Code pre-generation support (#23763)
* Start implementing a pregenerator
* Start moving pregenerate logic into a separate directory
* Start adding some ability to figure out pregeneration output locations
* Pregeneration for bridge seems to work
* Add missing files
* Restyle
* Better log level logic
* Pregeneration of java also exists
* Allow pregeneration of cpp app data. Full codegen usage is now enabled
* Move sdk root around
* Make things run
* Restyle
* Parallel code pregen - can do pregen in 1.08 seconds on my machine
* Restyle
* Make sure pregen folders are split
* Pregeneration compile when using GN works
* Restyle
* Direct pregen usage works now
* Restyle
* Minor sort
* Add support for both pregen and no pregen
* Restyle
* Fix pregen dir output logic
* Support pregenerated directory in build_examples.py
* Fix gn build logic
* Somewhat simpler code for parallel vs serial codegen
* Use imap_unordered to not care about actual parallel generation order
* Also fix java jni codegen with pregenerated data
* Fix java compilation deps: java codegen uses data model files
* NRF now can use pregen folder
* Allow telink to also use a pregen dir
* Better shell escape, make mbedos cmake flags work with pregen dir
* Restyle
* Add pregen support for esp32 as well
* Add a test for esp32 pregeneration support
* Also test pregen support in linux builds (to check gn builds)
* Remove unused file
* Fix spelling
* Fix esp32 compilation - gn arguments need to be passed from cmake
* Fix some define forwarding logic in codegen
* Make sure java build config (which includes header paths) is set as a config and applies to generated sources
* java build config should apply to all sources, not just transitively
* Restyle
* Replace codege with codegen.
* Fix naming typo
* Update text for build steps
* Update spacing logic to make the code cleaner. The nospace args is odd
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 276b1f2..4a3ce85 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -229,12 +229,28 @@
run: |
./scripts/run_in_build_env.sh \
"./scripts/build/build_examples.py --no-log-timestamps \
- --target linux-x64-all-clusters-ipv6only-clang \
- --target linux-x64-chip-tool-ipv6only-clang \
--target linux-x64-minmdns-ipv6only-clang \
--target linux-x64-rpc-console \
build \
"
+ - name: Create a pre-generate directory and ensure compile-time codegen would fail
+ run: |
+ ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated"
+ mv scripts/codegen.py scripts/codegen.py.renamed
+ - name: Build using build_examples.py (pregen)
+ timeout-minutes: 60
+ run: |
+ ./scripts/run_in_build_env.sh \
+ "./scripts/build/build_examples.py --no-log-timestamps \
+ --target linux-x64-all-clusters-ipv6only-clang \
+ --target linux-x64-chip-tool-ipv6only-clang \
+ --pregen-dir ./zzz_pregenerated \
+ build \
+ "
+ - name: Undo code pre-generation changes (make compile time codegen work again)
+ run: |
+ rm -rf ./zzz_pregenerated
+ mv scripts/codegen.py.renamed scripts/codegen.py
- name: Run fake linux tests with build_examples
timeout-minutes: 15
run: |
diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml
index 6755e3f..9299ac3 100644
--- a/.github/workflows/examples-esp32.yaml
+++ b/.github/workflows/examples-esp32.yaml
@@ -71,11 +71,29 @@
"./scripts/build/build_examples.py \
--enable-flashbundle \
--target esp32-m5stack-all-clusters \
- --target esp32-m5stack-all-clusters-minimal \
- --target esp32-m5stack-all-clusters-rpc-ipv6only \
build \
--copy-artifacts-to out/artifacts \
"
+ - name: Prepare code pregen and ensure compile time pregen not possible
+ run: |
+ ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated"
+ mv scripts/codegen.py scripts/codegen.py.renamed
+ - name: Build some M5Stack variations with pregen
+ timeout-minutes: 60
+ run: |
+ ./scripts/run_in_build_env.sh \
+ "./scripts/build/build_examples.py \
+ --enable-flashbundle \
+ --target esp32-m5stack-all-clusters-minimal \
+ --target esp32-m5stack-all-clusters-rpc-ipv6only \
+ --pregen-dir ./zzz_pregenerated \
+ build \
+ --copy-artifacts-to out/artifacts \
+ "
+ - name: Undo code pregeneration changes
+ run: |
+ rm -rf ./zzz_pregenerated
+ mv scripts/codegen.py.renamed scripts/codegen.py
- name: Prepare bloat report
run: |
.environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \
diff --git a/build/chip/chip_codegen.cmake b/build/chip/chip_codegen.cmake
index 17e3844..885d189 100644
--- a/build/chip/chip_codegen.cmake
+++ b/build/chip/chip_codegen.cmake
@@ -31,40 +31,71 @@
${ARGN}
)
- set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
+ set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.")
- string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
+ if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "")
+ set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
- file(MAKE_DIRECTORY "${GEN_FOLDER}")
- file(GENERATE
- OUTPUT "${GEN_FOLDER}/expected.outputs"
- CONTENT "${OUTPUT_AS_NEWLINES}"
- )
+ string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
+
+ file(MAKE_DIRECTORY "${GEN_FOLDER}")
+ file(GENERATE
+ OUTPUT "${GEN_FOLDER}/expected.outputs"
+ CONTENT "${OUTPUT_AS_NEWLINES}"
+ )
- set(OUT_NAMES)
- foreach(NAME IN LISTS ARG_OUTPUTS)
- list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
- endforeach()
+ set(OUT_NAMES)
+ foreach(NAME IN LISTS ARG_OUTPUTS)
+ list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+ endforeach()
- # Python is expected to be in the path
- #
- # find_package(Python3 REQUIRED)
- add_custom_command(
- OUTPUT ${OUT_NAMES}
- COMMAND "${CHIP_ROOT}/scripts/codegen.py"
- ARGS "--generator" "${ARG_GENERATOR}"
- "--output-dir" "${GEN_FOLDER}"
- "--expected-outputs" "${GEN_FOLDER}/expected.outputs"
- "${ARG_INPUT}"
- DEPENDS
- "${ARG_INPUT}"
- VERBATIM
- )
+ # Python is expected to be in the path
+ #
+ # find_package(Python3 REQUIRED)
+ add_custom_command(
+ OUTPUT ${OUT_NAMES}
+ COMMAND "${CHIP_ROOT}/scripts/codegen.py"
+ ARGS "--generator" "${ARG_GENERATOR}"
+ "--output-dir" "${GEN_FOLDER}"
+ "--expected-outputs" "${GEN_FOLDER}/expected.outputs"
+ "${ARG_INPUT}"
+ DEPENDS
+ "${ARG_INPUT}"
+ VERBATIM
+ )
- add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
+ add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
- # Forward outputs to the parent
- set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
- set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+ # Forward outputs to the parent
+ set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+ set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+ else()
+ # Gets a path such as:
+ # examples/lock-app/lock-common/lock-app.matter
+ file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT})
+
+ # Removes the trailing file extension to get something like:
+ # examples/lock-app/lock-common/lock-app
+ string(REGEX REPLACE "\.matter$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}")
+
+
+ # Build the final location within the pregen directory
+ set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/codegen/${ARG_GENERATOR}")
+
+ # TODO: build a fake target of ${TARGET_NAME}
+
+ # Here we have ${CHIP_CODEGEN_PREGEN_DIR}
+ set(OUT_NAMES)
+ foreach(NAME IN LISTS ARG_OUTPUTS)
+ list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+ endforeach()
+
+
+ set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+ set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+
+ # allow adding dependencies to a phony target since no codegen is done
+ add_custom_target(${TARGET_NAME})
+ endif()
endfunction()
diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni
index bab7993..fd5d69c 100644
--- a/build/chip/chip_codegen.gni
+++ b/build/chip/chip_codegen.gni
@@ -18,41 +18,15 @@
import("$dir_pw_build/python.gni")
-# Defines a target that runs code generation based on
-# scripts/codegen.py
+declare_args() {
+ # Location where code has been pre-generated
+ chip_code_pre_generated_directory = ""
+}
+
+# Code generation that will happen at build time.
#
-# Arguments:
-# input
-# The ".matter" file to use to start the code generation
#
-# generator
-# Name of the generator to use (e.g. java, cpp-app)
-#
-# outputs
-# Explicit names of the expected outputs. Enforced to validate that
-# expected outputs are generated when processing input files.
-#
-# NOTE: content of "outputs" is verified to match the output of codegen.py
-# exactly. It is not inferred on purpose, to make build-rules explicit
-# and verifiable (even though codege.py can at runtime report its outputs)
-#
-# To find the list of generated files, you can run codegen.py with the
-# "--name-only" argument
-#
-# Example usage:
-#
-# chip_codegen("java-jni-generate") {
-# input = "controller-clusters.matter"
-# generator = "java"
-#
-# outputs = [
-# "jni/IdentifyClient-ReadImpl.cpp",
-# "jni/IdentifyClient-InvokeSubscribeImpl.cpp",
-# # ... more to follow
-# ]
-# }
-#
-template("chip_codegen") {
+template("_chip_build_time_codegen") {
_name = target_name
_generator = invoker.generator
@@ -60,7 +34,7 @@
include_dirs = [ target_gen_dir ]
}
- pw_python_action(_name) {
+ pw_python_action("${_name}_codegen") {
script = "${chip_root}/scripts/codegen.py"
_idl_file = invoker.input
@@ -79,7 +53,6 @@
]
deps = [ "${chip_root}/scripts/idl" ]
- public_configs = [ ":${_name}_config" ]
inputs = [
_idl_file,
@@ -92,4 +65,119 @@
outputs += [ "${target_gen_dir}/${name}" ]
}
}
+
+ source_set(_name) {
+ sources = []
+ foreach(name, invoker.outputs) {
+ sources += [ "${target_gen_dir}/${name}" ]
+ }
+
+ public_configs = [ ":${_name}_config" ]
+
+ if (defined(invoker.public_configs)) {
+ public_configs += invoker.public_configs
+ }
+
+ forward_variables_from(invoker, [ "deps" ])
+
+ if (!defined(deps)) {
+ deps = []
+ }
+ deps += [ ":${_name}_codegen" ]
+ }
+}
+
+# Defines a target that runs code generation based on
+# scripts/codegen.py
+#
+# Arguments:
+# input
+# The ".matter" file to use to start the code generation
+#
+# generator
+# Name of the generator to use (e.g. java, cpp-app)
+#
+# outputs
+# Explicit names of the expected outputs. Enforced to validate that
+# expected outputs are generated when processing input files.
+#
+# deps, public_configs
+# Forwarded to the resulting source set
+#
+# Command line parameters:
+#
+# chip_code_pre_generated_directory:
+# - If this is set, generation will NOT happen at compile time but rather
+# the code generation is assumed to have already happened and reside in
+# the given location.
+# - The TOP LEVEL directory is assumed to be given. Actual location for
+# individual generators is expected to be of the form
+# <top_dir>/<matter_path>/<generator>
+#
+# NOTE: content of "outputs" is verified to match the output of codegen.py
+# exactly. It is not inferred on purpose, to make build-rules explicit
+# and verifiable (even though codegen.py can at runtime report its outputs)
+#
+# To find the list of generated files, you can run codegen.py with the
+# "--name-only" argument
+#
+# NOTE:
+# the result of the target_name WILL BE a `source_set`. Treat it as such.
+#
+# Example usage:
+#
+# chip_codegen("java-jni-generate") {
+# input = "controller-clusters.matter"
+# generator = "java"
+#
+# outputs = [
+# "jni/IdentifyClient-ReadImpl.cpp",
+# "jni/IdentifyClient-InvokeSubscribeImpl.cpp",
+# # ... more to follow
+# ]
+# }
+#
+template("chip_codegen") {
+ if (chip_code_pre_generated_directory == "") {
+ _chip_build_time_codegen(target_name) {
+ forward_variables_from(invoker,
+ [
+ "deps",
+ "generator",
+ "input",
+ "outputs",
+ "public_configs",
+ ])
+ }
+ } else {
+ _name = target_name
+
+ # This contstructs a path like:
+ # FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/)
+ # USING "cpp-app" for generator:
+ # => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app
+ _generation_dir =
+ chip_code_pre_generated_directory + "/" +
+ string_replace(rebase_path(invoker.input, chip_root), ".matter", "") +
+ "/codegen/" + invoker.generator
+
+ config("${_name}_config") {
+ include_dirs = [ "${_generation_dir}" ]
+ }
+
+ source_set(_name) {
+ public_configs = [ ":${_name}_config" ]
+
+ if (defined(invoker.public_configs)) {
+ public_configs += invoker.public_configs
+ }
+
+ forward_variables_from(invoker, [ "deps" ])
+
+ sources = []
+ foreach(name, invoker.outputs) {
+ sources += [ "${_generation_dir}/${name}" ]
+ }
+ }
+ }
}
diff --git a/config/esp32/components/chip/CMakeLists.txt b/config/esp32/components/chip/CMakeLists.txt
index 44f9238..d7cb69a 100644
--- a/config/esp32/components/chip/CMakeLists.txt
+++ b/config/esp32/components/chip/CMakeLists.txt
@@ -101,6 +101,10 @@
chip_gn_arg_append("chip_inet_config_enable_ipv4" "false")
endif()
+if(CHIP_CODEGEN_PREGEN_DIR)
+ chip_gn_arg_append("chip_code_pre_generated_directory" "\"${CHIP_CODEGEN_PREGEN_DIR}\"")
+endif()
+
if(CONFIG_ENABLE_PW_RPC)
string(APPEND chip_gn_args "import(\"//build_overrides/pigweed.gni\")\n")
chip_gn_arg_append("remove_default_configs" "[\"//third_party/connectedhomeip/third_party/pigweed/repo/pw_build:toolchain_cpp_standard\"]")
diff --git a/docs/code_generation.md b/docs/code_generation.md
index 51a615f..051a215 100644
--- a/docs/code_generation.md
+++ b/docs/code_generation.md
@@ -176,12 +176,32 @@
### `*.matter` code generation
-Currently `*.matter` code generation is done at compile time.
+`*.matter` code generation can be done either at compile time or it can use
+pre-generated output.
Rules for how `codegen.py` is invoked and how includes/sources are set are
defined at:
- `src/app/chip_data_model.cmake`
-- `build/chip/esp32/esp32_codegen.cmake` (support for 2-pass cmake builds used
- by the Espressif `idf.py` build system)
- `src/app/chip_data_model.gni`
+
+Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for
+the 2-pass cmake builds used by the Espressif `idf.py` build system.
+
+## Pre-generation
+
+Code pre-generation can be used:
+
+- when compile-time code generation is not desirable. This may be for
+ importing into build systems that do not have the pre-requisites to run code
+ generation at build time or to save the code generation time at the expense
+ of running code generation for every possible zap/generation type
+- To check changes in generated code across versions, beyond the comparisons
+ of golden image tests in `scripts/idl/tests`
+
+The script to trigger code pre-generation is `scripts/code_pregenerate.py` and
+requires the pre-generation output directory as an argument
+
+```bash
+scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
+```
diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py
index 5792bfa..780241f 100755
--- a/scripts/build/build_examples.py
+++ b/scripts/build/build_examples.py
@@ -88,6 +88,11 @@
type=click.Path(file_okay=False, resolve_path=True),
help='Prefix for the generated file output.')
@click.option(
+ '--pregen-dir',
+ default=None,
+ type=click.Path(file_okay=False, resolve_path=True),
+ help='Directory where generated files have been pre-generated.')
+@click.option(
'--clean',
default=False,
is_flag=True,
@@ -114,7 +119,7 @@
'for using ccache when building examples.'))
@click.pass_context
def main(context, log_level, target, repo,
- out_prefix, clean, dry_run, dry_run_output, enable_flashbundle,
+ out_prefix, pregen_dir, clean, dry_run, dry_run_output, enable_flashbundle,
no_log_timestamps, pw_command_launcher):
# Ensures somewhat pretty logging of what is going on
log_fmt = '%(asctime)s %(levelname)-7s %(message)s'
@@ -143,6 +148,7 @@
context.obj.SetupBuilders(targets=requested_targets, options=BuilderOptions(
enable_flashbundle=enable_flashbundle,
pw_command_launcher=pw_command_launcher,
+ pregen_dir=pregen_dir,
))
if clean:
diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py
index 99bc925..dc420fc 100644
--- a/scripts/build/builders/builder.py
+++ b/scripts/build/builders/builder.py
@@ -24,9 +24,13 @@
class BuilderOptions:
# Enable flashbundle generation stage
enable_flashbundle: bool = False
+
# Allow to wrap default build command
pw_command_launcher: str = None
+ # Locations where files are pre-generated
+ pregen_dir: str = None
+
class Builder(ABC):
"""Generic builder base class for CHIP.
diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py
index d84e8f5..54c47a4 100644
--- a/scripts/build/builders/esp32.py
+++ b/scripts/build/builders/esp32.py
@@ -176,11 +176,19 @@
self._Execute(
['bash', '-c', 'echo -e "\\nCONFIG_DISABLE_IPV4=y\\n" >>%s' % shlex.quote(defaults_out)])
- cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out} reconfigure".format(
- defaults=shlex.quote(defaults_out),
- example_path=self.ExamplePath,
- out=shlex.quote(self.output_dir)
- )
+ cmake_flags = []
+
+ if self.options.pregen_dir:
+ cmake_flags.append(
+ f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}")
+
+ cmake_args = ['-C', self.ExamplePath, '-B',
+ shlex.quote(self.output_dir)] + cmake_flags
+
+ cmake_args = " ".join(cmake_args)
+ defaults = shlex.quote(defaults_out)
+
+ cmd = f"\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py {cmake_args} reconfigure"
# This will do a 'cmake reconfigure' which will create ninja files without rebuilding
self._IdfEnvExecute(cmd)
diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py
index bacd87d..9e859f6 100644
--- a/scripts/build/builders/gn.py
+++ b/scripts/build/builders/gn.py
@@ -61,8 +61,13 @@
]
extra_args = []
+
if self.options.pw_command_launcher:
extra_args.append('pw_command_launcher="%s"' % self.options.pw_command_launcher)
+
+ if self.options.pregen_dir:
+ extra_args.append('chip_code_pre_generated_directory="%s"' % self.options.pregen_dir)
+
extra_args.extend(self.GnBuildArgs() or [])
if extra_args:
cmd += ['--args=%s' % ' '.join(extra_args)]
diff --git a/scripts/build/builders/mbed.py b/scripts/build/builders/mbed.py
index 73fc802..46b3ead 100644
--- a/scripts/build/builders/mbed.py
+++ b/scripts/build/builders/mbed.py
@@ -121,14 +121,16 @@
'--mbed-os-path', self.mbed_os_path,
], title='Generating config ' + self.identifier)
- self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), '-GNinja',
- '-DCMAKE_BUILD_TYPE={}'.format(
- self.profile.ProfileName.lower()),
- '-DMBED_OS_PATH={}'.format(
- shlex.quote(self.mbed_os_path)),
- '-DMBED_OS_POSIX_SOCKET_PATH={}'.format(
- shlex.quote(self.mbed_os_posix_socket_path)),
- ], title='Generating ' + self.identifier)
+ flags = []
+ flags.append(f"-DMBED_OS_PATH={shlex.quote(self.mbed_os_path)}")
+ flags.append(f"-DMBED_OS_PATH={shlex.quote(self.mbed_os_path)}")
+ flags.append(f"-DMBED_OS_POSIX_SOCKET_PATH={shlex.quote(self.mbed_os_posix_socket_path)}")
+
+ if self.options.pregen_dir:
+ flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}")
+
+ self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir),
+ '-GNinja'] + flags, title='Generating ' + self.identifier)
def _build(self):
# Remove old artifacts to force linking
diff --git a/scripts/build/builders/nrf.py b/scripts/build/builders/nrf.py
index 03298cc..0037a38 100644
--- a/scripts/build/builders/nrf.py
+++ b/scripts/build/builders/nrf.py
@@ -179,6 +179,9 @@
if self.board == NrfBoard.NRF52840DONGLE and self.app != NrfApp.ALL_CLUSTERS and self.app != NrfApp.ALL_CLUSTERS_MINIMAL:
flags.append("-DCONF_FILE=prj_no_dfu.conf")
+ if self.options.pregen_dir:
+ flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}")
+
build_flags = " -- " + " ".join(flags) if len(flags) > 0 else ""
cmd = '''
diff --git a/scripts/build/builders/telink.py b/scripts/build/builders/telink.py
index a85ec7c..32a9bf8 100644
--- a/scripts/build/builders/telink.py
+++ b/scripts/build/builders/telink.py
@@ -100,15 +100,21 @@
if os.path.exists(self.output_dir):
return
+ flags = []
+ if self.options.pregen_dir:
+ flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}")
+
+ build_flags = " -- " + " ".join(flags) if len(flags) > 0 else ""
+
cmd = self.get_cmd_prefixes()
cmd += '''
source "$ZEPHYR_BASE/zephyr-env.sh";
-west build --cmake-only -d {outdir} -b {board} {sourcedir}
+west build --cmake-only -d {outdir} -b {board} {sourcedir}{build_flags}
'''.format(
- outdir=shlex.quote(
- self.output_dir), board=self.board.GnArgName(), sourcedir=shlex.quote(
- os.path.join(
- self.root, 'examples', self.app.ExampleName(), 'telink'))).strip()
+ outdir=shlex.quote(self.output_dir),
+ board=self.board.GnArgName(),
+ sourcedir=shlex.quote(os.path.join(self.root, 'examples', self.app.ExampleName(), 'telink')),
+ build_flags=build_flags).strip()
self._Execute(['bash', '-c', cmd],
title='Generating ' + self.identifier)
diff --git a/scripts/codepregen.py b/scripts/codepregen.py
new file mode 100755
index 0000000..13ca9a6
--- /dev/null
+++ b/scripts/codepregen.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import logging
+import multiprocessing
+import itertools
+import enum
+import os
+import sys
+
+
+try:
+ from pregenerate import FindPregenerationTargets
+except:
+ import os
+ sys.path.append(os.path.abspath(os.path.dirname(__file__)))
+ from pregenerate import FindPregenerationTargets
+
+try:
+ import coloredlogs
+ _has_coloredlogs = True
+except:
+ _has_coloredlogs = False
+
+# Supported log levels, mapping string values required for argument
+# parsing into logging constants
+__LOG_LEVELS__ = {
+ 'debug': logging.DEBUG,
+ 'info': logging.INFO,
+ 'warn': logging.WARN,
+ 'fatal': logging.FATAL,
+}
+
+
+def _ParallelGenerateOne(arg):
+ """
+ Helper method to be passed to multiprocessing parallel generation of
+ items.
+ """
+ arg[0].Generate(arg[1])
+
+
+@click.command()
+@click.option(
+ '--log-level',
+ default='INFO',
+ type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False),
+ help='Determines the verbosity of script output')
+@click.option(
+ '--parallel/--no-parallel',
+ default=True,
+ help='Do parallel/multiprocessing codegen.')
+@click.option(
+ '--sdk-root',
+ default=None,
+ help='Path to the SDK root (where .zap/.matter files exist).')
+@click.argument('output_dir')
+def main(log_level, parallel, sdk_root, output_dir):
+ if _has_coloredlogs:
+ coloredlogs.install(level=__LOG_LEVELS__[
+ log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
+ else:
+ logging.basicConfig(
+ level=__LOG_LEVELS__[log_level],
+ format='%(asctime)s %(levelname)-7s %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S'
+ )
+
+ if not sdk_root:
+ sdk_root = os.path.join(os.path.dirname(
+ os.path.realpath(__file__)), '..')
+
+ sdk_root = os.path.abspath(sdk_root)
+
+ if not output_dir:
+ raise Exception("Missing output directory")
+
+ output_dir = os.path.abspath(output_dir)
+
+ logging.info(f"Pre-generating {sdk_root} data into {output_dir}")
+
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ targets = FindPregenerationTargets(sdk_root)
+
+ if parallel:
+ target_and_dir = zip(targets, itertools.repeat(output_dir))
+ with multiprocessing.Pool() as pool:
+ for _ in pool.imap_unordered(_ParallelGenerateOne, target_and_dir):
+ pass
+ else:
+ for target in targets:
+ target.Generate(output_dir)
+
+ logging.info("Done")
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py
new file mode 100644
index 0000000..eab3205
--- /dev/null
+++ b/scripts/pregenerate/__init__.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import os
+
+from typing import Iterator
+
+from .types import InputIdlFile, IdlFileType
+
+
+from .pregenerators import CodegenJavaPregenerator
+from .pregenerators import CodegenBridgePregenerator
+from .pregenerators import CodegenCppAppPregenerator
+
+
+def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]:
+ relevant_subdirs = [
+ 'examples', # all example apps
+ 'src', # realistically only controller/data_model
+ ]
+
+ while sdk_root.endswith('/'):
+ sdk_root = sdk_root[:-1]
+ sdk_root_length = len(sdk_root)
+
+ for subdir_name in relevant_subdirs:
+ top_directory_name = os.path.join(sdk_root, subdir_name)
+ logging.debug(f"Searching {top_directory_name}")
+ for root, dirs, files in os.walk(top_directory_name):
+ for file in files:
+ if file.endswith('.zap'):
+ yield InputIdlFile(file_type=IdlFileType.ZAP,
+ relative_path=os.path.join(root[sdk_root_length+1:], file))
+ if file.endswith('.matter'):
+ yield InputIdlFile(file_type=IdlFileType.MATTER,
+ relative_path=os.path.join(root[sdk_root_length+1:], file))
+
+
+def FindPregenerationTargets(sdk_root: str):
+ """Finds all relevand pre-generation targets in the given
+ SDK root.
+
+ Pre-generation targets are based on zap and matter files with options
+ on what rules to pregenerate and how.
+ """
+
+ generators = [
+ CodegenBridgePregenerator(sdk_root),
+ CodegenJavaPregenerator(sdk_root),
+ CodegenCppAppPregenerator(sdk_root),
+ ]
+
+ for idl in FindAllIdls(sdk_root):
+ for generator in generators:
+ if generator.Accept(idl):
+ yield generator.CreateTarget(idl)
diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py
new file mode 100644
index 0000000..3d6658d
--- /dev/null
+++ b/scripts/pregenerate/pregenerators.py
@@ -0,0 +1,102 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import shlex
+import subprocess
+
+from .types import InputIdlFile, IdlFileType
+
+CODEGEN_PY_PATH = os.path.join(os.path.dirname(__file__), '..', 'codegen.py')
+
+
+class CodegenTarget:
+ """A target that uses `scripts/codegen.py` to generate files."""
+
+ def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str):
+ self.idl = idl
+ self.generator = generator
+ self.sdk_root = sdk_root
+
+ if idl.file_type != IdlFileType.MATTER:
+ raise Exception(f"Can only code generate for `*.matter` input files, not for {idl}")
+
+ def Generate(self, output_root: str):
+ '''Runs codegen.py to generate in the specified directory'''
+
+ output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generator)
+
+ logging.info(f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}")
+
+ cmd = [
+ CODEGEN_PY_PATH,
+ '--log-level', 'fatal',
+ '--generator', self.generator,
+ '--output-dir', output_dir,
+ os.path.join(self.sdk_root, self.idl.relative_path)
+ ]
+
+ logging.debug(f"Executing {cmd}")
+ subprocess.check_call(cmd)
+
+
+class CodegenBridgePregenerator:
+ """Pregeneration logic for "bridge" codegen.py outputs"""
+
+ def __init__(self, sdk_root):
+ self.sdk_root = sdk_root
+
+ def Accept(self, idl: InputIdlFile):
+ # Bridge is highly specific, a single path is acceptable for dynamic
+ # bridge codegen
+ return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter"
+
+ def CreateTarget(self, idl: InputIdlFile):
+ return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge")
+
+
+class CodegenJavaPregenerator:
+ """Pregeneration logic for "java" codegen.py outputs"""
+
+ def __init__(self, sdk_root):
+ self.sdk_root = sdk_root
+
+ def Accept(self, idl: InputIdlFile):
+ # Java is highly specific, a single path is acceptable for dynamic
+ # bridge codegen
+ return idl.relative_path == "src/controller/data_model/controller-clusters.matter"
+
+ def CreateTarget(self, idl: InputIdlFile):
+ return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java")
+
+
+class CodegenCppAppPregenerator:
+ """Pregeneration logic for "cpp-app" codegen.py outputs"""
+
+ def __init__(self, sdk_root):
+ self.sdk_root = sdk_root
+
+ def Accept(self, idl: InputIdlFile):
+ if idl.file_type != IdlFileType.MATTER:
+ return False
+
+ # we should not be checked for these, but verify just in case
+ if '/tests/' in idl.relative_path:
+ return False
+
+ return True
+
+ def CreateTarget(self, idl: InputIdlFile):
+ return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app")
diff --git a/scripts/pregenerate/types.py b/scripts/pregenerate/types.py
new file mode 100644
index 0000000..3d8c0bc
--- /dev/null
+++ b/scripts/pregenerate/types.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from dataclasses import dataclass
+from enum import Enum, auto
+
+
+class IdlFileType(Enum):
+ ZAP = auto()
+ MATTER = auto()
+
+
+@dataclass
+class InputIdlFile:
+ file_type: IdlFileType
+ relative_path: str
+
+ @property
+ def pregen_subdir(self):
+ '''
+ Returns the relative path inside the pregenerate directory where
+ data for this IDL file should be pregenerated.
+ '''
+ top_dir = os.path.splitext(self.relative_path)[0]
+
+ if self.file_type == IdlFileType.MATTER:
+ return os.path.join(top_dir, "codegen")
+ elif self.file_type == IdlFileType.ZAP:
+ return os.path.join(top_dir, "zap")
+ else:
+ raise Exception("Unknown file type for self")
diff --git a/src/app/chip_data_model.gni b/src/app/chip_data_model.gni
index 68aabd7..ecab5a9 100644
--- a/src/app/chip_data_model.gni
+++ b/src/app/chip_data_model.gni
@@ -51,15 +51,6 @@
_idl = string_replace(invoker.zap_file, ".zap", ".matter")
}
- chip_codegen("${_data_model_name}_codegen") {
- input = _idl
- generator = "cpp-app"
- outputs = [
- "app/PluginApplicationCallbacks.h",
- "app/callback-stub.cpp",
- ]
- }
-
config("${_data_model_name}_config") {
include_dirs = []
@@ -68,6 +59,24 @@
}
}
+ chip_codegen("${_data_model_name}_codegen") {
+ input = _idl
+ generator = "cpp-app"
+
+ outputs = [
+ "app/PluginApplicationCallbacks.h",
+ "app/callback-stub.cpp",
+ ]
+
+ public_configs = [ ":${_data_model_name}_config" ]
+
+ if (!defined(deps)) {
+ deps = []
+ }
+
+ deps += [ "${chip_root}/src/app/common:cluster-objects" ]
+ }
+
_use_default_im_dispatch = !defined(invoker.use_default_im_dispatch) ||
invoker.use_default_im_dispatch
@@ -84,7 +93,10 @@
sources = []
}
- sources += get_target_outputs(":${_data_model_name}_codegen")
+ if (!defined(deps)) {
+ deps = []
+ }
+ deps += [ ":${_data_model_name}_codegen" ]
sources += [
"${_app_root}/clusters/barrier-control-server/barrier-control-server.h",
diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn
index 011669d..95b0e6c 100644
--- a/src/controller/data_model/BUILD.gn
+++ b/src/controller/data_model/BUILD.gn
@@ -31,6 +31,12 @@
}
if (current_os == "android" || build_java_matter_controller) {
+ config("java-build-config") {
+ if (build_java_matter_controller) {
+ include_dirs = java_matter_controller_dependent_paths
+ }
+ }
+
chip_codegen("java-jni-generate") {
input = "controller-clusters.matter"
generator = "java"
@@ -167,12 +173,21 @@
"jni/UnitTestingClient-ReadImpl.cpp",
"jni/UnitTestingClient-InvokeSubscribeImpl.cpp",
]
+
+ deps = [
+ ":data_model",
+ "${chip_root}/src/platform:platform_buildconfig",
+ ]
+
+ public_configs = [ ":java-build-config" ]
}
source_set("java-jni-sources") {
- sources = get_target_outputs(":java-jni-generate")
+ public_configs = [
+ ":java-build-config",
+ "${chip_root}/src:includes",
+ ]
- public_configs = [ "${chip_root}/src:includes" ]
deps = [
":data_model",
":java-jni-generate",
@@ -182,7 +197,6 @@
]
if (build_java_matter_controller) {
- include_dirs = java_matter_controller_dependent_paths
deps += [ "${chip_root}/src/platform/Linux" ]
} else {
deps += [ "${chip_root}/src/platform/android" ]
