Use Pigweed's new Python build system (#21202)
This change updates Matter to use Pigweed's new Python build system.
This reduces the number of pip installs performed during bootstrap and
builds to just a single pigweed package along with any others
specified in //BUILD.gn.
All python_actions performed duing a GN build do not rely on any pip
install of any in-tree python packages including Pigweed ones. Instead
build a venv is created in the out directory with only 3rd party deps
installed. All Python imports of in-tree packages are accomplished by
setting PYTHONPATH for each python_action. This allows scaling up the
number of in-tree Python packages without a significant impact to
build speed.
Bootstrap time is slightly reduced however all Pigweed Python modules
are installed allong with the Python packages from:
- //examples/common/pigweed/rpc_console
- //examples/chef
- //integrations/mobly
Before:
```
Setting up Python environment.....done (1m34.7s)
```
After:
```
Setting up Python environment.....done (1m14.7s)
```
diff --git a/.github/workflows/full-android.yaml b/.github/workflows/full-android.yaml
index bb453ee..cb5eb7f 100644
--- a/.github/workflows/full-android.yaml
+++ b/.github/workflows/full-android.yaml
@@ -72,7 +72,7 @@
- name: Run Android build rule tests
run: |
./scripts/run_in_build_env.sh \
- "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
+ "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test"
- name: Clean out build output
run: rm -rf ./out
# - name: Build Android Studio build (arm64 only)
diff --git a/.github/workflows/smoketest-android.yaml b/.github/workflows/smoketest-android.yaml
index 4fa7492..4e6571b 100644
--- a/.github/workflows/smoketest-android.yaml
+++ b/.github/workflows/smoketest-android.yaml
@@ -67,4 +67,4 @@
- name: Run Android build rule tests
run: |
./scripts/run_in_build_env.sh \
- "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
+ "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test"
diff --git a/.gn b/.gn
index 6df7f3c..5da05d8 100644
--- a/.gn
+++ b/.gn
@@ -25,5 +25,13 @@
default_args = {
pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"
- pw_build_PIP_CONSTRAINTS = ["//scripts/constraints.txt"]
+
+ pw_build_PIP_CONSTRAINTS = [ "//scripts/constraints.txt" ]
+ pw_build_PIP_REQUIREMENTS = [ "//scripts/requirements.txt" ]
+
+ # Use the new Python build and merged 'pigweed' Python package.
+ pw_build_USE_NEW_PYTHON_BUILD = true
+
+ # GN target to use for the default Python build venv.
+ pw_build_PYTHON_BUILD_VENV = "//:matter_build_venv"
}
diff --git a/BUILD.gn b/BUILD.gn
index d0018dd..30e24a7 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -23,6 +23,8 @@
import("//src/lwip/lwip.gni")
import("//src/platform/device.gni")
import("$dir_pw_build/python.gni")
+import("$dir_pw_build/python_dist.gni")
+import("$dir_pw_build/python_venv.gni")
# This build file should not be used in superproject builds.
assert(chip_root == "//")
@@ -53,29 +55,85 @@
}
}
- # Python packages for supporting specific targets.
+ # Pigweed Python packages expected to be used in the :matter_build_venv
+ # target. If all packages are needed this list should match
+ # _pigweed_python_deps in:
+ # https://cs.opensource.google/pigweed/pigweed/+/master:pw_env_setup/BUILD.gn?q=_pigweed_python_deps
+ _pigweed_python_packages = [
+ "$dir_pw_allocator/py",
+ "$dir_pw_arduino_build/py",
+ "$dir_pw_bloat/py",
+ "$dir_pw_build/py",
+ "$dir_pw_build_info/py",
+ "$dir_pw_build_mcuxpresso/py",
+ "$dir_pw_cli/py",
+ "$dir_pw_console/py",
+ "$dir_pw_cpu_exception_cortex_m/py",
+ "$dir_pw_docgen/py",
+ "$dir_pw_doctor/py",
+ "$dir_pw_env_setup/py",
+ "$dir_pw_hdlc/py",
+ "$dir_pw_log:protos.python",
+ "$dir_pw_log_tokenized/py",
+ "$dir_pw_metric/py",
+ "$dir_pw_module/py",
+ "$dir_pw_package/py",
+ "$dir_pw_presubmit/py",
+ "$dir_pw_protobuf/py",
+ "$dir_pw_protobuf_compiler/py",
+ "$dir_pw_rpc/py",
+ "$dir_pw_snapshot/py:pw_snapshot",
+ "$dir_pw_snapshot/py:pw_snapshot_metadata",
+ "$dir_pw_software_update/py",
+ "$dir_pw_status/py",
+ "$dir_pw_stm32cube_build/py",
+ "$dir_pw_symbolizer/py",
+ "$dir_pw_system/py",
+ "$dir_pw_thread/py",
+ "$dir_pw_tls_client/py",
+ "$dir_pw_tokenizer/py",
+ "$dir_pw_toolchain/py",
+ "$dir_pw_trace/py",
+ "$dir_pw_trace_tokenized/py",
+ "$dir_pw_transfer/py",
+ "$dir_pw_unit_test/py",
+ "$dir_pw_watch/py",
+ ]
+
+ # Matter's in-tree pw_python_package or pw_create_python_source_tree targets.
+ _matter_python_packages = [
+ "//integrations/mobly:chip_mobly",
+ "//examples/chef",
+ "//examples/common/pigweed/rpc_console/py:chip_rpc",
+ ]
+
+ pw_python_venv("matter_build_venv") {
+ path = "$root_build_dir/python-venv"
+ constraints = pw_build_PIP_CONSTRAINTS
+ requirements = pw_build_PIP_REQUIREMENTS
+
+ # Packages available to import within GN's build venv.
+ source_packages = _matter_python_packages + _pigweed_python_packages
+ }
+
+ # Python packages installed during bootstrap.
pw_python_group("python_packages") {
python_deps = [
- "$dir_pw_build/py",
- "$dir_pw_doctor/py",
- "$dir_pw_env_setup/py",
- "$dir_pw_hdlc/py",
- "$dir_pw_log:protos.python",
- "$dir_pw_module/py",
- "$dir_pw_protobuf/py",
- "$dir_pw_protobuf_compiler/py",
- "$dir_pw_rpc/py",
- "$dir_pw_status/py",
- "$dir_pw_toolchain/py",
- "$dir_pw_trace/py",
- "$dir_pw_trace_tokenized/py",
- "$dir_pw_unit_test/py",
- "$dir_pw_watch/py",
- "integrations/mobly:chip_mobly",
- "scripts:requirements",
+ ":pip_install_editable_matter_packages",
+ "$dir_pw_env_setup:pip_install_pigweed_package",
]
}
+ # These pw_python_package targets will be installed using 'pip install --editable'
+ pw_internal_pip_install("pip_install_editable_matter_packages") {
+ packages = [
+ "//integrations/mobly:chip_mobly",
+ "//examples/chef",
+ "//examples/common/pigweed/rpc_console/py:chip_rpc",
+ ]
+ editable = true
+ }
+
# This is a real toolchain. Build CHIP.
group("default") {
deps = [
diff --git a/build/chip/chip_test.gni b/build/chip/chip_test.gni
index 5671567..08fbb3e 100644
--- a/build/chip/chip_test.gni
+++ b/build/chip/chip_test.gni
@@ -44,7 +44,11 @@
pw_python_action(_test_name + "_run") {
deps = [ ":${_test_name}" ]
inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
- script = "$dir_pw_unit_test/py/pw_unit_test/test_runner.py"
+ module = "pw_unit_test.test_runner"
+ python_deps = [
+ "$dir_pw_cli/py",
+ "$dir_pw_unit_test/py",
+ ]
args = [
"--runner",
rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),
diff --git a/build/chip/java/tests/BUILD.gn b/build/chip/java/tests/BUILD.gn
index d5e97c0..8a7c8dd 100644
--- a/build/chip/java/tests/BUILD.gn
+++ b/build/chip/java/tests/BUILD.gn
@@ -17,7 +17,7 @@
import("$dir_pw_build/python.gni")
import("${chip_root}/build/chip/java/rules.gni")
-pw_python_script("java_build_test") {
+pw_python_action("java_build_test") {
inputs = [
"expected_output/child_library_2_expected.json",
"expected_output/grandchild_library_expected.json",
@@ -26,8 +26,9 @@
"expected_output/child_prebuilt_expected.json",
"expected_output/java_prebuilt_expected.json",
]
- other_deps = [ ":java_library" ]
- tests = [ "test.py" ]
+ deps = [ ":java_library" ]
+ script = "test.py"
+ stamp = true
}
java_library("java_library") {
diff --git a/build/chip/java/tests/expected_output/child_library_2_expected.json b/build/chip/java/tests/expected_output/child_library_2_expected.json
index 9563f85..065593b 100644
--- a/build/chip/java/tests/expected_output/child_library_2_expected.json
+++ b/build/chip/java/tests/expected_output/child_library_2_expected.json
@@ -1,10 +1,8 @@
{
"deps_info": {
"name": "child_library_2.json",
- "jar_path": "python/lib/build/chip/java/tests/child_library_2.jar",
- "deps_configs": [
- "python/gen/build/chip/java/tests/grandchild_library.json"
- ],
- "deps_jars": ["python/lib/build/chip/java/tests/grandchild_library.jar"]
+ "jar_path": "lib/build/chip/java/tests/child_library_2.jar",
+ "deps_configs": ["gen/build/chip/java/tests/grandchild_library.json"],
+ "deps_jars": ["lib/build/chip/java/tests/grandchild_library.jar"]
}
}
diff --git a/build/chip/java/tests/expected_output/child_library_expected.json b/build/chip/java/tests/expected_output/child_library_expected.json
index 279542f..e84ff94 100644
--- a/build/chip/java/tests/expected_output/child_library_expected.json
+++ b/build/chip/java/tests/expected_output/child_library_expected.json
@@ -1,7 +1,7 @@
{
"deps_info": {
"name": "child_library.json",
- "jar_path": "python/lib/build/chip/java/tests/child_library.jar",
+ "jar_path": "lib/build/chip/java/tests/child_library.jar",
"deps_configs": [],
"deps_jars": []
}
diff --git a/build/chip/java/tests/expected_output/child_prebuilt_expected.json b/build/chip/java/tests/expected_output/child_prebuilt_expected.json
index c276a7b..77e4ba0 100644
--- a/build/chip/java/tests/expected_output/child_prebuilt_expected.json
+++ b/build/chip/java/tests/expected_output/child_prebuilt_expected.json
@@ -1,7 +1,7 @@
{
"deps_info": {
"name": "child_prebuilt.json",
- "jar_path": "python/lib/build/chip/java/tests/child_jar.jar",
+ "jar_path": "lib/build/chip/java/tests/child_jar.jar",
"deps_configs": [],
"deps_jars": []
}
diff --git a/build/chip/java/tests/expected_output/grandchild_library_expected.json b/build/chip/java/tests/expected_output/grandchild_library_expected.json
index 54c6c33..c7d0659 100644
--- a/build/chip/java/tests/expected_output/grandchild_library_expected.json
+++ b/build/chip/java/tests/expected_output/grandchild_library_expected.json
@@ -1,7 +1,7 @@
{
"deps_info": {
"name": "grandchild_library.json",
- "jar_path": "python/lib/build/chip/java/tests/grandchild_library.jar",
+ "jar_path": "lib/build/chip/java/tests/grandchild_library.jar",
"deps_configs": [],
"deps_jars": []
}
diff --git a/build/chip/java/tests/expected_output/java_library_expected.json b/build/chip/java/tests/expected_output/java_library_expected.json
index 2806ed6..40df563 100644
--- a/build/chip/java/tests/expected_output/java_library_expected.json
+++ b/build/chip/java/tests/expected_output/java_library_expected.json
@@ -1,18 +1,18 @@
{
"deps_info": {
"name": "java_library.json",
- "jar_path": "python/lib/build/chip/java/tests/java_library.jar",
+ "jar_path": "lib/build/chip/java/tests/java_library.jar",
"deps_configs": [
- "python/gen/build/chip/java/tests/child_library.json",
- "python/gen/build/chip/java/tests/child_library_2.json",
- "python/gen/build/chip/java/tests/java_prebuilt.json"
+ "gen/build/chip/java/tests/child_library.json",
+ "gen/build/chip/java/tests/child_library_2.json",
+ "gen/build/chip/java/tests/java_prebuilt.json"
],
"deps_jars": [
- "python/lib/build/chip/java/tests/child_library.jar",
- "python/lib/build/chip/java/tests/child_library_2.jar",
- "python/lib/build/chip/java/tests/grandchild_library.jar",
- "python/lib/build/chip/java/tests/prebuilt_jar.jar",
- "python/lib/build/chip/java/tests/child_jar.jar"
+ "lib/build/chip/java/tests/child_library.jar",
+ "lib/build/chip/java/tests/child_library_2.jar",
+ "lib/build/chip/java/tests/grandchild_library.jar",
+ "lib/build/chip/java/tests/prebuilt_jar.jar",
+ "lib/build/chip/java/tests/child_jar.jar"
]
}
}
diff --git a/build/chip/java/tests/expected_output/java_prebuilt_expected.json b/build/chip/java/tests/expected_output/java_prebuilt_expected.json
index 10ce6a5..d11f72d 100644
--- a/build/chip/java/tests/expected_output/java_prebuilt_expected.json
+++ b/build/chip/java/tests/expected_output/java_prebuilt_expected.json
@@ -1,10 +1,8 @@
{
"deps_info": {
"name": "java_prebuilt.json",
- "jar_path": "python/lib/build/chip/java/tests/prebuilt_jar.jar",
- "deps_configs": [
- "python/gen/build/chip/java/tests/child_prebuilt.json"
- ],
- "deps_jars": ["python/lib/build/chip/java/tests/child_jar.jar"]
+ "jar_path": "lib/build/chip/java/tests/prebuilt_jar.jar",
+ "deps_configs": ["gen/build/chip/java/tests/child_prebuilt.json"],
+ "deps_jars": ["lib/build/chip/java/tests/child_jar.jar"]
}
}
diff --git a/build/chip/java/tests/test.py b/build/chip/java/tests/test.py
index 4beff28..68ca31c 100755
--- a/build/chip/java/tests/test.py
+++ b/build/chip/java/tests/test.py
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
"""
Test for GN Java build rules. This test should be executed using ninja.
"""
@@ -29,8 +28,8 @@
local_test_dir = '/build/chip/java/tests'
test_dir = chip_root + local_test_dir
- jars_dir = 'python/lib' + local_test_dir
- configs_dir = 'python/gen' + local_test_dir
+ jars_dir = 'lib' + local_test_dir
+ configs_dir = 'gen' + local_test_dir
# Target names in the BUILD.gn
targets_to_check = [
@@ -39,10 +38,7 @@
'child_library_2',
'grandchild_library',
]
- prebuilt_targets_to_check = [
- 'java_prebuilt',
- 'child_prebuilt'
- ]
+ prebuilt_targets_to_check = ['java_prebuilt', 'child_prebuilt']
def testExpectedJarsCreated(self):
jars_dir = JavaBuildTest.jars_dir
@@ -58,18 +54,22 @@
configs_dir = JavaBuildTest.configs_dir
expected_dir = JavaBuildTest.test_dir + '/expected_output'
- for target in (JavaBuildTest.targets_to_check + JavaBuildTest.prebuilt_targets_to_check):
- with open(expected_dir + '/' + target + '_expected.json', 'r') as expected_config, open(configs_dir + '/' + target + '.json', 'r') as actual_config:
+ for target in (JavaBuildTest.targets_to_check +
+ JavaBuildTest.prebuilt_targets_to_check):
+ with open(expected_dir + '/' + target + '_expected.json',
+ 'r') as expected_config, open(
+ configs_dir + '/' + target + '.json',
+ 'r') as actual_config:
expected_json = json.load(expected_config)['deps_info']
actual_json = json.load(actual_config)['deps_info']
self.assertEqual(expected_json['name'], actual_json['name'])
- self.assertEqual(
- expected_json['jar_path'], actual_json['jar_path'])
- self.assertCountEqual(
- expected_json['deps_configs'], actual_json['deps_configs'])
- self.assertCountEqual(
- expected_json['deps_jars'], actual_json['deps_jars'])
+ self.assertEqual(expected_json['jar_path'],
+ actual_json['jar_path'])
+ self.assertCountEqual(expected_json['deps_configs'],
+ actual_json['deps_configs'])
+ self.assertCountEqual(expected_json['deps_jars'],
+ actual_json['deps_jars'])
if __name__ == '__main__':
diff --git a/examples/build_overrides/pigweed_environment.gni b/examples/build_overrides/pigweed_environment.gni
index fb1dee4..1d72ca2 100644
--- a/examples/build_overrides/pigweed_environment.gni
+++ b/examples/build_overrides/pigweed_environment.gni
@@ -26,4 +26,4 @@
dir_cipd_python =
get_path_info("${_bootstrap_root}/${dir_cipd_python}", "abspath")
dir_virtual_env =
- get_path_info("${_bootstrap_root}/${dir_virtual_env}", "abspath")
+ get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", "abspath")
diff --git a/examples/chef/BUILD.gn b/examples/chef/BUILD.gn
index 5c3280c..4fb787d 100644
--- a/examples/chef/BUILD.gn
+++ b/examples/chef/BUILD.gn
@@ -19,7 +19,11 @@
import("$dir_pw_build/python.gni")
pw_python_package("chef") {
- setup = [ "setup.py" ]
+ setup = [
+ "pyproject.toml",
+ "setup.cfg",
+ "setup.py",
+ ]
inputs = [
"sample_app_util/test_files/sample_zap_file.zap",
diff --git a/examples/chef/README.md b/examples/chef/README.md
index b9641be..96003b5 100644
--- a/examples/chef/README.md
+++ b/examples/chef/README.md
@@ -148,7 +148,7 @@
- name: Checkout submodules
run: scripts/checkout_submodules.py --shallow --platform $PLATFORM
- name: Bootstrap
- timeout-minutes: 10
+ timeout-minutes: 25
run: scripts/build/gn_bootstrap.sh
- name: CI Examples $PLATFORM
shell: bash
diff --git a/examples/chef/chef.py b/examples/chef/chef.py
index f62f40d..469ba9a 100755
--- a/examples/chef/chef.py
+++ b/examples/chef/chef.py
@@ -239,7 +239,8 @@
shutil.copy(src_item, dest_item)
-def main(argv: Sequence[str]) -> None:
+def main() -> int:
+
check_python_version()
config = load_config()
cicd_config = load_cicd_config()
@@ -340,7 +341,7 @@
parser.add_option(
"", "--cpu_type", help="CPU type to compile for. Linux only.", choices=["arm64", "arm", "x64"])
- options, _ = parser.parse_args(argv)
+ options, _ = parser.parse_args(sys.argv[1:])
splash()
@@ -795,7 +796,8 @@
shell.run_cmd(f"python3 -m chip_rpc.console --device {config['silabs-thread']['CU']} -b 115200")
flush_print("Done")
+ return 0
if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
+ sys.exit(main())
diff --git a/examples/chef/pyproject.toml b/examples/chef/pyproject.toml
new file mode 100644
index 0000000..bc6b895
--- /dev/null
+++ b/examples/chef/pyproject.toml
@@ -0,0 +1,16 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[build-system]
+requires = ['setuptools', 'wheel']
+build-backend = 'setuptools.build_meta'
diff --git a/examples/chef/setup.cfg b/examples/chef/setup.cfg
new file mode 100644
index 0000000..cc1e5b1
--- /dev/null
+++ b/examples/chef/setup.cfg
@@ -0,0 +1,30 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[metadata]
+name = chef
+version = 0.0.1
+author = Project CHIP Authors
+description = Build custom sample apps for supported platforms
+
+[options]
+packages = find:
+zip_safe = False
+
+[options.package_data]
+sample_app_util =
+ test_files/sample_zap_file.zap
+ test_files/sample_zap_file_hashmeta.yaml
+
+[options.entry_points]
+console_scripts = chef = chef:main
\ No newline at end of file
diff --git a/examples/chef/setup.py b/examples/chef/setup.py
index e80c86f..ae5a663 100644
--- a/examples/chef/setup.py
+++ b/examples/chef/setup.py
@@ -11,18 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-
"""The chef package."""
import setuptools # type: ignore
-setuptools.setup(
- name='chef',
- version='0.0.1',
- author='Project CHIP Authors',
- description='Build custom sample apps for supported platforms',
- packages=setuptools.find_packages(),
- package_data={'chef': ['py.typed']},
- zip_safe=False,
-)
+setuptools.setup() # Package definition in setup.cfg
diff --git a/examples/common/pigweed/BUILD.gn b/examples/common/pigweed/BUILD.gn
index 06775bf..389c3fb 100644
--- a/examples/common/pigweed/BUILD.gn
+++ b/examples/common/pigweed/BUILD.gn
@@ -24,84 +24,82 @@
]
}
-if (chip_enable_pw_rpc) {
- import("//build_overrides/pigweed.gni")
- import("$dir_pw_protobuf_compiler/proto.gni")
+import("//build_overrides/pigweed.gni")
+import("$dir_pw_protobuf_compiler/proto.gni")
- pw_proto_library("echo_service") {
- sources = [ "$dir_pw_rpc/echo.proto" ]
- inputs = [ "$dir_pw_rpc/echo.options" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "$dir_pw_rpc"
- prefix = "echo_service"
- }
+pw_proto_library("echo_service") {
+ sources = [ "$dir_pw_rpc/echo.proto" ]
+ inputs = [ "$dir_pw_rpc/echo.options" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "$dir_pw_rpc"
+ prefix = "echo_service"
+}
- pw_proto_library("attributes_service") {
- sources = [ "protos/attributes_service.proto" ]
- inputs = [ "protos/attributes_service.options" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "attributes_service"
- }
+pw_proto_library("attributes_service") {
+ sources = [ "protos/attributes_service.proto" ]
+ inputs = [ "protos/attributes_service.options" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "attributes_service"
+}
- pw_proto_library("device_service") {
- sources = [ "protos/device_service.proto" ]
- inputs = [ "protos/device_service.options" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "device_service"
- }
+pw_proto_library("device_service") {
+ sources = [ "protos/device_service.proto" ]
+ inputs = [ "protos/device_service.options" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "device_service"
+}
- pw_proto_library("descriptor_service") {
- sources = [ "protos/descriptor_service.proto" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "descriptor_service"
- }
+pw_proto_library("descriptor_service") {
+ sources = [ "protos/descriptor_service.proto" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "descriptor_service"
+}
- pw_proto_library("button_service") {
- sources = [ "protos/button_service.proto" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "button_service"
- }
+pw_proto_library("button_service") {
+ sources = [ "protos/button_service.proto" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "button_service"
+}
- pw_proto_library("lighting_service") {
- sources = [ "protos/lighting_service.proto" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "lighting_service"
- }
+pw_proto_library("lighting_service") {
+ sources = [ "protos/lighting_service.proto" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "lighting_service"
+}
- pw_proto_library("locking_service") {
- sources = [ "protos/locking_service.proto" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "locking_service"
- }
+pw_proto_library("locking_service") {
+ sources = [ "protos/locking_service.proto" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "locking_service"
+}
- pw_proto_library("ot_cli_service") {
- sources = [ "protos/ot_cli_service.proto" ]
- inputs = [ "protos/ot_cli_service.options" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "ot_cli_service"
- }
+pw_proto_library("ot_cli_service") {
+ sources = [ "protos/ot_cli_service.proto" ]
+ inputs = [ "protos/ot_cli_service.options" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "ot_cli_service"
+}
- pw_proto_library("thread_service") {
- sources = [ "protos/thread_service.proto" ]
- inputs = [ "protos/thread_service.options" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "thread_service"
- }
+pw_proto_library("thread_service") {
+ sources = [ "protos/thread_service.proto" ]
+ inputs = [ "protos/thread_service.options" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "thread_service"
+}
- pw_proto_library("wifi_service") {
- sources = [ "protos/wifi_service.proto" ]
- deps = [ "$dir_pw_protobuf:common_protos" ]
- strip_prefix = "protos"
- prefix = "wifi_service"
- }
+pw_proto_library("wifi_service") {
+ sources = [ "protos/wifi_service.proto" ]
+ deps = [ "$dir_pw_protobuf:common_protos" ]
+ strip_prefix = "protos"
+ prefix = "wifi_service"
}
pw_source_set("rpc_services") {
diff --git a/examples/common/pigweed/rpc_console/.gn b/examples/common/pigweed/rpc_console/.gn
index 22e0c41..066dd0c 100644
--- a/examples/common/pigweed/rpc_console/.gn
+++ b/examples/common/pigweed/rpc_console/.gn
@@ -19,4 +19,11 @@
default_args = {
chip_enable_pw_rpc = true
+
+ pw_build_PIP_CONSTRAINTS =
+ [ "//third_party/connectedhomeip/scripts/constraints.txt" ]
+ pw_build_PIP_REQUIREMENTS =
+ [ "//third_party/connectedhomeip/scripts/requirements.txt" ]
+
+ pw_build_USE_NEW_PYTHON_BUILD = true
}
diff --git a/examples/common/pigweed/rpc_console/BUILD.gn b/examples/common/pigweed/rpc_console/BUILD.gn
index 4001499..fd62a9d 100644
--- a/examples/common/pigweed/rpc_console/BUILD.gn
+++ b/examples/common/pigweed/rpc_console/BUILD.gn
@@ -13,10 +13,31 @@
# limitations under the License.
import("//build_overrides/chip.gni")
import("//build_overrides/pigweed.gni")
+import("$dir_pw_build/python_dist.gni")
group("default") {
deps = [
- "py:chip_rpc.install",
+ ":chip_rpc_distribution.wheel",
+ ":pip_install_editable_chip_rpc",
+ "py:chip_rpc._build_wheel",
"py:chip_rpc_wheel",
]
}
+
+# This target creates a single Python package and wheel for chip-rpc. It will
+# merge all Python dependencies from Pigweed and Matter. The output is located
+# in:
+# out/obj/chip_rpc_distribution/ <- source files here
+# out/obj/chip_rpc_distribution._build_wheel/chip_rpc-0.0.1-py3-none-any.whl
+pw_create_python_source_tree("chip_rpc_distribution") {
+ packages = [ "py:chip_rpc" ]
+ generate_setup_cfg = {
+ common_config_file = "common_setup.cfg"
+ }
+ extra_files = [ "py/pyproject.toml > pyproject.toml" ]
+}
+
+pw_internal_pip_install("pip_install_editable_chip_rpc") {
+ packages = [ "py:chip_rpc" ]
+ editable = true
+}
diff --git a/examples/common/pigweed/rpc_console/common_setup.cfg b/examples/common/pigweed/rpc_console/common_setup.cfg
new file mode 100644
index 0000000..af1401a
--- /dev/null
+++ b/examples/common/pigweed/rpc_console/common_setup.cfg
@@ -0,0 +1,19 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[metadata]
+name = chip_rpc
+version = 0.0.1
+
+[options]
+zip_safe = False
diff --git a/examples/common/pigweed/rpc_console/py/setup.cfg b/examples/common/pigweed/rpc_console/py/setup.cfg
index a60214e..c2189cb 100644
--- a/examples/common/pigweed/rpc_console/py/setup.cfg
+++ b/examples/common/pigweed/rpc_console/py/setup.cfg
@@ -19,11 +19,8 @@
packages = find:
zip_safe = False
install_requires =
- pw_console
- pw_hdlc
- pw_protobuf_compiler
- pw_rpc
- pw_tokenizer
+ pyserial
+ prompt_toolkit
[options.entry_points]
console_scripts = chip-console = chip_rpc.console:main
diff --git a/examples/common/pigweed/rpc_console/py/setup.py b/examples/common/pigweed/rpc_console/py/setup.py
index ba25589..515889f 100644
--- a/examples/common/pigweed/rpc_console/py/setup.py
+++ b/examples/common/pigweed/rpc_console/py/setup.py
@@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+"""The chip-rpc package."""
import setuptools # type: ignore
diff --git a/integrations/mobly/BUILD.gn b/integrations/mobly/BUILD.gn
index 731e84e..57699c4 100644
--- a/integrations/mobly/BUILD.gn
+++ b/integrations/mobly/BUILD.gn
@@ -17,5 +17,18 @@
import("$dir_pw_build/python.gni")
pw_python_package("chip_mobly") {
- setup = [ "setup.py" ]
+ setup = [
+ "setup.py",
+ "setup.cfg",
+ "pyproject.toml",
+ ]
+ sources = [
+ "chip_mobly/__init__.py",
+ "chip_mobly/pigweed_device.py",
+ ]
+ tests = [ "hello_world_test.py" ]
+ python_deps = [
+ "$dir_pw_hdlc/py",
+ "$dir_pw_rpc/py",
+ ]
}
diff --git a/integrations/mobly/chip_mobly/py.typed b/integrations/mobly/chip_mobly/py.typed
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/integrations/mobly/chip_mobly/py.typed
diff --git a/integrations/mobly/pyproject.toml b/integrations/mobly/pyproject.toml
new file mode 100644
index 0000000..bc6b895
--- /dev/null
+++ b/integrations/mobly/pyproject.toml
@@ -0,0 +1,16 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[build-system]
+requires = ['setuptools', 'wheel']
+build-backend = 'setuptools.build_meta'
diff --git a/integrations/mobly/setup.cfg b/integrations/mobly/setup.cfg
new file mode 100644
index 0000000..ffb6251
--- /dev/null
+++ b/integrations/mobly/setup.cfg
@@ -0,0 +1,27 @@
+# Copyright (c) 2020 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[metadata]
+name = chip_mobly
+version = 0.0.1
+author = CHIP Authors
+description = Integration of Mobly with CHIP devices
+
+[options]
+packages = find:
+zip_safe = False
+install_requires =
+ mobly
+
+[options.package_data]
+chip_modly = py.typed
diff --git a/integrations/mobly/setup.py b/integrations/mobly/setup.py
index 2e681af..9f39679 100644
--- a/integrations/mobly/setup.py
+++ b/integrations/mobly/setup.py
@@ -11,15 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
"""chip_mobly"""
import setuptools # type: ignore
-setuptools.setup(
- name='chip_mobly',
- version='0.0.1',
- author='CHIP Authors',
- description='Integration of Mobly with CHIP devices',
- packages=setuptools.find_packages(),
-)
+setuptools.setup() # Package definition in setup.cfg
diff --git a/scripts/BUILD.gn b/scripts/BUILD.gn
index 3c7080f..141e262 100644
--- a/scripts/BUILD.gn
+++ b/scripts/BUILD.gn
@@ -15,7 +15,3 @@
import("//build_overrides/pigweed.gni")
import("$dir_pw_build/python.gni")
-
-pw_python_requirements("requirements") {
- files = [ "requirements.txt" ]
-}
diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
index be3a2d7..af1b567 100644
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -35,20 +35,8 @@
git submodule update --init
fi
- local _MATTER_BANNER="$(
- cat <<-EOF
-
- █
- █
- ▄ █ ▄ █ ▌
- ▀▀█████▀▀ ▄▓▀▀▀▄,▄▀▀▀▀▄ ╓▄▀▀▀▀▄█ ▀▀█▀▀▀▀▀█▀▀ ,▄▀▀▀▀▄ ▄▀▀
- ▀█▄ ▄█▀ █ █ █ ▐▌ █ █ █ .█▄▄▄▄▄▄█⌐ ▐▌
- ▀█▄ ▄█▀ █ █ █ ▐▌ █ █ █ █ ▐▌
- ▄██▀▀█ █▀▀██▄ █ █ █ ╙▀▄▄▄Φ▀█ ▀▄▄ ▀▄▄ ▀▄▄▄▄▀^ ▐▌
-▀▀ █ █ ▀▀
-
-EOF
- )"
+ PW_BRANDING_BANNER="$_CHIP_ROOT/scripts/matter_banner.txt"
+ export PW_BRANDING_BANNER
PW_PROJECT_ROOT="$_CHIP_ROOT"
export PW_PROJECT_ROOT
@@ -60,7 +48,8 @@
_chip_bootstrap_banner() {
if [ -z "$PW_ENVSETUP_QUIET" ] && [ -z "$PW_ENVSETUP_NO_BANNER" ]; then
- pw_bold_white "$_MATTER_BANNER\n"
+ cat "$PW_BRANDING_BANNER"
+ echo
fi
}
diff --git a/scripts/constraints.esp32.txt b/scripts/constraints.esp32.txt
new file mode 100644
index 0000000..6451f1b
--- /dev/null
+++ b/scripts/constraints.esp32.txt
@@ -0,0 +1,3 @@
+cryptography>=2.1.4
+pyelftools>=0.22
+pyserial>=3.3
diff --git a/scripts/constraints.txt b/scripts/constraints.txt
index 9188515..d80489a 100644
--- a/scripts/constraints.txt
+++ b/scripts/constraints.txt
@@ -27,11 +27,11 @@
# via flask-compress
cbor==1.0.0
# via -r requirements.txt
-cbor2==5.4.0
+cbor2==5.4.3
# via -r requirements.txt
certifi==2021.5.30
# via requests
-cffi==1.14.5
+cffi==1.15.1
# via cryptography
chardet==4.0.0
# via requests
@@ -58,7 +58,7 @@
# schema
cryptography==3.4.7
# via
- # -r requirements.esp32.txt
+ # -c constraints.esp32.txt
# -r requirements.txt
cxxfilt==0.2.2
# via -r requirements.txt
@@ -193,7 +193,7 @@
# via ipython
protobuf==3.17.3
# via -r requirements.txt
-psutil==5.8.0
+psutil==5.9.1
# via
# -r requirements.txt
# mobly
@@ -203,8 +203,6 @@
# via pytest
pycparser==2.20
# via cffi
-pyelftools==0.27
- # via -r requirements.esp32.txt
pygdbmi==0.9.0.2
# via
# -r requirements.esp32.txt
@@ -223,7 +221,7 @@
# via jsonschema
pyserial==3.5
# via
- # -r requirements.esp32.txt
+ # -c constraints.esp32.txt
# mbed-os-tools
# mobly
pytest==6.2.5 ; platform_machine != "aarch64" and sys_platform == "linux"
@@ -244,7 +242,7 @@
# via pandas
pyudev==0.23.2 ; platform_machine != "aarch64" and sys_platform == "linux"
# via -r requirements.mbed.txt
-pyyaml==5.4.1
+pyyaml==6.0
# via
# idf-component-manager
# mobly
diff --git a/scripts/environment.json b/scripts/environment.json
index f68d976..e86227f 100644
--- a/scripts/environment.json
+++ b/scripts/environment.json
@@ -6,7 +6,9 @@
],
"virtualenv": {
"gn_root": ".",
- "gn_targets": [":python_packages.install"]
+ "gn_targets": [":python_packages.install"],
+ "requirements": ["scripts/requirements.txt"],
+ "constraints": ["scripts/constraints.txt"]
},
"required_submodules": ["third_party/pigweed/repo"],
"rosetta": "never",
diff --git a/scripts/environment_no_cipd.json b/scripts/environment_no_cipd.json
index dbb3106..8a98fd1 100644
--- a/scripts/environment_no_cipd.json
+++ b/scripts/environment_no_cipd.json
@@ -1,7 +1,9 @@
{
"virtualenv": {
"gn_root": ".",
- "gn_targets": [":python_packages.install"]
+ "gn_targets": [":python_packages.install"],
+ "requirements": ["scripts/requirements.txt"],
+ "constraints": ["scripts/constraints.txt"]
},
"required_submodules": ["third_party/pigweed/repo"],
"rosetta": "never",
diff --git a/scripts/matter_banner.txt b/scripts/matter_banner.txt
new file mode 100644
index 0000000..401c8ce
--- /dev/null
+++ b/scripts/matter_banner.txt
@@ -0,0 +1,8 @@
+[0;37m █[0m
+[0;37m █[0m
+[0;37m ▄ █ ▄ █ █[0m
+[0;37m ▀▀█████▀▀ ▄▀▀▀▄ ▄▀▀▀▄ ▄▀▀▀▀▄█ ▀▀█▀▀▀▀▀█▀▀ ▄▀▀▀▀▄ ▄▀▀[0m
+[0;37m ▀█▄ ▄█▀ █ █ █ █ █ █ █ █▄▄▄▄▄▄█ █ [0m
+[0;37m ▀█▄ ▄█▀ █ █ █ █ █ █ █ █ █ [0m
+[0;37m ▄██▀▀█ █▀▀██▄ █ █ █ ▀▄▄▄▄▀█ ▀▄▄ ▀▄▄ ▀▄▄▄▄▀ █ [0m
+[0;37m ▀▀ █ █ ▀▀[0m
diff --git a/scripts/requirements.esp32.txt b/scripts/requirements.esp32.txt
index 1b48a89..c870b03 100644
--- a/scripts/requirements.esp32.txt
+++ b/scripts/requirements.esp32.txt
@@ -1,10 +1,6 @@
-setuptools>=21
click>=7.0
-pyserial>=3.3
future>=0.15.2
-cryptography>=2.1.4
pyparsing>=2.0.3,<2.4.0
-pyelftools>=0.22
idf-component-manager>=0.2.99-beta
gdbgui==0.13.2.0
pygdbmi<=0.9.0.2
diff --git a/scripts/requirements.nrfconnect.txt b/scripts/requirements.nrfconnect.txt
index 779079d..ace7727 100644
--- a/scripts/requirements.nrfconnect.txt
+++ b/scripts/requirements.nrfconnect.txt
@@ -1,2 +1,2 @@
jsonschema>=4.4.0
-cbor2>=5.4.0
\ No newline at end of file
+cbor2>=5.4.3
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index abbc877..7932454 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -2,6 +2,7 @@
virtualenv
# esp-idf
+-c constraints.esp32.txt
-r requirements.esp32.txt
# mbed-os
diff --git a/src/test_driver/efr32/py/setup.cfg b/src/test_driver/efr32/py/setup.cfg
index fe65246..77108ab 100644
--- a/src/test_driver/efr32/py/setup.cfg
+++ b/src/test_driver/efr32/py/setup.cfg
@@ -18,7 +18,3 @@
[options]
packages = find:
zip_safe = False
-install_requires =
- pw_hdlc
- pw_protobuf_compiler
- pw_rpc
diff --git a/third_party/pigweed/repo b/third_party/pigweed/repo
index 9ac358b..9235aeb 160000
--- a/third_party/pigweed/repo
+++ b/third_party/pigweed/repo
@@ -1 +1 @@
-Subproject commit 9ac358b4baaad897545b01f2d616ffd51858914b
+Subproject commit 9235aeb653e684a6f0b7b563965d85c747281a0f
