python: Gonk bundle Create a Gonk tools bundle containing Python wheels that include the firmware image and FPGA bitstream file. write_fpga.py and flash.py now default to using binaries included in the gonk_firmware python package. Change-Id: I28fe6746d9319dadda889520dc4b52476533ca95 Reviewed-on: https://pigweed-review.googlesource.com/c/gonk/+/194770 Reviewed-by: Keir Mierle <keir@google.com> Commit-Queue: Anthony DiGirolamo <tonymd@google.com>
diff --git a/BUILD.gn b/BUILD.gn index 90eb91d..ecd7a90 100644 --- a/BUILD.gn +++ b/BUILD.gn
@@ -47,10 +47,7 @@ # This group requires the following tools for the FPGA build: # yosys, nextpnr-ice40, icepack, icetime group("fpga") { - deps = [ - ":pip_install_gonk_fpga", - "//fpga", - ] + deps = [ "//fpga" ] } # Python Targets @@ -75,7 +72,7 @@ } # Bundle of all Gonk Python packages and their Pigweed dependencies -pw_python_distribution("gonk_python_distribution") { +pw_python_distribution("gonk_python_tools") { packages = _gonk_python_packages + _gonk_proto_packages generate_setup_cfg = { name = "gonk-dist" @@ -86,15 +83,15 @@ } # Pip install target for Gonk Python tools and protos. -pw_python_pip_install("pip_install_gonk_dist") { - packages = [ ":gonk_python_distribution" ] +pw_python_pip_install("pip_install_gonk_tools") { + packages = [ ":gonk_python_tools" ] } -# Python data package for the FPGA build. -pw_python_distribution("gonk_fpga_distribution") { +# Python data package for Gonk firmware build. +pw_python_distribution("gonk_firmware") { packages = [] generate_setup_cfg = { - name = "gonk-fpga" + name = "gonk-firmware" version = "0.0.1" append_date_to_version = true include_default_pyproject_file = true @@ -102,28 +99,57 @@ auto_create_package_data_init_py_files = true } - public_deps = [] + public_deps = [ + "//applications/fpga_config:fpga_config(//targets/stm32f730r8tx_arduino:arduino_size_optimized)", + "//applications/fpga_config:fpga_config.elf(//targets/stm32f730r8tx_arduino:arduino_size_optimized)", + ] + + _gonk_bin = "$root_build_dir/arduino_size_optimized/obj/applications/fpga_config/fpga_config.bin" + _gonk_elf = "$root_build_dir/arduino_size_optimized/obj/applications/fpga_config/bin/fpga_config.elf" + extra_files = [ + "$_gonk_bin > gonk_firmware/gonk.bin", + "$_gonk_elf > gonk_firmware/gonk.elf", + ] # The FPGA image is only built on Linux so far. if (host_os == "linux") { _fpga_artifacts_dir = "$root_build_dir/obj/fpga/toplevel" public_deps += [ "//fpga:toplevel._bin($default_toolchain)" ] - extra_files = - [ "$_fpga_artifacts_dir/toplevel.bin > gonk_fpga/toplevel.bin" ] + extra_files += + [ "$_fpga_artifacts_dir/toplevel.bin > gonk_firmware/fpga.bin" ] } } -# Pip install target for the FPGA datapackage only. This is sparate since the -# FPGA toolchain is an optional build component. -pw_python_pip_install("pip_install_gonk_fpga") { - packages = [ ":gonk_fpga_distribution" ] +# Pip install target for the firmware datapackage only. This is separate since +# the FPGA toolchain is an optional build component. +pw_python_pip_install("pip_install_gonk_firmware") { + packages = [ ":gonk_firmware" ] } # Python group used durring bootstrap. pw_python_group("python") { python_deps = [ - ":pip_install_gonk_dist", + ":pip_install_gonk_tools", "$dir_pw_env_setup:pip_install_pigweed_package", ] } + +_gonk_bundle_packages = + _gonk_python_packages + _gonk_proto_packages + [ ":gonk_firmware" ] + +pw_python_venv("gonk_bundle_venv") { + path = "$root_build_dir/gonk-bundle-venv" + source_packages = _gonk_bundle_packages +} + +pw_python_zip_with_setup("gonk_bundle") { + # Include vendored wheels for this virtualenv. + venv = ":gonk_bundle_venv" + + # Gonk packages to include + packages = [ + ":gonk_firmware", + ":gonk_python_tools", + ] +}
diff --git a/README.md b/README.md index af729c0..48e3532 100644 --- a/README.md +++ b/README.md
@@ -111,6 +111,38 @@ ./scripts/flash-with-blackmagic-probe.sh ./out/gn/arduino_size_optimized/obj/applications/spi_flash_test/bin/spi_flash_test.elf ``` +## Generating the Gonk Python Bundle + +On Linux with the FPGA toolchain available run: + +```sh +pw build +``` + +The zip file containing all the dependencies for the gonk python tooling is located in: + +```sh +out/gn/obj/gonk_bundle.zip +``` + +Inside are the Python wheels for `gonk_dist`, `gonk_firmware`, and all +third_party dependencies. + +``` +gonk_bundle +├── python_wheels +│ ├── appdirs-1.4.4-py2.py3-none-any.whl +│ ├── astroid-3.0.1-py3-none-any.whl +│ ├── ... +│ ├── gonk_dist-0.0.1+20240305140627-py3-none-any.whl +│ ├── gonk_firmware-0.0.1+20240305140542-py3-none-any.whl +│ ├── ... +│ └── wheel-0.40.0-py3-none-any.whl +├── requirements.txt +├── setup.bat +└── setup.sh +``` + ## pw_system Example ### Run on Host
diff --git a/tools/BUILD.gn b/tools/BUILD.gn index d7027cd..117a1fe 100644 --- a/tools/BUILD.gn +++ b/tools/BUILD.gn
@@ -26,6 +26,7 @@ "gonk_tools/binary_handler.py", "gonk_tools/build_project.py", "gonk_tools/find_serial_port.py", + "gonk_tools/firmware_files.py", "gonk_tools/flash.py", "gonk_tools/gonk_log_stream.py", "gonk_tools/presubmit_checks.py",
diff --git a/tools/gonk_tools/build_project.py b/tools/gonk_tools/build_project.py index 1b1c874..c07ab9b 100644 --- a/tools/gonk_tools/build_project.py +++ b/tools/gonk_tools/build_project.py
@@ -59,13 +59,15 @@ '--export-compile-commands', ] - default_gn_args: dict[str, str] = dict() + default_gn_args: dict[str, str] = {} default_gn_gen_command.append(gn_args(**default_gn_args)) default_targets = ['default'] if _ENABLE_FPGA_BUILD: - default_targets += ['fpga'] + # Build the FPGA image, gonk tools bundle, and pip install the + # gonk_firmware package into the bootstrap venv. + default_targets += ['fpga', 'gonk_bundle', 'pip_install_gonk_firmware'] else: _LOG.warning( 'FPGA build disabled. The following tools were not found: '
diff --git a/tools/gonk_tools/firmware_files.py b/tools/gonk_tools/firmware_files.py new file mode 100644 index 0000000..5048a48 --- /dev/null +++ b/tools/gonk_tools/firmware_files.py
@@ -0,0 +1,71 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Checks for firmware files bundled into the gonk_firmware module.""" + +import importlib.resources + +FIRMWARE_PY_PACKAGE = 'gonk_firmware' + +BUNDLED_FPGA_BINFILE_NAME = 'fpga.bin' +BUNDLED_FPGA_BINFILE = '' +BUNDLED_ELF_NAME = 'gonk.elf' +BUNDLED_ELF = '' +BUNDLED_BIN_NAME = 'gonk.bin' +BUNDLED_BIN = '' + +# Check for a bundled Gonk firmware and FPGA bitstream file. +try: + with importlib.resources.as_file( + importlib.resources.files(FIRMWARE_PY_PACKAGE) / + BUNDLED_FPGA_BINFILE_NAME) as bin_path: + if bin_path.is_file(): + BUNDLED_FPGA_BINFILE = BUNDLED_FPGA_BINFILE_NAME + + with importlib.resources.as_file( + importlib.resources.files(FIRMWARE_PY_PACKAGE) / + BUNDLED_ELF_NAME) as elf_path: + if elf_path.is_file(): + BUNDLED_ELF = BUNDLED_ELF_NAME + + with importlib.resources.as_file( + importlib.resources.files(FIRMWARE_PY_PACKAGE) / + BUNDLED_BIN_NAME) as bin_path: + if bin_path.is_file(): + BUNDLED_BIN = BUNDLED_BIN_NAME +except ModuleNotFoundError: + pass + + +def bundled_bin_path(): + try: + return importlib.resources.files(FIRMWARE_PY_PACKAGE).joinpath( + BUNDLED_BIN) + except ModuleNotFoundError: + return None + + +def bundled_elf_path(): + try: + return importlib.resources.files(FIRMWARE_PY_PACKAGE).joinpath( + BUNDLED_ELF) + except ModuleNotFoundError: + return None + + +def bundled_fpga_binfile_path(): + try: + return importlib.resources.files(FIRMWARE_PY_PACKAGE).joinpath( + BUNDLED_FPGA_BINFILE) + except ModuleNotFoundError: + return None
diff --git a/tools/gonk_tools/flash.py b/tools/gonk_tools/flash.py index 7b7c2c1..2887116 100644 --- a/tools/gonk_tools/flash.py +++ b/tools/gonk_tools/flash.py
@@ -19,9 +19,12 @@ import signal import subprocess import sys +from typing import Optional import pw_cli.color +from gonk_tools.firmware_files import bundled_bin_path + _COLOR = pw_cli.color.colors() DFU_SERIAL_STRING = 'STM32FxSTM32' @@ -36,7 +39,7 @@ ) parser.add_argument( - 'bin_file', + '--bin-file', help='Binary to flash with dfu-util', type=Path, ) @@ -44,8 +47,8 @@ def main( - bin_file: Path, dfu_serial_string: str, + bin_file: Optional[Path], ) -> int: """Flash Gonk via dfu-util.""" @@ -53,8 +56,12 @@ if not dfu_util_binary or not Path(dfu_util_binary).is_file(): raise FileNotFoundError('Unable to find "dfu-util"') - if not bin_file.is_file(): - raise FileNotFoundError(f'Unable to find the file "{bin_file}"') + if not bin_file: + bin_file = bundled_bin_path() + + if not bin_file or not bin_file.is_file(): + raise FileNotFoundError( + f'Unable to find the binary file to flash: "{bin_file}"') dfu_flash_args = [ dfu_util_binary,
diff --git a/tools/gonk_tools/write_fpga.py b/tools/gonk_tools/write_fpga.py index fd879f7..55c3653 100644 --- a/tools/gonk_tools/write_fpga.py +++ b/tools/gonk_tools/write_fpga.py
@@ -14,7 +14,6 @@ """Write a binary over serial to provision the Gonk FPGA.""" import argparse -import importlib.resources from itertools import islice import logging import operator @@ -38,6 +37,12 @@ ) from gonk_tools.gonk_log_stream import GonkLogStream +from gonk_tools.firmware_files import ( + BUNDLED_ELF, + BUNDLED_FPGA_BINFILE, + bundled_elf_path, + bundled_fpga_binfile_path, +) _ROOT_LOG = logging.getLogger() _LOG = logging.getLogger('host') @@ -45,19 +50,6 @@ _COLOR = pw_cli.color.colors() -_BUNDLED_FPGA_BINFILE = '' -_BUNDLED_FPGA_BINFILE_NAME = 'toplevel.bin' - -# Check for a bundled FPGA bitstream file. -try: - with importlib.resources.as_file( - importlib.resources.files('gonk_fpga') / - _BUNDLED_FPGA_BINFILE_NAME) as bin_path: - if bin_path.is_file(): - _BUNDLED_FPGA_BINFILE = _BUNDLED_FPGA_BINFILE_NAME -except ModuleNotFoundError: - pass - def _parse_args(): parser = argparse.ArgumentParser( @@ -259,7 +251,7 @@ except serial.SerialException: self.alive = False self.console.cancel() - raise # XXX handle instead of re-raise? + raise def load_bitstream_file(bitstream_file: Path) -> bytes: @@ -267,11 +259,13 @@ bitstream_bytes = b'' if str(bitstream_file) == 'DEFAULT': - if not _BUNDLED_FPGA_BINFILE: + if not BUNDLED_FPGA_BINFILE: raise FileNotFoundError('No default bitstream file is available.') - bitstream_path = importlib.resources.files('gonk_fpga').joinpath( - _BUNDLED_FPGA_BINFILE) + bitstream_path = bundled_fpga_binfile_path() + if not bitstream_path: + raise FileNotFoundError('No default bitstream file is available.') + bitstream_bytes = bitstream_path.read_bytes() else: if not bitstream_file.is_file(): @@ -322,6 +316,13 @@ ) -> int: """Write a bitstream file over serial while monitoring output.""" + if not databases and BUNDLED_ELF: + databases = [] + elf_file = bundled_elf_path() + if elf_file: + databases.append( + pw_tokenizer_database.load_token_database(elf_file)) + if not logfile: # Create a temp logfile to prevent logs from appearing over stdout. This # would corrupt the prompt toolkit UI.