python: Gonk bundle
Create a Gonk tools bundle containing Python wheels that include the
firmware image and FPGA bitstream file.
write_fpga.py and flash.py now default to using binaries included in
the gonk_firmware python package.
Change-Id: I28fe6746d9319dadda889520dc4b52476533ca95
Reviewed-on: https://pigweed-review.googlesource.com/c/gonk/+/194770
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Anthony DiGirolamo <tonymd@google.com>
diff --git a/BUILD.gn b/BUILD.gn
index 90eb91d..ecd7a90 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -47,10 +47,7 @@
# This group requires the following tools for the FPGA build:
# yosys, nextpnr-ice40, icepack, icetime
group("fpga") {
- deps = [
- ":pip_install_gonk_fpga",
- "//fpga",
- ]
+ deps = [ "//fpga" ]
}
# Python Targets
@@ -75,7 +72,7 @@
}
# Bundle of all Gonk Python packages and their Pigweed dependencies
-pw_python_distribution("gonk_python_distribution") {
+pw_python_distribution("gonk_python_tools") {
packages = _gonk_python_packages + _gonk_proto_packages
generate_setup_cfg = {
name = "gonk-dist"
@@ -86,15 +83,15 @@
}
# Pip install target for Gonk Python tools and protos.
-pw_python_pip_install("pip_install_gonk_dist") {
- packages = [ ":gonk_python_distribution" ]
+pw_python_pip_install("pip_install_gonk_tools") {
+ packages = [ ":gonk_python_tools" ]
}
-# Python data package for the FPGA build.
-pw_python_distribution("gonk_fpga_distribution") {
+# Python data package for Gonk firmware build.
+pw_python_distribution("gonk_firmware") {
packages = []
generate_setup_cfg = {
- name = "gonk-fpga"
+ name = "gonk-firmware"
version = "0.0.1"
append_date_to_version = true
include_default_pyproject_file = true
@@ -102,28 +99,57 @@
auto_create_package_data_init_py_files = true
}
- public_deps = []
+ public_deps = [
+ "//applications/fpga_config:fpga_config(//targets/stm32f730r8tx_arduino:arduino_size_optimized)",
+ "//applications/fpga_config:fpga_config.elf(//targets/stm32f730r8tx_arduino:arduino_size_optimized)",
+ ]
+
+ _gonk_bin = "$root_build_dir/arduino_size_optimized/obj/applications/fpga_config/fpga_config.bin"
+ _gonk_elf = "$root_build_dir/arduino_size_optimized/obj/applications/fpga_config/bin/fpga_config.elf"
+ extra_files = [
+ "$_gonk_bin > gonk_firmware/gonk.bin",
+ "$_gonk_elf > gonk_firmware/gonk.elf",
+ ]
# The FPGA image is only built on Linux so far.
if (host_os == "linux") {
_fpga_artifacts_dir = "$root_build_dir/obj/fpga/toplevel"
public_deps += [ "//fpga:toplevel._bin($default_toolchain)" ]
- extra_files =
- [ "$_fpga_artifacts_dir/toplevel.bin > gonk_fpga/toplevel.bin" ]
+ extra_files +=
+ [ "$_fpga_artifacts_dir/toplevel.bin > gonk_firmware/fpga.bin" ]
}
}
-# Pip install target for the FPGA datapackage only. This is sparate since the
-# FPGA toolchain is an optional build component.
-pw_python_pip_install("pip_install_gonk_fpga") {
- packages = [ ":gonk_fpga_distribution" ]
+# Pip install target for the firmware datapackage only. This is separate since
+# the FPGA toolchain is an optional build component.
+pw_python_pip_install("pip_install_gonk_firmware") {
+ packages = [ ":gonk_firmware" ]
}
# Python group used durring bootstrap.
pw_python_group("python") {
python_deps = [
- ":pip_install_gonk_dist",
+ ":pip_install_gonk_tools",
"$dir_pw_env_setup:pip_install_pigweed_package",
]
}
+
+_gonk_bundle_packages =
+ _gonk_python_packages + _gonk_proto_packages + [ ":gonk_firmware" ]
+
+pw_python_venv("gonk_bundle_venv") {
+ path = "$root_build_dir/gonk-bundle-venv"
+ source_packages = _gonk_bundle_packages
+}
+
+pw_python_zip_with_setup("gonk_bundle") {
+ # Include vendored wheels for this virtualenv.
+ venv = ":gonk_bundle_venv"
+
+ # Gonk packages to include
+ packages = [
+ ":gonk_firmware",
+ ":gonk_python_tools",
+ ]
+}
diff --git a/README.md b/README.md
index af729c0..48e3532 100644
--- a/README.md
+++ b/README.md
@@ -111,6 +111,38 @@
./scripts/flash-with-blackmagic-probe.sh ./out/gn/arduino_size_optimized/obj/applications/spi_flash_test/bin/spi_flash_test.elf
```
+## Generating the Gonk Python Bundle
+
+On Linux with the FPGA toolchain available run:
+
+```sh
+pw build
+```
+
+The zip file containing all the dependencies for the gonk python tooling is located in:
+
+```sh
+out/gn/obj/gonk_bundle.zip
+```
+
+Inside are the Python wheels for `gonk_dist`, `gonk_firmware`, and all
+third_party dependencies.
+
+```
+gonk_bundle
+├── python_wheels
+│ ├── appdirs-1.4.4-py2.py3-none-any.whl
+│ ├── astroid-3.0.1-py3-none-any.whl
+│ ├── ...
+│ ├── gonk_dist-0.0.1+20240305140627-py3-none-any.whl
+│ ├── gonk_firmware-0.0.1+20240305140542-py3-none-any.whl
+│ ├── ...
+│ └── wheel-0.40.0-py3-none-any.whl
+├── requirements.txt
+├── setup.bat
+└── setup.sh
+```
+
## pw_system Example
### Run on Host
diff --git a/tools/BUILD.gn b/tools/BUILD.gn
index d7027cd..117a1fe 100644
--- a/tools/BUILD.gn
+++ b/tools/BUILD.gn
@@ -26,6 +26,7 @@
"gonk_tools/binary_handler.py",
"gonk_tools/build_project.py",
"gonk_tools/find_serial_port.py",
+ "gonk_tools/firmware_files.py",
"gonk_tools/flash.py",
"gonk_tools/gonk_log_stream.py",
"gonk_tools/presubmit_checks.py",
diff --git a/tools/gonk_tools/build_project.py b/tools/gonk_tools/build_project.py
index 1b1c874..c07ab9b 100644
--- a/tools/gonk_tools/build_project.py
+++ b/tools/gonk_tools/build_project.py
@@ -59,13 +59,15 @@
'--export-compile-commands',
]
- default_gn_args: dict[str, str] = dict()
+ default_gn_args: dict[str, str] = {}
default_gn_gen_command.append(gn_args(**default_gn_args))
default_targets = ['default']
if _ENABLE_FPGA_BUILD:
- default_targets += ['fpga']
+ # Build the FPGA image, gonk tools bundle, and pip install the
+ # gonk_firmware package into the bootstrap venv.
+ default_targets += ['fpga', 'gonk_bundle', 'pip_install_gonk_firmware']
else:
_LOG.warning(
'FPGA build disabled. The following tools were not found: '
diff --git a/tools/gonk_tools/firmware_files.py b/tools/gonk_tools/firmware_files.py
new file mode 100644
index 0000000..5048a48
--- /dev/null
+++ b/tools/gonk_tools/firmware_files.py
@@ -0,0 +1,71 @@
+# Copyright 2023 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Checks for firmware files bundled into the gonk_firmware module."""
+
+import importlib.resources
+
+FIRMWARE_PY_PACKAGE = 'gonk_firmware'
+
+BUNDLED_FPGA_BINFILE_NAME = 'fpga.bin'
+BUNDLED_FPGA_BINFILE = ''
+BUNDLED_ELF_NAME = 'gonk.elf'
+BUNDLED_ELF = ''
+BUNDLED_BIN_NAME = 'gonk.bin'
+BUNDLED_BIN = ''
+
+# Check for a bundled Gonk firmware and FPGA bitstream file.
+try:
+ with importlib.resources.as_file(
+ importlib.resources.files(FIRMWARE_PY_PACKAGE) /
+ BUNDLED_FPGA_BINFILE_NAME) as bin_path:
+ if bin_path.is_file():
+ BUNDLED_FPGA_BINFILE = BUNDLED_FPGA_BINFILE_NAME
+
+ with importlib.resources.as_file(
+ importlib.resources.files(FIRMWARE_PY_PACKAGE) /
+ BUNDLED_ELF_NAME) as elf_path:
+ if elf_path.is_file():
+ BUNDLED_ELF = BUNDLED_ELF_NAME
+
+ with importlib.resources.as_file(
+ importlib.resources.files(FIRMWARE_PY_PACKAGE) /
+ BUNDLED_BIN_NAME) as bin_path:
+ if bin_path.is_file():
+ BUNDLED_BIN = BUNDLED_BIN_NAME
+except ModuleNotFoundError:
+ pass
+
+
+def bundled_bin_path():
+ try:
+ return importlib.resources.files(FIRMWARE_PY_PACKAGE).joinpath(
+ BUNDLED_BIN)
+ except ModuleNotFoundError:
+ return None
+
+
+def bundled_elf_path():
+ try:
+ return importlib.resources.files(FIRMWARE_PY_PACKAGE).joinpath(
+ BUNDLED_ELF)
+ except ModuleNotFoundError:
+ return None
+
+
+def bundled_fpga_binfile_path():
+ try:
+ return importlib.resources.files(FIRMWARE_PY_PACKAGE).joinpath(
+ BUNDLED_FPGA_BINFILE)
+ except ModuleNotFoundError:
+ return None
diff --git a/tools/gonk_tools/flash.py b/tools/gonk_tools/flash.py
index 7b7c2c1..2887116 100644
--- a/tools/gonk_tools/flash.py
+++ b/tools/gonk_tools/flash.py
@@ -19,9 +19,12 @@
import signal
import subprocess
import sys
+from typing import Optional
import pw_cli.color
+from gonk_tools.firmware_files import bundled_bin_path
+
_COLOR = pw_cli.color.colors()
DFU_SERIAL_STRING = 'STM32FxSTM32'
@@ -36,7 +39,7 @@
)
parser.add_argument(
- 'bin_file',
+ '--bin-file',
help='Binary to flash with dfu-util',
type=Path,
)
@@ -44,8 +47,8 @@
def main(
- bin_file: Path,
dfu_serial_string: str,
+ bin_file: Optional[Path],
) -> int:
"""Flash Gonk via dfu-util."""
@@ -53,8 +56,12 @@
if not dfu_util_binary or not Path(dfu_util_binary).is_file():
raise FileNotFoundError('Unable to find "dfu-util"')
- if not bin_file.is_file():
- raise FileNotFoundError(f'Unable to find the file "{bin_file}"')
+ if not bin_file:
+ bin_file = bundled_bin_path()
+
+ if not bin_file or not bin_file.is_file():
+ raise FileNotFoundError(
+ f'Unable to find the binary file to flash: "{bin_file}"')
dfu_flash_args = [
dfu_util_binary,
diff --git a/tools/gonk_tools/write_fpga.py b/tools/gonk_tools/write_fpga.py
index fd879f7..55c3653 100644
--- a/tools/gonk_tools/write_fpga.py
+++ b/tools/gonk_tools/write_fpga.py
@@ -14,7 +14,6 @@
"""Write a binary over serial to provision the Gonk FPGA."""
import argparse
-import importlib.resources
from itertools import islice
import logging
import operator
@@ -38,6 +37,12 @@
)
from gonk_tools.gonk_log_stream import GonkLogStream
+from gonk_tools.firmware_files import (
+ BUNDLED_ELF,
+ BUNDLED_FPGA_BINFILE,
+ bundled_elf_path,
+ bundled_fpga_binfile_path,
+)
_ROOT_LOG = logging.getLogger()
_LOG = logging.getLogger('host')
@@ -45,19 +50,6 @@
_COLOR = pw_cli.color.colors()
-_BUNDLED_FPGA_BINFILE = ''
-_BUNDLED_FPGA_BINFILE_NAME = 'toplevel.bin'
-
-# Check for a bundled FPGA bitstream file.
-try:
- with importlib.resources.as_file(
- importlib.resources.files('gonk_fpga') /
- _BUNDLED_FPGA_BINFILE_NAME) as bin_path:
- if bin_path.is_file():
- _BUNDLED_FPGA_BINFILE = _BUNDLED_FPGA_BINFILE_NAME
-except ModuleNotFoundError:
- pass
-
def _parse_args():
parser = argparse.ArgumentParser(
@@ -259,7 +251,7 @@
except serial.SerialException:
self.alive = False
self.console.cancel()
- raise # XXX handle instead of re-raise?
+ raise
def load_bitstream_file(bitstream_file: Path) -> bytes:
@@ -267,11 +259,13 @@
bitstream_bytes = b''
if str(bitstream_file) == 'DEFAULT':
- if not _BUNDLED_FPGA_BINFILE:
+ if not BUNDLED_FPGA_BINFILE:
raise FileNotFoundError('No default bitstream file is available.')
- bitstream_path = importlib.resources.files('gonk_fpga').joinpath(
- _BUNDLED_FPGA_BINFILE)
+ bitstream_path = bundled_fpga_binfile_path()
+ if not bitstream_path:
+ raise FileNotFoundError('No default bitstream file is available.')
+
bitstream_bytes = bitstream_path.read_bytes()
else:
if not bitstream_file.is_file():
@@ -322,6 +316,13 @@
) -> int:
"""Write a bitstream file over serial while monitoring output."""
+ if not databases and BUNDLED_ELF:
+ databases = []
+ elf_file = bundled_elf_path()
+ if elf_file:
+ databases.append(
+ pw_tokenizer_database.load_token_database(elf_file))
+
if not logfile:
# Create a temp logfile to prevent logs from appearing over stdout. This
# would corrupt the prompt toolkit UI.
