Arduino: python cleanup
- Move code in arduinobuilder.py to their own modules:
- __main__.py
- builder.py
- core_installer.py
- file_operations.py
- New log.py module
- os.system -> subprocess.run
- Move --run-* args in the 'show' subcommand under the 'run' subcommand.
This allows running multiple steps in the same invocation. For
example, this will execute 3 steps:
arduino_builder run \
--run-objcopy \
--run-postbuild \
--run-upload-command teensyloader
Change-Id: Ibe5493689f39aa6e6aaeecf5bc7065929111499d
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/19143
Reviewed-by: Wyatt Hepler <hepler@google.com>
Commit-Queue: Anthony DiGirolamo <tonymd@google.com>
diff --git a/pw_arduino_build/arduino.gni b/pw_arduino_build/arduino.gni
index bf30cd4..c214632 100644
--- a/pw_arduino_build/arduino.gni
+++ b/pw_arduino_build/arduino.gni
@@ -32,7 +32,7 @@
}
arduino_builder_script =
- get_path_info("py/pw_arduino_build/arduinobuilder.py", "abspath")
+ get_path_info("py/pw_arduino_build/__main__.py", "abspath")
_arduino_core_path =
rebase_path("../third_party/arduino/cores/$arduino_core_name")
@@ -46,12 +46,20 @@
arduino_package_name,
"--compiler-path-override",
_compiler_path_override,
- "show",
- "--delimit-with-newlines",
+]
+
+arduino_board_args = [
"--build-path",
rebase_path(root_build_dir),
"--board",
arduino_board,
"--menu-options",
]
-arduino_global_args += arduino_menu_options
+arduino_board_args += arduino_menu_options
+
+arduino_show_command_args = arduino_global_args + [
+ "show",
+ "--delimit-with-newlines",
+ ] + arduino_board_args
+
+arduino_run_command_args = arduino_global_args + [ "run" ] + arduino_board_args
diff --git a/pw_arduino_build/py/pw_arduino_build/arduinobuilder_test.py b/pw_arduino_build/py/builder_test.py
similarity index 100%
rename from pw_arduino_build/py/pw_arduino_build/arduinobuilder_test.py
rename to pw_arduino_build/py/builder_test.py
diff --git a/pw_arduino_build/py/file_operations_test.py b/pw_arduino_build/py/file_operations_test.py
new file mode 100644
index 0000000..7306fbb
--- /dev/null
+++ b/pw_arduino_build/py/file_operations_test.py
@@ -0,0 +1,103 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests for file_operations module."""
+
+import os
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+from parameterized import parameterized
+
+import pw_arduino_build.file_operations as file_operations
+
+
+def file_set():
+ return [
+ "app.ino",
+ "core/asm.S",
+ "core/asm.s",
+ "core/pwm/pulse.c",
+ "core/pwm/pulse.h",
+ "libraries/a.c",
+ "libraries/b.cpp",
+ "libraries/c.cc",
+ "libraries/c.h",
+ ]
+
+
+def create_files(root_dir, file_names):
+ for file_name in file_names:
+ folder_path = Path(root_dir) / Path(os.path.dirname(file_name))
+ folder_path.mkdir(parents=True, exist_ok=True)
+ file_path = Path(root_dir) / Path(file_name)
+ file_path.touch(exist_ok=True)
+
+
+class TestFileOperations(unittest.TestCase):
+ """Tests to ensure arduino core library source files can be found."""
+ def setUp(self):
+ self.test_dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.test_dir)
+
+ @parameterized.expand([
+ (
+ "sources recursive", file_set(), ["**/*.ino", "**/*.h", "**/*.cpp"],
+ [
+ "app.ino",
+ "core/pwm/pulse.h",
+ "libraries/b.cpp",
+ "libraries/c.h",
+ ]
+ ),
+ (
+ "directories recursive", file_set(), ["**"],
+ [
+ "core",
+ "core/pwm",
+ "libraries",
+ ]
+ ),
+ (
+ "directories one level deep", file_set(), ["*"],
+ [
+ "core",
+ "libraries",
+ ]
+ ),
+ (
+ "items one level deep", file_set(), ["*"],
+ [
+ "app.ino",
+ "core",
+ "libraries",
+ ]
+ )
+ ]) # yapf: disable
+ def test_find_files(self, test_case, base_fileset, patterns,
+ expected_results):
+ """Test find_files on source files and directories.
+ """
+ create_files(self.test_dir, base_fileset)
+ result = file_operations.find_files(self.test_dir,
+ patterns,
+ directories_only=("directories"
+ in test_case))
+ self.assertSequenceEqual(expected_results, result)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pw_arduino_build/py/pw_arduino_build/__main__.py b/pw_arduino_build/py/pw_arduino_build/__main__.py
new file mode 100644
index 0000000..0bdbcf1
--- /dev/null
+++ b/pw_arduino_build/py/pw_arduino_build/__main__.py
@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Command line interface for arduino_builder."""
+
+import argparse
+import glob
+import logging
+import os
+import pprint
+import shlex
+import subprocess
+import sys
+from typing import List
+
+from pw_arduino_build import core_installer, log
+from pw_arduino_build.builder import ArduinoBuilder
+
+_LOG = logging.getLogger(__name__)
+
+_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint
+_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat
+
+
+def list_boards_command(unused_args, builder):
+ # list-boards subcommand
+ # (does not need a selected board or default menu options)
+
+ # TODO(tonymd): Print this sorted with auto-ljust columns
+ longest_name_length = 0
+ for board_name, board_dict in builder.board.items():
+ if len(board_name) > longest_name_length:
+ longest_name_length = len(board_name)
+ longest_name_length += 2
+
+ print("Board Name".ljust(longest_name_length), "Description")
+ for board_name, board_dict in builder.board.items():
+ print(board_name.ljust(longest_name_length), board_dict['name'])
+ sys.exit(0)
+
+
+def list_menu_options_command(args, builder):
+ # List all menu options for the selected board.
+ builder.select_board(args.board)
+
+ print("All Options")
+ all_options, all_column_widths = builder.get_menu_options()
+ separator = "-" * (all_column_widths[0] + all_column_widths[1] + 2)
+ print(separator)
+
+ for name, description in all_options:
+ print(name.ljust(all_column_widths[0] + 1), description)
+
+ print("\nDefault Options")
+ print(separator)
+
+ default_options, unused_column_widths = builder.get_default_menu_options()
+ for name, description in default_options:
+ print(name.ljust(all_column_widths[0] + 1), description)
+
+
+def show_command_print_string_list(args, string_list: List[str]):
+ join_token = " "
+ if args.delimit_with_newlines:
+ join_token = "\n"
+ print(join_token.join(string_list))
+
+
+def show_command_print_flag_string(args, flag_string):
+ if args.delimit_with_newlines:
+ flag_string_with_newlines = shlex.split(flag_string)
+ print("\n".join(flag_string_with_newlines))
+ else:
+ print(flag_string)
+
+
+def run_command_lines(args, command_lines: List[str]):
+ for command_line in command_lines:
+ if not args.quiet:
+ print(command_line)
+ # TODO(tonymd): Exit with sub command exit code.
+ command_line_args = shlex.split(command_line)
+ process = subprocess.run(command_line_args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ if process.returncode != 0:
+ _LOG.error('Command failed with exit code %d.', process.returncode)
+ _LOG.error('Full command:')
+ _LOG.error('')
+ _LOG.error(' %s', command_line)
+ _LOG.error('')
+ _LOG.error('Process output:')
+ print(flush=True)
+ sys.stdout.buffer.write(process.stdout)
+ print(flush=True)
+ _LOG.error('')
+
+
+def run_command(args, builder):
+ """Run sub command function.
+
+ Runs Arduino recipes.
+ """
+
+ if args.run_prebuilds:
+ run_command_lines(args, builder.get_prebuild_steps())
+
+ if args.run_link:
+ line = builder.get_link_line()
+ archive_file_path = args.run_link[0] # pylint: disable=unused-variable
+ object_files = args.run_link[1:]
+ line = line.replace("{object_files}", " ".join(object_files), 1)
+ run_command_lines(args, [line])
+
+ if args.run_objcopy:
+ run_command_lines(args, builder.get_objcopy_steps())
+
+ if args.run_postbuilds:
+ run_command_lines(args, builder.get_postbuild_steps())
+
+ if args.run_upload_command:
+ command = builder.get_upload_line(args.run_upload_command,
+ args.serial_port)
+ run_command_lines(args, [command])
+
+
+# pylint: disable=too-many-branches
+def show_command(args, builder):
+ """Show sub command function.
+
+ Prints compiler info and flags.
+ """
+ if args.cc_binary:
+ print(builder.get_cc_binary())
+
+ elif args.cxx_binary:
+ print(builder.get_cxx_binary())
+
+ elif args.objcopy_binary:
+ print(builder.get_objcopy_binary())
+
+ elif args.ar_binary:
+ print(builder.get_ar_binary())
+
+ elif args.size_binary:
+ print(builder.get_size_binary())
+
+ elif args.c_compile:
+ print(builder.get_c_compile_line())
+
+ elif args.cpp_compile:
+ print(builder.get_cpp_compile_line())
+
+ elif args.link:
+ print(builder.get_link_line())
+
+ elif args.objcopy:
+ print(builder.get_objcopy(args.objcopy))
+
+ elif args.objcopy_flags:
+ objcopy_flags = builder.get_objcopy_flags(args.objcopy_flags)
+ show_command_print_flag_string(args, objcopy_flags)
+
+ elif args.c_flags:
+ cflags = builder.get_c_flags()
+ show_command_print_flag_string(args, cflags)
+
+ elif args.s_flags:
+ sflags = builder.get_s_flags()
+ show_command_print_flag_string(args, sflags)
+
+ elif args.cpp_flags:
+ cppflags = builder.get_cpp_flags()
+ show_command_print_flag_string(args, cppflags)
+
+ elif args.ld_flags:
+ ldflags = builder.get_ld_flags()
+ show_command_print_flag_string(args, ldflags)
+
+ elif args.ld_libs:
+ print(builder.get_ld_libs())
+
+ elif args.ar_flags:
+ ar_flags = builder.get_ar_flags()
+ show_command_print_flag_string(args, ar_flags)
+
+ elif args.core_path:
+ print(builder.get_core_path())
+
+ elif args.postbuild:
+ print(builder.get_postbuild_line(args.postbuild))
+
+ elif args.upload_command:
+ print(builder.get_upload_line(args.upload_command, args.serial_port))
+
+ elif args.upload_tools:
+ tools = builder.get_upload_tool_names()
+ for tool_name in tools:
+ print(tool_name)
+
+ elif args.library_includes:
+ show_command_print_string_list(args, builder.library_includes())
+
+ elif args.library_c_files:
+ show_command_print_string_list(args, builder.library_c_files())
+
+ elif args.library_cpp_files:
+ show_command_print_string_list(args, builder.library_cpp_files())
+
+ elif args.core_c_files:
+ show_command_print_string_list(args, builder.core_c_files())
+
+ elif args.core_s_files:
+ show_command_print_string_list(args, builder.core_s_files())
+
+ elif args.core_cpp_files:
+ show_command_print_string_list(args, builder.core_cpp_files())
+
+ elif args.variant_c_files:
+ vfiles = builder.variant_c_files()
+ if vfiles:
+ show_command_print_string_list(args, vfiles)
+
+ elif args.variant_s_files:
+ vfiles = builder.variant_s_files()
+ if vfiles:
+ show_command_print_string_list(args, vfiles)
+
+ elif args.variant_cpp_files:
+ vfiles = builder.variant_cpp_files()
+ if vfiles:
+ show_command_print_string_list(args, vfiles)
+
+
+def add_common_options(parser, serial_port, build_path, build_project_name,
+ project_path, project_source_path):
+ """Add command line options common to the run and show commands."""
+ parser.add_argument(
+ "--serial-port",
+ default=serial_port,
+ help="Serial port for flashing flash. Default: '{}'".format(
+ serial_port))
+ parser.add_argument(
+ "--build-path",
+ default=build_path,
+ help="Build directory. Default: '{}'".format(build_path))
+ parser.add_argument(
+ "--project-path",
+ default=project_path,
+ help="Project directory. Default: '{}'".format(project_path))
+ parser.add_argument(
+ "--project-source-path",
+ default=project_source_path,
+ help="Project directory. Default: '{}'".format(project_source_path))
+ parser.add_argument(
+ "--build-project-name",
+ default=build_project_name,
+ help="Project name. Default: '{}'".format(build_project_name))
+ parser.add_argument("--board",
+ required=True,
+ help="Name of the board to use.")
+ # nargs="+" is one or more args, e.g:
+ # --menu-options menu.usb.serialhid menu.speed.150
+ parser.add_argument("--menu-options", nargs="+", type=str)
+
+
+def main():
+ """Main command line function.
+
+ Parses command line args and dispatches to sub `*_command()` functions.
+ """
+ def log_level(arg: str) -> int:
+ try:
+ return getattr(logging, arg.upper())
+ except AttributeError:
+ raise argparse.ArgumentTypeError(
+ f'{arg.upper()} is not a valid log level')
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-q",
+ "--quiet",
+ help="hide run command output",
+ action="store_true")
+ parser.add_argument('-l',
+ '--loglevel',
+ type=log_level,
+ default=logging.INFO,
+ help='Set the log level '
+ '(debug, info, warning, error, critical)')
+
+ build_path = os.path.realpath(
+ os.path.expanduser(os.path.expandvars("./build")))
+ project_path = os.path.realpath(
+ os.path.expanduser(os.path.expandvars("./")))
+ project_source_path = os.path.join(project_path, "src")
+ build_project_name = os.path.basename(project_path)
+
+ serial_port = "UNKNOWN"
+ # TODO(tonymd): Temp solution to passing in serial port. It should use
+ # arduino core discovery tools.
+ possible_serial_ports = glob.glob("/dev/ttyACM*") + glob.glob(
+ "/dev/ttyUSB*")
+ if possible_serial_ports:
+ serial_port = possible_serial_ports[0]
+
+ # Global command line options
+ parser.add_argument("--arduino-package-path",
+ help="Path to the arduino IDE install location.")
+ parser.add_argument("--arduino-package-name",
+ help="Name of the Arduino board package to use.")
+ parser.add_argument("--compiler-path-override",
+ help="Path to arm-none-eabi-gcc bin folder. "
+ "Default: Arduino core specified gcc")
+
+ # Subcommands
+ subparsers = parser.add_subparsers(title="subcommand",
+ description="valid subcommands",
+ help="sub-command help",
+ dest="subcommand",
+ required=True)
+
+ # install-core command
+ install_core_parser = subparsers.add_parser(
+ "install-core", help="Download and install arduino cores")
+ install_core_parser.set_defaults(func=core_installer.install_core_command)
+ install_core_parser.add_argument("--prefix",
+ required=True,
+ help="Path to install core files.")
+ install_core_parser.add_argument(
+ "--core-name",
+ required=True,
+ choices=core_installer.supported_cores(),
+ help="Name of the arduino core to install.")
+
+ # list-boards command
+ list_boards_parser = subparsers.add_parser("list-boards",
+ help="show supported boards")
+ list_boards_parser.set_defaults(func=list_boards_command)
+
+ # list-menu-options command
+ list_menu_options_parser = subparsers.add_parser(
+ "list-menu-options",
+ help="show available menu options for selected board")
+ list_menu_options_parser.set_defaults(func=list_menu_options_command)
+ list_menu_options_parser.add_argument("--board",
+ required=True,
+ help="Name of the board to use.")
+
+ # show command
+ show_parser = subparsers.add_parser("show",
+ help="Return compiler information.")
+ add_common_options(show_parser, serial_port, build_path,
+ build_project_name, project_path, project_source_path)
+ show_parser.add_argument("--delimit-with-newlines",
+ help="Separate flag output with newlines.",
+ action="store_true")
+
+ output_group = show_parser.add_mutually_exclusive_group(required=True)
+ output_group.add_argument("--c-compile", action="store_true")
+ output_group.add_argument("--cpp-compile", action="store_true")
+ output_group.add_argument("--link", action="store_true")
+ output_group.add_argument("--c-flags", action="store_true")
+ output_group.add_argument("--s-flags", action="store_true")
+ output_group.add_argument("--cpp-flags", action="store_true")
+ output_group.add_argument("--ld-flags", action="store_true")
+ output_group.add_argument("--ar-flags", action="store_true")
+ output_group.add_argument("--ld-libs", action="store_true")
+ output_group.add_argument("--objcopy", help="objcopy step for SUFFIX")
+ output_group.add_argument("--objcopy-flags",
+ help="objcopy flags for SUFFIX")
+ output_group.add_argument("--core-path", action="store_true")
+ output_group.add_argument("--cc-binary", action="store_true")
+ output_group.add_argument("--cxx-binary", action="store_true")
+ output_group.add_argument("--ar-binary", action="store_true")
+ output_group.add_argument("--objcopy-binary", action="store_true")
+ output_group.add_argument("--size-binary", action="store_true")
+ output_group.add_argument("--postbuild",
+ help="Show recipe.hooks.postbuild.*.pattern")
+ output_group.add_argument("--upload-tools", action="store_true")
+ output_group.add_argument("--upload-command")
+ output_group.add_argument("--library-includes", action="store_true")
+ output_group.add_argument("--library-c-files", action="store_true")
+ output_group.add_argument("--library-cpp-files", action="store_true")
+ output_group.add_argument("--core-c-files", action="store_true")
+ output_group.add_argument("--core-s-files", action="store_true")
+ output_group.add_argument("--core-cpp-files", action="store_true")
+ output_group.add_argument("--variant-c-files", action="store_true")
+ output_group.add_argument("--variant-s-files", action="store_true")
+ output_group.add_argument("--variant-cpp-files", action="store_true")
+
+ show_parser.set_defaults(func=show_command)
+
+ # run command
+ run_parser = subparsers.add_parser("run", help="Run Arduino recipes.")
+ add_common_options(run_parser, serial_port, build_path, build_project_name,
+ project_path, project_source_path)
+ run_parser.add_argument("--run-link",
+ nargs="+",
+ type=str,
+ help="Run the link command. Expected arguments: "
+ "the archive file followed by all obj files.")
+ run_parser.add_argument("--run-objcopy", action="store_true")
+ run_parser.add_argument("--run-prebuilds", action="store_true")
+ run_parser.add_argument("--run-postbuilds", action="store_true")
+ run_parser.add_argument("--run-upload-command")
+
+ run_parser.set_defaults(func=run_command)
+
+ args = parser.parse_args()
+
+ log.install()
+ log.set_level(args.loglevel)
+
+ _LOG.debug(_pretty_format(args))
+
+ # Check for and set alternate compiler path.
+ compiler_path_override = False
+ if args.compiler_path_override:
+ compiler_path_override = os.path.realpath(
+ os.path.expanduser(os.path.expandvars(
+ args.compiler_path_override)))
+
+ if args.subcommand == "install-core":
+ args.func(args)
+ elif args.subcommand in ["list-boards", "list-menu-options"]:
+ builder = ArduinoBuilder(args.arduino_package_path,
+ args.arduino_package_name)
+ builder.load_board_definitions()
+ args.func(args, builder)
+ else:
+ builder = ArduinoBuilder(args.arduino_package_path,
+ args.arduino_package_name,
+ build_path=args.build_path,
+ build_project_name=args.build_project_name,
+ project_path=args.project_path,
+ project_source_path=args.project_source_path,
+ compiler_path_override=compiler_path_override)
+ builder.load_board_definitions()
+ builder.select_board(args.board, args.menu_options)
+ args.func(args, builder)
+
+ sys.exit(0)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pw_arduino_build/py/pw_arduino_build/arduinobuilder.py b/pw_arduino_build/py/pw_arduino_build/arduinobuilder.py
deleted file mode 100755
index 239517d..0000000
--- a/pw_arduino_build/py/pw_arduino_build/arduinobuilder.py
+++ /dev/null
@@ -1,1894 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2020 The Pigweed Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-"""Extracts build information from Arduino cores."""
-
-import argparse
-import glob
-import hashlib
-import logging
-import os
-import platform
-import pprint
-import re
-import shutil
-import shlex
-import stat
-import sys
-import tarfile
-import time
-import urllib.request
-import zipfile
-from collections import OrderedDict
-from pathlib import Path
-from typing import List, Dict
-
-_LOG = logging.getLogger(__name__)
-_STDERR_HANDLER = logging.StreamHandler()
-
-_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint
-_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat
-# yapf: disable
-_ARDUINO_CORE_ARTIFACTS = {
- # pylint: disable=line-too-long
- "teensy": {
- "Linux": {
- "arduino-ide": {
- "url": "https://downloads.arduino.cc/arduino-1.8.13-linux64.tar.xz",
- "file_name": "arduino-1.8.13-linux64.tar.xz",
- "sha256": "1b20d0ec850a2a63488009518725f058668bb6cb48c321f82dcf47dc4299b4ad",
- },
- "teensyduino": {
- "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linux64",
- "file_name": "TeensyduinoInstall.linux64",
- "sha256": "2e6cd99a757bc80593ea3de006de4cc934bcb0a6ec74cad8ec327f0289d40f0b",
- },
- },
- # TODO(tonymd): Handle 32-bit Linux Install?
- "Linux32": {
- "arduino-ide": {
- "url": "https://downloads.arduino.cc/arduino-1.8.13-linux32.tar.xz",
- "file_name": "arduino-1.8.13-linux32.tar.xz",
- "sha256": "",
- },
- "teensyduino": {
- "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linux32",
- "file_name": "TeensyduinoInstall.linux32",
- "sha256": "",
- },
- },
- # TODO(tonymd): Handle ARM32 (Raspberry Pi) Install?
- "LinuxARM32": {
- "arduino-ide": {
- "url": "https://downloads.arduino.cc/arduino-1.8.13-linuxarm.tar.xz",
- "file_name": "arduino-1.8.13-linuxarm.tar.xz",
- "sha256": "",
- },
- "teensyduino": {
- "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linuxarm",
- "file_name": "TeensyduinoInstall.linuxarm",
- "sha256": "",
- },
- },
- # TODO(tonymd): Handle ARM64 Install?
- "LinuxARM64": {
- "arduino-ide": {
- "url": "https://downloads.arduino.cc/arduino-1.8.13-linuxaarch64.tar.xz",
- "file_name": "arduino-1.8.13-linuxaarch64.tar.xz",
- "sha256": "",
- },
- "teensyduino": {
- "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linuxaarch64",
- "file_name": "TeensyduinoInstall.linuxaarch64",
- "sha256": "",
- },
- },
- "Darwin": {
- "teensyduino": {
- "url": "https://www.pjrc.com/teensy/td_153/Teensyduino_MacOS_Catalina.zip",
- "file_name": "Teensyduino_MacOS_Catalina.zip",
- "sha256": "401ef42c6e83e621cdda20191a4ef9b7db8a214bede5a94a9e26b45f79c64fe2",
- },
- },
- "Windows": {
- "arduino-ide": {
- "url": "https://downloads.arduino.cc/arduino-1.8.13-windows.zip",
- "file_name": "arduino-1.8.13-windows.zip",
- "sha256": "78d3e96827b9e9b31b43e516e601c38d670d29f12483e88cbf6d91a0f89ef524",
- },
- "teensyduino": {
- "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.exe",
- "file_name": "TeensyduinoInstall.exe",
- "sha256": "88f58681e5c4772c54e462bc88280320e4276e5b316dcab592fe38d96db990a1",
- },
- }
- },
- "adafruit-samd": {
- "all": {
- "core": {
- "url": "https://github.com/adafruit/ArduinoCore-samd/archive/1.6.2.tar.gz",
- "sha256": "5875f5bc05904c10e6313f02653f28f2f716db639d3d43f5a1d8a83d15339d64",
- }
- },
- "Linux": {},
- "Darwin": {},
- "Windows": {},
- },
- "stm32duino": {
- "all": {
- "core": {
- "url": "https://github.com/stm32duino/Arduino_Core_STM32/archive/1.9.0.tar.gz",
- "sha256": "4f75ba7a117d90392e8f67c58d31d22393749b9cdd3279bc21e7261ec06c62bf",
- }
- },
- "Linux": {},
- "Darwin": {},
- "Windows": {},
- },
-} # type: Dict[str, Dict]
-# yapf: enable
-
-
-def arduino_runtime_os_string():
- arduno_platform = {
- "Linux": "linux",
- "Windows": "windows",
- "Darwin": "macosx"
- }
- return arduno_platform[platform.system()]
-
-
-class FileOperations:
- """File helper functions."""
- @staticmethod
- def find_files(starting_dir: str,
- patterns: List[str],
- directories_only=False) -> List[str]:
- # ["**/*.S", "**/*.ino", "**/*.h", "**/*.c", "**/*.cpp"]
-
- original_working_dir = os.getcwd()
- if not (os.path.exists(starting_dir) and os.path.isdir(starting_dir)):
- _LOG.error("Directory '%s' does not exist.", starting_dir)
- raise FileNotFoundError
-
- os.chdir(starting_dir)
- files = []
- for pattern in patterns:
- for file_path in glob.glob(pattern, recursive=True):
- if not directories_only or (directories_only
- and os.path.isdir(file_path)):
- files.append(file_path)
- os.chdir(original_working_dir)
- return sorted(files)
-
- @staticmethod
- def sha256_sum(file_name):
- hash_sha256 = hashlib.sha256()
- with open(file_name, "rb") as file_handle:
- for chunk in iter(lambda: file_handle.read(4096), b""):
- hash_sha256.update(chunk)
- return hash_sha256.hexdigest()
-
- @staticmethod
- def md5_sum(file_name):
- hash_md5 = hashlib.md5()
- with open(file_name, "rb") as file_handle:
- for chunk in iter(lambda: file_handle.read(4096), b""):
- hash_md5.update(chunk)
- return hash_md5.hexdigest()
-
- @staticmethod
- def verify_file_checksum(file_path,
- expected_checksum,
- sum_function=sha256_sum):
- downloaded_checksum = sum_function(file_path)
- if downloaded_checksum != expected_checksum:
- _LOG.error("Error: Invalid %s", sum_function.__name__)
- _LOG.error("%s %s", downloaded_checksum,
- os.path.basename(file_path))
- _LOG.error("%s (expected)", expected_checksum)
- return sys.exit(1)
-
- _LOG.info(" %s:", sum_function.__name__)
- _LOG.info(" %s %s", downloaded_checksum, os.path.basename(file_path))
- return True
-
- @staticmethod
- def download_to_cache(url: str,
- expected_md5sum=None,
- expected_sha256sum=None,
- cache_directory=".cache") -> str:
-
- cache_dir = os.path.realpath(
- os.path.expanduser(os.path.expandvars(cache_directory)))
- downloaded_file = os.path.join(cache_dir, url.split("/")[-1])
-
- if not os.path.exists(downloaded_file):
- _LOG.info("Downloading: %s", url)
- urllib.request.urlretrieve(url, filename=downloaded_file)
-
- if os.path.exists(downloaded_file):
- _LOG.info("Downloaded: %s", downloaded_file)
- if expected_sha256sum:
- FileOperations.verify_file_checksum(
- downloaded_file,
- expected_sha256sum,
- sum_function=FileOperations.sha256_sum)
- elif expected_md5sum:
- FileOperations.verify_file_checksum(
- downloaded_file,
- expected_md5sum,
- sum_function=FileOperations.md5_sum)
-
- return downloaded_file
-
- @staticmethod
- def extract_zipfile(archive_file: str, dest_dir: str):
- with zipfile.ZipFile(archive_file) as archive:
- archive.extractall(path=dest_dir)
-
- @staticmethod
- def extract_tarfile(archive_file: str, dest_dir: str):
- with tarfile.open(archive_file, 'r') as archive:
- archive.extractall(path=dest_dir)
-
- @staticmethod
- def extract_archive(archive_file: str,
- dest_dir: str,
- cache_dir: str,
- remove_single_toplevel_folder=True):
- """Extract a tar or zip file.
-
- Args:
- archive_file (str): Absolute path to the archive file.
- dest_dir (str): Extraction destination directory.
- cache_dir (str): Directory where temp files can be created.
- remove_single_toplevel_folder (bool): If the archive contains only a
- single folder move the contents of that into the destination
- directory.
- """
- # Make a temporary directory to extract files into
- temp_extract_dir = os.path.join(cache_dir,
- "." + os.path.basename(archive_file))
- os.makedirs(temp_extract_dir, exist_ok=True)
-
- _LOG.info("Extracting: %s", archive_file)
- if zipfile.is_zipfile(archive_file):
- FileOperations.extract_zipfile(archive_file, temp_extract_dir)
- elif tarfile.is_tarfile(archive_file):
- FileOperations.extract_tarfile(archive_file, temp_extract_dir)
- else:
- _LOG.error("Unknown archive format: %s", archive_file)
- return sys.exit(1)
-
- _LOG.info("Installing into: %s", dest_dir)
- path_to_extracted_files = temp_extract_dir
-
- extracted_top_level_files = os.listdir(temp_extract_dir)
- # Check if tarfile has only one folder
- # If yes, make that the new path_to_extracted_files
- if remove_single_toplevel_folder and len(
- extracted_top_level_files) == 1:
- path_to_extracted_files = os.path.join(
- temp_extract_dir, extracted_top_level_files[0])
-
- # Move extracted files to dest_dir
- extracted_files = os.listdir(path_to_extracted_files)
- for file_name in extracted_files:
- source_file = os.path.join(path_to_extracted_files, file_name)
- dest_file = os.path.join(dest_dir, file_name)
- shutil.move(source_file, dest_file)
-
- # rm -rf temp_extract_dir
- shutil.rmtree(temp_extract_dir, ignore_errors=True)
-
- # Return List of extracted files
- return list(Path(dest_dir).rglob("*"))
-
- @staticmethod
- def remove_empty_directories(directory):
- """Recursively remove empty directories."""
-
- for path in sorted(Path(directory).rglob("*"), reverse=True):
- # If broken symlink
- if path.is_symlink() and not path.exists():
- path.unlink()
- # if empty directory
- elif path.is_dir() and len(os.listdir(path)) == 0:
- path.rmdir()
-
-
-class ArduinoCoreInstaller:
- """Simple Arduino core installer."""
- @staticmethod
- def install_teensy_core_windows(install_prefix, install_dir, cache_dir):
- """Download and install Teensyduino artifacts for Windows."""
- teensy_artifacts = _ARDUINO_CORE_ARTIFACTS["teensy"][platform.system()]
-
- arduino_artifact = teensy_artifacts["arduino-ide"]
- arduino_zipfile = FileOperations.download_to_cache(
- url=arduino_artifact["url"],
- expected_sha256sum=arduino_artifact["sha256"],
- cache_directory=cache_dir)
-
- teensyduino_artifact = teensy_artifacts["teensyduino"]
- teensyduino_installer = FileOperations.download_to_cache(
- url=teensyduino_artifact["url"],
- expected_sha256sum=teensyduino_artifact["sha256"],
- cache_directory=cache_dir)
-
- FileOperations.extract_archive(arduino_zipfile, install_dir, cache_dir)
-
- # "teensy" here should match args.core_name
- teensy_core_dir = os.path.join(install_prefix, "teensy")
-
- # Change working directory for installation
- original_working_dir = os.getcwd()
- os.chdir(install_prefix)
-
- _LOG.info("Installing Teensyduino to: %s", teensy_core_dir)
-
- install_command = "{} \"--dir={}\"".format(teensyduino_installer,
- "teensy")
- _LOG.info(" Running: %s", install_command)
- _LOG.info(" Please click yes on the Windows 'User Account Control' "
- "dialog.")
- _LOG.info(" You should see: 'Verified publisher: PRJC.COM LLC'")
-
- os.system(install_command)
- if not os.path.exists(
- os.path.join(teensy_core_dir, "hardware", "teensy")):
- _LOG.error(
- "Error: Installation Failed.\n"
- "Please try again and ensure Teensyduino is installed in "
- "the folder:\n"
- "%s", teensy_core_dir)
- sys.exit(1)
- else:
- _LOG.info(" Install complete!")
-
- FileOperations.remove_empty_directories(install_dir)
- os.chdir(original_working_dir)
-
- @staticmethod
- def install_teensy_core_mac(unused_install_prefix, install_dir, cache_dir):
- """Download and install Teensyduino artifacts for Mac."""
- teensy_artifacts = _ARDUINO_CORE_ARTIFACTS["teensy"][platform.system()]
-
- teensyduino_artifact = teensy_artifacts["teensyduino"]
- teensyduino_zip = FileOperations.download_to_cache(
- url=teensyduino_artifact["url"],
- expected_sha256sum=teensyduino_artifact["sha256"],
- cache_directory=cache_dir)
-
- extracted_files = FileOperations.extract_archive(
- teensyduino_zip,
- install_dir,
- cache_dir,
- remove_single_toplevel_folder=False)
- toplevel_folder = sorted(extracted_files)[0]
- os.symlink(os.path.join(toplevel_folder, "Contents", "Java",
- "hardware"),
- os.path.join(install_dir, "hardware"),
- target_is_directory=True)
-
- @staticmethod
- def install_teensy_core_linux(install_prefix, install_dir, cache_dir):
- """Download and install Teensyduino artifacts for Windows."""
- teensy_artifacts = _ARDUINO_CORE_ARTIFACTS["teensy"][platform.system()]
-
- arduino_artifact = teensy_artifacts["arduino-ide"]
- arduino_tarfile = FileOperations.download_to_cache(
- url=arduino_artifact["url"],
- expected_sha256sum=arduino_artifact["sha256"],
- cache_directory=cache_dir)
-
- teensyduino_artifact = teensy_artifacts["teensyduino"]
- teensyduino_installer = FileOperations.download_to_cache(
- url=teensyduino_artifact["url"],
- expected_sha256sum=teensyduino_artifact["sha256"],
- cache_directory=cache_dir)
-
- extracted_files = FileOperations.extract_archive(
- arduino_tarfile, install_dir, cache_dir)
- os.chmod(teensyduino_installer,
- os.stat(teensyduino_installer).st_mode | stat.S_IEXEC)
-
- original_working_dir = os.getcwd()
- os.chdir(install_prefix)
- # "teensy" here should match args.core_name
- os.system("{} --dir={}".format(teensyduino_installer, "teensy"))
-
- # Remove original arduino IDE files
- for efile in extracted_files:
- if efile.is_file():
- efile.unlink()
-
- FileOperations.remove_empty_directories(install_dir)
- os.chdir(original_working_dir)
-
- @staticmethod
- def install_arduino_samd_core(install_prefix: str, install_dir: str,
- cache_dir: str):
- # TODO(tonymd): Fetch core/tools as specified by:
- # http://downloads.arduino.cc/packages/package_index.json
- pass
-
- @staticmethod
- def install_adafruit_samd_core(install_prefix: str, install_dir: str,
- cache_dir: str):
- artifacts = _ARDUINO_CORE_ARTIFACTS["adafruit-samd"]["all"]["core"]
- core_tarfile = FileOperations.download_to_cache(
- url=artifacts["url"],
- expected_sha256sum=artifacts["sha256"],
- cache_directory=cache_dir)
-
- package_path = os.path.join(
- install_dir, "hardware", "samd",
- os.path.basename(core_tarfile).replace(".tar.gz", ""))
- os.makedirs(package_path, exist_ok=True)
- FileOperations.extract_archive(core_tarfile, package_path, cache_dir)
-
- original_working_dir = os.getcwd()
- os.chdir(install_prefix)
- # TODO(tonymd): Fetch platform specific tools as specified by:
- # https://adafruit.github.io/arduino-board-index/package_adafruit_index.json
- # Specifically:
- # https://github.com/ARM-software/CMSIS_5/archive/5.4.0.tar.gz
- os.chdir(original_working_dir)
- return True
-
- @staticmethod
- def install_stm32duino_core(install_prefix, install_dir, cache_dir):
- artifacts = _ARDUINO_CORE_ARTIFACTS["stm32duino"]["all"]["core"]
- core_tarfile = FileOperations.download_to_cache(
- url=artifacts["url"],
- expected_sha256sum=artifacts["sha256"],
- cache_directory=cache_dir)
-
- package_path = os.path.join(
- install_dir, "hardware", "stm32",
- os.path.basename(core_tarfile).replace(".tar.gz", ""))
- os.makedirs(package_path, exist_ok=True)
- FileOperations.extract_archive(core_tarfile, package_path, cache_dir)
- original_working_dir = os.getcwd()
- os.chdir(install_prefix)
- # TODO(tonymd): Fetch platform specific tools as specified by:
- # https://github.com/stm32duino/BoardManagerFiles/raw/master/STM32/package_stm_index.json
- os.chdir(original_working_dir)
- return True
-
-
-class ArduinoBuilder:
- """Used to interpret arduino boards.txt and platform.txt files."""
- # pylint: disable=too-many-instance-attributes,too-many-public-methods
-
- board_menu_regex = re.compile(
- r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE)
-
- board_name_regex = re.compile(
- r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE)
-
- variable_regex = re.compile(r"^(?P<name>[^\s#=]+)=(?P<value>.*)$",
- re.MULTILINE)
-
- menu_option_regex = re.compile(
- r"^menu\." # starts with "menu"
- r"(?P<menu_option_name>[^.]+)\." # first token after .
- r"(?P<menu_option_value>[^.]+)$") # second (final) token after .
-
- tool_name_regex = re.compile(
- r"^tools\." # starts with "tools"
- r"(?P<tool_name>[^.]+)\.") # first token after .
-
- interpolated_variable_regex = re.compile(r"{[^}]+}", re.MULTILINE)
-
- objcopy_step_name_regex = re.compile(r"^recipe.objcopy.([^.]+).pattern$")
-
- def __init__(self,
- arduino_path,
- package_name,
- build_path=None,
- project_path=None,
- project_source_path=None,
- build_project_name=None,
- compiler_path_override=False):
- self.arduino_path = arduino_path
- self.arduino_package_name = package_name
- self.selected_board = None
- self.build_path = build_path
- self.project_path = project_path
- self.project_source_path = project_source_path
- self.build_project_name = build_project_name
- self.compiler_path_override = compiler_path_override
- self.variant_includes = ""
- self.build_variant_path = False
-
- self.compiler_path_override_binaries = []
- if self.compiler_path_override:
- self.compiler_path_override_binaries = FileOperations.find_files(
- self.compiler_path_override, "*")
-
- # Container dicts for boards.txt and platform.txt file data.
- self.board = OrderedDict()
- self.platform = OrderedDict()
- self.menu_options = OrderedDict({
- "global_options": {},
- "default_board_values": {},
- "selected": {}
- })
- self.tools_variables = {}
-
- # Set and check for valid hardware folder.
- self.hardware_path = os.path.join(self.arduino_path, "hardware")
-
- if not os.path.exists(self.hardware_path):
- _LOG.error("Error: Arduino package path '%s' does not exist.",
- self.arduino_path)
- raise FileNotFoundError
-
- # Set and check for valid package name
- self.package_path = os.path.join(self.arduino_path, "hardware",
- package_name)
- # {build.arch} is the first folder name of the package (upcased)
- self.build_arch = os.path.split(package_name)[0].upper()
-
- if not os.path.exists(self.package_path):
- _LOG.error("Error: Arduino package name '%s' does not exist.",
- package_name)
- _LOG.error("Did you mean:\n")
- # TODO(tonymd): On Windows concatenating "/" may not work
- possible_alternatives = [
- d.replace(self.hardware_path + os.sep, "", 1)
- for d in glob.glob(self.hardware_path + "/*/*")
- ]
- _LOG.error("\n".join(possible_alternatives))
- sys.exit(1)
-
- # Grab all folder names in the cores directory. These are typically
- # sub-core source files.
- self.sub_core_folders = os.listdir(
- os.path.join(self.package_path, "cores"))
-
- self._find_tools_variables()
-
- self.boards_txt = os.path.join(self.package_path, "boards.txt")
- self.platform_txt = os.path.join(self.package_path, "platform.txt")
-
- def select_board(self, board_name, menu_option_overrides=False):
- self.selected_board = board_name
-
- # Load default menu options for a selected board.
- if not self.selected_board in self.board.keys():
- _LOG.error("Error board: '%s' not supported.", self.selected_board)
- # TODO(tonymd): Print supported boards here
- sys.exit(1)
-
- # Override default menu options if any are specified.
- if menu_option_overrides:
- for moption in menu_option_overrides:
- if not self.set_menu_option(moption):
- # TODO(tonymd): Print supported menu options here
- sys.exit(1)
-
- self._copy_default_menu_options_to_build_variables()
- self._apply_recipe_overrides()
- self._substitute_variables()
-
- def _apply_recipe_overrides(self):
- # Override link recipes with per-core exceptions
- # Teensyduino cores
- if self.build_arch == 'TEENSY':
- # Change {build.path}/{archive_file}
- # To {archive_file_path} (which should contain the core.a file)
- new_link_line = self.platform["recipe.c.combine.pattern"].replace(
- "{object_files} \"{build.path}/{archive_file}\"",
- "{object_files} {archive_file_path}", 1)
- # Add the teensy provided toolchain lib folder for link access to
- # libarm_cortexM*_math.a
- new_link_line = new_link_line.replace(
- "\"-L{build.path}\"",
- "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib",
- 1)
- self.platform["recipe.c.combine.pattern"] = new_link_line
-
- # Adafruit-samd core
- # TODO(tonymd): This build_arch may clash with Arduino-SAMD core
- elif self.build_arch == 'SAMD':
- new_link_line = self.platform["recipe.c.combine.pattern"].replace(
- "\"{build.path}/{archive_file}\" -Wl,--end-group",
- "{archive_file_path} -Wl,--end-group", 1)
- self.platform["recipe.c.combine.pattern"] = new_link_line
-
- # STM32L4 Core:
- # https://github.com/GrumpyOldPizza/arduino-STM32L4
- elif self.build_arch == 'STM32L4':
- # TODO(tonymd): {build.path}/{archive_file} for the link step always
- # seems to be core.a (except STM32 core)
- line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\""
- new_link_line = self.platform["recipe.c.combine.pattern"].replace(
- line_to_delete, "-Wl,--start-group {archive_file_path}", 1)
- self.platform["recipe.c.combine.pattern"] = new_link_line
-
- # stm32duino core
- elif self.build_arch == 'STM32':
- pass
-
- def _copy_default_menu_options_to_build_variables(self):
- # Clear existing options
- self.menu_options["selected"] = {}
- # Set default menu options for selected board
- for menu_key, menu_dict in self.menu_options["default_board_values"][
- self.selected_board].items():
- for name, var in self.board[self.selected_board].items():
- starting_key = "{}.{}.".format(menu_key, menu_dict["name"])
- if name.startswith(starting_key):
- new_var_name = name.replace(starting_key, "", 1)
- self.menu_options["selected"][new_var_name] = var
-
- def set_menu_option(self, moption):
- if moption not in self.board[self.selected_board]:
- _LOG.error("Error: '%s' is not a valid menu option.", moption)
- return False
-
- # Override default menu option with new value.
- menu_match_result = self.menu_option_regex.match(moption)
- if menu_match_result:
- menu_match = menu_match_result.groupdict()
- menu_value = menu_match["menu_option_value"]
- menu_key = "menu.{}".format(menu_match["menu_option_name"])
- self.menu_options["default_board_values"][
- self.selected_board][menu_key]["name"] = menu_value
-
- # Update build variables
- self._copy_default_menu_options_to_build_variables()
- return True
-
- def _set_global_arduino_variables(self):
- """Set some global variables defined by the Arduino-IDE.
-
- See Docs:
- https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties
- """
-
- for current_board_name in self.board.keys():
- if self.build_path:
- self.board[current_board_name]["build.path"] = self.build_path
- if self.build_project_name:
- self.board[current_board_name][
- "build.project_name"] = self.build_project_name
- # {archive_file} is the final *.elf
- archive_file = "{}.elf".format(self.build_project_name)
- self.board[current_board_name]["archive_file"] = archive_file
- # {archive_file_path} is the final core.a archive
- if self.build_path:
- self.board[current_board_name][
- "archive_file_path"] = os.path.join(
- self.build_path, "core.a")
- if self.project_source_path:
- self.board[current_board_name][
- "build.source.path"] = self.project_source_path
-
- self.board[current_board_name]["extra.time.local"] = str(
- int(time.time()))
- self.board[current_board_name]["runtime.ide.version"] = "10812"
- self.board[current_board_name][
- "runtime.hardware.path"] = self.hardware_path
-
- # Copy {runtime.tools.TOOL_NAME.path} vars
- self._set_tools_variables(self.board[current_board_name])
-
- self.board[current_board_name][
- "runtime.platform.path"] = self.package_path
- if self.platform["name"] == "Teensyduino":
- # Teensyduino is installed into the arduino IDE folder
- # rather than ~/.arduino15/packages/
- self.board[current_board_name][
- "runtime.hardware.path"] = os.path.join(
- self.hardware_path, "teensy")
-
- self.board[current_board_name]["build.system.path"] = os.path.join(
- self.package_path, "system")
-
- # Set the {build.core.path} variable that pointing to a sub-core
- # folder. For Teensys this is:
- # 'teensy/hardware/teensy/avr/cores/teensy{3,4}'. For other cores
- # it's typically just the 'arduino' folder. For example:
- # 'arduino-samd/hardware/samd/1.8.8/cores/arduino'
- core_path = os.path.join(
- self.package_path, "cores",
- self.board[current_board_name].get("build.core",
- self.sub_core_folders[0]))
- self.board[current_board_name]["build.core.path"] = core_path
-
- self.board[current_board_name]["build.arch"] = self.build_arch
-
- for name, var in self.board[current_board_name].items():
- self.board[current_board_name][name] = var.replace(
- "{build.core.path}", core_path)
-
- def load_board_definitions(self):
- """Loads Arduino boards.txt and platform.txt files into dictionaries.
-
- Populates the following dictionaries:
- self.menu_options
- self.boards
- self.platform
- """
- # Load platform.txt
- with open(self.platform_txt, "r") as pfile:
- platform_file = pfile.read()
- platform_var_matches = self.variable_regex.finditer(platform_file)
- for p_match in [m.groupdict() for m in platform_var_matches]:
- self.platform[p_match["name"]] = p_match["value"]
-
- # Load boards.txt
- with open(self.boards_txt, "r") as bfile:
- board_file = bfile.read()
- # Get all top-level menu options, e.g. menu.usb=USB Type
- board_menu_matches = self.board_menu_regex.finditer(board_file)
- for menuitem in [m.groupdict() for m in board_menu_matches]:
- self.menu_options["global_options"][menuitem["name"]] = {
- "description": menuitem["description"]
- }
-
- # Get all board names, e.g. teensy40.name=Teensy 4.0
- board_name_matches = self.board_name_regex.finditer(board_file)
- for b_match in [m.groupdict() for m in board_name_matches]:
- self.board[b_match["name"]] = OrderedDict()
- self.menu_options["default_board_values"][
- b_match["name"]] = OrderedDict()
-
- # Get all board variables, e.g. teensy40.*
- for current_board_name in self.board.keys():
- board_line_matches = re.finditer(
- fr"^\s*{current_board_name}\."
- fr"(?P<key>[^#=]+)=(?P<value>.*)$", board_file,
- re.MULTILINE)
- for b_match in [m.groupdict() for m in board_line_matches]:
- # Check if this line is a menu option
- # (e.g. 'menu.usb.serial') and save as default if it's the
- # first one seen.
- ArduinoBuilder.save_default_menu_option(
- current_board_name, b_match["key"], b_match["value"],
- self.menu_options)
- self.board[current_board_name][
- b_match["key"]] = b_match["value"].strip()
-
- self._set_global_arduino_variables()
-
- @staticmethod
- def save_default_menu_option(current_board_name, key, value, menu_options):
- """Save a given menu option as the default.
-
- Saves the key and value into menu_options["default_board_values"]
- if it doesn't already exist. Assumes menu options are added in the order
- specified in boards.txt. The first value for a menu key is the default.
- """
- # Check if key is a menu option
- # e.g. menu.usb.serial
- # menu.usb.serial.build.usbtype
- menu_match_result = re.match(
- r'^menu\.' # starts with "menu"
- r'(?P<menu_option_name>[^.]+)\.' # first token after .
- r'(?P<menu_option_value>[^.]+)' # second token after .
- r'(\.(?P<rest>.+))?', # optionally any trailing tokens after a .
- key)
- if menu_match_result:
- menu_match = menu_match_result.groupdict()
- current_menu_key = "menu.{}".format(menu_match["menu_option_name"])
- # If this is the first menu option seen for current_board_name, save
- # as the default.
- if current_menu_key not in menu_options["default_board_values"][
- current_board_name]:
- menu_options["default_board_values"][current_board_name][
- current_menu_key] = {
- "name": menu_match["menu_option_value"],
- "description": value
- }
-
- def _replace_variables(self, line, variable_lookup_source):
- """Replace {variables} from loaded boards.txt or platform.txt.
-
- Replace interpolated variables surrounded by curly braces in line with
- definitions from variable_lookup_source.
- """
- new_line = line
- for current_var_match in self.interpolated_variable_regex.findall(
- line):
- # {build.flags.c} --> build.flags.c
- current_var = current_var_match.strip("{}")
-
- # check for matches in board definition
- if current_var in variable_lookup_source:
- replacement = variable_lookup_source.get(current_var, "")
- new_line = new_line.replace(current_var_match, replacement)
- return new_line
-
- def _find_tools_variables(self):
- # Gather tool directories in order of increasing precedence
- runtime_tool_paths = []
-
- # Check for tools installed in ~/.arduino15/packages/arduino/tools/
- # TODO(tonymd): Is this Mac & Linux specific?
- runtime_tool_paths += glob.glob(
- os.path.join(
- os.path.realpath(os.path.expanduser(os.path.expandvars("~"))),
- ".arduino15", "packages", "arduino", "tools", "*"))
-
- # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES>
- runtime_tool_paths += glob.glob(
- os.path.join(self.arduino_path, "tools",
- arduino_runtime_os_string(), "*"))
- # <ARDUINO_PATH>/tools/<TOOL_NAMES>
- # This will grab linux/windows/macosx/share as <TOOL_NAMES>.
- runtime_tool_paths += glob.glob(
- os.path.join(self.arduino_path, "tools", "*"))
-
- # Process package tools after arduino tools.
- # They should overwrite vars & take precedence.
-
- # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES>
- runtime_tool_paths += glob.glob(
- os.path.join(self.package_path, "tools",
- arduino_runtime_os_string(), "*"))
- # <PACKAGE_PATH>/tools/<TOOL_NAMES>
- # This will grab linux/windows/macosx/share as <TOOL_NAMES>.
- runtime_tool_paths += glob.glob(
- os.path.join(self.package_path, "tools", "*"))
-
- for path in runtime_tool_paths:
- # Make sure TOOL_NAME is not an OS string
- if not (path.endswith("linux") or path.endswith("windows")
- or path.endswith("macosx") or path.endswith("share")):
- # TODO(tonymd): Check if a file & do nothing?
-
- # Check if it's a directory with subdir(s) as a version string
- # create all 'runtime.tools.{tool_folder}-{version.path}'
- # (for each version)
- # create 'runtime.tools.{tool_folder}.path'
- # (with latest version)
- if os.path.isdir(path):
- # Grab the tool name (folder) by itself.
- tool_folder = os.path.basename(path)
- # Sort so that [-1] is the latest version.
- version_paths = sorted(glob.glob(os.path.join(path, "*")))
- # Check if all sub folders start with a version string.
- if len(version_paths) == sum(
- bool(re.match(r"^[0-9.]+", os.path.basename(vp)))
- for vp in version_paths):
- for version_path in version_paths:
- version_string = os.path.basename(version_path)
- var_name = "runtime.tools.{}-{}.path".format(
- tool_folder, version_string)
- self.tools_variables[var_name] = os.path.join(
- path, version_string)
- var_name = "runtime.tools.{}.path".format(tool_folder)
- self.tools_variables[var_name] = os.path.join(
- path, os.path.basename(version_paths[-1]))
- # Else set toolpath to path.
- else:
- var_name = "runtime.tools.{}.path".format(tool_folder)
- self.tools_variables[var_name] = path
-
- _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables))
-
- # Copy self.tools_variables into destination
- def _set_tools_variables(self, destination):
- for key, value in self.tools_variables.items():
- destination[key] = value
-
- def _substitute_variables(self):
- """Perform variable substitution in board and platform metadata."""
-
- # menu -> board
- # Copy selected menu variables into board definiton
- for name, value in self.menu_options["selected"].items():
- self.board[self.selected_board][name] = value
-
- # board -> board
- # Replace any {vars} in the selected board with values defined within
- # (and from copied in menu options).
- for var, value in self.board[self.selected_board].items():
- self.board[self.selected_board][var] = self._replace_variables(
- value, self.board[self.selected_board])
-
- # Check for build.variant variable
- # This will be set in selected board after menu options substitution
- build_variant = self.board[self.selected_board].get(
- "build.variant", None)
- if build_variant:
- # Set build.variant.path
- bvp = os.path.join(self.package_path, "variants", build_variant)
- self.build_variant_path = bvp
- self.board[self.selected_board]["build.variant.path"] = bvp
- # Add the variant folder as an include directory
- # (used in stm32l4 core)
- self.variant_includes = "-I{}".format(bvp)
-
- _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform))
-
- # board -> platform
- # Replace {vars} in platform from the selected board definition
- for var, value in self.platform.items():
- self.platform[var] = self._replace_variables(
- value, self.board[self.selected_board])
-
- # platform -> platform
- # Replace any remaining {vars} in platform from platform
- for var, value in self.platform.items():
- self.platform[var] = self._replace_variables(value, self.platform)
-
- # Repeat platform -> platform for any lingering variables
- # Example: {build.opt.name} in STM32 core
- for var, value in self.platform.items():
- self.platform[var] = self._replace_variables(value, self.platform)
-
- _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options))
- _LOG.debug("SELECTED_BOARD: %s",
- _pretty_format(self.board[self.selected_board]))
- _LOG.debug("PLATFORM: %s", _pretty_format(self.platform))
-
- def selected_board_spec(self):
- return self.board[self.selected_board]
-
- def get_menu_options(self):
- all_options = []
- max_string_length = [0, 0]
-
- for key_name, description in self.board[self.selected_board].items():
- menu_match_result = self.menu_option_regex.match(key_name)
- if menu_match_result:
- menu_match = menu_match_result.groupdict()
- name = "menu.{}.{}".format(menu_match["menu_option_name"],
- menu_match["menu_option_value"])
- if len(name) > max_string_length[0]:
- max_string_length[0] = len(name)
- if len(description) > max_string_length[1]:
- max_string_length[1] = len(description)
- all_options.append((name, description))
-
- return all_options, max_string_length
-
- def get_default_menu_options(self):
- default_options = []
- max_string_length = [0, 0]
-
- for key_name, value in self.menu_options["default_board_values"][
- self.selected_board].items():
- full_key = key_name + "." + value["name"]
- if len(full_key) > max_string_length[0]:
- max_string_length[0] = len(full_key)
- if len(value["description"]) > max_string_length[1]:
- max_string_length[1] = len(value["description"])
- default_options.append((full_key, value["description"]))
-
- return default_options, max_string_length
-
- @staticmethod
- def split_binary_from_arguments(compile_line):
- compile_binary = None
- rest_of_line = compile_line
-
- compile_binary_match = re.search(r'^("[^"]+") ', compile_line)
- if compile_binary_match:
- compile_binary = compile_binary_match[1]
- rest_of_line = compile_line.replace(compile_binary_match[0], "", 1)
-
- return compile_binary, rest_of_line
-
- def _strip_includes_source_file_object_file_vars(self, compile_line):
- line = compile_line
- if self.variant_includes:
- line = compile_line.replace(
- "{includes} \"{source_file}\" -o \"{object_file}\"",
- self.variant_includes, 1)
- else:
- line = compile_line.replace(
- "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1)
- return line
-
- def _get_tool_name(self, line):
- tool_match_result = self.tool_name_regex.match(line)
- if tool_match_result:
- return tool_match_result[1]
- return False
-
- def get_upload_tool_names(self):
- return [
- self._get_tool_name(t) for t in self.platform.keys()
- if self.tool_name_regex.match(t) and 'upload.pattern' in t
- ]
-
- # TODO(tonymd): Use these getters in _replace_variables() or
- # _substitute_variables()
-
- def _get_platform_variable(self, variable):
- # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name,
- # compare with platform.system() and return that instead.
- return self.platform.get(variable, False)
-
- def _get_platform_variable_with_substitutions(self, variable, namespace):
- line = self.platform.get(variable, False)
- # Get all unique variables used in this line in line.
- unique_vars = sorted(
- set(self.interpolated_variable_regex.findall(line)))
- # Search for each unique_vars in namespace and global.
- for var in unique_vars:
- v_raw_name = var.strip("{}")
-
- # Check for namespace.variable
- # eg: 'tools.stm32CubeProg.cmd'
- possible_var_name = "{}.{}".format(namespace, v_raw_name)
- result = self._get_platform_variable(possible_var_name)
- # Check for os overriden variable
- # eg:
- # ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'),
- # ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'),
- possible_var_name = "{}.{}.{}".format(namespace, v_raw_name,
- arduino_runtime_os_string())
- os_override_result = self._get_platform_variable(possible_var_name)
-
- if os_override_result:
- line = line.replace(var, os_override_result)
- elif result:
- line = line.replace(var, result)
- # Check for variable at top level?
- # elif self._get_platform_variable(v_raw_name):
- # line = line.replace(self._get_platform_variable(v_raw_name),
- # result)
- return line
-
- def get_upload_line(self, tool_name, serial_port=False):
- # TODO(tonymd): Error if tool_name does not exist
- tool_namespace = "tools.{}".format(tool_name)
- pattern = "tools.{}.upload.pattern".format(tool_name)
-
- if not self._get_platform_variable(pattern):
- _LOG.error("Error: upload tool '%s' does not exist.", tool_name)
- tools = self.get_upload_tool_names()
- _LOG.error("Valid tools: %s", ", ".join(tools))
- return sys.exit(1)
-
- line = self._get_platform_variable_with_substitutions(
- pattern, tool_namespace)
-
- # TODO(tonymd): Teensy specific tool overrides.
- if tool_name == "teensyloader":
- # Remove un-necessary lines
- # {serial.port.label} and {serial.port.protocol} are returned by
- # the teensy_ports binary.
- line = line.replace("\"-portlabel={serial.port.label}\"", "", 1)
- line = line.replace("\"-portprotocol={serial.port.protocol}\"", "",
- 1)
-
- if serial_port:
- line = line.replace("{serial.port}", serial_port, 1)
-
- return line
-
- def _get_binary_path(self, variable_pattern):
- compile_line = self.replace_compile_binary_with_override_path(
- self._get_platform_variable(variable_pattern))
- compile_binary, _ = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- return compile_binary
-
- def get_cc_binary(self):
- return self._get_binary_path("recipe.c.o.pattern")
-
- def get_cxx_binary(self):
- return self._get_binary_path("recipe.cpp.o.pattern")
-
- def get_objcopy_binary(self):
- objcopy_step_name = self.get_objcopy_step_names()[0]
- objcopy_binary = self._get_binary_path(objcopy_step_name)
- return objcopy_binary
-
- def get_ar_binary(self):
- return self._get_binary_path("recipe.ar.pattern")
-
- def get_size_binary(self):
- return self._get_binary_path("recipe.size.pattern")
-
- def replace_command_args_with_compiler_override_path(self, compile_line):
- if not self.compiler_path_override:
- return compile_line
- replacement_line = compile_line
- replacement_line_args = compile_line.split()
- for arg in replacement_line_args:
- compile_binary_basename = os.path.basename(arg.strip("\""))
- if compile_binary_basename in self.compiler_path_override_binaries:
- new_compiler = os.path.join(self.compiler_path_override,
- compile_binary_basename)
- replacement_line = replacement_line.replace(
- arg, new_compiler, 1)
- return replacement_line
-
- def replace_compile_binary_with_override_path(self, compile_line):
- replacement_compile_line = compile_line
-
- # Change the compiler path if there's an override path set
- if self.compiler_path_override:
- compile_binary, line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- compile_binary_basename = os.path.basename(
- compile_binary.strip("\""))
- new_compiler = os.path.join(self.compiler_path_override,
- compile_binary_basename)
- if platform.system() == "Windows" and not re.match(
- r".*\.exe$", new_compiler, flags=re.IGNORECASE):
- new_compiler += ".exe"
-
- if os.path.isfile(new_compiler):
- replacement_compile_line = "\"{}\" {}".format(
- new_compiler, line)
-
- return replacement_compile_line
-
- def get_c_compile_line(self):
- _LOG.debug("ARDUINO_C_COMPILE: %s",
- _pretty_format(self.platform["recipe.c.o.pattern"]))
-
- compile_line = self.platform["recipe.c.o.pattern"]
- compile_line = self._strip_includes_source_file_object_file_vars(
- compile_line)
- compile_line += " -I{}".format(
- self.board[self.selected_board]["build.core.path"])
-
- compile_line = self.replace_compile_binary_with_override_path(
- compile_line)
- return compile_line
-
- def get_s_compile_line(self):
- _LOG.debug("ARDUINO_S_COMPILE %s",
- _pretty_format(self.platform["recipe.S.o.pattern"]))
-
- compile_line = self.platform["recipe.S.o.pattern"]
- compile_line = self._strip_includes_source_file_object_file_vars(
- compile_line)
- compile_line += " -I{}".format(
- self.board[self.selected_board]["build.core.path"])
-
- compile_line = self.replace_compile_binary_with_override_path(
- compile_line)
- return compile_line
-
- def get_ar_compile_line(self):
- _LOG.debug("ARDUINO_AR_COMPILE: %s",
- _pretty_format(self.platform["recipe.ar.pattern"]))
-
- compile_line = self.platform["recipe.ar.pattern"].replace(
- "\"{object_file}\"", "", 1)
-
- compile_line = self.replace_compile_binary_with_override_path(
- compile_line)
- return compile_line
-
- def get_cpp_compile_line(self):
- _LOG.debug("ARDUINO_CPP_COMPILE: %s",
- _pretty_format(self.platform["recipe.cpp.o.pattern"]))
-
- compile_line = self.platform["recipe.cpp.o.pattern"]
- compile_line = self._strip_includes_source_file_object_file_vars(
- compile_line)
- compile_line += " -I{}".format(
- self.board[self.selected_board]["build.core.path"])
-
- compile_line = self.replace_compile_binary_with_override_path(
- compile_line)
- return compile_line
-
- def get_link_line(self):
- _LOG.debug("ARDUINO_LINK: %s",
- _pretty_format(self.platform["recipe.c.combine.pattern"]))
-
- compile_line = self.platform["recipe.c.combine.pattern"]
-
- compile_line = self.replace_compile_binary_with_override_path(
- compile_line)
- return compile_line
-
- def get_objcopy_step_names(self):
- names = [
- name for name, line in self.platform.items()
- if self.objcopy_step_name_regex.match(name)
- ]
- return names
-
- def get_objcopy_steps(self) -> List[str]:
- lines = [
- line for name, line in self.platform.items()
- if self.objcopy_step_name_regex.match(name)
- ]
- lines = [
- self.replace_compile_binary_with_override_path(line)
- for line in lines
- ]
- return lines
-
- # TODO(tonymd): These recipes are probably run in sorted order
- def get_objcopy(self, suffix):
- # Expected vars:
- # teensy:
- # recipe.objcopy.eep.pattern
- # recipe.objcopy.hex.pattern
-
- pattern = "recipe.objcopy.{}.pattern".format(suffix)
- objcopy_step_names = self.get_objcopy_step_names()
-
- objcopy_suffixes = [
- m[1] for m in [
- self.objcopy_step_name_regex.match(line)
- for line in objcopy_step_names
- ] if m
- ]
- if pattern not in objcopy_step_names:
- _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix)
- _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes))
- return sys.exit(1)
-
- line = self._get_platform_variable(pattern)
-
- _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line)
-
- line = self.replace_compile_binary_with_override_path(line)
-
- return line
-
- def get_objcopy_flags(self, suffix):
- # TODO(tonymd): Possibly teensy specific variables.
- flags = ""
- if suffix == "hex":
- flags = self.platform.get("compiler.elf2hex.flags", "")
- elif suffix == "bin":
- flags = self.platform.get("compiler.elf2bin.flags", "")
- elif suffix == "eep":
- flags = self.platform.get("compiler.objcopy.eep.flags", "")
- return flags
-
- # TODO(tonymd): There are more recipe hooks besides postbuild.
- # They are run in sorted order.
- # TODO(tonymd): Rename this to get_hooks(hook_name, step).
- # TODO(tonymd): Add a list-hooks and or run-hooks command
- def get_postbuild_line(self, step_number):
- line = self.platform["recipe.hooks.postbuild.{}.pattern".format(
- step_number)]
- line = self.replace_command_args_with_compiler_override_path(line)
- return line
-
- def get_prebuild_steps(self) -> List[str]:
- # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern
- # stm32 core uses recipe.hooks.prebuild.1.pattern
- # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows
- # (should override non-windows key)
- lines = [
- line for name, line in self.platform.items() if re.match(
- r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name)
- ]
- # TODO(tonymd): Write a function to fetch/replace OS specific patterns
- # (ending in an OS string)
- lines = [
- self.replace_compile_binary_with_override_path(line)
- for line in lines
- ]
- return lines
-
- def get_postbuild_steps(self) -> List[str]:
- lines = [
- line for name, line in self.platform.items()
- if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name)
- ]
-
- lines = [
- self.replace_command_args_with_compiler_override_path(line)
- for line in lines
- ]
- return lines
-
- def get_s_flags(self):
- compile_line = self.get_s_compile_line()
- _, compile_line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- compile_line = compile_line.replace("-c", "", 1)
- return compile_line.strip()
-
- def get_c_flags(self):
- compile_line = self.get_c_compile_line()
- _, compile_line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- compile_line = compile_line.replace("-c", "", 1)
- return compile_line.strip()
-
- def get_cpp_flags(self):
- compile_line = self.get_cpp_compile_line()
- _, compile_line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- compile_line = compile_line.replace("-c", "", 1)
- return compile_line.strip()
-
- def get_ar_flags(self):
- compile_line = self.get_ar_compile_line()
- _, compile_line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- return compile_line.strip()
-
- def get_ld_flags(self):
- compile_line = self.get_link_line()
- _, compile_line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
-
- # TODO(tonymd): This is teensy specific
- line_to_delete = "-o \"{build.path}/{build.project_name}.elf\" " \
- "{object_files} \"-L{build.path}\""
- if self.build_path:
- line_to_delete = line_to_delete.replace("{build.path}",
- self.build_path)
- if self.build_project_name:
- line_to_delete = line_to_delete.replace("{build.project_name}",
- self.build_project_name)
-
- compile_line = compile_line.replace(line_to_delete, "", 1)
- libs = re.findall(r'(-l[^ ]+ ?)', compile_line)
- for lib in libs:
- compile_line = compile_line.replace(lib, "", 1)
- libs = [lib.strip() for lib in libs]
-
- return compile_line.strip()
-
- def get_ld_libs(self):
- compile_line = self.get_link_line()
- _, compile_line = ArduinoBuilder.split_binary_from_arguments(
- compile_line)
- # TODO(tonymd): This replacement is teensy specific
- compile_line = compile_line.replace(
- "-o \"{build.path}/{build.project_name}.elf\" "
- "{object_files} \"-L{build.path}\"", "", 1)
- libs = re.findall(r'(-l[^ ]+ ?)', compile_line)
- for lib in libs:
- compile_line = compile_line.replace(lib, "", 1)
- libs = [lib.strip() for lib in libs]
-
- return " ".join(libs)
-
- def library_folders(self):
- # Arduino library format documentation:
- # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files
- # - If src folder exists,
- # use that as the root include directory -Ilibraries/libname/src
- # - Else lib folder as root include -Ilibraries/libname
- # (exclude source files in the examples folder in this case)
-
- library_path = os.path.join(self.project_path, "libraries")
-
- library_folders = FileOperations.find_files(library_path, ["*"],
- directories_only=True)
- library_source_root_folders = []
- for lib in library_folders:
- lib_dir = os.path.join(library_path, lib)
- src_dir = os.path.join(lib_dir, "src")
- if os.path.exists(src_dir) and os.path.isdir(src_dir):
- library_source_root_folders.append(src_dir)
- else:
- library_source_root_folders.append(lib_dir)
-
- return library_source_root_folders
-
- def library_includes(self):
- include_args = []
- library_folders = self.library_folders()
- for lib_dir in library_folders:
- include_args.append("-I{}".format(os.path.relpath(lib_dir)))
- return include_args
-
- def library_files(self, pattern):
- sources = []
- library_folders = self.library_folders()
- for lib_dir in library_folders:
- for file_path in FileOperations.find_files(lib_dir, [pattern]):
- if not file_path.startswith("examples"):
- sources.append(
- os.path.relpath(os.path.join(lib_dir, file_path)))
- return sources
-
- def library_c_files(self):
- return self.library_files("**/*.c")
-
- def library_cpp_files(self):
- return self.library_files("**/*.cpp")
-
- def get_core_path(self):
- return self.board[self.selected_board]["build.core.path"]
-
- def core_files(self, pattern):
- sources = []
- for file_path in FileOperations.find_files(self.get_core_path(),
- [pattern]):
- sources.append(os.path.join(self.get_core_path(), file_path))
- return sources
-
- def core_c_files(self):
- return self.core_files("**/*.c")
-
- def core_s_files(self):
- return self.core_files("**/*.S")
-
- def core_cpp_files(self):
- return self.core_files("**/*.cpp")
-
- def get_variant_path(self):
- return self.build_variant_path
-
- def variant_files(self, pattern):
- sources = []
- if self.build_variant_path:
- for file_path in FileOperations.find_files(self.get_variant_path(),
- [pattern]):
- sources.append(os.path.join(self.get_variant_path(),
- file_path))
- return sources
-
- def variant_c_files(self):
- return self.variant_files("**/*.c")
-
- def variant_s_files(self):
- return self.variant_files("**/*.S")
-
- def variant_cpp_files(self):
- return self.variant_files("**/*.cpp")
-
- def project_files(self, pattern):
- sources = []
- for file_path in FileOperations.find_files(self.project_path,
- [pattern]):
- if not file_path.startswith(
- "examples") and not file_path.startswith("libraries"):
- sources.append(file_path)
- return sources
-
- def project_c_files(self):
- return self.project_files("**/*.c")
-
- def project_cpp_files(self):
- return self.project_files("**/*.cpp")
-
- def project_ino_files(self):
- return self.project_files("**/*.ino")
-
-
-def install_core_command(args):
- install_prefix = os.path.realpath(
- os.path.expanduser(os.path.expandvars(args.prefix)))
- install_dir = os.path.join(install_prefix, args.core_name)
- cache_dir = os.path.join(install_prefix, ".cache", args.core_name)
-
- shutil.rmtree(install_dir, ignore_errors=True)
- os.makedirs(install_dir, exist_ok=True)
- os.makedirs(cache_dir, exist_ok=True)
-
- if args.core_name == "teensy":
- if platform.system() == "Linux":
- ArduinoCoreInstaller.install_teensy_core_linux(
- install_prefix, install_dir, cache_dir)
- elif platform.system() == "Darwin":
- ArduinoCoreInstaller.install_teensy_core_mac(
- install_prefix, install_dir, cache_dir)
- elif platform.system() == "Windows":
- ArduinoCoreInstaller.install_teensy_core_windows(
- install_prefix, install_dir, cache_dir)
- elif args.core_name == "adafruit-samd":
- ArduinoCoreInstaller.install_adafruit_samd_core(
- install_prefix, install_dir, cache_dir)
- elif args.core_name == "stm32duino":
- ArduinoCoreInstaller.install_stm32duino_core(install_prefix,
- install_dir, cache_dir)
-
- sys.exit(0)
-
-
-def list_boards_command(unused_args, builder):
- # list-boards subcommand
- # (does not need a selected board or default menu options)
-
- # TODO(tonymd): Print this sorted with auto-ljust columns
- longest_name_length = 0
- for board_name, board_dict in builder.board.items():
- if len(board_name) > longest_name_length:
- longest_name_length = len(board_name)
- longest_name_length += 2
-
- print("Board Name".ljust(longest_name_length), "Description")
- for board_name, board_dict in builder.board.items():
- print(board_name.ljust(longest_name_length), board_dict['name'])
- sys.exit(0)
-
-
-def list_menu_options_command(args, builder):
- # List all menu options for the selected board.
- builder.select_board(args.board)
-
- print("All Options")
- all_options, all_column_widths = builder.get_menu_options()
- separator = "-" * (all_column_widths[0] + all_column_widths[1] + 2)
- print(separator)
-
- for name, description in all_options:
- print(name.ljust(all_column_widths[0] + 1), description)
-
- print("\nDefault Options")
- print(separator)
-
- default_options, unused_column_widths = builder.get_default_menu_options()
- for name, description in default_options:
- print(name.ljust(all_column_widths[0] + 1), description)
-
-
-def show_command_print_string_list(args, string_list: List[str]):
- join_token = " "
- if args.delimit_with_newlines:
- join_token = "\n"
- print(join_token.join(string_list))
-
-
-def show_command_print_flag_string(args, flag_string):
- if args.delimit_with_newlines:
- flag_string_with_newlines = shlex.split(flag_string)
- print("\n".join(flag_string_with_newlines))
- else:
- print(flag_string)
-
-
-def run_command_lines(args, command_lines: List[str]):
- for command_line in command_lines:
- if not args.quiet:
- print(command_line)
- # TODO(tonymd): Exit with sub command exit code.
- os.system(command_line)
-
-
-# pylint: disable=too-many-branches
-def show_command(args, builder):
- """Show sub command function.
-
- Prints compile flags and runs Arduino recipes.
- """
- builder.select_board(args.board, args.menu_options)
-
- if args.cc_binary:
- print(builder.get_cc_binary())
-
- elif args.cxx_binary:
- print(builder.get_cxx_binary())
-
- elif args.objcopy_binary:
- print(builder.get_objcopy_binary())
-
- elif args.ar_binary:
- print(builder.get_ar_binary())
-
- elif args.size_binary:
- print(builder.get_size_binary())
-
- elif args.run_c_compile:
- print(builder.get_c_compile_line())
-
- elif args.run_cpp_compile:
- print(builder.get_cpp_compile_line())
-
- elif args.run_link:
- line = builder.get_link_line()
- archive_file_path = args.run_link[0] # pylint: disable=unused-variable
- object_files = args.run_link[1:]
- line = line.replace("{object_files}", " ".join(object_files), 1)
- run_command_lines(args, [line])
-
- elif args.run_objcopy:
- run_command_lines(args, builder.get_objcopy_steps())
-
- elif args.objcopy:
- print(builder.get_objcopy(args.objcopy))
-
- elif args.objcopy_flags:
- objcopy_flags = builder.get_objcopy_flags(args.objcopy_flags)
- show_command_print_flag_string(args, objcopy_flags)
-
- elif args.c_flags:
- cflags = builder.get_c_flags()
- show_command_print_flag_string(args, cflags)
-
- elif args.s_flags:
- sflags = builder.get_s_flags()
- show_command_print_flag_string(args, sflags)
-
- elif args.cpp_flags:
- cppflags = builder.get_cpp_flags()
- show_command_print_flag_string(args, cppflags)
-
- elif args.ld_flags:
- ldflags = builder.get_ld_flags()
- show_command_print_flag_string(args, ldflags)
-
- elif args.ld_libs:
- print(builder.get_ld_libs())
-
- elif args.ar_flags:
- ar_flags = builder.get_ar_flags()
- show_command_print_flag_string(args, ar_flags)
-
- elif args.core_path:
- print(builder.get_core_path())
-
- elif args.run_prebuilds:
- run_command_lines(args, builder.get_prebuild_steps())
-
- elif args.run_postbuilds:
- run_command_lines(args, builder.get_postbuild_steps())
-
- elif args.postbuild:
- print(builder.get_postbuild_line(args.postbuild))
-
- elif args.run_upload_command:
- command = builder.get_upload_line(args.run_upload_command,
- args.serial_port)
- run_command_lines(args, [command])
-
- elif args.upload_command:
- print(builder.get_upload_line(args.upload_command, args.serial_port))
-
- elif args.upload_tools:
- tools = builder.get_upload_tool_names()
- for tool_name in tools:
- print(tool_name)
-
- elif args.library_includes:
- show_command_print_string_list(args, builder.library_includes())
-
- elif args.library_c_files:
- show_command_print_string_list(args, builder.library_c_files())
-
- elif args.library_cpp_files:
- show_command_print_string_list(args, builder.library_cpp_files())
-
- elif args.core_c_files:
- show_command_print_string_list(args, builder.core_c_files())
-
- elif args.core_s_files:
- show_command_print_string_list(args, builder.core_s_files())
-
- elif args.core_cpp_files:
- show_command_print_string_list(args, builder.core_cpp_files())
-
- elif args.variant_c_files:
- vfiles = builder.variant_c_files()
- if vfiles:
- show_command_print_string_list(args, vfiles)
-
- elif args.variant_s_files:
- vfiles = builder.variant_s_files()
- if vfiles:
- show_command_print_string_list(args, vfiles)
-
- elif args.variant_cpp_files:
- vfiles = builder.variant_cpp_files()
- if vfiles:
- show_command_print_string_list(args, vfiles)
-
-
-def main():
- """Main command line function.
-
- Parses command line args and dispatches to sub `*_command()` functions.
- """
- def log_level(arg: str) -> int:
- try:
- return getattr(logging, arg.upper())
- except AttributeError:
- raise argparse.ArgumentTypeError(
- f'{arg.upper()} is not a valid log level')
-
- parser = argparse.ArgumentParser()
- parser.add_argument("-q",
- "--quiet",
- help="hide run command output",
- action="store_true")
- parser.add_argument('-l',
- '--loglevel',
- type=log_level,
- default=logging.INFO,
- help='Set the log level '
- '(debug, info, warning, error, critical)')
-
- build_path = os.path.realpath(
- os.path.expanduser(os.path.expandvars("./build")))
- project_path = os.path.realpath(
- os.path.expanduser(os.path.expandvars("./")))
- project_source_path = os.path.join(project_path, "src")
- build_project_name = os.path.basename(project_path)
-
- serial_port = "UNKNOWN"
- # TODO(tonymd): Temp solution to passing in serial port. It should use
- # arduino core discovery tools.
- possible_serial_ports = glob.glob("/dev/ttyACM*") + glob.glob(
- "/dev/ttyUSB*")
- if possible_serial_ports:
- serial_port = possible_serial_ports[0]
-
- project_name = os.path.basename(
- os.path.realpath(os.path.expanduser(os.path.expandvars("./"))))
-
- # Global command line options
- parser.add_argument("--arduino-package-path",
- help="Path to the arduino IDE install location.")
- parser.add_argument("--arduino-package-name",
- help="Name of the Arduino board package to use.")
- parser.add_argument("--compiler-path-override",
- help="Path to arm-none-eabi-gcc bin folder. "
- "Default: Arduino core specified gcc")
-
- # Subcommands
- subparsers = parser.add_subparsers(title="subcommand",
- description="valid subcommands",
- help="sub-command help",
- dest="subcommand",
- required=True)
-
- # install-core command
- install_core_parser = subparsers.add_parser(
- "install-core", help="Download and install arduino cores")
- install_core_parser.set_defaults(func=install_core_command)
- install_core_parser.add_argument("--prefix",
- required=True,
- help="Path to install core files.")
- install_core_parser.add_argument(
- "--core-name",
- required=True,
- choices=[
- "teensy",
- "stm32duino",
- "adafruit-samd",
- ],
- help="Name of the arduino core to install.")
-
- # list-boards command
- list_boards_parser = subparsers.add_parser("list-boards",
- help="show supported boards")
- list_boards_parser.set_defaults(func=list_boards_command)
-
- # list-menu-options command
- list_menu_options_parser = subparsers.add_parser(
- "list-menu-options",
- help="show available menu options for selected board")
- list_menu_options_parser.set_defaults(func=list_menu_options_command)
- list_menu_options_parser.add_argument("--board",
- required=True,
- help="Name of the board to use.")
-
- # show command
- show_parser = subparsers.add_parser("show",
- help="Return compiler information.")
- show_parser.add_argument(
- "--serial-port",
- default=serial_port,
- help="Serial port for flashing flash. Default: '{}'".format(
- serial_port))
- show_parser.add_argument(
- "--build-path",
- default=build_path,
- help="Build directory. Default: '{}'".format(build_path))
- show_parser.add_argument(
- "--project-path",
- default=build_path,
- help="Project directory. Default: '{}'".format(project_path))
- show_parser.add_argument(
- "--project-source-path",
- default=project_source_path,
- help="Project directory. Default: '{}'".format(project_source_path))
- show_parser.add_argument(
- "--build-project-name",
- default=project_name,
- help="Project name. Default: '{}'".format(build_project_name))
- show_parser.add_argument("--board",
- required=True,
- help="Name of the board to use.")
- show_parser.add_argument("--delimit-with-newlines",
- help="Separate flags output with newlines.",
- action="store_true")
-
- output_group = show_parser.add_mutually_exclusive_group(required=True)
- output_group.add_argument("--run-c-compile", action="store_true")
- output_group.add_argument("--run-cpp-compile", action="store_true")
- output_group.add_argument("--run-link",
- nargs="+",
- type=str,
- help="Run the link command. Expected arguments: "
- "the archive file followed by all obj files.")
- output_group.add_argument("--run-objcopy", action="store_true")
- output_group.add_argument("--run-prebuilds", action="store_true")
- output_group.add_argument("--run-postbuilds", action="store_true")
- output_group.add_argument("--c-flags", action="store_true")
- output_group.add_argument("--s-flags", action="store_true")
- output_group.add_argument("--cpp-flags", action="store_true")
- output_group.add_argument("--ld-flags", action="store_true")
- output_group.add_argument("--ar-flags", action="store_true")
- output_group.add_argument("--ld-libs", action="store_true")
- output_group.add_argument("--objcopy", help="objcopy step for SUFFIX")
- output_group.add_argument("--objcopy-flags",
- help="objcopy flags for SUFFIX")
- output_group.add_argument("--core-path", action="store_true")
- output_group.add_argument("--cc-binary", action="store_true")
- output_group.add_argument("--cxx-binary", action="store_true")
- output_group.add_argument("--ar-binary", action="store_true")
- output_group.add_argument("--objcopy-binary", action="store_true")
- output_group.add_argument("--size-binary", action="store_true")
- output_group.add_argument("--postbuild",
- help="Show recipe.hooks.postbuild.*.pattern")
- output_group.add_argument("--upload-tools", action="store_true")
- output_group.add_argument("--upload-command")
- output_group.add_argument("--run-upload-command")
- output_group.add_argument("--library-includes", action="store_true")
- output_group.add_argument("--library-c-files", action="store_true")
- output_group.add_argument("--library-cpp-files", action="store_true")
- output_group.add_argument("--core-c-files", action="store_true")
- output_group.add_argument("--core-s-files", action="store_true")
- output_group.add_argument("--core-cpp-files", action="store_true")
- output_group.add_argument("--variant-c-files", action="store_true")
- output_group.add_argument("--variant-s-files", action="store_true")
- output_group.add_argument("--variant-cpp-files", action="store_true")
-
- # nargs="+" is one or more args, e.g:
- # --menu-options menu.usb.serialhid menu.speed.150
- show_parser.add_argument("--menu-options", nargs="+", type=str)
- show_parser.set_defaults(func=show_command)
-
- args = parser.parse_args()
-
- # Set logging level and output handler
- _LOG.setLevel(args.loglevel)
- _STDERR_HANDLER.setLevel(args.loglevel)
- _STDERR_HANDLER.setFormatter(
- logging.Formatter("[%(asctime)s] "
- "%(levelname)s %(message)s", "%Y%m%d %H:%M:%S"))
- _LOG.addHandler(_STDERR_HANDLER)
-
- _LOG.debug(_pretty_format(args))
-
- compiler_path_override = False
- if args.compiler_path_override:
- compiler_path_override = os.path.realpath(
- os.path.expanduser(os.path.expandvars(
- args.compiler_path_override)))
-
- if args.subcommand == "install-core":
- args.func(args)
- elif args.subcommand in ["list-boards", "list-menu-options"]:
- builder = ArduinoBuilder(args.arduino_package_path,
- args.arduino_package_name)
- builder.load_board_definitions()
- args.func(args, builder)
- else:
- builder = ArduinoBuilder(args.arduino_package_path,
- args.arduino_package_name,
- build_path=args.build_path,
- build_project_name=args.build_project_name,
- project_path=args.project_path,
- project_source_path=args.project_source_path,
- compiler_path_override=compiler_path_override)
- builder.load_board_definitions()
- args.func(args, builder)
-
- sys.exit(0)
-
-
-if __name__ == '__main__':
- main()
diff --git a/pw_arduino_build/py/pw_arduino_build/builder.py b/pw_arduino_build/py/pw_arduino_build/builder.py
new file mode 100755
index 0000000..06c4117
--- /dev/null
+++ b/pw_arduino_build/py/pw_arduino_build/builder.py
@@ -0,0 +1,1039 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Extracts build information from Arduino cores."""
+
+import glob
+import logging
+import os
+import platform
+import pprint
+import re
+import sys
+import time
+from collections import OrderedDict
+from typing import List
+
+from pw_arduino_build import file_operations
+
+_LOG = logging.getLogger(__name__)
+
+_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint
+_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat
+
+
+def arduino_runtime_os_string():
+ arduno_platform = {
+ "Linux": "linux",
+ "Windows": "windows",
+ "Darwin": "macosx"
+ }
+ return arduno_platform[platform.system()]
+
+
+class ArduinoBuilder:
+ """Used to interpret arduino boards.txt and platform.txt files."""
+ # pylint: disable=too-many-instance-attributes,too-many-public-methods
+
+ board_menu_regex = re.compile(
+ r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE)
+
+ board_name_regex = re.compile(
+ r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE)
+
+ variable_regex = re.compile(r"^(?P<name>[^\s#=]+)=(?P<value>.*)$",
+ re.MULTILINE)
+
+ menu_option_regex = re.compile(
+ r"^menu\." # starts with "menu"
+ r"(?P<menu_option_name>[^.]+)\." # first token after .
+ r"(?P<menu_option_value>[^.]+)$") # second (final) token after .
+
+ tool_name_regex = re.compile(
+ r"^tools\." # starts with "tools"
+ r"(?P<tool_name>[^.]+)\.") # first token after .
+
+ interpolated_variable_regex = re.compile(r"{[^}]+}", re.MULTILINE)
+
+ objcopy_step_name_regex = re.compile(r"^recipe.objcopy.([^.]+).pattern$")
+
+ def __init__(self,
+ arduino_path,
+ package_name,
+ build_path=None,
+ project_path=None,
+ project_source_path=None,
+ build_project_name=None,
+ compiler_path_override=False):
+ self.arduino_path = arduino_path
+ self.arduino_package_name = package_name
+ self.selected_board = None
+ self.build_path = build_path
+ self.project_path = project_path
+ self.project_source_path = project_source_path
+ self.build_project_name = build_project_name
+ self.compiler_path_override = compiler_path_override
+ self.variant_includes = ""
+ self.build_variant_path = False
+
+ self.compiler_path_override_binaries = []
+ if self.compiler_path_override:
+ self.compiler_path_override_binaries = file_operations.find_files(
+ self.compiler_path_override, "*")
+
+ # Container dicts for boards.txt and platform.txt file data.
+ self.board = OrderedDict()
+ self.platform = OrderedDict()
+ self.menu_options = OrderedDict({
+ "global_options": {},
+ "default_board_values": {},
+ "selected": {}
+ })
+ self.tools_variables = {}
+
+ # Set and check for valid hardware folder.
+ self.hardware_path = os.path.join(self.arduino_path, "hardware")
+
+ if not os.path.exists(self.hardware_path):
+ _LOG.error("Error: Arduino package path '%s' does not exist.",
+ self.arduino_path)
+ raise FileNotFoundError
+
+ # Set and check for valid package name
+ self.package_path = os.path.join(self.arduino_path, "hardware",
+ package_name)
+ # {build.arch} is the first folder name of the package (upcased)
+ self.build_arch = os.path.split(package_name)[0].upper()
+
+ if not os.path.exists(self.package_path):
+ _LOG.error("Error: Arduino package name '%s' does not exist.",
+ package_name)
+ _LOG.error("Did you mean:\n")
+ # TODO(tonymd): On Windows concatenating "/" may not work
+ possible_alternatives = [
+ d.replace(self.hardware_path + os.sep, "", 1)
+ for d in glob.glob(self.hardware_path + "/*/*")
+ ]
+ _LOG.error("\n".join(possible_alternatives))
+ sys.exit(1)
+
+ # Grab all folder names in the cores directory. These are typically
+ # sub-core source files.
+ self.sub_core_folders = os.listdir(
+ os.path.join(self.package_path, "cores"))
+
+ self._find_tools_variables()
+
+ self.boards_txt = os.path.join(self.package_path, "boards.txt")
+ self.platform_txt = os.path.join(self.package_path, "platform.txt")
+
+ def select_board(self, board_name, menu_option_overrides=False):
+ self.selected_board = board_name
+
+ # Load default menu options for a selected board.
+ if not self.selected_board in self.board.keys():
+ _LOG.error("Error board: '%s' not supported.", self.selected_board)
+ # TODO(tonymd): Print supported boards here
+ sys.exit(1)
+
+ # Override default menu options if any are specified.
+ if menu_option_overrides:
+ for moption in menu_option_overrides:
+ if not self.set_menu_option(moption):
+ # TODO(tonymd): Print supported menu options here
+ sys.exit(1)
+
+ self._copy_default_menu_options_to_build_variables()
+ self._apply_recipe_overrides()
+ self._substitute_variables()
+
+ def _apply_recipe_overrides(self):
+ # Override link recipes with per-core exceptions
+ # Teensyduino cores
+ if self.build_arch == 'TEENSY':
+ # Change {build.path}/{archive_file}
+ # To {archive_file_path} (which should contain the core.a file)
+ new_link_line = self.platform["recipe.c.combine.pattern"].replace(
+ "{object_files} \"{build.path}/{archive_file}\"",
+ "{object_files} {archive_file_path}", 1)
+ # Add the teensy provided toolchain lib folder for link access to
+ # libarm_cortexM*_math.a
+ new_link_line = new_link_line.replace(
+ "\"-L{build.path}\"",
+ "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib",
+ 1)
+ self.platform["recipe.c.combine.pattern"] = new_link_line
+
+ # Adafruit-samd core
+ # TODO(tonymd): This build_arch may clash with Arduino-SAMD core
+ elif self.build_arch == 'SAMD':
+ new_link_line = self.platform["recipe.c.combine.pattern"].replace(
+ "\"{build.path}/{archive_file}\" -Wl,--end-group",
+ "{archive_file_path} -Wl,--end-group", 1)
+ self.platform["recipe.c.combine.pattern"] = new_link_line
+
+ # STM32L4 Core:
+ # https://github.com/GrumpyOldPizza/arduino-STM32L4
+ elif self.build_arch == 'STM32L4':
+ # TODO(tonymd): {build.path}/{archive_file} for the link step always
+ # seems to be core.a (except STM32 core)
+ line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\""
+ new_link_line = self.platform["recipe.c.combine.pattern"].replace(
+ line_to_delete, "-Wl,--start-group {archive_file_path}", 1)
+ self.platform["recipe.c.combine.pattern"] = new_link_line
+
+ # stm32duino core
+ elif self.build_arch == 'STM32':
+ pass
+
+ def _copy_default_menu_options_to_build_variables(self):
+ # Clear existing options
+ self.menu_options["selected"] = {}
+ # Set default menu options for selected board
+ for menu_key, menu_dict in self.menu_options["default_board_values"][
+ self.selected_board].items():
+ for name, var in self.board[self.selected_board].items():
+ starting_key = "{}.{}.".format(menu_key, menu_dict["name"])
+ if name.startswith(starting_key):
+ new_var_name = name.replace(starting_key, "", 1)
+ self.menu_options["selected"][new_var_name] = var
+
+ def set_menu_option(self, moption):
+ if moption not in self.board[self.selected_board]:
+ _LOG.error("Error: '%s' is not a valid menu option.", moption)
+ return False
+
+ # Override default menu option with new value.
+ menu_match_result = self.menu_option_regex.match(moption)
+ if menu_match_result:
+ menu_match = menu_match_result.groupdict()
+ menu_value = menu_match["menu_option_value"]
+ menu_key = "menu.{}".format(menu_match["menu_option_name"])
+ self.menu_options["default_board_values"][
+ self.selected_board][menu_key]["name"] = menu_value
+
+ # Update build variables
+ self._copy_default_menu_options_to_build_variables()
+ return True
+
+ def _set_global_arduino_variables(self):
+ """Set some global variables defined by the Arduino-IDE.
+
+ See Docs:
+ https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties
+ """
+
+ for current_board_name in self.board.keys():
+ if self.build_path:
+ self.board[current_board_name]["build.path"] = self.build_path
+ if self.build_project_name:
+ self.board[current_board_name][
+ "build.project_name"] = self.build_project_name
+ # {archive_file} is the final *.elf
+ archive_file = "{}.elf".format(self.build_project_name)
+ self.board[current_board_name]["archive_file"] = archive_file
+ # {archive_file_path} is the final core.a archive
+ if self.build_path:
+ self.board[current_board_name][
+ "archive_file_path"] = os.path.join(
+ self.build_path, "core.a")
+ if self.project_source_path:
+ self.board[current_board_name][
+ "build.source.path"] = self.project_source_path
+
+ self.board[current_board_name]["extra.time.local"] = str(
+ int(time.time()))
+ self.board[current_board_name]["runtime.ide.version"] = "10812"
+ self.board[current_board_name][
+ "runtime.hardware.path"] = self.hardware_path
+
+ # Copy {runtime.tools.TOOL_NAME.path} vars
+ self._set_tools_variables(self.board[current_board_name])
+
+ self.board[current_board_name][
+ "runtime.platform.path"] = self.package_path
+ if self.platform["name"] == "Teensyduino":
+ # Teensyduino is installed into the arduino IDE folder
+ # rather than ~/.arduino15/packages/
+ self.board[current_board_name][
+ "runtime.hardware.path"] = os.path.join(
+ self.hardware_path, "teensy")
+
+ self.board[current_board_name]["build.system.path"] = os.path.join(
+ self.package_path, "system")
+
+ # Set the {build.core.path} variable that pointing to a sub-core
+ # folder. For Teensys this is:
+ # 'teensy/hardware/teensy/avr/cores/teensy{3,4}'. For other cores
+ # it's typically just the 'arduino' folder. For example:
+ # 'arduino-samd/hardware/samd/1.8.8/cores/arduino'
+ core_path = os.path.join(
+ self.package_path, "cores",
+ self.board[current_board_name].get("build.core",
+ self.sub_core_folders[0]))
+ self.board[current_board_name]["build.core.path"] = core_path
+
+ self.board[current_board_name]["build.arch"] = self.build_arch
+
+ for name, var in self.board[current_board_name].items():
+ self.board[current_board_name][name] = var.replace(
+ "{build.core.path}", core_path)
+
+ def load_board_definitions(self):
+ """Loads Arduino boards.txt and platform.txt files into dictionaries.
+
+ Populates the following dictionaries:
+ self.menu_options
+ self.boards
+ self.platform
+ """
+ # Load platform.txt
+ with open(self.platform_txt, "r") as pfile:
+ platform_file = pfile.read()
+ platform_var_matches = self.variable_regex.finditer(platform_file)
+ for p_match in [m.groupdict() for m in platform_var_matches]:
+ self.platform[p_match["name"]] = p_match["value"]
+
+ # Load boards.txt
+ with open(self.boards_txt, "r") as bfile:
+ board_file = bfile.read()
+ # Get all top-level menu options, e.g. menu.usb=USB Type
+ board_menu_matches = self.board_menu_regex.finditer(board_file)
+ for menuitem in [m.groupdict() for m in board_menu_matches]:
+ self.menu_options["global_options"][menuitem["name"]] = {
+ "description": menuitem["description"]
+ }
+
+ # Get all board names, e.g. teensy40.name=Teensy 4.0
+ board_name_matches = self.board_name_regex.finditer(board_file)
+ for b_match in [m.groupdict() for m in board_name_matches]:
+ self.board[b_match["name"]] = OrderedDict()
+ self.menu_options["default_board_values"][
+ b_match["name"]] = OrderedDict()
+
+ # Get all board variables, e.g. teensy40.*
+ for current_board_name in self.board.keys():
+ board_line_matches = re.finditer(
+ fr"^\s*{current_board_name}\."
+ fr"(?P<key>[^#=]+)=(?P<value>.*)$", board_file,
+ re.MULTILINE)
+ for b_match in [m.groupdict() for m in board_line_matches]:
+ # Check if this line is a menu option
+ # (e.g. 'menu.usb.serial') and save as default if it's the
+ # first one seen.
+ ArduinoBuilder.save_default_menu_option(
+ current_board_name, b_match["key"], b_match["value"],
+ self.menu_options)
+ self.board[current_board_name][
+ b_match["key"]] = b_match["value"].strip()
+
+ self._set_global_arduino_variables()
+
+ @staticmethod
+ def save_default_menu_option(current_board_name, key, value, menu_options):
+ """Save a given menu option as the default.
+
+ Saves the key and value into menu_options["default_board_values"]
+ if it doesn't already exist. Assumes menu options are added in the order
+ specified in boards.txt. The first value for a menu key is the default.
+ """
+ # Check if key is a menu option
+ # e.g. menu.usb.serial
+ # menu.usb.serial.build.usbtype
+ menu_match_result = re.match(
+ r'^menu\.' # starts with "menu"
+ r'(?P<menu_option_name>[^.]+)\.' # first token after .
+ r'(?P<menu_option_value>[^.]+)' # second token after .
+ r'(\.(?P<rest>.+))?', # optionally any trailing tokens after a .
+ key)
+ if menu_match_result:
+ menu_match = menu_match_result.groupdict()
+ current_menu_key = "menu.{}".format(menu_match["menu_option_name"])
+ # If this is the first menu option seen for current_board_name, save
+ # as the default.
+ if current_menu_key not in menu_options["default_board_values"][
+ current_board_name]:
+ menu_options["default_board_values"][current_board_name][
+ current_menu_key] = {
+ "name": menu_match["menu_option_value"],
+ "description": value
+ }
+
+ def _replace_variables(self, line, variable_lookup_source):
+ """Replace {variables} from loaded boards.txt or platform.txt.
+
+ Replace interpolated variables surrounded by curly braces in line with
+ definitions from variable_lookup_source.
+ """
+ new_line = line
+ for current_var_match in self.interpolated_variable_regex.findall(
+ line):
+ # {build.flags.c} --> build.flags.c
+ current_var = current_var_match.strip("{}")
+
+ # check for matches in board definition
+ if current_var in variable_lookup_source:
+ replacement = variable_lookup_source.get(current_var, "")
+ new_line = new_line.replace(current_var_match, replacement)
+ return new_line
+
+ def _find_tools_variables(self):
+ # Gather tool directories in order of increasing precedence
+ runtime_tool_paths = []
+
+ # Check for tools installed in ~/.arduino15/packages/arduino/tools/
+ # TODO(tonymd): Is this Mac & Linux specific?
+ runtime_tool_paths += glob.glob(
+ os.path.join(
+ os.path.realpath(os.path.expanduser(os.path.expandvars("~"))),
+ ".arduino15", "packages", "arduino", "tools", "*"))
+
+ # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES>
+ runtime_tool_paths += glob.glob(
+ os.path.join(self.arduino_path, "tools",
+ arduino_runtime_os_string(), "*"))
+ # <ARDUINO_PATH>/tools/<TOOL_NAMES>
+ # This will grab linux/windows/macosx/share as <TOOL_NAMES>.
+ runtime_tool_paths += glob.glob(
+ os.path.join(self.arduino_path, "tools", "*"))
+
+ # Process package tools after arduino tools.
+ # They should overwrite vars & take precedence.
+
+ # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES>
+ runtime_tool_paths += glob.glob(
+ os.path.join(self.package_path, "tools",
+ arduino_runtime_os_string(), "*"))
+ # <PACKAGE_PATH>/tools/<TOOL_NAMES>
+ # This will grab linux/windows/macosx/share as <TOOL_NAMES>.
+ runtime_tool_paths += glob.glob(
+ os.path.join(self.package_path, "tools", "*"))
+
+ for path in runtime_tool_paths:
+ # Make sure TOOL_NAME is not an OS string
+ if not (path.endswith("linux") or path.endswith("windows")
+ or path.endswith("macosx") or path.endswith("share")):
+ # TODO(tonymd): Check if a file & do nothing?
+
+ # Check if it's a directory with subdir(s) as a version string
+ # create all 'runtime.tools.{tool_folder}-{version.path}'
+ # (for each version)
+ # create 'runtime.tools.{tool_folder}.path'
+ # (with latest version)
+ if os.path.isdir(path):
+ # Grab the tool name (folder) by itself.
+ tool_folder = os.path.basename(path)
+ # Sort so that [-1] is the latest version.
+ version_paths = sorted(glob.glob(os.path.join(path, "*")))
+ # Check if all sub folders start with a version string.
+ if len(version_paths) == sum(
+ bool(re.match(r"^[0-9.]+", os.path.basename(vp)))
+ for vp in version_paths):
+ for version_path in version_paths:
+ version_string = os.path.basename(version_path)
+ var_name = "runtime.tools.{}-{}.path".format(
+ tool_folder, version_string)
+ self.tools_variables[var_name] = os.path.join(
+ path, version_string)
+ var_name = "runtime.tools.{}.path".format(tool_folder)
+ self.tools_variables[var_name] = os.path.join(
+ path, os.path.basename(version_paths[-1]))
+ # Else set toolpath to path.
+ else:
+ var_name = "runtime.tools.{}.path".format(tool_folder)
+ self.tools_variables[var_name] = path
+
+ _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables))
+
+ # Copy self.tools_variables into destination
+ def _set_tools_variables(self, destination):
+ for key, value in self.tools_variables.items():
+ destination[key] = value
+
+ def _substitute_variables(self):
+ """Perform variable substitution in board and platform metadata."""
+
+ # menu -> board
+ # Copy selected menu variables into board definiton
+ for name, value in self.menu_options["selected"].items():
+ self.board[self.selected_board][name] = value
+
+ # board -> board
+ # Replace any {vars} in the selected board with values defined within
+ # (and from copied in menu options).
+ for var, value in self.board[self.selected_board].items():
+ self.board[self.selected_board][var] = self._replace_variables(
+ value, self.board[self.selected_board])
+
+ # Check for build.variant variable
+ # This will be set in selected board after menu options substitution
+ build_variant = self.board[self.selected_board].get(
+ "build.variant", None)
+ if build_variant:
+ # Set build.variant.path
+ bvp = os.path.join(self.package_path, "variants", build_variant)
+ self.build_variant_path = bvp
+ self.board[self.selected_board]["build.variant.path"] = bvp
+ # Add the variant folder as an include directory
+ # (used in stm32l4 core)
+ self.variant_includes = "-I{}".format(bvp)
+
+ _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform))
+
+ # board -> platform
+ # Replace {vars} in platform from the selected board definition
+ for var, value in self.platform.items():
+ self.platform[var] = self._replace_variables(
+ value, self.board[self.selected_board])
+
+ # platform -> platform
+ # Replace any remaining {vars} in platform from platform
+ for var, value in self.platform.items():
+ self.platform[var] = self._replace_variables(value, self.platform)
+
+ # Repeat platform -> platform for any lingering variables
+ # Example: {build.opt.name} in STM32 core
+ for var, value in self.platform.items():
+ self.platform[var] = self._replace_variables(value, self.platform)
+
+ _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options))
+ _LOG.debug("SELECTED_BOARD: %s",
+ _pretty_format(self.board[self.selected_board]))
+ _LOG.debug("PLATFORM: %s", _pretty_format(self.platform))
+
+ def selected_board_spec(self):
+ return self.board[self.selected_board]
+
+ def get_menu_options(self):
+ all_options = []
+ max_string_length = [0, 0]
+
+ for key_name, description in self.board[self.selected_board].items():
+ menu_match_result = self.menu_option_regex.match(key_name)
+ if menu_match_result:
+ menu_match = menu_match_result.groupdict()
+ name = "menu.{}.{}".format(menu_match["menu_option_name"],
+ menu_match["menu_option_value"])
+ if len(name) > max_string_length[0]:
+ max_string_length[0] = len(name)
+ if len(description) > max_string_length[1]:
+ max_string_length[1] = len(description)
+ all_options.append((name, description))
+
+ return all_options, max_string_length
+
+ def get_default_menu_options(self):
+ default_options = []
+ max_string_length = [0, 0]
+
+ for key_name, value in self.menu_options["default_board_values"][
+ self.selected_board].items():
+ full_key = key_name + "." + value["name"]
+ if len(full_key) > max_string_length[0]:
+ max_string_length[0] = len(full_key)
+ if len(value["description"]) > max_string_length[1]:
+ max_string_length[1] = len(value["description"])
+ default_options.append((full_key, value["description"]))
+
+ return default_options, max_string_length
+
+ @staticmethod
+ def split_binary_from_arguments(compile_line):
+ compile_binary = None
+ rest_of_line = compile_line
+
+ compile_binary_match = re.search(r'^("[^"]+") ', compile_line)
+ if compile_binary_match:
+ compile_binary = compile_binary_match[1]
+ rest_of_line = compile_line.replace(compile_binary_match[0], "", 1)
+
+ return compile_binary, rest_of_line
+
+ def _strip_includes_source_file_object_file_vars(self, compile_line):
+ line = compile_line
+ if self.variant_includes:
+ line = compile_line.replace(
+ "{includes} \"{source_file}\" -o \"{object_file}\"",
+ self.variant_includes, 1)
+ else:
+ line = compile_line.replace(
+ "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1)
+ return line
+
+ def _get_tool_name(self, line):
+ tool_match_result = self.tool_name_regex.match(line)
+ if tool_match_result:
+ return tool_match_result[1]
+ return False
+
+ def get_upload_tool_names(self):
+ return [
+ self._get_tool_name(t) for t in self.platform.keys()
+ if self.tool_name_regex.match(t) and 'upload.pattern' in t
+ ]
+
+ # TODO(tonymd): Use these getters in _replace_variables() or
+ # _substitute_variables()
+
+ def _get_platform_variable(self, variable):
+ # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name,
+ # compare with platform.system() and return that instead.
+ return self.platform.get(variable, False)
+
+ def _get_platform_variable_with_substitutions(self, variable, namespace):
+ line = self.platform.get(variable, False)
+ # Get all unique variables used in this line in line.
+ unique_vars = sorted(
+ set(self.interpolated_variable_regex.findall(line)))
+ # Search for each unique_vars in namespace and global.
+ for var in unique_vars:
+ v_raw_name = var.strip("{}")
+
+ # Check for namespace.variable
+ # eg: 'tools.stm32CubeProg.cmd'
+ possible_var_name = "{}.{}".format(namespace, v_raw_name)
+ result = self._get_platform_variable(possible_var_name)
+ # Check for os overriden variable
+ # eg:
+ # ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'),
+ # ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'),
+ possible_var_name = "{}.{}.{}".format(namespace, v_raw_name,
+ arduino_runtime_os_string())
+ os_override_result = self._get_platform_variable(possible_var_name)
+
+ if os_override_result:
+ line = line.replace(var, os_override_result)
+ elif result:
+ line = line.replace(var, result)
+ # Check for variable at top level?
+ # elif self._get_platform_variable(v_raw_name):
+ # line = line.replace(self._get_platform_variable(v_raw_name),
+ # result)
+ return line
+
+ def get_upload_line(self, tool_name, serial_port=False):
+ # TODO(tonymd): Error if tool_name does not exist
+ tool_namespace = "tools.{}".format(tool_name)
+ pattern = "tools.{}.upload.pattern".format(tool_name)
+
+ if not self._get_platform_variable(pattern):
+ _LOG.error("Error: upload tool '%s' does not exist.", tool_name)
+ tools = self.get_upload_tool_names()
+ _LOG.error("Valid tools: %s", ", ".join(tools))
+ return sys.exit(1)
+
+ line = self._get_platform_variable_with_substitutions(
+ pattern, tool_namespace)
+
+ # TODO(tonymd): Teensy specific tool overrides.
+ if tool_name == "teensyloader":
+ # Remove un-necessary lines
+ # {serial.port.label} and {serial.port.protocol} are returned by
+ # the teensy_ports binary.
+ line = line.replace("\"-portlabel={serial.port.label}\"", "", 1)
+ line = line.replace("\"-portprotocol={serial.port.protocol}\"", "",
+ 1)
+
+ if serial_port:
+ line = line.replace("{serial.port}", serial_port, 1)
+
+ return line
+
+ def _get_binary_path(self, variable_pattern):
+ compile_line = self.replace_compile_binary_with_override_path(
+ self._get_platform_variable(variable_pattern))
+ compile_binary, _ = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ return compile_binary
+
+ def get_cc_binary(self):
+ return self._get_binary_path("recipe.c.o.pattern")
+
+ def get_cxx_binary(self):
+ return self._get_binary_path("recipe.cpp.o.pattern")
+
+ def get_objcopy_binary(self):
+ objcopy_step_name = self.get_objcopy_step_names()[0]
+ objcopy_binary = self._get_binary_path(objcopy_step_name)
+ return objcopy_binary
+
+ def get_ar_binary(self):
+ return self._get_binary_path("recipe.ar.pattern")
+
+ def get_size_binary(self):
+ return self._get_binary_path("recipe.size.pattern")
+
+ def replace_command_args_with_compiler_override_path(self, compile_line):
+ if not self.compiler_path_override:
+ return compile_line
+ replacement_line = compile_line
+ replacement_line_args = compile_line.split()
+ for arg in replacement_line_args:
+ compile_binary_basename = os.path.basename(arg.strip("\""))
+ if compile_binary_basename in self.compiler_path_override_binaries:
+ new_compiler = os.path.join(self.compiler_path_override,
+ compile_binary_basename)
+ replacement_line = replacement_line.replace(
+ arg, new_compiler, 1)
+ return replacement_line
+
+ def replace_compile_binary_with_override_path(self, compile_line):
+ replacement_compile_line = compile_line
+
+ # Change the compiler path if there's an override path set
+ if self.compiler_path_override:
+ compile_binary, line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ compile_binary_basename = os.path.basename(
+ compile_binary.strip("\""))
+ new_compiler = os.path.join(self.compiler_path_override,
+ compile_binary_basename)
+ if platform.system() == "Windows" and not re.match(
+ r".*\.exe$", new_compiler, flags=re.IGNORECASE):
+ new_compiler += ".exe"
+
+ if os.path.isfile(new_compiler):
+ replacement_compile_line = "\"{}\" {}".format(
+ new_compiler, line)
+
+ return replacement_compile_line
+
+ def get_c_compile_line(self):
+ _LOG.debug("ARDUINO_C_COMPILE: %s",
+ _pretty_format(self.platform["recipe.c.o.pattern"]))
+
+ compile_line = self.platform["recipe.c.o.pattern"]
+ compile_line = self._strip_includes_source_file_object_file_vars(
+ compile_line)
+ compile_line += " -I{}".format(
+ self.board[self.selected_board]["build.core.path"])
+
+ compile_line = self.replace_compile_binary_with_override_path(
+ compile_line)
+ return compile_line
+
+ def get_s_compile_line(self):
+ _LOG.debug("ARDUINO_S_COMPILE %s",
+ _pretty_format(self.platform["recipe.S.o.pattern"]))
+
+ compile_line = self.platform["recipe.S.o.pattern"]
+ compile_line = self._strip_includes_source_file_object_file_vars(
+ compile_line)
+ compile_line += " -I{}".format(
+ self.board[self.selected_board]["build.core.path"])
+
+ compile_line = self.replace_compile_binary_with_override_path(
+ compile_line)
+ return compile_line
+
+ def get_ar_compile_line(self):
+ _LOG.debug("ARDUINO_AR_COMPILE: %s",
+ _pretty_format(self.platform["recipe.ar.pattern"]))
+
+ compile_line = self.platform["recipe.ar.pattern"].replace(
+ "\"{object_file}\"", "", 1)
+
+ compile_line = self.replace_compile_binary_with_override_path(
+ compile_line)
+ return compile_line
+
+ def get_cpp_compile_line(self):
+ _LOG.debug("ARDUINO_CPP_COMPILE: %s",
+ _pretty_format(self.platform["recipe.cpp.o.pattern"]))
+
+ compile_line = self.platform["recipe.cpp.o.pattern"]
+ compile_line = self._strip_includes_source_file_object_file_vars(
+ compile_line)
+ compile_line += " -I{}".format(
+ self.board[self.selected_board]["build.core.path"])
+
+ compile_line = self.replace_compile_binary_with_override_path(
+ compile_line)
+ return compile_line
+
+ def get_link_line(self):
+ _LOG.debug("ARDUINO_LINK: %s",
+ _pretty_format(self.platform["recipe.c.combine.pattern"]))
+
+ compile_line = self.platform["recipe.c.combine.pattern"]
+
+ compile_line = self.replace_compile_binary_with_override_path(
+ compile_line)
+ return compile_line
+
+ def get_objcopy_step_names(self):
+ names = [
+ name for name, line in self.platform.items()
+ if self.objcopy_step_name_regex.match(name)
+ ]
+ return names
+
+ def get_objcopy_steps(self) -> List[str]:
+ lines = [
+ line for name, line in self.platform.items()
+ if self.objcopy_step_name_regex.match(name)
+ ]
+ lines = [
+ self.replace_compile_binary_with_override_path(line)
+ for line in lines
+ ]
+ return lines
+
+ # TODO(tonymd): These recipes are probably run in sorted order
+ def get_objcopy(self, suffix):
+ # Expected vars:
+ # teensy:
+ # recipe.objcopy.eep.pattern
+ # recipe.objcopy.hex.pattern
+
+ pattern = "recipe.objcopy.{}.pattern".format(suffix)
+ objcopy_step_names = self.get_objcopy_step_names()
+
+ objcopy_suffixes = [
+ m[1] for m in [
+ self.objcopy_step_name_regex.match(line)
+ for line in objcopy_step_names
+ ] if m
+ ]
+ if pattern not in objcopy_step_names:
+ _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix)
+ _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes))
+ return sys.exit(1)
+
+ line = self._get_platform_variable(pattern)
+
+ _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line)
+
+ line = self.replace_compile_binary_with_override_path(line)
+
+ return line
+
+ def get_objcopy_flags(self, suffix):
+ # TODO(tonymd): Possibly teensy specific variables.
+ flags = ""
+ if suffix == "hex":
+ flags = self.platform.get("compiler.elf2hex.flags", "")
+ elif suffix == "bin":
+ flags = self.platform.get("compiler.elf2bin.flags", "")
+ elif suffix == "eep":
+ flags = self.platform.get("compiler.objcopy.eep.flags", "")
+ return flags
+
+ # TODO(tonymd): There are more recipe hooks besides postbuild.
+ # They are run in sorted order.
+ # TODO(tonymd): Rename this to get_hooks(hook_name, step).
+ # TODO(tonymd): Add a list-hooks and or run-hooks command
+ def get_postbuild_line(self, step_number):
+ line = self.platform["recipe.hooks.postbuild.{}.pattern".format(
+ step_number)]
+ line = self.replace_command_args_with_compiler_override_path(line)
+ return line
+
+ def get_prebuild_steps(self) -> List[str]:
+ # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern
+ # stm32 core uses recipe.hooks.prebuild.1.pattern
+ # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows
+ # (should override non-windows key)
+ lines = [
+ line for name, line in self.platform.items() if re.match(
+ r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name)
+ ]
+ # TODO(tonymd): Write a function to fetch/replace OS specific patterns
+ # (ending in an OS string)
+ lines = [
+ self.replace_compile_binary_with_override_path(line)
+ for line in lines
+ ]
+ return lines
+
+ def get_postbuild_steps(self) -> List[str]:
+ lines = [
+ line for name, line in self.platform.items()
+ if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name)
+ ]
+
+ lines = [
+ self.replace_command_args_with_compiler_override_path(line)
+ for line in lines
+ ]
+ return lines
+
+ def get_s_flags(self):
+ compile_line = self.get_s_compile_line()
+ _, compile_line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ compile_line = compile_line.replace("-c", "", 1)
+ return compile_line.strip()
+
+ def get_c_flags(self):
+ compile_line = self.get_c_compile_line()
+ _, compile_line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ compile_line = compile_line.replace("-c", "", 1)
+ return compile_line.strip()
+
+ def get_cpp_flags(self):
+ compile_line = self.get_cpp_compile_line()
+ _, compile_line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ compile_line = compile_line.replace("-c", "", 1)
+ return compile_line.strip()
+
+ def get_ar_flags(self):
+ compile_line = self.get_ar_compile_line()
+ _, compile_line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ return compile_line.strip()
+
+ def get_ld_flags(self):
+ compile_line = self.get_link_line()
+ _, compile_line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+
+ # TODO(tonymd): This is teensy specific
+ line_to_delete = "-o \"{build.path}/{build.project_name}.elf\" " \
+ "{object_files} \"-L{build.path}\""
+ if self.build_path:
+ line_to_delete = line_to_delete.replace("{build.path}",
+ self.build_path)
+ if self.build_project_name:
+ line_to_delete = line_to_delete.replace("{build.project_name}",
+ self.build_project_name)
+
+ compile_line = compile_line.replace(line_to_delete, "", 1)
+ libs = re.findall(r'(-l[^ ]+ ?)', compile_line)
+ for lib in libs:
+ compile_line = compile_line.replace(lib, "", 1)
+ libs = [lib.strip() for lib in libs]
+
+ return compile_line.strip()
+
+ def get_ld_libs(self):
+ compile_line = self.get_link_line()
+ _, compile_line = ArduinoBuilder.split_binary_from_arguments(
+ compile_line)
+ # TODO(tonymd): This replacement is teensy specific
+ compile_line = compile_line.replace(
+ "-o \"{build.path}/{build.project_name}.elf\" "
+ "{object_files} \"-L{build.path}\"", "", 1)
+ libs = re.findall(r'(-l[^ ]+ ?)', compile_line)
+ for lib in libs:
+ compile_line = compile_line.replace(lib, "", 1)
+ libs = [lib.strip() for lib in libs]
+
+ return " ".join(libs)
+
+ def library_folders(self):
+ # Arduino library format documentation:
+ # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files
+ # - If src folder exists,
+ # use that as the root include directory -Ilibraries/libname/src
+ # - Else lib folder as root include -Ilibraries/libname
+ # (exclude source files in the examples folder in this case)
+
+ library_path = os.path.join(self.project_path, "libraries")
+
+ library_folders = file_operations.find_files(library_path, ["*"],
+ directories_only=True)
+ library_source_root_folders = []
+ for lib in library_folders:
+ lib_dir = os.path.join(library_path, lib)
+ src_dir = os.path.join(lib_dir, "src")
+ if os.path.exists(src_dir) and os.path.isdir(src_dir):
+ library_source_root_folders.append(src_dir)
+ else:
+ library_source_root_folders.append(lib_dir)
+
+ return library_source_root_folders
+
+ def library_includes(self):
+ include_args = []
+ library_folders = self.library_folders()
+ for lib_dir in library_folders:
+ include_args.append("-I{}".format(os.path.relpath(lib_dir)))
+ return include_args
+
+ def library_files(self, pattern):
+ sources = []
+ library_folders = self.library_folders()
+ for lib_dir in library_folders:
+ for file_path in file_operations.find_files(lib_dir, [pattern]):
+ if not file_path.startswith("examples"):
+ sources.append(
+ os.path.relpath(os.path.join(lib_dir, file_path)))
+ return sources
+
+ def library_c_files(self):
+ return self.library_files("**/*.c")
+
+ def library_cpp_files(self):
+ return self.library_files("**/*.cpp")
+
+ def get_core_path(self):
+ return self.board[self.selected_board]["build.core.path"]
+
+ def core_files(self, pattern):
+ sources = []
+ for file_path in file_operations.find_files(self.get_core_path(),
+ [pattern]):
+ sources.append(os.path.join(self.get_core_path(), file_path))
+ return sources
+
+ def core_c_files(self):
+ return self.core_files("**/*.c")
+
+ def core_s_files(self):
+ return self.core_files("**/*.S")
+
+ def core_cpp_files(self):
+ return self.core_files("**/*.cpp")
+
+ def get_variant_path(self):
+ return self.build_variant_path
+
+ def variant_files(self, pattern):
+ sources = []
+ if self.build_variant_path:
+ for file_path in file_operations.find_files(
+ self.get_variant_path(), [pattern]):
+ sources.append(os.path.join(self.get_variant_path(),
+ file_path))
+ return sources
+
+ def variant_c_files(self):
+ return self.variant_files("**/*.c")
+
+ def variant_s_files(self):
+ return self.variant_files("**/*.S")
+
+ def variant_cpp_files(self):
+ return self.variant_files("**/*.cpp")
+
+ def project_files(self, pattern):
+ sources = []
+ for file_path in file_operations.find_files(self.project_path,
+ [pattern]):
+ if not file_path.startswith(
+ "examples") and not file_path.startswith("libraries"):
+ sources.append(file_path)
+ return sources
+
+ def project_c_files(self):
+ return self.project_files("**/*.c")
+
+ def project_cpp_files(self):
+ return self.project_files("**/*.cpp")
+
+ def project_ino_files(self):
+ return self.project_files("**/*.ino")
diff --git a/pw_arduino_build/py/pw_arduino_build/core_installer.py b/pw_arduino_build/py/pw_arduino_build/core_installer.py
new file mode 100644
index 0000000..a1e3d3c
--- /dev/null
+++ b/pw_arduino_build/py/pw_arduino_build/core_installer.py
@@ -0,0 +1,346 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Arduino Core Installer."""
+
+import argparse
+import logging
+import os
+import platform
+import stat
+import sys
+import shutil
+import subprocess
+from typing import Dict
+
+import pw_arduino_build.file_operations as file_operations
+
+_LOG = logging.getLogger(__name__)
+
+# yapf: disable
+_ARDUINO_CORE_ARTIFACTS: Dict[str, Dict] = {
+ # pylint: disable=line-too-long
+ "teensy": {
+ "Linux": {
+ "arduino-ide": {
+ "url": "https://downloads.arduino.cc/arduino-1.8.13-linux64.tar.xz",
+ "file_name": "arduino-1.8.13-linux64.tar.xz",
+ "sha256": "1b20d0ec850a2a63488009518725f058668bb6cb48c321f82dcf47dc4299b4ad",
+ },
+ "teensyduino": {
+ "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linux64",
+ "file_name": "TeensyduinoInstall.linux64",
+ "sha256": "2e6cd99a757bc80593ea3de006de4cc934bcb0a6ec74cad8ec327f0289d40f0b",
+ },
+ },
+ # TODO(tonymd): Handle 32-bit Linux Install?
+ "Linux32": {
+ "arduino-ide": {
+ "url": "https://downloads.arduino.cc/arduino-1.8.13-linux32.tar.xz",
+ "file_name": "arduino-1.8.13-linux32.tar.xz",
+ "sha256": "",
+ },
+ "teensyduino": {
+ "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linux32",
+ "file_name": "TeensyduinoInstall.linux32",
+ "sha256": "",
+ },
+ },
+ # TODO(tonymd): Handle ARM32 (Raspberry Pi) Install?
+ "LinuxARM32": {
+ "arduino-ide": {
+ "url": "https://downloads.arduino.cc/arduino-1.8.13-linuxarm.tar.xz",
+ "file_name": "arduino-1.8.13-linuxarm.tar.xz",
+ "sha256": "",
+ },
+ "teensyduino": {
+ "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linuxarm",
+ "file_name": "TeensyduinoInstall.linuxarm",
+ "sha256": "",
+ },
+ },
+ # TODO(tonymd): Handle ARM64 Install?
+ "LinuxARM64": {
+ "arduino-ide": {
+ "url": "https://downloads.arduino.cc/arduino-1.8.13-linuxaarch64.tar.xz",
+ "file_name": "arduino-1.8.13-linuxaarch64.tar.xz",
+ "sha256": "",
+ },
+ "teensyduino": {
+ "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.linuxaarch64",
+ "file_name": "TeensyduinoInstall.linuxaarch64",
+ "sha256": "",
+ },
+ },
+ "Darwin": {
+ "teensyduino": {
+ "url": "https://www.pjrc.com/teensy/td_153/Teensyduino_MacOS_Catalina.zip",
+ "file_name": "Teensyduino_MacOS_Catalina.zip",
+ "sha256": "401ef42c6e83e621cdda20191a4ef9b7db8a214bede5a94a9e26b45f79c64fe2",
+ },
+ },
+ "Windows": {
+ "arduino-ide": {
+ "url": "https://downloads.arduino.cc/arduino-1.8.13-windows.zip",
+ "file_name": "arduino-1.8.13-windows.zip",
+ "sha256": "78d3e96827b9e9b31b43e516e601c38d670d29f12483e88cbf6d91a0f89ef524",
+ },
+ "teensyduino": {
+ "url": "https://www.pjrc.com/teensy/td_153/TeensyduinoInstall.exe",
+ "file_name": "TeensyduinoInstall.exe",
+ "sha256": "88f58681e5c4772c54e462bc88280320e4276e5b316dcab592fe38d96db990a1",
+ },
+ }
+ },
+ "adafruit-samd": {
+ "all": {
+ "core": {
+ "version": "1.6.2",
+ "url": "https://github.com/adafruit/ArduinoCore-samd/archive/1.6.2.tar.gz",
+ "sha256": "5875f5bc05904c10e6313f02653f28f2f716db639d3d43f5a1d8a83d15339d64",
+ }
+ },
+ "Linux": {},
+ "Darwin": {},
+ "Windows": {},
+ },
+ "arduino-samd": {
+ "all": {
+ "core": {
+ "version": "1.8.8",
+ "url": "http://downloads.arduino.cc/cores/samd-1.8.8.tar.bz2",
+ "file_name": "samd-1.8.8.tar.bz2",
+ "sha256": "7b93eb705cba9125d9ee52eba09b51fb5fe34520ada351508f4253abbc9f27fa",
+ }
+ },
+ "Linux": {},
+ "Darwin": {},
+ "Windows": {},
+ },
+ "stm32duino": {
+ "all": {
+ "core": {
+ "version": "1.9.0",
+ "url": "https://github.com/stm32duino/Arduino_Core_STM32/archive/1.9.0.tar.gz",
+ "sha256": "4f75ba7a117d90392e8f67c58d31d22393749b9cdd3279bc21e7261ec06c62bf",
+ }
+ },
+ "Linux": {},
+ "Darwin": {},
+ "Windows": {},
+ },
+}
+# yapf: enable
+
+
+def install_core_command(args: argparse.Namespace):
+ install_prefix = os.path.realpath(
+ os.path.expanduser(os.path.expandvars(args.prefix)))
+ install_dir = os.path.join(install_prefix, args.core_name)
+ cache_dir = os.path.join(install_prefix, ".cache", args.core_name)
+
+ if args.core_name in supported_cores():
+ shutil.rmtree(install_dir, ignore_errors=True)
+ os.makedirs(install_dir, exist_ok=True)
+ os.makedirs(cache_dir, exist_ok=True)
+
+ if args.core_name == "teensy":
+ if platform.system() == "Linux":
+ install_teensy_core_linux(install_prefix, install_dir, cache_dir)
+ elif platform.system() == "Darwin":
+ install_teensy_core_mac(install_prefix, install_dir, cache_dir)
+ elif platform.system() == "Windows":
+ install_teensy_core_windows(install_prefix, install_dir, cache_dir)
+ elif args.core_name == "adafruit-samd":
+ install_adafruit_samd_core(install_prefix, install_dir, cache_dir)
+ elif args.core_name == "stm32duino":
+ install_stm32duino_core(install_prefix, install_dir, cache_dir)
+ elif args.core_name == "arduino-samd":
+ install_arduino_samd_core(install_prefix, install_dir, cache_dir)
+ else:
+ _LOG.error("Invalid core '%s'. Supported cores: %s", args.core_name,
+ ", ".join(supported_cores()))
+ sys.exit(1)
+
+
+def supported_cores():
+ return _ARDUINO_CORE_ARTIFACTS.keys()
+
+
+def install_teensy_core_windows(install_prefix, install_dir, cache_dir):
+ """Download and install Teensyduino artifacts for Windows."""
+ teensy_artifacts = _ARDUINO_CORE_ARTIFACTS["teensy"][platform.system()]
+
+ arduino_artifact = teensy_artifacts["arduino-ide"]
+ arduino_zipfile = file_operations.download_to_cache(
+ url=arduino_artifact["url"],
+ expected_sha256sum=arduino_artifact["sha256"],
+ cache_directory=cache_dir)
+
+ teensyduino_artifact = teensy_artifacts["teensyduino"]
+ teensyduino_installer = file_operations.download_to_cache(
+ url=teensyduino_artifact["url"],
+ expected_sha256sum=teensyduino_artifact["sha256"],
+ cache_directory=cache_dir)
+
+ file_operations.extract_archive(arduino_zipfile, install_dir, cache_dir)
+
+ # "teensy" here should match args.core_name
+ teensy_core_dir = os.path.join(install_prefix, "teensy")
+
+ # Change working directory for installation
+ original_working_dir = os.getcwd()
+ os.chdir(install_prefix)
+
+ _LOG.info("Installing Teensyduino to: %s", teensy_core_dir)
+
+ install_command = [teensyduino_installer, "--dir=teensy"]
+ _LOG.info(" Running: %s", " ".join(install_command))
+ _LOG.info(" Please click yes on the Windows 'User Account Control' "
+ "dialog.")
+ _LOG.info(" You should see: 'Verified publisher: PRJC.COM LLC'")
+
+ install_command = [teensyduino_installer, "--dir=teensy"]
+ subprocess.run(install_command)
+ if not os.path.exists(os.path.join(teensy_core_dir, "hardware", "teensy")):
+ _LOG.error(
+ "Error: Installation Failed.\n"
+ "Please try again and ensure Teensyduino is installed in "
+ "the folder:\n"
+ "%s", teensy_core_dir)
+ sys.exit(1)
+ else:
+ _LOG.info(" Install complete!")
+
+ file_operations.remove_empty_directories(install_dir)
+ os.chdir(original_working_dir)
+
+
+def install_teensy_core_mac(unused_install_prefix, install_dir, cache_dir):
+ """Download and install Teensyduino artifacts for Mac."""
+ teensy_artifacts = _ARDUINO_CORE_ARTIFACTS["teensy"][platform.system()]
+
+ teensyduino_artifact = teensy_artifacts["teensyduino"]
+ teensyduino_zip = file_operations.download_to_cache(
+ url=teensyduino_artifact["url"],
+ expected_sha256sum=teensyduino_artifact["sha256"],
+ cache_directory=cache_dir)
+
+ extracted_files = file_operations.extract_archive(
+ teensyduino_zip,
+ install_dir,
+ cache_dir,
+ remove_single_toplevel_folder=False)
+ toplevel_folder = sorted(extracted_files)[0]
+ os.symlink(os.path.join(toplevel_folder, "Contents", "Java", "hardware"),
+ os.path.join(install_dir, "hardware"),
+ target_is_directory=True)
+
+
+def install_teensy_core_linux(install_prefix, install_dir, cache_dir):
+ """Download and install Teensyduino artifacts for Windows."""
+ teensy_artifacts = _ARDUINO_CORE_ARTIFACTS["teensy"][platform.system()]
+
+ arduino_artifact = teensy_artifacts["arduino-ide"]
+ arduino_tarfile = file_operations.download_to_cache(
+ url=arduino_artifact["url"],
+ expected_sha256sum=arduino_artifact["sha256"],
+ cache_directory=cache_dir)
+
+ teensyduino_artifact = teensy_artifacts["teensyduino"]
+ teensyduino_installer = file_operations.download_to_cache(
+ url=teensyduino_artifact["url"],
+ expected_sha256sum=teensyduino_artifact["sha256"],
+ cache_directory=cache_dir)
+
+ extracted_files = file_operations.extract_archive(arduino_tarfile,
+ install_dir, cache_dir)
+ os.chmod(teensyduino_installer,
+ os.stat(teensyduino_installer).st_mode | stat.S_IEXEC)
+
+ original_working_dir = os.getcwd()
+ os.chdir(install_prefix)
+ # "teensy" here should match args.core_name
+ install_command = [teensyduino_installer, "--dir=teensy"]
+ subprocess.run(install_command)
+
+ # Remove original arduino IDE files
+ for efile in extracted_files:
+ if efile.is_file():
+ efile.unlink()
+
+ file_operations.remove_empty_directories(install_dir)
+ os.chdir(original_working_dir)
+
+
+def install_arduino_samd_core(install_prefix: str, install_dir: str,
+ cache_dir: str):
+ artifacts = _ARDUINO_CORE_ARTIFACTS["arduino-samd"]["all"]["core"]
+ core_tarfile = file_operations.download_to_cache(
+ url=artifacts["url"],
+ expected_sha256sum=artifacts["sha256"],
+ cache_directory=cache_dir)
+
+ package_path = os.path.join(install_dir, "hardware", "samd",
+ artifacts["version"])
+ os.makedirs(package_path, exist_ok=True)
+ file_operations.extract_archive(core_tarfile, package_path, cache_dir)
+ original_working_dir = os.getcwd()
+ os.chdir(install_prefix)
+ # TODO(tonymd): Fetch core/tools as specified by:
+ # http://downloads.arduino.cc/packages/package_index.json
+ os.chdir(original_working_dir)
+ return True
+
+
+def install_adafruit_samd_core(install_prefix: str, install_dir: str,
+ cache_dir: str):
+ artifacts = _ARDUINO_CORE_ARTIFACTS["adafruit-samd"]["all"]["core"]
+ core_tarfile = file_operations.download_to_cache(
+ url=artifacts["url"],
+ expected_sha256sum=artifacts["sha256"],
+ cache_directory=cache_dir)
+
+ package_path = os.path.join(install_dir, "hardware", "samd",
+ artifacts["version"])
+ os.makedirs(package_path, exist_ok=True)
+ file_operations.extract_archive(core_tarfile, package_path, cache_dir)
+
+ original_working_dir = os.getcwd()
+ os.chdir(install_prefix)
+ # TODO(tonymd): Fetch platform specific tools as specified by:
+ # https://adafruit.github.io/arduino-board-index/package_adafruit_index.json
+ # Specifically:
+ # https://github.com/ARM-software/CMSIS_5/archive/5.4.0.tar.gz
+ os.chdir(original_working_dir)
+ return True
+
+
+def install_stm32duino_core(install_prefix, install_dir, cache_dir):
+ artifacts = _ARDUINO_CORE_ARTIFACTS["stm32duino"]["all"]["core"]
+ core_tarfile = file_operations.download_to_cache(
+ url=artifacts["url"],
+ expected_sha256sum=artifacts["sha256"],
+ cache_directory=cache_dir)
+
+ package_path = os.path.join(install_dir, "hardware", "stm32",
+ artifacts["version"])
+ os.makedirs(package_path, exist_ok=True)
+ file_operations.extract_archive(core_tarfile, package_path, cache_dir)
+ original_working_dir = os.getcwd()
+ os.chdir(install_prefix)
+ # TODO(tonymd): Fetch platform specific tools as specified by:
+ # https://github.com/stm32duino/BoardManagerFiles/raw/master/STM32/package_stm_index.json
+ os.chdir(original_working_dir)
+ return True
diff --git a/pw_arduino_build/py/pw_arduino_build/file_operations.py b/pw_arduino_build/py/pw_arduino_build/file_operations.py
new file mode 100644
index 0000000..2bcdb69
--- /dev/null
+++ b/pw_arduino_build/py/pw_arduino_build/file_operations.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""File Helper Functions."""
+
+import glob
+import hashlib
+import logging
+import os
+import shutil
+import sys
+import tarfile
+import urllib.request
+import zipfile
+from pathlib import Path
+from typing import List
+
+_LOG = logging.getLogger(__name__)
+
+
+class InvalidChecksumError(Exception):
+ pass
+
+
+def find_files(starting_dir: str,
+ patterns: List[str],
+ directories_only=False) -> List[str]:
+ original_working_dir = os.getcwd()
+ if not (os.path.exists(starting_dir) and os.path.isdir(starting_dir)):
+ _LOG.error("Directory '%s' does not exist.", starting_dir)
+ raise FileNotFoundError
+
+ os.chdir(starting_dir)
+ files = []
+ for pattern in patterns:
+ for file_path in glob.glob(pattern, recursive=True):
+ if not directories_only or (directories_only
+ and os.path.isdir(file_path)):
+ files.append(file_path)
+ os.chdir(original_working_dir)
+ return sorted(files)
+
+
+def sha256_sum(file_name):
+ hash_sha256 = hashlib.sha256()
+ with open(file_name, "rb") as file_handle:
+ for chunk in iter(lambda: file_handle.read(4096), b""):
+ hash_sha256.update(chunk)
+ return hash_sha256.hexdigest()
+
+
+def md5_sum(file_name):
+ hash_md5 = hashlib.md5()
+ with open(file_name, "rb") as file_handle:
+ for chunk in iter(lambda: file_handle.read(4096), b""):
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+
+
+def verify_file_checksum(file_path,
+ expected_checksum,
+ sum_function=sha256_sum):
+ downloaded_checksum = sum_function(file_path)
+ try:
+ if downloaded_checksum != expected_checksum:
+ raise InvalidChecksumError
+ except InvalidChecksumError:
+ _LOG.exception("Invalid %s\n"
+ "%s %s\n"
+ "%s (expected)",
+ sum_function.__name__, downloaded_checksum,
+ os.path.basename(file_path), expected_checksum)
+ # Exit to stop installation
+ return sys.exit(1)
+
+ _LOG.info(" %s:", sum_function.__name__)
+ _LOG.info(" %s %s", downloaded_checksum, os.path.basename(file_path))
+ return True
+
+
+def download_to_cache(url: str,
+ expected_md5sum=None,
+ expected_sha256sum=None,
+ cache_directory=".cache") -> str:
+
+ cache_dir = os.path.realpath(
+ os.path.expanduser(os.path.expandvars(cache_directory)))
+ downloaded_file = os.path.join(cache_dir, url.split("/")[-1])
+
+ if not os.path.exists(downloaded_file):
+ _LOG.info("Downloading: %s", url)
+ urllib.request.urlretrieve(url, filename=downloaded_file)
+
+ if os.path.exists(downloaded_file):
+ _LOG.info("Downloaded: %s", downloaded_file)
+ if expected_sha256sum:
+ verify_file_checksum(downloaded_file,
+ expected_sha256sum,
+ sum_function=sha256_sum)
+ elif expected_md5sum:
+ verify_file_checksum(downloaded_file,
+ expected_md5sum,
+ sum_function=md5_sum)
+
+ return downloaded_file
+
+
+def extract_zipfile(archive_file: str, dest_dir: str):
+ with zipfile.ZipFile(archive_file) as archive:
+ archive.extractall(path=dest_dir)
+
+
+def extract_tarfile(archive_file: str, dest_dir: str):
+ with tarfile.open(archive_file, 'r') as archive:
+ archive.extractall(path=dest_dir)
+
+
+def extract_archive(archive_file: str,
+ dest_dir: str,
+ cache_dir: str,
+ remove_single_toplevel_folder=True):
+ """Extract a tar or zip file.
+
+ Args:
+ archive_file (str): Absolute path to the archive file.
+ dest_dir (str): Extraction destination directory.
+ cache_dir (str): Directory where temp files can be created.
+ remove_single_toplevel_folder (bool): If the archive contains only a
+ single folder move the contents of that into the destination
+ directory.
+ """
+ # Make a temporary directory to extract files into
+ temp_extract_dir = os.path.join(cache_dir,
+ "." + os.path.basename(archive_file))
+ os.makedirs(temp_extract_dir, exist_ok=True)
+
+ _LOG.info("Extracting: %s", archive_file)
+ if zipfile.is_zipfile(archive_file):
+ extract_zipfile(archive_file, temp_extract_dir)
+ elif tarfile.is_tarfile(archive_file):
+ extract_tarfile(archive_file, temp_extract_dir)
+ else:
+ _LOG.error("Unknown archive format: %s", archive_file)
+ return sys.exit(1)
+
+ _LOG.info("Installing into: %s", dest_dir)
+ path_to_extracted_files = temp_extract_dir
+
+ extracted_top_level_files = os.listdir(temp_extract_dir)
+ # Check if tarfile has only one folder
+ # If yes, make that the new path_to_extracted_files
+ if remove_single_toplevel_folder and len(extracted_top_level_files) == 1:
+ path_to_extracted_files = os.path.join(temp_extract_dir,
+ extracted_top_level_files[0])
+
+ # Move extracted files to dest_dir
+ extracted_files = os.listdir(path_to_extracted_files)
+ for file_name in extracted_files:
+ source_file = os.path.join(path_to_extracted_files, file_name)
+ dest_file = os.path.join(dest_dir, file_name)
+ shutil.move(source_file, dest_file)
+
+ # rm -rf temp_extract_dir
+ shutil.rmtree(temp_extract_dir, ignore_errors=True)
+
+ # Return List of extracted files
+ return list(Path(dest_dir).rglob("*"))
+
+
+def remove_empty_directories(directory):
+ """Recursively remove empty directories."""
+
+ for path in sorted(Path(directory).rglob("*"), reverse=True):
+ # If broken symlink
+ if path.is_symlink() and not path.exists():
+ path.unlink()
+ # if empty directory
+ elif path.is_dir() and len(os.listdir(path)) == 0:
+ path.rmdir()
diff --git a/pw_arduino_build/py/pw_arduino_build/log.py b/pw_arduino_build/py/pw_arduino_build/log.py
new file mode 100644
index 0000000..d5ed9f7
--- /dev/null
+++ b/pw_arduino_build/py/pw_arduino_build/log.py
@@ -0,0 +1,39 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Configure the system logger for the default pw command log format."""
+
+import logging
+
+_LOG = logging.getLogger(__name__)
+_STDERR_HANDLER = logging.StreamHandler()
+
+
+def install(level: int = logging.INFO) -> None:
+ """Configure the system logger for the arduino_builder log format."""
+
+ # Set log level on root logger to debug, otherwise any higher levels
+ # elsewhere are ignored.
+ root = logging.getLogger()
+ root.setLevel(logging.DEBUG)
+
+ _STDERR_HANDLER.setLevel(level)
+ _STDERR_HANDLER.setFormatter(
+ logging.Formatter("[%(asctime)s] "
+ "%(levelname)s %(message)s", "%Y%m%d %H:%M:%S"))
+ root.addHandler(_STDERR_HANDLER)
+
+
+def set_level(log_level: int):
+ """Sets the log level for logs to stderr."""
+ _STDERR_HANDLER.setLevel(log_level)
diff --git a/pw_arduino_build/py/setup.py b/pw_arduino_build/py/setup.py
index 5c47295..1cb074f 100644
--- a/pw_arduino_build/py/setup.py
+++ b/pw_arduino_build/py/setup.py
@@ -24,8 +24,7 @@
packages=setuptools.find_packages(),
entry_points={
'console_scripts':
- ['arduino_builder = '
- ' pw_arduino_build.arduinobuilder:main']
+ ['arduino_builder = pw_arduino_build.__main__:main']
},
install_requires=[
'pyserial',
diff --git a/targets/arduino/BUILD.gn b/targets/arduino/BUILD.gn
index 7459ba8..93939d0 100644
--- a/targets/arduino/BUILD.gn
+++ b/targets/arduino/BUILD.gn
@@ -31,32 +31,32 @@
config("arduino_build") {
# Debug: Print out arduinobuilder.py args
- # print(string_join(" ", [rebase_path(arduino_builder_script)] + arduino_global_args))
+ # print(string_join(" ", [rebase_path(arduino_builder_script)] + arduino_show_command_args))
# Run prebuilds
# TODO(tonymd) This only needs to be run once but it's happening multiple times.
exec_script(arduino_builder_script,
- arduino_global_args + [ "--run-prebuilds" ],
+ arduino_run_command_args + [ "--run-prebuilds" ],
"string")
_exclude_flags = [ "-std=gnu++14" ]
_cflags = exec_script(arduino_builder_script,
- arduino_global_args + [ "--c-flags" ],
+ arduino_show_command_args + [ "--c-flags" ],
"list lines")
cflags = filter_exclude(_cflags, _exclude_flags)
asmflags = exec_script(arduino_builder_script,
- arduino_global_args + [ "--s-flags" ],
+ arduino_show_command_args + [ "--s-flags" ],
"list lines")
_cflags_cc = exec_script(arduino_builder_script,
- arduino_global_args + [ "--cpp-flags" ],
+ arduino_show_command_args + [ "--cpp-flags" ],
"list lines")
cflags_cc = filter_exclude(_cflags_cc, _exclude_flags)
_ldflags = exec_script(arduino_builder_script,
- arduino_global_args + [ "--ld-flags" ],
+ arduino_show_command_args + [ "--ld-flags" ],
"list lines")
ldflags =
filter_exclude(_ldflags,
diff --git a/targets/arduino/target_docs.rst b/targets/arduino/target_docs.rst
index a583539..8b91b4e 100644
--- a/targets/arduino/target_docs.rst
+++ b/targets/arduino/target_docs.rst
@@ -140,21 +140,20 @@
ninja -C out arduino
SERIAL_PORT=/dev/ttyACM0
+
for f in $(find out/arduino_debug/obj/ -iname "*.elf"); do
BUILD_PATH=$(dirname $f)
PROJECT_NAME=$(basename -s .elf $f)
- COMMON_ARGS="--quiet --arduino-package-path ./third_party/arduino/cores/teensy
- --arduino-package-name teensy/avr
- --compiler-path-override ./.environment/cipd/pigweed/bin
- show
- --build-path ${BUILD_PATH} --build-project-name ${PROJECT_NAME}
- --board teensy40 --menu-options menu.usb.serial menu.keys.en-us"
- echo "==> OBJCOPY" $f
- arduino_builder $COMMON_ARGS --run-objcopy
- # Optional
- # arduino_builder $COMMON_ARGS --run-postbuild
- echo "==> FLASH" $f
- arduino_builder --serial-port $SERIAL_PORT $COMMON_ARGS --run-upload-command teensyloader
+ CORE_ARGS="--quiet --arduino-package-path ./third_party/arduino/cores/teensy
+ --arduino-package-name teensy/avr
+ --compiler-path-override ./.environment/cipd/pigweed/bin"
+ BOARD_ARGS="--build-path ${BUILD_PATH} \
+ --build-project-name ${PROJECT_NAME}
+ --board teensy40
+ --menu-options menu.usb.serial menu.keys.en-us"
+ # Run objcopy, postbuild, and upload (flash) steps
+ arduino_builder $CORE_ARGS run $BOARD_ARGS --serial-port $SERIAL_PORT \
+ --run-objcopy --run-postbuild --run-upload-command teensyloader
while true; do
sleep .1; ls $SERIAL_PORT 2>/dev/null && break
done
diff --git a/third_party/arduino/BUILD.gn b/third_party/arduino/BUILD.gn
index d0adc06..7894f35 100644
--- a/third_party/arduino/BUILD.gn
+++ b/third_party/arduino/BUILD.gn
@@ -21,26 +21,34 @@
pw_source_set("arduino_core_sources") {
remove_configs = [ "$dir_pw_build:strict_warnings" ]
- _core_c_files = exec_script(arduino_builder_script,
- arduino_global_args + [ "--core-c-files" ],
- "list lines")
- _core_s_files = exec_script(arduino_builder_script,
- arduino_global_args + [ "--core-s-files" ],
- "list lines")
- _core_cpp_files = exec_script(arduino_builder_script,
- arduino_global_args + [ "--core-cpp-files" ],
- "list lines")
+ _core_c_files =
+ exec_script(arduino_builder_script,
+ arduino_show_command_args + [ "--core-c-files" ],
+ "list lines")
+ _core_s_files = filter_exclude(
+ exec_script(arduino_builder_script,
+ arduino_show_command_args + [ "--core-s-files" ],
+ "list lines"),
+ # TODO(tonymd): Conditionally remove this source file unless building for cortex-m0.
+ # Exception for adafruit-samd core
+ # pulse_asm.S is for: '.cpu cortex-m0plus .fpu softvfp'
+ [ "*pulse_asm.S\b" ])
+
+ _core_cpp_files =
+ exec_script(arduino_builder_script,
+ arduino_show_command_args + [ "--core-cpp-files" ],
+ "list lines")
_variant_c_files =
exec_script(arduino_builder_script,
- arduino_global_args + [ "--variant-c-files" ],
+ arduino_show_command_args + [ "--variant-c-files" ],
"list lines")
_variant_s_files =
exec_script(arduino_builder_script,
- arduino_global_args + [ "--variant-s-files" ],
+ arduino_show_command_args + [ "--variant-s-files" ],
"list lines")
_variant_cpp_files =
exec_script(arduino_builder_script,
- arduino_global_args + [ "--variant-cpp-files" ],
+ arduino_show_command_args + [ "--variant-cpp-files" ],
"list lines")
sources = _core_c_files + _core_s_files + _core_cpp_files +
