| # Copyright (c) 2024 TOKITA Hiroshi |
| # |
| # SPDX-License-Identifier: Apache-2.0 |
| |
| import argparse |
| import hashlib |
| import os |
| import patoolib |
| import platform |
| import re |
| import requests |
| import semver |
| import shutil |
| import subprocess |
| import tempfile |
| import textwrap |
| import zcmake |
| from pathlib import Path |
| |
| from west.commands import WestCommand |
| |
| |
| class Sdk(WestCommand): |
| def __init__(self): |
| super().__init__( |
| "sdk", |
| "manage Zephyr SDK", |
| "List and Install Zephyr SDK", |
| ) |
| |
| def do_add_parser(self, parser_adder): |
| parser = parser_adder.add_parser( |
| self.name, |
| help=self.help, |
| description=self.description, |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| epilog=textwrap.dedent( |
| """ |
| Listing SDKs: |
| |
| Run 'west sdk' or 'west sdk list' to list installed SDKs. |
| See 'west sdk list --help' for details. |
| |
| |
| Installing SDK: |
| |
| Run 'west sdk install' to install Zephyr SDK. |
| See 'west sdk install --help' for details. |
| """ |
| ), |
| ) |
| |
| subparsers_gen = parser.add_subparsers( |
| metavar="<subcommand>", |
| dest="subcommand", |
| help="select a subcommand. If omitted, treat it as the 'list' selected.", |
| ) |
| |
| subparsers_gen.add_parser( |
| "list", |
| help="list installed Zephyr SDKs", |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| epilog=textwrap.dedent( |
| """ |
| Listing SDKs: |
| |
| Run 'west sdk' or 'west sdk list' command information about available SDKs is displayed. |
| """ |
| ), |
| ) |
| |
| install_args_parser = subparsers_gen.add_parser( |
| "install", |
| help="install Zephyr SDK", |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| epilog=textwrap.dedent( |
| """ |
| Installing SDK: |
| |
| Run 'west sdk install' to install Zephyr SDK. |
| |
| Set --version option to install a specific version of the SDK. |
| If not specified, the install version is detected from "${ZEPHYR_BASE}/SDK_VERSION file. |
| SDKs older than 0.14.1 are not supported. |
| |
| You can specify the installation directory with --install-dir or --install-base. |
| If the specified version of the SDK is already installed, |
| the already installed SDK will be used regardless of the settings of |
| --install-dir and --install-base. |
| |
| Typically, Zephyr SDK archives contain only one directory named zephyr-sdk-<version> |
| at the top level. |
| The SDK archive is extracted to the home directory if both --install-dir and --install-base |
| are not specified. |
| In this case, SDK will install into ${HOME}/zephyr-sdk-<version>. |
| If --install-base is specified, the archive will be extracted under the specified path. |
| In this case, SDK will install into <BASE>/zephyr-sdk-<version> . |
| If --install-dir is specified, the directory contained in the archive will be renamed |
| and placed to the specified location. |
| |
| --interactive, --toolchains, --no-toolchains and --no-hosttools options |
| specify the behavior of the installer. Please see the description of each option. |
| |
| --personal-access-token specifies the GitHub personal access token. |
| This helps to relax the limits on the number of REST API calls. |
| |
| --api-url specifies the REST API endpoint for GitHub releases information |
| when installing the SDK from a different GitHub repository. |
| """ |
| ), |
| ) |
| |
| install_args_parser.add_argument( |
| "--version", |
| default=None, |
| nargs="?", |
| metavar="SDK_VER", |
| help="version of the Zephyr SDK to install. " |
| "If not specified, the install version is detected from " |
| "${ZEPHYR_BASE}/SDK_VERSION file.", |
| ) |
| install_args_parser.add_argument( |
| "-b", |
| "--install-base", |
| default=None, |
| metavar="BASE", |
| help="Base directory to SDK install. " |
| "The subdirectory created by extracting the archive in <BASE> will be the SDK installation directory. " |
| "For example, -b /foo/bar will install the SDK in `/foo/bar/zephyr-sdk-<version>'." |
| ) |
| install_args_parser.add_argument( |
| "-d", |
| "--install-dir", |
| default=None, |
| metavar="DIR", |
| help="SDK install destination directory. " |
| "The SDK will be installed on the specified path. " |
| "The directory contained in the archive will be renamed and installed for the specified directory. " |
| "For example, if you specify -b /foo/bar/baz, The archive's zephyr-sdk-<version> directory will be renamed baz and placed under /foo/bar. " |
| "If this option is specified, the --install-base option is ignored. " |
| ) |
| install_args_parser.add_argument( |
| "-i", |
| "--interactive", |
| action="store_true", |
| help="launches installer in interactive mode. " |
| "--toolchains, --no-toolchains and --no-hosttools will be ignored if this option is enabled.", |
| ) |
| install_args_parser.add_argument( |
| "-t", |
| "--toolchains", |
| metavar="toolchain_name", |
| nargs="+", |
| help="toolchain(s) to install (e.g. 'arm-zephyr-eabi'). " |
| "If this option is not given, toolchains for all architectures will be installed.", |
| ) |
| install_args_parser.add_argument( |
| "-T", |
| "--no-toolchains", |
| action="store_true", |
| help="do not install toolchains. " |
| "--toolchains will be ignored if this option is enabled.", |
| ) |
| install_args_parser.add_argument( |
| "-H", |
| "--no-hosttools", |
| action="store_true", |
| help="do not install host-tools.", |
| ) |
| install_args_parser.add_argument( |
| "--personal-access-token", help="GitHub personal access token." |
| ) |
| install_args_parser.add_argument( |
| "--api-url", |
| default="https://api.github.com/repos/zephyrproject-rtos/sdk-ng/releases", |
| help="GitHub releases API endpoint used to look for Zephyr SDKs.", |
| ) |
| |
| return parser |
| |
| def os_arch_name(self): |
| system = platform.system() |
| machine = platform.machine() |
| |
| if system == "Linux": |
| osname = "linux" |
| elif system == "Darwin": |
| osname = "macos" |
| elif system == "Windows": |
| osname = "windows" |
| else: |
| self.die(f"Unsupported system: {system}") |
| |
| if machine in ["aarch64", "arm64"]: |
| arch = "aarch64" |
| elif machine in ["x86_64", "AMD64"]: |
| arch = "x86_64" |
| else: |
| self.die(f"Unsupported machine: {machine}") |
| |
| return (osname, arch) |
| |
| def detect_version(self, args): |
| if args.version: |
| version = args.version |
| else: |
| if os.environ["ZEPHYR_BASE"]: |
| zephyr_base = Path(os.environ["ZEPHYR_BASE"]) |
| else: |
| zephyr_base = Path(__file__).parents[2] |
| |
| sdk_version_file = zephyr_base / "SDK_VERSION" |
| |
| if not sdk_version_file.exists(): |
| self.die(f"{str(sdk_version_file)} does not exist.") |
| |
| with open(sdk_version_file) as f: |
| version = f.readlines()[0].strip() |
| self.inf( |
| f"Found '{str(sdk_version_file)}', installing version {version}." |
| ) |
| |
| try: |
| semver.Version.parse(version) |
| except Exception: |
| self.die(f"Invalid version format: {version}") |
| |
| if semver.compare(version, "0.14.1") < 0: |
| self.die(f"Versions older than v0.14.1 are not supported.") |
| |
| return version |
| |
| def fetch_releases(self, url, req_headers): |
| """fetch releases data via GitHub REST API""" |
| |
| releases = [] |
| page = 1 |
| |
| while True: |
| params = {"page": page, "per_page": 100} |
| resp = requests.get(url, headers=req_headers, params=params) |
| if resp.status_code != 200: |
| raise Exception(f"Failed to fetch: {resp.status_code}, {resp.text}") |
| |
| data = resp.json() |
| if not data: |
| break |
| |
| releases.extend(data) |
| page += 1 |
| |
| return releases |
| |
| def minimal_sdk_filename(self, release): |
| (osname, arch) = self.os_arch_name() |
| version = re.sub(r"^v", "", release["tag_name"]) |
| |
| if semver.compare(version, "0.16.0") < 0: |
| if osname == "windows": |
| ext = ".zip" |
| else: |
| ext = ".tar.gz" |
| else: |
| if osname == "windows": |
| ext = ".7z" |
| else: |
| ext = ".tar.xz" |
| |
| return f"zephyr-sdk-{version}_{osname}-{arch}_minimal{ext}" |
| |
| def minimal_sdk_sha256(self, sha256_list, release): |
| name = self.minimal_sdk_filename(release) |
| tuples = [(re.split(r"\s+", t)) for t in sha256_list.splitlines()] |
| hashtable = {t[1]: t[0] for t in tuples} |
| |
| return hashtable[name] |
| |
| def minimal_sdk_url(self, release): |
| name = self.minimal_sdk_filename(release) |
| assets = release.get("assets", []) |
| minimal_sdk_asset = next(filter(lambda x: x["name"] == name, assets)) |
| |
| return minimal_sdk_asset["browser_download_url"] |
| |
| def sha256_sum_url(self, release): |
| assets = release.get("assets", []) |
| minimal_sdk_asset = next(filter(lambda x: x["name"] == "sha256.sum", assets)) |
| |
| return minimal_sdk_asset["browser_download_url"] |
| |
| def download_and_extract(self, base_dir, dir_name, target_release, req_headers): |
| self.inf("Fetching sha256...") |
| sha256_url = self.sha256_sum_url(target_release) |
| resp = requests.get(sha256_url, headers=req_headers, stream=True) |
| if resp.status_code != 200: |
| raise Exception(f"Failed to download {sha256_url}: {resp.status_code}") |
| |
| sha256 = self.minimal_sdk_sha256(resp.content.decode("UTF-8"), target_release) |
| |
| archive_url = self.minimal_sdk_url(target_release) |
| self.inf(f"Downloading {archive_url}...") |
| resp = requests.get(archive_url, headers=req_headers, stream=True) |
| if resp.status_code != 200: |
| raise Exception(f"Failed to download {archive_url}: {resp.status_code}") |
| |
| try: |
| Path(base_dir).mkdir(parents=True, exist_ok=True) |
| |
| with tempfile.TemporaryDirectory(dir=base_dir) as tempdir: |
| # download archive file |
| filename = Path(tempdir) / re.sub(r"^.*/", "", archive_url) |
| file = open(filename, mode="wb") |
| total_length = int(resp.headers["Content-Length"]) |
| count = 0 |
| |
| for chunk in resp.iter_content(chunk_size=8192): |
| file.write(chunk) |
| count = count + len(chunk) |
| self.inf(f"\r {count}/{total_length}", end="") |
| self.inf() |
| self.inf(f"Downloaded: {file.name}") |
| file.close() |
| |
| # check sha256 hash |
| with open(file.name, "rb") as sha256file: |
| digest = hashlib.sha256(sha256file.read()).hexdigest() |
| if sha256 != digest: |
| raise Exception(f"sha256 mismatched: {sha256}:{digest}") |
| |
| # extract archive file |
| self.inf(f"Extract: {file.name}") |
| patoolib.extract_archive(file.name, outdir=tempdir) |
| |
| # We expect that only the zephyr-sdk-x.y.z directory will be present in the archive. |
| extracted_dirs = [d for d in Path(tempdir).iterdir() if d.is_dir()] |
| if len(extracted_dirs) != 1: |
| raise Exception("Unexpected archive format") |
| |
| # move to destination dir |
| if dir_name: |
| dest_dir = Path(base_dir / dir_name) |
| else: |
| dest_dir = Path(base_dir / extracted_dirs[0].name) |
| |
| Path(dest_dir).parent.mkdir(parents=True, exist_ok=True) |
| |
| self.inf(f"Move: {str(extracted_dirs[0])} to {dest_dir}.") |
| shutil.move(extracted_dirs[0], dest_dir) |
| |
| return dest_dir |
| except PermissionError as pe: |
| self.die(pe) |
| |
| def run_setup(self, args, sdk_dir): |
| if "Windows" == platform.system(): |
| setup = Path(sdk_dir) / "setup.cmd" |
| optsep = "/" |
| else: |
| setup = Path(sdk_dir) / "setup.sh" |
| optsep = "-" |
| |
| # Associate installed SDK so that it can be found. |
| cmds_cmake_pkg = [str(setup), f"{optsep}c"] |
| self.dbg("Run: ", cmds_cmake_pkg) |
| result = subprocess.run(cmds_cmake_pkg) |
| if result.returncode != 0: |
| self.die(f"command \"{' '.join(cmds_cmake_pkg)}\" failed") |
| |
| cmds = [str(setup)] |
| |
| if not args.interactive and not args.no_toolchains: |
| if not args.toolchains: |
| cmds.extend([f"{optsep}t", "all"]) |
| else: |
| for tc in args.toolchains: |
| cmds.extend([f"{optsep}t", tc]) |
| |
| if not args.interactive and not args.no_hosttools: |
| cmds.extend([f"{optsep}h"]) |
| |
| if args.interactive or len(cmds) != 1: |
| self.dbg("Run: ", cmds) |
| result = subprocess.run(cmds) |
| if result.returncode != 0: |
| self.die(f"command \"{' '.join(cmds)}\" failed") |
| |
| def install_sdk(self, args, user_args): |
| version = self.detect_version(args) |
| (osname, arch) = self.os_arch_name() |
| |
| if args.personal_access_token: |
| req_headers = { |
| "Authorization": f"Bearer {args.personal_access_token}", |
| } |
| else: |
| req_headers = {} |
| |
| self.inf("Fetching Zephyr SDK list...") |
| releases = self.fetch_releases(args.api_url, req_headers) |
| self.dbg("releases: ", "\n".join([x["tag_name"] for x in releases])) |
| |
| # checking version |
| def check_semver(version): |
| try: |
| semver.Version.parse(version) |
| return True |
| except Exception: |
| return False |
| |
| available_versions = [ |
| re.sub(r"^v", "", x["tag_name"]) |
| for x in releases |
| if check_semver(re.sub(r"^v", "", x["tag_name"])) |
| ] |
| |
| if not version in available_versions: |
| self.die( |
| f"Unavailable SDK version: {version}." |
| + "Please select from the list below:\n" |
| + "\n".join(available_versions) |
| ) |
| |
| target_release = [x for x in releases if x["tag_name"] == f"v{version}"][0] |
| |
| # checking toolchains parameters |
| assets = target_release["assets"] |
| self.dbg("assets: ", "\n".join([x["browser_download_url"] for x in assets])) |
| |
| prefix = f"toolchain_{osname}-{arch}_" |
| available_toolchains = [ |
| re.sub(r"\..*", "", x["name"].replace(prefix, "")) |
| for x in assets |
| if x["name"].startswith(prefix) |
| ] |
| |
| if args.toolchains: |
| for tc in args.toolchains: |
| if not tc in available_toolchains: |
| self.die( |
| f"toolchain {tc} is not available.\n" |
| + "Please select from the list below:\n" |
| + "\n".join(available_toolchains) |
| ) |
| |
| installed_info = [v for (k, v) in self.fetch_sdk_info().items() if k == version] |
| if len(installed_info) == 0: |
| if args.install_dir: |
| base_dir = Path(args.install_dir).parent |
| dir_name = Path(args.install_dir).name |
| elif args.install_base: |
| base_dir = Path(args.install_base) |
| dir_name = None |
| else: |
| base_dir = Path("~").expanduser() |
| dir_name = None |
| |
| sdk_dir = self.download_and_extract( |
| base_dir, dir_name, target_release, req_headers |
| ) |
| else: |
| sdk_dir = Path(installed_info[0]["path"]) |
| self.inf( |
| f"Zephyr SDK version {version} is already installed at {str(sdk_dir)}. Using it." |
| ) |
| |
| self.run_setup(args, sdk_dir) |
| |
| def fetch_sdk_info(self): |
| sdk_lines = [] |
| try: |
| cmds = [ |
| "-P", |
| str(Path(__file__).parent / "sdk" / "listsdk.cmake"), |
| ] |
| |
| output = zcmake.run_cmake(cmds, capture_output=True) |
| if output: |
| # remove '-- Zephyr-sdk,' leader |
| sdk_lines = [l[15:] for l in output if l.startswith("-- Zephyr-sdk,")] |
| else: |
| sdk_lines = [] |
| |
| except Exception as e: |
| self.die(e) |
| |
| def parse_sdk_entry(line): |
| class SdkEntry: |
| def __init__(self): |
| self.version = None |
| self.path = None |
| |
| info = SdkEntry() |
| for ent in line.split(","): |
| kv = ent.split("=") |
| if kv[0].strip() == "ver": |
| info.version = kv[1].strip() |
| elif kv[0].strip() == "dir": |
| info.path = kv[1].strip() |
| |
| return info |
| |
| sdk_info = {} |
| for sdk_ent in [parse_sdk_entry(l) for l in reversed(sdk_lines)]: |
| entry = {} |
| |
| ver = None |
| sdk_path = Path(sdk_ent.path) |
| sdk_version_path = sdk_path / "sdk_version" |
| if sdk_version_path.exists(): |
| with open(str(sdk_version_path)) as f: |
| ver = f.readline().strip() |
| else: |
| continue |
| |
| entry["path"] = sdk_path |
| |
| if (sdk_path / "sysroots").exists(): |
| entry["hosttools"] = "installed" |
| |
| # Identify toolchain directory by the existence of <toolchain>/bin/<toolchain>-gcc |
| if "Windows" == platform.system(): |
| gcc_postfix = "-gcc.exe" |
| else: |
| gcc_postfix = "-gcc" |
| |
| toolchains = [ |
| tc.name |
| for tc in sdk_path.iterdir() |
| if (sdk_path / tc / "bin" / (tc.name + gcc_postfix)).exists() |
| ] |
| |
| if len(toolchains) > 0: |
| entry["toolchains"] = toolchains |
| |
| if ver: |
| sdk_info[ver] = entry |
| |
| return sdk_info |
| |
| def list_sdk(self): |
| sdk_info = self.fetch_sdk_info() |
| |
| if len(sdk_info) == 0: |
| self.die("No Zephyr SDK installed.") |
| |
| for k, v in sdk_info.items(): |
| self.inf(f"{k}:") |
| self.inf(f" path: {v['path']}") |
| if "hosttools" in v: |
| self.inf(f" hosttools: {v['hosttools']}") |
| if "toolchains" in v: |
| self.inf(" installed-toolchains:") |
| for tc in v["toolchains"]: |
| self.inf(f" - {tc}") |
| |
| # Since version 0.15.2, the sdk_toolchains file is included, |
| # so we can get information about available toolchains from there. |
| if (Path(v["path"]) / "sdk_toolchains").exists(): |
| with open(Path(v["path"]) / "sdk_toolchains") as f: |
| all_tcs = [l.strip() for l in f.readlines()] |
| |
| self.inf(" available-toolchains:") |
| for tc in all_tcs: |
| if tc not in v["toolchains"]: |
| self.inf(f" - {tc}") |
| |
| self.inf() |
| |
| def do_run(self, args, user_args): |
| self.dbg("args: ", args) |
| if args.subcommand == "install": |
| self.install_sdk(args, user_args) |
| elif args.subcommand == "list" or not args.subcommand: |
| self.list_sdk() |
