pw_env_setup: Explicitly list CIPD platforms
Explicitly list CIPD package platforms in JSON files. This makes it
easier to follow which packages are loaded for each platform and lets us
be clearer about which platforms are supported.
This will also simplify roller configs because the rollers will no
longer need to list platforms in builder properties. Instead, they can
be processed from the JSON files. As a result there should be fewer
issues where rollers are misconfigured.
Roller recipe and config CLs will follow. Since recipe environment setup
now uses pw_env_setup no changes need to be made to it to support this.
Change-Id: I80db2d8926b259c880887765d110f287bc7a5ed3
Bug: 385
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/46180
Reviewed-by: Wyatt Hepler <hepler@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst
index e6d9fb7..06e3295 100644
--- a/pw_env_setup/docs.rst
+++ b/pw_env_setup/docs.rst
@@ -167,8 +167,23 @@
.. _sample project: https://pigweed.googlesource.com/pigweed/sample_project/+/master
``cipd_package_files``
- CIPD package file. JSON file consisting of a list of dictionaries with "path"
- and "tags" keys, where "tags" is a list of strings.
+ CIPD package file. JSON file consisting of a list of dictionaries with "path",
+ "platforms", and "tags" keys. An example is below.
+
+.. code-block:: json
+
+ {
+ "path": "infra/3pp/tools/go/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
+ "tags": [
+ "version:2@1.16.3"
+ ]
+ },
``virtualenv.gn_targets``
Target for installing Python packages. Downstream projects will need to
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
index 240a571..ef62430 100644
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
@@ -1,42 +1,83 @@
[
{
"path": "infra/tools/bb/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
},
{
"path": "infra/tools/luci-auth/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
},
{
"path": "infra/tools/luci/gerrit/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
},
{
"path": "infra/tools/luci/gitiles/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
},
{
"path": "infra/tools/luci/cas/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
},
{
"path": "infra/tools/luci/led/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
},
{
- "path": "infra/tools/luci/logdog/logdog/${os}-${arch=amd64}",
+ "path": "infra/tools/luci/logdog/logdog/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"latest"
]
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
index 2ca677c..0eb60f0 100644
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
@@ -1,120 +1,207 @@
[
{
- "path": "gn/gn/${os}-${arch=amd64,arm64}",
+ "path": "gn/gn/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"git_revision:39a87c0b36310bdf06b692c098f199a0d97fc810"
]
},
{
- "path": "infra/ninja/${os}-${arch=amd64}",
+ "path": "infra/ninja/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:1.9.0"
]
},
{
- "path": "infra/cmake/${os}-${arch=amd64}",
+ "path": "infra/cmake/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:3.16.1"
]
},
{
"_comment": "TODO(pwbug/93): Package Bazel for Windows.",
- "path": "fuchsia/third_party/bazel/${os=linux,mac}-${arch=amd64}",
+ "path": "fuchsia/third_party/bazel/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64"
+ ],
"tags": [
"version:4.0.0"
]
},
{
- "path": "pigweed/third_party/bloaty-embedded/${os=linux,mac}-${arch=amd64}",
+ "path": "pigweed/third_party/bloaty-embedded/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64"
+ ],
"tags": [
"git_revision:2d87d204057b419f5290f8d38b61b9c2c5b4fb52-2"
]
},
{
- "path": "pigweed/third_party/gcc-arm-none-eabi/${os}-${arch=amd64}",
+ "path": "pigweed/third_party/gcc-arm-none-eabi/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:10-2020-q4-major"
]
},
{
- "path": "infra/3pp/tools/protoc/${os}-${arch}",
+ "path": "infra/3pp/tools/protoc/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:3.14.0"
]
},
{
"_comment": "TODO(pwbug/70) Put clang in cipd for Windows.",
- "path": "fuchsia/third_party/clang/${os=linux,mac}-${arch}",
+ "path": "fuchsia/third_party/clang/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64"
+ ],
"tags": [
"git_revision:3dc24bc31edbc01dea085b24a6a6b024d7ae531c"
]
},
{
"_comment": "When bumping the minor version (e.g., to 3.9.x) also update env_setup/virtualenv/init.py to check for the new version.",
- "path": "infra/3pp/tools/cpython3/${os}-${arch}",
+ "path": "infra/3pp/tools/cpython3/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:3.8.2.chromium.10"
]
},
{
"path": "infra/3pp/tools/go/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:2@1.16.3"
]
},
{
- "path": "pigweed/third_party/protoc-gen-go/${os}-${arch=amd64}",
+ "path": "pigweed/third_party/protoc-gen-go/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"version:1.3.2"
]
},
{
"_comment": "TODO(pwbug/66) Put openocd in cipd for Windows.",
- "path": "pigweed/third_party/openocd/${os=linux,mac}-${arch=amd64}",
+ "path": "pigweed/third_party/openocd/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64"
+ ],
"tags": [
"git_revision:e41c0f4906e46d1076ce62a0da5518aa1ca280b8"
]
},
{
- "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${os=windows}-${arch=amd64}",
+ "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${platform}",
+ "platforms": [
+ "windows-amd64"
+ ],
"tags": [
"version:10.2.0-11"
]
},
{
- "path": "pigweed/host_tools/${os}-${arch=amd64}",
+ "path": "pigweed/host_tools/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "mac-amd64",
+ "windows-amd64"
+ ],
"tags": [
"git_revision:3f00a3eb4957306eab148fe3651844e46df5b379"
]
},
{
- "path": "infra/rbe/client/${os=linux,windows}-${arch=amd64}",
+ "path": "infra/rbe/client/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "windows-amd64"
+ ],
"tags": [
"git_revision:8e834982e020c12b7c3c1f106961cead3d65dec1"
]
},
{
- "path": "fuchsia/third_party/qemu/${os=linux,mac}-${arch}",
+ "path": "fuchsia/third_party/qemu/${platform}",
+ "platforms": [
+ "linux-amd64",
+ "linux-arm64",
+ "mac-amd64"
+ ],
"tags": [
"git_revision:841f14e74f5af7886cf49cfcd4fed264911ae58e"
]
},
{
"path": "fuchsia/third_party/kythe",
+ "platforms": [
+ "linux-amd64"
+ ],
"subdir": "kythe",
"tags": [
"version:1.0.2"
]
},
{
- "path": "fuchsia/third_party/kythe-libs/${os=linux}-${arch=amd64}",
+ "path": "fuchsia/third_party/kythe-libs/${platform}",
+ "platforms": [
+ "linux-amd64"
+ ],
"subdir": "kythe",
"tags": [
"version:2020-08-05"
]
},
{
- "path": "infra/3pp/tools/renode/${os=linux}-${arch=amd64}",
+ "path": "infra/3pp/tools/renode/${platform}",
+ "platforms": [
+ "linux-amd64"
+ ],
"tags": [
"version:2@renode-1.11.0+20210306gite7897c1"
]
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
index c4301ec..922d589 100755
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
@@ -24,6 +24,7 @@
import argparse
import json
import os
+import platform
import re
import shutil
import subprocess
@@ -148,9 +149,28 @@
return True
+def _platform():
+ osname = {
+ 'darwin': 'mac',
+ 'linux': 'linux',
+ 'windows': 'windows',
+ }[platform.system().lower()]
+
+ if platform.machine().startswith(('aarch64', 'armv8')):
+ arch = 'arm64'
+ elif platform.machine() == 'x86_64':
+ arch = 'amd64'
+ elif platform.machine() == 'i686':
+ arch = 'i386'
+ else:
+ arch = platform.machine()
+
+ return '{}-{}'.format(osname, arch).lower()
+
+
def write_ensure_file(package_file, ensure_file):
with open(package_file, 'r') as ins:
- data = json.load(ins)
+ packages = json.load(ins)
# TODO(pwbug/103) Remove 30 days after bug fixed.
if os.path.isdir(ensure_file):
@@ -161,10 +181,14 @@
'$VerifiedPlatform mac-amd64\n'
'$ParanoidMode CheckPresence\n')
- for entry in data:
- outs.write('@Subdir {}\n'.format(entry.get('subdir', '')))
- outs.write('{} {}\n'.format(entry['path'],
- ' '.join(entry['tags'])))
+ for pkg in packages:
+ # If this is a new-style package manifest platform handling must
+ # be done here instead of by the cipd executable.
+ if 'platforms' in pkg and _platform() not in pkg['platforms']:
+ continue
+
+ outs.write('@Subdir {}\n'.format(pkg.get('subdir', '')))
+ outs.write('{} {}\n'.format(pkg['path'], ' '.join(pkg['tags'])))
def update(
