pw_env_setup: Skip downloading overridden packages (reland)
This is a reland of commit fa1dc6208a48e844ddf52c7e8252331ada9ced9a
dict.values() returns a thin wrapper of a dictionary in Python 3 and an
iterator in Python 2. The wrapper from Python 3 can be passed into
reversed(), but the iterator cannot. Explicitly convert this to a list
so it can be passed into reversed() no matter the Python version.
Original change's description:
> pw_env_setup: Skip downloading overridden packages
>
> Bug: b/246971220
> Change-Id: I31db4cad2519289ecc80e87448536a46a279dfd6
Bug: b/246971220
Change-Id: I61fd4847d73dc924c71824fae5a7686465fb8c6d
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/112172
Reviewed-by: Wyatt Hepler <hepler@google.com>
Reviewed-by: Erick Pfeifer <erickpfeifer@google.com>
Commit-Queue: Erick Pfeifer <erickpfeifer@google.com>
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
index 73140a4..5659130 100755
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
@@ -21,6 +21,7 @@
from __future__ import print_function
+import collections
import hashlib
import json
import os
@@ -191,9 +192,8 @@
return result
-def write_ensure_file(package_files, ensure_file, platform): # pylint: disable=redefined-outer-name
+def all_packages(package_files):
packages = []
-
for package_file in package_files:
name = package_file_name(package_file)
with open(package_file, 'r') as ins:
@@ -204,13 +204,28 @@
else:
package['subdir'] = name
packages.extend(file_packages)
+ return packages
+
+
+def deduplicate_packages(packages):
+ deduped = collections.OrderedDict()
+ for package in reversed(packages):
+ if package['path'] in deduped:
+ del deduped[package['path']]
+ deduped[package['path']] = package
+ return reversed(list(deduped.values()))
+
+
+def write_ensure_file(package_files, ensure_file, platform): # pylint: disable=redefined-outer-name
+ packages = all_packages(package_files)
+ deduped_packages = deduplicate_packages(packages)
with open(ensure_file, 'w') as outs:
outs.write('$VerifiedPlatform linux-amd64\n'
'$VerifiedPlatform mac-amd64\n'
'$ParanoidMode CheckPresence\n')
- for pkg in packages:
+ for pkg in deduped_packages:
# If this is a new-style package manifest platform handling must
# be done here instead of by the cipd executable.
if 'platforms' in pkg and platform not in pkg['platforms']:
