collect_deps: Initial commit Bug: b/522852278 Change-Id: I518cb7cc07805ce6e7a02589d99dfe12177b1325 Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/426875
diff --git a/recipes/collect_deps.proto b/recipes/collect_deps.proto new file mode 100644 index 0000000..a885782 --- /dev/null +++ b/recipes/collect_deps.proto
@@ -0,0 +1,22 @@ +// Copyright 2026 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +syntax = "proto3"; + +package recipes.pigweed.collect_deps; + +message InputProperties { + // LUCI project names to fetch config for. + repeated string projects = 1; +}
diff --git a/recipes/collect_deps.py b/recipes/collect_deps.py new file mode 100644 index 0000000..832d57d --- /dev/null +++ b/recipes/collect_deps.py
@@ -0,0 +1,392 @@ +# Copyright 2026 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Recipe to collect dependencies from downstream builders.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator + +from google.protobuf import json_format + +from recipe_engine import post_process, recipe_api, recipe_test_api + +from PB.go.chromium.org.luci.buildbucket.proto import ( + builder_common as builder_common_pb, +) +from PB.go.chromium.org.luci.buildbucket.proto import ( + builds_service as builds_service_pb, +) +from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb +from PB.go.chromium.org.luci.buildbucket.proto import ( + project_config as project_config_pb, +) +from PB.recipe_engine import result as result_pb +from PB.recipes.pigweed.collect_deps import InputProperties + +from RECIPE_MODULES.pigweed.raw_result import api as raw_result_api + +DEPS = [ + 'pigweed/raw_result', + 'recipe_engine/buildbucket', + 'recipe_engine/json', + 'recipe_engine/luci_config', + 'recipe_engine/properties', + 'recipe_engine/step', +] + +PROPERTIES = InputProperties + + +@raw_result_api.wrap_run_steps +def RunSteps( + api: recipe_api.RecipeScriptApi, props: InputProperties +) -> result_pb.RawResult | None: + projects = list(props.projects) + if not projects: + project = api.buildbucket.build.builder.project + assert project, 'No project specified' + projects = [project] + + builders = [] + for project in projects: + cfg = api.luci_config.buildbucket(project=project) + + for bucket in cfg.buckets: + if 'dev' in bucket.name.split('.'): + continue + if bucket.swarming and bucket.swarming.builders: + for builder in bucket.swarming.builders: + properties = {} + if builder.properties: + properties = api.json.loads(builder.properties) + + recipe = '' + if builder.recipe and builder.recipe.name: + recipe = builder.recipe.name + elif '$recipe_engine/recipe' in properties: + recipe = properties['$recipe_engine/recipe'] + elif 'recipe' in properties: + recipe = properties['recipe'] + + if recipe in ('module_deps', 'pigweed/module_deps'): + builders.append((project, bucket.name, builder.name)) + + if not builders: + return result_pb.RawResult( + summary_markdown='No module_deps builders found in configs.', + status=common_pb.SUCCESS, + ) + + # Map: module -> list of projects + deps_map: dict[str, list[str]] = {} + + with api.step.nest('collect builder results'): + for project, bucket_name, builder_name in builders: + with api.step.nest(f'{project}/{bucket_name}/{builder_name}'): + builder_id = builder_common_pb.BuilderID( + project=project, + bucket=bucket_name, + builder=builder_name, + ) + predicate = builds_service_pb.BuildPredicate( + builder=builder_id, + status=common_pb.SUCCESS, + experiments=['-pigweed.non_production'], + ) + + builds = api.buildbucket.search( + predicate, + limit=1, + step_name='search', + ) + + if builds: + build = builds[0] + output_props = ( + json_format.MessageToDict(build.output.properties) or {} + ) + deps = output_props.get('deps', []) + + input_props = ( + json_format.MessageToDict(build.input.properties) or {} + ) + remote = input_props.get('checkout_options', {}).get( + 'remote', '' + ) + + project_name = builder_name + if remote: + project_name = remote.split('/')[-1] + if project_name.endswith('.git'): + project_name = project_name[:-4] + + for dep in deps: + deps_map.setdefault(dep, []).append(project_name) + + # Sort the project lists for consistency + for dep in deps_map: + deps_map[dep] = sorted(set(deps_map[dep])) + + # Output properties + pres = api.step.empty('output properties').presentation + pres.properties['deps_map'] = deps_map + + # Summary + summary = ['Collected dependencies:'] + for dep, projects in sorted(deps_map.items()): + summary.append(f'* **{dep}**: {", ".join(projects)}') + + return result_pb.RawResult( + summary_markdown='\n'.join(summary), + status=common_pb.SUCCESS, + ) + + +def GenTests( + api: recipe_test_api.RecipeTestApi, +) -> Iterator[recipe_test_api.TestData]: + def mock_cfg_data( + builders_info: list[tuple[str, str, str, str]], + ) -> project_config_pb.BuildbucketCfg: + cfg = project_config_pb.BuildbucketCfg() + buckets = {} + for bucket_name, builder_name, recipe_name, style in builders_info: + bucket = buckets.setdefault( + bucket_name, cfg.buckets.add(name=bucket_name) + ) + builder = bucket.swarming.builders.add(name=builder_name) + if style == 'modern': + builder.recipe.name = recipe_name + elif style == 'legacy_dollar': + builder.properties = json.dumps( + {'$recipe_engine/recipe': recipe_name} + ) + elif style == 'legacy_plain': + builder.properties = json.dumps({'recipe': recipe_name}) + return cfg + + # 1. Success case + cfg_success = mock_cfg_data( + [ + ('ci', 'project-a-deps', 'pigweed/module_deps', 'modern'), + ('ci', 'project-b-build', 'pigweed/build', 'legacy_dollar'), + ('ci', 'project-d-other', 'other_recipe', 'legacy_plain'), + ('dev', 'dev-project-deps', 'module_deps', 'modern'), + ] + ) + + build_a = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder='project-a-deps', + ) + build_a.output.properties.update( + { + 'deps': ['pw_analog', 'pw_assert'], + } + ) + build_a.input.properties.update( + { + 'checkout_options': { + 'remote': 'https://pigweed.googlesource.com/project-a.git', + }, + } + ) + + yield api.test( + 'basic', + api.properties(projects=['pigweed']), + api.luci_config.mock_config( + 'pigweed', + 'cr-buildbucket.cfg', + cfg_success, + ), + api.buildbucket.simulated_search_results( + [build_a], + step_name='collect builder results.pigweed/ci/project-a-deps.search', + ), + api.post_check( + post_process.PropertyEquals, + 'deps_map', + { + 'pw_analog': ['project-a'], + 'pw_assert': ['project-a'], + }, + ), + api.post_process(post_process.DropExpectation), + ) + + # 2. No builders + cfg_empty = project_config_pb.BuildbucketCfg() + yield api.test( + 'no_builders', + api.properties(projects=['pigweed']), + api.luci_config.mock_config( + 'pigweed', + 'cr-buildbucket.cfg', + cfg_empty, + ), + api.post_process( + post_process.SummaryMarkdown, + 'No module_deps builders found in configs.', + ), + api.post_process(post_process.DropExpectation), + ) + + # 3. Config fetch failed (Infra Failure) + yield api.test( + 'config_fetch_failed', + api.properties(projects=['pigweed']), + api.step_data( + 'fetch pigweed cr-buildbucket.cfg.get', + retcode=1, + ), + api.post_process(post_process.DropExpectation), + status='INFRA_FAILURE', + ) + + # 4. Fallback to buildbucket project (when props.projects is empty) + yield api.test( + 'fallback_to_buildbucket_project', + api.buildbucket.ci_build(project='my-project'), + api.luci_config.mock_config( + 'my-project', + 'cr-buildbucket.cfg', + cfg_success, + ), + api.buildbucket.simulated_search_results( + [build_a], + step_name='collect builder results.my-project/ci/project-a-deps.search', + ), + api.post_check( + post_process.PropertyEquals, + 'deps_map', + { + 'pw_analog': ['project-a'], + 'pw_assert': ['project-a'], + }, + ), + api.post_process(post_process.DropExpectation), + ) + + # 5. Multiple builders, some empty results, custom project in properties + cfg_multi = mock_cfg_data( + [ + ('ci', 'project-a-deps', 'pigweed/module_deps', 'modern'), + ('ci', 'project-c-deps', 'module_deps', 'modern'), + ] + ) + build_c = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder='project-c-deps', + ) + build_c.output.properties.update( + { + 'deps': ['pw_assert', 'pw_log'], + } + ) + + yield api.test( + 'multiple_builders', + api.properties(projects=['custom-project']), + api.luci_config.mock_config( + 'custom-project', + 'cr-buildbucket.cfg', + cfg_multi, + ), + api.buildbucket.simulated_search_results( + [build_a], + step_name='collect builder results.custom-project/ci/project-a-deps.search', + ), + api.buildbucket.simulated_search_results( + [build_c], + step_name='collect builder results.custom-project/ci/project-c-deps.search', + ), + api.post_check( + post_process.PropertyEquals, + 'deps_map', + { + 'pw_analog': ['project-a'], + 'pw_assert': ['project-a', 'project-c-deps'], + 'pw_log': ['project-c-deps'], + }, + ), + api.post_process(post_process.DropExpectation), + ) + + # 6. Multiple projects + cfg_proj1 = mock_cfg_data( + [('ci', 'builder-1-deps', 'module_deps', 'modern')] + ) + cfg_proj2 = mock_cfg_data( + [('ci', 'builder-2-deps', 'module_deps', 'modern')] + ) + + build_1 = api.buildbucket.ci_build_message( + project='project-1', + bucket='ci', + builder='builder-1-deps', + ) + build_1.output.properties.update({'deps': ['pw_analog']}) + build_1.input.properties.update( + { + 'checkout_options': { + 'remote': 'https://pigweed.googlesource.com/proj1.git', + } + } + ) + + build_2 = api.buildbucket.ci_build_message( + project='project-2', + bucket='ci', + builder='builder-2-deps', + ) + build_2.output.properties.update({'deps': ['pw_assert']}) + build_2.input.properties.update( + { + 'checkout_options': { + 'remote': 'https://pigweed.googlesource.com/proj2.git', + } + } + ) + + yield api.test( + 'multiple_projects', + api.properties(projects=['project-1', 'project-2']), + api.luci_config.mock_config( + 'project-1', 'cr-buildbucket.cfg', cfg_proj1 + ), + api.luci_config.mock_config( + 'project-2', 'cr-buildbucket.cfg', cfg_proj2 + ), + api.buildbucket.simulated_search_results( + [build_1], + step_name='collect builder results.project-1/ci/builder-1-deps.search', + ), + api.buildbucket.simulated_search_results( + [build_2], + step_name='collect builder results.project-2/ci/builder-2-deps.search', + ), + api.post_check( + post_process.PropertyEquals, + 'deps_map', + { + 'pw_analog': ['proj1'], + 'pw_assert': ['proj2'], + }, + ), + api.post_process(post_process.DropExpectation), + )