scripts: zephyr_module: Add URL, version to SPDX
Improve the SPDX with the current values:
- URL: extracted from `git remote`. If more than one remote, URL is not
set.
- Version: extracted from `git rev-parse` (commit id).
- PURL and CPE for Zephyr: generated from URL and version.
For zephyr, the tag is extracted, if present, and replace the commit id for
the version field.
Since official modules does not have tags, tags are not yet extracted for
modules.
To track vulnerabilities from modules dependencies, a new SBOM,
`modules-deps.spdx` was created. It contains the `external-references`
provided by the modules. It allows to easily track vulnerabilities from
these external dependencies.
Signed-off-by: Thomas Gagneret <thomas.gagneret@hexploy.com>
diff --git a/scripts/zephyr_module.py b/scripts/zephyr_module.py
index 2bb72e8..20955cd 100755
--- a/scripts/zephyr_module.py
+++ b/scripts/zephyr_module.py
@@ -152,6 +152,15 @@
doc-url:
required: false
type: str
+ security:
+ required: false
+ type: map
+ mapping:
+ external-references:
+ required: false
+ type: seq
+ sequence:
+ - type: str
'''
MODULE_YML_PATH = PurePath('zephyr/module.yml')
@@ -408,24 +417,7 @@
return out
-def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
- propagate_state=False):
- # Process zephyr_base, projects, and modules and create a dictionary
- # with meta information for each input.
- #
- # The dictionary will contain meta info in the following lists:
- # - zephyr: path and revision
- # - modules: name, path, and revision
- # - west-projects: path and revision
- #
- # returns the dictionary with said lists
-
- meta = {'zephyr': None, 'modules': None, 'workspace': None}
-
- workspace_dirty = False
- workspace_extra = extra_modules is not None
- workspace_off = False
-
+def _create_meta_project(project_path):
def git_revision(path):
rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
stdout=subprocess.PIPE,
@@ -453,77 +445,213 @@
return revision, False
return None, False
- zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
- zephyr_project = {'path': zephyr_base,
- 'revision': zephyr_revision}
+ def git_remote(path):
+ popen = subprocess.Popen(['git', 'remote'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=path)
+ stdout, stderr = popen.communicate()
+ stdout = stdout.decode('utf-8')
+
+ remotes_name = []
+ if not (popen.returncode or stderr):
+ remotes_name = stdout.rstrip().split('\n')
+
+ remote_url = None
+
+ # If more than one remote, do not return any remote
+ if len(remotes_name) == 1:
+ remote = remotes_name[0]
+ popen = subprocess.Popen(['git', 'remote', 'get-url', remote],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=path)
+ stdout, stderr = popen.communicate()
+ stdout = stdout.decode('utf-8')
+
+ if not (popen.returncode or stderr):
+ remote_url = stdout.rstrip()
+
+ return remote_url
+
+ def git_tags(path, revision):
+ if not revision or len(revision) == 0:
+ return None
+
+ popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=path)
+ stdout, stderr = popen.communicate()
+ stdout = stdout.decode('utf-8')
+
+ tags = None
+ if not (popen.returncode or stderr):
+ tags = stdout.rstrip().splitlines()
+
+ return tags
+
+ workspace_dirty = False
+ path = PurePath(project_path).as_posix()
+
+ revision, dirty = git_revision(path)
+ workspace_dirty |= dirty
+ remote = git_remote(path)
+ tags = git_tags(path, revision)
+
+ meta_project = {'path': path,
+ 'revision': revision}
+
+ if remote:
+ meta_project['remote'] = remote
+
+ if tags:
+ meta_project['tags'] = tags
+
+ return meta_project, workspace_dirty
+
+
+def _get_meta_project(meta_projects_list, project_path):
+ projects = [ prj for prj in meta_projects_list[1:] if prj["path"] == project_path ]
+
+ return projects[0] if len(projects) == 1 else None
+
+
+def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
+ propagate_state=False):
+ # Process zephyr_base, projects, and modules and create a dictionary
+ # with meta information for each input.
+ #
+ # The dictionary will contain meta info in the following lists:
+ # - zephyr: path and revision
+ # - modules: name, path, and revision
+ # - west-projects: path and revision
+ #
+ # returns the dictionary with said lists
+
+ meta = {'zephyr': None, 'modules': None, 'workspace': None}
+
+ zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base)
+ zephyr_off = zephyr_project.get("remote") is None
+
+ workspace_dirty = zephyr_dirty
+ workspace_extra = extra_modules is not None
+ workspace_off = zephyr_off
+
+ if zephyr_off:
+ zephyr_project['revision'] += '-off'
+
meta['zephyr'] = zephyr_project
meta['workspace'] = {}
- workspace_dirty |= zephyr_dirty
if west_projs is not None:
from west.manifest import MANIFEST_REV_BRANCH
projects = west_projs['projects']
meta_projects = []
- # Special treatment of manifest project.
- manifest_proj_path = PurePath(projects[0].posixpath).as_posix()
- manifest_revision, manifest_dirty = git_revision(manifest_proj_path)
- workspace_dirty |= manifest_dirty
- manifest_project = {'path': manifest_proj_path,
- 'revision': manifest_revision}
- meta_projects.append(manifest_project)
+ manifest_path = projects[0].posixpath
+ # Special treatment of manifest project
+ # Git information (remote/revision) are not provided by west for the Manifest (west.yml)
+ # To mitigate this, we check if we don't use the manifest from the zephyr repository or an other project.
+ # If it's from zephyr, reuse zephyr information
+ # If it's from an other project, ignore it, it will be added later
+ # If it's not found, we extract data manually (remote/revision) from the directory
+
+ manifest_project = None
+ manifest_dirty = False
+ manifest_off = False
+
+ if zephyr_base == manifest_path:
+ manifest_project = zephyr_project
+ manifest_dirty = zephyr_dirty
+ manifest_off = zephyr_off
+ elif not [ prj for prj in projects[1:] if prj.posixpath == manifest_path ]:
+ manifest_project, manifest_dirty = _create_meta_project(
+ projects[0].posixpath)
+ manifest_off = manifest_project.get("remote") is None
+ if manifest_off:
+ manifest_project["revision"] += "-off"
+
+ if manifest_project:
+ workspace_off |= manifest_off
+ workspace_dirty |= manifest_dirty
+ meta_projects.append(manifest_project)
+
+ # Iterates on all projects except the first one (manifest)
for project in projects[1:]:
- project_path = PurePath(project.posixpath).as_posix()
- revision, dirty = git_revision(project_path)
+ meta_project, dirty = _create_meta_project(project.posixpath)
workspace_dirty |= dirty
- if project.sha(MANIFEST_REV_BRANCH) != revision:
- revision += '-off'
- workspace_off = True
- meta_project = {'path': project_path,
- 'revision': revision}
meta_projects.append(meta_project)
+ off = False
+ if not meta_project.get("remote") or project.sha(MANIFEST_REV_BRANCH) != meta_project['revision'].removesuffix("-dirty"):
+ off = True
+ if not meta_project.get('remote') or project.url != meta_project['remote']:
+ # Force manifest URL and set commit as 'off'
+ meta_project['url'] = project.url
+ off = True
+
+ if off:
+ meta_project['revision'] += '-off'
+ workspace_off |= off
+
+ # If manifest is in project, updates related variables
+ if project.posixpath == manifest_path:
+ manifest_dirty |= dirty
+ manifest_off |= off
+ manifest_project = meta_project
+
meta.update({'west': {'manifest': west_projs['manifest_path'],
'projects': meta_projects}})
meta['workspace'].update({'off': workspace_off})
- meta_projects = []
+ # Iterates on all modules
+ meta_modules = []
for module in modules:
- module_path = PurePath(module.project).as_posix()
- revision, dirty = git_revision(module_path)
- workspace_dirty |= dirty
- meta_project = {'name': module.meta['name'],
- 'path': module_path,
- 'revision': revision}
- meta_projects.append(meta_project)
- meta['modules'] = meta_projects
+ # Check if modules is not in projects
+ # It allows to have the "-off" flag since `modules` variable` does not provide URL/remote
+ meta_module = _get_meta_project(meta_projects, module.project)
+
+ if not meta_module:
+ meta_module, dirty = _create_meta_project(module.project)
+ workspace_dirty |= dirty
+
+ meta_module['name'] = module.meta.get('name')
+
+ if module.meta.get('security'):
+ meta_module['security'] = module.meta.get('security')
+ meta_modules.append(meta_module)
+
+ meta['modules'] = meta_modules
meta['workspace'].update({'dirty': workspace_dirty,
'extra': workspace_extra})
if propagate_state:
+ zephyr_revision = zephyr_project['revision']
if workspace_dirty and not zephyr_dirty:
zephyr_revision += '-dirty'
if workspace_extra:
zephyr_revision += '-extra'
- if workspace_off:
+ if workspace_off and not zephyr_off:
zephyr_revision += '-off'
zephyr_project.update({'revision': zephyr_revision})
if west_projs is not None:
+ manifest_revision = manifest_project['revision']
if workspace_dirty and not manifest_dirty:
manifest_revision += '-dirty'
if workspace_extra:
manifest_revision += '-extra'
- if workspace_off:
+ if workspace_off and not manifest_off:
manifest_revision += '-off'
manifest_project.update({'revision': manifest_revision})
return meta
-def west_projects(manifest = None):
+def west_projects(manifest=None):
manifest_path = None
projects = []
# West is imported here, as it is optional
@@ -691,7 +819,8 @@
for module in modules:
kconfig += process_kconfig(module.project, module.meta)
cmake += process_cmake(module.project, module.meta)
- sysbuild_kconfig += process_sysbuildkconfig(module.project, module.meta)
+ sysbuild_kconfig += process_sysbuildkconfig(
+ module.project, module.meta)
sysbuild_cmake += process_sysbuildcmake(module.project, module.meta)
settings += process_settings(module.project, module.meta)
twister += process_twister(module.project, module.meta)
@@ -735,6 +864,8 @@
args.extra_modules, args.meta_state_propagate)
with open(args.meta_out, 'w', encoding="utf-8") as fp:
+ # Ignore references and insert data instead
+ yaml.Dumper.ignore_aliases = lambda self, data: True
fp.write(yaml.dump(meta))
