scripts/snippets.py - third_party/github/zephyrproject-rtos/zephyr - Git at Google

 #!/usr/bin/env python3
 #
 # Copyright (c) 2022, Nordic Semiconductor ASA
 #
 # SPDX-License-Identifier: Apache-2.0

 '''Internal snippets tool.
 This is part of the build system's support for snippets.
 It is not meant for use outside of the build system.

 Output CMake variables:

 - SNIPPET_NAMES: CMake list of discovered snippet names
 - SNIPPET_FOUND_{snippet}: one per discovered snippet
 '''

 from collections import defaultdict, UserDict
 from dataclasses import dataclass, field
 from pathlib import Path, PurePosixPath
 from typing import Dict, Iterable, List, Set
 import argparse
 import logging
 import os
 import pykwalify.core
 import pykwalify.errors
 import re
 import sys
 import textwrap
 import yaml
 import platform

 # Marker type for an 'append:' configuration. Maps variables
 # to the list of values to append to them.
 Appends = Dict[str, List[str]]

 def _new_append():
     return defaultdict(list)

 def _new_board2appends():
     return defaultdict(_new_append)

 @dataclass
 class Snippet:
     '''Class for keeping track of all the settings discovered for an
     individual snippet.'''

     name: str
     appends: Appends = field(default_factory=_new_append)
     board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)

     def process_data(self, pathobj: Path, snippet_data: dict):
         '''Process the data in a snippet.yml file, after it is loaded into a
         python object and validated by pykwalify.'''
         def append_value(variable, value):
             if variable in ('EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
                 path = pathobj.parent / value
                 if not path.is_file():
                     _err(f'snippet file {pathobj}: {variable}: file not found: {path}')
                 return f'"{path}"'
             _err(f'unknown append variable: {variable}')

         for variable, value in snippet_data.get('append', {}).items():
             self.appends[variable].append(append_value(variable, value))
         for board, settings in snippet_data.get('boards', {}).items():
             if board.startswith('/') and not board.endswith('/'):
                 _err(f"snippet file {pathobj}: board {board} starts with '/', so "
                      "it must end with '/' to use a regular expression")
             for variable, value in settings.get('append', {}).items():
                 self.board2appends[board][variable].append(
                     append_value(variable, value))

 class Snippets(UserDict):
     '''Type for all the information we have discovered about all snippets.
     As a dict, this maps a snippet's name onto the Snippet object.
     Any additional global attributes about all snippets go here as
     instance attributes.'''

     def __init__(self, requested: Iterable[str] = None):
         super().__init__()
         self.paths: Set[Path] = set()
         self.requested: List[str] = list(requested or [])

 class SnippetsError(Exception):
     '''Class for signalling expected errors'''

     def __init__(self, msg):
         self.msg = msg

 class SnippetToCMakePrinter:
     '''Helper class for printing a Snippets's semantics to a .cmake
     include file for use by snippets.cmake.'''

     def __init__(self, snippets: Snippets, out_file):
         self.snippets = snippets
         self.out_file = out_file
         self.section = '#' * 79

     def print_cmake(self):
         '''Print to the output file provided to the constructor.'''
         # TODO: add source file info
         snippets = self.snippets
         snippet_names = sorted(snippets.keys())

         if platform.system() == "Windows":
             # Change to linux-style paths for windows to avoid cmake escape character code issues
             snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))

             for this_snippet in snippets:
                 for snippet_append in (snippets[this_snippet].appends):
                     snippets[this_snippet].appends[snippet_append] = \
                         set(map(lambda x: str(x.replace("\\", "/")), \
                             snippets[this_snippet].appends[snippet_append]))

         snippet_path_list = " ".join(
             sorted(f'"{path}"' for path in snippets.paths))

         self.print('''\
 # WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
 #
 # This file contains build system settings derived from your snippets.
 # Its contents are an implementation detail that should not be used outside
 # of Zephyr's snippets CMake module.
 #
 # See the Snippets guide in the Zephyr documentation for more information.
 ''')

         self.print(f'''\
 {self.section}
 # Global information about all snippets.

 # The name of every snippet that was discovered.
 set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
 # The paths to all the snippet.yml files. One snippet
 # can have multiple snippet.yml files.
 set(SNIPPET_PATHS {snippet_path_list})

 # Create variable scope for snippets build variables
 zephyr_create_scope(snippets)
 ''')

         for snippet_name in snippets.requested:
             self.print_cmake_for(snippets[snippet_name])
             self.print()

     def print_cmake_for(self, snippet: Snippet):
         self.print(f'''\
 {self.section}
 # Snippet '{snippet.name}'

 # Common variable appends.''')
         self.print_appends(snippet.appends, 0)
         for board, appends in snippet.board2appends.items():
             self.print_appends_for_board(board, appends)

     def print_appends_for_board(self, board: str, appends: Appends):
         if board.startswith('/'):
             board_re = board[1:-1]
             self.print(f'''\
 # Appends for board regular expression '{board_re}'
 if("${{BOARD}}" MATCHES "^{board_re}$")''')
         else:
             self.print(f'''\
 # Appends for board '{board}'
 if("${{BOARD}}" STREQUAL "{board}")''')
         self.print_appends(appends, 1)
         self.print('endif()')

     def print_appends(self, appends: Appends, indent: int):
         space = '  ' * indent
         for name, values in appends.items():
             for value in values:
                 self.print(f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)')

     def print(self, *args, **kwargs):
         kwargs['file'] = self.out_file
         print(*args, **kwargs)

 # Name of the file containing the pykwalify schema for snippet.yml
 # files.
 SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
 with open(SCHEMA_PATH, 'r') as f:
     SNIPPET_SCHEMA = yaml.safe_load(f.read())

 # The name of the file which contains metadata about the snippets
 # being defined in a directory.
 SNIPPET_YML = 'snippet.yml'

 # Regular expression for validating snippet names. Snippet names must
 # begin with an alphanumeric character, and may contain alphanumeric
 # characters or underscores. This is intentionally very restrictive to
 # keep things consistent and easy to type and remember. We can relax
 # this a bit later if needed.
 SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')

 # Logger for this module.
 LOG = logging.getLogger('snippets')

 def _err(msg):
     raise SnippetsError(f'error: {msg}')

 def parse_args():
     parser = argparse.ArgumentParser(description='snippets helper',
                                      allow_abbrev=False)
     parser.add_argument('--snippet-root', default=[], action='append', type=Path,
                         help='''a SNIPPET_ROOT element; may be given
                         multiple times''')
     parser.add_argument('--snippet', dest='snippets', default=[], action='append',
                         help='''a SNIPPET element; may be given
                         multiple times''')
     parser.add_argument('--cmake-out', type=Path,
                         help='''file to write cmake output to; include()
                         this file after calling this script''')
     return parser.parse_args()

 def setup_logging():
     # Silence validation errors from pykwalify, which are logged at
     # logging.ERROR level. We want to handle those ourselves as
     # needed.
     logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
     logging.basicConfig(level=logging.INFO,
                         format='  %(name)s: %(message)s')

 def process_snippets(args: argparse.Namespace) -> Snippets:
     '''Process snippet.yml files under each *snippet_root*
     by recursive search. Return a Snippets object describing
     the results of the search.
     '''
     # This will contain information about all the snippets
     # we discover in each snippet_root element.
     snippets = Snippets(requested=args.snippets)

     # Process each path in snippet_root in order, adjusting
     # snippets as needed for each one.
     for root in args.snippet_root:
         process_snippets_in(root, snippets)

     return snippets

 def process_snippets_in(root_dir: Path, snippets: Snippets) -> None:
     '''Process snippet.yml files in *root_dir*,
     updating *snippets* as needed.'''

     if not root_dir.is_dir():
         LOG.warning(f'SNIPPET_ROOT {root_dir} '
                     'is not a directory; ignoring it')
         return

     snippets_dir = root_dir / 'snippets'
     if not snippets_dir.is_dir():
         return

     for dirpath, _, filenames in os.walk(snippets_dir):
         if SNIPPET_YML not in filenames:
             continue

         snippet_yml = Path(dirpath) / SNIPPET_YML
         snippet_data = load_snippet_yml(snippet_yml)
         name = snippet_data['name']
         if name not in snippets:
             snippets[name] = Snippet(name=name)
         snippets[name].process_data(snippet_yml, snippet_data)
         snippets.paths.add(snippet_yml)

 def load_snippet_yml(snippet_yml: Path) -> dict:
     '''Load a snippet.yml file *snippet_yml*, validate the contents
     against the schema, and do other basic checks. Return the dict
     of the resulting YAML data.'''

     with open(snippet_yml, 'r') as f:
         try:
             snippet_data = yaml.safe_load(f.read())
         except yaml.scanner.ScannerError:
             _err(f'snippets file {snippet_yml} is invalid YAML')

     def pykwalify_err(e):
         return f'''\
 invalid {SNIPPET_YML} file: {snippet_yml}
 {textwrap.indent(e.msg, '  ')}
 '''

     try:
         pykwalify.core.Core(source_data=snippet_data,
                             schema_data=SNIPPET_SCHEMA).validate()
     except pykwalify.errors.PyKwalifyException as e:
         _err(pykwalify_err(e))

     name = snippet_data['name']
     if not SNIPPET_NAME_RE.fullmatch(name):
         _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
              'snippet names must begin with a letter '
              'or number, and may only contain letters, numbers, '
              'dashes (-), and underscores (_)')

     return snippet_data

 def check_for_errors(snippets: Snippets) -> None:
     unknown_snippets = sorted(snippet for snippet in snippets.requested
                               if snippet not in snippets)
     if unknown_snippets:
         all_snippets = '\n    '.join(sorted(snippets))
         _err(f'''\
 snippets not found: {', '.join(unknown_snippets)}
   Please choose from among the following snippets:
     {all_snippets}''')

 def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
     '''Write a cmake include file to *cmake_out* which
     reflects the information in *snippets*.

     The contents of this file should be considered an implementation
     detail and are not meant to be used outside of snippets.cmake.'''
     if not cmake_out.parent.exists():
         cmake_out.parent.mkdir()
     with open(cmake_out, 'w') as f:
         SnippetToCMakePrinter(snippets, f).print_cmake()

 def main():
     args = parse_args()
     setup_logging()
     try:
         snippets = process_snippets(args)
         check_for_errors(snippets)
     except SnippetsError as e:
         LOG.critical(e.msg)
         sys.exit(1)
     write_cmake_out(snippets, args.cmake_out)

 if __name__ == "__main__":
     main()
	#!/usr/bin/env python3
	#
	# Copyright (c) 2022, Nordic Semiconductor ASA
	#
	# SPDX-License-Identifier: Apache-2.0

	'''Internal snippets tool.
	This is part of the build system's support for snippets.
	It is not meant for use outside of the build system.

	Output CMake variables:

	- SNIPPET_NAMES: CMake list of discovered snippet names
	- SNIPPET_FOUND_{snippet}: one per discovered snippet
	'''

	from collections import defaultdict, UserDict
	from dataclasses import dataclass, field
	from pathlib import Path, PurePosixPath
	from typing import Dict, Iterable, List, Set
	import argparse
	import logging
	import os
	import pykwalify.core
	import pykwalify.errors
	import re
	import sys
	import textwrap
	import yaml
	import platform

	# Marker type for an 'append:' configuration. Maps variables
	# to the list of values to append to them.
	Appends = Dict[str, List[str]]

	def _new_append():
	return defaultdict(list)

	def _new_board2appends():
	return defaultdict(_new_append)

	@dataclass
	class Snippet:
	'''Class for keeping track of all the settings discovered for an
	individual snippet.'''

	name: str
	appends: Appends = field(default_factory=_new_append)
	board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)

	def process_data(self, pathobj: Path, snippet_data: dict):
	'''Process the data in a snippet.yml file, after it is loaded into a
	python object and validated by pykwalify.'''
	def append_value(variable, value):
	if variable in ('EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
	path = pathobj.parent / value
	if not path.is_file():
	_err(f'snippet file {pathobj}: {variable}: file not found: {path}')
	return f'"{path}"'
	_err(f'unknown append variable: {variable}')

	for variable, value in snippet_data.get('append', {}).items():
	self.appends[variable].append(append_value(variable, value))
	for board, settings in snippet_data.get('boards', {}).items():
	if board.startswith('/') and not board.endswith('/'):
	_err(f"snippet file {pathobj}: board {board} starts with '/', so "
	"it must end with '/' to use a regular expression")
	for variable, value in settings.get('append', {}).items():
	self.board2appends[board][variable].append(
	append_value(variable, value))

	class Snippets(UserDict):
	'''Type for all the information we have discovered about all snippets.
	As a dict, this maps a snippet's name onto the Snippet object.
	Any additional global attributes about all snippets go here as
	instance attributes.'''

	def __init__(self, requested: Iterable[str] = None):
	super().__init__()
	self.paths: Set[Path] = set()
	self.requested: List[str] = list(requested or [])

	class SnippetsError(Exception):
	'''Class for signalling expected errors'''

	def __init__(self, msg):
	self.msg = msg

	class SnippetToCMakePrinter:
	'''Helper class for printing a Snippets's semantics to a .cmake
	include file for use by snippets.cmake.'''

	def __init__(self, snippets: Snippets, out_file):
	self.snippets = snippets
	self.out_file = out_file
	self.section = '#' * 79

	def print_cmake(self):
	'''Print to the output file provided to the constructor.'''
	# TODO: add source file info
	snippets = self.snippets
	snippet_names = sorted(snippets.keys())

	if platform.system() == "Windows":
	# Change to linux-style paths for windows to avoid cmake escape character code issues
	snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))

	for this_snippet in snippets:
	for snippet_append in (snippets[this_snippet].appends):
	snippets[this_snippet].appends[snippet_append] = \
	set(map(lambda x: str(x.replace("\\", "/")), \
	snippets[this_snippet].appends[snippet_append]))

	snippet_path_list = " ".join(
	sorted(f'"{path}"' for path in snippets.paths))

	self.print('''\
	# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
	#
	# This file contains build system settings derived from your snippets.
	# Its contents are an implementation detail that should not be used outside
	# of Zephyr's snippets CMake module.
	#
	# See the Snippets guide in the Zephyr documentation for more information.
	''')

	self.print(f'''\
	{self.section}
	# Global information about all snippets.

	# The name of every snippet that was discovered.
	set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
	# The paths to all the snippet.yml files. One snippet
	# can have multiple snippet.yml files.
	set(SNIPPET_PATHS {snippet_path_list})

	# Create variable scope for snippets build variables
	zephyr_create_scope(snippets)
	''')

	for snippet_name in snippets.requested:
	self.print_cmake_for(snippets[snippet_name])
	self.print()

	def print_cmake_for(self, snippet: Snippet):
	self.print(f'''\
	{self.section}
	# Snippet '{snippet.name}'

	# Common variable appends.''')
	self.print_appends(snippet.appends, 0)
	for board, appends in snippet.board2appends.items():
	self.print_appends_for_board(board, appends)

	def print_appends_for_board(self, board: str, appends: Appends):
	if board.startswith('/'):
	board_re = board[1:-1]
	self.print(f'''\
	# Appends for board regular expression '{board_re}'
	if("${{BOARD}}" MATCHES "^{board_re}$")''')
	else:
	self.print(f'''\
	# Appends for board '{board}'
	if("${{BOARD}}" STREQUAL "{board}")''')
	self.print_appends(appends, 1)
	self.print('endif()')

	def print_appends(self, appends: Appends, indent: int):
	space = ' ' * indent
	for name, values in appends.items():
	for value in values:
	self.print(f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)')

	def print(self, args, *kwargs):
	kwargs['file'] = self.out_file
	print(args, *kwargs)

	# Name of the file containing the pykwalify schema for snippet.yml
	# files.
	SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
	with open(SCHEMA_PATH, 'r') as f:
	SNIPPET_SCHEMA = yaml.safe_load(f.read())

	# The name of the file which contains metadata about the snippets
	# being defined in a directory.
	SNIPPET_YML = 'snippet.yml'

	# Regular expression for validating snippet names. Snippet names must
	# begin with an alphanumeric character, and may contain alphanumeric
	# characters or underscores. This is intentionally very restrictive to
	# keep things consistent and easy to type and remember. We can relax
	# this a bit later if needed.
	SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')

	# Logger for this module.
	LOG = logging.getLogger('snippets')

	def _err(msg):
	raise SnippetsError(f'error: {msg}')

	def parse_args():
	parser = argparse.ArgumentParser(description='snippets helper',
	allow_abbrev=False)
	parser.add_argument('--snippet-root', default=[], action='append', type=Path,
	help='''a SNIPPET_ROOT element; may be given
	multiple times''')
	parser.add_argument('--snippet', dest='snippets', default=[], action='append',
	help='''a SNIPPET element; may be given
	multiple times''')
	parser.add_argument('--cmake-out', type=Path,
	help='''file to write cmake output to; include()
	this file after calling this script''')
	return parser.parse_args()

	def setup_logging():
	# Silence validation errors from pykwalify, which are logged at
	# logging.ERROR level. We want to handle those ourselves as
	# needed.
	logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
	logging.basicConfig(level=logging.INFO,
	format=' %(name)s: %(message)s')

	def process_snippets(args: argparse.Namespace) -> Snippets:
	'''Process snippet.yml files under each snippet_root
	by recursive search. Return a Snippets object describing
	the results of the search.
	'''
	# This will contain information about all the snippets
	# we discover in each snippet_root element.
	snippets = Snippets(requested=args.snippets)

	# Process each path in snippet_root in order, adjusting
	# snippets as needed for each one.
	for root in args.snippet_root:
	process_snippets_in(root, snippets)

	return snippets

	def process_snippets_in(root_dir: Path, snippets: Snippets) -> None:
	'''Process snippet.yml files in root_dir,
	updating snippets as needed.'''

	if not root_dir.is_dir():
	LOG.warning(f'SNIPPET_ROOT {root_dir} '
	'is not a directory; ignoring it')
	return

	snippets_dir = root_dir / 'snippets'
	if not snippets_dir.is_dir():
	return

	for dirpath, _, filenames in os.walk(snippets_dir):
	if SNIPPET_YML not in filenames:
	continue

	snippet_yml = Path(dirpath) / SNIPPET_YML
	snippet_data = load_snippet_yml(snippet_yml)
	name = snippet_data['name']
	if name not in snippets:
	snippets[name] = Snippet(name=name)
	snippets[name].process_data(snippet_yml, snippet_data)
	snippets.paths.add(snippet_yml)

	def load_snippet_yml(snippet_yml: Path) -> dict:
	'''Load a snippet.yml file snippet_yml, validate the contents
	against the schema, and do other basic checks. Return the dict
	of the resulting YAML data.'''

	with open(snippet_yml, 'r') as f:
	try:
	snippet_data = yaml.safe_load(f.read())
	except yaml.scanner.ScannerError:
	_err(f'snippets file {snippet_yml} is invalid YAML')

	def pykwalify_err(e):
	return f'''\
	invalid {SNIPPET_YML} file: {snippet_yml}
	{textwrap.indent(e.msg, ' ')}
	'''

	try:
	pykwalify.core.Core(source_data=snippet_data,
	schema_data=SNIPPET_SCHEMA).validate()
	except pykwalify.errors.PyKwalifyException as e:
	_err(pykwalify_err(e))

	name = snippet_data['name']
	if not SNIPPET_NAME_RE.fullmatch(name):
	_err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
	'snippet names must begin with a letter '
	'or number, and may only contain letters, numbers, '
	'dashes (-), and underscores (_)')

	return snippet_data

	def check_for_errors(snippets: Snippets) -> None:
	unknown_snippets = sorted(snippet for snippet in snippets.requested
	if snippet not in snippets)
	if unknown_snippets:
	all_snippets = '\n '.join(sorted(snippets))
	_err(f'''\
	snippets not found: {', '.join(unknown_snippets)}
	Please choose from among the following snippets:
	{all_snippets}''')

	def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
	'''Write a cmake include file to cmake_out which
	reflects the information in snippets.

	The contents of this file should be considered an implementation
	detail and are not meant to be used outside of snippets.cmake.'''
	if not cmake_out.parent.exists():
	cmake_out.parent.mkdir()
	with open(cmake_out, 'w') as f:
	SnippetToCMakePrinter(snippets, f).print_cmake()

	def main():
	args = parse_args()
	setup_logging()
	try:
	snippets = process_snippets(args)
	check_for_errors(snippets)
	except SnippetsError as e:
	LOG.critical(e.msg)
	sys.exit(1)
	write_cmake_out(snippets, args.cmake_out)

	if __name__ == "__main__":
	main()