pw_build: Add GnWriter and GnFile
This CL adds Python utilities for writing GN build and import files.
Change-Id: Ie0feb1e378cfb8d163e129137f3dfc748977d15f
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/138702
Commit-Queue: Aaron Green <aarongreen@google.com>
Reviewed-by: Carlos Chinchilla <cachinchilla@google.com>
diff --git a/pw_build/py/BUILD.gn b/pw_build/py/BUILD.gn
index 8fd9a7e..780fe98 100644
--- a/pw_build/py/BUILD.gn
+++ b/pw_build/py/BUILD.gn
@@ -44,6 +44,7 @@
"pw_build/gn_resolver.py",
"pw_build/gn_target.py",
"pw_build/gn_utils.py",
+ "pw_build/gn_writer.py",
"pw_build/host_tool.py",
"pw_build/merge_profraws.py",
"pw_build/mirror_tree.py",
@@ -68,6 +69,7 @@
"gn_config_test.py",
"gn_target_test.py",
"gn_utils_test.py",
+ "gn_writer_test.py",
"project_builder_prefs_test.py",
"python_runner_test.py",
"zip_test.py",
diff --git a/pw_build/py/gn_writer_test.py b/pw_build/py/gn_writer_test.py
new file mode 100644
index 0000000..b2139d8
--- /dev/null
+++ b/pw_build/py/gn_writer_test.py
@@ -0,0 +1,295 @@
+# Copyright 2023 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests for the pw_build.gn_writer module."""
+
+import os
+import unittest
+
+from io import StringIO
+from pathlib import PurePath
+from tempfile import TemporaryDirectory
+
+from pw_build.gn_config import GnConfig
+from pw_build.gn_writer import (
+ COPYRIGHT_HEADER,
+ GnFile,
+ GnWriter,
+)
+from pw_build.gn_target import GnTarget
+from pw_build.gn_utils import MalformedGnError
+
+
+class TestGnWriter(unittest.TestCase):
+ """Tests for gn_writer.GnWriter."""
+
+ def setUp(self):
+ """Creates a GnWriter that writes to a StringIO."""
+ self.reset()
+
+ def reset(self):
+ """Resets the writer and output."""
+ self.output = StringIO()
+ self.writer = GnWriter(self.output)
+
+ def test_write_comment(self):
+ """Writes a GN comment."""
+ self.writer.write_comment('hello, world!')
+ self.assertEqual(
+ self.output.getvalue(),
+ '# hello, world!\n',
+ )
+
+ def test_write_comment_wrap(self):
+ """Writes a GN comment that is exactly 80 characters."""
+ extra_long = (
+ "This line is a " + ("really, " * 5) + "REALLY extra long comment"
+ )
+ self.writer.write_comment(extra_long)
+ self.assertEqual(
+ self.output.getvalue(),
+ '# This line is a really, really, really, really, really, REALLY '
+ 'extra long\n# comment\n',
+ )
+
+ def test_write_comment_nowrap(self):
+ """Writes a long GN comment without whitespace to wrap on."""
+ no_breaks = 'A' + ('a' * 76) + 'h!'
+ self.writer.write_comment(no_breaks)
+ self.assertEqual(
+ self.output.getvalue(),
+ '# Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+ 'aaaaaaaaaaaaah!\n',
+ )
+
+ def test_write_imports(self):
+ """Writes GN import statements."""
+ self.writer.write_import('foo.gni')
+ self.writer.write_imports(['bar.gni', 'baz.gni'])
+ lines = [
+ 'import("foo.gni")',
+ 'import("bar.gni")',
+ 'import("baz.gni")',
+ ]
+ self.assertEqual('\n'.join(lines), self.output.getvalue().strip())
+
+ def test_write_config(self):
+ """Writes a GN config."""
+ config = GnConfig(
+ json='''{
+ "label": "$dir_3p/test:my-config",
+ "cflags": ["-frobinator", "-fizzbuzzer"],
+ "defines": ["KEY=VAL"],
+ "public": true,
+ "usages": 1
+ }'''
+ )
+ self.writer.write_config(config)
+ lines = [
+ 'config("my-config") {',
+ ' cflags = [',
+ ' "-fizzbuzzer",',
+ ' "-frobinator",',
+ ' ]',
+ ' defines = [',
+ ' "KEY=VAL",',
+ ' ]',
+ '}',
+ ]
+ self.assertEqual('\n'.join(lines), self.output.getvalue().strip())
+
+ def test_write_target(self):
+ """Tests writing the target using a GnWriter."""
+ target = GnTarget(
+ '$build',
+ '$src',
+ json='''{
+ "target_type": "custom_type",
+ "target_name": "my-target",
+ "package": "my-package"
+ }''',
+ )
+ target.add_visibility(bazel='//visibility:private')
+ target.add_visibility(bazel='//foo:__subpackages__')
+ target.add_path('public', bazel='//foo:my-header.h')
+ target.add_path('sources', bazel='//foo:my-source.cc')
+ target.add_path('inputs', bazel='//bar:my.data')
+ target.config.add('cflags', '-frobinator')
+ target.add_dep(public=True, bazel='//my-package:foo')
+ target.add_dep(public=True, bazel='@com_corp_repo//bar')
+ target.add_dep(bazel='//other-pkg/baz')
+ target.add_dep(bazel='@com_corp_repo//:top-level')
+
+ output = StringIO()
+ writer = GnWriter(output)
+ writer.repos = {'com_corp_repo': 'repo'}
+ writer.aliases = {'$build/other-pkg/baz': '$build/another-pkg/baz'}
+ writer.write_target(target)
+
+ self.assertEqual(
+ output.getvalue(),
+ '''
+# Generated from //my-package:my-target
+custom_type("my-target") {
+ visibility = [
+ "../foo/*",
+ ":*",
+ ]
+ public = [
+ "$src/foo/my-header.h",
+ ]
+ sources = [
+ "$src/foo/my-source.cc",
+ ]
+ inputs = [
+ "$src/bar/my.data",
+ ]
+ cflags = [
+ "-frobinator",
+ ]
+ public_deps = [
+ "$dir_pw_third_party/repo/bar",
+ ":foo",
+ ]
+ deps = [
+ "$dir_pw_third_party/repo:top-level",
+ "../another-pkg/baz",
+ ]
+}
+'''.lstrip(),
+ )
+
+ def test_write_list(self):
+ """Writes a GN list assigned to a variable."""
+ self.writer.write_list('empty', [])
+ self.writer.write_list('items', ['foo', 'bar', 'baz'])
+ lines = [
+ 'items = [',
+ ' "bar",',
+ ' "baz",',
+ ' "foo",',
+ ']',
+ ]
+ self.assertEqual('\n'.join(lines), self.output.getvalue().strip())
+
+ def test_write_scope(self):
+ """Writes a GN scope assigned to a variable."""
+ self.writer.write_scope('outer')
+ self.writer.write('key1 = "val1"')
+ self.writer.write_scope('inner')
+ self.writer.write('key2 = "val2"')
+ self.writer.write_end()
+ self.writer.write('key3 = "val3"')
+ self.writer.write_end()
+ lines = [
+ 'outer = {',
+ ' key1 = "val1"',
+ ' inner = {',
+ ' key2 = "val2"',
+ ' }',
+ '',
+ ' key3 = "val3"',
+ '}',
+ ]
+ self.assertEqual('\n'.join(lines), self.output.getvalue().strip())
+
+ def test_write_if_else_end(self):
+ """Writes GN conditional statements."""
+ self.writer.write_if('current_os == "linux"')
+ self.writer.write('mascot = "penguin"')
+ self.writer.write_else_if('current_os == "mac"')
+ self.writer.write('mascot = "dogcow"')
+ self.writer.write_else_if('current_os == "win"')
+ self.writer.write('mascot = "clippy"')
+ self.writer.write_else()
+ self.writer.write('mascot = "dropbear"')
+ self.writer.write_end()
+ lines = [
+ 'if (current_os == "linux") {',
+ ' mascot = "penguin"',
+ '} else if (current_os == "mac") {',
+ ' mascot = "dogcow"',
+ '} else if (current_os == "win") {',
+ ' mascot = "clippy"',
+ '} else {',
+ ' mascot = "dropbear"',
+ '}',
+ ]
+ self.assertEqual('\n'.join(lines), self.output.getvalue().strip())
+
+ def test_write_unclosed_target(self):
+ """Triggers an error from an unclosed GN scope."""
+ self.writer.write_target_start('unclosed', 'target')
+ with self.assertRaises(MalformedGnError):
+ self.writer.seal()
+
+ def test_write_unclosed_scope(self):
+ """Triggers an error from an unclosed GN scope."""
+ self.writer.write_scope('unclosed_scope')
+ with self.assertRaises(MalformedGnError):
+ self.writer.seal()
+
+ def test_write_unclosed_if(self):
+ """Triggers an error from an unclosed GN condition."""
+ self.writer.write_if('var == "unclosed-if"')
+ with self.assertRaises(MalformedGnError):
+ self.writer.seal()
+
+ def test_write_unclosed_else_if(self):
+ """Triggers an error from an unclosed GN condition."""
+ self.writer.write_if('var == "closed-if"')
+ self.writer.write_else_if('var == "unclosed-else-if"')
+ with self.assertRaises(MalformedGnError):
+ self.writer.seal()
+
+ def test_write_unclosed_else(self):
+ """Triggers an error from an unclosed GN condition."""
+ self.writer.write_if('var == "closed-if"')
+ self.writer.write_else_if('var == "closed-else-if"')
+ self.writer.write_else()
+ with self.assertRaises(MalformedGnError):
+ self.writer.seal()
+
+
+class TestGnFile(unittest.TestCase):
+ """Tests for gn_writer.GnFile."""
+
+ def test_format_on_close(self):
+ """Verifies the GN file is formatted when the file is closed."""
+ with TemporaryDirectory() as tmpdirname:
+ with GnFile(PurePath(tmpdirname, 'BUILD.gn')) as build_gn:
+ build_gn.write(' correct = "indent"')
+ build_gn.write_comment('newline before comment')
+ build_gn.write_scope('no_newline_before_item')
+ build_gn.write_list('single_item', ['is.inlined'])
+ build_gn.write_end()
+
+ filename = PurePath('pw_build', 'gn_writer.py')
+ expected = (
+ COPYRIGHT_HEADER
+ + f'''
+# This file was automatically generated by {filename}
+
+correct = "indent"
+
+# newline before comment
+no_newline_before_item = {{
+ single_item = [ "is.inlined" ]
+}}'''
+ )
+ with open(os.path.join(tmpdirname, 'BUILD.gn'), 'r') as build_gn:
+ self.assertEqual(expected.strip(), build_gn.read().strip())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pw_build/py/pw_build/gn_writer.py b/pw_build/py/pw_build/gn_writer.py
new file mode 100644
index 0000000..515ca7b
--- /dev/null
+++ b/pw_build/py/pw_build/gn_writer.py
@@ -0,0 +1,368 @@
+# Copyright 2023 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Writes a formatted BUILD.gn _file."""
+
+import os
+import subprocess
+
+from datetime import datetime
+from pathlib import PurePath, PurePosixPath
+from types import TracebackType
+from typing import Dict, IO, Iterable, Iterator, List, Optional, Type, Union
+
+from pw_build.gn_config import GnConfig, GN_CONFIG_FLAGS
+from pw_build.gn_target import GnTarget
+from pw_build.gn_utils import GnLabel, GnPath, MalformedGnError
+
+COPYRIGHT_HEADER = f'''
+# Copyright {datetime.now().year} The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# DO NOT MANUALLY EDIT!'''
+
+
+class GnWriter:
+ """Represents a partial BUILD.gn file being constructed.
+
+ Except for testing , callers should prefer using `GnFile`. That
+ object wraps this one, and ensures that the GN file produced includes the
+ relevant copyright header and is formatted correctly.
+
+ Attributes:
+ repos: A mapping of repository names to build args. These are used to
+ replace repository names when writing labels.
+ aliases: A mapping of label names to build args. These can be used to
+ rewrite labels with alternate names, e.g. "gtest" to "googletest".
+ """
+
+ def __init__(self, file: IO) -> None:
+ self._file: IO = file
+ self._scopes: List[str] = []
+ self._margin: str = ''
+ self._needs_blank: bool = False
+ self.repos: Dict[str, str] = {}
+ self.aliases: Dict[str, str] = {}
+
+ def write_comment(self, comment: Optional[str] = None) -> None:
+ """Adds a GN comment.
+
+ Args:
+ comment: The comment string to write.
+ """
+ if not comment:
+ self.write('#')
+ return
+ while len(comment) > 78:
+ index = comment.rfind(' ', 0, 78)
+ if index < 0:
+ break
+ self.write(f'# {comment[:index]}')
+ comment = comment[index + 1 :]
+ self.write(f'# {comment}')
+
+ def write_import(self, gni: Union[str, PurePosixPath, GnPath]) -> None:
+ """Adds a GN import.
+
+ Args:
+ gni: The source-relative path to a GN import file.
+ """
+ self._needs_blank = False
+ self.write(f'import("{str(gni)}")')
+ self._needs_blank = True
+
+ def write_imports(self, imports: Iterable[str]) -> None:
+ """Adds a list of GN imports.
+
+ Args:
+ imports: A list of GN import files.
+ """
+ for gni in imports:
+ self.write_import(gni)
+
+ def write_config(self, config: GnConfig) -> None:
+ """Adds a GN config.
+
+ Args:
+ config: The GN config data to write.
+ """
+ if not config:
+ return
+ if not config.label:
+ raise MalformedGnError('missing label for `config`')
+ self.write_target_start('config', config.label.name())
+ for flag in GN_CONFIG_FLAGS:
+ self.write_list(flag, config.get(flag))
+ self.write_end()
+
+ def write_target(self, target: GnTarget) -> None:
+ """Write a GN target.
+
+ Args:
+ target: The GN target data to write.
+ """
+ self.write_comment(
+ f'Generated from //{target.package()}:{target.name()}'
+ )
+ self.write_target_start(target.type(), target.name())
+ visibility = [
+ target.make_relative(scope) for scope in target.visibility
+ ]
+ self.write_list('visibility', visibility)
+ self.write_list('public', [str(path) for path in target.public])
+ self.write_list('sources', [str(path) for path in target.sources])
+ self.write_list('inputs', [str(path) for path in target.inputs])
+ for flag in GN_CONFIG_FLAGS:
+ self.write_list(flag, target.config.get(flag))
+ self._write_relative('public_configs', target, target.public_configs)
+ self._write_relative('configs', target, target.configs)
+ self._write_relative('remove_configs', target, target.remove_configs)
+ self._write_relative('public_deps', target, target.public_deps)
+ self._write_relative('deps', target, target.deps)
+ self.write_end()
+
+ def _write_relative(
+ self, var_name: str, target: GnTarget, labels: Iterable[GnLabel]
+ ) -> None:
+ """Write a list of labels relative to a target.
+
+ Args:
+ var_name: The name of the GN list variable.
+ target: The GN target to rebase the labels to.
+ labels: The labels to write to the list.
+ """
+ self.write_list(var_name, self._resolve(target, labels))
+
+ def _resolve(
+ self, target: GnTarget, labels: Iterable[GnLabel]
+ ) -> Iterator[str]:
+ """Returns rewritten labels.
+
+ If this label has a repo, it must be a key in this object's `repos` and
+ will be replaced by the corresponding value. If this label is a key in
+ this object's `aliases`, it will be replaced by the corresponding value.
+
+ Args:
+ labels: The labels to resolve.
+ """
+ for label in labels:
+ repo = label.repo()
+ if repo:
+ label.resolve_repo(self.repos[repo])
+ label = GnLabel(self.aliases.get(str(label), str(label)))
+ yield target.make_relative(label)
+
+ def write_target_start(
+ self, target_type: str, target_name: Optional[str] = None
+ ) -> None:
+ """Begins a GN target of the given type.
+
+ Args:
+ target_type: The type of the GN target.
+ target_name: The name of the GN target.
+ """
+ if target_name:
+ self.write(f'{target_type}("{target_name}") {{')
+ self._indent(target_name)
+ else:
+ self.write(f'{target_type}() {{')
+ self._indent(target_type)
+
+ def write_list(
+ self, var_name: str, items: Iterable[str], reorder: bool = True
+ ) -> None:
+ """Adds a named GN list of the given items, if non-empty.
+
+ Args:
+ var_name: The name of the GN list variable.
+ items: The list items to write as strings.
+ reorder: If true, the list is sorted lexicographically.
+ """
+ items = list(items)
+ if not items:
+ return
+ self.write(f'{var_name} = [')
+ self._indent(var_name)
+ if reorder:
+ items = sorted(items)
+ for item in items:
+ self.write(f'"{str(item)}",')
+ self._outdent()
+ self.write(']')
+
+ def write_scope(self, var_name: str) -> None:
+ """Begins a named GN scope.
+
+ Args:
+ var_name: The name of the GN scope variable.
+ """
+ self.write(f'{var_name} = {{')
+ self._indent(var_name)
+
+ def write_if(self, cond: str) -> None:
+ """Begins a GN 'if' condition.
+
+ Args:
+ cond: The conditional expression.
+ """
+ self.write(f'if ({cond}) {{')
+ self._indent(cond)
+
+ def write_else_if(self, cond: str) -> None:
+ """Adds another GN 'if' condition to a previous 'if' condition.
+
+ Args:
+ cond: The conditional expression.
+ """
+ self._outdent()
+ self.write(f'}} else if ({cond}) {{')
+ self._indent(cond)
+
+ def write_else(self) -> None:
+ """Adds a GN 'else' clause to a previous 'if' condition."""
+ last = self._outdent()
+ self.write('} else {')
+ self._indent(f'!({last})')
+
+ def write_end(self) -> None:
+ """Ends a target, scope, or 'if' condition'."""
+ self._outdent()
+ self.write('}')
+ self._needs_blank = True
+
+ def write_blank(self) -> None:
+ """Adds a blank line."""
+ print('', file=self._file)
+ self._needs_blank = False
+
+ def write_preformatted(self, preformatted: str) -> None:
+ """Adds text with minimal formatting.
+
+ The only formatting applied to the given text is to strip any leading
+ whitespace. This allows calls to be more readable by allowing
+ preformatted text to start on a new line, e.g.
+
+ _write_preformatted('''
+ preformatted line 1
+ preformatted line 2
+ preformatted line 3''')
+
+ Args:
+ preformatted: The text to write.
+ """
+ print(preformatted.lstrip(), file=self._file)
+
+ def write(self, text: str) -> None:
+ """Writes to the file, appropriately indented.
+
+ Args:
+ text: The text to indent and write.
+ """
+ if self._needs_blank:
+ self.write_blank()
+ print(f'{self._margin}{text}', file=self._file)
+
+ def _indent(self, scope: str) -> None:
+ """Increases the current margin.
+
+ Saves the scope of indent to aid in debugging. For example, trying to
+ use incorrect code such as
+
+ ```
+ self.write_if('foo')
+ self.write_comment('bar')
+ self.write_else_if('baz')
+ ```
+
+ will throw an exception due to the missing `write_end`. The exception
+ will note that 'baz' was opened but not closed.
+
+ Args:
+ scope: The name of the scope (for debugging).
+ """
+ self._scopes.append(scope)
+ self._margin += ' '
+
+ def _outdent(self) -> str:
+ """Decreases the current margin."""
+ if not self._scopes:
+ raise MalformedGnError('scope closed unexpectedly')
+ last = self._scopes.pop()
+ self._margin = self._margin[2:]
+ self._needs_blank = False
+ return last
+
+ def seal(self) -> None:
+ """Instructs the object that no more writes will occur."""
+ if self._scopes:
+ raise MalformedGnError(f'unclosed scope(s): {self._scopes}')
+
+
+class GnFile:
+ """Represents an open BUILD.gn file that is formatted on close.
+
+ Typical usage:
+
+ with GnFile('/path/to/BUILD.gn', 'my-package') as build_gn:
+ build_gn.write_...
+
+ where "write_..." refers to any of the "write" methods of `GnWriter`.
+ """
+
+ def __init__(
+ self, pathname: PurePath, package: Optional[str] = None
+ ) -> None:
+ if pathname.name != 'BUILD.gn' and pathname.suffix != '.gni':
+ raise MalformedGnError(f'invalid GN filename: {pathname}')
+ os.makedirs(pathname.parent, exist_ok=True)
+ self._pathname: PurePath = pathname
+ self._package: Optional[str] = package
+ self._file: IO
+ self._writer: GnWriter
+
+ def __enter__(self) -> GnWriter:
+ """Opens the GN file."""
+ self._file = open(self._pathname, 'w+')
+ self._writer = GnWriter(self._file)
+ self._writer.write_preformatted(COPYRIGHT_HEADER)
+ file = PurePath(*PurePath(__file__).parts[-2:])
+ self._writer.write_comment(
+ f'This file was automatically generated by {file}'
+ )
+ if self._package:
+ self._writer.write_comment(
+ f'It contains GN build targets for {self._package}.'
+ )
+ self._writer.write_blank()
+ return self._writer
+
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType],
+ ) -> None:
+ """Closes the GN file and formats it."""
+ self._file.close()
+ subprocess.check_call(['gn', 'format', self._pathname])
