| #! /usr/bin/env python3 |
| # |
| # Protocol Buffers - Google's data interchange format |
| # Copyright 2015 Google Inc. All rights reserved. |
| # |
| # Use of this source code is governed by a BSD-style |
| # license that can be found in the LICENSE file or at |
| # https://developers.google.com/open-source/licenses/bsd |
| |
| """PDDM - Poor Developers' Debug-able Macros |
| |
| A simple markup that can be added in comments of source so they can then be |
| expanded out into code. Most of this could be done with CPP macros, but then |
| developers can't really step through them in the debugger, this way they are |
| expanded to the same code, but you can debug them. |
| |
| Any file can be processed, but the syntax is designed around a C based compiler. |
| Processed lines start with "//%". There are three types of sections you can |
| create: Text (left alone), Macro Definitions, and Macro Expansions. There is |
| no order required between definitions and expansions, all definitions are read |
| before any expansions are processed (thus, if desired, definitions can be put |
| at the end of the file to keep them out of the way of the code). |
| |
| Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines |
| afterwards that start with "//%" are included in the definition. Multiple |
| macros can be defined in one block by just using a another "//%PDDM-DEFINE" |
| line to start the next macro. Optionally, a macro can be ended with |
| "//%PDDM-DEFINE-END", this can be useful when you want to make it clear that |
| trailing blank lines are included in the macro. You can also end a definition |
| with an expansion. |
| |
| Macro Expansions are started by single lines containing |
| "//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another |
| expansions. All lines in-between are replaced by the result of the expansion. |
| The first line of the expansion is always a blank like just for readability. |
| |
| Expansion itself is pretty simple, one macro can invoke another macro, but |
| you cannot nest the invoke of a macro in another macro (i.e. - can't do |
| "foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() |
| within its expansion. |
| |
| When macros are expanded, the arg references can also add "$O" suffix to the |
| name (i.e. - "NAME$O") to specify an option to be applied. The options are: |
| |
| $S - Replace each character in the value with a space. |
| $l - Lowercase the first letter of the value. |
| $L - Lowercase the whole value. |
| $u - Uppercase the first letter of the value. |
| $U - Uppercase the whole value. |
| |
| Within a macro you can use ## to cause things to get joined together after |
| expansion (i.e. - "a##b" within a macro will become "ab"). |
| |
| Example: |
| |
| int foo(MyEnum x) { |
| switch (x) { |
| //%PDDM-EXPAND case(Enum_Left, 1) |
| //%PDDM-EXPAND case(Enum_Center, 2) |
| //%PDDM-EXPAND case(Enum_Right, 3) |
| //%PDDM-EXPAND-END |
| } |
| |
| //%PDDM-DEFINE case(_A, _B) |
| //% case _A: |
| //% return _B; |
| |
| A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END |
| can be used to avoid adding extra blank lines/returns (or make it clear when |
| it is desired). |
| |
| One macro can invoke another by simply using its name NAME(ARGS). You cannot |
| nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). |
| |
| Within a macro you can use ## to cause things to get joined together after |
| processing (i.e. - "a##b" within a macro will become "ab"). |
| |
| |
| """ |
| |
| import optparse |
| import os |
| import re |
| import sys |
| |
| |
| # Regex for macro definition. |
| _MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)') |
| # Regex for macro's argument definition. |
| _MACRO_ARG_NAME_RE = re.compile(r'^\w+$') |
| |
| # Line inserted after each EXPAND. |
| _GENERATED_CODE_LINE = ( |
| '// This block of code is generated, do not edit it directly.' |
| ) |
| |
| |
| def _MacroRefRe(macro_names): |
| # Takes in a list of macro names and makes a regex that will match invokes |
| # of those macros. |
| return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' % |
| '|'.join(macro_names)) |
| |
| |
| def _MacroArgRefRe(macro_arg_names): |
| # Takes in a list of macro arg names and makes a regex that will match |
| # uses of those args. |
| return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' % |
| '|'.join(macro_arg_names)) |
| |
| |
| class PDDMError(Exception): |
| """Error thrown by pddm.""" |
| |
| def __init__(self, message="Error"): |
| self.message = message |
| super().__init__(self.message) |
| |
| |
| class MacroCollection(object): |
| """Hold a set of macros and can resolve/expand them.""" |
| |
| def __init__(self, a_file=None): |
| """Initializes the collection. |
| |
| Args: |
| a_file: The file like stream to parse. |
| |
| Raises: |
| PDDMError if there are any issues. |
| """ |
| self._macros = dict() |
| if a_file: |
| self.ParseInput(a_file) |
| |
| class MacroDefinition(object): |
| """Holds a macro definition.""" |
| |
| def __init__(self, name, arg_names): |
| self._name = name |
| self._args = tuple(arg_names) |
| self._body = '' |
| self._needNewLine = False |
| |
| def AppendLine(self, line): |
| if self._needNewLine: |
| self._body += '\n' |
| self._body += line |
| self._needNewLine = not line.endswith('\n') |
| |
| @property |
| def name(self): |
| return self._name |
| |
| @property |
| def args(self): |
| return self._args |
| |
| @property |
| def body(self): |
| return self._body |
| |
| def ParseInput(self, a_file): |
| """Consumes input extracting definitions. |
| |
| Args: |
| a_file: The file like stream to parse. |
| |
| Raises: |
| PDDMError if there are any issues. |
| """ |
| input_lines = a_file.read().splitlines() |
| self.ParseLines(input_lines) |
| |
| def ParseLines(self, input_lines): |
| """Parses list of lines. |
| |
| Args: |
| input_lines: A list of strings of input to parse (no newlines on the |
| strings). |
| |
| Raises: |
| PDDMError if there are any issues. |
| """ |
| current_macro = None |
| for line in input_lines: |
| if line.startswith('PDDM-'): |
| directive = line.split(' ', 1)[0] |
| if directive == 'PDDM-DEFINE': |
| name, args = self._ParseDefineLine(line) |
| if self._macros.get(name): |
| raise PDDMError('Attempt to redefine macro: "%s"' % line) |
| current_macro = self.MacroDefinition(name, args) |
| self._macros[name] = current_macro |
| continue |
| if directive == 'PDDM-DEFINE-END': |
| if not current_macro: |
| raise PDDMError('Got DEFINE-END directive without an active macro:' |
| ' "%s"' % line) |
| current_macro = None |
| continue |
| raise PDDMError('Hit a line with an unknown directive: "%s"' % line) |
| |
| if current_macro: |
| current_macro.AppendLine(line) |
| continue |
| |
| # Allow blank lines between macro definitions. |
| if line.strip() == '': |
| continue |
| |
| raise PDDMError('Hit a line that wasn\'t a directive and no open macro' |
| ' definition: "%s"' % line) |
| |
| def _ParseDefineLine(self, input_line): |
| assert input_line.startswith('PDDM-DEFINE') |
| line = input_line[12:].strip() |
| match = _MACRO_RE.match(line) |
| # Must match full line |
| if match is None or match.group(0) != line: |
| raise PDDMError('Failed to parse macro definition: "%s"' % input_line) |
| name = match.group('name') |
| args_str = match.group('args').strip() |
| args = [] |
| if args_str: |
| for part in args_str.split(','): |
| arg = part.strip() |
| if arg == '': |
| raise PDDMError('Empty arg name in macro definition: "%s"' |
| % input_line) |
| if not _MACRO_ARG_NAME_RE.match(arg): |
| raise PDDMError('Invalid arg name "%s" in macro definition: "%s"' |
| % (arg, input_line)) |
| if arg in args: |
| raise PDDMError('Arg name "%s" used more than once in macro' |
| ' definition: "%s"' % (arg, input_line)) |
| args.append(arg) |
| return (name, tuple(args)) |
| |
| def Expand(self, macro_ref_str): |
| """Expands the macro reference. |
| |
| Args: |
| macro_ref_str: String of a macro reference (i.e. foo(a, b)). |
| |
| Returns: |
| The text from the expansion. |
| |
| Raises: |
| PDDMError if there are any issues. |
| """ |
| match = _MACRO_RE.match(macro_ref_str) |
| if match is None or match.group(0) != macro_ref_str: |
| raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str) |
| if match.group('name') not in self._macros: |
| raise PDDMError('No macro named "%s".' % match.group('name')) |
| return self._Expand(match, [], macro_ref_str) |
| |
| def _FormatStack(self, macro_ref_stack): |
| result = '' |
| for _, macro_ref in reversed(macro_ref_stack): |
| result += '\n...while expanding "%s".' % macro_ref |
| return result |
| |
| def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None): |
| if macro_ref_str is None: |
| macro_ref_str = macro_ref_match.group('macro_ref') |
| name = macro_ref_match.group('name') |
| for prev_name, prev_macro_ref in macro_stack: |
| if name == prev_name: |
| raise PDDMError('Found macro recursion, invoking "%s":%s' % |
| (macro_ref_str, self._FormatStack(macro_stack))) |
| macro = self._macros[name] |
| args_str = macro_ref_match.group('args').strip() |
| args = [] |
| if args_str or len(macro.args): |
| args = [x.strip() for x in args_str.split(',')] |
| if len(args) != len(macro.args): |
| raise PDDMError('Expected %d args, got: "%s".%s' % |
| (len(macro.args), macro_ref_str, |
| self._FormatStack(macro_stack))) |
| # Replace args usages. |
| result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack) |
| # Expand any macro invokes. |
| new_macro_stack = macro_stack + [(name, macro_ref_str)] |
| while True: |
| eval_result = self._EvalMacrosRefs(result, new_macro_stack) |
| # Consume all ## directives to glue things together. |
| eval_result = eval_result.replace('##', '') |
| if eval_result == result: |
| break |
| result = eval_result |
| return result |
| |
| def _ReplaceArgValues(self, |
| macro, arg_values, macro_ref_to_report, macro_stack): |
| if len(arg_values) == 0: |
| # Nothing to do |
| return macro.body |
| assert len(arg_values) == len(macro.args) |
| args = dict(list(zip(macro.args, arg_values))) |
| |
| def _lookupArg(match): |
| val = args[match.group('name')] |
| opt = match.group('option') |
| if opt: |
| if opt == 'S': # Spaces for the length |
| return ' ' * len(val) |
| elif opt == 'l': # Lowercase first character |
| if val: |
| return val[0].lower() + val[1:] |
| else: |
| return val |
| elif opt == 'L': # All Lowercase |
| return val.lower() |
| elif opt == 'u': # Uppercase first character |
| if val: |
| return val[0].upper() + val[1:] |
| else: |
| return val |
| elif opt == 'U': # All Uppercase |
| return val.upper() |
| else: |
| raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s' |
| % (match.group('name'), match.group('option'), |
| macro_ref_to_report, |
| self._FormatStack(macro_stack))) |
| return val |
| # Let the regex do the work! |
| macro_arg_ref_re = _MacroArgRefRe(macro.args) |
| return macro_arg_ref_re.sub(_lookupArg, macro.body) |
| |
| def _EvalMacrosRefs(self, text, macro_stack): |
| macro_ref_re = _MacroRefRe(list(self._macros.keys())) |
| |
| def _resolveMacro(match): |
| return self._Expand(match, macro_stack) |
| return macro_ref_re.sub(_resolveMacro, text) |
| |
| |
| class SourceFile(object): |
| """Represents a source file with PDDM directives in it.""" |
| |
| def __init__(self, a_file, import_resolver=None): |
| """Initializes the file reading in the file. |
| |
| Args: |
| a_file: The file to read in. |
| import_resolver: a function that given a path will return a stream for |
| the contents. |
| |
| Raises: |
| PDDMError if there are any issues. |
| """ |
| self._sections = [] |
| self._original_content = a_file.read() |
| self._import_resolver = import_resolver |
| self._processed_content = None |
| |
| class SectionBase(object): |
| |
| def __init__(self, first_line_num): |
| self._lines = [] |
| self._first_line_num = first_line_num |
| |
| def TryAppend(self, line, line_num): |
| """Try appending a line. |
| |
| Args: |
| line: The line to append. |
| line_num: The number of the line. |
| |
| Returns: |
| A tuple of (SUCCESS, CAN_ADD_MORE). If SUCCESS if False, the line |
| wasn't append. If SUCCESS is True, then CAN_ADD_MORE is True/False to |
| indicate if more lines can be added after this one. |
| """ |
| assert False, "subclass should have overridden" |
| return (False, False) |
| |
| def HitEOF(self): |
| """Called when the EOF was reached for for a given section.""" |
| pass |
| |
| def BindMacroCollection(self, macro_collection): |
| """Binds the chunk to a macro collection. |
| |
| Args: |
| macro_collection: The collection to bind too. |
| """ |
| pass |
| |
| def Append(self, line): |
| self._lines.append(line) |
| |
| @property |
| def lines(self): |
| return self._lines |
| |
| @property |
| def num_lines_captured(self): |
| return len(self._lines) |
| |
| @property |
| def first_line_num(self): |
| return self._first_line_num |
| |
| @property |
| def first_line(self): |
| if not self._lines: |
| return '' |
| return self._lines[0] |
| |
| @property |
| def text(self): |
| return '\n'.join(self.lines) + '\n' |
| |
| class TextSection(SectionBase): |
| """Text section that is echoed out as is.""" |
| |
| def TryAppend(self, line, line_num): |
| if line.startswith('//%PDDM'): |
| return (False, False) |
| self.Append(line) |
| return (True, True) |
| |
| class ExpansionSection(SectionBase): |
| """Section that is the result of an macro expansion.""" |
| |
| def __init__(self, first_line_num): |
| SourceFile.SectionBase.__init__(self, first_line_num) |
| self._macro_collection = None |
| |
| def TryAppend(self, line, line_num): |
| if line.startswith('//%PDDM'): |
| directive = line.split(' ', 1)[0] |
| if directive == '//%PDDM-EXPAND': |
| self.Append(line) |
| return (True, True) |
| if directive == '//%PDDM-EXPAND-END': |
| assert self.num_lines_captured > 0 |
| return (True, False) |
| raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % |
| (directive, line_num, self.first_line)) |
| # Eat other lines. |
| return (True, True) |
| |
| def HitEOF(self): |
| raise PDDMError('Hit the end of the file while in "%s".' % |
| self.first_line) |
| |
| def BindMacroCollection(self, macro_collection): |
| self._macro_collection = macro_collection |
| |
| @property |
| def lines(self): |
| captured_lines = SourceFile.SectionBase.lines.fget(self) |
| directive_len = len('//%PDDM-EXPAND') |
| result = [] |
| for line in captured_lines: |
| result.append(line) |
| if self._macro_collection: |
| # Always add a blank line, seems to read better. (If need be, add an |
| # option to the EXPAND to indicate if this should be done.) |
| result.extend([_GENERATED_CODE_LINE, '']) |
| macro = line[directive_len:].strip() |
| try: |
| expand_result = self._macro_collection.Expand(macro) |
| # Since expansions are line oriented, strip trailing whitespace |
| # from the lines. |
| lines = [x.rstrip() for x in expand_result.split('\n')] |
| result.append('\n'.join(lines)) |
| except PDDMError as e: |
| raise PDDMError('%s\n...while expanding "%s" from the section' |
| ' that started:\n Line %d: %s' % |
| (e.message, macro, |
| self.first_line_num, self.first_line)) |
| |
| # Add the ending marker. |
| if len(captured_lines) == 1: |
| result.append('//%%PDDM-EXPAND-END %s' % |
| captured_lines[0][directive_len:].strip()) |
| else: |
| result.append('//%%PDDM-EXPAND-END (%s expansions)' % |
| len(captured_lines)) |
| return result |
| |
| class DefinitionSection(SectionBase): |
| """Section containing macro definitions""" |
| |
| def TryAppend(self, line, line_num): |
| if not line.startswith('//%'): |
| return (False, False) |
| if line.startswith('//%PDDM'): |
| directive = line.split(' ', 1)[0] |
| if directive == "//%PDDM-EXPAND": |
| return False, False |
| if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'): |
| raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % |
| (directive, line_num, self.first_line)) |
| self.Append(line) |
| return (True, True) |
| |
| def BindMacroCollection(self, macro_collection): |
| if macro_collection: |
| try: |
| # Parse the lines after stripping the prefix. |
| macro_collection.ParseLines([x[3:] for x in self.lines]) |
| except PDDMError as e: |
| raise PDDMError('%s\n...while parsing section that started:\n' |
| ' Line %d: %s' % |
| (e.message, self.first_line_num, self.first_line)) |
| |
| class ImportDefinesSection(SectionBase): |
| """Section containing an import of PDDM-DEFINES from an external file.""" |
| |
| def __init__(self, first_line_num, import_resolver): |
| SourceFile.SectionBase.__init__(self, first_line_num) |
| self._import_resolver = import_resolver |
| |
| def TryAppend(self, line, line_num): |
| if not line.startswith('//%PDDM-IMPORT-DEFINES '): |
| return (False, False) |
| assert self.num_lines_captured == 0 |
| self.Append(line) |
| return (True, False) |
| |
| def BindMacroCollection(self, macro_collection): |
| if not macro_collection: |
| return |
| if self._import_resolver is None: |
| raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):' |
| ' "%s".' % (self.first_line_num, self.first_line)) |
| import_name = self.first_line.split(' ', 1)[1].strip() |
| imported_file = self._import_resolver(import_name) |
| if imported_file is None: |
| raise PDDMError('Resolver failed to find "%s" (line %d):' |
| ' "%s".' % |
| (import_name, self.first_line_num, self.first_line)) |
| try: |
| imported_src_file = SourceFile(imported_file, self._import_resolver) |
| imported_src_file._ParseFile() |
| for section in imported_src_file._sections: |
| section.BindMacroCollection(macro_collection) |
| except PDDMError as e: |
| raise PDDMError('%s\n...while importing defines:\n' |
| ' Line %d: %s' % |
| (e.message, self.first_line_num, self.first_line)) |
| |
| def _ParseFile(self): |
| self._sections = [] |
| lines = self._original_content.splitlines() |
| cur_section = None |
| for line_num, line in enumerate(lines, 1): |
| if not cur_section: |
| cur_section = self._MakeSection(line, line_num) |
| was_added, accept_more = cur_section.TryAppend(line, line_num) |
| if not was_added: |
| cur_section = self._MakeSection(line, line_num) |
| was_added, accept_more = cur_section.TryAppend(line, line_num) |
| assert was_added |
| if not accept_more: |
| cur_section = None |
| |
| if cur_section: |
| cur_section.HitEOF() |
| |
| def _MakeSection(self, line, line_num): |
| if not line.startswith('//%PDDM'): |
| section = self.TextSection(line_num) |
| else: |
| directive = line.split(' ', 1)[0] |
| if directive == '//%PDDM-EXPAND': |
| section = self.ExpansionSection(line_num) |
| elif directive == '//%PDDM-DEFINE': |
| section = self.DefinitionSection(line_num) |
| elif directive == '//%PDDM-IMPORT-DEFINES': |
| section = self.ImportDefinesSection(line_num, self._import_resolver) |
| else: |
| raise PDDMError('Unexpected line %d: "%s".' % (line_num, line)) |
| self._sections.append(section) |
| return section |
| |
| def ProcessContent(self, strip_expansion=False): |
| """Processes the file contents.""" |
| self._ParseFile() |
| if strip_expansion: |
| # Without a collection the expansions become blank, removing them. |
| collection = None |
| else: |
| collection = MacroCollection() |
| for section in self._sections: |
| section.BindMacroCollection(collection) |
| result = '' |
| for section in self._sections: |
| result += section.text |
| self._processed_content = result |
| |
| @property |
| def original_content(self): |
| return self._original_content |
| |
| @property |
| def processed_content(self): |
| return self._processed_content |
| |
| |
| def main(args): |
| usage = '%prog [OPTIONS] PATH ...' |
| description = ( |
| 'Processes PDDM directives in the given paths and write them back out.' |
| ) |
| parser = optparse.OptionParser(usage=usage, description=description) |
| parser.add_option('--dry-run', |
| default=False, action='store_true', |
| help='Don\'t write back to the file(s), just report if the' |
| ' contents needs an update and exit with a value of 1.') |
| parser.add_option('--verbose', |
| default=False, action='store_true', |
| help='Reports is a file is already current.') |
| parser.add_option('--collapse', |
| default=False, action='store_true', |
| help='Removes all the generated code.') |
| opts, extra_args = parser.parse_args(args) |
| |
| if not extra_args: |
| parser.error('Need at least one file to process') |
| |
| result = 0 |
| for a_path in extra_args: |
| if not os.path.exists(a_path): |
| sys.stderr.write('ERROR: File not found: %s\n' % a_path) |
| return 100 |
| |
| def _ImportResolver(name): |
| # resolve based on the file being read. |
| a_dir = os.path.dirname(a_path) |
| import_path = os.path.join(a_dir, name) |
| if not os.path.exists(import_path): |
| return None |
| return open(import_path, 'r') |
| |
| with open(a_path, 'r') as f: |
| src_file = SourceFile(f, _ImportResolver) |
| |
| try: |
| src_file.ProcessContent(strip_expansion=opts.collapse) |
| except PDDMError as e: |
| sys.stderr.write('ERROR: %s\n...While processing "%s"\n' % |
| (e.message, a_path)) |
| return 101 |
| |
| if src_file.processed_content != src_file.original_content: |
| if not opts.dry_run: |
| print('Updating for "%s".' % a_path) |
| with open(a_path, 'w') as f: |
| f.write(src_file.processed_content) |
| else: |
| # Special result to indicate things need updating. |
| print('Update needed for "%s".' % a_path) |
| result = 1 |
| elif opts.verbose: |
| print('No update for "%s".' % a_path) |
| |
| return result |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |
