Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Protocol Buffers - Google's data interchange format |
| 3 | # Copyright 2008 Google Inc. All rights reserved. |
| 4 | # https://developers.google.com/protocol-buffers/ |
| 5 | # |
| 6 | # Redistribution and use in source and binary forms, with or without |
| 7 | # modification, are permitted provided that the following conditions are |
| 8 | # met: |
| 9 | # |
| 10 | # * Redistributions of source code must retain the above copyright |
| 11 | # notice, this list of conditions and the following disclaimer. |
| 12 | # * Redistributions in binary form must reproduce the above |
| 13 | # copyright notice, this list of conditions and the following disclaimer |
| 14 | # in the documentation and/or other materials provided with the |
| 15 | # distribution. |
| 16 | # * Neither the name of Google Inc. nor the names of its |
| 17 | # contributors may be used to endorse or promote products derived from |
| 18 | # this software without specific prior written permission. |
| 19 | # |
| 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 31 | |
| 32 | """Script to generate a list of all modules to use in autosummary. |
| 33 | |
| 34 | This script creates a ReStructured Text file for each public module in the |
| 35 | protobuf Python package. The script also updates the table of contents in |
| 36 | ``docs/index.rst`` to point to these module references. |
| 37 | |
| 38 | To build the docs with Sphinx: |
| 39 | |
| 40 | 1. Install the needed packages (``sphinx``, ``sphinxcontrib-napoleon`` for |
| 41 | Google-style docstring support). I've created a conda environment file to |
| 42 | make this easier: |
| 43 | |
| 44 | .. code:: bash |
| 45 | |
| 46 | conda env create -f python/docs/environment.yml |
| 47 | |
| 48 | 2. (Optional) Generate reference docs files and regenerate index: |
| 49 | |
| 50 | .. code:: bash |
| 51 | |
| 52 | cd python/docs |
| 53 | python generate_docs.py |
| 54 | |
| 55 | 3. Run Sphinx. |
| 56 | |
| 57 | .. code:: bash |
| 58 | |
| 59 | make html |
| 60 | """ |
| 61 | |
| 62 | import pathlib |
| 63 | import re |
| 64 | |
| 65 | |
| 66 | DOCS_DIR = pathlib.Path(__file__).parent.resolve() |
| 67 | PYTHON_DIR = DOCS_DIR.parent |
| 68 | SOURCE_DIR = PYTHON_DIR / "google" / "protobuf" |
| 69 | SOURCE_POSIX = SOURCE_DIR.as_posix() |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 70 | |
| 71 | # Modules which are always included: |
| 72 | INCLUDED_MODULES = ( |
| 73 | "google.protobuf.internal.containers", |
| 74 | ) |
| 75 | |
| 76 | # Packages to ignore, including all modules (unless in INCLUDED_MODULES): |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 77 | IGNORED_PACKAGES = ( |
| 78 | "compiler", |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 79 | "docs", |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 80 | "internal", |
| 81 | "pyext", |
| 82 | "util", |
| 83 | ) |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 84 | |
| 85 | # Ignored module stems in all packages (unless in INCLUDED_MODULES): |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 86 | IGNORED_MODULES = ( |
| 87 | "any_test_pb2", |
| 88 | "api_pb2", |
| 89 | "unittest", |
| 90 | "source_context_pb2", |
| 91 | "test_messages_proto3_pb2", |
| 92 | "test_messages_proto2", |
| 93 | ) |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 94 | |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 95 | TOC_REGEX = re.compile( |
| 96 | r"\.\. START REFTOC.*\.\. END REFTOC\.\n", |
| 97 | flags=re.DOTALL, |
| 98 | ) |
| 99 | TOC_TEMPLATE = """.. START REFTOC, generated by generate_docs.py. |
| 100 | .. toctree:: |
| 101 | |
| 102 | {toctree} |
| 103 | |
| 104 | .. END REFTOC. |
| 105 | """ |
| 106 | |
| 107 | AUTOMODULE_TEMPLATE = """.. DO NOT EDIT, generated by generate_docs.py. |
| 108 | |
Tim Swast | 35a2bf9 | 2019-08-15 16:22:02 -0700 | [diff] [blame] | 109 | .. ifconfig:: build_env == 'readthedocs' |
| 110 | |
| 111 | .. warning:: |
| 112 | |
| 113 | You are reading the documentation for the `latest committed changes |
noahdietz | 5abf802 | 2022-04-12 10:25:08 -0700 | [diff] [blame] | 114 | <https://github.com/protocolbuffers/protobuf/tree/main/python>`_ of |
Tim Swast | 35a2bf9 | 2019-08-15 16:22:02 -0700 | [diff] [blame] | 115 | the `Protocol Buffers package for Python |
| 116 | <https://developers.google.com/protocol-buffers/docs/pythontutorial>`_. |
| 117 | Some features may not yet be released. Read the documentation for the |
| 118 | latest released package at `googleapis.dev |
| 119 | <https://googleapis.dev/python/protobuf/latest/>`_. |
| 120 | |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 121 | {module} |
| 122 | {underline} |
| 123 | |
| 124 | .. automodule:: {module} |
| 125 | :members: |
| 126 | :inherited-members: |
| 127 | :undoc-members: |
| 128 | """ |
| 129 | |
| 130 | |
| 131 | def find_modules(): |
| 132 | modules = [] |
| 133 | for module_path in SOURCE_DIR.glob("**/*.py"): |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 134 | # Determine the (dotted) relative package and module names. |
| 135 | package_path = module_path.parent.relative_to(PYTHON_DIR) |
| 136 | if package_path == SOURCE_DIR: |
| 137 | package_name = "" |
| 138 | module_name = module_path.stem |
| 139 | else: |
| 140 | package_name = package_path.as_posix().replace("/", ".") |
| 141 | module_name = package_name + "." + module_path.stem |
| 142 | |
| 143 | # Filter: first, accept anything in the whitelist; then, reject anything |
| 144 | # at package level, then module name level. |
| 145 | if any(include == module_name for include in INCLUDED_MODULES): |
| 146 | pass |
| 147 | elif any(ignored in package_name for ignored in IGNORED_PACKAGES): |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 148 | continue |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 149 | elif any(ignored in module_path.stem for ignored in IGNORED_MODULES): |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 150 | continue |
| 151 | |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 152 | if module_path.name == "__init__.py": |
| 153 | modules.append(package_name) |
| 154 | else: |
David L. Jones | c953182 | 2020-04-22 21:41:12 -0700 | [diff] [blame] | 155 | modules.append(module_name) |
Tim Swast | 29c83ba | 2020-02-11 13:40:17 -0600 | [diff] [blame] | 156 | |
| 157 | return modules |
| 158 | |
| 159 | |
| 160 | def write_automodule(module): |
| 161 | contents = AUTOMODULE_TEMPLATE.format(module=module, underline="=" * len(module),) |
| 162 | automodule_path = DOCS_DIR.joinpath(*module.split(".")).with_suffix(".rst") |
| 163 | try: |
| 164 | automodule_path.parent.mkdir(parents=True) |
| 165 | except FileExistsError: |
| 166 | pass |
| 167 | with open(automodule_path, "w") as automodule_file: |
| 168 | automodule_file.write(contents) |
| 169 | |
| 170 | |
| 171 | def replace_toc(modules): |
| 172 | toctree = [module.replace(".", "/") for module in modules] |
| 173 | with open(DOCS_DIR / "index.rst", "r") as index_file: |
| 174 | index_contents = index_file.read() |
| 175 | toc = TOC_TEMPLATE.format( |
| 176 | toctree="\n ".join(toctree) |
| 177 | ) |
| 178 | index_contents = re.sub(TOC_REGEX, toc, index_contents) |
| 179 | with open(DOCS_DIR / "index.rst", "w") as index_file: |
| 180 | index_file.write(index_contents) |
| 181 | |
| 182 | |
| 183 | def main(): |
| 184 | modules = list(sorted(find_modules())) |
| 185 | for module in modules: |
| 186 | print("Generating reference for {}".format(module)) |
| 187 | write_automodule(module) |
| 188 | print("Generating index.rst") |
| 189 | replace_toc(modules) |
| 190 | |
| 191 | if __name__ == "__main__": |
| 192 | main() |