blob: 6b38398fe1c1befafed17b50ee80d4c355a62ba1 [file] [log] [blame]
Tim Swast29c83ba2020-02-11 13:40:17 -06001#!/usr/bin/env python
2# Protocol Buffers - Google's data interchange format
3# Copyright 2008 Google Inc. All rights reserved.
4# https://developers.google.com/protocol-buffers/
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met:
9#
10# * Redistributions of source code must retain the above copyright
11# notice, this list of conditions and the following disclaimer.
12# * Redistributions in binary form must reproduce the above
13# copyright notice, this list of conditions and the following disclaimer
14# in the documentation and/or other materials provided with the
15# distribution.
16# * Neither the name of Google Inc. nor the names of its
17# contributors may be used to endorse or promote products derived from
18# this software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32"""Script to generate a list of all modules to use in autosummary.
33
34This script creates a ReStructured Text file for each public module in the
35protobuf Python package. The script also updates the table of contents in
36``docs/index.rst`` to point to these module references.
37
38To build the docs with Sphinx:
39
401. Install the needed packages (``sphinx``, ``sphinxcontrib-napoleon`` for
41 Google-style docstring support). I've created a conda environment file to
42 make this easier:
43
44.. code:: bash
45
46 conda env create -f python/docs/environment.yml
47
482. (Optional) Generate reference docs files and regenerate index:
49
50.. code:: bash
51
52 cd python/docs
53 python generate_docs.py
54
553. Run Sphinx.
56
57.. code:: bash
58
59 make html
60"""
61
62import pathlib
63import re
64
65
66DOCS_DIR = pathlib.Path(__file__).parent.resolve()
67PYTHON_DIR = DOCS_DIR.parent
68SOURCE_DIR = PYTHON_DIR / "google" / "protobuf"
69SOURCE_POSIX = SOURCE_DIR.as_posix()
David L. Jonesc9531822020-04-22 21:41:12 -070070
71# Modules which are always included:
72INCLUDED_MODULES = (
73 "google.protobuf.internal.containers",
74)
75
76# Packages to ignore, including all modules (unless in INCLUDED_MODULES):
Tim Swast29c83ba2020-02-11 13:40:17 -060077IGNORED_PACKAGES = (
78 "compiler",
David L. Jonesc9531822020-04-22 21:41:12 -070079 "docs",
Tim Swast29c83ba2020-02-11 13:40:17 -060080 "internal",
81 "pyext",
82 "util",
83)
David L. Jonesc9531822020-04-22 21:41:12 -070084
85# Ignored module stems in all packages (unless in INCLUDED_MODULES):
Tim Swast29c83ba2020-02-11 13:40:17 -060086IGNORED_MODULES = (
87 "any_test_pb2",
88 "api_pb2",
89 "unittest",
90 "source_context_pb2",
91 "test_messages_proto3_pb2",
92 "test_messages_proto2",
93)
David L. Jonesc9531822020-04-22 21:41:12 -070094
Tim Swast29c83ba2020-02-11 13:40:17 -060095TOC_REGEX = re.compile(
96 r"\.\. START REFTOC.*\.\. END REFTOC\.\n",
97 flags=re.DOTALL,
98)
99TOC_TEMPLATE = """.. START REFTOC, generated by generate_docs.py.
100.. toctree::
101
102 {toctree}
103
104.. END REFTOC.
105"""
106
107AUTOMODULE_TEMPLATE = """.. DO NOT EDIT, generated by generate_docs.py.
108
Tim Swast35a2bf92019-08-15 16:22:02 -0700109.. ifconfig:: build_env == 'readthedocs'
110
111 .. warning::
112
113 You are reading the documentation for the `latest committed changes
noahdietz5abf8022022-04-12 10:25:08 -0700114 <https://github.com/protocolbuffers/protobuf/tree/main/python>`_ of
Tim Swast35a2bf92019-08-15 16:22:02 -0700115 the `Protocol Buffers package for Python
116 <https://developers.google.com/protocol-buffers/docs/pythontutorial>`_.
117 Some features may not yet be released. Read the documentation for the
118 latest released package at `googleapis.dev
119 <https://googleapis.dev/python/protobuf/latest/>`_.
120
Tim Swast29c83ba2020-02-11 13:40:17 -0600121{module}
122{underline}
123
124.. automodule:: {module}
125 :members:
126 :inherited-members:
127 :undoc-members:
128"""
129
130
131def find_modules():
132 modules = []
133 for module_path in SOURCE_DIR.glob("**/*.py"):
David L. Jonesc9531822020-04-22 21:41:12 -0700134 # Determine the (dotted) relative package and module names.
135 package_path = module_path.parent.relative_to(PYTHON_DIR)
136 if package_path == SOURCE_DIR:
137 package_name = ""
138 module_name = module_path.stem
139 else:
140 package_name = package_path.as_posix().replace("/", ".")
141 module_name = package_name + "." + module_path.stem
142
143 # Filter: first, accept anything in the whitelist; then, reject anything
144 # at package level, then module name level.
145 if any(include == module_name for include in INCLUDED_MODULES):
146 pass
147 elif any(ignored in package_name for ignored in IGNORED_PACKAGES):
Tim Swast29c83ba2020-02-11 13:40:17 -0600148 continue
David L. Jonesc9531822020-04-22 21:41:12 -0700149 elif any(ignored in module_path.stem for ignored in IGNORED_MODULES):
Tim Swast29c83ba2020-02-11 13:40:17 -0600150 continue
151
Tim Swast29c83ba2020-02-11 13:40:17 -0600152 if module_path.name == "__init__.py":
153 modules.append(package_name)
154 else:
David L. Jonesc9531822020-04-22 21:41:12 -0700155 modules.append(module_name)
Tim Swast29c83ba2020-02-11 13:40:17 -0600156
157 return modules
158
159
160def write_automodule(module):
161 contents = AUTOMODULE_TEMPLATE.format(module=module, underline="=" * len(module),)
162 automodule_path = DOCS_DIR.joinpath(*module.split(".")).with_suffix(".rst")
163 try:
164 automodule_path.parent.mkdir(parents=True)
165 except FileExistsError:
166 pass
167 with open(automodule_path, "w") as automodule_file:
168 automodule_file.write(contents)
169
170
171def replace_toc(modules):
172 toctree = [module.replace(".", "/") for module in modules]
173 with open(DOCS_DIR / "index.rst", "r") as index_file:
174 index_contents = index_file.read()
175 toc = TOC_TEMPLATE.format(
176 toctree="\n ".join(toctree)
177 )
178 index_contents = re.sub(TOC_REGEX, toc, index_contents)
179 with open(DOCS_DIR / "index.rst", "w") as index_file:
180 index_file.write(index_contents)
181
182
183def main():
184 modules = list(sorted(find_modules()))
185 for module in modules:
186 print("Generating reference for {}".format(module))
187 write_automodule(module)
188 print("Generating index.rst")
189 replace_toc(modules)
190
191if __name__ == "__main__":
192 main()