docs/_extensions/external_content.py - third_party/github/project-chip/connectedhomeip - Git at Google

 """
 External content
 ################

 Copyright (c) 2021 Nordic Semiconductor ASA
 SPDX-License-Identifier: Apache-2.0

 Introduction
 ============

 This extension allows to import sources from directories out of the Sphinx
 source directory. They are copied to the source directory before starting the
 build. Note that the copy is *smart*, that is, only updated files are actually
 copied. Therefore, incremental builds detect changes correctly and behave as
 expected.

 Links to external content not included in the generated documentation are
 transformed to external links as needed.

 Configuration options
 =====================

 - ``external_content_contents``: A list of external contents. Each entry is
   a tuple with two fields: the external base directory and a file glob pattern.
 - ``external_content_link_prefixes``: A list of link prefixes out of scope.
   All links to content with these prefixes are made external.
 - ``external_content_link_extensions``: A list of file extensions in scope of
   the documentation. All links to content without these file extensions are
   made external.
 - ``external_content_keep``: A list of file globs (relative to the destination
   directory) that should be kept even if they do not exist in the source
   directory. This option can be useful for auto-generated files in the
   destination directory.
 """

 import filecmp
 import os
 from pathlib import Path
 import re
 import shutil
 import tempfile
 from typing import Dict, Any, List, Optional

 from sphinx.application import Sphinx

 __version__ = "0.1.0"

 DIRECTIVES = ("figure", "image", "include", "literalinclude")
 """Default directives for included content."""

 EXTERNAL_LINK_URL_PREFIX = (
     "https://github.com/project-chip/connectedhomeip/blob/master/"
 )


 def adjust_includes(
     fname: Path,
     basepath: Path,
     encoding: str,
     link_prefixes: List[str],
     extensions: List[str],
     targets: List[Path],
     dstpath: Optional[Path] = None,
 ) -> None:
     """Adjust included content paths.

     Args:
         fname: File to be processed.
         basepath: Base path to be used to resolve content location.
         encoding: Sources encoding.
         link_prefixes: Prefixes of links that are made external.
         extensions: Filename extensions links to which are not made external.
         targets: List of all files that are being copied.
         dstpath: Destination path for fname if its path is not the actual destination.
     """

     if fname.suffix != ".md":
         return

     dstpath = dstpath or fname.parent

     def _adjust_path(path):
         # ignore absolute paths, section links, hyperlinks and same folder
         if path.startswith(("/", "#", "http", "www")) or not "/" in path:
             return path

         # for files that are being copied modify reference to and out of /docs
         filepath = path.split("#")[0]
         absolute = (basepath / filepath).resolve()
         if absolute in targets:
             if "docs/" in path:
                 path = path.replace("docs/", "")
             elif "../examples" in path:
                 path = path.replace("../", "", 1)
             return path

         # otherwise change links to point to their targets' original location
         return Path(os.path.relpath(basepath / path, dstpath)).as_posix()

     def _adjust_links(m):
         displayed, fpath = m.groups()
         fpath_adj = _adjust_path(fpath)
         return f"[{displayed}]({fpath_adj})"

     def _adjust_external(m):
         displayed, target = m.groups()
         return f"[{displayed}]({EXTERNAL_LINK_URL_PREFIX}{target})"

     def _adjust_filetype(m):
         displayed, target, extension = m.groups()
         if extension.lower() in extensions or target.startswith("http"):
             return m.group(0)

         return f"[{displayed}]({EXTERNAL_LINK_URL_PREFIX}{target})"

     def _adjust_image_link(m):
         prefix, fpath, postfix = m.groups()
         fpath_adj = _adjust_path(fpath)
         return f"{prefix}{fpath_adj}{postfix}"

     rules = [
         # Find any links and adjust the path
         (r"\[([^\[\]]*)\]\s*\((.*)\)", _adjust_links),

         # Find links that lead to an external folder and transform it
         # into an external link.
         (
             r"\[([^\[\]]*)\]\s*\((?:\.\./)*((?:" + "|".join(link_prefixes) + r")[^)]*)\)",
             _adjust_external,
         ),

         # Find links that lead to a non-presentable filetype and transform
         # it into an external link.
         (
             r"\[([^\[\]]*)\]\s*\((?:\.\./)*((?:[^()]+?/)*[^.()]+?(\.[^)/#]+))(?:#[^)]+)?\)",
             _adjust_filetype,
         ),

         # Find links that lead to a folder and transform it into an external link.
         (
             r"\[([^\[\]]*)\]\s*\((?:\.\./)*((?:[^()]+?/)+[^).#/]+)(\))",
             _adjust_filetype,
         ),

         # Find image links in img tags and adjust them
         (r"(<img [^>]*src=[\"'])([^ >]+)([\"'][^>]*>)", _adjust_image_link)
     ]

     with open(fname, "r+", encoding=encoding) as f:
         content = f.read()
         modified = False

         for pattern, sub_func in rules:
             content, changes_made = re.subn(pattern, sub_func, content)
             modified = modified or changes_made

         if modified:
             f.seek(0)
             f.write(content)
             f.truncate()


 def sync_contents(app: Sphinx) -> None:
     """Synhronize external contents.

     Args:
         app: Sphinx application instance.
     """

     srcdir = Path(app.srcdir).resolve()
     to_copy = []
     to_delete = set(f for f in srcdir.glob("**/*") if not f.is_dir())
     to_keep = set(
         f
         for k in app.config.external_content_keep
         for f in srcdir.glob(k)
         if not f.is_dir()
     )

     for content in app.config.external_content_contents:
         prefix_src, glob = content
         for src in prefix_src.glob(glob):
             if src.is_dir():
                 to_copy.extend(
                     [(f, prefix_src) for f in src.glob("**/*") if not f.is_dir()]
                 )
             else:
                 to_copy.append((src, prefix_src))

     list_of_destinations = [f for f, _ in to_copy]

     for entry in to_copy:
         src, prefix_src = entry
         dst = (srcdir / src.relative_to(prefix_src)).resolve()

         if dst in to_delete:
             to_delete.remove(dst)

         if not dst.parent.exists():
             dst.parent.mkdir(parents=True)

         # just copy if it does not exist
         if not dst.exists():
             shutil.copy(src, dst)
             adjust_includes(
                 dst,
                 src.parent,
                 app.config.source_encoding,
                 app.config.external_content_link_prefixes,
                 app.config.external_content_link_extensions,
                 list_of_destinations,
             )
         # if origin file is modified only copy if different
         elif src.stat().st_mtime > dst.stat().st_mtime:
             with tempfile.TemporaryDirectory() as td:
                 # adjust origin includes before comparing
                 src_adjusted = Path(td) / src.name
                 shutil.copy(src, src_adjusted)
                 adjust_includes(
                     src_adjusted,
                     src.parent,
                     app.config.source_encoding,
                     app.config.external_content_link_prefixes,
                     app.config.external_content_link_extensions,
                     list_of_destinations,
                     dstpath=dst.parent,
                 )

                 if not filecmp.cmp(src_adjusted, dst):
                     dst.unlink()
                     shutil.move(os.fspath(src_adjusted), os.fspath(dst))

     # remove any previously copied file not present in the origin folder,
     # excepting those marked to be kept.
     for file in to_delete - to_keep:
         file.unlink()


 def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value("external_content_contents", [], "env")
     app.add_config_value("external_content_keep", [], "")
     app.add_config_value("external_content_link_prefixes", [], "env")
     app.add_config_value("external_content_link_extensions", [], "env")

     app.connect("builder-inited", sync_contents)

     return {
         "version": __version__,
         "parallel_read_safe": True,
         "parallel_write_safe": True,
     }
	"""
	External content
	################

	Copyright (c) 2021 Nordic Semiconductor ASA
	SPDX-License-Identifier: Apache-2.0

	Introduction
	============

	This extension allows to import sources from directories out of the Sphinx
	source directory. They are copied to the source directory before starting the
	build. Note that the copy is smart, that is, only updated files are actually
	copied. Therefore, incremental builds detect changes correctly and behave as
	expected.

	Links to external content not included in the generated documentation are
	transformed to external links as needed.

	Configuration options
	=====================

	- ``external_content_contents``: A list of external contents. Each entry is
	a tuple with two fields: the external base directory and a file glob pattern.
	- ``external_content_link_prefixes``: A list of link prefixes out of scope.
	All links to content with these prefixes are made external.
	- ``external_content_link_extensions``: A list of file extensions in scope of
	the documentation. All links to content without these file extensions are
	made external.
	- ``external_content_keep``: A list of file globs (relative to the destination
	directory) that should be kept even if they do not exist in the source
	directory. This option can be useful for auto-generated files in the
	destination directory.
	"""

	import filecmp
	import os
	from pathlib import Path
	import re
	import shutil
	import tempfile
	from typing import Dict, Any, List, Optional

	from sphinx.application import Sphinx

	__version__ = "0.1.0"

	DIRECTIVES = ("figure", "image", "include", "literalinclude")
	"""Default directives for included content."""

	EXTERNAL_LINK_URL_PREFIX = (
	"https://github.com/project-chip/connectedhomeip/blob/master/"
	)


	def adjust_includes(
	fname: Path,
	basepath: Path,
	encoding: str,
	link_prefixes: List[str],
	extensions: List[str],
	targets: List[Path],
	dstpath: Optional[Path] = None,
	) -> None:
	"""Adjust included content paths.

	Args:
	fname: File to be processed.
	basepath: Base path to be used to resolve content location.
	encoding: Sources encoding.
	link_prefixes: Prefixes of links that are made external.
	extensions: Filename extensions links to which are not made external.
	targets: List of all files that are being copied.
	dstpath: Destination path for fname if its path is not the actual destination.
	"""

	if fname.suffix != ".md":
	return

	dstpath = dstpath or fname.parent

	def _adjust_path(path):
	# ignore absolute paths, section links, hyperlinks and same folder
	if path.startswith(("/", "#", "http", "www")) or not "/" in path:
	return path

	# for files that are being copied modify reference to and out of /docs
	filepath = path.split("#")[0]
	absolute = (basepath / filepath).resolve()
	if absolute in targets:
	if "docs/" in path:
	path = path.replace("docs/", "")
	elif "../examples" in path:
	path = path.replace("../", "", 1)
	return path

	# otherwise change links to point to their targets' original location
	return Path(os.path.relpath(basepath / path, dstpath)).as_posix()

	def _adjust_links(m):
	displayed, fpath = m.groups()
	fpath_adj = _adjust_path(fpath)
	return f"[{displayed}]({fpath_adj})"

	def _adjust_external(m):
	displayed, target = m.groups()
	return f"[{displayed}]({EXTERNAL_LINK_URL_PREFIX}{target})"

	def _adjust_filetype(m):
	displayed, target, extension = m.groups()
	if extension.lower() in extensions or target.startswith("http"):
	return m.group(0)

	return f"[{displayed}]({EXTERNAL_LINK_URL_PREFIX}{target})"

	def _adjust_image_link(m):
	prefix, fpath, postfix = m.groups()
	fpath_adj = _adjust_path(fpath)
	return f"{prefix}{fpath_adj}{postfix}"

	rules = [
	# Find any links and adjust the path
	(r"\[([^\[\]])\]\s\((.*)\)", _adjust_links),

	# Find links that lead to an external folder and transform it
	# into an external link.
	(
	r"\[([^\[\]])\]\s\((?:\.\./)((?:" + "\|".join(link_prefixes) + r")[^)])\)",
	_adjust_external,
	),

	# Find links that lead to a non-presentable filetype and transform
	# it into an external link.
	(
	r"\[([^\[\]])\]\s\((?:\.\./)((?:[^()]+?/)[^.()]+?(\.[^)/#]+))(?:#[^)]+)?\)",
	_adjust_filetype,
	),

	# Find links that lead to a folder and transform it into an external link.
	(
	r"\[([^\[\]])\]\s\((?:\.\./)*((?:[^()]+?/)+[^).#/]+)(\))",
	_adjust_filetype,
	),

	# Find image links in img tags and adjust them
	(r"(<img [^>]src=[\"'])([^ >]+)([\"'][^>]>)", _adjust_image_link)
	]

	with open(fname, "r+", encoding=encoding) as f:
	content = f.read()
	modified = False

	for pattern, sub_func in rules:
	content, changes_made = re.subn(pattern, sub_func, content)
	modified = modified or changes_made

	if modified:
	f.seek(0)
	f.write(content)
	f.truncate()


	def sync_contents(app: Sphinx) -> None:
	"""Synhronize external contents.

	Args:
	app: Sphinx application instance.
	"""

	srcdir = Path(app.srcdir).resolve()
	to_copy = []
	to_delete = set(f for f in srcdir.glob("*/") if not f.is_dir())
	to_keep = set(
	f
	for k in app.config.external_content_keep
	for f in srcdir.glob(k)
	if not f.is_dir()
	)

	for content in app.config.external_content_contents:
	prefix_src, glob = content
	for src in prefix_src.glob(glob):
	if src.is_dir():
	to_copy.extend(
	[(f, prefix_src) for f in src.glob("*/") if not f.is_dir()]
	)
	else:
	to_copy.append((src, prefix_src))

	list_of_destinations = [f for f, _ in to_copy]

	for entry in to_copy:
	src, prefix_src = entry
	dst = (srcdir / src.relative_to(prefix_src)).resolve()

	if dst in to_delete:
	to_delete.remove(dst)

	if not dst.parent.exists():
	dst.parent.mkdir(parents=True)

	# just copy if it does not exist
	if not dst.exists():
	shutil.copy(src, dst)
	adjust_includes(
	dst,
	src.parent,
	app.config.source_encoding,
	app.config.external_content_link_prefixes,
	app.config.external_content_link_extensions,
	list_of_destinations,
	)
	# if origin file is modified only copy if different
	elif src.stat().st_mtime > dst.stat().st_mtime:
	with tempfile.TemporaryDirectory() as td:
	# adjust origin includes before comparing
	src_adjusted = Path(td) / src.name
	shutil.copy(src, src_adjusted)
	adjust_includes(
	src_adjusted,
	src.parent,
	app.config.source_encoding,
	app.config.external_content_link_prefixes,
	app.config.external_content_link_extensions,
	list_of_destinations,
	dstpath=dst.parent,
	)

	if not filecmp.cmp(src_adjusted, dst):
	dst.unlink()
	shutil.move(os.fspath(src_adjusted), os.fspath(dst))

	# remove any previously copied file not present in the origin folder,
	# excepting those marked to be kept.
	for file in to_delete - to_keep:
	file.unlink()


	def setup(app: Sphinx) -> Dict[str, Any]:
	app.add_config_value("external_content_contents", [], "env")
	app.add_config_value("external_content_keep", [], "")
	app.add_config_value("external_content_link_prefixes", [], "env")
	app.add_config_value("external_content_link_extensions", [], "env")

	app.connect("builder-inited", sync_contents)

	return {
	"version": __version__,
	"parallel_read_safe": True,
	"parallel_write_safe": True,
	}