blob: f8c2f48a2207baf3e7208472d73857376a4afa13 [file] [log] [blame]
"""
External content
################
Copyright (c) 2021 Nordic Semiconductor ASA
SPDX-License-Identifier: Apache-2.0
Introduction
============
This extension allows to import sources from directories out of the Sphinx
source directory. They are copied to the source directory before starting the
build. Note that the copy is *smart*, that is, only updated files are actually
copied. Therefore, incremental builds detect changes correctly and behave as
expected.
Links to external content not included in the generated documentation are
transformed to external links as needed.
Configuration options
=====================
- ``external_content_contents``: A list of external contents. Each entry is
a tuple with two fields: the external base directory and a file glob pattern.
- ``external_content_link_prefixes``: A list of link prefixes out of scope.
All links to content with these prefixes are made external.
- ``external_content_link_extensions``: A list of file extensions in scope of
the documentation. All links to content without these file extensions are
made external.
- ``external_content_keep``: A list of file globs (relative to the destination
directory) that should be kept even if they do not exist in the source
directory. This option can be useful for auto-generated files in the
destination directory.
"""
import filecmp
import os
from pathlib import Path
import re
import shutil
import tempfile
from typing import Dict, Any, List, Optional
from sphinx.application import Sphinx
__version__ = "0.1.0"
DIRECTIVES = ("figure", "image", "include", "literalinclude")
"""Default directives for included content."""
EXTERNAL_LINK_URL_PREFIX = (
"https://github.com/project-chip/connectedhomeip/blob/master/"
)
def adjust_includes(
fname: Path,
basepath: Path,
encoding: str,
link_prefixes: List[str],
extensions: List[str],
targets: List[Path],
dstpath: Optional[Path] = None,
) -> None:
"""Adjust included content paths.
Args:
fname: File to be processed.
basepath: Base path to be used to resolve content location.
encoding: Sources encoding.
link_prefixes: Prefixes of links that are made external.
extensions: Filename extensions links to which are not made external.
targets: List of all files that are being copied.
dstpath: Destination path for fname if its path is not the actual destination.
"""
if fname.suffix != ".md":
return
dstpath = dstpath or fname.parent
def _adjust_path(path):
# ignore absolute paths, section links, hyperlinks and same folder
if path.startswith(("/", "#", "http", "www")) or not "/" in path:
return path
# for files that are being copied modify reference to and out of /docs
filepath = path.split("#")[0]
absolute = (basepath / filepath).resolve()
if absolute in targets:
if "docs/" in path:
path = path.replace("docs/", "")
elif "../examples" in path:
path = path.replace("../", "", 1)
return path
# otherwise change links to point to their targets' original location
return Path(os.path.relpath(basepath / path, dstpath)).as_posix()
def _adjust_links(m):
displayed, fpath = m.groups()
fpath_adj = _adjust_path(fpath)
return f"[{displayed}]({fpath_adj})"
def _adjust_external(m):
displayed, target = m.groups()
return f"[{displayed}]({EXTERNAL_LINK_URL_PREFIX}{target})"
def _adjust_filetype(m):
displayed, target, extension = m.groups()
if extension.lower() in extensions or target.startswith("http"):
return m.group(0)
return f"[{displayed}]({EXTERNAL_LINK_URL_PREFIX}{target})"
def _adjust_image_link(m):
prefix, fpath, postfix = m.groups()
fpath_adj = _adjust_path(fpath)
return f"{prefix}{fpath_adj}{postfix}"
rules = [
# Find any links and adjust the path
(r"\[([^\[\]]*)\]\s*\((.*)\)", _adjust_links),
# Find links that lead to an external folder and transform it
# into an external link.
(
r"\[([^\[\]]*)\]\s*\((?:\.\./)*((?:" + "|".join(link_prefixes) + r")[^)]*)\)",
_adjust_external,
),
# Find links that lead to a non-presentable filetype and transform
# it into an external link.
(
r"\[([^\[\]]*)\]\s*\((?:\.\./)*((?:[^()]+?/)*[^.()]+?(\.[^)/#]+))(?:#[^)]+)?\)",
_adjust_filetype,
),
# Find links that lead to a folder and transform it into an external link.
(
r"\[([^\[\]]*)\]\s*\((?:\.\./)*((?:[^()]+?/)+[^).#/]+)(\))",
_adjust_filetype,
),
# Find image links in img tags and adjust them
(r"(<img [^>]*src=[\"'])([^ >]+)([\"'][^>]*>)", _adjust_image_link)
]
with open(fname, "r+", encoding=encoding) as f:
content = f.read()
modified = False
for pattern, sub_func in rules:
content, changes_made = re.subn(pattern, sub_func, content)
modified = modified or changes_made
if modified:
f.seek(0)
f.write(content)
f.truncate()
def sync_contents(app: Sphinx) -> None:
"""Synhronize external contents.
Args:
app: Sphinx application instance.
"""
srcdir = Path(app.srcdir).resolve()
to_copy = []
to_delete = set(f for f in srcdir.glob("**/*") if not f.is_dir())
to_keep = set(
f
for k in app.config.external_content_keep
for f in srcdir.glob(k)
if not f.is_dir()
)
for content in app.config.external_content_contents:
prefix_src, glob = content
for src in prefix_src.glob(glob):
if src.is_dir():
to_copy.extend(
[(f, prefix_src) for f in src.glob("**/*") if not f.is_dir()]
)
else:
to_copy.append((src, prefix_src))
list_of_destinations = [f for f, _ in to_copy]
for entry in to_copy:
src, prefix_src = entry
dst = (srcdir / src.relative_to(prefix_src)).resolve()
if dst in to_delete:
to_delete.remove(dst)
if not dst.parent.exists():
dst.parent.mkdir(parents=True)
# just copy if it does not exist
if not dst.exists():
shutil.copy(src, dst)
adjust_includes(
dst,
src.parent,
app.config.source_encoding,
app.config.external_content_link_prefixes,
app.config.external_content_link_extensions,
list_of_destinations,
)
# if origin file is modified only copy if different
elif src.stat().st_mtime > dst.stat().st_mtime:
with tempfile.TemporaryDirectory() as td:
# adjust origin includes before comparing
src_adjusted = Path(td) / src.name
shutil.copy(src, src_adjusted)
adjust_includes(
src_adjusted,
src.parent,
app.config.source_encoding,
app.config.external_content_link_prefixes,
app.config.external_content_link_extensions,
list_of_destinations,
dstpath=dst.parent,
)
if not filecmp.cmp(src_adjusted, dst):
dst.unlink()
shutil.move(os.fspath(src_adjusted), os.fspath(dst))
# remove any previously copied file not present in the origin folder,
# excepting those marked to be kept.
for file in to_delete - to_keep:
file.unlink()
def setup(app: Sphinx) -> Dict[str, Any]:
app.add_config_value("external_content_contents", [], "env")
app.add_config_value("external_content_keep", [], "")
app.add_config_value("external_content_link_prefixes", [], "env")
app.add_config_value("external_content_link_extensions", [], "env")
app.connect("builder-inited", sync_contents)
return {
"version": __version__,
"parallel_read_safe": True,
"parallel_write_safe": True,
}