Refactor config loading and git tree parsing

sphinx_multiversion/git.py

@@ -5,23 +5,20 @@ import subprocess
 import re
 import tarfile
 
-from . import sphinx
-
-VersionRef = collections.namedtuple('VersionRef', [
+GitRef = collections.namedtuple('VersionRef', [
     'name',
     'commit',
     'source',
     'is_remote',
     'refname',
-    'version',
-    'release',
 ])
 
 
-def get_refs(gitroot):
+def get_all_refs(gitroot):
     cmd = ("git", "for-each-ref", "--format", "%(objectname) %(refname)", "refs")
     output = subprocess.check_output(cmd, cwd=gitroot).decode()
     for line in output.splitlines():
+        is_remote = False
         line = line.strip()
         # Parse refname
         matchobj = re.match(r"^(\w+) refs/(heads|tags|remotes/[^/]+)/(\S+)$", line)
@@ -31,40 +28,30 @@ def get_refs(gitroot):
         source = matchobj.group(2)
         name = matchobj.group(3)
         refname = line.partition(' ')[2]
+        if source.startswith('remotes/'):
+            is_remote = True
 
-        yield (name, commit, source, refname)
-
-
-def get_conf(gitroot, refname, confpath):
-    objectname = "{}:{}".format(refname, confpath)
-    cmd = ("git", "show", objectname)
-    return subprocess.check_output(cmd, cwd=gitroot).decode()
+        yield GitRef(name, commit, source, is_remote, refname)
 
 
-def find_versions(gitroot, confpath, tag_whitelist, branch_whitelist, remote_whitelist):
-    for name, commit, source, refname in get_refs(gitroot):
-        is_remote = False
-        if source == 'tags':
-            if tag_whitelist is None or not re.match(tag_whitelist, name):
+def get_refs(gitroot, tag_whitelist, branch_whitelist, remote_whitelist):
+    for ref in get_all_refs(gitroot):
+        if ref.source == 'tags':
+            if tag_whitelist is None or not re.match(tag_whitelist, ref.name):
                 continue
-        elif source == 'heads':
-            if branch_whitelist is None or not re.match(branch_whitelist, name):
+        elif ref.source == 'heads':
+            if branch_whitelist is None or not re.match(branch_whitelist, ref.name):
                 continue
-        elif source.startswith('remotes/') and remote_whitelist is not None:
-            is_remote = True
-            remote_name = source.partition('/')[2]
+        elif ref.is_remote and remote_whitelist is not None:
+            remote_name = ref.source.partition('/')[2]
             if not re.match(remote_whitelist, remote_name):
                 continue
-            if branch_whitelist is None or not re.match(branch_whitelist, name):
+            if branch_whitelist is None or not re.match(branch_whitelist, ref.name):
                 continue
         else:
             continue
 
-        conf = get_conf(gitroot, refname, confpath)
-        config = sphinx.parse_conf(conf)
-        version = config['version']
-        release = config['release']
-        yield VersionRef(name, commit, source, is_remote, refname, version, release)
+        yield ref
 
 
 def copy_tree(src, dst, reference, sourcepath='.'):

sphinx_multiversion/main.py

@@ -1,13 +1,16 @@
 # -*- coding: utf-8 -*-
-import os
-import re
+import argparse
 import json
+import logging
+import os
 import pathlib
+import re
 import subprocess
 import sys
 import tempfile
 
 from sphinx.cmd import build as sphinx_build
+from sphinx import config as sphinx_config
 from sphinx import project as sphinx_project
 
 from . import sphinx
@@ -18,89 +21,115 @@ def main(argv=None):
     if not argv:
         argv = sys.argv[1:]
 
-    parser = sphinx_build.get_parser()
-    args = parser.parse_args(argv)
-
-    # Find the indices
-    srcdir_index = None
-    outdir_index = None
-    for i, value in enumerate(argv):
-        if value == args.sourcedir:
-            argv[i] = '{{{SOURCEDIR}}}'
-            test_args = parser.parse_args(argv)
-            if test_args.sourcedir == argv[i]:
-                srcdir_index = i
-            argv[i] = args.sourcedir
-
-        if value == args.outputdir:
-            argv[i] = '{{{OUTPUTDIR}}}'
-            test_args = parser.parse_args(argv)
-            if test_args.outputdir == argv[i]:
-                outdir_index = i
-            argv[i] = args.outputdir
-
-    if srcdir_index is None:
-        raise ValueError("Failed to find srcdir index")
-    if outdir_index is None:
-        raise ValueError("Failed to find outdir index")
-
-    # Parse config
-    confpath = os.path.join(args.confdir, 'conf.py')
-    with open(confpath, mode='r') as f:
-        config = sphinx.parse_conf(f.read())
-
+    parser = argparse.ArgumentParser()
+    parser.add_argument('sourcedir', help='path to documentation source files')
+    parser.add_argument('outputdir', help='path to output directory')
+    parser.add_argument('filenames', nargs='*', help='a list of specific files to rebuild. Ignored if -a is specified')
+    parser.add_argument('-c', metavar='PATH', dest='confdir', help='path where configuration file (conf.py) is located (default: same as SOURCEDIR)')
+    parser.add_argument('-C', action='store_true', dest='noconfig', help='use no config file at all, only -D options')
+    parser.add_argument('-D', metavar='setting=value', action='append', dest='define', default=[], help='override a setting in configuration file')
+    parser.add_argument('--dump-metadata', action='store_true', help='dump generated metadata and exit')
+    args, argv = parser.parse_known_args(argv)
+    if args.noconfig:
+        return 1
+
+    # Conf-overrides
+    confoverrides = {}
     for d in args.define:
         key, _, value = d.partition('=')
-        config[key] = value
-
-    tag_whitelist = config.get('smv_tag_whitelist', sphinx.DEFAULT_TAG_WHITELIST)
-    branch_whitelist = config.get('smv_branch_whitelist', sphinx.DEFAULT_BRANCH_WHITELIST)
-    remote_whitelist = config.get('smv_remote_whitelist', sphinx.DEFAULT_REMOTE_WHITELIST)
-    released_pattern = config.get('smv_released_pattern', sphinx.DEFAULT_RELEASED_PATTERN)
-    outputdir_format = config.get('smv_outputdir_format', sphinx.DEFAULT_OUTPUTDIR_FORMAT)
+        confoverrides[key] = value
 
+    # Parse config
+    config = sphinx_config.Config.read(
+        os.path.abspath(args.confdir if args.confdir else args.sourcedir),
+        confoverrides,
+    )
+    config.add("smv_tag_whitelist", sphinx.DEFAULT_TAG_WHITELIST, "html", str)
+    config.add("smv_branch_whitelist", sphinx.DEFAULT_TAG_WHITELIST, "html", str)
+    config.add("smv_remote_whitelist", sphinx.DEFAULT_REMOTE_WHITELIST, "html", str)
+    config.add("smv_released_pattern", sphinx.DEFAULT_RELEASED_PATTERN, "html", str)
+    config.add("smv_outputdir_format", sphinx.DEFAULT_OUTPUTDIR_FORMAT, "html", str)
+
+    # Get git references
     gitroot = pathlib.Path('.').resolve()
-    versions = git.find_versions(str(gitroot), 'source/conf.py', tag_whitelist, branch_whitelist, remote_whitelist)
+    gitrefs = git.get_refs(
+        str(gitroot),
+        config.smv_tag_whitelist,
+        config.smv_branch_whitelist,
+        config.smv_remote_whitelist,
+    )
+
+    logger = logging.getLogger(__name__)
+
+    # Get Sourcedir
+    sourcedir = os.path.relpath(args.sourcedir, str(gitroot))
+    if args.confdir:
+        confdir = os.path.relpath(args.confdir, str(gitroot))
+    else:
+        confdir = sourcedir
 
     with tempfile.TemporaryDirectory() as tmp:
         # Generate Metadata
         metadata = {}
         outputdirs = set()
-        sourcedir = os.path.relpath(args.sourcedir, str(gitroot))
-        for versionref in versions:
-            # Ensure that there are not duplicate output dirs
-            outputdir = sphinx.format_outputdir(
-                outputdir_format, versionref, language=config["language"])
-            if outputdir in outputdirs:
-                print("outputdir '%s' of version %r conflicts with other versions!"
-                      % (outputdir, versionref))
-                continue
-            outputdirs.add(outputdir)
-
+        for gitref in gitrefs:
             # Clone Git repo
-            repopath = os.path.join(tmp, str(hash(versionref)))
-            srcdir = os.path.join(repopath, sourcedir)
+            repopath = os.path.join(tmp, gitref.commit)
             try:
-                git.copy_tree(gitroot.as_uri(), repopath, versionref)
+                git.copy_tree(gitroot.as_uri(), repopath, gitref)
             except (OSError, subprocess.CalledProcessError):
-                outputdirs.remove(outputdir)
+                logger.error(
+                    "Failed to copy git tree for %s to %s",
+                    gitref.refname, repopath)
+                continue
+
+            # Find config
+            confpath = os.path.join(repopath, confdir)
+            try:
+                current_config = sphinx_config.Config.read(
+                    confpath,
+                    confoverrides,
+                )
+            except sphinx_config.ConfigError:
+                logger.error(
+                    "Failed load config for %s from %s",
+                    gitref.refname, confpath)
+                continue
+
+            # Ensure that there are not duplicate output dirs
+            outputdir = config.smv_outputdir_format.format(
+                ref=gitref,
+                config=current_config,
+            )
+            if outputdir in outputdirs:
+                logger.warning(
+                    "outputdir '%s' for %s conflicts with other versions",
+                    outputdir, gitref.name)
                 continue
+            outputdirs.add(outputdir)
 
             # Get List of files
-            source_suffixes = config.get("source_suffix", "")
+            source_suffixes = current_config.source_suffix
             if isinstance(source_suffixes, str):
-                source_suffixes = [source_suffixes]
-            project = sphinx_project.Project(srcdir, source_suffixes)
-            metadata[versionref.name] = {
-                "name": versionref.name,
-                "version": versionref.version,
-                "release": versionref.release,
-                "is_released": bool(re.match(released_pattern, versionref.refname)),
-                "source": versionref.source,
-                "sourcedir": srcdir,
+                source_suffixes = [current_config.source_suffix]
+            project = sphinx_project.Project(sourcedir, source_suffixes)
+            metadata[gitref.name] = {
+                "name": gitref.name,
+                "version": current_config.version,
+                "release": current_config.release,
+                "is_released": bool(
+                    re.match(config.smv_released_pattern, gitref.refname)),
+                "source": gitref.source,
+                "sourcedir": sourcedir,
                 "outputdir": outputdir,
                 "docnames": list(project.discover())
             }
+
+        if args.dump_metadata:
+            print(json.dumps(metadata, indent=2))
+            return
+
+        # Write Metadata
         metadata_path = os.path.abspath(os.path.join(tmp, "versions.json"))
         with open(metadata_path, mode='w') as fp:
             json.dump(metadata, fp, indent=2)
@@ -108,15 +137,18 @@ def main(argv=None):
         # Run Sphinx
         argv.extend(["-D", "smv_metadata_path={}".format(metadata_path)])
         for version_name, data in metadata.items():
+            outdir = os.path.join(args.outputdir, data["outputdir"])
+            os.makedirs(outdir, exist_ok=True)
+
             current_argv = argv.copy()
             current_argv.extend([
+                *args.define,
                 "-D", "smv_current_version={}".format(version_name),
+                "-c", args.confdir,
+                data["sourcedir"],
+                outdir,
+                *args.filenames,
             ])
-
-            outdir = os.path.join(args.outputdir, data["outputdir"])
-            current_argv[srcdir_index] = data["sourcedir"]
-            current_argv[outdir_index] = outdir
-            os.makedirs(outdir, exist_ok=True)
             status = sphinx_build.build_main(current_argv)
             if status not in (0, None):
                 break

sphinx_multiversion/sphinx.py

@@ -13,7 +13,7 @@ DEFAULT_TAG_WHITELIST = r'^.*$'
 DEFAULT_BRANCH_WHITELIST = r'^.*$'
 DEFAULT_REMOTE_WHITELIST = None
 DEFAULT_RELEASED_PATTERN = r'^tags/.*$'
-DEFAULT_OUTPUTDIR_FORMAT = r'{version.version}/{language}'
+DEFAULT_OUTPUTDIR_FORMAT = r'{config.version}/{config.language}'
 
 Version = collections.namedtuple('Version', [
     'name',
@@ -107,10 +107,6 @@ def parse_conf(config):
     return module
 
 
-def format_outputdir(fmt, versionref, language):
-    return fmt.format(version=versionref, language=language)
-
-
 def html_page_context(app, pagename, templatename, context, doctree):
     versioninfo = VersionInfo(
         app, context, app.config.smv_metadata, app.config.smv_current_version)