From fec416142c01714efe229e35cfa1c3c9534a7832 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Dec 2019 11:20:06 -0800 Subject: [PATCH 1/5] DOC: Deduplicate code for Sphinx's APIdoc generation The ``InterfaceHelpWriter`` class was practically a clone of ``ApiDocWriter``. Both have been merged into one single module, having the Interface helper inherit from the ApiDocWriter. --- tools/apigen.py | 395 +++++++++++++++++++++---- tools/build_interface_docs.py | 42 +-- tools/interfacedocgen.py | 529 ---------------------------------- 3 files changed, 366 insertions(+), 600 deletions(-) delete mode 100644 tools/interfacedocgen.py diff --git a/tools/apigen.py b/tools/apigen.py index e3198664f3..19e47b5c20 100644 --- a/tools/apigen.py +++ b/tools/apigen.py @@ -21,25 +21,59 @@ project. """ import os +import sys import re +import tempfile +import warnings + +from nipype.interfaces.base import BaseInterface +from nipype.pipeline.engine import Workflow +from nipype.utils.misc import trim + +from github import get_file_url + +RST_SECTION_LEVELS = ("*", "=", "-", "~", "^") + +RST_CLASS_BLOCK = """ +.. _{uri}.{cls}: + +.. index:: {cls} + +{cls} +{underline} +`Link to code <{code_url}>`__ + +{body} +""" + +RST_FUNC_BLOCK = """ +.. _{uri}.{name}: + +:func:`{name}` +{underline} +`Link to code <{code_url}>`__ + +{body} + +""" # Functions and classes class ApiDocWriter(object): - """ Class for automatic detection and parsing of API docs - to Sphinx-parsable reST format""" + """Write reST documents for API docs.""" # only separating first two levels - rst_section_levels = ["*", "=", "-", "~", "^"] + rst_section_levels = RST_SECTION_LEVELS def __init__( self, package_name, rst_extension=".rst", - package_skip_patterns=None, - module_skip_patterns=None, + package_skip_patterns=(r"\.tests$",), + module_skip_patterns=(r"\.setup$", r"\._"), ): - """ Initialize package for parsing + r""" + Initialize package for parsing. Parameters ---------- @@ -55,7 +89,7 @@ def __init__( if *package_name* is ``sphinx``, then ``sphinx.util`` will result in ``.util`` being passed for earching by these regexps. If is None, gives default. Default is: - ['\.tests$'] + ``('\.tests$', )``. module_skip_patterns : None or sequence Sequence of strings giving URIs of modules to be excluded Operates on the module name including preceding URI path, @@ -63,22 +97,24 @@ def __init__( ``sphinx.util.console`` results in the string to search of ``.util.console`` If is None, gives default. Default is: - ['\.setup$', '\._'] + ``('\.setup$', '\._')``. + """ - if package_skip_patterns is None: - package_skip_patterns = ["\\.tests$"] - if module_skip_patterns is None: - module_skip_patterns = ["\\.setup$", "\\._"] - self.package_name = package_name + self._skip_patterns = {} self.rst_extension = rst_extension + self.package_name = package_name self.package_skip_patterns = package_skip_patterns self.module_skip_patterns = module_skip_patterns - def get_package_name(self): + @property + def package_name(self): + """Get package name.""" return self._package_name - def set_package_name(self, package_name): - """ Set package_name + @package_name.setter + def package_name(self, name): + """ + Set package_name. >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx @@ -88,19 +124,36 @@ def set_package_name(self, package_name): >>> import docutils >>> docwriter.root_path == docutils.__path__[0] True + """ # It's also possible to imagine caching the module parsing here - self._package_name = package_name - self.root_module = __import__(package_name) + self._package_name = name + self.root_module = __import__(name) self.root_path = self.root_module.__path__[0] self.written_modules = None - package_name = property( - get_package_name, set_package_name, None, "get/set package_name" - ) + @property + def package_skip_patterns(self): + """Get package skip patterns.""" + return self._skip_patterns['package'] + + @package_skip_patterns.setter + def package_skip_patterns(self, pattern): + self._skip_patterns['package'] = _parse_patterns(pattern) + + @property + def module_skip_patterns(self): + """Get module skip patterns.""" + return self._skip_patterns['module'] + + @module_skip_patterns.setter + def module_skip_patterns(self, pattern): + self._skip_patterns['module'] = _parse_patterns(pattern) def _get_object_name(self, line): - """ Get second token in line + """ + Get second token in line. + >>> docwriter = ApiDocWriter('sphinx') >>> docwriter._get_object_name(" def func(): ") u'func' @@ -115,7 +168,8 @@ def _get_object_name(self, line): return name.rstrip(":") def _uri2path(self, uri): - """ Convert uri to absolute filepath + """ + Convert uri to absolute filepath. Parameters ---------- @@ -157,25 +211,25 @@ def _uri2path(self, uri): return path def _path2uri(self, dirpath): - """ Convert directory path to uri """ + """Convert directory path to uri.""" relpath = dirpath.replace(self.root_path, self.package_name) if relpath.startswith(os.path.sep): relpath = relpath[1:] return relpath.replace(os.path.sep, ".") def _parse_module(self, uri): - """ Parse module defined in *uri* """ + """Parse module defined in ``uri``.""" filename = self._uri2path(uri) if filename is None: # nothing that we could handle here. return ([], []) f = open(filename, "rt") - functions, classes = self._parse_lines(f) + functions, classes = self._parse_lines(f, uri) f.close() return functions, classes - def _parse_lines(self, linesource): - """ Parse lines of text for functions and classes """ + def _parse_lines(self, linesource, module=None): + """Parse lines of text for functions and classes.""" functions = [] classes = [] for line in linesource: @@ -196,7 +250,8 @@ def _parse_lines(self, linesource): return functions, classes def generate_api_doc(self, uri): - """Make autodoc documentation template string for a module + """ + Make autodoc documentation template string for a module. Parameters ---------- @@ -207,6 +262,7 @@ def generate_api_doc(self, uri): ------- S : string Contents of API doc + """ # get the names of all classes and functions functions, classes = self._parse_module(uri) @@ -272,7 +328,8 @@ def generate_api_doc(self, uri): return ad def _survives_exclude(self, matchstr, match_type): - """ Returns True if *matchstr* does not match patterns + r""" + Return ``True`` if ``matchstr`` does not match patterns. ``self.package_name`` removed from front of string if present @@ -281,41 +338,38 @@ def _survives_exclude(self, matchstr, match_type): >>> dw = ApiDocWriter('sphinx') >>> dw._survives_exclude('sphinx.okpkg', 'package') True - >>> dw.package_skip_patterns.append('^\\.badpkg$') + >>> dw.package_skip_patterns.append(r'^\.badpkg$') >>> dw._survives_exclude('sphinx.badpkg', 'package') False >>> dw._survives_exclude('sphinx.badpkg', 'module') True >>> dw._survives_exclude('sphinx.badmod', 'module') True - >>> dw.module_skip_patterns.append('^\\.badmod$') + >>> dw.module_skip_patterns.append(r'^\.badmod$') >>> dw._survives_exclude('sphinx.badmod', 'module') False + """ - if match_type == "module": - patterns = self.module_skip_patterns - elif match_type == "package": - patterns = self.package_skip_patterns - else: + patterns = self._skip_patterns.get(match_type) + if patterns is None: raise ValueError('Cannot interpret match type "%s"' % match_type) + # Match to URI without package name L = len(self.package_name) if matchstr[:L] == self.package_name: matchstr = matchstr[L:] for pat in patterns: - # print (pat, matchstr, match_type) #dbg try: pat.search except AttributeError: pat = re.compile(pat) - # print (pat.search(matchstr)) #dbg if pat.search(matchstr): return False return True - def discover_modules(self): - """ Return module sequence discovered from ``self.package_name`` - + def discover_modules(self, empty_start=True): + r""" + Return module sequence discovered from ``self.package_name``. Parameters ---------- @@ -336,8 +390,9 @@ def discover_modules(self): >>> 'sphinx.util' in dw.discover_modules() False >>> + """ - modules = [] + modules = [] if empty_start else [self.package_name] # raw directory parsing for dirpath, dirnames, filenames in os.walk(self.root_path): # Check directory names for packages @@ -358,11 +413,10 @@ def discover_modules(self): module_uri, "module" ): modules.append(module_uri) - # print sorted(modules) #dbg return sorted(modules) def write_modules_api(self, modules, outdir): - # write the list + """Generate the list of modules.""" written_modules = [] for m in modules: api_str = self.generate_api_doc(m) @@ -377,7 +431,8 @@ def write_modules_api(self, modules, outdir): self.written_modules = written_modules def write_api_docs(self, outdir): - """Generate API reST files. + """ + Generate API reST files. Parameters ---------- @@ -391,7 +446,8 @@ def write_api_docs(self, outdir): Notes ----- - Sets self.written_modules to list of written modules + Sets ``self.written_modules`` to list of written modules + """ if not os.path.exists(outdir): os.mkdir(outdir) @@ -399,8 +455,10 @@ def write_api_docs(self, outdir): modules = self.discover_modules() self.write_modules_api(modules, outdir) - def write_index(self, outdir, froot="gen", relative_to=None): - """Make a reST API index file from written files + def write_index(self, outdir, froot="gen", relative_to=None, + maxdepth=None): + """ + Make a reST API index file from written files. Parameters ---------- @@ -416,6 +474,7 @@ def write_index(self, outdir, froot="gen", relative_to=None): component of the written file path will be removed from outdir, in the generated index. Default is None, meaning, leave path as it is. + """ if self.written_modules is None: raise ValueError("No modules written") @@ -429,7 +488,243 @@ def write_index(self, outdir, froot="gen", relative_to=None): idx = open(path, "wt") w = idx.write w(".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n") - w(".. toctree::\n\n") + if maxdepth is None: + w(".. toctree::\n\n") + else: + w(".. toctree::\n") + w(" :maxdepth: %d\n\n" % maxdepth) for f in self.written_modules: w(" %s\n" % os.path.join(relpath, f)) idx.close() + + +class InterfaceHelpWriter(ApiDocWriter): + """Convert interface specs to rST.""" + + def __init__( + self, + package_name, + class_skip_patterns=None, + **kwargs + ): + """ + Initialize an :py:mod:`ApiDocWriter` for interface specs. + + Additional Parameters + --------------------- + class_skip_patterns : None or sequence + Sequence of strings giving classes to be excluded + Default is: None + + """ + super().__init__(package_name, **kwargs) + self.class_skip_patterns = class_skip_patterns + + @property + def class_skip_patterns(self): + """Get class skip patterns.""" + return self._skip_patterns['class'] + + @class_skip_patterns.setter + def class_skip_patterns(self, pattern): + self._skip_patterns['class'] = _parse_patterns(pattern) + + def _parse_lines(self, linesource, module=None): + """Parse lines of text for functions and classes.""" + functions = [] + classes = [] + for line in linesource: + if line.startswith("def ") and line.count("("): + # exclude private stuff + name = self._get_object_name(line) + if not name.startswith("_"): + functions.append(name) + elif line.startswith("class "): + # exclude private stuff + name = self._get_object_name(line) + if not name.startswith("_") and self._survives_exclude( + ".".join((module, name)), "class" + ): + classes.append(name) + else: + pass + functions.sort() + classes.sort() + return functions, classes + + def _write_graph_section(self, fname, title): + ad = "\n%s\n%s\n\n" % (title, self.rst_section_levels[3] * len(title)) + ad += ".. graphviz::\n\n" + fhandle = open(fname) + for line in fhandle: + ad += "\t" + line + "\n" + + fhandle.close() + os.remove(fname) + bitmap_fname = "{}.png".format(os.path.splitext(fname)[0]) + os.remove(bitmap_fname) + return ad + + def generate_api_doc(self, uri): + """ + Make autodoc documentation template string for a module. + + Parameters + ---------- + uri : string + python location of module - e.g 'sphinx.builder' + + Returns + ------- + S : string + Contents of API doc + + """ + # get the names of all classes and functions + functions, classes = self._parse_module(uri) + workflows = [] + helper_functions = [] + for function in functions: + + try: + __import__(uri) + finst = sys.modules[uri].__dict__[function] + except TypeError: + continue + try: + workflow = finst() + except Exception: + helper_functions.append((function, finst)) + continue + + if isinstance(workflow, Workflow): + workflows.append((workflow, function, finst)) + + if not classes and not workflows and not helper_functions: + print("WARNING: Empty -", uri) # dbg + return "" + + # Make a shorter version of the uri that omits the package name for + # titles + uri_short = re.sub(r"^%s\." % self.package_name, "", uri) + # uri_short = uri + + ad = ".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n" + + chap_title = uri_short + ad += chap_title + "\n" + self.rst_section_levels[1] * len(chap_title) + "\n\n" + + # Set the chapter title to read 'module' for all modules except for the + # main packages + # if '.' in uri: + # title = 'Module: :mod:`' + uri_short + '`' + # else: + # title = ':mod:`' + uri_short + '`' + # ad += title + '\n' + self.rst_section_levels[2] * len(title) + + # ad += '\n' + 'Classes' + '\n' + \ + # self.rst_section_levels[2] * 7 + '\n' + for c in classes: + __import__(uri) + print(c) + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + classinst = sys.modules[uri].__dict__[c] + except Exception as inst: + print(inst) + continue + + if not issubclass(classinst, BaseInterface): + continue + + ad += RST_CLASS_BLOCK.format( + uri=uri, + cls=c, + underline=self.rst_section_levels[2] * len(c), + code_url=get_file_url(classinst), + body=trim(classinst.help(returnhelp=True), self.rst_section_levels[3]) + ) + + if workflows or helper_functions: + ad += "\n.. module:: %s\n\n" % uri + + for workflow, name, finst in workflows: + ad += RST_FUNC_BLOCK.format( + uri=uri, + name=name, + underline=self.rst_section_levels[2] * (len(name) + 8), + code_url=get_file_url(finst), + body=trim(finst.__doc__, self.rst_section_levels[3]) + ) + """ + # use sphinx autodoc for function signature + ad += '\n.. _%s:\n\n' % (uri + '.' + name) + ad += '.. autofunction:: %s\n\n' % name + """ + + (_, fname) = tempfile.mkstemp(suffix=".dot") + workflow.write_graph(dotfilename=fname, graph2use="hierarchical") + ad += self._write_graph_section(fname, "Graph") + "\n" + + for name, finst in helper_functions: + ad += RST_FUNC_BLOCK.format( + uri=uri, + name=name, + underline=self.rst_section_levels[2] * (len(name) + 8), + code_url=get_file_url(finst), + body=trim(finst.__doc__, self.rst_section_levels[3]) + ) + return ad + + def discover_modules(self, empty_start=True): + """Return module sequence discovered from ``self.package_name``.""" + return super().discover_modules(empty_start=False) + + def write_modules_api(self, modules, outdir): + """Generate the list of modules.""" + written_modules = [] + for m in modules: + api_str = self.generate_api_doc(m) + if not api_str: + continue + # write out to file + mvalues = m.split(".") + if len(mvalues) > 3: + index_prefix = ".".join(mvalues[1:3]) + index_dir = os.path.join(outdir, index_prefix) + index_file = index_dir + self.rst_extension + if not os.path.exists(index_dir): + os.makedirs(index_dir) + header = """.. AUTO-GENERATED FILE -- DO NOT EDIT! + +{name} +{underline} + +.. toctree:: + :maxdepth: 1 + :glob: + + {name}/* + """.format( + name=index_prefix, underline="=" * len(index_prefix) + ) + with open(index_file, "wt") as fp: + fp.write(header) + m = os.path.join(index_prefix, ".".join(mvalues[3:])) + outfile = os.path.join(outdir, m + self.rst_extension) + fileobj = open(outfile, "wt") + fileobj.write(api_str) + fileobj.close() + written_modules.append(m) + self.written_modules = written_modules + + +def _parse_patterns(pattern): + if pattern is None: + return [] + if isinstance(pattern, str): + return [pattern] + if isinstance(pattern, tuple): + return list(pattern) + return pattern diff --git a/tools/build_interface_docs.py b/tools/build_interface_docs.py index f53c7a0419..d21d19428a 100755 --- a/tools/build_interface_docs.py +++ b/tools/build_interface_docs.py @@ -12,36 +12,36 @@ nipypepath = os.path.abspath("..") sys.path.insert(1, nipypepath) # local imports - from interfacedocgen import InterfaceHelpWriter + from apigen import InterfaceHelpWriter package = "nipype" outdir = os.path.join("interfaces", "generated") docwriter = InterfaceHelpWriter(package) # Packages that should not be included in generated API docs. docwriter.package_skip_patterns += [ - "\.external$", - "\.fixes$", - "\.utils$", - "\.pipeline", - "\.testing", - "\.caching", - "\.scripts", + r"\.external$", + r"\.fixes$", + r"\.utils$", + r"\.pipeline", + r"\.testing", + r"\.caching", + r"\.scripts", ] # Modules that should not be included in generated API docs. docwriter.module_skip_patterns += [ - "\.version$", - "\.interfaces\.base$", - "\.interfaces\.matlab$", - "\.interfaces\.rest$", - "\.interfaces\.pymvpa$", - "\.interfaces\.slicer\.generate_classes$", - "\.interfaces\.spm\.base$", - "\.interfaces\.traits", - "\.pipeline\.alloy$", - "\.pipeline\.s3_node_wrapper$", - "\.testing", - "\.scripts", - "\.conftest", + r"\.version$", + r"\.interfaces\.base$", + r"\.interfaces\.matlab$", + r"\.interfaces\.rest$", + r"\.interfaces\.pymvpa$", + r"\.interfaces\.slicer\.generate_classes$", + r"\.interfaces\.spm\.base$", + r"\.interfaces\.traits", + r"\.pipeline\.alloy$", + r"\.pipeline\.s3_node_wrapper$", + r"\.testing", + r"\.scripts", + r"\.conftest", ] docwriter.class_skip_patterns += [ "AFNICommand", diff --git a/tools/interfacedocgen.py b/tools/interfacedocgen.py deleted file mode 100644 index 27f45ec887..0000000000 --- a/tools/interfacedocgen.py +++ /dev/null @@ -1,529 +0,0 @@ -# -*- coding: utf-8 -*- -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -"""Attempt to generate templates for module reference with Sphinx - -XXX - we exclude extension modules - -To include extension modules, first identify them as valid in the -``_uri2path`` method, then handle them in the ``_parse_module`` script. - -We get functions and classes by parsing the text of .py files. -Alternatively we could import the modules for discovery, and we'd have -to do that for extension modules. This would involve changing the -``_parse_module`` method to work via import and introspection, and -might involve changing ``discover_modules`` (which determines which -files are modules, and therefore which module URIs will be passed to -``_parse_module``). - -NOTE: this is a modified version of a script originally shipped with the -PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed -project.""" - -# Stdlib imports -import inspect -import os -import re -import sys -import tempfile -import warnings - -from nipype.interfaces.base import BaseInterface -from nipype.pipeline.engine import Workflow -from nipype.utils.misc import trim - -from github import get_file_url - -# Functions and classes - - -class InterfaceHelpWriter(object): - """ Class for automatic detection and parsing of API docs - to Sphinx-parsable reST format""" - - # only separating first two levels - rst_section_levels = ["*", "=", "-", "~", "^"] - - def __init__( - self, - package_name, - rst_extension=".rst", - package_skip_patterns=None, - module_skip_patterns=None, - class_skip_patterns=None, - ): - """ Initialize package for parsing - - Parameters - ---------- - package_name : string - Name of the top-level package. *package_name* must be the - name of an importable package - rst_extension : string, optional - Extension for reST files, default '.rst' - package_skip_patterns : None or sequence of {strings, regexps} - Sequence of strings giving URIs of packages to be excluded - Operates on the package path, starting at (including) the - first dot in the package path, after *package_name* - so, - if *package_name* is ``sphinx``, then ``sphinx.util`` will - result in ``.util`` being passed for earching by these - regexps. If is None, gives default. Default is: - ['\.tests$'] - module_skip_patterns : None or sequence - Sequence of strings giving URIs of modules to be excluded - Operates on the module name including preceding URI path, - back to the first dot after *package_name*. For example - ``sphinx.util.console`` results in the string to search of - ``.util.console`` - If is None, gives default. Default is: - ['\.setup$', '\._'] - class_skip_patterns : None or sequence - Sequence of strings giving classes to be excluded - Default is: None - - """ - if package_skip_patterns is None: - package_skip_patterns = ["\\.tests$"] - if module_skip_patterns is None: - module_skip_patterns = ["\\.setup$", "\\._"] - if class_skip_patterns: - self.class_skip_patterns = class_skip_patterns - else: - self.class_skip_patterns = [] - self.package_name = package_name - self.rst_extension = rst_extension - self.package_skip_patterns = package_skip_patterns - self.module_skip_patterns = module_skip_patterns - - def get_package_name(self): - return self._package_name - - def set_package_name(self, package_name): - """ Set package_name - - >>> docwriter = ApiDocWriter('sphinx') - >>> import sphinx - >>> docwriter.root_path == sphinx.__path__[0] - True - >>> docwriter.package_name = 'docutils' - >>> import docutils - >>> docwriter.root_path == docutils.__path__[0] - True - """ - # It's also possible to imagine caching the module parsing here - self._package_name = package_name - self.root_module = __import__(package_name) - self.root_path = self.root_module.__path__[0] - self.written_modules = None - - package_name = property( - get_package_name, set_package_name, None, "get/set package_name" - ) - - def _get_object_name(self, line): - """ Get second token in line - >>> docwriter = ApiDocWriter('sphinx') - >>> docwriter._get_object_name(" def func(): ") - u'func' - >>> docwriter._get_object_name(" class Klass(object): ") - 'Klass' - >>> docwriter._get_object_name(" class Klass: ") - 'Klass' - """ - name = line.split()[1].split("(")[0].strip() - # in case we have classes which are not derived from object - # ie. old style classes - return name.rstrip(":") - - def _uri2path(self, uri): - """ Convert uri to absolute filepath - - Parameters - ---------- - uri : string - URI of python module to return path for - - Returns - ------- - path : None or string - Returns None if there is no valid path for this URI - Otherwise returns absolute file system path for URI - - Examples - -------- - >>> docwriter = ApiDocWriter('sphinx') - >>> import sphinx - >>> modpath = sphinx.__path__[0] - >>> res = docwriter._uri2path('sphinx.builder') - >>> res == os.path.join(modpath, 'builder.py') - True - >>> res = docwriter._uri2path('sphinx') - >>> res == os.path.join(modpath, '__init__.py') - True - >>> docwriter._uri2path('sphinx.does_not_exist') - - """ - if uri == self.package_name: - return os.path.join(self.root_path, "__init__.py") - path = uri.replace(".", os.path.sep) - path = path.replace(self.package_name + os.path.sep, "") - path = os.path.join(self.root_path, path) - # XXX maybe check for extensions as well? - if os.path.exists(path + ".py"): # file - path += ".py" - elif os.path.exists(os.path.join(path, "__init__.py")): - path = os.path.join(path, "__init__.py") - else: - return None - return path - - def _path2uri(self, dirpath): - """ Convert directory path to uri """ - relpath = dirpath.replace(self.root_path, self.package_name) - if relpath.startswith(os.path.sep): - relpath = relpath[1:] - return relpath.replace(os.path.sep, ".") - - def _parse_module(self, uri): - """ Parse module defined in *uri* """ - filename = self._uri2path(uri) - if filename is None: - # nothing that we could handle here. - return ([], []) - f = open(filename, "rt") - functions, classes = self._parse_lines(f, uri) - f.close() - return functions, classes - - def _parse_lines(self, linesource, module): - """ Parse lines of text for functions and classes """ - functions = [] - classes = [] - for line in linesource: - if line.startswith("def ") and line.count("("): - # exclude private stuff - name = self._get_object_name(line) - if not name.startswith("_"): - functions.append(name) - elif line.startswith("class "): - # exclude private stuff - name = self._get_object_name(line) - if not name.startswith("_") and self._survives_exclude( - ".".join((module, name)), "class" - ): - classes.append(name) - else: - pass - functions.sort() - classes.sort() - return functions, classes - - def _write_graph_section(self, fname, title): - ad = "\n%s\n%s\n\n" % (title, self.rst_section_levels[3] * len(title)) - ad += ".. graphviz::\n\n" - fhandle = open(fname) - for line in fhandle: - ad += "\t" + line + "\n" - - fhandle.close() - os.remove(fname) - bitmap_fname = "{}.png".format(os.path.splitext(fname)[0]) - os.remove(bitmap_fname) - return ad - - def generate_api_doc(self, uri): - """Make autodoc documentation template string for a module - - Parameters - ---------- - uri : string - python location of module - e.g 'sphinx.builder' - - Returns - ------- - S : string - Contents of API doc - """ - # get the names of all classes and functions - functions, classes = self._parse_module(uri) - workflows = [] - helper_functions = [] - for function in functions: - - try: - __import__(uri) - finst = sys.modules[uri].__dict__[function] - except TypeError: - continue - try: - workflow = finst() - except Exception: - helper_functions.append((function, finst)) - continue - - if isinstance(workflow, Workflow): - workflows.append((workflow, function, finst)) - - if not classes and not workflows and not helper_functions: - print("WARNING: Empty -", uri) # dbg - return "" - - # Make a shorter version of the uri that omits the package name for - # titles - uri_short = re.sub(r"^%s\." % self.package_name, "", uri) - # uri_short = uri - - ad = ".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n" - - chap_title = uri_short - ad += chap_title + "\n" + self.rst_section_levels[1] * len(chap_title) + "\n\n" - - # Set the chapter title to read 'module' for all modules except for the - # main packages - # if '.' in uri: - # title = 'Module: :mod:`' + uri_short + '`' - # else: - # title = ':mod:`' + uri_short + '`' - # ad += title + '\n' + self.rst_section_levels[2] * len(title) - - # ad += '\n' + 'Classes' + '\n' + \ - # self.rst_section_levels[2] * 7 + '\n' - for c in classes: - __import__(uri) - print(c) - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - classinst = sys.modules[uri].__dict__[c] - except Exception as inst: - print(inst) - continue - - if not issubclass(classinst, BaseInterface): - continue - - label = uri + "." + c + ":" - ad += "\n.. _%s\n\n" % label - ad += "\n.. index:: %s\n\n" % c - ad += c + "\n" + self.rst_section_levels[2] * len(c) + "\n\n" - ad += "`Link to code <%s>`__\n\n" % get_file_url(classinst) - ad += ( - trim(classinst.help(returnhelp=True), self.rst_section_levels[3]) + "\n" - ) - - if workflows or helper_functions: - ad += "\n.. module:: %s\n\n" % uri - - for workflow, name, finst in workflows: - label = ":func:`" + name + "`" - ad += "\n.. _%s:\n\n" % (uri + "." + name) - ad += "\n".join((label, self.rst_section_levels[2] * len(label))) - ad += "\n\n`Link to code <%s>`__\n\n" % get_file_url(finst) - helpstr = trim(finst.__doc__, self.rst_section_levels[3]) - ad += "\n\n" + helpstr + "\n\n" - """ - # use sphinx autodoc for function signature - ad += '\n.. _%s:\n\n' % (uri + '.' + name) - ad += '.. autofunction:: %s\n\n' % name - """ - - (_, fname) = tempfile.mkstemp(suffix=".dot") - workflow.write_graph(dotfilename=fname, graph2use="hierarchical") - - ad += self._write_graph_section(fname, "Graph") + "\n" - - for name, finst in helper_functions: - label = ":func:`" + name + "`" - ad += "\n.. _%s:\n\n" % (uri + "." + name) - ad += "\n".join((label, self.rst_section_levels[2] * len(label))) - ad += "\n\n`Link to code <%s>`__\n\n" % get_file_url(finst) - helpstr = trim(finst.__doc__, self.rst_section_levels[3]) - ad += "\n\n" + helpstr + "\n\n" - - return ad - - def _survives_exclude(self, matchstr, match_type): - """ Returns True if *matchstr* does not match patterns - - ``self.package_name`` removed from front of string if present - - Examples - -------- - >>> dw = ApiDocWriter('sphinx') - >>> dw._survives_exclude('sphinx.okpkg', 'package') - True - >>> dw.package_skip_patterns.append('^\\.badpkg$') - >>> dw._survives_exclude('sphinx.badpkg', 'package') - False - >>> dw._survives_exclude('sphinx.badpkg', 'module') - True - >>> dw._survives_exclude('sphinx.badmod', 'module') - True - >>> dw.module_skip_patterns.append('^\\.badmod$') - >>> dw._survives_exclude('sphinx.badmod', 'module') - False - """ - if match_type == "module": - patterns = self.module_skip_patterns - elif match_type == "package": - patterns = self.package_skip_patterns - elif match_type == "class": - patterns = self.class_skip_patterns - else: - raise ValueError('Cannot interpret match type "%s"' % match_type) - # Match to URI without package name - L = len(self.package_name) - if matchstr[:L] == self.package_name: - matchstr = matchstr[L:] - for pat in patterns: - try: - pat.search - except AttributeError: - pat = re.compile(pat) - if pat.search(matchstr): - return False - return True - - def discover_modules(self): - """ Return module sequence discovered from ``self.package_name`` - - - Parameters - ---------- - None - - Returns - ------- - mods : sequence - Sequence of module names within ``self.package_name`` - - Examples - -------- - >>> dw = ApiDocWriter('sphinx') - >>> mods = dw.discover_modules() - >>> 'sphinx.util' in mods - True - >>> dw.package_skip_patterns.append('\.util$') - >>> 'sphinx.util' in dw.discover_modules() - False - >>> - """ - modules = [self.package_name] - # raw directory parsing - for dirpath, dirnames, filenames in os.walk(self.root_path): - # Check directory names for packages - root_uri = self._path2uri(os.path.join(self.root_path, dirpath)) - for dirname in dirnames[:]: # copy list - we modify inplace - package_uri = ".".join((root_uri, dirname)) - if self._uri2path(package_uri) and self._survives_exclude( - package_uri, "package" - ): - modules.append(package_uri) - else: - dirnames.remove(dirname) - # Check filenames for modules - for filename in filenames: - module_name = filename[:-3] - module_uri = ".".join((root_uri, module_name)) - if self._uri2path(module_uri) and self._survives_exclude( - module_uri, "module" - ): - modules.append(module_uri) - return sorted(modules) - - def write_modules_api(self, modules, outdir): - # write the list - written_modules = [] - for m in modules: - api_str = self.generate_api_doc(m) - if not api_str: - continue - # write out to file - mvalues = m.split(".") - if len(mvalues) > 3: - index_prefix = ".".join(mvalues[1:3]) - index_dir = os.path.join(outdir, index_prefix) - index_file = index_dir + self.rst_extension - if not os.path.exists(index_dir): - os.makedirs(index_dir) - header = """.. AUTO-GENERATED FILE -- DO NOT EDIT! - -{name} -{underline} - -.. toctree:: - :maxdepth: 1 - :glob: - - {name}/* - """.format( - name=index_prefix, underline="=" * len(index_prefix) - ) - with open(index_file, "wt") as fp: - fp.write(header) - m = os.path.join(index_prefix, ".".join(mvalues[3:])) - outfile = os.path.join(outdir, m + self.rst_extension) - fileobj = open(outfile, "wt") - fileobj.write(api_str) - fileobj.close() - written_modules.append(m) - self.written_modules = written_modules - - def write_api_docs(self, outdir): - """Generate API reST files. - - Parameters - ---------- - outdir : string - Directory name in which to store files - We create automatic filenames for each module - - Returns - ------- - None - - Notes - ----- - Sets self.written_modules to list of written modules - """ - if not os.path.exists(outdir): - os.mkdir(outdir) - # compose list of modules - modules = self.discover_modules() - self.write_modules_api(modules, outdir) - - def write_index(self, outdir, froot="gen", relative_to=None): - """Make a reST API index file from written files - - Parameters - ---------- - path : string - Filename to write index to - outdir : string - Directory to which to write generated index file - froot : string, optional - root (filename without extension) of filename to write to - Defaults to 'gen'. We add ``self.rst_extension``. - relative_to : string - path to which written filenames are relative. This - component of the written file path will be removed from - outdir, in the generated index. Default is None, meaning, - leave path as it is. - """ - if self.written_modules is None: - raise ValueError("No modules written") - # Get full filename path - path = os.path.join(outdir, froot + self.rst_extension) - # Path written into index is relative to rootpath - if relative_to is not None: - relpath = outdir.replace(relative_to + os.path.sep, "") - else: - relpath = outdir - idx = open(path, "wt") - w = idx.write - w(".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n") - w(".. toctree::\n") - w(" :maxdepth: 2\n\n") - for f in self.written_modules: - w(" %s\n" % os.path.join(relpath, f)) - idx.close() From ef25a77ea735f667e24093adcf8b429340a8c7a7 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Dec 2019 17:11:25 -0800 Subject: [PATCH 2/5] DOC: Deep revision of documentation building This PR follows up on #3119 (merge that one first for a clean diff, or diff against ``oesteban:maint/dedup-apigen-code``). In practice, this PR fixes several broken points of our documentation (e.g., the workflows list was empty and now it has been updated, changelog not rendered, API of pure python code not rendered by the Nipype API parser was missing, etc.). CHANGES ------- * Replaced the ``numpydoc`` sphinx extension with ``sphinxcontrib-napoleon``. * Removed autosummary sphinx extension, required by numpydoc * Cleared up ``docs/sphinxext/*``, as nothing is now used from there * Use current sphinx-apidoc/autodoc/autosummary * Removed the modref generation tooling, as it is not necessary anymore after re-enabling apidoc. * Cut building warnings down to 321 - just those we incur because our API generator. This required some fixes of some docstrings. Beyond those corresponding to the Nipype API generator, only missing links remain as warnings (for sections in the navbar). * Updated changelogs to be reStructuredText. --- doc/Makefile | 13 +- doc/api/index.rst | 13 +- .../{0.X.X-changelog => 0.X.X-changelog.rst} | 8 +- .../{1.X.X-changelog => 1.X.X-changelog.rst} | 42 +-- doc/changes.rst | 4 +- doc/conf.py | 112 +++++--- doc/devel/cmd_interface_devel.rst | 2 +- doc/devel/interface_specs.rst | 3 +- doc/devel/testing_nipype.rst | 2 +- doc/documentation.rst | 31 ++- doc/interfaces/index.rst | 1 - doc/links_names.txt | 2 +- doc/sphinxext/README.txt | 16 -- doc/sphinxext/autosummary_generate.py | 240 ------------------ doc/sphinxext/ipython_console_highlighting.py | 101 -------- doc/users/install.rst | 6 +- doc/version.rst | 6 +- nipype/__init__.py | 8 +- nipype/pipeline/plugins/base.py | 54 ++-- nipype/utils/config.py | 4 +- nipype/utils/filemanip.py | 6 +- nipype/utils/nipype2boutiques.py | 51 ++-- tools/README | 15 -- tools/build_interface_docs.py | 15 +- tools/build_modref_templates.py | 44 ---- tools/update_changes.sh | 2 +- 26 files changed, 234 insertions(+), 567 deletions(-) rename doc/changelog/{0.X.X-changelog => 0.X.X-changelog.rst} (99%) rename doc/changelog/{1.X.X-changelog => 1.X.X-changelog.rst} (94%) delete mode 100644 doc/sphinxext/README.txt delete mode 100755 doc/sphinxext/autosummary_generate.py delete mode 100644 doc/sphinxext/ipython_console_highlighting.py delete mode 100644 tools/README delete mode 100755 tools/build_modref_templates.py diff --git a/doc/Makefile b/doc/Makefile index abe329a57a..25acfeb122 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -5,18 +5,19 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = +PYTHONPATH = $(PWD) # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d _build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html api htmlonly latex changes linkcheck doctest +.PHONY: help clean html nipypeapi htmlonly latex changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html make the HTML documentation" - @echo " api make API documents only" + @echo " nipypeapi make interface API documents only" @echo " latex make the LaTeX, you can set PAPER=a4 or PAPER=letter" @echo " pdf make and run the PDF generation" @echo " changes make an overview of all changed/added/deprecated" \ @@ -33,14 +34,12 @@ htmlonly: @echo @echo "Build finished. The HTML pages are in _build/html." -api: - rm -rf api/generated - python -u ../tools/build_modref_templates.py +nipypeapi: rm -rf interfaces/generated python -u ../tools/build_interface_docs.py @echo "Build API docs finished." -html: clean examples2rst api htmlonly +html: clean examples2rst nipypeapi htmlonly @echo "Build HTML and API finished." examples2rst: @@ -48,7 +47,7 @@ examples2rst: ../tools/make_examples.py --no-exec @echo "examples2rst finished." -latex: api +latex: nipypeapi $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex @echo @echo "Build finished; the LaTeX files are in _build/latex." diff --git a/doc/api/index.rst b/doc/api/index.rst index 0e40dbf4ab..0cc9d87e32 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -1,10 +1,15 @@ .. _api-index: -### -API -### +########################################### +Library API (application program interface) +########################################### + +Information on specific functions, classes, and methods. :Release: |version| :Date: |today| -.. include:: generated/gen.rst +.. toctree:: + :glob: + + generated/* diff --git a/doc/changelog/0.X.X-changelog b/doc/changelog/0.X.X-changelog.rst similarity index 99% rename from doc/changelog/0.X.X-changelog rename to doc/changelog/0.X.X-changelog.rst index b1c16318b2..0c007cade7 100644 --- a/doc/changelog/0.X.X-changelog +++ b/doc/changelog/0.X.X-changelog.rst @@ -1,7 +1,7 @@ 0.14.0 (November 29, 2017) ========================== -###### [Full changelog](https://github.com/nipy/nipype/milestone/13) +(`Full changelog `__) * FIX+MAINT: Revision of the resource monitor (https://github.com/nipy/nipype/pull/2285) * FIX: MultiProc mishandling crashes (https://github.com/nipy/nipype/pull/2301) @@ -385,8 +385,10 @@ Release 0.9.0 (December 20, 2013) - camino.QBallMX - camino.LinRecon - camino.SFPeaks - One outdated interface no longer part of Camino was removed: + + One outdated interface no longer part of Camino was removed: - camino.Conmap + * ENH: Three new mrtrix interfaces were added: - mrtrix.GenerateDirections - mrtrix.FindShPeaks @@ -713,7 +715,7 @@ Features added * General: - - Type checking of inputs and outputs using Traits from ETS_. + - Type checking of inputs and outputs using Traits from ETS. - Support for nested workflows. - Preliminary Slicer and AFNI support. - New flexible DataGrabber node. diff --git a/doc/changelog/1.X.X-changelog b/doc/changelog/1.X.X-changelog.rst similarity index 94% rename from doc/changelog/1.X.X-changelog rename to doc/changelog/1.X.X-changelog.rst index 99d26fa4a8..0e0d661f61 100644 --- a/doc/changelog/1.X.X-changelog +++ b/doc/changelog/1.X.X-changelog.rst @@ -26,7 +26,7 @@ 1.3.0 (November 11, 2019) ========================= -##### [Full changelog](https://github.com/nipy/nipype/milestone/34?closed=1) +(`Full changelog `__) * FIX: Fixed typo in QwarpInputSpec Trait description (https://github.com/nipy/nipype/pull/3079) * FIX: Restore ``AFNICommand._get_fname``, required by some interfaces (https://github.com/nipy/nipype/pull/3071) @@ -52,7 +52,7 @@ Python 1.2.3 will be the last version to support Python 3.4. -##### [Full changelog](https://github.com/nipy/nipype/milestone/35?closed=1) +(`Full changelog `__) * FIX: Patch Path.mkdir for Python 2 (https://github.com/nipy/nipype/pull/3037) * FIX: Drop deprecated message argument to ``FileNotFoundError`` (https://github.com/nipy/nipype/pull/3035) @@ -71,7 +71,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.2.2 (September 07, 2019) ========================== -##### [Full changelog](https://github.com/nipy/nipype/milestone/33?closed=1) +(`Full changelog `__) * FIX: Ensure ``loadpkl`` returns a not None value (https://github.com/nipy/nipype/pull/3020) * FIX: ``loadpkl`` failed when pklz file contained versioning info (https://github.com/nipy/nipype/pull/3017) @@ -87,7 +87,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.2.1 (August 19, 2019) ======================= -##### [Full changelog](https://github.com/nipy/nipype/milestone/32?closed=1) +(`Full changelog `__) * FIX: Resolve/rebase paths from/to results files (https://github.com/nipy/nipype/pull/2971) * FIX: Use ``load_resultfile`` when loading a results pickle (https://github.com/nipy/nipype/pull/2985) @@ -97,7 +97,7 @@ Python 1.2.3 will be the last version to support Python 3.4. * FIX: Docker build (https://github.com/nipy/nipype/pull/2963) * FIX: Remove '=' signs from EddyQuad argument specifications (https://github.com/nipy/nipype/pull/2941) * FIX: Set input model to bedpostx for camino.TrackBedpostxProba (https://github.com/nipy/nipype/pull/2947) - * FIX: Allow ``max_sh``not to be set (auto mode) (https://github.com/nipy/nipype/pull/2940) + * FIX: Allow ``max_sh`` to not be set (auto mode) (https://github.com/nipy/nipype/pull/2940) * ENH: Update mrtrix reconst.py EstimateFOD max_sh to be able to accept list (https://github.com/nipy/nipype/pull/2990) * ENH: Let ``indirectory`` handle ``nipype.utils.filemanip.Path`` (https://github.com/nipy/nipype/pull/2989) * ENH: Add resolve/rebase ``BasePath`` traits methods & tests (https://github.com/nipy/nipype/pull/2970) @@ -114,7 +114,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.2.0 (May 09, 2019) ==================== -##### [Full changelog](https://github.com/nipy/nipype/milestone/31?closed=1) +(`Full changelog `__) * FIX: Parsing of filename in AlignEpiAnatPy when filename does not have + (https://github.com/nipy/nipype/pull/2909) * FIX: Import nibabel reorientation bug fix (https://github.com/nipy/nipype/pull/2912) @@ -133,7 +133,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.1.9 (February 25, 2019) ========================= -##### [Full changelog](https://github.com/nipy/nipype/milestone/30?closed=1) +(`Full changelog `__) * FIX: Make positional arguments to LaplacianThickness require previous argument (https://github.com/nipy/nipype/pull/2848) * FIX: Import math and csv modules for bids_gen_info (https://github.com/nipy/nipype/pull/2881) @@ -149,7 +149,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.1.8 (January 28, 2019) ======================== -##### [Full changelog](https://github.com/nipy/nipype/milestone/29?closed=1) +(`Full changelog `__) * FIX: ANTS LaplacianThickness cmdline opts fixed up (https://github.com/nipy/nipype/pull/2846) * FIX: Resolve LinAlgError during SVD (https://github.com/nipy/nipype/pull/2838) @@ -171,7 +171,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.1.7 (December 17, 2018) ========================= -##### [Full changelog](https://github.com/nipy/nipype/milestone/28?closed=1) +(`Full changelog `__) * FIX: Copy node list before generating a flat graph (https://github.com/nipy/nipype/pull/2828) * FIX: Update pytest req'd version to 3.6 (https://github.com/nipy/nipype/pull/2827) @@ -193,7 +193,7 @@ Python 1.2.3 will be the last version to support Python 3.4. 1.1.6 (November 26, 2018) ========================= -##### [Full changelog](https://github.com/nipy/nipype/milestone/27?closed=1) +(`Full changelog `__) * FIX: MapNodes fail when ``MultiProcPlugin`` passed by instance (https://github.com/nipy/nipype/pull/2786) * FIX: --fineTune arguments order for MeshFix command (https://github.com/nipy/nipype/pull/2780) @@ -216,7 +216,7 @@ Python 1.2.3 will be the last version to support Python 3.4. Hotfix release. -##### [Full changelog](https://github.com/nipy/nipype/milestone/26?closed=1) +(`Full changelog `__) * ENH: Allow timeouts during SLURM job status checks (https://github.com/nipy/nipype/pull/2767) * RF: Subclass non-daemon variants of all multiprocessing contexts (https://github.com/nipy/nipype/pull/2771) @@ -225,7 +225,7 @@ Hotfix release. 1.1.4 (October 31, 2018) ======================== -##### [Full changelog](https://github.com/nipy/nipype/milestone/25?closed=1) +(`Full changelog `__) * FIX: Python 2.7-3.7.1 compatible NonDaemonPool (https://github.com/nipy/nipype/pull/2754) * FIX: VRML typo (VMRL) in MeshFix (https://github.com/nipy/nipype/pull/2757) @@ -253,7 +253,7 @@ Hotfix release. 1.1.3 (September 24, 2018) ========================== -##### [Full changelog](https://github.com/nipy/nipype/milestone/24?closed=1) +(`Full changelog `__) * FIX: Return afni.Qwarp outputs as absolute paths (https://github.com/nipy/nipype/pull/2705) * FIX: Add informative error for interfaces that fail to return valid runtime object (https://github.com/nipy/nipype/pull/2692) @@ -272,7 +272,7 @@ Hotfix release. Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. -##### [Full changelog](https://github.com/nipy/nipype/milestone/23?closed=1) +(`Full changelog `__) * FIX: Read BIDS config.json under grabbids or layout (https://github.com/nipy/nipype/pull/2679) * FIX: Node __repr__ and detailed graph expansion (https://github.com/nipy/nipype/pull/2669) @@ -287,7 +287,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.1.1 (July 30, 2018) ===================== -##### [Full changelog](https://github.com/nipy/nipype/milestone/22?closed=1) +(`Full changelog `__) * FIX: Un-set incorrect default options in TOPUP (https://github.com/nipy/nipype/pull/2637) * FIX: Copy FSCommand.version to ReconAll.version (https://github.com/nipy/nipype/pull/2656) @@ -309,7 +309,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.1.0 (July 04, 2018) ===================== -###### [Full changelog](https://github.com/nipy/nipype/milestone/21?closed=1) +(`Full changelog `__) * RF: Futures-based MultiProc (https://github.com/nipy/nipype/pull/2598) * FIX: Avoid closing file descriptors on Windows (https://github.com/nipy/nipype/pull/2617) @@ -326,7 +326,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.0.4 (May 29, 2018) ==================== -###### [Full changelog](https://github.com/nipy/nipype/milestone/20?closed=1) +(`Full changelog `__) * FIX: Update logging levels in enable_debug_mode (https://github.com/nipy/nipype/pull/2595) * FIX: Set default result in DistributedPluginBase._clean_queue (https://github.com/nipy/nipype/pull/2596) @@ -349,7 +349,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.0.3 (April 30, 2018) ====================== -###### [Full changelog](https://github.com/nipy/nipype/milestone/19?closed=1) +(`Full changelog `__) * FIX: Propagate explicit Workflow config to Nodes (https://github.com/nipy/nipype/pull/2559) * FIX: Return non-enhanced volumes from dwi_flirt (https://github.com/nipy/nipype/pull/2547) @@ -376,7 +376,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.0.2 (March 27, 2018) ====================== -###### [Full changelog](https://github.com/nipy/nipype/milestone/16?closed=1) +(`Full changelog `__) * FIX: dcm2niix interface (https://github.com/nipy/nipype/pull/2498) * FIX: mark .niml.dset as special extension in utils.filemanip (https://github.com/nipy/nipype/pull/2495) @@ -399,7 +399,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.0.1 (February 27, 2018) ========================= -###### [Full changelog](https://github.com/nipy/nipype/milestone/16?closed=1) +(`Full changelog `__) * FIX: Small bug in freesurfer label2annot fill_thresh specs [#2377](https://github.com/nipy/nipype/pull/2377) * FIX: Error creating gradients in DTIRecon [#2460](https://github.com/nipy/nipype/pull/2460) @@ -432,7 +432,7 @@ Hot-fix release, resolving incorrect dependencies in 1.1.1 wheel. 1.0.0 (January 24, 2018) ======================== -###### [Full changelog](https://github.com/nipy/nipype/milestone/16?closed=1) +(`Full changelog `__) * FIX: Change to interface workdir within ``Interface.run()`` instead Node (https://github.com/nipy/nipype/pull/2384) * FIX: PBS plugin submissions (https://github.com/nipy/nipype/pull/2344) diff --git a/doc/changes.rst b/doc/changes.rst index 4585c58af8..858a907691 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -6,6 +6,8 @@ Changes in Nipype ================= -.. include:: ../CHANGES +.. include:: changelog/1.X.X-changelog.rst + +.. include:: changelog/0.X.X-changelog.rst .. include:: links_names.txt diff --git a/doc/conf.py b/doc/conf.py index c49f20e514..45bd46b97b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,49 +12,65 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os -from shutil import rmtree - -nipypepath = os.path.abspath('..') -sys.path.insert(1, nipypepath) - +from packaging.version import Version import nipype -if not os.path.exists('users/examples'): - os.mkdir('users/examples') -os.system('python ../tools/make_examples.py --no-exec') +# if not os.path.exists('users/examples'): +# os.mkdir('users/examples') +# os.system('python ../tools/make_examples.py --no-exec') -if os.path.exists('api/generated'): - rmtree('api/generated') -os.system('python ../tools/build_modref_templates.py') -if os.path.exists('interfaces/generated'): - rmtree('interfaces/generated') -os.system('python ../tools/build_interface_docs.py') +# if os.path.exists('interfaces/generated'): +# rmtree('interfaces/generated') +# os.system('python ../tools/build_interface_docs.py') # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('sphinxext')) +# sys.path.append(os.path.abspath('sphinxext')) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.todo', - 'sphinx.ext.imgmath', - 'sphinx.ext.inheritance_diagram', - 'sphinx.ext.graphviz', - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.autosummary', - 'numpydoc', - 'matplotlib.sphinxext.plot_directive', - #'matplotlib.sphinxext.only_directives', - 'nipype.sphinxext.plot_workflow', - #'IPython.sphinxext.ipython_directive', - #'IPython.sphinxext.ipython_console_highlighting' - ] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.graphviz', + 'sphinx.ext.mathjax', + 'sphinx.ext.inheritance_diagram', + 'sphinx.ext.todo', + 'sphinxcontrib.apidoc', + 'sphinxcontrib.napoleon', + 'matplotlib.sphinxext.plot_directive', + 'nipype.sphinxext.plot_workflow', +] + +autodoc_mock_imports = [ + 'matplotlib', + 'nilearn', + 'nipy', + 'nitime', + 'numpy', + 'pandas', + 'seaborn', + 'skimage', + 'svgutils', + 'transforms3d', +] + +# Accept custom section names to be parsed for numpy-style docstrings +# of parameters. +# Requires pinning sphinxcontrib-napoleon to a specific commit while +# https://github.com/sphinx-contrib/napoleon/pull/10 is merged. +napoleon_use_param = False +napoleon_custom_sections = [ + ('Inputs', 'Parameters'), + ('Outputs', 'Parameters'), + ('Attributes', 'Parameters'), +] + + on_rtd = os.environ.get('READTHEDOCS') == 'True' if on_rtd: extensions.append('readthedocs_ext.readthedocs') @@ -80,9 +96,9 @@ # built documents. # # The short X.Y version. -version = nipype.__version__ +version = Version(nipype.__version__).public # The full version, including alpha/beta/rc tags. -release = "1.3.0-rc1" +release = nipype.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -101,6 +117,15 @@ # for source files. exclude_trees = ['_build'] +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + '_build', 'Thumbs.db', '.DS_Store', + 'api/generated/gen.rst', + 'interfaces/generated/gen.rst' +] + # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None @@ -248,8 +273,29 @@ # If false, no module index is generated. #latex_use_modindex = True +# -- apidoc extension configuration ------------------------------------------ +apidoc_module_dir = '../nipype' +apidoc_output_dir = 'api/generated' +apidoc_excluded_paths = [ + '*/tests/*', 'tests/*', + 'algorithms/*', + 'external/*', + 'fixes/*', + 'interfaces/*', + 'scripts/*', + 'sphinxext/*', + 'testing/*', + 'workflows/*', + 'conftest.py', + 'info.py', + 'pkg_info.py', + 'refs.py', +] +apidoc_separate_modules = True +apidoc_extra_args = ['--module-first', '-d 1', '-T'] + +# -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} -exclude_patterns = ['interfaces/generated/gen.rst', 'api/generated/gen.rst'] diff --git a/doc/devel/cmd_interface_devel.rst b/doc/devel/cmd_interface_devel.rst index b53dfe9f5b..e0153cf678 100644 --- a/doc/devel/cmd_interface_devel.rst +++ b/doc/devel/cmd_interface_devel.rst @@ -170,7 +170,7 @@ names as arguments on the command line. We have simplified this procedure with three additional metadata terms: ``name_source``, ``name_template``, ``keep_extension``. -For example in the :ref:`InvWarp ` class, the +For example in the :ref:`InvWarp ` class, the ``inverse_warp`` parameter is the name of the output file that is created by the routine. diff --git a/doc/devel/interface_specs.rst b/doc/devel/interface_specs.rst index 26623266c6..13d44e1528 100644 --- a/doc/devel/interface_specs.rst +++ b/doc/devel/interface_specs.rst @@ -470,8 +470,7 @@ If you used genfile: And optionally: -* ``_redirect_x``: If set to True it will make Nipype start Xvfb before running the interface and redirect X output to it. This is useful for -commandlines that spawn a graphical user interface. +* ``_redirect_x``: If set to True it will make Nipype start Xvfb before running the interface and redirect X output to it. This is useful for commandlines that spawn a graphical user interface. * ``_format_arg(name, spec, value)``: For extra formatting of the input values before passing them to generic ``_parse_inputs()`` method. diff --git a/doc/devel/testing_nipype.rst b/doc/devel/testing_nipype.rst index 0cce8b4671..5713f6727b 100644 --- a/doc/devel/testing_nipype.rst +++ b/doc/devel/testing_nipype.rst @@ -90,7 +90,7 @@ Testing Nipype using Docker Nipype is tested inside Docker containers and users can use nipype images to test local versions. First, install the `Docker Engine `_. Nipype has one base docker image called nipype/nipype:base, that contains several useful tools - (FreeSurfer, AFNI, FSL, ANTs, etc.), and additional test images +(FreeSurfer, AFNI, FSL, ANTs, etc.), and additional test images for specific Python versions: py27 for Python 2.7 and py36 for Python 3.6. Users can pull the nipype image for Python 3.6 as follows:: diff --git a/doc/documentation.rst b/doc/documentation.rst index 5b4216f8a7..1cf275d630 100644 --- a/doc/documentation.rst +++ b/doc/documentation.rst @@ -4,10 +4,8 @@ Documentation ============= -.. htmlonly:: - - :Release: |version| - :Date: |today| +:Release: |version| +:Date: |today| Previous versions: `1.3.0 `_ `1.2.3 `_ @@ -16,35 +14,42 @@ Previous versions: `1.3.0 `_ `1.2.3 `_. + Be sure to read `Michael's excellent tutorials `__. + + .. admonition:: Nipype Workflows - .. admonition:: Interfaces, Workflows and Examples + The workflows that used to live as a module under + ``nipype.workflows`` have been migrated to the + new project `NiFlows `__. + + .. admonition:: Interfaces and Examples .. hlist:: :columns: 2 - * Workflows + * *In-house* interfaces .. toctree:: :maxdepth: 1 :glob: - interfaces/generated/*workflows* - * Examples + interfaces/generated/*algorithms* + + * Interfaces to third-party tools .. toctree:: :maxdepth: 1 :glob: - users/examples/* - * Interfaces + interfaces/generated/*interfaces* + + * Examples .. toctree:: :maxdepth: 1 :glob: - interfaces/generated/*algorithms* - interfaces/generated/*interfaces* + users/examples/* .. admonition:: Developer Guides diff --git a/doc/interfaces/index.rst b/doc/interfaces/index.rst index 77b9541100..14deeec063 100644 --- a/doc/interfaces/index.rst +++ b/doc/interfaces/index.rst @@ -7,4 +7,3 @@ Interfaces and Algorithms :Release: |version| :Date: |today| -.. include:: generated/gen.rst diff --git a/doc/links_names.txt b/doc/links_names.txt index 4cf07795f7..1a51a6dea3 100644 --- a/doc/links_names.txt +++ b/doc/links_names.txt @@ -74,7 +74,7 @@ .. _EPD: http://www.enthought.com/products/epd.php .. _Traits: http://code.enthought.com/projects/traits/ .. _Miniconda: https://conda.io/miniconda.html -.. _neurodocker: https://github.com/kaczmarj/neurodocker +.. _NeuroDocker: https://github.com/kaczmarj/neurodocker .. Python imaging projects .. _PyMVPA: http://www.pymvpa.org diff --git a/doc/sphinxext/README.txt b/doc/sphinxext/README.txt deleted file mode 100644 index 08bcbe9a60..0000000000 --- a/doc/sphinxext/README.txt +++ /dev/null @@ -1,16 +0,0 @@ -=================== - Sphinx Extensions -=================== - -We've copied these sphinx extensions over from nipy-core. Any edits -should be done upstream in nipy-core, not here in nipype! - -These a are a few sphinx extensions we are using to build the nipy -documentation. In this file we list where they each come from, since we intend -to always push back upstream any modifications or improvements we make to them. - -* From numpy: - * numpy_ext - -* From ipython - * ipython_console_highlighting diff --git a/doc/sphinxext/autosummary_generate.py b/doc/sphinxext/autosummary_generate.py deleted file mode 100755 index 658c50e4a4..0000000000 --- a/doc/sphinxext/autosummary_generate.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -r""" -autosummary_generate.py OPTIONS FILES - -Generate automatic RST source files for items referred to in -autosummary:: directives. - -Each generated RST file contains a single auto*:: directive which -extracts the docstring of the referred item. - -Example Makefile rule:: - - generate: - ./ext/autosummary_generate.py -o source/generated source/*.rst - -""" -from __future__ import print_function, unicode_literals -from builtins import open - -import re -import inspect -import os -import optparse -import pydoc -from autosummary import import_by_name - -try: - from phantom_import import import_phantom_module -except ImportError: - import_phantom_module = lambda x: x - - -def main(): - p = optparse.OptionParser(__doc__.strip()) - p.add_option("-p", "--phantom", action="store", type="string", - dest="phantom", default=None, - help="Phantom import modules from a file") - p.add_option("-o", "--output-dir", action="store", type="string", - dest="output_dir", default=None, - help=("Write all output files to the given directory " - "(instead of writing them as specified in the " - "autosummary:: directives)")) - options, args = p.parse_args() - - if len(args) == 0: - p.error("wrong number of arguments") - - if options.phantom and os.path.isfile(options.phantom): - import_phantom_module(options.phantom) - - # read - names = {} - for name, loc in list(get_documented(args).items()): - for (filename, sec_title, keyword, toctree) in loc: - if toctree is not None: - path = os.path.join(os.path.dirname(filename), toctree) - names[name] = os.path.abspath(path) - - # write - for name, path in sorted(names.items()): - if options.output_dir is not None: - path = options.output_dir - - if not os.path.isdir(path): - os.makedirs(path) - - try: - obj, name = import_by_name(name) - except ImportError as e: - print("Failed to import '%s': %s" % (name, e)) - continue - - fn = os.path.join(path, '%s.rst' % name) - - if os.path.exists(fn): - # skip - continue - - f = open(fn, 'w') - - try: - f.write('%s\n%s\n\n' % (name, '=' * len(name))) - - if inspect.isclass(obj): - if issubclass(obj, Exception): - f.write(format_modulemember(name, 'autoexception')) - else: - f.write(format_modulemember(name, 'autoclass')) - elif inspect.ismodule(obj): - f.write(format_modulemember(name, 'automodule')) - elif inspect.ismethod(obj) or inspect.ismethoddescriptor(obj): - f.write(format_classmember(name, 'automethod')) - elif callable(obj): - f.write(format_modulemember(name, 'autofunction')) - elif hasattr(obj, '__get__'): - f.write(format_classmember(name, 'autoattribute')) - else: - f.write(format_modulemember(name, 'autofunction')) - finally: - f.close() - - -def format_modulemember(name, directive): - parts = name.split('.') - mod, name = '.'.join(parts[:-1]), parts[-1] - return ".. currentmodule:: %s\n\n.. %s:: %s\n" % (mod, directive, name) - - -def format_classmember(name, directive): - parts = name.split('.') - mod, name = '.'.join(parts[:-2]), '.'.join(parts[-2:]) - return ".. currentmodule:: %s\n\n.. %s:: %s\n" % (mod, directive, name) - - -def get_documented(filenames): - """ - Find out what items are documented in source/*.rst - See `get_documented_in_lines`. - - """ - documented = {} - for filename in filenames: - f = open(filename, 'r') - lines = f.read().splitlines() - documented.update(get_documented_in_lines(lines, filename=filename)) - f.close() - return documented - - -def get_documented_in_docstring(name, module=None, filename=None): - """ - Find out what items are documented in the given object's docstring. - See `get_documented_in_lines`. - - """ - try: - obj, real_name = import_by_name(name) - lines = pydoc.getdoc(obj).splitlines() - return get_documented_in_lines(lines, module=name, filename=filename) - except AttributeError: - pass - except ImportError as e: - print("Failed to import '%s': %s" % (name, e)) - return {} - - -def get_documented_in_lines(lines, module=None, filename=None): - """ - Find out what items are documented in the given lines - - Returns - ------- - documented : dict of list of (filename, title, keyword, toctree) - Dictionary whose keys are documented names of objects. - The value is a list of locations where the object was documented. - Each location is a tuple of filename, the current section title, - the name of the directive, and the value of the :toctree: argument - (if present) of the directive. - - """ - title_underline_re = re.compile("^[-=*_^#]{3,}\s*$") - autodoc_re = re.compile( - ".. auto(function|method|attribute|class|exception|module)::" - "\s*([A-Za-z0-9_.]+)\s*$") - autosummary_re = re.compile(r'^\.\.\s+autosummary::\s*') - module_re = re.compile( - r'^\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$') - autosummary_item_re = re.compile(r'^\s+([_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?') - toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') - - documented = {} - - current_title = [] - last_line = None - toctree = None - current_module = module - in_autosummary = False - - for line in lines: - try: - if in_autosummary: - m = toctree_arg_re.match(line) - if m: - toctree = m.group(1) - continue - - if line.strip().startswith(':'): - continue # skip options - - m = autosummary_item_re.match(line) - if m: - name = m.group(1).strip() - if current_module and not name.startswith( - current_module + '.'): - name = "%s.%s" % (current_module, name) - documented.setdefault(name, []).append( - (filename, current_title, 'autosummary', toctree)) - continue - if line.strip() == '': - continue - in_autosummary = False - - m = autosummary_re.match(line) - if m: - in_autosummary = True - continue - - m = autodoc_re.search(line) - if m: - name = m.group(2).strip() - if m.group(1) == "module": - current_module = name - documented.update(get_documented_in_docstring( - name, filename=filename)) - elif current_module and not name.startswith( - current_module + '.'): - name = "%s.%s" % (current_module, name) - documented.setdefault(name, []).append( - (filename, current_title, "auto" + m.group(1), None)) - continue - - m = title_underline_re.match(line) - if m and last_line: - current_title = last_line.strip() - continue - - m = module_re.match(line) - if m: - current_module = m.group(2) - continue - finally: - last_line = line - - return documented - - -if __name__ == "__main__": - main() diff --git a/doc/sphinxext/ipython_console_highlighting.py b/doc/sphinxext/ipython_console_highlighting.py deleted file mode 100644 index a400d3c9c1..0000000000 --- a/doc/sphinxext/ipython_console_highlighting.py +++ /dev/null @@ -1,101 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -"""reST directive for syntax-highlighting ipython interactive sessions. -""" - -# ----------------------------------------------------------------------------- -# Needed modules - -# Standard library -import re - -# Third party -from pygments.lexer import Lexer, do_insertions -from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer, - PythonTracebackLexer) -from pygments.token import Comment, Generic - -from sphinx import highlighting - - -# ----------------------------------------------------------------------------- -# Global constants -line_re = re.compile('.*?\n') - -# ----------------------------------------------------------------------------- -# Code begins - classes and functions - - -class IPythonConsoleLexer(Lexer): - """ - For IPython console output or doctests, such as: - - .. sourcecode:: ipython - - In [1]: a = 'foo' - - In [2]: a - Out[2]: 'foo' - - In [3]: print a - foo - - In [4]: 1 / 0 - - Notes: - - - Tracebacks are not currently supported. - - - It assumes the default IPython prompts, not customized ones. - """ - - name = 'IPython console session' - aliases = ['ipython'] - mimetypes = ['text/x-ipython-console'] - input_prompt = re.compile("(In \[[0-9]+\]: )|( \.\.\.+:)") - output_prompt = re.compile("(Out\[[0-9]+\]: )|( \.\.\.+:)") - continue_prompt = re.compile(" \.\.\.+:") - tb_start = re.compile("\-+") - - def get_tokens_unprocessed(self, text): - pylexer = PythonLexer(**self.options) - - curcode = '' - insertions = [] - for match in line_re.finditer(text): - line = match.group() - input_prompt = self.input_prompt.match(line) - continue_prompt = self.continue_prompt.match(line.rstrip()) - output_prompt = self.output_prompt.match(line) - if line.startswith("#"): - insertions.append((len(curcode), - [(0, Comment, line)])) - elif input_prompt is not None: - insertions.append((len( - curcode), [(0, Generic.Prompt, input_prompt.group())])) - curcode += line[input_prompt.end():] - elif continue_prompt is not None: - insertions.append((len( - curcode), [(0, Generic.Prompt, continue_prompt.group())])) - curcode += line[continue_prompt.end():] - elif output_prompt is not None: - insertions.append((len( - curcode), [(0, Generic.Output, output_prompt.group())])) - curcode += line[output_prompt.end():] - else: - if curcode: - for item in do_insertions(insertions, - pylexer.get_tokens_unprocessed( - curcode)): - yield item - curcode = '' - insertions = [] - yield match.start(), Generic.Output, line - if curcode: - for item in do_insertions(insertions, - pylexer.get_tokens_unprocessed(curcode)): - yield item - -# ----------------------------------------------------------------------------- -# Register the extension as a valid pygments lexer -highlighting.lexers['ipython'] = IPythonConsoleLexer() diff --git a/doc/users/install.rst b/doc/users/install.rst index 3a710088e9..a16d41c5df 100644 --- a/doc/users/install.rst +++ b/doc/users/install.rst @@ -16,7 +16,7 @@ image from Docker hub:: docker pull nipype/nipype You may also build custom docker containers with specific versions of software -using Neurodocker_ (see the `Neurodocker tutorial +using NeuroDocker_ (see the `Neurodocker tutorial `_). Using conda @@ -61,7 +61,7 @@ listed below:: Debian and Ubuntu ~~~~~~~~~~~~~~~~~ -Add the `NeuroDebian `_ repository and install +Add the NeuroDebian_ repository and install the ``python-nipype`` package using ``apt-get`` or your favorite package manager. @@ -111,7 +111,7 @@ Interface Dependencies Nipype provides wrappers around many neuroimaging tools and contains some algorithms. These tools will need to be installed for Nipype to run. You can create containers with different versions of these tools installed using -Neurodocker_ (see the :doc:`neurodocker`). +NeuroDocker_. Installation for developers --------------------------- diff --git a/doc/version.rst b/doc/version.rst index c795d991c6..35e3e0a60f 100644 --- a/doc/version.rst +++ b/doc/version.rst @@ -1,6 +1,4 @@ .. _version: -.. htmlonly:: - - :Release: |version| - :Date: |today| +:Release: |version| +:Date: |today| diff --git a/nipype/__init__.py b/nipype/__init__.py index 76b2ba58f9..74c6a42dd7 100644 --- a/nipype/__init__.py +++ b/nipype/__init__.py @@ -67,10 +67,12 @@ def get_info(): def check_latest_version(raise_exception=False): - """Check for the latest version of the library + """ + Check for the latest version of the library. - parameters: - raise_exception: boolean + Parameters + ---------- + raise_exception: bool Raise a RuntimeError if a bad version is being used """ import etelemetry diff --git a/nipype/pipeline/plugins/base.py b/nipype/pipeline/plugins/base.py index f7fcb6dab1..4be8eb232b 100644 --- a/nipype/pipeline/plugins/base.py +++ b/nipype/pipeline/plugins/base.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Common graph operations for execution -""" +"""Common graph operations for execution.""" import sys from copy import deepcopy from glob import glob @@ -23,10 +22,7 @@ class PluginBase(object): - """ - Base class for plugins - - """ + """Base class for plugins.""" def __init__(self, plugin_args=None): if plugin_args is None: @@ -37,15 +33,20 @@ def __init__(self, plugin_args=None): def run(self, graph, config, updatehash=False): """ + Instruct the plugin to execute the workflow graph. + The core plugin member that should be implemented by all plugins. - graph: a networkx, flattened :abbr:`DAG (Directed Acyclic Graph)` - to be executed - - config: a nipype.config object - - updatehash: + Parameters + ---------- + graph : + a networkx, flattened :abbr:`DAG (Directed Acyclic Graph)` + to be executed + config : :obj:`~nipype.config` + a nipype.config object + updatehash : :obj:`bool` + whether cached nodes with stale hash should be just updated. """ raise NotImplementedError @@ -55,19 +56,7 @@ class DistributedPluginBase(PluginBase): """ Execute workflow with a distribution engine - Relevant class attributes - ------------------------- - - procs: list (N) of underlying interface elements to be processed - proc_done: a boolean numpy array (N,) signifying whether a process has been - submitted for execution - proc_pending: a boolean numpy array (N,) signifying whether a - process is currently running. - depidx: a boolean matrix (NxN) storing the dependency structure accross - processes. Process dependencies are derived from each column. - - Combinations of ``proc_done`` and ``proc_pending`` - -------------------------------------------------- + Combinations of ``proc_done`` and ``proc_pending``: +------------+---------------+--------------------------------+ | proc_done | proc_pending | outcome | @@ -80,6 +69,21 @@ class DistributedPluginBase(PluginBase): +------------+---------------+--------------------------------+ | False | True | INVALID COMBINATION | +------------+---------------+--------------------------------+ + + Attributes + ---------- + procs : :obj:`list` + list (N) of underlying interface elements to be processed + proc_done : :obj:`numpy.ndarray` + a boolean numpy array (N,) signifying whether a process has been + submitted for execution + proc_pending : :obj:`numpy.ndarray` + a boolean numpy array (N,) signifying whether a + process is currently running. + depidx : :obj:`numpy.matrix` + a boolean matrix (NxN) storing the dependency structure accross + processes. Process dependencies are derived from each column. + """ def __init__(self, plugin_args=None): diff --git a/nipype/utils/config.py b/nipype/utils/config.py index 999ee76307..6f1b385672 100644 --- a/nipype/utils/config.py +++ b/nipype/utils/config.py @@ -71,7 +71,7 @@ [check] interval = 1209600 -""".format +""" def mkdir_p(path): @@ -130,7 +130,7 @@ def cwd(self): def set_default_config(self): """Read default settings template and set into config object""" - default_cfg = DEFAULT_CONFIG_TPL( + default_cfg = DEFAULT_CONFIG_TPL.format( log_dir=os.path.expanduser("~"), # Get $HOME in a platform-agnostic way crashdump_dir=self.cwd, # Read cached cwd ) diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index b0d6c4a0c1..735cc610b6 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -284,7 +284,8 @@ def _generate_cifs_table(): def on_cifs(fname): - """ Checks whether a file path is on a CIFS filesystem mounted in a POSIX + """ + Checks whether a file path is on a CIFS filesystem mounted in a POSIX host (i.e., has the ``mount`` command). On Windows, Docker mounts host directories into containers through CIFS @@ -292,9 +293,10 @@ def on_cifs(fname): the CIFS driver exposes to the OS as symlinks. We have found that under concurrent access to the filesystem, this feature can result in failures to create or read recently-created symlinks, - leading to inconsistent behavior and ``FileNotFoundError``s. + leading to inconsistent behavior and ``FileNotFoundError``. This check is written to support disabling symlinks on CIFS shares. + """ # Only the first match (most recent parent) counts for fspath, fstype in _cifs_table: diff --git a/nipype/utils/nipype2boutiques.py b/nipype/utils/nipype2boutiques.py index 4013714bc2..0a12e59f28 100644 --- a/nipype/utils/nipype2boutiques.py +++ b/nipype/utils/nipype2boutiques.py @@ -34,25 +34,42 @@ def generate_boutiques_descriptor( tags=None, ): """ - Returns a JSON string containing a JSON Boutiques description of a - Nipype interface. - Arguments: - * module: module where the Nipype interface is declared. - * interface_name: name of Nipype interface. - * container_image: name of the container image where the tool is installed - * container_type: type of container image (Docker or Singularity) - * container_index: optional index where the image is available - * verbose: print information messages - * save: True if you want to save descriptor to a file - * save_path: file path for the saved descriptor (defaults to name of the + Generate a JSON Boutiques description of a Nipype interface. + + Arguments + --------- + module : + module where the Nipype interface is declared. + interface_name : + name of Nipype interface. + container_image : + name of the container image where the tool is installed + container_type : + type of container image (Docker or Singularity) + container_index : + optional index where the image is available + verbose : + print information messages + save : + True if you want to save descriptor to a file + save_path : + file path for the saved descriptor (defaults to name of the interface in current directory) - * author: author of the tool (required for publishing) - * ignore_inputs: list of interface inputs to not include in the descriptor - * tags: JSON object containing tags to include in the descriptor, - e.g. "{\"key1\": \"value1\"}" (note: the tags 'domain:neuroinformatics' - and 'interface-type:nipype' are included by default) - """ + author : + author of the tool (required for publishing) + ignore_inputs : + list of interface inputs to not include in the descriptor + tags : + JSON object containing tags to include in the descriptor, + e.g. ``{"key1": "value1"}`` (note: the tags 'domain:neuroinformatics' + and 'interface-type:nipype' are included by default) + + Returns + ------- + boutiques : str + string containing a Boutiques' JSON object + """ if not module: raise Exception("Undefined module.") diff --git a/tools/README b/tools/README deleted file mode 100644 index 8d987d4e00..0000000000 --- a/tools/README +++ /dev/null @@ -1,15 +0,0 @@ -============== - Nipype Tools -============== - -This directory contains various tools used by the nipype developers. -Only install tools here that are unique to the nipype project. Any -tools shared with our parent project, nipy, should go in the -nipy/tools directory. - -Exceptions ----------- - -* apigen.py: This is not importable from nipy, so I copied it. -* build_modref_templates.py: This was copied and modified to work with nipype. - diff --git a/tools/build_interface_docs.py b/tools/build_interface_docs.py index d21d19428a..f42adc7904 100755 --- a/tools/build_interface_docs.py +++ b/tools/build_interface_docs.py @@ -26,22 +26,25 @@ r"\.testing", r"\.caching", r"\.scripts", + r"\.sphinxext$", + r"\.workflows" ] # Modules that should not be included in generated API docs. docwriter.module_skip_patterns += [ - r"\.version$", + r"\.conftest", r"\.interfaces\.base$", r"\.interfaces\.matlab$", - r"\.interfaces\.rest$", r"\.interfaces\.pymvpa$", + r"\.interfaces\.rest$", r"\.interfaces\.slicer\.generate_classes$", r"\.interfaces\.spm\.base$", r"\.interfaces\.traits", r"\.pipeline\.alloy$", r"\.pipeline\.s3_node_wrapper$", - r"\.testing", + r"\.pkg_info" r"\.scripts", - r"\.conftest", + r"\.testing", + r"\.version$", ] docwriter.class_skip_patterns += [ "AFNICommand", @@ -52,12 +55,12 @@ "^SPM", "Tester", "Spec$", - "Numpy" + "Numpy", # NipypeTester raises an # exception when instantiated in # InterfaceHelpWriter.generate_api_doc "NipypeTester", ] docwriter.write_api_docs(outdir) - docwriter.write_index(outdir, "gen", relative_to="interfaces") + # docwriter.write_index(outdir, "gen") print("%d files written" % len(docwriter.written_modules)) diff --git a/tools/build_modref_templates.py b/tools/build_modref_templates.py deleted file mode 100755 index 8a4c480f51..0000000000 --- a/tools/build_modref_templates.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -"""Script to auto-generate our API docs. -""" -# stdlib imports -import os -import sys - -# ***************************************************************************** -if __name__ == "__main__": - nipypepath = os.path.abspath("..") - sys.path.insert(1, nipypepath) - package = "nipype" - # local imports - from apigen import ApiDocWriter - - outdir = os.path.join("api", "generated") - docwriter = ApiDocWriter(package) - # Packages that should not be included in generated API docs. - docwriter.package_skip_patterns += [ - "\.external$", - "\.utils$", - "\.interfaces\.", - "\.workflows$", - "\.pipeline\.plugins$", - "\.testing$", - "\.fixes$", - "\.algorithms$", - "\.scripts$", - ] - # Modules that should not be included in generated API docs. - docwriter.module_skip_patterns += [ - "\.version$", - "info", - "\.interfaces\.(?!(base|matlab))", - "\.pipeline\.utils$", - "\.interfaces\.slicer\.generate_classes$", - "\.interfaces\.pymvpa$", - "\.scripts$", - ] - docwriter.write_api_docs(outdir) - docwriter.write_index(outdir, "gen", relative_to="api") - print("%d files written" % len(docwriter.written_modules)) diff --git a/tools/update_changes.sh b/tools/update_changes.sh index 1ba3528b1f..7a12a2d1b4 100755 --- a/tools/update_changes.sh +++ b/tools/update_changes.sh @@ -13,7 +13,7 @@ set -u # Treat unset variables as an error when substituting. set -x # Print command traces before executing command. ROOT=$( git rev-parse --show-toplevel ) -CHANGES=$ROOT/doc/changelog/1.X.X-changelog +CHANGES=$ROOT/doc/changelog/1.X.X-changelog.rst # Check whether the Upcoming release header is present head -1 $CHANGES | grep -q Upcoming From e24c38aecb3489acf6635478fec2b8e8f3f63191 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Dec 2019 17:43:57 -0800 Subject: [PATCH 3/5] fix: last release changelog --- doc/changelog/1.X.X-changelog.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/changelog/1.X.X-changelog.rst b/doc/changelog/1.X.X-changelog.rst index 0e0d661f61..3af9ed8ca4 100644 --- a/doc/changelog/1.X.X-changelog.rst +++ b/doc/changelog/1.X.X-changelog.rst @@ -1,7 +1,6 @@ 1.4.0 (December 20, 2019) ========================= - -##### [Full changelog](https://github.com/nipy/nipype/milestone/37?closed=1) +(`Full changelog `__) * FIX: Mark strings containing regex escapes as raw (https://github.com/nipy/nipype/pull/3106) * ENH: Pacify DeprecationWarnings caused by nibabel 3 pre-release (https://github.com/nipy/nipype/pull/3099) @@ -16,16 +15,13 @@ * MNT: Fix Dorota Jarecka ORCID (https://github.com/nipy/nipype/pull/3100) * MNT: Drop Python 2 support (https://github.com/nipy/nipype/pull/2654) - 1.3.1 (November 12, 2019) ========================= - * FIX: Restore checking traits or bunch (https://github.com/nipy/nipype/pull/3094) 1.3.0 (November 11, 2019) ========================= - (`Full changelog `__) * FIX: Fixed typo in QwarpInputSpec Trait description (https://github.com/nipy/nipype/pull/3079) From 690b783e3c4a41737f21f4f5c67f4eee4064ed63 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Dec 2019 17:50:19 -0800 Subject: [PATCH 4/5] enh: accept a milestone number in the ``update_changes.sh`` script --- tools/update_changes.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/update_changes.sh b/tools/update_changes.sh index 7a12a2d1b4..387ec3442f 100755 --- a/tools/update_changes.sh +++ b/tools/update_changes.sh @@ -23,7 +23,10 @@ UPCOMING=$? HEADER="$1 ($(date '+%B %d, %Y'))" echo $HEADER >> newchanges echo $( printf "%${#HEADER}s" | tr " " "=" ) >> newchanges -echo "" >> newchanges + +if [[ "x$2" != "x" ]]; then + echo "(\`Full changelog \`__)" >> newchanges +fi # Search for PRs since previous release git log --grep="Merge pull request" `git describe --tags --abbrev=0`..HEAD --pretty='format: * %b %s' | sed 's+Merge pull request \#\([^\d]*\)\ from\ .*+(https://github.com/nipy/nipype/pull/\1)+' >> newchanges From 9f569722fe9c59c92b727dd149a37d295051340a Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Dec 2019 17:54:38 -0800 Subject: [PATCH 5/5] fix: remove unnecessary line [skip ci] --- doc/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 25acfeb122..2c96edd38b 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -5,7 +5,6 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = -PYTHONPATH = $(PWD) # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4