1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
|
"""
sphinx.ext.apidoc
~~~~~~~~~~~~~~~~~
Parses a directory tree looking for Python modules and packages and creates
ReST files appropriately to create code documentation with Sphinx. It also
creates a modules index (named modules.<suffix>).
This is derived from the "sphinx-autopackage" script, which is:
Copyright 2008 Société des arts technologiques (SAT),
https://sat.qc.ca/
:copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import argparse
import glob
import locale
import os
import sys
from copy import copy
from fnmatch import fnmatch
from importlib.machinery import EXTENSION_SUFFIXES
from os import path
from typing import Any, Generator, List, Tuple
import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.cmd.quickstart import EXTENSIONS
from sphinx.locale import __
from sphinx.util.osutil import FileAvoidWrite, ensuredir
from sphinx.util.template import ReSTRenderer
# automodule options
if 'SPHINX_APIDOC_OPTIONS' in os.environ:
OPTIONS = os.environ['SPHINX_APIDOC_OPTIONS'].split(',')
else:
OPTIONS = [
'members',
'undoc-members',
# 'inherited-members', # disabled because there's a bug in sphinx
'show-inheritance',
]
PY_SUFFIXES = ('.py', '.pyx') + tuple(EXTENSION_SUFFIXES)
template_dir = path.join(package_dir, 'templates', 'apidoc')
def is_initpy(filename: str) -> bool:
"""Check *filename* is __init__ file or not."""
basename = path.basename(filename)
for suffix in sorted(PY_SUFFIXES, key=len, reverse=True):
if basename == '__init__' + suffix:
return True
else:
return False
def module_join(*modnames: str) -> str:
"""Join module names with dots."""
return '.'.join(filter(None, modnames))
def is_packagedir(dirname: str = None, files: List[str] = None) -> bool:
"""Check given *files* contains __init__ file."""
if files is None and dirname is None:
return False
if files is None:
files = os.listdir(dirname)
return any(f for f in files if is_initpy(f))
def write_file(name: str, text: str, opts: Any) -> None:
"""Write the output file for module/package <name>."""
quiet = getattr(opts, 'quiet', None)
fname = path.join(opts.destdir, '%s.%s' % (name, opts.suffix))
if opts.dryrun:
if not quiet:
print(__('Would create file %s.') % fname)
return
if not opts.force and path.isfile(fname):
if not quiet:
print(__('File %s already exists, skipping.') % fname)
else:
if not quiet:
print(__('Creating file %s.') % fname)
with FileAvoidWrite(fname) as f:
f.write(text)
def create_module_file(package: str, basename: str, opts: Any,
user_template_dir: str = None) -> None:
"""Build the text of the file and write the file."""
options = copy(OPTIONS)
if opts.includeprivate and 'private-members' not in options:
options.append('private-members')
qualname = module_join(package, basename)
context = {
'show_headings': not opts.noheadings,
'basename': basename,
'qualname': qualname,
'automodule_options': options,
}
text = ReSTRenderer([user_template_dir, template_dir]).render('module.rst_t', context)
write_file(qualname, text, opts)
def create_package_file(root: str, master_package: str, subroot: str, py_files: List[str],
opts: Any, subs: List[str], is_namespace: bool,
excludes: List[str] = [], user_template_dir: str = None) -> None:
"""Build the text of the file and write the file."""
# build a list of sub packages (directories containing an __init__ file)
subpackages = [module_join(master_package, subroot, pkgname)
for pkgname in subs
if not is_skipped_package(path.join(root, pkgname), opts, excludes)]
# build a list of sub modules
submodules = [sub.split('.')[0] for sub in py_files
if not is_skipped_module(path.join(root, sub), opts, excludes) and
not is_initpy(sub)]
submodules = [module_join(master_package, subroot, modname)
for modname in submodules]
options = copy(OPTIONS)
if opts.includeprivate and 'private-members' not in options:
options.append('private-members')
pkgname = module_join(master_package, subroot)
context = {
'pkgname': pkgname,
'subpackages': subpackages,
'submodules': submodules,
'is_namespace': is_namespace,
'modulefirst': opts.modulefirst,
'separatemodules': opts.separatemodules,
'automodule_options': options,
'show_headings': not opts.noheadings,
'maxdepth': opts.maxdepth,
}
text = ReSTRenderer([user_template_dir, template_dir]).render('package.rst_t', context)
write_file(pkgname, text, opts)
if submodules and opts.separatemodules:
for submodule in submodules:
create_module_file(None, submodule, opts, user_template_dir)
def create_modules_toc_file(modules: List[str], opts: Any, name: str = 'modules',
user_template_dir: str = None) -> None:
"""Create the module's index."""
modules.sort()
prev_module = ''
for module in modules[:]:
# look if the module is a subpackage and, if yes, ignore it
if module.startswith(prev_module + '.'):
modules.remove(module)
else:
prev_module = module
context = {
'header': opts.header,
'maxdepth': opts.maxdepth,
'docnames': modules,
}
text = ReSTRenderer([user_template_dir, template_dir]).render('toc.rst_t', context)
write_file(name, text, opts)
def is_skipped_package(dirname: str, opts: Any, excludes: List[str] = []) -> bool:
"""Check if we want to skip this module."""
if not path.isdir(dirname):
return False
files = glob.glob(path.join(dirname, '*.py'))
regular_package = any(f for f in files if is_initpy(f))
if not regular_package and not opts.implicit_namespaces:
# *dirname* is not both a regular package and an implicit namespace pacage
return True
# Check there is some showable module inside package
if all(is_excluded(path.join(dirname, f), excludes) for f in files):
# all submodules are excluded
return True
else:
return False
def is_skipped_module(filename: str, opts: Any, excludes: List[str]) -> bool:
"""Check if we want to skip this module."""
if not path.exists(filename):
# skip if the file doesn't exist
return True
elif path.basename(filename).startswith('_') and not opts.includeprivate:
# skip if the module has a "private" name
return True
else:
return False
def walk(rootpath: str, excludes: List[str], opts: Any
) -> Generator[Tuple[str, List[str], List[str]], None, None]:
"""Walk through the directory and list files and subdirectories up."""
followlinks = getattr(opts, 'followlinks', False)
includeprivate = getattr(opts, 'includeprivate', False)
for root, subs, files in os.walk(rootpath, followlinks=followlinks):
# document only Python module files (that aren't excluded)
files = sorted(f for f in files
if f.endswith(PY_SUFFIXES) and
not is_excluded(path.join(root, f), excludes))
# remove hidden ('.') and private ('_') directories, as well as
# excluded dirs
if includeprivate:
exclude_prefixes: Tuple[str, ...] = ('.',)
else:
exclude_prefixes = ('.', '_')
subs[:] = sorted(sub for sub in subs if not sub.startswith(exclude_prefixes) and
not is_excluded(path.join(root, sub), excludes))
yield root, subs, files
def has_child_module(rootpath: str, excludes: List[str], opts: Any) -> bool:
"""Check the given directory contains child modules at least one."""
for root, subs, files in walk(rootpath, excludes, opts):
if files:
return True
return False
def recurse_tree(rootpath: str, excludes: List[str], opts: Any,
user_template_dir: str = None) -> List[str]:
"""
Look for every file in the directory tree and create the corresponding
ReST files.
"""
implicit_namespaces = getattr(opts, 'implicit_namespaces', False)
# check if the base directory is a package and get its name
if is_packagedir(rootpath) or implicit_namespaces:
root_package = rootpath.split(path.sep)[-1]
else:
# otherwise, the base is a directory with packages
root_package = None
toplevels = []
for root, subs, files in walk(rootpath, excludes, opts):
is_pkg = is_packagedir(None, files)
is_namespace = not is_pkg and implicit_namespaces
if is_pkg:
for f in files[:]:
if is_initpy(f):
files.remove(f)
files.insert(0, f)
elif root != rootpath:
# only accept non-package at toplevel unless using implicit namespaces
if not implicit_namespaces:
del subs[:]
continue
if is_pkg or is_namespace:
# we are in a package with something to document
if subs or len(files) > 1 or not is_skipped_package(root, opts):
subpackage = root[len(rootpath):].lstrip(path.sep).\
replace(path.sep, '.')
# if this is not a namespace or
# a namespace and there is something there to document
if not is_namespace or has_child_module(root, excludes, opts):
create_package_file(root, root_package, subpackage,
files, opts, subs, is_namespace, excludes,
user_template_dir)
toplevels.append(module_join(root_package, subpackage))
else:
# if we are at the root level, we don't require it to be a package
assert root == rootpath and root_package is None
for py_file in files:
if not is_skipped_module(path.join(rootpath, py_file), opts, excludes):
module = py_file.split('.')[0]
create_module_file(root_package, module, opts, user_template_dir)
toplevels.append(module)
return toplevels
def is_excluded(root: str, excludes: List[str]) -> bool:
"""Check if the directory is in the exclude list.
Note: by having trailing slashes, we avoid common prefix issues, like
e.g. an exclude "foo" also accidentally excluding "foobar".
"""
for exclude in excludes:
if fnmatch(root, exclude):
return True
return False
def get_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
usage='%(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> '
'[EXCLUDE_PATTERN, ...]',
epilog=__('For more information, visit <http://sphinx-doc.org/>.'),
description=__("""
Look recursively in <MODULE_PATH> for Python modules and packages and create
one reST file with automodule directives per package in the <OUTPUT_PATH>.
The <EXCLUDE_PATTERN>s can be file and/or directory patterns that will be
excluded from generation.
Note: By default this script will not overwrite already created files."""))
parser.add_argument('--version', action='version', dest='show_version',
version='%%(prog)s %s' % __display_version__)
parser.add_argument('module_path',
help=__('path to module to document'))
parser.add_argument('exclude_pattern', nargs='*',
help=__('fnmatch-style file and/or directory patterns '
'to exclude from generation'))
parser.add_argument('-o', '--output-dir', action='store', dest='destdir',
required=True,
help=__('directory to place all output'))
parser.add_argument('-q', action='store_true', dest='quiet',
help=__('no output on stdout, just warnings on stderr'))
parser.add_argument('-d', '--maxdepth', action='store', dest='maxdepth',
type=int, default=4,
help=__('maximum depth of submodules to show in the TOC '
'(default: 4)'))
parser.add_argument('-f', '--force', action='store_true', dest='force',
help=__('overwrite existing files'))
parser.add_argument('-l', '--follow-links', action='store_true',
dest='followlinks', default=False,
help=__('follow symbolic links. Powerful when combined '
'with collective.recipe.omelette.'))
parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun',
help=__('run the script without creating files'))
parser.add_argument('-e', '--separate', action='store_true',
dest='separatemodules',
help=__('put documentation for each module on its own page'))
parser.add_argument('-P', '--private', action='store_true',
dest='includeprivate',
help=__('include "_private" modules'))
parser.add_argument('--tocfile', action='store', dest='tocfile', default='modules',
help=__("filename of table of contents (default: modules)"))
parser.add_argument('-T', '--no-toc', action='store_false', dest='tocfile',
help=__("don't create a table of contents file"))
parser.add_argument('-E', '--no-headings', action='store_true',
dest='noheadings',
help=__("don't create headings for the module/package "
"packages (e.g. when the docstrings already "
"contain them)"))
parser.add_argument('-M', '--module-first', action='store_true',
dest='modulefirst',
help=__('put module documentation before submodule '
'documentation'))
parser.add_argument('--implicit-namespaces', action='store_true',
dest='implicit_namespaces',
help=__('interpret module paths according to PEP-0420 '
'implicit namespaces specification'))
parser.add_argument('-s', '--suffix', action='store', dest='suffix',
default='rst',
help=__('file suffix (default: rst)'))
parser.add_argument('-F', '--full', action='store_true', dest='full',
help=__('generate a full project with sphinx-quickstart'))
parser.add_argument('-a', '--append-syspath', action='store_true',
dest='append_syspath',
help=__('append module_path to sys.path, used when --full is given'))
parser.add_argument('-H', '--doc-project', action='store', dest='header',
help=__('project name (default: root module name)'))
parser.add_argument('-A', '--doc-author', action='store', dest='author',
help=__('project author(s), used when --full is given'))
parser.add_argument('-V', '--doc-version', action='store', dest='version',
help=__('project version, used when --full is given'))
parser.add_argument('-R', '--doc-release', action='store', dest='release',
help=__('project release, used when --full is given, '
'defaults to --doc-version'))
group = parser.add_argument_group(__('extension options'))
group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
action='append', help=__('enable arbitrary extensions'))
for ext in EXTENSIONS:
group.add_argument('--ext-%s' % ext, action='append_const',
const='sphinx.ext.%s' % ext, dest='extensions',
help=__('enable %s extension') % ext)
group = parser.add_argument_group(__('Project templating'))
group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
dest='templatedir',
help=__('template directory for template files'))
return parser
def main(argv: List[str] = sys.argv[1:]) -> int:
"""Parse and check the command line arguments."""
sphinx.locale.setlocale(locale.LC_ALL, '')
sphinx.locale.init_console(os.path.join(package_dir, 'locale'), 'sphinx')
parser = get_parser()
args = parser.parse_args(argv)
rootpath = path.abspath(args.module_path)
# normalize opts
if args.header is None:
args.header = rootpath.split(path.sep)[-1]
if args.suffix.startswith('.'):
args.suffix = args.suffix[1:]
if not path.isdir(rootpath):
print(__('%s is not a directory.') % rootpath, file=sys.stderr)
sys.exit(1)
if not args.dryrun:
ensuredir(args.destdir)
excludes = [path.abspath(exclude) for exclude in args.exclude_pattern]
modules = recurse_tree(rootpath, excludes, args, args.templatedir)
if args.full:
from sphinx.cmd import quickstart as qs
modules.sort()
prev_module = ''
text = ''
for module in modules:
if module.startswith(prev_module + '.'):
continue
prev_module = module
text += ' %s\n' % module
d = {
'path': args.destdir,
'sep': False,
'dot': '_',
'project': args.header,
'author': args.author or 'Author',
'version': args.version or '',
'release': args.release or args.version or '',
'suffix': '.' + args.suffix,
'master': 'index',
'epub': True,
'extensions': ['sphinx.ext.autodoc', 'sphinx.ext.viewcode',
'sphinx.ext.todo'],
'makefile': True,
'batchfile': True,
'make_mode': True,
'mastertocmaxdepth': args.maxdepth,
'mastertoctree': text,
'language': 'en',
'module_path': rootpath,
'append_syspath': args.append_syspath,
}
if args.extensions:
d['extensions'].extend(args.extensions)
if args.quiet:
d['quiet'] = True
for ext in d['extensions'][:]:
if ',' in ext:
d['extensions'].remove(ext)
d['extensions'].extend(ext.split(','))
if not args.dryrun:
qs.generate(d, silent=True, overwrite=args.force,
templatedir=args.templatedir)
elif args.tocfile:
create_modules_toc_file(modules, args, args.tocfile, args.templatedir)
return 0
# So program can be started with "python -m sphinx.apidoc ..."
if __name__ == "__main__":
main()
|