== Python API docs ==

New command line options:

-l: logs bpy dumping problems in OUTPUT_DIR/.bpy.log

-L: logs sphinx|latex problems in:
    * OUTPUT_DIR/.sphinx-build.log      (sphinx building html problems)
    * OUTPUT_DIR/.sphinx-build_pdf.log  (sphinx building latex problems)
    * OUTPUT_DIR/.latex_make.log        (latex make problems)

-P: builds the pdf

-R: pack the files in a dir ready for online deployment
    (including the zip and the pdf eventually)


Example usage:
./cmake/bin/blender -b -P ./blender/doc/python_api/sphinx_doc_gen.py -- -p bmesh* -l -o ./python_api -B -P -R -L

1 files changed, 257 insertions, 79 deletions

diff --git a/doc/python_api/sphinx_doc_gen.py b/doc/python_api/sphinx_doc_gen.py
index b15fcd9303b..52024b04c93 100644
--- a/doc/python_api/sphinx_doc_gen.py
+++ b/doc/python_api/sphinx_doc_gen.py
@@ -35,7 +35,7 @@ API dump in RST files
     ./blender.bin -b -P doc/python_api/sphinx_doc_gen.py -- -o ../python_api
 
   For quick builds:
-    ./blender.bin -b -P doc/python_api/sphinx_doc_gen.py -- -q
+    ./blender.bin -b -P doc/python_api/sphinx_doc_gen.py -- -p
 
 
 Sphinx: HTML generation
@@ -75,6 +75,7 @@ import os
 import sys
 import inspect
 import shutil
+import logging
 
 from platform import platform
 PLATFORM = platform().split('-')[0].lower()    # 'linux', 'darwin', 'windows'
@@ -94,27 +95,45 @@ def handle_args():
         usage=SCRIPT_HELP_MSG
     )
 
-    # optional arguments    
-    parser.add_argument("-o", "--output",
-                        dest="output_dir",
+    # optional arguments
+    parser.add_argument("-p", "--partial",
+                        dest="partial",
                         type=str,
-                        default=SCRIPT_DIR,
-                        # XXX, THIS ISNT WORKING, EXAMPLE SCRIPTS WILL NOT BE FOUND
-                        help="Path of the API docs (default=<script dir>).",
+                        default="",
+                        help="Use a wildcard to only build specific module(s)\n"
+                             "Example: --partial bmesh*\n",
                         required=False)
 
-    parser.add_argument("-B", "--sphinx-build",
-                        dest="sphinx_build",
+    parser.add_argument("-f", "--fullrebuild",
+                        dest="full_rebuild",
                         default=False,
                         action='store_true',
-                        help="Run sphinx-build SPHINX_IN SPHINX_OUT (default=False)",
+                        help="Rewrite all rst files in sphinx-in/ "
+                             "(default=False)",
                         required=False)
 
-    parser.add_argument("-N", "--sphinx-named-output",
-                        dest="sphinx_named_output",
+    parser.add_argument("-b", "--bpy",
+                        dest="bpy",
                         default=False,
                         action='store_true',
-                        help="Add the theme name to the html dir name (default=False)",
+                        help="Write the rst file of the bpy module "
+                             "(default=False)",
+                        required=False)
+
+    parser.add_argument("-o", "--output",
+                        dest="output_dir",
+                        type=str,
+                        default=SCRIPT_DIR,
+                        help="Path of the API docs (default=<script dir>)",
+                        required=False)
+
+    parser.add_argument("-l", "--bpy-log",
+                        dest="bpy_log",
+                        default=False,
+                        action='store_true',
+                        help=
+                        "Log the output of the api dump (default=False).\n"
+                        "If given, save logs in OUTPUT_DIR/.bpy.log",
                         required=False)
 
     parser.add_argument("-T", "--sphinx-theme",
@@ -135,25 +154,51 @@ def handle_args():
 #                                 'sphinxdoc', 'traditional'],   # sphinx
                         required=False)
 
-    parser.add_argument("-f", "--fullrebuild",
-                        dest="full_rebuild",
+    parser.add_argument("-N", "--sphinx-named-output",
+                        dest="sphinx_named_output",
                         default=False,
                         action='store_true',
-                        help="Rewrite all rst files in sphinx-in/ (default=False)",
+                        help="Add the theme name to the html dir name.\n"
+                             "Example: \"sphinx-out_haiku\" (default=False)",
                         required=False)
 
-    parser.add_argument("-p", "--partial",
-                        dest="partial",
-                        type=str,
-                        default="",
-                        help="Use a wildcard to only build spesific module(s)",
+    parser.add_argument("-B", "--sphinx-build",
+                        dest="sphinx_build",
+                        default=False,
+                        action='store_true',
+                        help="Build the html docs by running:\n"
+                             "sphinx-build SPHINX_IN SPHINX_OUT\n"
+                             "(default=False; does not depend on -P)",
                         required=False)
 
-    parser.add_argument("-b", "--bpy",
-                        dest="bpy",
+    parser.add_argument("-P", "--sphinx-build-pdf",
+                        dest="sphinx_build_pdf",
+                        default=False,
+                        action='store_true',
+                        help="Build the pdf by running:\n"
+                             "sphinx-build -b latex SPHINX_IN SPHINX_OUT_PDF\n"
+                             "(default=False; does not depend on -B)",
+                        required=False)
+
+    parser.add_argument("-R", "--pack-reference",
+                        dest="pack_reference",
+                        default=False,
+                        action='store_true',
+                        help="Pack all necessary files in the deployed dir.\n"
+                             "(default=False; use with -B and -P)",
+                        required=False)
+
+    parser.add_argument("-L", "--sphinx-log",
+                        dest="sphinx_log",
                         default=False,
                         action='store_true',
-                        help="Write the rst file of the bpy module (default=False)",
+                        help=
+                        "Log warnings and errors (default=False).\n"
+                        "If given, eventually save logs in:\n"
+                        "* OUTPUT_DIR/.sphinx-build.log\n"
+                        "* OUTPUT_DIR/.sphinx-build_pdf.log\n"
+                        "* OUTPUT_DIR/.latex_make.log",
+                        # see SPHINX_BUILD_LOG
                         required=False)
 
     # parse only the args passed after '--'
@@ -168,11 +213,29 @@ ARGS = handle_args()
 
 # ----------------------------------BPY-----------------------------------------
 
+BPY_LOGGER = logging.getLogger('bpy')
+BPY_LOGGER.setLevel(logging.DEBUG)
+
+if ARGS.bpy_log:
+    BPY_LOGFILE = os.path.join(ARGS.output_dir,".bpy.log")
+    bpy_logfilehandler = logging.FileHandler(BPY_LOGFILE, mode="w")
+    bpy_logfilehandler.setLevel(logging.DEBUG)
+    BPY_LOGGER.addHandler(bpy_logfilehandler)
+
+    # using a FileHandler seems to disable the stdout, so we add a StreamHandler
+    bpy_log_stdout_handler = logging.StreamHandler(stream=sys.stdout)
+    bpy_log_stdout_handler.setLevel(logging.DEBUG)
+    BPY_LOGGER.addHandler(bpy_log_stdout_handler)
+
 """
 # for quick rebuilds
 rm -rf /b/doc/python_api/sphinx-* && \
-./blender.bin --background -noaudio --factory-startup --python  doc/python_api/sphinx_doc_gen.py && \
+./blender.bin -b -noaudio --factory-startup -P doc/python_api/sphinx_doc_gen.py && \
 sphinx-build doc/python_api/sphinx-in doc/python_api/sphinx-out
+
+or
+
+./blender.bin -b -noaudio --factory-startup -P doc/python_api/sphinx_doc_gen.py -- -f -B
 """
 
 # Switch for quick testing so doc-builds don't take so long
@@ -228,7 +291,7 @@ else:
     del m
     del fnmatch
 
-    print("Partial Doc Build, Skipping: %s\n" % "\n                             ".join(sorted(EXCLUDE_MODULES)))
+    BPY_LOGGER.debug("Partial Doc Build, Skipping: %s\n" % "\n                             ".join(sorted(EXCLUDE_MODULES)))
 
     #
     # done filtering
@@ -237,7 +300,7 @@ else:
 try:
     __import__("aud")
 except ImportError:
-    print("Warning: Built without 'aud' module, docs incomplete...")
+    BPY_LOGGER.debug("Warning: Built without 'aud' module, docs incomplete...")
     EXCLUDE_MODULES = EXCLUDE_MODULES + ("aud", )
 
 # examples
@@ -248,7 +311,7 @@ for f in os.listdir(EXAMPLES_DIR):
         EXAMPLE_SET.add(os.path.splitext(f)[0])
 EXAMPLE_SET_USED = set()
 
-#rst files dir
+# rst files dir
 RST_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, "rst"))
 
 # extra info, not api reference docs
@@ -264,10 +327,36 @@ INFO_DOCS = (
 # only support for properties atm.
 RNA_BLACKLIST = {
     # XXX messes up PDF!, really a bug but for now just workaround.
-    "UserPreferencesSystem": {"language", }
+    "UserPreferencesSystem": {"language" }
     }
 
 
+# --------------------configure compile time options----------------------------
+
+# -------------------------------BLENDER----------------------------------------
+
+blender_version_strings = [str(v) for v in bpy.app.version]
+
+# converting bytes to strings, due to #30154
+BLENDER_REVISION = str(bpy.app.build_revision, 'utf_8')
+BLENDER_DATE = str(bpy.app.build_date, 'utf_8')
+
+BLENDER_VERSION_DOTS = ".".join(blender_version_strings)    # '2.62.1'
+if BLENDER_REVISION != "Unknown":
+    BLENDER_VERSION_DOTS += " r" + BLENDER_REVISION         # '2.62.1 r44584'
+
+BLENDER_VERSION_PATH = "_".join(blender_version_strings)    # '2_62_1'
+if bpy.app.version_cycle == "release":
+    BLENDER_VERSION_PATH = "%s%s_release" % ("_".join(blender_version_strings[:2]),
+                                             bpy.app.version_char)   # '2_62_release'
+
+# --------------------------DOWNLOADABLE FILES----------------------------------
+
+REFERENCE_NAME = "blender_python_reference_%s" % BLENDER_VERSION_PATH
+REFERENCE_PATH = os.path.join(ARGS.output_dir, REFERENCE_NAME)
+BLENDER_PDF_FILENAME = "%s.pdf" % REFERENCE_NAME
+BLENDER_ZIP_FILENAME = "%s.zip" % REFERENCE_NAME
+
 # -------------------------------SPHINX-----------------------------------------
 
 SPHINX_THEMES = {'bf': ['blender-org'], # , 'naiad',
@@ -286,37 +375,43 @@ if ARGS.sphinx_theme not in available_themes:
     print ("Please choose a theme among: %s" % ', '.join(available_themes))
     sys.exit()
 
+if ARGS.sphinx_theme in SPHINX_THEMES['bf']:
+    SPHINX_THEME_DIR = os.path.join(ARGS.output_dir, ARGS.sphinx_theme)
+    SPHINX_THEME_SVN_DIR = os.path.join(SCRIPT_DIR, ARGS.sphinx_theme)
+
 SPHINX_IN = os.path.join(ARGS.output_dir, "sphinx-in")
 SPHINX_IN_TMP = SPHINX_IN + "-tmp"
 SPHINX_OUT = os.path.join(ARGS.output_dir, "sphinx-out")
 if ARGS.sphinx_named_output:
     SPHINX_OUT += "_%s" % ARGS.sphinx_theme
 
-if ARGS.sphinx_theme in SPHINX_THEMES['bf']:
-    SPHINX_THEME_DIR = os.path.join(ARGS.output_dir, ARGS.sphinx_theme)
-    SPHINX_THEME_SVN_DIR = os.path.join(SCRIPT_DIR, ARGS.sphinx_theme)
-
-# ------------------------------------------------------------------------------
-
-# configure compile time options
-
-# -------------------------------BLENDER----------------------------------------
-
-'''
-blender version
-'''
-version_strings = [str(v) for v in bpy.app.version]
-
-BLENDER_VERSION_DOTS = ".".join(version_strings)    # '2.62.1'
-if bpy.app.build_revision != b"Unknown":
-    # converting bytes to strings, due to #30154
-    BLENDER_VERSION_DOTS += " r" + str(bpy.app.build_revision, 'utf_8')    # '2.62.1 r44584'
-
-BLENDER_VERSION_PDF = "_".join(version_strings)    # '2_62_1'
-if bpy.app.version_cycle == "release":
-    BLENDER_VERSION_PDF = "%s%s_release" % ("_".join(version_strings[:2]),
-                                            bpy.app.version_char)   # '2_62_release'
-
+# html build
+if ARGS.sphinx_build:
+    SPHINX_BUILD = ["sphinx-build", SPHINX_IN, SPHINX_OUT]
+
+    if ARGS.sphinx_log:
+        SPHINX_BUILD_LOG = os.path.join(ARGS.output_dir, ".sphinx-build.log")
+        SPHINX_BUILD = ["sphinx-build",
+                        "-w", SPHINX_BUILD_LOG,
+                        SPHINX_IN, SPHINX_OUT]
+
+# pdf build
+if ARGS.sphinx_build_pdf:
+    SPHINX_OUT_PDF = os.path.join(ARGS.output_dir, "sphinx-out_pdf")
+    SPHINX_BUILD_PDF = ["sphinx-build",
+                        "-b", "latex",
+                        SPHINX_IN, SPHINX_OUT_PDF]
+    SPHINX_MAKE_PDF = ["make", "-C", SPHINX_OUT_PDF]
+    SPHINX_MAKE_PDF_STDOUT = None
+
+    if ARGS.sphinx_log:
+        SPHINX_BUILD_PDF_LOG = os.path.join(ARGS.output_dir, ".sphinx-build_pdf.log")
+        SPHINX_BUILD_PDF = ["sphinx-build", "-b", "latex",
+                            "-w", SPHINX_BUILD_PDF_LOG,
+                            SPHINX_IN, SPHINX_OUT_PDF]
+
+        sphinx_make_pdf_log = os.path.join(ARGS.output_dir, ".latex_make.log")
+        SPHINX_MAKE_PDF_STDOUT = open(sphinx_make_pdf_log, "w", encoding="utf-8")
 
 # --------------------------------API DUMP--------------------------------------
 
@@ -416,7 +511,7 @@ def write_example_ref(ident, fw, example_id, ext="py"):
         EXAMPLE_SET_USED.add(example_id)
     else:
         if bpy.app.debug:
-            print("\tskipping example:", example_id)
+            BPY_LOGGER.debug("\tskipping example: " + example_id)
 
     # Support for numbered files bpy.types.Operator -> bpy.types.Operator.1.py
     i = 1
@@ -703,7 +798,7 @@ def pymodule2sphinx(basepath, module_name, module, title):
             write_indented_lines("   ", fw, "constant value %s" % repr(value), False)
             fw("\n")
         else:
-            print("\tnot documenting %s.%s of %r type" % (module_name, attribute, value_type.__name__))
+            BPY_LOGGER.debug("\tnot documenting %s.%s of %r type" % (module_name, attribute, value_type.__name__))
             continue
 
         attribute_set.add(attribute)
@@ -711,7 +806,7 @@ def pymodule2sphinx(basepath, module_name, module, title):
     del module_dir_value_type
 
     # TODO, bpy_extras does this already, mathutils not.
-    """
+    '''
     if submodules:
         fw("\n"
            "**********\n"
@@ -722,7 +817,7 @@ def pymodule2sphinx(basepath, module_name, module, title):
         for attribute, submod in submodules:
             fw("* :mod:`%s.%s`\n" % (module_name, attribute))
         fw("\n")
-    """
+    '''
 
     # write collected classes now
     for (type_name, value) in classes:
@@ -871,8 +966,9 @@ def pycontext2sphinx(basepath):
 
 
 def pyrna_enum2sphinx(prop, use_empty_descriptions=False):
-    """ write a bullet point list of enum + descrptons
-    """
+    '''
+    write a bullet point list of enum + descriptions
+    '''
 
     if use_empty_descriptions:
         ok = True
@@ -895,8 +991,9 @@ def pyrna_enum2sphinx(prop, use_empty_descriptions=False):
 
 
 def pyrna2sphinx(basepath):
-    """ bpy.types and bpy.ops
-    """
+    '''
+    bpy.types and bpy.ops
+    '''
     structs, funcs, ops, props = rna_info.BuildRNAInfo()
     if FILTER_BPY_TYPES is not None:
         structs = {k: v for k, v in structs.items() if k[1] in FILTER_BPY_TYPES}
@@ -1313,11 +1410,11 @@ def write_rst_contents(basepath):
     fw(" Blender Documentation contents\n")
     fw("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n")
     fw("\n")
-    fw("Welcome, this document is an API reference for Blender %s. built %s.\n" % (BLENDER_VERSION_DOTS, str(bpy.app.build_date, 'utf_8')))
+    fw("Welcome, this document is an API reference for Blender %s, built %s.\n" % (BLENDER_VERSION_DOTS, BLENDER_DATE))
     fw("\n")
 
-    # fw("`A PDF version of this document is also available <blender_python_reference_%s.pdf>`_\n" % BLENDER_VERSION_PDF)
-    fw("`A compressed ZIP file of this site is available <blender_python_reference_%s.zip>`_\n" % BLENDER_VERSION_PDF)
+    # fw("`A PDF version of this document is also available <%s>`_\n" % BLENDER_PDF_FILENAME)
+    fw("`A compressed ZIP file of this site is available <%s>`_\n" % BLENDER_ZIP_FILENAME)
 
     fw("\n")
 
@@ -1604,7 +1701,7 @@ def align_sphinx_in_to_sphinx_in_tmp():
     # remove deprecated files that have been removed
     for f in sorted(sphinx_in_files):
         if f not in sphinx_in_tmp_files:
-            print("\tdeprecated: %s" % f)
+            BPY_LOGGER.debug("\tdeprecated: %s" % f)
             os.remove(os.path.join(SPHINX_IN, f))
 
     # freshen with new files.
@@ -1618,10 +1715,26 @@ def align_sphinx_in_to_sphinx_in_tmp():
                 do_copy = False
 
         if do_copy:
-            print("\tupdating: %s" % f)
+            BPY_LOGGER.debug("\tupdating: %s" % f)
             shutil.copy(f_from, f_to)
 
 
+def refactor_sphinx_log(sphinx_logfile):
+    refactored_log = []
+    with open(sphinx_logfile, "r", encoding="utf-8") as original_logfile:
+        lines = set(original_logfile.readlines())
+        for line in lines:
+            if 'warning' in line.lower() or 'error' in line.lower():
+                line = line.strip().split(None, 2)
+                if len(line) == 3:
+                    location, kind, msg = line
+                    location = os.path.relpath(location, start=SPHINX_IN)
+                    refactored_log.append((kind, location, msg))
+    with open(sphinx_logfile, "w", encoding="utf-8") as refactored_logfile:
+        for log in sorted(refactored_log):
+            refactored_logfile.write("%-12s %s\n             %s\n" % log)
+
+
 def main():
 
     # dirs preparation
@@ -1630,27 +1743,36 @@ def main():
             os.mkdir(dir_path)
 
     # dump the api in rst files
+    shutil.rmtree(SPHINX_IN_TMP, True)
+    rna2sphinx(SPHINX_IN_TMP)
     if ARGS.full_rebuild:
         # only for full updates
         shutil.rmtree(SPHINX_IN, True)
-        shutil.rmtree(SPHINX_OUT, True)
-        rna2sphinx(SPHINX_IN_TMP)
         shutil.copytree(SPHINX_IN_TMP,
                         SPHINX_IN,
                         copy_function=shutil.copy)
+        if ARGS.sphinx_build:
+            shutil.rmtree(SPHINX_OUT, True)
+        if ARGS.sphinx_build_pdf:
+            shutil.rmtree(SPHINX_OUT_PDF, True)
     else:
-        # write here, then move
-        shutil.rmtree(SPHINX_IN_TMP, True)
-        rna2sphinx(SPHINX_IN_TMP)
+        # move changed files in SPHINX_IN
         align_sphinx_in_to_sphinx_in_tmp()
 
     # report which example files weren't used
     EXAMPLE_SET_UNUSED = EXAMPLE_SET - EXAMPLE_SET_USED
     if EXAMPLE_SET_UNUSED:
-        print("\nUnused examples found in '%s'..." % EXAMPLES_DIR)
+        BPY_LOGGER.debug("\nUnused examples found in '%s'..." % EXAMPLES_DIR)
         for f in EXAMPLE_SET_UNUSED:
-            print("    %s.py" % f)
-        print("  %d total\n" % len(EXAMPLE_SET_UNUSED))
+            BPY_LOGGER.debug("    %s.py" % f)
+        BPY_LOGGER.debug("  %d total\n" % len(EXAMPLE_SET_UNUSED))
+
+    # copy the examples dir
+    examples_dir = os.path.join(ARGS.output_dir, "examples")
+    shutil.rmtree(examples_dir, True)
+    shutil.copytree(EXAMPLES_DIR,
+                    examples_dir,
+                    ignore=shutil.ignore_patterns(*(".svn",)))
 
     # eventually, copy the theme in the output directory
     if ARGS.sphinx_theme in SPHINX_THEMES['bf']:
@@ -1659,13 +1781,69 @@ def main():
                             SPHINX_THEME_DIR,
                             copy_function=shutil.copy)
 
-    # eventually, build the docs
+    # eventually, build the html docs
     if ARGS.sphinx_build:
         import subprocess
-        sphinx_build_command = "sphinx-build %s %s" % (SPHINX_IN, SPHINX_OUT)
-        print ('\n%s\n' % sphinx_build_command)
-        subprocess.call(sphinx_build_command, shell=True)
+        subprocess.call(SPHINX_BUILD)
 
+        # sphinx-build log cleanup+sort
+        if ARGS.sphinx_log:
+            if os.stat(SPHINX_BUILD_LOG).st_size:
+                refactor_sphinx_log(SPHINX_BUILD_LOG)
+
+    # eventually, build the pdf docs
+    if ARGS.sphinx_build_pdf:
+        import subprocess
+        subprocess.call(SPHINX_BUILD_PDF)
+        subprocess.call(SPHINX_MAKE_PDF, stdout=SPHINX_MAKE_PDF_STDOUT)
+        # sphinx-build log cleanup+sort
+        if ARGS.sphinx_log:
+            if os.stat(SPHINX_BUILD_PDF_LOG).st_size:
+                refactor_sphinx_log(SPHINX_BUILD_PDF_LOG)
+
+    # eventually, prepare the dir to be deployed online (REFERENCE_PATH)
+    if ARGS.pack_reference:
+
+        if ARGS.sphinx_build:
+            # delete REFERENCE_PATH
+            if os.path.exists(REFERENCE_PATH):
+                shutil.rmtree(REFERENCE_PATH, True)
+            
+            # copy SPHINX_OUT to the REFERENCE_PATH
+            ignores = ('.doctrees', 'objects.inv', '.buildinfo')
+            shutil.copytree(SPHINX_OUT,
+                            REFERENCE_PATH,
+                            ignore=shutil.ignore_patterns(*ignores))
+
+            # zip REFERENCE_PATH
+            from zipfile import ZipFile
+            # zip out of REFERENCE_PATH to avoid infinite loop
+            zip_tmp_path = os.path.join(ARGS.output_dir, BLENDER_ZIP_FILENAME)
+            zip_final_path = os.path.join(REFERENCE_PATH, BLENDER_ZIP_FILENAME)
+            with ZipFile(zip_tmp_path, 'w') as tmpzip:
+                for base, dirs, files in os.walk(REFERENCE_PATH):
+                    zipbase = os.path.relpath(base, ARGS.output_dir)
+                    for filename in files:
+                        tmpzip.write(os.path.join(base, filename),
+                                     arcname = os.path.join(zipbase, filename))
+
+            # move the zip to REFERENCE_PATH
+            os.rename(zip_tmp_path, zip_final_path)
+
+        if ARGS.sphinx_build_pdf:
+            # copy the pdf to REFERENCE_PATH
+            shutil.copy(os.path.join(SPHINX_OUT_PDF, "contents.pdf"),
+                        os.path.join(REFERENCE_PATH, BLENDER_PDF_FILENAME))
+        '''
+        TODO
+        1) uncomment the link to the pdf in write_rst_contents()
+        2) do we want to zip also zip the pdf too?
+           if so move the zipping after the pdf generation 
+        
+        3) Unused examples found in '/home/user/my/blender/builds/blender/doc/python_api/examples'...
+           check against patterns in -p
+        '''
+        
     sys.exit()