import/export automated testing using CMake's CTest

Will need to write full docs on this on the wiki.
basic info.
- 21 tests, OBJ/3DS/X3D/FBX, 3 tests per format import export. STL, PLY, BVH are TODO.
- uses files in ../lib/tests (checkout separate)
- run with CMake Makefiles "make test" or "ctest"
- currently checks against basic MD5 hash on scene import and file MD5 hash on export (realize this wont work predictably on binary formats *TODO*).
- currently uses a generic script for all tests with arguments to specify command to run, expected output, testing method, files to check against etc.

Has already proved useful, found a number of bugs in import export and some in blender too.

3 files changed, 399 insertions, 0 deletions

diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index c13abb862b7..46e1405df84 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -34,3 +34,5 @@ endif()
 if(WINDOWS)
 	add_subdirectory(icons)
 endif()
+
+add_subdirectory(test)
diff --git a/source/test/CMakeLists.txt b/source/test/CMakeLists.txt
new file mode 100644
index 00000000000..8e5dca9386d
--- /dev/null
+++ b/source/test/CMakeLists.txt
@@ -0,0 +1,202 @@
+# -*- mode: cmake; indent-tabs-mode: t; -*-
+# $Id: CMakeLists.txt 34198 2011-01-09 15:12:08Z campbellbarton $
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# Contributor(s): Jacques Beaurain.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+# --env-system-scripts allows to run without WITH_INSTALL
+
+# Use '--write-blend=/tmp/test.blend' to view output
+
+
+set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../lib/tests)
+set(TEST_OUT_DIR ${CMAKE_BINARY_DIR}/tests)
+
+#~ if(NOT IS_DIRECTORY ${TEST_SRC_DIR})
+#~ 	message(FATAL_ERROR "CMake test directory not found!")
+#~ endif()
+
+# all calls to blender use this
+set(GENERIC_ARGS --background --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
+
+
+# OBJ Import tests
+add_test(import_obj_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/cube.obj'\)
+	--md5=4d090508b812b5e08168aa2614746bda --md5_method=SCENE
+)
+
+add_test(import_obj_nurbs_cyclic ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/nurbs_cyclic.obj'\)
+	--md5=9e0da7b65b4c4f818a203d56af2d3a4b --md5_method=SCENE
+	--write-blend=/root/foo99.blend
+)
+
+add_test(import_obj_makehuman ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/makehuman.obj'\)
+	--md5=e0829dc078b0789e1d81f1071235bc4f --md5_method=SCENE
+)
+
+# OBJ Export tests
+add_test(export_obj_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_cube.obj',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_obj_cube.obj
+	--md5_source=${TEST_OUT_DIR}/export_obj_cube.mtl
+	--md5=70bdc394c2726203ad26c085176e3484 --md5_method=FILE
+)
+
+add_test(export_obj_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_nurbs.obj',use_selection=False,use_nurbs=True\)
+	--md5_source=${TEST_OUT_DIR}/export_obj_nurbs.obj
+	--md5_source=${TEST_OUT_DIR}/export_obj_nurbs.mtl
+	--md5=a733ae4fa4a591ea9b0912da3af042de --md5_method=FILE
+)
+
+add_test(export_obj_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_all_objects.obj',use_selection=False,use_nurbs=True\)
+	--md5_source=${TEST_OUT_DIR}/export_obj_all_objects.obj
+	--md5_source=${TEST_OUT_DIR}/export_obj_all_objects.mtl
+	--md5=c835899ca8993495af8a13c2f229629b --md5_method=FILE
+)
+
+# X3D Import
+add_test(import_x3d_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/color_cube.x3d'\)
+	--md5=c80538e272812c9d765d43df269d8a9b --md5_method=SCENE
+)
+
+add_test(import_x3d_teapot ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/teapot.x3d'\)
+	--md5=fa19713ff71d4b3893dcbe0ab3a73955 --md5_method=SCENE
+	--write-blend=/root/foo99.blend
+)
+
+add_test(import_x3d_suzanne_material ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/suzanne_material.x3d'\)
+	--md5=52a59dcf731904ac49953dd82c020ae5 --md5_method=SCENE
+)
+
+# X3D Export
+add_test(export_x3d_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_cube.x3d',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_x3d_cube.x3d
+	--md5=560ba3762a6604669994f661235ef93c --md5_method=FILE
+)
+
+add_test(export_x3d_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_nurbs.x3d',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_x3d_nurbs.x3d
+	--md5=078c0ca5a08f123cd2cdac48afb54853 --md5_method=FILE
+)
+
+add_test(export_x3d_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_all_objects.x3d',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_x3d_all_objects.x3d
+	--md5=b4bddb55efd8e34af673ffb42bf4c372 --md5_method=FILE
+)
+
+# 3DS Import
+add_test(import_3ds_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/cube.3ds'\)
+	--md5=cb5a45c35a343c3f5beca2a918472951 --md5_method=SCENE
+)
+
+add_test(import_3ds_hierarchy_lara ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/hierarchy_lara.3ds'\)
+	--md5=2e9812099b26ad607fdcf4c7be918c71 --md5_method=SCENE
+	--write-blend=/root/foo99.blend
+)
+
+add_test(import_3ds_hierarchy_greek_trireme ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/hierarchy_greek_trireme.3ds'\)
+	--md5=d05b922d7be20356d8409d1f768a3a9a --md5_method=SCENE
+)
+
+# 3DS Export
+add_test(export_3ds_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_cube.3ds',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_3ds_cube.3ds
+	--md5=0df6cfb130052d01e31ef77d391d4cc0 --md5_method=FILE
+)
+
+add_test(export_3ds_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_nurbs.3ds',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_3ds_nurbs.3ds
+	--md5=ba1a6d43346fee3bcadc7e30e3c95935 --md5_method=FILE
+)
+
+add_test(export_3ds_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_all_objects.3ds',use_selection=False\)
+	--md5_source=${TEST_OUT_DIR}/export_3ds_all_objects.3ds
+	--md5=1523ca2e31cf7d781c7de1e17bd14520   --md5_method=FILE
+)
+
+
+# FBX Export
+# 'use_metadata=False' for reliable md5's
+add_test(export_fbx_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_cube.fbx',use_selection=False,use_metadata=False\)
+	--md5_source=${TEST_OUT_DIR}/export_fbx_cube.fbx
+	--md5=ce937e605e493958464d62e6de4a2f9f --md5_method=FILE
+)
+
+add_test(export_fbx_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_nurbs.fbx',use_selection=False,use_metadata=False\)
+	--md5_source=${TEST_OUT_DIR}/export_fbx_nurbs.fbx
+	--md5=e02f0147afba2a4ce1ae110567ac3531 --md5_method=FILE
+)
+
+add_test(export_fbx_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
+	${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
+	--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
+	--run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_all_objects.fbx',use_selection=False,use_metadata=False\)
+	--md5_source=${TEST_OUT_DIR}/export_fbx_all_objects.fbx
+	--md5=c29a3aa600d2e432e4a521cc1e513ba8   --md5_method=FILE
+)
\ No newline at end of file
diff --git a/source/test/bl_test.py b/source/test/bl_test.py
new file mode 100644
index 00000000000..86c5cf81e79
--- /dev/null
+++ b/source/test/bl_test.py
@@ -0,0 +1,195 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+# <pep8 compliant>
+
+import sys
+import os
+
+
+# may split this out into a new file
+def replace_bpy_app_version():
+    """ So MD5's are pradictable from output which uses blenders versions.
+    """
+
+    import bpy
+
+    app = bpy.app
+    app_fake = type(bpy)("bpy.app")
+
+    for attr in dir(app):
+        if not attr.startswith("_"):
+            setattr(app_fake, attr, getattr(app, attr))
+
+    app_fake.version = 0, 0, 0
+    app_fake.version_string = "0.00 (sub 0)"
+    bpy.app = app_fake
+
+
+def clear_startup_blend():
+    import bpy
+
+    for scene in bpy.data.scenes:
+        for obj in scene.objects:
+            scene.objects.unlink(obj)
+
+
+def blend_to_md5():
+    import bpy
+    scene = bpy.context.scene
+    ROUND = 4
+
+    def matrix2str(matrix):
+        return "".join([str(round(axis, ROUND)) for vector in matrix for axis in vector]).encode('ASCII')
+
+    def coords2str(seq, attr):
+        return "".join([str(round(axis, ROUND)) for vertex in seq for axis in getattr(vertex, attr)]).encode('ASCII')
+
+    import hashlib
+
+    md5 = hashlib.new("md5")
+    md5_update = md5.update
+
+    for obj in scene.objects:
+        md5_update(matrix2str(obj.matrix_world))
+        data = obj.data
+
+        if type(data) == bpy.types.Mesh:
+            md5_update(coords2str(data.vertices, "co"))
+        elif type(data) == bpy.types.Curve:
+            for spline in data.splines:
+                md5_update(coords2str(spline.bezier_points, "co"))
+                md5_update(coords2str(spline.points, "co"))
+
+    return md5.hexdigest()
+
+
+def main():
+    argv = sys.argv
+    print("  args:", " ".join(argv))
+    argv = argv[argv.index("--") + 1:]
+
+    def arg_extract(arg, optional=True, array=False):
+        arg += "="
+        if array:
+            value = []
+        else:
+            value = None
+
+        i = 0
+        while i < len(argv):
+            if argv[i].startswith(arg):
+                item = argv[i][len(arg):]
+                del argv[i]
+                i -= 1
+
+                if array:
+                    value.append(item)
+                else:
+                    value = item
+                    break
+
+            i += 1
+
+        if (not value) and (not optional):
+            print("  '%s' not set" % arg)
+            sys.exit(1)
+
+        return value
+
+    run = arg_extract("--run", optional=False)
+    md5 = arg_extract("--md5", optional=False)
+    md5_method = arg_extract("--md5_method", optional=False)  # 'SCENE' / 'FILE'
+
+    # only when md5_method is 'FILE'
+    md5_source = arg_extract("--md5_source", optional=True, array=True)
+
+    # save blend file, for testing
+    write_blend = arg_extract("--write-blend", optional=True)
+
+    # ensure files are written anew
+    for f in md5_source:
+        if os.path.exists(f):
+            os.remove(f)
+
+    import bpy
+
+    replace_bpy_app_version()
+    if not bpy.data.filepath:
+        clear_startup_blend()
+
+    print("  Running: '%s'" % run)
+    print("  MD5: '%s'!" % md5)
+
+    try:
+        result = eval(run)
+    except:
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+    if write_blend is not None:
+        print("  Writing Blend: %s" % write_blend)
+        bpy.ops.wm.save_mainfile(filepath=write_blend, check_existing=False)
+
+    print("  Result: '%s'" % str(result))
+    if not result:
+        print("  Running: %s -> False" % run)
+        sys.exit(1)
+
+    if md5_method == 'SCENE':
+        md5_new = blend_to_md5()
+    elif md5_method == 'FILE':
+        if not md5_source:
+            print("  Missing --md5_source argument")
+            sys.exit(1)
+
+        for f in md5_source:
+            if not os.path.exists(f):
+                print("  Missing --md5_source=%r argument does not point to a file")
+                sys.exit(1)
+
+        import hashlib
+
+        md5_instance = hashlib.new("md5")
+        md5_update = md5_instance.update
+
+        for f in md5_source:
+            md5_update(open(f, "rb").read())
+
+        md5_new = md5_instance.hexdigest()
+
+    else:
+        print("  Invalid --md5_method=%s argument is not a valid source")
+        sys.exit(1)
+
+    if md5 != md5_new:
+        print("  Running: %s\n    MD5 Recieved: %s\n    MD5 Expected: %s" % (run, md5_new, md5))
+        sys.exit(1)
+
+    print("  Success: %s" % run)
+
+
+if __name__ == "__main__":
+    # So a python error exits(1)
+    try:
+        main()
+    except:
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)