From 8a3beb0012ddc90cefad162539930d96c3d59458 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Fri, 21 Jan 2011 00:06:30 +0000 Subject: import/export automated testing using CMake's CTest Will need to write full docs on this on the wiki. basic info. - 21 tests, OBJ/3DS/X3D/FBX, 3 tests per format import export. STL, PLY, BVH are TODO. - uses files in ../lib/tests (checkout separate) - run with CMake Makefiles "make test" or "ctest" - currently checks against basic MD5 hash on scene import and file MD5 hash on export (realize this wont work predictably on binary formats *TODO*). - currently uses a generic script for all tests with arguments to specify command to run, expected output, testing method, files to check against etc. Has already proved useful, found a number of bugs in import export and some in blender too. --- CMakeLists.txt | 2 + source/CMakeLists.txt | 2 + source/test/CMakeLists.txt | 202 +++++++++++++++++++++++++++++++++++++++++++++ source/test/bl_test.py | 195 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 401 insertions(+) create mode 100644 source/test/CMakeLists.txt create mode 100644 source/test/bl_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a17eaf3e93..bad6b878d54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,8 @@ cmake_minimum_required(VERSION 2.6) project(Blender) +enable_testing() + #----------------------------------------------------------------------------- # Redirect output files diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index c13abb862b7..46e1405df84 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -34,3 +34,5 @@ endif() if(WINDOWS) add_subdirectory(icons) endif() + +add_subdirectory(test) diff --git a/source/test/CMakeLists.txt b/source/test/CMakeLists.txt new file mode 100644 index 00000000000..8e5dca9386d --- /dev/null +++ b/source/test/CMakeLists.txt @@ -0,0 +1,202 @@ +# -*- mode: cmake; indent-tabs-mode: t; -*- +# $Id: CMakeLists.txt 34198 2011-01-09 15:12:08Z campbellbarton $ +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Contributor(s): Jacques Beaurain. +# +# ***** END GPL LICENSE BLOCK ***** + +# --env-system-scripts allows to run without WITH_INSTALL + +# Use '--write-blend=/tmp/test.blend' to view output + + +set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../lib/tests) +set(TEST_OUT_DIR ${CMAKE_BINARY_DIR}/tests) + +#~ if(NOT IS_DIRECTORY ${TEST_SRC_DIR}) +#~ message(FATAL_ERROR "CMake test directory not found!") +#~ endif() + +# all calls to blender use this +set(GENERIC_ARGS --background --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) + + +# OBJ Import tests +add_test(import_obj_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/cube.obj'\) + --md5=4d090508b812b5e08168aa2614746bda --md5_method=SCENE +) + +add_test(import_obj_nurbs_cyclic ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/nurbs_cyclic.obj'\) + --md5=9e0da7b65b4c4f818a203d56af2d3a4b --md5_method=SCENE + --write-blend=/root/foo99.blend +) + +add_test(import_obj_makehuman ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/makehuman.obj'\) + --md5=e0829dc078b0789e1d81f1071235bc4f --md5_method=SCENE +) + +# OBJ Export tests +add_test(export_obj_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_cube.obj',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_obj_cube.obj + --md5_source=${TEST_OUT_DIR}/export_obj_cube.mtl + --md5=70bdc394c2726203ad26c085176e3484 --md5_method=FILE +) + +add_test(export_obj_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_nurbs.obj',use_selection=False,use_nurbs=True\) + --md5_source=${TEST_OUT_DIR}/export_obj_nurbs.obj + --md5_source=${TEST_OUT_DIR}/export_obj_nurbs.mtl + --md5=a733ae4fa4a591ea9b0912da3af042de --md5_method=FILE +) + +add_test(export_obj_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_all_objects.obj',use_selection=False,use_nurbs=True\) + --md5_source=${TEST_OUT_DIR}/export_obj_all_objects.obj + --md5_source=${TEST_OUT_DIR}/export_obj_all_objects.mtl + --md5=c835899ca8993495af8a13c2f229629b --md5_method=FILE +) + +# X3D Import +add_test(import_x3d_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/color_cube.x3d'\) + --md5=c80538e272812c9d765d43df269d8a9b --md5_method=SCENE +) + +add_test(import_x3d_teapot ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/teapot.x3d'\) + --md5=fa19713ff71d4b3893dcbe0ab3a73955 --md5_method=SCENE + --write-blend=/root/foo99.blend +) + +add_test(import_x3d_suzanne_material ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/suzanne_material.x3d'\) + --md5=52a59dcf731904ac49953dd82c020ae5 --md5_method=SCENE +) + +# X3D Export +add_test(export_x3d_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_cube.x3d',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_x3d_cube.x3d + --md5=560ba3762a6604669994f661235ef93c --md5_method=FILE +) + +add_test(export_x3d_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_nurbs.x3d',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_x3d_nurbs.x3d + --md5=078c0ca5a08f123cd2cdac48afb54853 --md5_method=FILE +) + +add_test(export_x3d_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_all_objects.x3d',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_x3d_all_objects.x3d + --md5=b4bddb55efd8e34af673ffb42bf4c372 --md5_method=FILE +) + +# 3DS Import +add_test(import_3ds_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/cube.3ds'\) + --md5=cb5a45c35a343c3f5beca2a918472951 --md5_method=SCENE +) + +add_test(import_3ds_hierarchy_lara ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/hierarchy_lara.3ds'\) + --md5=2e9812099b26ad607fdcf4c7be918c71 --md5_method=SCENE + --write-blend=/root/foo99.blend +) + +add_test(import_3ds_hierarchy_greek_trireme ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/hierarchy_greek_trireme.3ds'\) + --md5=d05b922d7be20356d8409d1f768a3a9a --md5_method=SCENE +) + +# 3DS Export +add_test(export_3ds_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_cube.3ds',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_3ds_cube.3ds + --md5=0df6cfb130052d01e31ef77d391d4cc0 --md5_method=FILE +) + +add_test(export_3ds_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_nurbs.3ds',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_3ds_nurbs.3ds + --md5=ba1a6d43346fee3bcadc7e30e3c95935 --md5_method=FILE +) + +add_test(export_3ds_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_all_objects.3ds',use_selection=False\) + --md5_source=${TEST_OUT_DIR}/export_3ds_all_objects.3ds + --md5=1523ca2e31cf7d781c7de1e17bd14520 --md5_method=FILE +) + + +# FBX Export +# 'use_metadata=False' for reliable md5's +add_test(export_fbx_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_cube.fbx',use_selection=False,use_metadata=False\) + --md5_source=${TEST_OUT_DIR}/export_fbx_cube.fbx + --md5=ce937e605e493958464d62e6de4a2f9f --md5_method=FILE +) + +add_test(export_fbx_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_nurbs.fbx',use_selection=False,use_metadata=False\) + --md5_source=${TEST_OUT_DIR}/export_fbx_nurbs.fbx + --md5=e02f0147afba2a4ce1ae110567ac3531 --md5_method=FILE +) + +add_test(export_fbx_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS} + ${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend + --python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py -- + --run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_all_objects.fbx',use_selection=False,use_metadata=False\) + --md5_source=${TEST_OUT_DIR}/export_fbx_all_objects.fbx + --md5=c29a3aa600d2e432e4a521cc1e513ba8 --md5_method=FILE +) \ No newline at end of file diff --git a/source/test/bl_test.py b/source/test/bl_test.py new file mode 100644 index 00000000000..86c5cf81e79 --- /dev/null +++ b/source/test/bl_test.py @@ -0,0 +1,195 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# + +import sys +import os + + +# may split this out into a new file +def replace_bpy_app_version(): + """ So MD5's are pradictable from output which uses blenders versions. + """ + + import bpy + + app = bpy.app + app_fake = type(bpy)("bpy.app") + + for attr in dir(app): + if not attr.startswith("_"): + setattr(app_fake, attr, getattr(app, attr)) + + app_fake.version = 0, 0, 0 + app_fake.version_string = "0.00 (sub 0)" + bpy.app = app_fake + + +def clear_startup_blend(): + import bpy + + for scene in bpy.data.scenes: + for obj in scene.objects: + scene.objects.unlink(obj) + + +def blend_to_md5(): + import bpy + scene = bpy.context.scene + ROUND = 4 + + def matrix2str(matrix): + return "".join([str(round(axis, ROUND)) for vector in matrix for axis in vector]).encode('ASCII') + + def coords2str(seq, attr): + return "".join([str(round(axis, ROUND)) for vertex in seq for axis in getattr(vertex, attr)]).encode('ASCII') + + import hashlib + + md5 = hashlib.new("md5") + md5_update = md5.update + + for obj in scene.objects: + md5_update(matrix2str(obj.matrix_world)) + data = obj.data + + if type(data) == bpy.types.Mesh: + md5_update(coords2str(data.vertices, "co")) + elif type(data) == bpy.types.Curve: + for spline in data.splines: + md5_update(coords2str(spline.bezier_points, "co")) + md5_update(coords2str(spline.points, "co")) + + return md5.hexdigest() + + +def main(): + argv = sys.argv + print(" args:", " ".join(argv)) + argv = argv[argv.index("--") + 1:] + + def arg_extract(arg, optional=True, array=False): + arg += "=" + if array: + value = [] + else: + value = None + + i = 0 + while i < len(argv): + if argv[i].startswith(arg): + item = argv[i][len(arg):] + del argv[i] + i -= 1 + + if array: + value.append(item) + else: + value = item + break + + i += 1 + + if (not value) and (not optional): + print(" '%s' not set" % arg) + sys.exit(1) + + return value + + run = arg_extract("--run", optional=False) + md5 = arg_extract("--md5", optional=False) + md5_method = arg_extract("--md5_method", optional=False) # 'SCENE' / 'FILE' + + # only when md5_method is 'FILE' + md5_source = arg_extract("--md5_source", optional=True, array=True) + + # save blend file, for testing + write_blend = arg_extract("--write-blend", optional=True) + + # ensure files are written anew + for f in md5_source: + if os.path.exists(f): + os.remove(f) + + import bpy + + replace_bpy_app_version() + if not bpy.data.filepath: + clear_startup_blend() + + print(" Running: '%s'" % run) + print(" MD5: '%s'!" % md5) + + try: + result = eval(run) + except: + import traceback + traceback.print_exc() + sys.exit(1) + + if write_blend is not None: + print(" Writing Blend: %s" % write_blend) + bpy.ops.wm.save_mainfile(filepath=write_blend, check_existing=False) + + print(" Result: '%s'" % str(result)) + if not result: + print(" Running: %s -> False" % run) + sys.exit(1) + + if md5_method == 'SCENE': + md5_new = blend_to_md5() + elif md5_method == 'FILE': + if not md5_source: + print(" Missing --md5_source argument") + sys.exit(1) + + for f in md5_source: + if not os.path.exists(f): + print(" Missing --md5_source=%r argument does not point to a file") + sys.exit(1) + + import hashlib + + md5_instance = hashlib.new("md5") + md5_update = md5_instance.update + + for f in md5_source: + md5_update(open(f, "rb").read()) + + md5_new = md5_instance.hexdigest() + + else: + print(" Invalid --md5_method=%s argument is not a valid source") + sys.exit(1) + + if md5 != md5_new: + print(" Running: %s\n MD5 Recieved: %s\n MD5 Expected: %s" % (run, md5_new, md5)) + sys.exit(1) + + print(" Success: %s" % run) + + +if __name__ == "__main__": + # So a python error exits(1) + try: + main() + except: + import traceback + traceback.print_exc() + sys.exit(1) -- cgit v1.2.3