From 54f8a5dd737388d89332f3074eba01edc9efa8c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 11 Jan 2021 11:29:30 +0100 Subject: Tests: run suites instead of individual test cases Group all tests of a test suite into a single test command invocation. This reduces the number of invocations by `ctest` by an order of magnitude. Since rB56aa5b0d8c6b663, `bin/tests/blender_test` was run for every individual test. Having over a 1000 tests made testing slower than necessary. Individual tests can still be run if desired by invocation of `bin/tests/blender_test --gtest_filter=suitename.testname`. NOTE: For this commit to have an immediate effect, it may be necessary to remove the `tests` and `Testing` directories and some CMake files from your build directory and rebuild. Run `ctest -N` to see the list of tests; there should be less than 200. Reviewed By: sergey, LazyDodo, sebbas Maniphest Tasks: T83222 Differential Revision: https://developer.blender.org/D9649 --- build_files/cmake/Modules/GTest.cmake | 178 +++-------------------- build_files/cmake/Modules/GTestAddTests.cmake | 194 -------------------------- build_files/cmake/Modules/GTestTesting.cmake | 19 ++- build_files/cmake/macros.cmake | 55 ++++++-- 4 files changed, 78 insertions(+), 368 deletions(-) delete mode 100644 build_files/cmake/Modules/GTestAddTests.cmake (limited to 'build_files') diff --git a/build_files/cmake/Modules/GTest.cmake b/build_files/cmake/Modules/GTest.cmake index 6981c1ddc55..d32e49d2145 100644 --- a/build_files/cmake/Modules/GTest.cmake +++ b/build_files/cmake/Modules/GTest.cmake @@ -330,6 +330,9 @@ function(gtest_add_tests) set(gtest_case_name_regex ".*\\( *([A-Za-z_0-9]+) *, *([A-Za-z_0-9]+) *\\).*") set(gtest_test_type_regex "(TYPED_TEST|TEST_?[FP]?)") + # This will get a filter for each test suite. + set(test_filters "") + foreach(source IN LISTS ARGS_SOURCES) if(NOT ARGS_SKIP_DEPENDENCY) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${source}) @@ -376,175 +379,32 @@ function(gtest_add_tests) list(APPEND testList ${ctest_test_name}) endif() else() - set(ctest_test_name ${ARGS_TEST_PREFIX}${gtest_test_name}${ARGS_TEST_SUFFIX}) - add_test(NAME ${ctest_test_name} - ${workDir} - COMMAND ${ARGS_TARGET} - --gtest_filter=${gtest_test_name} - ${ARGS_EXTRA_ARGS} - ) - list(APPEND testList ${ctest_test_name}) + # BLENDER: collect tests named "suite.testcase" as list of "suite.*" filters. + string(REGEX REPLACE "\\..*$" "" gtest_suite_name ${gtest_test_name}) + list(APPEND test_filters "${gtest_suite_name}.*") endif() endforeach() endforeach() - if(ARGS_TEST_LIST) - set(${ARGS_TEST_LIST} ${testList} PARENT_SCOPE) - endif() - -endfunction() - -#------------------------------------------------------------------------------ - -function(gtest_discover_tests TARGET) - cmake_parse_arguments( - "" - "NO_PRETTY_TYPES;NO_PRETTY_VALUES" - "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;DISCOVERY_TIMEOUT;XML_OUTPUT_DIR;DISCOVERY_MODE" - "EXTRA_ARGS;PROPERTIES" - ${ARGN} - ) - - if(NOT _WORKING_DIRECTORY) - set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") - endif() - if(NOT _TEST_LIST) - set(_TEST_LIST ${TARGET}_TESTS) - endif() - if(NOT _DISCOVERY_TIMEOUT) - set(_DISCOVERY_TIMEOUT 5) - endif() - if(NOT _DISCOVERY_MODE) - if(NOT CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE) - set(CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE "POST_BUILD") - endif() - set(_DISCOVERY_MODE ${CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE}) - endif() - - get_property( - has_counter - TARGET ${TARGET} - PROPERTY CTEST_DISCOVERED_TEST_COUNTER - SET - ) - if(has_counter) - get_property( - counter - TARGET ${TARGET} - PROPERTY CTEST_DISCOVERED_TEST_COUNTER - ) - math(EXPR counter "${counter} + 1") - else() - set(counter 1) - endif() - set_property( - TARGET ${TARGET} - PROPERTY CTEST_DISCOVERED_TEST_COUNTER - ${counter} + # Join all found GTest suite names into one big filter. + list(REMOVE_DUPLICATES test_filters) + list(JOIN test_filters ":" gtest_filter) + add_test(NAME ${ARGS_TEST_PREFIX} + ${workDir} + COMMAND ${ARGS_TARGET} + --gtest_filter=${gtest_filter} + ${ARGS_EXTRA_ARGS} ) + list(APPEND testList ${ARGS_TEST_PREFIX}) - # Define rule to generate test list for aforementioned test executable - # Blender: use _ instead of [] to avoid problems with zsh regex. - set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_${counter}_") - set(ctest_include_file "${ctest_file_base}_include.cmake") - set(ctest_tests_file "${ctest_file_base}_tests.cmake") - get_property(crosscompiling_emulator - TARGET ${TARGET} - PROPERTY CROSSCOMPILING_EMULATOR - ) - - if(_DISCOVERY_MODE STREQUAL "POST_BUILD") - add_custom_command( - TARGET ${TARGET} POST_BUILD - BYPRODUCTS "${ctest_tests_file}" - COMMAND "${CMAKE_COMMAND}" - -D "TEST_TARGET=${TARGET}" - -D "TEST_EXECUTABLE=$" - -D "TEST_EXECUTOR=${crosscompiling_emulator}" - -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" - -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" - -D "TEST_PROPERTIES=${_PROPERTIES}" - -D "TEST_PREFIX=${_TEST_PREFIX}" - -D "TEST_SUFFIX=${_TEST_SUFFIX}" - -D "NO_PRETTY_TYPES=${_NO_PRETTY_TYPES}" - -D "NO_PRETTY_VALUES=${_NO_PRETTY_VALUES}" - -D "TEST_LIST=${_TEST_LIST}" - -D "CTEST_FILE=${ctest_tests_file}" - -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}" - -D "TEST_XML_OUTPUT_DIR=${_XML_OUTPUT_DIR}" - -P "${_GOOGLETEST_DISCOVER_TESTS_SCRIPT}" - VERBATIM - ) - - file(WRITE "${ctest_include_file}" - "if(EXISTS \"${ctest_tests_file}\")\n" - " include(\"${ctest_tests_file}\")\n" - "else()\n" - " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n" - "endif()\n" - ) - elseif(_DISCOVERY_MODE STREQUAL "PRE_TEST") - - get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL - PROPERTY GENERATOR_IS_MULTI_CONFIG - ) - - if(GENERATOR_IS_MULTI_CONFIG) - set(ctest_tests_file "${ctest_file_base}_tests-$.cmake") - endif() - - string(CONCAT ctest_include_content - "if(EXISTS \"$\")" "\n" - " if(\"$\" IS_NEWER_THAN \"${ctest_tests_file}\")" "\n" - " include(\"${_GOOGLETEST_DISCOVER_TESTS_SCRIPT}\")" "\n" - " gtest_discover_tests_impl(" "\n" - " TEST_EXECUTABLE" " [==[" "$" "]==]" "\n" - " TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n" - " TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n" - " TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n" - " TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n" - " TEST_PREFIX" " [==[" "${_TEST_PREFIX}" "]==]" "\n" - " TEST_SUFFIX" " [==[" "${_TEST_SUFFIX}" "]==]" "\n" - " NO_PRETTY_TYPES" " [==[" "${_NO_PRETTY_TYPES}" "]==]" "\n" - " NO_PRETTY_VALUES" " [==[" "${_NO_PRETTY_VALUES}" "]==]" "\n" - " TEST_LIST" " [==[" "${_TEST_LIST}" "]==]" "\n" - " CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n" - " TEST_DISCOVERY_TIMEOUT" " [==[" "${_DISCOVERY_TIMEOUT}" "]==]" "\n" - " TEST_XML_OUTPUT_DIR" " [==[" "${_XML_OUTPUT_DIR}" "]==]" "\n" - " )" "\n" - " endif()" "\n" - " include(\"${ctest_tests_file}\")" "\n" - "else()" "\n" - " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n" - "endif()" "\n" - ) - - if(GENERATOR_IS_MULTI_CONFIG) - foreach(_config ${CMAKE_CONFIGURATION_TYPES}) - file(GENERATE OUTPUT "${ctest_file_base}_include-${_config}.cmake" CONTENT "${ctest_include_content}" CONDITION $) - endforeach() - file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")") - else() - file(GENERATE OUTPUT "${ctest_file_base}_include.cmake" CONTENT "${ctest_include_content}") - file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include.cmake\")") - endif() - - else() - message(FATAL_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}") + if(ARGS_TEST_LIST) + set(${ARGS_TEST_LIST} ${testList} PARENT_SCOPE) endif() - # Add discovered tests to directory TEST_INCLUDE_FILES - set_property(DIRECTORY - APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" - ) - endfunction() -############################################################################### - -set(_GOOGLETEST_DISCOVER_TESTS_SCRIPT - ${CMAKE_CURRENT_LIST_DIR}/GTestAddTests.cmake -) +# BLENDER: remove the discovery function gtest_discover_tests(). It's not used, +# as it generates too many test invocations. # Restore project's policies cmake_policy(POP) diff --git a/build_files/cmake/Modules/GTestAddTests.cmake b/build_files/cmake/Modules/GTestAddTests.cmake deleted file mode 100644 index 116531bb07b..00000000000 --- a/build_files/cmake/Modules/GTestAddTests.cmake +++ /dev/null @@ -1,194 +0,0 @@ -# Distributed under the OSI-approved BSD 3-Clause License, -# see accompanying file BSD-3-Clause-license.txt for details. - -# Changes made to this script have been marked with "BLENDER". - - -# BLENDER: disable ASAN leak detection when trying to discover tests. -set(ENV{ASAN_OPTIONS} "detect_leaks=0") - -cmake_minimum_required(VERSION ${CMAKE_VERSION}) - -# Overwrite possibly existing ${_CTEST_FILE} with empty file -set(flush_tests_MODE WRITE) - -# Flushes script to ${_CTEST_FILE} -macro(flush_script) - file(${flush_tests_MODE} "${_CTEST_FILE}" "${script}") - set(flush_tests_MODE APPEND) - - set(script "") -endmacro() - -# Flushes tests_buffer to tests -macro(flush_tests_buffer) - list(APPEND tests "${tests_buffer}") - set(tests_buffer "") -endmacro() - -macro(add_command NAME) - set(_args "") - foreach(_arg ${ARGN}) - if(_arg MATCHES "[^-./:a-zA-Z0-9_]") - string(APPEND _args " [==[${_arg}]==]") - else() - string(APPEND _args " ${_arg}") - endif() - endforeach() - string(APPEND script "${NAME}(${_args})\n") - string(LENGTH "${script}" _script_len) - if(${_script_len} GREATER "50000") - flush_script() - endif() - # Unsets macro local variables to prevent leakage outside of this macro. - unset(_args) - unset(_script_len) -endmacro() - -function(gtest_discover_tests_impl) - - cmake_parse_arguments( - "" - "" - "NO_PRETTY_TYPES;NO_PRETTY_VALUES;TEST_EXECUTABLE;TEST_EXECUTOR;TEST_WORKING_DIR;TEST_PREFIX;TEST_SUFFIX;TEST_LIST;CTEST_FILE;TEST_DISCOVERY_TIMEOUT;TEST_XML_OUTPUT_DIR" - "TEST_EXTRA_ARGS;TEST_PROPERTIES" - ${ARGN} - ) - - set(prefix "${_TEST_PREFIX}") - set(suffix "${_TEST_SUFFIX}") - set(extra_args ${_TEST_EXTRA_ARGS}) - set(properties ${_TEST_PROPERTIES}) - set(script) - set(suite) - set(tests) - set(tests_buffer) - - # Run test executable to get list of available tests - if(NOT EXISTS "${_TEST_EXECUTABLE}") - message(FATAL_ERROR - "Specified test executable does not exist.\n" - " Path: '${_TEST_EXECUTABLE}'" - ) - endif() - execute_process( - COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" --gtest_list_tests - WORKING_DIRECTORY "${_TEST_WORKING_DIR}" - TIMEOUT ${_TEST_DISCOVERY_TIMEOUT} - OUTPUT_VARIABLE output - RESULT_VARIABLE result - ) - if(NOT ${result} EQUAL 0) - string(REPLACE "\n" "\n " output "${output}") - message(FATAL_ERROR - "Error running test executable.\n" - " Path: '${_TEST_EXECUTABLE}'\n" - " Result: ${result}\n" - " Output:\n" - " ${output}\n" - ) - endif() - - # Preserve semicolon in test-parameters - string(REPLACE [[;]] [[\;]] output "${output}") - string(REPLACE "\n" ";" output "${output}") - - # Parse output - foreach(line ${output}) - # Skip header - if(NOT line MATCHES "gtest_main\\.cc") - # Do we have a module name or a test name? - if(NOT line MATCHES "^ ") - # Module; remove trailing '.' to get just the name... - string(REGEX REPLACE "\\.( *#.*)?" "" suite "${line}") - if(line MATCHES "#" AND NOT _NO_PRETTY_TYPES) - string(REGEX REPLACE "/[0-9]\\.+ +#.*= +" "/" pretty_suite "${line}") - else() - set(pretty_suite "${suite}") - endif() - string(REGEX REPLACE "^DISABLED_" "" pretty_suite "${pretty_suite}") - else() - # Test name; strip spaces and comments to get just the name... - string(REGEX REPLACE " +" "" test "${line}") - if(test MATCHES "#" AND NOT _NO_PRETTY_VALUES) - string(REGEX REPLACE "/[0-9]+#GetParam..=" "/" pretty_test "${test}") - else() - string(REGEX REPLACE "#.*" "" pretty_test "${test}") - endif() - string(REGEX REPLACE "^DISABLED_" "" pretty_test "${pretty_test}") - string(REGEX REPLACE "#.*" "" test "${test}") - if(NOT "${_TEST_XML_OUTPUT_DIR}" STREQUAL "") - set(TEST_XML_OUTPUT_PARAM "--gtest_output=xml:${_TEST_XML_OUTPUT_DIR}/${prefix}${suite}.${test}${suffix}.xml") - else() - unset(TEST_XML_OUTPUT_PARAM) - endif() - - # sanitize test name for further processing downstream - set(testname "${prefix}${pretty_suite}.${pretty_test}${suffix}") - # escape \ - string(REPLACE [[\]] [[\\]] testname "${testname}") - # escape ; - string(REPLACE [[;]] [[\;]] testname "${testname}") - # escape $ - string(REPLACE [[$]] [[\$]] testname "${testname}") - - # ...and add to script - add_command(add_test - "${testname}" - ${_TEST_EXECUTOR} - "${_TEST_EXECUTABLE}" - "--gtest_filter=${suite}.${test}" - "--gtest_also_run_disabled_tests" - ${TEST_XML_OUTPUT_PARAM} - ${extra_args} - ) - if(suite MATCHES "^DISABLED" OR test MATCHES "^DISABLED") - add_command(set_tests_properties - "${testname}" - PROPERTIES DISABLED TRUE - ) - endif() - add_command(set_tests_properties - "${testname}" - PROPERTIES - WORKING_DIRECTORY "${_TEST_WORKING_DIR}" - SKIP_REGULAR_EXPRESSION "\\\\[ SKIPPED \\\\]" - ${properties} - ) - list(APPEND tests_buffer "${testname}") - list(LENGTH tests_buffer tests_buffer_length) - if(${tests_buffer_length} GREATER "250") - flush_tests_buffer() - endif() - endif() - endif() - endforeach() - - - # Create a list of all discovered tests, which users may use to e.g. set - # properties on the tests - flush_tests_buffer() - add_command(set ${_TEST_LIST} ${tests}) - - # Write CTest script - flush_script() - -endfunction() - -if(CMAKE_SCRIPT_MODE_FILE) - gtest_discover_tests_impl( - NO_PRETTY_TYPES ${NO_PRETTY_TYPES} - NO_PRETTY_VALUES ${NO_PRETTY_VALUES} - TEST_EXECUTABLE ${TEST_EXECUTABLE} - TEST_EXECUTOR ${TEST_EXECUTOR} - TEST_WORKING_DIR ${TEST_WORKING_DIR} - TEST_PREFIX ${TEST_PREFIX} - TEST_SUFFIX ${TEST_SUFFIX} - TEST_LIST ${TEST_LIST} - CTEST_FILE ${CTEST_FILE} - TEST_DISCOVERY_TIMEOUT ${TEST_DISCOVERY_TIMEOUT} - TEST_XML_OUTPUT_DIR ${TEST_XML_OUTPUT_DIR} - TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS} - TEST_PROPERTIES ${TEST_PROPERTIES} - ) -endif() diff --git a/build_files/cmake/Modules/GTestTesting.cmake b/build_files/cmake/Modules/GTestTesting.cmake index 6406dd6aa28..70f48fc33ea 100644 --- a/build_files/cmake/Modules/GTestTesting.cmake +++ b/build_files/cmake/Modules/GTestTesting.cmake @@ -8,6 +8,17 @@ # #============================================================================= +function(GET_BLENDER_TEST_INSTALL_DIR VARIABLE_NAME) + get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if(GENERATOR_IS_MULTI_CONFIG) + string(REPLACE "\${BUILD_TYPE}" "$" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) + else() + string(REPLACE "\${BUILD_TYPE}" "" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) + endif() + set(${VARIABLE_NAME} "${TEST_INSTALL_DIR}" PARENT_SCOPE) +endfunction() + + macro(BLENDER_SRC_GTEST_EX) if(WITH_GTESTS) set(options SKIP_ADD_TEST) @@ -75,13 +86,7 @@ macro(BLENDER_SRC_GTEST_EX) target_link_libraries(${TARGET_NAME} ${GMP_LIBRARIES}) endif() - get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) - if(GENERATOR_IS_MULTI_CONFIG) - string(REPLACE "\${BUILD_TYPE}" "$" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) - else() - string(REPLACE "\${BUILD_TYPE}" "" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) - endif() - + GET_BLENDER_TEST_INSTALL_DIR(TEST_INSTALL_DIR) set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${TESTS_OUTPUT_DIR}" RUNTIME_OUTPUT_DIRECTORY_RELEASE "${TESTS_OUTPUT_DIR}" diff --git a/build_files/cmake/macros.cmake b/build_files/cmake/macros.cmake index 9febeaa6889..f4b36597611 100644 --- a/build_files/cmake/macros.cmake +++ b/build_files/cmake/macros.cmake @@ -388,6 +388,43 @@ function(blender_add_lib set_property(GLOBAL APPEND PROPERTY BLENDER_LINK_LIBS ${name}) endfunction() +function(blender_add_test_suite) + if (ARGC LESS 1) + message(FATAL_ERROR "No arguments supplied to blender_add_test_suite()") + endif() + + # Parse the arguments + set(oneValueArgs TARGET SUITE_NAME) + set(multiValueArgs SOURCES) + cmake_parse_arguments(ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + # Figure out the release dir, as some tests need files from there. + GET_BLENDER_TEST_INSTALL_DIR(TEST_INSTALL_DIR) + if(APPLE) + set(_test_release_dir ${TEST_INSTALL_DIR}/Blender.app/Contents/Resources/${BLENDER_VERSION}) + else() + if(WIN32 OR WITH_INSTALL_PORTABLE) + set(_test_release_dir ${TEST_INSTALL_DIR}/${BLENDER_VERSION}) + else() + set(_test_release_dir ${TEST_INSTALL_DIR}/share/blender/${BLENDER_VERSION}) + endif() + endif() + + # Define a test case with our custom gtest_add_tests() command. + include(GTest) + gtest_add_tests( + TARGET ${ARGS_TARGET} + SOURCES "${ARGS_SOURCES}" + TEST_PREFIX ${ARGS_SUITE_NAME} + WORKING_DIRECTORY "${TEST_INSTALL_DIR}" + EXTRA_ARGS + --test-assets-dir "${CMAKE_SOURCE_DIR}/../lib/tests" + --test-release-dir "${_test_release_dir}" + ) + + unset(_test_release_dir) +endfunction() + # Add tests for a Blender library, to be called in tandem with blender_add_lib(). # The tests will be part of the blender_test executable (see tests/gtests/runner). function(blender_add_test_lib @@ -421,6 +458,12 @@ function(blender_add_test_lib blender_add_lib__impl(${name} "${sources}" "${includes}" "${includes_sys}" "${library_deps}") set_property(GLOBAL APPEND PROPERTY BLENDER_TEST_LIBS ${name}) + + blender_add_test_suite( + TARGET blender_test + SUITE_NAME ${name} + SOURCES "${sources}" + ) endfunction() @@ -454,14 +497,10 @@ function(blender_add_test_executable SKIP_ADD_TEST ) - include(GTest) - set(_GOOGLETEST_DISCOVER_TESTS_SCRIPT - ${CMAKE_SOURCE_DIR}/build_files/cmake/Modules/GTestAddTests.cmake - ) - - gtest_discover_tests(${name}_test - DISCOVERY_MODE PRE_TEST - WORKING_DIRECTORY "${TEST_INSTALL_DIR}" + blender_add_test_suite( + TARGET ${name}_test + SUITE_NAME ${name} + SOURCES "${sources}" ) endfunction() -- cgit v1.2.3