diff options
author | Viktor Kirilov <vik.kirilov@gmail.com> | 2021-01-09 18:43:50 +0300 |
---|---|---|
committer | Viktor Kirilov <vik.kirilov@gmail.com> | 2021-02-02 17:59:16 +0300 |
commit | 8eccc83edd1498094d0c455e6c7fde916a6b7684 (patch) | |
tree | 819fa5ab89e6cb60b767ca934faf409534ecd0f3 | |
parent | 98685b0e86b1c94e01eaf89cd2aa17e1d39315d2 (diff) |
added the no_output decorator and also fixed a bug with the recently introduced no_breaks decorator
-rw-r--r-- | doc/markdown/testcases.md | 1 | ||||
-rw-r--r-- | doctest/doctest.h | 18 | ||||
-rw-r--r-- | doctest/parts/doctest.cpp | 15 | ||||
-rw-r--r-- | doctest/parts/doctest_fwd.h | 3 | ||||
-rw-r--r-- | examples/all_features/test_cases_and_suites.cpp | 2 | ||||
-rw-r--r-- | examples/all_features/test_output/filter_2.txt | 2 | ||||
-rw-r--r-- | examples/all_features/test_output/filter_2_xml.txt | 3 | ||||
-rw-r--r-- | examples/all_features/test_output/order_1.txt | 4 | ||||
-rw-r--r-- | examples/all_features/test_output/order_1_junit.txt | 3 | ||||
-rw-r--r-- | examples/all_features/test_output/order_1_xml.txt | 10 | ||||
-rw-r--r-- | examples/all_features/test_output/order_2.txt | 4 | ||||
-rw-r--r-- | examples/all_features/test_output/order_2_junit.txt | 3 | ||||
-rw-r--r-- | examples/all_features/test_output/order_2_xml.txt | 10 | ||||
-rw-r--r-- | examples/all_features/test_output/test_cases_and_suites.cpp.txt | 4 | ||||
-rw-r--r-- | examples/all_features/test_output/test_cases_and_suites.cpp_junit.txt | 3 | ||||
-rw-r--r-- | examples/all_features/test_output/test_cases_and_suites.cpp_xml.txt | 12 |
16 files changed, 78 insertions, 19 deletions
diff --git a/doc/markdown/testcases.md b/doc/markdown/testcases.md index 762f7170..0744aa66 100644 --- a/doc/markdown/testcases.md +++ b/doc/markdown/testcases.md @@ -121,6 +121,7 @@ Multiple decorators can be used at the same time. These are the currently suppor - **```skip(bool = true)```** - marks the test case to be skipped from execution - unless the ```--no-skip``` option is used - **```no_breaks(bool = true)```** - no breaking into the debugger for asserts in the test case - useful in combination with `may_fail`/`should_fail`/`expected_failures` +- **```no_output(bool = true)```** - no output from asserts in the test case - useful in combination with `may_fail`/`should_fail`/`expected_failures` - **```may_fail(bool = true)```** - doesn't fail the test if any given assertion fails (but still reports it) - this can be useful to flag a work-in-progress, or a known issue that you don't want to immediately fix but still want to track in the your tests - **```should_fail(bool = true)```** - like **```may_fail()```** but fails the test if it passes - his can be useful if you want to be notified of accidental, or third-party, fixes - **```expected_failures(int)```** - defines the number of assertions that are expected to fail within the test case - reported as failure when the number of failed assertions is different than the declared expected number of failures diff --git a/doctest/doctest.h b/doctest/doctest.h index 8cce7985..41758a86 100644 --- a/doctest/doctest.h +++ b/doctest/doctest.h @@ -663,6 +663,7 @@ struct DOCTEST_INTERFACE TestCaseData const char* m_description; bool m_skip; bool m_no_breaks; + bool m_no_output; bool m_may_fail; bool m_should_fail; int m_expected_failures; @@ -1254,6 +1255,7 @@ namespace detail { const char* m_description; bool m_skip; bool m_no_breaks; + bool m_no_output; bool m_may_fail; bool m_should_fail; int m_expected_failures; @@ -1594,6 +1596,7 @@ DOCTEST_DEFINE_DECORATOR(test_suite, const char*, ""); DOCTEST_DEFINE_DECORATOR(description, const char*, ""); DOCTEST_DEFINE_DECORATOR(skip, bool, true); DOCTEST_DEFINE_DECORATOR(no_breaks, bool, true); +DOCTEST_DEFINE_DECORATOR(no_output, bool, true); DOCTEST_DEFINE_DECORATOR(timeout, double, 0); DOCTEST_DEFINE_DECORATOR(may_fail, bool, true); DOCTEST_DEFINE_DECORATOR(should_fail, bool, true); @@ -3764,6 +3767,8 @@ namespace detail { // clear state m_description = nullptr; m_skip = false; + m_no_breaks = false; + m_no_output = false; m_may_fail = false; m_should_fail = false; m_expected_failures = 0; @@ -3779,6 +3784,8 @@ namespace detail { m_test_suite = test_suite.m_test_suite; m_description = test_suite.m_description; m_skip = test_suite.m_skip; + m_no_breaks = test_suite.m_no_breaks; + m_no_output = test_suite.m_no_output; m_may_fail = test_suite.m_may_fail; m_should_fail = test_suite.m_should_fail; m_expected_failures = test_suite.m_expected_failures; @@ -5773,6 +5780,9 @@ namespace { } void test_case_end(const CurrentTestCaseStats& st) override { + if(tc->m_no_output) + return; + // log the preamble of the test case only if there is something // else to print - something other than that an assert has failed if(opt.duration || @@ -5807,6 +5817,9 @@ namespace { } void test_case_exception(const TestCaseException& e) override { + if(tc->m_no_output) + return; + logTestStart(); file_line_to_stream(tc->m_file.c_str(), tc->m_line, " "); @@ -5841,7 +5854,7 @@ namespace { } void log_assert(const AssertData& rb) override { - if(!rb.m_failed && !opt.success) + if((!rb.m_failed && !opt.success) || tc->m_no_output) return; std::lock_guard<std::mutex> lock(mutex); @@ -5857,6 +5870,9 @@ namespace { } void log_message(const MessageData& mb) override { + if(tc->m_no_output) + return; + std::lock_guard<std::mutex> lock(mutex); logTestStart(); diff --git a/doctest/parts/doctest.cpp b/doctest/parts/doctest.cpp index eb8ad234..ca840e35 100644 --- a/doctest/parts/doctest.cpp +++ b/doctest/parts/doctest.cpp @@ -1098,6 +1098,8 @@ namespace detail { // clear state m_description = nullptr; m_skip = false; + m_no_breaks = false; + m_no_output = false; m_may_fail = false; m_should_fail = false; m_expected_failures = 0; @@ -1113,6 +1115,8 @@ namespace detail { m_test_suite = test_suite.m_test_suite; m_description = test_suite.m_description; m_skip = test_suite.m_skip; + m_no_breaks = test_suite.m_no_breaks; + m_no_output = test_suite.m_no_output; m_may_fail = test_suite.m_may_fail; m_should_fail = test_suite.m_should_fail; m_expected_failures = test_suite.m_expected_failures; @@ -3107,6 +3111,9 @@ namespace { } void test_case_end(const CurrentTestCaseStats& st) override { + if(tc->m_no_output) + return; + // log the preamble of the test case only if there is something // else to print - something other than that an assert has failed if(opt.duration || @@ -3141,6 +3148,9 @@ namespace { } void test_case_exception(const TestCaseException& e) override { + if(tc->m_no_output) + return; + logTestStart(); file_line_to_stream(tc->m_file.c_str(), tc->m_line, " "); @@ -3175,7 +3185,7 @@ namespace { } void log_assert(const AssertData& rb) override { - if(!rb.m_failed && !opt.success) + if((!rb.m_failed && !opt.success) || tc->m_no_output) return; std::lock_guard<std::mutex> lock(mutex); @@ -3191,6 +3201,9 @@ namespace { } void log_message(const MessageData& mb) override { + if(tc->m_no_output) + return; + std::lock_guard<std::mutex> lock(mutex); logTestStart(); diff --git a/doctest/parts/doctest_fwd.h b/doctest/parts/doctest_fwd.h index c07853db..19a3d619 100644 --- a/doctest/parts/doctest_fwd.h +++ b/doctest/parts/doctest_fwd.h @@ -660,6 +660,7 @@ struct DOCTEST_INTERFACE TestCaseData const char* m_description; bool m_skip; bool m_no_breaks; + bool m_no_output; bool m_may_fail; bool m_should_fail; int m_expected_failures; @@ -1251,6 +1252,7 @@ namespace detail { const char* m_description; bool m_skip; bool m_no_breaks; + bool m_no_output; bool m_may_fail; bool m_should_fail; int m_expected_failures; @@ -1591,6 +1593,7 @@ DOCTEST_DEFINE_DECORATOR(test_suite, const char*, ""); DOCTEST_DEFINE_DECORATOR(description, const char*, ""); DOCTEST_DEFINE_DECORATOR(skip, bool, true); DOCTEST_DEFINE_DECORATOR(no_breaks, bool, true); +DOCTEST_DEFINE_DECORATOR(no_output, bool, true); DOCTEST_DEFINE_DECORATOR(timeout, double, 0); DOCTEST_DEFINE_DECORATOR(may_fail, bool, true); DOCTEST_DEFINE_DECORATOR(should_fail, bool, true); diff --git a/examples/all_features/test_cases_and_suites.cpp b/examples/all_features/test_cases_and_suites.cpp index f227479a..43aad5d6 100644 --- a/examples/all_features/test_cases_and_suites.cpp +++ b/examples/all_features/test_cases_and_suites.cpp @@ -70,3 +70,5 @@ TEST_SUITE("test suite with a description" * doctest::description("regarding fai FAIL_CHECK(""); } } + +TEST_CASE("should fail and no output" * doctest::should_fail() * doctest::no_breaks() * doctest::no_output()) { FAIL(""); } diff --git a/examples/all_features/test_output/filter_2.txt b/examples/all_features/test_output/filter_2.txt index 5d1ce0fe..18fe1062 100644 --- a/examples/all_features/test_output/filter_2.txt +++ b/examples/all_features/test_output/filter_2.txt @@ -1,6 +1,6 @@ [doctest] run with "--help" for options =============================================================================== -[doctest] test cases: 0 | 0 passed | 0 failed | 80 skipped +[doctest] test cases: 0 | 0 passed | 0 failed | 81 skipped [doctest] assertions: 0 | 0 passed | 0 failed | [doctest] Status: SUCCESS! Program code. diff --git a/examples/all_features/test_output/filter_2_xml.txt b/examples/all_features/test_output/filter_2_xml.txt index b05b1e97..fde7c970 100644 --- a/examples/all_features/test_output/filter_2_xml.txt +++ b/examples/all_features/test_output/filter_2_xml.txt @@ -88,6 +88,7 @@ <TestCase name="part of some TS" filename="test_cases_and_suites.cpp" line="0" skipped="true"/> </TestSuite> <TestSuite> + <TestCase name="should fail and no output" filename="test_cases_and_suites.cpp" line="0" should_fail="true" skipped="true"/> <TestCase name="should fail because of an exception" filename="test_cases_and_suites.cpp" line="0" skipped="true"/> <TestCase name="signed integers stuff<int>" filename="templated_test_cases.cpp" line="0" skipped="true"/> <TestCase name="signed integers stuff<short int>" filename="templated_test_cases.cpp" line="0" skipped="true"/> @@ -118,6 +119,6 @@ <TestCase name="will end from an unknown exception" filename="coverage_maxout.cpp" line="0" skipped="true"/> </TestSuite> <OverallResultsAsserts successes="0" failures="0"/> - <OverallResultsTestCases successes="0" failures="0" skipped="80"/> + <OverallResultsTestCases successes="0" failures="0" skipped="81"/> </doctest> Program code. diff --git a/examples/all_features/test_output/order_1.txt b/examples/all_features/test_output/order_1.txt index 29c2da2d..2429335b 100644 --- a/examples/all_features/test_output/order_1.txt +++ b/examples/all_features/test_output/order_1.txt @@ -102,7 +102,7 @@ test_cases_and_suites.cpp(0): MESSAGE: failing because of the timeout decorator! Test case exceeded time limit of 0.000001! =============================================================================== -[doctest] test cases: 15 | 5 passed | 10 failed | -[doctest] assertions: 12 | 1 passed | 11 failed | +[doctest] test cases: 16 | 6 passed | 10 failed | +[doctest] assertions: 13 | 1 passed | 12 failed | [doctest] Status: FAILURE! Program code. diff --git a/examples/all_features/test_output/order_1_junit.txt b/examples/all_features/test_output/order_1_junit.txt index f48940b7..68640a86 100644 --- a/examples/all_features/test_output/order_1_junit.txt +++ b/examples/all_features/test_output/order_1_junit.txt @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuites> - <testsuite name="all_features" errors="1" failures="1" tests="12"> + <testsuite name="all_features" errors="1" failures="1" tests="13"> <testcase classname="test_cases_and_suites.cpp" name="an empty test that will succeed - not part of a test suite" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="should fail because of an exception" status="run"> <error message="exception"> @@ -15,6 +15,7 @@ CHECK( data == 85 ) is NOT correct! </failure> </testcase> + <testcase classname="test_cases_and_suites.cpp" name="should fail and no output" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="part of scoped" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="part of scoped 2" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="unskipped" status="run"/> diff --git a/examples/all_features/test_output/order_1_xml.txt b/examples/all_features/test_output/order_1_xml.txt index 7759c95d..d47aba95 100644 --- a/examples/all_features/test_output/order_1_xml.txt +++ b/examples/all_features/test_output/order_1_xml.txt @@ -22,6 +22,12 @@ </Expression> <OverallResultsAsserts successes="0" failures="1"/> </TestCase> + <TestCase name="should fail and no output" filename="test_cases_and_suites.cpp" line="0" should_fail="true"> + <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0"> + <Text/> + </Message> + <OverallResultsAsserts successes="0" failures="1"/> + </TestCase> </TestSuite> <TestSuite name="scoped test suite"> <TestCase name="part of scoped" filename="test_cases_and_suites.cpp" line="0"> @@ -104,7 +110,7 @@ <OverallResultsAsserts successes="0" failures="0"/> </TestCase> </TestSuite> - <OverallResultsAsserts successes="1" failures="11"/> - <OverallResultsTestCases successes="5" failures="10"/> + <OverallResultsAsserts successes="1" failures="12"/> + <OverallResultsTestCases successes="6" failures="10"/> </doctest> Program code. diff --git a/examples/all_features/test_output/order_2.txt b/examples/all_features/test_output/order_2.txt index a9137253..f9d76c27 100644 --- a/examples/all_features/test_output/order_2.txt +++ b/examples/all_features/test_output/order_2.txt @@ -95,7 +95,7 @@ TEST CASE: unskipped test_cases_and_suites.cpp(0): FATAL ERROR: =============================================================================== -[doctest] test cases: 14 | 5 passed | 9 failed | -[doctest] assertions: 11 | 1 passed | 10 failed | +[doctest] test cases: 15 | 6 passed | 9 failed | +[doctest] assertions: 12 | 1 passed | 11 failed | [doctest] Status: FAILURE! Program code. diff --git a/examples/all_features/test_output/order_2_junit.txt b/examples/all_features/test_output/order_2_junit.txt index e93970ed..b70cfbe6 100644 --- a/examples/all_features/test_output/order_2_junit.txt +++ b/examples/all_features/test_output/order_2_junit.txt @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuites> - <testsuite name="all_features" errors="1" failures="1" tests="11"> + <testsuite name="all_features" errors="1" failures="1" tests="12"> <testcase classname="test_cases_and_suites.cpp" name="an empty test that will succeed - not part of a test suite" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="doesn't fail but it should have" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="doesn't fail which is fine" status="run"/> @@ -20,6 +20,7 @@ CHECK( data == 85 ) is NOT correct! <testcase classname="test_cases_and_suites.cpp" name="part of scoped" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="part of scoped 2" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="part of some TS" status="run"/> + <testcase classname="test_cases_and_suites.cpp" name="should fail and no output" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="should fail because of an exception" status="run"> <error message="exception"> 0 diff --git a/examples/all_features/test_output/order_2_xml.txt b/examples/all_features/test_output/order_2_xml.txt index 053bca6b..a09907ba 100644 --- a/examples/all_features/test_output/order_2_xml.txt +++ b/examples/all_features/test_output/order_2_xml.txt @@ -87,6 +87,12 @@ </TestCase> </TestSuite> <TestSuite> + <TestCase name="should fail and no output" filename="test_cases_and_suites.cpp" line="0" should_fail="true"> + <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0"> + <Text/> + </Message> + <OverallResultsAsserts successes="0" failures="1"/> + </TestCase> <TestCase name="should fail because of an exception" filename="test_cases_and_suites.cpp" line="0"> <Exception crash="false"> 0 @@ -102,7 +108,7 @@ <OverallResultsAsserts successes="0" failures="1"/> </TestCase> </TestSuite> - <OverallResultsAsserts successes="1" failures="10"/> - <OverallResultsTestCases successes="5" failures="9"/> + <OverallResultsAsserts successes="1" failures="11"/> + <OverallResultsTestCases successes="6" failures="9"/> </doctest> Program code. diff --git a/examples/all_features/test_output/test_cases_and_suites.cpp.txt b/examples/all_features/test_output/test_cases_and_suites.cpp.txt index 9a3ec59f..c7e12f06 100644 --- a/examples/all_features/test_output/test_cases_and_suites.cpp.txt +++ b/examples/all_features/test_output/test_cases_and_suites.cpp.txt @@ -95,7 +95,7 @@ test_cases_and_suites.cpp(0): ERROR: Didn't fail exactly 1 times so marking it as failed! =============================================================================== -[doctest] test cases: 14 | 5 passed | 9 failed | -[doctest] assertions: 11 | 1 passed | 10 failed | +[doctest] test cases: 15 | 6 passed | 9 failed | +[doctest] assertions: 12 | 1 passed | 11 failed | [doctest] Status: FAILURE! Program code. diff --git a/examples/all_features/test_output/test_cases_and_suites.cpp_junit.txt b/examples/all_features/test_output/test_cases_and_suites.cpp_junit.txt index 78551eaa..47e66d09 100644 --- a/examples/all_features/test_output/test_cases_and_suites.cpp_junit.txt +++ b/examples/all_features/test_output/test_cases_and_suites.cpp_junit.txt @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuites> - <testsuite name="all_features" errors="1" failures="1" tests="11"> + <testsuite name="all_features" errors="1" failures="1" tests="12"> <testcase classname="test_cases_and_suites.cpp" name="an empty test that will succeed - not part of a test suite" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="should fail because of an exception" status="run"> <error message="exception"> @@ -26,6 +26,7 @@ CHECK( data == 85 ) is NOT correct! <testcase classname="test_cases_and_suites.cpp" name="doesn't fail but it should have" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="fails 1 time as it should" status="run"/> <testcase classname="test_cases_and_suites.cpp" name="fails more times as it should" status="run"/> + <testcase classname="test_cases_and_suites.cpp" name="should fail and no output" status="run"/> </testsuite> </testsuites> Program code. diff --git a/examples/all_features/test_output/test_cases_and_suites.cpp_xml.txt b/examples/all_features/test_output/test_cases_and_suites.cpp_xml.txt index f13d64c9..ece0fb6d 100644 --- a/examples/all_features/test_output/test_cases_and_suites.cpp_xml.txt +++ b/examples/all_features/test_output/test_cases_and_suites.cpp_xml.txt @@ -100,7 +100,15 @@ <OverallResultsAsserts successes="0" failures="2" expected_failures="1"/> </TestCase> </TestSuite> - <OverallResultsAsserts successes="1" failures="10"/> - <OverallResultsTestCases successes="5" failures="9"/> + <TestSuite> + <TestCase name="should fail and no output" filename="test_cases_and_suites.cpp" line="0" should_fail="true"> + <Message type="FATAL ERROR" filename="test_cases_and_suites.cpp" line="0"> + <Text/> + </Message> + <OverallResultsAsserts successes="0" failures="1"/> + </TestCase> + </TestSuite> + <OverallResultsAsserts successes="1" failures="11"/> + <OverallResultsTestCases successes="6" failures="9"/> </doctest> Program code. |