From 89da6543076c0491094d19bd83af9ef98e1362fe Mon Sep 17 00:00:00 2001 From: onqtam Date: Tue, 2 May 2017 21:10:42 +0300 Subject: more work on test coverage --- examples/all_features/CMakeLists.txt | 39 ++++++++++------------ examples/all_features/coverage_maxout.cpp | 35 ++++++++++++++----- examples/all_features/main.cpp | 6 ++-- examples/all_features/test_output/abort_after.txt | 24 +++++++++++++ examples/all_features/test_output/count.txt | 2 +- .../test_output/coverage_maxout.cpp.txt | 37 ++++++++++++++++++-- examples/all_features/test_output/filter_1.txt | 5 +++ examples/all_features/test_output/filter_2.txt | 5 +++ examples/all_features/test_output/first_last.txt | 29 ++++++++++++++++ .../all_features/test_output/list_test_cases.txt | 3 +- .../all_features/test_output/list_test_suites.txt | 2 +- examples/all_features/test_output/version.txt | 1 + examples/exe_with_static_libs/CMakeLists.txt | 2 +- examples/executable_dll_and_plugin/CMakeLists.txt | 2 +- 14 files changed, 153 insertions(+), 39 deletions(-) create mode 100644 examples/all_features/test_output/abort_after.txt create mode 100644 examples/all_features/test_output/filter_1.txt create mode 100644 examples/all_features/test_output/filter_2.txt create mode 100644 examples/all_features/test_output/first_last.txt (limited to 'examples') diff --git a/examples/all_features/CMakeLists.txt b/examples/all_features/CMakeLists.txt index a0f9194..4946b76 100644 --- a/examples/all_features/CMakeLists.txt +++ b/examples/all_features/CMakeLists.txt @@ -20,9 +20,10 @@ set(files ) doctest_add_executable(${PROJECT_NAME} ${files}) -target_compile_definitions(${PROJECT_NAME} PRIVATE DOCTEST_CONFIG_COLORS_NONE DOCTEST_CONFIG_NO_POSIX_SIGNALS) # easy way to fix test coverage +# easy way to fix test coverage - disable colors and signals +target_compile_definitions(${PROJECT_NAME} PRIVATE DOCTEST_CONFIG_COLORS_NONE DOCTEST_CONFIG_NO_POSIX_SIGNALS) -set(common_test_args COMMAND $ --no-skipped-summary=1) +set(common_test_args COMMAND $ --no-skipped-summary --no-version) # add per-file tests foreach(f ${files}) @@ -32,27 +33,23 @@ endforeach() set(common_cov_args ${common_test_args} -sf=*coverage*) # queries -doctest_add_test(NAME help ${common_cov_args} -h) -doctest_add_test(NAME version ${common_cov_args} -v) -doctest_add_test(NAME count ${common_cov_args} -c) -doctest_add_test(NAME list_test_cases ${common_cov_args} -ltc) -doctest_add_test(NAME list_test_suites ${common_cov_args} -lts) +doctest_add_test(NAME version COMMAND $ -v) +doctest_add_test(NAME help ${common_cov_args} -h) +doctest_add_test(NAME count ${common_cov_args} -c) +doctest_add_test(NAME list_test_cases ${common_cov_args} -ltc) +doctest_add_test(NAME list_test_suites ${common_cov_args} -lts) -doctest_add_test(NAME all_binary_success ${common_test_args} -tc=all?binary* -s) +# get all the binary asserts to be printed - to exercise all cases in getAssertString() +doctest_add_test(NAME all_binary_success ${common_test_args} -tc=all?binary* -s) -## options -#doctest_add_test(NAME successful ${common_cov_args} -s=true -e=Off -sfe=*main*) -#doctest_add_test(NAME abort_after ${common_cov_args} -aa=3 -no-colors) -#doctest_add_test(NAME first_last ${common_cov_args} -f=2 -l=5) -#doctest_add_test(NAME filter_1 ${common_cov_args} -tc=assert*********) -#doctest_add_test(NAME filter_2 ${common_cov_args} -tce=???ert*) -#doctest_add_test(NAME filter_3 ${common_cov_args} -sf=asgghdgsa) -#doctest_add_test(NAME filter_4 ${common_cov_args} -sfe=*test.cpp*) -#doctest_add_test(NAME filter_5 ${common_cov_args} -ts=ts1) -#doctest_add_test(NAME filter_6 ${common_cov_args} -tse=ts1) -doctest_add_test(NAME order_1 ${common_test_args} -sf=*test_cases_and_suites.cpp -ob=suite) -doctest_add_test(NAME order_2 ${common_test_args} -sf=*test_cases_and_suites.cpp -ob=name) -doctest_add_test(NAME order_3 ${common_test_args} -sf=*test_cases_and_suites.cpp -ob=rand -rs=324 -sfe=*) # sfe=* to exclude all tests for no output +# options +doctest_add_test(NAME abort_after ${common_cov_args} -aa=2 -e=off) # abort after 2 failed asserts (and parse negative) +doctest_add_test(NAME first_last ${common_cov_args} -f=2 -l=4) # run a range +doctest_add_test(NAME filter_1 ${common_cov_args} -ts=nonexistent) # should filter out all +doctest_add_test(NAME filter_2 ${common_cov_args} -tse=*) # should filter out all +doctest_add_test(NAME order_1 ${common_test_args} -ob=suite -sf=*test_cases_and_suites*) +doctest_add_test(NAME order_2 ${common_test_args} -ob=name -sf=*test_cases_and_suites*) +doctest_add_test(NAME order_3 ${common_test_args} -ob=rand -sfe=*) # exclude everything for no output ################################################################################ ## VARIATION OF THE BUILD WITH DOCTEST DISABLED - SHOULD STILL COMPILE diff --git a/examples/all_features/coverage_maxout.cpp b/examples/all_features/coverage_maxout.cpp index 68e2484..1bbc26c 100644 --- a/examples/all_features/coverage_maxout.cpp +++ b/examples/all_features/coverage_maxout.cpp @@ -15,17 +15,30 @@ // !!! THESE ARE NOT PROPER EXAMPLES OF LIBRARY USAGE !!! THESE ARE MEANT FOR CODE COVERAGE ONLY !!! // ================================================================================================= +// forward declarations of internals +namespace doctest +{ +namespace detail +{ + const char* fileForOutput(const char* file); + void reportFatal(const std::string&); + int wildcmp(const char* str, const char* wild, bool caseSensitive); +} // namespace detail +} // namespace doctest + TEST_CASE("doctest internals") { using namespace doctest; + detail::reportFatal(""); + detail::wildcmp("str", "str*", false); // trigger code path for string with nullptr - doctest::String a(0); - const doctest::String const_str("omgomgomg"); + String a(0); + const String const_str("omgomgomg"); a = const_str.c_str(); CHECK(a.size() == const_str.size()); CHECK(a.length() == const_str.length()); - CHECK(a.compare(const_str, true) == 0); - CHECK(a.compare("omgomgomg", false) == 0); + CHECK_MESSAGE(a.compare(const_str, true) != 0, "should fail"); + CHECK_MESSAGE(a.compare("omgomgomg", false) != 0, "should fail"); // toString a += toString("aaa") // @@ -42,18 +55,18 @@ TEST_CASE("doctest internals") { + toString(static_cast(1)); // others - //a += doctest::detail::fileForOutput("c:\\a"); - //a += doctest::detail::fileForOutput("c:/a"); - //a += doctest::detail::fileForOutput("a"); + a += detail::fileForOutput("c:\\a"); + a += detail::fileForOutput("c:/a"); + a += detail::fileForOutput("a"); std::ostringstream oss; // trigger code path for String to ostream through operator<< oss << a; // trigger code path for assert string of a non-existent assert type - oss << doctest::detail::getAssertString(static_cast(3)); + oss << detail::getAssertString(static_cast(3)); a += oss.str().c_str(); // trigger code path for rawMemoryToString - CHECK(doctest::detail::rawMemoryToString(a).length() > 0u); + CHECK_MESSAGE(detail::rawMemoryToString(a).length() == 0u, "should fail"); } TEST_CASE("will end from a std::string exception") { @@ -64,4 +77,8 @@ TEST_CASE("will end from a const char* exception") { throw_if(true, "const char*!"); } +TEST_CASE("will end from an unknown exception") { + throw_if(true, doctest::String("unknown :(")); +} + #endif // DOCTEST_CONFIG_DISABLE diff --git a/examples/all_features/main.cpp b/examples/all_features/main.cpp index e5d227a..513dfe0 100644 --- a/examples/all_features/main.cpp +++ b/examples/all_features/main.cpp @@ -12,8 +12,8 @@ int main(int argc, char** argv) { // defaults context.addFilter("test-case-exclude", "*math*"); // exclude test cases with "math" in the name - //context.setOption("abort-after", 5); // stop test execution after 5 failed assertions - context.setOption("order-by", "file"); // sort the test cases by file and line + context.setOption("rand-seed", 324); // if order-by is set to "rand" ise this seed + context.setOption("order-by", "file"); // sort the test cases by file and line context.applyCommandLine(argc, argv); @@ -25,6 +25,8 @@ int main(int argc, char** argv) { if(context.shouldExit()) // important - query flags (and --exit) rely on the user doing this return res; // propagate the result of the tests + context.clearFilters(); // removes all filters added up to this point + int client_stuff_return_code = program(); // your program - if the testing framework is integrated in your production code diff --git a/examples/all_features/test_output/abort_after.txt b/examples/all_features/test_output/abort_after.txt new file mode 100644 index 0000000..d649b04 --- /dev/null +++ b/examples/all_features/test_output/abort_after.txt @@ -0,0 +1,24 @@ +[doctest] run with "--help" for options +=============================================================================== +coverage_maxout.cpp(0) +TEST CASE: doctest internals + +coverage_maxout.cpp(0) ERROR! + CHECK( a.compare(const_str, true) != 0 ) +with expansion: + CHECK( 0 != 0 ) +with context: + should fail + +coverage_maxout.cpp(0) ERROR! + CHECK( a.compare("omgomgomg", false) != 0 ) +with expansion: + CHECK( 0 != 0 ) +with context: + should fail + +Aborting - too many failed asserts! +=============================================================================== +[doctest] test cases: 1 | 0 passed | 1 failed | +[doctest] assertions: 4 | 2 passed | 2 failed | +Program code. diff --git a/examples/all_features/test_output/count.txt b/examples/all_features/test_output/count.txt index 544b183..eac190d 100644 --- a/examples/all_features/test_output/count.txt +++ b/examples/all_features/test_output/count.txt @@ -1,3 +1,3 @@ [doctest] run with "--help" for options =============================================================================== -[doctest] number of tests passing the current filters: 3 +[doctest] number of tests passing the current filters: 4 diff --git a/examples/all_features/test_output/coverage_maxout.cpp.txt b/examples/all_features/test_output/coverage_maxout.cpp.txt index 319efea..6b827dc 100644 --- a/examples/all_features/test_output/coverage_maxout.cpp.txt +++ b/examples/all_features/test_output/coverage_maxout.cpp.txt @@ -1,4 +1,29 @@ [doctest] run with "--help" for options +=============================================================================== +coverage_maxout.cpp(0) +TEST CASE: doctest internals + +coverage_maxout.cpp(0) ERROR! + CHECK( a.compare(const_str, true) != 0 ) +with expansion: + CHECK( 0 != 0 ) +with context: + should fail + +coverage_maxout.cpp(0) ERROR! + CHECK( a.compare("omgomgomg", false) != 0 ) +with expansion: + CHECK( 0 != 0 ) +with context: + should fail + +coverage_maxout.cpp(0) ERROR! + CHECK( detail::rawMemoryToString(a).length() == 0u ) +with expansion: + CHECK( 18 == 0 ) +with context: + should fail + =============================================================================== coverage_maxout.cpp(0) TEST CASE: will end from a std::string exception @@ -16,6 +41,14 @@ threw exception: const char*! =============================================================================== -[doctest] test cases: 3 | 1 passed | 2 failed | -[doctest] assertions: 5 | 5 passed | 0 failed | +coverage_maxout.cpp(0) +TEST CASE: will end from an unknown exception + +TEST CASE FAILED! +threw exception: + unknown exception + +=============================================================================== +[doctest] test cases: 4 | 0 passed | 4 failed | +[doctest] assertions: 5 | 2 passed | 3 failed | Program code. diff --git a/examples/all_features/test_output/filter_1.txt b/examples/all_features/test_output/filter_1.txt new file mode 100644 index 0000000..eca69f0 --- /dev/null +++ b/examples/all_features/test_output/filter_1.txt @@ -0,0 +1,5 @@ +[doctest] run with "--help" for options +=============================================================================== +[doctest] test cases: 0 | 0 passed | 0 failed | +[doctest] assertions: 0 | 0 passed | 0 failed | +Program code. diff --git a/examples/all_features/test_output/filter_2.txt b/examples/all_features/test_output/filter_2.txt new file mode 100644 index 0000000..eca69f0 --- /dev/null +++ b/examples/all_features/test_output/filter_2.txt @@ -0,0 +1,5 @@ +[doctest] run with "--help" for options +=============================================================================== +[doctest] test cases: 0 | 0 passed | 0 failed | +[doctest] assertions: 0 | 0 passed | 0 failed | +Program code. diff --git a/examples/all_features/test_output/first_last.txt b/examples/all_features/test_output/first_last.txt new file mode 100644 index 0000000..e0df6f7 --- /dev/null +++ b/examples/all_features/test_output/first_last.txt @@ -0,0 +1,29 @@ +[doctest] run with "--help" for options +=============================================================================== +coverage_maxout.cpp(0) +TEST CASE: will end from a std::string exception + +TEST CASE FAILED! +threw exception: + std::string! + +=============================================================================== +coverage_maxout.cpp(0) +TEST CASE: will end from a const char* exception + +TEST CASE FAILED! +threw exception: + const char*! + +=============================================================================== +coverage_maxout.cpp(0) +TEST CASE: will end from an unknown exception + +TEST CASE FAILED! +threw exception: + unknown exception + +=============================================================================== +[doctest] test cases: 4 | 1 passed | 3 failed | +[doctest] assertions: 0 | 0 passed | 0 failed | +Program code. diff --git a/examples/all_features/test_output/list_test_cases.txt b/examples/all_features/test_output/list_test_cases.txt index 2f9dc13..af64168 100644 --- a/examples/all_features/test_output/list_test_cases.txt +++ b/examples/all_features/test_output/list_test_cases.txt @@ -3,5 +3,6 @@ doctest internals will end from a std::string exception will end from a const char* exception +will end from an unknown exception =============================================================================== -[doctest] number of tests passing the current filters: 3 +[doctest] number of tests passing the current filters: 4 diff --git a/examples/all_features/test_output/list_test_suites.txt b/examples/all_features/test_output/list_test_suites.txt index 540b37b..1cfe96c 100644 --- a/examples/all_features/test_output/list_test_suites.txt +++ b/examples/all_features/test_output/list_test_suites.txt @@ -2,4 +2,4 @@ [doctest] listing all test suites =============================================================================== -[doctest] number of tests passing the current filters: 3 +[doctest] number of tests passing the current filters: 4 diff --git a/examples/all_features/test_output/version.txt b/examples/all_features/test_output/version.txt index e69de29..12266d4 100644 --- a/examples/all_features/test_output/version.txt +++ b/examples/all_features/test_output/version.txt @@ -0,0 +1 @@ +[doctest] doctest version is "1.1.4" diff --git a/examples/exe_with_static_libs/CMakeLists.txt b/examples/exe_with_static_libs/CMakeLists.txt index 7e53f95..d4b1cb8 100644 --- a/examples/exe_with_static_libs/CMakeLists.txt +++ b/examples/exe_with_static_libs/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries(${PROJECT_NAME} lib_1) target_link_libraries(${PROJECT_NAME} lib_2) set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER ${PROJECT_NAME}) -doctest_add_test(NAME ${PROJECT_NAME} COMMAND $) +doctest_add_test(NAME ${PROJECT_NAME} COMMAND $ --no-version) # pulls the force linking functionality include(doctest_force_link_static_lib_in_target.cmake) diff --git a/examples/executable_dll_and_plugin/CMakeLists.txt b/examples/executable_dll_and_plugin/CMakeLists.txt index 25be7fa..f746405 100644 --- a/examples/executable_dll_and_plugin/CMakeLists.txt +++ b/examples/executable_dll_and_plugin/CMakeLists.txt @@ -32,4 +32,4 @@ endif() # have the executable depend on the plugin so it gets built as well when building/starting only the executable add_dependencies(${PROJECT_NAME} plugin) -doctest_add_test(NAME ${PROJECT_NAME} COMMAND $) +doctest_add_test(NAME ${PROJECT_NAME} COMMAND $ --no-version) -- cgit v1.2.3