more work on test coverage

12 files changed, 151 insertions, 37 deletions

diff --git a/examples/all_features/CMakeLists.txt b/examples/all_features/CMakeLists.txt
index a0f9194..4946b76 100644
--- a/examples/all_features/CMakeLists.txt
+++ b/examples/all_features/CMakeLists.txt
@@ -20,9 +20,10 @@ set(files
 )
 
 doctest_add_executable(${PROJECT_NAME} ${files})
-target_compile_definitions(${PROJECT_NAME} PRIVATE DOCTEST_CONFIG_COLORS_NONE DOCTEST_CONFIG_NO_POSIX_SIGNALS) # easy way to fix test coverage
+# easy way to fix test coverage - disable colors and signals
+target_compile_definitions(${PROJECT_NAME} PRIVATE DOCTEST_CONFIG_COLORS_NONE DOCTEST_CONFIG_NO_POSIX_SIGNALS)
 
-set(common_test_args COMMAND $<TARGET_FILE:${PROJECT_NAME}> --no-skipped-summary=1)
+set(common_test_args COMMAND $<TARGET_FILE:${PROJECT_NAME}> --no-skipped-summary --no-version)
 
 # add per-file tests
 foreach(f ${files})
@@ -32,27 +33,23 @@ endforeach()
 set(common_cov_args ${common_test_args} -sf=*coverage*)
 
 # queries
-doctest_add_test(NAME help                  ${common_cov_args} -h)
-doctest_add_test(NAME version               ${common_cov_args} -v)
-doctest_add_test(NAME count                 ${common_cov_args} -c)
-doctest_add_test(NAME list_test_cases       ${common_cov_args} -ltc)
-doctest_add_test(NAME list_test_suites      ${common_cov_args} -lts)
+doctest_add_test(NAME version COMMAND $<TARGET_FILE:${PROJECT_NAME}> -v)
+doctest_add_test(NAME help               ${common_cov_args} -h)
+doctest_add_test(NAME count              ${common_cov_args} -c)
+doctest_add_test(NAME list_test_cases    ${common_cov_args} -ltc)
+doctest_add_test(NAME list_test_suites   ${common_cov_args} -lts)
 
-doctest_add_test(NAME all_binary_success    ${common_test_args} -tc=all?binary* -s)
+# get all the binary asserts to be printed - to exercise all cases in getAssertString()
+doctest_add_test(NAME all_binary_success ${common_test_args} -tc=all?binary* -s)
 
-## options
-#doctest_add_test(NAME successful            ${common_cov_args} -s=true -e=Off -sfe=*main*)
-#doctest_add_test(NAME abort_after           ${common_cov_args} -aa=3 -no-colors)
-#doctest_add_test(NAME first_last            ${common_cov_args} -f=2 -l=5)
-#doctest_add_test(NAME filter_1              ${common_cov_args} -tc=assert*********)
-#doctest_add_test(NAME filter_2              ${common_cov_args} -tce=???ert*)
-#doctest_add_test(NAME filter_3              ${common_cov_args} -sf=asgghdgsa)
-#doctest_add_test(NAME filter_4              ${common_cov_args} -sfe=*test.cpp*)
-#doctest_add_test(NAME filter_5              ${common_cov_args} -ts=ts1)
-#doctest_add_test(NAME filter_6              ${common_cov_args} -tse=ts1)
-doctest_add_test(NAME order_1               ${common_test_args} -sf=*test_cases_and_suites.cpp -ob=suite)
-doctest_add_test(NAME order_2               ${common_test_args} -sf=*test_cases_and_suites.cpp -ob=name)
-doctest_add_test(NAME order_3               ${common_test_args} -sf=*test_cases_and_suites.cpp -ob=rand -rs=324 -sfe=*) # sfe=* to exclude all tests for no output
+# options
+doctest_add_test(NAME abort_after        ${common_cov_args} -aa=2 -e=off)    # abort after 2 failed asserts (and parse negative)
+doctest_add_test(NAME first_last         ${common_cov_args} -f=2 -l=4)       # run a range
+doctest_add_test(NAME filter_1           ${common_cov_args} -ts=nonexistent) # should filter out all
+doctest_add_test(NAME filter_2           ${common_cov_args} -tse=*)          # should filter out all
+doctest_add_test(NAME order_1            ${common_test_args} -ob=suite -sf=*test_cases_and_suites*)
+doctest_add_test(NAME order_2            ${common_test_args} -ob=name  -sf=*test_cases_and_suites*)
+doctest_add_test(NAME order_3            ${common_test_args} -ob=rand  -sfe=*) # exclude everything for no output
 
 ################################################################################
 ## VARIATION OF THE BUILD WITH DOCTEST DISABLED - SHOULD STILL COMPILE
diff --git a/examples/all_features/coverage_maxout.cpp b/examples/all_features/coverage_maxout.cpp
index 68e2484..1bbc26c 100644
--- a/examples/all_features/coverage_maxout.cpp
+++ b/examples/all_features/coverage_maxout.cpp
@@ -15,17 +15,30 @@
 // !!! THESE ARE NOT PROPER EXAMPLES OF LIBRARY USAGE !!! THESE ARE MEANT FOR CODE COVERAGE ONLY !!!
 // =================================================================================================
 
+// forward declarations of internals
+namespace doctest
+{
+namespace detail
+{
+    const char* fileForOutput(const char* file);
+    void reportFatal(const std::string&);
+    int wildcmp(const char* str, const char* wild, bool caseSensitive);
+} // namespace detail
+} // namespace doctest
+
 TEST_CASE("doctest internals") {
     using namespace doctest;
+    detail::reportFatal("");
+    detail::wildcmp("str", "str*", false);
 
     // trigger code path for string with nullptr
-    doctest::String       a(0);
-    const doctest::String const_str("omgomgomg");
+    String       a(0);
+    const String const_str("omgomgomg");
     a = const_str.c_str();
     CHECK(a.size() == const_str.size());
     CHECK(a.length() == const_str.length());
-    CHECK(a.compare(const_str, true) == 0);
-    CHECK(a.compare("omgomgomg", false) == 0);
+    CHECK_MESSAGE(a.compare(const_str, true) != 0, "should fail");
+    CHECK_MESSAGE(a.compare("omgomgomg", false) != 0, "should fail");
 
     // toString
     a += toString("aaa")                           //
@@ -42,18 +55,18 @@ TEST_CASE("doctest internals") {
          + toString(static_cast<unsigned short>(1));
 
     // others
-    //a += doctest::detail::fileForOutput("c:\\a");
-    //a += doctest::detail::fileForOutput("c:/a");
-    //a += doctest::detail::fileForOutput("a");
+    a += detail::fileForOutput("c:\\a");
+    a += detail::fileForOutput("c:/a");
+    a += detail::fileForOutput("a");
 
     std::ostringstream oss;
     // trigger code path for String to ostream through operator<<
     oss << a;
     // trigger code path for assert string of a non-existent assert type
-    oss << doctest::detail::getAssertString(static_cast<doctest::detail::assertType::Enum>(3));
+    oss << detail::getAssertString(static_cast<detail::assertType::Enum>(3));
     a += oss.str().c_str();
     // trigger code path for rawMemoryToString
-    CHECK(doctest::detail::rawMemoryToString(a).length() > 0u);
+    CHECK_MESSAGE(detail::rawMemoryToString(a).length() == 0u, "should fail");
 }
 
 TEST_CASE("will end from a std::string exception") {
@@ -64,4 +77,8 @@ TEST_CASE("will end from a const char* exception") {
     throw_if(true, "const char*!");
 }
 
+TEST_CASE("will end from an unknown exception") {
+    throw_if(true, doctest::String("unknown :("));
+}
+
 #endif // DOCTEST_CONFIG_DISABLE
diff --git a/examples/all_features/main.cpp b/examples/all_features/main.cpp
index e5d227a..513dfe0 100644
--- a/examples/all_features/main.cpp
+++ b/examples/all_features/main.cpp
@@ -12,8 +12,8 @@ int main(int argc, char** argv) {
 
     // defaults
     context.addFilter("test-case-exclude", "*math*"); // exclude test cases with "math" in the name
-    //context.setOption("abort-after", 5);   // stop test execution after 5 failed assertions
-    context.setOption("order-by", "file"); // sort the test cases by file and line
+    context.setOption("rand-seed", 324);              // if order-by is set to "rand" ise this seed
+    context.setOption("order-by", "file");            // sort the test cases by file and line
 
     context.applyCommandLine(argc, argv);
 
@@ -25,6 +25,8 @@ int main(int argc, char** argv) {
     if(context.shouldExit()) // important - query flags (and --exit) rely on the user doing this
         return res;          // propagate the result of the tests
 
+    context.clearFilters(); // removes all filters added up to this point
+
     int client_stuff_return_code = program();
     // your program - if the testing framework is integrated in your production code
 
diff --git a/examples/all_features/test_output/abort_after.txt b/examples/all_features/test_output/abort_after.txt
new file mode 100644
index 0000000..d649b04
--- /dev/null
+++ b/examples/all_features/test_output/abort_after.txt
@@ -0,0 +1,24 @@
+[doctest] run with "--help" for options
+===============================================================================
+coverage_maxout.cpp(0)
+TEST CASE:  doctest internals
+
+coverage_maxout.cpp(0) ERROR!
+  CHECK( a.compare(const_str, true) != 0 )
+with expansion:
+  CHECK( 0 != 0 )
+with context:
+  should fail
+
+coverage_maxout.cpp(0) ERROR!
+  CHECK( a.compare("omgomgomg", false) != 0 )
+with expansion:
+  CHECK( 0 != 0 )
+with context:
+  should fail
+
+Aborting - too many failed asserts!
+===============================================================================
+[doctest] test cases:    1 |    0 passed |    1 failed | 
+[doctest] assertions:    4 |    2 passed |    2 failed |
+Program code.
diff --git a/examples/all_features/test_output/count.txt b/examples/all_features/test_output/count.txt
index 544b183..eac190d 100644
--- a/examples/all_features/test_output/count.txt
+++ b/examples/all_features/test_output/count.txt
@@ -1,3 +1,3 @@
 [doctest] run with "--help" for options
 ===============================================================================
-[doctest] number of tests passing the current filters: 3
+[doctest] number of tests passing the current filters: 4
diff --git a/examples/all_features/test_output/coverage_maxout.cpp.txt b/examples/all_features/test_output/coverage_maxout.cpp.txt
index 319efea..6b827dc 100644
--- a/examples/all_features/test_output/coverage_maxout.cpp.txt
+++ b/examples/all_features/test_output/coverage_maxout.cpp.txt
@@ -1,6 +1,31 @@
 [doctest] run with "--help" for options
 ===============================================================================
 coverage_maxout.cpp(0)
+TEST CASE:  doctest internals
+
+coverage_maxout.cpp(0) ERROR!
+  CHECK( a.compare(const_str, true) != 0 )
+with expansion:
+  CHECK( 0 != 0 )
+with context:
+  should fail
+
+coverage_maxout.cpp(0) ERROR!
+  CHECK( a.compare("omgomgomg", false) != 0 )
+with expansion:
+  CHECK( 0 != 0 )
+with context:
+  should fail
+
+coverage_maxout.cpp(0) ERROR!
+  CHECK( detail::rawMemoryToString(a).length() == 0u )
+with expansion:
+  CHECK( 18 == 0 )
+with context:
+  should fail
+
+===============================================================================
+coverage_maxout.cpp(0)
 TEST CASE:  will end from a std::string exception
 
 TEST CASE FAILED!
@@ -16,6 +41,14 @@ threw exception:
   const char*!
 
 ===============================================================================
-[doctest] test cases:    3 |    1 passed |    2 failed | 
-[doctest] assertions:    5 |    5 passed |    0 failed |
+coverage_maxout.cpp(0)
+TEST CASE:  will end from an unknown exception
+
+TEST CASE FAILED!
+threw exception:
+  unknown exception
+
+===============================================================================
+[doctest] test cases:    4 |    0 passed |    4 failed | 
+[doctest] assertions:    5 |    2 passed |    3 failed |
 Program code.
diff --git a/examples/all_features/test_output/filter_1.txt b/examples/all_features/test_output/filter_1.txt
new file mode 100644
index 0000000..eca69f0
--- /dev/null
+++ b/examples/all_features/test_output/filter_1.txt
@@ -0,0 +1,5 @@
+[doctest] run with "--help" for options
+===============================================================================
+[doctest] test cases:    0 |    0 passed |    0 failed | 
+[doctest] assertions:    0 |    0 passed |    0 failed |
+Program code.
diff --git a/examples/all_features/test_output/filter_2.txt b/examples/all_features/test_output/filter_2.txt
new file mode 100644
index 0000000..eca69f0
--- /dev/null
+++ b/examples/all_features/test_output/filter_2.txt
@@ -0,0 +1,5 @@
+[doctest] run with "--help" for options
+===============================================================================
+[doctest] test cases:    0 |    0 passed |    0 failed | 
+[doctest] assertions:    0 |    0 passed |    0 failed |
+Program code.
diff --git a/examples/all_features/test_output/first_last.txt b/examples/all_features/test_output/first_last.txt
new file mode 100644
index 0000000..e0df6f7
--- /dev/null
+++ b/examples/all_features/test_output/first_last.txt
@@ -0,0 +1,29 @@
+[doctest] run with "--help" for options
+===============================================================================
+coverage_maxout.cpp(0)
+TEST CASE:  will end from a std::string exception
+
+TEST CASE FAILED!
+threw exception:
+  std::string!
+
+===============================================================================
+coverage_maxout.cpp(0)
+TEST CASE:  will end from a const char* exception
+
+TEST CASE FAILED!
+threw exception:
+  const char*!
+
+===============================================================================
+coverage_maxout.cpp(0)
+TEST CASE:  will end from an unknown exception
+
+TEST CASE FAILED!
+threw exception:
+  unknown exception
+
+===============================================================================
+[doctest] test cases:    4 |    1 passed |    3 failed | 
+[doctest] assertions:    0 |    0 passed |    0 failed |
+Program code.
diff --git a/examples/all_features/test_output/list_test_cases.txt b/examples/all_features/test_output/list_test_cases.txt
index 2f9dc13..af64168 100644
--- a/examples/all_features/test_output/list_test_cases.txt
+++ b/examples/all_features/test_output/list_test_cases.txt
@@ -3,5 +3,6 @@
 doctest internals
 will end from a std::string exception
 will end from a const char* exception
+will end from an unknown exception
 ===============================================================================
-[doctest] number of tests passing the current filters: 3
+[doctest] number of tests passing the current filters: 4
diff --git a/examples/all_features/test_output/list_test_suites.txt b/examples/all_features/test_output/list_test_suites.txt
index 540b37b..1cfe96c 100644
--- a/examples/all_features/test_output/list_test_suites.txt
+++ b/examples/all_features/test_output/list_test_suites.txt
@@ -2,4 +2,4 @@
 [doctest] listing all test suites
 
 ===============================================================================
-[doctest] number of tests passing the current filters: 3
+[doctest] number of tests passing the current filters: 4
diff --git a/examples/all_features/test_output/version.txt b/examples/all_features/test_output/version.txt
index e69de29..12266d4 100644
--- a/examples/all_features/test_output/version.txt
+++ b/examples/all_features/test_output/version.txt
@@ -0,0 +1 @@
+[doctest] doctest version is "1.1.4"