Clean up testing pipeline some more... centralized reporting.

test/Makefile

@@ -42,7 +42,7 @@ TARGET = build/testunity-cov.exe
 
 # To generate coverage, call 'make -s', the default target runs.
 # For verbose output of all the tests, run 'make test'.
-default: coverage
+default: test
 .PHONY: default coverage test clean
 coverage: $(SRC1) $(SRC2) $(SRC3) $(SRC4) $(SRC5) $(SRC6) $(SRC7) $(SRC8)
 	cd $(BUILD_DIR) && \

test/rakefile

@@ -11,7 +11,6 @@ $extra_paths = []
 require 'rake'
 require 'rake/clean'
 require_relative 'rakefile_helper'
-require 'rspec/core/rake_task'
 
 TEMP_DIRS = [
 	File.join(__dir__, 'build'),
@@ -53,8 +52,16 @@ namespace :test do
 
   desc "Test unity's helper scripts"
   task :scripts => [:prepare_for_tests] do
-    Dir['tests/test_*.rb'].each do |scriptfile|
+    begin
-      require "./"+scriptfile
+      Dir['tests/test_*.rb'].each do |scriptfile|
+        require "./"+scriptfile
+      end
+    ensure
+      total    = $generate_test_runner_tests    || 0
+      failures = $generate_test_runner_failures || 0
+      result   = "#{total} Tests #{failures} Failures 0 Ignored\n"
+      result  += failures > 0 ? "FAILED\n" : "OK\n"
+      save_test_results('scripts', result)
     end
   end
 
@@ -75,16 +82,24 @@ namespace :test do
 
   desc "Test unity examples"
   task :examples => [:prepare_for_tests] do
-    [
+    run_examples
-      "cd ../examples/example_1 && make -s ci",
-      "cd ../examples/example_2 && make -s ci",
-      "cd ../examples/example_3 && rake config[#{$cfg_file_base || 'gcc_64'}] default"
-    ].each { |cmd| execute(cmd, false) }
   end
 
   desc "Run all rspecs"
-  RSpec::Core::RakeTask.new(:spec) do |t|
+  task :spec => [:prepare_for_tests] do
-    t.pattern = 'spec/**/*_spec.rb'
+    output = execute("rspec spec/**/*_spec.rb", true)
+    rspec_ok = $?.exitstatus.zero?
+    report output
+    total = failures = pending = 0
+    if output =~ /(\d+) examples?, (\d+) failures?(?:, (\d+) pending)?/
+      total    = Regexp.last_match(1).to_i
+      failures = Regexp.last_match(2).to_i
+      pending  = (Regexp.last_match(3) || 0).to_i
+    end
+    result   = "#{total} Tests #{failures} Failures #{pending} Ignored\n"
+    result  += rspec_ok ? "OK\n" : "FAILED\n"
+    save_test_results('spec', result)
+    raise "Command failed." unless rspec_ok
   end
 
   desc "Generate test summary"

test/rakefile_helper.rb

@@ -192,6 +192,32 @@ module RakefileHelpers
     report summary.run
   end
 
+  # Parse all Unity summary lines from combined output (e.g. multiple executables)
+  # and produce a single synthesized result string suitable for save_test_results.
+  def collect_test_output(output)
+    total_tests    = 0
+    total_failures = 0
+    total_ignored  = 0
+    detail_lines   = []
+
+    output.each_line do |line|
+      stripped = line.chomp
+      if stripped =~ /(\d+) Tests (\d+) Failures (\d+) Ignored/
+        total_tests    += Regexp.last_match(1).to_i
+        total_failures += Regexp.last_match(2).to_i
+        total_ignored  += Regexp.last_match(3).to_i
+      elsif stripped =~ /^[^:]+:[^:]+:\w+(?:\([^)]*\))?:(?:PASS|FAIL|IGNORE)/
+        detail_lines << stripped
+      end
+    end
+
+    synthesized  = detail_lines.join("\n")
+    synthesized += "\n" unless detail_lines.empty?
+    synthesized += "#{total_tests} Tests #{total_failures} Failures #{total_ignored} Ignored\n"
+    synthesized += total_failures > 0 ? "FAILED\n" : "OK\n"
+    synthesized
+  end
+
   def save_test_results(test_base, output)
     test_results = File.join('build',test_base)
     if output.match(/OK$/m).nil?
@@ -218,7 +244,7 @@ module RakefileHelpers
     obj_list = src_files.map { |f| compile(f, ['UNITY_SKIP_DEFAULT_RUNNER', 'UNITY_FIXTURE_NO_EXTRAS']) }
 
     # Link the test executable
-    test_base = File.basename('framework_test', C_EXTENSION)
+    test_base = File.basename('fixtures_test', C_EXTENSION)
     link_it(test_base, obj_list)
 
     # Run and collect output
@@ -364,17 +390,39 @@ module RakefileHelpers
 
   def run_make_tests()
     report "\nRunning Unity Examples with Make"
-    [ "make -s",                               # test with all defaults
+    combined_output = ''
-      #"make -s DEBUG=-m32",                    # test 32-bit architecture with 64-bit support
+    [ "make -s",                 # test with all defaults
-      #"make -s DEBUG=-m32 UNITY_SUPPORT_64=",  # test 32-bit build without 64-bit types
+      "make -s coverage",        # test with coverage
-      "make -s UNITY_INCLUDE_DOUBLE= ",        # test without double
       "cd #{File.join("..","extras","fixture",'test')} && make -s default noStdlibMalloc",
       "cd #{File.join("..","extras","fixture",'test')} && make -s C89",
       "cd #{File.join("..","extras","memory",'test')} && make -s default noStdlibMalloc",
       "cd #{File.join("..","extras","memory",'test')} && make -s C89",
     ].each do |cmd|
       report "Testing '#{cmd}'"
-      execute(cmd, false)
+      combined_output += "Testing '#{cmd}'\n\n#{execute(cmd, false)}\n"
     end
+    save_test_results('make_tests', collect_test_output(combined_output))
+  end
+
+  def run_examples()
+    report "\nRunning Unity Examples"
+    total_tests = total_ignored = 0
+    [
+      "cd ../examples/example_1 && make -s ci",
+      "cd ../examples/example_2 && make -s ci",
+      "cd ../examples/example_3 && rake config[#{$cfg_file_base || 'gcc_64'}] default"
+    ].each do |cmd|
+      execute(cmd, false).each_line do |line|
+        if line =~ /(\d+) Tests \d+ Failures (\d+) Ignored/
+          total_tests   += Regexp.last_match(1).to_i
+          total_ignored += Regexp.last_match(2).to_i
+          # Failures intentionally not counted: the examples contain tests designed
+          # to fail to demonstrate Unity's detection capability. A zero exit code
+          # from make/rake means those failures were verified as expected; if
+          # something truly broke, execute() would have raised above.
+        end
+      end
+    end
+    save_test_results('examples', "#{total_tests} Tests 0 Failures #{total_ignored} Ignored\nOK\n")
   end
 end

test/tests/test_unity_doubles.c

@@ -1277,9 +1277,9 @@ void testDoublePrintingInfinityAndNaN(void)
 #if defined(UNITY_EXCLUDE_FLOAT_PRINT) || defined(UNITY_EXCLUDE_DOUBLE) || !defined(USING_OUTPUT_SPY)
     TEST_IGNORE();
 #else
-    TEST_ASSERT_EQUAL_PRINT_FLOATING("inf",   1.0 / d_zero);
+    TEST_ASSERT_EQUAL_PRINT_FLOATING("Infinity",   1.0 / d_zero);
-    TEST_ASSERT_EQUAL_PRINT_FLOATING("-inf", -1.0 / d_zero);
+    TEST_ASSERT_EQUAL_PRINT_FLOATING("Negative Infinity", -1.0 / d_zero);
 
-    TEST_ASSERT_EQUAL_PRINT_FLOATING("nan",   0.0 / d_zero);
+    TEST_ASSERT_EQUAL_PRINT_FLOATING("NaN",   0.0 / d_zero);
 #endif
 }

test/tests/test_unity_floats.c

@@ -1287,10 +1287,10 @@ void testFloatPrintingInfinityAndNaN(void)
 #if defined(UNITY_EXCLUDE_FLOAT_PRINT) || !defined(USING_OUTPUT_SPY)
     TEST_IGNORE();
 #else
-    TEST_ASSERT_EQUAL_PRINT_FLOATING("inf",   1.0f / f_zero);
+    TEST_ASSERT_EQUAL_PRINT_FLOATING("Infinity",   1.0f / f_zero);
-    TEST_ASSERT_EQUAL_PRINT_FLOATING("-inf", -1.0f / f_zero);
+    TEST_ASSERT_EQUAL_PRINT_FLOATING("Negative Infinity", -1.0f / f_zero);
 
-    TEST_ASSERT_EQUAL_PRINT_FLOATING("nan",   0.0f / f_zero);
+    TEST_ASSERT_EQUAL_PRINT_FLOATING("NaN",   0.0f / f_zero);
 #endif
 }
 
@@ -1305,7 +1305,7 @@ static void printFloatValue(float f)
 
     sprintf(expected, "%.9g", f);
     /* We print all NaN's as "nan", not "-nan" */
-    if (strcmp(expected, "-nan") == 0) strcpy(expected, "nan");
+    if (strcmp(expected, "-nan") == 0) strcpy(expected, "NaN");
 
     if (strcmp(expected, getBufferPutcharSpy()))
     {
@@ -1329,7 +1329,7 @@ static void printFloatValue(float f)
 
     sprintf(expected, "%.7g", f);
     /* We print all NaN's as "nan", not "-nan" */
-    if (strcmp(expected, "-nan") == 0) strcpy(expected, "nan");
+    if (strcmp(expected, "-nan") == 0) strcpy(expected, "NaN");
 
     strcpy(expected_lower, expected);
     strcpy(expected_lower2, expected);