first version of true system testing

git-svn-id: http://cmock.svn.sourceforge.net/svnroot/cmock/trunk@73 bf332499-1b4d-0410-844d-d2d48d5cc64c

config/environment.rb

@@ -5,7 +5,8 @@ ROOT_PATH = File.expand_path(File.dirname(__FILE__) + "/../")
   'lib',
   'vendor/behaviors/lib',
   'vendor/hardmock/lib',
-  'vendor/unity/auto/'
+  'vendor/unity/auto/',
+  'test/system/'
 ].each do |dir|
   $LOAD_PATH.unshift(File.join(ROOT_PATH, dir))
 end

gcc.yml

@@ -1,17 +1,18 @@
+---
 compiler:
   path: gcc
-  source_path:    'examples/src/'
+  source_path:     &systest_generated_path 'test/system/generated/'
   unit_tests_path: &unit_tests_path 'examples/test/'
-  mocks_path:     'examples/mocks/'
-  build_path:      &build_path 'examples/build/'
+  mocks_path:      &systest_mocks_path 'test/system/generated/'
+  build_path:      &systest_build_path 'test/system/build/'
   options:
     - -c
   includes:
     prefix: '-I'
     items:
-      - 'examples/src/'
+      - *systest_generated_path
       - *unit_tests_path
-      - 'examples/mocks/'
+      - *systest_mocks_path
       - 'vendor/unity/src/'
   defines:
     prefix: '-D'
@@ -19,7 +20,8 @@ compiler:
   object_files:
     prefix: '-o'
     extension: '.o'
-    destination: *build_path
+    destination: *systest_build_path
+    
 linker:
   path: gcc
   options:
@@ -27,17 +29,11 @@ linker:
   includes:
     prefix: '-I'
   object_files:
-    path: *build_path
+    path: *systest_build_path
     extension: '.o'
   bin_files:
     prefix: '-o'
     extension: '.exe'
-    destination: *build_path
-:cmock:
-  :mock_path: 'examples/mocks/'
-  :includes:
-    - 'Types.h'
-    - 'UnityHelper.h'
-  :plugins:
-    - 'ignore'
-  :unity_helper: 'vendor/unity/src/UnityHelper.h'
+    destination: *systest_build_path
+
+

rakefile.rb

@@ -10,16 +10,12 @@ include RakefileHelpers
 
 DEFAULT_CONFIG_FILE = 'gcc.yml'
 
+configure_clean
 configure_toolchain(DEFAULT_CONFIG_FILE)
 
-task :default => [:clobber, 'test:all', :app]
+task :default => ['test:all']
 task :cruise => [:default]
 
-desc "Build application"
-task :app do
-  build_application('Main')
-end
-
 desc "Load configuration"
 task :config, :config_file do |t, args|
   args = {:config_file => DEFAULT_CONFIG_FILE} if args[:config_file].nil?
@@ -30,31 +26,19 @@ end
 namespace :test do
 
   desc "Run CMock and example application tests"
-  task :all => [:clean, :units, :example, :app]
+  task :all => ['test:units', 'test:system']
 
   Rake::TestTask.new('units') do |t|
     t.pattern = 'test/unit/*_test.rb'
     t.verbose = true
   end
   
-  desc "Run example unit tests"
-  task :example => [:clean] do
-    systest_test_files = get_unit_test_files
-    run_systests(systest_test_files)
-    tests_failed = report_summary
-    raise "Unit tests failed." if (tests_failed > 0)
+  desc "Run system tests"
+  task :system => [:clobber] do
+    report 'Running system tests'
+
+    tests_failed = run_systests(FileList['test/system/cases/*.yml'])
+    raise "System tests failed." if (tests_failed > 0)
   end
-  
-  get_unit_test_files.each do |test_file|
-    file_name = File.basename(test_file)
-    module_name = file_name.sub(/Test/,'')
-    task file_name do
-      run_systests(test_file)
-    end
-    desc "Test #{module_name}"
-    task module_name do
-      run_systests(test_file)
-    end
-  end
-  
+    
 end

rakefile_helper.rb

@@ -3,10 +3,15 @@ require 'fileutils'
 require 'cmock'
 require 'generate_test_runner'
 require 'unity_test_summary'
+require 'systest_generator'
 
 module RakefileHelpers
 
+  SYSTEST_GENERATED_FILES_PATH = 'test/system/generated/'
+  SYSTEST_BUILD_FILES_PATH     = 'test/system/build/'
+
   C_EXTENSION = '.c'
+  RESULT_EXTENSION = '.result'
   
   def report(message)
     puts message
@@ -20,8 +25,8 @@ module RakefileHelpers
   end
   
   def configure_clean
-    CLEAN.include($cfg['compiler']['mocks_path'] + '*.*') unless $cfg['compiler']['mocks_path'].nil?
-    CLEAN.include($cfg['compiler']['build_path'] + '*.*') unless $cfg['compiler']['mocks_path'].nil?
+    CLEAN.include(SYSTEST_GENERATED_FILES_PATH + '*.*')
+    CLEAN.include(SYSTEST_BUILD_FILES_PATH + '*.*')
   end
   
   def configure_toolchain(config_file)
@@ -29,12 +34,6 @@ module RakefileHelpers
     configure_clean
   end
   
-  def get_unit_test_files
-    path = $cfg['compiler']['unit_tests_path'] + 'Test*' + C_EXTENSION
-    path.gsub!(/\\/, '/')
-    FileList.new(path)
-  end
-  
   def get_local_include_dirs
     include_dirs = $cfg['compiler']['includes']['items'].dup
     include_dirs.delete_if {|dir| dir.is_a?(Array)}
@@ -146,11 +145,11 @@ module RakefileHelpers
   end
   
   def execute(command_string, verbose=true)
-    report command_string
+#    report command_string
     output = `#{command_string}`.chomp
     report(output) if (verbose && !output.nil? && (output.length > 0))
     if $?.exitstatus != 0
-      raise "Command failed. (Returned #{$?.exitstatus})"
+      raise "#{command_string} failed. (Returned #{$?.exitstatus})"
     end
     return output
   end
@@ -165,50 +164,50 @@ module RakefileHelpers
     summary.run
   end
   
-  def run_systests(test_files)
+  def run_systests(test_case_files)    
+    SystemTestGenerator.new.generate_files(test_case_files)
+    test_files = FileList.new(SYSTEST_GENERATED_FILES_PATH + 'test*.c')
     
-    report 'Running system tests...'
-    
-    # Tack on TEST define for compiling unit tests
     load_configuration($cfg_file)
-    test_defines = ['TEST']
     $cfg['compiler']['defines']['items'] = [] if $cfg['compiler']['defines']['items'].nil?
-    $cfg['compiler']['defines']['items'] << 'TEST'
     
     include_dirs = get_local_include_dirs
-    
+        
     # Build and execute each unit test
     test_files.each do |test|
-    
+
       obj_list = []
       
+      test_base    = File.basename(test, C_EXTENSION)
+      cmock_config = test_base.gsub(/test_/, '') + '_cmock.yml'
+      
       # Detect dependencies and build required required modules
       extract_headers(test).each do |header|
+
         # Generate mock if a mock was included
-        if header =~ /^Mock(.*)\.h/i
+        if header =~ /^mock_(.*)\.h/i
           module_name = $1
-          cmock = CMock.new($cfg_file)
+          cmock = CMock.new(SYSTEST_GENERATED_FILES_PATH + cmock_config)
           cmock.setup_mocks("#{$cfg['compiler']['source_path']}#{module_name}.h")
         end
         # Compile corresponding source file if it exists
         src_file = find_source_file(header, include_dirs)
         if !src_file.nil?
-          compile(src_file, test_defines)
+          compile(src_file)
           obj_list << header.ext($cfg['compiler']['object_files']['extension'])
         end
       end
-      
+
       # Generate and build the test suite runner
-      test_base = File.basename(test, C_EXTENSION)
-      runner_name = test_base + '_Runner.c'
-      runner_path = $cfg['compiler']['build_path'] + runner_name
+      runner_name = test_base + '_runner.c'
+      runner_path = $cfg['compiler']['source_path'] + runner_name
       test_gen = UnityTestRunnerGenerator.new
-      test_gen.run(test, runner_path, ['Types'])
-      compile(runner_path, test_defines)
+      test_gen.run(test, runner_path, [])
+      compile(runner_path)
       obj_list << runner_name.ext($cfg['compiler']['object_files']['extension'])
       
       # Build the test module
-      compile(test, test_defines)
+      compile(test)
       obj_list << test_base.ext($cfg['compiler']['object_files']['extension'])
       
       # Link the test executable
@@ -222,44 +221,48 @@ module RakefileHelpers
       else
         cmd_str = "#{simulator[:command]} #{simulator[:pre_support]} #{executable} #{simulator[:post_support]}"
       end
-      output = execute(cmd_str)
-      test_results = $cfg['compiler']['build_path'] + test_base
-      if output.match(/OK$/m).nil?
-        test_results += '.testfail'
-      else
-        test_results += '.testpass'
-      end
+      output = execute(cmd_str, false)
+      test_results = $cfg['compiler']['build_path'] + test_base + RESULT_EXTENSION
       File.open(test_results, 'w') { |f| f.print output }
-      
     end
-  end
-  
-  def build_application(main)
-  
-    report "Building application..."
     
-    obj_list = []
-    load_configuration($cfg_file)
-    main_path = $cfg['compiler']['source_path'] + main + C_EXTENSION
+    # Parse and report test results
+    total_tests = 0
+    total_failures = 0
+    failure_messages = []
 
-    # Detect dependencies and build required required modules
-    include_dirs = get_local_include_dirs
-    extract_headers(main_path).each do |header|
-      src_file = find_source_file(header, include_dirs)
-      if !src_file.nil?
-        compile(src_file)
-        obj_list << header.ext($cfg['compiler']['object_files']['extension'])
+    test_case_files.each do |test_case|      
+      tests = (YAML.load_file(test_case))[:systest][:tests][:units]
+      total_tests += tests.size
+
+      test_file    = 'test_' + File.basename(test_case).ext(C_EXTENSION)
+      result_file  = test_file.ext(RESULT_EXTENSION)
+      test_results = File.read(SYSTEST_BUILD_FILES_PATH + result_file)
+
+      tests.each_with_index do |test, index|
+        # compare test's intended pass/fail state with pass/fail state in actual results;
+        # if they don't match, the system test has failed
+        if (test[:pass] != !((test_results =~ /test#{index+1}::: PASS/).nil?))
+          total_failures += 1
+          failure_messages << "#{test_file}:test#{index+1}:should #{test[:should]}"
+        end
       end
     end
     
-    # Build the main source file
-    main_base = File.basename(main_path, C_EXTENSION)
-    compile(main_path)
-    obj_list << main_base.ext($cfg['compiler']['object_files']['extension'])
+    puts "\n"
+    puts "--------------------------\n"
+    puts "SYSTEM TEST SUMMARY\n"
+    puts "--------------------------\n"
+    puts "TOTAL TESTS: #{total_tests} TOTAL FAILURES: #{total_failures}\n"
+    puts "\n"
     
-    # Create the executable
-    link(main_base, obj_list)
+    failure_messages.each do |failure|
+      puts failure
+    end
+    
+    puts ''
+
+    return total_failures
   end
-  
 end
 

test/system/cases/simple.yml

@@ -0,0 +1,54 @@
+---
+:cmock:
+  :plugins:
+  - 'ignore'
+
+:systest:
+  :types: |
+    #define UINT32 unsigned int
+    
+    typedef signed int custom_type;
+
+  :mockable: |
+    UINT32 foo(custom_type a);
+    UINT32 bar(custom_type b);
+
+  :source: 
+    :header: |
+      unsigned int foobar(int a, int b);
+    :code: |
+      unsigned int foobar(int a, int b)    
+      {
+        return foo((custom_type)a) + bar((custom_type)b);
+      }    
+
+  :tests:
+    :common: |
+      void setUp(void) {}
+      void tearDown(void) {}
+    :units:
+    - :pass: TRUE
+      :should: 'exercise two simple ExpectAndReturn mock calls'
+      :code: |
+        test()
+        {
+          foo_ExpectAndReturn((custom_type)1, 10);
+          bar_ExpectAndReturn((custom_type)2, 20);
+          TEST_ASSERT_EQUAL(30, foobar(1, 2));
+        }
+      
+    - :pass: FALSE
+      :should: 'fail because bar() is called but not expected'
+      :code: |
+        test()
+        {
+          foo_ExpectAndReturn((custom_type)1, 10);
+          TEST_ASSERT_EQUAL(30, foobar(1, 2));
+        }
+      
+  # :unity_helper:
+  #   :header: |
+  #     // header stuff
+  #   :code:
+  #     // code stuff
+  #     

test/system/systest_generator.rb

@@ -0,0 +1,172 @@
+SYS_TEST_GEN_ROOT = File.expand_path( File.dirname( __FILE__ ) ) + '/'
+
+require 'yaml'
+
+GENERATED_PATH = SYS_TEST_GEN_ROOT + 'generated/'
+BUILD_PATH     = SYS_TEST_GEN_ROOT + 'build/'
+CASES_PATH     = SYS_TEST_GEN_ROOT + 'cases/'
+
+TYPES_H          = 'types.h'
+UNITY_H          = 'unity.h'
+UNITY_HELPER_H   = 'unity_helper.h'
+UNITY_HELPER_C   = 'unity_helper.c'
+MOCKABLE_H       = 'mockable.h'
+
+YAML_EXTENSION = '.yml'
+TEST_PREFIX    = 'test_'
+MOCK_PREFIX    = 'mock_'
+H_EXTENSION    = '.h'
+C_EXTENSION    = '.c'
+
+
+class SystemTestGenerator
+  
+  def generate_files(test_cases)
+    test_cases.each do |filename|
+      yaml_hash = YAML.load_file(filename)
+      
+      name = File.basename(filename, YAML_EXTENSION)
+      namix = "#{name}_"
+      
+      generate_cmock_config(yaml_hash, namix)
+      generate_code(yaml_hash, namix, name)
+    end
+  end
+
+  private
+  
+  def generate_cmock_config(yaml_hash, namix)
+    cmock_yaml = yaml_hash.clone
+    cmock_yaml.delete(:systest)
+    cmock = cmock_yaml[:cmock]
+    
+    cmock[:mock_path]   = GENERATED_PATH
+    cmock[:includes]    = [namix + TYPES_H]
+    cmock[:mock_prefix] = MOCK_PREFIX
+    if not yaml_hash[:systest][:unity_helper].nil?
+      cmock[:includes]     << namix + UNITY_HELPER_H 
+      cmock[:unity_helper]  = namix + UNITY_HELPER_C
+    end
+
+    File.open(GENERATED_PATH + namix + 'cmock' + YAML_EXTENSION, 'w') do |out|
+      YAML.dump(cmock_yaml, out)
+    end
+  end
+  
+  def generate_code(yaml_hash, namix, name)    
+    generate_types_file(yaml_hash, namix)
+    generate_mockable_file(yaml_hash, namix)
+    generate_unity_helper_files(yaml_hash, namix)
+    
+    generate_test_file(yaml_hash, namix, name)
+    generate_source_file(yaml_hash, namix, name)
+  end
+  
+  def generate_types_file(yaml_hash, namix)
+    types = yaml_hash[:systest][:types]
+    return if types.nil?
+    
+    write_header_file(GENERATED_PATH + namix + TYPES_H, namix.upcase + 'TYPES_H') do |out|
+      out.puts(types)
+    end
+  end
+  
+  def generate_mockable_file(yaml_hash, namix)
+    mockable = yaml_hash[:systest][:mockable]
+    return if mockable.nil?
+    
+    write_header_file(GENERATED_PATH + namix + MOCKABLE_H, namix.upcase + 'MOCKABLE_H', [namix + TYPES_H]) do |out|
+      out.puts(mockable)
+    end
+  end
+
+  def generate_unity_helper_files(yaml_hash, namix)
+    unity_helper = yaml_hash[:systest][:unity_helper]
+    return if unity_helper.nil?
+    
+    write_header_file(GENERATED_PATH + namix + UNITY_HELPER_H, namix.upcase + 'UNITY_HELPER_H', [namix + TYPES_H]) do |out|
+      out.puts(unity_helper[:header])
+    end
+
+    write_source_file(GENERATED_PATH + namix + UNITY_HELPER_C, [namix + UNITY_HELPER_H]) do |out|
+      out.puts(unity_helper[:code])
+    end
+  end
+
+  def generate_test_file(yaml_hash, namix, name)
+    tests = yaml_hash[:systest][:tests]
+    return if tests.nil?
+    
+    includes = [UNITY_H]
+    includes << (namix + UNITY_HELPER_H) if not yaml_hash[:systest][:unity_helper].nil?
+    includes << [MOCK_PREFIX + namix + MOCKABLE_H]
+    includes << [name + H_EXTENSION]
+    
+    write_source_file(GENERATED_PATH + TEST_PREFIX + name + C_EXTENSION, includes) do |out|
+      out.puts(tests[:common])
+      out.puts('')
+
+      tests[:units].each_with_index do |test, index|
+        out.puts('// ' + test[:should])
+        out.puts(test[:code].gsub!(/test\(\)/, "void test#{index+1}(void)"))
+        out.puts('')
+      end
+    end    
+  end
+
+  def generate_source_file(yaml_hash, namix, name)
+    source = yaml_hash[:systest][:source]
+    return if source.nil?
+    
+    header_file = name + H_EXTENSION
+    
+    includes = (namix + TYPES_H) if not yaml_hash[:systest][:types].nil?
+
+    write_header_file(GENERATED_PATH + header_file, name.upcase, includes) do |out|
+      out.puts(source[:header])
+    end    
+
+    includes = []
+    includes << (namix + TYPES_H) if not yaml_hash[:systest][:types].nil?
+    includes << (namix + MOCKABLE_H) if not yaml_hash[:systest][:mockable].nil?
+    includes << header_file
+    
+    write_source_file(GENERATED_PATH + name + C_EXTENSION, includes) do |out|
+      out.puts(source[:code])
+    end    
+  end
+
+  def write_header_file(filename, upcase_name, include_list=[])
+    File.open(filename, 'w') do |out|
+      out.puts("#ifndef _#{upcase_name}")
+      out.puts("#define _#{upcase_name}")
+      out.puts('')
+      include_list.each do |include|
+        out.puts("#include \"#{include}\"")
+      end
+      out.puts('')
+      yield(out)
+      out.puts('')
+      out.puts("#endif // _#{upcase_name}")
+      out.puts('')
+    end    
+  end
+
+  def write_source_file(filename, include_list=[])
+    File.open(filename, 'w') do |out|
+      include_list.each do |include|
+        out.puts("#include \"#{include}\"")
+      end
+      out.puts('')
+      yield(out)
+      out.puts('')
+    end    
+  end
+  
+end
+
+
+if ($0 == __FILE__)
+  SystemTestGenerator.new.generate_files(Dir[CASES_PATH + "*#{YAML_EXTENSION}"])
+end
+