diff --git a/tools/harness-automation/README.rst b/tools/harness-automation/README.rst index 4399961bc..25fe76d64 100644 --- a/tools/harness-automation/README.rst +++ b/tools/harness-automation/README.rst @@ -2,7 +2,7 @@ Harness Automation Tool ======================= -This is a tool to automate testing openthread with GRL Thread-Test-Harness1.1-Alpha v1.0-Release_19.0. +This is a tool to automate testing openthread with GRL Thread-Test-Harness1.1-Alpha v1.0-Release_40.0. ----------- Quick Start @@ -33,14 +33,20 @@ Other options:: -h, --help Show help message and exit. + --blacklist BLACKLIST_FILE, -b BLACKLIST_FILE + skip test cases listed in BLACKLIST_FILE. + --pattern PATTERN, -p PATTERN File name pattern, Default to all python files. --delete-blacklist, -d - Clear blacklist on startup. By default, golden devices failed to be connected are kept in a blacklist automatically. Add this option to clear blacklist on startup. + Clear golden device blacklist on startup. By default, golden devices failed to be connected are kept in a blacklist automatically. Add this option to clear blacklist on startup. + + --name-greps NAME_GREPS, -g NAME_GREPS + Filter case by its name using filename matching syntax. Multiple this options are OR-ed to allow more tests. --skip SKIP, -k SKIP - Type of test case status to skip. ``e`` for error, ``f`` for fail, ``p`` for pass. Default to "efp". If test case names are given by ``NAME``, this option will not work. + Type of test case status to skip. ``e`` for error, ``f`` for fail, ``p`` for pass. If test case names are given by ``NAME``, this option will not work. --dry-run, -n Just show what test case will be run. diff --git a/tools/harness-automation/autothreadharness/harness_case.py b/tools/harness-automation/autothreadharness/harness_case.py index 2ad2ac6bc..9d8fd766a 100644 --- a/tools/harness-automation/autothreadharness/harness_case.py +++ b/tools/harness-automation/autothreadharness/harness_case.py @@ -116,6 +116,10 @@ class HarnessCase(unittest.TestCase): """int: Child timeout in seconds """ + sed_polling_interval = settings.THREAD_SED_POLLING_INTERVAL + """int: SED polling interval in seconds + """ + manual_reset = False """bool: whether reset manually""" @@ -124,6 +128,7 @@ class HarnessCase(unittest.TestCase): timeout = hasattr(settings, 'TIMEOUT') and settings.TIMEOUT or DEFAULT_TIMEOUT + started = 0 def wait_until(self, what, times=-1): """Wait until `what` return True @@ -168,10 +173,6 @@ class HarnessCase(unittest.TestCase): raw_input('Reset golden devices and press enter to continue..') return elif not settings.PDU_CONTROLLER_TYPE: - if settings.GOLDEN_DEVICE_TYPE != 'OpenThread': - logger.warning('All golden devices may not be resetted') - return - if settings.AUTO_DUT: return @@ -189,7 +190,7 @@ class HarnessCase(unittest.TestCase): tries = 3 pdu_factory = PduControllerFactory() - + while True: try: pdu = pdu_factory.create_pdu_controller(settings.PDU_CONTROLLER_TYPE) @@ -322,6 +323,13 @@ class HarnessCase(unittest.TestCase): def _setup_page(self): """Do sniffer settings and general settings """ + if not self.started: + self.started = time.time() + + if time.time() - self.started > 30: + self._browser.refresh() + return + # Detect Sniffer try: dialog = self._browser.find_element_by_id('capture-Setup-modal') @@ -374,25 +382,28 @@ class HarnessCase(unittest.TestCase): # General Setup try: - button = self._browser.find_element_by_id('general-Setup') - button.click() - time.sleep(2) + if self.child_timeout or self.sed_polling_interval: + button = self._browser.find_element_by_id('general-Setup') + button.click() + time.sleep(2) - dialog = self._browser.find_element_by_id('general-Setup-modal') - if dialog.get_attribute('aria-hidden') != 'false': - raise Exception('Missing General Setup dialog') + dialog = self._browser.find_element_by_id('general-Setup-modal') + if dialog.get_attribute('aria-hidden') != 'false': + raise Exception('Missing General Setup dialog') - field = dialog.find_element_by_id('inp_general_child_update_wait_time') - field.clear() - field.send_keys(str(self.child_timeout)) + field = dialog.find_element_by_id('inp_general_child_update_wait_time') + field.clear() + if self.child_timeout: + field.send_keys(str(self.child_timeout)) - field = dialog.find_element_by_id('inp_general_sed_polling_rate') - field.clear() - field.send_keys(str(settings.THREAD_SED_POLLING_INTERVAL)) + field = dialog.find_element_by_id('inp_general_sed_polling_rate') + field.clear() + if self.sed_polling_interval: + field.send_keys(str(self.sed_polling_interval)) - button = dialog.find_element_by_id('saveGeneralSettings') - button.click() - time.sleep(1) + button = dialog.find_element_by_id('saveGeneralSettings') + button.click() + time.sleep(1) except: logger.exception('Failed to do general setup') @@ -470,6 +481,18 @@ class HarnessCase(unittest.TestCase): if settings.DUT_DEVICE: self._add_device(*settings.DUT_DEVICE) + # enable AUTO DUT + if self.auto_dut: + checkbox_auto_dut = browser.find_element_by_id('EnableAutoDutSelection') + if not checkbox_auto_dut.is_selected(): + checkbox_auto_dut.click() + time.sleep(1) + + if settings.DUT_DEVICE: + radio_auto_dut = browser.find_element_by_class_name('AutoDUT_RadBtns') + if not radio_auto_dut.is_selected(): + radio_auto_dut.click() + while True: try: self._connect_devices() @@ -477,6 +500,7 @@ class HarnessCase(unittest.TestCase): if not self.wait_until(lambda: 'disabled' not in button_next.get_attribute('class'), times=(30 + 4 * self.golden_devices_required)): bad_ones = [] + selected_hw_set = test_bed.find_elements_by_class_name('selected-hw') for selected_hw in selected_hw_set: form_inputs = selected_hw.find_elements_by_tag_name('input') form_port = form_inputs[0] @@ -486,7 +510,7 @@ class HarnessCase(unittest.TestCase): for selected_hw in bad_ones: port = form_port.get_attribute('value').encode('utf8') if settings.DUT_DEVICE and port == settings.DUT_DEVICE[0]: - raise Exception('DUT device failed') + raise SystemExit('DUT device failed') if not settings.APC_HOST: # port cannot recover without power off @@ -508,12 +532,7 @@ class HarnessCase(unittest.TestCase): logger.info('Try again with new golden devices') continue - if self.auto_dut: - checkbox_auto_dut = browser.find_element_by_id('EnableAutoDutSelection') - if not checkbox_auto_dut.is_selected(): - checkbox_auto_dut.click() - - time.sleep(1) + if self.auto_dut and not settings.DUT_DEVICE: radio_auto_dut = browser.find_element_by_class_name('AutoDUT_RadBtns') if not radio_auto_dut.is_selected(): radio_auto_dut.click() @@ -765,13 +784,19 @@ class HarnessCase(unittest.TestCase): inp.clear() inp.send_keys(ml64) - elif title.startswith('Sheild DUT'): + elif title.startswith('Shield Devices') or title.startswith('Sheild DUT'): # FIXME should find better way to simulate - self.dut.channel = (self.channel == THREAD_CHANNEL_MAX - and THREAD_CHANNEL_MIN) or (self.channel + 1) + if self.dut and settings.SHIELD_SIMULATION: + self.dut.channel = (self.channel == THREAD_CHANNEL_MAX + and THREAD_CHANNEL_MIN) or (self.channel + 1) + else: + raw_input('Shield DUT and press enter to continue..') - elif title.startswith('Bring DUT Back to network'): - self.dut.channel = self.channel + elif title.startswith('Unshield Devices') or title.startswith('Bring DUT Back to network'): + if self.dut and settings.SHIELD_SIMULATION: + self.dut.channel = self.channel + else: + raw_input('Bring DUT and press enter to continue..') elif title.startswith('Configure Prefix on DUT'): body = dialog.find_element_by_id('cnfrmMsg').text @@ -815,6 +840,10 @@ class HarnessCase(unittest.TestCase): except UnexpectedAlertPresentException: logger.exception('Failed to connect to harness server') raise SystemExit() + except SystemExit: + raise + except: + logger.exception('Something wrong') self._select_case(self.role, self.case) diff --git a/tools/harness-automation/autothreadharness/runner.py b/tools/harness-automation/autothreadharness/runner.py index 183a4ee74..d08e06204 100644 --- a/tools/harness-automation/autothreadharness/runner.py +++ b/tools/harness-automation/autothreadharness/runner.py @@ -52,9 +52,7 @@ logger.setLevel(logging.INFO) RESUME_SCRIPT_PATH = '%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\' \ 'Startup\\continue_harness.bat' - class SimpleTestResult(unittest.TestResult): - def __init__(self, path, auto_reboot_args=None, manual_reset=False): '''Record test results in json file @@ -144,36 +142,23 @@ class SimpleTestResult(unittest.TestResult): super(SimpleTestResult, self).addError(test, err) self.add_result(test, None, str(err[1])) - -def __print_fw_version_of_device_connected_to_port(port): - '''Print firmware version of a device connected to the COM port''' - try: - with OpenThreadController(port) as otc: - print('%s: %s' % (port, otc.version)) - except: - logger.exception('failed to get version of %s' % port) - - def list_devices(names=None, continue_from=None, **kwargs): - '''List devices in settings file and print versions''' + """List devices in settings file and print versions""" - ports = [] - - if names: - ports = list(names) + if not names: + names = [device for device, _type in settings.GOLDEN_DEVICES if _type == 'OpenThread'] + if continue_from: + continue_from = names.index(continue_from) else: - ports = [port for port, _type in settings.GOLDEN_DEVICES if _type == 'OpenThread'] - - if continue_from: - continue_from = ports.index(continue_from) - else: - continue_from = 0 - - ports = list(ports[continue_from:]) - - map(__print_fw_version_of_device_connected_to_port, ports) + continue_from = 0 + for port in names[continue_from:]: + try: + with OpenThreadController(port) as otc: + print('%s: %s' % (port, otc.version)) + except: + logger.exception('failed to get version of %s' % port) def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=None, name_greps=None, manual_reset=False, delete_history=False, max_devices=0, @@ -193,9 +178,8 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist= if blacklist: try: - excludes = filter(lambda line: not line.startswith('#'), - map(lambda line: line.strip('\n'), - open(blacklist, 'r').readlines())) + excludes = [line.strip('\n') for line in open(blacklist, 'r').readlines() + if not line.startswith('#')] except: logger.exception('Failed to open test case black list file') raise @@ -294,7 +278,6 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist= suite.run(result) - def main(): parser = argparse.ArgumentParser(description='Thread harness test case runner') parser.add_argument('--auto-reboot', '-a', action='store_true', default=False, @@ -302,7 +285,7 @@ def main(): parser.add_argument('names', metavar='NAME', type=str, nargs='*', default=None, help='test case name, omit to test all') parser.add_argument('--blacklist', '-b', metavar='BLACKLIST_FILE', type=str, - help='file to list test cases to skipt', default=None) + help='file to list test cases to skip', default=None) parser.add_argument('--continue-from', '-c', type=str, default=None, help='first case to test') parser.add_argument('--delete-history', '-d', action='store_true', default=False, @@ -313,8 +296,7 @@ def main(): help='file to list cases names to test') parser.add_argument('--skip', '-k', metavar='SKIP', type=str, help='type of results to skip.' \ - 'e for error, f for fail, p for pass. default to "efp"', - default='') + 'e for error, f for fail, p for pass.', default='') parser.add_argument('--list-devices', '-l', action='store_true', default=False, help='list devices') parser.add_argument('--manual-reset', '-m', action='store_true', default=False, @@ -332,9 +314,8 @@ def main(): if args['list_file']: try: - names = filter(lambda line: not line.startswith('#'), - map(lambda line: line.strip('\n'), - open(args['list_file'], 'r').readlines())) + names = [line.strip('\n') for line in open(args['list_file'], 'r').readlines() + if not line.startswith('#')] except: logger.exception('Failed to open test case list file') raise diff --git a/tools/harness-automation/autothreadharness/settings_sample.py b/tools/harness-automation/autothreadharness/settings_sample.py index 1384f2356..6d7d1944e 100644 --- a/tools/harness-automation/autothreadharness/settings_sample.py +++ b/tools/harness-automation/autothreadharness/settings_sample.py @@ -28,11 +28,11 @@ # -AUTO_DUT = False +AUTO_DUT = True """bool: Whether use the auto DUT feature of thread harness.""" DUT_DEVICE = ('COM16', 'OpenThread') -"""(str,str): The first element is serial port of the DUT, and the second is +"""(str, str): The first element is serial port of the DUT, and the second is the device type. This must be set if AUTO_DUT=False.""" DUT_VERSION = 'g12345' @@ -53,11 +53,11 @@ THREAD_NETWORKNAME = 'GRL' THREAD_EXTPANID = '000db80000000000' """str: Thread extended PAN ID""" -THREAD_CHILD_TIMEOUT = 100 -"""int: Child timeout in seconds""" +THREAD_CHILD_TIMEOUT = 0 +"""int: Child timeout in seconds. Set to 0 to use Harness's default value.""" -THREAD_SED_POLLING_INTERVAL = 100 -"""int: SED polling interval in seconds""" +THREAD_SED_POLLING_INTERVAL = 0 +"""int: SED polling interval in seconds. Set to 0 to use Harness's default value.""" HARNESS_HOME = 'C:\\GRL\\Thread1.1' """str: Harness installation path, e.g. ``C:\GRL\Thread1.1``""" @@ -74,11 +74,18 @@ TESTER_REMARKS = 'OpenThread is great' GOLDEN_DEVICES = [] """[(str, str)]: devices list. -It should be something like [('COM1', 'OpenThread'), ('COM2', 'ARM')] on Windows and can be found on Windows Device Manager.""" +It should be something like [('COM1', 'OpenThread'), ('COM2', 'ARM')] for devices connected to Windows. + +For OpenThread golden devices, ser2net is also supported, just use IP:PORT for the name. For example, +('192.168.1.2:5001', 'OpenThread'). +""" OUTPUT_PATH = '.\\output' """str: Path to store results and logs, MUST be writable.""" +SHIELD_SIMULATION = False +"""bool: whether to simulate RF shield by changing channel""" + PDU_CONTROLLER_TYPE = None """str: Type of connected PDU controller. @@ -109,9 +116,6 @@ Example parameters for the 'APC_PDU_CONTROLLER': Example parameters for the 'NORDIC_BOARD_PDU_CONTOLLER': {'boards_serial_numbers': ('12345123', ...)} """ + HARNESS_VERSION = 35 """int: Version of the installed Thread Harness.""" - -GOLDEN_DEVICE_TYPE = 'OpenThread' -"""str: Type of the Golden Device.""" - diff --git a/tools/harness-automation/cases/router_5_2_7.py b/tools/harness-automation/cases/router_5_2_7.py new file mode 100644 index 000000000..38e00f1a5 --- /dev/null +++ b/tools/harness-automation/cases/router_5_2_7.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016, The OpenThread Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + + +import unittest + +from autothreadharness.harness_case import HarnessCase + +class Router_5_2_7(HarnessCase): + role = HarnessCase.ROLE_ROUTER + case = '5 2 7' + golden_devices_required = 16 + def on_dialog(self, dialog, title): + pass + +if __name__ == '__main__': + unittest.main() diff --git a/tools/harness-automation/doc/quickstart.rst b/tools/harness-automation/doc/quickstart.rst index 057c42348..09ef8361e 100644 --- a/tools/harness-automation/doc/quickstart.rst +++ b/tools/harness-automation/doc/quickstart.rst @@ -4,7 +4,7 @@ Thread Harness Automation Quick Start Setup ----- -#. Install Thread-Test-Harness1.1-Alpha v1.0-Release_13.0 +#. Install Thread-Test-Harness1.1-Alpha v1.0-Release_40.0 #. Install python 2.7 #. Get the OpenThread and switch to the harness automation path::