diff --git a/tools/harness-automation/autothreadharness/exceptions.py b/tools/harness-automation/autothreadharness/exceptions.py new file mode 100644 index 000000000..e299cc7a8 --- /dev/null +++ b/tools/harness-automation/autothreadharness/exceptions.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016, The OpenThread Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +class FailError(Exception): + """This error class is the base for all errors that cause the current test case fails. + """ + pass + +class FatalError(FailError): + """This error class is the base for all errors that cause the whole test procedure stops. + """ + pass + +class GoldenDeviceNotEnoughError(FailError): + """Raised when no more golden devices are available for trying. + """ + pass diff --git a/tools/harness-automation/autothreadharness/harness_case.py b/tools/harness-automation/autothreadharness/harness_case.py index 2082fb491..36da5d3d7 100644 --- a/tools/harness-automation/autothreadharness/harness_case.py +++ b/tools/harness-automation/autothreadharness/harness_case.py @@ -42,10 +42,11 @@ from selenium.webdriver.support.ui import Select from selenium.common.exceptions import UnexpectedAlertPresentException from autothreadharness import settings -from autothreadharness.pdu_controller_factory import PduControllerFactory +from autothreadharness.exceptions import FailError, FatalError, GoldenDeviceNotEnoughError from autothreadharness.harness_controller import HarnessController from autothreadharness.helpers import HistoryHelper from autothreadharness.open_thread_controller import OpenThreadController +from autothreadharness.pdu_controller_factory import PduControllerFactory logger = logging.getLogger(__name__) @@ -145,9 +146,6 @@ class HarnessCase(unittest.TestCase): """int: SED polling interval in seconds """ - manual_reset = False - """bool: whether reset manually""" - auto_dut = settings.AUTO_DUT """bool: whether use harness auto dut feature""" @@ -172,10 +170,7 @@ class HarnessCase(unittest.TestCase): Note: If PDU_CONTROLLER_TYPE is not valid, usb devices is not rebooted. """ - if self.manual_reset: - raw_input('Reset golden devices and press enter to continue..') - return - elif not settings.PDU_CONTROLLER_TYPE: + if not settings.PDU_CONTROLLER_TYPE: if settings.AUTO_DUT: return @@ -212,7 +207,7 @@ class HarnessCase(unittest.TestCase): pdu.close() break - time.sleep(20) + time.sleep(len(settings.GOLDEN_DEVICES)) def _init_harness(self): """Restart harness backend service. @@ -252,9 +247,6 @@ class HarnessCase(unittest.TestCase): dut = OpenThreadController(dut_port) self.dut = dut - if not settings.PDU_CONTROLLER_TYPE or self.manual_reset: - self.dut.reset() - def _destroy_dut(self): self.dut = None @@ -345,7 +337,7 @@ class HarnessCase(unittest.TestCase): logger.exception('Failed to get dialog.') else: if dialog and dialog.get_attribute('aria-hidden') == 'false': - times = 100 # FIXME better to be a more meaningful value + times = 60 while times: status = dialog.find_element_by_class_name('status-notify').text if 'Searching' in status: @@ -476,7 +468,7 @@ class HarnessCase(unittest.TestCase): golden_devices_required += 1 if len(devices) < golden_devices_required: - raise Exception('Golden devices is not enough') + raise GoldenDeviceNotEnoughError() # add golden devices while golden_devices_required: @@ -518,10 +510,14 @@ class HarnessCase(unittest.TestCase): form_port = form_inputs[0] port = form_port.get_attribute('value').encode('utf8') if settings.DUT_DEVICE and port == settings.DUT_DEVICE[0]: - raise SystemExit('DUT device failed') + if settings.PDU_CONTROLLER_TYPE is None: + # connection error cannot recover without power cycling + raise FatalError('Failed to connect to DUT') + else: + raise FailError('Failed to connect to DUT') - if not settings.PDU_CONTROLLER_TYPE: - # port cannot recover without power off + if settings.PDU_CONTROLLER_TYPE is None: + # port cannot recover without power cycling self.history.mark_bad_golden_device(port) # remove the bad one @@ -535,7 +531,7 @@ class HarnessCase(unittest.TestCase): if devices is None: logger.warning('Golden devices not enough') - raise SystemExit() + raise GoldenDeviceNotEnoughError() else: logger.info('Try again with new golden devices') continue @@ -548,7 +544,7 @@ class HarnessCase(unittest.TestCase): time.sleep(5) button_next.click() - except SystemExit: + except FailError: raise except: logger.exception('Unexpected error') @@ -669,7 +665,7 @@ class HarnessCase(unittest.TestCase): error = True if done is None: - raise Exception('Unexpected dialog occurred') + raise FailError('Unexpected dialog occurred') dialog.find_element_by_id('ConfirmOk').click() @@ -702,7 +698,7 @@ class HarnessCase(unittest.TestCase): wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30) if error: - raise Exception('Fail for previous exceptions') + raise FailError('Fail for previous exceptions') def _handle_dialog(self, dialog, title): """Handle a dialog. @@ -752,7 +748,7 @@ class HarnessCase(unittest.TestCase): break if not ll64: - raise Exception('No link local address found') + raise FailError('No link local address found') logger.info('Link local address is %s', ll64) inp = dialog.find_element_by_id('cnfrmInpText') @@ -847,7 +843,11 @@ class HarnessCase(unittest.TestCase): except UnexpectedAlertPresentException: logger.exception('Failed to connect to harness server') raise SystemExit() - except SystemExit: + except FatalError: + logger.exception('Test stopped for fatal error') + raise SystemExit() + except FailError: + logger.exception('Test failed') raise except: logger.exception('Something wrong') diff --git a/tools/harness-automation/autothreadharness/pdu_controller.py b/tools/harness-automation/autothreadharness/pdu_controller.py index 66d8e5137..4950cf894 100644 --- a/tools/harness-automation/autothreadharness/pdu_controller.py +++ b/tools/harness-automation/autothreadharness/pdu_controller.py @@ -179,3 +179,14 @@ class NordicBoardPduController(PduController): def close(self): pass + +class ManualPduController(PduController): + + def open(self, **kwargs): + pass + + def reboot(self, **kwargs): + raw_input('Reset all devices and press enter to continue..') + + def close(self): + pass diff --git a/tools/harness-automation/autothreadharness/pdu_controller_factory.py b/tools/harness-automation/autothreadharness/pdu_controller_factory.py index 5407bb58e..d585c059b 100644 --- a/tools/harness-automation/autothreadharness/pdu_controller_factory.py +++ b/tools/harness-automation/autothreadharness/pdu_controller_factory.py @@ -36,5 +36,7 @@ class PduControllerFactory(object): return pdu_controller.NordicBoardPduController() elif _type == 'APC_PDU_CONTROLLER': return pdu_controller.ApcPduController() + elif _type == 'MANUAL_PDU_CONTROLLER': + return pdu_controller.ManualPduController() else: return pdu_controller.DummyPduController() diff --git a/tools/harness-automation/autothreadharness/runner.py b/tools/harness-automation/autothreadharness/runner.py index a4a4917b3..d8417ea23 100644 --- a/tools/harness-automation/autothreadharness/runner.py +++ b/tools/harness-automation/autothreadharness/runner.py @@ -53,7 +53,7 @@ RESUME_SCRIPT_PATH = '%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\' \ 'Startup\\continue_harness.bat' class SimpleTestResult(unittest.TestResult): - def __init__(self, path, auto_reboot_args=None, manual_reset=False): + def __init__(self, path, auto_reboot_args=None): """Record test results in json file Args: @@ -62,7 +62,6 @@ class SimpleTestResult(unittest.TestResult): """ super(SimpleTestResult, self).__init__() self.path = path - self.manual_reset = manual_reset self.auto_reboot_args = auto_reboot_args self.result = json.load(open(self.path, 'r')) self.log_handler = None @@ -82,8 +81,6 @@ class SimpleTestResult(unittest.TestResult): # record start timestamp self.started = time.strftime('%Y-%m-%dT%H:%M:%S') - # manual reset - test.manual_reset = self.manual_reset os.system('mkdir %s' % test.result_dir) self.log_handler = logging.FileHandler('%s\\auto-%s.log' % (test.result_dir, time.strftime('%Y%m%d%H%M%S'))) self.log_handler.setLevel(logging.DEBUG) @@ -268,7 +265,13 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist= auto_reboot_args = None os.system('del "%s"' % RESUME_SCRIPT_PATH) - result = SimpleTestResult(result_file, auto_reboot_args, manual_reset) + # manual reset + if manual_reset: + settings.PDU_CONTROLLER_TYPE = 'MANUAL_PDU_CONTROLLER' + settings.PDU_CONTROLLER_OPEN_PARAMS = {} + settings.PDU_CONTROLLER_REBOOT_PARAMS = {} + + result = SimpleTestResult(result_file, auto_reboot_args) for case in suite: logger.info(case.__class__.__name__)