[Harness Automation]: enhance exception handling and refine manual reset feature (#1369)

* refine exception handling

* implement manual pdu controller

tools/harness-automation/autothreadharness/exceptions.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016, The OpenThread Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the
+#    names of its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+class FailError(Exception):
+    """This error class is the base for all errors that cause the current test case fails.
+    """
+    pass
+
+class FatalError(FailError):
+    """This error class is the base for all errors that cause the whole test procedure stops.
+    """
+    pass
+
+class GoldenDeviceNotEnoughError(FailError):
+    """Raised when no more golden devices are available for trying.
+    """
+    pass

tools/harness-automation/autothreadharness/harness_case.py

@@ -42,10 +42,11 @@ from selenium.webdriver.support.ui import Select
 from selenium.common.exceptions import UnexpectedAlertPresentException
 
 from autothreadharness import settings
-from autothreadharness.pdu_controller_factory import PduControllerFactory
+from autothreadharness.exceptions import FailError, FatalError, GoldenDeviceNotEnoughError
 from autothreadharness.harness_controller import HarnessController
 from autothreadharness.helpers import HistoryHelper
 from autothreadharness.open_thread_controller import OpenThreadController
+from autothreadharness.pdu_controller_factory import PduControllerFactory
 
 logger = logging.getLogger(__name__)
 
@@ -145,9 +146,6 @@ class HarnessCase(unittest.TestCase):
     """int: SED polling interval in seconds
     """
 
-    manual_reset = False
-    """bool: whether reset manually"""
-
     auto_dut = settings.AUTO_DUT
     """bool: whether use harness auto dut feature"""
 
@@ -172,10 +170,7 @@ class HarnessCase(unittest.TestCase):
         Note:
             If PDU_CONTROLLER_TYPE is not valid, usb devices is not rebooted.
         """
-        if self.manual_reset:
-            raw_input('Reset golden devices and press enter to continue..')
-            return
-        elif not settings.PDU_CONTROLLER_TYPE:
+        if not settings.PDU_CONTROLLER_TYPE:
             if settings.AUTO_DUT:
                 return
 
@@ -212,7 +207,7 @@ class HarnessCase(unittest.TestCase):
                 pdu.close()
                 break
 
-        time.sleep(20)
+        time.sleep(len(settings.GOLDEN_DEVICES))
 
     def _init_harness(self):
         """Restart harness backend service.
@@ -252,9 +247,6 @@ class HarnessCase(unittest.TestCase):
         dut = OpenThreadController(dut_port)
         self.dut = dut
 
-        if not settings.PDU_CONTROLLER_TYPE or self.manual_reset:
-            self.dut.reset()
-
     def _destroy_dut(self):
         self.dut = None
 
@@ -345,7 +337,7 @@ class HarnessCase(unittest.TestCase):
             logger.exception('Failed to get dialog.')
         else:
             if dialog and dialog.get_attribute('aria-hidden') == 'false':
-                times = 100 # FIXME better to be a more meaningful value
+                times = 60
                 while times:
                     status = dialog.find_element_by_class_name('status-notify').text
                     if 'Searching' in status:
@@ -476,7 +468,7 @@ class HarnessCase(unittest.TestCase):
             golden_devices_required += 1
 
         if len(devices) < golden_devices_required:
-            raise Exception('Golden devices is not enough')
+            raise GoldenDeviceNotEnoughError()
 
         # add golden devices
         while golden_devices_required:
@@ -518,10 +510,14 @@ class HarnessCase(unittest.TestCase):
                         form_port = form_inputs[0]
                         port = form_port.get_attribute('value').encode('utf8')
                         if settings.DUT_DEVICE and port == settings.DUT_DEVICE[0]:
-                            raise SystemExit('DUT device failed')
+                            if settings.PDU_CONTROLLER_TYPE is None:
+                                # connection error cannot recover without power cycling
+                                raise FatalError('Failed to connect to DUT')
+                            else:
+                                raise FailError('Failed to connect to DUT')
 
-                        if not settings.PDU_CONTROLLER_TYPE:
-                            # port cannot recover without power off
+                        if settings.PDU_CONTROLLER_TYPE is None:
+                            # port cannot recover without power cycling
                             self.history.mark_bad_golden_device(port)
 
                         # remove the bad one
@@ -535,7 +531,7 @@ class HarnessCase(unittest.TestCase):
 
                     if devices is None:
                         logger.warning('Golden devices not enough')
-                        raise SystemExit()
+                        raise GoldenDeviceNotEnoughError()
                     else:
                         logger.info('Try again with new golden devices')
                         continue
@@ -548,7 +544,7 @@ class HarnessCase(unittest.TestCase):
                     time.sleep(5)
 
                 button_next.click()
-            except SystemExit:
+            except FailError:
                 raise
             except:
                 logger.exception('Unexpected error')
@@ -669,7 +665,7 @@ class HarnessCase(unittest.TestCase):
                         error = True
 
                     if done is None:
-                        raise Exception('Unexpected dialog occurred')
+                        raise FailError('Unexpected dialog occurred')
 
                     dialog.find_element_by_id('ConfirmOk').click()
 
@@ -702,7 +698,7 @@ class HarnessCase(unittest.TestCase):
         wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30)
 
         if error:
-            raise Exception('Fail for previous exceptions')
+            raise FailError('Fail for previous exceptions')
 
     def _handle_dialog(self, dialog, title):
         """Handle a dialog.
@@ -752,7 +748,7 @@ class HarnessCase(unittest.TestCase):
                     break
 
             if not ll64:
-                raise Exception('No link local address found')
+                raise FailError('No link local address found')
 
             logger.info('Link local address is %s', ll64)
             inp = dialog.find_element_by_id('cnfrmInpText')
@@ -847,7 +843,11 @@ class HarnessCase(unittest.TestCase):
         except UnexpectedAlertPresentException:
             logger.exception('Failed to connect to harness server')
             raise SystemExit()
-        except SystemExit:
+        except FatalError:
+            logger.exception('Test stopped for fatal error')
+            raise SystemExit()
+        except FailError:
+            logger.exception('Test failed')
             raise
         except:
             logger.exception('Something wrong')

tools/harness-automation/autothreadharness/pdu_controller.py

@@ -179,3 +179,14 @@ class NordicBoardPduController(PduController):
 
     def close(self):
         pass
+
+class ManualPduController(PduController):
+
+    def open(self, **kwargs):
+        pass
+
+    def reboot(self, **kwargs):
+        raw_input('Reset all devices and press enter to continue..')
+
+    def close(self):
+        pass

tools/harness-automation/autothreadharness/pdu_controller_factory.py

@@ -36,5 +36,7 @@ class PduControllerFactory(object):
             return pdu_controller.NordicBoardPduController()
         elif _type == 'APC_PDU_CONTROLLER':
             return pdu_controller.ApcPduController()
+        elif _type == 'MANUAL_PDU_CONTROLLER':
+            return pdu_controller.ManualPduController()
         else:
             return pdu_controller.DummyPduController()

tools/harness-automation/autothreadharness/runner.py

@@ -53,7 +53,7 @@ RESUME_SCRIPT_PATH = '%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\' \
                      'Startup\\continue_harness.bat'
 
 class SimpleTestResult(unittest.TestResult):
-    def __init__(self, path, auto_reboot_args=None, manual_reset=False):
+    def __init__(self, path, auto_reboot_args=None):
         """Record test results in json file
 
         Args:
@@ -62,7 +62,6 @@ class SimpleTestResult(unittest.TestResult):
         """
         super(SimpleTestResult, self).__init__()
         self.path = path
-        self.manual_reset = manual_reset
         self.auto_reboot_args = auto_reboot_args
         self.result = json.load(open(self.path, 'r'))
         self.log_handler = None
@@ -82,8 +81,6 @@ class SimpleTestResult(unittest.TestResult):
         # record start timestamp
         self.started = time.strftime('%Y-%m-%dT%H:%M:%S')
 
-        # manual reset
-        test.manual_reset = self.manual_reset
         os.system('mkdir %s' % test.result_dir)
         self.log_handler = logging.FileHandler('%s\\auto-%s.log' % (test.result_dir, time.strftime('%Y%m%d%H%M%S')))
         self.log_handler.setLevel(logging.DEBUG)
@@ -268,7 +265,13 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=
         auto_reboot_args = None
         os.system('del "%s"' % RESUME_SCRIPT_PATH)
 
-    result = SimpleTestResult(result_file, auto_reboot_args, manual_reset)
+    # manual reset
+    if manual_reset:
+        settings.PDU_CONTROLLER_TYPE = 'MANUAL_PDU_CONTROLLER'
+        settings.PDU_CONTROLLER_OPEN_PARAMS = {}
+        settings.PDU_CONTROLLER_REBOOT_PARAMS = {}
+
+    result = SimpleTestResult(result_file, auto_reboot_args)
     for case in suite:
         logger.info(case.__class__.__name__)