From aeda7f89704463212eb2e3826a0cc340d94017da Mon Sep 17 00:00:00 2001 From: Buke Po Date: Tue, 21 Feb 2017 01:06:41 +0800 Subject: [PATCH] [Harness Automation] Fix retrying golden device issue and clean code (#1350) * fix bad golden device issue * clean code * fix small issues reported by pylint --- .../autothreadharness/__init__.py | 3 - .../autothreadharness/harness_case.py | 121 +++++++++--------- .../autothreadharness/harness_controller.py | 63 +++++---- .../open_thread_controller.py | 16 ++- .../autothreadharness/pdu_controller.py | 9 +- .../pdu_controller_factory.py | 2 +- .../autothreadharness/runner.py | 15 +-- tools/harness-thci/OpenThread.py | 22 ++-- 8 files changed, 123 insertions(+), 128 deletions(-) diff --git a/tools/harness-automation/autothreadharness/__init__.py b/tools/harness-automation/autothreadharness/__init__.py index 91b9b9f1e..a10b5bf43 100644 --- a/tools/harness-automation/autothreadharness/__init__.py +++ b/tools/harness-automation/autothreadharness/__init__.py @@ -26,6 +26,3 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # - - - diff --git a/tools/harness-automation/autothreadharness/harness_case.py b/tools/harness-automation/autothreadharness/harness_case.py index 9d8fd766a..f383ca5bb 100644 --- a/tools/harness-automation/autothreadharness/harness_case.py +++ b/tools/harness-automation/autothreadharness/harness_case.py @@ -27,12 +27,6 @@ # POSSIBILITY OF SUCH DAMAGE. # - -from selenium import webdriver -from selenium.webdriver import ActionChains -from selenium.webdriver.support.ui import Select -from selenium.common.exceptions import UnexpectedAlertPresentException - import json import logging import os @@ -41,6 +35,11 @@ import re import time import unittest +from selenium import webdriver +from selenium.webdriver import ActionChains +from selenium.webdriver.support.ui import Select +from selenium.common.exceptions import UnexpectedAlertPresentException + from autothreadharness import settings from autothreadharness.pdu_controller_factory import PduControllerFactory from autothreadharness.harness_controller import HarnessController @@ -58,6 +57,31 @@ THREAD_CHANNEL_MIN = 11 DEFAULT_TIMEOUT = 2700 """Timeout for each test case in seconds""" +def wait_until(what, times=-1): + """Wait until `what` return True + + Args: + what (Callable[bool]): Call `wait()` again and again until it returns True + times (int): Maximum times of trials before giving up + + Returns: + True if success, False if times threshold reached + + """ + while times: + logger.info('Waiting times left %d', times) + try: + if what() is True: + return True + except: + logger.exception('Wait failed') + else: + logger.warning('Trial[%d] failed', times) + times -= 1 + time.sleep(1) + + return False + class HarnessCase(unittest.TestCase): """This is the case class of all automation test cases. @@ -127,32 +151,10 @@ class HarnessCase(unittest.TestCase): """bool: whether use harness auto dut feature""" timeout = hasattr(settings, 'TIMEOUT') and settings.TIMEOUT or DEFAULT_TIMEOUT + """number: timeout in seconds to stop running this test case""" started = 0 - def wait_until(self, what, times=-1): - """Wait until `what` return True - - Args: - what (Callable[bool]): Call `wait()` again and again until it returns True - times (int): Maximum times of trials before giving up - - Returns: - True if success, False if times threshold reached - - """ - while times: - logger.info('Waiting times left %d', times) - try: - if what() is True: - return True - except: - logger.exception('Wait failed') - else: - logger.warning('Trial[%d] failed', times) - times -= 1 - time.sleep(1) - - return False + """number: test case started timestamp""" def __init__(self, *args, **kwargs): self.dut = None @@ -167,7 +169,7 @@ class HarnessCase(unittest.TestCase): """Reboot all usb devices. Note: - If APC_HOST is not valid, usb devices is not rebooted. + If PDU_CONTROLLER_TYPE is not valid, usb devices is not rebooted. """ if self.manual_reset: raw_input('Reset golden devices and press enter to continue..') @@ -180,10 +182,10 @@ class HarnessCase(unittest.TestCase): port, _ = device try: with OpenThreadController(port) as otc: - logger.info('Resetting %s' % port) + logger.info('Resetting %s', port) otc.reset() except: - logger.exception('Failed to reset device %s' % port) + logger.exception('Failed to reset device %s', port) self.history.mark_bad_golden_device(device) return @@ -244,7 +246,7 @@ class HarnessCase(unittest.TestCase): dut = OpenThreadController(dut_port) self.dut = dut - if not settings.APC_HOST or self.manual_reset: + if not settings.PDU_CONTROLLER_TYPE or self.manual_reset: self.dut.reset() def _destroy_dut(self): @@ -271,7 +273,7 @@ class HarnessCase(unittest.TestCase): browser.maximize_window() browser.get(settings.HARNESS_URL) self._browser = browser - if not self.wait_until(lambda: 'Thread' in browser.title, 30): + if not wait_until(lambda: 'Thread' in browser.title, 30): self.assertIn('Thread', browser.title) def _destroy_browser(self): @@ -457,11 +459,9 @@ class HarnessCase(unittest.TestCase): remove_button.click() selected_hw_num = selected_hw_num - 1 - devices = list(settings.GOLDEN_DEVICES) - for index, device in enumerate(devices): - port = device[0] - if (self.history.is_bad_golden_device(port) or (settings.DUT_DEVICE and port == settings.DUT_DEVICE[0])): - devices.remove(device) + devices = [device for device in settings.GOLDEN_DEVICES + if not self.history.is_bad_golden_device(device[0]) and \ + not (settings.DUT_DEVICE and device[0] == settings.DUT_DEVICE[0])] logger.info('Available golden devices: %s', json.dumps(devices, indent=2)) golden_devices_required = self.golden_devices_required @@ -497,8 +497,8 @@ class HarnessCase(unittest.TestCase): try: self._connect_devices() button_next = browser.find_element_by_id('nextBtn') - if not self.wait_until(lambda: 'disabled' not in button_next.get_attribute('class'), - times=(30 + 4 * self.golden_devices_required)): + if not wait_until(lambda: 'disabled' not in button_next.get_attribute('class'), + times=(30 + 4 * self.golden_devices_required)): bad_ones = [] selected_hw_set = test_bed.find_elements_by_class_name('selected-hw') for selected_hw in selected_hw_set: @@ -508,11 +508,13 @@ class HarnessCase(unittest.TestCase): bad_ones.append(selected_hw) for selected_hw in bad_ones: + form_inputs = selected_hw.find_elements_by_tag_name('input') + form_port = form_inputs[0] port = form_port.get_attribute('value').encode('utf8') if settings.DUT_DEVICE and port == settings.DUT_DEVICE[0]: raise SystemExit('DUT device failed') - if not settings.APC_HOST: + if not settings.PDU_CONTROLLER_TYPE: # port cannot recover without power off self.history.mark_bad_golden_device(port) @@ -556,7 +558,7 @@ class HarnessCase(unittest.TestCase): time.sleep(1) checkbox = None - self.wait_until(lambda: self._browser.find_elements_by_css_selector('.tree-node .tree-title') and True) + wait_until(lambda: self._browser.find_elements_by_css_selector('.tree-node .tree-title') and True) elems = self._browser.find_elements_by_css_selector('.tree-node .tree-title') finder = re.compile(r'.*\b' + case + r'\b') finder_dotted = re.compile(r'.*\b' + case.replace(' ', r'\.') + r'\b') @@ -574,13 +576,13 @@ class HarnessCase(unittest.TestCase): time.sleep(5) raise Exception('Failed to find the case') - self._browser.execute_script("$('.overview').css('left', '0')"); + self._browser.execute_script("$('.overview').css('left', '0')") checkbox.click() time.sleep(1) elem = self._browser.find_element_by_id('runTest') elem.click() - if not self.wait_until(lambda: self._browser.find_element_by_id('stopTest') and True, 10): + if not wait_until(lambda: self._browser.find_element_by_id('stopTest') and True, 10): raise Exception('Failed to start test case') def _collect_result(self): @@ -612,22 +614,21 @@ class HarnessCase(unittest.TestCase): # generate excel self._browser.find_element_by_class_name('save-excel').click() time.sleep(1) - for wh in self._browser.window_handles: - if wh != main_window: - self._browser.switch_to.window(wh) + for window_handle in self._browser.window_handles: + if window_handle != main_window: + self._browser.switch_to.window(window_handle) self._browser.close() self._browser.switch_to.window(main_window) # save pcap self._browser.find_element_by_class_name('save-wireshark').click() time.sleep(1) - for wh in self._browser.window_handles: - if wh != main_window: - self._browser.switch_to.window(wh) + for window_handle in self._browser.window_handles: + if window_handle != main_window: + self._browser.switch_to.window(window_handle) self._browser.close() self._browser.switch_to.window(main_window) - timestamp = time.strftime('%Y%m%d%H%M%S') os.system('copy "%%HOMEPATH%%\\Downloads\\NewPdf_*.pdf" %s\\' % self.result_dir) os.system('copy "%%HOMEPATH%%\\Downloads\\ExcelReport_*.xlsx" %s\\' @@ -658,7 +659,7 @@ class HarnessCase(unittest.TestCase): try: done = self._handle_dialog(dialog, title) except: - logger.exception('Error handling dialog: %s' % title) + logger.exception('Error handling dialog: %s', title) error = True if done is None: @@ -686,12 +687,13 @@ class HarnessCase(unittest.TestCase): lines = self._hc.tail() if 'SUCCESS: The process "dumpcap.exe" with PID ' in lines: logger.info('Tshark should be ended now, lets wait at most 30 seconds.') - if not self.wait_until(lambda: 'tshark.exe' not in subprocess.check_output('tasklist'), 30): - res = subprocess.check_output('taskkill /t /f /im tshark.exe', stderr=subprocess.STDOUT, shell=True) + if not wait_until(lambda: 'tshark.exe' not in subprocess.check_output('tasklist'), 30): + res = subprocess.check_output('taskkill /t /f /im tshark.exe', + stderr=subprocess.STDOUT, shell=True) logger.info(res) # Wait until case really stopped - self.wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30) + wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30) if error: raise Exception('Fail for previous exceptions') @@ -785,10 +787,9 @@ class HarnessCase(unittest.TestCase): inp.send_keys(ml64) elif title.startswith('Shield Devices') or title.startswith('Sheild DUT'): - # FIXME should find better way to simulate if self.dut and settings.SHIELD_SIMULATION: self.dut.channel = (self.channel == THREAD_CHANNEL_MAX - and THREAD_CHANNEL_MIN) or (self.channel + 1) + and THREAD_CHANNEL_MIN) or (self.channel + 1) else: raw_input('Shield DUT and press enter to continue..') @@ -802,7 +803,7 @@ class HarnessCase(unittest.TestCase): body = dialog.find_element_by_id('cnfrmMsg').text body = body.split(': ')[1] params = reduce(lambda params, param: params.update(((param[0].strip(' '), param[1]),)) or params, - map(lambda it: it.split('='), body.split(', ')), {}) + [it.split('=') for it in body.split(', ')], {}) prefix = params['P_Prefix'].strip('\0\r\n\t ') flags = [] if params.get('P_slaac_preferred', 0) == '1': diff --git a/tools/harness-automation/autothreadharness/harness_controller.py b/tools/harness-automation/autothreadharness/harness_controller.py index 41c4030b0..c6dbeae78 100644 --- a/tools/harness-automation/autothreadharness/harness_controller.py +++ b/tools/harness-automation/autothreadharness/harness_controller.py @@ -37,6 +37,25 @@ from autothreadharness import settings logger = logging.getLogger(__name__) +def _try_kill(proc): + logger.info('Try kill process') + times = 1 + + while proc.poll() is None: + proc.kill() + + time.sleep(5) + + if proc.poll() is not None: + logger.info('Process has been killed') + break + + logger.info('Trial %d failed', times) + times += 1 + + if times > 3: + raise SystemExit() + class HarnessController(object): """Harness service control @@ -61,12 +80,12 @@ class HarnessController(object): % (settings.HARNESS_HOME, settings.HARNESS_HOME)) self.harness_file = '%s\\harness-%s.log' % (self.result_dir, time.strftime('%Y%m%d%H%M%S')) - with open(self.harness_file, 'w') as harnessOut: + with open(self.harness_file, 'w') as harness_out: self.harness = subprocess.Popen([settings.HARNESS_HOME + '\\Python27\\python.exe', settings.HARNESS_HOME + '\\Thread_Harness\\Run.py'], cwd=settings.HARNESS_HOME, - stdout=harnessOut, - stderr=harnessOut, + stdout=harness_out, + stderr=harness_out, env=env) time.sleep(2) @@ -76,17 +95,17 @@ class HarnessController(object): if self.miniweb: logger.warning('Miniweb already started') else: - with open('%s\\miniweb-%s.log' % (self.result_dir, time.strftime('%Y%m%d%H%M%S')), 'w') as miniwebOut: + with open('%s\\miniweb-%s.log' % (self.result_dir, time.strftime('%Y%m%d%H%M%S')), 'w') as miniweb_out: self.miniweb = subprocess.Popen([settings.HARNESS_HOME + '\\MiniWeb\\miniweb.exe'], - stdout=miniwebOut, - stderr=miniwebOut, + stdout=miniweb_out, + stderr=miniweb_out, cwd=settings.HARNESS_HOME + '\\MiniWeb') def stop(self): logger.info('Stopping harness service') if self.harness: - self._try_kill(self.harness) + _try_kill(self.harness) self.harness = None else: logger.warning('Harness not started yet') @@ -95,37 +114,15 @@ class HarnessController(object): return if self.miniweb: - self._try_kill(self.miniweb) + _try_kill(self.miniweb) self.miniweb = None else: logger.warning('Miniweb not started yet') def tail(self): - with open(self.harness_file) as harnessOut: - harnessOut.seek(-100, 2) - return ''.join(harnessOut.readlines()) - - def _try_kill(self, proc): - logger.info('Try kill process') - times = 1 - - while proc.poll() is None: - proc.kill() - - time.sleep(5) - - if proc.poll() is not None: - logger.info('Process has been killed') - break - - logger.info('Trial {} failed'.format(times)) - times += 1 - - if times > 3: - raise SystemExit() + with open(self.harness_file) as harness_out: + harness_out.seek(-100, 2) + return ''.join(harness_out.readlines()) def __del__(self): self.stop() - -if __name__ == '__main__': - hc = HarnessController() diff --git a/tools/harness-automation/autothreadharness/open_thread_controller.py b/tools/harness-automation/autothreadharness/open_thread_controller.py index c87be917b..e86479e5c 100644 --- a/tools/harness-automation/autothreadharness/open_thread_controller.py +++ b/tools/harness-automation/autothreadharness/open_thread_controller.py @@ -30,11 +30,12 @@ import logging import re -import serial import socket import threading import time +import serial + from . import settings __all__ = ['OpenThreadController'] @@ -56,6 +57,7 @@ class OpenThreadController(threading.Thread): super(OpenThreadController, self).__init__() self.port = port self.handle = None + self.lines = [] self._log = log self._is_net = False self._init() @@ -89,10 +91,10 @@ class OpenThreadController(threading.Thread): self.handle = None def _connect(self): - logger.debug('My port is %s' % self.port) + logger.debug('My port is %s', self.port) if self.port.startswith('NET'): portnum = settings.SER2NET_PORTBASE + int(self.port.split('NET')[1]) - logger.debug('My port num is %d' % portnum) + logger.debug('My port num is %d', portnum) address = (settings.SER2NET_HOSTNAME, portnum) self.handle = socket.create_connection(address) self.handle.setblocking(0) @@ -125,9 +127,9 @@ class OpenThreadController(threading.Thread): expected str: the expected string times int: number of trials """ - logger.debug('[%s] Expecting [%s]' % (self.port, expected)) + logger.debug('[%s] Expecting [%s]', self.port, expected) retry_times = 10 - for i in range(0, times): + while times: if not retry_times: break @@ -140,6 +142,8 @@ class OpenThreadController(threading.Thread): retry_times -= 1 time.sleep(0.1) + times -= 1 + raise Exception('failed to find expected string[%s]' % expected) def _readline(self): @@ -177,7 +181,7 @@ class OpenThreadController(threading.Thread): except socket.error: logging.debug('Nothing cleared') - logger.debug('sending [%s]' % line) + logger.debug('sending [%s]', line) self._write(line + '\r\n') # wait for write to complete diff --git a/tools/harness-automation/autothreadharness/pdu_controller.py b/tools/harness-automation/autothreadharness/pdu_controller.py index 46a61dea0..66d8e5137 100644 --- a/tools/harness-automation/autothreadharness/pdu_controller.py +++ b/tools/harness-automation/autothreadharness/pdu_controller.py @@ -58,7 +58,7 @@ class DummyPduController(PduController): pass def reboot(self, **params): - print('No PDU controller connected.') + logger.info('No PDU controller connected.') def close(self): pass @@ -174,13 +174,8 @@ class NordicBoardPduController(PduController): boards_serial_numbers = params['boards_serial_numbers'] for serial_number in boards_serial_numbers: - print('Resetting board with the serial number: {}'.format(serial_number)) + logger.info('Resetting board with the serial number: %s', serial_number) self._pin_reset(serial_number) def close(self): pass - - -if __name__ == '__main__': - apc = ApcPduController('192.168.1.88') - apc.reboot() diff --git a/tools/harness-automation/autothreadharness/pdu_controller_factory.py b/tools/harness-automation/autothreadharness/pdu_controller_factory.py index 9bfd2a247..5407bb58e 100644 --- a/tools/harness-automation/autothreadharness/pdu_controller_factory.py +++ b/tools/harness-automation/autothreadharness/pdu_controller_factory.py @@ -25,7 +25,7 @@ # POSSIBILITY OF SUCH DAMAGE. # -import pdu_controller +from . import pdu_controller class PduControllerFactory(object): diff --git a/tools/harness-automation/autothreadharness/runner.py b/tools/harness-automation/autothreadharness/runner.py index d08e06204..a4a4917b3 100644 --- a/tools/harness-automation/autothreadharness/runner.py +++ b/tools/harness-automation/autothreadharness/runner.py @@ -45,7 +45,7 @@ from autothreadharness import settings logging.basicConfig(level=logging.INFO) logger = logging.getLogger() -'''Logger: The global logger''' +"""Logger: The global logger""" logger.setLevel(logging.INFO) @@ -54,12 +54,12 @@ RESUME_SCRIPT_PATH = '%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\' \ class SimpleTestResult(unittest.TestResult): def __init__(self, path, auto_reboot_args=None, manual_reset=False): - '''Record test results in json file + """Record test results in json file Args: path (str): File path to record the results auto_reboot (bool): Whether reboot when harness die - ''' + """ super(SimpleTestResult, self).__init__() self.path = path self.manual_reset = manual_reset @@ -91,12 +91,12 @@ class SimpleTestResult(unittest.TestResult): logger.addHandler(self.log_handler) def add_result(self, test, passed, error=None): - '''Record test result into json file + """Record test result into json file Args: test (TestCase): The test just run passed (bool): Whether the case is passed - ''' + """ self.result[unicode(test.__class__.__name__)] = { 'started': self.started, 'stopped': time.strftime('%Y-%m-%dT%H:%M:%S'), @@ -163,13 +163,13 @@ def list_devices(names=None, continue_from=None, **kwargs): def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=None, name_greps=None, manual_reset=False, delete_history=False, max_devices=0, continue_from=None, result_file='./result.json', auto_reboot=False): - '''Discover all test cases and skip those passed + """Discover all test cases and skip those passed Args: pattern (str): Pattern to match case modules, refer python's unittest documentation for more details skip (str): types cases to skip - ''' + """ if not os.path.exists(settings.OUTPUT_PATH): os.mkdir(settings.OUTPUT_PATH) @@ -192,7 +192,6 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist= log = json.load(open(result_file, 'r')) except: logger.exception('Failed to open result file') - pass if not log: log = {} diff --git a/tools/harness-thci/OpenThread.py b/tools/harness-thci/OpenThread.py index 9a6904ea9..0e5cede9b 100644 --- a/tools/harness-thci/OpenThread.py +++ b/tools/harness-thci/OpenThread.py @@ -26,21 +26,21 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -''' +""" >> Thread Host Controller Interface >> Device : OpenThread THCI >> Class : OpenThread -''' +""" import re -import sys import time -import serial import socket import logging + +import serial from IThci import IThci from GRLLibs.UtilityModules.Test import Thread_Device_Role, Device_Data_Requirement, MacType -from GRLLibs.UtilityModules.enums import PlatformDiagnosticPacket_Direction, PlatformDiagnosticPacket_Type, AddressType +from GRLLibs.UtilityModules.enums import PlatformDiagnosticPacket_Direction, PlatformDiagnosticPacket_Type from GRLLibs.UtilityModules.ModuleHelper import ModuleHelper, ThreadRunner from GRLLibs.ThreadPacket.PlatformPackets import PlatformDiagnosticPacket, PlatformPackets from Queue import Queue @@ -112,7 +112,7 @@ class OpenThread(IThci): print '[%s] Expecting [%s]' % (self.port, expected) retry_times = 10 - for i in range(0, times): + while times: if not retry_times: break @@ -127,6 +127,8 @@ class OpenThread(IThci): retry_times -= 1 time.sleep(0.1) + times -= 1 + raise Exception('failed to find expected string[%s]' % expected) def _read(self, size=512): @@ -206,15 +208,15 @@ class OpenThread(IThci): try: # command retransmit times - retryTimes = 3 - while retryTimes: - retryTimes -= 1 + retry_times = 3 + while retry_times: + retry_times -= 1 try: self._sendline(cmd) self._expect(cmd) except Exception as e: logging.exception('%s: failed to send command[%s]: %s', self.port, cmd, str(e)) - if not retryTimes: + if not retry_times: raise else: break