espressif/openthread
1
0
Fork 0
mirror of https://github.com/espressif/openthread.git synced 2026-08-17 16:09:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Harness Automation] Fix retrying golden device issue and clean code (#1350)

Browse Source 
* fix bad golden device issue

* clean code

* fix small issues reported by pylint
This commit is contained in:
Buke Po
2017-02-20 09:06:41 -08:00
committed by Jonathan Hui
parent d12401a649
commit aeda7f8970
8 changed files with 123 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/harness-automation/autothreadharness/__init__.py
-3
View File
@@ -26,6 +26,3 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
tools/harness-automation/autothreadharness/harness_case.py
+61 -60
View File
@@ -27,12 +27,6 @@
# POSSIBILITY OF SUCH DAMAGE.
#
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import UnexpectedAlertPresentException
import json
import logging
import os
@@ -41,6 +35,11 @@ import re
import time
import unittest
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import UnexpectedAlertPresentException
from autothreadharness import settings
from autothreadharness.pdu_controller_factory import PduControllerFactory
from autothreadharness.harness_controller import HarnessController
@@ -58,6 +57,31 @@ THREAD_CHANNEL_MIN = 11
DEFAULT_TIMEOUT = 2700
"""Timeout for each test case in seconds"""
def wait_until(what, times=-1):
"""Wait until `what` return True
Args:
what (Callable[bool]): Call `wait()` again and again until it returns True
times (int): Maximum times of trials before giving up
Returns:
True if success, False if times threshold reached
"""
while times:
logger.info('Waiting times left %d', times)
try:
if what() is True:
return True
except:
logger.exception('Wait failed')
else:
logger.warning('Trial[%d] failed', times)
times -= 1
time.sleep(1)
return False
class HarnessCase(unittest.TestCase):
"""This is the case class of all automation test cases.
@@ -127,32 +151,10 @@ class HarnessCase(unittest.TestCase):
"""bool: whether use harness auto dut feature"""
timeout = hasattr(settings, 'TIMEOUT') and settings.TIMEOUT or DEFAULT_TIMEOUT
"""number: timeout in seconds to stop running this test case"""
started = 0
def wait_until(self, what, times=-1):
"""Wait until `what` return True
Args:
what (Callable[bool]): Call `wait()` again and again until it returns True
times (int): Maximum times of trials before giving up
Returns:
True if success, False if times threshold reached
"""
while times:
logger.info('Waiting times left %d', times)
try:
if what() is True:
return True
except:
logger.exception('Wait failed')
else:
logger.warning('Trial[%d] failed', times)
times -= 1
time.sleep(1)
return False
"""number: test case started timestamp"""
def __init__(self, *args, **kwargs):
self.dut = None
@@ -167,7 +169,7 @@ class HarnessCase(unittest.TestCase):
"""Reboot all usb devices.
Note:
If APC_HOST is not valid, usb devices is not rebooted.
If PDU_CONTROLLER_TYPE is not valid, usb devices is not rebooted.
"""
if self.manual_reset:
raw_input('Reset golden devices and press enter to continue..')
@@ -180,10 +182,10 @@ class HarnessCase(unittest.TestCase):
port, _ = device
try:
with OpenThreadController(port) as otc:
logger.info('Resetting %s' % port)
logger.info('Resetting %s', port)
otc.reset()
except:
logger.exception('Failed to reset device %s' % port)
logger.exception('Failed to reset device %s', port)
self.history.mark_bad_golden_device(device)
return
@@ -244,7 +246,7 @@ class HarnessCase(unittest.TestCase):
dut = OpenThreadController(dut_port)
self.dut = dut
if not settings.APC_HOST or self.manual_reset:
if not settings.PDU_CONTROLLER_TYPE or self.manual_reset:
self.dut.reset()
def _destroy_dut(self):
@@ -271,7 +273,7 @@ class HarnessCase(unittest.TestCase):
browser.maximize_window()
browser.get(settings.HARNESS_URL)
self._browser = browser
if not self.wait_until(lambda: 'Thread' in browser.title, 30):
if not wait_until(lambda: 'Thread' in browser.title, 30):
self.assertIn('Thread', browser.title)
def _destroy_browser(self):
@@ -457,11 +459,9 @@ class HarnessCase(unittest.TestCase):
remove_button.click()
selected_hw_num = selected_hw_num - 1
devices = list(settings.GOLDEN_DEVICES)
for index, device in enumerate(devices):
port = device[0]
if (self.history.is_bad_golden_device(port) or (settings.DUT_DEVICE and port == settings.DUT_DEVICE[0])):
devices.remove(device)
devices = [device for device in settings.GOLDEN_DEVICES
if not self.history.is_bad_golden_device(device[0]) and \
not (settings.DUT_DEVICE and device[0] == settings.DUT_DEVICE[0])]
logger.info('Available golden devices: %s', json.dumps(devices, indent=2))
golden_devices_required = self.golden_devices_required
@@ -497,8 +497,8 @@ class HarnessCase(unittest.TestCase):
try:
self._connect_devices()
button_next = browser.find_element_by_id('nextBtn')
if not self.wait_until(lambda: 'disabled' not in button_next.get_attribute('class'),
times=(30 + 4 * self.golden_devices_required)):
if not wait_until(lambda: 'disabled' not in button_next.get_attribute('class'),
times=(30 + 4 * self.golden_devices_required)):
bad_ones = []
selected_hw_set = test_bed.find_elements_by_class_name('selected-hw')
for selected_hw in selected_hw_set:
@@ -508,11 +508,13 @@ class HarnessCase(unittest.TestCase):
bad_ones.append(selected_hw)
for selected_hw in bad_ones:
form_inputs = selected_hw.find_elements_by_tag_name('input')
form_port = form_inputs[0]
port = form_port.get_attribute('value').encode('utf8')
if settings.DUT_DEVICE and port == settings.DUT_DEVICE[0]:
raise SystemExit('DUT device failed')
if not settings.APC_HOST:
if not settings.PDU_CONTROLLER_TYPE:
# port cannot recover without power off
self.history.mark_bad_golden_device(port)
@@ -556,7 +558,7 @@ class HarnessCase(unittest.TestCase):
time.sleep(1)
checkbox = None
self.wait_until(lambda: self._browser.find_elements_by_css_selector('.tree-node .tree-title') and True)
wait_until(lambda: self._browser.find_elements_by_css_selector('.tree-node .tree-title') and True)
elems = self._browser.find_elements_by_css_selector('.tree-node .tree-title')
finder = re.compile(r'.*\b' + case + r'\b')
finder_dotted = re.compile(r'.*\b' + case.replace(' ', r'\.') + r'\b')
@@ -574,13 +576,13 @@ class HarnessCase(unittest.TestCase):
time.sleep(5)
raise Exception('Failed to find the case')
self._browser.execute_script("$('.overview').css('left', '0')");
self._browser.execute_script("$('.overview').css('left', '0')")
checkbox.click()
time.sleep(1)
elem = self._browser.find_element_by_id('runTest')
elem.click()
if not self.wait_until(lambda: self._browser.find_element_by_id('stopTest') and True, 10):
if not wait_until(lambda: self._browser.find_element_by_id('stopTest') and True, 10):
raise Exception('Failed to start test case')
def _collect_result(self):
@@ -612,22 +614,21 @@ class HarnessCase(unittest.TestCase):
# generate excel
self._browser.find_element_by_class_name('save-excel').click()
time.sleep(1)
for wh in self._browser.window_handles:
if wh != main_window:
self._browser.switch_to.window(wh)
for window_handle in self._browser.window_handles:
if window_handle != main_window:
self._browser.switch_to.window(window_handle)
self._browser.close()
self._browser.switch_to.window(main_window)
# save pcap
self._browser.find_element_by_class_name('save-wireshark').click()
time.sleep(1)
for wh in self._browser.window_handles:
if wh != main_window:
self._browser.switch_to.window(wh)
for window_handle in self._browser.window_handles:
if window_handle != main_window:
self._browser.switch_to.window(window_handle)
self._browser.close()
self._browser.switch_to.window(main_window)
timestamp = time.strftime('%Y%m%d%H%M%S')
os.system('copy "%%HOMEPATH%%\\Downloads\\NewPdf_*.pdf" %s\\'
% self.result_dir)
os.system('copy "%%HOMEPATH%%\\Downloads\\ExcelReport_*.xlsx" %s\\'
@@ -658,7 +659,7 @@ class HarnessCase(unittest.TestCase):
try:
done = self._handle_dialog(dialog, title)
except:
logger.exception('Error handling dialog: %s' % title)
logger.exception('Error handling dialog: %s', title)
error = True
if done is None:
@@ -686,12 +687,13 @@ class HarnessCase(unittest.TestCase):
lines = self._hc.tail()
if 'SUCCESS: The process "dumpcap.exe" with PID ' in lines:
logger.info('Tshark should be ended now, lets wait at most 30 seconds.')
if not self.wait_until(lambda: 'tshark.exe' not in subprocess.check_output('tasklist'), 30):
res = subprocess.check_output('taskkill /t /f /im tshark.exe', stderr=subprocess.STDOUT, shell=True)
if not wait_until(lambda: 'tshark.exe' not in subprocess.check_output('tasklist'), 30):
res = subprocess.check_output('taskkill /t /f /im tshark.exe',
stderr=subprocess.STDOUT, shell=True)
logger.info(res)
# Wait until case really stopped
self.wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30)
wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30)
if error:
raise Exception('Fail for previous exceptions')
@@ -785,10 +787,9 @@ class HarnessCase(unittest.TestCase):
inp.send_keys(ml64)
elif title.startswith('Shield Devices') or title.startswith('Sheild DUT'):
# FIXME should find better way to simulate
if self.dut and settings.SHIELD_SIMULATION:
self.dut.channel = (self.channel == THREAD_CHANNEL_MAX
and THREAD_CHANNEL_MIN) or (self.channel + 1)
and THREAD_CHANNEL_MIN) or (self.channel + 1)
else:
raw_input('Shield DUT and press enter to continue..')
@@ -802,7 +803,7 @@ class HarnessCase(unittest.TestCase):
body = dialog.find_element_by_id('cnfrmMsg').text
body = body.split(': ')[1]
params = reduce(lambda params, param: params.update(((param[0].strip(' '), param[1]),)) or params,
map(lambda it: it.split('='), body.split(', ')), {})
[it.split('=') for it in body.split(', ')], {})
prefix = params['P_Prefix'].strip('\0\r\n\t ')
flags = []
if params.get('P_slaac_preferred', 0) == '1':
tools/harness-automation/autothreadharness/harness_controller.py
+30 -33
View File
@@ -37,6 +37,25 @@ from autothreadharness import settings
logger = logging.getLogger(__name__)
def _try_kill(proc):
logger.info('Try kill process')
times = 1
while proc.poll() is None:
proc.kill()
time.sleep(5)
if proc.poll() is not None:
logger.info('Process has been killed')
break
logger.info('Trial %d failed', times)
times += 1
if times > 3:
raise SystemExit()
class HarnessController(object):
"""Harness service control
@@ -61,12 +80,12 @@ class HarnessController(object):
% (settings.HARNESS_HOME, settings.HARNESS_HOME))
self.harness_file = '%s\\harness-%s.log' % (self.result_dir, time.strftime('%Y%m%d%H%M%S'))
with open(self.harness_file, 'w') as harnessOut:
with open(self.harness_file, 'w') as harness_out:
self.harness = subprocess.Popen([settings.HARNESS_HOME + '\\Python27\\python.exe',
settings.HARNESS_HOME + '\\Thread_Harness\\Run.py'],
cwd=settings.HARNESS_HOME,
stdout=harnessOut,
stderr=harnessOut,
stdout=harness_out,
stderr=harness_out,
env=env)
time.sleep(2)
@@ -76,17 +95,17 @@ class HarnessController(object):
if self.miniweb:
logger.warning('Miniweb already started')
else:
with open('%s\\miniweb-%s.log' % (self.result_dir, time.strftime('%Y%m%d%H%M%S')), 'w') as miniwebOut:
with open('%s\\miniweb-%s.log' % (self.result_dir, time.strftime('%Y%m%d%H%M%S')), 'w') as miniweb_out:
self.miniweb = subprocess.Popen([settings.HARNESS_HOME + '\\MiniWeb\\miniweb.exe'],
stdout=miniwebOut,
stderr=miniwebOut,
stdout=miniweb_out,
stderr=miniweb_out,
cwd=settings.HARNESS_HOME + '\\MiniWeb')
def stop(self):
logger.info('Stopping harness service')
if self.harness:
self._try_kill(self.harness)
_try_kill(self.harness)
self.harness = None
else:
logger.warning('Harness not started yet')
@@ -95,37 +114,15 @@ class HarnessController(object):
return
if self.miniweb:
self._try_kill(self.miniweb)
_try_kill(self.miniweb)
self.miniweb = None
else:
logger.warning('Miniweb not started yet')
def tail(self):
with open(self.harness_file) as harnessOut:
harnessOut.seek(-100, 2)
return ''.join(harnessOut.readlines())
def _try_kill(self, proc):
logger.info('Try kill process')
times = 1
while proc.poll() is None:
proc.kill()
time.sleep(5)
if proc.poll() is not None:
logger.info('Process has been killed')
break
logger.info('Trial {} failed'.format(times))
times += 1
if times > 3:
raise SystemExit()
with open(self.harness_file) as harness_out:
harness_out.seek(-100, 2)
return ''.join(harness_out.readlines())
def __del__(self):
self.stop()
if __name__ == '__main__':
hc = HarnessController()
tools/harness-automation/autothreadharness/open_thread_controller.py
+10 -6
View File
@@ -30,11 +30,12 @@
import logging
import re
import serial
import socket
import threading
import time
import serial
from . import settings
__all__ = ['OpenThreadController']
@@ -56,6 +57,7 @@ class OpenThreadController(threading.Thread):
super(OpenThreadController, self).__init__()
self.port = port
self.handle = None
self.lines = []
self._log = log
self._is_net = False
self._init()
@@ -89,10 +91,10 @@ class OpenThreadController(threading.Thread):
self.handle = None
def _connect(self):
logger.debug('My port is %s' % self.port)
logger.debug('My port is %s', self.port)
if self.port.startswith('NET'):
portnum = settings.SER2NET_PORTBASE + int(self.port.split('NET')[1])
logger.debug('My port num is %d' % portnum)
logger.debug('My port num is %d', portnum)
address = (settings.SER2NET_HOSTNAME, portnum)
self.handle = socket.create_connection(address)
self.handle.setblocking(0)
@@ -125,9 +127,9 @@ class OpenThreadController(threading.Thread):
expected str: the expected string
times int: number of trials
"""
logger.debug('[%s] Expecting [%s]' % (self.port, expected))
logger.debug('[%s] Expecting [%s]', self.port, expected)
retry_times = 10
for i in range(0, times):
while times:
if not retry_times:
break
@@ -140,6 +142,8 @@ class OpenThreadController(threading.Thread):
retry_times -= 1
time.sleep(0.1)
times -= 1
raise Exception('failed to find expected string[%s]' % expected)
def _readline(self):
@@ -177,7 +181,7 @@ class OpenThreadController(threading.Thread):
except socket.error:
logging.debug('Nothing cleared')
logger.debug('sending [%s]' % line)
logger.debug('sending [%s]', line)
self._write(line + '\r\n')
# wait for write to complete
tools/harness-automation/autothreadharness/pdu_controller.py
+2 -7
View File
@@ -58,7 +58,7 @@ class DummyPduController(PduController):
pass
def reboot(self, **params):
print('No PDU controller connected.')
logger.info('No PDU controller connected.')
def close(self):
pass
@@ -174,13 +174,8 @@ class NordicBoardPduController(PduController):
boards_serial_numbers = params['boards_serial_numbers']
for serial_number in boards_serial_numbers:
print('Resetting board with the serial number: {}'.format(serial_number))
logger.info('Resetting board with the serial number: %s', serial_number)
self._pin_reset(serial_number)
def close(self):
pass
if __name__ == '__main__':
apc = ApcPduController('192.168.1.88')
apc.reboot()
tools/harness-automation/autothreadharness/pdu_controller_factory.py
+1 -1
View File
@@ -25,7 +25,7 @@
# POSSIBILITY OF SUCH DAMAGE.
#
import pdu_controller
from . import pdu_controller
class PduControllerFactory(object):
tools/harness-automation/autothreadharness/runner.py
+7 -8
View File
@@ -45,7 +45,7 @@ from autothreadharness import settings
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
'''Logger: The global logger'''
"""Logger: The global logger"""
logger.setLevel(logging.INFO)
@@ -54,12 +54,12 @@ RESUME_SCRIPT_PATH = '%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\' \
class SimpleTestResult(unittest.TestResult):
def __init__(self, path, auto_reboot_args=None, manual_reset=False):
'''Record test results in json file
"""Record test results in json file
Args:
path (str): File path to record the results
auto_reboot (bool): Whether reboot when harness die
'''
"""
super(SimpleTestResult, self).__init__()
self.path = path
self.manual_reset = manual_reset
@@ -91,12 +91,12 @@ class SimpleTestResult(unittest.TestResult):
logger.addHandler(self.log_handler)
def add_result(self, test, passed, error=None):
'''Record test result into json file
"""Record test result into json file
Args:
test (TestCase): The test just run
passed (bool): Whether the case is passed
'''
"""
self.result[unicode(test.__class__.__name__)] = {
'started': self.started,
'stopped': time.strftime('%Y-%m-%dT%H:%M:%S'),
@@ -163,13 +163,13 @@ def list_devices(names=None, continue_from=None, **kwargs):
def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=None, name_greps=None,
manual_reset=False, delete_history=False, max_devices=0,
continue_from=None, result_file='./result.json', auto_reboot=False):
'''Discover all test cases and skip those passed
"""Discover all test cases and skip those passed
Args:
pattern (str): Pattern to match case modules, refer python's unittest
documentation for more details
skip (str): types cases to skip
'''
"""
if not os.path.exists(settings.OUTPUT_PATH):
os.mkdir(settings.OUTPUT_PATH)
@@ -192,7 +192,6 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=
log = json.load(open(result_file, 'r'))
except:
logger.exception('Failed to open result file')
pass
if not log:
log = {}
tools/harness-thci/OpenThread.py
+12 -10
View File
@@ -26,21 +26,21 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
'''
"""
>> Thread Host Controller Interface
>> Device : OpenThread THCI
>> Class : OpenThread
'''
"""
import re
import sys
import time
import serial
import socket
import logging
import serial
from IThci import IThci
from GRLLibs.UtilityModules.Test import Thread_Device_Role, Device_Data_Requirement, MacType
from GRLLibs.UtilityModules.enums import PlatformDiagnosticPacket_Direction, PlatformDiagnosticPacket_Type, AddressType
from GRLLibs.UtilityModules.enums import PlatformDiagnosticPacket_Direction, PlatformDiagnosticPacket_Type
from GRLLibs.UtilityModules.ModuleHelper import ModuleHelper, ThreadRunner
from GRLLibs.ThreadPacket.PlatformPackets import PlatformDiagnosticPacket, PlatformPackets
from Queue import Queue
@@ -112,7 +112,7 @@ class OpenThread(IThci):
print '[%s] Expecting [%s]' % (self.port, expected)
retry_times = 10
for i in range(0, times):
while times:
if not retry_times:
break
@@ -127,6 +127,8 @@ class OpenThread(IThci):
retry_times -= 1
time.sleep(0.1)
times -= 1
raise Exception('failed to find expected string[%s]' % expected)
def _read(self, size=512):
@@ -206,15 +208,15 @@ class OpenThread(IThci):
try:
# command retransmit times
retryTimes = 3
while retryTimes:
retryTimes -= 1
retry_times = 3
while retry_times:
retry_times -= 1
try:
self._sendline(cmd)
self._expect(cmd)
except Exception as e:
logging.exception('%s: failed to send command[%s]: %s', self.port, cmd, str(e))
if not retryTimes:
if not retry_times:
raise
else:
break