espressif/openthread
1
0
Fork 0
mirror of https://github.com/espressif/openthread.git synced 2026-08-18 16:39:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix issues until harness r40 (#1348)

Browse Source
This commit is contained in:
Buke Po
2017-02-19 23:09:15 -08:00
committed by Jonathan Hui
parent fec2e7e27e
commit 1973a03865
6 changed files with 147 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/harness-automation/README.rst
+9 -3
View File
@@ -2,7 +2,7 @@
Harness Automation Tool
=======================
This is a tool to automate testing openthread with GRL Thread-Test-Harness1.1-Alpha v1.0-Release_19.0.
This is a tool to automate testing openthread with GRL Thread-Test-Harness1.1-Alpha v1.0-Release_40.0.
-----------
Quick Start
@@ -33,14 +33,20 @@ Other options::
-h, --help
Show help message and exit.
--blacklist BLACKLIST_FILE, -b BLACKLIST_FILE
skip test cases listed in BLACKLIST_FILE.
--pattern PATTERN, -p PATTERN
File name pattern, Default to all python files.
--delete-blacklist, -d
Clear blacklist on startup. By default, golden devices failed to be connected are kept in a blacklist automatically. Add this option to clear blacklist on startup.
Clear golden device blacklist on startup. By default, golden devices failed to be connected are kept in a blacklist automatically. Add this option to clear blacklist on startup.
--name-greps NAME_GREPS, -g NAME_GREPS
Filter case by its name using filename matching syntax. Multiple this options are OR-ed to allow more tests.
--skip SKIP, -k SKIP
Type of test case status to skip. ``e`` for error, ``f`` for fail, ``p`` for pass. Default to "efp". If test case names are given by ``NAME``, this option will not work.
Type of test case status to skip. ``e`` for error, ``f`` for fail, ``p`` for pass. If test case names are given by ``NAME``, this option will not work.
--dry-run, -n
Just show what test case will be run.
tools/harness-automation/autothreadharness/harness_case.py
+61 -32
View File
@@ -116,6 +116,10 @@ class HarnessCase(unittest.TestCase):
"""int: Child timeout in seconds
"""
sed_polling_interval = settings.THREAD_SED_POLLING_INTERVAL
"""int: SED polling interval in seconds
"""
manual_reset = False
"""bool: whether reset manually"""
@@ -124,6 +128,7 @@ class HarnessCase(unittest.TestCase):
timeout = hasattr(settings, 'TIMEOUT') and settings.TIMEOUT or DEFAULT_TIMEOUT
started = 0
def wait_until(self, what, times=-1):
"""Wait until `what` return True
@@ -168,10 +173,6 @@ class HarnessCase(unittest.TestCase):
raw_input('Reset golden devices and press enter to continue..')
return
elif not settings.PDU_CONTROLLER_TYPE:
if settings.GOLDEN_DEVICE_TYPE != 'OpenThread':
logger.warning('All golden devices may not be resetted')
return
if settings.AUTO_DUT:
return
@@ -189,7 +190,7 @@ class HarnessCase(unittest.TestCase):
tries = 3
pdu_factory = PduControllerFactory()
while True:
try:
pdu = pdu_factory.create_pdu_controller(settings.PDU_CONTROLLER_TYPE)
@@ -322,6 +323,13 @@ class HarnessCase(unittest.TestCase):
def _setup_page(self):
"""Do sniffer settings and general settings
"""
if not self.started:
self.started = time.time()
if time.time() - self.started > 30:
self._browser.refresh()
return
# Detect Sniffer
try:
dialog = self._browser.find_element_by_id('capture-Setup-modal')
@@ -374,25 +382,28 @@ class HarnessCase(unittest.TestCase):
# General Setup
try:
button = self._browser.find_element_by_id('general-Setup')
button.click()
time.sleep(2)
if self.child_timeout or self.sed_polling_interval:
button = self._browser.find_element_by_id('general-Setup')
button.click()
time.sleep(2)
dialog = self._browser.find_element_by_id('general-Setup-modal')
if dialog.get_attribute('aria-hidden') != 'false':
raise Exception('Missing General Setup dialog')
dialog = self._browser.find_element_by_id('general-Setup-modal')
if dialog.get_attribute('aria-hidden') != 'false':
raise Exception('Missing General Setup dialog')
field = dialog.find_element_by_id('inp_general_child_update_wait_time')
field.clear()
field.send_keys(str(self.child_timeout))
field = dialog.find_element_by_id('inp_general_child_update_wait_time')
field.clear()
if self.child_timeout:
field.send_keys(str(self.child_timeout))
field = dialog.find_element_by_id('inp_general_sed_polling_rate')
field.clear()
field.send_keys(str(settings.THREAD_SED_POLLING_INTERVAL))
field = dialog.find_element_by_id('inp_general_sed_polling_rate')
field.clear()
if self.sed_polling_interval:
field.send_keys(str(self.sed_polling_interval))
button = dialog.find_element_by_id('saveGeneralSettings')
button.click()
time.sleep(1)
button = dialog.find_element_by_id('saveGeneralSettings')
button.click()
time.sleep(1)
except:
logger.exception('Failed to do general setup')
@@ -470,6 +481,18 @@ class HarnessCase(unittest.TestCase):
if settings.DUT_DEVICE:
self._add_device(*settings.DUT_DEVICE)
# enable AUTO DUT
if self.auto_dut:
checkbox_auto_dut = browser.find_element_by_id('EnableAutoDutSelection')
if not checkbox_auto_dut.is_selected():
checkbox_auto_dut.click()
time.sleep(1)
if settings.DUT_DEVICE:
radio_auto_dut = browser.find_element_by_class_name('AutoDUT_RadBtns')
if not radio_auto_dut.is_selected():
radio_auto_dut.click()
while True:
try:
self._connect_devices()
@@ -477,6 +500,7 @@ class HarnessCase(unittest.TestCase):
if not self.wait_until(lambda: 'disabled' not in button_next.get_attribute('class'),
times=(30 + 4 * self.golden_devices_required)):
bad_ones = []
selected_hw_set = test_bed.find_elements_by_class_name('selected-hw')
for selected_hw in selected_hw_set:
form_inputs = selected_hw.find_elements_by_tag_name('input')
form_port = form_inputs[0]
@@ -486,7 +510,7 @@ class HarnessCase(unittest.TestCase):
for selected_hw in bad_ones:
port = form_port.get_attribute('value').encode('utf8')
if settings.DUT_DEVICE and port == settings.DUT_DEVICE[0]:
raise Exception('DUT device failed')
raise SystemExit('DUT device failed')
if not settings.APC_HOST:
# port cannot recover without power off
@@ -508,12 +532,7 @@ class HarnessCase(unittest.TestCase):
logger.info('Try again with new golden devices')
continue
if self.auto_dut:
checkbox_auto_dut = browser.find_element_by_id('EnableAutoDutSelection')
if not checkbox_auto_dut.is_selected():
checkbox_auto_dut.click()
time.sleep(1)
if self.auto_dut and not settings.DUT_DEVICE:
radio_auto_dut = browser.find_element_by_class_name('AutoDUT_RadBtns')
if not radio_auto_dut.is_selected():
radio_auto_dut.click()
@@ -765,13 +784,19 @@ class HarnessCase(unittest.TestCase):
inp.clear()
inp.send_keys(ml64)
elif title.startswith('Sheild DUT'):
elif title.startswith('Shield Devices') or title.startswith('Sheild DUT'):
# FIXME should find better way to simulate
self.dut.channel = (self.channel == THREAD_CHANNEL_MAX
and THREAD_CHANNEL_MIN) or (self.channel + 1)
if self.dut and settings.SHIELD_SIMULATION:
self.dut.channel = (self.channel == THREAD_CHANNEL_MAX
and THREAD_CHANNEL_MIN) or (self.channel + 1)
else:
raw_input('Shield DUT and press enter to continue..')
elif title.startswith('Bring DUT Back to network'):
self.dut.channel = self.channel
elif title.startswith('Unshield Devices') or title.startswith('Bring DUT Back to network'):
if self.dut and settings.SHIELD_SIMULATION:
self.dut.channel = self.channel
else:
raw_input('Bring DUT and press enter to continue..')
elif title.startswith('Configure Prefix on DUT'):
body = dialog.find_element_by_id('cnfrmMsg').text
@@ -815,6 +840,10 @@ class HarnessCase(unittest.TestCase):
except UnexpectedAlertPresentException:
logger.exception('Failed to connect to harness server')
raise SystemExit()
except SystemExit:
raise
except:
logger.exception('Something wrong')
self._select_case(self.role, self.case)
tools/harness-automation/autothreadharness/runner.py
+18 -37
View File
@@ -52,9 +52,7 @@ logger.setLevel(logging.INFO)
RESUME_SCRIPT_PATH = '%appdata%\\Microsoft\\Windows\\Start Menu\\Programs\\' \
'Startup\\continue_harness.bat'
class SimpleTestResult(unittest.TestResult):
def __init__(self, path, auto_reboot_args=None, manual_reset=False):
'''Record test results in json file
@@ -144,36 +142,23 @@ class SimpleTestResult(unittest.TestResult):
super(SimpleTestResult, self).addError(test, err)
self.add_result(test, None, str(err[1]))
def __print_fw_version_of_device_connected_to_port(port):
'''Print firmware version of a device connected to the COM port'''
try:
with OpenThreadController(port) as otc:
print('%s: %s' % (port, otc.version))
except:
logger.exception('failed to get version of %s' % port)
def list_devices(names=None, continue_from=None, **kwargs):
'''List devices in settings file and print versions'''
"""List devices in settings file and print versions"""
ports = []
if names:
ports = list(names)
if not names:
names = [device for device, _type in settings.GOLDEN_DEVICES if _type == 'OpenThread']
if continue_from:
continue_from = names.index(continue_from)
else:
ports = [port for port, _type in settings.GOLDEN_DEVICES if _type == 'OpenThread']
if continue_from:
continue_from = ports.index(continue_from)
else:
continue_from = 0
ports = list(ports[continue_from:])
map(__print_fw_version_of_device_connected_to_port, ports)
continue_from = 0
for port in names[continue_from:]:
try:
with OpenThreadController(port) as otc:
print('%s: %s' % (port, otc.version))
except:
logger.exception('failed to get version of %s' % port)
def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=None, name_greps=None,
manual_reset=False, delete_history=False, max_devices=0,
@@ -193,9 +178,8 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=
if blacklist:
try:
excludes = filter(lambda line: not line.startswith('#'),
map(lambda line: line.strip('\n'),
open(blacklist, 'r').readlines()))
excludes = [line.strip('\n') for line in open(blacklist, 'r').readlines()
if not line.startswith('#')]
except:
logger.exception('Failed to open test case black list file')
raise
@@ -294,7 +278,6 @@ def discover(names=None, pattern=['*.py'], skip='efp', dry_run=False, blacklist=
suite.run(result)
def main():
parser = argparse.ArgumentParser(description='Thread harness test case runner')
parser.add_argument('--auto-reboot', '-a', action='store_true', default=False,
@@ -302,7 +285,7 @@ def main():
parser.add_argument('names', metavar='NAME', type=str, nargs='*', default=None,
help='test case name, omit to test all')
parser.add_argument('--blacklist', '-b', metavar='BLACKLIST_FILE', type=str,
help='file to list test cases to skipt', default=None)
help='file to list test cases to skip', default=None)
parser.add_argument('--continue-from', '-c', type=str, default=None,
help='first case to test')
parser.add_argument('--delete-history', '-d', action='store_true', default=False,
@@ -313,8 +296,7 @@ def main():
help='file to list cases names to test')
parser.add_argument('--skip', '-k', metavar='SKIP', type=str,
help='type of results to skip.' \
'e for error, f for fail, p for pass. default to "efp"',
default='')
'e for error, f for fail, p for pass.', default='')
parser.add_argument('--list-devices', '-l', action='store_true', default=False,
help='list devices')
parser.add_argument('--manual-reset', '-m', action='store_true', default=False,
@@ -332,9 +314,8 @@ def main():
if args['list_file']:
try:
names = filter(lambda line: not line.startswith('#'),
map(lambda line: line.strip('\n'),
open(args['list_file'], 'r').readlines()))
names = [line.strip('\n') for line in open(args['list_file'], 'r').readlines()
if not line.startswith('#')]
except:
logger.exception('Failed to open test case list file')
raise
tools/harness-automation/autothreadharness/settings_sample.py
+15 -11
View File
@@ -28,11 +28,11 @@
#
AUTO_DUT = False
AUTO_DUT = True
"""bool: Whether use the auto DUT feature of thread harness."""
DUT_DEVICE = ('COM16', 'OpenThread')
"""(str,str): The first element is serial port of the DUT, and the second is
"""(str, str): The first element is serial port of the DUT, and the second is
the device type. This must be set if AUTO_DUT=False."""
DUT_VERSION = 'g12345'
@@ -53,11 +53,11 @@ THREAD_NETWORKNAME = 'GRL'
THREAD_EXTPANID = '000db80000000000'
"""str: Thread extended PAN ID"""
THREAD_CHILD_TIMEOUT = 100
"""int: Child timeout in seconds"""
THREAD_CHILD_TIMEOUT = 0
"""int: Child timeout in seconds. Set to 0 to use Harness's default value."""
THREAD_SED_POLLING_INTERVAL = 100
"""int: SED polling interval in seconds"""
THREAD_SED_POLLING_INTERVAL = 0
"""int: SED polling interval in seconds. Set to 0 to use Harness's default value."""
HARNESS_HOME = 'C:\\GRL\\Thread1.1'
"""str: Harness installation path, e.g. ``C:\GRL\Thread1.1``"""
@@ -74,11 +74,18 @@ TESTER_REMARKS = 'OpenThread is great'
GOLDEN_DEVICES = []
"""[(str, str)]: devices list.
It should be something like [('COM1', 'OpenThread'), ('COM2', 'ARM')] on Windows and can be found on Windows Device Manager."""
It should be something like [('COM1', 'OpenThread'), ('COM2', 'ARM')] for devices connected to Windows.
For OpenThread golden devices, ser2net is also supported, just use IP:PORT for the name. For example,
('192.168.1.2:5001', 'OpenThread').
"""
OUTPUT_PATH = '.\\output'
"""str: Path to store results and logs, MUST be writable."""
SHIELD_SIMULATION = False
"""bool: whether to simulate RF shield by changing channel"""
PDU_CONTROLLER_TYPE = None
"""str: Type of connected PDU controller.
@@ -109,9 +116,6 @@ Example parameters for the 'APC_PDU_CONTROLLER':
Example parameters for the 'NORDIC_BOARD_PDU_CONTOLLER':
{'boards_serial_numbers': ('12345123', ...)}
"""
HARNESS_VERSION = 35
"""int: Version of the installed Thread Harness."""
GOLDEN_DEVICE_TYPE = 'OpenThread'
"""str: Type of the Golden Device."""
tools/harness-automation/cases/router_5_2_7.py
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python
#
# Copyright (c) 2016, The OpenThread Authors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
import unittest
from autothreadharness.harness_case import HarnessCase
class Router_5_2_7(HarnessCase):
role = HarnessCase.ROLE_ROUTER
case = '5 2 7'
golden_devices_required = 16
def on_dialog(self, dialog, title):
pass
if __name__ == '__main__':
unittest.main()
tools/harness-automation/doc/quickstart.rst
+1 -1
View File
@@ -4,7 +4,7 @@ Thread Harness Automation Quick Start
Setup
-----
#. Install Thread-Test-Harness1.1-Alpha v1.0-Release_13.0
#. Install Thread-Test-Harness1.1-Alpha v1.0-Release_40.0
#. Install python 2.7
#. Get the OpenThread and switch to the harness automation path::