espressif/openthread
1
0
Fork 0
mirror of https://github.com/espressif/openthread.git synced 2026-08-12 05:37:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

support Thread Harness 1.1 R19 (#520)

Browse Source
This commit is contained in:
Buke Po
2016-09-06 09:33:19 -07:00
committed by Jonathan Hui
parent d3f8b5f242
commit a7ed23c16c
16 changed files with 257 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/harness-automation/README.rst
+1 -1
View File
@@ -2,7 +2,7 @@
Harness Automation Tool
=======================
This is a tool to automate testing openthread with GRL Thread-Test-Harness1.1-Alpha v1.0-Release_13.0.
This is a tool to automate testing openthread with GRL Thread-Test-Harness1.1-Alpha v1.0-Release_19.0.
-----------
Quick Start
tools/harness-automation/autothreadharness/harness_case.py
+59 -19
View File
@@ -57,7 +57,7 @@ THREAD_CHANNEL_MIN = 11
class HarnessCase(unittest.TestCase):
"""This is the case class of all automation test cases.
All test case classes MUST define properties `suite`, `case` and `golden_devices_needed`
All test case classes MUST define properties `suite`, `case` and `golden_devices_required`
"""
channel = settings.THREAD_CHANNEL
@@ -85,7 +85,7 @@ class HarnessCase(unittest.TestCase):
"""str: Case id, e.g. '6 5 1'.
"""
golden_devices_needed = 0
golden_devices_required = 0
"""int: Golden devices needed to finish the test
"""
@@ -96,6 +96,9 @@ class HarnessCase(unittest.TestCase):
manual_reset = False
"""bool: whether reset manually"""
auto_dut = settings.AUTO_DUT
"""bool: whether use harness auto dut feature"""
def wait_until(self, what, times=-1):
"""Wait until `what` return True
@@ -125,9 +128,7 @@ class HarnessCase(unittest.TestCase):
self._browser = None
self._hc = None
self.result_dir = '%s\\%s' % (settings.OUTPUT_PATH, self.__class__.__name__)
# create directory
if self.__class__ is not HarnessCase:
os.system('mkdir %s' % self.result_dir)
self.history = HistoryHelper()
super(HarnessCase, self).__init__(*args, **kwargs)
@@ -141,6 +142,19 @@ class HarnessCase(unittest.TestCase):
raw_input('Reset golden devices and press enter to continue..')
return
elif not settings.APC_HOST:
if settings.GOLDEN_DEVICE_TYPE != 'OpenThread':
logger.warning('All golden devices may not be resetted')
return
for device in settings.GOLDEN_DEVICES:
try:
with OpenThreadController(device) as otc:
logger.info('Resetting %s' % device)
otc.reset()
except:
logger.exception('Failed to reset device %s' % device)
self.history.mark_bad_golden_device(device)
return
tries = 3
@@ -187,6 +201,10 @@ class HarnessCase(unittest.TestCase):
DUT will be restarted. and openthread will started.
"""
if self.auto_dut:
self.dut = None
return
dut_port = settings.DUT_DEVICE
dut = OpenThreadController(dut_port)
self.dut = dut
@@ -247,6 +265,8 @@ class HarnessCase(unittest.TestCase):
logger.info('Empty files in temps')
os.system('del /q "%s\\Thread_Harness\\temp\\*.*"' % settings.HARNESS_HOME)
# create directory
os.system('mkdir %s' % self.result_dir)
self._init_harness()
self._init_devices()
self._init_dut()
@@ -351,7 +371,6 @@ class HarnessCase(unittest.TestCase):
browser = self._browser
test_bed = browser.find_element_by_id('test-bed')
time.sleep(3)
history = HistoryHelper()
selected_hw_set = test_bed.find_elements_by_class_name('selected-hw')
selected_hw_num = len(selected_hw_set)
@@ -361,17 +380,26 @@ class HarnessCase(unittest.TestCase):
remove_button.click()
selected_hw_num = selected_hw_num - 1
devices = filter(lambda port: not history.is_bad_golden_device(port),
devices = filter(lambda port: not (self.history.is_bad_golden_device(port) or (not self.auto_dut and port == settings.DUT_DEVICE )),
settings.GOLDEN_DEVICES)
logger.info('Available golden devices: %s', json.dumps(devices, indent=2))
if len(devices) < self.golden_devices_required:
raise Exception('Golden devices is not enough')
golden_devices_required = self.golden_devices_required
if self.auto_dut:
golden_devices_required = golden_devices_required + 1
if len(devices) < golden_devices_required:
raise Exception('Golden devices is not enough')
device_type_id = settings.GOLDEN_DEVICE_TYPE
if device_type_id == 'OpenThread':
device_type_id = 'ARM'
while golden_devices_required:
golden_device = browser.find_element_by_id(settings.GOLDEN_DEVICE_TYPE)
freescale = browser.find_element_by_id(device_type_id)
# drag
action_chains = ActionChains(browser)
action_chains.click_and_hold(golden_device)
action_chains.click_and_hold(freescale)
action_chains.move_to_element(test_bed).perform()
time.sleep(1)
@@ -395,15 +423,15 @@ class HarnessCase(unittest.TestCase):
while True:
try:
self._connect_devices()
elem = browser.find_element_by_id('nextBtn')
if not self.wait_until(lambda: 'disabled' not in elem.get_attribute('class'),
times=60):
button_next = browser.find_element_by_id('nextBtn')
if not self.wait_until(lambda: 'disabled' not in button_next.get_attribute('class'),
times=120):
for selected_hw in selected_hw_set:
form_inputs = selected_hw.find_elements_by_tag_name('input')
form_port = form_inputs[0]
if form_port.is_enabled():
port = form_port.get_attribute('value').encode('utf8')
history.mark_bad_golden_device(port)
self.history.mark_bad_golden_device(port)
if devices:
device = devices.pop()
form_port.clear()
@@ -418,7 +446,16 @@ class HarnessCase(unittest.TestCase):
logger.info('Try again with new golden devices')
continue
elem.click()
if self.auto_dut:
checkbox_auto_dut = browser.find_element_by_id('EnableAutoDutSelection')
if not checkbox_auto_dut.is_selected():
checkbox_auto_dut.click()
radio_auto_dut = browser.find_element_by_class_name('AutoDUT_RadBtns')
if not radio_auto_dut.is_selected():
radio_auto_dut.click()
button_next.click()
except SystemExit:
raise
except:
@@ -435,6 +472,7 @@ class HarnessCase(unittest.TestCase):
time.sleep(1)
checkbox = None
self.wait_until(lambda: self._browser.find_elements_by_css_selector('.tree-node .tree-title') and True)
elems = self._browser.find_elements_by_css_selector('.tree-node .tree-title')
for elem in elems:
action_chains = ActionChains(self._browser)
@@ -474,8 +512,9 @@ class HarnessCase(unittest.TestCase):
if dialog.get_attribute('aria-hidden') != 'false':
raise Exception('Test information dialog not ready')
version = self.auto_dut and settings.DUT_VERSION or self.dut.version
dialog.find_element_by_id('inp_dut_manufacturer').send_keys(settings.DUT_MANUFACTURER)
dialog.find_element_by_id('inp_dut_firmware_version').send_keys(self.dut.version)
dialog.find_element_by_id('inp_dut_firmware_version').send_keys(version)
dialog.find_element_by_id('inp_tester_name').send_keys(settings.TESTER_NAME)
dialog.find_element_by_id('inp_remarks').send_keys(settings.TESTER_REMARKS)
dialog.find_element_by_id('generatePdf').click()
@@ -552,7 +591,8 @@ class HarnessCase(unittest.TestCase):
time.sleep(5)
done = True
time.sleep(5)
# Wait until case really stopped
self.wait_until(lambda: self._browser.find_element_by_id('runTest') and True, 30)
if error:
raise Exception('Fail for previous exceptions')
@@ -709,5 +749,5 @@ class HarnessCase(unittest.TestCase):
# get case result
status = self._browser.find_element_by_class_name('title-test').text
logger.info(status)
success = 'Fail' not in status
success = 'Pass' in status
self.assertTrue(success)
tools/harness-automation/autothreadharness/harness_controller.py
+1
View File
@@ -65,6 +65,7 @@ class HarnessController(object):
stdout=fout,
stderr=fout,
env=env)
time.sleep(2)
if self.miniweb:
logger.warning('Miniweb already started')
tools/harness-automation/autothreadharness/helpers.py
+3
View File
@@ -56,3 +56,6 @@ class HistoryHelper(object):
def is_bad_golden_device(self, port):
return port in self.data['golden_device_black_list']
def __str__(self):
return json.dumps(self.data, indent=2)
tools/harness-automation/autothreadharness/open_thread_controller.py
+73 -14
View File
@@ -30,7 +30,7 @@
import logging
import time
import threading
import serial
from pexpect_serial import SerialSpawn
@@ -40,23 +40,45 @@ logger = logging.getLogger(__name__)
class OpenThreadController(object):
"""This is an simple wrapper to communicate with openthread"""
def __init__(self, port):
def __init__(self, port, log=False):
"""Initialize the controller
Args:
port (str): serial port's path or name(windows)
"""
self.port = port
self.ss = None
self._log = log
self._ss = None
self._lv = None
self._init()
def _init(self):
ser = serial.Serial(self.port, 115200, timeout=2)
self.ss = SerialSpawn(ser, timeout=2)
self._ss = SerialSpawn(ser, timeout=2)
if not self._log:
return
if self._lv:
self._lv.stop()
self._lv = OpenThreadLogViewer(ss=self._ss)
self._lv.start()
def __del__(self):
if self.ss:
self.ss.close()
self.close()
def close(self):
if self._lv and self._lv.is_alive():
self._lv.viewing = False
self._lv.join()
if self._ss:
self._ss.close()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def is_started(self):
"""check if openthread is started
@@ -82,11 +104,10 @@ class OpenThreadController(object):
def reset(self):
"""Reset openthread device, not equivalent to stop and start
"""
self.ss.sendline('reset')
time.sleep(1)
self.ss.close()
time.sleep(1)
self._init()
logger.info('DUT> reset')
self._log and self._lv.pause()
self._ss.sendline('reset')
self._log and self._lv.resume()
def _req(self, req):
"""Send command and wait for response.
@@ -100,13 +121,14 @@ class OpenThreadController(object):
[str]: The output lines
"""
logger.info('DUT> %s', req)
self._log and self._lv.pause()
times = 3
while times:
times = times - 1
try:
self.ss.sendline(req)
self.ss.expect(req + self.ss.linesep)
self._ss.sendline(req)
self._ss.expect(req + self._ss.linesep)
except:
logger.exception('Failed to send command')
else:
@@ -116,7 +138,7 @@ class OpenThreadController(object):
res = []
while True:
line = self.ss.readline().strip('\0\r\n\t ')
line = self._ss.readline().strip('\0\r\n\t ')
logger.debug(line)
if line:
@@ -124,6 +146,7 @@ class OpenThreadController(object):
break
res.append(line)
self._log and self._lv.resume()
return res
@property
@@ -218,3 +241,39 @@ class OpenThreadController(object):
self._req('prefix remove %s' % prefix)
time.sleep(1)
self._req('netdataregister')
def enable_blacklist(self):
"""Enable blacklist feature"""
self._req('blacklist enable')
def add_blacklist(self, mac):
"""Add a mac address to blacklist"""
self._req('blacklist add %s' % mac)
class OpenThreadLogViewer(threading.Thread):
_lock = threading.Lock()
viewing = False
def __init__(self, *args, **kwargs):
self._ss = kwargs.pop('ss')
super(OpenThreadLogViewer, self).__init__(*args, **kwargs)
def run(self):
self.viewing = True
while self.viewing and self._lock.acquire():
try:
line = self._ss.readline().strip('\0\r\n\t ')
except:
pass
else:
logger.info(line)
self._lock.release()
time.sleep(0)
def resume(self):
"""Start dumping logs"""
self._lock.release()
def pause(self):
"""Start dumping logs"""
self._lock.acquire()
tools/harness-automation/autothreadharness/runner.py
+12 -2
View File
@@ -83,6 +83,7 @@ class SimpleTestResult(unittest.TestResult):
# manual reset
test.manual_reset = self.manual_reset
os.system('mkdir %s' % test.result_dir)
self.log_handler = logging.FileHandler('%s\\auto-%s.log' % (test.result_dir, time.strftime('%Y%m%d%H%M%S')))
self.log_handler.setLevel(logging.DEBUG)
self.log_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
@@ -149,10 +150,17 @@ def discover(names=None, pattern='*.py', skip='efp', dry_run=False,
'''
if list_devices:
for port in names or settings.GOLDEN_DEVICES:
if continue_from:
continue_from = settings.GOLDEN_DEVICES.index(continue_from)
else:
continue_from = 0
for port in names or settings.GOLDEN_DEVICES[continue_from:]:
print('%s: %s' % (port, OpenThreadController(port).version))
return
if delete_blacklist:
os.system('del history.json')
log = None
if os.path.isfile(result_file):
try:
@@ -167,6 +175,9 @@ def discover(names=None, pattern='*.py', skip='efp', dry_run=False,
suite = unittest.TestSuite()
discovered = unittest.defaultTestLoader.discover('cases', pattern)
if names and continue_from:
names = names[names.index(continue_from):]
for s1 in discovered:
for s2 in s1:
for case in s2:
@@ -226,7 +237,6 @@ def discover(names=None, pattern='*.py', skip='efp', dry_run=False,
if dry_run:
return
os.system('del history.json')
suite.run(result)
def main():
tools/harness-automation/autothreadharness/settings_sample.py
+13 -8
View File
@@ -28,8 +28,14 @@
#
AUTO_DUT = False
"""bool: Whether use the auto DUT feature of thread harness."""
DUT_DEVICE = 'COM16'
"""str: Serial port of the DUT"""
"""str: Serial port of the DUT, must be set if AUTO_DUT=False."""
DUT_VERSION = 'g12345'
"""str: Version of DUT, must be set if AUTO_DUT=False."""
DUT_MANUFACTURER = 'Open Thread'
"""str: Manufacturer of the DUT"""
@@ -53,7 +59,7 @@ THREAD_SED_POLLING_INTERVAL = 100
"""int: SED polling interval in seconds"""
HARNESS_HOME = 'C:\\GRL\\Thread1.1'
"""str: Harness installation path"""
"""str: Harness installation path, e.g. ``C:\GRL\Thread1.1``"""
HARNESS_URL = 'http://127.0.0.1:8000'
"""str: Harness front-end url"""
@@ -65,7 +71,7 @@ Keep this None if no APC PDU available.
"""
APC_OUTLET = 1
"""int: PDU outlet"""
"""int: PDU outlet, only needed when APC_HOST is not None."""
TESTER_NAME = 'Thread Open'
"""str: Who are you"""
@@ -73,14 +79,13 @@ TESTER_NAME = 'Thread Open'
TESTER_REMARKS = 'OpenThread is great'
"""str: Any comments in the final PDF"""
GOLDEN_DEVICE_TYPE = 'ARM'
"""str: Golden device type"""
GOLDEN_DEVICE_TYPE = 'OpenThread'
"""str: Golden device type. Possible values are: `OpenThread`, `ARM`, `SiLabs` and `Freescale`."""
GOLDEN_DEVICES = []
"""[str]: Golden devices list.
It should be something like ['COM1', 'COM2'] on Windows
"""
It should be something like ['COM1', 'COM2'] on Windows and can be found on Windows Device Manager."""
OUTPUT_PATH = '.\\output'
"""str: Path to store results and logs"""
"""str: Path to store results and logs, MUST be writable."""
tools/harness-automation/cases/leader_5_3_6.py
+1 -1
View File
@@ -34,7 +34,7 @@ import unittest
class Leader_5_3_6(HarnessCase):
suite = 1
case = '5 3 6'
golden_devices_required = 2
golden_devices_required = 5
def on_dialog(self, dialog, title):
pass
tools/harness-automation/cases/ed_6_3_2.py → tools/harness-automation/cases/med_6_3_2.py
+1 -1
View File
@@ -33,7 +33,7 @@ import unittest
from autothreadharness.harness_case import HarnessCase
class ED_6_3_2(HarnessCase):
suite = 32
suite = 1024
case = '6 3 2'
golden_devices_required = 1
def on_dialog(self, dialog, title):
tools/harness-automation/cases/reed_5_2_4.py
+1 -1
View File
@@ -35,7 +35,7 @@ from autothreadharness.harness_case import HarnessCase
class REED_5_2_4(HarnessCase):
suite = 16
case = '5 2 4'
golden_devices_required = 17
golden_devices_required = 18
def on_dialog(self, dialog, title):
pass
tools/harness-automation/cases/router_5_1_7.py
+1 -1
View File
@@ -35,7 +35,7 @@ from autothreadharness.harness_case import HarnessCase
class Router_5_1_7(HarnessCase):
suite = 2
case = '5 1 7'
golden_devices_required = 7
golden_devices_required = 11
def on_dialog(self, dialog, title):
if title.startswith('Enter Router Max Child Count'):
inp = dialog.find_element_by_id('cnfrmInpText')
tools/harness-automation/cases/router_5_3_11.py
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python
#
# Copyright (c) 2016, Nest Labs, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
from autothreadharness.harness_case import HarnessCase
import unittest
class Router_5_3_11(HarnessCase):
suite = 2
case = '5 3 11'
golden_devices_required = 2
def on_dialog(self, dialog, title):
pass
if __name__ == '__main__':
unittest.main()
tools/harness-automation/cases/router_5_3_3.py
+4 -1
View File
@@ -36,6 +36,9 @@ class Router_5_3_3(HarnessCase):
case = '5 3 3'
golden_devices_required = 4
def on_dialog(self, dialog, title):
pass
if title.startswith('Start DUT'):
self.dut.enable_blacklist()
self.dut.add_blacklist('166e0a000000005')
if __name__ == '__main__':
unittest.main()
tools/harness-automation/cases/router_5_7_1.py
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python
#
# Copyright (c) 2016, Nest Labs, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
import unittest
from autothreadharness.harness_case import HarnessCase
class Router_5_7_1(HarnessCase):
suite = 2
case = '5 7 1'
golden_devices_required = 4
def on_dialog(self, dialog, title):
pass
if __name__ == '__main__':
unittest.main()
tools/harness-automation/cases/router_7_1_6.py
+1 -1
View File
@@ -35,7 +35,7 @@ from autothreadharness.harness_case import HarnessCase
class Router_7_1_6(HarnessCase):
suite = 2
case = '7 1 6'
golden_devices_required = 3
golden_devices_required = 25
def on_dialog(self, dialog, title):
pass
tools/harness-automation/doc/quickstart.rst
+2 -14
View File
@@ -6,7 +6,7 @@ Setup
#. Install Thread-Test-Harness1.1-Alpha v1.0-Release_13.0
#. Install python 2.7
#. Get OpenThread and switch to the harness automation path::
#. Get the OpenThread and switch to the harness automation path::
git clone https://github.com/openthread/openthread.git
cd openthread/tools/harness-automation
@@ -17,22 +17,10 @@ Setup
#. Update settings.
Just copy the sample and modify::
Just copy the sample and modify according to the comments carefully::
cp autothreadharness/settings_sample.py autothreadharness/settings.py
APC_HOST
MUST be set to None if no APC PDU available.
GOLDEN_DEVICES
can be found on Windows Device Manager. You have to filter out the ports of sniffer device and DUT by yourself.
OUTPUT_PATH
MUST be set to a writable directory.
HARNESS_HOME
MUST be set to Thread Harness installation directory. e.g. ``C:\GRL\Thread``
Run single case
---------------