sniffer.py: remove scapy dependency. (#921)

Added to spinel.codec a method to register a callback for a given message type.
Pulled all IPv6 packet parsing from spinel.codec using the new callback mechanism.

tools/spinel-cli/README.md

@@ -1,13 +1,13 @@
 # Spinel CLI Reference
 
-The Spinel CLI exposes the OpenThread configuration and management APIs 
+The Spinel CLI exposes the OpenThread configuration and management APIs
 running on an NCP build via a command line interface.  Spinel CLI is primarily
 targeted for driving the automated continuous integration tests, and is
 suitable for manual experimentation with controlling OpenThread NCP instances.
 For a production grade host driver, see [wpantund]: https://github.com/openthread/wpantund.
 
-Use the CLI to play with NCP builds of OpenThread on a Linux or Mac OS 
-platform, including starting a basic tunnel interface to allow IPv6 
+Use the CLI to play with NCP builds of OpenThread on a Linux or Mac OS
+platform, including starting a basic tunnel interface to allow IPv6
 applications to run on the HOST and use the Thread network.
 
 The power of this tool is three fold:
@@ -31,9 +31,9 @@ The power of this tool is three fold:
 
 ```
 sudo easy_install pip
-sudo pip install blessed
-sudo pip install ipaddress
-sudo pip install scapy
+sudo pip install --user pyserial
+sudo pip install --user ipaddress
+sudo pip install --user scapy==2.3.2
 ```
 
 ## Usage
@@ -56,7 +56,7 @@ sudo pip install scapy
 
     -p <PIPE>, --pipe=<PIPE>
         Open a piped process connection to the OpenThread NCP device
-        where <PIPE> is the command to start an emulator, such as 
+        where <PIPE> is the command to start an emulator, such as
         "ot-ncp".  Spinel-cli will communicate with the child process
         via stdin/stdout.
 
@@ -80,9 +80,9 @@ sudo pip install scapy
 ## Quick Start
 
 The spinel-cli tool provides an intuitive command line interface, including
-all the standard OpenThread CLI commands, plus full history accessible by 
+all the standard OpenThread CLI commands, plus full history accessible by
 pressing the up/down keys, or searchable via ^R.  There are a few commands
-that spinel-cli provides as well that aren't part of the standard set 
+that spinel-cli provides as well that aren't part of the standard set
 documented in the command reference section.
 
 ```
@@ -118,7 +118,7 @@ spinel-cli >
 
 ## Running the NCP Tests
 
-The OpenThread automated test suite can be run against any of the following 
+The OpenThread automated test suite can be run against any of the following
 node types by passing the NODE_TYPE environment variable:
 
 | NODE_TYPE     |    Description                                     |
@@ -152,7 +152,7 @@ NODE_TYPE=ncp-sim BUILD_TARGET=posix-distcheck DISTCHECK_CONFIGURE_FLAGS="--with
 ### OpenThread CLI Commands
 
 The primary intent of spinel-cli is to support the exact syntax and output
-of the OpenThread CLI command set in order to seamlessly reapply the 
+of the OpenThread CLI command set in order to seamlessly reapply the
 thread-cert automated test suite against NCP targets.
 
 See [cli module][1] for more information on these commands.
@@ -161,7 +161,7 @@ See [cli module][1] for more information on these commands.
 
 ### Diagnostics CLI Commands
 
-The Diagnostics module is enabled only when building OpenThread with 
+The Diagnostics module is enabled only when building OpenThread with
 the --enable-diag configure option.
 
 See [diag module][2] for more information on these commands.
@@ -171,7 +171,7 @@ See [diag module][2] for more information on these commands.
 
 ### NCP CLI Commands
 
-These commands extend beyond the core OpenThread CLI, and are specific to 
+These commands extend beyond the core OpenThread CLI, and are specific to
 the spinel-cli tool for the purposes of debugging, access to NCP-specific
 Spinel parameters, and support of advanced configurations.
 
@@ -287,8 +287,8 @@ DEBUG_ENABLE = 0
 spinel-cli > debug 1
 DEBUG_ENABLE = 1
 spinel-cli > version
-TX Pay: (3) ['81', '02', '02'] 
-RX Pay: (53) ['81', '06', '02', '4F', '50', '45', '4E', '54', '48', '52', '45', '41', '44', '2F', '67', '38', '62', '63', '34', '62', '31', '64', '2D', '64', '69', '72', '74', '79', '3B', '20', '41', '75', '67', '20', '33', '31', '20', '32', '30', '31', '36', '20', '31', '30', '3A', '34', '38', '3A', '35', '33', '00', '40', '33'] 
+TX Pay: (3) ['81', '02', '02']
+RX Pay: (53) ['81', '06', '02', '4F', '50', '45', '4E', '54', '48', '52', '45', '41', '44', '2F', '67', '38', '62', '63', '34', '62', '31', '64', '2D', '64', '69', '72', '74', '79', '3B', '20', '41', '75', '67', '20', '33', '31', '20', '32', '30', '31', '36', '20', '31', '30', '3A', '34', '38', '3A', '35', '33', '00', '40', '33']
 OPENTHREAD/g8bc4b1d-dirty; Aug 31 2016 10:48:53
 Done
 ```
@@ -306,7 +306,7 @@ Set whether debug terminal title bar is enabled.
 Control sideband tunnel interface.
 
 #### ncp-tun up
-        
+
 Bring up Thread TUN interface.
 
 ```bash
@@ -315,7 +315,7 @@ Done
 ```
 
 #### ncp-tun down
-        
+
 Bring down Thread TUN interface.
 
 ```bash
@@ -335,7 +335,7 @@ Done
 #### ncp-tun del \<ipaddr\>
 
 Delete an IPv6 address from the Thread TUN interface.
-        
+
 ```bash
 spinel-cli > ncp-tun del 2001::dead:beef:cafe
 Done

tools/spinel-cli/SNIFFER.md

@@ -1,6 +1,6 @@
 # Spinel Sniffer Reference
 
-Any Spinel NCP node can be made into a promiscuous packet sniffer, and this 
+Any Spinel NCP node can be made into a promiscuous packet sniffer, and this
 tool both intializes a device into this mode and outputs a pcap stream that
 can be saved or piped directly into Wireshark.
 
@@ -21,9 +21,8 @@ The tool has been tested on the following platforms:
 
 ```
 sudo easy_install pip
-sudo pip install ipaddress
-sudo pip install scapy
-sudo pip install pyserial
+sudo pip install --user pyserial
+sudo pip install --user ipaddress
 ```
 
 ## Usage
@@ -46,7 +45,7 @@ sudo pip install pyserial
 
     -p <PIPE>, --pipe=<PIPE>
         Open a piped process connection to the OpenThread NCP device
-        where <PIPE> is the command to start an emulator, such as 
+        where <PIPE> is the command to start an emulator, such as
         "ot-ncp".  Spinel-cli will communicate with the child process
         via stdin/stdout.
 
@@ -90,9 +89,9 @@ From openthread root:
     sudo ./tools/spinel-cli/sniffer.py -c 11 -n 1 -u /dev/ttyUSB0 | wireshark -k -i -
 ```
 
-This will connect to stock openthread ncp firmware over the given UART, 
-make the node into a promiscuous mode sniffer on the given channel, 
-open up wireshark, and start streaming packets into wireshark. 
+This will connect to stock openthread ncp firmware over the given UART,
+make the node into a promiscuous mode sniffer on the given channel,
+open up wireshark, and start streaming packets into wireshark.
 
 ## Troubleshooting
 Q: sniffer.py throws ```ImportError: No module named dnet``` on OSX
@@ -107,4 +106,3 @@ echo 'import site; site.addsitedir("/usr/local/lib/python2.7/site-packages")' >>
 you may need to reinstall the scapy pip dependency listed above
 
 you can read more about this issue here: http://stackoverflow.com/questions/26229057/scapy-installation-fails-on-osx-with-dnet-import-error
-

tools/spinel-cli/spinel-cli.py

@@ -57,6 +57,7 @@ import traceback
 
 import optparse
 
+import struct
 import string
 import textwrap
 
@@ -65,7 +66,6 @@ import logging.config
 import logging.handlers
 
 from cmd import Cmd
-from struct import pack
 
 from spinel.const import SPINEL
 from spinel.const import kThread
@@ -79,6 +79,7 @@ import ipaddress
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 from scapy.layers.inet6 import IPv6
+from scapy.layers.inet6 import ICMPv6EchoReply
 from scapy.layers.inet6 import ICMPv6EchoRequest
 
 
@@ -96,9 +97,12 @@ class SpinelCliCmd(Cmd, SpinelCodec):
 
     def __init__(self, stream_desc, nodeid, *_a, **kw):
 
+        self.nodeid = kw.get('nodeid', '1')
+
         self.wpan_api = WpanApi(stream_desc, nodeid)
         self.wpan_api.queue_register(SPINEL.HEADER_DEFAULT)
-        self.nodeid = kw.get('nodeid', '1')
+        self.wpan_api.callback_register(SPINEL.PROP_STREAM_NET,
+                                        self.wpan_callback)
 
         Cmd.__init__(self)
         Cmd.identchars = string.ascii_letters + string.digits + '-'
@@ -203,6 +207,26 @@ class SpinelCliCmd(Cmd, SpinelCodec):
 
     ]
 
+    @classmethod
+    def wpan_callback(cls, prop, value, tid):
+        consumed = False
+
+        if prop == SPINEL.PROP_STREAM_NET:
+            consumed = True
+
+            pkt = IPv6(value[2:])
+
+            if CONFIG.DEBUG_LOG_PKT:
+                pkt.show()
+
+            if ICMPv6EchoReply in pkt:
+                timenow = int(round(time.time() * 1000)) & 0xFFFFFFFF
+                timedelta = (timenow - struct.unpack('>I', pkt.data)[0])
+                print("\n%d bytes from %s: icmp_seq=%d hlim=%d time=%dms" % (
+                    pkt.plen, pkt.src, pkt.seq, pkt.hlim, timedelta))
+
+        return consumed
+
     @classmethod
     def log(cls, text):
         """ Common log handler. """
@@ -908,11 +932,11 @@ class SpinelCliCmd(Cmd, SpinelCodec):
         args = line.split(" ")
 
         if args[0] == "counter":
-            newline = line.replace("counter","")
+            newline = line.replace("counter", "")
             self.handle_property(newline, SPINEL.PROP_NET_KEY_SEQUENCE_COUNTER, 'L')
 
         elif args[0] == "guardtime":
-            newline = line.replace("guardtime","")
+            newline = line.replace("guardtime", "")
             self.handle_property(newline, SPINEL.PROP_NET_KEY_SWITCH_GUARDTIME, 'L')
 
     def do_leaderdata(self, line):
@@ -1028,7 +1052,7 @@ class SpinelCliCmd(Cmd, SpinelCodec):
             try:
                 # remap string state names to integer
                 line = map_arg_name[line]
-            except KeyError, _ex:
+            except KeyError:
                 print("Error")
                 return
 
@@ -1138,7 +1162,7 @@ class SpinelCliCmd(Cmd, SpinelCodec):
             ml64 = self.prop_get_value(SPINEL.PROP_IPV6_ML_ADDR)
             ml64 = str(ipaddress.IPv6Address(ml64))
             timenow = int(round(time.time() * 1000)) & 0xFFFFFFFF
-            timenow = pack('>I', timenow)
+            timenow = struct.pack('>I', timenow)
             ping_req = str(IPv6(src=ml64, dst=addr) /
                            ICMPv6EchoRequest() / timenow)
             self.wpan_api.ip_send(ping_req)
@@ -1625,13 +1649,13 @@ class SpinelCliCmd(Cmd, SpinelCodec):
                 rssi = int(params[2])
             except:
                 rssi = SPINEL.RSSI_OVERRIDE
-            arr += pack('b', rssi)
+            arr += struct.pack('b', rssi)
             value = self.prop_insert_value(SPINEL.PROP_MAC_WHITELIST, arr,
                                            str(len(arr)) + 's')
 
         elif params[0] == "remove":
             arr = util.hex_to_bytes(params[1])
-            arr += pack('b', SPINEL.RSSI_OVERRIDE)
+            arr += struct.pack('b', SPINEL.RSSI_OVERRIDE)
             value = self.prop_remove_value(SPINEL.PROP_MAC_WHITELIST, arr,
                                            str(len(arr)) + 's')
 

tools/spinel-cli/spinel/codec.py

@@ -29,13 +29,20 @@
 Module providing a Spienl coder / decoder class.
 """
 
+from __future__ import print_function
+
 import os
+import sys
 import time
 import logging
 import threading
 import traceback
 
-import Queue
+is_py2 = sys.version[0] == '2'
+if is_py2:
+    import Queue as Queue
+else:
+    import queue as Queue
 
 from struct import pack
 from struct import unpack
@@ -44,10 +51,6 @@ from collections import defaultdict
 
 import ipaddress
 
-logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
-from scapy.layers.inet6 import IPv6
-from scapy.layers.inet6 import ICMPv6EchoReply
-
 import spinel.util as util
 import spinel.config as CONFIG
 from spinel.const import kThread
@@ -161,8 +164,8 @@ class SpinelCodec(object):
         }
         try:
             return map_decode[spinel_format[0]](payload)
-        except KeyError, _ex:
-            print traceback.format_exc()
+        except KeyError:
+            print(traceback.format_exc())
             return None
 
     @classmethod
@@ -232,8 +235,8 @@ class SpinelCodec(object):
         }
         try:
             return map_encode[code](value)
-        except KeyError, _ex:
-            print traceback.format_exc()
+        except KeyError:
+            print(traceback.format_exc())
             return None
 
     def next_code(self, spinel_format):
@@ -400,7 +403,7 @@ class SpinelPropertyHandler(SpinelCodec):
         ipaddr_str = str(ipaddress.IPv6Address(prefix)) + \
             str(self.wpan_api.nodeid)
         if CONFIG.DEBUG_LOG_PROP:
-            print "\n>>>> new PREFIX add ipaddr: " + ipaddr_str
+            print("\n>>>> new PREFIX add ipaddr: " + ipaddr_str)
 
         valid = 1
         preferred = 1
@@ -443,10 +446,10 @@ class SpinelPropertyHandler(SpinelCodec):
             self.handle_ipaddr_insert(*prefix)
 
         if CONFIG.DEBUG_LOG_PROP:
-            print "\n========= PREFIX ============"
-            print "ipaddrs: " + str(self.autoAddresses)
-            print "slaac prefix set: " + str(slaacPrefixSet)
-            print "==============================\n"
+            print("\n========= PREFIX ============")
+            print("ipaddrs: " + str(self.autoAddresses))
+            print("slaac prefix set: " + str(slaacPrefixSet))
+            print("==============================\n")
 
         # ==> ipaddrs - query current addresses
         #
@@ -584,8 +587,6 @@ class SpinelCommandHandler(SpinelCodec):
                 if ((prop_id == SPINEL.PROP_STREAM_NET) or
                         (prop_id == SPINEL.PROP_STREAM_NET_INSECURE)):
                     logging.debug("PROP_VALUE_" + name + ": " + prop_name)
-                    pkt = IPv6(prop_value[2:])
-                    pkt.show()
 
                 elif prop_id == SPINEL.PROP_STREAM_DEBUG:
                     logging.debug("DEBUG: " + prop_value)
@@ -593,12 +594,12 @@ class SpinelCommandHandler(SpinelCodec):
             if wpan_api:
                 wpan_api.queue_add(prop_id, prop_value, tid)
             else:
-                print "no wpan_api"
+                print("no wpan_api")
 
         except Exception as _ex:
             prop_name = "Property Unknown"
             logging.info("\n%s (%i): ", prop_name, prop_id)
-            print traceback.format_exc()
+            print(traceback.format_exc())
 
     def PROP_VALUE_IS(self, wpan_api, payload, tid):
         self.handle_prop(wpan_api, "IS", payload, tid)
@@ -734,11 +735,12 @@ class WpanApi(SpinelCodec):
 
         # PARSER state
         self.rx_pkt = []
+        self.callback = defaultdict(list)  # Map prop_id to list of callbacks.
 
         # Fire up threads
         self._reader_alive = True
         self.tid_filter = set()
-        self.__queue_prop = defaultdict(Queue.Queue)
+        self.__queue_prop = defaultdict(Queue.Queue)  # Map tid to Queue.
         self.queue_register()
         self.__start_reader()
 
@@ -789,7 +791,7 @@ class WpanApi(SpinelCodec):
             handler(self, payload, tid)
 
         except Exception as _ex:
-            print traceback.format_exc()
+            print(traceback.format_exc())
             cmd_name = "CB_Unknown"
             logging.info("\n%s (%i): ", cmd_name, cmd_id)
 
@@ -821,6 +823,9 @@ class WpanApi(SpinelCodec):
             self.value = value
             self.tid = tid
 
+    def callback_register(self, prop, cb):
+        self.callback[prop].append(cb)
+
     def queue_register(self, tid=SPINEL.HEADER_DEFAULT):
         self.tid_filter.add(tid)
         return self.__queue_prop[tid]
@@ -829,15 +834,12 @@ class WpanApi(SpinelCodec):
         self.queue_clear(tid)
 
     def queue_add(self, prop, value, tid):
-        # Asynchronous handlers don't actually add to queue.
-        if prop == SPINEL.PROP_STREAM_NET:
-            pkt = IPv6(value[2:])
-            if ICMPv6EchoReply in pkt:
-                timenow = int(round(time.time() * 1000)) & 0xFFFFFFFF
-                timedelta = (timenow - unpack('>I', pkt.data)[0])
-                print "\n%d bytes from %s: icmp_seq=%d hlim=%d time=%dms" % (
-                    pkt.plen, pkt.src, pkt.seq, pkt.hlim, timedelta)
-            return
+        cb_list = self.callback[prop]
+
+        # Asynchronous handlers can consume message and not add to queue.
+        if len(cb_list) > 0:
+            consumed = cb_list[0](prop, value, tid)
+            if consumed: return
 
         if tid not in self.tid_filter:
             return
@@ -861,7 +863,7 @@ class WpanApi(SpinelCodec):
         if os.geteuid() == 0:
             self.tun_if = TunInterface(nodeid)
         else:
-            print "Warning: superuser required to start tun interface."
+            print("Warning: superuser required to start tun interface.")
 
     def if_down(self):
         if self.tun_if:
@@ -922,7 +924,7 @@ class WpanApi(SpinelCodec):
         if CONFIG.DEBUG_LOG_PROP:
             handler = SPINEL_PROP_DISPATCH[prop_id]
             prop_name = handler.__name__
-            print "PROP_VALUE_GET [tid=%d]: %s" % (tid & 0xF, prop_name)
+            print("PROP_VALUE_GET [tid=%d]: %s" % (tid & 0xF, prop_name))
         return self.__prop_change_value(SPINEL.CMD_PROP_VALUE_GET, prop_id,
                                         None, None, tid)
 
@@ -932,7 +934,7 @@ class WpanApi(SpinelCodec):
         if CONFIG.DEBUG_LOG_PROP:
             handler = SPINEL_PROP_DISPATCH[prop_id]
             prop_name = handler.__name__
-            print "PROP_VALUE_SET [tid=%d]: %s" % (tid & 0xF, prop_name)
+            print("PROP_VALUE_SET [tid=%d]: %s" % (tid & 0xF, prop_name))
         return self.__prop_change_value(SPINEL.CMD_PROP_VALUE_SET, prop_id,
                                         value, py_format, tid)
 
@@ -942,7 +944,7 @@ class WpanApi(SpinelCodec):
         if CONFIG.DEBUG_LOG_PROP:
             handler = SPINEL_PROP_DISPATCH[prop_id]
             prop_name = handler.__name__
-            print "PROP_VALUE_INSERT [tid=%d]: %s" % (tid & 0xF, prop_name)
+            print("PROP_VALUE_INSERT [tid=%d]: %s" % (tid & 0xF, prop_name))
         return self.__prop_change_value(SPINEL.CMD_PROP_VALUE_INSERT, prop_id,
                                         value, py_format, tid)
 
@@ -952,7 +954,7 @@ class WpanApi(SpinelCodec):
         if CONFIG.DEBUG_LOG_PROP:
             handler = SPINEL_PROP_DISPATCH[prop_id]
             prop_name = handler.__name__
-            print "PROP_VALUE_REMOVE [tid=%d]: %s" % (tid & 0xF, prop_name)
+            print("PROP_VALUE_REMOVE [tid=%d]: %s" % (tid & 0xF, prop_name))
         return self.__prop_change_value(SPINEL.CMD_PROP_VALUE_REMOVE, prop_id,
                                         value, py_format, tid)
 

tools/spinel-cli/spinel/config.py

@@ -110,4 +110,4 @@ def debug_set_level(level):
             DEBUG_STREAM_RX = 1
             DEBUG_STREAM_TX = 1
 
-    print "DEBUG_ENABLE = " + str(DEBUG_ENABLE)
+    print("DEBUG_ENABLE = " + str(DEBUG_ENABLE))

tools/spinel-cli/spinel/stream.py

@@ -31,6 +31,8 @@ Module providing a generic stream interface.
 Also includes adapter implementations for serial, socket, and pipes.
 """
 
+from __future__ import print_function
+
 import sys
 import logging
 import traceback
@@ -67,7 +69,7 @@ class StreamSerial(IStream):
             self.serial = serial.Serial(dev, baudrate)
         except:
             logging.error("Couldn't open " + dev)
-            print traceback.format_exc()
+            traceback.print_exc()
 
     def write(self, data):
         self.serial.write(data)
@@ -113,7 +115,7 @@ class StreamPipe(IStream):
                                          stderr=sys.stdout.fileno())
         except:
             logging.error("Couldn't open " + filename)
-            print traceback.format_exc()
+            traceback.print_exc()
 
     def write(self, data):
         if CONFIG.DEBUG_STREAM_TX:
@@ -151,21 +153,21 @@ def StreamOpen(stream_type, descriptor, verbose=True):
 
     if stream_type == 'p':
         if verbose:
-            print "Opening pipe to " + str(descriptor)
+            print("Opening pipe to " + str(descriptor))
         return StreamPipe(descriptor)
 
     elif stream_type == 's':
         port = int(descriptor)
         hostname = "localhost"
         if verbose:
-            print "Opening socket to " + hostname + ":" + str(port)
+            print("Opening socket to " + hostname + ":" + str(port))
         return StreamSocket(hostname, port)
 
     elif stream_type == 'u':
         dev = str(descriptor)
         baudrate = 115200
         if verbose:
-            print "Opening serial to " + dev + " @ " + str(baudrate)
+            print("Opening serial to " + dev + " @ " + str(baudrate))
         return StreamSerial(dev, baudrate)
 
     else:

tools/spinel-cli/spinel/test_codec.py

@@ -27,6 +27,7 @@
 #
 """ Unittest for spinel.codec module. """
 
+import time
 import unittest
 
 from spinel.const import SPINEL
@@ -69,3 +70,27 @@ class TestCodec(unittest.TestCase):
             # print "value "+util.hexify_str(value)
             # print "truth "+util.hexify_str(truth_value)
             self.failUnless(value == truth_value)
+
+    def cb_test_callback(self, prop, value, tid):
+        self.test_callback_pass = True
+
+    def test_callback(self):
+        """ Unit test of WpanApi.callback_register. """
+
+        vector = [
+            "800672340060000000000c3a40fe80000000000000020d6f00055715d3fddead00beef0000cd9bb7814c5619ea8100b0ca00000000267fc789"  # PROP_STREAM_NET
+        ]
+
+        mock_stream = MockStream({})
+        nodeid = 1
+        use_hdlc = False
+        wpan_api = WpanApi(mock_stream, nodeid, use_hdlc)
+
+        self.test_callback_pass = False
+        wpan_api.callback_register(SPINEL.PROP_STREAM_NET, self.cb_test_callback)
+
+        for pkt in vector:
+            mock_stream.write_child_hex(pkt)
+            time.sleep(0.1)
+
+        self.failUnless(self.test_callback_pass)

tools/spinel-cli/spinel/tun.py

@@ -27,6 +27,8 @@
 #
 """ Utility class for creating TUN network interfaces on Linux and OSX. """
 
+from __future__ import print_function
+
 import os
 import sys
 import fcntl
@@ -103,7 +105,7 @@ class TunInterface(object):
     def ping6(self, args):
         """ Ping an address. """
         cmd = 'ping6 ' + args
-        print cmd
+        print(cmd)
         self.command(cmd)
 
     def addr_add(self, addr):
@@ -137,7 +139,7 @@ class TunInterface(object):
                                       util.hexify_str(packet))
                     self.write(packet)
             except:
-                print traceback.format_exc()
+                traceback.print_exc()
                 break
 
         logging.info("TUN: exiting")