From 6de8d778b784d2ef6af409e6b85d511cbc8fc3b5 Mon Sep 17 00:00:00 2001
From: Yakun Xu <xyk@google.com>
Date: Wed, 3 Jun 2020 12:32:00 +0800
Subject: [PATCH] [tests] skip running expected fail tests (#5038)

This commit skips running expected fail tests so that it will be easier
to find real failures and saves some time.
---
 .../thread-cert/Cert_5_3_10_AddressQuery.py   |  2 ++
 .../Cert_5_7_01_CoapDiagCommands_A.py         |  2 ++
 .../thread-cert/Cert_8_1_01_Commissioning.py  |  2 ++
 .../thread-cert/Cert_8_1_02_Commissioning.py  |  2 ++
 .../thread-cert/Cert_8_2_01_JoinerRouter.py   |  2 ++
 .../thread-cert/Cert_8_2_02_JoinerRouter.py   |  2 ++
 .../Cert_9_2_02_MGMTCommissionerSet.py        |  2 ++
 .../thread-cert/Cert_9_2_04_ActiveDataset.py  |  2 ++
 .../thread-cert/Cert_9_2_07_DelayTimer.py     |  2 ++
 .../Cert_9_2_08_PersistentDatasets.py         |  2 ++
 .../Cert_9_2_09_PendingPartition.py           |  2 ++
 .../Cert_9_2_10_PendingPartition.py           |  2 ++
 .../thread-cert/Cert_9_2_11_MasterKey.py      |  2 ++
 .../thread-cert/Cert_9_2_12_Announce.py       |  2 ++
 .../thread-cert/Cert_9_2_13_EnergyScan.py     |  2 ++
 .../thread-cert/Cert_9_2_14_PanIdQuery.py     |  2 ++
 .../Cert_9_2_15_PendingPartition.py           |  2 ++
 .../Cert_9_2_16_ActivePendingPartition.py     |  2 ++
 .../scripts/thread-cert/Cert_9_2_17_Orphan.py |  2 ++
 .../Cert_9_2_18_RollBackActiveTimestamp.py    |  2 ++
 tests/scripts/thread-cert/Makefile.am         | 32 -------------------
 .../scripts/thread-cert/test_coap_observe.py  |  3 ++
 tests/scripts/thread-cert/test_coaps.py       |  2 ++
 tests/scripts/thread-cert/test_diag.py        |  8 +++--
 .../thread-cert/test_ipv6_fragmentation.py    |  2 ++
 .../thread-cert/test_ipv6_source_selection.py |  2 ++
 .../test_reed_address_solicit_rejected.py     |  2 ++
 tests/scripts/thread-cert/test_service.py     |  2 ++
 tests/scripts/thread-cert/thread_cert.py      | 17 +++++++++-
 29 files changed, 74 insertions(+), 36 deletions(-)

diff --git a/tests/scripts/thread-cert/Cert_5_3_10_AddressQuery.py b/tests/scripts/thread-cert/Cert_5_3_10_AddressQuery.py
index fce8d3838..9ce42d467 100755
--- a/tests/scripts/thread-cert/Cert_5_3_10_AddressQuery.py
+++ b/tests/scripts/thread-cert/Cert_5_3_10_AddressQuery.py
@@ -43,6 +43,8 @@ MED1 = 5
 
 
 class Cert_5_3_10_AddressQuery(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/Cert_5_7_01_CoapDiagCommands_A.py b/tests/scripts/thread-cert/Cert_5_7_01_CoapDiagCommands_A.py
index 49862bb94..c243420d3 100755
--- a/tests/scripts/thread-cert/Cert_5_7_01_CoapDiagCommands_A.py
+++ b/tests/scripts/thread-cert/Cert_5_7_01_CoapDiagCommands_A.py
@@ -48,6 +48,8 @@ MTDS = [MED1, SED1]
 
 
 class Cert_5_7_01_CoapDiagCommands_A(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'whitelist': [ROUTER1],
diff --git a/tests/scripts/thread-cert/Cert_8_1_01_Commissioning.py b/tests/scripts/thread-cert/Cert_8_1_01_Commissioning.py
index 7c3416479..ae9c490f4 100755
--- a/tests/scripts/thread-cert/Cert_8_1_01_Commissioning.py
+++ b/tests/scripts/thread-cert/Cert_8_1_01_Commissioning.py
@@ -41,6 +41,8 @@ JOINER = 2
 
 
 class Cert_8_1_01_Commissioning(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'masterkey': '00112233445566778899aabbccddeeff',
diff --git a/tests/scripts/thread-cert/Cert_8_1_02_Commissioning.py b/tests/scripts/thread-cert/Cert_8_1_02_Commissioning.py
index 3c3357cdf..706d6a314 100755
--- a/tests/scripts/thread-cert/Cert_8_1_02_Commissioning.py
+++ b/tests/scripts/thread-cert/Cert_8_1_02_Commissioning.py
@@ -36,6 +36,8 @@ JOINER = 2
 
 
 class Cert_8_1_02_Commissioning(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'masterkey': 'deadbeefdeadbeefdeadbeefdeadbeef',
diff --git a/tests/scripts/thread-cert/Cert_8_2_01_JoinerRouter.py b/tests/scripts/thread-cert/Cert_8_2_01_JoinerRouter.py
index af87cfe91..830252b41 100755
--- a/tests/scripts/thread-cert/Cert_8_2_01_JoinerRouter.py
+++ b/tests/scripts/thread-cert/Cert_8_2_01_JoinerRouter.py
@@ -37,6 +37,8 @@ JOINER = 3
 
 
 class Cert_8_2_01_JoinerRouter(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'masterkey': 'deadbeefdeadbeefdeadbeefdeadbeef',
diff --git a/tests/scripts/thread-cert/Cert_8_2_02_JoinerRouter.py b/tests/scripts/thread-cert/Cert_8_2_02_JoinerRouter.py
index 710f42b1e..650ed18f2 100755
--- a/tests/scripts/thread-cert/Cert_8_2_02_JoinerRouter.py
+++ b/tests/scripts/thread-cert/Cert_8_2_02_JoinerRouter.py
@@ -37,6 +37,8 @@ JOINER = 3
 
 
 class Cert_8_2_02_JoinerRouter(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'masterkey': 'deadbeefdeadbeefdeadbeefdeadbeef',
diff --git a/tests/scripts/thread-cert/Cert_9_2_02_MGMTCommissionerSet.py b/tests/scripts/thread-cert/Cert_9_2_02_MGMTCommissionerSet.py
index 8023d0b3a..5f1335a68 100755
--- a/tests/scripts/thread-cert/Cert_9_2_02_MGMTCommissionerSet.py
+++ b/tests/scripts/thread-cert/Cert_9_2_02_MGMTCommissionerSet.py
@@ -40,6 +40,8 @@ LEADER = 2
 
 
 class Cert_9_2_02_MGMTCommissionerSet(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/Cert_9_2_04_ActiveDataset.py b/tests/scripts/thread-cert/Cert_9_2_04_ActiveDataset.py
index 99786bb2f..c1d1f9b7b 100755
--- a/tests/scripts/thread-cert/Cert_9_2_04_ActiveDataset.py
+++ b/tests/scripts/thread-cert/Cert_9_2_04_ActiveDataset.py
@@ -36,6 +36,8 @@ LEADER = 2
 
 
 class Cert_9_2_04_ActiveDataset(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_07_DelayTimer.py b/tests/scripts/thread-cert/Cert_9_2_07_DelayTimer.py
index c7129af1e..01f32a066 100755
--- a/tests/scripts/thread-cert/Cert_9_2_07_DelayTimer.py
+++ b/tests/scripts/thread-cert/Cert_9_2_07_DelayTimer.py
@@ -47,6 +47,8 @@ COMMISSIONER_PENDING_PANID = 0xafce
 
 
 class Cert_9_2_7_DelayTimer(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_08_PersistentDatasets.py b/tests/scripts/thread-cert/Cert_9_2_08_PersistentDatasets.py
index 587c2eb28..e394ca0fc 100755
--- a/tests/scripts/thread-cert/Cert_9_2_08_PersistentDatasets.py
+++ b/tests/scripts/thread-cert/Cert_9_2_08_PersistentDatasets.py
@@ -49,6 +49,8 @@ MTDS = [ED, SED]
 
 
 class Cert_9_2_8_PersistentDatasets(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_09_PendingPartition.py b/tests/scripts/thread-cert/Cert_9_2_09_PendingPartition.py
index 3b26b08ba..4f194b028 100755
--- a/tests/scripts/thread-cert/Cert_9_2_09_PendingPartition.py
+++ b/tests/scripts/thread-cert/Cert_9_2_09_PendingPartition.py
@@ -44,6 +44,8 @@ ROUTER2 = 4
 
 
 class Cert_9_2_09_PendingPartition(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_10_PendingPartition.py b/tests/scripts/thread-cert/Cert_9_2_10_PendingPartition.py
index bea8506b2..0dafa5434 100755
--- a/tests/scripts/thread-cert/Cert_9_2_10_PendingPartition.py
+++ b/tests/scripts/thread-cert/Cert_9_2_10_PendingPartition.py
@@ -48,6 +48,8 @@ MTDS = [ED1, SED1]
 
 
 class Cert_9_2_10_PendingPartition(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_11_MasterKey.py b/tests/scripts/thread-cert/Cert_9_2_11_MasterKey.py
index 6a22c0899..7d140f619 100755
--- a/tests/scripts/thread-cert/Cert_9_2_11_MasterKey.py
+++ b/tests/scripts/thread-cert/Cert_9_2_11_MasterKey.py
@@ -48,6 +48,8 @@ MTDS = [ED1, SED1]
 
 
 class Cert_9_2_11_MasterKey(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_12_Announce.py b/tests/scripts/thread-cert/Cert_9_2_12_Announce.py
index 52255351c..6680e72ec 100755
--- a/tests/scripts/thread-cert/Cert_9_2_12_Announce.py
+++ b/tests/scripts/thread-cert/Cert_9_2_12_Announce.py
@@ -47,6 +47,8 @@ DATASET2_PANID = 0xafce
 
 
 class Cert_9_2_12_Announce(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER1: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_13_EnergyScan.py b/tests/scripts/thread-cert/Cert_9_2_13_EnergyScan.py
index e0df5efb5..7d7153c9b 100755
--- a/tests/scripts/thread-cert/Cert_9_2_13_EnergyScan.py
+++ b/tests/scripts/thread-cert/Cert_9_2_13_EnergyScan.py
@@ -38,6 +38,8 @@ ED1 = 4
 
 
 class Cert_9_2_13_EnergyScan(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/Cert_9_2_14_PanIdQuery.py b/tests/scripts/thread-cert/Cert_9_2_14_PanIdQuery.py
index cb2b96348..ec2e2981a 100755
--- a/tests/scripts/thread-cert/Cert_9_2_14_PanIdQuery.py
+++ b/tests/scripts/thread-cert/Cert_9_2_14_PanIdQuery.py
@@ -38,6 +38,8 @@ LEADER2 = 4
 
 
 class Cert_9_2_14_PanIdQuery(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/Cert_9_2_15_PendingPartition.py b/tests/scripts/thread-cert/Cert_9_2_15_PendingPartition.py
index 385a5cbb5..d72661b3e 100755
--- a/tests/scripts/thread-cert/Cert_9_2_15_PendingPartition.py
+++ b/tests/scripts/thread-cert/Cert_9_2_15_PendingPartition.py
@@ -43,6 +43,8 @@ ROUTER2 = 4
 
 
 class Cert_9_2_15_PendingPartition(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_16_ActivePendingPartition.py b/tests/scripts/thread-cert/Cert_9_2_16_ActivePendingPartition.py
index 30ecdae80..72e645998 100755
--- a/tests/scripts/thread-cert/Cert_9_2_16_ActivePendingPartition.py
+++ b/tests/scripts/thread-cert/Cert_9_2_16_ActivePendingPartition.py
@@ -44,6 +44,8 @@ ROUTER2 = 4
 
 
 class Cert_9_2_16_ActivePendingPartition(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_17_Orphan.py b/tests/scripts/thread-cert/Cert_9_2_17_Orphan.py
index 83fd7ad89..1ccec7add 100755
--- a/tests/scripts/thread-cert/Cert_9_2_17_Orphan.py
+++ b/tests/scripts/thread-cert/Cert_9_2_17_Orphan.py
@@ -43,6 +43,8 @@ ED1 = 3
 
 
 class Cert_9_2_17_Orphan(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER1: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Cert_9_2_18_RollBackActiveTimestamp.py b/tests/scripts/thread-cert/Cert_9_2_18_RollBackActiveTimestamp.py
index 72b1de463..f4a469b74 100755
--- a/tests/scripts/thread-cert/Cert_9_2_18_RollBackActiveTimestamp.py
+++ b/tests/scripts/thread-cert/Cert_9_2_18_RollBackActiveTimestamp.py
@@ -49,6 +49,8 @@ MTDS = [ED1, SED1]
 
 
 class Cert_9_2_18_RollBackActiveTimestamp(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         COMMISSIONER: {
             'active_dataset': {
diff --git a/tests/scripts/thread-cert/Makefile.am b/tests/scripts/thread-cert/Makefile.am
index e355c999c..63f34e62d 100644
--- a/tests/scripts/thread-cert/Makefile.am
+++ b/tests/scripts/thread-cert/Makefile.am
@@ -291,36 +291,4 @@ TESTS                                                              = \
     $(check_SCRIPTS)                                                 \
     $(NULL)
 
-XFAIL_NCP_TESTS                                                    = \
-    test_coaps.py                                                    \
-    test_coap_observe.py                                             \
-    test_diag.py                                                     \
-    test_ipv6_fragmentation.py                                       \
-    test_ipv6_source_selection.py                                    \
-    test_reed_address_solicit_rejected.py                            \
-    test_service.py                                                  \
-    Cert_5_3_10_AddressQuery.py                                      \
-    Cert_5_7_01_CoapDiagCommands_A.py                                \
-    Cert_8_1_01_Commissioning.py                                     \
-    Cert_8_1_02_Commissioning.py                                     \
-    Cert_8_2_01_JoinerRouter.py                                      \
-    Cert_8_2_02_JoinerRouter.py                                      \
-    Cert_9_2_02_MGMTCommissionerSet.py                               \
-    Cert_9_2_04_ActiveDataset.py                                     \
-    Cert_9_2_07_DelayTimer.py                                        \
-    Cert_9_2_08_PersistentDatasets.py                                \
-    Cert_9_2_09_PendingPartition.py                                  \
-    Cert_9_2_10_PendingPartition.py                                  \
-    Cert_9_2_11_MasterKey.py                                         \
-    Cert_9_2_12_Announce.py                                          \
-    Cert_9_2_13_EnergyScan.py                                        \
-    Cert_9_2_14_PanIdQuery.py                                        \
-    Cert_9_2_15_PendingPartition.py                                  \
-    Cert_9_2_16_ActivePendingPartition.py                            \
-    Cert_9_2_17_Orphan.py                                            \
-    Cert_9_2_18_RollBackActiveTimestamp.py                           \
-    $(NULL)
-
-XFAIL_TESTS = $(if $(filter $(NODE_TYPE),ncp-sim),$(XFAIL_NCP_TESTS))
-
 include $(abs_top_nlbuild_autotools_dir)/automake/post.am
diff --git a/tests/scripts/thread-cert/test_coap_observe.py b/tests/scripts/thread-cert/test_coap_observe.py
index 8edf4355b..f93c4f825 100755
--- a/tests/scripts/thread-cert/test_coap_observe.py
+++ b/tests/scripts/thread-cert/test_coap_observe.py
@@ -41,6 +41,9 @@ class TestCoapObserve(thread_cert.TestCase):
     """
     Test suite for CoAP Observations (RFC7641).
     """
+
+    support_ncp = False
+
     topology = {
         LEADER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/test_coaps.py b/tests/scripts/thread-cert/test_coaps.py
index d71e9c596..8dbe85d9e 100755
--- a/tests/scripts/thread-cert/test_coaps.py
+++ b/tests/scripts/thread-cert/test_coaps.py
@@ -37,6 +37,8 @@ ROUTER = 2
 
 
 class TestCoaps(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/test_diag.py b/tests/scripts/thread-cert/test_diag.py
index ae5b42f97..053f23752 100755
--- a/tests/scripts/thread-cert/test_diag.py
+++ b/tests/scripts/thread-cert/test_diag.py
@@ -34,10 +34,12 @@ import thread_cert
 
 
 class TestDiag(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {1: None}
 
     def test(self):
-        self.node = self.nodes[1]
+        node = self.nodes[1]
 
         cases = [
             ('diag\n', 'diagnostics mode is disabled\r\n'),
@@ -88,11 +90,11 @@ class TestDiag(thread_cert.TestCase):
         ]
 
         for case in cases:
-            self.node.send_command(case[0])
+            node.send_command(case[0])
             self.simulator.go(1)
             if type(self.simulator).__name__ == 'VirtualTime':
                 time.sleep(0.1)
-            self.node._expect(case[1])
+            node._expect(case[1])
 
 
 if __name__ == '__main__':
diff --git a/tests/scripts/thread-cert/test_ipv6_fragmentation.py b/tests/scripts/thread-cert/test_ipv6_fragmentation.py
index 00506d205..f9e830397 100755
--- a/tests/scripts/thread-cert/test_ipv6_fragmentation.py
+++ b/tests/scripts/thread-cert/test_ipv6_fragmentation.py
@@ -38,6 +38,8 @@ ROUTER = 2
 
 
 class TestIPv6Fragmentation(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/test_ipv6_source_selection.py b/tests/scripts/thread-cert/test_ipv6_source_selection.py
index ee59d9629..39c94c205 100755
--- a/tests/scripts/thread-cert/test_ipv6_source_selection.py
+++ b/tests/scripts/thread-cert/test_ipv6_source_selection.py
@@ -37,6 +37,8 @@ ROUTER = 2
 
 
 class TestIPv6SourceSelection(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/test_reed_address_solicit_rejected.py b/tests/scripts/thread-cert/test_reed_address_solicit_rejected.py
index 713c44dc9..b5f776291 100755
--- a/tests/scripts/thread-cert/test_reed_address_solicit_rejected.py
+++ b/tests/scripts/thread-cert/test_reed_address_solicit_rejected.py
@@ -47,6 +47,8 @@ SRV_0_SERVER_DATA = 'bar'
 
 
 class TestREEDAddressSolicitRejected(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'mode': 'rsdn',
diff --git a/tests/scripts/thread-cert/test_service.py b/tests/scripts/thread-cert/test_service.py
index c1c69e7bb..d5d39e18f 100755
--- a/tests/scripts/thread-cert/test_service.py
+++ b/tests/scripts/thread-cert/test_service.py
@@ -49,6 +49,8 @@ SRV_1_SERVER_DATA = 'qux'
 
 
 class Test_Service(thread_cert.TestCase):
+    support_ncp = False
+
     topology = {
         LEADER: {
             'channel': 12,
diff --git a/tests/scripts/thread-cert/thread_cert.py b/tests/scripts/thread-cert/thread_cert.py
index 7604384f0..f467998ea 100644
--- a/tests/scripts/thread-cert/thread_cert.py
+++ b/tests/scripts/thread-cert/thread_cert.py
@@ -28,6 +28,7 @@
 #
 
 import os
+import sys
 import unittest
 
 import config
@@ -50,7 +51,21 @@ EXTENDED_ADDRESS_BASE = 0x166e0a0000000000
 """Extended address base to keep U/L bit 1. The value is borrowed from Thread Test Harness."""
 
 
-class TestCase(unittest.TestCase):
+class NcpSupportMixin():
+    """ The mixin to check whether a test case supports NCP.
+    """
+
+    support_ncp = True
+
+    def __init__(self, *args, **kwargs):
+        if os.getenv('NODE_TYPE', 'sim') == 'ncp-sim' and not self.support_ncp:
+            # 77 means skip this test case in automake tests
+            sys.exit(77)
+
+        super().__init__(*args, **kwargs)
+
+
+class TestCase(NcpSupportMixin, unittest.TestCase):
     """The base class for all thread certification test cases.
 
     The `topology` member of sub-class is used to create test topology.