[sub-mac] redo security processing for every (re)transmission (#13093)

Retransmissions of frames containing time-dependent header Information
Elements (IEs), such as CSL or Time Sync, require updates to these
IEs to reflect the exact time of sending. If the frame counter is not
incremented for these retransmissions, it leads to nonce reuse in
AES-CCM encryption, which is a security vulnerability.

This commit addresses this issue by ensuring that every transmission
attempt (initial or retry) uses a fresh frame counter:
- Deferred security processing from `SubMac::Send()` to
  `SubMac::BeginTransmit()`.
- Upon retransmission in `SubMac::HandleTransmitDone()`, the frame is
  restored to plaintext via `TxFrame::DecryptTransmitAesCcm()` and
  security flags are cleared.
- This allows time-dependent IEs to be updated and a new frame counter
  to be assigned for every attempt.

Added a Nexus test case `retransmission_security` to verify that both
CSL and standard MAC retransmissions use incrementing frame counters
and updated CSL phases.

src/core/config/mac.h

@@ -382,6 +382,15 @@
 #define OPENTHREAD_CONFIG_MAC_SOFTWARE_RX_ON_WHEN_IDLE_ENABLE 0
 #endif
 
+/**
+ * @def OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+ *
+ * Define to 1 to enable software retransmission security logic.
+ */
+#ifndef OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+#define OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE 1
+#endif
+
 /**
  * @def OPENTHREAD_CONFIG_MAC_CSL_TRANSMITTER_ENABLE
  *

src/core/mac/mac_frame.cpp

@@ -41,7 +41,8 @@
 #include "common/log.hpp"
 #include "common/num_utils.hpp"
 #include "radio/trel_link.hpp"
-#if OPENTHREAD_FTD || OPENTHREAD_MTD || OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_SECURITY_ENABLE
+#if OPENTHREAD_FTD || OPENTHREAD_MTD || OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_SECURITY_ENABLE || \
+    (OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE)
 #include "crypto/aes_ccm.hpp"
 #endif
 
@@ -1404,6 +1405,39 @@ exit:
 #endif // OPENTHREAD_FTD || OPENTHREAD_MTD || OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_SECURITY_ENABLE
 }
 
+#if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+void TxFrame::DecryptTransmitAesCcm(const ExtAddress &aExtAddress)
+{
+    uint32_t       frameCounter = 0;
+    uint8_t        securityLevel;
+    uint8_t        nonce[Crypto::AesCcm::kNonceSize];
+    uint8_t        tagLength;
+    Crypto::AesCcm aesCcm;
+
+    VerifyOrExit(GetSecurityEnabled() && IsSecurityProcessed());
+
+    SuccessOrExit(GetSecurityLevel(securityLevel));
+    SuccessOrExit(GetFrameCounter(frameCounter));
+
+    Crypto::AesCcm::GenerateNonce(aExtAddress, frameCounter, securityLevel, nonce);
+
+    aesCcm.SetKey(GetAesKey());
+    tagLength = GetFooterLength() - GetFcsSize();
+
+    aesCcm.Init(GetHeaderLength(), GetPayloadLength(), tagLength, nonce, sizeof(nonce));
+    aesCcm.Header(GetHeader(), GetHeaderLength());
+    aesCcm.Payload(GetPayload(), GetPayload(), GetPayloadLength(), Crypto::AesCcm::kDecrypt);
+    // Note: We skip aesCcm.Finalize() checking because we are only decrypting back to plaintext,
+    // and we know the ciphertext was generated correctly by us previously.
+
+    SetIsSecurityProcessed(false);
+    SetIsHeaderUpdated(false);
+
+exit:
+    return;
+}
+#endif // OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+
 void TxFrame::GenerateImmAck(const RxFrame &aFrame, bool aIsFramePending)
 {
     uint16_t fcf = static_cast<uint16_t>(kTypeAck) | aFrame.GetVersion();

src/core/mac/mac_frame.hpp

@@ -626,6 +626,14 @@ public:
 #endif // OPENTHREAD_CONFIG_TIME_SYNC_ENABLE
 
 #if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT
+    /**
+     * Indicates whether the frame contains header IEs.
+     *
+     * @retval TRUE   The frame contains header IEs.
+     * @retval FALSE  The frame contains no header IEs.
+     */
+    bool HasHeaderIe(void) const { return FindHeaderIeIndex() != kInvalidIndex; }
+
     /**
      * Returns a pointer to the Header IE.
      *
@@ -1240,6 +1248,14 @@ public:
      */
     void ProcessTransmitAesCcm(const ExtAddress &aExtAddress);
 
+    /**
+     * Decrypts the frame which was previously encrypted.
+     *
+     * @param[in]  aExtAddress  A reference to the extended address, which will be used to generate nonce
+     *                          for AES CCM computation.
+     */
+    void DecryptTransmitAesCcm(const ExtAddress &aExtAddress);
+
     /**
      * Indicates whether or not the frame has security processed.
      *

src/core/mac/sub_mac.cpp

@@ -63,6 +63,10 @@ SubMac::SubMac(Instance &aInstance)
     mCslParentAccuracy.Init();
 #endif
 
+#if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && !OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+    // Assuming the platform must deal with the retransmission security correctly.
+    OT_ASSERT(mRadioCaps & OT_RADIO_CAPS_TRANSMIT_RETRIES);
+#endif
     Init();
 }
 
@@ -601,6 +605,15 @@ void SubMac::HandleTransmitDone(TxFrame &aFrame, RxFrame *aAckFrame, Error aErro
         mTransmitRetries++;
         aFrame.SetIsARetransmission(true);
 
+#if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+        if (aFrame.GetSecurityEnabled() && aFrame.IsSecurityProcessed() && aFrame.HasHeaderIe())
+        {
+            aFrame.DecryptTransmitAesCcm(GetExtAddress());
+        }
+
+        ProcessTransmitSecurity();
+#endif
+
 #if OPENTHREAD_CONFIG_MAC_ADD_DELAY_ON_NO_ACK_ERROR_BEFORE_RETRY
         if (aError == kErrorNoAck)
         {