[link-raw] do not disable link raw during transmit/scan operation (#6026)

src/core/mac/link_raw.cpp

@@ -66,9 +66,10 @@ LinkRaw::LinkRaw(Instance &aInstance)
 
 otError LinkRaw::SetReceiveDone(otLinkRawReceiveDone aCallback)
 {
-    otError error = OT_ERROR_NONE;
+    otError error  = OT_ERROR_NONE;
+    bool    enable = aCallback != nullptr;
 
-    otLogDebgMac("LinkRaw::Enabled(%s)", (aCallback != nullptr ? "true" : "false"));
+    otLogDebgMac("LinkRaw::Enabled(%s)", (enable ? "true" : "false"));
 
 #if OPENTHREAD_MTD || OPENTHREAD_FTD
     VerifyOrExit(!Get<ThreadNetif>().IsUp(), error = OT_ERROR_INVALID_STATE);
@@ -78,9 +79,17 @@ otError LinkRaw::SetReceiveDone(otLinkRawReceiveDone aCallback)
     // avoid any conflict in control of radio and sub-mac between `Mac` and
     // `LinkRaw`. in RADIO build, we directly enable/disable sub-mac.
 
-    Get<Mac>().SetEnabled(aCallback == nullptr);
+    if (!enable)
+    {
+        // When disabling link-raw, make sure there is no ongoing
+        // transmit or scan operation. Otherwise Mac will attempt to
+        // handle an unexpected "done" callback.
+        VerifyOrExit(!mSubMac.IsTransmittingOrScanning(), error = OT_ERROR_BUSY);
+    }
+
+    Get<Mac>().SetEnabled(!enable);
 #else
-    if (aCallback)
+    if (enable)
     {
         SuccessOrExit(error = mSubMac.Enable());
     }

src/core/mac/sub_mac.hpp

@@ -287,6 +287,15 @@ public:
      */
     otError Sleep(void);
 
+    /**
+     * This method indicates whether the sub-mac is busy transmitting or scanning.
+     *
+     * @retval TRUE if the sub-mac is busy transmitting or scanning.
+     * @retval FALSE if the sub-mac is not busy transmitting or scanning.
+     *
+     */
+    bool IsTransmittingOrScanning(void) const { return (mState == kStateTransmit) || (mState == kStateEnergyScan); }
+
     /**
      * This method transitions the radio to Receive.
      *