[mesh-forwarder] remove message if no pending tx in SendMessage() (#9495)

This commit updates `MeshForwarder::SendMessage()` on FTD to check if
the message is marked for direct transmission and/or indirect
transmission to a sleepy child. If there is no pending transmission,
the message is removed.

This situation can occur if the message destination is a multicast
address larger than realm-local scope. In such a case, `SendMessage()`
skips `SetDirectTransmission()` on the message(since such message
will be forwarded using IP-in-IP encapsulation by `Ip6` module) and
assumes the message is for a sleepy child. However, if none of the
children are subscribed to this address, the message will not be
marked for indirect transmission either. Without the fix in this
commit, such messages would have remained in the `mSendQueue` and not
been removed or freed, as messages are only checked for removal after
a direct or indirect transmission attempt.

src/core/thread/mesh_forwarder.cpp

@@ -1320,8 +1320,10 @@ exit:
     mScheduleTransmissionTask.Post();
 }
 
-void MeshForwarder::RemoveMessageIfNoPendingTx(Message &aMessage)
+bool MeshForwarder::RemoveMessageIfNoPendingTx(Message &aMessage)
 {
+    bool didRemove = false;
+
 #if OPENTHREAD_FTD
     VerifyOrExit(!aMessage.IsDirectTransmission() && !aMessage.IsChildPending());
 #else
@@ -1335,9 +1337,10 @@ void MeshForwarder::RemoveMessageIfNoPendingTx(Message &aMessage)
     }
 
     mSendQueue.DequeueAndFree(aMessage);
+    didRemove = true;
 
 exit:
-    return;
+    return didRemove;
 }
 
 void MeshForwarder::HandleReceivedFrame(Mac::RxFrame &aFrame)

src/core/thread/mesh_forwarder.hpp

@@ -567,7 +567,7 @@ private:
     void UpdateNeighborLinkFailures(Neighbor &aNeighbor, Error aError, bool aAllowNeighborRemove, uint8_t aFailLimit);
     void HandleSentFrame(Mac::TxFrame &aFrame, Error aError);
     void UpdateSendMessage(Error aFrameTxError, Mac::Address &aMacDest, Neighbor *aNeighbor);
-    void RemoveMessageIfNoPendingTx(Message &aMessage);
+    bool RemoveMessageIfNoPendingTx(Message &aMessage);
 
     void HandleTimeTick(void);
     void ScheduleTransmissionTask(void);

src/core/thread/mesh_forwarder_ftd.cpp

@@ -135,12 +135,21 @@ Error MeshForwarder::SendMessage(Message &aMessage)
         break;
     }
 
+    // Ensure that the message is marked for direct tx and/or for indirect tx
+    // to a sleepy child. Otherwise, remove the message.
+
+    if (RemoveMessageIfNoPendingTx(aMessage))
+    {
+        ExitNow();
+    }
+
 #if (OPENTHREAD_CONFIG_MAX_FRAMES_IN_DIRECT_TX_QUEUE > 0)
     ApplyDirectTxQueueLimit(aMessage);
 #endif
 
     mScheduleTransmissionTask.Post();
 
+exit:
     return error;
 }