[border-agent] improve DTLS session resource management (#13078)

Every DTLS ClientHello from an unseen port previously allocated a
dynamic CoapDtlsSession on the heap before DTLS cookie verification.
This allowed multiple connection attempts to leave allocated sessions
active indefinitely, leading to high memory utilization.

To resolve this:
- Enforce a 15-second handshake timeout on newly allocated sessions.
  Connecting sessions that do not successfully finish the handshake
  within 15 seconds are cleanly disconnected and freed.
- Enforce a session limit cap of 16 concurrent secure sessions on the
  Border Agent. Reaching this limit immediately rejects new session
  connection requests before triggering heap allocation.
- Implement Nexus test case TestBorderAgentSessionsLimit to robustly
  verify both session limit rejection and handshake timeout behavior.

src/core/meshcop/border_agent.cpp

@@ -35,6 +35,7 @@
 
 #if OPENTHREAD_CONFIG_BORDER_AGENT_ENABLE
 
+#include "common/num_utils.hpp"
 #include "instance/instance.hpp"
 #include "meshcop/border_agent_txt_data.hpp"
 
@@ -257,7 +258,18 @@ SecureSession *Manager::HandleAcceptSession(void *aContext, const Ip6::MessageIn
 
 Manager::CoapDtlsSession *Manager::HandleAcceptSession(void)
 {
-    return CoapDtlsSession::Allocate(GetInstance(), mDtlsTransport);
+    CoapDtlsSession *session = nullptr;
+
+    if (mDtlsTransport.GetSessions().CountAllEntries() >= kMaxSessions)
+    {
+        LogWarn("Accept session failed: reached max concurrent secure sessions limit (%lu)", ToUlong(kMaxSessions));
+        ExitNow();
+    }
+
+    session = CoapDtlsSession::Allocate(GetInstance(), mDtlsTransport);
+
+exit:
+    return session;
 }
 
 void Manager::HandleRemoveSession(void *aContext, SecureSession &aSession)
@@ -579,7 +591,8 @@ Manager::CoapDtlsSession::CoapDtlsSession(Instance &aInstance, Dtls::Transport &
     SetResourceHandler(&HandleResource);
     SetConnectCallback(&HandleConnected, this);
 
-    LogInfo("Allocating session %u", mIndex);
+    LogInfo("Allocating session %u - starting handshake timer for %lu ms", mIndex, ToUlong(kHandshakeTimeout));
+    mTimer.Start(kHandshakeTimeout);
 }
 
 Error Manager::CoapDtlsSession::SendMessage(OwnedPtr<Coap::Message> aMessage)
@@ -1149,8 +1162,13 @@ void Manager::CoapDtlsSession::HandleTimer(void)
         ResignEnroller();
 #endif
         LogInfo("Session %u timed out - disconnecting", mIndex);
-        DisconnectTimeout();
     }
+    else
+    {
+        LogInfo("Session %u handshake timeout - disconnecting", mIndex);
+    }
+
+    DisconnectTimeout();
 }
 
 void Manager::CoapDtlsSession::CopyInfoTo(SessionInfo &aInfo, UptimeMsec aUptimeNow) const

src/core/meshcop/border_agent.hpp

@@ -264,6 +264,8 @@ public:
 private:
     static constexpr uint16_t kUdpPort          = OPENTHREAD_CONFIG_BORDER_AGENT_UDP_PORT;
     static constexpr uint32_t kKeepAliveTimeout = 50 * 1000; // Timeout to reject a commissioner (in msec)
+    static constexpr uint32_t kHandshakeTimeout = 15 * 1000; // Handshake timeout (in msec)
+    static constexpr uint32_t kMaxSessions      = 16;        // Max concurrent secure sessions
 
 #if OPENTHREAD_CONFIG_BORDER_AGENT_MESHCOP_SERVICE_ENABLE
     static constexpr uint16_t kDummyUdpPort          = 49152;

tests/nexus/test_border_agent.cpp

@@ -2574,6 +2574,137 @@ void TestBorderAgentServiceRegistrationRename(void)
     node1.Get<Dns::Multicast::Core>().FreeIterator(*iterator);
 }
 
+void TestBorderAgentSessionsLimit(void)
+{
+    Core                     nexus;
+    Node                    &node0 = nexus.CreateNode();
+    Node                    *nodes[20];
+    Ip6::SockAddr            sockAddr;
+    Pskc                     pskc;
+    Manager::SessionIterator iter;
+    Manager::SessionInfo     sessionInfo;
+    uint8_t                  sessionCount;
+
+    Log("------------------------------------------------------------------------------------------------------");
+    Log("TestBorderAgentSessionsLimit");
+
+    nexus.AdvanceTime(0);
+    SuccessOrQuit(node0.SetLogLevel(kLogLevelInfo));
+
+    node0.Form();
+
+    nexus.AdvanceTime(50 * Time::kOneSecondInMsec);
+    VerifyOrQuit(node0.Get<Mle::Mle>().IsLeader());
+
+    for (uint8_t i = 0; i < 20; i++)
+    {
+        nodes[i] = &nexus.CreateNode();
+        SuccessOrQuit(nodes[i]->Get<Mac::Mac>().SetPanChannel(node0.Get<Mac::Mac>().GetPanChannel()));
+        nodes[i]->Get<Mac::Mac>().SetPanId(node0.Get<Mac::Mac>().GetPanId());
+        nodes[i]->Get<ThreadNetif>().Up();
+    }
+
+    VerifyOrQuit(node0.Get<Manager>().IsEnabled());
+    VerifyOrQuit(node0.Get<Manager>().IsRunning());
+
+    SuccessOrQuit(node0.Get<Ip6::Filter>().AddUnsecurePort(node0.Get<Manager>().GetUdpPort()));
+
+    sockAddr.SetAddress(node0.Get<Mle::Mle>().GetLinkLocalAddress());
+    sockAddr.SetPort(node0.Get<Manager>().GetUdpPort());
+    node0.Get<KeyManager>().GetPskc(pskc);
+
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    Log("Establish 16 concurrent secure sessions (the limit)");
+
+    for (uint8_t i = 0; i < 16; i++)
+    {
+        SuccessOrQuit(nodes[i]->Get<Tmf::SecureAgent>().SetPsk(pskc.m8, Pskc::kSize));
+        SuccessOrQuit(nodes[i]->Get<Tmf::SecureAgent>().Open(0));
+        SuccessOrQuit(nodes[i]->Get<Tmf::SecureAgent>().Connect(sockAddr));
+    }
+
+    nexus.AdvanceTime(2 * Time::kOneSecondInMsec);
+
+    for (uint8_t i = 0; i < 16; i++)
+    {
+        VerifyOrQuit(nodes[i]->Get<Tmf::SecureAgent>().IsConnected());
+    }
+
+    VerifyOrQuit(node0.Get<Manager>().GetCounters().mPskcSecureSessionSuccesses == 16);
+
+    // Verify exactly 16 sessions are connected
+    sessionCount = 0;
+    iter.Init(node0.GetInstance());
+    while (iter.GetNextSessionInfo(sessionInfo) == kErrorNone)
+    {
+        VerifyOrQuit(sessionInfo.mIsConnected);
+        sessionCount++;
+    }
+    VerifyOrQuit(sessionCount == 16);
+
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    Log("Try to establish a 17th secure session, should be rejected");
+
+    SuccessOrQuit(nodes[16]->Get<Tmf::SecureAgent>().SetPsk(pskc.m8, Pskc::kSize));
+    SuccessOrQuit(nodes[16]->Get<Tmf::SecureAgent>().Open(0));
+    SuccessOrQuit(nodes[16]->Get<Tmf::SecureAgent>().Connect(sockAddr));
+
+    nexus.AdvanceTime(2 * Time::kOneSecondInMsec);
+
+    VerifyOrQuit(!nodes[16]->Get<Tmf::SecureAgent>().IsConnected());
+    VerifyOrQuit(node0.Get<Manager>().GetCounters().mPskcSecureSessionSuccesses == 16);
+
+    // Verify count remains 16
+    sessionCount = 0;
+    iter.Init(node0.GetInstance());
+    while (iter.GetNextSessionInfo(sessionInfo) == kErrorNone)
+    {
+        sessionCount++;
+    }
+    VerifyOrQuit(sessionCount == 16);
+
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    Log("Close all sessions to clean up");
+
+    for (uint8_t i = 0; i < 17; i++)
+    {
+        nodes[i]->Get<Tmf::SecureAgent>().Close();
+    }
+
+    nexus.AdvanceTime(5 * Time::kOneSecondInMsec);
+
+    iter.Init(node0.GetInstance());
+    VerifyOrQuit(iter.GetNextSessionInfo(sessionInfo) == kErrorNotFound);
+
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    Log("Verify Handshake Timeout: start a session but block the client right after first packet");
+
+    SuccessOrQuit(nodes[0]->Get<Tmf::SecureAgent>().SetPsk(pskc.m8, Pskc::kSize));
+    SuccessOrQuit(nodes[0]->Get<Tmf::SecureAgent>().Open(0));
+    SuccessOrQuit(nodes[0]->Get<Tmf::SecureAgent>().Connect(sockAddr));
+
+    // Drop client's interface immediately so it never receives HelloVerifyRequest
+    nodes[0]->Get<ThreadNetif>().Down();
+
+    // Wait for 14 seconds (handshake timeout is 15 seconds)
+    nexus.AdvanceTime(14 * Time::kOneSecondInMsec);
+
+    // The session should still be in connecting state on node0
+    iter.Init(node0.GetInstance());
+    SuccessOrQuit(iter.GetNextSessionInfo(sessionInfo));
+    VerifyOrQuit(!sessionInfo.mIsConnected);
+    VerifyOrQuit(iter.GetNextSessionInfo(sessionInfo) == kErrorNotFound);
+
+    // Wait for additional 4 seconds (total 18 seconds, past the 15-second timeout + 2-second guard time)
+    nexus.AdvanceTime(4 * Time::kOneSecondInMsec);
+
+    // The session should have timed out and been removed
+    iter.Init(node0.GetInstance());
+    VerifyOrQuit(iter.GetNextSessionInfo(sessionInfo) == kErrorNotFound);
+
+    Log("TestBorderAgentSessionsLimit passed successfully!");
+}
+
 } // namespace Nexus
 } // namespace ot
 
@@ -2586,6 +2717,7 @@ int main(void)
     ot::Nexus::TestBorderAgentTxtDataCallback();
     ot::Nexus::TestBorderAgentServiceRegistration();
     ot::Nexus::TestBorderAgentServiceRegistrationRename();
+    ot::Nexus::TestBorderAgentSessionsLimit();
     printf("All tests passed\n");
     return 0;
 }