[tcplp] add support for TCP Fast Open (without cookie management) (#9165)

This commit adds support for TCP Fast Open, without cookie management.

To add support for this, I looked at the FreeBSD codebase and brought
in some code from FreeBSD 12.0 that implements TCP Fast Open --- the
version of FreeBSD that TCPlp is based on did not fully support TCP
Fast Open.

Normally, a part of TFO is cookie management --- the server generates
a cookie and includes it in the initial handshake, and client is
expected to present this cookie on future handshakes. This part is not
yet implemented, and I changed the logic from FreeBSD to allow data to
be exchanged in the TFO handshake even if the client does not present
a cookie. If we implement this functionality for TFO later, it is
probably worth departing from FreeBSD's data structures and policies
for maintaining cookie state in favor of something that is simpler and
more memory-efficient.

include/openthread/instance.h

@@ -53,7 +53,7 @@ extern "C" {
  * @note This number versions both OpenThread platform and user APIs.
  *
  */
-#define OPENTHREAD_API_VERSION (350)
+#define OPENTHREAD_API_VERSION (351)
 
 /**
  * @addtogroup api-instance

include/openthread/tcp.h

@@ -225,7 +225,7 @@ typedef void (*otTcpDisconnected)(otTcpEndpoint *aEndpoint, otTcpDisconnectedRea
  * opaque in its declaration, is treated as struct tcpcb in the TCP
  * implementation.
  */
-#define OT_TCP_ENDPOINT_TCB_SIZE_BASE 368
+#define OT_TCP_ENDPOINT_TCB_SIZE_BASE 392
 #define OT_TCP_ENDPOINT_TCB_NUM_PTR 36
 
 /**
@@ -401,11 +401,16 @@ enum
 /**
  * Records the remote host and port for this connection.
  *
- * Caller must wait for `otTcpEstablished` callback indicating that TCP
- * connection establishment handshake is done before it can start sending data
- * e.g., calling `otTcpSendByReference()`.
+ * TCP Fast Open must be enabled or disabled using @p aFlags. If it is
+ * disabled, then the TCP connection establishment handshake is initiated
+ * immediately. If it is enabled, then this function merely records the
+ * the remote host and port, and the TCP connection establishment handshake
+ * only happens on the first call to `otTcpSendByReference()`.
  *
- * The TCP Fast Open is not yet supported and @p aFlags is ignored.
+ * If TCP Fast Open is disabled, then the caller must wait for the
+ * `otTcpEstablished` callback indicating that TCP connection establishment
+ * handshake is done before it can start sending data e.g., by calling
+ * `otTcpSendByReference()`.
  *
  * @param[in]  aEndpoint  A pointer to the TCP endpoint structure to connect.
  * @param[in]  aSockName  The IP address and port of the host to which to connect.

src/cli/README_TCP.md

@@ -58,7 +58,7 @@ For a more in-depth example, see [this video](https://youtu.be/ppZ784YUKlI).
 - [init](#init-size)
 - [deinit](#deinit)
 - [bind](#bind-ip-port)
-- [connect](#connect-ip-port)
+- [connect](#connect-ip-port-fastopen)
 - [send](#send-message)
 - [benchmark](#benchmark-run-size)
 - [sendend](#sendend)
@@ -118,7 +118,7 @@ Associates a name (i.e. IPv6 address and port) to the example TCP endpoint.
 Done
 ```
 
-### connect \<ip\> \<port\>
+### connect \<ip\> \<port\> [\<fastopen\>]
 
 Establishes a connection with the specified peer.
 
@@ -126,6 +126,7 @@ If the connection establishment is successful, the resulting TCP connection is a
 
 - ip: the peer's IP address.
 - port: the peer's TCP port.
+- fastopen: if "fast", TCP Fast Open is enabled for this connection; if "slow", it is not. Defaults to "slow".
 
 ```bash
 > tcp connect fe80:0:0:0:a8df:580a:860:ffa4 30000

src/cli/cli_tcp.cpp

@@ -64,6 +64,7 @@ TcpExample::TcpExample(otInstance *aInstance, OutputImplementer &aOutputImplemen
     : Output(aInstance, aOutputImplementer)
     , mInitialized(false)
     , mEndpointConnected(false)
+    , mEndpointConnectedFastOpen(false)
     , mSendBusy(false)
     , mUseCircularSendBuffer(true)
     , mUseTls(false)
@@ -294,6 +295,7 @@ template <> otError TcpExample::Process<Cmd("connect")>(Arg aArgs[])
     otError    error;
     otSockAddr sockaddr;
     bool       nat64SynthesizedAddress;
+    uint32_t   flags;
 
     VerifyOrExit(mInitialized, error = OT_ERROR_INVALID_STATE);
 
@@ -306,7 +308,26 @@ template <> otError TcpExample::Process<Cmd("connect")>(Arg aArgs[])
     }
 
     SuccessOrExit(error = aArgs[1].ParseAsUint16(sockaddr.mPort));
-    VerifyOrExit(aArgs[2].IsEmpty(), error = OT_ERROR_INVALID_ARGS);
+    if (aArgs[2].IsEmpty())
+    {
+        flags = OT_TCP_CONNECT_NO_FAST_OPEN;
+    }
+    else
+    {
+        if (aArgs[2] == "slow")
+        {
+            flags = OT_TCP_CONNECT_NO_FAST_OPEN;
+        }
+        else if (aArgs[2] == "fast")
+        {
+            flags = 0;
+        }
+        else
+        {
+            ExitNow(error = OT_ERROR_INVALID_ARGS);
+        }
+        VerifyOrExit(aArgs[3].IsEmpty(), error = OT_ERROR_INVALID_ARGS);
+    }
 
 #if OPENTHREAD_CONFIG_TLS_ENABLE
     if (mUseTls)
@@ -320,8 +341,17 @@ template <> otError TcpExample::Process<Cmd("connect")>(Arg aArgs[])
     }
 #endif // OPENTHREAD_CONFIG_TLS_ENABLE
 
-    SuccessOrExit(error = otTcpConnect(&mEndpoint, &sockaddr, OT_TCP_CONNECT_NO_FAST_OPEN));
-    mEndpointConnected = true;
+    SuccessOrExit(error = otTcpConnect(&mEndpoint, &sockaddr, flags));
+    mEndpointConnected         = true;
+    mEndpointConnectedFastOpen = ((flags & OT_TCP_CONNECT_NO_FAST_OPEN) == 0);
+
+#if OPENTHREAD_CONFIG_TLS_ENABLE
+    if (mUseTls && mEndpointConnectedFastOpen)
+    {
+        PrepareTlsHandshake();
+        ContinueTlsHandshake();
+    }
+#endif
 
 exit:
     return error;
@@ -472,7 +502,8 @@ template <> otError TcpExample::Process<Cmd("abort")>(Arg aArgs[])
     VerifyOrExit(mInitialized, error = OT_ERROR_INVALID_STATE);
 
     SuccessOrExit(error = otTcpAbort(&mEndpoint));
-    mEndpointConnected = false;
+    mEndpointConnected         = false;
+    mEndpointConnectedFastOpen = false;
 
 exit:
     return error;
@@ -591,24 +622,10 @@ void TcpExample::HandleTcpEstablished(otTcpEndpoint *aEndpoint)
     OT_UNUSED_VARIABLE(aEndpoint);
     OutputLine("TCP: Connection established");
 #if OPENTHREAD_CONFIG_TLS_ENABLE
-    if (mUseTls)
+    if (mUseTls && !mEndpointConnectedFastOpen)
     {
-        int rv;
-        rv = mbedtls_ssl_set_hostname(&mSslContext, "localhost");
-        if (rv != 0)
-        {
-            OutputLine("mbedtls_ssl_set_hostname returned %d", rv);
-        }
-        rv = mbedtls_ssl_set_hs_ecjpake_password(
-            &mSslContext, reinterpret_cast<const unsigned char *>(sEcjpakePassword), sEcjpakePasswordLength);
-        if (rv != 0)
-        {
-            OutputLine("mbedtls_ssl_set_hs_ecjpake_password returned %d", rv);
-        }
-        mbedtls_ssl_set_bio(&mSslContext, &mEndpointAndCircularSendBuffer, otTcpMbedTlsSslSendCallback,
-                            otTcpMbedTlsSslRecvCallback, nullptr);
-        mTlsHandshakeComplete = false;
-        ContinueTLSHandshake();
+        PrepareTlsHandshake();
+        ContinueTlsHandshake();
     }
 #endif // OPENTHREAD_CONFIG_TLS_ENABLE
 }
@@ -660,7 +677,7 @@ void TcpExample::HandleTcpForwardProgress(otTcpEndpoint *aEndpoint, size_t aInSe
 #if OPENTHREAD_CONFIG_TLS_ENABLE
     if (mUseTls)
     {
-        ContinueTLSHandshake();
+        ContinueTlsHandshake();
     }
 #endif
 
@@ -688,8 +705,22 @@ void TcpExample::HandleTcpReceiveAvailable(otTcpEndpoint *aEndpoint,
     OT_UNUSED_VARIABLE(aBytesRemaining);
     OT_ASSERT(aEndpoint == &mEndpoint);
 
+    /* If we get data before the handshake completes, then this is a TFO connection. */
+    if (!mEndpointConnected)
+    {
+        mEndpointConnected         = true;
+        mEndpointConnectedFastOpen = true;
+
 #if OPENTHREAD_CONFIG_TLS_ENABLE
-    if (mUseTls && ContinueTLSHandshake())
+        if (mUseTls)
+        {
+            PrepareTlsHandshake();
+        }
+#endif
+    }
+
+#if OPENTHREAD_CONFIG_TLS_ENABLE
+    if (mUseTls && ContinueTlsHandshake())
     {
         return;
     }
@@ -773,8 +804,9 @@ void TcpExample::HandleTcpDisconnected(otTcpEndpoint *aEndpoint, otTcpDisconnect
     // We set this to false even for the TIME-WAIT state, so that we can reuse
     // the active socket if an incoming connection comes in instead of waiting
     // for the 2MSL timeout.
-    mEndpointConnected = false;
-    mSendBusy          = false;
+    mEndpointConnected         = false;
+    mEndpointConnectedFastOpen = false;
+    mSendBusy                  = false;
 
     // Mark the benchmark as inactive if the connection was disconnected.
     mBenchmarkBytesTotal  = 0;
@@ -803,20 +835,11 @@ otTcpIncomingConnectionAction TcpExample::HandleTcpAcceptReady(otTcpListener
     *aAcceptInto = &mEndpoint;
     action       = OT_TCP_INCOMING_CONNECTION_ACTION_ACCEPT;
 
-exit:
-    return action;
-}
-
-void TcpExample::HandleTcpAcceptDone(otTcpListener *aListener, otTcpEndpoint *aEndpoint, const otSockAddr *aPeer)
-{
-    OT_UNUSED_VARIABLE(aListener);
-    OT_UNUSED_VARIABLE(aEndpoint);
-
-    mEndpointConnected = true;
-    OutputFormat("Accepted connection from ");
-    OutputSockAddrLine(*aPeer);
-
 #if OPENTHREAD_CONFIG_TLS_ENABLE
+    /*
+     * Natural to wait until the AcceptDone callback but with TFO we could get data before that
+     * so it doesn't make sense to wait until then.
+     */
     if (mUseTls)
     {
         int rv;
@@ -835,6 +858,19 @@ void TcpExample::HandleTcpAcceptDone(otTcpListener *aListener, otTcpEndpoint *aE
         }
     }
 #endif // OPENTHREAD_CONFIG_TLS_ENABLE
+
+exit:
+    return action;
+}
+
+void TcpExample::HandleTcpAcceptDone(otTcpListener *aListener, otTcpEndpoint *aEndpoint, const otSockAddr *aPeer)
+{
+    OT_UNUSED_VARIABLE(aListener);
+    OT_UNUSED_VARIABLE(aEndpoint);
+
+    mEndpointConnected = true;
+    OutputFormat("Accepted connection from ");
+    OutputSockAddrLine(*aPeer);
 }
 
 otError TcpExample::ContinueBenchmarkCircularSend(void)
@@ -908,7 +944,26 @@ void TcpExample::CompleteBenchmark(void)
 }
 
 #if OPENTHREAD_CONFIG_TLS_ENABLE
-bool TcpExample::ContinueTLSHandshake(void)
+void TcpExample::PrepareTlsHandshake(void)
+{
+    int rv;
+    rv = mbedtls_ssl_set_hostname(&mSslContext, "localhost");
+    if (rv != 0)
+    {
+        OutputLine("mbedtls_ssl_set_hostname returned %d", rv);
+    }
+    rv = mbedtls_ssl_set_hs_ecjpake_password(&mSslContext, reinterpret_cast<const unsigned char *>(sEcjpakePassword),
+                                             sEcjpakePasswordLength);
+    if (rv != 0)
+    {
+        OutputLine("mbedtls_ssl_set_hs_ecjpake_password returned %d", rv);
+    }
+    mbedtls_ssl_set_bio(&mSslContext, &mEndpointAndCircularSendBuffer, otTcpMbedTlsSslSendCallback,
+                        otTcpMbedTlsSslRecvCallback, nullptr);
+    mTlsHandshakeComplete = false;
+}
+
+bool TcpExample::ContinueTlsHandshake(void)
 {
     bool wasNotAlreadyDone = false;
     int  rv;

src/cli/cli_tcp.hpp

@@ -96,7 +96,8 @@ private:
     void    CompleteBenchmark(void);
 
 #if OPENTHREAD_CONFIG_TLS_ENABLE
-    bool ContinueTLSHandshake(void);
+    void PrepareTlsHandshake(void);
+    bool ContinueTlsHandshake(void);
 #endif
 
     static void HandleTcpEstablishedCallback(otTcpEndpoint *aEndpoint);
@@ -138,6 +139,7 @@ private:
 
     bool mInitialized;
     bool mEndpointConnected;
+    bool mEndpointConnectedFastOpen;
     bool mSendBusy;
     bool mUseCircularSendBuffer;
     bool mUseTls;

src/core/net/tcp6.cpp

@@ -168,17 +168,28 @@ exit:
 
 Error Tcp::Endpoint::Connect(const SockAddr &aSockName, uint32_t aFlags)
 {
-    Error               error = kErrorNone;
-    struct tcpcb       &tp    = GetTcb();
-    struct sockaddr_in6 sin6p;
-
-    OT_UNUSED_VARIABLE(aFlags);
+    Error         error = kErrorNone;
+    struct tcpcb &tp    = GetTcb();
 
     VerifyOrExit(tp.t_state == TCP6S_CLOSED, error = kErrorInvalidState);
 
-    memcpy(&sin6p.sin6_addr, &aSockName.mAddress, sizeof(sin6p.sin6_addr));
-    sin6p.sin6_port = HostSwap16(aSockName.mPort);
-    error           = BsdErrorToOtError(tcp6_usr_connect(&tp, &sin6p));
+    if (aFlags & OT_TCP_CONNECT_NO_FAST_OPEN)
+    {
+        struct sockaddr_in6 sin6p;
+
+        tp.t_flags &= ~TF_FASTOPEN;
+        memcpy(&sin6p.sin6_addr, &aSockName.mAddress, sizeof(sin6p.sin6_addr));
+        sin6p.sin6_port = HostSwap16(aSockName.mPort);
+        error           = BsdErrorToOtError(tcp6_usr_connect(&tp, &sin6p));
+    }
+    else
+    {
+        tp.t_flags |= TF_FASTOPEN;
+
+        /* Stash the destination address in tp. */
+        memcpy(&tp.faddr, &aSockName.mAddress, sizeof(tp.faddr));
+        tp.fport = HostSwap16(aSockName.mPort);
+    }
 
 exit:
     return error;
@@ -192,7 +203,17 @@ Error Tcp::Endpoint::SendByReference(otLinkedBuffer &aBuffer, uint32_t aFlags)
     size_t backlogBefore = GetBacklogBytes();
     size_t sent          = aBuffer.mLength;
 
-    SuccessOrExit(error = BsdErrorToOtError(tcp_usr_send(&tp, (aFlags & OT_TCP_SEND_MORE_TO_COME) != 0, &aBuffer, 0)));
+    struct sockaddr_in6  sin6p;
+    struct sockaddr_in6 *name = nullptr;
+
+    if (IS_FASTOPEN(tp.t_flags))
+    {
+        memcpy(&sin6p.sin6_addr, &tp.faddr, sizeof(sin6p.sin6_addr));
+        sin6p.sin6_port = tp.fport;
+        name            = &sin6p;
+    }
+    SuccessOrExit(
+        error = BsdErrorToOtError(tcp_usr_send(&tp, (aFlags & OT_TCP_SEND_MORE_TO_COME) != 0, &aBuffer, 0, name)));
 
     PostCallbacksAfterSend(sent, backlogBefore);
 
@@ -208,9 +229,19 @@ Error Tcp::Endpoint::SendByExtension(size_t aNumBytes, uint32_t aFlags)
     size_t        backlogBefore = GetBacklogBytes();
     int           bsdError;
 
+    struct sockaddr_in6  sin6p;
+    struct sockaddr_in6 *name = nullptr;
+
     VerifyOrExit(lbuf_head(&tp.sendbuf) != nullptr, error = kErrorInvalidState);
 
-    bsdError = tcp_usr_send(&tp, moreToCome ? 1 : 0, nullptr, aNumBytes);
+    if (IS_FASTOPEN(tp.t_flags))
+    {
+        memcpy(&sin6p.sin6_addr, &tp.faddr, sizeof(sin6p.sin6_addr));
+        sin6p.sin6_port = tp.fport;
+        name            = &sin6p;
+    }
+
+    bsdError = tcp_usr_send(&tp, moreToCome ? 1 : 0, nullptr, aNumBytes, name);
     SuccessOrExit(error = BsdErrorToOtError(bsdError));
 
     PostCallbacksAfterSend(aNumBytes, backlogBefore);
@@ -614,6 +645,9 @@ Error Tcp::HandleMessage(ot::Ip6::Header &aIp6Header, Message &aMessage, Message
     Listener *listener;
     Listener *listenerPrev;
 
+    struct tcplp_signals sig;
+    int                  nextAction;
+
     VerifyOrExit(length == aMessage.GetLength() - aMessage.GetOffset(), error = kErrorParse);
     VerifyOrExit(length >= sizeof(Tcp::Header), error = kErrorParse);
     SuccessOrExit(error = aMessage.Read(aMessage.GetOffset() + offsetof(struct tcphdr, th_off_x2), headerSize));
@@ -634,9 +668,7 @@ Error Tcp::HandleMessage(ot::Ip6::Header &aIp6Header, Message &aMessage, Message
     endpoint = mEndpoints.FindMatching(aMessageInfo, endpointPrev);
     if (endpoint != nullptr)
     {
-        struct tcplp_signals sig;
-        int                  nextAction;
-        struct tcpcb        *tp = &endpoint->GetTcb();
+        struct tcpcb *tp = &endpoint->GetTcb();
 
         otLinkedBuffer *priorHead    = lbuf_head(&tp->sendbuf);
         size_t          priorBacklog = endpoint->GetSendBufferBytes() - endpoint->GetInFlightBytes();
@@ -656,7 +688,13 @@ Error Tcp::HandleMessage(ot::Ip6::Header &aIp6Header, Message &aMessage, Message
     {
         struct tcpcb_listen *tpl = &listener->GetTcbListen();
 
-        tcp_input(ip6Header, tcpHeader, &aMessage, nullptr, tpl, nullptr);
+        memset(&sig, 0x00, sizeof(sig));
+        nextAction = tcp_input(ip6Header, tcpHeader, &aMessage, nullptr, tpl, &sig);
+        OT_ASSERT(nextAction != RELOOKUP_REQUIRED);
+        if (sig.accepted_connection != nullptr)
+        {
+            ProcessSignals(Tcp::Endpoint::FromTcb(*sig.accepted_connection), nullptr, 0, sig);
+        }
         ExitNow();
     }
 
@@ -683,6 +721,8 @@ void Tcp::ProcessSignals(Endpoint             &aEndpoint,
     {
         otLinkedBuffer *curr = aPriorHead;
 
+        OT_ASSERT(curr != nullptr || aSignals.links_popped == 0);
+
         for (uint32_t i = 0; i != aSignals.links_popped; i++)
         {
             otLinkedBuffer *next = curr->mNext;

tests/scripts/expect/cli-tcp-tfo-tls.exp

@@ -0,0 +1,112 @@
+#!/usr/bin/expect -f
+#
+#  Copyright (c) 2023, The OpenThread Authors.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#  3. Neither the name of the copyright holder nor the
+#     names of its contributors may be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGE.
+#
+
+source "tests/scripts/expect/_common.exp"
+source "tests/scripts/expect/_multinode.exp"
+
+spawn_node 2 "cli"
+spawn_node 1 "cli"
+setup_leader
+setup_node 2 "rnd" "router"
+
+switch_node 1
+send "tcp init tls\n"
+expect_line "Done"
+
+switch_node 2
+send "tcp init tls\n"
+expect_line "Done"
+set addr_2 [get_ipaddr mleid]
+send "tcp listen :: 30000\n"
+expect_line "Done"
+send "tcp stoplistening\n"
+expect_line "Done"
+
+switch_node 1
+send "tcp connect $addr_2 30000\n"
+expect_line "Done"
+expect "TCP: Connection refused"
+
+switch_node 2
+send "tcp listen :: 30000\n"
+expect_line "Done"
+
+switch_node 1
+set addr_1 [get_ipaddr mleid]
+send "tcp bind $addr_1 25000\n"
+expect_line "Done"
+send "tcp connect $addr_2 30000 fast\n"
+expect_line "Done"
+expect "TCP: Connection established"
+expect "TLS Handshake Complete"
+
+switch_node 2
+expect "Accepted connection from \\\[$addr_1\\\]:25000"
+expect "TCP: Connection established"
+expect "TLS Handshake Complete"
+
+switch_node 1
+send "tcp send hello\n"
+expect_line "Done"
+
+switch_node 2
+expect "TLS: Received 5 bytes: hello"
+expect "(TCP: Received 26 bytes)"
+send "tcp send world\n"
+expect_line "Done"
+
+switch_node 1
+expect "TLS: Received 5 bytes: world"
+expect "(TCP: Received 26 bytes)"
+send "tcp sendend\n"
+expect_line "Done"
+send "tcp send more\n"
+expect_line "Error 1: Failed"
+
+switch_node 2
+expect "TCP: Reached end of stream"
+send "tcp send goodbye\n"
+expect_line "Done"
+
+switch_node 1
+expect "TLS: Received 7 bytes: goodbye"
+expect "(TCP: Received 28 bytes)"
+
+switch_node 2
+send "tcp sendend\n"
+expect_line "Done"
+expect "TCP: Disconnected"
+
+switch_node 1
+expect "TCP: Reached end of stream"
+expect "TCP: Entered TIME-WAIT state"
+set timeout 130
+expect "TCP: Disconnected"
+
+dispose_all

tests/scripts/expect/cli-tcp-tfo.exp

@@ -0,0 +1,103 @@
+#!/usr/bin/expect -f
+#
+#  Copyright (c) 2023, The OpenThread Authors.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#  3. Neither the name of the copyright holder nor the
+#     names of its contributors may be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGE.
+#
+
+source "tests/scripts/expect/_common.exp"
+source "tests/scripts/expect/_multinode.exp"
+
+spawn_node 2 "cli"
+spawn_node 1 "cli"
+setup_leader
+setup_node 2 "rnd" "router"
+
+switch_node 1
+send "tcp init circular\n"
+expect_line "Done"
+
+switch_node 2
+send "tcp init linked\n"
+expect_line "Done"
+set addr_2 [get_ipaddr mleid]
+send "tcp listen :: 30000\n"
+expect_line "Done"
+send "tcp stoplistening\n"
+expect_line "Done"
+
+switch_node 1
+send "tcp connect $addr_2 30000\n"
+expect_line "Done"
+expect "TCP: Connection refused"
+
+switch_node 2
+send "tcp listen :: 30000\n"
+expect_line "Done"
+
+switch_node 1
+set addr_1 [get_ipaddr mleid]
+send "tcp bind $addr_1 25000\n"
+expect_line "Done"
+send "tcp connect $addr_2 30000 fast\n"
+expect_line "Done"
+send "tcp send hello\n"
+expect_line "Done"
+expect "TCP: Connection established"
+
+switch_node 2
+expect "TCP: Received 5 bytes: hello"
+expect "Accepted connection from \\\[$addr_1\\\]:25000"
+expect "TCP: Connection established"
+send "tcp send world\n"
+expect_line "Done"
+
+switch_node 1
+expect "TCP: Received 5 bytes: world"
+send "tcp sendend\n"
+expect_line "Done"
+send "tcp send more\n"
+expect_line "Error 1: Failed"
+
+switch_node 2
+expect "TCP: Reached end of stream"
+send "tcp send goodbye\n"
+expect_line "Done"
+
+switch_node 1
+expect "TCP: Received 7 bytes: goodbye"
+
+switch_node 2
+send "tcp sendend\n"
+expect_line "Done"
+expect "TCP: Disconnected"
+
+switch_node 1
+expect "TCP: Reached end of stream"
+expect "TCP: Entered TIME-WAIT state"
+set timeout 130
+expect "TCP: Disconnected"
+
+dispose_all

third_party/tcplp/CMakeLists.txt

@@ -30,6 +30,7 @@ project("TCPlp" C)
 
 set(src_tcplp
     bsdtcp/cc/cc_newreno.c
+    bsdtcp/tcp_fastopen.c
     bsdtcp/tcp_input.c
     bsdtcp/tcp_output.c
     bsdtcp/tcp_reass.c

third_party/tcplp/bsdtcp/tcp.h

@@ -118,6 +118,8 @@ struct tcphdr {
 #define    TCPOLEN_TSTAMP_APPA		(TCPOLEN_TIMESTAMP+2) /* appendix A */
 #define	TCPOPT_SIGNATURE	19		/* Keyed MD5: RFC 2385 */
 #define	   TCPOLEN_SIGNATURE		18
+#define	TCPOPT_FAST_OPEN	34
+#define	   TCPOLEN_FAST_OPEN_EMPTY	2
 
 /* Miscellaneous constants */
 #define	MAX_SACK_BLKS	6	/* Max # SACK blocks stored at receiver side */
@@ -167,6 +169,9 @@ struct tcphdr {
 #define TCP_MAXHLEN	(0xf<<2)	/* max length of header in bytes */
 #define TCP_MAXOLEN	(TCP_MAXHLEN - sizeof(struct tcphdr))
 					/* max space left for options */
+#define TCP_FASTOPEN_MIN_COOKIE_LEN	4	/* Per RFC7413 */
+#define TCP_FASTOPEN_MAX_COOKIE_LEN	16	/* Per RFC7413 */
+#define TCP_FASTOPEN_PSK_LEN		16	/* Same as TCP_FASTOPEN_KEY_LEN */
 
 /*
  * User-settable options (used with setsockopt).  These are discrete

third_party/tcplp/bsdtcp/tcp_const.h

@@ -95,9 +95,17 @@ enum tcp_subr_consts {
 enum tcp_timer_consts {
 //	V_tcp_v6pmtud_blackhole_mss = FRAMECAP_6LOWPAN - sizeof(struct ip6_hdr) - sizeof(struct tcphdr), // Doesn't matter unless blackhole_detect is 1.
 	tcp_rexmit_drop_options = 0, // drop options after a few retransmits
-	always_keepalive = 1,
+	always_keepalive = 1
 };
 
+enum tcp_fastopen_consts {
+	V_tcp_fastopen_client_enable = 1,
+	V_tcp_fastopen_server_enable = 1,
+	V_tcp_fastopen_acceptany = 1,
+	V_tcp_fastopen_numkeys = 4
+};
+#define TCP_RFC7413
+
 /*
  * Force a time value to be in a certain range.
  */

third_party/tcplp/bsdtcp/tcp_fastopen.h

@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2015-2017 Patrick Kelsey
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef TCPLP_TCP_FASTOPEN_H_
+#define TCPLP_TCP_FASTOPEN_H_
+
+// #include "opt_inet.h"
+#include "tcp_const.h"
+#include "tcp_var.h"
+
+#define	TCP_FASTOPEN_COOKIE_LEN		8	/* SipHash24 64-bit output */
+
+#if 0
+
+#ifdef TCP_RFC7413
+VNET_DECLARE(unsigned int, tcp_fastopen_client_enable);
+#define	V_tcp_fastopen_client_enable	VNET(tcp_fastopen_client_enable)
+
+VNET_DECLARE(unsigned int, tcp_fastopen_server_enable);
+#define	V_tcp_fastopen_server_enable	VNET(tcp_fastopen_server_enable)
+#else
+#define	V_tcp_fastopen_client_enable	0
+#define	V_tcp_fastopen_server_enable	0
+#endif  /* TCP_RFC7413 */
+
+union tcp_fastopen_ip_addr {
+	struct in_addr v4;
+	struct in6_addr v6;
+};
+
+struct tcp_fastopen_ccache_entry {
+	TAILQ_ENTRY(tcp_fastopen_ccache_entry) cce_link;
+	union tcp_fastopen_ip_addr cce_client_ip;	/* network byte order */
+	union tcp_fastopen_ip_addr cce_server_ip;	/* network byte order */
+	uint16_t server_port;				/* network byte order */
+	uint16_t server_mss;				/* host byte order */
+	uint8_t af;
+	uint8_t cookie_len;
+	uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN];
+	sbintime_t disable_time; /* non-zero value means path is disabled */
+};
+
+struct tcp_fastopen_ccache;
+
+struct tcp_fastopen_ccache_bucket {
+	struct mtx	ccb_mtx;
+	TAILQ_HEAD(bucket_entries, tcp_fastopen_ccache_entry) ccb_entries;
+	int		ccb_num_entries;
+	struct tcp_fastopen_ccache *ccb_ccache;
+};
+
+struct tcp_fastopen_ccache {
+	uma_zone_t 	zone;
+	struct tcp_fastopen_ccache_bucket *base;
+	unsigned int 	bucket_limit;
+	unsigned int 	buckets;
+	unsigned int 	mask;
+	uint32_t 	secret;
+};
+
+#endif
+
+#ifdef TCP_RFC7413
+void	tcp_fastopen_init(void);
+void	tcp_fastopen_destroy(void);
+unsigned int *tcp_fastopen_alloc_counter(void);
+void	tcp_fastopen_decrement_counter(unsigned int *);
+/* samkumar: changed type of first argument from "struct in_conninfo *"" to "struct tcpcb*"". */
+int	tcp_fastopen_check_cookie(struct tcpcb*, uint8_t *, unsigned int,
+	    uint64_t *);
+void	tcp_fastopen_connect(struct tcpcb *);
+void	tcp_fastopen_disable_path(struct tcpcb *);
+void	tcp_fastopen_update_cache(struct tcpcb *, uint16_t, uint8_t,
+	    uint8_t *);
+#else
+#define tcp_fastopen_init()			((void)0)
+#define tcp_fastopen_destroy()			((void)0)
+#define tcp_fastopen_alloc_counter()		NULL
+#define tcp_fastopen_decrement_counter(c)	((void)0)
+#define tcp_fastopen_check_cookie(i, c, l, lc)	(-1)
+#define tcp_fastopen_connect(t)			((void)0)
+#define tcp_fastopen_disable_path(t)		((void)0)
+#define tcp_fastopen_update_cache(t, m, l, c)	((void)0)
+#endif /* TCP_RFC7413 */
+
+#endif /* _TCP_FASTOPEN_H_ */

third_party/tcplp/bsdtcp/tcp_input.c

@@ -81,6 +81,7 @@
 #include "tcp_seq.h"
 #include "tcp_timer.h"
 #include "tcp_var.h"
+#include "tcp_fastopen.h"
 #include "../lib/bitmap.h"
 #include "../lib/cbuf.h"
 #include "icmp_var.h"
@@ -640,6 +641,10 @@ tcp_input(struct ip6_hdr* ip6, struct tcphdr* th, otMessage* msg, struct tcpcb*
 	 */
 
 	if (/*so->so_options & SO_ACCEPTCONN*/tp == NULL) {
+		int tfo_cookie_valid = 0;
+		uint64_t tfo_response_cookie;
+		// int tfo_response_cookie_valid = 0;
+
 		/* samkumar: NULL check isn't needed but prevents a compiler warning */
 		KASSERT(tpl != NULL && tpl->t_state == TCP6S_LISTEN, ("listen socket must be in listening state!"));
 
@@ -768,8 +773,43 @@ tcp_input(struct ip6_hdr* ip6, struct tcphdr* th, otMessage* msg, struct tcpcb*
 		 * syncache, so we initialize the new socket right away. The code to
 		 * initialize the socket is taken from the syncache_socket function.
 		 */
-
+		/*
+		 * samkumar: As of FreeBSD 10.3, the syncache_add function returns
+		 * a flag indicating if a "fast open" code path should be taken.
+		 * In that case, there is a "goto" statement to the removed logic
+		 * above that calls tcp_do_segment after expanding a syncache entry.
+		 * Analogous logic is implemented below.
+		 */
 		tcp_dooptions(&to, optp, optlen, TO_SYN);
+
+		/*
+		 * samkumar: TCP Fast Open logic taken from syncache_add in
+		 * FreeBSD 12.0.
+		 */
+		if (V_tcp_fastopen_server_enable && /*IS_FASTOPEN(tp->t_flags) &&
+			(tp->t_tfo_pending != NULL) && */
+			(to.to_flags & TOF_FASTOPEN)) {
+			/*
+			 * Limit the number of pending TFO connections to
+			 * approximately half of the queue limit.  This prevents TFO
+			 * SYN floods from starving the service by filling the
+			 * listen queue with bogus TFO connections.
+			 */
+			/*
+			 * samkumar: Since we let the application handle the listen
+			 * queue it doesn't make sense to limit the number of pending
+			 * TFO connections as above. Long term, I think the best fix
+			 * is to let applications know if an incoming connection is
+			 * TFO, so that they can handle the case appropriately (e.g.,
+			 * by disabling TFO or by declining the connection).
+			 */
+			int result = tcp_fastopen_check_cookie(NULL,
+				to.to_tfo_cookie, to.to_tfo_len,
+				&tfo_response_cookie);
+			tfo_cookie_valid = (result > 0);
+			// tfo_response_cookie_valid = (result >= 0);
+		}
+
 		tp = tcplp_sys_accept_ready(tpl, &ip6->ip6_src, th->th_sport); // Try to allocate an active socket to accept into
 		if (tp == NULL) {
 			/* If we couldn't allocate, just ignore the SYN. */
@@ -780,14 +820,23 @@ tcp_input(struct ip6_hdr* ip6, struct tcphdr* th, otMessage* msg, struct tcpcb*
 			tp = NULL;
 			goto dropwithreset;
 		}
+		sig->accepted_connection = tp;
 		tcp_state_change(tp, TCPS_SYN_RECEIVED);
 		tpmarkpassiveopen(tp);
-		tp->t_flags |= TF_ACKNOW; // samkumar: my addition
 		tp->iss = tcp_new_isn(tp);
 		tp->irs = th->th_seq;
 		tcp_rcvseqinit(tp);
 		tcp_sendseqinit(tp);
 		tp->snd_wl1 = th->th_seq;
+		/*
+		 * samkumar: We remove the "+ 1"s below since we use
+		 * tcp_output to send the appropriate SYN-ACK. For
+		 * example, syncache_tfo_expand eliminates the "+ 1"s
+		 * too. My understanding is that syncache_socket has
+		 * the "+ 1"s because it's normally called once the
+		 * SYN-ACK has already been ACKed, which is not how
+		 * TCPlp operates.
+		 */
 		tp->snd_max = tp->iss/* + 1*/;
 		tp->snd_nxt = tp->iss/* + 1*/;
 		tp->rcv_up = th->th_seq + 1;
@@ -890,6 +939,26 @@ tcp_input(struct ip6_hdr* ip6, struct tcphdr* th, otMessage* msg, struct tcpcb*
 		 */
 		tcp_mss(tp, /*sc->sc_peer_mss*/(to.to_flags & TOF_MSS) ? to.to_mss : 0);
 
+		if (tfo_cookie_valid) {
+			/*
+			 * samkumar: The code below is taken from syncache_tfo_socket.
+			 * It calls syncache_socket (upon which the above code is based)
+			 * so it makes sense for this logic to go here.
+			 */
+			tp->t_flags |= TF_FASTOPEN;
+			tp->t_tfo_cookie.server = tfo_response_cookie;
+			tp->snd_max = tp->iss;
+			tp->snd_nxt = tp->iss;
+			// tp->tfo_pending = pending_counter;
+			/* This would normally "goto" labeled code that calls tcp_do_segment. */
+			tcp_do_segment(ip6, th, msg, tp, drop_hdrlen, tlen, iptos, sig);
+
+			tp->accepted_from = tpl;
+			return (IPPROTO_DONE);
+		} else {
+			tp->t_flags |= TF_ACKNOW; // samkumar: my addition
+		}
+
 		tcp_output(tp); // to send the SYN-ACK
 
 		tp->accepted_from = tpl;
@@ -964,6 +1033,7 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 	int rstreason, todrop, win;
 	uint64_t tiwin;
 	struct tcpopt to;
+	int tfo_syn;
 	uint32_t ticks = tcplp_sys_get_ticks();
 	otInstance* instance = tp->instance;
 	thflags = th->th_flags;
@@ -1093,6 +1163,29 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 		if ((tp->t_flags & TF_SACK_PERMIT) &&
 		    (to.to_flags & TOF_SACKPERM) == 0)
 			tp->t_flags &= ~TF_SACK_PERMIT;
+		/*
+		 * samkumar: TCP Fast Open logic from FreeBSD 12.0.
+		 */
+		if (IS_FASTOPEN(tp->t_flags)) {
+			if (to.to_flags & TOF_FASTOPEN) {
+				uint16_t mss;
+
+				if (to.to_flags & TOF_MSS)
+					mss = to.to_mss;
+				else
+					/*
+					 * samkumar: The original code here would set
+					 * mss to either TCP6_MSS or TCP_MSS depending
+					 * on whether the INP_IPV6 flag is present in
+					 * tp->t_inpcb->inp_vflag. In TCPlp, we always
+					 * assume IPv6.
+					 */
+					mss = TCP6_MSS;
+				tcp_fastopen_update_cache(tp, mss,
+				    to.to_tfo_len, to.to_tfo_cookie);
+			} else
+				tcp_fastopen_disable_path(tp);
+		}
 	}
 	/*
 	 * Header prediction: check for the two common cases
@@ -1403,9 +1496,32 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 		     SEQ_GT(th->th_ack, tp->snd_max))) {
 				rstreason = BANDLIM_RST_OPENPORT;
 				goto dropwithreset;
-		} else if ((thflags & TH_SYN) && !(thflags & TH_ACK) && (th->th_seq == tp->irs)) {
+		} else if (!IS_FASTOPEN(tp->t_flags) && (thflags & TH_SYN) && !(thflags & TH_ACK) && (th->th_seq == tp->irs)) {
 			tp->t_flags |= TF_ACKNOW;
 		}
+		/*
+		 * samkumar: TCP Fast Open Logic from FreeBSD 12.0.
+		 */
+		if (IS_FASTOPEN(tp->t_flags)) {
+			/*
+			 * When a TFO connection is in SYN_RECEIVED, the
+			 * only valid packets are the initial SYN, a
+			 * retransmit/copy of the initial SYN (possibly with
+			 * a subset of the original data), a valid ACK, a
+			 * FIN, or a RST.
+			 */
+			if ((thflags & (TH_SYN|TH_ACK)) == (TH_SYN|TH_ACK)) {
+				rstreason = BANDLIM_RST_OPENPORT;
+				goto dropwithreset;
+			} else if (thflags & TH_SYN) {
+				/* non-initial SYN is ignored */
+				if ((tcp_timer_active(tp, TT_DELACK) || 
+				     tcp_timer_active(tp, TT_REXMT)))
+					goto drop;
+			} else if (!(thflags & (TH_ACK|TH_FIN|TH_RST))) {
+				goto drop;
+			}
+		}
 		break;
 
 	/*
@@ -1440,6 +1556,8 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 		tp->irs = th->th_seq;
 		tcp_rcvseqinit(tp);
 		if (thflags & TH_ACK) {
+			int tfo_partial_ack = 0;
+
 			/*
 			 * samkumar: Removed call to soisconnected(so), since TCPlp has its
 			 * own buffering.
@@ -1453,11 +1571,20 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 			tp->rcv_adv += imin(tp->rcv_wnd,
 			    TCP_MAXWIN << tp->rcv_scale);
 			tp->snd_una++;		/* SYN is acked */
+			/*
+			 * If not all the data that was sent in the TFO SYN
+			 * has been acked, resend the remainder right away.
+			 */
+			if (IS_FASTOPEN(tp->t_flags) &&
+			    (tp->snd_una != tp->snd_max)) {
+				tp->snd_nxt = th->th_ack;
+				tfo_partial_ack = 1;
+			}
 			/*
 			 * If there's data, delay ACK; if there's also a FIN
 			 * ACKNOW will be turned on later.
 			 */
-			if (DELAY_ACK(tp, tlen) && tlen != 0)
+			if (DELAY_ACK(tp, tlen) && tlen != 0 && !tfo_partial_ack)
 				tcp_timer_activate(tp, TT_DELACK,
 				    tcp_delacktime);
 			else
@@ -1802,9 +1929,14 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 	 */
 	if ((thflags & TH_ACK) == 0) {
 		if (tp->t_state == TCPS_SYN_RECEIVED ||
-		    (tp->t_flags & TF_NEEDSYN))
+		    (tp->t_flags & TF_NEEDSYN)) {
+			if (tp->t_state == TCPS_SYN_RECEIVED &&
+			    IS_FASTOPEN(tp->t_flags)) {
+				tp->snd_wnd = tiwin;
+				cc_conn_init(tp);
+			}
 			goto step6;
-		else if (tp->t_flags & TF_ACKNOW)
+		} else if (tp->t_flags & TF_ACKNOW)
 			goto dropafterack;
 		else
 			goto drop;
@@ -1839,6 +1971,21 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 		 *      SYN-RECEIVED* -> FIN-WAIT-1
 		 */
 		tp->t_starttime = ticks;
+		/*
+		 * samkumar: I'm eliminating the TFO pending counter.
+		 */
+		if (IS_FASTOPEN(tp->t_flags)/* && tp->t_tfo_pending */) {\
+			/*
+			tcp_fastopen_decrement_counter(tp->t_tfo_pending);
+			tp->t_tfo_pending = NULL;
+			*/
+
+			/*
+			 * Account for the ACK of our SYN prior to
+			 * regular ACK processing below.
+			 */ 
+			tp->snd_una++;
+		}
 		if (tp->t_flags & TF_NEEDFIN) {
 			tcp_state_change(tp, TCPS_FIN_WAIT_1);
 			tp->t_flags &= ~TF_NEEDFIN;
@@ -1846,7 +1993,15 @@ tcp_do_segment(struct ip6_hdr* ip6, struct tcphdr *th, otMessage* msg,
 			tcp_state_change(tp, TCPS_ESTABLISHED);
 			/* samkumar: Set conn_established signal for TCPlp. */
 			sig->conn_established = true;
-			cc_conn_init(tp);
+			/*
+			 * TFO connections call cc_conn_init() during SYN
+			 * processing.  Calling it again here for such
+			 * connections is not harmless as it would undo the
+			 * snd_cwnd reduction that occurs when a TFO SYN|ACK
+			 * is retransmitted.
+			 */
+			if (!IS_FASTOPEN(tp->t_flags))
+				cc_conn_init(tp);
 			tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp));
 			/*
 			 * samkumar: I added this check to account for simultaneous open.
@@ -2383,7 +2538,9 @@ step6:
 	 * case PRU_RCVD).  If a FIN has already been received on this
 	 * connection then we just ignore the text.
 	 */
-	if ((tlen || (thflags & TH_FIN)) &&
+	tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
+		   IS_FASTOPEN(tp->t_flags));
+	if ((tlen || (thflags & TH_FIN) || tfo_syn) &&
 	    TCPS_HAVERCVDFIN(tp->t_state) == 0) {
 		tcp_seq save_start = th->th_seq;
 		/*
@@ -2411,8 +2568,9 @@ step6:
 		 */
 		if (th->th_seq == tp->rcv_nxt &&
 		    (tpiscantrcv(tp) || bmp_isempty(tp->reassbmp, REASSBMP_SIZE(tp))) &&
-		    TCPS_HAVEESTABLISHED(tp->t_state)) {
-			if (DELAY_ACK(tp, tlen))
+		    (TCPS_HAVEESTABLISHED(tp->t_state) ||
+			 tfo_syn)) {
+			if (DELAY_ACK(tp, tlen) || tfo_syn)
 				tp->t_flags |= TF_DELACK;
 			else
 				tp->t_flags |= TF_ACKNOW;
@@ -2694,6 +2852,21 @@ tcp_dooptions(struct tcpopt *to, uint8_t *cp, int cnt, int flags)
 			to->to_nsacks = (optlen - 2) / TCPOLEN_SACK;
 			to->to_sacks = cp + 2;
 			break;
+		case TCPOPT_FAST_OPEN:
+			/*
+			 * Cookie length validation is performed by the
+			 * server side cookie checking code or the client
+			 * side cookie cache update code.
+			 */
+			if (!(flags & TO_SYN))
+				continue;
+			if (!V_tcp_fastopen_client_enable &&
+			    !V_tcp_fastopen_server_enable)
+				continue;
+			to->to_flags |= TOF_FASTOPEN;
+			to->to_tfo_len = optlen - 2;
+			to->to_tfo_cookie = to->to_tfo_len ? cp + 2 : NULL;
+			break;
 		default:
 			continue;
 		}

third_party/tcplp/bsdtcp/tcp_output.c

@@ -34,6 +34,7 @@
 
 #include "../tcplp.h"
 #include "tcp.h"
+#include "tcp_fastopen.h"
 #include "tcp_fsm.h"
 #include "tcp_var.h"
 #include "tcp_seq.h"
@@ -118,11 +119,25 @@ tcp_output(struct tcpcb *tp)
 	struct sackhole* p;
 	unsigned ipoptlen, optlen, hdrlen;
 	struct tcpopt to;
+	unsigned int wanted_cookie = 0;
+	unsigned int dont_sendalot = 0;
 	uint8_t opt[TCP_MAXOLEN];
 	uint32_t ticks = tcplp_sys_get_ticks();
 
 	/* samkumar: Code for TCP offload has been removed. */
 
+	/*
+	 * For TFO connections in SYN_SENT or SYN_RECEIVED,
+	 * only allow the initial SYN or SYN|ACK and those sent
+	 * by the retransmit timer.
+	 */
+	if (IS_FASTOPEN(tp->t_flags) &&
+	    ((tp->t_state == TCPS_SYN_SENT) ||
+	     (tp->t_state == TCPS_SYN_RECEIVED)) &&
+	    SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
+	    (tp->snd_nxt != tp->snd_una))       /* not a retransmit */
+		return (0);
+
 	/*
 	 * Determine length of data that should be transmitted,
 	 * and flags that will be used.
@@ -323,6 +338,13 @@ after_sack_rexmit:
 	if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
 		if (tp->t_state != TCPS_SYN_RECEIVED)
 			flags &= ~TH_SYN;
+		/*
+		 * When sending additional segments following a TFO SYN|ACK,
+		 * do not include the SYN bit.
+		 */
+		if (IS_FASTOPEN(tp->t_flags) &&
+		    (tp->t_state == TCPS_SYN_RECEIVED))
+			flags &= ~TH_SYN;
 		off--, len++;
 	}
 
@@ -336,6 +358,29 @@ after_sack_rexmit:
 		flags &= ~TH_FIN;
 	}
 
+	/*
+	 * On TFO sockets, ensure no data is sent in the following cases:
+	 *
+	 *  - When retransmitting SYN|ACK on a passively-created socket
+	 *
+	 *  - When retransmitting SYN on an actively created socket
+	 *
+	 *  - When sending a zero-length cookie (cookie request) on an
+	 *    actively created socket
+	 *
+	 *  - When the socket is in the CLOSED state (RST is being sent)
+	 */
+	/*
+	 * samkumar: I commented out the check to ensure no data is sent
+	 * on a TFO cookie request. As far as I am aware, this is still
+	 * compliant with the RFC.
+	 */
+	if (IS_FASTOPEN(tp->t_flags) &&
+	    (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
+	     /*((tp->t_state == TCPS_SYN_SENT) &&
+	      (tp->t_tfo_client_cookie_len == 0)) ||*/
+	     (flags & TH_RST)))
+		len = 0;
 	if (len <= 0) {
 		/*
 		 * If FIN has been sent but not acked,
@@ -675,16 +720,52 @@ send:
 	 * We only have to care about SYN and established connection
 	 * segments.  Options for SYN-ACK segments are handled in TCP
 	 * syncache.
-	 * Sam: I've done away with the syncache. However, it seems that
-	 * the existing logic works fine for SYN-ACK as well
 	 */
+	/*
+	 * samkumar: I've done away with the syncache. However, it
+	 * seems that the existing logic works fine for SYN-ACK as
+	 * well.
+	 */
+	to.to_flags = 0;
 	if ((tp->t_flags & TF_NOOPT) == 0) {
-		to.to_flags = 0;
 		/* Maximum segment size. */
 		if (flags & TH_SYN) {
 			tp->snd_nxt = tp->iss;
 			to.to_mss = tcp_mssopt(tp);
 			to.to_flags |= TOF_MSS;
+
+			/*
+			 * On SYN or SYN|ACK transmits on TFO connections,
+			 * only include the TFO option if it is not a
+			 * retransmit, as the presence of the TFO option may
+			 * have caused the original SYN or SYN|ACK to have
+			 * been dropped by a middlebox.
+			 */
+			if (IS_FASTOPEN(tp->t_flags) &&
+			    (tp->t_rxtshift == 0)) {
+				if (tp->t_state == TCPS_SYN_RECEIVED) {
+					to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
+					to.to_tfo_cookie =
+					    (u_int8_t *)&tp->t_tfo_cookie.server;
+					to.to_flags |= TOF_FASTOPEN;
+					wanted_cookie = 1;
+				} else if (tp->t_state == TCPS_SYN_SENT) {
+					to.to_tfo_len =
+					    tp->t_tfo_client_cookie_len;
+					to.to_tfo_cookie =
+					    tp->t_tfo_cookie.client;
+					to.to_flags |= TOF_FASTOPEN;
+					wanted_cookie = 1;
+					/*
+					 * If we wind up having more data to
+					 * send with the SYN than can fit in
+					 * one segment, don't send any more
+					 * until the SYN|ACK comes back from
+					 * the other end.
+					 */
+					dont_sendalot = 1;
+				}
+			}
 		}
 		/* Window scaling. */
 		if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
@@ -724,6 +805,13 @@ send:
 
 		/* Processing the options. */
 		hdrlen += optlen = tcp_addoptions(&to, opt);
+		/*
+		 * If we wanted a TFO option to be added, but it was unable
+		 * to fit, ensure no data is sent.
+		 */
+		if (IS_FASTOPEN(tp->t_flags) && wanted_cookie &&
+		    !(to.to_flags & TOF_FASTOPEN))
+			len = 0;
 	}
 	/*
 	 * samkumar: This used to be set to ip6_optlen(tp->t_inpcb), instead of 0,
@@ -746,6 +834,8 @@ send:
 		 */
 		len = tp->t_maxopd - optlen - ipoptlen;
 		sendalot = 1;
+		if (dont_sendalot)
+				sendalot = 0;
 	}
 	/*
 	 * samkumar: The else case of the above "if" statement would set tso to 0.
@@ -1453,6 +1543,25 @@ tcp_addoptions(struct tcpopt *to, uint8_t *optp)
 			/* samkumar: Removed TCPSTAT_INC(tcps_sack_send_blocks); */
 			break;
 			}
+		case TOF_FASTOPEN:
+			{
+			int total_len;
+
+			/* XXX is there any point to aligning this option? */
+			total_len = TCPOLEN_FAST_OPEN_EMPTY + to->to_tfo_len;
+			if (TCP_MAXOLEN - optlen < total_len) {
+				to->to_flags &= ~TOF_FASTOPEN;
+				continue;
+			}
+			*optp++ = TCPOPT_FAST_OPEN;
+			*optp++ = total_len;
+			if (to->to_tfo_len > 0) {
+				bcopy(to->to_tfo_cookie, optp, to->to_tfo_len);
+				optp += to->to_tfo_len;
+			}
+			optlen += total_len;
+			break;
+			}
 		default:
 			tcplp_sys_panic("PANIC: %s: unknown TCP option type", __func__);
 			break;

third_party/tcplp/bsdtcp/tcp_subr.c

@@ -45,6 +45,7 @@
 #include "../lib/bitmap.h"
 #include "../lib/cbuf.h"
 #include "cc.h"
+#include "tcp_fastopen.h"
 
 #include "tcp_const.h"
 
@@ -60,6 +61,7 @@ tcp_seq tcp_new_isn(struct tcpcb* tp) {
  * samkumar: There used to be a function, void tcp_init(void), that would
  * initialize global state for TCP, including a hash table to store TCBs,
  * allocating memory zones for sockets, and setting global configurable state.
+ * In FreeBSD 12.0, it also makes a call to the function tcp_fastopen_init.
  * None of that is needed for TCPlp: TCB allocation and matching is done by
  * the host system and global configurable state is removed with hardcoded
  * values in order to save memory, for example. Thus, I've removed the function
@@ -179,6 +181,13 @@ tcp_discardcb(struct tcpcb *tp)
 struct tcpcb *
 tcp_close(struct tcpcb *tp)
 {
+	/* samkumar: Eliminate the TFO pending counter. */
+	/*
+	if (tp->t_tfo_pending) {
+		tcp_fastopen_decrement_counter(tp->t_tfo_pending);
+		tp->t_tfo_pending = NULL;
+	}
+	*/
 	tcp_state_change(tp, TCP6S_CLOSED); // for the print statement
 	tcp_discardcb(tp);
 	// Don't reset the TCB by calling initialize_tcb, since that overwrites the buffer contents.
@@ -190,7 +199,8 @@ tcp_close(struct tcpcb *tp)
  * Allocates an mbuf and fills in a skeletal tcp/ip header.  The only
  * use for this function is in keepalives, which use tcp_respond.
  */
-/* samkumar: I changed the signature of this function. Instead of allocating
+/*
+ * samkumar: I changed the signature of this function. Instead of allocating
  * the struct tcptemp using malloc, populating it, and then returning it, I
  * have the caller allocate it. This function merely populates it now.
  */

third_party/tcplp/bsdtcp/tcp_usrreq.c

@@ -44,7 +44,7 @@
 #include "tcp_seq.h"
 #include "tcp_var.h"
 #include "tcp_timer.h"
-//#include <sys/socket.h>
+#include "tcp_fastopen.h"
 #include "ip6.h"
 
 #include "tcp_const.h"
@@ -251,14 +251,15 @@ out:
  * to extend the final linked buffer of the send buffer. Either DATA should be
  * NULL, or EXTENDBY should be 0.
  */
-int tcp_usr_send(struct tcpcb* tp, int moretocome, otLinkedBuffer* data, size_t extendby)
+int tcp_usr_send(struct tcpcb* tp, int moretocome, otLinkedBuffer* data, size_t extendby, struct sockaddr_in6* nam)
 {
 	int error = 0;
+	int do_fastopen_implied_connect = (nam != NULL) && IS_FASTOPEN(tp->t_flags) && tp->t_state < TCPS_SYN_SENT;
 
 	/*
 	 * samkumar: This if statement and the next are checks that I added
 	 */
-	if (tp->t_state < TCPS_ESTABLISHED) {
+	if (tp->t_state < TCPS_ESTABLISHED && !IS_FASTOPEN(tp->t_flags)) {
 		error = ENOTCONN;
 		goto out;
 	}
@@ -273,9 +274,9 @@ int tcp_usr_send(struct tcpcb* tp, int moretocome, otLinkedBuffer* data, size_t
 	 * INP_TIMEWAIT and INP_DROPPED flags on inp->inp_flags, and handled the
 	 * control mbuf passed as an argument (which would result in an error since
 	 * TCP doesn't support control information). I've deleted that code, but
-	 * left the following if block.
+	 * added the following if block based on those checks.
 	 */
-	 if ((tp->t_state == TCPS_TIME_WAIT) || (tp->t_state == TCPS_CLOSED)) {
+	 if ((tp->t_state == TCPS_TIME_WAIT) || (tp->t_state == TCPS_CLOSED && !do_fastopen_implied_connect)) {
 		error = ECONNRESET;
 		goto out;
 	}
@@ -283,13 +284,13 @@ int tcp_usr_send(struct tcpcb* tp, int moretocome, otLinkedBuffer* data, size_t
 	/*
 	 * The following code used to be wrapped in an if statement:
 	 * "if (!(flags & PRUS_OOB))", that only executed it if the "out of band"
-	 * flag was not set. In TCB, "out of band" data is conveyed via the urgent
+	 * flag was not set. In TCP, "out of band" data is conveyed via the urgent
 	 * pointer, and TCPlp does not support the urgent pointer. Therefore, I
 	 * removed the "if" check and put its body below.
 	 */
 
 	/*
-	 * samkumar; The FreeBSD code calls sbappendstream(&so->so_snd, m, flags);
+	 * samkumar: The FreeBSD code calls sbappendstream(&so->so_snd, m, flags);
 	 * I've replaced it with the following logic, which appends to the
 	 * send buffer according to TCPlp's data structures.
 	 */
@@ -308,9 +309,16 @@ int tcp_usr_send(struct tcpcb* tp, int moretocome, otLinkedBuffer* data, size_t
 	/*
 	 * samkumar: There used to be code here to handle "implied connect,"
 	 * which initiates the TCP handshake if sending data on a socket that
-	 * isn't yet connected. TCPlp doesn't support this at the moment, but
-	 * it might be worth revisiting  when implementing TCP Fast Open.
+	 * isn't yet connected. For now, I've special-cased this code to work
+	 * only for TCP Fast Open for IPv6 (since implied connect is the only
+	 * way to initiate a connection with TCP Fast Open).
 	 */
+	if (do_fastopen_implied_connect) {
+		error = tcp6_connect(tp, nam);
+		if (error)
+			goto out;
+		tcp_fastopen_connect(tp);
+	}
 
 	/*
 	 * samkumar: There used to be code here handling the PRUS_EOF flag in
@@ -362,6 +370,17 @@ tcp_usr_rcvd(struct tcpcb* tp)
 		goto out;
 	}
 
+	/*
+	 * For passively-created TFO connections, don't attempt a window
+	 * update while still in SYN_RECEIVED as this may trigger an early
+	 * SYN|ACK.  It is preferable to have the SYN|ACK be sent along with
+	 * application response data, or failing that, when the DELACK timer
+	 * expires.
+	 */
+	if (IS_FASTOPEN(tp->t_flags) &&
+	    (tp->t_state == TCPS_SYN_RECEIVED))
+		goto out;
+
 	tcp_output(tp);
 
 out:

third_party/tcplp/bsdtcp/tcp_var.h

@@ -50,6 +50,8 @@
 #include "types.h"
 #include "ip6.h"
 
+#define TCP_RFC7413
+
 /* Implement byte-order-specific functions using OpenThread. */
 uint16_t tcplp_sys_hostswap16(uint16_t hostport);
 uint32_t tcplp_sys_hostswap32(uint32_t hostport);
@@ -351,6 +353,14 @@ struct tcpcb {
 
 //	uint32_t t_ispare[8];		/* 5 UTO, 3 TBD */
 //	void	*t_pspare2[4];		/* 1 TCP_SIGNATURE, 3 TBD */
+
+    /* samkumar: TFO fields from FreeBSD 12.0. */
+	uint64_t t_tfo_client_cookie_len; /* TCP Fast Open client cookie length */
+//	unsigned int *t_tfo_pending;	/* TCP Fast Open server pending counter */
+	union {
+		uint8_t client[TCP_FASTOPEN_MAX_COOKIE_LEN];
+		uint64_t server;
+	} t_tfo_cookie;			/* TCP Fast Open cookie to send */
 #if 0
 #if defined(_KERNEL) && defined(TCPPCAP)
 	struct mbufq t_inpkts;		/* List of saved input packets. */
@@ -384,7 +394,7 @@ void	cc_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type);
 
 /* Added, since there is no header file for tcp_usrreq.c. */
 int tcp6_usr_connect(struct tcpcb* tp, struct sockaddr_in6* sinp6);
-int tcp_usr_send(struct tcpcb* tp, int moretocome, struct otLinkedBuffer* data, size_t extendby);
+int tcp_usr_send(struct tcpcb* tp, int moretocome, struct otLinkedBuffer* data, size_t extendby, struct sockaddr_in6* nam);
 int tcp_usr_rcvd(struct tcpcb* tp);
 int tcp_usr_shutdown(struct tcpcb* tp);
 void tcp_usr_abort(struct tcpcb* tp);
@@ -421,6 +431,7 @@ void tcp_usr_abort(struct tcpcb* tp);
 #define	TF_ECN_SND_ECE	0x10000000	/* ECN ECE in queue */
 #define	TF_CONGRECOVERY	0x20000000	/* congestion recovery mode */
 #define	TF_WASCRECOVERY	0x40000000	/* was in congestion recovery */
+#define	TF_FASTOPEN	0x80000000	/* TCP Fast Open indication */
 
 #define	IN_FASTRECOVERY(t_flags)	(t_flags & TF_FASTRECOVERY)
 #define	ENTER_FASTRECOVERY(t_flags)	t_flags |= TF_FASTRECOVERY
@@ -434,6 +445,12 @@ void tcp_usr_abort(struct tcpcb* tp);
 #define	ENTER_RECOVERY(t_flags) t_flags |= (TF_CONGRECOVERY | TF_FASTRECOVERY)
 #define	EXIT_RECOVERY(t_flags) t_flags &= ~(TF_CONGRECOVERY | TF_FASTRECOVERY)
 
+#ifndef TCP_RFC7413
+#define	IS_FASTOPEN(t_flags)		(false)
+#else
+#define	IS_FASTOPEN(t_flags)		(t_flags & TF_FASTOPEN)
+#endif
+
 #define	BYTES_THIS_ACK(tp, th)	(th->th_ack - tp->snd_una)
 
 /*
@@ -480,14 +497,17 @@ struct tcpopt {
 #define	TOF_TS		0x0010		/* timestamp */
 #define	TOF_SIGNATURE	0x0040		/* TCP-MD5 signature option (RFC2385) */
 #define	TOF_SACK	0x0080		/* Peer sent SACK option */
-#define	TOF_MAXOPT	0x0100
+#define	TOF_FASTOPEN	0x0100		/* TCP Fast Open (TFO) cookie */
+#define	TOF_MAXOPT	0x0200
 	u_int32_t	to_tsval;	/* new timestamp */
 	u_int32_t	to_tsecr;	/* reflected timestamp */
 	uint8_t		*to_sacks;	/* pointer to the first SACK blocks */
 	uint8_t		*to_signature;	/* pointer to the TCP-MD5 signature */
+	u_int8_t	*to_tfo_cookie; /* pointer to the TFO cookie */
 	u_int16_t	to_mss;		/* maximum segment size */
 	u_int8_t	to_wscale;	/* window scaling */
 	u_int8_t	to_nsacks;	/* number of SACK blocks */
+	u_int8_t	to_tfo_len;	/* TFO cookie length */
 	u_int32_t	to_spare;	/* UTO */
 };
 

third_party/tcplp/tcplp.h

@@ -55,11 +55,12 @@ extern "C" {
 
 struct tcplp_signals
 {
-    uint32_t links_popped;
-    uint32_t bytes_acked;
-    bool     conn_established;
-    bool     recvbuf_added;
-    bool     rcvd_fin;
+    struct tcpcb* accepted_connection;
+    uint32_t      links_popped;
+    uint32_t      bytes_acked;
+    bool          conn_established;
+    bool          recvbuf_added;
+    bool          rcvd_fin;
 };
 
 /*