From 6b6c67eebf12bf795bf03ee3503e21b0a5e6cab2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Krzysztof=20Kopy=C5=9Bci=C5=84ski?=
 <krzysztof.kopyscinski@codecoup.pl>
Date: Tue, 3 Nov 2020 08:11:20 +0100
Subject: [PATCH] mesh: Slab based segmentation handling

Allocates segmented message buffers as slabs in a common pool for RX and
TX. This reduces memory requirements for both TX and RX, as TX messages
can be stored without the network and advertising buffer overhead, and
RX can use only the slabs it needs, instead of allocating a full size
segmented message. This approach also removes the need for decrypting
the segments for each retransmission, reducing overall processing load.

Slab based segmentation for tx also introduces queuing of segmented
messages, which allows the application layer to send multiple messages
to the same destination without violating Bluetooth Mesh specification
v1.0.1, section 3.6.4.1. This mechanism is provided through a flag that
blocks segmented messages to a destination which a message is already
being sent to until the previous message finishes.

This changes the SDU size configuration to a symmetrical
RX_SEG_MAX/TX_SEG_MAX pair of configurations, plus a new segment pool
side configuration. It also removes the binding between the TX_SEG_MAX
config and the advertising buffers, reducing the minimum advertising
buffer count from 6 to 3.

this is port of 901b5b2b2b0a33f5426f621277e7238e8b83740b and
7443c1bd3ded598b56db3c931614072098af84e1
---
 nimble/host/mesh/include/mesh/glue.h          |  58 ++
 nimble/host/mesh/src/friend.c                 |   4 +-
 nimble/host/mesh/src/glue.c                   |  25 +
 nimble/host/mesh/src/lpn.c                    |  10 +-
 nimble/host/mesh/src/net.c                    |  56 --
 nimble/host/mesh/src/net.h                    |   4 -
 nimble/host/mesh/src/transport.c              | 811 +++++++++++-------
 nimble/host/mesh/src/transport.h              |  10 +-
 nimble/host/mesh/syscfg.yml                   |  53 +-
 .../linux_blemesh/include/syscfg/syscfg.h     |  12 +-
 10 files changed, 626 insertions(+), 417 deletions(-)

diff --git a/nimble/host/mesh/include/mesh/glue.h b/nimble/host/mesh/include/mesh/glue.h
index a84c53e2c..643a36b60 100644
--- a/nimble/host/mesh/include/mesh/glue.h
+++ b/nimble/host/mesh/include/mesh/glue.h
@@ -242,6 +242,10 @@ static inline void net_buf_simple_init(struct os_mbuf *buf,
     buf->om_len = 0;
 }
 
+#define net_buf_simple_init_with_data(buf, data, size)  \
+    buf = NET_BUF_SIMPLE(size); \
+    os_mbuf_copyinto(buf, 0, data, size);
+
 static inline void net_buf_simple_reset(struct os_mbuf *om)
 {
     om->om_len = 0;
@@ -413,6 +417,7 @@ static inline unsigned int find_msb_set(u32_t op)
 
 #define CONFIG_BT_MESH_LPN_GROUPS           MYNEWT_VAL(BLE_MESH_LPN_GROUPS)
 #define CONFIG_BT_MESH_ADV_BUF_COUNT        MYNEWT_VAL(BLE_MESH_ADV_BUF_COUNT)
+#define CONFIG_BT_MESH_SEG_BUFS             MYNEWT_VAL(BLE_MESH_SEG_BUFS )
 #define CONFIG_BT_MESH_FRIEND_QUEUE_SIZE    MYNEWT_VAL(BLE_MESH_FRIEND_QUEUE_SIZE)
 #define CONFIG_BT_MESH_FRIEND_RECV_WIN      MYNEWT_VAL(BLE_MESH_FRIEND_RECV_WIN)
 #define CONFIG_BT_MESH_LPN_POLL_TIMEOUT     MYNEWT_VAL(BLE_MESH_LPN_POLL_TIMEOUT)
@@ -427,7 +432,10 @@ static inline unsigned int find_msb_set(u32_t op)
 #define CONFIG_BT_MESH_STORE_TIMEOUT        MYNEWT_VAL(BLE_MESH_STORE_TIMEOUT)
 #define CONFIG_BT_MESH_IVU_DIVIDER          MYNEWT_VAL(BLE_MESH_IVU_DIVIDER)
 #define CONFIG_BT_DEVICE_NAME               MYNEWT_VAL(BLE_MESH_DEVICE_NAME)
+#define CONFIG_BT_RX_SEG_MAX                MYNEWT_VAL(BLE_MESH_RX_SEG_MAX)
 #define CONFIG_BT_MESH_TX_SEG_MAX           MYNEWT_VAL(BLE_MESH_TX_SEG_MAX)
+#define CONFIG_BT_MESH_RX_SEG_MAX           MYNEWT_VAL(BLE_MESH_RX_SEG_MAX)
+#define CONFIG_BT_MESH_RX_SEG_MSG_COUNT     MYNEWT_VAL(BLE_MESH_RX_SEG_MSG_COUNT)
 #define CONFIG_BT_MESH_LABEL_COUNT          MYNEWT_VAL(BLE_MESH_LABEL_COUNT)
 #define CONFIG_BT_MESH_NODE_COUNT           MYNEWT_VAL(BLE_MESH_CDB_NODE_COUNT)
 #define CONFIG_BT_MESH_CDB                  BLE_MESH_CDB
@@ -515,6 +523,56 @@ settings_load(void)
 
 #define BUILD_ASSERT(cond) _Static_assert(cond, "")
 
+
+/* Memory slabs/blocks */
+
+/** Memory slab structure */
+struct k_mem_slab {
+    /**
+     * _wait_q_t is not required now, as we don't implement zephyr timeouts -
+     * if slab couldn't be allocated, we simply return error  
+     */
+    uint32_t num_blocks; /** number of memory blocks available for allocation */
+    size_t block_size; /** size of single block */
+	/**
+     * buffer for blocks - must be alligned to N-byte, where N is a power of 2.
+     * Minimal size of buffer is num_blocks * block_size
+     */
+    char *buffer;
+	char *free_list; /** list of free memory blocks */
+	uint32_t num_used; /** count of used memory blocks */
+};
+
+struct k_mem_block_id {
+	uint32_t pool : 8;
+	uint32_t level : 4;
+	uint32_t block : 20;
+};
+
+struct k_mem_block {
+	void *data;
+	struct k_mem_block_id id;
+};
+
+extern void k_mem_slab_free(struct k_mem_slab *slab, void **mem);
+extern int k_mem_slab_alloc(struct k_mem_slab *slab, void **mem);
+static inline uint32_t k_mem_slab_num_free_get(struct k_mem_slab *slab)
+{
+	return slab->num_blocks - slab->num_used;
+}
+
+/** slab_align must be power of 2 */
+#define K_MEM_SLAB_DEFINE(name, slab_block_size, slab_num_blocks, slab_align) \
+    char _k_mem_slab_buf_##name[slab_num_blocks * slab_block_size]; \
+    struct k_mem_slab name = { \
+        slab_num_blocks, \
+        slab_block_size, \
+        _k_mem_slab_buf_##name, \
+        NULL, \
+        0 \
+    };
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/nimble/host/mesh/src/friend.c b/nimble/host/mesh/src/friend.c
index 75ae0b20e..8ab4f5895 100644
--- a/nimble/host/mesh/src/friend.c
+++ b/nimble/host/mesh/src/friend.c
@@ -259,7 +259,7 @@ int bt_mesh_friend_clear(struct bt_mesh_net_rx *rx, struct os_mbuf *buf)
 	cfm.lpn_counter = msg->lpn_counter;
 
 	bt_mesh_ctl_send(&tx, TRANS_CTL_OP_FRIEND_CLEAR_CFM, &cfm,
-			 sizeof(cfm), NULL, NULL, NULL);
+			 sizeof(cfm), NULL, NULL);
 
 	friend_clear(frnd);
 
@@ -784,7 +784,7 @@ static void send_friend_clear(struct bt_mesh_friend *frnd)
 	BT_DBG("");
 
 	bt_mesh_ctl_send(&tx, TRANS_CTL_OP_FRIEND_CLEAR, &req,
-			 sizeof(req), NULL, &clear_sent_cb, frnd);
+			 sizeof(req), &clear_sent_cb, frnd);
 }
 
 static void clear_timeout(struct ble_npl_event *work)
diff --git a/nimble/host/mesh/src/glue.c b/nimble/host/mesh/src/glue.c
index a4e9ca85f..82025bd55 100644
--- a/nimble/host/mesh/src/glue.c
+++ b/nimble/host/mesh/src/glue.c
@@ -870,6 +870,31 @@ void net_buf_slist_merge_slist(struct net_buf_slist_t *list,
 	}
 }
 
+/** Memory slab methods */
+extern void  k_mem_slab_free(struct k_mem_slab *slab, void **mem)
+{
+    **(char ***)mem = slab->free_list;
+    slab->free_list = *(char **)mem;
+    slab->num_used--;
+}
+
+extern int k_mem_slab_alloc(struct k_mem_slab *slab, void **mem)
+{
+	int result;
+
+	if (slab->free_list != NULL) {
+		/* take a free block */
+		*mem = slab->free_list;
+		slab->free_list = *(char **)(slab->free_list);
+		slab->num_used++;
+		result = 0;
+	} else {
+		*mem = NULL;
+		result = -ENOMEM;
+	}
+	return result;
+}
+
 #if MYNEWT_VAL(BLE_MESH_SETTINGS)
 
 int settings_bytes_from_str(char *val_str, void *vp, int *len)
diff --git a/nimble/host/mesh/src/lpn.c b/nimble/host/mesh/src/lpn.c
index 39e54331f..49c366e8d 100644
--- a/nimble/host/mesh/src/lpn.c
+++ b/nimble/host/mesh/src/lpn.c
@@ -190,7 +190,7 @@ static int send_friend_clear(void)
 	BT_DBG("");
 
 	return bt_mesh_ctl_send(&tx, TRANS_CTL_OP_FRIEND_CLEAR, &req,
-				sizeof(req), NULL, &clear_sent_cb, NULL);
+				sizeof(req), &clear_sent_cb, NULL);
 }
 
 static void clear_friendship(bool force, bool disable)
@@ -308,7 +308,7 @@ static int send_friend_req(struct bt_mesh_lpn *lpn)
 	BT_DBG("");
 
 	return bt_mesh_ctl_send(&tx, TRANS_CTL_OP_FRIEND_REQ, &req,
-				sizeof(req), NULL, &friend_req_sent_cb, NULL);
+				sizeof(req), &friend_req_sent_cb, NULL);
 }
 
 static void req_sent(u16_t duration, int err, void *user_data)
@@ -378,7 +378,7 @@ static int send_friend_poll(void)
 	}
 
 	err = bt_mesh_ctl_send(&tx, TRANS_CTL_OP_FRIEND_POLL, &fsn, 1,
-			       NULL, &req_sent_cb, NULL);
+			       		   &req_sent_cb, NULL);
 	if (err == 0) {
 		lpn->pending_poll = 0;
 		lpn->sent_req = TRANS_CTL_OP_FRIEND_POLL;
@@ -689,8 +689,8 @@ static bool sub_update(u8_t op)
 
 	req.xact = lpn->xact_next++;
 
-	if (bt_mesh_ctl_send(&tx, op, &req, 1 + g * 2, NULL,
-			     &req_sent_cb, NULL) < 0) {
+	if (bt_mesh_ctl_send(&tx, op, &req, 1 + g * 2, 
+						 &req_sent_cb, NULL) < 0) {
 		group_zero(lpn->pending);
 		return false;
 	}
diff --git a/nimble/host/mesh/src/net.c b/nimble/host/mesh/src/net.c
index 276082741..ca561a17e 100644
--- a/nimble/host/mesh/src/net.c
+++ b/nimble/host/mesh/src/net.c
@@ -724,62 +724,6 @@ u32_t bt_mesh_next_seq(void)
 	return seq;
 }
 
-int bt_mesh_net_resend(struct bt_mesh_subnet *sub, struct os_mbuf *buf,
-		       bool new_key, const struct bt_mesh_send_cb *cb,
-		       void *cb_data)
-{
-	const u8_t *enc, *priv;
-	u32_t seq;
-	u16_t dst;
-	int err;
-
-	BT_DBG("net_idx 0x%04x new_key %u len %u", sub->net_idx, new_key,
-	       buf->om_len);
-
-	enc = sub->keys[new_key].enc;
-	priv = sub->keys[new_key].privacy;
-
-	err = bt_mesh_net_obfuscate(buf->om_data, BT_MESH_NET_IVI_TX, priv);
-	if (err) {
-		BT_ERR("deobfuscate failed (err %d)", err);
-		return err;
-	}
-
-	err = bt_mesh_net_decrypt(enc, buf, BT_MESH_NET_IVI_TX, false);
-	if (err) {
-		BT_ERR("decrypt failed (err %d)", err);
-		return err;
-	}
-
-	seq = bt_mesh_next_seq();
-	sys_put_be24(seq, &buf->om_data[2]);
-
-	/* Get destination, in case it's a proxy client */
-	dst = DST(buf->om_data);
-
-	err = bt_mesh_net_encrypt(enc, buf, BT_MESH_NET_IVI_TX, false);
-	if (err) {
-		BT_ERR("encrypt failed (err %d)", err);
-		return err;
-	}
-
-	err = bt_mesh_net_obfuscate(buf->om_data, BT_MESH_NET_IVI_TX, priv);
-	if (err) {
-		BT_ERR("obfuscate failed (err %d)", err);
-		return err;
-	}
-
-	if (IS_ENABLED(CONFIG_BT_MESH_GATT_PROXY) &&
-		bt_mesh_proxy_relay(buf, dst) &&
-	    BT_MESH_ADDR_IS_UNICAST(dst)) {
-			send_cb_finalize(cb, cb_data);
-	} else {
-		bt_mesh_adv_send(buf, cb, cb_data);
-	}
-
-	return 0;
-}
-
 static void bt_mesh_net_local(struct ble_npl_event *work)
 {
 	struct os_mbuf *buf;
diff --git a/nimble/host/mesh/src/net.h b/nimble/host/mesh/src/net.h
index cef09bb2c..529314ce6 100644
--- a/nimble/host/mesh/src/net.h
+++ b/nimble/host/mesh/src/net.h
@@ -344,10 +344,6 @@ int bt_mesh_net_encode(struct bt_mesh_net_tx *tx, struct os_mbuf *buf,
 int bt_mesh_net_send(struct bt_mesh_net_tx *tx, struct os_mbuf *buf,
 		     const struct bt_mesh_send_cb *cb, void *cb_data);
 
-int bt_mesh_net_resend(struct bt_mesh_subnet *sub, struct os_mbuf *buf,
-		       bool new_key, const struct bt_mesh_send_cb *cb,
-		       void *cb_data);
-
 int bt_mesh_net_decode(struct os_mbuf *data, enum bt_mesh_net_if net_if,
 		       struct bt_mesh_net_rx *rx, struct os_mbuf *buf);
 
diff --git a/nimble/host/mesh/src/transport.c b/nimble/host/mesh/src/transport.c
index 8d68dfe1c..36fd513c0 100644
--- a/nimble/host/mesh/src/transport.c
+++ b/nimble/host/mesh/src/transport.c
@@ -13,6 +13,7 @@
 #include <string.h>
 
 #include "mesh/mesh.h"
+#include "mesh/glue.h"
 #include "mesh_priv.h"
 
 #include "crypto.h"
@@ -26,13 +27,6 @@
 #include "transport.h"
 #include "testing.h"
 
-/* The transport layer needs at least three buffers for itself to avoid
- * deadlocks. Ensure that there are a sufficient number of advertising
- * buffers available compared to the maximum supported outgoing segment
- * count.
- */
-BUILD_ASSERT(CONFIG_BT_MESH_ADV_BUF_COUNT >= (CONFIG_BT_MESH_TX_SEG_MAX + 3));
-
 #define AID_MASK                    ((u8_t)(BIT_MASK(6)))
 
 #define SEG(data)                   ((data)[0] >> 7)
@@ -40,7 +34,7 @@ BUILD_ASSERT(CONFIG_BT_MESH_ADV_BUF_COUNT >= (CONFIG_BT_MESH_TX_SEG_MAX + 3));
 #define AID(data)                   ((data)[0] & AID_MASK)
 #define ASZMIC(data)                (((data)[1] >> 7) & 1)
 
-#define APP_MIC_LEN(aszmic)         ((aszmic) ? 8 : 4)
+#define APP_MIC_LEN(aszmic) ((aszmic) ? BT_MESH_MIC_LONG : BT_MESH_MIC_SHORT)
 
 #define UNSEG_HDR(akf, aid)         ((akf << 6) | (aid & AID_MASK))
 #define SEG_HDR(akf, aid)           (UNSEG_HDR(akf, aid) | 0x80)
@@ -72,11 +66,21 @@ BUILD_ASSERT(CONFIG_BT_MESH_ADV_BUF_COUNT >= (CONFIG_BT_MESH_TX_SEG_MAX + 3));
 
 static struct seg_tx {
 	struct bt_mesh_subnet   *sub;
-	struct os_mbuf          *seg[CONFIG_BT_MESH_TX_SEG_MAX];
+	void		            *seg[CONFIG_BT_MESH_TX_SEG_MAX];
 	u64_t                    seq_auth;
+	u16_t                    src;
 	u16_t                    dst;
+	u16_t                    len;
+	u8_t                     hdr;
+	u8_t                     xmit;
 	u8_t                     seg_n:5,       /* Last segment index */
-				 new_key:1;     /* New/old key */
+							 ctl:1,
+							 aszmic:1,
+							 friend_cred:1;
+	u8_t                     seg_o:5,
+							 started:1,     /* Start cb called */
+							 sending:1,     /* Sending is in progress */
+							 blocked:1;     /* Blocked by ongoing tx */
 	u8_t                     nack_count;    /* Number of unacked segs */
 	u8_t                     ttl;
 	u8_t                     seg_pending:5, /* Number of segments pending */
@@ -88,22 +92,23 @@ static struct seg_tx {
 
 static struct seg_rx {
 	struct bt_mesh_subnet   *sub;
+	void                    *seg[CONFIG_BT_MESH_TX_SEG_MAX];
 	u64_t                    seq_auth;
-	u8_t                     seg_n:5,
-				 ctl:1,
-				 in_use:1,
-				 obo:1;
-	u8_t                     hdr;
-	u8_t                     ttl;
 	u16_t                    src;
 	u16_t                    dst;
+	u16_t                    len;
+	u8_t                     hdr;
+	u8_t                     seg_n:5,
+							 ctl:1,
+							 in_use:1,
+							 obo:1;
+	u8_t                     ttl;
 	u32_t                    block;
 	u32_t                    last;
 	struct k_delayed_work    ack;
-	struct os_mbuf    *buf;
-} seg_rx[MYNEWT_VAL(BLE_MESH_RX_SEG_MSG_COUNT)] = {
-	[0 ... (MYNEWT_VAL(BLE_MESH_RX_SEG_MSG_COUNT) - 1)] = { 0 },
-};
+} seg_rx[CONFIG_BT_MESH_RX_SEG_MSG_COUNT];
+
+K_MEM_SLAB_DEFINE(segs, BT_MESH_APP_SEG_SDU_MAX, CONFIG_BT_MESH_SEG_BUFS, 4);
 
 static u16_t hb_sub_dst = BT_MESH_ADDR_UNASSIGNED;
 
@@ -113,13 +118,11 @@ void bt_mesh_set_hb_sub_dst(u16_t addr)
 }
 
 static int send_unseg(struct bt_mesh_net_tx *tx, struct os_mbuf *sdu,
-		      const struct bt_mesh_send_cb *cb, void *cb_data)
+		      const struct bt_mesh_send_cb *cb, void *cb_data,
+		      const u8_t *ctl_op)
 {
 	struct os_mbuf *buf;
 
-	BT_DBG("src 0x%04x dst 0x%04x app_idx 0x%04x sdu_len %u",
-	       tx->src, tx->ctx->addr, tx->ctx->app_idx, sdu->om_len);
-
 	buf = bt_mesh_adv_create(BT_MESH_ADV_DATA, tx->xmit, BUF_TIMEOUT);
 	if (!buf) {
 		BT_ERR("Out of network buffers");
@@ -128,7 +131,9 @@ static int send_unseg(struct bt_mesh_net_tx *tx, struct os_mbuf *sdu,
 
 	net_buf_reserve(buf, BT_MESH_NET_HDR_LEN);
 
-	if (BT_MESH_IS_DEV_KEY(tx->ctx->app_idx)) {
+	if (ctl_op) {
+		net_buf_add_u8(buf, TRANS_CTL_HDR(*ctl_op, 0));
+	} else if (BT_MESH_IS_DEV_KEY(tx->ctx->app_idx)) {
 		net_buf_add_u8(buf, UNSEG_HDR(0, 0));
 	} else {
 		net_buf_add_u8(buf, UNSEG_HDR(1, tx->aid));
@@ -166,6 +171,15 @@ send:
 	return bt_mesh_net_send(tx, buf, cb, cb_data);
 }
 
+static inline u8_t seg_len(bool ctl)
+{
+	if (ctl) {
+		return BT_MESH_CTL_SEG_SDU_MAX;
+	} else {
+		return BT_MESH_APP_SEG_SDU_MAX;
+	}
+}
+
 bool bt_mesh_tx_in_progress(void)
 {
 	int i;
@@ -181,12 +195,39 @@ bool bt_mesh_tx_in_progress(void)
 
 static void seg_tx_done(struct seg_tx *tx, u8_t seg_idx)
 {
-	BT_MESH_ADV(tx->seg[seg_idx])->busy = 0U;
-	net_buf_unref(tx->seg[seg_idx]);
+	k_mem_slab_free(&segs, (void **)&tx->seg[seg_idx]);
 	tx->seg[seg_idx] = NULL;
 	tx->nack_count--;
 }
 
+static bool seg_tx_blocks(struct seg_tx *tx, u16_t src, u16_t dst)
+{
+	return (tx->src == src) && (tx->dst == dst);
+}
+
+static void seg_tx_unblock_check(struct seg_tx *tx)
+{
+	struct seg_tx *blocked = NULL;
+	int i;
+
+	/* Unblock the first blocked tx with the same params. */
+	for (i = 0; i < ARRAY_SIZE(seg_tx); ++i) {
+		if (&seg_tx[i] != tx &&
+		    seg_tx[i].blocked &&
+		    seg_tx_blocks(tx, seg_tx[i].src, seg_tx[i].dst) &&
+		    (!blocked || seg_tx[i].seq_auth < blocked->seq_auth)) {
+			blocked = &seg_tx[i];
+		}
+	}
+
+	if (blocked) {
+		BT_DBG("Unblocked 0x%04x",
+		       (u16_t)(blocked->seq_auth & TRANS_SEQ_ZERO_MASK));
+		blocked->blocked = false;
+		k_delayed_work_submit(&blocked->retransmit, 0);
+	}
+}
+
 static void seg_tx_reset(struct seg_tx *tx)
 {
 	int i;
@@ -197,7 +238,9 @@ static void seg_tx_reset(struct seg_tx *tx)
 	tx->cb_data = NULL;
 	tx->seq_auth = 0;
 	tx->sub = NULL;
+	tx->src = BT_MESH_ADDR_UNASSIGNED;
 	tx->dst = BT_MESH_ADDR_UNASSIGNED;
+	tx->blocked = false;
 
 	if (!tx->nack_count) {
 		return;
@@ -211,6 +254,8 @@ static void seg_tx_reset(struct seg_tx *tx)
 		seg_tx_done(tx, i);
 	}
 
+	tx->nack_count = 0;
+
 	if (atomic_test_and_clear_bit(bt_mesh.flags, BT_MESH_IVU_PENDING)) {
 		BT_DBG("Proceding with pending IV Update");
 
@@ -228,6 +273,8 @@ static inline void seg_tx_complete(struct seg_tx *tx, int err)
 	const struct bt_mesh_send_cb *cb = tx->cb;
 	void *cb_data = tx->cb_data;
 
+	seg_tx_unblock_check(tx);
+
 	seg_tx_reset(tx);
 
 	if (cb && cb->end) {
@@ -237,32 +284,49 @@ static inline void seg_tx_complete(struct seg_tx *tx, int err)
 
 static void schedule_retransmit(struct seg_tx *tx)
 {
+	if (!tx->nack_count) {
+		return;
+	}
+
 	if (--tx->seg_pending) {
 		return;
 	}
 
-	if (!BT_MESH_ADDR_IS_UNICAST(tx->dst) && !tx->attempts) {
+	/* Group address sending should stop when the last segment goes out for
+	 * the last time. Normally, this happens after the attempts counter has
+	 * been set to 0. For self-rx, this callback is called inline, which
+	 * means that the attempt counter is still 1. Since this is group based
+	 * sar, we know that segments aren't removed by acks, so seg_o will be
+	 * the last segment that goes out.
+	 */
+	if (!BT_MESH_ADDR_IS_UNICAST(tx->dst) &&
+	    (!tx->attempts ||
+	     (tx->sending && tx->attempts == 1 && tx->seg_n == tx->seg_o))) {
 		BT_DBG("SDU TX complete");
 		seg_tx_complete(tx, 0);
 		return;
 	}
 
-	k_delayed_work_submit(&tx->retransmit, SEG_RETRANSMIT_TIMEOUT(tx));
-}
-
-static void seg_first_send_start(u16_t duration, int err, void *user_data)
-{
-	struct seg_tx *tx = user_data;
-
-	if (tx->cb && tx->cb->start) {
-		tx->cb->start(duration, err, tx->cb_data);
-	}
+	/* If we haven't gone through all the segments for this attempt yet,
+	 * (likely because of a buffer allocation failure or because we
+	 * called this from inside bt_mesh_net_send), we should continue the
+	 * retransmit immediately, as we just freed up a tx buffer.
+	 */
+	k_delayed_work_submit(&tx->retransmit,
+			      (tx->sending || !tx->seg_o) ?
+				      SEG_RETRANSMIT_TIMEOUT(tx) :
+				      0);
 }
 
 static void seg_send_start(u16_t duration, int err, void *user_data)
 {
 	struct seg_tx *tx = user_data;
 
+	if (!tx->started && tx->cb && tx->cb->start) {
+		tx->cb->start(duration, err, tx->cb_data);
+		tx->started = 1U;
+	}
+
 	/* If there's an error in transmitting the 'sent' callback will never
 	 * be called. Make sure that we kick the retransmit timer also in this
 	 * case since otherwise we risk the transmission of becoming stale.
@@ -279,52 +343,90 @@ static void seg_sent(int err, void *user_data)
 	schedule_retransmit(tx);
 }
 
-static const struct bt_mesh_send_cb first_sent_cb = {
-	.start = seg_first_send_start,
-	.end = seg_sent,
-};
-
 static const struct bt_mesh_send_cb seg_sent_cb = {
 	.start = seg_send_start,
 	.end = seg_sent,
 };
 
+static void seg_tx_buf_build(struct seg_tx *tx, u8_t seg_o,
+			     struct os_mbuf *buf)
+{
+	u16_t seq_zero = tx->seq_auth & TRANS_SEQ_ZERO_MASK;
+	u8_t len = MIN(seg_len(tx->ctl), tx->len - (seg_len(tx->ctl) * seg_o));
+
+	net_buf_simple_add_u8(buf, tx->hdr);
+	net_buf_simple_add_u8(buf, (tx->aszmic << 7) | seq_zero >> 6);
+	net_buf_simple_add_u8(buf, (((seq_zero & 0x3f) << 2) | (seg_o >> 3)));
+	net_buf_simple_add_u8(buf, ((seg_o & 0x07) << 5) | tx->seg_n);
+	net_buf_simple_add_mem(buf, tx->seg[seg_o], len);
+}
+
 static void seg_tx_send_unacked(struct seg_tx *tx)
 {
-	int i, err;
+	struct bt_mesh_msg_ctx ctx = {
+		.net_idx = tx->sub->net_idx,
+		/* App idx only used by network to detect control messages: */
+		.app_idx = (tx->ctl ? BT_MESH_KEY_UNUSED : 0),
+		.addr = tx->dst,
+		.send_rel = true,
+		.send_ttl = tx->ttl,
+	};
+	struct bt_mesh_net_tx net_tx = {
+		.sub = tx->sub,
+		.ctx = &ctx,
+		.src = tx->src,
+		.xmit = tx->xmit,
+		.friend_cred = tx->friend_cred,
+		.aid = tx->hdr & AID_MASK,
+	};
 
-	if (!(tx->attempts--)) {
+	if (!tx->attempts) {
 		BT_ERR("Ran out of retransmit attempts");
 		seg_tx_complete(tx, -ETIMEDOUT);
 		return;
 	}
 
-	BT_DBG("Attempts: %u", tx->attempts);
+	BT_DBG("SeqZero: 0x%04x Attempts: %u",
+	       (u16_t)(tx->seq_auth & TRANS_SEQ_ZERO_MASK), tx->attempts);
 
-	for (i = 0; i <= tx->seg_n; i++) {
-		struct os_mbuf *seg = tx->seg[i];
+	tx->sending = 1U;
 
+	for (; tx->seg_o <= tx->seg_n; tx->seg_o++) {
+		struct os_mbuf *seg;
+		int err;
+
+		if (!tx->seg[tx->seg_o]) {
+			continue;
+		}
+
+		seg = bt_mesh_adv_create(BT_MESH_ADV_DATA, tx->xmit,
+					 BUF_TIMEOUT);
 		if (!seg) {
-			continue;
+			BT_DBG("Allocating segment failed");
+			tx->sending = 0U;
+			return;
 		}
 
-		if (BT_MESH_ADV(seg)->busy) {
-			BT_DBG("Skipping segment that's still advertising");
-			continue;
-		}
+		net_buf_reserve(seg, BT_MESH_NET_HDR_LEN);
+		seg_tx_buf_build(tx, tx->seg_o, seg);
 
 		tx->seg_pending++;
 
-		BT_DBG("resending %u/%u", i, tx->seg_n);
+		BT_DBG("Sending %u/%u", tx->seg_o, tx->seg_n);
 
-		err = bt_mesh_net_resend(tx->sub, seg, tx->new_key,
-					 &seg_sent_cb, tx);
+		err = bt_mesh_net_send(&net_tx, seg, &seg_sent_cb, tx);
 		if (err) {
 			BT_ERR("Sending segment failed");
 			seg_tx_complete(tx, -EIO);
+			BT_DBG("Sending segment failed");
+			tx->seg_pending--;
+			tx->sending = 0U;
 			return;
 		}
 	}
+	tx->sending = 0U;
+	tx->seg_o = 0U;
+	tx->attempts--;
 }
 
 static void seg_retransmit(struct ble_npl_event *work)
@@ -334,31 +436,25 @@ static void seg_retransmit(struct ble_npl_event *work)
 }
 
 static int send_seg(struct bt_mesh_net_tx *net_tx, struct os_mbuf *sdu,
-		    const struct bt_mesh_send_cb *cb, void *cb_data)
+		    const struct bt_mesh_send_cb *cb, void *cb_data,
+		    u8_t *ctl_op)
 {
-	u8_t seg_hdr, seg_o;
-	u16_t seq_zero;
+{
+	bool blocked = false;
 	struct seg_tx *tx;
+	u8_t seg_o;
 	int i;
 
 	BT_DBG("src 0x%04x dst 0x%04x app_idx 0x%04x aszmic %u sdu_len %u",
 	       net_tx->src, net_tx->ctx->addr, net_tx->ctx->app_idx,
 	       net_tx->aszmic, sdu->om_len);
 
-	if (sdu->om_len < 1) {
-		BT_ERR("Zero-length SDU not allowed");
-		return -EINVAL;
-	}
-
-	if (sdu->om_len > BT_MESH_TX_SDU_MAX) {
-		BT_ERR("Not enough segment buffers for length %u", sdu->om_len);
-		return -EMSGSIZE;
-	}
-
 	for (tx = NULL, i = 0; i < ARRAY_SIZE(seg_tx); i++) {
-		if (!seg_tx[i].nack_count) {
+		if (seg_tx[i].nack_count) {
+			blocked |= seg_tx_blocks(&seg_tx[i], net_tx->src,
+						 net_tx->ctx->addr);
+		} else if (!tx) {
 			tx = &seg_tx[i];
-			break;
 		}
 	}
 
@@ -367,23 +463,32 @@ static int send_seg(struct bt_mesh_net_tx *net_tx, struct os_mbuf *sdu,
 		return -EBUSY;
 	}
 
-	if (BT_MESH_IS_DEV_KEY(net_tx->ctx->app_idx)) {
-		seg_hdr = SEG_HDR(0, 0);
+	if (ctl_op) {
+		tx->hdr = TRANS_CTL_HDR(*ctl_op, 1);
+	} else if (BT_MESH_IS_DEV_KEY(net_tx->ctx->app_idx)) {
+		tx->hdr = SEG_HDR(0, 0);
 	} else {
-		seg_hdr = SEG_HDR(1, net_tx->aid);
+		tx->hdr = SEG_HDR(1, net_tx->aid);
 	}
 
-	seg_o = 0;
+	tx->src = net_tx->src;
 	tx->dst = net_tx->ctx->addr;
-	tx->seg_n = (sdu->om_len - 1) / 12;
+	tx->seg_n = (sdu->om_len - 1) / seg_len(!!ctl_op);
+	tx->seg_o = 0;
+	tx->len = sdu->om_len;
 	tx->nack_count = tx->seg_n + 1;
 	tx->seq_auth = SEQ_AUTH(BT_MESH_NET_IVI_TX, bt_mesh.seq);
 	tx->sub = net_tx->sub;
-	tx->new_key = net_tx->sub->kr_flag;
 	tx->cb = cb;
 	tx->cb_data = cb_data;
 	tx->attempts = SEG_RETRANSMIT_ATTEMPTS;
 	tx->seg_pending = 0;
+	tx->xmit = net_tx->xmit;
+	tx->aszmic = net_tx->aszmic;
+	tx->friend_cred = net_tx->friend_cred;
+	tx->blocked = blocked;
+	tx->started = 0;
+	tx->ctl = !!ctl_op;
 
 	if (net_tx->ctx->send_ttl == BT_MESH_TTL_DEFAULT) {
 		tx->ttl = bt_mesh_default_ttl_get();
@@ -391,9 +496,8 @@ static int send_seg(struct bt_mesh_net_tx *net_tx, struct os_mbuf *sdu,
 		tx->ttl = net_tx->ctx->send_ttl;
 	}
 
-	seq_zero = tx->seq_auth & TRANS_SEQ_ZERO_MASK;
-
-	BT_DBG("SeqZero 0x%04x", seq_zero);
+	BT_DBG("SeqZero 0x%04x (segs: %u)",
+	       (u16_t)(tx->seq_auth & TRANS_SEQ_ZERO_MASK), tx->nack_count);
 
 	if (IS_ENABLED(CONFIG_BT_MESH_FRIEND) &&
 	    !bt_mesh_friend_queue_has_space(tx->sub->net_idx, net_tx->src,
@@ -407,65 +511,46 @@ static int send_seg(struct bt_mesh_net_tx *net_tx, struct os_mbuf *sdu,
 	}
 
 	for (seg_o = 0; sdu->om_len; seg_o++) {
-		struct os_mbuf *seg;
+		void *buf;
 		u16_t len;
 		int err;
 
-		seg = bt_mesh_adv_create(BT_MESH_ADV_DATA, net_tx->xmit,
-					 BUF_TIMEOUT);
-		if (!seg) {
+		err = k_mem_slab_alloc(&segs, &buf);
+		if (err) {
 			BT_ERR("Out of segment buffers");
 			seg_tx_reset(tx);
 			return -ENOBUFS;
 		}
 
-		net_buf_reserve(seg, BT_MESH_NET_HDR_LEN);
-
-		net_buf_add_u8(seg, seg_hdr);
-		net_buf_add_u8(seg, (net_tx->aszmic << 7) | seq_zero >> 6);
-		net_buf_add_u8(seg, (((seq_zero & 0x3f) << 2) |
-				     (seg_o >> 3)));
-		net_buf_add_u8(seg, ((seg_o & 0x07) << 5) | tx->seg_n);
-
-		len = min(sdu->om_len, 12);
-		net_buf_add_mem(seg, sdu->om_data, len);
-		net_buf_simple_pull(sdu, len);
+		len = MIN(sdu->om_len, seg_len(!!ctl_op));
+		memcpy(buf, net_buf_simple_pull_mem(sdu, len), len);
+		BT_DBG("seg %u: %s", seg_o, bt_hex(buf, len));
 
 		if (IS_ENABLED(CONFIG_BT_MESH_FRIEND)) {
 			enum bt_mesh_friend_pdu_type type;
 
+			struct os_mbuf *seg = NET_BUF_SIMPLE(16);
+			seg_tx_buf_build(tx, seg_o, seg);
+
 			if (seg_o == tx->seg_n) {
 				type = BT_MESH_FRIEND_PDU_COMPLETE;
 			} else {
 				type = BT_MESH_FRIEND_PDU_PARTIAL;
 			}
 
-			if (bt_mesh_friend_enqueue_tx(net_tx, type,
-						      &tx->seq_auth,
-						      tx->seg_n + 1,
-						      seg) &&
+			if (bt_mesh_friend_enqueue_tx(
+				    net_tx, type, ctl_op ? NULL : &tx->seq_auth,
+				    tx->seg_n + 1, seg) &&
 			    BT_MESH_ADDR_IS_UNICAST(net_tx->ctx->addr)) {
 				/* PDUs for a specific Friend should only go
 				 * out through the Friend Queue.
 				 */
-				net_buf_unref(seg);
+				k_mem_slab_free(&segs, &buf);
 				continue;
 			}
 		}
 
-		tx->seg[seg_o] = net_buf_ref(seg);
-
-		BT_DBG("Sending %u/%u", seg_o, tx->seg_n);
-		tx->seg_pending++;
-
-		err = bt_mesh_net_send(net_tx, seg,
-				       seg_o ? &seg_sent_cb : &first_sent_cb,
-				       tx);
-		if (err) {
-			BT_ERR("Sending segment failed");
-			seg_tx_reset(tx);
-			return err;
-		}
+		tx->seg[seg_o] = buf;
 	}
 
 	/* This can happen if segments only went into the Friend Queue */
@@ -477,6 +562,21 @@ static int send_seg(struct bt_mesh_net_tx *net_tx, struct os_mbuf *sdu,
 		 * with the Friend Queue.
 		 */
 		send_cb_finalize(cb, cb_data);
+		return 0;
+	}
+
+	if (blocked) {
+		/* Move the sequence number, so we don't end up creating
+		 * another segmented transmission with the same SeqZero while
+		 * this one is blocked.
+		 */
+		bt_mesh_next_seq();
+		BT_DBG("Blocked.");
+		return 0;
+	}
+
+	seg_tx_send_unacked(tx);
+
 	}
 
 	if (IS_ENABLED(CONFIG_BT_MESH_LOW_POWER) &&
@@ -511,12 +611,22 @@ int bt_mesh_trans_send(struct bt_mesh_net_tx *tx, struct os_mbuf *msg,
 	u8_t aid;
 	int err;
 
+	if (msg->om_len < 1) {
+		BT_ERR("Zero-length SDU not allowed");
+		return -EINVAL;
+	}
+
+	if (msg->om_len > BT_MESH_TX_SDU_MAX) {
+		BT_ERR("Not enough segment buffers for length %u", msg->om_len);
+		return -EMSGSIZE;
+	}
+
 	if (net_buf_simple_tailroom(msg) < 4) {
 		BT_ERR("Insufficient tailroom for Transport MIC");
 		return -EINVAL;
 	}
 
-	if (msg->om_len > 11) {
+	if (msg->om_len > BT_MESH_SDU_UNSEG_MAX) {
 		tx->ctx->send_rel = 1;
 		tx->ctx->send_rel = true;
 	}
@@ -553,9 +663,9 @@ int bt_mesh_trans_send(struct bt_mesh_net_tx *tx, struct os_mbuf *msg,
 	}
 
 	if (tx->ctx->send_rel) {
-		err = send_seg(tx, msg, cb, cb_data);
+		err = send_seg(tx, msg, cb, cb_data, NULL);
 	} else {
-		err = send_unseg(tx, msg, cb, cb_data);
+		err = send_unseg(tx, msg, cb, cb_data, NULL);
 	}
 
 	return err;
@@ -630,28 +740,118 @@ static bool is_replay(struct bt_mesh_net_rx *rx, struct bt_mesh_rpl **match)
 	return true;
 }
 
-static int sdu_recv(struct bt_mesh_net_rx *rx, u32_t seq, u8_t hdr,
-		    u8_t aszmic, struct os_mbuf *buf)
+static void seg_rx_assemble(struct seg_rx *rx, struct os_mbuf *buf,
+			    u8_t aszmic)
+{
+	int i;
+
+	net_buf_simple_reset(buf);
+
+	for (i = 0; i <= rx->seg_n; i++) {
+		net_buf_simple_add_mem(buf, rx->seg[i],
+				       MIN(seg_len(rx->ctl),
+					   rx->len - (i * seg_len(rx->ctl))));
+	}
+
+	/* Adjust the length to not contain the MIC at the end */
+	if (!rx->ctl) {
+		buf->om_len -= APP_MIC_LEN(aszmic);
+	}
+}
+
+static int remote_devkey_decrypt(struct bt_mesh_net_rx *rx, u32_t seq, u8_t *ad,
+				 u8_t aszmic, struct os_mbuf *buf,
+				 struct os_mbuf *sdu)
+{
+	struct bt_mesh_cdb_node *node;
+	int err;
+
+	if (!IS_ENABLED(CONFIG_BT_MESH_PROVISIONER)) {
+		return -ENOENT;
+	}
+
+	/* We will try our local devkey separately. */
+	if (bt_mesh_elem_find(rx->ctx.addr)) {
+		return -ENOENT;
+	}
+
+	/*
+	 * There is no way of knowing if we should use our
+	 * local DevKey or the remote DevKey to decrypt the
+	 * message so we must try both.
+	 */
+	node = bt_mesh_cdb_node_get(rx->ctx.addr);
+	if (node == NULL) {
+		BT_ERR("No node found for addr 0x%04x", rx->ctx.addr);
+		return -EINVAL;
+	}
+
+	err = bt_mesh_app_decrypt(node->dev_key, true, aszmic, buf, sdu, ad,
+				  rx->ctx.addr, rx->ctx.recv_dst, seq,
+				  BT_MESH_NET_IVI_RX(rx));
+	if (err) {
+		BT_DBG("Unable to decrypt with node DevKey");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int app_key_decrypt(struct bt_mesh_net_rx *rx,
+			   struct bt_mesh_app_key *key, u32_t seq, u8_t *ad,
+			   u8_t hdr, u8_t aszmic, struct os_mbuf *buf,
+			   struct os_mbuf *sdu)
+{
+	struct bt_mesh_app_keys *keys;
+	int err;
+
+	/* Check that this AppKey matches received net_idx */
+	if (key->net_idx != rx->sub->net_idx) {
+		return -EINVAL;
+	}
+
+	if (rx->new_key && key->updated) {
+		keys = &key->keys[1];
+	} else {
+		keys = &key->keys[0];
+	}
+
+	/* Check that the AppKey ID matches */
+	if (AID(&hdr) != keys->id) {
+		return -EINVAL;
+	}
+
+	err = bt_mesh_app_decrypt(keys->val, false, aszmic, buf, sdu, ad,
+				  rx->ctx.addr, rx->ctx.recv_dst, seq,
+				  BT_MESH_NET_IVI_RX(rx));
+	if (err) {
+		BT_WARN("Unable to decrypt with AppKey 0x%03x", key->app_idx);
+	}
+
+	return err;
+}
+
+static int sdu_recv_unseg(struct bt_mesh_net_rx *rx, u8_t hdr,
+			  struct os_mbuf *buf)
 {
 	struct os_mbuf *sdu =
-		NET_BUF_SIMPLE(MYNEWT_VAL(BLE_MESH_RX_SDU_MAX) - 4);
+		NET_BUF_SIMPLE(BT_MESH_SDU_UNSEG_MAX);
 	u8_t *ad;
 	u16_t i;
 	int err = 0;
 
-	BT_DBG("ASZMIC %u AKF %u AID 0x%02x", aszmic, AKF(&hdr), AID(&hdr));
+	BT_DBG("AKF %u AID 0x%02x", AKF(&hdr), AID(&hdr));
 	BT_DBG("len %u: %s", buf->om_len, bt_hex(buf->om_data, buf->om_len));
 
-	if (buf->om_len < 1 + APP_MIC_LEN(aszmic)) {
+	if (buf->om_len < 1 + APP_MIC_LEN(0)) {
 		BT_ERR("Too short SDU + MIC");
-		err = -EINVAL;
-		goto done;
+		return -EINVAL;
 	}
 
 	if (IS_ENABLED(CONFIG_BT_MESH_FRIEND) && !rx->local_match) {
 		BT_DBG("Ignoring PDU for LPN 0x%04x of this Friend",
 		       rx->ctx.recv_dst);
-		goto done;
+		return 0;
 	}
 
 	if (BT_MESH_ADDR_IS_VIRTUAL(rx->ctx.recv_dst)) {
@@ -661,86 +861,140 @@ static int sdu_recv(struct bt_mesh_net_rx *rx, u32_t seq, u8_t hdr,
 	}
 
 	/* Adjust the length to not contain the MIC at the end */
-	buf->om_len -= APP_MIC_LEN(aszmic);
+	buf->om_len -= APP_MIC_LEN(0);
 
 	if (!AKF(&hdr)) {
-		net_buf_simple_init(sdu, 0);
-		err = bt_mesh_app_decrypt(bt_mesh.dev_key, true, aszmic, buf,
-					  sdu, ad, rx->ctx.addr,
-					  rx->ctx.recv_dst, seq,
-					  BT_MESH_NET_IVI_RX(rx));
-		if (err) {
-			BT_WARN("Unable to decrypt with local DevKey");
-		} else {
-			rx->ctx.app_idx = BT_MESH_KEY_DEV_LOCAL;
-			bt_mesh_model_recv(rx, sdu);
-			goto done;
-		}
-
-		if (IS_ENABLED(CONFIG_BT_MESH_PROVISIONER)) {
-			struct bt_mesh_cdb_node *node;
-
-			/*
-			 * There is no way of knowing if we should use our
-			 * local DevKey or the remote DevKey to decrypt the
-			 * message so we must try both.
-			 */
-
-			node = bt_mesh_cdb_node_get(rx->ctx.addr);
-			if (node == NULL) {
-				BT_ERR("No node found for addr 0x%04x",
-				       rx->ctx.addr);
-				return -EINVAL;
-			}
-
-			net_buf_simple_init(sdu, 0);
-			err = bt_mesh_app_decrypt(node->dev_key, true, aszmic,
-						  buf, sdu, ad, rx->ctx.addr,
-						  rx->ctx.recv_dst, seq,
-						  BT_MESH_NET_IVI_RX(rx));
-			if (err) {
-				BT_ERR("Unable to decrypt with node DevKey");
-				return -EINVAL;
-			}
-
+		/* Attempt remote dev key first, as that is only available for
+		 * provisioner devices, which normally don't interact with nodes
+		 * that know their local dev key.
+		 */
+		net_buf_simple_reset(sdu);
+		err = remote_devkey_decrypt(rx, rx->seq, ad, 0, buf, sdu);
+		if (!err) {
 			rx->ctx.app_idx = BT_MESH_KEY_DEV_REMOTE;
 			bt_mesh_model_recv(rx, sdu);
 			return 0;
 		}
 
+		net_buf_simple_reset(sdu);
+		err = bt_mesh_app_decrypt(bt_mesh.dev_key, true, 0, buf, sdu,
+					  ad, rx->ctx.addr, rx->ctx.recv_dst,
+					  rx->seq, BT_MESH_NET_IVI_RX(rx));
+		if (err) {
+			BT_ERR("Unable to decrypt with local DevKey");
+			return err;
+		}
+
+		rx->ctx.app_idx = BT_MESH_KEY_DEV_LOCAL;
+		bt_mesh_model_recv(rx, sdu);
+		return 0;
+	}
+
+	for (i = 0U; i < ARRAY_SIZE(bt_mesh.app_keys); i++) {
+		struct bt_mesh_app_key *key = &bt_mesh.app_keys[i];
+
+		net_buf_simple_reset(sdu);
+		err = app_key_decrypt(rx, &bt_mesh.app_keys[i], rx->seq, ad,
+				      hdr, 0, buf, sdu);
+
+		if (err) {
+			continue;
+		}
+
+		rx->ctx.app_idx = key->app_idx;
+
+		bt_mesh_model_recv(rx, sdu);
+		return 0;
+	}
+
+	BT_WARN("No matching AppKey");
+
+	return -EINVAL;
+}
+
+static int sdu_recv_seg(struct seg_rx *seg, u8_t hdr, u8_t aszmic,
+			struct bt_mesh_net_rx *rx)
+{
+	struct os_mbuf *buf = NET_BUF_SIMPLE(BT_MESH_RX_SDU_MAX);
+	struct os_mbuf *sdu;
+	u32_t seq = (seg->seq_auth & 0xffffff);
+	u8_t *ad;
+	u16_t i;
+	int err;
+
+	BT_DBG("ASZMIC %u AKF %u AID 0x%02x", aszmic, AKF(&hdr), AID(&hdr));
+
+	if (seg->len < 1 + APP_MIC_LEN(aszmic)) {
+		BT_ERR("Too short SDU + MIC");
 		return -EINVAL;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(bt_mesh.app_keys); i++) {
-		struct bt_mesh_app_key *key = &bt_mesh.app_keys[i];
-		struct bt_mesh_app_keys *keys;
+	if (IS_ENABLED(CONFIG_BT_MESH_FRIEND) && !rx->local_match) {
+		BT_DBG("Ignoring PDU for LPN 0x%04x of this Friend",
+		       rx->ctx.recv_dst);
+		return 0;
+	}
 
-		/* Check that this AppKey matches received net_idx */
-		if (key->net_idx != rx->sub->net_idx) {
-			continue;
-		}
+	if (BT_MESH_ADDR_IS_VIRTUAL(rx->ctx.recv_dst)) {
+		ad = bt_mesh_label_uuid_get(rx->ctx.recv_dst);
+	} else {
+		ad = NULL;
+	}
 
-		if (rx->new_key && key->updated) {
-			keys = &key->keys[1];
-		} else {
-			keys = &key->keys[0];
-		}
+	/* Decrypting in place to avoid creating two assembly buffers.
+	 * We'll reassemble the buffer from the segments before each decryption
+	 * attempt.
+	 */
+	if (!AKF(&hdr)) {
+		/* Attempt remote dev key first, as that is only available for
+		 * provisioner devices, which normally don't interact with nodes
+		 * that know their local dev key.
+		 */
+		seg_rx_assemble(seg, buf, aszmic);
+
+		net_buf_simple_init_with_data(sdu, buf->om_data,
+					      seg->len - APP_MIC_LEN(aszmic));
+		sdu->om_len = 0;
 
 		/* Check that the AppKey ID matches */
-		if (AID(&hdr) != keys->id) {
-			continue;
+		err = remote_devkey_decrypt(rx, seq, ad, aszmic, buf, sdu);
+		if (!err) {
+			rx->ctx.app_idx = BT_MESH_KEY_DEV_REMOTE;
+			bt_mesh_model_recv(rx, sdu);
+			return 0;
 		}
 
-		net_buf_simple_init(sdu, 0);
-		err = bt_mesh_app_decrypt(keys->val, false, aszmic, buf,
+		seg_rx_assemble(seg, buf, aszmic);
+		net_buf_simple_init_with_data(sdu, buf->om_data,
+					      seg->len - APP_MIC_LEN(aszmic));
+		sdu->om_len = 0;
+
+		err = bt_mesh_app_decrypt(bt_mesh.dev_key, true, aszmic, buf,
 					  sdu, ad, rx->ctx.addr,
 					  rx->ctx.recv_dst, seq,
 					  BT_MESH_NET_IVI_RX(rx));
 		if (err) {
-			BT_WARN("Unable to decrypt with AppKey 0x%03x",
-				key->app_idx);
-			continue;
+			BT_ERR("Unable to decrypt with local DevKey");
+			return err;
+		}
 
+		rx->ctx.app_idx = BT_MESH_KEY_DEV_LOCAL;
+		bt_mesh_model_recv(rx, sdu);
+		return 0;
+	}
+
+	for (i = 0U; i < ARRAY_SIZE(bt_mesh.app_keys); i++) {
+		struct bt_mesh_app_key *key = &bt_mesh.app_keys[i];
+
+		seg_rx_assemble(seg, buf, aszmic);
+		net_buf_simple_init_with_data(sdu, buf->om_data,
+					      seg->len - APP_MIC_LEN(aszmic));
+		sdu->om_len = 0;
+
+		err = app_key_decrypt(rx, &bt_mesh.app_keys[i], seq, ad, hdr,
+				      aszmic, buf, sdu);
+		if (err) {
+			continue;
 		}
 
 		rx->ctx.app_idx = key->app_idx;
@@ -985,7 +1239,7 @@ static int trans_unseg(struct os_mbuf *buf, struct bt_mesh_net_rx *rx,
 			return 0;
 		}
 
-		return sdu_recv(rx, rx->seq, hdr, 0, buf);
+		return sdu_recv_unseg(rx, hdr, buf);
 	}
 }
 
@@ -1014,138 +1268,27 @@ static inline s32_t ack_timeout(struct seg_rx *rx)
 	return max(to, K_MSEC(400));
 }
 
-static int ctl_send_unseg(struct bt_mesh_net_tx *tx, u8_t ctl_op, void *data,
-			  size_t data_len, u64_t *seq_auth,
-			  const struct bt_mesh_send_cb *cb, void *cb_data)
+int bt_mesh_ctl_send(struct bt_mesh_net_tx *tx, u8_t ctl_op, void *data,
+		     size_t data_len, const struct bt_mesh_send_cb *cb, void *cb_data)
 {
 	struct os_mbuf *buf;
 
-	buf = bt_mesh_adv_create(BT_MESH_ADV_DATA, tx->xmit, BUF_TIMEOUT);
-	if (!buf) {
-		BT_ERR("Out of transport buffers");
-		return -ENOBUFS;
+	net_buf_simple_init_with_data(buf, data, data_len);
+
+	if (data_len > BT_MESH_SDU_UNSEG_MAX) {
+		tx->ctx->send_rel = true;
 	}
 
-	net_buf_reserve(buf, BT_MESH_NET_HDR_LEN);
+	tx->ctx->app_idx = BT_MESH_KEY_UNUSED;
 
-	net_buf_add_u8(buf, TRANS_CTL_HDR(ctl_op, 0));
-
-	net_buf_add_mem(buf, data, data_len);
-
-	if (IS_ENABLED(CONFIG_BT_MESH_FRIEND)) {
-		if (bt_mesh_friend_enqueue_tx(tx, BT_MESH_FRIEND_PDU_SINGLE,
-					      seq_auth, 1, buf) &&
-		    BT_MESH_ADDR_IS_UNICAST(tx->ctx->addr)) {
-			/* PDUs for a specific Friend should only go
-			 * out through the Friend Queue.
-			 */
-			net_buf_unref(buf);
-			return 0;
-		}
-	}
-
-	return bt_mesh_net_send(tx, buf, cb, cb_data);
-}
-
-static int ctl_send_seg(struct bt_mesh_net_tx *tx, u8_t ctl_op,
-			void *data, size_t data_len, u64_t *seq_auth,
-			const struct bt_mesh_send_cb *cb, void *cb_data)
-{
-	u8_t seg_o = 0;
-	u16_t seq_zero;
-	struct seg_tx *tx_seg;
-	int i;
-	u16_t unsent = data_len;
-
-	for (tx_seg = NULL, i = 0; i < ARRAY_SIZE(seg_tx); i++) {
-		if (!seg_tx[i].nack_count) {
-			tx_seg = &seg_tx[i];
-			break;
-		}
-	}
-
-	if (!tx_seg) {
-		BT_ERR("No multi-segment message contexts available");
-		return -EBUSY;
-	}
-
-	tx_seg->dst = tx->ctx->addr;
-	tx_seg->seg_n = (data_len - 1) / 8;
-	tx_seg->nack_count = tx_seg->seg_n + 1;
-	tx_seg->seq_auth = SEQ_AUTH(BT_MESH_NET_IVI_TX, bt_mesh.seq);
-	tx_seg->sub = tx->sub;
-	tx_seg->new_key = tx->sub->kr_flag;
-	tx_seg->cb = cb;
-	tx_seg->cb_data = cb_data;
-	tx_seg->attempts = SEG_RETRANSMIT_ATTEMPTS;
-	tx_seg->seg_pending = 0;
-
-	if (tx->ctx->send_ttl == BT_MESH_TTL_DEFAULT) {
-		tx_seg->ttl = bt_mesh_default_ttl_get();
-	} else {
-		tx_seg->ttl = tx->ctx->send_ttl;
-	}
-
-	seq_zero = tx_seg->seq_auth & TRANS_SEQ_ZERO_MASK;
-
-	for (seg_o = 0; seg_o <= tx_seg->seg_n; seg_o++) {
-		struct os_mbuf *seg;
-		u16_t len;
-		int err;
-
-		seg = bt_mesh_adv_create(BT_MESH_ADV_DATA, tx->xmit,
-					 BUF_TIMEOUT);
-		if (!seg) {
-			BT_ERR("Out of segment buffers");
-			seg_tx_reset(tx_seg);
-			return -ENOBUFS;
-		}
-
-		net_buf_reserve(seg, BT_MESH_NET_HDR_LEN);
-		net_buf_add_u8(seg, TRANS_CTL_HDR(ctl_op, 1));
-		net_buf_add_u8(seg, (tx->aszmic << 7) | seq_zero >> 6);
-		net_buf_add_u8(seg, (((seq_zero & 0x3f) << 2) | (seg_o >> 3)));
-		net_buf_add_u8(seg, ((seg_o & 0x07) << 5) | tx_seg->seg_n);
-		if (unsent < 8) {
-			len = unsent;
-		} else {
-			len = 8;
-		}			
-
-		net_buf_add_mem(seg, (char *)data + (data_len - unsent), len);
-		unsent -= len;
-		tx_seg->seg[seg_o] = net_buf_ref(seg);
-
-		BT_DBG("Sending %u/%u", seg_o, tx_seg->seg_n);
-		tx_seg->seg_pending++;
-
-		err = bt_mesh_net_send(tx, seg,
-				       seg_o ? &seg_sent_cb : &first_sent_cb,
-				       tx_seg);
-		if (err) {
-			BT_ERR("Sending segment failed");
-			seg_tx_reset(tx_seg);
-			return err;
-		}
-	}
-
-	return 0;
-}
-
-int bt_mesh_ctl_send(struct bt_mesh_net_tx *tx, u8_t ctl_op, void *data,
-		     size_t data_len, u64_t *seq_auth,
-		     const struct bt_mesh_send_cb *cb, void *cb_data)
-{
 	BT_DBG("src 0x%04x dst 0x%04x ttl 0x%02x ctl 0x%02x", tx->src,
 	       tx->ctx->addr, tx->ctx->send_ttl, ctl_op);
 	BT_DBG("len %zu: %s", data_len, bt_hex(data, data_len));
 
-	if (data_len <= 11) {
-		return ctl_send_unseg(tx, ctl_op, data, data_len, seq_auth,
-				      cb, cb_data);
+	if (tx->ctx->send_rel) {
+		return send_seg(tx, buf, cb, cb_data, &ctl_op);
 	} else {
-		return ctl_send_seg(tx, ctl_op, data, data_len, seq_auth,
-				    cb, cb_data);
+		return send_unseg(tx, buf, cb, cb_data, &ctl_op);
 	}
 }
 
@@ -1187,11 +1330,13 @@ static int send_ack(struct bt_mesh_subnet *sub, u16_t src, u16_t dst,
 	sys_put_be32(block, &buf[2]);
 
 	return bt_mesh_ctl_send(&tx, TRANS_CTL_OP_ACK, buf, sizeof(buf),
-				NULL, NULL, NULL);
+				NULL, NULL);
 }
 
 static void seg_rx_reset(struct seg_rx *rx, bool full_reset)
 {
+	int i;
+
 	BT_DBG("rx %p", rx);
 
 	k_delayed_work_cancel(&rx->ack);
@@ -1203,6 +1348,11 @@ static void seg_rx_reset(struct seg_rx *rx, bool full_reset)
 						&rx->seq_auth);
 	}
 
+	for (i = 0; i <= rx->seg_n; i++) {
+		k_mem_slab_free(&segs, &rx->seg[i]);
+		rx->seg[i] = NULL;
+	}
+
 	rx->in_use = 0;
 
 	/* We don't always reset these values since we need to be able to
@@ -1240,18 +1390,9 @@ static void seg_ack(struct ble_npl_event *work)
 	k_delayed_work_submit(&rx->ack, ack_timeout(rx));
 }
 
-static inline u8_t seg_len(bool ctl)
-{
-	if (ctl) {
-		return 8;
-	} else {
-		return 12;
-	}
-}
-
 static inline bool sdu_len_is_ok(bool ctl, u8_t seg_n)
 {
-	return ((seg_n * seg_len(ctl) + 1) <= MYNEWT_VAL(BLE_MESH_RX_SDU_MAX));
+	return (seg_n < CONFIG_BT_MESH_RX_SEG_MAX);
 }
 
 static struct seg_rx *seg_rx_find(struct bt_mesh_net_rx *net_rx,
@@ -1316,7 +1457,15 @@ static struct seg_rx *seg_rx_alloc(struct bt_mesh_net_rx *net_rx,
 				   const u8_t *hdr, const u64_t *seq_auth,
 				   u8_t seg_n)
 {
-	int i;
+	int i, j;
+
+	/* No race condition on this check, as this function only executes in
+	 * the collaborative Bluetooth rx thread:
+	 */
+	if (k_mem_slab_num_free_get(&segs) < seg_n) {
+		BT_WARN("Not enough segments for incoming message");
+		return NULL;
+	}
 
 	for (i = 0; i < ARRAY_SIZE(seg_rx); i++) {
 		struct seg_rx *rx = &seg_rx[i];
@@ -1326,7 +1475,6 @@ static struct seg_rx *seg_rx_alloc(struct bt_mesh_net_rx *net_rx,
 		}
 
 		rx->in_use = 1;
-		net_buf_simple_init(rx->buf, 0);
 		rx->sub = net_rx->sub;
 		rx->ctl = net_rx->ctl;
 		rx->seq_auth = *seq_auth;
@@ -1337,6 +1485,14 @@ static struct seg_rx *seg_rx_alloc(struct bt_mesh_net_rx *net_rx,
 		rx->dst = net_rx->ctx.recv_dst;
 		rx->block = 0;
 
+		/* Allocating everything at the beginning to avoid any failures
+		 * after initial rx. Could reduce overall buffer usage by
+		 * allocating as we go?
+		 */
+		for (j = 0; j <= seg_n; j++) {
+			k_mem_slab_alloc(&segs, &rx->seg[j]);
+		}
+
 		BT_DBG("New RX context. Block Complete 0x%08x",
 		       (unsigned) BLOCK_COMPLETE(seg_n));
 
@@ -1498,11 +1654,11 @@ found_rx:
 	 */
 	if (seg_o == seg_n) {
 		/* Set the expected final buffer length */
-		rx->buf->om_len = seg_n * seg_len(rx->ctl) + buf->om_len;
+		rx->len = seg_n * seg_len(rx->ctl) + buf->om_len;
 		BT_DBG("Target len %u * %u + %u = %u", seg_n, seg_len(rx->ctl),
-		       buf->om_len, rx->buf->om_len);
+		       buf->om_len, rx->len);
 
-		if (rx->buf->om_len > MYNEWT_VAL(BLE_MESH_RX_SDU_MAX)) {
+		if (rx->len > BT_MESH_RX_SDU_MAX) {
 			BT_ERR("Too large SDU len");
 			send_ack(net_rx->sub, net_rx->ctx.recv_dst,
 				 net_rx->ctx.addr, net_rx->ctx.send_ttl,
@@ -1525,8 +1681,7 @@ found_rx:
 		k_delayed_work_submit(&rx->ack, ack_timeout(rx));
 	}
 
-	/* Location in buffer can be calculated based on seg_o & rx->ctl */
-	memcpy(rx->buf->om_data + (seg_o * seg_len(rx->ctl)), buf->om_data, buf->om_len);
+	memcpy(rx->seg[seg_o], buf->om_data, buf->om_len);
 
 	BT_DBG("Received %u/%u", seg_o, seg_n);
 
@@ -1551,10 +1706,11 @@ found_rx:
 		 net_rx->ctx.send_ttl, seq_auth, rx->block, rx->obo);
 
 	if (net_rx->ctl) {
-		err = ctl_recv(net_rx, *hdr, rx->buf, seq_auth);
+		struct os_mbuf *sdu = NET_BUF_SIMPLE(BT_MESH_RX_CTL_MAX);
+		seg_rx_assemble(rx, sdu, 0U);
+		err = ctl_recv(net_rx, *hdr, sdu, seq_auth);
 	} else {
-		err = sdu_recv(net_rx, (rx->seq_auth & 0xffffff), *hdr,
-			       ASZMIC(hdr), rx->buf);
+		err = sdu_recv_seg(rx, *hdr, ASZMIC(hdr), net_rx);
 	}
 
 	seg_rx_reset(rx, false);
@@ -1694,7 +1850,6 @@ void bt_mesh_trans_init(void)
 	 *  For now we increase MSYS_1_BLOCK_COUNT
 	 */
 	for (i = 0; i < ARRAY_SIZE(seg_rx); i++) {
-	    seg_rx[i].buf = NET_BUF_SIMPLE(MYNEWT_VAL(BLE_MESH_RX_SDU_MAX));
 		k_delayed_work_init(&seg_rx[i].ack, seg_ack);
 		k_delayed_work_add_arg(&seg_rx[i].ack, &seg_rx[i]);
 	}
@@ -1755,7 +1910,7 @@ int bt_mesh_heartbeat_send(const struct bt_mesh_send_cb *cb, void *cb_data)
 	BT_DBG("InitTTL %u feat 0x%04x", cfg->hb_pub.ttl, feat);
 
 	return bt_mesh_ctl_send(&tx, TRANS_CTL_OP_HEARTBEAT, &hb, sizeof(hb),
-			 NULL, cb, cb_data);
+			cb, cb_data);
 }
 
 int bt_mesh_app_key_get(const struct bt_mesh_subnet *subnet, u16_t app_idx,
diff --git a/nimble/host/mesh/src/transport.h b/nimble/host/mesh/src/transport.h
index 8bcbff071..d9b4781a6 100644
--- a/nimble/host/mesh/src/transport.h
+++ b/nimble/host/mesh/src/transport.h
@@ -11,7 +11,12 @@
 
 #define TRANS_SEQ_AUTH_NVAL 0xffffffffffffffff
 
-#define BT_MESH_TX_SDU_MAX             (CONFIG_BT_MESH_TX_SEG_MAX * 12)
+#define BT_MESH_SDU_UNSEG_MAX          11
+#define BT_MESH_CTL_SEG_SDU_MAX        8
+#define BT_MESH_APP_SEG_SDU_MAX        12
+#define BT_MESH_TX_SDU_MAX (CONFIG_BT_MESH_TX_SEG_MAX * BT_MESH_APP_SEG_SDU_MAX)
+#define BT_MESH_RX_SDU_MAX (CONFIG_BT_MESH_RX_SEG_MAX * BT_MESH_APP_SEG_SDU_MAX)
+#define BT_MESH_RX_CTL_MAX (CONFIG_BT_MESH_RX_SEG_MAX * BT_MESH_CTL_SEG_SDU_MAX)
 
 #define TRANS_SEQ_ZERO_MASK            ((u16_t)BIT_MASK(13))
 #define TRANS_CTL_OP_MASK              ((u8_t)BIT_MASK(7))
@@ -87,8 +92,7 @@ void bt_mesh_rx_reset(void);
 void bt_mesh_tx_reset(void);
 
 int bt_mesh_ctl_send(struct bt_mesh_net_tx *tx, u8_t ctl_op, void *data,
-		     size_t data_len, u64_t *seq_auth,
-		     const struct bt_mesh_send_cb *cb, void *cb_data);
+		     size_t data_len, const struct bt_mesh_send_cb *cb, void *cb_data);
 
 int bt_mesh_trans_send(struct bt_mesh_net_tx *tx, struct os_mbuf *msg,
 		       const struct bt_mesh_send_cb *cb, void *cb_data);
diff --git a/nimble/host/mesh/syscfg.yml b/nimble/host/mesh/syscfg.yml
index a61cbc54f..6339f305f 100644
--- a/nimble/host/mesh/syscfg.yml
+++ b/nimble/host/mesh/syscfg.yml
@@ -186,7 +186,7 @@ syscfg.defs:
         description: >
             Maximum number of simultaneous outgoing multi-segment and/or
             reliable messages.
-        value: 4
+        value: 1
 
     BLE_MESH_RX_SEG_MSG_COUNT:
         description: >
@@ -194,15 +194,39 @@ syscfg.defs:
             reliable messages.
         value: 2
 
-    BLE_MESH_RX_SDU_MAX:
+    BLE_MESH_SEG_BUFS:
         description: >
-            Maximum incoming Upper Transport Access PDU length. This
-            determines also how many segments incoming segmented messages
-            can have. Each segment can contain 12 bytes, so this value should
-            be set to a multiple of 12 to avoid wasted memory. The minimum
-            requirement is 2 segments (24 bytes) whereas the maximum supported
-            by the Mesh specification is 32 segments (384 bytes).
-        value: 72
+            The incoming and outgoing segmented messages allocate their
+            segments from the same pool. Each segment is a 12 byte block,
+            and may only be used by one message at the time.
+
+            Outgoing messages will allocate their segments at the start of the
+            transmission, and release them one by one as soon as they have been
+            acknowledged by the receiver. Incoming messages allocate all their
+            segments at the start of the transaction, and won't release them until
+            the message is fully received.
+        value:
+            64
+
+    BLE_MESH_RX_SEG_MAX:
+        description: >
+            Maximum number of segments supported for incoming messages.
+            This value should typically be fine-tuned based on what
+            models the local node supports, i.e. what's the largest
+            message payload that the node needs to be able to receive.
+            This value affects memory and call stack consumption, which
+            is why the default is lower than the maximum that the
+            specification would allow (32 segments).
+
+            The maximum incoming SDU size is 12 times this number (out of
+            which 4 or 8 bytes is used for the Transport Layer MIC). For
+            example, 5 segments means the maximum SDU size is 60 bytes,
+            which leaves 56 bytes for application layer data using a
+            4-byte MIC and 52 bytes using an 8-byte MIC.
+        value:
+            3
+
+
 
     BLE_MESH_TX_SEG_MAX:
         description: >
@@ -210,20 +234,15 @@ syscfg.defs:
             This value should typically be fine-tuned based on what
             models the local node supports, i.e. what's the largest
             message payload that the node needs to be able to send.
-            This value affects memory and call stack consumption, which
-            is why the default is lower than the maximum that the
-            specification would allow (32 segments).
+            This value affects memory consumption, which is why the
+            default is lower than the maximum that the specification
+            would allow (32 segments).
 
             The maximum outgoing SDU size is 12 times this number (out of
             which 4 or 8 bytes is used for the Transport Layer MIC). For
             example, 5 segments means the maximum SDU size is 60 bytes,
             which leaves 56 bytes for application layer data using a
             4-byte MIC and 52 bytes using an 8-byte MIC.
-
-            Be sure to specify a sufficient number of advertising buffers
-            when setting this option to a higher value. There must be at
-            least three more advertising buffers (BT_MESH_ADV_BUF_COUNT)
-            as there are outgoing segments.
         value: 3
 
     BLE_MESH_SEG_RETRANSMIT_ATTEMPTS:
diff --git a/porting/examples/linux_blemesh/include/syscfg/syscfg.h b/porting/examples/linux_blemesh/include/syscfg/syscfg.h
index 7880d3de6..341a1dcf7 100644
--- a/porting/examples/linux_blemesh/include/syscfg/syscfg.h
+++ b/porting/examples/linux_blemesh/include/syscfg/syscfg.h
@@ -1070,8 +1070,12 @@
 #define MYNEWT_VAL_BLE_MESH_RPL_STORE_TIMEOUT (5)
 #endif
 
-#ifndef MYNEWT_VAL_BLE_MESH_RX_SDU_MAX
-#define MYNEWT_VAL_BLE_MESH_RX_SDU_MAX (72)
+#ifndef MYNEWT_VAL_BLE_MESH_SEG_BUFS
+#define MYNEWT_VAL_BLE_MESH_SEG_BUFS (72)
+#endif
+
+#ifndef MYNEWT_VAL_BLE_MESH_RX_SEG_MAX
+#define MYNEWT_VAL_BLE_MESH_RX_SEG_MAX (3)
 #endif
 
 #ifndef MYNEWT_VAL_BLE_MESH_RX_SEG_MSG_COUNT
@@ -1143,6 +1147,10 @@
 #define MYNEWT_VAL_BLE_MESH_TX_SEG_MAX (6)
 #endif
 
+#ifndef MYNEWT_VAL_BLE_MESH_RX_SEG_MSG_COUNT
+#define MYNEWT_VAL_BLE_MESH_RX_SEG_MSG_COUNT (3)
+#endif
+
 #ifndef MYNEWT_VAL_BLE_MESH_TX_SEG_MSG_COUNT
 #define MYNEWT_VAL_BLE_MESH_TX_SEG_MSG_COUNT (4)
 #endif