host/mesh: Add proxy send callback

Zephyr Bluetooth Mesh did not check whether the proxy message
was actually sent out, so that the response message could
not be received during reset.

This is port of 7531d2e3c8ffa73c9e1d1fcf33d2b26991ea5f8c

nimble/host/mesh/src/adv.c

@@ -66,15 +66,6 @@ static struct bt_mesh_adv *adv_alloc(int id)
 	return &adv_pool[id];
 }
 
-static inline void adv_send_start(uint16_t duration, int err,
-				  const struct bt_mesh_send_cb *cb,
-				  void *cb_data)
-{
-	if (cb && cb->start) {
-		cb->start(duration, err, cb_data);
-	}
-}
-
 static inline void adv_send_end(int err, const struct bt_mesh_send_cb *cb,
 				void *cb_data)
 {

nimble/host/mesh/src/adv.h

@@ -34,6 +34,16 @@ enum bt_mesh_adv_type
 typedef void (*bt_mesh_adv_func_t)(struct os_mbuf *buf, uint16_t duration,
 				   int err, void *user_data);
 
+
+static inline void adv_send_start(uint16_t duration, int err,
+				  const struct bt_mesh_send_cb *cb,
+				  	void *cb_data)
+{
+	if (cb && cb->start) {
+		cb->start(duration, err, cb_data);
+	}
+}
+
 struct bt_mesh_adv {
 	/** Fragments associated with this buffer. */
 	struct os_mbuf *frags;

nimble/host/mesh/src/cfg_srv.c

@@ -2023,11 +2023,27 @@ done:
 	os_mbuf_free_chain(msg);
 }
 
+static void reset_send_start(uint16_t duration, int err, void *cb_data)
+{
+	if (err) {
+		BT_ERR("Sending Node Reset Status failed (err %d)", err);
+		bt_mesh_reset();
+	}
+}
+
+static void reset_send_end(int err, void *cb_data)
+{
+	bt_mesh_reset();
+}
+
 static void node_reset(struct bt_mesh_model *model,
 		       struct bt_mesh_msg_ctx *ctx,
 		       struct os_mbuf *buf)
 {
-	static struct bt_mesh_proxy_idle_cb proxy_idle = {.cb = bt_mesh_reset};
+	static const struct bt_mesh_send_cb reset_cb = {
+		.start = reset_send_start,
+		.end = reset_send_end,
+	};
 
 	struct os_mbuf *msg = BT_MESH_MODEL_BUF(OP_NODE_RESET_STATUS, 0);
 
@@ -2035,26 +2051,13 @@ static void node_reset(struct bt_mesh_model *model,
 	       ctx->net_idx, ctx->app_idx, ctx->addr, buf->om_len,
 	       bt_hex(buf->om_data, buf->om_len));
 
-
 	bt_mesh_model_msg_init(msg, OP_NODE_RESET_STATUS);
 
-	/* Send the response first since we wont have any keys left to
-	 * send it later.
-	 */
-	if (bt_mesh_model_send(model, ctx, msg, NULL, NULL)) {
+	if (bt_mesh_model_send(model, ctx, msg, &reset_cb, NULL)) {
 		BT_ERR("Unable to send Node Reset Status");
 	}
 
-	if (!IS_ENABLED(CONFIG_BT_MESH_GATT_PROXY)) {
-		bt_mesh_reset();
-		return;
-	}
-
-	/* If the response goes to a proxy node, we'll wait for the sending to
-	 * complete before moving on.
-	 */
-	bt_mesh_proxy_on_idle(&proxy_idle);
-    os_mbuf_free_chain(msg);
+	os_mbuf_free_chain(msg);
 }
 
 static void send_friend_status(struct bt_mesh_model *model,

nimble/host/mesh/src/net.c

@@ -534,12 +534,13 @@ int bt_mesh_net_send(struct bt_mesh_net_tx *tx, struct os_mbuf *buf,
 		goto done;
 	}
 
+	BT_MESH_ADV(buf)->cb = cb;
+	BT_MESH_ADV(buf)->cb_data = cb_data;
+
 	/* Deliver to GATT Proxy Clients if necessary. */
 	if (IS_ENABLED(CONFIG_BT_MESH_GATT_PROXY) &&
 	    bt_mesh_proxy_relay(buf, tx->ctx->addr) &&
 	    BT_MESH_ADDR_IS_UNICAST(tx->ctx->addr)) {
-		/* Notify completion if this only went through the Mesh Proxy */
-		send_cb_finalize(cb, cb_data);
 
 		err = 0;
 		goto done;

nimble/host/mesh/src/pb_gatt.c

@@ -15,10 +15,16 @@
 #include "adv.h"
 #include "prov.h"
 
+struct prov_bearer_send_cb {
+	prov_bearer_send_complete_t cb;
+	void *cb_data;
+};
+
 struct prov_link {
 	uint16_t conn_handle;
 	const struct prov_bearer_cb *cb;
 	void *cb_data;
+	struct prov_bearer_send_cb comp;
 	struct {
 		uint8_t  id;        /* Transaction ID */
 		uint8_t  prev_id;   /* Previous Transaction ID */
@@ -130,6 +136,13 @@ static int link_accept(const struct prov_bearer_cb *cb, void *cb_data)
 	return 0;
 }
 
+static void buf_send_end(uint16_t conn_handle, void *user_data)
+{
+	if (link.comp.cb) {
+		link.comp.cb(0, link.comp.cb_data);
+	}
+}
+
 static int buf_send(struct os_mbuf *buf, prov_bearer_send_complete_t cb,
 		    void *cb_data)
 {
@@ -137,9 +150,12 @@ static int buf_send(struct os_mbuf *buf, prov_bearer_send_complete_t cb,
 		return -ENOTCONN;
 	}
 
+	link.comp.cb = cb;
+	link.comp.cb_data = cb_data;
+
 	k_work_reschedule(&link.prot_timer, PROTOCOL_TIMEOUT);
 
-	return bt_mesh_proxy_send(link.conn_handle, BT_MESH_PROXY_PROV, buf);
+	return bt_mesh_pb_gatt_send(link.conn_handle, buf, buf_send_end, NULL);
 }
 
 static void clear_tx(void)

nimble/host/mesh/src/proxy.c

@@ -139,9 +139,6 @@ static struct bt_mesh_proxy_client {
 	[0 ... (MYNEWT_VAL(BLE_MAX_CONNECTIONS) - 1)] = { 0 },
 };
 
-static sys_slist_t idle_waiters;
-static atomic_t pending_notifications;
-
 /* Track which service is enabled */
 static enum {
 	MESH_GATT_NONE,
@@ -213,7 +210,8 @@ static struct bt_mesh_proxy_client *find_client(uint16_t conn_handle)
 static struct bt_mesh_subnet *beacon_sub;
 
 static int proxy_segment_and_send(uint16_t conn_handle, uint8_t type,
-				  struct os_mbuf *msg);
+				  struct os_mbuf *msg,
+				  void (*end)(uint16_t, void *), void *user_data);
 
 static int filter_set(struct bt_mesh_proxy_client *client,
 		      struct os_mbuf *buf)
@@ -328,7 +326,8 @@ static void send_filter_status(struct bt_mesh_proxy_client *client,
 		return;
 	}
 
-	err = proxy_segment_and_send(client->conn_handle, BT_MESH_PROXY_CONFIG, buf);
+	err = proxy_segment_and_send(client->conn_handle, BT_MESH_PROXY_CONFIG, buf,
+				     NULL, NULL);
 	if (err) {
 		BT_ERR("Failed to send proxy cfg message (err %d)", err);
 	}
@@ -407,7 +406,8 @@ static int beacon_send(uint16_t conn_handle, struct bt_mesh_subnet *sub)
 	net_buf_simple_init(buf, 1);
 	bt_mesh_beacon_create(sub, buf);
 
-	rc = proxy_segment_and_send(conn_handle, BT_MESH_PROXY_BEACON, buf);
+	rc = proxy_segment_and_send(conn_handle, BT_MESH_PROXY_BEACON, buf,
+				    NULL, NULL);
 	os_mbuf_free_chain(buf);
 	return rc;
 }
@@ -953,10 +953,19 @@ static bool client_filter_match(struct bt_mesh_proxy_client *client,
 	return false;
 }
 
+static void buf_send_end(uint16_t conn_handle, void *user_data)
+{
+	struct os_mbuf *buf = user_data;
+
+	net_buf_unref(buf);
+}
+
 bool bt_mesh_proxy_relay(struct os_mbuf *buf, uint16_t dst)
 {
+	const struct bt_mesh_send_cb *cb = BT_MESH_ADV(buf)->cb;
+	void *cb_data = BT_MESH_ADV(buf)->cb_data;
 	bool relayed = false;
-	int i;
+	int i, err;
 
 	BT_DBG("%u bytes to dst 0x%04x", buf->om_len, dst);
 
@@ -972,6 +981,11 @@ bool bt_mesh_proxy_relay(struct os_mbuf *buf, uint16_t dst)
 			continue;
 		}
 
+		if (client->filter_type == PROV) {
+			BT_ERR("Invalid PDU type for Proxy Client");
+			return -EINVAL;
+		}
+
 		/* Proxy PDU sending modifies the original buffer,
 		 * so we need to make a copy.
 		 */
@@ -979,7 +993,14 @@ bool bt_mesh_proxy_relay(struct os_mbuf *buf, uint16_t dst)
 		net_buf_simple_init(msg, 1);
 		net_buf_simple_add_mem(msg, buf->om_data, buf->om_len);
 
-		bt_mesh_proxy_send(client->conn_handle, BT_MESH_PROXY_NET_PDU, msg);
+		err = proxy_segment_and_send(client->conn_handle, BT_MESH_PROXY_NET_PDU,
+					     msg, buf_send_end, net_buf_ref(buf));
+
+		adv_send_start(0, err, cb, cb_data);
+		if (err) {
+			BT_ERR("Failed to send proxy message (err %d)", err);
+			continue;
+		}
 		os_mbuf_free_chain(msg);
 		relayed = true;
 	}
@@ -989,22 +1010,9 @@ bool bt_mesh_proxy_relay(struct os_mbuf *buf, uint16_t dst)
 
 #endif /* MYNEWT_VAL(BLE_MESH_GATT_PROXY) */
 
-static void notify_complete(void)
-{
-	sys_snode_t *n;
-
-	if (atomic_dec(&pending_notifications) > 1) {
-		return;
-	}
-
-	BT_DBG("");
-
-	while ((n = sys_slist_get(&idle_waiters))) {
-		CONTAINER_OF(n, struct bt_mesh_proxy_idle_cb, n)->cb();
-	}
-}
-
-static int proxy_send(uint16_t conn_handle, const void *data, uint16_t len)
+static int proxy_send(uint16_t conn_handle,
+		      const void *data, uint16_t len,
+		      void (*end)(uint16_t, void *), void *user_data)
 {
 	struct os_mbuf *om;
 	int err = 0;
@@ -1016,7 +1024,7 @@ static int proxy_send(uint16_t conn_handle, const void *data, uint16_t len)
 		om = ble_hs_mbuf_from_flat(data, len);
 		assert(om);
 		err = ble_gattc_notify_custom(conn_handle, svc_handles.proxy_data_out_h, om);
-		notify_complete();
+		/* We do not pass cb into ble_gattc_notify_custom - execute it at the end */
 	}
 #endif
 
@@ -1025,19 +1033,16 @@ static int proxy_send(uint16_t conn_handle, const void *data, uint16_t len)
 		om = ble_hs_mbuf_from_flat(data, len);
 		assert(om);
 		err = ble_gattc_notify_custom(conn_handle, svc_handles.prov_data_out_h, om);
-		notify_complete();
 	}
 #endif
 
-	if (!err) {
-			atomic_inc(&pending_notifications);
-	}
-
+	end(conn_handle, user_data);
 	return err;
 }
 
 static int proxy_segment_and_send(uint16_t conn_handle, uint8_t type,
-				  struct os_mbuf *msg)
+				  struct os_mbuf *msg,
+				  void (*end)(uint16_t, void *), void *user_data)
 {
 	uint16_t mtu;
 
@@ -1048,30 +1053,30 @@ static int proxy_segment_and_send(uint16_t conn_handle, uint8_t type,
 	mtu = ble_att_mtu(conn_handle) - 3;
 	if (mtu > msg->om_len) {
 		net_buf_simple_push_u8(msg, PDU_HDR(SAR_COMPLETE, type));
-		return proxy_send(conn_handle, msg->om_data, msg->om_len);
+		return proxy_send(conn_handle, msg->om_data, msg->om_len, end, user_data);
 	}
 
 	net_buf_simple_push_u8(msg, PDU_HDR(SAR_FIRST, type));
-	proxy_send(conn_handle, msg->om_data, mtu);
+	proxy_send(conn_handle, msg->om_data, mtu, NULL, NULL);
 	net_buf_simple_pull_mem(msg, mtu);
 
 	while (msg->om_len) {
 		if (msg->om_len + 1 < mtu) {
 			net_buf_simple_push_u8(msg, PDU_HDR(SAR_LAST, type));
-			proxy_send(conn_handle, msg->om_data, msg->om_len);
+			proxy_send(conn_handle, msg->om_data, msg->om_len, end, user_data);
 			break;
 		}
 
 		net_buf_simple_push_u8(msg, PDU_HDR(SAR_CONT, type));
-		proxy_send(conn_handle, msg->om_data, mtu);
+		proxy_send(conn_handle, msg->om_data, mtu, NULL, NULL);
 		net_buf_simple_pull_mem(msg, mtu);
 	}
 
 	return 0;
 }
 
-int bt_mesh_proxy_send(uint16_t conn_handle, uint8_t type,
-		       struct os_mbuf *msg)
+int bt_mesh_pb_gatt_send(uint16_t conn_handle, struct os_mbuf *buf,
+			 void (*end)(uint16_t, void *), void *user_data)
 {
 	struct bt_mesh_proxy_client *client = find_client(conn_handle);
 
@@ -1080,12 +1085,12 @@ int bt_mesh_proxy_send(uint16_t conn_handle, uint8_t type,
 		return -ENOTCONN;
 	}
 
-	if ((client->filter_type == PROV) != (type == BT_MESH_PROXY_PROV)) {
+	if (client->filter_type == PROV) {
 		BT_ERR("Invalid PDU type for Proxy Client");
 		return -EINVAL;
 	}
 
-	return proxy_segment_and_send(conn_handle, type, msg);
+	return proxy_segment_and_send(conn_handle, BT_MESH_PROXY_PROV, buf, end, user_data);
 }
 
 #if (MYNEWT_VAL(BLE_MESH_PB_GATT))
@@ -1610,14 +1615,4 @@ int bt_mesh_proxy_init(void)
 	return 0;
 }
 
-void bt_mesh_proxy_on_idle(struct bt_mesh_proxy_idle_cb *cb)
-{
-	if (!atomic_get(&pending_notifications)) {
-		cb->cb();
-		return;
-	}
-
-	sys_slist_append(&idle_waiters, &cb->n);
-}
-
 #endif /* MYNEWT_VAL(BLE_MESH_PROXY) */

nimble/host/mesh/src/proxy.h

@@ -6,8 +6,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-#ifndef __PROXY_H__
-#define __PROXY_H__
+#ifndef ZEPHYR_SUBSYS_BLUETOOTH_MESH_PROXY_H_
+#define ZEPHYR_SUBSYS_BLUETOOTH_MESH_PROXY_H_
+
+int bt_mesh_pb_gatt_send(uint16_t conn_handle, struct os_mbuf *buf,
+			 void (*end)(uint16_t, void *), void *user_data);
 
 #define BT_MESH_PROXY_NET_PDU   0x00
 #define BT_MESH_PROXY_BEACON    0x01
@@ -17,13 +20,6 @@
 #include "mesh/mesh.h"
 #include "mesh/slist.h"
 
-struct bt_mesh_proxy_idle_cb {
-	sys_snode_t n;
-	void (*cb)(void);
-};
-
-int bt_mesh_proxy_send(uint16_t conn_handle, uint8_t type, struct os_mbuf *msg);
-
 int bt_mesh_proxy_prov_enable(void);
 int bt_mesh_proxy_prov_disable(bool disconnect);
 
@@ -45,8 +41,7 @@ bool bt_mesh_proxy_relay(struct os_mbuf *buf, uint16_t dst);
 void bt_mesh_proxy_addr_add(struct os_mbuf *buf, uint16_t addr);
 
 int bt_mesh_proxy_init(void);
-void bt_mesh_proxy_on_idle(struct bt_mesh_proxy_idle_cb *cb);
 
 int ble_mesh_proxy_gap_event(struct ble_gap_event *event, void *arg);
 
-#endif
+#endif /* ZEPHYR_SUBSYS_BLUETOOTH_MESH_PROXY_H_ */