tipc-discussion

[tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang Le <hoa...@de...>]

Currently, updating binding table (add service binding to
name table/withdraw a service binding) is being sent over replicast.
However, if we are scaling up clusters to > 100 nodes/containers this
method is less affection because of looping through nodes in a cluster one
by one.

It is worth to use broadcast to update a binding service. Then binding
table updates in all nodes for one shot.

The mechanism is backward compatible because of sending side changing.

Signed-off-by: Hoang Le <hoa...@de...>
---
 net/tipc/bcast.c      | 13 +++++++++++++
 net/tipc/bcast.h      |  2 ++
 net/tipc/name_table.c |  4 ++--
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
index f41096a759fa..18431fa897ab 100644
--- a/net/tipc/bcast.c
+++ b/net/tipc/bcast.c
@@ -843,3 +843,16 @@ void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
 		__skb_queue_tail(inputq, _skb);
 	}
 }
+
+int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb)
+{
+	struct sk_buff_head xmitq;
+	u16 cong_link_cnt;
+	int rc = 0;
+
+	__skb_queue_head_init(&xmitq);
+	__skb_queue_tail(&xmitq, skb);
+	rc = tipc_bcast_xmit(net, &xmitq, &cong_link_cnt);
+	__skb_queue_purge(&xmitq);
+	return rc;
+}
diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
index dadad953e2be..a100da3800fc 100644
--- a/net/tipc/bcast.h
+++ b/net/tipc/bcast.h
@@ -101,6 +101,8 @@ int tipc_bclink_reset_stats(struct net *net);
 u32 tipc_bcast_get_broadcast_mode(struct net *net);
 u32 tipc_bcast_get_broadcast_ratio(struct net *net);
 
+int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb);
+
 void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
 			   struct sk_buff_head *inputq);
 
diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
index 66a65c2cdb23..9e9c61f7c999 100644
--- a/net/tipc/name_table.c
+++ b/net/tipc/name_table.c
@@ -633,7 +633,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb)
-		tipc_node_broadcast(net, skb);
+		tipc_bcast_named_publish(net, skb);
 	return p;
 }
 
@@ -664,7 +664,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb) {
-		tipc_node_broadcast(net, skb);
+		tipc_bcast_named_publish(net, skb);
 		return 1;
 	}
 	return 0;
-- 
2.20.1

[tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

Currently, updating binding table (add service binding to
name table/withdraw a service binding) is being sent over replicast.
However, if we are scaling up clusters to > 100 nodes/containers this
method is less affection because of looping through nodes in a cluster one
by one.

It is worth to use broadcast to update a binding service. Then binding
table updates in all nodes for one shot.

The mechanism is backward compatible as sync message slient dropped.

v2: resolve synchronization problem when switching from unicast to
broadcast

v5: - never use broadcast if there is a single node not understanding it
    - always use broadcast otherwise
    - add sequence numbering to non-bulk messages

Signed-off-by: Hoang Huu Le <hoa...@de...>
---
 net/tipc/bcast.c      |   6 +--
 net/tipc/bcast.h      |   4 +-
 net/tipc/link.c       |   2 +-
 net/tipc/msg.h        |  40 +++++++++++++++
 net/tipc/name_distr.c | 115 +++++++++++++++++++++++++++++++-----------
 net/tipc/name_distr.h |   9 ++--
 net/tipc/name_table.c |   9 +++-
 net/tipc/name_table.h |   2 +
 net/tipc/node.c       |  28 +++++++---
 net/tipc/node.h       |   8 +--
 10 files changed, 175 insertions(+), 48 deletions(-)

diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
index 4c20be08b9c4..9d085ad6f0cf 100644
--- a/net/tipc/bcast.c
+++ b/net/tipc/bcast.c
@@ -249,8 +249,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
  * Consumes the buffer chain.
  * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
  */
-static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
-			   u16 *cong_link_cnt)
+int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
+		    u16 *cong_link_cnt)
 {
 	struct tipc_link *l = tipc_bc_sndlink(net);
 	struct sk_buff_head xmitq;
@@ -746,7 +746,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
 	nl->local = false;
 }
 
-u32 tipc_bcast_get_broadcast_mode(struct net *net)
+u32 tipc_bcast_get_mode(struct net *net)
 {
 	struct tipc_bc_base *bb = tipc_bc_base(net);
 
diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
index 9e847d9617d3..b3b883e2a823 100644
--- a/net/tipc/bcast.h
+++ b/net/tipc/bcast.h
@@ -89,6 +89,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
 int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
 		    struct tipc_mc_method *method, struct tipc_nlist *dests,
 		    u16 *cong_link_cnt);
+int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
+		    u16 *cong_link_cnt);
 int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
 void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
 			struct tipc_msg *hdr);
@@ -98,7 +100,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
 int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
 int tipc_bclink_reset_stats(struct net *net);
 
-u32 tipc_bcast_get_broadcast_mode(struct net *net);
+u32 tipc_bcast_get_mode(struct net *net);
 u32 tipc_bcast_get_broadcast_ratio(struct net *net);
 
 void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
diff --git a/net/tipc/link.c b/net/tipc/link.c
index d4675e922a8f..da0b30733549 100644
--- a/net/tipc/link.c
+++ b/net/tipc/link.c
@@ -2646,7 +2646,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg)
 	struct nlattr *attrs;
 	struct nlattr *prop;
 	struct tipc_net *tn = net_generic(net, tipc_net_id);
-	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
+	u32 bc_mode = tipc_bcast_get_mode(net);
 	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
 	struct tipc_link *bcl = tn->bcl;
 
diff --git a/net/tipc/msg.h b/net/tipc/msg.h
index 871feadbbc19..d53914316684 100644
--- a/net/tipc/msg.h
+++ b/net/tipc/msg.h
@@ -409,6 +409,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
 	msg_set_bits(m, 1, 25, 0xf, err);
 }
 
+static inline void msg_set_bulk(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 28, 0x1, 1);
+}
+
+static inline u32 msg_is_bulk(struct tipc_msg *m)
+{
+	return msg_bits(m, 1, 28, 0x1);
+}
+
+static inline void msg_set_last_bulk(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 27, 0x1, 1);
+}
+
+static inline u32 msg_is_last_bulk(struct tipc_msg *m)
+{
+	return msg_bits(m, 1, 27, 0x1);
+}
+
+static inline void msg_set_non_legacy(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 26, 0x1, 1);
+}
+
+static inline u32 msg_is_legacy(struct tipc_msg *m)
+{
+	return !msg_bits(m, 1, 26, 0x1);
+}
+
 static inline u32 msg_reroute_cnt(struct tipc_msg *m)
 {
 	return msg_bits(m, 1, 21, 0xf);
@@ -538,6 +568,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
 	msg_set_word(m, 4, p);
 }
 
+static inline u16 msg_named_seqno(struct tipc_msg *m)
+{
+	return msg_bits(m, 4, 0, 0xffff);
+}
+
+static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
+{
+	msg_set_bits(m, 4, 0, 0xffff, n);
+}
+
 static inline u32 msg_destport(struct tipc_msg *m)
 {
 	return msg_word(m, 5);
diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
index 5feaf3b67380..f4bbb8e792c9 100644
--- a/net/tipc/name_distr.c
+++ b/net/tipc/name_distr.c
@@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
 		pr_warn("Publication distribution failure\n");
 		return NULL;
 	}
-
+	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
+	msg_set_non_legacy(buf_msg(skb));
 	item = (struct distr_item *)msg_data(buf_msg(skb));
 	publ_to_item(item, publ);
 	return skb;
@@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
 struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
 {
 	struct name_table *nt = tipc_name_table(net);
-	struct sk_buff *buf;
 	struct distr_item *item;
+	struct sk_buff *skb;
 
 	write_lock_bh(&nt->cluster_scope_lock);
 	list_del(&publ->binding_node);
@@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
 	if (publ->scope == TIPC_NODE_SCOPE)
 		return NULL;
 
-	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
-	if (!buf) {
+	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
+	if (!skb) {
 		pr_warn("Withdrawal distribution failure\n");
 		return NULL;
 	}
-
-	item = (struct distr_item *)msg_data(buf_msg(buf));
+	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
+	msg_set_non_legacy(buf_msg(skb));
+	item = (struct distr_item *)msg_data(buf_msg(skb));
 	publ_to_item(item, publ);
-	return buf;
+	return skb;
 }
 
 /**
@@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
  * @pls: linked list of publication items to be packed into buffer chain
  */
 static void named_distribute(struct net *net, struct sk_buff_head *list,
-			     u32 dnode, struct list_head *pls)
+			     u32 dnode, struct list_head *pls, u16 seqno)
 {
 	struct publication *publ;
 	struct sk_buff *skb = NULL;
@@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
 			ITEM_SIZE) * ITEM_SIZE;
 	u32 msg_rem = msg_dsz;
+	struct tipc_msg *hdr;
 
 	list_for_each_entry(publ, pls, binding_node) {
 		/* Prepare next buffer: */
@@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 				pr_warn("Bulk publication failure\n");
 				return;
 			}
-			msg_set_bc_ack_invalid(buf_msg(skb), true);
-			item = (struct distr_item *)msg_data(buf_msg(skb));
+			hdr = buf_msg(skb);
+			msg_set_bc_ack_invalid(hdr, true);
+			msg_set_bulk(hdr);
+			msg_set_non_legacy(hdr);
+			item = (struct distr_item *)msg_data(hdr);
 		}
 
 		/* Pack publication into message: */
@@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 		}
 	}
 	if (skb) {
-		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
+		hdr = buf_msg(skb);
+		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
 		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
 		__skb_queue_tail(list, skb);
 	}
+	hdr = buf_msg(skb_peek_tail(list));
+	msg_set_last_bulk(hdr);
+	msg_set_named_seqno(hdr, seqno);
 }
 
 /**
  * tipc_named_node_up - tell specified node about all publications by this node
  */
-void tipc_named_node_up(struct net *net, u32 dnode)
+void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
 {
 	struct name_table *nt = tipc_name_table(net);
+	struct tipc_net *tn = tipc_net(net);
 	struct sk_buff_head head;
+	u16 seqno;
 
 	__skb_queue_head_init(&head);
+	spin_lock_bh(&tn->nametbl_lock);
+	if (!(capabilities & TIPC_NAMED_BCAST))
+		nt->rc_dests++;
+	seqno = nt->snd_nxt;
+	spin_unlock_bh(&tn->nametbl_lock);
 
 	read_lock_bh(&nt->cluster_scope_lock);
-	named_distribute(net, &head, dnode, &nt->cluster_scope);
+	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
 	tipc_node_xmit(net, &head, dnode, 0);
 	read_unlock_bh(&nt->cluster_scope_lock);
 }
@@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
 	spin_unlock_bh(&tn->nametbl_lock);
 }
 
-void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
+void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
+		      u32 addr, u16 capabilities)
 {
+	struct name_table *nt = tipc_name_table(net);
+	struct tipc_net *tn = tipc_net(net);
+
 	struct publication *publ, *tmp;
 
 	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
 		tipc_publ_purge(net, publ, addr);
 	tipc_dist_queue_purge(net, addr);
+	spin_lock_bh(&tn->nametbl_lock);
+	if (!(capabilities & TIPC_NAMED_BCAST))
+		nt->rc_dests--;
+	spin_unlock_bh(&tn->nametbl_lock);
 }
 
 /**
@@ -295,29 +320,61 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
 	return false;
 }
 
+struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
+				   u16 *rcv_nxt, bool *open)
+{
+	struct tipc_msg *hdr;
+	struct sk_buff *skb, *tmp;
+	u16 seqno;
+
+	skb_queue_walk_safe(namedq, skb, tmp) {
+		skb_linearize(skb);
+		hdr = buf_msg(skb);
+		seqno = msg_named_seqno(hdr);
+		if (msg_is_last_bulk(hdr)) {
+			*rcv_nxt = seqno;
+			*open = true;
+		}
+		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
+			__skb_unlink(skb, namedq);
+			return skb;
+		}
+
+		if (*open && (*rcv_nxt == seqno)) {
+			(*rcv_nxt)++;
+			__skb_unlink(skb, namedq);
+			return skb;
+		}
+
+		if (less(seqno, *rcv_nxt)) {
+			__skb_unlink(skb, namedq);
+			kfree_skb(skb);
+			continue;
+		}
+	}
+	return NULL;
+}
+
 /**
  * tipc_named_rcv - process name table update messages sent by another node
  */
-void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
+void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
+		    u16 *rcv_nxt, bool *open)
 {
-	struct tipc_net *tn = net_generic(net, tipc_net_id);
-	struct tipc_msg *msg;
+	struct tipc_net *tn = tipc_net(net);
 	struct distr_item *item;
-	uint count;
-	u32 node;
+	struct tipc_msg *hdr;
 	struct sk_buff *skb;
-	int mtype;
+	u32 count, node = 0;
 
 	spin_lock_bh(&tn->nametbl_lock);
-	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
-		skb_linearize(skb);
-		msg = buf_msg(skb);
-		mtype = msg_type(msg);
-		item = (struct distr_item *)msg_data(msg);
-		count = msg_data_sz(msg) / ITEM_SIZE;
-		node = msg_orignode(msg);
+	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
+		hdr = buf_msg(skb);
+		node = msg_orignode(hdr);
+		item = (struct distr_item *)msg_data(hdr);
+		count = msg_data_sz(hdr) / ITEM_SIZE;
 		while (count--) {
-			tipc_update_nametbl(net, item, node, mtype);
+			tipc_update_nametbl(net, item, node, msg_type(hdr));
 			item++;
 		}
 		kfree_skb(skb);
@@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
 		publ->node = self;
 	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
 		publ->node = self;
-
+	nt->rc_dests = 0;
 	spin_unlock_bh(&tn->nametbl_lock);
 }
diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
index 63fc73e0fa6c..092323158f06 100644
--- a/net/tipc/name_distr.h
+++ b/net/tipc/name_distr.h
@@ -67,11 +67,14 @@ struct distr_item {
 	__be32 key;
 };
 
+void tipc_named_bcast(struct net *net, struct sk_buff *skb);
 struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
 struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
-void tipc_named_node_up(struct net *net, u32 dnode);
-void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
+void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
+void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
+		    u16 *rcv_nxt, bool *open);
 void tipc_named_reinit(struct net *net);
-void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
+void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
+		      u32 addr, u16 capabilities);
 
 #endif
diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
index 359b2bc888cf..2ac33d32edc2 100644
--- a/net/tipc/name_table.c
+++ b/net/tipc/name_table.c
@@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 	struct tipc_net *tn = tipc_net(net);
 	struct publication *p = NULL;
 	struct sk_buff *skb = NULL;
+	u32 rc_dests;
 
 	spin_lock_bh(&tn->nametbl_lock);
 
@@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 		nt->local_publ_count++;
 		skb = tipc_named_publish(net, p);
 	}
+	rc_dests = nt->rc_dests;
 exit:
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb)
-		tipc_node_broadcast(net, skb);
+		tipc_node_broadcast(net, skb, rc_dests);
 	return p;
+
 }
 
 /**
@@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 	u32 self = tipc_own_addr(net);
 	struct sk_buff *skb = NULL;
 	struct publication *p;
+	u32 rc_dests;
 
 	spin_lock_bh(&tn->nametbl_lock);
 
@@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
 		       type, lower, upper, key);
 	}
+	rc_dests = nt->rc_dests;
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb) {
-		tipc_node_broadcast(net, skb);
+		tipc_node_broadcast(net, skb, rc_dests);
 		return 1;
 	}
 	return 0;
diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
index 728bc7016c38..8064e1986e2c 100644
--- a/net/tipc/name_table.h
+++ b/net/tipc/name_table.h
@@ -106,6 +106,8 @@ struct name_table {
 	struct list_head cluster_scope;
 	rwlock_t cluster_scope_lock;
 	u32 local_publ_count;
+	u32 rc_dests;
+	u32 snd_nxt;
 };
 
 int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/tipc/node.c b/net/tipc/node.c
index 803a3a6d0f50..ad8d7bce1f98 100644
--- a/net/tipc/node.c
+++ b/net/tipc/node.c
@@ -75,6 +75,8 @@ struct tipc_bclink_entry {
 	struct sk_buff_head arrvq;
 	struct sk_buff_head inputq2;
 	struct sk_buff_head namedq;
+	u16 named_rcv_nxt;
+	bool named_open;
 };
 
 /**
@@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
 	write_unlock_bh(&n->lock);
 
 	if (flags & TIPC_NOTIFY_NODE_DOWN)
-		tipc_publ_notify(net, publ_list, addr);
+		tipc_publ_notify(net, publ_list, addr, n->capabilities);
 
 	if (flags & TIPC_NOTIFY_NODE_UP)
-		tipc_named_node_up(net, addr);
+		tipc_named_node_up(net, addr, n->capabilities);
 
 	if (flags & TIPC_NOTIFY_LINK_UP) {
 		tipc_mon_peer_up(net, addr, bearer_id);
@@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
 	return 0;
 }
 
-void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
+void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
 {
+	struct sk_buff_head xmitq;
 	struct sk_buff *txskb;
 	struct tipc_node *n;
+	u16 dummy;
 	u32 dst;
 
+	/* Use broadcast if all nodes support it */
+	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
+		__skb_queue_head_init(&xmitq);
+		__skb_queue_tail(&xmitq, skb);
+		tipc_bcast_xmit(net, &xmitq, &dummy);
+		return;
+	}
+
+	/* Otherwise use legacy replicast method */
 	rcu_read_lock();
 	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
 		dst = n->addr;
@@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
 		tipc_node_xmit_skb(net, txskb, dst, 0);
 	}
 	rcu_read_unlock();
-
 	kfree_skb(skb);
 }
 
@@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
 
 	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
 	if (!skb_queue_empty(&n->bc_entry.namedq))
-		tipc_named_rcv(net, &n->bc_entry.namedq);
+		tipc_named_rcv(net, &n->bc_entry.namedq,
+			       &n->bc_entry.named_rcv_nxt,
+			       &n->bc_entry.named_open);
 
 	/* If reassembly or retransmission failure => reset all links to peer */
 	if (rc & TIPC_LINK_DOWN_EVT)
@@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
 		tipc_node_link_down(n, bearer_id, false);
 
 	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
-		tipc_named_rcv(net, &n->bc_entry.namedq);
+		tipc_named_rcv(net, &n->bc_entry.namedq,
+			       &n->bc_entry.named_rcv_nxt,
+			       &n->bc_entry.named_open);
 
 	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
 		tipc_node_mcast_rcv(n);
diff --git a/net/tipc/node.h b/net/tipc/node.h
index a6803b449a2c..9f6f13f1604f 100644
--- a/net/tipc/node.h
+++ b/net/tipc/node.h
@@ -55,7 +55,8 @@ enum {
 	TIPC_MCAST_RBCTL      = (1 << 7),
 	TIPC_GAP_ACK_BLOCK    = (1 << 8),
 	TIPC_TUNNEL_ENHANCED  = (1 << 9),
-	TIPC_NAGLE            = (1 << 10)
+	TIPC_NAGLE            = (1 << 10),
+	TIPC_NAMED_BCAST      = (1 << 11)
 };
 
 #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
@@ -68,7 +69,8 @@ enum {
 				TIPC_MCAST_RBCTL       |   \
 				TIPC_GAP_ACK_BLOCK     |   \
 				TIPC_TUNNEL_ENHANCED   |   \
-				TIPC_NAGLE)
+				TIPC_NAGLE             |   \
+				TIPC_NAMED_BCAST)
 
 #define INVALID_BEARER_ID -1
 
@@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
 		       u32 selector);
 void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
 void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
-void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
+void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
 int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
 void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
 int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);
-- 
2.25.1

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/4/20 5:21 AM, Hoang Huu Le wrote:
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
>
> It is worth to use broadcast to update a binding service.
s/Then binding table updates in/all nodes in one shoy. This way, the 
binding table can be updated on all peer nodes in one shot./

Broadcast is used when all peer nodes, as indicated by a new capability 
flag TIPC_NAMED_BCAST, support reception of this message type.

Four problems need to be considered when introducing this feature.
1) When establishing a link to a new peer node we still update this by a 
unicast 'bulk' update. This may lead to race conditions, where a later 
broadcast publication/withdrawal bypass the 'bulk', resulting in 
disordered publications, or even that a withdrawal may arrive before the 
corresponding publication. We solve this by adding an 'is_last_bulk' bit 
in the last bulk messages so that it can be distinguished from all other 
messages. Only when this message has arrived do we open up for reception 
of broadcast publications/withdrawals.
2) When a first legacy node is added to the cluster all distribution 
will switch over to use the legacy 'replicast' method, while the 
opposite happens when the last legacy node leaves the cluster. This 
entails another risk of message disordering that has to be handled. We 
solve this by adding a sequence number to the broadcast/replicast 
messages, so that disordering can be discovered and corrected. Note 
however that we don't need to consider potential message loss or 
duplication at this protocol level.
3) Bulk messages don't contain any sequence numbers, and will always 
arrive in order. Hence we must exempt those from the sequence number 
control and deliver them unconditionally. We solve this by adding a new 
'is_bulk' bit in those messages so that they can be recognized.
4) Legacy messages, which don't contain any new bits or sequence 
numbers, but neither can arrive out of order, also need to be exempt 
from the initial synchronization and sequence number check, and 
delivered unconditionally. Therefore, we add another 'is_not_legacy' bit 
to all new messages so that those can be distinguished from legacy 
messages and the latter delivered directly.

> Signed-off-by: Hoang Huu Le <hoa...@de...>
> ---
>   net/tipc/bcast.c      |   6 +--
>   net/tipc/bcast.h      |   4 +-
>   net/tipc/link.c       |   2 +-
>   net/tipc/msg.h        |  40 +++++++++++++++
>   net/tipc/name_distr.c | 115 +++++++++++++++++++++++++++++++-----------
>   net/tipc/name_distr.h |   9 ++--
>   net/tipc/name_table.c |   9 +++-
>   net/tipc/name_table.h |   2 +
>   net/tipc/node.c       |  28 +++++++---
>   net/tipc/node.h       |   8 +--
>   10 files changed, 175 insertions(+), 48 deletions(-)
>
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index 4c20be08b9c4..9d085ad6f0cf 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -249,8 +249,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
>    * Consumes the buffer chain.
>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>    */
> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> -			   u16 *cong_link_cnt)
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt)
>   {
>   	struct tipc_link *l = tipc_bc_sndlink(net);
>   	struct sk_buff_head xmitq;
> @@ -746,7 +746,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>   	nl->local = false;
>   }
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net)
> +u32 tipc_bcast_get_mode(struct net *net)
>   {
>   	struct tipc_bc_base *bb = tipc_bc_base(net);
>   
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index 9e847d9617d3..b3b883e2a823 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -89,6 +89,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>   		    struct tipc_mc_method *method, struct tipc_nlist *dests,
>   		    u16 *cong_link_cnt);
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt);
>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>   			struct tipc_msg *hdr);
> @@ -98,7 +100,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>   int tipc_bclink_reset_stats(struct net *net);
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net);
> +u32 tipc_bcast_get_mode(struct net *net);
>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>   
>   void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> diff --git a/net/tipc/link.c b/net/tipc/link.c
> index d4675e922a8f..da0b30733549 100644
> --- a/net/tipc/link.c
> +++ b/net/tipc/link.c
> @@ -2646,7 +2646,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg)
>   	struct nlattr *attrs;
>   	struct nlattr *prop;
>   	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
> +	u32 bc_mode = tipc_bcast_get_mode(net);
>   	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>   	struct tipc_link *bcl = tn->bcl;
>   
> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
> index 871feadbbc19..d53914316684 100644
> --- a/net/tipc/msg.h
> +++ b/net/tipc/msg.h
> @@ -409,6 +409,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
>   	msg_set_bits(m, 1, 25, 0xf, err);
>   }
>   
> +static inline void msg_set_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 28, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 28, 0x1);
> +}
> +
> +static inline void msg_set_last_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 27, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 27, 0x1);
> +}
> +
> +static inline void msg_set_non_legacy(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 26, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_legacy(struct tipc_msg *m)
> +{
> +	return !msg_bits(m, 1, 26, 0x1);
> +}
> +
>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>   {
>   	return msg_bits(m, 1, 21, 0xf);
> @@ -538,6 +568,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
>   	msg_set_word(m, 4, p);
>   }
>   
> +static inline u16 msg_named_seqno(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 4, 0, 0xffff);
> +}
> +
> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
> +{
> +	msg_set_bits(m, 4, 0, 0xffff, n);
> +}
> +
>   static inline u32 msg_destport(struct tipc_msg *m)
>   {
>   	return msg_word(m, 5);
> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
> index 5feaf3b67380..f4bbb8e792c9 100644
> --- a/net/tipc/name_distr.c
> +++ b/net/tipc/name_distr.c
> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   		pr_warn("Publication distribution failure\n");
>   		return NULL;
>   	}
> -
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
>   	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
>   	return skb;
> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> -	struct sk_buff *buf;
>   	struct distr_item *item;
> +	struct sk_buff *skb;
>   
>   	write_lock_bh(&nt->cluster_scope_lock);
>   	list_del(&publ->binding_node);
> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   	if (publ->scope == TIPC_NODE_SCOPE)
>   		return NULL;
>   
> -	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> -	if (!buf) {
> +	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> +	if (!skb) {
>   		pr_warn("Withdrawal distribution failure\n");
>   		return NULL;
>   	}
> -
> -	item = (struct distr_item *)msg_data(buf_msg(buf));
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
> +	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
> -	return buf;
> +	return skb;
>   }
>   
>   /**
> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>    * @pls: linked list of publication items to be packed into buffer chain
>    */
>   static void named_distribute(struct net *net, struct sk_buff_head *list,
> -			     u32 dnode, struct list_head *pls)
> +			     u32 dnode, struct list_head *pls, u16 seqno)
>   {
>   	struct publication *publ;
>   	struct sk_buff *skb = NULL;
> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
>   			ITEM_SIZE) * ITEM_SIZE;
>   	u32 msg_rem = msg_dsz;
> +	struct tipc_msg *hdr;
>   
>   	list_for_each_entry(publ, pls, binding_node) {
>   		/* Prepare next buffer: */
> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   				pr_warn("Bulk publication failure\n");
>   				return;
>   			}
> -			msg_set_bc_ack_invalid(buf_msg(skb), true);
> -			item = (struct distr_item *)msg_data(buf_msg(skb));
> +			hdr = buf_msg(skb);
> +			msg_set_bc_ack_invalid(hdr, true);
> +			msg_set_bulk(hdr);
> +			msg_set_non_legacy(hdr);
> +			item = (struct distr_item *)msg_data(hdr);
>   		}
>   
>   		/* Pack publication into message: */
> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   		}
>   	}
>   	if (skb) {
> -		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
> +		hdr = buf_msg(skb);
> +		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>   		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>   		__skb_queue_tail(list, skb);
>   	}
> +	hdr = buf_msg(skb_peek_tail(list));
> +	msg_set_last_bulk(hdr);
> +	msg_set_named_seqno(hdr, seqno);
>   }
>   
>   /**
>    * tipc_named_node_up - tell specified node about all publications by this node
>    */
> -void tipc_named_node_up(struct net *net, u32 dnode)
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
>   	struct sk_buff_head head;
> +	u16 seqno;
>   
>   	__skb_queue_head_init(&head);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests++;
> +	seqno = nt->snd_nxt;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	read_lock_bh(&nt->cluster_scope_lock);
> -	named_distribute(net, &head, dnode, &nt->cluster_scope);
> +	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>   	tipc_node_xmit(net, &head, dnode, 0);
>   	read_unlock_bh(&nt->cluster_scope_lock);
>   }
> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities)
>   {
> +	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
> +
>   	struct publication *publ, *tmp;
>   
>   	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>   		tipc_publ_purge(net, publ, addr);
>   	tipc_dist_queue_purge(net, addr);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests--;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
>   /**
> @@ -295,29 +320,61 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
>   	return false;
>   }
>   
> +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
> +				   u16 *rcv_nxt, bool *open)
> +{
> +	struct tipc_msg *hdr;
> +	struct sk_buff *skb, *tmp;
> +	u16 seqno;
> +
> +	skb_queue_walk_safe(namedq, skb, tmp) {
> +		skb_linearize(skb);
> +		hdr = buf_msg(skb);
> +		seqno = msg_named_seqno(hdr);
> +		if (msg_is_last_bulk(hdr)) {
> +			*rcv_nxt = seqno;
> +			*open = true;
> +		}
> +		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (*open && (*rcv_nxt == seqno)) {
> +			(*rcv_nxt)++;
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (less(seqno, *rcv_nxt)) {
> +			__skb_unlink(skb, namedq);
> +			kfree_skb(skb);
> +			continue;
> +		}
This test is not needed, since we will never receive duplicate messages.

> +	}
> +	return NULL;
> +}
> +
>   /**
>    * tipc_named_rcv - process name table update messages sent by another node
>    */
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open)
>   {
> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	struct tipc_msg *msg;
> +	struct tipc_net *tn = tipc_net(net);
>   	struct distr_item *item;
> -	uint count;
> -	u32 node;
> +	struct tipc_msg *hdr;
>   	struct sk_buff *skb;
> -	int mtype;
> +	u32 count, node = 0;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
> -		skb_linearize(skb);
> -		msg = buf_msg(skb);
> -		mtype = msg_type(msg);
> -		item = (struct distr_item *)msg_data(msg);
> -		count = msg_data_sz(msg) / ITEM_SIZE;
> -		node = msg_orignode(msg);
> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
> +		hdr = buf_msg(skb);
> +		node = msg_orignode(hdr);
> +		item = (struct distr_item *)msg_data(hdr);
> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>   		while (count--) {
> -			tipc_update_nametbl(net, item, node, mtype);
> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>   			item++;
>   		}
>   		kfree_skb(skb);
> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>   		publ->node = self;
>   	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>   		publ->node = self;
> -
> +	nt->rc_dests = 0;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
> index 63fc73e0fa6c..092323158f06 100644
> --- a/net/tipc/name_distr.h
> +++ b/net/tipc/name_distr.h
> @@ -67,11 +67,14 @@ struct distr_item {
>   	__be32 key;
>   };
>   
> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>   struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
> -void tipc_named_node_up(struct net *net, u32 dnode);
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open);
>   void tipc_named_reinit(struct net *net);
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities);
>   
>   #endif
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 359b2bc888cf..2ac33d32edc2 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   	struct tipc_net *tn = tipc_net(net);
>   	struct publication *p = NULL;
>   	struct sk_buff *skb = NULL;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   		nt->local_publ_count++;
>   		skb = tipc_named_publish(net, p);
>   	}
> +	rc_dests = nt->rc_dests;
>   exit:
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   	return p;
> +
>   }
>   
>   /**
> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   	u32 self = tipc_own_addr(net);
>   	struct sk_buff *skb = NULL;
>   	struct publication *p;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>   		       type, lower, upper, key);
>   	}
> +	rc_dests = nt->rc_dests;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   		return 1;
>   	}
>   	return 0;
> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
> index 728bc7016c38..8064e1986e2c 100644
> --- a/net/tipc/name_table.h
> +++ b/net/tipc/name_table.h
> @@ -106,6 +106,8 @@ struct name_table {
>   	struct list_head cluster_scope;
>   	rwlock_t cluster_scope_lock;
>   	u32 local_publ_count;
> +	u32 rc_dests;
> +	u32 snd_nxt;
>   };
>   
>   int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
> diff --git a/net/tipc/node.c b/net/tipc/node.c
> index 803a3a6d0f50..ad8d7bce1f98 100644
> --- a/net/tipc/node.c
> +++ b/net/tipc/node.c
> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>   	struct sk_buff_head arrvq;
>   	struct sk_buff_head inputq2;
>   	struct sk_buff_head namedq;
> +	u16 named_rcv_nxt;
> +	bool named_open;
>   };
>   
>   /**
> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>   	write_unlock_bh(&n->lock);
>   
>   	if (flags & TIPC_NOTIFY_NODE_DOWN)
> -		tipc_publ_notify(net, publ_list, addr);
> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_NODE_UP)
> -		tipc_named_node_up(net, addr);
> +		tipc_named_node_up(net, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_LINK_UP) {
>   		tipc_mon_peer_up(net, addr, bearer_id);
> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>   	return 0;
>   }
>   
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>   {
> +	struct sk_buff_head xmitq;
>   	struct sk_buff *txskb;
>   	struct tipc_node *n;
> +	u16 dummy;
>   	u32 dst;
>   
> +	/* Use broadcast if all nodes support it */
> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
> +		__skb_queue_head_init(&xmitq);
> +		__skb_queue_tail(&xmitq, skb);
> +		tipc_bcast_xmit(net, &xmitq, &dummy);
> +		return;
> +	}
> +
> +	/* Otherwise use legacy replicast method */
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>   		dst = n->addr;
> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>   		tipc_node_xmit_skb(net, txskb, dst, 0);
>   	}
>   	rcu_read_unlock();
> -
>   	kfree_skb(skb);
>   }
>   
> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>   
>   	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>   	if (!skb_queue_empty(&n->bc_entry.namedq))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	/* If reassembly or retransmission failure => reset all links to peer */
>   	if (rc & TIPC_LINK_DOWN_EVT)
> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>   		tipc_node_link_down(n, bearer_id, false);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>   		tipc_node_mcast_rcv(n);
> diff --git a/net/tipc/node.h b/net/tipc/node.h
> index a6803b449a2c..9f6f13f1604f 100644
> --- a/net/tipc/node.h
> +++ b/net/tipc/node.h
> @@ -55,7 +55,8 @@ enum {
>   	TIPC_MCAST_RBCTL      = (1 << 7),
>   	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>   	TIPC_TUNNEL_ENHANCED  = (1 << 9),
> -	TIPC_NAGLE            = (1 << 10)
> +	TIPC_NAGLE            = (1 << 10),
> +	TIPC_NAMED_BCAST      = (1 << 11)
>   };
>   
>   #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
> @@ -68,7 +69,8 @@ enum {
>   				TIPC_MCAST_RBCTL       |   \
>   				TIPC_GAP_ACK_BLOCK     |   \
>   				TIPC_TUNNEL_ENHANCED   |   \
> -				TIPC_NAGLE)
> +				TIPC_NAGLE             |   \
> +				TIPC_NAMED_BCAST)
>   
>   #define INVALID_BEARER_ID -1
>   
> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>   		       u32 selector);
>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>   void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);
Provided the testing didn't reveal any new problems;
Acked-by: Jon Maloy <jm...@re...>

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

Hi Jon,

See my inline comment.

Regards,
Hoang
-----Original Message-----
From: Jon Maloy <jm...@re...> 
Sent: Thursday, June 4, 2020 8:42 PM
To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
Subject: Re: [net-next] tipc: update a binding service via broadcast



On 6/4/20 5:21 AM, Hoang Huu Le wrote:
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
>
> It is worth to use broadcast to update a binding service.
s/Then binding table updates in/all nodes in one shoy. This way, the 
binding table can be updated on all peer nodes in one shot./

Broadcast is used when all peer nodes, as indicated by a new capability 
flag TIPC_NAMED_BCAST, support reception of this message type.

Four problems need to be considered when introducing this feature.
1) When establishing a link to a new peer node we still update this by a 
unicast 'bulk' update. This may lead to race conditions, where a later 
broadcast publication/withdrawal bypass the 'bulk', resulting in 
disordered publications, or even that a withdrawal may arrive before the 
corresponding publication. We solve this by adding an 'is_last_bulk' bit 
in the last bulk messages so that it can be distinguished from all other 
messages. Only when this message has arrived do we open up for reception 
of broadcast publications/withdrawals.
2) When a first legacy node is added to the cluster all distribution 
will switch over to use the legacy 'replicast' method, while the 
opposite happens when the last legacy node leaves the cluster. This 
entails another risk of message disordering that has to be handled. We 
solve this by adding a sequence number to the broadcast/replicast 
messages, so that disordering can be discovered and corrected. Note 
however that we don't need to consider potential message loss or 
duplication at this protocol level.
3) Bulk messages don't contain any sequence numbers, and will always 
arrive in order. Hence we must exempt those from the sequence number 
control and deliver them unconditionally. We solve this by adding a new 
'is_bulk' bit in those messages so that they can be recognized.
4) Legacy messages, which don't contain any new bits or sequence 
numbers, but neither can arrive out of order, also need to be exempt 
from the initial synchronization and sequence number check, and 
delivered unconditionally. Therefore, we add another 'is_not_legacy' bit 
to all new messages so that those can be distinguished from legacy 
messages and the latter delivered directly.

> Signed-off-by: Hoang Huu Le <hoa...@de...>
> ---
>   net/tipc/bcast.c      |   6 +--
>   net/tipc/bcast.h      |   4 +-
>   net/tipc/link.c       |   2 +-
>   net/tipc/msg.h        |  40 +++++++++++++++
>   net/tipc/name_distr.c | 115 +++++++++++++++++++++++++++++++-----------
>   net/tipc/name_distr.h |   9 ++--
>   net/tipc/name_table.c |   9 +++-
>   net/tipc/name_table.h |   2 +
>   net/tipc/node.c       |  28 +++++++---
>   net/tipc/node.h       |   8 +--
>   10 files changed, 175 insertions(+), 48 deletions(-)
>
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index 4c20be08b9c4..9d085ad6f0cf 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -249,8 +249,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
>    * Consumes the buffer chain.
>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>    */
> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> -			   u16 *cong_link_cnt)
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt)
>   {
>   	struct tipc_link *l = tipc_bc_sndlink(net);
>   	struct sk_buff_head xmitq;
> @@ -746,7 +746,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>   	nl->local = false;
>   }
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net)
> +u32 tipc_bcast_get_mode(struct net *net)
>   {
>   	struct tipc_bc_base *bb = tipc_bc_base(net);
>   
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index 9e847d9617d3..b3b883e2a823 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -89,6 +89,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>   		    struct tipc_mc_method *method, struct tipc_nlist *dests,
>   		    u16 *cong_link_cnt);
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt);
>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>   			struct tipc_msg *hdr);
> @@ -98,7 +100,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>   int tipc_bclink_reset_stats(struct net *net);
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net);
> +u32 tipc_bcast_get_mode(struct net *net);
>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>   
>   void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> diff --git a/net/tipc/link.c b/net/tipc/link.c
> index d4675e922a8f..da0b30733549 100644
> --- a/net/tipc/link.c
> +++ b/net/tipc/link.c
> @@ -2646,7 +2646,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg)
>   	struct nlattr *attrs;
>   	struct nlattr *prop;
>   	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
> +	u32 bc_mode = tipc_bcast_get_mode(net);
>   	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>   	struct tipc_link *bcl = tn->bcl;
>   
> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
> index 871feadbbc19..d53914316684 100644
> --- a/net/tipc/msg.h
> +++ b/net/tipc/msg.h
> @@ -409,6 +409,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
>   	msg_set_bits(m, 1, 25, 0xf, err);
>   }
>   
> +static inline void msg_set_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 28, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 28, 0x1);
> +}
> +
> +static inline void msg_set_last_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 27, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 27, 0x1);
> +}
> +
> +static inline void msg_set_non_legacy(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 26, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_legacy(struct tipc_msg *m)
> +{
> +	return !msg_bits(m, 1, 26, 0x1);
> +}
> +
>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>   {
>   	return msg_bits(m, 1, 21, 0xf);
> @@ -538,6 +568,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
>   	msg_set_word(m, 4, p);
>   }
>   
> +static inline u16 msg_named_seqno(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 4, 0, 0xffff);
> +}
> +
> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
> +{
> +	msg_set_bits(m, 4, 0, 0xffff, n);
> +}
> +
>   static inline u32 msg_destport(struct tipc_msg *m)
>   {
>   	return msg_word(m, 5);
> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
> index 5feaf3b67380..f4bbb8e792c9 100644
> --- a/net/tipc/name_distr.c
> +++ b/net/tipc/name_distr.c
> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   		pr_warn("Publication distribution failure\n");
>   		return NULL;
>   	}
> -
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
>   	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
>   	return skb;
> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> -	struct sk_buff *buf;
>   	struct distr_item *item;
> +	struct sk_buff *skb;
>   
>   	write_lock_bh(&nt->cluster_scope_lock);
>   	list_del(&publ->binding_node);
> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   	if (publ->scope == TIPC_NODE_SCOPE)
>   		return NULL;
>   
> -	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> -	if (!buf) {
> +	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> +	if (!skb) {
>   		pr_warn("Withdrawal distribution failure\n");
>   		return NULL;
>   	}
> -
> -	item = (struct distr_item *)msg_data(buf_msg(buf));
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
> +	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
> -	return buf;
> +	return skb;
>   }
>   
>   /**
> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>    * @pls: linked list of publication items to be packed into buffer chain
>    */
>   static void named_distribute(struct net *net, struct sk_buff_head *list,
> -			     u32 dnode, struct list_head *pls)
> +			     u32 dnode, struct list_head *pls, u16 seqno)
>   {
>   	struct publication *publ;
>   	struct sk_buff *skb = NULL;
> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
>   			ITEM_SIZE) * ITEM_SIZE;
>   	u32 msg_rem = msg_dsz;
> +	struct tipc_msg *hdr;
>   
>   	list_for_each_entry(publ, pls, binding_node) {
>   		/* Prepare next buffer: */
> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   				pr_warn("Bulk publication failure\n");
>   				return;
>   			}
> -			msg_set_bc_ack_invalid(buf_msg(skb), true);
> -			item = (struct distr_item *)msg_data(buf_msg(skb));
> +			hdr = buf_msg(skb);
> +			msg_set_bc_ack_invalid(hdr, true);
> +			msg_set_bulk(hdr);
> +			msg_set_non_legacy(hdr);
> +			item = (struct distr_item *)msg_data(hdr);
>   		}
>   
>   		/* Pack publication into message: */
> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   		}
>   	}
>   	if (skb) {
> -		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
> +		hdr = buf_msg(skb);
> +		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>   		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>   		__skb_queue_tail(list, skb);
>   	}
> +	hdr = buf_msg(skb_peek_tail(list));
> +	msg_set_last_bulk(hdr);
> +	msg_set_named_seqno(hdr, seqno);
>   }
>   
>   /**
>    * tipc_named_node_up - tell specified node about all publications by this node
>    */
> -void tipc_named_node_up(struct net *net, u32 dnode)
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
>   	struct sk_buff_head head;
> +	u16 seqno;
>   
>   	__skb_queue_head_init(&head);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests++;
> +	seqno = nt->snd_nxt;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	read_lock_bh(&nt->cluster_scope_lock);
> -	named_distribute(net, &head, dnode, &nt->cluster_scope);
> +	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>   	tipc_node_xmit(net, &head, dnode, 0);
>   	read_unlock_bh(&nt->cluster_scope_lock);
>   }
> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities)
>   {
> +	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
> +
>   	struct publication *publ, *tmp;
>   
>   	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>   		tipc_publ_purge(net, publ, addr);
>   	tipc_dist_queue_purge(net, addr);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests--;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
>   /**
> @@ -295,29 +320,61 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
>   	return false;
>   }
>   
> +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
> +				   u16 *rcv_nxt, bool *open)
> +{
> +	struct tipc_msg *hdr;
> +	struct sk_buff *skb, *tmp;
> +	u16 seqno;
> +
> +	skb_queue_walk_safe(namedq, skb, tmp) {
> +		skb_linearize(skb);
> +		hdr = buf_msg(skb);
> +		seqno = msg_named_seqno(hdr);
> +		if (msg_is_last_bulk(hdr)) {
> +			*rcv_nxt = seqno;
> +			*open = true;
> +		}
> +		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (*open && (*rcv_nxt == seqno)) {
> +			(*rcv_nxt)++;
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (less(seqno, *rcv_nxt)) {
> +			__skb_unlink(skb, namedq);
> +			kfree_skb(skb);
> +			continue;
> +		}
This test is not needed, since we will never receive duplicate messages.
[Hoang] No, it could happen as below scenario (not dup):
- rcv_next: 40
- replicast: 40..49 -> in transmit queue
- broadcast: 50..60 -> in transmit queue
Node lost,  re-established, sync rcv_nxt: 60 (via bulk and last bulk), *open = false
Where as in namedq: [40..60] <- those should be dropped.
Sum-up, these publications in namedq become obsoleted when received 'last_bulk'

> +	}
> +	return NULL;
> +}
> +
>   /**
>    * tipc_named_rcv - process name table update messages sent by another node
>    */
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open)
>   {
> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	struct tipc_msg *msg;
> +	struct tipc_net *tn = tipc_net(net);
>   	struct distr_item *item;
> -	uint count;
> -	u32 node;
> +	struct tipc_msg *hdr;
>   	struct sk_buff *skb;
> -	int mtype;
> +	u32 count, node = 0;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
> -		skb_linearize(skb);
> -		msg = buf_msg(skb);
> -		mtype = msg_type(msg);
> -		item = (struct distr_item *)msg_data(msg);
> -		count = msg_data_sz(msg) / ITEM_SIZE;
> -		node = msg_orignode(msg);
> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
> +		hdr = buf_msg(skb);
> +		node = msg_orignode(hdr);
> +		item = (struct distr_item *)msg_data(hdr);
> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>   		while (count--) {
> -			tipc_update_nametbl(net, item, node, mtype);
> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>   			item++;
>   		}
>   		kfree_skb(skb);
> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>   		publ->node = self;
>   	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>   		publ->node = self;
> -
> +	nt->rc_dests = 0;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
> index 63fc73e0fa6c..092323158f06 100644
> --- a/net/tipc/name_distr.h
> +++ b/net/tipc/name_distr.h
> @@ -67,11 +67,14 @@ struct distr_item {
>   	__be32 key;
>   };
>   
> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>   struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
> -void tipc_named_node_up(struct net *net, u32 dnode);
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open);
>   void tipc_named_reinit(struct net *net);
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities);
>   
>   #endif
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 359b2bc888cf..2ac33d32edc2 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   	struct tipc_net *tn = tipc_net(net);
>   	struct publication *p = NULL;
>   	struct sk_buff *skb = NULL;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   		nt->local_publ_count++;
>   		skb = tipc_named_publish(net, p);
>   	}
> +	rc_dests = nt->rc_dests;
>   exit:
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   	return p;
> +
>   }
>   
>   /**
> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   	u32 self = tipc_own_addr(net);
>   	struct sk_buff *skb = NULL;
>   	struct publication *p;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>   		       type, lower, upper, key);
>   	}
> +	rc_dests = nt->rc_dests;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   		return 1;
>   	}
>   	return 0;
> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
> index 728bc7016c38..8064e1986e2c 100644
> --- a/net/tipc/name_table.h
> +++ b/net/tipc/name_table.h
> @@ -106,6 +106,8 @@ struct name_table {
>   	struct list_head cluster_scope;
>   	rwlock_t cluster_scope_lock;
>   	u32 local_publ_count;
> +	u32 rc_dests;
> +	u32 snd_nxt;
>   };
>   
>   int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
> diff --git a/net/tipc/node.c b/net/tipc/node.c
> index 803a3a6d0f50..ad8d7bce1f98 100644
> --- a/net/tipc/node.c
> +++ b/net/tipc/node.c
> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>   	struct sk_buff_head arrvq;
>   	struct sk_buff_head inputq2;
>   	struct sk_buff_head namedq;
> +	u16 named_rcv_nxt;
> +	bool named_open;
>   };
>   
>   /**
> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>   	write_unlock_bh(&n->lock);
>   
>   	if (flags & TIPC_NOTIFY_NODE_DOWN)
> -		tipc_publ_notify(net, publ_list, addr);
> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_NODE_UP)
> -		tipc_named_node_up(net, addr);
> +		tipc_named_node_up(net, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_LINK_UP) {
>   		tipc_mon_peer_up(net, addr, bearer_id);
> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>   	return 0;
>   }
>   
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>   {
> +	struct sk_buff_head xmitq;
>   	struct sk_buff *txskb;
>   	struct tipc_node *n;
> +	u16 dummy;
>   	u32 dst;
>   
> +	/* Use broadcast if all nodes support it */
> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
> +		__skb_queue_head_init(&xmitq);
> +		__skb_queue_tail(&xmitq, skb);
> +		tipc_bcast_xmit(net, &xmitq, &dummy);
> +		return;
> +	}
> +
> +	/* Otherwise use legacy replicast method */
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>   		dst = n->addr;
> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>   		tipc_node_xmit_skb(net, txskb, dst, 0);
>   	}
>   	rcu_read_unlock();
> -
>   	kfree_skb(skb);
>   }
>   
> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>   
>   	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>   	if (!skb_queue_empty(&n->bc_entry.namedq))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	/* If reassembly or retransmission failure => reset all links to peer */
>   	if (rc & TIPC_LINK_DOWN_EVT)
> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>   		tipc_node_link_down(n, bearer_id, false);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>   		tipc_node_mcast_rcv(n);
> diff --git a/net/tipc/node.h b/net/tipc/node.h
> index a6803b449a2c..9f6f13f1604f 100644
> --- a/net/tipc/node.h
> +++ b/net/tipc/node.h
> @@ -55,7 +55,8 @@ enum {
>   	TIPC_MCAST_RBCTL      = (1 << 7),
>   	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>   	TIPC_TUNNEL_ENHANCED  = (1 << 9),
> -	TIPC_NAGLE            = (1 << 10)
> +	TIPC_NAGLE            = (1 << 10),
> +	TIPC_NAMED_BCAST      = (1 << 11)
>   };
>   
>   #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
> @@ -68,7 +69,8 @@ enum {
>   				TIPC_MCAST_RBCTL       |   \
>   				TIPC_GAP_ACK_BLOCK     |   \
>   				TIPC_TUNNEL_ENHANCED   |   \
> -				TIPC_NAGLE)
> +				TIPC_NAGLE             |   \
> +				TIPC_NAMED_BCAST)
>   
>   #define INVALID_BEARER_ID -1
>   
> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>   		       u32 selector);
>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>   void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);
Provided the testing didn't reveal any new problems;
Acked-by: Jon Maloy <jm...@re...>

[tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

Currently, updating binding table (add service binding to
name table/withdraw a service binding) is being sent over replicast.
However, if we are scaling up clusters to > 100 nodes/containers this
method is less affection because of looping through nodes in a cluster one
by one.

It is worth to use broadcast to update a binding service. This way, the
binding table can be updated on all peer nodes in one shot.

Broadcast is used when all peer nodes, as indicated by a new capability
flag TIPC_NAMED_BCAST, support reception of this message type.

Four problems need to be considered when introducing this feature.
1) When establishing a link to a new peer node we still update this by a
unicast 'bulk' update. This may lead to race conditions, where a later
broadcast publication/withdrawal bypass the 'bulk', resulting in
disordered publications, or even that a withdrawal may arrive before the
corresponding publication. We solve this by adding an 'is_last_bulk' bit
in the last bulk messages so that it can be distinguished from all other
messages. Only when this message has arrived do we open up for reception
of broadcast publications/withdrawals.
2) When a first legacy node is added to the cluster all distribution
will switch over to use the legacy 'replicast' method, while the
opposite happens when the last legacy node leaves the cluster. This
entails another risk of message disordering that has to be handled. We
solve this by adding a sequence number to the broadcast/replicast
messages, so that disordering can be discovered and corrected. Note
however that we don't need to consider potential message loss or
duplication at this protocol level.
3) Bulk messages don't contain any sequence numbers, and will always
arrive in order. Hence we must exempt those from the sequence number
control and deliver them unconditionally. We solve this by adding a new
'is_bulk' bit in those messages so that they can be recognized.
4) Legacy messages, which don't contain any new bits or sequence
numbers, but neither can arrive out of order, also need to be exempt
from the initial synchronization and sequence number check, and
delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
to all new messages so that those can be distinguished from legacy
messages and the latter delivered directly.

v2: resolve synchronization problem when switching from unicast to
broadcast

v5: - never use broadcast if there is a single node not understanding it
    - always use broadcast otherwise
    - add sequence numbering to non-bulk messages

v6: update Jon's comment

Signed-off-by: Hoang Huu Le <hoa...@de...>
Acked-by: Jon Maloy <jm...@re...>
---
 net/tipc/bcast.c      |   6 +--
 net/tipc/bcast.h      |   4 +-
 net/tipc/link.c       |   2 +-
 net/tipc/msg.h        |  40 +++++++++++++++
 net/tipc/name_distr.c | 115 +++++++++++++++++++++++++++++++-----------
 net/tipc/name_distr.h |   9 ++--
 net/tipc/name_table.c |   9 +++-
 net/tipc/name_table.h |   2 +
 net/tipc/node.c       |  28 +++++++---
 net/tipc/node.h       |   8 +--
 10 files changed, 175 insertions(+), 48 deletions(-)

diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
index 4c20be08b9c4..9d085ad6f0cf 100644
--- a/net/tipc/bcast.c
+++ b/net/tipc/bcast.c
@@ -249,8 +249,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
  * Consumes the buffer chain.
  * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
  */
-static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
-			   u16 *cong_link_cnt)
+int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
+		    u16 *cong_link_cnt)
 {
 	struct tipc_link *l = tipc_bc_sndlink(net);
 	struct sk_buff_head xmitq;
@@ -746,7 +746,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
 	nl->local = false;
 }
 
-u32 tipc_bcast_get_broadcast_mode(struct net *net)
+u32 tipc_bcast_get_mode(struct net *net)
 {
 	struct tipc_bc_base *bb = tipc_bc_base(net);
 
diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
index 9e847d9617d3..b3b883e2a823 100644
--- a/net/tipc/bcast.h
+++ b/net/tipc/bcast.h
@@ -89,6 +89,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
 int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
 		    struct tipc_mc_method *method, struct tipc_nlist *dests,
 		    u16 *cong_link_cnt);
+int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
+		    u16 *cong_link_cnt);
 int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
 void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
 			struct tipc_msg *hdr);
@@ -98,7 +100,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
 int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
 int tipc_bclink_reset_stats(struct net *net);
 
-u32 tipc_bcast_get_broadcast_mode(struct net *net);
+u32 tipc_bcast_get_mode(struct net *net);
 u32 tipc_bcast_get_broadcast_ratio(struct net *net);
 
 void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
diff --git a/net/tipc/link.c b/net/tipc/link.c
index d4675e922a8f..da0b30733549 100644
--- a/net/tipc/link.c
+++ b/net/tipc/link.c
@@ -2646,7 +2646,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg)
 	struct nlattr *attrs;
 	struct nlattr *prop;
 	struct tipc_net *tn = net_generic(net, tipc_net_id);
-	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
+	u32 bc_mode = tipc_bcast_get_mode(net);
 	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
 	struct tipc_link *bcl = tn->bcl;
 
diff --git a/net/tipc/msg.h b/net/tipc/msg.h
index 871feadbbc19..d53914316684 100644
--- a/net/tipc/msg.h
+++ b/net/tipc/msg.h
@@ -409,6 +409,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
 	msg_set_bits(m, 1, 25, 0xf, err);
 }
 
+static inline void msg_set_bulk(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 28, 0x1, 1);
+}
+
+static inline u32 msg_is_bulk(struct tipc_msg *m)
+{
+	return msg_bits(m, 1, 28, 0x1);
+}
+
+static inline void msg_set_last_bulk(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 27, 0x1, 1);
+}
+
+static inline u32 msg_is_last_bulk(struct tipc_msg *m)
+{
+	return msg_bits(m, 1, 27, 0x1);
+}
+
+static inline void msg_set_non_legacy(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 26, 0x1, 1);
+}
+
+static inline u32 msg_is_legacy(struct tipc_msg *m)
+{
+	return !msg_bits(m, 1, 26, 0x1);
+}
+
 static inline u32 msg_reroute_cnt(struct tipc_msg *m)
 {
 	return msg_bits(m, 1, 21, 0xf);
@@ -538,6 +568,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
 	msg_set_word(m, 4, p);
 }
 
+static inline u16 msg_named_seqno(struct tipc_msg *m)
+{
+	return msg_bits(m, 4, 0, 0xffff);
+}
+
+static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
+{
+	msg_set_bits(m, 4, 0, 0xffff, n);
+}
+
 static inline u32 msg_destport(struct tipc_msg *m)
 {
 	return msg_word(m, 5);
diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
index 5feaf3b67380..6cbd4b271768 100644
--- a/net/tipc/name_distr.c
+++ b/net/tipc/name_distr.c
@@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
 		pr_warn("Publication distribution failure\n");
 		return NULL;
 	}
-
+	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
+	msg_set_non_legacy(buf_msg(skb));
 	item = (struct distr_item *)msg_data(buf_msg(skb));
 	publ_to_item(item, publ);
 	return skb;
@@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
 struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
 {
 	struct name_table *nt = tipc_name_table(net);
-	struct sk_buff *buf;
 	struct distr_item *item;
+	struct sk_buff *skb;
 
 	write_lock_bh(&nt->cluster_scope_lock);
 	list_del(&publ->binding_node);
@@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
 	if (publ->scope == TIPC_NODE_SCOPE)
 		return NULL;
 
-	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
-	if (!buf) {
+	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
+	if (!skb) {
 		pr_warn("Withdrawal distribution failure\n");
 		return NULL;
 	}
-
-	item = (struct distr_item *)msg_data(buf_msg(buf));
+	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
+	msg_set_non_legacy(buf_msg(skb));
+	item = (struct distr_item *)msg_data(buf_msg(skb));
 	publ_to_item(item, publ);
-	return buf;
+	return skb;
 }
 
 /**
@@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
  * @pls: linked list of publication items to be packed into buffer chain
  */
 static void named_distribute(struct net *net, struct sk_buff_head *list,
-			     u32 dnode, struct list_head *pls)
+			     u32 dnode, struct list_head *pls, u16 seqno)
 {
 	struct publication *publ;
 	struct sk_buff *skb = NULL;
@@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
 			ITEM_SIZE) * ITEM_SIZE;
 	u32 msg_rem = msg_dsz;
+	struct tipc_msg *hdr;
 
 	list_for_each_entry(publ, pls, binding_node) {
 		/* Prepare next buffer: */
@@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 				pr_warn("Bulk publication failure\n");
 				return;
 			}
-			msg_set_bc_ack_invalid(buf_msg(skb), true);
-			item = (struct distr_item *)msg_data(buf_msg(skb));
+			hdr = buf_msg(skb);
+			msg_set_bc_ack_invalid(hdr, true);
+			msg_set_bulk(hdr);
+			msg_set_non_legacy(hdr);
+			item = (struct distr_item *)msg_data(hdr);
 		}
 
 		/* Pack publication into message: */
@@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 		}
 	}
 	if (skb) {
-		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
+		hdr = buf_msg(skb);
+		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
 		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
 		__skb_queue_tail(list, skb);
 	}
+	hdr = buf_msg(skb_peek_tail(list));
+	msg_set_last_bulk(hdr);
+	msg_set_named_seqno(hdr, seqno);
 }
 
 /**
  * tipc_named_node_up - tell specified node about all publications by this node
  */
-void tipc_named_node_up(struct net *net, u32 dnode)
+void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
 {
 	struct name_table *nt = tipc_name_table(net);
+	struct tipc_net *tn = tipc_net(net);
 	struct sk_buff_head head;
+	u16 seqno;
 
 	__skb_queue_head_init(&head);
+	spin_lock_bh(&tn->nametbl_lock);
+	if (!(capabilities & TIPC_NAMED_BCAST))
+		nt->rc_dests++;
+	seqno = nt->snd_nxt;
+	spin_unlock_bh(&tn->nametbl_lock);
 
 	read_lock_bh(&nt->cluster_scope_lock);
-	named_distribute(net, &head, dnode, &nt->cluster_scope);
+	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
 	tipc_node_xmit(net, &head, dnode, 0);
 	read_unlock_bh(&nt->cluster_scope_lock);
 }
@@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
 	spin_unlock_bh(&tn->nametbl_lock);
 }
 
-void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
+void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
+		      u32 addr, u16 capabilities)
 {
+	struct name_table *nt = tipc_name_table(net);
+	struct tipc_net *tn = tipc_net(net);
+
 	struct publication *publ, *tmp;
 
 	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
 		tipc_publ_purge(net, publ, addr);
 	tipc_dist_queue_purge(net, addr);
+	spin_lock_bh(&tn->nametbl_lock);
+	if (!(capabilities & TIPC_NAMED_BCAST))
+		nt->rc_dests--;
+	spin_unlock_bh(&tn->nametbl_lock);
 }
 
 /**
@@ -295,29 +320,61 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
 	return false;
 }
 
+struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
+				   u16 *rcv_nxt, bool *open)
+{
+	struct sk_buff *skb, *tmp;
+	struct tipc_msg *hdr;
+	u16 seqno;
+
+	skb_queue_walk_safe(namedq, skb, tmp) {
+		skb_linearize(skb);
+		hdr = buf_msg(skb);
+		seqno = msg_named_seqno(hdr);
+		if (msg_is_last_bulk(hdr)) {
+			*rcv_nxt = seqno;
+			*open = true;
+		}
+		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
+			__skb_unlink(skb, namedq);
+			return skb;
+		}
+
+		if (*open && (*rcv_nxt == seqno)) {
+			(*rcv_nxt)++;
+			__skb_unlink(skb, namedq);
+			return skb;
+		}
+
+		if (less(seqno, *rcv_nxt)) {
+			__skb_unlink(skb, namedq);
+			kfree_skb(skb);
+			continue;
+		}
+	}
+	return NULL;
+}
+
 /**
  * tipc_named_rcv - process name table update messages sent by another node
  */
-void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
+void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
+		    u16 *rcv_nxt, bool *open)
 {
-	struct tipc_net *tn = net_generic(net, tipc_net_id);
-	struct tipc_msg *msg;
+	struct tipc_net *tn = tipc_net(net);
 	struct distr_item *item;
-	uint count;
-	u32 node;
+	struct tipc_msg *hdr;
 	struct sk_buff *skb;
-	int mtype;
+	u32 count, node = 0;
 
 	spin_lock_bh(&tn->nametbl_lock);
-	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
-		skb_linearize(skb);
-		msg = buf_msg(skb);
-		mtype = msg_type(msg);
-		item = (struct distr_item *)msg_data(msg);
-		count = msg_data_sz(msg) / ITEM_SIZE;
-		node = msg_orignode(msg);
+	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
+		hdr = buf_msg(skb);
+		node = msg_orignode(hdr);
+		item = (struct distr_item *)msg_data(hdr);
+		count = msg_data_sz(hdr) / ITEM_SIZE;
 		while (count--) {
-			tipc_update_nametbl(net, item, node, mtype);
+			tipc_update_nametbl(net, item, node, msg_type(hdr));
 			item++;
 		}
 		kfree_skb(skb);
@@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
 		publ->node = self;
 	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
 		publ->node = self;
-
+	nt->rc_dests = 0;
 	spin_unlock_bh(&tn->nametbl_lock);
 }
diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
index 63fc73e0fa6c..092323158f06 100644
--- a/net/tipc/name_distr.h
+++ b/net/tipc/name_distr.h
@@ -67,11 +67,14 @@ struct distr_item {
 	__be32 key;
 };
 
+void tipc_named_bcast(struct net *net, struct sk_buff *skb);
 struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
 struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
-void tipc_named_node_up(struct net *net, u32 dnode);
-void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
+void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
+void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
+		    u16 *rcv_nxt, bool *open);
 void tipc_named_reinit(struct net *net);
-void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
+void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
+		      u32 addr, u16 capabilities);
 
 #endif
diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
index 359b2bc888cf..2ac33d32edc2 100644
--- a/net/tipc/name_table.c
+++ b/net/tipc/name_table.c
@@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 	struct tipc_net *tn = tipc_net(net);
 	struct publication *p = NULL;
 	struct sk_buff *skb = NULL;
+	u32 rc_dests;
 
 	spin_lock_bh(&tn->nametbl_lock);
 
@@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 		nt->local_publ_count++;
 		skb = tipc_named_publish(net, p);
 	}
+	rc_dests = nt->rc_dests;
 exit:
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb)
-		tipc_node_broadcast(net, skb);
+		tipc_node_broadcast(net, skb, rc_dests);
 	return p;
+
 }
 
 /**
@@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 	u32 self = tipc_own_addr(net);
 	struct sk_buff *skb = NULL;
 	struct publication *p;
+	u32 rc_dests;
 
 	spin_lock_bh(&tn->nametbl_lock);
 
@@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
 		       type, lower, upper, key);
 	}
+	rc_dests = nt->rc_dests;
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb) {
-		tipc_node_broadcast(net, skb);
+		tipc_node_broadcast(net, skb, rc_dests);
 		return 1;
 	}
 	return 0;
diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
index 728bc7016c38..8064e1986e2c 100644
--- a/net/tipc/name_table.h
+++ b/net/tipc/name_table.h
@@ -106,6 +106,8 @@ struct name_table {
 	struct list_head cluster_scope;
 	rwlock_t cluster_scope_lock;
 	u32 local_publ_count;
+	u32 rc_dests;
+	u32 snd_nxt;
 };
 
 int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/tipc/node.c b/net/tipc/node.c
index 803a3a6d0f50..ad8d7bce1f98 100644
--- a/net/tipc/node.c
+++ b/net/tipc/node.c
@@ -75,6 +75,8 @@ struct tipc_bclink_entry {
 	struct sk_buff_head arrvq;
 	struct sk_buff_head inputq2;
 	struct sk_buff_head namedq;
+	u16 named_rcv_nxt;
+	bool named_open;
 };
 
 /**
@@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
 	write_unlock_bh(&n->lock);
 
 	if (flags & TIPC_NOTIFY_NODE_DOWN)
-		tipc_publ_notify(net, publ_list, addr);
+		tipc_publ_notify(net, publ_list, addr, n->capabilities);
 
 	if (flags & TIPC_NOTIFY_NODE_UP)
-		tipc_named_node_up(net, addr);
+		tipc_named_node_up(net, addr, n->capabilities);
 
 	if (flags & TIPC_NOTIFY_LINK_UP) {
 		tipc_mon_peer_up(net, addr, bearer_id);
@@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
 	return 0;
 }
 
-void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
+void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
 {
+	struct sk_buff_head xmitq;
 	struct sk_buff *txskb;
 	struct tipc_node *n;
+	u16 dummy;
 	u32 dst;
 
+	/* Use broadcast if all nodes support it */
+	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
+		__skb_queue_head_init(&xmitq);
+		__skb_queue_tail(&xmitq, skb);
+		tipc_bcast_xmit(net, &xmitq, &dummy);
+		return;
+	}
+
+	/* Otherwise use legacy replicast method */
 	rcu_read_lock();
 	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
 		dst = n->addr;
@@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
 		tipc_node_xmit_skb(net, txskb, dst, 0);
 	}
 	rcu_read_unlock();
-
 	kfree_skb(skb);
 }
 
@@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
 
 	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
 	if (!skb_queue_empty(&n->bc_entry.namedq))
-		tipc_named_rcv(net, &n->bc_entry.namedq);
+		tipc_named_rcv(net, &n->bc_entry.namedq,
+			       &n->bc_entry.named_rcv_nxt,
+			       &n->bc_entry.named_open);
 
 	/* If reassembly or retransmission failure => reset all links to peer */
 	if (rc & TIPC_LINK_DOWN_EVT)
@@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
 		tipc_node_link_down(n, bearer_id, false);
 
 	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
-		tipc_named_rcv(net, &n->bc_entry.namedq);
+		tipc_named_rcv(net, &n->bc_entry.namedq,
+			       &n->bc_entry.named_rcv_nxt,
+			       &n->bc_entry.named_open);
 
 	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
 		tipc_node_mcast_rcv(n);
diff --git a/net/tipc/node.h b/net/tipc/node.h
index a6803b449a2c..9f6f13f1604f 100644
--- a/net/tipc/node.h
+++ b/net/tipc/node.h
@@ -55,7 +55,8 @@ enum {
 	TIPC_MCAST_RBCTL      = (1 << 7),
 	TIPC_GAP_ACK_BLOCK    = (1 << 8),
 	TIPC_TUNNEL_ENHANCED  = (1 << 9),
-	TIPC_NAGLE            = (1 << 10)
+	TIPC_NAGLE            = (1 << 10),
+	TIPC_NAMED_BCAST      = (1 << 11)
 };
 
 #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
@@ -68,7 +69,8 @@ enum {
 				TIPC_MCAST_RBCTL       |   \
 				TIPC_GAP_ACK_BLOCK     |   \
 				TIPC_TUNNEL_ENHANCED   |   \
-				TIPC_NAGLE)
+				TIPC_NAGLE             |   \
+				TIPC_NAMED_BCAST)
 
 #define INVALID_BEARER_ID -1
 
@@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
 		       u32 selector);
 void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
 void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
-void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
+void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
 int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
 void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
 int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);
-- 
2.25.1

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/5/20 3:52 AM, Hoang Huu Le wrote:
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
>
> It is worth to use broadcast to update a binding service. This way, the
> binding table can be updated on all peer nodes in one shot.
>
> Broadcast is used when all peer nodes, as indicated by a new capability
> flag TIPC_NAMED_BCAST, support reception of this message type.
>
> Four problems need to be considered when introducing this feature.
> 1) When establishing a link to a new peer node we still update this by a
> unicast 'bulk' update. This may lead to race conditions, where a later
> broadcast publication/withdrawal bypass the 'bulk', resulting in
> disordered publications, or even that a withdrawal may arrive before the
> corresponding publication. We solve this by adding an 'is_last_bulk' bit
> in the last bulk messages so that it can be distinguished from all other
> messages. Only when this message has arrived do we open up for reception
> of broadcast publications/withdrawals.
> 2) When a first legacy node is added to the cluster all distribution
> will switch over to use the legacy 'replicast' method, while the
> opposite happens when the last legacy node leaves the cluster. This
> entails another risk of message disordering that has to be handled. We
> solve this by adding a sequence number to the broadcast/replicast
> messages, so that disordering can be discovered and corrected. Note
> however that we don't need to consider potential message loss or
> duplication at this protocol level.
> 3) Bulk messages don't contain any sequence numbers, and will always
> arrive in order. Hence we must exempt those from the sequence number
> control and deliver them unconditionally. We solve this by adding a new
> 'is_bulk' bit in those messages so that they can be recognized.
> 4) Legacy messages, which don't contain any new bits or sequence
> numbers, but neither can arrive out of order, also need to be exempt
> from the initial synchronization and sequence number check, and
> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
> to all new messages so that those can be distinguished from legacy
> messages and the latter delivered directly.
---
If you add the above three dashes before the version info that will not 
by accident be considered part of the commit log.
But of course our internal versioning is irrelevant to David M, so you 
should just remove this info before posting.
> v2: resolve synchronization problem when switching from unicast to
> broadcast
>
> v5: - never use broadcast if there is a single node not understanding it
>      - always use broadcast otherwise
>      - add sequence numbering to non-bulk messages
>
> v6: update Jon's comment
>
> Signed-off-by: Hoang Huu Le <hoa...@de...>
> Acked-by: Jon Maloy <jm...@re...>
> ---
>   net/tipc/bcast.c      |   6 +--
>   net/tipc/bcast.h      |   4 +-
>   net/tipc/link.c       |   2 +-
>   net/tipc/msg.h        |  40 +++++++++++++++
>   net/tipc/name_distr.c | 115 +++++++++++++++++++++++++++++++-----------
>   net/tipc/name_distr.h |   9 ++--
>   net/tipc/name_table.c |   9 +++-
>   net/tipc/name_table.h |   2 +
>   net/tipc/node.c       |  28 +++++++---
>   net/tipc/node.h       |   8 +--
>   10 files changed, 175 insertions(+), 48 deletions(-)
>
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index 4c20be08b9c4..9d085ad6f0cf 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -249,8 +249,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
>    * Consumes the buffer chain.
>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>    */
> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> -			   u16 *cong_link_cnt)
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt)
>   {
>   	struct tipc_link *l = tipc_bc_sndlink(net);
>   	struct sk_buff_head xmitq;
> @@ -746,7 +746,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>   	nl->local = false;
>   }
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net)
> +u32 tipc_bcast_get_mode(struct net *net)
>   {
>   	struct tipc_bc_base *bb = tipc_bc_base(net);
>   
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index 9e847d9617d3..b3b883e2a823 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -89,6 +89,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>   		    struct tipc_mc_method *method, struct tipc_nlist *dests,
>   		    u16 *cong_link_cnt);
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt);
>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>   			struct tipc_msg *hdr);
> @@ -98,7 +100,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>   int tipc_bclink_reset_stats(struct net *net);
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net);
> +u32 tipc_bcast_get_mode(struct net *net);
>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>   
>   void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> diff --git a/net/tipc/link.c b/net/tipc/link.c
> index d4675e922a8f..da0b30733549 100644
> --- a/net/tipc/link.c
> +++ b/net/tipc/link.c
> @@ -2646,7 +2646,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg)
>   	struct nlattr *attrs;
>   	struct nlattr *prop;
>   	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
> +	u32 bc_mode = tipc_bcast_get_mode(net);
>   	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>   	struct tipc_link *bcl = tn->bcl;
>   
> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
> index 871feadbbc19..d53914316684 100644
> --- a/net/tipc/msg.h
> +++ b/net/tipc/msg.h
> @@ -409,6 +409,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
>   	msg_set_bits(m, 1, 25, 0xf, err);
>   }
>   
> +static inline void msg_set_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 28, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 28, 0x1);
> +}
> +
> +static inline void msg_set_last_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 27, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 27, 0x1);
> +}
> +
> +static inline void msg_set_non_legacy(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 26, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_legacy(struct tipc_msg *m)
> +{
> +	return !msg_bits(m, 1, 26, 0x1);
> +}
> +
>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>   {
>   	return msg_bits(m, 1, 21, 0xf);
> @@ -538,6 +568,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
>   	msg_set_word(m, 4, p);
>   }
>   
> +static inline u16 msg_named_seqno(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 4, 0, 0xffff);
> +}
> +
> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
> +{
> +	msg_set_bits(m, 4, 0, 0xffff, n);
> +}
> +
>   static inline u32 msg_destport(struct tipc_msg *m)
>   {
>   	return msg_word(m, 5);
> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
> index 5feaf3b67380..6cbd4b271768 100644
> --- a/net/tipc/name_distr.c
> +++ b/net/tipc/name_distr.c
> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   		pr_warn("Publication distribution failure\n");
>   		return NULL;
>   	}
> -
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
>   	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
>   	return skb;
> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> -	struct sk_buff *buf;
>   	struct distr_item *item;
> +	struct sk_buff *skb;
>   
>   	write_lock_bh(&nt->cluster_scope_lock);
>   	list_del(&publ->binding_node);
> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   	if (publ->scope == TIPC_NODE_SCOPE)
>   		return NULL;
>   
> -	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> -	if (!buf) {
> +	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> +	if (!skb) {
>   		pr_warn("Withdrawal distribution failure\n");
>   		return NULL;
>   	}
> -
> -	item = (struct distr_item *)msg_data(buf_msg(buf));
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
> +	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
> -	return buf;
> +	return skb;
>   }
>   
>   /**
> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>    * @pls: linked list of publication items to be packed into buffer chain
>    */
>   static void named_distribute(struct net *net, struct sk_buff_head *list,
> -			     u32 dnode, struct list_head *pls)
> +			     u32 dnode, struct list_head *pls, u16 seqno)
>   {
>   	struct publication *publ;
>   	struct sk_buff *skb = NULL;
> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
>   			ITEM_SIZE) * ITEM_SIZE;
>   	u32 msg_rem = msg_dsz;
> +	struct tipc_msg *hdr;
>   
>   	list_for_each_entry(publ, pls, binding_node) {
>   		/* Prepare next buffer: */
> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   				pr_warn("Bulk publication failure\n");
>   				return;
>   			}
> -			msg_set_bc_ack_invalid(buf_msg(skb), true);
> -			item = (struct distr_item *)msg_data(buf_msg(skb));
> +			hdr = buf_msg(skb);
> +			msg_set_bc_ack_invalid(hdr, true);
> +			msg_set_bulk(hdr);
> +			msg_set_non_legacy(hdr);
> +			item = (struct distr_item *)msg_data(hdr);
>   		}
>   
>   		/* Pack publication into message: */
> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   		}
>   	}
>   	if (skb) {
> -		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
> +		hdr = buf_msg(skb);
> +		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>   		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>   		__skb_queue_tail(list, skb);
>   	}
> +	hdr = buf_msg(skb_peek_tail(list));
> +	msg_set_last_bulk(hdr);
> +	msg_set_named_seqno(hdr, seqno);
>   }
>   
>   /**
>    * tipc_named_node_up - tell specified node about all publications by this node
>    */
> -void tipc_named_node_up(struct net *net, u32 dnode)
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
>   	struct sk_buff_head head;
> +	u16 seqno;
>   
>   	__skb_queue_head_init(&head);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests++;
> +	seqno = nt->snd_nxt;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	read_lock_bh(&nt->cluster_scope_lock);
> -	named_distribute(net, &head, dnode, &nt->cluster_scope);
> +	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>   	tipc_node_xmit(net, &head, dnode, 0);
>   	read_unlock_bh(&nt->cluster_scope_lock);
>   }
> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities)
>   {
> +	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
> +
>   	struct publication *publ, *tmp;
>   
>   	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>   		tipc_publ_purge(net, publ, addr);
>   	tipc_dist_queue_purge(net, addr);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests--;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
>   /**
> @@ -295,29 +320,61 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
>   	return false;
>   }
>   
> +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
> +				   u16 *rcv_nxt, bool *open)
> +{
> +	struct sk_buff *skb, *tmp;
> +	struct tipc_msg *hdr;
> +	u16 seqno;
> +
> +	skb_queue_walk_safe(namedq, skb, tmp) {
> +		skb_linearize(skb);
> +		hdr = buf_msg(skb);
> +		seqno = msg_named_seqno(hdr);
> +		if (msg_is_last_bulk(hdr)) {
> +			*rcv_nxt = seqno;
> +			*open = true;
> +		}
> +		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (*open && (*rcv_nxt == seqno)) {
> +			(*rcv_nxt)++;
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (less(seqno, *rcv_nxt)) {
> +			__skb_unlink(skb, namedq);
> +			kfree_skb(skb);
> +			continue;
Still not needed. This queue should be flushed in 
tipc_node_lost_contact(), which I now see we don't do.
This has to e fixed too.
///jon
> +		}
> +	}
> +	return NULL;
> +}
> +
>   /**
>    * tipc_named_rcv - process name table update messages sent by another node
>    */
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open)
>   {
> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	struct tipc_msg *msg;
> +	struct tipc_net *tn = tipc_net(net);
>   	struct distr_item *item;
> -	uint count;
> -	u32 node;
> +	struct tipc_msg *hdr;
>   	struct sk_buff *skb;
> -	int mtype;
> +	u32 count, node = 0;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
> -		skb_linearize(skb);
> -		msg = buf_msg(skb);
> -		mtype = msg_type(msg);
> -		item = (struct distr_item *)msg_data(msg);
> -		count = msg_data_sz(msg) / ITEM_SIZE;
> -		node = msg_orignode(msg);
> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
> +		hdr = buf_msg(skb);
> +		node = msg_orignode(hdr);
> +		item = (struct distr_item *)msg_data(hdr);
> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>   		while (count--) {
> -			tipc_update_nametbl(net, item, node, mtype);
> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>   			item++;
>   		}
>   		kfree_skb(skb);
> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>   		publ->node = self;
>   	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>   		publ->node = self;
> -
> +	nt->rc_dests = 0;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
> index 63fc73e0fa6c..092323158f06 100644
> --- a/net/tipc/name_distr.h
> +++ b/net/tipc/name_distr.h
> @@ -67,11 +67,14 @@ struct distr_item {
>   	__be32 key;
>   };
>   
> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>   struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
> -void tipc_named_node_up(struct net *net, u32 dnode);
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open);
>   void tipc_named_reinit(struct net *net);
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities);
>   
>   #endif
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 359b2bc888cf..2ac33d32edc2 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   	struct tipc_net *tn = tipc_net(net);
>   	struct publication *p = NULL;
>   	struct sk_buff *skb = NULL;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   		nt->local_publ_count++;
>   		skb = tipc_named_publish(net, p);
>   	}
> +	rc_dests = nt->rc_dests;
>   exit:
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   	return p;
> +
>   }
>   
>   /**
> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   	u32 self = tipc_own_addr(net);
>   	struct sk_buff *skb = NULL;
>   	struct publication *p;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>   		       type, lower, upper, key);
>   	}
> +	rc_dests = nt->rc_dests;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   		return 1;
>   	}
>   	return 0;
> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
> index 728bc7016c38..8064e1986e2c 100644
> --- a/net/tipc/name_table.h
> +++ b/net/tipc/name_table.h
> @@ -106,6 +106,8 @@ struct name_table {
>   	struct list_head cluster_scope;
>   	rwlock_t cluster_scope_lock;
>   	u32 local_publ_count;
> +	u32 rc_dests;
> +	u32 snd_nxt;
>   };
>   
>   int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
> diff --git a/net/tipc/node.c b/net/tipc/node.c
> index 803a3a6d0f50..ad8d7bce1f98 100644
> --- a/net/tipc/node.c
> +++ b/net/tipc/node.c
> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>   	struct sk_buff_head arrvq;
>   	struct sk_buff_head inputq2;
>   	struct sk_buff_head namedq;
> +	u16 named_rcv_nxt;
> +	bool named_open;
>   };
>   
>   /**
> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>   	write_unlock_bh(&n->lock);
>   
>   	if (flags & TIPC_NOTIFY_NODE_DOWN)
> -		tipc_publ_notify(net, publ_list, addr);
> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_NODE_UP)
> -		tipc_named_node_up(net, addr);
> +		tipc_named_node_up(net, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_LINK_UP) {
>   		tipc_mon_peer_up(net, addr, bearer_id);
> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>   	return 0;
>   }
>   
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>   {
> +	struct sk_buff_head xmitq;
>   	struct sk_buff *txskb;
>   	struct tipc_node *n;
> +	u16 dummy;
>   	u32 dst;
>   
> +	/* Use broadcast if all nodes support it */
> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
> +		__skb_queue_head_init(&xmitq);
> +		__skb_queue_tail(&xmitq, skb);
> +		tipc_bcast_xmit(net, &xmitq, &dummy);
> +		return;
> +	}
> +
> +	/* Otherwise use legacy replicast method */
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>   		dst = n->addr;
> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>   		tipc_node_xmit_skb(net, txskb, dst, 0);
>   	}
>   	rcu_read_unlock();
> -
>   	kfree_skb(skb);
>   }
>   
> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>   
>   	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>   	if (!skb_queue_empty(&n->bc_entry.namedq))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	/* If reassembly or retransmission failure => reset all links to peer */
>   	if (rc & TIPC_LINK_DOWN_EVT)
> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>   		tipc_node_link_down(n, bearer_id, false);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>   		tipc_node_mcast_rcv(n);
> diff --git a/net/tipc/node.h b/net/tipc/node.h
> index a6803b449a2c..9f6f13f1604f 100644
> --- a/net/tipc/node.h
> +++ b/net/tipc/node.h
> @@ -55,7 +55,8 @@ enum {
>   	TIPC_MCAST_RBCTL      = (1 << 7),
>   	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>   	TIPC_TUNNEL_ENHANCED  = (1 << 9),
> -	TIPC_NAGLE            = (1 << 10)
> +	TIPC_NAGLE            = (1 << 10),
> +	TIPC_NAMED_BCAST      = (1 << 11)
>   };
>   
>   #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
> @@ -68,7 +69,8 @@ enum {
>   				TIPC_MCAST_RBCTL       |   \
>   				TIPC_GAP_ACK_BLOCK     |   \
>   				TIPC_TUNNEL_ENHANCED   |   \
> -				TIPC_NAGLE)
> +				TIPC_NAGLE             |   \
> +				TIPC_NAMED_BCAST)
>   
>   #define INVALID_BEARER_ID -1
>   
> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>   		       u32 selector);
>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>   void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

-----Original Message-----
From: Jon Maloy <jm...@re...> 
Sent: Friday, June 5, 2020 8:03 PM
To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
Subject: Re: [next-net v6] tipc: update a binding service via broadcast



On 6/5/20 3:52 AM, Hoang Huu Le wrote:
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
>
> It is worth to use broadcast to update a binding service. This way, the
> binding table can be updated on all peer nodes in one shot.
>
> Broadcast is used when all peer nodes, as indicated by a new capability
> flag TIPC_NAMED_BCAST, support reception of this message type.
>
> Four problems need to be considered when introducing this feature.
> 1) When establishing a link to a new peer node we still update this by a
> unicast 'bulk' update. This may lead to race conditions, where a later
> broadcast publication/withdrawal bypass the 'bulk', resulting in
> disordered publications, or even that a withdrawal may arrive before the
> corresponding publication. We solve this by adding an 'is_last_bulk' bit
> in the last bulk messages so that it can be distinguished from all other
> messages. Only when this message has arrived do we open up for reception
> of broadcast publications/withdrawals.
> 2) When a first legacy node is added to the cluster all distribution
> will switch over to use the legacy 'replicast' method, while the
> opposite happens when the last legacy node leaves the cluster. This
> entails another risk of message disordering that has to be handled. We
> solve this by adding a sequence number to the broadcast/replicast
> messages, so that disordering can be discovered and corrected. Note
> however that we don't need to consider potential message loss or
> duplication at this protocol level.
> 3) Bulk messages don't contain any sequence numbers, and will always
> arrive in order. Hence we must exempt those from the sequence number
> control and deliver them unconditionally. We solve this by adding a new
> 'is_bulk' bit in those messages so that they can be recognized.
> 4) Legacy messages, which don't contain any new bits or sequence
> numbers, but neither can arrive out of order, also need to be exempt
> from the initial synchronization and sequence number check, and
> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
> to all new messages so that those can be distinguished from legacy
> messages and the latter delivered directly.
---
If you add the above three dashes before the version info that will not 
by accident be considered part of the commit log.
But of course our internal versioning is irrelevant to David M, so you 
should just remove this info before posting.
[Hoang] Thanks Jon. Frankly, I didn't know that.

> v2: resolve synchronization problem when switching from unicast to
> broadcast
>
> v5: - never use broadcast if there is a single node not understanding it
>      - always use broadcast otherwise
>      - add sequence numbering to non-bulk messages
>
> v6: update Jon's comment
>
> Signed-off-by: Hoang Huu Le <hoa...@de...>
> Acked-by: Jon Maloy <jm...@re...>
> ---
>   net/tipc/bcast.c      |   6 +--
>   net/tipc/bcast.h      |   4 +-
>   net/tipc/link.c       |   2 +-
>   net/tipc/msg.h        |  40 +++++++++++++++
>   net/tipc/name_distr.c | 115 +++++++++++++++++++++++++++++++-----------
>   net/tipc/name_distr.h |   9 ++--
>   net/tipc/name_table.c |   9 +++-
>   net/tipc/name_table.h |   2 +
>   net/tipc/node.c       |  28 +++++++---
>   net/tipc/node.h       |   8 +--
>   10 files changed, 175 insertions(+), 48 deletions(-)
>
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index 4c20be08b9c4..9d085ad6f0cf 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -249,8 +249,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
>    * Consumes the buffer chain.
>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>    */
> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> -			   u16 *cong_link_cnt)
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt)
>   {
>   	struct tipc_link *l = tipc_bc_sndlink(net);
>   	struct sk_buff_head xmitq;
> @@ -746,7 +746,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>   	nl->local = false;
>   }
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net)
> +u32 tipc_bcast_get_mode(struct net *net)
>   {
>   	struct tipc_bc_base *bb = tipc_bc_base(net);
>   
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index 9e847d9617d3..b3b883e2a823 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -89,6 +89,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>   		    struct tipc_mc_method *method, struct tipc_nlist *dests,
>   		    u16 *cong_link_cnt);
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt);
>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>   			struct tipc_msg *hdr);
> @@ -98,7 +100,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>   int tipc_bclink_reset_stats(struct net *net);
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net);
> +u32 tipc_bcast_get_mode(struct net *net);
>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>   
>   void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> diff --git a/net/tipc/link.c b/net/tipc/link.c
> index d4675e922a8f..da0b30733549 100644
> --- a/net/tipc/link.c
> +++ b/net/tipc/link.c
> @@ -2646,7 +2646,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg)
>   	struct nlattr *attrs;
>   	struct nlattr *prop;
>   	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
> +	u32 bc_mode = tipc_bcast_get_mode(net);
>   	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>   	struct tipc_link *bcl = tn->bcl;
>   
> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
> index 871feadbbc19..d53914316684 100644
> --- a/net/tipc/msg.h
> +++ b/net/tipc/msg.h
> @@ -409,6 +409,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
>   	msg_set_bits(m, 1, 25, 0xf, err);
>   }
>   
> +static inline void msg_set_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 28, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 28, 0x1);
> +}
> +
> +static inline void msg_set_last_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 27, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 27, 0x1);
> +}
> +
> +static inline void msg_set_non_legacy(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 26, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_legacy(struct tipc_msg *m)
> +{
> +	return !msg_bits(m, 1, 26, 0x1);
> +}
> +
>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>   {
>   	return msg_bits(m, 1, 21, 0xf);
> @@ -538,6 +568,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
>   	msg_set_word(m, 4, p);
>   }
>   
> +static inline u16 msg_named_seqno(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 4, 0, 0xffff);
> +}
> +
> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
> +{
> +	msg_set_bits(m, 4, 0, 0xffff, n);
> +}
> +
>   static inline u32 msg_destport(struct tipc_msg *m)
>   {
>   	return msg_word(m, 5);
> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
> index 5feaf3b67380..6cbd4b271768 100644
> --- a/net/tipc/name_distr.c
> +++ b/net/tipc/name_distr.c
> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   		pr_warn("Publication distribution failure\n");
>   		return NULL;
>   	}
> -
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
>   	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
>   	return skb;
> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> -	struct sk_buff *buf;
>   	struct distr_item *item;
> +	struct sk_buff *skb;
>   
>   	write_lock_bh(&nt->cluster_scope_lock);
>   	list_del(&publ->binding_node);
> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   	if (publ->scope == TIPC_NODE_SCOPE)
>   		return NULL;
>   
> -	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> -	if (!buf) {
> +	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> +	if (!skb) {
>   		pr_warn("Withdrawal distribution failure\n");
>   		return NULL;
>   	}
> -
> -	item = (struct distr_item *)msg_data(buf_msg(buf));
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
> +	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
> -	return buf;
> +	return skb;
>   }
>   
>   /**
> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>    * @pls: linked list of publication items to be packed into buffer chain
>    */
>   static void named_distribute(struct net *net, struct sk_buff_head *list,
> -			     u32 dnode, struct list_head *pls)
> +			     u32 dnode, struct list_head *pls, u16 seqno)
>   {
>   	struct publication *publ;
>   	struct sk_buff *skb = NULL;
> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
>   			ITEM_SIZE) * ITEM_SIZE;
>   	u32 msg_rem = msg_dsz;
> +	struct tipc_msg *hdr;
>   
>   	list_for_each_entry(publ, pls, binding_node) {
>   		/* Prepare next buffer: */
> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   				pr_warn("Bulk publication failure\n");
>   				return;
>   			}
> -			msg_set_bc_ack_invalid(buf_msg(skb), true);
> -			item = (struct distr_item *)msg_data(buf_msg(skb));
> +			hdr = buf_msg(skb);
> +			msg_set_bc_ack_invalid(hdr, true);
> +			msg_set_bulk(hdr);
> +			msg_set_non_legacy(hdr);
> +			item = (struct distr_item *)msg_data(hdr);
>   		}
>   
>   		/* Pack publication into message: */
> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   		}
>   	}
>   	if (skb) {
> -		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
> +		hdr = buf_msg(skb);
> +		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>   		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>   		__skb_queue_tail(list, skb);
>   	}
> +	hdr = buf_msg(skb_peek_tail(list));
> +	msg_set_last_bulk(hdr);
> +	msg_set_named_seqno(hdr, seqno);
>   }
>   
>   /**
>    * tipc_named_node_up - tell specified node about all publications by this node
>    */
> -void tipc_named_node_up(struct net *net, u32 dnode)
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
>   	struct sk_buff_head head;
> +	u16 seqno;
>   
>   	__skb_queue_head_init(&head);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests++;
> +	seqno = nt->snd_nxt;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	read_lock_bh(&nt->cluster_scope_lock);
> -	named_distribute(net, &head, dnode, &nt->cluster_scope);
> +	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>   	tipc_node_xmit(net, &head, dnode, 0);
>   	read_unlock_bh(&nt->cluster_scope_lock);
>   }
> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities)
>   {
> +	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
> +
>   	struct publication *publ, *tmp;
>   
>   	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>   		tipc_publ_purge(net, publ, addr);
>   	tipc_dist_queue_purge(net, addr);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests--;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
>   /**
> @@ -295,29 +320,61 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
>   	return false;
>   }
>   
> +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
> +				   u16 *rcv_nxt, bool *open)
> +{
> +	struct sk_buff *skb, *tmp;
> +	struct tipc_msg *hdr;
> +	u16 seqno;
> +
> +	skb_queue_walk_safe(namedq, skb, tmp) {
> +		skb_linearize(skb);
> +		hdr = buf_msg(skb);
> +		seqno = msg_named_seqno(hdr);
> +		if (msg_is_last_bulk(hdr)) {
> +			*rcv_nxt = seqno;
> +			*open = true;
> +		}
> +		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (*open && (*rcv_nxt == seqno)) {
> +			(*rcv_nxt)++;
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (less(seqno, *rcv_nxt)) {
> +			__skb_unlink(skb, namedq);
> +			kfree_skb(skb);
> +			continue;
Still not needed. This queue should be flushed in 
tipc_node_lost_contact(), which I now see we don't do.
[Hoang] Yes, that's right. I will verify and send it out.

This has to e fixed too.
///jon
> +		}
> +	}
> +	return NULL;
> +}
> +
>   /**
>    * tipc_named_rcv - process name table update messages sent by another node
>    */
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open)
>   {
> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	struct tipc_msg *msg;
> +	struct tipc_net *tn = tipc_net(net);
>   	struct distr_item *item;
> -	uint count;
> -	u32 node;
> +	struct tipc_msg *hdr;
>   	struct sk_buff *skb;
> -	int mtype;
> +	u32 count, node = 0;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
> -		skb_linearize(skb);
> -		msg = buf_msg(skb);
> -		mtype = msg_type(msg);
> -		item = (struct distr_item *)msg_data(msg);
> -		count = msg_data_sz(msg) / ITEM_SIZE;
> -		node = msg_orignode(msg);
> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
> +		hdr = buf_msg(skb);
> +		node = msg_orignode(hdr);
> +		item = (struct distr_item *)msg_data(hdr);
> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>   		while (count--) {
> -			tipc_update_nametbl(net, item, node, mtype);
> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>   			item++;
>   		}
>   		kfree_skb(skb);
> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>   		publ->node = self;
>   	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>   		publ->node = self;
> -
> +	nt->rc_dests = 0;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
> index 63fc73e0fa6c..092323158f06 100644
> --- a/net/tipc/name_distr.h
> +++ b/net/tipc/name_distr.h
> @@ -67,11 +67,14 @@ struct distr_item {
>   	__be32 key;
>   };
>   
> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>   struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
> -void tipc_named_node_up(struct net *net, u32 dnode);
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open);
>   void tipc_named_reinit(struct net *net);
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities);
>   
>   #endif
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 359b2bc888cf..2ac33d32edc2 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   	struct tipc_net *tn = tipc_net(net);
>   	struct publication *p = NULL;
>   	struct sk_buff *skb = NULL;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   		nt->local_publ_count++;
>   		skb = tipc_named_publish(net, p);
>   	}
> +	rc_dests = nt->rc_dests;
>   exit:
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   	return p;
> +
>   }
>   
>   /**
> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   	u32 self = tipc_own_addr(net);
>   	struct sk_buff *skb = NULL;
>   	struct publication *p;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>   		       type, lower, upper, key);
>   	}
> +	rc_dests = nt->rc_dests;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   		return 1;
>   	}
>   	return 0;
> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
> index 728bc7016c38..8064e1986e2c 100644
> --- a/net/tipc/name_table.h
> +++ b/net/tipc/name_table.h
> @@ -106,6 +106,8 @@ struct name_table {
>   	struct list_head cluster_scope;
>   	rwlock_t cluster_scope_lock;
>   	u32 local_publ_count;
> +	u32 rc_dests;
> +	u32 snd_nxt;
>   };
>   
>   int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
> diff --git a/net/tipc/node.c b/net/tipc/node.c
> index 803a3a6d0f50..ad8d7bce1f98 100644
> --- a/net/tipc/node.c
> +++ b/net/tipc/node.c
> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>   	struct sk_buff_head arrvq;
>   	struct sk_buff_head inputq2;
>   	struct sk_buff_head namedq;
> +	u16 named_rcv_nxt;
> +	bool named_open;
>   };
>   
>   /**
> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>   	write_unlock_bh(&n->lock);
>   
>   	if (flags & TIPC_NOTIFY_NODE_DOWN)
> -		tipc_publ_notify(net, publ_list, addr);
> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_NODE_UP)
> -		tipc_named_node_up(net, addr);
> +		tipc_named_node_up(net, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_LINK_UP) {
>   		tipc_mon_peer_up(net, addr, bearer_id);
> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>   	return 0;
>   }
>   
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>   {
> +	struct sk_buff_head xmitq;
>   	struct sk_buff *txskb;
>   	struct tipc_node *n;
> +	u16 dummy;
>   	u32 dst;
>   
> +	/* Use broadcast if all nodes support it */
> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
> +		__skb_queue_head_init(&xmitq);
> +		__skb_queue_tail(&xmitq, skb);
> +		tipc_bcast_xmit(net, &xmitq, &dummy);
> +		return;
> +	}
> +
> +	/* Otherwise use legacy replicast method */
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>   		dst = n->addr;
> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>   		tipc_node_xmit_skb(net, txskb, dst, 0);
>   	}
>   	rcu_read_unlock();
> -
>   	kfree_skb(skb);
>   }
>   
> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>   
>   	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>   	if (!skb_queue_empty(&n->bc_entry.namedq))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	/* If reassembly or retransmission failure => reset all links to peer */
>   	if (rc & TIPC_LINK_DOWN_EVT)
> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>   		tipc_node_link_down(n, bearer_id, false);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>   		tipc_node_mcast_rcv(n);
> diff --git a/net/tipc/node.h b/net/tipc/node.h
> index a6803b449a2c..9f6f13f1604f 100644
> --- a/net/tipc/node.h
> +++ b/net/tipc/node.h
> @@ -55,7 +55,8 @@ enum {
>   	TIPC_MCAST_RBCTL      = (1 << 7),
>   	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>   	TIPC_TUNNEL_ENHANCED  = (1 << 9),
> -	TIPC_NAGLE            = (1 << 10)
> +	TIPC_NAGLE            = (1 << 10),
> +	TIPC_NAMED_BCAST      = (1 << 11)
>   };
>   
>   #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
> @@ -68,7 +69,8 @@ enum {
>   				TIPC_MCAST_RBCTL       |   \
>   				TIPC_GAP_ACK_BLOCK     |   \
>   				TIPC_TUNNEL_ENHANCED   |   \
> -				TIPC_NAGLE)
> +				TIPC_NAGLE             |   \
> +				TIPC_NAMED_BCAST)
>   
>   #define INVALID_BEARER_ID -1
>   
> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>   		       u32 selector);
>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>   void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/6/20 11:10 PM, Hoang Huu Le wrote:
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Friday, June 5, 2020 8:03 PM
> To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
> Subject: Re: [next-net v6] tipc: update a binding service via broadcast
>
>
>
> On 6/5/20 3:52 AM, Hoang Huu Le wrote:
>> Currently, updating binding table (add service binding to
>> name table/withdraw a service binding) is being sent over replicast.
>> However, if we are scaling up clusters to > 100 nodes/containers this
>> method is less affection because of looping through nodes in a cluster one
>> by one.
>>
[...]
> Still not needed. This queue should be flushed in
> tipc_node_lost_contact(), which I now see we don't do.
> [Hoang] Yes, that's right. I will verify and send it out.
Actually, this might explain the mysterious "Failed to withdraw" 
printouts you observed during testing earlier.
Those withdraw items might be from a previous session just lingering in 
the queue.
On the other hand, such a bug is so obvious and would have such grave 
consequences (what if there are old 'publish' items in the queue?) that 
I find it hard to believe that it can have remain unnoticed all this time.
Are you sure we are not cleaning up this queue somewhere else?

If it really is so we must also issue a correction patch to 'net' for 
this issue.

///jon


>
> This has to e fixed too.
> ///jon
>> +		}
>> +	}
>> +	return NULL;
>> +}
>> +
>>    /**
>>     * tipc_named_rcv - process name table update messages sent by another node
>>     */
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open)
>>    {
>> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
>> -	struct tipc_msg *msg;
>> +	struct tipc_net *tn = tipc_net(net);
>>    	struct distr_item *item;
>> -	uint count;
>> -	u32 node;
>> +	struct tipc_msg *hdr;
>>    	struct sk_buff *skb;
>> -	int mtype;
>> +	u32 count, node = 0;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>> -		skb_linearize(skb);
>> -		msg = buf_msg(skb);
>> -		mtype = msg_type(msg);
>> -		item = (struct distr_item *)msg_data(msg);
>> -		count = msg_data_sz(msg) / ITEM_SIZE;
>> -		node = msg_orignode(msg);
>> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>> +		hdr = buf_msg(skb);
>> +		node = msg_orignode(hdr);
>> +		item = (struct distr_item *)msg_data(hdr);
>> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>>    		while (count--) {
>> -			tipc_update_nametbl(net, item, node, mtype);
>> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>>    			item++;
>>    		}
>>    		kfree_skb(skb);
>> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>>    		publ->node = self;
>>    	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>    		publ->node = self;
>> -
>> +	nt->rc_dests = 0;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    }
>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>> index 63fc73e0fa6c..092323158f06 100644
>> --- a/net/tipc/name_distr.h
>> +++ b/net/tipc/name_distr.h
>> @@ -67,11 +67,14 @@ struct distr_item {
>>    	__be32 key;
>>    };
>>    
>> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>    struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
>> -void tipc_named_node_up(struct net *net, u32 dnode);
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open);
>>    void tipc_named_reinit(struct net *net);
>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +		      u32 addr, u16 capabilities);
>>    
>>    #endif
>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>> index 359b2bc888cf..2ac33d32edc2 100644
>> --- a/net/tipc/name_table.c
>> +++ b/net/tipc/name_table.c
>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    	struct tipc_net *tn = tipc_net(net);
>>    	struct publication *p = NULL;
>>    	struct sk_buff *skb = NULL;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    		nt->local_publ_count++;
>>    		skb = tipc_named_publish(net, p);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    exit:
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb)
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    	return p;
>> +
>>    }
>>    
>>    /**
>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    	u32 self = tipc_own_addr(net);
>>    	struct sk_buff *skb = NULL;
>>    	struct publication *p;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>    		       type, lower, upper, key);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb) {
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    		return 1;
>>    	}
>>    	return 0;
>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>> index 728bc7016c38..8064e1986e2c 100644
>> --- a/net/tipc/name_table.h
>> +++ b/net/tipc/name_table.h
>> @@ -106,6 +106,8 @@ struct name_table {
>>    	struct list_head cluster_scope;
>>    	rwlock_t cluster_scope_lock;
>>    	u32 local_publ_count;
>> +	u32 rc_dests;
>> +	u32 snd_nxt;
>>    };
>>    
>>    int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>> index 803a3a6d0f50..ad8d7bce1f98 100644
>> --- a/net/tipc/node.c
>> +++ b/net/tipc/node.c
>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>    	struct sk_buff_head arrvq;
>>    	struct sk_buff_head inputq2;
>>    	struct sk_buff_head namedq;
>> +	u16 named_rcv_nxt;
>> +	bool named_open;
>>    };
>>    
>>    /**
>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>>    	write_unlock_bh(&n->lock);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_DOWN)
>> -		tipc_publ_notify(net, publ_list, addr);
>> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_UP)
>> -		tipc_named_node_up(net, addr);
>> +		tipc_named_node_up(net, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_LINK_UP) {
>>    		tipc_mon_peer_up(net, addr, bearer_id);
>> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>>    	return 0;
>>    }
>>    
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>>    {
>> +	struct sk_buff_head xmitq;
>>    	struct sk_buff *txskb;
>>    	struct tipc_node *n;
>> +	u16 dummy;
>>    	u32 dst;
>>    
>> +	/* Use broadcast if all nodes support it */
>> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>> +		__skb_queue_head_init(&xmitq);
>> +		__skb_queue_tail(&xmitq, skb);
>> +		tipc_bcast_xmit(net, &xmitq, &dummy);
>> +		return;
>> +	}
>> +
>> +	/* Otherwise use legacy replicast method */
>>    	rcu_read_lock();
>>    	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>    		dst = n->addr;
>> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>    		tipc_node_xmit_skb(net, txskb, dst, 0);
>>    	}
>>    	rcu_read_unlock();
>> -
>>    	kfree_skb(skb);
>>    }
>>    
>> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>>    
>>    	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>    	if (!skb_queue_empty(&n->bc_entry.namedq))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	/* If reassembly or retransmission failure => reset all links to peer */
>>    	if (rc & TIPC_LINK_DOWN_EVT)
>> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>>    		tipc_node_link_down(n, bearer_id, false);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>    		tipc_node_mcast_rcv(n);
>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>> index a6803b449a2c..9f6f13f1604f 100644
>> --- a/net/tipc/node.h
>> +++ b/net/tipc/node.h
>> @@ -55,7 +55,8 @@ enum {
>>    	TIPC_MCAST_RBCTL      = (1 << 7),
>>    	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>    	TIPC_TUNNEL_ENHANCED  = (1 << 9),
>> -	TIPC_NAGLE            = (1 << 10)
>> +	TIPC_NAGLE            = (1 << 10),
>> +	TIPC_NAMED_BCAST      = (1 << 11)
>>    };
>>    
>>    #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>> @@ -68,7 +69,8 @@ enum {
>>    				TIPC_MCAST_RBCTL       |   \
>>    				TIPC_GAP_ACK_BLOCK     |   \
>>    				TIPC_TUNNEL_ENHANCED   |   \
>> -				TIPC_NAGLE)
>> +				TIPC_NAGLE             |   \
>> +				TIPC_NAMED_BCAST)
>>    
>>    #define INVALID_BEARER_ID -1
>>    
>> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>>    		       u32 selector);
>>    void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>>    void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>>    int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>>    void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>    int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

-----Original Message-----
From: Jon Maloy <jm...@re...> 
Sent: Monday, June 8, 2020 2:12 AM
To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
Subject: Re: [next-net v6] tipc: update a binding service via broadcast



On 6/6/20 11:10 PM, Hoang Huu Le wrote:
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Friday, June 5, 2020 8:03 PM
> To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
> Subject: Re: [next-net v6] tipc: update a binding service via broadcast
>
>
>
> On 6/5/20 3:52 AM, Hoang Huu Le wrote:
>> Currently, updating binding table (add service binding to
>> name table/withdraw a service binding) is being sent over replicast.
>> However, if we are scaling up clusters to > 100 nodes/containers this
>> method is less affection because of looping through nodes in a cluster one
>> by one.
>>
[...]
> Still not needed. This queue should be flushed in
> tipc_node_lost_contact(), which I now see we don't do.
> [Hoang] Yes, that's right. I will verify and send it out.
Actually, this might explain the mysterious "Failed to withdraw" 
printouts you observed during testing earlier.
Those withdraw items might be from a previous session just lingering in 
the queue.
On the other hand, such a bug is so obvious and would have such grave 
consequences (what if there are old 'publish' items in the queue?) that 
I find it hard to believe that it can have remain unnoticed all this time.
Are you sure we are not cleaning up this queue somewhere else?

If it really is so we must also issue a correction patch to 'net' for 
this issue.
[Hoang] Yes, I checked and already do the clean stuff at v2-3 in this feature ... So, we should apply this for 'net' too.

///jon


>
> This has to e fixed too.
> ///jon
>> +		}
>> +	}
>> +	return NULL;
>> +}
>> +
>>    /**
>>     * tipc_named_rcv - process name table update messages sent by another node
>>     */
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open)
>>    {
>> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
>> -	struct tipc_msg *msg;
>> +	struct tipc_net *tn = tipc_net(net);
>>    	struct distr_item *item;
>> -	uint count;
>> -	u32 node;
>> +	struct tipc_msg *hdr;
>>    	struct sk_buff *skb;
>> -	int mtype;
>> +	u32 count, node = 0;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>> -		skb_linearize(skb);
>> -		msg = buf_msg(skb);
>> -		mtype = msg_type(msg);
>> -		item = (struct distr_item *)msg_data(msg);
>> -		count = msg_data_sz(msg) / ITEM_SIZE;
>> -		node = msg_orignode(msg);
>> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>> +		hdr = buf_msg(skb);
>> +		node = msg_orignode(hdr);
>> +		item = (struct distr_item *)msg_data(hdr);
>> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>>    		while (count--) {
>> -			tipc_update_nametbl(net, item, node, mtype);
>> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>>    			item++;
>>    		}
>>    		kfree_skb(skb);
>> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>>    		publ->node = self;
>>    	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>    		publ->node = self;
>> -
>> +	nt->rc_dests = 0;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    }
>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>> index 63fc73e0fa6c..092323158f06 100644
>> --- a/net/tipc/name_distr.h
>> +++ b/net/tipc/name_distr.h
>> @@ -67,11 +67,14 @@ struct distr_item {
>>    	__be32 key;
>>    };
>>    
>> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>    struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
>> -void tipc_named_node_up(struct net *net, u32 dnode);
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open);
>>    void tipc_named_reinit(struct net *net);
>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +		      u32 addr, u16 capabilities);
>>    
>>    #endif
>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>> index 359b2bc888cf..2ac33d32edc2 100644
>> --- a/net/tipc/name_table.c
>> +++ b/net/tipc/name_table.c
>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    	struct tipc_net *tn = tipc_net(net);
>>    	struct publication *p = NULL;
>>    	struct sk_buff *skb = NULL;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    		nt->local_publ_count++;
>>    		skb = tipc_named_publish(net, p);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    exit:
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb)
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    	return p;
>> +
>>    }
>>    
>>    /**
>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    	u32 self = tipc_own_addr(net);
>>    	struct sk_buff *skb = NULL;
>>    	struct publication *p;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>    		       type, lower, upper, key);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb) {
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    		return 1;
>>    	}
>>    	return 0;
>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>> index 728bc7016c38..8064e1986e2c 100644
>> --- a/net/tipc/name_table.h
>> +++ b/net/tipc/name_table.h
>> @@ -106,6 +106,8 @@ struct name_table {
>>    	struct list_head cluster_scope;
>>    	rwlock_t cluster_scope_lock;
>>    	u32 local_publ_count;
>> +	u32 rc_dests;
>> +	u32 snd_nxt;
>>    };
>>    
>>    int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>> index 803a3a6d0f50..ad8d7bce1f98 100644
>> --- a/net/tipc/node.c
>> +++ b/net/tipc/node.c
>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>    	struct sk_buff_head arrvq;
>>    	struct sk_buff_head inputq2;
>>    	struct sk_buff_head namedq;
>> +	u16 named_rcv_nxt;
>> +	bool named_open;
>>    };
>>    
>>    /**
>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>>    	write_unlock_bh(&n->lock);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_DOWN)
>> -		tipc_publ_notify(net, publ_list, addr);
>> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_UP)
>> -		tipc_named_node_up(net, addr);
>> +		tipc_named_node_up(net, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_LINK_UP) {
>>    		tipc_mon_peer_up(net, addr, bearer_id);
>> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>>    	return 0;
>>    }
>>    
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>>    {
>> +	struct sk_buff_head xmitq;
>>    	struct sk_buff *txskb;
>>    	struct tipc_node *n;
>> +	u16 dummy;
>>    	u32 dst;
>>    
>> +	/* Use broadcast if all nodes support it */
>> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>> +		__skb_queue_head_init(&xmitq);
>> +		__skb_queue_tail(&xmitq, skb);
>> +		tipc_bcast_xmit(net, &xmitq, &dummy);
>> +		return;
>> +	}
>> +
>> +	/* Otherwise use legacy replicast method */
>>    	rcu_read_lock();
>>    	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>    		dst = n->addr;
>> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>    		tipc_node_xmit_skb(net, txskb, dst, 0);
>>    	}
>>    	rcu_read_unlock();
>> -
>>    	kfree_skb(skb);
>>    }
>>    
>> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>>    
>>    	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>    	if (!skb_queue_empty(&n->bc_entry.namedq))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	/* If reassembly or retransmission failure => reset all links to peer */
>>    	if (rc & TIPC_LINK_DOWN_EVT)
>> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>>    		tipc_node_link_down(n, bearer_id, false);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>    		tipc_node_mcast_rcv(n);
>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>> index a6803b449a2c..9f6f13f1604f 100644
>> --- a/net/tipc/node.h
>> +++ b/net/tipc/node.h
>> @@ -55,7 +55,8 @@ enum {
>>    	TIPC_MCAST_RBCTL      = (1 << 7),
>>    	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>    	TIPC_TUNNEL_ENHANCED  = (1 << 9),
>> -	TIPC_NAGLE            = (1 << 10)
>> +	TIPC_NAGLE            = (1 << 10),
>> +	TIPC_NAMED_BCAST      = (1 << 11)
>>    };
>>    
>>    #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>> @@ -68,7 +69,8 @@ enum {
>>    				TIPC_MCAST_RBCTL       |   \
>>    				TIPC_GAP_ACK_BLOCK     |   \
>>    				TIPC_TUNNEL_ENHANCED   |   \
>> -				TIPC_NAGLE)
>> +				TIPC_NAGLE             |   \
>> +				TIPC_NAMED_BCAST)
>>    
>>    #define INVALID_BEARER_ID -1
>>    
>> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>>    		       u32 selector);
>>    void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>>    void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>>    int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>>    void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>    int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/6/20 11:10 PM, Hoang Huu Le wrote:
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Friday, June 5, 2020 8:03 PM
> To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
> Subject: Re: [next-net v6] tipc: update a binding service via broadcast
>
>
>
> On 6/5/20 3:52 AM, Hoang Huu Le wrote:
>> Currently, updating binding table (add service binding to
>> name table/withdraw a service binding) is being sent over replicast.
>> However, if we are scaling up clusters to > 100 nodes/containers this
>> method is less affection because of looping through nodes in a cluster one
>> by one.
[...]
>> +		if (*open && (*rcv_nxt == seqno)) {
>> +			(*rcv_nxt)++;
>> +			__skb_unlink(skb, namedq);
>> +			return skb;
>> +		}
>> +
>> +		if (less(seqno, *rcv_nxt)) {
>> +			__skb_unlink(skb, namedq);
>> +			kfree_skb(skb);
>> +			continue;
> Still not needed. This queue should be flushed in
> tipc_node_lost_contact(), which I now see we don't do.
> [Hoang] Yes, that's right. I will verify and send it out.
>
> This has to e fixed too.
> ///jon
I hate to admit it, but we might actually need this test after all. 
Imagine that somebody has done 'publish' just after the broadcast link 
came up (in tipc_bcast_add_peer()) , but before tipc_named_node_up() is 
called. The context of those two calls is not atomic, so I think it is 
possible that this publication might end up both in the bcast_link 
backlog queue and in the bulk distribution.
This publication message will have a named_seqno that is lower than the 
agreed synch point, and should be dropped at reception.

Given the crucial role of the binding table for the overall TIPC 
functionality I think it is better be safe than sorry here, and keep 
this test.

///jon

>> +		}
>> +	}
>> +	return NULL;
>> +}
>> +
>>    /**
>>     * tipc_named_rcv - process name table update messages sent by another node
>>     */
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open)
>>    {
>> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
>> -	struct tipc_msg *msg;
>> +	struct tipc_net *tn = tipc_net(net);
>>    	struct distr_item *item;
>> -	uint count;
>> -	u32 node;
>> +	struct tipc_msg *hdr;
>>    	struct sk_buff *skb;
>> -	int mtype;
>> +	u32 count, node = 0;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>> -		skb_linearize(skb);
>> -		msg = buf_msg(skb);
>> -		mtype = msg_type(msg);
>> -		item = (struct distr_item *)msg_data(msg);
>> -		count = msg_data_sz(msg) / ITEM_SIZE;
>> -		node = msg_orignode(msg);
>> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>> +		hdr = buf_msg(skb);
>> +		node = msg_orignode(hdr);
>> +		item = (struct distr_item *)msg_data(hdr);
>> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>>    		while (count--) {
>> -			tipc_update_nametbl(net, item, node, mtype);
>> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>>    			item++;
>>    		}
>>    		kfree_skb(skb);
>> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>>    		publ->node = self;
>>    	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>    		publ->node = self;
>> -
>> +	nt->rc_dests = 0;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    }
>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>> index 63fc73e0fa6c..092323158f06 100644
>> --- a/net/tipc/name_distr.h
>> +++ b/net/tipc/name_distr.h
>> @@ -67,11 +67,14 @@ struct distr_item {
>>    	__be32 key;
>>    };
>>    
>> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>    struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
>> -void tipc_named_node_up(struct net *net, u32 dnode);
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open);
>>    void tipc_named_reinit(struct net *net);
>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +		      u32 addr, u16 capabilities);
>>    
>>    #endif
>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>> index 359b2bc888cf..2ac33d32edc2 100644
>> --- a/net/tipc/name_table.c
>> +++ b/net/tipc/name_table.c
>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    	struct tipc_net *tn = tipc_net(net);
>>    	struct publication *p = NULL;
>>    	struct sk_buff *skb = NULL;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    		nt->local_publ_count++;
>>    		skb = tipc_named_publish(net, p);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    exit:
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb)
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    	return p;
>> +
>>    }
>>    
>>    /**
>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    	u32 self = tipc_own_addr(net);
>>    	struct sk_buff *skb = NULL;
>>    	struct publication *p;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>    		       type, lower, upper, key);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb) {
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    		return 1;
>>    	}
>>    	return 0;
>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>> index 728bc7016c38..8064e1986e2c 100644
>> --- a/net/tipc/name_table.h
>> +++ b/net/tipc/name_table.h
>> @@ -106,6 +106,8 @@ struct name_table {
>>    	struct list_head cluster_scope;
>>    	rwlock_t cluster_scope_lock;
>>    	u32 local_publ_count;
>> +	u32 rc_dests;
>> +	u32 snd_nxt;
>>    };
>>    
>>    int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>> index 803a3a6d0f50..ad8d7bce1f98 100644
>> --- a/net/tipc/node.c
>> +++ b/net/tipc/node.c
>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>    	struct sk_buff_head arrvq;
>>    	struct sk_buff_head inputq2;
>>    	struct sk_buff_head namedq;
>> +	u16 named_rcv_nxt;
>> +	bool named_open;
>>    };
>>    
>>    /**
>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>>    	write_unlock_bh(&n->lock);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_DOWN)
>> -		tipc_publ_notify(net, publ_list, addr);
>> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_UP)
>> -		tipc_named_node_up(net, addr);
>> +		tipc_named_node_up(net, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_LINK_UP) {
>>    		tipc_mon_peer_up(net, addr, bearer_id);
>> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>>    	return 0;
>>    }
>>    
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>>    {
>> +	struct sk_buff_head xmitq;
>>    	struct sk_buff *txskb;
>>    	struct tipc_node *n;
>> +	u16 dummy;
>>    	u32 dst;
>>    
>> +	/* Use broadcast if all nodes support it */
>> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>> +		__skb_queue_head_init(&xmitq);
>> +		__skb_queue_tail(&xmitq, skb);
>> +		tipc_bcast_xmit(net, &xmitq, &dummy);
>> +		return;
>> +	}
>> +
>> +	/* Otherwise use legacy replicast method */
>>    	rcu_read_lock();
>>    	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>    		dst = n->addr;
>> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>    		tipc_node_xmit_skb(net, txskb, dst, 0);
>>    	}
>>    	rcu_read_unlock();
>> -
>>    	kfree_skb(skb);
>>    }
>>    
>> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>>    
>>    	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>    	if (!skb_queue_empty(&n->bc_entry.namedq))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	/* If reassembly or retransmission failure => reset all links to peer */
>>    	if (rc & TIPC_LINK_DOWN_EVT)
>> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>>    		tipc_node_link_down(n, bearer_id, false);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>    		tipc_node_mcast_rcv(n);
>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>> index a6803b449a2c..9f6f13f1604f 100644
>> --- a/net/tipc/node.h
>> +++ b/net/tipc/node.h
>> @@ -55,7 +55,8 @@ enum {
>>    	TIPC_MCAST_RBCTL      = (1 << 7),
>>    	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>    	TIPC_TUNNEL_ENHANCED  = (1 << 9),
>> -	TIPC_NAGLE            = (1 << 10)
>> +	TIPC_NAGLE            = (1 << 10),
>> +	TIPC_NAMED_BCAST      = (1 << 11)
>>    };
>>    
>>    #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>> @@ -68,7 +69,8 @@ enum {
>>    				TIPC_MCAST_RBCTL       |   \
>>    				TIPC_GAP_ACK_BLOCK     |   \
>>    				TIPC_TUNNEL_ENHANCED   |   \
>> -				TIPC_NAGLE)
>> +				TIPC_NAGLE             |   \
>> +				TIPC_NAMED_BCAST)
>>    
>>    #define INVALID_BEARER_ID -1
>>    
>> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>>    		       u32 selector);
>>    void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>>    void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>>    int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>>    void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>    int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

-----Original Message-----
From: Jon Maloy <jm...@re...> 
Sent: Monday, June 8, 2020 8:33 PM
To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
Subject: Re: [next-net v6] tipc: update a binding service via broadcast



On 6/6/20 11:10 PM, Hoang Huu Le wrote:
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Friday, June 5, 2020 8:03 PM
> To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
> Subject: Re: [next-net v6] tipc: update a binding service via broadcast
>
>
>
> On 6/5/20 3:52 AM, Hoang Huu Le wrote:
>> Currently, updating binding table (add service binding to
>> name table/withdraw a service binding) is being sent over replicast.
>> However, if we are scaling up clusters to > 100 nodes/containers this
>> method is less affection because of looping through nodes in a cluster one
>> by one.
[...]
>> +		if (*open && (*rcv_nxt == seqno)) {
>> +			(*rcv_nxt)++;
>> +			__skb_unlink(skb, namedq);
>> +			return skb;
>> +		}
>> +
>> +		if (less(seqno, *rcv_nxt)) {
>> +			__skb_unlink(skb, namedq);
>> +			kfree_skb(skb);
>> +			continue;
> Still not needed. This queue should be flushed in
> tipc_node_lost_contact(), which I now see we don't do.
> [Hoang] Yes, that's right. I will verify and send it out.
>
> This has to e fixed too.
> ///jon
I hate to admit it, but we might actually need this test after all. 
Imagine that somebody has done 'publish' just after the broadcast link 
came up (in tipc_bcast_add_peer()) , but before tipc_named_node_up() is 
called. The context of those two calls is not atomic, so I think it is 
possible that this publication might end up both in the bcast_link 
backlog queue and in the bulk distribution.
This publication message will have a named_seqno that is lower than the 
agreed synch point, and should be dropped at reception.

Given the crucial role of the binding table for the overall TIPC 
functionality I think it is better be safe than sorry here, and keep 
this test.
[Hoang] Finally, I'm able to reproduce the problem as same as above scene:
<code>
357                 if (less(seqno, *rcv_nxt)) {
358                         pr_info("DROP[%x->%x]: %s blk %d lblk %d nxt %d legacy %d seqno %u bc %u hsz %u dsz %u qlen %u\n",
359                                 msg_orignode(hdr), tipc_own_addr(net),
360                                 msg_type(hdr) == PUBLICATION ? "PUBL":"DRAW",
361                                 msg_is_bulk(hdr), msg_is_last_bulk(hdr),
362                                 *rcv_nxt, msg_is_legacy(hdr),
363                                 msg_named_seqno(hdr), msg_non_seq(hdr),
364                                 msg_hdr_sz(hdr), msg_data_sz(hdr),
365                                 skb_queue_len(namedq));
366
367                         __skb_unlink(skb, namedq);
368                         kfree_skb(skb);
369                         continue;
370                 }
</code>
-----------
[12528.036895] tipc: Established link <1001024:eth0-1001001:brtipc> on network plane A
[12528.043857] tipc: Established link <1001002:brtipc-1001001:brtipc> on network plane A
[12528.136462] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3878 bc 0 hsz 40 dsz 20 qlen 23
[12528.140864] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3879 bc 0 hsz 40 dsz 20 qlen 22
[...]
[12528.210959] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3893 bc 0 hsz 40 dsz 20 qlen 8
[12528.218903] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3894 bc 0 hsz 40 dsz 20 qlen 7
[12528.227214] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3878 bc 0 hsz 40 dsz 20 qlen 23
[12528.231285] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3879 bc 0 hsz 40 dsz 20 qlen 22
[...]
[12528.277445] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3893 bc 0 hsz 40 dsz 20 qlen 8
[12528.280847] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3894 bc 0 hsz 40 dsz 20 qlen 7 
---  
I will re-post the patch including the test as well.

///jon

>> +		}
>> +	}
>> +	return NULL;
>> +}
>> +
>>    /**
>>     * tipc_named_rcv - process name table update messages sent by another node
>>     */
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open)
>>    {
>> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
>> -	struct tipc_msg *msg;
>> +	struct tipc_net *tn = tipc_net(net);
>>    	struct distr_item *item;
>> -	uint count;
>> -	u32 node;
>> +	struct tipc_msg *hdr;
>>    	struct sk_buff *skb;
>> -	int mtype;
>> +	u32 count, node = 0;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>> -		skb_linearize(skb);
>> -		msg = buf_msg(skb);
>> -		mtype = msg_type(msg);
>> -		item = (struct distr_item *)msg_data(msg);
>> -		count = msg_data_sz(msg) / ITEM_SIZE;
>> -		node = msg_orignode(msg);
>> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>> +		hdr = buf_msg(skb);
>> +		node = msg_orignode(hdr);
>> +		item = (struct distr_item *)msg_data(hdr);
>> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>>    		while (count--) {
>> -			tipc_update_nametbl(net, item, node, mtype);
>> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>>    			item++;
>>    		}
>>    		kfree_skb(skb);
>> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>>    		publ->node = self;
>>    	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>    		publ->node = self;
>> -
>> +	nt->rc_dests = 0;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    }
>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>> index 63fc73e0fa6c..092323158f06 100644
>> --- a/net/tipc/name_distr.h
>> +++ b/net/tipc/name_distr.h
>> @@ -67,11 +67,14 @@ struct distr_item {
>>    	__be32 key;
>>    };
>>    
>> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>    struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
>> -void tipc_named_node_up(struct net *net, u32 dnode);
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +		    u16 *rcv_nxt, bool *open);
>>    void tipc_named_reinit(struct net *net);
>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +		      u32 addr, u16 capabilities);
>>    
>>    #endif
>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>> index 359b2bc888cf..2ac33d32edc2 100644
>> --- a/net/tipc/name_table.c
>> +++ b/net/tipc/name_table.c
>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    	struct tipc_net *tn = tipc_net(net);
>>    	struct publication *p = NULL;
>>    	struct sk_buff *skb = NULL;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>    		nt->local_publ_count++;
>>    		skb = tipc_named_publish(net, p);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    exit:
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb)
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    	return p;
>> +
>>    }
>>    
>>    /**
>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    	u32 self = tipc_own_addr(net);
>>    	struct sk_buff *skb = NULL;
>>    	struct publication *p;
>> +	u32 rc_dests;
>>    
>>    	spin_lock_bh(&tn->nametbl_lock);
>>    
>> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>    		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>    		       type, lower, upper, key);
>>    	}
>> +	rc_dests = nt->rc_dests;
>>    	spin_unlock_bh(&tn->nametbl_lock);
>>    
>>    	if (skb) {
>> -		tipc_node_broadcast(net, skb);
>> +		tipc_node_broadcast(net, skb, rc_dests);
>>    		return 1;
>>    	}
>>    	return 0;
>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>> index 728bc7016c38..8064e1986e2c 100644
>> --- a/net/tipc/name_table.h
>> +++ b/net/tipc/name_table.h
>> @@ -106,6 +106,8 @@ struct name_table {
>>    	struct list_head cluster_scope;
>>    	rwlock_t cluster_scope_lock;
>>    	u32 local_publ_count;
>> +	u32 rc_dests;
>> +	u32 snd_nxt;
>>    };
>>    
>>    int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>> index 803a3a6d0f50..ad8d7bce1f98 100644
>> --- a/net/tipc/node.c
>> +++ b/net/tipc/node.c
>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>    	struct sk_buff_head arrvq;
>>    	struct sk_buff_head inputq2;
>>    	struct sk_buff_head namedq;
>> +	u16 named_rcv_nxt;
>> +	bool named_open;
>>    };
>>    
>>    /**
>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>>    	write_unlock_bh(&n->lock);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_DOWN)
>> -		tipc_publ_notify(net, publ_list, addr);
>> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_NODE_UP)
>> -		tipc_named_node_up(net, addr);
>> +		tipc_named_node_up(net, addr, n->capabilities);
>>    
>>    	if (flags & TIPC_NOTIFY_LINK_UP) {
>>    		tipc_mon_peer_up(net, addr, bearer_id);
>> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>>    	return 0;
>>    }
>>    
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>>    {
>> +	struct sk_buff_head xmitq;
>>    	struct sk_buff *txskb;
>>    	struct tipc_node *n;
>> +	u16 dummy;
>>    	u32 dst;
>>    
>> +	/* Use broadcast if all nodes support it */
>> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>> +		__skb_queue_head_init(&xmitq);
>> +		__skb_queue_tail(&xmitq, skb);
>> +		tipc_bcast_xmit(net, &xmitq, &dummy);
>> +		return;
>> +	}
>> +
>> +	/* Otherwise use legacy replicast method */
>>    	rcu_read_lock();
>>    	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>    		dst = n->addr;
>> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>    		tipc_node_xmit_skb(net, txskb, dst, 0);
>>    	}
>>    	rcu_read_unlock();
>> -
>>    	kfree_skb(skb);
>>    }
>>    
>> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>>    
>>    	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>    	if (!skb_queue_empty(&n->bc_entry.namedq))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	/* If reassembly or retransmission failure => reset all links to peer */
>>    	if (rc & TIPC_LINK_DOWN_EVT)
>> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>>    		tipc_node_link_down(n, bearer_id, false);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>> +			       &n->bc_entry.named_rcv_nxt,
>> +			       &n->bc_entry.named_open);
>>    
>>    	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>    		tipc_node_mcast_rcv(n);
>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>> index a6803b449a2c..9f6f13f1604f 100644
>> --- a/net/tipc/node.h
>> +++ b/net/tipc/node.h
>> @@ -55,7 +55,8 @@ enum {
>>    	TIPC_MCAST_RBCTL      = (1 << 7),
>>    	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>    	TIPC_TUNNEL_ENHANCED  = (1 << 9),
>> -	TIPC_NAGLE            = (1 << 10)
>> +	TIPC_NAGLE            = (1 << 10),
>> +	TIPC_NAMED_BCAST      = (1 << 11)
>>    };
>>    
>>    #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>> @@ -68,7 +69,8 @@ enum {
>>    				TIPC_MCAST_RBCTL       |   \
>>    				TIPC_GAP_ACK_BLOCK     |   \
>>    				TIPC_TUNNEL_ENHANCED   |   \
>> -				TIPC_NAGLE)
>> +				TIPC_NAGLE             |   \
>> +				TIPC_NAMED_BCAST)
>>    
>>    #define INVALID_BEARER_ID -1
>>    
>> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>>    		       u32 selector);
>>    void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>>    void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>>    int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>>    void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>    int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [next-net v6] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/9/20 5:20 AM, Hoang Huu Le wrote:
>
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Monday, June 8, 2020 8:33 PM
> To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
> Subject: Re: [next-net v6] tipc: update a binding service via broadcast
>
>
>
> On 6/6/20 11:10 PM, Hoang Huu Le wrote:
>> -----Original Message-----
>> From: Jon Maloy <jm...@re...>
>> Sent: Friday, June 5, 2020 8:03 PM
>> To: Hoang Huu Le <hoa...@de...>; ma...@do...; yin...@wi...; tip...@li...
>> Subject: Re: [next-net v6] tipc: update a binding service via broadcast
>>
>>
>>
>> On 6/5/20 3:52 AM, Hoang Huu Le wrote:
>>> Currently, updating binding table (add service binding to
>>> name table/withdraw a service binding) is being sent over replicast.
>>> However, if we are scaling up clusters to > 100 nodes/containers this
>>> method is less affection because of looping through nodes in a cluster one
>>> by one.
> [...]
>>> +		if (*open && (*rcv_nxt == seqno)) {
>>> +			(*rcv_nxt)++;
>>> +			__skb_unlink(skb, namedq);
>>> +			return skb;
>>> +		}
>>> +
>>> +		if (less(seqno, *rcv_nxt)) {
>>> +			__skb_unlink(skb, namedq);
>>> +			kfree_skb(skb);
>>> +			continue;
>> Still not needed. This queue should be flushed in
>> tipc_node_lost_contact(), which I now see we don't do.
>> [Hoang] Yes, that's right. I will verify and send it out.
>>
>> This has to e fixed too.
>> ///jon
> I hate to admit it, but we might actually need this test after all.
> Imagine that somebody has done 'publish' just after the broadcast link
> came up (in tipc_bcast_add_peer()) , but before tipc_named_node_up() is
> called. The context of those two calls is not atomic, so I think it is
> possible that this publication might end up both in the bcast_link
> backlog queue and in the bulk distribution.
> This publication message will have a named_seqno that is lower than the
> agreed synch point, and should be dropped at reception.
>
> Given the crucial role of the binding table for the overall TIPC
> functionality I think it is better be safe than sorry here, and keep
> this test.
> [Hoang] Finally, I'm able to reproduce the problem as same as above scene:
> <code>
> 357                 if (less(seqno, *rcv_nxt)) {
> 358                         pr_info("DROP[%x->%x]: %s blk %d lblk %d nxt %d legacy %d seqno %u bc %u hsz %u dsz %u qlen %u\n",
> 359                                 msg_orignode(hdr), tipc_own_addr(net),
> 360                                 msg_type(hdr) == PUBLICATION ? "PUBL":"DRAW",
> 361                                 msg_is_bulk(hdr), msg_is_last_bulk(hdr),
> 362                                 *rcv_nxt, msg_is_legacy(hdr),
> 363                                 msg_named_seqno(hdr), msg_non_seq(hdr),
> 364                                 msg_hdr_sz(hdr), msg_data_sz(hdr),
> 365                                 skb_queue_len(namedq));
> 366
> 367                         __skb_unlink(skb, namedq);
> 368                         kfree_skb(skb);
> 369                         continue;
> 370                 }
> </code>
> -----------
> [12528.036895] tipc: Established link <1001024:eth0-1001001:brtipc> on network plane A
> [12528.043857] tipc: Established link <1001002:brtipc-1001001:brtipc> on network plane A
> [12528.136462] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3878 bc 0 hsz 40 dsz 20 qlen 23
> [12528.140864] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3879 bc 0 hsz 40 dsz 20 qlen 22
> [...]
> [12528.210959] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3893 bc 0 hsz 40 dsz 20 qlen 8
> [12528.218903] tipc: DROP[1001001->1001002]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3894 bc 0 hsz 40 dsz 20 qlen 7
> [12528.227214] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3878 bc 0 hsz 40 dsz 20 qlen 23
> [12528.231285] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3879 bc 0 hsz 40 dsz 20 qlen 22
> [...]
> [12528.277445] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3893 bc 0 hsz 40 dsz 20 qlen 8
> [12528.280847] tipc: DROP[1001001->1001024]: DRAW blk 0 lblk 0 nxt 3895 legacy 0 seqno 3894 bc 0 hsz 40 dsz 20 qlen 7
> ---
> I will re-post the patch including the test as well.
Great. I remember took us several years to get the broadcast link start 
synchronization right, so we'd better be careful and build on that 
experience. But now I think we are ok.

///jon
>
> ///jon
>
>>> +		}
>>> +	}
>>> +	return NULL;
>>> +}
>>> +
>>>     /**
>>>      * tipc_named_rcv - process name table update messages sent by another node
>>>      */
>>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>>> +		    u16 *rcv_nxt, bool *open)
>>>     {
>>> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
>>> -	struct tipc_msg *msg;
>>> +	struct tipc_net *tn = tipc_net(net);
>>>     	struct distr_item *item;
>>> -	uint count;
>>> -	u32 node;
>>> +	struct tipc_msg *hdr;
>>>     	struct sk_buff *skb;
>>> -	int mtype;
>>> +	u32 count, node = 0;
>>>     
>>>     	spin_lock_bh(&tn->nametbl_lock);
>>> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>>> -		skb_linearize(skb);
>>> -		msg = buf_msg(skb);
>>> -		mtype = msg_type(msg);
>>> -		item = (struct distr_item *)msg_data(msg);
>>> -		count = msg_data_sz(msg) / ITEM_SIZE;
>>> -		node = msg_orignode(msg);
>>> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>>> +		hdr = buf_msg(skb);
>>> +		node = msg_orignode(hdr);
>>> +		item = (struct distr_item *)msg_data(hdr);
>>> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>>>     		while (count--) {
>>> -			tipc_update_nametbl(net, item, node, mtype);
>>> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>>>     			item++;
>>>     		}
>>>     		kfree_skb(skb);
>>> @@ -345,6 +402,6 @@ void tipc_named_reinit(struct net *net)
>>>     		publ->node = self;
>>>     	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>>     		publ->node = self;
>>> -
>>> +	nt->rc_dests = 0;
>>>     	spin_unlock_bh(&tn->nametbl_lock);
>>>     }
>>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>>> index 63fc73e0fa6c..092323158f06 100644
>>> --- a/net/tipc/name_distr.h
>>> +++ b/net/tipc/name_distr.h
>>> @@ -67,11 +67,14 @@ struct distr_item {
>>>     	__be32 key;
>>>     };
>>>     
>>> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>>     struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>>>     struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
>>> -void tipc_named_node_up(struct net *net, u32 dnode);
>>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>>> +		    u16 *rcv_nxt, bool *open);
>>>     void tipc_named_reinit(struct net *net);
>>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
>>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> +		      u32 addr, u16 capabilities);
>>>     
>>>     #endif
>>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>>> index 359b2bc888cf..2ac33d32edc2 100644
>>> --- a/net/tipc/name_table.c
>>> +++ b/net/tipc/name_table.c
>>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>>     	struct tipc_net *tn = tipc_net(net);
>>>     	struct publication *p = NULL;
>>>     	struct sk_buff *skb = NULL;
>>> +	u32 rc_dests;
>>>     
>>>     	spin_lock_bh(&tn->nametbl_lock);
>>>     
>>> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>>     		nt->local_publ_count++;
>>>     		skb = tipc_named_publish(net, p);
>>>     	}
>>> +	rc_dests = nt->rc_dests;
>>>     exit:
>>>     	spin_unlock_bh(&tn->nametbl_lock);
>>>     
>>>     	if (skb)
>>> -		tipc_node_broadcast(net, skb);
>>> +		tipc_node_broadcast(net, skb, rc_dests);
>>>     	return p;
>>> +
>>>     }
>>>     
>>>     /**
>>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>>     	u32 self = tipc_own_addr(net);
>>>     	struct sk_buff *skb = NULL;
>>>     	struct publication *p;
>>> +	u32 rc_dests;
>>>     
>>>     	spin_lock_bh(&tn->nametbl_lock);
>>>     
>>> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>>>     		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>>     		       type, lower, upper, key);
>>>     	}
>>> +	rc_dests = nt->rc_dests;
>>>     	spin_unlock_bh(&tn->nametbl_lock);
>>>     
>>>     	if (skb) {
>>> -		tipc_node_broadcast(net, skb);
>>> +		tipc_node_broadcast(net, skb, rc_dests);
>>>     		return 1;
>>>     	}
>>>     	return 0;
>>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>>> index 728bc7016c38..8064e1986e2c 100644
>>> --- a/net/tipc/name_table.h
>>> +++ b/net/tipc/name_table.h
>>> @@ -106,6 +106,8 @@ struct name_table {
>>>     	struct list_head cluster_scope;
>>>     	rwlock_t cluster_scope_lock;
>>>     	u32 local_publ_count;
>>> +	u32 rc_dests;
>>> +	u32 snd_nxt;
>>>     };
>>>     
>>>     int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
>>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>>> index 803a3a6d0f50..ad8d7bce1f98 100644
>>> --- a/net/tipc/node.c
>>> +++ b/net/tipc/node.c
>>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>>     	struct sk_buff_head arrvq;
>>>     	struct sk_buff_head inputq2;
>>>     	struct sk_buff_head namedq;
>>> +	u16 named_rcv_nxt;
>>> +	bool named_open;
>>>     };
>>>     
>>>     /**
>>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>>>     	write_unlock_bh(&n->lock);
>>>     
>>>     	if (flags & TIPC_NOTIFY_NODE_DOWN)
>>> -		tipc_publ_notify(net, publ_list, addr);
>>> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>>     
>>>     	if (flags & TIPC_NOTIFY_NODE_UP)
>>> -		tipc_named_node_up(net, addr);
>>> +		tipc_named_node_up(net, addr, n->capabilities);
>>>     
>>>     	if (flags & TIPC_NOTIFY_LINK_UP) {
>>>     		tipc_mon_peer_up(net, addr, bearer_id);
>>> @@ -1729,12 +1731,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>>>     	return 0;
>>>     }
>>>     
>>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>>>     {
>>> +	struct sk_buff_head xmitq;
>>>     	struct sk_buff *txskb;
>>>     	struct tipc_node *n;
>>> +	u16 dummy;
>>>     	u32 dst;
>>>     
>>> +	/* Use broadcast if all nodes support it */
>>> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>>> +		__skb_queue_head_init(&xmitq);
>>> +		__skb_queue_tail(&xmitq, skb);
>>> +		tipc_bcast_xmit(net, &xmitq, &dummy);
>>> +		return;
>>> +	}
>>> +
>>> +	/* Otherwise use legacy replicast method */
>>>     	rcu_read_lock();
>>>     	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>>     		dst = n->addr;
>>> @@ -1749,7 +1762,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>>     		tipc_node_xmit_skb(net, txskb, dst, 0);
>>>     	}
>>>     	rcu_read_unlock();
>>> -
>>>     	kfree_skb(skb);
>>>     }
>>>     
>>> @@ -1844,7 +1856,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>>>     
>>>     	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>>     	if (!skb_queue_empty(&n->bc_entry.namedq))
>>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>>> +			       &n->bc_entry.named_rcv_nxt,
>>> +			       &n->bc_entry.named_open);
>>>     
>>>     	/* If reassembly or retransmission failure => reset all links to peer */
>>>     	if (rc & TIPC_LINK_DOWN_EVT)
>>> @@ -2109,7 +2123,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>>>     		tipc_node_link_down(n, bearer_id, false);
>>>     
>>>     	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>>> -		tipc_named_rcv(net, &n->bc_entry.namedq);
>>> +		tipc_named_rcv(net, &n->bc_entry.namedq,
>>> +			       &n->bc_entry.named_rcv_nxt,
>>> +			       &n->bc_entry.named_open);
>>>     
>>>     	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>>     		tipc_node_mcast_rcv(n);
>>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>>> index a6803b449a2c..9f6f13f1604f 100644
>>> --- a/net/tipc/node.h
>>> +++ b/net/tipc/node.h
>>> @@ -55,7 +55,8 @@ enum {
>>>     	TIPC_MCAST_RBCTL      = (1 << 7),
>>>     	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>>     	TIPC_TUNNEL_ENHANCED  = (1 << 9),
>>> -	TIPC_NAGLE            = (1 << 10)
>>> +	TIPC_NAGLE            = (1 << 10),
>>> +	TIPC_NAMED_BCAST      = (1 << 11)
>>>     };
>>>     
>>>     #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>>> @@ -68,7 +69,8 @@ enum {
>>>     				TIPC_MCAST_RBCTL       |   \
>>>     				TIPC_GAP_ACK_BLOCK     |   \
>>>     				TIPC_TUNNEL_ENHANCED   |   \
>>> -				TIPC_NAGLE)
>>> +				TIPC_NAGLE             |   \
>>> +				TIPC_NAMED_BCAST)
>>>     
>>>     #define INVALID_BEARER_ID -1
>>>     
>>> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>>>     		       u32 selector);
>>>     void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>>>     void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
>>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>>>     int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>>>     void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>>     int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

[tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

Currently, updating binding table (add service binding to
name table/withdraw a service binding) is being sent over replicast.
However, if we are scaling up clusters to > 100 nodes/containers this
method is less affection because of looping through nodes in a cluster one
by one.

It is worth to use broadcast to update a binding service. This way, the
binding table can be updated on all peer nodes in one shot.

Broadcast is used when all peer nodes, as indicated by a new capability
flag TIPC_NAMED_BCAST, support reception of this message type.

Four problems need to be considered when introducing this feature.
1) When establishing a link to a new peer node we still update this by a
unicast 'bulk' update. This may lead to race conditions, where a later
broadcast publication/withdrawal bypass the 'bulk', resulting in
disordered publications, or even that a withdrawal may arrive before the
corresponding publication. We solve this by adding an 'is_last_bulk' bit
in the last bulk messages so that it can be distinguished from all other
messages. Only when this message has arrived do we open up for reception
of broadcast publications/withdrawals.
2) When a first legacy node is added to the cluster all distribution
will switch over to use the legacy 'replicast' method, while the
opposite happens when the last legacy node leaves the cluster. This
entails another risk of message disordering that has to be handled. We
solve this by adding a sequence number to the broadcast/replicast
messages, so that disordering can be discovered and corrected. Note
however that we don't need to consider potential message loss or
duplication at this protocol level.
3) Bulk messages don't contain any sequence numbers, and will always
arrive in order. Hence we must exempt those from the sequence number
control and deliver them unconditionally. We solve this by adding a new
'is_bulk' bit in those messages so that they can be recognized.
4) Legacy messages, which don't contain any new bits or sequence
numbers, but neither can arrive out of order, also need to be exempt
from the initial synchronization and sequence number check, and
delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
to all new messages so that those can be distinguished from legacy
messages and the latter delivered directly.

Signed-off-by: Hoang Huu Le <hoa...@de...>
Acked-by: Jon Maloy <jm...@re...>
---
 net/tipc/bcast.c      |   6 +--
 net/tipc/bcast.h      |   4 +-
 net/tipc/link.c       |   2 +-
 net/tipc/msg.h        |  40 ++++++++++++++++
 net/tipc/name_distr.c | 109 +++++++++++++++++++++++++++++++-----------
 net/tipc/name_distr.h |   9 ++--
 net/tipc/name_table.c |   9 +++-
 net/tipc/name_table.h |   2 +
 net/tipc/node.c       |  29 ++++++++---
 net/tipc/node.h       |   8 ++--
 10 files changed, 170 insertions(+), 48 deletions(-)

diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
index 383f87bc1061..940d176e0e87 100644
--- a/net/tipc/bcast.c
+++ b/net/tipc/bcast.c
@@ -250,8 +250,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
  * Consumes the buffer chain.
  * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
  */
-static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
-			   u16 *cong_link_cnt)
+int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
+		    u16 *cong_link_cnt)
 {
 	struct tipc_link *l = tipc_bc_sndlink(net);
 	struct sk_buff_head xmitq;
@@ -752,7 +752,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
 	nl->local = false;
 }
 
-u32 tipc_bcast_get_broadcast_mode(struct net *net)
+u32 tipc_bcast_get_mode(struct net *net)
 {
 	struct tipc_bc_base *bb = tipc_bc_base(net);
 
diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
index 4240c95188b1..2d9352dc7b0e 100644
--- a/net/tipc/bcast.h
+++ b/net/tipc/bcast.h
@@ -90,6 +90,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
 int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
 		    struct tipc_mc_method *method, struct tipc_nlist *dests,
 		    u16 *cong_link_cnt);
+int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
+		    u16 *cong_link_cnt);
 int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
 void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
 			struct tipc_msg *hdr);
@@ -101,7 +103,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg,
 int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
 int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l);
 
-u32 tipc_bcast_get_broadcast_mode(struct net *net);
+u32 tipc_bcast_get_mode(struct net *net);
 u32 tipc_bcast_get_broadcast_ratio(struct net *net);
 
 void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
diff --git a/net/tipc/link.c b/net/tipc/link.c
index ee3b8d0576b8..eac89a3e22ce 100644
--- a/net/tipc/link.c
+++ b/net/tipc/link.c
@@ -2745,7 +2745,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg,
 	void *hdr;
 	struct nlattr *attrs;
 	struct nlattr *prop;
-	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
+	u32 bc_mode = tipc_bcast_get_mode(net);
 	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
 
 	if (!bcl)
diff --git a/net/tipc/msg.h b/net/tipc/msg.h
index 58660d56bc83..65119e81ff0c 100644
--- a/net/tipc/msg.h
+++ b/net/tipc/msg.h
@@ -438,6 +438,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
 	msg_set_bits(m, 1, 25, 0xf, err);
 }
 
+static inline void msg_set_bulk(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 28, 0x1, 1);
+}
+
+static inline u32 msg_is_bulk(struct tipc_msg *m)
+{
+	return msg_bits(m, 1, 28, 0x1);
+}
+
+static inline void msg_set_last_bulk(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 27, 0x1, 1);
+}
+
+static inline u32 msg_is_last_bulk(struct tipc_msg *m)
+{
+	return msg_bits(m, 1, 27, 0x1);
+}
+
+static inline void msg_set_non_legacy(struct tipc_msg *m)
+{
+	msg_set_bits(m, 1, 26, 0x1, 1);
+}
+
+static inline u32 msg_is_legacy(struct tipc_msg *m)
+{
+	return !msg_bits(m, 1, 26, 0x1);
+}
+
 static inline u32 msg_reroute_cnt(struct tipc_msg *m)
 {
 	return msg_bits(m, 1, 21, 0xf);
@@ -567,6 +597,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
 	msg_set_word(m, 4, p);
 }
 
+static inline u16 msg_named_seqno(struct tipc_msg *m)
+{
+	return msg_bits(m, 4, 0, 0xffff);
+}
+
+static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
+{
+	msg_set_bits(m, 4, 0, 0xffff, n);
+}
+
 static inline u32 msg_destport(struct tipc_msg *m)
 {
 	return msg_word(m, 5);
diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
index 5feaf3b67380..481d480609f0 100644
--- a/net/tipc/name_distr.c
+++ b/net/tipc/name_distr.c
@@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
 		pr_warn("Publication distribution failure\n");
 		return NULL;
 	}
-
+	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
+	msg_set_non_legacy(buf_msg(skb));
 	item = (struct distr_item *)msg_data(buf_msg(skb));
 	publ_to_item(item, publ);
 	return skb;
@@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
 struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
 {
 	struct name_table *nt = tipc_name_table(net);
-	struct sk_buff *buf;
 	struct distr_item *item;
+	struct sk_buff *skb;
 
 	write_lock_bh(&nt->cluster_scope_lock);
 	list_del(&publ->binding_node);
@@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
 	if (publ->scope == TIPC_NODE_SCOPE)
 		return NULL;
 
-	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
-	if (!buf) {
+	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
+	if (!skb) {
 		pr_warn("Withdrawal distribution failure\n");
 		return NULL;
 	}
-
-	item = (struct distr_item *)msg_data(buf_msg(buf));
+	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
+	msg_set_non_legacy(buf_msg(skb));
+	item = (struct distr_item *)msg_data(buf_msg(skb));
 	publ_to_item(item, publ);
-	return buf;
+	return skb;
 }
 
 /**
@@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
  * @pls: linked list of publication items to be packed into buffer chain
  */
 static void named_distribute(struct net *net, struct sk_buff_head *list,
-			     u32 dnode, struct list_head *pls)
+			     u32 dnode, struct list_head *pls, u16 seqno)
 {
 	struct publication *publ;
 	struct sk_buff *skb = NULL;
@@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
 			ITEM_SIZE) * ITEM_SIZE;
 	u32 msg_rem = msg_dsz;
+	struct tipc_msg *hdr;
 
 	list_for_each_entry(publ, pls, binding_node) {
 		/* Prepare next buffer: */
@@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 				pr_warn("Bulk publication failure\n");
 				return;
 			}
-			msg_set_bc_ack_invalid(buf_msg(skb), true);
-			item = (struct distr_item *)msg_data(buf_msg(skb));
+			hdr = buf_msg(skb);
+			msg_set_bc_ack_invalid(hdr, true);
+			msg_set_bulk(hdr);
+			msg_set_non_legacy(hdr);
+			item = (struct distr_item *)msg_data(hdr);
 		}
 
 		/* Pack publication into message: */
@@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
 		}
 	}
 	if (skb) {
-		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
+		hdr = buf_msg(skb);
+		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
 		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
 		__skb_queue_tail(list, skb);
 	}
+	hdr = buf_msg(skb_peek_tail(list));
+	msg_set_last_bulk(hdr);
+	msg_set_named_seqno(hdr, seqno);
 }
 
 /**
  * tipc_named_node_up - tell specified node about all publications by this node
  */
-void tipc_named_node_up(struct net *net, u32 dnode)
+void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
 {
 	struct name_table *nt = tipc_name_table(net);
+	struct tipc_net *tn = tipc_net(net);
 	struct sk_buff_head head;
+	u16 seqno;
 
 	__skb_queue_head_init(&head);
+	spin_lock_bh(&tn->nametbl_lock);
+	if (!(capabilities & TIPC_NAMED_BCAST))
+		nt->rc_dests++;
+	seqno = nt->snd_nxt;
+	spin_unlock_bh(&tn->nametbl_lock);
 
 	read_lock_bh(&nt->cluster_scope_lock);
-	named_distribute(net, &head, dnode, &nt->cluster_scope);
+	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
 	tipc_node_xmit(net, &head, dnode, 0);
 	read_unlock_bh(&nt->cluster_scope_lock);
 }
@@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
 	spin_unlock_bh(&tn->nametbl_lock);
 }
 
-void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
+void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
+		      u32 addr, u16 capabilities)
 {
+	struct name_table *nt = tipc_name_table(net);
+	struct tipc_net *tn = tipc_net(net);
+
 	struct publication *publ, *tmp;
 
 	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
 		tipc_publ_purge(net, publ, addr);
 	tipc_dist_queue_purge(net, addr);
+	spin_lock_bh(&tn->nametbl_lock);
+	if (!(capabilities & TIPC_NAMED_BCAST))
+		nt->rc_dests--;
+	spin_unlock_bh(&tn->nametbl_lock);
 }
 
 /**
@@ -295,29 +320,55 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
 	return false;
 }
 
+struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
+				   u16 *rcv_nxt, bool *open)
+{
+	struct sk_buff *skb, *tmp;
+	struct tipc_msg *hdr;
+	u16 seqno;
+
+	skb_queue_walk_safe(namedq, skb, tmp) {
+		skb_linearize(skb);
+		hdr = buf_msg(skb);
+		seqno = msg_named_seqno(hdr);
+		if (msg_is_last_bulk(hdr)) {
+			*rcv_nxt = seqno;
+			*open = true;
+		}
+		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
+			__skb_unlink(skb, namedq);
+			return skb;
+		}
+
+		if (*open && (*rcv_nxt == seqno)) {
+			(*rcv_nxt)++;
+			__skb_unlink(skb, namedq);
+			return skb;
+		}
+	}
+	return NULL;
+}
+
 /**
  * tipc_named_rcv - process name table update messages sent by another node
  */
-void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
+void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
+		    u16 *rcv_nxt, bool *open)
 {
-	struct tipc_net *tn = net_generic(net, tipc_net_id);
-	struct tipc_msg *msg;
+	struct tipc_net *tn = tipc_net(net);
 	struct distr_item *item;
-	uint count;
-	u32 node;
+	struct tipc_msg *hdr;
 	struct sk_buff *skb;
-	int mtype;
+	u32 count, node = 0;
 
 	spin_lock_bh(&tn->nametbl_lock);
-	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
-		skb_linearize(skb);
-		msg = buf_msg(skb);
-		mtype = msg_type(msg);
-		item = (struct distr_item *)msg_data(msg);
-		count = msg_data_sz(msg) / ITEM_SIZE;
-		node = msg_orignode(msg);
+	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
+		hdr = buf_msg(skb);
+		node = msg_orignode(hdr);
+		item = (struct distr_item *)msg_data(hdr);
+		count = msg_data_sz(hdr) / ITEM_SIZE;
 		while (count--) {
-			tipc_update_nametbl(net, item, node, mtype);
+			tipc_update_nametbl(net, item, node, msg_type(hdr));
 			item++;
 		}
 		kfree_skb(skb);
@@ -345,6 +396,6 @@ void tipc_named_reinit(struct net *net)
 		publ->node = self;
 	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
 		publ->node = self;
-
+	nt->rc_dests = 0;
 	spin_unlock_bh(&tn->nametbl_lock);
 }
diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
index 63fc73e0fa6c..092323158f06 100644
--- a/net/tipc/name_distr.h
+++ b/net/tipc/name_distr.h
@@ -67,11 +67,14 @@ struct distr_item {
 	__be32 key;
 };
 
+void tipc_named_bcast(struct net *net, struct sk_buff *skb);
 struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
 struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
-void tipc_named_node_up(struct net *net, u32 dnode);
-void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
+void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
+void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
+		    u16 *rcv_nxt, bool *open);
 void tipc_named_reinit(struct net *net);
-void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
+void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
+		      u32 addr, u16 capabilities);
 
 #endif
diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
index 359b2bc888cf..2ac33d32edc2 100644
--- a/net/tipc/name_table.c
+++ b/net/tipc/name_table.c
@@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 	struct tipc_net *tn = tipc_net(net);
 	struct publication *p = NULL;
 	struct sk_buff *skb = NULL;
+	u32 rc_dests;
 
 	spin_lock_bh(&tn->nametbl_lock);
 
@@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 		nt->local_publ_count++;
 		skb = tipc_named_publish(net, p);
 	}
+	rc_dests = nt->rc_dests;
 exit:
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb)
-		tipc_node_broadcast(net, skb);
+		tipc_node_broadcast(net, skb, rc_dests);
 	return p;
+
 }
 
 /**
@@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 	u32 self = tipc_own_addr(net);
 	struct sk_buff *skb = NULL;
 	struct publication *p;
+	u32 rc_dests;
 
 	spin_lock_bh(&tn->nametbl_lock);
 
@@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
 		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
 		       type, lower, upper, key);
 	}
+	rc_dests = nt->rc_dests;
 	spin_unlock_bh(&tn->nametbl_lock);
 
 	if (skb) {
-		tipc_node_broadcast(net, skb);
+		tipc_node_broadcast(net, skb, rc_dests);
 		return 1;
 	}
 	return 0;
diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
index 728bc7016c38..8064e1986e2c 100644
--- a/net/tipc/name_table.h
+++ b/net/tipc/name_table.h
@@ -106,6 +106,8 @@ struct name_table {
 	struct list_head cluster_scope;
 	rwlock_t cluster_scope_lock;
 	u32 local_publ_count;
+	u32 rc_dests;
+	u32 snd_nxt;
 };
 
 int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/tipc/node.c b/net/tipc/node.c
index a4c2816c3746..030a51c4d1fa 100644
--- a/net/tipc/node.c
+++ b/net/tipc/node.c
@@ -75,6 +75,8 @@ struct tipc_bclink_entry {
 	struct sk_buff_head arrvq;
 	struct sk_buff_head inputq2;
 	struct sk_buff_head namedq;
+	u16 named_rcv_nxt;
+	bool named_open;
 };
 
 /**
@@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
 	write_unlock_bh(&n->lock);
 
 	if (flags & TIPC_NOTIFY_NODE_DOWN)
-		tipc_publ_notify(net, publ_list, addr);
+		tipc_publ_notify(net, publ_list, addr, n->capabilities);
 
 	if (flags & TIPC_NOTIFY_NODE_UP)
-		tipc_named_node_up(net, addr);
+		tipc_named_node_up(net, addr, n->capabilities);
 
 	if (flags & TIPC_NOTIFY_LINK_UP) {
 		tipc_mon_peer_up(net, addr, bearer_id);
@@ -1483,6 +1485,7 @@ static void node_lost_contact(struct tipc_node *n,
 
 	/* Clean up broadcast state */
 	tipc_bcast_remove_peer(n->net, n->bc_entry.link);
+	__skb_queue_purge(&n->bc_entry.namedq);
 
 	/* Abort any ongoing link failover */
 	for (i = 0; i < MAX_BEARERS; i++) {
@@ -1729,12 +1732,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
 	return 0;
 }
 
-void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
+void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
 {
+	struct sk_buff_head xmitq;
 	struct sk_buff *txskb;
 	struct tipc_node *n;
+	u16 dummy;
 	u32 dst;
 
+	/* Use broadcast if all nodes support it */
+	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
+		__skb_queue_head_init(&xmitq);
+		__skb_queue_tail(&xmitq, skb);
+		tipc_bcast_xmit(net, &xmitq, &dummy);
+		return;
+	}
+
+	/* Otherwise use legacy replicast method */
 	rcu_read_lock();
 	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
 		dst = n->addr;
@@ -1749,7 +1763,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
 		tipc_node_xmit_skb(net, txskb, dst, 0);
 	}
 	rcu_read_unlock();
-
 	kfree_skb(skb);
 }
 
@@ -1844,7 +1857,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
 
 	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
 	if (!skb_queue_empty(&n->bc_entry.namedq))
-		tipc_named_rcv(net, &n->bc_entry.namedq);
+		tipc_named_rcv(net, &n->bc_entry.namedq,
+			       &n->bc_entry.named_rcv_nxt,
+			       &n->bc_entry.named_open);
 
 	/* If reassembly or retransmission failure => reset all links to peer */
 	if (rc & TIPC_LINK_DOWN_EVT)
@@ -2114,7 +2129,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
 		tipc_node_link_down(n, bearer_id, false);
 
 	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
-		tipc_named_rcv(net, &n->bc_entry.namedq);
+		tipc_named_rcv(net, &n->bc_entry.namedq,
+			       &n->bc_entry.named_rcv_nxt,
+			       &n->bc_entry.named_open);
 
 	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
 		tipc_node_mcast_rcv(n);
diff --git a/net/tipc/node.h b/net/tipc/node.h
index a6803b449a2c..9f6f13f1604f 100644
--- a/net/tipc/node.h
+++ b/net/tipc/node.h
@@ -55,7 +55,8 @@ enum {
 	TIPC_MCAST_RBCTL      = (1 << 7),
 	TIPC_GAP_ACK_BLOCK    = (1 << 8),
 	TIPC_TUNNEL_ENHANCED  = (1 << 9),
-	TIPC_NAGLE            = (1 << 10)
+	TIPC_NAGLE            = (1 << 10),
+	TIPC_NAMED_BCAST      = (1 << 11)
 };
 
 #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
@@ -68,7 +69,8 @@ enum {
 				TIPC_MCAST_RBCTL       |   \
 				TIPC_GAP_ACK_BLOCK     |   \
 				TIPC_TUNNEL_ENHANCED   |   \
-				TIPC_NAGLE)
+				TIPC_NAGLE             |   \
+				TIPC_NAMED_BCAST)
 
 #define INVALID_BEARER_ID -1
 
@@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
 		       u32 selector);
 void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
 void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
-void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
+void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
 int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
 void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
 int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);
-- 
2.25.1

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/7/20 12:24 AM, Hoang Huu Le wrote:
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
>
> It is worth to use broadcast to update a binding service. This way, the
> binding table can be updated on all peer nodes in one shot.
>
> Broadcast is used when all peer nodes, as indicated by a new capability
> flag TIPC_NAMED_BCAST, support reception of this message type.
>
> Four problems need to be considered when introducing this feature.
> 1) When establishing a link to a new peer node we still update this by a
> unicast 'bulk' update. This may lead to race conditions, where a later
> broadcast publication/withdrawal bypass the 'bulk', resulting in
> disordered publications, or even that a withdrawal may arrive before the
> corresponding publication. We solve this by adding an 'is_last_bulk' bit
> in the last bulk messages so that it can be distinguished from all other
> messages. Only when this message has arrived do we open up for reception
> of broadcast publications/withdrawals.
Add a line feed between these paragraphs before you send the patch.
Otherwise, still acked by me.

///join

> 2) When a first legacy node is added to the cluster all distribution
> will switch over to use the legacy 'replicast' method, while the
> opposite happens when the last legacy node leaves the cluster. This
> entails another risk of message disordering that has to be handled. We
> solve this by adding a sequence number to the broadcast/replicast
> messages, so that disordering can be discovered and corrected. Note
> however that we don't need to consider potential message loss or
> duplication at this protocol level.
> 3) Bulk messages don't contain any sequence numbers, and will always
> arrive in order. Hence we must exempt those from the sequence number
> control and deliver them unconditionally. We solve this by adding a new
> 'is_bulk' bit in those messages so that they can be recognized.
> 4) Legacy messages, which don't contain any new bits or sequence
> numbers, but neither can arrive out of order, also need to be exempt
> from the initial synchronization and sequence number check, and
> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
> to all new messages so that those can be distinguished from legacy
> messages and the latter delivered directly.
>
> Signed-off-by: Hoang Huu Le <hoa...@de...>
> Acked-by: Jon Maloy <jm...@re...>
> ---
>   net/tipc/bcast.c      |   6 +--
>   net/tipc/bcast.h      |   4 +-
>   net/tipc/link.c       |   2 +-
>   net/tipc/msg.h        |  40 ++++++++++++++++
>   net/tipc/name_distr.c | 109 +++++++++++++++++++++++++++++++-----------
>   net/tipc/name_distr.h |   9 ++--
>   net/tipc/name_table.c |   9 +++-
>   net/tipc/name_table.h |   2 +
>   net/tipc/node.c       |  29 ++++++++---
>   net/tipc/node.h       |   8 ++--
>   10 files changed, 170 insertions(+), 48 deletions(-)
>
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index 383f87bc1061..940d176e0e87 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -250,8 +250,8 @@ static void tipc_bcast_select_xmit_method(struct net *net, int dests,
>    * Consumes the buffer chain.
>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>    */
> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> -			   u16 *cong_link_cnt)
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt)
>   {
>   	struct tipc_link *l = tipc_bc_sndlink(net);
>   	struct sk_buff_head xmitq;
> @@ -752,7 +752,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>   	nl->local = false;
>   }
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net)
> +u32 tipc_bcast_get_mode(struct net *net)
>   {
>   	struct tipc_bc_base *bb = tipc_bc_base(net);
>   
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index 4240c95188b1..2d9352dc7b0e 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -90,6 +90,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool supp);
>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>   		    struct tipc_mc_method *method, struct tipc_nlist *dests,
>   		    u16 *cong_link_cnt);
> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
> +		    u16 *cong_link_cnt);
>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb);
>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>   			struct tipc_msg *hdr);
> @@ -101,7 +103,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg,
>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>   int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l);
>   
> -u32 tipc_bcast_get_broadcast_mode(struct net *net);
> +u32 tipc_bcast_get_mode(struct net *net);
>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>   
>   void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> diff --git a/net/tipc/link.c b/net/tipc/link.c
> index ee3b8d0576b8..eac89a3e22ce 100644
> --- a/net/tipc/link.c
> +++ b/net/tipc/link.c
> @@ -2745,7 +2745,7 @@ int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg,
>   	void *hdr;
>   	struct nlattr *attrs;
>   	struct nlattr *prop;
> -	u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
> +	u32 bc_mode = tipc_bcast_get_mode(net);
>   	u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>   
>   	if (!bcl)
> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
> index 58660d56bc83..65119e81ff0c 100644
> --- a/net/tipc/msg.h
> +++ b/net/tipc/msg.h
> @@ -438,6 +438,36 @@ static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
>   	msg_set_bits(m, 1, 25, 0xf, err);
>   }
>   
> +static inline void msg_set_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 28, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 28, 0x1);
> +}
> +
> +static inline void msg_set_last_bulk(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 27, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 1, 27, 0x1);
> +}
> +
> +static inline void msg_set_non_legacy(struct tipc_msg *m)
> +{
> +	msg_set_bits(m, 1, 26, 0x1, 1);
> +}
> +
> +static inline u32 msg_is_legacy(struct tipc_msg *m)
> +{
> +	return !msg_bits(m, 1, 26, 0x1);
> +}
> +
>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>   {
>   	return msg_bits(m, 1, 21, 0xf);
> @@ -567,6 +597,16 @@ static inline void msg_set_origport(struct tipc_msg *m, u32 p)
>   	msg_set_word(m, 4, p);
>   }
>   
> +static inline u16 msg_named_seqno(struct tipc_msg *m)
> +{
> +	return msg_bits(m, 4, 0, 0xffff);
> +}
> +
> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
> +{
> +	msg_set_bits(m, 4, 0, 0xffff, n);
> +}
> +
>   static inline u32 msg_destport(struct tipc_msg *m)
>   {
>   	return msg_word(m, 5);
> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
> index 5feaf3b67380..481d480609f0 100644
> --- a/net/tipc/name_distr.c
> +++ b/net/tipc/name_distr.c
> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   		pr_warn("Publication distribution failure\n");
>   		return NULL;
>   	}
> -
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
>   	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
>   	return skb;
> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ)
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> -	struct sk_buff *buf;
>   	struct distr_item *item;
> +	struct sk_buff *skb;
>   
>   	write_lock_bh(&nt->cluster_scope_lock);
>   	list_del(&publ->binding_node);
> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>   	if (publ->scope == TIPC_NODE_SCOPE)
>   		return NULL;
>   
> -	buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> -	if (!buf) {
> +	skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
> +	if (!skb) {
>   		pr_warn("Withdrawal distribution failure\n");
>   		return NULL;
>   	}
> -
> -	item = (struct distr_item *)msg_data(buf_msg(buf));
> +	msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
> +	msg_set_non_legacy(buf_msg(skb));
> +	item = (struct distr_item *)msg_data(buf_msg(skb));
>   	publ_to_item(item, publ);
> -	return buf;
> +	return skb;
>   }
>   
>   /**
> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ)
>    * @pls: linked list of publication items to be packed into buffer chain
>    */
>   static void named_distribute(struct net *net, struct sk_buff_head *list,
> -			     u32 dnode, struct list_head *pls)
> +			     u32 dnode, struct list_head *pls, u16 seqno)
>   {
>   	struct publication *publ;
>   	struct sk_buff *skb = NULL;
> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   	u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - INT_H_SIZE) /
>   			ITEM_SIZE) * ITEM_SIZE;
>   	u32 msg_rem = msg_dsz;
> +	struct tipc_msg *hdr;
>   
>   	list_for_each_entry(publ, pls, binding_node) {
>   		/* Prepare next buffer: */
> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   				pr_warn("Bulk publication failure\n");
>   				return;
>   			}
> -			msg_set_bc_ack_invalid(buf_msg(skb), true);
> -			item = (struct distr_item *)msg_data(buf_msg(skb));
> +			hdr = buf_msg(skb);
> +			msg_set_bc_ack_invalid(hdr, true);
> +			msg_set_bulk(hdr);
> +			msg_set_non_legacy(hdr);
> +			item = (struct distr_item *)msg_data(hdr);
>   		}
>   
>   		/* Pack publication into message: */
> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>   		}
>   	}
>   	if (skb) {
> -		msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
> +		hdr = buf_msg(skb);
> +		msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>   		skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>   		__skb_queue_tail(list, skb);
>   	}
> +	hdr = buf_msg(skb_peek_tail(list));
> +	msg_set_last_bulk(hdr);
> +	msg_set_named_seqno(hdr, seqno);
>   }
>   
>   /**
>    * tipc_named_node_up - tell specified node about all publications by this node
>    */
> -void tipc_named_node_up(struct net *net, u32 dnode)
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>   {
>   	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
>   	struct sk_buff_head head;
> +	u16 seqno;
>   
>   	__skb_queue_head_init(&head);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests++;
> +	seqno = nt->snd_nxt;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	read_lock_bh(&nt->cluster_scope_lock);
> -	named_distribute(net, &head, dnode, &nt->cluster_scope);
> +	named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>   	tipc_node_xmit(net, &head, dnode, 0);
>   	read_unlock_bh(&nt->cluster_scope_lock);
>   }
> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net *net, u32 addr)
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr)
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities)
>   {
> +	struct name_table *nt = tipc_name_table(net);
> +	struct tipc_net *tn = tipc_net(net);
> +
>   	struct publication *publ, *tmp;
>   
>   	list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>   		tipc_publ_purge(net, publ, addr);
>   	tipc_dist_queue_purge(net, addr);
> +	spin_lock_bh(&tn->nametbl_lock);
> +	if (!(capabilities & TIPC_NAMED_BCAST))
> +		nt->rc_dests--;
> +	spin_unlock_bh(&tn->nametbl_lock);
>   }
>   
>   /**
> @@ -295,29 +320,55 @@ static bool tipc_update_nametbl(struct net *net, struct distr_item *i,
>   	return false;
>   }
>   
> +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
> +				   u16 *rcv_nxt, bool *open)
> +{
> +	struct sk_buff *skb, *tmp;
> +	struct tipc_msg *hdr;
> +	u16 seqno;
> +
> +	skb_queue_walk_safe(namedq, skb, tmp) {
> +		skb_linearize(skb);
> +		hdr = buf_msg(skb);
> +		seqno = msg_named_seqno(hdr);
> +		if (msg_is_last_bulk(hdr)) {
> +			*rcv_nxt = seqno;
> +			*open = true;
> +		}
> +		if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +
> +		if (*open && (*rcv_nxt == seqno)) {
> +			(*rcv_nxt)++;
> +			__skb_unlink(skb, namedq);
> +			return skb;
> +		}
> +	}
> +	return NULL;
> +}
> +
>   /**
>    * tipc_named_rcv - process name table update messages sent by another node
>    */
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open)
>   {
> -	struct tipc_net *tn = net_generic(net, tipc_net_id);
> -	struct tipc_msg *msg;
> +	struct tipc_net *tn = tipc_net(net);
>   	struct distr_item *item;
> -	uint count;
> -	u32 node;
> +	struct tipc_msg *hdr;
>   	struct sk_buff *skb;
> -	int mtype;
> +	u32 count, node = 0;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
> -	for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
> -		skb_linearize(skb);
> -		msg = buf_msg(skb);
> -		mtype = msg_type(msg);
> -		item = (struct distr_item *)msg_data(msg);
> -		count = msg_data_sz(msg) / ITEM_SIZE;
> -		node = msg_orignode(msg);
> +	while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
> +		hdr = buf_msg(skb);
> +		node = msg_orignode(hdr);
> +		item = (struct distr_item *)msg_data(hdr);
> +		count = msg_data_sz(hdr) / ITEM_SIZE;
>   		while (count--) {
> -			tipc_update_nametbl(net, item, node, mtype);
> +			tipc_update_nametbl(net, item, node, msg_type(hdr));
>   			item++;
>   		}
>   		kfree_skb(skb);
> @@ -345,6 +396,6 @@ void tipc_named_reinit(struct net *net)
>   		publ->node = self;
>   	list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>   		publ->node = self;
> -
> +	nt->rc_dests = 0;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   }
> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
> index 63fc73e0fa6c..092323158f06 100644
> --- a/net/tipc/name_distr.h
> +++ b/net/tipc/name_distr.h
> @@ -67,11 +67,14 @@ struct distr_item {
>   	__be32 key;
>   };
>   
> +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>   struct sk_buff *tipc_named_publish(struct net *net, struct publication *publ);
>   struct sk_buff *tipc_named_withdraw(struct net *net, struct publication *publ);
> -void tipc_named_node_up(struct net *net, u32 dnode);
> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
> +		    u16 *rcv_nxt, bool *open);
>   void tipc_named_reinit(struct net *net);
> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, u32 addr);
> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
> +		      u32 addr, u16 capabilities);
>   
>   #endif
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 359b2bc888cf..2ac33d32edc2 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   	struct tipc_net *tn = tipc_net(net);
>   	struct publication *p = NULL;
>   	struct sk_buff *skb = NULL;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -743,12 +744,14 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>   		nt->local_publ_count++;
>   		skb = tipc_named_publish(net, p);
>   	}
> +	rc_dests = nt->rc_dests;
>   exit:
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   	return p;
> +
>   }
>   
>   /**
> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   	u32 self = tipc_own_addr(net);
>   	struct sk_buff *skb = NULL;
>   	struct publication *p;
> +	u32 rc_dests;
>   
>   	spin_lock_bh(&tn->nametbl_lock);
>   
> @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>   		pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>   		       type, lower, upper, key);
>   	}
> +	rc_dests = nt->rc_dests;
>   	spin_unlock_bh(&tn->nametbl_lock);
>   
>   	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_node_broadcast(net, skb, rc_dests);
>   		return 1;
>   	}
>   	return 0;
> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
> index 728bc7016c38..8064e1986e2c 100644
> --- a/net/tipc/name_table.h
> +++ b/net/tipc/name_table.h
> @@ -106,6 +106,8 @@ struct name_table {
>   	struct list_head cluster_scope;
>   	rwlock_t cluster_scope_lock;
>   	u32 local_publ_count;
> +	u32 rc_dests;
> +	u32 snd_nxt;
>   };
>   
>   int tipc_nl_name_table_dump(struct sk_buff *skb, struct netlink_callback *cb);
> diff --git a/net/tipc/node.c b/net/tipc/node.c
> index a4c2816c3746..030a51c4d1fa 100644
> --- a/net/tipc/node.c
> +++ b/net/tipc/node.c
> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>   	struct sk_buff_head arrvq;
>   	struct sk_buff_head inputq2;
>   	struct sk_buff_head namedq;
> +	u16 named_rcv_nxt;
> +	bool named_open;
>   };
>   
>   /**
> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct tipc_node *n)
>   	write_unlock_bh(&n->lock);
>   
>   	if (flags & TIPC_NOTIFY_NODE_DOWN)
> -		tipc_publ_notify(net, publ_list, addr);
> +		tipc_publ_notify(net, publ_list, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_NODE_UP)
> -		tipc_named_node_up(net, addr);
> +		tipc_named_node_up(net, addr, n->capabilities);
>   
>   	if (flags & TIPC_NOTIFY_LINK_UP) {
>   		tipc_mon_peer_up(net, addr, bearer_id);
> @@ -1483,6 +1485,7 @@ static void node_lost_contact(struct tipc_node *n,
>   
>   	/* Clean up broadcast state */
>   	tipc_bcast_remove_peer(n->net, n->bc_entry.link);
> +	__skb_queue_purge(&n->bc_entry.namedq);
>   
>   	/* Abort any ongoing link failover */
>   	for (i = 0; i < MAX_BEARERS; i++) {
> @@ -1729,12 +1732,23 @@ int tipc_node_distr_xmit(struct net *net, struct sk_buff_head *xmitq)
>   	return 0;
>   }
>   
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests)
>   {
> +	struct sk_buff_head xmitq;
>   	struct sk_buff *txskb;
>   	struct tipc_node *n;
> +	u16 dummy;
>   	u32 dst;
>   
> +	/* Use broadcast if all nodes support it */
> +	if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
> +		__skb_queue_head_init(&xmitq);
> +		__skb_queue_tail(&xmitq, skb);
> +		tipc_bcast_xmit(net, &xmitq, &dummy);
> +		return;
> +	}
> +
> +	/* Otherwise use legacy replicast method */
>   	rcu_read_lock();
>   	list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>   		dst = n->addr;
> @@ -1749,7 +1763,6 @@ void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>   		tipc_node_xmit_skb(net, txskb, dst, 0);
>   	}
>   	rcu_read_unlock();
> -
>   	kfree_skb(skb);
>   }
>   
> @@ -1844,7 +1857,9 @@ static void tipc_node_bc_rcv(struct net *net, struct sk_buff *skb, int bearer_id
>   
>   	/* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>   	if (!skb_queue_empty(&n->bc_entry.namedq))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	/* If reassembly or retransmission failure => reset all links to peer */
>   	if (rc & TIPC_LINK_DOWN_EVT)
> @@ -2114,7 +2129,9 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b)
>   		tipc_node_link_down(n, bearer_id, false);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
> -		tipc_named_rcv(net, &n->bc_entry.namedq);
> +		tipc_named_rcv(net, &n->bc_entry.namedq,
> +			       &n->bc_entry.named_rcv_nxt,
> +			       &n->bc_entry.named_open);
>   
>   	if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>   		tipc_node_mcast_rcv(n);
> diff --git a/net/tipc/node.h b/net/tipc/node.h
> index a6803b449a2c..9f6f13f1604f 100644
> --- a/net/tipc/node.h
> +++ b/net/tipc/node.h
> @@ -55,7 +55,8 @@ enum {
>   	TIPC_MCAST_RBCTL      = (1 << 7),
>   	TIPC_GAP_ACK_BLOCK    = (1 << 8),
>   	TIPC_TUNNEL_ENHANCED  = (1 << 9),
> -	TIPC_NAGLE            = (1 << 10)
> +	TIPC_NAGLE            = (1 << 10),
> +	TIPC_NAMED_BCAST      = (1 << 11)
>   };
>   
>   #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
> @@ -68,7 +69,8 @@ enum {
>   				TIPC_MCAST_RBCTL       |   \
>   				TIPC_GAP_ACK_BLOCK     |   \
>   				TIPC_TUNNEL_ENHANCED   |   \
> -				TIPC_NAGLE)
> +				TIPC_NAGLE             |   \
> +				TIPC_NAMED_BCAST)
>   
>   #define INVALID_BEARER_ID -1
>   
> @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct sk_buff *skb, u32 dest,
>   		       u32 selector);
>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, u32 addr);
>   void tipc_node_unsubscribe(struct net *net, struct list_head *subscr, u32 addr);
> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int rc_dests);
>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port);
>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected);

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

On 6/7/20 3:03 PM, Jon Maloy wrote:
>
>
> On 6/7/20 12:24 AM, Hoang Huu Le wrote:
>> Currently, updating binding table (add service binding to
>> name table/withdraw a service binding) is being sent over replicast.
>> However, if we are scaling up clusters to > 100 nodes/containers this
>> method is less affection because of looping through nodes in a 
>> cluster one
>> by one.
>>
>> It is worth to use broadcast to update a binding service. This way, the
>> binding table can be updated on all peer nodes in one shot.
>>
>> Broadcast is used when all peer nodes, as indicated by a new capability
>> flag TIPC_NAMED_BCAST, support reception of this message type.
>>
>> Four problems need to be considered when introducing this feature.
>> 1) When establishing a link to a new peer node we still update this by a
>> unicast 'bulk' update. This may lead to race conditions, where a later
>> broadcast publication/withdrawal bypass the 'bulk', resulting in
>> disordered publications, or even that a withdrawal may arrive before the
>> corresponding publication. We solve this by adding an 'is_last_bulk' bit
>> in the last bulk messages so that it can be distinguished from all other
>> messages. Only when this message has arrived do we open up for reception
>> of broadcast publications/withdrawals.
> Add a line feed between these paragraphs before you send the patch.
> Otherwise, still acked by me.
>
> ///jon
Oh, already posted... Just ignore my comment above.
///jon
>
>> 2) When a first legacy node is added to the cluster all distribution
>> will switch over to use the legacy 'replicast' method, while the
>> opposite happens when the last legacy node leaves the cluster. This
>> entails another risk of message disordering that has to be handled. We
>> solve this by adding a sequence number to the broadcast/replicast
>> messages, so that disordering can be discovered and corrected. Note
>> however that we don't need to consider potential message loss or
>> duplication at this protocol level.
>> 3) Bulk messages don't contain any sequence numbers, and will always
>> arrive in order. Hence we must exempt those from the sequence number
>> control and deliver them unconditionally. We solve this by adding a new
>> 'is_bulk' bit in those messages so that they can be recognized.
>> 4) Legacy messages, which don't contain any new bits or sequence
>> numbers, but neither can arrive out of order, also need to be exempt
>> from the initial synchronization and sequence number check, and
>> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
>> to all new messages so that those can be distinguished from legacy
>> messages and the latter delivered directly.
>>
>> Signed-off-by: Hoang Huu Le <hoa...@de...>
>> Acked-by: Jon Maloy <jm...@re...>
>> ---
>>   net/tipc/bcast.c      |   6 +--
>>   net/tipc/bcast.h      |   4 +-
>>   net/tipc/link.c       |   2 +-
>>   net/tipc/msg.h        |  40 ++++++++++++++++
>>   net/tipc/name_distr.c | 109 +++++++++++++++++++++++++++++++-----------
>>   net/tipc/name_distr.h |   9 ++--
>>   net/tipc/name_table.c |   9 +++-
>>   net/tipc/name_table.h |   2 +
>>   net/tipc/node.c       |  29 ++++++++---
>>   net/tipc/node.h       |   8 ++--
>>   10 files changed, 170 insertions(+), 48 deletions(-)
>>
>> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
>> index 383f87bc1061..940d176e0e87 100644
>> --- a/net/tipc/bcast.c
>> +++ b/net/tipc/bcast.c
>> @@ -250,8 +250,8 @@ static void tipc_bcast_select_xmit_method(struct 
>> net *net, int dests,
>>    * Consumes the buffer chain.
>>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>>    */
>> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>> -               u16 *cong_link_cnt)
>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>> +            u16 *cong_link_cnt)
>>   {
>>       struct tipc_link *l = tipc_bc_sndlink(net);
>>       struct sk_buff_head xmitq;
>> @@ -752,7 +752,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>>       nl->local = false;
>>   }
>>   -u32 tipc_bcast_get_broadcast_mode(struct net *net)
>> +u32 tipc_bcast_get_mode(struct net *net)
>>   {
>>       struct tipc_bc_base *bb = tipc_bc_base(net);
>>   diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
>> index 4240c95188b1..2d9352dc7b0e 100644
>> --- a/net/tipc/bcast.h
>> +++ b/net/tipc/bcast.h
>> @@ -90,6 +90,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool 
>> supp);
>>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>               struct tipc_mc_method *method, struct tipc_nlist *dests,
>>               u16 *cong_link_cnt);
>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>> +            u16 *cong_link_cnt);
>>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct 
>> sk_buff *skb);
>>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>>               struct tipc_msg *hdr);
>> @@ -101,7 +103,7 @@ int tipc_nl_add_bc_link(struct net *net, struct 
>> tipc_nl_msg *msg,
>>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>>   int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l);
>>   -u32 tipc_bcast_get_broadcast_mode(struct net *net);
>> +u32 tipc_bcast_get_mode(struct net *net);
>>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>>     void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head 
>> *defq,
>> diff --git a/net/tipc/link.c b/net/tipc/link.c
>> index ee3b8d0576b8..eac89a3e22ce 100644
>> --- a/net/tipc/link.c
>> +++ b/net/tipc/link.c
>> @@ -2745,7 +2745,7 @@ int tipc_nl_add_bc_link(struct net *net, struct 
>> tipc_nl_msg *msg,
>>       void *hdr;
>>       struct nlattr *attrs;
>>       struct nlattr *prop;
>> -    u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
>> +    u32 bc_mode = tipc_bcast_get_mode(net);
>>       u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>>         if (!bcl)
>> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
>> index 58660d56bc83..65119e81ff0c 100644
>> --- a/net/tipc/msg.h
>> +++ b/net/tipc/msg.h
>> @@ -438,6 +438,36 @@ static inline void msg_set_errcode(struct 
>> tipc_msg *m, u32 err)
>>       msg_set_bits(m, 1, 25, 0xf, err);
>>   }
>>   +static inline void msg_set_bulk(struct tipc_msg *m)
>> +{
>> +    msg_set_bits(m, 1, 28, 0x1, 1);
>> +}
>> +
>> +static inline u32 msg_is_bulk(struct tipc_msg *m)
>> +{
>> +    return msg_bits(m, 1, 28, 0x1);
>> +}
>> +
>> +static inline void msg_set_last_bulk(struct tipc_msg *m)
>> +{
>> +    msg_set_bits(m, 1, 27, 0x1, 1);
>> +}
>> +
>> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
>> +{
>> +    return msg_bits(m, 1, 27, 0x1);
>> +}
>> +
>> +static inline void msg_set_non_legacy(struct tipc_msg *m)
>> +{
>> +    msg_set_bits(m, 1, 26, 0x1, 1);
>> +}
>> +
>> +static inline u32 msg_is_legacy(struct tipc_msg *m)
>> +{
>> +    return !msg_bits(m, 1, 26, 0x1);
>> +}
>> +
>>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>>   {
>>       return msg_bits(m, 1, 21, 0xf);
>> @@ -567,6 +597,16 @@ static inline void msg_set_origport(struct 
>> tipc_msg *m, u32 p)
>>       msg_set_word(m, 4, p);
>>   }
>>   +static inline u16 msg_named_seqno(struct tipc_msg *m)
>> +{
>> +    return msg_bits(m, 4, 0, 0xffff);
>> +}
>> +
>> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
>> +{
>> +    msg_set_bits(m, 4, 0, 0xffff, n);
>> +}
>> +
>>   static inline u32 msg_destport(struct tipc_msg *m)
>>   {
>>       return msg_word(m, 5);
>> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
>> index 5feaf3b67380..481d480609f0 100644
>> --- a/net/tipc/name_distr.c
>> +++ b/net/tipc/name_distr.c
>> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net 
>> *net, struct publication *publ)
>>           pr_warn("Publication distribution failure\n");
>>           return NULL;
>>       }
>> -
>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>> +    msg_set_non_legacy(buf_msg(skb));
>>       item = (struct distr_item *)msg_data(buf_msg(skb));
>>       publ_to_item(item, publ);
>>       return skb;
>> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net 
>> *net, struct publication *publ)
>>   struct sk_buff *tipc_named_withdraw(struct net *net, struct 
>> publication *publ)
>>   {
>>       struct name_table *nt = tipc_name_table(net);
>> -    struct sk_buff *buf;
>>       struct distr_item *item;
>> +    struct sk_buff *skb;
>>         write_lock_bh(&nt->cluster_scope_lock);
>>       list_del(&publ->binding_node);
>> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net 
>> *net, struct publication *publ)
>>       if (publ->scope == TIPC_NODE_SCOPE)
>>           return NULL;
>>   -    buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>> -    if (!buf) {
>> +    skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>> +    if (!skb) {
>>           pr_warn("Withdrawal distribution failure\n");
>>           return NULL;
>>       }
>> -
>> -    item = (struct distr_item *)msg_data(buf_msg(buf));
>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>> +    msg_set_non_legacy(buf_msg(skb));
>> +    item = (struct distr_item *)msg_data(buf_msg(skb));
>>       publ_to_item(item, publ);
>> -    return buf;
>> +    return skb;
>>   }
>>     /**
>> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net 
>> *net, struct publication *publ)
>>    * @pls: linked list of publication items to be packed into buffer 
>> chain
>>    */
>>   static void named_distribute(struct net *net, struct sk_buff_head 
>> *list,
>> -                 u32 dnode, struct list_head *pls)
>> +                 u32 dnode, struct list_head *pls, u16 seqno)
>>   {
>>       struct publication *publ;
>>       struct sk_buff *skb = NULL;
>> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, 
>> struct sk_buff_head *list,
>>       u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - 
>> INT_H_SIZE) /
>>               ITEM_SIZE) * ITEM_SIZE;
>>       u32 msg_rem = msg_dsz;
>> +    struct tipc_msg *hdr;
>>         list_for_each_entry(publ, pls, binding_node) {
>>           /* Prepare next buffer: */
>> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, 
>> struct sk_buff_head *list,
>>                   pr_warn("Bulk publication failure\n");
>>                   return;
>>               }
>> -            msg_set_bc_ack_invalid(buf_msg(skb), true);
>> -            item = (struct distr_item *)msg_data(buf_msg(skb));
>> +            hdr = buf_msg(skb);
>> +            msg_set_bc_ack_invalid(hdr, true);
>> +            msg_set_bulk(hdr);
>> +            msg_set_non_legacy(hdr);
>> +            item = (struct distr_item *)msg_data(hdr);
>>           }
>>             /* Pack publication into message: */
>> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, 
>> struct sk_buff_head *list,
>>           }
>>       }
>>       if (skb) {
>> -        msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
>> +        hdr = buf_msg(skb);
>> +        msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>>           skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>>           __skb_queue_tail(list, skb);
>>       }
>> +    hdr = buf_msg(skb_peek_tail(list));
>> +    msg_set_last_bulk(hdr);
>> +    msg_set_named_seqno(hdr, seqno);
>>   }
>>     /**
>>    * tipc_named_node_up - tell specified node about all publications 
>> by this node
>>    */
>> -void tipc_named_node_up(struct net *net, u32 dnode)
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>>   {
>>       struct name_table *nt = tipc_name_table(net);
>> +    struct tipc_net *tn = tipc_net(net);
>>       struct sk_buff_head head;
>> +    u16 seqno;
>>         __skb_queue_head_init(&head);
>> +    spin_lock_bh(&tn->nametbl_lock);
>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>> +        nt->rc_dests++;
>> +    seqno = nt->snd_nxt;
>> +    spin_unlock_bh(&tn->nametbl_lock);
>>         read_lock_bh(&nt->cluster_scope_lock);
>> -    named_distribute(net, &head, dnode, &nt->cluster_scope);
>> +    named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>>       tipc_node_xmit(net, &head, dnode, 0);
>>       read_unlock_bh(&nt->cluster_scope_lock);
>>   }
>> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net 
>> *net, u32 addr)
>>       spin_unlock_bh(&tn->nametbl_lock);
>>   }
>>   -void tipc_publ_notify(struct net *net, struct list_head 
>> *nsub_list, u32 addr)
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +              u32 addr, u16 capabilities)
>>   {
>> +    struct name_table *nt = tipc_name_table(net);
>> +    struct tipc_net *tn = tipc_net(net);
>> +
>>       struct publication *publ, *tmp;
>>         list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>>           tipc_publ_purge(net, publ, addr);
>>       tipc_dist_queue_purge(net, addr);
>> +    spin_lock_bh(&tn->nametbl_lock);
>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>> +        nt->rc_dests--;
>> +    spin_unlock_bh(&tn->nametbl_lock);
>>   }
>>     /**
>> @@ -295,29 +320,55 @@ static bool tipc_update_nametbl(struct net 
>> *net, struct distr_item *i,
>>       return false;
>>   }
>>   +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
>> +                   u16 *rcv_nxt, bool *open)
>> +{
>> +    struct sk_buff *skb, *tmp;
>> +    struct tipc_msg *hdr;
>> +    u16 seqno;
>> +
>> +    skb_queue_walk_safe(namedq, skb, tmp) {
>> +        skb_linearize(skb);
>> +        hdr = buf_msg(skb);
>> +        seqno = msg_named_seqno(hdr);
>> +        if (msg_is_last_bulk(hdr)) {
>> +            *rcv_nxt = seqno;
>> +            *open = true;
>> +        }
>> +        if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
>> +            __skb_unlink(skb, namedq);
>> +            return skb;
>> +        }
>> +
>> +        if (*open && (*rcv_nxt == seqno)) {
>> +            (*rcv_nxt)++;
>> +            __skb_unlink(skb, namedq);
>> +            return skb;
>> +        }
>> +    }
>> +    return NULL;
>> +}
>> +
>>   /**
>>    * tipc_named_rcv - process name table update messages sent by 
>> another node
>>    */
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +            u16 *rcv_nxt, bool *open)
>>   {
>> -    struct tipc_net *tn = net_generic(net, tipc_net_id);
>> -    struct tipc_msg *msg;
>> +    struct tipc_net *tn = tipc_net(net);
>>       struct distr_item *item;
>> -    uint count;
>> -    u32 node;
>> +    struct tipc_msg *hdr;
>>       struct sk_buff *skb;
>> -    int mtype;
>> +    u32 count, node = 0;
>>         spin_lock_bh(&tn->nametbl_lock);
>> -    for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>> -        skb_linearize(skb);
>> -        msg = buf_msg(skb);
>> -        mtype = msg_type(msg);
>> -        item = (struct distr_item *)msg_data(msg);
>> -        count = msg_data_sz(msg) / ITEM_SIZE;
>> -        node = msg_orignode(msg);
>> +    while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>> +        hdr = buf_msg(skb);
>> +        node = msg_orignode(hdr);
>> +        item = (struct distr_item *)msg_data(hdr);
>> +        count = msg_data_sz(hdr) / ITEM_SIZE;
>>           while (count--) {
>> -            tipc_update_nametbl(net, item, node, mtype);
>> +            tipc_update_nametbl(net, item, node, msg_type(hdr));
>>               item++;
>>           }
>>           kfree_skb(skb);
>> @@ -345,6 +396,6 @@ void tipc_named_reinit(struct net *net)
>>           publ->node = self;
>>       list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>           publ->node = self;
>> -
>> +    nt->rc_dests = 0;
>>       spin_unlock_bh(&tn->nametbl_lock);
>>   }
>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>> index 63fc73e0fa6c..092323158f06 100644
>> --- a/net/tipc/name_distr.h
>> +++ b/net/tipc/name_distr.h
>> @@ -67,11 +67,14 @@ struct distr_item {
>>       __be32 key;
>>   };
>>   +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>   struct sk_buff *tipc_named_publish(struct net *net, struct 
>> publication *publ);
>>   struct sk_buff *tipc_named_withdraw(struct net *net, struct 
>> publication *publ);
>> -void tipc_named_node_up(struct net *net, u32 dnode);
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +            u16 *rcv_nxt, bool *open);
>>   void tipc_named_reinit(struct net *net);
>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, 
>> u32 addr);
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +              u32 addr, u16 capabilities);
>>     #endif
>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>> index 359b2bc888cf..2ac33d32edc2 100644
>> --- a/net/tipc/name_table.c
>> +++ b/net/tipc/name_table.c
>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct 
>> net *net, u32 type, u32 lower,
>>       struct tipc_net *tn = tipc_net(net);
>>       struct publication *p = NULL;
>>       struct sk_buff *skb = NULL;
>> +    u32 rc_dests;
>>         spin_lock_bh(&tn->nametbl_lock);
>>   @@ -743,12 +744,14 @@ struct publication 
>> *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>           nt->local_publ_count++;
>>           skb = tipc_named_publish(net, p);
>>       }
>> +    rc_dests = nt->rc_dests;
>>   exit:
>>       spin_unlock_bh(&tn->nametbl_lock);
>>         if (skb)
>> -        tipc_node_broadcast(net, skb);
>> +        tipc_node_broadcast(net, skb, rc_dests);
>>       return p;
>> +
>>   }
>>     /**
>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 
>> type, u32 lower,
>>       u32 self = tipc_own_addr(net);
>>       struct sk_buff *skb = NULL;
>>       struct publication *p;
>> +    u32 rc_dests;
>>         spin_lock_bh(&tn->nametbl_lock);
>>   @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, 
>> u32 type, u32 lower,
>>           pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>                  type, lower, upper, key);
>>       }
>> +    rc_dests = nt->rc_dests;
>>       spin_unlock_bh(&tn->nametbl_lock);
>>         if (skb) {
>> -        tipc_node_broadcast(net, skb);
>> +        tipc_node_broadcast(net, skb, rc_dests);
>>           return 1;
>>       }
>>       return 0;
>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>> index 728bc7016c38..8064e1986e2c 100644
>> --- a/net/tipc/name_table.h
>> +++ b/net/tipc/name_table.h
>> @@ -106,6 +106,8 @@ struct name_table {
>>       struct list_head cluster_scope;
>>       rwlock_t cluster_scope_lock;
>>       u32 local_publ_count;
>> +    u32 rc_dests;
>> +    u32 snd_nxt;
>>   };
>>     int tipc_nl_name_table_dump(struct sk_buff *skb, struct 
>> netlink_callback *cb);
>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>> index a4c2816c3746..030a51c4d1fa 100644
>> --- a/net/tipc/node.c
>> +++ b/net/tipc/node.c
>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>       struct sk_buff_head arrvq;
>>       struct sk_buff_head inputq2;
>>       struct sk_buff_head namedq;
>> +    u16 named_rcv_nxt;
>> +    bool named_open;
>>   };
>>     /**
>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct 
>> tipc_node *n)
>>       write_unlock_bh(&n->lock);
>>         if (flags & TIPC_NOTIFY_NODE_DOWN)
>> -        tipc_publ_notify(net, publ_list, addr);
>> +        tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>         if (flags & TIPC_NOTIFY_NODE_UP)
>> -        tipc_named_node_up(net, addr);
>> +        tipc_named_node_up(net, addr, n->capabilities);
>>         if (flags & TIPC_NOTIFY_LINK_UP) {
>>           tipc_mon_peer_up(net, addr, bearer_id);
>> @@ -1483,6 +1485,7 @@ static void node_lost_contact(struct tipc_node *n,
>>         /* Clean up broadcast state */
>>       tipc_bcast_remove_peer(n->net, n->bc_entry.link);
>> +    __skb_queue_purge(&n->bc_entry.namedq);
>>         /* Abort any ongoing link failover */
>>       for (i = 0; i < MAX_BEARERS; i++) {
>> @@ -1729,12 +1732,23 @@ int tipc_node_distr_xmit(struct net *net, 
>> struct sk_buff_head *xmitq)
>>       return 0;
>>   }
>>   -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int 
>> rc_dests)
>>   {
>> +    struct sk_buff_head xmitq;
>>       struct sk_buff *txskb;
>>       struct tipc_node *n;
>> +    u16 dummy;
>>       u32 dst;
>>   +    /* Use broadcast if all nodes support it */
>> +    if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>> +        __skb_queue_head_init(&xmitq);
>> +        __skb_queue_tail(&xmitq, skb);
>> +        tipc_bcast_xmit(net, &xmitq, &dummy);
>> +        return;
>> +    }
>> +
>> +    /* Otherwise use legacy replicast method */
>>       rcu_read_lock();
>>       list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>           dst = n->addr;
>> @@ -1749,7 +1763,6 @@ void tipc_node_broadcast(struct net *net, 
>> struct sk_buff *skb)
>>           tipc_node_xmit_skb(net, txskb, dst, 0);
>>       }
>>       rcu_read_unlock();
>> -
>>       kfree_skb(skb);
>>   }
>>   @@ -1844,7 +1857,9 @@ static void tipc_node_bc_rcv(struct net *net, 
>> struct sk_buff *skb, int bearer_id
>>         /* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>       if (!skb_queue_empty(&n->bc_entry.namedq))
>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>> +                   &n->bc_entry.named_rcv_nxt,
>> +                   &n->bc_entry.named_open);
>>         /* If reassembly or retransmission failure => reset all links 
>> to peer */
>>       if (rc & TIPC_LINK_DOWN_EVT)
>> @@ -2114,7 +2129,9 @@ void tipc_rcv(struct net *net, struct sk_buff 
>> *skb, struct tipc_bearer *b)
>>           tipc_node_link_down(n, bearer_id, false);
>>         if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>> +                   &n->bc_entry.named_rcv_nxt,
>> +                   &n->bc_entry.named_open);
>>         if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>           tipc_node_mcast_rcv(n);
>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>> index a6803b449a2c..9f6f13f1604f 100644
>> --- a/net/tipc/node.h
>> +++ b/net/tipc/node.h
>> @@ -55,7 +55,8 @@ enum {
>>       TIPC_MCAST_RBCTL      = (1 << 7),
>>       TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>       TIPC_TUNNEL_ENHANCED  = (1 << 9),
>> -    TIPC_NAGLE            = (1 << 10)
>> +    TIPC_NAGLE            = (1 << 10),
>> +    TIPC_NAMED_BCAST      = (1 << 11)
>>   };
>>     #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>> @@ -68,7 +69,8 @@ enum {
>>                   TIPC_MCAST_RBCTL       |   \
>>                   TIPC_GAP_ACK_BLOCK     |   \
>>                   TIPC_TUNNEL_ENHANCED   |   \
>> -                TIPC_NAGLE)
>> +                TIPC_NAGLE             |   \
>> +                TIPC_NAMED_BCAST)
>>     #define INVALID_BEARER_ID -1
>>   @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct 
>> sk_buff *skb, u32 dest,
>>                  u32 selector);
>>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, 
>> u32 addr);
>>   void tipc_node_unsubscribe(struct net *net, struct list_head 
>> *subscr, u32 addr);
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int 
>> rc_dests);
>>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 
>> peer_port);
>>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool 
>> connected);
>
>
>
> _______________________________________________
> tipc-discussion mailing list
> tip...@li...
> https://lists.sourceforge.net/lists/listinfo/tipc-discussion
>

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[David M. <da...@da...>]

net-next is CLOSED, thank you

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

-----Original Message-----
From: Jon Maloy <jm...@re...> 
Sent: Monday, June 8, 2020 2:14 AM
To: tip...@li...
Subject: Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast



On 6/7/20 3:03 PM, Jon Maloy wrote:
>
>
> On 6/7/20 12:24 AM, Hoang Huu Le wrote:
>> Currently, updating binding table (add service binding to
>> name table/withdraw a service binding) is being sent over replicast.
>> However, if we are scaling up clusters to > 100 nodes/containers this
>> method is less affection because of looping through nodes in a 
>> cluster one
>> by one.
>>
>> It is worth to use broadcast to update a binding service. This way, the
>> binding table can be updated on all peer nodes in one shot.
>>
>> Broadcast is used when all peer nodes, as indicated by a new capability
>> flag TIPC_NAMED_BCAST, support reception of this message type.
>>
>> Four problems need to be considered when introducing this feature.
>> 1) When establishing a link to a new peer node we still update this by a
>> unicast 'bulk' update. This may lead to race conditions, where a later
>> broadcast publication/withdrawal bypass the 'bulk', resulting in
>> disordered publications, or even that a withdrawal may arrive before the
>> corresponding publication. We solve this by adding an 'is_last_bulk' bit
>> in the last bulk messages so that it can be distinguished from all other
>> messages. Only when this message has arrived do we open up for reception
>> of broadcast publications/withdrawals.
> Add a line feed between these paragraphs before you send the patch.
> Otherwise, still acked by me.
>
> ///jon
Oh, already posted... Just ignore my comment above.
[Hoang] net-next is closed. I will re-post the patch later with your suggestion.

///jon
>
>> 2) When a first legacy node is added to the cluster all distribution
>> will switch over to use the legacy 'replicast' method, while the
>> opposite happens when the last legacy node leaves the cluster. This
>> entails another risk of message disordering that has to be handled. We
>> solve this by adding a sequence number to the broadcast/replicast
>> messages, so that disordering can be discovered and corrected. Note
>> however that we don't need to consider potential message loss or
>> duplication at this protocol level.
>> 3) Bulk messages don't contain any sequence numbers, and will always
>> arrive in order. Hence we must exempt those from the sequence number
>> control and deliver them unconditionally. We solve this by adding a new
>> 'is_bulk' bit in those messages so that they can be recognized.
>> 4) Legacy messages, which don't contain any new bits or sequence
>> numbers, but neither can arrive out of order, also need to be exempt
>> from the initial synchronization and sequence number check, and
>> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
>> to all new messages so that those can be distinguished from legacy
>> messages and the latter delivered directly.
>>
>> Signed-off-by: Hoang Huu Le <hoa...@de...>
>> Acked-by: Jon Maloy <jm...@re...>
>> ---
>>   net/tipc/bcast.c      |   6 +--
>>   net/tipc/bcast.h      |   4 +-
>>   net/tipc/link.c       |   2 +-
>>   net/tipc/msg.h        |  40 ++++++++++++++++
>>   net/tipc/name_distr.c | 109 +++++++++++++++++++++++++++++++-----------
>>   net/tipc/name_distr.h |   9 ++--
>>   net/tipc/name_table.c |   9 +++-
>>   net/tipc/name_table.h |   2 +
>>   net/tipc/node.c       |  29 ++++++++---
>>   net/tipc/node.h       |   8 ++--
>>   10 files changed, 170 insertions(+), 48 deletions(-)
>>
>> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
>> index 383f87bc1061..940d176e0e87 100644
>> --- a/net/tipc/bcast.c
>> +++ b/net/tipc/bcast.c
>> @@ -250,8 +250,8 @@ static void tipc_bcast_select_xmit_method(struct 
>> net *net, int dests,
>>    * Consumes the buffer chain.
>>    * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>>    */
>> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>> -               u16 *cong_link_cnt)
>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>> +            u16 *cong_link_cnt)
>>   {
>>       struct tipc_link *l = tipc_bc_sndlink(net);
>>       struct sk_buff_head xmitq;
>> @@ -752,7 +752,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>>       nl->local = false;
>>   }
>>   -u32 tipc_bcast_get_broadcast_mode(struct net *net)
>> +u32 tipc_bcast_get_mode(struct net *net)
>>   {
>>       struct tipc_bc_base *bb = tipc_bc_base(net);
>>   diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
>> index 4240c95188b1..2d9352dc7b0e 100644
>> --- a/net/tipc/bcast.h
>> +++ b/net/tipc/bcast.h
>> @@ -90,6 +90,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool 
>> supp);
>>   int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>               struct tipc_mc_method *method, struct tipc_nlist *dests,
>>               u16 *cong_link_cnt);
>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>> +            u16 *cong_link_cnt);
>>   int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct 
>> sk_buff *skb);
>>   void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>>               struct tipc_msg *hdr);
>> @@ -101,7 +103,7 @@ int tipc_nl_add_bc_link(struct net *net, struct 
>> tipc_nl_msg *msg,
>>   int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>>   int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l);
>>   -u32 tipc_bcast_get_broadcast_mode(struct net *net);
>> +u32 tipc_bcast_get_mode(struct net *net);
>>   u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>>     void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head 
>> *defq,
>> diff --git a/net/tipc/link.c b/net/tipc/link.c
>> index ee3b8d0576b8..eac89a3e22ce 100644
>> --- a/net/tipc/link.c
>> +++ b/net/tipc/link.c
>> @@ -2745,7 +2745,7 @@ int tipc_nl_add_bc_link(struct net *net, struct 
>> tipc_nl_msg *msg,
>>       void *hdr;
>>       struct nlattr *attrs;
>>       struct nlattr *prop;
>> -    u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
>> +    u32 bc_mode = tipc_bcast_get_mode(net);
>>       u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>>         if (!bcl)
>> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
>> index 58660d56bc83..65119e81ff0c 100644
>> --- a/net/tipc/msg.h
>> +++ b/net/tipc/msg.h
>> @@ -438,6 +438,36 @@ static inline void msg_set_errcode(struct 
>> tipc_msg *m, u32 err)
>>       msg_set_bits(m, 1, 25, 0xf, err);
>>   }
>>   +static inline void msg_set_bulk(struct tipc_msg *m)
>> +{
>> +    msg_set_bits(m, 1, 28, 0x1, 1);
>> +}
>> +
>> +static inline u32 msg_is_bulk(struct tipc_msg *m)
>> +{
>> +    return msg_bits(m, 1, 28, 0x1);
>> +}
>> +
>> +static inline void msg_set_last_bulk(struct tipc_msg *m)
>> +{
>> +    msg_set_bits(m, 1, 27, 0x1, 1);
>> +}
>> +
>> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
>> +{
>> +    return msg_bits(m, 1, 27, 0x1);
>> +}
>> +
>> +static inline void msg_set_non_legacy(struct tipc_msg *m)
>> +{
>> +    msg_set_bits(m, 1, 26, 0x1, 1);
>> +}
>> +
>> +static inline u32 msg_is_legacy(struct tipc_msg *m)
>> +{
>> +    return !msg_bits(m, 1, 26, 0x1);
>> +}
>> +
>>   static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>>   {
>>       return msg_bits(m, 1, 21, 0xf);
>> @@ -567,6 +597,16 @@ static inline void msg_set_origport(struct 
>> tipc_msg *m, u32 p)
>>       msg_set_word(m, 4, p);
>>   }
>>   +static inline u16 msg_named_seqno(struct tipc_msg *m)
>> +{
>> +    return msg_bits(m, 4, 0, 0xffff);
>> +}
>> +
>> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
>> +{
>> +    msg_set_bits(m, 4, 0, 0xffff, n);
>> +}
>> +
>>   static inline u32 msg_destport(struct tipc_msg *m)
>>   {
>>       return msg_word(m, 5);
>> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
>> index 5feaf3b67380..481d480609f0 100644
>> --- a/net/tipc/name_distr.c
>> +++ b/net/tipc/name_distr.c
>> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net 
>> *net, struct publication *publ)
>>           pr_warn("Publication distribution failure\n");
>>           return NULL;
>>       }
>> -
>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>> +    msg_set_non_legacy(buf_msg(skb));
>>       item = (struct distr_item *)msg_data(buf_msg(skb));
>>       publ_to_item(item, publ);
>>       return skb;
>> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net 
>> *net, struct publication *publ)
>>   struct sk_buff *tipc_named_withdraw(struct net *net, struct 
>> publication *publ)
>>   {
>>       struct name_table *nt = tipc_name_table(net);
>> -    struct sk_buff *buf;
>>       struct distr_item *item;
>> +    struct sk_buff *skb;
>>         write_lock_bh(&nt->cluster_scope_lock);
>>       list_del(&publ->binding_node);
>> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net 
>> *net, struct publication *publ)
>>       if (publ->scope == TIPC_NODE_SCOPE)
>>           return NULL;
>>   -    buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>> -    if (!buf) {
>> +    skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>> +    if (!skb) {
>>           pr_warn("Withdrawal distribution failure\n");
>>           return NULL;
>>       }
>> -
>> -    item = (struct distr_item *)msg_data(buf_msg(buf));
>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>> +    msg_set_non_legacy(buf_msg(skb));
>> +    item = (struct distr_item *)msg_data(buf_msg(skb));
>>       publ_to_item(item, publ);
>> -    return buf;
>> +    return skb;
>>   }
>>     /**
>> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net 
>> *net, struct publication *publ)
>>    * @pls: linked list of publication items to be packed into buffer 
>> chain
>>    */
>>   static void named_distribute(struct net *net, struct sk_buff_head 
>> *list,
>> -                 u32 dnode, struct list_head *pls)
>> +                 u32 dnode, struct list_head *pls, u16 seqno)
>>   {
>>       struct publication *publ;
>>       struct sk_buff *skb = NULL;
>> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net, 
>> struct sk_buff_head *list,
>>       u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) - 
>> INT_H_SIZE) /
>>               ITEM_SIZE) * ITEM_SIZE;
>>       u32 msg_rem = msg_dsz;
>> +    struct tipc_msg *hdr;
>>         list_for_each_entry(publ, pls, binding_node) {
>>           /* Prepare next buffer: */
>> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net, 
>> struct sk_buff_head *list,
>>                   pr_warn("Bulk publication failure\n");
>>                   return;
>>               }
>> -            msg_set_bc_ack_invalid(buf_msg(skb), true);
>> -            item = (struct distr_item *)msg_data(buf_msg(skb));
>> +            hdr = buf_msg(skb);
>> +            msg_set_bc_ack_invalid(hdr, true);
>> +            msg_set_bulk(hdr);
>> +            msg_set_non_legacy(hdr);
>> +            item = (struct distr_item *)msg_data(hdr);
>>           }
>>             /* Pack publication into message: */
>> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net, 
>> struct sk_buff_head *list,
>>           }
>>       }
>>       if (skb) {
>> -        msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
>> +        hdr = buf_msg(skb);
>> +        msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>>           skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>>           __skb_queue_tail(list, skb);
>>       }
>> +    hdr = buf_msg(skb_peek_tail(list));
>> +    msg_set_last_bulk(hdr);
>> +    msg_set_named_seqno(hdr, seqno);
>>   }
>>     /**
>>    * tipc_named_node_up - tell specified node about all publications 
>> by this node
>>    */
>> -void tipc_named_node_up(struct net *net, u32 dnode)
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>>   {
>>       struct name_table *nt = tipc_name_table(net);
>> +    struct tipc_net *tn = tipc_net(net);
>>       struct sk_buff_head head;
>> +    u16 seqno;
>>         __skb_queue_head_init(&head);
>> +    spin_lock_bh(&tn->nametbl_lock);
>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>> +        nt->rc_dests++;
>> +    seqno = nt->snd_nxt;
>> +    spin_unlock_bh(&tn->nametbl_lock);
>>         read_lock_bh(&nt->cluster_scope_lock);
>> -    named_distribute(net, &head, dnode, &nt->cluster_scope);
>> +    named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>>       tipc_node_xmit(net, &head, dnode, 0);
>>       read_unlock_bh(&nt->cluster_scope_lock);
>>   }
>> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net 
>> *net, u32 addr)
>>       spin_unlock_bh(&tn->nametbl_lock);
>>   }
>>   -void tipc_publ_notify(struct net *net, struct list_head 
>> *nsub_list, u32 addr)
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +              u32 addr, u16 capabilities)
>>   {
>> +    struct name_table *nt = tipc_name_table(net);
>> +    struct tipc_net *tn = tipc_net(net);
>> +
>>       struct publication *publ, *tmp;
>>         list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>>           tipc_publ_purge(net, publ, addr);
>>       tipc_dist_queue_purge(net, addr);
>> +    spin_lock_bh(&tn->nametbl_lock);
>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>> +        nt->rc_dests--;
>> +    spin_unlock_bh(&tn->nametbl_lock);
>>   }
>>     /**
>> @@ -295,29 +320,55 @@ static bool tipc_update_nametbl(struct net 
>> *net, struct distr_item *i,
>>       return false;
>>   }
>>   +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
>> +                   u16 *rcv_nxt, bool *open)
>> +{
>> +    struct sk_buff *skb, *tmp;
>> +    struct tipc_msg *hdr;
>> +    u16 seqno;
>> +
>> +    skb_queue_walk_safe(namedq, skb, tmp) {
>> +        skb_linearize(skb);
>> +        hdr = buf_msg(skb);
>> +        seqno = msg_named_seqno(hdr);
>> +        if (msg_is_last_bulk(hdr)) {
>> +            *rcv_nxt = seqno;
>> +            *open = true;
>> +        }
>> +        if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
>> +            __skb_unlink(skb, namedq);
>> +            return skb;
>> +        }
>> +
>> +        if (*open && (*rcv_nxt == seqno)) {
>> +            (*rcv_nxt)++;
>> +            __skb_unlink(skb, namedq);
>> +            return skb;
>> +        }
>> +    }
>> +    return NULL;
>> +}
>> +
>>   /**
>>    * tipc_named_rcv - process name table update messages sent by 
>> another node
>>    */
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +            u16 *rcv_nxt, bool *open)
>>   {
>> -    struct tipc_net *tn = net_generic(net, tipc_net_id);
>> -    struct tipc_msg *msg;
>> +    struct tipc_net *tn = tipc_net(net);
>>       struct distr_item *item;
>> -    uint count;
>> -    u32 node;
>> +    struct tipc_msg *hdr;
>>       struct sk_buff *skb;
>> -    int mtype;
>> +    u32 count, node = 0;
>>         spin_lock_bh(&tn->nametbl_lock);
>> -    for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>> -        skb_linearize(skb);
>> -        msg = buf_msg(skb);
>> -        mtype = msg_type(msg);
>> -        item = (struct distr_item *)msg_data(msg);
>> -        count = msg_data_sz(msg) / ITEM_SIZE;
>> -        node = msg_orignode(msg);
>> +    while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>> +        hdr = buf_msg(skb);
>> +        node = msg_orignode(hdr);
>> +        item = (struct distr_item *)msg_data(hdr);
>> +        count = msg_data_sz(hdr) / ITEM_SIZE;
>>           while (count--) {
>> -            tipc_update_nametbl(net, item, node, mtype);
>> +            tipc_update_nametbl(net, item, node, msg_type(hdr));
>>               item++;
>>           }
>>           kfree_skb(skb);
>> @@ -345,6 +396,6 @@ void tipc_named_reinit(struct net *net)
>>           publ->node = self;
>>       list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>           publ->node = self;
>> -
>> +    nt->rc_dests = 0;
>>       spin_unlock_bh(&tn->nametbl_lock);
>>   }
>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>> index 63fc73e0fa6c..092323158f06 100644
>> --- a/net/tipc/name_distr.h
>> +++ b/net/tipc/name_distr.h
>> @@ -67,11 +67,14 @@ struct distr_item {
>>       __be32 key;
>>   };
>>   +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>   struct sk_buff *tipc_named_publish(struct net *net, struct 
>> publication *publ);
>>   struct sk_buff *tipc_named_withdraw(struct net *net, struct 
>> publication *publ);
>> -void tipc_named_node_up(struct net *net, u32 dnode);
>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>> +            u16 *rcv_nxt, bool *open);
>>   void tipc_named_reinit(struct net *net);
>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list, 
>> u32 addr);
>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>> +              u32 addr, u16 capabilities);
>>     #endif
>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>> index 359b2bc888cf..2ac33d32edc2 100644
>> --- a/net/tipc/name_table.c
>> +++ b/net/tipc/name_table.c
>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct 
>> net *net, u32 type, u32 lower,
>>       struct tipc_net *tn = tipc_net(net);
>>       struct publication *p = NULL;
>>       struct sk_buff *skb = NULL;
>> +    u32 rc_dests;
>>         spin_lock_bh(&tn->nametbl_lock);
>>   @@ -743,12 +744,14 @@ struct publication 
>> *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>           nt->local_publ_count++;
>>           skb = tipc_named_publish(net, p);
>>       }
>> +    rc_dests = nt->rc_dests;
>>   exit:
>>       spin_unlock_bh(&tn->nametbl_lock);
>>         if (skb)
>> -        tipc_node_broadcast(net, skb);
>> +        tipc_node_broadcast(net, skb, rc_dests);
>>       return p;
>> +
>>   }
>>     /**
>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 
>> type, u32 lower,
>>       u32 self = tipc_own_addr(net);
>>       struct sk_buff *skb = NULL;
>>       struct publication *p;
>> +    u32 rc_dests;
>>         spin_lock_bh(&tn->nametbl_lock);
>>   @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net, 
>> u32 type, u32 lower,
>>           pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>                  type, lower, upper, key);
>>       }
>> +    rc_dests = nt->rc_dests;
>>       spin_unlock_bh(&tn->nametbl_lock);
>>         if (skb) {
>> -        tipc_node_broadcast(net, skb);
>> +        tipc_node_broadcast(net, skb, rc_dests);
>>           return 1;
>>       }
>>       return 0;
>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>> index 728bc7016c38..8064e1986e2c 100644
>> --- a/net/tipc/name_table.h
>> +++ b/net/tipc/name_table.h
>> @@ -106,6 +106,8 @@ struct name_table {
>>       struct list_head cluster_scope;
>>       rwlock_t cluster_scope_lock;
>>       u32 local_publ_count;
>> +    u32 rc_dests;
>> +    u32 snd_nxt;
>>   };
>>     int tipc_nl_name_table_dump(struct sk_buff *skb, struct 
>> netlink_callback *cb);
>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>> index a4c2816c3746..030a51c4d1fa 100644
>> --- a/net/tipc/node.c
>> +++ b/net/tipc/node.c
>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>       struct sk_buff_head arrvq;
>>       struct sk_buff_head inputq2;
>>       struct sk_buff_head namedq;
>> +    u16 named_rcv_nxt;
>> +    bool named_open;
>>   };
>>     /**
>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct 
>> tipc_node *n)
>>       write_unlock_bh(&n->lock);
>>         if (flags & TIPC_NOTIFY_NODE_DOWN)
>> -        tipc_publ_notify(net, publ_list, addr);
>> +        tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>         if (flags & TIPC_NOTIFY_NODE_UP)
>> -        tipc_named_node_up(net, addr);
>> +        tipc_named_node_up(net, addr, n->capabilities);
>>         if (flags & TIPC_NOTIFY_LINK_UP) {
>>           tipc_mon_peer_up(net, addr, bearer_id);
>> @@ -1483,6 +1485,7 @@ static void node_lost_contact(struct tipc_node *n,
>>         /* Clean up broadcast state */
>>       tipc_bcast_remove_peer(n->net, n->bc_entry.link);
>> +    __skb_queue_purge(&n->bc_entry.namedq);
>>         /* Abort any ongoing link failover */
>>       for (i = 0; i < MAX_BEARERS; i++) {
>> @@ -1729,12 +1732,23 @@ int tipc_node_distr_xmit(struct net *net, 
>> struct sk_buff_head *xmitq)
>>       return 0;
>>   }
>>   -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int 
>> rc_dests)
>>   {
>> +    struct sk_buff_head xmitq;
>>       struct sk_buff *txskb;
>>       struct tipc_node *n;
>> +    u16 dummy;
>>       u32 dst;
>>   +    /* Use broadcast if all nodes support it */
>> +    if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>> +        __skb_queue_head_init(&xmitq);
>> +        __skb_queue_tail(&xmitq, skb);
>> +        tipc_bcast_xmit(net, &xmitq, &dummy);
>> +        return;
>> +    }
>> +
>> +    /* Otherwise use legacy replicast method */
>>       rcu_read_lock();
>>       list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>           dst = n->addr;
>> @@ -1749,7 +1763,6 @@ void tipc_node_broadcast(struct net *net, 
>> struct sk_buff *skb)
>>           tipc_node_xmit_skb(net, txskb, dst, 0);
>>       }
>>       rcu_read_unlock();
>> -
>>       kfree_skb(skb);
>>   }
>>   @@ -1844,7 +1857,9 @@ static void tipc_node_bc_rcv(struct net *net, 
>> struct sk_buff *skb, int bearer_id
>>         /* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>       if (!skb_queue_empty(&n->bc_entry.namedq))
>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>> +                   &n->bc_entry.named_rcv_nxt,
>> +                   &n->bc_entry.named_open);
>>         /* If reassembly or retransmission failure => reset all links 
>> to peer */
>>       if (rc & TIPC_LINK_DOWN_EVT)
>> @@ -2114,7 +2129,9 @@ void tipc_rcv(struct net *net, struct sk_buff 
>> *skb, struct tipc_bearer *b)
>>           tipc_node_link_down(n, bearer_id, false);
>>         if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>> +                   &n->bc_entry.named_rcv_nxt,
>> +                   &n->bc_entry.named_open);
>>         if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>           tipc_node_mcast_rcv(n);
>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>> index a6803b449a2c..9f6f13f1604f 100644
>> --- a/net/tipc/node.h
>> +++ b/net/tipc/node.h
>> @@ -55,7 +55,8 @@ enum {
>>       TIPC_MCAST_RBCTL      = (1 << 7),
>>       TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>       TIPC_TUNNEL_ENHANCED  = (1 << 9),
>> -    TIPC_NAGLE            = (1 << 10)
>> +    TIPC_NAGLE            = (1 << 10),
>> +    TIPC_NAMED_BCAST      = (1 << 11)
>>   };
>>     #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>> @@ -68,7 +69,8 @@ enum {
>>                   TIPC_MCAST_RBCTL       |   \
>>                   TIPC_GAP_ACK_BLOCK     |   \
>>                   TIPC_TUNNEL_ENHANCED   |   \
>> -                TIPC_NAGLE)
>> +                TIPC_NAGLE             |   \
>> +                TIPC_NAMED_BCAST)
>>     #define INVALID_BEARER_ID -1
>>   @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct 
>> sk_buff *skb, u32 dest,
>>                  u32 selector);
>>   void tipc_node_subscribe(struct net *net, struct list_head *subscr, 
>> u32 addr);
>>   void tipc_node_unsubscribe(struct net *net, struct list_head 
>> *subscr, u32 addr);
>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int 
>> rc_dests);
>>   int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 
>> peer_port);
>>   void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>   int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool 
>> connected);
>
>
>
> _______________________________________________
> tipc-discussion mailing list
> tip...@li...
> https://lists.sourceforge.net/lists/listinfo/tipc-discussion
>



_______________________________________________
tipc-discussion mailing list
tip...@li...
https://lists.sourceforge.net/lists/listinfo/tipc-discussion

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Jon M. <jm...@re...>]

In case you are not aware of this, check this link before you send 
anything to net-next.

http://vger.kernel.org/~davem/net-next.html

///jon



On 6/7/20 9:50 PM, Hoang Huu Le wrote:
>
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Monday, June 8, 2020 2:14 AM
> To: tip...@li...
> Subject: Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast
>
>
>
> On 6/7/20 3:03 PM, Jon Maloy wrote:
>>
>> On 6/7/20 12:24 AM, Hoang Huu Le wrote:
>>> Currently, updating binding table (add service binding to
>>> name table/withdraw a service binding) is being sent over replicast.
>>> However, if we are scaling up clusters to > 100 nodes/containers this
>>> method is less affection because of looping through nodes in a
>>> cluster one
>>> by one.
>>>
>>> It is worth to use broadcast to update a binding service. This way, the
>>> binding table can be updated on all peer nodes in one shot.
>>>
>>> Broadcast is used when all peer nodes, as indicated by a new capability
>>> flag TIPC_NAMED_BCAST, support reception of this message type.
>>>
>>> Four problems need to be considered when introducing this feature.
>>> 1) When establishing a link to a new peer node we still update this by a
>>> unicast 'bulk' update. This may lead to race conditions, where a later
>>> broadcast publication/withdrawal bypass the 'bulk', resulting in
>>> disordered publications, or even that a withdrawal may arrive before the
>>> corresponding publication. We solve this by adding an 'is_last_bulk' bit
>>> in the last bulk messages so that it can be distinguished from all other
>>> messages. Only when this message has arrived do we open up for reception
>>> of broadcast publications/withdrawals.
>> Add a line feed between these paragraphs before you send the patch.
>> Otherwise, still acked by me.
>>
>> ///jon
> Oh, already posted... Just ignore my comment above.
> [Hoang] net-next is closed. I will re-post the patch later with your suggestion.
>
> ///jon
>>> 2) When a first legacy node is added to the cluster all distribution
>>> will switch over to use the legacy 'replicast' method, while the
>>> opposite happens when the last legacy node leaves the cluster. This
>>> entails another risk of message disordering that has to be handled. We
>>> solve this by adding a sequence number to the broadcast/replicast
>>> messages, so that disordering can be discovered and corrected. Note
>>> however that we don't need to consider potential message loss or
>>> duplication at this protocol level.
>>> 3) Bulk messages don't contain any sequence numbers, and will always
>>> arrive in order. Hence we must exempt those from the sequence number
>>> control and deliver them unconditionally. We solve this by adding a new
>>> 'is_bulk' bit in those messages so that they can be recognized.
>>> 4) Legacy messages, which don't contain any new bits or sequence
>>> numbers, but neither can arrive out of order, also need to be exempt
>>> from the initial synchronization and sequence number check, and
>>> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
>>> to all new messages so that those can be distinguished from legacy
>>> messages and the latter delivered directly.
>>>
>>> Signed-off-by: Hoang Huu Le <hoa...@de...>
>>> Acked-by: Jon Maloy <jm...@re...>
>>> ---
>>>    net/tipc/bcast.c      |   6 +--
>>>    net/tipc/bcast.h      |   4 +-
>>>    net/tipc/link.c       |   2 +-
>>>    net/tipc/msg.h        |  40 ++++++++++++++++
>>>    net/tipc/name_distr.c | 109 +++++++++++++++++++++++++++++++-----------
>>>    net/tipc/name_distr.h |   9 ++--
>>>    net/tipc/name_table.c |   9 +++-
>>>    net/tipc/name_table.h |   2 +
>>>    net/tipc/node.c       |  29 ++++++++---
>>>    net/tipc/node.h       |   8 ++--
>>>    10 files changed, 170 insertions(+), 48 deletions(-)
>>>
>>> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
>>> index 383f87bc1061..940d176e0e87 100644
>>> --- a/net/tipc/bcast.c
>>> +++ b/net/tipc/bcast.c
>>> @@ -250,8 +250,8 @@ static void tipc_bcast_select_xmit_method(struct
>>> net *net, int dests,
>>>     * Consumes the buffer chain.
>>>     * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>>>     */
>>> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>> -               u16 *cong_link_cnt)
>>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>> +            u16 *cong_link_cnt)
>>>    {
>>>        struct tipc_link *l = tipc_bc_sndlink(net);
>>>        struct sk_buff_head xmitq;
>>> @@ -752,7 +752,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>>>        nl->local = false;
>>>    }
>>>    -u32 tipc_bcast_get_broadcast_mode(struct net *net)
>>> +u32 tipc_bcast_get_mode(struct net *net)
>>>    {
>>>        struct tipc_bc_base *bb = tipc_bc_base(net);
>>>    diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
>>> index 4240c95188b1..2d9352dc7b0e 100644
>>> --- a/net/tipc/bcast.h
>>> +++ b/net/tipc/bcast.h
>>> @@ -90,6 +90,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool
>>> supp);
>>>    int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>>                struct tipc_mc_method *method, struct tipc_nlist *dests,
>>>                u16 *cong_link_cnt);
>>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>> +            u16 *cong_link_cnt);
>>>    int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct
>>> sk_buff *skb);
>>>    void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>>>                struct tipc_msg *hdr);
>>> @@ -101,7 +103,7 @@ int tipc_nl_add_bc_link(struct net *net, struct
>>> tipc_nl_msg *msg,
>>>    int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>>>    int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l);
>>>    -u32 tipc_bcast_get_broadcast_mode(struct net *net);
>>> +u32 tipc_bcast_get_mode(struct net *net);
>>>    u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>>>      void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head
>>> *defq,
>>> diff --git a/net/tipc/link.c b/net/tipc/link.c
>>> index ee3b8d0576b8..eac89a3e22ce 100644
>>> --- a/net/tipc/link.c
>>> +++ b/net/tipc/link.c
>>> @@ -2745,7 +2745,7 @@ int tipc_nl_add_bc_link(struct net *net, struct
>>> tipc_nl_msg *msg,
>>>        void *hdr;
>>>        struct nlattr *attrs;
>>>        struct nlattr *prop;
>>> -    u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
>>> +    u32 bc_mode = tipc_bcast_get_mode(net);
>>>        u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>>>          if (!bcl)
>>> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
>>> index 58660d56bc83..65119e81ff0c 100644
>>> --- a/net/tipc/msg.h
>>> +++ b/net/tipc/msg.h
>>> @@ -438,6 +438,36 @@ static inline void msg_set_errcode(struct
>>> tipc_msg *m, u32 err)
>>>        msg_set_bits(m, 1, 25, 0xf, err);
>>>    }
>>>    +static inline void msg_set_bulk(struct tipc_msg *m)
>>> +{
>>> +    msg_set_bits(m, 1, 28, 0x1, 1);
>>> +}
>>> +
>>> +static inline u32 msg_is_bulk(struct tipc_msg *m)
>>> +{
>>> +    return msg_bits(m, 1, 28, 0x1);
>>> +}
>>> +
>>> +static inline void msg_set_last_bulk(struct tipc_msg *m)
>>> +{
>>> +    msg_set_bits(m, 1, 27, 0x1, 1);
>>> +}
>>> +
>>> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
>>> +{
>>> +    return msg_bits(m, 1, 27, 0x1);
>>> +}
>>> +
>>> +static inline void msg_set_non_legacy(struct tipc_msg *m)
>>> +{
>>> +    msg_set_bits(m, 1, 26, 0x1, 1);
>>> +}
>>> +
>>> +static inline u32 msg_is_legacy(struct tipc_msg *m)
>>> +{
>>> +    return !msg_bits(m, 1, 26, 0x1);
>>> +}
>>> +
>>>    static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>>>    {
>>>        return msg_bits(m, 1, 21, 0xf);
>>> @@ -567,6 +597,16 @@ static inline void msg_set_origport(struct
>>> tipc_msg *m, u32 p)
>>>        msg_set_word(m, 4, p);
>>>    }
>>>    +static inline u16 msg_named_seqno(struct tipc_msg *m)
>>> +{
>>> +    return msg_bits(m, 4, 0, 0xffff);
>>> +}
>>> +
>>> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
>>> +{
>>> +    msg_set_bits(m, 4, 0, 0xffff, n);
>>> +}
>>> +
>>>    static inline u32 msg_destport(struct tipc_msg *m)
>>>    {
>>>        return msg_word(m, 5);
>>> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
>>> index 5feaf3b67380..481d480609f0 100644
>>> --- a/net/tipc/name_distr.c
>>> +++ b/net/tipc/name_distr.c
>>> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net
>>> *net, struct publication *publ)
>>>            pr_warn("Publication distribution failure\n");
>>>            return NULL;
>>>        }
>>> -
>>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>>> +    msg_set_non_legacy(buf_msg(skb));
>>>        item = (struct distr_item *)msg_data(buf_msg(skb));
>>>        publ_to_item(item, publ);
>>>        return skb;
>>> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net
>>> *net, struct publication *publ)
>>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct
>>> publication *publ)
>>>    {
>>>        struct name_table *nt = tipc_name_table(net);
>>> -    struct sk_buff *buf;
>>>        struct distr_item *item;
>>> +    struct sk_buff *skb;
>>>          write_lock_bh(&nt->cluster_scope_lock);
>>>        list_del(&publ->binding_node);
>>> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net
>>> *net, struct publication *publ)
>>>        if (publ->scope == TIPC_NODE_SCOPE)
>>>            return NULL;
>>>    -    buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>>> -    if (!buf) {
>>> +    skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>>> +    if (!skb) {
>>>            pr_warn("Withdrawal distribution failure\n");
>>>            return NULL;
>>>        }
>>> -
>>> -    item = (struct distr_item *)msg_data(buf_msg(buf));
>>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>>> +    msg_set_non_legacy(buf_msg(skb));
>>> +    item = (struct distr_item *)msg_data(buf_msg(skb));
>>>        publ_to_item(item, publ);
>>> -    return buf;
>>> +    return skb;
>>>    }
>>>      /**
>>> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net
>>> *net, struct publication *publ)
>>>     * @pls: linked list of publication items to be packed into buffer
>>> chain
>>>     */
>>>    static void named_distribute(struct net *net, struct sk_buff_head
>>> *list,
>>> -                 u32 dnode, struct list_head *pls)
>>> +                 u32 dnode, struct list_head *pls, u16 seqno)
>>>    {
>>>        struct publication *publ;
>>>        struct sk_buff *skb = NULL;
>>> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net,
>>> struct sk_buff_head *list,
>>>        u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) -
>>> INT_H_SIZE) /
>>>                ITEM_SIZE) * ITEM_SIZE;
>>>        u32 msg_rem = msg_dsz;
>>> +    struct tipc_msg *hdr;
>>>          list_for_each_entry(publ, pls, binding_node) {
>>>            /* Prepare next buffer: */
>>> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net,
>>> struct sk_buff_head *list,
>>>                    pr_warn("Bulk publication failure\n");
>>>                    return;
>>>                }
>>> -            msg_set_bc_ack_invalid(buf_msg(skb), true);
>>> -            item = (struct distr_item *)msg_data(buf_msg(skb));
>>> +            hdr = buf_msg(skb);
>>> +            msg_set_bc_ack_invalid(hdr, true);
>>> +            msg_set_bulk(hdr);
>>> +            msg_set_non_legacy(hdr);
>>> +            item = (struct distr_item *)msg_data(hdr);
>>>            }
>>>              /* Pack publication into message: */
>>> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net,
>>> struct sk_buff_head *list,
>>>            }
>>>        }
>>>        if (skb) {
>>> -        msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
>>> +        hdr = buf_msg(skb);
>>> +        msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>>>            skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>>>            __skb_queue_tail(list, skb);
>>>        }
>>> +    hdr = buf_msg(skb_peek_tail(list));
>>> +    msg_set_last_bulk(hdr);
>>> +    msg_set_named_seqno(hdr, seqno);
>>>    }
>>>      /**
>>>     * tipc_named_node_up - tell specified node about all publications
>>> by this node
>>>     */
>>> -void tipc_named_node_up(struct net *net, u32 dnode)
>>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>>>    {
>>>        struct name_table *nt = tipc_name_table(net);
>>> +    struct tipc_net *tn = tipc_net(net);
>>>        struct sk_buff_head head;
>>> +    u16 seqno;
>>>          __skb_queue_head_init(&head);
>>> +    spin_lock_bh(&tn->nametbl_lock);
>>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>>> +        nt->rc_dests++;
>>> +    seqno = nt->snd_nxt;
>>> +    spin_unlock_bh(&tn->nametbl_lock);
>>>          read_lock_bh(&nt->cluster_scope_lock);
>>> -    named_distribute(net, &head, dnode, &nt->cluster_scope);
>>> +    named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>>>        tipc_node_xmit(net, &head, dnode, 0);
>>>        read_unlock_bh(&nt->cluster_scope_lock);
>>>    }
>>> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net
>>> *net, u32 addr)
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>    }
>>>    -void tipc_publ_notify(struct net *net, struct list_head
>>> *nsub_list, u32 addr)
>>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> +              u32 addr, u16 capabilities)
>>>    {
>>> +    struct name_table *nt = tipc_name_table(net);
>>> +    struct tipc_net *tn = tipc_net(net);
>>> +
>>>        struct publication *publ, *tmp;
>>>          list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>>>            tipc_publ_purge(net, publ, addr);
>>>        tipc_dist_queue_purge(net, addr);
>>> +    spin_lock_bh(&tn->nametbl_lock);
>>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>>> +        nt->rc_dests--;
>>> +    spin_unlock_bh(&tn->nametbl_lock);
>>>    }
>>>      /**
>>> @@ -295,29 +320,55 @@ static bool tipc_update_nametbl(struct net
>>> *net, struct distr_item *i,
>>>        return false;
>>>    }
>>>    +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
>>> +                   u16 *rcv_nxt, bool *open)
>>> +{
>>> +    struct sk_buff *skb, *tmp;
>>> +    struct tipc_msg *hdr;
>>> +    u16 seqno;
>>> +
>>> +    skb_queue_walk_safe(namedq, skb, tmp) {
>>> +        skb_linearize(skb);
>>> +        hdr = buf_msg(skb);
>>> +        seqno = msg_named_seqno(hdr);
>>> +        if (msg_is_last_bulk(hdr)) {
>>> +            *rcv_nxt = seqno;
>>> +            *open = true;
>>> +        }
>>> +        if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
>>> +            __skb_unlink(skb, namedq);
>>> +            return skb;
>>> +        }
>>> +
>>> +        if (*open && (*rcv_nxt == seqno)) {
>>> +            (*rcv_nxt)++;
>>> +            __skb_unlink(skb, namedq);
>>> +            return skb;
>>> +        }
>>> +    }
>>> +    return NULL;
>>> +}
>>> +
>>>    /**
>>>     * tipc_named_rcv - process name table update messages sent by
>>> another node
>>>     */
>>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>>> +            u16 *rcv_nxt, bool *open)
>>>    {
>>> -    struct tipc_net *tn = net_generic(net, tipc_net_id);
>>> -    struct tipc_msg *msg;
>>> +    struct tipc_net *tn = tipc_net(net);
>>>        struct distr_item *item;
>>> -    uint count;
>>> -    u32 node;
>>> +    struct tipc_msg *hdr;
>>>        struct sk_buff *skb;
>>> -    int mtype;
>>> +    u32 count, node = 0;
>>>          spin_lock_bh(&tn->nametbl_lock);
>>> -    for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>>> -        skb_linearize(skb);
>>> -        msg = buf_msg(skb);
>>> -        mtype = msg_type(msg);
>>> -        item = (struct distr_item *)msg_data(msg);
>>> -        count = msg_data_sz(msg) / ITEM_SIZE;
>>> -        node = msg_orignode(msg);
>>> +    while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>>> +        hdr = buf_msg(skb);
>>> +        node = msg_orignode(hdr);
>>> +        item = (struct distr_item *)msg_data(hdr);
>>> +        count = msg_data_sz(hdr) / ITEM_SIZE;
>>>            while (count--) {
>>> -            tipc_update_nametbl(net, item, node, mtype);
>>> +            tipc_update_nametbl(net, item, node, msg_type(hdr));
>>>                item++;
>>>            }
>>>            kfree_skb(skb);
>>> @@ -345,6 +396,6 @@ void tipc_named_reinit(struct net *net)
>>>            publ->node = self;
>>>        list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>>            publ->node = self;
>>> -
>>> +    nt->rc_dests = 0;
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>    }
>>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>>> index 63fc73e0fa6c..092323158f06 100644
>>> --- a/net/tipc/name_distr.h
>>> +++ b/net/tipc/name_distr.h
>>> @@ -67,11 +67,14 @@ struct distr_item {
>>>        __be32 key;
>>>    };
>>>    +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>>    struct sk_buff *tipc_named_publish(struct net *net, struct
>>> publication *publ);
>>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct
>>> publication *publ);
>>> -void tipc_named_node_up(struct net *net, u32 dnode);
>>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>>> +            u16 *rcv_nxt, bool *open);
>>>    void tipc_named_reinit(struct net *net);
>>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> u32 addr);
>>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> +              u32 addr, u16 capabilities);
>>>      #endif
>>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>>> index 359b2bc888cf..2ac33d32edc2 100644
>>> --- a/net/tipc/name_table.c
>>> +++ b/net/tipc/name_table.c
>>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct
>>> net *net, u32 type, u32 lower,
>>>        struct tipc_net *tn = tipc_net(net);
>>>        struct publication *p = NULL;
>>>        struct sk_buff *skb = NULL;
>>> +    u32 rc_dests;
>>>          spin_lock_bh(&tn->nametbl_lock);
>>>    @@ -743,12 +744,14 @@ struct publication
>>> *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>>            nt->local_publ_count++;
>>>            skb = tipc_named_publish(net, p);
>>>        }
>>> +    rc_dests = nt->rc_dests;
>>>    exit:
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>          if (skb)
>>> -        tipc_node_broadcast(net, skb);
>>> +        tipc_node_broadcast(net, skb, rc_dests);
>>>        return p;
>>> +
>>>    }
>>>      /**
>>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32
>>> type, u32 lower,
>>>        u32 self = tipc_own_addr(net);
>>>        struct sk_buff *skb = NULL;
>>>        struct publication *p;
>>> +    u32 rc_dests;
>>>          spin_lock_bh(&tn->nametbl_lock);
>>>    @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net,
>>> u32 type, u32 lower,
>>>            pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>>                   type, lower, upper, key);
>>>        }
>>> +    rc_dests = nt->rc_dests;
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>          if (skb) {
>>> -        tipc_node_broadcast(net, skb);
>>> +        tipc_node_broadcast(net, skb, rc_dests);
>>>            return 1;
>>>        }
>>>        return 0;
>>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>>> index 728bc7016c38..8064e1986e2c 100644
>>> --- a/net/tipc/name_table.h
>>> +++ b/net/tipc/name_table.h
>>> @@ -106,6 +106,8 @@ struct name_table {
>>>        struct list_head cluster_scope;
>>>        rwlock_t cluster_scope_lock;
>>>        u32 local_publ_count;
>>> +    u32 rc_dests;
>>> +    u32 snd_nxt;
>>>    };
>>>      int tipc_nl_name_table_dump(struct sk_buff *skb, struct
>>> netlink_callback *cb);
>>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>>> index a4c2816c3746..030a51c4d1fa 100644
>>> --- a/net/tipc/node.c
>>> +++ b/net/tipc/node.c
>>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>>        struct sk_buff_head arrvq;
>>>        struct sk_buff_head inputq2;
>>>        struct sk_buff_head namedq;
>>> +    u16 named_rcv_nxt;
>>> +    bool named_open;
>>>    };
>>>      /**
>>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct
>>> tipc_node *n)
>>>        write_unlock_bh(&n->lock);
>>>          if (flags & TIPC_NOTIFY_NODE_DOWN)
>>> -        tipc_publ_notify(net, publ_list, addr);
>>> +        tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>>          if (flags & TIPC_NOTIFY_NODE_UP)
>>> -        tipc_named_node_up(net, addr);
>>> +        tipc_named_node_up(net, addr, n->capabilities);
>>>          if (flags & TIPC_NOTIFY_LINK_UP) {
>>>            tipc_mon_peer_up(net, addr, bearer_id);
>>> @@ -1483,6 +1485,7 @@ static void node_lost_contact(struct tipc_node *n,
>>>          /* Clean up broadcast state */
>>>        tipc_bcast_remove_peer(n->net, n->bc_entry.link);
>>> +    __skb_queue_purge(&n->bc_entry.namedq);
>>>          /* Abort any ongoing link failover */
>>>        for (i = 0; i < MAX_BEARERS; i++) {
>>> @@ -1729,12 +1732,23 @@ int tipc_node_distr_xmit(struct net *net,
>>> struct sk_buff_head *xmitq)
>>>        return 0;
>>>    }
>>>    -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int
>>> rc_dests)
>>>    {
>>> +    struct sk_buff_head xmitq;
>>>        struct sk_buff *txskb;
>>>        struct tipc_node *n;
>>> +    u16 dummy;
>>>        u32 dst;
>>>    +    /* Use broadcast if all nodes support it */
>>> +    if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>>> +        __skb_queue_head_init(&xmitq);
>>> +        __skb_queue_tail(&xmitq, skb);
>>> +        tipc_bcast_xmit(net, &xmitq, &dummy);
>>> +        return;
>>> +    }
>>> +
>>> +    /* Otherwise use legacy replicast method */
>>>        rcu_read_lock();
>>>        list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>>            dst = n->addr;
>>> @@ -1749,7 +1763,6 @@ void tipc_node_broadcast(struct net *net,
>>> struct sk_buff *skb)
>>>            tipc_node_xmit_skb(net, txskb, dst, 0);
>>>        }
>>>        rcu_read_unlock();
>>> -
>>>        kfree_skb(skb);
>>>    }
>>>    @@ -1844,7 +1857,9 @@ static void tipc_node_bc_rcv(struct net *net,
>>> struct sk_buff *skb, int bearer_id
>>>          /* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>>        if (!skb_queue_empty(&n->bc_entry.namedq))
>>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>>> +                   &n->bc_entry.named_rcv_nxt,
>>> +                   &n->bc_entry.named_open);
>>>          /* If reassembly or retransmission failure => reset all links
>>> to peer */
>>>        if (rc & TIPC_LINK_DOWN_EVT)
>>> @@ -2114,7 +2129,9 @@ void tipc_rcv(struct net *net, struct sk_buff
>>> *skb, struct tipc_bearer *b)
>>>            tipc_node_link_down(n, bearer_id, false);
>>>          if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>>> +                   &n->bc_entry.named_rcv_nxt,
>>> +                   &n->bc_entry.named_open);
>>>          if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>>            tipc_node_mcast_rcv(n);
>>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>>> index a6803b449a2c..9f6f13f1604f 100644
>>> --- a/net/tipc/node.h
>>> +++ b/net/tipc/node.h
>>> @@ -55,7 +55,8 @@ enum {
>>>        TIPC_MCAST_RBCTL      = (1 << 7),
>>>        TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>>        TIPC_TUNNEL_ENHANCED  = (1 << 9),
>>> -    TIPC_NAGLE            = (1 << 10)
>>> +    TIPC_NAGLE            = (1 << 10),
>>> +    TIPC_NAMED_BCAST      = (1 << 11)
>>>    };
>>>      #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>>> @@ -68,7 +69,8 @@ enum {
>>>                    TIPC_MCAST_RBCTL       |   \
>>>                    TIPC_GAP_ACK_BLOCK     |   \
>>>                    TIPC_TUNNEL_ENHANCED   |   \
>>> -                TIPC_NAGLE)
>>> +                TIPC_NAGLE             |   \
>>> +                TIPC_NAMED_BCAST)
>>>      #define INVALID_BEARER_ID -1
>>>    @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct
>>> sk_buff *skb, u32 dest,
>>>                   u32 selector);
>>>    void tipc_node_subscribe(struct net *net, struct list_head *subscr,
>>> u32 addr);
>>>    void tipc_node_unsubscribe(struct net *net, struct list_head
>>> *subscr, u32 addr);
>>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int
>>> rc_dests);
>>>    int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32
>>> peer_port);
>>>    void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>>    int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool
>>> connected);
>>
>>
>> _______________________________________________
>> tipc-discussion mailing list
>> tip...@li...
>> https://lists.sourceforge.net/lists/listinfo/tipc-discussion
>>
>
>
> _______________________________________________
> tipc-discussion mailing list
> tip...@li...
> https://lists.sourceforge.net/lists/listinfo/tipc-discussion

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang H. Le <hoa...@de...>]

Thanks Jon!
Hoang
-----Original Message-----
From: Jon Maloy <jm...@re...> 
Sent: Monday, June 8, 2020 8:57 AM
To: Hoang Huu Le <hoa...@de...>; tip...@li...; tipc-dek <tip...@de...>
Subject: Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

In case you are not aware of this, check this link before you send 
anything to net-next.

http://vger.kernel.org/~davem/net-next.html

///jon



On 6/7/20 9:50 PM, Hoang Huu Le wrote:
>
> -----Original Message-----
> From: Jon Maloy <jm...@re...>
> Sent: Monday, June 8, 2020 2:14 AM
> To: tip...@li...
> Subject: Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast
>
>
>
> On 6/7/20 3:03 PM, Jon Maloy wrote:
>>
>> On 6/7/20 12:24 AM, Hoang Huu Le wrote:
>>> Currently, updating binding table (add service binding to
>>> name table/withdraw a service binding) is being sent over replicast.
>>> However, if we are scaling up clusters to > 100 nodes/containers this
>>> method is less affection because of looping through nodes in a
>>> cluster one
>>> by one.
>>>
>>> It is worth to use broadcast to update a binding service. This way, the
>>> binding table can be updated on all peer nodes in one shot.
>>>
>>> Broadcast is used when all peer nodes, as indicated by a new capability
>>> flag TIPC_NAMED_BCAST, support reception of this message type.
>>>
>>> Four problems need to be considered when introducing this feature.
>>> 1) When establishing a link to a new peer node we still update this by a
>>> unicast 'bulk' update. This may lead to race conditions, where a later
>>> broadcast publication/withdrawal bypass the 'bulk', resulting in
>>> disordered publications, or even that a withdrawal may arrive before the
>>> corresponding publication. We solve this by adding an 'is_last_bulk' bit
>>> in the last bulk messages so that it can be distinguished from all other
>>> messages. Only when this message has arrived do we open up for reception
>>> of broadcast publications/withdrawals.
>> Add a line feed between these paragraphs before you send the patch.
>> Otherwise, still acked by me.
>>
>> ///jon
> Oh, already posted... Just ignore my comment above.
> [Hoang] net-next is closed. I will re-post the patch later with your suggestion.
>
> ///jon
>>> 2) When a first legacy node is added to the cluster all distribution
>>> will switch over to use the legacy 'replicast' method, while the
>>> opposite happens when the last legacy node leaves the cluster. This
>>> entails another risk of message disordering that has to be handled. We
>>> solve this by adding a sequence number to the broadcast/replicast
>>> messages, so that disordering can be discovered and corrected. Note
>>> however that we don't need to consider potential message loss or
>>> duplication at this protocol level.
>>> 3) Bulk messages don't contain any sequence numbers, and will always
>>> arrive in order. Hence we must exempt those from the sequence number
>>> control and deliver them unconditionally. We solve this by adding a new
>>> 'is_bulk' bit in those messages so that they can be recognized.
>>> 4) Legacy messages, which don't contain any new bits or sequence
>>> numbers, but neither can arrive out of order, also need to be exempt
>>> from the initial synchronization and sequence number check, and
>>> delivered unconditionally. Therefore, we add another 'is_not_legacy' bit
>>> to all new messages so that those can be distinguished from legacy
>>> messages and the latter delivered directly.
>>>
>>> Signed-off-by: Hoang Huu Le <hoa...@de...>
>>> Acked-by: Jon Maloy <jm...@re...>
>>> ---
>>>    net/tipc/bcast.c      |   6 +--
>>>    net/tipc/bcast.h      |   4 +-
>>>    net/tipc/link.c       |   2 +-
>>>    net/tipc/msg.h        |  40 ++++++++++++++++
>>>    net/tipc/name_distr.c | 109 +++++++++++++++++++++++++++++++-----------
>>>    net/tipc/name_distr.h |   9 ++--
>>>    net/tipc/name_table.c |   9 +++-
>>>    net/tipc/name_table.h |   2 +
>>>    net/tipc/node.c       |  29 ++++++++---
>>>    net/tipc/node.h       |   8 ++--
>>>    10 files changed, 170 insertions(+), 48 deletions(-)
>>>
>>> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
>>> index 383f87bc1061..940d176e0e87 100644
>>> --- a/net/tipc/bcast.c
>>> +++ b/net/tipc/bcast.c
>>> @@ -250,8 +250,8 @@ static void tipc_bcast_select_xmit_method(struct
>>> net *net, int dests,
>>>     * Consumes the buffer chain.
>>>     * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
>>>     */
>>> -static int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>> -               u16 *cong_link_cnt)
>>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>> +            u16 *cong_link_cnt)
>>>    {
>>>        struct tipc_link *l = tipc_bc_sndlink(net);
>>>        struct sk_buff_head xmitq;
>>> @@ -752,7 +752,7 @@ void tipc_nlist_purge(struct tipc_nlist *nl)
>>>        nl->local = false;
>>>    }
>>>    -u32 tipc_bcast_get_broadcast_mode(struct net *net)
>>> +u32 tipc_bcast_get_mode(struct net *net)
>>>    {
>>>        struct tipc_bc_base *bb = tipc_bc_base(net);
>>>    diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
>>> index 4240c95188b1..2d9352dc7b0e 100644
>>> --- a/net/tipc/bcast.h
>>> +++ b/net/tipc/bcast.h
>>> @@ -90,6 +90,8 @@ void tipc_bcast_toggle_rcast(struct net *net, bool
>>> supp);
>>>    int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>>                struct tipc_mc_method *method, struct tipc_nlist *dests,
>>>                u16 *cong_link_cnt);
>>> +int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
>>> +            u16 *cong_link_cnt);
>>>    int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct
>>> sk_buff *skb);
>>>    void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
>>>                struct tipc_msg *hdr);
>>> @@ -101,7 +103,7 @@ int tipc_nl_add_bc_link(struct net *net, struct
>>> tipc_nl_msg *msg,
>>>    int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]);
>>>    int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l);
>>>    -u32 tipc_bcast_get_broadcast_mode(struct net *net);
>>> +u32 tipc_bcast_get_mode(struct net *net);
>>>    u32 tipc_bcast_get_broadcast_ratio(struct net *net);
>>>      void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head
>>> *defq,
>>> diff --git a/net/tipc/link.c b/net/tipc/link.c
>>> index ee3b8d0576b8..eac89a3e22ce 100644
>>> --- a/net/tipc/link.c
>>> +++ b/net/tipc/link.c
>>> @@ -2745,7 +2745,7 @@ int tipc_nl_add_bc_link(struct net *net, struct
>>> tipc_nl_msg *msg,
>>>        void *hdr;
>>>        struct nlattr *attrs;
>>>        struct nlattr *prop;
>>> -    u32 bc_mode = tipc_bcast_get_broadcast_mode(net);
>>> +    u32 bc_mode = tipc_bcast_get_mode(net);
>>>        u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);
>>>          if (!bcl)
>>> diff --git a/net/tipc/msg.h b/net/tipc/msg.h
>>> index 58660d56bc83..65119e81ff0c 100644
>>> --- a/net/tipc/msg.h
>>> +++ b/net/tipc/msg.h
>>> @@ -438,6 +438,36 @@ static inline void msg_set_errcode(struct
>>> tipc_msg *m, u32 err)
>>>        msg_set_bits(m, 1, 25, 0xf, err);
>>>    }
>>>    +static inline void msg_set_bulk(struct tipc_msg *m)
>>> +{
>>> +    msg_set_bits(m, 1, 28, 0x1, 1);
>>> +}
>>> +
>>> +static inline u32 msg_is_bulk(struct tipc_msg *m)
>>> +{
>>> +    return msg_bits(m, 1, 28, 0x1);
>>> +}
>>> +
>>> +static inline void msg_set_last_bulk(struct tipc_msg *m)
>>> +{
>>> +    msg_set_bits(m, 1, 27, 0x1, 1);
>>> +}
>>> +
>>> +static inline u32 msg_is_last_bulk(struct tipc_msg *m)
>>> +{
>>> +    return msg_bits(m, 1, 27, 0x1);
>>> +}
>>> +
>>> +static inline void msg_set_non_legacy(struct tipc_msg *m)
>>> +{
>>> +    msg_set_bits(m, 1, 26, 0x1, 1);
>>> +}
>>> +
>>> +static inline u32 msg_is_legacy(struct tipc_msg *m)
>>> +{
>>> +    return !msg_bits(m, 1, 26, 0x1);
>>> +}
>>> +
>>>    static inline u32 msg_reroute_cnt(struct tipc_msg *m)
>>>    {
>>>        return msg_bits(m, 1, 21, 0xf);
>>> @@ -567,6 +597,16 @@ static inline void msg_set_origport(struct
>>> tipc_msg *m, u32 p)
>>>        msg_set_word(m, 4, p);
>>>    }
>>>    +static inline u16 msg_named_seqno(struct tipc_msg *m)
>>> +{
>>> +    return msg_bits(m, 4, 0, 0xffff);
>>> +}
>>> +
>>> +static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
>>> +{
>>> +    msg_set_bits(m, 4, 0, 0xffff, n);
>>> +}
>>> +
>>>    static inline u32 msg_destport(struct tipc_msg *m)
>>>    {
>>>        return msg_word(m, 5);
>>> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
>>> index 5feaf3b67380..481d480609f0 100644
>>> --- a/net/tipc/name_distr.c
>>> +++ b/net/tipc/name_distr.c
>>> @@ -102,7 +102,8 @@ struct sk_buff *tipc_named_publish(struct net
>>> *net, struct publication *publ)
>>>            pr_warn("Publication distribution failure\n");
>>>            return NULL;
>>>        }
>>> -
>>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>>> +    msg_set_non_legacy(buf_msg(skb));
>>>        item = (struct distr_item *)msg_data(buf_msg(skb));
>>>        publ_to_item(item, publ);
>>>        return skb;
>>> @@ -114,8 +115,8 @@ struct sk_buff *tipc_named_publish(struct net
>>> *net, struct publication *publ)
>>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct
>>> publication *publ)
>>>    {
>>>        struct name_table *nt = tipc_name_table(net);
>>> -    struct sk_buff *buf;
>>>        struct distr_item *item;
>>> +    struct sk_buff *skb;
>>>          write_lock_bh(&nt->cluster_scope_lock);
>>>        list_del(&publ->binding_node);
>>> @@ -123,15 +124,16 @@ struct sk_buff *tipc_named_withdraw(struct net
>>> *net, struct publication *publ)
>>>        if (publ->scope == TIPC_NODE_SCOPE)
>>>            return NULL;
>>>    -    buf = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>>> -    if (!buf) {
>>> +    skb = named_prepare_buf(net, WITHDRAWAL, ITEM_SIZE, 0);
>>> +    if (!skb) {
>>>            pr_warn("Withdrawal distribution failure\n");
>>>            return NULL;
>>>        }
>>> -
>>> -    item = (struct distr_item *)msg_data(buf_msg(buf));
>>> +    msg_set_named_seqno(buf_msg(skb), nt->snd_nxt++);
>>> +    msg_set_non_legacy(buf_msg(skb));
>>> +    item = (struct distr_item *)msg_data(buf_msg(skb));
>>>        publ_to_item(item, publ);
>>> -    return buf;
>>> +    return skb;
>>>    }
>>>      /**
>>> @@ -141,7 +143,7 @@ struct sk_buff *tipc_named_withdraw(struct net
>>> *net, struct publication *publ)
>>>     * @pls: linked list of publication items to be packed into buffer
>>> chain
>>>     */
>>>    static void named_distribute(struct net *net, struct sk_buff_head
>>> *list,
>>> -                 u32 dnode, struct list_head *pls)
>>> +                 u32 dnode, struct list_head *pls, u16 seqno)
>>>    {
>>>        struct publication *publ;
>>>        struct sk_buff *skb = NULL;
>>> @@ -149,6 +151,7 @@ static void named_distribute(struct net *net,
>>> struct sk_buff_head *list,
>>>        u32 msg_dsz = ((tipc_node_get_mtu(net, dnode, 0, false) -
>>> INT_H_SIZE) /
>>>                ITEM_SIZE) * ITEM_SIZE;
>>>        u32 msg_rem = msg_dsz;
>>> +    struct tipc_msg *hdr;
>>>          list_for_each_entry(publ, pls, binding_node) {
>>>            /* Prepare next buffer: */
>>> @@ -159,8 +162,11 @@ static void named_distribute(struct net *net,
>>> struct sk_buff_head *list,
>>>                    pr_warn("Bulk publication failure\n");
>>>                    return;
>>>                }
>>> -            msg_set_bc_ack_invalid(buf_msg(skb), true);
>>> -            item = (struct distr_item *)msg_data(buf_msg(skb));
>>> +            hdr = buf_msg(skb);
>>> +            msg_set_bc_ack_invalid(hdr, true);
>>> +            msg_set_bulk(hdr);
>>> +            msg_set_non_legacy(hdr);
>>> +            item = (struct distr_item *)msg_data(hdr);
>>>            }
>>>              /* Pack publication into message: */
>>> @@ -176,24 +182,35 @@ static void named_distribute(struct net *net,
>>> struct sk_buff_head *list,
>>>            }
>>>        }
>>>        if (skb) {
>>> -        msg_set_size(buf_msg(skb), INT_H_SIZE + (msg_dsz - msg_rem));
>>> +        hdr = buf_msg(skb);
>>> +        msg_set_size(hdr, INT_H_SIZE + (msg_dsz - msg_rem));
>>>            skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>>>            __skb_queue_tail(list, skb);
>>>        }
>>> +    hdr = buf_msg(skb_peek_tail(list));
>>> +    msg_set_last_bulk(hdr);
>>> +    msg_set_named_seqno(hdr, seqno);
>>>    }
>>>      /**
>>>     * tipc_named_node_up - tell specified node about all publications
>>> by this node
>>>     */
>>> -void tipc_named_node_up(struct net *net, u32 dnode)
>>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities)
>>>    {
>>>        struct name_table *nt = tipc_name_table(net);
>>> +    struct tipc_net *tn = tipc_net(net);
>>>        struct sk_buff_head head;
>>> +    u16 seqno;
>>>          __skb_queue_head_init(&head);
>>> +    spin_lock_bh(&tn->nametbl_lock);
>>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>>> +        nt->rc_dests++;
>>> +    seqno = nt->snd_nxt;
>>> +    spin_unlock_bh(&tn->nametbl_lock);
>>>          read_lock_bh(&nt->cluster_scope_lock);
>>> -    named_distribute(net, &head, dnode, &nt->cluster_scope);
>>> +    named_distribute(net, &head, dnode, &nt->cluster_scope, seqno);
>>>        tipc_node_xmit(net, &head, dnode, 0);
>>>        read_unlock_bh(&nt->cluster_scope_lock);
>>>    }
>>> @@ -245,13 +262,21 @@ static void tipc_dist_queue_purge(struct net
>>> *net, u32 addr)
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>    }
>>>    -void tipc_publ_notify(struct net *net, struct list_head
>>> *nsub_list, u32 addr)
>>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> +              u32 addr, u16 capabilities)
>>>    {
>>> +    struct name_table *nt = tipc_name_table(net);
>>> +    struct tipc_net *tn = tipc_net(net);
>>> +
>>>        struct publication *publ, *tmp;
>>>          list_for_each_entry_safe(publ, tmp, nsub_list, binding_node)
>>>            tipc_publ_purge(net, publ, addr);
>>>        tipc_dist_queue_purge(net, addr);
>>> +    spin_lock_bh(&tn->nametbl_lock);
>>> +    if (!(capabilities & TIPC_NAMED_BCAST))
>>> +        nt->rc_dests--;
>>> +    spin_unlock_bh(&tn->nametbl_lock);
>>>    }
>>>      /**
>>> @@ -295,29 +320,55 @@ static bool tipc_update_nametbl(struct net
>>> *net, struct distr_item *i,
>>>        return false;
>>>    }
>>>    +struct sk_buff *tipc_named_dequeue(struct sk_buff_head *namedq,
>>> +                   u16 *rcv_nxt, bool *open)
>>> +{
>>> +    struct sk_buff *skb, *tmp;
>>> +    struct tipc_msg *hdr;
>>> +    u16 seqno;
>>> +
>>> +    skb_queue_walk_safe(namedq, skb, tmp) {
>>> +        skb_linearize(skb);
>>> +        hdr = buf_msg(skb);
>>> +        seqno = msg_named_seqno(hdr);
>>> +        if (msg_is_last_bulk(hdr)) {
>>> +            *rcv_nxt = seqno;
>>> +            *open = true;
>>> +        }
>>> +        if (msg_is_bulk(hdr) || msg_is_legacy(hdr)) {
>>> +            __skb_unlink(skb, namedq);
>>> +            return skb;
>>> +        }
>>> +
>>> +        if (*open && (*rcv_nxt == seqno)) {
>>> +            (*rcv_nxt)++;
>>> +            __skb_unlink(skb, namedq);
>>> +            return skb;
>>> +        }
>>> +    }
>>> +    return NULL;
>>> +}
>>> +
>>>    /**
>>>     * tipc_named_rcv - process name table update messages sent by
>>> another node
>>>     */
>>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *inputq)
>>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>>> +            u16 *rcv_nxt, bool *open)
>>>    {
>>> -    struct tipc_net *tn = net_generic(net, tipc_net_id);
>>> -    struct tipc_msg *msg;
>>> +    struct tipc_net *tn = tipc_net(net);
>>>        struct distr_item *item;
>>> -    uint count;
>>> -    u32 node;
>>> +    struct tipc_msg *hdr;
>>>        struct sk_buff *skb;
>>> -    int mtype;
>>> +    u32 count, node = 0;
>>>          spin_lock_bh(&tn->nametbl_lock);
>>> -    for (skb = skb_dequeue(inputq); skb; skb = skb_dequeue(inputq)) {
>>> -        skb_linearize(skb);
>>> -        msg = buf_msg(skb);
>>> -        mtype = msg_type(msg);
>>> -        item = (struct distr_item *)msg_data(msg);
>>> -        count = msg_data_sz(msg) / ITEM_SIZE;
>>> -        node = msg_orignode(msg);
>>> +    while ((skb = tipc_named_dequeue(namedq, rcv_nxt, open))) {
>>> +        hdr = buf_msg(skb);
>>> +        node = msg_orignode(hdr);
>>> +        item = (struct distr_item *)msg_data(hdr);
>>> +        count = msg_data_sz(hdr) / ITEM_SIZE;
>>>            while (count--) {
>>> -            tipc_update_nametbl(net, item, node, mtype);
>>> +            tipc_update_nametbl(net, item, node, msg_type(hdr));
>>>                item++;
>>>            }
>>>            kfree_skb(skb);
>>> @@ -345,6 +396,6 @@ void tipc_named_reinit(struct net *net)
>>>            publ->node = self;
>>>        list_for_each_entry_rcu(publ, &nt->cluster_scope, binding_node)
>>>            publ->node = self;
>>> -
>>> +    nt->rc_dests = 0;
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>    }
>>> diff --git a/net/tipc/name_distr.h b/net/tipc/name_distr.h
>>> index 63fc73e0fa6c..092323158f06 100644
>>> --- a/net/tipc/name_distr.h
>>> +++ b/net/tipc/name_distr.h
>>> @@ -67,11 +67,14 @@ struct distr_item {
>>>        __be32 key;
>>>    };
>>>    +void tipc_named_bcast(struct net *net, struct sk_buff *skb);
>>>    struct sk_buff *tipc_named_publish(struct net *net, struct
>>> publication *publ);
>>>    struct sk_buff *tipc_named_withdraw(struct net *net, struct
>>> publication *publ);
>>> -void tipc_named_node_up(struct net *net, u32 dnode);
>>> -void tipc_named_rcv(struct net *net, struct sk_buff_head *msg_queue);
>>> +void tipc_named_node_up(struct net *net, u32 dnode, u16 capabilities);
>>> +void tipc_named_rcv(struct net *net, struct sk_buff_head *namedq,
>>> +            u16 *rcv_nxt, bool *open);
>>>    void tipc_named_reinit(struct net *net);
>>> -void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> u32 addr);
>>> +void tipc_publ_notify(struct net *net, struct list_head *nsub_list,
>>> +              u32 addr, u16 capabilities);
>>>      #endif
>>> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
>>> index 359b2bc888cf..2ac33d32edc2 100644
>>> --- a/net/tipc/name_table.c
>>> +++ b/net/tipc/name_table.c
>>> @@ -729,6 +729,7 @@ struct publication *tipc_nametbl_publish(struct
>>> net *net, u32 type, u32 lower,
>>>        struct tipc_net *tn = tipc_net(net);
>>>        struct publication *p = NULL;
>>>        struct sk_buff *skb = NULL;
>>> +    u32 rc_dests;
>>>          spin_lock_bh(&tn->nametbl_lock);
>>>    @@ -743,12 +744,14 @@ struct publication
>>> *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>>>            nt->local_publ_count++;
>>>            skb = tipc_named_publish(net, p);
>>>        }
>>> +    rc_dests = nt->rc_dests;
>>>    exit:
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>          if (skb)
>>> -        tipc_node_broadcast(net, skb);
>>> +        tipc_node_broadcast(net, skb, rc_dests);
>>>        return p;
>>> +
>>>    }
>>>      /**
>>> @@ -762,6 +765,7 @@ int tipc_nametbl_withdraw(struct net *net, u32
>>> type, u32 lower,
>>>        u32 self = tipc_own_addr(net);
>>>        struct sk_buff *skb = NULL;
>>>        struct publication *p;
>>> +    u32 rc_dests;
>>>          spin_lock_bh(&tn->nametbl_lock);
>>>    @@ -775,10 +779,11 @@ int tipc_nametbl_withdraw(struct net *net,
>>> u32 type, u32 lower,
>>>            pr_err("Failed to remove local publication {%u,%u,%u}/%u\n",
>>>                   type, lower, upper, key);
>>>        }
>>> +    rc_dests = nt->rc_dests;
>>>        spin_unlock_bh(&tn->nametbl_lock);
>>>          if (skb) {
>>> -        tipc_node_broadcast(net, skb);
>>> +        tipc_node_broadcast(net, skb, rc_dests);
>>>            return 1;
>>>        }
>>>        return 0;
>>> diff --git a/net/tipc/name_table.h b/net/tipc/name_table.h
>>> index 728bc7016c38..8064e1986e2c 100644
>>> --- a/net/tipc/name_table.h
>>> +++ b/net/tipc/name_table.h
>>> @@ -106,6 +106,8 @@ struct name_table {
>>>        struct list_head cluster_scope;
>>>        rwlock_t cluster_scope_lock;
>>>        u32 local_publ_count;
>>> +    u32 rc_dests;
>>> +    u32 snd_nxt;
>>>    };
>>>      int tipc_nl_name_table_dump(struct sk_buff *skb, struct
>>> netlink_callback *cb);
>>> diff --git a/net/tipc/node.c b/net/tipc/node.c
>>> index a4c2816c3746..030a51c4d1fa 100644
>>> --- a/net/tipc/node.c
>>> +++ b/net/tipc/node.c
>>> @@ -75,6 +75,8 @@ struct tipc_bclink_entry {
>>>        struct sk_buff_head arrvq;
>>>        struct sk_buff_head inputq2;
>>>        struct sk_buff_head namedq;
>>> +    u16 named_rcv_nxt;
>>> +    bool named_open;
>>>    };
>>>      /**
>>> @@ -396,10 +398,10 @@ static void tipc_node_write_unlock(struct
>>> tipc_node *n)
>>>        write_unlock_bh(&n->lock);
>>>          if (flags & TIPC_NOTIFY_NODE_DOWN)
>>> -        tipc_publ_notify(net, publ_list, addr);
>>> +        tipc_publ_notify(net, publ_list, addr, n->capabilities);
>>>          if (flags & TIPC_NOTIFY_NODE_UP)
>>> -        tipc_named_node_up(net, addr);
>>> +        tipc_named_node_up(net, addr, n->capabilities);
>>>          if (flags & TIPC_NOTIFY_LINK_UP) {
>>>            tipc_mon_peer_up(net, addr, bearer_id);
>>> @@ -1483,6 +1485,7 @@ static void node_lost_contact(struct tipc_node *n,
>>>          /* Clean up broadcast state */
>>>        tipc_bcast_remove_peer(n->net, n->bc_entry.link);
>>> +    __skb_queue_purge(&n->bc_entry.namedq);
>>>          /* Abort any ongoing link failover */
>>>        for (i = 0; i < MAX_BEARERS; i++) {
>>> @@ -1729,12 +1732,23 @@ int tipc_node_distr_xmit(struct net *net,
>>> struct sk_buff_head *xmitq)
>>>        return 0;
>>>    }
>>>    -void tipc_node_broadcast(struct net *net, struct sk_buff *skb)
>>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int
>>> rc_dests)
>>>    {
>>> +    struct sk_buff_head xmitq;
>>>        struct sk_buff *txskb;
>>>        struct tipc_node *n;
>>> +    u16 dummy;
>>>        u32 dst;
>>>    +    /* Use broadcast if all nodes support it */
>>> +    if (!rc_dests && tipc_bcast_get_mode(net) != BCLINK_MODE_RCAST) {
>>> +        __skb_queue_head_init(&xmitq);
>>> +        __skb_queue_tail(&xmitq, skb);
>>> +        tipc_bcast_xmit(net, &xmitq, &dummy);
>>> +        return;
>>> +    }
>>> +
>>> +    /* Otherwise use legacy replicast method */
>>>        rcu_read_lock();
>>>        list_for_each_entry_rcu(n, tipc_nodes(net), list) {
>>>            dst = n->addr;
>>> @@ -1749,7 +1763,6 @@ void tipc_node_broadcast(struct net *net,
>>> struct sk_buff *skb)
>>>            tipc_node_xmit_skb(net, txskb, dst, 0);
>>>        }
>>>        rcu_read_unlock();
>>> -
>>>        kfree_skb(skb);
>>>    }
>>>    @@ -1844,7 +1857,9 @@ static void tipc_node_bc_rcv(struct net *net,
>>> struct sk_buff *skb, int bearer_id
>>>          /* Handle NAME_DISTRIBUTOR messages sent from 1.7 nodes */
>>>        if (!skb_queue_empty(&n->bc_entry.namedq))
>>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>>> +                   &n->bc_entry.named_rcv_nxt,
>>> +                   &n->bc_entry.named_open);
>>>          /* If reassembly or retransmission failure => reset all links
>>> to peer */
>>>        if (rc & TIPC_LINK_DOWN_EVT)
>>> @@ -2114,7 +2129,9 @@ void tipc_rcv(struct net *net, struct sk_buff
>>> *skb, struct tipc_bearer *b)
>>>            tipc_node_link_down(n, bearer_id, false);
>>>          if (unlikely(!skb_queue_empty(&n->bc_entry.namedq)))
>>> -        tipc_named_rcv(net, &n->bc_entry.namedq);
>>> +        tipc_named_rcv(net, &n->bc_entry.namedq,
>>> +                   &n->bc_entry.named_rcv_nxt,
>>> +                   &n->bc_entry.named_open);
>>>          if (unlikely(!skb_queue_empty(&n->bc_entry.inputq1)))
>>>            tipc_node_mcast_rcv(n);
>>> diff --git a/net/tipc/node.h b/net/tipc/node.h
>>> index a6803b449a2c..9f6f13f1604f 100644
>>> --- a/net/tipc/node.h
>>> +++ b/net/tipc/node.h
>>> @@ -55,7 +55,8 @@ enum {
>>>        TIPC_MCAST_RBCTL      = (1 << 7),
>>>        TIPC_GAP_ACK_BLOCK    = (1 << 8),
>>>        TIPC_TUNNEL_ENHANCED  = (1 << 9),
>>> -    TIPC_NAGLE            = (1 << 10)
>>> +    TIPC_NAGLE            = (1 << 10),
>>> +    TIPC_NAMED_BCAST      = (1 << 11)
>>>    };
>>>      #define TIPC_NODE_CAPABILITIES (TIPC_SYN_BIT           |  \
>>> @@ -68,7 +69,8 @@ enum {
>>>                    TIPC_MCAST_RBCTL       |   \
>>>                    TIPC_GAP_ACK_BLOCK     |   \
>>>                    TIPC_TUNNEL_ENHANCED   |   \
>>> -                TIPC_NAGLE)
>>> +                TIPC_NAGLE             |   \
>>> +                TIPC_NAMED_BCAST)
>>>      #define INVALID_BEARER_ID -1
>>>    @@ -101,7 +103,7 @@ int tipc_node_xmit_skb(struct net *net, struct
>>> sk_buff *skb, u32 dest,
>>>                   u32 selector);
>>>    void tipc_node_subscribe(struct net *net, struct list_head *subscr,
>>> u32 addr);
>>>    void tipc_node_unsubscribe(struct net *net, struct list_head
>>> *subscr, u32 addr);
>>> -void tipc_node_broadcast(struct net *net, struct sk_buff *skb);
>>> +void tipc_node_broadcast(struct net *net, struct sk_buff *skb, int
>>> rc_dests);
>>>    int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32
>>> peer_port);
>>>    void tipc_node_remove_conn(struct net *net, u32 dnode, u32 port);
>>>    int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool
>>> connected);
>>
>>
>> _______________________________________________
>> tipc-discussion mailing list
>> tip...@li...
>> https://lists.sourceforge.net/lists/listinfo/tipc-discussion
>>
>
>
> _______________________________________________
> tipc-discussion mailing list
> tip...@li...
> https://lists.sourceforge.net/lists/listinfo/tipc-discussion

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Jon M. <jon...@er...>]

Hi Hoang,
This is good, but you have missed the point about the synchronization problem I have been talking about.

1) A new node comes up
2) The "bulk" binding table update is sent, as a series of packets over the new unicast link. This may take some time.
3) The owner of one of the bindings in the bulk (on this node) does unbind.
4) This is sent as broadcast withdraw to all nodes, and arrives before the last packets of the unicast bulk to the newly connected node.
5) Since there is no corresponding publication in the peer node's binding table yet, the withdraw is ignored.
6) The last bulk unicasts arrive at the new peer, and the now invalid publication is added to its binding table.
7) This publication will stay there forever.

We need to find a way to synchronize so that we know that all the bulk publications are in place in the binding table before any broadcast publications/withdraws can be accepted.
Obviously, we could create a backlog queue in the name table, but I hope we can find a simpler and neater solution.

Regards
///jon

> -----Original Message-----
> From: Hoang Le <hoa...@de...>
> Sent: 13-Nov-19 02:35
> To: Jon Maloy <jon...@er...>; ma...@do...; tip...@li...
> Subject: [net-next] tipc: update a binding service via broadcast
> 
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
> 
> It is worth to use broadcast to update a binding service. Then binding
> table updates in all nodes for one shot.
> 
> The mechanism is backward compatible because of sending side changing.
> 
> Signed-off-by: Hoang Le <hoa...@de...>
> ---
>  net/tipc/bcast.c      | 13 +++++++++++++
>  net/tipc/bcast.h      |  2 ++
>  net/tipc/name_table.c |  4 ++--
>  3 files changed, 17 insertions(+), 2 deletions(-)
> 
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index f41096a759fa..18431fa897ab 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -843,3 +843,16 @@ void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
>  		__skb_queue_tail(inputq, _skb);
>  	}
>  }
> +
> +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb)
> +{
> +	struct sk_buff_head xmitq;
> +	u16 cong_link_cnt;
> +	int rc = 0;
> +
> +	__skb_queue_head_init(&xmitq);
> +	__skb_queue_tail(&xmitq, skb);
> +	rc = tipc_bcast_xmit(net, &xmitq, &cong_link_cnt);
> +	__skb_queue_purge(&xmitq);
> +	return rc;
> +}
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index dadad953e2be..a100da3800fc 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -101,6 +101,8 @@ int tipc_bclink_reset_stats(struct net *net);
>  u32 tipc_bcast_get_broadcast_mode(struct net *net);
>  u32 tipc_bcast_get_broadcast_ratio(struct net *net);
> 
> +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb);
> +
>  void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
>  			   struct sk_buff_head *inputq);
> 
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 66a65c2cdb23..9e9c61f7c999 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -633,7 +633,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>  	spin_unlock_bh(&tn->nametbl_lock);
> 
>  	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_bcast_named_publish(net, skb);
>  	return p;
>  }
> 
> @@ -664,7 +664,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>  	spin_unlock_bh(&tn->nametbl_lock);
> 
>  	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_bcast_named_publish(net, skb);
>  		return 1;
>  	}
>  	return 0;
> --
> 2.20.1

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Hoang L. <hoa...@de...>]

Hi Jon,

Please take a look at v2. The mechanism looks the same as I did before in commit:
c55c8edafa91 ("tipc: smooth change between replicast and broadcast")
However, in this case we handle only one direction:  replicast -> broadcast.
Then, it is still backward compatible.

[...]
>From ae2ee6a7064de3ec1dc2c7df2db241d22b0d129f Mon Sep 17 00:00:00 2001
From: Hoang Le <hoa...@de...>
Date: Wed, 13 Nov 2019 14:01:03 +0700
Subject: [PATCH] tipc: update a binding service via broadcast

Currently, updating binding table (add service binding to
name table/withdraw a service binding) is being sent over replicast.
However, if we are scaling up clusters to > 100 nodes/containers this
method is less affection because of looping through nodes in a cluster one
by one.

It is worth to use broadcast to update a binding service. Then binding
table updates in all nodes for one shot.

The mechanism is backward compatible because of sending side changing.

v2: resolve synchronization problem when switching from unicast to
broadcast

Signed-off-by: Hoang Le <hoa...@de...>
---
 net/tipc/bcast.c      | 13 +++++++++++++
 net/tipc/bcast.h      |  2 ++
 net/tipc/link.c       | 16 ++++++++++++++++
 net/tipc/name_distr.c |  8 ++++++++
 net/tipc/name_table.c |  9 ++++++---
 5 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
index f41096a759fa..18431fa897ab 100644
--- a/net/tipc/bcast.c
+++ b/net/tipc/bcast.c
@@ -843,3 +843,16 @@ void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
                __skb_queue_tail(inputq, _skb);
        }
 }
+
+int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb)
+{
+       struct sk_buff_head xmitq;
+       u16 cong_link_cnt;
+       int rc = 0;
+
+       __skb_queue_head_init(&xmitq);
+       __skb_queue_tail(&xmitq, skb);
+       rc = tipc_bcast_xmit(net, &xmitq, &cong_link_cnt);
+       __skb_queue_purge(&xmitq);
+       return rc;
+}
diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
index dadad953e2be..a100da3800fc 100644
--- a/net/tipc/bcast.h
+++ b/net/tipc/bcast.h
@@ -101,6 +101,8 @@ int tipc_bclink_reset_stats(struct net *net);
 u32 tipc_bcast_get_broadcast_mode(struct net *net);
 u32 tipc_bcast_get_broadcast_ratio(struct net *net);

+int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb);
+
 void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
                           struct sk_buff_head *inputq);

diff --git a/net/tipc/link.c b/net/tipc/link.c
index fb72031228c9..22f1854435df 100644
--- a/net/tipc/link.c
+++ b/net/tipc/link.c
@@ -187,6 +187,9 @@ struct tipc_link {
        struct tipc_link *bc_sndlink;
        u8 nack_state;
        bool bc_peer_is_up;
+       bool named_sync;
+       struct sk_buff_head defer_namedq;
+

        /* Statistics */
        struct tipc_stats stats;
@@ -363,6 +366,7 @@ void tipc_link_remove_bc_peer(struct tipc_link *snd_l,
        trace_tipc_link_reset(rcv_l, TIPC_DUMP_ALL, "bclink removed!");
        tipc_link_reset(rcv_l);
        rcv_l->state = LINK_RESET;
+       rcv_l->named_sync = false;
        if (!snd_l->ackers) {
                trace_tipc_link_reset(snd_l, TIPC_DUMP_ALL, "zero ackers!");
                tipc_link_reset(snd_l);
@@ -508,6 +512,7 @@ bool tipc_link_create(struct net *net, char *if_name, int bearer_id,
        __skb_queue_head_init(&l->failover_deferdq);
        skb_queue_head_init(&l->wakeupq);
        skb_queue_head_init(l->inputq);
+       __skb_queue_head_init(&l->defer_namedq);
        return true;
 }

@@ -932,6 +937,8 @@ void tipc_link_reset(struct tipc_link *l)
        l->silent_intv_cnt = 0;
        l->rst_cnt = 0;
        l->bc_peer_is_up = false;
+       l->named_sync = false;
+       __skb_queue_purge(&l->defer_namedq);
        memset(&l->mon_state, 0, sizeof(l->mon_state));
        tipc_link_reset_stats(l);
 }
@@ -1210,6 +1217,15 @@ static bool tipc_data_input(struct tipc_link *l, struct sk_buff *skb,
                return true;
        case NAME_DISTRIBUTOR:
                l->bc_rcvlink->state = LINK_ESTABLISHED;
+               if (msg_is_syn(hdr)) {
+                       l->bc_rcvlink->named_sync = true;
+                       skb_queue_splice_tail_init(&l->defer_namedq, l->namedq);
+                       return true;
+               }
+               if (msg_is_rcast(hdr) && !l->bc_rcvlink->named_sync) {
+                       skb_queue_tail(&l->defer_namedq, skb);
+                       return true;
+               }
                skb_queue_tail(l->namedq, skb);
                return true;
        case MSG_BUNDLER:
diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
index 5feaf3b67380..419b3f0f102d 100644
--- a/net/tipc/name_distr.c
+++ b/net/tipc/name_distr.c
@@ -180,6 +180,14 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
                skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
                __skb_queue_tail(list, skb);
        }
+
+       /* Allocate dummy message in order to synchronize w/bcast */
+       skb = named_prepare_buf(net, PUBLICATION, 0, dnode);
+       if (skb) {
+               /* Preparing for 'synching' header */
+               msg_set_syn(buf_msg(skb), 1);
+               __skb_queue_tail(list, skb);
+       }
 }

 /**
diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
index 66a65c2cdb23..4ba6d73e5c4c 100644
--- a/net/tipc/name_table.c
+++ b/net/tipc/name_table.c
@@ -632,8 +632,10 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
 exit:
        spin_unlock_bh(&tn->nametbl_lock);

-       if (skb)
-               tipc_node_broadcast(net, skb);
+       if (skb) {
+               msg_set_is_rcast(buf_msg(skb), true);
+               tipc_bcast_named_publish(net, skb);
+       }
        return p;
 }

@@ -664,7 +666,8 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
        spin_unlock_bh(&tn->nametbl_lock);

        if (skb) {
-               tipc_node_broadcast(net, skb);
+               msg_set_is_rcast(buf_msg(skb), true);
+               tipc_bcast_named_publish(net, skb);
                return 1;
        }
        return 0;
--
2.20.1

-----Original Message-----
From: Jon Maloy <jon...@er...> 
Sent: Wednesday, November 13, 2019 7:02 PM
To: Hoang Huu Le <hoa...@de...>; ma...@do...; tip...@li...
Cc: 'Ying Xue' <yin...@wi...>
Subject: RE: [net-next] tipc: update a binding service via broadcast

Hi Hoang,
This is good, but you have missed the point about the synchronization problem I have been talking about.

1) A new node comes up
2) The "bulk" binding table update is sent, as a series of packets over the new unicast link. This may take some time.
3) The owner of one of the bindings in the bulk (on this node) does unbind.
4) This is sent as broadcast withdraw to all nodes, and arrives before the last packets of the unicast bulk to the newly connected node.
5) Since there is no corresponding publication in the peer node's binding table yet, the withdraw is ignored.
6) The last bulk unicasts arrive at the new peer, and the now invalid publication is added to its binding table.
7) This publication will stay there forever.

We need to find a way to synchronize so that we know that all the bulk publications are in place in the binding table before any broadcast publications/withdraws can be accepted.
Obviously, we could create a backlog queue in the name table, but I hope we can find a simpler and neater solution.

Regards
///jon

> -----Original Message-----
> From: Hoang Le <hoa...@de...>
> Sent: 13-Nov-19 02:35
> To: Jon Maloy <jon...@er...>; ma...@do...; tip...@li...
> Subject: [net-next] tipc: update a binding service via broadcast
> 
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
> 
> It is worth to use broadcast to update a binding service. Then binding
> table updates in all nodes for one shot.
> 
> The mechanism is backward compatible because of sending side changing.
> 
> Signed-off-by: Hoang Le <hoa...@de...>
> ---
>  net/tipc/bcast.c      | 13 +++++++++++++
>  net/tipc/bcast.h      |  2 ++
>  net/tipc/name_table.c |  4 ++--
>  3 files changed, 17 insertions(+), 2 deletions(-)
> 
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index f41096a759fa..18431fa897ab 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -843,3 +843,16 @@ void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
>  		__skb_queue_tail(inputq, _skb);
>  	}
>  }
> +
> +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb)
> +{
> +	struct sk_buff_head xmitq;
> +	u16 cong_link_cnt;
> +	int rc = 0;
> +
> +	__skb_queue_head_init(&xmitq);
> +	__skb_queue_tail(&xmitq, skb);
> +	rc = tipc_bcast_xmit(net, &xmitq, &cong_link_cnt);
> +	__skb_queue_purge(&xmitq);
> +	return rc;
> +}
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index dadad953e2be..a100da3800fc 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -101,6 +101,8 @@ int tipc_bclink_reset_stats(struct net *net);
>  u32 tipc_bcast_get_broadcast_mode(struct net *net);
>  u32 tipc_bcast_get_broadcast_ratio(struct net *net);
> 
> +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb);
> +
>  void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
>  			   struct sk_buff_head *inputq);
> 
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 66a65c2cdb23..9e9c61f7c999 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -633,7 +633,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>  	spin_unlock_bh(&tn->nametbl_lock);
> 
>  	if (skb)
> -		tipc_node_broadcast(net, skb);
> +		tipc_bcast_named_publish(net, skb);
>  	return p;
>  }
> 
> @@ -664,7 +664,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>  	spin_unlock_bh(&tn->nametbl_lock);
> 
>  	if (skb) {
> -		tipc_node_broadcast(net, skb);
> +		tipc_bcast_named_publish(net, skb);
>  		return 1;
>  	}
>  	return 0;
> --
> 2.20.1

Re: [tipc-discussion] [net-next] tipc: update a binding service via broadcast

[Jon M. <jon...@er...>]

Hi Hoang,
You are complicating this too much. The 'syn' and 'rcast'  bits should remain internal for the bcast.c   rcast/bcast code, and we should not mess with those elsewhere.

Instead, you should add a bit in the NAME_DISTR header that marks if the message is bcast/rcast or bulk. For compatibility, this bit should be inverted, so that 1 means bcast/rcast ('not_bulk' or similar) and 0 means 'bulk'.  Then, you add another bit, in bulk messages only, meaning 'not_last'. 
The last bulk message has this bit set to zero, and at that moment you can set the link->named_sync and deliver the deferred queue.

As long as the link_named_sync bit is not set, you sort the messages as follows:

- Incoming !not_bulk/not_last messages go to the head of the named_deferred queue.
- Incoming not_bulk messages go to the tail of the named_deferred queue, irrespective of the not_last bit (which should be always zero anyway)
- Incoming !not_bulk/!not_last message is sorted in after the last !not_bulk/not_last message in the queue, and named_sync is set.

 Because all old messages will be !not_bulk/!not_last (0/0) those will never go into the deferred queue, and named_sync will be set from the start, so we have guaranteed backwards compatibility.

I leave it to you to find better name for those bits.

I think that should solve our problem in a reasonably simple way.

BR
///jon

> -----Original Message-----
> From: Hoang Le <hoa...@de...>
> Sent: 14-Nov-19 03:09
> To: Jon Maloy <jon...@er...>; ma...@do...; tip...@li...
> Cc: 'Ying Xue' <yin...@wi...>
> Subject: RE: [net-next] tipc: update a binding service via broadcast
> 
> Hi Jon,
> 
> Please take a look at v2. The mechanism looks the same as I did before in commit:
> c55c8edafa91 ("tipc: smooth change between replicast and broadcast")
> However, in this case we handle only one direction:  replicast -> broadcast.
> Then, it is still backward compatible.
> 
> [...]
> From ae2ee6a7064de3ec1dc2c7df2db241d22b0d129f Mon Sep 17 00:00:00 2001
> From: Hoang Le <hoa...@de...>
> Date: Wed, 13 Nov 2019 14:01:03 +0700
> Subject: [PATCH] tipc: update a binding service via broadcast
> 
> Currently, updating binding table (add service binding to
> name table/withdraw a service binding) is being sent over replicast.
> However, if we are scaling up clusters to > 100 nodes/containers this
> method is less affection because of looping through nodes in a cluster one
> by one.
> 
> It is worth to use broadcast to update a binding service. Then binding
> table updates in all nodes for one shot.
> 
> The mechanism is backward compatible because of sending side changing.
> 
> v2: resolve synchronization problem when switching from unicast to
> broadcast
> 
> Signed-off-by: Hoang Le <hoa...@de...>
> ---
>  net/tipc/bcast.c      | 13 +++++++++++++
>  net/tipc/bcast.h      |  2 ++
>  net/tipc/link.c       | 16 ++++++++++++++++
>  net/tipc/name_distr.c |  8 ++++++++
>  net/tipc/name_table.c |  9 ++++++---
>  5 files changed, 45 insertions(+), 3 deletions(-)
> 
> diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> index f41096a759fa..18431fa897ab 100644
> --- a/net/tipc/bcast.c
> +++ b/net/tipc/bcast.c
> @@ -843,3 +843,16 @@ void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
>                 __skb_queue_tail(inputq, _skb);
>         }
>  }
> +
> +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb)
> +{
> +       struct sk_buff_head xmitq;
> +       u16 cong_link_cnt;
> +       int rc = 0;
> +
> +       __skb_queue_head_init(&xmitq);
> +       __skb_queue_tail(&xmitq, skb);
> +       rc = tipc_bcast_xmit(net, &xmitq, &cong_link_cnt);
> +       __skb_queue_purge(&xmitq);
> +       return rc;
> +}
> diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> index dadad953e2be..a100da3800fc 100644
> --- a/net/tipc/bcast.h
> +++ b/net/tipc/bcast.h
> @@ -101,6 +101,8 @@ int tipc_bclink_reset_stats(struct net *net);
>  u32 tipc_bcast_get_broadcast_mode(struct net *net);
>  u32 tipc_bcast_get_broadcast_ratio(struct net *net);
> 
> +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb);
> +
>  void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
>                            struct sk_buff_head *inputq);
> 
> diff --git a/net/tipc/link.c b/net/tipc/link.c
> index fb72031228c9..22f1854435df 100644
> --- a/net/tipc/link.c
> +++ b/net/tipc/link.c
> @@ -187,6 +187,9 @@ struct tipc_link {
>         struct tipc_link *bc_sndlink;
>         u8 nack_state;
>         bool bc_peer_is_up;
> +       bool named_sync;
> +       struct sk_buff_head defer_namedq;
> +
> 
>         /* Statistics */
>         struct tipc_stats stats;
> @@ -363,6 +366,7 @@ void tipc_link_remove_bc_peer(struct tipc_link *snd_l,
>         trace_tipc_link_reset(rcv_l, TIPC_DUMP_ALL, "bclink removed!");
>         tipc_link_reset(rcv_l);
>         rcv_l->state = LINK_RESET;
> +       rcv_l->named_sync = false;
>         if (!snd_l->ackers) {
>                 trace_tipc_link_reset(snd_l, TIPC_DUMP_ALL, "zero ackers!");
>                 tipc_link_reset(snd_l);
> @@ -508,6 +512,7 @@ bool tipc_link_create(struct net *net, char *if_name, int bearer_id,
>         __skb_queue_head_init(&l->failover_deferdq);
>         skb_queue_head_init(&l->wakeupq);
>         skb_queue_head_init(l->inputq);
> +       __skb_queue_head_init(&l->defer_namedq);
>         return true;
>  }
> 
> @@ -932,6 +937,8 @@ void tipc_link_reset(struct tipc_link *l)
>         l->silent_intv_cnt = 0;
>         l->rst_cnt = 0;
>         l->bc_peer_is_up = false;
> +       l->named_sync = false;
> +       __skb_queue_purge(&l->defer_namedq);
>         memset(&l->mon_state, 0, sizeof(l->mon_state));
>         tipc_link_reset_stats(l);
>  }
> @@ -1210,6 +1217,15 @@ static bool tipc_data_input(struct tipc_link *l, struct sk_buff *skb,
>                 return true;
>         case NAME_DISTRIBUTOR:
>                 l->bc_rcvlink->state = LINK_ESTABLISHED;
> +               if (msg_is_syn(hdr)) {
> +                       l->bc_rcvlink->named_sync = true;
> +                       skb_queue_splice_tail_init(&l->defer_namedq, l->namedq);

I think you would leak the skb here. It is not added to the queue and not deleted.

> +                       return true;
> +               }
> +               if (msg_is_rcast(hdr) && !l->bc_rcvlink->named_sync) {
> +                       skb_queue_tail(&l->defer_namedq, skb);
> +                       return true;
> +               }
>                 skb_queue_tail(l->namedq, skb);
>                 return true;
>         case MSG_BUNDLER:
> diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
> index 5feaf3b67380..419b3f0f102d 100644
> --- a/net/tipc/name_distr.c
> +++ b/net/tipc/name_distr.c
> @@ -180,6 +180,14 @@ static void named_distribute(struct net *net, struct sk_buff_head *list,
>                 skb_trim(skb, INT_H_SIZE + (msg_dsz - msg_rem));
>                 __skb_queue_tail(list, skb);
>         }
> +
> +       /* Allocate dummy message in order to synchronize w/bcast */
> +       skb = named_prepare_buf(net, PUBLICATION, 0, dnode);
> +       if (skb) {
> +               /* Preparing for 'synching' header */
> +               msg_set_syn(buf_msg(skb), 1);
> +               __skb_queue_tail(list, skb);
> +       }
>  }
> 
>  /**
> diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> index 66a65c2cdb23..4ba6d73e5c4c 100644
> --- a/net/tipc/name_table.c
> +++ b/net/tipc/name_table.c
> @@ -632,8 +632,10 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
>  exit:
>         spin_unlock_bh(&tn->nametbl_lock);
> 
> -       if (skb)
> -               tipc_node_broadcast(net, skb);
> +       if (skb) {
> +               msg_set_is_rcast(buf_msg(skb), true);
> +               tipc_bcast_named_publish(net, skb);
> +       }
>         return p;
>  }
> 
> @@ -664,7 +666,8 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
>         spin_unlock_bh(&tn->nametbl_lock);
> 
>         if (skb) {
> -               tipc_node_broadcast(net, skb);
> +               msg_set_is_rcast(buf_msg(skb), true);
> +               tipc_bcast_named_publish(net, skb);
>                 return 1;
>         }
>         return 0;
> --
> 2.20.1
> 
> -----Original Message-----
> From: Jon Maloy <jon...@er...>
> Sent: Wednesday, November 13, 2019 7:02 PM
> To: Hoang Huu Le <hoa...@de...>; ma...@do...; tipc-
> dis...@li...
> Cc: 'Ying Xue' <yin...@wi...>
> Subject: RE: [net-next] tipc: update a binding service via broadcast
> 
> Hi Hoang,
> This is good, but you have missed the point about the synchronization problem I have been talking about.
> 
> 1) A new node comes up
> 2) The "bulk" binding table update is sent, as a series of packets over the new unicast link. This may take
> some time.
> 3) The owner of one of the bindings in the bulk (on this node) does unbind.
> 4) This is sent as broadcast withdraw to all nodes, and arrives before the last packets of the unicast bulk to
> the newly connected node.
> 5) Since there is no corresponding publication in the peer node's binding table yet, the withdraw is ignored.
> 6) The last bulk unicasts arrive at the new peer, and the now invalid publication is added to its binding
> table.
> 7) This publication will stay there forever.
> 
> We need to find a way to synchronize so that we know that all the bulk publications are in place in the
> binding table before any broadcast publications/withdraws can be accepted.
> Obviously, we could create a backlog queue in the name table, but I hope we can find a simpler and neater
> solution.
> 
> Regards
> ///jon
> 
> > -----Original Message-----
> > From: Hoang Le <hoa...@de...>
> > Sent: 13-Nov-19 02:35
> > To: Jon Maloy <jon...@er...>; ma...@do...; tip...@li...
> > Subject: [net-next] tipc: update a binding service via broadcast
> >
> > Currently, updating binding table (add service binding to
> > name table/withdraw a service binding) is being sent over replicast.
> > However, if we are scaling up clusters to > 100 nodes/containers this
> > method is less affection because of looping through nodes in a cluster one
> > by one.
> >
> > It is worth to use broadcast to update a binding service. Then binding
> > table updates in all nodes for one shot.
> >
> > The mechanism is backward compatible because of sending side changing.
> >
> > Signed-off-by: Hoang Le <hoa...@de...>
> > ---
> >  net/tipc/bcast.c      | 13 +++++++++++++
> >  net/tipc/bcast.h      |  2 ++
> >  net/tipc/name_table.c |  4 ++--
> >  3 files changed, 17 insertions(+), 2 deletions(-)
> >
> > diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c
> > index f41096a759fa..18431fa897ab 100644
> > --- a/net/tipc/bcast.c
> > +++ b/net/tipc/bcast.c
> > @@ -843,3 +843,16 @@ void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> >  		__skb_queue_tail(inputq, _skb);
> >  	}
> >  }
> > +
> > +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb)
> > +{
> > +	struct sk_buff_head xmitq;
> > +	u16 cong_link_cnt;
> > +	int rc = 0;
> > +
> > +	__skb_queue_head_init(&xmitq);
> > +	__skb_queue_tail(&xmitq, skb);
> > +	rc = tipc_bcast_xmit(net, &xmitq, &cong_link_cnt);
> > +	__skb_queue_purge(&xmitq);
> > +	return rc;
> > +}
> > diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h
> > index dadad953e2be..a100da3800fc 100644
> > --- a/net/tipc/bcast.h
> > +++ b/net/tipc/bcast.h
> > @@ -101,6 +101,8 @@ int tipc_bclink_reset_stats(struct net *net);
> >  u32 tipc_bcast_get_broadcast_mode(struct net *net);
> >  u32 tipc_bcast_get_broadcast_ratio(struct net *net);
> >
> > +int tipc_bcast_named_publish(struct net *net, struct sk_buff *skb);
> > +
> >  void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
> >  			   struct sk_buff_head *inputq);
> >
> > diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c
> > index 66a65c2cdb23..9e9c61f7c999 100644
> > --- a/net/tipc/name_table.c
> > +++ b/net/tipc/name_table.c
> > @@ -633,7 +633,7 @@ struct publication *tipc_nametbl_publish(struct net *net, u32 type, u32 lower,
> >  	spin_unlock_bh(&tn->nametbl_lock);
> >
> >  	if (skb)
> > -		tipc_node_broadcast(net, skb);
> > +		tipc_bcast_named_publish(net, skb);
> >  	return p;
> >  }
> >
> > @@ -664,7 +664,7 @@ int tipc_nametbl_withdraw(struct net *net, u32 type, u32 lower,
> >  	spin_unlock_bh(&tn->nametbl_lock);
> >
> >  	if (skb) {
> > -		tipc_node_broadcast(net, skb);
> > +		tipc_bcast_named_publish(net, skb);
> >  		return 1;
> >  	}
> >  	return 0;
> > --
> > 2.20.1
>