From: fendo <fendo@bi12info.com>
Date: Tue, 19 Feb 2013 21:47:43 +0000 (+0100)
Subject: Add grouping session functionality for load balancing and failover
X-Git-Tag: 2.2.1-2sames3.8~11
X-Git-Url: http://git.sameswireless.fr/l2tpns.git/commitdiff_plain/b5273cc1c65f9496c1a7c2135b61979dd98fcf08?ds=sidebyside;hp=-c

Add grouping session functionality for load balancing and failover
---

b5273cc1c65f9496c1a7c2135b61979dd98fcf08
diff --git a/Makefile b/Makefile
index 21221ab..ee05e2c 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,7 @@ INSTALL = install -c -D -o root -g root
 l2tpns.LIBS = -lm -lcli -ldl
 
 OBJS = arp.o cli.o cluster.o constants.o control.o icmp.o l2tpns.o \
-    ll.o md5.o ppp.o radius.o tbf.o util.o pppoe.o
+    ll.o md5.o ppp.o radius.o tbf.o util.o pppoe.o grpsess.o
 
 PROGRAMS = l2tpns nsctl
 PLUGINS = autosnoop.so autothrottle.so garden.so sessionctl.so \
@@ -129,6 +129,7 @@ radius.o: radius.c md5.h constants.h l2tpns.h plugin.h util.h cluster.h \
 tbf.o: tbf.c l2tpns.h util.h tbf.h
 util.o: util.c l2tpns.h bgp.h
 pppoe.o: pppoe.c l2tpns.h cluster.h constants.h md5.h util.h
+grpsess.o: grpsess.c l2tpns.h util.h bgp.h
 bgp.o: bgp.c l2tpns.h bgp.h util.h
 l2tplac.o: l2tplac.c md5.h l2tpns.h util.h cluster.h l2tplac.h pppoe.h
 autosnoop.so: autosnoop.c l2tpns.h plugin.h
diff --git a/arp.c b/arp.c
index 6d55756..e9c7584 100644
--- a/arp.c
+++ b/arp.c
@@ -5,6 +5,8 @@
 #include <net/ethernet.h>
 #include <net/if_arp.h>
 #include <linux/if_packet.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 
diff --git a/autosnoop.c b/autosnoop.c
index ddc699f..5633824 100644
--- a/autosnoop.c
+++ b/autosnoop.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 
diff --git a/autothrottle.c b/autothrottle.c
index 73f9a94..007f926 100644
--- a/autothrottle.c
+++ b/autothrottle.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 
diff --git a/bgp.c b/bgp.c
index 7c1e6c0..36894be 100644
--- a/bgp.c
+++ b/bgp.c
@@ -20,6 +20,7 @@
 #include <arpa/inet.h>
 #include <netdb.h>
 #include <fcntl.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #include "bgp.h"
diff --git a/cli.c b/cli.c
index 3f950e0..65afb43 100644
--- a/cli.c
+++ b/cli.c
@@ -21,6 +21,7 @@
 #include <dlfcn.h>
 #include <netdb.h>
 #include <libcli.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #include "constants.h"
diff --git a/cluster.c b/cluster.c
index 78de01b..4b85f67 100644
--- a/cluster.c
+++ b/cluster.c
@@ -16,6 +16,7 @@
 #include <malloc.h>
 #include <errno.h>
 #include <libcli.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #include "cluster.h"
@@ -709,6 +710,7 @@ void cluster_check_master(void)
 		increment_counter(&session[i].cout, &session[i].cout_wrap, sess_local[i].cout);
 		session[i].cin_delta += sess_local[i].cin;
 		session[i].cout_delta += sess_local[i].cout;
+		session[i].coutgrp_delta += sess_local[i].cout;
 
 		session[i].pin += sess_local[i].pin;
 		session[i].pout += sess_local[i].pout;
@@ -1263,6 +1265,7 @@ static int cluster_handle_bytes(uint8_t *data, int size)
 
 		session[b->sid].cin_delta += b->cin;
 		session[b->sid].cout_delta += b->cout;
+		session[b->sid].coutgrp_delta += b->cout;
 
 		if (b->cin)
 			session[b->sid].last_packet = session[b->sid].last_data = time_now;
diff --git a/control.c b/control.c
index 3d38b22..092ab8e 100644
--- a/control.c
+++ b/control.c
@@ -1,6 +1,9 @@
 // L2TPNS: control
 
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "control.h"
 
diff --git a/debian/changelog b/debian/changelog
index 325002c..5dcfe00 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+l2tpns (2.2.1-2sames3.6) unstable; urgency=low
+
+  * Sames l2tpns version.
+
+ -- Fernando Alves <fernando.alves@sameswireless.fr>  Tue, 12 Feb 2013 20:20:17 +0100
+
 l2tpns (2.2.1-2fdn3.5) unstable; urgency=low
 
   * Update debian/changelog
diff --git a/dictionary.sames b/dictionary.sames
new file mode 100644
index 0000000..a8a244d
--- /dev/null
+++ b/dictionary.sames
@@ -0,0 +1,18 @@
+# -*- text -*-
+##############################################################################
+#	Vendor specific attributes SamesWireless
+#	http://www.sameswireless.fr
+#
+#	$Id$
+#
+##############################################################################
+
+VENDOR		SAMES				64520
+
+BEGIN-VENDOR	SAMES
+
+ATTRIBUTE	SAMES-Group-Framed-Route		22	string
+ATTRIBUTE	SAMES-Group-Session-Weight		23	string
+
+END-VENDOR SAMES
+
diff --git a/garden.c b/garden.c
index 994ecb5..0b7d763 100644
--- a/garden.c
+++ b/garden.c
@@ -3,6 +3,9 @@
 #include <stdlib.h>
 #include <sys/wait.h>
 #include <sys/types.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 #include "control.h"
@@ -219,7 +222,7 @@ int garden_session(sessiont *s, int flag, char *newuser)
 	/* Clean up counters */
 	s->pin = s->pout = 0;
 	s->cin = s->cout = 0;
-	s->cin_delta = s->cout_delta = 0;
+	s->cin_delta = s->cout_delta = s->coutgrp_delta = 0;
 	s->cin_wrap = s->cout_wrap = 0;
 
 	snprintf(cmd, sizeof(cmd),
diff --git a/grpsess.c b/grpsess.c
new file mode 100644
index 0000000..13c6505
--- /dev/null
+++ b/grpsess.c
@@ -0,0 +1,520 @@
+/*
+ * Fernando ALVES 2013
+ * Grouped session for load balancing and fail-over
+ * GPL licenced
+ */
+
+#include <errno.h>
+#include <ctype.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
+#include "l2tpns.h"
+#include "util.h"
+
+#ifdef BGP
+#include "bgp.h"
+#endif
+
+union grp_iphash {
+	groupidt grp;
+	union grp_iphash *idx;
+} grp_ip_hash[256];			// Mapping from IP address to group structures.
+
+static groupidt gnextgrpid = 0;
+
+// Find gruop by IP, < 1 for not found
+//
+// Confusingly enough, this 'ip' must be
+// in _network_ order. This being the common
+// case when looking it up from IP packet headers.
+static groupidt grp_lookup_ipmap(in_addr_t ip)
+{
+	uint8_t *a = (uint8_t *) &ip;
+	union grp_iphash *h = grp_ip_hash;
+
+	if (!(h = h[*a++].idx)) return 0;
+	if (!(h = h[*a++].idx)) return 0;
+	if (!(h = h[*a++].idx)) return 0;
+
+	return h[*a].grp;
+}
+
+//
+// Take an IP address in HOST byte order and
+// add it to the grouid by IP cache.
+//
+// (It's actually cached in network order)
+//
+static void grp_cache_ipmap(in_addr_t ip, groupidt g)
+{
+	in_addr_t nip = htonl(ip);	// MUST be in network order. I.e. MSB must in be ((char *) (&ip))[0]
+	uint8_t *a = (uint8_t *) &nip;
+	union grp_iphash *h = grp_ip_hash;
+	int i;
+
+	for (i = 0; i < 3; i++)
+	{
+		if (!(h[a[i]].idx || (h[a[i]].idx = calloc(256, sizeof(union grp_iphash)))))
+			return;
+
+		h = h[a[i]].idx;
+	}
+
+	h[a[3]].grp = g;
+
+	if (g > 0)
+		LOG(4, 0, 0, "Caching Group:%d ip address %s\n", g, fmtaddr(nip, 0));
+	else if (g == 0)
+		LOG(4, 0, 0, "Un-caching Group ip address %s\n", fmtaddr(nip, 0));
+}
+
+groupidt grp_groupbyip(in_addr_t ip)
+{
+	groupidt g = grp_lookup_ipmap(ip);
+
+	if (g > 0 && g < MAXGROUPE)
+		return g;
+
+	return 0;
+}
+
+// Add a route
+//
+// This adds it to the routing table, advertises it
+// via BGP if enabled, and stuffs it into the
+// 'sessionbyip' cache.
+//
+// 'ip' must be in _host_ order.
+//
+static void grp_routeset(groupidt g, in_addr_t ip, int prefixlen, int add)
+{
+	struct {
+		struct nlmsghdr nh;
+		struct rtmsg rt;
+		char buf[32];
+	} req;
+	int i;
+	in_addr_t n_ip;
+
+	if (!prefixlen) prefixlen = 32;
+
+	ip &= 0xffffffff << (32 - prefixlen);;	// Force the ip to be the first one in the route.
+
+	memset(&req, 0, sizeof(req));
+
+	if (add)
+	{
+		req.nh.nlmsg_type = RTM_NEWROUTE;
+		req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_REPLACE;
+	}
+	else
+	{
+		req.nh.nlmsg_type = RTM_DELROUTE;
+		req.nh.nlmsg_flags = NLM_F_REQUEST;
+	}
+
+	req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.rt));
+
+	req.rt.rtm_family = AF_INET;
+	req.rt.rtm_dst_len = prefixlen;
+	req.rt.rtm_table = RT_TABLE_MAIN;
+	req.rt.rtm_protocol = 42;
+	req.rt.rtm_scope = RT_SCOPE_LINK;
+	req.rt.rtm_type = RTN_UNICAST;
+
+	netlink_addattr(&req.nh, RTA_OIF, &tunidx, sizeof(int));
+	n_ip = htonl(ip);
+	netlink_addattr(&req.nh, RTA_DST, &n_ip, sizeof(n_ip));
+
+	LOG(1, 0, 0, "Route (Group) %s %s/%d\n", add ? "add" : "del", fmtaddr(htonl(ip), 0), prefixlen);
+
+	if (netlink_send(&req.nh) < 0)
+		LOG(0, 0, 0, "grp_routeset() error in sending netlink message: %s\n", strerror(errno));
+
+#ifdef BGP
+	if (add)
+		bgp_add_route(htonl(ip), prefixlen);
+	else
+		bgp_del_route(htonl(ip), prefixlen);
+#endif /* BGP */
+
+	// Add/Remove the IPs to the 'groupbyip' cache.
+	// Note that we add the zero address in the case of
+	// a network route. Roll on CIDR.
+
+	// Note that 'g == 0' implies this is the address pool.
+	// We still cache it here, because it will pre-fill
+	// the malloc'ed tree.
+	if (g)
+	{
+		if (!add)	// Are we deleting a route?
+			g = 0;	// Caching the session as '0' is the same as uncaching.
+
+		for (i = ip; i < ip+(1<<(32-prefixlen)) ; ++i)
+			grp_cache_ipmap(i, g);
+	}
+}
+
+// Set all route of a group
+void grp_setgrouproute(groupidt g, int add)
+{
+	int i;
+	for (i = 0; i < grpsession[g].nbroutesgrp; i++)
+	{
+		if (grpsession[g].route[i].ip != 0)
+		{
+			grp_routeset(g, grpsession[g].route[i].ip, grpsession[g].route[i].prefixlen, add);
+		}
+	}
+}
+
+// return group id by session
+groupidt grp_groupbysession(sessionidt s)
+{
+	groupidt g = 0;
+	int i;
+	for (g = gnextgrpid; g != 0; g = grpsession[g].prev)
+	{
+		for (i = 0; i < grpsession[g].nbsession; i++)
+		{
+			if (grpsession[g].sesslist[i].sid == s)
+			{	// session found in group
+				return g;
+			}
+		}
+	}
+
+	return 0;
+}
+
+// Remove a session to a group
+// return 1 if OK
+void grp_removesession(groupidt g, sessionidt s)
+{
+	int i;
+
+	if (grpsession[g].nbsession <= 0)
+		return;
+
+	for (i = 0; i < grpsession[g].nbsession; i++)
+	{
+		if (grpsession[g].sesslist[i].sid == s)
+		{	// session found on group
+			--grpsession[g].nbsession;
+			if (grpsession[g].nbsession == 0)
+			{
+				// Group is empty, remove it
+
+				// Del all routes
+				grp_setgrouproute(g, 0);
+
+				if (gnextgrpid == g)
+				{
+					gnextgrpid = 0;
+				}
+				else
+				{
+					groupidt g2;
+					for (g2 = gnextgrpid; g2 != 0; g2 = grpsession[g2].prev)
+					{
+						if (grpsession[g2].prev == g)
+						{
+							grpsession[g2].prev = grpsession[g].prev;
+							break;
+						}
+					}
+				}
+
+				memset(&grpsession[g], 0, sizeof(grpsession[0]));
+			}
+			else
+			{
+				// remove the session
+				memmove(&grpsession[g].sesslist[i],
+						&grpsession[g].sesslist[i+1],
+						(grpsession[g].nbsession - i) * sizeof(grpsession[g].sesslist[i]));
+			}
+			return;
+		}
+	}
+}
+
+// Add a session to a group
+// return 1 if OK
+static int grp_addsession(groupidt g, sessionidt s, uint8_t weight)
+{
+	int i;
+
+	for (i = 0; i < grpsession[g].nbsession; i++)
+	{
+		if (grpsession[g].sesslist[i].sid == s)
+		{	// already in group
+			if ((!grpsession[g].sesslist[i].weight) || (weight > 1))
+				grpsession[g].sesslist[i].weight = weight; // update Weight session (for load-balancing)
+
+			return 1;
+		}
+	}
+
+	if (i >= MAXSESSINGRP)
+	{
+		LOG(1, s, session[s].tunnel, "   Too many session for Group %d\n", g);
+		return 0;
+	}
+	else
+	{	// add session id to group
+		if (i == 0)
+		{
+			// it's the first session of the group, set to next group
+			grpsession[g].prev = gnextgrpid;
+			gnextgrpid = g;
+		}
+		grpsession[g].sesslist[i].sid = s;
+		grpsession[g].sesslist[i].weight = weight;
+		grpsession[g].nbsession++;
+
+		return 1;
+	}
+	return 0;
+}
+
+// Add a route to a group
+// return 1 if OK
+static int grp_addroute(groupidt g, sessionidt s, in_addr_t ip, int prefixlen)
+{
+	int i;
+
+	for (i = 0; i < MAXROUTEINGRP; i++)
+	{
+		if ((i >= grpsession[g].nbroutesgrp))
+		{
+			LOG(3, s, session[s].tunnel, "   Radius reply Group %d contains route for %s/%d\n",
+				g, fmtaddr(htonl(ip), 0), prefixlen);
+
+			grpsession[g].route[i].ip = ip;
+			grpsession[g].route[i].prefixlen = prefixlen;
+			grpsession[g].nbroutesgrp++;
+			return 1;
+		}
+		else if ((grpsession[g].route[i].ip == ip) && (grpsession[g].route[i].prefixlen == prefixlen))
+		{
+			// route already defined in group
+			LOG(3, s, session[s].tunnel,
+				"   Radius reply Group %d contains route for %s/%d (this already defined)\n",
+				g, fmtaddr(htonl(ip), 0), prefixlen);
+
+			return 1;
+		}
+		else if (grpsession[g].route[i].ip == 0)
+		{
+			LOG(3, s, session[s].tunnel, "   Radius reply Group %d contains route for %s/%d (find empty on list!!!)\n",
+				g, fmtaddr(htonl(ip), 0), prefixlen);
+
+			grpsession[g].route[i].ip = ip;
+			grpsession[g].route[i].prefixlen = prefixlen;
+			return 1;
+		}
+	}
+
+	if (i >= MAXROUTEINGRP)
+	{
+		LOG(1, s, session[s].tunnel, "   Too many routes for Group %d\n", g);
+	}
+	return 0;
+}
+
+// Process Sames vendor specific attribut radius
+void grp_processvendorspecific(sessionidt s, uint8_t *pvs)
+{
+	uint8_t attrib = *pvs;
+	groupidt grpid = 0;
+	uint8_t *n = pvs + 2;
+	uint8_t *e = pvs + pvs[1];
+
+	if ((attrib >= 22) && (attrib <= 23))
+	{
+		while (n < e && isdigit(*n))
+		{
+			grpid = grpid * 10 + *n - '0';
+			n++;
+		}
+		if ((grpid == 0) || (grpid >= MAXGROUPE))
+		{
+			LOG(1, s, session[s].tunnel, "   Group Attribute Id %d not allowed\n", grpid);
+			return;
+		}
+		else if (*n != ',')
+		{
+			LOG(1, s, session[s].tunnel, "   Group Attribute Id not defined\n");
+			return;
+		}
+
+		if (!grp_addsession(grpid, s, 1))
+		{
+			return;
+		}
+
+		n++;
+	}
+
+	//process, Sames vendor-specific 64520
+	if (attrib == 22) //SAMES-Group-Framed-Route
+	{
+		in_addr_t ip = 0;
+		uint8_t u = 0;
+		uint8_t bits = 0;
+
+		while (n < e && (isdigit(*n) || *n == '.'))
+		{
+			if (*n == '.')
+			{
+				ip = (ip << 8) + u;
+				u = 0;
+			}
+			else
+				u = u * 10 + *n - '0';
+			n++;
+		}
+		ip = (ip << 8) + u;
+		if (*n == '/')
+		{
+			n++;
+			while (n < e && isdigit(*n))
+				bits = bits * 10 + *n++ - '0';
+		}
+		else if ((ip >> 24) < 128)
+			bits = 8;
+		else if ((ip >> 24) < 192)
+			bits = 16;
+		else
+			bits = 24;
+
+		if (!grp_addroute(grpid, s, ip, bits))
+			return;
+	}
+	else if (attrib == 23) //SAMES-Group-Session-Weight
+	{
+		uint8_t weight = 0;
+
+		while (n < e && isdigit(*n))
+			weight = weight * 10 + *n++ - '0';
+
+		if (!weight)
+		{
+			LOG(1, s, session[s].tunnel, "   Group-Session-Weight 0 GroupId %d not allowed\n", grpid);
+			return;
+		}
+		if (!grp_addsession(grpid, s, weight))
+		{
+			return;
+		}
+	}
+	else
+	{
+		LOG(3, s, session[s].tunnel, "   Unknown vendor-specific: 64520, Attrib: %d\n", attrib);
+	}
+}
+
+// Init data structures
+void grp_initdata()
+{
+	// Set default value (10s)
+	config->grp_txrate_average_time = 10;
+
+	if (!(grpsession = shared_malloc(sizeof(groupsesst) * MAXGROUPE)))
+	{
+		LOG(0, 0, 0, "Error doing malloc for grouped session: %s\n", strerror(errno));
+		exit(1);
+	}
+
+	memset(grpsession, 0, sizeof(grpsession[0]) * MAXGROUPE);
+}
+
+// Update time_changed of the group
+void grp_time_changed()
+{
+	groupidt g;
+
+	for (g = gnextgrpid; g != 0; g = grpsession[g].prev)
+	{
+		grpsession[g].time_changed++;
+	}
+}
+
+// return the next session can be used on the group
+sessionidt grp_getnextsession(groupidt g, in_addr_t ip)
+{
+	sessionidt s = 0, s2 = 0, s3 = 0;
+	int i;
+	uint32_t ltime_changed = 0;
+	uint32_t mintxrate = 0xFFFFFFFF;
+
+	if (g >= MAXGROUPE)
+		return 0;
+
+	if ((s = sessionbyip(ip)))
+	{
+		if ( (session[s].ppp.phase > Establish) &&
+			 (time_now - session[s].last_packet <= (config->echo_timeout + 2)) )
+		{
+			int recaltxrate = 0;
+
+			for (i = 0; i < grpsession[g].nbsession; i++)
+			{
+				if (s == grpsession[g].sesslist[i].sid)
+				{
+					if ((time_now - grpsession[g].sesslist[i].mark_time) > config->grp_txrate_average_time)
+					{
+						grpsession[g].sesslist[i].mark_time = time_now;
+						recaltxrate = 1;
+						break;
+					}
+				}
+			}
+
+			if (!recaltxrate)
+				return s;
+		}
+	}
+
+	if (grpsession[g].time_changed > config->grp_txrate_average_time)
+	{
+		ltime_changed = grpsession[g].time_changed;
+		grpsession[g].time_changed = 1;
+	}
+
+	for (i = 0; i < grpsession[g].nbsession; i++)
+	{
+		if ((s2 = grpsession[g].sesslist[i].sid))
+		{
+			s3 = s2;
+			if (ltime_changed)
+			{
+				grpsession[g].sesslist[i].tx_rate = session[s2].coutgrp_delta/ltime_changed;
+				session[s2].coutgrp_delta = grpsession[g].sesslist[i].tx_rate;
+				LOG(3, s2, session[s2].tunnel, "TX Rate: %d session weight: %d\n", grpsession[g].sesslist[i].tx_rate, grpsession[g].sesslist[i].weight);
+			}
+
+			if ( session[s2].ppp.phase > Establish &&
+				(time_now - session[s2].last_packet <= (config->echo_timeout + 2)) )
+			{
+				uint32_t txrate = grpsession[g].sesslist[i].tx_rate/grpsession[g].sesslist[i].weight;
+				if (txrate < mintxrate)
+				{
+					s = s2;
+					mintxrate = txrate;
+				}
+			}
+		}
+	}
+
+	if (!s)
+		s = s3;
+
+	if (s)
+		cache_ipmap(ntohl(ip), s);
+
+	return s;
+}
diff --git a/icmp.c b/icmp.c
index fa947b7..5f05127 100644
--- a/icmp.c
+++ b/icmp.c
@@ -13,6 +13,7 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <memory.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #include "pppoe.h"
diff --git a/l2tplac.c b/l2tplac.c
index 6ad4683..442f08a 100644
--- a/l2tplac.c
+++ b/l2tplac.c
@@ -7,6 +7,8 @@
 
 #include <errno.h>
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
 
 #include "md5.h"
 #include "l2tpns.h"
@@ -525,6 +527,7 @@ int lac_session_forward(uint8_t *buf, int len, sessionidt sess, uint16_t proto,
 		// Update STAT OUT
 		increment_counter(&session[s].cout, &session[s].cout_wrap, len); // byte count
 		session[s].cout_delta += len;
+		session[s].coutgrp_delta += len;
 		session[s].pout++;
 		sess_local[s].cout += len;
 		sess_local[s].pout++;
diff --git a/l2tpns.c b/l2tpns.c
index 1a37a0a..52872a5 100644
--- a/l2tpns.c
+++ b/l2tpns.c
@@ -81,7 +81,7 @@ int cluster_sockfd = -1;	// Intra-cluster communications socket.
 int epollfd = -1;		// event polling
 time_t basetime = 0;		// base clock
 char hostname[MAXHOSTNAME] = "";	// us.
-static int tunidx;		// ifr_ifindex of tun device
+int tunidx;				// ifr_ifindex of tun device
 int nlseqnum = 0;		// netlink sequence number
 int min_initok_nlseqnum = 0;	// minimun seq number for messages after init is ok
 static int syslog_log = 0;	// are we logging to syslog
@@ -192,6 +192,7 @@ config_descriptt config_values[] = {
 	CONFIG("pppoe_ac_name", pppoe_ac_name, STRING),
 	CONFIG("disable_sending_hello", disable_sending_hello, BOOL),
 	CONFIG("disable_no_spoof", disable_no_spoof, BOOL),
+	CONFIG("grp_txrate_average_time", grp_txrate_average_time, INT),
 	{ NULL, 0, 0, 0 }
 };
 
@@ -220,6 +221,7 @@ tunnelt *tunnel = NULL;			// Array of tunnel structures.
 bundlet *bundle = NULL;			// Array of bundle structures.
 fragmentationt *frag = NULL;		// Array of fragmentation structures.
 sessiont *session = NULL;		// Array of session structures.
+groupsesst *grpsession = NULL;		// Array of groupsesst structures.
 sessionlocalt *sess_local = NULL;	// Array of local per-session counters.
 radiust *radius = NULL;			// Array of radius structures.
 ippoolt *ip_address_pool = NULL;	// Array of dynamic IP addresses.
@@ -230,9 +232,6 @@ struct Tstats *_statistics = NULL;
 struct Tringbuffer *ringbuffer = NULL;
 #endif
 
-static ssize_t netlink_send(struct nlmsghdr *nh);
-static void netlink_addattr(struct nlmsghdr *nh, int type, const void *data, int alen);
-static void cache_ipmap(in_addr_t ip, sessionidt s);
 static void uncache_ipmap(in_addr_t ip);
 static void cache_ipv6map(struct in6_addr ip, int prefixlen, sessionidt s);
 static void free_ip_address(sessionidt s);
@@ -261,8 +260,9 @@ static clockt now(double *f)
 	if (f) *f = t.tv_sec + t.tv_usec / 1000000.0;
 	if (t.tv_sec != time_now)
 	{
-	    time_now = t.tv_sec;
-	    time_changed++;
+		time_now = t.tv_sec;
+		time_changed++;
+		grp_time_changed();
 	}
 
 	// Time in milliseconds
@@ -621,7 +621,7 @@ static void initnetlink(void)
 	}
 }
 
-static ssize_t netlink_send(struct nlmsghdr *nh)
+ssize_t netlink_send(struct nlmsghdr *nh)
 {
 	struct sockaddr_nl nladdr;
 	struct iovec iov;
@@ -657,7 +657,7 @@ static ssize_t netlink_recv(void *buf, ssize_t len)
 }
 
 /* adapted from iproute2 */
-static void netlink_addattr(struct nlmsghdr *nh, int type, const void *data, int alen)
+void netlink_addattr(struct nlmsghdr *nh, int type, const void *data, int alen)
 {
 	int len = RTA_LENGTH(alen);
 	struct rtattr *rta;
@@ -1003,7 +1003,7 @@ sessionidt sessionbyipv6(struct in6_addr ip)
 //
 // (It's actually cached in network order)
 //
-static void cache_ipmap(in_addr_t ip, sessionidt s)
+void cache_ipmap(in_addr_t ip, sessionidt s)
 {
 	in_addr_t nip = htonl(ip);	// MUST be in network order. I.e. MSB must in be ((char *) (&ip))[0]
 	uint8_t *a = (uint8_t *) &nip;
@@ -1375,6 +1375,7 @@ static void update_session_out_stat(sessionidt s, sessiont *sp, int len)
 {
 	increment_counter(&sp->cout, &sp->cout_wrap, len); // byte count
 	sp->cout_delta += len;
+	sp->coutgrp_delta += len;
 	sp->pout++;
 	sp->last_data = time_now;
 
@@ -1388,6 +1389,7 @@ static void update_session_out_stat(sessionidt s, sessiont *sp, int len)
 void processipout(uint8_t *buf, int len)
 {
 	sessionidt s;
+	groupidt g;
 	sessiont *sp;
 	tunnelidt t;
 	in_addr_t ip;
@@ -1424,7 +1426,31 @@ void processipout(uint8_t *buf, int len)
 	}
 
 	ip = *(uint32_t *)(buf + 16);
-	if (!(s = sessionbyip(ip)))
+	if ((g = grp_groupbyip(ip)))
+	{
+		s = grp_getnextsession(g, ip);
+		if (!s)
+		{
+			// Is this a packet for a session that doesn't exist?
+			static int rate = 0;	// Number of ICMP packets we've sent this second.
+			static int last = 0;	// Last time we reset the ICMP packet counter 'rate'.
+
+			if (last != time_now)
+			{
+				last = time_now;
+				rate = 0;
+			}
+
+			if (rate++ < config->icmp_rate) // Only send a max of icmp_rate per second.
+			{
+				LOG(4, 0, 0, "IP: Sending ICMP host unreachable to %s\n", fmtaddr(*(in_addr_t *)(buf + 12), 0));
+				host_unreachable(*(in_addr_t *)(buf + 12), *(uint16_t *)(buf + 4),
+					config->bind_address ? config->bind_address : my_address, buf, len);
+			}
+			return;
+		}
+	}
+	else if (!(s = sessionbyip(ip)))
 	{
 		// Is this a packet for a session that doesn't exist?
 		static int rate = 0;	// Number of ICMP packets we've sent this second.
@@ -1759,6 +1785,7 @@ static void processipv6out(uint8_t * buf, int len)
 
 	increment_counter(&sp->cout, &sp->cout_wrap, len); // byte count
 	sp->cout_delta += len;
+	sp->coutgrp_delta += len;
 	sp->pout++;
 	udp_tx += len;
 
@@ -1808,6 +1835,7 @@ static void send_ipout(sessionidt s, uint8_t *buf, int len)
 
 	increment_counter(&sp->cout, &sp->cout_wrap, len); // byte count
 	sp->cout_delta += len;
+	sp->coutgrp_delta += len;
 	sp->pout++;
 	udp_tx += len;
 
@@ -2054,7 +2082,7 @@ void sessionshutdown(sessionidt s, char const *reason, int cdn_result, int cdn_e
 		session[s].die = TIME + 150; // Clean up in 15 seconds
 
 	if (session[s].ip)
-	{                          // IP allocated, clear and unroute
+	{	// IP allocated, clear and unroute
 		int r;
 		int routed = 0;
 		for (r = 0; r < MAXROUTE && session[s].route[r].ip; r++)
@@ -2254,6 +2282,8 @@ static void sessionclear(sessionidt s)
 // kill a session now
 void sessionkill(sessionidt s, char *reason)
 {
+	groupidt g;
+
 	CSTAT(sessionkill);
 
 	if (!session[s].opened) // not alive
@@ -2284,6 +2314,12 @@ void sessionkill(sessionidt s, char *reason)
 #endif
 
 	LOG(2, s, session[s].tunnel, "Kill session %d (%s): %s\n", s, session[s].user, reason);
+
+	if ((g = grp_groupbysession(s)))
+	{
+		grp_removesession(g, s);
+	}
+
 	sessionclear(s);
 	cluster_send_session(s);
 }
@@ -3682,7 +3718,7 @@ static void regular_cleanups(double period)
 
 		// No data in ECHO_TIMEOUT seconds, send LCP ECHO
 		if (session[s].ppp.phase >= Establish && (time_now - session[s].last_packet >= config->echo_timeout) &&
-			(time_now - sess_local[s].last_echo >= ECHO_TIMEOUT))
+			(time_now - sess_local[s].last_echo >= config->echo_timeout))
 		{
 			uint8_t b[MAXETHER];
 
@@ -4661,6 +4697,8 @@ static void initdata(int optdebug, char *optconfig)
 #ifdef LAC
 	lac_initremotelnsdata();
 #endif
+
+	grp_initdata();
 }
 
 static int assign_ip_address(sessionidt s)
@@ -5582,6 +5620,7 @@ int sessionsetup(sessionidt s, tunnelidt t)
 	if (!session[s].bundle || (bundle[session[s].bundle].num_of_links == 1))
 	{
 		int routed = 0;
+		groupidt g;
 
 		// Add the route for this session.
 		for (r = 0; r < MAXROUTE && session[s].route[r].ip; r++)
@@ -5604,6 +5643,11 @@ int sessionsetup(sessionidt s, tunnelidt t)
 		}
 		else
 			cache_ipmap(session[s].ip, s);
+
+		if ((g = grp_groupbysession(s)))
+		{
+			grp_setgrouproute(g, 1);
+		}
 	}
 
 	sess_local[s].lcp_authtype = 0; // RADIUS authentication complete
diff --git a/l2tpns.h b/l2tpns.h
index 56f8fa1..989ddd9 100644
--- a/l2tpns.h
+++ b/l2tpns.h
@@ -24,6 +24,10 @@
 #define MAXADDRESS	20		// Maximum length for the Endpoint Discrminiator address
 #define MAXSESSION	60000		// could be up to 65535
 #define MAXTBFS		6000		// Maximum token bucket filters. Might need up to 2 * session.
+#define MAXSESSINGRP	12		// Maximum number of member links in grouped session
+#define MAXGRPINSESS	12		// Maximum number of member links in session group
+#define MAXGROUPE		300	// could be up to 65535, Maximum number of grouped session
+#define MAXROUTEINGRP	15		// max static routes per group
 
 // Tunnel Id reserved for pppoe
 #define TUNNEL_ID_PPPOE	1
@@ -207,6 +211,7 @@ enum {
 typedef uint16_t sessionidt;
 typedef uint16_t bundleidt;
 typedef uint16_t tunnelidt;
+typedef uint16_t groupidt;
 typedef uint32_t clockt;
 typedef uint8_t hasht[16];
 
@@ -328,14 +333,37 @@ typedef struct
 #ifdef LAC
 	sessionidt forwardtosession;	// LNS id_session to forward
 	uint8_t src_hwaddr[ETH_ALEN];	// MAC addr source (for pppoe sessions 6 bytes)
-	char reserved[4];		// Space to expand structure without changing HB_VERSION
+	uint32_t coutgrp_delta;
 #else
 	uint8_t src_hwaddr[ETH_ALEN];	// MAC addr source (for pppoe sessions 6 bytes)
-	char reserved[6];		// Space to expand structure without changing HB_VERSION
+	uint32_t coutgrp_delta;
+	char reserved[2];		// Space to expand structure without changing HB_VERSION
 #endif
 }
 sessiont;
 
+typedef struct
+{
+	uint32_t tx_rate;
+	time_t mark_time;
+	sessionidt sid;
+	uint8_t weight;
+}
+groupsessionidt;
+
+typedef struct
+{
+	//uint16_t indexprevsessid;
+	uint32_t time_changed;
+	//time_t mark_time;
+	groupidt prev;
+	groupsessionidt sesslist[MAXSESSINGRP];
+	routet route[MAXROUTEINGRP];		// static routes
+	uint8_t nbroutesgrp;
+	uint8_t nbsession;
+}
+groupsesst;
+
 typedef struct
 {
         int state;                              // current state (bundlestate enum)
@@ -781,6 +809,7 @@ typedef struct
 	uint8_t pppoe_hwaddr[ETH_ALEN];	// MAC addr of interface pppoe to bind
 	int disable_sending_hello; // Disable l2tp sending HELLO message for Apple compatibility.
 	int disable_no_spoof; // Disable no spoof (permit load balancing client --> internet)
+	int grp_txrate_average_time; // caculation txrate average time (default 10s)
 } configt;
 
 enum config_typet { INT, STRING, UNSIGNED_LONG, SHORT, BOOL, IPv4, IPv6 };
@@ -946,6 +975,10 @@ int ip_filter(uint8_t *buf, int len, uint8_t filter);
 int cmd_show_ipcache(struct cli_def *cli, char *command, char **argv, int argc);
 int cmd_show_hist_idle(struct cli_def *cli, char *command, char **argv, int argc);
 int cmd_show_hist_open(struct cli_def *cli, char *command, char **argv, int argc);
+void netlink_addattr(struct nlmsghdr *nh, int type, const void *data, int alen);
+ssize_t netlink_send(struct nlmsghdr *nh);
+void cache_ipmap(in_addr_t ip, sessionidt s);
+
 #ifdef LAC
 tunnelidt lac_new_tunnel();
 void lac_tunnelclear(tunnelidt t);
@@ -954,6 +987,16 @@ void lac_send_ICRQ(tunnelidt t, sessionidt s);
 void lac_tunnelshutdown(tunnelidt t, char *reason, int result, int error, char *msg);
 #endif
 
+// grpsess.c
+sessionidt grp_getnextsession(groupidt g, in_addr_t ip);
+void grp_initdata(void);
+void grp_processvendorspecific(sessionidt s, uint8_t *pvs);
+groupidt grp_groupbysession(sessionidt s);
+groupidt grp_groupbyip(in_addr_t ip);
+void grp_setgrouproute(groupidt g, int add);
+void grp_time_changed(void);
+void grp_removesession(groupidt g, sessionidt s);
+
 #undef LOG
 #undef LOG_HEX
 #define LOG(D, s, t, f, ...)	({ if (D <= config->debug) _log(D, s, t, f, ## __VA_ARGS__); })
@@ -988,6 +1031,7 @@ extern bundlet *bundle;
 extern sessiont *session;
 extern sessionlocalt *sess_local;
 extern ippoolt *ip_address_pool;
+extern groupsesst *grpsession;
 #define sessionfree (session[0].next)
 
 
@@ -1000,6 +1044,7 @@ extern struct Tstats *_statistics;
 extern in_addr_t my_address;
 extern int clifd;
 extern int epollfd;
+extern int tunidx;		// ifr_ifindex of tun device
 
 struct event_data {
 	enum {
diff --git a/nsctl.c b/nsctl.c
index 59ad0c6..98785cd 100644
--- a/nsctl.c
+++ b/nsctl.c
@@ -7,6 +7,8 @@
 #include <string.h>
 #include <netdb.h>
 #include <signal.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #include "control.h"
diff --git a/ppp.c b/ppp.c
index 893ceb9..5c4e51b 100644
--- a/ppp.c
+++ b/ppp.c
@@ -5,6 +5,9 @@
 #include <unistd.h>
 #include <errno.h>
 #include <stdlib.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "constants.h"
 #include "plugin.h"
diff --git a/pppoe.c b/pppoe.c
index 4b196ea..ca1c5e4 100644
--- a/pppoe.c
+++ b/pppoe.c
@@ -21,6 +21,7 @@
 #include <netpacket/packet.h>
 #include <arpa/inet.h>
 #include <linux/if_pppox.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #include "cluster.h"
@@ -956,6 +957,7 @@ static void pppoe_forwardto_session_rmlns(uint8_t *pack, int size, sessionidt se
 		// Update STAT OUT
 		increment_counter(&session[s].cout, &session[s].cout_wrap, ll2tp); // byte count
 		session[s].cout_delta += ll2tp;
+		session[s].coutgrp_delta += ll2tp;
 		session[s].pout++;
 		sess_local[s].cout += ll2tp;
 		sess_local[s].pout++;
@@ -1027,6 +1029,7 @@ void pppoe_forwardto_session_pppoe(uint8_t *pack, int size, sessionidt sess, uin
 		// Update STAT OUT
 		increment_counter(&session[s].cout, &session[s].cout_wrap, lpppoe); // byte count
 		session[s].cout_delta += lpppoe;
+		session[s].coutgrp_delta += lpppoe;
 		session[s].pout++;
 		sess_local[s].cout += lpppoe;
 		sess_local[s].pout++;
diff --git a/radius.c b/radius.c
index 34806c5..6b6be70 100644
--- a/radius.c
+++ b/radius.c
@@ -11,6 +11,7 @@
 #include <ctype.h>
 #include <netinet/in.h>
 #include <errno.h>
+#include <linux/rtnetlink.h>
 
 #include "md5.h"
 #include "constants.h"
@@ -668,11 +669,19 @@ void processrad(uint8_t *buf, int len, char socket_index)
 						else if (vendor == 529 && attrib >= 135 && attrib <= 136) // Ascend
 						{
 							// handle old-format ascend DNS attributes below
-						    	p += 6;
+							p += 6;
+						}
+						else if (vendor == 64520) // Sames
+						{
+							//Sames vendor-specific 64520
+							uint8_t *pvs = p + 6; // pvs set to begin to attribute
+							LOG(3, s, session[s].tunnel, "   Sames vendor-specific: %d, Attrib: %d, lenght: %d\n", vendor, attrib, attrib_length);
+							grp_processvendorspecific(s, pvs);
+							continue;
 						}
 						else
 						{
-							LOG(3, s, session[s].tunnel, "      Unknown vendor-specific\n");
+							LOG(3, s, session[s].tunnel, "   Unknown vendor-specific: %d, Attrib: %d\n", vendor, attrib);
 							continue;
 						}
 					}
diff --git a/sessionctl.c b/sessionctl.c
index 805b794..a8c11bd 100644
--- a/sessionctl.c
+++ b/sessionctl.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 #include "control.h"
diff --git a/setrxspeed.c b/setrxspeed.c
index 52fcf4a..f987378 100644
--- a/setrxspeed.c
+++ b/setrxspeed.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 
diff --git a/snoopctl.c b/snoopctl.c
index d538a23..97d4bd0 100644
--- a/snoopctl.c
+++ b/snoopctl.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 #include "control.h"
diff --git a/stripdomain.c b/stripdomain.c
index 877617e..c251dba 100644
--- a/stripdomain.c
+++ b/stripdomain.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 
diff --git a/tbf.c b/tbf.c
index c6273ce..8b488d6 100644
--- a/tbf.c
+++ b/tbf.c
@@ -1,6 +1,9 @@
 // L2TPNS: token bucket filters
 
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "util.h"
 #include "tbf.h"
diff --git a/throttlectl.c b/throttlectl.c
index adcff27..bcc0478 100644
--- a/throttlectl.c
+++ b/throttlectl.c
@@ -1,4 +1,7 @@
 #include <string.h>
+#include <sys/socket.h>
+#include <linux/rtnetlink.h>
+
 #include "l2tpns.h"
 #include "plugin.h"
 #include "control.h"
diff --git a/util.c b/util.c
index d4dbd4f..65f8dc3 100644
--- a/util.c
+++ b/util.c
@@ -8,6 +8,7 @@
 #include <arpa/inet.h>
 #include <string.h>
 #include <sys/mman.h>
+#include <linux/rtnetlink.h>
 
 #include "l2tpns.h"
 #ifdef BGP