From bd2ec601493cece0de9e565e7bd497b129bece10 Mon Sep 17 00:00:00 2001
From: Brendan O'Dea <bod@optus.net>
Date: Thu, 2 Jun 2005 11:32:30 +0000
Subject: [PATCH 1/1] merge in changes from 2.0 branch; fix byte counters in
 accounting records, add gigawords

---
 Changes               |   7 ++
 Docs/manual.html      |   4 -
 Docs/startup-config.5 |  10 +-
 cli.c                 |   8 +-
 cluster.c             |  69 ++++++----
 cluster.h             |   8 +-
 garden.c              |   6 +-
 l2tpns.c              | 283 +++++++++++++++++++++++++++---------------
 l2tpns.h              |  18 ++-
 ppp.c                 |  57 +++++----
 radius.c              |  49 +++++---
 11 files changed, 331 insertions(+), 188 deletions(-)

diff --git a/Changes b/Changes
index 4b07b11..6fc5543 100644
--- a/Changes
+++ b/Changes
@@ -79,6 +79,13 @@
   routes.
 - New config option: allow_duplicate_users which determines whether
   or not to kill older sessions with the same username.
+- Show session open time in "show session"/"show user" detailed output.
+- Have slaves with BGP configured drop BGP on receipt of a shutdown
+  signal, but hang about for an additional 5s to process any remaining
+  traffic.
+- Run regular_cleanups after processing the results of the select,
+  looking at a sufficient slice of each table to ensure that all
+  entries are examined at least once per second.
 
 * Fri Dec 17 2004 Brendan O'Dea <bod@optusnet.com.au> 2.0.13
 - Better cluster master collision resolution: keep a counter of state
diff --git a/Docs/manual.html b/Docs/manual.html
index ad4bbec..4db3a83 100644
--- a/Docs/manual.html
+++ b/Docs/manual.html
@@ -285,10 +285,6 @@ second.  Even if this is disabled, you can see this information by running
 the <EM>uptime</EM> command on the CLI.
 </LI>
 
-<LI><B>cleanup_interval</B> (int)<BR>
-Interval between regular cleanups (in seconds).
-</LI>
-
 <LI><B>multi_read_count</B> (int)<BR>
 Number of packets to read off each of the UDP and TUN fds when
 returned as readable by select (default:  10).  Avoids incurring the
diff --git a/Docs/startup-config.5 b/Docs/startup-config.5
index cf70f50..61c2dc2 100644
--- a/Docs/startup-config.5
+++ b/Docs/startup-config.5
@@ -2,7 +2,7 @@
 .de Id
 .ds Dt \\$4 \\$5
 ..
-.Id $Id: startup-config.5,v 1.9 2005-06-02 04:04:08 bodea Exp $
+.Id $Id: startup-config.5,v 1.10 2005-06-02 11:32:33 bodea Exp $
 .TH STARTUP-CONFIG 5 "\*(Dt" L2TPNS "File Formats and Conventions"
 .SH NAME
 startup\-config \- configuration file for l2tpns
@@ -91,10 +91,7 @@ record when the session is closed.
 If
 .B radius_accounting
 is on, defines the interval between sending of RADIUS interim
-accounting records (in seconds).  Note: checking of this interval
-occurs no more frequently than
-.B cleanup_interval
-seconds (see below).
+accounting records (in seconds).
 .TP
 .B radius_secret
 Secret to be used in RADIUS packets.
@@ -145,9 +142,6 @@ by running the
 .B uptime
 command on the CLI.
 .TP
-.B cleanup_interval
-Interval between regular cleanups (in seconds).
-.TP
 .B multi_read_count
 Number of packets to read off each of the UDP and TUN fds when
 returned as readable by select (default: 10).  Avoids incurring the
diff --git a/cli.c b/cli.c
index 61a4419..bf0987f 100644
--- a/cli.c
+++ b/cli.c
@@ -2,7 +2,7 @@
 // vim: sw=8 ts=8
 
 char const *cvs_name = "$Name:  $";
-char const *cvs_id_cli = "$Id: cli.c,v 1.59 2005-06-02 03:52:46 bodea Exp $";
+char const *cvs_id_cli = "$Id: cli.c,v 1.60 2005-06-02 11:32:30 bodea Exp $";
 
 #include <stdio.h>
 #include <stdarg.h>
@@ -411,7 +411,7 @@ static int cmd_show_session(struct cli_def *cli, char *command, char **argv, int
 			cli_print(cli, "\tIdle time:\t%u seconds", abs(time_now - session[s].last_packet));
 			cli_print(cli, "\tNext Recv:\t%u", session[s].nr);
 			cli_print(cli, "\tNext Send:\t%u", session[s].ns);
-			cli_print(cli, "\tBytes In/Out:\t%u/%u", session[s].total_cout, session[s].total_cin);
+			cli_print(cli, "\tBytes In/Out:\t%u/%u", session[s].cout, session[s].cin);
 			cli_print(cli, "\tPkts In/Out:\t%u/%u", session[s].pout, session[s].pin);
 			cli_print(cli, "\tMRU:\t\t%d", session[s].mru);
 			cli_print(cli, "\tRx Speed:\t%u", session[s].rx_connect_speed);
@@ -507,8 +507,8 @@ static int cmd_show_session(struct cli_def *cli, char *command, char **argv, int
 				(session[i].walled_garden) ? "Y" : "N",
 				(session[i].flags & SF_IPV6CP_ACKED) ? "Y" : "N",
 				abs(time_now - (unsigned long)session[i].opened),
-				(unsigned long)session[i].total_cout,
-				(unsigned long)session[i].total_cin,
+				(unsigned long)session[i].cout,
+				(unsigned long)session[i].cin,
 				abs(time_now - (session[i].last_packet ? session[i].last_packet : time_now)),
 				fmtaddr(htonl(tunnel[ session[i].tunnel ].ip), 1),
 				session[i].calling[0] ? session[i].calling : "*");
diff --git a/cluster.c b/cluster.c
index a9e8209..8444b78 100644
--- a/cluster.c
+++ b/cluster.c
@@ -1,6 +1,6 @@
 // L2TPNS Clustering Stuff
 
-char const *cvs_id_cluster = "$Id: cluster.c,v 1.39 2005-05-26 12:17:30 bodea Exp $";
+char const *cvs_id_cluster = "$Id: cluster.c,v 1.40 2005-06-02 11:32:30 bodea Exp $";
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -287,7 +287,7 @@ int master_forward_packet(char *data, int size, in_addr_t addr, int port)
 	LOG(4, 0, 0, "Forwarding packet from %s to master (size %d)\n", fmtaddr(addr, 0), size);
 
 	STAT(c_forwarded);
-	add_type(&p, C_FORWARD, addr, (char *) &port, sizeof(port));
+	add_type(&p, C_FORWARD, addr, (char *) &port, sizeof(port)); // ick. should be uint16_t
 	memcpy(p, data, size);
 	p += size;
 
@@ -402,10 +402,16 @@ void master_update_counts(void)
 	if (config->cluster_iam_master)		// Only happens on the slaves.
 		return;
 
-	if (!config->cluster_master_address)		// If we don't have a master, skip it for a while.
+	if (!config->cluster_master_address)	// If we don't have a master, skip it for a while.
 		return;
 
-	i = MAX_B_RECS * 5; // Examine max 2000 sessions;
+	// C_BYTES format changed in 2.1.0 (cluster version 5)
+	// during upgrade from previous versions, hang onto our counters
+	// for a bit until the new master comes up
+	if (config->cluster_last_hb_ver < 5)
+		return;
+
+	i = MAX_B_RECS * 5; // Examine max 3000 sessions;
 	if (config->cluster_highest_sessionid > i)
 		i = config->cluster_highest_sessionid;
 
@@ -416,17 +422,20 @@ void master_update_counts(void)
 			walk_session_number = 1;
 
 		if (!sess_local[walk_session_number].cin && !sess_local[walk_session_number].cout)
-			continue; // Unused. Skip it.
+			continue; // Unchanged. Skip it.
 
 		b[c].sid = walk_session_number;
-		b[c].in = sess_local[walk_session_number].cin;
-		b[c].out = sess_local[walk_session_number].cout;
-
-		if (++c > MAX_B_RECS)	// Send a max of 400 elements in a packet.
-			break;
+		b[c].pin = sess_local[walk_session_number].pin;
+		b[c].pout = sess_local[walk_session_number].pout;
+		b[c].cin = sess_local[walk_session_number].cin;
+		b[c].cout = sess_local[walk_session_number].cout;
 
 			// Reset counters.
+		sess_local[walk_session_number].pin = sess_local[walk_session_number].pout = 0;
 		sess_local[walk_session_number].cin = sess_local[walk_session_number].cout = 0;
+
+		if (++c > MAX_B_RECS)	// Send a max of 600 elements in a packet.
+			break;
 	}
 
 	if (!c)		// Didn't find any that changes. Get out of here!
@@ -527,6 +536,10 @@ void cluster_check_master(void)
 
 	LOG(0, 0, 0, "Master timed out! Holding election...\n");
 
+	// In the process of shutting down, can't be master
+	if (main_quit)
+		return;
+
 	for (i = have_peers = 0; i < num_peers; i++)
 	{
 		if ((peers[i].timestamp + config->cluster_hb_timeout) < t)
@@ -610,13 +623,17 @@ void cluster_check_master(void)
 			// Reset die relative to our uptime rather than the old master's
 		if (session[i].die) session[i].die = TIME;
 
-			// Accumulate un-sent byte counters.
-		session[i].cin += sess_local[i].cin;
-		session[i].cout += sess_local[i].cout;
-		session[i].total_cin += sess_local[i].cin;
-		session[i].total_cout += sess_local[i].cout;
+			// Accumulate un-sent byte/packet counters.
+		increment_counter(&session[i].cin, &session[i].cin_wrap, sess_local[i].cin);
+		increment_counter(&session[i].cout, &session[i].cout_wrap, sess_local[i].cout);
+		session[i].cin_delta += sess_local[i].cin;
+		session[i].cout_delta += sess_local[i].cout;
+
+		session[i].pin += sess_local[i].pin;
+		session[i].pout += sess_local[i].pout;
 
 		sess_local[i].cin = sess_local[i].cout = 0;
+		sess_local[i].pin = sess_local[i].pout = 0;
 
 		sess_local[i].radius = 0;	// Reset authentication as the radius blocks aren't up to date.
 
@@ -1078,13 +1095,16 @@ static int cluster_handle_bytes(char *data, int size)
 			return -1; /* Abort processing */
 		}
 
-		session[b->sid].total_cin += b->in;
-		session[b->sid].total_cout += b->out;
+		session[b->sid].pin += b->pin;
+		session[b->sid].pout += b->pout;
+
+		increment_counter(&session[b->sid].cin, &session[b->sid].cin_wrap, b->cin);
+		increment_counter(&session[b->sid].cout, &session[b->sid].cout_wrap, b->cout);
 
-		session[b->sid].cin += b->in;
-		session[b->sid].cout += b->out;
+		session[b->sid].cin_delta += b->cin;
+		session[b->sid].cout_delta += b->cout;
 
-		if (b->in)
+		if (b->cin)
 			session[b->sid].last_packet = time_now; // Reset idle timer!
 
 		size -= sizeof(*b);
@@ -1223,12 +1243,12 @@ static uint8_t *convert_session(struct oldsession *old)
 	new.nr = old->nr;
 	new.ns = old->ns;
 	new.magic = old->magic;
-	new.cin = old->cin;
-	new.cout = old->cout;
 	new.pin = old->pin;
 	new.pout = old->pout;
-	new.total_cin = old->total_cin;
-	new.total_cout = old->total_cout;
+	new.cin = old->total_cin;
+	new.cout = old->total_cout;
+	new.cin_delta = old->cin;
+	new.cout_delta = old->cout;
 	new.throttle_in = old->throttle_in;
 	new.throttle_out = old->throttle_out;
 	new.filter_in = old->filter_in;
@@ -1360,6 +1380,7 @@ static int cluster_process_heartbeat(uint8_t *data, int size, int more, uint8_t
 		config->cluster_seq_number = h->seq;
 
 	config->cluster_last_hb = TIME;	// Reset to ensure that we don't become master!!
+	config->cluster_last_hb_ver = hb_ver; // remember what cluster version the master is using
 
 	if (config->cluster_seq_number != h->seq) {	// Out of sequence heartbeat!
 		static int lastseen_seq = 0;
diff --git a/cluster.h b/cluster.h
index f4e2a40..00e8e9b 100644
--- a/cluster.h
+++ b/cluster.h
@@ -1,5 +1,5 @@
 // L2TPNS Clustering Stuff
-// $Id: cluster.h,v 1.11 2005-05-26 12:17:30 bodea Exp $
+// $Id: cluster.h,v 1.12 2005-06-02 11:32:30 bodea Exp $
 
 #ifndef __CLUSTER_H__
 #define __CLUSTER_H__
@@ -57,8 +57,10 @@ typedef struct {
 typedef struct {		/* Used to update byte counters on the */
 				/* master. */
 	uint32_t sid;
-	uint32_t in;
-	uint32_t out;
+	uint32_t pin;
+	uint32_t pout;
+	uint32_t cin;
+	uint32_t cout;
 } bytest;
 
 typedef struct {
diff --git a/garden.c b/garden.c
index b552ff2..07f1f41 100644
--- a/garden.c
+++ b/garden.c
@@ -9,7 +9,7 @@
 
 /* walled garden */
 
-char const *cvs_id = "$Id: garden.c,v 1.22 2005-05-07 08:17:25 bodea Exp $";
+char const *cvs_id = "$Id: garden.c,v 1.23 2005-06-02 11:32:30 bodea Exp $";
 
 int plugin_api_version = PLUGIN_API_VERSION;
 static struct pluginfuncs *p = 0;
@@ -206,8 +206,10 @@ int garden_session(sessiont *s, int flag, char *newuser)
 			p->sessionkill(other, "Duplicate session when user released from walled garden");
 		}
 		/* Clean up counters */
-		s->cin = s->cout = 0;
 		s->pin = s->pout = 0;
+		s->cin = s->cout = 0;
+		s->cin_delta = s->cout_delta = 0;
+		s->cin_wrap = s->cout_wrap = 0;
 
 		snprintf(cmd, sizeof(cmd), "iptables -t nat -D garden_users -s %s -j garden", p->fmtaddr(htonl(s->ip), 0));
 		p->log(3, sess, s->tunnel, "%s\n", cmd);
diff --git a/l2tpns.c b/l2tpns.c
index beb6df5..9942cf9 100644
--- a/l2tpns.c
+++ b/l2tpns.c
@@ -4,7 +4,7 @@
 // Copyright (c) 2002 FireBrick (Andrews & Arnold Ltd / Watchfront Ltd) - GPL licenced
 // vim: sw=8 ts=8
 
-char const *cvs_id_l2tpns = "$Id: l2tpns.c,v 1.106 2005-06-02 04:04:07 bodea Exp $";
+char const *cvs_id_l2tpns = "$Id: l2tpns.c,v 1.107 2005-06-02 11:32:30 bodea Exp $";
 
 #include <arpa/inet.h>
 #include <assert.h>
@@ -90,7 +90,7 @@ uint32_t eth_tx = 0;
 static uint32_t ip_pool_size = 1;	// Size of the pool of addresses used for dynamic address allocation.
 time_t time_now = 0;			// Current time in seconds since epoch.
 static char time_now_string[64] = {0};	// Current time as a string.
-static char main_quit = 0;		// True if we're in the process of exiting.
+char main_quit = 0;			// True if we're in the process of exiting.
 linked_list *loaded_plugins;
 linked_list *plugins[MAX_PLUGIN_TYPES];
 
@@ -122,7 +122,6 @@ config_descriptt config_values[] = {
 	CONFIG("accounting_dir", accounting_dir, STRING),
 	CONFIG("setuid", target_uid, INT),
 	CONFIG("dump_speed", dump_speed, BOOL),
-	CONFIG("cleanup_interval", cleanup_interval, INT),
 	CONFIG("multi_read_count", multi_read_count, INT),
 	CONFIG("scheduler_fifo", scheduler_fifo, BOOL),
 	CONFIG("lock_pages", lock_pages, BOOL),
@@ -191,11 +190,15 @@ static void processcontrol(uint8_t *buf, int len, struct sockaddr_in *addr, int
 static tunnelidt new_tunnel(void);
 static void unhide_value(uint8_t *value, size_t len, uint16_t type, uint8_t *vector, size_t vec_len);
 
-// return internal time (10ths since process startup)
-static clockt now(void)
+// on slaves, alow BGP to withdraw cleanly before exiting
+#define QUIT_DELAY	5
+
+// return internal time (10ths since process startup), set f if given
+static clockt now(double *f)
 {
 	struct timeval t;
 	gettimeofday(&t, 0);
+	if (f) *f = t.tv_sec + t.tv_usec / 1000000.0;
 	return (t.tv_sec - basetime) * 10 + t.tv_usec / 100000 + 1;
 }
 
@@ -205,7 +208,7 @@ static clockt now(void)
 clockt backoff(uint8_t try)
 {
 	if (try > 5) try = 5;                  // max backoff
-	return now() + 10 * (1 << try);
+	return now(NULL) + 10 * (1 << try);
 }
 
 
@@ -299,6 +302,16 @@ void _log_hex(int level, const char *title, const char *data, int maxsize)
 	}
 }
 
+// update a counter, accumulating 2^32 wraps
+void increment_counter(uint32_t *counter, uint32_t *wrap, uint32_t delta)
+{
+	uint32_t new = *counter + delta;
+	if (new < *counter)
+		(*wrap)++;
+
+	*counter = new;
+}
+
 // initialise the random generator
 static void initrandom(char *source)
 {
@@ -1078,11 +1091,13 @@ static void processipout(uint8_t * buf, int len)
 	if (sp->snoop_ip && sp->snoop_port)
 		snoop_send_packet(buf, len, sp->snoop_ip, sp->snoop_port);
 
-	sp->cout += len; // byte count
-	sp->total_cout += len; // byte count
+	increment_counter(&sp->cout, &sp->cout_wrap, len); // byte count
+	sp->cout_delta += len;
 	sp->pout++;
 	udp_tx += len;
+
 	sess_local[s].cout += len;	// To send to master..
+	sess_local[s].pout++;
 }
 
 // process outgoing (to tunnel) IPv6
@@ -1187,11 +1202,13 @@ static void processipv6out(uint8_t * buf, int len)
 	if (sp->snoop_ip && sp->snoop_port)
 		snoop_send_packet(buf, len, sp->snoop_ip, sp->snoop_port);
 
-	sp->cout += len; // byte count
-	sp->total_cout += len; // byte count
+	increment_counter(&sp->cout, &sp->cout_wrap, len); // byte count
+	sp->cout_delta += len;
 	sp->pout++;
 	udp_tx += len;
+
 	sess_local[s].cout += len;	// To send to master..
+	sess_local[s].pout++;
 }
 
 //
@@ -1237,11 +1254,13 @@ static void send_ipout(sessionidt s, uint8_t *buf, int len)
 	if (sp->snoop_ip && sp->snoop_port)
 		snoop_send_packet(buf, len, sp->snoop_ip, sp->snoop_port);
 
-	sp->cout += len; // byte count
-	sp->total_cout += len; // byte count
+	increment_counter(&sp->cout, &sp->cout_wrap, len); // byte count
+	sp->cout_delta += len;
 	sp->pout++;
 	udp_tx += len;
+
 	sess_local[s].cout += len;	// To send to master..
+	sess_local[s].pout++;
 }
 
 // add an AVP (16 bit)
@@ -2552,42 +2571,61 @@ static void processtun(uint8_t * buf, int len)
 	// Else discard.
 }
 
-//
-// Maximum number of actions to complete.
-// This is to avoid sending out too many packets
-// at once.
-#define MAX_ACTIONS 500
-
-static int regular_cleanups(void)
+// Handle retries, timeouts.  Runs every 1/10th sec, want to ensure
+// that we look at the whole of the tunnel, radius and session tables
+// every second
+static void regular_cleanups(double period)
 {
-	static sessionidt s = 0;	// Next session to check for actions on.
-	tunnelidt t;
-	int count=0,i;
-	uint16_t r;
-	static clockt next_acct = 0;
-	static clockt next_shut_acct = 0;
+	// Next tunnel, radius and session to check for actions on.
+	static tunnelidt t = 0;
+	static int r = 0;
+	static sessionidt s = 0;
+
+	int t_actions = 0;
+	int r_actions = 0;
+	int s_actions = 0;
+
+	int t_slice;
+	int r_slice;
+	int s_slice;
+
+	int i;
 	int a;
 
-	LOG(3, 0, 0, "Begin regular cleanup\n");
+	// divide up tables into slices based on the last run
+	t_slice = config->cluster_highest_tunnelid  * period;
+	r_slice = (MAXRADIUS - 1)                   * period;
+	s_slice = config->cluster_highest_sessionid * period;
 
-	for (r = 1; r < MAXRADIUS; r++)
-	{
-		if (!radius[r].state)
-			continue;
-		if (radius[r].retry)
-		{
-			if (radius[r].retry <= TIME)
-				radiusretry(r);
-		} else
-			radius[r].retry = backoff(radius[r].try+1);	// Is this really needed? --mo
-	}
-	for (t = 1; t <= config->cluster_highest_tunnelid; t++)
+	if (t_slice < 1)
+	    t_slice = 1;
+	else if (t_slice > config->cluster_highest_tunnelid)
+	    t_slice = config->cluster_highest_tunnelid;
+
+	if (r_slice < 1)
+	    r_slice = 1;
+	else if (r_slice > (MAXRADIUS - 1))
+	    r_slice = MAXRADIUS - 1;
+
+	if (s_slice < 1)
+	    s_slice = 1;
+	else if (s_slice > config->cluster_highest_sessionid)
+	    s_slice = config->cluster_highest_sessionid;
+
+	LOG(4, 0, 0, "Begin regular cleanup (last %f seconds ago)\n", period);
+
+	for (i = 0; i < t_slice; i++)
 	{
+		t++;
+		if (t > config->cluster_highest_tunnelid)
+			t = 1;
+
 		// check for expired tunnels
 		if (tunnel[t].die && tunnel[t].die <= TIME)
 		{
 			STAT(tunnel_timeout);
 			tunnelkill(t, "Expired");
+			t_actions++;
 			continue;
 		}
 		// check for message resend
@@ -2607,6 +2645,8 @@ static int regular_cleanups(void)
 						tunnelsend(c->buf, c->length, t);
 						c = c->next;
 					}
+
+				t_actions++;
 			}
 		}
 		// Send hello
@@ -2615,6 +2655,7 @@ static int regular_cleanups(void)
 			controlt *c = controlnew(6); // sending HELLO
 			controladd(c, t, 0); // send the message
 			LOG(3, 0, t, "Sending HELLO message\n");
+			t_actions++;
 		}
 
 		// Check for tunnel changes requested from the CLI
@@ -2625,13 +2666,28 @@ static int regular_cleanups(void)
 			{
 				LOG(2, 0, t, "Dropping tunnel by CLI\n");
 				tunnelshutdown(t, "Requested by administrator", 1, 0, 0);
+				t_actions++;
 			}
 		}
+	}
+
+	for (i = 0; i < r_slice; i++)
+	{
+		r++;
+		if (r >= MAXRADIUS)
+			r = 1;
+
+		if (!radius[r].state)
+			continue;
 
+		if (radius[r].retry <= TIME)
+		{
+			radiusretry(r);
+			r_actions++;
+		}
 	}
 
-	count = 0;
-	for (i = 1; i <= config->cluster_highest_sessionid; i++)
+	for (i = 0; i < s_slice; i++)
 	{
 		s++;
 		if (s > config->cluster_highest_sessionid)
@@ -2646,7 +2702,7 @@ static int regular_cleanups(void)
 			if (session[s].die <= TIME)
 			{
 				sessionkill(s, "Expired");
-				if (++count >= MAX_ACTIONS) break;
+				s_actions++;
 			}
 			continue;
 		}
@@ -2656,6 +2712,7 @@ static int regular_cleanups(void)
 			// IPCP has not completed yet. Resend
 			LOG(3, s, session[s].tunnel, "No ACK for initial IPCP ConfigReq... resending\n");
 			sendipcp(session[s].tunnel, s);
+			s_actions++;
 		}
 
 		// Drop sessions who have not responded within IDLE_TIMEOUT seconds
@@ -2663,7 +2720,7 @@ static int regular_cleanups(void)
 		{
 			sessionshutdown(s, "No response to LCP ECHO requests.", 3, 0);
 			STAT(session_timeout);
-			if (++count >= MAX_ACTIONS) break;
+			s_actions++;
 			continue;
 		}
 
@@ -2683,7 +2740,7 @@ static int regular_cleanups(void)
 			LOG(4, s, session[s].tunnel, "No data in %d seconds, sending LCP ECHO\n",
 					(int)(time_now - session[s].last_packet));
 			tunnelsend(b, 24, session[s].tunnel); // send it
-			if (++count >= MAX_ACTIONS) break;
+			s_actions++;
 		}
 
 		// Check for actions requested from the CLI
@@ -2697,6 +2754,7 @@ static int regular_cleanups(void)
 				LOG(2, s, session[s].tunnel, "Dropping session by CLI\n");
 				sessionshutdown(s, "Requested by administrator.", 3, 0);
 				a = 0; // dead, no need to check for other actions
+				s_actions++;
 			}
 
 			if (a & CLI_SESS_NOSNOOP)
@@ -2704,6 +2762,7 @@ static int regular_cleanups(void)
 				LOG(2, s, session[s].tunnel, "Unsnooping session by CLI\n");
 				session[s].snoop_ip = 0;
 				session[s].snoop_port = 0;
+				s_actions++;
 				send++;
 			}
 			else if (a & CLI_SESS_SNOOP)
@@ -2714,6 +2773,7 @@ static int regular_cleanups(void)
 
 				session[s].snoop_ip = cli_session_actions[s].snoop_ip;
 				session[s].snoop_port = cli_session_actions[s].snoop_port;
+				s_actions++;
 				send++;
 			}
 
@@ -2721,6 +2781,7 @@ static int regular_cleanups(void)
 			{
 				LOG(2, s, session[s].tunnel, "Un-throttling session by CLI\n");
 				throttle_session(s, 0, 0);
+				s_actions++;
 				send++;
 			}
 			else if (a & CLI_SESS_THROTTLE)
@@ -2730,6 +2791,7 @@ static int regular_cleanups(void)
 				    cli_session_actions[s].throttle_out);
 
 				throttle_session(s, cli_session_actions[s].throttle_in, cli_session_actions[s].throttle_out);
+				s_actions++;
 				send++;
 			}
 
@@ -2737,6 +2799,7 @@ static int regular_cleanups(void)
 			{
 				LOG(2, s, session[s].tunnel, "Un-filtering session by CLI\n");
 				filter_session(s, 0, 0);
+				s_actions++;
 				send++;
 			}
 			else if (a & CLI_SESS_FILTER)
@@ -2746,13 +2809,12 @@ static int regular_cleanups(void)
 				    cli_session_actions[s].filter_out);
 
 				filter_session(s, cli_session_actions[s].filter_in, cli_session_actions[s].filter_out);
+				s_actions++;
 				send++;
 			}
 
 			if (send)
 				cluster_send_session(s);
-
-			if (++count >= MAX_ACTIONS) break;
 		}
 
 		// RADIUS interim accounting
@@ -2773,38 +2835,14 @@ static int regular_cleanups(void)
 
 			radiussend(r, RADIUSINTERIM);
 			sess_local[s].last_interim = time_now;
-
-			if (++count >= MAX_ACTIONS)
-				break;
-		}
-	}
-
-	if (*config->accounting_dir)
-	{
-		if (next_acct <= TIME)
-		{
-			// Dump accounting data
-			next_acct = TIME + ACCT_TIME;
-			next_shut_acct = TIME + ACCT_SHUT_TIME;
-			dump_acct_info(1);
-		}
-		else if (next_shut_acct <= TIME)
-		{
-			// Dump accounting data for shutdown sessions
-			next_shut_acct = TIME + ACCT_SHUT_TIME;
-			if (shut_acct_n)
-				dump_acct_info(0);
+			s_actions++;
 		}
 	}
 
-	if (count >= MAX_ACTIONS)
-		return 1;	// Didn't finish!
-
-	LOG(3, 0, 0, "End regular cleanup (%d actions), next in %d seconds\n", count, config->cleanup_interval);
-	return 0;
+	LOG(4, 0, 0, "End regular cleanup: checked %d/%d/%d tunnels/radius/sessions; %d/%d/%d actions\n",
+		t_slice, r_slice, s_slice, t_actions, r_actions, s_actions);
 }
 
-
 //
 // Are we in the middle of a tunnel update, or radius
 // requests??
@@ -2812,8 +2850,39 @@ static int regular_cleanups(void)
 static int still_busy(void)
 {
 	int i;
+	static time_t stopped_bgp = 0;
 	static clockt last_talked = 0;
 	static clockt start_busy_wait = 0;
+
+	if (!config->cluster_iam_master)
+	{
+#ifdef BGP
+	    	if (bgp_configured)
+		{
+			if (!stopped_bgp)
+			{
+			    	LOG(1, 0, 0, "Shutting down in %d seconds, stopping BGP...\n", QUIT_DELAY);
+
+				for (i = 0; i < BGP_NUM_PEERS; i++)
+					if (bgp_peers[i].state == Established)
+						bgp_stop(&bgp_peers[i]);
+
+				stopped_bgp = time_now;
+
+				// we don't want to become master
+				cluster_send_ping(0);
+
+				return 1;
+			}
+
+			if (time_now < (stopped_bgp + QUIT_DELAY))
+				return 1;
+		}
+#endif /* BGP */
+
+		return 0;
+	}
+
 	if (start_busy_wait == 0)
 		start_busy_wait = TIME;
 
@@ -2865,7 +2934,6 @@ static void mainloop(void)
 	uint8_t buf[65536];
 	struct timeval to;
 	clockt next_cluster_ping = 0;	// send initial ping immediately
-	time_t next_clean = time_now + config->cleanup_interval;
 
 	LOG(4, 0, 0, "Beginning of main loop.  udpfd=%d, tunfd=%d, cluster_sockfd=%d, controlfd=%d\n",
 		udpfd, tunfd, cluster_sockfd, controlfd);
@@ -2890,6 +2958,7 @@ static void mainloop(void)
 		fd_set w;
 		int bgp_set[BGP_NUM_PEERS];
 #endif /* BGP */
+		int more = 0;
 
 		if (config->reload_config)
 		{
@@ -2928,7 +2997,7 @@ static void mainloop(void)
 
 		STAT(select_called);
 
-		TIME = now();
+		TIME = now(NULL);
 		if (n < 0)
 		{
 			if (errno == EINTR ||
@@ -3055,6 +3124,7 @@ static void mainloop(void)
 					config->multi_read_count, udp_pkts, tun_pkts, cluster_pkts);
 
 				STAT(multi_read_exceeded);
+				more++;
 			}
 		}
 
@@ -3077,9 +3147,11 @@ static void mainloop(void)
 				next_cluster_ping = TIME + config->cluster_hb_interval;
 		}
 
+		if (!config->cluster_iam_master)
+			continue;
+
 			// Run token bucket filtering queue..
 			// Only run it every 1/10th of a second.
-			// Runs on all machines both master and slave.
 		{
 			static clockt last_run = 0;
 			if (last_run != TIME)
@@ -3089,20 +3161,42 @@ static void mainloop(void)
 			}
 		}
 
-		/* Handle timeouts. Make sure that this gets run anyway, even if there was
-		 * something to read, else under load this will never actually run....
-		 *
-		 */
-		if (config->cluster_iam_master && next_clean <= time_now)
+			// Handle timeouts, retries etc.
 		{
-			if (regular_cleanups())
+			static double last_clean = 0;
+			double this_clean;
+			double diff;
+
+			TIME = now(&this_clean);
+			diff = this_clean - last_clean;
+
+			// Run during idle time (after we've handled
+			// all incoming packets) or every 1/10th sec
+			if (!more || diff > 0.1)
 			{
-				// Did it finish?
-				next_clean = time_now + 1 ;	// Didn't finish. Check quickly.
+				regular_cleanups(diff);
+				last_clean = this_clean;
 			}
-			else
+		}
+
+		if (*config->accounting_dir)
+		{
+			static clockt next_acct = 0;
+			static clockt next_shut_acct = 0;
+
+			if (next_acct <= TIME)
+			{
+				// Dump accounting data
+				next_acct = TIME + ACCT_TIME;
+				next_shut_acct = TIME + ACCT_SHUT_TIME;
+				dump_acct_info(1);
+			}
+			else if (next_shut_acct <= TIME)
 			{
-				next_clean = time_now + config->cleanup_interval; // Did. Move to next interval.
+				// Dump accounting data for shutdown sessions
+				next_shut_acct = TIME + ACCT_SHUT_TIME;
+				if (shut_acct_n)
+					dump_acct_info(0);
 			}
 		}
 	}
@@ -3117,6 +3211,7 @@ static void mainloop(void)
 
 	//
 	// Important!!! We MUST not process any packets past this point!
+	LOG(1, 0, 0, "Clean shutdown complete\n");
 }
 
 static void stripdomain(char *host)
@@ -3591,7 +3686,7 @@ void snoop_send_packet(char *packet, uint16_t size, in_addr_t destination, uint1
 
 static int dump_session(FILE **f, sessiont *s)
 {
-	if (!s->opened || !s->ip || !(s->cin || s->cout) || !*s->user || s->walled_garden)
+	if (!s->opened || !s->ip || !(s->cin_delta || s->cout_delta) || !*s->user || s->walled_garden)
 		return 1;
 
 	if (!*f)
@@ -3625,11 +3720,10 @@ static int dump_session(FILE **f, sessiont *s)
 		s->user,						// username
 		fmtaddr(htonl(s->ip), 0),				// ip
 		(s->throttle_in || s->throttle_out) ? 2 : 1,		// qos
-		(uint32_t) s->cin,					// uptxoctets
-		(uint32_t) s->cout);					// downrxoctets
+		(uint32_t) s->cin_delta,				// uptxoctets
+		(uint32_t) s->cout_delta);				// downrxoctets
 
-	s->pin = s->cin = 0;
-	s->pout = s->cout = 0;
+	s->cin_delta = s->cout_delta = 0;
 
 	return 1;
 }
@@ -3800,14 +3894,6 @@ int main(int argc, char *argv[])
 
 	mainloop();
 
-#ifdef BGP
-	/* try to shut BGP down cleanly; with luck the sockets will be
-	   writable since we're out of the select */
-	for (i = 0; i < BGP_NUM_PEERS; i++)
-		if (bgp_peers[i].state == Established)
-			bgp_stop(&bgp_peers[i]);
-#endif /* BGP */
-
 	/* remove plugins (so cleanup code gets run) */
 	plugins_done();
 
@@ -4084,7 +4170,6 @@ static void update_config()
 	}
 
 	memcpy(config->old_plugins, config->plugins, sizeof(config->plugins));
-	if (!config->cleanup_interval) config->cleanup_interval = 10;
 	if (!config->multi_read_count) config->multi_read_count = 10;
 	if (!config->cluster_address) config->cluster_address = inet_addr(DEFAULT_MCAST_ADDR);
 	if (!*config->cluster_interface)
diff --git a/l2tpns.h b/l2tpns.h
index 423f105..040861c 100644
--- a/l2tpns.h
+++ b/l2tpns.h
@@ -1,5 +1,5 @@
 // L2TPNS Global Stuff
-// $Id: l2tpns.h,v 1.74 2005-06-02 04:04:07 bodea Exp $
+// $Id: l2tpns.h,v 1.75 2005-06-02 11:32:31 bodea Exp $
 
 #ifndef __L2TPNS_H__
 #define __L2TPNS_H__
@@ -173,10 +173,10 @@ typedef struct
 	uint16_t nr;			// next receive
 	uint16_t ns;			// next send
 	uint32_t magic;			// ppp magic number
-	uint32_t cin, cout;		// byte counts
 	uint32_t pin, pout;		// packet counts
-	uint32_t total_cin;		// This counter is never reset while a session is open
-	uint32_t total_cout;		// This counter is never reset while a session is open
+	uint32_t cin, cout;		// byte counts
+	uint32_t cin_wrap, cout_wrap;	// byte counter wrap count (RADIUS accounting giagawords)
+	uint32_t cin_delta, cout_delta;	// byte count changes (for dump_session())
 	uint16_t throttle_in;		// upstream throttle rate (kbps)
 	uint16_t throttle_out;		// downstream throttle rate
 	uint8_t filter_in;		// input filter index (to ip_filters[N-1]; 0 if none)
@@ -201,7 +201,7 @@ typedef struct
 	uint8_t walled_garden;		// is this session gardened?
 	uint8_t ipv6prefixlen;		// IPv6 route prefix length
 	struct in6_addr ipv6route;	// Static IPv6 route
-	char reserved[24];		// Space to expand structure without changing HB_VERSION
+	char reserved[16];		// Space to expand structure without changing HB_VERSION
 }
 sessiont;
 
@@ -217,6 +217,10 @@ sessiont;
 
 typedef struct
 {
+	// packet counters
+	uint32_t pin;
+	uint32_t pout;
+
 	// byte counters
 	uint32_t cin;
 	uint32_t cout;
@@ -437,7 +441,6 @@ typedef struct
 
 	char		config_file[128];
 	int		reload_config;			// flag to re-read config (set by cli)
-	int		cleanup_interval;		// interval between regular cleanups (in seconds)
 	int		multi_read_count;		// amount of packets to read per fd in processing loop
 
 	char		tundevice[10];			// tun device name
@@ -496,6 +499,7 @@ typedef struct
 	int		cluster_highest_sessionid;
 	int		cluster_highest_tunnelid;
 	clockt		cluster_last_hb;		// Last time we saw a heartbeat from the master.
+	int		cluster_last_hb_ver;		// Heartbeat version last seen from master
 	int		cluster_num_changes;		// Number of changes queued.
 
 	int		cluster_hb_interval;		// How often to send a heartbeat.
@@ -616,6 +620,7 @@ void route6set(sessionidt s, struct in6_addr ip, int prefixlen, int add);
 sessionidt sessionbyip(in_addr_t ip);
 sessionidt sessionbyipv6(struct in6_addr ip);
 sessionidt sessionbyuser(char *username);
+void increment_counter(uint32_t *counter, uint32_t *wrap, uint32_t delta);
 void random_data(uint8_t *buf, int len);
 void sessionkill(sessionidt s, char *reason);
 void sessionshutdown(sessionidt s, char *reason, int result, int error);
@@ -681,6 +686,7 @@ if (count++ < max) { \
 extern configt *config;
 extern time_t basetime;		// Time when this process started.
 extern time_t time_now;		// Seconds since EPOCH.
+extern char main_quit;
 extern uint32_t last_id;
 extern struct Tstats *_statistics;
 extern in_addr_t my_address;
diff --git a/ppp.c b/ppp.c
index 3408d86..38eb7ae 100644
--- a/ppp.c
+++ b/ppp.c
@@ -1,6 +1,6 @@
 // L2TPNS PPP Stuff
 
-char const *cvs_id_ppp = "$Id: ppp.c,v 1.61 2005-05-13 09:23:00 bodea Exp $";
+char const *cvs_id_ppp = "$Id: ppp.c,v 1.62 2005-06-02 11:32:31 bodea Exp $";
 
 #include <stdio.h>
 #include <string.h>
@@ -1015,21 +1015,26 @@ void processipin(tunnelidt t, sessionidt s, uint8_t *p, uint16_t l)
 		return;
 	}
 
+	p += 4;
+	l -= 4;
+
 	if (session[s].snoop_ip && session[s].snoop_port)
 	{
 		// Snooping this session
-		snoop_send_packet(p + 4, l - 4, session[s].snoop_ip, session[s].snoop_port);
+		snoop_send_packet(p, l, session[s].snoop_ip, session[s].snoop_port);
 	}
 
-	session[s].cin += l - 4;
-	session[s].total_cin += l - 4;
-	sess_local[s].cin += l - 4;
-
+	increment_counter(&session[s].cin, &session[s].cin_wrap, l);
+	session[s].cin_delta += l;
 	session[s].pin++;
-	eth_tx += l - 4;
+
+	sess_local[s].cin += l;
+	sess_local[s].pin++;
+
+	eth_tx += l;
 
 	STAT(tun_tx_packets);
-	INC_STAT(tun_tx_bytes, l - 4);
+	INC_STAT(tun_tx_bytes, l);
 }
 
 // process IPv6 packet received
@@ -1103,21 +1108,26 @@ void processipv6in(tunnelidt t, sessionidt s, uint8_t *p, uint16_t l)
 		return;
 	}
 
+	p += 4;
+	l -= 4;
+
 	if (session[s].snoop_ip && session[s].snoop_port)
 	{
 		// Snooping this session
-		snoop_send_packet(p + 4, l - 4, session[s].snoop_ip, session[s].snoop_port);
+		snoop_send_packet(p, l, session[s].snoop_ip, session[s].snoop_port);
 	}
 
-	session[s].cin += l - 4;
-	session[s].total_cin += l - 4;
-	sess_local[s].cin += l - 4;
-
+	increment_counter(&session[s].cin, &session[s].cin_wrap, l);
+	session[s].cin_delta += l;
 	session[s].pin++;
-	eth_tx += l - 4;
+
+	sess_local[s].cin += l;
+	sess_local[s].pin++;
+
+	eth_tx += l;
 
 	STAT(tun_tx_packets);
-	INC_STAT(tun_tx_bytes, l - 4);
+	INC_STAT(tun_tx_bytes, l);
 }
 
 //
@@ -1137,19 +1147,24 @@ void send_ipin(sessionidt s, uint8_t *buf, int len)
 		return;
 	}
 
+	buf += 4;
+	len -= 4;
+
 	if (session[s].snoop_ip && session[s].snoop_port)
 	{
 		// Snooping this session
-		snoop_send_packet(buf + 4, len - 4, session[s].snoop_ip, session[s].snoop_port);
+		snoop_send_packet(buf, len, session[s].snoop_ip, session[s].snoop_port);
 	}
 
 	// Increment packet counters
-	session[s].cin += len - 4;
-	session[s].total_cin += len - 4;
-	sess_local[s].cin += len - 4;
-
+	increment_counter(&session[s].cin, &session[s].cin_wrap, len);
+	session[s].cin_delta += len;
 	session[s].pin++;
-	eth_tx += len - 4;
+
+	sess_local[s].cin += len;
+	sess_local[s].pin++;
+
+	eth_tx += len;
 
 	STAT(tun_tx_packets);
 	INC_STAT(tun_tx_bytes, len - 4);
diff --git a/radius.c b/radius.c
index 2c5fe60..0569c20 100644
--- a/radius.c
+++ b/radius.c
@@ -1,6 +1,6 @@
 // L2TPNS Radius Stuff
 
-char const *cvs_id_radius = "$Id: radius.c,v 1.31 2005-05-16 04:51:16 bodea Exp $";
+char const *cvs_id_radius = "$Id: radius.c,v 1.32 2005-06-02 11:32:32 bodea Exp $";
 
 #include <time.h>
 #include <stdio.h>
@@ -132,7 +132,7 @@ void radiussend(uint16_t r, uint8_t state)
 		radius[r].try = 0;
 
 	radius[r].state = state;
-	radius[r].retry = backoff(radius[r].try++);
+	radius[r].retry = backoff(radius[r].try++) + 20; // 3s, 4s, 6s, 10s...
 	LOG(4, s, session[s].tunnel, "Send RADIUS id %d sock %d state %s try %d\n",
 		r >> RADIUS_SHIFT, r & RADIUS_MASK,
 		radius_state(radius[r].state), radius[r].try);
@@ -233,47 +233,63 @@ void radiussend(uint16_t r, uint8_t state)
 		}
 	}
 	else if (state == RADIUSSTART || state == RADIUSSTOP || state == RADIUSINTERIM)
-	{                          // accounting
-		*p = 40;                // accounting type
+	{			// accounting
+		*p = 40;	// accounting type
 		p[1] = 6;
 		*(uint32_t *) (p + 2) = htonl(state - RADIUSSTART + 1); // start=1, stop=2, interim=3
 		p += p[1];
 		if (s)
 		{
-			*p = 44;           // session ID
+			*p = 44;	// session ID
 			p[1] = 18;
 			sprintf(p + 2, "%08X%08X", session[s].unique_id, session[s].opened);
 			p += p[1];
 			if (state == RADIUSSTART)
-			{                // start
-				*p = 41;      // delay
+			{			// start
+				*p = 41;	// delay
 				p[1] = 6;
 				*(uint32_t *) (p + 2) = htonl(time(NULL) - session[s].opened);
 				p += p[1];
 				sess_local[s].last_interim = time_now; // Setup "first" Interim
 			}
 			else
-			{                // stop, interim
-				*p = 42;      // input octets
+			{			// stop, interim
+				*p = 42;	// input octets
 				p[1] = 6;
 				*(uint32_t *) (p + 2) = htonl(session[s].cin);
 				p += p[1];
-				*p = 43;      // output octets
+
+				*p = 43;	// output octets
 				p[1] = 6;
 				*(uint32_t *) (p + 2) = htonl(session[s].cout);
 				p += p[1];
-				*p = 46;      // session time
-				p[1] = 6;
-				*(uint32_t *) (p + 2) = htonl(time(NULL) - session[s].opened);
-				p += p[1];
-				*p = 47;      // input packets
+				if (state == RADIUSSTOP)
+				{
+					*p = 46;	// session time
+					p[1] = 6;
+					*(uint32_t *) (p + 2) = htonl(time(NULL) - session[s].opened);
+					p += p[1];
+				}
+
+				*p = 47;	// input packets
 				p[1] = 6;
 				*(uint32_t *) (p + 2) = htonl(session[s].pin);
 				p += p[1];
-				*p = 48;      // output spackets
+
+				*p = 48;	// output packets
 				p[1] = 6;
 				*(uint32_t *) (p + 2) = htonl(session[s].pout);
 				p += p[1];
+
+				*p = 52;	// input gigawords
+				p[1] = 6;
+				*(uint32_t *) (p + 2) = htonl(session[s].cin_wrap);
+				p += p[1];
+
+				*p = 53;	// output gigawords
+				p[1] = 6;
+				*(uint32_t *) (p + 2) = htonl(session[s].cout_wrap);
+				p += p[1];
 			}
 
 			if (session[s].snoop_ip && session[s].snoop_port)
@@ -711,7 +727,6 @@ void radiusretry(uint16_t r)
 
 	if (s) t = session[s].tunnel;
 
-	radius[r].retry = backoff(radius[r].try + 1);
 	switch (radius[r].state)
 	{
 		case RADIUSCHAP:	// sending CHAP down PPP
-- 
2.20.1