better cluster master collision resolution

diff --git a/Changes b/Changes
index 8a7cd21..676e2aa 100644 (file)
--- a/Changes
+++ b/Changes
@@ -1,3 +1,9 @@
+* Fri Dec 3 2004 Brendan O'Dea <bod@optusnet.com.au> 2.0.13
+- Better cluster master collision resolution: keep a counter of state
+  changes, propagated in the heartbeats; the master with the highest #
+  of changes (that has kept in contact with the LAC through the
+  outage) prevails.
+
 * Wed Dec 1 2004 Brendan O'Dea <bod@optusnet.com.au> 2.0.12
 - The "This time, for sure!" release.
 - Fix throttlectl plugin argument parsing.

diff --git a/cluster.c b/cluster.c
index 7a83858..28b878e 100644 (file)
--- a/cluster.c
+++ b/cluster.c
@@ -1,6 +1,6 @@
 // L2TPNS Clustering Stuff
 
-char const *cvs_id_cluster = "$Id: cluster.c,v 1.19 2004/11/29 02:17:17 bodea Exp $";
+char const *cvs_id_cluster = "$Id: cluster.c,v 1.20 2004/12/03 06:40:02 bodea Exp $";
 
 #include <stdio.h>
 #include <sys/file.h>
@@ -744,6 +744,8 @@ void cluster_heartbeat()
        if (!config->cluster_iam_master)        // Only the master does this.
                return;
 
+       config->cluster_table_version += config->cluster_num_changes;
+
        // Fill out the heartbeat header.
        memset(&h, 0, sizeof(h));
 
@@ -759,6 +761,7 @@ void cluster_heartbeat()
        h.size_tunn = sizeof(tunnelt);
        h.interval = config->cluster_hb_interval;
        h.timeout  = config->cluster_hb_timeout;
+       h.table_version = config->cluster_table_version;
 
        add_type(&p, C_HEARTBEAT, HB_VERSION, (char*) &h, sizeof(h));
 
@@ -817,8 +820,10 @@ void cluster_heartbeat()
                exit(1);
        }
 
-       LOG(3, 0, 0, "Sending heartbeat #%d with %d changes (%d x-sess, %d x-tunnels, %d highsess, %d hightun, size %d)\n",
-           h.seq, config->cluster_num_changes, count, tcount, config->cluster_highest_sessionid,
+       LOG(3, 0, 0, "Sending v%d heartbeat #%d, change #%llu with %d changes "
+                    "(%d x-sess, %d x-tunnels, %d highsess, %d hightun, size %d)\n",
+           HB_VERSION, h.seq, h.table_version, config->cluster_num_changes,
+           count, tcount, config->cluster_highest_sessionid,
            config->cluster_highest_tunnelid, (p-buff));
 
        config->cluster_num_changes = 0;
@@ -1096,18 +1101,20 @@ static int cluster_recv_tunnel(int more, u8 *p)
 //
 // Process a heartbeat..
 //
+// v3: added interval, timeout
+// v4: added table_version
 static int cluster_process_heartbeat(u8 * data, int size, int more, u8 * p, u32 addr)
 {
        heartt * h;
        int s = size - (p-data);
        int i, type;
 
-#if HB_VERSION != 3
+#if HB_VERSION != 4
 # error "need to update cluster_process_heartbeat()"
 #endif
 
-       // we handle version 2+
-       if (more < 2 || more > HB_VERSION) {
+       // we handle versions 3 through 4
+       if (more < 3 || more > HB_VERSION) {
                LOG(0, 0, 0, "Received a heartbeat version that I don't support (%d)!\n", more);
                return -1; // Ignore it??
        }
@@ -1126,13 +1133,20 @@ static int cluster_process_heartbeat(u8 * data, int size, int more, u8 * p, u32
        if (config->cluster_iam_master) {       // Sanity...
                                // Note that this MUST match the election process above!
 
-               LOG(0, 0, 0, "I just got a packet claiming to be from a master but _I_ am the master!\n");
+               LOG(0, 0, 0, "I just got a heartbeat from master %s, but _I_ am the master!\n", fmtaddr(addr, 0));
                if (!h->basetime) {
-                       LOG(0, 0, 0, "Heartbeat from addr %s with zero basetime!\n", fmtaddr(addr, 0));
+                       LOG(0, 0, 0, "Heartbeat with zero basetime!  Ignoring\n");
                        return -1; // Skip it.
                }
+               if (more >= 4 && h->table_version > config->cluster_table_version) {
+                       LOG(0, 0, 0, "They've seen more state changes (%llu vs my %llu) so I'm gone!\n",
+                               h->table_version, config->cluster_table_version);
+
+                       kill(0, SIGTERM);
+                       exit(1);
+               }
                if (basetime > h->basetime) {
-                       LOG(0, 0, 0, "They're (%s) an older master than me so I'm gone!\n", fmtaddr(addr, 0));
+                       LOG(0, 0, 0, "They're an older master than me so I'm gone!\n");
                        kill(0, SIGTERM);
                        exit(1);
                }
@@ -1173,23 +1187,20 @@ static int cluster_process_heartbeat(u8 * data, int size, int more, u8 * p, u32
                        // that the free session pointer is correct.
        cluster_check_sessions(h->highsession, h->freesession, h->hightunnel);
 
-       if (more > 2) // reserved section of heartt was not initialized prior to v3
+       if (h->interval != config->cluster_hb_interval)
        {
-               if (h->interval != config->cluster_hb_interval)
-               {
-                       LOG(2, 0, 0, "Master set ping/heartbeat interval to %u (was %u)\n",
-                               h->interval, config->cluster_hb_interval);
+               LOG(2, 0, 0, "Master set ping/heartbeat interval to %u (was %u)\n",
+                       h->interval, config->cluster_hb_interval);
 
-                       config->cluster_hb_interval = h->interval;
-               }
+               config->cluster_hb_interval = h->interval;
+       }
 
-               if (h->timeout != config->cluster_hb_timeout)
-               {
-                       LOG(2, 0, 0, "Master set heartbeat timeout to %u (was %u)\n",
-                               h->timeout, config->cluster_hb_timeout);
+       if (h->timeout != config->cluster_hb_timeout)
+       {
+               LOG(2, 0, 0, "Master set heartbeat timeout to %u (was %u)\n",
+                       h->timeout, config->cluster_hb_timeout);
 
-                       config->cluster_hb_timeout = h->timeout;
-               }
+               config->cluster_hb_timeout = h->timeout;
        }
 
                // Ok. process the packet...
@@ -1273,6 +1284,7 @@ static int cluster_process_heartbeat(u8 * data, int size, int more, u8 * p, u32
        }
 
        config->cluster_last_hb = TIME; // Successfully received a heartbeat!
+       config->cluster_table_version = h->table_version;
        return 0;
 
 shortpacket:
@@ -1414,10 +1426,12 @@ int cmd_show_cluster(struct cli_def *cli, char *command, char **argv, int argc)
                                : "Not defined",
                        0.1 * (TIME - config->cluster_last_hb));
                cli_print(cli, "Uptodate         : %s", config->cluster_iam_uptodate ? "Yes" : "No");
+               cli_print(cli, "Table version #  : %llu", config->cluster_table_version);
                cli_print(cli, "Next sequence number expected: %d", config->cluster_seq_number);
                cli_print(cli, "%d sessions undefined of %d", config->cluster_undefined_sessions, config->cluster_highest_sessionid);
                cli_print(cli, "%d tunnels undefined of %d", config->cluster_undefined_tunnels, config->cluster_highest_tunnelid);
        } else {
+               cli_print(cli, "Table version #  : %llu", config->cluster_table_version);
                cli_print(cli, "Next heartbeat # : %d", config->cluster_seq_number);
                cli_print(cli, "Highest session  : %d", config->cluster_highest_sessionid);
                cli_print(cli, "Highest tunnel   : %d", config->cluster_highest_tunnelid);

diff --git a/cluster.h b/cluster.h
index e2ccd46..f986ab1 100644 (file)
--- a/cluster.h
+++ b/cluster.h
@@ -1,5 +1,5 @@
 // L2TPNS Clustering Stuff
-// $Id: cluster.h,v 1.7 2004/11/16 07:54:32 bodea Exp $
+// $Id: cluster.h,v 1.8 2004/12/03 06:40:02 bodea Exp $
 
 #ifndef __CLUSTER_H__
 #define __CLUSTER_H__
@@ -20,7 +20,7 @@
 #define C_CTUNNEL              13      // Compressed tunnel structure.
 #define C_GARDEN               14      // Gardened packet
 
-#define HB_VERSION             3       // Protocol version number..
+#define HB_VERSION             4       // Protocol version number..
 #define HB_MAX_SEQ             (1<<30) // Maximum sequence number. (MUST BE A POWER OF 2!)
 #define HB_HISTORY_SIZE                64      // How many old heartbeats we remember?? (Must be a factor of HB_MAX_SEQ)
 
@@ -45,10 +45,12 @@ typedef struct {
        u32     size_sess;      // Size of the session structure.
 
        u32     size_tunn;      // size of the tunnel structure.
-       u32     interval;       // ping/heartbeat interval (if changed)
-       u32     timeout;        // heartbeat timeout (if changed)
+       u32     interval;       // ping/heartbeat interval
+       u32     timeout;        // heartbeat timeout
 
-       char reserved[128 - 11*sizeof(u32)];    // Pad out to 128 bytes.
+       u64     table_version;  // # state changes processed by cluster
+
+       char reserved[128 - 13*sizeof(u32)];    // Pad out to 128 bytes.
 } heartt;
 
 typedef struct {               /* Used to update byte counters on the */

diff --git a/l2tpns.h b/l2tpns.h
index d1ab05e..a0026ae 100644 (file)
--- a/l2tpns.h
+++ b/l2tpns.h
@@ -1,5 +1,5 @@
 // L2TPNS Global Stuff
-// $Id: l2tpns.h,v 1.44 2004/12/01 04:14:55 bodea Exp $
+// $Id: l2tpns.h,v 1.45 2004/12/03 06:40:02 bodea Exp $
 
 #ifndef __L2TPNS_H__
 #define __L2TPNS_H__
@@ -15,7 +15,7 @@
 #include <sys/types.h>
 #include <libcli.h>
 
-#define VERSION        "2.0.12"
+#define VERSION        "2.0.13"
 
 // Limits
 #define MAXTUNNEL      500             // could be up to 65535
@@ -109,6 +109,7 @@ enum
 // Types
 typedef unsigned short u16;
 typedef unsigned int u32;
+typedef unsigned long long u64;
 typedef unsigned char u8;
 typedef u32 ipt;
 typedef u16 portt;
@@ -465,6 +466,7 @@ typedef struct
 
        int             cluster_hb_interval;            // How often to send a heartbeat.
        int             cluster_hb_timeout;             // How many missed heartbeats trigger an election.
+       u64             cluster_table_version;          // # state changes processed by cluster
 
 #ifdef BGP
 #define BGP_NUM_PEERS  2

diff --git a/l2tpns.spec b/l2tpns.spec
index 383daf8..7edc33f 100644 (file)
--- a/l2tpns.spec
+++ b/l2tpns.spec
@@ -1,6 +1,6 @@
 Summary: A high-speed clustered L2TP LNS
 Name: l2tpns
-Version: 2.0.12
+Version: 2.0.13
 Release: 1
 Copyright: GPL
 Group: System Environment/Daemons
@@ -43,5 +43,5 @@ rm -rf %{buildroot}
 %attr(644,root,root) /usr/share/man/man[58]/*
 
 %changelog
-* Wed Dec  1 2004 Brendan O'Dea <bod@optusnet.com.au> 2.0.12-1
-- 2.0.12 release, see /usr/share/doc/l2tpns-2.0.12/Changes
+* Fri Dec 3 2004 Brendan O'Dea <bod@optusnet.com.au> 2.0.13-1
+- 2.0.13 release, see /usr/share/doc/l2tpns-2.0.13/Changes