multicast disaster mitigation

diff --git a/Changes b/Changes
index 2188dfb..12cb3f8 100644 (file)
--- a/Changes
+++ b/Changes
@@ -1,3 +1,17 @@
+* Sat May 21 2005 Brendan O'Dea <bod@optusnet.com.au> 2.0.21
+- Cluster changes from Michael, intended to prevent a stray master
+  from trashing a cluster:
+  + Ignore heartbeats from peers claiming to be the master before the
+    timeout on the old master has expired.
+  + A master receiving a stray heartbeat sends a unicast HB back, which
+    should cause the rogue to die due to the tie-breaker code.
+  + Keep probing the master for late heartbeats.
+  + Drop BGP as soon as we become master with peers (TODO: pre-emptively
+    drop routes if the master is late and we are the best candidate).
+  + A slave (presumably a recently restarted master) receiving a
+    LASTSEEN message sends a zero basetime ping (as sent by the master
+    on clean shutdown).
+  
 * Mon May 16 2005 Brendan O'Dea <bod@optusnet.com.au> 2.0.20
 - Add handling of "throttle=N" RADIUS attributes.
 - Fix RADIUS indexing (should have 16K entries with 64 sockets).
@@ -113,7 +127,7 @@
 - Revise BGP config to use "router bgp AS" syntax (requires libcli >= 1.8.2)
 
 * Tue Nov 9 2004 Brendan O'Dea <bod@optusnet.com.au> 2.0.5
-- Handle routing properly in lone-master case 
+- Handle routing properly in lone-master case
 - Fix intercepts:  don't double-snoop throttled customers, ensure
   byte/packet counts are only updated once
 - Add a callback to allow plugins to fetch values from the running config

diff --git a/cluster.c b/cluster.c
index e3e7f01..6f72394 100644 (file)
--- a/cluster.c
+++ b/cluster.c
@@ -1,6 +1,6 @@
 // L2TPNS Clustering Stuff
 
-char const *cvs_id_cluster = "$Id: cluster.c,v 1.26.2.5 2005/05/02 09:03:27 bodea Exp $";
+char const *cvs_id_cluster = "$Id: cluster.c,v 1.26.2.6 2005/05/21 13:05:36 bodea Exp $";
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -485,14 +485,15 @@ void cluster_check_master(void)
 
        // If the master is late (missed 2 hearbeats by a second and a
        // hair) it may be that the switch has dropped us from the
-       // multicast group, try unicasting one probe to the master
+       // multicast group, try unicasting probes to the master
        // which will hopefully respond with a unicast heartbeat that
        // will allow us to limp along until the querier next runs.
-       if (TIME > (config->cluster_last_hb + 2 * config->cluster_hb_interval + 11))
+       if (config->cluster_master_address
+           && TIME > (config->cluster_last_hb + 2 * config->cluster_hb_interval + 11))
        {
-               if (!probed && config->cluster_master_address)
+               if (!probed || (TIME > (probed + 2 * config->cluster_hb_interval)))
                {
-                       probed = 1;
+                       probed = TIME;
                        LOG(1, 0, 0, "Heartbeat from master %.1fs late, probing...\n",
                                0.1 * (TIME - (config->cluster_last_hb + config->cluster_hb_interval)));
 
@@ -621,6 +622,11 @@ void cluster_check_master(void)
 
        if (!num_peers) // lone master
                advertise();
+#ifdef BGP
+       else if (bgp_configured)
+               bgp_enable_routing(0);
+#endif /* BGP */
+
 
        // FIXME. We need to fix up the tunnel control message
        // queue here! There's a number of other variables we
@@ -896,6 +902,14 @@ static int cluster_catchup_slave(int seq, in_addr_t slave)
        int diff;
 
        LOG(1, 0, 0, "Slave %s sent LASTSEEN with seq %d\n", fmtaddr(slave, 0), seq);
+       if (!config->cluster_iam_master) {
+               LOG(1, 0, 0, "Got LASTSEEN but I'm not a master! Sending a null PING.\n");
+                       // Send a ping to the stray slave saying that we're a master that's
+                       // just shutdown. This should force the slave to listen for the real
+                       // master.
+               peer_send_message(slave, C_PING, 0, NULL, 0);
+               return 0;
+       }
 
        diff = config->cluster_seq_number - seq;        // How many packet do we need to send?
        if (diff < 0)
@@ -907,9 +921,11 @@ static int cluster_catchup_slave(int seq, in_addr_t slave)
                return peer_send_message(slave, C_KILL, seq, NULL, 0);// Kill the slave. Nothing else to do.
        }
 
+       LOG(1, 0, 0, "Sending %d catchup packets to slave %s\n", diff, fmtaddr(slave, 0) );
+
                // Now resend every packet that it missed, in order.
        while (seq != config->cluster_seq_number) {
-               s = seq%HB_HISTORY_SIZE;
+               s = seq % HB_HISTORY_SIZE;
                if (seq != past_hearts[s].seq) {
                        LOG(0, 0, 0, "Tried to re-send heartbeat for %s but %d doesn't match %d! (%d,%d)\n",
                                fmtaddr(slave, 0), seq, past_hearts[s].seq, s, config->cluster_seq_number);
@@ -1174,9 +1190,33 @@ static int cluster_process_heartbeat(uint8_t *data, int size, int more, uint8_t
                        exit(1);
                }
 
+                       //
+                       // Send it a unicast heartbeat to see give it a chance to die.
+                       // NOTE: It's actually safe to do seq-number - 1 without checking
+                       // for wrap around.
+                       //
+               cluster_catchup_slave(config->cluster_seq_number - 1, addr);
+
                return -1; // Skip it.
        }
 
+               //
+               // Try and guard against a stray master appearing.
+               //
+               // Ignore heartbeats received from another master before the
+               // timeout (less a smidgen) for the old master has elapsed.
+               //
+               // Note that after a clean failover, the cluster_master_address
+               // is cleared, so this doesn't run. 
+               //
+       if (config->cluster_master_address && addr != config->cluster_master_address
+           && (config->cluster_last_hb + config->cluster_hb_timeout - 11) > TIME) {
+                   LOG(0, 0, 0, "Ignoring stray heartbeat from %s, current master %s has not yet timed out (last heartbeat %.1f seconds ago).\n",
+                           fmtaddr(addr, 0), fmtaddr(config->cluster_master_address, 1),
+                           0.1 * (TIME - config->cluster_last_hb));
+                   return -1; // ignore
+       }
+
        if (config->cluster_seq_number == -1)   // Don't have one. Just align to the master...
                config->cluster_seq_number = h->seq;
 

diff --git a/l2tpns.h b/l2tpns.h
index d5b1afb..068ec96 100644 (file)
--- a/l2tpns.h
+++ b/l2tpns.h
@@ -1,5 +1,5 @@
 // L2TPNS Global Stuff
-// $Id: l2tpns.h,v 1.49.2.11 2005/05/16 05:56:50 bodea Exp $
+// $Id: l2tpns.h,v 1.49.2.12 2005/05/21 13:05:36 bodea Exp $
 
 #ifndef __L2TPNS_H__
 #define __L2TPNS_H__
@@ -15,7 +15,7 @@
 #include <sys/types.h>
 #include <libcli.h>
 
-#define VERSION        "2.0.20"
+#define VERSION        "2.0.21"
 
 // Limits
 #define MAXTUNNEL      500             // could be up to 65535

diff --git a/l2tpns.spec b/l2tpns.spec
index 6700645..c05c025 100644 (file)
--- a/l2tpns.spec
+++ b/l2tpns.spec
@@ -1,6 +1,6 @@
 Summary: A high-speed clustered L2TP LNS
 Name: l2tpns
-Version: 2.0.20
+Version: 2.0.21
 Release: 1
 Copyright: GPL
 Group: System Environment/Daemons
@@ -43,5 +43,5 @@ rm -rf %{buildroot}
 %attr(644,root,root) /usr/share/man/man[58]/*
 
 %changelog
-* Mon May 16 2005 Brendan O'Dea <bod@optusnet.com.au> 2.0.20-1
-- 2.0.20 release, see /usr/share/doc/l2tpns-2.0.20/Changes
+* Sat May 21 2005 Brendan O'Dea <bod@optusnet.com.au> 2.0.21-1
+- 2.0.21 release, see /usr/share/doc/l2tpns-2.0.21/Changes