Update lox-context to handle Removed bridges, add test for spare removals

crates/lox-distributor/src/lox_context.rs

@@ -95,7 +95,6 @@ impl LoxServerContext {
             }
         }
 
-
         accounted_for_bridges
     }
 
@@ -143,15 +142,16 @@ impl LoxServerContext {
         // Next, handle the failing bridges. If resource last passed tests >= ACCEPTED_HOURS_OF_FAILURE ago,
         // it should be replaced with a working resource and be removed from the bridgetable.
         for bridge in failing {
-            let res = self.replace_with_new(bridge);
+            match self.replace_with_new(bridge) {
-            if res == lox_library::ReplaceSuccess::Replaced {
+                lox_library::ReplaceSuccess::Replaced => {
                     println!(
                         "Failing BridgeLine {:?} successfully replaced.",
                         bridge.uid_fingerprint
                     );
                     accounted_for_bridges.push(bridge.uid_fingerprint);
                     self.metrics.removed_bridges.inc();
-            } else if res == lox_library::ReplaceSuccess::NotReplaced {
+                }
+                lox_library::ReplaceSuccess::NotReplaced => {
                     // Add the bridge to the list of to_be_replaced bridges in the Lox context and try
                     // again to replace at the next update (nothing changes in the Lox Authority)
                     println!(
@@ -160,16 +160,19 @@ impl LoxServerContext {
                     );
                     self.metrics.existing_or_updated_bridges.inc();
                     accounted_for_bridges.push(bridge.uid_fingerprint);
-            } else {
+                }
-                // NotFound
+                lox_library::ReplaceSuccess::Removed => {
-                assert!(
-                    res == lox_library::ReplaceSuccess::NotFound,
-                    "ReplaceSuccess incorrectly set"
-                );
                     println!(
-                    "Failing BridgeLine {:?} not found in bridge table.",
+                        "Failing BridgeLine {:?} successfully removed.",
                         bridge.uid_fingerprint
                     );
+                    accounted_for_bridges.push(bridge.uid_fingerprint);
+                    self.metrics.removed_bridges.inc();
+                }
+                lox_library::ReplaceSuccess::NotFound => println!(
+                    "Failing BridgeLine {:?} not found in bridge table.",
+                    bridge.uid_fingerprint
+                ),
             }
         }
         accounted_for_bridges
@@ -193,11 +196,23 @@ impl LoxServerContext {
                 accounted_for_bridges,
             );
         }
-        let unaccounted_for = self.ba.lock().unwrap().find_and_remove_unaccounted_for_bridges(accounted_for_bridges);
+
+        let unaccounted_for = self
+            .ba
+            .lock()
+            .unwrap()
+            .find_and_remove_unaccounted_for_bridges(accounted_for_bridges);
         for bridgeline in unaccounted_for {
             match self.replace_with_new(bridgeline) {
                 lox_library::ReplaceSuccess::Replaced => {
-                        println!("BridgeLine {:?} not found in rdsys update was successfully replaced.", bridgeline.uid_fingerprint);
+                    println!(
+                        "BridgeLine {:?} not found in rdsys update was successfully replaced.",
+                        bridgeline.uid_fingerprint
+                    );
+                    self.metrics.removed_bridges.inc();
+                }
+                lox_library::ReplaceSuccess::Removed => {
+                    println!("BridgeLine {:?} not found in rdsys update was not distributed to a bucket so was removed", bridgeline.uid_fingerprint);
                     self.metrics.removed_bridges.inc();
                 }
                 lox_library::ReplaceSuccess::NotReplaced => {
@@ -218,7 +233,12 @@ impl LoxServerContext {
             // TODO: Decide the circumstances under which a bridge is allocated to an open_inv or spare bucket,
             // eventually also do some more fancy grouping of new resources, i.e., by type or region
             let mut db_obj = self.db.lock().unwrap();
-            match self.ba.lock().unwrap().add_spare_bucket(bucket, &mut db_obj) {
+            match self
+                .ba
+                .lock()
+                .unwrap()
+                .add_spare_bucket(bucket, &mut db_obj)
+            {
                 Ok(_) => (),
                 Err(e) => {
                     println!("Error: {:?}", e);
@@ -799,7 +819,7 @@ mod tests {
         // Extra bridges should be cleared from the Lox Context after each sync
         assert!(
             th.context.extra_bridges.lock().unwrap().is_empty(),
-            "Extra bridges should be empty after syncUnexpected number of extra bridges"
+            "Extra bridges should be empty after sync"
         );
     }
 
@@ -831,11 +851,72 @@ mod tests {
             reachable_expected_length,
             "Unexpected number of reachable bridges"
         );
+        // Extra bridges should be cleared from the Lox Context after each sync
+        assert!(
+            th.context.extra_bridges.lock().unwrap().is_empty(),
+            "Extra bridges should be empty after sync"
+        );
+    }
+
+    #[test]
+    fn test_sync_with_preloaded_obsolete_bridgetable() {
+        // Tests the case where all bridges in the bridgetable are no longer in rdsys.
+        // In this case, all bridges should be replaced. If it's a bridge in a spare bucket, just remove the other bridges
+        // from the spare bucket and delete the bridge
+        let bridge_config = get_config();
+        // Sync bridges to non-empty bridge table with disparate sets of bridges
+        let th_with_bridges = TestHarness::new_with_bridges(); //Creates 5 open invitation and 5 hot spare buckets, so 30 total buckets to be replaced
+        let mut rs = TestResourceState::default();
+        for _ in 0..5 {
+            rs.add_working_resource();
+        }
+        assert_ne!(rs.rstate.working, None);
+        assert_eq!(rs.rstate.not_working, None);
+
+        assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.reachable.len(), 15+15, "Unexpected number of reachable bridges should equal the number of open invitation bridges plus the number of spares added: 2x5x3");
         assert_eq!(
-            th.context.extra_bridges.lock().unwrap().len(),
+            th_with_bridges
+                .context
+                .ba
+                .lock()
+                .unwrap()
+                .bridge_table
+                .spares
+                .len(),
+            5,
+            "Unexpected number of spare bridges, should be 5"
+        );
+
+        // All potentially distributed resources (i.e., those assigned to open invitation/trusted buckets)
+        // not found in the rdsys update will first be replaced with any new resources coming in from rdsys then
+        // by bridges from the hot spare buckets. In this case, the hot spare buckets are also not in the bridge table
+        // so will also be replaced.
+        // Since there are fewer working resources than resources that have populated the bridge table, this update will
+        // exhaust the spare buckets and leave some obsolete bridges. The set of open invitation/trusted buckets should be
+        // preserved (5 open invitation buckets * 3)
+        th_with_bridges
+            .context
+            .sync_with_bridgetable(bridge_config.watched_blockages, rs.rstate.clone());
+        assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.reachable.len(), 15, "Unexpected number of reachable bridges should equal the number of open invitation bridges added: 5x3");
+        assert_eq!(
+            th_with_bridges
+                .context
+                .ba
+                .lock()
+                .unwrap()
+                .bridge_table
+                .spares
+                .len(),
+            0,
+            "Unexpected number of spare bridges, should be exhausted"
+        );
+
+        assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.unallocated_bridges.len(), 0, "Unexpected number of unallocated bridges, should be 0 (All spare buckets and new resources for replacement exhausted)"
+        );
+        assert_eq!(
+            th_with_bridges.context.extra_bridges.lock().unwrap().len(),
             0,
             "Unexpected number of extra bridges"
         );
     }
-
 }