diff --git a/crates/lox-distributor/src/lox_context.rs b/crates/lox-distributor/src/lox_context.rs index 7bd92cd..7c5ba9d 100644 --- a/crates/lox-distributor/src/lox_context.rs +++ b/crates/lox-distributor/src/lox_context.rs @@ -95,7 +95,6 @@ impl LoxServerContext { } } - accounted_for_bridges } @@ -143,33 +142,37 @@ impl LoxServerContext { // Next, handle the failing bridges. If resource last passed tests >= ACCEPTED_HOURS_OF_FAILURE ago, // it should be replaced with a working resource and be removed from the bridgetable. for bridge in failing { - let res = self.replace_with_new(bridge); - if res == lox_library::ReplaceSuccess::Replaced { - println!( - "Failing BridgeLine {:?} successfully replaced.", - bridge.uid_fingerprint - ); - accounted_for_bridges.push(bridge.uid_fingerprint); - self.metrics.removed_bridges.inc(); - } else if res == lox_library::ReplaceSuccess::NotReplaced { - // Add the bridge to the list of to_be_replaced bridges in the Lox context and try - // again to replace at the next update (nothing changes in the Lox Authority) - println!( - "Failing BridgeLine {:?} NOT replaced, saved for next update!", - bridge.uid_fingerprint - ); - self.metrics.existing_or_updated_bridges.inc(); - accounted_for_bridges.push(bridge.uid_fingerprint); - } else { - // NotFound - assert!( - res == lox_library::ReplaceSuccess::NotFound, - "ReplaceSuccess incorrectly set" - ); - println!( + match self.replace_with_new(bridge) { + lox_library::ReplaceSuccess::Replaced => { + println!( + "Failing BridgeLine {:?} successfully replaced.", + bridge.uid_fingerprint + ); + accounted_for_bridges.push(bridge.uid_fingerprint); + self.metrics.removed_bridges.inc(); + } + lox_library::ReplaceSuccess::NotReplaced => { + // Add the bridge to the list of to_be_replaced bridges in the Lox context and try + // again to replace at the next update (nothing changes in the Lox Authority) + println!( + "Failing BridgeLine {:?} NOT replaced, saved for next update!", + bridge.uid_fingerprint + ); + self.metrics.existing_or_updated_bridges.inc(); + accounted_for_bridges.push(bridge.uid_fingerprint); + } + lox_library::ReplaceSuccess::Removed => { + println!( + "Failing BridgeLine {:?} successfully removed.", + bridge.uid_fingerprint + ); + accounted_for_bridges.push(bridge.uid_fingerprint); + self.metrics.removed_bridges.inc(); + } + lox_library::ReplaceSuccess::NotFound => println!( "Failing BridgeLine {:?} not found in bridge table.", bridge.uid_fingerprint - ); + ), } } accounted_for_bridges @@ -193,32 +196,49 @@ impl LoxServerContext { accounted_for_bridges, ); } - let unaccounted_for = self.ba.lock().unwrap().find_and_remove_unaccounted_for_bridges(accounted_for_bridges); + + let unaccounted_for = self + .ba + .lock() + .unwrap() + .find_and_remove_unaccounted_for_bridges(accounted_for_bridges); for bridgeline in unaccounted_for { match self.replace_with_new(bridgeline) { - lox_library::ReplaceSuccess::Replaced => { - println!("BridgeLine {:?} not found in rdsys update was successfully replaced.", bridgeline.uid_fingerprint); - self.metrics.removed_bridges.inc(); - } - lox_library::ReplaceSuccess::NotReplaced => { - // Try again to replace at the next update (nothing changes in the Lox Authority) - println!("BridgeLine {:?} not found in rdsys update NOT replaced, saved for next update!", - bridgeline.uid_fingerprint); - self.metrics.existing_or_updated_bridges.inc(); - } - lox_library::ReplaceSuccess::NotFound => println!( - "BridgeLine {:?} no longer in reachable bridges.", - bridgeline.uid_fingerprint - ), - } + lox_library::ReplaceSuccess::Replaced => { + println!( + "BridgeLine {:?} not found in rdsys update was successfully replaced.", + bridgeline.uid_fingerprint + ); + self.metrics.removed_bridges.inc(); } + lox_library::ReplaceSuccess::Removed => { + println!("BridgeLine {:?} not found in rdsys update was not distributed to a bucket so was removed", bridgeline.uid_fingerprint); + self.metrics.removed_bridges.inc(); + } + lox_library::ReplaceSuccess::NotReplaced => { + // Try again to replace at the next update (nothing changes in the Lox Authority) + println!("BridgeLine {:?} not found in rdsys update NOT replaced, saved for next update!", + bridgeline.uid_fingerprint); + self.metrics.existing_or_updated_bridges.inc(); + } + lox_library::ReplaceSuccess::NotFound => println!( + "BridgeLine {:?} no longer in reachable bridges.", + bridgeline.uid_fingerprint + ), + } + } // Finally, assign any extra_bridges to new buckets if there are enough while self.extra_bridges.lock().unwrap().len() >= MAX_BRIDGES_PER_BUCKET { let bucket = self.remove_extra_bridges(); // TODO: Decide the circumstances under which a bridge is allocated to an open_inv or spare bucket, // eventually also do some more fancy grouping of new resources, i.e., by type or region let mut db_obj = self.db.lock().unwrap(); - match self.ba.lock().unwrap().add_spare_bucket(bucket, &mut db_obj) { + match self + .ba + .lock() + .unwrap() + .add_spare_bucket(bucket, &mut db_obj) + { Ok(_) => (), Err(e) => { println!("Error: {:?}", e); @@ -765,7 +785,7 @@ mod tests { } } - fn get_config()-> BridgeConfig { + fn get_config() -> BridgeConfig { env::set_var("BRIDGE_CONFIG_PATH", "bridge_config.json"); let path = env::var("BRIDGE_CONFIG_PATH").unwrap(); let config_file = fs::File::open(&path).unwrap(); @@ -799,7 +819,7 @@ mod tests { // Extra bridges should be cleared from the Lox Context after each sync assert!( th.context.extra_bridges.lock().unwrap().is_empty(), - "Extra bridges should be empty after syncUnexpected number of extra bridges" + "Extra bridges should be empty after sync" ); } @@ -831,11 +851,72 @@ mod tests { reachable_expected_length, "Unexpected number of reachable bridges" ); + // Extra bridges should be cleared from the Lox Context after each sync + assert!( + th.context.extra_bridges.lock().unwrap().is_empty(), + "Extra bridges should be empty after sync" + ); + } + + #[test] + fn test_sync_with_preloaded_obsolete_bridgetable() { + // Tests the case where all bridges in the bridgetable are no longer in rdsys. + // In this case, all bridges should be replaced. If it's a bridge in a spare bucket, just remove the other bridges + // from the spare bucket and delete the bridge + let bridge_config = get_config(); + // Sync bridges to non-empty bridge table with disparate sets of bridges + let th_with_bridges = TestHarness::new_with_bridges(); //Creates 5 open invitation and 5 hot spare buckets, so 30 total buckets to be replaced + let mut rs = TestResourceState::default(); + for _ in 0..5 { + rs.add_working_resource(); + } + assert_ne!(rs.rstate.working, None); + assert_eq!(rs.rstate.not_working, None); + + assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.reachable.len(), 15+15, "Unexpected number of reachable bridges should equal the number of open invitation bridges plus the number of spares added: 2x5x3"); assert_eq!( - th.context.extra_bridges.lock().unwrap().len(), + th_with_bridges + .context + .ba + .lock() + .unwrap() + .bridge_table + .spares + .len(), + 5, + "Unexpected number of spare bridges, should be 5" + ); + + // All potentially distributed resources (i.e., those assigned to open invitation/trusted buckets) + // not found in the rdsys update will first be replaced with any new resources coming in from rdsys then + // by bridges from the hot spare buckets. In this case, the hot spare buckets are also not in the bridge table + // so will also be replaced. + // Since there are fewer working resources than resources that have populated the bridge table, this update will + // exhaust the spare buckets and leave some obsolete bridges. The set of open invitation/trusted buckets should be + // preserved (5 open invitation buckets * 3) + th_with_bridges + .context + .sync_with_bridgetable(bridge_config.watched_blockages, rs.rstate.clone()); + assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.reachable.len(), 15, "Unexpected number of reachable bridges should equal the number of open invitation bridges added: 5x3"); + assert_eq!( + th_with_bridges + .context + .ba + .lock() + .unwrap() + .bridge_table + .spares + .len(), + 0, + "Unexpected number of spare bridges, should be exhausted" + ); + + assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.unallocated_bridges.len(), 0, "Unexpected number of unallocated bridges, should be 0 (All spare buckets and new resources for replacement exhausted)" + ); + assert_eq!( + th_with_bridges.context.extra_bridges.lock().unwrap().len(), 0, "Unexpected number of extra bridges" ); } - }