vecna
/
lox
mirror of https://gitlab.torproject.org/vecna/lox.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge upstream changes

This commit is contained in:
Vecna 2024-04-26 23:05:00 -04:00
parent 0644431dee 95573f1193
commit 5c0376cd56
20 changed files with 1612 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1236
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

9
crates/lox-distributor/Cargo.toml
View File

@ -16,11 +16,12 @@ readme = "README.md"
[dependencies]
julianday = "1.2.0"
base64 = "0.21.7"
base64 = "0.22.0"
hyper = { version = "0.14.28", features = ["deprecated", "backports","server"] }
hex = "0.4.3"
hex_fmt = "0.3"
futures = "0.3.30"
time = "0.3.34"
time = "0.3.36"
tokio = { version = "1", features = ["full", "macros", "signal"] }
rand = "0.8.5"
reqwest = { version = "0.11", features = ["json", "stream"]}
@ -30,7 +31,7 @@ lox-zkp = { git = "https://gitlab.torproject.org/onyinyang/lox-zkp", version = "
lox-library = { path = "../lox-library", version = "0.1.0"}
lox_utils = { path = "../lox-utils", version = "0.1.0"}
rdsys_backend = { path = "../rdsys-backend-api", version = "0.2"}
clap = { version = "4.5.2", features = ["derive"] }
clap = { version = "4.5.4", features = ["derive"] }
serde_json = "1.0.113"
prometheus = "0.13.3"
sled = "0.34.7"
@ -42,5 +43,5 @@ array-bytes = "6.2.0"
sha1 = "0.10"
[dependencies.chrono]
version = "0.4.34"
version = "0.4.38"
features = ["serde"]

6
crates/lox-distributor/bridge_config.json Normal file
View File

@ -0,0 +1,6 @@
{
"watched_blockages": [
"RU"
],
"percent_spares": 50
}

2
crates/lox-distributor/src/db_handler.rs
View File

@ -81,7 +81,7 @@ impl DB {
extra_bridges: Arc::new(Mutex::new(Vec::new())),
to_be_replaced_bridges: Arc::new(Mutex::new(Vec::new())),
tp_bridge_infos: Arc::new(Mutex::new(HashMap::<
[u8; 20],
String,
BridgeVerificationInfo,
>::new())),
metrics,

151
crates/lox-distributor/src/fake_resource_state.rs Normal file
View File

@ -0,0 +1,151 @@
use crate::resource_parser::ACCEPTED_HOURS_OF_FAILURE;
use chrono::{Duration, Utc};
use rand::{Rng, RngCore};
use rdsys_backend::proto::{Resource, ResourceState, TestResults};
use std::collections::HashMap;
#[derive(Default)]
pub struct TestResourceState {
pub rstate: ResourceState,
}
impl TestResourceState {
// A previously working resources become not_working but within accepted failure time
pub fn working_with_accepted_failures(&mut self) {
match &mut self.rstate.working {
Some(resources) => {
if let Some(resource) = resources.pop() {
self.add_not_working_to_rstate(resource)
}
}
None => {
panic!("rstate.working Empty")
}
}
assert_ne!(self.rstate.working, None);
assert_eq!(self.rstate.not_working, None);
}
// Block resources that are working. Targeted blocked regions are specified in bridge_config.json
pub fn block_working(&mut self) {
match &mut self.rstate.working {
Some(resources) => {
for resource in resources {
resource.blocked_in = HashMap::from([
("AS".to_owned(), true),
("IR".to_owned(), false),
("RU".to_owned(), false),
("CN".to_owned(), false),
("SA".to_owned(), false),
]);
}
}
None => {
panic!("rstate.working Empty")
}
}
assert_ne!(self.rstate.working, None);
assert_eq!(self.rstate.not_working, None);
}
// Add a resource that is working
pub fn add_working_resource(&mut self) {
let working_resource = make_resource(
HashMap::from([
("AS".to_owned(), false),
("IR".to_owned(), false),
("RU".to_owned(), false),
("CN".to_owned(), false),
("SA".to_owned(), false),
]),
ACCEPTED_HOURS_OF_FAILURE - 12,
);
self.add_working_to_rstate(working_resource);
}
// Add a not-working resource that has been failing for 1 hour longer than the accepted threshold
pub fn add_not_working_resource(&mut self) {
let not_working_resource = make_resource(
HashMap::from([
("AS".to_owned(), false),
("IR".to_owned(), false),
("RU".to_owned(), false),
("CN".to_owned(), false),
("SA".to_owned(), false),
]),
ACCEPTED_HOURS_OF_FAILURE + 1,
);
self.add_not_working_to_rstate(not_working_resource);
}
// Add resource to rstate's working field
pub fn add_working_to_rstate(&mut self, working_resource: Resource) {
match &mut self.rstate.working {
Some(resources) => {
resources.push(working_resource);
}
None => {
self.rstate.working = Some(vec![working_resource]);
}
}
}
// Add resource to rstate's not_working field
pub fn add_not_working_to_rstate(&mut self, not_working_resource: Resource) {
match &mut self.rstate.not_working {
Some(resources) => {
resources.push(not_working_resource);
}
None => {
self.rstate.not_working = Some(vec![not_working_resource]);
}
}
}
}
pub fn make_resource(blocked_in: HashMap<String, bool>, last_passed: i64) -> Resource {
let mut flags = HashMap::new();
flags.insert(String::from("fast"), true);
flags.insert(String::from("stable"), true);
let mut params = HashMap::new();
params.insert(
String::from("password"),
String::from("ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"),
);
Resource {
r#type: String::from("obfs4"),
blocked_in,
test_result: TestResults {
last_passed: Utc::now() - Duration::hours(last_passed),
},
protocol: String::from("tcp"),
address: gen_ip(),
port: gen_port(),
fingerprint: gen_fingerprint(),
or_addresses: None,
distribution: String::from("https"),
flags: Some(flags),
params: Some(params),
}
}
pub fn gen_fingerprint() -> String {
let mut rng = rand::thread_rng();
let mut fingerprint_array: [u8; 16] = [0; 16];
rng.fill_bytes(&mut fingerprint_array);
hex::encode_upper(fingerprint_array)
}
pub fn gen_port() -> u16 {
rand::thread_rng().gen_range(0..u16::MAX)
}
pub fn gen_ip() -> String {
let i = rand::thread_rng().gen_range(1..u8::MAX);
let ii = rand::thread_rng().gen_range(1..u8::MAX);
let iii = rand::thread_rng().gen_range(1..u8::MAX);
let iv = rand::thread_rng().gen_range(1..u8::MAX);
format!("{}.{}.{}.{}", i, ii, iii, iv)
}

355
crates/lox-distributor/src/lox_context.rs
View File

@ -38,7 +38,7 @@ pub struct LoxServerContext {
pub extra_bridges: Arc<Mutex<Vec<BridgeLine>>>,
pub to_be_replaced_bridges: Arc<Mutex<Vec<BridgeLine>>>,
// Map of bridge fingerprint to values needed to verify TP reports
pub tp_bridge_infos: Arc<Mutex<HashMap<[u8; 20], BridgeVerificationInfo>>>,
pub tp_bridge_infos: Arc<Mutex<HashMap<String, BridgeVerificationInfo>>>,
#[serde(skip)]
pub metrics: Metrics,
}
@ -82,7 +82,10 @@ impl LoxServerContext {
for bridge in blocked_bridgelines {
let res = self.mark_blocked(bridge);
if res {
println!("BridgeLine {:?} successfully marked unreachable", bridge);
println!(
"BridgeLine {:?} successfully marked unreachable",
bridge.uid_fingerprint
);
self.metrics.blocked_bridges.inc();
} else {
println!(
@ -107,6 +110,7 @@ impl LoxServerContext {
self.metrics.new_bridges.inc();
}
}
accounted_for_bridges
}
@ -127,7 +131,7 @@ impl LoxServerContext {
if res {
println!(
"Blocked BridgeLine {:?} successfully marked unreachable",
bridge
bridge.uid_fingerprint
);
self.metrics.blocked_bridges.inc();
} else {
@ -154,33 +158,37 @@ impl LoxServerContext {
// Next, handle the failing bridges. If resource last passed tests >= ACCEPTED_HOURS_OF_FAILURE ago,
// it should be replaced with a working resource and be removed from the bridgetable.
for bridge in failing {
let res = self.replace_with_new(bridge);
if res == lox_library::ReplaceSuccess::Replaced {
println!(
"Failing BridgeLine {:?} successfully replaced.",
bridge.uid_fingerprint
);
accounted_for_bridges.push(bridge.uid_fingerprint);
self.metrics.removed_bridges.inc();
} else if res == lox_library::ReplaceSuccess::NotReplaced {
// Add the bridge to the list of to_be_replaced bridges in the Lox context and try
// again to replace at the next update (nothing changes in the Lox Authority)
println!(
"Failing BridgeLine {:?} NOT replaced, saved for next update!",
bridge.uid_fingerprint
);
self.metrics.existing_or_updated_bridges.inc();
accounted_for_bridges.push(bridge.uid_fingerprint);
} else {
// NotFound
assert!(
res == lox_library::ReplaceSuccess::NotFound,
"ReplaceSuccess incorrectly set"
);
println!(
match self.replace_with_new(bridge) {
lox_library::ReplaceSuccess::Replaced => {
println!(
"Failing BridgeLine {:?} successfully replaced.",
bridge.uid_fingerprint
);
accounted_for_bridges.push(bridge.uid_fingerprint);
self.metrics.removed_bridges.inc();
}
lox_library::ReplaceSuccess::NotReplaced => {
// Add the bridge to the list of to_be_replaced bridges in the Lox context and try
// again to replace at the next update (nothing changes in the Lox Authority)
println!(
"Failing BridgeLine {:?} NOT replaced, saved for next update!",
bridge.uid_fingerprint
);
self.metrics.existing_or_updated_bridges.inc();
accounted_for_bridges.push(bridge.uid_fingerprint);
}
lox_library::ReplaceSuccess::Removed => {
println!(
"Failing BridgeLine {:?} successfully removed.",
bridge.uid_fingerprint
);
accounted_for_bridges.push(bridge.uid_fingerprint);
self.metrics.removed_bridges.inc();
}
lox_library::ReplaceSuccess::NotFound => println!(
"Failing BridgeLine {:?} not found in bridge table.",
bridge.uid_fingerprint
);
),
}
}
accounted_for_bridges
@ -204,33 +212,36 @@ impl LoxServerContext {
accounted_for_bridges,
);
}
let mut ba_clone = self.ba.lock().unwrap();
let total_reachable = ba_clone.bridge_table.reachable.len();
match total_reachable.cmp(&accounted_for_bridges.len()) {
Ordering::Greater => {
let unaccounted_for = ba_clone.find_and_remove_unaccounted_for_bridges(accounted_for_bridges);
for bridgeline in unaccounted_for {
match self.replace_with_new(bridgeline) {
lox_library::ReplaceSuccess::Replaced => {
println!("BridgeLine {:?} not found in rdsys update was successfully replaced.", bridgeline.uid_fingerprint);
self.metrics.removed_bridges.inc();
}
lox_library::ReplaceSuccess::NotReplaced => {
// Try again to replace at the next update (nothing changes in the Lox Authority)
println!("BridgeLine {:?} not found in rdsys update NOT replaced, saved for next update!",
bridgeline.uid_fingerprint);
self.metrics.existing_or_updated_bridges.inc();
}
lox_library::ReplaceSuccess::NotFound => println!(
"BridgeLine {:?} no longer in reachable bridges.",
bridgeline.uid_fingerprint
),
}
}
}
Ordering::Less => println!("Something unexpected occurred: The number of reachable bridges should not be less than those updated from rdsys"),
_ => (),
let unaccounted_for = self
.ba
.lock()
.unwrap()
.find_and_remove_unaccounted_for_bridges(accounted_for_bridges);
for bridgeline in unaccounted_for {
match self.replace_with_new(bridgeline) {
lox_library::ReplaceSuccess::Replaced => {
println!(
"BridgeLine {:?} not found in rdsys update was successfully replaced.",
bridgeline.uid_fingerprint
);
self.metrics.removed_bridges.inc();
}
lox_library::ReplaceSuccess::Removed => {
println!("BridgeLine {:?} not found in rdsys update was not distributed to a bucket so was removed", bridgeline.uid_fingerprint);
self.metrics.removed_bridges.inc();
}
lox_library::ReplaceSuccess::NotReplaced => {
// Try again to replace at the next update (nothing changes in the Lox Authority)
println!("BridgeLine {:?} not found in rdsys update NOT replaced, saved for next update!",
bridgeline.uid_fingerprint);
self.metrics.existing_or_updated_bridges.inc();
}
lox_library::ReplaceSuccess::NotFound => println!(
"BridgeLine {:?} no longer in reachable bridges.",
bridgeline.uid_fingerprint
),
}
}
// Finally, assign any extra_bridges to new buckets if there are enough
while self.extra_bridges.lock().unwrap().len() >= MAX_BRIDGES_PER_BUCKET {
@ -238,7 +249,12 @@ impl LoxServerContext {
// TODO: Decide the circumstances under which a bridge is allocated to an open_inv or spare bucket,
// eventually also do some more fancy grouping of new resources, i.e., by type or region
let mut db_obj = self.db.lock().unwrap();
match ba_clone.add_spare_bucket(bucket, &mut db_obj) {
match self
.ba
.lock()
.unwrap()
.add_spare_bucket(bucket, &mut db_obj)
{
Ok(_) => (),
Err(e) => {
println!("Error: {:?}", e);
@ -251,6 +267,10 @@ impl LoxServerContext {
// Regenerate tables for verifying TP reports
self.generate_tp_bridge_infos();
// Any remaining extra bridges should be cleared from the Lox Context after each sync
// Currently bridgetable updating behaviour does not occur without receiving a resource list
// from rdsys so if the extra bridge is still working, it can be added to the table later
self.extra_bridges.lock().unwrap().clear();
}
pub fn append_extra_bridges(&self, bridge: BridgeLine) {
@ -362,11 +382,12 @@ impl LoxServerContext {
let mut hasher = Sha1::new();
hasher.update(&bridge.fingerprint);
let fingerprint: [u8; 20] = hasher.finalize().into();
let fingerprint_str = array_bytes::bytes2hex("", fingerprint);
// Add bucket to existing entry or add new entry
if tp_bridge_infos.contains_key(&fingerprint) {
if tp_bridge_infos.contains_key(&fingerprint_str) {
tp_bridge_infos
.get_mut(&fingerprint)
.get_mut(&fingerprint_str)
.unwrap()
.buckets
.insert(bucket);
@ -374,7 +395,7 @@ impl LoxServerContext {
let mut buckets = HashSet::<Scalar>::new();
buckets.insert(bucket);
tp_bridge_infos.insert(
fingerprint,
fingerprint_str,
BridgeVerificationInfo {
bridge_line: *bridge,
buckets: buckets,
@ -807,13 +828,12 @@ impl LoxServerContext {
};
// TODO: Forward this information to rdsys
for bridge_str in blocked_bridges.keys() {
let bridge = array_bytes::hex2array(bridge_str).unwrap();
if self.tp_bridge_infos.lock().unwrap().contains_key(&bridge) {
if self.tp_bridge_infos.lock().unwrap().contains_key(bridge_str) {
let bl = self
.tp_bridge_infos
.lock()
.unwrap()
.get(&bridge)
.get(bridge_str)
.unwrap()
.bridge_line;
self.mark_blocked(bl);
@ -828,7 +848,7 @@ impl LoxServerContext {
.tp_bridge_infos
.lock()
.unwrap()
.get(&report.fingerprint)
.get(&array_bytes::bytes2hex("", report.fingerprint))
{
Some(bridge_info) => report.verify(&bridge_info),
None => false,
@ -871,7 +891,7 @@ impl LoxServerContext {
.tp_bridge_infos
.lock()
.unwrap()
.get(&report.fingerprint)
.get(&array_bytes::bytes2hex("", report.fingerprint))
{
Some(bridge_info) => report.verify(la, &bridge_info, &Htable),
None => false,
@ -920,3 +940,212 @@ fn prepare_header(response: String) -> Response<Body> {
.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*"));
resp
}
#[cfg(test)]
mod tests {
use crate::{fake_resource_state::TestResourceState, metrics::Metrics, BridgeConfig};
use lox_library::{bridge_table::MAX_BRIDGES_PER_BUCKET, BridgeAuth, BridgeDb};
use std::{
collections::HashMap,
env, fs,
sync::{Arc, Mutex},
};
use troll_patrol::bridge_verification_info::BridgeVerificationInfo;
use super::LoxServerContext;
struct TestHarness {
context: LoxServerContext,
}
impl TestHarness {
fn new() -> Self {
let bridgedb = BridgeDb::new();
let mut lox_auth = BridgeAuth::new(bridgedb.pubkey);
lox_auth.enc_bridge_table();
let context = LoxServerContext {
db: Arc::new(Mutex::new(bridgedb)),
ba: Arc::new(Mutex::new(lox_auth)),
extra_bridges: Arc::new(Mutex::new(Vec::new())),
to_be_replaced_bridges: Arc::new(Mutex::new(Vec::new())),
tp_bridge_infos: Arc::new(Mutex::new(
HashMap::<String, BridgeVerificationInfo>::new(),
)),
metrics: Metrics::default(),
};
Self { context }
}
fn new_with_bridges() -> Self {
let mut bridgedb = BridgeDb::new();
let mut lox_auth = BridgeAuth::new(bridgedb.pubkey);
// Make 3 x num_buckets open invitation bridges, in sets of 3
for _ in 0..5 {
let bucket = [
lox_utils::random(),
lox_utils::random(),
lox_utils::random(),
];
let _ = lox_auth.add_openinv_bridges(bucket, &mut bridgedb);
}
// Add hot_spare more hot spare buckets
for _ in 0..5 {
let bucket = [
lox_utils::random(),
lox_utils::random(),
lox_utils::random(),
];
let _ = lox_auth.add_spare_bucket(bucket, &mut bridgedb);
}
// Create the encrypted bridge table
lox_auth.enc_bridge_table();
let context = LoxServerContext {
db: Arc::new(Mutex::new(bridgedb)),
ba: Arc::new(Mutex::new(lox_auth)),
extra_bridges: Arc::new(Mutex::new(Vec::new())),
to_be_replaced_bridges: Arc::new(Mutex::new(Vec::new())),
tp_bridge_infos: Arc::new(Mutex::new(
HashMap::<String, BridgeVerificationInfo>::new(),
)),
metrics: Metrics::default(),
};
Self { context }
}
}
fn get_config() -> BridgeConfig {
env::set_var("BRIDGE_CONFIG_PATH", "bridge_config.json");
let path = env::var("BRIDGE_CONFIG_PATH").unwrap();
let config_file = fs::File::open(&path).unwrap();
serde_json::from_reader(config_file).unwrap()
}
#[test]
fn test_sync_with_bridgetable_only_working_resources() {
let bridge_config = get_config();
// Add bridges to empty bridge table and update with changed bridge state
let th = TestHarness::new();
let mut rs = TestResourceState::default();
for _ in 0..5 {
rs.add_working_resource();
}
assert_ne!(rs.rstate.working, None);
assert_eq!(rs.rstate.not_working, None);
th.context
.sync_with_bridgetable(bridge_config.watched_blockages.clone(), rs.rstate.clone());
let mut reachable_expected_length = rs.rstate.clone().working.unwrap().len();
let expected_extra_bridges = reachable_expected_length % MAX_BRIDGES_PER_BUCKET;
if expected_extra_bridges != 0 {
reachable_expected_length = reachable_expected_length - expected_extra_bridges;
}
assert_eq!(
th.context.ba.lock().unwrap().bridge_table.reachable.len(),
reachable_expected_length,
"Unexpected number of reachable bridges"
);
// Extra bridges should be cleared from the Lox Context after each sync
assert!(
th.context.extra_bridges.lock().unwrap().is_empty(),
"Extra bridges should be empty after sync"
);
}
#[test]
fn test_sync_with_bridgetable_working_and_not_working_resources() {
let bridge_config = get_config();
// Add bridges to empty bridge table and update with changed bridge state
let th = TestHarness::new();
let mut rs = TestResourceState::default();
for _ in 0..5 {
rs.add_working_resource();
}
for _ in 0..5 {
rs.add_not_working_resource()
}
assert_ne!(rs.rstate.working, None);
assert_ne!(rs.rstate.not_working, None);
th.context
.sync_with_bridgetable(bridge_config.watched_blockages.clone(), rs.rstate.clone());
let mut reachable_expected_length = rs.rstate.clone().working.unwrap().len();
let expected_extra_bridges = reachable_expected_length % MAX_BRIDGES_PER_BUCKET;
if expected_extra_bridges != 0 {
reachable_expected_length = reachable_expected_length - expected_extra_bridges;
}
assert_eq!(
th.context.ba.lock().unwrap().bridge_table.reachable.len(),
reachable_expected_length,
"Unexpected number of reachable bridges"
);
// Extra bridges should be cleared from the Lox Context after each sync
assert!(
th.context.extra_bridges.lock().unwrap().is_empty(),
"Extra bridges should be empty after sync"
);
}
#[test]
fn test_sync_with_preloaded_obsolete_bridgetable() {
// Tests the case where all bridges in the bridgetable are no longer in rdsys.
// In this case, all bridges should be replaced. If it's a bridge in a spare bucket, just remove the other bridges
// from the spare bucket and delete the bridge
let bridge_config = get_config();
// Sync bridges to non-empty bridge table with disparate sets of bridges
let th_with_bridges = TestHarness::new_with_bridges(); //Creates 5 open invitation and 5 hot spare buckets, so 30 total buckets to be replaced
let mut rs = TestResourceState::default();
for _ in 0..5 {
rs.add_working_resource();
}
assert_ne!(rs.rstate.working, None);
assert_eq!(rs.rstate.not_working, None);
assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.reachable.len(), 15+15, "Unexpected number of reachable bridges should equal the number of open invitation bridges plus the number of spares added: 2x5x3");
assert_eq!(
th_with_bridges
.context
.ba
.lock()
.unwrap()
.bridge_table
.spares
.len(),
5,
"Unexpected number of spare bridges, should be 5"
);
// All potentially distributed resources (i.e., those assigned to open invitation/trusted buckets)
// not found in the rdsys update will first be replaced with any new resources coming in from rdsys then
// by bridges from the hot spare buckets. In this case, the hot spare buckets are also not in the bridge table
// so will also be replaced.
// Since there are fewer working resources than resources that have populated the bridge table, this update will
// exhaust the spare buckets and leave some obsolete bridges. The set of open invitation/trusted buckets should be
// preserved (5 open invitation buckets * 3)
th_with_bridges
.context
.sync_with_bridgetable(bridge_config.watched_blockages, rs.rstate.clone());
assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.reachable.len(), 15, "Unexpected number of reachable bridges should equal the number of open invitation bridges added: 5x3");
assert_eq!(
th_with_bridges
.context
.ba
.lock()
.unwrap()
.bridge_table
.spares
.len(),
0,
"Unexpected number of spare bridges, should be exhausted"
);
assert_eq!(th_with_bridges.context.ba.lock().unwrap().bridge_table.unallocated_bridges.len(), 0, "Unexpected number of unallocated bridges, should be 0 (All spare buckets and new resources for replacement exhausted)"
);
assert_eq!(
th_with_bridges.context.extra_bridges.lock().unwrap().len(),
0,
"Unexpected number of extra bridges"
);
}
}

1
crates/lox-distributor/src/main.rs
View File

@ -26,6 +26,7 @@ use std::{
mod db_handler;
use db_handler::DB;
mod fake_resource_state;
mod lox_context;
mod metrics;
use metrics::Metrics;

22
crates/lox-distributor/src/request_handler.rs
View File

@ -74,14 +74,10 @@ mod tests {
use super::*;
use base64::{engine::general_purpose, Engine as _};
use base64::{engine::general_purpose, Engine};
use chrono::{Duration, Utc};
use julianday::JulianDay;
use lox_library::{
bridge_table::{self, BridgeLine},
cred::BucketReachability,
proto, BridgeAuth, BridgeDb,
};
use lox_library::{bridge_table, bridge_table::BridgeLine, cred::BucketReachability, proto, BridgeAuth, BridgeDb};
use rand::RngCore;
use std::sync::{Arc, Mutex};
@ -226,13 +222,21 @@ mod tests {
// Make 3 x num_buckets open invitation bridges, in sets of 3
for _ in 0..5 {
let bucket = [random(), random(), random()];
let bucket = [
lox_utils::random(),
lox_utils::random(),
lox_utils::random(),
];
let _ = lox_auth.add_openinv_bridges(bucket, &mut bridgedb);
}
// Add hot_spare more hot spare buckets
for _ in 0..5 {
let bucket = [random(), random(), random()];
let bucket = [
lox_utils::random(),
lox_utils::random(),
lox_utils::random(),
];
let _ = lox_auth.add_spare_bucket(bucket, &mut bridgedb);
}
// Create the encrypted bridge table
@ -243,7 +247,7 @@ mod tests {
extra_bridges: Arc::new(Mutex::new(Vec::new())),
to_be_replaced_bridges: Arc::new(Mutex::new(Vec::new())),
tp_bridge_infos: Arc::new(Mutex::new(std::collections::HashMap::<
[u8; 20],
String,
troll_patrol::bridge_verification_info::BridgeVerificationInfo,
>::new())),
metrics: Metrics::default(),

6
crates/lox-distributor/src/resource_parser.rs
View File

@ -189,7 +189,7 @@ mod tests {
"123.456.789.100".to_owned(),
3002,
"BE84A97D02130470A1C77839954392BA979F7EE1".to_owned(),
ACCEPTED_HOURS_OF_FAILURE-1,
ACCEPTED_HOURS_OF_FAILURE - 1,
);
let resource_two = make_resource(
"https".to_owned(),
@ -203,7 +203,7 @@ mod tests {
"123.222.333.444".to_owned(),
6002,
"C56B9EF202130470A1C77839954392BA979F7FF9".to_owned(),
ACCEPTED_HOURS_OF_FAILURE+2,
ACCEPTED_HOURS_OF_FAILURE + 2,
);
let resource_three = make_resource(
"scramblesuit".to_owned(),
@ -217,7 +217,7 @@ mod tests {
"443.288.222.100".to_owned(),
3042,
"5E3A8BD902130470A1C77839954392BA979F7B46".to_owned(),
ACCEPTED_HOURS_OF_FAILURE+1,
ACCEPTED_HOURS_OF_FAILURE + 1,
);
let resource_four = make_resource(
"https".to_owned(),

2
crates/lox-distributor/src/troll_patrol_handler.rs
View File

@ -142,7 +142,7 @@ mod tests {
extra_bridges: Arc::new(Mutex::new(Vec::new())),
to_be_replaced_bridges: Arc::new(Mutex::new(Vec::new())),
tp_bridge_infos: Arc::new(Mutex::new(
HashMap::<[u8; 20], BridgeVerificationInfo>::new(),
HashMap::<String, BridgeVerificationInfo>::new(),
)),
metrics: Metrics::default(),
};

8
crates/lox-library/Cargo.toml
View File

@ -16,7 +16,7 @@ readme = "README.md"
[dependencies]
curve25519-dalek = { version = "4", default-features = false, features = ["serde", "rand_core", "digest"] }
ed25519-dalek = { version = "2", features = ["serde", "rand_core"] }
bincode = "1"
bincode = "1.3.3"
chrono = "0.4"
rand = { version = "0.8", features = ["std_rng"]}
serde = "1.0.197"
@ -26,11 +26,11 @@ statistical = "1.0.0"
lazy_static = "1"
hex_fmt = "0.3"
aes-gcm = { version = "0.10", features =["aes"]}
base64 = "0.21"
time = "0.3.34"
base64 = "0.22.0"
time = "0.3.36"
prometheus = "0.13.3"
subtle = "2.5"
thiserror = "1.0.58"
thiserror = "1.0.59"
lox-zkp = { git = "https://gitlab.torproject.org/onyinyang/lox-zkp", version = "0.8.0" }
[features]

48
crates/lox-library/src/lib.rs
View File

@ -73,6 +73,7 @@ pub enum ReplaceSuccess {
NotFound = 0,
NotReplaced = 1,
Replaced = 2,
Removed = 3,
}
/// This error is thrown if the number of buckets/keys in the bridge table
@ -511,6 +512,17 @@ impl BridgeAuth {
return res;
}
}
// Also check the unallocated bridges just in case there is a bridge that should be updated there
let unallocated_bridges = self.bridge_table.unallocated_bridges.clone();
for (i, unallocated_bridge) in unallocated_bridges.iter().enumerate() {
if unallocated_bridge.uid_fingerprint == bridge.uid_fingerprint {
// Now we must remove the old bridge from the unallocated bridges and insert the new bridge
// in its place
self.bridge_table.unallocated_bridges.remove(i);
self.bridge_table.unallocated_bridges.push(*bridge);
res = true;
}
}
// If this is returned, we assume that the bridge wasn't found in the bridge table
// and therefore should be treated as a "new bridge"
res
@ -535,6 +547,25 @@ impl BridgeAuth {
Ok(())
}
// Remove an unallocated resource
pub fn remove_unallocated(&mut self, bridge: &BridgeLine) -> ReplaceSuccess {
// Check if the bridge is in the unallocated bridges and remove the bridge if so
// Bridges in spare buckets should have been moved to the unallocated bridges
if self.bridge_table.unallocated_bridges.contains(bridge) {
let index = self
.bridge_table
.unallocated_bridges
.iter()
.position(|x| x == bridge)
.unwrap();
self.bridge_table.unallocated_bridges.swap_remove(index);
// A bridge that is in the unallocated_bridges is not exactly replaced
// but is successfully handled and no further action is needed
return ReplaceSuccess::Removed;
}
ReplaceSuccess::NotFound
}
/// Attempt to remove a bridge that is failing tests and replace it with a bridge from
/// available_bridge or from a spare bucket
pub fn bridge_replace(
@ -544,8 +575,23 @@ impl BridgeAuth {
) -> ReplaceSuccess {
let reachable_bridges = &self.bridge_table.reachable.clone();
let Some(positions) = reachable_bridges.get(bridge) else {
return ReplaceSuccess::NotFound;
return self.remove_unallocated(bridge);
};
// Check if the bridge is in a spare bucket first, if it is, dissolve the bucket
if let Some(spare) = self
.bridge_table
.spares
.iter()
.find(|x| positions.iter().any(|(bucketnum, _)| &bucketnum == x))
.cloned()
{
let Ok(_) = self.dissolve_spare_bucket(spare) else {
return ReplaceSuccess::NotReplaced;
};
// Next Check if the bridge is in the unallocated bridges and remove the bridge if so
// Bridges in spare buckets should have been moved to the unallocated bridges
return self.remove_unallocated(bridge);
}
// select replacement:
// - first try the given bridge
// - second try to pick one from the set of available bridges

80
crates/lox-library/src/tests.rs
View File

@ -1060,22 +1060,39 @@ fn test_update_bridge() {
#[test]
fn test_bridge_replace() {
// Create 3 open invitation buckets and 3 spare buckets
let cases = vec!["not found", "available", "unallocated", "spare", "failed"];
let cases = vec![
"not found",
"available",
"unallocated",
"use_spare",
"remove_spare",
"failed",
];
let num_buckets = 5;
let hot_spare = 0;
for case in cases {
let table_size: usize;
let mut th: TestHarness;
if String::from(case) != "failed" {
th = TestHarness::new();
} else {
th = TestHarness::new_buckets(num_buckets, hot_spare);
match case {
"failed" => {
th = TestHarness::new_buckets(num_buckets, hot_spare);
table_size = th.ba.bridge_table.buckets.len();
}
"remove_spare" => {
th = TestHarness::new_buckets(0, 5);
table_size = th.ba.bridge_table.buckets.len();
}
_ => {
th = TestHarness::new();
// Ensure that a randomly selected bucket isn't taken from the set of spare bridges
table_size = th.ba.bridge_table.buckets.len() - 5;
}
}
// Randomly select a bridge to replace
let table_size = th.ba.bridge_table.buckets.len();
let mut num = 100000;
while !th.ba.bridge_table.buckets.contains_key(&num) {
num = rand::thread_rng().gen_range(0..th.ba.bridge_table.counter);
num = rand::thread_rng().gen_range(0..table_size as u32);
}
println!("chosen num is: {:?}", num);
let replaceable_bucket = *th.ba.bridge_table.buckets.get(&num).unwrap();
@ -1093,7 +1110,10 @@ fn test_bridge_replace() {
// Case zero: bridge to be replaced is not in the bridgetable
let random_bridgeline = BridgeLine::random();
assert!(
!th.ba.bridge_table.reachable.contains_key(&random_bridgeline),
!th.ba
.bridge_table
.reachable
.contains_key(&random_bridgeline),
"Random bridgeline happens to be in the bridge_table (and should not be)"
);
assert!(
@ -1133,8 +1153,10 @@ fn test_bridge_replace() {
.is_some(),
"Replacement bridge not added to reachable bridges"
);
println!("Table Size {:?}", table_size);
println!("Bucket length {:?}", th.ba.bridge_table.buckets.len() - 5);
assert!(
table_size == th.ba.bridge_table.buckets.len(),
table_size == th.ba.bridge_table.buckets.len() - 5,
"Number of buckets changed size"
);
assert!(
@ -1175,7 +1197,7 @@ fn test_bridge_replace() {
"Replacement bridge not added to reachable bridges"
);
assert!(
table_size == th.ba.bridge_table.buckets.len(),
table_size == th.ba.bridge_table.buckets.len() - 5,
"Number of buckets changed size"
);
assert!(
@ -1185,7 +1207,7 @@ fn test_bridge_replace() {
println!("Successfully added unallocated bridgeline");
}
"spare" => {
"use_spare" => {
// Case three: available_bridge == null and unallocated_bridges == null
assert!(
th.ba.bridge_table.unallocated_bridges.is_empty(),
@ -1205,7 +1227,7 @@ fn test_bridge_replace() {
);
// Remove a spare bucket to replace bridge, buckets decrease by 1
assert!(
(table_size - 1) == th.ba.bridge_table.buckets.len(),
(table_size - 1) == th.ba.bridge_table.buckets.len() - 5,
"Number of buckets changed size"
);
assert!(
@ -1215,6 +1237,40 @@ fn test_bridge_replace() {
println!("Successfully added bridgeline from spare");
}
"remove_spare" => {
// Case three: available_bridge == null and unallocated_bridges == null
assert!(
th.ba.bridge_table.unallocated_bridges.is_empty(),
"Unallocated bridges should have a length of 0"
);
assert!(
th.ba.bridge_replace(replacement_bridge, None) == ReplaceSuccess::Removed,
"Bridge was replaced with available spare, instead of being removed"
);
assert!(
th.ba.bridge_table.unallocated_bridges.len() == 2,
"Unallocated bridges should have a length of 2"
);
assert!(
th.ba
.bridge_table
.reachable
.get(replacement_bridge)
.is_none(),
"Replacement bridge still marked as reachable"
);
// Remove a spare bucket to replace bridge, buckets decrease by 1
assert!(
(table_size - 1) == th.ba.bridge_table.buckets.len(),
"Number of buckets changed size"
);
assert!(
th.ba.bridge_table.unallocated_bridges.len() == 2,
"Extra spare bridges not added to unallocated bridges"
);
println!("Successfully removed a spare bridgeline marked to be replaced");
}
"failed" => {
// Case four: available_bridge == None and unallocated_bridges == None and spare buckets == None
assert!(

4
crates/lox-utils/Cargo.toml
View File

@ -12,8 +12,10 @@ categories = ["rust-patterns"]
repository = "https://gitlab.torproject.org/tpo/anti-censorship/lox.git/"
[dependencies]
chrono = { version = "0.4.34", features = ["serde", "clock"] }
base64 = "0.22.0"
chrono = { version = "0.4.38", features = ["serde", "clock"] }
lox-library = {path = "../lox-library", version = "0.1.0"}
rand = "0.8.5"
serde = "1"
serde_json = "1.0.113"
serde_with = "3.7.0"

71
crates/lox-utils/src/lib.rs
View File

@ -1,3 +1,4 @@
use base64::{engine::general_purpose, Engine as _};
use chrono::{DateTime, Utc};
use lox_library::bridge_table::{
from_scalar, BridgeLine, BridgeTable, EncryptedBucket, MAX_BRIDGES_PER_BUCKET,
@ -5,18 +6,53 @@ use lox_library::bridge_table::{
use lox_library::cred::{BucketReachability, Invitation, Lox};
use lox_library::proto::{self, check_blockage, level_up, trust_promotion};
use lox_library::{IssuerPubKey, OPENINV_LENGTH};
use rand::RngCore;
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use std::array::TryFromSliceError;
use std::collections::HashMap;
#[serde_as]
const LOX_INVITE_TOKEN: &str = "loxinvite_";
#[derive(Serialize, Deserialize)]
pub struct Invite {
#[serde_as(as = "[_; OPENINV_LENGTH]")]
#[serde(with = "base64serde")]
pub invite: [u8; OPENINV_LENGTH],
}
mod base64serde {
use base64::{engine::general_purpose::STANDARD_NO_PAD, Engine as _};
use lox_library::OPENINV_LENGTH;
use serde::{Deserialize, Serialize};
use serde::{Deserializer, Serializer};
use crate::LOX_INVITE_TOKEN;
pub fn serialize<S: Serializer>(v: &[u8; OPENINV_LENGTH], s: S) -> Result<S::Ok, S::Error> {
let mut base64 = STANDARD_NO_PAD.encode(v);
base64.insert_str(0, LOX_INVITE_TOKEN);
String::serialize(&base64, s)
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; OPENINV_LENGTH], D::Error> {
let mut base64 = String::deserialize(d)?;
let encoded_str = base64.split_off(LOX_INVITE_TOKEN.len());
if base64 != LOX_INVITE_TOKEN {
return Err(serde::de::Error::custom("Token identifier does not match"))
}
match STANDARD_NO_PAD.decode(encoded_str) {
Ok(output) => {
let out: Result<[u8; OPENINV_LENGTH], D::Error> = match output.try_into() {
Ok(out) => Ok(out),
Err(e) => Err(serde::de::Error::custom(String::from_utf8(e).unwrap())),
};
out
}
Err(e) => Err(serde::de::Error::custom(e)),
}
}
}
#[derive(Deserialize, Serialize)]
pub struct OpenReqState {
pub request: proto::open_invite::Request,
@ -154,3 +190,34 @@ pub fn calc_test_days(trust_level: i64) -> i64 {
// }
total
}
pub fn random() -> BridgeLine {
let mut rng = rand::thread_rng();
let mut res: BridgeLine = BridgeLine::default();
// Pick a random 4-byte address
let mut addr: [u8; 4] = [0; 4];
rng.fill_bytes(&mut addr);
// If the leading byte is 224 or more, that's not a valid IPv4
// address. Choose an IPv6 address instead (but don't worry too
// much about it being well formed).
if addr[0] >= 224 {
rng.fill_bytes(&mut res.addr);
} else {
// Store an IPv4 address as a v4-mapped IPv6 address
res.addr[10] = 255;
res.addr[11] = 255;
res.addr[12..16].copy_from_slice(&addr);
};
let ports: [u16; 4] = [443, 4433, 8080, 43079];
let portidx = (rng.next_u32() % 4) as usize;
res.port = ports[portidx];
res.uid_fingerprint = rng.next_u64();
let mut cert: [u8; 52] = [0; 52];
rng.fill_bytes(&mut cert);
let infostr: String = format!(
"obfs4 cert={}, iat-mode=0",
general_purpose::STANDARD_NO_PAD.encode(cert)
);
res.info[..infostr.len()].copy_from_slice(infostr.as_bytes());
res
}

4
crates/lox-wasm/Cargo.toml
View File

@ -21,7 +21,7 @@ lazy_static = "1.4.0"
lox-library = { path = "../lox-library", version = "0.1.0" }
lox_utils = { path = "../lox-utils", version = "0.1.0" }
wasm-bindgen = "0.2"
time = "0.3.34"
time = "0.3.36"
serde_json = "1.0.113"
console_error_panic_hook = "0.1.7"
@ -30,5 +30,5 @@ rand = { version = "0.7", features = ["wasm-bindgen"] }
lox-zkp = { git = "https://gitlab.torproject.org/onyinyang/lox-zkp", version = "0.8.0" }
[dependencies.chrono]
version = "0.4.34"
version = "0.4.38"
features = ["serde", "wasmbind"]

2
crates/lox-wasm/index.js
View File

@ -199,7 +199,7 @@ function request_open_invite() {
return new Promise((fulfill, reject) => {
loxServerPostRequest("/invite", null).then((response) => {
console.log("Got invitation token: " + response.invite);
fulfill(response.invite);
fulfill(JSON.stringify(response));
return;
}).catch(() => {
console.log("Error requesting open invite from Lox server");

45
crates/lox-wasm/src/lib.rs
View File

@ -29,9 +29,13 @@ pub fn set_panic_hook() {
// Receives an invite and prepares an open_invite request, returning the
// Request and State
#[wasm_bindgen]
pub fn open_invite(invite: &[u8]) -> Result<String, JsValue> {
log(&format!("Using invite: {:?}", invite));
let token = match lox_utils::validate(invite) {
pub fn open_invite(base64_invite: String) -> Result<String, JsValue> {
log(&format!("Using invite: {:?}", base64_invite));
let invite: lox_utils::Invite = match serde_json::from_str(&base64_invite) {
Ok(invite) => invite,
Err(e) => return Err(JsValue::from(e.to_string())),
};
let token = match lox_utils::validate(&invite.invite) {
Ok(token) => token,
Err(e) => return Err(JsValue::from(e.to_string())),
};
@ -831,29 +835,32 @@ pub fn get_next_unlock(constants_str: String, lox_cred_str: String) -> Result<St
days_to_next_level + constants.level_interval[trust_level as usize + 1];
invitations_at_next_level = constants.level_invitations[trust_level as usize + 1];
}
let days_to_blockage_migration_unlock =
match trust_level < constants.min_blockage_migration_trust_level {
// If the credential is greater than the minimum level that enables
// migrating after a blockage, the time to unlock is 0, otherwise we
// add the time to upgrade until that level
true => {
let mut blockage_days = days_to_next_level;
let mut count = 1;
while trust_level + count < constants.min_blockage_migration_trust_level {
blockage_days += constants.level_interval[trust_level as usize + count as usize];
count += 1;
}
blockage_days
let days_to_blockage_migration_unlock = match trust_level
< constants.min_blockage_migration_trust_level
{
// If the credential is greater than the minimum level that enables
// migrating after a blockage, the time to unlock is 0, otherwise we
// add the time to upgrade until that level
true => {
let mut blockage_days = days_to_next_level;
let mut count = 1;
while trust_level + count < constants.min_blockage_migration_trust_level {
blockage_days += constants.level_interval[trust_level as usize + count as usize];
count += 1;
}
false => 0,
};
blockage_days
}
false => 0,
};
let day_of_level_unlock =
(scalar_u32(&lox_cred.lox_credential.level_since).unwrap() + days_to_next_level) as i32;
let level_unlock_date = JulianDay::new(day_of_level_unlock).to_date();
let day_of_invite_unlock =
(scalar_u32(&lox_cred.lox_credential.level_since).unwrap() + days_to_invite_inc) as i32;
let invite_unlock_date = JulianDay::new(day_of_invite_unlock).to_date();
let day_of_blockage_migration_unlock = (scalar_u32(&lox_cred.lox_credential.level_since).unwrap() + days_to_blockage_migration_unlock) as i32;
let day_of_blockage_migration_unlock = (scalar_u32(&lox_cred.lox_credential.level_since)
.unwrap()
+ days_to_blockage_migration_unlock) as i32;
let blockage_migration_unlock_date =
JulianDay::new(day_of_blockage_migration_unlock as i32).to_date();
let next_unlock: lox_utils::LoxNextUnlock = lox_utils::LoxNextUnlock {

2
crates/rdsys-backend-api/Cargo.toml
View File

@ -25,4 +25,4 @@ reqwest = { version = "0.11", features = ["json", "stream"]}
tokio-stream = "0.1.15"
futures = "0.3.30"
tokio-util = "0.7.10"
chrono = { version = "0.4.34", features = ["serde", "clock"] }
chrono = { version = "0.4.38", features = ["serde", "clock"] }

6
crates/rdsys-backend-api/src/proto.rs
View File

@ -10,13 +10,13 @@ pub struct ResourceRequest {
pub resource_types: Vec<String>,
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct TestResults {
pub last_passed: DateTime<Utc>,
}
/// Representation of a bridge resource
#[derive(Deserialize, PartialEq, Eq, Debug)]
#[derive(Clone, Deserialize, PartialEq, Eq, Debug)]
pub struct Resource {
pub r#type: String,
pub blocked_in: HashMap<String, bool>,
@ -57,7 +57,7 @@ impl Resource {
}
/// A ResourceState holds information about new, changed, or pruned resources
#[derive(Deserialize, PartialEq, Eq, Debug)]
#[derive(Clone, Deserialize, Default, PartialEq, Eq, Debug)]
pub struct ResourceState {
pub working: Option<Vec<Resource>>,
pub not_working: Option<Vec<Resource>>,