Cargo.toml

@@ -40,6 +40,3 @@ x25519-dalek = { version = "2", features = ["serde", "static_secrets"] }
 
 [dev-dependencies]
 base64 = "0.21.7"
-
-[features]
-simulation = []

src/analysis.rs

@@ -1,5 +1,5 @@
 use crate::{BridgeInfo, BridgeInfoType};
-use lox_library::proto::trust_promotion::UNTRUSTED_INTERVAL;
+use lox_library::proto::{level_up::LEVEL_INTERVAL, trust_promotion::UNTRUSTED_INTERVAL};
 use nalgebra::DVector;
 use statrs::distribution::{Continuous, MultivariateNormal, Normal};
 use std::{

src/extra_info.rs

@@ -5,13 +5,10 @@ Note, this is NOT a complete implementation of the document format.
 use chrono::DateTime;
 use julianday::JulianDay;
 use serde::{Deserialize, Serialize};
-use std::{
+use std::collections::{BTreeMap, HashMap, HashSet};
-    collections::{BTreeMap, HashMap, HashSet},
-    fmt,
-};
 
 /// Fields we need from extra-info document
-#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
+#[derive(Eq, PartialEq, Hash, Serialize, Deserialize)]
 pub struct ExtraInfo {
     /// Bridge nickname, probably unused
     pub nickname: String,
@@ -140,36 +137,3 @@ impl ExtraInfo {
         set
     }
 }
-
-/// Convert the ExtraInfo object to a string record, as in a downloaded file
-impl fmt::Display for ExtraInfo {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut str = String::from("@type bridge-extra-info 1.3");
-        str.push_str(
-            format!(
-                "\nextra-info {} {}",
-                self.nickname,
-                array_bytes::bytes2hex("", self.fingerprint).to_uppercase()
-            )
-            .as_str(),
-        );
-        let date = JulianDay::new(self.date.try_into().unwrap()).to_date();
-        str.push_str(format!("\nbridge-stats-end {} 23:59:59 (86400 s)", date).as_str());
-        str.push_str(format!("\npublished {} 23:59:59", date).as_str());
-
-        // These should be sorted in descending order by count, but that's not
-        // necessary for our purposes.
-        str.push_str("\nbridge-ips ");
-        let mut first_cc = true;
-        for (cc, count) in &self.bridge_ips {
-            if !first_cc {
-                str.push(',');
-            }
-            str.push_str(format!("{}={}", cc, count,).as_str());
-            first_cc = false;
-        }
-        str.push_str("\n");
-
-        write!(f, "{}", str)
-    }
-}

src/lib.rs

@@ -20,11 +20,6 @@ pub mod negative_report;
 pub mod positive_report;
 pub mod request_handler;
 
-#[cfg(any(test, feature = "simulation"))]
-pub mod simulation {
-    pub mod extra_infos_server;
-}
-
 use analysis::Analyzer;
 use extra_info::*;
 use negative_report::*;
@@ -210,15 +205,16 @@ pub fn add_bridge_to_db(db: &Db, fingerprint: [u8; 20]) {
 
 // Download a webpage and return it as a string
 pub async fn download(url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-    if url.starts_with("https://") {
     let https = hyper_rustls::HttpsConnectorBuilder::new()
         .with_native_roots()
         .expect("no native root CA certificates found")
         .https_only()
         .enable_http1()
         .build();
+
     let client: hyper_util::client::legacy::Client<_, Empty<Bytes>> =
         hyper_util::client::legacy::Client::builder(TokioExecutor::new()).build(https);
+
     println!("Downloading {}", url);
     let mut res = client.get(url.parse()?).await?;
     assert_eq!(res.status(), StatusCode::OK);
@@ -230,21 +226,6 @@ pub async fn download(url: &str) -> Result<String, Box<dyn std::error::Error + S
         }
     }
     Ok(body_str)
-    } else {
-        let client: hyper_util::client::legacy::Client<_, Empty<Bytes>> =
-            hyper_util::client::legacy::Client::builder(TokioExecutor::new()).build_http();
-        println!("Downloading {}", url);
-        let mut res = client.get(url.parse()?).await?;
-        assert_eq!(res.status(), StatusCode::OK);
-        let mut body_str = String::default();
-        while let Some(next) = res.frame().await {
-            let frame = next?;
-            if let Some(chunk) = frame.data_ref() {
-                body_str.push_str(&String::from_utf8(chunk.to_vec())?);
-            }
-        }
-        Ok(body_str)
-    }
 }
 
 // Process extra-infos

src/simulation/extra_infos_server.rs

@@ -1,163 +0,0 @@
-use crate::extra_info::ExtraInfo;
-
-use hyper::{
-    body::{self, Bytes},
-    header::HeaderValue,
-    server::conn::AddrStream,
-    service::{make_service_fn, service_fn},
-    Body, Method, Request, Response, Server,
-};
-use serde_json::json;
-use std::{collections::HashSet, convert::Infallible, net::SocketAddr, time::Duration};
-use tokio::{
-    spawn,
-    sync::{broadcast, mpsc, oneshot},
-    time::sleep,
-};
-
-async fn serve_extra_infos(
-    extra_infos_pages: &mut Vec<String>,
-    req: Request<Body>,
-) -> Result<Response<Body>, Infallible> {
-    match req.method() {
-        &Method::OPTIONS => Ok(Response::builder()
-            .header("Access-Control-Allow-Origin", HeaderValue::from_static("*"))
-            .header("Access-Control-Allow-Headers", "accept, content-type")
-            .header("Access-Control-Allow-Methods", "POST")
-            .status(200)
-            .body(Body::from("Allow POST"))
-            .unwrap()),
-        _ => match req.uri().path() {
-            "/" => Ok::<_, Infallible>(serve_index(&extra_infos_pages)),
-            "/add" => Ok::<_, Infallible>({
-                let bytes = body::to_bytes(req.into_body()).await.unwrap();
-                add_extra_infos(extra_infos_pages, bytes)
-            }),
-            path => Ok::<_, Infallible>({
-                // Serve the requested file
-                serve_extra_infos_file(&extra_infos_pages, path)
-            }),
-        },
-    }
-}
-
-pub async fn server() {
-    let (context_tx, context_rx) = mpsc::channel(32);
-    let request_tx = context_tx.clone();
-    let shutdown_cmd_tx = context_tx.clone();
-    let (shutdown_tx, mut shutdown_rx) = broadcast::channel(16);
-    let kill_context = shutdown_tx.subscribe();
-
-    let context_manager =
-        spawn(async move { create_context_manager(context_rx, kill_context).await });
-
-    let addr = SocketAddr::from(([127, 0, 0, 1], 8004));
-    let make_svc = make_service_fn(move |_conn: &AddrStream| {
-        let request_tx = request_tx.clone();
-        let service = service_fn(move |req| {
-            let request_tx = request_tx.clone();
-            let (response_tx, response_rx) = oneshot::channel();
-            let cmd = Command::Request {
-                req,
-                sender: response_tx,
-            };
-            async move {
-                request_tx.send(cmd).await.unwrap();
-                response_rx.await.unwrap()
-            }
-        });
-        async move { Ok::<_, Infallible>(service) }
-    });
-    let server = Server::bind(&addr).serve(make_svc);
-    println!("Listening on localhost:8004");
-    if let Err(e) = server.await {
-        eprintln!("server error: {}", e);
-    }
-}
-
-async fn create_context_manager(
-    context_rx: mpsc::Receiver<Command>,
-    mut kill: broadcast::Receiver<()>,
-) {
-    tokio::select! {
-        create_context = context_manager(context_rx) => create_context,
-        _ = kill.recv() => {println!("Shut down context_manager");},
-    }
-}
-
-async fn context_manager(mut context_rx: mpsc::Receiver<Command>) {
-    let mut extra_infos_pages = Vec::<String>::new();
-
-    while let Some(cmd) = context_rx.recv().await {
-        use Command::*;
-        match cmd {
-            Request { req, sender } => {
-                let response = serve_extra_infos(&mut extra_infos_pages, req).await;
-                if let Err(e) = sender.send(response) {
-                    eprintln!("Server Response Error: {:?}", e);
-                }
-                sleep(Duration::from_millis(1)).await;
-            }
-            Shutdown { shutdown_sig } => {
-                drop(shutdown_sig);
-            }
-        }
-    }
-}
-
-#[derive(Debug)]
-enum Command {
-    Request {
-        req: Request<Body>,
-        sender: oneshot::Sender<Result<Response<Body>, Infallible>>,
-    },
-    Shutdown {
-        shutdown_sig: broadcast::Sender<()>,
-    },
-}
-
-fn add_extra_infos(extra_infos_pages: &mut Vec<String>, request: Bytes) -> Response<Body> {
-    let extra_infos: HashSet<ExtraInfo> = match serde_json::from_slice(&request) {
-        Ok(req) => req,
-        Err(e) => {
-            let response = json!({"error": e.to_string()});
-            let val = serde_json::to_string(&response).unwrap();
-            return prepare_header(val);
-        }
-    };
-
-    let mut extra_infos_file = String::new();
-    for extra_info in extra_infos {
-        extra_infos_file.push_str(extra_info.to_string().as_str());
-    }
-    extra_infos_pages.push(extra_infos_file);
-    prepare_header("OK".to_string())
-}
-
-fn serve_index(extra_infos_pages: &Vec<String>) -> Response<Body> {
-    let mut body_str = String::new();
-    for i in 0..extra_infos_pages.len() {
-        body_str
-            .push_str(format!("<a href=\"{}-extra-infos\">{}-extra-infos</a>\n", i, i).as_str());
-    }
-    prepare_header(body_str)
-}
-
-fn serve_extra_infos_file(extra_infos_pages: &Vec<String>, path: &str) -> Response<Body> {
-    if path.ends_with("-extra-infos") {
-        if let Ok(index) = &path[1..(path.len() - "-extra-infos".len())].parse::<usize>() {
-            if extra_infos_pages.len() > *index {
-                return prepare_header(extra_infos_pages[*index].clone());
-            }
-        }
-    }
-    prepare_header("Not a valid file".to_string())
-}
-
-// Prepare HTTP Response for successful Server Request
-fn prepare_header(response: String) -> Response<Body> {
-    let mut resp = Response::new(Body::from(response));
-    resp.headers_mut()
-        .insert("Access-Control-Allow-Origin", HeaderValue::from_static("*"));
-    resp
-}

src/tests.rs

@@ -3,7 +3,6 @@
 use crate::{
     analysis::{blocked_in, Analyzer},
     bridge_verification_info::BridgeVerificationInfo,
-    simulation::extra_infos_server,
     *,
 };
 use lox_library::{
@@ -20,9 +19,7 @@ use sha1::{Digest, Sha1};
 use std::{
     collections::{BTreeMap, HashMap, HashSet},
     sync::{Arc, Mutex},
-    time::Duration,
 };
-use tokio::{spawn, time::sleep};
 use x25519_dalek::{PublicKey, StaticSecret};
 
 struct TestHarness {
@@ -153,12 +150,12 @@ pub fn random() -> BridgeLine {
 }
 
 #[tokio::test]
-async fn test_download_extra_infos() {
+async fn test_extra_infos() {
     let bridge_to_test =
         array_bytes::hex2array("72E12B89136B45BBC81D1EF0AC7DDDBB91B148DB").unwrap();
 
     // Open test database
-    let db: Db = sled::open("test_db_dei").unwrap();
+    let db: Db = sled::open("test_db_ei").unwrap();
 
     // Delete all data in test DB
     db.clear().unwrap();
@@ -183,110 +180,6 @@ async fn test_download_extra_infos() {
         bincode::deserialize(&db.get(bridge_to_test).unwrap().unwrap()).unwrap();
 }
 
-#[tokio::test]
-async fn test_simulate_extra_infos() {
-    let extra_info_str = r#"@type bridge-extra-info 1.3
-extra-info ElephantBridgeDE2 72E12B89136B45BBC81D1EF0AC7DDDBB91B148DB
-master-key-ed25519 eWxjRwAWW7n8BGG9fNa6rApmBFbe3f0xcD7dqwOICW8
-published 2024-04-06 03:51:04
-transport obfs4
-write-history 2024-04-05 04:55:22 (86400 s) 31665735680,14918491136,15423603712,36168353792,40396827648
-read-history 2024-04-05 04:55:22 (86400 s) 31799622656,15229917184,15479115776,36317251584,40444155904
-ipv6-write-history 2024-04-05 04:55:22 (86400 s) 5972127744,610078720,516897792,327949312,640708608
-ipv6-read-history 2024-04-05 04:55:22 (86400 s) 4156158976,4040448000,2935756800,4263080960,6513532928
-dirreq-write-history 2024-04-05 04:55:22 (86400 s) 625217536,646188032,618014720,584386560,600778752
-dirreq-read-history 2024-04-05 04:55:22 (86400 s) 18816000,19000320,18484224,17364992,18443264
-geoip-db-digest 44073997E1ED63E183B79DE2A1757E9997A834E3
-geoip6-db-digest C0BF46880C6C132D746683279CC90DD4B2D31786
-dirreq-stats-end 2024-04-05 06:51:23 (86400 s)
-dirreq-v3-ips ru=16,au=8,by=8,cn=8,gb=8,ir=8,mt=8,nl=8,pl=8,tn=8,tr=8,us=8
-dirreq-v3-reqs ru=72,gb=64,pl=32,cn=16,ir=16,us=16,au=8,by=8,mt=8,nl=8,tn=8,tr=8
-dirreq-v3-resp ok=216,not-enough-sigs=0,unavailable=0,not-found=0,not-modified=328,busy=0
-dirreq-v3-direct-dl complete=0,timeout=0,running=0
-dirreq-v3-tunneled-dl complete=212,timeout=4,running=0,min=21595,d1=293347,d2=1624137,q1=1911800,d3=2066929,d4=2415000,md=2888500,d6=3264000,d7=3851333,q3=41>
-hidserv-stats-end 2024-04-05 06:51:23 (86400 s)
-hidserv-rend-relayed-cells 7924 delta_f=2048 epsilon=0.30 bin_size=1024
-hidserv-dir-onions-seen -12 delta_f=8 epsilon=0.30 bin_size=8
-hidserv-v3-stats-end 2024-04-05 12:00:00 (86400 s)
-hidserv-rend-v3-relayed-cells -4785 delta_f=2048 epsilon=0.30 bin_size=1024
-hidserv-dir-v3-onions-seen 5 delta_f=8 epsilon=0.30 bin_size=8
-padding-counts 2024-04-05 06:51:42 (86400 s) bin-size=10000 write-drop=0 write-pad=80000 write-total=79980000 read-drop=0 read-pad=1110000 read-total=7989000>
-bridge-stats-end 2024-04-05 06:51:44 (86400 s)
-bridge-ips ru=40,us=32,??=8,au=8,br=8,by=8,cn=8,de=8,eg=8,eu=8,gb=8,ge=8,hr=8,ie=8,ir=8,kp=8,lt=8,mt=8,nl=8,pl=8,ro=8,sg=8,tn=8,tr=8,vn=8
-bridge-ip-versions v4=104,v6=8
-bridge-ip-transports <OR>=56,obfs4=56
-router-digest-sha256 zK0VMl3i0B2eaeQTR03e2hZ0i8ytkuhK/psgD2J1/lQ
-router-digest F30B38390C375E1EE74BFED844177804442569E0"#;
-
-    let extra_info_set = ExtraInfo::parse_file(&extra_info_str);
-    assert_eq!(extra_info_set.len(), 1);
-
-    let extra_info = extra_info_set.iter().next().unwrap().clone();
-
-    let extra_info_str = extra_info.to_string();
-
-    let extra_info_2 = ExtraInfo::parse_file(&extra_info_str)
-        .into_iter()
-        .next()
-        .unwrap()
-        .clone();
-    assert_eq!(extra_info, extra_info_2);
-
-    let bridge_to_test: [u8; 20] =
-        array_bytes::hex2array("72E12B89136B45BBC81D1EF0AC7DDDBB91B148DB").unwrap();
-
-    // Open test database
-    let db: Db = sled::open("test_db_sei").unwrap();
-
-    // Delete all data in test DB
-    db.clear().unwrap();
-    assert!(!db.contains_key("bridges").unwrap());
-    assert!(!db.contains_key(bridge_to_test).unwrap());
-
-    // Start web server
-    spawn(async move {
-        extra_infos_server::server().await;
-    });
-
-    // Give server time to start
-    sleep(Duration::new(1, 0)).await;
-
-    // Update extra-infos (no new data to add)
-    update_extra_infos(&db, "http://localhost:8004/")
-        .await
-        .unwrap();
-
-    // Check that database is still empty
-    assert!(!db.contains_key("bridges").unwrap());
-    assert!(!db.contains_key(bridge_to_test).unwrap());
-
-    // Add our extra-info to the server's records
-    {
-        use hyper::{Body, Client, Method, Request};
-        let client = Client::new();
-        let req = Request::builder()
-            .method(Method::POST)
-            .uri("http://localhost:8004/add".parse::<hyper::Uri>().unwrap())
-            .body(Body::from(serde_json::to_string(&extra_info_set).unwrap()))
-            .unwrap();
-        client.request(req).await.unwrap();
-    }
-
-    // Update extra-infos (add new record)
-    update_extra_infos(&db, "http://localhost:8004/")
-        .await
-        .unwrap();
-
-    // Check that DB now contains information on this bridge
-    assert!(db.contains_key("bridges").unwrap());
-    let bridges: HashSet<[u8; 20]> =
-        bincode::deserialize(&db.get("bridges").unwrap().unwrap()).unwrap();
-    assert!(bridges.contains(&bridge_to_test));
-    assert!(db.contains_key(bridge_to_test).unwrap());
-    let _bridge_info: BridgeInfo =
-        bincode::deserialize(&db.get(bridge_to_test).unwrap().unwrap()).unwrap();
-}
-
 #[test]
 fn test_negative_reports() {
     let mut th = TestHarness::new();