vecna
/
troll-patrol
mirror of https://git-crysp.uwaterloo.ca/vvecna/troll-patrol.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Track bridge info by country first, then date

This commit is contained in:
Vecna 2024-03-23 21:40:00 -04:00
parent 3ef9c04141
commit 12e699f979
1 changed files with 90 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
src/lib.rs
View File

@ -49,11 +49,8 @@ pub struct BridgeInfo {
/// first Julian date we started collecting data on this bridge /// first Julian date we started collecting data on this bridge
pub first_seen: u32, pub first_seen: u32,
/// list of countries where the bridge is believed to be blocked /// map of countries to data for this bridge in that country
pub blocked_in: Vec<String>, pub info_by_country: HashMap<String, BridgeCountryInfo>,
/// map of dates to data for that day
pub info_by_day: HashMap<u32, DailyBridgeInfo>,
} }
impl BridgeInfo { impl BridgeInfo {
@ -62,8 +59,7 @@ impl BridgeInfo {
fingerprint: fingerprint, fingerprint: fingerprint,
nickname: nickname.to_string(), nickname: nickname.to_string(),
first_seen: get_date(), first_seen: get_date(),
blocked_in: Vec::<String>::new(), info_by_country: HashMap::<String, BridgeCountryInfo>::new(),
info_by_day: HashMap::<u32, DailyBridgeInfo>::new(),
} }
} }
} }
@ -76,15 +72,11 @@ impl fmt::Display for BridgeInfo {
); );
str.push_str(format!("nickname: {}\n", self.nickname).as_str()); str.push_str(format!("nickname: {}\n", self.nickname).as_str());
str.push_str(format!("first_seen: {}\n", self.first_seen).as_str()); str.push_str(format!("first_seen: {}\n", self.first_seen).as_str());
str.push_str("blocked_in:"); str.push_str("info_by_country:");
for country in &self.blocked_in { for country in self.info_by_country.keys() {
str.push_str(format!("\n {}", country).as_str()); str.push_str(format!("\n country: {}", country).as_str());
} let country_info = self.info_by_country.get(country).unwrap();
str.push_str("info_by_day:"); for line in country_info.to_string().lines() {
for day in self.info_by_day.keys() {
str.push_str(format!("\n day: {}", day).as_str());
let daily_info = self.info_by_day.get(day).unwrap();
for line in daily_info.to_string().lines() {
str.push_str(format!("\n {}", line).as_str()); str.push_str(format!("\n {}", line).as_str());
} }
} }
@ -99,62 +91,49 @@ pub enum BridgeInfoType {
PositiveReports, PositiveReports,
} }
/// Information about bridge reachability, gathered daily /// Information about bridge reachability from a given country
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub struct DailyBridgeInfo { pub struct BridgeCountryInfo {
pub info_by_country: BTreeMap<String, BTreeMap<BridgeInfoType, u32>>, pub info_by_day: BTreeMap<u32, BTreeMap<BridgeInfoType, u32>>,
pub blocked: bool,
} }
impl DailyBridgeInfo { impl BridgeCountryInfo {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
info_by_country: BTreeMap::<String, BTreeMap<BridgeInfoType, u32>>::new(), info_by_day: BTreeMap::<u32, BTreeMap<BridgeInfoType, u32>>::new(),
blocked: false,
} }
} }
pub fn add_info( pub fn add_info(&mut self, info_type: BridgeInfoType, date: u32, count: u32) {
&mut self, if self.info_by_day.contains_key(&date) {
info_type: BridgeInfoType, let info = self.info_by_day.get_mut(&date).unwrap();
count_per_country: &BTreeMap<String, u32>, if !info.contains_key(&info_type) {
) { info.insert(info_type, count);
for country in count_per_country.keys() { } else if info_type == BridgeInfoType::BridgeIps {
if self.info_by_country.contains_key(country) { if *info.get(&info_type).unwrap() < count {
let info = self.info_by_country.get_mut(country).unwrap(); // Use highest value we've seen today
if !info.contains_key(&info_type) { info.insert(info_type, count);
info.insert(
info_type,
*count_per_country.get(&country.to_string()).unwrap(),
);
} else if info_type == BridgeInfoType::BridgeIps {
// Use newest value we've seen today
if info.get(&info_type).unwrap() < count_per_country.get(country).unwrap() {
info.insert(
BridgeInfoType::BridgeIps,
*count_per_country.get(&country.to_string()).unwrap(),
);
}
} else {
let new_count = info.get(&info_type).unwrap()
+ *count_per_country.get(&country.to_string()).unwrap();
info.insert(info_type, new_count);
} }
} else { } else {
let mut info = BTreeMap::<BridgeInfoType, u32>::new(); // Add count to previous count for reports
info.insert( let new_count = info.get(&info_type).unwrap() + count;
info_type, info.insert(info_type, new_count);
*count_per_country.get(&country.to_string()).unwrap(),
);
self.info_by_country.insert(country.to_string(), info);
} }
} else {
let mut info = BTreeMap::<BridgeInfoType, u32>::new();
info.insert(info_type, count);
self.info_by_day.insert(date, info);
} }
} }
} }
impl fmt::Display for DailyBridgeInfo { impl fmt::Display for BridgeCountryInfo {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut str = String::from("info:"); let mut str = String::from("info:");
for country in self.info_by_country.keys() { for date in self.info_by_day.keys() {
let info = self.info_by_country.get(country).unwrap(); let info = self.info_by_day.get(date).unwrap();
let ip_count = match info.get(&BridgeInfoType::BridgeIps) { let ip_count = match info.get(&BridgeInfoType::BridgeIps) {
Some(v) => v, Some(v) => v,
None => &0, None => &0,
@ -170,8 +149,8 @@ impl fmt::Display for DailyBridgeInfo {
if ip_count > &0 || nr_count > &0 || pr_count > &0 { if ip_count > &0 || nr_count > &0 || pr_count > &0 {
str.push_str( str.push_str(
format!( format!(
"\n cc: {}\n connections: {}\n negative reports: {}\n positive reports: {}", "\n date: {}\n connections: {}\n negative reports: {}\n positive reports: {}",
country, date,
ip_count, ip_count,
nr_count, nr_count,
pr_count, pr_count,
@ -196,20 +175,29 @@ pub fn add_extra_info_to_db(db: &Db, extra_info: ExtraInfo) {
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
None => BridgeInfo::new(fingerprint, &extra_info.nickname), None => BridgeInfo::new(fingerprint, &extra_info.nickname),
}; };
// If we already have an entry, compare it with the new one. For each for country in extra_info.bridge_ips.keys() {
// country:count mapping, use the greater of the two counts. if bridge_info.info_by_country.contains_key::<String>(country) {
if bridge_info.info_by_day.contains_key(&extra_info.date) { bridge_info
let daily_bridge_info = bridge_info.info_by_day.get_mut(&extra_info.date).unwrap(); .info_by_country
daily_bridge_info.add_info(BridgeInfoType::BridgeIps, &extra_info.bridge_ips); .get_mut(country)
} else { .unwrap()
// No existing entry; make a new one. .add_info(
let mut daily_bridge_info = DailyBridgeInfo { BridgeInfoType::BridgeIps,
info_by_country: BTreeMap::<String, BTreeMap<BridgeInfoType, u32>>::new(), extra_info.date,
}; *extra_info.bridge_ips.get(country).unwrap(),
daily_bridge_info.add_info(BridgeInfoType::BridgeIps, &extra_info.bridge_ips); );
bridge_info } else {
.info_by_day // No existing entry; make a new one.
.insert(extra_info.date, daily_bridge_info); let mut bridge_country_info = BridgeCountryInfo::new();
bridge_country_info.add_info(
BridgeInfoType::BridgeIps,
extra_info.date,
*extra_info.bridge_ips.get(country).unwrap(),
);
bridge_info
.info_by_country
.insert(country.to_string(), bridge_country_info);
}
} }
// Commit changes to database // Commit changes to database
db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap()) db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
@ -260,11 +248,12 @@ pub fn save_negative_report_to_process(db: &Db, nr: NegativeReport) {
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
None => BTreeMap::<String, BTreeMap<String, u32>>::new(), None => BTreeMap::<String, BTreeMap<String, u32>>::new(),
}; };
// Store to-be-processed reports with key [fingerprint]_[date] // Store to-be-processed reports with key [fingerprint]_[country]_[date]
let map_key = format!( let map_key = format!(
"{}_{}", "{}_{}_{}",
array_bytes::bytes2hex("", &nr.fingerprint), array_bytes::bytes2hex("", &nr.fingerprint),
&nr.date &nr.country,
&nr.date,
); );
let serialized_nr = nr.to_json(); let serialized_nr = nr.to_json();
if reports.contains_key(&map_key) { if reports.contains_key(&map_key) {
@ -322,12 +311,9 @@ pub async fn update_negative_reports(db: &Db, distributors: &BTreeMap<BridgeDist
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
None => BTreeMap::<String, BTreeMap<String, u32>>::new(), None => BTreeMap::<String, BTreeMap<String, u32>>::new(),
}; };
for bridge_date in all_negative_reports.keys() { // Key is [fingerprint]_[country]_[date]
// We could parse the fingerprint and date: for bridge_country_date in all_negative_reports.keys() {
//let fingerprint: [u8; 20] = array_bytes::hex2array(&bridge_date[0..40]).unwrap(); let reports = all_negative_reports.get(bridge_country_date).unwrap();
//let date: u32 = &bridge_date[41..].parse().unwrap();
// but instead, let's just get it from the first report
let reports = all_negative_reports.get(bridge_date).unwrap();
if !reports.is_empty() { if !reports.is_empty() {
let first_report: SerializableNegativeReport = let first_report: SerializableNegativeReport =
serde_json::from_str(reports.first_key_value().unwrap().0).unwrap(); serde_json::from_str(reports.first_key_value().unwrap().0).unwrap();
@ -335,8 +321,7 @@ pub async fn update_negative_reports(db: &Db, distributors: &BTreeMap<BridgeDist
let date = first_report.date; let date = first_report.date;
let country = first_report.country; let country = first_report.country;
let count_valid = verify_negative_reports(&distributors, reports).await; let count_valid = verify_negative_reports(&distributors, reports).await;
let mut count_per_country = BTreeMap::<String, u32>::new();
count_per_country.insert(country, count_valid).unwrap();
let mut bridge_info = match db.get(&fingerprint).unwrap() { let mut bridge_info = match db.get(&fingerprint).unwrap() {
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
// It should already exist, unless the bridge hasn't published // It should already exist, unless the bridge hasn't published
@ -344,17 +329,19 @@ pub async fn update_negative_reports(db: &Db, distributors: &BTreeMap<BridgeDist
None => BridgeInfo::new(fingerprint, &"".to_string()), None => BridgeInfo::new(fingerprint, &"".to_string()),
}; };
// Add the new report count to it // Add the new report count to it
if bridge_info.info_by_day.contains_key(&date) { if bridge_info.info_by_country.contains_key(&country) {
let daily_bridge_info = bridge_info.info_by_day.get_mut(&date).unwrap(); let bridge_country_info = bridge_info.info_by_country.get_mut(&country).unwrap();
daily_bridge_info.add_info(BridgeInfoType::NegativeReports, &count_per_country); bridge_country_info.add_info(BridgeInfoType::NegativeReports, date, count_valid);
// Commit changes to database // Commit changes to database
db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap()) db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
.unwrap(); .unwrap();
} else { } else {
// No existing entry; make a new one. // No existing entry; make a new one.
let mut daily_bridge_info = DailyBridgeInfo::new(); let mut bridge_country_info = BridgeCountryInfo::new();
daily_bridge_info.add_info(BridgeInfoType::NegativeReports, &count_per_country); bridge_country_info.add_info(BridgeInfoType::NegativeReports, date, count_valid);
bridge_info.info_by_day.insert(date, daily_bridge_info); bridge_info
.info_by_country
.insert(country, bridge_country_info);
// Commit changes to database // Commit changes to database
db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap()) db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
.unwrap(); .unwrap();
@ -380,11 +367,12 @@ pub fn save_positive_report_to_process(db: &Db, pr: PositiveReport) {
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
None => BTreeMap::<String, Vec<SerializablePositiveReport>>::new(), None => BTreeMap::<String, Vec<SerializablePositiveReport>>::new(),
}; };
// Store to-be-processed reports with key [fingerprint]_[date] // Store to-be-processed reports with key [fingerprint]_[country]_[date]
let map_key = format!( let map_key = format!(
"{}_{}", "{}_{}_{}",
array_bytes::bytes2hex("", &pr.fingerprint), array_bytes::bytes2hex("", &pr.fingerprint),
&pr.date &pr.country,
&pr.date,
); );
if reports.contains_key(&map_key) { if reports.contains_key(&map_key) {
reports reports
@ -437,20 +425,15 @@ pub async fn update_positive_reports(db: &Db, distributors: &BTreeMap<BridgeDist
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
None => BTreeMap::<String, Vec<SerializablePositiveReport>>::new(), None => BTreeMap::<String, Vec<SerializablePositiveReport>>::new(),
}; };
for bridge_date in all_positive_reports.keys() { // Key is [fingerprint]_[country]_[date]
// We could parse the fingerprint and date: for bridge_country_date in all_positive_reports.keys() {
//let fingerprint: [u8; 20] = array_bytes::hex2array(&bridge_date[0..40]).unwrap(); let reports = all_positive_reports.get(bridge_country_date).unwrap();
//let date: u32 = &bridge_date[41..].parse().unwrap();
// but instead, let's just get it from the first report
let reports = all_positive_reports.get(bridge_date).unwrap();
if !reports.is_empty() { if !reports.is_empty() {
let first_report = &reports[0]; let first_report = &reports[0];
let fingerprint = first_report.fingerprint; let fingerprint = first_report.fingerprint;
let date = first_report.date; let date = first_report.date;
let country = first_report.country.clone(); let country = first_report.country.clone();
let count_valid = verify_positive_reports(&distributors, reports).await; let count_valid = verify_positive_reports(&distributors, reports).await;
let mut count_per_country = BTreeMap::<String, u32>::new();
count_per_country.insert(country, count_valid).unwrap();
let mut bridge_info = match db.get(&fingerprint).unwrap() { let mut bridge_info = match db.get(&fingerprint).unwrap() {
Some(v) => bincode::deserialize(&v).unwrap(), Some(v) => bincode::deserialize(&v).unwrap(),
// It should already exist, unless the bridge hasn't published // It should already exist, unless the bridge hasn't published
@ -458,17 +441,19 @@ pub async fn update_positive_reports(db: &Db, distributors: &BTreeMap<BridgeDist
None => BridgeInfo::new(fingerprint, &"".to_string()), None => BridgeInfo::new(fingerprint, &"".to_string()),
}; };
// Add the new report count to it // Add the new report count to it
if bridge_info.info_by_day.contains_key(&date) { if bridge_info.info_by_country.contains_key(&country) {
let daily_bridge_info = bridge_info.info_by_day.get_mut(&date).unwrap(); let bridge_country_info = bridge_info.info_by_country.get_mut(&country).unwrap();
daily_bridge_info.add_info(BridgeInfoType::PositiveReports, &count_per_country); bridge_country_info.add_info(BridgeInfoType::PositiveReports, date, count_valid);
// Commit changes to database // Commit changes to database
db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap()) db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
.unwrap(); .unwrap();
} else { } else {
// No existing entry; make a new one. // No existing entry; make a new one.
let mut daily_bridge_info = DailyBridgeInfo::new(); let mut bridge_country_info = BridgeCountryInfo::new();
daily_bridge_info.add_info(BridgeInfoType::PositiveReports, &count_per_country); bridge_country_info.add_info(BridgeInfoType::PositiveReports, date, count_valid);
bridge_info.info_by_day.insert(date, daily_bridge_info); bridge_info
.info_by_country
.insert(country, bridge_country_info);
// Commit changes to database // Commit changes to database
db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap()) db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
.unwrap(); .unwrap();