Add script and configurations for reproducing Troll Patrol results
This commit is contained in:
parent
0f703db1fd
commit
c945f72cac
|
|
@ -53,7 +53,6 @@ RUN cp /home/user/build/config.toml .cargo/
|
|||
WORKDIR /home/user/build
|
||||
RUN git clone https://git-crysp.uwaterloo.ca/vvecna/troll-patrol.git
|
||||
WORKDIR /home/user/build/troll-patrol
|
||||
# Commit on analysis branch
|
||||
RUN git checkout 7acba0a6f00c6ffdb429b4993ee109a8e125b466
|
||||
RUN mkdir -p .cargo
|
||||
RUN cp /home/user/build/config.toml .cargo/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
Overt 2 0.25
|
||||
Flooding 2 0.25
|
||||
Overt 2 0
|
||||
Flooding 2 0.01
|
||||
Overt 2 1.0
|
||||
Flooding 2 1.0
|
||||
Overt 2 0.5
|
||||
Flooding 2 0.5
|
||||
Overt 2 0.8
|
||||
Flooding 2 0.8
|
||||
Overt 2 0.15
|
||||
Flooding 2 0.15
|
||||
Overt 2 0.4
|
||||
Flooding 2 0.4
|
||||
Overt 2 0.9
|
||||
Flooding 2 0.9
|
||||
Overt 2 0.05
|
||||
Flooding 2 0.05
|
||||
Overt 2 0.3
|
||||
Flooding 2 0.3
|
||||
Overt 2 0.6
|
||||
Flooding 2 0.6
|
||||
Overt 2 0.1
|
||||
Flooding 2 0.1
|
||||
Overt 2 0.2
|
||||
Flooding 2 0.2
|
||||
Overt 2 0.35
|
||||
Flooding 2 0.35
|
||||
Overt 2 0.45
|
||||
Flooding 2 0.45
|
||||
Overt 2 0.7
|
||||
Flooding 2 0.7
|
||||
Overt 2 0.01
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
Overt 0 0.25
|
||||
Flooding 0 0.25
|
||||
Overt 4 0.25
|
||||
Flooding 4 0.25
|
||||
Overt 1 0.25
|
||||
Flooding 1 0.25
|
||||
Overt 3 0.25
|
||||
Flooding 3 0.25
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"la_port": 8001,
|
||||
"la_test_port": 8005,
|
||||
"tp_port": 8003,
|
||||
"tp_test_port": 8123,
|
||||
"bootstrapping_period_duration": 180,
|
||||
"censor_secrecy": CENSOR_SECRECY,
|
||||
"censor_max_connections": 30000,
|
||||
"censor_max_pr": 1,
|
||||
"censor_speed": "Fast",
|
||||
"censor_event_duration": 14,
|
||||
"censor_totality": "Full",
|
||||
"censor_partial_blocking_percent": 0.5,
|
||||
"country": "ru",
|
||||
"min_new_users_per_day": 0,
|
||||
"max_new_users_per_day": 20,
|
||||
"num_connection_retries": 3,
|
||||
"num_days": 500,
|
||||
"one_positive_report_per_cred": true,
|
||||
"prob_censor_gets_invite": 0.01,
|
||||
"prob_connection_fails": 0.0028,
|
||||
"prob_user_connects": 0.5,
|
||||
"prob_user_invites_friend": 0.25,
|
||||
"prob_user_submits_reports": PROB_USER_SUBMITS_REPORTS,
|
||||
"prob_user_treats_throttling_as_blocking": 0.25
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"db": {
|
||||
"db_path": "server_db"
|
||||
},
|
||||
"distributors": {
|
||||
"Lox": "http://127.0.0.1:8002"
|
||||
},
|
||||
"extra_infos_base_url": "http://127.0.0.1:8004/",
|
||||
"confidence": 0.95,
|
||||
"max_threshold": HARSHNESS,
|
||||
"scaling_factor": 0.25,
|
||||
"min_historical_days": 30,
|
||||
"max_historical_days": 30,
|
||||
"port": 8003,
|
||||
"require_bridge_token": false,
|
||||
"updater_port": 8123,
|
||||
"updater_schedule": "* * 22 * * * *"
|
||||
}
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
#!/bin/bash
|
||||
|
||||
# cd into the directory containing this script (from the bash faq 028)
|
||||
if [[ $BASH_SOURCE = */* ]]; then
|
||||
cd -- "${BASH_SOURCE%/*}/" || exit 1
|
||||
fi
|
||||
|
||||
# Check for the python dependencies we need at the end
|
||||
./scripts/check-dependencies.py
|
||||
if [ $? != 0 ]; then
|
||||
echo "Please make sure all dependencies are installed before running this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Number of simulation runs to do in parallel
|
||||
parallel=""
|
||||
|
||||
# Number of simulation runs in each configuration
|
||||
n=""
|
||||
|
||||
# Number of configurations in experiment 1
|
||||
exp1=""
|
||||
# Experiment 1 beginning
|
||||
e1b=1
|
||||
# Experiment 1 end
|
||||
e1e=33
|
||||
|
||||
# Number of configurations in experiment 2
|
||||
exp2=""
|
||||
# Experiment 2 beginning
|
||||
e2b=1
|
||||
# Experiment 2 end
|
||||
e2e=8
|
||||
|
||||
# Max observed memory use for 1 trial
|
||||
mem_use=688
|
||||
|
||||
# Proceed without confirmation
|
||||
noninteractive=false
|
||||
|
||||
# By default, run experiments and process results
|
||||
run_experiments=true
|
||||
process_results=true
|
||||
|
||||
# Get parameters
|
||||
while getopts ":p:n:1:2:yer" opt; do
|
||||
case ${opt} in
|
||||
p)
|
||||
parallel="${OPTARG}"
|
||||
;;
|
||||
n)
|
||||
n="${OPTARG}"
|
||||
;;
|
||||
1)
|
||||
exp1="${OPTARG}"
|
||||
if [ "$exp1" != "" ]; then
|
||||
# If the user specified a range, use that range
|
||||
if [[ "$exp1" == *"-"* ]]; then
|
||||
e1b="${exp1%%-*}"
|
||||
e1e="${exp1##*-}"
|
||||
exp1=$((e1e - e1b + 1))
|
||||
else
|
||||
e1b=1
|
||||
e1e="$exp1"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
2)
|
||||
exp2="${OPTARG}"
|
||||
if [ "$exp2" != "" ]; then
|
||||
# If the user specified a range, use that range
|
||||
if [[ "$exp2" == *"-"* ]]; then
|
||||
e2b="${exp2%%-*}"
|
||||
e2e="${exp2##*-}"
|
||||
exp2=$((e2e - e2b + 1))
|
||||
else
|
||||
e2b=1
|
||||
e2e="$exp2"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
y)
|
||||
noninteractive=true
|
||||
;;
|
||||
e)
|
||||
run_experiments=true
|
||||
process_results=false
|
||||
;;
|
||||
r)
|
||||
run_experiments=false
|
||||
process_results=true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Run experiments unless -r flag was used
|
||||
if [ "$run_experiments" == true ]; then
|
||||
|
||||
# Ask user for values they didn't already specify
|
||||
|
||||
if [ "$parallel" == "" ]; then
|
||||
read -e -p "How many simulation runs should we perform in parallel? (We suggest the number of CPU cores you have.) " parallel
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ "$n" == "" ]; then
|
||||
read -e -p "How many trials should we do in each configuration? [5] " n
|
||||
|
||||
# Default to 5
|
||||
if [ "$n" == "" ]; then
|
||||
n=5
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ "$exp1" == "" ]; then
|
||||
read -e -p "How many configurations should we use in the first experiment? [33] " exp1
|
||||
|
||||
# Default to 33 and max at 33
|
||||
if [[ "$exp1" == "" || "$exp1" -gt 33 ]]; then
|
||||
exp1=33
|
||||
# Min 0
|
||||
elif [[ "$exp1" -lt 0 ]]; then
|
||||
exp1=0
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Begining and end
|
||||
e1b=1
|
||||
e1e="$exp1"
|
||||
fi
|
||||
|
||||
if [ "$exp2" == "" ]; then
|
||||
read -e -p "How many configurations should we use in the second experiment? [8] " exp2
|
||||
|
||||
# Default to 10 and max at 10
|
||||
if [[ "$exp2" == "" || "$exp2" -gt 8 ]]; then
|
||||
exp2=8
|
||||
# Min 0
|
||||
elif [[ "$exp2" -lt 0 ]]; then
|
||||
exp2=0
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Beginning and end
|
||||
e2b=1
|
||||
e2e="$exp2"
|
||||
fi
|
||||
|
||||
num_configs=$((exp1 + exp2))
|
||||
num_trials=$((num_configs * n))
|
||||
batches=$(( (num_trials + parallel - 1) / parallel))
|
||||
|
||||
if [[ "$parallel" -gt "$num_trials" ]]; then
|
||||
parallel=$num_trials
|
||||
fi
|
||||
|
||||
echo "We will test Troll Patrol in ${num_configs} configurations."
|
||||
echo "We will run the simulation ${n} times in each configuration."
|
||||
echo "This results in a total of ${num_trials} simulation runs."
|
||||
echo "We will do ${parallel} runs in parallel, so this is ${batches} batches."
|
||||
echo "It is recommended that you have at least ${parallel} CPU cores and $((parallel * mem_use)) MB of RAM."
|
||||
echo "If you don't have enough cores or RAM, try reducing the number of parallel simulation runs."
|
||||
echo "It is anticipated that this will take from $((batches + batches / 2)) to $((batches * 2)) days to complete."
|
||||
if [ "$noninteractive" == false ]; then
|
||||
read -e -p "Is this okay? (y/N) " res
|
||||
if [[ "$res" != "Y" && "$res" != "y" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
./scripts/run-experiments.sh "$parallel" "$n" "$e1b" "$e1e" "$e2b" "$e2e"
|
||||
fi
|
||||
|
||||
# Process results unless -e flag was used
|
||||
if [ "$process_results" == true ]; then
|
||||
|
||||
# Parse out bridge info we want to plot
|
||||
for i in results/*/*-simulation; do
|
||||
sed -n '/^Full stats per bridge:$/,/^End full stats per bridge$/{//!p;}' \
|
||||
"$i" > "${i%-simulation}-bridges.csv"
|
||||
sim_begin=$(grep -Po '(?<=Simulation began on day )(.*)(?=$)' "$i")
|
||||
censor_begin=$(grep -Po '(?<=Censor began on day )(.*)(?=$)' "$i")
|
||||
echo "$sim_begin,$censor_begin" > "${i%-simulation}-start.csv"
|
||||
done
|
||||
|
||||
if [ "$n" == "" ]; then
|
||||
read -e -p "How many trials did we do in each configuration? [5] " n
|
||||
|
||||
# Default to 5
|
||||
if [ "$n" == "" ]; then
|
||||
n=5
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
./scripts/plot-results.py 1 $n results/1/*-bridges.csv
|
||||
./scripts/plot-results.py 2 $n results/2/*-bridges.csv
|
||||
fi
|
||||
|
||||
echo "Done. See the results directory for the output."
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import matplotlib
|
||||
import matplotlib.pyplot as pyplot
|
||||
from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes,mark_inset
|
||||
import math
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Generate the configuration files for an experiment we're about to do
|
||||
|
||||
exp_num="$1"
|
||||
secrecy="$2"
|
||||
harshness="$3"
|
||||
prob="$4"
|
||||
|
||||
f1="configs/troll_patrol_config.json"
|
||||
cp "${f1}.template" "${f1}"
|
||||
f2="configs/simulation_config.json"
|
||||
cp "${f2}.template" "${f2}"
|
||||
|
||||
# Troll Patrol config
|
||||
sed -i "s/HARSHNESS/$harshness/" "${f1}"
|
||||
|
||||
# Lox Simulation config
|
||||
sed -i "s/CENSOR_SECRECY/\"$secrecy\"/" "${f2}"
|
||||
sed -i "s/PROB_USER_SUBMITS_REPORTS/$prob/" "${f2}"
|
||||
|
|
@ -0,0 +1,488 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import matplotlib
|
||||
import matplotlib.pyplot as pyplot
|
||||
from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes,mark_inset
|
||||
import math
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
|
||||
# Pass experiment number as first arg
|
||||
experiment_num = int(sys.argv[1])
|
||||
|
||||
# Pass number of trials as second arg (used to average results)
|
||||
num_trials = int(sys.argv[2])
|
||||
|
||||
# (Pass list of *-bridge.csv files as remaining args)
|
||||
|
||||
# Artificially truncate to this many days if we ran for longer
|
||||
num_days = 500
|
||||
|
||||
# Max number of days for Troll Patrol to detect censorship. If it
|
||||
# doesn't detect it within this time, we count it as a false negative.
|
||||
max_number_of_days_to_detect = 10
|
||||
|
||||
# Use bigger font size
|
||||
if experiment_num == 1:
|
||||
matplotlib.rcParams.update({'font.size': 14})
|
||||
else:
|
||||
matplotlib.rcParams.update({'font.size': 14})
|
||||
|
||||
# Adjust width of experiment 1 figures
|
||||
width = 7.2
|
||||
|
||||
# Get mean of list of numbers
|
||||
def mean(my_list):
|
||||
if len(my_list) == 0:
|
||||
return None
|
||||
|
||||
sum = 0
|
||||
for i in my_list:
|
||||
sum += i
|
||||
return sum / len(my_list)
|
||||
|
||||
# Get stddev of list of numbers
|
||||
def std_dev(my_list):
|
||||
if len(my_list) == 0:
|
||||
return None
|
||||
|
||||
avg = mean(my_list)
|
||||
sum = 0
|
||||
for i in my_list:
|
||||
sum += (i - avg)**2
|
||||
sum /= len(my_list)
|
||||
return math.sqrt(sum)
|
||||
|
||||
# Independent variable
|
||||
if experiment_num == 1:
|
||||
# Probability user submits reports
|
||||
# (note flooding does not use 0, so plot from index 1)
|
||||
ind_var = [0.0, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
|
||||
else:
|
||||
# Harshness
|
||||
ind_var = [0, 1, 2, 3, 4]
|
||||
|
||||
# Raw detection times for violin plots
|
||||
overt = [ [] for i in range(len(ind_var)) ]
|
||||
flooding = [ [] for i in range(len(ind_var)) ]
|
||||
|
||||
# Get {True,False} {Positives,Negatives} for our trials
|
||||
overt_tp = [ [] for i in range(len(ind_var)) ]
|
||||
overt_tn = [ [] for i in range(len(ind_var)) ]
|
||||
overt_fp = [ [] for i in range(len(ind_var)) ]
|
||||
overt_fn = [ [] for i in range(len(ind_var)) ]
|
||||
|
||||
flooding_tp = [ [] for i in range(len(ind_var)) ]
|
||||
flooding_tn = [ [] for i in range(len(ind_var)) ]
|
||||
flooding_fp = [ [] for i in range(len(ind_var)) ]
|
||||
flooding_fn = [ [] for i in range(len(ind_var)) ]
|
||||
|
||||
# Remaining arguments should be *-bridges.csv files containing info on bridges
|
||||
for bfile in sys.argv[3:]:
|
||||
with open(bfile,'r') as bcsv:
|
||||
# Read data on bridges from CSV
|
||||
bridges = csv.reader(bcsv, delimiter=',')
|
||||
|
||||
# Get censor_secrecy and ind_var from simulation config
|
||||
sfile = bfile[:-(len("bridges.csv"))] + "simulation_config.json"
|
||||
with open(sfile,'r') as sjson:
|
||||
config = json.load(sjson)
|
||||
secrecy = config["censor_secrecy"]
|
||||
if experiment_num == 1:
|
||||
var = config["prob_user_submits_reports"]
|
||||
index = ind_var.index(var)
|
||||
else:
|
||||
tfile = bfile[:-(len("bridges.csv"))] + "troll_patrol_config.json"
|
||||
with open(tfile,'r') as tjson:
|
||||
tconfig = json.load(tjson)
|
||||
# max_threshold used as harshness
|
||||
var = tconfig["max_threshold"]
|
||||
index = ind_var.index(var)
|
||||
|
||||
# Get start date so we can ignore events after 500 days
|
||||
startfile = bfile[:-(len("bridges.csv"))] + "start.csv"
|
||||
with open(startfile,'r') as startcsv:
|
||||
start_dates = csv.reader(startcsv, delimiter=',')
|
||||
start_row = next(start_dates)
|
||||
start_date = int(start_row[0])
|
||||
end_date = start_date + num_days - 1
|
||||
|
||||
# Raw detection times for violin plot
|
||||
detection_times = []
|
||||
|
||||
# {True,False} {Positives,Negatives}
|
||||
true_pos = 0
|
||||
true_neg = 0
|
||||
false_pos = 0
|
||||
false_neg = 0
|
||||
|
||||
for row in bridges:
|
||||
if row[0] == "Full stats per bridge:" or row[0] == "Fingerprint":
|
||||
continue
|
||||
|
||||
# row[0] is the bridge fingerprint
|
||||
first_distributed = int(row[1])
|
||||
first_real_user = int(row[2])
|
||||
first_blocked = int(row[3])
|
||||
first_detected_blocked = int(row[4])
|
||||
# row[5] is first positive report
|
||||
|
||||
# Treat anything after the end date like it didn't happen
|
||||
if first_distributed > end_date:
|
||||
first_distributed = 0
|
||||
if first_real_user > end_date:
|
||||
first_real_user = 0
|
||||
if first_blocked > end_date:
|
||||
first_blocked = 0
|
||||
if first_detected_blocked > end_date:
|
||||
first_detected_blocked = 0
|
||||
|
||||
# Ignore bridges with no users
|
||||
if first_real_user == 0:
|
||||
continue
|
||||
|
||||
# Did we identify correctly?
|
||||
|
||||
# Negative classification
|
||||
if first_detected_blocked == 0:
|
||||
if first_blocked == 0:
|
||||
true_neg += 1
|
||||
else:
|
||||
false_neg += 1
|
||||
# Positive classification
|
||||
else:
|
||||
if first_blocked == 0 or first_detected_blocked < first_blocked:
|
||||
false_pos += 1
|
||||
# If we didn't detect it in time, consider it a false
|
||||
# negative, even if we eventually detected it
|
||||
elif first_detected_blocked - first_blocked > max_number_of_days_to_detect:
|
||||
false_neg += 1
|
||||
else:
|
||||
true_pos += 1
|
||||
|
||||
# Add data point to plot in violin plot
|
||||
detection_times.append(first_detected_blocked - first_blocked)
|
||||
|
||||
if secrecy == "Flooding":
|
||||
# Add raw data for violin plot
|
||||
flooding[index].extend(detection_times)
|
||||
|
||||
flooding_tp[index].append(true_pos)
|
||||
flooding_tn[index].append(true_neg)
|
||||
flooding_fp[index].append(false_pos)
|
||||
flooding_fn[index].append(false_neg)
|
||||
else:
|
||||
# Add raw data for violin plot
|
||||
overt[index].extend(detection_times)
|
||||
|
||||
overt_tp[index].append(true_pos)
|
||||
overt_tn[index].append(true_neg)
|
||||
overt_fp[index].append(false_pos)
|
||||
overt_fn[index].append(false_neg)
|
||||
|
||||
# We may not have results for all values of the independent variable. If
|
||||
# we have a smaller set of values, track them.
|
||||
ind_var_overt = []
|
||||
ind_var_flooding = []
|
||||
|
||||
# Get precision and recall for each trial
|
||||
|
||||
overt_precision_means = []
|
||||
overt_precision_stddevs = []
|
||||
overt_recall_means = []
|
||||
overt_recall_stddevs = []
|
||||
|
||||
# Get mean and stddev precision and recall
|
||||
for i in range(len(ind_var)):
|
||||
precisions = []
|
||||
recalls = []
|
||||
|
||||
# If we have data, add its index to the list
|
||||
if len(overt_tp[i]) > 0:
|
||||
ind_var_overt.append(i)
|
||||
|
||||
# Compute precision and recall for each trial
|
||||
for j in range(len(overt_tp[i])):
|
||||
precisions.append(overt_tp[i][j] / (overt_tp[i][j] + overt_fp[i][j]))
|
||||
recalls.append(overt_tp[i][j] / (overt_tp[i][j] + overt_fn[i][j]))
|
||||
|
||||
# Add their means and stddevs to the appropriate lists
|
||||
overt_precision_means.append(mean(precisions))
|
||||
overt_precision_stddevs.append(std_dev(precisions))
|
||||
overt_recall_means.append(mean(recalls))
|
||||
overt_recall_stddevs.append(std_dev(recalls))
|
||||
|
||||
flooding_precision_means = []
|
||||
flooding_precision_stddevs = []
|
||||
flooding_recall_means = []
|
||||
flooding_recall_stddevs = []
|
||||
|
||||
# Get mean and stddev precision and recall
|
||||
for i in range(len(ind_var)):
|
||||
precisions = []
|
||||
recalls = []
|
||||
|
||||
# If we have data, add its index to the list
|
||||
if len(flooding_tp[i]) > 0:
|
||||
ind_var_flooding.append(i)
|
||||
|
||||
# Compute precision and recall for each trial
|
||||
for j in range(len(flooding_tp[i])):
|
||||
precisions.append(flooding_tp[i][j] / (flooding_tp[i][j] + flooding_fp[i][j]))
|
||||
recalls.append(flooding_tp[i][j] / (flooding_tp[i][j] + flooding_fn[i][j]))
|
||||
|
||||
# Add their means and stddevs to the appropriate lists
|
||||
flooding_precision_means.append(mean(precisions))
|
||||
flooding_precision_stddevs.append(std_dev(precisions))
|
||||
flooding_recall_means.append(mean(recalls))
|
||||
flooding_recall_stddevs.append(std_dev(recalls))
|
||||
|
||||
# Plot our data
|
||||
|
||||
# Violin plots
|
||||
|
||||
# Overt censor
|
||||
if experiment_num == 1:
|
||||
pyplot.violinplot([overt[i] for i in ind_var_overt], positions=[ind_var[i] for i in ind_var_overt], widths=0.04)
|
||||
pyplot.title("Time to Detect Censorship (Overt Censor)")
|
||||
pyplot.xlabel("Probability of users submitting reports")
|
||||
pyplot.ylabel("Days to detect censorship")
|
||||
pyplot.ylim(bottom=0)
|
||||
pyplot.savefig("results/figure-2b.png")
|
||||
pyplot.cla()
|
||||
|
||||
else:
|
||||
pyplot.violinplot([overt[i] for i in ind_var_overt], positions=[ind_var[i] for i in ind_var_overt])
|
||||
pyplot.title("Time to Detect Censorship (Overt Censor)")
|
||||
pyplot.xlabel("Harshness")
|
||||
pyplot.xticks(ind_var)
|
||||
pyplot.ylabel("Days to detect censorship")
|
||||
pyplot.ylim(bottom=0)
|
||||
pyplot.savefig("results/figure-3b.png")
|
||||
pyplot.cla()
|
||||
|
||||
# Flooding censor (should be orange)
|
||||
if experiment_num == 1:
|
||||
#pyplot.figure().set_figwidth(width)
|
||||
fv = pyplot.violinplot([flooding[i] for i in ind_var_flooding], positions=[ind_var[i] for i in ind_var_flooding], widths=0.045)
|
||||
else:
|
||||
fv = pyplot.violinplot([flooding[i] for i in ind_var_flooding], positions=[ind_var[i] for i in ind_var_flooding])
|
||||
|
||||
# Make it orange regardless of experiment number
|
||||
for pc in fv["bodies"]:
|
||||
pc.set_facecolor("orange")
|
||||
pc.set_edgecolor("orange")
|
||||
for part in ("cbars", "cmins", "cmaxes"):
|
||||
fv[part].set_edgecolor("orange")
|
||||
|
||||
if experiment_num == 1:
|
||||
pyplot.title("Time to Detect Censorship (Flooding Censor)")
|
||||
pyplot.xlabel("Probability of users submitting reports")
|
||||
pyplot.ylabel("Days to detect censorship")
|
||||
pyplot.ylim(bottom=0)
|
||||
pyplot.savefig("results/figure-2c.png")
|
||||
pyplot.cla()
|
||||
|
||||
else:
|
||||
pyplot.title("Time to Detect Censorship (Flooding Censor)")
|
||||
pyplot.xlabel("Harshness")
|
||||
pyplot.xticks(ind_var)
|
||||
pyplot.ylabel("Days to detect censorship")
|
||||
pyplot.ylim(bottom=0)
|
||||
pyplot.savefig("results/figure-3c.png")
|
||||
pyplot.cla()
|
||||
|
||||
# Precision vs. Recall
|
||||
|
||||
if experiment_num == 1:
|
||||
# Also plot recall alone
|
||||
pyplot.ylim(0,1)
|
||||
ax = pyplot
|
||||
ax.errorbar([ind_var[i] for i in ind_var_overt], [overt_recall_means[i] for i in ind_var_overt], [overt_recall_stddevs[i] for i in ind_var_overt], linestyle="solid", marker='o', capsize=3)
|
||||
ax.errorbar([ind_var[i] for i in ind_var_flooding], [flooding_recall_means[i] for i in ind_var_flooding], [flooding_recall_stddevs[i] for i in ind_var_flooding], linestyle="dotted", marker='v', capsize=3)
|
||||
pyplot.xlabel("Probability of users submitting reports")
|
||||
pyplot.xlim(0,1)
|
||||
pyplot.ylabel("Recall")
|
||||
pyplot.ylim(0,1)
|
||||
pyplot.title("Proportion of Blocked Bridges Detected")
|
||||
pyplot.legend(["Overt censor", "Flooding censor"], loc = "lower right")
|
||||
pyplot.savefig("results/figure-2a.png")
|
||||
pyplot.cla()
|
||||
|
||||
else:
|
||||
pyplot.xlim(0,1)
|
||||
pyplot.ylim(0,1.02)
|
||||
ax = pyplot.axes()
|
||||
ax.errorbar([overt_recall_means[i] for i in ind_var_overt], [overt_precision_means[i] for i in ind_var_overt], xerr=[overt_recall_stddevs[i] for i in ind_var_overt], yerr=[overt_precision_stddevs[i] for i in ind_var_overt], marker='o', capsize=3, linestyle="solid")
|
||||
ax.errorbar([flooding_recall_means[i] for i in ind_var_flooding], [flooding_precision_means[i] for i in ind_var_flooding], xerr=[flooding_recall_stddevs[i] for i in ind_var_flooding], yerr=[flooding_precision_stddevs[i] for i in ind_var_flooding], marker='v', capsize=3, linestyle="dotted")
|
||||
pyplot.xlabel("Recall")
|
||||
pyplot.xlim(0,1)
|
||||
pyplot.ylabel("Precision")
|
||||
pyplot.ylim(0,1.02)
|
||||
pyplot.title("Precision vs. Recall")
|
||||
pyplot.legend(["Overt censor", "Flooding censor"], loc = "lower left")
|
||||
|
||||
# Zoom in on relevant part
|
||||
axins = zoomed_inset_axes(ax, zoom=1.75, bbox_to_anchor=(-0.325, -0.125, 1, 1), bbox_transform=ax.transAxes)
|
||||
axins.errorbar([overt_recall_means[i] for i in ind_var_overt], [overt_precision_means[i] for i in ind_var_overt], xerr=[overt_recall_stddevs[i] for i in ind_var_overt], yerr=[overt_precision_stddevs[i] for i in ind_var_overt], marker='o', capsize=3, linestyle="solid")
|
||||
axins.errorbar([flooding_recall_means[i] for i in ind_var_flooding], [flooding_precision_means[i] for i in ind_var_flooding], xerr=[flooding_recall_stddevs[i] for i in ind_var_flooding], yerr=[flooding_precision_stddevs[i] for i in ind_var_flooding], marker='v', capsize=3, linestyle="dotted")
|
||||
pyplot.xlim(0.75,1)
|
||||
pyplot.ylim(0.7,1.02)
|
||||
mark_inset(ax, axins, loc1=2, loc2=4)
|
||||
pyplot.savefig("results/figure-3a.png")
|
||||
pyplot.cla()
|
||||
|
||||
# Format mean +- standard deviation with correct sigfigs and rounding.
|
||||
# I couldn't find an existing solution for this, so here's my awkward approach.
|
||||
def fmt(data, multiple_trials=True):
|
||||
# If we only run one trial, just use the count without standard deviation
|
||||
if not multiple_trials:
|
||||
return f"{data[0]}"
|
||||
|
||||
# Get mean and standard deviation
|
||||
m = mean(data)
|
||||
s = std_dev(data)
|
||||
|
||||
if s == 0:
|
||||
return f"{round(m)}$\\pm$0"
|
||||
|
||||
# We have max 3600 bridges, so we will certainly never see this many.
|
||||
n = 10000
|
||||
while round(s / n) < 1:
|
||||
n /= 10
|
||||
s = round(s / n) * n
|
||||
m = round(m / n) * n
|
||||
|
||||
if s >= 1:
|
||||
s = int(round(s))
|
||||
elif s >= 0.1:
|
||||
s = int(round(s*10)) / 10
|
||||
|
||||
# We have a pesky 0.6000000...1 that causes problems. This is to handle that.
|
||||
if m >= 1:
|
||||
m = int(round(m))
|
||||
elif m >= 0.1:
|
||||
m = int(round(m*10)) / 10
|
||||
|
||||
return f"{m}$\\pm${s}"
|
||||
|
||||
def fmt_pr(m, s, multiple_trials=True):
|
||||
# If we only run one trial, round to 3 decimal places and don't
|
||||
# include standard deviations
|
||||
if not multiple_trials:
|
||||
m = int(round(m*1000)) / 1000
|
||||
return f"{m}"
|
||||
|
||||
n = 1.0
|
||||
while s > 0 and round(s / n) < 1:
|
||||
n /= 10
|
||||
s = round(s / n) * n
|
||||
m = round(m / n) * n
|
||||
|
||||
if s >= 0.1:
|
||||
s = int(round(s*10)) / 10
|
||||
m = int(round(m*10)) / 10
|
||||
elif s >= 0.01:
|
||||
s = int(round(s*100)) / 100
|
||||
m = int(round(m*100)) / 100
|
||||
elif s >= 0.001:
|
||||
s = int(round(s*1000)) / 1000
|
||||
m = int(round(m*1000)) / 1000
|
||||
elif s >= 0.0001:
|
||||
s = int(round(s*10000)) / 10000
|
||||
m = int(round(m*10000)) / 10000
|
||||
elif s >= 0.00001:
|
||||
s = int(round(s*100000)) / 100000
|
||||
m = int(round(m*100000)) / 100000
|
||||
elif s >= 0.000001:
|
||||
s = int(round(s*1000000)) / 1000000
|
||||
m = int(round(m*1000000)) / 1000000
|
||||
|
||||
return f"{m}$\\pm${s}"
|
||||
|
||||
# Output raw data as lines of table
|
||||
|
||||
standalone_table_preamble = """\\documentclass{article}
|
||||
\\usepackage{standalone}
|
||||
\\usepackage{array}
|
||||
\\newcolumntype{C}[1]{>{\\centering\\arraybackslash}p{#1}}
|
||||
\\begin{document}"""
|
||||
|
||||
# Use appropriate variables for this experiment
|
||||
if experiment_num == 1:
|
||||
ind_var_str = "Prob. users submit reports"
|
||||
|
||||
# Make 2 different tables, one for overt censor and one for flooding censor
|
||||
with open("results/experiment-1-table-overt.tex", 'w') as f:
|
||||
print(standalone_table_preamble, file=f)
|
||||
print("""\\begin{table*}
|
||||
\\caption[Results of experiment 1 with overt censor]{Results of the first experiment with the \\textbf{overt censor}, specifically the mean and standard deviation number of true positives, true negatives, false positives, and false negatives for each set of trials. The independent variable in this experiment is the probability of users submitting reports.}
|
||||
\\label{experiment-1-results-overt}
|
||||
\\centering
|
||||
\\begin{tabular}[p]{|C{0.1\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|c|c|}""", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\textbf{" + ind_var_str + "} & \\textbf{True positives} & \\textbf{True negatives} & \\textbf{False positives} & \\textbf{False negatives} & \\textbf{Precision} & \\textbf{Recall} \\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\hline", file=f)
|
||||
for i in ind_var_overt:
|
||||
print(f"{ind_var[i]} & {fmt(overt_tp[i], num_trials>1)} & {fmt(overt_tn[i], num_trials>1)} & {fmt(overt_fp[i], num_trials>1)} & {fmt(overt_fn[i], num_trials>1)} & {fmt_pr(overt_precision_means[i], overt_precision_stddevs[i], num_trials>1)} & {fmt_pr(overt_recall_means[i], overt_recall_stddevs[i], num_trials>1)}\\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\end{tabular}", file=f)
|
||||
print("\\end{table*}", file=f)
|
||||
print("\\end{document}", file=f)
|
||||
|
||||
with open("results/experiment-1-table-flooding.tex", 'w') as f:
|
||||
print(standalone_table_preamble, file=f)
|
||||
print("""\\begin{table*}
|
||||
\\caption[Results of experiment 1 with flooding censor]{Results of the first experiment with the \\textbf{flooding censor}, specifically the mean and standard deviation number of true positives, true negatives, false positives, and false negatives for each set of trials. The independent variable in this experiment is the probability of users submitting reports. When Troll Patrol does not detect that bridges are blocked, Lox does not allow users to migrate to new bridges, so the number of overall bridges in the simulation does not grow. This accounts for the low number of overall bridges when the number of positive classifications (both true and false) is low.}
|
||||
\\label{experiment-1-results-flooding}
|
||||
\\centering
|
||||
\\begin{tabular}[p]{|C{0.1\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|c|c|}""", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\textbf{" + ind_var_str + "} & \\textbf{True positives} & \\textbf{True negatives} & \\textbf{False positives} & \\textbf{False negatives} & \\textbf{Precision} & \\textbf{Recall} \\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\hline", file=f)
|
||||
for i in ind_var_flooding:
|
||||
print(f"{ind_var[i]} & {fmt(flooding_tp[i], num_trials>1)} & {fmt(flooding_tn[i], num_trials>1)} & {fmt(flooding_fp[i], num_trials>1)} & {fmt(flooding_fn[i], num_trials>1)} & {fmt_pr(flooding_precision_means[i], flooding_precision_stddevs[i], num_trials>1)} & {fmt_pr(flooding_recall_means[i], flooding_recall_stddevs[i], num_trials>1)} \\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\end{tabular}", file=f)
|
||||
print("\\end{table*}", file=f)
|
||||
print("\\end{document}", file=f)
|
||||
else:
|
||||
# Make 2 tables for experiment 2
|
||||
with open("results/experiment-2-table-overt.tex", 'w') as f:
|
||||
print(standalone_table_preamble, file=f)
|
||||
print("""\\begin{table*}
|
||||
\\caption[Results of experiment 2 with overt censor]{Results of the second experiment with the \\textbf{overt censor}, specifically the mean and standard deviation number of true positives, true negatives, false positives, and false negatives for each set of trials. The independent variable in this experiment is the harshness of the classifier.}
|
||||
\\label{experiment-2-results-overt}
|
||||
\\centering
|
||||
\\begin{tabular}[t]{|C{0.115\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|c|c|}""", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\textbf{Harshness} & \\textbf{True positives} & \\textbf{True negatives} & \\textbf{False positives} & \\textbf{False negatives} & \\textbf{Precision} & \\textbf{Recall} \\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\hline", file=f)
|
||||
for i in ind_var_overt:
|
||||
print(f"{ind_var[i]} & {fmt(overt_tp[i], num_trials>1)} & {fmt(overt_tn[i], num_trials>1)} & {fmt(overt_fp[i], num_trials>1)} & {fmt(overt_fn[i], num_trials>1)} & {fmt_pr(overt_precision_means[i], overt_precision_stddevs[i], num_trials>1)} & {fmt_pr(overt_recall_means[i], overt_recall_stddevs[i], num_trials>1)}\\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\end{tabular}", file=f)
|
||||
print("\\end{table*}", file=f)
|
||||
print("\\end{document}", file=f)
|
||||
|
||||
with open("results/experiment-2-table-flooding.tex", 'w') as f:
|
||||
print(standalone_table_preamble, file=f)
|
||||
print("""\\begin{table*}
|
||||
\\caption[Results of experiment 2 with flooding censor]{Results of the second experiment with the \\textbf{flooding censor}, specifically the mean and standard deviation number of true positives, true negatives, false positives, and false negatives for each set of trials. The independent variable in this experiment is the harshness of the classifier.}
|
||||
\\label{experiment-2-results-flooding}
|
||||
\\centering
|
||||
\\begin{tabular}[t]{|C{0.115\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|C{0.1\\textwidth}|C{0.105\\textwidth}|c|c|}""", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\textbf{Harshness} & \\textbf{True positives} & \\textbf{True negatives} & \\textbf{False positives} & \\textbf{False negatives} & \\textbf{Precision} & \\textbf{Recall} \\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\hline", file=f)
|
||||
for i in ind_var_flooding:
|
||||
print(f"{ind_var[i]} & {fmt(flooding_tp[i], num_trials>1)} & {fmt(flooding_tn[i], num_trials>1)} & {fmt(flooding_fp[i], num_trials>1)} & {fmt(flooding_fn[i], num_trials>1)} & {fmt_pr(flooding_precision_means[i], flooding_precision_stddevs[i], num_trials>1)} & {fmt_pr(flooding_recall_means[i], flooding_recall_stddevs[i], num_trials>1)} \\\\", file=f)
|
||||
print("\\hline", file=f)
|
||||
print("\\end{tabular}", file=f)
|
||||
print("\\end{table*}", file=f)
|
||||
print("\\end{document}", file=f)
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
#!/bin/bash
|
||||
|
||||
img="troll-patrol"
|
||||
exp_num="$1"
|
||||
uuid="$2"
|
||||
|
||||
container=$(docker run --rm -d -i $img)
|
||||
|
||||
# Create results directory if it doesn't already exist
|
||||
mkdir -p results/${exp_num}
|
||||
|
||||
docker cp configs/troll_patrol_config.json $container:/home/user/troll-patrol/
|
||||
cat configs/troll_patrol_config.json >> "results/${exp_num}/${uuid}"-troll_patrol_config.json
|
||||
docker cp configs/simulation_config.json $container:/home/user/simulation/
|
||||
cat configs/simulation_config.json >> "results/${exp_num}/${uuid}"-simulation_config.json
|
||||
|
||||
# Run rdsys to give bridges to LA
|
||||
docker exec $container sh \
|
||||
-c "cd /home/user/rdsys/ && /usr/bin/time -v ./rdsys -config conf/config.json" \
|
||||
2>&1 | grep -P '\t' > "results/${exp_num}/${uuid}"-rdsys &
|
||||
|
||||
# Give rdsys time to start up
|
||||
sleep 5
|
||||
|
||||
# Run LA, filtering a lot of the output because we don't need it
|
||||
docker exec $container sh \
|
||||
-c "cd /home/user/lox-distributor/ && ./lox-distributor" \
|
||||
| grep -v BridgeLine \
|
||||
| grep -v "Added bridge with fingerprint" \
|
||||
| grep -v "Today's date according to server" \
|
||||
&> "results/${exp_num}/${uuid}"-lox-distributor &
|
||||
|
||||
# Give LA time to start up
|
||||
sleep 5
|
||||
|
||||
# Run Troll Patrol
|
||||
docker exec $container sh \
|
||||
-c "cd /home/user/troll-patrol/ && ./troll-patrol --config troll_patrol_config.json" \
|
||||
&> "results/${exp_num}/${uuid}"-troll-patrol &
|
||||
|
||||
# Give Troll Patrol time to start up
|
||||
sleep 5
|
||||
|
||||
# Run simulation and then kill other processes
|
||||
docker exec $container sh \
|
||||
-c "cd /home/user/simulation/ && ./simulation && killall -s INT rdsys lox-distributor troll-patrol" \
|
||||
| tee "results/${exp_num}/${uuid}"-simulation 2>&1
|
||||
|
||||
# Stop the container once it's done
|
||||
echo "Stopping docker container... It should be removed."
|
||||
docker stop $container
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Number to run in parallel
|
||||
parallel="$1"
|
||||
|
||||
# Number of runs in each configuration
|
||||
n="$2"
|
||||
|
||||
# First and last configuration of experiment 1
|
||||
e1b="$3"
|
||||
e1e="$4"
|
||||
|
||||
# First and last configuration of experiment 2
|
||||
e2b="$5"
|
||||
e2e="$6"
|
||||
|
||||
# Build docker container
|
||||
docker build -t troll-patrol .
|
||||
|
||||
# Parameters should be:
|
||||
# $1: experiment number (1 or 2)
|
||||
# $2: censor secrecy (Overt or Flooding)
|
||||
# $3: harshness (0-4)
|
||||
# $4: probability of users submitting reports (0.0-1.0)
|
||||
run_docker() {
|
||||
# Get a UUID so each simulation run stores its output in a different file
|
||||
uuid=$(cat /proc/sys/kernel/random/uuid)
|
||||
|
||||
./scripts/gen-configs.sh $1 $2 $3 $4
|
||||
./scripts/run-container.sh $1 $uuid
|
||||
|
||||
# If harshness = 2, probability of users submitting reports=0.25,
|
||||
# experiment number = 1, then copy the results to experiment 2
|
||||
# directory.
|
||||
if [[ "$3" == 2 && "$4" == 0.25 && "$1" == 1 ]]; then
|
||||
mkdir -p results/2
|
||||
cp results/1/${uuid}-* results/2/
|
||||
fi
|
||||
}
|
||||
|
||||
# Make list of all configurations to use
|
||||
configs=()
|
||||
|
||||
# Experiment 1
|
||||
for i in $(seq $e1b $e1e); do
|
||||
line=$(sed -n "${i}p" configs/experiment-1)
|
||||
for j in $(seq $n); do
|
||||
configs+=( "1 $line" )
|
||||
done
|
||||
done
|
||||
|
||||
# Experiment 2
|
||||
for i in $(seq $e2b $e2e); do
|
||||
line=$(sed -n "${i}p" configs/experiment-2)
|
||||
for j in $(seq $n); do
|
||||
configs+=( "2 $line" )
|
||||
done
|
||||
done
|
||||
|
||||
# Go through configs in batches of $n
|
||||
index=0
|
||||
while [[ $index -lt ${#configs[@]} ]]; do
|
||||
# Note: Elements contain multiple tokens. Don't put this in quotes.
|
||||
run_docker ${configs[$index]} &
|
||||
|
||||
index=$((index + 1))
|
||||
|
||||
if [[ $(($index % parallel)) == 0 ]]; then
|
||||
# Finish this batch before starting the next one
|
||||
wait
|
||||
fi
|
||||
done
|
||||
|
||||
# Finish the final batch
|
||||
wait
|
||||
Loading…
Reference in New Issue