2.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit 6f0c6f0a authored by Philipp Meyer's avatar Philipp Meyer
Browse files

Configured benchmark scripts to either reproduce PODC results or complete results

parent 5f542a3d
all: main.pdf all: results-complete.pdf
program: pp-print podc: results-podc.pdf
program: pp-print
@echo 'Compiling pp-print' @echo 'Compiling pp-print'
@$(MAKE) -C pp-print @$(MAKE) -C pp-print
...@@ -11,17 +13,26 @@ peregrine: ...@@ -11,17 +13,26 @@ peregrine:
protocols: program protocols: program
@./generate_protocols.sh @./generate_protocols.sh
results/results.csv: protocols peregrine results-podc.csv: protocols peregrine
@./run_benchmarks.sh @./run_benchmarks.sh $@ termination consensus
results-complete.csv: protocols peregrine
@./run_benchmarks.sh $@ termination consensus correctness
table-podc.tex: results-podc.csv
python3 make_table.py $< >$@
table-complete.tex: results-complete.csv
python3 make_table.py $< >$@
table.tex: results/results.csv results-podc.pdf: results-podc.tex table-podc.tex
python3 make_table.py >table.tex pdflatex $<
main.pdf: table.tex results-complete.pdf: results-complete.tex table-complete.tex
pdflatex main.tex pdflatex $<
clean: clean:
@rm -f *.aux *.log *.pdf @rm -f *.aux *.log *.pdf
@rm -rf results protocols out @rm -rf results-podc.csv results-complete.csv protocols out
@rm -f table.tex @rm -f table.tex table-podc.tex table-complete.tex
@$(MAKE) clean -C pp-print @$(MAKE) clean -C pp-print
import sys import sys
import pandas as pd import pandas as pd
benchmark_dir='.' prop_steps = {}
results_dir='results' prop_steps['termination'] = '$|\mathcal{P}|$'
prop_steps['consensus'] = '$|R|$'
properties = [ 'termination', 'consensus', 'correctness' ] prop_steps['correctness'] = '$|R|$'
prop_steps = [ '$|\mathcal{P}|$', '$|R|$', '$|R|$' ]
max_params = 2 max_params = 2
long_table = True long_table = True
short_table = False short_table = False
results_file = 'results/results.csv'
if len(sys.argv) >= 2: if len(sys.argv) >= 2:
long_table = False
short_table = False
for arg in sys.argv[1:]: for arg in sys.argv[1:]:
if arg == '-s': if arg[0] == '-':
short_table = True if arg == '-s':
elif arg == '-l': long_table = False
long_table = True short_table = True
elif arg == '-l':
long_table = True
short_table = False
else:
print("unknown argument: %s" % arg, file=sys.stderr)
sys.exit(-1)
else: else:
print("unknown argument: %s" % arg, file=sys.stderr) results_file = sys.argv[1]
sys.exit(-1)
# long table # long table
if long_table: if long_table:
results_file='results.csv'
results = pd.read_csv(benchmark_dir + '/' + results_dir + '/' + results_file) results = pd.read_csv(results_file)
properties = list(map(lambda s: s[:-7], filter(lambda s: s[-7:] == "_result", results.columns.values, )))
alignrow = "\\begin{longtable}{l" alignrow = "\\begin{longtable}{l"
for i in range(max_params): for i in range(max_params):
...@@ -42,8 +47,8 @@ if long_table: ...@@ -42,8 +47,8 @@ if long_table:
print("\\toprule") print("\\toprule")
headerrow = "protocol & \\multicolumn{%d}{c}{params} & $|Q|$ & $|T|$" % max_params headerrow = "protocol & \\multicolumn{%d}{c}{params} & $|Q|$ & $|T|$" % max_params
for prop, prop_step in zip(properties, prop_steps): for prop in properties:
headerrow += " & %s & %s" % (prop, prop_step) headerrow += " & %s & %s" % (prop, prop_steps[prop])
headerrow += " & total \\\\" headerrow += " & total \\\\"
print(headerrow) print(headerrow)
...@@ -133,7 +138,9 @@ if long_table: ...@@ -133,7 +138,9 @@ if long_table:
if short_table: if short_table:
results_file='results.csv' results_file='results.csv'
results = pd.read_csv(benchmark_dir + '/' + results_dir + '/' + results_file) results = pd.read_csv(results_file)
properties = list(map(lambda s: s[:-7], filter(lambda s: s[-7:] == "_result", results.columns.values, )))
last_protocol = '' last_protocol = ''
last_params = [] last_params = []
......
\documentclass{scrartcl}
\usepackage[a4paper, top=1cm, bottom=1cm, left=1cm, right=1cm]{geometry}
\usepackage[table]{xcolor}
\usepackage{booktabs}
\usepackage{longtable}
\usepackage{multirow}
\begin{document}
\input{table-complete.tex}
\begin{tabular}{ll}
$|Q|$ & Number of states. \\
$|T|$ & Number of non-silent transitions. \\
termination & Time in seconds to prove layered termination. \\
$|\mathcal{P}|$ & Number of layers in the partition for layered termination. \\
consensus & Time in seconds to prove strong consensus. \\
correctness & Time in seconds to prove strong consensus with correctness. \\
$|\mathcal{R}|$ & Number of refinement steps (i.e.\ number of traps or siphons) for strong consensus and correctness. \\
total & Time in seconds to prove layered termination, strong consensus and strong consensus with correctness. \\
\end{tabular}
\end{document}
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
\begin{document} \begin{document}
\input{table.tex} \input{table-podc.tex}
\begin{tabular}{ll} \begin{tabular}{ll}
$|Q|$ & Number of states. \\ $|Q|$ & Number of states. \\
......
#!/bin/bash #!/bin/bash
benchmark_dir='.'
protocols_dir='protocols' protocols_dir='protocols'
out_dir='out' out_dir='out'
results_dir='results'
extension='pp' extension='pp'
executable_dir='../dist/build/peregrine' executable_dir='../dist/build/peregrine'
...@@ -13,82 +11,83 @@ options='-i -v' ...@@ -13,82 +11,83 @@ options='-i -v'
#1 hour #1 hour
timelimit=$((1 * 3600)) timelimit=$((1 * 3600))
properties=( 'termination' 'consensus' 'correctness' ) results_file="$1"
prop_options=( '--layered-termination' '--strong-consensus' '--correctness' ) properties="${@:2}"
prop_refinements=( 'Checking SAT of layered termination' 'Checking SAT of trap' 'Checking SAT of trap' )
structure_option='--structure' declare -A prop_options
prop_options['termination']='--layered-termination'
prop_options['consensus']='--strong-consensus'
prop_options['correctness']='--correctness'
mkdir -p $benchmark_dir/$out_dir declare -A prop_refinements
mkdir -p $benchmark_dir/$results_dir prop_refinements['termination']='Checking SAT of layered termination'
prop_refinements['consensus']='Checking SAT of trap'
prop_refinements['correctness']='Checking SAT of trap'
>$benchmark_dir/$results_dir/results.csv structure_option='--structure'
echo -n "protocol,file,states,transitions" >>$benchmark_dir/$results_dir/results.csv
for (( propi=0;propi<${#properties[@]};propi++)); do mkdir -p $out_dir
prop=${properties[$propi]}
echo -n ",${prop}_result,${prop}_time,${prop}_refinements" >>$benchmark_dir/$results_dir/results.csv >$results_file
>$benchmark_dir/$results_dir/$prop.positive.list echo -n "protocol,file,states,transitions" >>$results_file
>$benchmark_dir/$results_dir/$prop.dontknow.list for prop in $properties; do
>$benchmark_dir/$results_dir/$prop.timeout.list echo -n ",${prop}_result,${prop}_time,${prop}_refinements" >>$results_file
>$benchmark_dir/$results_dir/$prop.error.list
done done
echo >>$benchmark_dir/$results_dir/results.csv echo >>$results_file
for protocol_dir in $(find $benchmark_dir/$protocols_dir -mindepth 1 -maxdepth 1 -type d); do for protocol_dir in $(find $protocols_dir -mindepth 1 -maxdepth 1 -type d); do
protocol=$(basename $protocol_dir) protocol=$(basename $protocol_dir)
echo echo
echo "Benchmarking $protocol protocol" echo "Benchmarking $protocol protocol"
echo echo
mkdir -p $benchmark_dir/$out_dir/$protocol mkdir -p $out_dir/$protocol
timeout_reached='false' timeout_reached='false'
for param in $(seq 1 1000); do for param in $(seq 1 1000); do
if [ $timeout_reached == 'false' ]; then if [ $timeout_reached == 'false' ]; then
for filename in $(find $protocol_dir -mindepth 1 -maxdepth 1 -name "${protocol}_?${param}_*.$extension"); do for filename in $(find $protocol_dir -mindepth 1 -maxdepth 1 -name "${protocol}_?${param}_*.$extension"); do
file=$(basename $filename) file=$(basename $filename)
echo -n "$protocol,$file" >>$benchmark_dir/$results_dir/results.csv echo -n "$protocol,$file" >>$results_file
( (
$executable_dir/$executable $options $structure_option $filename 2>&1 | tee $benchmark_dir/$out_dir/$protocol/$file.out $executable_dir/$executable $options $structure_option $filename 2>&1 | tee $out_dir/$protocol/$file.out
) )
n_states=$(grep -e '^States *: ' "$benchmark_dir/$out_dir/$protocol/$file.out" | sed -e 's/^.*: \([0-9]*\)$/\1/') n_states=$(grep -e '^States *: ' "$out_dir/$protocol/$file.out" | sed -e 's/^.*: \([0-9]*\)$/\1/')
n_transitions=$(grep -e '^Transitions *: ' "$benchmark_dir/$out_dir/$protocol/$file.out" | sed -e 's/^.*: \([0-9]*\)$/\1/') n_transitions=$(grep -e '^Transitions *: ' "$out_dir/$protocol/$file.out" | sed -e 's/^.*: \([0-9]*\)$/\1/')
echo -n ",$n_states,$n_transitions" >>$benchmark_dir/$results_dir/results.csv echo -n ",$n_states,$n_transitions" >>$results_file
for (( propi=0;propi<${#properties[@]};propi++)); do for prop in $properties; do
prop=${properties[$propi]} prop_option=${prop_options[$prop]}
prop_option=${prop_options[$propi]} prop_refinement=${prop_refinements[$prop]}
prop_refinement=${prop_refinements[$propi]}
echo "Benchmarking $file with property $prop" echo "Benchmarking $file with property $prop"
timing="$(date +%s%N)" timing="$(date +%s%N)"
( (
set -o pipefail set -o pipefail
echo timeout $timelimit $executable_dir/$executable $options $prop_option $filename echo timeout $timelimit $executable_dir/$executable $options $prop_option $filename
timeout $timelimit $executable_dir/$executable $options $prop_option $filename 2>&1 | tee $benchmark_dir/$out_dir/$protocol/$file.$prop.out timeout $timelimit $executable_dir/$executable $options $prop_option $filename 2>&1 | tee $out_dir/$protocol/$file.$prop.out
) )
result=$? result=$?
timing=$(($(date +%s%N)-timing)) timing=$(($(date +%s%N)-timing))
if [[ result -eq 0 ]]; then if [[ result -eq 0 ]]; then
list='positive' res='positive'
elif [[ result -eq 2 ]]; then elif [[ result -eq 2 ]]; then
list='dontknow' res='dontknow'
elif [[ result -eq 124 || result -eq 137 ]]; then elif [[ result -eq 124 || result -eq 137 ]]; then
list='timeout' res='timeout'
else else
list='error' res='error'
fi fi
# get number of refinement steps # get number of refinement steps
n_refinements=$(grep -c -e "$prop_refinement" "$benchmark_dir/$out_dir/$protocol/$file.$prop.out") n_refinements=$(grep -c -e "$prop_refinement" "$out_dir/$protocol/$file.$prop.out")
echo "$file,$timing" >>$benchmark_dir/$results_dir/$prop.$list.list echo -n ",$res,$timing,$n_refinements" >>$results_file
echo -n ",$list,$timing,$n_refinements" >>$benchmark_dir/$results_dir/results.csv if [ $res == 'timeout' ]; then
if [ $list == 'timeout' ]; then
timeout_reached='true' timeout_reached='true'
fi fi
done done
echo >>$benchmark_dir/$results_dir/results.csv echo >>$results_file
done done
fi fi
done done
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment