Commit 7f494119 authored by Philipp Meyer's avatar Philipp Meyer

Unified and cleaned up benchmark running scripts

parent b3727a3e
#!/bin/bash
function sort_file {
cat $1 | \
sed -e 's/^[[:digit:]]* //' \
-e 's/\.pl$//' \
-e 's/\.pnet$//' \
-e 's/\.spec$//' \
-e 's/\.tts$//' | \
sort \
>$1.sorted
}
results_our_tool=( positive dontknow error timeout )
results_other_tool=( positive negative error timeout )
our_tool=slapnet
benchmark_dirs=( 'found-in-mist-repo' 'given-by-daniel-kroening' 'ic3-soter' )
benchmark_tools=( 'mist' 'bfc' 'iic' )
for (( benchmark=0;benchmark<${#benchmark_dirs[@]};benchmark++)); do
benchmark_dir='benchmarks/'${benchmark_dirs[$benchmark]}
other_tool=${benchmark_tools[$benchmark]}
echo "$our_tool on $benchmark_dir compared with $other_tool"
for result in "${results_our_tool[@]}"; do
sort_file $benchmark_dir/$result-$our_tool.list
done
for result in "${results_other_tool[@]}"; do
sort_file $benchmark_dir/$result-$other_tool.list
done
echo -n "other\\our| "
for ((rour=0;rour<${#results_our_tool[@]};rour++)); do
printf "%8s | " ${results_our_tool[$rour]}
sums_our_tool[$rour]=0
done
echo
echo -n "---------+-"
for rour in "${results_our_tool[@]}"; do
echo -n "---------+-"
done
echo "---------"
for rother in "${results_other_tool[@]}"; do
printf "%8s | " $rother
sum_other_tool=0
for ((rour=0;rour<${#results_our_tool[@]};rour++)); do
n=`comm -12 $benchmark_dir/${results_our_tool[$rour]}-$our_tool.list.sorted $benchmark_dir/$rother-$other_tool.list.sorted | wc -l`
printf "%8d | " $n
sum_other_tool=$((sum_other_tool + n))
sums_our_tool[$rour]=$((${sums_our_tool[$rour]} + n))
done
printf "%8d\n" $sum_other_tool
done
echo -n "---------+-"
for rour in "${results_our_tool[@]}"; do
echo -n "---------+-"
done
echo "---------"
total_sum=0
echo -n " our sum | "
for ((rour=0;rour<${#results_our_tool[@]};rour++)); do
printf "%8d | " ${sums_our_tool[$rour]}
total_sum=$((total_sum + ${sums_our_tool[$rour]}))
done
printf "%8d\n" $total_sum
echo -n "---------+-"
for rour in "${results_our_tool[@]}"; do
echo -n "---------+-"
done
echo "---------"
total_time=0
echo -n "our time | "
for ((rour=0;rour<${#results_our_tool[@]};rour++)); do
result=${results_our_tool[$rour]}
result_time=0
while read T file; do
result_time=$((result_time + T))
done <$benchmark_dir/$result-$our_tool.list
printf "%1.2e | " $result_time
total_time=$((total_time + $result_time))
done
printf "%1.2e\n" $total_time
echo
done
......@@ -18,9 +18,9 @@ results_other_tool=( positive negative error timeout )
our_tool=slapnet
#benchmark_dirs=( 'ibm-soundness' )
benchmark_dirs=( 'sap-reference' )
benchmark_tools=( 'lola' )
#benchmark_dirs=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
benchmark_dirs=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' )
benchmark_tools=( 'sara' 'sara' 'sara ') # TODO: use positive/negative lists
for (( benchmark=0;benchmark<${#benchmark_dirs[@]};benchmark++)); do
benchmark_dir=${benchmark_dirs[$benchmark]}
other_tool=${benchmark_tools[$benchmark]}
......
#!/bin/bash
#benchmarks=( 'mist' 'wahl-kroening' 'soter' )
benchmarks=( 'soter' )
benchmarks=( 'mist' 'wahl-kroening' 'soter' )
extensions=( 'spec' )
executable='../../slapnet'
......@@ -16,7 +15,7 @@ for benchmark in ${benchmarks[@]}; do
T="$(date +%s%N)"
(
set -o pipefail;
timeout 60 $executable --$ext $file | tee $file.out
timeout 60 $executable --$ext --no-given-properties --termination-by-reachability -o $file.terminating $file | tee $file.out
)
result=$?
T=$(($(date +%s%N)-T))
......
#!/bin/bash
#benchmarks=( 'ibm-soundness' 'sap-reference' )
benchmarks=( 'sap-reference' )
#benchmarks=( 'ibm-soundness' )
#extensions=( 'pnet' 'tpn' 'lola' )
extensions=( 'tpn' )
#benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' )
extensions=( 'pnet' 'tpn' 'lola' 'spec' )
executable='/home/philipp/local/lola-2.0/src/lola'
for benchmark in ${benchmarks[@]}; do
......
#!/bin/bash
#benchmarks=( 'ibm-soundness' 'sap-reference' )
benchmarks=( 'sap-reference' )
#benchmarks=( 'ibm-soundness' )
extensions=( 'pnet' 'tpn' 'lola' )
#benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' )
extensions=( 'pnet' 'tpn' 'lola' 'spec' )
executable='/home/philipp/local/sara-1.0/src/sara'
for benchmark in ${benchmarks[@]}; do
......
#!/bin/bash
#benchmarks=( 'ibm-soundness' 'sap-reference' )
benchmarks=( 'sap-reference' )
#benchmarks=( 'ibm-soundness' )
#extensions=( 'pnet' 'tpn' 'lola' )
extensions=( 'tpn' )
executable='../../slapnet'
#benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' )
extensions=( 'pnet' 'tpn' 'lola' 'spec' )
executable='../slapnet'
for benchmark in ${benchmarks[@]}; do
benchmark_dir="$benchmark"
......
#!/bin/bash
benchmarks=( 'found-in-mist-repo' 'given-by-daniel-kroening' 'ic3-soter' )
executable='./slapnet'
for benchmark in ${benchmarks[@]}; do
benchmark_dir="benchmarks/$benchmark"
>$benchmark_dir/positive-slapnet.list
>$benchmark_dir/dontknow-slapnet.list
>$benchmark_dir/timeout-slapnet.list
>$benchmark_dir/error-slapnet.list
for pnet_file in `find $benchmark_dir -name "*.pnet"`; do
T="$(date +%s%N)"
(
set -o pipefail;
timeout 60 $executable $pnet_file | tee $pnet_file.out
)
result=$?
T=$(($(date +%s%N)-T))
if [[ result -eq 0 ]]; then
list='positive'
elif [[ result -eq 2 ]]; then
list='dontknow'
elif [[ result -eq 124 || result -eq 137 ]]; then
list='timeout'
else
list='error'
fi
echo $T $pnet_file >>$benchmark_dir/$list-slapnet.list
done
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment