In January 2021 we will introduce a 10 GB quota for project repositories. Higher limits for individual projects will be available on request. Please see https://doku.lrz.de/display/PUBLIC/GitLab for more information.

Commit a68e024c authored by Philipp Meyer's avatar Philipp Meyer

Ran more benchmarks for scalable examples

parent 859d32e7
cav-benchmarks/bug_tracking/x0_PENDING_q3_buggy
cav-benchmarks/bug_tracking/x0_BUG_REPORT_q1
cav-benchmarks/bug_tracking/x0_BUG_REPORT_q3
cav-benchmarks/bug_tracking/x0_BUG_REPORT_q5
cav-benchmarks/bug_tracking/x0_BUG_REPORT_q6
cav-benchmarks/bug_tracking/x0_BUG_REPORT_q8
cav-benchmarks/bug_tracking/x0_CLOSED_q1
cav-benchmarks/bug_tracking/x0_CLOSED_q3
cav-benchmarks/bug_tracking/x0_CLOSED_q5
cav-benchmarks/bug_tracking/x0_CLOSED_q6
cav-benchmarks/bug_tracking/x0_CLOSED_q8
cav-benchmarks/bug_tracking/x0_FIXED_q1
cav-benchmarks/bug_tracking/x0_FIXED_q3
cav-benchmarks/bug_tracking/x0_FIXED_q5
cav-benchmarks/bug_tracking/x0_FIXED_q6
cav-benchmarks/bug_tracking/x0_FIXED_q8
cav-benchmarks/bug_tracking/x0_FIX_AGAIN_q1
cav-benchmarks/bug_tracking/x0_FIX_AGAIN_q3
cav-benchmarks/bug_tracking/x0_FIX_AGAIN_q5
cav-benchmarks/bug_tracking/x0_FIX_AGAIN_q6
cav-benchmarks/bug_tracking/x0_FIX_AGAIN_q8
cav-benchmarks/bug_tracking/x0_MORE_INFO_q1
cav-benchmarks/bug_tracking/x0_MORE_INFO_q3
cav-benchmarks/bug_tracking/x0_MORE_INFO_q5
cav-benchmarks/bug_tracking/x0_MORE_INFO_q6
cav-benchmarks/bug_tracking/x0_MORE_INFO_q8
cav-benchmarks/bug_tracking/x0_MUST_FIX_q1
cav-benchmarks/bug_tracking/x0_MUST_FIX_q3
cav-benchmarks/bug_tracking/x0_MUST_FIX_q5
cav-benchmarks/bug_tracking/x0_MUST_FIX_q6
cav-benchmarks/bug_tracking/x0_MUST_FIX_q8
cav-benchmarks/bug_tracking/x0_PENDING_q1
cav-benchmarks/bug_tracking/x0_PENDING_q3
cav-benchmarks/bug_tracking/x0_PENDING_q5
cav-benchmarks/bug_tracking/x0_PENDING_q6
cav-benchmarks/bug_tracking/x0_PENDING_q8
cav-benchmarks/bug_tracking/x0_VERIFIED_q1
cav-benchmarks/bug_tracking/x0_VERIFIED_q3
cav-benchmarks/bug_tracking/x0_VERIFIED_q5
cav-benchmarks/bug_tracking/x0_VERIFIED_q6
cav-benchmarks/bug_tracking/x0_VERIFIED_q8
cav-benchmarks/medical/x0_AA_q1.spec
cav-benchmarks/medical/x0_AA_q2.spec
cav-benchmarks/medical/x0_AA_q5.spec
cav-benchmarks/medical/x0_AR_q1.spec
cav-benchmarks/medical/x0_AR_q2.spec
cav-benchmarks/medical/x0_AR_q5.spec
cav-benchmarks/medical/x0_HA_q1.spec
cav-benchmarks/medical/x0_HA_q2.spec
cav-benchmarks/medical/x0_HA_q5.spec
cav-benchmarks/medical/x0_HQ_q1.spec
cav-benchmarks/medical/x0_HQ_q2.spec
cav-benchmarks/medical/x0_HQ_q5.spec
cav-benchmarks/mist/PN/kanban
cav-benchmarks/mist/PN/leabasicapproach
cav-benchmarks/mist/PN/pncsacover
cav-benchmarks/mist/PN/pncsasemiliv
cav-benchmarks/mist/PN/MultiME
cav-benchmarks/mist/PN/basicME
cav-benchmarks/mist/PN/bingham_h150
cav-benchmarks/mist/PN/bingham_h25
cav-benchmarks/mist/PN/bingham_h250
cav-benchmarks/mist/PN/bingham_h250_attic
cav-benchmarks/mist/PN/bingham_h50
cav-benchmarks/mist/PN/csm
cav-benchmarks/mist/PN/extendedread-write
cav-benchmarks/mist/PN/extendedread-write-smallconsts
cav-benchmarks/mist/PN/fms
cav-benchmarks/mist/PN/fms_attic
cav-benchmarks/mist/PN/manufacturing
cav-benchmarks/mist/PN/mesh2x2
cav-benchmarks/mist/PN/mesh3x2
cav-benchmarks/mist/PN/multipool
cav-benchmarks/mist/PN/pingpong
cav-benchmarks/mist/boundedPN/kanban
cav-benchmarks/mist/boundedPN/lamport
cav-benchmarks/mist/boundedPN/newdekker
cav-benchmarks/mist/boundedPN/newrtp
cav-benchmarks/mist/boundedPN/peterson
cav-benchmarks/mist/boundedPN/read-write
cav-benchmarks/soter/finite_leader__single_leader__depth_0
cav-benchmarks/soter/firewall__no_pred_called_with_zero__depth_0
cav-benchmarks/soter/howait__all_workers_finished_if_wait_over__depth_0
cav-benchmarks/soter/howait__all_workers_finished_if_wait_over__depth_1
cav-benchmarks/soter/howait__all_workers_finished_if_wait_over__depth_2
cav-benchmarks/soter/safe_send__sending_to_non-pid__depth_0
cav-benchmarks/soter/stutter__we_abhorr_as__depth_0
cav-benchmarks/soter/stutter__we_abhorr_as__depth_1
cav-benchmarks/soter/stutter__we_abhorr_as__depth_2
cav-benchmarks/soter/unsafe_send__sending_to_non-pid__depth_0
cav-benchmarks/soter/unsafe_send__sending_to_non-pid__depth_1
cav-benchmarks/soter/unsafe_send__sending_to_non-pid__depth_2
cav-benchmarks/soter/concdb__single_client_writes__depth_0
cav-benchmarks/soter/concdb__single_client_writes__depth_1
cav-benchmarks/soter/concdb__single_client_writes__depth_2
cav-benchmarks/soter/finite_leader__single_leader__depth_1
cav-benchmarks/soter/finite_leader__single_leader__depth_2
cav-benchmarks/soter/firewall__no_pred_called_with_zero__depth_1
cav-benchmarks/soter/firewall__no_pred_called_with_zero__depth_2
cav-benchmarks/soter/parikh__should_already_be_initialized__depth_0
cav-benchmarks/soter/parikh__should_already_be_initialized__depth_1
cav-benchmarks/soter/parikh__should_already_be_initialized__depth_2
cav-benchmarks/soter/pipe__single_message_in_mailbox__depth_0
cav-benchmarks/soter/pipe__single_message_in_mailbox__depth_1
cav-benchmarks/soter/pipe__single_message_in_mailbox__depth_2
cav-benchmarks/soter/reslock__critical__depth_0
cav-benchmarks/soter/reslock__critical__depth_1
cav-benchmarks/soter/reslock__critical__depth_2
cav-benchmarks/soter/reslockbeh__critical__depth_0
cav-benchmarks/soter/reslockbeh__critical__depth_1
cav-benchmarks/soter/reslockbeh__critical__depth_2
cav-benchmarks/soter/ring__single_message_in_mailbox__depth_0
cav-benchmarks/soter/ring__single_message_in_mailbox__depth_1
cav-benchmarks/soter/ring__single_message_in_mailbox__depth_2
cav-benchmarks/soter/safe_send__sending_to_non-pid_1__depth_1
cav-benchmarks/soter/safe_send__sending_to_non-pid_1__depth_2
cav-benchmarks/soter/safe_send__sending_to_non-pid_2__depth_1
cav-benchmarks/soter/safe_send__sending_to_non-pid_2__depth_2
cav-benchmarks/soter/safe_send__sending_to_non-pid_3__depth_1
cav-benchmarks/soter/safe_send__sending_to_non-pid_3__depth_2
cav-benchmarks/soter/safe_send__sending_to_non-pid_4__depth_1
cav-benchmarks/soter/safe_send__sending_to_non-pid_4__depth_2
cav-benchmarks/soter/sieve__single_message_in_counter_mailbox__depth_0
cav-benchmarks/soter/sieve__single_message_in_counter_mailbox__depth_1
cav-benchmarks/soter/sieve__single_message_in_counter_mailbox__depth_2
cav-benchmarks/soter/sieve__single_message_in_filter_mailbox__depth_0
cav-benchmarks/soter/sieve__single_message_in_sieve_mailbox__depth_0
cav-benchmarks/soter/state_factory__after_receive_if_no_mail__depth_0
cav-benchmarks/soter/state_factory__single_message_in_mailbox__depth_0
cav-benchmarks/soter/state_factory__single_message_in_mailbox__depth_1
cav-benchmarks/wahl-kroening/Boop_simple_vf_satabs.1/main
cav-benchmarks/wahl-kroening/Boop_simple_vf_satabs.2/main
cav-benchmarks/wahl-kroening/Function_Pointer3_vs_satabs.1/main
cav-benchmarks/wahl-kroening/Function_Pointer3_vs_satabs.2/main
cav-benchmarks/wahl-kroening/Function_Pointer3_vs_satabs.3/main
cav-benchmarks/wahl-kroening/buggy_spaghetti_vf_satabs.1/main
cav-benchmarks/wahl-kroening/buggy_spaghetti_vf_satabs.2/main
cav-benchmarks/wahl-kroening/conditionals_vs_satabs.1/main
cav-benchmarks/wahl-kroening/constants_vf_satabs.1/main
cav-benchmarks/wahl-kroening/constants_vf_satabs.2/main
cav-benchmarks/wahl-kroening/dekker_vs_satabs.1/main
cav-benchmarks/wahl-kroening/dekker_vs_satabs.2/main
cav-benchmarks/wahl-kroening/double_lock_p1_vs_satabs.1/main
cav-benchmarks/wahl-kroening/double_lock_p1_vs_satabs.2/main
cav-benchmarks/wahl-kroening/double_lock_p1_vs_satabs.3/main
cav-benchmarks/wahl-kroening/double_lock_p2_vs_satabs.1/main
cav-benchmarks/wahl-kroening/double_lock_p2_vs_satabs.2/main
cav-benchmarks/wahl-kroening/double_lock_p3_vs_satabs.1/main
cav-benchmarks/wahl-kroening/double_lock_p3_vs_satabs.2/main
cav-benchmarks/wahl-kroening/double_lock_p3_vs_satabs.3/main
cav-benchmarks/wahl-kroening/lu-fig2_fixed_vs_satabs.1/main
cav-benchmarks/wahl-kroening/lu-fig2_fixed_vs_satabs.2/main
cav-benchmarks/wahl-kroening/lu-fig2_fixed_vs_satabs.3/main
cav-benchmarks/wahl-kroening/peterson_vs_satabs.1/main
cav-benchmarks/wahl-kroening/peterson_vs_satabs.2/main
cav-benchmarks/wahl-kroening/pthread5_vs_satabs.1/main
cav-benchmarks/wahl-kroening/pthread5_vs_satabs.2/main
cav-benchmarks/wahl-kroening/pthread5_vs_satabs.3/main
cav-benchmarks/wahl-kroening/pthread5_vs_satabs.4/main
cav-benchmarks/wahl-kroening/rand_cas_vs_satabs.1/main
cav-benchmarks/wahl-kroening/rand_lock_p0_vs_satabs.1/main
cav-benchmarks/wahl-kroening/rand_lock_p0_vs_satabs.2/main
cav-benchmarks/wahl-kroening/rand_lock_p0_vs_satabs.3/main
cav-benchmarks/wahl-kroening/simple_loop5_vs_satabs.1/main
cav-benchmarks/wahl-kroening/simple_loop5_vs_satabs.2/main
cav-benchmarks/wahl-kroening/spin2003_vs_satabs.1/main
cav-benchmarks/wahl-kroening/spin2003_vs_satabs.2/main
cav-benchmarks/wahl-kroening/stack_cas_p0_vs_satabs.1/main
cav-benchmarks/wahl-kroening/stack_cas_p0_vs_satabs.2/main
cav-benchmarks/wahl-kroening/stack_cas_p0_vs_satabs.3/main
cav-benchmarks/wahl-kroening/stack_lock_p0_vs_satabs.1/main
cav-benchmarks/wahl-kroening/stack_lock_p0_vs_satabs.2/main
cav-benchmarks/wahl-kroening/szymanski_vs_satabs.1/main
cav-benchmarks/wahl-kroening/szymanski_vs_satabs.2/main
cav-benchmarks/wahl-kroening/conditionals_vs_satabs.2/main
cav-benchmarks/wahl-kroening/rand_cas_vs_satabs.2/main
......@@ -2,7 +2,8 @@
#benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
#benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' 'cav-benchmarks/medical' 'cav-benchmarks/bug_tracking' )
benchmarks=( 'cav-benchmarks/soter' )
benchmarks=( 'service-tech/ibm-soundness' )
#benchmarks=( 'service-tech/sap-reference' )
extensions=( 'pnet' 'tpn' 'lola' 'spec' )
tool='lola'
......@@ -25,8 +26,7 @@ for benchmark in ${benchmarks[@]}; do
>$benchmark_dir/$prop-timeout-$tool.list
>$benchmark_dir/$prop-error-$tool.list
for ext in ${extensions[@]}; do
#for file in `find $benchmark_dir -name "*.$ext"`; do
for file in $(cat $benchmark_dir/$prop.dontknow); do
for file in `find $benchmark_dir -name "*.$ext"`; do
timing="$(date +%s%N)"
(
ulimit -S -t $time_soft
......@@ -34,7 +34,8 @@ for benchmark in ${benchmarks[@]}; do
ulimit -S -v $mem_soft
ulimit -H -v $mem_hard
set -o pipefail;
$executable -f $file.spec.$prop.task1 $file.spec.$prop 2>&1 | tee $file.out
echo $executable -f $file.$prop.task1 $file.$prop
$executable -f $file.$prop.task1 $file.$prop 2>&1 | tee $file.out
)
result=$?
ryes=$(grep "result: yes" $file.out)
......
#!/bin/bash
#benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' )
#benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' )
benchmarks=( 'service-tech/sap-reference' )
extensions=( 'pnet' 'tpn' 'lola' 'spec' )
executable='/home/philipp/local/sara-1.0/src/sara'
prop=( 'safe' )
for benchmark in ${benchmarks[@]}; do
benchmark_dir="$benchmark"
......@@ -20,7 +22,7 @@ for benchmark in ${benchmarks[@]}; do
echo "testing $file with $base"
(
set -o pipefail;
timeout 60 $executable -i $base.terminating.sara 2>&1 | tee $base.out
timeout 60 $executable -i $base.$prop.sara 2>&1 | tee $base.out
)
popd
result=$?
......
#!/bin/bash
#benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' 'cav-benchmarks/bug_tracking' 'cav-benchmarks/medical' )
benchmarks=( 'service-tech/ibm-soundness' 'service-tech/sap-reference' )
#benchmarks=( 'service-tech/sap-reference' )
#benchmarks=( 'cav-benchmarks/mist' 'cav-benchmarks/wahl-kroening' 'cav-benchmarks/soter' 'cav-benchmarks/bug_tracking' 'cav-benchmarks/medical' )
extensions=( 'pnet' 'lola' 'tpn' 'spec' )
executable='../slapnet'
......@@ -9,14 +10,13 @@ executable='../slapnet'
#3 hours
time_soft=$(expr 3 \* 3600)
properties=( 'deadlock_free' )
prop_options=( '--no-given-properties --safe' )
properties=( 'terminating' )
prop_options=( '--no-given-properties --terminating' )
for benchmark in ${benchmarks[@]}; do
benchmark_dir="$benchmark"
for (( propi=0;propi<${#properties[@]};propi++)); do
prop=${properties[$propi]}
prop_options=${property_options[$propi]}
>$benchmark_dir/$prop-positive-slapnet.list
>$benchmark_dir/$prop-dontknow-slapnet.list
>$benchmark_dir/$prop-timeout-slapnet.list
......@@ -28,7 +28,9 @@ for benchmark in ${benchmarks[@]}; do
timing="$(date +%s%N)"
(
set -o pipefail
timeout $time_soft $executable $prop_options --$ext $file -o $file.$prop 2>&1 | tee $file.$prop.out
echo timeout $time_soft $executable $prop_options --$ext $file -o $file.$prop
timeout $time_soft $executable $prop_options --$ext $file 2>&1 | tee $file.$prop.out
)
result=$?
timing=$(($(date +%s%N)-timing))
......
#!/usr/bin/bash
#folder="LeaderElectionDKR82"
folder="Snapshot"
suffix="-comm"
folder="LeaderElectionCR79"
#folder="Snapshot"
suffix="-ord"
#suffix="-comm"
n=100
m=2
m=3
bf="${folder}/benchmark${suffix}.out"
echo "n user system memory" >$bf
for i in $(seq 1 $n); do
for i in $(seq 10 10 100); do
net="${folder}/n${i}${suffix}.pnet"
echo "Testing net $net"
for j in $(seq 1 $m); do
echo -n "$i " >>$bf
/usr/bin/time -f "%U %S %M" -a -o $bf ../slapnet $net > "${net}.out"
/usr/bin/time -f "%U %S %M" -a -o $bf ../../slapnet $net > "${net}.out"
done
done
......@@ -4,7 +4,8 @@ library("data.table")
library("Hmisc")
#folders <- c("LeaderElectionCR79", "LeaderElectionDKR82")
folders <- c("LeaderElectionDKR82")
#folders <- c("LeaderElectionCR79")
folders <- c("Snapshot")
for (folder in folders) {
infile <- paste(folder, "benchmark.out", sep="/")
......@@ -16,7 +17,8 @@ for (folder in folders) {
datatable <- data.table(dataframe)
data <- datatable[,list(time=mean(user+system),err_time=sd(user+system),memory=mean(memory),err_memory=sd(memory)),by=n]
data <- datatable[,list(time=mean(user+system),memory=mean(memory/1024)),by=n]
#data <- datatable[,list(time=mean(user+system),err_time=sd(user+system),memory=mean(memory/1024),err_memory=sd(memory)),by=n]
write.table(data, file=outfile, row.names=FALSE)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment