Commit fcfba200 authored by Salomon Sickert-Zehnter's avatar Salomon Sickert-Zehnter

Merge branch 'regression' into 'master'

Update Testing

See merge request i7/owl!364
parents 5eecd8e6 2311794a
......@@ -30,7 +30,9 @@ JAVA_HOME=/opt/graalvm-ce-java11-20.1.0/
If GraalVM (native-image) is not available, the project can also be built with a reduced set of features on any JDK that supports at least Java 11. See below for instructions.
## Building on GraalVM
## Building
### GraalVM
The standard distribution can be obtained with:
......@@ -40,7 +42,7 @@ $ ./gradlew distZip
The resulting `.zip` is located in `build/distributions`. It includes the scripts for the CLI tools, Jars usable as a Java library, and a C library.
## Building on OpenJDK
### OpenJDK
If GraalVM is not available, building the native executable and library can be skipped by executing:
......@@ -48,6 +50,27 @@ If GraalVM is not available, building the native executable and library can be s
$ ./gradlew distZip -Pdisable-native
```
### Docker
In case you want to build or test locally using docker (recommended on Windows), first build the docker image with
```
cd docker-build-environment && docker build -t owl .
```
Then, run the build process via
```
docker run --rm -it -v <path>/owl:/dir -w /dir owl ./gradlew build
```
or similar. Optionally, you can pass `GRADLE_HOME=./.gradle` and `GRADLE_USER_HOME=./.gradle` to speed up subsequent builds.
## Testing
To test locally, run `gradle localEnvironment` to update the folder and then `python scripts/util.py test <name>` to run the respective test.
On Windows, installing WSL is required and the testing commands (scripts in the `script` folder) should be run from within WSL.
To test inside docker, run
```
docker run -e GRADLE_HOME=./.gradle -e GRADLE_USER_HOME=./.gradle --rm -it -v <path>/owl:/dir -w /dir owl ./gradlew localEnvironment && python scripts/util.py test <name>
```
## Citing
Please see the [citing section of the official website for an updated list](https://owl.model.in.tum.de/#citing).
......@@ -17,6 +17,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import org.apache.tools.ant.taskdefs.condition.Os
plugins {
// https://github.com/tbroyer/gradle-errorprone-plugin
// https://plugins.gradle.org/plugin/net.ltgt.errorprone
......@@ -56,10 +58,23 @@ tasks.withType(JavaCompile) {
// options.errorprone.disableWarningsInGeneratedCode = true
}
def disablePandoc = project.hasProperty("disable-pandoc")
def disableNative = project.hasProperty("disable-native")
def buildMarkdown = !project.hasProperty("disable-pandoc")
def buildOwlNative = !project.hasProperty("disable-native")
def includeJar = !project.hasProperty("exclude-jar")
def enableNativeAssertions = project.hasProperty("enable-native-assertions")
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
if (project.hasProperty("disable-default")) {
project.logger.quiet("Running on windows, disable native compilations with -Pdisable-native -Pdisable-pandoc")
} else {
project.logger.quiet("Running on windows, disabling unsupported native compilations, specify -Pdisable-default to override")
buildMarkdown = false;
buildOwlNative = false;
}
}
apply from: 'gradle/analysis.gradle'
apply from: 'gradle/antlr.gradle'
......@@ -113,7 +128,7 @@ dependencies {
// https://mvnrepository.com/artifact/com.google.auto.value/auto-value
implementation 'com.google.auto.value:auto-value-annotations:1.7.1'
implementation 'com.google.auto.value:auto-value-annotations:1.7.1'
annotationProcessor 'com.google.auto.value:auto-value:1.7.1'
}
......@@ -145,23 +160,6 @@ tasks.withType(Test) {
maxHeapSize = "16G"
}
model {
components {
owlClient(NativeExecutableSpec) {
baseName "owl-client"
sources {
c {
source {
srcDir "src/main/c"
include "owl-client.c"
}
}
}
}
}
}
// ---------------- Startup Scripts ----------------
// Create startup scripts for each tool
......@@ -197,23 +195,7 @@ def scripts = [
'nba2ldba' : 'owl.translations.nba2ldba.NBA2LDBA',
'dra2dpa' : 'owl.translations.dra2dpa.IARBuilder',
// 'owl-server': 'owl.run.ServerCli'
]
def rabinizerScripts = [
// LTL -> LDBA
'ltl2ldba' : 'owl.translations.ltl2ldba.LTL2LDBAModule',
'ltl2ldgba' : 'owl.translations.ltl2ldba.LTL2LDGBAModule',
// LTL -> DRA
'ltl2dra' : 'owl.translations.rabinizer.RabinizerDegeneralizeMain',
'ltl2dgra' : 'owl.translations.rabinizer.RabinizerMain',
// LTL -> DPA
'ltl2dpa' : 'owl.translations.LTL2DPA',
// DRA -> DPA
'dra2dpa' : 'owl.translations.dra2dpa.IARBuilder'
'owl-server': 'owl.run.ServerCli'
]
ext.createStartScriptTasks = { name, cp, map ->
......@@ -223,7 +205,7 @@ ext.createStartScriptTasks = { name, cp, map ->
mainClassName = className
applicationName = scriptName
// CreateStartScripts is stupid; it.outputs.files = outputDir, not the created scripts.
outputDir = file(project.buildDir.path + '/scripts/' + scriptName)
outputDir = file(project.buildDir.path + '/scripts/' + scriptName)
classpath = cp
}
}
......@@ -257,9 +239,10 @@ task mavenJars(dependsOn: [jar, sourcesJar, javadocJar])
task buildNativeLibrary(type: Exec, dependsOn: [classes]) {
mkdir "${buildDir}/native-library"
workingDir "${buildDir}/native-library"
onlyIf { buildOwlNative }
def graalHome = System.getenv("GRAAL_HOME")
def command = (graalHome != null ? graalHome + "/bin/" : "") + "native-image"
def command = (graalHome == null ? "" : graalHome + "/bin/") + "native-image";
commandLine command,
"owl.cinterface.CInterface", "libowl",
......@@ -279,9 +262,10 @@ task buildNativeLibrary(type: Exec, dependsOn: [classes]) {
task buildNativeExecutable(type: Exec, dependsOn: [classes]) {
mkdir "${buildDir}/native-executable"
workingDir "${buildDir}/native-executable"
onlyIf { buildOwlNative }
def graalHome = System.getenv("GRAAL_HOME")
def command = (graalHome != null ? graalHome + "/bin/" : "") + "native-image"
def command = (graalHome == null ? "" : graalHome + "/bin/") + "native-image";
// TODO: remove dependency on header files.
commandLine command,
......@@ -300,36 +284,20 @@ task buildNativeExecutable(type: Exec, dependsOn: [classes]) {
// ---------------- Script Jobs ----------------
def allStartScripts = createStartScriptTasks("", jar.outputs.files + project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME), scripts)
allStartScripts.each() {
it.dependsOn(jar)
}
def rabinizerStartScripts = createStartScriptTasks("Rabinizer", jar.outputs.files + project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME), rabinizerScripts)
rabinizerStartScripts.each() {
def otherStartScripts = createStartScriptTasks("", jar.outputs.files + project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME), scripts)
otherStartScripts.each() {
it.dependsOn(jar)
}
// ---------------- Documentation ----------------
// Compile the markdown files
import org.apache.tools.ant.taskdefs.condition.Os
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
tasks.create(name: "compileMarkdown") {
doLast {
logger.warn("Cannot compile markdown on windows, skipping")
}
}.onlyIf {
!disablePandoc
}
} else {
tasks.create(name: "compileMarkdown", type: Exec) {
executable = 'scripts/render-markdown.sh'
outputs.dir "$project.docsDir/markdown"
args = ["$project.docsDir/markdown"]
}.onlyIf {
!disablePandoc
}
tasks.create(name: "compileMarkdown", type: Exec) {
executable = 'scripts/render-markdown.sh'
outputs.dir "$project.docsDir/markdown"
args = ["$project.docsDir/markdown"]
}.onlyIf {
buildMarkdown
}
javadoc {
......@@ -357,50 +325,67 @@ distributions {
from compileMarkdown
into("bin") {
from allStartScripts
from otherStartScripts
fileMode = 0755
}
if (!disableNative) {
into("clib") {
from "$projectDir/src/main/headers"
from "$buildDir/native-library" include "*.h"
from "$buildDir/native-library" include "*.so"
}
if (buildOwlNative) {
into("clib") {
from "$projectDir/src/main/headers"
from "$buildDir/native-library" include "*.h"
from "$buildDir/native-library" include "*.so"
}
into("bin") {
from "$buildDir/native-executable" include "owl-native"
}
into("bin") {
from "$buildDir/native-executable" include "owl-native"
}
}
into("jars") {
from jar
from javadocJar
from sourcesJar
if (includeJar) {
into("jars") {
from jar
from javadocJar
from sourcesJar
}
}
}
}
}
rabinizer {
distributionBaseName = 'rabinizer'
task localEnvironment(type: Copy) {
destinationDir = project.buildDir
dependsOn(tasks.getByPath(":buildNativeLibrary"))
dependsOn(tasks.getByPath(":buildNativeExecutable"))
contents {
from "AUTHORS"
from "LICENSE"
into("bin") {
from startScripts
from otherStartScripts
fileMode = 0755
}
into("bin") {
from rabinizerStartScripts
fileMode = 0755
}
into("lib") {
from configurations.default
from jar
}
if (buildOwlNative) {
into("clib") {
from "$projectDir/src/main/headers"
from "$buildDir/native-library" include "*.h"
from "$buildDir/native-library" include "*.so"
}
into("bin") {
from "$buildDir/native-executable" include "owl-native"
}
}
}
if (!disableNative) {
tasks.getByPath(":distZip").dependsOn(tasks.getByPath(":buildNativeExecutable"))
tasks.getByPath(":distTar").dependsOn(tasks.getByPath(":buildNativeExecutable"))
tasks.getByPath(":distZip").dependsOn(tasks.getByPath(":buildNativeLibrary"))
tasks.getByPath(":distTar").dependsOn(tasks.getByPath(":buildNativeLibrary"))
}
tasks.getByPath(":distZip").dependsOn(tasks.getByPath(":buildNativeExecutable"))
tasks.getByPath(":distTar").dependsOn(tasks.getByPath(":buildNativeExecutable"))
tasks.getByPath(":distZip").dependsOn(tasks.getByPath(":buildNativeLibrary"))
tasks.getByPath(":distTar").dependsOn(tasks.getByPath(":buildNativeLibrary"))
apply from: 'gradle/idea.gradle'
......@@ -6,6 +6,9 @@
<trusted-artifacts>
<trust file=".*-javadoc[.]jar" regex="true"/>
<trust file=".*-sources[.]jar" regex="true"/>
<!-- These verifications fail on windows, probably due to line endings? -->
<trust file=".*[.]pom" regex="true"/>
<trust file=".*[.]module" regex="true"/>
</trusted-artifacts>
</configuration>
<components>
......
......@@ -72,7 +72,7 @@ if [ ${#} -eq 0 ]; then
fi
shift
tool_executable="$(unix_path "$1")"
tool_executable="$1"
if ! [ -e "${tool_executable}" ] && ! command -v "${tool_executable}" 1>/dev/null 2>&1; then
echo "Specified tool executable $tool_executable not found"
exit 1
......@@ -87,12 +87,7 @@ if [ "$update" = "1" ]; then
echo "Updating binaries"
(
cd "${project_folder}"
if [[ $os == "windows" ]]; then
gradle="gradlew.bat"
else
gradle="./gradlew"
fi
if ! output=$($gradle --no-daemon buildBin); then
if ! output=$(./gradlew --no-daemon buildBin); then
echo "Gradle failed, output:"
echo "${output}"
exit 1
......@@ -147,7 +142,7 @@ echo "(total of $(wc -l < "${formula_file}") formulas)"
# shellcheck disable=SC2145
echo "Command: ${tool_executions[@]}"
tool_executions=( "${tool_executions[@]/\%F/$(win_path "${formula_file}")}" )
tool_executions=( "${tool_executions[@]/\%F/"${formula_file}"}" )
if [ "$method" = "time" ]; then
......
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import socket
if __name__ == "__main__":
if len(sys.argv) == 1 and sys.argv[0] == "--version":
print("owl python client version 1.0")
sys.exit(0)
if not 3 <= len(sys.argv) <= 4:
print("Args: <hostname> <port> <input>?")
hostname, port = sys.argv[1], int(sys.argv[2])
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((hostname, port))
if len(sys.argv) == 4:
s.sendall(sys.argv[3].encode())
else:
s.sendall(sys.stdin.buffer.read())
s.shutdown(socket.SHUT_WR)
data = b''
while True:
recv = s.recv(1024)
if not recv:
break
data += recv
print(data.decode(), end="")
......@@ -10,7 +10,7 @@ echo "Running bench with args $@"
source "$(dirname "$0")/vars.sh"
results_folder="${project_folder}/build/results"
evaluation_script=$(win_path "${script_folder}/spotcross-eval.py")
evaluation_script="${script_folder}/spotcross-eval.py"
timeout_sec="300"
declare -a tool_names
......@@ -69,7 +69,7 @@ while [ ${#} -gt 0 ]; do
"${tools_invocation[@]}" 2> >(tee "$ltlcross_output_file")
dataset_names+=("$dataset_name")
csv_files+=("$(win_path "$csv_file")")
csv_files+=( "$csv_file" )
done
if [ ${num_datasets} -gt 1 ]; then
......
......@@ -10,7 +10,7 @@ echo "Running test with args $@"
source "$(dirname "$0")/vars.sh"
results_folder="${project_folder}/build/results"
evaluation_script=$(win_path "${script_folder}/spotcross-eval.py")
evaluation_script="${script_folder}/spotcross-eval.py"
timeout_sec="300"
any_error=0
......@@ -104,8 +104,6 @@ while [ ${#} -gt 0 ]; do
else
any_error=1
[ "$os" == "windows" ] && dos2unix -q "$grind_file"
faulty_formula="$(tail -n1 "$grind_file")"
if [ -z "$faulty_formula" ]; then
echo "Not generating automaton image as no faulty formula is present"
......@@ -177,7 +175,8 @@ while [ ${#} -gt 0 ]; do
function make_image() {
formula="$1"
destination="$2"
declare -a highlight=("${!3}")
shift 2
declare -a highlight=( "$@" )
if ! destination_file=$(mktemp -t owl.tmp.XXXXXXXXXX); then
echo "Failed to obtain temprorary file"
......@@ -210,7 +209,7 @@ while [ ${#} -gt 0 ]; do
export SPOT_DOTEXTRA="edge[arrowhead=vee, arrowsize=.7]"
# highlight-word fails for fin acceptance, have to strip it
if [ ${#highlight[@]} -eq 0 ]; then
if [ ${#} -eq 0 ]; then
autfilt --dot <"${destination_file}" >"$destination.dot"
elif ! autfilt --dot "${highlight[@]}" <"${destination_file}" \
>"$destination.dot" 2>/dev/null; then
......@@ -228,8 +227,8 @@ while [ ${#} -gt 0 ]; do
}
if [ ${tool_error} = "1" ]; then
make_image "$faulty_formula" "$results_folder/error-pos" pos_highlight[@]
make_image "!($faulty_formula)" "$results_folder/error-neg" neg_highlight[@]
make_image "$faulty_formula" "$results_folder/error-pos" ${pos_highlight[@]}
make_image "!($faulty_formula)" "$results_folder/error-neg" ${neg_highlight[@]}
fi
done
fi
......@@ -237,12 +236,12 @@ while [ ${#} -gt 0 ]; do
if [ ${num_datasets} -gt 1 ]; then
echo ""
echo "Stats for dataset $dataset_name"
! ${python} "$evaluation_script" "ltl" "$(win_path "$csv_file")" |
! ${python} "$evaluation_script" "ltl" "$csv_file" |
tee "$results_folder/stats-$dataset_name.txt"
echo ""
fi
csv_files+=("$(win_path "$csv_file")")
csv_files+=( "$csv_file" )
if [ "$dataset_error" = "1" ]; then
# If this dataset failed, don't evaluate more sets
......
......@@ -33,13 +33,18 @@ def run(servers, env=None):
pass
if open_ports:
print("Open ports: {0}".format(", ".join(map(str, open_ports))))
sys.exit(1)
server_processes = dict()
for port, server in servers.items():
server_process = subprocess.Popen(server, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=None, env=env)
server_processes[port] = server_process
try:
server_process = subprocess.Popen(server, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=None, env=env)
server_processes[port] = server_process
except FileNotFoundError as e:
print(f"Could not run process {' '.join(server)}")
raise e
for port, process in server_processes.items():
while not is_open(port):
......@@ -52,10 +57,7 @@ def run(servers, env=None):
def stop(server_processes):
for server_process in server_processes.values():
if os.name == 'nt':
os.kill(server_process.pid, signal.CTRL_C_EVENT)
else:
server_process.terminate()
server_process.terminate()
for server_process in server_processes.values():
try:
......
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import os
import sys
from os.path import dirname, join, exists, relpath
def get_bin_dir():
return join(dirname(dirname(dirname(__file__))), "build", "bin")
def is_native_available():
return exists(join(get_bin_dir(), "owl-native"))
def get_client_executable():
return relpath(join(dirname(dirname(__file__)), "client.py"))
def get_owl_executable():
bin_dir = get_bin_dir()
native_path = join(bin_dir, "owl-native")
if os.path.exists(native_path):
return relpath(native_path)
return relpath(join(bin_dir, "owl"))
def get_owl_server_executable():
return relpath(join(get_bin_dir(), "owl-server"))
class Tool(object):
def __init__(self, name, flags):
......@@ -80,30 +106,20 @@ class OwlTool(Tool):
return pipeline
def get_server_execution(self, port=None):
if os.name == 'nt':
tool_execution = ["build\\bin\\owl-server.bat"]
else:
tool_execution = ["build/bin/owl-server"]
tool_execution = [get_owl_server_executable()]
if port is not None:
tool_execution.extend(["--port", str(port)])
tool_execution.extend(self.get_pipeline())
return tool_execution
def get_file_execution(self, file):
if os.name == 'nt':
tool_execution = ["build\\bin\\owl.bat"]
else:
tool_execution = ["build/bin/owl"]
tool_execution = [get_owl_executable()]
tool_execution.extend(["-I", file])
tool_execution.extend(self.get_pipeline())
return tool_execution
def get_input_execution(self, static_input):
if os.name == 'nt':
tool_execution = ["build\\bin\\owl.bat"]
else:
tool_execution = ["build/bin/owl-native"]
tool_execution = [get_owl_executable()]
tool_execution.extend(["-i", static_input])
tool_execution.extend(self.get_pipeline())
return tool_execution
......
......@@ -5,14 +5,12 @@ import os
import os.path as path
import subprocess
import sys
import socket
import time
import signal
from os.path import relpath
import owlpy.defaults as owl_defaults
import owlpy.formula as owl_formula
import owlpy.tool as owl_tool
import owlpy.run_servers as run_servers
import owlpy.tool as owl_tool
def _test(args, check):
......@@ -66,13 +64,10 @@ def _test(args, check):
if check:
reference = test_json["reference"]
test_arguments = [owl_defaults.get_script_path("ltlcross-run.sh"),
reference["name"], " ".join(reference["exec"])]
test_arguments = [str(relpath(owl_defaults.get_script_path("ltlcross-run.sh"))),
reference["name"], " ".join(reference["exec"])]
else:
test_arguments = [owl_defaults.get_script_path("ltlcross-bench.sh")]
if os.name == 'nt':
test_arguments = ["bash"] + test_arguments
test_arguments = [str(relpath(owl_defaults.get_script_path("ltlcross-bench.sh")))]
loaded_tools = []
......@@ -95,7 +90,7 @@ def _test(args, check):
else:
raise TypeError("Unknown tools type {0!s}".format(type(tools)))
enable_server = False
enable_server = not owl_tool.is_native_available()
port = 6060
servers = {}
for test_tool in loaded_tools:
......@@ -112,8 +107,8 @@ def _test(args, check):
if type(loaded_tool) is owl_tool.OwlTool:
if enable_server:
servers[port] = loaded_tool.get_server_execution(port)
test_arguments.append("build/bin/owl-client"
+ " localhost " + str(port) + " %f")
test_arguments.append(f"{owl_tool.get_client_executable()} "
f"localhost {port} %f")
port += 1
else:
test_arguments.append(" ".join(loaded_tool.get_input_execution("%f")))
......@@ -159,7 +154,6 @@ def _test(args, check):
sub_env["JAVA_OPTS"] = "-Xss64M"
if servers:
from contextlib import closing
print("Servers:")
for server in servers.values():
......@@ -260,9 +254,6 @@ def _benchmark(args):
benchmark_script = [owl_defaults.get_script_path("benchmark.sh"), "--stdin",
"--repeat", str(repeat)]
if os.name == 'nt':
benchmark_script = ["bash"] + benchmark_script
if update:
benchmark_script += ["--update"]
if perf:
......
......@@ -11,33 +11,4 @@ if command -v python3 >/dev/null 2>&1; then
python="python3"
else
python="python"
fi
case "$(uname -s)" in
CYGWIN*)
os="windows"
;;
*)
os="linux"
;;
esac
if [[ $os == "windows" ]]; then
function win_path() {
for path in "$@"; do
cygpath -w "$path"
done
}
function unix_path() {
for path in "$@"; do
cygpath -u "$path"
done
}
else
function win_path() {
echo "$@"
}
function unix_path() {
echo "$@"
}
fi
fi
\ No newline at end of file
// Based on http://www.linuxhowtos.org socket tutorial
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <errno.h>
#define BUFFER_SIZE 8192
void error(const char *msg)
{
perror(msg);
exit(1);
}
int main(int argc, char *argv[])
{
int sockfd, portno;
struct sockaddr_in serv_addr;
struct hostent *server;