Manual Merge branch 'toolkit2' of gitlab.lrz.de:exahype/ExaHyPE-Engine into toolkit2 (d92031f5) · Commits · ExaHyPE / ExaHyPE-Engine

Toolkit2/exahype-specfile.schema.json

+24 −14

Original line number	Diff line number	Diff line
		@@ -646,6 +646,14 @@
		"old_format" : { "detect_variable_list_after" : "material-parameters" }
		},
		"point_sources" : {
		"oneOf" : [
		{
		"type" : "integer",
		"scope" : "compile-time",
		"used-by" : "toolkit",
		"$comment" : "Specify number of point sources"
		},
		{
		"type" : "array",
		"scope" : "compile-time",
		"used-by" : "toolkit",
		@@ -661,6 +669,8 @@
		"minItems" : 2,
		"maxItems" : 3
		}
		}
		]
		},
		"global_observables" : {
		"$ref" : "#/definitions/variables",

Toolkit2/exahype/specfiles/mexa.py

+10 −10

Original line number	Diff line number	Diff line
		@@ -603,12 +603,12 @@ class mexagraph:
		for right, attr in self.G[starting_from].iteritems():
		if attr['op'] == self.P:
		if right.value == first:
		#print "Looking for %s, Found %s" % (left,first)
		#print("Looking for %s, Found %s" % (left,first))
		return self.get_path_down(left.except_base(), right, silent_failure=silent_failure)
		else:
		#print "No success: %s != %s" % (right,first)
		#print("No success: %s != %s" % (right,first))
		pass
		#print "Looked for %s (%s), found nothing" % (left,first)
		#print("Looked for %s (%s), found nothing" % (left,first))
		if not silent_failure:
		raise ValueError("Symbol %s not found in graph, searching from %s." % (left,starting_from))

		@@ -743,13 +743,13 @@ class mexagraph:
		#assert isa(root, term)
		ret = []
		for right, attr in self.G[left].iteritems():
		#print "%s,%s" % (right,attr)
		#print("%s,%s" % (right,attr))
		if attr['op'] == self.P:
		right_path = symbol(right.value).add_prefix(left_path)# if isa(right,term) else str(right))
		#print "At %s, %s: Visiting %s, %s" % (left, left_path, right, right_path)
		#print("At %s, %s: Visiting %s, %s" % (left, left_path, right, right_path))
		ret += self.to_mexafile(right, right_path)
		else:
		#print "%s: Attr is %s" % (right, attr)
		#print("%s: Attr is %s" % (right, attr))
		if isa(right, term):
		right = self.get_path_down(right) # TODO: Resolve the term -> symbol here.
		ret += [ Rel(left_path, right, attr['op'], attr['src']) ]
		@@ -789,7 +789,7 @@ class mexagraph:
		# put onto the stack:
		newstack = stack + tuplize(Rel(left, right, self.P, attr['src']))
		#right_path = symbol(right.value).add_prefix(left_path)
		#print "At %s, %s: Visiting %s, %s" % (left, left_path, right, right_path)
		#print("At %s, %s: Visiting %s, %s" % (left, left_path, right, right_path))
		ret += self.evaluate_to_mexafile(right, newstack, max_rec=max_rec-1, sort_by=sort_by)
		elif attr['op'] == self.E:
		if isa(right, term):
		@@ -806,7 +806,7 @@ class mexagraph:
		newstack = stack + tuplize(Rel(left, right, self.E, attr['src']))
		pathlst = [ rel.r.value for rel in newstack if rel.op == self.P ]
		srclst = [ rel.src for rel in newstack if rel.op == self.E and not rel.l.isRoot() ]
		#print "At left=%s, newstack=%s I composed path=%s, srclist=%s" % (left,newstack,pathlst,srclst)
		#print("At left=%s, newstack=%s I composed path=%s, srclist=%s" % (left,newstack,pathlst,srclst))
		ret += [ Rel(symbol(pathlst), right, "equals", source(srclst)) ] # could also use "define"
		else:
		# TODO: Should instead let all other operations pass.
		@@ -1765,9 +1765,9 @@ class mexafile_to_exahype_specfile(io_mexafile):
		return jinja_env.get_template(self.tplfile).render(ctx)
		# without exception chaining, we loose the stack trace:
		#except Exception as e:
		# print "Debuggin context:"
		# print("Debuggin context:")
		# pprint.pprint(ctx)
		# print "Exception:"
		# print("Exception:")
		# print str(e)
		# raise e

Toolkit2/exahype/specfiles/specfile1_reader.py

+538 −530

Original line number	Diff line number	Diff line
		@@ -249,7 +249,6 @@ class SpecFile1Reader():
		if term=="pointsources":
		try:
		n_point_sources = int(token_s.split(":")[-1])
		self.log.warning("WARNING: Found 'pointsources' term. Ensure that you specify field 'point_sources' in the generated JSON file.")
		found_token = True
		except:
		raise SpecFile1ParserError("Number of point sources could not be parsed in original ExaHyPE specification file (is: '%s'. expected: 'pointsources':<int>)!" % token_s)
		@@ -335,15 +334,19 @@ class SpecFile1Reader():
		return int(variables.strip())
		else:
		result=[]
		it = re.finditer("(\s\|,\|^)([^\s,]+)\s:\s([^\s,]+)",variables)
		for m in it:
		for token in variables.split(","):
		token_s=token.strip()
		m = re.match("([^\s,]+)\s:\s([^\s,]+)",token_s)
		if m:
		multiplicity = None
		try:
		multiplicity = int(m.group(3))
		multiplicity = int(m.group(2))
		except:
		multiplicity = m.group(3)
		multiplicity = m.group(2)

		result.append(collections.OrderedDict([ ( "name",m.group(2) ), ( "multiplicity", multiplicity ) ]))
		result.append(collections.OrderedDict([ ( "name",m.group(1) ), ( "multiplicity", multiplicity ) ]))
		else:
		raise SpecFile1ParserError("variables\|parameters\|global_observables: Token '%s' does not have structure '<string>:<integer>'." % token_s)
		if result:
		return result
		else:
		@@ -353,15 +356,19 @@ class SpecFile1Reader():
		# TODO
		def map_constants(self,constants):
		result = collections.OrderedDict()
		it = re.finditer("(\s\|,\|^)([^\s,]+)\s:\s([^\s,]+)",constants)
		for m in it:
		for token in constants.split(","):
		token_s=token.strip()
		m = re.match("([^\s,]+)\s:\s([^\s,]+)",token_s)
		if m:
		try:
		result[m.group(2)]=int(m.group(3))
		result[m.group(1)]=int(m.group(2))
		except:
		try:
		result[m.group(2)]=float(m.group(3))
		result[m.group(1)]=float(m.group(2))
		except:
		result[m.group(2)]=m.group(3)
		result[m.group(1)]=m.group(2)
		else:
		raise SpecFile1ParserError("constants\|select: Token '%s' does not have structure '<string>:<integer>'." % token_s)
		if result:
		return result
		else:
		@@ -424,6 +431,7 @@ class SpecFile1Reader():
		# terms
		result, n_point_sources = self.map_kernel_terms(aderdg_kernel_terms)
		solver["aderdg_kernel"]["terms"]=result
		solver["point_sources"]=n_point_sources
		# opts
		solver["aderdg_kernel"].update(self.map_aderdg_kernel_opts(aderdg_kernel_opts))
		# limiter

Toolkit2/exahype/toolkit/controller.py

+18 −7

Original line number	Diff line number	Diff line
		@@ -214,6 +214,7 @@ class Controller():
		except Exception as e:
		self.log.error("Could not read specification file '%s': %s" % (self.specfileName, str(e)))
		self.log.error("In order to fix this problem, please fix the format of your file with the command line flag --format=XXX where XXX is a supported specification file format.")
		if self.debug:
		self.log.exception(e)
		sys.exit(-3)

		@@ -309,7 +310,7 @@ class Controller():
		nVar = helper.count_variables(helper.parse_variables(solver,"variables"))
		nParam = helper.count_variables(helper.parse_variables(solver,"material_parameters"))
		nGlobalObs = helper.count_variables(helper.parse_variables(solver,"global_observables"))
		nPointSources = len(solver.get("point_sources",[]))
		nPointSources = solver["point_sources"] if type(solver.get("point_sources",[])) is int else len(solver.get("point_sources",[]))

		context["numberOfVariables"] = nVar
		context["numberOfMaterialParameters"] = nParam
		@@ -430,8 +431,8 @@ class Controller():
		context["solvers"].append(solverContext)

		context["specfileName"] = self.specfileName
		context["spec_file_as_hex"] = kernelCallsModel.KernelCallsModel.specfile_as_hex(self.spec)
		context["external_parser_command"], context["external_parser_strings"] = kernelCallsModel.KernelCallsModel.get_exahype_external_parser_command()
		context["specFileAsHex"] = self.specfileAsHex(self.spec)
		context["externalParserCommand"] = "%s/%s %s" % ( Configuration.pathToExaHyPERoot, "Toolkit2/toolkit.sh","--format=any --validate-only")
		context["subPaths"] = []
		# todo(JM) optimised kernels subPaths
		# todo(JM) profiler
		@@ -440,6 +441,16 @@ class Controller():

		return context

		def specfileAsHex(self,spec):
		"""
		Given a native python nested dict/list object, dump it as string and then hex-encode that string
		character by character. This is safest way to include something in C++ without dealing with
		character sets or anything.
		"""
		text = json.dumps(spec, sort_keys=True, indent=4)
		hex_tokens = [ "0x%02x"%ord(char) for char in text ] + ["0x00"] # null-terminated list of hex numbers
		return ", ".join(hex_tokens)

		def checkEnvVariable(self):
		"""
		Check environment variables as an ultimate step. Should only be called when actually calling the toolkit,

Toolkit2/exahype/toolkit/models/kernelCallsModel.py

+0 −56

Original line number	Diff line number	Diff line
		@@ -30,62 +30,6 @@ escape = shlex.quote # always escape your strings, please
		formatter_keys = lambda template: [i[1] for i in Formatter().parse(template)]

		class KernelCallsModel(AbstractModelBaseClass):
		@staticmethod
		def obtainRuntimeProperties():
		data = {}
		data['binary'] = sys.executable

		if not data['binary']:
		# well, erhm.
		data['binary'] = "python3"

		data["env"] = {
		'PYTHONPATH': ":".join(sys.path)
		}

		# assuming that no chdir since startup
		data["working_directory"] = escape(os.getcwd())
		data["script"] = os.path.abspath(sys.argv[0])
		data["args"] = [ data['script'], "--format=any", "--validate-only"]

		return data
		#print("Stopping for debugging")
		#import ipdb; ipdb.set_trace()

		@classmethod
		def get_exahype_external_parser_command(cls):
		"""
		Note: Considering KernelCallsImplementation.template, you'll notice that the filename of a specification file
		will be appended to this command.
		"""
		runtimeProperties = cls.obtainRuntimeProperties()
		# the environment string is really "fingers crossed", without proper escaping.
		runtimeProperties["env_string"] = " ".join([ "%s='%s'" % kv for kv in runtimeProperties["env"].items() ])
		runtimeProperties["args_string"] = " ".join(runtimeProperties["args"])
		# stay readable in the Jinja. We could just do command.format(**runtimeProperties),
		# but instead we delegate the insertion until C++ sees it.

		# running it from the toolkit directory would break paths from the ExahyPE executable to it's specfile.
		# To avoid this, we could pass the specfile as stdin to the toolkit, thought. However, this is only the
		# last resort.

		# command = "cd {working_directory} &&
		command = "{env_string} {binary} {args_string}"
		command_strings = {k:v for k,v in runtimeProperties.items() if k in formatter_keys(command) }
		command_template = command.format(**{k:'" + ' + k + ' + "' for k in runtimeProperties.keys() if k in formatter_keys(command) })
		#import ipdb; ipdb.set_trace()
		return (command_template, command_strings)

		@staticmethod
		def specfile_as_hex(spec):
		"""
		Given a native python nested dict/list object, dump it as string and then hex-encode that string
		character by character. This is safest way to include something in C++ without dealing with
		character sets or anything.
		"""
		text = json.dumps(spec, sort_keys=True, indent=4)
		hex_tokens = [ "0x%02x"%ord(char) for char in text ] + ["0x00"] # null-terminated list of hex numbers
		return ", ".join(hex_tokens)

		def generateCode(self):
		return self.render("KernelCallsImplementation.template", "KernelCalls.cpp") #return path to generated file