Skip to content

GitLab

Notice to GitKraken users: A vulnerability has been found in the SSH key generation of GitKraken versions 7.6.0 to 8.0.0 (https://www.gitkraken.com/blog/weak-ssh-key-fix). If you use GitKraken and have generated a SSH key using one of these versions, please remove it both from your local workstation and from your LRZ GitLab profile.

21.10.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit f929321b authored by Edmond Irani Liu's avatar Edmond Irani Liu 🏂
Browse files

update batch processing to handle errors

parent bcf1500b
Hide whitespace changes
Inline Side-by-side
GSMP/motion_automata/automata/HelperFunctions.py
View file @ f929321b
import sys
sys.path.append("../../GSMP/tools/")
sys.path.append("../../GSMP/tools/commonroad-collision-checker")
sys.path.append("../../GSMP/tools/commonroad-road-boundary")
sys.path.append("../../GSMP/motion_automata/vehicle_model")
import os
path_commonroad_search = "../../"
sys.path.append(os.path.join(path_commonroad_search, "GSMP/tools/"))
sys.path.append(os.path.join(path_commonroad_search, "GSMP/tools/commonroad-collision-checker"))
sys.path.append(os.path.join(path_commonroad_search, "GSMP/tools/commonroad-road-boundary"))
sys.path.append(os.path.join(path_commonroad_search, "GSMP/motion_automata/vehicle_model"))
import time
from multiprocessing import Manager, Process
......@@ -56,15 +59,25 @@ def generate_automata(veh_type_id: int):
# step 1
automata = MotionAutomata(veh_type_id)
prefix = '../../GSMP/motion_automata/motion_primitives/'
print("Reading motion primitives...")
if veh_type_id == 1:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-0.91_0.91_SAstep_0.23_T_0.5_Model_FORD_ESCORT.xml')
elif veh_type_id == 2:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-1.066_1.066_SAstep_0.27_T_0.5_Model_BMW320i.xml')
elif veh_type_id == 3:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-1.023_1.023_SAstep_0.26_T_0.5_Model_VW_VANAGON.xml')
try:
prefix = '../../GSMP/motion_automata/motion_primitives/'
if veh_type_id == 1:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-0.91_0.91_SAstep_0.23_T_0.5_Model_FORD_ESCORT.xml')
elif veh_type_id == 2:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-1.066_1.066_SAstep_0.27_T_0.5_Model_BMW320i.xml')
elif veh_type_id == 3:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-1.023_1.023_SAstep_0.26_T_0.5_Model_VW_VANAGON.xml')
except Exception:
prefix = '../../../GSMP/motion_automata/motion_primitives/'
if veh_type_id == 1:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-0.91_0.91_SAstep_0.23_T_0.5_Model_FORD_ESCORT.xml')
elif veh_type_id == 2:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-1.066_1.066_SAstep_0.27_T_0.5_Model_BMW320i.xml')
elif veh_type_id == 3:
automata.readFromXML(prefix + 'V_0.0_20.0_Vstep_1.0_SA_-1.023_1.023_SAstep_0.26_T_0.5_Model_VW_VANAGON.xml')
# step 2
automata.createConnectivityLists()
......
notebooks/batch_processing/batch_processing.ipynb deleted 100644 → 0
View file @ bcf1500b
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Batch Processing"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This is a script that helps you to batch process your code to solve for solution to different scenarios. It is composed of a configuration file (.yaml) and the codes below. Before you run this code, you should make sure the configuration file is set correctly. Here are the explanation for each of the parameters in the configuration file:\n",
"* input_path: the input directory of CommonRoad scenarios that you indend to solve.\n",
"* output_path: the output directory of the solution files.\n",
"* overwrite: the falg to determine whether to overwrite existing solution files.\n",
"* timeout: timeout time for your motion planner, unit in seconds\n",
"* trajectory_planner_path: input directory where the module containing the function to execute your motion planner is located\n",
"* trajectory_planner_module_name: name of the module taht contains the function to execute your motion planner\n",
"* trajectory_planner_function_name: name of the function that executes your motion planner\n",
"* default: the parameters specified under this will be applied to all scenarios. if you wish to specify a different paramter for specific scenarios, simply copy the section and replace 'default' with the id of your scenario.\n",
"* vehicle_model: model of the vehicle, its value could be PM, KS, ST and MB.\n",
"* vehicle_type type of the vehicle, its value could be FORD_ESCORT, BMW_320i and VW_VANAGON.\n",
"* cost_function: identifier of cost function. Please refer to [Cost Functions](https://gitlab.lrz.de/tum-cps/commonroad-cost-functions/blob/master/costFunctions_commonRoad.pdf) for more information."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Helper functions"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import pathlib\n",
"import multiprocessing\n",
"import yaml\n",
"import sys\n",
"import warnings\n",
"\n",
"from commonroad.common.file_reader import CommonRoadFileReader\n",
"from commonroad.common.solution_writer import CommonRoadSolutionWriter, VehicleModel, VehicleType, CostFunction\n",
"\n",
"def parse_vehicle_model(model):\n",
" if model == 'PM':\n",
" cr_model = VehicleModel.PM\n",
" elif model == 'ST':\n",
" cr_model = VehicleModel.ST\n",
" elif model == 'KS':\n",
" cr_model = VehicleModel.KS\n",
" elif model == 'MB':\n",
" cr_model = VehicleModel.MB\n",
" else:\n",
" raise ValueError('Selected vehicle model is not valid: {}.'.format(model))\n",
" return cr_model\n",
"\n",
"\n",
"def parse_vehicle_type(type):\n",
" if type == 'FORD_ESCORT':\n",
" cr_type = VehicleType.FORD_ESCORT\n",
" cr_type_id = 1\n",
" elif type == 'BMW_320i':\n",
" cr_type = VehicleType.BMW_320i\n",
" cr_type_id = 2\n",
" elif type == 'VW_VANAGON':\n",
" cr_type = VehicleType.VW_VANAGON\n",
" cr_type_id = 3\n",
" else:\n",
" raise ValueError('Selected vehicle type is not valid: {}.'.format(type))\n",
" \n",
" return cr_type, cr_type_id\n",
"\n",
"\n",
"def parse_cost_function(cost):\n",
" if cost == 'JB1':\n",
" cr_cost = CostFunction.JB1\n",
" elif cost == 'SA1':\n",
" cr_cost = CostFunction.SA1\n",
" elif cost == 'WX1':\n",
" cr_cost = CostFunction.WX1\n",
" elif cost == 'SM1':\n",
" cr_cost = CostFunction.SM1\n",
" elif cost == 'SM2':\n",
" cr_cost = CostFunction.SM2\n",
" elif cost == 'SM3':\n",
" cr_cost = CostFunction.SM3\n",
" else:\n",
" raise ValueError('Selected cost function is not valid: {}.'.format(cost))\n",
" return cr_cost\n",
"\n",
"\n",
"def call_trajectory_planner(queue, function, scenario, planning_problem_set, vehicle_type_id):\n",
" queue.put(function(scenario, planning_problem_set, vehicle_type_id))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Read configuration"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# open config file\n",
"with open('batch_processing_config.yaml', 'r') as stream:\n",
" try:\n",
" settings = yaml.load(stream)\n",
" except yaml.YAMLError as exc:\n",
" print(exc)\n",
"\n",
"# get planning wrapper function\n",
"sys.path.append(os.getcwd() + os.path.dirname(settings['trajectory_planner_path']))\n",
"module = __import__(settings['trajectory_planner_module_name'])\n",
"function = getattr(module, settings['trajectory_planner_function_name'])\n",
"time_timeout = settings['timeout']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Start Processing"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# iterate through scenarios\n",
"num_files = len(os.listdir(settings['input_path']))\n",
"\n",
"print(\"Total number of files to be processed: {}\".format(num_files))\n",
"print(\"Timeout setting: {} seconds\\n\".format(time_timeout))\n",
" \n",
"count_processed = 0\n",
"for filename in os.listdir(settings['input_path']):\n",
" count_processed += 1\n",
" print(\"File No. {} / {}\".format(count_processed, num_files))\n",
" \n",
" if not filename.endswith('.xml'):\n",
" continue\n",
" \n",
" fullname = os.path.join(settings['input_path'], filename)\n",
"\n",
" print(\"Started processing scenario {}\".format(filename))\n",
" scenario, planning_problem_set = CommonRoadFileReader(fullname).open()\n",
"\n",
" # get settings for each scenario\n",
" if scenario.benchmark_id in settings.keys():\n",
" # specific\n",
" vehicle_model = parse_vehicle_model(settings[scenario.benchmark_id]['vehicle_model'])\n",
" vehicle_type,vehicle_type_id = parse_vehicle_type(settings[scenario.benchmark_id]['vehicle_type'])\n",
" cost_function = parse_cost_function(settings[scenario.benchmark_id]['cost_function'])\n",
" else:\n",
" # default\n",
" vehicle_model = parse_vehicle_model(settings['default']['vehicle_model'])\n",
" vehicle_type, vehicle_type_id = parse_vehicle_type(settings['default']['vehicle_type'])\n",
" cost_function = parse_cost_function(settings['default']['cost_function'])\n",
" \n",
" queue = multiprocessing.Queue()\n",
" # create process, pass in required arguements\n",
" p = multiprocessing.Process(target=call_trajectory_planner, name=\"trajectory_planner\",\n",
" args=(queue, function, scenario, planning_problem_set, vehicle_type_id))\n",
" # start planning\n",
" p.start()\n",
" \n",
" # wait till process ends or skip if timed out\n",
" p.join(timeout=time_timeout)\n",
"\n",
" if p.is_alive():\n",
" print(\"===> Trajectory planner timeout.\")\n",
" p.terminate()\n",
" p.join()\n",
" solution_trajectories = {}\n",
" else:\n",
" print(\"Planning finished.\")\n",
" solution_trajectories = queue.get()\n",
"\n",
" # create path for solutions\n",
" pathlib.Path(settings['output_path']).mkdir(parents=True, exist_ok=True)\n",
"\n",
" error = False\n",
" cr_solution_writer = CommonRoadSolutionWriter(settings['output_path'], \n",
" scenario.benchmark_id, \n",
" scenario.dt,\n",
" vehicle_type, \n",
" vehicle_model, \n",
" cost_function)\n",
" \n",
" # inspect whether all planning problems are solved\n",
" for planning_problem_id, planning_problem in planning_problem_set.planning_problem_dict.items():\n",
" if planning_problem_id not in solution_trajectories.keys():\n",
" print('Solution for planning problem with ID={} is not provided for scenario {}. Solution skipped.'.format(\n",
" planning_problem_id, filename))\n",
" error = True\n",
" break\n",
" else:\n",
" cr_solution_writer.add_solution_trajectory(\n",
" solution_trajectories[planning_problem_id], planning_problem_id)\n",
" if not error:\n",
" cr_solution_writer.write_to_file(overwrite=settings['overwrite'])\n",
"\n",
" print(\"=========================================================\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python (cr37)",
"language": "python",
"name": "cr37"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
notebooks/batch_processing/batch_processing.py deleted 100644 → 0
View file @ bcf1500b
import os
import pathlib
import multiprocessing
import yaml
import sys
import warnings
from commonroad.common.file_reader import CommonRoadFileReader
from commonroad.common.solution_writer import CommonRoadSolutionWriter, VehicleModel, VehicleType, CostFunction
def parse_vehicle_model(model):
if model == 'PM':
cr_model = VehicleModel.PM
elif model == 'ST':
cr_model = VehicleModel.ST
elif model == 'KS':
cr_model = VehicleModel.KS
elif model == 'MB':
cr_model = VehicleModel.MB
else:
raise ValueError('Selected vehicle model is not valid: {}.'.format(model))
return cr_model
def parse_vehicle_type(type):
if type == 'FORD_ESCORT':
cr_type = VehicleType.FORD_ESCORT
cr_type_id = 1
elif type == 'BMW_320i':
cr_type = VehicleType.BMW_320i
cr_type_id = 2
elif type == 'VW_VANAGON':
cr_type = VehicleType.VW_VANAGON
cr_type_id = 3
else:
raise ValueError('Selected vehicle type is not valid: {}.'.format(type))
return cr_type, cr_type_id
def parse_cost_function(cost):
if cost == 'JB1':
cr_cost = CostFunction.JB1
elif cost == 'SA1':
cr_cost = CostFunction.SA1
elif cost == 'WX1':
cr_cost = CostFunction.WX1
elif cost == 'SM1':
cr_cost = CostFunction.SM1
elif cost == 'SM2':
cr_cost = CostFunction.SM2
elif cost == 'SM3':
cr_cost = CostFunction.SM3
else:
raise ValueError('Selected cost function is not valid: {}.'.format(cost))
return cr_cost
def call_trajectory_planner(queue, function, scenario, planning_problem_set, vehicle_type_id):
queue.put(function(scenario, planning_problem_set, vehicle_type_id))
# open config file
with open('batch_processing_config.yaml', 'r') as stream:
try:
settings = yaml.load(stream)
except yaml.YAMLError as exc:
print(exc)
# get planning wrapper function
sys.path.append(os.getcwd() + os.path.dirname(settings['trajectory_planner_path']))
module = __import__(settings['trajectory_planner_module_name'])
function = getattr(module, settings['trajectory_planner_function_name'])
if __name__ == '__main__':
# iterate through scenarios
for filename in os.listdir(settings['input_path']):
if not filename.endswith('.xml'):
continue
fullname = os.path.join(settings['input_path'], filename)
print("Started processing scenario {}".format(filename))
scenario, planning_problem_set = CommonRoadFileReader(fullname).open()
# get settings for each scenario
if scenario.benchmark_id in settings.keys():
# specific
vehicle_model = parse_vehicle_model(settings[scenario.benchmark_id]['vehicle_model'])
vehicle_type,vehicle_type_id = parse_vehicle_type(settings[scenario.benchmark_id]['vehicle_type'])
cost_function = parse_cost_function(settings[scenario.benchmark_id]['cost_function'])
else:
# default
vehicle_model = parse_vehicle_model(settings['default']['vehicle_model'])
vehicle_type, vehicle_type_id = parse_vehicle_type(settings['default']['vehicle_type'])
cost_function = parse_cost_function(settings['default']['cost_function'])
queue = multiprocessing.Queue()
# create process, pass in required arguements
p = multiprocessing.Process(target=call_trajectory_planner, name="trajectory_planner",
args=(queue, function, scenario, planning_problem_set, vehicle_type_id))
# start planning
p.start()
# wait till process ends or skip if timed out
p.join(timeout=settings['timeout'])
if p.is_alive():
print("Trajectory planner timeout.")
p.terminate()
p.join()
solution_trajectories = {}
else:
print("Planning finished.")
solution_trajectories = queue.get()
# create path for solutions
pathlib.Path(settings['output_path']).mkdir(parents=True, exist_ok=True)
error = False
cr_solution_writer = CommonRoadSolutionWriter(settings['output_path'],
scenario.benchmark_id,
scenario.dt,
vehicle_type,
vehicle_model,
cost_function)
# inspect whether all planning problems are solved
for planning_problem_id, planning_problem in planning_problem_set.planning_problem_dict.items():
if planning_problem_id not in solution_trajectories.keys():
warnings.warn('Solution for planning problem with ID={} is not provided for scenario {}. Solution skipped.'.format(
planning_problem_id, filename))
error = True
break
else:
cr_solution_writer.add_solution_trajectory(
solution_trajectories[planning_problem_id], planning_problem_id)
if not error:
cr_solution_writer.write_to_file(overwrite=settings['overwrite'])
print("Solution written.")
print("=========================================================")
\ No newline at end of file
notebooks/batch_processing/batch_processing_config.yaml deleted 100644 → 0
View file @ bcf1500b
# all paths need to be specified relative to file batch_processing.py
# input directory of your intended CommonRoad scenarios
input_path: ../../scenarios/exercise/
# output directory for CommonRoad solution files
output_path: ../../solutions/
# overwrite solution file if it already exists
overwrite: True
# timeout time for trajectory planner [s]
timeout: 120
# name of function which calls the trajectory planner
# Inputs: scenario and planning problem set
# Output: dictionary, key: planning problem ID, value: commonroad trajectory
trajectory_planner_path: ./
trajectory_planner_module_name: batch_search
trajectory_planner_function_name: execute_search_batch
# benchmark evaluation default parameters
# change 'defult' to scenario id to specify exclusive paramters for that scenario
# e.g. change 'default' to 'USA_US101-6_2_T-1'
default:
# vehicle model, e.g., kinematic single-track model
vehicle_model: KS
# vehicle type, e.g, BMW 320i
vehicle_type: BMW_320i
# cost function
cost_function: SM1
notebooks/batch_processing/configuration/batch_processing_config.yaml 0 → 100644
View file @ f929321b
# paths can be either relative or absolute
# input directory of your intended CommonRoad scenarios
input_path: ../../../scenarios/exercise
# output directory of CommonRoad solution files
output_path: ../../../solutions/
# overwrite solution file if it already exists
overwrite: True
# timeout time for motion planner [s]
timeout: 60
# name of the function which calls the motion planner
motion_planner_path: ../target/
motion_planner_module_name: batch_search
motion_planner_function_name: execute_search_batch
# benchmark evaluation parameters
# change 'defult' to scenario id to specify exclusive parameters for that scenario
# e.g. change 'default' to 'USA_US101-6_2_T-1'
default:
# vehicle model, e.g., kinematic single-track model
vehicle_model: KS
# vehicle type, e.g, BMW 320i
vehicle_type: BMW_320i
# cost function
cost_function: SM1
# used to specify id of planner. in this example, we have
# 1 = Greedy Best First Search, 2 = A*, else = Your own planner
planner_id: 1
# planner problem index. for cooperative scenarios: 0, 1, 2, ..., otherwise: 0
planning_problem_idx: 0
# maximum permissible depth of the search tree
max_tree_depth: 100
notebooks/batch_processing/script/.ipynb_checkpoints/batch_processing-checkpoint.ipynb 0 → 100644
View file @ f929321b
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Tutorial: Batch Processing"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This tutorial shows how to solve for solutions to different scenarios in a batch processing fashion. You can find all realted files in [this](https://gitlab.lrz.de/tum-cps/commonroad_io/tree/batch_processing) repository. The main components are:\n",
"* a configuration file (.yaml format),\n",
"* some helper functions,\n",
"* and the script below to call the planer function.\n",
"\n",
"Before proceeding, you should make sure the configuration file is set correctly. Here are the explanation for each of the parameters in the configuration file:\n",
"* **input_path**: input directory of CommonRoad scenarios that you indend to solve.\n",
"* **output_path**: output directory of the solution files.\n",
"* **overwrite**: flag to determine whether the existing solution files should be overwritten.\n",
"* **timeout**: timeout time for your motion planner, unit in seconds\n",
"* **motion_planner_path**: directory where the module containing the function to execute your motion planner is located\n",
"* **motion_planner_module_name**: name of the module that contains the function to execute your motion planner\n",
"* **motion_planner_function_name**: name of the function that executes your motion planner\n",
"* **default**: parameters specified under this section will be applied to all scenarios. If you wish to specify a different paramter for specific scenarios, simply copy the section and replace 'default' with the id of your scenario.\n",
"* **vehicle_model**: model of the vehicle, valid values: **PM, KS, ST and MB**. Please refer to [Vehicle Models](https://gitlab.lrz.de/tum-cps/commonroad-vehicle-models/blob/master/vehicleModels_commonRoad.pdf) for more information.\n",
"* **vehicle_type**: type of the vehicle, valid values: **FORD_ESCORT, BMW_320i and VW_VANAGON**.\n",
"* **cost_function**: identifier of cost function. Please refer to [Cost Functions](https://gitlab.lrz.de/tum-cps/commonroad-cost-functions/blob/master/costFunctions_commonRoad.pdf) for more information.\n",
"* **planner_id**: id of the planner that is used to solve for solutions. in this example, 1 = Greedy Best First Search, 2 = A*, else = Your own planner.\n",
"* **planning_problem_idx**: planner problem index. for cooperative scenarios: 0, 1, 2, ..., otherwise: 0\n",
"* **max_tree_depth**: maximum permissible depth of the search tree\n",
"\n",
"Note: the paths can be either **relative** to this script or **absolute**.\n",
"\n",
"Simply run the following script to start batch processing. The results here is just exemplary, you will see different output as your configuration varies."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# import helper functions\n",
"from helper_functions import *\n",
"\n",
"# specify path to configuration file\n",
"path_file_config = \"../configuration/batch_processing_config.yaml\"\n",
"\n",
"# load configuration file\n",
"configuration = load_configuration(path_file_config)\n",
"\n",
"# get target function\n",
"function_target = get_target_function(configuration)\n",
"\n",
"# get a list of scenario files\n",
"list_files_input = get_input_files(configuration)\n",
"\n",
"# get length of the list and time before timeout\n",
"num_files = len(list_files_input)\n",
"time_timeout = configuration['timeout']\n",
"\n",
"print(\"Total number of files to be processed: {}\".format(num_files))\n",
"print(\"Timeout setting: {} seconds\\n\".format(time_timeout))\n",
"count_processed = 0\n",
"\n",
"# iterate through scenarios\n",
"for file_scenario in list_files_input:\n",
" count_processed += 1\n",
" print(\"Processing file No. {} / {}. Scenario ID: {}\".format(count_processed, num_files, file_scenario[:-4]))\n",
" # parse the scenario file\n",
" result_parse = parse_scenario_file(configuration, file_scenario)\n",
" # execute target function\n",
" solution_trajectories = execute_target_function(function_target, result_parse, time_timeout)\n",
" # save solution file\n",
" save_solution(configuration, solution_trajectories, result_parse)\n",
" print(\"{:=^70s}\".format(''))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python (cr37)",
"language": "python",
"name": "cr37"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}