DHParser/server.py: bug fixes for slow systems and Python 3.5

DHParser/compile.py

@@ -373,7 +373,7 @@ def process_tree(tp: TreeProcessor, tree: RootNode) -> RootNode:
     """Process a tree with the tree-processor `tp` only if no fatal error
     has occurred so far. Process, but catch any Python exceptions, in case
     any normal errors have occurred earlier in the processing pipeline.
-    Don't catch Python-exceptions if not errors have occurred earlier.
+    Don't catch Python-exceptions if no errors have occurred earlier.
 
     This behaviour is based on the assumption that given any non-fatal
     errors have occurred earlier, the tree passed through the pipeline
@@ -407,4 +407,4 @@ def process_tree(tp: TreeProcessor, tree: RootNode) -> RootNode:
 
 
 # TODO: Verify compiler against grammar, i.e. make sure that for all on_X()-methods, `X` is the name of a parser
-# TODO: AST validation against an ASDSL-Specification
\ No newline at end of file
+# TODO: AST validation against an ASDSL-Specification

DHParser/server.py

@@ -42,7 +42,7 @@ For JSON see:
 """
 
 import asyncio
-from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor, CancelledError
+from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
 from concurrent.futures.process import BrokenProcessPool
 from functools import partial
 import json
@@ -195,7 +195,7 @@ def asyncio_run(coroutine: Coroutine, loop=None) -> Any:
 
 
 async def asyncio_connect(host: str = USE_DEFAULT_HOST, port: int = USE_DEFAULT_PORT,
-                    retry_timeout: float = 1.0):
+                    retry_timeout: float = 5.0):
     """
     Backwards compatible version of Python3.8's `asyncio.connect()`, with the
     variant that it returns a reader, writer pair instead of just one stream.
@@ -398,7 +398,7 @@ class Server:
             response = JSONRPC_HEADER.format(length=len(response)).encode() + response
         append_log(self.log_file, 'RESPONSE: ', response.decode(), '\n\n', echo=self.echo_log)
         # print('returned: ', response)
-        if sys.version_info >= (3, 8):
+        if sys.version_info >= (3, 9):
             await writer.write(response)
         else:
             writer.write(response)
@@ -566,8 +566,7 @@ class Server:
                         data = data[:k + l]
 
             append_log(self.log_file, 'RECEIVE: ', data.decode(), '\n', echo=self.echo_log)
-            # if data:
-            #     print(data)
+
             if not data and reader.at_eof():
                 break
 
@@ -632,7 +631,7 @@ class Server:
             self.exit_connection = False  # reset flag
 
         if self.kill_switch:
-            # TODO: terminate processes and threads! Is this needed??
+            # TODO: terminate processes and threads! Is this needed?
             self.stage.value = SERVER_TERMINATE
             if sys.version_info >= (3, 7):
                 await writer.wait_closed()
@@ -731,10 +730,9 @@ class Server:
                 self.serve_py35(host, port, loop)
         except KeyboardInterrupt:
             print("KeyboardInterrupt")
-        except CancelledError:
-            pass
-        self.pp_executor = None
-        self.tp_executor = None
+        except asyncio.CancelledError:
+            if self.stage.value != SERVER_TERMINATE:
+                raise
         # self.server_messages.put(SERVER_OFFLINE)
         self.stage.value = SERVER_OFFLINE
 
@@ -754,9 +752,8 @@ if len(path) < 20:
 
 def run_server(host, port):
     from DHParser.server import asyncio_run, Server, stop_server, has_server_stopped
-    stop_server(host, port)
-    # asyncio_run(has_server_stopped(host, port))
-    server = Server({PARAMETERS})
+    stop_server(host, port, 1.0)
+    server = Server({PARAMETERS})   
     server.run_server(host, port)
 
 if __name__ == '__main__':
@@ -776,28 +773,36 @@ def spawn_server(host: str = USE_DEFAULT_HOST,
     Start DHParser-Server in a separate process and return.
     Useful for writing test code.
     """
+    async def wait_for_connection(host, port):
+        reader, writer = await asyncio_connect(host, port)  # wait until server online
+        writer.close()
+        # if sys.version_info >= (3, 7):
+        #     await writer.wait_closed()
+
     global python_interpreter_name_cached
     host, port = substitute_default_host_and_port(host, port)
     null_device = " >/dev/null" if platform.system() != "Windows" else " > NUL"
     if python_interpreter_name_cached:
         interpreter = python_interpreter_name_cached
     else:
-        # interpreter = 'python3' if os.system('python3 -V' + null_device) == 0 else 'python'
-        interpreter = '/home/eckhart/.local/bin/python3.5'
+        interpreter = 'python3' if os.system('python3 -V' + null_device) == 0 else 'python'
+        # interpreter = '/home/eckhart/.local/bin/python3.8'
         # interpreter = "pypy3"
         python_interpreter_name_cached = interpreter
     run_server_script = RUN_SERVER_SCRIPT_TEMPLATE.format(
         HOST=host, PORT=port, INITIALIZATION=initialization,
         PARAMETERS=parameters, IMPORT_PATH=import_path.replace('\\', '\\\\'))
     subprocess.Popen([interpreter, '-c', run_server_script])
+    asyncio_run(wait_for_connection(host, port))
 
 
 async def has_server_stopped(host: str = USE_DEFAULT_HOST,
                              port: int = USE_DEFAULT_PORT,
-                             timeout: float = 1.0) -> bool:
+                             timeout: float = 5.0) -> bool:
     """
     Returns True, if no server is running or any server that is running
-    has stopped within the given timeout.
+    has stopped within the given timeout. Returns False, if server has
+    not stopped and is still running.
     """
     host, port = substitute_default_host_and_port(host, port)
     delay = timeout / 2**7  if timeout > 0.0 else timeout - 0.001
@@ -813,7 +818,6 @@ async def has_server_stopped(host: str = USE_DEFAULT_HOST,
                 delay = timeout  # exit while loop
         return False
     except ConnectionRefusedError:
-
         return True
 
 
@@ -832,7 +836,9 @@ def stop_server(host: str = USE_DEFAULT_HOST, port: int = USE_DEFAULT_PORT,
             if sys.version_info >= (3, 7):
                 await writer.wait_closed()
             if timeout > 0.0:
-                await has_server_stopped(host, port)
+                if not await has_server_stopped(host, port, timeout):
+                    raise AssertionError('Could not stop server on host %s port %i '
+                                         'within timeout %f !' % (host, port, timeout))
         except ConnectionRefusedError as error:
             return error
         except ConnectionResetError as error:
@@ -884,6 +890,3 @@ def gen_lsp_table(lsp_funcs_or_instance: Union[Sequence[Callable], Iterator[Call
             name = name.replace('_', '/').replace('S/', '$/')
         rpc_table[name] = func
     return rpc_table
-
-
-

DHParser/testing.py

@@ -763,6 +763,14 @@ def runner(tests, namespace):
     "Test" and methods, the name of which starts with "test" contained
     in such classes or functions, the name of which starts with "test".
 
+    if `tests` is either the empty string or an empty sequence, runner
+    checks sys.argv for specified tests. In case sys.argv[0] (i.e. the
+    script's file name) starts with 'test' any argument in sys.argv[1:]
+    (i.e. the rest of the command line) that starts with 'test' or
+    'Test' is considered the name of a test function or test method
+    (of a test-class) that shall be run. Test-Methods are specified in
+    the form: class_name.method.name e.g. "TestServer.test_connection".
+
     Args:
         tests: String or list of strings with the names of tests to
             run. If empty, runner searches by itself all objects the
@@ -794,7 +802,9 @@ def runner(tests, namespace):
             tests = tests.split(' ')
         assert all(test.lower().startswith('test') for test in tests)
     else:
-        tests = [name for name in sys.argv[1:] if name.lower().startswith('test')]
+        tests = []
+        if sys.argv[0].lower().startswith('test'):
+            tests = [name for name in sys.argv[1:] if name.lower().startswith('test')]
         if not tests:
             tests = [name for name in namespace.keys() if name.lower().startswith('test')]
 

examples/json/jsonServer.py

@@ -263,9 +263,9 @@ def assert_if(cond: bool, message: str):
 
 
 if __name__ == "__main__":
-    print(os.getcwd())
+    # print(os.getcwd())
     sys.path.append(os.path.abspath(os.path.join(scriptpath, '..', '..')))
-    print(sys.path)
+    # print(sys.path)
 
     host, port = '', -1
 

test/test_server.py

@@ -58,16 +58,13 @@ def long_running(duration: str) -> str:
 def send_request(request: str, expect_response: bool = True) -> str:
     response = ''
     async def send(request):
-        try:
-            nonlocal response
-            reader, writer = await asyncio_connect('127.0.0.1', TEST_PORT)
-            writer.write(request.encode())
-            if expect_response:
-                response = (await reader.read(8192)).decode()
-            writer.close()
-            if sys.version_info >= (3, 7):  await writer.wait_closed()
-        except ConnectionRefusedError:
-            pass
+        nonlocal response
+        reader, writer = await asyncio_connect('127.0.0.1', TEST_PORT)
+        writer.write(request.encode())
+        if expect_response:
+            response = (await reader.read(8192)).decode()
+        writer.close()
+        if sys.version_info >= (3, 7):  await writer.wait_closed()
 
     asyncio_run(send(request))
     return response
@@ -114,7 +111,7 @@ class TestServer:
         finally:
             stop_server('127.0.0.1', TEST_PORT)
 
-    def test_indentify(self):
+    def test_identify(self):
         """Test server's 'identify/'-command."""
         async def send_request(request):
             reader, writer = await asyncio_connect('127.0.0.1', TEST_PORT)
@@ -385,7 +382,7 @@ class TestLanguageServer:
         response = send_request(json_rpc('exit', {}))
         assert response == '', response
 
-    def test_initializion_sequence(self):
+    def test_initialization_sequence(self):
         self.start_server()
         async def initialization_sequence():
             reader, writer = await asyncio_connect('127.0.0.1', TEST_PORT)
@@ -395,7 +392,7 @@ class TestLanguageServer:
                                    'capabilities': {}}).encode())
             response = (await reader.read(8192)).decode()
             i = response.find('{')
-            print(len(response), response)
+            # print(len(response), response)
             res = json.loads(response[i:])
             assert 'result' in res and 'capabilities' in res['result'], str(res)
 
@@ -404,10 +401,16 @@ class TestLanguageServer:
             writer.write(json_rpc('custom', {'test': 1}).encode())
             response = (await reader.read(8192)).decode()
             assert response.find('test') >= 0
+            writer.close()
+            if sys.version_info >= (3, 7):  await writer.wait_closed()
 
         asyncio_run(initialization_sequence())
 
 
 if __name__ == "__main__":
+    if "--killserver" in sys.argv:
+        result = stop_server('127.0.0.1', TEST_PORT)
+        print('server stopped' if result is None else "server wasn't running")
+        sys.exit(0)
     from DHParser.testing import runner
     runner("", globals())