Explicitly set multiprocessing to use spawn not fork.

kurtschelfthout · kurtschelfthout · commit d357956f0e6b · 2022-01-03T17:35:27.000Z
Enable all db and flow tests. On linux, multiprocessing's default is fork, which causes gRPC to fail because its default polling mechanism is epoll. See grpc/grpc#23796
diff --git a/.github/workflows/check and test.yml b/.github/workflows/check and test.yml
@@ -59,5 +59,4 @@ jobs:
         PYTHONDEVMODE: 1 # see https://docs.python.org/3/library/devmode.html#devmode
         GRPC_VERBOSITY: debug
       run: |
-        poetry run pytest tests/test_meadowdb tests/test_meadowflow/test_event_log.py tests/test_meadowflow/test_time_events.py
-    
+        poetry run pytest tests/test_meadowdb tests/test_meadowflow
diff --git a/src/meadowflow/local_job_runner.py b/src/meadowflow/local_job_runner.py
@@ -2,6 +2,7 @@
 TODO capabilities
 TODO checking for and restarting requested but not running jobs
 """
+import multiprocessing
 import traceback
 from concurrent.futures.process import BrokenProcessPool
 from typing import Dict, Iterable, Callable, Any, Tuple, Sequence, Optional
@@ -41,7 +42,8 @@ class LocalJobRunner(JobRunner):
 
     def __init__(self, event_log: EventLog):
         self._running: Dict[str, Future] = {}
-        self._executor = ProcessPoolExecutor(max_workers=5)
+        ctx = multiprocessing.get_context("spawn")
+        self._executor = ProcessPoolExecutor(max_workers=5, mp_context=ctx)
         self._event_log = event_log
 
     async def run(
diff --git a/src/meadowflow/server/server_main.py b/src/meadowflow/server/server_main.py
@@ -38,7 +38,8 @@ def main_in_child_process(
     logs, etc. If there's an existing server already running, the child process will
     just die immediately without doing anything.
     """
-    server_process = multiprocessing.Process(
+    ctx = multiprocessing.get_context("spawn")
+    server_process = ctx.Process(
         target=main, args=(host, port, job_runner_poll_delay_seconds)
     )
     server_process.start()
diff --git a/src/meadowgrid/coordinator_main.py b/src/meadowgrid/coordinator_main.py
@@ -41,9 +41,8 @@ def main_in_child_process(
     it, see logs, etc. If there's an existing server already running, the child process
     will just die immediately without doing anything.
     """
-    server_process = multiprocessing.Process(
-        target=main, args=(host, port, meadowflow_address)
-    )
+    ctx = multiprocessing.get_context("spawn")
+    server_process = ctx.Process(target=main, args=(host, port, meadowflow_address))
     server_process.start()
 
     try:
diff --git a/src/meadowgrid/job_worker_main.py b/src/meadowgrid/job_worker_main.py
@@ -57,7 +57,8 @@ def main_in_child_process(
     see logs, etc. If there's an existing worker already running, the child process will
     just die immediately without doing anything.
     """
-    server_process = multiprocessing.Process(
+    ctx = multiprocessing.get_context("spawn")
+    server_process = ctx.Process(
         target=main,
         args=(working_folder, available_resources, coordinator_host, coordinator_port),
     )
