Add platform testing script.

TODO: fix pool support in future python versions (>3.7)

Author: iffiX

Jenkinsfile

@@ -52,7 +52,7 @@ pipeline {
                             sh 'pip install .'
                             sh 'pip install ./test_lib/multiagent-particle-envs/'
                             sh 'pip install "gym[atari, box2d, classic_control]"'
-                            sh 'pip install mock pytest==6.0.0 pytest-cov==2.10.0 allure-pytest==2.8.16 pytest-xvfb==2.0.0 pytest-html==1.22.1 pytest-repeat==0.8.0'
+                            sh 'pip install mock pytest==6.0.1 pytest-cov==2.10.0 allure-pytest==2.8.16 pytest-xvfb==2.0.0 pytest-html==1.22.1 pytest-repeat==0.8.0'
                             // This line must be included, otherwise matplotlib will
                             // segfault when it tries to build the font cache.
                             sh "python3 -c 'import matplotlib.pyplot as plt'"

README.md

@@ -140,12 +140,16 @@ conda activate some_env
 pip install machin
 ```
 
-**Note**: Currently only a fraction of all functions is supported on Windows, to test whether
-the code is running correctly, you can run the corresponding test script in the root directory:
+**Note**: Currently only a fraction of all functions is supported on platforms other than linux
+(mainly distributed algorithms), to test whether the code is running correctly, you can run the 
+corresponding test script for your platform in the root directory:
 ```
 run_win_test.bat
-run_lunux_test.sh
+run_linux_test.sh
+run_macos_test.sh
 ```
+Some errors may occur due to incorrect setup of libraries, make sure you have installed `graphviz` 
+etc.
 
 ### Contributing
 ---

machin/parallel/pool.py

@@ -1,4 +1,6 @@
 import os
+import sys
+import warnings
 import threading
 import multiprocessing.pool as pool
 from multiprocessing.pool import TERMINATE
@@ -8,6 +10,9 @@
 from .pickle import dumps, loads
 from .queue import SimpleQueue, MultiP2PQueue
 
+# TODO: fix pool support pools in python 3.8 etc.
+# Since future implementations are different.
+
 
 def proxy_caller(*input_):
     """
@@ -53,6 +58,10 @@ class Pool(pool.Pool):
      2. Ability to select the tensor serialize scheme.
     """
 
+    def Process(self, *args, **kwds):
+        # Prevent future versions of pool from changing this implementation.
+        return self._ctx.Process(*args, **kwds)
+
     def __init__(
         self,
         processes=None,
@@ -101,19 +110,27 @@ def __init__(
             processes = os.cpu_count() or 1
         if processes < 1:
             raise ValueError("Number of processes must be at least 1")
-        self._processes = processes
-
-        self._is_recursive = is_recursive
-        self._is_daemon = is_daemon
-        self._is_copy_tensor = is_copy_tensor
-        self._caller = proxy_caller
 
         context = get_context("spawn")
-        if not is_copy_tensor:
+        if sys.platform.startswith("linux") and not is_copy_tensor:
             if share_method not in ("cpu", "cuda"):
                 raise RuntimeError(f'Invalid share method: "{share_method}"')
             if share_method == "cpu":
                 context = get_context("fork")
+        else:
+            warnings.warn(
+                "Sharing but not copying a tensor is not supported "
+                "on platforms other than linux."
+            )
+            is_copy_tensor = True
+
+        self._ctx = context
+        self._processes = processes
+
+        self._is_recursive = is_recursive
+        self._is_daemon = is_daemon
+        self._is_copy_tensor = is_copy_tensor
+        self._caller = proxy_caller
 
         super().__init__(
             processes=processes,

run_linux_test.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
+# if you are using any non-standard python 3 installation (>= 3.6)
+# please replace first three python with your absolute path
 python --version
 python -m pip install virtualenv
-virtualenv venv
+python -m virtualenv venv
 source venv/bin/activate
 pip install .
 pip install ./test_lib/multiagent-particle-envs/
-pip install mock pytest==6.0.0 pytest-html==1.22.1 pytest-repeat==0.8.0
-python -m pytest -s --assert=plain -k "not full_train" --html=test_results/test_api.html --self-contained-html ./test/
+pip install mock pytest==6.0.1 pytest-html==1.22.1 pytest-repeat==0.8.0
+python -m pytest -s --assert=plain -k "not full_train" --html=test_api.html --self-contained-html ./test/

run_macos_test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# if you are using any non-standard python 3 installation (>= 3.6)
+# please replace first three python with your absolute path
+python --version
+python -m pip install virtualenv
+python -m virtualenv venv
+source venv/bin/activate
+pip install .
+pip install ./test_lib/multiagent-particle-envs/
+pip install mock pytest==6.0.1 pytest-html==1.22.1 pytest-repeat==0.8.0
+python -m pytest -s --assert=plain -k "not full_train" --html=test_api.html --self-contained-html ./test/

run_win_test.bat

@@ -1,9 +1,13 @@
 @ECHO OFF
+REM if you are using any non-standard python 3 installation (>= 3.6)
+REM please replace first three python with your absolute path
+
 python --version
 python -m pip install virtualenv
-virtualenv venv
-.\venv\Scripts\activate
+python -m virtualenv venv
+call .\venv\Scripts\activate
 pip install .
 pip install .\test_lib\multiagent-particle-envs\
-pip install mock pytest==6.0.0 pytest-html==1.22.1 pytest-repeat==0.8.0
-python -m pytest -s --assert=plain -k "not full_train" --html=test_results\test_api.html --self-contained-html .\test\
+pip install mock pytest==6.0.1 pytest-html==1.22.1 pytest-repeat==0.8.0
+python -m pytest -s --assert=plain -k "not full_train" --html=test_api.html --self-contained-html .\test\
+pause

test/parallel/test_pickle.py

@@ -2,6 +2,7 @@
 from machin.parallel.process import Process
 import torch as t
 from multiprocessing import Pipe, get_context
+from test.util_platforms import linux_only
 
 
 def subproc_test_dumps_copy_tensor(pipe):
@@ -25,6 +26,7 @@ def subproc_test_dumps_not_copy_tensor(pipe):
     pipe.send(dumps(tensor, copy_tensor=False))
 
 
+@linux_only
 def test_dumps_not_copy_tensor():
     pipe_0, pipe_1 = Pipe(duplex=True)
     ctx = get_context("fork")
@@ -49,6 +51,7 @@ def local_func():
     pipe.send(dumps(local_func, copy_tensor=False))
 
 
+@linux_only
 def test_dumps_local_func():
     pipe_0, pipe_1 = Pipe(duplex=True)
     ctx = get_context("fork")

test/parallel/test_pool.py

@@ -8,6 +8,7 @@
 import torch as t
 
 from test.util_fixtures import *
+from test.util_fixtures import linux_only
 
 # enable pool logging
 # log_to_stderr(DEBUG)
@@ -163,6 +164,7 @@ def test_starmap_async(self):
     #     pool.join()
     #     logger.info("Pool 2 joined.")
 
+    @linux_only
     def test_cpu_shared_tensor(self):
         x = [t.ones([10]) * i for i in range(5)]
         for xx in x: