Merge branch 'master' into dev

Author: iffiX

Jenkinsfile

@@ -10,8 +10,8 @@ pipeline {
     }
     environment {
         PYPI_CREDS = credentials('pypi_username_password')
-        TWINE_USERNAME = '${env.PYPI_CREDS_USR}'
-        TWINE_PASSWORD = '${env.PYPI_CREDS_PSW}'
+        TWINE_USERNAME = "${env.PYPI_CREDS_USR}"
+        TWINE_PASSWORD = "${env.PYPI_CREDS_PSW}"
         // See https://github.com/pytorch/pytorch/issues/37377
         MKL_SERVICE_FORCE_INTEL = "1"
     }

machin/frame/algorithms/trpo.py

@@ -241,7 +241,10 @@ def fvp(v):
 
             # usually 1e-15 is low enough
             if t.allclose(loss_grad, t.zeros_like(loss_grad), atol=1e-15):
-                default_logger.warning("TRPO detects zero gradient.")
+                default_logger.warning(
+                    "TRPO detects zero gradient, update step skipped."
+                )
+                return 0, 0
 
             step_dir = self._conjugate_gradients(
                 fvp,

machin/model/algorithms/trpo.py

@@ -26,7 +26,8 @@ def sample(self, probability: t.tensor, action=None):
             Action log probability tensor of shape ``[batch, 1]``.
         """
         batch_size = probability.shape[0]
-        self.action_param = probability
+        # dx (xlnx) = lnx + 1, x must > 0
+        self.action_param = probability + 1e-6
         dist = Categorical(probs=probability)
         if action is None:
             action = dist.sample()
@@ -41,7 +42,7 @@ def get_kl(self, *args, **kwargs):
         self.forward(*args, **kwargs)
         action_prob1 = self.action_param
         action_prob0 = action_prob1.detach()
-        kl = action_prob0 * (t.log(action_prob0) - t.log(action_prob1))
+        kl = action_prob0 * (t.log(action_prob0 / action_prob1))
         return kl.sum(1, keepdim=True)
 
     def compare_kl(self, params: t.tensor, *args, **kwargs):

test/auto/env/test_openai_gym.py

@@ -1,4 +1,10 @@
-from test.util_platforms import linux_only_forall
+from torch.distributions import Categorical, Normal
+from pytorch_lightning.callbacks import Callback
+from pytorch_lightning.utilities.distributed import ReduceOp
+from machin.parallel.distributed import get_cur_rank
+from machin.parallel.thread import Thread
+from machin.parallel.queue import SimpleQueue, TimeoutError
+from machin.utils.logging import default_logger
 from machin.auto.config import (
     generate_training_config,
     generate_algorithm_config,
@@ -11,25 +17,18 @@
     gym_env_dataset_creator,
     launch,
 )
+from test.util_run_multi import *
+from test.util_fixtures import *
+from test.util_platforms import linux_only_forall
+
 import os
 import pickle
 import os.path as p
 import gym
-import pytest
 import torch as t
 import torch.nn as nn
 import torch.nn.functional as F
 import subprocess as sp
-import multiprocessing as mp
-from test.util_run_multi import *
-from test.util_fixtures import *
-from pytorch_lightning.callbacks import Callback
-from torch.distributions import Categorical, Normal
-from machin.parallel.distributed import get_cur_rank
-from machin.parallel.thread import Thread
-from machin.parallel.queue import SimpleQueue, TimeoutError
-from machin.utils.logging import default_logger
-from pytorch_lightning.utilities.distributed import ReduceOp
 
 linux_only_forall()
 

test/auto/test_launcher.py

@@ -1,7 +1,7 @@
+from unittest import mock
 from machin.frame.algorithms import DQN
 from machin.auto.launcher import Launcher
 from machin.auto.dataset import RLDataset, DatasetResult
-from unittest import mock
 import pytest
 import torch as t
 import torch.nn as nn

test/auto/test_pl_logger.py

@@ -1,6 +1,6 @@
-from machin.auto.pl_logger import LocalMediaLogger
-from pytorch_lightning.loggers.base import DummyExperiment
 from PIL import Image
+from pytorch_lightning.loggers.base import DummyExperiment
+from machin.auto.pl_logger import LocalMediaLogger
 import os
 import matplotlib.pyplot as plt
 

test/conftest.py

@@ -3,5 +3,10 @@ def pytest_addoption(parser):
         "--gpu_device",
         action="store",
         default=None,
-        help="Gpu device descriptor in pytorch",
+        help="GPU device descriptor in pytorch",
+    )
+    parser.addoption(
+        "--multiprocess_method",
+        default="forkserver",
+        help="spawn or forkserver, default is forkserver",
     )

test/data/all.py

@@ -1,7 +1,7 @@
-import os
-import re
 from . import generators, ROOT
 from .archive import Archive
+import os
+import re
 
 
 def first(iterable, condition=lambda x: True):

test/data/archive.py

@@ -1,7 +1,7 @@
-import torch as t
 import os
 import re
 import datetime
+import torch as t
 
 
 class Archive:

test/data/generators/generate_gail.py

@@ -1,6 +1,6 @@
+from torch.distributions import Categorical
 from machin.frame.algorithms import PPO
 from machin.utils.logging import default_logger as logger
-from torch.distributions import Categorical
 from test.data import ROOT
 from test.data.archive import Archive, get_time_string
 import os

test/env/wrappers/test_openai_gym.py

@@ -6,17 +6,16 @@
 
 Submit us a issue if you have found any problem.
 """
-from test.util_platforms import linux_only_forall
-
-linux_only_forall()
-
+from random import choice, sample
 from machin.env.wrappers import openai_gym
 from machin.utils.logging import default_logger
-from random import choice, sample
+from test.util_platforms import linux_only_forall
+
 import pytest
 import gym
 import numpy as np
 
+linux_only_forall()
 ENV_NUM = 2
 SAMPLE_NUM = 2
 WORKER_NUM = 2
@@ -26,69 +25,15 @@ def mock_action(action_space: gym.spaces.Space):
     return action_space.sample()
 
 
-def prepare_envs(env_list):
-    for env in env_list:
-        env.reset()
-
-
-def should_skip(spec):
-    # From gym/envs/tests/spec_list.py
-    # Used to check whether a gym environment should be tested.
-
-    # We skip tests for envs that require dependencies or are otherwise
-    # troublesome to run frequently
-    ep = spec.entry_point
-
-    # No need to test unittest environments
-    if ep.startswith("gym.envs.unittest"):
-        return True
-
-    # Skip not renderable tests
-    if ep.startswith("gym.envs.algorithmic") or ep.startswith("gym.envs.toy_text"):
-        return True
-
-    # Skip mujoco tests
-    if ep.startswith("gym.envs.mujoco") or ep.startswith("gym.envs.robotics:"):
-        return True
-
-    # Skip atari tests
-    if ep.startswith("gym.envs.atari"):
-        return True
-
-    # Skip other tests
-    if "GoEnv" in ep or "HexEnv" in ep or "CarRacing" in ep:
-        return True
-
-    # Conditionally skip box2d tests
-    try:
-        import Box2D
-    except ImportError:
-        if ep.startswith("gym.envs.box2d"):
-            return True
-
-    return False
-
-
 @pytest.fixture(scope="module", autouse=True)
 def envs():
-    all_envs = []
-    env_map = {}
-    # Find the newest version of non-skippable environments.
-    for env_raw_name, env_spec in gym.envs.registry.env_specs.items():
-        if not should_skip(env_spec):
-            env_name, env_version = env_raw_name.split("-v")
-            if env_name not in env_version or int(env_version) > env_map[env_name]:
-                env_map[env_name] = int(env_version)
+    names = ["CartPole-v0"]
+    creators = []
 
     # Create environments.
-    for env_name, env_version in env_map.items():
-        env_name = env_name + "-v" + str(env_version)
-        default_logger.info(f"OpenAI gym {env_name} added")
-        all_envs.append([lambda *_: gym.make(env_name) for _ in range(ENV_NUM)])
-    default_logger.info(
-        "{} OpenAI gym environments to be tested.".format(len(all_envs))
-    )
-    return all_envs
+    for name in names:
+        creators.append([lambda *_: gym.make(name) for _ in range(ENV_NUM)])
+    return names, creators
 
 
 class TestParallelWrapperDummy:
@@ -104,8 +49,9 @@ class TestParallelWrapperDummy:
 
     @pytest.mark.parametrize("idx,reset_num", param_test_reset)
     def test_reset(self, envs, idx, reset_num):
-        for env_list in envs:
-            dummy_wrapper = openai_gym.ParallelWrapperDummy(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            dummy_wrapper = openai_gym.ParallelWrapperDummy(creators)
             obsrvs = dummy_wrapper.reset(idx)
             dummy_wrapper.close()
 
@@ -129,8 +75,9 @@ def test_reset(self, envs, idx, reset_num):
 
     @pytest.mark.parametrize("idx,act_num", param_test_step)
     def test_step(self, envs, idx, act_num):
-        for env_list in envs:
-            dummy_wrapper = openai_gym.ParallelWrapperDummy(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            dummy_wrapper = openai_gym.ParallelWrapperDummy(creators)
             action = [mock_action(dummy_wrapper.action_space) for _ in range(act_num)]
             dummy_wrapper.reset(idx)
             obsrvs, reward, terminal, info = dummy_wrapper.step(action, idx)
@@ -159,8 +106,9 @@ def test_step(self, envs, idx, act_num):
 
     @pytest.mark.parametrize("idx", param_test_seed)
     def test_seed(self, envs, idx):
-        for env_list in envs:
-            dummy_wrapper = openai_gym.ParallelWrapperDummy(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            dummy_wrapper = openai_gym.ParallelWrapperDummy(creators)
             seeds = dummy_wrapper.seed()
             dummy_wrapper.close()
             assert len(seeds) == ENV_NUM
@@ -177,8 +125,9 @@ def test_seed(self, envs, idx):
 
     @pytest.mark.parametrize("idx,render_num", param_test_render)
     def test_render(self, envs, idx, render_num):
-        for env_list in envs:
-            dummy_wrapper = openai_gym.ParallelWrapperDummy(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            dummy_wrapper = openai_gym.ParallelWrapperDummy(creators)
             dummy_wrapper.reset(idx)
             rendered = dummy_wrapper.render(idx)
             dummy_wrapper.close()
@@ -190,16 +139,18 @@ def test_render(self, envs, idx, render_num):
     # Test for ParallelWrapperDummy.close
     ########################################################################
     def test_close(self, envs):
-        for env_list in envs:
-            dummy_wrapper = openai_gym.ParallelWrapperDummy(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            dummy_wrapper = openai_gym.ParallelWrapperDummy(creators)
             dummy_wrapper.close()
 
     ########################################################################
     # Test for ParallelWrapperDummy.active
     ########################################################################
     def test_active(self, envs):
-        for env_list in envs:
-            dummy_wrapper = openai_gym.ParallelWrapperDummy(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            dummy_wrapper = openai_gym.ParallelWrapperDummy(creators)
             dummy_wrapper.reset()
             active = dummy_wrapper.active()
             dummy_wrapper.close()
@@ -209,7 +160,7 @@ def test_active(self, envs):
     # Test for ParallelWrapperDummy.size
     ########################################################################
     def test_size(self, envs):
-        dummy_wrapper = openai_gym.ParallelWrapperDummy(envs[0])
+        dummy_wrapper = openai_gym.ParallelWrapperDummy(envs[1][0])
         assert dummy_wrapper.size() == ENV_NUM
         dummy_wrapper.close()
 
@@ -227,8 +178,9 @@ class TestParallelWrapperSubProc:
 
     @pytest.mark.parametrize("idx,reset_num", param_test_reset)
     def test_reset(self, envs, idx, reset_num):
-        for env_list in envs:
-            subproc_wrapper = openai_gym.ParallelWrapperSubProc(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            subproc_wrapper = openai_gym.ParallelWrapperSubProc(creators)
             obsrvs = subproc_wrapper.reset(idx)
             subproc_wrapper.close()
 
@@ -252,8 +204,9 @@ def test_reset(self, envs, idx, reset_num):
 
     @pytest.mark.parametrize("idx,act_num", param_test_step)
     def test_step(self, envs, idx, act_num):
-        for env_list in envs:
-            subproc_wrapper = openai_gym.ParallelWrapperSubProc(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            subproc_wrapper = openai_gym.ParallelWrapperSubProc(creators)
             action = [mock_action(subproc_wrapper.action_space) for _ in range(act_num)]
             subproc_wrapper.reset(idx)
             obsrvs, reward, terminal, info = subproc_wrapper.step(action, idx)
@@ -282,8 +235,9 @@ def test_step(self, envs, idx, act_num):
 
     @pytest.mark.parametrize("idx", param_test_seed)
     def test_seed(self, envs, idx):
-        for env_list in envs:
-            subproc_wrapper = openai_gym.ParallelWrapperSubProc(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            subproc_wrapper = openai_gym.ParallelWrapperSubProc(creators)
             seeds = subproc_wrapper.seed()
             subproc_wrapper.close()
             assert len(seeds) == ENV_NUM
@@ -300,8 +254,9 @@ def test_seed(self, envs, idx):
 
     @pytest.mark.parametrize("idx,render_num", param_test_render)
     def test_render(self, envs, idx, render_num):
-        for env_list in envs:
-            subproc_wrapper = openai_gym.ParallelWrapperSubProc(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            subproc_wrapper = openai_gym.ParallelWrapperSubProc(creators)
             subproc_wrapper.reset(idx)
             rendered = subproc_wrapper.render(idx)
             subproc_wrapper.close()
@@ -313,22 +268,24 @@ def test_render(self, envs, idx, render_num):
     # Test for ParallelWrapperSubProc.close
     ########################################################################
     def test_close(self, envs):
-        for env_list in envs:
-            subproc_wrapper = openai_gym.ParallelWrapperSubProc(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            subproc_wrapper = openai_gym.ParallelWrapperSubProc(creators)
             subproc_wrapper.close()
 
     ########################################################################
     # Test for ParallelWrapperSubProc.active
     ########################################################################
     def test_active(self, envs):
-        for env_list in envs:
-            subproc_wrapper = openai_gym.ParallelWrapperSubProc(env_list)
+        for name, creators in zip(*envs):
+            default_logger.info(f"Testing on env {name}")
+            subproc_wrapper = openai_gym.ParallelWrapperSubProc(creators)
             subproc_wrapper.reset()
             active = subproc_wrapper.active()
             subproc_wrapper.close()
             assert len(active) == ENV_NUM
 
     def test_size(self, envs):
-        subproc_wrapper = openai_gym.ParallelWrapperSubProc(envs[0])
+        subproc_wrapper = openai_gym.ParallelWrapperSubProc(envs[1][0])
         assert subproc_wrapper.size() == ENV_NUM
         subproc_wrapper.close()