init v0.2.0

init v0.2.0

Author: huangshiyu13

README.md

@@ -28,7 +28,7 @@
 [![Embark](https://img.shields.io/badge/discord-OpenRL-%237289da.svg?logo=discord)](https://discord.gg/qMbVT2qBhr)
 [![slack badge](https://img.shields.io/badge/Slack-join-blueviolet?logo=slack&amp)](https://join.slack.com/t/openrlhq/shared_invite/zt-1tqwpvthd-Eeh0IxQ~DIaGqYXoW2IUQg)
 
-OpenRL-v0.1.10 is updated on Oct 27, 2023
+OpenRL-v0.2.0 is updated on Dec 20, 2023
 
 The main branch is the latest version of OpenRL, which is under active development. If you just want to have a try with
 OpenRL, you can switch to the stable branch.

examples/envpool/envpool_wrappers.py

@@ -9,8 +9,7 @@
 from packaging import version
 from stable_baselines3.common.vec_env import VecEnvWrapper as BaseWrapper
 from stable_baselines3.common.vec_env import VecMonitor
-from stable_baselines3.common.vec_env.base_vec_env import (VecEnvObs,
-                                                           VecEnvStepReturn)
+from stable_baselines3.common.vec_env.base_vec_env import VecEnvObs, VecEnvStepReturn
 
 is_legacy_gym = version.parse(gym.__version__) < version.parse("0.26.0")
 
@@ -114,9 +113,9 @@ def __init__(
 
         if is_wrapped_with_monitor:
             warnings.warn(
-                "The environment is already wrapped with a `Monitor` wrapper"
-                "but you are wrapping it with a `VecMonitor` wrapper, the `Monitor` statistics will be"
-                "overwritten by the `VecMonitor` ones.",
+                "The environment is already wrapped with a `Monitor` wrapperbut you are"
+                " wrapping it with a `VecMonitor` wrapper, the `Monitor` statistics"
+                " will beoverwritten by the `VecMonitor` ones.",
                 UserWarning,
             )
 

examples/envpool/make_env.py

@@ -5,9 +5,12 @@
 import envpool
 from gymnasium import Env
 
-
-from openrl.envs.vec_env import (AsyncVectorEnv, RewardWrapper,
-                                 SyncVectorEnv, VecMonitorWrapper)
+from openrl.envs.vec_env import (
+    AsyncVectorEnv,
+    RewardWrapper,
+    SyncVectorEnv,
+    VecMonitorWrapper,
+)
 from openrl.envs.vec_env.vec_info import VecInfoFactory
 from openrl.envs.wrappers.base_wrapper import BaseWrapper
 from openrl.rewards import RewardFactory
@@ -76,7 +79,7 @@ def make_envpool_envs(
     assert kwargs.get("env_type") in ["gym", "dm", "gymnasium"]
     kwargs["envpool"] = True
 
-    if 'env_wrappers' in kwargs:
+    if "env_wrappers" in kwargs:
         env_wrappers = kwargs.pop("env_wrappers")
     else:
         env_wrappers = []

examples/envpool/train_ppo.py

@@ -16,10 +16,10 @@
 
 """"""
 import numpy as np
-
-from openrl.configs.config import create_config_parser
 from make_env import make
+
 from examples.envpool.envpool_wrappers import VecAdapter, VecMonitor
+from openrl.configs.config import create_config_parser
 from openrl.modules.common import PPONet as Net
 from openrl.modules.common.ppo_net import PPONet as Net
 from openrl.runners.common import PPOAgent as Agent

openrl/__init__.py

@@ -1,5 +1,5 @@
 __TITLE__ = "openrl"
-__VERSION__ = "v0.1.10"
+__VERSION__ = "v0.2.0"
 __DESCRIPTION__ = "Distributed Deep RL Framework"
 __AUTHOR__ = "OpenRL Contributors"
 __EMAIL__ = "huangshiyu@4paradigm.com"

openrl/envs/common/build_envs.py

@@ -69,4 +69,4 @@ def _make_env() -> Env:
         return _make_env
 
     env_fns = [create_env(env_id, env_num, need_env_id) for env_id in range(env_num)]
-    return env_fns
+    return env_fns

openrl/envs/common/registration.py

@@ -173,4 +173,4 @@ def make(
         vec_info_class = VecInfoFactory.get_vec_info_class(vec_info_class, env)
         env = VecMonitorWrapper(vec_info_class, env)
 
-    return env
+    return env

openrl/envs/nlp/daily_dialog_env.py

@@ -72,16 +72,18 @@ def __init__(
         # set the observation and action space here
         self._vocab_size = self.tokenizer.vocab_size
 
-        self.observation_space = DictSpace({
-            "input_encoded_pt": spaces.Box(
-                low=0,
-                high=self._vocab_size,
-                shape=(self._max_text_length + self.max_steps,),
-            ),
-            "input_attention_mask_pt": spaces.Box(
-                low=0, high=1, shape=(self._max_text_length + self.max_steps,)
-            ),
-        })
+        self.observation_space = DictSpace(
+            {
+                "input_encoded_pt": spaces.Box(
+                    low=0,
+                    high=self._vocab_size,
+                    shape=(self._max_text_length + self.max_steps,),
+                ),
+                "input_attention_mask_pt": spaces.Box(
+                    low=0, high=1, shape=(self._max_text_length + self.max_steps,)
+                ),
+            }
+        )
         self.action_space = Discrete(n=self._vocab_size)
         # see https://github.com/huggingface/transformers/issues/4875 : rounding up to nearest power of 2 for better GPU efficiency
 
@@ -112,7 +114,6 @@ def __init__(
         self.reward_function = None
 
     def set_reward(self, reward_fn=None):
-
         self.reward_function = reward_fn
 
     def step_word(self, word: str) -> Tuple[Dict[str, torch.tensor], int, bool, dict]:

openrl/envs/nlp/fake_dialog_env.py

@@ -30,16 +30,18 @@ def __init__(
         # set the observation and action space here
         self._vocab_size = 2
 
-        self.observation_space = DictSpace({
-            "input_encoded_pt": spaces.Box(
-                low=0,
-                high=self._vocab_size,
-                shape=(self._max_text_length + self.max_steps,),
-            ),
-            "input_attention_mask_pt": spaces.Box(
-                low=0, high=1, shape=(self._max_text_length + self.max_steps,)
-            ),
-        })
+        self.observation_space = DictSpace(
+            {
+                "input_encoded_pt": spaces.Box(
+                    low=0,
+                    high=self._vocab_size,
+                    shape=(self._max_text_length + self.max_steps,),
+                ),
+                "input_attention_mask_pt": spaces.Box(
+                    low=0, high=1, shape=(self._max_text_length + self.max_steps,)
+                ),
+            }
+        )
         self.action_space = Discrete(n=self._vocab_size)
 
         n = 2

openrl/envs/nlp/rewards/intent.py

@@ -41,10 +41,9 @@ def __init__(
         self.use_model_parallel = False
 
         if intent_model == "builtin_intent":
-            
             self._device = "cpu"
-            self.use_data_parallel = False 
-            
+            self.use_data_parallel = False
+
             from transformers import GPT2Config, GPT2LMHeadModel
 
             class TestTokenizer: