Add support for python 3.10 (and remove pybullet) (vwxyzjn#386)

* Add support for python 3.10 (and remove pybullet)

* Revert pyproject.toml to just remove pybullet

* Update poetry and remove MacOS from PettingZoo CI

* Update requirements

* Update installation.md and README.md for python version

.github/workflows/tests.yaml

@@ -93,34 +93,6 @@ jobs:
         if: runner.os == 'Linux' || runner.os == 'macOS'
         run: poetry run pytest tests/test_atari_jax_gymnasium.py
 
-  test-pybullet-envs:
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.8]
-        poetry-version: [1.3]
-        os: [ubuntu-22.04, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Run image
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-
-      # pybullet tests
-      - name: Install core dependencies
-        run: poetry install -E pytest
-      - name: Install pybullet dependencies
-        run: poetry install -E "pytest pybullet"
-      - name: Downgrade setuptools
-        run: poetry run pip install setuptools==59.5.0
-      - name: Run pybullet tests
-        run: poetry run pytest tests/test_pybullet.py
-
   test-procgen-envs:
     strategy:
       fail-fast: false
@@ -144,7 +116,7 @@ jobs:
         run: poetry install -E "pytest procgen"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
-      - name: Run pybullet tests
+      - name: Run procgen tests
         run: poetry run pytest tests/test_procgen.py
 
   test-mujoco-envs:
@@ -283,7 +255,7 @@ jobs:
 
       # mujoco_py tests
       - name: Install dependencies
-        run: poetry install -E "pytest pybullet mujoco_py mujoco jax"
+        run: poetry install -E "pytest mujoco_py mujoco jax"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: install mujoco_py dependencies
@@ -316,7 +288,7 @@ jobs:
 
       # mujoco_py tests
       - name: Install dependencies
-        run: poetry install -E "pytest pybullet mujoco_py mujoco jax"
+        run: poetry install -E "pytest mujoco_py mujoco jax"
       - name: Run gymnasium migration dependencies
         run: poetry run pip install "stable_baselines3==2.0.0a1"
       - name: Downgrade setuptools
@@ -389,7 +361,7 @@ jobs:
       matrix:
         python-version: [3.8]
         poetry-version: [1.3]
-        os: [ubuntu-22.04, macos-latest]
+        os: [ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -48,10 +48,6 @@ repos:
         name: poetry-export requirements-atari.txt
         args: ["--without-hashes", "-o", "requirements/requirements-atari.txt", "-E", "atari"]
         stages: [manual]
-      - id: poetry-export
-        name: poetry-export requirements-pybullet.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-pybullet.txt", "-E", "pybullet"]
-        stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-mujoco.txt
         args: ["--without-hashes", "-o", "requirements/requirements-mujoco.txt", "-E", "mujoco"]

Dockerfile

@@ -19,7 +19,7 @@ RUN apt-get -y install wget unzip software-properties-common \
     libgl1-mesa-glx \
     libglew-dev \
     libosmesa6-dev patchelf
-RUN poetry install -E "atari mujoco_py pybullet"
+RUN poetry install -E "atari mujoco_py"
 RUN poetry run python -c "import mujoco_py"
 
 COPY entrypoint.sh /usr/local/bin/

README.md

@@ -38,7 +38,7 @@ CleanRL only contains implementations of **online** deep reinforcement learning
 ## Get started
 
 Prerequisites:
-* Python >=3.7.1,<3.10 (not yet 3.10)
+* Python >=3.7.1,<3.11
 * [Poetry 1.2.1+](https://python-poetry.org)
 
 To run experiments locally, give the following a try:
@@ -77,7 +77,6 @@ pip install -r requirements/requirements.txt
 
 # optional dependencies
 pip install -r requirements/requirements-atari.txt
-pip install -r requirements/requirements-pybullet.txt
 pip install -r requirements/requirements-mujoco.txt
 pip install -r requirements/requirements-mujoco_py.txt
 pip install -r requirements/requirements-mujoco.txt
@@ -112,12 +111,6 @@ python cleanrl/ppo_atari_envpool.py --env-id BreakoutNoFrameskip-v4
 # Side effects such as lower sample efficiency might occur
 poetry run python ppo_atari_envpool.py --clip-coef=0.2 --num-envs=16 --num-minibatches=8 --num-steps=128 --update-epochs=3
 
-# pybullet
-poetry install -E pybullet
-python cleanrl/td3_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
-python cleanrl/ddpg_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
-python cleanrl/sac_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
-
 # procgen
 poetry install -E procgen
 python cleanrl/ppo_procgen.py --env-id starpilot

benchmark/ddpg.sh

@@ -1,12 +1,12 @@
-poetry install -E "mujoco_py pybullet"
+poetry install -E "mujoco_py"
 python -c "import mujoco_py"
 xvfb-run -a python -m cleanrl_utils.benchmark \
     --env-ids HalfCheetah-v2 Walker2d-v2 Hopper-v2 InvertedPendulum-v2 Humanoid-v2 Pusher-v2 \
     --command "poetry run python cleanrl/ddpg_continuous_action.py --track --capture-video" \
     --num-seeds 3 \
     --workers 1
 
-poetry install -E "mujoco_py pybullet jax"
+poetry install -E "mujoco_py jax"
 poetry run pip install --upgrade "jax[cuda]==0.3.17" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 poetry run python -c "import mujoco_py"
 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \

benchmark/sac.sh

@@ -1,4 +1,4 @@
-poetry install -E "mujoco_py pybullet"
+poetry install -E mujoco_py
 poetry run python -c "import mujoco_py"
 OMP_NUM_THREADS=1 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \
     --env-ids HalfCheetah-v2 Walker2d-v2 Hopper-v2 \

benchmark/td3.sh

@@ -1,12 +1,12 @@
-poetry install -E "mujoco_py pybullet"
+poetry install -E mujoco_py
 python -c "import mujoco_py"
 OMP_NUM_THREADS=1 xvfb-run -a python -m cleanrl_utils.benchmark \
     --env-ids HalfCheetah-v2 Walker2d-v2 Hopper-v2 InvertedPendulum-v2 Humanoid-v2 Pusher-v2 \
     --command "poetry run python cleanrl/td3_continuous_action.py --track --capture-video" \
     --num-seeds 3 \
     --workers 1
 
-poetry install -E "mujoco_py pybullet jax"
+poetry install -E "mujoco_py jax"
 poetry run pip install --upgrade "jax[cuda]==0.3.17" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 poetry run python -c "import mujoco_py"
 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \

cleanrl/ppo_continuous_action_isaacgym/isaacgym/pyproject.toml

@@ -9,7 +9,7 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.7.1,<3.10"
+python = ">=3.7.1"
 gym = "0.23.1"
 torch = "^1.12.0"
 torchvision = "^0.13.0"

cleanrl/sac_continuous_action.py

@@ -7,7 +7,6 @@
 
 import gym
 import numpy as np
-import pybullet_envs  # noqa
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -37,7 +36,7 @@ def parse_args():
         help="whether to capture videos of the agent performances (check out `videos` folder)")
 
     # Algorithm specific arguments
-    parser.add_argument("--env-id", type=str, default="HopperBulletEnv-v0",
+    parser.add_argument("--env-id", type=str, default="Hopper-v4",
         help="the id of the environment")
     parser.add_argument("--total-timesteps", type=int, default=1000000,
         help="total timesteps of the experiments")

cleanrl/td3_continuous_action.py

@@ -7,7 +7,6 @@
 
 import gym
 import numpy as np
-import pybullet_envs  # noqa
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -37,7 +36,7 @@ def parse_args():
         help="whether to capture videos of the agent performances (check out `videos` folder)")
 
     # Algorithm specific arguments
-    parser.add_argument("--env-id", type=str, default="HopperBulletEnv-v0",
+    parser.add_argument("--env-id", type=str, default="Hopper-v4",
         help="the id of the environment")
     parser.add_argument("--total-timesteps", type=int, default=1000000,
         help="total timesteps of the experiments")

cleanrl/td3_continuous_action_jax.py

@@ -13,7 +13,6 @@
 import jax.numpy as jnp
 import numpy as np
 import optax
-import pybullet_envs  # noqa
 from flax.training.train_state import TrainState
 from stable_baselines3.common.buffers import ReplayBuffer
 from torch.utils.tensorboard import SummaryWriter

docs/contribution.md

@@ -77,7 +77,7 @@ Importantly, **regardless of the slight difference in performance-impacting chan
 Given a new feature, we create a PR and then run the benchmark experiments through [`benchmark.py`](https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl_utils/benchmark.py), such as the following:
 
 ```bash
-poetry install -E "docs pybullet mujoco_py"
+poetry install -E "docs mujoco_py"
 python -c "import mujoco_py"
 xvfb-run -a python -m cleanrl_utils.benchmark \
     --env-ids HalfCheetah-v2 Walker2d-v2 Hopper-v2 \

docs/get-started/examples.md

@@ -32,16 +32,6 @@ python cleanrl/ppo.py --env-id CartPole-v1
 python cleanrl/c51.py --env-id CartPole-v1
 ```
 
-## PyBullet
-```
-poetry shell
-
-poetry install -E pybullet
-python cleanrl/td3_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
-python cleanrl/ddpg_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
-python cleanrl/sac_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
-```
-
 ## Procgen 
 ```
 poetry shell

docs/get-started/installation.md

@@ -2,7 +2,7 @@
 
 ## Prerequisites
 
-* Python >=3.7.1,<3.10 (not yet 3.10)
+* Python >=3.7.1,<3.11
 * [Poetry 1.2.1+](https://python-poetry.org)
 
 Simply run the following command for a quick start
@@ -56,7 +56,6 @@ pip install -r requirements/requirements.txt
 
 # optional dependencies
 pip install -r requirements/requirements-atari.txt
-pip install -r requirements/requirements-pybullet.txt
 pip install -r requirements/requirements-mujoco.txt
 pip install -r requirements/requirements-mujoco_py.txt
 pip install -r requirements/requirements-mujoco.txt
@@ -84,11 +83,6 @@ ale-py = "0.7.4"
 AutoROM = {extras = ["accept-rom-license"], version = "^0.4.2"}
 opencv-python = "^4.6.0.66"
 
-[tool.poetry.group.pybullet]
-optional = true
-[tool.poetry.group.pybullet.dependencies]
-pybullet = "3.1.8"
-
 [tool.poetry.group.procgen]
 optional = true
 [tool.poetry.group.procgen.dependencies]
@@ -99,7 +93,6 @@ You can install them using the following command
 
 ```bash
 poetry install -E atari
-poetry install -E pybullet
 poetry install -E mujoco
 poetry install -E mujoco_py
 poetry install -E dm_control

docs/rl-algorithms/ddpg.md

@@ -40,21 +40,21 @@ The [ddpg_continuous_action.py](https://github.com/vwxyzjn/cleanrl/blob/master/c
 
     ```bash
     poetry install
-    poetry install -E pybullet
     poetry run python cleanrl/ddpg_continuous_action.py --help
-    poetry run python cleanrl/ddpg_continuous_action.py --env-id HopperBulletEnv-v0
     poetry install -E mujoco_py # only works in Linux
     poetry run python cleanrl/ddpg_continuous_action.py --env-id Hopper-v2
+    poetry install -E mujoco
+    poetry run python cleanrl/ddpg_continuous_action.py --env-id Hopper-v4
     ```
 
 === "pip"
 
     ```bash
-    pip install -r requirements/requirements-pybullet.txt
     python cleanrl/ddpg_continuous_action.py --help
-    python cleanrl/ddpg_continuous_action.py --env-id HopperBulletEnv-v0
     pip install -r requirements/requirements-mujoco_py.txt # only works in Linux, you have to pick either `mujoco` or `mujoco_py`
     python cleanrl/ddpg_continuous_action.py --env-id Hopper-v2
+    pip install -r requirements/requirements-mujoco.txt
+    python cleanrl/ddpg_continuous_actions.py --env-id Hopper-v4
     ```
 
 ### Explanation of the logged metrics
@@ -277,21 +277,21 @@ The [ddpg_continuous_action_jax.py](https://github.com/vwxyzjn/cleanrl/blob/mast
 
     ```bash
     poetry install
-    poetry install -E "pybullet jax"
+    poetry install -E "mujoco jax"
     poetry run python cleanrl/ddpg_continuous_action_jax.py --help
-    poetry run python cleanrl/ddpg_continuous_action_jax.py --env-id HopperBulletEnv-v0
+    poetry run python cleanrl/ddpg_continuous_action_jax.py --env-id Hopper-v4
     poetry install -E mujoco_py # only works in Linux
     poetry run python cleanrl/ddpg_continuous_action_jax.py --env-id Hopper-v2
     ```
 
 === "pip"
 
     ```bash
-    pip install -r requirements/requirements-pybullet.txt
+    pip install -r requirements/requirements-mujoco.txt
     pip install -r requirements/requirements-jax.txt
     python cleanrl/ddpg_continuous_action_jax.py --help
-    python cleanrl/ddpg_continuous_action_jax.py --env-id HopperBulletEnv-v0
-    pip install -r requirements/requirements-mujoco_py.txt # only works in Linux, you have to pick either `mujoco` or `mujoco_py`
+    python cleanrl/ddpg_continuous_action_jax.py --env-id Hopper-v4
+    pip install -r requirements/requirements-mujoco_py.txt # only works in Linux
     python cleanrl/ddpg_continuous_action_jax.py --env-id Hopper-v2
     ```
 

docs/rl-algorithms/ppo.md

@@ -249,7 +249,7 @@ The [ppo_continuous_action.py](https://github.com/vwxyzjn/cleanrl/blob/master/cl
     poetry install -E "mujoco dm_control"
     python cleanrl/ppo_continuous_action.py --env-id dm_control/cartpole-balance-v0
     # backwards compatibility with mujoco v2 environments
-    poetry install -E mujoco_py
+    poetry install -E mujoco_py  # only works in Linux
     python cleanrl/ppo_continuous_action.py --env-id Hopper-v2
     ```
 

docs/rl-algorithms/sac.md

@@ -50,23 +50,23 @@ The [sac_continuous_action.py](https://github.com/vwxyzjn/cleanrl/blob/master/cl
 
     ```bash
     poetry install
-    poetry install -E pybullet
+    poetry install -E mujoco
     poetry run python cleanrl/sac_continuous_action.py --help
-    poetry run python cleanrl/sac_continuous_action.py --env-id HopperBulletEnv-v0
+    poetry run python cleanrl/sac_continuous_action.py --env-id Hopper-v4
     poetry install -E mujoco_py # only works in Linux
     poetry run python cleanrl/sac_continuous_action.py --env-id Hopper-v2
-    poetry run python cleanrl/sac_continuous_action.py --env-id HopperBulletEnv-v0 --autotune False --alpha 0.2 ## Without Automatic entropy coef. tuning
+    poetry run python cleanrl/sac_continuous_action.py --env-id Hopper-v2 --autotune False --alpha 0.2 ## Without Automatic entropy coef. tuning
     ```
 
 === "pip"
 
     ```bash
-    pip install -r requirements/requirements-pybullet.txt
+    pip install -r requirements/requirements-mujoco.txt
     python cleanrl/sac_continuous_action.py --help
-    python cleanrl/sac_continuous_action.py --env-id HopperBulletEnv-v0
-    pip install -r requirements/requirements-mujoco_py.txt # only works in Linux, you have to pick either `mujoco` or `mujoco_py`
+    python cleanrl/sac_continuous_action.py --env-id Mujoco-v4
+    pip install -r requirements/requirements-mujoco_py.txt # only works in Linux
     python cleanrl/sac_continuous_action.py --env-id Hopper-v2
-    python cleanrl/sac_continuous_action.py --env-id HopperBulletEnv-v0 --autotune False --alpha 0.2 ## Without Automatic entropy coef. tuning
+    python cleanrl/sac_continuous_action.py --env-id Hopper-v2 --autotune False --alpha 0.2 ## Without Automatic entropy coef. tuning
     ```
 
 ### Explanation of the logged metrics