
# **PPO** Agent playing **LunarLander-v2**
This is a trained model of a **PPO** agent playing **LunarLander-v2**
using the [stable-baselines3 library](https://github.com/DLR-RM/stable-baselines3).

## Usage (with Stable-baselines3)


```python
import gymnasium as gym
from stable_baselines3 import PPO
from stable_baselines3.ppo.policies import MlpPolicy
from stable_baselines3.common.evaluation import evaluate_policy
from stable_baselines3.common.logger import configure
from stable_baselines3.common.monitor import Monitor
from stable_baselines3.common.callbacks import BaseCallback
from lib import score

from stable_baselines3 import PPO
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3.common.env_util import make_vec_env

from huggingface_sb3 import package_to_hub

class MyCallback(BaseCallback):
    def __init__(self, verbose=0):
        super(MyCallback, self).__init__(verbose)
    
    def _on_step(self) -> bool:
        current_step = self.num_timesteps
        if(current_step == 90_000):
            self.model.learning_rate = 0.001
        elif(current_step == 130_000):
            self.model.learning_rate = 0.0003
        return True

v_last_max_score = 150
v_target_score = 249
def training () :
    global v_last_max_score, v_target_score
    v_is_succeed = False
    v_total_steps = 1_000_000
    gym_name = "LunarLander-v2"
    model_name = "ppo-LunarLander-v2"
    # env_1 = 
    env = Monitor(gym.make(gym_name, render_mode="rgb_array"), 'test_monitor')

    ppo_config = {
        'policy': 'MlpPolicy', 
        'env': env             
        # , 'learning_rate' : learning_rate_schedule #0.0003
        , 'learning_rate' : 0.01
        , 'n_steps' :1024 #2048
        , 'batch_size' :512 #64
        , 'n_epochs' :20 #10
        , 'gamma' :0.999 #0.99
        , 'gae_lambda' :0.95
        , 'clip_range' :0.2 #0.15
        , 'clip_range_vf' :None
        , 'normalize_advantage' :True
        , 'ent_coef' : 0.000021 #0.0
        , 'vf_coef' :0.5
        , 'max_grad_norm' :0.5
        , 'use_sde' :False
        , 'sde_sample_freq' :4 #-1
        , 'rollout_buffer_class' :None
        , 'rollout_buffer_kwargs' :None
        , 'target_kl' :None
        , 'stats_window_size' :100
        , 'tensorboard_log' :None
        , 'policy_kwargs' : None
        , 'verbose' :1
        , 'seed' :None
        , 'device' :'auto'
        , '_init_setup_model' :True
    }

    model = PPO(**ppo_config)

    tmp_path = "/home/ubuntu/log/"
    new_logger = configure(tmp_path, ["stdout"])
    model.set_logger(new_logger)

    callback = MyCallback()
    model.learn(total_timesteps=v_total_steps, callback=callback)
    mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=10, deterministic=True)
    new_logger.log(f"mean_reward={mean_reward:.2f} +/- {std_reward}")

    #save for report
    v_score = mean_reward - std_reward
    if v_score>v_last_max_score:
        v_last_max_score = v_score
        model.save(model_name)
        score.record_video(gym_name, model, video_length=500, prefix="ppo-lunar")

        #upload
        upload_env_id = "LunarLander-v2"
        upload_model_architecture = "PPO"
        upload_repo_id = "dev-cuai/ppo-LunarLander-v2"
        upload_commit_message = "Upload PPO LunarLander-v2 trained agent"

        package_to_hub(
            model=model,  # Our trained model
            model_name=model_name,  # The name of our trained model
            model_architecture=upload_model_architecture,  # The model architecture we used: in our case PPO
            env_id=upload_env_id,  # Name of the environment
            eval_env=env,  # Evaluation Environment
            repo_id=upload_repo_id,  # id of the model repository from the Hugging Face Hub (repo_id = {organization}/{repo_name} for instance ThomasSimonini/ppo-LunarLander-v2
            commit_message=upload_commit_message,
        )

        v_is_succeed = v_score > v_target_score

    else :
        print('Too Low Mean_reward({}<{}), model.learning_rate:{}'.format(v_score, v_last_max_score, model.learning_rate))

    return v_is_succeed

v_loop_cnt = 0
while(True):
    v_loop_cnt += 1
    for i in range(3):
        print("OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO")
    print("run loop {} times. last score was {}".format(v_loop_cnt, v_last_max_score))
    v_ret = training ()
    if(v_ret):
        break

...
```

{"library_name": "stable-baselines3", "tags": ["LunarLander-v2", "deep-reinforcement-learning", "reinforcement-learning", "stable-baselines3"], "model-index": [{"name": "PPO", "results": [{"task": {"type": "reinforcement-learning", "name": "reinforcement-learning"}, "dataset": {"name": "LunarLander-v2", "type": "LunarLander-v2"}, "metrics": [{"type": "mean_reward", "value": "256.40 +/- 19.752047868963977", "name": "mean_reward", "verified": false}]}]}]}
stable-baselines3 LunarLander-v2 deep-reinforcement-learning reinforcement-learning model-index region:us