0

我想使用稳定的基线 RL 实现并使用自定义模型。我简化了我的情况。我有三个问题：

  • 为什么它不学会预测 2？根据初始化它预测 4, 7, 3, ...
  • 我假设 CustomCombinedExtractor 在前向传递中产生最终的离散预测。所以这将是 10 维。但稳定的基线要求它输出 64 暗向量。这是为什么？之后是否应用了进一步的模型？我怎样才能停用它？
  • 我们有哪些明智的选择：“lr_schedule”？

这里的代码：

import gym
from gym import spaces
from stable_baselines3 import DQN
from stable_baselines3.dqn import MultiInputPolicy
import numpy as np
import torch.nn as nn
import torch


class CustomEnv(gym.Env):
    """Custom Environment that follows gym interface"""
    metadata = {'render.modes': ['human']}

    def __init__(self):
        super(CustomEnv, self).__init__()
        self.action_space = spaces.Discrete(10)
        self.observation_space = spaces.Dict({
            "vector1": spaces.Box(low=0, high=10, shape=(10,), dtype=np.float32),
            "vector2": spaces.Box(low=0, high=10, shape=(10,), dtype=np.float32)
        })

    def obs(self):
        return dict({
            "vector1": 5*np.ones(10),
            "vector2": 5*np.ones(10)})

    def step(self, action):
        if action == 2:
            reward = 20
        else:
            reward = 0
        return self.obs(), reward, False, dict({})

    def reset(self):
        return self.obs()

    def render(self, mode='human'):
        return None

    def close(self):
        pass

env = CustomEnv()

class CustomCombinedExtractor(MultiInputPolicy):
    def __init__(self, observation_space, action_space, lr_schedule):
        super().__init__(observation_space, action_space, lr_schedule)

        extractors = {}

        total_concat_size = 0
        for key, subspace in observation_space.spaces.items():
            elif key == "vector"1:
                extractors[key] = nn.Linear(subspace.shape[0], 64)
                total_concat_size += 64
            elif key == "vector2":
                extractors[key] = nn.Linear(subspace.shape[0], 64)
                total_concat_size += 64

        self.extractors = nn.ModuleDict(extractors)
        self._features_dim = 1
        self.features_dim = 1

    def forward(self, observations):
        encoded_tensor_list = []

        x = self.extractors["vector"](observations["vector"])
        return x.T


def lr_schedule(x): return 1/x
policy_kwargs = dict(
    features_extractor_class=CustomCombinedExtractor,
    features_extractor_kwargs=dict(
        action_space=spaces.Discrete(10), lr_schedule=lr_schedule),
)

model = DQN(MultiInputPolicy, env, verbose=1,
            buffer_size=1000, policy_kwargs=policy_kwargs)

model.learn(total_timesteps=25000)
model.save("ppo_cartpole")

del model  # remove to demonstrate saving and loading

model = DQN.load("ppo_cartpole")

obs = env.reset()
while True:
    action, _states = model.predict(obs)
    print(action)
    obs, rewards, dones, info = env.step(action)
    env.render()
4

0 回答 0