1

我正在尝试使用 Pytorch 在 CarPole 环境中实现 DQN。我不知道为什么，但是无论我尝试训练代理多久，即使分数普遍增加，它们也只是波动而没有保持高分。该代码来自为 tensorflow 编写的 DQN 教程，该教程运行正常，但是当我尝试转换为 Pytorch 时，它无法学习。这是模型：

class Net(nn.Module):
def __init__(self, state_size, action_size):
    super(Net, self).__init__()
    self.fc1 = nn.Linear(state_size, 24)
    self.fc2 = nn.Linear(24, 24)
    self.fc3 = nn.Linear(24, action_size)
    
def forward(self, inputs):
    x = torch.from_numpy(inputs)
    x = F.relu(self.fc1(x.float()))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    return x

class DQNAgent(nn.Module):
def __init__(self, state_size, action_size):
    super(DQNAgent, self).__init__()
    self.state_size = state_size
    self.action_size = action_size
    self.memory = deque(maxlen=2000)
    self.model, self.criterion, self.optimizer = self.build_model()
    
    self.epsilon = 1.0
    self.epsilon_decay = 0.995
    self.epsilon_min = 0.01
    self.gamma = 0.95
    
def build_model(self):
    model = Net(state_size, action_size)
    model = model.float()
    
    criterion = nn.MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=0.001) # might need to return criterion and optimizer
    
    return model, criterion, optimizer

def remember(self, state, action, reward, next_state, done):
    self.memory.append((state, action, reward, next_state, done))
    
def act(self, state):
    if np.random.rand() <= self.epsilon:
        return random.randrange(self.action_size)
    act_values = self.model(state)
    return np.argmax(act_values.detach().numpy())

def replay(self, batch_size):
    minibatch = random.sample(self.memory, batch_size)
    for state, action, reward, next_state, done in minibatch:
        if done:
            target = reward
        elif not done:
            target = reward + self.gamma*torch.max(self.model(next_state)) # --> a tensor
            
            
        target_f = self.model(state)
        target_f[0][action] = target
        
        # self.model.fit(state, target_f, epochs=1, verbose=0)
        loss = self.criterion(self.model(state), target_f)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()
        
    if self.epsilon > self.epsilon_min:
        self.epsilon = self.epsilon*self.epsilon_decay
        
def load(self, name):
    pass

def save(self, name):
    pass

...并训练：

for e in range(n_episodes):
state = env.reset()
state = np.reshape(state, [1, state_size])

for time in range(5000):
    # env.render()
    action = agent.act(state)
    
    next_state, reward, done, _ = env.step(action)
    reward = reward if not done else -10
    
    next_state = np.reshape(next_state, [1, state_size])
    
    agent.remember(state, action, reward, next_state, done)
    
    state = next_state
    
    if done:
        print("Episode {}/{}, score: {}, e: {:.2}".format(e, n_episodes, time, agent.epsilon))
        break
        
if len(agent.memory) > batch_size:
    agent.replay(batch_size)
    memory = agent.memory

如果有人能给我任何建议/建议，将不胜感激！！！现在很迷茫。谢谢！

4

0 回答 0