我正在尝试编写马尔可夫决策过程（MDP），但遇到了一些问题。您能否检查我的代码并找出它不起作用的原因
我试图用一些小数据来做它，它可以工作并给我必要的结果，我觉得这是正确的。但我的问题是对这段代码的概括。是的，我知道 MDP 库，但我需要编写这个库。此代码有效，我希望在课堂上得到相同的结果：
import pandas as pd
data = [['3 0', 'UP', 0.6, '3 1', 5, 'YES'], ['3 0', 'UP', 0.4, '3 2', -10, 'YES'], \
['3 0', 'RIGHT', 1, '3 3', 10, 'YES'], ['3 1', 'RIGHT', 1, '3 3', 4, 'NO'], \
['3 2', 'DOWN', 0.6, '3 3', 3, 'NO'], ['3 2', 'DOWN', 0.4, '3 1', 5, 'NO'], \
['3 3', 'RIGHT', 1, 'EXIT', 7, 'NO'], ['EXIT', 'NO', 1, 'EXIT', 0, 'NO']]
df = pd.DataFrame(data, columns = ['Start', 'Action', 'Probability', 'End', 'Reward', 'Policy'], \
dtype = float) #initial matrix
point_3_0, point_3_1, point_3_2, point_3_3, point_EXIT = 0, 0, 0, 0, 0
gamma = 0.9 #it is a discount factor
for i in range(100):
point_3_0 = gamma * max(0.6 * (point_3_1 + 5) + 0.4 * (point_3_2 - 10), point_3_3 + 10)
point_3_1 = gamma * (point_3_3 + 4)
point_3_2 = gamma * (0.6 * (point_3_3 + 3) + 0.4 * (point_3_1 + 5))
point_3_3 = gamma * (point_EXIT + 7)
print(point_3_0, point_3_1, point_3_2, point_3_3, point_EXIT)
但是在这里我在某个地方有一个错误，它看起来太复杂了？你能帮我解决这个问题吗？！
gamma = 0.9
class MDP:
def __init__(self, gamma, table):
self.gamma = gamma
self.table = table
def Action(self, state):
return self.table[self.table.Start == state].Action.values
def Probability(self, state):
return self.table[self.table.Start == state].Probability.values
def End(self, state):
return self.table[self.table.Start == state].End.values
def Reward(self, state):
return self.table[self.table.Start == state].Reward.values
def Policy(self, state):
return self.table[self.table.Start == state].Policy.values
mdp = MDP(gamma = gamma, table = df)
def value_iteration():
states = mdp.table.Start.values
actions = mdp.Action
probabilities = mdp.Probability
ends = mdp.End
rewards = mdp.Reward
policies = mdp.Policy
V1 = {s: 0 for s in states}
for i in range(100):
V = V1.copy()
for s in states:
if policies(s) == 'YES':
V1[s] = gamma * max(rewards(s) + [sum([p * V[s1] for (p, s1) \
in zip(probabilities(s), ends(s))][actions(s)==a]) for a in set(actions(s))])
else:
sum(probabilities[s] * ends(s))
return V
value_iteration()
我希望每一点都有值，但是得到： ValueError：具有多个元素的数组的真值是不明确的。使用 a.any() 或 a.all()