题目:基于gym构建如下迷宫世界:
全部代码:
maze_mdp.py
python">import logging #日志模块
import numpy
import random
from gym import spaces
import gym
logging = logging.getLogger(__name__)
# Set this in SOME subclasses
class MazeEnv(gym.Env):
metadata = {
'render.modes':['human','rgb_array'],
'video.frames_per_second':2
}
def _seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def __init__(self):
# 状态空间
self.states = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]
# 设置机器人的位置,机器人的位置根据所处状态不同位置也不同。
# 事先计算出每个状态点机器人位置的中心坐标,并存储到两个向量中,在类初始化中给出
self.x = [125, 175, 225, 325, 125, 175, 225, 325, 225, 275, 325, 125, 175, 225, 275, 325, 125, 175]
self.y = [325, 325, 325, 325, 275, 275, 275, 275, 225, 225, 225, 175, 175, 175, 175, 175, 125, 125]
self.terminate_states = dict() #终止状态为字典格式
self.terminate_states[11] = 1
# 动作空间
self.actions = ['n','e','s','w']
# 回报函数
self.rewards = dict(); #回报的数据结构为字典
self.rewards['18_s'] = 20.0
self.rewards['10_e'] = 20.0
self.rewards['16_n'] = 20.0
# 状态转移概率
self.t = dict();
self.t['1_e'] = 2
self.t['1_s'] = 5
self.t['2_w'] = 1
self.t['2_e'] = 3
self.t['2_s'] = 6
self.t['3_w'] = 2
self.t['3_s'] = 7
self.t['4_s'] = 8
self.t['5_e'] = 6
self.t['5_n'] = 1
self.t['6_w'] = 5
self.t['6_e'] = 7
self.t['6_n'] = 2
self.t['7_w'] = 6
self.t['7_s'] = 9
self.t['7_n'] = 3
self.t['8_s'] = 11
self.t['8_n'] = 4
self.t['9_e'] = 10
self.t['9_s'] = 14
self.t['9_n'] = 7
self.t['10_w'] = 9
self.t['10_e'] = 11
self.t['10_s'] = 15
self.t['11_w'] = 10
self.t['11_s'] = 16
self.t['11_n'] = 8
self.t['12_e'] = 13
self.t['12_s'] = 17
self.t['13_w'] = 12
self.t['13_e'] = 14
self.t['13_s'] = 18
self.t['14_w'] = 13
self.t['14_e'] = 15
self.t['14_n'] = 9
self.t['15_w'] = 14
self.t['15_e'] = 16
self.t['15_n'] = 10
self.t['16_w'] = 15
self.t['16_n'] = 11
self.t['17_e'] = 18
self.t['17_n'] = 12
self.t['18_w'] = 17
self.t['18_n'] = 13
#折扣因子
self.gamma = 0.8
self.viewer = None
self.state = None
def getTerminal(self):
return self.terminate_states
def getGamma(self):
return self.gamma
def getStates(self):
return self.states
def getAction(self):
return self.actions
def getTerminate_states(self):
return self.terminate_states
def setAction(self,s):
self.state = s
# step()函数输入是动作,
# 输出是下一时刻的动作、回报、是否终止、调试信息
# 没有的用{}表示
def _step(self,action):
# 系统当前状态
state = self.state
# 判断系统当前状态是否为终止状态
if state in self.terminate_states:
return state,0,True,{}
#将状态与动作组成字典的键值
key = "%d_%s"(state,action)
#状态转移
if key in self.t:
next_state = self.t[key]
else:
next_state = state
self.state = next_state
is_terminal = False
if next_state in self.terminate_states:
is_terminal = True
if key not in self.rewards:
r = -1.0
else:
r = self.rewards[key]
return next_state,r,is_terminal,{}
# reset函数建立
# reset常常用随机的方法初始机器人状态
def _reset(self):
self.state = self.states[int(random.random()*len(self.states))]
return self.state
# render函数建立
def _render(self, mode='human', close=False):
if close:
if self.viewer is not None:
self.viewer.close()
self.viewer = None
return
screen_width = 450
screen_height = 450
if self.viewer is None:
# 调用rendering的画图函数
from gym.envs.classic_control import rendering
self.viewer = rendering.Viewer(screen_width, screen_height)
# 创建网格世界,一共12条直线
self.line1 = rendering.Line((100, 350), (350, 350))
self.line2 = rendering.Line((100, 300), (350, 300))
self.line3 = rendering.Line((100, 250), (350, 250))
self.line4 = rendering.Line((100, 200), (350, 200))
self.line5 = rendering.Line((100, 150), (350, 150))
self.line6 = rendering.Line((100, 100), (350, 100))
self.line7 = rendering.Line((100, 350), (100, 100))
self.line8 = rendering.Line((150, 350), (150, 100))
self.line9 = rendering.Line((200, 350), (200, 100))
self.line10 = rendering.Line((250, 350), (250, 100))
self.line11 = rendering.Line((300, 350), (300, 100))
self.line12 = rendering.Line((350, 350), (350, 100))
# 接下来写墙块,黑色填满代表墙块
# 创建第一个墙块
self.wall1 = rendering.make_polygon([(250,350),(300,350), (300,300),(250,300)],filled=True)
self.wall1.set_color(0, 0, 0)
# 创建第二个墙块
self.wall2 = rendering.make_polygon([(250,300),(300,300),(300,250),(250,250)],filled=True)
self.wall2.set_color(0, 0, 0)
# 创建第三个墙块
self.wall3 = rendering.make_polygon([(100, 250), (150, 250), (150,200),(100,200)], filled=True)
self.wall3.set_color(0, 0, 0)
# 创建第四个墙块
self.wall4 = rendering.make_polygon([(150, 250), (200, 250), (200,200),(150,200)], filled=True)
self.wall4.set_color(0, 0, 0)
# 创建第五个墙块
self.wall5 = rendering.make_polygon([(200, 150), (250, 150), (250, 100),(200, 100)], filled=True)
self.wall5.set_color(0, 0, 0)
# 创建第六个墙块
self.wall6 = rendering.make_polygon([(250, 150), (300, 150), (300, 100),(250, 100)], filled=True)
self.wall6.set_color(0, 0, 0)
# 创建第七个墙块
self.wall7 = rendering.make_polygon([(300, 150), (350, 150), (350, 100),(300, 100)], filled=True)
self.wall7.set_color(0, 0, 0)
# 创建出口,用三角形表示
self.goal = rendering.make_polygon([(325,245),(300,205),(350,205)],
filled=True)
self.goal.set_color(1, 0.9, 0)
# 创建机器人,用不同颜色的圆表示
self.robot = rendering.make_circle(15)
self.robotrans = rendering.Transform()
self.robot.add_attr(self.robotrans)
self.robot.set_color(0.8, 0.6, 0.4)
# 给11条直线设置颜色,并将这些创建的对象添加到几何中
self.line1.set_color(0, 0, 0)
self.line2.set_color(0, 0, 0)
self.line3.set_color(0, 0, 0)
self.line4.set_color(0, 0, 0)
self.line5.set_color(0, 0, 0)
self.line6.set_color(0, 0, 0)
self.line7.set_color(0, 0, 0)
self.line8.set_color(0, 0, 0)
self.line9.set_color(0, 0, 0)
self.line10.set_color(0, 0, 0)
self.line11.set_color(0, 0, 0)
self.line12.set_color(0, 0, 0)
self.viewer.add_geom(self.line1)
self.viewer.add_geom(self.line2)
self.viewer.add_geom(self.line3)
self.viewer.add_geom(self.line4)
self.viewer.add_geom(self.line5)
self.viewer.add_geom(self.line6)
self.viewer.add_geom(self.line7)
self.viewer.add_geom(self.line8)
self.viewer.add_geom(self.line9)
self.viewer.add_geom(self.line10)
self.viewer.add_geom(self.line11)
self.viewer.add_geom(self.line12)
self.viewer.add_geom(self.wall1)
self.viewer.add_geom(self.wall2)
self.viewer.add_geom(self.wall3)
self.viewer.add_geom(self.wall4)
self.viewer.add_geom(self.wall5)
self.viewer.add_geom(self.wall6)
self.viewer.add_geom(self.wall7)
self.viewer.add_geom(self.goal)
self.viewer.add_geom(self.robot)
# 根据这两个向量和机器人当前状态,就可以设置机器人当前的圆心坐标
if self.state is None: return None
self.robotrans.set_translation(self.x[self.state - 1], self.y[self.state - 1])
return self.viewer.render(return_rgb_array=mode =='rgb_array')
- 在
classic_control\_init_.py
中添加
python">from gym.envs.classic_control.maze_mdp import MazeEnv
- 在
envs/_init_.py
中添加
python">register(
id='MazeWorld-v0',
entry_point = 'gym.envs.classic_control:MazeEnv',
max_episode_steps=200,
reward_threshold=100.0,
)
- 启动
Anaconda Prompt
- 遇到的困难
墙块代码我采用的画多边形的函数make_polygon
,但是一直缺一块口子
最后发现。。。。
是因为四个顶点的顺序搞错了。。。。。。