fix bug and add mirroring
这个提交包含在:
@ -29,7 +29,7 @@
|
|||||||
<site pos="0.15 0 0" size="0.003 0.003 0.03" type="box" name="cali_left_site1" rgba="0 0 1 0"/>
|
<site pos="0.15 0 0" size="0.003 0.003 0.03" type="box" name="cali_left_site1" rgba="0 0 1 0"/>
|
||||||
<site pos="0.15 0 0" size="0.003 0.03 0.003" type="box" name="cali_left_site2" rgba="0 0 1 0"/>
|
<site pos="0.15 0 0" size="0.003 0.03 0.003" type="box" name="cali_left_site2" rgba="0 0 1 0"/>
|
||||||
<site pos="0.15 0 0" size="0.03 0.003 0.003" type="box" name="cali_left_site3" rgba="0 0 1 0"/>
|
<site pos="0.15 0 0" size="0.03 0.003 0.003" type="box" name="cali_left_site3" rgba="0 0 1 0"/>
|
||||||
<camera name="left_wrist" pos="-0.1 0 0.16" fovy="20" mode="targetbody" target="vx300s_left/camera_focus"/>
|
<camera name="left_wrist" pos="-0.1 0 0.16" fovy="20" mode="fixed" euler="0 -1.05 -1.5707" />
|
||||||
<inertial pos="0.0395662 -2.56311e-07 0.00400649" quat="0.62033 0.619916 -0.339682 0.339869" mass="0.251652" diaginertia="0.000689546 0.000650316 0.000468142" />
|
<inertial pos="0.0395662 -2.56311e-07 0.00400649" quat="0.62033 0.619916 -0.339682 0.339869" mass="0.251652" diaginertia="0.000689546 0.000650316 0.000468142" />
|
||||||
<joint name="vx300s_left/wrist_rotate" pos="0 0 0" axis="1 0 0" limited="true" range="-3.14158 3.14158" frictionloss="30" />
|
<joint name="vx300s_left/wrist_rotate" pos="0 0 0" axis="1 0 0" limited="true" range="-3.14158 3.14158" frictionloss="30" />
|
||||||
<geom pos="-0.02 0 0" quat="0.707107 0 0 0.707107" type="mesh" mesh="vx300s_7_gripper" name="vx300s_left/7_gripper" />
|
<geom pos="-0.02 0 0" quat="0.707107 0 0 0.707107" type="mesh" mesh="vx300s_7_gripper" name="vx300s_left/7_gripper" />
|
||||||
|
|||||||
@ -29,7 +29,7 @@
|
|||||||
<site pos="0.15 0 0" size="0.003 0.003 0.03" type="box" name="cali_right_site1" rgba="0 0 1 0"/>
|
<site pos="0.15 0 0" size="0.003 0.003 0.03" type="box" name="cali_right_site1" rgba="0 0 1 0"/>
|
||||||
<site pos="0.15 0 0" size="0.003 0.03 0.003" type="box" name="cali_right_site2" rgba="0 0 1 0"/>
|
<site pos="0.15 0 0" size="0.003 0.03 0.003" type="box" name="cali_right_site2" rgba="0 0 1 0"/>
|
||||||
<site pos="0.15 0 0" size="0.03 0.003 0.003" type="box" name="cali_right_site3" rgba="0 0 1 0"/>
|
<site pos="0.15 0 0" size="0.03 0.003 0.003" type="box" name="cali_right_site3" rgba="0 0 1 0"/>
|
||||||
<camera name="right_wrist" pos="-0.1 0 0.16" fovy="20" mode="targetbody" target="vx300s_right/camera_focus"/>
|
<camera name="right_wrist" pos="-0.1 0 0.16" fovy="20" mode="fixed" euler="0 -1.05 -1.5707" />
|
||||||
<inertial pos="0.0395662 -2.56311e-07 0.00400649" quat="0.62033 0.619916 -0.339682 0.339869" mass="0.251652" diaginertia="0.000689546 0.000650316 0.000468142" />
|
<inertial pos="0.0395662 -2.56311e-07 0.00400649" quat="0.62033 0.619916 -0.339682 0.339869" mass="0.251652" diaginertia="0.000689546 0.000650316 0.000468142" />
|
||||||
<joint name="vx300s_right/wrist_rotate" pos="0 0 0" axis="1 0 0" limited="true" range="-3.14158 3.14158" frictionloss="30" />
|
<joint name="vx300s_right/wrist_rotate" pos="0 0 0" axis="1 0 0" limited="true" range="-3.14158 3.14158" frictionloss="30" />
|
||||||
<geom pos="-0.02 0 0" quat="0.707107 0 0 0.707107" type="mesh" mesh="vx300s_7_gripper" name="vx300s_right/7_gripper" />
|
<geom pos="-0.02 0 0" quat="0.707107 0 0 0.707107" type="mesh" mesh="vx300s_7_gripper" name="vx300s_right/7_gripper" />
|
||||||
|
|||||||
47
commands.txt
47
commands.txt
@ -18,6 +18,7 @@ CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
|||||||
--hidden_dim 512 --batch_size 24 --dim_feedforward 3200 --num_epochs 5000 --lr 1e-4 --seed 0
|
--hidden_dim 512 --batch_size 24 --dim_feedforward 3200 --num_epochs 5000 --lr 1e-4 --seed 0
|
||||||
|
|
||||||
|
|
||||||
|
#### NOTE to reproduce this experiment, uncomment the sim data filtering in utils.py
|
||||||
conda activate mobile
|
conda activate mobile
|
||||||
cd /home/tonyzhao/Research/act-plus-plus
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||||
@ -27,4 +28,50 @@ CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
|||||||
--hidden_dim 512 --batch_size 24 --dim_feedforward 3200 --lr 1e-4 --seed 0 \
|
--hidden_dim 512 --batch_size 24 --dim_feedforward 3200 --lr 1e-4 --seed 0 \
|
||||||
--num_steps 1000000 --eval_every 10000000000 --validate_every 2000 --save_every 5000
|
--num_steps 1000000 --eval_every 10000000000 --validate_every 2000 --save_every 5000
|
||||||
|
|
||||||
|
# generate mirrored data
|
||||||
|
conda activate mobile
|
||||||
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
|
python3 record_sim_episodes.py --task_name sim_transfer_cube_scripted_mirror --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted_mirror --num_episodes 50
|
||||||
|
python3 postprocess_episodes.py --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted_mirror --num_episodes 50
|
||||||
|
# the sim_transfer_cube_scripted_mirror will have 100 episodes
|
||||||
|
# I then copy the whole dir to sim_transfer_cube_scripted then removed all mirrored episodes
|
||||||
|
# this gives sim_transfer_cube_scripted_mirror (100 episodes) and sim_transfer_cube_scripted (50 episodes)
|
||||||
|
|
||||||
|
# visualize the original data
|
||||||
|
python3 visualize_episodes.py --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted_mirror --episode_idx 0
|
||||||
|
# visualize the artificially mirrored data
|
||||||
|
python3 visualize_episodes.py --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted_mirror --episode_idx 0 --ismirror
|
||||||
|
|
||||||
|
# sanity check
|
||||||
|
# replay the mirrored data action in the original env
|
||||||
|
python3 replay_episodes.py --dataset_path /scr/tonyzhao/datasets/sim_transfer_cube_scripted_mirror/mirror_episode_0.hdf5
|
||||||
|
# replay the original data action in the original env
|
||||||
|
python3 replay_episodes.py --dataset_path /scr/tonyzhao/datasets/sim_transfer_cube_scripted_mirror/episode_0.hdf5
|
||||||
|
|
||||||
|
|
||||||
|
# launch experiment on original data
|
||||||
|
conda activate mobile
|
||||||
|
export MUJOCO_GL=egl
|
||||||
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
|
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||||
|
--task_name sim_transfer_cube_scripted \
|
||||||
|
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted \
|
||||||
|
--policy_class ACT --kl_weight 10 --chunk_size 50 \
|
||||||
|
--hidden_dim 512 --batch_size 12 --dim_feedforward 3200 --lr 1e-5 --seed 0 \
|
||||||
|
--num_steps 100000 --eval_every 2000 --validate_every 2000 --save_every 2000
|
||||||
|
|
||||||
|
|
||||||
|
# launch experiment on all data
|
||||||
|
conda activate mobile
|
||||||
|
export MUJOCO_GL=egl
|
||||||
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
|
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||||
|
--task_name sim_transfer_cube_scripted_mirror \
|
||||||
|
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_mirror \
|
||||||
|
--policy_class ACT --kl_weight 10 --chunk_size 50 \
|
||||||
|
--hidden_dim 512 --batch_size 12 --dim_feedforward 3200 --lr 1e-5 --seed 0 \
|
||||||
|
--num_steps 50000 --eval_every 2000 --validate_every 2000 --save_every 2000
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
10
constants.py
10
constants.py
@ -7,7 +7,7 @@ SIM_TASK_CONFIGS = {
|
|||||||
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',
|
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',
|
||||||
'num_episodes': 50,
|
'num_episodes': 50,
|
||||||
'episode_len': 400,
|
'episode_len': 400,
|
||||||
'camera_names': ['top']
|
'camera_names': ['top', 'left_wrist', 'right_wrist']
|
||||||
},
|
},
|
||||||
|
|
||||||
'sim_transfer_cube_human':{
|
'sim_transfer_cube_human':{
|
||||||
@ -36,6 +36,14 @@ SIM_TASK_CONFIGS = {
|
|||||||
'episode_len': None,
|
'episode_len': None,
|
||||||
'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist']
|
'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist']
|
||||||
},
|
},
|
||||||
|
|
||||||
|
'sim_transfer_cube_scripted_mirror':{
|
||||||
|
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted_mirror',
|
||||||
|
'num_episodes': 50,
|
||||||
|
'episode_len': 400,
|
||||||
|
'camera_names': ['top', 'left_wrist', 'right_wrist']
|
||||||
|
},
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
### Simulation envs fixed constants
|
### Simulation envs fixed constants
|
||||||
|
|||||||
@ -136,8 +136,8 @@ class BimanualViperXEETask(base.Task):
|
|||||||
obs['env_state'] = self.get_env_state(physics)
|
obs['env_state'] = self.get_env_state(physics)
|
||||||
obs['images'] = dict()
|
obs['images'] = dict()
|
||||||
obs['images']['top'] = physics.render(height=480, width=640, camera_id='top')
|
obs['images']['top'] = physics.render(height=480, width=640, camera_id='top')
|
||||||
obs['images']['angle'] = physics.render(height=480, width=640, camera_id='angle')
|
# obs['images']['angle'] = physics.render(height=480, width=640, camera_id='angle')
|
||||||
obs['images']['vis'] = physics.render(height=480, width=640, camera_id='front_close')
|
# obs['images']['vis'] = physics.render(height=480, width=640, camera_id='front_close')
|
||||||
# used in scripted policy to obtain starting pose
|
# used in scripted policy to obtain starting pose
|
||||||
obs['mocap_pose_left'] = np.concatenate([physics.data.mocap_pos[0], physics.data.mocap_quat[0]]).copy()
|
obs['mocap_pose_left'] = np.concatenate([physics.data.mocap_pos[0], physics.data.mocap_quat[0]]).copy()
|
||||||
obs['mocap_pose_right'] = np.concatenate([physics.data.mocap_pos[1], physics.data.mocap_quat[1]]).copy()
|
obs['mocap_pose_right'] = np.concatenate([physics.data.mocap_pos[1], physics.data.mocap_quat[1]]).copy()
|
||||||
|
|||||||
@ -105,7 +105,8 @@ def main(args):
|
|||||||
if not os.path.isdir(ckpt_dir):
|
if not os.path.isdir(ckpt_dir):
|
||||||
os.makedirs(ckpt_dir)
|
os.makedirs(ckpt_dir)
|
||||||
config_path = os.path.join(ckpt_dir, 'config.pkl')
|
config_path = os.path.join(ckpt_dir, 'config.pkl')
|
||||||
wandb.init(project="mobile-aloha", reinit=True, entity="tonyzhao")
|
expr_name = ckpt_dir.split('/')[-1]
|
||||||
|
wandb.init(project="mobile-aloha", reinit=True, entity="tonyzhao", name=expr_name)
|
||||||
wandb.config.update(config)
|
wandb.config.update(config)
|
||||||
with open(config_path, 'wb') as f:
|
with open(config_path, 'wb') as f:
|
||||||
pickle.dump(config, f)
|
pickle.dump(config, f)
|
||||||
@ -113,7 +114,7 @@ def main(args):
|
|||||||
ckpt_names = [f'policy_last.ckpt']
|
ckpt_names = [f'policy_last.ckpt']
|
||||||
results = []
|
results = []
|
||||||
for ckpt_name in ckpt_names:
|
for ckpt_name in ckpt_names:
|
||||||
success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True)
|
success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50)
|
||||||
wandb.log({'success_rate': success_rate, 'avg_return': avg_return})
|
wandb.log({'success_rate': success_rate, 'avg_return': avg_return})
|
||||||
results.append([ckpt_name, success_rate, avg_return])
|
results.append([ckpt_name, success_rate, avg_return])
|
||||||
|
|
||||||
@ -169,7 +170,7 @@ def get_image(ts, camera_names):
|
|||||||
return curr_image
|
return curr_image
|
||||||
|
|
||||||
|
|
||||||
def eval_bc(config, ckpt_name, save_episode=True):
|
def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
|
||||||
set_seed(1000)
|
set_seed(1000)
|
||||||
ckpt_dir = config['ckpt_dir']
|
ckpt_dir = config['ckpt_dir']
|
||||||
state_dim = config['state_dim']
|
state_dim = config['state_dim']
|
||||||
@ -227,7 +228,6 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
|||||||
|
|
||||||
max_timesteps = int(max_timesteps * 1) # may increase for real-world tasks
|
max_timesteps = int(max_timesteps * 1) # may increase for real-world tasks
|
||||||
|
|
||||||
num_rollouts = 50
|
|
||||||
episode_returns = []
|
episode_returns = []
|
||||||
highest_rewards = []
|
highest_rewards = []
|
||||||
for rollout_id in range(num_rollouts):
|
for rollout_id in range(num_rollouts):
|
||||||
@ -311,7 +311,10 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
|||||||
base_action = action[-2:]
|
base_action = action[-2:]
|
||||||
|
|
||||||
### step the environment
|
### step the environment
|
||||||
ts = env.step(target_qpos, base_action)
|
if real_robot:
|
||||||
|
ts = env.step(target_qpos, base_action)
|
||||||
|
else:
|
||||||
|
ts = env.step(target_qpos)
|
||||||
|
|
||||||
### for visualization
|
### for visualization
|
||||||
qpos_list.append(qpos_numpy)
|
qpos_list.append(qpos_numpy)
|
||||||
@ -409,10 +412,12 @@ def train_bc(train_dataloader, val_dataloader, config):
|
|||||||
|
|
||||||
# evaluation
|
# evaluation
|
||||||
if (step > 0) and (step % eval_every == 0):
|
if (step > 0) and (step % eval_every == 0):
|
||||||
state_dict = deepcopy(policy.state_dict())
|
# first save then eval
|
||||||
success_with_te, _ = eval_bc(config, f'TE_step{step}', state_dict, temporal_agg=True, save_episode=True, num_rollouts=5)
|
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
|
||||||
success_without_te, _ = eval_bc(config, f'step{step}', state_dict, temporal_agg=False, save_episode=True, num_rollouts=5)
|
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
wandb.log({'success_with_te':success_with_te, 'success_without_te':success_without_te}, step=step)
|
torch.save(policy.state_dict(), ckpt_path)
|
||||||
|
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=10)
|
||||||
|
wandb.log({'success': success}, step=step)
|
||||||
|
|
||||||
# training
|
# training
|
||||||
policy.train()
|
policy.train()
|
||||||
|
|||||||
94
postprocess_episodes.py
普通文件
94
postprocess_episodes.py
普通文件
@ -0,0 +1,94 @@
|
|||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import cv2
|
||||||
|
import h5py
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
from visualize_episodes import visualize_joints, visualize_timestamp, save_videos
|
||||||
|
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from constants import DT
|
||||||
|
|
||||||
|
import IPython
|
||||||
|
e = IPython.embed
|
||||||
|
|
||||||
|
JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"]
|
||||||
|
STATE_NAMES = JOINT_NAMES + ["gripper"]
|
||||||
|
|
||||||
|
MIRROR_STATE_MULTIPLY = np.array([-1, 1, 1, -1, 1, -1, 1])[None, ...].astype('float32')
|
||||||
|
|
||||||
|
|
||||||
|
def load_hdf5(dataset_dir, dataset_name):
|
||||||
|
dataset_path = os.path.join(dataset_dir, dataset_name + '.hdf5')
|
||||||
|
if not os.path.isfile(dataset_path):
|
||||||
|
print(f'Dataset does not exist at \n{dataset_path}\n')
|
||||||
|
exit()
|
||||||
|
|
||||||
|
with h5py.File(dataset_path, 'r') as root:
|
||||||
|
is_sim = root.attrs['sim']
|
||||||
|
qpos = root['/observations/qpos'][()]
|
||||||
|
qvel = root['/observations/qvel'][()]
|
||||||
|
action = root['/action'][()]
|
||||||
|
image_dict = dict()
|
||||||
|
for cam_name in root[f'/observations/images/'].keys():
|
||||||
|
image_dict[cam_name] = root[f'/observations/images/{cam_name}'][()]
|
||||||
|
|
||||||
|
return qpos, qvel, action, image_dict
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
dataset_dir = args['dataset_dir']
|
||||||
|
num_episodes = args['num_episodes']
|
||||||
|
|
||||||
|
for episode_idx in range(num_episodes):
|
||||||
|
dataset_name = f'episode_{episode_idx}'
|
||||||
|
|
||||||
|
qpos, qvel, action, image_dict = load_hdf5(dataset_dir, dataset_name)
|
||||||
|
|
||||||
|
# process proprioception
|
||||||
|
qpos = np.concatenate([qpos[:, 7:] * MIRROR_STATE_MULTIPLY, qpos[:, :7] * MIRROR_STATE_MULTIPLY], axis=1)
|
||||||
|
qvel = np.concatenate([qvel[:, 7:] * MIRROR_STATE_MULTIPLY, qvel[:, :7] * MIRROR_STATE_MULTIPLY], axis=1)
|
||||||
|
action = np.concatenate([action[:, 7:] * MIRROR_STATE_MULTIPLY, action[:, :7] * MIRROR_STATE_MULTIPLY], axis=1)
|
||||||
|
|
||||||
|
# mirror image obs
|
||||||
|
image_dict['left_wrist'], image_dict['right_wrist'] = image_dict['right_wrist'][:, :, ::-1], image_dict['left_wrist'][:, :, ::-1]
|
||||||
|
image_dict['top'] = image_dict['top'][:, :, ::-1]
|
||||||
|
|
||||||
|
# saving
|
||||||
|
data_dict = {
|
||||||
|
'/observations/qpos': qpos,
|
||||||
|
'/observations/qvel': qvel,
|
||||||
|
'/action': action,
|
||||||
|
}
|
||||||
|
for cam_name in image_dict.keys():
|
||||||
|
data_dict[f'/observations/images/{cam_name}'] = image_dict[cam_name]
|
||||||
|
max_timesteps = len(qpos)
|
||||||
|
|
||||||
|
# HDF5
|
||||||
|
t0 = time.time()
|
||||||
|
dataset_path = os.path.join(dataset_dir, f'mirror_episode_{episode_idx}')
|
||||||
|
with h5py.File(dataset_path + '.hdf5', 'w', rdcc_nbytes=1024 ** 2 * 2) as root:
|
||||||
|
root.attrs['sim'] = True
|
||||||
|
obs = root.create_group('observations')
|
||||||
|
image = obs.create_group('images')
|
||||||
|
for cam_name in image_dict.keys():
|
||||||
|
_ = image.create_dataset(cam_name, (max_timesteps, 480, 640, 3), dtype='uint8',
|
||||||
|
chunks=(1, 480, 640, 3), )
|
||||||
|
# compression='gzip',compression_opts=2,)
|
||||||
|
# compression=32001, compression_opts=(0, 0, 0, 0, 9, 1, 1), shuffle=False)
|
||||||
|
qpos = obs.create_dataset('qpos', (max_timesteps, 14))
|
||||||
|
qvel = obs.create_dataset('qvel', (max_timesteps, 14))
|
||||||
|
action = root.create_dataset('action', (max_timesteps, 14))
|
||||||
|
|
||||||
|
for name, array in data_dict.items():
|
||||||
|
root[name][...] = array
|
||||||
|
print(f'Saving {dataset_path}: {time.time() - t0:.1f} secs\n')
|
||||||
|
|
||||||
|
# save_videos(image_dict, DT, video_path=os.path.join(dataset_dir, dataset_name + '_mirror_video.mp4'))
|
||||||
|
# visualize_joints(qpos, action, plot_path=os.path.join(dataset_dir, dataset_name + '_mirror_qpos.png'))
|
||||||
|
# visualize_timestamp(t_list, dataset_path) # TODO addn timestamp back
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--dataset_dir', action='store', type=str, help='Dataset dir.', required=True)
|
||||||
|
parser.add_argument('--num_episodes', action='store', type=int, help='Number of episodes.', required=True)
|
||||||
|
main(vars(parser.parse_args()))
|
||||||
@ -28,7 +28,7 @@ def main(args):
|
|||||||
num_episodes = args['num_episodes']
|
num_episodes = args['num_episodes']
|
||||||
onscreen_render = args['onscreen_render']
|
onscreen_render = args['onscreen_render']
|
||||||
inject_noise = False
|
inject_noise = False
|
||||||
render_cam_name = 'angle'
|
render_cam_name = 'top'
|
||||||
|
|
||||||
if not os.path.isdir(dataset_dir):
|
if not os.path.isdir(dataset_dir):
|
||||||
os.makedirs(dataset_dir, exist_ok=True)
|
os.makedirs(dataset_dir, exist_ok=True)
|
||||||
@ -39,6 +39,8 @@ def main(args):
|
|||||||
policy_cls = PickAndTransferPolicy
|
policy_cls = PickAndTransferPolicy
|
||||||
elif task_name == 'sim_insertion_scripted':
|
elif task_name == 'sim_insertion_scripted':
|
||||||
policy_cls = InsertionPolicy
|
policy_cls = InsertionPolicy
|
||||||
|
elif task_name == 'sim_transfer_cube_scripted_mirror':
|
||||||
|
policy_cls = PickAndTransferPolicy
|
||||||
else:
|
else:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|||||||
48
replay_episodes.py
普通文件
48
replay_episodes.py
普通文件
@ -0,0 +1,48 @@
|
|||||||
|
import os
|
||||||
|
import h5py
|
||||||
|
import argparse
|
||||||
|
from collections import defaultdict
|
||||||
|
from sim_env import make_sim_env
|
||||||
|
from utils import sample_box_pose, sample_insertion_pose
|
||||||
|
from sim_env import BOX_POSE
|
||||||
|
from constants import DT
|
||||||
|
from visualize_episodes import save_videos
|
||||||
|
|
||||||
|
import IPython
|
||||||
|
e = IPython.embed
|
||||||
|
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
dataset_path = args['dataset_path']
|
||||||
|
|
||||||
|
|
||||||
|
if not os.path.isfile(dataset_path):
|
||||||
|
print(f'Dataset does not exist at \n{dataset_path}\n')
|
||||||
|
exit()
|
||||||
|
|
||||||
|
with h5py.File(dataset_path, 'r') as root:
|
||||||
|
actions = root['/action'][()]
|
||||||
|
|
||||||
|
env = make_sim_env('sim_transfer_cube')
|
||||||
|
BOX_POSE[0] = sample_box_pose() # used in sim reset
|
||||||
|
ts = env.reset()
|
||||||
|
episode_replay = [ts]
|
||||||
|
for action in actions:
|
||||||
|
ts = env.step(action)
|
||||||
|
episode_replay.append(ts)
|
||||||
|
|
||||||
|
# saving
|
||||||
|
image_dict = defaultdict(lambda: [])
|
||||||
|
while episode_replay:
|
||||||
|
ts = episode_replay.pop(0)
|
||||||
|
for cam_name, image in ts.observation['images'].items():
|
||||||
|
image_dict[cam_name].append(image)
|
||||||
|
|
||||||
|
video_path = dataset_path.replace('episode_', 'replay_episode_').replace('hdf5', 'mp4')
|
||||||
|
save_videos(image_dict, DT, video_path=video_path)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--dataset_path', action='store', type=str, help='Dataset path.', required=True)
|
||||||
|
main(vars(parser.parse_args()))
|
||||||
@ -108,8 +108,10 @@ class BimanualViperXTask(base.Task):
|
|||||||
obs['env_state'] = self.get_env_state(physics)
|
obs['env_state'] = self.get_env_state(physics)
|
||||||
obs['images'] = dict()
|
obs['images'] = dict()
|
||||||
obs['images']['top'] = physics.render(height=480, width=640, camera_id='top')
|
obs['images']['top'] = physics.render(height=480, width=640, camera_id='top')
|
||||||
obs['images']['angle'] = physics.render(height=480, width=640, camera_id='angle')
|
obs['images']['left_wrist'] = physics.render(height=480, width=640, camera_id='left_wrist')
|
||||||
obs['images']['vis'] = physics.render(height=480, width=640, camera_id='front_close')
|
obs['images']['right_wrist'] = physics.render(height=480, width=640, camera_id='right_wrist')
|
||||||
|
# obs['images']['angle'] = physics.render(height=480, width=640, camera_id='angle')
|
||||||
|
# obs['images']['vis'] = physics.render(height=480, width=640, camera_id='front_close')
|
||||||
|
|
||||||
return obs
|
return obs
|
||||||
|
|
||||||
|
|||||||
2
utils.py
2
utils.py
@ -38,7 +38,7 @@ class EpisodicDataset(torch.utils.data.Dataset):
|
|||||||
try:
|
try:
|
||||||
# print(dataset_path)
|
# print(dataset_path)
|
||||||
with h5py.File(dataset_path, 'r') as root:
|
with h5py.File(dataset_path, 'r') as root:
|
||||||
is_sim = False # root.attrs['sim']
|
is_sim = root.attrs['sim']
|
||||||
if '/base_action' in root:
|
if '/base_action' in root:
|
||||||
base_action = root['/base_action'][()]
|
base_action = root['/base_action'][()]
|
||||||
base_action = preprocess_base_action(base_action)
|
base_action = preprocess_base_action(base_action)
|
||||||
|
|||||||
@ -33,7 +33,11 @@ def load_hdf5(dataset_dir, dataset_name):
|
|||||||
def main(args):
|
def main(args):
|
||||||
dataset_dir = args['dataset_dir']
|
dataset_dir = args['dataset_dir']
|
||||||
episode_idx = args['episode_idx']
|
episode_idx = args['episode_idx']
|
||||||
dataset_name = f'episode_{episode_idx}'
|
ismirror = args['ismirror']
|
||||||
|
if ismirror:
|
||||||
|
dataset_name = f'mirror_episode_{episode_idx}'
|
||||||
|
else:
|
||||||
|
dataset_name = f'episode_{episode_idx}'
|
||||||
|
|
||||||
qpos, qvel, action, image_dict = load_hdf5(dataset_dir, dataset_name)
|
qpos, qvel, action, image_dict = load_hdf5(dataset_dir, dataset_name)
|
||||||
save_videos(image_dict, DT, video_path=os.path.join(dataset_dir, dataset_name + '_video.mp4'))
|
save_videos(image_dict, DT, video_path=os.path.join(dataset_dir, dataset_name + '_video.mp4'))
|
||||||
@ -44,6 +48,7 @@ def main(args):
|
|||||||
def save_videos(video, dt, video_path=None):
|
def save_videos(video, dt, video_path=None):
|
||||||
if isinstance(video, list):
|
if isinstance(video, list):
|
||||||
cam_names = list(video[0].keys())
|
cam_names = list(video[0].keys())
|
||||||
|
cam_names = sorted(cam_names)
|
||||||
h, w, _ = video[0][cam_names[0]].shape
|
h, w, _ = video[0][cam_names[0]].shape
|
||||||
w = w * len(cam_names)
|
w = w * len(cam_names)
|
||||||
fps = int(1/dt)
|
fps = int(1/dt)
|
||||||
@ -60,6 +65,7 @@ def save_videos(video, dt, video_path=None):
|
|||||||
print(f'Saved video to: {video_path}')
|
print(f'Saved video to: {video_path}')
|
||||||
elif isinstance(video, dict):
|
elif isinstance(video, dict):
|
||||||
cam_names = list(video.keys())
|
cam_names = list(video.keys())
|
||||||
|
cam_names = sorted(cam_names)
|
||||||
all_cam_videos = []
|
all_cam_videos = []
|
||||||
for cam_name in cam_names:
|
for cam_name in cam_names:
|
||||||
all_cam_videos.append(video[cam_name])
|
all_cam_videos.append(video[cam_name])
|
||||||
@ -144,4 +150,5 @@ if __name__ == '__main__':
|
|||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument('--dataset_dir', action='store', type=str, help='Dataset dir.', required=True)
|
parser.add_argument('--dataset_dir', action='store', type=str, help='Dataset dir.', required=True)
|
||||||
parser.add_argument('--episode_idx', action='store', type=int, help='Episode index.', required=False)
|
parser.add_argument('--episode_idx', action='store', type=int, help='Episode index.', required=False)
|
||||||
|
parser.add_argument('--ismirror', action='store_true')
|
||||||
main(vars(parser.parse_args()))
|
main(vars(parser.parse_args()))
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户