vinn_eval pipeline

这个提交包含在:
Zipeng Fu
2023-12-11 21:52:59 -08:00
父节点 0247d3669c
当前提交 61ffa1bdb1
共有 4 个文件被更改,包括 70 次插入 和 24 次删除

子模块 byol_pytorch 已更新:e782c4b828...bcd1b066c3

查看文件

@ -248,6 +248,28 @@ CUDA_VISIBLE_DEVICES=1 python3 train.py --task aloha_mobile_elevator_truncated_c
# eval on real robot
conda activate aloha
cd /home/mobile-aloha/interbotix_ws/src/act
TASK_NAME=aloha_mobile_wipe_wine
python3 vinn_cache_feature.py --ckpt_path /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/byol-${TASK_NAME}-DUMMY-seed-0.pt
TASK_NAME=aloha_mobile_wipe_wine
python3 vinn_select_k.py \
--dataset_dir /home/mobile-aloha/data/${TASK_NAME} \
--ckpt_dir /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/VINN-eval-seed-0-test \
TASK_NAME=aloha_mobile_wipe_wine
python3 vinn_eval.py \
--dataset_dir /home/mobile-aloha/data/${TASK_NAME} \
--model_dir /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/byol-${TASK_NAME}-DUMMY-seed-0.pt \
--ckpt_dir /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/VINN-eval-seed-0-test \
--task_name $TASK_NAME
--------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------
NOTE: chunk size cannot be any number, try before launching NOTE: chunk size cannot be any number, try before launching

查看文件

@ -37,7 +37,7 @@ def main(args):
repr_type = ckpt_name.split('-')[0] repr_type = ckpt_name.split('-')[0]
seed = int(ckpt_name.split('-')[-1][:-3]) seed = int(ckpt_name.split('-')[-1][:-3])
dataset_dir = f'/scr/tonyzhao/mobile_aloha_datasets/{dataset_name}' dataset_dir = f'/home/mobile-aloha/data/{dataset_name}'
episode_idxs = [int(name.split('_')[1].split('.')[0]) for name in os.listdir(dataset_dir) if ('.hdf5' in name) and ('features' not in name)] episode_idxs = [int(name.split('_')[1].split('.')[0]) for name in os.listdir(dataset_dir) if ('.hdf5' in name) and ('features' not in name)]
episode_idxs.sort() episode_idxs.sort()
@ -100,6 +100,7 @@ def main(args):
all_features = [] all_features = []
with torch.inference_mode(): with torch.inference_mode():
for batch in chunks(processed_images, batch_size): for batch in chunks(processed_images, batch_size):
print('inference')
features = feature_extractors[cam_name](batch) features = feature_extractors[cam_name](batch)
features = features.squeeze(axis=3).squeeze(axis=2) features = features.squeeze(axis=3).squeeze(axis=2)
all_features.append(features) all_features.append(features)

查看文件

@ -17,8 +17,9 @@ from sim_env import BOX_POSE
from constants import DT from constants import DT
from imitate_episodes import save_videos from imitate_episodes import save_videos
from einops import rearrange from einops import rearrange
import time
DT = 0.02
import IPython import IPython
e = IPython.embed e = IPython.embed
@ -56,7 +57,7 @@ def main(args):
# TODO ###################### # TODO ######################
k = None # for scripted box transfer k = None # for scripted box transfer
skip = 100 skip = 100
real_robot = False real_robot = True
save_episode = True save_episode = True
# TODO ###################### # TODO ######################
onscreen_cam = 'main' onscreen_cam = 'main'
@ -84,6 +85,12 @@ def main(args):
env_max_reward = 1 env_max_reward = 1
ks = [71] ks = [71]
state_weight = 0 state_weight = 0
elif task_name == 'aloha_mobile_wipe_wine':
sim_episode_len = 1300
env_max_reward = 4
ks = [2, 2, 2]
state_weight = 5
print(f'{state_weight=}')
else: else:
raise NotImplementedError raise NotImplementedError
@ -92,6 +99,10 @@ def main(args):
k = ks[seed] k = ks[seed]
if real_robot:
BASE_DELAY = 15
query_freq = skip - BASE_DELAY
# load train data # load train data
vis_features = [] vis_features = []
state_features = [] state_features = []
@ -100,6 +111,8 @@ def main(args):
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5') dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
with h5py.File(dataset_path, 'r') as root: with h5py.File(dataset_path, 'r') as root:
action = root['/action'][:] action = root['/action'][:]
base_action = root['/base_action'][:]
action = np.concatenate([action, base_action], axis=1)
camera_names = list(root[f'/observations/images/'].keys()) camera_names = list(root[f'/observations/images/'].keys())
# Visual feature # Visual feature
@ -151,10 +164,10 @@ def main(args):
# load environment # load environment
if real_robot: if real_robot:
from real_env import make_real_env from aloha_scripts.real_env import make_real_env #### TODO TODO
env = make_real_env(init_node=True) env = make_real_env(init_node=True, setup_robots=True, setup_base=True)
max_timesteps = EPISODE_LEN max_timesteps = sim_episode_len
camera_names = CAMERA_NAMES camera_names = ['cam_high', 'cam_left_wrist', 'cam_right_wrist']
else: else:
from sim_env import make_sim_env from sim_env import make_sim_env
env = make_sim_env(task_name) env = make_sim_env(task_name)
@ -177,19 +190,21 @@ def main(args):
rewards = [] rewards = []
with torch.inference_mode(): with torch.inference_mode():
for t in range(sim_episode_len): for t in range(sim_episode_len):
### process previous timestep to get qpos and image_list start_time = time.time()
obs = ts.observation if t % 100 == 0: print(t)
if 'images' in obs: if t % query_freq == 0:
image_list.append(obs['images']) ### process previous timestep to get qpos and image_list
else: obs = ts.observation
image_list.append({'main': obs['image']}) if 'images' in obs:
qpos_numpy = np.array(obs['qpos']) image_list.append(obs['images'])
# qpos = pre_process(qpos_numpy) else:
qpos = torch.from_numpy(qpos_numpy).float().cuda().unsqueeze(0) image_list.append({'main': obs['image']})
qpos_history[:, t] = qpos qpos_numpy = np.array(obs['qpos'])
_, curr_image_raw = get_image(ts, camera_names) # qpos = pre_process(qpos_numpy)
qpos = torch.from_numpy(qpos_numpy).float().cuda().unsqueeze(0)
qpos_history[:, t] = qpos
_, curr_image_raw = get_image(ts, camera_names)
if t % skip == 0:
image_size = 120 image_size = 120
transform = transforms.Compose([ transform = transforms.Compose([
transforms.Resize(image_size), # will scale the image transforms.Resize(image_size), # will scale the image
@ -220,9 +235,14 @@ def main(args):
### Both features ### Both features
curr_feature = [curr_image_feature, qpos] curr_feature = [curr_image_feature, qpos]
all_target_qpos = calculate_nearest_neighbors(curr_feature, train_inputs, train_targets, k, state_weight) # TODO use this action = calculate_nearest_neighbors(curr_feature, train_inputs, train_targets, k, state_weight) # TODO use this
all_target_qpos = all_target_qpos.squeeze(0).cpu().numpy() action = action.squeeze(0).cpu().numpy()
target_qpos = all_target_qpos[t % skip] action = np.concatenate([action[:-BASE_DELAY, :-2], action[BASE_DELAY:, -2:]], axis=1)
print(f'Query: {(time.time() - start_time):.3f}s')
curr_action = action[t % query_freq]
target_qpos = curr_action[:-2]
base_action = curr_action[-2:]
# ### SAFETY # ### SAFETY
# max_a = 0.05 # max_a = 0.05
@ -231,7 +251,10 @@ def main(args):
# ### SAFETY # ### SAFETY
### step the environment ### step the environment
ts = env.step(target_qpos) ts = env.step(target_qpos, base_action=base_action)
duration = time.time() - start_time
# print(f'{duration:.3f}')
time.sleep(max(0, DT - duration))
### save things for visualization ### save things for visualization
qpos_list.append(qpos_numpy) qpos_list.append(qpos_numpy)