vinn_eval pipeline
这个提交包含在:
子模块 byol_pytorch 已更新:e782c4b828...bcd1b066c3
22
commands.txt
22
commands.txt
@ -248,6 +248,28 @@ CUDA_VISIBLE_DEVICES=1 python3 train.py --task aloha_mobile_elevator_truncated_c
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# eval on real robot
|
||||||
|
|
||||||
|
conda activate aloha
|
||||||
|
cd /home/mobile-aloha/interbotix_ws/src/act
|
||||||
|
TASK_NAME=aloha_mobile_wipe_wine
|
||||||
|
python3 vinn_cache_feature.py --ckpt_path /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/byol-${TASK_NAME}-DUMMY-seed-0.pt
|
||||||
|
|
||||||
|
|
||||||
|
TASK_NAME=aloha_mobile_wipe_wine
|
||||||
|
python3 vinn_select_k.py \
|
||||||
|
--dataset_dir /home/mobile-aloha/data/${TASK_NAME} \
|
||||||
|
--ckpt_dir /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/VINN-eval-seed-0-test \
|
||||||
|
|
||||||
|
|
||||||
|
TASK_NAME=aloha_mobile_wipe_wine
|
||||||
|
python3 vinn_eval.py \
|
||||||
|
--dataset_dir /home/mobile-aloha/data/${TASK_NAME} \
|
||||||
|
--model_dir /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/byol-${TASK_NAME}-DUMMY-seed-0.pt \
|
||||||
|
--ckpt_dir /home/mobile-aloha/interbotix_ws/src/act/ckpts/vinn_ckpts/VINN-eval-seed-0-test \
|
||||||
|
--task_name $TASK_NAME
|
||||||
|
|
||||||
|
|
||||||
---------------------------------------------------------------------------------------
|
---------------------------------------------------------------------------------------
|
||||||
|
|
||||||
NOTE: chunk size cannot be any number, try before launching
|
NOTE: chunk size cannot be any number, try before launching
|
||||||
|
|||||||
@ -37,7 +37,7 @@ def main(args):
|
|||||||
repr_type = ckpt_name.split('-')[0]
|
repr_type = ckpt_name.split('-')[0]
|
||||||
seed = int(ckpt_name.split('-')[-1][:-3])
|
seed = int(ckpt_name.split('-')[-1][:-3])
|
||||||
|
|
||||||
dataset_dir = f'/scr/tonyzhao/mobile_aloha_datasets/{dataset_name}'
|
dataset_dir = f'/home/mobile-aloha/data/{dataset_name}'
|
||||||
|
|
||||||
episode_idxs = [int(name.split('_')[1].split('.')[0]) for name in os.listdir(dataset_dir) if ('.hdf5' in name) and ('features' not in name)]
|
episode_idxs = [int(name.split('_')[1].split('.')[0]) for name in os.listdir(dataset_dir) if ('.hdf5' in name) and ('features' not in name)]
|
||||||
episode_idxs.sort()
|
episode_idxs.sort()
|
||||||
@ -100,6 +100,7 @@ def main(args):
|
|||||||
all_features = []
|
all_features = []
|
||||||
with torch.inference_mode():
|
with torch.inference_mode():
|
||||||
for batch in chunks(processed_images, batch_size):
|
for batch in chunks(processed_images, batch_size):
|
||||||
|
print('inference')
|
||||||
features = feature_extractors[cam_name](batch)
|
features = feature_extractors[cam_name](batch)
|
||||||
features = features.squeeze(axis=3).squeeze(axis=2)
|
features = features.squeeze(axis=3).squeeze(axis=2)
|
||||||
all_features.append(features)
|
all_features.append(features)
|
||||||
|
|||||||
67
vinn_eval.py
67
vinn_eval.py
@ -17,8 +17,9 @@ from sim_env import BOX_POSE
|
|||||||
from constants import DT
|
from constants import DT
|
||||||
from imitate_episodes import save_videos
|
from imitate_episodes import save_videos
|
||||||
from einops import rearrange
|
from einops import rearrange
|
||||||
|
import time
|
||||||
|
|
||||||
|
DT = 0.02
|
||||||
import IPython
|
import IPython
|
||||||
e = IPython.embed
|
e = IPython.embed
|
||||||
|
|
||||||
@ -56,7 +57,7 @@ def main(args):
|
|||||||
# TODO ######################
|
# TODO ######################
|
||||||
k = None # for scripted box transfer
|
k = None # for scripted box transfer
|
||||||
skip = 100
|
skip = 100
|
||||||
real_robot = False
|
real_robot = True
|
||||||
save_episode = True
|
save_episode = True
|
||||||
# TODO ######################
|
# TODO ######################
|
||||||
onscreen_cam = 'main'
|
onscreen_cam = 'main'
|
||||||
@ -84,6 +85,12 @@ def main(args):
|
|||||||
env_max_reward = 1
|
env_max_reward = 1
|
||||||
ks = [71]
|
ks = [71]
|
||||||
state_weight = 0
|
state_weight = 0
|
||||||
|
elif task_name == 'aloha_mobile_wipe_wine':
|
||||||
|
sim_episode_len = 1300
|
||||||
|
env_max_reward = 4
|
||||||
|
ks = [2, 2, 2]
|
||||||
|
state_weight = 5
|
||||||
|
print(f'{state_weight=}')
|
||||||
else:
|
else:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@ -92,6 +99,10 @@ def main(args):
|
|||||||
|
|
||||||
k = ks[seed]
|
k = ks[seed]
|
||||||
|
|
||||||
|
if real_robot:
|
||||||
|
BASE_DELAY = 15
|
||||||
|
query_freq = skip - BASE_DELAY
|
||||||
|
|
||||||
# load train data
|
# load train data
|
||||||
vis_features = []
|
vis_features = []
|
||||||
state_features = []
|
state_features = []
|
||||||
@ -100,6 +111,8 @@ def main(args):
|
|||||||
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
|
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
|
||||||
with h5py.File(dataset_path, 'r') as root:
|
with h5py.File(dataset_path, 'r') as root:
|
||||||
action = root['/action'][:]
|
action = root['/action'][:]
|
||||||
|
base_action = root['/base_action'][:]
|
||||||
|
action = np.concatenate([action, base_action], axis=1)
|
||||||
camera_names = list(root[f'/observations/images/'].keys())
|
camera_names = list(root[f'/observations/images/'].keys())
|
||||||
|
|
||||||
# Visual feature
|
# Visual feature
|
||||||
@ -151,10 +164,10 @@ def main(args):
|
|||||||
|
|
||||||
# load environment
|
# load environment
|
||||||
if real_robot:
|
if real_robot:
|
||||||
from real_env import make_real_env
|
from aloha_scripts.real_env import make_real_env #### TODO TODO
|
||||||
env = make_real_env(init_node=True)
|
env = make_real_env(init_node=True, setup_robots=True, setup_base=True)
|
||||||
max_timesteps = EPISODE_LEN
|
max_timesteps = sim_episode_len
|
||||||
camera_names = CAMERA_NAMES
|
camera_names = ['cam_high', 'cam_left_wrist', 'cam_right_wrist']
|
||||||
else:
|
else:
|
||||||
from sim_env import make_sim_env
|
from sim_env import make_sim_env
|
||||||
env = make_sim_env(task_name)
|
env = make_sim_env(task_name)
|
||||||
@ -177,19 +190,21 @@ def main(args):
|
|||||||
rewards = []
|
rewards = []
|
||||||
with torch.inference_mode():
|
with torch.inference_mode():
|
||||||
for t in range(sim_episode_len):
|
for t in range(sim_episode_len):
|
||||||
### process previous timestep to get qpos and image_list
|
start_time = time.time()
|
||||||
obs = ts.observation
|
if t % 100 == 0: print(t)
|
||||||
if 'images' in obs:
|
if t % query_freq == 0:
|
||||||
image_list.append(obs['images'])
|
### process previous timestep to get qpos and image_list
|
||||||
else:
|
obs = ts.observation
|
||||||
image_list.append({'main': obs['image']})
|
if 'images' in obs:
|
||||||
qpos_numpy = np.array(obs['qpos'])
|
image_list.append(obs['images'])
|
||||||
# qpos = pre_process(qpos_numpy)
|
else:
|
||||||
qpos = torch.from_numpy(qpos_numpy).float().cuda().unsqueeze(0)
|
image_list.append({'main': obs['image']})
|
||||||
qpos_history[:, t] = qpos
|
qpos_numpy = np.array(obs['qpos'])
|
||||||
_, curr_image_raw = get_image(ts, camera_names)
|
# qpos = pre_process(qpos_numpy)
|
||||||
|
qpos = torch.from_numpy(qpos_numpy).float().cuda().unsqueeze(0)
|
||||||
|
qpos_history[:, t] = qpos
|
||||||
|
_, curr_image_raw = get_image(ts, camera_names)
|
||||||
|
|
||||||
if t % skip == 0:
|
|
||||||
image_size = 120
|
image_size = 120
|
||||||
transform = transforms.Compose([
|
transform = transforms.Compose([
|
||||||
transforms.Resize(image_size), # will scale the image
|
transforms.Resize(image_size), # will scale the image
|
||||||
@ -220,9 +235,14 @@ def main(args):
|
|||||||
### Both features
|
### Both features
|
||||||
curr_feature = [curr_image_feature, qpos]
|
curr_feature = [curr_image_feature, qpos]
|
||||||
|
|
||||||
all_target_qpos = calculate_nearest_neighbors(curr_feature, train_inputs, train_targets, k, state_weight) # TODO use this
|
action = calculate_nearest_neighbors(curr_feature, train_inputs, train_targets, k, state_weight) # TODO use this
|
||||||
all_target_qpos = all_target_qpos.squeeze(0).cpu().numpy()
|
action = action.squeeze(0).cpu().numpy()
|
||||||
target_qpos = all_target_qpos[t % skip]
|
action = np.concatenate([action[:-BASE_DELAY, :-2], action[BASE_DELAY:, -2:]], axis=1)
|
||||||
|
print(f'Query: {(time.time() - start_time):.3f}s')
|
||||||
|
|
||||||
|
curr_action = action[t % query_freq]
|
||||||
|
target_qpos = curr_action[:-2]
|
||||||
|
base_action = curr_action[-2:]
|
||||||
|
|
||||||
# ### SAFETY
|
# ### SAFETY
|
||||||
# max_a = 0.05
|
# max_a = 0.05
|
||||||
@ -231,7 +251,10 @@ def main(args):
|
|||||||
# ### SAFETY
|
# ### SAFETY
|
||||||
|
|
||||||
### step the environment
|
### step the environment
|
||||||
ts = env.step(target_qpos)
|
ts = env.step(target_qpos, base_action=base_action)
|
||||||
|
duration = time.time() - start_time
|
||||||
|
# print(f'{duration:.3f}')
|
||||||
|
time.sleep(max(0, DT - duration))
|
||||||
|
|
||||||
### save things for visualization
|
### save things for visualization
|
||||||
qpos_list.append(qpos_numpy)
|
qpos_list.append(qpos_numpy)
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户