VINN works for sim transfer cube

这个提交包含在:
Tony Zhao
2023-12-10 20:18:46 -08:00
父节点 c460687c33
当前提交 91f1bbf6e2
共有 6 个文件被更改,包括 621 次插入 和 0 次删除

1
byol_pytorch 子模块

子模块 byol_pytorch 已添加到 25e5b3fbf8

查看文件

@ -113,6 +113,56 @@ CUDA_VISIBLE_DEVICES=1 python3 imitate_episodes.py \
--batch_size 32 --lr 1e-4 --seed 0 \ --batch_size 32 --lr 1e-4 --seed 0 \
--num_steps 100000 --eval_every 2000 --validate_every 2000 --save_every 2000 --num_steps 100000 --eval_every 2000 --validate_every 2000 --save_every 2000
# Dec 10
######################## more diffusion ########################
conda activate mobile
export MUJOCO_GL=egl
cd /home/tonyzhao/Research/act-plus-plus
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
--task_name sim_transfer_cube_scripted \
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_3_chunk64 \
--policy_class Diffusion --chunk_size 64 \
--batch_size 32 --lr 1e-4 --seed 0 \
--num_steps 200000 --eval_every 4000 --validate_every 4000 --save_every 4000
######################## VINN ########################
conda activate mobile
cd /home/tonyzhao/Research/act-plus-plus/byol_pytorch/examples/lightning
CUDA_VISIBLE_DEVICES=1 python3 train.py --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted --cam_name top --seed 0
conda activate mobile
cd /home/tonyzhao/Research/act-plus-plus/byol_pytorch/examples/lightning
CUDA_VISIBLE_DEVICES=0 python3 train.py --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted --cam_name left_wrist --seed 0
conda activate mobile
cd /home/tonyzhao/Research/act-plus-plus/byol_pytorch/examples/lightning
CUDA_VISIBLE_DEVICES=1 python3 train.py --dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted --cam_name right_wrist --seed 0
conda activate mobile
cd /home/tonyzhao/Research/act-plus-plus
TASK_NAME=sim_transfer_cube_scripted
python3 vinn_cache_feature.py --ckpt_path /home/tonyzhao/Research/act-plus-plus/byol_pytorch/examples/lightning/byol-${TASK_NAME}-DUMMY-seed-0.pt
TASK_NAME=sim_transfer_cube_scripted
python3 vinn_select_k.py \
--dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted \
--ckpt_dir /scr/tonyzhao/train_logs/VINN-eval-seed-0-test
python3 vinn_eval.py \
--dataset_dir /scr/tonyzhao/datasets/sim_transfer_cube_scripted \
--model_dir /home/tonyzhao/Research/act-plus-plus/byol_pytorch/examples/lightning/byol-${TASK_NAME}-DUMMY-seed-0.pt \
--ckpt_dir /scr/tonyzhao/train_logs/VINN-eval-seed-0-test \
--task_name $TASK_NAME
## TODO
make sure env is consistent
tune a bit more
--------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------

查看文件

@ -175,6 +175,7 @@ def find_all_hdf5(dataset_dir, skip_mirrored_data):
hdf5_files = [] hdf5_files = []
for root, dirs, files in os.walk(dataset_dir): for root, dirs, files in os.walk(dataset_dir):
for filename in fnmatch.filter(files, '*.hdf5'): for filename in fnmatch.filter(files, '*.hdf5'):
if 'features' in filename: continue
if skip_mirrored_data and 'mirror' in filename: if skip_mirrored_data and 'mirror' in filename:
continue continue
hdf5_files.append(os.path.join(root, filename)) hdf5_files.append(os.path.join(root, filename))

132
vinn_cache_feature.py 普通文件
查看文件

@ -0,0 +1,132 @@
import torch
import argparse
import pathlib
from torch import nn
import torchvision
import os
import time
import h5py
import h5py
from torchvision import models, transforms
from PIL import Image
import IPython
e = IPython.embed
def chunks(lst, n):
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(lst), n):
yield lst[i:i + n]
def expand_greyscale(t):
return t.expand(3, -1, -1)
def main(args):
#################################################
batch_size = 256
#################################################
ckpt_path = args.ckpt_path
ckpt_name = pathlib.PurePath(ckpt_path).name
dataset_name = ckpt_name.split('-')[1]
repr_type = ckpt_name.split('-')[0]
seed = int(ckpt_name.split('-')[-1][:-3])
dataset_dir = f'/scr/tonyzhao/datasets/{dataset_name}'
episode_idxs = [int(name.split('_')[1].split('.')[0]) for name in os.listdir(dataset_dir) if ('.hdf5' in name) and ('features' not in name)]
episode_idxs.sort()
assert len(episode_idxs) == episode_idxs[-1] + 1 # no holes
num_episodes = len(episode_idxs)
feature_extractors = {}
for episode_idx in range(num_episodes):
# load all images
dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5')
with h5py.File(dataset_path, 'r') as root:
image_dict = {}
camera_names = list(root[f'/observations/images/'].keys())
print(f'Camera names: {camera_names}')
for cam_name in camera_names:
image_dict[cam_name] = root[f'/observations/images/{cam_name}'][:]
# load pretrain nets after cam names are known
if not feature_extractors:
for cam_name in camera_names:
resnet = torchvision.models.resnet18(pretrained=True)
loading_status = resnet.load_state_dict(torch.load(ckpt_path.replace('DUMMY', cam_name)))
print(cam_name, loading_status)
resnet = nn.Sequential(*list(resnet.children())[:-1])
resnet = resnet.cuda()
resnet.eval()
feature_extractors[cam_name] = resnet
# inference with resnet
feature_dict = {}
for cam_name, images in image_dict.items():
# Preprocess images
image_size = 120 # TODO NOTICE: reduced resolution
transform = transforms.Compose([
transforms.Resize(image_size), # will scale the image
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Lambda(expand_greyscale),
transforms.Normalize(
mean=torch.tensor([0.485, 0.456, 0.406]),
std=torch.tensor([0.229, 0.224, 0.225])),
])
processed_images = []
for image in images:
image = Image.fromarray(image)
image = transform(image)
processed_images.append(image)
processed_images = torch.stack(processed_images).cuda()
# query the model
all_features = []
with torch.inference_mode():
for batch in chunks(processed_images, batch_size):
features = feature_extractors[cam_name](batch)
features = features.squeeze(axis=3).squeeze(axis=2)
all_features.append(features)
all_features = torch.cat(all_features, axis=0)
max_timesteps = all_features.shape[0]
feature_dict[cam_name] = all_features
# TODO START diagnostics
# first_image = images[0]
# first_processed_image = processed_images[0].cpu().numpy()
# first_feature = all_features[0].cpu().numpy()
# import numpy as np
# np.save('first_image.npy', first_image)
# np.save('first_processed_image.npy', first_processed_image)
# np.save('first_feature.npy', first_feature)
# torch.save(resnet.state_dict(), 'rn.ckpt')
# e()
# exit()
# TODO END diagnostics
# save
dataset_path = os.path.join(dataset_dir, f'{repr_type}_features_seed{seed}_episode_{episode_idx}.hdf5')
print(dataset_path)
# HDF5
t0 = time.time()
with h5py.File(dataset_path, 'w', rdcc_nbytes=1024 ** 2 * 2) as root:
features = root.create_group('features')
for cam_name, array in feature_dict.items():
cam_feature = features.create_dataset(cam_name, (max_timesteps, 512))
features[cam_name][...] = array.cpu().numpy()
print(f'Saving: {time.time() - t0:.1f} secs\n')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='cache features')
parser.add_argument('--ckpt_path', type=str, required=True, help='ckpt_path')
args = parser.parse_args()
main(args)

308
vinn_eval.py 普通文件
查看文件

@ -0,0 +1,308 @@
import torch
from torch import nn
import torch.nn.functional as F
import numpy as np
import h5py
import pathlib
import os
import argparse
import matplotlib.pyplot as plt
from PIL import Image
import torchvision
from torchvision import transforms
# from visualize_episodes import visualize_joints
from utils import set_seed, sample_box_pose
# from imitate_episodes import get_image
from sim_env import BOX_POSE
from constants import DT
from imitate_episodes import save_videos
from einops import rearrange
import IPython
e = IPython.embed
# modified from https://github.com/jyopari/VINN/blob/main/nearest-neighbor-eval/handle_nn.ipynb
def calculate_nearest_neighbors(curr_feature, support_inputs, support_targets, k, state_weight):
has_skip = len(support_targets.shape) == 3
if has_skip: # when there is action skip
num_targets, skip, a_dim = support_targets.shape
support_targets = support_targets.view((num_targets, -1))
curr_vis_feature, curr_s_feature = curr_feature
support_vis_feature, support_s_feature = support_inputs
pairwise_dist_vis = torch.norm(curr_vis_feature - support_vis_feature, dim=1).unsqueeze(0)
pairwise_dist_s = torch.norm(curr_s_feature - support_s_feature, dim=1).unsqueeze(0)
pairwise_dist = pairwise_dist_vis + pairwise_dist_s * state_weight
sorted_dist, index = torch.sort(pairwise_dist, dim=1) # sort the support axis
permuted_support_targets = support_targets[index]
topk_dist = pairwise_dist[:, :k]
topk_support_targets = permuted_support_targets[:, :k]
weights = F.softmax(-topk_dist, dim=1)
weighted_support_targets = weights.unsqueeze(2) * topk_support_targets
prediction = torch.sum(weighted_support_targets, dim=1)
if has_skip:
num_predictions = prediction.shape[0]
prediction = prediction.reshape((num_predictions, skip, a_dim))
return prediction
def main(args):
# TODO ######################
k = None # for scripted box transfer
skip = 100
real_robot = False
save_episode = True
# TODO ######################
onscreen_cam = 'main'
state_dim = 14
dataset_dir = args['dataset_dir']
onscreen_render = args['onscreen_render']
ckpt_dir = args['ckpt_dir']
model_dir = args['model_dir']
task_name = args['task_name']
if 'insertion' in task_name:
sim_episode_len = 400
env_max_reward = 4
ks = [None]
elif 'transfer_cube' in task_name:
sim_episode_len = 400
env_max_reward = 4
ks = [1, 1, 1]
if 'human' in dataset_dir:
state_weight = 5
else:
state_weight = 10
print(f'{state_weight=}')
elif task_name == 'ziploc_slide':
env_max_reward = 1
ks = [71]
state_weight = 0
else:
raise NotImplementedError
model_name = pathlib.PurePath(model_dir).name
seed = int(model_name.split('-')[-1][:-3])
k = ks[seed]
# load train data
vis_features = []
state_features = []
Y = []
for episode_id in range(0, 40):
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
with h5py.File(dataset_path, 'r') as root:
action = root['/action'][:]
camera_names = list(root[f'/observations/images/'].keys())
# Visual feature
all_cam_feature = []
for cam_name in camera_names:
feature_dataset_path = os.path.join(dataset_dir, f'byol_features_seed{seed}_episode_{episode_id}.hdf5')
with h5py.File(feature_dataset_path, 'r') as root:
cam_feature = root[f'/features/{cam_name}'][:]
all_cam_feature.append(cam_feature)
vis_fea = np.concatenate(all_cam_feature, axis=1)
## State feature
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
with h5py.File(dataset_path, 'r') as root:
s_fea = root['/observations/qpos'][:]
# stack actions together
eps_len = len(action)
indices = np.tile(np.arange(skip), eps_len).reshape(eps_len, skip) # each row is 0, 1, ... skip
offset = np.expand_dims(np.arange(eps_len), axis=1)
indices = indices + offset # row1: 0, 1, ... skip; row2: 1, 2, ... skip+1
# indices will exceed eps_len, thus clamp to eps_len-1
indices = np.clip(indices, 0, eps_len-1)
# stack action
action = action[indices] # new shape: eps_len, skip, a_dim
vis_features.append(vis_fea)
state_features.append(s_fea)
Y.append(action)
vis_features = np.concatenate(vis_features)
state_features = np.concatenate(state_features)
Y = np.concatenate(Y)
train_inputs = [torch.from_numpy(vis_features).cuda(), torch.from_numpy(state_features).cuda()]
train_targets = torch.from_numpy(Y).cuda()
set_seed(1000)
feature_extractors = {}
for cam_name in camera_names:
resnet = torchvision.models.resnet18(pretrained=True)
loading_status = resnet.load_state_dict(torch.load(model_dir.replace('DUMMY', cam_name)))
print(cam_name, loading_status)
resnet = nn.Sequential(*list(resnet.children())[:-1])
resnet = resnet.cuda()
resnet.eval()
feature_extractors[cam_name] = resnet
# load environment
if real_robot:
from real_env import make_real_env
env = make_real_env(init_node=True)
max_timesteps = EPISODE_LEN
camera_names = CAMERA_NAMES
else:
from sim_env import make_sim_env
env = make_sim_env(task_name)
max_timesteps = sim_episode_len
num_rollouts = 50
episode_returns = []
max_rewards = []
for rollout_id in range(num_rollouts):
### set task
BOX_POSE[0] = sample_box_pose() # used in sim reset
ts = env.reset()
### evaluation loop
qpos_history = torch.zeros((1, max_timesteps, state_dim)).cuda()
image_list = [] # for visualization
qpos_list = []
target_qpos_list = []
rewards = []
with torch.inference_mode():
for t in range(sim_episode_len):
### process previous timestep to get qpos and image_list
obs = ts.observation
if 'images' in obs:
image_list.append(obs['images'])
else:
image_list.append({'main': obs['image']})
qpos_numpy = np.array(obs['qpos'])
# qpos = pre_process(qpos_numpy)
qpos = torch.from_numpy(qpos_numpy).float().cuda().unsqueeze(0)
qpos_history[:, t] = qpos
_, curr_image_raw = get_image(ts, camera_names)
if t % skip == 0:
image_size = 120
transform = transforms.Compose([
transforms.Resize(image_size), # will scale the image
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Lambda(expand_greyscale),
transforms.Normalize(
mean=torch.tensor([0.485, 0.456, 0.406]),
std=torch.tensor([0.229, 0.224, 0.225])),
])
all_cam_features = []
for cam_id, curr_image in enumerate(curr_image_raw):
curr_image = Image.fromarray(curr_image) # TODO only one camera
curr_image = transform(curr_image)
curr_image = curr_image.unsqueeze(dim=0).cuda()
curr_image_feature = feature_extractors[camera_names[cam_id]](curr_image)
curr_image_feature = curr_image_feature.squeeze(3).squeeze(2)
all_cam_features.append(curr_image_feature)
curr_image_feature = torch.cat(all_cam_features, dim=1)
### Visual feature
# curr_feature = curr_image_feature
### State feature
# curr_feature = qpos
### Both features
curr_feature = [curr_image_feature, qpos]
all_target_qpos = calculate_nearest_neighbors(curr_feature, train_inputs, train_targets, k, state_weight) # TODO use this
all_target_qpos = all_target_qpos.squeeze(0).cpu().numpy()
target_qpos = all_target_qpos[t % skip]
# ### SAFETY
# max_a = 0.05
# curr_qpos = qpos.squeeze().cpu().numpy()
# target_qpos = target_qpos.clip(curr_qpos - max_a, curr_qpos + max_a)
# ### SAFETY
### step the environment
ts = env.step(target_qpos)
### save things for visualization
qpos_list.append(qpos_numpy)
target_qpos_list.append(target_qpos)
rewards.append(ts.reward)
# if real_robot and t != 0 and t % 60 == 0:
# e()
plt.close()
if real_robot:
env.puppet_bot_left.dxl.robot_set_operating_modes("single", "gripper", "position")
env.puppet_bot_right.dxl.robot_set_operating_modes("single", "gripper", "position")
env.puppet_bot_left.dxl.robot_set_operating_modes("single", "gripper", "pwm")
env.puppet_bot_right.dxl.robot_set_operating_modes("single", "gripper", "pwm")
rewards = np.array(rewards)
episode_return = np.sum(rewards[rewards!=None])
episode_returns.append(episode_return)
max_reward = np.max(rewards)
max_rewards.append(max_reward)
print(f'{episode_return=}, {max_reward=}')
if save_episode:
save_videos(image_list, DT, video_path=os.path.join(ckpt_dir, f'video{rollout_id}.mp4'))
# visualize_joints(qpos_list, target_qpos_list, plot_path=os.path.join(ckpt_dir, f'qpos{rollout_id}.png'))
# visualize_joints(qpos_list, example_qpos, plot_path=os.path.join(ckpt_dir, f'qpos_reference{rollout_id}.png'), label_overwrite=("policy", "dataset"))
success_rate = np.mean(np.array(max_rewards) == env_max_reward)
avg_return = np.mean(episode_returns)
summary_str = f'\nSuccess rate: {success_rate}\nAverage return: {avg_return}\n\n'
for r in range(env_max_reward+1):
more_or_equal_r = (np.array(max_rewards) >= r).sum()
more_or_equal_r_rate = more_or_equal_r / num_rollouts
summary_str += f'Reward >= {r}: {more_or_equal_r}/{num_rollouts} = {more_or_equal_r_rate*100}%\n'
print(summary_str)
# save success rate to txt
result_file_name = f'result_{skip}_{k}' + '.txt'
with open(os.path.join(ckpt_dir, result_file_name), 'w') as f:
f.write(summary_str)
f.write(repr(episode_returns))
f.write('\n\n')
f.write(repr(max_rewards))
return success_rate, avg_return
def get_image(ts, camera_names):
if 'images' in ts.observation:
curr_images = []
for cam_name in camera_names:
curr_image = rearrange(ts.observation['images'][cam_name], 'h w c -> c h w')
curr_images.append(curr_image)
curr_image_raw = np.stack(curr_images, axis=0)
else:
curr_image_raw = rearrange(ts.observation['image'], 'h w c -> c h w')
curr_image = torch.from_numpy(curr_image_raw / 255.0).float().cuda().unsqueeze(0)
curr_image_raw = rearrange(curr_image_raw, 'b c h w -> b h w c')
return curr_image, curr_image_raw
def expand_greyscale(t):
return t.expand(3, -1, -1)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--onscreen_render', action='store_true')
parser.add_argument('--dataset_dir', action='store', type=str, help='The text to parse.', required=True)
parser.add_argument('--model_dir', action='store', type=str, help='model_dir', required=True)
parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True)
parser.add_argument('--ckpt_dir', action='store', type=str, help='The text to parse.', required=True)
main(vars(parser.parse_args()))

129
vinn_select_k.py 普通文件
查看文件

@ -0,0 +1,129 @@
import torch
import torch.nn.functional as F
import numpy as np
import h5py
import pathlib
import os
import argparse
import matplotlib.pyplot as plt
import IPython
e = IPython.embed
# modified from https://github.com/jyopari/VINN/blob/main/nearest-neighbor-eval/handle_nn.ipynb
def calculate_nearest_neighbors(query_inputs, query_targets, support_inputs, support_targets, max_k):
with torch.no_grad():
pairwise_dist = []
for q_in in query_inputs:
diff = support_inputs - q_in.unsqueeze(0)
dist = torch.norm(diff, dim=1)
pairwise_dist.append(dist)
pairwise_dist = torch.stack(pairwise_dist)
sorted_dist, index = torch.sort(pairwise_dist, dim=1) # sort the support axis
permuted_support_targets = support_targets[index]
errors = []
for k in range(1, max_k):
topk_dist = pairwise_dist[:, :k]
topk_support_targets = permuted_support_targets[:, :k]
weights = F.softmax(-topk_dist, dim=1)
weighted_support_targets = weights.unsqueeze(2) * topk_support_targets
prediction = torch.sum(weighted_support_targets, dim=1)
error = F.mse_loss(prediction, query_targets)
errors.append(error)
return errors
def chunks(lst, n):
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(lst), n):
yield lst[i:i + n]
def main(args):
# TODO ######################
dataset_dir = args['dataset_dir']
ckpt_dir = args['ckpt_dir']
seed = 0
max_k = 400
batch_size = 100
# TODO ######################
if not os.path.isdir(ckpt_dir):
os.makedirs(ckpt_dir)
episode_idxs = [int(name.split('_')[1].split('.')[0]) for name in os.listdir(dataset_dir) if ('.hdf5' in name) and ('features' not in name)]
episode_idxs.sort()
assert len(episode_idxs) == episode_idxs[-1] + 1 # no holes
num_episodes = len(episode_idxs)
val_split = int(num_episodes * 0.8)
# load train data
X = []
Y = []
for episode_id in range(0, val_split):
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
with h5py.File(dataset_path, 'r') as root:
action = root['/action'][:]
camera_names = list(root[f'/observations/images/'].keys())
all_cam_feature = []
feature_dataset_path = os.path.join(dataset_dir, f'byol_features_seed{seed}_episode_{episode_id}.hdf5')
with h5py.File(feature_dataset_path, 'r') as root:
for cam_name in camera_names:
cam_feature = root[f'/features/{cam_name}'][:]
all_cam_feature.append(cam_feature)
cam_feature = np.concatenate(all_cam_feature, axis=1)
X.append(cam_feature)
Y.append(action)
X = np.concatenate(X)
Y = np.concatenate(Y)
train_inputs = torch.from_numpy(X).cuda()
train_targets = torch.from_numpy(Y).cuda()
print(f'All features: {train_inputs.shape}')
# load test data
X = []
Y = []
for episode_id in range(val_split, num_episodes):
dataset_path = os.path.join(dataset_dir, f'episode_{episode_id}.hdf5')
with h5py.File(dataset_path, 'r') as root:
action = root['/action'][:]
all_cam_feature = []
feature_dataset_path = os.path.join(dataset_dir, f'byol_features_seed{seed}_episode_{episode_id}.hdf5')
with h5py.File(feature_dataset_path, 'r') as root:
for cam_name in camera_names:
cam_feature = root[f'/features/{cam_name}'][:]
all_cam_feature.append(cam_feature)
cam_feature = np.concatenate(all_cam_feature, axis=1)
X.append(cam_feature)
Y.append(action)
X = np.concatenate(X)
Y = np.concatenate(Y)
val_inputs = torch.from_numpy(X).cuda()
val_targets = torch.from_numpy(Y).cuda()
val_losses = []
for inputs, targets in zip(chunks(val_inputs, batch_size), chunks(val_targets, batch_size)):
val_loss = calculate_nearest_neighbors(inputs, targets, train_inputs, train_targets, max_k)
val_loss = torch.stack(val_loss)
val_losses.append(val_loss)
val_losses = torch.mean(torch.stack(val_losses), dim=0)
val_loss = val_losses
val_loss = torch.tensor(val_loss).cpu().numpy()
print(f'min val loss of {np.min(val_loss)} at k={np.argmin(val_loss)}')
plt.plot(np.arange(1, max_k), val_loss)
plt.savefig(os.path.join(ckpt_dir, f'k_select-seed{seed}.png'))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--dataset_dir', action='store', type=str, help='The text to parse.', required=True)
parser.add_argument('--ckpt_dir', action='store', type=str, help='The text to parse.', required=True)
main(vars(parser.parse_args()))