1 次代码提交

作者 SHA1 备注 提交日期
f163ce50b9 pretraining on rws1 2023-10-28 23:31:25 -07:00
共有 5 个文件被更改,包括 255 次插入 和 160 次删除

30
commands.txt 普通文件
查看文件

@ -0,0 +1,30 @@
conda activate mimic
export MUJOCO_GL=egl
cd /home/tonyzhao/Research/act-plus-plus
python3 imitate_episodes.py \
--task_name sim_transfer_cube_human \
--ckpt_dir /scr/tonyzhao/train_logs/vq_test \
--policy_class ACT --kl_weight 10 --chunk_size 100 \
--hidden_dim 512 --batch_size 8 --dim_feedforward 3200 \
--num_epochs 10000 --lr 1e-5 --seed 0 --vq
conda activate mobile
cd /home/tonyzhao/Research/act-plus-plus
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
--task_name all \
--ckpt_dir /scr/tonyzhao/train_logs/pretrain_all \
--policy_class ACT --kl_weight 10 --chunk_size 50 \
--hidden_dim 512 --batch_size 24 --dim_feedforward 3200 --num_epochs 5000 --lr 1e-4 --seed 0
conda activate mobile
cd /home/tonyzhao/Research/act-plus-plus
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
--task_name all \
--ckpt_dir /scr/tonyzhao/train_logs/pretrain_all \
--policy_class ACT --kl_weight 10 --chunk_size 50 \
--hidden_dim 512 --batch_size 24 --dim_feedforward 3200 --lr 1e-4 --seed 0 \
--num_steps 1000000 --eval_every 10000000000 --validate_every 2000 --save_every 5000

查看文件

@ -1,7 +1,7 @@
import pathlib
### Task parameters
DATA_DIR = 'data'
DATA_DIR = '/scr/tonyzhao/datasets'
SIM_TASK_CONFIGS = {
'sim_transfer_cube_scripted':{
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',
@ -30,6 +30,15 @@ SIM_TASK_CONFIGS = {
'episode_len': 500,
'camera_names': ['top']
},
'all': {
'dataset_dir': DATA_DIR + '/',
'num_episodes': None,
'episode_len': None,
'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist']
},
}
### Simulation envs fixed constants

查看文件

@ -59,7 +59,7 @@ def get_args_parser():
parser.add_argument('--policy_class', action='store', type=str, help='policy_class, capitalize', required=True)
parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True)
parser.add_argument('--seed', action='store', type=int, help='seed', required=True)
parser.add_argument('--num_epochs', action='store', type=int, help='num_epochs', required=True)
parser.add_argument('--num_steps', action='store', type=int, help='num_epochs', required=True)
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
parser.add_argument('--temporal_agg', action='store_true')
@ -68,6 +68,9 @@ def get_args_parser():
parser.add_argument('--vq_class', action='store', type=int, help='vq_class', required=False)
parser.add_argument('--vq_dim', action='store', type=int, help='vq_dim', required=False)
parser.add_argument('--action_dim', action='store', type=int, required=False)
parser.add_argument('--eval_every', action='store', type=int, default=500, help='eval_every', required=False)
parser.add_argument('--validate_every', action='store', type=int, default=500, help='validate_every', required=False)
parser.add_argument('--save_every', action='store', type=int, default=500, help='save_every', required=False)
return parser

查看文件

@ -5,8 +5,10 @@ import pickle
import argparse
import matplotlib.pyplot as plt
from copy import deepcopy
from itertools import repeat
from tqdm import tqdm
from einops import rearrange
import wandb
from constants import DT
from constants import PUPPET_GRIPPER_JOINT_OPEN
@ -33,16 +35,21 @@ def main(args):
task_name = args['task_name']
batch_size_train = args['batch_size']
batch_size_val = args['batch_size']
num_epochs = args['num_epochs']
num_steps = args['num_steps']
eval_every = args['eval_every']
validate_every = args['validate_every']
save_every = args['save_every']
# get task parameters
is_sim = task_name[:4] == 'sim_'
if is_sim:
# if is_sim:
# from constants import SIM_TASK_CONFIGS
# task_config = SIM_TASK_CONFIGS[task_name]
# else:
# from aloha_scripts.constants import TASK_CONFIGS
# task_config = TASK_CONFIGS[task_name]
from constants import SIM_TASK_CONFIGS
task_config = SIM_TASK_CONFIGS[task_name]
else:
from aloha_scripts.constants import TASK_CONFIGS
task_config = TASK_CONFIGS[task_name]
dataset_dir = task_config['dataset_dir']
num_episodes = task_config['num_episodes']
episode_len = task_config['episode_len']
@ -79,7 +86,10 @@ def main(args):
raise NotImplementedError
config = {
'num_epochs': num_epochs,
'num_steps': num_steps,
'eval_every': eval_every,
'validate_every': validate_every,
'save_every': save_every,
'ckpt_dir': ckpt_dir,
'episode_len': episode_len,
'state_dim': state_dim,
@ -94,11 +104,19 @@ def main(args):
'real_robot': not is_sim
}
if not os.path.isdir(ckpt_dir):
os.makedirs(ckpt_dir)
config_path = os.path.join(ckpt_dir, 'config.pkl')
wandb.init(project="mobile-aloha", reinit=True, entity="tonyzhao")
wandb.config.update(config)
with open(config_path, 'wb') as f:
pickle.dump(config, f)
if is_eval:
ckpt_names = [f'policy_last.ckpt']
results = []
for ckpt_name in ckpt_names:
success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True)
wandb.log({'success_rate': success_rate, 'avg_return': avg_return})
results.append([ckpt_name, success_rate, avg_return])
for ckpt_name, success_rate, avg_return in results:
@ -106,22 +124,21 @@ def main(args):
print()
exit()
train_dataloader, val_dataloader, stats, _ = load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val)
train_dataloader, val_dataloader, stats, _ = load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val, args['chunk_size'])
# save dataset stats
if not os.path.isdir(ckpt_dir):
os.makedirs(ckpt_dir)
stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
with open(stats_path, 'wb') as f:
pickle.dump(stats, f)
best_ckpt_info = train_bc(train_dataloader, val_dataloader, config)
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
best_step, min_val_loss, best_state_dict = best_ckpt_info
# save best checkpoint
ckpt_path = os.path.join(ckpt_dir, f'policy_best.ckpt')
torch.save(best_state_dict, ckpt_path)
print(f'Best ckpt, val loss {min_val_loss:.6f} @ epoch{best_epoch}')
print(f'Best ckpt, val loss {min_val_loss:.6f} @ step{best_step}')
wandb.finish()
def make_policy(policy_class, policy_config):
@ -346,11 +363,14 @@ def forward_pass(data, policy):
def train_bc(train_dataloader, val_dataloader, config):
num_epochs = config['num_epochs']
num_steps = config['num_steps']
ckpt_dir = config['ckpt_dir']
seed = config['seed']
policy_class = config['policy_class']
policy_config = config['policy_config']
eval_every = config['eval_every']
validate_every = config['validate_every']
save_every = config['save_every']
set_seed(seed)
@ -358,85 +378,76 @@ def train_bc(train_dataloader, val_dataloader, config):
policy.cuda()
optimizer = make_optimizer(policy_class, policy)
train_history = []
validation_history = []
min_val_loss = np.inf
best_ckpt_info = None
for epoch in tqdm(range(num_epochs)):
print(f'\nEpoch {epoch}')
train_dataloader = repeater(train_dataloader)
for step in tqdm(range(num_steps+1)):
# validation
if step % validate_every == 0:
print('validating')
with torch.inference_mode():
policy.eval()
epoch_dicts = []
validation_dicts = []
for batch_idx, data in enumerate(val_dataloader):
forward_dict = forward_pass(data, policy)
epoch_dicts.append(forward_dict)
epoch_summary = compute_dict_mean(epoch_dicts)
validation_history.append(epoch_summary)
validation_dicts.append(forward_dict)
epoch_val_loss = epoch_summary['loss']
validation_summary = compute_dict_mean(validation_dicts)
epoch_val_loss = validation_summary['loss']
if epoch_val_loss < min_val_loss:
min_val_loss = epoch_val_loss
best_ckpt_info = (epoch, min_val_loss, deepcopy(policy.state_dict()))
best_ckpt_info = (step, min_val_loss, deepcopy(policy.state_dict()))
for k in validation_summary.keys():
validation_summary[f'val_{k}'] = validation_summary.pop(k)
wandb.log(validation_summary, step=step)
print(f'Val loss: {epoch_val_loss:.5f}')
summary_string = ''
for k, v in epoch_summary.items():
for k, v in validation_summary.items():
summary_string += f'{k}: {v.item():.3f} '
print(summary_string)
# evaluation
if (step > 0) and (step % eval_every == 0):
state_dict = deepcopy(policy.state_dict())
success_with_te, _ = eval_bc(config, f'TE_step{step}', state_dict, temporal_agg=True, save_episode=True, num_rollouts=5)
success_without_te, _ = eval_bc(config, f'step{step}', state_dict, temporal_agg=False, save_episode=True, num_rollouts=5)
wandb.log({'success_with_te':success_with_te, 'success_without_te':success_without_te}, step=step)
# training
policy.train()
optimizer.zero_grad()
for batch_idx, data in enumerate(train_dataloader):
data = next(train_dataloader)
forward_dict = forward_pass(data, policy)
# backward
loss = forward_dict['loss']
loss.backward()
optimizer.step()
optimizer.zero_grad()
train_history.append(detach_dict(forward_dict))
epoch_summary = compute_dict_mean(train_history[(batch_idx+1)*epoch:(batch_idx+1)*(epoch+1)])
epoch_train_loss = epoch_summary['loss']
print(f'Train loss: {epoch_train_loss:.5f}')
summary_string = ''
for k, v in epoch_summary.items():
summary_string += f'{k}: {v.item():.3f} '
print(summary_string)
wandb.log(forward_dict, step=step) # not great, make training 1-2% slower
if epoch % 1000 == 0:
ckpt_path = os.path.join(ckpt_dir, f'policy_epoch_{epoch}_seed_{seed}.ckpt')
if step % save_every == 0:
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{step}_seed_{seed}.ckpt')
torch.save(policy.state_dict(), ckpt_path)
plot_history(train_history, validation_history, epoch, ckpt_dir, seed)
ckpt_path = os.path.join(ckpt_dir, f'policy_last.ckpt')
torch.save(policy.state_dict(), ckpt_path)
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
ckpt_path = os.path.join(ckpt_dir, f'policy_epoch_{best_epoch}_seed_{seed}.ckpt')
best_step, min_val_loss, best_state_dict = best_ckpt_info
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{best_step}_seed_{seed}.ckpt')
torch.save(best_state_dict, ckpt_path)
print(f'Training finished:\nSeed {seed}, val loss {min_val_loss:.6f} at epoch {best_epoch}')
# save training curves
plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed)
print(f'Training finished:\nSeed {seed}, val loss {min_val_loss:.6f} at step {best_step}')
return best_ckpt_info
def plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed):
# save training curves
for key in train_history[0]:
plot_path = os.path.join(ckpt_dir, f'train_val_{key}_seed_{seed}.png')
plt.figure()
train_values = [summary[key].item() for summary in train_history]
val_values = [summary[key].item() for summary in validation_history]
plt.plot(np.linspace(0, num_epochs-1, len(train_history)), train_values, label='train')
plt.plot(np.linspace(0, num_epochs-1, len(validation_history)), val_values, label='validation')
# plt.ylim([-0.1, 1])
plt.tight_layout()
plt.legend()
plt.title(key)
plt.savefig(plot_path)
print(f'Saved plots to {ckpt_dir}')
def repeater(data_loader):
epoch = 0
for loader in repeat(data_loader):
for data in loader:
yield data
print(f'Epoch {epoch} done')
epoch += 1
if __name__ == '__main__':
@ -448,8 +459,11 @@ if __name__ == '__main__':
parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True)
parser.add_argument('--batch_size', action='store', type=int, help='batch_size', required=True)
parser.add_argument('--seed', action='store', type=int, help='seed', required=True)
parser.add_argument('--num_epochs', action='store', type=int, help='num_epochs', required=True)
parser.add_argument('--num_steps', action='store', type=int, help='num_steps', required=True)
parser.add_argument('--lr', action='store', type=float, help='lr', required=True)
parser.add_argument('--eval_every', action='store', type=int, default=500, help='eval_every', required=False)
parser.add_argument('--validate_every', action='store', type=int, default=500, help='validate_every', required=False)
parser.add_argument('--save_every', action='store', type=int, default=500, help='save_every', required=False)
# for ACT
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)

119
utils.py
查看文件

@ -2,43 +2,53 @@ import numpy as np
import torch
import os
import h5py
import fnmatch
from torch.utils.data import TensorDataset, DataLoader
import IPython
e = IPython.embed
class EpisodicDataset(torch.utils.data.Dataset):
def __init__(self, episode_ids, dataset_dir, camera_names, norm_stats):
def __init__(self, dataset_path_list, camera_names, norm_stats, episode_ids, episode_len, chunk_size):
super(EpisodicDataset).__init__()
self.episode_ids = episode_ids
self.dataset_dir = dataset_dir
self.dataset_path_list = dataset_path_list
self.camera_names = camera_names
self.norm_stats = norm_stats
self.is_sim = None
self.episode_len = episode_len
self.chunk_size = chunk_size
self.cumulative_len = np.cumsum(self.episode_len)
self.max_episode_len = max(episode_len)
self.__getitem__(0) # initialize self.is_sim
self.is_sim = False
def __len__(self):
return len(self.episode_ids)
return sum(self.episode_len)
def _locate_transition(self, index):
assert index < self.cumulative_len[-1]
episode_index = np.argmax(self.cumulative_len > index) # argmax returns first True index
start_ts = index - (self.cumulative_len[episode_index] - self.episode_len[episode_index])
episode_id = self.episode_ids[episode_index]
return episode_id, start_ts
def __getitem__(self, index):
sample_full_episode = False # hardcode
episode_id = self.episode_ids[index]
dataset_path = os.path.join(self.dataset_dir, f'episode_{episode_id}.hdf5')
episode_id, start_ts = self._locate_transition(index)
dataset_path = self.dataset_path_list[episode_id]
try:
# print(dataset_path)
with h5py.File(dataset_path, 'r') as root:
is_sim = root.attrs['sim']
is_sim = False # root.attrs['sim']
if '/base_action' in root:
base_action = root['/base_action'][()]
base_action = preprocess_base_action(base_action)
action = np.concatenate([root['/action'][()], base_action], axis=-1)
else:
action = root['/action'][()]
dummy_base_action = np.zeros([action.shape[0], 2])
action = np.concatenate([action, dummy_base_action], axis=-1)
original_action_shape = action.shape
episode_len = original_action_shape[0]
if sample_full_episode:
start_ts = 0
else:
start_ts = np.random.choice(episode_len)
# get observation at start_ts only
qpos = root['/observations/qpos'][start_ts]
qvel = root['/observations/qvel'][start_ts]
@ -53,12 +63,15 @@ class EpisodicDataset(torch.utils.data.Dataset):
action = action[max(0, start_ts - 1):] # hack, to make timesteps more aligned
action_len = episode_len - max(0, start_ts - 1) # hack, to make timesteps more aligned
self.is_sim = is_sim
padded_action = np.zeros(original_action_shape, dtype=np.float32)
# self.is_sim = is_sim
padded_action = np.zeros((self.max_episode_len, original_action_shape[1]), dtype=np.float32)
padded_action[:action_len] = action
is_pad = np.zeros(episode_len)
is_pad = np.zeros(self.max_episode_len)
is_pad[action_len:] = 1
padded_action = padded_action[:self.chunk_size]
padded_action = padded_action[:self.chunk_size]
# new axis for different cameras
all_cam_images = []
for cam_name in self.camera_names:
@ -79,66 +92,92 @@ class EpisodicDataset(torch.utils.data.Dataset):
action_data = (action_data - self.norm_stats["action_mean"]) / self.norm_stats["action_std"]
qpos_data = (qpos_data - self.norm_stats["qpos_mean"]) / self.norm_stats["qpos_std"]
except:
print(f'Error loading {dataset_path}')
quit()
# print(image_data.dtype, qpos_data.dtype, action_data.dtype, is_pad.dtype)
return image_data, qpos_data, action_data, is_pad
def get_norm_stats(dataset_dir, num_episodes):
def get_norm_stats(dataset_path_list):
all_qpos_data = []
all_action_data = []
for episode_idx in range(num_episodes):
dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5')
all_episode_len = []
for dataset_path in dataset_path_list:
try:
with h5py.File(dataset_path, 'r') as root:
qpos = root['/observations/qpos'][()]
qvel = root['/observations/qvel'][()]
if '/base_action' in root:
base_action = root['/base_action'][()]
base_action = preprocess_base_action(base_action)
action = np.concatenate([root['/action'][()], base_action], axis=-1)
else:
action = root['/action'][()]
dummy_base_action = np.zeros([action.shape[0], 2])
action = np.concatenate([action, dummy_base_action], axis=-1)
except:
print(f'Error loading {dataset_path}')
quit()
all_qpos_data.append(torch.from_numpy(qpos))
all_action_data.append(torch.from_numpy(action))
all_qpos_data = torch.stack(all_qpos_data)
all_action_data = torch.stack(all_action_data)
all_action_data = all_action_data
all_episode_len.append(len(qpos))
all_qpos_data = torch.cat(all_qpos_data, dim=0)
all_action_data = torch.cat(all_action_data, dim=0)
# normalize action data
action_mean = all_action_data.mean(dim=[0, 1], keepdim=True)
action_std = all_action_data.std(dim=[0, 1], keepdim=True)
action_mean = all_action_data.mean(dim=[0]).float()
action_std = all_action_data.std(dim=[0]).float()
action_std = torch.clip(action_std, 1e-2, np.inf) # clipping
# normalize qpos data
qpos_mean = all_qpos_data.mean(dim=[0, 1], keepdim=True)
qpos_std = all_qpos_data.std(dim=[0, 1], keepdim=True)
qpos_mean = all_qpos_data.mean(dim=[0]).float()
qpos_std = all_qpos_data.std(dim=[0]).float()
qpos_std = torch.clip(qpos_std, 1e-2, np.inf) # clipping
stats = {"action_mean": action_mean.numpy().squeeze(), "action_std": action_std.numpy().squeeze(),
"qpos_mean": qpos_mean.numpy().squeeze(), "qpos_std": qpos_std.numpy().squeeze(),
stats = {"action_mean": action_mean.numpy(), "action_std": action_std.numpy(),
"qpos_mean": qpos_mean.numpy(), "qpos_std": qpos_std.numpy(),
"example_qpos": qpos}
return stats
return stats, all_episode_len
def find_all_hdf5(dataset_dir):
hdf5_files = []
for root, dirs, files in os.walk(dataset_dir):
for filename in fnmatch.filter(files, '*.hdf5'):
hdf5_files.append(os.path.join(root, filename))
return hdf5_files
def load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val):
print(f'\nData from: {dataset_dir}\n')
def load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val, chunk_size):
del num_episodes
dataset_path_list = find_all_hdf5(dataset_dir)
dataset_path_list = [p for p in dataset_path_list if 'sim' not in p] ### TODO filter out sim data
num_episodes = len(dataset_path_list)
# obtain train test split
train_ratio = 0.8
shuffled_indices = np.random.permutation(num_episodes)
train_indices = shuffled_indices[:int(train_ratio * num_episodes)]
val_indices = shuffled_indices[int(train_ratio * num_episodes):]
train_ratio = 0.998
shuffled_episode_ids = np.random.permutation(num_episodes)
train_episode_ids = shuffled_episode_ids[:int(train_ratio * num_episodes)]
val_episode_ids = shuffled_episode_ids[int(train_ratio * num_episodes):]
print(f'\n\nData from: {dataset_dir}\n- Train on {len(train_episode_ids)} episodes\n- Test on {len(val_episode_ids)} episodes\n\n')
# obtain normalization stats for qpos and action
norm_stats = get_norm_stats(dataset_dir, num_episodes)
norm_stats, all_episode_len = get_norm_stats(dataset_path_list)
train_episode_len = [all_episode_len[i] for i in train_episode_ids]
val_episode_len = [all_episode_len[i] for i in val_episode_ids]
# construct dataset and dataloader
train_dataset = EpisodicDataset(train_indices, dataset_dir, camera_names, norm_stats)
val_dataset = EpisodicDataset(val_indices, dataset_dir, camera_names, norm_stats)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size_train, shuffle=True, pin_memory=True, num_workers=1, prefetch_factor=1, persistent_workers=True)
val_dataloader = DataLoader(val_dataset, batch_size=batch_size_val, shuffle=True, pin_memory=True, num_workers=1, prefetch_factor=1, persistent_workers=True)
train_dataset = EpisodicDataset(dataset_path_list, camera_names, norm_stats, train_episode_ids, train_episode_len, chunk_size)
val_dataset = EpisodicDataset(dataset_path_list, camera_names, norm_stats, val_episode_ids, val_episode_len, chunk_size)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size_train, shuffle=True, pin_memory=True, num_workers=1, prefetch_factor=1)
val_dataloader = DataLoader(val_dataset, batch_size=batch_size_val, shuffle=True, pin_memory=True, num_workers=1, prefetch_factor=1)
return train_dataloader, val_dataloader, norm_stats, train_dataset.is_sim
def calibrate_linear_vel(base_action):
c = 0.19
v = base_action[:, 0]