这个提交包含在:
Zipeng Fu
2023-10-13 17:57:59 -04:00
父节点 694c606940
当前提交 b37712fd2f
共有 10 个文件被更改,包括 658 次插入 和 41 次删除

4
.gitignore vendored
查看文件

@ -137,4 +137,6 @@ celerybeat.pid
dmypy.json dmypy.json
# Pyre type checker # Pyre type checker
.pyre/ .pyre/
ckpts/

查看文件

@ -57,10 +57,7 @@ To set up a new terminal, run:
We use ``sim_transfer_cube_scripted`` task in the examples below. Another option is ``sim_insertion_scripted``. We use ``sim_transfer_cube_scripted`` task in the examples below. Another option is ``sim_insertion_scripted``.
To generated 50 episodes of scripted data, run: To generated 50 episodes of scripted data, run:
python3 record_sim_episodes.py \ python3 record_sim_episodes.py --task_name sim_transfer_cube_scripted --dataset_dir <data save dir> --num_episodes 50
--task_name sim_transfer_cube_scripted \
--dataset_dir <data save dir> \
--num_episodes 50
To can add the flag ``--onscreen_render`` to see real-time rendering. To can add the flag ``--onscreen_render`` to see real-time rendering.
To visualize the episode after it is collected, run To visualize the episode after it is collected, run
@ -70,12 +67,7 @@ To visualize the episode after it is collected, run
To train ACT: To train ACT:
# Transfer Cube task # Transfer Cube task
python3 imitate_episodes.py \ python3 imitate_episodes.py --task_name sim_transfer_cube_scripted --ckpt_dir <ckpt dir> --policy_class ACT --kl_weight 10 --chunk_size 100 --hidden_dim 512 --batch_size 8 --dim_feedforward 3200 --num_epochs 2000 --lr 1e-5 --seed 0
--task_name sim_transfer_cube_scripted \
--ckpt_dir <ckpt dir> \
--policy_class ACT --kl_weight 10 --chunk_size 100 --hidden_dim 512 --batch_size 8 --dim_feedforward 3200 \
--num_epochs 2000 --lr 1e-5 \
--seed 0
To evaluate the policy, run the same command but add ``--eval``. This loads the best validation checkpoint. To evaluate the policy, run the same command but add ``--eval``. This loads the best validation checkpoint.

查看文件

@ -1,7 +1,7 @@
import pathlib import pathlib
### Task parameters ### Task parameters
DATA_DIR = '<put your data dir here>' DATA_DIR = 'data'
SIM_TASK_CONFIGS = { SIM_TASK_CONFIGS = {
'sim_transfer_cube_scripted':{ 'sim_transfer_cube_scripted':{
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted', 'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',

查看文件

@ -63,6 +63,7 @@ def get_args_parser():
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False) parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False) parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
parser.add_argument('--temporal_agg', action='store_true') parser.add_argument('--temporal_agg', action='store_true')
parser.add_argument('--vq', action='store_true')
return parser return parser

查看文件

@ -5,6 +5,7 @@ DETR model and criterion classes.
import torch import torch
from torch import nn from torch import nn
from torch.autograd import Variable from torch.autograd import Variable
import torch.nn.functional as F
from .backbone import build_backbone from .backbone import build_backbone
from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer
@ -33,7 +34,7 @@ def get_sinusoid_encoding_table(n_position, d_hid):
class DETRVAE(nn.Module): class DETRVAE(nn.Module):
""" This is the DETR module that performs object detection """ """ This is the DETR module that performs object detection """
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names): def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names, vq, vq_class, vq_dim, ):
""" Initializes the model. """ Initializes the model.
Parameters: Parameters:
backbones: torch module of the backbone to be used. See backbone.py backbones: torch module of the backbone to be used. See backbone.py
@ -48,6 +49,7 @@ class DETRVAE(nn.Module):
self.camera_names = camera_names self.camera_names = camera_names
self.transformer = transformer self.transformer = transformer
self.encoder = encoder self.encoder = encoder
self.vq, self.vq_class, self.vq_dim = vq, vq_class, vq_dim
hidden_dim = transformer.d_model hidden_dim = transformer.d_model
self.action_head = nn.Linear(hidden_dim, state_dim) self.action_head = nn.Linear(hidden_dim, state_dim)
self.is_pad_head = nn.Linear(hidden_dim, 1) self.is_pad_head = nn.Linear(hidden_dim, 1)
@ -68,20 +70,24 @@ class DETRVAE(nn.Module):
self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding
self.encoder_action_proj = nn.Linear(14, hidden_dim) # project action to embedding self.encoder_action_proj = nn.Linear(14, hidden_dim) # project action to embedding
self.encoder_joint_proj = nn.Linear(14, hidden_dim) # project qpos to embedding self.encoder_joint_proj = nn.Linear(14, hidden_dim) # project qpos to embedding
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
print('Use VQ: ', self.vq)
if self.vq:
self.latent_proj = nn.Linear(hidden_dim, self.vq_class * self.vq_dim)
else:
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
self.register_buffer('pos_table', get_sinusoid_encoding_table(1+1+num_queries, hidden_dim)) # [CLS], qpos, a_seq self.register_buffer('pos_table', get_sinusoid_encoding_table(1+1+num_queries, hidden_dim)) # [CLS], qpos, a_seq
# decoder extra parameters # decoder extra parameters
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding if self.vq:
self.latent_out_proj = nn.Linear(self.vq_class * self.vq_dim, hidden_dim)
else:
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent
def forward(self, qpos, image, env_state, actions=None, is_pad=None):
""" def encode(self, qpos, actions=None, is_pad=None, vq_sample=None):
qpos: batch, qpos_dim # cvae encoder
image: batch, num_cam, channel, height, width
env_state: None
actions: batch, seq, action_dim
"""
is_training = actions is not None # train or val is_training = actions is not None # train or val
bs, _ = qpos.shape bs, _ = qpos.shape
### Obtain latent z from action sequence ### Obtain latent z from action sequence
@ -104,15 +110,43 @@ class DETRVAE(nn.Module):
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad) encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
encoder_output = encoder_output[0] # take cls output only encoder_output = encoder_output[0] # take cls output only
latent_info = self.latent_proj(encoder_output) latent_info = self.latent_proj(encoder_output)
mu = latent_info[:, :self.latent_dim]
logvar = latent_info[:, self.latent_dim:] if self.vq:
latent_sample = reparametrize(mu, logvar) logits = latent_info.reshape([*latent_info.shape[:-1], self.vq_class, self.vq_dim])
latent_input = self.latent_out_proj(latent_sample) probs = torch.softmax(logits, dim=-1)
else: binaries = F.one_hot(torch.multinomial(probs.view(-1, self.vq_dim), 1).squeeze(-1), self.vq_dim).view(-1, self.vq_class, self.vq_dim).float()
mu = logvar = None binaries_flat = binaries.view(-1, self.vq_class * self.vq_dim)
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device) probs_flat = probs.view(-1, self.vq_class * self.vq_dim)
latent_input = self.latent_out_proj(latent_sample) straigt_through = binaries_flat - probs_flat.detach() + probs_flat
latent_input = self.latent_out_proj(straigt_through)
mu = logvar = None
else:
mu = latent_info[:, :self.latent_dim]
logvar = latent_info[:, self.latent_dim:]
latent_sample = reparametrize(mu, logvar)
latent_input = self.latent_out_proj(latent_sample)
else:
mu = logvar = binaries = probs = None
if self.vq:
latent_input = self.latent_out_proj(vq_sample.view(-1, self.vq_class * self.vq_dim))
else:
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
latent_input = self.latent_out_proj(latent_sample)
return latent_input, probs, binaries, mu, logvar
def forward(self, qpos, image, env_state, actions=None, is_pad=None, vq_sample=None):
"""
qpos: batch, qpos_dim
image: batch, num_cam, channel, height, width
env_state: None
actions: batch, seq, action_dim
"""
latent_input, probs, binaries, mu, logvar = self.encode(qpos, actions, is_pad, vq_sample)
# cvae decoder
if self.backbones is not None: if self.backbones is not None:
# Image observation features and position embeddings # Image observation features and position embeddings
all_cam_features = [] all_cam_features = []
@ -136,7 +170,7 @@ class DETRVAE(nn.Module):
hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0] hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0]
a_hat = self.action_head(hs) a_hat = self.action_head(hs)
is_pad_hat = self.is_pad_head(hs) is_pad_hat = self.is_pad_head(hs)
return a_hat, is_pad_hat, [mu, logvar] return a_hat, is_pad_hat, [mu, logvar], probs, binaries
@ -247,6 +281,9 @@ def build(args):
state_dim=state_dim, state_dim=state_dim,
num_queries=args.num_queries, num_queries=args.num_queries,
camera_names=args.camera_names, camera_names=args.camera_names,
vq=args.vq,
vq_class=args.vq_class,
vq_dim=args.vq_dim,
) )
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)

73
detr/models/latent_model.py 普通文件
查看文件

@ -0,0 +1,73 @@
import torch.nn as nn
from torch.nn import functional as F
import torch
DROPOUT_RATE = 0.1
# a causal transformer block
class Causal_Transformer_Block(nn.Module):
def __init__(self, seq_len, latent_dim, num_head) -> None:
super().__init__()
self.num_head = num_head
self.latent_dim = latent_dim
self.ln_1 = nn.LayerNorm(latent_dim)
self.attn = nn.MultiheadAttention(latent_dim, num_head, dropout=DROPOUT_RATE, batch_first=True)
self.ln_2 = nn.LayerNorm(latent_dim)
self.mlp = nn.Sequential(
nn.Linear(latent_dim, 4 * latent_dim),
nn.GELU(),
nn.Linear(4 * latent_dim, latent_dim),
nn.Dropout(DROPOUT_RATE),
)
# self.register_buffer("attn_mask", torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool())
def forward(self, x):
attn_mask = torch.triu(torch.ones(x.shape[1], x.shape[1], device=x.device, dtype=torch.bool), diagonal=1)
x = self.ln_1(x)
x = x + self.attn(x, x, x, attn_mask=attn_mask)[0]
x = self.ln_2(x)
x = x + self.mlp(x)
return x
# use self-attention instead of RNN to model the latent space sequence
class Latent_Model_Transformer(nn.Module):
def __init__(self, input_dim, output_dim, seq_len, latent_dim=256, num_head=8, num_layer=3) -> None:
super().__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.seq_len = seq_len
self.latent_dim = latent_dim
self.num_head = num_head
self.num_layer = num_layer
self.input_layer = nn.Linear(input_dim, latent_dim)
self.weight_pos_embed = nn.Embedding(seq_len, latent_dim)
self.attention_blocks = nn.Sequential(
nn.Dropout(DROPOUT_RATE),
*[Causal_Transformer_Block(seq_len, latent_dim, num_head) for _ in range(num_layer)],
nn.LayerNorm(latent_dim)
)
self.output_layer = nn.Linear(latent_dim, output_dim)
def forward(self, x):
x = self.input_layer(x)
x = x + self.weight_pos_embed(torch.arange(x.shape[1], device=x.device))
x = self.attention_blocks(x)
logits = self.output_layer(x)
return logits
@torch.no_grad()
def generate(self, n, temperature=0.1, x=None):
if x is None:
x = torch.zeros((n, 1, self.input_dim), device=self.weight_pos_embed.weight.device)
for i in range(self.seq_len):
logits = self.forward(x)[:, -1]
probs = torch.softmax(logits / temperature, dim=-1)
samples = torch.multinomial(probs, num_samples=1)[..., 0]
samples_one_hot = F.one_hot(samples.long(), num_classes=self.output_dim).float()
x = torch.cat([x, samples_one_hot[:, None, :]], dim=1)
return x[:, 1:, :]

查看文件

@ -16,6 +16,8 @@ from utils import compute_dict_mean, set_seed, detach_dict # helper functions
from policy import ACTPolicy, CNNMLPPolicy from policy import ACTPolicy, CNNMLPPolicy
from visualize_episodes import save_videos from visualize_episodes import save_videos
from detr.models.latent_model import Latent_Model_Transformer
from sim_env import BOX_POSE from sim_env import BOX_POSE
import IPython import IPython
@ -54,6 +56,8 @@ def main(args):
enc_layers = 4 enc_layers = 4
dec_layers = 7 dec_layers = 7
nheads = 8 nheads = 8
vq_class = 10
vq_dim = 10
policy_config = {'lr': args['lr'], policy_config = {'lr': args['lr'],
'num_queries': args['chunk_size'], 'num_queries': args['chunk_size'],
'kl_weight': args['kl_weight'], 'kl_weight': args['kl_weight'],
@ -65,6 +69,9 @@ def main(args):
'dec_layers': dec_layers, 'dec_layers': dec_layers,
'nheads': nheads, 'nheads': nheads,
'camera_names': camera_names, 'camera_names': camera_names,
'vq': args['vq'],
'vq_class': vq_class,
'vq_dim': vq_dim,
} }
elif policy_class == 'CNNMLP': elif policy_class == 'CNNMLP':
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1, policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
@ -161,6 +168,7 @@ def eval_bc(config, ckpt_name, save_episode=True):
task_name = config['task_name'] task_name = config['task_name']
temporal_agg = config['temporal_agg'] temporal_agg = config['temporal_agg']
onscreen_cam = 'angle' onscreen_cam = 'angle'
vq = config['policy_config']['vq']
# load policy and stats # load policy and stats
ckpt_path = os.path.join(ckpt_dir, ckpt_name) ckpt_path = os.path.join(ckpt_dir, ckpt_name)
@ -169,7 +177,16 @@ def eval_bc(config, ckpt_name, save_episode=True):
print(loading_status) print(loading_status)
policy.cuda() policy.cuda()
policy.eval() policy.eval()
print(f'Loaded: {ckpt_path}') if vq:
vq_dim = config['policy_config']['vq_dim']
vq_class = config['policy_config']['vq_class']
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_best.ckpt')
latent_model.load_state_dict(torch.load(latent_model_ckpt_path))
latent_model.cuda()
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
else:
print(f'Loaded: {ckpt_path}')
stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl') stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
with open(stats_path, 'rb') as f: with open(stats_path, 'rb') as f:
stats = pickle.load(f) stats = pickle.load(f)
@ -246,7 +263,11 @@ def eval_bc(config, ckpt_name, save_episode=True):
### query policy ### query policy
if config['policy_class'] == "ACT": if config['policy_class'] == "ACT":
if t % query_frequency == 0: if t % query_frequency == 0:
all_actions = policy(qpos, curr_image) if vq:
vq_sample = latent_model.generate(1, temperature=0.01, x=None)
all_actions = policy(qpos, curr_image, vq_sample=vq_sample)
else:
all_actions = policy(qpos, curr_image)
if temporal_agg: if temporal_agg:
all_time_actions[[t], t:t+num_queries] = all_actions all_time_actions[[t], t:t+num_queries] = all_actions
actions_for_curr_step = all_time_actions[:, t] actions_for_curr_step = all_time_actions[:, t]
@ -431,5 +452,6 @@ if __name__ == '__main__':
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False) parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False) parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
parser.add_argument('--temporal_agg', action='store_true') parser.add_argument('--temporal_agg', action='store_true')
parser.add_argument('--vq', action='store_true')
main(vars(parser.parse_args())) main(vars(parser.parse_args()))

查看文件

@ -1,6 +1,7 @@
import torch.nn as nn import torch.nn as nn
from torch.nn import functional as F from torch.nn import functional as F
import torchvision.transforms as transforms import torchvision.transforms as transforms
import torch
from detr.main import build_ACT_model_and_optimizer, build_CNNMLP_model_and_optimizer from detr.main import build_ACT_model_and_optimizer, build_CNNMLP_model_and_optimizer
import IPython import IPython
@ -13,9 +14,10 @@ class ACTPolicy(nn.Module):
self.model = model # CVAE decoder self.model = model # CVAE decoder
self.optimizer = optimizer self.optimizer = optimizer
self.kl_weight = args_override['kl_weight'] self.kl_weight = args_override['kl_weight']
self.vq = args_override['vq']
print(f'KL Weight {self.kl_weight}') print(f'KL Weight {self.kl_weight}')
def __call__(self, qpos, image, actions=None, is_pad=None): def __call__(self, qpos, image, actions=None, is_pad=None, vq_sample=None):
env_state = None env_state = None
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]) std=[0.229, 0.224, 0.225])
@ -24,9 +26,14 @@ class ACTPolicy(nn.Module):
actions = actions[:, :self.model.num_queries] actions = actions[:, :self.model.num_queries]
is_pad = is_pad[:, :self.model.num_queries] is_pad = is_pad[:, :self.model.num_queries]
a_hat, is_pad_hat, (mu, logvar) = self.model(qpos, image, env_state, actions, is_pad)
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
loss_dict = dict() loss_dict = dict()
print(vq_sample)
a_hat, is_pad_hat, (mu, logvar), probs, binaries = self.model(qpos, image, env_state, actions, is_pad, vq_sample)
if self.vq:
total_kld = [torch.tensor(0.0)]
loss_dict['vq_loss'] = F.l1_loss(binaries, probs, reduction='mean')
else:
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
all_l1 = F.l1_loss(actions, a_hat, reduction='none') all_l1 = F.l1_loss(actions, a_hat, reduction='none')
l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean() l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean()
loss_dict['l1'] = l1 loss_dict['l1'] = l1
@ -34,12 +41,23 @@ class ACTPolicy(nn.Module):
loss_dict['loss'] = loss_dict['l1'] + loss_dict['kl'] * self.kl_weight loss_dict['loss'] = loss_dict['l1'] + loss_dict['kl'] * self.kl_weight
return loss_dict return loss_dict
else: # inference time else: # inference time
a_hat, _, (_, _) = self.model(qpos, image, env_state) # no action, sample from prior a_hat, _, (_, _), _, _ = self.model(qpos, image, env_state, vq_sample=vq_sample) # no action, sample from prior
return a_hat return a_hat
def configure_optimizers(self): def configure_optimizers(self):
return self.optimizer return self.optimizer
@torch.no_grad()
def vq_encode(self, qpos, actions, is_pad):
actions = actions[:, :self.model.num_queries]
is_pad = is_pad[:, :self.model.num_queries]
_, _, binaries, _, _ = self.model.encode(qpos, actions, is_pad)
return binaries
class CNNMLPPolicy(nn.Module): class CNNMLPPolicy(nn.Module):
def __init__(self, args_override): def __init__(self, args_override):

468
train_latent_model.py 普通文件
查看文件

@ -0,0 +1,468 @@
import torch
import numpy as np
import os
import pickle
import argparse
import matplotlib.pyplot as plt
from copy import deepcopy
from tqdm import tqdm
from einops import rearrange
import torch.nn.functional as F
from constants import DT
from constants import PUPPET_GRIPPER_JOINT_OPEN
from utils import load_data # data functions
from utils import sample_box_pose, sample_insertion_pose # robot functions
from utils import compute_dict_mean, set_seed, detach_dict # helper functions
from policy import ACTPolicy, CNNMLPPolicy
from visualize_episodes import save_videos
from detr.models.latent_model import Latent_Model_Transformer
from sim_env import BOX_POSE
import IPython
e = IPython.embed
def main(args):
set_seed(1)
# command line parameters
is_eval = args['eval']
ckpt_dir = args['ckpt_dir']
policy_class = args['policy_class']
onscreen_render = args['onscreen_render']
task_name = args['task_name']
batch_size_train = args['batch_size']
batch_size_val = args['batch_size']
num_epochs = args['num_epochs']
# get task parameters
is_sim = task_name[:4] == 'sim_'
if is_sim:
from constants import SIM_TASK_CONFIGS
task_config = SIM_TASK_CONFIGS[task_name]
else:
from aloha_scripts.constants import TASK_CONFIGS
task_config = TASK_CONFIGS[task_name]
dataset_dir = task_config['dataset_dir']
num_episodes = task_config['num_episodes']
episode_len = task_config['episode_len']
camera_names = task_config['camera_names']
# fixed parameters
state_dim = 14
lr_backbone = 1e-5
backbone = 'resnet18'
if policy_class == 'ACT':
enc_layers = 4
dec_layers = 7
nheads = 8
vq_class = 10
vq_dim = 10
policy_config = {'lr': args['lr'],
'num_queries': args['chunk_size'],
'kl_weight': args['kl_weight'],
'hidden_dim': args['hidden_dim'],
'dim_feedforward': args['dim_feedforward'],
'lr_backbone': lr_backbone,
'backbone': backbone,
'enc_layers': enc_layers,
'dec_layers': dec_layers,
'nheads': nheads,
'camera_names': camera_names,
'vq': True,
'vq_class': vq_class,
'vq_dim': vq_dim,
}
elif policy_class == 'CNNMLP':
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
'camera_names': camera_names,}
else:
raise NotImplementedError
config = {
'num_epochs': num_epochs,
'ckpt_dir': ckpt_dir,
'episode_len': episode_len,
'state_dim': state_dim,
'lr': args['lr'],
'policy_class': policy_class,
'onscreen_render': onscreen_render,
'policy_config': policy_config,
'task_name': task_name,
'seed': args['seed'],
'temporal_agg': args['temporal_agg'],
'camera_names': camera_names,
'real_robot': not is_sim
}
# if is_eval:
# ckpt_names = [f'policy_best.ckpt']
# results = []
# for ckpt_name in ckpt_names:
# success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True)
# results.append([ckpt_name, success_rate, avg_return])
# for ckpt_name, success_rate, avg_return in results:
# print(f'{ckpt_name}: {success_rate=} {avg_return=}')
# print()
# exit()
train_dataloader, val_dataloader, stats, _ = load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val)
# save dataset stats
# if not os.path.isdir(ckpt_dir):
# os.makedirs(ckpt_dir)
# stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
# with open(stats_path, 'wb') as f:
# pickle.dump(stats, f)
ckpt_name = f'policy_best.ckpt'
best_ckpt_info = train_bc(train_dataloader, val_dataloader, config, ckpt_name)
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
# save best checkpoint
ckpt_path = os.path.join(ckpt_dir, f'latent_model_best.ckpt')
torch.save(best_state_dict, ckpt_path)
print(f'Best ckpt, val loss {min_val_loss:.6f} @ epoch{best_epoch}')
def make_policy(policy_class, policy_config):
if policy_class == 'ACT':
policy = ACTPolicy(policy_config)
elif policy_class == 'CNNMLP':
policy = CNNMLPPolicy(policy_config)
else:
raise NotImplementedError
return policy
# def make_optimizer(policy_class, policy):
# if policy_class == 'ACT':
# optimizer = policy.configure_optimizers()
# elif policy_class == 'CNNMLP':
# optimizer = policy.configure_optimizers()
# else:
# raise NotImplementedError
# return optimizer
def get_image(ts, camera_names):
curr_images = []
for cam_name in camera_names:
curr_image = rearrange(ts.observation['images'][cam_name], 'h w c -> c h w')
curr_images.append(curr_image)
curr_image = np.stack(curr_images, axis=0)
curr_image = torch.from_numpy(curr_image / 255.0).float().cuda().unsqueeze(0)
return curr_image
# def eval_bc(config, ckpt_name, save_episode=True):
# set_seed(1000)
# ckpt_dir = config['ckpt_dir']
# state_dim = config['state_dim']
# real_robot = config['real_robot']
# policy_class = config['policy_class']
# onscreen_render = config['onscreen_render']
# policy_config = config['policy_config']
# camera_names = config['camera_names']
# max_timesteps = config['episode_len']
# task_name = config['task_name']
# temporal_agg = config['temporal_agg']
# onscreen_cam = 'angle'
# # load policy and stats
# ckpt_path = os.path.join(ckpt_dir, ckpt_name)
# policy = make_policy(policy_class, policy_config)
# loading_status = policy.load_state_dict(torch.load(ckpt_path))
# print(loading_status)
# policy.cuda()
# policy.eval()
# print(f'Loaded: {ckpt_path}')
# stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
# with open(stats_path, 'rb') as f:
# stats = pickle.load(f)
# pre_process = lambda s_qpos: (s_qpos - stats['qpos_mean']) / stats['qpos_std']
# post_process = lambda a: a * stats['action_std'] + stats['action_mean']
# # load environment
# if real_robot:
# from aloha_scripts.robot_utils import move_grippers # requires aloha
# from aloha_scripts.real_env import make_real_env # requires aloha
# env = make_real_env(init_node=True)
# env_max_reward = 0
# else:
# from sim_env import make_sim_env
# env = make_sim_env(task_name)
# env_max_reward = env.task.max_reward
# query_frequency = policy_config['num_queries']
# if temporal_agg:
# query_frequency = 1
# num_queries = policy_config['num_queries']
# max_timesteps = int(max_timesteps * 1) # may increase for real-world tasks
# num_rollouts = 50
# episode_returns = []
# highest_rewards = []
# for rollout_id in range(num_rollouts):
# rollout_id += 0
# ### set task
# if 'sim_transfer_cube' in task_name:
# BOX_POSE[0] = sample_box_pose() # used in sim reset
# elif 'sim_insertion' in task_name:
# BOX_POSE[0] = np.concatenate(sample_insertion_pose()) # used in sim reset
# ts = env.reset()
# ### onscreen render
# if onscreen_render:
# ax = plt.subplot()
# plt_img = ax.imshow(env._physics.render(height=480, width=640, camera_id=onscreen_cam))
# plt.ion()
# ### evaluation loop
# if temporal_agg:
# all_time_actions = torch.zeros([max_timesteps, max_timesteps+num_queries, state_dim]).cuda()
# qpos_history = torch.zeros((1, max_timesteps, state_dim)).cuda()
# image_list = [] # for visualization
# qpos_list = []
# target_qpos_list = []
# rewards = []
# with torch.inference_mode():
# for t in range(max_timesteps):
# ### update onscreen render and wait for DT
# if onscreen_render:
# image = env._physics.render(height=480, width=640, camera_id=onscreen_cam)
# plt_img.set_data(image)
# plt.pause(DT)
# ### process previous timestep to get qpos and image_list
# obs = ts.observation
# if 'images' in obs:
# image_list.append(obs['images'])
# else:
# image_list.append({'main': obs['image']})
# qpos_numpy = np.array(obs['qpos'])
# qpos = pre_process(qpos_numpy)
# qpos = torch.from_numpy(qpos).float().cuda().unsqueeze(0)
# qpos_history[:, t] = qpos
# curr_image = get_image(ts, camera_names)
# ### query policy
# if config['policy_class'] == "ACT":
# if t % query_frequency == 0:
# all_actions = policy(qpos, curr_image)
# if temporal_agg:
# all_time_actions[[t], t:t+num_queries] = all_actions
# actions_for_curr_step = all_time_actions[:, t]
# actions_populated = torch.all(actions_for_curr_step != 0, axis=1)
# actions_for_curr_step = actions_for_curr_step[actions_populated]
# k = 0.01
# exp_weights = np.exp(-k * np.arange(len(actions_for_curr_step)))
# exp_weights = exp_weights / exp_weights.sum()
# exp_weights = torch.from_numpy(exp_weights).cuda().unsqueeze(dim=1)
# raw_action = (actions_for_curr_step * exp_weights).sum(dim=0, keepdim=True)
# else:
# raw_action = all_actions[:, t % query_frequency]
# elif config['policy_class'] == "CNNMLP":
# raw_action = policy(qpos, curr_image)
# else:
# raise NotImplementedError
# ### post-process actions
# raw_action = raw_action.squeeze(0).cpu().numpy()
# action = post_process(raw_action)
# target_qpos = action
# ### step the environment
# ts = env.step(target_qpos)
# ### for visualization
# qpos_list.append(qpos_numpy)
# target_qpos_list.append(target_qpos)
# rewards.append(ts.reward)
# plt.close()
# if real_robot:
# move_grippers([env.puppet_bot_left, env.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) # open
# pass
# rewards = np.array(rewards)
# episode_return = np.sum(rewards[rewards!=None])
# episode_returns.append(episode_return)
# episode_highest_reward = np.max(rewards)
# highest_rewards.append(episode_highest_reward)
# print(f'Rollout {rollout_id}\n{episode_return=}, {episode_highest_reward=}, {env_max_reward=}, Success: {episode_highest_reward==env_max_reward}')
# if save_episode:
# save_videos(image_list, DT, video_path=os.path.join(ckpt_dir, f'video{rollout_id}.mp4'))
# success_rate = np.mean(np.array(highest_rewards) == env_max_reward)
# avg_return = np.mean(episode_returns)
# summary_str = f'\nSuccess rate: {success_rate}\nAverage return: {avg_return}\n\n'
# for r in range(env_max_reward+1):
# more_or_equal_r = (np.array(highest_rewards) >= r).sum()
# more_or_equal_r_rate = more_or_equal_r / num_rollouts
# summary_str += f'Reward >= {r}: {more_or_equal_r}/{num_rollouts} = {more_or_equal_r_rate*100}%\n'
# print(summary_str)
# # save success rate to txt
# result_file_name = 'result_' + ckpt_name.split('.')[0] + '.txt'
# with open(os.path.join(ckpt_dir, result_file_name), 'w') as f:
# f.write(summary_str)
# f.write(repr(episode_returns))
# f.write('\n\n')
# f.write(repr(highest_rewards))
# return success_rate, avg_return
def forward_pass(data, policy, latent_model):
image_data, qpos_data, action_data, is_pad = data
image_data, qpos_data, action_data, is_pad = image_data.cuda(), qpos_data.cuda(), action_data.cuda(), is_pad.cuda()
forward_dict = {}
gt_labels = policy.vq_encode(qpos_data, action_data, is_pad)
inputs = torch.cat([torch.zeros_like(gt_labels)[:, [0]], gt_labels[:, :-1]], dim=1)
output_logits = latent_model(inputs)
ce_loss = F.cross_entropy(output_logits, gt_labels)
with torch.no_grad():
output_labels = F.one_hot(torch.argmax(output_logits, dim=-1), num_classes=gt_labels.shape[-1]).float()
# output_latents = F.softmax(output_logits, dim=-1)
l1_error = F.l1_loss(output_labels, gt_labels, reduction='mean')
# l1_errors = []
# for i in range(l1_errors.shape[1]):
# l1_errors.append(torch.mean(l1_errors[:, i]).item())
forward_dict['loss'] = ce_loss
forward_dict['l1_error'] = l1_error
return forward_dict
def train_bc(train_dataloader, val_dataloader, config, ckpt_name):
num_epochs = config['num_epochs']
ckpt_dir = config['ckpt_dir']
seed = config['seed']
policy_class = config['policy_class']
policy_config = config['policy_config']
set_seed(seed)
vq_dim = config['policy_config']['vq_dim']
vq_class = config['policy_config']['vq_class']
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
latent_model.cuda()
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
policy = make_policy(policy_class, policy_config)
loading_status = policy.load_state_dict(torch.load(ckpt_path))
policy.eval()
policy.cuda()
optimizer = torch.optim.AdamW(latent_model.parameters(), lr=config['lr'])
train_history = []
validation_history = []
min_val_loss = np.inf
best_ckpt_info = None
for epoch in tqdm(range(num_epochs)):
print(f'\nEpoch {epoch}')
# validation
with torch.inference_mode():
latent_model.eval()
epoch_dicts = []
for batch_idx, data in enumerate(val_dataloader):
forward_dict = forward_pass(data, policy, latent_model)
epoch_dicts.append(forward_dict)
epoch_summary = compute_dict_mean(epoch_dicts)
validation_history.append(epoch_summary)
epoch_val_loss = epoch_summary['loss']
if epoch_val_loss < min_val_loss:
min_val_loss = epoch_val_loss
best_ckpt_info = (epoch, min_val_loss, deepcopy(latent_model.state_dict()))
print(f'Val loss: {epoch_val_loss:.5f}')
summary_string = ''
for k, v in epoch_summary.items():
summary_string += f'{k}: {v.item():.3f} '
print(summary_string)
# training
optimizer.zero_grad()
for batch_idx, data in enumerate(train_dataloader):
forward_dict = forward_pass(data, policy, latent_model)
# backward
loss = forward_dict['loss']
loss.backward()
optimizer.step()
optimizer.zero_grad()
train_history.append(detach_dict(forward_dict))
epoch_summary = compute_dict_mean(train_history[(batch_idx+1)*epoch:(batch_idx+1)*(epoch+1)])
epoch_train_loss = epoch_summary['loss']
print(f'Train loss: {epoch_train_loss:.5f}')
summary_string = ''
for k, v in epoch_summary.items():
summary_string += f'{k}: {v.item():.3f} '
print(summary_string)
if epoch % 100 == 0:
ckpt_path = os.path.join(ckpt_dir, f'latent_model_epoch_{epoch}_seed_{seed}.ckpt')
torch.save(latent_model.state_dict(), ckpt_path)
plot_history(train_history, validation_history, epoch, ckpt_dir, seed)
ckpt_path = os.path.join(ckpt_dir, f'latent_model_last.ckpt')
torch.save(latent_model.state_dict(), ckpt_path)
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
ckpt_path = os.path.join(ckpt_dir, f'latent_model_epoch_{best_epoch}_seed_{seed}.ckpt')
torch.save(best_state_dict, ckpt_path)
print(f'Training finished:\nSeed {seed}, val loss {min_val_loss:.6f} at epoch {best_epoch}')
# save training curves
plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed)
return best_ckpt_info
def plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed):
# save training curves
for key in train_history[0]:
plot_path = os.path.join(ckpt_dir, f'latent_model_val_{key}_seed_{seed}.png')
plt.figure()
train_values = [summary[key].item() for summary in train_history]
val_values = [summary[key].item() for summary in validation_history]
plt.plot(np.linspace(0, num_epochs-1, len(train_history)), train_values, label='train')
plt.plot(np.linspace(0, num_epochs-1, len(validation_history)), val_values, label='validation')
# plt.ylim([-0.1, 1])
plt.tight_layout()
plt.legend()
plt.title(key)
plt.savefig(plot_path)
print(f'Saved plots to {ckpt_dir}')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--eval', action='store_true')
parser.add_argument('--onscreen_render', action='store_true')
parser.add_argument('--ckpt_dir', action='store', type=str, help='ckpt_dir', required=True)
parser.add_argument('--policy_class', action='store', type=str, help='policy_class, capitalize', required=True)
parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True)
parser.add_argument('--batch_size', action='store', type=int, help='batch_size', required=True)
parser.add_argument('--seed', action='store', type=int, help='seed', required=True)
parser.add_argument('--num_epochs', action='store', type=int, help='num_epochs', required=True)
parser.add_argument('--lr', action='store', type=float, help='lr', required=True)
# for ACT
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
parser.add_argument('--temporal_agg', action='store_true')
main(vars(parser.parse_args()))

查看文件

@ -81,10 +81,14 @@ def get_norm_stats(dataset_dir, num_episodes):
all_action_data = [] all_action_data = []
for episode_idx in range(num_episodes): for episode_idx in range(num_episodes):
dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5') dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5')
with h5py.File(dataset_path, 'r') as root: try:
qpos = root['/observations/qpos'][()] with h5py.File(dataset_path, 'r') as root:
qvel = root['/observations/qvel'][()] qpos = root['/observations/qpos'][()]
action = root['/action'][()] qvel = root['/observations/qvel'][()]
action = root['/action'][()]
except:
print(f'Error loading {dataset_path}')
quit()
all_qpos_data.append(torch.from_numpy(qpos)) all_qpos_data.append(torch.from_numpy(qpos))
all_action_data.append(torch.from_numpy(action)) all_action_data.append(torch.from_numpy(action))
all_qpos_data = torch.stack(all_qpos_data) all_qpos_data = torch.stack(all_qpos_data)