vq init
这个提交包含在:
4
.gitignore
vendored
4
.gitignore
vendored
@ -137,4 +137,6 @@ celerybeat.pid
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
.pyre/
|
||||
|
||||
ckpts/
|
||||
12
README.md
12
README.md
@ -57,10 +57,7 @@ To set up a new terminal, run:
|
||||
We use ``sim_transfer_cube_scripted`` task in the examples below. Another option is ``sim_insertion_scripted``.
|
||||
To generated 50 episodes of scripted data, run:
|
||||
|
||||
python3 record_sim_episodes.py \
|
||||
--task_name sim_transfer_cube_scripted \
|
||||
--dataset_dir <data save dir> \
|
||||
--num_episodes 50
|
||||
python3 record_sim_episodes.py --task_name sim_transfer_cube_scripted --dataset_dir <data save dir> --num_episodes 50
|
||||
|
||||
To can add the flag ``--onscreen_render`` to see real-time rendering.
|
||||
To visualize the episode after it is collected, run
|
||||
@ -70,12 +67,7 @@ To visualize the episode after it is collected, run
|
||||
To train ACT:
|
||||
|
||||
# Transfer Cube task
|
||||
python3 imitate_episodes.py \
|
||||
--task_name sim_transfer_cube_scripted \
|
||||
--ckpt_dir <ckpt dir> \
|
||||
--policy_class ACT --kl_weight 10 --chunk_size 100 --hidden_dim 512 --batch_size 8 --dim_feedforward 3200 \
|
||||
--num_epochs 2000 --lr 1e-5 \
|
||||
--seed 0
|
||||
python3 imitate_episodes.py --task_name sim_transfer_cube_scripted --ckpt_dir <ckpt dir> --policy_class ACT --kl_weight 10 --chunk_size 100 --hidden_dim 512 --batch_size 8 --dim_feedforward 3200 --num_epochs 2000 --lr 1e-5 --seed 0
|
||||
|
||||
|
||||
To evaluate the policy, run the same command but add ``--eval``. This loads the best validation checkpoint.
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import pathlib
|
||||
|
||||
### Task parameters
|
||||
DATA_DIR = '<put your data dir here>'
|
||||
DATA_DIR = 'data'
|
||||
SIM_TASK_CONFIGS = {
|
||||
'sim_transfer_cube_scripted':{
|
||||
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',
|
||||
|
||||
@ -63,6 +63,7 @@ def get_args_parser():
|
||||
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
|
||||
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
|
||||
parser.add_argument('--temporal_agg', action='store_true')
|
||||
parser.add_argument('--vq', action='store_true')
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@ DETR model and criterion classes.
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.autograd import Variable
|
||||
import torch.nn.functional as F
|
||||
from .backbone import build_backbone
|
||||
from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer
|
||||
|
||||
@ -33,7 +34,7 @@ def get_sinusoid_encoding_table(n_position, d_hid):
|
||||
|
||||
class DETRVAE(nn.Module):
|
||||
""" This is the DETR module that performs object detection """
|
||||
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names):
|
||||
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names, vq, vq_class, vq_dim, ):
|
||||
""" Initializes the model.
|
||||
Parameters:
|
||||
backbones: torch module of the backbone to be used. See backbone.py
|
||||
@ -48,6 +49,7 @@ class DETRVAE(nn.Module):
|
||||
self.camera_names = camera_names
|
||||
self.transformer = transformer
|
||||
self.encoder = encoder
|
||||
self.vq, self.vq_class, self.vq_dim = vq, vq_class, vq_dim
|
||||
hidden_dim = transformer.d_model
|
||||
self.action_head = nn.Linear(hidden_dim, state_dim)
|
||||
self.is_pad_head = nn.Linear(hidden_dim, 1)
|
||||
@ -68,20 +70,24 @@ class DETRVAE(nn.Module):
|
||||
self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding
|
||||
self.encoder_action_proj = nn.Linear(14, hidden_dim) # project action to embedding
|
||||
self.encoder_joint_proj = nn.Linear(14, hidden_dim) # project qpos to embedding
|
||||
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
|
||||
|
||||
print('Use VQ: ', self.vq)
|
||||
if self.vq:
|
||||
self.latent_proj = nn.Linear(hidden_dim, self.vq_class * self.vq_dim)
|
||||
else:
|
||||
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
|
||||
self.register_buffer('pos_table', get_sinusoid_encoding_table(1+1+num_queries, hidden_dim)) # [CLS], qpos, a_seq
|
||||
|
||||
# decoder extra parameters
|
||||
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
|
||||
if self.vq:
|
||||
self.latent_out_proj = nn.Linear(self.vq_class * self.vq_dim, hidden_dim)
|
||||
else:
|
||||
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
|
||||
self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent
|
||||
|
||||
def forward(self, qpos, image, env_state, actions=None, is_pad=None):
|
||||
"""
|
||||
qpos: batch, qpos_dim
|
||||
image: batch, num_cam, channel, height, width
|
||||
env_state: None
|
||||
actions: batch, seq, action_dim
|
||||
"""
|
||||
|
||||
def encode(self, qpos, actions=None, is_pad=None, vq_sample=None):
|
||||
# cvae encoder
|
||||
is_training = actions is not None # train or val
|
||||
bs, _ = qpos.shape
|
||||
### Obtain latent z from action sequence
|
||||
@ -104,15 +110,43 @@ class DETRVAE(nn.Module):
|
||||
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
|
||||
encoder_output = encoder_output[0] # take cls output only
|
||||
latent_info = self.latent_proj(encoder_output)
|
||||
mu = latent_info[:, :self.latent_dim]
|
||||
logvar = latent_info[:, self.latent_dim:]
|
||||
latent_sample = reparametrize(mu, logvar)
|
||||
latent_input = self.latent_out_proj(latent_sample)
|
||||
else:
|
||||
mu = logvar = None
|
||||
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
||||
latent_input = self.latent_out_proj(latent_sample)
|
||||
|
||||
if self.vq:
|
||||
logits = latent_info.reshape([*latent_info.shape[:-1], self.vq_class, self.vq_dim])
|
||||
probs = torch.softmax(logits, dim=-1)
|
||||
binaries = F.one_hot(torch.multinomial(probs.view(-1, self.vq_dim), 1).squeeze(-1), self.vq_dim).view(-1, self.vq_class, self.vq_dim).float()
|
||||
binaries_flat = binaries.view(-1, self.vq_class * self.vq_dim)
|
||||
probs_flat = probs.view(-1, self.vq_class * self.vq_dim)
|
||||
straigt_through = binaries_flat - probs_flat.detach() + probs_flat
|
||||
latent_input = self.latent_out_proj(straigt_through)
|
||||
mu = logvar = None
|
||||
else:
|
||||
mu = latent_info[:, :self.latent_dim]
|
||||
logvar = latent_info[:, self.latent_dim:]
|
||||
latent_sample = reparametrize(mu, logvar)
|
||||
latent_input = self.latent_out_proj(latent_sample)
|
||||
|
||||
else:
|
||||
mu = logvar = binaries = probs = None
|
||||
if self.vq:
|
||||
latent_input = self.latent_out_proj(vq_sample.view(-1, self.vq_class * self.vq_dim))
|
||||
else:
|
||||
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
||||
latent_input = self.latent_out_proj(latent_sample)
|
||||
|
||||
|
||||
return latent_input, probs, binaries, mu, logvar
|
||||
|
||||
def forward(self, qpos, image, env_state, actions=None, is_pad=None, vq_sample=None):
|
||||
"""
|
||||
qpos: batch, qpos_dim
|
||||
image: batch, num_cam, channel, height, width
|
||||
env_state: None
|
||||
actions: batch, seq, action_dim
|
||||
"""
|
||||
latent_input, probs, binaries, mu, logvar = self.encode(qpos, actions, is_pad, vq_sample)
|
||||
|
||||
# cvae decoder
|
||||
if self.backbones is not None:
|
||||
# Image observation features and position embeddings
|
||||
all_cam_features = []
|
||||
@ -136,7 +170,7 @@ class DETRVAE(nn.Module):
|
||||
hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0]
|
||||
a_hat = self.action_head(hs)
|
||||
is_pad_hat = self.is_pad_head(hs)
|
||||
return a_hat, is_pad_hat, [mu, logvar]
|
||||
return a_hat, is_pad_hat, [mu, logvar], probs, binaries
|
||||
|
||||
|
||||
|
||||
@ -247,6 +281,9 @@ def build(args):
|
||||
state_dim=state_dim,
|
||||
num_queries=args.num_queries,
|
||||
camera_names=args.camera_names,
|
||||
vq=args.vq,
|
||||
vq_class=args.vq_class,
|
||||
vq_dim=args.vq_dim,
|
||||
)
|
||||
|
||||
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||
|
||||
73
detr/models/latent_model.py
普通文件
73
detr/models/latent_model.py
普通文件
@ -0,0 +1,73 @@
|
||||
import torch.nn as nn
|
||||
from torch.nn import functional as F
|
||||
import torch
|
||||
|
||||
DROPOUT_RATE = 0.1
|
||||
|
||||
# a causal transformer block
|
||||
class Causal_Transformer_Block(nn.Module):
|
||||
def __init__(self, seq_len, latent_dim, num_head) -> None:
|
||||
super().__init__()
|
||||
self.num_head = num_head
|
||||
self.latent_dim = latent_dim
|
||||
self.ln_1 = nn.LayerNorm(latent_dim)
|
||||
self.attn = nn.MultiheadAttention(latent_dim, num_head, dropout=DROPOUT_RATE, batch_first=True)
|
||||
self.ln_2 = nn.LayerNorm(latent_dim)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(latent_dim, 4 * latent_dim),
|
||||
nn.GELU(),
|
||||
nn.Linear(4 * latent_dim, latent_dim),
|
||||
nn.Dropout(DROPOUT_RATE),
|
||||
)
|
||||
|
||||
# self.register_buffer("attn_mask", torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool())
|
||||
|
||||
def forward(self, x):
|
||||
attn_mask = torch.triu(torch.ones(x.shape[1], x.shape[1], device=x.device, dtype=torch.bool), diagonal=1)
|
||||
x = self.ln_1(x)
|
||||
x = x + self.attn(x, x, x, attn_mask=attn_mask)[0]
|
||||
x = self.ln_2(x)
|
||||
x = x + self.mlp(x)
|
||||
|
||||
return x
|
||||
|
||||
# use self-attention instead of RNN to model the latent space sequence
|
||||
class Latent_Model_Transformer(nn.Module):
|
||||
def __init__(self, input_dim, output_dim, seq_len, latent_dim=256, num_head=8, num_layer=3) -> None:
|
||||
super().__init__()
|
||||
self.input_dim = input_dim
|
||||
self.output_dim = output_dim
|
||||
self.seq_len = seq_len
|
||||
self.latent_dim = latent_dim
|
||||
self.num_head = num_head
|
||||
self.num_layer = num_layer
|
||||
self.input_layer = nn.Linear(input_dim, latent_dim)
|
||||
self.weight_pos_embed = nn.Embedding(seq_len, latent_dim)
|
||||
self.attention_blocks = nn.Sequential(
|
||||
nn.Dropout(DROPOUT_RATE),
|
||||
*[Causal_Transformer_Block(seq_len, latent_dim, num_head) for _ in range(num_layer)],
|
||||
nn.LayerNorm(latent_dim)
|
||||
)
|
||||
self.output_layer = nn.Linear(latent_dim, output_dim)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.input_layer(x)
|
||||
x = x + self.weight_pos_embed(torch.arange(x.shape[1], device=x.device))
|
||||
x = self.attention_blocks(x)
|
||||
logits = self.output_layer(x)
|
||||
|
||||
return logits
|
||||
|
||||
@torch.no_grad()
|
||||
def generate(self, n, temperature=0.1, x=None):
|
||||
if x is None:
|
||||
x = torch.zeros((n, 1, self.input_dim), device=self.weight_pos_embed.weight.device)
|
||||
for i in range(self.seq_len):
|
||||
logits = self.forward(x)[:, -1]
|
||||
probs = torch.softmax(logits / temperature, dim=-1)
|
||||
samples = torch.multinomial(probs, num_samples=1)[..., 0]
|
||||
samples_one_hot = F.one_hot(samples.long(), num_classes=self.output_dim).float()
|
||||
x = torch.cat([x, samples_one_hot[:, None, :]], dim=1)
|
||||
|
||||
return x[:, 1:, :]
|
||||
|
||||
@ -16,6 +16,8 @@ from utils import compute_dict_mean, set_seed, detach_dict # helper functions
|
||||
from policy import ACTPolicy, CNNMLPPolicy
|
||||
from visualize_episodes import save_videos
|
||||
|
||||
from detr.models.latent_model import Latent_Model_Transformer
|
||||
|
||||
from sim_env import BOX_POSE
|
||||
|
||||
import IPython
|
||||
@ -54,6 +56,8 @@ def main(args):
|
||||
enc_layers = 4
|
||||
dec_layers = 7
|
||||
nheads = 8
|
||||
vq_class = 10
|
||||
vq_dim = 10
|
||||
policy_config = {'lr': args['lr'],
|
||||
'num_queries': args['chunk_size'],
|
||||
'kl_weight': args['kl_weight'],
|
||||
@ -65,6 +69,9 @@ def main(args):
|
||||
'dec_layers': dec_layers,
|
||||
'nheads': nheads,
|
||||
'camera_names': camera_names,
|
||||
'vq': args['vq'],
|
||||
'vq_class': vq_class,
|
||||
'vq_dim': vq_dim,
|
||||
}
|
||||
elif policy_class == 'CNNMLP':
|
||||
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
||||
@ -161,6 +168,7 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
||||
task_name = config['task_name']
|
||||
temporal_agg = config['temporal_agg']
|
||||
onscreen_cam = 'angle'
|
||||
vq = config['policy_config']['vq']
|
||||
|
||||
# load policy and stats
|
||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||
@ -169,7 +177,16 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
||||
print(loading_status)
|
||||
policy.cuda()
|
||||
policy.eval()
|
||||
print(f'Loaded: {ckpt_path}')
|
||||
if vq:
|
||||
vq_dim = config['policy_config']['vq_dim']
|
||||
vq_class = config['policy_config']['vq_class']
|
||||
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
||||
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_best.ckpt')
|
||||
latent_model.load_state_dict(torch.load(latent_model_ckpt_path))
|
||||
latent_model.cuda()
|
||||
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
|
||||
else:
|
||||
print(f'Loaded: {ckpt_path}')
|
||||
stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
||||
with open(stats_path, 'rb') as f:
|
||||
stats = pickle.load(f)
|
||||
@ -246,7 +263,11 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
||||
### query policy
|
||||
if config['policy_class'] == "ACT":
|
||||
if t % query_frequency == 0:
|
||||
all_actions = policy(qpos, curr_image)
|
||||
if vq:
|
||||
vq_sample = latent_model.generate(1, temperature=0.01, x=None)
|
||||
all_actions = policy(qpos, curr_image, vq_sample=vq_sample)
|
||||
else:
|
||||
all_actions = policy(qpos, curr_image)
|
||||
if temporal_agg:
|
||||
all_time_actions[[t], t:t+num_queries] = all_actions
|
||||
actions_for_curr_step = all_time_actions[:, t]
|
||||
@ -431,5 +452,6 @@ if __name__ == '__main__':
|
||||
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
|
||||
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
|
||||
parser.add_argument('--temporal_agg', action='store_true')
|
||||
parser.add_argument('--vq', action='store_true')
|
||||
|
||||
main(vars(parser.parse_args()))
|
||||
|
||||
26
policy.py
26
policy.py
@ -1,6 +1,7 @@
|
||||
import torch.nn as nn
|
||||
from torch.nn import functional as F
|
||||
import torchvision.transforms as transforms
|
||||
import torch
|
||||
|
||||
from detr.main import build_ACT_model_and_optimizer, build_CNNMLP_model_and_optimizer
|
||||
import IPython
|
||||
@ -13,9 +14,10 @@ class ACTPolicy(nn.Module):
|
||||
self.model = model # CVAE decoder
|
||||
self.optimizer = optimizer
|
||||
self.kl_weight = args_override['kl_weight']
|
||||
self.vq = args_override['vq']
|
||||
print(f'KL Weight {self.kl_weight}')
|
||||
|
||||
def __call__(self, qpos, image, actions=None, is_pad=None):
|
||||
def __call__(self, qpos, image, actions=None, is_pad=None, vq_sample=None):
|
||||
env_state = None
|
||||
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
|
||||
std=[0.229, 0.224, 0.225])
|
||||
@ -24,9 +26,14 @@ class ACTPolicy(nn.Module):
|
||||
actions = actions[:, :self.model.num_queries]
|
||||
is_pad = is_pad[:, :self.model.num_queries]
|
||||
|
||||
a_hat, is_pad_hat, (mu, logvar) = self.model(qpos, image, env_state, actions, is_pad)
|
||||
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
|
||||
loss_dict = dict()
|
||||
print(vq_sample)
|
||||
a_hat, is_pad_hat, (mu, logvar), probs, binaries = self.model(qpos, image, env_state, actions, is_pad, vq_sample)
|
||||
if self.vq:
|
||||
total_kld = [torch.tensor(0.0)]
|
||||
loss_dict['vq_loss'] = F.l1_loss(binaries, probs, reduction='mean')
|
||||
else:
|
||||
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
|
||||
all_l1 = F.l1_loss(actions, a_hat, reduction='none')
|
||||
l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean()
|
||||
loss_dict['l1'] = l1
|
||||
@ -34,12 +41,23 @@ class ACTPolicy(nn.Module):
|
||||
loss_dict['loss'] = loss_dict['l1'] + loss_dict['kl'] * self.kl_weight
|
||||
return loss_dict
|
||||
else: # inference time
|
||||
a_hat, _, (_, _) = self.model(qpos, image, env_state) # no action, sample from prior
|
||||
a_hat, _, (_, _), _, _ = self.model(qpos, image, env_state, vq_sample=vq_sample) # no action, sample from prior
|
||||
return a_hat
|
||||
|
||||
def configure_optimizers(self):
|
||||
return self.optimizer
|
||||
|
||||
@torch.no_grad()
|
||||
def vq_encode(self, qpos, actions, is_pad):
|
||||
actions = actions[:, :self.model.num_queries]
|
||||
is_pad = is_pad[:, :self.model.num_queries]
|
||||
|
||||
_, _, binaries, _, _ = self.model.encode(qpos, actions, is_pad)
|
||||
|
||||
return binaries
|
||||
|
||||
|
||||
|
||||
|
||||
class CNNMLPPolicy(nn.Module):
|
||||
def __init__(self, args_override):
|
||||
|
||||
468
train_latent_model.py
普通文件
468
train_latent_model.py
普通文件
@ -0,0 +1,468 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
import os
|
||||
import pickle
|
||||
import argparse
|
||||
import matplotlib.pyplot as plt
|
||||
from copy import deepcopy
|
||||
from tqdm import tqdm
|
||||
from einops import rearrange
|
||||
import torch.nn.functional as F
|
||||
|
||||
from constants import DT
|
||||
from constants import PUPPET_GRIPPER_JOINT_OPEN
|
||||
from utils import load_data # data functions
|
||||
from utils import sample_box_pose, sample_insertion_pose # robot functions
|
||||
from utils import compute_dict_mean, set_seed, detach_dict # helper functions
|
||||
from policy import ACTPolicy, CNNMLPPolicy
|
||||
from visualize_episodes import save_videos
|
||||
from detr.models.latent_model import Latent_Model_Transformer
|
||||
|
||||
from sim_env import BOX_POSE
|
||||
|
||||
import IPython
|
||||
e = IPython.embed
|
||||
|
||||
def main(args):
|
||||
set_seed(1)
|
||||
# command line parameters
|
||||
is_eval = args['eval']
|
||||
ckpt_dir = args['ckpt_dir']
|
||||
policy_class = args['policy_class']
|
||||
onscreen_render = args['onscreen_render']
|
||||
task_name = args['task_name']
|
||||
batch_size_train = args['batch_size']
|
||||
batch_size_val = args['batch_size']
|
||||
num_epochs = args['num_epochs']
|
||||
|
||||
# get task parameters
|
||||
is_sim = task_name[:4] == 'sim_'
|
||||
if is_sim:
|
||||
from constants import SIM_TASK_CONFIGS
|
||||
task_config = SIM_TASK_CONFIGS[task_name]
|
||||
else:
|
||||
from aloha_scripts.constants import TASK_CONFIGS
|
||||
task_config = TASK_CONFIGS[task_name]
|
||||
dataset_dir = task_config['dataset_dir']
|
||||
num_episodes = task_config['num_episodes']
|
||||
episode_len = task_config['episode_len']
|
||||
camera_names = task_config['camera_names']
|
||||
|
||||
# fixed parameters
|
||||
state_dim = 14
|
||||
lr_backbone = 1e-5
|
||||
backbone = 'resnet18'
|
||||
if policy_class == 'ACT':
|
||||
enc_layers = 4
|
||||
dec_layers = 7
|
||||
nheads = 8
|
||||
vq_class = 10
|
||||
vq_dim = 10
|
||||
policy_config = {'lr': args['lr'],
|
||||
'num_queries': args['chunk_size'],
|
||||
'kl_weight': args['kl_weight'],
|
||||
'hidden_dim': args['hidden_dim'],
|
||||
'dim_feedforward': args['dim_feedforward'],
|
||||
'lr_backbone': lr_backbone,
|
||||
'backbone': backbone,
|
||||
'enc_layers': enc_layers,
|
||||
'dec_layers': dec_layers,
|
||||
'nheads': nheads,
|
||||
'camera_names': camera_names,
|
||||
'vq': True,
|
||||
'vq_class': vq_class,
|
||||
'vq_dim': vq_dim,
|
||||
}
|
||||
elif policy_class == 'CNNMLP':
|
||||
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
||||
'camera_names': camera_names,}
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
config = {
|
||||
'num_epochs': num_epochs,
|
||||
'ckpt_dir': ckpt_dir,
|
||||
'episode_len': episode_len,
|
||||
'state_dim': state_dim,
|
||||
'lr': args['lr'],
|
||||
'policy_class': policy_class,
|
||||
'onscreen_render': onscreen_render,
|
||||
'policy_config': policy_config,
|
||||
'task_name': task_name,
|
||||
'seed': args['seed'],
|
||||
'temporal_agg': args['temporal_agg'],
|
||||
'camera_names': camera_names,
|
||||
'real_robot': not is_sim
|
||||
}
|
||||
|
||||
# if is_eval:
|
||||
# ckpt_names = [f'policy_best.ckpt']
|
||||
# results = []
|
||||
# for ckpt_name in ckpt_names:
|
||||
# success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True)
|
||||
# results.append([ckpt_name, success_rate, avg_return])
|
||||
|
||||
# for ckpt_name, success_rate, avg_return in results:
|
||||
# print(f'{ckpt_name}: {success_rate=} {avg_return=}')
|
||||
# print()
|
||||
# exit()
|
||||
|
||||
train_dataloader, val_dataloader, stats, _ = load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val)
|
||||
|
||||
# save dataset stats
|
||||
# if not os.path.isdir(ckpt_dir):
|
||||
# os.makedirs(ckpt_dir)
|
||||
# stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
||||
# with open(stats_path, 'wb') as f:
|
||||
# pickle.dump(stats, f)
|
||||
|
||||
ckpt_name = f'policy_best.ckpt'
|
||||
best_ckpt_info = train_bc(train_dataloader, val_dataloader, config, ckpt_name)
|
||||
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
|
||||
|
||||
# save best checkpoint
|
||||
ckpt_path = os.path.join(ckpt_dir, f'latent_model_best.ckpt')
|
||||
torch.save(best_state_dict, ckpt_path)
|
||||
print(f'Best ckpt, val loss {min_val_loss:.6f} @ epoch{best_epoch}')
|
||||
|
||||
|
||||
def make_policy(policy_class, policy_config):
|
||||
if policy_class == 'ACT':
|
||||
policy = ACTPolicy(policy_config)
|
||||
elif policy_class == 'CNNMLP':
|
||||
policy = CNNMLPPolicy(policy_config)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
return policy
|
||||
|
||||
|
||||
# def make_optimizer(policy_class, policy):
|
||||
# if policy_class == 'ACT':
|
||||
# optimizer = policy.configure_optimizers()
|
||||
# elif policy_class == 'CNNMLP':
|
||||
# optimizer = policy.configure_optimizers()
|
||||
# else:
|
||||
# raise NotImplementedError
|
||||
# return optimizer
|
||||
|
||||
|
||||
def get_image(ts, camera_names):
|
||||
curr_images = []
|
||||
for cam_name in camera_names:
|
||||
curr_image = rearrange(ts.observation['images'][cam_name], 'h w c -> c h w')
|
||||
curr_images.append(curr_image)
|
||||
curr_image = np.stack(curr_images, axis=0)
|
||||
curr_image = torch.from_numpy(curr_image / 255.0).float().cuda().unsqueeze(0)
|
||||
return curr_image
|
||||
|
||||
|
||||
# def eval_bc(config, ckpt_name, save_episode=True):
|
||||
# set_seed(1000)
|
||||
# ckpt_dir = config['ckpt_dir']
|
||||
# state_dim = config['state_dim']
|
||||
# real_robot = config['real_robot']
|
||||
# policy_class = config['policy_class']
|
||||
# onscreen_render = config['onscreen_render']
|
||||
# policy_config = config['policy_config']
|
||||
# camera_names = config['camera_names']
|
||||
# max_timesteps = config['episode_len']
|
||||
# task_name = config['task_name']
|
||||
# temporal_agg = config['temporal_agg']
|
||||
# onscreen_cam = 'angle'
|
||||
|
||||
# # load policy and stats
|
||||
# ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||
# policy = make_policy(policy_class, policy_config)
|
||||
# loading_status = policy.load_state_dict(torch.load(ckpt_path))
|
||||
# print(loading_status)
|
||||
# policy.cuda()
|
||||
# policy.eval()
|
||||
# print(f'Loaded: {ckpt_path}')
|
||||
# stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
||||
# with open(stats_path, 'rb') as f:
|
||||
# stats = pickle.load(f)
|
||||
|
||||
# pre_process = lambda s_qpos: (s_qpos - stats['qpos_mean']) / stats['qpos_std']
|
||||
# post_process = lambda a: a * stats['action_std'] + stats['action_mean']
|
||||
|
||||
# # load environment
|
||||
# if real_robot:
|
||||
# from aloha_scripts.robot_utils import move_grippers # requires aloha
|
||||
# from aloha_scripts.real_env import make_real_env # requires aloha
|
||||
# env = make_real_env(init_node=True)
|
||||
# env_max_reward = 0
|
||||
# else:
|
||||
# from sim_env import make_sim_env
|
||||
# env = make_sim_env(task_name)
|
||||
# env_max_reward = env.task.max_reward
|
||||
|
||||
# query_frequency = policy_config['num_queries']
|
||||
# if temporal_agg:
|
||||
# query_frequency = 1
|
||||
# num_queries = policy_config['num_queries']
|
||||
|
||||
# max_timesteps = int(max_timesteps * 1) # may increase for real-world tasks
|
||||
|
||||
# num_rollouts = 50
|
||||
# episode_returns = []
|
||||
# highest_rewards = []
|
||||
# for rollout_id in range(num_rollouts):
|
||||
# rollout_id += 0
|
||||
# ### set task
|
||||
# if 'sim_transfer_cube' in task_name:
|
||||
# BOX_POSE[0] = sample_box_pose() # used in sim reset
|
||||
# elif 'sim_insertion' in task_name:
|
||||
# BOX_POSE[0] = np.concatenate(sample_insertion_pose()) # used in sim reset
|
||||
|
||||
# ts = env.reset()
|
||||
|
||||
# ### onscreen render
|
||||
# if onscreen_render:
|
||||
# ax = plt.subplot()
|
||||
# plt_img = ax.imshow(env._physics.render(height=480, width=640, camera_id=onscreen_cam))
|
||||
# plt.ion()
|
||||
|
||||
# ### evaluation loop
|
||||
# if temporal_agg:
|
||||
# all_time_actions = torch.zeros([max_timesteps, max_timesteps+num_queries, state_dim]).cuda()
|
||||
|
||||
# qpos_history = torch.zeros((1, max_timesteps, state_dim)).cuda()
|
||||
# image_list = [] # for visualization
|
||||
# qpos_list = []
|
||||
# target_qpos_list = []
|
||||
# rewards = []
|
||||
# with torch.inference_mode():
|
||||
# for t in range(max_timesteps):
|
||||
# ### update onscreen render and wait for DT
|
||||
# if onscreen_render:
|
||||
# image = env._physics.render(height=480, width=640, camera_id=onscreen_cam)
|
||||
# plt_img.set_data(image)
|
||||
# plt.pause(DT)
|
||||
|
||||
# ### process previous timestep to get qpos and image_list
|
||||
# obs = ts.observation
|
||||
# if 'images' in obs:
|
||||
# image_list.append(obs['images'])
|
||||
# else:
|
||||
# image_list.append({'main': obs['image']})
|
||||
# qpos_numpy = np.array(obs['qpos'])
|
||||
# qpos = pre_process(qpos_numpy)
|
||||
# qpos = torch.from_numpy(qpos).float().cuda().unsqueeze(0)
|
||||
# qpos_history[:, t] = qpos
|
||||
# curr_image = get_image(ts, camera_names)
|
||||
|
||||
# ### query policy
|
||||
# if config['policy_class'] == "ACT":
|
||||
# if t % query_frequency == 0:
|
||||
# all_actions = policy(qpos, curr_image)
|
||||
# if temporal_agg:
|
||||
# all_time_actions[[t], t:t+num_queries] = all_actions
|
||||
# actions_for_curr_step = all_time_actions[:, t]
|
||||
# actions_populated = torch.all(actions_for_curr_step != 0, axis=1)
|
||||
# actions_for_curr_step = actions_for_curr_step[actions_populated]
|
||||
# k = 0.01
|
||||
# exp_weights = np.exp(-k * np.arange(len(actions_for_curr_step)))
|
||||
# exp_weights = exp_weights / exp_weights.sum()
|
||||
# exp_weights = torch.from_numpy(exp_weights).cuda().unsqueeze(dim=1)
|
||||
# raw_action = (actions_for_curr_step * exp_weights).sum(dim=0, keepdim=True)
|
||||
# else:
|
||||
# raw_action = all_actions[:, t % query_frequency]
|
||||
# elif config['policy_class'] == "CNNMLP":
|
||||
# raw_action = policy(qpos, curr_image)
|
||||
# else:
|
||||
# raise NotImplementedError
|
||||
|
||||
# ### post-process actions
|
||||
# raw_action = raw_action.squeeze(0).cpu().numpy()
|
||||
# action = post_process(raw_action)
|
||||
# target_qpos = action
|
||||
|
||||
# ### step the environment
|
||||
# ts = env.step(target_qpos)
|
||||
|
||||
# ### for visualization
|
||||
# qpos_list.append(qpos_numpy)
|
||||
# target_qpos_list.append(target_qpos)
|
||||
# rewards.append(ts.reward)
|
||||
|
||||
# plt.close()
|
||||
# if real_robot:
|
||||
# move_grippers([env.puppet_bot_left, env.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) # open
|
||||
# pass
|
||||
|
||||
# rewards = np.array(rewards)
|
||||
# episode_return = np.sum(rewards[rewards!=None])
|
||||
# episode_returns.append(episode_return)
|
||||
# episode_highest_reward = np.max(rewards)
|
||||
# highest_rewards.append(episode_highest_reward)
|
||||
# print(f'Rollout {rollout_id}\n{episode_return=}, {episode_highest_reward=}, {env_max_reward=}, Success: {episode_highest_reward==env_max_reward}')
|
||||
|
||||
# if save_episode:
|
||||
# save_videos(image_list, DT, video_path=os.path.join(ckpt_dir, f'video{rollout_id}.mp4'))
|
||||
|
||||
# success_rate = np.mean(np.array(highest_rewards) == env_max_reward)
|
||||
# avg_return = np.mean(episode_returns)
|
||||
# summary_str = f'\nSuccess rate: {success_rate}\nAverage return: {avg_return}\n\n'
|
||||
# for r in range(env_max_reward+1):
|
||||
# more_or_equal_r = (np.array(highest_rewards) >= r).sum()
|
||||
# more_or_equal_r_rate = more_or_equal_r / num_rollouts
|
||||
# summary_str += f'Reward >= {r}: {more_or_equal_r}/{num_rollouts} = {more_or_equal_r_rate*100}%\n'
|
||||
|
||||
# print(summary_str)
|
||||
|
||||
# # save success rate to txt
|
||||
# result_file_name = 'result_' + ckpt_name.split('.')[0] + '.txt'
|
||||
# with open(os.path.join(ckpt_dir, result_file_name), 'w') as f:
|
||||
# f.write(summary_str)
|
||||
# f.write(repr(episode_returns))
|
||||
# f.write('\n\n')
|
||||
# f.write(repr(highest_rewards))
|
||||
|
||||
# return success_rate, avg_return
|
||||
|
||||
|
||||
def forward_pass(data, policy, latent_model):
|
||||
image_data, qpos_data, action_data, is_pad = data
|
||||
image_data, qpos_data, action_data, is_pad = image_data.cuda(), qpos_data.cuda(), action_data.cuda(), is_pad.cuda()
|
||||
forward_dict = {}
|
||||
gt_labels = policy.vq_encode(qpos_data, action_data, is_pad)
|
||||
inputs = torch.cat([torch.zeros_like(gt_labels)[:, [0]], gt_labels[:, :-1]], dim=1)
|
||||
output_logits = latent_model(inputs)
|
||||
ce_loss = F.cross_entropy(output_logits, gt_labels)
|
||||
|
||||
with torch.no_grad():
|
||||
output_labels = F.one_hot(torch.argmax(output_logits, dim=-1), num_classes=gt_labels.shape[-1]).float()
|
||||
# output_latents = F.softmax(output_logits, dim=-1)
|
||||
l1_error = F.l1_loss(output_labels, gt_labels, reduction='mean')
|
||||
# l1_errors = []
|
||||
# for i in range(l1_errors.shape[1]):
|
||||
# l1_errors.append(torch.mean(l1_errors[:, i]).item())
|
||||
|
||||
forward_dict['loss'] = ce_loss
|
||||
forward_dict['l1_error'] = l1_error
|
||||
|
||||
return forward_dict
|
||||
|
||||
|
||||
def train_bc(train_dataloader, val_dataloader, config, ckpt_name):
|
||||
num_epochs = config['num_epochs']
|
||||
ckpt_dir = config['ckpt_dir']
|
||||
seed = config['seed']
|
||||
policy_class = config['policy_class']
|
||||
policy_config = config['policy_config']
|
||||
|
||||
set_seed(seed)
|
||||
|
||||
vq_dim = config['policy_config']['vq_dim']
|
||||
vq_class = config['policy_config']['vq_class']
|
||||
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
||||
latent_model.cuda()
|
||||
|
||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||
policy = make_policy(policy_class, policy_config)
|
||||
loading_status = policy.load_state_dict(torch.load(ckpt_path))
|
||||
policy.eval()
|
||||
policy.cuda()
|
||||
|
||||
optimizer = torch.optim.AdamW(latent_model.parameters(), lr=config['lr'])
|
||||
|
||||
train_history = []
|
||||
validation_history = []
|
||||
min_val_loss = np.inf
|
||||
best_ckpt_info = None
|
||||
for epoch in tqdm(range(num_epochs)):
|
||||
print(f'\nEpoch {epoch}')
|
||||
# validation
|
||||
with torch.inference_mode():
|
||||
latent_model.eval()
|
||||
epoch_dicts = []
|
||||
for batch_idx, data in enumerate(val_dataloader):
|
||||
forward_dict = forward_pass(data, policy, latent_model)
|
||||
epoch_dicts.append(forward_dict)
|
||||
epoch_summary = compute_dict_mean(epoch_dicts)
|
||||
validation_history.append(epoch_summary)
|
||||
|
||||
epoch_val_loss = epoch_summary['loss']
|
||||
if epoch_val_loss < min_val_loss:
|
||||
min_val_loss = epoch_val_loss
|
||||
best_ckpt_info = (epoch, min_val_loss, deepcopy(latent_model.state_dict()))
|
||||
print(f'Val loss: {epoch_val_loss:.5f}')
|
||||
summary_string = ''
|
||||
for k, v in epoch_summary.items():
|
||||
summary_string += f'{k}: {v.item():.3f} '
|
||||
print(summary_string)
|
||||
|
||||
# training
|
||||
optimizer.zero_grad()
|
||||
for batch_idx, data in enumerate(train_dataloader):
|
||||
forward_dict = forward_pass(data, policy, latent_model)
|
||||
# backward
|
||||
loss = forward_dict['loss']
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
train_history.append(detach_dict(forward_dict))
|
||||
epoch_summary = compute_dict_mean(train_history[(batch_idx+1)*epoch:(batch_idx+1)*(epoch+1)])
|
||||
epoch_train_loss = epoch_summary['loss']
|
||||
print(f'Train loss: {epoch_train_loss:.5f}')
|
||||
summary_string = ''
|
||||
for k, v in epoch_summary.items():
|
||||
summary_string += f'{k}: {v.item():.3f} '
|
||||
print(summary_string)
|
||||
|
||||
if epoch % 100 == 0:
|
||||
ckpt_path = os.path.join(ckpt_dir, f'latent_model_epoch_{epoch}_seed_{seed}.ckpt')
|
||||
torch.save(latent_model.state_dict(), ckpt_path)
|
||||
plot_history(train_history, validation_history, epoch, ckpt_dir, seed)
|
||||
|
||||
ckpt_path = os.path.join(ckpt_dir, f'latent_model_last.ckpt')
|
||||
torch.save(latent_model.state_dict(), ckpt_path)
|
||||
|
||||
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
|
||||
ckpt_path = os.path.join(ckpt_dir, f'latent_model_epoch_{best_epoch}_seed_{seed}.ckpt')
|
||||
torch.save(best_state_dict, ckpt_path)
|
||||
print(f'Training finished:\nSeed {seed}, val loss {min_val_loss:.6f} at epoch {best_epoch}')
|
||||
|
||||
# save training curves
|
||||
plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed)
|
||||
|
||||
return best_ckpt_info
|
||||
|
||||
|
||||
def plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed):
|
||||
# save training curves
|
||||
for key in train_history[0]:
|
||||
plot_path = os.path.join(ckpt_dir, f'latent_model_val_{key}_seed_{seed}.png')
|
||||
plt.figure()
|
||||
train_values = [summary[key].item() for summary in train_history]
|
||||
val_values = [summary[key].item() for summary in validation_history]
|
||||
plt.plot(np.linspace(0, num_epochs-1, len(train_history)), train_values, label='train')
|
||||
plt.plot(np.linspace(0, num_epochs-1, len(validation_history)), val_values, label='validation')
|
||||
# plt.ylim([-0.1, 1])
|
||||
plt.tight_layout()
|
||||
plt.legend()
|
||||
plt.title(key)
|
||||
plt.savefig(plot_path)
|
||||
print(f'Saved plots to {ckpt_dir}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--eval', action='store_true')
|
||||
parser.add_argument('--onscreen_render', action='store_true')
|
||||
parser.add_argument('--ckpt_dir', action='store', type=str, help='ckpt_dir', required=True)
|
||||
parser.add_argument('--policy_class', action='store', type=str, help='policy_class, capitalize', required=True)
|
||||
parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True)
|
||||
parser.add_argument('--batch_size', action='store', type=int, help='batch_size', required=True)
|
||||
parser.add_argument('--seed', action='store', type=int, help='seed', required=True)
|
||||
parser.add_argument('--num_epochs', action='store', type=int, help='num_epochs', required=True)
|
||||
parser.add_argument('--lr', action='store', type=float, help='lr', required=True)
|
||||
|
||||
# for ACT
|
||||
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
|
||||
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
|
||||
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
|
||||
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
|
||||
parser.add_argument('--temporal_agg', action='store_true')
|
||||
|
||||
main(vars(parser.parse_args()))
|
||||
12
utils.py
12
utils.py
@ -81,10 +81,14 @@ def get_norm_stats(dataset_dir, num_episodes):
|
||||
all_action_data = []
|
||||
for episode_idx in range(num_episodes):
|
||||
dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5')
|
||||
with h5py.File(dataset_path, 'r') as root:
|
||||
qpos = root['/observations/qpos'][()]
|
||||
qvel = root['/observations/qvel'][()]
|
||||
action = root['/action'][()]
|
||||
try:
|
||||
with h5py.File(dataset_path, 'r') as root:
|
||||
qpos = root['/observations/qpos'][()]
|
||||
qvel = root['/observations/qvel'][()]
|
||||
action = root['/action'][()]
|
||||
except:
|
||||
print(f'Error loading {dataset_path}')
|
||||
quit()
|
||||
all_qpos_data.append(torch.from_numpy(qpos))
|
||||
all_action_data.append(torch.from_numpy(action))
|
||||
all_qpos_data = torch.stack(all_qpos_data)
|
||||
|
||||
在新工单中引用
屏蔽一个用户