vq init
这个提交包含在:
4
.gitignore
vendored
4
.gitignore
vendored
@ -137,4 +137,6 @@ celerybeat.pid
|
|||||||
dmypy.json
|
dmypy.json
|
||||||
|
|
||||||
# Pyre type checker
|
# Pyre type checker
|
||||||
.pyre/
|
.pyre/
|
||||||
|
|
||||||
|
ckpts/
|
||||||
12
README.md
12
README.md
@ -57,10 +57,7 @@ To set up a new terminal, run:
|
|||||||
We use ``sim_transfer_cube_scripted`` task in the examples below. Another option is ``sim_insertion_scripted``.
|
We use ``sim_transfer_cube_scripted`` task in the examples below. Another option is ``sim_insertion_scripted``.
|
||||||
To generated 50 episodes of scripted data, run:
|
To generated 50 episodes of scripted data, run:
|
||||||
|
|
||||||
python3 record_sim_episodes.py \
|
python3 record_sim_episodes.py --task_name sim_transfer_cube_scripted --dataset_dir <data save dir> --num_episodes 50
|
||||||
--task_name sim_transfer_cube_scripted \
|
|
||||||
--dataset_dir <data save dir> \
|
|
||||||
--num_episodes 50
|
|
||||||
|
|
||||||
To can add the flag ``--onscreen_render`` to see real-time rendering.
|
To can add the flag ``--onscreen_render`` to see real-time rendering.
|
||||||
To visualize the episode after it is collected, run
|
To visualize the episode after it is collected, run
|
||||||
@ -70,12 +67,7 @@ To visualize the episode after it is collected, run
|
|||||||
To train ACT:
|
To train ACT:
|
||||||
|
|
||||||
# Transfer Cube task
|
# Transfer Cube task
|
||||||
python3 imitate_episodes.py \
|
python3 imitate_episodes.py --task_name sim_transfer_cube_scripted --ckpt_dir <ckpt dir> --policy_class ACT --kl_weight 10 --chunk_size 100 --hidden_dim 512 --batch_size 8 --dim_feedforward 3200 --num_epochs 2000 --lr 1e-5 --seed 0
|
||||||
--task_name sim_transfer_cube_scripted \
|
|
||||||
--ckpt_dir <ckpt dir> \
|
|
||||||
--policy_class ACT --kl_weight 10 --chunk_size 100 --hidden_dim 512 --batch_size 8 --dim_feedforward 3200 \
|
|
||||||
--num_epochs 2000 --lr 1e-5 \
|
|
||||||
--seed 0
|
|
||||||
|
|
||||||
|
|
||||||
To evaluate the policy, run the same command but add ``--eval``. This loads the best validation checkpoint.
|
To evaluate the policy, run the same command but add ``--eval``. This loads the best validation checkpoint.
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
import pathlib
|
import pathlib
|
||||||
|
|
||||||
### Task parameters
|
### Task parameters
|
||||||
DATA_DIR = '<put your data dir here>'
|
DATA_DIR = 'data'
|
||||||
SIM_TASK_CONFIGS = {
|
SIM_TASK_CONFIGS = {
|
||||||
'sim_transfer_cube_scripted':{
|
'sim_transfer_cube_scripted':{
|
||||||
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',
|
'dataset_dir': DATA_DIR + '/sim_transfer_cube_scripted',
|
||||||
|
|||||||
@ -63,6 +63,7 @@ def get_args_parser():
|
|||||||
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
|
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
|
||||||
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
|
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
|
||||||
parser.add_argument('--temporal_agg', action='store_true')
|
parser.add_argument('--temporal_agg', action='store_true')
|
||||||
|
parser.add_argument('--vq', action='store_true')
|
||||||
|
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|||||||
@ -5,6 +5,7 @@ DETR model and criterion classes.
|
|||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from torch.autograd import Variable
|
from torch.autograd import Variable
|
||||||
|
import torch.nn.functional as F
|
||||||
from .backbone import build_backbone
|
from .backbone import build_backbone
|
||||||
from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer
|
from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer
|
||||||
|
|
||||||
@ -33,7 +34,7 @@ def get_sinusoid_encoding_table(n_position, d_hid):
|
|||||||
|
|
||||||
class DETRVAE(nn.Module):
|
class DETRVAE(nn.Module):
|
||||||
""" This is the DETR module that performs object detection """
|
""" This is the DETR module that performs object detection """
|
||||||
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names):
|
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names, vq, vq_class, vq_dim, ):
|
||||||
""" Initializes the model.
|
""" Initializes the model.
|
||||||
Parameters:
|
Parameters:
|
||||||
backbones: torch module of the backbone to be used. See backbone.py
|
backbones: torch module of the backbone to be used. See backbone.py
|
||||||
@ -48,6 +49,7 @@ class DETRVAE(nn.Module):
|
|||||||
self.camera_names = camera_names
|
self.camera_names = camera_names
|
||||||
self.transformer = transformer
|
self.transformer = transformer
|
||||||
self.encoder = encoder
|
self.encoder = encoder
|
||||||
|
self.vq, self.vq_class, self.vq_dim = vq, vq_class, vq_dim
|
||||||
hidden_dim = transformer.d_model
|
hidden_dim = transformer.d_model
|
||||||
self.action_head = nn.Linear(hidden_dim, state_dim)
|
self.action_head = nn.Linear(hidden_dim, state_dim)
|
||||||
self.is_pad_head = nn.Linear(hidden_dim, 1)
|
self.is_pad_head = nn.Linear(hidden_dim, 1)
|
||||||
@ -68,20 +70,24 @@ class DETRVAE(nn.Module):
|
|||||||
self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding
|
self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding
|
||||||
self.encoder_action_proj = nn.Linear(14, hidden_dim) # project action to embedding
|
self.encoder_action_proj = nn.Linear(14, hidden_dim) # project action to embedding
|
||||||
self.encoder_joint_proj = nn.Linear(14, hidden_dim) # project qpos to embedding
|
self.encoder_joint_proj = nn.Linear(14, hidden_dim) # project qpos to embedding
|
||||||
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
|
|
||||||
|
print('Use VQ: ', self.vq)
|
||||||
|
if self.vq:
|
||||||
|
self.latent_proj = nn.Linear(hidden_dim, self.vq_class * self.vq_dim)
|
||||||
|
else:
|
||||||
|
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
|
||||||
self.register_buffer('pos_table', get_sinusoid_encoding_table(1+1+num_queries, hidden_dim)) # [CLS], qpos, a_seq
|
self.register_buffer('pos_table', get_sinusoid_encoding_table(1+1+num_queries, hidden_dim)) # [CLS], qpos, a_seq
|
||||||
|
|
||||||
# decoder extra parameters
|
# decoder extra parameters
|
||||||
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
|
if self.vq:
|
||||||
|
self.latent_out_proj = nn.Linear(self.vq_class * self.vq_dim, hidden_dim)
|
||||||
|
else:
|
||||||
|
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
|
||||||
self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent
|
self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent
|
||||||
|
|
||||||
def forward(self, qpos, image, env_state, actions=None, is_pad=None):
|
|
||||||
"""
|
def encode(self, qpos, actions=None, is_pad=None, vq_sample=None):
|
||||||
qpos: batch, qpos_dim
|
# cvae encoder
|
||||||
image: batch, num_cam, channel, height, width
|
|
||||||
env_state: None
|
|
||||||
actions: batch, seq, action_dim
|
|
||||||
"""
|
|
||||||
is_training = actions is not None # train or val
|
is_training = actions is not None # train or val
|
||||||
bs, _ = qpos.shape
|
bs, _ = qpos.shape
|
||||||
### Obtain latent z from action sequence
|
### Obtain latent z from action sequence
|
||||||
@ -104,15 +110,43 @@ class DETRVAE(nn.Module):
|
|||||||
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
|
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
|
||||||
encoder_output = encoder_output[0] # take cls output only
|
encoder_output = encoder_output[0] # take cls output only
|
||||||
latent_info = self.latent_proj(encoder_output)
|
latent_info = self.latent_proj(encoder_output)
|
||||||
mu = latent_info[:, :self.latent_dim]
|
|
||||||
logvar = latent_info[:, self.latent_dim:]
|
if self.vq:
|
||||||
latent_sample = reparametrize(mu, logvar)
|
logits = latent_info.reshape([*latent_info.shape[:-1], self.vq_class, self.vq_dim])
|
||||||
latent_input = self.latent_out_proj(latent_sample)
|
probs = torch.softmax(logits, dim=-1)
|
||||||
else:
|
binaries = F.one_hot(torch.multinomial(probs.view(-1, self.vq_dim), 1).squeeze(-1), self.vq_dim).view(-1, self.vq_class, self.vq_dim).float()
|
||||||
mu = logvar = None
|
binaries_flat = binaries.view(-1, self.vq_class * self.vq_dim)
|
||||||
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
probs_flat = probs.view(-1, self.vq_class * self.vq_dim)
|
||||||
latent_input = self.latent_out_proj(latent_sample)
|
straigt_through = binaries_flat - probs_flat.detach() + probs_flat
|
||||||
|
latent_input = self.latent_out_proj(straigt_through)
|
||||||
|
mu = logvar = None
|
||||||
|
else:
|
||||||
|
mu = latent_info[:, :self.latent_dim]
|
||||||
|
logvar = latent_info[:, self.latent_dim:]
|
||||||
|
latent_sample = reparametrize(mu, logvar)
|
||||||
|
latent_input = self.latent_out_proj(latent_sample)
|
||||||
|
|
||||||
|
else:
|
||||||
|
mu = logvar = binaries = probs = None
|
||||||
|
if self.vq:
|
||||||
|
latent_input = self.latent_out_proj(vq_sample.view(-1, self.vq_class * self.vq_dim))
|
||||||
|
else:
|
||||||
|
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
||||||
|
latent_input = self.latent_out_proj(latent_sample)
|
||||||
|
|
||||||
|
|
||||||
|
return latent_input, probs, binaries, mu, logvar
|
||||||
|
|
||||||
|
def forward(self, qpos, image, env_state, actions=None, is_pad=None, vq_sample=None):
|
||||||
|
"""
|
||||||
|
qpos: batch, qpos_dim
|
||||||
|
image: batch, num_cam, channel, height, width
|
||||||
|
env_state: None
|
||||||
|
actions: batch, seq, action_dim
|
||||||
|
"""
|
||||||
|
latent_input, probs, binaries, mu, logvar = self.encode(qpos, actions, is_pad, vq_sample)
|
||||||
|
|
||||||
|
# cvae decoder
|
||||||
if self.backbones is not None:
|
if self.backbones is not None:
|
||||||
# Image observation features and position embeddings
|
# Image observation features and position embeddings
|
||||||
all_cam_features = []
|
all_cam_features = []
|
||||||
@ -136,7 +170,7 @@ class DETRVAE(nn.Module):
|
|||||||
hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0]
|
hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0]
|
||||||
a_hat = self.action_head(hs)
|
a_hat = self.action_head(hs)
|
||||||
is_pad_hat = self.is_pad_head(hs)
|
is_pad_hat = self.is_pad_head(hs)
|
||||||
return a_hat, is_pad_hat, [mu, logvar]
|
return a_hat, is_pad_hat, [mu, logvar], probs, binaries
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -247,6 +281,9 @@ def build(args):
|
|||||||
state_dim=state_dim,
|
state_dim=state_dim,
|
||||||
num_queries=args.num_queries,
|
num_queries=args.num_queries,
|
||||||
camera_names=args.camera_names,
|
camera_names=args.camera_names,
|
||||||
|
vq=args.vq,
|
||||||
|
vq_class=args.vq_class,
|
||||||
|
vq_dim=args.vq_dim,
|
||||||
)
|
)
|
||||||
|
|
||||||
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||||
|
|||||||
73
detr/models/latent_model.py
普通文件
73
detr/models/latent_model.py
普通文件
@ -0,0 +1,73 @@
|
|||||||
|
import torch.nn as nn
|
||||||
|
from torch.nn import functional as F
|
||||||
|
import torch
|
||||||
|
|
||||||
|
DROPOUT_RATE = 0.1
|
||||||
|
|
||||||
|
# a causal transformer block
|
||||||
|
class Causal_Transformer_Block(nn.Module):
|
||||||
|
def __init__(self, seq_len, latent_dim, num_head) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.num_head = num_head
|
||||||
|
self.latent_dim = latent_dim
|
||||||
|
self.ln_1 = nn.LayerNorm(latent_dim)
|
||||||
|
self.attn = nn.MultiheadAttention(latent_dim, num_head, dropout=DROPOUT_RATE, batch_first=True)
|
||||||
|
self.ln_2 = nn.LayerNorm(latent_dim)
|
||||||
|
self.mlp = nn.Sequential(
|
||||||
|
nn.Linear(latent_dim, 4 * latent_dim),
|
||||||
|
nn.GELU(),
|
||||||
|
nn.Linear(4 * latent_dim, latent_dim),
|
||||||
|
nn.Dropout(DROPOUT_RATE),
|
||||||
|
)
|
||||||
|
|
||||||
|
# self.register_buffer("attn_mask", torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool())
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
attn_mask = torch.triu(torch.ones(x.shape[1], x.shape[1], device=x.device, dtype=torch.bool), diagonal=1)
|
||||||
|
x = self.ln_1(x)
|
||||||
|
x = x + self.attn(x, x, x, attn_mask=attn_mask)[0]
|
||||||
|
x = self.ln_2(x)
|
||||||
|
x = x + self.mlp(x)
|
||||||
|
|
||||||
|
return x
|
||||||
|
|
||||||
|
# use self-attention instead of RNN to model the latent space sequence
|
||||||
|
class Latent_Model_Transformer(nn.Module):
|
||||||
|
def __init__(self, input_dim, output_dim, seq_len, latent_dim=256, num_head=8, num_layer=3) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.input_dim = input_dim
|
||||||
|
self.output_dim = output_dim
|
||||||
|
self.seq_len = seq_len
|
||||||
|
self.latent_dim = latent_dim
|
||||||
|
self.num_head = num_head
|
||||||
|
self.num_layer = num_layer
|
||||||
|
self.input_layer = nn.Linear(input_dim, latent_dim)
|
||||||
|
self.weight_pos_embed = nn.Embedding(seq_len, latent_dim)
|
||||||
|
self.attention_blocks = nn.Sequential(
|
||||||
|
nn.Dropout(DROPOUT_RATE),
|
||||||
|
*[Causal_Transformer_Block(seq_len, latent_dim, num_head) for _ in range(num_layer)],
|
||||||
|
nn.LayerNorm(latent_dim)
|
||||||
|
)
|
||||||
|
self.output_layer = nn.Linear(latent_dim, output_dim)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
x = self.input_layer(x)
|
||||||
|
x = x + self.weight_pos_embed(torch.arange(x.shape[1], device=x.device))
|
||||||
|
x = self.attention_blocks(x)
|
||||||
|
logits = self.output_layer(x)
|
||||||
|
|
||||||
|
return logits
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
def generate(self, n, temperature=0.1, x=None):
|
||||||
|
if x is None:
|
||||||
|
x = torch.zeros((n, 1, self.input_dim), device=self.weight_pos_embed.weight.device)
|
||||||
|
for i in range(self.seq_len):
|
||||||
|
logits = self.forward(x)[:, -1]
|
||||||
|
probs = torch.softmax(logits / temperature, dim=-1)
|
||||||
|
samples = torch.multinomial(probs, num_samples=1)[..., 0]
|
||||||
|
samples_one_hot = F.one_hot(samples.long(), num_classes=self.output_dim).float()
|
||||||
|
x = torch.cat([x, samples_one_hot[:, None, :]], dim=1)
|
||||||
|
|
||||||
|
return x[:, 1:, :]
|
||||||
|
|
||||||
@ -16,6 +16,8 @@ from utils import compute_dict_mean, set_seed, detach_dict # helper functions
|
|||||||
from policy import ACTPolicy, CNNMLPPolicy
|
from policy import ACTPolicy, CNNMLPPolicy
|
||||||
from visualize_episodes import save_videos
|
from visualize_episodes import save_videos
|
||||||
|
|
||||||
|
from detr.models.latent_model import Latent_Model_Transformer
|
||||||
|
|
||||||
from sim_env import BOX_POSE
|
from sim_env import BOX_POSE
|
||||||
|
|
||||||
import IPython
|
import IPython
|
||||||
@ -54,6 +56,8 @@ def main(args):
|
|||||||
enc_layers = 4
|
enc_layers = 4
|
||||||
dec_layers = 7
|
dec_layers = 7
|
||||||
nheads = 8
|
nheads = 8
|
||||||
|
vq_class = 10
|
||||||
|
vq_dim = 10
|
||||||
policy_config = {'lr': args['lr'],
|
policy_config = {'lr': args['lr'],
|
||||||
'num_queries': args['chunk_size'],
|
'num_queries': args['chunk_size'],
|
||||||
'kl_weight': args['kl_weight'],
|
'kl_weight': args['kl_weight'],
|
||||||
@ -65,6 +69,9 @@ def main(args):
|
|||||||
'dec_layers': dec_layers,
|
'dec_layers': dec_layers,
|
||||||
'nheads': nheads,
|
'nheads': nheads,
|
||||||
'camera_names': camera_names,
|
'camera_names': camera_names,
|
||||||
|
'vq': args['vq'],
|
||||||
|
'vq_class': vq_class,
|
||||||
|
'vq_dim': vq_dim,
|
||||||
}
|
}
|
||||||
elif policy_class == 'CNNMLP':
|
elif policy_class == 'CNNMLP':
|
||||||
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
||||||
@ -161,6 +168,7 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
|||||||
task_name = config['task_name']
|
task_name = config['task_name']
|
||||||
temporal_agg = config['temporal_agg']
|
temporal_agg = config['temporal_agg']
|
||||||
onscreen_cam = 'angle'
|
onscreen_cam = 'angle'
|
||||||
|
vq = config['policy_config']['vq']
|
||||||
|
|
||||||
# load policy and stats
|
# load policy and stats
|
||||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
@ -169,7 +177,16 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
|||||||
print(loading_status)
|
print(loading_status)
|
||||||
policy.cuda()
|
policy.cuda()
|
||||||
policy.eval()
|
policy.eval()
|
||||||
print(f'Loaded: {ckpt_path}')
|
if vq:
|
||||||
|
vq_dim = config['policy_config']['vq_dim']
|
||||||
|
vq_class = config['policy_config']['vq_class']
|
||||||
|
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
||||||
|
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_best.ckpt')
|
||||||
|
latent_model.load_state_dict(torch.load(latent_model_ckpt_path))
|
||||||
|
latent_model.cuda()
|
||||||
|
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
|
||||||
|
else:
|
||||||
|
print(f'Loaded: {ckpt_path}')
|
||||||
stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
||||||
with open(stats_path, 'rb') as f:
|
with open(stats_path, 'rb') as f:
|
||||||
stats = pickle.load(f)
|
stats = pickle.load(f)
|
||||||
@ -246,7 +263,11 @@ def eval_bc(config, ckpt_name, save_episode=True):
|
|||||||
### query policy
|
### query policy
|
||||||
if config['policy_class'] == "ACT":
|
if config['policy_class'] == "ACT":
|
||||||
if t % query_frequency == 0:
|
if t % query_frequency == 0:
|
||||||
all_actions = policy(qpos, curr_image)
|
if vq:
|
||||||
|
vq_sample = latent_model.generate(1, temperature=0.01, x=None)
|
||||||
|
all_actions = policy(qpos, curr_image, vq_sample=vq_sample)
|
||||||
|
else:
|
||||||
|
all_actions = policy(qpos, curr_image)
|
||||||
if temporal_agg:
|
if temporal_agg:
|
||||||
all_time_actions[[t], t:t+num_queries] = all_actions
|
all_time_actions[[t], t:t+num_queries] = all_actions
|
||||||
actions_for_curr_step = all_time_actions[:, t]
|
actions_for_curr_step = all_time_actions[:, t]
|
||||||
@ -431,5 +452,6 @@ if __name__ == '__main__':
|
|||||||
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
|
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
|
||||||
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
|
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
|
||||||
parser.add_argument('--temporal_agg', action='store_true')
|
parser.add_argument('--temporal_agg', action='store_true')
|
||||||
|
parser.add_argument('--vq', action='store_true')
|
||||||
|
|
||||||
main(vars(parser.parse_args()))
|
main(vars(parser.parse_args()))
|
||||||
|
|||||||
26
policy.py
26
policy.py
@ -1,6 +1,7 @@
|
|||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from torch.nn import functional as F
|
from torch.nn import functional as F
|
||||||
import torchvision.transforms as transforms
|
import torchvision.transforms as transforms
|
||||||
|
import torch
|
||||||
|
|
||||||
from detr.main import build_ACT_model_and_optimizer, build_CNNMLP_model_and_optimizer
|
from detr.main import build_ACT_model_and_optimizer, build_CNNMLP_model_and_optimizer
|
||||||
import IPython
|
import IPython
|
||||||
@ -13,9 +14,10 @@ class ACTPolicy(nn.Module):
|
|||||||
self.model = model # CVAE decoder
|
self.model = model # CVAE decoder
|
||||||
self.optimizer = optimizer
|
self.optimizer = optimizer
|
||||||
self.kl_weight = args_override['kl_weight']
|
self.kl_weight = args_override['kl_weight']
|
||||||
|
self.vq = args_override['vq']
|
||||||
print(f'KL Weight {self.kl_weight}')
|
print(f'KL Weight {self.kl_weight}')
|
||||||
|
|
||||||
def __call__(self, qpos, image, actions=None, is_pad=None):
|
def __call__(self, qpos, image, actions=None, is_pad=None, vq_sample=None):
|
||||||
env_state = None
|
env_state = None
|
||||||
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
|
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
|
||||||
std=[0.229, 0.224, 0.225])
|
std=[0.229, 0.224, 0.225])
|
||||||
@ -24,9 +26,14 @@ class ACTPolicy(nn.Module):
|
|||||||
actions = actions[:, :self.model.num_queries]
|
actions = actions[:, :self.model.num_queries]
|
||||||
is_pad = is_pad[:, :self.model.num_queries]
|
is_pad = is_pad[:, :self.model.num_queries]
|
||||||
|
|
||||||
a_hat, is_pad_hat, (mu, logvar) = self.model(qpos, image, env_state, actions, is_pad)
|
|
||||||
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
|
|
||||||
loss_dict = dict()
|
loss_dict = dict()
|
||||||
|
print(vq_sample)
|
||||||
|
a_hat, is_pad_hat, (mu, logvar), probs, binaries = self.model(qpos, image, env_state, actions, is_pad, vq_sample)
|
||||||
|
if self.vq:
|
||||||
|
total_kld = [torch.tensor(0.0)]
|
||||||
|
loss_dict['vq_loss'] = F.l1_loss(binaries, probs, reduction='mean')
|
||||||
|
else:
|
||||||
|
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
|
||||||
all_l1 = F.l1_loss(actions, a_hat, reduction='none')
|
all_l1 = F.l1_loss(actions, a_hat, reduction='none')
|
||||||
l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean()
|
l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean()
|
||||||
loss_dict['l1'] = l1
|
loss_dict['l1'] = l1
|
||||||
@ -34,12 +41,23 @@ class ACTPolicy(nn.Module):
|
|||||||
loss_dict['loss'] = loss_dict['l1'] + loss_dict['kl'] * self.kl_weight
|
loss_dict['loss'] = loss_dict['l1'] + loss_dict['kl'] * self.kl_weight
|
||||||
return loss_dict
|
return loss_dict
|
||||||
else: # inference time
|
else: # inference time
|
||||||
a_hat, _, (_, _) = self.model(qpos, image, env_state) # no action, sample from prior
|
a_hat, _, (_, _), _, _ = self.model(qpos, image, env_state, vq_sample=vq_sample) # no action, sample from prior
|
||||||
return a_hat
|
return a_hat
|
||||||
|
|
||||||
def configure_optimizers(self):
|
def configure_optimizers(self):
|
||||||
return self.optimizer
|
return self.optimizer
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
def vq_encode(self, qpos, actions, is_pad):
|
||||||
|
actions = actions[:, :self.model.num_queries]
|
||||||
|
is_pad = is_pad[:, :self.model.num_queries]
|
||||||
|
|
||||||
|
_, _, binaries, _, _ = self.model.encode(qpos, actions, is_pad)
|
||||||
|
|
||||||
|
return binaries
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class CNNMLPPolicy(nn.Module):
|
class CNNMLPPolicy(nn.Module):
|
||||||
def __init__(self, args_override):
|
def __init__(self, args_override):
|
||||||
|
|||||||
468
train_latent_model.py
普通文件
468
train_latent_model.py
普通文件
@ -0,0 +1,468 @@
|
|||||||
|
import torch
|
||||||
|
import numpy as np
|
||||||
|
import os
|
||||||
|
import pickle
|
||||||
|
import argparse
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from copy import deepcopy
|
||||||
|
from tqdm import tqdm
|
||||||
|
from einops import rearrange
|
||||||
|
import torch.nn.functional as F
|
||||||
|
|
||||||
|
from constants import DT
|
||||||
|
from constants import PUPPET_GRIPPER_JOINT_OPEN
|
||||||
|
from utils import load_data # data functions
|
||||||
|
from utils import sample_box_pose, sample_insertion_pose # robot functions
|
||||||
|
from utils import compute_dict_mean, set_seed, detach_dict # helper functions
|
||||||
|
from policy import ACTPolicy, CNNMLPPolicy
|
||||||
|
from visualize_episodes import save_videos
|
||||||
|
from detr.models.latent_model import Latent_Model_Transformer
|
||||||
|
|
||||||
|
from sim_env import BOX_POSE
|
||||||
|
|
||||||
|
import IPython
|
||||||
|
e = IPython.embed
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
set_seed(1)
|
||||||
|
# command line parameters
|
||||||
|
is_eval = args['eval']
|
||||||
|
ckpt_dir = args['ckpt_dir']
|
||||||
|
policy_class = args['policy_class']
|
||||||
|
onscreen_render = args['onscreen_render']
|
||||||
|
task_name = args['task_name']
|
||||||
|
batch_size_train = args['batch_size']
|
||||||
|
batch_size_val = args['batch_size']
|
||||||
|
num_epochs = args['num_epochs']
|
||||||
|
|
||||||
|
# get task parameters
|
||||||
|
is_sim = task_name[:4] == 'sim_'
|
||||||
|
if is_sim:
|
||||||
|
from constants import SIM_TASK_CONFIGS
|
||||||
|
task_config = SIM_TASK_CONFIGS[task_name]
|
||||||
|
else:
|
||||||
|
from aloha_scripts.constants import TASK_CONFIGS
|
||||||
|
task_config = TASK_CONFIGS[task_name]
|
||||||
|
dataset_dir = task_config['dataset_dir']
|
||||||
|
num_episodes = task_config['num_episodes']
|
||||||
|
episode_len = task_config['episode_len']
|
||||||
|
camera_names = task_config['camera_names']
|
||||||
|
|
||||||
|
# fixed parameters
|
||||||
|
state_dim = 14
|
||||||
|
lr_backbone = 1e-5
|
||||||
|
backbone = 'resnet18'
|
||||||
|
if policy_class == 'ACT':
|
||||||
|
enc_layers = 4
|
||||||
|
dec_layers = 7
|
||||||
|
nheads = 8
|
||||||
|
vq_class = 10
|
||||||
|
vq_dim = 10
|
||||||
|
policy_config = {'lr': args['lr'],
|
||||||
|
'num_queries': args['chunk_size'],
|
||||||
|
'kl_weight': args['kl_weight'],
|
||||||
|
'hidden_dim': args['hidden_dim'],
|
||||||
|
'dim_feedforward': args['dim_feedforward'],
|
||||||
|
'lr_backbone': lr_backbone,
|
||||||
|
'backbone': backbone,
|
||||||
|
'enc_layers': enc_layers,
|
||||||
|
'dec_layers': dec_layers,
|
||||||
|
'nheads': nheads,
|
||||||
|
'camera_names': camera_names,
|
||||||
|
'vq': True,
|
||||||
|
'vq_class': vq_class,
|
||||||
|
'vq_dim': vq_dim,
|
||||||
|
}
|
||||||
|
elif policy_class == 'CNNMLP':
|
||||||
|
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
||||||
|
'camera_names': camera_names,}
|
||||||
|
else:
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
config = {
|
||||||
|
'num_epochs': num_epochs,
|
||||||
|
'ckpt_dir': ckpt_dir,
|
||||||
|
'episode_len': episode_len,
|
||||||
|
'state_dim': state_dim,
|
||||||
|
'lr': args['lr'],
|
||||||
|
'policy_class': policy_class,
|
||||||
|
'onscreen_render': onscreen_render,
|
||||||
|
'policy_config': policy_config,
|
||||||
|
'task_name': task_name,
|
||||||
|
'seed': args['seed'],
|
||||||
|
'temporal_agg': args['temporal_agg'],
|
||||||
|
'camera_names': camera_names,
|
||||||
|
'real_robot': not is_sim
|
||||||
|
}
|
||||||
|
|
||||||
|
# if is_eval:
|
||||||
|
# ckpt_names = [f'policy_best.ckpt']
|
||||||
|
# results = []
|
||||||
|
# for ckpt_name in ckpt_names:
|
||||||
|
# success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True)
|
||||||
|
# results.append([ckpt_name, success_rate, avg_return])
|
||||||
|
|
||||||
|
# for ckpt_name, success_rate, avg_return in results:
|
||||||
|
# print(f'{ckpt_name}: {success_rate=} {avg_return=}')
|
||||||
|
# print()
|
||||||
|
# exit()
|
||||||
|
|
||||||
|
train_dataloader, val_dataloader, stats, _ = load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val)
|
||||||
|
|
||||||
|
# save dataset stats
|
||||||
|
# if not os.path.isdir(ckpt_dir):
|
||||||
|
# os.makedirs(ckpt_dir)
|
||||||
|
# stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
||||||
|
# with open(stats_path, 'wb') as f:
|
||||||
|
# pickle.dump(stats, f)
|
||||||
|
|
||||||
|
ckpt_name = f'policy_best.ckpt'
|
||||||
|
best_ckpt_info = train_bc(train_dataloader, val_dataloader, config, ckpt_name)
|
||||||
|
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
|
||||||
|
|
||||||
|
# save best checkpoint
|
||||||
|
ckpt_path = os.path.join(ckpt_dir, f'latent_model_best.ckpt')
|
||||||
|
torch.save(best_state_dict, ckpt_path)
|
||||||
|
print(f'Best ckpt, val loss {min_val_loss:.6f} @ epoch{best_epoch}')
|
||||||
|
|
||||||
|
|
||||||
|
def make_policy(policy_class, policy_config):
|
||||||
|
if policy_class == 'ACT':
|
||||||
|
policy = ACTPolicy(policy_config)
|
||||||
|
elif policy_class == 'CNNMLP':
|
||||||
|
policy = CNNMLPPolicy(policy_config)
|
||||||
|
else:
|
||||||
|
raise NotImplementedError
|
||||||
|
return policy
|
||||||
|
|
||||||
|
|
||||||
|
# def make_optimizer(policy_class, policy):
|
||||||
|
# if policy_class == 'ACT':
|
||||||
|
# optimizer = policy.configure_optimizers()
|
||||||
|
# elif policy_class == 'CNNMLP':
|
||||||
|
# optimizer = policy.configure_optimizers()
|
||||||
|
# else:
|
||||||
|
# raise NotImplementedError
|
||||||
|
# return optimizer
|
||||||
|
|
||||||
|
|
||||||
|
def get_image(ts, camera_names):
|
||||||
|
curr_images = []
|
||||||
|
for cam_name in camera_names:
|
||||||
|
curr_image = rearrange(ts.observation['images'][cam_name], 'h w c -> c h w')
|
||||||
|
curr_images.append(curr_image)
|
||||||
|
curr_image = np.stack(curr_images, axis=0)
|
||||||
|
curr_image = torch.from_numpy(curr_image / 255.0).float().cuda().unsqueeze(0)
|
||||||
|
return curr_image
|
||||||
|
|
||||||
|
|
||||||
|
# def eval_bc(config, ckpt_name, save_episode=True):
|
||||||
|
# set_seed(1000)
|
||||||
|
# ckpt_dir = config['ckpt_dir']
|
||||||
|
# state_dim = config['state_dim']
|
||||||
|
# real_robot = config['real_robot']
|
||||||
|
# policy_class = config['policy_class']
|
||||||
|
# onscreen_render = config['onscreen_render']
|
||||||
|
# policy_config = config['policy_config']
|
||||||
|
# camera_names = config['camera_names']
|
||||||
|
# max_timesteps = config['episode_len']
|
||||||
|
# task_name = config['task_name']
|
||||||
|
# temporal_agg = config['temporal_agg']
|
||||||
|
# onscreen_cam = 'angle'
|
||||||
|
|
||||||
|
# # load policy and stats
|
||||||
|
# ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
|
# policy = make_policy(policy_class, policy_config)
|
||||||
|
# loading_status = policy.load_state_dict(torch.load(ckpt_path))
|
||||||
|
# print(loading_status)
|
||||||
|
# policy.cuda()
|
||||||
|
# policy.eval()
|
||||||
|
# print(f'Loaded: {ckpt_path}')
|
||||||
|
# stats_path = os.path.join(ckpt_dir, f'dataset_stats.pkl')
|
||||||
|
# with open(stats_path, 'rb') as f:
|
||||||
|
# stats = pickle.load(f)
|
||||||
|
|
||||||
|
# pre_process = lambda s_qpos: (s_qpos - stats['qpos_mean']) / stats['qpos_std']
|
||||||
|
# post_process = lambda a: a * stats['action_std'] + stats['action_mean']
|
||||||
|
|
||||||
|
# # load environment
|
||||||
|
# if real_robot:
|
||||||
|
# from aloha_scripts.robot_utils import move_grippers # requires aloha
|
||||||
|
# from aloha_scripts.real_env import make_real_env # requires aloha
|
||||||
|
# env = make_real_env(init_node=True)
|
||||||
|
# env_max_reward = 0
|
||||||
|
# else:
|
||||||
|
# from sim_env import make_sim_env
|
||||||
|
# env = make_sim_env(task_name)
|
||||||
|
# env_max_reward = env.task.max_reward
|
||||||
|
|
||||||
|
# query_frequency = policy_config['num_queries']
|
||||||
|
# if temporal_agg:
|
||||||
|
# query_frequency = 1
|
||||||
|
# num_queries = policy_config['num_queries']
|
||||||
|
|
||||||
|
# max_timesteps = int(max_timesteps * 1) # may increase for real-world tasks
|
||||||
|
|
||||||
|
# num_rollouts = 50
|
||||||
|
# episode_returns = []
|
||||||
|
# highest_rewards = []
|
||||||
|
# for rollout_id in range(num_rollouts):
|
||||||
|
# rollout_id += 0
|
||||||
|
# ### set task
|
||||||
|
# if 'sim_transfer_cube' in task_name:
|
||||||
|
# BOX_POSE[0] = sample_box_pose() # used in sim reset
|
||||||
|
# elif 'sim_insertion' in task_name:
|
||||||
|
# BOX_POSE[0] = np.concatenate(sample_insertion_pose()) # used in sim reset
|
||||||
|
|
||||||
|
# ts = env.reset()
|
||||||
|
|
||||||
|
# ### onscreen render
|
||||||
|
# if onscreen_render:
|
||||||
|
# ax = plt.subplot()
|
||||||
|
# plt_img = ax.imshow(env._physics.render(height=480, width=640, camera_id=onscreen_cam))
|
||||||
|
# plt.ion()
|
||||||
|
|
||||||
|
# ### evaluation loop
|
||||||
|
# if temporal_agg:
|
||||||
|
# all_time_actions = torch.zeros([max_timesteps, max_timesteps+num_queries, state_dim]).cuda()
|
||||||
|
|
||||||
|
# qpos_history = torch.zeros((1, max_timesteps, state_dim)).cuda()
|
||||||
|
# image_list = [] # for visualization
|
||||||
|
# qpos_list = []
|
||||||
|
# target_qpos_list = []
|
||||||
|
# rewards = []
|
||||||
|
# with torch.inference_mode():
|
||||||
|
# for t in range(max_timesteps):
|
||||||
|
# ### update onscreen render and wait for DT
|
||||||
|
# if onscreen_render:
|
||||||
|
# image = env._physics.render(height=480, width=640, camera_id=onscreen_cam)
|
||||||
|
# plt_img.set_data(image)
|
||||||
|
# plt.pause(DT)
|
||||||
|
|
||||||
|
# ### process previous timestep to get qpos and image_list
|
||||||
|
# obs = ts.observation
|
||||||
|
# if 'images' in obs:
|
||||||
|
# image_list.append(obs['images'])
|
||||||
|
# else:
|
||||||
|
# image_list.append({'main': obs['image']})
|
||||||
|
# qpos_numpy = np.array(obs['qpos'])
|
||||||
|
# qpos = pre_process(qpos_numpy)
|
||||||
|
# qpos = torch.from_numpy(qpos).float().cuda().unsqueeze(0)
|
||||||
|
# qpos_history[:, t] = qpos
|
||||||
|
# curr_image = get_image(ts, camera_names)
|
||||||
|
|
||||||
|
# ### query policy
|
||||||
|
# if config['policy_class'] == "ACT":
|
||||||
|
# if t % query_frequency == 0:
|
||||||
|
# all_actions = policy(qpos, curr_image)
|
||||||
|
# if temporal_agg:
|
||||||
|
# all_time_actions[[t], t:t+num_queries] = all_actions
|
||||||
|
# actions_for_curr_step = all_time_actions[:, t]
|
||||||
|
# actions_populated = torch.all(actions_for_curr_step != 0, axis=1)
|
||||||
|
# actions_for_curr_step = actions_for_curr_step[actions_populated]
|
||||||
|
# k = 0.01
|
||||||
|
# exp_weights = np.exp(-k * np.arange(len(actions_for_curr_step)))
|
||||||
|
# exp_weights = exp_weights / exp_weights.sum()
|
||||||
|
# exp_weights = torch.from_numpy(exp_weights).cuda().unsqueeze(dim=1)
|
||||||
|
# raw_action = (actions_for_curr_step * exp_weights).sum(dim=0, keepdim=True)
|
||||||
|
# else:
|
||||||
|
# raw_action = all_actions[:, t % query_frequency]
|
||||||
|
# elif config['policy_class'] == "CNNMLP":
|
||||||
|
# raw_action = policy(qpos, curr_image)
|
||||||
|
# else:
|
||||||
|
# raise NotImplementedError
|
||||||
|
|
||||||
|
# ### post-process actions
|
||||||
|
# raw_action = raw_action.squeeze(0).cpu().numpy()
|
||||||
|
# action = post_process(raw_action)
|
||||||
|
# target_qpos = action
|
||||||
|
|
||||||
|
# ### step the environment
|
||||||
|
# ts = env.step(target_qpos)
|
||||||
|
|
||||||
|
# ### for visualization
|
||||||
|
# qpos_list.append(qpos_numpy)
|
||||||
|
# target_qpos_list.append(target_qpos)
|
||||||
|
# rewards.append(ts.reward)
|
||||||
|
|
||||||
|
# plt.close()
|
||||||
|
# if real_robot:
|
||||||
|
# move_grippers([env.puppet_bot_left, env.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) # open
|
||||||
|
# pass
|
||||||
|
|
||||||
|
# rewards = np.array(rewards)
|
||||||
|
# episode_return = np.sum(rewards[rewards!=None])
|
||||||
|
# episode_returns.append(episode_return)
|
||||||
|
# episode_highest_reward = np.max(rewards)
|
||||||
|
# highest_rewards.append(episode_highest_reward)
|
||||||
|
# print(f'Rollout {rollout_id}\n{episode_return=}, {episode_highest_reward=}, {env_max_reward=}, Success: {episode_highest_reward==env_max_reward}')
|
||||||
|
|
||||||
|
# if save_episode:
|
||||||
|
# save_videos(image_list, DT, video_path=os.path.join(ckpt_dir, f'video{rollout_id}.mp4'))
|
||||||
|
|
||||||
|
# success_rate = np.mean(np.array(highest_rewards) == env_max_reward)
|
||||||
|
# avg_return = np.mean(episode_returns)
|
||||||
|
# summary_str = f'\nSuccess rate: {success_rate}\nAverage return: {avg_return}\n\n'
|
||||||
|
# for r in range(env_max_reward+1):
|
||||||
|
# more_or_equal_r = (np.array(highest_rewards) >= r).sum()
|
||||||
|
# more_or_equal_r_rate = more_or_equal_r / num_rollouts
|
||||||
|
# summary_str += f'Reward >= {r}: {more_or_equal_r}/{num_rollouts} = {more_or_equal_r_rate*100}%\n'
|
||||||
|
|
||||||
|
# print(summary_str)
|
||||||
|
|
||||||
|
# # save success rate to txt
|
||||||
|
# result_file_name = 'result_' + ckpt_name.split('.')[0] + '.txt'
|
||||||
|
# with open(os.path.join(ckpt_dir, result_file_name), 'w') as f:
|
||||||
|
# f.write(summary_str)
|
||||||
|
# f.write(repr(episode_returns))
|
||||||
|
# f.write('\n\n')
|
||||||
|
# f.write(repr(highest_rewards))
|
||||||
|
|
||||||
|
# return success_rate, avg_return
|
||||||
|
|
||||||
|
|
||||||
|
def forward_pass(data, policy, latent_model):
|
||||||
|
image_data, qpos_data, action_data, is_pad = data
|
||||||
|
image_data, qpos_data, action_data, is_pad = image_data.cuda(), qpos_data.cuda(), action_data.cuda(), is_pad.cuda()
|
||||||
|
forward_dict = {}
|
||||||
|
gt_labels = policy.vq_encode(qpos_data, action_data, is_pad)
|
||||||
|
inputs = torch.cat([torch.zeros_like(gt_labels)[:, [0]], gt_labels[:, :-1]], dim=1)
|
||||||
|
output_logits = latent_model(inputs)
|
||||||
|
ce_loss = F.cross_entropy(output_logits, gt_labels)
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
output_labels = F.one_hot(torch.argmax(output_logits, dim=-1), num_classes=gt_labels.shape[-1]).float()
|
||||||
|
# output_latents = F.softmax(output_logits, dim=-1)
|
||||||
|
l1_error = F.l1_loss(output_labels, gt_labels, reduction='mean')
|
||||||
|
# l1_errors = []
|
||||||
|
# for i in range(l1_errors.shape[1]):
|
||||||
|
# l1_errors.append(torch.mean(l1_errors[:, i]).item())
|
||||||
|
|
||||||
|
forward_dict['loss'] = ce_loss
|
||||||
|
forward_dict['l1_error'] = l1_error
|
||||||
|
|
||||||
|
return forward_dict
|
||||||
|
|
||||||
|
|
||||||
|
def train_bc(train_dataloader, val_dataloader, config, ckpt_name):
|
||||||
|
num_epochs = config['num_epochs']
|
||||||
|
ckpt_dir = config['ckpt_dir']
|
||||||
|
seed = config['seed']
|
||||||
|
policy_class = config['policy_class']
|
||||||
|
policy_config = config['policy_config']
|
||||||
|
|
||||||
|
set_seed(seed)
|
||||||
|
|
||||||
|
vq_dim = config['policy_config']['vq_dim']
|
||||||
|
vq_class = config['policy_config']['vq_class']
|
||||||
|
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
||||||
|
latent_model.cuda()
|
||||||
|
|
||||||
|
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
|
policy = make_policy(policy_class, policy_config)
|
||||||
|
loading_status = policy.load_state_dict(torch.load(ckpt_path))
|
||||||
|
policy.eval()
|
||||||
|
policy.cuda()
|
||||||
|
|
||||||
|
optimizer = torch.optim.AdamW(latent_model.parameters(), lr=config['lr'])
|
||||||
|
|
||||||
|
train_history = []
|
||||||
|
validation_history = []
|
||||||
|
min_val_loss = np.inf
|
||||||
|
best_ckpt_info = None
|
||||||
|
for epoch in tqdm(range(num_epochs)):
|
||||||
|
print(f'\nEpoch {epoch}')
|
||||||
|
# validation
|
||||||
|
with torch.inference_mode():
|
||||||
|
latent_model.eval()
|
||||||
|
epoch_dicts = []
|
||||||
|
for batch_idx, data in enumerate(val_dataloader):
|
||||||
|
forward_dict = forward_pass(data, policy, latent_model)
|
||||||
|
epoch_dicts.append(forward_dict)
|
||||||
|
epoch_summary = compute_dict_mean(epoch_dicts)
|
||||||
|
validation_history.append(epoch_summary)
|
||||||
|
|
||||||
|
epoch_val_loss = epoch_summary['loss']
|
||||||
|
if epoch_val_loss < min_val_loss:
|
||||||
|
min_val_loss = epoch_val_loss
|
||||||
|
best_ckpt_info = (epoch, min_val_loss, deepcopy(latent_model.state_dict()))
|
||||||
|
print(f'Val loss: {epoch_val_loss:.5f}')
|
||||||
|
summary_string = ''
|
||||||
|
for k, v in epoch_summary.items():
|
||||||
|
summary_string += f'{k}: {v.item():.3f} '
|
||||||
|
print(summary_string)
|
||||||
|
|
||||||
|
# training
|
||||||
|
optimizer.zero_grad()
|
||||||
|
for batch_idx, data in enumerate(train_dataloader):
|
||||||
|
forward_dict = forward_pass(data, policy, latent_model)
|
||||||
|
# backward
|
||||||
|
loss = forward_dict['loss']
|
||||||
|
loss.backward()
|
||||||
|
optimizer.step()
|
||||||
|
optimizer.zero_grad()
|
||||||
|
train_history.append(detach_dict(forward_dict))
|
||||||
|
epoch_summary = compute_dict_mean(train_history[(batch_idx+1)*epoch:(batch_idx+1)*(epoch+1)])
|
||||||
|
epoch_train_loss = epoch_summary['loss']
|
||||||
|
print(f'Train loss: {epoch_train_loss:.5f}')
|
||||||
|
summary_string = ''
|
||||||
|
for k, v in epoch_summary.items():
|
||||||
|
summary_string += f'{k}: {v.item():.3f} '
|
||||||
|
print(summary_string)
|
||||||
|
|
||||||
|
if epoch % 100 == 0:
|
||||||
|
ckpt_path = os.path.join(ckpt_dir, f'latent_model_epoch_{epoch}_seed_{seed}.ckpt')
|
||||||
|
torch.save(latent_model.state_dict(), ckpt_path)
|
||||||
|
plot_history(train_history, validation_history, epoch, ckpt_dir, seed)
|
||||||
|
|
||||||
|
ckpt_path = os.path.join(ckpt_dir, f'latent_model_last.ckpt')
|
||||||
|
torch.save(latent_model.state_dict(), ckpt_path)
|
||||||
|
|
||||||
|
best_epoch, min_val_loss, best_state_dict = best_ckpt_info
|
||||||
|
ckpt_path = os.path.join(ckpt_dir, f'latent_model_epoch_{best_epoch}_seed_{seed}.ckpt')
|
||||||
|
torch.save(best_state_dict, ckpt_path)
|
||||||
|
print(f'Training finished:\nSeed {seed}, val loss {min_val_loss:.6f} at epoch {best_epoch}')
|
||||||
|
|
||||||
|
# save training curves
|
||||||
|
plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed)
|
||||||
|
|
||||||
|
return best_ckpt_info
|
||||||
|
|
||||||
|
|
||||||
|
def plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed):
|
||||||
|
# save training curves
|
||||||
|
for key in train_history[0]:
|
||||||
|
plot_path = os.path.join(ckpt_dir, f'latent_model_val_{key}_seed_{seed}.png')
|
||||||
|
plt.figure()
|
||||||
|
train_values = [summary[key].item() for summary in train_history]
|
||||||
|
val_values = [summary[key].item() for summary in validation_history]
|
||||||
|
plt.plot(np.linspace(0, num_epochs-1, len(train_history)), train_values, label='train')
|
||||||
|
plt.plot(np.linspace(0, num_epochs-1, len(validation_history)), val_values, label='validation')
|
||||||
|
# plt.ylim([-0.1, 1])
|
||||||
|
plt.tight_layout()
|
||||||
|
plt.legend()
|
||||||
|
plt.title(key)
|
||||||
|
plt.savefig(plot_path)
|
||||||
|
print(f'Saved plots to {ckpt_dir}')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--eval', action='store_true')
|
||||||
|
parser.add_argument('--onscreen_render', action='store_true')
|
||||||
|
parser.add_argument('--ckpt_dir', action='store', type=str, help='ckpt_dir', required=True)
|
||||||
|
parser.add_argument('--policy_class', action='store', type=str, help='policy_class, capitalize', required=True)
|
||||||
|
parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True)
|
||||||
|
parser.add_argument('--batch_size', action='store', type=int, help='batch_size', required=True)
|
||||||
|
parser.add_argument('--seed', action='store', type=int, help='seed', required=True)
|
||||||
|
parser.add_argument('--num_epochs', action='store', type=int, help='num_epochs', required=True)
|
||||||
|
parser.add_argument('--lr', action='store', type=float, help='lr', required=True)
|
||||||
|
|
||||||
|
# for ACT
|
||||||
|
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
|
||||||
|
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
|
||||||
|
parser.add_argument('--hidden_dim', action='store', type=int, help='hidden_dim', required=False)
|
||||||
|
parser.add_argument('--dim_feedforward', action='store', type=int, help='dim_feedforward', required=False)
|
||||||
|
parser.add_argument('--temporal_agg', action='store_true')
|
||||||
|
|
||||||
|
main(vars(parser.parse_args()))
|
||||||
12
utils.py
12
utils.py
@ -81,10 +81,14 @@ def get_norm_stats(dataset_dir, num_episodes):
|
|||||||
all_action_data = []
|
all_action_data = []
|
||||||
for episode_idx in range(num_episodes):
|
for episode_idx in range(num_episodes):
|
||||||
dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5')
|
dataset_path = os.path.join(dataset_dir, f'episode_{episode_idx}.hdf5')
|
||||||
with h5py.File(dataset_path, 'r') as root:
|
try:
|
||||||
qpos = root['/observations/qpos'][()]
|
with h5py.File(dataset_path, 'r') as root:
|
||||||
qvel = root['/observations/qvel'][()]
|
qpos = root['/observations/qpos'][()]
|
||||||
action = root['/action'][()]
|
qvel = root['/observations/qvel'][()]
|
||||||
|
action = root['/action'][()]
|
||||||
|
except:
|
||||||
|
print(f'Error loading {dataset_path}')
|
||||||
|
quit()
|
||||||
all_qpos_data.append(torch.from_numpy(qpos))
|
all_qpos_data.append(torch.from_numpy(qpos))
|
||||||
all_action_data.append(torch.from_numpy(action))
|
all_action_data.append(torch.from_numpy(action))
|
||||||
all_qpos_data = torch.stack(all_qpos_data)
|
all_qpos_data = torch.stack(all_qpos_data)
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户