这个提交包含在:
Zipeng Fu
2023-10-13 17:57:59 -04:00
父节点 694c606940
当前提交 b37712fd2f
共有 10 个文件被更改,包括 658 次插入 和 41 次删除

查看文件

@ -63,6 +63,7 @@ def get_args_parser():
parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False)
parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False)
parser.add_argument('--temporal_agg', action='store_true')
parser.add_argument('--vq', action='store_true')
return parser

查看文件

@ -5,6 +5,7 @@ DETR model and criterion classes.
import torch
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
from .backbone import build_backbone
from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer
@ -33,7 +34,7 @@ def get_sinusoid_encoding_table(n_position, d_hid):
class DETRVAE(nn.Module):
""" This is the DETR module that performs object detection """
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names):
def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names, vq, vq_class, vq_dim, ):
""" Initializes the model.
Parameters:
backbones: torch module of the backbone to be used. See backbone.py
@ -48,6 +49,7 @@ class DETRVAE(nn.Module):
self.camera_names = camera_names
self.transformer = transformer
self.encoder = encoder
self.vq, self.vq_class, self.vq_dim = vq, vq_class, vq_dim
hidden_dim = transformer.d_model
self.action_head = nn.Linear(hidden_dim, state_dim)
self.is_pad_head = nn.Linear(hidden_dim, 1)
@ -68,20 +70,24 @@ class DETRVAE(nn.Module):
self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding
self.encoder_action_proj = nn.Linear(14, hidden_dim) # project action to embedding
self.encoder_joint_proj = nn.Linear(14, hidden_dim) # project qpos to embedding
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
print('Use VQ: ', self.vq)
if self.vq:
self.latent_proj = nn.Linear(hidden_dim, self.vq_class * self.vq_dim)
else:
self.latent_proj = nn.Linear(hidden_dim, self.latent_dim*2) # project hidden state to latent std, var
self.register_buffer('pos_table', get_sinusoid_encoding_table(1+1+num_queries, hidden_dim)) # [CLS], qpos, a_seq
# decoder extra parameters
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
if self.vq:
self.latent_out_proj = nn.Linear(self.vq_class * self.vq_dim, hidden_dim)
else:
self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding
self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent
def forward(self, qpos, image, env_state, actions=None, is_pad=None):
"""
qpos: batch, qpos_dim
image: batch, num_cam, channel, height, width
env_state: None
actions: batch, seq, action_dim
"""
def encode(self, qpos, actions=None, is_pad=None, vq_sample=None):
# cvae encoder
is_training = actions is not None # train or val
bs, _ = qpos.shape
### Obtain latent z from action sequence
@ -104,15 +110,43 @@ class DETRVAE(nn.Module):
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
encoder_output = encoder_output[0] # take cls output only
latent_info = self.latent_proj(encoder_output)
mu = latent_info[:, :self.latent_dim]
logvar = latent_info[:, self.latent_dim:]
latent_sample = reparametrize(mu, logvar)
latent_input = self.latent_out_proj(latent_sample)
else:
mu = logvar = None
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
latent_input = self.latent_out_proj(latent_sample)
if self.vq:
logits = latent_info.reshape([*latent_info.shape[:-1], self.vq_class, self.vq_dim])
probs = torch.softmax(logits, dim=-1)
binaries = F.one_hot(torch.multinomial(probs.view(-1, self.vq_dim), 1).squeeze(-1), self.vq_dim).view(-1, self.vq_class, self.vq_dim).float()
binaries_flat = binaries.view(-1, self.vq_class * self.vq_dim)
probs_flat = probs.view(-1, self.vq_class * self.vq_dim)
straigt_through = binaries_flat - probs_flat.detach() + probs_flat
latent_input = self.latent_out_proj(straigt_through)
mu = logvar = None
else:
mu = latent_info[:, :self.latent_dim]
logvar = latent_info[:, self.latent_dim:]
latent_sample = reparametrize(mu, logvar)
latent_input = self.latent_out_proj(latent_sample)
else:
mu = logvar = binaries = probs = None
if self.vq:
latent_input = self.latent_out_proj(vq_sample.view(-1, self.vq_class * self.vq_dim))
else:
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
latent_input = self.latent_out_proj(latent_sample)
return latent_input, probs, binaries, mu, logvar
def forward(self, qpos, image, env_state, actions=None, is_pad=None, vq_sample=None):
"""
qpos: batch, qpos_dim
image: batch, num_cam, channel, height, width
env_state: None
actions: batch, seq, action_dim
"""
latent_input, probs, binaries, mu, logvar = self.encode(qpos, actions, is_pad, vq_sample)
# cvae decoder
if self.backbones is not None:
# Image observation features and position embeddings
all_cam_features = []
@ -136,7 +170,7 @@ class DETRVAE(nn.Module):
hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0]
a_hat = self.action_head(hs)
is_pad_hat = self.is_pad_head(hs)
return a_hat, is_pad_hat, [mu, logvar]
return a_hat, is_pad_hat, [mu, logvar], probs, binaries
@ -247,6 +281,9 @@ def build(args):
state_dim=state_dim,
num_queries=args.num_queries,
camera_names=args.camera_names,
vq=args.vq,
vq_class=args.vq_class,
vq_dim=args.vq_dim,
)
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)

73
detr/models/latent_model.py 普通文件
查看文件

@ -0,0 +1,73 @@
import torch.nn as nn
from torch.nn import functional as F
import torch
DROPOUT_RATE = 0.1
# a causal transformer block
class Causal_Transformer_Block(nn.Module):
def __init__(self, seq_len, latent_dim, num_head) -> None:
super().__init__()
self.num_head = num_head
self.latent_dim = latent_dim
self.ln_1 = nn.LayerNorm(latent_dim)
self.attn = nn.MultiheadAttention(latent_dim, num_head, dropout=DROPOUT_RATE, batch_first=True)
self.ln_2 = nn.LayerNorm(latent_dim)
self.mlp = nn.Sequential(
nn.Linear(latent_dim, 4 * latent_dim),
nn.GELU(),
nn.Linear(4 * latent_dim, latent_dim),
nn.Dropout(DROPOUT_RATE),
)
# self.register_buffer("attn_mask", torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool())
def forward(self, x):
attn_mask = torch.triu(torch.ones(x.shape[1], x.shape[1], device=x.device, dtype=torch.bool), diagonal=1)
x = self.ln_1(x)
x = x + self.attn(x, x, x, attn_mask=attn_mask)[0]
x = self.ln_2(x)
x = x + self.mlp(x)
return x
# use self-attention instead of RNN to model the latent space sequence
class Latent_Model_Transformer(nn.Module):
def __init__(self, input_dim, output_dim, seq_len, latent_dim=256, num_head=8, num_layer=3) -> None:
super().__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.seq_len = seq_len
self.latent_dim = latent_dim
self.num_head = num_head
self.num_layer = num_layer
self.input_layer = nn.Linear(input_dim, latent_dim)
self.weight_pos_embed = nn.Embedding(seq_len, latent_dim)
self.attention_blocks = nn.Sequential(
nn.Dropout(DROPOUT_RATE),
*[Causal_Transformer_Block(seq_len, latent_dim, num_head) for _ in range(num_layer)],
nn.LayerNorm(latent_dim)
)
self.output_layer = nn.Linear(latent_dim, output_dim)
def forward(self, x):
x = self.input_layer(x)
x = x + self.weight_pos_embed(torch.arange(x.shape[1], device=x.device))
x = self.attention_blocks(x)
logits = self.output_layer(x)
return logits
@torch.no_grad()
def generate(self, n, temperature=0.1, x=None):
if x is None:
x = torch.zeros((n, 1, self.input_dim), device=self.weight_pos_embed.weight.device)
for i in range(self.seq_len):
logits = self.forward(x)[:, -1]
probs = torch.softmax(logits / temperature, dim=-1)
samples = torch.multinomial(probs, num_samples=1)[..., 0]
samples_one_hot = F.one_hot(samples.long(), num_classes=self.output_dim).float()
x = torch.cat([x, samples_one_hot[:, None, :]], dim=1)
return x[:, 1:, :]