no encoder option
这个提交包含在:
@ -72,6 +72,7 @@ def get_args_parser():
|
|||||||
parser.add_argument('--validate_every', action='store', type=int, default=500, help='validate_every', required=False)
|
parser.add_argument('--validate_every', action='store', type=int, default=500, help='validate_every', required=False)
|
||||||
parser.add_argument('--save_every', action='store', type=int, default=500, help='save_every', required=False)
|
parser.add_argument('--save_every', action='store', type=int, default=500, help='save_every', required=False)
|
||||||
parser.add_argument('--resume_ckpt_path', action='store', type=str, help='load_ckpt_path', required=False)
|
parser.add_argument('--resume_ckpt_path', action='store', type=str, help='load_ckpt_path', required=False)
|
||||||
|
parser.add_argument('--no_encoder', action='store_true')
|
||||||
|
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|||||||
@ -88,55 +88,59 @@ class DETRVAE(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
def encode(self, qpos, actions=None, is_pad=None, vq_sample=None):
|
def encode(self, qpos, actions=None, is_pad=None, vq_sample=None):
|
||||||
# cvae encoder
|
|
||||||
is_training = actions is not None # train or val
|
|
||||||
bs, _ = qpos.shape
|
bs, _ = qpos.shape
|
||||||
### Obtain latent z from action sequence
|
if self.encoder is None:
|
||||||
if is_training:
|
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
||||||
# project action sequence to embedding dim, and concat with a CLS token
|
latent_input = self.latent_out_proj(latent_sample)
|
||||||
action_embed = self.encoder_action_proj(actions) # (bs, seq, hidden_dim)
|
probs = binaries = mu = logvar = None
|
||||||
qpos_embed = self.encoder_joint_proj(qpos) # (bs, hidden_dim)
|
|
||||||
qpos_embed = torch.unsqueeze(qpos_embed, axis=1) # (bs, 1, hidden_dim)
|
|
||||||
cls_embed = self.cls_embed.weight # (1, hidden_dim)
|
|
||||||
cls_embed = torch.unsqueeze(cls_embed, axis=0).repeat(bs, 1, 1) # (bs, 1, hidden_dim)
|
|
||||||
encoder_input = torch.cat([cls_embed, qpos_embed, action_embed], axis=1) # (bs, seq+1, hidden_dim)
|
|
||||||
encoder_input = encoder_input.permute(1, 0, 2) # (seq+1, bs, hidden_dim)
|
|
||||||
# do not mask cls token
|
|
||||||
cls_joint_is_pad = torch.full((bs, 2), False).to(qpos.device) # False: not a padding
|
|
||||||
is_pad = torch.cat([cls_joint_is_pad, is_pad], axis=1) # (bs, seq+1)
|
|
||||||
# obtain position embedding
|
|
||||||
pos_embed = self.pos_table.clone().detach()
|
|
||||||
pos_embed = pos_embed.permute(1, 0, 2) # (seq+1, 1, hidden_dim)
|
|
||||||
# query model
|
|
||||||
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
|
|
||||||
encoder_output = encoder_output[0] # take cls output only
|
|
||||||
latent_info = self.latent_proj(encoder_output)
|
|
||||||
|
|
||||||
if self.vq:
|
|
||||||
logits = latent_info.reshape([*latent_info.shape[:-1], self.vq_class, self.vq_dim])
|
|
||||||
probs = torch.softmax(logits, dim=-1)
|
|
||||||
binaries = F.one_hot(torch.multinomial(probs.view(-1, self.vq_dim), 1).squeeze(-1), self.vq_dim).view(-1, self.vq_class, self.vq_dim).float()
|
|
||||||
binaries_flat = binaries.view(-1, self.vq_class * self.vq_dim)
|
|
||||||
probs_flat = probs.view(-1, self.vq_class * self.vq_dim)
|
|
||||||
straigt_through = binaries_flat - probs_flat.detach() + probs_flat
|
|
||||||
latent_input = self.latent_out_proj(straigt_through)
|
|
||||||
mu = logvar = None
|
|
||||||
else:
|
|
||||||
probs = binaries = None
|
|
||||||
mu = latent_info[:, :self.latent_dim]
|
|
||||||
logvar = latent_info[:, self.latent_dim:]
|
|
||||||
latent_sample = reparametrize(mu, logvar)
|
|
||||||
latent_input = self.latent_out_proj(latent_sample)
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
mu = logvar = binaries = probs = None
|
# cvae encoder
|
||||||
if self.vq:
|
is_training = actions is not None # train or val
|
||||||
latent_input = self.latent_out_proj(vq_sample.view(-1, self.vq_class * self.vq_dim))
|
### Obtain latent z from action sequence
|
||||||
else:
|
if is_training:
|
||||||
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
# project action sequence to embedding dim, and concat with a CLS token
|
||||||
latent_input = self.latent_out_proj(latent_sample)
|
action_embed = self.encoder_action_proj(actions) # (bs, seq, hidden_dim)
|
||||||
|
qpos_embed = self.encoder_joint_proj(qpos) # (bs, hidden_dim)
|
||||||
|
qpos_embed = torch.unsqueeze(qpos_embed, axis=1) # (bs, 1, hidden_dim)
|
||||||
|
cls_embed = self.cls_embed.weight # (1, hidden_dim)
|
||||||
|
cls_embed = torch.unsqueeze(cls_embed, axis=0).repeat(bs, 1, 1) # (bs, 1, hidden_dim)
|
||||||
|
encoder_input = torch.cat([cls_embed, qpos_embed, action_embed], axis=1) # (bs, seq+1, hidden_dim)
|
||||||
|
encoder_input = encoder_input.permute(1, 0, 2) # (seq+1, bs, hidden_dim)
|
||||||
|
# do not mask cls token
|
||||||
|
cls_joint_is_pad = torch.full((bs, 2), False).to(qpos.device) # False: not a padding
|
||||||
|
is_pad = torch.cat([cls_joint_is_pad, is_pad], axis=1) # (bs, seq+1)
|
||||||
|
# obtain position embedding
|
||||||
|
pos_embed = self.pos_table.clone().detach()
|
||||||
|
pos_embed = pos_embed.permute(1, 0, 2) # (seq+1, 1, hidden_dim)
|
||||||
|
# query model
|
||||||
|
encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad)
|
||||||
|
encoder_output = encoder_output[0] # take cls output only
|
||||||
|
latent_info = self.latent_proj(encoder_output)
|
||||||
|
|
||||||
|
if self.vq:
|
||||||
|
logits = latent_info.reshape([*latent_info.shape[:-1], self.vq_class, self.vq_dim])
|
||||||
|
probs = torch.softmax(logits, dim=-1)
|
||||||
|
binaries = F.one_hot(torch.multinomial(probs.view(-1, self.vq_dim), 1).squeeze(-1), self.vq_dim).view(-1, self.vq_class, self.vq_dim).float()
|
||||||
|
binaries_flat = binaries.view(-1, self.vq_class * self.vq_dim)
|
||||||
|
probs_flat = probs.view(-1, self.vq_class * self.vq_dim)
|
||||||
|
straigt_through = binaries_flat - probs_flat.detach() + probs_flat
|
||||||
|
latent_input = self.latent_out_proj(straigt_through)
|
||||||
|
mu = logvar = None
|
||||||
|
else:
|
||||||
|
probs = binaries = None
|
||||||
|
mu = latent_info[:, :self.latent_dim]
|
||||||
|
logvar = latent_info[:, self.latent_dim:]
|
||||||
|
latent_sample = reparametrize(mu, logvar)
|
||||||
|
latent_input = self.latent_out_proj(latent_sample)
|
||||||
|
|
||||||
|
else:
|
||||||
|
mu = logvar = binaries = probs = None
|
||||||
|
if self.vq:
|
||||||
|
latent_input = self.latent_out_proj(vq_sample.view(-1, self.vq_class * self.vq_dim))
|
||||||
|
else:
|
||||||
|
latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device)
|
||||||
|
latent_input = self.latent_out_proj(latent_sample)
|
||||||
|
|
||||||
|
|
||||||
return latent_input, probs, binaries, mu, logvar
|
return latent_input, probs, binaries, mu, logvar
|
||||||
|
|
||||||
def forward(self, qpos, image, env_state, actions=None, is_pad=None, vq_sample=None):
|
def forward(self, qpos, image, env_state, actions=None, is_pad=None, vq_sample=None):
|
||||||
@ -275,7 +279,10 @@ def build(args):
|
|||||||
|
|
||||||
transformer = build_transformer(args)
|
transformer = build_transformer(args)
|
||||||
|
|
||||||
encoder = build_encoder(args)
|
if args.no_encoder:
|
||||||
|
encoder = None
|
||||||
|
else:
|
||||||
|
encoder = build_transformer(args)
|
||||||
|
|
||||||
model = DETRVAE(
|
model = DETRVAE(
|
||||||
backbones,
|
backbones,
|
||||||
|
|||||||
@ -77,7 +77,8 @@ def main(args):
|
|||||||
'vq': args['use_vq'],
|
'vq': args['use_vq'],
|
||||||
'vq_class': args['vq_class'],
|
'vq_class': args['vq_class'],
|
||||||
'vq_dim': args['vq_dim'],
|
'vq_dim': args['vq_dim'],
|
||||||
'action_dim': 16
|
'action_dim': 16,
|
||||||
|
'no_encoder': args['no_encoder'],
|
||||||
}
|
}
|
||||||
elif policy_class == 'CNNMLP':
|
elif policy_class == 'CNNMLP':
|
||||||
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
policy_config = {'lr': args['lr'], 'lr_backbone': lr_backbone, 'backbone' : backbone, 'num_queries': 1,
|
||||||
@ -422,7 +423,7 @@ def train_bc(train_dataloader, val_dataloader, config):
|
|||||||
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
|
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
|
||||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
torch.save(policy.state_dict(), ckpt_path)
|
torch.save(policy.state_dict(), ckpt_path)
|
||||||
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=10)
|
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=20)
|
||||||
wandb.log({'success': success}, step=step)
|
wandb.log({'success': success}, step=step)
|
||||||
|
|
||||||
# training
|
# training
|
||||||
@ -484,5 +485,6 @@ if __name__ == '__main__':
|
|||||||
parser.add_argument('--use_vq', action='store_true')
|
parser.add_argument('--use_vq', action='store_true')
|
||||||
parser.add_argument('--vq_class', action='store', type=int, help='vq_class')
|
parser.add_argument('--vq_class', action='store', type=int, help='vq_class')
|
||||||
parser.add_argument('--vq_dim', action='store', type=int, help='vq_dim')
|
parser.add_argument('--vq_dim', action='store', type=int, help='vq_dim')
|
||||||
|
parser.add_argument('--no_encoder', action='store_true')
|
||||||
|
|
||||||
main(vars(parser.parse_args()))
|
main(vars(parser.parse_args()))
|
||||||
|
|||||||
@ -28,11 +28,12 @@ class ACTPolicy(nn.Module):
|
|||||||
|
|
||||||
loss_dict = dict()
|
loss_dict = dict()
|
||||||
a_hat, is_pad_hat, (mu, logvar), probs, binaries = self.model(qpos, image, env_state, actions, is_pad, vq_sample)
|
a_hat, is_pad_hat, (mu, logvar), probs, binaries = self.model(qpos, image, env_state, actions, is_pad, vq_sample)
|
||||||
if self.vq:
|
if self.vq or self.model.encoder is None:
|
||||||
total_kld = [torch.tensor(0.0)]
|
total_kld = [torch.tensor(0.0)]
|
||||||
loss_dict['vq_discrepancy'] = F.l1_loss(probs, binaries, reduction='mean')
|
|
||||||
else:
|
else:
|
||||||
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
|
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar)
|
||||||
|
if self.vq:
|
||||||
|
loss_dict['vq_discrepancy'] = F.l1_loss(probs, binaries, reduction='mean')
|
||||||
all_l1 = F.l1_loss(actions, a_hat, reduction='none')
|
all_l1 = F.l1_loss(actions, a_hat, reduction='none')
|
||||||
l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean()
|
l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean()
|
||||||
loss_dict['l1'] = l1
|
loss_dict['l1'] = l1
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户