fix diffusion EMA

这个提交包含在:
Tony Zhao
2023-12-10 14:40:35 -08:00
父节点 66e07fff99
当前提交 c460687c33
共有 2 个文件被更改,包括 61 次插入 和 32 次删除

查看文件

@ -225,7 +225,7 @@ def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
# load policy and stats
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
policy = make_policy(policy_class, policy_config)
loading_status = policy.load_state_dict(torch.load(ckpt_path))
loading_status = policy.deserialize(torch.load(ckpt_path))
print(loading_status)
policy.cuda()
policy.eval()
@ -234,7 +234,7 @@ def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
vq_class = config['policy_config']['vq_class']
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_last.ckpt')
latent_model.load_state_dict(torch.load(latent_model_ckpt_path))
latent_model.deserialize(torch.load(latent_model_ckpt_path))
latent_model.eval()
latent_model.cuda()
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
@ -506,10 +506,10 @@ def train_bc(train_dataloader, val_dataloader, config):
policy = make_policy(policy_class, policy_config)
if config['load_pretrain']:
loading_status = policy.load_state_dict(torch.load(os.path.join('/home/zfu/interbotix_ws/src/act/ckpts/pretrain_all', 'policy_step_50000_seed_0.ckpt')))
loading_status = policy.deserialize(torch.load(os.path.join('/home/zfu/interbotix_ws/src/act/ckpts/pretrain_all', 'policy_step_50000_seed_0.ckpt')))
print(f'loaded! {loading_status}')
if config['resume_ckpt_path'] is not None:
loading_status = policy.load_state_dict(torch.load(config['resume_ckpt_path']))
loading_status = policy.deserialize(torch.load(config['resume_ckpt_path']))
print(f'Resume policy from: {config["resume_ckpt_path"]}, Status: {loading_status}')
policy.cuda()
optimizer = make_optimizer(policy_class, policy)
@ -537,7 +537,7 @@ def train_bc(train_dataloader, val_dataloader, config):
epoch_val_loss = validation_summary['loss']
if epoch_val_loss < min_val_loss:
min_val_loss = epoch_val_loss
best_ckpt_info = (step, min_val_loss, deepcopy(policy.state_dict()))
best_ckpt_info = (step, min_val_loss, deepcopy(policy.serialize()))
for k in list(validation_summary.keys()):
validation_summary[f'val_{k}'] = validation_summary.pop(k)
wandb.log(validation_summary, step=step)
@ -552,7 +552,7 @@ def train_bc(train_dataloader, val_dataloader, config):
# first save then eval
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
torch.save(policy.state_dict(), ckpt_path)
torch.save(policy.serialize(), ckpt_path)
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=10)
wandb.log({'success': success}, step=step)
@ -569,10 +569,10 @@ def train_bc(train_dataloader, val_dataloader, config):
if step % save_every == 0:
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{step}_seed_{seed}.ckpt')
torch.save(policy.state_dict(), ckpt_path)
torch.save(policy.serialize(), ckpt_path)
ckpt_path = os.path.join(ckpt_dir, f'policy_last.ckpt')
torch.save(policy.state_dict(), ckpt_path)
torch.save(policy.serialize(), ckpt_path)
best_step, min_val_loss, best_state_dict = best_ckpt_info
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{best_step}_seed_{seed}.ckpt')

查看文件

@ -43,31 +43,35 @@ class DiffusionPolicy(nn.Module):
backbones.append(ResNet18Conv(**{'input_channel': 3, 'pretrained': False, 'input_coord_conv': False}))
pools.append(SpatialSoftmax(**{'input_shape': [512, 15, 20], 'num_kp': self.num_kp, 'temperature': 1.0, 'learnable_temperature': False, 'noise_std': 0.0}))
linears.append(torch.nn.Linear(int(np.prod([self.num_kp, 2])), self.feature_dimension))
self.backbones = nn.ModuleList(backbones)
self.pools = nn.ModuleList(pools)
self.linears = nn.ModuleList(linears)
backbones = nn.ModuleList(backbones)
pools = nn.ModuleList(pools)
linears = nn.ModuleList(linears)
self.backbones = replace_bn_with_gn(self.backbones) # TODO
backbones = replace_bn_with_gn(backbones) # TODO
self.noise_pred_net = ConditionalUnet1D(
noise_pred_net = ConditionalUnet1D(
input_dim=self.ac_dim,
global_cond_dim=self.obs_dim*self.observation_horizon
)
nets = nn.ModuleDict({
'policy': nn.ModuleDict({
'backbones': self.backbones,
'pools': self.pools,
'linears': self.linears,
'noise_pred_net': self.noise_pred_net
'backbones': backbones,
'pools': pools,
'linears': linears,
'noise_pred_net': noise_pred_net
})
})
nets = nets.float()
self.ema = EMAModel(model=nets, power=self.ema_power)
nets = nets.float().cuda()
ENABLE_EMA = True
if ENABLE_EMA:
ema = EMAModel(model=nets, power=self.ema_power)
else:
ema = None
self.nets = nets
self.ema = ema
# setup noise scheduler
self.noise_scheduler = DDIMScheduler(
@ -79,6 +83,9 @@ class DiffusionPolicy(nn.Module):
prediction_type='epsilon'
)
n_parameters = sum(p.numel() for p in self.parameters())
print("number of parameters: %.2fM" % (n_parameters/1e6,))
def configure_optimizers(self):
optimizer = torch.optim.AdamW(self.nets.parameters(), lr=self.lr, weight_decay=self.weight_decay)
@ -88,13 +95,14 @@ class DiffusionPolicy(nn.Module):
def __call__(self, qpos, image, actions=None, is_pad=None):
B = qpos.shape[0]
if actions is not None: # training time
nets = self.nets
all_features = []
for cam_id in range(len(self.camera_names)):
cam_image = image[:, cam_id]
cam_features = self.backbones[cam_id](cam_image)
pool_features = self.pools[cam_id](cam_features)
cam_features = nets['policy']['backbones'][cam_id](cam_image)
pool_features = nets['policy']['pools'][cam_id](cam_features)
pool_features = torch.flatten(pool_features, start_dim=1)
out_features = self.linears[cam_id](pool_features)
out_features = nets['policy']['linears'][cam_id](pool_features)
all_features.append(out_features)
obs_cond = torch.cat(all_features + [qpos], dim=1)
@ -114,7 +122,7 @@ class DiffusionPolicy(nn.Module):
actions, noise, timesteps)
# predict the noise residual
noise_pred = self.noise_pred_net(noisy_actions, timesteps, global_cond=obs_cond)
noise_pred = nets['policy']['noise_pred_net'](noisy_actions, timesteps, global_cond=obs_cond)
# L2 loss
loss = F.mse_loss(noise_pred, noise)
@ -122,6 +130,9 @@ class DiffusionPolicy(nn.Module):
loss_dict = {}
loss_dict['l2_loss'] = loss
loss_dict['loss'] = loss
if self.training and self.ema is not None:
self.ema.step(nets)
return loss_dict
else: # inference time
To = self.observation_horizon
@ -129,17 +140,17 @@ class DiffusionPolicy(nn.Module):
Tp = self.prediction_horizon
action_dim = self.ac_dim
# select network ### TODO TODO
# nets = self.nets
# if self.ema is not None:
# nets = self.ema.averaged_model
nets = self.nets
if self.ema is not None:
nets = self.ema.averaged_model
all_features = []
for cam_id in range(len(self.camera_names)):
cam_image = image[:, cam_id]
cam_features = self.backbones[cam_id](cam_image)
pool_features = self.pools[cam_id](cam_features)
cam_features = nets['policy']['backbones'][cam_id](cam_image)
pool_features = nets['policy']['pools'][cam_id](cam_features)
pool_features = torch.flatten(pool_features, start_dim=1)
out_features = self.linears[cam_id](pool_features)
out_features = nets['policy']['linears'][cam_id](pool_features)
all_features.append(out_features)
obs_cond = torch.cat(all_features + [qpos], dim=1)
@ -154,7 +165,7 @@ class DiffusionPolicy(nn.Module):
for k in self.noise_scheduler.timesteps:
# predict noise
noise_pred = self.noise_pred_net(
noise_pred = nets['policy']['noise_pred_net'](
sample=naction,
timestep=k,
global_cond=obs_cond
@ -169,6 +180,20 @@ class DiffusionPolicy(nn.Module):
return naction
def serialize(self):
return {
"nets": self.nets.state_dict(),
"ema": self.ema.averaged_model.state_dict() if self.ema is not None else None,
}
def deserialize(self, model_dict):
status = self.nets.load_state_dict(model_dict["nets"])
print('Loaded model')
if model_dict.get("ema", None) is not None:
print('Loaded EMA')
status_ema = self.ema.averaged_model.load_state_dict(model_dict["ema"])
status = [status, status_ema]
return status
class ACTPolicy(nn.Module):
def __init__(self, args_override):
@ -219,7 +244,11 @@ class ACTPolicy(nn.Module):
return binaries
def serialize(self):
return self.state_dict()
def deserialize(self, model_dict):
return self.load_state_dict(model_dict)
class CNNMLPPolicy(nn.Module):