ema fixed
这个提交包含在:
36
commands.txt
36
commands.txt
@ -114,6 +114,42 @@ CUDA_VISIBLE_DEVICES=1 python3 imitate_episodes.py \
|
|||||||
--num_steps 100000 --eval_every 2000 --validate_every 2000 --save_every 2000
|
--num_steps 100000 --eval_every 2000 --validate_every 2000 --save_every 2000
|
||||||
|
|
||||||
|
|
||||||
|
# use EMA and train for longer (code changes)
|
||||||
|
|
||||||
|
conda activate mobile
|
||||||
|
export MUJOCO_GL=egl
|
||||||
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
|
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||||
|
--task_name sim_transfer_cube_scripted \
|
||||||
|
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_3_ema \
|
||||||
|
--policy_class Diffusion --chunk_size 32 \
|
||||||
|
--batch_size 32 --lr 1e-4 --seed 0 \
|
||||||
|
--num_steps 300000 --eval_every 6000 --validate_every 6000 --save_every 6000
|
||||||
|
|
||||||
|
# code regression test: no EMA
|
||||||
|
conda activate mobile
|
||||||
|
export MUJOCO_GL=egl
|
||||||
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
|
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||||
|
--task_name sim_transfer_cube_scripted \
|
||||||
|
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_4_regression_no_ema \
|
||||||
|
--policy_class Diffusion --chunk_size 32 \
|
||||||
|
--batch_size 32 --lr 1e-4 --seed 0 \
|
||||||
|
--num_steps 300000 --eval_every 6000 --validate_every 6000 --save_every 6000
|
||||||
|
|
||||||
|
conda activate mobile
|
||||||
|
export MUJOCO_GL=egl
|
||||||
|
cd /home/tonyzhao/Research/act-plus-plus
|
||||||
|
CUDA_VISIBLE_DEVICES=1 python3 imitate_episodes.py \
|
||||||
|
--task_name sim_transfer_cube_scripted \
|
||||||
|
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_5_ema_again \
|
||||||
|
--policy_class Diffusion --chunk_size 32 \
|
||||||
|
--batch_size 32 --lr 1e-4 --seed 0 \
|
||||||
|
--num_steps 300000 --eval_every 6000 --validate_every 6000 --save_every 6000
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---------------------------------------------------------------------------------------
|
---------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|||||||
@ -26,7 +26,10 @@ def compress_dataset(input_dataset_path, output_dataset_path):
|
|||||||
# Create the compressed dataset
|
# Create the compressed dataset
|
||||||
with h5py.File(output_dataset_path, 'w') as outfile:
|
with h5py.File(output_dataset_path, 'w') as outfile:
|
||||||
|
|
||||||
outfile.attrs['sim'] = infile.attrs['sim']
|
if 'sim' in infile.attrs:
|
||||||
|
outfile.attrs['sim'] = infile.attrs['sim']
|
||||||
|
else:
|
||||||
|
outfile.attrs['sim'] = False
|
||||||
outfile.attrs['compress'] = True
|
outfile.attrs['compress'] = True
|
||||||
|
|
||||||
# Copy non-image data directly
|
# Copy non-image data directly
|
||||||
|
|||||||
@ -211,7 +211,7 @@ def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
|
|||||||
# load policy and stats
|
# load policy and stats
|
||||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
policy = make_policy(policy_class, policy_config)
|
policy = make_policy(policy_class, policy_config)
|
||||||
loading_status = policy.load_state_dict(torch.load(ckpt_path))
|
loading_status = policy.deserialize(torch.load(ckpt_path))
|
||||||
print(loading_status)
|
print(loading_status)
|
||||||
policy.cuda()
|
policy.cuda()
|
||||||
policy.eval()
|
policy.eval()
|
||||||
@ -220,7 +220,7 @@ def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
|
|||||||
vq_class = config['policy_config']['vq_class']
|
vq_class = config['policy_config']['vq_class']
|
||||||
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
||||||
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_last.ckpt')
|
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_last.ckpt')
|
||||||
latent_model.load_state_dict(torch.load(latent_model_ckpt_path))
|
latent_model.deserialize(torch.load(latent_model_ckpt_path))
|
||||||
latent_model.eval()
|
latent_model.eval()
|
||||||
latent_model.cuda()
|
latent_model.cuda()
|
||||||
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
|
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
|
||||||
@ -410,10 +410,10 @@ def train_bc(train_dataloader, val_dataloader, config):
|
|||||||
|
|
||||||
policy = make_policy(policy_class, policy_config)
|
policy = make_policy(policy_class, policy_config)
|
||||||
if config['load_pretrain']:
|
if config['load_pretrain']:
|
||||||
loading_status = policy.load_state_dict(torch.load(os.path.join('/home/zfu/interbotix_ws/src/act/ckpts/pretrain_all', 'policy_step_50000_seed_0.ckpt')))
|
loading_status = policy.deserialize(torch.load(os.path.join('/home/zfu/interbotix_ws/src/act/ckpts/pretrain_all', 'policy_step_50000_seed_0.ckpt')))
|
||||||
print(f'loaded! {loading_status}')
|
print(f'loaded! {loading_status}')
|
||||||
if config['resume_ckpt_path'] is not None:
|
if config['resume_ckpt_path'] is not None:
|
||||||
loading_status = policy.load_state_dict(torch.load(config['resume_ckpt_path']))
|
loading_status = policy.deserialize(torch.load(config['resume_ckpt_path']))
|
||||||
print(f'Resume policy from: {config["resume_ckpt_path"]}, Status: {loading_status}')
|
print(f'Resume policy from: {config["resume_ckpt_path"]}, Status: {loading_status}')
|
||||||
policy.cuda()
|
policy.cuda()
|
||||||
optimizer = make_optimizer(policy_class, policy)
|
optimizer = make_optimizer(policy_class, policy)
|
||||||
@ -439,7 +439,7 @@ def train_bc(train_dataloader, val_dataloader, config):
|
|||||||
epoch_val_loss = validation_summary['loss']
|
epoch_val_loss = validation_summary['loss']
|
||||||
if epoch_val_loss < min_val_loss:
|
if epoch_val_loss < min_val_loss:
|
||||||
min_val_loss = epoch_val_loss
|
min_val_loss = epoch_val_loss
|
||||||
best_ckpt_info = (step, min_val_loss, deepcopy(policy.state_dict()))
|
best_ckpt_info = (step, min_val_loss, deepcopy(policy.serialize()))
|
||||||
for k in list(validation_summary.keys()):
|
for k in list(validation_summary.keys()):
|
||||||
validation_summary[f'val_{k}'] = validation_summary.pop(k)
|
validation_summary[f'val_{k}'] = validation_summary.pop(k)
|
||||||
wandb.log(validation_summary, step=step)
|
wandb.log(validation_summary, step=step)
|
||||||
@ -454,7 +454,7 @@ def train_bc(train_dataloader, val_dataloader, config):
|
|||||||
# first save then eval
|
# first save then eval
|
||||||
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
|
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
|
||||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||||
torch.save(policy.state_dict(), ckpt_path)
|
torch.save(policy.serialize(), ckpt_path)
|
||||||
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=10)
|
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=10)
|
||||||
wandb.log({'success': success}, step=step)
|
wandb.log({'success': success}, step=step)
|
||||||
|
|
||||||
@ -471,10 +471,10 @@ def train_bc(train_dataloader, val_dataloader, config):
|
|||||||
|
|
||||||
if step % save_every == 0:
|
if step % save_every == 0:
|
||||||
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{step}_seed_{seed}.ckpt')
|
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{step}_seed_{seed}.ckpt')
|
||||||
torch.save(policy.state_dict(), ckpt_path)
|
torch.save(policy.serialize(), ckpt_path)
|
||||||
|
|
||||||
ckpt_path = os.path.join(ckpt_dir, f'policy_last.ckpt')
|
ckpt_path = os.path.join(ckpt_dir, f'policy_last.ckpt')
|
||||||
torch.save(policy.state_dict(), ckpt_path)
|
torch.save(policy.serialize(), ckpt_path)
|
||||||
|
|
||||||
best_step, min_val_loss, best_state_dict = best_ckpt_info
|
best_step, min_val_loss, best_state_dict = best_ckpt_info
|
||||||
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{best_step}_seed_{seed}.ckpt')
|
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{best_step}_seed_{seed}.ckpt')
|
||||||
|
|||||||
77
policy.py
77
policy.py
@ -43,31 +43,35 @@ class DiffusionPolicy(nn.Module):
|
|||||||
backbones.append(ResNet18Conv(**{'input_channel': 3, 'pretrained': False, 'input_coord_conv': False}))
|
backbones.append(ResNet18Conv(**{'input_channel': 3, 'pretrained': False, 'input_coord_conv': False}))
|
||||||
pools.append(SpatialSoftmax(**{'input_shape': [512, 15, 20], 'num_kp': self.num_kp, 'temperature': 1.0, 'learnable_temperature': False, 'noise_std': 0.0}))
|
pools.append(SpatialSoftmax(**{'input_shape': [512, 15, 20], 'num_kp': self.num_kp, 'temperature': 1.0, 'learnable_temperature': False, 'noise_std': 0.0}))
|
||||||
linears.append(torch.nn.Linear(int(np.prod([self.num_kp, 2])), self.feature_dimension))
|
linears.append(torch.nn.Linear(int(np.prod([self.num_kp, 2])), self.feature_dimension))
|
||||||
self.backbones = nn.ModuleList(backbones)
|
backbones = nn.ModuleList(backbones)
|
||||||
self.pools = nn.ModuleList(pools)
|
pools = nn.ModuleList(pools)
|
||||||
self.linears = nn.ModuleList(linears)
|
linears = nn.ModuleList(linears)
|
||||||
|
|
||||||
self.backbones = replace_bn_with_gn(self.backbones) # TODO
|
backbones = replace_bn_with_gn(backbones) # TODO
|
||||||
|
|
||||||
|
|
||||||
self.noise_pred_net = ConditionalUnet1D(
|
noise_pred_net = ConditionalUnet1D(
|
||||||
input_dim=self.ac_dim,
|
input_dim=self.ac_dim,
|
||||||
global_cond_dim=self.obs_dim*self.observation_horizon
|
global_cond_dim=self.obs_dim*self.observation_horizon
|
||||||
)
|
)
|
||||||
|
|
||||||
nets = nn.ModuleDict({
|
nets = nn.ModuleDict({
|
||||||
'policy': nn.ModuleDict({
|
'policy': nn.ModuleDict({
|
||||||
'backbones': self.backbones,
|
'backbones': backbones,
|
||||||
'pools': self.pools,
|
'pools': pools,
|
||||||
'linears': self.linears,
|
'linears': linears,
|
||||||
'noise_pred_net': self.noise_pred_net
|
'noise_pred_net': noise_pred_net
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
nets = nets.float()
|
nets = nets.float().cuda()
|
||||||
|
ENABLE_EMA = True
|
||||||
self.ema = EMAModel(model=nets, power=self.ema_power)
|
if ENABLE_EMA:
|
||||||
|
ema = EMAModel(model=nets, power=self.ema_power)
|
||||||
|
else:
|
||||||
|
ema = None
|
||||||
self.nets = nets
|
self.nets = nets
|
||||||
|
self.ema = ema
|
||||||
|
|
||||||
# setup noise scheduler
|
# setup noise scheduler
|
||||||
self.noise_scheduler = DDIMScheduler(
|
self.noise_scheduler = DDIMScheduler(
|
||||||
@ -79,6 +83,9 @@ class DiffusionPolicy(nn.Module):
|
|||||||
prediction_type='epsilon'
|
prediction_type='epsilon'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
n_parameters = sum(p.numel() for p in self.parameters())
|
||||||
|
print("number of parameters: %.2fM" % (n_parameters/1e6,))
|
||||||
|
|
||||||
|
|
||||||
def configure_optimizers(self):
|
def configure_optimizers(self):
|
||||||
optimizer = torch.optim.AdamW(self.nets.parameters(), lr=self.lr, weight_decay=self.weight_decay)
|
optimizer = torch.optim.AdamW(self.nets.parameters(), lr=self.lr, weight_decay=self.weight_decay)
|
||||||
@ -88,13 +95,14 @@ class DiffusionPolicy(nn.Module):
|
|||||||
def __call__(self, qpos, image, actions=None, is_pad=None):
|
def __call__(self, qpos, image, actions=None, is_pad=None):
|
||||||
B = qpos.shape[0]
|
B = qpos.shape[0]
|
||||||
if actions is not None: # training time
|
if actions is not None: # training time
|
||||||
|
nets = self.nets
|
||||||
all_features = []
|
all_features = []
|
||||||
for cam_id in range(len(self.camera_names)):
|
for cam_id in range(len(self.camera_names)):
|
||||||
cam_image = image[:, cam_id]
|
cam_image = image[:, cam_id]
|
||||||
cam_features = self.backbones[cam_id](cam_image)
|
cam_features = nets['policy']['backbones'][cam_id](cam_image)
|
||||||
pool_features = self.pools[cam_id](cam_features)
|
pool_features = nets['policy']['pools'][cam_id](cam_features)
|
||||||
pool_features = torch.flatten(pool_features, start_dim=1)
|
pool_features = torch.flatten(pool_features, start_dim=1)
|
||||||
out_features = self.linears[cam_id](pool_features)
|
out_features = nets['policy']['linears'][cam_id](pool_features)
|
||||||
all_features.append(out_features)
|
all_features.append(out_features)
|
||||||
|
|
||||||
obs_cond = torch.cat(all_features + [qpos], dim=1)
|
obs_cond = torch.cat(all_features + [qpos], dim=1)
|
||||||
@ -114,7 +122,7 @@ class DiffusionPolicy(nn.Module):
|
|||||||
actions, noise, timesteps)
|
actions, noise, timesteps)
|
||||||
|
|
||||||
# predict the noise residual
|
# predict the noise residual
|
||||||
noise_pred = self.noise_pred_net(noisy_actions, timesteps, global_cond=obs_cond)
|
noise_pred = nets['policy']['noise_pred_net'](noisy_actions, timesteps, global_cond=obs_cond)
|
||||||
|
|
||||||
# L2 loss
|
# L2 loss
|
||||||
loss = F.mse_loss(noise_pred, noise)
|
loss = F.mse_loss(noise_pred, noise)
|
||||||
@ -122,6 +130,9 @@ class DiffusionPolicy(nn.Module):
|
|||||||
loss_dict = {}
|
loss_dict = {}
|
||||||
loss_dict['l2_loss'] = loss
|
loss_dict['l2_loss'] = loss
|
||||||
loss_dict['loss'] = loss
|
loss_dict['loss'] = loss
|
||||||
|
|
||||||
|
if self.training and self.ema is not None:
|
||||||
|
self.ema.step(nets)
|
||||||
return loss_dict
|
return loss_dict
|
||||||
else: # inference time
|
else: # inference time
|
||||||
To = self.observation_horizon
|
To = self.observation_horizon
|
||||||
@ -129,17 +140,17 @@ class DiffusionPolicy(nn.Module):
|
|||||||
Tp = self.prediction_horizon
|
Tp = self.prediction_horizon
|
||||||
action_dim = self.ac_dim
|
action_dim = self.ac_dim
|
||||||
|
|
||||||
# select network ### TODO TODO
|
nets = self.nets
|
||||||
# nets = self.nets
|
if self.ema is not None:
|
||||||
# if self.ema is not None:
|
nets = self.ema.averaged_model
|
||||||
# nets = self.ema.averaged_model
|
|
||||||
all_features = []
|
all_features = []
|
||||||
for cam_id in range(len(self.camera_names)):
|
for cam_id in range(len(self.camera_names)):
|
||||||
cam_image = image[:, cam_id]
|
cam_image = image[:, cam_id]
|
||||||
cam_features = self.backbones[cam_id](cam_image)
|
cam_features = nets['policy']['backbones'][cam_id](cam_image)
|
||||||
pool_features = self.pools[cam_id](cam_features)
|
pool_features = nets['policy']['pools'][cam_id](cam_features)
|
||||||
pool_features = torch.flatten(pool_features, start_dim=1)
|
pool_features = torch.flatten(pool_features, start_dim=1)
|
||||||
out_features = self.linears[cam_id](pool_features)
|
out_features = nets['policy']['linears'][cam_id](pool_features)
|
||||||
all_features.append(out_features)
|
all_features.append(out_features)
|
||||||
|
|
||||||
obs_cond = torch.cat(all_features + [qpos], dim=1)
|
obs_cond = torch.cat(all_features + [qpos], dim=1)
|
||||||
@ -154,7 +165,7 @@ class DiffusionPolicy(nn.Module):
|
|||||||
|
|
||||||
for k in self.noise_scheduler.timesteps:
|
for k in self.noise_scheduler.timesteps:
|
||||||
# predict noise
|
# predict noise
|
||||||
noise_pred = self.noise_pred_net(
|
noise_pred = nets['policy']['noise_pred_net'](
|
||||||
sample=naction,
|
sample=naction,
|
||||||
timestep=k,
|
timestep=k,
|
||||||
global_cond=obs_cond
|
global_cond=obs_cond
|
||||||
@ -169,6 +180,20 @@ class DiffusionPolicy(nn.Module):
|
|||||||
|
|
||||||
return naction
|
return naction
|
||||||
|
|
||||||
|
def serialize(self):
|
||||||
|
return {
|
||||||
|
"nets": self.nets.state_dict(),
|
||||||
|
"ema": self.ema.averaged_model.state_dict() if self.ema is not None else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
def deserialize(self, model_dict):
|
||||||
|
status = self.nets.load_state_dict(model_dict["nets"])
|
||||||
|
print('Loaded model')
|
||||||
|
if model_dict.get("ema", None) is not None:
|
||||||
|
print('Loaded EMA')
|
||||||
|
status_ema = self.ema.averaged_model.load_state_dict(model_dict["ema"])
|
||||||
|
status = [status, status_ema]
|
||||||
|
return status
|
||||||
|
|
||||||
class ACTPolicy(nn.Module):
|
class ACTPolicy(nn.Module):
|
||||||
def __init__(self, args_override):
|
def __init__(self, args_override):
|
||||||
@ -219,7 +244,11 @@ class ACTPolicy(nn.Module):
|
|||||||
|
|
||||||
return binaries
|
return binaries
|
||||||
|
|
||||||
|
def serialize(self):
|
||||||
|
return self.state_dict()
|
||||||
|
|
||||||
|
def deserialize(self, model_dict):
|
||||||
|
return self.load_state_dict(model_dict)
|
||||||
|
|
||||||
|
|
||||||
class CNNMLPPolicy(nn.Module):
|
class CNNMLPPolicy(nn.Module):
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户