比较提交
1 次代码提交
| 作者 | SHA1 | 提交日期 | |
|---|---|---|---|
| 637d4f9a93 |
36
commands.txt
36
commands.txt
@ -114,6 +114,42 @@ CUDA_VISIBLE_DEVICES=1 python3 imitate_episodes.py \
|
||||
--num_steps 100000 --eval_every 2000 --validate_every 2000 --save_every 2000
|
||||
|
||||
|
||||
# use EMA and train for longer (code changes)
|
||||
|
||||
conda activate mobile
|
||||
export MUJOCO_GL=egl
|
||||
cd /home/tonyzhao/Research/act-plus-plus
|
||||
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||
--task_name sim_transfer_cube_scripted \
|
||||
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_3_ema \
|
||||
--policy_class Diffusion --chunk_size 32 \
|
||||
--batch_size 32 --lr 1e-4 --seed 0 \
|
||||
--num_steps 300000 --eval_every 6000 --validate_every 6000 --save_every 6000
|
||||
|
||||
# code regression test: no EMA
|
||||
conda activate mobile
|
||||
export MUJOCO_GL=egl
|
||||
cd /home/tonyzhao/Research/act-plus-plus
|
||||
CUDA_VISIBLE_DEVICES=0 python3 imitate_episodes.py \
|
||||
--task_name sim_transfer_cube_scripted \
|
||||
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_4_regression_no_ema \
|
||||
--policy_class Diffusion --chunk_size 32 \
|
||||
--batch_size 32 --lr 1e-4 --seed 0 \
|
||||
--num_steps 300000 --eval_every 6000 --validate_every 6000 --save_every 6000
|
||||
|
||||
conda activate mobile
|
||||
export MUJOCO_GL=egl
|
||||
cd /home/tonyzhao/Research/act-plus-plus
|
||||
CUDA_VISIBLE_DEVICES=1 python3 imitate_episodes.py \
|
||||
--task_name sim_transfer_cube_scripted \
|
||||
--ckpt_dir /scr/tonyzhao/train_logs/cube_scripted_diffusion_sweep_5_ema_again \
|
||||
--policy_class Diffusion --chunk_size 32 \
|
||||
--batch_size 32 --lr 1e-4 --seed 0 \
|
||||
--num_steps 300000 --eval_every 6000 --validate_every 6000 --save_every 6000
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@ -26,7 +26,10 @@ def compress_dataset(input_dataset_path, output_dataset_path):
|
||||
# Create the compressed dataset
|
||||
with h5py.File(output_dataset_path, 'w') as outfile:
|
||||
|
||||
outfile.attrs['sim'] = infile.attrs['sim']
|
||||
if 'sim' in infile.attrs:
|
||||
outfile.attrs['sim'] = infile.attrs['sim']
|
||||
else:
|
||||
outfile.attrs['sim'] = False
|
||||
outfile.attrs['compress'] = True
|
||||
|
||||
# Copy non-image data directly
|
||||
|
||||
@ -211,7 +211,7 @@ def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
|
||||
# load policy and stats
|
||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||
policy = make_policy(policy_class, policy_config)
|
||||
loading_status = policy.load_state_dict(torch.load(ckpt_path))
|
||||
loading_status = policy.deserialize(torch.load(ckpt_path))
|
||||
print(loading_status)
|
||||
policy.cuda()
|
||||
policy.eval()
|
||||
@ -220,7 +220,7 @@ def eval_bc(config, ckpt_name, save_episode=True, num_rollouts=50):
|
||||
vq_class = config['policy_config']['vq_class']
|
||||
latent_model = Latent_Model_Transformer(vq_dim, vq_dim, vq_class)
|
||||
latent_model_ckpt_path = os.path.join(ckpt_dir, 'latent_model_last.ckpt')
|
||||
latent_model.load_state_dict(torch.load(latent_model_ckpt_path))
|
||||
latent_model.deserialize(torch.load(latent_model_ckpt_path))
|
||||
latent_model.eval()
|
||||
latent_model.cuda()
|
||||
print(f'Loaded policy from: {ckpt_path}, latent model from: {latent_model_ckpt_path}')
|
||||
@ -410,10 +410,10 @@ def train_bc(train_dataloader, val_dataloader, config):
|
||||
|
||||
policy = make_policy(policy_class, policy_config)
|
||||
if config['load_pretrain']:
|
||||
loading_status = policy.load_state_dict(torch.load(os.path.join('/home/zfu/interbotix_ws/src/act/ckpts/pretrain_all', 'policy_step_50000_seed_0.ckpt')))
|
||||
loading_status = policy.deserialize(torch.load(os.path.join('/home/zfu/interbotix_ws/src/act/ckpts/pretrain_all', 'policy_step_50000_seed_0.ckpt')))
|
||||
print(f'loaded! {loading_status}')
|
||||
if config['resume_ckpt_path'] is not None:
|
||||
loading_status = policy.load_state_dict(torch.load(config['resume_ckpt_path']))
|
||||
loading_status = policy.deserialize(torch.load(config['resume_ckpt_path']))
|
||||
print(f'Resume policy from: {config["resume_ckpt_path"]}, Status: {loading_status}')
|
||||
policy.cuda()
|
||||
optimizer = make_optimizer(policy_class, policy)
|
||||
@ -439,7 +439,7 @@ def train_bc(train_dataloader, val_dataloader, config):
|
||||
epoch_val_loss = validation_summary['loss']
|
||||
if epoch_val_loss < min_val_loss:
|
||||
min_val_loss = epoch_val_loss
|
||||
best_ckpt_info = (step, min_val_loss, deepcopy(policy.state_dict()))
|
||||
best_ckpt_info = (step, min_val_loss, deepcopy(policy.serialize()))
|
||||
for k in list(validation_summary.keys()):
|
||||
validation_summary[f'val_{k}'] = validation_summary.pop(k)
|
||||
wandb.log(validation_summary, step=step)
|
||||
@ -454,7 +454,7 @@ def train_bc(train_dataloader, val_dataloader, config):
|
||||
# first save then eval
|
||||
ckpt_name = f'policy_step_{step}_seed_{seed}.ckpt'
|
||||
ckpt_path = os.path.join(ckpt_dir, ckpt_name)
|
||||
torch.save(policy.state_dict(), ckpt_path)
|
||||
torch.save(policy.serialize(), ckpt_path)
|
||||
success, _ = eval_bc(config, ckpt_name, save_episode=True, num_rollouts=10)
|
||||
wandb.log({'success': success}, step=step)
|
||||
|
||||
@ -471,10 +471,10 @@ def train_bc(train_dataloader, val_dataloader, config):
|
||||
|
||||
if step % save_every == 0:
|
||||
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{step}_seed_{seed}.ckpt')
|
||||
torch.save(policy.state_dict(), ckpt_path)
|
||||
torch.save(policy.serialize(), ckpt_path)
|
||||
|
||||
ckpt_path = os.path.join(ckpt_dir, f'policy_last.ckpt')
|
||||
torch.save(policy.state_dict(), ckpt_path)
|
||||
torch.save(policy.serialize(), ckpt_path)
|
||||
|
||||
best_step, min_val_loss, best_state_dict = best_ckpt_info
|
||||
ckpt_path = os.path.join(ckpt_dir, f'policy_step_{best_step}_seed_{seed}.ckpt')
|
||||
|
||||
77
policy.py
77
policy.py
@ -43,31 +43,35 @@ class DiffusionPolicy(nn.Module):
|
||||
backbones.append(ResNet18Conv(**{'input_channel': 3, 'pretrained': False, 'input_coord_conv': False}))
|
||||
pools.append(SpatialSoftmax(**{'input_shape': [512, 15, 20], 'num_kp': self.num_kp, 'temperature': 1.0, 'learnable_temperature': False, 'noise_std': 0.0}))
|
||||
linears.append(torch.nn.Linear(int(np.prod([self.num_kp, 2])), self.feature_dimension))
|
||||
self.backbones = nn.ModuleList(backbones)
|
||||
self.pools = nn.ModuleList(pools)
|
||||
self.linears = nn.ModuleList(linears)
|
||||
backbones = nn.ModuleList(backbones)
|
||||
pools = nn.ModuleList(pools)
|
||||
linears = nn.ModuleList(linears)
|
||||
|
||||
self.backbones = replace_bn_with_gn(self.backbones) # TODO
|
||||
backbones = replace_bn_with_gn(backbones) # TODO
|
||||
|
||||
|
||||
self.noise_pred_net = ConditionalUnet1D(
|
||||
noise_pred_net = ConditionalUnet1D(
|
||||
input_dim=self.ac_dim,
|
||||
global_cond_dim=self.obs_dim*self.observation_horizon
|
||||
)
|
||||
|
||||
nets = nn.ModuleDict({
|
||||
'policy': nn.ModuleDict({
|
||||
'backbones': self.backbones,
|
||||
'pools': self.pools,
|
||||
'linears': self.linears,
|
||||
'noise_pred_net': self.noise_pred_net
|
||||
'backbones': backbones,
|
||||
'pools': pools,
|
||||
'linears': linears,
|
||||
'noise_pred_net': noise_pred_net
|
||||
})
|
||||
})
|
||||
|
||||
nets = nets.float()
|
||||
|
||||
self.ema = EMAModel(model=nets, power=self.ema_power)
|
||||
nets = nets.float().cuda()
|
||||
ENABLE_EMA = True
|
||||
if ENABLE_EMA:
|
||||
ema = EMAModel(model=nets, power=self.ema_power)
|
||||
else:
|
||||
ema = None
|
||||
self.nets = nets
|
||||
self.ema = ema
|
||||
|
||||
# setup noise scheduler
|
||||
self.noise_scheduler = DDIMScheduler(
|
||||
@ -79,6 +83,9 @@ class DiffusionPolicy(nn.Module):
|
||||
prediction_type='epsilon'
|
||||
)
|
||||
|
||||
n_parameters = sum(p.numel() for p in self.parameters())
|
||||
print("number of parameters: %.2fM" % (n_parameters/1e6,))
|
||||
|
||||
|
||||
def configure_optimizers(self):
|
||||
optimizer = torch.optim.AdamW(self.nets.parameters(), lr=self.lr, weight_decay=self.weight_decay)
|
||||
@ -88,13 +95,14 @@ class DiffusionPolicy(nn.Module):
|
||||
def __call__(self, qpos, image, actions=None, is_pad=None):
|
||||
B = qpos.shape[0]
|
||||
if actions is not None: # training time
|
||||
nets = self.nets
|
||||
all_features = []
|
||||
for cam_id in range(len(self.camera_names)):
|
||||
cam_image = image[:, cam_id]
|
||||
cam_features = self.backbones[cam_id](cam_image)
|
||||
pool_features = self.pools[cam_id](cam_features)
|
||||
cam_features = nets['policy']['backbones'][cam_id](cam_image)
|
||||
pool_features = nets['policy']['pools'][cam_id](cam_features)
|
||||
pool_features = torch.flatten(pool_features, start_dim=1)
|
||||
out_features = self.linears[cam_id](pool_features)
|
||||
out_features = nets['policy']['linears'][cam_id](pool_features)
|
||||
all_features.append(out_features)
|
||||
|
||||
obs_cond = torch.cat(all_features + [qpos], dim=1)
|
||||
@ -114,7 +122,7 @@ class DiffusionPolicy(nn.Module):
|
||||
actions, noise, timesteps)
|
||||
|
||||
# predict the noise residual
|
||||
noise_pred = self.noise_pred_net(noisy_actions, timesteps, global_cond=obs_cond)
|
||||
noise_pred = nets['policy']['noise_pred_net'](noisy_actions, timesteps, global_cond=obs_cond)
|
||||
|
||||
# L2 loss
|
||||
loss = F.mse_loss(noise_pred, noise)
|
||||
@ -122,6 +130,9 @@ class DiffusionPolicy(nn.Module):
|
||||
loss_dict = {}
|
||||
loss_dict['l2_loss'] = loss
|
||||
loss_dict['loss'] = loss
|
||||
|
||||
if self.training and self.ema is not None:
|
||||
self.ema.step(nets)
|
||||
return loss_dict
|
||||
else: # inference time
|
||||
To = self.observation_horizon
|
||||
@ -129,17 +140,17 @@ class DiffusionPolicy(nn.Module):
|
||||
Tp = self.prediction_horizon
|
||||
action_dim = self.ac_dim
|
||||
|
||||
# select network ### TODO TODO
|
||||
# nets = self.nets
|
||||
# if self.ema is not None:
|
||||
# nets = self.ema.averaged_model
|
||||
nets = self.nets
|
||||
if self.ema is not None:
|
||||
nets = self.ema.averaged_model
|
||||
|
||||
all_features = []
|
||||
for cam_id in range(len(self.camera_names)):
|
||||
cam_image = image[:, cam_id]
|
||||
cam_features = self.backbones[cam_id](cam_image)
|
||||
pool_features = self.pools[cam_id](cam_features)
|
||||
cam_features = nets['policy']['backbones'][cam_id](cam_image)
|
||||
pool_features = nets['policy']['pools'][cam_id](cam_features)
|
||||
pool_features = torch.flatten(pool_features, start_dim=1)
|
||||
out_features = self.linears[cam_id](pool_features)
|
||||
out_features = nets['policy']['linears'][cam_id](pool_features)
|
||||
all_features.append(out_features)
|
||||
|
||||
obs_cond = torch.cat(all_features + [qpos], dim=1)
|
||||
@ -154,7 +165,7 @@ class DiffusionPolicy(nn.Module):
|
||||
|
||||
for k in self.noise_scheduler.timesteps:
|
||||
# predict noise
|
||||
noise_pred = self.noise_pred_net(
|
||||
noise_pred = nets['policy']['noise_pred_net'](
|
||||
sample=naction,
|
||||
timestep=k,
|
||||
global_cond=obs_cond
|
||||
@ -169,6 +180,20 @@ class DiffusionPolicy(nn.Module):
|
||||
|
||||
return naction
|
||||
|
||||
def serialize(self):
|
||||
return {
|
||||
"nets": self.nets.state_dict(),
|
||||
"ema": self.ema.averaged_model.state_dict() if self.ema is not None else None,
|
||||
}
|
||||
|
||||
def deserialize(self, model_dict):
|
||||
status = self.nets.load_state_dict(model_dict["nets"])
|
||||
print('Loaded model')
|
||||
if model_dict.get("ema", None) is not None:
|
||||
print('Loaded EMA')
|
||||
status_ema = self.ema.averaged_model.load_state_dict(model_dict["ema"])
|
||||
status = [status, status_ema]
|
||||
return status
|
||||
|
||||
class ACTPolicy(nn.Module):
|
||||
def __init__(self, args_override):
|
||||
@ -219,7 +244,11 @@ class ACTPolicy(nn.Module):
|
||||
|
||||
return binaries
|
||||
|
||||
def serialize(self):
|
||||
return self.state_dict()
|
||||
|
||||
def deserialize(self, model_dict):
|
||||
return self.load_state_dict(model_dict)
|
||||
|
||||
|
||||
class CNNMLPPolicy(nn.Module):
|
||||
|
||||
在新工单中引用
屏蔽一个用户