镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
feat(evo1): merge stage2 training optimizations
这个提交包含在:
@ -33,6 +33,17 @@ Validated in this repo / recent experiments:
|
|||||||
- `training_stage=stage2` finetunes both the VLM and the action head.
|
- `training_stage=stage2` finetunes both the VLM and the action head.
|
||||||
- In this repo, `resume_pretrain=true` is the Evo1-style stage handoff: load weights only, but start a fresh optimizer/scheduler state.
|
- In this repo, `resume_pretrain=true` is the Evo1-style stage handoff: load weights only, but start a fresh optimizer/scheduler state.
|
||||||
- We currently recommend `dataset.image_transforms.enable=false` in the validated training commands because the image augmentation pipeline was not stable in the tested environment.
|
- We currently recommend `dataset.image_transforms.enable=false` in the validated training commands because the image augmentation pipeline was not stable in the tested environment.
|
||||||
|
- Stage2 now defaults to `embedder_tensor_fastpath=true`, `enable_vlm_gradient_checkpointing=true`, and
|
||||||
|
`gradient_checkpointing_use_reentrant=false`.
|
||||||
|
|
||||||
|
## Optimization Summary
|
||||||
|
|
||||||
|
The current Evo1 integration includes a small set of stage2-oriented training optimizations merged from the Evo_2 branch while preserving the LeRobot/Evo-RL structure:
|
||||||
|
|
||||||
|
- Runtime config switches were added for `embedder_tensor_fastpath`, `enable_vlm_gradient_checkpointing`, and `gradient_checkpointing_use_reentrant`.
|
||||||
|
- Evo1 image collection during training no longer forces a `GPU -> CPU -> GPU` round trip before the VLM embedder.
|
||||||
|
- InternVL3 stage2 now enables a more memory-friendly checkpointing path and disables `use_cache` when checkpointing is active.
|
||||||
|
- The main `lerobot_train` entrypoint was kept intact; only lightweight Evo1 stage2 runtime logging was added.
|
||||||
|
|
||||||
## Stage 1 Command
|
## Stage 1 Command
|
||||||
|
|
||||||
|
|||||||
@ -56,6 +56,9 @@ class Evo1Config(PreTrainedConfig):
|
|||||||
vlm_num_layers: int | None = 14
|
vlm_num_layers: int | None = 14
|
||||||
vlm_dtype: str = "bfloat16"
|
vlm_dtype: str = "bfloat16"
|
||||||
use_flash_attn: bool = True
|
use_flash_attn: bool = True
|
||||||
|
embedder_tensor_fastpath: bool = True
|
||||||
|
enable_vlm_gradient_checkpointing: bool | None = None
|
||||||
|
gradient_checkpointing_use_reentrant: bool = False
|
||||||
action_head: str = "flowmatching"
|
action_head: str = "flowmatching"
|
||||||
embed_dim: int = 896
|
embed_dim: int = 896
|
||||||
hidden_dim: int = 1024
|
hidden_dim: int = 1024
|
||||||
@ -97,6 +100,9 @@ class Evo1Config(PreTrainedConfig):
|
|||||||
self.finetune_vlm = True
|
self.finetune_vlm = True
|
||||||
self.finetune_action_head = True
|
self.finetune_action_head = True
|
||||||
|
|
||||||
|
if self.enable_vlm_gradient_checkpointing is None:
|
||||||
|
self.enable_vlm_gradient_checkpointing = self.training_stage == "stage2"
|
||||||
|
|
||||||
if self.n_action_steps > self.chunk_size:
|
if self.n_action_steps > self.chunk_size:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"n_action_steps ({self.n_action_steps}) must be <= chunk_size ({self.chunk_size})"
|
f"n_action_steps ({self.n_action_steps}) must be <= chunk_size ({self.chunk_size})"
|
||||||
|
|||||||
@ -22,6 +22,9 @@ class EVO1(nn.Module):
|
|||||||
num_language_layers=config.get("vlm_num_layers", 14),
|
num_language_layers=config.get("vlm_num_layers", 14),
|
||||||
model_dtype=config.get("vlm_dtype", "bfloat16"),
|
model_dtype=config.get("vlm_dtype", "bfloat16"),
|
||||||
use_flash_attn=config.get("use_flash_attn", True),
|
use_flash_attn=config.get("use_flash_attn", True),
|
||||||
|
enable_gradient_checkpointing=config.get("enable_vlm_gradient_checkpointing", False),
|
||||||
|
enable_tensor_fastpath=config.get("embedder_tensor_fastpath", True),
|
||||||
|
gradient_checkpointing_use_reentrant=config.get("gradient_checkpointing_use_reentrant", False),
|
||||||
)
|
)
|
||||||
|
|
||||||
action_head_type = config.get("action_head", "flowmatching").lower()
|
action_head_type = config.get("action_head", "flowmatching").lower()
|
||||||
|
|||||||
@ -88,12 +88,18 @@ class InternVL3Embedder(nn.Module):
|
|||||||
num_language_layers: int | None = 14,
|
num_language_layers: int | None = 14,
|
||||||
model_dtype: str | torch.dtype = "bfloat16",
|
model_dtype: str | torch.dtype = "bfloat16",
|
||||||
use_flash_attn: bool = True,
|
use_flash_attn: bool = True,
|
||||||
|
enable_gradient_checkpointing: bool = False,
|
||||||
|
enable_tensor_fastpath: bool = True,
|
||||||
|
gradient_checkpointing_use_reentrant: bool = False,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._requested_device = device
|
self._requested_device = device
|
||||||
self.image_size = image_size
|
self.image_size = image_size
|
||||||
self.num_language_layers = num_language_layers
|
self.num_language_layers = num_language_layers
|
||||||
self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens
|
self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens
|
||||||
|
self.enable_gradient_checkpointing = bool(enable_gradient_checkpointing)
|
||||||
|
self.enable_tensor_fastpath = bool(enable_tensor_fastpath)
|
||||||
|
self.gradient_checkpointing_use_reentrant = bool(gradient_checkpointing_use_reentrant)
|
||||||
self.transform = build_transform(image_size)
|
self.transform = build_transform(image_size)
|
||||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
|
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
|
||||||
if isinstance(model_dtype, str):
|
if isinstance(model_dtype, str):
|
||||||
@ -123,12 +129,53 @@ class InternVL3Embedder(nn.Module):
|
|||||||
else:
|
else:
|
||||||
self.model.language_model.layers = torch.nn.ModuleList(layers)
|
self.model.language_model.layers = torch.nn.ModuleList(layers)
|
||||||
self.model.language_model.lm_head = torch.nn.Identity()
|
self.model.language_model.lm_head = torch.nn.Identity()
|
||||||
|
self._configure_memory_features()
|
||||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
|
||||||
self.model.vision_model.encoder.gradient_checkpointing = False
|
|
||||||
|
|
||||||
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
||||||
|
|
||||||
|
def _configure_memory_features(self) -> None:
|
||||||
|
checkpoint_kwargs = {"use_reentrant": self.gradient_checkpointing_use_reentrant}
|
||||||
|
|
||||||
|
def _enable_ckpt(module) -> bool:
|
||||||
|
if module is None or not hasattr(module, "gradient_checkpointing_enable"):
|
||||||
|
return False
|
||||||
|
module.gradient_checkpointing_enable(gradient_checkpointing_kwargs=checkpoint_kwargs)
|
||||||
|
return True
|
||||||
|
|
||||||
|
if not self.enable_gradient_checkpointing:
|
||||||
|
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||||
|
self.model.vision_model.encoder.gradient_checkpointing = False
|
||||||
|
return
|
||||||
|
|
||||||
|
enabled_any = False
|
||||||
|
enabled_any = _enable_ckpt(self.model) or enabled_any
|
||||||
|
|
||||||
|
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||||
|
self.model.vision_model.encoder.gradient_checkpointing = True
|
||||||
|
enabled_any = True
|
||||||
|
|
||||||
|
if hasattr(self.model, "language_model"):
|
||||||
|
language_model = self.model.language_model
|
||||||
|
enabled_any = _enable_ckpt(language_model) or enabled_any
|
||||||
|
if hasattr(language_model, "model"):
|
||||||
|
enabled_any = _enable_ckpt(language_model.model) or enabled_any
|
||||||
|
if hasattr(language_model, "config"):
|
||||||
|
language_model.config.use_cache = False
|
||||||
|
|
||||||
|
if hasattr(self.model, "config"):
|
||||||
|
self.model.config.use_cache = False
|
||||||
|
|
||||||
|
if hasattr(self.model, "enable_input_require_grads"):
|
||||||
|
self.model.enable_input_require_grads()
|
||||||
|
|
||||||
|
if enabled_any:
|
||||||
|
logging.info(
|
||||||
|
"Enabled InternVL3 gradient checkpointing (use_reentrant=%s).",
|
||||||
|
self.gradient_checkpointing_use_reentrant,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.warning("Requested InternVL3 gradient checkpointing, but no checkpointable module was found.")
|
||||||
|
|
||||||
def _dynamic_preprocess_tensor(
|
def _dynamic_preprocess_tensor(
|
||||||
self, image_t, min_num=1, max_num=1, use_thumbnail=False
|
self, image_t, min_num=1, max_num=1, use_thumbnail=False
|
||||||
):
|
):
|
||||||
@ -191,7 +238,7 @@ class InternVL3Embedder(nn.Module):
|
|||||||
for image_tensors in image_tensors_batch:
|
for image_tensors in image_tensors_batch:
|
||||||
num_tiles_list = []
|
num_tiles_list = []
|
||||||
for image in image_tensors:
|
for image in image_tensors:
|
||||||
if isinstance(image, torch.Tensor):
|
if isinstance(image, torch.Tensor) and self.enable_tensor_fastpath:
|
||||||
if (
|
if (
|
||||||
image.ndim == 3
|
image.ndim == 3
|
||||||
and image.shape[0] == 3
|
and image.shape[0] == 3
|
||||||
@ -216,6 +263,8 @@ class InternVL3Embedder(nn.Module):
|
|||||||
tiles = self._dynamic_preprocess_tensor(image_t)
|
tiles = self._dynamic_preprocess_tensor(image_t)
|
||||||
tile_tensors = (tiles - mean) / std
|
tile_tensors = (tiles - mean) / std
|
||||||
else:
|
else:
|
||||||
|
if isinstance(image, torch.Tensor):
|
||||||
|
image = to_pil_image(image.detach().to(device="cpu"))
|
||||||
image_t = TF.to_tensor(image).to(device=self.device, dtype=torch.float32)
|
image_t = TF.to_tensor(image).to(device=self.device, dtype=torch.float32)
|
||||||
tiles = self._dynamic_preprocess_tensor(image_t)
|
tiles = self._dynamic_preprocess_tensor(image_t)
|
||||||
tile_tensors = (tiles - mean) / std
|
tile_tensors = (tiles - mean) / std
|
||||||
|
|||||||
@ -70,6 +70,9 @@ class EVO1Policy(PreTrainedPolicy):
|
|||||||
"vlm_num_layers": config.vlm_num_layers,
|
"vlm_num_layers": config.vlm_num_layers,
|
||||||
"vlm_dtype": config.vlm_dtype,
|
"vlm_dtype": config.vlm_dtype,
|
||||||
"use_flash_attn": config.use_flash_attn,
|
"use_flash_attn": config.use_flash_attn,
|
||||||
|
"embedder_tensor_fastpath": config.embedder_tensor_fastpath,
|
||||||
|
"enable_vlm_gradient_checkpointing": config.enable_vlm_gradient_checkpointing,
|
||||||
|
"gradient_checkpointing_use_reentrant": config.gradient_checkpointing_use_reentrant,
|
||||||
"action_head": config.action_head,
|
"action_head": config.action_head,
|
||||||
"action_horizon": config.chunk_size,
|
"action_horizon": config.chunk_size,
|
||||||
"per_action_dim": config.max_action_dim,
|
"per_action_dim": config.max_action_dim,
|
||||||
@ -300,7 +303,7 @@ class EVO1Policy(PreTrainedPolicy):
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"EVO1 batch mismatch across cameras: key={camera_key} batch={image.shape[0]} expected={batch_size}"
|
f"EVO1 batch mismatch across cameras: key={camera_key} batch={image.shape[0]} expected={batch_size}"
|
||||||
)
|
)
|
||||||
sample_images.append(image[batch_index].detach().cpu())
|
sample_images.append(image[batch_index])
|
||||||
if not sample_images:
|
if not sample_images:
|
||||||
raise ValueError("EVO1 received a batch without any image tensor.")
|
raise ValueError("EVO1 received a batch without any image tensor.")
|
||||||
while len(sample_images) < self.config.max_views:
|
while len(sample_images) < self.config.max_views:
|
||||||
|
|||||||
@ -273,6 +273,14 @@ def train(
|
|||||||
" scheduler, and global step will start fresh.",
|
" scheduler, and global step will start fresh.",
|
||||||
cfg.policy.pretrained_path,
|
cfg.policy.pretrained_path,
|
||||||
)
|
)
|
||||||
|
if cfg.policy.type == "evo1" and getattr(cfg.policy, "training_stage", None) == "stage2":
|
||||||
|
logging.info(
|
||||||
|
"Evo1 stage2 runtime settings: embedder_tensor_fastpath=%s "
|
||||||
|
"enable_vlm_gradient_checkpointing=%s gradient_checkpointing_use_reentrant=%s",
|
||||||
|
getattr(cfg.policy, "embedder_tensor_fastpath", None),
|
||||||
|
getattr(cfg.policy, "enable_vlm_gradient_checkpointing", None),
|
||||||
|
getattr(cfg.policy, "gradient_checkpointing_use_reentrant", None),
|
||||||
|
)
|
||||||
policy = make_policy(
|
policy = make_policy(
|
||||||
cfg=cfg.policy,
|
cfg=cfg.policy,
|
||||||
ds_meta=dataset.meta,
|
ds_meta=dataset.meta,
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户