镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
fix(evo1): address review follow-ups
这个提交包含在:
@ -1,12 +1,13 @@
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from torch.optim import Optimizer
|
||||
from torch.optim.lr_scheduler import LambdaLR
|
||||
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
|
||||
from lerobot.optim.optimizers import AdamWConfig
|
||||
from lerobot.optim.schedulers import LRSchedulerConfig
|
||||
from torch.optim import Optimizer
|
||||
from torch.optim.lr_scheduler import LambdaLR
|
||||
from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE
|
||||
|
||||
|
||||
@ -52,6 +53,9 @@ class Evo1Config(PreTrainedConfig):
|
||||
)
|
||||
|
||||
vlm_model_name: str = "OpenGVLab/InternVL3-1B"
|
||||
vlm_num_layers: int | None = 14
|
||||
vlm_dtype: str = "bfloat16"
|
||||
use_flash_attn: bool = True
|
||||
action_head: str = "flowmatching"
|
||||
embed_dim: int = 896
|
||||
hidden_dim: int = 1024
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
from types import SimpleNamespace
|
||||
from typing import Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@ -16,7 +15,14 @@ class EVO1(nn.Module):
|
||||
self._device = config.get("device", "cuda")
|
||||
self.return_cls_only = config.get("return_cls_only", False)
|
||||
vlm_name = config.get("vlm_name", "OpenGVLab/InternVL3-1B")
|
||||
self.embedder = InternVL3Embedder(model_name=vlm_name, device=self._device)
|
||||
self.embedder = InternVL3Embedder(
|
||||
model_name=vlm_name,
|
||||
image_size=config.get("image_size", 448),
|
||||
device=self._device,
|
||||
num_language_layers=config.get("vlm_num_layers", 14),
|
||||
model_dtype=config.get("vlm_dtype", "bfloat16"),
|
||||
use_flash_attn=config.get("use_flash_attn", True),
|
||||
)
|
||||
|
||||
action_head_type = config.get("action_head", "flowmatching").lower()
|
||||
if action_head_type != "flowmatching":
|
||||
@ -30,11 +36,6 @@ class EVO1(nn.Module):
|
||||
config["per_action_dim"] = per_action_dim
|
||||
config["action_dim"] = action_dim
|
||||
|
||||
if action_dim != horizon * per_action_dim:
|
||||
raise ValueError(
|
||||
f"action_dim ({action_dim}) must equal horizon ({horizon}) * per_action_dim ({per_action_dim})"
|
||||
)
|
||||
|
||||
self.horizon = horizon
|
||||
self.per_action_dim = per_action_dim
|
||||
self.action_head = FlowmatchingActionHead(
|
||||
@ -56,7 +57,7 @@ class EVO1(nn.Module):
|
||||
|
||||
def get_vl_embeddings(
|
||||
self,
|
||||
images: list[Union[Image.Image, torch.Tensor]],
|
||||
images: list[Image.Image | torch.Tensor],
|
||||
image_mask: torch.Tensor,
|
||||
prompt: str = "",
|
||||
return_cls_only: bool | None = None,
|
||||
@ -72,7 +73,7 @@ class EVO1(nn.Module):
|
||||
return_cls_only=return_cls_only,
|
||||
)
|
||||
|
||||
def prepare_state(self, state_input: Union[list, torch.Tensor]) -> torch.Tensor:
|
||||
def prepare_state(self, state_input: list | torch.Tensor) -> torch.Tensor:
|
||||
if isinstance(state_input, list):
|
||||
state_tensor = torch.tensor(state_input)
|
||||
elif isinstance(state_input, torch.Tensor):
|
||||
@ -111,10 +112,10 @@ class EVO1(nn.Module):
|
||||
@torch.no_grad()
|
||||
def run_inference(
|
||||
self,
|
||||
images: list[Union[Image.Image, torch.Tensor]],
|
||||
images: list[Image.Image | torch.Tensor],
|
||||
image_mask: torch.Tensor,
|
||||
prompt: str,
|
||||
state_input: Union[list, torch.Tensor],
|
||||
state_input: list | torch.Tensor,
|
||||
return_cls_only: bool | None = None,
|
||||
action_mask: torch.Tensor | None = None,
|
||||
embodiment_ids: torch.Tensor | None = None,
|
||||
|
||||
@ -212,6 +212,7 @@ class FlowmatchingActionHead(nn.Module):
|
||||
)
|
||||
|
||||
self.action_encoder = None
|
||||
self.single_action_proj = None
|
||||
if horizon > 1:
|
||||
per_action_dim = getattr(self.config, "per_action_dim", None)
|
||||
if per_action_dim is None:
|
||||
@ -223,6 +224,8 @@ class FlowmatchingActionHead(nn.Module):
|
||||
horizon=horizon,
|
||||
num_categories=num_categories,
|
||||
)
|
||||
else:
|
||||
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@ -261,9 +264,9 @@ class FlowmatchingActionHead(nn.Module):
|
||||
|
||||
if action_mask is not None:
|
||||
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
|
||||
assert action_mask.shape == noise.shape, (
|
||||
f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
|
||||
)
|
||||
if action_mask.shape != noise.shape:
|
||||
raise ValueError(f"action_mask shape {action_mask.shape} != noise shape {noise.shape}")
|
||||
actions_gt_seq = actions_gt_seq * action_mask
|
||||
noise = noise * action_mask
|
||||
|
||||
if self.horizon > 1:
|
||||
@ -278,8 +281,8 @@ class FlowmatchingActionHead(nn.Module):
|
||||
if self.horizon > 1 and self.action_encoder is not None:
|
||||
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
|
||||
else:
|
||||
if not hasattr(self, "single_action_proj"):
|
||||
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
|
||||
if self.single_action_proj is None:
|
||||
raise RuntimeError("single_action_proj is not initialized for single-step inference/training.")
|
||||
action_tokens = self.single_action_proj(action_intermediate_seq)
|
||||
|
||||
x = action_tokens
|
||||
@ -290,9 +293,6 @@ class FlowmatchingActionHead(nn.Module):
|
||||
|
||||
if self.horizon > 1:
|
||||
x_flat = x.reshape(batch_size, -1)
|
||||
|
||||
if not hasattr(self, "seq_pool_proj"):
|
||||
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
|
||||
x_pooled = self.seq_pool_proj(x_flat)
|
||||
else:
|
||||
x_pooled = x.squeeze(1)
|
||||
@ -308,6 +308,9 @@ class FlowmatchingActionHead(nn.Module):
|
||||
embodiment_id: torch.LongTensor = None,
|
||||
action_mask: torch.Tensor = None,
|
||||
):
|
||||
if action_mask is None:
|
||||
raise ValueError("action_mask must be provided for inference with flow matching.")
|
||||
|
||||
batch_size = fused_tokens.size(0)
|
||||
device = fused_tokens.device
|
||||
if embodiment_id is None:
|
||||
@ -335,15 +338,10 @@ class FlowmatchingActionHead(nn.Module):
|
||||
action_seq = action.view(batch_size, 1, per_action_dim)
|
||||
|
||||
action_mask = action_mask.view(batch_size, 1, per_action_dim).repeat(1, self.horizon, 1)
|
||||
|
||||
if action_mask is not None:
|
||||
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
|
||||
assert action_mask.shape == action_seq.shape, (
|
||||
f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
|
||||
)
|
||||
action_seq = action_seq * action_mask
|
||||
else:
|
||||
raise ValueError("action_mask must be provided for inference with flow matching.")
|
||||
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
|
||||
if action_mask.shape != action_seq.shape:
|
||||
raise ValueError(f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}")
|
||||
action_seq = action_seq * action_mask
|
||||
|
||||
num_steps = int(getattr(self.config, "num_inference_timesteps", 32))
|
||||
dt = 1.0 / num_steps
|
||||
@ -358,11 +356,9 @@ class FlowmatchingActionHead(nn.Module):
|
||||
action_seq = action_seq * action_mask
|
||||
action_tokens = self.action_encoder(action_seq, embodiment_id)
|
||||
else:
|
||||
if hasattr(self, "single_action_proj"):
|
||||
action_tokens = self.single_action_proj(action_seq)
|
||||
else:
|
||||
self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device)
|
||||
action_tokens = self.single_action_proj(action_seq)
|
||||
if self.single_action_proj is None:
|
||||
raise RuntimeError("single_action_proj is not initialized for single-step inference/training.")
|
||||
action_tokens = self.single_action_proj(action_seq)
|
||||
|
||||
x = action_tokens
|
||||
for block in self.transformer_blocks:
|
||||
@ -382,6 +378,7 @@ class FlowmatchingActionHead(nn.Module):
|
||||
pred = self.mlp_head(x_pooled, embodiment_id)
|
||||
|
||||
action = action + dt * pred
|
||||
action = action * action_mask.view(batch_size, -1)
|
||||
|
||||
if self.horizon > 1:
|
||||
action_seq = action.view(batch_size, self.horizon, per_action_dim)
|
||||
|
||||
@ -78,28 +78,43 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
|
||||
|
||||
|
||||
class InternVL3Embedder(nn.Module):
|
||||
def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"):
|
||||
def __init__(
|
||||
self,
|
||||
model_name="OpenGVLab/InternVL3-1B",
|
||||
image_size=448,
|
||||
device="cuda",
|
||||
num_language_layers: int | None = 14,
|
||||
model_dtype: str | torch.dtype = "bfloat16",
|
||||
use_flash_attn: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.device = device
|
||||
self._requested_device = device
|
||||
self.image_size = image_size
|
||||
self.num_language_layers = num_language_layers
|
||||
self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens
|
||||
self.transform = build_transform(image_size)
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
|
||||
if isinstance(model_dtype, str):
|
||||
try:
|
||||
model_dtype = getattr(torch, model_dtype)
|
||||
except AttributeError as exc:
|
||||
raise ValueError(f"Unsupported EVO1 vlm_dtype '{model_dtype}'") from exc
|
||||
self.model = AutoModel.from_pretrained(
|
||||
model_name,
|
||||
torch_dtype=torch.bfloat16,
|
||||
torch_dtype=model_dtype,
|
||||
trust_remote_code=True,
|
||||
use_flash_attn=True,
|
||||
use_flash_attn=use_flash_attn,
|
||||
low_cpu_mem_usage=True,
|
||||
_fast_init=False,
|
||||
).to(self.device)
|
||||
).to(self._requested_device)
|
||||
|
||||
if hasattr(self.model.language_model, "model"):
|
||||
layers = self.model.language_model.model.layers
|
||||
|
||||
else:
|
||||
layers = self.model.language_model.layers
|
||||
layers = layers[:14]
|
||||
if self.num_language_layers is not None:
|
||||
layers = layers[: self.num_language_layers]
|
||||
|
||||
if hasattr(self.model.language_model, "model"):
|
||||
self.model.language_model.model.layers = torch.nn.ModuleList(layers)
|
||||
@ -110,6 +125,8 @@ class InternVL3Embedder(nn.Module):
|
||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||
self.model.vision_model.encoder.gradient_checkpointing = False
|
||||
|
||||
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
||||
|
||||
def _preprocess_images(
|
||||
self, image_tensors: list[Image.Image | torch.Tensor]
|
||||
) -> tuple[torch.Tensor, list[int]]:
|
||||
@ -132,7 +149,6 @@ class InternVL3Embedder(nn.Module):
|
||||
prompt += f"Image-{i + 1}: <image>\n"
|
||||
prompt += text_prompt.strip()
|
||||
|
||||
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
||||
for tile_count in num_tiles_list:
|
||||
token_count = self.model.num_image_token * tile_count
|
||||
image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * token_count + IMG_END_TOKEN
|
||||
@ -231,3 +247,7 @@ class InternVL3Embedder(nn.Module):
|
||||
fused_hidden = outputs.hidden_states[-1].to(torch.float32)
|
||||
|
||||
return fused_hidden[:, 0, :] if return_cls_only else fused_hidden
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return next(self.model.parameters()).device
|
||||
|
||||
@ -67,6 +67,9 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
"device": config.device,
|
||||
"return_cls_only": config.return_cls_only,
|
||||
"vlm_name": config.vlm_model_name,
|
||||
"vlm_num_layers": config.vlm_num_layers,
|
||||
"vlm_dtype": config.vlm_dtype,
|
||||
"use_flash_attn": config.use_flash_attn,
|
||||
"action_head": config.action_head,
|
||||
"action_horizon": config.chunk_size,
|
||||
"per_action_dim": config.max_action_dim,
|
||||
@ -85,10 +88,7 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
|
||||
@property
|
||||
def _camera_keys(self) -> list[str]:
|
||||
keys = list(self.config.image_features)
|
||||
if keys:
|
||||
return keys
|
||||
return []
|
||||
return list(self.config.image_features)
|
||||
|
||||
@property
|
||||
def _env_action_dim(self) -> int:
|
||||
@ -323,7 +323,7 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
reduction: str,
|
||||
) -> Tensor:
|
||||
flat_mask = action_mask.view(action_mask.shape[0], -1).to(dtype=pred_velocity.dtype)
|
||||
sq_error = ((pred_velocity * flat_mask) - target_velocity).pow(2)
|
||||
sq_error = ((pred_velocity - target_velocity) * flat_mask).pow(2)
|
||||
active = flat_mask.sum(dim=1).clamp_min(1.0)
|
||||
per_sample_loss = sq_error.sum(dim=1) / active
|
||||
if reduction == "none":
|
||||
@ -350,7 +350,8 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
action_mask=action_mask.to(device=self.config.device, dtype=self._compute_dtype),
|
||||
embodiment_ids=embodiment_ids,
|
||||
)
|
||||
target_velocity = (actions_gt - noise).view(actions_gt.shape[0], -1)
|
||||
flat_action_mask = action_mask.view(action_mask.shape[0], -1).to(dtype=actions_gt.dtype)
|
||||
target_velocity = (actions_gt - noise).view(actions_gt.shape[0], -1) * flat_action_mask
|
||||
loss = self._compute_masked_loss(pred_velocity, target_velocity, action_mask, reduction)
|
||||
loss_mean = loss.mean().item() if loss.ndim > 0 else loss.item()
|
||||
return loss, {
|
||||
@ -371,7 +372,11 @@ class EVO1Policy(PreTrainedPolicy):
|
||||
embodiment_ids = self._get_embodiment_ids(batch, states.shape[0])
|
||||
action_mask = self._prepare_inference_action_mask(states.shape[0])
|
||||
|
||||
with torch.autocast(device_type="cuda", dtype=torch.bfloat16) if self.config.use_amp and str(self.config.device).startswith("cuda") else nullcontext():
|
||||
with (
|
||||
torch.autocast(device_type="cuda", dtype=torch.bfloat16)
|
||||
if self.config.use_amp and str(self.config.device).startswith("cuda")
|
||||
else nullcontext()
|
||||
):
|
||||
actions = self.model(
|
||||
fused_tokens,
|
||||
state=states,
|
||||
|
||||
在新工单中引用
屏蔽一个用户