镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
support Evo1 policy (#11)
* feat(evo1): integrate evo1 with lerobot training * chore(evo1): drop v21 export and local tests * docs(evo1): add setup notes and keep only 4gpu config * chore(evo1): clean up quality issues * fix(evo1): use batch_size consistently * docs(evo1): add handoff notes * docs(evo1): clarify resume pretrain semantics * fix(evo1): address review follow-ups * docs(evo1): refresh inherited issues note * fix(evo1): update Evo1 model to higher effience * fix(evo1): GPU version of dynamic process and ratio cache * fix(evo1): support stage2 checkpoint resume --------- Co-authored-by: t0mo0n <2423843425@qq.com>
这个提交包含在:
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fd4a3d6d0266205fe3f725f048645be66592c23564db22e08b0807d5569b2c64
|
||||
size 607384
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9ee6e1d67c0241f42a0dbaf5c676d19230375ed078ee1943bb8c4c37ab303d14
|
||||
size 447784
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0d37675b699b643f2e15348de99aed535fac10dc5f09a79b8e4b0acfaf4e0378
|
||||
size 3658384
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e8ae51f17c0c7f6d67b753922e2cc269f5e8372a65734f859231639a64fe8fad
|
||||
size 1845784
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:efa33022976218f246f736a6133494b5d0e5486e8b50b52900c42f285ae85558
|
||||
size 877984
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a1c69d1c7f6f8c60e2c82331c0f444725200ed4053d3e5ec23aa82cb5db5b29f
|
||||
size 833684
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:88374973bc3bd9e7389b5853cd6e555af9731c30b658b1ee6f85be34a126cb92
|
||||
size 58884
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a93c1e3fc245d9d336a427fcc506cb1fc15615268053392b249816f90c011e50
|
||||
size 8273
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0861e122e406f891a57a550551ad1ce0ef4ca5babf58f476cf8914395e144319
|
||||
size 162084
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e78dee99bc84857c09183463f3fb3fdea292ea7570f1bfb0001a79674bb69d45
|
||||
size 2323489
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:97e9079580af6e423317c33b2a0f4fe0f6dbc67c3e1d9e6f4836c87c6c8f34a2
|
||||
size 14480060
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:db5b39eef745655a67e57870d8ee56f26a03651d2161635ca2ec3c58c9269907
|
||||
size 5098355
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ccc08d3ef1cb8bdfb8ab5395ff65893d13386ad526d7bc954d468b7dc885d557
|
||||
size 2037373
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:78718679ffdf9d81130047c4267ed66a081315eff36f216833bc3781c6241417
|
||||
size 2155609
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:486dddcd07f072b2e3978087dc2f9b641a91722bccb1bcb3421c4c7ce64a39ed
|
||||
size 1001764
|
||||
@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d6dc6d31b0fa56341b2323fee5e8a375c2500ae6fb02e31fb52e8e917b608439
|
||||
size 8169
|
||||
@ -161,8 +161,25 @@ class PreTrainedConfig(draccus.ChoiceRegistry, HubMixin, abc.ABC): # type: igno
|
||||
return None
|
||||
|
||||
def _save_pretrained(self, save_directory: Path) -> None:
|
||||
with open(save_directory / CONFIG_NAME, "w") as f, draccus.config_type("json"):
|
||||
draccus.dump(self, f, indent=4)
|
||||
with tempfile.NamedTemporaryFile("w+", delete=False, suffix=".json") as tmp, draccus.config_type("json"):
|
||||
draccus.dump(self, tmp, indent=4)
|
||||
tmp_path = tmp.name
|
||||
|
||||
with open(tmp_path) as f:
|
||||
config = json.load(f)
|
||||
|
||||
if "type" not in config:
|
||||
matching_choices = [
|
||||
name for name in self.get_known_choices() if self.get_choice_class(name) is self.__class__
|
||||
]
|
||||
if len(matching_choices) != 1:
|
||||
raise ValueError(
|
||||
f"Expected exactly one registered choice for {self.__class__.__name__}, got {matching_choices}"
|
||||
)
|
||||
config["type"] = matching_choices[0]
|
||||
|
||||
with open(save_directory / CONFIG_NAME, "w") as f:
|
||||
json.dump(config, f, indent=4)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
@ -207,16 +224,34 @@ class PreTrainedConfig(draccus.ChoiceRegistry, HubMixin, abc.ABC): # type: igno
|
||||
# apply cli overrides.
|
||||
# This is very ugly, ideally we'd like to be able to do that natively with draccus
|
||||
# something like --policy.path (in addition to --policy.type)
|
||||
with draccus.config_type("json"):
|
||||
orig_config = draccus.parse(cls, config_file, args=[])
|
||||
|
||||
if config_file is None:
|
||||
raise FileNotFoundError(f"{CONFIG_NAME} not found in {model_id}")
|
||||
|
||||
with open(config_file) as f:
|
||||
config = json.load(f)
|
||||
|
||||
config.pop("type")
|
||||
if "type" in config:
|
||||
with draccus.config_type("json"):
|
||||
orig_config = draccus.parse(cls, config_file, args=[])
|
||||
else:
|
||||
parse_successes = []
|
||||
for choice_name in cls.get_known_choices():
|
||||
config_cls = cls.get_choice_class(choice_name)
|
||||
try:
|
||||
with draccus.config_type("json"):
|
||||
parsed_config = draccus.parse(config_cls, config_file, args=[])
|
||||
except Exception:
|
||||
continue
|
||||
parse_successes.append((choice_name, parsed_config))
|
||||
|
||||
if len(parse_successes) != 1:
|
||||
raise ValueError(
|
||||
f"Expected to infer exactly one policy type from {config_file}, got "
|
||||
f"{[choice for choice, _ in parse_successes]}"
|
||||
)
|
||||
orig_config = parse_successes[0][1]
|
||||
|
||||
config.pop("type", None)
|
||||
with tempfile.NamedTemporaryFile("w+", delete=False, suffix=".json") as f:
|
||||
json.dump(config, f)
|
||||
config_file = f.name
|
||||
|
||||
@ -54,6 +54,9 @@ class TrainPipelineConfig(HubMixin):
|
||||
# Note that when resuming a run, the default behavior is to use the configuration from the checkpoint,
|
||||
# regardless of what's provided with the training command at the time of resumption.
|
||||
resume: bool = False
|
||||
# `resume_pretrain` mirrors Evo-1's stage2 handoff semantics: initialize from policy weights only while
|
||||
# starting a fresh training run with a new optimizer, scheduler and step counter.
|
||||
resume_pretrain: bool = False
|
||||
# `seed` is used for training (eg: model initialization, dataset shuffling)
|
||||
# AND for the evaluation environments.
|
||||
seed: int | None = 1000
|
||||
@ -89,6 +92,16 @@ class TrainPipelineConfig(HubMixin):
|
||||
def validate(self) -> None:
|
||||
# HACK: We parse again the cli args here to get the pretrained paths if there was some.
|
||||
policy_path = parser.get_path_arg("policy")
|
||||
if self.resume and self.resume_pretrain:
|
||||
raise ValueError("'resume' and 'resume_pretrain' are mutually exclusive.")
|
||||
|
||||
if self.resume_pretrain and not policy_path and (
|
||||
self.policy is None or self.policy.pretrained_path is None
|
||||
):
|
||||
raise ValueError(
|
||||
"'resume_pretrain=true' expects weights via '--policy.path' or 'policy.pretrained_path'."
|
||||
)
|
||||
|
||||
if policy_path:
|
||||
# Only load the policy config
|
||||
cli_overrides = parser.get_cli_overrides("policy")
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
|
||||
from .act.configuration_act import ACTConfig as ACTConfig
|
||||
from .diffusion.configuration_diffusion import DiffusionConfig as DiffusionConfig
|
||||
from .evo1.configuration_evo1 import Evo1Config as Evo1Config
|
||||
from .groot.configuration_groot import GrootConfig as GrootConfig
|
||||
from .pi0.configuration_pi0 import PI0Config as PI0Config
|
||||
from .pi0_fast.configuration_pi0_fast import PI0FastConfig as PI0FastConfig
|
||||
@ -28,6 +29,7 @@ from .xvla.configuration_xvla import XVLAConfig as XVLAConfig
|
||||
__all__ = [
|
||||
"ACTConfig",
|
||||
"DiffusionConfig",
|
||||
"Evo1Config",
|
||||
"PI0Config",
|
||||
"PI05Config",
|
||||
"PI0FastConfig",
|
||||
|
||||
@ -0,0 +1,51 @@
|
||||
# Evo1 Known Issues
|
||||
|
||||
## 尚未完善的部分
|
||||
|
||||
- 多 embodiment 训练目前还没有完整打通。
|
||||
- 模型侧保留了 `embodiment_id`、`num_categories`、`default_embodiment_id` 这些接口。
|
||||
- 但当前常用数据集没有稳定的 `embodiment_id` 字段,训练时实际上仍然会回落到单一 `default_embodiment_id=0`。
|
||||
- 如果后续要做多机器人/多 embodiment 联训,还需要同时补齐:
|
||||
- 数据集里的稳定 `embodiment_id` 来源
|
||||
- 多 embodiment 的归一化统计,而不是单份聚合 stats
|
||||
- training config 层面对多数据源联训的明确支持
|
||||
|
||||
- 目前接入主要面向 Evo-RL / LeRobot 风格的 checkpoint。
|
||||
- 也就是带有 `config.json`、`model.safetensors`、`policy_preprocessor.json`、`policy_postprocessor.json` 的目录。
|
||||
- 原始 Evo1 / DeepSpeed checkpoint 没有直接作为 Evo-RL 训练续训格式来兼容。
|
||||
|
||||
- 当前验证过、最稳定的训练路径是 4 卡 bf16。
|
||||
- 目录里只保留了四卡 accelerate 配置。
|
||||
- 单卡、双卡虽然理论上可以再配,但当前没有作为维护目标继续保留。
|
||||
|
||||
- 图像增强路径当前不作为默认推荐配置。
|
||||
- 已验证训练命令里默认使用 `dataset.image_transforms.enable=false`。
|
||||
- 原因不是 Evo1 算法本身依赖关闭增强,而是当前实验环境里的 transforms 路径没有作为稳定组合继续维护。
|
||||
|
||||
## 相比原始 Evo1 的工程差异
|
||||
|
||||
- 训练入口不再走原始 Evo1 的“大训练脚本”。
|
||||
- 现在是把 Evo1 接到了 LeRobot 统一训练框架里,通过 `lerobot_train` 启动。
|
||||
- `stage1/stage2`、checkpoint 保存、processor、dataset 读取、日志和训练配置都尽量复用 LeRobot 现有基础设施。
|
||||
|
||||
- `stage1 -> stage2` 的衔接方式改成了 Evo-RL 里的 `resume_pretrain=true` 语义。
|
||||
- 含义是:只加载权重,不恢复 optimizer / scheduler / global step。
|
||||
- 这和原始 Evo1 的“两阶段训练切换”意图是一致的,但实现方式更贴近 LeRobot 的配置体系。
|
||||
|
||||
- 混合精度执行边界比原始 Evo1 更统一。
|
||||
- 原始 Evo1 里,VLM embedding 和后面的 action head 前向不在同一个统一的训练 AMP 作用域里。
|
||||
- 现在 Evo-RL 版是在更大的 `accelerator.autocast()` 范围中执行整个 policy forward。
|
||||
- 这属于工程上的规整化,主要影响的是执行路径、速度和数值边界;在 `stage2` finetune VLM 时尤其值得注意。
|
||||
|
||||
- 分布式训练实现不同。
|
||||
- 原始 Evo1 训练主要依赖 DeepSpeed。
|
||||
- 现在的 Evo-RL 接入版默认走 Accelerate bf16 配置。
|
||||
- 这不改变 Evo1 的核心模型结构和两阶段训练思路,但会让工程实现、混精边界、分布式包装方式与原始仓库有所不同。
|
||||
|
||||
- processor / preprocessor / postprocessor 被纳入了 LeRobot 的标准保存格式。
|
||||
- 这样推理、录制和训练可以复用同一套加载接口。
|
||||
- 也意味着 Evo1 不再只是一套训练脚本,而是完整地成为 LeRobot policy registry 里的一个 policy。
|
||||
|
||||
- 数据读取不再依赖原始 Evo1 自己的大型 dataset 训练脚本。
|
||||
- 当前训练直接复用 LeRobot 数据集接口。
|
||||
- 为了对照复现原始 Evo1 时,额外写过一个 v3.0 数据到原始 Evo1 loader 可读格式的适配脚本;但这条适配链不是 Evo-RL 日常训练的默认路径。
|
||||
112
src/lerobot/policies/evo1/README.md
普通文件
112
src/lerobot/policies/evo1/README.md
普通文件
@ -0,0 +1,112 @@
|
||||
# EVO1 Notes
|
||||
|
||||
This directory contains the Evo1 policy integration for LeRobot/Evo-RL.
|
||||
|
||||
## Extra Dependencies
|
||||
|
||||
Compared with the default LeRobot policy stack, Evo1 usually needs these extras:
|
||||
|
||||
- `flash-attn`
|
||||
- InternVL3 model weights
|
||||
- `transformers` support for the InternVL3 remote code path
|
||||
|
||||
The most important compatibility constraint is:
|
||||
|
||||
- `flash-attn` must match your `torch` version, CUDA version, Python version, and wheel ABI.
|
||||
|
||||
## Known Working Combinations
|
||||
|
||||
Validated in this repo / recent experiments:
|
||||
|
||||
- Evo-RL A800 environment:
|
||||
- `torch==2.4.1+cu118`
|
||||
- `flash_attn==2.6.3`
|
||||
- Original Evo1-style environment discussed during reproduction:
|
||||
- `python==3.10`
|
||||
- `torch==2.5.1`
|
||||
- `torchvision==0.20.1`
|
||||
- `transformers==4.39.0`
|
||||
- `flash-attn==2.8.3`
|
||||
## Training Notes
|
||||
|
||||
- `training_stage=stage1` freezes the VLM and trains the action head.
|
||||
- `training_stage=stage2` finetunes both the VLM and the action head.
|
||||
- In this repo, `resume_pretrain=true` is the Evo1-style stage handoff: load weights only, but start a fresh optimizer/scheduler state.
|
||||
- We currently recommend `dataset.image_transforms.enable=false` in the validated training commands because the image augmentation pipeline was not stable in the tested environment.
|
||||
|
||||
## Stage 1 Command
|
||||
|
||||
Example: 4 GPUs, effective batch size `16` (`4 x 4`), `5000` steps, save every `2500`.
|
||||
|
||||
```bash
|
||||
cd /root/private_data/code/Evo-RL
|
||||
PYTHONPATH=src /opt/conda/bin/accelerate launch \
|
||||
--config_file src/lerobot/policies/evo1/accelerate_four_gpu_bf16.yaml \
|
||||
-m lerobot.scripts.lerobot_train \
|
||||
--dataset.repo_id=Elvinky/bi_so101_fold_towel_d0 \
|
||||
--dataset.root=/root/private_data/data/bi_so101_fold_towel_d0 \
|
||||
--dataset.image_transforms.enable=false \
|
||||
--policy.type=evo1 \
|
||||
--policy.training_stage=stage1 \
|
||||
--policy.vlm_model_name=/root/private_data/models/InternVL3-1B \
|
||||
--policy.device=cuda \
|
||||
--policy.push_to_hub=false \
|
||||
--policy.dropout=0.2 \
|
||||
--policy.optimizer_lr=1e-5 \
|
||||
--policy.optimizer_weight_decay=1e-3 \
|
||||
--policy.optimizer_grad_clip_norm=1.0 \
|
||||
--policy.scheduler_warmup_steps=1000 \
|
||||
--policy.num_layers=8 \
|
||||
--policy.chunk_size=50 \
|
||||
--policy.n_action_steps=50 \
|
||||
--policy.max_action_dim=24 \
|
||||
--policy.max_state_dim=24 \
|
||||
--batch_size=4 \
|
||||
--num_workers=4 \
|
||||
--steps=5000 \
|
||||
--log_freq=10 \
|
||||
--save_checkpoint=true \
|
||||
--save_freq=2500 \
|
||||
--eval_freq=0 \
|
||||
--wandb.enable=false \
|
||||
--output_dir=/root/private_data/code/Evo-RL/outputs/train/evo1_stage1
|
||||
```
|
||||
|
||||
## Stage 2 Command
|
||||
|
||||
Example: start stage2 from the stage1 `5000`-step checkpoint, `80000` steps, save every `10000`.
|
||||
|
||||
```bash
|
||||
cd /root/private_data/code/Evo-RL
|
||||
PYTHONPATH=src /opt/conda/bin/accelerate launch \
|
||||
--config_file src/lerobot/policies/evo1/accelerate_four_gpu_bf16.yaml \
|
||||
-m lerobot.scripts.lerobot_train \
|
||||
--dataset.repo_id=Elvinky/bi_so101_fold_towel_d0 \
|
||||
--dataset.root=/root/private_data/data/bi_so101_fold_towel_d0 \
|
||||
--dataset.image_transforms.enable=false \
|
||||
--policy.path=/root/private_data/code/Evo-RL/outputs/train/evo1_stage1/checkpoints/005000/pretrained_model \
|
||||
--policy.training_stage=stage2 \
|
||||
--resume_pretrain=true \
|
||||
--policy.vlm_model_name=/root/private_data/models/InternVL3-1B \
|
||||
--policy.device=cuda \
|
||||
--policy.push_to_hub=false \
|
||||
--policy.dropout=0.2 \
|
||||
--policy.optimizer_lr=1e-5 \
|
||||
--policy.optimizer_weight_decay=1e-3 \
|
||||
--policy.optimizer_grad_clip_norm=1.0 \
|
||||
--policy.scheduler_warmup_steps=1000 \
|
||||
--policy.num_layers=8 \
|
||||
--policy.chunk_size=50 \
|
||||
--policy.n_action_steps=50 \
|
||||
--policy.max_action_dim=24 \
|
||||
--policy.max_state_dim=24 \
|
||||
--batch_size=4 \
|
||||
--num_workers=4 \
|
||||
--steps=80000 \
|
||||
--log_freq=10 \
|
||||
--save_checkpoint=true \
|
||||
--save_freq=10000 \
|
||||
--eval_freq=0 \
|
||||
--wandb.enable=false \
|
||||
--output_dir=/root/private_data/code/Evo-RL/outputs/train/evo1_stage2
|
||||
```
|
||||
@ -0,0 +1,5 @@
|
||||
from .configuration_evo1 import Evo1Config
|
||||
from .modeling_evo1 import EVO1Policy
|
||||
from .processor_evo1 import make_evo1_pre_post_processors
|
||||
|
||||
__all__ = ["Evo1Config", "EVO1Policy", "make_evo1_pre_post_processors"]
|
||||
@ -0,0 +1,16 @@
|
||||
compute_environment: LOCAL_MACHINE
|
||||
debug: false
|
||||
distributed_type: MULTI_GPU
|
||||
downcast_bf16: "no"
|
||||
enable_cpu_affinity: false
|
||||
machine_rank: 0
|
||||
main_training_function: main
|
||||
mixed_precision: bf16
|
||||
num_machines: 1
|
||||
num_processes: 4
|
||||
rdzv_backend: static
|
||||
same_network: true
|
||||
tpu_env: []
|
||||
tpu_use_cluster: false
|
||||
tpu_use_sudo: false
|
||||
use_cpu: false
|
||||
@ -0,0 +1,155 @@
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from torch.optim import Optimizer
|
||||
from torch.optim.lr_scheduler import LambdaLR
|
||||
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
|
||||
from lerobot.optim.optimizers import AdamWConfig
|
||||
from lerobot.optim.schedulers import LRSchedulerConfig
|
||||
from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE
|
||||
|
||||
|
||||
@LRSchedulerConfig.register_subclass("evo1_exact")
|
||||
@dataclass
|
||||
class Evo1SchedulerConfig(LRSchedulerConfig):
|
||||
num_warmup_steps: int
|
||||
|
||||
def build(self, optimizer: Optimizer, num_training_steps: int) -> LambdaLR:
|
||||
def lr_lambda(current_step: int) -> float:
|
||||
if current_step < self.num_warmup_steps:
|
||||
return current_step / max(1, self.num_warmup_steps)
|
||||
progress = (current_step - self.num_warmup_steps) / max(
|
||||
1, num_training_steps - self.num_warmup_steps
|
||||
)
|
||||
return max(0.0, 0.5 * (1.0 + math.cos(math.pi * progress)))
|
||||
|
||||
return LambdaLR(optimizer, lr_lambda, -1)
|
||||
|
||||
|
||||
@PreTrainedConfig.register_subclass("evo1")
|
||||
@dataclass
|
||||
class Evo1Config(PreTrainedConfig):
|
||||
training_stage: str = "stage1"
|
||||
use_amp: bool = True
|
||||
|
||||
n_obs_steps: int = 1
|
||||
chunk_size: int = 50
|
||||
n_action_steps: int = 50
|
||||
|
||||
max_state_dim: int = 24
|
||||
max_action_dim: int = 24
|
||||
max_views: int = 3
|
||||
image_resolution: tuple[int, int] = (448, 448)
|
||||
empty_cameras: int = 0
|
||||
|
||||
normalization_mapping: dict[str, NormalizationMode] = field(
|
||||
default_factory=lambda: {
|
||||
"VISUAL": NormalizationMode.IDENTITY,
|
||||
"STATE": NormalizationMode.MIN_MAX,
|
||||
"ACTION": NormalizationMode.MIN_MAX,
|
||||
}
|
||||
)
|
||||
|
||||
vlm_model_name: str = "OpenGVLab/InternVL3-1B"
|
||||
vlm_num_layers: int | None = 14
|
||||
vlm_dtype: str = "bfloat16"
|
||||
use_flash_attn: bool = True
|
||||
action_head: str = "flowmatching"
|
||||
embed_dim: int = 896
|
||||
hidden_dim: int = 1024
|
||||
state_hidden_dim: int = 1024
|
||||
num_heads: int = 8
|
||||
num_layers: int = 8
|
||||
dropout: float = 0.0
|
||||
num_inference_timesteps: int = 32
|
||||
num_categories: int = 1
|
||||
return_cls_only: bool = False
|
||||
|
||||
finetune_vlm: bool = False
|
||||
finetune_action_head: bool = True
|
||||
|
||||
task_field: str = "task"
|
||||
embodiment_id_field: str | None = None
|
||||
default_embodiment_id: int = 0
|
||||
|
||||
optimizer_lr: float = 1e-5
|
||||
optimizer_betas: tuple[float, float] = (0.9, 0.999)
|
||||
optimizer_eps: float = 1e-8
|
||||
optimizer_weight_decay: float = 1e-5
|
||||
optimizer_grad_clip_norm: float = 1.0
|
||||
|
||||
scheduler_warmup_steps: int = 300
|
||||
drop_last: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.training_stage not in {"stage1", "stage2"}:
|
||||
raise ValueError(
|
||||
f"Unsupported EVO1 training_stage '{self.training_stage}', expected 'stage1' or 'stage2'"
|
||||
)
|
||||
|
||||
if self.training_stage == "stage1":
|
||||
self.finetune_vlm = False
|
||||
self.finetune_action_head = True
|
||||
elif self.training_stage == "stage2":
|
||||
self.finetune_vlm = True
|
||||
self.finetune_action_head = True
|
||||
|
||||
if self.n_action_steps > self.chunk_size:
|
||||
raise ValueError(
|
||||
f"n_action_steps ({self.n_action_steps}) must be <= chunk_size ({self.chunk_size})"
|
||||
)
|
||||
|
||||
def validate_features(self) -> None:
|
||||
if self.input_features is None:
|
||||
self.input_features = {}
|
||||
if self.output_features is None:
|
||||
self.output_features = {}
|
||||
|
||||
for i in range(self.empty_cameras):
|
||||
key = OBS_IMAGES + f".empty_camera_{i}"
|
||||
if key not in self.input_features:
|
||||
self.input_features[key] = PolicyFeature(
|
||||
type=FeatureType.VISUAL,
|
||||
shape=(3, *self.image_resolution),
|
||||
)
|
||||
|
||||
if OBS_STATE not in self.input_features:
|
||||
self.input_features[OBS_STATE] = PolicyFeature(
|
||||
type=FeatureType.STATE,
|
||||
shape=(self.max_state_dim,),
|
||||
)
|
||||
|
||||
if ACTION not in self.output_features:
|
||||
self.output_features[ACTION] = PolicyFeature(
|
||||
type=FeatureType.ACTION,
|
||||
shape=(self.max_action_dim,),
|
||||
)
|
||||
|
||||
def get_optimizer_preset(self) -> AdamWConfig:
|
||||
return AdamWConfig(
|
||||
lr=self.optimizer_lr,
|
||||
betas=self.optimizer_betas,
|
||||
eps=self.optimizer_eps,
|
||||
weight_decay=self.optimizer_weight_decay,
|
||||
grad_clip_norm=self.optimizer_grad_clip_norm,
|
||||
)
|
||||
|
||||
def get_scheduler_preset(self):
|
||||
return Evo1SchedulerConfig(
|
||||
num_warmup_steps=self.scheduler_warmup_steps,
|
||||
)
|
||||
|
||||
@property
|
||||
def observation_delta_indices(self) -> list[int]:
|
||||
return [0]
|
||||
|
||||
@property
|
||||
def action_delta_indices(self) -> list[int]:
|
||||
return list(range(self.chunk_size))
|
||||
|
||||
@property
|
||||
def reward_delta_indices(self) -> None:
|
||||
return None
|
||||
@ -0,0 +1,84 @@
|
||||
# Evo1 原始实现继承问题
|
||||
|
||||
这份文档记录的是:在 Evo-RL 接入 Evo1 过程中发现、但对照原始 Evo1 仓库后确认**并非迁移新引入**,而是原始 Evo1 本身已有的实现问题或工程限制。
|
||||
|
||||
## 已在当前 Evo-RL 接入层修复
|
||||
|
||||
下面这些问题最初来自原始 Evo1,但已经在当前 Evo-RL 端口里修掉,不再属于当前分支的残留问题:
|
||||
|
||||
### 1. `get_action()` 中 `action_mask` 判空顺序不安全
|
||||
|
||||
- 原始 Evo1:先 `action_mask.view(...)`,后做 `None` 检查。
|
||||
- 当前端口:已改成先判空,再处理 shape/view。
|
||||
|
||||
### 2. masked loss / target velocity 在一般 mask 场景下不严谨
|
||||
|
||||
- 原始 Evo1:
|
||||
- `target_velocity = actions_gt - noise`
|
||||
- `loss = MSE(pred_velocity * mask, target_velocity)`
|
||||
- 当前端口:
|
||||
- loss 改成先算 `(pred_velocity - target_velocity)` 再乘 mask;
|
||||
- `target_velocity` 和训练分支里的 `actions_gt_seq` 也会显式跟随 mask。
|
||||
|
||||
### 3. `forward/get_action` 中动态创建 `single_action_proj`
|
||||
|
||||
- 原始 Evo1:按需在前向里构造 `nn.Linear`。
|
||||
- 当前端口:改成在 `__init__` 中一次性定义,避免参数集合在运行时变化。
|
||||
|
||||
### 4. InternVL3 层数硬编码为 14
|
||||
|
||||
- 原始 Evo1:语言模型层数硬编码截断到 14。
|
||||
- 当前端口:改成配置项 `vlm_num_layers`,默认仍保持 14。
|
||||
|
||||
### 5. `use_flash_attn=True` 与 `torch.bfloat16` 硬编码
|
||||
|
||||
- 原始 Evo1:直接写死。
|
||||
- 当前端口:改成配置项 `use_flash_attn` 和 `vlm_dtype`。
|
||||
|
||||
### 6. 其他已顺手修掉的工程性问题
|
||||
|
||||
- 删除了原始 Evo1 里永假的 `action_dim != horizon * per_action_dim` 检查。
|
||||
- `InternVL3Embedder` 不再把 `device` 固定保存成字符串语义,而是运行时从模型参数读取实际 device。
|
||||
|
||||
## 相较原始 Evo1 的性能优化(已落地)
|
||||
|
||||
### 1. VLM 图文融合改为 batch 处理(主要收益)
|
||||
|
||||
- 由逐样本循环调用改为整批处理,减少 Python 循环和重复前向开销,提升吞吐。
|
||||
- 优化位置:
|
||||
- `src/lerobot/policies/evo1/modeling_evo1.py`:`_compute_fused_tokens`
|
||||
- `src/lerobot/policies/evo1/evo1_model.py`:`get_vl_embeddings`
|
||||
- `src/lerobot/policies/evo1/internvl3_embedder.py`:`_build_multimodal_prompt`、`_prepare_and_fuse_embeddings`、`get_fused_image_text_embedding_from_tensor_images`
|
||||
|
||||
### 2. 图像预处理避免不必要的 PIL 转换(主要收益)
|
||||
|
||||
- 对已是 `[3, image_size, image_size]` 的 Tensor 走纯 Tensor 快路径,直接归一化,不再强制 Tensor->PIL->Tensor。
|
||||
- 优化位置:
|
||||
- `src/lerobot/policies/evo1/internvl3_embedder.py`:`_preprocess_images`
|
||||
|
||||
### 3. 次要优化(收益次于以上两项)
|
||||
|
||||
- `repeat` 改 `expand`/广播,减少中间张量分配。
|
||||
- 位置:`src/lerobot/policies/evo1/flow_matching.py`:`MultiEmbodimentActionEncoder.forward`
|
||||
- 推理阶段时间嵌入预计算,减少循环内重复计算。
|
||||
- 位置:`src/lerobot/policies/evo1/flow_matching.py`:`FlowmatchingActionHead.get_action`
|
||||
- 强化 dtype/device 对齐,减少隐式 cast。
|
||||
- 位置:`src/lerobot/policies/evo1/flow_matching.py`、`src/lerobot/policies/evo1/internvl3_embedder.py`
|
||||
|
||||
## 当前仍保留的原始 Evo1 遗留问题
|
||||
|
||||
下面这些问题在当前 Evo-RL 端口里仍然基本沿用原始 Evo1 的实现,后续如果要继续打磨,建议由 Evo1 本体维护侧统一评估:
|
||||
|
||||
### 1. 配置链较绕
|
||||
|
||||
- 现象:`dict -> SimpleNamespace` 形式在模型层继续传递配置。
|
||||
- 影响:
|
||||
- 配置可读性和可追踪性一般。
|
||||
- 归因:原始 Evo1 即如此实现。
|
||||
- 建议:后续可考虑直接传更强约束的配置对象。
|
||||
|
||||
## 说明
|
||||
|
||||
以上问题是 Evo-RL 接入时发现的“原始实现继承问题”,不等同于本次迁移引入的行为不一致。
|
||||
|
||||
当前 Evo-RL 接入侧已经优先修掉了会影响训练正确性或接口健壮性的部分;剩余项更多是性能与工程可维护性问题。
|
||||
@ -0,0 +1,207 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from PIL import Image
|
||||
|
||||
from lerobot.policies.evo1.flow_matching import FlowmatchingActionHead
|
||||
from lerobot.policies.evo1.internvl3_embedder import InternVL3Embedder
|
||||
|
||||
|
||||
class EVO1(nn.Module):
|
||||
def __init__(self, config: dict):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self._device = config.get("device", "cuda")
|
||||
self.return_cls_only = config.get("return_cls_only", False)
|
||||
vlm_name = config.get("vlm_name", "OpenGVLab/InternVL3-1B")
|
||||
self.embedder = InternVL3Embedder(
|
||||
model_name=vlm_name,
|
||||
image_size=config.get("image_size", 448),
|
||||
device=self._device,
|
||||
num_language_layers=config.get("vlm_num_layers", 14),
|
||||
model_dtype=config.get("vlm_dtype", "bfloat16"),
|
||||
use_flash_attn=config.get("use_flash_attn", True),
|
||||
)
|
||||
|
||||
action_head_type = config.get("action_head", "flowmatching").lower()
|
||||
if action_head_type != "flowmatching":
|
||||
raise NotImplementedError(f"Unknown action_head: {action_head_type}")
|
||||
|
||||
horizon = config.get("action_horizon", config.get("horizon", 16))
|
||||
per_action_dim = config.get("per_action_dim", 7)
|
||||
action_dim = horizon * per_action_dim
|
||||
|
||||
config["horizon"] = horizon
|
||||
config["per_action_dim"] = per_action_dim
|
||||
config["action_dim"] = action_dim
|
||||
|
||||
self.horizon = horizon
|
||||
self.per_action_dim = per_action_dim
|
||||
self.action_head = FlowmatchingActionHead(
|
||||
config=SimpleNamespace(
|
||||
embed_dim=config.get("embed_dim", 896),
|
||||
hidden_dim=config.get("hidden_dim", 1024),
|
||||
action_dim=action_dim,
|
||||
horizon=horizon,
|
||||
per_action_dim=per_action_dim,
|
||||
state_dim=config.get("state_dim", 7),
|
||||
state_hidden_dim=config.get("state_hidden_dim", 1024),
|
||||
num_heads=config.get("num_heads", 8),
|
||||
num_layers=config.get("num_layers", 8),
|
||||
dropout=config.get("dropout", 0.0),
|
||||
num_inference_timesteps=config.get("num_inference_timesteps", 50),
|
||||
num_categories=config.get("num_categories", 1),
|
||||
)
|
||||
).to(self._device)
|
||||
|
||||
def get_vl_embeddings(
|
||||
self,
|
||||
images: list[list[Image.Image | torch.Tensor]],
|
||||
image_mask: torch.Tensor,
|
||||
prompt: str | list[str] = "",
|
||||
return_cls_only: bool | None = None,
|
||||
) -> torch.Tensor:
|
||||
if return_cls_only is None:
|
||||
return_cls_only = self.return_cls_only
|
||||
|
||||
if not images:
|
||||
raise ValueError("EVO1 expects at least one batch item.")
|
||||
if not isinstance(images[0], list):
|
||||
raise TypeError("EVO1 expects batched images as list[list[Image|Tensor]].")
|
||||
|
||||
batch_size = len(images)
|
||||
if image_mask.ndim == 1:
|
||||
image_mask = image_mask.unsqueeze(0)
|
||||
if image_mask.ndim != 2:
|
||||
raise ValueError(f"EVO1 expects image_mask with shape [B, V], got {tuple(image_mask.shape)}")
|
||||
if image_mask.shape[0] != batch_size:
|
||||
raise ValueError(
|
||||
f"EVO1 batch mismatch: num_images_batch={batch_size} image_mask_batch={image_mask.shape[0]}"
|
||||
)
|
||||
|
||||
if isinstance(prompt, list) and len(prompt) != batch_size:
|
||||
raise ValueError(
|
||||
f"EVO1 batch mismatch: num_prompts={len(prompt)} num_images_batch={batch_size}"
|
||||
)
|
||||
|
||||
return self.embedder.get_fused_image_text_embedding_from_tensor_images(
|
||||
image_tensors=images,
|
||||
image_mask=image_mask,
|
||||
text_prompt=prompt,
|
||||
return_cls_only=return_cls_only,
|
||||
)
|
||||
|
||||
def prepare_state(self, state_input: list | torch.Tensor) -> torch.Tensor:
|
||||
if isinstance(state_input, list):
|
||||
state_tensor = torch.tensor(state_input)
|
||||
elif isinstance(state_input, torch.Tensor):
|
||||
state_tensor = state_input
|
||||
else:
|
||||
raise TypeError(f"Unsupported state input type: {type(state_input)}")
|
||||
|
||||
if state_tensor.ndim == 1:
|
||||
state_tensor = state_tensor.unsqueeze(0)
|
||||
elif state_tensor.ndim == 3:
|
||||
state_tensor = state_tensor[:, -1]
|
||||
elif state_tensor.ndim != 2:
|
||||
raise ValueError(f"EVO1 expects state tensor with shape [B, D], got {tuple(state_tensor.shape)}")
|
||||
|
||||
return state_tensor.to(self._device)
|
||||
|
||||
def predict_action(
|
||||
self,
|
||||
fused_tokens: torch.Tensor,
|
||||
state: torch.Tensor | None,
|
||||
actions_gt: torch.Tensor | None = None,
|
||||
action_mask: torch.Tensor | None = None,
|
||||
embodiment_ids: torch.Tensor | None = None,
|
||||
):
|
||||
batch_size = fused_tokens.shape[0]
|
||||
|
||||
if state is not None and state.shape[0] != batch_size:
|
||||
raise ValueError(f"EVO1 batch mismatch: state_batch={state.shape[0]} fused_batch={batch_size}")
|
||||
if actions_gt is not None and actions_gt.shape[0] != batch_size:
|
||||
raise ValueError(f"EVO1 batch mismatch: actions_gt_batch={actions_gt.shape[0]} fused_batch={batch_size}")
|
||||
if action_mask is not None and action_mask.shape[0] != batch_size:
|
||||
raise ValueError(f"EVO1 batch mismatch: action_mask_batch={action_mask.shape[0]} fused_batch={batch_size}")
|
||||
|
||||
if embodiment_ids is not None:
|
||||
if embodiment_ids.ndim == 0:
|
||||
embodiment_ids = embodiment_ids.unsqueeze(0)
|
||||
elif embodiment_ids.ndim > 1:
|
||||
embodiment_ids = embodiment_ids[:, -1]
|
||||
if embodiment_ids.shape[0] != batch_size:
|
||||
raise ValueError(
|
||||
f"EVO1 batch mismatch: embodiment_ids_batch={embodiment_ids.shape[0]} fused_batch={batch_size}"
|
||||
)
|
||||
|
||||
if actions_gt is None:
|
||||
return self.action_head.get_action(
|
||||
fused_tokens,
|
||||
state=state,
|
||||
action_mask=action_mask,
|
||||
embodiment_id=embodiment_ids,
|
||||
)
|
||||
return self.action_head(
|
||||
fused_tokens,
|
||||
state=state,
|
||||
actions_gt=actions_gt,
|
||||
action_mask=action_mask,
|
||||
embodiment_id=embodiment_ids,
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def run_inference(
|
||||
self,
|
||||
images: list[list[Image.Image | torch.Tensor]],
|
||||
image_mask: torch.Tensor,
|
||||
prompt: str | list[str],
|
||||
state_input: list | torch.Tensor,
|
||||
return_cls_only: bool | None = None,
|
||||
action_mask: torch.Tensor | None = None,
|
||||
embodiment_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
fused_tokens = self.get_vl_embeddings(
|
||||
images=images,
|
||||
image_mask=image_mask,
|
||||
prompt=prompt,
|
||||
return_cls_only=return_cls_only,
|
||||
)
|
||||
state_tensor = self.prepare_state(state_input)
|
||||
if state_tensor.shape[0] != fused_tokens.shape[0]:
|
||||
raise ValueError(
|
||||
f"EVO1 batch mismatch: state_batch={state_tensor.shape[0]} fused_batch={fused_tokens.shape[0]}"
|
||||
)
|
||||
action = self.predict_action(
|
||||
fused_tokens,
|
||||
state_tensor,
|
||||
action_mask=action_mask,
|
||||
embodiment_ids=embodiment_ids,
|
||||
)
|
||||
|
||||
if isinstance(action, torch.Tensor) and action.dtype == torch.bfloat16:
|
||||
action = action.to(torch.float32)
|
||||
|
||||
return action
|
||||
|
||||
def forward(
|
||||
self,
|
||||
fused_tokens: torch.Tensor,
|
||||
state: torch.Tensor | None = None,
|
||||
actions_gt: torch.Tensor | None = None,
|
||||
action_mask: torch.Tensor | None = None,
|
||||
embodiment_ids: torch.Tensor | None = None,
|
||||
):
|
||||
return self.predict_action(fused_tokens, state, actions_gt, action_mask, embodiment_ids)
|
||||
|
||||
def _freeze_module(self, module: nn.Module):
|
||||
for param in module.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
def set_finetune_flags(self):
|
||||
if not self.config.get("finetune_vlm", False):
|
||||
self._freeze_module(self.embedder)
|
||||
|
||||
if not self.config.get("finetune_action_head", False):
|
||||
self._freeze_module(self.action_head)
|
||||
@ -0,0 +1,408 @@
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class SinusoidalPositionalEncoding(nn.Module):
|
||||
def __init__(self, dim: int, max_len: int = 1000):
|
||||
super().__init__()
|
||||
pe = torch.zeros(max_len, dim)
|
||||
position = torch.arange(0, max_len).unsqueeze(1)
|
||||
div_term = torch.exp(torch.arange(0, dim, 2) * -(math.log(10000.0) / dim))
|
||||
pe[:, 0::2] = torch.sin(position * div_term)
|
||||
pe[:, 1::2] = torch.cos(position * div_term)
|
||||
pe = pe.unsqueeze(0)
|
||||
self.register_buffer("pe", pe)
|
||||
|
||||
def forward(self, seq_len: int):
|
||||
if seq_len > self.pe.size(1):
|
||||
self._extend_pe(seq_len)
|
||||
return self.pe[:, :seq_len, :]
|
||||
|
||||
def _extend_pe(self, new_max_len):
|
||||
old_max_len, dim = self.pe.size(1), self.pe.size(2)
|
||||
if new_max_len <= old_max_len:
|
||||
return
|
||||
extra_positions = torch.arange(old_max_len, new_max_len, dtype=torch.float).unsqueeze(1)
|
||||
div_term = torch.exp(torch.arange(0, dim, 2, dtype=torch.float) * -(math.log(10000.0) / dim))
|
||||
extra_pe = torch.zeros(new_max_len - old_max_len, dim)
|
||||
extra_pe[:, 0::2] = torch.sin(extra_positions * div_term)
|
||||
extra_pe[:, 1::2] = torch.cos(extra_positions * div_term)
|
||||
extra_pe = extra_pe.unsqueeze(0)
|
||||
new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1)
|
||||
self.pe = new_pe
|
||||
|
||||
|
||||
class CategorySpecificLinear(nn.Module):
|
||||
def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1):
|
||||
super().__init__()
|
||||
self.num_categories = num_categories
|
||||
if num_categories <= 1:
|
||||
self.linear = nn.Linear(in_dim, out_dim)
|
||||
else:
|
||||
self.weight = nn.Parameter(torch.randn(num_categories, in_dim, out_dim))
|
||||
self.bias = nn.Parameter(torch.randn(num_categories, out_dim))
|
||||
|
||||
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
|
||||
if self.num_categories <= 1:
|
||||
if x.dtype != self.linear.weight.dtype:
|
||||
x = x.to(dtype=self.linear.weight.dtype)
|
||||
return self.linear(x)
|
||||
|
||||
if x.dtype != self.weight.dtype:
|
||||
x = x.to(dtype=self.weight.dtype)
|
||||
|
||||
orig_shape = x.shape
|
||||
x_flat = x.reshape(-1, orig_shape[-1])
|
||||
if category_id.dim() == 0:
|
||||
cid = category_id.item()
|
||||
out = x_flat @ self.weight[cid] + self.bias[cid]
|
||||
else:
|
||||
category_id = category_id.view(-1)
|
||||
weight_selected = self.weight[category_id]
|
||||
bias_selected = self.bias[category_id]
|
||||
out = torch.bmm(x_flat.unsqueeze(1), weight_selected).squeeze(1) + bias_selected
|
||||
out_shape = orig_shape[:-1] + (out.shape[-1],)
|
||||
return out.view(out_shape)
|
||||
|
||||
|
||||
class CategorySpecificMLP(nn.Module):
|
||||
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1):
|
||||
super().__init__()
|
||||
self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories)
|
||||
self.fc2 = CategorySpecificLinear(hidden_dim, output_dim, num_categories)
|
||||
self.activation = nn.ReLU(inplace=True)
|
||||
|
||||
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
|
||||
out = self.activation(self.fc1(x, category_id))
|
||||
out = self.fc2(out, category_id)
|
||||
return out
|
||||
|
||||
|
||||
class MultiEmbodimentActionEncoder(nn.Module):
|
||||
def __init__(
|
||||
self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1
|
||||
):
|
||||
super().__init__()
|
||||
self.horizon = horizon
|
||||
self.embed_dim = embed_dim
|
||||
self.num_categories = num_categories
|
||||
|
||||
self.W1 = CategorySpecificLinear(action_dim, hidden_dim, num_categories)
|
||||
self.W2 = CategorySpecificLinear(hidden_dim, hidden_dim, num_categories)
|
||||
self.W3 = CategorySpecificLinear(hidden_dim, embed_dim, num_categories)
|
||||
|
||||
self.pos_encoding = SinusoidalPositionalEncoding(hidden_dim, max_len=horizon)
|
||||
self.activation = nn.ReLU(inplace=True)
|
||||
|
||||
def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor):
|
||||
batch_size, horizon, action_dim = action_seq.shape
|
||||
assert self.horizon == horizon, "Action sequence length must match horizon"
|
||||
|
||||
x = action_seq.reshape(batch_size * horizon, action_dim)
|
||||
|
||||
if category_id.dim() == 0:
|
||||
cat_ids = category_id.expand(horizon * batch_size)
|
||||
else:
|
||||
cat_ids = category_id.unsqueeze(1).expand(batch_size, horizon).reshape(batch_size * horizon)
|
||||
out = self.activation(self.W1(x, cat_ids))
|
||||
|
||||
pos_enc = self.pos_encoding(horizon).to(device=out.device, dtype=out.dtype)
|
||||
out = out.view(batch_size, horizon, -1) + pos_enc
|
||||
out = out.view(batch_size * horizon, -1)
|
||||
out = self.activation(self.W2(out, cat_ids))
|
||||
out = self.W3(out, cat_ids)
|
||||
out = out.view(batch_size, horizon, self.embed_dim)
|
||||
return out
|
||||
|
||||
|
||||
class BasicTransformerBlock(nn.Module):
|
||||
def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0):
|
||||
super().__init__()
|
||||
self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
|
||||
self.norm1 = nn.LayerNorm(embed_dim)
|
||||
self.norm2 = nn.LayerNorm(embed_dim)
|
||||
self.ff = nn.Sequential(nn.Linear(embed_dim, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, embed_dim))
|
||||
|
||||
def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor):
|
||||
x = self.norm1(action_tokens)
|
||||
attn_out, _ = self.attn(x, context_tokens, context_tokens)
|
||||
|
||||
x = action_tokens + attn_out
|
||||
|
||||
x2 = self.norm2(x)
|
||||
|
||||
if time_emb is not None:
|
||||
x2 = x2 + time_emb.unsqueeze(1)
|
||||
ff_out = self.ff(x2)
|
||||
x = x + ff_out
|
||||
return x
|
||||
|
||||
|
||||
class FlowmatchingActionHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config=None,
|
||||
embed_dim: int = 896,
|
||||
hidden_dim: int = 1024,
|
||||
action_dim: int = 16 * 7,
|
||||
horizon: int = 16,
|
||||
per_action_dim: int = 7,
|
||||
num_heads: int = 8,
|
||||
num_layers: int = 8,
|
||||
dropout: float = 0.0,
|
||||
num_inference_timesteps: int = 20,
|
||||
num_categories: int = 1,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
if config is not None:
|
||||
embed_dim = getattr(config, "embed_dim", embed_dim)
|
||||
hidden_dim = getattr(config, "hidden_dim", hidden_dim)
|
||||
action_dim = getattr(config, "action_dim", action_dim)
|
||||
horizon = getattr(config, "horizon", horizon)
|
||||
num_heads = getattr(config, "num_heads", num_heads)
|
||||
num_layers = getattr(config, "num_layers", num_layers)
|
||||
dropout = getattr(config, "dropout", dropout)
|
||||
num_inference_timesteps = getattr(config, "num_inference_timesteps", num_inference_timesteps)
|
||||
num_categories = getattr(config, "num_categories", num_categories)
|
||||
self.config = config
|
||||
else:
|
||||
from types import SimpleNamespace
|
||||
|
||||
self.config = SimpleNamespace(
|
||||
embed_dim=embed_dim,
|
||||
hidden_dim=hidden_dim,
|
||||
action_dim=action_dim,
|
||||
horizon=horizon,
|
||||
num_heads=num_heads,
|
||||
num_layers=num_layers,
|
||||
dropout=dropout,
|
||||
num_inference_timesteps=num_inference_timesteps,
|
||||
num_categories=num_categories,
|
||||
)
|
||||
self.embed_dim = embed_dim
|
||||
self.horizon = horizon
|
||||
self.per_action_dim = getattr(self.config, "per_action_dim", per_action_dim)
|
||||
self.action_dim = getattr(self.config, "action_dim", action_dim)
|
||||
|
||||
self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000)
|
||||
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
BasicTransformerBlock(
|
||||
embed_dim=embed_dim, num_heads=num_heads, hidden_dim=embed_dim * 4, dropout=dropout
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
self.norm_out = nn.LayerNorm(embed_dim)
|
||||
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim)
|
||||
|
||||
self.mlp_head = CategorySpecificMLP(
|
||||
input_dim=embed_dim, hidden_dim=hidden_dim, output_dim=action_dim, num_categories=num_categories
|
||||
)
|
||||
|
||||
self.state_encoder = None
|
||||
if hasattr(self.config, "state_dim") and self.config.state_dim is not None:
|
||||
state_hidden = getattr(self.config, "state_hidden_dim", embed_dim)
|
||||
|
||||
self.state_encoder = CategorySpecificMLP(
|
||||
input_dim=self.config.state_dim,
|
||||
hidden_dim=state_hidden,
|
||||
output_dim=embed_dim,
|
||||
num_categories=num_categories,
|
||||
)
|
||||
|
||||
self.action_encoder = None
|
||||
self.single_action_proj = None
|
||||
if horizon > 1:
|
||||
per_action_dim = getattr(self.config, "per_action_dim", None)
|
||||
if per_action_dim is None:
|
||||
per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim
|
||||
self.action_encoder = MultiEmbodimentActionEncoder(
|
||||
action_dim=per_action_dim,
|
||||
embed_dim=embed_dim,
|
||||
hidden_dim=embed_dim,
|
||||
horizon=horizon,
|
||||
num_categories=num_categories,
|
||||
)
|
||||
else:
|
||||
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
fused_tokens: torch.Tensor,
|
||||
state: torch.Tensor = None,
|
||||
actions_gt: torch.Tensor = None,
|
||||
embodiment_id: torch.LongTensor = None,
|
||||
state_mask: torch.Tensor = None,
|
||||
action_mask: torch.Tensor = None,
|
||||
):
|
||||
if actions_gt is None:
|
||||
return self.get_action(
|
||||
fused_tokens,
|
||||
state=state,
|
||||
embodiment_id=embodiment_id,
|
||||
action_mask=action_mask,
|
||||
)
|
||||
batch_size = fused_tokens.size(0)
|
||||
device = fused_tokens.device
|
||||
|
||||
if embodiment_id is None:
|
||||
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
|
||||
|
||||
context_tokens = fused_tokens
|
||||
if state is not None and self.state_encoder is not None:
|
||||
state_emb = self.state_encoder(state, embodiment_id)
|
||||
state_emb = state_emb.unsqueeze(1)
|
||||
|
||||
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
|
||||
|
||||
t = (
|
||||
torch.distributions.Beta(2, 2).sample((batch_size,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
|
||||
)
|
||||
|
||||
time_index = (t * 1000).long()
|
||||
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0).to(dtype=context_tokens.dtype)
|
||||
|
||||
actions_gt_seq = actions_gt
|
||||
|
||||
noise = torch.rand_like(actions_gt) * 2 - 1
|
||||
|
||||
if action_mask is not None:
|
||||
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
|
||||
if action_mask.shape != noise.shape:
|
||||
raise ValueError(f"action_mask shape {action_mask.shape} != noise shape {noise.shape}")
|
||||
actions_gt_seq = actions_gt_seq * action_mask
|
||||
noise = noise * action_mask
|
||||
|
||||
if self.horizon > 1:
|
||||
noise_seq = noise.view(batch_size, self.horizon, self.per_action_dim)
|
||||
actions_gt_seq = actions_gt_seq.view(batch_size, self.horizon, self.per_action_dim)
|
||||
else:
|
||||
noise_seq = noise.view(batch_size, 1, self.per_action_dim)
|
||||
actions_gt_seq = actions_gt_seq.view(batch_size, 1, self.per_action_dim)
|
||||
|
||||
t_broadcast = t.view(batch_size, 1, 1)
|
||||
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
|
||||
|
||||
if self.horizon > 1 and self.action_encoder is not None:
|
||||
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
|
||||
else:
|
||||
if self.single_action_proj is None:
|
||||
raise RuntimeError("single_action_proj is not initialized for single-step inference/training.")
|
||||
action_tokens = self.single_action_proj(action_intermediate_seq)
|
||||
|
||||
x = action_tokens
|
||||
for block in self.transformer_blocks:
|
||||
x = block(x, context_tokens, time_emb)
|
||||
|
||||
x = self.norm_out(x)
|
||||
|
||||
if self.horizon > 1:
|
||||
x_flat = x.reshape(batch_size, -1)
|
||||
x_pooled = self.seq_pool_proj(x_flat)
|
||||
else:
|
||||
x_pooled = x.squeeze(1)
|
||||
|
||||
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
|
||||
|
||||
return pred_velocity, noise
|
||||
|
||||
def get_action(
|
||||
self,
|
||||
fused_tokens: torch.Tensor,
|
||||
state: torch.Tensor = None,
|
||||
embodiment_id: torch.LongTensor = None,
|
||||
action_mask: torch.Tensor = None,
|
||||
):
|
||||
if action_mask is None:
|
||||
raise ValueError("action_mask must be provided for inference with flow matching.")
|
||||
|
||||
batch_size = fused_tokens.size(0)
|
||||
device = fused_tokens.device
|
||||
if embodiment_id is None:
|
||||
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
|
||||
|
||||
context_tokens = fused_tokens
|
||||
if state is not None and self.state_encoder is not None:
|
||||
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
|
||||
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
|
||||
|
||||
action_dim_total = getattr(self.config, "action_dim", None)
|
||||
if action_dim_total is None:
|
||||
action_dim_total = self.action_dim
|
||||
|
||||
if self.horizon > 1:
|
||||
per_action_dim = getattr(self.config, "per_action_dim", action_dim_total // self.horizon)
|
||||
else:
|
||||
per_action_dim = action_dim_total
|
||||
|
||||
action = torch.rand(batch_size, action_dim_total, device=device, dtype=context_tokens.dtype) * 2 - 1
|
||||
if self.horizon > 1:
|
||||
action_seq = action.view(batch_size, self.horizon, per_action_dim)
|
||||
|
||||
else:
|
||||
action_seq = action.view(batch_size, 1, per_action_dim)
|
||||
|
||||
action_mask = action_mask.view(batch_size, 1, per_action_dim).repeat(1, self.horizon, 1)
|
||||
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
|
||||
if action_mask.shape != action_seq.shape:
|
||||
raise ValueError(f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}")
|
||||
|
||||
flat_action_mask = action_mask.view(batch_size, -1)
|
||||
action_seq = action_seq * action_mask
|
||||
|
||||
num_steps = int(getattr(self.config, "num_inference_timesteps", 32))
|
||||
dt = 1.0 / num_steps
|
||||
t_vals = torch.arange(num_steps, device=device, dtype=torch.float32) / num_steps
|
||||
time_indices = (t_vals * 1000).long()
|
||||
all_time_embs = self.time_pos_enc(1000)[0, time_indices, :].to(device=device, dtype=context_tokens.dtype)
|
||||
|
||||
for i in range(num_steps):
|
||||
time_emb = all_time_embs[i].unsqueeze(0).expand(batch_size, -1)
|
||||
|
||||
if self.horizon > 1 and self.action_encoder is not None:
|
||||
action_tokens = self.action_encoder(action_seq, embodiment_id)
|
||||
else:
|
||||
if self.single_action_proj is None:
|
||||
raise RuntimeError("single_action_proj is not initialized for single-step inference/training.")
|
||||
action_tokens = self.single_action_proj(action_seq)
|
||||
|
||||
x = action_tokens
|
||||
for block in self.transformer_blocks:
|
||||
x = block(x, context_tokens, time_emb)
|
||||
x = self.norm_out(x)
|
||||
|
||||
if self.horizon > 1:
|
||||
x_flat = x.reshape(batch_size, -1)
|
||||
if hasattr(self, "seq_pool_proj"):
|
||||
x_pooled = self.seq_pool_proj(x_flat)
|
||||
else:
|
||||
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
|
||||
x_pooled = self.seq_pool_proj(x_flat)
|
||||
else:
|
||||
x_pooled = x.squeeze(1)
|
||||
|
||||
pred = self.mlp_head(x_pooled, embodiment_id)
|
||||
|
||||
action = action + dt * pred
|
||||
action = action * flat_action_mask
|
||||
|
||||
if self.horizon > 1:
|
||||
action_seq = action.view(batch_size, self.horizon, per_action_dim)
|
||||
else:
|
||||
action_seq = action.view(batch_size, 1, per_action_dim)
|
||||
|
||||
return action
|
||||
|
||||
@property
|
||||
def device(self):
|
||||
return next(self.parameters()).device
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return next(self.parameters()).dtype
|
||||
@ -0,0 +1,412 @@
|
||||
import functools
|
||||
import logging
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torchvision.transforms as T
|
||||
import torchvision.transforms.functional as TF
|
||||
from PIL import Image
|
||||
from torchvision.transforms.functional import InterpolationMode, to_pil_image
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
|
||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
IMAGENET_STD = (0.229, 0.224, 0.225)
|
||||
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>" # nosec B105
|
||||
IMG_START_TOKEN = "<img>" # nosec B105
|
||||
IMG_END_TOKEN = "</img>" # nosec B105
|
||||
|
||||
|
||||
# === Image Transformations ===
|
||||
def build_transform(input_size):
|
||||
return T.Compose(
|
||||
[
|
||||
T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
|
||||
T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
|
||||
T.ToTensor(),
|
||||
T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# === Aspect Ratio Handling ===
|
||||
@functools.lru_cache(maxsize=10000)
|
||||
def get_target_aspect_ratio(orig_width, orig_height, image_size, min_num, max_num):
|
||||
aspect_ratio = orig_width / orig_height
|
||||
target_ratios = {
|
||||
(i, j)
|
||||
for n in range(min_num, max_num + 1)
|
||||
for i in range(1, n + 1)
|
||||
for j in range(1, n + 1)
|
||||
if i * j <= max_num and i * j >= min_num
|
||||
}
|
||||
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
||||
|
||||
best_ratio_diff = float("inf")
|
||||
best_ratio = (1, 1)
|
||||
area = orig_width * orig_height
|
||||
for ratio in target_ratios:
|
||||
target_ar = ratio[0] / ratio[1]
|
||||
diff = abs(aspect_ratio - target_ar)
|
||||
if diff < best_ratio_diff:
|
||||
best_ratio_diff = diff
|
||||
best_ratio = ratio
|
||||
elif diff == best_ratio_diff and area > 0.5 * image_size**2 * ratio[0] * ratio[1]:
|
||||
best_ratio = ratio
|
||||
return best_ratio
|
||||
|
||||
|
||||
def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False):
|
||||
orig_width, orig_height = image.size
|
||||
target_aspect_ratio = get_target_aspect_ratio(orig_width, orig_height, image_size, min_num, max_num)
|
||||
target_width = image_size * target_aspect_ratio[0]
|
||||
target_height = image_size * target_aspect_ratio[1]
|
||||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||||
resized_img = image.resize((target_width, target_height))
|
||||
processed_images = []
|
||||
for i in range(blocks):
|
||||
box = (
|
||||
(i % (target_width // image_size)) * image_size,
|
||||
(i // (target_width // image_size)) * image_size,
|
||||
((i % (target_width // image_size)) + 1) * image_size,
|
||||
((i // (target_width // image_size)) + 1) * image_size,
|
||||
)
|
||||
split_img = resized_img.crop(box)
|
||||
processed_images.append(split_img)
|
||||
assert len(processed_images) == blocks
|
||||
if use_thumbnail and len(processed_images) != 1:
|
||||
thumbnail_img = image.resize((image_size, image_size))
|
||||
processed_images.append(thumbnail_img)
|
||||
return processed_images
|
||||
|
||||
|
||||
class InternVL3Embedder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
model_name="OpenGVLab/InternVL3-1B",
|
||||
image_size=448,
|
||||
device="cuda",
|
||||
num_language_layers: int | None = 14,
|
||||
model_dtype: str | torch.dtype = "bfloat16",
|
||||
use_flash_attn: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self._requested_device = device
|
||||
self.image_size = image_size
|
||||
self.num_language_layers = num_language_layers
|
||||
self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens
|
||||
self.transform = build_transform(image_size)
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
|
||||
if isinstance(model_dtype, str):
|
||||
try:
|
||||
model_dtype = getattr(torch, model_dtype)
|
||||
except AttributeError as exc:
|
||||
raise ValueError(f"Unsupported EVO1 vlm_dtype '{model_dtype}'") from exc
|
||||
self.model = AutoModel.from_pretrained(
|
||||
model_name,
|
||||
torch_dtype=model_dtype,
|
||||
trust_remote_code=True,
|
||||
use_flash_attn=use_flash_attn,
|
||||
low_cpu_mem_usage=True,
|
||||
_fast_init=False,
|
||||
).to(self._requested_device)
|
||||
|
||||
if hasattr(self.model.language_model, "model"):
|
||||
layers = self.model.language_model.model.layers
|
||||
|
||||
else:
|
||||
layers = self.model.language_model.layers
|
||||
if self.num_language_layers is not None:
|
||||
layers = layers[: self.num_language_layers]
|
||||
|
||||
if hasattr(self.model.language_model, "model"):
|
||||
self.model.language_model.model.layers = torch.nn.ModuleList(layers)
|
||||
else:
|
||||
self.model.language_model.layers = torch.nn.ModuleList(layers)
|
||||
self.model.language_model.lm_head = torch.nn.Identity()
|
||||
|
||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||
self.model.vision_model.encoder.gradient_checkpointing = False
|
||||
|
||||
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
||||
|
||||
def _dynamic_preprocess_tensor(
|
||||
self, image_t, min_num=1, max_num=1, use_thumbnail=False
|
||||
):
|
||||
# image_t shape expected: [C, H, W]
|
||||
C, orig_height, orig_width = image_t.shape
|
||||
|
||||
# get ratio by cache
|
||||
target_aspect_ratio = get_target_aspect_ratio(
|
||||
orig_width, orig_height, self.image_size, min_num, max_num
|
||||
)
|
||||
|
||||
ratio_w, ratio_h = target_aspect_ratio[0], target_aspect_ratio[1]
|
||||
target_width = self.image_size * ratio_w
|
||||
target_height = self.image_size * ratio_h
|
||||
blocks = ratio_w * ratio_h
|
||||
|
||||
# Resize on GPU
|
||||
# image_t expected shape for interpolate is [C, H, W] mapping to size
|
||||
resized_img = TF.resize(
|
||||
image_t,
|
||||
size=[target_height, target_width],
|
||||
interpolation=InterpolationMode.BICUBIC,
|
||||
antialias=True,
|
||||
)
|
||||
|
||||
# Eliminate the for-loop using view and permute for zero-copy strided tensor tiling
|
||||
# resized_img shape: [C, ratio_h * image_size, ratio_w * image_size]
|
||||
# 1. view to -> [C, ratio_h, image_size, ratio_w, image_size]
|
||||
reshaped = resized_img.view(C, ratio_h, self.image_size, ratio_w, self.image_size)
|
||||
# 2. permute to -> [ratio_h, ratio_w, C, image_size, image_size]
|
||||
permuted = reshaped.permute(1, 3, 0, 2, 4)
|
||||
# 3. reshape to -> [blocks, C, image_size, image_size]
|
||||
stacked_tiles = permuted.reshape(blocks, C, self.image_size, self.image_size)
|
||||
|
||||
if use_thumbnail and blocks != 1:
|
||||
thumbnail_img = TF.resize(
|
||||
image_t,
|
||||
size=[self.image_size, self.image_size],
|
||||
interpolation=InterpolationMode.BICUBIC,
|
||||
antialias=True,
|
||||
)
|
||||
# Concat the thumbnail directly along the batch dimension
|
||||
stacked_tiles = torch.cat([stacked_tiles, thumbnail_img.unsqueeze(0)], dim=0)
|
||||
|
||||
return stacked_tiles
|
||||
|
||||
def _preprocess_images(
|
||||
self, image_tensors_batch: list[list[Image.Image | torch.Tensor]]
|
||||
) -> tuple[torch.Tensor, list[list[int]]]:
|
||||
pixel_values_list = []
|
||||
batch_num_tiles_list = []
|
||||
model_dtype = next(self.model.parameters()).dtype
|
||||
mean = torch.tensor(IMAGENET_MEAN, device=self.device, dtype=torch.float32).view(
|
||||
1, 3, 1, 1
|
||||
)
|
||||
std = torch.tensor(IMAGENET_STD, device=self.device, dtype=torch.float32).view(
|
||||
1, 3, 1, 1
|
||||
)
|
||||
|
||||
for image_tensors in image_tensors_batch:
|
||||
num_tiles_list = []
|
||||
for image in image_tensors:
|
||||
if isinstance(image, torch.Tensor):
|
||||
if (
|
||||
image.ndim == 3
|
||||
and image.shape[0] == 3
|
||||
and image.shape[1] == self.image_size
|
||||
and image.shape[2] == self.image_size
|
||||
):
|
||||
image_t = image.to(device=self.device)
|
||||
if not torch.is_floating_point(image_t):
|
||||
image_t = image_t.to(torch.float32) / 255.0
|
||||
else:
|
||||
image_t = image_t.to(torch.float32)
|
||||
if float(image_t.max().item()) > 1.0:
|
||||
image_t = image_t / 255.0
|
||||
tile_tensors = (image_t.unsqueeze(0) - mean) / std
|
||||
else:
|
||||
image_t = image.to(device=self.device, dtype=torch.float32)
|
||||
if not torch.is_floating_point(image):
|
||||
image_t = image_t / 255.0
|
||||
else:
|
||||
if float(image_t.max().item()) > 1.0:
|
||||
image_t = image_t / 255.0
|
||||
tiles = self._dynamic_preprocess_tensor(image_t)
|
||||
tile_tensors = (tiles - mean) / std
|
||||
else:
|
||||
image_t = TF.to_tensor(image).to(device=self.device, dtype=torch.float32)
|
||||
tiles = self._dynamic_preprocess_tensor(image_t)
|
||||
tile_tensors = (tiles - mean) / std
|
||||
|
||||
tile_tensors = tile_tensors.to(dtype=model_dtype)
|
||||
pixel_values_list.append(tile_tensors)
|
||||
num_tiles_list.append(tile_tensors.shape[0])
|
||||
batch_num_tiles_list.append(num_tiles_list)
|
||||
|
||||
if pixel_values_list:
|
||||
pixel_values = torch.cat(pixel_values_list, dim=0)
|
||||
else:
|
||||
pixel_values = torch.empty(
|
||||
0,
|
||||
3,
|
||||
self.image_size,
|
||||
self.image_size,
|
||||
dtype=model_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
return pixel_values, batch_num_tiles_list
|
||||
|
||||
def _build_multimodal_prompt(
|
||||
self,
|
||||
batch_num_tiles_list: list[list[int]],
|
||||
text_prompts: list[str],
|
||||
) -> list[str]:
|
||||
if len(batch_num_tiles_list) != len(text_prompts):
|
||||
raise ValueError(
|
||||
f"InternVL3 batch mismatch: num_image_batches={len(batch_num_tiles_list)} num_text_prompts={len(text_prompts)}"
|
||||
)
|
||||
|
||||
prompts = []
|
||||
for num_tiles_list, text_prompt in zip(batch_num_tiles_list, text_prompts, strict=True):
|
||||
prompt_segments = []
|
||||
for i, tile_count in enumerate(num_tiles_list):
|
||||
token_count = self.model.num_image_token * tile_count
|
||||
image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * token_count + IMG_END_TOKEN
|
||||
prompt_segments.append(f"Image-{i + 1}: {image_tokens}\n")
|
||||
prompts.append("".join(prompt_segments) + text_prompt.strip())
|
||||
|
||||
return prompts
|
||||
|
||||
def _prepare_and_fuse_embeddings(
|
||||
self,
|
||||
prompts: list[str],
|
||||
vit_embeds: torch.Tensor,
|
||||
image_masks: torch.Tensor,
|
||||
batch_num_tiles_list: list[list[int]],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
untruncated_ids = self.tokenizer(prompts, padding=False, truncation=False)["input_ids"]
|
||||
true_sequence_length = max((len(ids) for ids in untruncated_ids), default=0)
|
||||
|
||||
if true_sequence_length > self.max_text_length:
|
||||
logging.warning(
|
||||
"InternVL3 prompt truncated in batch: max_length=%s actual_max_length=%s",
|
||||
self.max_text_length,
|
||||
true_sequence_length,
|
||||
)
|
||||
|
||||
model_inputs = self.tokenizer(
|
||||
prompts,
|
||||
return_tensors="pt",
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
max_length=self.max_text_length,
|
||||
).to(self.device)
|
||||
input_ids = model_inputs["input_ids"]
|
||||
attention_mask = model_inputs["attention_mask"]
|
||||
|
||||
img_token_mask = input_ids == self.img_context_token_id
|
||||
input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone()
|
||||
batch_size, _, channels = input_embeds.shape
|
||||
vit_embeds = vit_embeds.reshape(-1, channels).to(device=input_embeds.device, dtype=input_embeds.dtype)
|
||||
|
||||
tokens_per_tile = self.model.num_image_token
|
||||
vit_idx = 0
|
||||
actual_vis_tokens_list = img_token_mask.sum(dim=1).tolist()
|
||||
|
||||
for batch_idx in range(batch_size):
|
||||
expected_vis_tokens = sum(batch_num_tiles_list[batch_idx]) * tokens_per_tile
|
||||
actual_vis_tokens = int(actual_vis_tokens_list[batch_idx])
|
||||
if actual_vis_tokens > expected_vis_tokens:
|
||||
raise ValueError(
|
||||
"InternVL3 detected more image placeholder tokens than expected in prompt construction: "
|
||||
f"batch_idx={batch_idx}, actual={actual_vis_tokens}, expected={expected_vis_tokens}"
|
||||
)
|
||||
|
||||
if vit_idx + expected_vis_tokens > vit_embeds.shape[0]:
|
||||
raise ValueError(
|
||||
"InternVL3 produced fewer image tokens than expected for batch fusion: "
|
||||
f"need_up_to={vit_idx + expected_vis_tokens}, got={vit_embeds.shape[0]}"
|
||||
)
|
||||
|
||||
item_vit_embeds = vit_embeds[vit_idx : vit_idx + expected_vis_tokens]
|
||||
vit_idx += expected_vis_tokens
|
||||
|
||||
if actual_vis_tokens > 0:
|
||||
input_embeds[batch_idx, img_token_mask[batch_idx]] = item_vit_embeds[:actual_vis_tokens]
|
||||
|
||||
current_token_idx = 0
|
||||
img_token_locations = torch.where(img_token_mask[batch_idx])[0]
|
||||
for image_idx, num_tiles_for_this_image in enumerate(batch_num_tiles_list[batch_idx]):
|
||||
num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile
|
||||
if not bool(image_masks[batch_idx, image_idx].item()):
|
||||
start_offset = current_token_idx
|
||||
end_offset = min(
|
||||
current_token_idx + num_tokens_for_this_image,
|
||||
int(img_token_locations.shape[0]),
|
||||
)
|
||||
if start_offset < end_offset:
|
||||
masked_token_indices = img_token_locations[start_offset:end_offset]
|
||||
attention_mask[batch_idx, masked_token_indices] = 0
|
||||
current_token_idx += num_tokens_for_this_image
|
||||
|
||||
return input_embeds, attention_mask
|
||||
|
||||
def get_fused_image_text_embedding_from_tensor_images(
|
||||
self,
|
||||
image_tensors: list[Image.Image | torch.Tensor] | list[list[Image.Image | torch.Tensor]],
|
||||
image_mask: torch.Tensor,
|
||||
text_prompt: str | list[str],
|
||||
return_cls_only: bool = True,
|
||||
):
|
||||
is_batched_input = bool(image_tensors) and isinstance(image_tensors[0], list)
|
||||
if is_batched_input:
|
||||
image_tensors_batch = image_tensors
|
||||
else:
|
||||
image_tensors_batch = [image_tensors]
|
||||
|
||||
batch_size = len(image_tensors_batch)
|
||||
if batch_size == 0:
|
||||
raise ValueError("InternVL3 expects at least one batch item.")
|
||||
|
||||
if isinstance(text_prompt, str):
|
||||
text_prompts = [text_prompt] * batch_size
|
||||
else:
|
||||
text_prompts = text_prompt
|
||||
if len(text_prompts) != batch_size:
|
||||
raise ValueError(
|
||||
f"InternVL3 batch mismatch: num_text_prompts={len(text_prompts)} num_batches={batch_size}"
|
||||
)
|
||||
|
||||
if image_mask.ndim == 1:
|
||||
image_masks = image_mask.unsqueeze(0)
|
||||
else:
|
||||
image_masks = image_mask
|
||||
|
||||
if image_masks.shape[0] != batch_size:
|
||||
raise ValueError(
|
||||
f"InternVL3 batch mismatch: image_mask_batch={image_masks.shape[0]} num_batches={batch_size}"
|
||||
)
|
||||
|
||||
for batch_idx, image_list in enumerate(image_tensors_batch):
|
||||
if len(image_list) == 0:
|
||||
raise ValueError(f"InternVL3 expects at least one image per batch item, got empty at batch_idx={batch_idx}")
|
||||
if int(image_masks.shape[1]) != len(image_list):
|
||||
raise ValueError(
|
||||
"InternVL3 mask/image count mismatch: "
|
||||
f"batch_idx={batch_idx}, num_masks={image_masks.shape[1]}, num_images={len(image_list)}"
|
||||
)
|
||||
|
||||
image_masks = image_masks.to(device=self.device, dtype=torch.bool)
|
||||
pixel_values, batch_num_tiles_list = self._preprocess_images(image_tensors_batch)
|
||||
|
||||
if pixel_values.shape[0] == 0:
|
||||
logging.warning("InternVL3 received an empty image batch after preprocessing.")
|
||||
hidden_size = self.model.language_model.get_input_embeddings().embedding_dim
|
||||
vit_embeds = torch.empty(0, hidden_size, dtype=next(self.model.parameters()).dtype, device=self.device)
|
||||
else:
|
||||
vit_embeds = self.model.extract_feature(pixel_values)
|
||||
|
||||
prompts = self._build_multimodal_prompt(batch_num_tiles_list, text_prompts)
|
||||
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(
|
||||
prompts,
|
||||
vit_embeds,
|
||||
image_masks,
|
||||
batch_num_tiles_list,
|
||||
)
|
||||
|
||||
outputs = self.model.language_model(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=attention_mask,
|
||||
output_hidden_states=True,
|
||||
return_dict=True,
|
||||
)
|
||||
fused_hidden = outputs.hidden_states[-1].to(torch.float32)
|
||||
|
||||
return fused_hidden[:, 0, :] if return_cls_only else fused_hidden
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return next(self.model.parameters()).device
|
||||
@ -0,0 +1,404 @@
|
||||
import builtins
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
|
||||
from lerobot.policies.evo1.evo1_model import EVO1
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy, T
|
||||
from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE
|
||||
|
||||
|
||||
class EVO1Policy(PreTrainedPolicy):
|
||||
config_class = Evo1Config
|
||||
name = "evo1"
|
||||
|
||||
def __init__(self, config: Evo1Config, **kwargs):
|
||||
super().__init__(config)
|
||||
config.validate_features()
|
||||
|
||||
if len(config.image_features) > config.max_views:
|
||||
raise ValueError(
|
||||
f"EVO1 supports at most {config.max_views} camera streams, got {len(config.image_features)}"
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.model = EVO1(self._build_model_config(config))
|
||||
self.model.set_finetune_flags()
|
||||
self.reset()
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
cls: builtins.type[T],
|
||||
pretrained_name_or_path: str | Path,
|
||||
*,
|
||||
config: PreTrainedConfig | None = None,
|
||||
force_download: bool = False,
|
||||
resume_download: bool | None = None,
|
||||
proxies: dict | None = None,
|
||||
token: str | bool | None = None,
|
||||
cache_dir: str | Path | None = None,
|
||||
local_files_only: bool = False,
|
||||
revision: str | None = None,
|
||||
strict: bool = True,
|
||||
**kwargs,
|
||||
) -> T:
|
||||
return super().from_pretrained(
|
||||
pretrained_name_or_path=pretrained_name_or_path,
|
||||
config=config,
|
||||
force_download=force_download,
|
||||
resume_download=resume_download,
|
||||
proxies=proxies,
|
||||
token=token,
|
||||
cache_dir=cache_dir,
|
||||
local_files_only=local_files_only,
|
||||
revision=revision,
|
||||
strict=strict,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_model_config(config: Evo1Config) -> dict:
|
||||
return {
|
||||
"device": config.device,
|
||||
"return_cls_only": config.return_cls_only,
|
||||
"vlm_name": config.vlm_model_name,
|
||||
"vlm_num_layers": config.vlm_num_layers,
|
||||
"vlm_dtype": config.vlm_dtype,
|
||||
"use_flash_attn": config.use_flash_attn,
|
||||
"action_head": config.action_head,
|
||||
"action_horizon": config.chunk_size,
|
||||
"per_action_dim": config.max_action_dim,
|
||||
"state_dim": config.max_state_dim,
|
||||
"embed_dim": config.embed_dim,
|
||||
"hidden_dim": config.hidden_dim,
|
||||
"state_hidden_dim": config.state_hidden_dim,
|
||||
"num_heads": config.num_heads,
|
||||
"num_layers": config.num_layers,
|
||||
"dropout": config.dropout,
|
||||
"num_inference_timesteps": config.num_inference_timesteps,
|
||||
"num_categories": config.num_categories,
|
||||
"finetune_vlm": config.finetune_vlm,
|
||||
"finetune_action_head": config.finetune_action_head,
|
||||
}
|
||||
|
||||
@property
|
||||
def _camera_keys(self) -> list[str]:
|
||||
return list(self.config.image_features)
|
||||
|
||||
@property
|
||||
def _env_action_dim(self) -> int:
|
||||
action_feature = self.config.action_feature
|
||||
if action_feature is None:
|
||||
return self.config.max_action_dim
|
||||
return int(action_feature.shape[0])
|
||||
|
||||
@property
|
||||
def _compute_dtype(self) -> torch.dtype:
|
||||
return next(self.model.action_head.parameters()).dtype
|
||||
|
||||
@property
|
||||
def _training_compute_dtype(self) -> torch.dtype:
|
||||
if str(self.config.device).startswith("cuda"):
|
||||
return torch.bfloat16
|
||||
return self._compute_dtype
|
||||
|
||||
@property
|
||||
def _inference_compute_dtype(self) -> torch.dtype:
|
||||
if str(self.config.device).startswith("cuda") and self.config.use_amp:
|
||||
return torch.bfloat16
|
||||
return self._compute_dtype
|
||||
|
||||
def get_optim_params(self) -> list[dict]:
|
||||
decay, no_decay = [], []
|
||||
for name, param in self.named_parameters():
|
||||
if not param.requires_grad:
|
||||
continue
|
||||
is_bias = name.endswith("bias") or ".bias" in name
|
||||
is_norm = param.dim() == 1 or "norm" in name.lower()
|
||||
if is_bias or is_norm:
|
||||
no_decay.append(param)
|
||||
else:
|
||||
decay.append(param)
|
||||
return [
|
||||
{"params": decay, "weight_decay": self.config.optimizer_weight_decay},
|
||||
{"params": no_decay, "weight_decay": 0.0},
|
||||
]
|
||||
|
||||
def reset(self):
|
||||
self._action_queue = deque([], maxlen=self.config.n_action_steps)
|
||||
|
||||
def _normalize_task_batch(self, batch: dict[str, Tensor | list[str] | str]) -> list[str]:
|
||||
prompts = batch.get(self.config.task_field)
|
||||
if prompts is None and self.config.task_field != "task":
|
||||
prompts = batch.get("task")
|
||||
if prompts is None:
|
||||
raise ValueError(f"EVO1 expects a '{self.config.task_field}' text field in the batch.")
|
||||
if isinstance(prompts, str):
|
||||
return [prompts]
|
||||
if isinstance(prompts, (list, tuple)):
|
||||
return [str(prompt) for prompt in prompts]
|
||||
raise TypeError(f"Unsupported prompt batch type: {type(prompts)}")
|
||||
|
||||
def _prepare_state(self, batch: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
|
||||
if OBS_STATE not in batch:
|
||||
raise ValueError(f"EVO1 requires '{OBS_STATE}' in the batch.")
|
||||
state = batch[OBS_STATE]
|
||||
if state.dim() == 1:
|
||||
state = state.unsqueeze(0)
|
||||
elif state.dim() == 3:
|
||||
state = state[:, -1]
|
||||
elif state.dim() != 2:
|
||||
raise ValueError(f"Unsupported state tensor shape for EVO1: {tuple(state.shape)}")
|
||||
batch_size, state_dim = state.shape
|
||||
if state_dim > self.config.max_state_dim:
|
||||
raise ValueError(
|
||||
f"State dim {state_dim} exceeds configured max_state_dim {self.config.max_state_dim}"
|
||||
)
|
||||
explicit_mask = batch.get("state_mask")
|
||||
if explicit_mask is not None:
|
||||
if explicit_mask.dim() == 1:
|
||||
explicit_mask = explicit_mask.unsqueeze(0)
|
||||
elif explicit_mask.dim() == 3:
|
||||
explicit_mask = explicit_mask[:, -1]
|
||||
elif explicit_mask.dim() != 2:
|
||||
raise ValueError(
|
||||
f"Unsupported state_mask tensor shape for EVO1: {tuple(explicit_mask.shape)}"
|
||||
)
|
||||
if explicit_mask.shape != (batch_size, state_dim):
|
||||
raise ValueError(
|
||||
f"state_mask shape {tuple(explicit_mask.shape)} does not match state shape {(batch_size, state_dim)}"
|
||||
)
|
||||
padded = torch.zeros(
|
||||
batch_size,
|
||||
self.config.max_state_dim,
|
||||
dtype=state.dtype,
|
||||
device=self.config.device,
|
||||
)
|
||||
padded[:, :state_dim] = state.to(device=self.config.device)
|
||||
mask = torch.zeros(
|
||||
batch_size,
|
||||
self.config.max_state_dim,
|
||||
dtype=torch.bool,
|
||||
device=self.config.device,
|
||||
)
|
||||
if explicit_mask is None:
|
||||
mask[:, :state_dim] = True
|
||||
else:
|
||||
mask[:, :state_dim] = explicit_mask.to(device=self.config.device, dtype=torch.bool)
|
||||
return padded.to(dtype=self._compute_dtype), mask
|
||||
|
||||
def _prepare_actions(self, batch: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
|
||||
if ACTION not in batch:
|
||||
raise ValueError(f"EVO1 requires '{ACTION}' in the batch for training.")
|
||||
action = batch[ACTION]
|
||||
if action.dim() == 2:
|
||||
action = action.unsqueeze(1)
|
||||
batch_size, horizon, action_dim = action.shape
|
||||
if horizon != self.config.chunk_size:
|
||||
raise ValueError(
|
||||
f"EVO1 expects chunk_size={self.config.chunk_size}, got action horizon {horizon}"
|
||||
)
|
||||
if action_dim > self.config.max_action_dim:
|
||||
raise ValueError(
|
||||
f"Action dim {action_dim} exceeds configured max_action_dim {self.config.max_action_dim}"
|
||||
)
|
||||
explicit_mask = batch.get("action_mask")
|
||||
if explicit_mask is not None:
|
||||
if explicit_mask.dim() == 2:
|
||||
explicit_mask = explicit_mask.unsqueeze(0)
|
||||
elif explicit_mask.dim() != 3:
|
||||
raise ValueError(
|
||||
f"Unsupported action_mask tensor shape for EVO1: {tuple(explicit_mask.shape)}"
|
||||
)
|
||||
if explicit_mask.shape != (batch_size, horizon, action_dim):
|
||||
raise ValueError(
|
||||
"action_mask shape "
|
||||
f"{tuple(explicit_mask.shape)} does not match action shape {(batch_size, horizon, action_dim)}"
|
||||
)
|
||||
padded = torch.zeros(
|
||||
batch_size,
|
||||
horizon,
|
||||
self.config.max_action_dim,
|
||||
dtype=action.dtype,
|
||||
device=self.config.device,
|
||||
)
|
||||
padded[:, :, :action_dim] = action.to(device=self.config.device)
|
||||
mask = torch.zeros(
|
||||
batch_size,
|
||||
horizon,
|
||||
self.config.max_action_dim,
|
||||
dtype=torch.bool,
|
||||
device=self.config.device,
|
||||
)
|
||||
if explicit_mask is None:
|
||||
mask[:, :, :action_dim] = True
|
||||
else:
|
||||
mask[:, :, :action_dim] = explicit_mask.to(device=self.config.device, dtype=torch.bool)
|
||||
return padded.to(dtype=self._compute_dtype), mask
|
||||
|
||||
def _prepare_inference_action_mask(self, batch_size: int) -> Tensor:
|
||||
mask = torch.zeros(
|
||||
batch_size,
|
||||
self.config.max_action_dim,
|
||||
dtype=torch.bool,
|
||||
device=self.config.device,
|
||||
)
|
||||
mask[:, : self._env_action_dim] = True
|
||||
return mask
|
||||
|
||||
def _get_embodiment_ids(self, batch: dict[str, Tensor], batch_size: int) -> Tensor:
|
||||
embodiment_ids = batch.get("embodiment_id")
|
||||
if embodiment_ids is None and self.config.embodiment_id_field:
|
||||
embodiment_ids = batch.get(self.config.embodiment_id_field)
|
||||
if embodiment_ids is None:
|
||||
return torch.full(
|
||||
(batch_size,),
|
||||
self.config.default_embodiment_id,
|
||||
dtype=torch.long,
|
||||
device=self.config.device,
|
||||
)
|
||||
if embodiment_ids.dim() == 0:
|
||||
embodiment_ids = embodiment_ids.unsqueeze(0)
|
||||
elif embodiment_ids.dim() > 1:
|
||||
embodiment_ids = embodiment_ids[:, -1]
|
||||
return embodiment_ids.to(device=self.config.device, dtype=torch.long)
|
||||
|
||||
def _collect_image_batches(self, batch: dict[str, Tensor]) -> tuple[list[list[Tensor]], Tensor]:
|
||||
camera_keys = self._camera_keys or sorted(key for key in batch if key.startswith(f"{OBS_IMAGES}."))
|
||||
if not camera_keys:
|
||||
raise ValueError("EVO1 requires at least one visual observation feature.")
|
||||
first_image = batch[camera_keys[0]]
|
||||
if first_image.dim() == 3:
|
||||
batch_size = 1
|
||||
elif first_image.dim() in (4, 5):
|
||||
batch_size = first_image.shape[0]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported image tensor shape for EVO1: key={camera_keys[0]} shape={tuple(first_image.shape)}"
|
||||
)
|
||||
image_batches: list[list[Tensor]] = []
|
||||
image_masks = torch.zeros(batch_size, self.config.max_views, dtype=torch.bool)
|
||||
|
||||
for batch_index in range(batch_size):
|
||||
sample_images: list[Tensor] = []
|
||||
for camera_key in camera_keys[: self.config.max_views]:
|
||||
image = batch[camera_key]
|
||||
if image.dim() == 3:
|
||||
image = image.unsqueeze(0)
|
||||
elif image.dim() == 5:
|
||||
image = image[:, -1]
|
||||
elif image.dim() != 4:
|
||||
raise ValueError(
|
||||
f"Unsupported image tensor shape for EVO1: key={camera_key} shape={tuple(image.shape)}"
|
||||
)
|
||||
if image.shape[0] != batch_size:
|
||||
raise ValueError(
|
||||
f"EVO1 batch mismatch across cameras: key={camera_key} batch={image.shape[0]} expected={batch_size}"
|
||||
)
|
||||
sample_images.append(image[batch_index].detach().cpu())
|
||||
if not sample_images:
|
||||
raise ValueError("EVO1 received a batch without any image tensor.")
|
||||
while len(sample_images) < self.config.max_views:
|
||||
sample_images.append(torch.zeros_like(sample_images[0]))
|
||||
image_batches.append(sample_images[: self.config.max_views])
|
||||
image_masks[batch_index, : min(len(camera_keys), self.config.max_views)] = True
|
||||
|
||||
return image_batches, image_masks
|
||||
|
||||
def _compute_fused_tokens(
|
||||
self,
|
||||
prompts: list[str],
|
||||
image_batches: list[list[Tensor]],
|
||||
image_masks: Tensor,
|
||||
) -> Tensor:
|
||||
fused_tokens = self.model.get_vl_embeddings(
|
||||
images=image_batches,
|
||||
image_mask=image_masks,
|
||||
prompt=prompts,
|
||||
return_cls_only=self.config.return_cls_only,
|
||||
)
|
||||
return fused_tokens.to(device=self.config.device, dtype=self._compute_dtype)
|
||||
|
||||
def _compute_masked_loss(
|
||||
self,
|
||||
pred_velocity: Tensor,
|
||||
target_velocity: Tensor,
|
||||
action_mask: Tensor,
|
||||
reduction: str,
|
||||
) -> Tensor:
|
||||
flat_mask = action_mask.view(action_mask.shape[0], -1).to(dtype=pred_velocity.dtype)
|
||||
sq_error = ((pred_velocity - target_velocity) * flat_mask).pow(2)
|
||||
active = flat_mask.sum(dim=1).clamp_min(1.0)
|
||||
per_sample_loss = sq_error.sum(dim=1) / active
|
||||
if reduction == "none":
|
||||
return per_sample_loss
|
||||
if reduction != "mean":
|
||||
raise ValueError(f"Unsupported reduction '{reduction}'")
|
||||
return sq_error.sum() / active.sum()
|
||||
|
||||
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
|
||||
prompts = self._normalize_task_batch(batch)
|
||||
image_batches, image_masks = self._collect_image_batches(batch)
|
||||
states, _state_mask = self._prepare_state(batch)
|
||||
actions_gt, action_mask = self._prepare_actions(batch)
|
||||
fused_tokens = self._compute_fused_tokens(prompts, image_batches, image_masks)
|
||||
states = states.to(dtype=self._training_compute_dtype)
|
||||
actions_gt = actions_gt.to(dtype=self._training_compute_dtype)
|
||||
fused_tokens = fused_tokens.to(dtype=self._training_compute_dtype)
|
||||
embodiment_ids = self._get_embodiment_ids(batch, states.shape[0])
|
||||
|
||||
pred_velocity, noise = self.model(
|
||||
fused_tokens,
|
||||
state=states,
|
||||
actions_gt=actions_gt,
|
||||
action_mask=action_mask.to(device=self.config.device, dtype=self._compute_dtype),
|
||||
embodiment_ids=embodiment_ids,
|
||||
)
|
||||
flat_action_mask = action_mask.view(action_mask.shape[0], -1).to(dtype=actions_gt.dtype)
|
||||
target_velocity = (actions_gt - noise).view(actions_gt.shape[0], -1) * flat_action_mask
|
||||
loss = self._compute_masked_loss(pred_velocity, target_velocity, action_mask, reduction)
|
||||
loss_mean = loss.mean().item() if loss.ndim > 0 else loss.item()
|
||||
return loss, {
|
||||
"loss": loss_mean,
|
||||
"active_action_dims": float(action_mask.sum(dim=(1, 2)).float().mean().item()),
|
||||
}
|
||||
|
||||
@torch.no_grad()
|
||||
def predict_action_chunk(self, batch: dict[str, Tensor], **kwargs) -> Tensor:
|
||||
self.eval()
|
||||
|
||||
prompts = self._normalize_task_batch(batch)
|
||||
image_batches, image_masks = self._collect_image_batches(batch)
|
||||
states, _state_mask = self._prepare_state(batch)
|
||||
fused_tokens = self._compute_fused_tokens(prompts, image_batches, image_masks)
|
||||
states = states.to(dtype=self._inference_compute_dtype)
|
||||
fused_tokens = fused_tokens.to(dtype=self._inference_compute_dtype)
|
||||
embodiment_ids = self._get_embodiment_ids(batch, states.shape[0])
|
||||
action_mask = self._prepare_inference_action_mask(states.shape[0])
|
||||
|
||||
with (
|
||||
torch.autocast(device_type="cuda", dtype=torch.bfloat16)
|
||||
if self.config.use_amp and str(self.config.device).startswith("cuda")
|
||||
else nullcontext()
|
||||
):
|
||||
actions = self.model(
|
||||
fused_tokens,
|
||||
state=states,
|
||||
action_mask=action_mask,
|
||||
embodiment_ids=embodiment_ids,
|
||||
)
|
||||
actions = actions.view(states.shape[0], self.config.chunk_size, self.config.max_action_dim)
|
||||
return actions[:, :, : self._env_action_dim]
|
||||
|
||||
@torch.no_grad()
|
||||
def select_action(self, batch: dict[str, Tensor], **kwargs) -> Tensor:
|
||||
self.eval()
|
||||
if len(self._action_queue) == 0:
|
||||
action_chunk = self.predict_action_chunk(batch)[:, : self.config.n_action_steps]
|
||||
self._action_queue.extend(action_chunk.transpose(0, 1))
|
||||
return self._action_queue.popleft()
|
||||
@ -0,0 +1,90 @@
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
|
||||
from lerobot.processor import (
|
||||
AddBatchDimensionProcessorStep,
|
||||
DeviceProcessorStep,
|
||||
NormalizerProcessorStep,
|
||||
PolicyAction,
|
||||
PolicyProcessorPipeline,
|
||||
RenameObservationsProcessorStep,
|
||||
UnnormalizerProcessorStep,
|
||||
)
|
||||
from lerobot.processor.converters import (
|
||||
batch_to_transition,
|
||||
create_transition,
|
||||
policy_action_to_transition,
|
||||
transition_to_policy_action,
|
||||
)
|
||||
from lerobot.utils.constants import (
|
||||
ACTION,
|
||||
DONE,
|
||||
INFO,
|
||||
OBS_PREFIX,
|
||||
POLICY_POSTPROCESSOR_DEFAULT_NAME,
|
||||
POLICY_PREPROCESSOR_DEFAULT_NAME,
|
||||
REWARD,
|
||||
TRUNCATED,
|
||||
)
|
||||
|
||||
|
||||
def evo1_batch_to_transition(batch: dict[str, Any]):
|
||||
transition = batch_to_transition(batch)
|
||||
complementary_data = dict(transition.get("complementary_data") or {})
|
||||
reserved = {ACTION, REWARD, DONE, TRUNCATED, INFO}
|
||||
for key, value in batch.items():
|
||||
if key in reserved or key.startswith(OBS_PREFIX):
|
||||
continue
|
||||
complementary_data.setdefault(key, value)
|
||||
return create_transition(
|
||||
observation=transition.get("observation"),
|
||||
action=transition.get("action"),
|
||||
reward=transition.get("reward", 0.0),
|
||||
done=transition.get("done", False),
|
||||
truncated=transition.get("truncated", False),
|
||||
info=transition.get("info", {}),
|
||||
complementary_data=complementary_data,
|
||||
)
|
||||
|
||||
|
||||
def make_evo1_pre_post_processors(
|
||||
config: Evo1Config,
|
||||
dataset_stats: dict[str, dict[str, torch.Tensor]] | None = None,
|
||||
) -> tuple[
|
||||
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
]:
|
||||
input_steps = [
|
||||
RenameObservationsProcessorStep(rename_map={}),
|
||||
AddBatchDimensionProcessorStep(),
|
||||
NormalizerProcessorStep(
|
||||
features={**config.input_features, **config.output_features},
|
||||
norm_map=config.normalization_mapping,
|
||||
stats=dataset_stats,
|
||||
),
|
||||
DeviceProcessorStep(device=config.device),
|
||||
]
|
||||
output_steps = [
|
||||
UnnormalizerProcessorStep(
|
||||
features=config.output_features,
|
||||
norm_map=config.normalization_mapping,
|
||||
stats=dataset_stats,
|
||||
),
|
||||
DeviceProcessorStep(device="cpu"),
|
||||
]
|
||||
|
||||
return (
|
||||
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
|
||||
steps=input_steps,
|
||||
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
|
||||
to_transition=evo1_batch_to_transition,
|
||||
),
|
||||
PolicyProcessorPipeline[PolicyAction, PolicyAction](
|
||||
steps=output_steps,
|
||||
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
|
||||
to_transition=policy_action_to_transition,
|
||||
to_output=transition_to_policy_action,
|
||||
),
|
||||
)
|
||||
117
src/lerobot/policies/evo1/移植说明.md
普通文件
117
src/lerobot/policies/evo1/移植说明.md
普通文件
@ -0,0 +1,117 @@
|
||||
# Evo1 移植说明
|
||||
|
||||
这份说明主要给后续接手同学快速了解:Evo1 是怎么接入到 Evo-RL / LeRobot 训练体系里的,以及这次移植时主要做了什么。
|
||||
|
||||
## 1. 总体思路
|
||||
|
||||
这次移植没有重写 Evo1 的模型主体,而是沿用了原始 Evo1 已经比较贴近 LeRobot 的那一部分设计:
|
||||
|
||||
- policy 本体仍然是 Evo1
|
||||
- VLM 仍然是 InternVL3
|
||||
- action head 仍然是 flow matching
|
||||
- `stage1 / stage2` 两阶段训练逻辑仍然保留
|
||||
|
||||
真正的工作重点不在“改模型”,而在于把原始仓库里那套比较集中的训练逻辑,拆开并接到 LeRobot 现有训练基础设施里。
|
||||
|
||||
## 2. 为什么训练接入相对自然
|
||||
|
||||
原始 Evo1 在推理侧本来就已经有一些 LeRobot 风格的组织方式,比如:
|
||||
|
||||
- policy / processor 的概念是比较明确的
|
||||
- checkpoint 也能整理成标准的模型目录
|
||||
- 两阶段训练本身就是围绕同一个 policy 展开的
|
||||
|
||||
所以这次接入并不是从零抽象一个新 policy,而更像是:
|
||||
|
||||
1. 保留 Evo1 模型实现
|
||||
2. 把原始大训练脚本中的训练逻辑抽出来
|
||||
3. 接到 LeRobot 的 `make_policy`、`lerobot_train`、processor 保存与加载、dataset 接口、checkpoint 体系里
|
||||
|
||||
## 3. 具体做了什么
|
||||
|
||||
可以把这次工作理解为三层。
|
||||
|
||||
### 模型层
|
||||
|
||||
- 在 `src/lerobot/policies/evo1/` 下接入 Evo1 config、model、processor、flow matching action head。
|
||||
- 保留了原始 Evo1 的关键配置项,比如:
|
||||
- `training_stage`
|
||||
- `vlm_model_name`
|
||||
- `num_layers`
|
||||
- `chunk_size`
|
||||
- `n_action_steps`
|
||||
- `max_action_dim`
|
||||
- `max_state_dim`
|
||||
|
||||
### 训练层
|
||||
|
||||
- 不再使用原始 Evo1 那个集中的训练脚本作为日常训练入口。
|
||||
- 改为通过 LeRobot 统一训练入口 `lerobot_train` 启动 Evo1。
|
||||
- 在训练配置层面补了:
|
||||
- `stage1`
|
||||
- `stage2`
|
||||
- `resume_pretrain`
|
||||
|
||||
其中:
|
||||
|
||||
- `stage1`:冻结 VLM,只训练 action head
|
||||
- `stage2`:继续训练 VLM + action head
|
||||
- `resume_pretrain=true`:用于从 stage1 权重切到 stage2,但不恢复 optimizer / scheduler / step
|
||||
|
||||
### 工程集成层
|
||||
|
||||
- Evo1 被接进了 LeRobot 的 policy registry
|
||||
- 训练、推理、record 都能通过统一的 `make_policy` / processor 体系加载
|
||||
- checkpoint 会保存成 LeRobot 标准格式:
|
||||
- `config.json`
|
||||
- `model.safetensors`
|
||||
- `policy_preprocessor.json`
|
||||
- `policy_postprocessor.json`
|
||||
|
||||
这样后面:
|
||||
|
||||
- 训练完成后直接推理
|
||||
- human-in-loop record 里加载 policy
|
||||
- stage1 切 stage2
|
||||
|
||||
都会更自然,不需要再写很多原始 Evo1 仓库特有的胶水代码。
|
||||
|
||||
## 4. 与原始 Evo1 训练实现的关系
|
||||
|
||||
要注意,这次移植不是逐行复刻原始训练脚本,而是做了“语义尽量一致、工程上接入 LeRobot”的版本。
|
||||
|
||||
换句话说:
|
||||
|
||||
- 核心模型和两阶段训练思路是沿用原始 Evo1 的
|
||||
- 但训练执行框架已经变成 Evo-RL / LeRobot 这套
|
||||
|
||||
这意味着:
|
||||
|
||||
- 训练配置方式不同
|
||||
- checkpoint 组织方式不同
|
||||
- mixed precision / distributed wrapper 的工程边界也会和原始仓库不完全一样
|
||||
|
||||
所以如果后续要做“原始 Evo1 vs 接入版 Evo-RL Evo1”的严格对照实验,需要把:
|
||||
|
||||
- 数据
|
||||
- batch size
|
||||
- 卡数
|
||||
- precision
|
||||
- stage 切换方式
|
||||
|
||||
都尽量对齐后再看结果。
|
||||
|
||||
## 5. 当前建议的理解方式
|
||||
|
||||
最适合把现在这版 Evo1 接入看成:
|
||||
|
||||
- **不是一个临时脚本适配**
|
||||
- 而是把 Evo1 真正注册成了 LeRobot 里的一个 policy
|
||||
|
||||
这样后续无论是:
|
||||
|
||||
- 继续补多 embodiment
|
||||
- 对齐原始 Evo1 的训练细节
|
||||
- 修复 processor / tokenizer / checkpoint 可移植性问题
|
||||
|
||||
都可以在 LeRobot 统一框架里继续演进,而不用回到原始仓库那种“大而全训练脚本”上反复打补丁。
|
||||
@ -31,6 +31,7 @@ from lerobot.envs.configs import EnvConfig
|
||||
from lerobot.envs.utils import env_to_policy_features
|
||||
from lerobot.policies.act.configuration_act import ACTConfig
|
||||
from lerobot.policies.diffusion.configuration_diffusion import DiffusionConfig
|
||||
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
|
||||
from lerobot.policies.groot.configuration_groot import GrootConfig
|
||||
from lerobot.policies.pi0.configuration_pi0 import PI0Config
|
||||
from lerobot.policies.pi05.configuration_pi05 import PI05Config
|
||||
@ -103,6 +104,10 @@ def get_policy_class(name: str) -> type[PreTrainedPolicy]:
|
||||
from lerobot.policies.pi05.modeling_pi05 import PI05Policy
|
||||
|
||||
return PI05Policy
|
||||
elif name == "evo1":
|
||||
from lerobot.policies.evo1.modeling_evo1 import EVO1Policy
|
||||
|
||||
return EVO1Policy
|
||||
elif name == "sac":
|
||||
from lerobot.policies.sac.modeling_sac import SACPolicy
|
||||
|
||||
@ -169,6 +174,8 @@ def make_policy_config(policy_type: str, **kwargs) -> PreTrainedConfig:
|
||||
return PI0Config(**kwargs)
|
||||
elif policy_type == "pi05":
|
||||
return PI05Config(**kwargs)
|
||||
elif policy_type == "evo1":
|
||||
return Evo1Config(**kwargs)
|
||||
elif policy_type == "sac":
|
||||
return SACConfig(**kwargs)
|
||||
elif policy_type == "smolvla":
|
||||
@ -333,6 +340,14 @@ def make_pre_post_processors(
|
||||
dataset_stats=kwargs.get("dataset_stats"),
|
||||
)
|
||||
|
||||
elif isinstance(policy_cfg, Evo1Config):
|
||||
from lerobot.policies.evo1.processor_evo1 import make_evo1_pre_post_processors
|
||||
|
||||
processors = make_evo1_pre_post_processors(
|
||||
config=policy_cfg,
|
||||
dataset_stats=kwargs.get("dataset_stats"),
|
||||
)
|
||||
|
||||
elif isinstance(policy_cfg, SACConfig):
|
||||
from lerobot.policies.sac.processor_sac import make_sac_pre_post_processors
|
||||
|
||||
|
||||
@ -267,6 +267,12 @@ def train(
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating policy")
|
||||
if cfg.resume_pretrain:
|
||||
logging.info(
|
||||
"resume_pretrain=true: policy weights will be initialized from %s, while optimizer,"
|
||||
" scheduler, and global step will start fresh.",
|
||||
cfg.policy.pretrained_path,
|
||||
)
|
||||
policy = make_policy(
|
||||
cfg=cfg.policy,
|
||||
ds_meta=dataset.meta,
|
||||
@ -282,7 +288,9 @@ def train(
|
||||
# Wait for all processes to finish policy creation before continuing
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
# Create processors - only provide dataset_stats if not resuming from saved processors
|
||||
# Create processors. `resume_pretrain` intentionally follows the fresh-start path:
|
||||
# load policy weights from `pretrained_path`, but rebuild processor state from the
|
||||
# current dataset and do not restore optimizer / scheduler / global step.
|
||||
processor_kwargs = {}
|
||||
postprocessor_kwargs = {}
|
||||
if (cfg.policy.pretrained_path and not cfg.resume) or not cfg.policy.pretrained_path:
|
||||
@ -393,7 +401,7 @@ def train(
|
||||
shuffle=shuffle and not cfg.dataset.streaming,
|
||||
sampler=sampler,
|
||||
pin_memory=device.type == "cuda",
|
||||
drop_last=False,
|
||||
drop_last=getattr(cfg.policy, "drop_last", False),
|
||||
prefetch_factor=2 if cfg.num_workers > 0 else None,
|
||||
)
|
||||
|
||||
@ -402,6 +410,14 @@ def train(
|
||||
policy, optimizer, dataloader, lr_scheduler = accelerator.prepare(
|
||||
policy, optimizer, dataloader, lr_scheduler
|
||||
)
|
||||
if (
|
||||
accelerator.num_processes > 1
|
||||
and cfg.policy.type == "evo1"
|
||||
and getattr(cfg.policy, "training_stage", None) == "stage2"
|
||||
and hasattr(policy, "_set_static_graph")
|
||||
):
|
||||
logging.info("Enabling DDP static graph for Evo1 stage2.")
|
||||
policy._set_static_graph()
|
||||
dl_iter = cycle(dataloader)
|
||||
|
||||
policy.train()
|
||||
|
||||
在新工单中引用
屏蔽一个用户