feat(evo1): integrate evo1 with lerobot training

这个提交包含在:
javadcc_mac
2026-03-20 13:05:02 +08:00
父节点 0b720b1e0b
当前提交 078216b032
共有 17 个文件被更改,包括 2432 次插入 和 1 次删除

查看文件

@ -54,6 +54,9 @@ class TrainPipelineConfig(HubMixin):
# Note that when resuming a run, the default behavior is to use the configuration from the checkpoint,
# regardless of what's provided with the training command at the time of resumption.
resume: bool = False
# `resume_pretrain` mirrors Evo-1's stage2 handoff semantics: initialize from policy weights only while
# starting a fresh training run with a new optimizer, scheduler and step counter.
resume_pretrain: bool = False
# `seed` is used for training (eg: model initialization, dataset shuffling)
# AND for the evaluation environments.
seed: int | None = 1000
@ -89,6 +92,16 @@ class TrainPipelineConfig(HubMixin):
def validate(self) -> None:
# HACK: We parse again the cli args here to get the pretrained paths if there was some.
policy_path = parser.get_path_arg("policy")
if self.resume and self.resume_pretrain:
raise ValueError("'resume' and 'resume_pretrain' are mutually exclusive.")
if self.resume_pretrain and not policy_path and (
self.policy is None or self.policy.pretrained_path is None
):
raise ValueError(
"'resume_pretrain=true' expects weights via '--policy.path' or 'policy.pretrained_path'."
)
if policy_path:
# Only load the policy config
cli_overrides = parser.get_cli_overrides("policy")

查看文件

@ -0,0 +1 @@
"""LeRobot v2.1 compatibility helpers."""

查看文件

@ -0,0 +1,529 @@
#!/usr/bin/env python
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Export a LeRobot v3.0 dataset into a legacy v2.1-style layout that Evo1 can train on.
This script is intentionally pragmatic: it restores the files that Evo1's current dataset
loader actually consumes.
It recreates:
- meta/tasks.jsonl
- meta/episodes.jsonl
- meta/episodes_stats.jsonl
- meta/stats.json
- data/chunk-xxx/episode_xxxxxx.parquet
- videos/chunk-xxx/<camera>/episode_xxxxxx.mp4
It can also optionally rewrite task strings using ACP indicators so the exported dataset
preserves Evo-RL's prompt-conditioning semantics without changing Evo1's training code.
"""
import argparse
import copy
import json
import logging
import shutil
from collections import Counter
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.utils import (
DEFAULT_CHUNK_SIZE,
LEGACY_EPISODES_PATH,
LEGACY_EPISODES_STATS_PATH,
LEGACY_TASKS_PATH,
load_info,
load_stats,
load_tasks,
serialize_dict,
unflatten_dict,
write_info,
write_stats,
)
from lerobot.datasets.dataset_tools import _keep_episodes_from_video_with_av
from lerobot.rl.acp_tags import build_acp_tagged_task
from lerobot.utils.constants import HF_LEROBOT_HOME
from lerobot.utils.utils import init_logging
V21 = "v2.1"
V30 = "v3.0"
def write_jsonl(records: list[dict[str, Any]], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as f:
for record in records:
f.write(json.dumps(make_json_ready(record), ensure_ascii=False))
f.write("\n")
def make_json_ready(value: Any) -> Any:
if isinstance(value, dict):
return {key: make_json_ready(subvalue) for key, subvalue in value.items()}
if hasattr(value, "tolist") and not isinstance(value, (str, bytes)):
converted = value.tolist()
if converted is not value:
return make_json_ready(converted)
if isinstance(value, list):
return [make_json_ready(item) for item in value]
if isinstance(value, tuple):
return [make_json_ready(item) for item in value]
if isinstance(value, np.ndarray):
return value.tolist()
if isinstance(value, np.generic):
return value.item()
return value
def legacy_chunk_dir(episode_index: int) -> str:
return f"chunk-{episode_index // DEFAULT_CHUNK_SIZE:03d}"
def legacy_data_file_path(root: Path, episode_index: int) -> Path:
return root / "data" / legacy_chunk_dir(episode_index) / f"episode_{episode_index:06d}.parquet"
def legacy_video_file_path(root: Path, video_key: str, episode_index: int) -> Path:
return root / "videos" / legacy_chunk_dir(episode_index) / video_key / f"episode_{episode_index:06d}.mp4"
def resolve_source_root(
src_root: str | None,
repo_id: str | None,
root: str | None,
episodes: list[int] | None,
) -> Path:
if src_root is not None:
resolved = Path(src_root).expanduser().resolve()
if not resolved.exists():
raise FileNotFoundError(f"Source dataset root does not exist: {resolved}")
return resolved
if repo_id is None:
raise ValueError("Provide either --src-root or --repo-id.")
base_root = Path(root).expanduser().resolve() if root is not None else HF_LEROBOT_HOME.resolve()
dataset_root = base_root / repo_id
LeRobotDataset(repo_id=repo_id, root=dataset_root, episodes=episodes, revision=V30, download_videos=True)
return dataset_root.resolve()
def build_exported_task_maps(
tasks_df: pd.DataFrame, acp_indicator_field: str | None
) -> tuple[dict[int, str], dict[tuple[int, int], int] | None]:
source_tasks: list[tuple[int, str]] = []
ordered_tasks = tasks_df.sort_values("task_index")
for task_text, row in ordered_tasks.iterrows():
source_tasks.append((int(row["task_index"]), "" if task_text is None else str(task_text)))
if acp_indicator_field is None:
task_by_index = {task_index: task_text for task_index, task_text in source_tasks}
return task_by_index, None
task_by_index: dict[int, str] = {}
pair_to_exported: dict[tuple[int, int], int] = {}
next_task_index = 0
for task_index, task_text in source_tasks:
for indicator in (1, 0):
pair_to_exported[(task_index, indicator)] = next_task_index
task_by_index[next_task_index] = build_acp_tagged_task(task_text, is_positive=bool(indicator))
next_task_index += 1
return task_by_index, pair_to_exported
def ensure_selected_episodes(episodes_df: pd.DataFrame, episodes: list[int] | None) -> pd.DataFrame:
episodes_df = episodes_df.sort_values("episode_index").reset_index(drop=True)
if episodes is None:
return episodes_df
selected = set(episodes)
filtered = episodes_df[episodes_df["episode_index"].isin(selected)].copy()
found = {int(ep) for ep in filtered["episode_index"].tolist()}
missing = sorted(selected - found)
if missing:
raise ValueError(f"Requested episode indices not found in metadata: {missing}")
return filtered.sort_values("episode_index").reset_index(drop=True)
def load_episodes_dataframe(src_root: Path) -> pd.DataFrame:
episode_paths = sorted((src_root / "meta" / "episodes").glob("*/*.parquet"))
if not episode_paths:
raise FileNotFoundError(f"No episode metadata parquet files found under {src_root / 'meta' / 'episodes'}")
episode_frames = [pd.read_parquet(path) for path in episode_paths]
return pd.concat(episode_frames, ignore_index=True)
def normalize_indicator_series(series: pd.Series, field_name: str) -> pd.Series:
if series.isna().any():
raise ValueError(f"ACP indicator field '{field_name}' contains null values.")
invalid = series[~series.isin([0, 1])]
if not invalid.empty:
values = sorted({repr(value) for value in invalid.tolist()[:8]})
raise ValueError(f"ACP indicator field '{field_name}' must contain only 0/1 values, got {values}")
return series.astype(int)
def export_data_files(
src_root: Path,
dst_root: Path,
info: dict[str, Any],
episodes: list[int] | None,
task_by_index: dict[int, str],
acp_indicator_field: str | None,
acp_task_remap: dict[tuple[int, int], int] | None,
) -> tuple[dict[int, list[str]], set[int]]:
selected = set(episodes) if episodes is not None else None
visual_feature_columns = {
key for key, feature in info["features"].items() if feature["dtype"] in {"image", "video"}
}
episode_tasks: dict[int, list[str]] = {}
used_task_indices: set[int] = set()
for parquet_path in sorted((src_root / "data").glob("*/*.parquet")):
df = pd.read_parquet(parquet_path)
if selected is not None:
df = df[df["episode_index"].isin(selected)]
if df.empty:
continue
drop_columns = [column for column in visual_feature_columns if column in df.columns]
if drop_columns:
df = df.drop(columns=drop_columns)
if "task_index" not in df.columns:
raise KeyError(f"Expected 'task_index' column in {parquet_path}")
if acp_indicator_field is not None:
if acp_task_remap is None:
raise ValueError("ACP task remap is missing while ACP export is enabled.")
if acp_indicator_field not in df.columns:
raise KeyError(f"ACP indicator field '{acp_indicator_field}' not found in {parquet_path}")
indicators = normalize_indicator_series(df[acp_indicator_field], acp_indicator_field)
if df["task_index"].isna().any():
raise ValueError(f"'task_index' contains null values in {parquet_path}")
remapped_indices = [
acp_task_remap[(int(task_index), int(indicator))]
for task_index, indicator in zip(df["task_index"].tolist(), indicators.tolist(), strict=False)
]
df = df.copy()
df["task_index"] = remapped_indices
for episode_index, episode_df in df.groupby("episode_index", sort=True):
episode_index = int(episode_index)
output_path = legacy_data_file_path(dst_root, episode_index)
output_path.parent.mkdir(parents=True, exist_ok=True)
episode_df.to_parquet(output_path, index=False)
task_indices = [int(task_index) for task_index in episode_df["task_index"].tolist()]
used_task_indices.update(task_indices)
episode_tasks[episode_index] = list(
dict.fromkeys(task_by_index[task_index] for task_index in task_indices if task_index in task_by_index)
)
return episode_tasks, used_task_indices
def export_tasks_jsonl(dst_root: Path, task_by_index: dict[int, str], used_task_indices: set[int]) -> None:
records = [
{"task_index": int(task_index), "task": task_by_index[task_index]}
for task_index in sorted(used_task_indices)
]
write_jsonl(records, dst_root / LEGACY_TASKS_PATH)
def flatten_episode_tasks(task_value: Any) -> list[str]:
if hasattr(task_value, "tolist") and not isinstance(task_value, (str, bytes)):
return flatten_episode_tasks(task_value.tolist())
if isinstance(task_value, list):
flattened: list[str] = []
for value in task_value:
flattened.extend(flatten_episode_tasks(value))
return flattened
if task_value is None:
return []
return [str(task_value)]
def export_episodes_jsonl(dst_root: Path, episodes_df: pd.DataFrame, episode_tasks: dict[int, list[str]]) -> None:
records: list[dict[str, Any]] = []
for _, row in episodes_df.iterrows():
episode_index = int(row["episode_index"])
tasks = episode_tasks.get(episode_index)
if tasks is None:
tasks = flatten_episode_tasks(row.get("tasks"))
records.append(
{
"episode_index": episode_index,
"tasks": tasks,
"length": int(row["length"]),
}
)
write_jsonl(records, dst_root / LEGACY_EPISODES_PATH)
def export_episode_stats_jsonl(src_root: Path, dst_root: Path, episodes: list[int] | None) -> None:
selected = set(episodes) if episodes is not None else None
records: list[dict[str, Any]] = []
for parquet_path in sorted((src_root / "meta" / "episodes").glob("*/*.parquet")):
df = pd.read_parquet(parquet_path)
if selected is not None:
df = df[df["episode_index"].isin(selected)]
if df.empty:
continue
stats_columns = [column for column in df.columns if column.startswith("stats/")]
if not stats_columns:
continue
for _, row in df.sort_values("episode_index").iterrows():
flat_stats = {column.removeprefix("stats/"): row[column] for column in stats_columns}
records.append(
{
"episode_index": int(row["episode_index"]),
"stats": serialize_dict(unflatten_dict(flat_stats)),
}
)
if records:
write_jsonl(records, dst_root / LEGACY_EPISODES_STATS_PATH)
def export_videos(
src_root: Path,
dst_root: Path,
info: dict[str, Any],
episodes_df: pd.DataFrame,
vcodec: str,
pix_fmt: str,
) -> None:
video_keys = sorted([key for key, feature in info["features"].items() if feature["dtype"] == "video"])
if not video_keys:
logging.warning("Source dataset has no video features; skipping video export.")
return
if info["video_path"] is None:
raise ValueError("Source dataset declares video features but info['video_path'] is None.")
fps = float(info["fps"])
file_usage: dict[str, Counter[tuple[int, int]]] = {video_key: Counter() for video_key in video_keys}
for _, row in episodes_df.iterrows():
for video_key in video_keys:
file_usage[video_key][
(int(row[f"videos/{video_key}/chunk_index"]), int(row[f"videos/{video_key}/file_index"]))
] += 1
for video_key in video_keys:
logging.info(f"Exporting legacy videos for {video_key}")
for _, row in episodes_df.iterrows():
episode_index = int(row["episode_index"])
chunk_index = int(row[f"videos/{video_key}/chunk_index"])
file_index = int(row[f"videos/{video_key}/file_index"])
from_timestamp = float(row[f"videos/{video_key}/from_timestamp"])
to_timestamp = float(row[f"videos/{video_key}/to_timestamp"])
src_video_rel = info["video_path"].format(
video_key=video_key,
chunk_index=chunk_index,
file_index=file_index,
)
src_video_path = src_root / src_video_rel
if not src_video_path.exists():
raise FileNotFoundError(f"Missing source video file: {src_video_path}")
dst_video_path = legacy_video_file_path(dst_root, video_key, episode_index)
dst_video_path.parent.mkdir(parents=True, exist_ok=True)
if file_usage[video_key][(chunk_index, file_index)] == 1 and abs(from_timestamp) < 1e-6:
shutil.copy(src_video_path, dst_video_path)
continue
_keep_episodes_from_video_with_av(
input_path=src_video_path,
output_path=dst_video_path,
episodes_to_keep=[(from_timestamp, to_timestamp)],
fps=fps,
vcodec=vcodec,
pix_fmt=pix_fmt,
)
def build_legacy_info(
info: dict[str, Any],
episodes_df: pd.DataFrame,
used_task_indices: set[int],
acp_indicator_field: str | None,
) -> dict[str, Any]:
legacy_info = copy.deepcopy(info)
legacy_info["codebase_version"] = V21
legacy_info["total_episodes"] = int(len(episodes_df))
legacy_info["total_frames"] = int(episodes_df["length"].sum())
legacy_info["total_tasks"] = int(len(used_task_indices))
legacy_info["total_chunks"] = int((max(episodes_df["episode_index"]) // DEFAULT_CHUNK_SIZE) + 1) if len(episodes_df) else 0
legacy_info["total_videos"] = int(
len([key for key, feature in info["features"].items() if feature["dtype"] == "video"]) * len(episodes_df)
)
legacy_info["data_path"] = "data/chunk-{chunk_index:03d}/episode_{episode_index:06d}.parquet"
legacy_info["video_path"] = (
"videos/chunk-{chunk_index:03d}/{video_key}/episode_{episode_index:06d}.mp4"
if info["video_path"] is not None
else None
)
if acp_indicator_field is not None:
legacy_info["acp_export_indicator_field"] = acp_indicator_field
return legacy_info
def convert_dataset(
src_root: Path,
dst_root: Path,
episodes: list[int] | None,
acp_indicator_field: str | None,
force: bool,
vcodec: str,
pix_fmt: str,
) -> None:
info = load_info(src_root)
if info.get("codebase_version") != V30:
raise ValueError(
f"Source dataset at {src_root} has codebase_version={info.get('codebase_version')!r}, expected {V30!r}."
)
tasks_df = load_tasks(src_root)
episodes_df = ensure_selected_episodes(load_episodes_dataframe(src_root), episodes)
if dst_root.exists():
if not force:
raise FileExistsError(f"Destination already exists: {dst_root}. Use --force to overwrite.")
shutil.rmtree(dst_root)
dst_root.mkdir(parents=True, exist_ok=True)
task_by_index, acp_task_remap = build_exported_task_maps(tasks_df, acp_indicator_field)
episode_tasks, used_task_indices = export_data_files(
src_root=src_root,
dst_root=dst_root,
info=info,
episodes=episodes,
task_by_index=task_by_index,
acp_indicator_field=acp_indicator_field,
acp_task_remap=acp_task_remap,
)
if not used_task_indices:
raise ValueError("No frames were exported. Check --episodes or the source dataset contents.")
export_tasks_jsonl(dst_root, task_by_index, used_task_indices)
export_episodes_jsonl(dst_root, episodes_df, episode_tasks)
stats = load_stats(src_root)
if stats is not None:
write_stats(stats, dst_root)
export_episode_stats_jsonl(src_root, dst_root, episodes)
export_videos(
src_root=src_root,
dst_root=dst_root,
info=info,
episodes_df=episodes_df,
vcodec=vcodec,
pix_fmt=pix_fmt,
)
legacy_info = build_legacy_info(info, episodes_df, used_task_indices, acp_indicator_field)
write_info(legacy_info, dst_root)
logging.info("Export completed")
logging.info(f" source: {src_root}")
logging.info(f" output: {dst_root}")
logging.info(f" episodes: {len(episodes_df)}")
logging.info(f" tasks: {len(used_task_indices)}")
def main() -> None:
init_logging()
parser = argparse.ArgumentParser()
parser.add_argument("--src-root", type=str, default=None, help="Local v3.0 dataset root to export.")
parser.add_argument("--repo-id", type=str, default=None, help="Optional HF dataset repo id to download first.")
parser.add_argument(
"--root",
type=str,
default=None,
help="Base directory for HF downloads when --repo-id is used. Defaults to HF_LEROBOT_HOME.",
)
parser.add_argument(
"--dst-root",
type=str,
required=True,
help="Output directory for the exported legacy dataset.",
)
parser.add_argument(
"--episodes",
type=int,
nargs="*",
default=None,
help="Optional episode indices to export. Useful for smoke tests and partial exports.",
)
parser.add_argument(
"--acp-indicator-field",
type=str,
default=None,
help="If set, rewrite task strings/task_index using this 0/1 frame field, e.g. complementary_info.acp_indicator.",
)
parser.add_argument(
"--force",
action="store_true",
help="Overwrite the destination directory if it already exists.",
)
parser.add_argument(
"--vcodec",
type=str,
default="h264",
choices=["h264", "hevc", "libsvtav1"],
help="Codec used when slicing shared source video files into per-episode legacy files.",
)
parser.add_argument(
"--pix-fmt",
type=str,
default="yuv420p",
help="Pixel format used when re-encoding legacy video clips.",
)
args = parser.parse_args()
src_root = resolve_source_root(args.src_root, args.repo_id, args.root, args.episodes)
dst_root = Path(args.dst_root).expanduser().resolve()
convert_dataset(
src_root=src_root,
dst_root=dst_root,
episodes=args.episodes,
acp_indicator_field=args.acp_indicator_field,
force=args.force,
vcodec=args.vcodec,
pix_fmt=args.pix_fmt,
)
if __name__ == "__main__":
main()

查看文件

@ -14,6 +14,7 @@
from .act.configuration_act import ACTConfig as ACTConfig
from .diffusion.configuration_diffusion import DiffusionConfig as DiffusionConfig
from .evo1.configuration_evo1 import Evo1Config as Evo1Config
from .groot.configuration_groot import GrootConfig as GrootConfig
from .pi0.configuration_pi0 import PI0Config as PI0Config
from .pi0_fast.configuration_pi0_fast import PI0FastConfig as PI0FastConfig
@ -28,6 +29,7 @@ from .xvla.configuration_xvla import XVLAConfig as XVLAConfig
__all__ = [
"ACTConfig",
"DiffusionConfig",
"Evo1Config",
"PI0Config",
"PI05Config",
"PI0FastConfig",

查看文件

@ -0,0 +1,20 @@
# Evo1 Known Issues
## Current scope
- Evo1 in Evo-RL only targets new LeRobot-format checkpoints saved with `config.json`, `model.safetensors`, and processor configs.
- Legacy Evo1 / DeepSpeed checkpoints are intentionally out of scope and are no longer supported by this integration.
## Precision
- Evo1 training is expected to run with `bf16` mixed precision to match the original Evo1 recipe.
- Use the dedicated accelerate config in this directory when launching training to keep the precision path aligned.
## Multi-embodiment training
- The model path still accepts `embodiment_id` and `num_categories`, but the current towel dataset does not contain an `embodiment_id` field.
- With the current data, Evo1 training always falls back to `default_embodiment_id=0`.
- Multi-repo multi-robot training is still blocked at the training config layer.
- A full multi-embodiment setup would still need:
- a stable `embodiment_id` field in the dataset or a deterministic mapping from dataset identity
- per-robot normalization instead of a single aggregated stats table

查看文件

@ -0,0 +1,5 @@
from .configuration_evo1 import Evo1Config
from .modeling_evo1 import EVO1Policy
from .processor_evo1 import make_evo1_pre_post_processors
__all__ = ["Evo1Config", "EVO1Policy", "make_evo1_pre_post_processors"]

查看文件

@ -0,0 +1,16 @@
compute_environment: LOCAL_MACHINE
debug: false
distributed_type: MULTI_GPU
downcast_bf16: "no"
enable_cpu_affinity: false
machine_rank: 0
main_training_function: main
mixed_precision: bf16
num_machines: 1
num_processes: 4
rdzv_backend: static
same_network: true
tpu_env: []
tpu_use_cluster: false
tpu_use_sudo: false
use_cpu: false

查看文件

@ -0,0 +1,16 @@
compute_environment: LOCAL_MACHINE
debug: false
distributed_type: "NO"
downcast_bf16: "no"
enable_cpu_affinity: false
machine_rank: 0
main_training_function: main
mixed_precision: bf16
num_machines: 1
num_processes: 1
rdzv_backend: static
same_network: true
tpu_env: []
tpu_use_cluster: false
tpu_use_sudo: false
use_cpu: false

查看文件

@ -0,0 +1,151 @@
import math
from dataclasses import dataclass, field
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
from lerobot.optim.optimizers import AdamWConfig
from lerobot.optim.schedulers import LRSchedulerConfig
from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR
from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE
@LRSchedulerConfig.register_subclass("evo1_exact")
@dataclass
class Evo1SchedulerConfig(LRSchedulerConfig):
num_warmup_steps: int
def build(self, optimizer: Optimizer, num_training_steps: int) -> LambdaLR:
def lr_lambda(current_step: int) -> float:
if current_step < self.num_warmup_steps:
return current_step / max(1, self.num_warmup_steps)
progress = (current_step - self.num_warmup_steps) / max(
1, num_training_steps - self.num_warmup_steps
)
return max(0.0, 0.5 * (1.0 + math.cos(math.pi * progress)))
return LambdaLR(optimizer, lr_lambda, -1)
@PreTrainedConfig.register_subclass("evo1")
@dataclass
class Evo1Config(PreTrainedConfig):
training_stage: str = "stage1"
use_amp: bool = True
n_obs_steps: int = 1
chunk_size: int = 50
n_action_steps: int = 50
max_state_dim: int = 24
max_action_dim: int = 24
max_views: int = 3
image_resolution: tuple[int, int] = (448, 448)
empty_cameras: int = 0
normalization_mapping: dict[str, NormalizationMode] = field(
default_factory=lambda: {
"VISUAL": NormalizationMode.IDENTITY,
"STATE": NormalizationMode.MIN_MAX,
"ACTION": NormalizationMode.MIN_MAX,
}
)
vlm_model_name: str = "OpenGVLab/InternVL3-1B"
action_head: str = "flowmatching"
embed_dim: int = 896
hidden_dim: int = 1024
state_hidden_dim: int = 1024
num_heads: int = 8
num_layers: int = 8
dropout: float = 0.0
num_inference_timesteps: int = 32
num_categories: int = 1
return_cls_only: bool = False
finetune_vlm: bool = False
finetune_action_head: bool = True
task_field: str = "task"
embodiment_id_field: str | None = None
default_embodiment_id: int = 0
optimizer_lr: float = 1e-5
optimizer_betas: tuple[float, float] = (0.9, 0.999)
optimizer_eps: float = 1e-8
optimizer_weight_decay: float = 1e-5
optimizer_grad_clip_norm: float = 1.0
scheduler_warmup_steps: int = 300
drop_last: bool = True
def __post_init__(self):
super().__post_init__()
if self.training_stage not in {"stage1", "stage2"}:
raise ValueError(
f"Unsupported EVO1 training_stage '{self.training_stage}', expected 'stage1' or 'stage2'"
)
if self.training_stage == "stage1":
self.finetune_vlm = False
self.finetune_action_head = True
elif self.training_stage == "stage2":
self.finetune_vlm = True
self.finetune_action_head = True
if self.n_action_steps > self.chunk_size:
raise ValueError(
f"n_action_steps ({self.n_action_steps}) must be <= chunk_size ({self.chunk_size})"
)
def validate_features(self) -> None:
if self.input_features is None:
self.input_features = {}
if self.output_features is None:
self.output_features = {}
for i in range(self.empty_cameras):
key = OBS_IMAGES + f".empty_camera_{i}"
if key not in self.input_features:
self.input_features[key] = PolicyFeature(
type=FeatureType.VISUAL,
shape=(3, *self.image_resolution),
)
if OBS_STATE not in self.input_features:
self.input_features[OBS_STATE] = PolicyFeature(
type=FeatureType.STATE,
shape=(self.max_state_dim,),
)
if ACTION not in self.output_features:
self.output_features[ACTION] = PolicyFeature(
type=FeatureType.ACTION,
shape=(self.max_action_dim,),
)
def get_optimizer_preset(self) -> AdamWConfig:
return AdamWConfig(
lr=self.optimizer_lr,
betas=self.optimizer_betas,
eps=self.optimizer_eps,
weight_decay=self.optimizer_weight_decay,
grad_clip_norm=self.optimizer_grad_clip_norm,
)
def get_scheduler_preset(self):
return Evo1SchedulerConfig(
num_warmup_steps=self.scheduler_warmup_steps,
)
@property
def observation_delta_indices(self) -> list[int]:
return [0]
@property
def action_delta_indices(self) -> list[int]:
return list(range(self.chunk_size))
@property
def reward_delta_indices(self) -> None:
return None

查看文件

@ -0,0 +1,155 @@
from types import SimpleNamespace
from typing import Union
import torch
import torch.nn as nn
from PIL import Image
from lerobot.policies.evo1.flow_matching import FlowmatchingActionHead
from lerobot.policies.evo1.internvl3_embedder import InternVL3Embedder
class EVO1(nn.Module):
def __init__(self, config: dict):
super().__init__()
self.config = config
self._device = config.get("device", "cuda")
self.return_cls_only = config.get("return_cls_only", False)
vlm_name = config.get("vlm_name", "OpenGVLab/InternVL3-1B")
self.embedder = InternVL3Embedder(model_name=vlm_name, device=self._device)
action_head_type = config.get("action_head", "flowmatching").lower()
if action_head_type != "flowmatching":
raise NotImplementedError(f"Unknown action_head: {action_head_type}")
horizon = config.get("action_horizon", config.get("horizon", 16))
per_action_dim = config.get("per_action_dim", 7)
action_dim = horizon * per_action_dim
config["horizon"] = horizon
config["per_action_dim"] = per_action_dim
config["action_dim"] = action_dim
if action_dim != horizon * per_action_dim:
raise ValueError(
f"action_dim ({action_dim}) must equal horizon ({horizon}) * per_action_dim ({per_action_dim})"
)
self.horizon = horizon
self.per_action_dim = per_action_dim
self.action_head = FlowmatchingActionHead(
config=SimpleNamespace(
embed_dim=config.get("embed_dim", 896),
hidden_dim=config.get("hidden_dim", 1024),
action_dim=action_dim,
horizon=horizon,
per_action_dim=per_action_dim,
state_dim=config.get("state_dim", 7),
state_hidden_dim=config.get("state_hidden_dim", 1024),
num_heads=config.get("num_heads", 8),
num_layers=config.get("num_layers", 8),
dropout=config.get("dropout", 0.0),
num_inference_timesteps=config.get("num_inference_timesteps", 50),
num_categories=config.get("num_categories", 1),
)
).to(self._device)
def get_vl_embeddings(
self,
images: list[Union[Image.Image, torch.Tensor]],
image_mask: torch.Tensor,
prompt: str = "",
return_cls_only: bool | None = None,
) -> torch.Tensor:
if return_cls_only is None:
return_cls_only = self.return_cls_only
if not images:
raise ValueError("EVO1 expects at least one image per sample.")
return self.embedder.get_fused_image_text_embedding_from_tensor_images(
image_tensors=images,
image_mask=image_mask,
text_prompt=prompt,
return_cls_only=return_cls_only,
)
def prepare_state(self, state_input: Union[list, torch.Tensor]) -> torch.Tensor:
if isinstance(state_input, list):
state_tensor = torch.tensor(state_input)
elif isinstance(state_input, torch.Tensor):
state_tensor = state_input
else:
raise TypeError(f"Unsupported state input type: {type(state_input)}")
if state_tensor.ndim == 1:
state_tensor = state_tensor.unsqueeze(0)
return state_tensor.to(self._device)
def predict_action(
self,
fused_tokens: torch.Tensor,
state: torch.Tensor,
actions_gt: torch.Tensor | None = None,
action_mask: torch.Tensor | None = None,
embodiment_ids: torch.Tensor | None = None,
):
if actions_gt is None:
return self.action_head.get_action(
fused_tokens,
state=state,
action_mask=action_mask,
embodiment_id=embodiment_ids,
)
return self.action_head(
fused_tokens,
state=state,
actions_gt=actions_gt,
action_mask=action_mask,
embodiment_id=embodiment_ids,
)
@torch.no_grad()
def run_inference(
self,
images: list[Union[Image.Image, torch.Tensor]],
image_mask: torch.Tensor,
prompt: str,
state_input: Union[list, torch.Tensor],
return_cls_only: bool | None = None,
action_mask: torch.Tensor | None = None,
embodiment_ids: torch.Tensor | None = None,
) -> torch.Tensor:
fused_tokens = self.get_vl_embeddings(
images=images,
image_mask=image_mask,
prompt=prompt,
return_cls_only=return_cls_only,
)
state_tensor = self.prepare_state(state_input)
return self.predict_action(
fused_tokens,
state_tensor,
action_mask=action_mask,
embodiment_ids=embodiment_ids,
)
def forward(
self,
fused_tokens: torch.Tensor,
state: torch.Tensor | None = None,
actions_gt: torch.Tensor | None = None,
action_mask: torch.Tensor | None = None,
embodiment_ids: torch.Tensor | None = None,
):
return self.predict_action(fused_tokens, state, actions_gt, action_mask, embodiment_ids)
def _freeze_module(self, module: nn.Module):
for param in module.parameters():
param.requires_grad = False
def set_finetune_flags(self):
if not self.config.get("finetune_vlm", False):
self._freeze_module(self.embedder)
if not self.config.get("finetune_action_head", False):
self._freeze_module(self.action_head)

查看文件

@ -0,0 +1,400 @@
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class SinusoidalPositionalEncoding(nn.Module):
def __init__(self, dim: int, max_len: int = 1000):
super().__init__()
pe = torch.zeros(max_len, dim)
position = torch.arange(0, max_len).unsqueeze(1)
div_term = torch.exp(torch.arange(0, dim, 2) * -(math.log(10000.0) / dim))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer('pe', pe)
def forward(self, seq_len: int):
if seq_len > self.pe.size(1):
self._extend_pe(seq_len)
return self.pe[:, :seq_len, :]
def _extend_pe(self, new_max_len):
old_max_len, dim = self.pe.size(1), self.pe.size(2)
if new_max_len <= old_max_len:
return
extra_positions = torch.arange(old_max_len, new_max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(torch.arange(0, dim, 2, dtype=torch.float) * -(math.log(10000.0) / dim))
extra_pe = torch.zeros(new_max_len - old_max_len, dim)
extra_pe[:, 0::2] = torch.sin(extra_positions * div_term)
extra_pe[:, 1::2] = torch.cos(extra_positions * div_term)
extra_pe = extra_pe.unsqueeze(0)
new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1)
self.pe = new_pe
class CategorySpecificLinear(nn.Module):
def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1):
super().__init__()
self.num_categories = num_categories
if num_categories <= 1:
self.linear = nn.Linear(in_dim, out_dim)
else:
self.weight = nn.Parameter(torch.randn(num_categories, in_dim, out_dim))
self.bias = nn.Parameter(torch.randn(num_categories, out_dim))
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
if self.num_categories <= 1:
return self.linear(x)
orig_shape = x.shape
x_flat = x.reshape(-1, orig_shape[-1])
if category_id.dim() == 0:
cid = category_id.item()
out = x_flat @ self.weight[cid] + self.bias[cid]
else:
category_id = category_id.view(-1)
weight_selected = self.weight[category_id]
bias_selected = self.bias[category_id]
out = torch.bmm(x_flat.unsqueeze(1), weight_selected).squeeze(1) + bias_selected
out_shape = orig_shape[:-1] + (out.shape[-1],)
return out.view(out_shape)
class CategorySpecificMLP(nn.Module):
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1):
super().__init__()
self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories)
self.fc2 = CategorySpecificLinear(hidden_dim, output_dim, num_categories)
self.activation = nn.ReLU(inplace=True)
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
out = self.activation(self.fc1(x, category_id))
out = self.fc2(out, category_id)
return out
class MultiEmbodimentActionEncoder(nn.Module):
def __init__(self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1):
super().__init__()
self.horizon = horizon
self.embed_dim = embed_dim
self.num_categories = num_categories
self.W1 = CategorySpecificLinear(action_dim, hidden_dim, num_categories)
self.W2 = CategorySpecificLinear(hidden_dim, hidden_dim, num_categories)
self.W3 = CategorySpecificLinear(hidden_dim, embed_dim, num_categories)
self.pos_encoding = SinusoidalPositionalEncoding(hidden_dim, max_len=horizon)
self.activation = nn.ReLU(inplace=True)
def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor):
B, H, D = action_seq.shape
assert H == self.horizon, "Action sequence length must match horizon"
x = action_seq.reshape(B * H, D)
if category_id.dim() == 0:
cat_ids = category_id.repeat(H * B)
else:
cat_ids = category_id.unsqueeze(1).repeat(1, H).reshape(B * H)
out = self.activation(self.W1(x, cat_ids))
pos_enc = self.pos_encoding(H).to(out.device)
pos_enc = pos_enc.repeat(B, 1, 1).reshape(B * H, -1)
out = out + pos_enc
out = self.activation(self.W2(out, cat_ids))
out = self.W3(out, cat_ids)
out = out.view(B, H, self.embed_dim)
return out
class BasicTransformerBlock(nn.Module):
def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0):
super().__init__()
self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
self.norm1 = nn.LayerNorm(embed_dim)
self.norm2 = nn.LayerNorm(embed_dim)
self.ff = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.GELU(),
nn.Linear(hidden_dim, embed_dim)
)
def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor):
x = self.norm1(action_tokens)
attn_out, _ = self.attn(x, context_tokens, context_tokens)
x = action_tokens + attn_out
x2 = self.norm2(x)
if time_emb is not None:
x2 = x2 + time_emb.unsqueeze(1)
ff_out = self.ff(x2)
x = x + ff_out
return x
class FlowmatchingActionHead(nn.Module):
def __init__(self, config=None,
embed_dim: int = 896,
hidden_dim: int = 1024,
action_dim: int = 16*7,
horizon: int = 16,
per_action_dim: int = 7,
num_heads: int = 8,
num_layers: int = 8,
dropout: float = 0.0,
num_inference_timesteps: int = 20,
num_categories: int = 1):
super().__init__()
if config is not None:
embed_dim = getattr(config, "embed_dim", embed_dim)
hidden_dim = getattr(config, "hidden_dim", hidden_dim)
action_dim = getattr(config, "action_dim", action_dim)
horizon = getattr(config, "horizon", horizon)
num_heads = getattr(config, "num_heads", num_heads)
num_layers = getattr(config, "num_layers", num_layers)
dropout = getattr(config, "dropout", dropout)
num_inference_timesteps = getattr(config, "num_inference_timesteps", num_inference_timesteps)
num_categories = getattr(config, "num_categories", num_categories)
self.config = config
else:
from types import SimpleNamespace
self.config = SimpleNamespace(embed_dim=embed_dim, hidden_dim=hidden_dim,
action_dim=action_dim, horizon=horizon,
num_heads=num_heads, num_layers=num_layers,
dropout=dropout, num_inference_timesteps=num_inference_timesteps,
num_categories=num_categories)
self.embed_dim = embed_dim
self.horizon = horizon
self.per_action_dim = config.per_action_dim
self.action_dim = config.action_dim
self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000)
self.transformer_blocks = nn.ModuleList([
BasicTransformerBlock(embed_dim=embed_dim, num_heads=num_heads,
hidden_dim=embed_dim*4, dropout=dropout)
for _ in range(num_layers)
])
self.norm_out = nn.LayerNorm(embed_dim)
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim)
self.mlp_head = CategorySpecificMLP(input_dim=embed_dim, hidden_dim=hidden_dim,
output_dim=action_dim, num_categories=num_categories)
self.state_encoder = None
if hasattr(self.config, "state_dim") and self.config.state_dim is not None:
state_hidden = getattr(self.config, "state_hidden_dim", embed_dim)
self.state_encoder = CategorySpecificMLP(input_dim=self.config.state_dim,
hidden_dim=state_hidden,
output_dim=embed_dim,
num_categories=num_categories)
self.action_encoder = None
if horizon > 1:
per_action_dim = getattr(self.config, "per_action_dim", None)
if per_action_dim is None:
per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim
self.action_encoder = MultiEmbodimentActionEncoder(action_dim=per_action_dim,
embed_dim=embed_dim,
hidden_dim=embed_dim,
horizon=horizon,
num_categories=num_categories)
def forward(self, fused_tokens: torch.Tensor, state: torch.Tensor = None,
actions_gt: torch.Tensor = None, embodiment_id: torch.LongTensor = None,
state_mask: torch.Tensor = None, action_mask: torch.Tensor = None):
if actions_gt is None:
return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id)
B = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id)
state_emb = state_emb.unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
t = torch.distributions.Beta(2, 2).sample((B,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
time_index = (t * 1000).long()
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
action_shape = actions_gt.shape[1]
actions_gt_seq = actions_gt
noise = torch.rand_like(actions_gt) * 2 - 1
if action_mask is not None:
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
assert action_mask.shape == noise.shape, f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
noise = noise * action_mask
if self.horizon > 1:
noise_seq = noise.view(B, self.horizon, self.per_action_dim)
else:
noise_seq = noise.unsqueeze(1)
if self.horizon > 1:
t_broadcast = t.view(B, 1, 1)
else:
t_broadcast = t.view(B, 1)
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
if self.horizon > 1 and self.action_encoder is not None:
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
else:
if not hasattr(self, "single_action_proj"):
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_intermediate_seq)
x = action_tokens
for block in self.transformer_blocks:
x = block(x, context_tokens, time_emb)
x = self.norm_out(x)
if self.horizon > 1:
x_flat = x.reshape(B, -1)
if not hasattr(self, "seq_pool_proj"):
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
return pred_velocity, noise
def get_action(self, fused_tokens: torch.Tensor, state: torch.Tensor = None, embodiment_id: torch.LongTensor = None, action_mask: torch.Tensor = None):
B = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
action_dim_total = getattr(self.config, "action_dim", None)
if action_dim_total is None:
action_dim_total = self.action_dim
if self.horizon > 1:
per_action_dim = getattr(self.config, "per_action_dim", action_dim_total // self.horizon)
else:
per_action_dim = action_dim_total
action = (torch.rand(B, action_dim_total, device=device) * 2 - 1)
if self.horizon > 1:
action_seq = action.view(B, self.horizon, per_action_dim)
else:
action_seq = action.view(B, 1, per_action_dim)
action_mask = action_mask.view(B, 1, per_action_dim).repeat(1,self.horizon,1)
if action_mask is not None:
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
assert action_mask.shape == action_seq.shape, f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
action_seq = action_seq * action_mask
else:
raise ValueError("action_mask must be provided for inference with flow matching.")
N = int(getattr(self.config, "num_inference_timesteps", 32))
dt = 1.0 / N
for i in range(N):
t = i / N
time_index = int(t * 1000)
time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0)
time_emb = time_emb.unsqueeze(0).repeat(B, 1)
if self.horizon > 1 and self.action_encoder is not None:
action_seq = action_seq * action_mask
action_tokens = self.action_encoder(action_seq, embodiment_id)
else:
if hasattr(self, "single_action_proj"):
action_tokens = self.single_action_proj(action_seq)
else:
self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_seq)
x = action_tokens
for block in self.transformer_blocks:
x = block(x, context_tokens, time_emb)
x = self.norm_out(x)
if self.horizon > 1:
x_flat = x.reshape(B, -1)
if hasattr(self, "seq_pool_proj"):
x_pooled = self.seq_pool_proj(x_flat)
else:
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred = self.mlp_head(x_pooled, embodiment_id)
action = action + dt * pred
if self.horizon > 1:
action_seq = action.view(B, self.horizon, per_action_dim)
else:
action_seq = action.view(B, 1, per_action_dim)
return action
@property
def device(self):
return next(self.parameters()).device
@property
def dtype(self):
return next(self.parameters()).dtype

查看文件

@ -0,0 +1,241 @@
import logging
from typing import Union
import torch
import torch.nn as nn
import torchvision.transforms as T
from PIL import Image
from torchvision.transforms.functional import InterpolationMode
from torchvision.transforms.functional import to_pil_image
from transformers import AutoModel, AutoTokenizer
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
# === Image Transformations ===
def build_transform(input_size):
return T.Compose([
T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
T.ToTensor(),
T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
])
# === Aspect Ratio Handling ===
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
best_ratio_diff = float('inf')
best_ratio = (1, 1)
area = width * height
for ratio in target_ratios:
target_ar = ratio[0] / ratio[1]
diff = abs(aspect_ratio - target_ar)
if diff < best_ratio_diff:
best_ratio_diff = diff
best_ratio = ratio
elif diff == best_ratio_diff and area > 0.5 * image_size**2 * ratio[0] * ratio[1]:
best_ratio = ratio
return best_ratio
def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False):
orig_width, orig_height = image.size
aspect_ratio = orig_width / orig_height
target_ratios = set(
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
i * j <= max_num and i * j >= min_num)
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio, target_ratios, orig_width, orig_height, image_size)
target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
resized_img = image.resize((target_width, target_height))
processed_images = []
for i in range(blocks):
box = (
(i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size
)
split_img = resized_img.crop(box)
processed_images.append(split_img)
assert len(processed_images) == blocks
if use_thumbnail and len(processed_images) != 1:
thumbnail_img = image.resize((image_size, image_size))
processed_images.append(thumbnail_img)
return processed_images
class InternVL3Embedder(nn.Module):
def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"):
super().__init__()
self.device = device
self.image_size = image_size
self.max_text_length = 1024 # InternVL3 supports up to 1024 tokens
self.transform = build_transform(image_size)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_fast=False)
self.model = AutoModel.from_pretrained(
model_name,
torch_dtype=torch.bfloat16,
trust_remote_code=True,
use_flash_attn=True,
low_cpu_mem_usage=True,
_fast_init=False,
).to(self.device)
if hasattr(self.model.language_model, 'model'):
layers = self.model.language_model.model.layers
else:
layers = self.model.language_model.layers
layers = layers[:14]
if hasattr(self.model.language_model, 'model'):
self.model.language_model.model.layers = torch.nn.ModuleList(layers)
else:
self.model.language_model.layers = torch.nn.ModuleList(layers)
self.model.language_model.lm_head = torch.nn.Identity()
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
self.model.vision_model.encoder.gradient_checkpointing = False
def _preprocess_images(
self,
image_tensors: list[Union[Image.Image, torch.Tensor]]
) -> tuple[torch.Tensor, list[int]]:
pixel_values_list = []
for i, image in enumerate(image_tensors):
if isinstance(image, torch.Tensor):
image = to_pil_image(image)
tiles = dynamic_preprocess(image, image_size=self.image_size)
tile_tensors = torch.stack([self.transform(t) for t in tiles]) # (T_i, 3, 448, 448)
pixel_values_list.append(tile_tensors)
pixel_values = torch.cat(pixel_values_list, dim=0).to(dtype=torch.bfloat16, device=self.device)
num_tiles_list = [pv.shape[0] for pv in pixel_values_list]
return pixel_values, num_tiles_list
def _build_multimodal_prompt(
self,
num_tiles_list: list[int],
text_prompt: str
) -> str:
prompt = ''
for i in range(len(num_tiles_list)):
prompt += f"Image-{i+1}: <image>\n"
prompt += text_prompt.strip()
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
IMG_START_TOKEN = "<img>"
IMG_END_TOKEN = "</img>"
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
for tile_count in num_tiles_list:
token_count = self.model.num_image_token * tile_count
image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * token_count + IMG_END_TOKEN
prompt = prompt.replace("<image>", image_tokens, 1)
return prompt
def _prepare_and_fuse_embeddings(
self,
prompt: str,
vit_embeds: torch.Tensor,
image_mask: torch.Tensor,
num_tiles_list: list[int]
) -> tuple[torch.Tensor, torch.Tensor]:
untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
true_sequence_length = untruncated_ids.shape[1]
if true_sequence_length > self.max_text_length:
logging.warning(
"InternVL3 prompt truncated: max_length=%s actual_length=%s prompt_prefix=%r",
self.max_text_length,
true_sequence_length,
prompt[:100],
)
model_inputs = self.tokenizer(prompt, return_tensors="pt", padding='max_length', truncation=True, max_length=self.max_text_length).to(self.device)
input_ids = model_inputs["input_ids"]
attention_mask = model_inputs["attention_mask"]
img_token_mask = (input_ids == self.img_context_token_id)
img_token_locations = torch.where(img_token_mask)[1]
input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone()
B, N, C = input_embeds.shape
input_embeds = input_embeds.reshape(B * N, C)
input_ids = input_ids.reshape(B * N)
selected = (input_ids == self.img_context_token_id)
vit_embeds = vit_embeds.reshape(-1, C).to(device=input_embeds.device, dtype=input_embeds.dtype)
n_token = int(selected.sum().item())
if vit_embeds.shape[0] < n_token:
raise ValueError(
f"InternVL3 produced fewer image tokens than expected: got {vit_embeds.shape[0]}, need {n_token}"
)
input_embeds[selected] = vit_embeds[:n_token]
tokens_per_tile = self.model.num_image_token
current_token_idx = 0
for i in range(len(image_mask)):
num_tiles_for_this_image = num_tiles_list[i]
num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile
if not image_mask[i]:
start_idx = img_token_locations[current_token_idx]
end_idx = start_idx + num_tokens_for_this_image
attention_mask[0, start_idx:end_idx] = 0
current_token_idx += num_tokens_for_this_image
input_embeds = input_embeds.reshape(B, N, C)
return input_embeds, attention_mask
def get_fused_image_text_embedding_from_tensor_images(
self,
image_tensors: list[Union[Image.Image, torch.Tensor]],
image_mask: torch.Tensor,
text_prompt: str,
return_cls_only: bool = True,
):
pixel_values, num_tiles_list = self._preprocess_images(image_tensors)
if pixel_values.shape[0] == 0:
logging.warning("InternVL3 received an empty image batch after preprocessing.")
vit_embeds = self.model.extract_feature(pixel_values)
fused_embeds = vit_embeds
prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt)
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(prompt, fused_embeds, image_mask, num_tiles_list)
outputs = self.model.language_model(
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
output_hidden_states=True,
return_dict=True,
)
fused_hidden = outputs.hidden_states[-1].to(torch.float32)
return fused_hidden[:, 0, :] if return_cls_only else fused_hidden

查看文件

@ -0,0 +1,390 @@
import builtins
from collections import deque
from contextlib import nullcontext
from pathlib import Path
import torch
from torch import Tensor
from lerobot.configs.policies import PreTrainedConfig
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
from lerobot.policies.evo1.evo1_model import EVO1
from lerobot.policies.pretrained import PreTrainedPolicy, T
from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE
class EVO1Policy(PreTrainedPolicy):
config_class = Evo1Config
name = "evo1"
def __init__(self, config: Evo1Config, **kwargs):
super().__init__(config)
config.validate_features()
if len(config.image_features) > config.max_views:
raise ValueError(
f"EVO1 supports at most {config.max_views} camera streams, got {len(config.image_features)}"
)
self.config = config
self.model = EVO1(self._build_model_config(config))
self.model.set_finetune_flags()
self.reset()
@classmethod
def from_pretrained(
cls: builtins.type[T],
pretrained_name_or_path: str | Path,
*,
config: PreTrainedConfig | None = None,
force_download: bool = False,
resume_download: bool | None = None,
proxies: dict | None = None,
token: str | bool | None = None,
cache_dir: str | Path | None = None,
local_files_only: bool = False,
revision: str | None = None,
strict: bool = True,
**kwargs,
) -> T:
return super().from_pretrained(
pretrained_name_or_path=pretrained_name_or_path,
config=config,
force_download=force_download,
resume_download=resume_download,
proxies=proxies,
token=token,
cache_dir=cache_dir,
local_files_only=local_files_only,
revision=revision,
strict=strict,
**kwargs,
)
@staticmethod
def _build_model_config(config: Evo1Config) -> dict:
return {
"device": config.device,
"return_cls_only": config.return_cls_only,
"vlm_name": config.vlm_model_name,
"action_head": config.action_head,
"action_horizon": config.chunk_size,
"per_action_dim": config.max_action_dim,
"state_dim": config.max_state_dim,
"embed_dim": config.embed_dim,
"hidden_dim": config.hidden_dim,
"state_hidden_dim": config.state_hidden_dim,
"num_heads": config.num_heads,
"num_layers": config.num_layers,
"dropout": config.dropout,
"num_inference_timesteps": config.num_inference_timesteps,
"num_categories": config.num_categories,
"finetune_vlm": config.finetune_vlm,
"finetune_action_head": config.finetune_action_head,
}
@property
def _camera_keys(self) -> list[str]:
keys = list(self.config.image_features)
if keys:
return keys
return []
@property
def _env_action_dim(self) -> int:
action_feature = self.config.action_feature
if action_feature is None:
return self.config.max_action_dim
return int(action_feature.shape[0])
@property
def _compute_dtype(self) -> torch.dtype:
return next(self.model.action_head.parameters()).dtype
@property
def _training_compute_dtype(self) -> torch.dtype:
if str(self.config.device).startswith("cuda"):
return torch.bfloat16
return self._compute_dtype
@property
def _inference_compute_dtype(self) -> torch.dtype:
if str(self.config.device).startswith("cuda") and self.config.use_amp:
return torch.bfloat16
return self._compute_dtype
def get_optim_params(self) -> list[dict]:
decay, no_decay = [], []
for name, param in self.named_parameters():
if not param.requires_grad:
continue
is_bias = name.endswith("bias") or ".bias" in name
is_norm = param.dim() == 1 or "norm" in name.lower()
if is_bias or is_norm:
no_decay.append(param)
else:
decay.append(param)
return [
{"params": decay, "weight_decay": self.config.optimizer_weight_decay},
{"params": no_decay, "weight_decay": 0.0},
]
def reset(self):
self._action_queue = deque([], maxlen=self.config.n_action_steps)
def _normalize_task_batch(self, batch: dict[str, Tensor | list[str] | str]) -> list[str]:
prompts = batch.get(self.config.task_field)
if prompts is None and self.config.task_field != "task":
prompts = batch.get("task")
if prompts is None:
raise ValueError(f"EVO1 expects a '{self.config.task_field}' text field in the batch.")
if isinstance(prompts, str):
return [prompts]
if isinstance(prompts, (list, tuple)):
return [str(prompt) for prompt in prompts]
raise TypeError(f"Unsupported prompt batch type: {type(prompts)}")
def _prepare_state(self, batch: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
if OBS_STATE not in batch:
raise ValueError(f"EVO1 requires '{OBS_STATE}' in the batch.")
state = batch[OBS_STATE]
if state.dim() == 1:
state = state.unsqueeze(0)
elif state.dim() == 3:
state = state[:, -1]
elif state.dim() != 2:
raise ValueError(f"Unsupported state tensor shape for EVO1: {tuple(state.shape)}")
batch_size, state_dim = state.shape
if state_dim > self.config.max_state_dim:
raise ValueError(
f"State dim {state_dim} exceeds configured max_state_dim {self.config.max_state_dim}"
)
explicit_mask = batch.get("state_mask")
if explicit_mask is not None:
if explicit_mask.dim() == 1:
explicit_mask = explicit_mask.unsqueeze(0)
elif explicit_mask.dim() == 3:
explicit_mask = explicit_mask[:, -1]
elif explicit_mask.dim() != 2:
raise ValueError(
f"Unsupported state_mask tensor shape for EVO1: {tuple(explicit_mask.shape)}"
)
if explicit_mask.shape != (batch_size, state_dim):
raise ValueError(
f"state_mask shape {tuple(explicit_mask.shape)} does not match state shape {(batch_size, state_dim)}"
)
padded = torch.zeros(
batch_size,
self.config.max_state_dim,
dtype=state.dtype,
device=self.config.device,
)
padded[:, :state_dim] = state.to(device=self.config.device)
mask = torch.zeros(
batch_size,
self.config.max_state_dim,
dtype=torch.bool,
device=self.config.device,
)
if explicit_mask is None:
mask[:, :state_dim] = True
else:
mask[:, :state_dim] = explicit_mask.to(device=self.config.device, dtype=torch.bool)
return padded.to(dtype=self._compute_dtype), mask
def _prepare_actions(self, batch: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
if ACTION not in batch:
raise ValueError(f"EVO1 requires '{ACTION}' in the batch for training.")
action = batch[ACTION]
if action.dim() == 2:
action = action.unsqueeze(1)
batch_size, horizon, action_dim = action.shape
if horizon != self.config.chunk_size:
raise ValueError(
f"EVO1 expects chunk_size={self.config.chunk_size}, got action horizon {horizon}"
)
if action_dim > self.config.max_action_dim:
raise ValueError(
f"Action dim {action_dim} exceeds configured max_action_dim {self.config.max_action_dim}"
)
explicit_mask = batch.get("action_mask")
if explicit_mask is not None:
if explicit_mask.dim() == 2:
explicit_mask = explicit_mask.unsqueeze(0)
elif explicit_mask.dim() != 3:
raise ValueError(
f"Unsupported action_mask tensor shape for EVO1: {tuple(explicit_mask.shape)}"
)
if explicit_mask.shape != (batch_size, horizon, action_dim):
raise ValueError(
"action_mask shape "
f"{tuple(explicit_mask.shape)} does not match action shape {(batch_size, horizon, action_dim)}"
)
padded = torch.zeros(
batch_size,
horizon,
self.config.max_action_dim,
dtype=action.dtype,
device=self.config.device,
)
padded[:, :, :action_dim] = action.to(device=self.config.device)
mask = torch.zeros(
batch_size,
horizon,
self.config.max_action_dim,
dtype=torch.bool,
device=self.config.device,
)
if explicit_mask is None:
mask[:, :, :action_dim] = True
else:
mask[:, :, :action_dim] = explicit_mask.to(device=self.config.device, dtype=torch.bool)
return padded.to(dtype=self._compute_dtype), mask
def _prepare_inference_action_mask(self, batch_size: int) -> Tensor:
mask = torch.zeros(
batch_size,
self.config.max_action_dim,
dtype=torch.bool,
device=self.config.device,
)
mask[:, : self._env_action_dim] = True
return mask
def _get_embodiment_ids(self, batch: dict[str, Tensor], batch_size: int) -> Tensor:
embodiment_ids = batch.get("embodiment_id")
if embodiment_ids is None and self.config.embodiment_id_field:
embodiment_ids = batch.get(self.config.embodiment_id_field)
if embodiment_ids is None:
return torch.full(
(batch_size,),
self.config.default_embodiment_id,
dtype=torch.long,
device=self.config.device,
)
if embodiment_ids.dim() == 0:
embodiment_ids = embodiment_ids.unsqueeze(0)
elif embodiment_ids.dim() > 1:
embodiment_ids = embodiment_ids[:, -1]
return embodiment_ids.to(device=self.config.device, dtype=torch.long)
def _collect_image_batches(self, batch: dict[str, Tensor]) -> tuple[list[list[Tensor]], Tensor]:
camera_keys = self._camera_keys or sorted(key for key in batch if key.startswith(f"{OBS_IMAGES}."))
if not camera_keys:
raise ValueError("EVO1 requires at least one visual observation feature.")
batch_size = batch[camera_keys[0]].shape[0]
image_batches: list[list[Tensor]] = []
image_masks = torch.zeros(batch_size, self.config.max_views, dtype=torch.bool)
for batch_index in range(batch_size):
sample_images: list[Tensor] = []
for camera_key in camera_keys[: self.config.max_views]:
image = batch[camera_key]
if image.dim() == 3:
image = image.unsqueeze(0)
elif image.dim() == 5:
image = image[:, -1]
elif image.dim() != 4:
raise ValueError(
f"Unsupported image tensor shape for EVO1: key={camera_key} shape={tuple(image.shape)}"
)
sample_images.append(image[batch_index].detach().cpu())
if not sample_images:
raise ValueError("EVO1 received a batch without any image tensor.")
while len(sample_images) < self.config.max_views:
sample_images.append(torch.zeros_like(sample_images[0]))
image_batches.append(sample_images[: self.config.max_views])
image_masks[batch_index, : min(len(camera_keys), self.config.max_views)] = True
return image_batches, image_masks
def _compute_fused_tokens(
self,
prompts: list[str],
image_batches: list[list[Tensor]],
image_masks: Tensor,
) -> Tensor:
fused_tokens = []
for prompt, images, image_mask in zip(prompts, image_batches, image_masks, strict=True):
fused = self.model.get_vl_embeddings(
images=images,
image_mask=image_mask,
prompt=prompt,
return_cls_only=self.config.return_cls_only,
)
fused_tokens.append(fused.to(device=self.config.device, dtype=self._compute_dtype))
return torch.cat(fused_tokens, dim=0)
def _compute_masked_loss(
self,
pred_velocity: Tensor,
target_velocity: Tensor,
action_mask: Tensor,
reduction: str,
) -> Tensor:
flat_mask = action_mask.view(action_mask.shape[0], -1).to(dtype=pred_velocity.dtype)
sq_error = ((pred_velocity * flat_mask) - target_velocity).pow(2)
active = flat_mask.sum(dim=1).clamp_min(1.0)
per_sample_loss = sq_error.sum(dim=1) / active
if reduction == "none":
return per_sample_loss
if reduction != "mean":
raise ValueError(f"Unsupported reduction '{reduction}'")
return sq_error.sum() / active.sum()
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
prompts = self._normalize_task_batch(batch)
image_batches, image_masks = self._collect_image_batches(batch)
states, _state_mask = self._prepare_state(batch)
actions_gt, action_mask = self._prepare_actions(batch)
fused_tokens = self._compute_fused_tokens(prompts, image_batches, image_masks)
states = states.to(dtype=self._training_compute_dtype)
actions_gt = actions_gt.to(dtype=self._training_compute_dtype)
fused_tokens = fused_tokens.to(dtype=self._training_compute_dtype)
embodiment_ids = self._get_embodiment_ids(batch, states.shape[0])
pred_velocity, noise = self.model(
fused_tokens,
state=states,
actions_gt=actions_gt,
action_mask=action_mask.to(device=self.config.device, dtype=self._compute_dtype),
embodiment_ids=embodiment_ids,
)
target_velocity = (actions_gt - noise).view(actions_gt.shape[0], -1)
loss = self._compute_masked_loss(pred_velocity, target_velocity, action_mask, reduction)
loss_mean = loss.mean().item() if loss.ndim > 0 else loss.item()
return loss, {
"loss": loss_mean,
"active_action_dims": float(action_mask.sum(dim=(1, 2)).float().mean().item()),
}
@torch.no_grad()
def predict_action_chunk(self, batch: dict[str, Tensor], **kwargs) -> Tensor:
self.eval()
prompts = self._normalize_task_batch(batch)
image_batches, image_masks = self._collect_image_batches(batch)
states, _state_mask = self._prepare_state(batch)
fused_tokens = self._compute_fused_tokens(prompts, image_batches, image_masks)
states = states.to(dtype=self._inference_compute_dtype)
fused_tokens = fused_tokens.to(dtype=self._inference_compute_dtype)
embodiment_ids = self._get_embodiment_ids(batch, states.shape[0])
action_mask = self._prepare_inference_action_mask(states.shape[0])
with torch.autocast(device_type="cuda", dtype=torch.bfloat16) if self.config.use_amp and str(self.config.device).startswith("cuda") else nullcontext():
actions = self.model(
fused_tokens,
state=states,
action_mask=action_mask,
embodiment_ids=embodiment_ids,
)
actions = actions.view(states.shape[0], self.config.chunk_size, self.config.max_action_dim)
return actions[:, :, : self._env_action_dim]
@torch.no_grad()
def select_action(self, batch: dict[str, Tensor], **kwargs) -> Tensor:
self.eval()
if len(self._action_queue) == 0:
action_chunk = self.predict_action_chunk(batch)[:, : self.config.n_action_steps]
self._action_queue.extend(action_chunk.transpose(0, 1))
return self._action_queue.popleft()

查看文件

@ -0,0 +1,90 @@
from typing import Any
import torch
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
from lerobot.processor import (
AddBatchDimensionProcessorStep,
DeviceProcessorStep,
NormalizerProcessorStep,
PolicyAction,
PolicyProcessorPipeline,
RenameObservationsProcessorStep,
UnnormalizerProcessorStep,
)
from lerobot.processor.converters import (
batch_to_transition,
create_transition,
policy_action_to_transition,
transition_to_policy_action,
)
from lerobot.utils.constants import (
ACTION,
DONE,
INFO,
OBS_PREFIX,
POLICY_POSTPROCESSOR_DEFAULT_NAME,
POLICY_PREPROCESSOR_DEFAULT_NAME,
REWARD,
TRUNCATED,
)
def evo1_batch_to_transition(batch: dict[str, Any]):
transition = batch_to_transition(batch)
complementary_data = dict(transition.get("complementary_data") or {})
reserved = {ACTION, REWARD, DONE, TRUNCATED, INFO}
for key, value in batch.items():
if key in reserved or key.startswith(OBS_PREFIX):
continue
complementary_data.setdefault(key, value)
return create_transition(
observation=transition.get("observation"),
action=transition.get("action"),
reward=transition.get("reward", 0.0),
done=transition.get("done", False),
truncated=transition.get("truncated", False),
info=transition.get("info", {}),
complementary_data=complementary_data,
)
def make_evo1_pre_post_processors(
config: Evo1Config,
dataset_stats: dict[str, dict[str, torch.Tensor]] | None = None,
) -> tuple[
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
PolicyProcessorPipeline[PolicyAction, PolicyAction],
]:
input_steps = [
RenameObservationsProcessorStep(rename_map={}),
AddBatchDimensionProcessorStep(),
NormalizerProcessorStep(
features={**config.input_features, **config.output_features},
norm_map=config.normalization_mapping,
stats=dataset_stats,
),
DeviceProcessorStep(device=config.device),
]
output_steps = [
UnnormalizerProcessorStep(
features=config.output_features,
norm_map=config.normalization_mapping,
stats=dataset_stats,
),
DeviceProcessorStep(device="cpu"),
]
return (
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
steps=input_steps,
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
to_transition=evo1_batch_to_transition,
),
PolicyProcessorPipeline[PolicyAction, PolicyAction](
steps=output_steps,
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
to_transition=policy_action_to_transition,
to_output=transition_to_policy_action,
),
)

查看文件

@ -31,6 +31,7 @@ from lerobot.envs.configs import EnvConfig
from lerobot.envs.utils import env_to_policy_features
from lerobot.policies.act.configuration_act import ACTConfig
from lerobot.policies.diffusion.configuration_diffusion import DiffusionConfig
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
from lerobot.policies.groot.configuration_groot import GrootConfig
from lerobot.policies.pi0.configuration_pi0 import PI0Config
from lerobot.policies.pi05.configuration_pi05 import PI05Config
@ -103,6 +104,10 @@ def get_policy_class(name: str) -> type[PreTrainedPolicy]:
from lerobot.policies.pi05.modeling_pi05 import PI05Policy
return PI05Policy
elif name == "evo1":
from lerobot.policies.evo1.modeling_evo1 import EVO1Policy
return EVO1Policy
elif name == "sac":
from lerobot.policies.sac.modeling_sac import SACPolicy
@ -169,6 +174,8 @@ def make_policy_config(policy_type: str, **kwargs) -> PreTrainedConfig:
return PI0Config(**kwargs)
elif policy_type == "pi05":
return PI05Config(**kwargs)
elif policy_type == "evo1":
return Evo1Config(**kwargs)
elif policy_type == "sac":
return SACConfig(**kwargs)
elif policy_type == "smolvla":
@ -333,6 +340,14 @@ def make_pre_post_processors(
dataset_stats=kwargs.get("dataset_stats"),
)
elif isinstance(policy_cfg, Evo1Config):
from lerobot.policies.evo1.processor_evo1 import make_evo1_pre_post_processors
processors = make_evo1_pre_post_processors(
config=policy_cfg,
dataset_stats=kwargs.get("dataset_stats"),
)
elif isinstance(policy_cfg, SACConfig):
from lerobot.policies.sac.processor_sac import make_sac_pre_post_processors

查看文件

@ -393,7 +393,7 @@ def train(
shuffle=shuffle and not cfg.dataset.streaming,
sampler=sampler,
pin_memory=device.type == "cuda",
drop_last=False,
drop_last=getattr(cfg.policy, "drop_last", False),
prefetch_factor=2 if cfg.num_workers > 0 else None,
)

查看文件

@ -0,0 +1,387 @@
import math
import json
import pytest
import torch
import torch.nn.functional as F
from lerobot.configs.default import DatasetConfig
from lerobot.configs.train import TrainPipelineConfig
from lerobot.configs.types import FeatureType, PolicyFeature
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
from lerobot.policies.evo1.modeling_evo1 import EVO1Policy
from lerobot.policies.evo1.processor_evo1 import make_evo1_pre_post_processors
from lerobot.utils.constants import POLICY_PREPROCESSOR_DEFAULT_NAME
class DummyEmbedder(torch.nn.Module):
def __init__(self, model_name="dummy", image_size=32, device="cpu"):
super().__init__()
self.device = device
self.proj = torch.nn.Linear(4, 8)
def get_fused_image_text_embedding_from_tensor_images(
self,
image_tensors,
image_mask,
text_prompt,
return_cls_only=True,
):
token = self.proj(torch.ones(1, 4))
if return_cls_only:
return token
return token.unsqueeze(1).repeat(1, 3, 1)
def make_test_config() -> Evo1Config:
return Evo1Config(
device="cpu",
use_amp=False,
chunk_size=5,
n_action_steps=2,
max_state_dim=6,
max_action_dim=6,
max_views=2,
embed_dim=8,
hidden_dim=16,
state_hidden_dim=8,
num_heads=1,
num_layers=1,
num_inference_timesteps=2,
input_features={
"observation.images.front": PolicyFeature(type=FeatureType.VISUAL, shape=(3, 16, 16)),
"observation.images.wrist": PolicyFeature(type=FeatureType.VISUAL, shape=(3, 16, 16)),
"observation.state": PolicyFeature(type=FeatureType.STATE, shape=(6,)),
},
output_features={
"action": PolicyFeature(type=FeatureType.ACTION, shape=(6,)),
},
)
def test_evo1_policy_forward_and_inference_smoke(monkeypatch):
import lerobot.policies.evo1.evo1_model as evo1_model_module
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
policy = EVO1Policy(make_test_config())
batch = {
"task": ["pick cube", "place cube"],
"observation.state": torch.rand(2, 6),
"observation.images.front": torch.rand(2, 3, 16, 16),
"observation.images.wrist": torch.rand(2, 3, 16, 16),
"action": torch.rand(2, 5, 6),
}
loss, info = policy.forward(batch)
assert loss.ndim == 0
assert loss.item() >= 0
assert "loss" in info
action_chunk = policy.predict_action_chunk(batch)
assert action_chunk.shape == (2, 5, 6)
action = policy.select_action(batch)
assert action.shape == (2, 6)
def test_evo1_processors_keep_task_and_state_shape():
preprocessor, postprocessor = make_evo1_pre_post_processors(
make_test_config(),
dataset_stats={
"observation.state": {"min": torch.zeros(6), "max": torch.ones(6)},
"action": {"min": torch.zeros(6), "max": torch.ones(6)},
},
)
processed = preprocessor(
{
"task": "pick cube",
"observation.state": torch.full((6,), 0.5),
"observation.images.front": torch.ones(3, 16, 16),
"observation.images.wrist": torch.ones(3, 16, 16),
}
)
assert processed["task"] == ["pick cube"]
assert processed["observation.state"].shape == (1, 6)
action = postprocessor(torch.zeros(1, 6))
assert action.shape == (1, 6)
def test_evo1_scheduler_matches_original_lambda():
cfg = make_test_config()
cfg.scheduler_warmup_steps = 3
scheduler_cfg = cfg.get_scheduler_preset()
parameter = torch.nn.Parameter(torch.tensor(1.0))
optimizer = torch.optim.AdamW([parameter], lr=cfg.optimizer_lr)
scheduler = scheduler_cfg.build(optimizer, num_training_steps=20)
def old_lr_lambda(current_step: int) -> float:
if current_step < cfg.scheduler_warmup_steps:
return current_step / max(1, cfg.scheduler_warmup_steps)
progress = (current_step - cfg.scheduler_warmup_steps) / max(1, 20 - cfg.scheduler_warmup_steps)
return max(0.0, 0.5 * (1.0 + math.cos(math.pi * progress)))
for step in [0, 1, 2, 5, 10, 19, 20]:
assert scheduler.lr_lambdas[0](step) == pytest.approx(old_lr_lambda(step))
def test_evo1_stage_freeze_semantics_match_original_training(monkeypatch):
import lerobot.policies.evo1.evo1_model as evo1_model_module
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
stage1_cfg = make_test_config()
stage1_cfg.training_stage = "stage1"
stage1_cfg.__post_init__()
stage1_policy = EVO1Policy(stage1_cfg)
assert not any(param.requires_grad for param in stage1_policy.model.embedder.parameters())
assert any(param.requires_grad for param in stage1_policy.model.action_head.parameters())
stage2_cfg = make_test_config()
stage2_cfg.training_stage = "stage2"
stage2_cfg.__post_init__()
stage2_policy = EVO1Policy(stage2_cfg)
assert all(param.requires_grad for param in stage2_policy.model.embedder.parameters())
assert all(param.requires_grad for param in stage2_policy.model.action_head.parameters())
def test_evo1_training_stage_sets_expected_flags():
stage1_cfg = make_test_config()
stage1_cfg.training_stage = "stage1"
stage1_cfg.__post_init__()
assert stage1_cfg.finetune_vlm is False
assert stage1_cfg.finetune_action_head is True
assert stage1_cfg.optimizer_betas == (0.9, 0.999)
assert stage1_cfg.drop_last is True
stage2_cfg = make_test_config()
stage2_cfg.training_stage = "stage2"
stage2_cfg.__post_init__()
assert stage2_cfg.finetune_vlm is True
assert stage2_cfg.finetune_action_head is True
def test_evo1_batch_preparation_matches_original_dataset_contract(monkeypatch):
import lerobot.policies.evo1.evo1_model as evo1_model_module
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
cfg = make_test_config()
cfg.max_views = 3
policy = EVO1Policy(cfg)
batch = {
"task": ["fold towel", "stack cloth"],
"observation.state": torch.tensor(
[[[0.1, 0.2, 0.3, 0.4]], [[0.5, 0.6, 0.7, 0.8]]], dtype=torch.float32
),
"observation.images.front": torch.rand(2, 1, 3, 16, 16),
"observation.images.wrist": torch.rand(2, 1, 3, 16, 16),
"action": torch.arange(2 * 5 * 4, dtype=torch.float32).view(2, 5, 4),
}
prompts = policy._normalize_task_batch(batch)
state, state_mask = policy._prepare_state(batch)
actions, action_mask = policy._prepare_actions(batch)
image_batches, image_masks = policy._collect_image_batches(batch)
assert prompts == ["fold towel", "stack cloth"]
assert state.shape == (2, 6)
assert state_mask.shape == (2, 6)
assert torch.allclose(state[:, :4].float(), torch.tensor([[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]]))
assert torch.equal(state_mask[:, :4], torch.ones(2, 4, dtype=torch.bool))
assert torch.equal(state_mask[:, 4:], torch.zeros(2, 2, dtype=torch.bool))
assert actions.shape == (2, 5, 6)
assert action_mask.shape == (2, 5, 6)
assert torch.allclose(actions[:, :, :4].float(), batch["action"])
assert torch.equal(action_mask[:, :, :4], torch.ones(2, 5, 4, dtype=torch.bool))
assert torch.equal(action_mask[:, :, 4:], torch.zeros(2, 5, 2, dtype=torch.bool))
assert len(image_batches) == 2
assert all(len(sample) == 3 for sample in image_batches)
assert image_masks.shape == (2, 3)
assert torch.equal(image_masks, torch.tensor([[True, True, False], [True, True, False]]))
assert torch.count_nonzero(image_batches[0][2]) == 0
assert torch.count_nonzero(image_batches[1][2]) == 0
def test_evo1_explicit_masks_take_priority(monkeypatch):
import lerobot.policies.evo1.evo1_model as evo1_model_module
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
policy = EVO1Policy(make_test_config())
batch = {
"observation.state": torch.tensor([[0.1, 0.2, 0.3, 0.4]], dtype=torch.float32),
"state_mask": torch.tensor([[True, False, True, False]], dtype=torch.bool),
"action": torch.arange(5 * 4, dtype=torch.float32).view(1, 5, 4),
"action_mask": torch.tensor(
[[[True, False, True, False]] * 5],
dtype=torch.bool,
),
}
_state, state_mask = policy._prepare_state(batch)
_action, action_mask = policy._prepare_actions(batch)
assert torch.equal(state_mask[0, :4], torch.tensor([True, False, True, False]))
assert torch.equal(action_mask[0, :, :4], batch["action_mask"][0])
assert torch.equal(state_mask[0, 4:], torch.zeros(2, dtype=torch.bool))
assert torch.equal(action_mask[0, :, 4:], torch.zeros(5, 2, dtype=torch.bool))
def test_evo1_masked_loss_matches_original_training_formula(monkeypatch):
import lerobot.policies.evo1.evo1_model as evo1_model_module
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
policy = EVO1Policy(make_test_config())
pred_velocity = torch.tensor([[1.0, 2.0, 3.0, 4.0]], dtype=torch.float32)
target_velocity = torch.tensor([[0.5, 2.5, 1.0, 6.0]], dtype=torch.float32)
action_mask = torch.tensor([[[True, True], [False, False]]], dtype=torch.bool)
loss = policy._compute_masked_loss(pred_velocity, target_velocity, action_mask, reduction="mean")
flat_mask = action_mask.view(action_mask.shape[0], -1)
pred_masked = pred_velocity * flat_mask
old_loss = F.mse_loss(pred_masked, target_velocity, reduction="mean") * (
pred_velocity.numel() / flat_mask.sum().item()
)
assert loss.item() == pytest.approx(old_loss.item())
def test_resume_pretrain_uses_policy_weights_without_checkpoint_state(monkeypatch, tmp_path):
from lerobot.configs import train as train_config_module
pretrained_path = tmp_path / "stage1" / "pretrained_model"
pretrained_path.mkdir(parents=True)
loaded_cfg = make_test_config()
loaded_cfg.push_to_hub = False
monkeypatch.setattr(train_config_module.parser, "get_path_arg", lambda field_name: str(pretrained_path))
monkeypatch.setattr(train_config_module.parser, "get_cli_overrides", lambda field_name: [])
monkeypatch.setattr(
train_config_module.PreTrainedConfig,
"from_pretrained",
classmethod(lambda cls, path, cli_overrides=None: loaded_cfg),
)
cfg = TrainPipelineConfig(
dataset=DatasetConfig(repo_id="dummy/repo"),
policy=None,
output_dir=tmp_path / "fresh_run",
resume_pretrain=True,
)
cfg.validate()
assert cfg.policy is loaded_cfg
assert cfg.policy.pretrained_path == pretrained_path
assert cfg.checkpoint_path is None
assert cfg.resume is False
def test_resume_and_resume_pretrain_are_mutually_exclusive(tmp_path):
cfg = TrainPipelineConfig(
dataset=DatasetConfig(repo_id="dummy/repo"),
policy=make_test_config(),
output_dir=tmp_path / "fresh_run",
resume=True,
resume_pretrain=True,
)
with pytest.raises(ValueError, match="mutually exclusive"):
cfg.validate()
def test_resume_pretrain_requires_policy_weights(tmp_path):
cfg = TrainPipelineConfig(
dataset=DatasetConfig(repo_id="dummy/repo"),
policy=make_test_config(),
output_dir=tmp_path / "fresh_run",
resume_pretrain=True,
)
with pytest.raises(ValueError, match="expects weights"):
cfg.validate()
def test_evo1_new_preprocessor_saves_device_processor(tmp_path):
preprocessor, _ = make_evo1_pre_post_processors(
make_test_config(),
dataset_stats={
"observation.state": {"min": torch.zeros(6), "max": torch.ones(6)},
"action": {"min": torch.zeros(6), "max": torch.ones(6)},
},
)
preprocessor.save_pretrained(tmp_path)
with open(tmp_path / f"{POLICY_PREPROCESSOR_DEFAULT_NAME}.json") as f:
saved = json.load(f)
saved_step_keys = {step["registry_name"] for step in saved["steps"]}
assert "device_processor" in saved_step_keys
def test_evo1_training_forward_casts_core_tensors_to_bfloat16(monkeypatch):
import lerobot.policies.evo1.evo1_model as evo1_model_module
import lerobot.policies.evo1.modeling_evo1 as modeling_evo1_module
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
monkeypatch.setattr(
modeling_evo1_module.EVO1Policy,
"_training_compute_dtype",
property(lambda self: torch.bfloat16),
)
policy = EVO1Policy(make_test_config())
captured = {}
def fake_prepare_state(_batch):
return torch.randn(2, 6, dtype=torch.float32), torch.ones(2, 6, dtype=torch.bool)
def fake_prepare_actions(_batch):
return torch.randn(2, 5, 6, dtype=torch.float32), torch.ones(2, 5, 6, dtype=torch.bool)
def fake_compute_fused_tokens(_prompts, _image_batches, _image_masks):
return torch.randn(2, 3, 8, dtype=torch.float32)
def fake_get_embodiment_ids(_batch, _batch_size):
return torch.zeros(2, dtype=torch.long)
def fake_collect_image_batches(_batch):
return [[torch.zeros(3, 16, 16)] * 2 for _ in range(2)], torch.ones(2, 2, dtype=torch.bool)
def fake_model_forward(fused_tokens, state=None, actions_gt=None, action_mask=None, embodiment_ids=None):
captured["fused_tokens_dtype"] = fused_tokens.dtype
captured["state_dtype"] = state.dtype
captured["actions_gt_dtype"] = actions_gt.dtype
captured["action_mask_dtype"] = action_mask.dtype
batch_size = state.shape[0]
flat_dim = actions_gt.shape[1] * actions_gt.shape[2]
return torch.zeros(batch_size, flat_dim, dtype=torch.bfloat16), torch.zeros_like(actions_gt)
monkeypatch.setattr(policy, "_prepare_state", fake_prepare_state)
monkeypatch.setattr(policy, "_prepare_actions", fake_prepare_actions)
monkeypatch.setattr(policy, "_compute_fused_tokens", fake_compute_fused_tokens)
monkeypatch.setattr(policy, "_get_embodiment_ids", fake_get_embodiment_ids)
monkeypatch.setattr(policy, "_collect_image_batches", fake_collect_image_batches)
monkeypatch.setattr(policy.model, "forward", fake_model_forward)
loss, info = policy.forward({"task": ["a", "b"]})
assert captured["fused_tokens_dtype"] == torch.bfloat16
assert captured["state_dtype"] == torch.bfloat16
assert captured["actions_gt_dtype"] == torch.bfloat16
assert captured["action_mask_dtype"] == policy._compute_dtype
assert loss.dtype == torch.bfloat16
assert "loss" in info