镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
chore(evo1): drop v21 export and local tests
这个提交包含在:
@ -1 +0,0 @@
|
|||||||
"""LeRobot v2.1 compatibility helpers."""
|
|
||||||
@ -1,529 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
"""
|
|
||||||
Export a LeRobot v3.0 dataset into a legacy v2.1-style layout that Evo1 can train on.
|
|
||||||
|
|
||||||
This script is intentionally pragmatic: it restores the files that Evo1's current dataset
|
|
||||||
loader actually consumes.
|
|
||||||
|
|
||||||
It recreates:
|
|
||||||
- meta/tasks.jsonl
|
|
||||||
- meta/episodes.jsonl
|
|
||||||
- meta/episodes_stats.jsonl
|
|
||||||
- meta/stats.json
|
|
||||||
- data/chunk-xxx/episode_xxxxxx.parquet
|
|
||||||
- videos/chunk-xxx/<camera>/episode_xxxxxx.mp4
|
|
||||||
|
|
||||||
It can also optionally rewrite task strings using ACP indicators so the exported dataset
|
|
||||||
preserves Evo-RL's prompt-conditioning semantics without changing Evo1's training code.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import copy
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import shutil
|
|
||||||
from collections import Counter
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
|
||||||
from lerobot.datasets.utils import (
|
|
||||||
DEFAULT_CHUNK_SIZE,
|
|
||||||
LEGACY_EPISODES_PATH,
|
|
||||||
LEGACY_EPISODES_STATS_PATH,
|
|
||||||
LEGACY_TASKS_PATH,
|
|
||||||
load_info,
|
|
||||||
load_stats,
|
|
||||||
load_tasks,
|
|
||||||
serialize_dict,
|
|
||||||
unflatten_dict,
|
|
||||||
write_info,
|
|
||||||
write_stats,
|
|
||||||
)
|
|
||||||
from lerobot.datasets.dataset_tools import _keep_episodes_from_video_with_av
|
|
||||||
from lerobot.rl.acp_tags import build_acp_tagged_task
|
|
||||||
from lerobot.utils.constants import HF_LEROBOT_HOME
|
|
||||||
from lerobot.utils.utils import init_logging
|
|
||||||
|
|
||||||
V21 = "v2.1"
|
|
||||||
V30 = "v3.0"
|
|
||||||
|
|
||||||
|
|
||||||
def write_jsonl(records: list[dict[str, Any]], path: Path) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with open(path, "w") as f:
|
|
||||||
for record in records:
|
|
||||||
f.write(json.dumps(make_json_ready(record), ensure_ascii=False))
|
|
||||||
f.write("\n")
|
|
||||||
|
|
||||||
|
|
||||||
def make_json_ready(value: Any) -> Any:
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return {key: make_json_ready(subvalue) for key, subvalue in value.items()}
|
|
||||||
if hasattr(value, "tolist") and not isinstance(value, (str, bytes)):
|
|
||||||
converted = value.tolist()
|
|
||||||
if converted is not value:
|
|
||||||
return make_json_ready(converted)
|
|
||||||
if isinstance(value, list):
|
|
||||||
return [make_json_ready(item) for item in value]
|
|
||||||
if isinstance(value, tuple):
|
|
||||||
return [make_json_ready(item) for item in value]
|
|
||||||
if isinstance(value, np.ndarray):
|
|
||||||
return value.tolist()
|
|
||||||
if isinstance(value, np.generic):
|
|
||||||
return value.item()
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def legacy_chunk_dir(episode_index: int) -> str:
|
|
||||||
return f"chunk-{episode_index // DEFAULT_CHUNK_SIZE:03d}"
|
|
||||||
|
|
||||||
|
|
||||||
def legacy_data_file_path(root: Path, episode_index: int) -> Path:
|
|
||||||
return root / "data" / legacy_chunk_dir(episode_index) / f"episode_{episode_index:06d}.parquet"
|
|
||||||
|
|
||||||
|
|
||||||
def legacy_video_file_path(root: Path, video_key: str, episode_index: int) -> Path:
|
|
||||||
return root / "videos" / legacy_chunk_dir(episode_index) / video_key / f"episode_{episode_index:06d}.mp4"
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_source_root(
|
|
||||||
src_root: str | None,
|
|
||||||
repo_id: str | None,
|
|
||||||
root: str | None,
|
|
||||||
episodes: list[int] | None,
|
|
||||||
) -> Path:
|
|
||||||
if src_root is not None:
|
|
||||||
resolved = Path(src_root).expanduser().resolve()
|
|
||||||
if not resolved.exists():
|
|
||||||
raise FileNotFoundError(f"Source dataset root does not exist: {resolved}")
|
|
||||||
return resolved
|
|
||||||
|
|
||||||
if repo_id is None:
|
|
||||||
raise ValueError("Provide either --src-root or --repo-id.")
|
|
||||||
|
|
||||||
base_root = Path(root).expanduser().resolve() if root is not None else HF_LEROBOT_HOME.resolve()
|
|
||||||
dataset_root = base_root / repo_id
|
|
||||||
LeRobotDataset(repo_id=repo_id, root=dataset_root, episodes=episodes, revision=V30, download_videos=True)
|
|
||||||
return dataset_root.resolve()
|
|
||||||
|
|
||||||
|
|
||||||
def build_exported_task_maps(
|
|
||||||
tasks_df: pd.DataFrame, acp_indicator_field: str | None
|
|
||||||
) -> tuple[dict[int, str], dict[tuple[int, int], int] | None]:
|
|
||||||
source_tasks: list[tuple[int, str]] = []
|
|
||||||
ordered_tasks = tasks_df.sort_values("task_index")
|
|
||||||
for task_text, row in ordered_tasks.iterrows():
|
|
||||||
source_tasks.append((int(row["task_index"]), "" if task_text is None else str(task_text)))
|
|
||||||
|
|
||||||
if acp_indicator_field is None:
|
|
||||||
task_by_index = {task_index: task_text for task_index, task_text in source_tasks}
|
|
||||||
return task_by_index, None
|
|
||||||
|
|
||||||
task_by_index: dict[int, str] = {}
|
|
||||||
pair_to_exported: dict[tuple[int, int], int] = {}
|
|
||||||
next_task_index = 0
|
|
||||||
for task_index, task_text in source_tasks:
|
|
||||||
for indicator in (1, 0):
|
|
||||||
pair_to_exported[(task_index, indicator)] = next_task_index
|
|
||||||
task_by_index[next_task_index] = build_acp_tagged_task(task_text, is_positive=bool(indicator))
|
|
||||||
next_task_index += 1
|
|
||||||
|
|
||||||
return task_by_index, pair_to_exported
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_selected_episodes(episodes_df: pd.DataFrame, episodes: list[int] | None) -> pd.DataFrame:
|
|
||||||
episodes_df = episodes_df.sort_values("episode_index").reset_index(drop=True)
|
|
||||||
if episodes is None:
|
|
||||||
return episodes_df
|
|
||||||
|
|
||||||
selected = set(episodes)
|
|
||||||
filtered = episodes_df[episodes_df["episode_index"].isin(selected)].copy()
|
|
||||||
found = {int(ep) for ep in filtered["episode_index"].tolist()}
|
|
||||||
missing = sorted(selected - found)
|
|
||||||
if missing:
|
|
||||||
raise ValueError(f"Requested episode indices not found in metadata: {missing}")
|
|
||||||
return filtered.sort_values("episode_index").reset_index(drop=True)
|
|
||||||
|
|
||||||
|
|
||||||
def load_episodes_dataframe(src_root: Path) -> pd.DataFrame:
|
|
||||||
episode_paths = sorted((src_root / "meta" / "episodes").glob("*/*.parquet"))
|
|
||||||
if not episode_paths:
|
|
||||||
raise FileNotFoundError(f"No episode metadata parquet files found under {src_root / 'meta' / 'episodes'}")
|
|
||||||
|
|
||||||
episode_frames = [pd.read_parquet(path) for path in episode_paths]
|
|
||||||
return pd.concat(episode_frames, ignore_index=True)
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_indicator_series(series: pd.Series, field_name: str) -> pd.Series:
|
|
||||||
if series.isna().any():
|
|
||||||
raise ValueError(f"ACP indicator field '{field_name}' contains null values.")
|
|
||||||
invalid = series[~series.isin([0, 1])]
|
|
||||||
if not invalid.empty:
|
|
||||||
values = sorted({repr(value) for value in invalid.tolist()[:8]})
|
|
||||||
raise ValueError(f"ACP indicator field '{field_name}' must contain only 0/1 values, got {values}")
|
|
||||||
return series.astype(int)
|
|
||||||
|
|
||||||
|
|
||||||
def export_data_files(
|
|
||||||
src_root: Path,
|
|
||||||
dst_root: Path,
|
|
||||||
info: dict[str, Any],
|
|
||||||
episodes: list[int] | None,
|
|
||||||
task_by_index: dict[int, str],
|
|
||||||
acp_indicator_field: str | None,
|
|
||||||
acp_task_remap: dict[tuple[int, int], int] | None,
|
|
||||||
) -> tuple[dict[int, list[str]], set[int]]:
|
|
||||||
selected = set(episodes) if episodes is not None else None
|
|
||||||
visual_feature_columns = {
|
|
||||||
key for key, feature in info["features"].items() if feature["dtype"] in {"image", "video"}
|
|
||||||
}
|
|
||||||
|
|
||||||
episode_tasks: dict[int, list[str]] = {}
|
|
||||||
used_task_indices: set[int] = set()
|
|
||||||
|
|
||||||
for parquet_path in sorted((src_root / "data").glob("*/*.parquet")):
|
|
||||||
df = pd.read_parquet(parquet_path)
|
|
||||||
|
|
||||||
if selected is not None:
|
|
||||||
df = df[df["episode_index"].isin(selected)]
|
|
||||||
if df.empty:
|
|
||||||
continue
|
|
||||||
|
|
||||||
drop_columns = [column for column in visual_feature_columns if column in df.columns]
|
|
||||||
if drop_columns:
|
|
||||||
df = df.drop(columns=drop_columns)
|
|
||||||
|
|
||||||
if "task_index" not in df.columns:
|
|
||||||
raise KeyError(f"Expected 'task_index' column in {parquet_path}")
|
|
||||||
|
|
||||||
if acp_indicator_field is not None:
|
|
||||||
if acp_task_remap is None:
|
|
||||||
raise ValueError("ACP task remap is missing while ACP export is enabled.")
|
|
||||||
if acp_indicator_field not in df.columns:
|
|
||||||
raise KeyError(f"ACP indicator field '{acp_indicator_field}' not found in {parquet_path}")
|
|
||||||
indicators = normalize_indicator_series(df[acp_indicator_field], acp_indicator_field)
|
|
||||||
if df["task_index"].isna().any():
|
|
||||||
raise ValueError(f"'task_index' contains null values in {parquet_path}")
|
|
||||||
remapped_indices = [
|
|
||||||
acp_task_remap[(int(task_index), int(indicator))]
|
|
||||||
for task_index, indicator in zip(df["task_index"].tolist(), indicators.tolist(), strict=False)
|
|
||||||
]
|
|
||||||
df = df.copy()
|
|
||||||
df["task_index"] = remapped_indices
|
|
||||||
|
|
||||||
for episode_index, episode_df in df.groupby("episode_index", sort=True):
|
|
||||||
episode_index = int(episode_index)
|
|
||||||
output_path = legacy_data_file_path(dst_root, episode_index)
|
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
episode_df.to_parquet(output_path, index=False)
|
|
||||||
|
|
||||||
task_indices = [int(task_index) for task_index in episode_df["task_index"].tolist()]
|
|
||||||
used_task_indices.update(task_indices)
|
|
||||||
episode_tasks[episode_index] = list(
|
|
||||||
dict.fromkeys(task_by_index[task_index] for task_index in task_indices if task_index in task_by_index)
|
|
||||||
)
|
|
||||||
|
|
||||||
return episode_tasks, used_task_indices
|
|
||||||
|
|
||||||
|
|
||||||
def export_tasks_jsonl(dst_root: Path, task_by_index: dict[int, str], used_task_indices: set[int]) -> None:
|
|
||||||
records = [
|
|
||||||
{"task_index": int(task_index), "task": task_by_index[task_index]}
|
|
||||||
for task_index in sorted(used_task_indices)
|
|
||||||
]
|
|
||||||
write_jsonl(records, dst_root / LEGACY_TASKS_PATH)
|
|
||||||
|
|
||||||
|
|
||||||
def flatten_episode_tasks(task_value: Any) -> list[str]:
|
|
||||||
if hasattr(task_value, "tolist") and not isinstance(task_value, (str, bytes)):
|
|
||||||
return flatten_episode_tasks(task_value.tolist())
|
|
||||||
if isinstance(task_value, list):
|
|
||||||
flattened: list[str] = []
|
|
||||||
for value in task_value:
|
|
||||||
flattened.extend(flatten_episode_tasks(value))
|
|
||||||
return flattened
|
|
||||||
if task_value is None:
|
|
||||||
return []
|
|
||||||
return [str(task_value)]
|
|
||||||
|
|
||||||
|
|
||||||
def export_episodes_jsonl(dst_root: Path, episodes_df: pd.DataFrame, episode_tasks: dict[int, list[str]]) -> None:
|
|
||||||
records: list[dict[str, Any]] = []
|
|
||||||
for _, row in episodes_df.iterrows():
|
|
||||||
episode_index = int(row["episode_index"])
|
|
||||||
tasks = episode_tasks.get(episode_index)
|
|
||||||
if tasks is None:
|
|
||||||
tasks = flatten_episode_tasks(row.get("tasks"))
|
|
||||||
records.append(
|
|
||||||
{
|
|
||||||
"episode_index": episode_index,
|
|
||||||
"tasks": tasks,
|
|
||||||
"length": int(row["length"]),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
write_jsonl(records, dst_root / LEGACY_EPISODES_PATH)
|
|
||||||
|
|
||||||
|
|
||||||
def export_episode_stats_jsonl(src_root: Path, dst_root: Path, episodes: list[int] | None) -> None:
|
|
||||||
selected = set(episodes) if episodes is not None else None
|
|
||||||
records: list[dict[str, Any]] = []
|
|
||||||
|
|
||||||
for parquet_path in sorted((src_root / "meta" / "episodes").glob("*/*.parquet")):
|
|
||||||
df = pd.read_parquet(parquet_path)
|
|
||||||
if selected is not None:
|
|
||||||
df = df[df["episode_index"].isin(selected)]
|
|
||||||
if df.empty:
|
|
||||||
continue
|
|
||||||
|
|
||||||
stats_columns = [column for column in df.columns if column.startswith("stats/")]
|
|
||||||
if not stats_columns:
|
|
||||||
continue
|
|
||||||
|
|
||||||
for _, row in df.sort_values("episode_index").iterrows():
|
|
||||||
flat_stats = {column.removeprefix("stats/"): row[column] for column in stats_columns}
|
|
||||||
records.append(
|
|
||||||
{
|
|
||||||
"episode_index": int(row["episode_index"]),
|
|
||||||
"stats": serialize_dict(unflatten_dict(flat_stats)),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
if records:
|
|
||||||
write_jsonl(records, dst_root / LEGACY_EPISODES_STATS_PATH)
|
|
||||||
|
|
||||||
|
|
||||||
def export_videos(
|
|
||||||
src_root: Path,
|
|
||||||
dst_root: Path,
|
|
||||||
info: dict[str, Any],
|
|
||||||
episodes_df: pd.DataFrame,
|
|
||||||
vcodec: str,
|
|
||||||
pix_fmt: str,
|
|
||||||
) -> None:
|
|
||||||
video_keys = sorted([key for key, feature in info["features"].items() if feature["dtype"] == "video"])
|
|
||||||
if not video_keys:
|
|
||||||
logging.warning("Source dataset has no video features; skipping video export.")
|
|
||||||
return
|
|
||||||
|
|
||||||
if info["video_path"] is None:
|
|
||||||
raise ValueError("Source dataset declares video features but info['video_path'] is None.")
|
|
||||||
|
|
||||||
fps = float(info["fps"])
|
|
||||||
file_usage: dict[str, Counter[tuple[int, int]]] = {video_key: Counter() for video_key in video_keys}
|
|
||||||
for _, row in episodes_df.iterrows():
|
|
||||||
for video_key in video_keys:
|
|
||||||
file_usage[video_key][
|
|
||||||
(int(row[f"videos/{video_key}/chunk_index"]), int(row[f"videos/{video_key}/file_index"]))
|
|
||||||
] += 1
|
|
||||||
|
|
||||||
for video_key in video_keys:
|
|
||||||
logging.info(f"Exporting legacy videos for {video_key}")
|
|
||||||
for _, row in episodes_df.iterrows():
|
|
||||||
episode_index = int(row["episode_index"])
|
|
||||||
chunk_index = int(row[f"videos/{video_key}/chunk_index"])
|
|
||||||
file_index = int(row[f"videos/{video_key}/file_index"])
|
|
||||||
from_timestamp = float(row[f"videos/{video_key}/from_timestamp"])
|
|
||||||
to_timestamp = float(row[f"videos/{video_key}/to_timestamp"])
|
|
||||||
|
|
||||||
src_video_rel = info["video_path"].format(
|
|
||||||
video_key=video_key,
|
|
||||||
chunk_index=chunk_index,
|
|
||||||
file_index=file_index,
|
|
||||||
)
|
|
||||||
src_video_path = src_root / src_video_rel
|
|
||||||
if not src_video_path.exists():
|
|
||||||
raise FileNotFoundError(f"Missing source video file: {src_video_path}")
|
|
||||||
|
|
||||||
dst_video_path = legacy_video_file_path(dst_root, video_key, episode_index)
|
|
||||||
dst_video_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
if file_usage[video_key][(chunk_index, file_index)] == 1 and abs(from_timestamp) < 1e-6:
|
|
||||||
shutil.copy(src_video_path, dst_video_path)
|
|
||||||
continue
|
|
||||||
|
|
||||||
_keep_episodes_from_video_with_av(
|
|
||||||
input_path=src_video_path,
|
|
||||||
output_path=dst_video_path,
|
|
||||||
episodes_to_keep=[(from_timestamp, to_timestamp)],
|
|
||||||
fps=fps,
|
|
||||||
vcodec=vcodec,
|
|
||||||
pix_fmt=pix_fmt,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def build_legacy_info(
|
|
||||||
info: dict[str, Any],
|
|
||||||
episodes_df: pd.DataFrame,
|
|
||||||
used_task_indices: set[int],
|
|
||||||
acp_indicator_field: str | None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
legacy_info = copy.deepcopy(info)
|
|
||||||
legacy_info["codebase_version"] = V21
|
|
||||||
legacy_info["total_episodes"] = int(len(episodes_df))
|
|
||||||
legacy_info["total_frames"] = int(episodes_df["length"].sum())
|
|
||||||
legacy_info["total_tasks"] = int(len(used_task_indices))
|
|
||||||
legacy_info["total_chunks"] = int((max(episodes_df["episode_index"]) // DEFAULT_CHUNK_SIZE) + 1) if len(episodes_df) else 0
|
|
||||||
legacy_info["total_videos"] = int(
|
|
||||||
len([key for key, feature in info["features"].items() if feature["dtype"] == "video"]) * len(episodes_df)
|
|
||||||
)
|
|
||||||
legacy_info["data_path"] = "data/chunk-{chunk_index:03d}/episode_{episode_index:06d}.parquet"
|
|
||||||
legacy_info["video_path"] = (
|
|
||||||
"videos/chunk-{chunk_index:03d}/{video_key}/episode_{episode_index:06d}.mp4"
|
|
||||||
if info["video_path"] is not None
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
if acp_indicator_field is not None:
|
|
||||||
legacy_info["acp_export_indicator_field"] = acp_indicator_field
|
|
||||||
return legacy_info
|
|
||||||
|
|
||||||
|
|
||||||
def convert_dataset(
|
|
||||||
src_root: Path,
|
|
||||||
dst_root: Path,
|
|
||||||
episodes: list[int] | None,
|
|
||||||
acp_indicator_field: str | None,
|
|
||||||
force: bool,
|
|
||||||
vcodec: str,
|
|
||||||
pix_fmt: str,
|
|
||||||
) -> None:
|
|
||||||
info = load_info(src_root)
|
|
||||||
if info.get("codebase_version") != V30:
|
|
||||||
raise ValueError(
|
|
||||||
f"Source dataset at {src_root} has codebase_version={info.get('codebase_version')!r}, expected {V30!r}."
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks_df = load_tasks(src_root)
|
|
||||||
episodes_df = ensure_selected_episodes(load_episodes_dataframe(src_root), episodes)
|
|
||||||
|
|
||||||
if dst_root.exists():
|
|
||||||
if not force:
|
|
||||||
raise FileExistsError(f"Destination already exists: {dst_root}. Use --force to overwrite.")
|
|
||||||
shutil.rmtree(dst_root)
|
|
||||||
dst_root.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
task_by_index, acp_task_remap = build_exported_task_maps(tasks_df, acp_indicator_field)
|
|
||||||
|
|
||||||
episode_tasks, used_task_indices = export_data_files(
|
|
||||||
src_root=src_root,
|
|
||||||
dst_root=dst_root,
|
|
||||||
info=info,
|
|
||||||
episodes=episodes,
|
|
||||||
task_by_index=task_by_index,
|
|
||||||
acp_indicator_field=acp_indicator_field,
|
|
||||||
acp_task_remap=acp_task_remap,
|
|
||||||
)
|
|
||||||
if not used_task_indices:
|
|
||||||
raise ValueError("No frames were exported. Check --episodes or the source dataset contents.")
|
|
||||||
|
|
||||||
export_tasks_jsonl(dst_root, task_by_index, used_task_indices)
|
|
||||||
export_episodes_jsonl(dst_root, episodes_df, episode_tasks)
|
|
||||||
|
|
||||||
stats = load_stats(src_root)
|
|
||||||
if stats is not None:
|
|
||||||
write_stats(stats, dst_root)
|
|
||||||
export_episode_stats_jsonl(src_root, dst_root, episodes)
|
|
||||||
|
|
||||||
export_videos(
|
|
||||||
src_root=src_root,
|
|
||||||
dst_root=dst_root,
|
|
||||||
info=info,
|
|
||||||
episodes_df=episodes_df,
|
|
||||||
vcodec=vcodec,
|
|
||||||
pix_fmt=pix_fmt,
|
|
||||||
)
|
|
||||||
|
|
||||||
legacy_info = build_legacy_info(info, episodes_df, used_task_indices, acp_indicator_field)
|
|
||||||
write_info(legacy_info, dst_root)
|
|
||||||
|
|
||||||
logging.info("Export completed")
|
|
||||||
logging.info(f" source: {src_root}")
|
|
||||||
logging.info(f" output: {dst_root}")
|
|
||||||
logging.info(f" episodes: {len(episodes_df)}")
|
|
||||||
logging.info(f" tasks: {len(used_task_indices)}")
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
init_logging()
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("--src-root", type=str, default=None, help="Local v3.0 dataset root to export.")
|
|
||||||
parser.add_argument("--repo-id", type=str, default=None, help="Optional HF dataset repo id to download first.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--root",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="Base directory for HF downloads when --repo-id is used. Defaults to HF_LEROBOT_HOME.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--dst-root",
|
|
||||||
type=str,
|
|
||||||
required=True,
|
|
||||||
help="Output directory for the exported legacy dataset.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--episodes",
|
|
||||||
type=int,
|
|
||||||
nargs="*",
|
|
||||||
default=None,
|
|
||||||
help="Optional episode indices to export. Useful for smoke tests and partial exports.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--acp-indicator-field",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="If set, rewrite task strings/task_index using this 0/1 frame field, e.g. complementary_info.acp_indicator.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--force",
|
|
||||||
action="store_true",
|
|
||||||
help="Overwrite the destination directory if it already exists.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--vcodec",
|
|
||||||
type=str,
|
|
||||||
default="h264",
|
|
||||||
choices=["h264", "hevc", "libsvtav1"],
|
|
||||||
help="Codec used when slicing shared source video files into per-episode legacy files.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--pix-fmt",
|
|
||||||
type=str,
|
|
||||||
default="yuv420p",
|
|
||||||
help="Pixel format used when re-encoding legacy video clips.",
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
src_root = resolve_source_root(args.src_root, args.repo_id, args.root, args.episodes)
|
|
||||||
dst_root = Path(args.dst_root).expanduser().resolve()
|
|
||||||
|
|
||||||
convert_dataset(
|
|
||||||
src_root=src_root,
|
|
||||||
dst_root=dst_root,
|
|
||||||
episodes=args.episodes,
|
|
||||||
acp_indicator_field=args.acp_indicator_field,
|
|
||||||
force=args.force,
|
|
||||||
vcodec=args.vcodec,
|
|
||||||
pix_fmt=args.pix_fmt,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,387 +0,0 @@
|
|||||||
import math
|
|
||||||
import json
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import torch
|
|
||||||
import torch.nn.functional as F
|
|
||||||
|
|
||||||
from lerobot.configs.default import DatasetConfig
|
|
||||||
from lerobot.configs.train import TrainPipelineConfig
|
|
||||||
from lerobot.configs.types import FeatureType, PolicyFeature
|
|
||||||
from lerobot.policies.evo1.configuration_evo1 import Evo1Config
|
|
||||||
from lerobot.policies.evo1.modeling_evo1 import EVO1Policy
|
|
||||||
from lerobot.policies.evo1.processor_evo1 import make_evo1_pre_post_processors
|
|
||||||
from lerobot.utils.constants import POLICY_PREPROCESSOR_DEFAULT_NAME
|
|
||||||
|
|
||||||
|
|
||||||
class DummyEmbedder(torch.nn.Module):
|
|
||||||
def __init__(self, model_name="dummy", image_size=32, device="cpu"):
|
|
||||||
super().__init__()
|
|
||||||
self.device = device
|
|
||||||
self.proj = torch.nn.Linear(4, 8)
|
|
||||||
|
|
||||||
def get_fused_image_text_embedding_from_tensor_images(
|
|
||||||
self,
|
|
||||||
image_tensors,
|
|
||||||
image_mask,
|
|
||||||
text_prompt,
|
|
||||||
return_cls_only=True,
|
|
||||||
):
|
|
||||||
token = self.proj(torch.ones(1, 4))
|
|
||||||
if return_cls_only:
|
|
||||||
return token
|
|
||||||
return token.unsqueeze(1).repeat(1, 3, 1)
|
|
||||||
|
|
||||||
|
|
||||||
def make_test_config() -> Evo1Config:
|
|
||||||
return Evo1Config(
|
|
||||||
device="cpu",
|
|
||||||
use_amp=False,
|
|
||||||
chunk_size=5,
|
|
||||||
n_action_steps=2,
|
|
||||||
max_state_dim=6,
|
|
||||||
max_action_dim=6,
|
|
||||||
max_views=2,
|
|
||||||
embed_dim=8,
|
|
||||||
hidden_dim=16,
|
|
||||||
state_hidden_dim=8,
|
|
||||||
num_heads=1,
|
|
||||||
num_layers=1,
|
|
||||||
num_inference_timesteps=2,
|
|
||||||
input_features={
|
|
||||||
"observation.images.front": PolicyFeature(type=FeatureType.VISUAL, shape=(3, 16, 16)),
|
|
||||||
"observation.images.wrist": PolicyFeature(type=FeatureType.VISUAL, shape=(3, 16, 16)),
|
|
||||||
"observation.state": PolicyFeature(type=FeatureType.STATE, shape=(6,)),
|
|
||||||
},
|
|
||||||
output_features={
|
|
||||||
"action": PolicyFeature(type=FeatureType.ACTION, shape=(6,)),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_policy_forward_and_inference_smoke(monkeypatch):
|
|
||||||
import lerobot.policies.evo1.evo1_model as evo1_model_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
|
|
||||||
policy = EVO1Policy(make_test_config())
|
|
||||||
|
|
||||||
batch = {
|
|
||||||
"task": ["pick cube", "place cube"],
|
|
||||||
"observation.state": torch.rand(2, 6),
|
|
||||||
"observation.images.front": torch.rand(2, 3, 16, 16),
|
|
||||||
"observation.images.wrist": torch.rand(2, 3, 16, 16),
|
|
||||||
"action": torch.rand(2, 5, 6),
|
|
||||||
}
|
|
||||||
|
|
||||||
loss, info = policy.forward(batch)
|
|
||||||
assert loss.ndim == 0
|
|
||||||
assert loss.item() >= 0
|
|
||||||
assert "loss" in info
|
|
||||||
|
|
||||||
action_chunk = policy.predict_action_chunk(batch)
|
|
||||||
assert action_chunk.shape == (2, 5, 6)
|
|
||||||
|
|
||||||
action = policy.select_action(batch)
|
|
||||||
assert action.shape == (2, 6)
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_processors_keep_task_and_state_shape():
|
|
||||||
preprocessor, postprocessor = make_evo1_pre_post_processors(
|
|
||||||
make_test_config(),
|
|
||||||
dataset_stats={
|
|
||||||
"observation.state": {"min": torch.zeros(6), "max": torch.ones(6)},
|
|
||||||
"action": {"min": torch.zeros(6), "max": torch.ones(6)},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
processed = preprocessor(
|
|
||||||
{
|
|
||||||
"task": "pick cube",
|
|
||||||
"observation.state": torch.full((6,), 0.5),
|
|
||||||
"observation.images.front": torch.ones(3, 16, 16),
|
|
||||||
"observation.images.wrist": torch.ones(3, 16, 16),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
assert processed["task"] == ["pick cube"]
|
|
||||||
assert processed["observation.state"].shape == (1, 6)
|
|
||||||
|
|
||||||
action = postprocessor(torch.zeros(1, 6))
|
|
||||||
assert action.shape == (1, 6)
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_scheduler_matches_original_lambda():
|
|
||||||
cfg = make_test_config()
|
|
||||||
cfg.scheduler_warmup_steps = 3
|
|
||||||
scheduler_cfg = cfg.get_scheduler_preset()
|
|
||||||
parameter = torch.nn.Parameter(torch.tensor(1.0))
|
|
||||||
optimizer = torch.optim.AdamW([parameter], lr=cfg.optimizer_lr)
|
|
||||||
scheduler = scheduler_cfg.build(optimizer, num_training_steps=20)
|
|
||||||
|
|
||||||
def old_lr_lambda(current_step: int) -> float:
|
|
||||||
if current_step < cfg.scheduler_warmup_steps:
|
|
||||||
return current_step / max(1, cfg.scheduler_warmup_steps)
|
|
||||||
progress = (current_step - cfg.scheduler_warmup_steps) / max(1, 20 - cfg.scheduler_warmup_steps)
|
|
||||||
return max(0.0, 0.5 * (1.0 + math.cos(math.pi * progress)))
|
|
||||||
|
|
||||||
for step in [0, 1, 2, 5, 10, 19, 20]:
|
|
||||||
assert scheduler.lr_lambdas[0](step) == pytest.approx(old_lr_lambda(step))
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_stage_freeze_semantics_match_original_training(monkeypatch):
|
|
||||||
import lerobot.policies.evo1.evo1_model as evo1_model_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
|
|
||||||
|
|
||||||
stage1_cfg = make_test_config()
|
|
||||||
stage1_cfg.training_stage = "stage1"
|
|
||||||
stage1_cfg.__post_init__()
|
|
||||||
stage1_policy = EVO1Policy(stage1_cfg)
|
|
||||||
|
|
||||||
assert not any(param.requires_grad for param in stage1_policy.model.embedder.parameters())
|
|
||||||
assert any(param.requires_grad for param in stage1_policy.model.action_head.parameters())
|
|
||||||
|
|
||||||
stage2_cfg = make_test_config()
|
|
||||||
stage2_cfg.training_stage = "stage2"
|
|
||||||
stage2_cfg.__post_init__()
|
|
||||||
stage2_policy = EVO1Policy(stage2_cfg)
|
|
||||||
|
|
||||||
assert all(param.requires_grad for param in stage2_policy.model.embedder.parameters())
|
|
||||||
assert all(param.requires_grad for param in stage2_policy.model.action_head.parameters())
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_training_stage_sets_expected_flags():
|
|
||||||
stage1_cfg = make_test_config()
|
|
||||||
stage1_cfg.training_stage = "stage1"
|
|
||||||
stage1_cfg.__post_init__()
|
|
||||||
assert stage1_cfg.finetune_vlm is False
|
|
||||||
assert stage1_cfg.finetune_action_head is True
|
|
||||||
assert stage1_cfg.optimizer_betas == (0.9, 0.999)
|
|
||||||
assert stage1_cfg.drop_last is True
|
|
||||||
|
|
||||||
stage2_cfg = make_test_config()
|
|
||||||
stage2_cfg.training_stage = "stage2"
|
|
||||||
stage2_cfg.__post_init__()
|
|
||||||
assert stage2_cfg.finetune_vlm is True
|
|
||||||
assert stage2_cfg.finetune_action_head is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_batch_preparation_matches_original_dataset_contract(monkeypatch):
|
|
||||||
import lerobot.policies.evo1.evo1_model as evo1_model_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
|
|
||||||
|
|
||||||
cfg = make_test_config()
|
|
||||||
cfg.max_views = 3
|
|
||||||
policy = EVO1Policy(cfg)
|
|
||||||
|
|
||||||
batch = {
|
|
||||||
"task": ["fold towel", "stack cloth"],
|
|
||||||
"observation.state": torch.tensor(
|
|
||||||
[[[0.1, 0.2, 0.3, 0.4]], [[0.5, 0.6, 0.7, 0.8]]], dtype=torch.float32
|
|
||||||
),
|
|
||||||
"observation.images.front": torch.rand(2, 1, 3, 16, 16),
|
|
||||||
"observation.images.wrist": torch.rand(2, 1, 3, 16, 16),
|
|
||||||
"action": torch.arange(2 * 5 * 4, dtype=torch.float32).view(2, 5, 4),
|
|
||||||
}
|
|
||||||
|
|
||||||
prompts = policy._normalize_task_batch(batch)
|
|
||||||
state, state_mask = policy._prepare_state(batch)
|
|
||||||
actions, action_mask = policy._prepare_actions(batch)
|
|
||||||
image_batches, image_masks = policy._collect_image_batches(batch)
|
|
||||||
|
|
||||||
assert prompts == ["fold towel", "stack cloth"]
|
|
||||||
|
|
||||||
assert state.shape == (2, 6)
|
|
||||||
assert state_mask.shape == (2, 6)
|
|
||||||
assert torch.allclose(state[:, :4].float(), torch.tensor([[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]]))
|
|
||||||
assert torch.equal(state_mask[:, :4], torch.ones(2, 4, dtype=torch.bool))
|
|
||||||
assert torch.equal(state_mask[:, 4:], torch.zeros(2, 2, dtype=torch.bool))
|
|
||||||
|
|
||||||
assert actions.shape == (2, 5, 6)
|
|
||||||
assert action_mask.shape == (2, 5, 6)
|
|
||||||
assert torch.allclose(actions[:, :, :4].float(), batch["action"])
|
|
||||||
assert torch.equal(action_mask[:, :, :4], torch.ones(2, 5, 4, dtype=torch.bool))
|
|
||||||
assert torch.equal(action_mask[:, :, 4:], torch.zeros(2, 5, 2, dtype=torch.bool))
|
|
||||||
|
|
||||||
assert len(image_batches) == 2
|
|
||||||
assert all(len(sample) == 3 for sample in image_batches)
|
|
||||||
assert image_masks.shape == (2, 3)
|
|
||||||
assert torch.equal(image_masks, torch.tensor([[True, True, False], [True, True, False]]))
|
|
||||||
assert torch.count_nonzero(image_batches[0][2]) == 0
|
|
||||||
assert torch.count_nonzero(image_batches[1][2]) == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_explicit_masks_take_priority(monkeypatch):
|
|
||||||
import lerobot.policies.evo1.evo1_model as evo1_model_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
|
|
||||||
policy = EVO1Policy(make_test_config())
|
|
||||||
|
|
||||||
batch = {
|
|
||||||
"observation.state": torch.tensor([[0.1, 0.2, 0.3, 0.4]], dtype=torch.float32),
|
|
||||||
"state_mask": torch.tensor([[True, False, True, False]], dtype=torch.bool),
|
|
||||||
"action": torch.arange(5 * 4, dtype=torch.float32).view(1, 5, 4),
|
|
||||||
"action_mask": torch.tensor(
|
|
||||||
[[[True, False, True, False]] * 5],
|
|
||||||
dtype=torch.bool,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
_state, state_mask = policy._prepare_state(batch)
|
|
||||||
_action, action_mask = policy._prepare_actions(batch)
|
|
||||||
|
|
||||||
assert torch.equal(state_mask[0, :4], torch.tensor([True, False, True, False]))
|
|
||||||
assert torch.equal(action_mask[0, :, :4], batch["action_mask"][0])
|
|
||||||
assert torch.equal(state_mask[0, 4:], torch.zeros(2, dtype=torch.bool))
|
|
||||||
assert torch.equal(action_mask[0, :, 4:], torch.zeros(5, 2, dtype=torch.bool))
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_masked_loss_matches_original_training_formula(monkeypatch):
|
|
||||||
import lerobot.policies.evo1.evo1_model as evo1_model_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
|
|
||||||
policy = EVO1Policy(make_test_config())
|
|
||||||
|
|
||||||
pred_velocity = torch.tensor([[1.0, 2.0, 3.0, 4.0]], dtype=torch.float32)
|
|
||||||
target_velocity = torch.tensor([[0.5, 2.5, 1.0, 6.0]], dtype=torch.float32)
|
|
||||||
action_mask = torch.tensor([[[True, True], [False, False]]], dtype=torch.bool)
|
|
||||||
|
|
||||||
loss = policy._compute_masked_loss(pred_velocity, target_velocity, action_mask, reduction="mean")
|
|
||||||
|
|
||||||
flat_mask = action_mask.view(action_mask.shape[0], -1)
|
|
||||||
pred_masked = pred_velocity * flat_mask
|
|
||||||
old_loss = F.mse_loss(pred_masked, target_velocity, reduction="mean") * (
|
|
||||||
pred_velocity.numel() / flat_mask.sum().item()
|
|
||||||
)
|
|
||||||
|
|
||||||
assert loss.item() == pytest.approx(old_loss.item())
|
|
||||||
|
|
||||||
|
|
||||||
def test_resume_pretrain_uses_policy_weights_without_checkpoint_state(monkeypatch, tmp_path):
|
|
||||||
from lerobot.configs import train as train_config_module
|
|
||||||
|
|
||||||
pretrained_path = tmp_path / "stage1" / "pretrained_model"
|
|
||||||
pretrained_path.mkdir(parents=True)
|
|
||||||
loaded_cfg = make_test_config()
|
|
||||||
loaded_cfg.push_to_hub = False
|
|
||||||
|
|
||||||
monkeypatch.setattr(train_config_module.parser, "get_path_arg", lambda field_name: str(pretrained_path))
|
|
||||||
monkeypatch.setattr(train_config_module.parser, "get_cli_overrides", lambda field_name: [])
|
|
||||||
monkeypatch.setattr(
|
|
||||||
train_config_module.PreTrainedConfig,
|
|
||||||
"from_pretrained",
|
|
||||||
classmethod(lambda cls, path, cli_overrides=None: loaded_cfg),
|
|
||||||
)
|
|
||||||
|
|
||||||
cfg = TrainPipelineConfig(
|
|
||||||
dataset=DatasetConfig(repo_id="dummy/repo"),
|
|
||||||
policy=None,
|
|
||||||
output_dir=tmp_path / "fresh_run",
|
|
||||||
resume_pretrain=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
cfg.validate()
|
|
||||||
|
|
||||||
assert cfg.policy is loaded_cfg
|
|
||||||
assert cfg.policy.pretrained_path == pretrained_path
|
|
||||||
assert cfg.checkpoint_path is None
|
|
||||||
assert cfg.resume is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_resume_and_resume_pretrain_are_mutually_exclusive(tmp_path):
|
|
||||||
cfg = TrainPipelineConfig(
|
|
||||||
dataset=DatasetConfig(repo_id="dummy/repo"),
|
|
||||||
policy=make_test_config(),
|
|
||||||
output_dir=tmp_path / "fresh_run",
|
|
||||||
resume=True,
|
|
||||||
resume_pretrain=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="mutually exclusive"):
|
|
||||||
cfg.validate()
|
|
||||||
|
|
||||||
|
|
||||||
def test_resume_pretrain_requires_policy_weights(tmp_path):
|
|
||||||
cfg = TrainPipelineConfig(
|
|
||||||
dataset=DatasetConfig(repo_id="dummy/repo"),
|
|
||||||
policy=make_test_config(),
|
|
||||||
output_dir=tmp_path / "fresh_run",
|
|
||||||
resume_pretrain=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="expects weights"):
|
|
||||||
cfg.validate()
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_new_preprocessor_saves_device_processor(tmp_path):
|
|
||||||
preprocessor, _ = make_evo1_pre_post_processors(
|
|
||||||
make_test_config(),
|
|
||||||
dataset_stats={
|
|
||||||
"observation.state": {"min": torch.zeros(6), "max": torch.ones(6)},
|
|
||||||
"action": {"min": torch.zeros(6), "max": torch.ones(6)},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
preprocessor.save_pretrained(tmp_path)
|
|
||||||
|
|
||||||
with open(tmp_path / f"{POLICY_PREPROCESSOR_DEFAULT_NAME}.json") as f:
|
|
||||||
saved = json.load(f)
|
|
||||||
|
|
||||||
saved_step_keys = {step["registry_name"] for step in saved["steps"]}
|
|
||||||
assert "device_processor" in saved_step_keys
|
|
||||||
|
|
||||||
|
|
||||||
def test_evo1_training_forward_casts_core_tensors_to_bfloat16(monkeypatch):
|
|
||||||
import lerobot.policies.evo1.evo1_model as evo1_model_module
|
|
||||||
import lerobot.policies.evo1.modeling_evo1 as modeling_evo1_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(evo1_model_module, "InternVL3Embedder", DummyEmbedder)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
modeling_evo1_module.EVO1Policy,
|
|
||||||
"_training_compute_dtype",
|
|
||||||
property(lambda self: torch.bfloat16),
|
|
||||||
)
|
|
||||||
|
|
||||||
policy = EVO1Policy(make_test_config())
|
|
||||||
captured = {}
|
|
||||||
|
|
||||||
def fake_prepare_state(_batch):
|
|
||||||
return torch.randn(2, 6, dtype=torch.float32), torch.ones(2, 6, dtype=torch.bool)
|
|
||||||
|
|
||||||
def fake_prepare_actions(_batch):
|
|
||||||
return torch.randn(2, 5, 6, dtype=torch.float32), torch.ones(2, 5, 6, dtype=torch.bool)
|
|
||||||
|
|
||||||
def fake_compute_fused_tokens(_prompts, _image_batches, _image_masks):
|
|
||||||
return torch.randn(2, 3, 8, dtype=torch.float32)
|
|
||||||
|
|
||||||
def fake_get_embodiment_ids(_batch, _batch_size):
|
|
||||||
return torch.zeros(2, dtype=torch.long)
|
|
||||||
|
|
||||||
def fake_collect_image_batches(_batch):
|
|
||||||
return [[torch.zeros(3, 16, 16)] * 2 for _ in range(2)], torch.ones(2, 2, dtype=torch.bool)
|
|
||||||
|
|
||||||
def fake_model_forward(fused_tokens, state=None, actions_gt=None, action_mask=None, embodiment_ids=None):
|
|
||||||
captured["fused_tokens_dtype"] = fused_tokens.dtype
|
|
||||||
captured["state_dtype"] = state.dtype
|
|
||||||
captured["actions_gt_dtype"] = actions_gt.dtype
|
|
||||||
captured["action_mask_dtype"] = action_mask.dtype
|
|
||||||
batch_size = state.shape[0]
|
|
||||||
flat_dim = actions_gt.shape[1] * actions_gt.shape[2]
|
|
||||||
return torch.zeros(batch_size, flat_dim, dtype=torch.bfloat16), torch.zeros_like(actions_gt)
|
|
||||||
|
|
||||||
monkeypatch.setattr(policy, "_prepare_state", fake_prepare_state)
|
|
||||||
monkeypatch.setattr(policy, "_prepare_actions", fake_prepare_actions)
|
|
||||||
monkeypatch.setattr(policy, "_compute_fused_tokens", fake_compute_fused_tokens)
|
|
||||||
monkeypatch.setattr(policy, "_get_embodiment_ids", fake_get_embodiment_ids)
|
|
||||||
monkeypatch.setattr(policy, "_collect_image_batches", fake_collect_image_batches)
|
|
||||||
monkeypatch.setattr(policy.model, "forward", fake_model_forward)
|
|
||||||
|
|
||||||
loss, info = policy.forward({"task": ["a", "b"]})
|
|
||||||
|
|
||||||
assert captured["fused_tokens_dtype"] == torch.bfloat16
|
|
||||||
assert captured["state_dtype"] == torch.bfloat16
|
|
||||||
assert captured["actions_gt_dtype"] == torch.bfloat16
|
|
||||||
assert captured["action_mask_dtype"] == policy._compute_dtype
|
|
||||||
assert loss.dtype == torch.bfloat16
|
|
||||||
assert "loss" in info
|
|
||||||
在新工单中引用
屏蔽一个用户