snapshot: sync mint main from current workspace 2026-03-03 16:32:56 +0800

这个提交包含在:
Elvin
2026-03-03 16:32:56 +08:00
当前提交 908eb4bbbc
共有 677 个文件被更改,包括 167208 次插入 和 0 次删除

查看文件

@ -0,0 +1,102 @@
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Helper to recalibrate your device (robot or teleoperator).
Example:
```shell
lerobot-calibrate \
--teleop.type=so100_leader \
--teleop.port=/dev/tty.usbmodem58760431551 \
--teleop.id=blue
```
"""
import logging
from dataclasses import asdict, dataclass
from pprint import pformat
import draccus
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401
from lerobot.robots import ( # noqa: F401
Robot,
RobotConfig,
bi_openarm_follower,
bi_so_follower,
hope_jr,
koch_follower,
lekiwi,
make_robot_from_config,
omx_follower,
openarm_follower,
so_follower,
)
from lerobot.teleoperators import ( # noqa: F401
Teleoperator,
TeleoperatorConfig,
bi_openarm_leader,
bi_so_leader,
homunculus,
koch_leader,
make_teleoperator_from_config,
omx_leader,
openarm_leader,
so_leader,
unitree_g1,
)
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.utils import init_logging
@dataclass
class CalibrateConfig:
teleop: TeleoperatorConfig | None = None
robot: RobotConfig | None = None
def __post_init__(self):
if bool(self.teleop) == bool(self.robot):
raise ValueError("Choose either a teleop or a robot.")
self.device = self.robot if self.robot else self.teleop
@draccus.wrap()
def calibrate(cfg: CalibrateConfig):
init_logging()
logging.info(pformat(asdict(cfg)))
if isinstance(cfg.device, RobotConfig):
device = make_robot_from_config(cfg.device)
elif isinstance(cfg.device, TeleoperatorConfig):
device = make_teleoperator_from_config(cfg.device)
device.connect(calibrate=False)
try:
device.calibrate()
finally:
device.disconnect()
def main():
register_third_party_plugins()
calibrate()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,346 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Generate a structured report for a LeRobot dataset.
Examples:
```bash
lerobot-dataset-report --dataset local/eval_twl2_10_hil_auto
lerobot-dataset-report --dataset ~/.cache/huggingface/lerobot/local/eval_twl2_10_hil_auto --json
```
"""
import argparse
import json
from pathlib import Path
from typing import Any
import pyarrow.dataset as pa_ds
from lerobot.datasets.utils import load_episodes, load_info, load_tasks
from lerobot.utils.constants import HF_LEROBOT_HOME
def resolve_dataset_root(dataset: str, root: Path | None) -> Path:
dataset_path = Path(dataset).expanduser()
if dataset_path.exists():
return dataset_path.resolve()
base_root = root.expanduser().resolve() if root is not None else HF_LEROBOT_HOME.resolve()
candidates = [base_root / dataset]
if "/" not in dataset.strip("/"):
candidates.append(base_root / "local" / dataset)
for candidate in candidates:
if candidate.exists():
return candidate.resolve()
searched = "\n".join(f"- {c}" for c in candidates)
raise FileNotFoundError(f"Dataset path not found. Searched:\n{searched}")
def _to_float(value: Any) -> float:
if isinstance(value, (list, tuple)):
return _to_float(value[0]) if len(value) > 0 else 0.0
if value is None:
return 0.0
return float(value)
def _normalize_label(value: Any) -> str:
if value is None:
return "unlabeled"
label = str(value).strip().lower()
if label == "":
return "unlabeled"
return label
def _safe_ratio(numerator: int, denominator: int) -> float:
return float(numerator) / float(denominator) if denominator > 0 else 0.0
def _flatten_episode_tasks(task_value: Any) -> list[str]:
if hasattr(task_value, "tolist") and not isinstance(task_value, (str, bytes)):
return _flatten_episode_tasks(task_value.tolist())
if isinstance(task_value, list):
values: list[str] = []
for item in task_value:
values.extend(_flatten_episode_tasks(item))
return values
if task_value is None:
return []
return [str(task_value)]
def _build_episode_length_histogram(lengths: list[float], bins: int = 20) -> list[dict[str, float | int]]:
if bins <= 0:
raise ValueError(f"bins must be > 0, got {bins}")
if not lengths:
return [{"start": 0.0, "end": 0.0, "count": 0} for _ in range(bins)]
min_len = float(min(lengths))
max_len = float(max(lengths))
span = max_len - min_len
histogram = []
if span == 0.0:
histogram.append({"start": min_len, "end": max_len, "count": len(lengths)})
histogram.extend({"start": min_len, "end": max_len, "count": 0} for _ in range(bins - 1))
return histogram
bin_width = span / bins
for i in range(bins):
start = min_len + i * bin_width
end = min_len + (i + 1) * bin_width if i < bins - 1 else max_len
histogram.append({"start": start, "end": end, "count": 0})
for length in lengths:
idx = min(int((float(length) - min_len) / bin_width), bins - 1)
histogram[idx]["count"] = int(histogram[idx]["count"]) + 1
return histogram
def _format_ascii_histogram(histogram: list[dict[str, float | int]], bar_width: int = 40) -> list[str]:
max_count = max((bin_info["count"] for bin_info in histogram), default=0)
lines: list[str] = []
for bin_info in histogram:
count = int(bin_info["count"])
start = float(bin_info["start"])
end = float(bin_info["end"])
filled = 0 if max_count == 0 else int(round((count / max_count) * bar_width))
bar = "#" * filled
lines.append(f"- [{start:>7.2f}s, {end:>7.2f}s] | {count:>5} | {bar}")
return lines
def build_report(dataset_root: Path) -> dict[str, Any]:
info = load_info(dataset_root)
episodes_ds = load_episodes(dataset_root)
episodes_df = episodes_ds.to_pandas()
actual_episode_count = int(len(episodes_df))
episode_success_labels = episodes_df["episode_success"].tolist() if "episode_success" in episodes_df else []
normalized_labels = [_normalize_label(v) for v in episode_success_labels]
success_count = sum(1 for v in normalized_labels if v == "success")
failure_count = sum(1 for v in normalized_labels if v == "failure")
unlabeled_count = sum(1 for v in normalized_labels if v not in {"success", "failure"})
labeled_count = success_count + failure_count
data_dataset = pa_ds.dataset(dataset_root / "data", format="parquet")
data_schema_names = set(data_dataset.schema.names)
actual_frame_count = int(data_dataset.count_rows())
intervention_col = "complementary_info.is_intervention"
episode_index_col = "episode_index"
intervention_frames = 0
intervention_episode_count = 0
if intervention_col in data_schema_names and episode_index_col in data_schema_names and actual_frame_count > 0:
table = data_dataset.to_table(columns=[episode_index_col, intervention_col])
episode_indices = table[episode_index_col].to_pylist()
intervention_values = table[intervention_col].to_pylist()
intervention_episode_ids = set()
for ep_idx, raw_value in zip(episode_indices, intervention_values, strict=True):
if _to_float(raw_value) > 0.0:
intervention_frames += 1
intervention_episode_ids.add(int(ep_idx))
intervention_episode_count = len(intervention_episode_ids)
tasks_df = load_tasks(dataset_root)
unique_tasks: list[str] = []
seen_tasks: set[str] = set()
if "tasks" in episodes_df.columns:
for row_tasks in episodes_df["tasks"].tolist():
for task in _flatten_episode_tasks(row_tasks):
if task not in seen_tasks:
seen_tasks.add(task)
unique_tasks.append(task)
feature_rows = []
for key in sorted(info["features"].keys()):
feature = info["features"][key]
feature_rows.append(
{
"name": key,
"dtype": feature.get("dtype"),
"shape": list(feature.get("shape", ())),
"names_count": len(feature["names"]) if feature.get("names") else 0,
"is_video": feature.get("dtype") == "video",
}
)
fps = float(info.get("fps", 0) or 0)
episode_lengths_frames = (
episodes_df["length"].astype(int).tolist() if "length" in episodes_df and actual_episode_count > 0 else []
)
episode_lengths_seconds = [frames / fps for frames in episode_lengths_frames] if fps > 0 else []
length_mean = float(sum(episode_lengths_seconds) / len(episode_lengths_seconds)) if episode_lengths_seconds else 0.0
length_min = float(min(episode_lengths_seconds)) if episode_lengths_seconds else 0.0
length_max = float(max(episode_lengths_seconds)) if episode_lengths_seconds else 0.0
length_histogram = _build_episode_length_histogram(episode_lengths_seconds, bins=20)
return {
"dataset_root": str(dataset_root),
"meta": {
"robot_type": info.get("robot_type"),
"fps": info.get("fps"),
"codebase_version": info.get("codebase_version"),
"splits": info.get("splits", {}),
"declared_total_episodes": int(info.get("total_episodes", 0)),
"declared_total_frames": int(info.get("total_frames", 0)),
"declared_total_tasks": int(info.get("total_tasks", 0)),
},
"structure": {
"features": feature_rows,
"task_count": int(len(unique_tasks)),
"tasks": unique_tasks,
"tasks_parquet_columns": list(tasks_df.columns),
"tasks_parquet_row_count": int(len(tasks_df)),
},
"quality": {
"actual_episode_count": actual_episode_count,
"actual_frame_count": actual_frame_count,
"episode_length": {
"unit": "seconds",
"mean": length_mean,
"min": length_min,
"max": length_max,
"histogram_bins": length_histogram,
},
},
"success_metrics": {
"success_count": success_count,
"failure_count": failure_count,
"unlabeled_count": unlabeled_count,
"success_ratio_all_episodes": _safe_ratio(success_count, actual_episode_count),
"success_ratio_labeled_episodes": _safe_ratio(success_count, labeled_count),
},
"intervention_metrics": {
"intervention_frames": intervention_frames,
"intervention_frame_ratio": _safe_ratio(intervention_frames, actual_frame_count),
"episodes_with_intervention": intervention_episode_count,
"episode_intervention_ratio": _safe_ratio(intervention_episode_count, actual_episode_count),
"has_intervention_column": intervention_col in data_schema_names,
},
}
def format_text_report(report: dict[str, Any]) -> str:
lines: list[str] = []
lines.append("=== Dataset Report ===")
lines.append(f"Root: {report['dataset_root']}")
lines.append("")
meta = report["meta"]
lines.append("[Meta]")
lines.append(f"- robot_type: {meta['robot_type']}")
lines.append(f"- fps: {meta['fps']}")
lines.append(f"- codebase_version: {meta['codebase_version']}")
lines.append(f"- splits: {meta['splits']}")
lines.append(
f"- declared totals: episodes={meta['declared_total_episodes']}, "
f"frames={meta['declared_total_frames']}, tasks={meta['declared_total_tasks']}"
)
lines.append("")
quality = report["quality"]
lines.append("[Actual Data Stats]")
lines.append(
f"- actual totals: episodes={quality['actual_episode_count']}, frames={quality['actual_frame_count']}"
)
lines.append(
f"- episode length ({quality['episode_length']['unit']}): mean={quality['episode_length']['mean']:.2f}, "
f"min={quality['episode_length']['min']:.2f}, max={quality['episode_length']['max']:.2f}"
)
lines.append("- episode length histogram (20 bins, terminal view):")
lines.extend(_format_ascii_histogram(quality["episode_length"]["histogram_bins"]))
lines.append("")
success = report["success_metrics"]
lines.append("[Success Metrics]")
lines.append(
f"- counts: success={success['success_count']}, failure={success['failure_count']}, "
f"unlabeled={success['unlabeled_count']}"
)
lines.append(f"- success ratio (all episodes): {success['success_ratio_all_episodes']:.4f}")
lines.append(f"- success ratio (labeled episodes): {success['success_ratio_labeled_episodes']:.4f}")
lines.append("")
intervention = report["intervention_metrics"]
lines.append("[Intervention Metrics]")
lines.append(f"- has intervention field: {intervention['has_intervention_column']}")
lines.append(
f"- frame-level: intervention_frames={intervention['intervention_frames']}, "
f"ratio={intervention['intervention_frame_ratio']:.4f}"
)
lines.append(
f"- episode-level: episodes_with_intervention={intervention['episodes_with_intervention']}, "
f"ratio={intervention['episode_intervention_ratio']:.4f}"
)
lines.append("")
structure = report["structure"]
lines.append("[Task List]")
lines.append(f"- task_count: {structure['task_count']}")
for idx, task in enumerate(structure["tasks"]):
lines.append(f"- task[{idx}]: {task}")
lines.append("")
lines.append("[Feature Schema]")
for feature in structure["features"]:
lines.append(
f"- {feature['name']}: dtype={feature['dtype']}, shape={feature['shape']}, "
f"names_count={feature['names_count']}, is_video={feature['is_video']}"
)
return "\n".join(lines)
def main():
parser = argparse.ArgumentParser(description="Summarize LeRobot dataset schema and quality metrics.")
parser.add_argument(
"--dataset",
type=str,
required=True,
help="Dataset repo id (e.g. local/eval_xxx) or absolute/local filesystem path.",
)
parser.add_argument(
"--root",
type=Path,
default=None,
help="Optional root directory containing datasets. Defaults to HF_LEROBOT_HOME.",
)
parser.add_argument("--json", action="store_true", help="Output JSON instead of text.")
args = parser.parse_args()
dataset_root = resolve_dataset_root(args.dataset, args.root)
report = build_report(dataset_root)
if args.json:
print(json.dumps(report, indent=2, ensure_ascii=False))
else:
print(format_text_report(report))
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,287 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Visualize data of **all** frames of any episode of a dataset of type LeRobotDataset.
Note: The last frame of the episode doesn't always correspond to a final state.
That's because our datasets are composed of transition from state to state up to
the antepenultimate state associated to the ultimate action to arrive in the final state.
However, there might not be a transition from a final state to another state.
Note: This script aims to visualize the data used to train the neural networks.
~What you see is what you get~. When visualizing image modality, it is often expected to observe
lossy compression artifacts since these images have been decoded from compressed mp4 videos to
save disk space. The compression factor applied has been tuned to not affect success rate.
Examples:
- Visualize data stored on a local machine:
```
local$ lerobot-dataset-viz \
--repo-id lerobot/pusht \
--episode-index 0
```
- Visualize data stored on a distant machine with a local viewer:
```
distant$ lerobot-dataset-viz \
--repo-id lerobot/pusht \
--episode-index 0 \
--save 1 \
--output-dir path/to/directory
local$ scp distant:path/to/directory/lerobot_pusht_episode_0.rrd .
local$ rerun lerobot_pusht_episode_0.rrd
```
- Visualize data stored on a distant machine through streaming:
(You need to forward the websocket port to the distant machine, with
`ssh -L 9087:localhost:9087 username@remote-host`)
```
distant$ lerobot-dataset-viz \
--repo-id lerobot/pusht \
--episode-index 0 \
--mode distant \
--ws-port 9087
local$ rerun ws://localhost:9087
```
"""
import argparse
import gc
import logging
import time
from pathlib import Path
import numpy as np
import rerun as rr
import torch
import torch.utils.data
import tqdm
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.utils.constants import ACTION, DONE, OBS_STATE, REWARD
def to_hwc_uint8_numpy(chw_float32_torch: torch.Tensor) -> np.ndarray:
assert chw_float32_torch.dtype == torch.float32
assert chw_float32_torch.ndim == 3
c, h, w = chw_float32_torch.shape
assert c < h and c < w, f"expect channel first images, but instead {chw_float32_torch.shape}"
hwc_uint8_numpy = (chw_float32_torch * 255).type(torch.uint8).permute(1, 2, 0).numpy()
return hwc_uint8_numpy
def visualize_dataset(
dataset: LeRobotDataset,
episode_index: int,
batch_size: int = 32,
num_workers: int = 0,
mode: str = "local",
web_port: int = 9090,
ws_port: int = 9087,
save: bool = False,
output_dir: Path | None = None,
display_compressed_images: bool = False,
) -> Path | None:
if save:
assert output_dir is not None, (
"Set an output directory where to write .rrd files with `--output-dir path/to/directory`."
)
repo_id = dataset.repo_id
logging.info("Loading dataloader")
dataloader = torch.utils.data.DataLoader(
dataset,
num_workers=num_workers,
batch_size=batch_size,
)
logging.info("Starting Rerun")
if mode not in ["local", "distant"]:
raise ValueError(mode)
spawn_local_viewer = mode == "local" and not save
rr.init(f"{repo_id}/episode_{episode_index}", spawn=spawn_local_viewer)
# Manually call python garbage collector after `rr.init` to avoid hanging in a blocking flush
# when iterating on a dataloader with `num_workers` > 0
# TODO(rcadene): remove `gc.collect` when rerun version 0.16 is out, which includes a fix
gc.collect()
if mode == "distant":
rr.serve_web_viewer(open_browser=False, web_port=web_port)
logging.info("Logging to Rerun")
for batch in tqdm.tqdm(dataloader, total=len(dataloader)):
# iterate over the batch
for i in range(len(batch["index"])):
rr.set_time("frame_index", sequence=batch["frame_index"][i].item())
rr.set_time("timestamp", timestamp=batch["timestamp"][i].item())
# display each camera image
for key in dataset.meta.camera_keys:
img = to_hwc_uint8_numpy(batch[key][i])
img_entity = rr.Image(img).compress() if display_compressed_images else rr.Image(img)
rr.log(key, entity=img_entity)
# display each dimension of action space (e.g. actuators command)
if ACTION in batch:
for dim_idx, val in enumerate(batch[ACTION][i]):
rr.log(f"{ACTION}/{dim_idx}", rr.Scalars(val.item()))
# display each dimension of observed state space (e.g. agent position in joint space)
if OBS_STATE in batch:
for dim_idx, val in enumerate(batch[OBS_STATE][i]):
rr.log(f"state/{dim_idx}", rr.Scalars(val.item()))
if DONE in batch:
rr.log(DONE, rr.Scalars(batch[DONE][i].item()))
if REWARD in batch:
rr.log(REWARD, rr.Scalars(batch[REWARD][i].item()))
if "next.success" in batch:
rr.log("next.success", rr.Scalars(batch["next.success"][i].item()))
if mode == "local" and save:
# save .rrd locally
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
repo_id_str = repo_id.replace("/", "_")
rrd_path = output_dir / f"{repo_id_str}_episode_{episode_index}.rrd"
rr.save(rrd_path)
return rrd_path
elif mode == "distant":
# stop the process from exiting since it is serving the websocket connection
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print("Ctrl-C received. Exiting.")
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--repo-id",
type=str,
required=True,
help="Name of hugging face repository containing a LeRobotDataset dataset (e.g. `lerobot/pusht`).",
)
parser.add_argument(
"--episode-index",
type=int,
required=True,
help="Episode to visualize.",
)
parser.add_argument(
"--root",
type=Path,
default=None,
help="Root directory for the dataset stored locally (e.g. `--root data`). By default, the dataset will be loaded from hugging face cache folder, or downloaded from the hub if available.",
)
parser.add_argument(
"--output-dir",
type=Path,
default=None,
help="Directory path to write a .rrd file when `--save 1` is set.",
)
parser.add_argument(
"--batch-size",
type=int,
default=32,
help="Batch size loaded by DataLoader.",
)
parser.add_argument(
"--num-workers",
type=int,
default=4,
help="Number of processes of Dataloader for loading the data.",
)
parser.add_argument(
"--mode",
type=str,
default="local",
help=(
"Mode of viewing between 'local' or 'distant'. "
"'local' requires data to be on a local machine. It spawns a viewer to visualize the data locally. "
"'distant' creates a server on the distant machine where the data is stored. "
"Visualize the data by connecting to the server with `rerun ws://localhost:PORT` on the local machine."
),
)
parser.add_argument(
"--web-port",
type=int,
default=9090,
help="Web port for rerun.io when `--mode distant` is set.",
)
parser.add_argument(
"--ws-port",
type=int,
default=9087,
help="Web socket port for rerun.io when `--mode distant` is set.",
)
parser.add_argument(
"--save",
type=int,
default=0,
help=(
"Save a .rrd file in the directory provided by `--output-dir`. "
"It also deactivates the spawning of a viewer. "
"Visualize the data by running `rerun path/to/file.rrd` on your local machine."
),
)
parser.add_argument(
"--tolerance-s",
type=float,
default=1e-4,
help=(
"Tolerance in seconds used to ensure data timestamps respect the dataset fps value"
"This is argument passed to the constructor of LeRobotDataset and maps to its tolerance_s constructor argument"
"If not given, defaults to 1e-4."
),
)
parser.add_argument(
"--display-compressed-images",
type=bool,
required=True,
default=False,
help="If set, display compressed images in Rerun instead of uncompressed ones.",
)
args = parser.parse_args()
kwargs = vars(args)
repo_id = kwargs.pop("repo_id")
root = kwargs.pop("root")
tolerance_s = kwargs.pop("tolerance_s")
logging.info("Loading dataset")
dataset = LeRobotDataset(repo_id, episodes=[args.episode_index], root=root, tolerance_s=tolerance_s)
visualize_dataset(dataset, **vars(args))
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,466 @@
#!/usr/bin/env python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Edit LeRobot datasets using various transformation tools.
This script allows you to delete episodes, split datasets, merge datasets,
remove features, modify tasks, and convert image datasets to video format.
When new_repo_id is specified, creates a new dataset.
Usage Examples:
Delete episodes 0, 2, and 5 from a dataset:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type delete_episodes \
--operation.episode_indices "[0, 2, 5]"
Delete episodes and save to a new dataset:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--new_repo_id lerobot/pusht_filtered \
--operation.type delete_episodes \
--operation.episode_indices "[0, 2, 5]"
Split dataset by fractions:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type split \
--operation.splits '{"train": 0.8, "val": 0.2}'
Split dataset by episode indices:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type split \
--operation.splits '{"train": [0, 1, 2, 3], "val": [4, 5]}'
Split into more than two splits:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type split \
--operation.splits '{"train": 0.6, "val": 0.2, "test": 0.2}'
Merge multiple datasets:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht_merged \
--operation.type merge \
--operation.repo_ids "['lerobot/pusht_train', 'lerobot/pusht_val']"
Remove camera feature:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type remove_feature \
--operation.feature_names "['observation.images.top']"
Modify tasks - set a single task for all episodes (WARNING: modifies in-place):
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type modify_tasks \
--operation.new_task "Pick up the cube and place it"
Modify tasks - set different tasks for specific episodes (WARNING: modifies in-place):
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type modify_tasks \
--operation.episode_tasks '{"0": "Task A", "1": "Task B", "2": "Task A"}'
Modify tasks - set default task with overrides for specific episodes (WARNING: modifies in-place):
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht \
--operation.type modify_tasks \
--operation.new_task "Default task" \
--operation.episode_tasks '{"5": "Special task for episode 5"}'
Convert image dataset to video format and save locally:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht_image \
--operation.type convert_image_to_video \
--operation.output_dir /path/to/output/pusht_video
Convert image dataset to video format and save with new repo_id:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht_image \
--new_repo_id lerobot/pusht_video \
--operation.type convert_image_to_video
Convert image dataset to video format and push to hub:
python -m lerobot.scripts.lerobot_edit_dataset \
--repo_id lerobot/pusht_image \
--new_repo_id lerobot/pusht_video \
--operation.type convert_image_to_video \
--push_to_hub true
Using JSON config file:
python -m lerobot.scripts.lerobot_edit_dataset \
--config_path path/to/edit_config.json
"""
import abc
import logging
import shutil
from dataclasses import dataclass
from pathlib import Path
import draccus
from lerobot.configs import parser
from lerobot.datasets.dataset_tools import (
convert_image_to_video_dataset,
delete_episodes,
merge_datasets,
modify_tasks,
remove_feature,
split_dataset,
)
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.utils.constants import HF_LEROBOT_HOME
from lerobot.utils.utils import init_logging
@dataclass
class OperationConfig(draccus.ChoiceRegistry, abc.ABC):
@property
def type(self) -> str:
return self.get_choice_name(self.__class__)
@OperationConfig.register_subclass("delete_episodes")
@dataclass
class DeleteEpisodesConfig(OperationConfig):
episode_indices: list[int] | None = None
@OperationConfig.register_subclass("split")
@dataclass
class SplitConfig(OperationConfig):
splits: dict[str, float | list[int]] | None = None
@OperationConfig.register_subclass("merge")
@dataclass
class MergeConfig(OperationConfig):
repo_ids: list[str] | None = None
@OperationConfig.register_subclass("remove_feature")
@dataclass
class RemoveFeatureConfig(OperationConfig):
feature_names: list[str] | None = None
@OperationConfig.register_subclass("modify_tasks")
@dataclass
class ModifyTasksConfig(OperationConfig):
new_task: str | None = None
episode_tasks: dict[str, str] | None = None
@OperationConfig.register_subclass("convert_image_to_video")
@dataclass
class ConvertImageToVideoConfig(OperationConfig):
output_dir: str | None = None
vcodec: str = "libsvtav1"
pix_fmt: str = "yuv420p"
g: int = 2
crf: int = 30
fast_decode: int = 0
episode_indices: list[int] | None = None
num_workers: int = 4
max_episodes_per_batch: int | None = None
max_frames_per_batch: int | None = None
@dataclass
class EditDatasetConfig:
repo_id: str
operation: OperationConfig
root: str | None = None
new_repo_id: str | None = None
push_to_hub: bool = False
def get_output_path(repo_id: str, new_repo_id: str | None, root: Path | None) -> tuple[str, Path]:
if new_repo_id:
output_repo_id = new_repo_id
output_dir = root / new_repo_id if root else HF_LEROBOT_HOME / new_repo_id
else:
output_repo_id = repo_id
dataset_path = root / repo_id if root else HF_LEROBOT_HOME / repo_id
old_path = Path(str(dataset_path) + "_old")
if dataset_path.exists():
if old_path.exists():
shutil.rmtree(old_path)
shutil.move(str(dataset_path), str(old_path))
output_dir = dataset_path
return output_repo_id, output_dir
def handle_delete_episodes(cfg: EditDatasetConfig) -> None:
if not isinstance(cfg.operation, DeleteEpisodesConfig):
raise ValueError("Operation config must be DeleteEpisodesConfig")
if not cfg.operation.episode_indices:
raise ValueError("episode_indices must be specified for delete_episodes operation")
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
output_repo_id, output_dir = get_output_path(
cfg.repo_id, cfg.new_repo_id, Path(cfg.root) if cfg.root else None
)
if cfg.new_repo_id is None:
dataset.root = Path(str(dataset.root) + "_old")
logging.info(f"Deleting episodes {cfg.operation.episode_indices} from {cfg.repo_id}")
new_dataset = delete_episodes(
dataset,
episode_indices=cfg.operation.episode_indices,
output_dir=output_dir,
repo_id=output_repo_id,
)
logging.info(f"Dataset saved to {output_dir}")
logging.info(f"Episodes: {new_dataset.meta.total_episodes}, Frames: {new_dataset.meta.total_frames}")
if cfg.push_to_hub:
logging.info(f"Pushing to hub as {output_repo_id}")
LeRobotDataset(output_repo_id, root=output_dir).push_to_hub()
def handle_split(cfg: EditDatasetConfig) -> None:
if not isinstance(cfg.operation, SplitConfig):
raise ValueError("Operation config must be SplitConfig")
if not cfg.operation.splits:
raise ValueError(
"splits dict must be specified with split names as keys and fractions/episode lists as values"
)
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
logging.info(f"Splitting dataset {cfg.repo_id} with splits: {cfg.operation.splits}")
split_datasets = split_dataset(dataset, splits=cfg.operation.splits)
for split_name, split_ds in split_datasets.items():
split_repo_id = f"{cfg.repo_id}_{split_name}"
logging.info(
f"{split_name}: {split_ds.meta.total_episodes} episodes, {split_ds.meta.total_frames} frames"
)
if cfg.push_to_hub:
logging.info(f"Pushing {split_name} split to hub as {split_repo_id}")
LeRobotDataset(split_ds.repo_id, root=split_ds.root).push_to_hub()
def handle_merge(cfg: EditDatasetConfig) -> None:
if not isinstance(cfg.operation, MergeConfig):
raise ValueError("Operation config must be MergeConfig")
if not cfg.operation.repo_ids:
raise ValueError("repo_ids must be specified for merge operation")
if not cfg.repo_id:
raise ValueError("repo_id must be specified as the output repository for merged dataset")
logging.info(f"Loading {len(cfg.operation.repo_ids)} datasets to merge")
datasets = [LeRobotDataset(repo_id, root=cfg.root) for repo_id in cfg.operation.repo_ids]
output_dir = Path(cfg.root) / cfg.repo_id if cfg.root else HF_LEROBOT_HOME / cfg.repo_id
logging.info(f"Merging datasets into {cfg.repo_id}")
merged_dataset = merge_datasets(
datasets,
output_repo_id=cfg.repo_id,
output_dir=output_dir,
)
logging.info(f"Merged dataset saved to {output_dir}")
logging.info(
f"Episodes: {merged_dataset.meta.total_episodes}, Frames: {merged_dataset.meta.total_frames}"
)
if cfg.push_to_hub:
logging.info(f"Pushing to hub as {cfg.repo_id}")
LeRobotDataset(merged_dataset.repo_id, root=output_dir).push_to_hub()
def handle_remove_feature(cfg: EditDatasetConfig) -> None:
if not isinstance(cfg.operation, RemoveFeatureConfig):
raise ValueError("Operation config must be RemoveFeatureConfig")
if not cfg.operation.feature_names:
raise ValueError("feature_names must be specified for remove_feature operation")
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
output_repo_id, output_dir = get_output_path(
cfg.repo_id, cfg.new_repo_id, Path(cfg.root) if cfg.root else None
)
if cfg.new_repo_id is None:
dataset.root = Path(str(dataset.root) + "_old")
logging.info(f"Removing features {cfg.operation.feature_names} from {cfg.repo_id}")
new_dataset = remove_feature(
dataset,
feature_names=cfg.operation.feature_names,
output_dir=output_dir,
repo_id=output_repo_id,
)
logging.info(f"Dataset saved to {output_dir}")
logging.info(f"Remaining features: {list(new_dataset.meta.features.keys())}")
if cfg.push_to_hub:
logging.info(f"Pushing to hub as {output_repo_id}")
LeRobotDataset(output_repo_id, root=output_dir).push_to_hub()
def handle_modify_tasks(cfg: EditDatasetConfig) -> None:
if not isinstance(cfg.operation, ModifyTasksConfig):
raise ValueError("Operation config must be ModifyTasksConfig")
new_task = cfg.operation.new_task
episode_tasks_raw = cfg.operation.episode_tasks
if new_task is None and episode_tasks_raw is None:
raise ValueError("Must specify at least one of new_task or episode_tasks for modify_tasks operation")
# Warn about in-place modification behavior
if cfg.new_repo_id is not None:
logging.warning("modify_tasks modifies datasets in-place. The --new_repo_id parameter is ignored.")
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
logging.warning(f"Modifying dataset in-place at {dataset.root}. Original data will be overwritten.")
# Convert episode_tasks keys from string to int if needed (CLI passes strings)
episode_tasks: dict[int, str] | None = None
if episode_tasks_raw is not None:
episode_tasks = {int(k): v for k, v in episode_tasks_raw.items()}
logging.info(f"Modifying tasks in {cfg.repo_id}")
if new_task:
logging.info(f" Default task: '{new_task}'")
if episode_tasks:
logging.info(f" Episode-specific tasks: {episode_tasks}")
modified_dataset = modify_tasks(
dataset,
new_task=new_task,
episode_tasks=episode_tasks,
)
logging.info(f"Dataset modified at {dataset.root}")
logging.info(f"Tasks: {list(modified_dataset.meta.tasks.index)}")
if cfg.push_to_hub:
logging.info(f"Pushing to hub as {cfg.repo_id}")
modified_dataset.push_to_hub()
def handle_convert_image_to_video(cfg: EditDatasetConfig) -> None:
# Note: Parser may create any config type with the right fields, so we access fields directly
# instead of checking isinstance()
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
# Determine output directory and repo_id
# Priority: 1) new_repo_id, 2) operation.output_dir, 3) auto-generated name
output_dir_config = getattr(cfg.operation, "output_dir", None)
if cfg.new_repo_id:
# Use new_repo_id for both local storage and hub push
output_repo_id = cfg.new_repo_id
# Place new dataset as a sibling to the original dataset
# Get the parent of the actual dataset root (not cfg.root which might be the lerobot cache dir)
# Extract just the dataset name (after last slash) for the local directory
local_dir_name = cfg.new_repo_id.split("/")[-1]
output_dir = dataset.root.parent / local_dir_name
logging.info(f"Saving to new dataset: {cfg.new_repo_id} at {output_dir}")
elif output_dir_config:
# Use custom output directory for local-only storage
output_dir = Path(output_dir_config)
# Extract repo name from output_dir for the dataset
output_repo_id = output_dir.name
logging.info(f"Saving to local directory: {output_dir}")
else:
# Auto-generate name: append "_video" to original repo_id
output_repo_id = f"{cfg.repo_id}_video"
# Place new dataset as a sibling to the original dataset
# Extract just the dataset name (after last slash) for the local directory
local_dir_name = output_repo_id.split("/")[-1]
output_dir = dataset.root.parent / local_dir_name
logging.info(f"Saving to auto-generated location: {output_dir}")
logging.info(f"Converting dataset {cfg.repo_id} to video format")
new_dataset = convert_image_to_video_dataset(
dataset=dataset,
output_dir=output_dir,
repo_id=output_repo_id,
vcodec=getattr(cfg.operation, "vcodec", "libsvtav1"),
pix_fmt=getattr(cfg.operation, "pix_fmt", "yuv420p"),
g=getattr(cfg.operation, "g", 2),
crf=getattr(cfg.operation, "crf", 30),
fast_decode=getattr(cfg.operation, "fast_decode", 0),
episode_indices=getattr(cfg.operation, "episode_indices", None),
num_workers=getattr(cfg.operation, "num_workers", 4),
max_episodes_per_batch=getattr(cfg.operation, "max_episodes_per_batch", None),
max_frames_per_batch=getattr(cfg.operation, "max_frames_per_batch", None),
)
logging.info("Video dataset created successfully!")
logging.info(f"Location: {output_dir}")
logging.info(f"Episodes: {new_dataset.meta.total_episodes}")
logging.info(f"Frames: {new_dataset.meta.total_frames}")
if cfg.push_to_hub:
logging.info(f"Pushing to hub as {output_repo_id}...")
new_dataset.push_to_hub()
logging.info("✓ Successfully pushed to hub!")
else:
logging.info("Dataset saved locally (not pushed to hub)")
@parser.wrap()
def edit_dataset(cfg: EditDatasetConfig) -> None:
operation_type = cfg.operation.type
if operation_type == "delete_episodes":
handle_delete_episodes(cfg)
elif operation_type == "split":
handle_split(cfg)
elif operation_type == "merge":
handle_merge(cfg)
elif operation_type == "remove_feature":
handle_remove_feature(cfg)
elif operation_type == "modify_tasks":
handle_modify_tasks(cfg)
elif operation_type == "convert_image_to_video":
handle_convert_image_to_video(cfg)
else:
available = ", ".join(OperationConfig.get_known_choices())
raise ValueError(f"Unknown operation: {operation_type}\nAvailable operations: {available}")
def main() -> None:
init_logging()
edit_dataset()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,813 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Evaluate a policy on an environment by running rollouts and computing metrics.
Usage examples:
You want to evaluate a model from the hub (eg: https://huggingface.co/lerobot/diffusion_pusht)
for 10 episodes.
```
lerobot-eval \
--policy.path=lerobot/diffusion_pusht \
--env.type=pusht \
--eval.batch_size=10 \
--eval.n_episodes=10 \
--policy.use_amp=false \
--policy.device=cuda
```
OR, you want to evaluate a model checkpoint from the LeRobot training script for 10 episodes.
```
lerobot-eval \
--policy.path=outputs/train/diffusion_pusht/checkpoints/005000/pretrained_model \
--env.type=pusht \
--eval.batch_size=10 \
--eval.n_episodes=10 \
--policy.use_amp=false \
--policy.device=cuda
```
Note that in both examples, the repo/folder should contain at least `config.json` and `model.safetensors` files.
You can learn about the CLI options for this script in the `EvalPipelineConfig` in lerobot/configs/eval.py
"""
import concurrent.futures as cf
import json
import logging
import threading
import time
from collections import defaultdict
from collections.abc import Callable
from contextlib import nullcontext
from copy import deepcopy
from dataclasses import asdict
from functools import partial
from pathlib import Path
from pprint import pformat
from typing import Any, TypedDict
import einops
import gymnasium as gym
import numpy as np
import torch
from termcolor import colored
from torch import Tensor, nn
from tqdm import trange
from lerobot.configs import parser
from lerobot.configs.eval import EvalPipelineConfig
from lerobot.envs.factory import make_env, make_env_pre_post_processors
from lerobot.envs.utils import (
add_envs_task,
check_env_attributes_and_types,
close_envs,
preprocess_observation,
)
from lerobot.policies.factory import make_policy, make_pre_post_processors
from lerobot.policies.pretrained import PreTrainedPolicy
from lerobot.processor import PolicyAction, PolicyProcessorPipeline
from lerobot.utils.constants import ACTION, DONE, OBS_STR, REWARD
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.io_utils import write_video
from lerobot.utils.random_utils import set_seed
from lerobot.utils.utils import (
get_safe_torch_device,
init_logging,
inside_slurm,
)
def rollout(
env: gym.vector.VectorEnv,
policy: PreTrainedPolicy,
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
seeds: list[int] | None = None,
return_observations: bool = False,
render_callback: Callable[[gym.vector.VectorEnv], None] | None = None,
) -> dict:
"""Run a batched policy rollout once through a batch of environments.
Note that all environments in the batch are run until the last environment is done. This means some
data will probably need to be discarded (for environments that aren't the first one to be done).
The return dictionary contains:
(optional) "observation": A dictionary of (batch, sequence + 1, *) tensors mapped to observation
keys. NOTE that this has an extra sequence element relative to the other keys in the
dictionary. This is because an extra observation is included for after the environment is
terminated or truncated.
"action": A (batch, sequence, action_dim) tensor of actions applied based on the observations (not
including the last observations).
"reward": A (batch, sequence) tensor of rewards received for applying the actions.
"success": A (batch, sequence) tensor of success conditions (the only time this can be True is upon
environment termination/truncation).
"done": A (batch, sequence) tensor of **cumulative** done conditions. For any given batch element,
the first True is followed by True's all the way till the end. This can be used for masking
extraneous elements from the sequences above.
Args:
env: The batch of environments.
policy: The policy. Must be a PyTorch nn module.
seeds: The environments are seeded once at the start of the rollout. If provided, this argument
specifies the seeds for each of the environments.
return_observations: Whether to include all observations in the returned rollout data. Observations
are returned optionally because they typically take more memory to cache. Defaults to False.
render_callback: Optional rendering callback to be used after the environments are reset, and after
every step.
Returns:
The dictionary described above.
"""
assert isinstance(policy, nn.Module), "Policy must be a PyTorch nn module."
# Reset the policy and environments.
policy.reset()
observation, info = env.reset(seed=seeds)
if render_callback is not None:
render_callback(env)
all_observations = []
all_actions = []
all_rewards = []
all_successes = []
all_dones = []
step = 0
# Keep track of which environments are done.
done = np.array([False] * env.num_envs)
max_steps = env.call("_max_episode_steps")[0]
progbar = trange(
max_steps,
desc=f"Running rollout with at most {max_steps} steps",
disable=inside_slurm(), # we dont want progress bar when we use slurm, since it clutters the logs
leave=False,
)
check_env_attributes_and_types(env)
while not np.all(done) and step < max_steps:
# Numpy array to tensor and changing dictionary keys to LeRobot policy format.
observation = preprocess_observation(observation)
if return_observations:
all_observations.append(deepcopy(observation))
# Infer "task" from attributes of environments.
# TODO: works with SyncVectorEnv but not AsyncVectorEnv
observation = add_envs_task(env, observation)
# Apply environment-specific preprocessing (e.g., LiberoProcessorStep for LIBERO)
observation = env_preprocessor(observation)
observation = preprocessor(observation)
with torch.inference_mode():
action = policy.select_action(observation)
action = postprocessor(action)
action_transition = {ACTION: action}
action_transition = env_postprocessor(action_transition)
action = action_transition[ACTION]
# Convert to CPU / numpy.
action_numpy: np.ndarray = action.to("cpu").numpy()
assert action_numpy.ndim == 2, "Action dimensions should be (batch, action_dim)"
# Apply the next action.
observation, reward, terminated, truncated, info = env.step(action_numpy)
if render_callback is not None:
render_callback(env)
# VectorEnv stores is_success in `info["final_info"][env_index]["is_success"]`. "final_info" isn't
# available if none of the envs finished.
if "final_info" in info:
final_info = info["final_info"]
if not isinstance(final_info, dict):
raise RuntimeError(
"Unsupported `final_info` format: expected dict (Gymnasium >= 1.0). "
"You're likely using an older version of gymnasium (< 1.0). Please upgrade."
)
successes = final_info["is_success"].tolist()
else:
successes = [False] * env.num_envs
# Keep track of which environments are done so far.
# Mark the episode as done if we reach the maximum step limit.
# This ensures that the rollout always terminates cleanly at `max_steps`,
# and allows logging/saving (e.g., videos) to be triggered consistently.
done = terminated | truncated | done
if step + 1 == max_steps:
done = np.ones_like(done, dtype=bool)
all_actions.append(torch.from_numpy(action_numpy))
all_rewards.append(torch.from_numpy(reward))
all_dones.append(torch.from_numpy(done))
all_successes.append(torch.tensor(successes))
step += 1
running_success_rate = (
einops.reduce(torch.stack(all_successes, dim=1), "b n -> b", "any").numpy().mean()
)
progbar.set_postfix({"running_success_rate": f"{running_success_rate.item() * 100:.1f}%"})
progbar.update()
# Track the final observation.
if return_observations:
observation = preprocess_observation(observation)
all_observations.append(deepcopy(observation))
# Stack the sequence along the first dimension so that we have (batch, sequence, *) tensors.
ret = {
ACTION: torch.stack(all_actions, dim=1),
"reward": torch.stack(all_rewards, dim=1),
"success": torch.stack(all_successes, dim=1),
"done": torch.stack(all_dones, dim=1),
}
if return_observations:
stacked_observations = {}
for key in all_observations[0]:
stacked_observations[key] = torch.stack([obs[key] for obs in all_observations], dim=1)
ret[OBS_STR] = stacked_observations
if hasattr(policy, "use_original_modules"):
policy.use_original_modules()
return ret
def eval_policy(
env: gym.vector.VectorEnv,
policy: PreTrainedPolicy,
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
n_episodes: int,
max_episodes_rendered: int = 0,
videos_dir: Path | None = None,
return_episode_data: bool = False,
start_seed: int | None = None,
) -> dict:
"""
Args:
env: The batch of environments.
policy: The policy.
n_episodes: The number of episodes to evaluate.
max_episodes_rendered: Maximum number of episodes to render into videos.
videos_dir: Where to save rendered videos.
return_episode_data: Whether to return episode data for online training. Incorporates the data into
the "episodes" key of the returned dictionary.
start_seed: The first seed to use for the first individual rollout. For all subsequent rollouts the
seed is incremented by 1. If not provided, the environments are not manually seeded.
Returns:
Dictionary with metrics and data regarding the rollouts.
"""
if max_episodes_rendered > 0 and not videos_dir:
raise ValueError("If max_episodes_rendered > 0, videos_dir must be provided.")
if not isinstance(policy, PreTrainedPolicy):
exc = ValueError(
f"Policy of type 'PreTrainedPolicy' is expected, but type '{type(policy)}' was provided."
)
try:
from peft import PeftModel
if not isinstance(policy, PeftModel):
raise exc
except ImportError:
raise exc from None
start = time.time()
policy.eval()
# Determine how many batched rollouts we need to get n_episodes. Note that if n_episodes is not evenly
# divisible by env.num_envs we end up discarding some data in the last batch.
n_batches = n_episodes // env.num_envs + int((n_episodes % env.num_envs) != 0)
# Keep track of some metrics.
sum_rewards = []
max_rewards = []
all_successes = []
all_seeds = []
threads = [] # for video saving threads
n_episodes_rendered = 0 # for saving the correct number of videos
# Callback for visualization.
def render_frame(env: gym.vector.VectorEnv):
# noqa: B023
if n_episodes_rendered >= max_episodes_rendered:
return
n_to_render_now = min(max_episodes_rendered - n_episodes_rendered, env.num_envs)
if isinstance(env, gym.vector.SyncVectorEnv):
ep_frames.append(np.stack([env.envs[i].render() for i in range(n_to_render_now)])) # noqa: B023
elif isinstance(env, gym.vector.AsyncVectorEnv):
# Here we must render all frames and discard any we don't need.
ep_frames.append(np.stack(env.call("render")[:n_to_render_now]))
if max_episodes_rendered > 0:
video_paths: list[str] = []
if return_episode_data:
episode_data: dict | None = None
# we dont want progress bar when we use slurm, since it clutters the logs
progbar = trange(n_batches, desc="Stepping through eval batches", disable=inside_slurm())
for batch_ix in progbar:
# Cache frames for rendering videos. Each item will be (b, h, w, c), and the list indexes the rollout
# step.
if max_episodes_rendered > 0:
ep_frames: list[np.ndarray] = []
if start_seed is None:
seeds = None
else:
seeds = range(
start_seed + (batch_ix * env.num_envs), start_seed + ((batch_ix + 1) * env.num_envs)
)
rollout_data = rollout(
env=env,
policy=policy,
env_preprocessor=env_preprocessor,
env_postprocessor=env_postprocessor,
preprocessor=preprocessor,
postprocessor=postprocessor,
seeds=list(seeds) if seeds else None,
return_observations=return_episode_data,
render_callback=render_frame if max_episodes_rendered > 0 else None,
)
# Figure out where in each rollout sequence the first done condition was encountered (results after
# this won't be included).
n_steps = rollout_data["done"].shape[1]
# Note: this relies on a property of argmax: that it returns the first occurrence as a tiebreaker.
done_indices = torch.argmax(rollout_data["done"].to(int), dim=1)
# Make a mask with shape (batch, n_steps) to mask out rollout data after the first done
# (batch-element-wise). Note the `done_indices + 1` to make sure to keep the data from the done step.
mask = (torch.arange(n_steps) <= einops.repeat(done_indices + 1, "b -> b s", s=n_steps)).int()
# Extend metrics.
batch_sum_rewards = einops.reduce((rollout_data["reward"] * mask), "b n -> b", "sum")
sum_rewards.extend(batch_sum_rewards.tolist())
batch_max_rewards = einops.reduce((rollout_data["reward"] * mask), "b n -> b", "max")
max_rewards.extend(batch_max_rewards.tolist())
batch_successes = einops.reduce((rollout_data["success"] * mask), "b n -> b", "any")
all_successes.extend(batch_successes.tolist())
if seeds:
all_seeds.extend(seeds)
else:
all_seeds.append(None)
# FIXME: episode_data is either None or it doesn't exist
if return_episode_data:
this_episode_data = _compile_episode_data(
rollout_data,
done_indices,
start_episode_index=batch_ix * env.num_envs,
start_data_index=(0 if episode_data is None else (episode_data["index"][-1].item() + 1)),
fps=env.unwrapped.metadata["render_fps"],
)
if episode_data is None:
episode_data = this_episode_data
else:
# Some sanity checks to make sure we are correctly compiling the data.
assert episode_data["episode_index"][-1] + 1 == this_episode_data["episode_index"][0]
assert episode_data["index"][-1] + 1 == this_episode_data["index"][0]
# Concatenate the episode data.
episode_data = {k: torch.cat([episode_data[k], this_episode_data[k]]) for k in episode_data}
# Maybe render video for visualization.
if max_episodes_rendered > 0 and len(ep_frames) > 0:
batch_stacked_frames = np.stack(ep_frames, axis=1) # (b, t, *)
for stacked_frames, done_index in zip(
batch_stacked_frames, done_indices.flatten().tolist(), strict=False
):
if n_episodes_rendered >= max_episodes_rendered:
break
videos_dir.mkdir(parents=True, exist_ok=True)
video_path = videos_dir / f"eval_episode_{n_episodes_rendered}.mp4"
video_paths.append(str(video_path))
thread = threading.Thread(
target=write_video,
args=(
str(video_path),
stacked_frames[: done_index + 1], # + 1 to capture the last observation
env.unwrapped.metadata["render_fps"],
),
)
thread.start()
threads.append(thread)
n_episodes_rendered += 1
progbar.set_postfix(
{"running_success_rate": f"{np.mean(all_successes[:n_episodes]).item() * 100:.1f}%"}
)
# Wait till all video rendering threads are done.
for thread in threads:
thread.join()
# Compile eval info.
info = {
"per_episode": [
{
"episode_ix": i,
"sum_reward": sum_reward,
"max_reward": max_reward,
"success": success,
"seed": seed,
}
for i, (sum_reward, max_reward, success, seed) in enumerate(
zip(
sum_rewards[:n_episodes],
max_rewards[:n_episodes],
all_successes[:n_episodes],
all_seeds[:n_episodes],
strict=True,
)
)
],
"aggregated": {
"avg_sum_reward": float(np.nanmean(sum_rewards[:n_episodes])),
"avg_max_reward": float(np.nanmean(max_rewards[:n_episodes])),
"pc_success": float(np.nanmean(all_successes[:n_episodes]) * 100),
"eval_s": time.time() - start,
"eval_ep_s": (time.time() - start) / n_episodes,
},
}
if return_episode_data:
info["episodes"] = episode_data
if max_episodes_rendered > 0:
info["video_paths"] = video_paths
return info
def _compile_episode_data(
rollout_data: dict, done_indices: Tensor, start_episode_index: int, start_data_index: int, fps: float
) -> dict:
"""Convenience function for `eval_policy(return_episode_data=True)`
Compiles all the rollout data into a Hugging Face dataset.
Similar logic is implemented when datasets are pushed to hub (see: `push_to_hub`).
"""
ep_dicts = []
total_frames = 0
for ep_ix in range(rollout_data[ACTION].shape[0]):
# + 2 to include the first done frame and the last observation frame.
num_frames = done_indices[ep_ix].item() + 2
total_frames += num_frames
# Here we do `num_frames - 1` as we don't want to include the last observation frame just yet.
ep_dict = {
ACTION: rollout_data[ACTION][ep_ix, : num_frames - 1],
"episode_index": torch.tensor([start_episode_index + ep_ix] * (num_frames - 1)),
"frame_index": torch.arange(0, num_frames - 1, 1),
"timestamp": torch.arange(0, num_frames - 1, 1) / fps,
DONE: rollout_data["done"][ep_ix, : num_frames - 1],
"next.success": rollout_data["success"][ep_ix, : num_frames - 1],
REWARD: rollout_data["reward"][ep_ix, : num_frames - 1].type(torch.float32),
}
# For the last observation frame, all other keys will just be copy padded.
for k in ep_dict:
ep_dict[k] = torch.cat([ep_dict[k], ep_dict[k][-1:]])
for key in rollout_data[OBS_STR]:
ep_dict[key] = rollout_data[OBS_STR][key][ep_ix, :num_frames]
ep_dicts.append(ep_dict)
data_dict = {}
for key in ep_dicts[0]:
data_dict[key] = torch.cat([x[key] for x in ep_dicts])
data_dict["index"] = torch.arange(start_data_index, start_data_index + total_frames, 1)
return data_dict
@parser.wrap()
def eval_main(cfg: EvalPipelineConfig):
logging.info(pformat(asdict(cfg)))
# Check device is available
device = get_safe_torch_device(cfg.policy.device, log=True)
torch.backends.cudnn.benchmark = True
torch.backends.cuda.matmul.allow_tf32 = True
set_seed(cfg.seed)
logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
logging.info("Making environment.")
envs = make_env(
cfg.env,
n_envs=cfg.eval.batch_size,
use_async_envs=cfg.eval.use_async_envs,
trust_remote_code=cfg.trust_remote_code,
)
logging.info("Making policy.")
policy = make_policy(
cfg=cfg.policy,
env_cfg=cfg.env,
rename_map=cfg.rename_map,
)
policy.eval()
# The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
preprocessor_overrides = {
"device_processor": {"device": str(policy.config.device)},
"rename_observations_processor": {"rename_map": cfg.rename_map},
}
preprocessor, postprocessor = make_pre_post_processors(
policy_cfg=cfg.policy,
pretrained_path=cfg.policy.pretrained_path,
preprocessor_overrides=preprocessor_overrides,
)
# Create environment-specific preprocessor and postprocessor (e.g., for LIBERO environments)
env_preprocessor, env_postprocessor = make_env_pre_post_processors(env_cfg=cfg.env, policy_cfg=cfg.policy)
with torch.no_grad(), torch.autocast(device_type=device.type) if cfg.policy.use_amp else nullcontext():
info = eval_policy_all(
envs=envs,
policy=policy,
env_preprocessor=env_preprocessor,
env_postprocessor=env_postprocessor,
preprocessor=preprocessor,
postprocessor=postprocessor,
n_episodes=cfg.eval.n_episodes,
max_episodes_rendered=10,
videos_dir=Path(cfg.output_dir) / "videos",
start_seed=cfg.seed,
max_parallel_tasks=cfg.env.max_parallel_tasks,
)
print("Overall Aggregated Metrics:")
print(info["overall"])
# Print per-suite stats
for task_group, task_group_info in info.items():
print(f"\nAggregated Metrics for {task_group}:")
print(task_group_info)
# Close all vec envs
close_envs(envs)
# Save info
with open(Path(cfg.output_dir) / "eval_info.json", "w") as f:
json.dump(info, f, indent=2)
logging.info("End of eval")
# ---- typed payload returned by one task eval ----
class TaskMetrics(TypedDict):
sum_rewards: list[float]
max_rewards: list[float]
successes: list[bool]
video_paths: list[str]
ACC_KEYS = ("sum_rewards", "max_rewards", "successes", "video_paths")
def eval_one(
env: gym.vector.VectorEnv,
*,
policy: PreTrainedPolicy,
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
n_episodes: int,
max_episodes_rendered: int,
videos_dir: Path | None,
return_episode_data: bool,
start_seed: int | None,
) -> TaskMetrics:
"""Evaluates one task_id of one suite using the provided vec env."""
task_videos_dir = videos_dir
task_result = eval_policy(
env=env,
policy=policy,
env_preprocessor=env_preprocessor,
env_postprocessor=env_postprocessor,
preprocessor=preprocessor,
postprocessor=postprocessor,
n_episodes=n_episodes,
max_episodes_rendered=max_episodes_rendered,
videos_dir=task_videos_dir,
return_episode_data=return_episode_data,
start_seed=start_seed,
)
per_episode = task_result["per_episode"]
return TaskMetrics(
sum_rewards=[ep["sum_reward"] for ep in per_episode],
max_rewards=[ep["max_reward"] for ep in per_episode],
successes=[ep["success"] for ep in per_episode],
video_paths=task_result.get("video_paths", []),
)
def run_one(
task_group: str,
task_id: int,
env,
*,
policy,
env_preprocessor,
env_postprocessor,
preprocessor,
postprocessor,
n_episodes: int,
max_episodes_rendered: int,
videos_dir: Path | None,
return_episode_data: bool,
start_seed: int | None,
):
"""
Run eval_one for a single (task_group, task_id, env).
Returns (task_group, task_id, task_metrics_dict).
This function is intentionally module-level to make it easy to test.
"""
task_videos_dir = None
if videos_dir is not None:
task_videos_dir = videos_dir / f"{task_group}_{task_id}"
task_videos_dir.mkdir(parents=True, exist_ok=True)
# Call the existing eval_one (assumed to return TaskMetrics-like dict)
metrics = eval_one(
env,
policy=policy,
env_preprocessor=env_preprocessor,
env_postprocessor=env_postprocessor,
preprocessor=preprocessor,
postprocessor=postprocessor,
n_episodes=n_episodes,
max_episodes_rendered=max_episodes_rendered,
videos_dir=task_videos_dir,
return_episode_data=return_episode_data,
start_seed=start_seed,
)
# ensure we always provide video_paths key to simplify accumulation
if max_episodes_rendered > 0:
metrics.setdefault("video_paths", [])
return task_group, task_id, metrics
def eval_policy_all(
envs: dict[str, dict[int, gym.vector.VectorEnv]],
policy,
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
n_episodes: int,
*,
max_episodes_rendered: int = 0,
videos_dir: Path | None = None,
return_episode_data: bool = False,
start_seed: int | None = None,
max_parallel_tasks: int = 1,
) -> dict:
"""
Evaluate a nested `envs` dict: {task_group: {task_id: vec_env}}.
This implementation flattens tasks, runs them sequentially or via ThreadPoolExecutor,
accumulates per-group and overall statistics, and returns the same aggregate metrics
schema as the single-env evaluator (avg_sum_reward / avg_max_reward / pc_success / timings)
plus per-task infos.
"""
start_t = time.time()
# Flatten envs into list of (task_group, task_id, env)
tasks = [(tg, tid, vec) for tg, group in envs.items() for tid, vec in group.items()]
# accumulators: track metrics at both per-group level and across all groups
group_acc: dict[str, dict[str, list]] = defaultdict(lambda: {k: [] for k in ACC_KEYS})
overall: dict[str, list] = {k: [] for k in ACC_KEYS}
per_task_infos: list[dict] = []
# small inline helper to accumulate one task's metrics into accumulators
def _accumulate_to(group: str, metrics: dict):
# metrics expected to contain 'sum_rewards', 'max_rewards', 'successes', optionally 'video_paths'
# but eval_one may store per-episode lists; we assume metrics uses scalars averaged per task as before.
# To be robust, accept scalars or lists.
def _append(key, value):
if value is None:
return
if isinstance(value, list):
group_acc[group][key].extend(value)
overall[key].extend(value)
else:
group_acc[group][key].append(value)
overall[key].append(value)
_append("sum_rewards", metrics.get("sum_rewards"))
_append("max_rewards", metrics.get("max_rewards"))
_append("successes", metrics.get("successes"))
# video_paths is list-like
paths = metrics.get("video_paths", [])
if paths:
group_acc[group]["video_paths"].extend(paths)
overall["video_paths"].extend(paths)
# Choose runner (sequential vs threaded)
task_runner = partial(
run_one,
policy=policy,
env_preprocessor=env_preprocessor,
env_postprocessor=env_postprocessor,
preprocessor=preprocessor,
postprocessor=postprocessor,
n_episodes=n_episodes,
max_episodes_rendered=max_episodes_rendered,
videos_dir=videos_dir,
return_episode_data=return_episode_data,
start_seed=start_seed,
)
if max_parallel_tasks <= 1:
# sequential path (single accumulator path on the main thread)
# NOTE: keeping a single-threaded accumulator avoids concurrent list appends or locks
for task_group, task_id, env in tasks:
tg, tid, metrics = task_runner(task_group, task_id, env)
_accumulate_to(tg, metrics)
per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
else:
# threaded path: submit all tasks, consume completions on main thread and accumulate there
with cf.ThreadPoolExecutor(max_workers=max_parallel_tasks) as executor:
fut2meta = {}
for task_group, task_id, env in tasks:
fut = executor.submit(task_runner, task_group, task_id, env)
fut2meta[fut] = (task_group, task_id)
for fut in cf.as_completed(fut2meta):
tg, tid, metrics = fut.result()
_accumulate_to(tg, metrics)
per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
# compute aggregated metrics helper (robust to lists/scalars)
def _agg_from_list(xs):
if not xs:
return float("nan")
arr = np.array(xs, dtype=float)
return float(np.nanmean(arr))
# compute per-group aggregates
groups_aggregated = {}
for group, acc in group_acc.items():
groups_aggregated[group] = {
"avg_sum_reward": _agg_from_list(acc["sum_rewards"]),
"avg_max_reward": _agg_from_list(acc["max_rewards"]),
"pc_success": _agg_from_list(acc["successes"]) * 100 if acc["successes"] else float("nan"),
"n_episodes": len(acc["sum_rewards"]),
"video_paths": list(acc["video_paths"]),
}
# overall aggregates
overall_agg = {
"avg_sum_reward": _agg_from_list(overall["sum_rewards"]),
"avg_max_reward": _agg_from_list(overall["max_rewards"]),
"pc_success": _agg_from_list(overall["successes"]) * 100 if overall["successes"] else float("nan"),
"n_episodes": len(overall["sum_rewards"]),
"eval_s": time.time() - start_t,
"eval_ep_s": (time.time() - start_t) / max(1, len(overall["sum_rewards"])),
"video_paths": list(overall["video_paths"]),
}
return {
"per_task": per_task_infos,
"per_group": groups_aggregated,
"overall": overall_agg,
}
def main():
init_logging()
register_third_party_plugins()
eval_main()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,319 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Helper to find the camera devices available in your system.
Example:
```shell
lerobot-find-cameras
```
"""
# NOTE(Steven): RealSense can also be identified/opened as OpenCV cameras. If you know the camera is a RealSense, use the `lerobot-find-cameras realsense` flag to avoid confusion.
# NOTE(Steven): macOS cameras sometimes report different FPS at init time, not an issue here as we don't specify FPS when opening the cameras, but the information displayed might not be truthful.
import argparse
import concurrent.futures
import logging
import time
from pathlib import Path
from typing import Any
import numpy as np
from PIL import Image
from lerobot.cameras.configs import ColorMode
from lerobot.cameras.opencv.camera_opencv import OpenCVCamera
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig
from lerobot.cameras.realsense.camera_realsense import RealSenseCamera
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig
logger = logging.getLogger(__name__)
def find_all_opencv_cameras() -> list[dict[str, Any]]:
"""
Finds all available OpenCV cameras plugged into the system.
Returns:
A list of all available OpenCV cameras with their metadata.
"""
all_opencv_cameras_info: list[dict[str, Any]] = []
logger.info("Searching for OpenCV cameras...")
try:
opencv_cameras = OpenCVCamera.find_cameras()
for cam_info in opencv_cameras:
all_opencv_cameras_info.append(cam_info)
logger.info(f"Found {len(opencv_cameras)} OpenCV cameras.")
except Exception as e:
logger.error(f"Error finding OpenCV cameras: {e}")
return all_opencv_cameras_info
def find_all_realsense_cameras() -> list[dict[str, Any]]:
"""
Finds all available RealSense cameras plugged into the system.
Returns:
A list of all available RealSense cameras with their metadata.
"""
all_realsense_cameras_info: list[dict[str, Any]] = []
logger.info("Searching for RealSense cameras...")
try:
realsense_cameras = RealSenseCamera.find_cameras()
for cam_info in realsense_cameras:
all_realsense_cameras_info.append(cam_info)
logger.info(f"Found {len(realsense_cameras)} RealSense cameras.")
except ImportError:
logger.warning("Skipping RealSense camera search: pyrealsense2 library not found or not importable.")
except Exception as e:
logger.error(f"Error finding RealSense cameras: {e}")
return all_realsense_cameras_info
def find_and_print_cameras(camera_type_filter: str | None = None) -> list[dict[str, Any]]:
"""
Finds available cameras based on an optional filter and prints their information.
Args:
camera_type_filter: Optional string to filter cameras ("realsense" or "opencv").
If None, lists all cameras.
Returns:
A list of all available cameras matching the filter, with their metadata.
"""
all_cameras_info: list[dict[str, Any]] = []
if camera_type_filter:
camera_type_filter = camera_type_filter.lower()
if camera_type_filter is None or camera_type_filter == "opencv":
all_cameras_info.extend(find_all_opencv_cameras())
if camera_type_filter is None or camera_type_filter == "realsense":
all_cameras_info.extend(find_all_realsense_cameras())
if not all_cameras_info:
if camera_type_filter:
logger.warning(f"No {camera_type_filter} cameras were detected.")
else:
logger.warning("No cameras (OpenCV or RealSense) were detected.")
else:
print("\n--- Detected Cameras ---")
for i, cam_info in enumerate(all_cameras_info):
print(f"Camera #{i}:")
for key, value in cam_info.items():
if key == "default_stream_profile" and isinstance(value, dict):
print(f" {key.replace('_', ' ').capitalize()}:")
for sub_key, sub_value in value.items():
print(f" {sub_key.capitalize()}: {sub_value}")
else:
print(f" {key.replace('_', ' ').capitalize()}: {value}")
print("-" * 20)
return all_cameras_info
def save_image(
img_array: np.ndarray,
camera_identifier: str | int,
images_dir: Path,
camera_type: str,
):
"""
Saves a single image to disk using Pillow. Handles color conversion if necessary.
"""
try:
img = Image.fromarray(img_array, mode="RGB")
safe_identifier = str(camera_identifier).replace("/", "_").replace("\\", "_")
filename_prefix = f"{camera_type.lower()}_{safe_identifier}"
filename = f"{filename_prefix}.png"
path = images_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
img.save(str(path))
logger.info(f"Saved image: {path}")
except Exception as e:
logger.error(f"Failed to save image for camera {camera_identifier} (type {camera_type}): {e}")
def create_camera_instance(cam_meta: dict[str, Any]) -> dict[str, Any] | None:
"""Create and connect to a camera instance based on metadata."""
cam_type = cam_meta.get("type")
cam_id = cam_meta.get("id")
instance = None
logger.info(f"Preparing {cam_type} ID {cam_id} with default profile")
try:
if cam_type == "OpenCV":
cv_config = OpenCVCameraConfig(
index_or_path=cam_id,
color_mode=ColorMode.RGB,
)
instance = OpenCVCamera(cv_config)
elif cam_type == "RealSense":
rs_config = RealSenseCameraConfig(
serial_number_or_name=cam_id,
color_mode=ColorMode.RGB,
)
instance = RealSenseCamera(rs_config)
else:
logger.warning(f"Unknown camera type: {cam_type} for ID {cam_id}. Skipping.")
return None
if instance:
logger.info(f"Connecting to {cam_type} camera: {cam_id}...")
instance.connect(warmup=True)
return {"instance": instance, "meta": cam_meta}
except Exception as e:
logger.error(f"Failed to connect or configure {cam_type} camera {cam_id}: {e}")
if instance and instance.is_connected:
instance.disconnect()
return None
def process_camera_image(
cam_dict: dict[str, Any], output_dir: Path, current_time: float
) -> concurrent.futures.Future | None:
"""Capture and process an image from a single camera."""
cam = cam_dict["instance"]
meta = cam_dict["meta"]
cam_type_str = str(meta.get("type", "unknown"))
cam_id_str = str(meta.get("id", "unknown"))
try:
image_data = cam.read()
return save_image(
image_data,
cam_id_str,
output_dir,
cam_type_str,
)
except TimeoutError:
logger.warning(
f"Timeout reading from {cam_type_str} camera {cam_id_str} at time {current_time:.2f}s."
)
except Exception as e:
logger.error(f"Error reading from {cam_type_str} camera {cam_id_str}: {e}")
return None
def cleanup_cameras(cameras_to_use: list[dict[str, Any]]):
"""Disconnect all cameras."""
logger.info(f"Disconnecting {len(cameras_to_use)} cameras...")
for cam_dict in cameras_to_use:
try:
if cam_dict["instance"] and cam_dict["instance"].is_connected:
cam_dict["instance"].disconnect()
except Exception as e:
logger.error(f"Error disconnecting camera {cam_dict['meta'].get('id')}: {e}")
def save_images_from_all_cameras(
output_dir: Path,
record_time_s: float = 2.0,
camera_type: str | None = None,
):
"""
Connects to detected cameras (optionally filtered by type) and saves images from each.
Uses default stream profiles for width, height, and FPS.
Args:
output_dir: Directory to save images.
record_time_s: Duration in seconds to record images.
camera_type: Optional string to filter cameras ("realsense" or "opencv").
If None, uses all detected cameras.
"""
output_dir.mkdir(parents=True, exist_ok=True)
logger.info(f"Saving images to {output_dir}")
all_camera_metadata = find_and_print_cameras(camera_type_filter=camera_type)
if not all_camera_metadata:
logger.warning("No cameras detected matching the criteria. Cannot save images.")
return
cameras_to_use = []
for cam_meta in all_camera_metadata:
camera_instance = create_camera_instance(cam_meta)
if camera_instance:
cameras_to_use.append(camera_instance)
if not cameras_to_use:
logger.warning("No cameras could be connected. Aborting image save.")
return
logger.info(f"Starting image capture for {record_time_s} seconds from {len(cameras_to_use)} cameras.")
start_time = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=len(cameras_to_use) * 2) as executor:
try:
while time.perf_counter() - start_time < record_time_s:
futures = []
current_capture_time = time.perf_counter()
for cam_dict in cameras_to_use:
future = process_camera_image(cam_dict, output_dir, current_capture_time)
if future:
futures.append(future)
if futures:
concurrent.futures.wait(futures)
except KeyboardInterrupt:
logger.info("Capture interrupted by user.")
finally:
print("\nFinalizing image saving...")
executor.shutdown(wait=True)
cleanup_cameras(cameras_to_use)
print(f"Image capture finished. Images saved to {output_dir}")
def main():
parser = argparse.ArgumentParser(
description="Unified camera utility script for listing cameras and capturing images."
)
parser.add_argument(
"camera_type",
type=str,
nargs="?",
default=None,
choices=["realsense", "opencv"],
help="Specify camera type to capture from (e.g., 'realsense', 'opencv'). Captures from all if omitted.",
)
parser.add_argument(
"--output-dir",
type=Path,
default="outputs/captured_images",
help="Directory to save images. Default: outputs/captured_images",
)
parser.add_argument(
"--record-time-s",
type=float,
default=6.0,
help="Time duration to attempt capturing frames. Default: 6 seconds.",
)
args = parser.parse_args()
save_images_from_all_cameras(**vars(args))
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,221 @@
#!/usr/bin/env python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Script to find joint limits and end-effector bounds via teleoperation.
Example:
```shell
lerobot-find-joint-limits \
--robot.type=so100_follower \
--robot.port=/dev/tty.usbmodem58760432981 \
--robot.id=black \
--teleop.type=so100_leader \
--teleop.port=/dev/tty.usbmodem58760434471 \
--teleop.id=blue \
--urdf_path=<user>/SO-ARM100-main/Simulation/SO101/so101_new_calib.urdf \
--target_frame_name=gripper \
--teleop_time_s=30 \
--warmup_time_s=5 \
--control_loop_fps=30
```
"""
import time
from dataclasses import dataclass
import draccus
import numpy as np
from lerobot.model.kinematics import RobotKinematics
from lerobot.robots import ( # noqa: F401
RobotConfig,
bi_openarm_follower,
bi_so_follower,
koch_follower,
make_robot_from_config,
omx_follower,
openarm_follower,
so_follower,
)
from lerobot.teleoperators import ( # noqa: F401
TeleoperatorConfig,
bi_openarm_leader,
bi_so_leader,
gamepad,
koch_leader,
make_teleoperator_from_config,
omx_leader,
openarm_leader,
so_leader,
)
from lerobot.utils.robot_utils import precise_sleep
@dataclass
class FindJointLimitsConfig:
teleop: TeleoperatorConfig
robot: RobotConfig
# Path to URDF file for kinematics
# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo:
# https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
urdf_path: str
target_frame_name: str = "gripper"
# Duration of the recording phase in seconds
teleop_time_s: float = 30
# Duration of the warmup phase in seconds
warmup_time_s: float = 5
# Control loop frequency
control_loop_fps: int = 30
@draccus.wrap()
def find_joint_and_ee_bounds(cfg: FindJointLimitsConfig):
teleop = make_teleoperator_from_config(cfg.teleop)
robot = make_robot_from_config(cfg.robot)
print(f"Connecting to robot: {cfg.robot.type}...")
teleop.connect()
robot.connect()
print("Devices connected.")
# Initialize Kinematics
try:
kinematics = RobotKinematics(cfg.urdf_path, cfg.target_frame_name)
except Exception as e:
print(f"Error initializing kinematics: {e}")
print("Ensure URDF path and target frame name are correct.")
robot.disconnect()
teleop.disconnect()
return
# Initialize variables
max_pos = None
min_pos = None
max_ee = None
min_ee = None
start_t = time.perf_counter()
warmup_done = False
print("\n" + "=" * 40)
print(f" WARMUP PHASE ({cfg.warmup_time_s}s)")
print(" Move the robot freely to ensure control works.")
print(" Data is NOT being recorded yet.")
print("=" * 40 + "\n")
try:
while True:
t0 = time.perf_counter()
# 1. Teleoperation Control Loop
action = teleop.get_action()
robot.send_action(action)
# 2. Read Observations
observation = robot.get_observation()
joint_positions = np.array([observation[f"{key}.pos"] for key in robot.bus.motors])
# 3. Calculate Kinematics
# Forward kinematics to get (x, y, z) translation
ee_pos = kinematics.forward_kinematics(joint_positions)[:3, 3]
current_time = time.perf_counter()
elapsed = current_time - start_t
# 4. Handle Phases
if elapsed < cfg.warmup_time_s:
# Still in warmup
pass
else:
# Phase Transition: Warmup -> Recording
if not warmup_done:
print("\n" + "=" * 40)
print(" RECORDING STARTED")
print(" Move robot to ALL joint limits.")
print(" Press Ctrl+C to stop early and save results.")
print("=" * 40 + "\n")
# Initialize limits with current position at start of recording
max_pos = joint_positions.copy()
min_pos = joint_positions.copy()
max_ee = ee_pos.copy()
min_ee = ee_pos.copy()
warmup_done = True
# Update Limits
max_ee = np.maximum(max_ee, ee_pos)
min_ee = np.minimum(min_ee, ee_pos)
max_pos = np.maximum(max_pos, joint_positions)
min_pos = np.minimum(min_pos, joint_positions)
# Time check
recording_time = elapsed - cfg.warmup_time_s
remaining = cfg.teleop_time_s - recording_time
# Simple throttle for print statements (every ~1 sec)
if int(recording_time * 100) % 100 == 0:
print(f"Time remaining: {remaining:.1f}s", end="\r")
if recording_time > cfg.teleop_time_s:
print("\nTime limit reached.")
break
precise_sleep(max(1.0 / cfg.control_loop_fps - (time.perf_counter() - t0), 0.0))
except KeyboardInterrupt:
print("\n\nInterrupted by user. Stopping safely...")
finally:
# Safety: Disconnect devices
print("\nDisconnecting devices...")
robot.disconnect()
teleop.disconnect()
# Results Output
if max_pos is not None:
print("\n" + "=" * 40)
print("FINAL RESULTS")
print("=" * 40)
# Rounding for readability
r_max_ee = np.round(max_ee, 4).tolist()
r_min_ee = np.round(min_ee, 4).tolist()
r_max_pos = np.round(max_pos, 4).tolist()
r_min_pos = np.round(min_pos, 4).tolist()
print("\n# End Effector Bounds (x, y, z):")
print(f"max_ee = {r_max_ee}")
print(f"min_ee = {r_min_ee}")
print("\n# Joint Position Limits (radians):")
print(f"max_pos = {r_max_pos}")
print(f"min_pos = {r_min_pos}")
else:
print("No data recorded (exited during warmup).")
def main():
find_joint_and_ee_bounds()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,69 @@
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Helper to find the USB port associated with your MotorsBus.
Example:
```shell
lerobot-find-port
```
"""
import platform
import time
from pathlib import Path
def find_available_ports():
from serial.tools import list_ports # Part of pyserial library
if platform.system() == "Windows":
# List COM ports using pyserial
ports = [port.device for port in list_ports.comports()]
else: # Linux/macOS
# List /dev/tty* ports for Unix-based systems
ports = [str(path) for path in Path("/dev").glob("tty*")]
return ports
def find_port():
print("Finding all available ports for the MotorsBus.")
ports_before = find_available_ports()
print("Ports before disconnecting:", ports_before)
print("Remove the USB cable from your MotorsBus and press Enter when done.")
input() # Wait for user to disconnect the device
time.sleep(0.5) # Allow some time for port to be released
ports_after = find_available_ports()
ports_diff = list(set(ports_before) - set(ports_after))
if len(ports_diff) == 1:
port = ports_diff[0]
print(f"The port of this MotorsBus is '{port}'")
print("Reconnect the USB cable.")
elif len(ports_diff) == 0:
raise OSError(f"Could not detect the port. No difference was found ({ports_diff}).")
else:
raise OSError(f"Could not detect the port. More than one port was found ({ports_diff}).")
def main():
find_port()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,168 @@
#!/usr/bin/env python
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Records with policy execution and teleop-device action mirroring enabled.
Phase A/B goals:
- Same policy action is executed on the follower robot and mirrored to the teleop arm.
- Keyboard-toggled intervention lets teleop temporarily take over execution.
"""
import logging
import json
import time
from pathlib import Path
from typing import Any
from lerobot.configs import parser
from lerobot.scripts.lerobot_record import RecordConfig, record
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.constants import HF_LEROBOT_HOME
from lerobot.utils.recording_annotations import EPISODE_FAILURE, infer_collector_policy_version
def _default_failure_reset_pose_path(cfg: RecordConfig) -> Path:
robot_id = cfg.robot.id if cfg.robot.id else "default"
robot_type = cfg.robot.type if hasattr(cfg.robot, "type") else type(cfg.robot).__name__
return HF_LEROBOT_HOME / "failure_reset_pose" / f"{robot_type}_{robot_id}.json"
def _extract_joint_pos_from_observation(observation: dict[str, Any]) -> dict[str, float]:
return {key: float(value) for key, value in observation.items() if key.endswith(".pos")}
def _save_failure_reset_pose(robot: Any, pose_path: Path) -> dict[str, float]:
observation = robot.get_observation()
joint_pos = _extract_joint_pos_from_observation(observation)
if not joint_pos:
raise ValueError("Could not capture failure reset pose: no '.pos' joints found in observation.")
pose_path.parent.mkdir(parents=True, exist_ok=True)
payload = {"robot_type": robot.robot_type, "joint_pos": joint_pos}
with open(pose_path, "w") as f:
json.dump(payload, f, indent=2, sort_keys=True)
logging.info("Saved failure_reset_pose to %s", pose_path)
return joint_pos
def _load_failure_reset_pose(pose_path: Path) -> dict[str, float]:
with open(pose_path) as f:
payload = json.load(f)
joint_pos_raw = payload["joint_pos"] if isinstance(payload, dict) and "joint_pos" in payload else payload
if not isinstance(joint_pos_raw, dict):
raise ValueError(f"Invalid failure reset pose payload in {pose_path}: expected dict, got {type(joint_pos_raw)}")
joint_pos = {str(key): float(value) for key, value in joint_pos_raw.items() if str(key).endswith(".pos")}
if not joint_pos:
raise ValueError(f"Invalid failure reset pose payload in {pose_path}: no '.pos' joints found.")
logging.info("Loaded failure_reset_pose from %s", pose_path)
return joint_pos
def _slow_reset_all_arms_to_pose(
robot: Any,
teleop: Any,
target_pose: dict[str, float],
duration_s: float = 3.0,
) -> None:
joint_keys = [key for key in robot.action_features if key.endswith(".pos") and key in target_pose]
if not joint_keys:
logging.warning("No matching '.pos' joints found for failure reset pose.")
return
current_pose = _extract_joint_pos_from_observation(robot.get_observation())
start_pose = {key: current_pose.get(key, float(target_pose[key])) for key in joint_keys}
goal_pose = {key: float(target_pose[key]) for key in joint_keys}
if teleop is not None and not isinstance(teleop, list) and hasattr(teleop, "set_manual_control"):
teleop.set_manual_control(False)
step_dt_s = 0.05
steps = max(int(duration_s / step_dt_s), 1)
for idx in range(1, steps + 1):
alpha = idx / steps
action = {key: start_pose[key] + (goal_pose[key] - start_pose[key]) * alpha for key in joint_keys}
robot.send_action(action)
if teleop is not None and not isinstance(teleop, list):
teleop.send_feedback(action)
time.sleep(step_dt_s)
logging.info("Episode marked as failure. Arms returned to failure reset pose in %.1fs.", duration_s)
class _HumanInloopFailureResetController:
def __init__(self, cfg: RecordConfig):
self.pose_path = _default_failure_reset_pose_path(cfg)
self.failure_reset_pose: dict[str, float] | None = None
def on_record_connected(self, robot: Any, teleop: Any) -> None:
if self.pose_path.is_file():
self.failure_reset_pose = _load_failure_reset_pose(self.pose_path)
return
input(
"Human-inloop with policy detected.\n"
"Please ensure ALL robot arms are at reset position, then press ENTER to capture:\n"
f"{self.pose_path}\n"
)
self.failure_reset_pose = _save_failure_reset_pose(robot=robot, pose_path=self.pose_path)
def on_episode_outcome(self, robot: Any, teleop: Any, episode_success: str | None) -> None:
if episode_success == EPISODE_FAILURE and self.failure_reset_pose is not None:
_slow_reset_all_arms_to_pose(robot=robot, teleop=teleop, target_pose=self.failure_reset_pose)
@parser.wrap()
def human_inloop_record(cfg: RecordConfig):
if cfg.teleop is None:
raise ValueError("`lerobot-human-inloop-record` requires `teleop` config.")
cfg.policy_sync_to_teleop = cfg.policy is not None
cfg.intervention_state_machine_enabled = cfg.policy is not None
cfg.enable_episode_outcome_labeling = True
cfg.default_episode_success = "failure"
cfg.enable_collector_policy_id = True
if cfg.collector_policy_id_policy is None:
cfg.collector_policy_id_policy = infer_collector_policy_version(cfg.policy)
if cfg.policy is not None:
failure_reset_controller = _HumanInloopFailureResetController(cfg)
cfg._on_record_connected = failure_reset_controller.on_record_connected
cfg._on_record_episode_outcome = failure_reset_controller.on_episode_outcome
logging.info(
"Human-in-loop recording is enabled. Press '%s' to toggle takeover. "
"Press '%s' to mark success and end, '%s' to mark failure and end. "
"Recorded `action` is the executed action. "
"Policy output (when policy is enabled) is stored in `complementary_info.policy_action`. "
"Collector source is stored in `complementary_info.collector_policy_id`. "
"ACP inference: enable=%s use_cfg=%s cfg_beta=%.3f.",
cfg.intervention_toggle_key,
cfg.episode_success_key,
cfg.episode_failure_key,
cfg.acp_inference.enable,
cfg.acp_inference.use_cfg,
cfg.acp_inference.cfg_beta,
)
return record(cfg)
def main():
register_third_party_plugins()
human_inloop_record()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,134 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Visualize effects of image transforms for a given configuration.
This script will generate examples of transformed images as they are output by LeRobot dataset.
Additionally, each individual transform can be visualized separately as well as examples of combined transforms
Example:
```bash
lerobot-imgtransform-viz \
--repo_id=lerobot/pusht \
--episodes='[0]' \
--image_transforms.enable=True
```
"""
import logging
from copy import deepcopy
from dataclasses import replace
from pathlib import Path
import draccus
from torchvision.transforms import ToPILImage
from lerobot.configs.default import DatasetConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.transforms import (
ImageTransforms,
ImageTransformsConfig,
make_transform_from_config,
)
OUTPUT_DIR = Path("outputs/image_transforms")
to_pil = ToPILImage()
def save_all_transforms(cfg: ImageTransformsConfig, original_frame, output_dir, n_examples):
output_dir_all = output_dir / "all"
output_dir_all.mkdir(parents=True, exist_ok=True)
tfs = ImageTransforms(cfg)
for i in range(1, n_examples + 1):
transformed_frame = tfs(original_frame)
to_pil(transformed_frame).save(output_dir_all / f"{i}.png", quality=100)
print("Combined transforms examples saved to:")
print(f" {output_dir_all}")
def save_each_transform(cfg: ImageTransformsConfig, original_frame, output_dir, n_examples):
if not cfg.enable:
logging.warning(
"No single transforms will be saved, because `image_transforms.enable=False`. To enable, set `enable` to True in `ImageTransformsConfig` or in the command line with `--image_transforms.enable=True`."
)
return
print("Individual transforms examples saved to:")
for tf_name, tf_cfg in cfg.tfs.items():
# Apply a few transformation with random value in min_max range
output_dir_single = output_dir / tf_name
output_dir_single.mkdir(parents=True, exist_ok=True)
tf = make_transform_from_config(tf_cfg)
for i in range(1, n_examples + 1):
transformed_frame = tf(original_frame)
to_pil(transformed_frame).save(output_dir_single / f"{i}.png", quality=100)
# Apply min, max, average transformations
tf_cfg_kwgs_min = deepcopy(tf_cfg.kwargs)
tf_cfg_kwgs_max = deepcopy(tf_cfg.kwargs)
tf_cfg_kwgs_avg = deepcopy(tf_cfg.kwargs)
for key, (min_, max_) in tf_cfg.kwargs.items():
avg = (min_ + max_) / 2
tf_cfg_kwgs_min[key] = [min_, min_]
tf_cfg_kwgs_max[key] = [max_, max_]
tf_cfg_kwgs_avg[key] = [avg, avg]
tf_min = make_transform_from_config(replace(tf_cfg, **{"kwargs": tf_cfg_kwgs_min}))
tf_max = make_transform_from_config(replace(tf_cfg, **{"kwargs": tf_cfg_kwgs_max}))
tf_avg = make_transform_from_config(replace(tf_cfg, **{"kwargs": tf_cfg_kwgs_avg}))
tf_frame_min = tf_min(original_frame)
tf_frame_max = tf_max(original_frame)
tf_frame_avg = tf_avg(original_frame)
to_pil(tf_frame_min).save(output_dir_single / "min.png", quality=100)
to_pil(tf_frame_max).save(output_dir_single / "max.png", quality=100)
to_pil(tf_frame_avg).save(output_dir_single / "mean.png", quality=100)
print(f" {output_dir_single}")
@draccus.wrap()
def visualize_image_transforms(cfg: DatasetConfig, output_dir: Path = OUTPUT_DIR, n_examples: int = 5):
dataset = LeRobotDataset(
repo_id=cfg.repo_id,
episodes=cfg.episodes,
revision=cfg.revision,
video_backend=cfg.video_backend,
)
output_dir = output_dir / cfg.repo_id.split("/")[-1]
output_dir.mkdir(parents=True, exist_ok=True)
# Get 1st frame from 1st camera of 1st episode
original_frame = dataset[0][dataset.meta.camera_keys[0]]
to_pil(original_frame).save(output_dir / "original_frame.png", quality=100)
print("\nOriginal frame saved to:")
print(f" {output_dir / 'original_frame.png'}.")
save_all_transforms(cfg.image_transforms, original_frame, output_dir, n_examples)
save_each_transform(cfg.image_transforms, original_frame, output_dir, n_examples)
def main():
visualize_image_transforms()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,126 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Use this script to get a quick summary of your system config.
It should be able to run without any of LeRobot's dependencies or LeRobot itself installed.
Example:
```shell
lerobot-info
```
"""
import importlib
import platform
import shutil
import subprocess
from importlib.metadata import PackageNotFoundError, distribution
PACKAGE_NAME = "lerobot"
def get_ffmpeg_version() -> str:
"""Get the ffmpeg version if installed, otherwise return 'N/A'."""
command_path = shutil.which("ffmpeg")
if command_path is None:
return "N/A"
try:
result = subprocess.run([command_path, "-version"], capture_output=True, text=True, check=True)
first_line = result.stdout.splitlines()[0]
version_info = first_line.split(" ")[2]
return version_info
except (subprocess.SubprocessError, IndexError):
return "Installed (version parsing failed)"
def get_package_version(package_name: str) -> str:
"""Get the version of a package if it exists, otherwise return 'N/A'."""
try:
module = importlib.import_module(package_name)
return getattr(module, "__version__", "Installed (version not found)")
except ImportError:
return "N/A"
def get_sys_info() -> dict[str, str]:
"""Run this to get basic system info to help for tracking issues & bugs."""
# General package versions
info = {
"LeRobot version": get_package_version(PACKAGE_NAME),
"Platform": platform.platform(),
"Python version": platform.python_version(),
"Huggingface Hub version": get_package_version("huggingface_hub"),
"Datasets version": get_package_version("datasets"),
"Numpy version": get_package_version("numpy"),
"FFmpeg version": get_ffmpeg_version(),
}
# PyTorch and GPU specific information
torch_version = "N/A"
torch_cuda_available = "N/A"
cuda_version = "N/A"
gpu_model = "N/A"
try:
import torch
torch_version = str(torch.__version__)
torch_cuda_available = torch.cuda.is_available()
if torch_cuda_available:
cuda_version = str(torch.version.cuda)
# Gets the name of the first available GPU
gpu_model = torch.cuda.get_device_name(0)
except ImportError:
# If torch is not installed, the default "N/A" values will be used.
pass
info.update(
{
"PyTorch version": torch_version,
"Is PyTorch built with CUDA support?": str(torch_cuda_available),
"Cuda version": cuda_version,
"GPU model": gpu_model,
"Using GPU in script?": "<fill in>",
}
)
scripts = "N/A"
try:
dist = distribution(PACKAGE_NAME)
scripts = [ep.name for ep in dist.entry_points if ep.group == "console_scripts"]
except PackageNotFoundError:
pass
info.update({f"{PACKAGE_NAME} scripts": str(scripts)})
return info
def format_dict_for_markdown(d: dict[str, str]) -> str:
"""Formats a dictionary into a markdown-friendly bulleted list."""
return "\n".join([f"- {prop}: {val}" for prop, val in d.items()])
def main():
"""
Main function to print system info in markdown format.
"""
system_info = get_sys_info()
print(format_dict_for_markdown(system_info))
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,568 @@
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Records a dataset. Actions for the robot can be either generated by teleoperation or by a policy.
Example:
```shell
lerobot-record \
--robot.type=so100_follower \
--robot.port=/dev/tty.usbmodem58760431541 \
--robot.cameras="{laptop: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \
--robot.id=black \
--dataset.repo_id=<my_username>/<my_dataset_name> \
--dataset.num_episodes=2 \
--dataset.single_task="Grab the cube" \
--display_data=true
# <- Optional: specify video codec (h264, hevc, libsvtav1). Default is libsvtav1. \
# --dataset.vcodec=h264 \
# <- Teleop optional if you want to teleoperate to record or in between episodes with a policy \
# --teleop.type=so100_leader \
# --teleop.port=/dev/tty.usbmodem58760431551 \
# --teleop.id=blue \
# <- Policy optional if you want to record with a policy \
# --policy.path=${HF_USER}/my_policy \
```
Example recording with bimanual so100:
```shell
lerobot-record \
--robot.type=bi_so_follower \
--robot.left_arm_config.port=/dev/tty.usbmodem5A460822851 \
--robot.right_arm_config.port=/dev/tty.usbmodem5A460814411 \
--robot.id=bimanual_follower \
--robot.left_arm_config.cameras='{
wrist: {"type": "opencv", "index_or_path": 1, "width": 640, "height": 480, "fps": 30},
top: {"type": "opencv", "index_or_path": 3, "width": 640, "height": 480, "fps": 30},
}' --robot.right_arm_config.cameras='{
wrist: {"type": "opencv", "index_or_path": 2, "width": 640, "height": 480, "fps": 30},
front: {"type": "opencv", "index_or_path": 4, "width": 640, "height": 480, "fps": 30},
}' \
--teleop.type=bi_so_leader \
--teleop.left_arm_config.port=/dev/tty.usbmodem5A460852721 \
--teleop.right_arm_config.port=/dev/tty.usbmodem5A460819811 \
--teleop.id=bimanual_leader \
--display_data=true \
--dataset.repo_id=${HF_USER}/bimanual-so-handover-cube \
--dataset.num_episodes=25 \
--dataset.single_task="Grab and handover the red cube to the other arm"
```
"""
import logging
from dataclasses import asdict, dataclass, field
from pathlib import Path
from pprint import pformat
from lerobot.cameras import ( # noqa: F401
CameraConfig, # noqa: F401
)
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401
from lerobot.cameras.reachy2_camera.configuration_reachy2_camera import Reachy2CameraConfig # noqa: F401
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401
from lerobot.cameras.zmq.configuration_zmq import ZMQCameraConfig # noqa: F401
from lerobot.configs import parser
from lerobot.configs.policies import PreTrainedConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.pipeline_features import aggregate_pipeline_dataset_features, create_initial_features
from lerobot.datasets.utils import combine_feature_dicts
from lerobot.datasets.video_utils import VideoEncodingManager
from lerobot.policies.factory import make_policy, make_pre_post_processors
from lerobot.processor import make_default_processors
from lerobot.processor.rename_processor import rename_stats
from lerobot.robots import ( # noqa: F401
RobotConfig,
bi_openarm_follower,
bi_so_follower,
earthrover_mini_plus,
hope_jr,
koch_follower,
make_robot_from_config,
omx_follower,
openarm_follower,
reachy2,
so_follower,
unitree_g1 as unitree_g1_robot,
)
from lerobot.teleoperators import ( # noqa: F401
TeleoperatorConfig,
bi_openarm_leader,
bi_so_leader,
homunculus,
koch_leader,
make_teleoperator_from_config,
omx_leader,
openarm_leader,
reachy2_teleoperator,
so_leader,
unitree_g1,
)
from lerobot.scripts.recording_hil import (
ACPInferenceConfig,
PolicySyncDualArmExecutor,
_capture_policy_runtime_state, # noqa: F401
_predict_policy_action_with_acp_inference, # noqa: F401
)
from lerobot.scripts.recording_loop import record_loop
from lerobot.utils.constants import ACTION
from lerobot.utils.control_utils import (
init_keyboard_listener,
is_headless,
sanity_check_dataset_name,
sanity_check_dataset_robot_compatibility,
)
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.recording_annotations import (
infer_collector_policy_id,
normalize_episode_success_label,
resolve_episode_success_label,
)
from lerobot.utils.utils import (
init_logging,
log_say,
)
from lerobot.utils.visualization_utils import init_rerun
@dataclass
class DatasetRecordConfig:
# Dataset identifier. By convention it should match '{hf_username}/{dataset_name}' (e.g. `lerobot/test`).
repo_id: str
# A short but accurate description of the task performed during the recording (e.g. "Pick the Lego block and drop it in the box on the right.")
single_task: str
# Root directory where the dataset will be stored (e.g. 'dataset/path').
root: str | Path | None = None
# Limit the frames per second.
fps: int = 30
# Number of seconds for data recording for each episode.
episode_time_s: int | float = 60
# Number of seconds for resetting the environment after each episode.
reset_time_s: int | float = 60
# Number of episodes to record.
num_episodes: int = 50
# Encode frames in the dataset into video
video: bool = True
# Upload dataset to Hugging Face hub.
push_to_hub: bool = True
# Upload on private repository on the Hugging Face hub.
private: bool = False
# Add tags to your dataset on the hub.
tags: list[str] | None = None
# Number of subprocesses handling the saving of frames as PNG. Set to 0 to use threads only;
# set to ≥1 to use subprocesses, each using threads to write images. The best number of processes
# and threads depends on your system. We recommend 4 threads per camera with 0 processes.
# If fps is unstable, adjust the thread count. If still unstable, try using 1 or more subprocesses.
num_image_writer_processes: int = 0
# Number of threads writing the frames as png images on disk, per camera.
# Too many threads might cause unstable teleoperation fps due to main thread being blocked.
# Not enough threads might cause low camera fps.
num_image_writer_threads_per_camera: int = 4
# Number of episodes to record before batch encoding videos
# Set to 1 for immediate encoding (default behavior), or higher for batched encoding
video_encoding_batch_size: int = 1
# Video codec for encoding videos. Options: 'h264', 'hevc', 'libsvtav1'.
# Use 'h264' for faster encoding on systems where AV1 encoding is CPU-heavy.
vcodec: str = "libsvtav1"
# Rename map for the observation to override the image and state keys
rename_map: dict[str, str] = field(default_factory=dict)
def __post_init__(self):
if self.single_task is None:
raise ValueError("You need to provide a task as argument in `single_task`.")
@dataclass
class RecordConfig:
robot: RobotConfig
dataset: DatasetRecordConfig
# Whether to control the robot with a teleoperator
teleop: TeleoperatorConfig | None = None
# Whether to control the robot with a policy
policy: PreTrainedConfig | None = None
# Display all cameras on screen
display_data: bool = False
# Display data on a remote Rerun server
display_ip: str | None = None
# Port of the remote Rerun server
display_port: int | None = None
# Whether to display compressed images in Rerun
display_compressed_images: bool = False
# Use vocal synthesis to read events.
play_sounds: bool = True
# Resume recording on an existing dataset.
resume: bool = False
# In policy mode, broadcast the same robot action to the teleop arm via `teleop.send_feedback`.
policy_sync_to_teleop: bool = False
# Use parallel dispatch to reduce action broadcast latency when syncing policy to teleop.
policy_sync_parallel: bool = True
# Enable S0/S1/S2 intervention state machine when policy + teleop are both available.
intervention_state_machine_enabled: bool = True
# Keyboard key used to toggle entering/leaving intervention.
intervention_toggle_key: str = "i"
# Whether to capture episode-level success/failure labels from keyboard.
enable_episode_outcome_labeling: bool = False
# Keyboard key to mark the current episode as success and end it.
episode_success_key: str = "s"
# Keyboard key to mark the current episode as failure and end it.
episode_failure_key: str = "f"
# Optional fallback label used when no explicit success/failure key was pressed.
default_episode_success: str | None = None
# If True, require explicit or default episode labels before saving.
require_episode_success_label: bool = False
# Whether to store step-level collector policy provenance.
enable_collector_policy_id: bool = False
# Policy identifier used when action source is policy. If omitted, inferred from policy config.
collector_policy_id_policy: str | None = None
# Policy identifier used when action source is human/teleop.
collector_policy_id_human: str = "human"
# ACP inference controls for policy-driven recording.
acp_inference: ACPInferenceConfig = field(default_factory=ACPInferenceConfig)
# Retry timeout for transient communication errors (seconds). Set to 0 to fail immediately.
communication_retry_timeout_s: float = 2.0
# Sleep interval between communication retries (seconds).
communication_retry_interval_s: float = 0.1
def __post_init__(self):
# HACK: We parse again the cli args here to get the pretrained path if there was one.
policy_path = parser.get_path_arg("policy")
if policy_path:
cli_overrides = parser.get_cli_overrides("policy")
self.policy = PreTrainedConfig.from_pretrained(policy_path, cli_overrides=cli_overrides)
self.policy.pretrained_path = policy_path
if self.teleop is None and self.policy is None:
raise ValueError("Choose a policy, a teleoperator or both to control the robot")
if not self.intervention_toggle_key or len(self.intervention_toggle_key) != 1:
raise ValueError("`intervention_toggle_key` must be a single character.")
if self.enable_episode_outcome_labeling:
label_key_bindings = {
"episode_success_key": self.episode_success_key,
"episode_failure_key": self.episode_failure_key,
}
for key_name, key_value in label_key_bindings.items():
if not key_value or len(key_value) != 1:
raise ValueError(f"`{key_name}` must be a single character.")
normalized_keys = [
self.intervention_toggle_key.lower(),
self.episode_success_key.lower(),
self.episode_failure_key.lower(),
]
if len(set(normalized_keys)) != len(normalized_keys):
raise ValueError(
"`intervention_toggle_key`, `episode_success_key`, and `episode_failure_key` must be distinct."
)
if self.default_episode_success is not None:
self.default_episode_success = normalize_episode_success_label(self.default_episode_success)
if not self.collector_policy_id_human:
raise ValueError("`collector_policy_id_human` must be a non-empty string.")
if self.acp_inference.use_cfg and not self.acp_inference.enable:
raise ValueError("`acp_inference.use_cfg=true` requires `acp_inference.enable=true`.")
if self.acp_inference.cfg_beta < 0:
raise ValueError("`acp_inference.cfg_beta` must be >= 0.")
if self.communication_retry_timeout_s < 0:
raise ValueError("`communication_retry_timeout_s` must be >= 0.")
if self.communication_retry_interval_s <= 0:
raise ValueError("`communication_retry_interval_s` must be > 0.")
@classmethod
def __get_path_fields__(cls) -> list[str]:
"""This enables the parser to load config from the policy using `--policy.path=local/dir`"""
return ["policy"]
@parser.wrap()
def record(cfg: RecordConfig) -> LeRobotDataset:
init_logging()
if cfg.require_episode_success_label and not cfg.enable_episode_outcome_labeling:
raise ValueError(
"`require_episode_success_label=true` requires `enable_episode_outcome_labeling=true`."
)
logging.info(pformat(asdict(cfg)))
if cfg.display_data:
init_rerun(session_name="recording", ip=cfg.display_ip, port=cfg.display_port)
display_compressed_images = (
True
if (cfg.display_data and cfg.display_ip is not None and cfg.display_port is not None)
else cfg.display_compressed_images
)
robot = make_robot_from_config(cfg.robot)
teleop = make_teleoperator_from_config(cfg.teleop) if cfg.teleop is not None else None
teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
dataset_features = combine_feature_dicts(
aggregate_pipeline_dataset_features(
pipeline=teleop_action_processor,
initial_features=create_initial_features(
action=robot.action_features
), # TODO(steven, pepijn): in future this should be come from teleop or policy
use_videos=cfg.dataset.video,
),
aggregate_pipeline_dataset_features(
pipeline=robot_observation_processor,
initial_features=create_initial_features(observation=robot.observation_features),
use_videos=cfg.dataset.video,
),
)
if cfg.intervention_state_machine_enabled and cfg.policy is not None and cfg.teleop is not None:
action_names = dataset_features[ACTION]["names"]
if action_names is None:
action_names = list(robot.action_features)
else:
action_names = list(action_names)
dataset_features["complementary_info.policy_action"] = {
"dtype": "float32",
"shape": (len(action_names),),
"names": action_names,
}
dataset_features["complementary_info.is_intervention"] = {
"dtype": "float32",
"shape": (1,),
"names": ["is_intervention"],
}
dataset_features["complementary_info.state"] = {
"dtype": "float32",
"shape": (1,),
"names": ["state"],
}
if cfg.enable_collector_policy_id:
dataset_features["complementary_info.collector_policy_id"] = {
"dtype": "string",
"shape": (1,),
"names": ["collector_policy_id"],
}
dataset = None
listener = None
policy_sync_executor = None
try:
if cfg.resume:
dataset = LeRobotDataset(
cfg.dataset.repo_id,
root=cfg.dataset.root,
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
vcodec=cfg.dataset.vcodec,
)
if hasattr(robot, "cameras") and len(robot.cameras) > 0:
dataset.start_image_writer(
num_processes=cfg.dataset.num_image_writer_processes,
num_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras),
)
sanity_check_dataset_robot_compatibility(dataset, robot, cfg.dataset.fps, dataset_features)
else:
# Create empty dataset or load existing saved episodes
sanity_check_dataset_name(cfg.dataset.repo_id, cfg.policy)
dataset = LeRobotDataset.create(
cfg.dataset.repo_id,
cfg.dataset.fps,
root=cfg.dataset.root,
robot_type=robot.name,
features=dataset_features,
use_videos=cfg.dataset.video,
image_writer_processes=cfg.dataset.num_image_writer_processes,
image_writer_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras),
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
vcodec=cfg.dataset.vcodec,
)
# Load pretrained policy
policy = None if cfg.policy is None else make_policy(cfg.policy, ds_meta=dataset.meta)
preprocessor = None
postprocessor = None
if cfg.acp_inference.enable and cfg.policy is None:
raise ValueError("`acp_inference.enable=true` requires `policy` to be set.")
if cfg.policy is not None:
preprocessor, postprocessor = make_pre_post_processors(
policy_cfg=cfg.policy,
pretrained_path=cfg.policy.pretrained_path,
dataset_stats=rename_stats(dataset.meta.stats, cfg.dataset.rename_map),
preprocessor_overrides={
"device_processor": {"device": cfg.policy.device},
"rename_observations_processor": {"rename_map": cfg.dataset.rename_map},
},
)
collector_policy_id_policy = (
cfg.collector_policy_id_policy
if cfg.collector_policy_id_policy is not None
else infer_collector_policy_id(cfg.policy)
)
collector_policy_id_human = cfg.collector_policy_id_human
robot.connect()
if teleop is not None:
teleop.connect()
on_record_connected = getattr(cfg, "_on_record_connected", None)
if callable(on_record_connected):
on_record_connected(robot, teleop)
if cfg.policy_sync_to_teleop:
if cfg.policy is None:
raise ValueError("`policy_sync_to_teleop=true` requires `policy` to be set.")
if teleop is None or isinstance(teleop, list):
raise ValueError(
"`policy_sync_to_teleop=true` requires exactly one teleoperator with send_feedback support."
)
policy_sync_executor = PolicySyncDualArmExecutor(
robot=robot,
teleop=teleop,
parallel_dispatch=cfg.policy_sync_parallel,
)
listener, events = init_keyboard_listener(
intervention_toggle_key=cfg.intervention_toggle_key,
episode_success_key=cfg.episode_success_key if cfg.enable_episode_outcome_labeling else None,
episode_failure_key=cfg.episode_failure_key if cfg.enable_episode_outcome_labeling else None,
)
with VideoEncodingManager(dataset):
recorded_episodes = 0
while recorded_episodes < cfg.dataset.num_episodes and not events["stop_recording"]:
events["episode_outcome"] = None
log_say(f"Recording episode {dataset.num_episodes}", cfg.play_sounds)
record_loop(
robot=robot,
events=events,
fps=cfg.dataset.fps,
teleop_action_processor=teleop_action_processor,
robot_action_processor=robot_action_processor,
robot_observation_processor=robot_observation_processor,
teleop=teleop,
policy=policy,
preprocessor=preprocessor,
postprocessor=postprocessor,
dataset=dataset,
control_time_s=cfg.dataset.episode_time_s,
single_task=cfg.dataset.single_task,
display_data=cfg.display_data,
display_compressed_images=display_compressed_images,
policy_sync_executor=policy_sync_executor,
intervention_state_machine_enabled=cfg.intervention_state_machine_enabled,
collector_policy_id_policy=collector_policy_id_policy,
collector_policy_id_human=collector_policy_id_human,
acp_inference=cfg.acp_inference,
communication_retry_timeout_s=cfg.communication_retry_timeout_s,
communication_retry_interval_s=cfg.communication_retry_interval_s,
)
episode_success = None
if cfg.enable_episode_outcome_labeling:
episode_success = resolve_episode_success_label(
explicit_label=events.get("episode_outcome"),
default_label=cfg.default_episode_success,
require_label=cfg.require_episode_success_label,
)
if events.get("episode_outcome") is None and episode_success is not None:
logging.warning(
"Episode %s has no explicit success/failure label, defaulting to '%s'.",
dataset.num_episodes,
episode_success,
)
on_episode_outcome = getattr(cfg, "_on_record_episode_outcome", None)
if callable(on_episode_outcome):
on_episode_outcome(robot, teleop, episode_success)
# Execute a few seconds without recording to give time to manually reset the environment
# Skip reset for the last episode to be recorded
if not events["stop_recording"] and (
(recorded_episodes < cfg.dataset.num_episodes - 1) or events["rerecord_episode"]
):
log_say("Reset the environment", cfg.play_sounds)
# reset g1 robot
if robot.name == "unitree_g1":
robot.reset()
record_loop(
robot=robot,
events=events,
fps=cfg.dataset.fps,
teleop_action_processor=teleop_action_processor,
robot_action_processor=robot_action_processor,
robot_observation_processor=robot_observation_processor,
teleop=teleop,
control_time_s=cfg.dataset.reset_time_s,
single_task=cfg.dataset.single_task,
display_data=cfg.display_data,
policy_sync_executor=policy_sync_executor,
intervention_state_machine_enabled=cfg.intervention_state_machine_enabled,
collector_policy_id_policy=collector_policy_id_policy,
collector_policy_id_human=collector_policy_id_human,
acp_inference=cfg.acp_inference,
communication_retry_timeout_s=cfg.communication_retry_timeout_s,
communication_retry_interval_s=cfg.communication_retry_interval_s,
)
if events["rerecord_episode"]:
log_say("Re-record episode", cfg.play_sounds)
events["rerecord_episode"] = False
events["exit_early"] = False
events["episode_outcome"] = None
dataset.clear_episode_buffer()
continue
extra_episode_metadata = (
{"episode_success": episode_success}
if cfg.enable_episode_outcome_labeling
else None
)
dataset.save_episode(extra_episode_metadata=extra_episode_metadata)
recorded_episodes += 1
finally:
log_say("Stop recording", cfg.play_sounds, blocking=True)
if dataset:
dataset.finalize()
if policy_sync_executor is not None:
policy_sync_executor.shutdown()
if robot.is_connected:
robot.disconnect()
if teleop and teleop.is_connected:
teleop.disconnect()
if not is_headless() and listener:
listener.stop()
if cfg.dataset.push_to_hub:
if dataset is not None:
dataset.push_to_hub(tags=cfg.dataset.tags, private=cfg.dataset.private)
else:
logging.warning(
"`dataset.push_to_hub=true` was requested, but dataset was not initialized due to an earlier error."
)
log_say("Exiting", cfg.play_sounds)
return dataset
def main():
register_third_party_plugins()
record()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,141 @@
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Replays the actions of an episode from a dataset on a robot.
Examples:
```shell
lerobot-replay \
--robot.type=so100_follower \
--robot.port=/dev/tty.usbmodem58760431541 \
--robot.id=black \
--dataset.repo_id=aliberts/record-test \
--dataset.episode=0
```
Example replay with bimanual so100:
```shell
lerobot-replay \
--robot.type=bi_so_follower \
--robot.left_arm_port=/dev/tty.usbmodem5A460851411 \
--robot.right_arm_port=/dev/tty.usbmodem5A460812391 \
--robot.id=bimanual_follower \
--dataset.repo_id=${HF_USER}/bimanual-so100-handover-cube \
--dataset.episode=0
```
"""
import logging
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from pprint import pformat
from lerobot.configs import parser
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.processor import (
make_default_robot_action_processor,
)
from lerobot.robots import ( # noqa: F401
Robot,
RobotConfig,
bi_openarm_follower,
bi_so_follower,
earthrover_mini_plus,
hope_jr,
koch_follower,
make_robot_from_config,
omx_follower,
openarm_follower,
reachy2,
so_follower,
unitree_g1,
)
from lerobot.utils.constants import ACTION
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.robot_utils import precise_sleep
from lerobot.utils.utils import (
init_logging,
log_say,
)
@dataclass
class DatasetReplayConfig:
# Dataset identifier. By convention it should match '{hf_username}/{dataset_name}' (e.g. `lerobot/test`).
repo_id: str
# Episode to replay.
episode: int
# Root directory where the dataset will be stored (e.g. 'dataset/path').
root: str | Path | None = None
# Limit the frames per second. By default, uses the policy fps.
fps: int = 30
@dataclass
class ReplayConfig:
robot: RobotConfig
dataset: DatasetReplayConfig
# Use vocal synthesis to read events.
play_sounds: bool = True
@parser.wrap()
def replay(cfg: ReplayConfig):
init_logging()
logging.info(pformat(asdict(cfg)))
robot_action_processor = make_default_robot_action_processor()
robot = make_robot_from_config(cfg.robot)
dataset = LeRobotDataset(cfg.dataset.repo_id, root=cfg.dataset.root, episodes=[cfg.dataset.episode])
# Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == cfg.dataset.episode)
actions = episode_frames.select_columns(ACTION)
robot.connect()
try:
log_say("Replaying episode", cfg.play_sounds, blocking=True)
for idx in range(len(episode_frames)):
start_episode_t = time.perf_counter()
action_array = actions[idx][ACTION]
action = {}
for i, name in enumerate(dataset.features[ACTION]["names"]):
action[name] = action_array[i]
robot_obs = robot.get_observation()
processed_action = robot_action_processor((action, robot_obs))
_ = robot.send_action(processed_action)
dt_s = time.perf_counter() - start_episode_t
precise_sleep(max(1 / dataset.fps - dt_s, 0.0))
finally:
robot.disconnect()
def main():
register_third_party_plugins()
replay()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,360 @@
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Setup and debug CAN interfaces for Damiao motors (e.g., OpenArms).
Examples:
Setup CAN interfaces with CAN FD:
```shell
lerobot-setup-can --mode=setup --interfaces=can0,can1,can2,can3
```
Test motors on a single interface:
```shell
lerobot-setup-can --mode=test --interfaces=can0
```
Test motors on all interfaces:
```shell
lerobot-setup-can --mode=test --interfaces=can0,can1,can2,can3
```
Speed test:
```shell
lerobot-setup-can --mode=speed --interfaces=can0
```
"""
import subprocess
import sys
import time
from dataclasses import dataclass, field
import draccus
from lerobot.utils.import_utils import _can_available
MOTOR_NAMES = {
0x01: "joint_1",
0x02: "joint_2",
0x03: "joint_3",
0x04: "joint_4",
0x05: "joint_5",
0x06: "joint_6",
0x07: "joint_7",
0x08: "gripper",
}
@dataclass
class CANSetupConfig:
mode: str = "test"
interfaces: str = "can0" # Comma-separated, e.g. "can0,can1,can2,can3"
bitrate: int = 1000000
data_bitrate: int = 5000000
use_fd: bool = True
motor_ids: list[int] = field(default_factory=lambda: list(range(0x01, 0x09)))
timeout: float = 1.0
speed_iterations: int = 100
def get_interfaces(self) -> list[str]:
return [i.strip() for i in self.interfaces.split(",") if i.strip()]
def check_interface_status(interface: str) -> tuple[bool, str, bool]:
"""Check if CAN interface is UP and configured."""
try:
result = subprocess.run(["ip", "link", "show", interface], capture_output=True, text=True) # nosec B607
if result.returncode != 0:
return False, "Interface not found", False
output = result.stdout
is_up = "UP" in output
is_fd = "fd on" in output.lower() or "canfd" in output.lower()
status = "UP" if is_up else "DOWN"
if is_fd:
status += " (CAN FD)"
return is_up, status, is_fd
except FileNotFoundError:
return False, "ip command not found", False
def setup_interface(interface: str, bitrate: int, data_bitrate: int, use_fd: bool) -> bool:
"""Configure a CAN interface."""
try:
subprocess.run(["sudo", "ip", "link", "set", interface, "down"], check=False, capture_output=True) # nosec B607
cmd = ["sudo", "ip", "link", "set", interface, "type", "can", "bitrate", str(bitrate)]
if use_fd:
cmd.extend(["dbitrate", str(data_bitrate), "fd", "on"])
result = subprocess.run(cmd, capture_output=True, text=True) # nosec B607
if result.returncode != 0:
print(f" ✗ Failed to configure: {result.stderr}")
return False
result = subprocess.run( # nosec B607
["sudo", "ip", "link", "set", interface, "up"], capture_output=True, text=True
)
if result.returncode != 0:
print(f" ✗ Failed to bring up: {result.stderr}")
return False
return True
except Exception as e:
print(f" ✗ Error: {e}")
return False
def test_motor(bus, motor_id: int, timeout: float, use_fd: bool):
"""Test a single motor and return responses."""
import can
enable_msg = can.Message(
arbitration_id=motor_id,
data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFC],
is_extended_id=False,
is_fd=use_fd,
)
try:
bus.send(enable_msg)
except Exception as e:
return None, f"Send error: {e}"
responses = []
start_time = time.time()
while time.time() - start_time < timeout:
msg = bus.recv(timeout=0.1)
if msg:
responses.append((msg.arbitration_id, msg.data.hex(), getattr(msg, "is_fd", False)))
disable_msg = can.Message(
arbitration_id=motor_id,
data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD],
is_extended_id=False,
is_fd=use_fd,
)
try:
bus.send(disable_msg)
except Exception:
print(f"Error sending message to motor 0x{motor_id:02X}")
return responses, None
def test_interface(cfg: CANSetupConfig, interface: str):
"""Test all motors on a CAN interface."""
import can
is_up, status, _ = check_interface_status(interface)
print(f"\n{interface}: {status}")
if not is_up:
print(f" ⚠ Interface is not UP. Run: lerobot-setup-can --mode=setup --interfaces {interface}")
return {}
try:
kwargs = {"channel": interface, "interface": "socketcan", "bitrate": cfg.bitrate}
if cfg.use_fd:
kwargs.update({"data_bitrate": cfg.data_bitrate, "fd": True})
bus = can.interface.Bus(**kwargs)
except Exception as e:
print(f" ✗ Connection failed: {e}")
return {}
results = {}
try:
while bus.recv(timeout=0.01):
pass
for motor_id in cfg.motor_ids:
motor_name = MOTOR_NAMES.get(motor_id, f"motor_0x{motor_id:02X}")
responses, error = test_motor(bus, motor_id, cfg.timeout, cfg.use_fd)
if error:
print(f" Motor 0x{motor_id:02X} ({motor_name}): ✗ {error}")
results[motor_id] = {"found": False, "error": error}
elif responses:
print(f" Motor 0x{motor_id:02X} ({motor_name}): ✓ FOUND")
for resp_id, data, is_fd in responses:
fd_flag = " [FD]" if is_fd else ""
print(f" → Response 0x{resp_id:02X}{fd_flag}: {data}")
results[motor_id] = {"found": True, "responses": responses}
else:
print(f" Motor 0x{motor_id:02X} ({motor_name}): ✗ No response")
results[motor_id] = {"found": False}
time.sleep(0.05)
finally:
bus.shutdown()
found = sum(1 for r in results.values() if r.get("found"))
print(f"\n Summary: {found}/{len(cfg.motor_ids)} motors found")
return results
def speed_test(cfg: CANSetupConfig, interface: str):
"""Test communication speed with motors."""
import can
is_up, status, _ = check_interface_status(interface)
if not is_up:
print(f"{interface}: {status} - skipping")
return
print(f"\n{interface}: Running speed test ({cfg.speed_iterations} iterations)...")
try:
kwargs = {"channel": interface, "interface": "socketcan", "bitrate": cfg.bitrate}
if cfg.use_fd:
kwargs.update({"data_bitrate": cfg.data_bitrate, "fd": True})
bus = can.interface.Bus(**kwargs)
except Exception as e:
print(f" ✗ Connection failed: {e}")
return
responding_motor = None
for motor_id in cfg.motor_ids:
responses, _ = test_motor(bus, motor_id, 0.5, cfg.use_fd)
if responses:
responding_motor = motor_id
break
if not responding_motor:
print(" ✗ No responding motors found")
bus.shutdown()
return
print(f" Testing with motor 0x{responding_motor:02X}...")
latencies = []
for _ in range(cfg.speed_iterations):
start = time.perf_counter()
msg = can.Message(
arbitration_id=responding_motor,
data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFC],
is_extended_id=False,
is_fd=cfg.use_fd,
)
bus.send(msg)
resp = bus.recv(timeout=0.1)
if resp:
latencies.append((time.perf_counter() - start) * 1000)
bus.shutdown()
if latencies:
avg_latency = sum(latencies) / len(latencies)
hz = 1000.0 / avg_latency if avg_latency > 0 else 0
print(f" ✓ Success rate: {len(latencies)}/{cfg.speed_iterations}")
print(f" ✓ Avg latency: {avg_latency:.2f} ms")
print(f" ✓ Max frequency: {hz:.1f} Hz")
else:
print(" ✗ No successful responses")
def run_setup(cfg: CANSetupConfig):
"""Setup CAN interfaces."""
print("=" * 50)
print("CAN Interface Setup")
print("=" * 50)
print(f"Mode: {'CAN FD' if cfg.use_fd else 'CAN 2.0'}")
print(f"Bitrate: {cfg.bitrate / 1_000_000:.1f} Mbps")
if cfg.use_fd:
print(f"Data bitrate: {cfg.data_bitrate / 1_000_000:.1f} Mbps")
print()
interfaces = cfg.get_interfaces()
for interface in interfaces:
print(f"Configuring {interface}...")
if setup_interface(interface, cfg.bitrate, cfg.data_bitrate, cfg.use_fd):
is_up, status, _ = check_interface_status(interface)
print(f" ✓ {interface}: {status}")
else:
print(f" ✗ {interface}: Failed")
print("\nSetup complete!")
print("\nNext: Test motors with:")
print(f" lerobot-setup-can --mode=test --interfaces {','.join(interfaces)}")
def run_test(cfg: CANSetupConfig):
"""Test motors on CAN interfaces."""
print("=" * 50)
print("CAN Motor Test")
print("=" * 50)
print(f"Testing motors 0x{min(cfg.motor_ids):02X}-0x{max(cfg.motor_ids):02X}")
print(f"Mode: {'CAN FD' if cfg.use_fd else 'CAN 2.0'}")
print()
interfaces = cfg.get_interfaces()
all_results = {}
for interface in interfaces:
all_results[interface] = test_interface(cfg, interface)
total_found = sum(sum(1 for r in res.values() if r.get("found")) for res in all_results.values())
print("\n" + "=" * 50)
print("Summary")
print("=" * 50)
print(f"Total motors found: {total_found}")
if total_found == 0:
print("\n⚠ No motors found! Check:")
print(" 1. Motors are powered (24V)")
print(" 2. CAN wiring (CANH, CANL, GND)")
print(" 3. Motor timeout parameter > 0 (use Damiao tools)")
print(" 4. 120Ω termination at both cable ends")
print(f" 5. Interface configured: lerobot-setup-can --mode=setup --interfaces {interfaces[0]}")
def run_speed(cfg: CANSetupConfig):
"""Run speed tests on CAN interfaces."""
print("=" * 50)
print("CAN Speed Test")
print("=" * 50)
for interface in cfg.get_interfaces():
speed_test(cfg, interface)
@draccus.wrap()
def setup_can(cfg: CANSetupConfig):
if not _can_available:
print("Error: python-can not installed. Install with: pip install python-can")
sys.exit(1)
if cfg.mode == "setup":
run_setup(cfg)
elif cfg.mode == "test":
run_test(cfg)
elif cfg.mode == "speed":
run_speed(cfg)
else:
print(f"Unknown mode: {cfg.mode}")
print("Available modes: setup, test, speed")
sys.exit(1)
def main():
setup_can()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,92 @@
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Helper to set motor ids and baudrate.
Example:
```shell
lerobot-setup-motors \
--teleop.type=so100_leader \
--teleop.port=/dev/tty.usbmodem575E0031751
```
"""
from dataclasses import dataclass
import draccus
from lerobot.robots import ( # noqa: F401
RobotConfig,
bi_so_follower,
koch_follower,
lekiwi,
make_robot_from_config,
omx_follower,
so_follower,
)
from lerobot.teleoperators import ( # noqa: F401
TeleoperatorConfig,
bi_so_leader,
koch_leader,
make_teleoperator_from_config,
omx_leader,
so_leader,
)
COMPATIBLE_DEVICES = [
"koch_follower",
"koch_leader",
"omx_follower",
"omx_leader",
"so100_follower",
"so100_leader",
"so101_follower",
"so101_leader",
"lekiwi",
]
@dataclass
class SetupConfig:
teleop: TeleoperatorConfig | None = None
robot: RobotConfig | None = None
def __post_init__(self):
if bool(self.teleop) == bool(self.robot):
raise ValueError("Choose either a teleop or a robot.")
self.device = self.robot if self.robot else self.teleop
@draccus.wrap()
def setup_motors(cfg: SetupConfig):
if cfg.device.type not in COMPATIBLE_DEVICES:
raise NotImplementedError
if isinstance(cfg.device, RobotConfig):
device = make_robot_from_config(cfg.device)
else:
device = make_teleoperator_from_config(cfg.device)
device.setup_motors()
def main():
setup_motors()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,250 @@
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Simple script to control a robot from teleoperation.
Example:
```shell
lerobot-teleoperate \
--robot.type=so101_follower \
--robot.port=/dev/tty.usbmodem58760431541 \
--robot.cameras="{ front: {type: opencv, index_or_path: 0, width: 1920, height: 1080, fps: 30}}" \
--robot.id=black \
--teleop.type=so101_leader \
--teleop.port=/dev/tty.usbmodem58760431551 \
--teleop.id=blue \
--display_data=true
```
Example teleoperation with bimanual so100:
```shell
lerobot-teleoperate \
--robot.type=bi_so_follower \
--robot.left_arm_config.port=/dev/tty.usbmodem5A460822851 \
--robot.right_arm_config.port=/dev/tty.usbmodem5A460814411 \
--robot.id=bimanual_follower \
--robot.left_arm_config.cameras='{
wrist: {"type": "opencv", "index_or_path": 1, "width": 640, "height": 480, "fps": 30},
}' --robot.right_arm_config.cameras='{
wrist: {"type": "opencv", "index_or_path": 2, "width": 640, "height": 480, "fps": 30},
}' \
--teleop.type=bi_so_leader \
--teleop.left_arm_config.port=/dev/tty.usbmodem5A460852721 \
--teleop.right_arm_config.port=/dev/tty.usbmodem5A460819811 \
--teleop.id=bimanual_leader \
--display_data=true
```
"""
import logging
import time
from dataclasses import asdict, dataclass
from pprint import pformat
import rerun as rr
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401
from lerobot.configs import parser
from lerobot.processor import (
RobotAction,
RobotObservation,
RobotProcessorPipeline,
make_default_processors,
)
from lerobot.robots import ( # noqa: F401
Robot,
RobotConfig,
bi_openarm_follower,
bi_so_follower,
earthrover_mini_plus,
hope_jr,
koch_follower,
make_robot_from_config,
omx_follower,
openarm_follower,
reachy2,
so_follower,
unitree_g1 as unitree_g1_robot,
)
from lerobot.teleoperators import ( # noqa: F401
Teleoperator,
TeleoperatorConfig,
bi_openarm_leader,
bi_so_leader,
gamepad,
homunculus,
keyboard,
koch_leader,
make_teleoperator_from_config,
omx_leader,
openarm_leader,
reachy2_teleoperator,
so_leader,
unitree_g1,
)
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.robot_utils import precise_sleep
from lerobot.utils.utils import init_logging, move_cursor_up
from lerobot.utils.visualization_utils import init_rerun, log_rerun_data
@dataclass
class TeleoperateConfig:
# TODO: pepijn, steven: if more robots require multiple teleoperators (like lekiwi) its good to make this possibele in teleop.py and record.py with List[Teleoperator]
teleop: TeleoperatorConfig
robot: RobotConfig
# Limit the maximum frames per second.
fps: int = 60
teleop_time_s: float | None = None
# Display all cameras on screen
display_data: bool = False
# Display data on a remote Rerun server
display_ip: str | None = None
# Port of the remote Rerun server
display_port: int | None = None
# Whether to display compressed images in Rerun
display_compressed_images: bool = False
def teleop_loop(
teleop: Teleoperator,
robot: Robot,
fps: int,
teleop_action_processor: RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction],
robot_action_processor: RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction],
robot_observation_processor: RobotProcessorPipeline[RobotObservation, RobotObservation],
display_data: bool = False,
duration: float | None = None,
display_compressed_images: bool = False,
):
"""
This function continuously reads actions from a teleoperation device, processes them through optional
pipelines, sends them to a robot, and optionally displays the robot's state. The loop runs at a
specified frequency until a set duration is reached or it is manually interrupted.
Args:
teleop: The teleoperator device instance providing control actions.
robot: The robot instance being controlled.
fps: The target frequency for the control loop in frames per second.
display_data: If True, fetches robot observations and displays them in the console and Rerun.
display_compressed_images: If True, compresses images before sending them to Rerun for display.
duration: The maximum duration of the teleoperation loop in seconds. If None, the loop runs indefinitely.
teleop_action_processor: An optional pipeline to process raw actions from the teleoperator.
robot_action_processor: An optional pipeline to process actions before they are sent to the robot.
robot_observation_processor: An optional pipeline to process raw observations from the robot.
"""
display_len = max(len(key) for key in robot.action_features)
start = time.perf_counter()
while True:
loop_start = time.perf_counter()
# Get robot observation
# Not really needed for now other than for visualization
# teleop_action_processor can take None as an observation
# given that it is the identity processor as default
obs = robot.get_observation()
# Get teleop action
raw_action = teleop.get_action()
# Process teleop action through pipeline
teleop_action = teleop_action_processor((raw_action, obs))
# Process action for robot through pipeline
robot_action_to_send = robot_action_processor((teleop_action, obs))
# Send processed action to robot (robot_action_processor.to_output should return RobotAction)
_ = robot.send_action(robot_action_to_send)
if display_data:
# Process robot observation through pipeline
obs_transition = robot_observation_processor(obs)
log_rerun_data(
observation=obs_transition,
action=teleop_action,
compress_images=display_compressed_images,
)
print("\n" + "-" * (display_len + 10))
print(f"{'NAME':<{display_len}} | {'NORM':>7}")
# Display the final robot action that was sent
for motor, value in robot_action_to_send.items():
print(f"{motor:<{display_len}} | {value:>7.2f}")
move_cursor_up(len(robot_action_to_send) + 3)
dt_s = time.perf_counter() - loop_start
precise_sleep(max(1 / fps - dt_s, 0.0))
loop_s = time.perf_counter() - loop_start
print(f"Teleop loop time: {loop_s * 1e3:.2f}ms ({1 / loop_s:.0f} Hz)")
move_cursor_up(1)
if duration is not None and time.perf_counter() - start >= duration:
return
@parser.wrap()
def teleoperate(cfg: TeleoperateConfig):
init_logging()
logging.info(pformat(asdict(cfg)))
if cfg.display_data:
init_rerun(session_name="teleoperation", ip=cfg.display_ip, port=cfg.display_port)
display_compressed_images = (
True
if (cfg.display_data and cfg.display_ip is not None and cfg.display_port is not None)
else cfg.display_compressed_images
)
teleop = make_teleoperator_from_config(cfg.teleop)
robot = make_robot_from_config(cfg.robot)
teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
teleop.connect()
robot.connect()
try:
teleop_loop(
teleop=teleop,
robot=robot,
fps=cfg.fps,
display_data=cfg.display_data,
duration=cfg.teleop_time_s,
teleop_action_processor=teleop_action_processor,
robot_action_processor=robot_action_processor,
robot_observation_processor=robot_observation_processor,
display_compressed_images=display_compressed_images,
)
except KeyboardInterrupt:
pass
finally:
if cfg.display_data:
rr.rerun_shutdown()
teleop.disconnect()
robot.disconnect()
def main():
register_third_party_plugins()
teleoperate()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,600 @@
#!/usr/bin/env python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import dataclasses
import logging
import time
from contextlib import nullcontext
from pprint import pformat
from typing import Any
import torch
from accelerate import Accelerator
from termcolor import colored
from torch.optim import Optimizer
from lerobot.configs import parser
from lerobot.configs.train import TrainPipelineConfig
from lerobot.datasets.factory import make_dataset
from lerobot.datasets.sampler import EpisodeAwareSampler
from lerobot.datasets.utils import cycle
from lerobot.envs.factory import make_env, make_env_pre_post_processors
from lerobot.envs.utils import close_envs
from lerobot.optim.factory import make_optimizer_and_scheduler
from lerobot.policies.factory import make_policy, make_pre_post_processors
from lerobot.policies.pretrained import PreTrainedPolicy
from lerobot.rl.acp_dataset_stats import compute_acp_indicator_stats
from lerobot.rl.acp_hook import build_acp_raw_batch_hook
from lerobot.rl.wandb_utils import WandBLogger
from lerobot.scripts.lerobot_eval import eval_policy_all
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.logging_utils import AverageMeter, MetricsTracker
from lerobot.utils.random_utils import set_seed
from lerobot.utils.train_utils import (
get_step_checkpoint_dir,
get_step_identifier,
load_training_state,
save_checkpoint,
update_last_checkpoint,
)
from lerobot.utils.utils import (
format_big_number,
has_method,
init_logging,
)
def update_policy(
train_metrics: MetricsTracker,
policy: PreTrainedPolicy,
batch: Any,
optimizer: Optimizer,
grad_clip_norm: float,
accelerator: Accelerator,
lr_scheduler=None,
lock=None,
rabc_weights_provider=None,
) -> tuple[MetricsTracker, dict]:
"""
Performs a single training step to update the policy's weights.
This function executes the forward and backward passes, clips gradients, and steps the optimizer and
learning rate scheduler. Accelerator handles mixed-precision training automatically.
Args:
train_metrics: A MetricsTracker instance to record training statistics.
policy: The policy model to be trained.
batch: A batch of training data.
optimizer: The optimizer used to update the policy's parameters.
grad_clip_norm: The maximum norm for gradient clipping.
accelerator: The Accelerator instance for distributed training and mixed precision.
lr_scheduler: An optional learning rate scheduler.
lock: An optional lock for thread-safe optimizer updates.
rabc_weights_provider: Optional RABCWeights instance for sample weighting.
Returns:
A tuple containing:
- The updated MetricsTracker with new statistics for this step.
- A dictionary of outputs from the policy's forward pass, for logging purposes.
"""
start_time = time.perf_counter()
policy.train()
# Get RA-BC weights if enabled
rabc_batch_weights = None
rabc_batch_stats = None
if rabc_weights_provider is not None:
rabc_batch_weights, rabc_batch_stats = rabc_weights_provider.compute_batch_weights(batch)
# Let accelerator handle mixed precision
with accelerator.autocast():
# Use per-sample loss when RA-BC is enabled for proper weighting
if rabc_batch_weights is not None:
# Get per-sample losses
per_sample_loss, output_dict = policy.forward(batch, reduction="none")
# Apply RA-BC weights: L_RA-BC = Σ(w_i * l_i) / (Σw_i + ε)
# rabc_batch_weights is already normalized to sum to batch_size
epsilon = 1e-6
loss = (per_sample_loss * rabc_batch_weights).sum() / (rabc_batch_weights.sum() + epsilon)
# Log raw mean weight (before normalization) - this is the meaningful metric
output_dict["rabc_mean_weight"] = rabc_batch_stats["raw_mean_weight"]
output_dict["rabc_num_zero_weight"] = rabc_batch_stats["num_zero_weight"]
output_dict["rabc_num_full_weight"] = rabc_batch_stats["num_full_weight"]
else:
loss, output_dict = policy.forward(batch)
# TODO(rcadene): policy.unnormalize_outputs(out_dict)
# Use accelerator's backward method
accelerator.backward(loss)
# Clip gradients if specified
if grad_clip_norm > 0:
grad_norm = accelerator.clip_grad_norm_(policy.parameters(), grad_clip_norm)
else:
grad_norm = torch.nn.utils.clip_grad_norm_(
policy.parameters(), float("inf"), error_if_nonfinite=False
)
# Optimizer step
with lock if lock is not None else nullcontext():
optimizer.step()
optimizer.zero_grad()
# Step through pytorch scheduler at every batch instead of epoch
if lr_scheduler is not None:
lr_scheduler.step()
# Update internal buffers if policy has update method
if has_method(accelerator.unwrap_model(policy, keep_fp32_wrapper=True), "update"):
accelerator.unwrap_model(policy, keep_fp32_wrapper=True).update()
train_metrics.loss = loss.item()
train_metrics.grad_norm = grad_norm.item()
train_metrics.lr = optimizer.param_groups[0]["lr"]
train_metrics.update_s = time.perf_counter() - start_time
return train_metrics, output_dict
@parser.wrap()
def train(
cfg: TrainPipelineConfig,
accelerator: Accelerator | None = None,
):
"""
Main function to train a policy.
This function orchestrates the entire training pipeline, including:
- Setting up logging, seeding, and device configuration.
- Creating the dataset, evaluation environment (if applicable), policy, and optimizer.
- Handling resumption from a checkpoint.
- Running the main training loop, which involves fetching data batches and calling `update_policy`.
- Periodically logging metrics, saving model checkpoints, and evaluating the policy.
- Pushing the final trained model to the Hugging Face Hub if configured.
Args:
cfg: A `TrainPipelineConfig` object containing all training configurations.
accelerator: Optional Accelerator instance. If None, one will be created automatically.
"""
cfg.validate()
acp_raw_batch_hook = build_acp_raw_batch_hook(cfg.acp, cfg.seed)
# Create Accelerator if not provided
# It will automatically detect if running in distributed mode or single-process mode
# We set step_scheduler_with_optimizer=False to prevent accelerate from adjusting the lr_scheduler steps based on the num_processes
# We set find_unused_parameters=True to handle models with conditional computation
if accelerator is None:
from accelerate.utils import DistributedDataParallelKwargs
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
# Accelerate auto-detects the device based on the available hardware and ignores the policy.device setting.
# Force the device to be CPU when policy.device is set to CPU.
force_cpu = cfg.policy.device == "cpu"
accelerator = Accelerator(
step_scheduler_with_optimizer=False,
kwargs_handlers=[ddp_kwargs],
cpu=force_cpu,
)
init_logging(accelerator=accelerator)
# Determine if this is the main process (for logging and checkpointing)
# When using accelerate, only the main process should log to avoid duplicate outputs
is_main_process = accelerator.is_main_process
# Only log on main process
if is_main_process:
logging.info(pformat(cfg.to_dict()))
# Initialize wandb only on main process
if cfg.wandb.enable and cfg.wandb.project and is_main_process:
wandb_logger = WandBLogger(cfg)
else:
wandb_logger = None
if is_main_process:
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
if cfg.seed is not None:
set_seed(cfg.seed, accelerator=accelerator)
# Use accelerator's device
device = accelerator.device
torch.backends.cudnn.benchmark = True
torch.backends.cuda.matmul.allow_tf32 = True
# Dataset loading synchronization: main process downloads first to avoid race conditions
if is_main_process:
logging.info("Creating dataset")
dataset = make_dataset(cfg)
if cfg.acp.enable:
indicator_stats = compute_acp_indicator_stats(dataset, cfg.acp.indicator_field)
if indicator_stats is None:
logging.warning(
"ACP is enabled but indicator statistics are unavailable for field '%s'.",
cfg.acp.indicator_field,
)
else:
if indicator_stats.total_count >= 0:
logging.info(
"ACP indicator stats (%s): field='%s' ratio=%.6f positive=%d total=%d",
indicator_stats.source,
indicator_stats.indicator_field,
indicator_stats.positive_ratio,
indicator_stats.positive_count,
indicator_stats.total_count,
)
else:
logging.info(
"ACP indicator stats (%s): field='%s' ratio=%.6f",
indicator_stats.source,
indicator_stats.indicator_field,
indicator_stats.positive_ratio,
)
if indicator_stats.invalid_count > 0:
logging.warning(
"ACP indicator field '%s' contains %d non-binary values (expected only 0/1).",
indicator_stats.indicator_field,
indicator_stats.invalid_count,
)
accelerator.wait_for_everyone()
# Now all other processes can safely load the dataset
if not is_main_process:
dataset = make_dataset(cfg)
# Create environment used for evaluating checkpoints during training on simulation data.
# On real-world data, no need to create an environment as evaluations are done outside train.py,
# using the eval.py instead, with gym_dora environment and dora-rs.
eval_env = None
if cfg.eval_freq > 0 and cfg.env is not None and is_main_process:
logging.info("Creating env")
eval_env = make_env(cfg.env, n_envs=cfg.eval.batch_size, use_async_envs=cfg.eval.use_async_envs)
if is_main_process:
logging.info("Creating policy")
policy = make_policy(
cfg=cfg.policy,
ds_meta=dataset.meta,
rename_map=cfg.rename_map,
)
if cfg.peft is not None:
logging.info("Using PEFT! Wrapping model.")
# Convert CLI peft config to dict for overrides
peft_cli_overrides = dataclasses.asdict(cfg.peft)
policy = policy.wrap_with_peft(peft_cli_overrides=peft_cli_overrides)
# Wait for all processes to finish policy creation before continuing
accelerator.wait_for_everyone()
# Create processors - only provide dataset_stats if not resuming from saved processors
processor_kwargs = {}
postprocessor_kwargs = {}
if (cfg.policy.pretrained_path and not cfg.resume) or not cfg.policy.pretrained_path:
# Only provide dataset_stats when not resuming from saved processor state
processor_kwargs["dataset_stats"] = dataset.meta.stats
# For SARM, always provide dataset_meta for progress normalization
if cfg.policy.type == "sarm":
processor_kwargs["dataset_meta"] = dataset.meta
if cfg.policy.pretrained_path is not None:
processor_kwargs["preprocessor_overrides"] = {
"device_processor": {"device": device.type},
"normalizer_processor": {
"stats": dataset.meta.stats,
"features": {**policy.config.input_features, **policy.config.output_features},
"norm_map": policy.config.normalization_mapping,
},
}
processor_kwargs["preprocessor_overrides"]["rename_observations_processor"] = {
"rename_map": cfg.rename_map
}
postprocessor_kwargs["postprocessor_overrides"] = {
"unnormalizer_processor": {
"stats": dataset.meta.stats,
"features": policy.config.output_features,
"norm_map": policy.config.normalization_mapping,
},
}
preprocessor, postprocessor = make_pre_post_processors(
policy_cfg=cfg.policy,
pretrained_path=cfg.policy.pretrained_path,
**processor_kwargs,
**postprocessor_kwargs,
)
if is_main_process:
logging.info("Creating optimizer and scheduler")
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy)
# Load precomputed SARM progress for RA-BC if enabled
# Generate progress using: src/lerobot/policies/sarm/compute_rabc_weights.py
rabc_weights = None
if cfg.use_rabc:
from lerobot.utils.rabc import RABCWeights
# Get chunk_size from policy config
chunk_size = getattr(policy.config, "chunk_size", None)
if chunk_size is None:
raise ValueError("Chunk size is not found in policy config")
head_mode = getattr(cfg, "rabc_head_mode", "sparse")
logging.info(f"Loading SARM progress for RA-BC from {cfg.rabc_progress_path}")
logging.info(f"Using chunk_size={chunk_size} from policy config, head_mode={head_mode}")
rabc_weights = RABCWeights(
progress_path=cfg.rabc_progress_path,
chunk_size=chunk_size,
head_mode=head_mode,
kappa=getattr(cfg, "rabc_kappa", 0.01),
epsilon=getattr(cfg, "rabc_epsilon", 1e-6),
device=device,
)
step = 0 # number of policy updates (forward + backward + optim)
if cfg.resume:
step, optimizer, lr_scheduler = load_training_state(cfg.checkpoint_path, optimizer, lr_scheduler)
num_learnable_params = sum(p.numel() for p in policy.parameters() if p.requires_grad)
num_total_params = sum(p.numel() for p in policy.parameters())
if is_main_process:
logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
if cfg.env is not None:
logging.info(f"{cfg.env.task=}")
logging.info("Creating environment processors")
env_preprocessor, env_postprocessor = make_env_pre_post_processors(
env_cfg=cfg.env, policy_cfg=cfg.policy
)
logging.info(f"{cfg.steps=} ({format_big_number(cfg.steps)})")
logging.info(f"{dataset.num_frames=} ({format_big_number(dataset.num_frames)})")
logging.info(f"{dataset.num_episodes=}")
num_processes = accelerator.num_processes
effective_bs = cfg.batch_size * num_processes
logging.info(f"Effective batch size: {cfg.batch_size} x {num_processes} = {effective_bs}")
logging.info(f"{num_learnable_params=} ({format_big_number(num_learnable_params)})")
logging.info(f"{num_total_params=} ({format_big_number(num_total_params)})")
# create dataloader for offline training
if hasattr(cfg.policy, "drop_n_last_frames"):
shuffle = False
sampler = EpisodeAwareSampler(
dataset.meta.episodes["dataset_from_index"],
dataset.meta.episodes["dataset_to_index"],
episode_indices_to_use=dataset.episodes,
drop_n_last_frames=cfg.policy.drop_n_last_frames,
shuffle=True,
)
else:
shuffle = True
sampler = None
dataloader = torch.utils.data.DataLoader(
dataset,
num_workers=cfg.num_workers,
batch_size=cfg.batch_size,
shuffle=shuffle and not cfg.dataset.streaming,
sampler=sampler,
pin_memory=device.type == "cuda",
drop_last=False,
prefetch_factor=2 if cfg.num_workers > 0 else None,
)
# Prepare everything with accelerator
accelerator.wait_for_everyone()
policy, optimizer, dataloader, lr_scheduler = accelerator.prepare(
policy, optimizer, dataloader, lr_scheduler
)
dl_iter = cycle(dataloader)
policy.train()
train_metrics = {
"loss": AverageMeter("loss", ":.3f"),
"grad_norm": AverageMeter("grdn", ":.3f"),
"lr": AverageMeter("lr", ":0.1e"),
"update_s": AverageMeter("updt_s", ":.3f"),
"dataloading_s": AverageMeter("data_s", ":.3f"),
}
# Use effective batch size for proper epoch calculation in distributed training
effective_batch_size = cfg.batch_size * accelerator.num_processes
train_tracker = MetricsTracker(
effective_batch_size,
dataset.num_frames,
dataset.num_episodes,
train_metrics,
initial_step=step,
accelerator=accelerator,
)
if is_main_process:
logging.info(
f"Start offline training on a fixed dataset, with effective batch size: {effective_batch_size}"
)
logged_first_prompt = False
prompt_keys: list[str] = []
policy_task_field = getattr(cfg.policy, "task_field", None)
if isinstance(policy_task_field, str) and policy_task_field:
prompt_keys.append(policy_task_field)
for key in ("task", "subtask"):
if key not in prompt_keys:
prompt_keys.append(key)
for _ in range(step, cfg.steps):
start_time = time.perf_counter()
batch = next(dl_iter)
if acp_raw_batch_hook is not None:
batch = acp_raw_batch_hook(batch, step)
batch = preprocessor(batch)
if is_main_process and not logged_first_prompt:
for key in prompt_keys:
if key not in batch:
continue
prompt_batch = batch[key]
first_prompt = None
if isinstance(prompt_batch, str):
first_prompt = prompt_batch
elif isinstance(prompt_batch, (list, tuple)) and len(prompt_batch) > 0:
first_item = prompt_batch[0]
if isinstance(first_item, str):
first_prompt = first_item
if first_prompt is not None:
logging.info("First policy prompt (%s):\n%s", key, first_prompt)
logged_first_prompt = True
break
train_tracker.dataloading_s = time.perf_counter() - start_time
train_tracker, output_dict = update_policy(
train_tracker,
policy,
batch,
optimizer,
cfg.optimizer.grad_clip_norm,
accelerator=accelerator,
lr_scheduler=lr_scheduler,
rabc_weights_provider=rabc_weights,
)
# Note: eval and checkpoint happens *after* the `step`th training update has completed, so we
# increment `step` here.
step += 1
train_tracker.step()
is_log_step = cfg.log_freq > 0 and step % cfg.log_freq == 0 and is_main_process
is_saving_step = step % cfg.save_freq == 0 or step == cfg.steps
is_eval_step = cfg.eval_freq > 0 and step % cfg.eval_freq == 0
if is_log_step:
logging.info(train_tracker)
if wandb_logger:
wandb_log_dict = train_tracker.to_dict()
if output_dict:
wandb_log_dict.update(output_dict)
# Log RA-BC statistics if enabled
if rabc_weights is not None:
rabc_stats = rabc_weights.get_stats()
wandb_log_dict.update(
{
"rabc_delta_mean": rabc_stats["delta_mean"],
"rabc_delta_std": rabc_stats["delta_std"],
"rabc_num_frames": rabc_stats["num_frames"],
}
)
wandb_logger.log_dict(wandb_log_dict, step)
train_tracker.reset_averages()
if cfg.save_checkpoint and is_saving_step:
if is_main_process:
logging.info(f"Checkpoint policy after step {step}")
checkpoint_dir = get_step_checkpoint_dir(cfg.output_dir, cfg.steps, step)
save_checkpoint(
checkpoint_dir=checkpoint_dir,
step=step,
cfg=cfg,
policy=accelerator.unwrap_model(policy),
optimizer=optimizer,
scheduler=lr_scheduler,
preprocessor=preprocessor,
postprocessor=postprocessor,
)
update_last_checkpoint(checkpoint_dir)
if wandb_logger:
wandb_logger.log_policy(checkpoint_dir)
accelerator.wait_for_everyone()
if cfg.env and is_eval_step:
if is_main_process:
step_id = get_step_identifier(step, cfg.steps)
logging.info(f"Eval policy at step {step}")
with torch.no_grad(), accelerator.autocast():
eval_info = eval_policy_all(
envs=eval_env, # dict[suite][task_id] -> vec_env
policy=accelerator.unwrap_model(policy),
env_preprocessor=env_preprocessor,
env_postprocessor=env_postprocessor,
preprocessor=preprocessor,
postprocessor=postprocessor,
n_episodes=cfg.eval.n_episodes,
videos_dir=cfg.output_dir / "eval" / f"videos_step_{step_id}",
max_episodes_rendered=4,
start_seed=cfg.seed,
max_parallel_tasks=cfg.env.max_parallel_tasks,
)
# overall metrics (suite-agnostic)
aggregated = eval_info["overall"]
# optional: per-suite logging
for suite, suite_info in eval_info.items():
logging.info("Suite %s aggregated: %s", suite, suite_info)
# meters/tracker
eval_metrics = {
"avg_sum_reward": AverageMeter("∑rwrd", ":.3f"),
"pc_success": AverageMeter("success", ":.1f"),
"eval_s": AverageMeter("eval_s", ":.3f"),
}
eval_tracker = MetricsTracker(
cfg.batch_size,
dataset.num_frames,
dataset.num_episodes,
eval_metrics,
initial_step=step,
accelerator=accelerator,
)
eval_tracker.eval_s = aggregated.pop("eval_s")
eval_tracker.avg_sum_reward = aggregated.pop("avg_sum_reward")
eval_tracker.pc_success = aggregated.pop("pc_success")
if wandb_logger:
wandb_log_dict = {**eval_tracker.to_dict(), **eval_info}
wandb_logger.log_dict(wandb_log_dict, step, mode="eval")
wandb_logger.log_video(eval_info["overall"]["video_paths"][0], step, mode="eval")
accelerator.wait_for_everyone()
if eval_env:
close_envs(eval_env)
if is_main_process:
logging.info("End of training")
if cfg.policy.push_to_hub:
unwrapped_policy = accelerator.unwrap_model(policy)
if cfg.policy.use_peft:
unwrapped_policy.push_model_to_hub(cfg, peft_model=unwrapped_policy)
else:
unwrapped_policy.push_model_to_hub(cfg)
preprocessor.push_to_hub(cfg.policy.repo_id)
postprocessor.push_to_hub(cfg.policy.repo_id)
# Properly clean up the distributed process group
accelerator.wait_for_everyone()
accelerator.end_training()
def main():
register_third_party_plugins()
train()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,604 @@
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Train FAST tokenizer for action encoding.
This script:
1. Loads action chunks from LeRobotDataset (with episode sampling)
2. Optionally applies delta transforms (relative vs absolute actions)
3. Extracts specified action dimensions for encoding
4. Applies normalization (MEAN_STD, MIN_MAX, QUANTILES, or other modes)
5. Trains FAST tokenizer (BPE on DCT coefficients) on the action chunks
6. Saves tokenizer to output directory
7. Optionally pushes tokenizer to Hugging Face Hub
8. Reports compression statistics
Example:
```shell
lerobot-train-tokenizer \
--repo_id=user/dataset_name \
--action_horizon=10 \
--max_episodes=100 \
--sample_fraction=0.1 \
--encoded_dims="0:6" \
--delta_dims="0,1,2,3,4,5" \
--use_delta_transform=true \
--state_key="observation.state" \
--normalization_mode="QUANTILES" \
--vocab_size=1024 \
--scale=10.0 \
--output_dir="./fast_tokenizer_dataset_name" \
--push_to_hub=true \
--hub_repo_id="user/fast_tokenizer_dataset_name" \
--hub_private=false
"""
import json
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
import numpy as np
import torch
from huggingface_hub import HfApi
from lerobot.utils.import_utils import _transformers_available
if TYPE_CHECKING or _transformers_available:
from transformers import AutoProcessor
else:
AutoProcessor = None
from lerobot.configs import parser
from lerobot.configs.types import NormalizationMode
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.utils.constants import ACTION, OBS_STATE
@dataclass
class TokenizerTrainingConfig:
"""Configuration for training FAST tokenizer."""
# LeRobot dataset repository ID
repo_id: str
# Root directory for dataset (default: ~/.cache/huggingface/lerobot)
root: str | None = None
# Number of future actions in each chunk
action_horizon: int = 10
# Max episodes to use (None = all episodes in dataset)
max_episodes: int | None = None
# Fraction of chunks to sample per episode
sample_fraction: float = 0.1
# Comma-separated dimension ranges to encode (e.g., "0:6,7:23")
encoded_dims: str = "0:6,7:23"
# Comma-separated dimension indices for delta transform (e.g., "0,1,2,3,4,5")
delta_dims: str | None = None
# Whether to apply delta transform (relative actions vs absolute actions)
use_delta_transform: bool = False
# Dataset key for state observations (default: "observation.state")
state_key: str = OBS_STATE
# Normalization mode (MEAN_STD, MIN_MAX, QUANTILES, QUANTILE10, IDENTITY)
normalization_mode: str = "QUANTILES"
# FAST vocabulary size (BPE vocab size)
vocab_size: int = 1024
# DCT scaling factor (default: 10.0)
scale: float = 10.0
# Directory to save tokenizer (default: ./fast_tokenizer_{repo_id})
output_dir: str | None = None
# Whether to push the tokenizer to Hugging Face Hub
push_to_hub: bool = False
# Hub repository ID (e.g., "username/tokenizer-name"). If None, uses output_dir name
hub_repo_id: str | None = None
# Whether to create a private repository on the Hub
hub_private: bool = False
def apply_delta_transform(state: np.ndarray, actions: np.ndarray, delta_dims: list[int] | None) -> np.ndarray:
"""Apply delta transform to specified dimensions.
Args:
state: Current state [D]
actions: Future actions [D]
delta_dims: List of dimension indices to apply delta transform to
Returns:
Transformed actions [D]
"""
if delta_dims is None or len(delta_dims) == 0:
return actions
delta_actions = actions.copy()
for dim in delta_dims:
delta_actions[dim] = actions[dim] - state[dim]
return delta_actions
def apply_normalization(
data: np.ndarray,
stats: dict[str, np.ndarray],
mode: NormalizationMode,
eps: float = 1e-8,
) -> np.ndarray:
"""Apply normalization to data based on the specified mode.
Args:
data: Data to normalize [N, H, D] or [D]
stats: Dictionary of statistics (mean, std, min, max, q01, q99, q10, q90)
mode: Normalization mode to apply
eps: Small epsilon for numerical stability
Returns:
Normalized data with the same shape as input
"""
if mode == NormalizationMode.IDENTITY:
return data
if mode == NormalizationMode.MEAN_STD:
mean = stats.get("mean")
std = stats.get("std")
if mean is None or std is None:
raise ValueError("MEAN_STD mode requires 'mean' and 'std' in stats")
return (data - mean) / np.maximum(std, eps)
if mode == NormalizationMode.MIN_MAX:
min_val = stats.get("min")
max_val = stats.get("max")
if min_val is None or max_val is None:
raise ValueError("MIN_MAX mode requires 'min' and 'max' in stats")
denom = np.maximum(max_val - min_val, eps)
return 2.0 * (data - min_val) / denom - 1.0
if mode == NormalizationMode.QUANTILES:
q01 = stats.get("q01")
q99 = stats.get("q99")
if q01 is None or q99 is None:
raise ValueError("QUANTILES mode requires 'q01' and 'q99' in stats")
denom = np.maximum(q99 - q01, eps)
# Clip to quantile range then normalize to [-1, 1]
clipped = np.clip(data, q01, q99)
return 2.0 * (clipped - q01) / denom - 1.0
if mode == NormalizationMode.QUANTILE10:
q10 = stats.get("q10")
q90 = stats.get("q90")
if q10 is None or q90 is None:
raise ValueError("QUANTILE10 mode requires 'q10' and 'q90' in stats")
denom = np.maximum(q90 - q10, eps)
# Clip to quantile range then normalize to [-1, 1]
clipped = np.clip(data, q10, q90)
return 2.0 * (clipped - q10) / denom - 1.0
raise ValueError(f"Unsupported normalization mode: {mode}")
def process_episode(args):
"""Process single episode and return action chunks."""
dataset, ep_idx, action_horizon, delta_dims, sample_fraction, state_key, use_delta_transform = args
try:
# get episode info
ep_info = dataset.meta.episodes[ep_idx]
from_idx = ep_info["dataset_from_index"]
to_idx = ep_info["dataset_to_index"]
ep_length = to_idx - from_idx
if ep_length < action_horizon:
return None
# load all frames in episode
# if dataset has episode filtering, we need to use the mapping
states = []
actions = []
for abs_idx in range(from_idx, to_idx):
# map absolute index to relative index if needed
if dataset._absolute_to_relative_idx is not None:
if abs_idx not in dataset._absolute_to_relative_idx:
# this episode's frames aren't in the filtered dataset
return None
rel_idx = dataset._absolute_to_relative_idx[abs_idx]
else:
rel_idx = abs_idx
frame = dataset.hf_dataset[rel_idx]
# get state (could be from observation.state or other state key)
if state_key in frame:
state = (
frame[state_key].numpy()
if torch.is_tensor(frame[state_key])
else np.array(frame[state_key])
)
else:
# if no state key, use zeros (no delta transform)
state = np.zeros_like(
frame[ACTION].numpy() if torch.is_tensor(frame[ACTION]) else np.array(frame[ACTION])
)
action = frame[ACTION].numpy() if torch.is_tensor(frame[ACTION]) else np.array(frame[ACTION])
states.append(state)
actions.append(action)
states = np.array(states)
actions = np.array(actions)
# create action chunks (sliding window)
# all actions in a chunk are relative to the FIRST state in that chunk
action_chunks = []
for i in range(len(states) - action_horizon + 1):
current_state = states[i] # First state in chunk
future_absolute_actions = actions[i : i + action_horizon]
if use_delta_transform:
# relative actions
delta_chunk = np.zeros_like(future_absolute_actions)
for t in range(action_horizon):
delta_chunk[t] = apply_delta_transform(
current_state,
future_absolute_actions[t],
delta_dims,
)
action_chunks.append(delta_chunk)
else:
# absolute actions (no delta)
action_chunks.append(future_absolute_actions)
if len(action_chunks) == 0:
return None
action_chunks = np.array(action_chunks)
# sample chunks
if sample_fraction < 1.0:
n_chunks = len(action_chunks)
n_samples = max(1, int(n_chunks * sample_fraction))
episode_seed = hash(ep_idx) % (2**31)
rng = np.random.RandomState(episode_seed)
indices = rng.choice(n_chunks, size=n_samples, replace=False)
action_chunks = action_chunks[indices]
return action_chunks
except Exception as e:
print(f"Error processing episode {ep_idx}: {e}")
import traceback
traceback.print_exc()
return None
def train_fast_tokenizer(
action_chunks: np.ndarray,
vocab_size: int = 1024,
scale: float = 10.0,
) -> AutoProcessor:
"""
Train FAST tokenizer (BPE on DCT coefficients) on action chunks.
Uses the .fit() method to train a new tokenizer on the provided data.
Args:
action_chunks: Array of action chunks [N, H, D] where N=num_chunks, H=horizon, D=action_dim
vocab_size: BPE vocabulary size
scale: DCT scaling factor for quantization
Returns:
Trained FAST tokenizer
"""
print(f"Training FAST tokenizer on {len(action_chunks)} action chunks...")
print(f"Action chunk shape: {action_chunks.shape}")
print(f"Vocab size: {vocab_size}")
print(f"DCT scale: {scale}")
# download the tokenizer source code (not pretrained weights)
# we'll train a new tokenizer on our own data
base_tokenizer = AutoProcessor.from_pretrained("physical-intelligence/fast", trust_remote_code=True)
# convert action_chunks array to list of arrays (expected by .fit())
action_data_list = [action_chunks[i] for i in range(len(action_chunks))]
# train the new tokenizer on our action data using .fit()
# this trains the BPE tokenizer on DCT coefficients
print("Training new tokenizer (this may take a few minutes)...")
tokenizer = base_tokenizer.fit(
action_data_list,
scale=scale,
vocab_size=vocab_size,
time_horizon=action_chunks.shape[1], # action_horizon
action_dim=action_chunks.shape[2], # encoded dimensions
)
print("✓ Tokenizer training complete!")
# validate it works
sample_chunk = action_chunks[0]
encoded = tokenizer(sample_chunk[None])[0]
if isinstance(encoded, list):
encoded = np.array(encoded)
print(f"Sample encoding: {len(encoded)} tokens for chunk shape {sample_chunk.shape}")
return tokenizer
def compute_compression_stats(tokenizer, action_chunks: np.ndarray):
"""Compute compression statistics."""
print("\nComputing compression statistics...")
# sample for stats (use max 1000 chunks for speed)
sample_size = min(1000, len(action_chunks))
sample_indices = np.random.RandomState(42).choice(len(action_chunks), size=sample_size, replace=False)
sample_chunks = action_chunks[sample_indices]
token_lengths = []
for chunk in sample_chunks:
encoded = tokenizer(chunk[None])[0]
if isinstance(encoded, list):
token_lengths.append(len(encoded))
else:
token_lengths.append(encoded.shape[0] if hasattr(encoded, "shape") else len(encoded))
token_lengths = np.array(token_lengths)
# compression ratio: (H * D) / avg_tokens
input_size = action_chunks.shape[1] * action_chunks.shape[2]
avg_tokens = np.mean(token_lengths)
compression_ratio = input_size / avg_tokens
stats = {
"compression_ratio": float(compression_ratio),
"mean_token_length": float(np.mean(token_lengths)),
"p99_token_length": float(np.percentile(token_lengths, 99)),
"min_token_length": float(np.min(token_lengths)),
"max_token_length": float(np.max(token_lengths)),
}
print("Compression Statistics:")
print(f" Average compression ratio: {stats['compression_ratio']:.2f}x")
print(f" Mean token length: {stats['mean_token_length']:.1f}")
print(f" P99 token length: {stats['p99_token_length']:.0f}")
print(f" Min token length: {stats['min_token_length']:.0f}")
print(f" Max token length: {stats['max_token_length']:.0f}")
return stats
@parser.wrap()
def train_tokenizer(cfg: TokenizerTrainingConfig):
"""
Train FAST tokenizer for action encoding.
Args:
cfg: TokenizerTrainingConfig dataclass with all configuration parameters
"""
# load dataset
print(f"Loading dataset: {cfg.repo_id}")
dataset = LeRobotDataset(repo_id=cfg.repo_id, root=cfg.root)
print(f"Dataset loaded: {dataset.num_episodes} episodes, {dataset.num_frames} frames")
# parse normalization mode
try:
norm_mode = NormalizationMode(cfg.normalization_mode)
except ValueError as err:
raise ValueError(
f"Invalid normalization_mode: {cfg.normalization_mode}. "
f"Must be one of: {', '.join([m.value for m in NormalizationMode])}"
) from err
print(f"Normalization mode: {norm_mode.value}")
# parse encoded dimensions
encoded_dim_ranges = []
for range_str in cfg.encoded_dims.split(","):
start, end = map(int, range_str.strip().split(":"))
encoded_dim_ranges.append((start, end))
total_encoded_dims = sum(end - start for start, end in encoded_dim_ranges)
print(f"Encoding {total_encoded_dims} dimensions: {cfg.encoded_dims}")
# parse delta dimensions
delta_dim_list = None
if cfg.delta_dims is not None and cfg.delta_dims.strip():
delta_dim_list = [int(d.strip()) for d in cfg.delta_dims.split(",")]
print(f"Delta dimensions: {delta_dim_list}")
else:
print("No delta dimensions specified")
print(f"Use delta transform: {cfg.use_delta_transform}")
if cfg.use_delta_transform and (delta_dim_list is None or len(delta_dim_list) == 0):
print("Warning: use_delta_transform=True but no delta_dims specified. No delta will be applied.")
print(f"Action horizon: {cfg.action_horizon}")
print(f"State key: {cfg.state_key}")
# determine episodes to process
num_episodes = dataset.num_episodes
if cfg.max_episodes is not None:
num_episodes = min(cfg.max_episodes, num_episodes)
print(f"Processing {num_episodes} episodes...")
# process episodes sequentially (to avoid pickling issues with dataset)
all_chunks = []
for ep_idx in range(num_episodes):
if ep_idx % 10 == 0:
print(f" Processing episode {ep_idx}/{num_episodes}...")
chunks = process_episode(
(
dataset,
ep_idx,
cfg.action_horizon,
delta_dim_list,
cfg.sample_fraction,
cfg.state_key,
cfg.use_delta_transform,
)
)
if chunks is not None:
all_chunks.append(chunks)
# concatenate all chunks
all_chunks = np.concatenate(all_chunks, axis=0)
print(f"Collected {len(all_chunks)} action chunks")
# extract only encoded dimensions FIRST (before normalization)
encoded_chunks = []
for start, end in encoded_dim_ranges:
encoded_chunks.append(all_chunks[:, :, start:end])
encoded_chunks = np.concatenate(encoded_chunks, axis=-1) # [N, H, D_encoded]
print(f"Extracted {encoded_chunks.shape[-1]} encoded dimensions")
# apply normalization to encoded dimensions
print("\nBefore normalization - overall stats:")
print(f" Min: {np.min(encoded_chunks):.4f}, Max: {np.max(encoded_chunks):.4f}")
print(f" Mean: {np.mean(encoded_chunks):.4f}, Std: {np.std(encoded_chunks):.4f}")
# get normalization stats from dataset
norm_stats = dataset.meta.stats
if norm_stats is not None and ACTION in norm_stats:
action_stats = norm_stats[ACTION]
# build encoded dimension indices
encoded_dim_indices = []
for start, end in encoded_dim_ranges:
encoded_dim_indices.extend(range(start, end))
encoded_dim_indices = np.array(encoded_dim_indices)
# extract stats for encoded dimensions only
encoded_stats = {}
for stat_name, stat_values in action_stats.items():
if isinstance(stat_values, (list, np.ndarray)):
stat_array = np.array(stat_values)
if len(stat_array) > max(encoded_dim_indices):
encoded_stats[stat_name] = stat_array[encoded_dim_indices]
if encoded_stats:
print(f"\nNormalization stats for encoded dimensions (mode: {norm_mode.value}):")
for stat_name, stat_values in encoded_stats.items():
print(
f" {stat_name}: shape={stat_values.shape}, "
f"range=[{np.min(stat_values):.4f}, {np.max(stat_values):.4f}]"
)
# apply normalization based on mode
try:
encoded_chunks = apply_normalization(encoded_chunks, encoded_stats, norm_mode, eps=1e-8)
print(f"\nApplied {norm_mode.value} normalization")
except ValueError as e:
print(f"Warning: {e}. Using raw actions without normalization.")
print("\nAfter normalization - overall stats:")
print(f" Min: {np.min(encoded_chunks):.4f}, Max: {np.max(encoded_chunks):.4f}")
print(f" Mean: {np.mean(encoded_chunks):.4f}, Std: {np.std(encoded_chunks):.4f}")
print("\nPer-dimension stats (after normalization):")
for d in range(encoded_chunks.shape[-1]):
dim_data = encoded_chunks[:, :, d]
print(
f" Dim {d}: min={np.min(dim_data):7.4f}, max={np.max(dim_data):7.4f}, "
f"mean={np.mean(dim_data):7.4f}, std={np.std(dim_data):7.4f}"
)
else:
print("Warning: Could not extract stats for encoded dimensions, using raw actions")
else:
print("Warning: No normalization stats found in dataset, using raw actions")
print(f"Encoded chunks shape: {encoded_chunks.shape}")
# train FAST tokenizer
tokenizer = train_fast_tokenizer(
encoded_chunks,
vocab_size=cfg.vocab_size,
scale=cfg.scale,
)
# compute compression statistics
compression_stats = compute_compression_stats(tokenizer, encoded_chunks)
# save tokenizer
output_dir = cfg.output_dir
if output_dir is None:
output_dir = f"fast_tokenizer_{cfg.repo_id.replace('/', '_')}"
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
tokenizer.save_pretrained(output_path)
# save metadata
metadata = {
"repo_id": cfg.repo_id,
"vocab_size": cfg.vocab_size,
"scale": cfg.scale,
"encoded_dims": cfg.encoded_dims,
"encoded_dim_ranges": encoded_dim_ranges,
"total_encoded_dims": total_encoded_dims,
"delta_dims": cfg.delta_dims,
"delta_dim_list": delta_dim_list,
"use_delta_transform": cfg.use_delta_transform,
"state_key": cfg.state_key,
"normalization_mode": norm_mode.value,
"action_horizon": cfg.action_horizon,
"num_training_chunks": len(encoded_chunks),
"compression_stats": compression_stats,
}
with open(output_path / "metadata.json", "w") as f:
json.dump(metadata, f, indent=2)
print(f"\nSaved FAST tokenizer to {output_path}")
print(f"Metadata: {json.dumps(metadata, indent=2)}")
# push to Hugging Face Hub if requested
if cfg.push_to_hub:
# determine the hub repository ID
hub_repo_id = cfg.hub_repo_id
if hub_repo_id is None:
hub_repo_id = output_path.name
print(f"\nNo hub_repo_id provided, using: {hub_repo_id}")
print(f"\nPushing tokenizer to Hugging Face Hub: {hub_repo_id}")
print(f" Private: {cfg.hub_private}")
try:
# use the tokenizer's push_to_hub method
tokenizer.push_to_hub(
repo_id=hub_repo_id,
private=cfg.hub_private,
commit_message=f"Upload FAST tokenizer trained on {cfg.repo_id}",
)
# also upload the metadata.json file separately
api = HfApi()
api.upload_file(
path_or_fileobj=str(output_path / "metadata.json"),
path_in_repo="metadata.json",
repo_id=hub_repo_id,
repo_type="model",
commit_message="Upload tokenizer metadata",
)
print(f"Successfully pushed tokenizer to: https://huggingface.co/{hub_repo_id}")
except Exception as e:
print(f"Error pushing to hub: {e}")
print(" Make sure you're logged in with `huggingface-cli login`")
def main():
"""CLI entry point that parses arguments and runs the tokenizer training."""
train_tokenizer()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,720 @@
#!/usr/bin/env python
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import math
from pathlib import Path
from pprint import pformat
from typing import Any
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import torch
from accelerate import Accelerator
from accelerate.utils import DistributedDataParallelKwargs
from torch.utils.data import DataLoader, Sampler
from tqdm.auto import tqdm
from lerobot.configs import parser
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.value import ValueInferencePipelineConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.utils import load_info, write_info
from lerobot.policies.factory import make_policy, make_pre_post_processors
from lerobot.utils.constants import (
CHECKPOINTS_DIR,
LAST_CHECKPOINT_LINK,
PRETRAINED_MODEL_DIR,
)
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.random_utils import set_seed
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
from lerobot.utils.utils import init_logging, inside_slurm
from lerobot.scripts.value_infer_viz import (
_export_overlay_videos,
_get_episode_value_bounds,
_get_video_encode_options,
)
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
from lerobot.values.pistar06.modeling_pistar06 import (
EpisodeTargetInfo,
compute_normalized_value_targets,
)
class ContiguousDistributedEvalSampler(Sampler[int]):
"""Distributed eval sampler with contiguous per-rank shards and deterministic tail padding."""
def __init__(self, dataset_size: int, num_replicas: int, rank: int):
if dataset_size <= 0:
raise ValueError(f"'dataset_size' must be > 0, got {dataset_size}.")
if num_replicas <= 0:
raise ValueError(f"'num_replicas' must be > 0, got {num_replicas}.")
if rank < 0 or rank >= num_replicas:
raise ValueError(f"'rank' must be in [0, {num_replicas - 1}], got {rank}.")
self.dataset_size = int(dataset_size)
self.num_replicas = int(num_replicas)
self.rank = int(rank)
self.num_samples = int(math.ceil(self.dataset_size / self.num_replicas))
self.total_size = self.num_samples * self.num_replicas
def __iter__(self):
shard_start = self.rank * self.num_samples
shard_end = min(shard_start + self.num_samples, self.dataset_size)
if shard_start >= self.dataset_size:
indices: list[int] = []
else:
indices = list(range(shard_start, shard_end))
if len(indices) < self.num_samples:
indices.extend([self.dataset_size - 1] * (self.num_samples - len(indices)))
return iter(indices)
def __len__(self) -> int:
return self.num_samples
def _set_infer_logger_levels() -> None:
for logger_name in ["fsspec", "fsspec.local", "huggingface_hub", "datasets", "torchcodec"]:
logging.getLogger(logger_name).setLevel(logging.WARNING)
def _create_accelerator(cfg: ValueInferencePipelineConfig, accelerator: Accelerator | None) -> Accelerator:
if accelerator is not None:
return accelerator
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)
force_cpu = cfg.runtime.device == "cpu"
return Accelerator(step_scheduler_with_optimizer=False, kwargs_handlers=[ddp_kwargs], cpu=force_cpu)
def _resolve_pretrained_model_dir(checkpoint_path: str, checkpoint_ref: str) -> Path:
path = Path(checkpoint_path)
if not path.exists():
raise FileNotFoundError(f"Checkpoint path not found: {path}")
if (path / "model.safetensors").is_file() and (path / "config.json").is_file():
return path
if (path / PRETRAINED_MODEL_DIR / "model.safetensors").is_file() and (
path / PRETRAINED_MODEL_DIR / "config.json"
).is_file():
return path / PRETRAINED_MODEL_DIR
checkpoints_root = path / CHECKPOINTS_DIR if (path / CHECKPOINTS_DIR).is_dir() else path
step_ref = LAST_CHECKPOINT_LINK if checkpoint_ref == "last" else checkpoint_ref
step_dir = checkpoints_root / step_ref
if (step_dir / PRETRAINED_MODEL_DIR / "model.safetensors").is_file() and (
step_dir / PRETRAINED_MODEL_DIR / "config.json"
).is_file():
return step_dir / PRETRAINED_MODEL_DIR
if (step_dir / "model.safetensors").is_file() and (step_dir / "config.json").is_file():
return step_dir
raise FileNotFoundError(
f"Could not resolve pretrained model directory from checkpoint_path={path} checkpoint_ref={checkpoint_ref}."
)
def _load_dataset_distributed(cfg: ValueInferencePipelineConfig, accelerator: Accelerator) -> LeRobotDataset:
dataset_kwargs = {
"repo_id": cfg.dataset.repo_id,
"root": cfg.dataset.root,
"episodes": cfg.dataset.episodes,
"revision": cfg.dataset.revision,
"download_videos": cfg.dataset.download_videos,
}
if accelerator.is_main_process:
dataset = LeRobotDataset(**dataset_kwargs)
accelerator.wait_for_everyone()
if not accelerator.is_main_process:
dataset = LeRobotDataset(**dataset_kwargs)
return dataset
def _init_runtime(
cfg: ValueInferencePipelineConfig,
accelerator: Accelerator,
) -> tuple[Path, torch.device]:
output_dir = cfg.output_dir / "value"
if accelerator.is_main_process:
output_dir.mkdir(parents=True, exist_ok=True)
accelerator.wait_for_everyone()
log_file = output_dir / "value_infer.log" if accelerator.is_main_process else None
init_logging(log_file=log_file, file_level="INFO", accelerator=accelerator)
_set_infer_logger_levels()
if accelerator.is_main_process:
logging.info(pformat(cfg.to_dict()))
if cfg.seed is not None:
set_seed(cfg.seed, accelerator=accelerator)
device = accelerator.device
if device.type == "cuda":
torch.backends.cudnn.benchmark = True
torch.backends.cuda.matmul.allow_tf32 = True
return output_dir, device
def _build_episode_info(
dataset: LeRobotDataset,
success_field: str,
default_success: str,
) -> tuple[dict[int, EpisodeTargetInfo], dict[int, int]]:
episodes_ds = dataset.meta.episodes.with_format(None)
episodes = episodes_ds[:]
n_episodes = len(episodes_ds)
has_success = success_field in episodes_ds.column_names
episode_info: dict[int, EpisodeTargetInfo] = {}
task_max_length: dict[int, int] = {}
for i in range(n_episodes):
ep_idx = int(episodes["episode_index"][i])
ep_length = int(episodes["length"][i])
tasks = episodes["tasks"][i]
task_name = tasks[0] if isinstance(tasks, list) else tasks
if task_name not in dataset.meta.tasks.index:
raise KeyError(f"Episode {ep_idx} references unknown task '{task_name}'.")
task_index = int(dataset.meta.tasks.loc[task_name].task_index)
explicit_success = episodes[success_field][i] if has_success else None
resolved_success = resolve_episode_success_label(
explicit_success,
default_label=default_success,
require_label=True,
)
ep_success = resolved_success == EPISODE_SUCCESS
episode_info[ep_idx] = EpisodeTargetInfo(
episode_index=ep_idx,
task_index=task_index,
length=ep_length,
success=ep_success,
)
task_max_length[task_index] = max(task_max_length.get(task_index, 0), ep_length)
return episode_info, task_max_length
def _compute_dense_rewards_from_targets(
targets: np.ndarray,
episode_indices: np.ndarray,
frame_indices: np.ndarray,
) -> np.ndarray:
rewards = np.zeros_like(targets, dtype=np.float32)
n = targets.shape[0]
for i in range(n):
is_next_in_episode = (
i + 1 < n
and episode_indices[i + 1] == episode_indices[i]
and frame_indices[i + 1] == frame_indices[i] + 1
)
if is_next_in_episode:
rewards[i] = float(targets[i] - targets[i + 1])
else:
rewards[i] = float(targets[i])
return rewards
def _compute_n_step_advantages(
rewards: np.ndarray,
values: np.ndarray,
episode_indices: np.ndarray,
frame_indices: np.ndarray,
n_step: int,
) -> np.ndarray:
if n_step <= 0:
raise ValueError("'n_step' must be > 0.")
n = rewards.shape[0]
advantages = np.zeros(n, dtype=np.float32)
for i in range(n):
ep_i = episode_indices[i]
fi = frame_indices[i]
discounted_sum = 0.0
j = i
steps = 0
while steps < n_step and j < n:
same_episode = episode_indices[j] == ep_i
contiguous = frame_indices[j] == fi + steps
if not same_episode or not contiguous:
break
discounted_sum += float(rewards[j])
steps += 1
j += 1
if steps == n_step and j < n and episode_indices[j] == ep_i and frame_indices[j] == fi + n_step:
bootstrap = float(values[j])
else:
bootstrap = 0.0
advantages[i] = float(discounted_sum + bootstrap - values[i])
return advantages
def _compute_task_thresholds(
task_indices: np.ndarray,
advantages: np.ndarray,
positive_ratio: float,
) -> dict[int, float]:
if not 0.0 <= positive_ratio <= 1.0:
raise ValueError("'positive_ratio' must be within [0, 1].")
thresholds: dict[int, float] = {}
quantile = 1.0 - positive_ratio
for task_idx in np.unique(task_indices):
task_adv = advantages[task_indices == task_idx]
if task_adv.size == 0:
thresholds[int(task_idx)] = float("inf")
else:
thresholds[int(task_idx)] = float(np.quantile(task_adv, quantile))
return thresholds
def _binarize_advantages(
task_indices: np.ndarray,
advantages: np.ndarray,
thresholds: dict[int, float],
interventions: np.ndarray,
force_intervention_positive: bool,
) -> np.ndarray:
indicators = np.zeros_like(advantages, dtype=np.int64)
for i in range(advantages.shape[0]):
task_idx = int(task_indices[i])
thr = thresholds[task_idx]
indicators[i] = 1 if float(advantages[i]) >= thr else 0
if force_intervention_positive:
intervention_mask = interventions.astype(np.float32) > 0.5
indicators[intervention_mask] = 1
return indicators
def _update_feature_metadata(dataset_root: Path, feature_infos: dict[str, dict[str, Any]]) -> None:
info = load_info(dataset_root)
for feature_name, feature_info in feature_infos.items():
info["features"][feature_name] = {
"dtype": feature_info["dtype"],
"shape": tuple(feature_info["shape"]),
"names": feature_info.get("names"),
}
write_info(info, dataset_root)
def _write_columns_in_place(
dataset_root: Path,
absolute_indices: np.ndarray,
columns: dict[str, np.ndarray],
feature_infos: dict[str, dict[str, Any]],
) -> None:
if absolute_indices.ndim != 1:
raise ValueError("'absolute_indices' must be rank-1.")
max_index = int(np.max(absolute_indices))
selected = np.zeros(max_index + 1, dtype=np.bool_)
selected[absolute_indices] = True
lookups: dict[str, np.ndarray] = {}
for field, values in columns.items():
lookup_dtype = np.float32 if feature_infos[field]["dtype"] == "float32" else np.int64
lookup = np.zeros(max_index + 1, dtype=lookup_dtype)
lookup[absolute_indices] = values.astype(lookup_dtype, copy=False)
lookups[field] = lookup
data_files = sorted((dataset_root / "data").glob("chunk-*/file-*.parquet"))
if not data_files:
raise FileNotFoundError(f"No parquet data files found under {dataset_root / 'data'}")
for parquet_path in tqdm(data_files, desc="Writing annotations", leave=False):
table = pq.read_table(parquet_path)
idx_np = table["index"].to_numpy().astype(np.int64, copy=False)
in_range = (idx_np >= 0) & (idx_np <= max_index)
in_subset = np.zeros_like(in_range)
in_subset[in_range] = selected[idx_np[in_range]]
new_table = table
for field, lookup in lookups.items():
ftype = feature_infos[field]["dtype"]
if ftype == "float32":
default_value = np.nan
target_dtype = np.float32
pa_type = pa.float32()
elif ftype == "int64":
default_value = 0
target_dtype = np.int64
pa_type = pa.int64()
else:
raise ValueError(f"Unsupported annotation dtype '{ftype}' for field '{field}'.")
if field in new_table.schema.names:
current = new_table[field].to_numpy().astype(target_dtype, copy=True)
else:
current = np.full(idx_np.shape[0], default_value, dtype=target_dtype)
if np.any(in_subset):
subset_indices = idx_np[in_subset]
current[in_subset] = lookup[subset_indices]
array = pa.array(current, type=pa_type)
if field in new_table.schema.names:
col_idx = new_table.schema.names.index(field)
new_table = new_table.set_column(col_idx, field, array)
else:
new_table = new_table.append_column(field, array)
pq.write_table(new_table, parquet_path, compression="snappy")
_update_feature_metadata(dataset_root=dataset_root, feature_infos=feature_infos)
def _load_value_policy_and_processors(
cfg: ValueInferencePipelineConfig,
dataset: LeRobotDataset,
pretrained_dir: Path,
device: torch.device,
):
value_cfg = PreTrainedConfig.from_pretrained(pretrained_dir)
if not isinstance(value_cfg, Pistar06Config):
raise ValueError(
f"Unsupported value config type '{type(value_cfg)}'. lerobot-value-infer currently supports only 'pistar06'."
)
value_cfg.pretrained_path = pretrained_dir
value_cfg.device = device.type
value_policy = make_policy(
cfg=value_cfg,
ds_meta=dataset.meta,
rename_map=cfg.rename_map,
)
preprocessor, _ = make_pre_post_processors(
policy_cfg=value_cfg,
pretrained_path=pretrained_dir,
preprocessor_overrides={"device_processor": {"device": device.type}},
)
return value_policy, value_cfg, preprocessor
def _export_visualization_outputs(
dataset: LeRobotDataset,
cfg: ValueInferencePipelineConfig,
output_dir: Path,
) -> list[str]:
viz_output_dir = output_dir / "viz"
written_videos = _export_overlay_videos(
dataset=dataset,
value_field=cfg.acp.value_field,
advantage_field=cfg.acp.advantage_field,
indicator_field=cfg.acp.indicator_field,
viz_episodes=cfg.viz.episodes,
video_key=cfg.viz.video_key,
output_dir=viz_output_dir,
overwrite=cfg.viz.overwrite,
vcodec=cfg.viz.vcodec,
frame_storage_mode=cfg.viz.frame_storage_mode,
)
logging.info("Exported %d overlay videos to %s", len(written_videos), viz_output_dir)
return [str(path) for path in written_videos]
def run_value_inference_pipeline(
cfg: ValueInferencePipelineConfig,
accelerator: Accelerator | None = None,
) -> dict[str, Any]:
cfg.validate()
accelerator = _create_accelerator(cfg, accelerator)
output_dir, device = _init_runtime(cfg, accelerator)
dataset = _load_dataset_distributed(cfg, accelerator)
raw_frames = dataset.hf_dataset.with_format(None)
frame_count = len(raw_frames)
if frame_count == 0:
raise ValueError("Dataset has no frames.")
if not cfg.acp.enable:
viz_outputs: list[str] = []
if accelerator.is_main_process:
logging.info(
"ACP disabled; skipping value inference and reusing existing '%s' annotations.",
cfg.acp.value_field,
)
if cfg.acp.value_field not in raw_frames.column_names:
raise KeyError(
f"Missing value field '{cfg.acp.value_field}' in dataset while 'acp.enable=false'."
)
if cfg.viz.enable:
viz_outputs = _export_visualization_outputs(dataset=dataset, cfg=cfg, output_dir=output_dir)
accelerator.wait_for_everyone()
if not accelerator.is_main_process:
result: dict[str, Any] = {
"main_process": False,
"world_size": int(accelerator.num_processes),
}
accelerator.end_training()
return result
result = {
"main_process": True,
"world_size": int(accelerator.num_processes),
"num_frames": int(frame_count),
"checkpoint": None,
"value_field": cfg.acp.value_field,
"acp_enabled": False,
"value_inference_skipped": True,
"indicator_positive_ratio": None,
"thresholds": None,
"viz_outputs": viz_outputs,
}
accelerator.end_training()
return result
pretrained_dir = _resolve_pretrained_model_dir(
checkpoint_path=cfg.inference.checkpoint_path,
checkpoint_ref=cfg.inference.checkpoint_ref,
)
value_policy, value_cfg, preprocessor = _load_value_policy_and_processors(
cfg=cfg,
dataset=dataset,
pretrained_dir=pretrained_dir,
device=device,
)
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
if value_cfg.task_index_feature not in raw_frames.column_names:
raise KeyError(f"Missing task feature '{value_cfg.task_index_feature}' in dataset columns.")
task_indices = np.asarray(raw_frames[value_cfg.task_index_feature], dtype=np.int64)
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
if cfg.acp.intervention_field in raw_frames.column_names:
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
else:
interventions = np.zeros(frame_count, dtype=np.float32)
eval_loader = DataLoader(
dataset,
batch_size=cfg.runtime.batch_size,
shuffle=False,
num_workers=cfg.runtime.num_workers,
pin_memory=(device.type == "cuda"),
drop_last=False,
)
value_policy = accelerator.prepare(value_policy)
eval_loader = accelerator.prepare(eval_loader)
if accelerator.is_main_process:
max_abs_index = int(np.max(absolute_indices))
prediction_lookup = np.zeros(max_abs_index + 1, dtype=np.float32)
prediction_seen = np.zeros(max_abs_index + 1, dtype=np.bool_)
logging.info(
"Start value inference | world_size=%d batches=%d batch_size=%d checkpoint=%s",
accelerator.num_processes,
len(eval_loader),
cfg.runtime.batch_size,
pretrained_dir,
)
else:
prediction_lookup = None
prediction_seen = None
value_policy.eval()
eval_iter = tqdm(
eval_loader,
desc="Value inference",
total=len(eval_loader),
leave=False,
disable=(not accelerator.is_main_process) or inside_slurm(),
)
with torch.no_grad():
for raw_batch in eval_iter:
batch_indices = raw_batch["index"]
if not isinstance(batch_indices, torch.Tensor):
batch_indices = torch.as_tensor(batch_indices)
batch_indices = batch_indices.to(device=device, dtype=torch.long, non_blocking=True)
processed_batch = preprocessor(raw_batch)
with accelerator.autocast():
predicted_value = accelerator.unwrap_model(value_policy).predict_value(processed_batch)
gathered_idx = accelerator.gather_for_metrics(batch_indices)
gathered_val = accelerator.gather_for_metrics(predicted_value)
if accelerator.is_main_process:
idx_np = gathered_idx.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
val_np = gathered_val.detach().cpu().numpy().astype(np.float32, copy=False).reshape(-1)
prediction_lookup[idx_np] = val_np
prediction_seen[idx_np] = True
accelerator.wait_for_everyone()
if accelerator.is_main_process:
if prediction_lookup is None or prediction_seen is None:
raise RuntimeError("Prediction buffers unexpectedly missing on main process.")
missing_mask = ~prediction_seen[absolute_indices]
if bool(np.any(missing_mask)):
missing_count = int(np.sum(missing_mask))
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
predicted_values = prediction_lookup[absolute_indices]
logging.info(
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
float(np.min(predicted_values)),
float(np.max(predicted_values)),
float(np.mean(predicted_values)),
float(np.std(predicted_values)),
)
columns: dict[str, np.ndarray] = {
cfg.acp.value_field: predicted_values.astype(np.float32),
}
feature_infos: dict[str, dict[str, Any]] = {
cfg.acp.value_field: {"dtype": "float32", "shape": (1,), "names": None},
}
indicator_positive_ratio: float | None = None
thresholds: dict[int, float] | None = None
if cfg.acp.enable:
episode_info, task_max_lengths = _build_episode_info(
dataset=dataset,
success_field=cfg.dataset.success_field,
default_success=cfg.dataset.default_success,
)
value_targets = compute_normalized_value_targets(
episode_indices=episode_indices,
frame_indices=frame_indices,
episode_info=episode_info,
task_max_lengths=task_max_lengths,
c_fail_coef=cfg.acp.c_fail_coef,
clip_min=value_cfg.bin_min,
clip_max=value_cfg.bin_max,
)
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
advantages = _compute_n_step_advantages(
rewards=rewards,
values=predicted_values,
episode_indices=episode_indices,
frame_indices=frame_indices,
n_step=cfg.acp.n_step,
)
thresholds = _compute_task_thresholds(
task_indices=task_indices,
advantages=advantages,
positive_ratio=cfg.acp.positive_ratio,
)
indicators = _binarize_advantages(
task_indices=task_indices,
advantages=advantages,
thresholds=thresholds,
interventions=interventions,
force_intervention_positive=cfg.acp.force_intervention_positive,
)
indicator_positive_ratio = float(np.mean(indicators.astype(np.float32)))
logging.info(
"ACP stats | n_step=%d positive_ratio_target=%.4f positive_ratio_observed=%.4f",
cfg.acp.n_step,
cfg.acp.positive_ratio,
indicator_positive_ratio,
)
columns[cfg.acp.advantage_field] = advantages.astype(np.float32)
columns[cfg.acp.indicator_field] = indicators.astype(np.int64)
feature_infos[cfg.acp.advantage_field] = {"dtype": "float32", "shape": (1,), "names": None}
feature_infos[cfg.acp.indicator_field] = {"dtype": "int64", "shape": (1,), "names": None}
_write_columns_in_place(
dataset_root=Path(dataset.root),
absolute_indices=absolute_indices,
columns=columns,
feature_infos=feature_infos,
)
logging.info("Wrote value annotations to dataset root: %s", dataset.root)
# Sync computed columns into the in-memory hf_dataset so viz can read them
for field, values in columns.items():
if field in dataset.hf_dataset.column_names:
dataset.hf_dataset = dataset.hf_dataset.remove_columns([field])
dataset.hf_dataset = dataset.hf_dataset.add_column(field, values.tolist())
viz_outputs: list[str] = []
if cfg.viz.enable:
viz_outputs = _export_visualization_outputs(dataset=dataset, cfg=cfg, output_dir=output_dir)
result = {
"main_process": True,
"world_size": int(accelerator.num_processes),
"num_frames": int(frame_count),
"checkpoint": str(pretrained_dir),
"value_field": cfg.acp.value_field,
"acp_enabled": bool(cfg.acp.enable),
"value_inference_skipped": False,
"indicator_positive_ratio": indicator_positive_ratio,
"thresholds": thresholds,
"viz_outputs": viz_outputs,
}
else:
result = {
"main_process": False,
"world_size": int(accelerator.num_processes),
}
# Keep non-main ranks alive until main rank finishes optional slow visualization export.
accelerator.wait_for_everyone()
accelerator.end_training()
return result
@parser.wrap()
def value_infer(cfg: ValueInferencePipelineConfig):
return run_value_inference_pipeline(cfg)
def main():
register_third_party_plugins()
value_infer()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,329 @@
#!/usr/bin/env python
import logging
import time
from contextlib import nullcontext
from pprint import pformat
from typing import Any
import torch
from accelerate import Accelerator
from termcolor import colored
from torch.optim import Optimizer
from lerobot.configs import parser
from lerobot.configs.value_train import ValueTrainPipelineConfig
from lerobot.datasets.factory import make_dataset
from lerobot.datasets.utils import cycle
from lerobot.optim.factory import make_optimizer_and_scheduler
from lerobot.policies.factory import make_policy, make_pre_post_processors
from lerobot.policies.pretrained import PreTrainedPolicy
from lerobot.rl.wandb_utils import WandBLogger
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.logging_utils import AverageMeter, MetricsTracker
from lerobot.utils.random_utils import set_seed
from lerobot.utils.train_utils import (
get_step_checkpoint_dir,
load_training_state,
save_checkpoint,
update_last_checkpoint,
)
from lerobot.utils.utils import format_big_number, has_method, init_logging
def update_policy(
train_metrics: MetricsTracker,
policy: PreTrainedPolicy,
batch: Any,
optimizer: Optimizer,
grad_clip_norm: float,
accelerator: Accelerator,
lr_scheduler=None,
lock=None,
) -> tuple[MetricsTracker, dict]:
start_time = time.perf_counter()
policy.train()
with accelerator.autocast():
loss, output_dict = policy.forward(batch)
accelerator.backward(loss)
if grad_clip_norm > 0:
grad_norm = accelerator.clip_grad_norm_(policy.parameters(), grad_clip_norm)
else:
grad_norm = torch.nn.utils.clip_grad_norm_(policy.parameters(), float("inf"), error_if_nonfinite=False)
with lock if lock is not None else nullcontext():
optimizer.step()
optimizer.zero_grad()
if lr_scheduler is not None:
lr_scheduler.step()
if has_method(accelerator.unwrap_model(policy, keep_fp32_wrapper=True), "update"):
accelerator.unwrap_model(policy, keep_fp32_wrapper=True).update()
train_metrics.loss = loss.item()
train_metrics.grad_norm = grad_norm.item()
train_metrics.lr = optimizer.param_groups[0]["lr"]
train_metrics.update_s = time.perf_counter() - start_time
return train_metrics, output_dict
@parser.wrap()
def value_train(
cfg: ValueTrainPipelineConfig,
accelerator: Accelerator | None = None,
):
cfg.validate()
if accelerator is None:
from accelerate.utils import DistributedDataParallelKwargs
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)
force_cpu = cfg.value.device == "cpu"
accelerator = Accelerator(
step_scheduler_with_optimizer=False,
kwargs_handlers=[ddp_kwargs],
cpu=force_cpu,
)
init_logging(accelerator=accelerator)
is_main_process = accelerator.is_main_process
if is_main_process:
logging.info(pformat(cfg.to_dict()))
if cfg.wandb.enable and cfg.wandb.project and is_main_process:
wandb_logger = WandBLogger(cfg)
else:
wandb_logger = None
if is_main_process:
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
if cfg.seed is not None:
set_seed(cfg.seed, accelerator=accelerator)
device = accelerator.device
torch.backends.cudnn.benchmark = True
torch.backends.cuda.matmul.allow_tf32 = True
if is_main_process:
logging.info("Creating dataset")
dataset = make_dataset(cfg)
accelerator.wait_for_everyone()
if not is_main_process:
dataset = make_dataset(cfg)
if is_main_process:
logging.info("Creating value model")
value_model = make_policy(
cfg=cfg.value,
ds_meta=dataset.meta,
rename_map=cfg.rename_map,
)
value_target_raw_batch_hook = None
if has_method(value_model, "build_training_raw_batch_hook"):
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
dataset=dataset,
targets_cfg=cfg.targets,
)
if is_main_process:
logging.info("Using value model raw-batch hook for target construction.")
elif is_main_process:
logging.info("Value model does not define a raw-batch hook; using dataset targets as-is.")
accelerator.wait_for_everyone()
processor_kwargs = {}
postprocessor_kwargs = {}
if (cfg.value.pretrained_path and not cfg.resume) or not cfg.value.pretrained_path:
processor_kwargs["dataset_stats"] = dataset.meta.stats
if cfg.value.pretrained_path is not None:
processor_kwargs["preprocessor_overrides"] = {
"device_processor": {"device": device.type},
"normalizer_processor": {
"stats": dataset.meta.stats,
"features": {**(cfg.value.input_features or {}), **(cfg.value.output_features or {})},
"norm_map": cfg.value.normalization_mapping,
},
}
processor_kwargs["preprocessor_overrides"]["rename_observations_processor"] = {
"rename_map": cfg.rename_map
}
postprocessor_kwargs["postprocessor_overrides"] = {
"unnormalizer_processor": {
"stats": dataset.meta.stats,
"features": cfg.value.output_features or {},
"norm_map": cfg.value.normalization_mapping,
},
}
preprocessor, postprocessor = make_pre_post_processors(
policy_cfg=cfg.value,
pretrained_path=cfg.value.pretrained_path,
**processor_kwargs,
**postprocessor_kwargs,
)
if is_main_process:
logging.info("Creating optimizer and scheduler")
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, value_model)
step = 0
if cfg.resume:
if cfg.checkpoint_path is None:
raise ValueError("'checkpoint_path' is missing while resume=true.")
step, optimizer, lr_scheduler = load_training_state(cfg.checkpoint_path, optimizer, lr_scheduler)
num_learnable_params = sum(p.numel() for p in value_model.parameters() if p.requires_grad)
num_total_params = sum(p.numel() for p in value_model.parameters())
if is_main_process:
logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
logging.info(f"{cfg.steps=} ({format_big_number(cfg.steps)})")
logging.info(f"{dataset.num_frames=} ({format_big_number(dataset.num_frames)})")
logging.info(f"{dataset.num_episodes=}")
num_processes = accelerator.num_processes
effective_bs = cfg.batch_size * num_processes
logging.info(f"Effective batch size: {cfg.batch_size} x {num_processes} = {effective_bs}")
logging.info(f"{num_learnable_params=} ({format_big_number(num_learnable_params)})")
logging.info(f"{num_total_params=} ({format_big_number(num_total_params)})")
dataloader = torch.utils.data.DataLoader(
dataset,
num_workers=cfg.num_workers,
batch_size=cfg.batch_size,
shuffle=not cfg.dataset.streaming,
sampler=None,
pin_memory=device.type == "cuda",
drop_last=False,
prefetch_factor=2 if cfg.num_workers > 0 else None,
)
accelerator.wait_for_everyone()
value_model, optimizer, dataloader, lr_scheduler = accelerator.prepare(
value_model, optimizer, dataloader, lr_scheduler
)
dl_iter = cycle(dataloader)
value_model.train()
train_metrics = {
"loss": AverageMeter("loss", ":.3f"),
"grad_norm": AverageMeter("grdn", ":.3f"),
"lr": AverageMeter("lr", ":0.1e"),
"update_s": AverageMeter("updt_s", ":.3f"),
"dataloading_s": AverageMeter("data_s", ":.3f"),
}
effective_batch_size = cfg.batch_size * accelerator.num_processes
train_tracker = MetricsTracker(
effective_batch_size,
dataset.num_frames,
dataset.num_episodes,
train_metrics,
initial_step=step,
accelerator=accelerator,
)
if is_main_process:
logging.info(
f"Start value training on a fixed dataset, with effective batch size: {effective_batch_size}"
)
logged_first_prompt = False
for _ in range(step, cfg.steps):
start_time = time.perf_counter()
batch = next(dl_iter)
if value_target_raw_batch_hook is not None:
batch = value_target_raw_batch_hook(batch, step)
batch = preprocessor(batch)
if is_main_process and not logged_first_prompt and cfg.value.task_field in batch:
task_batch = batch[cfg.value.task_field]
if isinstance(task_batch, str):
first_prompt = task_batch
elif len(task_batch) > 0:
first_prompt = task_batch[0]
else:
first_prompt = None
if first_prompt is not None:
logging.info("First value prompt:\n%s", first_prompt)
logged_first_prompt = True
train_tracker.dataloading_s = time.perf_counter() - start_time
train_tracker, output_dict = update_policy(
train_tracker,
value_model,
batch,
optimizer,
cfg.optimizer.grad_clip_norm,
accelerator=accelerator,
lr_scheduler=lr_scheduler,
)
step += 1
train_tracker.step()
is_log_step = cfg.log_freq > 0 and step % cfg.log_freq == 0 and is_main_process
is_saving_step = step % cfg.save_freq == 0 or step == cfg.steps
if is_log_step:
logging.info(train_tracker)
if wandb_logger:
wandb_log_dict = train_tracker.to_dict()
if output_dict:
wandb_log_dict.update(output_dict)
wandb_logger.log_dict(wandb_log_dict, step)
train_tracker.reset_averages()
if cfg.save_checkpoint and is_saving_step:
if is_main_process:
logging.info(f"Checkpoint value model after step {step}")
checkpoint_dir = get_step_checkpoint_dir(cfg.output_dir, cfg.steps, step)
save_checkpoint(
checkpoint_dir=checkpoint_dir,
step=step,
cfg=cfg,
policy=accelerator.unwrap_model(value_model),
optimizer=optimizer,
scheduler=lr_scheduler,
preprocessor=preprocessor,
postprocessor=postprocessor,
)
update_last_checkpoint(checkpoint_dir)
if wandb_logger:
wandb_logger.log_policy(checkpoint_dir)
accelerator.wait_for_everyone()
if is_main_process:
logging.info("End of value training")
if cfg.value.push_to_hub:
unwrapped_value = accelerator.unwrap_model(value_model)
if cfg.value.use_peft:
unwrapped_value.push_model_to_hub(cfg, peft_model=unwrapped_value)
else:
unwrapped_value.push_model_to_hub(cfg)
preprocessor.push_to_hub(cfg.value.repo_id)
accelerator.wait_for_everyone()
accelerator.end_training()
def main():
register_third_party_plugins()
value_train()
if __name__ == "__main__":
main()

查看文件

@ -0,0 +1,209 @@
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Human-in-loop recording helpers used by `lerobot_record.py`."""
import logging
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from dataclasses import dataclass
from typing import Any
import numpy as np
import torch
from lerobot.policies.pretrained import PreTrainedPolicy
from lerobot.processor import PolicyAction, PolicyProcessorPipeline, RobotAction
from lerobot.rl.acp_tags import build_acp_tagged_task
from lerobot.robots import Robot
from lerobot.teleoperators import Teleoperator
from lerobot.utils.control_utils import predict_action
@dataclass
class ACPInferenceConfig:
enable: bool = False
use_cfg: bool = False
cfg_beta: float = 1.0
POLICY_RUNTIME_STATE_KEYS = ("_action_queue", "_queues", "_prev_mean")
INTERVENTION_STATE_POLICY = 0.0
INTERVENTION_STATE_ACTIVE = 1.0
INTERVENTION_STATE_RELEASE = 2.0
def _get_torch_rng_state(device: torch.device) -> tuple[torch.Tensor, torch.Tensor | None]:
cpu_state = torch.get_rng_state()
cuda_state = torch.cuda.get_rng_state(device) if device.type == "cuda" else None
return cpu_state, cuda_state
def _set_torch_rng_state(device: torch.device, cpu_state: torch.Tensor, cuda_state: torch.Tensor | None) -> None:
torch.set_rng_state(cpu_state)
if device.type == "cuda" and cuda_state is not None:
torch.cuda.set_rng_state(cuda_state, device)
def _clone_runtime_value(value: Any) -> Any:
if isinstance(value, torch.Tensor):
return value.detach().clone()
if isinstance(value, deque):
return deque((_clone_runtime_value(item) for item in value), maxlen=value.maxlen)
if isinstance(value, dict):
return {key: _clone_runtime_value(item) for key, item in value.items()}
if isinstance(value, list):
return [_clone_runtime_value(item) for item in value]
if isinstance(value, tuple):
return tuple(_clone_runtime_value(item) for item in value)
return deepcopy(value)
def _capture_policy_runtime_state(policy: PreTrainedPolicy) -> dict[str, Any]:
state: dict[str, Any] = {}
for key in POLICY_RUNTIME_STATE_KEYS:
if hasattr(policy, key):
state[key] = _clone_runtime_value(getattr(policy, key))
return state
def _restore_policy_runtime_state(policy: PreTrainedPolicy, state: dict[str, Any]) -> None:
for key, value in state.items():
setattr(policy, key, _clone_runtime_value(value))
def _predict_policy_action_with_runtime_state(
*,
observation_frame: dict[str, np.ndarray],
policy: PreTrainedPolicy,
device: torch.device,
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
use_amp: bool,
task: str | None,
robot_type: str | None,
runtime_state: dict[str, Any],
) -> PolicyAction:
_restore_policy_runtime_state(policy, runtime_state)
action = predict_action(
observation=observation_frame,
policy=policy,
device=device,
preprocessor=preprocessor,
postprocessor=postprocessor,
use_amp=use_amp,
task=task,
robot_type=robot_type,
)
runtime_state.clear()
runtime_state.update(_capture_policy_runtime_state(policy))
return action
def _predict_policy_action_with_acp_inference(
*,
observation_frame: dict[str, np.ndarray],
policy: PreTrainedPolicy,
device: torch.device,
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
use_amp: bool,
task: str | None,
robot_type: str | None,
acp_inference: ACPInferenceConfig,
cond_runtime_state: dict[str, Any] | None = None,
uncond_runtime_state: dict[str, Any] | None = None,
) -> PolicyAction:
if not acp_inference.enable:
return predict_action(
observation=observation_frame,
policy=policy,
device=device,
preprocessor=preprocessor,
postprocessor=postprocessor,
use_amp=use_amp,
task=task,
robot_type=robot_type,
)
conditional_task = build_acp_tagged_task(task, is_positive=True)
if not acp_inference.use_cfg:
return predict_action(
observation=observation_frame,
policy=policy,
device=device,
preprocessor=preprocessor,
postprocessor=postprocessor,
use_amp=use_amp,
task=conditional_task,
robot_type=robot_type,
)
if cond_runtime_state is None or uncond_runtime_state is None:
raise ValueError("CFG inference requires cond/uncond runtime states.")
cpu_state, cuda_state = _get_torch_rng_state(device)
action_cond = _predict_policy_action_with_runtime_state(
observation_frame=observation_frame,
policy=policy,
device=device,
preprocessor=preprocessor,
postprocessor=postprocessor,
use_amp=use_amp,
task=conditional_task,
robot_type=robot_type,
runtime_state=cond_runtime_state,
)
_set_torch_rng_state(device, cpu_state, cuda_state)
action_uncond = _predict_policy_action_with_runtime_state(
observation_frame=observation_frame,
policy=policy,
device=device,
preprocessor=preprocessor,
postprocessor=postprocessor,
use_amp=use_amp,
task=task,
robot_type=robot_type,
runtime_state=uncond_runtime_state,
)
return action_uncond + acp_inference.cfg_beta * (action_cond - action_uncond)
class PolicySyncDualArmExecutor:
"""Broadcast one policy-derived robot action to follower + teleop arm."""
def __init__(self, robot: Robot, teleop: Teleoperator, parallel_dispatch: bool = True):
self.robot = robot
self.teleop = teleop
self.parallel_dispatch = parallel_dispatch
self._pool = ThreadPoolExecutor(max_workers=2) if parallel_dispatch else None
def send_action(self, action: RobotAction) -> RobotAction:
if self._pool is None:
sent_action = self.robot.send_action(action)
self.teleop.send_feedback(action)
return sent_action
robot_future = self._pool.submit(self.robot.send_action, action)
teleop_future = self._pool.submit(self.teleop.send_feedback, action)
sent_action = robot_future.result()
teleop_future.result()
return sent_action
def shutdown(self) -> None:
if self._pool is not None:
self._pool.shutdown(wait=True)

查看文件

@ -0,0 +1,411 @@
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Core recording loop used by `lerobot_record.py`."""
import logging
import time
from collections.abc import Callable
from typing import Any, TypeVar
import numpy as np
from lerobot.datasets.image_writer import safe_stop_image_writer
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.utils import build_dataset_frame
from lerobot.policies.pretrained import PreTrainedPolicy
from lerobot.policies.utils import make_robot_action
from lerobot.processor import (
PolicyAction,
PolicyProcessorPipeline,
RobotAction,
RobotObservation,
RobotProcessorPipeline,
)
from lerobot.robots import Robot
from lerobot.scripts.recording_hil import (
ACPInferenceConfig,
INTERVENTION_STATE_ACTIVE,
INTERVENTION_STATE_POLICY,
INTERVENTION_STATE_RELEASE,
PolicySyncDualArmExecutor,
_capture_policy_runtime_state,
_predict_policy_action_with_acp_inference,
)
from lerobot.teleoperators import Teleoperator, koch_leader, omx_leader, so_leader
from lerobot.teleoperators.keyboard.teleop_keyboard import KeyboardTeleop
from lerobot.utils.constants import ACTION, OBS_STR
from lerobot.utils.recording_annotations import resolve_collector_policy_id
from lerobot.utils.robot_utils import precise_sleep
from lerobot.utils.utils import get_safe_torch_device
from lerobot.utils.visualization_utils import log_rerun_data
T = TypeVar("T")
""" --------------- record_loop() data flow --------------------------
[ Robot ]
V
[ robot.get_observation() ] ---> raw_obs
V
[ robot_observation_processor ] ---> processed_obs
V
.-----( ACTION LOGIC )------------------.
V V
[ From Teleoperator ] [ From Policy ]
| |
| [teleop.get_action] -> raw_action | [predict_action]
| | | |
| V | V
| [teleop_action_processor] | |
| | | |
'---> processed_teleop_action '---> processed_policy_action
| |
'-------------------------.-------------'
V
[ robot_action_processor ] --> robot_action_to_send
V
[ robot.send_action() ] -- (Robot Executes)
V
( Save to Dataset )
V
( Rerun Log / Loop Wait )
"""
@safe_stop_image_writer
def record_loop(
robot: Robot,
events: dict,
fps: int,
teleop_action_processor: RobotProcessorPipeline[
tuple[RobotAction, RobotObservation], RobotAction
], # runs after teleop
robot_action_processor: RobotProcessorPipeline[
tuple[RobotAction, RobotObservation], RobotAction
], # runs before robot
robot_observation_processor: RobotProcessorPipeline[
RobotObservation, RobotObservation
], # runs after robot
dataset: LeRobotDataset | None = None,
teleop: Teleoperator | list[Teleoperator] | None = None,
policy: PreTrainedPolicy | None = None,
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]] | None = None,
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction] | None = None,
control_time_s: int | None = None,
single_task: str | None = None,
display_data: bool = False,
display_compressed_images: bool = False,
policy_sync_executor: PolicySyncDualArmExecutor | None = None,
intervention_state_machine_enabled: bool = True,
collector_policy_id_policy: str = "policy",
collector_policy_id_human: str = "human",
acp_inference: ACPInferenceConfig | None = None,
communication_retry_timeout_s: float = 2.0,
communication_retry_interval_s: float = 0.1,
):
if acp_inference is None:
acp_inference = ACPInferenceConfig()
if dataset is not None and dataset.fps != fps:
raise ValueError(f"The dataset fps should be equal to requested fps ({dataset.fps} != {fps}).")
teleop_arm = teleop_keyboard = None
if isinstance(teleop, list):
teleop_keyboard = next((t for t in teleop if isinstance(t, KeyboardTeleop)), None)
teleop_arm = next(
(
t
for t in teleop
if isinstance(
t,
(
so_leader.SO100Leader
| so_leader.SO101Leader
| koch_leader.KochLeader
| omx_leader.OmxLeader
),
)
),
None,
)
if not (teleop_arm and teleop_keyboard and len(teleop) == 2 and robot.name == "lekiwi_client"):
raise ValueError(
"For multi-teleop, the list must contain exactly one KeyboardTeleop and one arm teleoperator. Currently only supported for LeKiwi robot."
)
if dataset is None and policy is not None:
raise ValueError("Policy-driven recording requires a dataset for feature mapping.")
action_feature_names = dataset.features[ACTION]["names"] if dataset is not None else None
if action_feature_names is None:
if hasattr(robot.action_features, "keys"):
action_feature_names = list(robot.action_features.keys())
else:
action_feature_names = list(robot.action_features)
zero_policy_action = {name: 0.0 for name in action_feature_names}
has_teleop = isinstance(teleop, Teleoperator) or isinstance(teleop, list)
intervention_enabled = intervention_state_machine_enabled and policy is not None and has_teleop
intervention_state = INTERVENTION_STATE_POLICY
last_teleop_action: RobotAction | None = None
teleop_fallback_warned = False
teleop_arm_for_mode_switch: Any | None = None
if isinstance(teleop, Teleoperator):
teleop_arm_for_mode_switch = teleop
elif isinstance(teleop, list):
teleop_arm_for_mode_switch = teleop_arm
def set_teleop_manual_control(enabled: bool) -> None:
if teleop_arm_for_mode_switch is None:
return
if not hasattr(teleop_arm_for_mode_switch, "set_manual_control"):
return
try:
teleop_arm_for_mode_switch.set_manual_control(enabled)
except Exception:
logging.exception("Failed to switch teleop manual-control mode to %s", enabled)
if policy is None:
# During reset/teleop-only loops keep leader backdrivable for manual dragging.
set_teleop_manual_control(True)
# Reset policy and processor if they are provided
if policy is not None and preprocessor is not None and postprocessor is not None:
policy.reset()
preprocessor.reset()
postprocessor.reset()
cond_policy_runtime_state: dict[str, Any] | None = None
uncond_policy_runtime_state: dict[str, Any] | None = None
if policy is not None and acp_inference.enable and acp_inference.use_cfg:
cond_policy_runtime_state = _capture_policy_runtime_state(policy)
uncond_policy_runtime_state = _capture_policy_runtime_state(policy)
if intervention_enabled:
# Start in S0: policy drives both arms, teleop arm should accept feedback commands.
set_teleop_manual_control(False)
def run_with_connection_retry(action_name: str, fn: Callable[[], T]) -> T:
timeout_s = max(communication_retry_timeout_s, 0.0)
interval_s = max(communication_retry_interval_s, 0.0)
deadline_t = time.perf_counter() + timeout_s
attempts = 0
first_error: ConnectionError | None = None
while True:
attempts += 1
try:
result = fn()
if attempts > 1:
elapsed_s = timeout_s - max(deadline_t - time.perf_counter(), 0.0)
logging.warning(
"%s recovered after %d retries in %.2fs.",
action_name,
attempts - 1,
elapsed_s,
)
return result
except ConnectionError as error:
if first_error is None:
first_error = error
logging.warning(
"%s failed with transient communication error; retrying for up to %.2fs (%s)",
action_name,
timeout_s,
error,
)
if timeout_s <= 0.0:
raise
remaining_s = deadline_t - time.perf_counter()
if remaining_s <= 0.0:
raise
sleep_s = interval_s if interval_s > 0.0 else remaining_s
time.sleep(min(sleep_s, remaining_s))
timestamp = 0
start_episode_t = time.perf_counter()
while timestamp < control_time_s:
start_loop_t = time.perf_counter()
if events["exit_early"]:
events["exit_early"] = False
break
if events.get("toggle_intervention", False):
events["toggle_intervention"] = False
if intervention_enabled:
if intervention_state == INTERVENTION_STATE_POLICY:
intervention_state = INTERVENTION_STATE_ACTIVE
set_teleop_manual_control(True)
logging.info("Intervention enabled (S1): teleop actions now override policy execution.")
else:
intervention_state = INTERVENTION_STATE_RELEASE
set_teleop_manual_control(False)
if policy is not None and preprocessor is not None and postprocessor is not None:
policy.reset()
preprocessor.reset()
postprocessor.reset()
if acp_inference.enable and acp_inference.use_cfg:
cond_policy_runtime_state = _capture_policy_runtime_state(policy)
uncond_policy_runtime_state = _capture_policy_runtime_state(policy)
if policy is not None and preprocessor is not None and postprocessor is not None:
logging.info("Policy cache reset on release: next policy action is recomputed.")
logging.info("Intervention release requested (S2): returning control to policy.")
else:
logging.info("Intervention toggle ignored because policy+teleop are not both active.")
# Get robot observation
obs = robot.get_observation()
# Applies a pipeline to the raw robot observation, default is IdentityProcessor
obs_processed = robot_observation_processor(obs)
if dataset is not None:
observation_frame = build_dataset_frame(dataset.features, obs_processed, prefix=OBS_STR)
# Get action from policy and/or teleop
act_processed_policy: RobotAction | None = None
act_processed_teleop: RobotAction | None = None
if policy is not None and preprocessor is not None and postprocessor is not None:
if not (intervention_enabled and intervention_state == INTERVENTION_STATE_ACTIVE):
policy_action = _predict_policy_action_with_acp_inference(
observation_frame=observation_frame,
policy=policy,
device=get_safe_torch_device(policy.config.device),
preprocessor=preprocessor,
postprocessor=postprocessor,
use_amp=policy.config.use_amp,
task=single_task,
robot_type=robot.robot_type,
acp_inference=acp_inference,
cond_runtime_state=cond_policy_runtime_state,
uncond_runtime_state=uncond_policy_runtime_state,
)
act_processed_policy = make_robot_action(policy_action, dataset.features)
if isinstance(teleop, Teleoperator):
act = run_with_connection_retry("teleop.get_action", teleop.get_action)
# Applies a pipeline to the raw teleop action, default is IdentityProcessor
act_processed_teleop = teleop_action_processor((act, obs))
elif isinstance(teleop, list):
arm_action = run_with_connection_retry("teleop_arm.get_action", teleop_arm.get_action)
arm_action = {f"arm_{k}": v for k, v in arm_action.items()}
keyboard_action = teleop_keyboard.get_action()
base_action = robot._from_keyboard_to_base_action(keyboard_action)
act = {**arm_action, **base_action} if len(base_action) > 0 else arm_action
act_processed_teleop = teleop_action_processor((act, obs))
if act_processed_policy is None and act_processed_teleop is None:
logging.info(
"No policy or teleoperator provided, skipping action generation."
"This is likely to happen when resetting the environment without a teleop device."
"The robot won't be at its rest position at the start of the next episode."
)
continue
if act_processed_teleop is not None:
last_teleop_action = act_processed_teleop
teleop_fallback_warned = False
policy_action_for_storage = act_processed_policy if act_processed_policy is not None else zero_policy_action
is_intervention = 0.0
if intervention_enabled and intervention_state == INTERVENTION_STATE_ACTIVE:
is_intervention = 1.0
if act_processed_teleop is not None:
action_values = act_processed_teleop
elif last_teleop_action is not None:
action_values = last_teleop_action
if not teleop_fallback_warned:
logging.warning(
"Intervention is active but no fresh teleop action is available; reusing last teleop action."
)
teleop_fallback_warned = True
elif act_processed_policy is not None:
action_values = act_processed_policy
if not teleop_fallback_warned:
logging.warning(
"Intervention is active but teleop action is unavailable; falling back to policy action."
)
teleop_fallback_warned = True
else:
action_values = zero_policy_action
if not teleop_fallback_warned:
logging.warning(
"Intervention is active but no teleop/policy action is available; sending zero action."
)
teleop_fallback_warned = True
else:
action_values = act_processed_policy if act_processed_policy is not None else act_processed_teleop
# Applies a pipeline to the action, default is IdentityProcessor
robot_action_to_send = robot_action_processor((action_values, obs))
# Send action to robot
# Action can eventually be clipped using `max_relative_target`,
# so action actually sent is saved in the dataset. action = postprocessor.process(action)
# TODO(steven, pepijn, adil): we should use a pipeline step to clip the action, so the sent action is the action that we input to the robot.
selected_from_policy = act_processed_policy is not None and action_values is act_processed_policy
if policy_sync_executor is not None and selected_from_policy:
_sent_action = run_with_connection_retry(
"policy_sync_executor.send_action",
lambda: policy_sync_executor.send_action(robot_action_to_send),
)
else:
_sent_action = run_with_connection_retry(
"robot.send_action",
lambda: robot.send_action(robot_action_to_send),
)
# Write to dataset
if dataset is not None:
action_frame = build_dataset_frame(dataset.features, action_values, prefix=ACTION)
policy_action_frame = build_dataset_frame(
dataset.features, policy_action_for_storage, prefix="complementary_info.policy_action"
)
frame = {**observation_frame, **action_frame, **policy_action_frame, "task": single_task}
if "complementary_info.is_intervention" in dataset.features:
frame["complementary_info.is_intervention"] = np.array([is_intervention], dtype=np.float32)
if "complementary_info.state" in dataset.features:
frame["complementary_info.state"] = np.array([intervention_state], dtype=np.float32)
if "complementary_info.collector_policy_id" in dataset.features:
frame["complementary_info.collector_policy_id"] = resolve_collector_policy_id(
intervention_enabled=intervention_enabled,
is_intervention=bool(is_intervention),
selected_from_policy=selected_from_policy,
policy_id=collector_policy_id_policy,
human_id=collector_policy_id_human,
)
dataset.add_frame(frame)
if display_data:
log_rerun_data(
observation=obs_processed, action=action_values, compress_images=display_compressed_images
)
if intervention_state == INTERVENTION_STATE_RELEASE:
intervention_state = INTERVENTION_STATE_POLICY
dt_s = time.perf_counter() - start_loop_t
precise_sleep(max(1 / fps - dt_s, 0.0))
timestamp = time.perf_counter() - start_episode_t

查看文件

@ -0,0 +1,477 @@
#!/usr/bin/env python
from pathlib import Path
import tempfile
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from tqdm.auto import tqdm
from lerobot.datasets.video_utils import decode_video_frames, encode_video_frames
from lerobot.datasets.lerobot_dataset import LeRobotDataset
def _select_video_key(camera_keys: list[str], requested_video_key: str | None) -> str:
if len(camera_keys) == 0:
raise ValueError("No camera key found in dataset.")
if requested_video_key is not None:
if requested_video_key not in camera_keys:
raise ValueError(f"Unknown video_key '{requested_video_key}'. Available camera keys: {camera_keys}")
return requested_video_key
for key in camera_keys:
lower = key.lower()
if ".front" in lower or "_front" in lower or lower.endswith("front"):
return key
return camera_keys[0]
def _parse_episodes_arg(episodes_arg: str, total_episodes: int) -> list[int]:
value = episodes_arg.strip().lower()
if value == "all":
return list(range(total_episodes))
parsed: set[int] = set()
for token in episodes_arg.split(","):
part = token.strip()
if not part:
continue
if "-" in part:
start_str, end_str = part.split("-", maxsplit=1)
start = int(start_str)
end = int(end_str)
if end < start:
raise ValueError(f"Invalid episode range '{part}'.")
parsed.update(range(start, end + 1))
else:
parsed.add(int(part))
episodes = sorted(parsed)
for ep in episodes:
if ep < 0 or ep >= total_episodes:
raise ValueError(f"Episode index out of range: {ep}, total_episodes={total_episodes}.")
return episodes
def _to_1d_float(values: list | np.ndarray) -> np.ndarray:
arr = np.asarray(values, dtype=np.float32)
if arr.ndim > 1:
arr = arr.reshape(arr.shape[0], -1)[:, 0]
return arr.reshape(-1)
def _to_1d_int(values: list | np.ndarray) -> np.ndarray:
arr = np.asarray(values, dtype=np.int64)
if arr.ndim > 1:
arr = arr.reshape(arr.shape[0], -1)[:, 0]
return arr.reshape(-1)
def _load_font(size: int) -> ImageFont.ImageFont | ImageFont.FreeTypeFont:
font_candidates = [
Path("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"),
Path("/usr/local/share/fonts/DejaVuSans.ttf"),
]
for font_path in font_candidates:
if font_path.exists():
return ImageFont.truetype(str(font_path), size=size)
return ImageFont.load_default()
def _curve_points(
values: np.ndarray,
current_step: int,
x0: int,
y0: int,
width: int,
height: int,
y_min: float,
y_max: float,
) -> list[tuple[int, int]]:
n = len(values)
if n == 0:
return []
denom_x = max(1, n - 1)
denom_y = max(1e-6, y_max - y_min)
points = []
last_step = min(current_step, n - 1)
for i in range(last_step + 1):
x = int(round(x0 + width * (i / denom_x)))
y_norm = np.clip((float(values[i]) - y_min) / denom_y, 0.0, 1.0)
y = int(round(y0 + (1.0 - y_norm) * height))
points.append((x, y))
return points
def _draw_overlay(
frame: Image.Image,
values: np.ndarray,
current_step: int,
advantage_t: float,
acp_t: int,
highlight_current_point: bool,
y_min: float,
y_max: float,
indicators: np.ndarray | None = None,
) -> Image.Image:
rgba = frame.convert("RGBA")
overlay = Image.new("RGBA", rgba.size, (0, 0, 0, 0))
draw = ImageDraw.Draw(overlay)
width, height = rgba.size
margin = max(10, width // 80)
chart_h = max(72, height // 4)
chart_w = width - 2 * margin
chart_x0 = margin
chart_y0 = height - margin - chart_h
draw.rectangle(
(chart_x0 - 4, chart_y0 - 4, chart_x0 + chart_w + 4, chart_y0 + chart_h + 4),
fill=(0, 0, 0, 110),
)
draw.line((chart_x0, chart_y0, chart_x0, chart_y0 + chart_h), fill=(200, 200, 200, 160), width=1)
draw.line(
(chart_x0, chart_y0 + chart_h, chart_x0 + chart_w, chart_y0 + chart_h),
fill=(200, 200, 200, 160),
width=1,
)
mid_y = chart_y0 + chart_h // 2
draw.line((chart_x0, mid_y, chart_x0 + chart_w, mid_y), fill=(120, 120, 120, 120), width=1)
points = _curve_points(values, current_step, chart_x0, chart_y0, chart_w, chart_h, y_min, y_max)
curve_width = max(2, width // 320)
white = (255, 255, 255, 255)
red = (255, 80, 80, 255)
if len(points) >= 2:
if indicators is not None:
for i in range(len(points) - 1):
seg_color = red if int(indicators[i]) == 1 or int(indicators[i + 1]) == 1 else white
draw.line([points[i], points[i + 1]], fill=seg_color, width=curve_width)
else:
draw.line(points, fill=white, width=curve_width)
point_color = red if highlight_current_point else white
if len(points) == 1:
x, y = points[0]
draw.ellipse((x - curve_width, y - curve_width, x + curve_width, y + curve_width), fill=point_color)
elif len(points) > 1:
x, y = points[-1]
radius = max(2, curve_width + 1)
draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=point_color)
font_size = max(18, height // 26)
font = _load_font(font_size)
lines = [f"advantage: {advantage_t:+.4f}", f"acp_indicator: {int(acp_t)}"]
line_sizes = [draw.textbbox((0, 0), text, font=font) for text in lines]
text_w = max(box[2] - box[0] for box in line_sizes)
text_h = sum(box[3] - box[1] for box in line_sizes) + max(4, font_size // 4)
box_pad = max(8, font_size // 3)
box_x1 = width - margin
box_x0 = box_x1 - text_w - 2 * box_pad
box_y0 = margin
box_y1 = box_y0 + text_h + 2 * box_pad
draw.rectangle((box_x0, box_y0, box_x1, box_y1), fill=(0, 0, 0, 150))
text_y = box_y0 + box_pad
for idx, text in enumerate(lines):
box = line_sizes[idx]
line_h = box[3] - box[1]
draw.text((box_x0 + box_pad, text_y), text, fill=(255, 255, 255, 255), font=font)
text_y += line_h + max(4, font_size // 4)
return Image.alpha_composite(rgba, overlay).convert("RGB")
def _build_output_video_path(output_dir: Path, repo_id: str, video_key: str, episode_index: int) -> Path:
repo_tag = repo_id.replace("/", "_")
key_tag = video_key.replace(".", "_")
return output_dir / f"{repo_tag}_episode_{episode_index:04d}_{key_tag}.mp4"
def _decode_frames_at_timestamps(
video_file: Path,
timestamps_s: np.ndarray,
tolerance_s: float,
backend: str | None,
) -> list[Image.Image]:
if timestamps_s.size == 0:
return []
frames = decode_video_frames(
video_path=video_file,
timestamps=timestamps_s.tolist(),
tolerance_s=tolerance_s,
backend=backend,
)
np_frames = frames.detach().cpu().numpy()
if np_frames.ndim != 4:
raise ValueError(f"Unexpected decoded frame tensor shape: {np_frames.shape}")
if np_frames.shape[1] in (1, 3):
np_frames = np.transpose(np_frames, (0, 2, 3, 1))
if np_frames.dtype != np.uint8:
if np.issubdtype(np_frames.dtype, np.floating):
max_val = float(np.max(np_frames)) if np_frames.size > 0 else 1.0
if max_val <= 1.0 + 1e-6:
np_frames = np.clip(np_frames, 0.0, 1.0) * 255.0
else:
np_frames = np.clip(np_frames, 0.0, 255.0)
else:
np_frames = np.clip(np_frames, 0, 255)
np_frames = np_frames.astype(np.uint8)
return [Image.fromarray(np_frames[i]) for i in range(np_frames.shape[0])]
def _get_episode_video_time_bounds(
dataset: LeRobotDataset,
episode_index: int,
video_key: str,
) -> tuple[float, float | None]:
episodes = getattr(dataset.meta, "episodes", None)
if episodes is None:
return 0.0, None
episodes_ds = episodes.with_format(None)
if "episode_index" not in episodes_ds.column_names:
return 0.0, None
episode_indices = np.asarray(episodes_ds["episode_index"], dtype=np.int64).reshape(-1)
matched = np.flatnonzero(episode_indices == episode_index)
if matched.size == 0:
return 0.0, None
row = int(matched[0])
from_col = f"videos/{video_key}/from_timestamp"
to_col = f"videos/{video_key}/to_timestamp"
from_ts = 0.0
to_ts: float | None = None
if from_col in episodes_ds.column_names:
from_ts = float(episodes_ds[from_col][row])
if to_col in episodes_ds.column_names:
to_ts = float(episodes_ds[to_col][row])
return from_ts, to_ts
def _get_video_encode_options(vcodec: str) -> tuple[dict[str, str], str]:
if vcodec == "libsvtav1":
return {"g": "2", "crf": "30", "preset": "12"}, "yuv420p"
if vcodec == "h264_nvenc":
return {
"preset": "p4",
"rc": "vbr",
"cq": "28",
"b": "0",
"g": "60",
}, "yuv420p"
return {"g": "2", "crf": "30"}, "yuv420p"
def _get_episode_value_bounds(ep_values: np.ndarray) -> tuple[float, float]:
if ep_values.shape[0] == 0:
raise ValueError("Cannot determine value bounds from an empty episode.")
return float(np.min(ep_values)), float(np.max(ep_values))
def _encode_pil_to_video(
frames: list[Image.Image],
video_path: Path,
fps: int,
vcodec: str,
) -> None:
"""Encode PIL frames directly to video without writing intermediate PNGs."""
import av as _av
video_options, pix_fmt = _get_video_encode_options(vcodec)
video_path.parent.mkdir(parents=True, exist_ok=True)
with _av.open(str(video_path), "w") as output:
out_stream = output.add_stream(vcodec, fps, options=video_options)
out_stream.pix_fmt = pix_fmt
out_stream.width = frames[0].width
out_stream.height = frames[0].height
for pil_img in frames:
av_frame = _av.VideoFrame.from_image(pil_img.convert("RGB"))
for packet in out_stream.encode(av_frame):
output.mux(packet)
for packet in out_stream.encode():
output.mux(packet)
def _export_single_episode(
src_video_path: Path,
dst_video_path: Path,
ep_values: np.ndarray,
ep_advantages: np.ndarray,
ep_indicators: np.ndarray,
episode_timestamps_s: np.ndarray,
fps: int,
vcodec: str,
tolerance_s: float,
video_backend: str | None,
frame_storage_mode: str = "memory",
temp_dir_root: Path | None = None,
) -> Path:
y_min, y_max = _get_episode_value_bounds(ep_values)
decoded_frames = _decode_frames_at_timestamps(
video_file=src_video_path,
timestamps_s=episode_timestamps_s,
tolerance_s=tolerance_s,
backend=video_backend,
)
n_frames = min(len(decoded_frames), len(ep_values))
if n_frames == 0:
raise ValueError(f"No decoded frames for video: {src_video_path}")
if frame_storage_mode == "disk":
with tempfile.TemporaryDirectory(
dir=str(temp_dir_root) if temp_dir_root is not None else None,
prefix=f"{dst_video_path.stem}-frames-",
) as temp_dir:
temp_path = Path(temp_dir)
for i in range(n_frames):
frame = decoded_frames[i]
composed = _draw_overlay(
frame=frame,
values=ep_values,
current_step=i,
advantage_t=float(ep_advantages[i]),
acp_t=int(ep_indicators[i]),
highlight_current_point=(int(ep_indicators[i]) == 1),
y_min=y_min,
y_max=y_max,
indicators=ep_indicators,
)
composed.save(temp_path / f"frame-{i:06d}.png")
encode_video_frames(
imgs_dir=temp_path,
video_path=dst_video_path,
fps=fps,
vcodec=vcodec,
overwrite=True,
)
return dst_video_path
composed_frames: list[Image.Image] = []
for i in range(n_frames):
composed = _draw_overlay(
frame=decoded_frames[i],
values=ep_values,
current_step=i,
advantage_t=float(ep_advantages[i]),
acp_t=int(ep_indicators[i]),
highlight_current_point=(int(ep_indicators[i]) == 1),
y_min=y_min,
y_max=y_max,
indicators=ep_indicators,
)
composed_frames.append(composed)
_encode_pil_to_video(composed_frames, dst_video_path, fps, vcodec)
return dst_video_path
def _export_overlay_videos(
dataset: LeRobotDataset,
value_field: str,
advantage_field: str,
indicator_field: str,
viz_episodes: str,
video_key: str | None,
output_dir: Path,
overwrite: bool,
vcodec: str,
frame_storage_mode: str = "memory",
) -> list[Path]:
selected_video_key = _select_video_key(dataset.meta.camera_keys, video_key)
raw_dataset = dataset.hf_dataset.with_format(None)
column_names = set(raw_dataset.column_names)
if value_field not in column_names:
raise KeyError(f"Missing value field '{value_field}' in dataset.")
values_all = _to_1d_float(raw_dataset[value_field])
if advantage_field in column_names:
advantages_all = _to_1d_float(raw_dataset[advantage_field])
else:
advantages_all = np.zeros_like(values_all, dtype=np.float32)
if indicator_field in column_names:
indicators_all = _to_1d_int(raw_dataset[indicator_field])
else:
indicators_all = np.zeros(values_all.shape[0], dtype=np.int64)
episode_indices_all = np.asarray(raw_dataset["episode_index"], dtype=np.int64).reshape(-1)
frame_indices_all = np.asarray(raw_dataset["frame_index"], dtype=np.int64).reshape(-1)
if "timestamp" in column_names:
timestamps_all = np.asarray(raw_dataset["timestamp"], dtype=np.float64).reshape(-1)
else:
timestamps_all = frame_indices_all.astype(np.float64) / float(dataset.fps)
if dataset.episodes is not None:
available_episodes = sorted(dataset.episodes)
else:
available_episodes = list(range(dataset.meta.total_episodes))
if viz_episodes.strip().lower() == "all":
episodes = available_episodes
else:
requested = _parse_episodes_arg(viz_episodes, dataset.meta.total_episodes)
episodes = [ep for ep in requested if ep in set(available_episodes)]
output_dir.mkdir(parents=True, exist_ok=True)
tasks = []
fps = int(dataset.fps)
for ep in episodes:
ep_positions = np.flatnonzero(episode_indices_all == ep)
if ep_positions.shape[0] > 1:
ep_frame_indices = frame_indices_all[ep_positions]
if bool(np.any(np.diff(ep_frame_indices) < 0)):
ep_positions = ep_positions[np.argsort(ep_frame_indices, kind="stable")]
ep_values = values_all[ep_positions]
if ep_values.shape[0] == 0:
continue
ep_timestamps = timestamps_all[ep_positions]
from_ts, to_ts = _get_episode_video_time_bounds(dataset, ep, selected_video_key)
ep_video_timestamps = from_ts + ep_timestamps
if to_ts is not None:
ep_video_timestamps = np.minimum(ep_video_timestamps, to_ts)
dst_path = _build_output_video_path(output_dir, dataset.repo_id, selected_video_key, ep)
if dst_path.exists() and not overwrite:
continue
src_path = Path(dataset.root) / dataset.meta.get_video_file_path(ep, selected_video_key)
tasks.append(
(
src_path,
dst_path,
ep_values,
advantages_all[ep_positions],
indicators_all[ep_positions],
ep_video_timestamps,
)
)
written_paths: list[Path] = []
for src, dst, vals, advs, inds, ts in tqdm(tasks, total=len(tasks), desc="Export value overlay videos", leave=False):
written_paths.append(
_export_single_episode(
src,
dst,
vals,
advs,
inds,
ts,
fps,
vcodec,
float(getattr(dataset, "tolerance_s", 1e-4)),
getattr(dataset, "video_backend", None),
frame_storage_mode,
output_dir,
)
)
return written_paths