镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
snapshot: sync mint main from current workspace 2026-03-03 16:32:56 +0800
这个提交包含在:
@ -0,0 +1,102 @@
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Helper to recalibrate your device (robot or teleoperator).
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-calibrate \
|
||||
--teleop.type=so100_leader \
|
||||
--teleop.port=/dev/tty.usbmodem58760431551 \
|
||||
--teleop.id=blue
|
||||
```
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import asdict, dataclass
|
||||
from pprint import pformat
|
||||
|
||||
import draccus
|
||||
|
||||
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401
|
||||
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401
|
||||
from lerobot.robots import ( # noqa: F401
|
||||
Robot,
|
||||
RobotConfig,
|
||||
bi_openarm_follower,
|
||||
bi_so_follower,
|
||||
hope_jr,
|
||||
koch_follower,
|
||||
lekiwi,
|
||||
make_robot_from_config,
|
||||
omx_follower,
|
||||
openarm_follower,
|
||||
so_follower,
|
||||
)
|
||||
from lerobot.teleoperators import ( # noqa: F401
|
||||
Teleoperator,
|
||||
TeleoperatorConfig,
|
||||
bi_openarm_leader,
|
||||
bi_so_leader,
|
||||
homunculus,
|
||||
koch_leader,
|
||||
make_teleoperator_from_config,
|
||||
omx_leader,
|
||||
openarm_leader,
|
||||
so_leader,
|
||||
unitree_g1,
|
||||
)
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.utils import init_logging
|
||||
|
||||
|
||||
@dataclass
|
||||
class CalibrateConfig:
|
||||
teleop: TeleoperatorConfig | None = None
|
||||
robot: RobotConfig | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if bool(self.teleop) == bool(self.robot):
|
||||
raise ValueError("Choose either a teleop or a robot.")
|
||||
|
||||
self.device = self.robot if self.robot else self.teleop
|
||||
|
||||
|
||||
@draccus.wrap()
|
||||
def calibrate(cfg: CalibrateConfig):
|
||||
init_logging()
|
||||
logging.info(pformat(asdict(cfg)))
|
||||
|
||||
if isinstance(cfg.device, RobotConfig):
|
||||
device = make_robot_from_config(cfg.device)
|
||||
elif isinstance(cfg.device, TeleoperatorConfig):
|
||||
device = make_teleoperator_from_config(cfg.device)
|
||||
|
||||
device.connect(calibrate=False)
|
||||
|
||||
try:
|
||||
device.calibrate()
|
||||
finally:
|
||||
device.disconnect()
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
calibrate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Generate a structured report for a LeRobot dataset.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
lerobot-dataset-report --dataset local/eval_twl2_10_hil_auto
|
||||
lerobot-dataset-report --dataset ~/.cache/huggingface/lerobot/local/eval_twl2_10_hil_auto --json
|
||||
```
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyarrow.dataset as pa_ds
|
||||
|
||||
from lerobot.datasets.utils import load_episodes, load_info, load_tasks
|
||||
from lerobot.utils.constants import HF_LEROBOT_HOME
|
||||
|
||||
|
||||
def resolve_dataset_root(dataset: str, root: Path | None) -> Path:
|
||||
dataset_path = Path(dataset).expanduser()
|
||||
if dataset_path.exists():
|
||||
return dataset_path.resolve()
|
||||
|
||||
base_root = root.expanduser().resolve() if root is not None else HF_LEROBOT_HOME.resolve()
|
||||
candidates = [base_root / dataset]
|
||||
if "/" not in dataset.strip("/"):
|
||||
candidates.append(base_root / "local" / dataset)
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
return candidate.resolve()
|
||||
|
||||
searched = "\n".join(f"- {c}" for c in candidates)
|
||||
raise FileNotFoundError(f"Dataset path not found. Searched:\n{searched}")
|
||||
|
||||
|
||||
def _to_float(value: Any) -> float:
|
||||
if isinstance(value, (list, tuple)):
|
||||
return _to_float(value[0]) if len(value) > 0 else 0.0
|
||||
if value is None:
|
||||
return 0.0
|
||||
return float(value)
|
||||
|
||||
|
||||
def _normalize_label(value: Any) -> str:
|
||||
if value is None:
|
||||
return "unlabeled"
|
||||
label = str(value).strip().lower()
|
||||
if label == "":
|
||||
return "unlabeled"
|
||||
return label
|
||||
|
||||
|
||||
def _safe_ratio(numerator: int, denominator: int) -> float:
|
||||
return float(numerator) / float(denominator) if denominator > 0 else 0.0
|
||||
|
||||
|
||||
def _flatten_episode_tasks(task_value: Any) -> list[str]:
|
||||
if hasattr(task_value, "tolist") and not isinstance(task_value, (str, bytes)):
|
||||
return _flatten_episode_tasks(task_value.tolist())
|
||||
if isinstance(task_value, list):
|
||||
values: list[str] = []
|
||||
for item in task_value:
|
||||
values.extend(_flatten_episode_tasks(item))
|
||||
return values
|
||||
if task_value is None:
|
||||
return []
|
||||
return [str(task_value)]
|
||||
|
||||
|
||||
def _build_episode_length_histogram(lengths: list[float], bins: int = 20) -> list[dict[str, float | int]]:
|
||||
if bins <= 0:
|
||||
raise ValueError(f"bins must be > 0, got {bins}")
|
||||
if not lengths:
|
||||
return [{"start": 0.0, "end": 0.0, "count": 0} for _ in range(bins)]
|
||||
|
||||
min_len = float(min(lengths))
|
||||
max_len = float(max(lengths))
|
||||
span = max_len - min_len
|
||||
|
||||
histogram = []
|
||||
if span == 0.0:
|
||||
histogram.append({"start": min_len, "end": max_len, "count": len(lengths)})
|
||||
histogram.extend({"start": min_len, "end": max_len, "count": 0} for _ in range(bins - 1))
|
||||
return histogram
|
||||
|
||||
bin_width = span / bins
|
||||
for i in range(bins):
|
||||
start = min_len + i * bin_width
|
||||
end = min_len + (i + 1) * bin_width if i < bins - 1 else max_len
|
||||
histogram.append({"start": start, "end": end, "count": 0})
|
||||
|
||||
for length in lengths:
|
||||
idx = min(int((float(length) - min_len) / bin_width), bins - 1)
|
||||
histogram[idx]["count"] = int(histogram[idx]["count"]) + 1
|
||||
|
||||
return histogram
|
||||
|
||||
|
||||
def _format_ascii_histogram(histogram: list[dict[str, float | int]], bar_width: int = 40) -> list[str]:
|
||||
max_count = max((bin_info["count"] for bin_info in histogram), default=0)
|
||||
lines: list[str] = []
|
||||
for bin_info in histogram:
|
||||
count = int(bin_info["count"])
|
||||
start = float(bin_info["start"])
|
||||
end = float(bin_info["end"])
|
||||
filled = 0 if max_count == 0 else int(round((count / max_count) * bar_width))
|
||||
bar = "#" * filled
|
||||
lines.append(f"- [{start:>7.2f}s, {end:>7.2f}s] | {count:>5} | {bar}")
|
||||
return lines
|
||||
|
||||
|
||||
def build_report(dataset_root: Path) -> dict[str, Any]:
|
||||
info = load_info(dataset_root)
|
||||
episodes_ds = load_episodes(dataset_root)
|
||||
episodes_df = episodes_ds.to_pandas()
|
||||
|
||||
actual_episode_count = int(len(episodes_df))
|
||||
episode_success_labels = episodes_df["episode_success"].tolist() if "episode_success" in episodes_df else []
|
||||
normalized_labels = [_normalize_label(v) for v in episode_success_labels]
|
||||
|
||||
success_count = sum(1 for v in normalized_labels if v == "success")
|
||||
failure_count = sum(1 for v in normalized_labels if v == "failure")
|
||||
unlabeled_count = sum(1 for v in normalized_labels if v not in {"success", "failure"})
|
||||
labeled_count = success_count + failure_count
|
||||
|
||||
data_dataset = pa_ds.dataset(dataset_root / "data", format="parquet")
|
||||
data_schema_names = set(data_dataset.schema.names)
|
||||
actual_frame_count = int(data_dataset.count_rows())
|
||||
|
||||
intervention_col = "complementary_info.is_intervention"
|
||||
episode_index_col = "episode_index"
|
||||
intervention_frames = 0
|
||||
intervention_episode_count = 0
|
||||
|
||||
if intervention_col in data_schema_names and episode_index_col in data_schema_names and actual_frame_count > 0:
|
||||
table = data_dataset.to_table(columns=[episode_index_col, intervention_col])
|
||||
episode_indices = table[episode_index_col].to_pylist()
|
||||
intervention_values = table[intervention_col].to_pylist()
|
||||
|
||||
intervention_episode_ids = set()
|
||||
for ep_idx, raw_value in zip(episode_indices, intervention_values, strict=True):
|
||||
if _to_float(raw_value) > 0.0:
|
||||
intervention_frames += 1
|
||||
intervention_episode_ids.add(int(ep_idx))
|
||||
intervention_episode_count = len(intervention_episode_ids)
|
||||
|
||||
tasks_df = load_tasks(dataset_root)
|
||||
unique_tasks: list[str] = []
|
||||
seen_tasks: set[str] = set()
|
||||
if "tasks" in episodes_df.columns:
|
||||
for row_tasks in episodes_df["tasks"].tolist():
|
||||
for task in _flatten_episode_tasks(row_tasks):
|
||||
if task not in seen_tasks:
|
||||
seen_tasks.add(task)
|
||||
unique_tasks.append(task)
|
||||
|
||||
feature_rows = []
|
||||
for key in sorted(info["features"].keys()):
|
||||
feature = info["features"][key]
|
||||
feature_rows.append(
|
||||
{
|
||||
"name": key,
|
||||
"dtype": feature.get("dtype"),
|
||||
"shape": list(feature.get("shape", ())),
|
||||
"names_count": len(feature["names"]) if feature.get("names") else 0,
|
||||
"is_video": feature.get("dtype") == "video",
|
||||
}
|
||||
)
|
||||
|
||||
fps = float(info.get("fps", 0) or 0)
|
||||
episode_lengths_frames = (
|
||||
episodes_df["length"].astype(int).tolist() if "length" in episodes_df and actual_episode_count > 0 else []
|
||||
)
|
||||
episode_lengths_seconds = [frames / fps for frames in episode_lengths_frames] if fps > 0 else []
|
||||
length_mean = float(sum(episode_lengths_seconds) / len(episode_lengths_seconds)) if episode_lengths_seconds else 0.0
|
||||
length_min = float(min(episode_lengths_seconds)) if episode_lengths_seconds else 0.0
|
||||
length_max = float(max(episode_lengths_seconds)) if episode_lengths_seconds else 0.0
|
||||
length_histogram = _build_episode_length_histogram(episode_lengths_seconds, bins=20)
|
||||
|
||||
return {
|
||||
"dataset_root": str(dataset_root),
|
||||
"meta": {
|
||||
"robot_type": info.get("robot_type"),
|
||||
"fps": info.get("fps"),
|
||||
"codebase_version": info.get("codebase_version"),
|
||||
"splits": info.get("splits", {}),
|
||||
"declared_total_episodes": int(info.get("total_episodes", 0)),
|
||||
"declared_total_frames": int(info.get("total_frames", 0)),
|
||||
"declared_total_tasks": int(info.get("total_tasks", 0)),
|
||||
},
|
||||
"structure": {
|
||||
"features": feature_rows,
|
||||
"task_count": int(len(unique_tasks)),
|
||||
"tasks": unique_tasks,
|
||||
"tasks_parquet_columns": list(tasks_df.columns),
|
||||
"tasks_parquet_row_count": int(len(tasks_df)),
|
||||
},
|
||||
"quality": {
|
||||
"actual_episode_count": actual_episode_count,
|
||||
"actual_frame_count": actual_frame_count,
|
||||
"episode_length": {
|
||||
"unit": "seconds",
|
||||
"mean": length_mean,
|
||||
"min": length_min,
|
||||
"max": length_max,
|
||||
"histogram_bins": length_histogram,
|
||||
},
|
||||
},
|
||||
"success_metrics": {
|
||||
"success_count": success_count,
|
||||
"failure_count": failure_count,
|
||||
"unlabeled_count": unlabeled_count,
|
||||
"success_ratio_all_episodes": _safe_ratio(success_count, actual_episode_count),
|
||||
"success_ratio_labeled_episodes": _safe_ratio(success_count, labeled_count),
|
||||
},
|
||||
"intervention_metrics": {
|
||||
"intervention_frames": intervention_frames,
|
||||
"intervention_frame_ratio": _safe_ratio(intervention_frames, actual_frame_count),
|
||||
"episodes_with_intervention": intervention_episode_count,
|
||||
"episode_intervention_ratio": _safe_ratio(intervention_episode_count, actual_episode_count),
|
||||
"has_intervention_column": intervention_col in data_schema_names,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def format_text_report(report: dict[str, Any]) -> str:
|
||||
lines: list[str] = []
|
||||
|
||||
lines.append("=== Dataset Report ===")
|
||||
lines.append(f"Root: {report['dataset_root']}")
|
||||
lines.append("")
|
||||
|
||||
meta = report["meta"]
|
||||
lines.append("[Meta]")
|
||||
lines.append(f"- robot_type: {meta['robot_type']}")
|
||||
lines.append(f"- fps: {meta['fps']}")
|
||||
lines.append(f"- codebase_version: {meta['codebase_version']}")
|
||||
lines.append(f"- splits: {meta['splits']}")
|
||||
lines.append(
|
||||
f"- declared totals: episodes={meta['declared_total_episodes']}, "
|
||||
f"frames={meta['declared_total_frames']}, tasks={meta['declared_total_tasks']}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
quality = report["quality"]
|
||||
lines.append("[Actual Data Stats]")
|
||||
lines.append(
|
||||
f"- actual totals: episodes={quality['actual_episode_count']}, frames={quality['actual_frame_count']}"
|
||||
)
|
||||
lines.append(
|
||||
f"- episode length ({quality['episode_length']['unit']}): mean={quality['episode_length']['mean']:.2f}, "
|
||||
f"min={quality['episode_length']['min']:.2f}, max={quality['episode_length']['max']:.2f}"
|
||||
)
|
||||
lines.append("- episode length histogram (20 bins, terminal view):")
|
||||
lines.extend(_format_ascii_histogram(quality["episode_length"]["histogram_bins"]))
|
||||
lines.append("")
|
||||
|
||||
success = report["success_metrics"]
|
||||
lines.append("[Success Metrics]")
|
||||
lines.append(
|
||||
f"- counts: success={success['success_count']}, failure={success['failure_count']}, "
|
||||
f"unlabeled={success['unlabeled_count']}"
|
||||
)
|
||||
lines.append(f"- success ratio (all episodes): {success['success_ratio_all_episodes']:.4f}")
|
||||
lines.append(f"- success ratio (labeled episodes): {success['success_ratio_labeled_episodes']:.4f}")
|
||||
lines.append("")
|
||||
|
||||
intervention = report["intervention_metrics"]
|
||||
lines.append("[Intervention Metrics]")
|
||||
lines.append(f"- has intervention field: {intervention['has_intervention_column']}")
|
||||
lines.append(
|
||||
f"- frame-level: intervention_frames={intervention['intervention_frames']}, "
|
||||
f"ratio={intervention['intervention_frame_ratio']:.4f}"
|
||||
)
|
||||
lines.append(
|
||||
f"- episode-level: episodes_with_intervention={intervention['episodes_with_intervention']}, "
|
||||
f"ratio={intervention['episode_intervention_ratio']:.4f}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
structure = report["structure"]
|
||||
lines.append("[Task List]")
|
||||
lines.append(f"- task_count: {structure['task_count']}")
|
||||
for idx, task in enumerate(structure["tasks"]):
|
||||
lines.append(f"- task[{idx}]: {task}")
|
||||
lines.append("")
|
||||
|
||||
lines.append("[Feature Schema]")
|
||||
for feature in structure["features"]:
|
||||
lines.append(
|
||||
f"- {feature['name']}: dtype={feature['dtype']}, shape={feature['shape']}, "
|
||||
f"names_count={feature['names_count']}, is_video={feature['is_video']}"
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Summarize LeRobot dataset schema and quality metrics.")
|
||||
parser.add_argument(
|
||||
"--dataset",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Dataset repo id (e.g. local/eval_xxx) or absolute/local filesystem path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional root directory containing datasets. Defaults to HF_LEROBOT_HOME.",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Output JSON instead of text.")
|
||||
args = parser.parse_args()
|
||||
|
||||
dataset_root = resolve_dataset_root(args.dataset, args.root)
|
||||
report = build_report(dataset_root)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
print(format_text_report(report))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Visualize data of **all** frames of any episode of a dataset of type LeRobotDataset.
|
||||
|
||||
Note: The last frame of the episode doesn't always correspond to a final state.
|
||||
That's because our datasets are composed of transition from state to state up to
|
||||
the antepenultimate state associated to the ultimate action to arrive in the final state.
|
||||
However, there might not be a transition from a final state to another state.
|
||||
|
||||
Note: This script aims to visualize the data used to train the neural networks.
|
||||
~What you see is what you get~. When visualizing image modality, it is often expected to observe
|
||||
lossy compression artifacts since these images have been decoded from compressed mp4 videos to
|
||||
save disk space. The compression factor applied has been tuned to not affect success rate.
|
||||
|
||||
Examples:
|
||||
|
||||
- Visualize data stored on a local machine:
|
||||
```
|
||||
local$ lerobot-dataset-viz \
|
||||
--repo-id lerobot/pusht \
|
||||
--episode-index 0
|
||||
```
|
||||
|
||||
- Visualize data stored on a distant machine with a local viewer:
|
||||
```
|
||||
distant$ lerobot-dataset-viz \
|
||||
--repo-id lerobot/pusht \
|
||||
--episode-index 0 \
|
||||
--save 1 \
|
||||
--output-dir path/to/directory
|
||||
|
||||
local$ scp distant:path/to/directory/lerobot_pusht_episode_0.rrd .
|
||||
local$ rerun lerobot_pusht_episode_0.rrd
|
||||
```
|
||||
|
||||
- Visualize data stored on a distant machine through streaming:
|
||||
(You need to forward the websocket port to the distant machine, with
|
||||
`ssh -L 9087:localhost:9087 username@remote-host`)
|
||||
```
|
||||
distant$ lerobot-dataset-viz \
|
||||
--repo-id lerobot/pusht \
|
||||
--episode-index 0 \
|
||||
--mode distant \
|
||||
--ws-port 9087
|
||||
|
||||
local$ rerun ws://localhost:9087
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import rerun as rr
|
||||
import torch
|
||||
import torch.utils.data
|
||||
import tqdm
|
||||
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.utils.constants import ACTION, DONE, OBS_STATE, REWARD
|
||||
|
||||
|
||||
def to_hwc_uint8_numpy(chw_float32_torch: torch.Tensor) -> np.ndarray:
|
||||
assert chw_float32_torch.dtype == torch.float32
|
||||
assert chw_float32_torch.ndim == 3
|
||||
c, h, w = chw_float32_torch.shape
|
||||
assert c < h and c < w, f"expect channel first images, but instead {chw_float32_torch.shape}"
|
||||
hwc_uint8_numpy = (chw_float32_torch * 255).type(torch.uint8).permute(1, 2, 0).numpy()
|
||||
return hwc_uint8_numpy
|
||||
|
||||
|
||||
def visualize_dataset(
|
||||
dataset: LeRobotDataset,
|
||||
episode_index: int,
|
||||
batch_size: int = 32,
|
||||
num_workers: int = 0,
|
||||
mode: str = "local",
|
||||
web_port: int = 9090,
|
||||
ws_port: int = 9087,
|
||||
save: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
display_compressed_images: bool = False,
|
||||
) -> Path | None:
|
||||
if save:
|
||||
assert output_dir is not None, (
|
||||
"Set an output directory where to write .rrd files with `--output-dir path/to/directory`."
|
||||
)
|
||||
|
||||
repo_id = dataset.repo_id
|
||||
|
||||
logging.info("Loading dataloader")
|
||||
dataloader = torch.utils.data.DataLoader(
|
||||
dataset,
|
||||
num_workers=num_workers,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
logging.info("Starting Rerun")
|
||||
|
||||
if mode not in ["local", "distant"]:
|
||||
raise ValueError(mode)
|
||||
|
||||
spawn_local_viewer = mode == "local" and not save
|
||||
rr.init(f"{repo_id}/episode_{episode_index}", spawn=spawn_local_viewer)
|
||||
|
||||
# Manually call python garbage collector after `rr.init` to avoid hanging in a blocking flush
|
||||
# when iterating on a dataloader with `num_workers` > 0
|
||||
# TODO(rcadene): remove `gc.collect` when rerun version 0.16 is out, which includes a fix
|
||||
gc.collect()
|
||||
|
||||
if mode == "distant":
|
||||
rr.serve_web_viewer(open_browser=False, web_port=web_port)
|
||||
|
||||
logging.info("Logging to Rerun")
|
||||
|
||||
for batch in tqdm.tqdm(dataloader, total=len(dataloader)):
|
||||
# iterate over the batch
|
||||
for i in range(len(batch["index"])):
|
||||
rr.set_time("frame_index", sequence=batch["frame_index"][i].item())
|
||||
rr.set_time("timestamp", timestamp=batch["timestamp"][i].item())
|
||||
|
||||
# display each camera image
|
||||
for key in dataset.meta.camera_keys:
|
||||
img = to_hwc_uint8_numpy(batch[key][i])
|
||||
img_entity = rr.Image(img).compress() if display_compressed_images else rr.Image(img)
|
||||
rr.log(key, entity=img_entity)
|
||||
|
||||
# display each dimension of action space (e.g. actuators command)
|
||||
if ACTION in batch:
|
||||
for dim_idx, val in enumerate(batch[ACTION][i]):
|
||||
rr.log(f"{ACTION}/{dim_idx}", rr.Scalars(val.item()))
|
||||
|
||||
# display each dimension of observed state space (e.g. agent position in joint space)
|
||||
if OBS_STATE in batch:
|
||||
for dim_idx, val in enumerate(batch[OBS_STATE][i]):
|
||||
rr.log(f"state/{dim_idx}", rr.Scalars(val.item()))
|
||||
|
||||
if DONE in batch:
|
||||
rr.log(DONE, rr.Scalars(batch[DONE][i].item()))
|
||||
|
||||
if REWARD in batch:
|
||||
rr.log(REWARD, rr.Scalars(batch[REWARD][i].item()))
|
||||
|
||||
if "next.success" in batch:
|
||||
rr.log("next.success", rr.Scalars(batch["next.success"][i].item()))
|
||||
|
||||
if mode == "local" and save:
|
||||
# save .rrd locally
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
repo_id_str = repo_id.replace("/", "_")
|
||||
rrd_path = output_dir / f"{repo_id_str}_episode_{episode_index}.rrd"
|
||||
rr.save(rrd_path)
|
||||
return rrd_path
|
||||
|
||||
elif mode == "distant":
|
||||
# stop the process from exiting since it is serving the websocket connection
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("Ctrl-C received. Exiting.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"--repo-id",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Name of hugging face repository containing a LeRobotDataset dataset (e.g. `lerobot/pusht`).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--episode-index",
|
||||
type=int,
|
||||
required=True,
|
||||
help="Episode to visualize.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Root directory for the dataset stored locally (e.g. `--root data`). By default, the dataset will be loaded from hugging face cache folder, or downloaded from the hub if available.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Directory path to write a .rrd file when `--save 1` is set.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=32,
|
||||
help="Batch size loaded by DataLoader.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-workers",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Number of processes of Dataloader for loading the data.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
type=str,
|
||||
default="local",
|
||||
help=(
|
||||
"Mode of viewing between 'local' or 'distant'. "
|
||||
"'local' requires data to be on a local machine. It spawns a viewer to visualize the data locally. "
|
||||
"'distant' creates a server on the distant machine where the data is stored. "
|
||||
"Visualize the data by connecting to the server with `rerun ws://localhost:PORT` on the local machine."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--web-port",
|
||||
type=int,
|
||||
default=9090,
|
||||
help="Web port for rerun.io when `--mode distant` is set.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ws-port",
|
||||
type=int,
|
||||
default=9087,
|
||||
help="Web socket port for rerun.io when `--mode distant` is set.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Save a .rrd file in the directory provided by `--output-dir`. "
|
||||
"It also deactivates the spawning of a viewer. "
|
||||
"Visualize the data by running `rerun path/to/file.rrd` on your local machine."
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--tolerance-s",
|
||||
type=float,
|
||||
default=1e-4,
|
||||
help=(
|
||||
"Tolerance in seconds used to ensure data timestamps respect the dataset fps value"
|
||||
"This is argument passed to the constructor of LeRobotDataset and maps to its tolerance_s constructor argument"
|
||||
"If not given, defaults to 1e-4."
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--display-compressed-images",
|
||||
type=bool,
|
||||
required=True,
|
||||
default=False,
|
||||
help="If set, display compressed images in Rerun instead of uncompressed ones.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
kwargs = vars(args)
|
||||
repo_id = kwargs.pop("repo_id")
|
||||
root = kwargs.pop("root")
|
||||
tolerance_s = kwargs.pop("tolerance_s")
|
||||
|
||||
logging.info("Loading dataset")
|
||||
dataset = LeRobotDataset(repo_id, episodes=[args.episode_index], root=root, tolerance_s=tolerance_s)
|
||||
|
||||
visualize_dataset(dataset, **vars(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,466 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Edit LeRobot datasets using various transformation tools.
|
||||
|
||||
This script allows you to delete episodes, split datasets, merge datasets,
|
||||
remove features, modify tasks, and convert image datasets to video format.
|
||||
When new_repo_id is specified, creates a new dataset.
|
||||
|
||||
Usage Examples:
|
||||
|
||||
Delete episodes 0, 2, and 5 from a dataset:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type delete_episodes \
|
||||
--operation.episode_indices "[0, 2, 5]"
|
||||
|
||||
Delete episodes and save to a new dataset:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--new_repo_id lerobot/pusht_filtered \
|
||||
--operation.type delete_episodes \
|
||||
--operation.episode_indices "[0, 2, 5]"
|
||||
|
||||
Split dataset by fractions:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type split \
|
||||
--operation.splits '{"train": 0.8, "val": 0.2}'
|
||||
|
||||
Split dataset by episode indices:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type split \
|
||||
--operation.splits '{"train": [0, 1, 2, 3], "val": [4, 5]}'
|
||||
|
||||
Split into more than two splits:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type split \
|
||||
--operation.splits '{"train": 0.6, "val": 0.2, "test": 0.2}'
|
||||
|
||||
Merge multiple datasets:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht_merged \
|
||||
--operation.type merge \
|
||||
--operation.repo_ids "['lerobot/pusht_train', 'lerobot/pusht_val']"
|
||||
|
||||
Remove camera feature:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type remove_feature \
|
||||
--operation.feature_names "['observation.images.top']"
|
||||
|
||||
Modify tasks - set a single task for all episodes (WARNING: modifies in-place):
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type modify_tasks \
|
||||
--operation.new_task "Pick up the cube and place it"
|
||||
|
||||
Modify tasks - set different tasks for specific episodes (WARNING: modifies in-place):
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type modify_tasks \
|
||||
--operation.episode_tasks '{"0": "Task A", "1": "Task B", "2": "Task A"}'
|
||||
|
||||
Modify tasks - set default task with overrides for specific episodes (WARNING: modifies in-place):
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht \
|
||||
--operation.type modify_tasks \
|
||||
--operation.new_task "Default task" \
|
||||
--operation.episode_tasks '{"5": "Special task for episode 5"}'
|
||||
|
||||
Convert image dataset to video format and save locally:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht_image \
|
||||
--operation.type convert_image_to_video \
|
||||
--operation.output_dir /path/to/output/pusht_video
|
||||
|
||||
Convert image dataset to video format and save with new repo_id:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht_image \
|
||||
--new_repo_id lerobot/pusht_video \
|
||||
--operation.type convert_image_to_video
|
||||
|
||||
Convert image dataset to video format and push to hub:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--repo_id lerobot/pusht_image \
|
||||
--new_repo_id lerobot/pusht_video \
|
||||
--operation.type convert_image_to_video \
|
||||
--push_to_hub true
|
||||
|
||||
Using JSON config file:
|
||||
python -m lerobot.scripts.lerobot_edit_dataset \
|
||||
--config_path path/to/edit_config.json
|
||||
"""
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import draccus
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.datasets.dataset_tools import (
|
||||
convert_image_to_video_dataset,
|
||||
delete_episodes,
|
||||
merge_datasets,
|
||||
modify_tasks,
|
||||
remove_feature,
|
||||
split_dataset,
|
||||
)
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.utils.constants import HF_LEROBOT_HOME
|
||||
from lerobot.utils.utils import init_logging
|
||||
|
||||
|
||||
@dataclass
|
||||
class OperationConfig(draccus.ChoiceRegistry, abc.ABC):
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return self.get_choice_name(self.__class__)
|
||||
|
||||
|
||||
@OperationConfig.register_subclass("delete_episodes")
|
||||
@dataclass
|
||||
class DeleteEpisodesConfig(OperationConfig):
|
||||
episode_indices: list[int] | None = None
|
||||
|
||||
|
||||
@OperationConfig.register_subclass("split")
|
||||
@dataclass
|
||||
class SplitConfig(OperationConfig):
|
||||
splits: dict[str, float | list[int]] | None = None
|
||||
|
||||
|
||||
@OperationConfig.register_subclass("merge")
|
||||
@dataclass
|
||||
class MergeConfig(OperationConfig):
|
||||
repo_ids: list[str] | None = None
|
||||
|
||||
|
||||
@OperationConfig.register_subclass("remove_feature")
|
||||
@dataclass
|
||||
class RemoveFeatureConfig(OperationConfig):
|
||||
feature_names: list[str] | None = None
|
||||
|
||||
|
||||
@OperationConfig.register_subclass("modify_tasks")
|
||||
@dataclass
|
||||
class ModifyTasksConfig(OperationConfig):
|
||||
new_task: str | None = None
|
||||
episode_tasks: dict[str, str] | None = None
|
||||
|
||||
|
||||
@OperationConfig.register_subclass("convert_image_to_video")
|
||||
@dataclass
|
||||
class ConvertImageToVideoConfig(OperationConfig):
|
||||
output_dir: str | None = None
|
||||
vcodec: str = "libsvtav1"
|
||||
pix_fmt: str = "yuv420p"
|
||||
g: int = 2
|
||||
crf: int = 30
|
||||
fast_decode: int = 0
|
||||
episode_indices: list[int] | None = None
|
||||
num_workers: int = 4
|
||||
max_episodes_per_batch: int | None = None
|
||||
max_frames_per_batch: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EditDatasetConfig:
|
||||
repo_id: str
|
||||
operation: OperationConfig
|
||||
root: str | None = None
|
||||
new_repo_id: str | None = None
|
||||
push_to_hub: bool = False
|
||||
|
||||
|
||||
def get_output_path(repo_id: str, new_repo_id: str | None, root: Path | None) -> tuple[str, Path]:
|
||||
if new_repo_id:
|
||||
output_repo_id = new_repo_id
|
||||
output_dir = root / new_repo_id if root else HF_LEROBOT_HOME / new_repo_id
|
||||
else:
|
||||
output_repo_id = repo_id
|
||||
dataset_path = root / repo_id if root else HF_LEROBOT_HOME / repo_id
|
||||
old_path = Path(str(dataset_path) + "_old")
|
||||
|
||||
if dataset_path.exists():
|
||||
if old_path.exists():
|
||||
shutil.rmtree(old_path)
|
||||
shutil.move(str(dataset_path), str(old_path))
|
||||
|
||||
output_dir = dataset_path
|
||||
|
||||
return output_repo_id, output_dir
|
||||
|
||||
|
||||
def handle_delete_episodes(cfg: EditDatasetConfig) -> None:
|
||||
if not isinstance(cfg.operation, DeleteEpisodesConfig):
|
||||
raise ValueError("Operation config must be DeleteEpisodesConfig")
|
||||
|
||||
if not cfg.operation.episode_indices:
|
||||
raise ValueError("episode_indices must be specified for delete_episodes operation")
|
||||
|
||||
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
|
||||
output_repo_id, output_dir = get_output_path(
|
||||
cfg.repo_id, cfg.new_repo_id, Path(cfg.root) if cfg.root else None
|
||||
)
|
||||
|
||||
if cfg.new_repo_id is None:
|
||||
dataset.root = Path(str(dataset.root) + "_old")
|
||||
|
||||
logging.info(f"Deleting episodes {cfg.operation.episode_indices} from {cfg.repo_id}")
|
||||
new_dataset = delete_episodes(
|
||||
dataset,
|
||||
episode_indices=cfg.operation.episode_indices,
|
||||
output_dir=output_dir,
|
||||
repo_id=output_repo_id,
|
||||
)
|
||||
|
||||
logging.info(f"Dataset saved to {output_dir}")
|
||||
logging.info(f"Episodes: {new_dataset.meta.total_episodes}, Frames: {new_dataset.meta.total_frames}")
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logging.info(f"Pushing to hub as {output_repo_id}")
|
||||
LeRobotDataset(output_repo_id, root=output_dir).push_to_hub()
|
||||
|
||||
|
||||
def handle_split(cfg: EditDatasetConfig) -> None:
|
||||
if not isinstance(cfg.operation, SplitConfig):
|
||||
raise ValueError("Operation config must be SplitConfig")
|
||||
|
||||
if not cfg.operation.splits:
|
||||
raise ValueError(
|
||||
"splits dict must be specified with split names as keys and fractions/episode lists as values"
|
||||
)
|
||||
|
||||
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
|
||||
|
||||
logging.info(f"Splitting dataset {cfg.repo_id} with splits: {cfg.operation.splits}")
|
||||
split_datasets = split_dataset(dataset, splits=cfg.operation.splits)
|
||||
|
||||
for split_name, split_ds in split_datasets.items():
|
||||
split_repo_id = f"{cfg.repo_id}_{split_name}"
|
||||
logging.info(
|
||||
f"{split_name}: {split_ds.meta.total_episodes} episodes, {split_ds.meta.total_frames} frames"
|
||||
)
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logging.info(f"Pushing {split_name} split to hub as {split_repo_id}")
|
||||
LeRobotDataset(split_ds.repo_id, root=split_ds.root).push_to_hub()
|
||||
|
||||
|
||||
def handle_merge(cfg: EditDatasetConfig) -> None:
|
||||
if not isinstance(cfg.operation, MergeConfig):
|
||||
raise ValueError("Operation config must be MergeConfig")
|
||||
|
||||
if not cfg.operation.repo_ids:
|
||||
raise ValueError("repo_ids must be specified for merge operation")
|
||||
|
||||
if not cfg.repo_id:
|
||||
raise ValueError("repo_id must be specified as the output repository for merged dataset")
|
||||
|
||||
logging.info(f"Loading {len(cfg.operation.repo_ids)} datasets to merge")
|
||||
datasets = [LeRobotDataset(repo_id, root=cfg.root) for repo_id in cfg.operation.repo_ids]
|
||||
|
||||
output_dir = Path(cfg.root) / cfg.repo_id if cfg.root else HF_LEROBOT_HOME / cfg.repo_id
|
||||
|
||||
logging.info(f"Merging datasets into {cfg.repo_id}")
|
||||
merged_dataset = merge_datasets(
|
||||
datasets,
|
||||
output_repo_id=cfg.repo_id,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
logging.info(f"Merged dataset saved to {output_dir}")
|
||||
logging.info(
|
||||
f"Episodes: {merged_dataset.meta.total_episodes}, Frames: {merged_dataset.meta.total_frames}"
|
||||
)
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logging.info(f"Pushing to hub as {cfg.repo_id}")
|
||||
LeRobotDataset(merged_dataset.repo_id, root=output_dir).push_to_hub()
|
||||
|
||||
|
||||
def handle_remove_feature(cfg: EditDatasetConfig) -> None:
|
||||
if not isinstance(cfg.operation, RemoveFeatureConfig):
|
||||
raise ValueError("Operation config must be RemoveFeatureConfig")
|
||||
|
||||
if not cfg.operation.feature_names:
|
||||
raise ValueError("feature_names must be specified for remove_feature operation")
|
||||
|
||||
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
|
||||
output_repo_id, output_dir = get_output_path(
|
||||
cfg.repo_id, cfg.new_repo_id, Path(cfg.root) if cfg.root else None
|
||||
)
|
||||
|
||||
if cfg.new_repo_id is None:
|
||||
dataset.root = Path(str(dataset.root) + "_old")
|
||||
|
||||
logging.info(f"Removing features {cfg.operation.feature_names} from {cfg.repo_id}")
|
||||
new_dataset = remove_feature(
|
||||
dataset,
|
||||
feature_names=cfg.operation.feature_names,
|
||||
output_dir=output_dir,
|
||||
repo_id=output_repo_id,
|
||||
)
|
||||
|
||||
logging.info(f"Dataset saved to {output_dir}")
|
||||
logging.info(f"Remaining features: {list(new_dataset.meta.features.keys())}")
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logging.info(f"Pushing to hub as {output_repo_id}")
|
||||
LeRobotDataset(output_repo_id, root=output_dir).push_to_hub()
|
||||
|
||||
|
||||
def handle_modify_tasks(cfg: EditDatasetConfig) -> None:
|
||||
if not isinstance(cfg.operation, ModifyTasksConfig):
|
||||
raise ValueError("Operation config must be ModifyTasksConfig")
|
||||
|
||||
new_task = cfg.operation.new_task
|
||||
episode_tasks_raw = cfg.operation.episode_tasks
|
||||
|
||||
if new_task is None and episode_tasks_raw is None:
|
||||
raise ValueError("Must specify at least one of new_task or episode_tasks for modify_tasks operation")
|
||||
|
||||
# Warn about in-place modification behavior
|
||||
if cfg.new_repo_id is not None:
|
||||
logging.warning("modify_tasks modifies datasets in-place. The --new_repo_id parameter is ignored.")
|
||||
|
||||
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
|
||||
logging.warning(f"Modifying dataset in-place at {dataset.root}. Original data will be overwritten.")
|
||||
|
||||
# Convert episode_tasks keys from string to int if needed (CLI passes strings)
|
||||
episode_tasks: dict[int, str] | None = None
|
||||
if episode_tasks_raw is not None:
|
||||
episode_tasks = {int(k): v for k, v in episode_tasks_raw.items()}
|
||||
|
||||
logging.info(f"Modifying tasks in {cfg.repo_id}")
|
||||
if new_task:
|
||||
logging.info(f" Default task: '{new_task}'")
|
||||
if episode_tasks:
|
||||
logging.info(f" Episode-specific tasks: {episode_tasks}")
|
||||
|
||||
modified_dataset = modify_tasks(
|
||||
dataset,
|
||||
new_task=new_task,
|
||||
episode_tasks=episode_tasks,
|
||||
)
|
||||
|
||||
logging.info(f"Dataset modified at {dataset.root}")
|
||||
logging.info(f"Tasks: {list(modified_dataset.meta.tasks.index)}")
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logging.info(f"Pushing to hub as {cfg.repo_id}")
|
||||
modified_dataset.push_to_hub()
|
||||
|
||||
|
||||
def handle_convert_image_to_video(cfg: EditDatasetConfig) -> None:
|
||||
# Note: Parser may create any config type with the right fields, so we access fields directly
|
||||
# instead of checking isinstance()
|
||||
dataset = LeRobotDataset(cfg.repo_id, root=cfg.root)
|
||||
|
||||
# Determine output directory and repo_id
|
||||
# Priority: 1) new_repo_id, 2) operation.output_dir, 3) auto-generated name
|
||||
output_dir_config = getattr(cfg.operation, "output_dir", None)
|
||||
|
||||
if cfg.new_repo_id:
|
||||
# Use new_repo_id for both local storage and hub push
|
||||
output_repo_id = cfg.new_repo_id
|
||||
# Place new dataset as a sibling to the original dataset
|
||||
# Get the parent of the actual dataset root (not cfg.root which might be the lerobot cache dir)
|
||||
# Extract just the dataset name (after last slash) for the local directory
|
||||
local_dir_name = cfg.new_repo_id.split("/")[-1]
|
||||
output_dir = dataset.root.parent / local_dir_name
|
||||
logging.info(f"Saving to new dataset: {cfg.new_repo_id} at {output_dir}")
|
||||
elif output_dir_config:
|
||||
# Use custom output directory for local-only storage
|
||||
output_dir = Path(output_dir_config)
|
||||
# Extract repo name from output_dir for the dataset
|
||||
output_repo_id = output_dir.name
|
||||
logging.info(f"Saving to local directory: {output_dir}")
|
||||
else:
|
||||
# Auto-generate name: append "_video" to original repo_id
|
||||
output_repo_id = f"{cfg.repo_id}_video"
|
||||
# Place new dataset as a sibling to the original dataset
|
||||
# Extract just the dataset name (after last slash) for the local directory
|
||||
local_dir_name = output_repo_id.split("/")[-1]
|
||||
output_dir = dataset.root.parent / local_dir_name
|
||||
logging.info(f"Saving to auto-generated location: {output_dir}")
|
||||
|
||||
logging.info(f"Converting dataset {cfg.repo_id} to video format")
|
||||
|
||||
new_dataset = convert_image_to_video_dataset(
|
||||
dataset=dataset,
|
||||
output_dir=output_dir,
|
||||
repo_id=output_repo_id,
|
||||
vcodec=getattr(cfg.operation, "vcodec", "libsvtav1"),
|
||||
pix_fmt=getattr(cfg.operation, "pix_fmt", "yuv420p"),
|
||||
g=getattr(cfg.operation, "g", 2),
|
||||
crf=getattr(cfg.operation, "crf", 30),
|
||||
fast_decode=getattr(cfg.operation, "fast_decode", 0),
|
||||
episode_indices=getattr(cfg.operation, "episode_indices", None),
|
||||
num_workers=getattr(cfg.operation, "num_workers", 4),
|
||||
max_episodes_per_batch=getattr(cfg.operation, "max_episodes_per_batch", None),
|
||||
max_frames_per_batch=getattr(cfg.operation, "max_frames_per_batch", None),
|
||||
)
|
||||
|
||||
logging.info("Video dataset created successfully!")
|
||||
logging.info(f"Location: {output_dir}")
|
||||
logging.info(f"Episodes: {new_dataset.meta.total_episodes}")
|
||||
logging.info(f"Frames: {new_dataset.meta.total_frames}")
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logging.info(f"Pushing to hub as {output_repo_id}...")
|
||||
new_dataset.push_to_hub()
|
||||
logging.info("✓ Successfully pushed to hub!")
|
||||
else:
|
||||
logging.info("Dataset saved locally (not pushed to hub)")
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def edit_dataset(cfg: EditDatasetConfig) -> None:
|
||||
operation_type = cfg.operation.type
|
||||
|
||||
if operation_type == "delete_episodes":
|
||||
handle_delete_episodes(cfg)
|
||||
elif operation_type == "split":
|
||||
handle_split(cfg)
|
||||
elif operation_type == "merge":
|
||||
handle_merge(cfg)
|
||||
elif operation_type == "remove_feature":
|
||||
handle_remove_feature(cfg)
|
||||
elif operation_type == "modify_tasks":
|
||||
handle_modify_tasks(cfg)
|
||||
elif operation_type == "convert_image_to_video":
|
||||
handle_convert_image_to_video(cfg)
|
||||
else:
|
||||
available = ", ".join(OperationConfig.get_known_choices())
|
||||
raise ValueError(f"Unknown operation: {operation_type}\nAvailable operations: {available}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_logging()
|
||||
edit_dataset()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
813
src/lerobot/scripts/lerobot_eval.py
普通文件
813
src/lerobot/scripts/lerobot_eval.py
普通文件
@ -0,0 +1,813 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Evaluate a policy on an environment by running rollouts and computing metrics.
|
||||
|
||||
Usage examples:
|
||||
|
||||
You want to evaluate a model from the hub (eg: https://huggingface.co/lerobot/diffusion_pusht)
|
||||
for 10 episodes.
|
||||
|
||||
```
|
||||
lerobot-eval \
|
||||
--policy.path=lerobot/diffusion_pusht \
|
||||
--env.type=pusht \
|
||||
--eval.batch_size=10 \
|
||||
--eval.n_episodes=10 \
|
||||
--policy.use_amp=false \
|
||||
--policy.device=cuda
|
||||
```
|
||||
|
||||
OR, you want to evaluate a model checkpoint from the LeRobot training script for 10 episodes.
|
||||
```
|
||||
lerobot-eval \
|
||||
--policy.path=outputs/train/diffusion_pusht/checkpoints/005000/pretrained_model \
|
||||
--env.type=pusht \
|
||||
--eval.batch_size=10 \
|
||||
--eval.n_episodes=10 \
|
||||
--policy.use_amp=false \
|
||||
--policy.device=cuda
|
||||
```
|
||||
|
||||
Note that in both examples, the repo/folder should contain at least `config.json` and `model.safetensors` files.
|
||||
|
||||
You can learn about the CLI options for this script in the `EvalPipelineConfig` in lerobot/configs/eval.py
|
||||
"""
|
||||
|
||||
import concurrent.futures as cf
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from contextlib import nullcontext
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
from typing import Any, TypedDict
|
||||
|
||||
import einops
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
import torch
|
||||
from termcolor import colored
|
||||
from torch import Tensor, nn
|
||||
from tqdm import trange
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.eval import EvalPipelineConfig
|
||||
from lerobot.envs.factory import make_env, make_env_pre_post_processors
|
||||
from lerobot.envs.utils import (
|
||||
add_envs_task,
|
||||
check_env_attributes_and_types,
|
||||
close_envs,
|
||||
preprocess_observation,
|
||||
)
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.processor import PolicyAction, PolicyProcessorPipeline
|
||||
from lerobot.utils.constants import ACTION, DONE, OBS_STR, REWARD
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.io_utils import write_video
|
||||
from lerobot.utils.random_utils import set_seed
|
||||
from lerobot.utils.utils import (
|
||||
get_safe_torch_device,
|
||||
init_logging,
|
||||
inside_slurm,
|
||||
)
|
||||
|
||||
|
||||
def rollout(
|
||||
env: gym.vector.VectorEnv,
|
||||
policy: PreTrainedPolicy,
|
||||
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
seeds: list[int] | None = None,
|
||||
return_observations: bool = False,
|
||||
render_callback: Callable[[gym.vector.VectorEnv], None] | None = None,
|
||||
) -> dict:
|
||||
"""Run a batched policy rollout once through a batch of environments.
|
||||
|
||||
Note that all environments in the batch are run until the last environment is done. This means some
|
||||
data will probably need to be discarded (for environments that aren't the first one to be done).
|
||||
|
||||
The return dictionary contains:
|
||||
(optional) "observation": A dictionary of (batch, sequence + 1, *) tensors mapped to observation
|
||||
keys. NOTE that this has an extra sequence element relative to the other keys in the
|
||||
dictionary. This is because an extra observation is included for after the environment is
|
||||
terminated or truncated.
|
||||
"action": A (batch, sequence, action_dim) tensor of actions applied based on the observations (not
|
||||
including the last observations).
|
||||
"reward": A (batch, sequence) tensor of rewards received for applying the actions.
|
||||
"success": A (batch, sequence) tensor of success conditions (the only time this can be True is upon
|
||||
environment termination/truncation).
|
||||
"done": A (batch, sequence) tensor of **cumulative** done conditions. For any given batch element,
|
||||
the first True is followed by True's all the way till the end. This can be used for masking
|
||||
extraneous elements from the sequences above.
|
||||
|
||||
Args:
|
||||
env: The batch of environments.
|
||||
policy: The policy. Must be a PyTorch nn module.
|
||||
seeds: The environments are seeded once at the start of the rollout. If provided, this argument
|
||||
specifies the seeds for each of the environments.
|
||||
return_observations: Whether to include all observations in the returned rollout data. Observations
|
||||
are returned optionally because they typically take more memory to cache. Defaults to False.
|
||||
render_callback: Optional rendering callback to be used after the environments are reset, and after
|
||||
every step.
|
||||
Returns:
|
||||
The dictionary described above.
|
||||
"""
|
||||
assert isinstance(policy, nn.Module), "Policy must be a PyTorch nn module."
|
||||
|
||||
# Reset the policy and environments.
|
||||
policy.reset()
|
||||
observation, info = env.reset(seed=seeds)
|
||||
if render_callback is not None:
|
||||
render_callback(env)
|
||||
|
||||
all_observations = []
|
||||
all_actions = []
|
||||
all_rewards = []
|
||||
all_successes = []
|
||||
all_dones = []
|
||||
|
||||
step = 0
|
||||
# Keep track of which environments are done.
|
||||
done = np.array([False] * env.num_envs)
|
||||
max_steps = env.call("_max_episode_steps")[0]
|
||||
progbar = trange(
|
||||
max_steps,
|
||||
desc=f"Running rollout with at most {max_steps} steps",
|
||||
disable=inside_slurm(), # we dont want progress bar when we use slurm, since it clutters the logs
|
||||
leave=False,
|
||||
)
|
||||
check_env_attributes_and_types(env)
|
||||
while not np.all(done) and step < max_steps:
|
||||
# Numpy array to tensor and changing dictionary keys to LeRobot policy format.
|
||||
observation = preprocess_observation(observation)
|
||||
if return_observations:
|
||||
all_observations.append(deepcopy(observation))
|
||||
|
||||
# Infer "task" from attributes of environments.
|
||||
# TODO: works with SyncVectorEnv but not AsyncVectorEnv
|
||||
observation = add_envs_task(env, observation)
|
||||
|
||||
# Apply environment-specific preprocessing (e.g., LiberoProcessorStep for LIBERO)
|
||||
observation = env_preprocessor(observation)
|
||||
|
||||
observation = preprocessor(observation)
|
||||
with torch.inference_mode():
|
||||
action = policy.select_action(observation)
|
||||
action = postprocessor(action)
|
||||
|
||||
action_transition = {ACTION: action}
|
||||
action_transition = env_postprocessor(action_transition)
|
||||
action = action_transition[ACTION]
|
||||
|
||||
# Convert to CPU / numpy.
|
||||
action_numpy: np.ndarray = action.to("cpu").numpy()
|
||||
assert action_numpy.ndim == 2, "Action dimensions should be (batch, action_dim)"
|
||||
|
||||
# Apply the next action.
|
||||
observation, reward, terminated, truncated, info = env.step(action_numpy)
|
||||
if render_callback is not None:
|
||||
render_callback(env)
|
||||
|
||||
# VectorEnv stores is_success in `info["final_info"][env_index]["is_success"]`. "final_info" isn't
|
||||
# available if none of the envs finished.
|
||||
if "final_info" in info:
|
||||
final_info = info["final_info"]
|
||||
if not isinstance(final_info, dict):
|
||||
raise RuntimeError(
|
||||
"Unsupported `final_info` format: expected dict (Gymnasium >= 1.0). "
|
||||
"You're likely using an older version of gymnasium (< 1.0). Please upgrade."
|
||||
)
|
||||
successes = final_info["is_success"].tolist()
|
||||
else:
|
||||
successes = [False] * env.num_envs
|
||||
|
||||
# Keep track of which environments are done so far.
|
||||
# Mark the episode as done if we reach the maximum step limit.
|
||||
# This ensures that the rollout always terminates cleanly at `max_steps`,
|
||||
# and allows logging/saving (e.g., videos) to be triggered consistently.
|
||||
done = terminated | truncated | done
|
||||
if step + 1 == max_steps:
|
||||
done = np.ones_like(done, dtype=bool)
|
||||
|
||||
all_actions.append(torch.from_numpy(action_numpy))
|
||||
all_rewards.append(torch.from_numpy(reward))
|
||||
all_dones.append(torch.from_numpy(done))
|
||||
all_successes.append(torch.tensor(successes))
|
||||
|
||||
step += 1
|
||||
running_success_rate = (
|
||||
einops.reduce(torch.stack(all_successes, dim=1), "b n -> b", "any").numpy().mean()
|
||||
)
|
||||
progbar.set_postfix({"running_success_rate": f"{running_success_rate.item() * 100:.1f}%"})
|
||||
progbar.update()
|
||||
|
||||
# Track the final observation.
|
||||
if return_observations:
|
||||
observation = preprocess_observation(observation)
|
||||
all_observations.append(deepcopy(observation))
|
||||
|
||||
# Stack the sequence along the first dimension so that we have (batch, sequence, *) tensors.
|
||||
ret = {
|
||||
ACTION: torch.stack(all_actions, dim=1),
|
||||
"reward": torch.stack(all_rewards, dim=1),
|
||||
"success": torch.stack(all_successes, dim=1),
|
||||
"done": torch.stack(all_dones, dim=1),
|
||||
}
|
||||
if return_observations:
|
||||
stacked_observations = {}
|
||||
for key in all_observations[0]:
|
||||
stacked_observations[key] = torch.stack([obs[key] for obs in all_observations], dim=1)
|
||||
ret[OBS_STR] = stacked_observations
|
||||
|
||||
if hasattr(policy, "use_original_modules"):
|
||||
policy.use_original_modules()
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def eval_policy(
|
||||
env: gym.vector.VectorEnv,
|
||||
policy: PreTrainedPolicy,
|
||||
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
n_episodes: int,
|
||||
max_episodes_rendered: int = 0,
|
||||
videos_dir: Path | None = None,
|
||||
return_episode_data: bool = False,
|
||||
start_seed: int | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Args:
|
||||
env: The batch of environments.
|
||||
policy: The policy.
|
||||
n_episodes: The number of episodes to evaluate.
|
||||
max_episodes_rendered: Maximum number of episodes to render into videos.
|
||||
videos_dir: Where to save rendered videos.
|
||||
return_episode_data: Whether to return episode data for online training. Incorporates the data into
|
||||
the "episodes" key of the returned dictionary.
|
||||
start_seed: The first seed to use for the first individual rollout. For all subsequent rollouts the
|
||||
seed is incremented by 1. If not provided, the environments are not manually seeded.
|
||||
Returns:
|
||||
Dictionary with metrics and data regarding the rollouts.
|
||||
"""
|
||||
if max_episodes_rendered > 0 and not videos_dir:
|
||||
raise ValueError("If max_episodes_rendered > 0, videos_dir must be provided.")
|
||||
|
||||
if not isinstance(policy, PreTrainedPolicy):
|
||||
exc = ValueError(
|
||||
f"Policy of type 'PreTrainedPolicy' is expected, but type '{type(policy)}' was provided."
|
||||
)
|
||||
try:
|
||||
from peft import PeftModel
|
||||
|
||||
if not isinstance(policy, PeftModel):
|
||||
raise exc
|
||||
except ImportError:
|
||||
raise exc from None
|
||||
|
||||
start = time.time()
|
||||
policy.eval()
|
||||
|
||||
# Determine how many batched rollouts we need to get n_episodes. Note that if n_episodes is not evenly
|
||||
# divisible by env.num_envs we end up discarding some data in the last batch.
|
||||
n_batches = n_episodes // env.num_envs + int((n_episodes % env.num_envs) != 0)
|
||||
|
||||
# Keep track of some metrics.
|
||||
sum_rewards = []
|
||||
max_rewards = []
|
||||
all_successes = []
|
||||
all_seeds = []
|
||||
threads = [] # for video saving threads
|
||||
n_episodes_rendered = 0 # for saving the correct number of videos
|
||||
|
||||
# Callback for visualization.
|
||||
def render_frame(env: gym.vector.VectorEnv):
|
||||
# noqa: B023
|
||||
if n_episodes_rendered >= max_episodes_rendered:
|
||||
return
|
||||
n_to_render_now = min(max_episodes_rendered - n_episodes_rendered, env.num_envs)
|
||||
if isinstance(env, gym.vector.SyncVectorEnv):
|
||||
ep_frames.append(np.stack([env.envs[i].render() for i in range(n_to_render_now)])) # noqa: B023
|
||||
elif isinstance(env, gym.vector.AsyncVectorEnv):
|
||||
# Here we must render all frames and discard any we don't need.
|
||||
ep_frames.append(np.stack(env.call("render")[:n_to_render_now]))
|
||||
|
||||
if max_episodes_rendered > 0:
|
||||
video_paths: list[str] = []
|
||||
|
||||
if return_episode_data:
|
||||
episode_data: dict | None = None
|
||||
|
||||
# we dont want progress bar when we use slurm, since it clutters the logs
|
||||
progbar = trange(n_batches, desc="Stepping through eval batches", disable=inside_slurm())
|
||||
for batch_ix in progbar:
|
||||
# Cache frames for rendering videos. Each item will be (b, h, w, c), and the list indexes the rollout
|
||||
# step.
|
||||
if max_episodes_rendered > 0:
|
||||
ep_frames: list[np.ndarray] = []
|
||||
|
||||
if start_seed is None:
|
||||
seeds = None
|
||||
else:
|
||||
seeds = range(
|
||||
start_seed + (batch_ix * env.num_envs), start_seed + ((batch_ix + 1) * env.num_envs)
|
||||
)
|
||||
rollout_data = rollout(
|
||||
env=env,
|
||||
policy=policy,
|
||||
env_preprocessor=env_preprocessor,
|
||||
env_postprocessor=env_postprocessor,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
seeds=list(seeds) if seeds else None,
|
||||
return_observations=return_episode_data,
|
||||
render_callback=render_frame if max_episodes_rendered > 0 else None,
|
||||
)
|
||||
|
||||
# Figure out where in each rollout sequence the first done condition was encountered (results after
|
||||
# this won't be included).
|
||||
n_steps = rollout_data["done"].shape[1]
|
||||
# Note: this relies on a property of argmax: that it returns the first occurrence as a tiebreaker.
|
||||
done_indices = torch.argmax(rollout_data["done"].to(int), dim=1)
|
||||
|
||||
# Make a mask with shape (batch, n_steps) to mask out rollout data after the first done
|
||||
# (batch-element-wise). Note the `done_indices + 1` to make sure to keep the data from the done step.
|
||||
mask = (torch.arange(n_steps) <= einops.repeat(done_indices + 1, "b -> b s", s=n_steps)).int()
|
||||
# Extend metrics.
|
||||
batch_sum_rewards = einops.reduce((rollout_data["reward"] * mask), "b n -> b", "sum")
|
||||
sum_rewards.extend(batch_sum_rewards.tolist())
|
||||
batch_max_rewards = einops.reduce((rollout_data["reward"] * mask), "b n -> b", "max")
|
||||
max_rewards.extend(batch_max_rewards.tolist())
|
||||
batch_successes = einops.reduce((rollout_data["success"] * mask), "b n -> b", "any")
|
||||
all_successes.extend(batch_successes.tolist())
|
||||
if seeds:
|
||||
all_seeds.extend(seeds)
|
||||
else:
|
||||
all_seeds.append(None)
|
||||
|
||||
# FIXME: episode_data is either None or it doesn't exist
|
||||
if return_episode_data:
|
||||
this_episode_data = _compile_episode_data(
|
||||
rollout_data,
|
||||
done_indices,
|
||||
start_episode_index=batch_ix * env.num_envs,
|
||||
start_data_index=(0 if episode_data is None else (episode_data["index"][-1].item() + 1)),
|
||||
fps=env.unwrapped.metadata["render_fps"],
|
||||
)
|
||||
if episode_data is None:
|
||||
episode_data = this_episode_data
|
||||
else:
|
||||
# Some sanity checks to make sure we are correctly compiling the data.
|
||||
assert episode_data["episode_index"][-1] + 1 == this_episode_data["episode_index"][0]
|
||||
assert episode_data["index"][-1] + 1 == this_episode_data["index"][0]
|
||||
# Concatenate the episode data.
|
||||
episode_data = {k: torch.cat([episode_data[k], this_episode_data[k]]) for k in episode_data}
|
||||
|
||||
# Maybe render video for visualization.
|
||||
if max_episodes_rendered > 0 and len(ep_frames) > 0:
|
||||
batch_stacked_frames = np.stack(ep_frames, axis=1) # (b, t, *)
|
||||
for stacked_frames, done_index in zip(
|
||||
batch_stacked_frames, done_indices.flatten().tolist(), strict=False
|
||||
):
|
||||
if n_episodes_rendered >= max_episodes_rendered:
|
||||
break
|
||||
|
||||
videos_dir.mkdir(parents=True, exist_ok=True)
|
||||
video_path = videos_dir / f"eval_episode_{n_episodes_rendered}.mp4"
|
||||
video_paths.append(str(video_path))
|
||||
thread = threading.Thread(
|
||||
target=write_video,
|
||||
args=(
|
||||
str(video_path),
|
||||
stacked_frames[: done_index + 1], # + 1 to capture the last observation
|
||||
env.unwrapped.metadata["render_fps"],
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
n_episodes_rendered += 1
|
||||
|
||||
progbar.set_postfix(
|
||||
{"running_success_rate": f"{np.mean(all_successes[:n_episodes]).item() * 100:.1f}%"}
|
||||
)
|
||||
|
||||
# Wait till all video rendering threads are done.
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Compile eval info.
|
||||
info = {
|
||||
"per_episode": [
|
||||
{
|
||||
"episode_ix": i,
|
||||
"sum_reward": sum_reward,
|
||||
"max_reward": max_reward,
|
||||
"success": success,
|
||||
"seed": seed,
|
||||
}
|
||||
for i, (sum_reward, max_reward, success, seed) in enumerate(
|
||||
zip(
|
||||
sum_rewards[:n_episodes],
|
||||
max_rewards[:n_episodes],
|
||||
all_successes[:n_episodes],
|
||||
all_seeds[:n_episodes],
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
],
|
||||
"aggregated": {
|
||||
"avg_sum_reward": float(np.nanmean(sum_rewards[:n_episodes])),
|
||||
"avg_max_reward": float(np.nanmean(max_rewards[:n_episodes])),
|
||||
"pc_success": float(np.nanmean(all_successes[:n_episodes]) * 100),
|
||||
"eval_s": time.time() - start,
|
||||
"eval_ep_s": (time.time() - start) / n_episodes,
|
||||
},
|
||||
}
|
||||
|
||||
if return_episode_data:
|
||||
info["episodes"] = episode_data
|
||||
|
||||
if max_episodes_rendered > 0:
|
||||
info["video_paths"] = video_paths
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def _compile_episode_data(
|
||||
rollout_data: dict, done_indices: Tensor, start_episode_index: int, start_data_index: int, fps: float
|
||||
) -> dict:
|
||||
"""Convenience function for `eval_policy(return_episode_data=True)`
|
||||
|
||||
Compiles all the rollout data into a Hugging Face dataset.
|
||||
|
||||
Similar logic is implemented when datasets are pushed to hub (see: `push_to_hub`).
|
||||
"""
|
||||
ep_dicts = []
|
||||
total_frames = 0
|
||||
for ep_ix in range(rollout_data[ACTION].shape[0]):
|
||||
# + 2 to include the first done frame and the last observation frame.
|
||||
num_frames = done_indices[ep_ix].item() + 2
|
||||
total_frames += num_frames
|
||||
|
||||
# Here we do `num_frames - 1` as we don't want to include the last observation frame just yet.
|
||||
ep_dict = {
|
||||
ACTION: rollout_data[ACTION][ep_ix, : num_frames - 1],
|
||||
"episode_index": torch.tensor([start_episode_index + ep_ix] * (num_frames - 1)),
|
||||
"frame_index": torch.arange(0, num_frames - 1, 1),
|
||||
"timestamp": torch.arange(0, num_frames - 1, 1) / fps,
|
||||
DONE: rollout_data["done"][ep_ix, : num_frames - 1],
|
||||
"next.success": rollout_data["success"][ep_ix, : num_frames - 1],
|
||||
REWARD: rollout_data["reward"][ep_ix, : num_frames - 1].type(torch.float32),
|
||||
}
|
||||
|
||||
# For the last observation frame, all other keys will just be copy padded.
|
||||
for k in ep_dict:
|
||||
ep_dict[k] = torch.cat([ep_dict[k], ep_dict[k][-1:]])
|
||||
|
||||
for key in rollout_data[OBS_STR]:
|
||||
ep_dict[key] = rollout_data[OBS_STR][key][ep_ix, :num_frames]
|
||||
|
||||
ep_dicts.append(ep_dict)
|
||||
|
||||
data_dict = {}
|
||||
for key in ep_dicts[0]:
|
||||
data_dict[key] = torch.cat([x[key] for x in ep_dicts])
|
||||
|
||||
data_dict["index"] = torch.arange(start_data_index, start_data_index + total_frames, 1)
|
||||
|
||||
return data_dict
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def eval_main(cfg: EvalPipelineConfig):
|
||||
logging.info(pformat(asdict(cfg)))
|
||||
|
||||
# Check device is available
|
||||
device = get_safe_torch_device(cfg.policy.device, log=True)
|
||||
|
||||
torch.backends.cudnn.benchmark = True
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
set_seed(cfg.seed)
|
||||
|
||||
logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
|
||||
|
||||
logging.info("Making environment.")
|
||||
envs = make_env(
|
||||
cfg.env,
|
||||
n_envs=cfg.eval.batch_size,
|
||||
use_async_envs=cfg.eval.use_async_envs,
|
||||
trust_remote_code=cfg.trust_remote_code,
|
||||
)
|
||||
|
||||
logging.info("Making policy.")
|
||||
|
||||
policy = make_policy(
|
||||
cfg=cfg.policy,
|
||||
env_cfg=cfg.env,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
|
||||
policy.eval()
|
||||
|
||||
# The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
|
||||
preprocessor_overrides = {
|
||||
"device_processor": {"device": str(policy.config.device)},
|
||||
"rename_observations_processor": {"rename_map": cfg.rename_map},
|
||||
}
|
||||
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.policy,
|
||||
pretrained_path=cfg.policy.pretrained_path,
|
||||
preprocessor_overrides=preprocessor_overrides,
|
||||
)
|
||||
|
||||
# Create environment-specific preprocessor and postprocessor (e.g., for LIBERO environments)
|
||||
env_preprocessor, env_postprocessor = make_env_pre_post_processors(env_cfg=cfg.env, policy_cfg=cfg.policy)
|
||||
|
||||
with torch.no_grad(), torch.autocast(device_type=device.type) if cfg.policy.use_amp else nullcontext():
|
||||
info = eval_policy_all(
|
||||
envs=envs,
|
||||
policy=policy,
|
||||
env_preprocessor=env_preprocessor,
|
||||
env_postprocessor=env_postprocessor,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
n_episodes=cfg.eval.n_episodes,
|
||||
max_episodes_rendered=10,
|
||||
videos_dir=Path(cfg.output_dir) / "videos",
|
||||
start_seed=cfg.seed,
|
||||
max_parallel_tasks=cfg.env.max_parallel_tasks,
|
||||
)
|
||||
print("Overall Aggregated Metrics:")
|
||||
print(info["overall"])
|
||||
|
||||
# Print per-suite stats
|
||||
for task_group, task_group_info in info.items():
|
||||
print(f"\nAggregated Metrics for {task_group}:")
|
||||
print(task_group_info)
|
||||
# Close all vec envs
|
||||
close_envs(envs)
|
||||
|
||||
# Save info
|
||||
with open(Path(cfg.output_dir) / "eval_info.json", "w") as f:
|
||||
json.dump(info, f, indent=2)
|
||||
|
||||
logging.info("End of eval")
|
||||
|
||||
|
||||
# ---- typed payload returned by one task eval ----
|
||||
class TaskMetrics(TypedDict):
|
||||
sum_rewards: list[float]
|
||||
max_rewards: list[float]
|
||||
successes: list[bool]
|
||||
video_paths: list[str]
|
||||
|
||||
|
||||
ACC_KEYS = ("sum_rewards", "max_rewards", "successes", "video_paths")
|
||||
|
||||
|
||||
def eval_one(
|
||||
env: gym.vector.VectorEnv,
|
||||
*,
|
||||
policy: PreTrainedPolicy,
|
||||
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
n_episodes: int,
|
||||
max_episodes_rendered: int,
|
||||
videos_dir: Path | None,
|
||||
return_episode_data: bool,
|
||||
start_seed: int | None,
|
||||
) -> TaskMetrics:
|
||||
"""Evaluates one task_id of one suite using the provided vec env."""
|
||||
|
||||
task_videos_dir = videos_dir
|
||||
|
||||
task_result = eval_policy(
|
||||
env=env,
|
||||
policy=policy,
|
||||
env_preprocessor=env_preprocessor,
|
||||
env_postprocessor=env_postprocessor,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
n_episodes=n_episodes,
|
||||
max_episodes_rendered=max_episodes_rendered,
|
||||
videos_dir=task_videos_dir,
|
||||
return_episode_data=return_episode_data,
|
||||
start_seed=start_seed,
|
||||
)
|
||||
|
||||
per_episode = task_result["per_episode"]
|
||||
return TaskMetrics(
|
||||
sum_rewards=[ep["sum_reward"] for ep in per_episode],
|
||||
max_rewards=[ep["max_reward"] for ep in per_episode],
|
||||
successes=[ep["success"] for ep in per_episode],
|
||||
video_paths=task_result.get("video_paths", []),
|
||||
)
|
||||
|
||||
|
||||
def run_one(
|
||||
task_group: str,
|
||||
task_id: int,
|
||||
env,
|
||||
*,
|
||||
policy,
|
||||
env_preprocessor,
|
||||
env_postprocessor,
|
||||
preprocessor,
|
||||
postprocessor,
|
||||
n_episodes: int,
|
||||
max_episodes_rendered: int,
|
||||
videos_dir: Path | None,
|
||||
return_episode_data: bool,
|
||||
start_seed: int | None,
|
||||
):
|
||||
"""
|
||||
Run eval_one for a single (task_group, task_id, env).
|
||||
Returns (task_group, task_id, task_metrics_dict).
|
||||
This function is intentionally module-level to make it easy to test.
|
||||
"""
|
||||
task_videos_dir = None
|
||||
if videos_dir is not None:
|
||||
task_videos_dir = videos_dir / f"{task_group}_{task_id}"
|
||||
task_videos_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Call the existing eval_one (assumed to return TaskMetrics-like dict)
|
||||
metrics = eval_one(
|
||||
env,
|
||||
policy=policy,
|
||||
env_preprocessor=env_preprocessor,
|
||||
env_postprocessor=env_postprocessor,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
n_episodes=n_episodes,
|
||||
max_episodes_rendered=max_episodes_rendered,
|
||||
videos_dir=task_videos_dir,
|
||||
return_episode_data=return_episode_data,
|
||||
start_seed=start_seed,
|
||||
)
|
||||
# ensure we always provide video_paths key to simplify accumulation
|
||||
if max_episodes_rendered > 0:
|
||||
metrics.setdefault("video_paths", [])
|
||||
return task_group, task_id, metrics
|
||||
|
||||
|
||||
def eval_policy_all(
|
||||
envs: dict[str, dict[int, gym.vector.VectorEnv]],
|
||||
policy,
|
||||
env_preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
env_postprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
n_episodes: int,
|
||||
*,
|
||||
max_episodes_rendered: int = 0,
|
||||
videos_dir: Path | None = None,
|
||||
return_episode_data: bool = False,
|
||||
start_seed: int | None = None,
|
||||
max_parallel_tasks: int = 1,
|
||||
) -> dict:
|
||||
"""
|
||||
Evaluate a nested `envs` dict: {task_group: {task_id: vec_env}}.
|
||||
This implementation flattens tasks, runs them sequentially or via ThreadPoolExecutor,
|
||||
accumulates per-group and overall statistics, and returns the same aggregate metrics
|
||||
schema as the single-env evaluator (avg_sum_reward / avg_max_reward / pc_success / timings)
|
||||
plus per-task infos.
|
||||
"""
|
||||
start_t = time.time()
|
||||
|
||||
# Flatten envs into list of (task_group, task_id, env)
|
||||
tasks = [(tg, tid, vec) for tg, group in envs.items() for tid, vec in group.items()]
|
||||
|
||||
# accumulators: track metrics at both per-group level and across all groups
|
||||
group_acc: dict[str, dict[str, list]] = defaultdict(lambda: {k: [] for k in ACC_KEYS})
|
||||
overall: dict[str, list] = {k: [] for k in ACC_KEYS}
|
||||
per_task_infos: list[dict] = []
|
||||
|
||||
# small inline helper to accumulate one task's metrics into accumulators
|
||||
def _accumulate_to(group: str, metrics: dict):
|
||||
# metrics expected to contain 'sum_rewards', 'max_rewards', 'successes', optionally 'video_paths'
|
||||
# but eval_one may store per-episode lists; we assume metrics uses scalars averaged per task as before.
|
||||
# To be robust, accept scalars or lists.
|
||||
def _append(key, value):
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, list):
|
||||
group_acc[group][key].extend(value)
|
||||
overall[key].extend(value)
|
||||
else:
|
||||
group_acc[group][key].append(value)
|
||||
overall[key].append(value)
|
||||
|
||||
_append("sum_rewards", metrics.get("sum_rewards"))
|
||||
_append("max_rewards", metrics.get("max_rewards"))
|
||||
_append("successes", metrics.get("successes"))
|
||||
# video_paths is list-like
|
||||
paths = metrics.get("video_paths", [])
|
||||
if paths:
|
||||
group_acc[group]["video_paths"].extend(paths)
|
||||
overall["video_paths"].extend(paths)
|
||||
|
||||
# Choose runner (sequential vs threaded)
|
||||
task_runner = partial(
|
||||
run_one,
|
||||
policy=policy,
|
||||
env_preprocessor=env_preprocessor,
|
||||
env_postprocessor=env_postprocessor,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
n_episodes=n_episodes,
|
||||
max_episodes_rendered=max_episodes_rendered,
|
||||
videos_dir=videos_dir,
|
||||
return_episode_data=return_episode_data,
|
||||
start_seed=start_seed,
|
||||
)
|
||||
|
||||
if max_parallel_tasks <= 1:
|
||||
# sequential path (single accumulator path on the main thread)
|
||||
# NOTE: keeping a single-threaded accumulator avoids concurrent list appends or locks
|
||||
for task_group, task_id, env in tasks:
|
||||
tg, tid, metrics = task_runner(task_group, task_id, env)
|
||||
_accumulate_to(tg, metrics)
|
||||
per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
|
||||
else:
|
||||
# threaded path: submit all tasks, consume completions on main thread and accumulate there
|
||||
with cf.ThreadPoolExecutor(max_workers=max_parallel_tasks) as executor:
|
||||
fut2meta = {}
|
||||
for task_group, task_id, env in tasks:
|
||||
fut = executor.submit(task_runner, task_group, task_id, env)
|
||||
fut2meta[fut] = (task_group, task_id)
|
||||
for fut in cf.as_completed(fut2meta):
|
||||
tg, tid, metrics = fut.result()
|
||||
_accumulate_to(tg, metrics)
|
||||
per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
|
||||
|
||||
# compute aggregated metrics helper (robust to lists/scalars)
|
||||
def _agg_from_list(xs):
|
||||
if not xs:
|
||||
return float("nan")
|
||||
arr = np.array(xs, dtype=float)
|
||||
return float(np.nanmean(arr))
|
||||
|
||||
# compute per-group aggregates
|
||||
groups_aggregated = {}
|
||||
for group, acc in group_acc.items():
|
||||
groups_aggregated[group] = {
|
||||
"avg_sum_reward": _agg_from_list(acc["sum_rewards"]),
|
||||
"avg_max_reward": _agg_from_list(acc["max_rewards"]),
|
||||
"pc_success": _agg_from_list(acc["successes"]) * 100 if acc["successes"] else float("nan"),
|
||||
"n_episodes": len(acc["sum_rewards"]),
|
||||
"video_paths": list(acc["video_paths"]),
|
||||
}
|
||||
|
||||
# overall aggregates
|
||||
overall_agg = {
|
||||
"avg_sum_reward": _agg_from_list(overall["sum_rewards"]),
|
||||
"avg_max_reward": _agg_from_list(overall["max_rewards"]),
|
||||
"pc_success": _agg_from_list(overall["successes"]) * 100 if overall["successes"] else float("nan"),
|
||||
"n_episodes": len(overall["sum_rewards"]),
|
||||
"eval_s": time.time() - start_t,
|
||||
"eval_ep_s": (time.time() - start_t) / max(1, len(overall["sum_rewards"])),
|
||||
"video_paths": list(overall["video_paths"]),
|
||||
}
|
||||
|
||||
return {
|
||||
"per_task": per_task_infos,
|
||||
"per_group": groups_aggregated,
|
||||
"overall": overall_agg,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
init_logging()
|
||||
register_third_party_plugins()
|
||||
eval_main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Helper to find the camera devices available in your system.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-find-cameras
|
||||
```
|
||||
"""
|
||||
|
||||
# NOTE(Steven): RealSense can also be identified/opened as OpenCV cameras. If you know the camera is a RealSense, use the `lerobot-find-cameras realsense` flag to avoid confusion.
|
||||
# NOTE(Steven): macOS cameras sometimes report different FPS at init time, not an issue here as we don't specify FPS when opening the cameras, but the information displayed might not be truthful.
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from lerobot.cameras.configs import ColorMode
|
||||
from lerobot.cameras.opencv.camera_opencv import OpenCVCamera
|
||||
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig
|
||||
from lerobot.cameras.realsense.camera_realsense import RealSenseCamera
|
||||
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def find_all_opencv_cameras() -> list[dict[str, Any]]:
|
||||
"""
|
||||
Finds all available OpenCV cameras plugged into the system.
|
||||
|
||||
Returns:
|
||||
A list of all available OpenCV cameras with their metadata.
|
||||
"""
|
||||
all_opencv_cameras_info: list[dict[str, Any]] = []
|
||||
logger.info("Searching for OpenCV cameras...")
|
||||
try:
|
||||
opencv_cameras = OpenCVCamera.find_cameras()
|
||||
for cam_info in opencv_cameras:
|
||||
all_opencv_cameras_info.append(cam_info)
|
||||
logger.info(f"Found {len(opencv_cameras)} OpenCV cameras.")
|
||||
except Exception as e:
|
||||
logger.error(f"Error finding OpenCV cameras: {e}")
|
||||
|
||||
return all_opencv_cameras_info
|
||||
|
||||
|
||||
def find_all_realsense_cameras() -> list[dict[str, Any]]:
|
||||
"""
|
||||
Finds all available RealSense cameras plugged into the system.
|
||||
|
||||
Returns:
|
||||
A list of all available RealSense cameras with their metadata.
|
||||
"""
|
||||
all_realsense_cameras_info: list[dict[str, Any]] = []
|
||||
logger.info("Searching for RealSense cameras...")
|
||||
try:
|
||||
realsense_cameras = RealSenseCamera.find_cameras()
|
||||
for cam_info in realsense_cameras:
|
||||
all_realsense_cameras_info.append(cam_info)
|
||||
logger.info(f"Found {len(realsense_cameras)} RealSense cameras.")
|
||||
except ImportError:
|
||||
logger.warning("Skipping RealSense camera search: pyrealsense2 library not found or not importable.")
|
||||
except Exception as e:
|
||||
logger.error(f"Error finding RealSense cameras: {e}")
|
||||
|
||||
return all_realsense_cameras_info
|
||||
|
||||
|
||||
def find_and_print_cameras(camera_type_filter: str | None = None) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Finds available cameras based on an optional filter and prints their information.
|
||||
|
||||
Args:
|
||||
camera_type_filter: Optional string to filter cameras ("realsense" or "opencv").
|
||||
If None, lists all cameras.
|
||||
|
||||
Returns:
|
||||
A list of all available cameras matching the filter, with their metadata.
|
||||
"""
|
||||
all_cameras_info: list[dict[str, Any]] = []
|
||||
|
||||
if camera_type_filter:
|
||||
camera_type_filter = camera_type_filter.lower()
|
||||
|
||||
if camera_type_filter is None or camera_type_filter == "opencv":
|
||||
all_cameras_info.extend(find_all_opencv_cameras())
|
||||
if camera_type_filter is None or camera_type_filter == "realsense":
|
||||
all_cameras_info.extend(find_all_realsense_cameras())
|
||||
|
||||
if not all_cameras_info:
|
||||
if camera_type_filter:
|
||||
logger.warning(f"No {camera_type_filter} cameras were detected.")
|
||||
else:
|
||||
logger.warning("No cameras (OpenCV or RealSense) were detected.")
|
||||
else:
|
||||
print("\n--- Detected Cameras ---")
|
||||
for i, cam_info in enumerate(all_cameras_info):
|
||||
print(f"Camera #{i}:")
|
||||
for key, value in cam_info.items():
|
||||
if key == "default_stream_profile" and isinstance(value, dict):
|
||||
print(f" {key.replace('_', ' ').capitalize()}:")
|
||||
for sub_key, sub_value in value.items():
|
||||
print(f" {sub_key.capitalize()}: {sub_value}")
|
||||
else:
|
||||
print(f" {key.replace('_', ' ').capitalize()}: {value}")
|
||||
print("-" * 20)
|
||||
return all_cameras_info
|
||||
|
||||
|
||||
def save_image(
|
||||
img_array: np.ndarray,
|
||||
camera_identifier: str | int,
|
||||
images_dir: Path,
|
||||
camera_type: str,
|
||||
):
|
||||
"""
|
||||
Saves a single image to disk using Pillow. Handles color conversion if necessary.
|
||||
"""
|
||||
try:
|
||||
img = Image.fromarray(img_array, mode="RGB")
|
||||
|
||||
safe_identifier = str(camera_identifier).replace("/", "_").replace("\\", "_")
|
||||
filename_prefix = f"{camera_type.lower()}_{safe_identifier}"
|
||||
filename = f"{filename_prefix}.png"
|
||||
|
||||
path = images_dir / filename
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
img.save(str(path))
|
||||
logger.info(f"Saved image: {path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save image for camera {camera_identifier} (type {camera_type}): {e}")
|
||||
|
||||
|
||||
def create_camera_instance(cam_meta: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Create and connect to a camera instance based on metadata."""
|
||||
cam_type = cam_meta.get("type")
|
||||
cam_id = cam_meta.get("id")
|
||||
instance = None
|
||||
|
||||
logger.info(f"Preparing {cam_type} ID {cam_id} with default profile")
|
||||
|
||||
try:
|
||||
if cam_type == "OpenCV":
|
||||
cv_config = OpenCVCameraConfig(
|
||||
index_or_path=cam_id,
|
||||
color_mode=ColorMode.RGB,
|
||||
)
|
||||
instance = OpenCVCamera(cv_config)
|
||||
elif cam_type == "RealSense":
|
||||
rs_config = RealSenseCameraConfig(
|
||||
serial_number_or_name=cam_id,
|
||||
color_mode=ColorMode.RGB,
|
||||
)
|
||||
instance = RealSenseCamera(rs_config)
|
||||
else:
|
||||
logger.warning(f"Unknown camera type: {cam_type} for ID {cam_id}. Skipping.")
|
||||
return None
|
||||
|
||||
if instance:
|
||||
logger.info(f"Connecting to {cam_type} camera: {cam_id}...")
|
||||
instance.connect(warmup=True)
|
||||
return {"instance": instance, "meta": cam_meta}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect or configure {cam_type} camera {cam_id}: {e}")
|
||||
if instance and instance.is_connected:
|
||||
instance.disconnect()
|
||||
return None
|
||||
|
||||
|
||||
def process_camera_image(
|
||||
cam_dict: dict[str, Any], output_dir: Path, current_time: float
|
||||
) -> concurrent.futures.Future | None:
|
||||
"""Capture and process an image from a single camera."""
|
||||
cam = cam_dict["instance"]
|
||||
meta = cam_dict["meta"]
|
||||
cam_type_str = str(meta.get("type", "unknown"))
|
||||
cam_id_str = str(meta.get("id", "unknown"))
|
||||
|
||||
try:
|
||||
image_data = cam.read()
|
||||
|
||||
return save_image(
|
||||
image_data,
|
||||
cam_id_str,
|
||||
output_dir,
|
||||
cam_type_str,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
f"Timeout reading from {cam_type_str} camera {cam_id_str} at time {current_time:.2f}s."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading from {cam_type_str} camera {cam_id_str}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_cameras(cameras_to_use: list[dict[str, Any]]):
|
||||
"""Disconnect all cameras."""
|
||||
logger.info(f"Disconnecting {len(cameras_to_use)} cameras...")
|
||||
for cam_dict in cameras_to_use:
|
||||
try:
|
||||
if cam_dict["instance"] and cam_dict["instance"].is_connected:
|
||||
cam_dict["instance"].disconnect()
|
||||
except Exception as e:
|
||||
logger.error(f"Error disconnecting camera {cam_dict['meta'].get('id')}: {e}")
|
||||
|
||||
|
||||
def save_images_from_all_cameras(
|
||||
output_dir: Path,
|
||||
record_time_s: float = 2.0,
|
||||
camera_type: str | None = None,
|
||||
):
|
||||
"""
|
||||
Connects to detected cameras (optionally filtered by type) and saves images from each.
|
||||
Uses default stream profiles for width, height, and FPS.
|
||||
|
||||
Args:
|
||||
output_dir: Directory to save images.
|
||||
record_time_s: Duration in seconds to record images.
|
||||
camera_type: Optional string to filter cameras ("realsense" or "opencv").
|
||||
If None, uses all detected cameras.
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Saving images to {output_dir}")
|
||||
all_camera_metadata = find_and_print_cameras(camera_type_filter=camera_type)
|
||||
|
||||
if not all_camera_metadata:
|
||||
logger.warning("No cameras detected matching the criteria. Cannot save images.")
|
||||
return
|
||||
|
||||
cameras_to_use = []
|
||||
for cam_meta in all_camera_metadata:
|
||||
camera_instance = create_camera_instance(cam_meta)
|
||||
if camera_instance:
|
||||
cameras_to_use.append(camera_instance)
|
||||
|
||||
if not cameras_to_use:
|
||||
logger.warning("No cameras could be connected. Aborting image save.")
|
||||
return
|
||||
|
||||
logger.info(f"Starting image capture for {record_time_s} seconds from {len(cameras_to_use)} cameras.")
|
||||
start_time = time.perf_counter()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(cameras_to_use) * 2) as executor:
|
||||
try:
|
||||
while time.perf_counter() - start_time < record_time_s:
|
||||
futures = []
|
||||
current_capture_time = time.perf_counter()
|
||||
|
||||
for cam_dict in cameras_to_use:
|
||||
future = process_camera_image(cam_dict, output_dir, current_capture_time)
|
||||
if future:
|
||||
futures.append(future)
|
||||
|
||||
if futures:
|
||||
concurrent.futures.wait(futures)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Capture interrupted by user.")
|
||||
finally:
|
||||
print("\nFinalizing image saving...")
|
||||
executor.shutdown(wait=True)
|
||||
cleanup_cameras(cameras_to_use)
|
||||
print(f"Image capture finished. Images saved to {output_dir}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Unified camera utility script for listing cameras and capturing images."
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"camera_type",
|
||||
type=str,
|
||||
nargs="?",
|
||||
default=None,
|
||||
choices=["realsense", "opencv"],
|
||||
help="Specify camera type to capture from (e.g., 'realsense', 'opencv'). Captures from all if omitted.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default="outputs/captured_images",
|
||||
help="Directory to save images. Default: outputs/captured_images",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--record-time-s",
|
||||
type=float,
|
||||
default=6.0,
|
||||
help="Time duration to attempt capturing frames. Default: 6 seconds.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
save_images_from_all_cameras(**vars(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Script to find joint limits and end-effector bounds via teleoperation.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-find-joint-limits \
|
||||
--robot.type=so100_follower \
|
||||
--robot.port=/dev/tty.usbmodem58760432981 \
|
||||
--robot.id=black \
|
||||
--teleop.type=so100_leader \
|
||||
--teleop.port=/dev/tty.usbmodem58760434471 \
|
||||
--teleop.id=blue \
|
||||
--urdf_path=<user>/SO-ARM100-main/Simulation/SO101/so101_new_calib.urdf \
|
||||
--target_frame_name=gripper \
|
||||
--teleop_time_s=30 \
|
||||
--warmup_time_s=5 \
|
||||
--control_loop_fps=30
|
||||
```
|
||||
"""
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import draccus
|
||||
import numpy as np
|
||||
|
||||
from lerobot.model.kinematics import RobotKinematics
|
||||
from lerobot.robots import ( # noqa: F401
|
||||
RobotConfig,
|
||||
bi_openarm_follower,
|
||||
bi_so_follower,
|
||||
koch_follower,
|
||||
make_robot_from_config,
|
||||
omx_follower,
|
||||
openarm_follower,
|
||||
so_follower,
|
||||
)
|
||||
from lerobot.teleoperators import ( # noqa: F401
|
||||
TeleoperatorConfig,
|
||||
bi_openarm_leader,
|
||||
bi_so_leader,
|
||||
gamepad,
|
||||
koch_leader,
|
||||
make_teleoperator_from_config,
|
||||
omx_leader,
|
||||
openarm_leader,
|
||||
so_leader,
|
||||
)
|
||||
from lerobot.utils.robot_utils import precise_sleep
|
||||
|
||||
|
||||
@dataclass
|
||||
class FindJointLimitsConfig:
|
||||
teleop: TeleoperatorConfig
|
||||
robot: RobotConfig
|
||||
|
||||
# Path to URDF file for kinematics
|
||||
# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo:
|
||||
# https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
|
||||
urdf_path: str
|
||||
target_frame_name: str = "gripper"
|
||||
|
||||
# Duration of the recording phase in seconds
|
||||
teleop_time_s: float = 30
|
||||
# Duration of the warmup phase in seconds
|
||||
warmup_time_s: float = 5
|
||||
# Control loop frequency
|
||||
control_loop_fps: int = 30
|
||||
|
||||
|
||||
@draccus.wrap()
|
||||
def find_joint_and_ee_bounds(cfg: FindJointLimitsConfig):
|
||||
teleop = make_teleoperator_from_config(cfg.teleop)
|
||||
robot = make_robot_from_config(cfg.robot)
|
||||
|
||||
print(f"Connecting to robot: {cfg.robot.type}...")
|
||||
teleop.connect()
|
||||
robot.connect()
|
||||
print("Devices connected.")
|
||||
|
||||
# Initialize Kinematics
|
||||
try:
|
||||
kinematics = RobotKinematics(cfg.urdf_path, cfg.target_frame_name)
|
||||
except Exception as e:
|
||||
print(f"Error initializing kinematics: {e}")
|
||||
print("Ensure URDF path and target frame name are correct.")
|
||||
robot.disconnect()
|
||||
teleop.disconnect()
|
||||
return
|
||||
|
||||
# Initialize variables
|
||||
max_pos = None
|
||||
min_pos = None
|
||||
max_ee = None
|
||||
min_ee = None
|
||||
|
||||
start_t = time.perf_counter()
|
||||
warmup_done = False
|
||||
|
||||
print("\n" + "=" * 40)
|
||||
print(f" WARMUP PHASE ({cfg.warmup_time_s}s)")
|
||||
print(" Move the robot freely to ensure control works.")
|
||||
print(" Data is NOT being recorded yet.")
|
||||
print("=" * 40 + "\n")
|
||||
|
||||
try:
|
||||
while True:
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# 1. Teleoperation Control Loop
|
||||
action = teleop.get_action()
|
||||
robot.send_action(action)
|
||||
|
||||
# 2. Read Observations
|
||||
observation = robot.get_observation()
|
||||
joint_positions = np.array([observation[f"{key}.pos"] for key in robot.bus.motors])
|
||||
|
||||
# 3. Calculate Kinematics
|
||||
# Forward kinematics to get (x, y, z) translation
|
||||
ee_pos = kinematics.forward_kinematics(joint_positions)[:3, 3]
|
||||
|
||||
current_time = time.perf_counter()
|
||||
elapsed = current_time - start_t
|
||||
|
||||
# 4. Handle Phases
|
||||
if elapsed < cfg.warmup_time_s:
|
||||
# Still in warmup
|
||||
pass
|
||||
|
||||
else:
|
||||
# Phase Transition: Warmup -> Recording
|
||||
if not warmup_done:
|
||||
print("\n" + "=" * 40)
|
||||
print(" RECORDING STARTED")
|
||||
print(" Move robot to ALL joint limits.")
|
||||
print(" Press Ctrl+C to stop early and save results.")
|
||||
print("=" * 40 + "\n")
|
||||
|
||||
# Initialize limits with current position at start of recording
|
||||
max_pos = joint_positions.copy()
|
||||
min_pos = joint_positions.copy()
|
||||
max_ee = ee_pos.copy()
|
||||
min_ee = ee_pos.copy()
|
||||
warmup_done = True
|
||||
|
||||
# Update Limits
|
||||
max_ee = np.maximum(max_ee, ee_pos)
|
||||
min_ee = np.minimum(min_ee, ee_pos)
|
||||
max_pos = np.maximum(max_pos, joint_positions)
|
||||
min_pos = np.minimum(min_pos, joint_positions)
|
||||
|
||||
# Time check
|
||||
recording_time = elapsed - cfg.warmup_time_s
|
||||
remaining = cfg.teleop_time_s - recording_time
|
||||
|
||||
# Simple throttle for print statements (every ~1 sec)
|
||||
if int(recording_time * 100) % 100 == 0:
|
||||
print(f"Time remaining: {remaining:.1f}s", end="\r")
|
||||
|
||||
if recording_time > cfg.teleop_time_s:
|
||||
print("\nTime limit reached.")
|
||||
break
|
||||
|
||||
precise_sleep(max(1.0 / cfg.control_loop_fps - (time.perf_counter() - t0), 0.0))
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nInterrupted by user. Stopping safely...")
|
||||
|
||||
finally:
|
||||
# Safety: Disconnect devices
|
||||
print("\nDisconnecting devices...")
|
||||
robot.disconnect()
|
||||
teleop.disconnect()
|
||||
|
||||
# Results Output
|
||||
if max_pos is not None:
|
||||
print("\n" + "=" * 40)
|
||||
print("FINAL RESULTS")
|
||||
print("=" * 40)
|
||||
|
||||
# Rounding for readability
|
||||
r_max_ee = np.round(max_ee, 4).tolist()
|
||||
r_min_ee = np.round(min_ee, 4).tolist()
|
||||
r_max_pos = np.round(max_pos, 4).tolist()
|
||||
r_min_pos = np.round(min_pos, 4).tolist()
|
||||
|
||||
print("\n# End Effector Bounds (x, y, z):")
|
||||
print(f"max_ee = {r_max_ee}")
|
||||
print(f"min_ee = {r_min_ee}")
|
||||
|
||||
print("\n# Joint Position Limits (radians):")
|
||||
print(f"max_pos = {r_max_pos}")
|
||||
print(f"min_pos = {r_min_pos}")
|
||||
|
||||
else:
|
||||
print("No data recorded (exited during warmup).")
|
||||
|
||||
|
||||
def main():
|
||||
find_joint_and_ee_bounds()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,69 @@
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Helper to find the USB port associated with your MotorsBus.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-find-port
|
||||
```
|
||||
"""
|
||||
|
||||
import platform
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def find_available_ports():
|
||||
from serial.tools import list_ports # Part of pyserial library
|
||||
|
||||
if platform.system() == "Windows":
|
||||
# List COM ports using pyserial
|
||||
ports = [port.device for port in list_ports.comports()]
|
||||
else: # Linux/macOS
|
||||
# List /dev/tty* ports for Unix-based systems
|
||||
ports = [str(path) for path in Path("/dev").glob("tty*")]
|
||||
return ports
|
||||
|
||||
|
||||
def find_port():
|
||||
print("Finding all available ports for the MotorsBus.")
|
||||
ports_before = find_available_ports()
|
||||
print("Ports before disconnecting:", ports_before)
|
||||
|
||||
print("Remove the USB cable from your MotorsBus and press Enter when done.")
|
||||
input() # Wait for user to disconnect the device
|
||||
|
||||
time.sleep(0.5) # Allow some time for port to be released
|
||||
ports_after = find_available_ports()
|
||||
ports_diff = list(set(ports_before) - set(ports_after))
|
||||
|
||||
if len(ports_diff) == 1:
|
||||
port = ports_diff[0]
|
||||
print(f"The port of this MotorsBus is '{port}'")
|
||||
print("Reconnect the USB cable.")
|
||||
elif len(ports_diff) == 0:
|
||||
raise OSError(f"Could not detect the port. No difference was found ({ports_diff}).")
|
||||
else:
|
||||
raise OSError(f"Could not detect the port. More than one port was found ({ports_diff}).")
|
||||
|
||||
|
||||
def main():
|
||||
find_port()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Records with policy execution and teleop-device action mirroring enabled.
|
||||
|
||||
Phase A/B goals:
|
||||
- Same policy action is executed on the follower robot and mirrored to the teleop arm.
|
||||
- Keyboard-toggled intervention lets teleop temporarily take over execution.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.scripts.lerobot_record import RecordConfig, record
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.constants import HF_LEROBOT_HOME
|
||||
from lerobot.utils.recording_annotations import EPISODE_FAILURE, infer_collector_policy_version
|
||||
|
||||
|
||||
def _default_failure_reset_pose_path(cfg: RecordConfig) -> Path:
|
||||
robot_id = cfg.robot.id if cfg.robot.id else "default"
|
||||
robot_type = cfg.robot.type if hasattr(cfg.robot, "type") else type(cfg.robot).__name__
|
||||
return HF_LEROBOT_HOME / "failure_reset_pose" / f"{robot_type}_{robot_id}.json"
|
||||
|
||||
|
||||
def _extract_joint_pos_from_observation(observation: dict[str, Any]) -> dict[str, float]:
|
||||
return {key: float(value) for key, value in observation.items() if key.endswith(".pos")}
|
||||
|
||||
|
||||
def _save_failure_reset_pose(robot: Any, pose_path: Path) -> dict[str, float]:
|
||||
observation = robot.get_observation()
|
||||
joint_pos = _extract_joint_pos_from_observation(observation)
|
||||
if not joint_pos:
|
||||
raise ValueError("Could not capture failure reset pose: no '.pos' joints found in observation.")
|
||||
|
||||
pose_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"robot_type": robot.robot_type, "joint_pos": joint_pos}
|
||||
with open(pose_path, "w") as f:
|
||||
json.dump(payload, f, indent=2, sort_keys=True)
|
||||
logging.info("Saved failure_reset_pose to %s", pose_path)
|
||||
return joint_pos
|
||||
|
||||
|
||||
def _load_failure_reset_pose(pose_path: Path) -> dict[str, float]:
|
||||
with open(pose_path) as f:
|
||||
payload = json.load(f)
|
||||
joint_pos_raw = payload["joint_pos"] if isinstance(payload, dict) and "joint_pos" in payload else payload
|
||||
if not isinstance(joint_pos_raw, dict):
|
||||
raise ValueError(f"Invalid failure reset pose payload in {pose_path}: expected dict, got {type(joint_pos_raw)}")
|
||||
joint_pos = {str(key): float(value) for key, value in joint_pos_raw.items() if str(key).endswith(".pos")}
|
||||
if not joint_pos:
|
||||
raise ValueError(f"Invalid failure reset pose payload in {pose_path}: no '.pos' joints found.")
|
||||
logging.info("Loaded failure_reset_pose from %s", pose_path)
|
||||
return joint_pos
|
||||
|
||||
|
||||
def _slow_reset_all_arms_to_pose(
|
||||
robot: Any,
|
||||
teleop: Any,
|
||||
target_pose: dict[str, float],
|
||||
duration_s: float = 3.0,
|
||||
) -> None:
|
||||
joint_keys = [key for key in robot.action_features if key.endswith(".pos") and key in target_pose]
|
||||
if not joint_keys:
|
||||
logging.warning("No matching '.pos' joints found for failure reset pose.")
|
||||
return
|
||||
|
||||
current_pose = _extract_joint_pos_from_observation(robot.get_observation())
|
||||
start_pose = {key: current_pose.get(key, float(target_pose[key])) for key in joint_keys}
|
||||
goal_pose = {key: float(target_pose[key]) for key in joint_keys}
|
||||
|
||||
if teleop is not None and not isinstance(teleop, list) and hasattr(teleop, "set_manual_control"):
|
||||
teleop.set_manual_control(False)
|
||||
|
||||
step_dt_s = 0.05
|
||||
steps = max(int(duration_s / step_dt_s), 1)
|
||||
for idx in range(1, steps + 1):
|
||||
alpha = idx / steps
|
||||
action = {key: start_pose[key] + (goal_pose[key] - start_pose[key]) * alpha for key in joint_keys}
|
||||
robot.send_action(action)
|
||||
if teleop is not None and not isinstance(teleop, list):
|
||||
teleop.send_feedback(action)
|
||||
time.sleep(step_dt_s)
|
||||
|
||||
logging.info("Episode marked as failure. Arms returned to failure reset pose in %.1fs.", duration_s)
|
||||
|
||||
|
||||
class _HumanInloopFailureResetController:
|
||||
def __init__(self, cfg: RecordConfig):
|
||||
self.pose_path = _default_failure_reset_pose_path(cfg)
|
||||
self.failure_reset_pose: dict[str, float] | None = None
|
||||
|
||||
def on_record_connected(self, robot: Any, teleop: Any) -> None:
|
||||
if self.pose_path.is_file():
|
||||
self.failure_reset_pose = _load_failure_reset_pose(self.pose_path)
|
||||
return
|
||||
|
||||
input(
|
||||
"Human-inloop with policy detected.\n"
|
||||
"Please ensure ALL robot arms are at reset position, then press ENTER to capture:\n"
|
||||
f"{self.pose_path}\n"
|
||||
)
|
||||
self.failure_reset_pose = _save_failure_reset_pose(robot=robot, pose_path=self.pose_path)
|
||||
|
||||
def on_episode_outcome(self, robot: Any, teleop: Any, episode_success: str | None) -> None:
|
||||
if episode_success == EPISODE_FAILURE and self.failure_reset_pose is not None:
|
||||
_slow_reset_all_arms_to_pose(robot=robot, teleop=teleop, target_pose=self.failure_reset_pose)
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def human_inloop_record(cfg: RecordConfig):
|
||||
if cfg.teleop is None:
|
||||
raise ValueError("`lerobot-human-inloop-record` requires `teleop` config.")
|
||||
|
||||
cfg.policy_sync_to_teleop = cfg.policy is not None
|
||||
cfg.intervention_state_machine_enabled = cfg.policy is not None
|
||||
cfg.enable_episode_outcome_labeling = True
|
||||
cfg.default_episode_success = "failure"
|
||||
cfg.enable_collector_policy_id = True
|
||||
if cfg.collector_policy_id_policy is None:
|
||||
cfg.collector_policy_id_policy = infer_collector_policy_version(cfg.policy)
|
||||
if cfg.policy is not None:
|
||||
failure_reset_controller = _HumanInloopFailureResetController(cfg)
|
||||
cfg._on_record_connected = failure_reset_controller.on_record_connected
|
||||
cfg._on_record_episode_outcome = failure_reset_controller.on_episode_outcome
|
||||
|
||||
logging.info(
|
||||
"Human-in-loop recording is enabled. Press '%s' to toggle takeover. "
|
||||
"Press '%s' to mark success and end, '%s' to mark failure and end. "
|
||||
"Recorded `action` is the executed action. "
|
||||
"Policy output (when policy is enabled) is stored in `complementary_info.policy_action`. "
|
||||
"Collector source is stored in `complementary_info.collector_policy_id`. "
|
||||
"ACP inference: enable=%s use_cfg=%s cfg_beta=%.3f.",
|
||||
cfg.intervention_toggle_key,
|
||||
cfg.episode_success_key,
|
||||
cfg.episode_failure_key,
|
||||
cfg.acp_inference.enable,
|
||||
cfg.acp_inference.use_cfg,
|
||||
cfg.acp_inference.cfg_beta,
|
||||
)
|
||||
return record(cfg)
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
human_inloop_record()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Visualize effects of image transforms for a given configuration.
|
||||
|
||||
This script will generate examples of transformed images as they are output by LeRobot dataset.
|
||||
Additionally, each individual transform can be visualized separately as well as examples of combined transforms
|
||||
|
||||
Example:
|
||||
```bash
|
||||
lerobot-imgtransform-viz \
|
||||
--repo_id=lerobot/pusht \
|
||||
--episodes='[0]' \
|
||||
--image_transforms.enable=True
|
||||
```
|
||||
"""
|
||||
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import draccus
|
||||
from torchvision.transforms import ToPILImage
|
||||
|
||||
from lerobot.configs.default import DatasetConfig
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.datasets.transforms import (
|
||||
ImageTransforms,
|
||||
ImageTransformsConfig,
|
||||
make_transform_from_config,
|
||||
)
|
||||
|
||||
OUTPUT_DIR = Path("outputs/image_transforms")
|
||||
to_pil = ToPILImage()
|
||||
|
||||
|
||||
def save_all_transforms(cfg: ImageTransformsConfig, original_frame, output_dir, n_examples):
|
||||
output_dir_all = output_dir / "all"
|
||||
output_dir_all.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tfs = ImageTransforms(cfg)
|
||||
for i in range(1, n_examples + 1):
|
||||
transformed_frame = tfs(original_frame)
|
||||
to_pil(transformed_frame).save(output_dir_all / f"{i}.png", quality=100)
|
||||
|
||||
print("Combined transforms examples saved to:")
|
||||
print(f" {output_dir_all}")
|
||||
|
||||
|
||||
def save_each_transform(cfg: ImageTransformsConfig, original_frame, output_dir, n_examples):
|
||||
if not cfg.enable:
|
||||
logging.warning(
|
||||
"No single transforms will be saved, because `image_transforms.enable=False`. To enable, set `enable` to True in `ImageTransformsConfig` or in the command line with `--image_transforms.enable=True`."
|
||||
)
|
||||
return
|
||||
|
||||
print("Individual transforms examples saved to:")
|
||||
for tf_name, tf_cfg in cfg.tfs.items():
|
||||
# Apply a few transformation with random value in min_max range
|
||||
output_dir_single = output_dir / tf_name
|
||||
output_dir_single.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tf = make_transform_from_config(tf_cfg)
|
||||
for i in range(1, n_examples + 1):
|
||||
transformed_frame = tf(original_frame)
|
||||
to_pil(transformed_frame).save(output_dir_single / f"{i}.png", quality=100)
|
||||
|
||||
# Apply min, max, average transformations
|
||||
tf_cfg_kwgs_min = deepcopy(tf_cfg.kwargs)
|
||||
tf_cfg_kwgs_max = deepcopy(tf_cfg.kwargs)
|
||||
tf_cfg_kwgs_avg = deepcopy(tf_cfg.kwargs)
|
||||
|
||||
for key, (min_, max_) in tf_cfg.kwargs.items():
|
||||
avg = (min_ + max_) / 2
|
||||
tf_cfg_kwgs_min[key] = [min_, min_]
|
||||
tf_cfg_kwgs_max[key] = [max_, max_]
|
||||
tf_cfg_kwgs_avg[key] = [avg, avg]
|
||||
|
||||
tf_min = make_transform_from_config(replace(tf_cfg, **{"kwargs": tf_cfg_kwgs_min}))
|
||||
tf_max = make_transform_from_config(replace(tf_cfg, **{"kwargs": tf_cfg_kwgs_max}))
|
||||
tf_avg = make_transform_from_config(replace(tf_cfg, **{"kwargs": tf_cfg_kwgs_avg}))
|
||||
|
||||
tf_frame_min = tf_min(original_frame)
|
||||
tf_frame_max = tf_max(original_frame)
|
||||
tf_frame_avg = tf_avg(original_frame)
|
||||
|
||||
to_pil(tf_frame_min).save(output_dir_single / "min.png", quality=100)
|
||||
to_pil(tf_frame_max).save(output_dir_single / "max.png", quality=100)
|
||||
to_pil(tf_frame_avg).save(output_dir_single / "mean.png", quality=100)
|
||||
|
||||
print(f" {output_dir_single}")
|
||||
|
||||
|
||||
@draccus.wrap()
|
||||
def visualize_image_transforms(cfg: DatasetConfig, output_dir: Path = OUTPUT_DIR, n_examples: int = 5):
|
||||
dataset = LeRobotDataset(
|
||||
repo_id=cfg.repo_id,
|
||||
episodes=cfg.episodes,
|
||||
revision=cfg.revision,
|
||||
video_backend=cfg.video_backend,
|
||||
)
|
||||
|
||||
output_dir = output_dir / cfg.repo_id.split("/")[-1]
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get 1st frame from 1st camera of 1st episode
|
||||
original_frame = dataset[0][dataset.meta.camera_keys[0]]
|
||||
to_pil(original_frame).save(output_dir / "original_frame.png", quality=100)
|
||||
print("\nOriginal frame saved to:")
|
||||
print(f" {output_dir / 'original_frame.png'}.")
|
||||
|
||||
save_all_transforms(cfg.image_transforms, original_frame, output_dir, n_examples)
|
||||
save_each_transform(cfg.image_transforms, original_frame, output_dir, n_examples)
|
||||
|
||||
|
||||
def main():
|
||||
visualize_image_transforms()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
126
src/lerobot/scripts/lerobot_info.py
普通文件
126
src/lerobot/scripts/lerobot_info.py
普通文件
@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Use this script to get a quick summary of your system config.
|
||||
It should be able to run without any of LeRobot's dependencies or LeRobot itself installed.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-info
|
||||
```
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
from importlib.metadata import PackageNotFoundError, distribution
|
||||
|
||||
PACKAGE_NAME = "lerobot"
|
||||
|
||||
|
||||
def get_ffmpeg_version() -> str:
|
||||
"""Get the ffmpeg version if installed, otherwise return 'N/A'."""
|
||||
command_path = shutil.which("ffmpeg")
|
||||
if command_path is None:
|
||||
return "N/A"
|
||||
try:
|
||||
result = subprocess.run([command_path, "-version"], capture_output=True, text=True, check=True)
|
||||
first_line = result.stdout.splitlines()[0]
|
||||
version_info = first_line.split(" ")[2]
|
||||
return version_info
|
||||
except (subprocess.SubprocessError, IndexError):
|
||||
return "Installed (version parsing failed)"
|
||||
|
||||
|
||||
def get_package_version(package_name: str) -> str:
|
||||
"""Get the version of a package if it exists, otherwise return 'N/A'."""
|
||||
try:
|
||||
module = importlib.import_module(package_name)
|
||||
return getattr(module, "__version__", "Installed (version not found)")
|
||||
except ImportError:
|
||||
return "N/A"
|
||||
|
||||
|
||||
def get_sys_info() -> dict[str, str]:
|
||||
"""Run this to get basic system info to help for tracking issues & bugs."""
|
||||
# General package versions
|
||||
info = {
|
||||
"LeRobot version": get_package_version(PACKAGE_NAME),
|
||||
"Platform": platform.platform(),
|
||||
"Python version": platform.python_version(),
|
||||
"Huggingface Hub version": get_package_version("huggingface_hub"),
|
||||
"Datasets version": get_package_version("datasets"),
|
||||
"Numpy version": get_package_version("numpy"),
|
||||
"FFmpeg version": get_ffmpeg_version(),
|
||||
}
|
||||
|
||||
# PyTorch and GPU specific information
|
||||
torch_version = "N/A"
|
||||
torch_cuda_available = "N/A"
|
||||
cuda_version = "N/A"
|
||||
gpu_model = "N/A"
|
||||
try:
|
||||
import torch
|
||||
|
||||
torch_version = str(torch.__version__)
|
||||
torch_cuda_available = torch.cuda.is_available()
|
||||
if torch_cuda_available:
|
||||
cuda_version = str(torch.version.cuda)
|
||||
# Gets the name of the first available GPU
|
||||
gpu_model = torch.cuda.get_device_name(0)
|
||||
except ImportError:
|
||||
# If torch is not installed, the default "N/A" values will be used.
|
||||
pass
|
||||
|
||||
info.update(
|
||||
{
|
||||
"PyTorch version": torch_version,
|
||||
"Is PyTorch built with CUDA support?": str(torch_cuda_available),
|
||||
"Cuda version": cuda_version,
|
||||
"GPU model": gpu_model,
|
||||
"Using GPU in script?": "<fill in>",
|
||||
}
|
||||
)
|
||||
scripts = "N/A"
|
||||
try:
|
||||
dist = distribution(PACKAGE_NAME)
|
||||
scripts = [ep.name for ep in dist.entry_points if ep.group == "console_scripts"]
|
||||
except PackageNotFoundError:
|
||||
pass
|
||||
|
||||
info.update({f"{PACKAGE_NAME} scripts": str(scripts)})
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def format_dict_for_markdown(d: dict[str, str]) -> str:
|
||||
"""Formats a dictionary into a markdown-friendly bulleted list."""
|
||||
return "\n".join([f"- {prop}: {val}" for prop, val in d.items()])
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to print system info in markdown format.
|
||||
"""
|
||||
system_info = get_sys_info()
|
||||
print(format_dict_for_markdown(system_info))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,568 @@
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Records a dataset. Actions for the robot can be either generated by teleoperation or by a policy.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-record \
|
||||
--robot.type=so100_follower \
|
||||
--robot.port=/dev/tty.usbmodem58760431541 \
|
||||
--robot.cameras="{laptop: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \
|
||||
--robot.id=black \
|
||||
--dataset.repo_id=<my_username>/<my_dataset_name> \
|
||||
--dataset.num_episodes=2 \
|
||||
--dataset.single_task="Grab the cube" \
|
||||
--display_data=true
|
||||
# <- Optional: specify video codec (h264, hevc, libsvtav1). Default is libsvtav1. \
|
||||
# --dataset.vcodec=h264 \
|
||||
# <- Teleop optional if you want to teleoperate to record or in between episodes with a policy \
|
||||
# --teleop.type=so100_leader \
|
||||
# --teleop.port=/dev/tty.usbmodem58760431551 \
|
||||
# --teleop.id=blue \
|
||||
# <- Policy optional if you want to record with a policy \
|
||||
# --policy.path=${HF_USER}/my_policy \
|
||||
```
|
||||
|
||||
Example recording with bimanual so100:
|
||||
```shell
|
||||
lerobot-record \
|
||||
--robot.type=bi_so_follower \
|
||||
--robot.left_arm_config.port=/dev/tty.usbmodem5A460822851 \
|
||||
--robot.right_arm_config.port=/dev/tty.usbmodem5A460814411 \
|
||||
--robot.id=bimanual_follower \
|
||||
--robot.left_arm_config.cameras='{
|
||||
wrist: {"type": "opencv", "index_or_path": 1, "width": 640, "height": 480, "fps": 30},
|
||||
top: {"type": "opencv", "index_or_path": 3, "width": 640, "height": 480, "fps": 30},
|
||||
}' --robot.right_arm_config.cameras='{
|
||||
wrist: {"type": "opencv", "index_or_path": 2, "width": 640, "height": 480, "fps": 30},
|
||||
front: {"type": "opencv", "index_or_path": 4, "width": 640, "height": 480, "fps": 30},
|
||||
}' \
|
||||
--teleop.type=bi_so_leader \
|
||||
--teleop.left_arm_config.port=/dev/tty.usbmodem5A460852721 \
|
||||
--teleop.right_arm_config.port=/dev/tty.usbmodem5A460819811 \
|
||||
--teleop.id=bimanual_leader \
|
||||
--display_data=true \
|
||||
--dataset.repo_id=${HF_USER}/bimanual-so-handover-cube \
|
||||
--dataset.num_episodes=25 \
|
||||
--dataset.single_task="Grab and handover the red cube to the other arm"
|
||||
```
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
|
||||
from lerobot.cameras import ( # noqa: F401
|
||||
CameraConfig, # noqa: F401
|
||||
)
|
||||
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401
|
||||
from lerobot.cameras.reachy2_camera.configuration_reachy2_camera import Reachy2CameraConfig # noqa: F401
|
||||
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401
|
||||
from lerobot.cameras.zmq.configuration_zmq import ZMQCameraConfig # noqa: F401
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.datasets.pipeline_features import aggregate_pipeline_dataset_features, create_initial_features
|
||||
from lerobot.datasets.utils import combine_feature_dicts
|
||||
from lerobot.datasets.video_utils import VideoEncodingManager
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.processor import make_default_processors
|
||||
from lerobot.processor.rename_processor import rename_stats
|
||||
from lerobot.robots import ( # noqa: F401
|
||||
RobotConfig,
|
||||
bi_openarm_follower,
|
||||
bi_so_follower,
|
||||
earthrover_mini_plus,
|
||||
hope_jr,
|
||||
koch_follower,
|
||||
make_robot_from_config,
|
||||
omx_follower,
|
||||
openarm_follower,
|
||||
reachy2,
|
||||
so_follower,
|
||||
unitree_g1 as unitree_g1_robot,
|
||||
)
|
||||
from lerobot.teleoperators import ( # noqa: F401
|
||||
TeleoperatorConfig,
|
||||
bi_openarm_leader,
|
||||
bi_so_leader,
|
||||
homunculus,
|
||||
koch_leader,
|
||||
make_teleoperator_from_config,
|
||||
omx_leader,
|
||||
openarm_leader,
|
||||
reachy2_teleoperator,
|
||||
so_leader,
|
||||
unitree_g1,
|
||||
)
|
||||
from lerobot.scripts.recording_hil import (
|
||||
ACPInferenceConfig,
|
||||
PolicySyncDualArmExecutor,
|
||||
_capture_policy_runtime_state, # noqa: F401
|
||||
_predict_policy_action_with_acp_inference, # noqa: F401
|
||||
)
|
||||
from lerobot.scripts.recording_loop import record_loop
|
||||
from lerobot.utils.constants import ACTION
|
||||
from lerobot.utils.control_utils import (
|
||||
init_keyboard_listener,
|
||||
is_headless,
|
||||
sanity_check_dataset_name,
|
||||
sanity_check_dataset_robot_compatibility,
|
||||
)
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.recording_annotations import (
|
||||
infer_collector_policy_id,
|
||||
normalize_episode_success_label,
|
||||
resolve_episode_success_label,
|
||||
)
|
||||
from lerobot.utils.utils import (
|
||||
init_logging,
|
||||
log_say,
|
||||
)
|
||||
from lerobot.utils.visualization_utils import init_rerun
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasetRecordConfig:
|
||||
# Dataset identifier. By convention it should match '{hf_username}/{dataset_name}' (e.g. `lerobot/test`).
|
||||
repo_id: str
|
||||
# A short but accurate description of the task performed during the recording (e.g. "Pick the Lego block and drop it in the box on the right.")
|
||||
single_task: str
|
||||
# Root directory where the dataset will be stored (e.g. 'dataset/path').
|
||||
root: str | Path | None = None
|
||||
# Limit the frames per second.
|
||||
fps: int = 30
|
||||
# Number of seconds for data recording for each episode.
|
||||
episode_time_s: int | float = 60
|
||||
# Number of seconds for resetting the environment after each episode.
|
||||
reset_time_s: int | float = 60
|
||||
# Number of episodes to record.
|
||||
num_episodes: int = 50
|
||||
# Encode frames in the dataset into video
|
||||
video: bool = True
|
||||
# Upload dataset to Hugging Face hub.
|
||||
push_to_hub: bool = True
|
||||
# Upload on private repository on the Hugging Face hub.
|
||||
private: bool = False
|
||||
# Add tags to your dataset on the hub.
|
||||
tags: list[str] | None = None
|
||||
# Number of subprocesses handling the saving of frames as PNG. Set to 0 to use threads only;
|
||||
# set to ≥1 to use subprocesses, each using threads to write images. The best number of processes
|
||||
# and threads depends on your system. We recommend 4 threads per camera with 0 processes.
|
||||
# If fps is unstable, adjust the thread count. If still unstable, try using 1 or more subprocesses.
|
||||
num_image_writer_processes: int = 0
|
||||
# Number of threads writing the frames as png images on disk, per camera.
|
||||
# Too many threads might cause unstable teleoperation fps due to main thread being blocked.
|
||||
# Not enough threads might cause low camera fps.
|
||||
num_image_writer_threads_per_camera: int = 4
|
||||
# Number of episodes to record before batch encoding videos
|
||||
# Set to 1 for immediate encoding (default behavior), or higher for batched encoding
|
||||
video_encoding_batch_size: int = 1
|
||||
# Video codec for encoding videos. Options: 'h264', 'hevc', 'libsvtav1'.
|
||||
# Use 'h264' for faster encoding on systems where AV1 encoding is CPU-heavy.
|
||||
vcodec: str = "libsvtav1"
|
||||
# Rename map for the observation to override the image and state keys
|
||||
rename_map: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.single_task is None:
|
||||
raise ValueError("You need to provide a task as argument in `single_task`.")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordConfig:
|
||||
robot: RobotConfig
|
||||
dataset: DatasetRecordConfig
|
||||
# Whether to control the robot with a teleoperator
|
||||
teleop: TeleoperatorConfig | None = None
|
||||
# Whether to control the robot with a policy
|
||||
policy: PreTrainedConfig | None = None
|
||||
# Display all cameras on screen
|
||||
display_data: bool = False
|
||||
# Display data on a remote Rerun server
|
||||
display_ip: str | None = None
|
||||
# Port of the remote Rerun server
|
||||
display_port: int | None = None
|
||||
# Whether to display compressed images in Rerun
|
||||
display_compressed_images: bool = False
|
||||
# Use vocal synthesis to read events.
|
||||
play_sounds: bool = True
|
||||
# Resume recording on an existing dataset.
|
||||
resume: bool = False
|
||||
# In policy mode, broadcast the same robot action to the teleop arm via `teleop.send_feedback`.
|
||||
policy_sync_to_teleop: bool = False
|
||||
# Use parallel dispatch to reduce action broadcast latency when syncing policy to teleop.
|
||||
policy_sync_parallel: bool = True
|
||||
# Enable S0/S1/S2 intervention state machine when policy + teleop are both available.
|
||||
intervention_state_machine_enabled: bool = True
|
||||
# Keyboard key used to toggle entering/leaving intervention.
|
||||
intervention_toggle_key: str = "i"
|
||||
# Whether to capture episode-level success/failure labels from keyboard.
|
||||
enable_episode_outcome_labeling: bool = False
|
||||
# Keyboard key to mark the current episode as success and end it.
|
||||
episode_success_key: str = "s"
|
||||
# Keyboard key to mark the current episode as failure and end it.
|
||||
episode_failure_key: str = "f"
|
||||
# Optional fallback label used when no explicit success/failure key was pressed.
|
||||
default_episode_success: str | None = None
|
||||
# If True, require explicit or default episode labels before saving.
|
||||
require_episode_success_label: bool = False
|
||||
# Whether to store step-level collector policy provenance.
|
||||
enable_collector_policy_id: bool = False
|
||||
# Policy identifier used when action source is policy. If omitted, inferred from policy config.
|
||||
collector_policy_id_policy: str | None = None
|
||||
# Policy identifier used when action source is human/teleop.
|
||||
collector_policy_id_human: str = "human"
|
||||
# ACP inference controls for policy-driven recording.
|
||||
acp_inference: ACPInferenceConfig = field(default_factory=ACPInferenceConfig)
|
||||
# Retry timeout for transient communication errors (seconds). Set to 0 to fail immediately.
|
||||
communication_retry_timeout_s: float = 2.0
|
||||
# Sleep interval between communication retries (seconds).
|
||||
communication_retry_interval_s: float = 0.1
|
||||
|
||||
def __post_init__(self):
|
||||
# HACK: We parse again the cli args here to get the pretrained path if there was one.
|
||||
policy_path = parser.get_path_arg("policy")
|
||||
|
||||
if policy_path:
|
||||
cli_overrides = parser.get_cli_overrides("policy")
|
||||
|
||||
self.policy = PreTrainedConfig.from_pretrained(policy_path, cli_overrides=cli_overrides)
|
||||
self.policy.pretrained_path = policy_path
|
||||
|
||||
if self.teleop is None and self.policy is None:
|
||||
raise ValueError("Choose a policy, a teleoperator or both to control the robot")
|
||||
if not self.intervention_toggle_key or len(self.intervention_toggle_key) != 1:
|
||||
raise ValueError("`intervention_toggle_key` must be a single character.")
|
||||
|
||||
if self.enable_episode_outcome_labeling:
|
||||
label_key_bindings = {
|
||||
"episode_success_key": self.episode_success_key,
|
||||
"episode_failure_key": self.episode_failure_key,
|
||||
}
|
||||
for key_name, key_value in label_key_bindings.items():
|
||||
if not key_value or len(key_value) != 1:
|
||||
raise ValueError(f"`{key_name}` must be a single character.")
|
||||
|
||||
normalized_keys = [
|
||||
self.intervention_toggle_key.lower(),
|
||||
self.episode_success_key.lower(),
|
||||
self.episode_failure_key.lower(),
|
||||
]
|
||||
if len(set(normalized_keys)) != len(normalized_keys):
|
||||
raise ValueError(
|
||||
"`intervention_toggle_key`, `episode_success_key`, and `episode_failure_key` must be distinct."
|
||||
)
|
||||
|
||||
if self.default_episode_success is not None:
|
||||
self.default_episode_success = normalize_episode_success_label(self.default_episode_success)
|
||||
|
||||
if not self.collector_policy_id_human:
|
||||
raise ValueError("`collector_policy_id_human` must be a non-empty string.")
|
||||
if self.acp_inference.use_cfg and not self.acp_inference.enable:
|
||||
raise ValueError("`acp_inference.use_cfg=true` requires `acp_inference.enable=true`.")
|
||||
if self.acp_inference.cfg_beta < 0:
|
||||
raise ValueError("`acp_inference.cfg_beta` must be >= 0.")
|
||||
if self.communication_retry_timeout_s < 0:
|
||||
raise ValueError("`communication_retry_timeout_s` must be >= 0.")
|
||||
if self.communication_retry_interval_s <= 0:
|
||||
raise ValueError("`communication_retry_interval_s` must be > 0.")
|
||||
|
||||
@classmethod
|
||||
def __get_path_fields__(cls) -> list[str]:
|
||||
"""This enables the parser to load config from the policy using `--policy.path=local/dir`"""
|
||||
return ["policy"]
|
||||
|
||||
@parser.wrap()
|
||||
def record(cfg: RecordConfig) -> LeRobotDataset:
|
||||
init_logging()
|
||||
if cfg.require_episode_success_label and not cfg.enable_episode_outcome_labeling:
|
||||
raise ValueError(
|
||||
"`require_episode_success_label=true` requires `enable_episode_outcome_labeling=true`."
|
||||
)
|
||||
logging.info(pformat(asdict(cfg)))
|
||||
if cfg.display_data:
|
||||
init_rerun(session_name="recording", ip=cfg.display_ip, port=cfg.display_port)
|
||||
display_compressed_images = (
|
||||
True
|
||||
if (cfg.display_data and cfg.display_ip is not None and cfg.display_port is not None)
|
||||
else cfg.display_compressed_images
|
||||
)
|
||||
|
||||
robot = make_robot_from_config(cfg.robot)
|
||||
teleop = make_teleoperator_from_config(cfg.teleop) if cfg.teleop is not None else None
|
||||
|
||||
teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
|
||||
|
||||
dataset_features = combine_feature_dicts(
|
||||
aggregate_pipeline_dataset_features(
|
||||
pipeline=teleop_action_processor,
|
||||
initial_features=create_initial_features(
|
||||
action=robot.action_features
|
||||
), # TODO(steven, pepijn): in future this should be come from teleop or policy
|
||||
use_videos=cfg.dataset.video,
|
||||
),
|
||||
aggregate_pipeline_dataset_features(
|
||||
pipeline=robot_observation_processor,
|
||||
initial_features=create_initial_features(observation=robot.observation_features),
|
||||
use_videos=cfg.dataset.video,
|
||||
),
|
||||
)
|
||||
if cfg.intervention_state_machine_enabled and cfg.policy is not None and cfg.teleop is not None:
|
||||
action_names = dataset_features[ACTION]["names"]
|
||||
if action_names is None:
|
||||
action_names = list(robot.action_features)
|
||||
else:
|
||||
action_names = list(action_names)
|
||||
dataset_features["complementary_info.policy_action"] = {
|
||||
"dtype": "float32",
|
||||
"shape": (len(action_names),),
|
||||
"names": action_names,
|
||||
}
|
||||
dataset_features["complementary_info.is_intervention"] = {
|
||||
"dtype": "float32",
|
||||
"shape": (1,),
|
||||
"names": ["is_intervention"],
|
||||
}
|
||||
dataset_features["complementary_info.state"] = {
|
||||
"dtype": "float32",
|
||||
"shape": (1,),
|
||||
"names": ["state"],
|
||||
}
|
||||
if cfg.enable_collector_policy_id:
|
||||
dataset_features["complementary_info.collector_policy_id"] = {
|
||||
"dtype": "string",
|
||||
"shape": (1,),
|
||||
"names": ["collector_policy_id"],
|
||||
}
|
||||
|
||||
dataset = None
|
||||
listener = None
|
||||
policy_sync_executor = None
|
||||
|
||||
try:
|
||||
if cfg.resume:
|
||||
dataset = LeRobotDataset(
|
||||
cfg.dataset.repo_id,
|
||||
root=cfg.dataset.root,
|
||||
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
|
||||
vcodec=cfg.dataset.vcodec,
|
||||
)
|
||||
|
||||
if hasattr(robot, "cameras") and len(robot.cameras) > 0:
|
||||
dataset.start_image_writer(
|
||||
num_processes=cfg.dataset.num_image_writer_processes,
|
||||
num_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras),
|
||||
)
|
||||
sanity_check_dataset_robot_compatibility(dataset, robot, cfg.dataset.fps, dataset_features)
|
||||
else:
|
||||
# Create empty dataset or load existing saved episodes
|
||||
sanity_check_dataset_name(cfg.dataset.repo_id, cfg.policy)
|
||||
dataset = LeRobotDataset.create(
|
||||
cfg.dataset.repo_id,
|
||||
cfg.dataset.fps,
|
||||
root=cfg.dataset.root,
|
||||
robot_type=robot.name,
|
||||
features=dataset_features,
|
||||
use_videos=cfg.dataset.video,
|
||||
image_writer_processes=cfg.dataset.num_image_writer_processes,
|
||||
image_writer_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras),
|
||||
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
|
||||
vcodec=cfg.dataset.vcodec,
|
||||
)
|
||||
|
||||
# Load pretrained policy
|
||||
policy = None if cfg.policy is None else make_policy(cfg.policy, ds_meta=dataset.meta)
|
||||
preprocessor = None
|
||||
postprocessor = None
|
||||
if cfg.acp_inference.enable and cfg.policy is None:
|
||||
raise ValueError("`acp_inference.enable=true` requires `policy` to be set.")
|
||||
if cfg.policy is not None:
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.policy,
|
||||
pretrained_path=cfg.policy.pretrained_path,
|
||||
dataset_stats=rename_stats(dataset.meta.stats, cfg.dataset.rename_map),
|
||||
preprocessor_overrides={
|
||||
"device_processor": {"device": cfg.policy.device},
|
||||
"rename_observations_processor": {"rename_map": cfg.dataset.rename_map},
|
||||
},
|
||||
)
|
||||
|
||||
collector_policy_id_policy = (
|
||||
cfg.collector_policy_id_policy
|
||||
if cfg.collector_policy_id_policy is not None
|
||||
else infer_collector_policy_id(cfg.policy)
|
||||
)
|
||||
collector_policy_id_human = cfg.collector_policy_id_human
|
||||
|
||||
robot.connect()
|
||||
if teleop is not None:
|
||||
teleop.connect()
|
||||
on_record_connected = getattr(cfg, "_on_record_connected", None)
|
||||
if callable(on_record_connected):
|
||||
on_record_connected(robot, teleop)
|
||||
|
||||
if cfg.policy_sync_to_teleop:
|
||||
if cfg.policy is None:
|
||||
raise ValueError("`policy_sync_to_teleop=true` requires `policy` to be set.")
|
||||
if teleop is None or isinstance(teleop, list):
|
||||
raise ValueError(
|
||||
"`policy_sync_to_teleop=true` requires exactly one teleoperator with send_feedback support."
|
||||
)
|
||||
policy_sync_executor = PolicySyncDualArmExecutor(
|
||||
robot=robot,
|
||||
teleop=teleop,
|
||||
parallel_dispatch=cfg.policy_sync_parallel,
|
||||
)
|
||||
|
||||
listener, events = init_keyboard_listener(
|
||||
intervention_toggle_key=cfg.intervention_toggle_key,
|
||||
episode_success_key=cfg.episode_success_key if cfg.enable_episode_outcome_labeling else None,
|
||||
episode_failure_key=cfg.episode_failure_key if cfg.enable_episode_outcome_labeling else None,
|
||||
)
|
||||
|
||||
with VideoEncodingManager(dataset):
|
||||
recorded_episodes = 0
|
||||
while recorded_episodes < cfg.dataset.num_episodes and not events["stop_recording"]:
|
||||
events["episode_outcome"] = None
|
||||
log_say(f"Recording episode {dataset.num_episodes}", cfg.play_sounds)
|
||||
record_loop(
|
||||
robot=robot,
|
||||
events=events,
|
||||
fps=cfg.dataset.fps,
|
||||
teleop_action_processor=teleop_action_processor,
|
||||
robot_action_processor=robot_action_processor,
|
||||
robot_observation_processor=robot_observation_processor,
|
||||
teleop=teleop,
|
||||
policy=policy,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
dataset=dataset,
|
||||
control_time_s=cfg.dataset.episode_time_s,
|
||||
single_task=cfg.dataset.single_task,
|
||||
display_data=cfg.display_data,
|
||||
display_compressed_images=display_compressed_images,
|
||||
policy_sync_executor=policy_sync_executor,
|
||||
intervention_state_machine_enabled=cfg.intervention_state_machine_enabled,
|
||||
collector_policy_id_policy=collector_policy_id_policy,
|
||||
collector_policy_id_human=collector_policy_id_human,
|
||||
acp_inference=cfg.acp_inference,
|
||||
communication_retry_timeout_s=cfg.communication_retry_timeout_s,
|
||||
communication_retry_interval_s=cfg.communication_retry_interval_s,
|
||||
)
|
||||
|
||||
episode_success = None
|
||||
if cfg.enable_episode_outcome_labeling:
|
||||
episode_success = resolve_episode_success_label(
|
||||
explicit_label=events.get("episode_outcome"),
|
||||
default_label=cfg.default_episode_success,
|
||||
require_label=cfg.require_episode_success_label,
|
||||
)
|
||||
if events.get("episode_outcome") is None and episode_success is not None:
|
||||
logging.warning(
|
||||
"Episode %s has no explicit success/failure label, defaulting to '%s'.",
|
||||
dataset.num_episodes,
|
||||
episode_success,
|
||||
)
|
||||
|
||||
on_episode_outcome = getattr(cfg, "_on_record_episode_outcome", None)
|
||||
if callable(on_episode_outcome):
|
||||
on_episode_outcome(robot, teleop, episode_success)
|
||||
|
||||
# Execute a few seconds without recording to give time to manually reset the environment
|
||||
# Skip reset for the last episode to be recorded
|
||||
if not events["stop_recording"] and (
|
||||
(recorded_episodes < cfg.dataset.num_episodes - 1) or events["rerecord_episode"]
|
||||
):
|
||||
log_say("Reset the environment", cfg.play_sounds)
|
||||
|
||||
# reset g1 robot
|
||||
if robot.name == "unitree_g1":
|
||||
robot.reset()
|
||||
|
||||
record_loop(
|
||||
robot=robot,
|
||||
events=events,
|
||||
fps=cfg.dataset.fps,
|
||||
teleop_action_processor=teleop_action_processor,
|
||||
robot_action_processor=robot_action_processor,
|
||||
robot_observation_processor=robot_observation_processor,
|
||||
teleop=teleop,
|
||||
control_time_s=cfg.dataset.reset_time_s,
|
||||
single_task=cfg.dataset.single_task,
|
||||
display_data=cfg.display_data,
|
||||
policy_sync_executor=policy_sync_executor,
|
||||
intervention_state_machine_enabled=cfg.intervention_state_machine_enabled,
|
||||
collector_policy_id_policy=collector_policy_id_policy,
|
||||
collector_policy_id_human=collector_policy_id_human,
|
||||
acp_inference=cfg.acp_inference,
|
||||
communication_retry_timeout_s=cfg.communication_retry_timeout_s,
|
||||
communication_retry_interval_s=cfg.communication_retry_interval_s,
|
||||
)
|
||||
|
||||
if events["rerecord_episode"]:
|
||||
log_say("Re-record episode", cfg.play_sounds)
|
||||
events["rerecord_episode"] = False
|
||||
events["exit_early"] = False
|
||||
events["episode_outcome"] = None
|
||||
dataset.clear_episode_buffer()
|
||||
continue
|
||||
|
||||
extra_episode_metadata = (
|
||||
{"episode_success": episode_success}
|
||||
if cfg.enable_episode_outcome_labeling
|
||||
else None
|
||||
)
|
||||
dataset.save_episode(extra_episode_metadata=extra_episode_metadata)
|
||||
recorded_episodes += 1
|
||||
finally:
|
||||
log_say("Stop recording", cfg.play_sounds, blocking=True)
|
||||
|
||||
if dataset:
|
||||
dataset.finalize()
|
||||
|
||||
if policy_sync_executor is not None:
|
||||
policy_sync_executor.shutdown()
|
||||
|
||||
if robot.is_connected:
|
||||
robot.disconnect()
|
||||
if teleop and teleop.is_connected:
|
||||
teleop.disconnect()
|
||||
|
||||
if not is_headless() and listener:
|
||||
listener.stop()
|
||||
|
||||
if cfg.dataset.push_to_hub:
|
||||
if dataset is not None:
|
||||
dataset.push_to_hub(tags=cfg.dataset.tags, private=cfg.dataset.private)
|
||||
else:
|
||||
logging.warning(
|
||||
"`dataset.push_to_hub=true` was requested, but dataset was not initialized due to an earlier error."
|
||||
)
|
||||
|
||||
log_say("Exiting", cfg.play_sounds)
|
||||
return dataset
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
record()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,141 @@
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Replays the actions of an episode from a dataset on a robot.
|
||||
|
||||
Examples:
|
||||
|
||||
```shell
|
||||
lerobot-replay \
|
||||
--robot.type=so100_follower \
|
||||
--robot.port=/dev/tty.usbmodem58760431541 \
|
||||
--robot.id=black \
|
||||
--dataset.repo_id=aliberts/record-test \
|
||||
--dataset.episode=0
|
||||
```
|
||||
|
||||
Example replay with bimanual so100:
|
||||
```shell
|
||||
lerobot-replay \
|
||||
--robot.type=bi_so_follower \
|
||||
--robot.left_arm_port=/dev/tty.usbmodem5A460851411 \
|
||||
--robot.right_arm_port=/dev/tty.usbmodem5A460812391 \
|
||||
--robot.id=bimanual_follower \
|
||||
--dataset.repo_id=${HF_USER}/bimanual-so100-handover-cube \
|
||||
--dataset.episode=0
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.processor import (
|
||||
make_default_robot_action_processor,
|
||||
)
|
||||
from lerobot.robots import ( # noqa: F401
|
||||
Robot,
|
||||
RobotConfig,
|
||||
bi_openarm_follower,
|
||||
bi_so_follower,
|
||||
earthrover_mini_plus,
|
||||
hope_jr,
|
||||
koch_follower,
|
||||
make_robot_from_config,
|
||||
omx_follower,
|
||||
openarm_follower,
|
||||
reachy2,
|
||||
so_follower,
|
||||
unitree_g1,
|
||||
)
|
||||
from lerobot.utils.constants import ACTION
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.robot_utils import precise_sleep
|
||||
from lerobot.utils.utils import (
|
||||
init_logging,
|
||||
log_say,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasetReplayConfig:
|
||||
# Dataset identifier. By convention it should match '{hf_username}/{dataset_name}' (e.g. `lerobot/test`).
|
||||
repo_id: str
|
||||
# Episode to replay.
|
||||
episode: int
|
||||
# Root directory where the dataset will be stored (e.g. 'dataset/path').
|
||||
root: str | Path | None = None
|
||||
# Limit the frames per second. By default, uses the policy fps.
|
||||
fps: int = 30
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReplayConfig:
|
||||
robot: RobotConfig
|
||||
dataset: DatasetReplayConfig
|
||||
# Use vocal synthesis to read events.
|
||||
play_sounds: bool = True
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def replay(cfg: ReplayConfig):
|
||||
init_logging()
|
||||
logging.info(pformat(asdict(cfg)))
|
||||
|
||||
robot_action_processor = make_default_robot_action_processor()
|
||||
|
||||
robot = make_robot_from_config(cfg.robot)
|
||||
dataset = LeRobotDataset(cfg.dataset.repo_id, root=cfg.dataset.root, episodes=[cfg.dataset.episode])
|
||||
|
||||
# Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
|
||||
episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == cfg.dataset.episode)
|
||||
actions = episode_frames.select_columns(ACTION)
|
||||
|
||||
robot.connect()
|
||||
|
||||
try:
|
||||
log_say("Replaying episode", cfg.play_sounds, blocking=True)
|
||||
for idx in range(len(episode_frames)):
|
||||
start_episode_t = time.perf_counter()
|
||||
|
||||
action_array = actions[idx][ACTION]
|
||||
action = {}
|
||||
for i, name in enumerate(dataset.features[ACTION]["names"]):
|
||||
action[name] = action_array[i]
|
||||
|
||||
robot_obs = robot.get_observation()
|
||||
|
||||
processed_action = robot_action_processor((action, robot_obs))
|
||||
|
||||
_ = robot.send_action(processed_action)
|
||||
|
||||
dt_s = time.perf_counter() - start_episode_t
|
||||
precise_sleep(max(1 / dataset.fps - dt_s, 0.0))
|
||||
finally:
|
||||
robot.disconnect()
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
replay()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,360 @@
|
||||
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Setup and debug CAN interfaces for Damiao motors (e.g., OpenArms).
|
||||
|
||||
Examples:
|
||||
|
||||
Setup CAN interfaces with CAN FD:
|
||||
```shell
|
||||
lerobot-setup-can --mode=setup --interfaces=can0,can1,can2,can3
|
||||
```
|
||||
|
||||
Test motors on a single interface:
|
||||
```shell
|
||||
lerobot-setup-can --mode=test --interfaces=can0
|
||||
```
|
||||
|
||||
Test motors on all interfaces:
|
||||
```shell
|
||||
lerobot-setup-can --mode=test --interfaces=can0,can1,can2,can3
|
||||
```
|
||||
|
||||
Speed test:
|
||||
```shell
|
||||
lerobot-setup-can --mode=speed --interfaces=can0
|
||||
```
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import draccus
|
||||
|
||||
from lerobot.utils.import_utils import _can_available
|
||||
|
||||
MOTOR_NAMES = {
|
||||
0x01: "joint_1",
|
||||
0x02: "joint_2",
|
||||
0x03: "joint_3",
|
||||
0x04: "joint_4",
|
||||
0x05: "joint_5",
|
||||
0x06: "joint_6",
|
||||
0x07: "joint_7",
|
||||
0x08: "gripper",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CANSetupConfig:
|
||||
mode: str = "test"
|
||||
interfaces: str = "can0" # Comma-separated, e.g. "can0,can1,can2,can3"
|
||||
bitrate: int = 1000000
|
||||
data_bitrate: int = 5000000
|
||||
use_fd: bool = True
|
||||
motor_ids: list[int] = field(default_factory=lambda: list(range(0x01, 0x09)))
|
||||
timeout: float = 1.0
|
||||
speed_iterations: int = 100
|
||||
|
||||
def get_interfaces(self) -> list[str]:
|
||||
return [i.strip() for i in self.interfaces.split(",") if i.strip()]
|
||||
|
||||
|
||||
def check_interface_status(interface: str) -> tuple[bool, str, bool]:
|
||||
"""Check if CAN interface is UP and configured."""
|
||||
try:
|
||||
result = subprocess.run(["ip", "link", "show", interface], capture_output=True, text=True) # nosec B607
|
||||
if result.returncode != 0:
|
||||
return False, "Interface not found", False
|
||||
|
||||
output = result.stdout
|
||||
is_up = "UP" in output
|
||||
is_fd = "fd on" in output.lower() or "canfd" in output.lower()
|
||||
status = "UP" if is_up else "DOWN"
|
||||
if is_fd:
|
||||
status += " (CAN FD)"
|
||||
|
||||
return is_up, status, is_fd
|
||||
except FileNotFoundError:
|
||||
return False, "ip command not found", False
|
||||
|
||||
|
||||
def setup_interface(interface: str, bitrate: int, data_bitrate: int, use_fd: bool) -> bool:
|
||||
"""Configure a CAN interface."""
|
||||
try:
|
||||
subprocess.run(["sudo", "ip", "link", "set", interface, "down"], check=False, capture_output=True) # nosec B607
|
||||
|
||||
cmd = ["sudo", "ip", "link", "set", interface, "type", "can", "bitrate", str(bitrate)]
|
||||
if use_fd:
|
||||
cmd.extend(["dbitrate", str(data_bitrate), "fd", "on"])
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True) # nosec B607
|
||||
if result.returncode != 0:
|
||||
print(f" ✗ Failed to configure: {result.stderr}")
|
||||
return False
|
||||
|
||||
result = subprocess.run( # nosec B607
|
||||
["sudo", "ip", "link", "set", interface, "up"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" ✗ Failed to bring up: {result.stderr}")
|
||||
return False
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ Error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def test_motor(bus, motor_id: int, timeout: float, use_fd: bool):
|
||||
"""Test a single motor and return responses."""
|
||||
import can
|
||||
|
||||
enable_msg = can.Message(
|
||||
arbitration_id=motor_id,
|
||||
data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFC],
|
||||
is_extended_id=False,
|
||||
is_fd=use_fd,
|
||||
)
|
||||
|
||||
try:
|
||||
bus.send(enable_msg)
|
||||
except Exception as e:
|
||||
return None, f"Send error: {e}"
|
||||
|
||||
responses = []
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
msg = bus.recv(timeout=0.1)
|
||||
if msg:
|
||||
responses.append((msg.arbitration_id, msg.data.hex(), getattr(msg, "is_fd", False)))
|
||||
|
||||
disable_msg = can.Message(
|
||||
arbitration_id=motor_id,
|
||||
data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD],
|
||||
is_extended_id=False,
|
||||
is_fd=use_fd,
|
||||
)
|
||||
try:
|
||||
bus.send(disable_msg)
|
||||
except Exception:
|
||||
print(f"Error sending message to motor 0x{motor_id:02X}")
|
||||
|
||||
return responses, None
|
||||
|
||||
|
||||
def test_interface(cfg: CANSetupConfig, interface: str):
|
||||
"""Test all motors on a CAN interface."""
|
||||
import can
|
||||
|
||||
is_up, status, _ = check_interface_status(interface)
|
||||
print(f"\n{interface}: {status}")
|
||||
|
||||
if not is_up:
|
||||
print(f" ⚠ Interface is not UP. Run: lerobot-setup-can --mode=setup --interfaces {interface}")
|
||||
return {}
|
||||
|
||||
try:
|
||||
kwargs = {"channel": interface, "interface": "socketcan", "bitrate": cfg.bitrate}
|
||||
if cfg.use_fd:
|
||||
kwargs.update({"data_bitrate": cfg.data_bitrate, "fd": True})
|
||||
bus = can.interface.Bus(**kwargs)
|
||||
except Exception as e:
|
||||
print(f" ✗ Connection failed: {e}")
|
||||
return {}
|
||||
|
||||
results = {}
|
||||
try:
|
||||
while bus.recv(timeout=0.01):
|
||||
pass
|
||||
|
||||
for motor_id in cfg.motor_ids:
|
||||
motor_name = MOTOR_NAMES.get(motor_id, f"motor_0x{motor_id:02X}")
|
||||
responses, error = test_motor(bus, motor_id, cfg.timeout, cfg.use_fd)
|
||||
|
||||
if error:
|
||||
print(f" Motor 0x{motor_id:02X} ({motor_name}): ✗ {error}")
|
||||
results[motor_id] = {"found": False, "error": error}
|
||||
elif responses:
|
||||
print(f" Motor 0x{motor_id:02X} ({motor_name}): ✓ FOUND")
|
||||
for resp_id, data, is_fd in responses:
|
||||
fd_flag = " [FD]" if is_fd else ""
|
||||
print(f" → Response 0x{resp_id:02X}{fd_flag}: {data}")
|
||||
results[motor_id] = {"found": True, "responses": responses}
|
||||
else:
|
||||
print(f" Motor 0x{motor_id:02X} ({motor_name}): ✗ No response")
|
||||
results[motor_id] = {"found": False}
|
||||
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
bus.shutdown()
|
||||
|
||||
found = sum(1 for r in results.values() if r.get("found"))
|
||||
print(f"\n Summary: {found}/{len(cfg.motor_ids)} motors found")
|
||||
return results
|
||||
|
||||
|
||||
def speed_test(cfg: CANSetupConfig, interface: str):
|
||||
"""Test communication speed with motors."""
|
||||
import can
|
||||
|
||||
is_up, status, _ = check_interface_status(interface)
|
||||
if not is_up:
|
||||
print(f"{interface}: {status} - skipping")
|
||||
return
|
||||
|
||||
print(f"\n{interface}: Running speed test ({cfg.speed_iterations} iterations)...")
|
||||
|
||||
try:
|
||||
kwargs = {"channel": interface, "interface": "socketcan", "bitrate": cfg.bitrate}
|
||||
if cfg.use_fd:
|
||||
kwargs.update({"data_bitrate": cfg.data_bitrate, "fd": True})
|
||||
bus = can.interface.Bus(**kwargs)
|
||||
except Exception as e:
|
||||
print(f" ✗ Connection failed: {e}")
|
||||
return
|
||||
|
||||
responding_motor = None
|
||||
for motor_id in cfg.motor_ids:
|
||||
responses, _ = test_motor(bus, motor_id, 0.5, cfg.use_fd)
|
||||
if responses:
|
||||
responding_motor = motor_id
|
||||
break
|
||||
|
||||
if not responding_motor:
|
||||
print(" ✗ No responding motors found")
|
||||
bus.shutdown()
|
||||
return
|
||||
|
||||
print(f" Testing with motor 0x{responding_motor:02X}...")
|
||||
latencies = []
|
||||
|
||||
for _ in range(cfg.speed_iterations):
|
||||
start = time.perf_counter()
|
||||
msg = can.Message(
|
||||
arbitration_id=responding_motor,
|
||||
data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFC],
|
||||
is_extended_id=False,
|
||||
is_fd=cfg.use_fd,
|
||||
)
|
||||
bus.send(msg)
|
||||
resp = bus.recv(timeout=0.1)
|
||||
if resp:
|
||||
latencies.append((time.perf_counter() - start) * 1000)
|
||||
|
||||
bus.shutdown()
|
||||
|
||||
if latencies:
|
||||
avg_latency = sum(latencies) / len(latencies)
|
||||
hz = 1000.0 / avg_latency if avg_latency > 0 else 0
|
||||
print(f" ✓ Success rate: {len(latencies)}/{cfg.speed_iterations}")
|
||||
print(f" ✓ Avg latency: {avg_latency:.2f} ms")
|
||||
print(f" ✓ Max frequency: {hz:.1f} Hz")
|
||||
else:
|
||||
print(" ✗ No successful responses")
|
||||
|
||||
|
||||
def run_setup(cfg: CANSetupConfig):
|
||||
"""Setup CAN interfaces."""
|
||||
print("=" * 50)
|
||||
print("CAN Interface Setup")
|
||||
print("=" * 50)
|
||||
print(f"Mode: {'CAN FD' if cfg.use_fd else 'CAN 2.0'}")
|
||||
print(f"Bitrate: {cfg.bitrate / 1_000_000:.1f} Mbps")
|
||||
if cfg.use_fd:
|
||||
print(f"Data bitrate: {cfg.data_bitrate / 1_000_000:.1f} Mbps")
|
||||
print()
|
||||
|
||||
interfaces = cfg.get_interfaces()
|
||||
for interface in interfaces:
|
||||
print(f"Configuring {interface}...")
|
||||
if setup_interface(interface, cfg.bitrate, cfg.data_bitrate, cfg.use_fd):
|
||||
is_up, status, _ = check_interface_status(interface)
|
||||
print(f" ✓ {interface}: {status}")
|
||||
else:
|
||||
print(f" ✗ {interface}: Failed")
|
||||
|
||||
print("\nSetup complete!")
|
||||
print("\nNext: Test motors with:")
|
||||
print(f" lerobot-setup-can --mode=test --interfaces {','.join(interfaces)}")
|
||||
|
||||
|
||||
def run_test(cfg: CANSetupConfig):
|
||||
"""Test motors on CAN interfaces."""
|
||||
print("=" * 50)
|
||||
print("CAN Motor Test")
|
||||
print("=" * 50)
|
||||
print(f"Testing motors 0x{min(cfg.motor_ids):02X}-0x{max(cfg.motor_ids):02X}")
|
||||
print(f"Mode: {'CAN FD' if cfg.use_fd else 'CAN 2.0'}")
|
||||
print()
|
||||
|
||||
interfaces = cfg.get_interfaces()
|
||||
all_results = {}
|
||||
for interface in interfaces:
|
||||
all_results[interface] = test_interface(cfg, interface)
|
||||
|
||||
total_found = sum(sum(1 for r in res.values() if r.get("found")) for res in all_results.values())
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("Summary")
|
||||
print("=" * 50)
|
||||
print(f"Total motors found: {total_found}")
|
||||
|
||||
if total_found == 0:
|
||||
print("\n⚠ No motors found! Check:")
|
||||
print(" 1. Motors are powered (24V)")
|
||||
print(" 2. CAN wiring (CANH, CANL, GND)")
|
||||
print(" 3. Motor timeout parameter > 0 (use Damiao tools)")
|
||||
print(" 4. 120Ω termination at both cable ends")
|
||||
print(f" 5. Interface configured: lerobot-setup-can --mode=setup --interfaces {interfaces[0]}")
|
||||
|
||||
|
||||
def run_speed(cfg: CANSetupConfig):
|
||||
"""Run speed tests on CAN interfaces."""
|
||||
print("=" * 50)
|
||||
print("CAN Speed Test")
|
||||
print("=" * 50)
|
||||
|
||||
for interface in cfg.get_interfaces():
|
||||
speed_test(cfg, interface)
|
||||
|
||||
|
||||
@draccus.wrap()
|
||||
def setup_can(cfg: CANSetupConfig):
|
||||
if not _can_available:
|
||||
print("Error: python-can not installed. Install with: pip install python-can")
|
||||
sys.exit(1)
|
||||
|
||||
if cfg.mode == "setup":
|
||||
run_setup(cfg)
|
||||
elif cfg.mode == "test":
|
||||
run_test(cfg)
|
||||
elif cfg.mode == "speed":
|
||||
run_speed(cfg)
|
||||
else:
|
||||
print(f"Unknown mode: {cfg.mode}")
|
||||
print("Available modes: setup, test, speed")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
setup_can()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,92 @@
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Helper to set motor ids and baudrate.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-setup-motors \
|
||||
--teleop.type=so100_leader \
|
||||
--teleop.port=/dev/tty.usbmodem575E0031751
|
||||
```
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import draccus
|
||||
|
||||
from lerobot.robots import ( # noqa: F401
|
||||
RobotConfig,
|
||||
bi_so_follower,
|
||||
koch_follower,
|
||||
lekiwi,
|
||||
make_robot_from_config,
|
||||
omx_follower,
|
||||
so_follower,
|
||||
)
|
||||
from lerobot.teleoperators import ( # noqa: F401
|
||||
TeleoperatorConfig,
|
||||
bi_so_leader,
|
||||
koch_leader,
|
||||
make_teleoperator_from_config,
|
||||
omx_leader,
|
||||
so_leader,
|
||||
)
|
||||
|
||||
COMPATIBLE_DEVICES = [
|
||||
"koch_follower",
|
||||
"koch_leader",
|
||||
"omx_follower",
|
||||
"omx_leader",
|
||||
"so100_follower",
|
||||
"so100_leader",
|
||||
"so101_follower",
|
||||
"so101_leader",
|
||||
"lekiwi",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SetupConfig:
|
||||
teleop: TeleoperatorConfig | None = None
|
||||
robot: RobotConfig | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if bool(self.teleop) == bool(self.robot):
|
||||
raise ValueError("Choose either a teleop or a robot.")
|
||||
|
||||
self.device = self.robot if self.robot else self.teleop
|
||||
|
||||
|
||||
@draccus.wrap()
|
||||
def setup_motors(cfg: SetupConfig):
|
||||
if cfg.device.type not in COMPATIBLE_DEVICES:
|
||||
raise NotImplementedError
|
||||
|
||||
if isinstance(cfg.device, RobotConfig):
|
||||
device = make_robot_from_config(cfg.device)
|
||||
else:
|
||||
device = make_teleoperator_from_config(cfg.device)
|
||||
|
||||
device.setup_motors()
|
||||
|
||||
|
||||
def main():
|
||||
setup_motors()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,250 @@
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Simple script to control a robot from teleoperation.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-teleoperate \
|
||||
--robot.type=so101_follower \
|
||||
--robot.port=/dev/tty.usbmodem58760431541 \
|
||||
--robot.cameras="{ front: {type: opencv, index_or_path: 0, width: 1920, height: 1080, fps: 30}}" \
|
||||
--robot.id=black \
|
||||
--teleop.type=so101_leader \
|
||||
--teleop.port=/dev/tty.usbmodem58760431551 \
|
||||
--teleop.id=blue \
|
||||
--display_data=true
|
||||
```
|
||||
|
||||
Example teleoperation with bimanual so100:
|
||||
|
||||
```shell
|
||||
lerobot-teleoperate \
|
||||
--robot.type=bi_so_follower \
|
||||
--robot.left_arm_config.port=/dev/tty.usbmodem5A460822851 \
|
||||
--robot.right_arm_config.port=/dev/tty.usbmodem5A460814411 \
|
||||
--robot.id=bimanual_follower \
|
||||
--robot.left_arm_config.cameras='{
|
||||
wrist: {"type": "opencv", "index_or_path": 1, "width": 640, "height": 480, "fps": 30},
|
||||
}' --robot.right_arm_config.cameras='{
|
||||
wrist: {"type": "opencv", "index_or_path": 2, "width": 640, "height": 480, "fps": 30},
|
||||
}' \
|
||||
--teleop.type=bi_so_leader \
|
||||
--teleop.left_arm_config.port=/dev/tty.usbmodem5A460852721 \
|
||||
--teleop.right_arm_config.port=/dev/tty.usbmodem5A460819811 \
|
||||
--teleop.id=bimanual_leader \
|
||||
--display_data=true
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pprint import pformat
|
||||
|
||||
import rerun as rr
|
||||
|
||||
from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401
|
||||
from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401
|
||||
from lerobot.configs import parser
|
||||
from lerobot.processor import (
|
||||
RobotAction,
|
||||
RobotObservation,
|
||||
RobotProcessorPipeline,
|
||||
make_default_processors,
|
||||
)
|
||||
from lerobot.robots import ( # noqa: F401
|
||||
Robot,
|
||||
RobotConfig,
|
||||
bi_openarm_follower,
|
||||
bi_so_follower,
|
||||
earthrover_mini_plus,
|
||||
hope_jr,
|
||||
koch_follower,
|
||||
make_robot_from_config,
|
||||
omx_follower,
|
||||
openarm_follower,
|
||||
reachy2,
|
||||
so_follower,
|
||||
unitree_g1 as unitree_g1_robot,
|
||||
)
|
||||
from lerobot.teleoperators import ( # noqa: F401
|
||||
Teleoperator,
|
||||
TeleoperatorConfig,
|
||||
bi_openarm_leader,
|
||||
bi_so_leader,
|
||||
gamepad,
|
||||
homunculus,
|
||||
keyboard,
|
||||
koch_leader,
|
||||
make_teleoperator_from_config,
|
||||
omx_leader,
|
||||
openarm_leader,
|
||||
reachy2_teleoperator,
|
||||
so_leader,
|
||||
unitree_g1,
|
||||
)
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.robot_utils import precise_sleep
|
||||
from lerobot.utils.utils import init_logging, move_cursor_up
|
||||
from lerobot.utils.visualization_utils import init_rerun, log_rerun_data
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeleoperateConfig:
|
||||
# TODO: pepijn, steven: if more robots require multiple teleoperators (like lekiwi) its good to make this possibele in teleop.py and record.py with List[Teleoperator]
|
||||
teleop: TeleoperatorConfig
|
||||
robot: RobotConfig
|
||||
# Limit the maximum frames per second.
|
||||
fps: int = 60
|
||||
teleop_time_s: float | None = None
|
||||
# Display all cameras on screen
|
||||
display_data: bool = False
|
||||
# Display data on a remote Rerun server
|
||||
display_ip: str | None = None
|
||||
# Port of the remote Rerun server
|
||||
display_port: int | None = None
|
||||
# Whether to display compressed images in Rerun
|
||||
display_compressed_images: bool = False
|
||||
|
||||
|
||||
def teleop_loop(
|
||||
teleop: Teleoperator,
|
||||
robot: Robot,
|
||||
fps: int,
|
||||
teleop_action_processor: RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction],
|
||||
robot_action_processor: RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction],
|
||||
robot_observation_processor: RobotProcessorPipeline[RobotObservation, RobotObservation],
|
||||
display_data: bool = False,
|
||||
duration: float | None = None,
|
||||
display_compressed_images: bool = False,
|
||||
):
|
||||
"""
|
||||
This function continuously reads actions from a teleoperation device, processes them through optional
|
||||
pipelines, sends them to a robot, and optionally displays the robot's state. The loop runs at a
|
||||
specified frequency until a set duration is reached or it is manually interrupted.
|
||||
|
||||
Args:
|
||||
teleop: The teleoperator device instance providing control actions.
|
||||
robot: The robot instance being controlled.
|
||||
fps: The target frequency for the control loop in frames per second.
|
||||
display_data: If True, fetches robot observations and displays them in the console and Rerun.
|
||||
display_compressed_images: If True, compresses images before sending them to Rerun for display.
|
||||
duration: The maximum duration of the teleoperation loop in seconds. If None, the loop runs indefinitely.
|
||||
teleop_action_processor: An optional pipeline to process raw actions from the teleoperator.
|
||||
robot_action_processor: An optional pipeline to process actions before they are sent to the robot.
|
||||
robot_observation_processor: An optional pipeline to process raw observations from the robot.
|
||||
"""
|
||||
|
||||
display_len = max(len(key) for key in robot.action_features)
|
||||
start = time.perf_counter()
|
||||
|
||||
while True:
|
||||
loop_start = time.perf_counter()
|
||||
|
||||
# Get robot observation
|
||||
# Not really needed for now other than for visualization
|
||||
# teleop_action_processor can take None as an observation
|
||||
# given that it is the identity processor as default
|
||||
obs = robot.get_observation()
|
||||
|
||||
# Get teleop action
|
||||
raw_action = teleop.get_action()
|
||||
|
||||
# Process teleop action through pipeline
|
||||
teleop_action = teleop_action_processor((raw_action, obs))
|
||||
|
||||
# Process action for robot through pipeline
|
||||
robot_action_to_send = robot_action_processor((teleop_action, obs))
|
||||
|
||||
# Send processed action to robot (robot_action_processor.to_output should return RobotAction)
|
||||
_ = robot.send_action(robot_action_to_send)
|
||||
|
||||
if display_data:
|
||||
# Process robot observation through pipeline
|
||||
obs_transition = robot_observation_processor(obs)
|
||||
|
||||
log_rerun_data(
|
||||
observation=obs_transition,
|
||||
action=teleop_action,
|
||||
compress_images=display_compressed_images,
|
||||
)
|
||||
|
||||
print("\n" + "-" * (display_len + 10))
|
||||
print(f"{'NAME':<{display_len}} | {'NORM':>7}")
|
||||
# Display the final robot action that was sent
|
||||
for motor, value in robot_action_to_send.items():
|
||||
print(f"{motor:<{display_len}} | {value:>7.2f}")
|
||||
move_cursor_up(len(robot_action_to_send) + 3)
|
||||
|
||||
dt_s = time.perf_counter() - loop_start
|
||||
precise_sleep(max(1 / fps - dt_s, 0.0))
|
||||
loop_s = time.perf_counter() - loop_start
|
||||
print(f"Teleop loop time: {loop_s * 1e3:.2f}ms ({1 / loop_s:.0f} Hz)")
|
||||
move_cursor_up(1)
|
||||
|
||||
if duration is not None and time.perf_counter() - start >= duration:
|
||||
return
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def teleoperate(cfg: TeleoperateConfig):
|
||||
init_logging()
|
||||
logging.info(pformat(asdict(cfg)))
|
||||
if cfg.display_data:
|
||||
init_rerun(session_name="teleoperation", ip=cfg.display_ip, port=cfg.display_port)
|
||||
display_compressed_images = (
|
||||
True
|
||||
if (cfg.display_data and cfg.display_ip is not None and cfg.display_port is not None)
|
||||
else cfg.display_compressed_images
|
||||
)
|
||||
|
||||
teleop = make_teleoperator_from_config(cfg.teleop)
|
||||
robot = make_robot_from_config(cfg.robot)
|
||||
teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
|
||||
|
||||
teleop.connect()
|
||||
robot.connect()
|
||||
|
||||
try:
|
||||
teleop_loop(
|
||||
teleop=teleop,
|
||||
robot=robot,
|
||||
fps=cfg.fps,
|
||||
display_data=cfg.display_data,
|
||||
duration=cfg.teleop_time_s,
|
||||
teleop_action_processor=teleop_action_processor,
|
||||
robot_action_processor=robot_action_processor,
|
||||
robot_observation_processor=robot_observation_processor,
|
||||
display_compressed_images=display_compressed_images,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
if cfg.display_data:
|
||||
rr.rerun_shutdown()
|
||||
teleop.disconnect()
|
||||
robot.disconnect()
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
teleoperate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
600
src/lerobot/scripts/lerobot_train.py
普通文件
600
src/lerobot/scripts/lerobot_train.py
普通文件
@ -0,0 +1,600 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import dataclasses
|
||||
import logging
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from accelerate import Accelerator
|
||||
from termcolor import colored
|
||||
from torch.optim import Optimizer
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.train import TrainPipelineConfig
|
||||
from lerobot.datasets.factory import make_dataset
|
||||
from lerobot.datasets.sampler import EpisodeAwareSampler
|
||||
from lerobot.datasets.utils import cycle
|
||||
from lerobot.envs.factory import make_env, make_env_pre_post_processors
|
||||
from lerobot.envs.utils import close_envs
|
||||
from lerobot.optim.factory import make_optimizer_and_scheduler
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.rl.acp_dataset_stats import compute_acp_indicator_stats
|
||||
from lerobot.rl.acp_hook import build_acp_raw_batch_hook
|
||||
from lerobot.rl.wandb_utils import WandBLogger
|
||||
from lerobot.scripts.lerobot_eval import eval_policy_all
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.logging_utils import AverageMeter, MetricsTracker
|
||||
from lerobot.utils.random_utils import set_seed
|
||||
from lerobot.utils.train_utils import (
|
||||
get_step_checkpoint_dir,
|
||||
get_step_identifier,
|
||||
load_training_state,
|
||||
save_checkpoint,
|
||||
update_last_checkpoint,
|
||||
)
|
||||
from lerobot.utils.utils import (
|
||||
format_big_number,
|
||||
has_method,
|
||||
init_logging,
|
||||
)
|
||||
|
||||
|
||||
def update_policy(
|
||||
train_metrics: MetricsTracker,
|
||||
policy: PreTrainedPolicy,
|
||||
batch: Any,
|
||||
optimizer: Optimizer,
|
||||
grad_clip_norm: float,
|
||||
accelerator: Accelerator,
|
||||
lr_scheduler=None,
|
||||
lock=None,
|
||||
rabc_weights_provider=None,
|
||||
) -> tuple[MetricsTracker, dict]:
|
||||
"""
|
||||
Performs a single training step to update the policy's weights.
|
||||
|
||||
This function executes the forward and backward passes, clips gradients, and steps the optimizer and
|
||||
learning rate scheduler. Accelerator handles mixed-precision training automatically.
|
||||
|
||||
Args:
|
||||
train_metrics: A MetricsTracker instance to record training statistics.
|
||||
policy: The policy model to be trained.
|
||||
batch: A batch of training data.
|
||||
optimizer: The optimizer used to update the policy's parameters.
|
||||
grad_clip_norm: The maximum norm for gradient clipping.
|
||||
accelerator: The Accelerator instance for distributed training and mixed precision.
|
||||
lr_scheduler: An optional learning rate scheduler.
|
||||
lock: An optional lock for thread-safe optimizer updates.
|
||||
rabc_weights_provider: Optional RABCWeights instance for sample weighting.
|
||||
|
||||
Returns:
|
||||
A tuple containing:
|
||||
- The updated MetricsTracker with new statistics for this step.
|
||||
- A dictionary of outputs from the policy's forward pass, for logging purposes.
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
policy.train()
|
||||
|
||||
# Get RA-BC weights if enabled
|
||||
rabc_batch_weights = None
|
||||
rabc_batch_stats = None
|
||||
if rabc_weights_provider is not None:
|
||||
rabc_batch_weights, rabc_batch_stats = rabc_weights_provider.compute_batch_weights(batch)
|
||||
|
||||
# Let accelerator handle mixed precision
|
||||
with accelerator.autocast():
|
||||
# Use per-sample loss when RA-BC is enabled for proper weighting
|
||||
if rabc_batch_weights is not None:
|
||||
# Get per-sample losses
|
||||
per_sample_loss, output_dict = policy.forward(batch, reduction="none")
|
||||
|
||||
# Apply RA-BC weights: L_RA-BC = Σ(w_i * l_i) / (Σw_i + ε)
|
||||
# rabc_batch_weights is already normalized to sum to batch_size
|
||||
epsilon = 1e-6
|
||||
loss = (per_sample_loss * rabc_batch_weights).sum() / (rabc_batch_weights.sum() + epsilon)
|
||||
# Log raw mean weight (before normalization) - this is the meaningful metric
|
||||
output_dict["rabc_mean_weight"] = rabc_batch_stats["raw_mean_weight"]
|
||||
output_dict["rabc_num_zero_weight"] = rabc_batch_stats["num_zero_weight"]
|
||||
output_dict["rabc_num_full_weight"] = rabc_batch_stats["num_full_weight"]
|
||||
else:
|
||||
loss, output_dict = policy.forward(batch)
|
||||
|
||||
# TODO(rcadene): policy.unnormalize_outputs(out_dict)
|
||||
|
||||
# Use accelerator's backward method
|
||||
accelerator.backward(loss)
|
||||
|
||||
# Clip gradients if specified
|
||||
if grad_clip_norm > 0:
|
||||
grad_norm = accelerator.clip_grad_norm_(policy.parameters(), grad_clip_norm)
|
||||
else:
|
||||
grad_norm = torch.nn.utils.clip_grad_norm_(
|
||||
policy.parameters(), float("inf"), error_if_nonfinite=False
|
||||
)
|
||||
|
||||
# Optimizer step
|
||||
with lock if lock is not None else nullcontext():
|
||||
optimizer.step()
|
||||
|
||||
optimizer.zero_grad()
|
||||
|
||||
# Step through pytorch scheduler at every batch instead of epoch
|
||||
if lr_scheduler is not None:
|
||||
lr_scheduler.step()
|
||||
|
||||
# Update internal buffers if policy has update method
|
||||
if has_method(accelerator.unwrap_model(policy, keep_fp32_wrapper=True), "update"):
|
||||
accelerator.unwrap_model(policy, keep_fp32_wrapper=True).update()
|
||||
|
||||
train_metrics.loss = loss.item()
|
||||
train_metrics.grad_norm = grad_norm.item()
|
||||
train_metrics.lr = optimizer.param_groups[0]["lr"]
|
||||
train_metrics.update_s = time.perf_counter() - start_time
|
||||
return train_metrics, output_dict
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def train(
|
||||
cfg: TrainPipelineConfig,
|
||||
accelerator: Accelerator | None = None,
|
||||
):
|
||||
"""
|
||||
Main function to train a policy.
|
||||
|
||||
This function orchestrates the entire training pipeline, including:
|
||||
- Setting up logging, seeding, and device configuration.
|
||||
- Creating the dataset, evaluation environment (if applicable), policy, and optimizer.
|
||||
- Handling resumption from a checkpoint.
|
||||
- Running the main training loop, which involves fetching data batches and calling `update_policy`.
|
||||
- Periodically logging metrics, saving model checkpoints, and evaluating the policy.
|
||||
- Pushing the final trained model to the Hugging Face Hub if configured.
|
||||
|
||||
Args:
|
||||
cfg: A `TrainPipelineConfig` object containing all training configurations.
|
||||
accelerator: Optional Accelerator instance. If None, one will be created automatically.
|
||||
"""
|
||||
cfg.validate()
|
||||
acp_raw_batch_hook = build_acp_raw_batch_hook(cfg.acp, cfg.seed)
|
||||
|
||||
# Create Accelerator if not provided
|
||||
# It will automatically detect if running in distributed mode or single-process mode
|
||||
# We set step_scheduler_with_optimizer=False to prevent accelerate from adjusting the lr_scheduler steps based on the num_processes
|
||||
# We set find_unused_parameters=True to handle models with conditional computation
|
||||
if accelerator is None:
|
||||
from accelerate.utils import DistributedDataParallelKwargs
|
||||
|
||||
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
|
||||
# Accelerate auto-detects the device based on the available hardware and ignores the policy.device setting.
|
||||
# Force the device to be CPU when policy.device is set to CPU.
|
||||
force_cpu = cfg.policy.device == "cpu"
|
||||
accelerator = Accelerator(
|
||||
step_scheduler_with_optimizer=False,
|
||||
kwargs_handlers=[ddp_kwargs],
|
||||
cpu=force_cpu,
|
||||
)
|
||||
|
||||
init_logging(accelerator=accelerator)
|
||||
|
||||
# Determine if this is the main process (for logging and checkpointing)
|
||||
# When using accelerate, only the main process should log to avoid duplicate outputs
|
||||
is_main_process = accelerator.is_main_process
|
||||
|
||||
# Only log on main process
|
||||
if is_main_process:
|
||||
logging.info(pformat(cfg.to_dict()))
|
||||
|
||||
# Initialize wandb only on main process
|
||||
if cfg.wandb.enable and cfg.wandb.project and is_main_process:
|
||||
wandb_logger = WandBLogger(cfg)
|
||||
else:
|
||||
wandb_logger = None
|
||||
if is_main_process:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
|
||||
if cfg.seed is not None:
|
||||
set_seed(cfg.seed, accelerator=accelerator)
|
||||
|
||||
# Use accelerator's device
|
||||
device = accelerator.device
|
||||
torch.backends.cudnn.benchmark = True
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
|
||||
# Dataset loading synchronization: main process downloads first to avoid race conditions
|
||||
if is_main_process:
|
||||
logging.info("Creating dataset")
|
||||
dataset = make_dataset(cfg)
|
||||
if cfg.acp.enable:
|
||||
indicator_stats = compute_acp_indicator_stats(dataset, cfg.acp.indicator_field)
|
||||
if indicator_stats is None:
|
||||
logging.warning(
|
||||
"ACP is enabled but indicator statistics are unavailable for field '%s'.",
|
||||
cfg.acp.indicator_field,
|
||||
)
|
||||
else:
|
||||
if indicator_stats.total_count >= 0:
|
||||
logging.info(
|
||||
"ACP indicator stats (%s): field='%s' ratio=%.6f positive=%d total=%d",
|
||||
indicator_stats.source,
|
||||
indicator_stats.indicator_field,
|
||||
indicator_stats.positive_ratio,
|
||||
indicator_stats.positive_count,
|
||||
indicator_stats.total_count,
|
||||
)
|
||||
else:
|
||||
logging.info(
|
||||
"ACP indicator stats (%s): field='%s' ratio=%.6f",
|
||||
indicator_stats.source,
|
||||
indicator_stats.indicator_field,
|
||||
indicator_stats.positive_ratio,
|
||||
)
|
||||
if indicator_stats.invalid_count > 0:
|
||||
logging.warning(
|
||||
"ACP indicator field '%s' contains %d non-binary values (expected only 0/1).",
|
||||
indicator_stats.indicator_field,
|
||||
indicator_stats.invalid_count,
|
||||
)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
# Now all other processes can safely load the dataset
|
||||
if not is_main_process:
|
||||
dataset = make_dataset(cfg)
|
||||
|
||||
# Create environment used for evaluating checkpoints during training on simulation data.
|
||||
# On real-world data, no need to create an environment as evaluations are done outside train.py,
|
||||
# using the eval.py instead, with gym_dora environment and dora-rs.
|
||||
eval_env = None
|
||||
if cfg.eval_freq > 0 and cfg.env is not None and is_main_process:
|
||||
logging.info("Creating env")
|
||||
eval_env = make_env(cfg.env, n_envs=cfg.eval.batch_size, use_async_envs=cfg.eval.use_async_envs)
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating policy")
|
||||
policy = make_policy(
|
||||
cfg=cfg.policy,
|
||||
ds_meta=dataset.meta,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
|
||||
if cfg.peft is not None:
|
||||
logging.info("Using PEFT! Wrapping model.")
|
||||
# Convert CLI peft config to dict for overrides
|
||||
peft_cli_overrides = dataclasses.asdict(cfg.peft)
|
||||
policy = policy.wrap_with_peft(peft_cli_overrides=peft_cli_overrides)
|
||||
|
||||
# Wait for all processes to finish policy creation before continuing
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
# Create processors - only provide dataset_stats if not resuming from saved processors
|
||||
processor_kwargs = {}
|
||||
postprocessor_kwargs = {}
|
||||
if (cfg.policy.pretrained_path and not cfg.resume) or not cfg.policy.pretrained_path:
|
||||
# Only provide dataset_stats when not resuming from saved processor state
|
||||
processor_kwargs["dataset_stats"] = dataset.meta.stats
|
||||
|
||||
# For SARM, always provide dataset_meta for progress normalization
|
||||
if cfg.policy.type == "sarm":
|
||||
processor_kwargs["dataset_meta"] = dataset.meta
|
||||
|
||||
if cfg.policy.pretrained_path is not None:
|
||||
processor_kwargs["preprocessor_overrides"] = {
|
||||
"device_processor": {"device": device.type},
|
||||
"normalizer_processor": {
|
||||
"stats": dataset.meta.stats,
|
||||
"features": {**policy.config.input_features, **policy.config.output_features},
|
||||
"norm_map": policy.config.normalization_mapping,
|
||||
},
|
||||
}
|
||||
processor_kwargs["preprocessor_overrides"]["rename_observations_processor"] = {
|
||||
"rename_map": cfg.rename_map
|
||||
}
|
||||
postprocessor_kwargs["postprocessor_overrides"] = {
|
||||
"unnormalizer_processor": {
|
||||
"stats": dataset.meta.stats,
|
||||
"features": policy.config.output_features,
|
||||
"norm_map": policy.config.normalization_mapping,
|
||||
},
|
||||
}
|
||||
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.policy,
|
||||
pretrained_path=cfg.policy.pretrained_path,
|
||||
**processor_kwargs,
|
||||
**postprocessor_kwargs,
|
||||
)
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating optimizer and scheduler")
|
||||
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy)
|
||||
|
||||
# Load precomputed SARM progress for RA-BC if enabled
|
||||
# Generate progress using: src/lerobot/policies/sarm/compute_rabc_weights.py
|
||||
rabc_weights = None
|
||||
if cfg.use_rabc:
|
||||
from lerobot.utils.rabc import RABCWeights
|
||||
|
||||
# Get chunk_size from policy config
|
||||
chunk_size = getattr(policy.config, "chunk_size", None)
|
||||
if chunk_size is None:
|
||||
raise ValueError("Chunk size is not found in policy config")
|
||||
|
||||
head_mode = getattr(cfg, "rabc_head_mode", "sparse")
|
||||
logging.info(f"Loading SARM progress for RA-BC from {cfg.rabc_progress_path}")
|
||||
logging.info(f"Using chunk_size={chunk_size} from policy config, head_mode={head_mode}")
|
||||
rabc_weights = RABCWeights(
|
||||
progress_path=cfg.rabc_progress_path,
|
||||
chunk_size=chunk_size,
|
||||
head_mode=head_mode,
|
||||
kappa=getattr(cfg, "rabc_kappa", 0.01),
|
||||
epsilon=getattr(cfg, "rabc_epsilon", 1e-6),
|
||||
device=device,
|
||||
)
|
||||
|
||||
step = 0 # number of policy updates (forward + backward + optim)
|
||||
|
||||
if cfg.resume:
|
||||
step, optimizer, lr_scheduler = load_training_state(cfg.checkpoint_path, optimizer, lr_scheduler)
|
||||
|
||||
num_learnable_params = sum(p.numel() for p in policy.parameters() if p.requires_grad)
|
||||
num_total_params = sum(p.numel() for p in policy.parameters())
|
||||
|
||||
if is_main_process:
|
||||
logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
|
||||
if cfg.env is not None:
|
||||
logging.info(f"{cfg.env.task=}")
|
||||
logging.info("Creating environment processors")
|
||||
env_preprocessor, env_postprocessor = make_env_pre_post_processors(
|
||||
env_cfg=cfg.env, policy_cfg=cfg.policy
|
||||
)
|
||||
logging.info(f"{cfg.steps=} ({format_big_number(cfg.steps)})")
|
||||
logging.info(f"{dataset.num_frames=} ({format_big_number(dataset.num_frames)})")
|
||||
logging.info(f"{dataset.num_episodes=}")
|
||||
num_processes = accelerator.num_processes
|
||||
effective_bs = cfg.batch_size * num_processes
|
||||
logging.info(f"Effective batch size: {cfg.batch_size} x {num_processes} = {effective_bs}")
|
||||
logging.info(f"{num_learnable_params=} ({format_big_number(num_learnable_params)})")
|
||||
logging.info(f"{num_total_params=} ({format_big_number(num_total_params)})")
|
||||
|
||||
# create dataloader for offline training
|
||||
if hasattr(cfg.policy, "drop_n_last_frames"):
|
||||
shuffle = False
|
||||
sampler = EpisodeAwareSampler(
|
||||
dataset.meta.episodes["dataset_from_index"],
|
||||
dataset.meta.episodes["dataset_to_index"],
|
||||
episode_indices_to_use=dataset.episodes,
|
||||
drop_n_last_frames=cfg.policy.drop_n_last_frames,
|
||||
shuffle=True,
|
||||
)
|
||||
else:
|
||||
shuffle = True
|
||||
sampler = None
|
||||
|
||||
dataloader = torch.utils.data.DataLoader(
|
||||
dataset,
|
||||
num_workers=cfg.num_workers,
|
||||
batch_size=cfg.batch_size,
|
||||
shuffle=shuffle and not cfg.dataset.streaming,
|
||||
sampler=sampler,
|
||||
pin_memory=device.type == "cuda",
|
||||
drop_last=False,
|
||||
prefetch_factor=2 if cfg.num_workers > 0 else None,
|
||||
)
|
||||
|
||||
# Prepare everything with accelerator
|
||||
accelerator.wait_for_everyone()
|
||||
policy, optimizer, dataloader, lr_scheduler = accelerator.prepare(
|
||||
policy, optimizer, dataloader, lr_scheduler
|
||||
)
|
||||
dl_iter = cycle(dataloader)
|
||||
|
||||
policy.train()
|
||||
|
||||
train_metrics = {
|
||||
"loss": AverageMeter("loss", ":.3f"),
|
||||
"grad_norm": AverageMeter("grdn", ":.3f"),
|
||||
"lr": AverageMeter("lr", ":0.1e"),
|
||||
"update_s": AverageMeter("updt_s", ":.3f"),
|
||||
"dataloading_s": AverageMeter("data_s", ":.3f"),
|
||||
}
|
||||
|
||||
# Use effective batch size for proper epoch calculation in distributed training
|
||||
effective_batch_size = cfg.batch_size * accelerator.num_processes
|
||||
train_tracker = MetricsTracker(
|
||||
effective_batch_size,
|
||||
dataset.num_frames,
|
||||
dataset.num_episodes,
|
||||
train_metrics,
|
||||
initial_step=step,
|
||||
accelerator=accelerator,
|
||||
)
|
||||
|
||||
if is_main_process:
|
||||
logging.info(
|
||||
f"Start offline training on a fixed dataset, with effective batch size: {effective_batch_size}"
|
||||
)
|
||||
|
||||
logged_first_prompt = False
|
||||
prompt_keys: list[str] = []
|
||||
policy_task_field = getattr(cfg.policy, "task_field", None)
|
||||
if isinstance(policy_task_field, str) and policy_task_field:
|
||||
prompt_keys.append(policy_task_field)
|
||||
for key in ("task", "subtask"):
|
||||
if key not in prompt_keys:
|
||||
prompt_keys.append(key)
|
||||
|
||||
for _ in range(step, cfg.steps):
|
||||
start_time = time.perf_counter()
|
||||
batch = next(dl_iter)
|
||||
if acp_raw_batch_hook is not None:
|
||||
batch = acp_raw_batch_hook(batch, step)
|
||||
batch = preprocessor(batch)
|
||||
|
||||
if is_main_process and not logged_first_prompt:
|
||||
for key in prompt_keys:
|
||||
if key not in batch:
|
||||
continue
|
||||
prompt_batch = batch[key]
|
||||
first_prompt = None
|
||||
if isinstance(prompt_batch, str):
|
||||
first_prompt = prompt_batch
|
||||
elif isinstance(prompt_batch, (list, tuple)) and len(prompt_batch) > 0:
|
||||
first_item = prompt_batch[0]
|
||||
if isinstance(first_item, str):
|
||||
first_prompt = first_item
|
||||
if first_prompt is not None:
|
||||
logging.info("First policy prompt (%s):\n%s", key, first_prompt)
|
||||
logged_first_prompt = True
|
||||
break
|
||||
train_tracker.dataloading_s = time.perf_counter() - start_time
|
||||
|
||||
train_tracker, output_dict = update_policy(
|
||||
train_tracker,
|
||||
policy,
|
||||
batch,
|
||||
optimizer,
|
||||
cfg.optimizer.grad_clip_norm,
|
||||
accelerator=accelerator,
|
||||
lr_scheduler=lr_scheduler,
|
||||
rabc_weights_provider=rabc_weights,
|
||||
)
|
||||
|
||||
# Note: eval and checkpoint happens *after* the `step`th training update has completed, so we
|
||||
# increment `step` here.
|
||||
step += 1
|
||||
train_tracker.step()
|
||||
is_log_step = cfg.log_freq > 0 and step % cfg.log_freq == 0 and is_main_process
|
||||
is_saving_step = step % cfg.save_freq == 0 or step == cfg.steps
|
||||
is_eval_step = cfg.eval_freq > 0 and step % cfg.eval_freq == 0
|
||||
|
||||
if is_log_step:
|
||||
logging.info(train_tracker)
|
||||
if wandb_logger:
|
||||
wandb_log_dict = train_tracker.to_dict()
|
||||
if output_dict:
|
||||
wandb_log_dict.update(output_dict)
|
||||
# Log RA-BC statistics if enabled
|
||||
if rabc_weights is not None:
|
||||
rabc_stats = rabc_weights.get_stats()
|
||||
wandb_log_dict.update(
|
||||
{
|
||||
"rabc_delta_mean": rabc_stats["delta_mean"],
|
||||
"rabc_delta_std": rabc_stats["delta_std"],
|
||||
"rabc_num_frames": rabc_stats["num_frames"],
|
||||
}
|
||||
)
|
||||
wandb_logger.log_dict(wandb_log_dict, step)
|
||||
train_tracker.reset_averages()
|
||||
|
||||
if cfg.save_checkpoint and is_saving_step:
|
||||
if is_main_process:
|
||||
logging.info(f"Checkpoint policy after step {step}")
|
||||
checkpoint_dir = get_step_checkpoint_dir(cfg.output_dir, cfg.steps, step)
|
||||
save_checkpoint(
|
||||
checkpoint_dir=checkpoint_dir,
|
||||
step=step,
|
||||
cfg=cfg,
|
||||
policy=accelerator.unwrap_model(policy),
|
||||
optimizer=optimizer,
|
||||
scheduler=lr_scheduler,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
)
|
||||
update_last_checkpoint(checkpoint_dir)
|
||||
if wandb_logger:
|
||||
wandb_logger.log_policy(checkpoint_dir)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
if cfg.env and is_eval_step:
|
||||
if is_main_process:
|
||||
step_id = get_step_identifier(step, cfg.steps)
|
||||
logging.info(f"Eval policy at step {step}")
|
||||
with torch.no_grad(), accelerator.autocast():
|
||||
eval_info = eval_policy_all(
|
||||
envs=eval_env, # dict[suite][task_id] -> vec_env
|
||||
policy=accelerator.unwrap_model(policy),
|
||||
env_preprocessor=env_preprocessor,
|
||||
env_postprocessor=env_postprocessor,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
n_episodes=cfg.eval.n_episodes,
|
||||
videos_dir=cfg.output_dir / "eval" / f"videos_step_{step_id}",
|
||||
max_episodes_rendered=4,
|
||||
start_seed=cfg.seed,
|
||||
max_parallel_tasks=cfg.env.max_parallel_tasks,
|
||||
)
|
||||
# overall metrics (suite-agnostic)
|
||||
aggregated = eval_info["overall"]
|
||||
|
||||
# optional: per-suite logging
|
||||
for suite, suite_info in eval_info.items():
|
||||
logging.info("Suite %s aggregated: %s", suite, suite_info)
|
||||
|
||||
# meters/tracker
|
||||
eval_metrics = {
|
||||
"avg_sum_reward": AverageMeter("∑rwrd", ":.3f"),
|
||||
"pc_success": AverageMeter("success", ":.1f"),
|
||||
"eval_s": AverageMeter("eval_s", ":.3f"),
|
||||
}
|
||||
eval_tracker = MetricsTracker(
|
||||
cfg.batch_size,
|
||||
dataset.num_frames,
|
||||
dataset.num_episodes,
|
||||
eval_metrics,
|
||||
initial_step=step,
|
||||
accelerator=accelerator,
|
||||
)
|
||||
eval_tracker.eval_s = aggregated.pop("eval_s")
|
||||
eval_tracker.avg_sum_reward = aggregated.pop("avg_sum_reward")
|
||||
eval_tracker.pc_success = aggregated.pop("pc_success")
|
||||
if wandb_logger:
|
||||
wandb_log_dict = {**eval_tracker.to_dict(), **eval_info}
|
||||
wandb_logger.log_dict(wandb_log_dict, step, mode="eval")
|
||||
wandb_logger.log_video(eval_info["overall"]["video_paths"][0], step, mode="eval")
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
if eval_env:
|
||||
close_envs(eval_env)
|
||||
|
||||
if is_main_process:
|
||||
logging.info("End of training")
|
||||
|
||||
if cfg.policy.push_to_hub:
|
||||
unwrapped_policy = accelerator.unwrap_model(policy)
|
||||
if cfg.policy.use_peft:
|
||||
unwrapped_policy.push_model_to_hub(cfg, peft_model=unwrapped_policy)
|
||||
else:
|
||||
unwrapped_policy.push_model_to_hub(cfg)
|
||||
preprocessor.push_to_hub(cfg.policy.repo_id)
|
||||
postprocessor.push_to_hub(cfg.policy.repo_id)
|
||||
|
||||
# Properly clean up the distributed process group
|
||||
accelerator.wait_for_everyone()
|
||||
accelerator.end_training()
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
train()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,604 @@
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Train FAST tokenizer for action encoding.
|
||||
|
||||
This script:
|
||||
1. Loads action chunks from LeRobotDataset (with episode sampling)
|
||||
2. Optionally applies delta transforms (relative vs absolute actions)
|
||||
3. Extracts specified action dimensions for encoding
|
||||
4. Applies normalization (MEAN_STD, MIN_MAX, QUANTILES, or other modes)
|
||||
5. Trains FAST tokenizer (BPE on DCT coefficients) on the action chunks
|
||||
6. Saves tokenizer to output directory
|
||||
7. Optionally pushes tokenizer to Hugging Face Hub
|
||||
8. Reports compression statistics
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
lerobot-train-tokenizer \
|
||||
--repo_id=user/dataset_name \
|
||||
--action_horizon=10 \
|
||||
--max_episodes=100 \
|
||||
--sample_fraction=0.1 \
|
||||
--encoded_dims="0:6" \
|
||||
--delta_dims="0,1,2,3,4,5" \
|
||||
--use_delta_transform=true \
|
||||
--state_key="observation.state" \
|
||||
--normalization_mode="QUANTILES" \
|
||||
--vocab_size=1024 \
|
||||
--scale=10.0 \
|
||||
--output_dir="./fast_tokenizer_dataset_name" \
|
||||
--push_to_hub=true \
|
||||
--hub_repo_id="user/fast_tokenizer_dataset_name" \
|
||||
--hub_private=false
|
||||
"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from huggingface_hub import HfApi
|
||||
|
||||
from lerobot.utils.import_utils import _transformers_available
|
||||
|
||||
if TYPE_CHECKING or _transformers_available:
|
||||
from transformers import AutoProcessor
|
||||
else:
|
||||
AutoProcessor = None
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.types import NormalizationMode
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.utils.constants import ACTION, OBS_STATE
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenizerTrainingConfig:
|
||||
"""Configuration for training FAST tokenizer."""
|
||||
|
||||
# LeRobot dataset repository ID
|
||||
repo_id: str
|
||||
# Root directory for dataset (default: ~/.cache/huggingface/lerobot)
|
||||
root: str | None = None
|
||||
# Number of future actions in each chunk
|
||||
action_horizon: int = 10
|
||||
# Max episodes to use (None = all episodes in dataset)
|
||||
max_episodes: int | None = None
|
||||
# Fraction of chunks to sample per episode
|
||||
sample_fraction: float = 0.1
|
||||
# Comma-separated dimension ranges to encode (e.g., "0:6,7:23")
|
||||
encoded_dims: str = "0:6,7:23"
|
||||
# Comma-separated dimension indices for delta transform (e.g., "0,1,2,3,4,5")
|
||||
delta_dims: str | None = None
|
||||
# Whether to apply delta transform (relative actions vs absolute actions)
|
||||
use_delta_transform: bool = False
|
||||
# Dataset key for state observations (default: "observation.state")
|
||||
state_key: str = OBS_STATE
|
||||
# Normalization mode (MEAN_STD, MIN_MAX, QUANTILES, QUANTILE10, IDENTITY)
|
||||
normalization_mode: str = "QUANTILES"
|
||||
# FAST vocabulary size (BPE vocab size)
|
||||
vocab_size: int = 1024
|
||||
# DCT scaling factor (default: 10.0)
|
||||
scale: float = 10.0
|
||||
# Directory to save tokenizer (default: ./fast_tokenizer_{repo_id})
|
||||
output_dir: str | None = None
|
||||
# Whether to push the tokenizer to Hugging Face Hub
|
||||
push_to_hub: bool = False
|
||||
# Hub repository ID (e.g., "username/tokenizer-name"). If None, uses output_dir name
|
||||
hub_repo_id: str | None = None
|
||||
# Whether to create a private repository on the Hub
|
||||
hub_private: bool = False
|
||||
|
||||
|
||||
def apply_delta_transform(state: np.ndarray, actions: np.ndarray, delta_dims: list[int] | None) -> np.ndarray:
|
||||
"""Apply delta transform to specified dimensions.
|
||||
|
||||
Args:
|
||||
state: Current state [D]
|
||||
actions: Future actions [D]
|
||||
delta_dims: List of dimension indices to apply delta transform to
|
||||
|
||||
Returns:
|
||||
Transformed actions [D]
|
||||
"""
|
||||
if delta_dims is None or len(delta_dims) == 0:
|
||||
return actions
|
||||
|
||||
delta_actions = actions.copy()
|
||||
for dim in delta_dims:
|
||||
delta_actions[dim] = actions[dim] - state[dim]
|
||||
|
||||
return delta_actions
|
||||
|
||||
|
||||
def apply_normalization(
|
||||
data: np.ndarray,
|
||||
stats: dict[str, np.ndarray],
|
||||
mode: NormalizationMode,
|
||||
eps: float = 1e-8,
|
||||
) -> np.ndarray:
|
||||
"""Apply normalization to data based on the specified mode.
|
||||
|
||||
Args:
|
||||
data: Data to normalize [N, H, D] or [D]
|
||||
stats: Dictionary of statistics (mean, std, min, max, q01, q99, q10, q90)
|
||||
mode: Normalization mode to apply
|
||||
eps: Small epsilon for numerical stability
|
||||
|
||||
Returns:
|
||||
Normalized data with the same shape as input
|
||||
"""
|
||||
if mode == NormalizationMode.IDENTITY:
|
||||
return data
|
||||
|
||||
if mode == NormalizationMode.MEAN_STD:
|
||||
mean = stats.get("mean")
|
||||
std = stats.get("std")
|
||||
if mean is None or std is None:
|
||||
raise ValueError("MEAN_STD mode requires 'mean' and 'std' in stats")
|
||||
return (data - mean) / np.maximum(std, eps)
|
||||
|
||||
if mode == NormalizationMode.MIN_MAX:
|
||||
min_val = stats.get("min")
|
||||
max_val = stats.get("max")
|
||||
if min_val is None or max_val is None:
|
||||
raise ValueError("MIN_MAX mode requires 'min' and 'max' in stats")
|
||||
denom = np.maximum(max_val - min_val, eps)
|
||||
return 2.0 * (data - min_val) / denom - 1.0
|
||||
|
||||
if mode == NormalizationMode.QUANTILES:
|
||||
q01 = stats.get("q01")
|
||||
q99 = stats.get("q99")
|
||||
if q01 is None or q99 is None:
|
||||
raise ValueError("QUANTILES mode requires 'q01' and 'q99' in stats")
|
||||
denom = np.maximum(q99 - q01, eps)
|
||||
# Clip to quantile range then normalize to [-1, 1]
|
||||
clipped = np.clip(data, q01, q99)
|
||||
return 2.0 * (clipped - q01) / denom - 1.0
|
||||
|
||||
if mode == NormalizationMode.QUANTILE10:
|
||||
q10 = stats.get("q10")
|
||||
q90 = stats.get("q90")
|
||||
if q10 is None or q90 is None:
|
||||
raise ValueError("QUANTILE10 mode requires 'q10' and 'q90' in stats")
|
||||
denom = np.maximum(q90 - q10, eps)
|
||||
# Clip to quantile range then normalize to [-1, 1]
|
||||
clipped = np.clip(data, q10, q90)
|
||||
return 2.0 * (clipped - q10) / denom - 1.0
|
||||
|
||||
raise ValueError(f"Unsupported normalization mode: {mode}")
|
||||
|
||||
|
||||
def process_episode(args):
|
||||
"""Process single episode and return action chunks."""
|
||||
dataset, ep_idx, action_horizon, delta_dims, sample_fraction, state_key, use_delta_transform = args
|
||||
|
||||
try:
|
||||
# get episode info
|
||||
ep_info = dataset.meta.episodes[ep_idx]
|
||||
from_idx = ep_info["dataset_from_index"]
|
||||
to_idx = ep_info["dataset_to_index"]
|
||||
ep_length = to_idx - from_idx
|
||||
|
||||
if ep_length < action_horizon:
|
||||
return None
|
||||
|
||||
# load all frames in episode
|
||||
# if dataset has episode filtering, we need to use the mapping
|
||||
states = []
|
||||
actions = []
|
||||
|
||||
for abs_idx in range(from_idx, to_idx):
|
||||
# map absolute index to relative index if needed
|
||||
if dataset._absolute_to_relative_idx is not None:
|
||||
if abs_idx not in dataset._absolute_to_relative_idx:
|
||||
# this episode's frames aren't in the filtered dataset
|
||||
return None
|
||||
rel_idx = dataset._absolute_to_relative_idx[abs_idx]
|
||||
else:
|
||||
rel_idx = abs_idx
|
||||
|
||||
frame = dataset.hf_dataset[rel_idx]
|
||||
|
||||
# get state (could be from observation.state or other state key)
|
||||
if state_key in frame:
|
||||
state = (
|
||||
frame[state_key].numpy()
|
||||
if torch.is_tensor(frame[state_key])
|
||||
else np.array(frame[state_key])
|
||||
)
|
||||
else:
|
||||
# if no state key, use zeros (no delta transform)
|
||||
state = np.zeros_like(
|
||||
frame[ACTION].numpy() if torch.is_tensor(frame[ACTION]) else np.array(frame[ACTION])
|
||||
)
|
||||
|
||||
action = frame[ACTION].numpy() if torch.is_tensor(frame[ACTION]) else np.array(frame[ACTION])
|
||||
|
||||
states.append(state)
|
||||
actions.append(action)
|
||||
|
||||
states = np.array(states)
|
||||
actions = np.array(actions)
|
||||
|
||||
# create action chunks (sliding window)
|
||||
# all actions in a chunk are relative to the FIRST state in that chunk
|
||||
action_chunks = []
|
||||
|
||||
for i in range(len(states) - action_horizon + 1):
|
||||
current_state = states[i] # First state in chunk
|
||||
future_absolute_actions = actions[i : i + action_horizon]
|
||||
|
||||
if use_delta_transform:
|
||||
# relative actions
|
||||
delta_chunk = np.zeros_like(future_absolute_actions)
|
||||
for t in range(action_horizon):
|
||||
delta_chunk[t] = apply_delta_transform(
|
||||
current_state,
|
||||
future_absolute_actions[t],
|
||||
delta_dims,
|
||||
)
|
||||
action_chunks.append(delta_chunk)
|
||||
else:
|
||||
# absolute actions (no delta)
|
||||
action_chunks.append(future_absolute_actions)
|
||||
|
||||
if len(action_chunks) == 0:
|
||||
return None
|
||||
|
||||
action_chunks = np.array(action_chunks)
|
||||
|
||||
# sample chunks
|
||||
if sample_fraction < 1.0:
|
||||
n_chunks = len(action_chunks)
|
||||
n_samples = max(1, int(n_chunks * sample_fraction))
|
||||
episode_seed = hash(ep_idx) % (2**31)
|
||||
rng = np.random.RandomState(episode_seed)
|
||||
indices = rng.choice(n_chunks, size=n_samples, replace=False)
|
||||
action_chunks = action_chunks[indices]
|
||||
|
||||
return action_chunks
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing episode {ep_idx}: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
|
||||
def train_fast_tokenizer(
|
||||
action_chunks: np.ndarray,
|
||||
vocab_size: int = 1024,
|
||||
scale: float = 10.0,
|
||||
) -> AutoProcessor:
|
||||
"""
|
||||
Train FAST tokenizer (BPE on DCT coefficients) on action chunks.
|
||||
|
||||
Uses the .fit() method to train a new tokenizer on the provided data.
|
||||
|
||||
Args:
|
||||
action_chunks: Array of action chunks [N, H, D] where N=num_chunks, H=horizon, D=action_dim
|
||||
vocab_size: BPE vocabulary size
|
||||
scale: DCT scaling factor for quantization
|
||||
|
||||
Returns:
|
||||
Trained FAST tokenizer
|
||||
"""
|
||||
print(f"Training FAST tokenizer on {len(action_chunks)} action chunks...")
|
||||
print(f"Action chunk shape: {action_chunks.shape}")
|
||||
print(f"Vocab size: {vocab_size}")
|
||||
print(f"DCT scale: {scale}")
|
||||
|
||||
# download the tokenizer source code (not pretrained weights)
|
||||
# we'll train a new tokenizer on our own data
|
||||
base_tokenizer = AutoProcessor.from_pretrained("physical-intelligence/fast", trust_remote_code=True)
|
||||
|
||||
# convert action_chunks array to list of arrays (expected by .fit())
|
||||
action_data_list = [action_chunks[i] for i in range(len(action_chunks))]
|
||||
|
||||
# train the new tokenizer on our action data using .fit()
|
||||
# this trains the BPE tokenizer on DCT coefficients
|
||||
print("Training new tokenizer (this may take a few minutes)...")
|
||||
tokenizer = base_tokenizer.fit(
|
||||
action_data_list,
|
||||
scale=scale,
|
||||
vocab_size=vocab_size,
|
||||
time_horizon=action_chunks.shape[1], # action_horizon
|
||||
action_dim=action_chunks.shape[2], # encoded dimensions
|
||||
)
|
||||
print("✓ Tokenizer training complete!")
|
||||
|
||||
# validate it works
|
||||
sample_chunk = action_chunks[0]
|
||||
encoded = tokenizer(sample_chunk[None])[0]
|
||||
if isinstance(encoded, list):
|
||||
encoded = np.array(encoded)
|
||||
print(f"Sample encoding: {len(encoded)} tokens for chunk shape {sample_chunk.shape}")
|
||||
|
||||
return tokenizer
|
||||
|
||||
|
||||
def compute_compression_stats(tokenizer, action_chunks: np.ndarray):
|
||||
"""Compute compression statistics."""
|
||||
print("\nComputing compression statistics...")
|
||||
|
||||
# sample for stats (use max 1000 chunks for speed)
|
||||
sample_size = min(1000, len(action_chunks))
|
||||
sample_indices = np.random.RandomState(42).choice(len(action_chunks), size=sample_size, replace=False)
|
||||
sample_chunks = action_chunks[sample_indices]
|
||||
|
||||
token_lengths = []
|
||||
for chunk in sample_chunks:
|
||||
encoded = tokenizer(chunk[None])[0]
|
||||
if isinstance(encoded, list):
|
||||
token_lengths.append(len(encoded))
|
||||
else:
|
||||
token_lengths.append(encoded.shape[0] if hasattr(encoded, "shape") else len(encoded))
|
||||
|
||||
token_lengths = np.array(token_lengths)
|
||||
|
||||
# compression ratio: (H * D) / avg_tokens
|
||||
input_size = action_chunks.shape[1] * action_chunks.shape[2]
|
||||
avg_tokens = np.mean(token_lengths)
|
||||
compression_ratio = input_size / avg_tokens
|
||||
|
||||
stats = {
|
||||
"compression_ratio": float(compression_ratio),
|
||||
"mean_token_length": float(np.mean(token_lengths)),
|
||||
"p99_token_length": float(np.percentile(token_lengths, 99)),
|
||||
"min_token_length": float(np.min(token_lengths)),
|
||||
"max_token_length": float(np.max(token_lengths)),
|
||||
}
|
||||
|
||||
print("Compression Statistics:")
|
||||
print(f" Average compression ratio: {stats['compression_ratio']:.2f}x")
|
||||
print(f" Mean token length: {stats['mean_token_length']:.1f}")
|
||||
print(f" P99 token length: {stats['p99_token_length']:.0f}")
|
||||
print(f" Min token length: {stats['min_token_length']:.0f}")
|
||||
print(f" Max token length: {stats['max_token_length']:.0f}")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def train_tokenizer(cfg: TokenizerTrainingConfig):
|
||||
"""
|
||||
Train FAST tokenizer for action encoding.
|
||||
|
||||
Args:
|
||||
cfg: TokenizerTrainingConfig dataclass with all configuration parameters
|
||||
"""
|
||||
# load dataset
|
||||
print(f"Loading dataset: {cfg.repo_id}")
|
||||
dataset = LeRobotDataset(repo_id=cfg.repo_id, root=cfg.root)
|
||||
print(f"Dataset loaded: {dataset.num_episodes} episodes, {dataset.num_frames} frames")
|
||||
|
||||
# parse normalization mode
|
||||
try:
|
||||
norm_mode = NormalizationMode(cfg.normalization_mode)
|
||||
except ValueError as err:
|
||||
raise ValueError(
|
||||
f"Invalid normalization_mode: {cfg.normalization_mode}. "
|
||||
f"Must be one of: {', '.join([m.value for m in NormalizationMode])}"
|
||||
) from err
|
||||
print(f"Normalization mode: {norm_mode.value}")
|
||||
|
||||
# parse encoded dimensions
|
||||
encoded_dim_ranges = []
|
||||
for range_str in cfg.encoded_dims.split(","):
|
||||
start, end = map(int, range_str.strip().split(":"))
|
||||
encoded_dim_ranges.append((start, end))
|
||||
|
||||
total_encoded_dims = sum(end - start for start, end in encoded_dim_ranges)
|
||||
print(f"Encoding {total_encoded_dims} dimensions: {cfg.encoded_dims}")
|
||||
|
||||
# parse delta dimensions
|
||||
delta_dim_list = None
|
||||
if cfg.delta_dims is not None and cfg.delta_dims.strip():
|
||||
delta_dim_list = [int(d.strip()) for d in cfg.delta_dims.split(",")]
|
||||
print(f"Delta dimensions: {delta_dim_list}")
|
||||
else:
|
||||
print("No delta dimensions specified")
|
||||
|
||||
print(f"Use delta transform: {cfg.use_delta_transform}")
|
||||
if cfg.use_delta_transform and (delta_dim_list is None or len(delta_dim_list) == 0):
|
||||
print("Warning: use_delta_transform=True but no delta_dims specified. No delta will be applied.")
|
||||
|
||||
print(f"Action horizon: {cfg.action_horizon}")
|
||||
print(f"State key: {cfg.state_key}")
|
||||
|
||||
# determine episodes to process
|
||||
num_episodes = dataset.num_episodes
|
||||
if cfg.max_episodes is not None:
|
||||
num_episodes = min(cfg.max_episodes, num_episodes)
|
||||
|
||||
print(f"Processing {num_episodes} episodes...")
|
||||
|
||||
# process episodes sequentially (to avoid pickling issues with dataset)
|
||||
all_chunks = []
|
||||
for ep_idx in range(num_episodes):
|
||||
if ep_idx % 10 == 0:
|
||||
print(f" Processing episode {ep_idx}/{num_episodes}...")
|
||||
|
||||
chunks = process_episode(
|
||||
(
|
||||
dataset,
|
||||
ep_idx,
|
||||
cfg.action_horizon,
|
||||
delta_dim_list,
|
||||
cfg.sample_fraction,
|
||||
cfg.state_key,
|
||||
cfg.use_delta_transform,
|
||||
)
|
||||
)
|
||||
if chunks is not None:
|
||||
all_chunks.append(chunks)
|
||||
|
||||
# concatenate all chunks
|
||||
all_chunks = np.concatenate(all_chunks, axis=0)
|
||||
print(f"Collected {len(all_chunks)} action chunks")
|
||||
|
||||
# extract only encoded dimensions FIRST (before normalization)
|
||||
encoded_chunks = []
|
||||
for start, end in encoded_dim_ranges:
|
||||
encoded_chunks.append(all_chunks[:, :, start:end])
|
||||
encoded_chunks = np.concatenate(encoded_chunks, axis=-1) # [N, H, D_encoded]
|
||||
print(f"Extracted {encoded_chunks.shape[-1]} encoded dimensions")
|
||||
|
||||
# apply normalization to encoded dimensions
|
||||
print("\nBefore normalization - overall stats:")
|
||||
print(f" Min: {np.min(encoded_chunks):.4f}, Max: {np.max(encoded_chunks):.4f}")
|
||||
print(f" Mean: {np.mean(encoded_chunks):.4f}, Std: {np.std(encoded_chunks):.4f}")
|
||||
|
||||
# get normalization stats from dataset
|
||||
norm_stats = dataset.meta.stats
|
||||
if norm_stats is not None and ACTION in norm_stats:
|
||||
action_stats = norm_stats[ACTION]
|
||||
|
||||
# build encoded dimension indices
|
||||
encoded_dim_indices = []
|
||||
for start, end in encoded_dim_ranges:
|
||||
encoded_dim_indices.extend(range(start, end))
|
||||
encoded_dim_indices = np.array(encoded_dim_indices)
|
||||
|
||||
# extract stats for encoded dimensions only
|
||||
encoded_stats = {}
|
||||
for stat_name, stat_values in action_stats.items():
|
||||
if isinstance(stat_values, (list, np.ndarray)):
|
||||
stat_array = np.array(stat_values)
|
||||
if len(stat_array) > max(encoded_dim_indices):
|
||||
encoded_stats[stat_name] = stat_array[encoded_dim_indices]
|
||||
|
||||
if encoded_stats:
|
||||
print(f"\nNormalization stats for encoded dimensions (mode: {norm_mode.value}):")
|
||||
for stat_name, stat_values in encoded_stats.items():
|
||||
print(
|
||||
f" {stat_name}: shape={stat_values.shape}, "
|
||||
f"range=[{np.min(stat_values):.4f}, {np.max(stat_values):.4f}]"
|
||||
)
|
||||
|
||||
# apply normalization based on mode
|
||||
try:
|
||||
encoded_chunks = apply_normalization(encoded_chunks, encoded_stats, norm_mode, eps=1e-8)
|
||||
print(f"\nApplied {norm_mode.value} normalization")
|
||||
except ValueError as e:
|
||||
print(f"Warning: {e}. Using raw actions without normalization.")
|
||||
|
||||
print("\nAfter normalization - overall stats:")
|
||||
print(f" Min: {np.min(encoded_chunks):.4f}, Max: {np.max(encoded_chunks):.4f}")
|
||||
print(f" Mean: {np.mean(encoded_chunks):.4f}, Std: {np.std(encoded_chunks):.4f}")
|
||||
|
||||
print("\nPer-dimension stats (after normalization):")
|
||||
for d in range(encoded_chunks.shape[-1]):
|
||||
dim_data = encoded_chunks[:, :, d]
|
||||
print(
|
||||
f" Dim {d}: min={np.min(dim_data):7.4f}, max={np.max(dim_data):7.4f}, "
|
||||
f"mean={np.mean(dim_data):7.4f}, std={np.std(dim_data):7.4f}"
|
||||
)
|
||||
else:
|
||||
print("Warning: Could not extract stats for encoded dimensions, using raw actions")
|
||||
else:
|
||||
print("Warning: No normalization stats found in dataset, using raw actions")
|
||||
|
||||
print(f"Encoded chunks shape: {encoded_chunks.shape}")
|
||||
|
||||
# train FAST tokenizer
|
||||
tokenizer = train_fast_tokenizer(
|
||||
encoded_chunks,
|
||||
vocab_size=cfg.vocab_size,
|
||||
scale=cfg.scale,
|
||||
)
|
||||
|
||||
# compute compression statistics
|
||||
compression_stats = compute_compression_stats(tokenizer, encoded_chunks)
|
||||
|
||||
# save tokenizer
|
||||
output_dir = cfg.output_dir
|
||||
if output_dir is None:
|
||||
output_dir = f"fast_tokenizer_{cfg.repo_id.replace('/', '_')}"
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tokenizer.save_pretrained(output_path)
|
||||
|
||||
# save metadata
|
||||
metadata = {
|
||||
"repo_id": cfg.repo_id,
|
||||
"vocab_size": cfg.vocab_size,
|
||||
"scale": cfg.scale,
|
||||
"encoded_dims": cfg.encoded_dims,
|
||||
"encoded_dim_ranges": encoded_dim_ranges,
|
||||
"total_encoded_dims": total_encoded_dims,
|
||||
"delta_dims": cfg.delta_dims,
|
||||
"delta_dim_list": delta_dim_list,
|
||||
"use_delta_transform": cfg.use_delta_transform,
|
||||
"state_key": cfg.state_key,
|
||||
"normalization_mode": norm_mode.value,
|
||||
"action_horizon": cfg.action_horizon,
|
||||
"num_training_chunks": len(encoded_chunks),
|
||||
"compression_stats": compression_stats,
|
||||
}
|
||||
|
||||
with open(output_path / "metadata.json", "w") as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
print(f"\nSaved FAST tokenizer to {output_path}")
|
||||
print(f"Metadata: {json.dumps(metadata, indent=2)}")
|
||||
|
||||
# push to Hugging Face Hub if requested
|
||||
if cfg.push_to_hub:
|
||||
# determine the hub repository ID
|
||||
hub_repo_id = cfg.hub_repo_id
|
||||
if hub_repo_id is None:
|
||||
hub_repo_id = output_path.name
|
||||
print(f"\nNo hub_repo_id provided, using: {hub_repo_id}")
|
||||
|
||||
print(f"\nPushing tokenizer to Hugging Face Hub: {hub_repo_id}")
|
||||
print(f" Private: {cfg.hub_private}")
|
||||
|
||||
try:
|
||||
# use the tokenizer's push_to_hub method
|
||||
tokenizer.push_to_hub(
|
||||
repo_id=hub_repo_id,
|
||||
private=cfg.hub_private,
|
||||
commit_message=f"Upload FAST tokenizer trained on {cfg.repo_id}",
|
||||
)
|
||||
|
||||
# also upload the metadata.json file separately
|
||||
api = HfApi()
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(output_path / "metadata.json"),
|
||||
path_in_repo="metadata.json",
|
||||
repo_id=hub_repo_id,
|
||||
repo_type="model",
|
||||
commit_message="Upload tokenizer metadata",
|
||||
)
|
||||
|
||||
print(f"Successfully pushed tokenizer to: https://huggingface.co/{hub_repo_id}")
|
||||
except Exception as e:
|
||||
print(f"Error pushing to hub: {e}")
|
||||
print(" Make sure you're logged in with `huggingface-cli login`")
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point that parses arguments and runs the tokenizer training."""
|
||||
train_tokenizer()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,720 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
import torch
|
||||
from accelerate import Accelerator
|
||||
from accelerate.utils import DistributedDataParallelKwargs
|
||||
from torch.utils.data import DataLoader, Sampler
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.value import ValueInferencePipelineConfig
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.datasets.utils import load_info, write_info
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.utils.constants import (
|
||||
CHECKPOINTS_DIR,
|
||||
LAST_CHECKPOINT_LINK,
|
||||
PRETRAINED_MODEL_DIR,
|
||||
)
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.random_utils import set_seed
|
||||
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
|
||||
from lerobot.utils.utils import init_logging, inside_slurm
|
||||
from lerobot.scripts.value_infer_viz import (
|
||||
_export_overlay_videos,
|
||||
_get_episode_value_bounds,
|
||||
_get_video_encode_options,
|
||||
)
|
||||
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
|
||||
from lerobot.values.pistar06.modeling_pistar06 import (
|
||||
EpisodeTargetInfo,
|
||||
compute_normalized_value_targets,
|
||||
)
|
||||
|
||||
|
||||
class ContiguousDistributedEvalSampler(Sampler[int]):
|
||||
"""Distributed eval sampler with contiguous per-rank shards and deterministic tail padding."""
|
||||
|
||||
def __init__(self, dataset_size: int, num_replicas: int, rank: int):
|
||||
if dataset_size <= 0:
|
||||
raise ValueError(f"'dataset_size' must be > 0, got {dataset_size}.")
|
||||
if num_replicas <= 0:
|
||||
raise ValueError(f"'num_replicas' must be > 0, got {num_replicas}.")
|
||||
if rank < 0 or rank >= num_replicas:
|
||||
raise ValueError(f"'rank' must be in [0, {num_replicas - 1}], got {rank}.")
|
||||
|
||||
self.dataset_size = int(dataset_size)
|
||||
self.num_replicas = int(num_replicas)
|
||||
self.rank = int(rank)
|
||||
self.num_samples = int(math.ceil(self.dataset_size / self.num_replicas))
|
||||
self.total_size = self.num_samples * self.num_replicas
|
||||
|
||||
def __iter__(self):
|
||||
shard_start = self.rank * self.num_samples
|
||||
shard_end = min(shard_start + self.num_samples, self.dataset_size)
|
||||
if shard_start >= self.dataset_size:
|
||||
indices: list[int] = []
|
||||
else:
|
||||
indices = list(range(shard_start, shard_end))
|
||||
|
||||
if len(indices) < self.num_samples:
|
||||
indices.extend([self.dataset_size - 1] * (self.num_samples - len(indices)))
|
||||
return iter(indices)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self.num_samples
|
||||
|
||||
|
||||
def _set_infer_logger_levels() -> None:
|
||||
for logger_name in ["fsspec", "fsspec.local", "huggingface_hub", "datasets", "torchcodec"]:
|
||||
logging.getLogger(logger_name).setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def _create_accelerator(cfg: ValueInferencePipelineConfig, accelerator: Accelerator | None) -> Accelerator:
|
||||
if accelerator is not None:
|
||||
return accelerator
|
||||
|
||||
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)
|
||||
force_cpu = cfg.runtime.device == "cpu"
|
||||
return Accelerator(step_scheduler_with_optimizer=False, kwargs_handlers=[ddp_kwargs], cpu=force_cpu)
|
||||
|
||||
|
||||
def _resolve_pretrained_model_dir(checkpoint_path: str, checkpoint_ref: str) -> Path:
|
||||
path = Path(checkpoint_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Checkpoint path not found: {path}")
|
||||
|
||||
if (path / "model.safetensors").is_file() and (path / "config.json").is_file():
|
||||
return path
|
||||
|
||||
if (path / PRETRAINED_MODEL_DIR / "model.safetensors").is_file() and (
|
||||
path / PRETRAINED_MODEL_DIR / "config.json"
|
||||
).is_file():
|
||||
return path / PRETRAINED_MODEL_DIR
|
||||
|
||||
checkpoints_root = path / CHECKPOINTS_DIR if (path / CHECKPOINTS_DIR).is_dir() else path
|
||||
step_ref = LAST_CHECKPOINT_LINK if checkpoint_ref == "last" else checkpoint_ref
|
||||
step_dir = checkpoints_root / step_ref
|
||||
|
||||
if (step_dir / PRETRAINED_MODEL_DIR / "model.safetensors").is_file() and (
|
||||
step_dir / PRETRAINED_MODEL_DIR / "config.json"
|
||||
).is_file():
|
||||
return step_dir / PRETRAINED_MODEL_DIR
|
||||
|
||||
if (step_dir / "model.safetensors").is_file() and (step_dir / "config.json").is_file():
|
||||
return step_dir
|
||||
|
||||
raise FileNotFoundError(
|
||||
f"Could not resolve pretrained model directory from checkpoint_path={path} checkpoint_ref={checkpoint_ref}."
|
||||
)
|
||||
|
||||
|
||||
def _load_dataset_distributed(cfg: ValueInferencePipelineConfig, accelerator: Accelerator) -> LeRobotDataset:
|
||||
dataset_kwargs = {
|
||||
"repo_id": cfg.dataset.repo_id,
|
||||
"root": cfg.dataset.root,
|
||||
"episodes": cfg.dataset.episodes,
|
||||
"revision": cfg.dataset.revision,
|
||||
"download_videos": cfg.dataset.download_videos,
|
||||
}
|
||||
|
||||
if accelerator.is_main_process:
|
||||
dataset = LeRobotDataset(**dataset_kwargs)
|
||||
accelerator.wait_for_everyone()
|
||||
if not accelerator.is_main_process:
|
||||
dataset = LeRobotDataset(**dataset_kwargs)
|
||||
return dataset
|
||||
|
||||
|
||||
def _init_runtime(
|
||||
cfg: ValueInferencePipelineConfig,
|
||||
accelerator: Accelerator,
|
||||
) -> tuple[Path, torch.device]:
|
||||
output_dir = cfg.output_dir / "value"
|
||||
if accelerator.is_main_process:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
log_file = output_dir / "value_infer.log" if accelerator.is_main_process else None
|
||||
init_logging(log_file=log_file, file_level="INFO", accelerator=accelerator)
|
||||
_set_infer_logger_levels()
|
||||
|
||||
if accelerator.is_main_process:
|
||||
logging.info(pformat(cfg.to_dict()))
|
||||
|
||||
if cfg.seed is not None:
|
||||
set_seed(cfg.seed, accelerator=accelerator)
|
||||
|
||||
device = accelerator.device
|
||||
if device.type == "cuda":
|
||||
torch.backends.cudnn.benchmark = True
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
|
||||
return output_dir, device
|
||||
|
||||
|
||||
def _build_episode_info(
|
||||
dataset: LeRobotDataset,
|
||||
success_field: str,
|
||||
default_success: str,
|
||||
) -> tuple[dict[int, EpisodeTargetInfo], dict[int, int]]:
|
||||
episodes_ds = dataset.meta.episodes.with_format(None)
|
||||
episodes = episodes_ds[:]
|
||||
n_episodes = len(episodes_ds)
|
||||
has_success = success_field in episodes_ds.column_names
|
||||
|
||||
episode_info: dict[int, EpisodeTargetInfo] = {}
|
||||
task_max_length: dict[int, int] = {}
|
||||
for i in range(n_episodes):
|
||||
ep_idx = int(episodes["episode_index"][i])
|
||||
ep_length = int(episodes["length"][i])
|
||||
tasks = episodes["tasks"][i]
|
||||
task_name = tasks[0] if isinstance(tasks, list) else tasks
|
||||
if task_name not in dataset.meta.tasks.index:
|
||||
raise KeyError(f"Episode {ep_idx} references unknown task '{task_name}'.")
|
||||
task_index = int(dataset.meta.tasks.loc[task_name].task_index)
|
||||
|
||||
explicit_success = episodes[success_field][i] if has_success else None
|
||||
resolved_success = resolve_episode_success_label(
|
||||
explicit_success,
|
||||
default_label=default_success,
|
||||
require_label=True,
|
||||
)
|
||||
ep_success = resolved_success == EPISODE_SUCCESS
|
||||
|
||||
episode_info[ep_idx] = EpisodeTargetInfo(
|
||||
episode_index=ep_idx,
|
||||
task_index=task_index,
|
||||
length=ep_length,
|
||||
success=ep_success,
|
||||
)
|
||||
task_max_length[task_index] = max(task_max_length.get(task_index, 0), ep_length)
|
||||
return episode_info, task_max_length
|
||||
|
||||
|
||||
def _compute_dense_rewards_from_targets(
|
||||
targets: np.ndarray,
|
||||
episode_indices: np.ndarray,
|
||||
frame_indices: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
rewards = np.zeros_like(targets, dtype=np.float32)
|
||||
n = targets.shape[0]
|
||||
|
||||
for i in range(n):
|
||||
is_next_in_episode = (
|
||||
i + 1 < n
|
||||
and episode_indices[i + 1] == episode_indices[i]
|
||||
and frame_indices[i + 1] == frame_indices[i] + 1
|
||||
)
|
||||
if is_next_in_episode:
|
||||
rewards[i] = float(targets[i] - targets[i + 1])
|
||||
else:
|
||||
rewards[i] = float(targets[i])
|
||||
|
||||
return rewards
|
||||
|
||||
|
||||
def _compute_n_step_advantages(
|
||||
rewards: np.ndarray,
|
||||
values: np.ndarray,
|
||||
episode_indices: np.ndarray,
|
||||
frame_indices: np.ndarray,
|
||||
n_step: int,
|
||||
) -> np.ndarray:
|
||||
if n_step <= 0:
|
||||
raise ValueError("'n_step' must be > 0.")
|
||||
|
||||
n = rewards.shape[0]
|
||||
advantages = np.zeros(n, dtype=np.float32)
|
||||
|
||||
for i in range(n):
|
||||
ep_i = episode_indices[i]
|
||||
fi = frame_indices[i]
|
||||
|
||||
discounted_sum = 0.0
|
||||
j = i
|
||||
steps = 0
|
||||
while steps < n_step and j < n:
|
||||
same_episode = episode_indices[j] == ep_i
|
||||
contiguous = frame_indices[j] == fi + steps
|
||||
if not same_episode or not contiguous:
|
||||
break
|
||||
|
||||
discounted_sum += float(rewards[j])
|
||||
steps += 1
|
||||
j += 1
|
||||
|
||||
if steps == n_step and j < n and episode_indices[j] == ep_i and frame_indices[j] == fi + n_step:
|
||||
bootstrap = float(values[j])
|
||||
else:
|
||||
bootstrap = 0.0
|
||||
|
||||
advantages[i] = float(discounted_sum + bootstrap - values[i])
|
||||
|
||||
return advantages
|
||||
|
||||
|
||||
def _compute_task_thresholds(
|
||||
task_indices: np.ndarray,
|
||||
advantages: np.ndarray,
|
||||
positive_ratio: float,
|
||||
) -> dict[int, float]:
|
||||
if not 0.0 <= positive_ratio <= 1.0:
|
||||
raise ValueError("'positive_ratio' must be within [0, 1].")
|
||||
|
||||
thresholds: dict[int, float] = {}
|
||||
quantile = 1.0 - positive_ratio
|
||||
|
||||
for task_idx in np.unique(task_indices):
|
||||
task_adv = advantages[task_indices == task_idx]
|
||||
if task_adv.size == 0:
|
||||
thresholds[int(task_idx)] = float("inf")
|
||||
else:
|
||||
thresholds[int(task_idx)] = float(np.quantile(task_adv, quantile))
|
||||
|
||||
return thresholds
|
||||
|
||||
|
||||
def _binarize_advantages(
|
||||
task_indices: np.ndarray,
|
||||
advantages: np.ndarray,
|
||||
thresholds: dict[int, float],
|
||||
interventions: np.ndarray,
|
||||
force_intervention_positive: bool,
|
||||
) -> np.ndarray:
|
||||
indicators = np.zeros_like(advantages, dtype=np.int64)
|
||||
|
||||
for i in range(advantages.shape[0]):
|
||||
task_idx = int(task_indices[i])
|
||||
thr = thresholds[task_idx]
|
||||
indicators[i] = 1 if float(advantages[i]) >= thr else 0
|
||||
|
||||
if force_intervention_positive:
|
||||
intervention_mask = interventions.astype(np.float32) > 0.5
|
||||
indicators[intervention_mask] = 1
|
||||
|
||||
return indicators
|
||||
|
||||
|
||||
def _update_feature_metadata(dataset_root: Path, feature_infos: dict[str, dict[str, Any]]) -> None:
|
||||
info = load_info(dataset_root)
|
||||
for feature_name, feature_info in feature_infos.items():
|
||||
info["features"][feature_name] = {
|
||||
"dtype": feature_info["dtype"],
|
||||
"shape": tuple(feature_info["shape"]),
|
||||
"names": feature_info.get("names"),
|
||||
}
|
||||
write_info(info, dataset_root)
|
||||
|
||||
|
||||
def _write_columns_in_place(
|
||||
dataset_root: Path,
|
||||
absolute_indices: np.ndarray,
|
||||
columns: dict[str, np.ndarray],
|
||||
feature_infos: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
if absolute_indices.ndim != 1:
|
||||
raise ValueError("'absolute_indices' must be rank-1.")
|
||||
|
||||
max_index = int(np.max(absolute_indices))
|
||||
selected = np.zeros(max_index + 1, dtype=np.bool_)
|
||||
selected[absolute_indices] = True
|
||||
|
||||
lookups: dict[str, np.ndarray] = {}
|
||||
for field, values in columns.items():
|
||||
lookup_dtype = np.float32 if feature_infos[field]["dtype"] == "float32" else np.int64
|
||||
lookup = np.zeros(max_index + 1, dtype=lookup_dtype)
|
||||
lookup[absolute_indices] = values.astype(lookup_dtype, copy=False)
|
||||
lookups[field] = lookup
|
||||
|
||||
data_files = sorted((dataset_root / "data").glob("chunk-*/file-*.parquet"))
|
||||
if not data_files:
|
||||
raise FileNotFoundError(f"No parquet data files found under {dataset_root / 'data'}")
|
||||
|
||||
for parquet_path in tqdm(data_files, desc="Writing annotations", leave=False):
|
||||
table = pq.read_table(parquet_path)
|
||||
idx_np = table["index"].to_numpy().astype(np.int64, copy=False)
|
||||
|
||||
in_range = (idx_np >= 0) & (idx_np <= max_index)
|
||||
in_subset = np.zeros_like(in_range)
|
||||
in_subset[in_range] = selected[idx_np[in_range]]
|
||||
|
||||
new_table = table
|
||||
for field, lookup in lookups.items():
|
||||
ftype = feature_infos[field]["dtype"]
|
||||
if ftype == "float32":
|
||||
default_value = np.nan
|
||||
target_dtype = np.float32
|
||||
pa_type = pa.float32()
|
||||
elif ftype == "int64":
|
||||
default_value = 0
|
||||
target_dtype = np.int64
|
||||
pa_type = pa.int64()
|
||||
else:
|
||||
raise ValueError(f"Unsupported annotation dtype '{ftype}' for field '{field}'.")
|
||||
|
||||
if field in new_table.schema.names:
|
||||
current = new_table[field].to_numpy().astype(target_dtype, copy=True)
|
||||
else:
|
||||
current = np.full(idx_np.shape[0], default_value, dtype=target_dtype)
|
||||
|
||||
if np.any(in_subset):
|
||||
subset_indices = idx_np[in_subset]
|
||||
current[in_subset] = lookup[subset_indices]
|
||||
|
||||
array = pa.array(current, type=pa_type)
|
||||
if field in new_table.schema.names:
|
||||
col_idx = new_table.schema.names.index(field)
|
||||
new_table = new_table.set_column(col_idx, field, array)
|
||||
else:
|
||||
new_table = new_table.append_column(field, array)
|
||||
|
||||
pq.write_table(new_table, parquet_path, compression="snappy")
|
||||
|
||||
_update_feature_metadata(dataset_root=dataset_root, feature_infos=feature_infos)
|
||||
|
||||
|
||||
def _load_value_policy_and_processors(
|
||||
cfg: ValueInferencePipelineConfig,
|
||||
dataset: LeRobotDataset,
|
||||
pretrained_dir: Path,
|
||||
device: torch.device,
|
||||
):
|
||||
value_cfg = PreTrainedConfig.from_pretrained(pretrained_dir)
|
||||
if not isinstance(value_cfg, Pistar06Config):
|
||||
raise ValueError(
|
||||
f"Unsupported value config type '{type(value_cfg)}'. lerobot-value-infer currently supports only 'pistar06'."
|
||||
)
|
||||
|
||||
value_cfg.pretrained_path = pretrained_dir
|
||||
value_cfg.device = device.type
|
||||
|
||||
value_policy = make_policy(
|
||||
cfg=value_cfg,
|
||||
ds_meta=dataset.meta,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
|
||||
preprocessor, _ = make_pre_post_processors(
|
||||
policy_cfg=value_cfg,
|
||||
pretrained_path=pretrained_dir,
|
||||
preprocessor_overrides={"device_processor": {"device": device.type}},
|
||||
)
|
||||
return value_policy, value_cfg, preprocessor
|
||||
|
||||
|
||||
def _export_visualization_outputs(
|
||||
dataset: LeRobotDataset,
|
||||
cfg: ValueInferencePipelineConfig,
|
||||
output_dir: Path,
|
||||
) -> list[str]:
|
||||
viz_output_dir = output_dir / "viz"
|
||||
written_videos = _export_overlay_videos(
|
||||
dataset=dataset,
|
||||
value_field=cfg.acp.value_field,
|
||||
advantage_field=cfg.acp.advantage_field,
|
||||
indicator_field=cfg.acp.indicator_field,
|
||||
viz_episodes=cfg.viz.episodes,
|
||||
video_key=cfg.viz.video_key,
|
||||
output_dir=viz_output_dir,
|
||||
overwrite=cfg.viz.overwrite,
|
||||
vcodec=cfg.viz.vcodec,
|
||||
frame_storage_mode=cfg.viz.frame_storage_mode,
|
||||
)
|
||||
logging.info("Exported %d overlay videos to %s", len(written_videos), viz_output_dir)
|
||||
return [str(path) for path in written_videos]
|
||||
|
||||
|
||||
def run_value_inference_pipeline(
|
||||
cfg: ValueInferencePipelineConfig,
|
||||
accelerator: Accelerator | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg.validate()
|
||||
|
||||
accelerator = _create_accelerator(cfg, accelerator)
|
||||
output_dir, device = _init_runtime(cfg, accelerator)
|
||||
|
||||
dataset = _load_dataset_distributed(cfg, accelerator)
|
||||
raw_frames = dataset.hf_dataset.with_format(None)
|
||||
frame_count = len(raw_frames)
|
||||
if frame_count == 0:
|
||||
raise ValueError("Dataset has no frames.")
|
||||
|
||||
if not cfg.acp.enable:
|
||||
viz_outputs: list[str] = []
|
||||
if accelerator.is_main_process:
|
||||
logging.info(
|
||||
"ACP disabled; skipping value inference and reusing existing '%s' annotations.",
|
||||
cfg.acp.value_field,
|
||||
)
|
||||
if cfg.acp.value_field not in raw_frames.column_names:
|
||||
raise KeyError(
|
||||
f"Missing value field '{cfg.acp.value_field}' in dataset while 'acp.enable=false'."
|
||||
)
|
||||
if cfg.viz.enable:
|
||||
viz_outputs = _export_visualization_outputs(dataset=dataset, cfg=cfg, output_dir=output_dir)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
if not accelerator.is_main_process:
|
||||
result: dict[str, Any] = {
|
||||
"main_process": False,
|
||||
"world_size": int(accelerator.num_processes),
|
||||
}
|
||||
accelerator.end_training()
|
||||
return result
|
||||
|
||||
result = {
|
||||
"main_process": True,
|
||||
"world_size": int(accelerator.num_processes),
|
||||
"num_frames": int(frame_count),
|
||||
"checkpoint": None,
|
||||
"value_field": cfg.acp.value_field,
|
||||
"acp_enabled": False,
|
||||
"value_inference_skipped": True,
|
||||
"indicator_positive_ratio": None,
|
||||
"thresholds": None,
|
||||
"viz_outputs": viz_outputs,
|
||||
}
|
||||
accelerator.end_training()
|
||||
return result
|
||||
|
||||
pretrained_dir = _resolve_pretrained_model_dir(
|
||||
checkpoint_path=cfg.inference.checkpoint_path,
|
||||
checkpoint_ref=cfg.inference.checkpoint_ref,
|
||||
)
|
||||
value_policy, value_cfg, preprocessor = _load_value_policy_and_processors(
|
||||
cfg=cfg,
|
||||
dataset=dataset,
|
||||
pretrained_dir=pretrained_dir,
|
||||
device=device,
|
||||
)
|
||||
|
||||
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
|
||||
|
||||
if value_cfg.task_index_feature not in raw_frames.column_names:
|
||||
raise KeyError(f"Missing task feature '{value_cfg.task_index_feature}' in dataset columns.")
|
||||
|
||||
task_indices = np.asarray(raw_frames[value_cfg.task_index_feature], dtype=np.int64)
|
||||
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
|
||||
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
||||
|
||||
if cfg.acp.intervention_field in raw_frames.column_names:
|
||||
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
|
||||
else:
|
||||
interventions = np.zeros(frame_count, dtype=np.float32)
|
||||
|
||||
eval_loader = DataLoader(
|
||||
dataset,
|
||||
batch_size=cfg.runtime.batch_size,
|
||||
shuffle=False,
|
||||
num_workers=cfg.runtime.num_workers,
|
||||
pin_memory=(device.type == "cuda"),
|
||||
drop_last=False,
|
||||
)
|
||||
|
||||
value_policy = accelerator.prepare(value_policy)
|
||||
eval_loader = accelerator.prepare(eval_loader)
|
||||
|
||||
if accelerator.is_main_process:
|
||||
max_abs_index = int(np.max(absolute_indices))
|
||||
prediction_lookup = np.zeros(max_abs_index + 1, dtype=np.float32)
|
||||
prediction_seen = np.zeros(max_abs_index + 1, dtype=np.bool_)
|
||||
logging.info(
|
||||
"Start value inference | world_size=%d batches=%d batch_size=%d checkpoint=%s",
|
||||
accelerator.num_processes,
|
||||
len(eval_loader),
|
||||
cfg.runtime.batch_size,
|
||||
pretrained_dir,
|
||||
)
|
||||
else:
|
||||
prediction_lookup = None
|
||||
prediction_seen = None
|
||||
|
||||
value_policy.eval()
|
||||
eval_iter = tqdm(
|
||||
eval_loader,
|
||||
desc="Value inference",
|
||||
total=len(eval_loader),
|
||||
leave=False,
|
||||
disable=(not accelerator.is_main_process) or inside_slurm(),
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
for raw_batch in eval_iter:
|
||||
batch_indices = raw_batch["index"]
|
||||
if not isinstance(batch_indices, torch.Tensor):
|
||||
batch_indices = torch.as_tensor(batch_indices)
|
||||
batch_indices = batch_indices.to(device=device, dtype=torch.long, non_blocking=True)
|
||||
|
||||
processed_batch = preprocessor(raw_batch)
|
||||
with accelerator.autocast():
|
||||
predicted_value = accelerator.unwrap_model(value_policy).predict_value(processed_batch)
|
||||
|
||||
gathered_idx = accelerator.gather_for_metrics(batch_indices)
|
||||
gathered_val = accelerator.gather_for_metrics(predicted_value)
|
||||
|
||||
if accelerator.is_main_process:
|
||||
idx_np = gathered_idx.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
|
||||
val_np = gathered_val.detach().cpu().numpy().astype(np.float32, copy=False).reshape(-1)
|
||||
prediction_lookup[idx_np] = val_np
|
||||
prediction_seen[idx_np] = True
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
if accelerator.is_main_process:
|
||||
if prediction_lookup is None or prediction_seen is None:
|
||||
raise RuntimeError("Prediction buffers unexpectedly missing on main process.")
|
||||
|
||||
missing_mask = ~prediction_seen[absolute_indices]
|
||||
if bool(np.any(missing_mask)):
|
||||
missing_count = int(np.sum(missing_mask))
|
||||
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
|
||||
|
||||
predicted_values = prediction_lookup[absolute_indices]
|
||||
logging.info(
|
||||
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
|
||||
float(np.min(predicted_values)),
|
||||
float(np.max(predicted_values)),
|
||||
float(np.mean(predicted_values)),
|
||||
float(np.std(predicted_values)),
|
||||
)
|
||||
|
||||
columns: dict[str, np.ndarray] = {
|
||||
cfg.acp.value_field: predicted_values.astype(np.float32),
|
||||
}
|
||||
feature_infos: dict[str, dict[str, Any]] = {
|
||||
cfg.acp.value_field: {"dtype": "float32", "shape": (1,), "names": None},
|
||||
}
|
||||
|
||||
indicator_positive_ratio: float | None = None
|
||||
thresholds: dict[int, float] | None = None
|
||||
|
||||
if cfg.acp.enable:
|
||||
episode_info, task_max_lengths = _build_episode_info(
|
||||
dataset=dataset,
|
||||
success_field=cfg.dataset.success_field,
|
||||
default_success=cfg.dataset.default_success,
|
||||
)
|
||||
|
||||
value_targets = compute_normalized_value_targets(
|
||||
episode_indices=episode_indices,
|
||||
frame_indices=frame_indices,
|
||||
episode_info=episode_info,
|
||||
task_max_lengths=task_max_lengths,
|
||||
c_fail_coef=cfg.acp.c_fail_coef,
|
||||
clip_min=value_cfg.bin_min,
|
||||
clip_max=value_cfg.bin_max,
|
||||
)
|
||||
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
|
||||
advantages = _compute_n_step_advantages(
|
||||
rewards=rewards,
|
||||
values=predicted_values,
|
||||
episode_indices=episode_indices,
|
||||
frame_indices=frame_indices,
|
||||
n_step=cfg.acp.n_step,
|
||||
)
|
||||
thresholds = _compute_task_thresholds(
|
||||
task_indices=task_indices,
|
||||
advantages=advantages,
|
||||
positive_ratio=cfg.acp.positive_ratio,
|
||||
)
|
||||
indicators = _binarize_advantages(
|
||||
task_indices=task_indices,
|
||||
advantages=advantages,
|
||||
thresholds=thresholds,
|
||||
interventions=interventions,
|
||||
force_intervention_positive=cfg.acp.force_intervention_positive,
|
||||
)
|
||||
|
||||
indicator_positive_ratio = float(np.mean(indicators.astype(np.float32)))
|
||||
logging.info(
|
||||
"ACP stats | n_step=%d positive_ratio_target=%.4f positive_ratio_observed=%.4f",
|
||||
cfg.acp.n_step,
|
||||
cfg.acp.positive_ratio,
|
||||
indicator_positive_ratio,
|
||||
)
|
||||
|
||||
columns[cfg.acp.advantage_field] = advantages.astype(np.float32)
|
||||
columns[cfg.acp.indicator_field] = indicators.astype(np.int64)
|
||||
feature_infos[cfg.acp.advantage_field] = {"dtype": "float32", "shape": (1,), "names": None}
|
||||
feature_infos[cfg.acp.indicator_field] = {"dtype": "int64", "shape": (1,), "names": None}
|
||||
|
||||
_write_columns_in_place(
|
||||
dataset_root=Path(dataset.root),
|
||||
absolute_indices=absolute_indices,
|
||||
columns=columns,
|
||||
feature_infos=feature_infos,
|
||||
)
|
||||
|
||||
logging.info("Wrote value annotations to dataset root: %s", dataset.root)
|
||||
|
||||
# Sync computed columns into the in-memory hf_dataset so viz can read them
|
||||
for field, values in columns.items():
|
||||
if field in dataset.hf_dataset.column_names:
|
||||
dataset.hf_dataset = dataset.hf_dataset.remove_columns([field])
|
||||
dataset.hf_dataset = dataset.hf_dataset.add_column(field, values.tolist())
|
||||
|
||||
viz_outputs: list[str] = []
|
||||
if cfg.viz.enable:
|
||||
viz_outputs = _export_visualization_outputs(dataset=dataset, cfg=cfg, output_dir=output_dir)
|
||||
|
||||
result = {
|
||||
"main_process": True,
|
||||
"world_size": int(accelerator.num_processes),
|
||||
"num_frames": int(frame_count),
|
||||
"checkpoint": str(pretrained_dir),
|
||||
"value_field": cfg.acp.value_field,
|
||||
"acp_enabled": bool(cfg.acp.enable),
|
||||
"value_inference_skipped": False,
|
||||
"indicator_positive_ratio": indicator_positive_ratio,
|
||||
"thresholds": thresholds,
|
||||
"viz_outputs": viz_outputs,
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
"main_process": False,
|
||||
"world_size": int(accelerator.num_processes),
|
||||
}
|
||||
|
||||
# Keep non-main ranks alive until main rank finishes optional slow visualization export.
|
||||
accelerator.wait_for_everyone()
|
||||
accelerator.end_training()
|
||||
return result
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def value_infer(cfg: ValueInferencePipelineConfig):
|
||||
return run_value_inference_pipeline(cfg)
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
value_infer()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import logging
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from accelerate import Accelerator
|
||||
from termcolor import colored
|
||||
from torch.optim import Optimizer
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.value_train import ValueTrainPipelineConfig
|
||||
from lerobot.datasets.factory import make_dataset
|
||||
from lerobot.datasets.utils import cycle
|
||||
from lerobot.optim.factory import make_optimizer_and_scheduler
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.rl.wandb_utils import WandBLogger
|
||||
from lerobot.utils.import_utils import register_third_party_plugins
|
||||
from lerobot.utils.logging_utils import AverageMeter, MetricsTracker
|
||||
from lerobot.utils.random_utils import set_seed
|
||||
from lerobot.utils.train_utils import (
|
||||
get_step_checkpoint_dir,
|
||||
load_training_state,
|
||||
save_checkpoint,
|
||||
update_last_checkpoint,
|
||||
)
|
||||
from lerobot.utils.utils import format_big_number, has_method, init_logging
|
||||
|
||||
|
||||
def update_policy(
|
||||
train_metrics: MetricsTracker,
|
||||
policy: PreTrainedPolicy,
|
||||
batch: Any,
|
||||
optimizer: Optimizer,
|
||||
grad_clip_norm: float,
|
||||
accelerator: Accelerator,
|
||||
lr_scheduler=None,
|
||||
lock=None,
|
||||
) -> tuple[MetricsTracker, dict]:
|
||||
start_time = time.perf_counter()
|
||||
policy.train()
|
||||
|
||||
with accelerator.autocast():
|
||||
loss, output_dict = policy.forward(batch)
|
||||
|
||||
accelerator.backward(loss)
|
||||
|
||||
if grad_clip_norm > 0:
|
||||
grad_norm = accelerator.clip_grad_norm_(policy.parameters(), grad_clip_norm)
|
||||
else:
|
||||
grad_norm = torch.nn.utils.clip_grad_norm_(policy.parameters(), float("inf"), error_if_nonfinite=False)
|
||||
|
||||
with lock if lock is not None else nullcontext():
|
||||
optimizer.step()
|
||||
|
||||
optimizer.zero_grad()
|
||||
|
||||
if lr_scheduler is not None:
|
||||
lr_scheduler.step()
|
||||
|
||||
if has_method(accelerator.unwrap_model(policy, keep_fp32_wrapper=True), "update"):
|
||||
accelerator.unwrap_model(policy, keep_fp32_wrapper=True).update()
|
||||
|
||||
train_metrics.loss = loss.item()
|
||||
train_metrics.grad_norm = grad_norm.item()
|
||||
train_metrics.lr = optimizer.param_groups[0]["lr"]
|
||||
train_metrics.update_s = time.perf_counter() - start_time
|
||||
return train_metrics, output_dict
|
||||
|
||||
|
||||
@parser.wrap()
|
||||
def value_train(
|
||||
cfg: ValueTrainPipelineConfig,
|
||||
accelerator: Accelerator | None = None,
|
||||
):
|
||||
cfg.validate()
|
||||
|
||||
if accelerator is None:
|
||||
from accelerate.utils import DistributedDataParallelKwargs
|
||||
|
||||
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)
|
||||
force_cpu = cfg.value.device == "cpu"
|
||||
accelerator = Accelerator(
|
||||
step_scheduler_with_optimizer=False,
|
||||
kwargs_handlers=[ddp_kwargs],
|
||||
cpu=force_cpu,
|
||||
)
|
||||
|
||||
init_logging(accelerator=accelerator)
|
||||
is_main_process = accelerator.is_main_process
|
||||
|
||||
if is_main_process:
|
||||
logging.info(pformat(cfg.to_dict()))
|
||||
|
||||
if cfg.wandb.enable and cfg.wandb.project and is_main_process:
|
||||
wandb_logger = WandBLogger(cfg)
|
||||
else:
|
||||
wandb_logger = None
|
||||
if is_main_process:
|
||||
logging.info(colored("Logs will be saved locally.", "yellow", attrs=["bold"]))
|
||||
|
||||
if cfg.seed is not None:
|
||||
set_seed(cfg.seed, accelerator=accelerator)
|
||||
|
||||
device = accelerator.device
|
||||
torch.backends.cudnn.benchmark = True
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating dataset")
|
||||
dataset = make_dataset(cfg)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
if not is_main_process:
|
||||
dataset = make_dataset(cfg)
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating value model")
|
||||
value_model = make_policy(
|
||||
cfg=cfg.value,
|
||||
ds_meta=dataset.meta,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
|
||||
value_target_raw_batch_hook = None
|
||||
if has_method(value_model, "build_training_raw_batch_hook"):
|
||||
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
|
||||
dataset=dataset,
|
||||
targets_cfg=cfg.targets,
|
||||
)
|
||||
if is_main_process:
|
||||
logging.info("Using value model raw-batch hook for target construction.")
|
||||
elif is_main_process:
|
||||
logging.info("Value model does not define a raw-batch hook; using dataset targets as-is.")
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
processor_kwargs = {}
|
||||
postprocessor_kwargs = {}
|
||||
if (cfg.value.pretrained_path and not cfg.resume) or not cfg.value.pretrained_path:
|
||||
processor_kwargs["dataset_stats"] = dataset.meta.stats
|
||||
|
||||
if cfg.value.pretrained_path is not None:
|
||||
processor_kwargs["preprocessor_overrides"] = {
|
||||
"device_processor": {"device": device.type},
|
||||
"normalizer_processor": {
|
||||
"stats": dataset.meta.stats,
|
||||
"features": {**(cfg.value.input_features or {}), **(cfg.value.output_features or {})},
|
||||
"norm_map": cfg.value.normalization_mapping,
|
||||
},
|
||||
}
|
||||
processor_kwargs["preprocessor_overrides"]["rename_observations_processor"] = {
|
||||
"rename_map": cfg.rename_map
|
||||
}
|
||||
postprocessor_kwargs["postprocessor_overrides"] = {
|
||||
"unnormalizer_processor": {
|
||||
"stats": dataset.meta.stats,
|
||||
"features": cfg.value.output_features or {},
|
||||
"norm_map": cfg.value.normalization_mapping,
|
||||
},
|
||||
}
|
||||
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.value,
|
||||
pretrained_path=cfg.value.pretrained_path,
|
||||
**processor_kwargs,
|
||||
**postprocessor_kwargs,
|
||||
)
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating optimizer and scheduler")
|
||||
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, value_model)
|
||||
|
||||
step = 0
|
||||
if cfg.resume:
|
||||
if cfg.checkpoint_path is None:
|
||||
raise ValueError("'checkpoint_path' is missing while resume=true.")
|
||||
step, optimizer, lr_scheduler = load_training_state(cfg.checkpoint_path, optimizer, lr_scheduler)
|
||||
|
||||
num_learnable_params = sum(p.numel() for p in value_model.parameters() if p.requires_grad)
|
||||
num_total_params = sum(p.numel() for p in value_model.parameters())
|
||||
|
||||
if is_main_process:
|
||||
logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
|
||||
logging.info(f"{cfg.steps=} ({format_big_number(cfg.steps)})")
|
||||
logging.info(f"{dataset.num_frames=} ({format_big_number(dataset.num_frames)})")
|
||||
logging.info(f"{dataset.num_episodes=}")
|
||||
num_processes = accelerator.num_processes
|
||||
effective_bs = cfg.batch_size * num_processes
|
||||
logging.info(f"Effective batch size: {cfg.batch_size} x {num_processes} = {effective_bs}")
|
||||
logging.info(f"{num_learnable_params=} ({format_big_number(num_learnable_params)})")
|
||||
logging.info(f"{num_total_params=} ({format_big_number(num_total_params)})")
|
||||
|
||||
dataloader = torch.utils.data.DataLoader(
|
||||
dataset,
|
||||
num_workers=cfg.num_workers,
|
||||
batch_size=cfg.batch_size,
|
||||
shuffle=not cfg.dataset.streaming,
|
||||
sampler=None,
|
||||
pin_memory=device.type == "cuda",
|
||||
drop_last=False,
|
||||
prefetch_factor=2 if cfg.num_workers > 0 else None,
|
||||
)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
value_model, optimizer, dataloader, lr_scheduler = accelerator.prepare(
|
||||
value_model, optimizer, dataloader, lr_scheduler
|
||||
)
|
||||
dl_iter = cycle(dataloader)
|
||||
|
||||
value_model.train()
|
||||
|
||||
train_metrics = {
|
||||
"loss": AverageMeter("loss", ":.3f"),
|
||||
"grad_norm": AverageMeter("grdn", ":.3f"),
|
||||
"lr": AverageMeter("lr", ":0.1e"),
|
||||
"update_s": AverageMeter("updt_s", ":.3f"),
|
||||
"dataloading_s": AverageMeter("data_s", ":.3f"),
|
||||
}
|
||||
|
||||
effective_batch_size = cfg.batch_size * accelerator.num_processes
|
||||
train_tracker = MetricsTracker(
|
||||
effective_batch_size,
|
||||
dataset.num_frames,
|
||||
dataset.num_episodes,
|
||||
train_metrics,
|
||||
initial_step=step,
|
||||
accelerator=accelerator,
|
||||
)
|
||||
|
||||
if is_main_process:
|
||||
logging.info(
|
||||
f"Start value training on a fixed dataset, with effective batch size: {effective_batch_size}"
|
||||
)
|
||||
|
||||
logged_first_prompt = False
|
||||
|
||||
for _ in range(step, cfg.steps):
|
||||
start_time = time.perf_counter()
|
||||
batch = next(dl_iter)
|
||||
if value_target_raw_batch_hook is not None:
|
||||
batch = value_target_raw_batch_hook(batch, step)
|
||||
batch = preprocessor(batch)
|
||||
|
||||
if is_main_process and not logged_first_prompt and cfg.value.task_field in batch:
|
||||
task_batch = batch[cfg.value.task_field]
|
||||
if isinstance(task_batch, str):
|
||||
first_prompt = task_batch
|
||||
elif len(task_batch) > 0:
|
||||
first_prompt = task_batch[0]
|
||||
else:
|
||||
first_prompt = None
|
||||
if first_prompt is not None:
|
||||
logging.info("First value prompt:\n%s", first_prompt)
|
||||
logged_first_prompt = True
|
||||
|
||||
train_tracker.dataloading_s = time.perf_counter() - start_time
|
||||
|
||||
train_tracker, output_dict = update_policy(
|
||||
train_tracker,
|
||||
value_model,
|
||||
batch,
|
||||
optimizer,
|
||||
cfg.optimizer.grad_clip_norm,
|
||||
accelerator=accelerator,
|
||||
lr_scheduler=lr_scheduler,
|
||||
)
|
||||
|
||||
step += 1
|
||||
train_tracker.step()
|
||||
is_log_step = cfg.log_freq > 0 and step % cfg.log_freq == 0 and is_main_process
|
||||
is_saving_step = step % cfg.save_freq == 0 or step == cfg.steps
|
||||
|
||||
if is_log_step:
|
||||
logging.info(train_tracker)
|
||||
if wandb_logger:
|
||||
wandb_log_dict = train_tracker.to_dict()
|
||||
if output_dict:
|
||||
wandb_log_dict.update(output_dict)
|
||||
wandb_logger.log_dict(wandb_log_dict, step)
|
||||
train_tracker.reset_averages()
|
||||
|
||||
if cfg.save_checkpoint and is_saving_step:
|
||||
if is_main_process:
|
||||
logging.info(f"Checkpoint value model after step {step}")
|
||||
checkpoint_dir = get_step_checkpoint_dir(cfg.output_dir, cfg.steps, step)
|
||||
save_checkpoint(
|
||||
checkpoint_dir=checkpoint_dir,
|
||||
step=step,
|
||||
cfg=cfg,
|
||||
policy=accelerator.unwrap_model(value_model),
|
||||
optimizer=optimizer,
|
||||
scheduler=lr_scheduler,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
)
|
||||
update_last_checkpoint(checkpoint_dir)
|
||||
if wandb_logger:
|
||||
wandb_logger.log_policy(checkpoint_dir)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
if is_main_process:
|
||||
logging.info("End of value training")
|
||||
|
||||
if cfg.value.push_to_hub:
|
||||
unwrapped_value = accelerator.unwrap_model(value_model)
|
||||
if cfg.value.use_peft:
|
||||
unwrapped_value.push_model_to_hub(cfg, peft_model=unwrapped_value)
|
||||
else:
|
||||
unwrapped_value.push_model_to_hub(cfg)
|
||||
preprocessor.push_to_hub(cfg.value.repo_id)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
accelerator.end_training()
|
||||
|
||||
|
||||
def main():
|
||||
register_third_party_plugins()
|
||||
value_train()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
209
src/lerobot/scripts/recording_hil.py
普通文件
209
src/lerobot/scripts/recording_hil.py
普通文件
@ -0,0 +1,209 @@
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Human-in-loop recording helpers used by `lerobot_record.py`."""
|
||||
|
||||
import logging
|
||||
from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.processor import PolicyAction, PolicyProcessorPipeline, RobotAction
|
||||
from lerobot.rl.acp_tags import build_acp_tagged_task
|
||||
from lerobot.robots import Robot
|
||||
from lerobot.teleoperators import Teleoperator
|
||||
from lerobot.utils.control_utils import predict_action
|
||||
|
||||
|
||||
@dataclass
|
||||
class ACPInferenceConfig:
|
||||
enable: bool = False
|
||||
use_cfg: bool = False
|
||||
cfg_beta: float = 1.0
|
||||
|
||||
|
||||
POLICY_RUNTIME_STATE_KEYS = ("_action_queue", "_queues", "_prev_mean")
|
||||
|
||||
|
||||
INTERVENTION_STATE_POLICY = 0.0
|
||||
INTERVENTION_STATE_ACTIVE = 1.0
|
||||
INTERVENTION_STATE_RELEASE = 2.0
|
||||
|
||||
|
||||
def _get_torch_rng_state(device: torch.device) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
cpu_state = torch.get_rng_state()
|
||||
cuda_state = torch.cuda.get_rng_state(device) if device.type == "cuda" else None
|
||||
return cpu_state, cuda_state
|
||||
|
||||
|
||||
def _set_torch_rng_state(device: torch.device, cpu_state: torch.Tensor, cuda_state: torch.Tensor | None) -> None:
|
||||
torch.set_rng_state(cpu_state)
|
||||
if device.type == "cuda" and cuda_state is not None:
|
||||
torch.cuda.set_rng_state(cuda_state, device)
|
||||
|
||||
|
||||
def _clone_runtime_value(value: Any) -> Any:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.detach().clone()
|
||||
if isinstance(value, deque):
|
||||
return deque((_clone_runtime_value(item) for item in value), maxlen=value.maxlen)
|
||||
if isinstance(value, dict):
|
||||
return {key: _clone_runtime_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_clone_runtime_value(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_clone_runtime_value(item) for item in value)
|
||||
return deepcopy(value)
|
||||
|
||||
|
||||
def _capture_policy_runtime_state(policy: PreTrainedPolicy) -> dict[str, Any]:
|
||||
state: dict[str, Any] = {}
|
||||
for key in POLICY_RUNTIME_STATE_KEYS:
|
||||
if hasattr(policy, key):
|
||||
state[key] = _clone_runtime_value(getattr(policy, key))
|
||||
return state
|
||||
|
||||
|
||||
def _restore_policy_runtime_state(policy: PreTrainedPolicy, state: dict[str, Any]) -> None:
|
||||
for key, value in state.items():
|
||||
setattr(policy, key, _clone_runtime_value(value))
|
||||
|
||||
|
||||
def _predict_policy_action_with_runtime_state(
|
||||
*,
|
||||
observation_frame: dict[str, np.ndarray],
|
||||
policy: PreTrainedPolicy,
|
||||
device: torch.device,
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
use_amp: bool,
|
||||
task: str | None,
|
||||
robot_type: str | None,
|
||||
runtime_state: dict[str, Any],
|
||||
) -> PolicyAction:
|
||||
_restore_policy_runtime_state(policy, runtime_state)
|
||||
action = predict_action(
|
||||
observation=observation_frame,
|
||||
policy=policy,
|
||||
device=device,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
use_amp=use_amp,
|
||||
task=task,
|
||||
robot_type=robot_type,
|
||||
)
|
||||
runtime_state.clear()
|
||||
runtime_state.update(_capture_policy_runtime_state(policy))
|
||||
return action
|
||||
|
||||
|
||||
def _predict_policy_action_with_acp_inference(
|
||||
*,
|
||||
observation_frame: dict[str, np.ndarray],
|
||||
policy: PreTrainedPolicy,
|
||||
device: torch.device,
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction],
|
||||
use_amp: bool,
|
||||
task: str | None,
|
||||
robot_type: str | None,
|
||||
acp_inference: ACPInferenceConfig,
|
||||
cond_runtime_state: dict[str, Any] | None = None,
|
||||
uncond_runtime_state: dict[str, Any] | None = None,
|
||||
) -> PolicyAction:
|
||||
if not acp_inference.enable:
|
||||
return predict_action(
|
||||
observation=observation_frame,
|
||||
policy=policy,
|
||||
device=device,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
use_amp=use_amp,
|
||||
task=task,
|
||||
robot_type=robot_type,
|
||||
)
|
||||
|
||||
conditional_task = build_acp_tagged_task(task, is_positive=True)
|
||||
if not acp_inference.use_cfg:
|
||||
return predict_action(
|
||||
observation=observation_frame,
|
||||
policy=policy,
|
||||
device=device,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
use_amp=use_amp,
|
||||
task=conditional_task,
|
||||
robot_type=robot_type,
|
||||
)
|
||||
|
||||
if cond_runtime_state is None or uncond_runtime_state is None:
|
||||
raise ValueError("CFG inference requires cond/uncond runtime states.")
|
||||
|
||||
cpu_state, cuda_state = _get_torch_rng_state(device)
|
||||
action_cond = _predict_policy_action_with_runtime_state(
|
||||
observation_frame=observation_frame,
|
||||
policy=policy,
|
||||
device=device,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
use_amp=use_amp,
|
||||
task=conditional_task,
|
||||
robot_type=robot_type,
|
||||
runtime_state=cond_runtime_state,
|
||||
)
|
||||
_set_torch_rng_state(device, cpu_state, cuda_state)
|
||||
action_uncond = _predict_policy_action_with_runtime_state(
|
||||
observation_frame=observation_frame,
|
||||
policy=policy,
|
||||
device=device,
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
use_amp=use_amp,
|
||||
task=task,
|
||||
robot_type=robot_type,
|
||||
runtime_state=uncond_runtime_state,
|
||||
)
|
||||
return action_uncond + acp_inference.cfg_beta * (action_cond - action_uncond)
|
||||
|
||||
|
||||
class PolicySyncDualArmExecutor:
|
||||
"""Broadcast one policy-derived robot action to follower + teleop arm."""
|
||||
|
||||
def __init__(self, robot: Robot, teleop: Teleoperator, parallel_dispatch: bool = True):
|
||||
self.robot = robot
|
||||
self.teleop = teleop
|
||||
self.parallel_dispatch = parallel_dispatch
|
||||
self._pool = ThreadPoolExecutor(max_workers=2) if parallel_dispatch else None
|
||||
|
||||
def send_action(self, action: RobotAction) -> RobotAction:
|
||||
if self._pool is None:
|
||||
sent_action = self.robot.send_action(action)
|
||||
self.teleop.send_feedback(action)
|
||||
return sent_action
|
||||
|
||||
robot_future = self._pool.submit(self.robot.send_action, action)
|
||||
teleop_future = self._pool.submit(self.teleop.send_feedback, action)
|
||||
sent_action = robot_future.result()
|
||||
teleop_future.result()
|
||||
return sent_action
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self._pool is not None:
|
||||
self._pool.shutdown(wait=True)
|
||||
@ -0,0 +1,411 @@
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Core recording loop used by `lerobot_record.py`."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import numpy as np
|
||||
|
||||
from lerobot.datasets.image_writer import safe_stop_image_writer
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.datasets.utils import build_dataset_frame
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
from lerobot.policies.utils import make_robot_action
|
||||
from lerobot.processor import (
|
||||
PolicyAction,
|
||||
PolicyProcessorPipeline,
|
||||
RobotAction,
|
||||
RobotObservation,
|
||||
RobotProcessorPipeline,
|
||||
)
|
||||
from lerobot.robots import Robot
|
||||
from lerobot.scripts.recording_hil import (
|
||||
ACPInferenceConfig,
|
||||
INTERVENTION_STATE_ACTIVE,
|
||||
INTERVENTION_STATE_POLICY,
|
||||
INTERVENTION_STATE_RELEASE,
|
||||
PolicySyncDualArmExecutor,
|
||||
_capture_policy_runtime_state,
|
||||
_predict_policy_action_with_acp_inference,
|
||||
)
|
||||
from lerobot.teleoperators import Teleoperator, koch_leader, omx_leader, so_leader
|
||||
from lerobot.teleoperators.keyboard.teleop_keyboard import KeyboardTeleop
|
||||
from lerobot.utils.constants import ACTION, OBS_STR
|
||||
from lerobot.utils.recording_annotations import resolve_collector_policy_id
|
||||
from lerobot.utils.robot_utils import precise_sleep
|
||||
from lerobot.utils.utils import get_safe_torch_device
|
||||
from lerobot.utils.visualization_utils import log_rerun_data
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
""" --------------- record_loop() data flow --------------------------
|
||||
[ Robot ]
|
||||
V
|
||||
[ robot.get_observation() ] ---> raw_obs
|
||||
V
|
||||
[ robot_observation_processor ] ---> processed_obs
|
||||
V
|
||||
.-----( ACTION LOGIC )------------------.
|
||||
V V
|
||||
[ From Teleoperator ] [ From Policy ]
|
||||
| |
|
||||
| [teleop.get_action] -> raw_action | [predict_action]
|
||||
| | | |
|
||||
| V | V
|
||||
| [teleop_action_processor] | |
|
||||
| | | |
|
||||
'---> processed_teleop_action '---> processed_policy_action
|
||||
| |
|
||||
'-------------------------.-------------'
|
||||
V
|
||||
[ robot_action_processor ] --> robot_action_to_send
|
||||
V
|
||||
[ robot.send_action() ] -- (Robot Executes)
|
||||
V
|
||||
( Save to Dataset )
|
||||
V
|
||||
( Rerun Log / Loop Wait )
|
||||
"""
|
||||
|
||||
|
||||
@safe_stop_image_writer
|
||||
def record_loop(
|
||||
robot: Robot,
|
||||
events: dict,
|
||||
fps: int,
|
||||
teleop_action_processor: RobotProcessorPipeline[
|
||||
tuple[RobotAction, RobotObservation], RobotAction
|
||||
], # runs after teleop
|
||||
robot_action_processor: RobotProcessorPipeline[
|
||||
tuple[RobotAction, RobotObservation], RobotAction
|
||||
], # runs before robot
|
||||
robot_observation_processor: RobotProcessorPipeline[
|
||||
RobotObservation, RobotObservation
|
||||
], # runs after robot
|
||||
dataset: LeRobotDataset | None = None,
|
||||
teleop: Teleoperator | list[Teleoperator] | None = None,
|
||||
policy: PreTrainedPolicy | None = None,
|
||||
preprocessor: PolicyProcessorPipeline[dict[str, Any], dict[str, Any]] | None = None,
|
||||
postprocessor: PolicyProcessorPipeline[PolicyAction, PolicyAction] | None = None,
|
||||
control_time_s: int | None = None,
|
||||
single_task: str | None = None,
|
||||
display_data: bool = False,
|
||||
display_compressed_images: bool = False,
|
||||
policy_sync_executor: PolicySyncDualArmExecutor | None = None,
|
||||
intervention_state_machine_enabled: bool = True,
|
||||
collector_policy_id_policy: str = "policy",
|
||||
collector_policy_id_human: str = "human",
|
||||
acp_inference: ACPInferenceConfig | None = None,
|
||||
communication_retry_timeout_s: float = 2.0,
|
||||
communication_retry_interval_s: float = 0.1,
|
||||
):
|
||||
if acp_inference is None:
|
||||
acp_inference = ACPInferenceConfig()
|
||||
|
||||
if dataset is not None and dataset.fps != fps:
|
||||
raise ValueError(f"The dataset fps should be equal to requested fps ({dataset.fps} != {fps}).")
|
||||
|
||||
teleop_arm = teleop_keyboard = None
|
||||
if isinstance(teleop, list):
|
||||
teleop_keyboard = next((t for t in teleop if isinstance(t, KeyboardTeleop)), None)
|
||||
teleop_arm = next(
|
||||
(
|
||||
t
|
||||
for t in teleop
|
||||
if isinstance(
|
||||
t,
|
||||
(
|
||||
so_leader.SO100Leader
|
||||
| so_leader.SO101Leader
|
||||
| koch_leader.KochLeader
|
||||
| omx_leader.OmxLeader
|
||||
),
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if not (teleop_arm and teleop_keyboard and len(teleop) == 2 and robot.name == "lekiwi_client"):
|
||||
raise ValueError(
|
||||
"For multi-teleop, the list must contain exactly one KeyboardTeleop and one arm teleoperator. Currently only supported for LeKiwi robot."
|
||||
)
|
||||
|
||||
if dataset is None and policy is not None:
|
||||
raise ValueError("Policy-driven recording requires a dataset for feature mapping.")
|
||||
|
||||
action_feature_names = dataset.features[ACTION]["names"] if dataset is not None else None
|
||||
if action_feature_names is None:
|
||||
if hasattr(robot.action_features, "keys"):
|
||||
action_feature_names = list(robot.action_features.keys())
|
||||
else:
|
||||
action_feature_names = list(robot.action_features)
|
||||
zero_policy_action = {name: 0.0 for name in action_feature_names}
|
||||
has_teleop = isinstance(teleop, Teleoperator) or isinstance(teleop, list)
|
||||
intervention_enabled = intervention_state_machine_enabled and policy is not None and has_teleop
|
||||
intervention_state = INTERVENTION_STATE_POLICY
|
||||
last_teleop_action: RobotAction | None = None
|
||||
teleop_fallback_warned = False
|
||||
|
||||
teleop_arm_for_mode_switch: Any | None = None
|
||||
if isinstance(teleop, Teleoperator):
|
||||
teleop_arm_for_mode_switch = teleop
|
||||
elif isinstance(teleop, list):
|
||||
teleop_arm_for_mode_switch = teleop_arm
|
||||
|
||||
def set_teleop_manual_control(enabled: bool) -> None:
|
||||
if teleop_arm_for_mode_switch is None:
|
||||
return
|
||||
if not hasattr(teleop_arm_for_mode_switch, "set_manual_control"):
|
||||
return
|
||||
try:
|
||||
teleop_arm_for_mode_switch.set_manual_control(enabled)
|
||||
except Exception:
|
||||
logging.exception("Failed to switch teleop manual-control mode to %s", enabled)
|
||||
|
||||
if policy is None:
|
||||
# During reset/teleop-only loops keep leader backdrivable for manual dragging.
|
||||
set_teleop_manual_control(True)
|
||||
|
||||
# Reset policy and processor if they are provided
|
||||
if policy is not None and preprocessor is not None and postprocessor is not None:
|
||||
policy.reset()
|
||||
preprocessor.reset()
|
||||
postprocessor.reset()
|
||||
|
||||
cond_policy_runtime_state: dict[str, Any] | None = None
|
||||
uncond_policy_runtime_state: dict[str, Any] | None = None
|
||||
if policy is not None and acp_inference.enable and acp_inference.use_cfg:
|
||||
cond_policy_runtime_state = _capture_policy_runtime_state(policy)
|
||||
uncond_policy_runtime_state = _capture_policy_runtime_state(policy)
|
||||
|
||||
if intervention_enabled:
|
||||
# Start in S0: policy drives both arms, teleop arm should accept feedback commands.
|
||||
set_teleop_manual_control(False)
|
||||
|
||||
def run_with_connection_retry(action_name: str, fn: Callable[[], T]) -> T:
|
||||
timeout_s = max(communication_retry_timeout_s, 0.0)
|
||||
interval_s = max(communication_retry_interval_s, 0.0)
|
||||
deadline_t = time.perf_counter() + timeout_s
|
||||
attempts = 0
|
||||
first_error: ConnectionError | None = None
|
||||
|
||||
while True:
|
||||
attempts += 1
|
||||
try:
|
||||
result = fn()
|
||||
if attempts > 1:
|
||||
elapsed_s = timeout_s - max(deadline_t - time.perf_counter(), 0.0)
|
||||
logging.warning(
|
||||
"%s recovered after %d retries in %.2fs.",
|
||||
action_name,
|
||||
attempts - 1,
|
||||
elapsed_s,
|
||||
)
|
||||
return result
|
||||
except ConnectionError as error:
|
||||
if first_error is None:
|
||||
first_error = error
|
||||
logging.warning(
|
||||
"%s failed with transient communication error; retrying for up to %.2fs (%s)",
|
||||
action_name,
|
||||
timeout_s,
|
||||
error,
|
||||
)
|
||||
|
||||
if timeout_s <= 0.0:
|
||||
raise
|
||||
|
||||
remaining_s = deadline_t - time.perf_counter()
|
||||
if remaining_s <= 0.0:
|
||||
raise
|
||||
|
||||
sleep_s = interval_s if interval_s > 0.0 else remaining_s
|
||||
time.sleep(min(sleep_s, remaining_s))
|
||||
|
||||
timestamp = 0
|
||||
start_episode_t = time.perf_counter()
|
||||
while timestamp < control_time_s:
|
||||
start_loop_t = time.perf_counter()
|
||||
|
||||
if events["exit_early"]:
|
||||
events["exit_early"] = False
|
||||
break
|
||||
|
||||
if events.get("toggle_intervention", False):
|
||||
events["toggle_intervention"] = False
|
||||
if intervention_enabled:
|
||||
if intervention_state == INTERVENTION_STATE_POLICY:
|
||||
intervention_state = INTERVENTION_STATE_ACTIVE
|
||||
set_teleop_manual_control(True)
|
||||
logging.info("Intervention enabled (S1): teleop actions now override policy execution.")
|
||||
else:
|
||||
intervention_state = INTERVENTION_STATE_RELEASE
|
||||
set_teleop_manual_control(False)
|
||||
if policy is not None and preprocessor is not None and postprocessor is not None:
|
||||
policy.reset()
|
||||
preprocessor.reset()
|
||||
postprocessor.reset()
|
||||
if acp_inference.enable and acp_inference.use_cfg:
|
||||
cond_policy_runtime_state = _capture_policy_runtime_state(policy)
|
||||
uncond_policy_runtime_state = _capture_policy_runtime_state(policy)
|
||||
if policy is not None and preprocessor is not None and postprocessor is not None:
|
||||
logging.info("Policy cache reset on release: next policy action is recomputed.")
|
||||
logging.info("Intervention release requested (S2): returning control to policy.")
|
||||
else:
|
||||
logging.info("Intervention toggle ignored because policy+teleop are not both active.")
|
||||
|
||||
# Get robot observation
|
||||
obs = robot.get_observation()
|
||||
|
||||
# Applies a pipeline to the raw robot observation, default is IdentityProcessor
|
||||
obs_processed = robot_observation_processor(obs)
|
||||
|
||||
if dataset is not None:
|
||||
observation_frame = build_dataset_frame(dataset.features, obs_processed, prefix=OBS_STR)
|
||||
|
||||
# Get action from policy and/or teleop
|
||||
act_processed_policy: RobotAction | None = None
|
||||
act_processed_teleop: RobotAction | None = None
|
||||
if policy is not None and preprocessor is not None and postprocessor is not None:
|
||||
if not (intervention_enabled and intervention_state == INTERVENTION_STATE_ACTIVE):
|
||||
policy_action = _predict_policy_action_with_acp_inference(
|
||||
observation_frame=observation_frame,
|
||||
policy=policy,
|
||||
device=get_safe_torch_device(policy.config.device),
|
||||
preprocessor=preprocessor,
|
||||
postprocessor=postprocessor,
|
||||
use_amp=policy.config.use_amp,
|
||||
task=single_task,
|
||||
robot_type=robot.robot_type,
|
||||
acp_inference=acp_inference,
|
||||
cond_runtime_state=cond_policy_runtime_state,
|
||||
uncond_runtime_state=uncond_policy_runtime_state,
|
||||
)
|
||||
act_processed_policy = make_robot_action(policy_action, dataset.features)
|
||||
|
||||
if isinstance(teleop, Teleoperator):
|
||||
act = run_with_connection_retry("teleop.get_action", teleop.get_action)
|
||||
|
||||
# Applies a pipeline to the raw teleop action, default is IdentityProcessor
|
||||
act_processed_teleop = teleop_action_processor((act, obs))
|
||||
|
||||
elif isinstance(teleop, list):
|
||||
arm_action = run_with_connection_retry("teleop_arm.get_action", teleop_arm.get_action)
|
||||
arm_action = {f"arm_{k}": v for k, v in arm_action.items()}
|
||||
keyboard_action = teleop_keyboard.get_action()
|
||||
base_action = robot._from_keyboard_to_base_action(keyboard_action)
|
||||
act = {**arm_action, **base_action} if len(base_action) > 0 else arm_action
|
||||
act_processed_teleop = teleop_action_processor((act, obs))
|
||||
|
||||
if act_processed_policy is None and act_processed_teleop is None:
|
||||
logging.info(
|
||||
"No policy or teleoperator provided, skipping action generation."
|
||||
"This is likely to happen when resetting the environment without a teleop device."
|
||||
"The robot won't be at its rest position at the start of the next episode."
|
||||
)
|
||||
continue
|
||||
|
||||
if act_processed_teleop is not None:
|
||||
last_teleop_action = act_processed_teleop
|
||||
teleop_fallback_warned = False
|
||||
|
||||
policy_action_for_storage = act_processed_policy if act_processed_policy is not None else zero_policy_action
|
||||
|
||||
is_intervention = 0.0
|
||||
if intervention_enabled and intervention_state == INTERVENTION_STATE_ACTIVE:
|
||||
is_intervention = 1.0
|
||||
if act_processed_teleop is not None:
|
||||
action_values = act_processed_teleop
|
||||
elif last_teleop_action is not None:
|
||||
action_values = last_teleop_action
|
||||
if not teleop_fallback_warned:
|
||||
logging.warning(
|
||||
"Intervention is active but no fresh teleop action is available; reusing last teleop action."
|
||||
)
|
||||
teleop_fallback_warned = True
|
||||
elif act_processed_policy is not None:
|
||||
action_values = act_processed_policy
|
||||
if not teleop_fallback_warned:
|
||||
logging.warning(
|
||||
"Intervention is active but teleop action is unavailable; falling back to policy action."
|
||||
)
|
||||
teleop_fallback_warned = True
|
||||
else:
|
||||
action_values = zero_policy_action
|
||||
if not teleop_fallback_warned:
|
||||
logging.warning(
|
||||
"Intervention is active but no teleop/policy action is available; sending zero action."
|
||||
)
|
||||
teleop_fallback_warned = True
|
||||
else:
|
||||
action_values = act_processed_policy if act_processed_policy is not None else act_processed_teleop
|
||||
|
||||
# Applies a pipeline to the action, default is IdentityProcessor
|
||||
robot_action_to_send = robot_action_processor((action_values, obs))
|
||||
|
||||
# Send action to robot
|
||||
# Action can eventually be clipped using `max_relative_target`,
|
||||
# so action actually sent is saved in the dataset. action = postprocessor.process(action)
|
||||
# TODO(steven, pepijn, adil): we should use a pipeline step to clip the action, so the sent action is the action that we input to the robot.
|
||||
selected_from_policy = act_processed_policy is not None and action_values is act_processed_policy
|
||||
if policy_sync_executor is not None and selected_from_policy:
|
||||
_sent_action = run_with_connection_retry(
|
||||
"policy_sync_executor.send_action",
|
||||
lambda: policy_sync_executor.send_action(robot_action_to_send),
|
||||
)
|
||||
else:
|
||||
_sent_action = run_with_connection_retry(
|
||||
"robot.send_action",
|
||||
lambda: robot.send_action(robot_action_to_send),
|
||||
)
|
||||
|
||||
# Write to dataset
|
||||
if dataset is not None:
|
||||
action_frame = build_dataset_frame(dataset.features, action_values, prefix=ACTION)
|
||||
policy_action_frame = build_dataset_frame(
|
||||
dataset.features, policy_action_for_storage, prefix="complementary_info.policy_action"
|
||||
)
|
||||
frame = {**observation_frame, **action_frame, **policy_action_frame, "task": single_task}
|
||||
|
||||
if "complementary_info.is_intervention" in dataset.features:
|
||||
frame["complementary_info.is_intervention"] = np.array([is_intervention], dtype=np.float32)
|
||||
if "complementary_info.state" in dataset.features:
|
||||
frame["complementary_info.state"] = np.array([intervention_state], dtype=np.float32)
|
||||
if "complementary_info.collector_policy_id" in dataset.features:
|
||||
frame["complementary_info.collector_policy_id"] = resolve_collector_policy_id(
|
||||
intervention_enabled=intervention_enabled,
|
||||
is_intervention=bool(is_intervention),
|
||||
selected_from_policy=selected_from_policy,
|
||||
policy_id=collector_policy_id_policy,
|
||||
human_id=collector_policy_id_human,
|
||||
)
|
||||
dataset.add_frame(frame)
|
||||
|
||||
if display_data:
|
||||
log_rerun_data(
|
||||
observation=obs_processed, action=action_values, compress_images=display_compressed_images
|
||||
)
|
||||
|
||||
if intervention_state == INTERVENTION_STATE_RELEASE:
|
||||
intervention_state = INTERVENTION_STATE_POLICY
|
||||
|
||||
dt_s = time.perf_counter() - start_loop_t
|
||||
precise_sleep(max(1 / fps - dt_s, 0.0))
|
||||
|
||||
timestamp = time.perf_counter() - start_episode_t
|
||||
@ -0,0 +1,477 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from lerobot.datasets.video_utils import decode_video_frames, encode_video_frames
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
|
||||
|
||||
def _select_video_key(camera_keys: list[str], requested_video_key: str | None) -> str:
|
||||
if len(camera_keys) == 0:
|
||||
raise ValueError("No camera key found in dataset.")
|
||||
if requested_video_key is not None:
|
||||
if requested_video_key not in camera_keys:
|
||||
raise ValueError(f"Unknown video_key '{requested_video_key}'. Available camera keys: {camera_keys}")
|
||||
return requested_video_key
|
||||
|
||||
for key in camera_keys:
|
||||
lower = key.lower()
|
||||
if ".front" in lower or "_front" in lower or lower.endswith("front"):
|
||||
return key
|
||||
return camera_keys[0]
|
||||
|
||||
|
||||
def _parse_episodes_arg(episodes_arg: str, total_episodes: int) -> list[int]:
|
||||
value = episodes_arg.strip().lower()
|
||||
if value == "all":
|
||||
return list(range(total_episodes))
|
||||
|
||||
parsed: set[int] = set()
|
||||
for token in episodes_arg.split(","):
|
||||
part = token.strip()
|
||||
if not part:
|
||||
continue
|
||||
if "-" in part:
|
||||
start_str, end_str = part.split("-", maxsplit=1)
|
||||
start = int(start_str)
|
||||
end = int(end_str)
|
||||
if end < start:
|
||||
raise ValueError(f"Invalid episode range '{part}'.")
|
||||
parsed.update(range(start, end + 1))
|
||||
else:
|
||||
parsed.add(int(part))
|
||||
|
||||
episodes = sorted(parsed)
|
||||
for ep in episodes:
|
||||
if ep < 0 or ep >= total_episodes:
|
||||
raise ValueError(f"Episode index out of range: {ep}, total_episodes={total_episodes}.")
|
||||
return episodes
|
||||
|
||||
|
||||
def _to_1d_float(values: list | np.ndarray) -> np.ndarray:
|
||||
arr = np.asarray(values, dtype=np.float32)
|
||||
if arr.ndim > 1:
|
||||
arr = arr.reshape(arr.shape[0], -1)[:, 0]
|
||||
return arr.reshape(-1)
|
||||
|
||||
|
||||
def _to_1d_int(values: list | np.ndarray) -> np.ndarray:
|
||||
arr = np.asarray(values, dtype=np.int64)
|
||||
if arr.ndim > 1:
|
||||
arr = arr.reshape(arr.shape[0], -1)[:, 0]
|
||||
return arr.reshape(-1)
|
||||
|
||||
|
||||
def _load_font(size: int) -> ImageFont.ImageFont | ImageFont.FreeTypeFont:
|
||||
font_candidates = [
|
||||
Path("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"),
|
||||
Path("/usr/local/share/fonts/DejaVuSans.ttf"),
|
||||
]
|
||||
for font_path in font_candidates:
|
||||
if font_path.exists():
|
||||
return ImageFont.truetype(str(font_path), size=size)
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def _curve_points(
|
||||
values: np.ndarray,
|
||||
current_step: int,
|
||||
x0: int,
|
||||
y0: int,
|
||||
width: int,
|
||||
height: int,
|
||||
y_min: float,
|
||||
y_max: float,
|
||||
) -> list[tuple[int, int]]:
|
||||
n = len(values)
|
||||
if n == 0:
|
||||
return []
|
||||
|
||||
denom_x = max(1, n - 1)
|
||||
denom_y = max(1e-6, y_max - y_min)
|
||||
|
||||
points = []
|
||||
last_step = min(current_step, n - 1)
|
||||
for i in range(last_step + 1):
|
||||
x = int(round(x0 + width * (i / denom_x)))
|
||||
y_norm = np.clip((float(values[i]) - y_min) / denom_y, 0.0, 1.0)
|
||||
y = int(round(y0 + (1.0 - y_norm) * height))
|
||||
points.append((x, y))
|
||||
return points
|
||||
|
||||
|
||||
def _draw_overlay(
|
||||
frame: Image.Image,
|
||||
values: np.ndarray,
|
||||
current_step: int,
|
||||
advantage_t: float,
|
||||
acp_t: int,
|
||||
highlight_current_point: bool,
|
||||
y_min: float,
|
||||
y_max: float,
|
||||
indicators: np.ndarray | None = None,
|
||||
) -> Image.Image:
|
||||
rgba = frame.convert("RGBA")
|
||||
overlay = Image.new("RGBA", rgba.size, (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(overlay)
|
||||
|
||||
width, height = rgba.size
|
||||
margin = max(10, width // 80)
|
||||
chart_h = max(72, height // 4)
|
||||
chart_w = width - 2 * margin
|
||||
chart_x0 = margin
|
||||
chart_y0 = height - margin - chart_h
|
||||
|
||||
draw.rectangle(
|
||||
(chart_x0 - 4, chart_y0 - 4, chart_x0 + chart_w + 4, chart_y0 + chart_h + 4),
|
||||
fill=(0, 0, 0, 110),
|
||||
)
|
||||
draw.line((chart_x0, chart_y0, chart_x0, chart_y0 + chart_h), fill=(200, 200, 200, 160), width=1)
|
||||
draw.line(
|
||||
(chart_x0, chart_y0 + chart_h, chart_x0 + chart_w, chart_y0 + chart_h),
|
||||
fill=(200, 200, 200, 160),
|
||||
width=1,
|
||||
)
|
||||
|
||||
mid_y = chart_y0 + chart_h // 2
|
||||
draw.line((chart_x0, mid_y, chart_x0 + chart_w, mid_y), fill=(120, 120, 120, 120), width=1)
|
||||
|
||||
points = _curve_points(values, current_step, chart_x0, chart_y0, chart_w, chart_h, y_min, y_max)
|
||||
curve_width = max(2, width // 320)
|
||||
white = (255, 255, 255, 255)
|
||||
red = (255, 80, 80, 255)
|
||||
if len(points) >= 2:
|
||||
if indicators is not None:
|
||||
for i in range(len(points) - 1):
|
||||
seg_color = red if int(indicators[i]) == 1 or int(indicators[i + 1]) == 1 else white
|
||||
draw.line([points[i], points[i + 1]], fill=seg_color, width=curve_width)
|
||||
else:
|
||||
draw.line(points, fill=white, width=curve_width)
|
||||
point_color = red if highlight_current_point else white
|
||||
if len(points) == 1:
|
||||
x, y = points[0]
|
||||
draw.ellipse((x - curve_width, y - curve_width, x + curve_width, y + curve_width), fill=point_color)
|
||||
elif len(points) > 1:
|
||||
x, y = points[-1]
|
||||
radius = max(2, curve_width + 1)
|
||||
draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=point_color)
|
||||
|
||||
font_size = max(18, height // 26)
|
||||
font = _load_font(font_size)
|
||||
lines = [f"advantage: {advantage_t:+.4f}", f"acp_indicator: {int(acp_t)}"]
|
||||
line_sizes = [draw.textbbox((0, 0), text, font=font) for text in lines]
|
||||
text_w = max(box[2] - box[0] for box in line_sizes)
|
||||
text_h = sum(box[3] - box[1] for box in line_sizes) + max(4, font_size // 4)
|
||||
box_pad = max(8, font_size // 3)
|
||||
box_x1 = width - margin
|
||||
box_x0 = box_x1 - text_w - 2 * box_pad
|
||||
box_y0 = margin
|
||||
box_y1 = box_y0 + text_h + 2 * box_pad
|
||||
draw.rectangle((box_x0, box_y0, box_x1, box_y1), fill=(0, 0, 0, 150))
|
||||
|
||||
text_y = box_y0 + box_pad
|
||||
for idx, text in enumerate(lines):
|
||||
box = line_sizes[idx]
|
||||
line_h = box[3] - box[1]
|
||||
draw.text((box_x0 + box_pad, text_y), text, fill=(255, 255, 255, 255), font=font)
|
||||
text_y += line_h + max(4, font_size // 4)
|
||||
|
||||
return Image.alpha_composite(rgba, overlay).convert("RGB")
|
||||
|
||||
|
||||
def _build_output_video_path(output_dir: Path, repo_id: str, video_key: str, episode_index: int) -> Path:
|
||||
repo_tag = repo_id.replace("/", "_")
|
||||
key_tag = video_key.replace(".", "_")
|
||||
return output_dir / f"{repo_tag}_episode_{episode_index:04d}_{key_tag}.mp4"
|
||||
|
||||
|
||||
def _decode_frames_at_timestamps(
|
||||
video_file: Path,
|
||||
timestamps_s: np.ndarray,
|
||||
tolerance_s: float,
|
||||
backend: str | None,
|
||||
) -> list[Image.Image]:
|
||||
if timestamps_s.size == 0:
|
||||
return []
|
||||
frames = decode_video_frames(
|
||||
video_path=video_file,
|
||||
timestamps=timestamps_s.tolist(),
|
||||
tolerance_s=tolerance_s,
|
||||
backend=backend,
|
||||
)
|
||||
np_frames = frames.detach().cpu().numpy()
|
||||
if np_frames.ndim != 4:
|
||||
raise ValueError(f"Unexpected decoded frame tensor shape: {np_frames.shape}")
|
||||
if np_frames.shape[1] in (1, 3):
|
||||
np_frames = np.transpose(np_frames, (0, 2, 3, 1))
|
||||
if np_frames.dtype != np.uint8:
|
||||
if np.issubdtype(np_frames.dtype, np.floating):
|
||||
max_val = float(np.max(np_frames)) if np_frames.size > 0 else 1.0
|
||||
if max_val <= 1.0 + 1e-6:
|
||||
np_frames = np.clip(np_frames, 0.0, 1.0) * 255.0
|
||||
else:
|
||||
np_frames = np.clip(np_frames, 0.0, 255.0)
|
||||
else:
|
||||
np_frames = np.clip(np_frames, 0, 255)
|
||||
np_frames = np_frames.astype(np.uint8)
|
||||
return [Image.fromarray(np_frames[i]) for i in range(np_frames.shape[0])]
|
||||
|
||||
|
||||
def _get_episode_video_time_bounds(
|
||||
dataset: LeRobotDataset,
|
||||
episode_index: int,
|
||||
video_key: str,
|
||||
) -> tuple[float, float | None]:
|
||||
episodes = getattr(dataset.meta, "episodes", None)
|
||||
if episodes is None:
|
||||
return 0.0, None
|
||||
episodes_ds = episodes.with_format(None)
|
||||
if "episode_index" not in episodes_ds.column_names:
|
||||
return 0.0, None
|
||||
|
||||
episode_indices = np.asarray(episodes_ds["episode_index"], dtype=np.int64).reshape(-1)
|
||||
matched = np.flatnonzero(episode_indices == episode_index)
|
||||
if matched.size == 0:
|
||||
return 0.0, None
|
||||
row = int(matched[0])
|
||||
|
||||
from_col = f"videos/{video_key}/from_timestamp"
|
||||
to_col = f"videos/{video_key}/to_timestamp"
|
||||
from_ts = 0.0
|
||||
to_ts: float | None = None
|
||||
if from_col in episodes_ds.column_names:
|
||||
from_ts = float(episodes_ds[from_col][row])
|
||||
if to_col in episodes_ds.column_names:
|
||||
to_ts = float(episodes_ds[to_col][row])
|
||||
return from_ts, to_ts
|
||||
|
||||
|
||||
def _get_video_encode_options(vcodec: str) -> tuple[dict[str, str], str]:
|
||||
if vcodec == "libsvtav1":
|
||||
return {"g": "2", "crf": "30", "preset": "12"}, "yuv420p"
|
||||
if vcodec == "h264_nvenc":
|
||||
return {
|
||||
"preset": "p4",
|
||||
"rc": "vbr",
|
||||
"cq": "28",
|
||||
"b": "0",
|
||||
"g": "60",
|
||||
}, "yuv420p"
|
||||
return {"g": "2", "crf": "30"}, "yuv420p"
|
||||
|
||||
|
||||
def _get_episode_value_bounds(ep_values: np.ndarray) -> tuple[float, float]:
|
||||
if ep_values.shape[0] == 0:
|
||||
raise ValueError("Cannot determine value bounds from an empty episode.")
|
||||
return float(np.min(ep_values)), float(np.max(ep_values))
|
||||
|
||||
|
||||
def _encode_pil_to_video(
|
||||
frames: list[Image.Image],
|
||||
video_path: Path,
|
||||
fps: int,
|
||||
vcodec: str,
|
||||
) -> None:
|
||||
"""Encode PIL frames directly to video without writing intermediate PNGs."""
|
||||
import av as _av
|
||||
|
||||
video_options, pix_fmt = _get_video_encode_options(vcodec)
|
||||
|
||||
video_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _av.open(str(video_path), "w") as output:
|
||||
out_stream = output.add_stream(vcodec, fps, options=video_options)
|
||||
out_stream.pix_fmt = pix_fmt
|
||||
out_stream.width = frames[0].width
|
||||
out_stream.height = frames[0].height
|
||||
for pil_img in frames:
|
||||
av_frame = _av.VideoFrame.from_image(pil_img.convert("RGB"))
|
||||
for packet in out_stream.encode(av_frame):
|
||||
output.mux(packet)
|
||||
for packet in out_stream.encode():
|
||||
output.mux(packet)
|
||||
|
||||
|
||||
def _export_single_episode(
|
||||
src_video_path: Path,
|
||||
dst_video_path: Path,
|
||||
ep_values: np.ndarray,
|
||||
ep_advantages: np.ndarray,
|
||||
ep_indicators: np.ndarray,
|
||||
episode_timestamps_s: np.ndarray,
|
||||
fps: int,
|
||||
vcodec: str,
|
||||
tolerance_s: float,
|
||||
video_backend: str | None,
|
||||
frame_storage_mode: str = "memory",
|
||||
temp_dir_root: Path | None = None,
|
||||
) -> Path:
|
||||
y_min, y_max = _get_episode_value_bounds(ep_values)
|
||||
decoded_frames = _decode_frames_at_timestamps(
|
||||
video_file=src_video_path,
|
||||
timestamps_s=episode_timestamps_s,
|
||||
tolerance_s=tolerance_s,
|
||||
backend=video_backend,
|
||||
)
|
||||
n_frames = min(len(decoded_frames), len(ep_values))
|
||||
if n_frames == 0:
|
||||
raise ValueError(f"No decoded frames for video: {src_video_path}")
|
||||
|
||||
if frame_storage_mode == "disk":
|
||||
with tempfile.TemporaryDirectory(
|
||||
dir=str(temp_dir_root) if temp_dir_root is not None else None,
|
||||
prefix=f"{dst_video_path.stem}-frames-",
|
||||
) as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
for i in range(n_frames):
|
||||
frame = decoded_frames[i]
|
||||
composed = _draw_overlay(
|
||||
frame=frame,
|
||||
values=ep_values,
|
||||
current_step=i,
|
||||
advantage_t=float(ep_advantages[i]),
|
||||
acp_t=int(ep_indicators[i]),
|
||||
highlight_current_point=(int(ep_indicators[i]) == 1),
|
||||
y_min=y_min,
|
||||
y_max=y_max,
|
||||
indicators=ep_indicators,
|
||||
)
|
||||
composed.save(temp_path / f"frame-{i:06d}.png")
|
||||
|
||||
encode_video_frames(
|
||||
imgs_dir=temp_path,
|
||||
video_path=dst_video_path,
|
||||
fps=fps,
|
||||
vcodec=vcodec,
|
||||
overwrite=True,
|
||||
)
|
||||
return dst_video_path
|
||||
|
||||
composed_frames: list[Image.Image] = []
|
||||
for i in range(n_frames):
|
||||
composed = _draw_overlay(
|
||||
frame=decoded_frames[i],
|
||||
values=ep_values,
|
||||
current_step=i,
|
||||
advantage_t=float(ep_advantages[i]),
|
||||
acp_t=int(ep_indicators[i]),
|
||||
highlight_current_point=(int(ep_indicators[i]) == 1),
|
||||
y_min=y_min,
|
||||
y_max=y_max,
|
||||
indicators=ep_indicators,
|
||||
)
|
||||
composed_frames.append(composed)
|
||||
|
||||
_encode_pil_to_video(composed_frames, dst_video_path, fps, vcodec)
|
||||
return dst_video_path
|
||||
|
||||
|
||||
def _export_overlay_videos(
|
||||
dataset: LeRobotDataset,
|
||||
value_field: str,
|
||||
advantage_field: str,
|
||||
indicator_field: str,
|
||||
viz_episodes: str,
|
||||
video_key: str | None,
|
||||
output_dir: Path,
|
||||
overwrite: bool,
|
||||
vcodec: str,
|
||||
frame_storage_mode: str = "memory",
|
||||
) -> list[Path]:
|
||||
selected_video_key = _select_video_key(dataset.meta.camera_keys, video_key)
|
||||
|
||||
raw_dataset = dataset.hf_dataset.with_format(None)
|
||||
column_names = set(raw_dataset.column_names)
|
||||
|
||||
if value_field not in column_names:
|
||||
raise KeyError(f"Missing value field '{value_field}' in dataset.")
|
||||
|
||||
values_all = _to_1d_float(raw_dataset[value_field])
|
||||
if advantage_field in column_names:
|
||||
advantages_all = _to_1d_float(raw_dataset[advantage_field])
|
||||
else:
|
||||
advantages_all = np.zeros_like(values_all, dtype=np.float32)
|
||||
|
||||
if indicator_field in column_names:
|
||||
indicators_all = _to_1d_int(raw_dataset[indicator_field])
|
||||
else:
|
||||
indicators_all = np.zeros(values_all.shape[0], dtype=np.int64)
|
||||
episode_indices_all = np.asarray(raw_dataset["episode_index"], dtype=np.int64).reshape(-1)
|
||||
frame_indices_all = np.asarray(raw_dataset["frame_index"], dtype=np.int64).reshape(-1)
|
||||
if "timestamp" in column_names:
|
||||
timestamps_all = np.asarray(raw_dataset["timestamp"], dtype=np.float64).reshape(-1)
|
||||
else:
|
||||
timestamps_all = frame_indices_all.astype(np.float64) / float(dataset.fps)
|
||||
|
||||
if dataset.episodes is not None:
|
||||
available_episodes = sorted(dataset.episodes)
|
||||
else:
|
||||
available_episodes = list(range(dataset.meta.total_episodes))
|
||||
|
||||
if viz_episodes.strip().lower() == "all":
|
||||
episodes = available_episodes
|
||||
else:
|
||||
requested = _parse_episodes_arg(viz_episodes, dataset.meta.total_episodes)
|
||||
episodes = [ep for ep in requested if ep in set(available_episodes)]
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tasks = []
|
||||
fps = int(dataset.fps)
|
||||
for ep in episodes:
|
||||
ep_positions = np.flatnonzero(episode_indices_all == ep)
|
||||
if ep_positions.shape[0] > 1:
|
||||
ep_frame_indices = frame_indices_all[ep_positions]
|
||||
if bool(np.any(np.diff(ep_frame_indices) < 0)):
|
||||
ep_positions = ep_positions[np.argsort(ep_frame_indices, kind="stable")]
|
||||
|
||||
ep_values = values_all[ep_positions]
|
||||
if ep_values.shape[0] == 0:
|
||||
continue
|
||||
|
||||
ep_timestamps = timestamps_all[ep_positions]
|
||||
from_ts, to_ts = _get_episode_video_time_bounds(dataset, ep, selected_video_key)
|
||||
ep_video_timestamps = from_ts + ep_timestamps
|
||||
if to_ts is not None:
|
||||
ep_video_timestamps = np.minimum(ep_video_timestamps, to_ts)
|
||||
|
||||
dst_path = _build_output_video_path(output_dir, dataset.repo_id, selected_video_key, ep)
|
||||
if dst_path.exists() and not overwrite:
|
||||
continue
|
||||
|
||||
src_path = Path(dataset.root) / dataset.meta.get_video_file_path(ep, selected_video_key)
|
||||
tasks.append(
|
||||
(
|
||||
src_path,
|
||||
dst_path,
|
||||
ep_values,
|
||||
advantages_all[ep_positions],
|
||||
indicators_all[ep_positions],
|
||||
ep_video_timestamps,
|
||||
)
|
||||
)
|
||||
|
||||
written_paths: list[Path] = []
|
||||
for src, dst, vals, advs, inds, ts in tqdm(tasks, total=len(tasks), desc="Export value overlay videos", leave=False):
|
||||
written_paths.append(
|
||||
_export_single_episode(
|
||||
src,
|
||||
dst,
|
||||
vals,
|
||||
advs,
|
||||
inds,
|
||||
ts,
|
||||
fps,
|
||||
vcodec,
|
||||
float(getattr(dataset, "tolerance_s", 1e-4)),
|
||||
getattr(dataset, "video_backend", None),
|
||||
frame_storage_mode,
|
||||
output_dir,
|
||||
)
|
||||
)
|
||||
|
||||
return written_paths
|
||||
在新工单中引用
屏蔽一个用户