snapshot: sync mint main from current workspace 2026-03-03 16:32:56 +0800

这个提交包含在:
Elvin
2026-03-03 16:32:56 +08:00
当前提交 908eb4bbbc
共有 677 个文件被更改,包括 167208 次插入 和 0 次删除

查看文件

@ -0,0 +1,88 @@
#!/usr/bin/env python
from types import SimpleNamespace
from lerobot.rl.acp_dataset_stats import compute_acp_indicator_stats
class DummyHFDataset:
def __init__(self, columns: dict[str, list]):
self._columns = columns
self.column_names = list(columns.keys())
def __getitem__(self, key: str):
return self._columns[key]
def test_compute_acp_indicator_stats_from_meta_stats():
dataset = SimpleNamespace(
meta=SimpleNamespace(
stats={
"complementary_info.acp_indicator": {
"mean": [0.3],
"count": [10],
}
}
)
)
stats = compute_acp_indicator_stats(dataset, "complementary_info.acp_indicator")
assert stats is not None
assert stats.source == "meta.stats"
assert stats.positive_ratio == 0.3
assert stats.total_count == 10
assert stats.positive_count == 3
assert stats.invalid_count == -1
def test_compute_acp_indicator_stats_from_dataset_scan():
dataset = SimpleNamespace(
meta=SimpleNamespace(stats={}),
hf_dataset=DummyHFDataset(
{
"complementary_info.acp_indicator": [[1], [0], [1], [2]],
}
),
)
stats = compute_acp_indicator_stats(dataset, "complementary_info.acp_indicator")
assert stats is not None
assert stats.source == "hf_dataset_scan"
assert stats.total_count == 4
assert stats.positive_count == 2
assert stats.positive_ratio == 0.5
assert stats.invalid_count == 1
def test_compute_acp_indicator_stats_returns_none_when_missing_field():
dataset = SimpleNamespace(
meta=SimpleNamespace(stats={}),
hf_dataset=DummyHFDataset({"task": ["pick", "place"]}),
)
stats = compute_acp_indicator_stats(dataset, "complementary_info.acp_indicator")
assert stats is None
def test_compute_acp_indicator_stats_prefers_meta_stats():
dataset = SimpleNamespace(
meta=SimpleNamespace(
stats={
"complementary_info.acp_indicator": {
"mean": [0.75],
"count": [8],
}
}
),
hf_dataset=DummyHFDataset({"complementary_info.acp_indicator": [0, 0, 0, 0]}),
)
stats = compute_acp_indicator_stats(dataset, "complementary_info.acp_indicator")
assert stats is not None
assert stats.source == "meta.stats"
assert stats.positive_ratio == 0.75
assert stats.total_count == 8
assert stats.positive_count == 6

查看文件

@ -0,0 +1,132 @@
#!/usr/bin/env python
from unittest.mock import patch
import torch
from lerobot.configs.train import ACPConfig
from lerobot.configs.types import FeatureType, PolicyFeature
from lerobot.policies.pi05.configuration_pi05 import PI05Config
from lerobot.policies.pi05.processor_pi05 import make_pi05_pre_post_processors
from lerobot.rl.acp_hook import build_acp_raw_batch_hook
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
from tests.utils import require_package
class TrackingTokenizer:
def __init__(self):
self.calls: list[list[str]] = []
def __call__(
self,
text,
max_length=512,
truncation=True,
padding="max_length",
padding_side="right",
return_tensors="pt",
**kwargs,
):
del truncation, padding, padding_side, return_tensors, kwargs
texts = [text] if isinstance(text, str) else list(text)
self.calls.append(texts)
batch_size = len(texts)
input_ids = torch.zeros(batch_size, max_length, dtype=torch.long)
attention_mask = torch.zeros(batch_size, max_length, dtype=torch.long)
for i, t in enumerate(texts):
seq_len = min(max_length, max(1, len(t.split())))
input_ids[i, :seq_len] = torch.arange(1, seq_len + 1, dtype=torch.long)
attention_mask[i, :seq_len] = 1
if isinstance(text, str):
return {
"input_ids": input_ids[0],
"attention_mask": attention_mask[0],
}
return {
"input_ids": input_ids,
"attention_mask": attention_mask,
}
def _make_pi05_preprocessor():
cfg = PI05Config(
max_state_dim=4,
max_action_dim=2,
dtype="float32",
device="cpu",
)
cfg.input_features = {
"observation.state": PolicyFeature(type=FeatureType.STATE, shape=(4,)),
}
cfg.output_features = {
"action": PolicyFeature(type=FeatureType.ACTION, shape=(2,)),
}
preprocessor, _ = make_pi05_pre_post_processors(config=cfg, dataset_stats=None)
return preprocessor
@require_package("transformers")
@patch("lerobot.processor.tokenizer_processor.AutoTokenizer")
def test_acp_prompt_reaches_pi05_tokenizer(mock_auto_tokenizer):
tracking_tokenizer = TrackingTokenizer()
mock_auto_tokenizer.from_pretrained.return_value = tracking_tokenizer
preprocessor = _make_pi05_preprocessor()
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="complementary_info.acp_indicator",
positive_tag="Advantage: positive",
negative_tag="Advantage: negative",
indicator_dropout_prob=0.0,
),
seed=123,
)
batch = {
"observation.state": torch.tensor(
[
[0.1, -0.2, 0.0, 0.3],
[-0.4, 0.5, -0.6, 0.7],
],
dtype=torch.float32,
),
"task": ["Pick bottle", "Place bottle"],
"complementary_info.acp_indicator": torch.tensor([1, 0], dtype=torch.int64),
}
conditioned_batch = hook(batch, 0)
out = preprocessor(conditioned_batch)
prompts = tracking_tokenizer.calls[-1]
assert len(prompts) == 2
assert "Advantage: positive" in prompts[0]
assert "Advantage: negative" in prompts[1]
assert prompts[0].startswith("Task: Pick bottle")
assert "State:" in prompts[0]
assert OBS_LANGUAGE_TOKENS in out
assert OBS_LANGUAGE_ATTENTION_MASK in out
assert out[OBS_LANGUAGE_TOKENS].shape[0] == 2
@require_package("transformers")
@patch("lerobot.processor.tokenizer_processor.AutoTokenizer")
def test_without_acp_no_advantage_tag_in_pi05_tokenizer(mock_auto_tokenizer):
tracking_tokenizer = TrackingTokenizer()
mock_auto_tokenizer.from_pretrained.return_value = tracking_tokenizer
preprocessor = _make_pi05_preprocessor()
batch = {
"observation.state": torch.tensor(
[[0.0, 0.1, -0.1, 0.2]],
dtype=torch.float32,
),
"task": ["Pick bottle"],
}
_ = preprocessor(batch)
prompts = tracking_tokenizer.calls[-1]
assert len(prompts) == 1
assert "Advantage:" not in prompts[0]
assert prompts[0].startswith("Task: Pick bottle")

查看文件

@ -0,0 +1,157 @@
#!/usr/bin/env python
import pytest
import torch
from torch.utils.data import DataLoader
from lerobot.configs.train import ACPConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.rl.acp_hook import build_acp_raw_batch_hook
def test_acp_hook_accepts_integer_indicators():
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="acp_indicator",
positive_tag="POS",
negative_tag="NEG",
indicator_dropout_prob=0.0,
),
seed=42,
)
batch = {
"task": ["pick bottle", "place bottle"],
"acp_indicator": torch.tensor([1, 0], dtype=torch.int64),
}
out = hook(batch, 0)
assert out["task"] == [
"pick bottle\nPOS",
"place bottle\nNEG",
]
def test_acp_hook_injects_tags():
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="complementary_info.acp_indicator",
positive_tag="POS",
negative_tag="NEG",
indicator_dropout_prob=0.0,
),
seed=42,
)
batch = {
"task": ["pick bottle", "place bottle"],
"complementary_info.acp_indicator": torch.tensor([1, 0], dtype=torch.int64),
}
out = hook(batch, 0)
assert out["task"] == ["pick bottle\nPOS", "place bottle\nNEG"]
def test_acp_hook_dropout_keeps_original_task():
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="complementary_info.acp_indicator",
positive_tag="POS",
negative_tag="NEG",
indicator_dropout_prob=1.0,
),
seed=42,
)
batch = {
"task": ["pick bottle", "place bottle"],
"complementary_info.acp_indicator": torch.tensor([1, 0], dtype=torch.int64),
}
out = hook(batch, 0)
assert out["task"] == ["pick bottle", "place bottle"]
def test_acp_hook_missing_indicator_skips():
hook = build_acp_raw_batch_hook(
ACPConfig(enable=True, indicator_field="missing_field"),
seed=42,
)
batch = {"task": ["pick bottle"]}
with pytest.raises(KeyError, match="missing_field"):
hook(batch, 0)
def test_acp_hook_non_integer_type_raises():
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="acp_indicator",
positive_tag="POS",
negative_tag="NEG",
indicator_dropout_prob=0.0,
),
seed=42,
)
batch = {
"task": ["pick bottle", "place bottle"],
"acp_indicator": torch.tensor([1.0, 0.0], dtype=torch.float32),
}
with pytest.raises(TypeError, match="integer 0/1"):
hook(batch, 0)
def test_acp_hook_non_binary_value_raises():
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="acp_indicator",
positive_tag="POS",
negative_tag="NEG",
indicator_dropout_prob=0.0,
),
seed=42,
)
batch = {
"task": ["pick bottle", "place bottle"],
"acp_indicator": torch.tensor([2, 0], dtype=torch.int64),
}
with pytest.raises(ValueError, match="must be 0 or 1"):
hook(batch, 0)
def _load_real_batch_or_skip(batch_size: int) -> dict:
repo_id = "local/eval_so101_pick_place_bottle_v0"
try:
dataset = LeRobotDataset(repo_id, download_videos=False)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0)
return next(iter(dataloader))
except Exception as e:
pytest.skip(f"Cannot load local dataset '{repo_id}': {e}")
@pytest.mark.parametrize("batch_size", [1, 3])
def test_acp_hook_with_real_local_dataset_batch(batch_size: int):
batch = _load_real_batch_or_skip(batch_size)
assert "task" in batch
assert "complementary_info.acp_indicator" not in batch
indicators = torch.randint(0, 2, (batch_size,), dtype=torch.int64)
batch["complementary_info.acp_indicator"] = indicators
hook = build_acp_raw_batch_hook(
ACPConfig(
enable=True,
indicator_field="complementary_info.acp_indicator",
positive_tag="POS",
negative_tag="NEG",
indicator_dropout_prob=0.0,
),
seed=42,
)
out = hook(batch, 0)
assert len(out["task"]) == batch_size
for i in range(batch_size):
expected_tag = "POS" if int(indicators[i].item()) == 1 else "NEG"
assert out["task"][i].endswith(expected_tag)

查看文件

@ -0,0 +1,211 @@
#!/usr/bin/env python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Multi-GPU Training Tests
This module tests multi-GPU training functionality with accelerate.
These tests are designed to run on machines with 2+ GPUs and are executed
in the nightly CI workflow.
The tests automatically generate accelerate configs and launch training
with subprocess to properly test the distributed training environment.
"""
import os
import subprocess
import tempfile
from pathlib import Path
import pytest
import torch
from lerobot.datasets.lerobot_dataset import LeRobotDataset
def get_num_available_gpus():
"""Returns the number of available GPUs."""
if not torch.cuda.is_available():
return 0
return torch.cuda.device_count()
def download_dataset(repo_id, episodes):
"""
Pre-download dataset to avoid race conditions in multi-GPU training.
Args:
repo_id: HuggingFace dataset repository ID
episodes: List of episode indices to download
"""
# Simply instantiating the dataset will download it
_ = LeRobotDataset(repo_id, episodes=episodes)
print(f"Dataset {repo_id} downloaded successfully")
def run_accelerate_training(config_args, num_processes=4, temp_dir=None):
"""
Helper function to run training with accelerate launch.
Args:
config_args: List of config arguments to pass to lerobot_train.py
num_processes: Number of processes (GPUs) to use
temp_dir: Temporary directory for outputs
Returns:
subprocess.CompletedProcess result
"""
config_path = Path(temp_dir) / "accelerate_config.yaml"
# Write YAML config
with open(config_path, "w") as f:
f.write("compute_environment: LOCAL_MACHINE\n")
f.write("distributed_type: MULTI_GPU\n")
f.write("mixed_precision: 'no'\n")
f.write(f"num_processes: {num_processes}\n")
f.write("use_cpu: false\n")
f.write("gpu_ids: all\n")
f.write("downcast_bf16: 'no'\n")
f.write("machine_rank: 0\n")
f.write("main_training_function: main\n")
f.write("num_machines: 1\n")
f.write("rdzv_backend: static\n")
f.write("same_network: true\n")
cmd = [
"accelerate",
"launch",
"--config_file",
str(config_path),
"-m",
"lerobot.scripts.lerobot_train",
] + config_args
result = subprocess.run(
cmd,
capture_output=True,
text=True,
env={**os.environ, "CUDA_VISIBLE_DEVICES": ",".join(map(str, range(num_processes)))},
)
return result
@pytest.mark.skipif(
get_num_available_gpus() < 2,
reason="Multi-GPU tests require at least 2 GPUs",
)
class TestMultiGPUTraining:
"""Test suite for multi-GPU training functionality."""
def test_basic_multi_gpu_training(self):
"""
Test that basic multi-GPU training runs successfully.
Verifies that the training completes without errors.
"""
# Pre-download dataset to avoid race conditions
download_dataset("lerobot/pusht", episodes=[0])
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = Path(temp_dir) / "outputs"
config_args = [
"--dataset.repo_id=lerobot/pusht",
"--dataset.episodes=[0]",
"--policy.type=act",
"--policy.device=cuda",
"--policy.push_to_hub=false",
f"--output_dir={output_dir}",
"--batch_size=4",
"--steps=10",
"--eval_freq=-1",
"--log_freq=5",
"--save_freq=10",
"--seed=42",
"--num_workers=0",
]
result = run_accelerate_training(config_args, num_processes=4, temp_dir=temp_dir)
# Check that training completed successfully
assert result.returncode == 0, (
f"Multi-GPU training failed with return code {result.returncode}\n"
f"STDOUT:\n{result.stdout}\n"
f"STDERR:\n{result.stderr}"
)
# Verify checkpoint was saved
checkpoints_dir = output_dir / "checkpoints"
assert checkpoints_dir.exists(), "Checkpoints directory was not created"
# Verify that training completed
assert "End of training" in result.stdout or "End of training" in result.stderr
def test_checkpoint_saving_multi_gpu(self):
"""
Test that checkpoints are correctly saved during multi-GPU training.
Only the main process (rank 0) should save checkpoints.
"""
# Pre-download dataset to avoid race conditions
download_dataset("lerobot/pusht", episodes=[0])
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = Path(temp_dir) / "outputs"
config_args = [
"--dataset.repo_id=lerobot/pusht",
"--dataset.episodes=[0]",
"--policy.type=act",
"--policy.device=cuda",
"--policy.push_to_hub=false",
f"--output_dir={output_dir}",
"--batch_size=4",
"--steps=20",
"--eval_freq=-1",
"--log_freq=5",
"--save_freq=10",
"--seed=42",
"--num_workers=0",
]
result = run_accelerate_training(config_args, num_processes=2, temp_dir=temp_dir)
assert result.returncode == 0, (
f"Training failed:\nSTDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}"
)
# Verify checkpoint directory exists
checkpoints_dir = output_dir / "checkpoints"
assert checkpoints_dir.exists(), "Checkpoints directory not created"
# Count checkpoint directories (should have checkpoint at step 10 and 20)
checkpoint_dirs = [d for d in checkpoints_dir.iterdir() if d.is_dir()]
assert len(checkpoint_dirs) >= 1, f"Expected at least 1 checkpoint, found {len(checkpoint_dirs)}"
# Verify checkpoint contents
for checkpoint_dir in checkpoint_dirs:
# Check for model files
model_files = list(checkpoint_dir.rglob("*.safetensors"))
assert len(model_files) > 0, f"No model files in checkpoint {checkpoint_dir}"
# Check for training state
training_state_dir = checkpoint_dir / "training_state"
assert training_state_dir.exists(), f"No training state in checkpoint {checkpoint_dir}"
# Verify optimizer state exists
optimizer_state = training_state_dir / "optimizer_state.safetensors"
assert optimizer_state.exists(), f"No optimizer state in checkpoint {checkpoint_dir}"

查看文件

@ -0,0 +1,157 @@
#!/usr/bin/env python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Visual Feature Consistency Tests
This module tests the `validate_visual_features_consistency` function,
which ensures that visual features (camera observations) in a dataset/env
match the expectations defined in a policy configuration.
The purpose of this check is to prevent mismatches between what a policy expects
(e.g., `observation.images.camera1`, `camera2`, `camera3`) and what a dataset or
environment actually provides (e.g., `observation.images.top`, `side`, or fewer cameras).
"""
from pathlib import Path
import numpy as np
import pytest
from lerobot.configs.default import DatasetConfig
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.train import TrainPipelineConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.policies.factory import make_policy_config
from lerobot.scripts.lerobot_train import train
from lerobot.utils.utils import auto_select_torch_device
pytest.importorskip("transformers")
DUMMY_REPO_ID = "dummy/repo"
@pytest.fixture
def temp_dir(tmp_path):
return tmp_path
DUMMY_STATE_DIM = 6
DUMMY_ACTION_DIM = 6
IMAGE_SIZE = 8
DEVICE = auto_select_torch_device()
def make_dummy_dataset(camera_keys, tmp_path):
"""Creates a minimal dummy dataset for testing rename_mapping logic."""
features = {
"action": {"dtype": "float32", "shape": (DUMMY_ACTION_DIM,), "names": None},
"observation.state": {"dtype": "float32", "shape": (DUMMY_STATE_DIM,), "names": None},
}
for cam in camera_keys:
features[f"observation.images.{cam}"] = {
"dtype": "image",
"shape": (IMAGE_SIZE, IMAGE_SIZE, 3),
"names": ["height", "width", "channel"],
}
dataset = LeRobotDataset.create(
repo_id=DUMMY_REPO_ID,
fps=30,
features=features,
root=tmp_path / "_dataset",
)
root = tmp_path / "_dataset"
for ep_idx in range(2):
for _ in range(3):
frame = {
"action": np.random.randn(DUMMY_ACTION_DIM).astype(np.float32),
"observation.state": np.random.randn(DUMMY_STATE_DIM).astype(np.float32),
}
for cam in camera_keys:
frame[f"observation.images.{cam}"] = np.random.randint(
0, 255, size=(IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.uint8
)
frame["task"] = f"task_{ep_idx}"
dataset.add_frame(frame)
dataset.save_episode()
dataset.finalize()
return dataset, root
def custom_validate(train_config: TrainPipelineConfig, policy_path: str, empty_cameras: int):
train_config.policy = PreTrainedConfig.from_pretrained(policy_path)
train_config.policy.pretrained_path = Path(policy_path)
# override empty_cameras and push_to_hub for testing
train_config.policy.empty_cameras = empty_cameras
train_config.policy.push_to_hub = False
if train_config.use_policy_training_preset:
train_config.optimizer = train_config.policy.get_optimizer_preset()
train_config.scheduler = train_config.policy.get_scheduler_preset()
return train_config
@pytest.mark.skip(reason="Skipping this test as it results OOM")
@pytest.mark.parametrize(
"camera_keys, empty_cameras, rename_map, expect_success",
[
# case 1: dataset has fewer cameras than policy (3 instead of 4), but we specify empty_cameras=1 for smolvla, pi0, pi05
(["camera1", "camera2", "camera3"], 1, {}, True),
# case 2: dataset has 2 cameras with different names, rename_mapping provided
(
["top", "side"],
0,
{
"observation.images.top": "observation.images.camera1",
"observation.images.side": "observation.images.camera2",
},
True,
),
# case 3: dataset has 2 cameras, policy expects 3, names do not match, no empty_cameras
(["top", "side"], 0, {}, False),
# TODO: case 4: dataset has 2 cameras, policy expects 3, no rename_map, no empty_cameras, should raise for smolvla
# (["camera1", "camera2"], 0, {}, False),
],
)
def test_train_with_camera_mismatch(camera_keys, empty_cameras, rename_map, expect_success, tmp_path):
"""Tests that training works or fails depending on camera/feature alignment."""
_dataset, root = make_dummy_dataset(camera_keys, tmp_path)
pretrained_path = "lerobot/smolvla_base"
dataset_config = DatasetConfig(repo_id=DUMMY_REPO_ID, root=root)
policy_config = make_policy_config(
"smolvla",
optimizer_lr=0.01,
push_to_hub=False,
pretrained_path=pretrained_path,
device=DEVICE,
)
policy_config.empty_cameras = empty_cameras
train_config = TrainPipelineConfig(
dataset=dataset_config,
policy=policy_config,
rename_map=rename_map,
output_dir=tmp_path / "_output",
steps=1,
)
train_config = custom_validate(train_config, policy_path=pretrained_path, empty_cameras=empty_cameras)
# HACK: disable the internal CLI validation step for tests, we did it with custom_validate
train_config.validate = lambda: None
if expect_success:
train(train_config)
else:
with pytest.raises(ValueError):
train(train_config)