feat: add vision-only values, takeover rewards and masked action loss

这个提交包含在:
Elvin-yk
2026-10-16 22:13:40 +08:00
父节点 25a23467d2
当前提交 ec9842d8a8
共有 11 个文件被更改,包括 366 次插入 和 115 次删除

查看文件

@ -9,6 +9,7 @@ from typing import Any
import draccus
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.utils.recording_annotations import normalize_episode_success_label
@ -118,6 +119,8 @@ class ValueInferencePipelineConfig:
)
runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig)
acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig)
# None inherits the trained checkpoint; old checkpoints resolve to legacy.
reward: ValueRewardConfig | None = None
viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig)
output_dir: Path | None = None
@ -131,6 +134,8 @@ class ValueInferencePipelineConfig:
self.inference.validate()
self.runtime.validate()
self.acp.validate()
if self.reward is not None:
self.reward.validate()
self.viz.validate()
if not self.job_name:

查看文件

@ -0,0 +1,19 @@
"""Reward settings shared by value training and inference."""
import math
from dataclasses import dataclass
@dataclass
class ValueRewardConfig:
mode: str = "legacy"
takeover_cost: float = 150.0
intervention_field: str = "complementary_info.is_intervention"
def validate(self) -> None:
if self.mode not in {"legacy", "time", "time_takeover"}:
raise ValueError("'reward.mode' must be legacy, time or time_takeover.")
if not math.isfinite(self.takeover_cost) or self.takeover_cost < 0:
raise ValueError("'reward.takeover_cost' must be finite and non-negative.")
if self.mode == "time_takeover" and not self.intervention_field:
raise ValueError("'reward.intervention_field' must be non-empty for time_takeover.")

查看文件

@ -14,6 +14,7 @@ from huggingface_hub.errors import HfHubHTTPError
from lerobot.configs import parser
from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.optim import OptimizerConfig
from lerobot.optim.schedulers import LRSchedulerConfig
from lerobot.utils.hub import HubMixin
@ -72,6 +73,7 @@ class ValueTrainPipelineConfig(HubMixin):
optimizer: OptimizerConfig | None = None
scheduler: LRSchedulerConfig | None = None
reward: ValueRewardConfig | None = None
targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig)
wandb: WandBConfig = field(default_factory=WandBConfig)
peft: PeftConfig | None = None
@ -113,6 +115,8 @@ class ValueTrainPipelineConfig(HubMixin):
)
self.targets.validate()
if self.reward is not None:
self.reward.validate()
if hasattr(self.value, "target_key"):
self.value.target_key = self.targets.target_field

查看文件

@ -71,6 +71,9 @@ class PI05Config(PreTrainedConfig):
}
)
# Exclude repeated episode-tail actions from flow-matching supervision.
mask_action_padding_loss: bool = False
# Training settings
gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization
compile_model: bool = False # Whether to use torch.compile for model optimization

查看文件

@ -1250,6 +1250,16 @@ class PI05Policy(PreTrainedPolicy):
- "mean": Return scalar mean loss (default, backward compatible)
- "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting
"""
action_is_pad = None
if self.config.mask_action_padding_loss:
action_is_pad = batch.get("action_is_pad")
if action_is_pad is None:
raise ValueError(
"mask_action_padding_loss=True requires action_is_pad in the training batch."
)
if not isinstance(action_is_pad, Tensor) or action_is_pad.dtype != torch.bool:
raise TypeError("action_is_pad must be a boolean tensor (True marks padding).")
# Prepare inputs
images, img_masks = self._preprocess_images(batch)
tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
@ -1263,6 +1273,9 @@ class PI05Policy(PreTrainedPolicy):
original_action_dim = self.config.output_features[ACTION].shape[0]
losses = losses[:, :, :original_action_dim]
if action_is_pad is not None:
return self._reduce_masked_action_losses(losses, action_is_pad, reduction)
loss_dict = {
"loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(),
}
@ -1278,6 +1291,27 @@ class PI05Policy(PreTrainedPolicy):
loss_dict["loss"] = loss.item()
return loss, loss_dict
@staticmethod
def _reduce_masked_action_losses(losses: Tensor, action_is_pad: Tensor, reduction: str):
if tuple(action_is_pad.shape) != tuple(losses.shape[:2]):
raise ValueError("action_is_pad must have shape [batch_size, action_horizon].")
valid = ~action_is_pad.to(device=losses.device)
valid_steps = valid.sum(dim=1)
if torch.any(valid_steps == 0):
raise ValueError("Every action chunk must contain at least one non-padding timestep.")
# Mask with where: even NaN/Inf in an unused loss slot must not enter the sum.
masked = torch.where(valid.unsqueeze(-1), losses, 0.0)
if reduction == "none":
loss = masked.sum(dim=(1, 2)) / (valid_steps * losses.shape[-1])
elif reduction == "mean":
loss = masked.sum() / (valid_steps.sum() * losses.shape[-1])
else:
raise ValueError("reduction must be 'mean' or 'none'.")
return loss, {
"loss": loss.detach().mean().item(),
"loss_per_dim": (masked.sum(dim=(0, 1)) / valid_steps.sum()).detach().cpu().tolist(),
}
def _get_default_peft_targets(self) -> dict[str, any]:
"""Return default PEFT target modules for PI0.5 fine-tuning."""
common_projections = (

查看文件

@ -32,6 +32,7 @@ from tqdm.auto import tqdm
from lerobot.configs import parser
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.value import ValueInferencePipelineConfig
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.utils import load_info, write_info
from lerobot.policies.factory import make_policy, make_pre_post_processors
@ -52,6 +53,7 @@ from lerobot.values.pistar06.modeling_pistar06 import (
EpisodeTargetInfo,
compute_normalized_value_targets,
)
from lerobot.values.rewards import build_successful_value_targets
class ContiguousDistributedEvalSampler(Sampler[int]):
@ -522,6 +524,23 @@ def run_value_inference_pipeline(
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
reward = value_cfg.reward or ValueRewardConfig()
if cfg.reward is not None and cfg.reward != reward:
raise ValueError("Inference reward settings must match the trained checkpoint.")
cfg.reward = reward
success_targets = None
if reward.mode != "legacy":
if not value_cfg.reward_scales:
raise ValueError(
"Checkpoint is missing training reward scales; refusing to fit on inference data."
)
success_targets, _ = build_successful_value_targets(
dataset,
reward,
cfg.dataset.success_field,
task_scales=value_cfg.reward_scales,
)
if cfg.acp.intervention_field in raw_frames.column_names:
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
else:
@ -595,6 +614,8 @@ def run_value_inference_pipeline(
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
predicted_values = prediction_lookup[absolute_indices]
if success_targets is not None:
predicted_values[success_targets == 0.0] = 0.0
logging.info(
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
float(np.min(predicted_values)),
@ -614,12 +635,12 @@ def run_value_inference_pipeline(
thresholds: dict[int, float] | None = None
if cfg.acp.enable:
if success_targets is None:
episode_info, task_max_lengths = _build_episode_info(
dataset=dataset,
success_field=cfg.dataset.success_field,
default_success=cfg.dataset.default_success,
)
value_targets = compute_normalized_value_targets(
episode_indices=episode_indices,
frame_indices=frame_indices,
@ -629,6 +650,8 @@ def run_value_inference_pipeline(
clip_min=value_cfg.bin_min,
clip_max=value_cfg.bin_max,
)
else:
value_targets = success_targets
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
advantages = _compute_n_step_advantages(
rewards=rewards,

查看文件

@ -131,7 +131,9 @@ def value_train(
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
dataset=dataset,
targets_cfg=cfg.targets,
reward_cfg=cfg.reward,
)
cfg.reward = value_model.config.reward
if is_main_process:
logging.info("Using value model raw-batch hook for target construction.")
elif is_main_process:
@ -163,6 +165,9 @@ def value_train(
"norm_map": cfg.value.normalization_mapping,
},
}
if cfg.value.mode == "vision_only":
processor_kwargs["preprocessor_overrides"].pop("normalizer_processor", None)
postprocessor_kwargs["postprocessor_overrides"].pop("unnormalizer_processor", None)
preprocessor, postprocessor = make_pre_post_processors(
policy_cfg=cfg.value,

查看文件

@ -4,6 +4,7 @@ from dataclasses import dataclass, field
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.types import NormalizationMode
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.optim.optimizers import AdamWConfig
from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
from lerobot.utils.constants import OBS_STATE
@ -12,7 +13,12 @@ from lerobot.utils.constants import OBS_STATE
@PreTrainedConfig.register_subclass("pistar06")
@dataclass
class Pistar06Config(PreTrainedConfig):
"""Value model config using the Pistar06 stack (SigLIP + Gemma)."""
"""SigLIP value model with optional Gemma language conditioning."""
# Old checkpoints without this field retain the language-conditioned path.
mode: str = "vision_language"
reward: ValueRewardConfig | None = None
reward_scales: dict[str, float] = field(default_factory=dict)
# Backbone components
vision_repo_id: str = "google/siglip-so400m-patch14-384"
@ -70,9 +76,13 @@ class Pistar06Config(PreTrainedConfig):
def __post_init__(self) -> None:
super().__post_init__()
if self.mode not in {"vision_language", "vision_only"}:
raise ValueError("'value.mode' must be vision_language or vision_only.")
if self.reward is not None:
self.reward.validate()
if not self.vision_repo_id:
raise ValueError("'value.vision_repo_id' must be non-empty.")
if not self.language_repo_id:
if self.mode == "vision_language" and not self.language_repo_id:
raise ValueError("'value.language_repo_id' must be non-empty.")
if not self.task_field:
raise ValueError("'value.task_field' must be non-empty.")

查看文件

@ -19,20 +19,29 @@ from huggingface_hub.errors import HfHubHTTPError
from safetensors.torch import save_file
from torch import Tensor, nn
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
from lerobot.utils.import_utils import _transformers_available
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY
from lerobot.values.rewards import build_successful_value_targets
if TYPE_CHECKING or _transformers_available:
from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoModelForCausalLM
from transformers import (
AutoConfig,
AutoImageProcessor,
AutoModel,
AutoModelForCausalLM,
SiglipVisionModel,
)
else:
AutoConfig = None
AutoImageProcessor = None
AutoModel = None
AutoModelForCausalLM = None
SiglipVisionModel = None
PISTAR06_SAVE_INFO = "pistar06_save_info.json"
@ -261,11 +270,15 @@ class Pistar06Model(nn.Module):
self.cfg = cfg
self.model_dtype = _resolve_load_dtype(cfg.dtype)
self.vision_encoder = AutoModel.from_pretrained(
# SiglipVisionModel loads only the visual weights; AutoModel on a full
# SigLIP repository would also create an unused text encoder under DDP.
vision_loader = SiglipVisionModel if cfg.mode == "vision_only" else AutoModel
self.vision_encoder = vision_loader.from_pretrained(
cfg.vision_repo_id,
revision=cfg.vision_revision,
torch_dtype=self.model_dtype,
)
if cfg.mode == "vision_language":
self.language_model = _load_language_model(
repo_id=cfg.language_repo_id,
revision=cfg.language_revision,
@ -292,31 +305,35 @@ class Pistar06Model(nn.Module):
)
vision_feature_size = _extract_vision_feature_size(self.vision_encoder)
language_hidden_size = _extract_hidden_size(self.language_model)
self.image_projector = nn.Sequential(
nn.Linear(vision_feature_size, cfg.fusion_hidden_dim),
nn.GELU(),
nn.Dropout(cfg.dropout),
)
head_input_dim = cfg.fusion_hidden_dim
if cfg.mode == "vision_language":
language_hidden_size = _extract_hidden_size(self.language_model)
self.language_projector = nn.Sequential(
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
nn.GELU(),
nn.Dropout(cfg.dropout),
)
self.final_norm = nn.LayerNorm(cfg.fusion_hidden_dim * 2)
head_input_dim *= 2
self.final_norm = nn.LayerNorm(head_input_dim)
self.value_head = nn.Sequential(
nn.Linear(cfg.fusion_hidden_dim * 2, cfg.fusion_hidden_dim),
nn.Linear(head_input_dim, cfg.fusion_hidden_dim),
nn.GELU(),
nn.Dropout(cfg.dropout),
nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins),
)
if cfg.use_gradient_checkpointing:
if cfg.mode == "vision_language":
_maybe_enable_gradient_checkpointing(self.language_model)
_maybe_enable_gradient_checkpointing(self.vision_encoder)
if cfg.freeze_language_model:
if cfg.mode == "vision_language" and cfg.freeze_language_model:
_freeze_module(self.language_model)
if cfg.freeze_vision_encoder:
_freeze_module(self.vision_encoder)
@ -358,6 +375,10 @@ class Pistar06Model(nn.Module):
if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras:
raise ValueError("Batch shape mismatch between images and image_attention_mask.")
if self.cfg.mode == "vision_only":
# Missing cameras must not influence detection of the input pixel range.
images = torch.where(image_attention_mask[:, :, None, None, None], images, 0)
if images.dtype == torch.uint8:
images = images.to(dtype=torch.float32) / 255.0
else:
@ -393,33 +414,26 @@ class Pistar06Model(nn.Module):
def forward(
self,
input_ids: Tensor,
attention_mask: Tensor,
images: Tensor,
image_attention_mask: Tensor,
input_ids: Tensor | None = None,
attention_mask: Tensor | None = None,
images: Tensor | None = None,
image_attention_mask: Tensor | None = None,
) -> Tensor:
if input_ids.ndim != 2:
raise ValueError(f"'input_ids' must have shape [B, T], got {tuple(input_ids.shape)}.")
if attention_mask.ndim != 2:
raise ValueError(f"'attention_mask' must have shape [B, T], got {tuple(attention_mask.shape)}.")
if images.ndim != 5:
raise ValueError(f"'images' must have shape [B, N, C, H, W], got {tuple(images.shape)}.")
if image_attention_mask.ndim != 2:
raise ValueError(
f"'image_attention_mask' must have shape [B, N], got {tuple(image_attention_mask.shape)}."
)
bsize = input_ids.shape[0]
if attention_mask.shape[0] != bsize:
raise ValueError("Batch size mismatch between input_ids and attention_mask.")
if images.shape[0] != bsize or image_attention_mask.shape[0] != bsize:
raise ValueError("Batch size mismatch between language and image inputs.")
if images.shape[1] == 0:
if images is None or images.ndim != 5:
raise ValueError("'images' must have shape [B, N, C, H, W].")
bsize, num_cameras = images.shape[:2]
if num_cameras == 0:
raise ValueError("At least one camera is required for Pistar06Model.")
if image_attention_mask is None or tuple(image_attention_mask.shape) != (bsize, num_cameras):
raise ValueError("'image_attention_mask' must have shape [B, N].")
image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device)
if not torch.all(image_attention_mask.any(dim=1)):
raise ValueError("Each sample must have at least one valid camera input.")
if self.cfg.mode == "vision_language":
if input_ids is None or attention_mask is None:
raise ValueError("vision_language requires input_ids and attention_mask.")
if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape or input_ids.shape[0] != bsize:
raise ValueError("Language tensors must have matching shape [B, T].")
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
if not torch.all(language_mask.any(dim=1)):
raise ValueError("Each sample must have at least one valid language token.")
@ -433,6 +447,7 @@ class Pistar06Model(nn.Module):
with image_context:
image_features = self._encode_images(flat_images)
if self.cfg.mode == "vision_language":
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
with language_context:
language_features = self._encode_language(
@ -441,7 +456,6 @@ class Pistar06Model(nn.Module):
feature_dtype = torch.float32
image_features = image_features.to(dtype=feature_dtype)
language_features = language_features.to(dtype=feature_dtype)
image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1)
camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype)
@ -451,10 +465,11 @@ class Pistar06Model(nn.Module):
image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0)
)
image_pooled = image_tokens.sum(dim=1) / camera_denominator
language_token = self.language_projector(language_features)
if self.cfg.mode == "vision_only":
return self.value_head(self.final_norm(image_pooled))
joint_features = torch.cat([image_pooled, language_token], dim=-1)
return self.value_head(self.final_norm(joint_features))
language_token = self.language_projector(language_features.to(dtype=feature_dtype))
return self.value_head(self.final_norm(torch.cat([image_pooled, language_token], dim=-1)))
class Pistar06Policy(PreTrainedPolicy):
@ -482,7 +497,7 @@ class Pistar06Policy(PreTrainedPolicy):
prefixes: list[str] = []
if self.config.freeze_vision_encoder:
prefixes.append("model.vision_encoder.")
if self.config.freeze_language_model:
if self.config.mode == "vision_language" and self.config.freeze_language_model:
prefixes.append("model.language_model.")
return prefixes
@ -626,8 +641,8 @@ class Pistar06Policy(PreTrainedPolicy):
raise RuntimeError("Pistar06Policy is a value model and does not support action selection.")
def predict_value(self, batch: dict[str, Tensor]) -> Tensor:
input_ids = batch[OBS_LANGUAGE_TOKENS]
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
input_ids = batch.get(OBS_LANGUAGE_TOKENS)
attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
images = batch[PISTAR06_IMAGES_KEY]
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
@ -640,7 +655,7 @@ class Pistar06Policy(PreTrainedPolicy):
bin_centers = self.bin_centers.to(device=logits.device)
return expected_value_from_logits(logits, bin_centers)
def build_training_raw_batch_hook(self, dataset, targets_cfg):
def build_training_raw_batch_hook(self, dataset, targets_cfg, reward_cfg=None):
raw_frames = dataset.hf_dataset.with_format(None)
frame_count = len(raw_frames)
if frame_count == 0:
@ -650,6 +665,42 @@ class Pistar06Policy(PreTrainedPolicy):
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
reward = reward_cfg or self.config.reward or ValueRewardConfig()
reward.validate()
if reward.mode != "legacy":
saved_scales = self.config.reward_scales if self.config.reward == reward else None
value_targets, scales = build_successful_value_targets(
dataset,
reward,
targets_cfg.success_field,
task_scales=saved_scales or None,
)
self.config.reward = reward
self.config.reward_scales = scales
else:
value_targets = self._legacy_training_targets(
dataset, targets_cfg, episode_indices, frame_indices
)
self.config.reward = reward
self.config.reward_scales = {}
max_index = int(np.max(absolute_indices))
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
target_key = targets_cfg.target_field
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
del step
batch_indices = batch.get("index")
if batch_indices is None:
raise KeyError("Missing 'index' in batch while building value targets.")
batch_indices = torch.as_tensor(batch_indices).detach().cpu().numpy().astype(np.int64).reshape(-1)
batch[target_key] = torch.from_numpy(value_target_lookup[batch_indices])
return batch
return value_target_hook
def _legacy_training_targets(self, dataset, targets_cfg, episode_indices, frame_indices):
episodes_ds = dataset.meta.episodes.with_format(None)
episodes = episodes_ds[:]
n_episodes = len(episodes_ds)
@ -692,26 +743,7 @@ class Pistar06Policy(PreTrainedPolicy):
clip_max=self.config.bin_max,
)
max_index = int(np.max(absolute_indices))
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
target_key = targets_cfg.target_field
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
del step
batch_indices = batch.get("index")
if batch_indices is None:
raise KeyError("Missing 'index' in batch while building value targets.")
if not isinstance(batch_indices, torch.Tensor):
batch_indices = torch.as_tensor(batch_indices)
batch_indices_np = batch_indices.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
target_values = torch.from_numpy(value_target_lookup[batch_indices_np]).to(dtype=torch.float32)
batch[target_key] = target_values
return batch
return value_target_hook
return value_targets
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
if self.config.target_key not in batch:
@ -720,8 +752,8 @@ class Pistar06Policy(PreTrainedPolicy):
"Make sure lerobot-value-train target hook is enabled."
)
input_ids = batch[OBS_LANGUAGE_TOKENS]
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
input_ids = batch.get(OBS_LANGUAGE_TOKENS)
attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
images = batch[PISTAR06_IMAGES_KEY]
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]

查看文件

@ -218,8 +218,33 @@ def make_pistar06_pre_post_processors(
if not camera_features:
camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)]
input_steps: list[ProcessorStep] = [
RenameObservationsProcessorStep(rename_map={}),
input_steps: list[ProcessorStep] = [RenameObservationsProcessorStep(rename_map={})]
if config.mode == "vision_language":
input_steps.extend(_language_steps(config, dataset_stats))
input_steps.extend(
[
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
DeviceProcessorStep(device=config.device),
]
)
output_steps: list[ProcessorStep] = [DeviceProcessorStep(device="cpu")]
return (
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
steps=input_steps,
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
),
PolicyProcessorPipeline[PolicyAction, PolicyAction](
steps=output_steps,
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
to_transition=policy_action_to_transition,
to_output=transition_to_policy_action,
),
)
def _language_steps(config: Pistar06Config, dataset_stats) -> list[ProcessorStep]:
return [
NormalizerProcessorStep(
features={**(config.input_features or {}), **(config.output_features or {})},
norm_map=config.normalization_mapping,
@ -241,23 +266,4 @@ def make_pistar06_pre_post_processors(
padding="max_length",
truncation=True,
),
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
DeviceProcessorStep(device=config.device),
]
output_steps: list[ProcessorStep] = [
DeviceProcessorStep(device="cpu"),
]
return (
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
steps=input_steps,
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
),
PolicyProcessorPipeline[PolicyAction, PolicyAction](
steps=output_steps,
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
to_transition=policy_action_to_transition,
to_output=transition_to_policy_action,
),
)

查看文件

@ -0,0 +1,110 @@
"""Success-only Monte Carlo targets with task scales fitted on training episodes."""
import math
import numpy as np
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, normalize_episode_success_label
def build_successful_value_targets(
dataset,
reward: ValueRewardConfig,
success_field: str,
*,
task_scales: dict[str, float] | None = None,
) -> tuple[np.ndarray, dict[str, float]]:
"""Return targets in raw frame order; reject partial, failed or unlabelled episodes.
Only episodes actually present in hf_dataset contribute to the fitted scale.
Task names, rather than dataset-local task indices, identify checkpoint scales.
Supplying scales reuses them without refitting or clipping out-of-range returns.
"""
reward.validate()
if reward.mode == "legacy":
raise ValueError("Use the legacy target builder for reward.mode=legacy.")
frames = dataset.hf_dataset.with_format(None)
episode_ids = np.asarray(frames["episode_index"], dtype=np.int64).reshape(-1)
frame_ids = np.asarray(frames["frame_index"], dtype=np.int64).reshape(-1)
if len(episode_ids) == 0:
raise ValueError("Dataset has no frames.")
episodes = dataset.meta.episodes.with_format(None)
if success_field not in episodes.column_names:
raise ValueError(f"reward.mode={reward.mode} requires explicit '{success_field}=success' labels.")
meta = episodes[:]
active_ids = set(episode_ids.tolist())
metadata = {}
for i, episode_id in enumerate(meta["episode_index"]):
episode_id = int(episode_id)
if episode_id not in active_ids:
continue
if episode_id in metadata:
raise ValueError(f"Duplicate episode metadata for episode {episode_id}.")
label = normalize_episode_success_label(meta[success_field][i])
if label != EPISODE_SUCCESS:
raise ValueError(
f"reward.mode={reward.mode} requires only explicitly successful episodes; "
f"episode {episode_id} has label {label!r}."
)
tasks = meta["tasks"][i]
if not isinstance(tasks, str):
if len(tasks) != 1:
raise ValueError(f"Episode {episode_id} must have exactly one task for reward normalization.")
tasks = tasks[0]
metadata[episode_id] = (int(meta["length"][i]), str(tasks))
if active_ids != set(metadata):
raise ValueError("Missing episode metadata for reward construction.")
interventions = None
if reward.mode == "time_takeover":
if reward.intervention_field not in frames.column_names:
raise ValueError(f"Missing intervention field '{reward.intervention_field}' for time_takeover.")
interventions = np.asarray(frames[reward.intervention_field]).reshape(-1)
if interventions.shape != episode_ids.shape or not np.isin(interventions, [0, 1]).all():
raise ValueError("Intervention flags must contain one boolean or binary 0/1 value per frame.")
interventions = interventions.astype(bool)
returns = np.empty(len(episode_ids), dtype=np.float64)
fitted_scales: dict[str, float] = {}
positions_by_task: dict[str, list[np.ndarray]] = {}
for episode_id in sorted(active_ids):
positions = np.flatnonzero(episode_ids == episode_id)
length, task = metadata[episode_id]
if (
length <= 0
or not np.array_equal(frame_ids[positions], np.arange(length))
or not np.all(np.diff(positions) == 1)
):
raise ValueError(
f"Episode {episode_id} must contain every frame exactly once, contiguously from 0 to length-1."
)
rewards = np.full(length, -1.0, dtype=np.float64)
rewards[-1] = 0.0
if interventions is not None and length > 2:
human = interventions[positions]
# Penalize the preceding autonomous frame. Entry into the terminal
# observation and initial human control are not takeover events.
takeovers = (~human[:-2]) & human[1:-1]
rewards[:-2] -= reward.takeover_cost * takeovers
episode_returns = np.cumsum(rewards[::-1])[::-1]
returns[positions] = episode_returns
# A one-frame successful episode has zero cost; use scale 1 to avoid 0/0.
fitted_scales[task] = max(fitted_scales.get(task, 1.0), float(-episode_returns[0]))
positions_by_task.setdefault(task, []).append(positions)
scales = dict(fitted_scales if task_scales is None else task_scales)
targets = np.empty(len(returns), dtype=np.float32)
for task, chunks in positions_by_task.items():
scale = scales.get(task)
if scale is None or not math.isfinite(scale) or scale <= 0:
raise ValueError(f"Missing or invalid saved reward scale for task {task!r}.")
positions = np.concatenate(chunks)
normalized = returns[positions] / scale
if np.any(normalized < -1.0) or np.any(normalized > 0.0):
raise ValueError(
f"Returns for task {task!r} exceed the saved reward scale; refusing to clip or refit."
)
targets[positions] = normalized
return targets, scales