From ec9842d8a8dd0371b4b50021547552e4608ec5ee Mon Sep 17 00:00:00 2001 From: Elvin-yk <180030703+Elvin-yk@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:13:40 +0800 Subject: [PATCH] feat: add vision-only values, takeover rewards and masked action loss --- src/lerobot/configs/value.py | 5 + src/lerobot/configs/value_reward.py | 19 ++ src/lerobot/configs/value_train.py | 4 + .../policies/pi05/configuration_pi05.py | 3 + src/lerobot/policies/pi05/modeling_pi05.py | 34 ++++ src/lerobot/scripts/lerobot_value_infer.py | 53 +++-- src/lerobot/scripts/lerobot_value_train.py | 5 + .../values/pistar06/configuration_pistar06.py | 14 +- .../values/pistar06/modeling_pistar06.py | 186 ++++++++++-------- .../values/pistar06/processor_pistar06.py | 48 +++-- src/lerobot/values/rewards.py | 110 +++++++++++ 11 files changed, 366 insertions(+), 115 deletions(-) create mode 100644 src/lerobot/configs/value_reward.py create mode 100644 src/lerobot/values/rewards.py diff --git a/src/lerobot/configs/value.py b/src/lerobot/configs/value.py index 8c47678..1f0232d 100644 --- a/src/lerobot/configs/value.py +++ b/src/lerobot/configs/value.py @@ -9,6 +9,7 @@ from typing import Any import draccus +from lerobot.configs.value_reward import ValueRewardConfig from lerobot.utils.recording_annotations import normalize_episode_success_label @@ -118,6 +119,8 @@ class ValueInferencePipelineConfig: ) runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig) acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig) + # None inherits the trained checkpoint; old checkpoints resolve to legacy. + reward: ValueRewardConfig | None = None viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig) output_dir: Path | None = None @@ -131,6 +134,8 @@ class ValueInferencePipelineConfig: self.inference.validate() self.runtime.validate() self.acp.validate() + if self.reward is not None: + self.reward.validate() self.viz.validate() if not self.job_name: diff --git a/src/lerobot/configs/value_reward.py b/src/lerobot/configs/value_reward.py new file mode 100644 index 0000000..9fca9b1 --- /dev/null +++ b/src/lerobot/configs/value_reward.py @@ -0,0 +1,19 @@ +"""Reward settings shared by value training and inference.""" + +import math +from dataclasses import dataclass + + +@dataclass +class ValueRewardConfig: + mode: str = "legacy" + takeover_cost: float = 150.0 + intervention_field: str = "complementary_info.is_intervention" + + def validate(self) -> None: + if self.mode not in {"legacy", "time", "time_takeover"}: + raise ValueError("'reward.mode' must be legacy, time or time_takeover.") + if not math.isfinite(self.takeover_cost) or self.takeover_cost < 0: + raise ValueError("'reward.takeover_cost' must be finite and non-negative.") + if self.mode == "time_takeover" and not self.intervention_field: + raise ValueError("'reward.intervention_field' must be non-empty for time_takeover.") diff --git a/src/lerobot/configs/value_train.py b/src/lerobot/configs/value_train.py index 1569331..327f8be 100644 --- a/src/lerobot/configs/value_train.py +++ b/src/lerobot/configs/value_train.py @@ -14,6 +14,7 @@ from huggingface_hub.errors import HfHubHTTPError from lerobot.configs import parser from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig from lerobot.configs.policies import PreTrainedConfig +from lerobot.configs.value_reward import ValueRewardConfig from lerobot.optim import OptimizerConfig from lerobot.optim.schedulers import LRSchedulerConfig from lerobot.utils.hub import HubMixin @@ -72,6 +73,7 @@ class ValueTrainPipelineConfig(HubMixin): optimizer: OptimizerConfig | None = None scheduler: LRSchedulerConfig | None = None + reward: ValueRewardConfig | None = None targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig) wandb: WandBConfig = field(default_factory=WandBConfig) peft: PeftConfig | None = None @@ -113,6 +115,8 @@ class ValueTrainPipelineConfig(HubMixin): ) self.targets.validate() + if self.reward is not None: + self.reward.validate() if hasattr(self.value, "target_key"): self.value.target_key = self.targets.target_field diff --git a/src/lerobot/policies/pi05/configuration_pi05.py b/src/lerobot/policies/pi05/configuration_pi05.py index b96e6d1..eba96c9 100644 --- a/src/lerobot/policies/pi05/configuration_pi05.py +++ b/src/lerobot/policies/pi05/configuration_pi05.py @@ -71,6 +71,9 @@ class PI05Config(PreTrainedConfig): } ) + # Exclude repeated episode-tail actions from flow-matching supervision. + mask_action_padding_loss: bool = False + # Training settings gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization compile_model: bool = False # Whether to use torch.compile for model optimization diff --git a/src/lerobot/policies/pi05/modeling_pi05.py b/src/lerobot/policies/pi05/modeling_pi05.py index 61e4990..67fcc4b 100644 --- a/src/lerobot/policies/pi05/modeling_pi05.py +++ b/src/lerobot/policies/pi05/modeling_pi05.py @@ -1250,6 +1250,16 @@ class PI05Policy(PreTrainedPolicy): - "mean": Return scalar mean loss (default, backward compatible) - "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting """ + action_is_pad = None + if self.config.mask_action_padding_loss: + action_is_pad = batch.get("action_is_pad") + if action_is_pad is None: + raise ValueError( + "mask_action_padding_loss=True requires action_is_pad in the training batch." + ) + if not isinstance(action_is_pad, Tensor) or action_is_pad.dtype != torch.bool: + raise TypeError("action_is_pad must be a boolean tensor (True marks padding).") + # Prepare inputs images, img_masks = self._preprocess_images(batch) tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"] @@ -1263,6 +1273,9 @@ class PI05Policy(PreTrainedPolicy): original_action_dim = self.config.output_features[ACTION].shape[0] losses = losses[:, :, :original_action_dim] + if action_is_pad is not None: + return self._reduce_masked_action_losses(losses, action_is_pad, reduction) + loss_dict = { "loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(), } @@ -1278,6 +1291,27 @@ class PI05Policy(PreTrainedPolicy): loss_dict["loss"] = loss.item() return loss, loss_dict + @staticmethod + def _reduce_masked_action_losses(losses: Tensor, action_is_pad: Tensor, reduction: str): + if tuple(action_is_pad.shape) != tuple(losses.shape[:2]): + raise ValueError("action_is_pad must have shape [batch_size, action_horizon].") + valid = ~action_is_pad.to(device=losses.device) + valid_steps = valid.sum(dim=1) + if torch.any(valid_steps == 0): + raise ValueError("Every action chunk must contain at least one non-padding timestep.") + # Mask with where: even NaN/Inf in an unused loss slot must not enter the sum. + masked = torch.where(valid.unsqueeze(-1), losses, 0.0) + if reduction == "none": + loss = masked.sum(dim=(1, 2)) / (valid_steps * losses.shape[-1]) + elif reduction == "mean": + loss = masked.sum() / (valid_steps.sum() * losses.shape[-1]) + else: + raise ValueError("reduction must be 'mean' or 'none'.") + return loss, { + "loss": loss.detach().mean().item(), + "loss_per_dim": (masked.sum(dim=(0, 1)) / valid_steps.sum()).detach().cpu().tolist(), + } + def _get_default_peft_targets(self) -> dict[str, any]: """Return default PEFT target modules for PI0.5 fine-tuning.""" common_projections = ( diff --git a/src/lerobot/scripts/lerobot_value_infer.py b/src/lerobot/scripts/lerobot_value_infer.py index c498df6..6b1dc68 100644 --- a/src/lerobot/scripts/lerobot_value_infer.py +++ b/src/lerobot/scripts/lerobot_value_infer.py @@ -32,6 +32,7 @@ from tqdm.auto import tqdm from lerobot.configs import parser from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.value import ValueInferencePipelineConfig +from lerobot.configs.value_reward import ValueRewardConfig from lerobot.datasets.lerobot_dataset import LeRobotDataset from lerobot.datasets.utils import load_info, write_info from lerobot.policies.factory import make_policy, make_pre_post_processors @@ -52,6 +53,7 @@ from lerobot.values.pistar06.modeling_pistar06 import ( EpisodeTargetInfo, compute_normalized_value_targets, ) +from lerobot.values.rewards import build_successful_value_targets class ContiguousDistributedEvalSampler(Sampler[int]): @@ -522,6 +524,23 @@ def run_value_inference_pipeline( episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64) frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64) + reward = value_cfg.reward or ValueRewardConfig() + if cfg.reward is not None and cfg.reward != reward: + raise ValueError("Inference reward settings must match the trained checkpoint.") + cfg.reward = reward + success_targets = None + if reward.mode != "legacy": + if not value_cfg.reward_scales: + raise ValueError( + "Checkpoint is missing training reward scales; refusing to fit on inference data." + ) + success_targets, _ = build_successful_value_targets( + dataset, + reward, + cfg.dataset.success_field, + task_scales=value_cfg.reward_scales, + ) + if cfg.acp.intervention_field in raw_frames.column_names: interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32) else: @@ -595,6 +614,8 @@ def run_value_inference_pipeline( raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.") predicted_values = prediction_lookup[absolute_indices] + if success_targets is not None: + predicted_values[success_targets == 0.0] = 0.0 logging.info( "Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f", float(np.min(predicted_values)), @@ -614,21 +635,23 @@ def run_value_inference_pipeline( thresholds: dict[int, float] | None = None if cfg.acp.enable: - episode_info, task_max_lengths = _build_episode_info( - dataset=dataset, - success_field=cfg.dataset.success_field, - default_success=cfg.dataset.default_success, - ) - - value_targets = compute_normalized_value_targets( - episode_indices=episode_indices, - frame_indices=frame_indices, - episode_info=episode_info, - task_max_lengths=task_max_lengths, - c_fail_coef=cfg.acp.c_fail_coef, - clip_min=value_cfg.bin_min, - clip_max=value_cfg.bin_max, - ) + if success_targets is None: + episode_info, task_max_lengths = _build_episode_info( + dataset=dataset, + success_field=cfg.dataset.success_field, + default_success=cfg.dataset.default_success, + ) + value_targets = compute_normalized_value_targets( + episode_indices=episode_indices, + frame_indices=frame_indices, + episode_info=episode_info, + task_max_lengths=task_max_lengths, + c_fail_coef=cfg.acp.c_fail_coef, + clip_min=value_cfg.bin_min, + clip_max=value_cfg.bin_max, + ) + else: + value_targets = success_targets rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices) advantages = _compute_n_step_advantages( rewards=rewards, diff --git a/src/lerobot/scripts/lerobot_value_train.py b/src/lerobot/scripts/lerobot_value_train.py index 91e4610..57aa2ec 100644 --- a/src/lerobot/scripts/lerobot_value_train.py +++ b/src/lerobot/scripts/lerobot_value_train.py @@ -131,7 +131,9 @@ def value_train( value_target_raw_batch_hook = value_model.build_training_raw_batch_hook( dataset=dataset, targets_cfg=cfg.targets, + reward_cfg=cfg.reward, ) + cfg.reward = value_model.config.reward if is_main_process: logging.info("Using value model raw-batch hook for target construction.") elif is_main_process: @@ -163,6 +165,9 @@ def value_train( "norm_map": cfg.value.normalization_mapping, }, } + if cfg.value.mode == "vision_only": + processor_kwargs["preprocessor_overrides"].pop("normalizer_processor", None) + postprocessor_kwargs["postprocessor_overrides"].pop("unnormalizer_processor", None) preprocessor, postprocessor = make_pre_post_processors( policy_cfg=cfg.value, diff --git a/src/lerobot/values/pistar06/configuration_pistar06.py b/src/lerobot/values/pistar06/configuration_pistar06.py index 665459f..ee85cd0 100644 --- a/src/lerobot/values/pistar06/configuration_pistar06.py +++ b/src/lerobot/values/pistar06/configuration_pistar06.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.types import NormalizationMode +from lerobot.configs.value_reward import ValueRewardConfig from lerobot.optim.optimizers import AdamWConfig from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig from lerobot.utils.constants import OBS_STATE @@ -12,7 +13,12 @@ from lerobot.utils.constants import OBS_STATE @PreTrainedConfig.register_subclass("pistar06") @dataclass class Pistar06Config(PreTrainedConfig): - """Value model config using the Pistar06 stack (SigLIP + Gemma).""" + """SigLIP value model with optional Gemma language conditioning.""" + + # Old checkpoints without this field retain the language-conditioned path. + mode: str = "vision_language" + reward: ValueRewardConfig | None = None + reward_scales: dict[str, float] = field(default_factory=dict) # Backbone components vision_repo_id: str = "google/siglip-so400m-patch14-384" @@ -70,9 +76,13 @@ class Pistar06Config(PreTrainedConfig): def __post_init__(self) -> None: super().__post_init__() + if self.mode not in {"vision_language", "vision_only"}: + raise ValueError("'value.mode' must be vision_language or vision_only.") + if self.reward is not None: + self.reward.validate() if not self.vision_repo_id: raise ValueError("'value.vision_repo_id' must be non-empty.") - if not self.language_repo_id: + if self.mode == "vision_language" and not self.language_repo_id: raise ValueError("'value.language_repo_id' must be non-empty.") if not self.task_field: raise ValueError("'value.task_field' must be non-empty.") diff --git a/src/lerobot/values/pistar06/modeling_pistar06.py b/src/lerobot/values/pistar06/modeling_pistar06.py index 91d4c9f..f86f31f 100644 --- a/src/lerobot/values/pistar06/modeling_pistar06.py +++ b/src/lerobot/values/pistar06/modeling_pistar06.py @@ -19,20 +19,29 @@ from huggingface_hub.errors import HfHubHTTPError from safetensors.torch import save_file from torch import Tensor, nn +from lerobot.configs.value_reward import ValueRewardConfig from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS from lerobot.utils.import_utils import _transformers_available from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY +from lerobot.values.rewards import build_successful_value_targets if TYPE_CHECKING or _transformers_available: - from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoModelForCausalLM + from transformers import ( + AutoConfig, + AutoImageProcessor, + AutoModel, + AutoModelForCausalLM, + SiglipVisionModel, + ) else: AutoConfig = None AutoImageProcessor = None AutoModel = None AutoModelForCausalLM = None + SiglipVisionModel = None PISTAR06_SAVE_INFO = "pistar06_save_info.json" @@ -261,16 +270,20 @@ class Pistar06Model(nn.Module): self.cfg = cfg self.model_dtype = _resolve_load_dtype(cfg.dtype) - self.vision_encoder = AutoModel.from_pretrained( + # SiglipVisionModel loads only the visual weights; AutoModel on a full + # SigLIP repository would also create an unused text encoder under DDP. + vision_loader = SiglipVisionModel if cfg.mode == "vision_only" else AutoModel + self.vision_encoder = vision_loader.from_pretrained( cfg.vision_repo_id, revision=cfg.vision_revision, torch_dtype=self.model_dtype, ) - self.language_model = _load_language_model( - repo_id=cfg.language_repo_id, - revision=cfg.language_revision, - dtype=self.model_dtype, - ) + if cfg.mode == "vision_language": + self.language_model = _load_language_model( + repo_id=cfg.language_repo_id, + revision=cfg.language_revision, + dtype=self.model_dtype, + ) image_processor = AutoImageProcessor.from_pretrained( cfg.vision_repo_id, @@ -292,31 +305,35 @@ class Pistar06Model(nn.Module): ) vision_feature_size = _extract_vision_feature_size(self.vision_encoder) - language_hidden_size = _extract_hidden_size(self.language_model) self.image_projector = nn.Sequential( nn.Linear(vision_feature_size, cfg.fusion_hidden_dim), nn.GELU(), nn.Dropout(cfg.dropout), ) - self.language_projector = nn.Sequential( - nn.Linear(language_hidden_size, cfg.fusion_hidden_dim), - nn.GELU(), - nn.Dropout(cfg.dropout), - ) - self.final_norm = nn.LayerNorm(cfg.fusion_hidden_dim * 2) + head_input_dim = cfg.fusion_hidden_dim + if cfg.mode == "vision_language": + language_hidden_size = _extract_hidden_size(self.language_model) + self.language_projector = nn.Sequential( + nn.Linear(language_hidden_size, cfg.fusion_hidden_dim), + nn.GELU(), + nn.Dropout(cfg.dropout), + ) + head_input_dim *= 2 + self.final_norm = nn.LayerNorm(head_input_dim) self.value_head = nn.Sequential( - nn.Linear(cfg.fusion_hidden_dim * 2, cfg.fusion_hidden_dim), + nn.Linear(head_input_dim, cfg.fusion_hidden_dim), nn.GELU(), nn.Dropout(cfg.dropout), nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins), ) if cfg.use_gradient_checkpointing: - _maybe_enable_gradient_checkpointing(self.language_model) + if cfg.mode == "vision_language": + _maybe_enable_gradient_checkpointing(self.language_model) _maybe_enable_gradient_checkpointing(self.vision_encoder) - if cfg.freeze_language_model: + if cfg.mode == "vision_language" and cfg.freeze_language_model: _freeze_module(self.language_model) if cfg.freeze_vision_encoder: _freeze_module(self.vision_encoder) @@ -358,6 +375,10 @@ class Pistar06Model(nn.Module): if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras: raise ValueError("Batch shape mismatch between images and image_attention_mask.") + if self.cfg.mode == "vision_only": + # Missing cameras must not influence detection of the input pixel range. + images = torch.where(image_attention_mask[:, :, None, None, None], images, 0) + if images.dtype == torch.uint8: images = images.to(dtype=torch.float32) / 255.0 else: @@ -393,36 +414,29 @@ class Pistar06Model(nn.Module): def forward( self, - input_ids: Tensor, - attention_mask: Tensor, - images: Tensor, - image_attention_mask: Tensor, + input_ids: Tensor | None = None, + attention_mask: Tensor | None = None, + images: Tensor | None = None, + image_attention_mask: Tensor | None = None, ) -> Tensor: - if input_ids.ndim != 2: - raise ValueError(f"'input_ids' must have shape [B, T], got {tuple(input_ids.shape)}.") - if attention_mask.ndim != 2: - raise ValueError(f"'attention_mask' must have shape [B, T], got {tuple(attention_mask.shape)}.") - if images.ndim != 5: - raise ValueError(f"'images' must have shape [B, N, C, H, W], got {tuple(images.shape)}.") - if image_attention_mask.ndim != 2: - raise ValueError( - f"'image_attention_mask' must have shape [B, N], got {tuple(image_attention_mask.shape)}." - ) - - bsize = input_ids.shape[0] - if attention_mask.shape[0] != bsize: - raise ValueError("Batch size mismatch between input_ids and attention_mask.") - if images.shape[0] != bsize or image_attention_mask.shape[0] != bsize: - raise ValueError("Batch size mismatch between language and image inputs.") - if images.shape[1] == 0: + if images is None or images.ndim != 5: + raise ValueError("'images' must have shape [B, N, C, H, W].") + bsize, num_cameras = images.shape[:2] + if num_cameras == 0: raise ValueError("At least one camera is required for Pistar06Model.") - + if image_attention_mask is None or tuple(image_attention_mask.shape) != (bsize, num_cameras): + raise ValueError("'image_attention_mask' must have shape [B, N].") image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device) if not torch.all(image_attention_mask.any(dim=1)): raise ValueError("Each sample must have at least one valid camera input.") - language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device) - if not torch.all(language_mask.any(dim=1)): - raise ValueError("Each sample must have at least one valid language token.") + if self.cfg.mode == "vision_language": + if input_ids is None or attention_mask is None: + raise ValueError("vision_language requires input_ids and attention_mask.") + if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape or input_ids.shape[0] != bsize: + raise ValueError("Language tensors must have matching shape [B, T].") + language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device) + if not torch.all(language_mask.any(dim=1)): + raise ValueError("Each sample must have at least one valid language token.") processed_images = self._preprocess_images(images, image_attention_mask) num_cameras = processed_images.shape[1] @@ -433,15 +447,15 @@ class Pistar06Model(nn.Module): with image_context: image_features = self._encode_images(flat_images) - language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext() - with language_context: - language_features = self._encode_language( - input_ids=input_ids, attention_mask=language_mask.long() - ) + if self.cfg.mode == "vision_language": + language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext() + with language_context: + language_features = self._encode_language( + input_ids=input_ids, attention_mask=language_mask.long() + ) feature_dtype = torch.float32 image_features = image_features.to(dtype=feature_dtype) - language_features = language_features.to(dtype=feature_dtype) image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1) camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype) @@ -451,10 +465,11 @@ class Pistar06Model(nn.Module): image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0) ) image_pooled = image_tokens.sum(dim=1) / camera_denominator - language_token = self.language_projector(language_features) + if self.cfg.mode == "vision_only": + return self.value_head(self.final_norm(image_pooled)) - joint_features = torch.cat([image_pooled, language_token], dim=-1) - return self.value_head(self.final_norm(joint_features)) + language_token = self.language_projector(language_features.to(dtype=feature_dtype)) + return self.value_head(self.final_norm(torch.cat([image_pooled, language_token], dim=-1))) class Pistar06Policy(PreTrainedPolicy): @@ -482,7 +497,7 @@ class Pistar06Policy(PreTrainedPolicy): prefixes: list[str] = [] if self.config.freeze_vision_encoder: prefixes.append("model.vision_encoder.") - if self.config.freeze_language_model: + if self.config.mode == "vision_language" and self.config.freeze_language_model: prefixes.append("model.language_model.") return prefixes @@ -626,8 +641,8 @@ class Pistar06Policy(PreTrainedPolicy): raise RuntimeError("Pistar06Policy is a value model and does not support action selection.") def predict_value(self, batch: dict[str, Tensor]) -> Tensor: - input_ids = batch[OBS_LANGUAGE_TOKENS] - attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK] + input_ids = batch.get(OBS_LANGUAGE_TOKENS) + attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK) images = batch[PISTAR06_IMAGES_KEY] image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY] @@ -640,7 +655,7 @@ class Pistar06Policy(PreTrainedPolicy): bin_centers = self.bin_centers.to(device=logits.device) return expected_value_from_logits(logits, bin_centers) - def build_training_raw_batch_hook(self, dataset, targets_cfg): + def build_training_raw_batch_hook(self, dataset, targets_cfg, reward_cfg=None): raw_frames = dataset.hf_dataset.with_format(None) frame_count = len(raw_frames) if frame_count == 0: @@ -650,6 +665,42 @@ class Pistar06Policy(PreTrainedPolicy): frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64) absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64) + reward = reward_cfg or self.config.reward or ValueRewardConfig() + reward.validate() + if reward.mode != "legacy": + saved_scales = self.config.reward_scales if self.config.reward == reward else None + value_targets, scales = build_successful_value_targets( + dataset, + reward, + targets_cfg.success_field, + task_scales=saved_scales or None, + ) + self.config.reward = reward + self.config.reward_scales = scales + else: + value_targets = self._legacy_training_targets( + dataset, targets_cfg, episode_indices, frame_indices + ) + self.config.reward = reward + self.config.reward_scales = {} + + max_index = int(np.max(absolute_indices)) + value_target_lookup = np.zeros(max_index + 1, dtype=np.float32) + value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False) + target_key = targets_cfg.target_field + + def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]: + del step + batch_indices = batch.get("index") + if batch_indices is None: + raise KeyError("Missing 'index' in batch while building value targets.") + batch_indices = torch.as_tensor(batch_indices).detach().cpu().numpy().astype(np.int64).reshape(-1) + batch[target_key] = torch.from_numpy(value_target_lookup[batch_indices]) + return batch + + return value_target_hook + + def _legacy_training_targets(self, dataset, targets_cfg, episode_indices, frame_indices): episodes_ds = dataset.meta.episodes.with_format(None) episodes = episodes_ds[:] n_episodes = len(episodes_ds) @@ -692,26 +743,7 @@ class Pistar06Policy(PreTrainedPolicy): clip_max=self.config.bin_max, ) - max_index = int(np.max(absolute_indices)) - value_target_lookup = np.zeros(max_index + 1, dtype=np.float32) - value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False) - - target_key = targets_cfg.target_field - - def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]: - del step - batch_indices = batch.get("index") - if batch_indices is None: - raise KeyError("Missing 'index' in batch while building value targets.") - if not isinstance(batch_indices, torch.Tensor): - batch_indices = torch.as_tensor(batch_indices) - - batch_indices_np = batch_indices.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1) - target_values = torch.from_numpy(value_target_lookup[batch_indices_np]).to(dtype=torch.float32) - batch[target_key] = target_values - return batch - - return value_target_hook + return value_targets def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]: if self.config.target_key not in batch: @@ -720,8 +752,8 @@ class Pistar06Policy(PreTrainedPolicy): "Make sure lerobot-value-train target hook is enabled." ) - input_ids = batch[OBS_LANGUAGE_TOKENS] - attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK] + input_ids = batch.get(OBS_LANGUAGE_TOKENS) + attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK) images = batch[PISTAR06_IMAGES_KEY] image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY] diff --git a/src/lerobot/values/pistar06/processor_pistar06.py b/src/lerobot/values/pistar06/processor_pistar06.py index 40d4088..49e7c9f 100644 --- a/src/lerobot/values/pistar06/processor_pistar06.py +++ b/src/lerobot/values/pistar06/processor_pistar06.py @@ -218,8 +218,33 @@ def make_pistar06_pre_post_processors( if not camera_features: camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)] - input_steps: list[ProcessorStep] = [ - RenameObservationsProcessorStep(rename_map={}), + input_steps: list[ProcessorStep] = [RenameObservationsProcessorStep(rename_map={})] + if config.mode == "vision_language": + input_steps.extend(_language_steps(config, dataset_stats)) + input_steps.extend( + [ + Pistar06PrepareImagesProcessorStep(camera_features=camera_features), + DeviceProcessorStep(device=config.device), + ] + ) + + output_steps: list[ProcessorStep] = [DeviceProcessorStep(device="cpu")] + return ( + PolicyProcessorPipeline[dict[str, Any], dict[str, Any]]( + steps=input_steps, + name=POLICY_PREPROCESSOR_DEFAULT_NAME, + ), + PolicyProcessorPipeline[PolicyAction, PolicyAction]( + steps=output_steps, + name=POLICY_POSTPROCESSOR_DEFAULT_NAME, + to_transition=policy_action_to_transition, + to_output=transition_to_policy_action, + ), + ) + + +def _language_steps(config: Pistar06Config, dataset_stats) -> list[ProcessorStep]: + return [ NormalizerProcessorStep( features={**(config.input_features or {}), **(config.output_features or {})}, norm_map=config.normalization_mapping, @@ -241,23 +266,4 @@ def make_pistar06_pre_post_processors( padding="max_length", truncation=True, ), - Pistar06PrepareImagesProcessorStep(camera_features=camera_features), - DeviceProcessorStep(device=config.device), ] - - output_steps: list[ProcessorStep] = [ - DeviceProcessorStep(device="cpu"), - ] - - return ( - PolicyProcessorPipeline[dict[str, Any], dict[str, Any]]( - steps=input_steps, - name=POLICY_PREPROCESSOR_DEFAULT_NAME, - ), - PolicyProcessorPipeline[PolicyAction, PolicyAction]( - steps=output_steps, - name=POLICY_POSTPROCESSOR_DEFAULT_NAME, - to_transition=policy_action_to_transition, - to_output=transition_to_policy_action, - ), - ) diff --git a/src/lerobot/values/rewards.py b/src/lerobot/values/rewards.py new file mode 100644 index 0000000..737be6d --- /dev/null +++ b/src/lerobot/values/rewards.py @@ -0,0 +1,110 @@ +"""Success-only Monte Carlo targets with task scales fitted on training episodes.""" + +import math + +import numpy as np + +from lerobot.configs.value_reward import ValueRewardConfig +from lerobot.utils.recording_annotations import EPISODE_SUCCESS, normalize_episode_success_label + + +def build_successful_value_targets( + dataset, + reward: ValueRewardConfig, + success_field: str, + *, + task_scales: dict[str, float] | None = None, +) -> tuple[np.ndarray, dict[str, float]]: + """Return targets in raw frame order; reject partial, failed or unlabelled episodes. + + Only episodes actually present in hf_dataset contribute to the fitted scale. + Task names, rather than dataset-local task indices, identify checkpoint scales. + Supplying scales reuses them without refitting or clipping out-of-range returns. + """ + reward.validate() + if reward.mode == "legacy": + raise ValueError("Use the legacy target builder for reward.mode=legacy.") + + frames = dataset.hf_dataset.with_format(None) + episode_ids = np.asarray(frames["episode_index"], dtype=np.int64).reshape(-1) + frame_ids = np.asarray(frames["frame_index"], dtype=np.int64).reshape(-1) + if len(episode_ids) == 0: + raise ValueError("Dataset has no frames.") + episodes = dataset.meta.episodes.with_format(None) + if success_field not in episodes.column_names: + raise ValueError(f"reward.mode={reward.mode} requires explicit '{success_field}=success' labels.") + meta = episodes[:] + active_ids = set(episode_ids.tolist()) + metadata = {} + for i, episode_id in enumerate(meta["episode_index"]): + episode_id = int(episode_id) + if episode_id not in active_ids: + continue + if episode_id in metadata: + raise ValueError(f"Duplicate episode metadata for episode {episode_id}.") + label = normalize_episode_success_label(meta[success_field][i]) + if label != EPISODE_SUCCESS: + raise ValueError( + f"reward.mode={reward.mode} requires only explicitly successful episodes; " + f"episode {episode_id} has label {label!r}." + ) + tasks = meta["tasks"][i] + if not isinstance(tasks, str): + if len(tasks) != 1: + raise ValueError(f"Episode {episode_id} must have exactly one task for reward normalization.") + tasks = tasks[0] + metadata[episode_id] = (int(meta["length"][i]), str(tasks)) + if active_ids != set(metadata): + raise ValueError("Missing episode metadata for reward construction.") + + interventions = None + if reward.mode == "time_takeover": + if reward.intervention_field not in frames.column_names: + raise ValueError(f"Missing intervention field '{reward.intervention_field}' for time_takeover.") + interventions = np.asarray(frames[reward.intervention_field]).reshape(-1) + if interventions.shape != episode_ids.shape or not np.isin(interventions, [0, 1]).all(): + raise ValueError("Intervention flags must contain one boolean or binary 0/1 value per frame.") + interventions = interventions.astype(bool) + + returns = np.empty(len(episode_ids), dtype=np.float64) + fitted_scales: dict[str, float] = {} + positions_by_task: dict[str, list[np.ndarray]] = {} + for episode_id in sorted(active_ids): + positions = np.flatnonzero(episode_ids == episode_id) + length, task = metadata[episode_id] + if ( + length <= 0 + or not np.array_equal(frame_ids[positions], np.arange(length)) + or not np.all(np.diff(positions) == 1) + ): + raise ValueError( + f"Episode {episode_id} must contain every frame exactly once, contiguously from 0 to length-1." + ) + rewards = np.full(length, -1.0, dtype=np.float64) + rewards[-1] = 0.0 + if interventions is not None and length > 2: + human = interventions[positions] + # Penalize the preceding autonomous frame. Entry into the terminal + # observation and initial human control are not takeover events. + takeovers = (~human[:-2]) & human[1:-1] + rewards[:-2] -= reward.takeover_cost * takeovers + episode_returns = np.cumsum(rewards[::-1])[::-1] + returns[positions] = episode_returns + # A one-frame successful episode has zero cost; use scale 1 to avoid 0/0. + fitted_scales[task] = max(fitted_scales.get(task, 1.0), float(-episode_returns[0])) + positions_by_task.setdefault(task, []).append(positions) + + scales = dict(fitted_scales if task_scales is None else task_scales) + targets = np.empty(len(returns), dtype=np.float32) + for task, chunks in positions_by_task.items(): + scale = scales.get(task) + if scale is None or not math.isfinite(scale) or scale <= 0: + raise ValueError(f"Missing or invalid saved reward scale for task {task!r}.") + positions = np.concatenate(chunks) + normalized = returns[positions] / scale + if np.any(normalized < -1.0) or np.any(normalized > 0.0): + raise ValueError( + f"Returns for task {task!r} exceed the saved reward scale; refusing to clip or refit." + ) + targets[positions] = normalized + return targets, scales