feat: add vision-only values, takeover rewards and masked action loss

这个提交包含在:
Elvin-yk
2026-10-16 22:13:40 +08:00
父节点 25a23467d2
当前提交 ec9842d8a8
共有 11 个文件被更改,包括 366 次插入 和 115 次删除

查看文件

@ -9,6 +9,7 @@ from typing import Any
import draccus import draccus
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.utils.recording_annotations import normalize_episode_success_label from lerobot.utils.recording_annotations import normalize_episode_success_label
@ -118,6 +119,8 @@ class ValueInferencePipelineConfig:
) )
runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig) runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig)
acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig) acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig)
# None inherits the trained checkpoint; old checkpoints resolve to legacy.
reward: ValueRewardConfig | None = None
viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig) viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig)
output_dir: Path | None = None output_dir: Path | None = None
@ -131,6 +134,8 @@ class ValueInferencePipelineConfig:
self.inference.validate() self.inference.validate()
self.runtime.validate() self.runtime.validate()
self.acp.validate() self.acp.validate()
if self.reward is not None:
self.reward.validate()
self.viz.validate() self.viz.validate()
if not self.job_name: if not self.job_name:

查看文件

@ -0,0 +1,19 @@
"""Reward settings shared by value training and inference."""
import math
from dataclasses import dataclass
@dataclass
class ValueRewardConfig:
mode: str = "legacy"
takeover_cost: float = 150.0
intervention_field: str = "complementary_info.is_intervention"
def validate(self) -> None:
if self.mode not in {"legacy", "time", "time_takeover"}:
raise ValueError("'reward.mode' must be legacy, time or time_takeover.")
if not math.isfinite(self.takeover_cost) or self.takeover_cost < 0:
raise ValueError("'reward.takeover_cost' must be finite and non-negative.")
if self.mode == "time_takeover" and not self.intervention_field:
raise ValueError("'reward.intervention_field' must be non-empty for time_takeover.")

查看文件

@ -14,6 +14,7 @@ from huggingface_hub.errors import HfHubHTTPError
from lerobot.configs import parser from lerobot.configs import parser
from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig
from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.optim import OptimizerConfig from lerobot.optim import OptimizerConfig
from lerobot.optim.schedulers import LRSchedulerConfig from lerobot.optim.schedulers import LRSchedulerConfig
from lerobot.utils.hub import HubMixin from lerobot.utils.hub import HubMixin
@ -72,6 +73,7 @@ class ValueTrainPipelineConfig(HubMixin):
optimizer: OptimizerConfig | None = None optimizer: OptimizerConfig | None = None
scheduler: LRSchedulerConfig | None = None scheduler: LRSchedulerConfig | None = None
reward: ValueRewardConfig | None = None
targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig) targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig)
wandb: WandBConfig = field(default_factory=WandBConfig) wandb: WandBConfig = field(default_factory=WandBConfig)
peft: PeftConfig | None = None peft: PeftConfig | None = None
@ -113,6 +115,8 @@ class ValueTrainPipelineConfig(HubMixin):
) )
self.targets.validate() self.targets.validate()
if self.reward is not None:
self.reward.validate()
if hasattr(self.value, "target_key"): if hasattr(self.value, "target_key"):
self.value.target_key = self.targets.target_field self.value.target_key = self.targets.target_field

查看文件

@ -71,6 +71,9 @@ class PI05Config(PreTrainedConfig):
} }
) )
# Exclude repeated episode-tail actions from flow-matching supervision.
mask_action_padding_loss: bool = False
# Training settings # Training settings
gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization
compile_model: bool = False # Whether to use torch.compile for model optimization compile_model: bool = False # Whether to use torch.compile for model optimization

查看文件

@ -1250,6 +1250,16 @@ class PI05Policy(PreTrainedPolicy):
- "mean": Return scalar mean loss (default, backward compatible) - "mean": Return scalar mean loss (default, backward compatible)
- "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting - "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting
""" """
action_is_pad = None
if self.config.mask_action_padding_loss:
action_is_pad = batch.get("action_is_pad")
if action_is_pad is None:
raise ValueError(
"mask_action_padding_loss=True requires action_is_pad in the training batch."
)
if not isinstance(action_is_pad, Tensor) or action_is_pad.dtype != torch.bool:
raise TypeError("action_is_pad must be a boolean tensor (True marks padding).")
# Prepare inputs # Prepare inputs
images, img_masks = self._preprocess_images(batch) images, img_masks = self._preprocess_images(batch)
tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"] tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
@ -1263,6 +1273,9 @@ class PI05Policy(PreTrainedPolicy):
original_action_dim = self.config.output_features[ACTION].shape[0] original_action_dim = self.config.output_features[ACTION].shape[0]
losses = losses[:, :, :original_action_dim] losses = losses[:, :, :original_action_dim]
if action_is_pad is not None:
return self._reduce_masked_action_losses(losses, action_is_pad, reduction)
loss_dict = { loss_dict = {
"loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(), "loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(),
} }
@ -1278,6 +1291,27 @@ class PI05Policy(PreTrainedPolicy):
loss_dict["loss"] = loss.item() loss_dict["loss"] = loss.item()
return loss, loss_dict return loss, loss_dict
@staticmethod
def _reduce_masked_action_losses(losses: Tensor, action_is_pad: Tensor, reduction: str):
if tuple(action_is_pad.shape) != tuple(losses.shape[:2]):
raise ValueError("action_is_pad must have shape [batch_size, action_horizon].")
valid = ~action_is_pad.to(device=losses.device)
valid_steps = valid.sum(dim=1)
if torch.any(valid_steps == 0):
raise ValueError("Every action chunk must contain at least one non-padding timestep.")
# Mask with where: even NaN/Inf in an unused loss slot must not enter the sum.
masked = torch.where(valid.unsqueeze(-1), losses, 0.0)
if reduction == "none":
loss = masked.sum(dim=(1, 2)) / (valid_steps * losses.shape[-1])
elif reduction == "mean":
loss = masked.sum() / (valid_steps.sum() * losses.shape[-1])
else:
raise ValueError("reduction must be 'mean' or 'none'.")
return loss, {
"loss": loss.detach().mean().item(),
"loss_per_dim": (masked.sum(dim=(0, 1)) / valid_steps.sum()).detach().cpu().tolist(),
}
def _get_default_peft_targets(self) -> dict[str, any]: def _get_default_peft_targets(self) -> dict[str, any]:
"""Return default PEFT target modules for PI0.5 fine-tuning.""" """Return default PEFT target modules for PI0.5 fine-tuning."""
common_projections = ( common_projections = (

查看文件

@ -32,6 +32,7 @@ from tqdm.auto import tqdm
from lerobot.configs import parser from lerobot.configs import parser
from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.value import ValueInferencePipelineConfig from lerobot.configs.value import ValueInferencePipelineConfig
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.datasets.lerobot_dataset import LeRobotDataset from lerobot.datasets.lerobot_dataset import LeRobotDataset
from lerobot.datasets.utils import load_info, write_info from lerobot.datasets.utils import load_info, write_info
from lerobot.policies.factory import make_policy, make_pre_post_processors from lerobot.policies.factory import make_policy, make_pre_post_processors
@ -52,6 +53,7 @@ from lerobot.values.pistar06.modeling_pistar06 import (
EpisodeTargetInfo, EpisodeTargetInfo,
compute_normalized_value_targets, compute_normalized_value_targets,
) )
from lerobot.values.rewards import build_successful_value_targets
class ContiguousDistributedEvalSampler(Sampler[int]): class ContiguousDistributedEvalSampler(Sampler[int]):
@ -522,6 +524,23 @@ def run_value_inference_pipeline(
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64) episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64) frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
reward = value_cfg.reward or ValueRewardConfig()
if cfg.reward is not None and cfg.reward != reward:
raise ValueError("Inference reward settings must match the trained checkpoint.")
cfg.reward = reward
success_targets = None
if reward.mode != "legacy":
if not value_cfg.reward_scales:
raise ValueError(
"Checkpoint is missing training reward scales; refusing to fit on inference data."
)
success_targets, _ = build_successful_value_targets(
dataset,
reward,
cfg.dataset.success_field,
task_scales=value_cfg.reward_scales,
)
if cfg.acp.intervention_field in raw_frames.column_names: if cfg.acp.intervention_field in raw_frames.column_names:
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32) interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
else: else:
@ -595,6 +614,8 @@ def run_value_inference_pipeline(
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.") raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
predicted_values = prediction_lookup[absolute_indices] predicted_values = prediction_lookup[absolute_indices]
if success_targets is not None:
predicted_values[success_targets == 0.0] = 0.0
logging.info( logging.info(
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f", "Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
float(np.min(predicted_values)), float(np.min(predicted_values)),
@ -614,12 +635,12 @@ def run_value_inference_pipeline(
thresholds: dict[int, float] | None = None thresholds: dict[int, float] | None = None
if cfg.acp.enable: if cfg.acp.enable:
if success_targets is None:
episode_info, task_max_lengths = _build_episode_info( episode_info, task_max_lengths = _build_episode_info(
dataset=dataset, dataset=dataset,
success_field=cfg.dataset.success_field, success_field=cfg.dataset.success_field,
default_success=cfg.dataset.default_success, default_success=cfg.dataset.default_success,
) )
value_targets = compute_normalized_value_targets( value_targets = compute_normalized_value_targets(
episode_indices=episode_indices, episode_indices=episode_indices,
frame_indices=frame_indices, frame_indices=frame_indices,
@ -629,6 +650,8 @@ def run_value_inference_pipeline(
clip_min=value_cfg.bin_min, clip_min=value_cfg.bin_min,
clip_max=value_cfg.bin_max, clip_max=value_cfg.bin_max,
) )
else:
value_targets = success_targets
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices) rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
advantages = _compute_n_step_advantages( advantages = _compute_n_step_advantages(
rewards=rewards, rewards=rewards,

查看文件

@ -131,7 +131,9 @@ def value_train(
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook( value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
dataset=dataset, dataset=dataset,
targets_cfg=cfg.targets, targets_cfg=cfg.targets,
reward_cfg=cfg.reward,
) )
cfg.reward = value_model.config.reward
if is_main_process: if is_main_process:
logging.info("Using value model raw-batch hook for target construction.") logging.info("Using value model raw-batch hook for target construction.")
elif is_main_process: elif is_main_process:
@ -163,6 +165,9 @@ def value_train(
"norm_map": cfg.value.normalization_mapping, "norm_map": cfg.value.normalization_mapping,
}, },
} }
if cfg.value.mode == "vision_only":
processor_kwargs["preprocessor_overrides"].pop("normalizer_processor", None)
postprocessor_kwargs["postprocessor_overrides"].pop("unnormalizer_processor", None)
preprocessor, postprocessor = make_pre_post_processors( preprocessor, postprocessor = make_pre_post_processors(
policy_cfg=cfg.value, policy_cfg=cfg.value,

查看文件

@ -4,6 +4,7 @@ from dataclasses import dataclass, field
from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.types import NormalizationMode from lerobot.configs.types import NormalizationMode
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.optim.optimizers import AdamWConfig from lerobot.optim.optimizers import AdamWConfig
from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
from lerobot.utils.constants import OBS_STATE from lerobot.utils.constants import OBS_STATE
@ -12,7 +13,12 @@ from lerobot.utils.constants import OBS_STATE
@PreTrainedConfig.register_subclass("pistar06") @PreTrainedConfig.register_subclass("pistar06")
@dataclass @dataclass
class Pistar06Config(PreTrainedConfig): class Pistar06Config(PreTrainedConfig):
"""Value model config using the Pistar06 stack (SigLIP + Gemma).""" """SigLIP value model with optional Gemma language conditioning."""
# Old checkpoints without this field retain the language-conditioned path.
mode: str = "vision_language"
reward: ValueRewardConfig | None = None
reward_scales: dict[str, float] = field(default_factory=dict)
# Backbone components # Backbone components
vision_repo_id: str = "google/siglip-so400m-patch14-384" vision_repo_id: str = "google/siglip-so400m-patch14-384"
@ -70,9 +76,13 @@ class Pistar06Config(PreTrainedConfig):
def __post_init__(self) -> None: def __post_init__(self) -> None:
super().__post_init__() super().__post_init__()
if self.mode not in {"vision_language", "vision_only"}:
raise ValueError("'value.mode' must be vision_language or vision_only.")
if self.reward is not None:
self.reward.validate()
if not self.vision_repo_id: if not self.vision_repo_id:
raise ValueError("'value.vision_repo_id' must be non-empty.") raise ValueError("'value.vision_repo_id' must be non-empty.")
if not self.language_repo_id: if self.mode == "vision_language" and not self.language_repo_id:
raise ValueError("'value.language_repo_id' must be non-empty.") raise ValueError("'value.language_repo_id' must be non-empty.")
if not self.task_field: if not self.task_field:
raise ValueError("'value.task_field' must be non-empty.") raise ValueError("'value.task_field' must be non-empty.")

查看文件

@ -19,20 +19,29 @@ from huggingface_hub.errors import HfHubHTTPError
from safetensors.torch import save_file from safetensors.torch import save_file
from torch import Tensor, nn from torch import Tensor, nn
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
from lerobot.utils.import_utils import _transformers_available from lerobot.utils.import_utils import _transformers_available
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY
from lerobot.values.rewards import build_successful_value_targets
if TYPE_CHECKING or _transformers_available: if TYPE_CHECKING or _transformers_available:
from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoModelForCausalLM from transformers import (
AutoConfig,
AutoImageProcessor,
AutoModel,
AutoModelForCausalLM,
SiglipVisionModel,
)
else: else:
AutoConfig = None AutoConfig = None
AutoImageProcessor = None AutoImageProcessor = None
AutoModel = None AutoModel = None
AutoModelForCausalLM = None AutoModelForCausalLM = None
SiglipVisionModel = None
PISTAR06_SAVE_INFO = "pistar06_save_info.json" PISTAR06_SAVE_INFO = "pistar06_save_info.json"
@ -261,11 +270,15 @@ class Pistar06Model(nn.Module):
self.cfg = cfg self.cfg = cfg
self.model_dtype = _resolve_load_dtype(cfg.dtype) self.model_dtype = _resolve_load_dtype(cfg.dtype)
self.vision_encoder = AutoModel.from_pretrained( # SiglipVisionModel loads only the visual weights; AutoModel on a full
# SigLIP repository would also create an unused text encoder under DDP.
vision_loader = SiglipVisionModel if cfg.mode == "vision_only" else AutoModel
self.vision_encoder = vision_loader.from_pretrained(
cfg.vision_repo_id, cfg.vision_repo_id,
revision=cfg.vision_revision, revision=cfg.vision_revision,
torch_dtype=self.model_dtype, torch_dtype=self.model_dtype,
) )
if cfg.mode == "vision_language":
self.language_model = _load_language_model( self.language_model = _load_language_model(
repo_id=cfg.language_repo_id, repo_id=cfg.language_repo_id,
revision=cfg.language_revision, revision=cfg.language_revision,
@ -292,31 +305,35 @@ class Pistar06Model(nn.Module):
) )
vision_feature_size = _extract_vision_feature_size(self.vision_encoder) vision_feature_size = _extract_vision_feature_size(self.vision_encoder)
language_hidden_size = _extract_hidden_size(self.language_model)
self.image_projector = nn.Sequential( self.image_projector = nn.Sequential(
nn.Linear(vision_feature_size, cfg.fusion_hidden_dim), nn.Linear(vision_feature_size, cfg.fusion_hidden_dim),
nn.GELU(), nn.GELU(),
nn.Dropout(cfg.dropout), nn.Dropout(cfg.dropout),
) )
head_input_dim = cfg.fusion_hidden_dim
if cfg.mode == "vision_language":
language_hidden_size = _extract_hidden_size(self.language_model)
self.language_projector = nn.Sequential( self.language_projector = nn.Sequential(
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim), nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
nn.GELU(), nn.GELU(),
nn.Dropout(cfg.dropout), nn.Dropout(cfg.dropout),
) )
self.final_norm = nn.LayerNorm(cfg.fusion_hidden_dim * 2) head_input_dim *= 2
self.final_norm = nn.LayerNorm(head_input_dim)
self.value_head = nn.Sequential( self.value_head = nn.Sequential(
nn.Linear(cfg.fusion_hidden_dim * 2, cfg.fusion_hidden_dim), nn.Linear(head_input_dim, cfg.fusion_hidden_dim),
nn.GELU(), nn.GELU(),
nn.Dropout(cfg.dropout), nn.Dropout(cfg.dropout),
nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins), nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins),
) )
if cfg.use_gradient_checkpointing: if cfg.use_gradient_checkpointing:
if cfg.mode == "vision_language":
_maybe_enable_gradient_checkpointing(self.language_model) _maybe_enable_gradient_checkpointing(self.language_model)
_maybe_enable_gradient_checkpointing(self.vision_encoder) _maybe_enable_gradient_checkpointing(self.vision_encoder)
if cfg.freeze_language_model: if cfg.mode == "vision_language" and cfg.freeze_language_model:
_freeze_module(self.language_model) _freeze_module(self.language_model)
if cfg.freeze_vision_encoder: if cfg.freeze_vision_encoder:
_freeze_module(self.vision_encoder) _freeze_module(self.vision_encoder)
@ -358,6 +375,10 @@ class Pistar06Model(nn.Module):
if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras: if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras:
raise ValueError("Batch shape mismatch between images and image_attention_mask.") raise ValueError("Batch shape mismatch between images and image_attention_mask.")
if self.cfg.mode == "vision_only":
# Missing cameras must not influence detection of the input pixel range.
images = torch.where(image_attention_mask[:, :, None, None, None], images, 0)
if images.dtype == torch.uint8: if images.dtype == torch.uint8:
images = images.to(dtype=torch.float32) / 255.0 images = images.to(dtype=torch.float32) / 255.0
else: else:
@ -393,33 +414,26 @@ class Pistar06Model(nn.Module):
def forward( def forward(
self, self,
input_ids: Tensor, input_ids: Tensor | None = None,
attention_mask: Tensor, attention_mask: Tensor | None = None,
images: Tensor, images: Tensor | None = None,
image_attention_mask: Tensor, image_attention_mask: Tensor | None = None,
) -> Tensor: ) -> Tensor:
if input_ids.ndim != 2: if images is None or images.ndim != 5:
raise ValueError(f"'input_ids' must have shape [B, T], got {tuple(input_ids.shape)}.") raise ValueError("'images' must have shape [B, N, C, H, W].")
if attention_mask.ndim != 2: bsize, num_cameras = images.shape[:2]
raise ValueError(f"'attention_mask' must have shape [B, T], got {tuple(attention_mask.shape)}.") if num_cameras == 0:
if images.ndim != 5:
raise ValueError(f"'images' must have shape [B, N, C, H, W], got {tuple(images.shape)}.")
if image_attention_mask.ndim != 2:
raise ValueError(
f"'image_attention_mask' must have shape [B, N], got {tuple(image_attention_mask.shape)}."
)
bsize = input_ids.shape[0]
if attention_mask.shape[0] != bsize:
raise ValueError("Batch size mismatch between input_ids and attention_mask.")
if images.shape[0] != bsize or image_attention_mask.shape[0] != bsize:
raise ValueError("Batch size mismatch between language and image inputs.")
if images.shape[1] == 0:
raise ValueError("At least one camera is required for Pistar06Model.") raise ValueError("At least one camera is required for Pistar06Model.")
if image_attention_mask is None or tuple(image_attention_mask.shape) != (bsize, num_cameras):
raise ValueError("'image_attention_mask' must have shape [B, N].")
image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device) image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device)
if not torch.all(image_attention_mask.any(dim=1)): if not torch.all(image_attention_mask.any(dim=1)):
raise ValueError("Each sample must have at least one valid camera input.") raise ValueError("Each sample must have at least one valid camera input.")
if self.cfg.mode == "vision_language":
if input_ids is None or attention_mask is None:
raise ValueError("vision_language requires input_ids and attention_mask.")
if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape or input_ids.shape[0] != bsize:
raise ValueError("Language tensors must have matching shape [B, T].")
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device) language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
if not torch.all(language_mask.any(dim=1)): if not torch.all(language_mask.any(dim=1)):
raise ValueError("Each sample must have at least one valid language token.") raise ValueError("Each sample must have at least one valid language token.")
@ -433,6 +447,7 @@ class Pistar06Model(nn.Module):
with image_context: with image_context:
image_features = self._encode_images(flat_images) image_features = self._encode_images(flat_images)
if self.cfg.mode == "vision_language":
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext() language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
with language_context: with language_context:
language_features = self._encode_language( language_features = self._encode_language(
@ -441,7 +456,6 @@ class Pistar06Model(nn.Module):
feature_dtype = torch.float32 feature_dtype = torch.float32
image_features = image_features.to(dtype=feature_dtype) image_features = image_features.to(dtype=feature_dtype)
language_features = language_features.to(dtype=feature_dtype)
image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1) image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1)
camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype) camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype)
@ -451,10 +465,11 @@ class Pistar06Model(nn.Module):
image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0) image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0)
) )
image_pooled = image_tokens.sum(dim=1) / camera_denominator image_pooled = image_tokens.sum(dim=1) / camera_denominator
language_token = self.language_projector(language_features) if self.cfg.mode == "vision_only":
return self.value_head(self.final_norm(image_pooled))
joint_features = torch.cat([image_pooled, language_token], dim=-1) language_token = self.language_projector(language_features.to(dtype=feature_dtype))
return self.value_head(self.final_norm(joint_features)) return self.value_head(self.final_norm(torch.cat([image_pooled, language_token], dim=-1)))
class Pistar06Policy(PreTrainedPolicy): class Pistar06Policy(PreTrainedPolicy):
@ -482,7 +497,7 @@ class Pistar06Policy(PreTrainedPolicy):
prefixes: list[str] = [] prefixes: list[str] = []
if self.config.freeze_vision_encoder: if self.config.freeze_vision_encoder:
prefixes.append("model.vision_encoder.") prefixes.append("model.vision_encoder.")
if self.config.freeze_language_model: if self.config.mode == "vision_language" and self.config.freeze_language_model:
prefixes.append("model.language_model.") prefixes.append("model.language_model.")
return prefixes return prefixes
@ -626,8 +641,8 @@ class Pistar06Policy(PreTrainedPolicy):
raise RuntimeError("Pistar06Policy is a value model and does not support action selection.") raise RuntimeError("Pistar06Policy is a value model and does not support action selection.")
def predict_value(self, batch: dict[str, Tensor]) -> Tensor: def predict_value(self, batch: dict[str, Tensor]) -> Tensor:
input_ids = batch[OBS_LANGUAGE_TOKENS] input_ids = batch.get(OBS_LANGUAGE_TOKENS)
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK] attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
images = batch[PISTAR06_IMAGES_KEY] images = batch[PISTAR06_IMAGES_KEY]
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY] image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
@ -640,7 +655,7 @@ class Pistar06Policy(PreTrainedPolicy):
bin_centers = self.bin_centers.to(device=logits.device) bin_centers = self.bin_centers.to(device=logits.device)
return expected_value_from_logits(logits, bin_centers) return expected_value_from_logits(logits, bin_centers)
def build_training_raw_batch_hook(self, dataset, targets_cfg): def build_training_raw_batch_hook(self, dataset, targets_cfg, reward_cfg=None):
raw_frames = dataset.hf_dataset.with_format(None) raw_frames = dataset.hf_dataset.with_format(None)
frame_count = len(raw_frames) frame_count = len(raw_frames)
if frame_count == 0: if frame_count == 0:
@ -650,6 +665,42 @@ class Pistar06Policy(PreTrainedPolicy):
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64) frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64) absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
reward = reward_cfg or self.config.reward or ValueRewardConfig()
reward.validate()
if reward.mode != "legacy":
saved_scales = self.config.reward_scales if self.config.reward == reward else None
value_targets, scales = build_successful_value_targets(
dataset,
reward,
targets_cfg.success_field,
task_scales=saved_scales or None,
)
self.config.reward = reward
self.config.reward_scales = scales
else:
value_targets = self._legacy_training_targets(
dataset, targets_cfg, episode_indices, frame_indices
)
self.config.reward = reward
self.config.reward_scales = {}
max_index = int(np.max(absolute_indices))
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
target_key = targets_cfg.target_field
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
del step
batch_indices = batch.get("index")
if batch_indices is None:
raise KeyError("Missing 'index' in batch while building value targets.")
batch_indices = torch.as_tensor(batch_indices).detach().cpu().numpy().astype(np.int64).reshape(-1)
batch[target_key] = torch.from_numpy(value_target_lookup[batch_indices])
return batch
return value_target_hook
def _legacy_training_targets(self, dataset, targets_cfg, episode_indices, frame_indices):
episodes_ds = dataset.meta.episodes.with_format(None) episodes_ds = dataset.meta.episodes.with_format(None)
episodes = episodes_ds[:] episodes = episodes_ds[:]
n_episodes = len(episodes_ds) n_episodes = len(episodes_ds)
@ -692,26 +743,7 @@ class Pistar06Policy(PreTrainedPolicy):
clip_max=self.config.bin_max, clip_max=self.config.bin_max,
) )
max_index = int(np.max(absolute_indices)) return value_targets
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
target_key = targets_cfg.target_field
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
del step
batch_indices = batch.get("index")
if batch_indices is None:
raise KeyError("Missing 'index' in batch while building value targets.")
if not isinstance(batch_indices, torch.Tensor):
batch_indices = torch.as_tensor(batch_indices)
batch_indices_np = batch_indices.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
target_values = torch.from_numpy(value_target_lookup[batch_indices_np]).to(dtype=torch.float32)
batch[target_key] = target_values
return batch
return value_target_hook
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]: def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
if self.config.target_key not in batch: if self.config.target_key not in batch:
@ -720,8 +752,8 @@ class Pistar06Policy(PreTrainedPolicy):
"Make sure lerobot-value-train target hook is enabled." "Make sure lerobot-value-train target hook is enabled."
) )
input_ids = batch[OBS_LANGUAGE_TOKENS] input_ids = batch.get(OBS_LANGUAGE_TOKENS)
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK] attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
images = batch[PISTAR06_IMAGES_KEY] images = batch[PISTAR06_IMAGES_KEY]
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY] image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]

查看文件

@ -218,8 +218,33 @@ def make_pistar06_pre_post_processors(
if not camera_features: if not camera_features:
camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)] camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)]
input_steps: list[ProcessorStep] = [ input_steps: list[ProcessorStep] = [RenameObservationsProcessorStep(rename_map={})]
RenameObservationsProcessorStep(rename_map={}), if config.mode == "vision_language":
input_steps.extend(_language_steps(config, dataset_stats))
input_steps.extend(
[
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
DeviceProcessorStep(device=config.device),
]
)
output_steps: list[ProcessorStep] = [DeviceProcessorStep(device="cpu")]
return (
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
steps=input_steps,
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
),
PolicyProcessorPipeline[PolicyAction, PolicyAction](
steps=output_steps,
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
to_transition=policy_action_to_transition,
to_output=transition_to_policy_action,
),
)
def _language_steps(config: Pistar06Config, dataset_stats) -> list[ProcessorStep]:
return [
NormalizerProcessorStep( NormalizerProcessorStep(
features={**(config.input_features or {}), **(config.output_features or {})}, features={**(config.input_features or {}), **(config.output_features or {})},
norm_map=config.normalization_mapping, norm_map=config.normalization_mapping,
@ -241,23 +266,4 @@ def make_pistar06_pre_post_processors(
padding="max_length", padding="max_length",
truncation=True, truncation=True,
), ),
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
DeviceProcessorStep(device=config.device),
] ]
output_steps: list[ProcessorStep] = [
DeviceProcessorStep(device="cpu"),
]
return (
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
steps=input_steps,
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
),
PolicyProcessorPipeline[PolicyAction, PolicyAction](
steps=output_steps,
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
to_transition=policy_action_to_transition,
to_output=transition_to_policy_action,
),
)

查看文件

@ -0,0 +1,110 @@
"""Success-only Monte Carlo targets with task scales fitted on training episodes."""
import math
import numpy as np
from lerobot.configs.value_reward import ValueRewardConfig
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, normalize_episode_success_label
def build_successful_value_targets(
dataset,
reward: ValueRewardConfig,
success_field: str,
*,
task_scales: dict[str, float] | None = None,
) -> tuple[np.ndarray, dict[str, float]]:
"""Return targets in raw frame order; reject partial, failed or unlabelled episodes.
Only episodes actually present in hf_dataset contribute to the fitted scale.
Task names, rather than dataset-local task indices, identify checkpoint scales.
Supplying scales reuses them without refitting or clipping out-of-range returns.
"""
reward.validate()
if reward.mode == "legacy":
raise ValueError("Use the legacy target builder for reward.mode=legacy.")
frames = dataset.hf_dataset.with_format(None)
episode_ids = np.asarray(frames["episode_index"], dtype=np.int64).reshape(-1)
frame_ids = np.asarray(frames["frame_index"], dtype=np.int64).reshape(-1)
if len(episode_ids) == 0:
raise ValueError("Dataset has no frames.")
episodes = dataset.meta.episodes.with_format(None)
if success_field not in episodes.column_names:
raise ValueError(f"reward.mode={reward.mode} requires explicit '{success_field}=success' labels.")
meta = episodes[:]
active_ids = set(episode_ids.tolist())
metadata = {}
for i, episode_id in enumerate(meta["episode_index"]):
episode_id = int(episode_id)
if episode_id not in active_ids:
continue
if episode_id in metadata:
raise ValueError(f"Duplicate episode metadata for episode {episode_id}.")
label = normalize_episode_success_label(meta[success_field][i])
if label != EPISODE_SUCCESS:
raise ValueError(
f"reward.mode={reward.mode} requires only explicitly successful episodes; "
f"episode {episode_id} has label {label!r}."
)
tasks = meta["tasks"][i]
if not isinstance(tasks, str):
if len(tasks) != 1:
raise ValueError(f"Episode {episode_id} must have exactly one task for reward normalization.")
tasks = tasks[0]
metadata[episode_id] = (int(meta["length"][i]), str(tasks))
if active_ids != set(metadata):
raise ValueError("Missing episode metadata for reward construction.")
interventions = None
if reward.mode == "time_takeover":
if reward.intervention_field not in frames.column_names:
raise ValueError(f"Missing intervention field '{reward.intervention_field}' for time_takeover.")
interventions = np.asarray(frames[reward.intervention_field]).reshape(-1)
if interventions.shape != episode_ids.shape or not np.isin(interventions, [0, 1]).all():
raise ValueError("Intervention flags must contain one boolean or binary 0/1 value per frame.")
interventions = interventions.astype(bool)
returns = np.empty(len(episode_ids), dtype=np.float64)
fitted_scales: dict[str, float] = {}
positions_by_task: dict[str, list[np.ndarray]] = {}
for episode_id in sorted(active_ids):
positions = np.flatnonzero(episode_ids == episode_id)
length, task = metadata[episode_id]
if (
length <= 0
or not np.array_equal(frame_ids[positions], np.arange(length))
or not np.all(np.diff(positions) == 1)
):
raise ValueError(
f"Episode {episode_id} must contain every frame exactly once, contiguously from 0 to length-1."
)
rewards = np.full(length, -1.0, dtype=np.float64)
rewards[-1] = 0.0
if interventions is not None and length > 2:
human = interventions[positions]
# Penalize the preceding autonomous frame. Entry into the terminal
# observation and initial human control are not takeover events.
takeovers = (~human[:-2]) & human[1:-1]
rewards[:-2] -= reward.takeover_cost * takeovers
episode_returns = np.cumsum(rewards[::-1])[::-1]
returns[positions] = episode_returns
# A one-frame successful episode has zero cost; use scale 1 to avoid 0/0.
fitted_scales[task] = max(fitted_scales.get(task, 1.0), float(-episode_returns[0]))
positions_by_task.setdefault(task, []).append(positions)
scales = dict(fitted_scales if task_scales is None else task_scales)
targets = np.empty(len(returns), dtype=np.float32)
for task, chunks in positions_by_task.items():
scale = scales.get(task)
if scale is None or not math.isfinite(scale) or scale <= 0:
raise ValueError(f"Missing or invalid saved reward scale for task {task!r}.")
positions = np.concatenate(chunks)
normalized = returns[positions] / scale
if np.any(normalized < -1.0) or np.any(normalized > 0.0):
raise ValueError(
f"Returns for task {task!r} exceed the saved reward scale; refusing to clip or refit."
)
targets[positions] = normalized
return targets, scales