镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
feat: add vision-only values, takeover rewards and masked action loss
这个提交包含在:
@ -9,6 +9,7 @@ from typing import Any
|
||||
|
||||
import draccus
|
||||
|
||||
from lerobot.configs.value_reward import ValueRewardConfig
|
||||
from lerobot.utils.recording_annotations import normalize_episode_success_label
|
||||
|
||||
|
||||
@ -118,6 +119,8 @@ class ValueInferencePipelineConfig:
|
||||
)
|
||||
runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig)
|
||||
acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig)
|
||||
# None inherits the trained checkpoint; old checkpoints resolve to legacy.
|
||||
reward: ValueRewardConfig | None = None
|
||||
viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig)
|
||||
|
||||
output_dir: Path | None = None
|
||||
@ -131,6 +134,8 @@ class ValueInferencePipelineConfig:
|
||||
self.inference.validate()
|
||||
self.runtime.validate()
|
||||
self.acp.validate()
|
||||
if self.reward is not None:
|
||||
self.reward.validate()
|
||||
self.viz.validate()
|
||||
|
||||
if not self.job_name:
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
"""Reward settings shared by value training and inference."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValueRewardConfig:
|
||||
mode: str = "legacy"
|
||||
takeover_cost: float = 150.0
|
||||
intervention_field: str = "complementary_info.is_intervention"
|
||||
|
||||
def validate(self) -> None:
|
||||
if self.mode not in {"legacy", "time", "time_takeover"}:
|
||||
raise ValueError("'reward.mode' must be legacy, time or time_takeover.")
|
||||
if not math.isfinite(self.takeover_cost) or self.takeover_cost < 0:
|
||||
raise ValueError("'reward.takeover_cost' must be finite and non-negative.")
|
||||
if self.mode == "time_takeover" and not self.intervention_field:
|
||||
raise ValueError("'reward.intervention_field' must be non-empty for time_takeover.")
|
||||
@ -14,6 +14,7 @@ from huggingface_hub.errors import HfHubHTTPError
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.value_reward import ValueRewardConfig
|
||||
from lerobot.optim import OptimizerConfig
|
||||
from lerobot.optim.schedulers import LRSchedulerConfig
|
||||
from lerobot.utils.hub import HubMixin
|
||||
@ -72,6 +73,7 @@ class ValueTrainPipelineConfig(HubMixin):
|
||||
optimizer: OptimizerConfig | None = None
|
||||
scheduler: LRSchedulerConfig | None = None
|
||||
|
||||
reward: ValueRewardConfig | None = None
|
||||
targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig)
|
||||
wandb: WandBConfig = field(default_factory=WandBConfig)
|
||||
peft: PeftConfig | None = None
|
||||
@ -113,6 +115,8 @@ class ValueTrainPipelineConfig(HubMixin):
|
||||
)
|
||||
|
||||
self.targets.validate()
|
||||
if self.reward is not None:
|
||||
self.reward.validate()
|
||||
|
||||
if hasattr(self.value, "target_key"):
|
||||
self.value.target_key = self.targets.target_field
|
||||
|
||||
@ -71,6 +71,9 @@ class PI05Config(PreTrainedConfig):
|
||||
}
|
||||
)
|
||||
|
||||
# Exclude repeated episode-tail actions from flow-matching supervision.
|
||||
mask_action_padding_loss: bool = False
|
||||
|
||||
# Training settings
|
||||
gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization
|
||||
compile_model: bool = False # Whether to use torch.compile for model optimization
|
||||
|
||||
@ -1250,6 +1250,16 @@ class PI05Policy(PreTrainedPolicy):
|
||||
- "mean": Return scalar mean loss (default, backward compatible)
|
||||
- "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting
|
||||
"""
|
||||
action_is_pad = None
|
||||
if self.config.mask_action_padding_loss:
|
||||
action_is_pad = batch.get("action_is_pad")
|
||||
if action_is_pad is None:
|
||||
raise ValueError(
|
||||
"mask_action_padding_loss=True requires action_is_pad in the training batch."
|
||||
)
|
||||
if not isinstance(action_is_pad, Tensor) or action_is_pad.dtype != torch.bool:
|
||||
raise TypeError("action_is_pad must be a boolean tensor (True marks padding).")
|
||||
|
||||
# Prepare inputs
|
||||
images, img_masks = self._preprocess_images(batch)
|
||||
tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
|
||||
@ -1263,6 +1273,9 @@ class PI05Policy(PreTrainedPolicy):
|
||||
original_action_dim = self.config.output_features[ACTION].shape[0]
|
||||
losses = losses[:, :, :original_action_dim]
|
||||
|
||||
if action_is_pad is not None:
|
||||
return self._reduce_masked_action_losses(losses, action_is_pad, reduction)
|
||||
|
||||
loss_dict = {
|
||||
"loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(),
|
||||
}
|
||||
@ -1278,6 +1291,27 @@ class PI05Policy(PreTrainedPolicy):
|
||||
loss_dict["loss"] = loss.item()
|
||||
return loss, loss_dict
|
||||
|
||||
@staticmethod
|
||||
def _reduce_masked_action_losses(losses: Tensor, action_is_pad: Tensor, reduction: str):
|
||||
if tuple(action_is_pad.shape) != tuple(losses.shape[:2]):
|
||||
raise ValueError("action_is_pad must have shape [batch_size, action_horizon].")
|
||||
valid = ~action_is_pad.to(device=losses.device)
|
||||
valid_steps = valid.sum(dim=1)
|
||||
if torch.any(valid_steps == 0):
|
||||
raise ValueError("Every action chunk must contain at least one non-padding timestep.")
|
||||
# Mask with where: even NaN/Inf in an unused loss slot must not enter the sum.
|
||||
masked = torch.where(valid.unsqueeze(-1), losses, 0.0)
|
||||
if reduction == "none":
|
||||
loss = masked.sum(dim=(1, 2)) / (valid_steps * losses.shape[-1])
|
||||
elif reduction == "mean":
|
||||
loss = masked.sum() / (valid_steps.sum() * losses.shape[-1])
|
||||
else:
|
||||
raise ValueError("reduction must be 'mean' or 'none'.")
|
||||
return loss, {
|
||||
"loss": loss.detach().mean().item(),
|
||||
"loss_per_dim": (masked.sum(dim=(0, 1)) / valid_steps.sum()).detach().cpu().tolist(),
|
||||
}
|
||||
|
||||
def _get_default_peft_targets(self) -> dict[str, any]:
|
||||
"""Return default PEFT target modules for PI0.5 fine-tuning."""
|
||||
common_projections = (
|
||||
|
||||
@ -32,6 +32,7 @@ from tqdm.auto import tqdm
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.value import ValueInferencePipelineConfig
|
||||
from lerobot.configs.value_reward import ValueRewardConfig
|
||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||
from lerobot.datasets.utils import load_info, write_info
|
||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||
@ -52,6 +53,7 @@ from lerobot.values.pistar06.modeling_pistar06 import (
|
||||
EpisodeTargetInfo,
|
||||
compute_normalized_value_targets,
|
||||
)
|
||||
from lerobot.values.rewards import build_successful_value_targets
|
||||
|
||||
|
||||
class ContiguousDistributedEvalSampler(Sampler[int]):
|
||||
@ -522,6 +524,23 @@ def run_value_inference_pipeline(
|
||||
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
|
||||
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
||||
|
||||
reward = value_cfg.reward or ValueRewardConfig()
|
||||
if cfg.reward is not None and cfg.reward != reward:
|
||||
raise ValueError("Inference reward settings must match the trained checkpoint.")
|
||||
cfg.reward = reward
|
||||
success_targets = None
|
||||
if reward.mode != "legacy":
|
||||
if not value_cfg.reward_scales:
|
||||
raise ValueError(
|
||||
"Checkpoint is missing training reward scales; refusing to fit on inference data."
|
||||
)
|
||||
success_targets, _ = build_successful_value_targets(
|
||||
dataset,
|
||||
reward,
|
||||
cfg.dataset.success_field,
|
||||
task_scales=value_cfg.reward_scales,
|
||||
)
|
||||
|
||||
if cfg.acp.intervention_field in raw_frames.column_names:
|
||||
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
|
||||
else:
|
||||
@ -595,6 +614,8 @@ def run_value_inference_pipeline(
|
||||
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
|
||||
|
||||
predicted_values = prediction_lookup[absolute_indices]
|
||||
if success_targets is not None:
|
||||
predicted_values[success_targets == 0.0] = 0.0
|
||||
logging.info(
|
||||
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
|
||||
float(np.min(predicted_values)),
|
||||
@ -614,21 +635,23 @@ def run_value_inference_pipeline(
|
||||
thresholds: dict[int, float] | None = None
|
||||
|
||||
if cfg.acp.enable:
|
||||
episode_info, task_max_lengths = _build_episode_info(
|
||||
dataset=dataset,
|
||||
success_field=cfg.dataset.success_field,
|
||||
default_success=cfg.dataset.default_success,
|
||||
)
|
||||
|
||||
value_targets = compute_normalized_value_targets(
|
||||
episode_indices=episode_indices,
|
||||
frame_indices=frame_indices,
|
||||
episode_info=episode_info,
|
||||
task_max_lengths=task_max_lengths,
|
||||
c_fail_coef=cfg.acp.c_fail_coef,
|
||||
clip_min=value_cfg.bin_min,
|
||||
clip_max=value_cfg.bin_max,
|
||||
)
|
||||
if success_targets is None:
|
||||
episode_info, task_max_lengths = _build_episode_info(
|
||||
dataset=dataset,
|
||||
success_field=cfg.dataset.success_field,
|
||||
default_success=cfg.dataset.default_success,
|
||||
)
|
||||
value_targets = compute_normalized_value_targets(
|
||||
episode_indices=episode_indices,
|
||||
frame_indices=frame_indices,
|
||||
episode_info=episode_info,
|
||||
task_max_lengths=task_max_lengths,
|
||||
c_fail_coef=cfg.acp.c_fail_coef,
|
||||
clip_min=value_cfg.bin_min,
|
||||
clip_max=value_cfg.bin_max,
|
||||
)
|
||||
else:
|
||||
value_targets = success_targets
|
||||
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
|
||||
advantages = _compute_n_step_advantages(
|
||||
rewards=rewards,
|
||||
|
||||
@ -131,7 +131,9 @@ def value_train(
|
||||
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
|
||||
dataset=dataset,
|
||||
targets_cfg=cfg.targets,
|
||||
reward_cfg=cfg.reward,
|
||||
)
|
||||
cfg.reward = value_model.config.reward
|
||||
if is_main_process:
|
||||
logging.info("Using value model raw-batch hook for target construction.")
|
||||
elif is_main_process:
|
||||
@ -163,6 +165,9 @@ def value_train(
|
||||
"norm_map": cfg.value.normalization_mapping,
|
||||
},
|
||||
}
|
||||
if cfg.value.mode == "vision_only":
|
||||
processor_kwargs["preprocessor_overrides"].pop("normalizer_processor", None)
|
||||
postprocessor_kwargs["postprocessor_overrides"].pop("unnormalizer_processor", None)
|
||||
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.value,
|
||||
|
||||
@ -4,6 +4,7 @@ from dataclasses import dataclass, field
|
||||
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.types import NormalizationMode
|
||||
from lerobot.configs.value_reward import ValueRewardConfig
|
||||
from lerobot.optim.optimizers import AdamWConfig
|
||||
from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
|
||||
from lerobot.utils.constants import OBS_STATE
|
||||
@ -12,7 +13,12 @@ from lerobot.utils.constants import OBS_STATE
|
||||
@PreTrainedConfig.register_subclass("pistar06")
|
||||
@dataclass
|
||||
class Pistar06Config(PreTrainedConfig):
|
||||
"""Value model config using the Pistar06 stack (SigLIP + Gemma)."""
|
||||
"""SigLIP value model with optional Gemma language conditioning."""
|
||||
|
||||
# Old checkpoints without this field retain the language-conditioned path.
|
||||
mode: str = "vision_language"
|
||||
reward: ValueRewardConfig | None = None
|
||||
reward_scales: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
# Backbone components
|
||||
vision_repo_id: str = "google/siglip-so400m-patch14-384"
|
||||
@ -70,9 +76,13 @@ class Pistar06Config(PreTrainedConfig):
|
||||
def __post_init__(self) -> None:
|
||||
super().__post_init__()
|
||||
|
||||
if self.mode not in {"vision_language", "vision_only"}:
|
||||
raise ValueError("'value.mode' must be vision_language or vision_only.")
|
||||
if self.reward is not None:
|
||||
self.reward.validate()
|
||||
if not self.vision_repo_id:
|
||||
raise ValueError("'value.vision_repo_id' must be non-empty.")
|
||||
if not self.language_repo_id:
|
||||
if self.mode == "vision_language" and not self.language_repo_id:
|
||||
raise ValueError("'value.language_repo_id' must be non-empty.")
|
||||
if not self.task_field:
|
||||
raise ValueError("'value.task_field' must be non-empty.")
|
||||
|
||||
@ -19,20 +19,29 @@ from huggingface_hub.errors import HfHubHTTPError
|
||||
from safetensors.torch import save_file
|
||||
from torch import Tensor, nn
|
||||
|
||||
from lerobot.configs.value_reward import ValueRewardConfig
|
||||
from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy
|
||||
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
|
||||
from lerobot.utils.import_utils import _transformers_available
|
||||
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
|
||||
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
|
||||
from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY
|
||||
from lerobot.values.rewards import build_successful_value_targets
|
||||
|
||||
if TYPE_CHECKING or _transformers_available:
|
||||
from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoModelForCausalLM
|
||||
from transformers import (
|
||||
AutoConfig,
|
||||
AutoImageProcessor,
|
||||
AutoModel,
|
||||
AutoModelForCausalLM,
|
||||
SiglipVisionModel,
|
||||
)
|
||||
else:
|
||||
AutoConfig = None
|
||||
AutoImageProcessor = None
|
||||
AutoModel = None
|
||||
AutoModelForCausalLM = None
|
||||
SiglipVisionModel = None
|
||||
|
||||
|
||||
PISTAR06_SAVE_INFO = "pistar06_save_info.json"
|
||||
@ -261,16 +270,20 @@ class Pistar06Model(nn.Module):
|
||||
self.cfg = cfg
|
||||
self.model_dtype = _resolve_load_dtype(cfg.dtype)
|
||||
|
||||
self.vision_encoder = AutoModel.from_pretrained(
|
||||
# SiglipVisionModel loads only the visual weights; AutoModel on a full
|
||||
# SigLIP repository would also create an unused text encoder under DDP.
|
||||
vision_loader = SiglipVisionModel if cfg.mode == "vision_only" else AutoModel
|
||||
self.vision_encoder = vision_loader.from_pretrained(
|
||||
cfg.vision_repo_id,
|
||||
revision=cfg.vision_revision,
|
||||
torch_dtype=self.model_dtype,
|
||||
)
|
||||
self.language_model = _load_language_model(
|
||||
repo_id=cfg.language_repo_id,
|
||||
revision=cfg.language_revision,
|
||||
dtype=self.model_dtype,
|
||||
)
|
||||
if cfg.mode == "vision_language":
|
||||
self.language_model = _load_language_model(
|
||||
repo_id=cfg.language_repo_id,
|
||||
revision=cfg.language_revision,
|
||||
dtype=self.model_dtype,
|
||||
)
|
||||
|
||||
image_processor = AutoImageProcessor.from_pretrained(
|
||||
cfg.vision_repo_id,
|
||||
@ -292,31 +305,35 @@ class Pistar06Model(nn.Module):
|
||||
)
|
||||
|
||||
vision_feature_size = _extract_vision_feature_size(self.vision_encoder)
|
||||
language_hidden_size = _extract_hidden_size(self.language_model)
|
||||
|
||||
self.image_projector = nn.Sequential(
|
||||
nn.Linear(vision_feature_size, cfg.fusion_hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Dropout(cfg.dropout),
|
||||
)
|
||||
self.language_projector = nn.Sequential(
|
||||
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Dropout(cfg.dropout),
|
||||
)
|
||||
self.final_norm = nn.LayerNorm(cfg.fusion_hidden_dim * 2)
|
||||
head_input_dim = cfg.fusion_hidden_dim
|
||||
if cfg.mode == "vision_language":
|
||||
language_hidden_size = _extract_hidden_size(self.language_model)
|
||||
self.language_projector = nn.Sequential(
|
||||
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Dropout(cfg.dropout),
|
||||
)
|
||||
head_input_dim *= 2
|
||||
self.final_norm = nn.LayerNorm(head_input_dim)
|
||||
self.value_head = nn.Sequential(
|
||||
nn.Linear(cfg.fusion_hidden_dim * 2, cfg.fusion_hidden_dim),
|
||||
nn.Linear(head_input_dim, cfg.fusion_hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Dropout(cfg.dropout),
|
||||
nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins),
|
||||
)
|
||||
|
||||
if cfg.use_gradient_checkpointing:
|
||||
_maybe_enable_gradient_checkpointing(self.language_model)
|
||||
if cfg.mode == "vision_language":
|
||||
_maybe_enable_gradient_checkpointing(self.language_model)
|
||||
_maybe_enable_gradient_checkpointing(self.vision_encoder)
|
||||
|
||||
if cfg.freeze_language_model:
|
||||
if cfg.mode == "vision_language" and cfg.freeze_language_model:
|
||||
_freeze_module(self.language_model)
|
||||
if cfg.freeze_vision_encoder:
|
||||
_freeze_module(self.vision_encoder)
|
||||
@ -358,6 +375,10 @@ class Pistar06Model(nn.Module):
|
||||
if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras:
|
||||
raise ValueError("Batch shape mismatch between images and image_attention_mask.")
|
||||
|
||||
if self.cfg.mode == "vision_only":
|
||||
# Missing cameras must not influence detection of the input pixel range.
|
||||
images = torch.where(image_attention_mask[:, :, None, None, None], images, 0)
|
||||
|
||||
if images.dtype == torch.uint8:
|
||||
images = images.to(dtype=torch.float32) / 255.0
|
||||
else:
|
||||
@ -393,36 +414,29 @@ class Pistar06Model(nn.Module):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
attention_mask: Tensor,
|
||||
images: Tensor,
|
||||
image_attention_mask: Tensor,
|
||||
input_ids: Tensor | None = None,
|
||||
attention_mask: Tensor | None = None,
|
||||
images: Tensor | None = None,
|
||||
image_attention_mask: Tensor | None = None,
|
||||
) -> Tensor:
|
||||
if input_ids.ndim != 2:
|
||||
raise ValueError(f"'input_ids' must have shape [B, T], got {tuple(input_ids.shape)}.")
|
||||
if attention_mask.ndim != 2:
|
||||
raise ValueError(f"'attention_mask' must have shape [B, T], got {tuple(attention_mask.shape)}.")
|
||||
if images.ndim != 5:
|
||||
raise ValueError(f"'images' must have shape [B, N, C, H, W], got {tuple(images.shape)}.")
|
||||
if image_attention_mask.ndim != 2:
|
||||
raise ValueError(
|
||||
f"'image_attention_mask' must have shape [B, N], got {tuple(image_attention_mask.shape)}."
|
||||
)
|
||||
|
||||
bsize = input_ids.shape[0]
|
||||
if attention_mask.shape[0] != bsize:
|
||||
raise ValueError("Batch size mismatch between input_ids and attention_mask.")
|
||||
if images.shape[0] != bsize or image_attention_mask.shape[0] != bsize:
|
||||
raise ValueError("Batch size mismatch between language and image inputs.")
|
||||
if images.shape[1] == 0:
|
||||
if images is None or images.ndim != 5:
|
||||
raise ValueError("'images' must have shape [B, N, C, H, W].")
|
||||
bsize, num_cameras = images.shape[:2]
|
||||
if num_cameras == 0:
|
||||
raise ValueError("At least one camera is required for Pistar06Model.")
|
||||
|
||||
if image_attention_mask is None or tuple(image_attention_mask.shape) != (bsize, num_cameras):
|
||||
raise ValueError("'image_attention_mask' must have shape [B, N].")
|
||||
image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device)
|
||||
if not torch.all(image_attention_mask.any(dim=1)):
|
||||
raise ValueError("Each sample must have at least one valid camera input.")
|
||||
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
|
||||
if not torch.all(language_mask.any(dim=1)):
|
||||
raise ValueError("Each sample must have at least one valid language token.")
|
||||
if self.cfg.mode == "vision_language":
|
||||
if input_ids is None or attention_mask is None:
|
||||
raise ValueError("vision_language requires input_ids and attention_mask.")
|
||||
if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape or input_ids.shape[0] != bsize:
|
||||
raise ValueError("Language tensors must have matching shape [B, T].")
|
||||
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
|
||||
if not torch.all(language_mask.any(dim=1)):
|
||||
raise ValueError("Each sample must have at least one valid language token.")
|
||||
|
||||
processed_images = self._preprocess_images(images, image_attention_mask)
|
||||
num_cameras = processed_images.shape[1]
|
||||
@ -433,15 +447,15 @@ class Pistar06Model(nn.Module):
|
||||
with image_context:
|
||||
image_features = self._encode_images(flat_images)
|
||||
|
||||
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
|
||||
with language_context:
|
||||
language_features = self._encode_language(
|
||||
input_ids=input_ids, attention_mask=language_mask.long()
|
||||
)
|
||||
if self.cfg.mode == "vision_language":
|
||||
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
|
||||
with language_context:
|
||||
language_features = self._encode_language(
|
||||
input_ids=input_ids, attention_mask=language_mask.long()
|
||||
)
|
||||
|
||||
feature_dtype = torch.float32
|
||||
image_features = image_features.to(dtype=feature_dtype)
|
||||
language_features = language_features.to(dtype=feature_dtype)
|
||||
|
||||
image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1)
|
||||
camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype)
|
||||
@ -451,10 +465,11 @@ class Pistar06Model(nn.Module):
|
||||
image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0)
|
||||
)
|
||||
image_pooled = image_tokens.sum(dim=1) / camera_denominator
|
||||
language_token = self.language_projector(language_features)
|
||||
if self.cfg.mode == "vision_only":
|
||||
return self.value_head(self.final_norm(image_pooled))
|
||||
|
||||
joint_features = torch.cat([image_pooled, language_token], dim=-1)
|
||||
return self.value_head(self.final_norm(joint_features))
|
||||
language_token = self.language_projector(language_features.to(dtype=feature_dtype))
|
||||
return self.value_head(self.final_norm(torch.cat([image_pooled, language_token], dim=-1)))
|
||||
|
||||
|
||||
class Pistar06Policy(PreTrainedPolicy):
|
||||
@ -482,7 +497,7 @@ class Pistar06Policy(PreTrainedPolicy):
|
||||
prefixes: list[str] = []
|
||||
if self.config.freeze_vision_encoder:
|
||||
prefixes.append("model.vision_encoder.")
|
||||
if self.config.freeze_language_model:
|
||||
if self.config.mode == "vision_language" and self.config.freeze_language_model:
|
||||
prefixes.append("model.language_model.")
|
||||
return prefixes
|
||||
|
||||
@ -626,8 +641,8 @@ class Pistar06Policy(PreTrainedPolicy):
|
||||
raise RuntimeError("Pistar06Policy is a value model and does not support action selection.")
|
||||
|
||||
def predict_value(self, batch: dict[str, Tensor]) -> Tensor:
|
||||
input_ids = batch[OBS_LANGUAGE_TOKENS]
|
||||
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
|
||||
input_ids = batch.get(OBS_LANGUAGE_TOKENS)
|
||||
attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
|
||||
images = batch[PISTAR06_IMAGES_KEY]
|
||||
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
|
||||
|
||||
@ -640,7 +655,7 @@ class Pistar06Policy(PreTrainedPolicy):
|
||||
bin_centers = self.bin_centers.to(device=logits.device)
|
||||
return expected_value_from_logits(logits, bin_centers)
|
||||
|
||||
def build_training_raw_batch_hook(self, dataset, targets_cfg):
|
||||
def build_training_raw_batch_hook(self, dataset, targets_cfg, reward_cfg=None):
|
||||
raw_frames = dataset.hf_dataset.with_format(None)
|
||||
frame_count = len(raw_frames)
|
||||
if frame_count == 0:
|
||||
@ -650,6 +665,42 @@ class Pistar06Policy(PreTrainedPolicy):
|
||||
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
||||
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
|
||||
|
||||
reward = reward_cfg or self.config.reward or ValueRewardConfig()
|
||||
reward.validate()
|
||||
if reward.mode != "legacy":
|
||||
saved_scales = self.config.reward_scales if self.config.reward == reward else None
|
||||
value_targets, scales = build_successful_value_targets(
|
||||
dataset,
|
||||
reward,
|
||||
targets_cfg.success_field,
|
||||
task_scales=saved_scales or None,
|
||||
)
|
||||
self.config.reward = reward
|
||||
self.config.reward_scales = scales
|
||||
else:
|
||||
value_targets = self._legacy_training_targets(
|
||||
dataset, targets_cfg, episode_indices, frame_indices
|
||||
)
|
||||
self.config.reward = reward
|
||||
self.config.reward_scales = {}
|
||||
|
||||
max_index = int(np.max(absolute_indices))
|
||||
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
|
||||
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
|
||||
target_key = targets_cfg.target_field
|
||||
|
||||
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
|
||||
del step
|
||||
batch_indices = batch.get("index")
|
||||
if batch_indices is None:
|
||||
raise KeyError("Missing 'index' in batch while building value targets.")
|
||||
batch_indices = torch.as_tensor(batch_indices).detach().cpu().numpy().astype(np.int64).reshape(-1)
|
||||
batch[target_key] = torch.from_numpy(value_target_lookup[batch_indices])
|
||||
return batch
|
||||
|
||||
return value_target_hook
|
||||
|
||||
def _legacy_training_targets(self, dataset, targets_cfg, episode_indices, frame_indices):
|
||||
episodes_ds = dataset.meta.episodes.with_format(None)
|
||||
episodes = episodes_ds[:]
|
||||
n_episodes = len(episodes_ds)
|
||||
@ -692,26 +743,7 @@ class Pistar06Policy(PreTrainedPolicy):
|
||||
clip_max=self.config.bin_max,
|
||||
)
|
||||
|
||||
max_index = int(np.max(absolute_indices))
|
||||
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
|
||||
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
|
||||
|
||||
target_key = targets_cfg.target_field
|
||||
|
||||
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
|
||||
del step
|
||||
batch_indices = batch.get("index")
|
||||
if batch_indices is None:
|
||||
raise KeyError("Missing 'index' in batch while building value targets.")
|
||||
if not isinstance(batch_indices, torch.Tensor):
|
||||
batch_indices = torch.as_tensor(batch_indices)
|
||||
|
||||
batch_indices_np = batch_indices.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
|
||||
target_values = torch.from_numpy(value_target_lookup[batch_indices_np]).to(dtype=torch.float32)
|
||||
batch[target_key] = target_values
|
||||
return batch
|
||||
|
||||
return value_target_hook
|
||||
return value_targets
|
||||
|
||||
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
|
||||
if self.config.target_key not in batch:
|
||||
@ -720,8 +752,8 @@ class Pistar06Policy(PreTrainedPolicy):
|
||||
"Make sure lerobot-value-train target hook is enabled."
|
||||
)
|
||||
|
||||
input_ids = batch[OBS_LANGUAGE_TOKENS]
|
||||
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
|
||||
input_ids = batch.get(OBS_LANGUAGE_TOKENS)
|
||||
attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
|
||||
images = batch[PISTAR06_IMAGES_KEY]
|
||||
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
|
||||
|
||||
|
||||
@ -218,8 +218,33 @@ def make_pistar06_pre_post_processors(
|
||||
if not camera_features:
|
||||
camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)]
|
||||
|
||||
input_steps: list[ProcessorStep] = [
|
||||
RenameObservationsProcessorStep(rename_map={}),
|
||||
input_steps: list[ProcessorStep] = [RenameObservationsProcessorStep(rename_map={})]
|
||||
if config.mode == "vision_language":
|
||||
input_steps.extend(_language_steps(config, dataset_stats))
|
||||
input_steps.extend(
|
||||
[
|
||||
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
|
||||
DeviceProcessorStep(device=config.device),
|
||||
]
|
||||
)
|
||||
|
||||
output_steps: list[ProcessorStep] = [DeviceProcessorStep(device="cpu")]
|
||||
return (
|
||||
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
|
||||
steps=input_steps,
|
||||
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
|
||||
),
|
||||
PolicyProcessorPipeline[PolicyAction, PolicyAction](
|
||||
steps=output_steps,
|
||||
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
|
||||
to_transition=policy_action_to_transition,
|
||||
to_output=transition_to_policy_action,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _language_steps(config: Pistar06Config, dataset_stats) -> list[ProcessorStep]:
|
||||
return [
|
||||
NormalizerProcessorStep(
|
||||
features={**(config.input_features or {}), **(config.output_features or {})},
|
||||
norm_map=config.normalization_mapping,
|
||||
@ -241,23 +266,4 @@ def make_pistar06_pre_post_processors(
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
),
|
||||
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
|
||||
DeviceProcessorStep(device=config.device),
|
||||
]
|
||||
|
||||
output_steps: list[ProcessorStep] = [
|
||||
DeviceProcessorStep(device="cpu"),
|
||||
]
|
||||
|
||||
return (
|
||||
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
|
||||
steps=input_steps,
|
||||
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
|
||||
),
|
||||
PolicyProcessorPipeline[PolicyAction, PolicyAction](
|
||||
steps=output_steps,
|
||||
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
|
||||
to_transition=policy_action_to_transition,
|
||||
to_output=transition_to_policy_action,
|
||||
),
|
||||
)
|
||||
|
||||
110
src/lerobot/values/rewards.py
普通文件
110
src/lerobot/values/rewards.py
普通文件
@ -0,0 +1,110 @@
|
||||
"""Success-only Monte Carlo targets with task scales fitted on training episodes."""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
from lerobot.configs.value_reward import ValueRewardConfig
|
||||
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, normalize_episode_success_label
|
||||
|
||||
|
||||
def build_successful_value_targets(
|
||||
dataset,
|
||||
reward: ValueRewardConfig,
|
||||
success_field: str,
|
||||
*,
|
||||
task_scales: dict[str, float] | None = None,
|
||||
) -> tuple[np.ndarray, dict[str, float]]:
|
||||
"""Return targets in raw frame order; reject partial, failed or unlabelled episodes.
|
||||
|
||||
Only episodes actually present in hf_dataset contribute to the fitted scale.
|
||||
Task names, rather than dataset-local task indices, identify checkpoint scales.
|
||||
Supplying scales reuses them without refitting or clipping out-of-range returns.
|
||||
"""
|
||||
reward.validate()
|
||||
if reward.mode == "legacy":
|
||||
raise ValueError("Use the legacy target builder for reward.mode=legacy.")
|
||||
|
||||
frames = dataset.hf_dataset.with_format(None)
|
||||
episode_ids = np.asarray(frames["episode_index"], dtype=np.int64).reshape(-1)
|
||||
frame_ids = np.asarray(frames["frame_index"], dtype=np.int64).reshape(-1)
|
||||
if len(episode_ids) == 0:
|
||||
raise ValueError("Dataset has no frames.")
|
||||
episodes = dataset.meta.episodes.with_format(None)
|
||||
if success_field not in episodes.column_names:
|
||||
raise ValueError(f"reward.mode={reward.mode} requires explicit '{success_field}=success' labels.")
|
||||
meta = episodes[:]
|
||||
active_ids = set(episode_ids.tolist())
|
||||
metadata = {}
|
||||
for i, episode_id in enumerate(meta["episode_index"]):
|
||||
episode_id = int(episode_id)
|
||||
if episode_id not in active_ids:
|
||||
continue
|
||||
if episode_id in metadata:
|
||||
raise ValueError(f"Duplicate episode metadata for episode {episode_id}.")
|
||||
label = normalize_episode_success_label(meta[success_field][i])
|
||||
if label != EPISODE_SUCCESS:
|
||||
raise ValueError(
|
||||
f"reward.mode={reward.mode} requires only explicitly successful episodes; "
|
||||
f"episode {episode_id} has label {label!r}."
|
||||
)
|
||||
tasks = meta["tasks"][i]
|
||||
if not isinstance(tasks, str):
|
||||
if len(tasks) != 1:
|
||||
raise ValueError(f"Episode {episode_id} must have exactly one task for reward normalization.")
|
||||
tasks = tasks[0]
|
||||
metadata[episode_id] = (int(meta["length"][i]), str(tasks))
|
||||
if active_ids != set(metadata):
|
||||
raise ValueError("Missing episode metadata for reward construction.")
|
||||
|
||||
interventions = None
|
||||
if reward.mode == "time_takeover":
|
||||
if reward.intervention_field not in frames.column_names:
|
||||
raise ValueError(f"Missing intervention field '{reward.intervention_field}' for time_takeover.")
|
||||
interventions = np.asarray(frames[reward.intervention_field]).reshape(-1)
|
||||
if interventions.shape != episode_ids.shape or not np.isin(interventions, [0, 1]).all():
|
||||
raise ValueError("Intervention flags must contain one boolean or binary 0/1 value per frame.")
|
||||
interventions = interventions.astype(bool)
|
||||
|
||||
returns = np.empty(len(episode_ids), dtype=np.float64)
|
||||
fitted_scales: dict[str, float] = {}
|
||||
positions_by_task: dict[str, list[np.ndarray]] = {}
|
||||
for episode_id in sorted(active_ids):
|
||||
positions = np.flatnonzero(episode_ids == episode_id)
|
||||
length, task = metadata[episode_id]
|
||||
if (
|
||||
length <= 0
|
||||
or not np.array_equal(frame_ids[positions], np.arange(length))
|
||||
or not np.all(np.diff(positions) == 1)
|
||||
):
|
||||
raise ValueError(
|
||||
f"Episode {episode_id} must contain every frame exactly once, contiguously from 0 to length-1."
|
||||
)
|
||||
rewards = np.full(length, -1.0, dtype=np.float64)
|
||||
rewards[-1] = 0.0
|
||||
if interventions is not None and length > 2:
|
||||
human = interventions[positions]
|
||||
# Penalize the preceding autonomous frame. Entry into the terminal
|
||||
# observation and initial human control are not takeover events.
|
||||
takeovers = (~human[:-2]) & human[1:-1]
|
||||
rewards[:-2] -= reward.takeover_cost * takeovers
|
||||
episode_returns = np.cumsum(rewards[::-1])[::-1]
|
||||
returns[positions] = episode_returns
|
||||
# A one-frame successful episode has zero cost; use scale 1 to avoid 0/0.
|
||||
fitted_scales[task] = max(fitted_scales.get(task, 1.0), float(-episode_returns[0]))
|
||||
positions_by_task.setdefault(task, []).append(positions)
|
||||
|
||||
scales = dict(fitted_scales if task_scales is None else task_scales)
|
||||
targets = np.empty(len(returns), dtype=np.float32)
|
||||
for task, chunks in positions_by_task.items():
|
||||
scale = scales.get(task)
|
||||
if scale is None or not math.isfinite(scale) or scale <= 0:
|
||||
raise ValueError(f"Missing or invalid saved reward scale for task {task!r}.")
|
||||
positions = np.concatenate(chunks)
|
||||
normalized = returns[positions] / scale
|
||||
if np.any(normalized < -1.0) or np.any(normalized > 0.0):
|
||||
raise ValueError(
|
||||
f"Returns for task {task!r} exceed the saved reward scale; refusing to clip or refit."
|
||||
)
|
||||
targets[positions] = normalized
|
||||
return targets, scales
|
||||
在新工单中引用
屏蔽一个用户