镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
feat: add vision-only values, takeover rewards and masked action loss
这个提交包含在:
@ -9,6 +9,7 @@ from typing import Any
|
|||||||
|
|
||||||
import draccus
|
import draccus
|
||||||
|
|
||||||
|
from lerobot.configs.value_reward import ValueRewardConfig
|
||||||
from lerobot.utils.recording_annotations import normalize_episode_success_label
|
from lerobot.utils.recording_annotations import normalize_episode_success_label
|
||||||
|
|
||||||
|
|
||||||
@ -118,6 +119,8 @@ class ValueInferencePipelineConfig:
|
|||||||
)
|
)
|
||||||
runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig)
|
runtime: ValueInferenceRuntimeConfig = field(default_factory=ValueInferenceRuntimeConfig)
|
||||||
acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig)
|
acp: ValueInferenceACPConfig = field(default_factory=ValueInferenceACPConfig)
|
||||||
|
# None inherits the trained checkpoint; old checkpoints resolve to legacy.
|
||||||
|
reward: ValueRewardConfig | None = None
|
||||||
viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig)
|
viz: ValueInferenceVizConfig = field(default_factory=ValueInferenceVizConfig)
|
||||||
|
|
||||||
output_dir: Path | None = None
|
output_dir: Path | None = None
|
||||||
@ -131,6 +134,8 @@ class ValueInferencePipelineConfig:
|
|||||||
self.inference.validate()
|
self.inference.validate()
|
||||||
self.runtime.validate()
|
self.runtime.validate()
|
||||||
self.acp.validate()
|
self.acp.validate()
|
||||||
|
if self.reward is not None:
|
||||||
|
self.reward.validate()
|
||||||
self.viz.validate()
|
self.viz.validate()
|
||||||
|
|
||||||
if not self.job_name:
|
if not self.job_name:
|
||||||
|
|||||||
@ -0,0 +1,19 @@
|
|||||||
|
"""Reward settings shared by value training and inference."""
|
||||||
|
|
||||||
|
import math
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ValueRewardConfig:
|
||||||
|
mode: str = "legacy"
|
||||||
|
takeover_cost: float = 150.0
|
||||||
|
intervention_field: str = "complementary_info.is_intervention"
|
||||||
|
|
||||||
|
def validate(self) -> None:
|
||||||
|
if self.mode not in {"legacy", "time", "time_takeover"}:
|
||||||
|
raise ValueError("'reward.mode' must be legacy, time or time_takeover.")
|
||||||
|
if not math.isfinite(self.takeover_cost) or self.takeover_cost < 0:
|
||||||
|
raise ValueError("'reward.takeover_cost' must be finite and non-negative.")
|
||||||
|
if self.mode == "time_takeover" and not self.intervention_field:
|
||||||
|
raise ValueError("'reward.intervention_field' must be non-empty for time_takeover.")
|
||||||
@ -14,6 +14,7 @@ from huggingface_hub.errors import HfHubHTTPError
|
|||||||
from lerobot.configs import parser
|
from lerobot.configs import parser
|
||||||
from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig
|
from lerobot.configs.default import DatasetConfig, PeftConfig, WandBConfig
|
||||||
from lerobot.configs.policies import PreTrainedConfig
|
from lerobot.configs.policies import PreTrainedConfig
|
||||||
|
from lerobot.configs.value_reward import ValueRewardConfig
|
||||||
from lerobot.optim import OptimizerConfig
|
from lerobot.optim import OptimizerConfig
|
||||||
from lerobot.optim.schedulers import LRSchedulerConfig
|
from lerobot.optim.schedulers import LRSchedulerConfig
|
||||||
from lerobot.utils.hub import HubMixin
|
from lerobot.utils.hub import HubMixin
|
||||||
@ -72,6 +73,7 @@ class ValueTrainPipelineConfig(HubMixin):
|
|||||||
optimizer: OptimizerConfig | None = None
|
optimizer: OptimizerConfig | None = None
|
||||||
scheduler: LRSchedulerConfig | None = None
|
scheduler: LRSchedulerConfig | None = None
|
||||||
|
|
||||||
|
reward: ValueRewardConfig | None = None
|
||||||
targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig)
|
targets: ValueTargetsConfig = field(default_factory=ValueTargetsConfig)
|
||||||
wandb: WandBConfig = field(default_factory=WandBConfig)
|
wandb: WandBConfig = field(default_factory=WandBConfig)
|
||||||
peft: PeftConfig | None = None
|
peft: PeftConfig | None = None
|
||||||
@ -113,6 +115,8 @@ class ValueTrainPipelineConfig(HubMixin):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.targets.validate()
|
self.targets.validate()
|
||||||
|
if self.reward is not None:
|
||||||
|
self.reward.validate()
|
||||||
|
|
||||||
if hasattr(self.value, "target_key"):
|
if hasattr(self.value, "target_key"):
|
||||||
self.value.target_key = self.targets.target_field
|
self.value.target_key = self.targets.target_field
|
||||||
|
|||||||
@ -71,6 +71,9 @@ class PI05Config(PreTrainedConfig):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Exclude repeated episode-tail actions from flow-matching supervision.
|
||||||
|
mask_action_padding_loss: bool = False
|
||||||
|
|
||||||
# Training settings
|
# Training settings
|
||||||
gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization
|
gradient_checkpointing: bool = False # Enable gradient checkpointing for memory optimization
|
||||||
compile_model: bool = False # Whether to use torch.compile for model optimization
|
compile_model: bool = False # Whether to use torch.compile for model optimization
|
||||||
|
|||||||
@ -1250,6 +1250,16 @@ class PI05Policy(PreTrainedPolicy):
|
|||||||
- "mean": Return scalar mean loss (default, backward compatible)
|
- "mean": Return scalar mean loss (default, backward compatible)
|
||||||
- "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting
|
- "none": Return per-sample losses of shape (batch_size,) for RA-BC weighting
|
||||||
"""
|
"""
|
||||||
|
action_is_pad = None
|
||||||
|
if self.config.mask_action_padding_loss:
|
||||||
|
action_is_pad = batch.get("action_is_pad")
|
||||||
|
if action_is_pad is None:
|
||||||
|
raise ValueError(
|
||||||
|
"mask_action_padding_loss=True requires action_is_pad in the training batch."
|
||||||
|
)
|
||||||
|
if not isinstance(action_is_pad, Tensor) or action_is_pad.dtype != torch.bool:
|
||||||
|
raise TypeError("action_is_pad must be a boolean tensor (True marks padding).")
|
||||||
|
|
||||||
# Prepare inputs
|
# Prepare inputs
|
||||||
images, img_masks = self._preprocess_images(batch)
|
images, img_masks = self._preprocess_images(batch)
|
||||||
tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
|
tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
|
||||||
@ -1263,6 +1273,9 @@ class PI05Policy(PreTrainedPolicy):
|
|||||||
original_action_dim = self.config.output_features[ACTION].shape[0]
|
original_action_dim = self.config.output_features[ACTION].shape[0]
|
||||||
losses = losses[:, :, :original_action_dim]
|
losses = losses[:, :, :original_action_dim]
|
||||||
|
|
||||||
|
if action_is_pad is not None:
|
||||||
|
return self._reduce_masked_action_losses(losses, action_is_pad, reduction)
|
||||||
|
|
||||||
loss_dict = {
|
loss_dict = {
|
||||||
"loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(),
|
"loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(),
|
||||||
}
|
}
|
||||||
@ -1278,6 +1291,27 @@ class PI05Policy(PreTrainedPolicy):
|
|||||||
loss_dict["loss"] = loss.item()
|
loss_dict["loss"] = loss.item()
|
||||||
return loss, loss_dict
|
return loss, loss_dict
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _reduce_masked_action_losses(losses: Tensor, action_is_pad: Tensor, reduction: str):
|
||||||
|
if tuple(action_is_pad.shape) != tuple(losses.shape[:2]):
|
||||||
|
raise ValueError("action_is_pad must have shape [batch_size, action_horizon].")
|
||||||
|
valid = ~action_is_pad.to(device=losses.device)
|
||||||
|
valid_steps = valid.sum(dim=1)
|
||||||
|
if torch.any(valid_steps == 0):
|
||||||
|
raise ValueError("Every action chunk must contain at least one non-padding timestep.")
|
||||||
|
# Mask with where: even NaN/Inf in an unused loss slot must not enter the sum.
|
||||||
|
masked = torch.where(valid.unsqueeze(-1), losses, 0.0)
|
||||||
|
if reduction == "none":
|
||||||
|
loss = masked.sum(dim=(1, 2)) / (valid_steps * losses.shape[-1])
|
||||||
|
elif reduction == "mean":
|
||||||
|
loss = masked.sum() / (valid_steps.sum() * losses.shape[-1])
|
||||||
|
else:
|
||||||
|
raise ValueError("reduction must be 'mean' or 'none'.")
|
||||||
|
return loss, {
|
||||||
|
"loss": loss.detach().mean().item(),
|
||||||
|
"loss_per_dim": (masked.sum(dim=(0, 1)) / valid_steps.sum()).detach().cpu().tolist(),
|
||||||
|
}
|
||||||
|
|
||||||
def _get_default_peft_targets(self) -> dict[str, any]:
|
def _get_default_peft_targets(self) -> dict[str, any]:
|
||||||
"""Return default PEFT target modules for PI0.5 fine-tuning."""
|
"""Return default PEFT target modules for PI0.5 fine-tuning."""
|
||||||
common_projections = (
|
common_projections = (
|
||||||
|
|||||||
@ -32,6 +32,7 @@ from tqdm.auto import tqdm
|
|||||||
from lerobot.configs import parser
|
from lerobot.configs import parser
|
||||||
from lerobot.configs.policies import PreTrainedConfig
|
from lerobot.configs.policies import PreTrainedConfig
|
||||||
from lerobot.configs.value import ValueInferencePipelineConfig
|
from lerobot.configs.value import ValueInferencePipelineConfig
|
||||||
|
from lerobot.configs.value_reward import ValueRewardConfig
|
||||||
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
from lerobot.datasets.lerobot_dataset import LeRobotDataset
|
||||||
from lerobot.datasets.utils import load_info, write_info
|
from lerobot.datasets.utils import load_info, write_info
|
||||||
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
from lerobot.policies.factory import make_policy, make_pre_post_processors
|
||||||
@ -52,6 +53,7 @@ from lerobot.values.pistar06.modeling_pistar06 import (
|
|||||||
EpisodeTargetInfo,
|
EpisodeTargetInfo,
|
||||||
compute_normalized_value_targets,
|
compute_normalized_value_targets,
|
||||||
)
|
)
|
||||||
|
from lerobot.values.rewards import build_successful_value_targets
|
||||||
|
|
||||||
|
|
||||||
class ContiguousDistributedEvalSampler(Sampler[int]):
|
class ContiguousDistributedEvalSampler(Sampler[int]):
|
||||||
@ -522,6 +524,23 @@ def run_value_inference_pipeline(
|
|||||||
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
|
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
|
||||||
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
||||||
|
|
||||||
|
reward = value_cfg.reward or ValueRewardConfig()
|
||||||
|
if cfg.reward is not None and cfg.reward != reward:
|
||||||
|
raise ValueError("Inference reward settings must match the trained checkpoint.")
|
||||||
|
cfg.reward = reward
|
||||||
|
success_targets = None
|
||||||
|
if reward.mode != "legacy":
|
||||||
|
if not value_cfg.reward_scales:
|
||||||
|
raise ValueError(
|
||||||
|
"Checkpoint is missing training reward scales; refusing to fit on inference data."
|
||||||
|
)
|
||||||
|
success_targets, _ = build_successful_value_targets(
|
||||||
|
dataset,
|
||||||
|
reward,
|
||||||
|
cfg.dataset.success_field,
|
||||||
|
task_scales=value_cfg.reward_scales,
|
||||||
|
)
|
||||||
|
|
||||||
if cfg.acp.intervention_field in raw_frames.column_names:
|
if cfg.acp.intervention_field in raw_frames.column_names:
|
||||||
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
|
interventions = np.asarray(raw_frames[cfg.acp.intervention_field], dtype=np.float32)
|
||||||
else:
|
else:
|
||||||
@ -595,6 +614,8 @@ def run_value_inference_pipeline(
|
|||||||
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
|
raise RuntimeError(f"Inference is missing predictions for {missing_count} frames.")
|
||||||
|
|
||||||
predicted_values = prediction_lookup[absolute_indices]
|
predicted_values = prediction_lookup[absolute_indices]
|
||||||
|
if success_targets is not None:
|
||||||
|
predicted_values[success_targets == 0.0] = 0.0
|
||||||
logging.info(
|
logging.info(
|
||||||
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
|
"Predicted value stats | min=%.6f max=%.6f mean=%.6f std=%.6f",
|
||||||
float(np.min(predicted_values)),
|
float(np.min(predicted_values)),
|
||||||
@ -614,12 +635,12 @@ def run_value_inference_pipeline(
|
|||||||
thresholds: dict[int, float] | None = None
|
thresholds: dict[int, float] | None = None
|
||||||
|
|
||||||
if cfg.acp.enable:
|
if cfg.acp.enable:
|
||||||
|
if success_targets is None:
|
||||||
episode_info, task_max_lengths = _build_episode_info(
|
episode_info, task_max_lengths = _build_episode_info(
|
||||||
dataset=dataset,
|
dataset=dataset,
|
||||||
success_field=cfg.dataset.success_field,
|
success_field=cfg.dataset.success_field,
|
||||||
default_success=cfg.dataset.default_success,
|
default_success=cfg.dataset.default_success,
|
||||||
)
|
)
|
||||||
|
|
||||||
value_targets = compute_normalized_value_targets(
|
value_targets = compute_normalized_value_targets(
|
||||||
episode_indices=episode_indices,
|
episode_indices=episode_indices,
|
||||||
frame_indices=frame_indices,
|
frame_indices=frame_indices,
|
||||||
@ -629,6 +650,8 @@ def run_value_inference_pipeline(
|
|||||||
clip_min=value_cfg.bin_min,
|
clip_min=value_cfg.bin_min,
|
||||||
clip_max=value_cfg.bin_max,
|
clip_max=value_cfg.bin_max,
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
value_targets = success_targets
|
||||||
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
|
rewards = _compute_dense_rewards_from_targets(value_targets, episode_indices, frame_indices)
|
||||||
advantages = _compute_n_step_advantages(
|
advantages = _compute_n_step_advantages(
|
||||||
rewards=rewards,
|
rewards=rewards,
|
||||||
|
|||||||
@ -131,7 +131,9 @@ def value_train(
|
|||||||
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
|
value_target_raw_batch_hook = value_model.build_training_raw_batch_hook(
|
||||||
dataset=dataset,
|
dataset=dataset,
|
||||||
targets_cfg=cfg.targets,
|
targets_cfg=cfg.targets,
|
||||||
|
reward_cfg=cfg.reward,
|
||||||
)
|
)
|
||||||
|
cfg.reward = value_model.config.reward
|
||||||
if is_main_process:
|
if is_main_process:
|
||||||
logging.info("Using value model raw-batch hook for target construction.")
|
logging.info("Using value model raw-batch hook for target construction.")
|
||||||
elif is_main_process:
|
elif is_main_process:
|
||||||
@ -163,6 +165,9 @@ def value_train(
|
|||||||
"norm_map": cfg.value.normalization_mapping,
|
"norm_map": cfg.value.normalization_mapping,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
if cfg.value.mode == "vision_only":
|
||||||
|
processor_kwargs["preprocessor_overrides"].pop("normalizer_processor", None)
|
||||||
|
postprocessor_kwargs["postprocessor_overrides"].pop("unnormalizer_processor", None)
|
||||||
|
|
||||||
preprocessor, postprocessor = make_pre_post_processors(
|
preprocessor, postprocessor = make_pre_post_processors(
|
||||||
policy_cfg=cfg.value,
|
policy_cfg=cfg.value,
|
||||||
|
|||||||
@ -4,6 +4,7 @@ from dataclasses import dataclass, field
|
|||||||
|
|
||||||
from lerobot.configs.policies import PreTrainedConfig
|
from lerobot.configs.policies import PreTrainedConfig
|
||||||
from lerobot.configs.types import NormalizationMode
|
from lerobot.configs.types import NormalizationMode
|
||||||
|
from lerobot.configs.value_reward import ValueRewardConfig
|
||||||
from lerobot.optim.optimizers import AdamWConfig
|
from lerobot.optim.optimizers import AdamWConfig
|
||||||
from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
|
from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
|
||||||
from lerobot.utils.constants import OBS_STATE
|
from lerobot.utils.constants import OBS_STATE
|
||||||
@ -12,7 +13,12 @@ from lerobot.utils.constants import OBS_STATE
|
|||||||
@PreTrainedConfig.register_subclass("pistar06")
|
@PreTrainedConfig.register_subclass("pistar06")
|
||||||
@dataclass
|
@dataclass
|
||||||
class Pistar06Config(PreTrainedConfig):
|
class Pistar06Config(PreTrainedConfig):
|
||||||
"""Value model config using the Pistar06 stack (SigLIP + Gemma)."""
|
"""SigLIP value model with optional Gemma language conditioning."""
|
||||||
|
|
||||||
|
# Old checkpoints without this field retain the language-conditioned path.
|
||||||
|
mode: str = "vision_language"
|
||||||
|
reward: ValueRewardConfig | None = None
|
||||||
|
reward_scales: dict[str, float] = field(default_factory=dict)
|
||||||
|
|
||||||
# Backbone components
|
# Backbone components
|
||||||
vision_repo_id: str = "google/siglip-so400m-patch14-384"
|
vision_repo_id: str = "google/siglip-so400m-patch14-384"
|
||||||
@ -70,9 +76,13 @@ class Pistar06Config(PreTrainedConfig):
|
|||||||
def __post_init__(self) -> None:
|
def __post_init__(self) -> None:
|
||||||
super().__post_init__()
|
super().__post_init__()
|
||||||
|
|
||||||
|
if self.mode not in {"vision_language", "vision_only"}:
|
||||||
|
raise ValueError("'value.mode' must be vision_language or vision_only.")
|
||||||
|
if self.reward is not None:
|
||||||
|
self.reward.validate()
|
||||||
if not self.vision_repo_id:
|
if not self.vision_repo_id:
|
||||||
raise ValueError("'value.vision_repo_id' must be non-empty.")
|
raise ValueError("'value.vision_repo_id' must be non-empty.")
|
||||||
if not self.language_repo_id:
|
if self.mode == "vision_language" and not self.language_repo_id:
|
||||||
raise ValueError("'value.language_repo_id' must be non-empty.")
|
raise ValueError("'value.language_repo_id' must be non-empty.")
|
||||||
if not self.task_field:
|
if not self.task_field:
|
||||||
raise ValueError("'value.task_field' must be non-empty.")
|
raise ValueError("'value.task_field' must be non-empty.")
|
||||||
|
|||||||
@ -19,20 +19,29 @@ from huggingface_hub.errors import HfHubHTTPError
|
|||||||
from safetensors.torch import save_file
|
from safetensors.torch import save_file
|
||||||
from torch import Tensor, nn
|
from torch import Tensor, nn
|
||||||
|
|
||||||
|
from lerobot.configs.value_reward import ValueRewardConfig
|
||||||
from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy
|
from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy
|
||||||
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
|
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
|
||||||
from lerobot.utils.import_utils import _transformers_available
|
from lerobot.utils.import_utils import _transformers_available
|
||||||
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
|
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
|
||||||
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
|
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
|
||||||
from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY
|
from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY
|
||||||
|
from lerobot.values.rewards import build_successful_value_targets
|
||||||
|
|
||||||
if TYPE_CHECKING or _transformers_available:
|
if TYPE_CHECKING or _transformers_available:
|
||||||
from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoModelForCausalLM
|
from transformers import (
|
||||||
|
AutoConfig,
|
||||||
|
AutoImageProcessor,
|
||||||
|
AutoModel,
|
||||||
|
AutoModelForCausalLM,
|
||||||
|
SiglipVisionModel,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
AutoConfig = None
|
AutoConfig = None
|
||||||
AutoImageProcessor = None
|
AutoImageProcessor = None
|
||||||
AutoModel = None
|
AutoModel = None
|
||||||
AutoModelForCausalLM = None
|
AutoModelForCausalLM = None
|
||||||
|
SiglipVisionModel = None
|
||||||
|
|
||||||
|
|
||||||
PISTAR06_SAVE_INFO = "pistar06_save_info.json"
|
PISTAR06_SAVE_INFO = "pistar06_save_info.json"
|
||||||
@ -261,11 +270,15 @@ class Pistar06Model(nn.Module):
|
|||||||
self.cfg = cfg
|
self.cfg = cfg
|
||||||
self.model_dtype = _resolve_load_dtype(cfg.dtype)
|
self.model_dtype = _resolve_load_dtype(cfg.dtype)
|
||||||
|
|
||||||
self.vision_encoder = AutoModel.from_pretrained(
|
# SiglipVisionModel loads only the visual weights; AutoModel on a full
|
||||||
|
# SigLIP repository would also create an unused text encoder under DDP.
|
||||||
|
vision_loader = SiglipVisionModel if cfg.mode == "vision_only" else AutoModel
|
||||||
|
self.vision_encoder = vision_loader.from_pretrained(
|
||||||
cfg.vision_repo_id,
|
cfg.vision_repo_id,
|
||||||
revision=cfg.vision_revision,
|
revision=cfg.vision_revision,
|
||||||
torch_dtype=self.model_dtype,
|
torch_dtype=self.model_dtype,
|
||||||
)
|
)
|
||||||
|
if cfg.mode == "vision_language":
|
||||||
self.language_model = _load_language_model(
|
self.language_model = _load_language_model(
|
||||||
repo_id=cfg.language_repo_id,
|
repo_id=cfg.language_repo_id,
|
||||||
revision=cfg.language_revision,
|
revision=cfg.language_revision,
|
||||||
@ -292,31 +305,35 @@ class Pistar06Model(nn.Module):
|
|||||||
)
|
)
|
||||||
|
|
||||||
vision_feature_size = _extract_vision_feature_size(self.vision_encoder)
|
vision_feature_size = _extract_vision_feature_size(self.vision_encoder)
|
||||||
language_hidden_size = _extract_hidden_size(self.language_model)
|
|
||||||
|
|
||||||
self.image_projector = nn.Sequential(
|
self.image_projector = nn.Sequential(
|
||||||
nn.Linear(vision_feature_size, cfg.fusion_hidden_dim),
|
nn.Linear(vision_feature_size, cfg.fusion_hidden_dim),
|
||||||
nn.GELU(),
|
nn.GELU(),
|
||||||
nn.Dropout(cfg.dropout),
|
nn.Dropout(cfg.dropout),
|
||||||
)
|
)
|
||||||
|
head_input_dim = cfg.fusion_hidden_dim
|
||||||
|
if cfg.mode == "vision_language":
|
||||||
|
language_hidden_size = _extract_hidden_size(self.language_model)
|
||||||
self.language_projector = nn.Sequential(
|
self.language_projector = nn.Sequential(
|
||||||
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
|
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
|
||||||
nn.GELU(),
|
nn.GELU(),
|
||||||
nn.Dropout(cfg.dropout),
|
nn.Dropout(cfg.dropout),
|
||||||
)
|
)
|
||||||
self.final_norm = nn.LayerNorm(cfg.fusion_hidden_dim * 2)
|
head_input_dim *= 2
|
||||||
|
self.final_norm = nn.LayerNorm(head_input_dim)
|
||||||
self.value_head = nn.Sequential(
|
self.value_head = nn.Sequential(
|
||||||
nn.Linear(cfg.fusion_hidden_dim * 2, cfg.fusion_hidden_dim),
|
nn.Linear(head_input_dim, cfg.fusion_hidden_dim),
|
||||||
nn.GELU(),
|
nn.GELU(),
|
||||||
nn.Dropout(cfg.dropout),
|
nn.Dropout(cfg.dropout),
|
||||||
nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins),
|
nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins),
|
||||||
)
|
)
|
||||||
|
|
||||||
if cfg.use_gradient_checkpointing:
|
if cfg.use_gradient_checkpointing:
|
||||||
|
if cfg.mode == "vision_language":
|
||||||
_maybe_enable_gradient_checkpointing(self.language_model)
|
_maybe_enable_gradient_checkpointing(self.language_model)
|
||||||
_maybe_enable_gradient_checkpointing(self.vision_encoder)
|
_maybe_enable_gradient_checkpointing(self.vision_encoder)
|
||||||
|
|
||||||
if cfg.freeze_language_model:
|
if cfg.mode == "vision_language" and cfg.freeze_language_model:
|
||||||
_freeze_module(self.language_model)
|
_freeze_module(self.language_model)
|
||||||
if cfg.freeze_vision_encoder:
|
if cfg.freeze_vision_encoder:
|
||||||
_freeze_module(self.vision_encoder)
|
_freeze_module(self.vision_encoder)
|
||||||
@ -358,6 +375,10 @@ class Pistar06Model(nn.Module):
|
|||||||
if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras:
|
if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras:
|
||||||
raise ValueError("Batch shape mismatch between images and image_attention_mask.")
|
raise ValueError("Batch shape mismatch between images and image_attention_mask.")
|
||||||
|
|
||||||
|
if self.cfg.mode == "vision_only":
|
||||||
|
# Missing cameras must not influence detection of the input pixel range.
|
||||||
|
images = torch.where(image_attention_mask[:, :, None, None, None], images, 0)
|
||||||
|
|
||||||
if images.dtype == torch.uint8:
|
if images.dtype == torch.uint8:
|
||||||
images = images.to(dtype=torch.float32) / 255.0
|
images = images.to(dtype=torch.float32) / 255.0
|
||||||
else:
|
else:
|
||||||
@ -393,33 +414,26 @@ class Pistar06Model(nn.Module):
|
|||||||
|
|
||||||
def forward(
|
def forward(
|
||||||
self,
|
self,
|
||||||
input_ids: Tensor,
|
input_ids: Tensor | None = None,
|
||||||
attention_mask: Tensor,
|
attention_mask: Tensor | None = None,
|
||||||
images: Tensor,
|
images: Tensor | None = None,
|
||||||
image_attention_mask: Tensor,
|
image_attention_mask: Tensor | None = None,
|
||||||
) -> Tensor:
|
) -> Tensor:
|
||||||
if input_ids.ndim != 2:
|
if images is None or images.ndim != 5:
|
||||||
raise ValueError(f"'input_ids' must have shape [B, T], got {tuple(input_ids.shape)}.")
|
raise ValueError("'images' must have shape [B, N, C, H, W].")
|
||||||
if attention_mask.ndim != 2:
|
bsize, num_cameras = images.shape[:2]
|
||||||
raise ValueError(f"'attention_mask' must have shape [B, T], got {tuple(attention_mask.shape)}.")
|
if num_cameras == 0:
|
||||||
if images.ndim != 5:
|
|
||||||
raise ValueError(f"'images' must have shape [B, N, C, H, W], got {tuple(images.shape)}.")
|
|
||||||
if image_attention_mask.ndim != 2:
|
|
||||||
raise ValueError(
|
|
||||||
f"'image_attention_mask' must have shape [B, N], got {tuple(image_attention_mask.shape)}."
|
|
||||||
)
|
|
||||||
|
|
||||||
bsize = input_ids.shape[0]
|
|
||||||
if attention_mask.shape[0] != bsize:
|
|
||||||
raise ValueError("Batch size mismatch between input_ids and attention_mask.")
|
|
||||||
if images.shape[0] != bsize or image_attention_mask.shape[0] != bsize:
|
|
||||||
raise ValueError("Batch size mismatch between language and image inputs.")
|
|
||||||
if images.shape[1] == 0:
|
|
||||||
raise ValueError("At least one camera is required for Pistar06Model.")
|
raise ValueError("At least one camera is required for Pistar06Model.")
|
||||||
|
if image_attention_mask is None or tuple(image_attention_mask.shape) != (bsize, num_cameras):
|
||||||
|
raise ValueError("'image_attention_mask' must have shape [B, N].")
|
||||||
image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device)
|
image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device)
|
||||||
if not torch.all(image_attention_mask.any(dim=1)):
|
if not torch.all(image_attention_mask.any(dim=1)):
|
||||||
raise ValueError("Each sample must have at least one valid camera input.")
|
raise ValueError("Each sample must have at least one valid camera input.")
|
||||||
|
if self.cfg.mode == "vision_language":
|
||||||
|
if input_ids is None or attention_mask is None:
|
||||||
|
raise ValueError("vision_language requires input_ids and attention_mask.")
|
||||||
|
if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape or input_ids.shape[0] != bsize:
|
||||||
|
raise ValueError("Language tensors must have matching shape [B, T].")
|
||||||
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
|
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
|
||||||
if not torch.all(language_mask.any(dim=1)):
|
if not torch.all(language_mask.any(dim=1)):
|
||||||
raise ValueError("Each sample must have at least one valid language token.")
|
raise ValueError("Each sample must have at least one valid language token.")
|
||||||
@ -433,6 +447,7 @@ class Pistar06Model(nn.Module):
|
|||||||
with image_context:
|
with image_context:
|
||||||
image_features = self._encode_images(flat_images)
|
image_features = self._encode_images(flat_images)
|
||||||
|
|
||||||
|
if self.cfg.mode == "vision_language":
|
||||||
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
|
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
|
||||||
with language_context:
|
with language_context:
|
||||||
language_features = self._encode_language(
|
language_features = self._encode_language(
|
||||||
@ -441,7 +456,6 @@ class Pistar06Model(nn.Module):
|
|||||||
|
|
||||||
feature_dtype = torch.float32
|
feature_dtype = torch.float32
|
||||||
image_features = image_features.to(dtype=feature_dtype)
|
image_features = image_features.to(dtype=feature_dtype)
|
||||||
language_features = language_features.to(dtype=feature_dtype)
|
|
||||||
|
|
||||||
image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1)
|
image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1)
|
||||||
camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype)
|
camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype)
|
||||||
@ -451,10 +465,11 @@ class Pistar06Model(nn.Module):
|
|||||||
image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0)
|
image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0)
|
||||||
)
|
)
|
||||||
image_pooled = image_tokens.sum(dim=1) / camera_denominator
|
image_pooled = image_tokens.sum(dim=1) / camera_denominator
|
||||||
language_token = self.language_projector(language_features)
|
if self.cfg.mode == "vision_only":
|
||||||
|
return self.value_head(self.final_norm(image_pooled))
|
||||||
|
|
||||||
joint_features = torch.cat([image_pooled, language_token], dim=-1)
|
language_token = self.language_projector(language_features.to(dtype=feature_dtype))
|
||||||
return self.value_head(self.final_norm(joint_features))
|
return self.value_head(self.final_norm(torch.cat([image_pooled, language_token], dim=-1)))
|
||||||
|
|
||||||
|
|
||||||
class Pistar06Policy(PreTrainedPolicy):
|
class Pistar06Policy(PreTrainedPolicy):
|
||||||
@ -482,7 +497,7 @@ class Pistar06Policy(PreTrainedPolicy):
|
|||||||
prefixes: list[str] = []
|
prefixes: list[str] = []
|
||||||
if self.config.freeze_vision_encoder:
|
if self.config.freeze_vision_encoder:
|
||||||
prefixes.append("model.vision_encoder.")
|
prefixes.append("model.vision_encoder.")
|
||||||
if self.config.freeze_language_model:
|
if self.config.mode == "vision_language" and self.config.freeze_language_model:
|
||||||
prefixes.append("model.language_model.")
|
prefixes.append("model.language_model.")
|
||||||
return prefixes
|
return prefixes
|
||||||
|
|
||||||
@ -626,8 +641,8 @@ class Pistar06Policy(PreTrainedPolicy):
|
|||||||
raise RuntimeError("Pistar06Policy is a value model and does not support action selection.")
|
raise RuntimeError("Pistar06Policy is a value model and does not support action selection.")
|
||||||
|
|
||||||
def predict_value(self, batch: dict[str, Tensor]) -> Tensor:
|
def predict_value(self, batch: dict[str, Tensor]) -> Tensor:
|
||||||
input_ids = batch[OBS_LANGUAGE_TOKENS]
|
input_ids = batch.get(OBS_LANGUAGE_TOKENS)
|
||||||
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
|
attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
|
||||||
images = batch[PISTAR06_IMAGES_KEY]
|
images = batch[PISTAR06_IMAGES_KEY]
|
||||||
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
|
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
|
||||||
|
|
||||||
@ -640,7 +655,7 @@ class Pistar06Policy(PreTrainedPolicy):
|
|||||||
bin_centers = self.bin_centers.to(device=logits.device)
|
bin_centers = self.bin_centers.to(device=logits.device)
|
||||||
return expected_value_from_logits(logits, bin_centers)
|
return expected_value_from_logits(logits, bin_centers)
|
||||||
|
|
||||||
def build_training_raw_batch_hook(self, dataset, targets_cfg):
|
def build_training_raw_batch_hook(self, dataset, targets_cfg, reward_cfg=None):
|
||||||
raw_frames = dataset.hf_dataset.with_format(None)
|
raw_frames = dataset.hf_dataset.with_format(None)
|
||||||
frame_count = len(raw_frames)
|
frame_count = len(raw_frames)
|
||||||
if frame_count == 0:
|
if frame_count == 0:
|
||||||
@ -650,6 +665,42 @@ class Pistar06Policy(PreTrainedPolicy):
|
|||||||
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
|
||||||
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
|
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
|
||||||
|
|
||||||
|
reward = reward_cfg or self.config.reward or ValueRewardConfig()
|
||||||
|
reward.validate()
|
||||||
|
if reward.mode != "legacy":
|
||||||
|
saved_scales = self.config.reward_scales if self.config.reward == reward else None
|
||||||
|
value_targets, scales = build_successful_value_targets(
|
||||||
|
dataset,
|
||||||
|
reward,
|
||||||
|
targets_cfg.success_field,
|
||||||
|
task_scales=saved_scales or None,
|
||||||
|
)
|
||||||
|
self.config.reward = reward
|
||||||
|
self.config.reward_scales = scales
|
||||||
|
else:
|
||||||
|
value_targets = self._legacy_training_targets(
|
||||||
|
dataset, targets_cfg, episode_indices, frame_indices
|
||||||
|
)
|
||||||
|
self.config.reward = reward
|
||||||
|
self.config.reward_scales = {}
|
||||||
|
|
||||||
|
max_index = int(np.max(absolute_indices))
|
||||||
|
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
|
||||||
|
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
|
||||||
|
target_key = targets_cfg.target_field
|
||||||
|
|
||||||
|
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
|
||||||
|
del step
|
||||||
|
batch_indices = batch.get("index")
|
||||||
|
if batch_indices is None:
|
||||||
|
raise KeyError("Missing 'index' in batch while building value targets.")
|
||||||
|
batch_indices = torch.as_tensor(batch_indices).detach().cpu().numpy().astype(np.int64).reshape(-1)
|
||||||
|
batch[target_key] = torch.from_numpy(value_target_lookup[batch_indices])
|
||||||
|
return batch
|
||||||
|
|
||||||
|
return value_target_hook
|
||||||
|
|
||||||
|
def _legacy_training_targets(self, dataset, targets_cfg, episode_indices, frame_indices):
|
||||||
episodes_ds = dataset.meta.episodes.with_format(None)
|
episodes_ds = dataset.meta.episodes.with_format(None)
|
||||||
episodes = episodes_ds[:]
|
episodes = episodes_ds[:]
|
||||||
n_episodes = len(episodes_ds)
|
n_episodes = len(episodes_ds)
|
||||||
@ -692,26 +743,7 @@ class Pistar06Policy(PreTrainedPolicy):
|
|||||||
clip_max=self.config.bin_max,
|
clip_max=self.config.bin_max,
|
||||||
)
|
)
|
||||||
|
|
||||||
max_index = int(np.max(absolute_indices))
|
return value_targets
|
||||||
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
|
|
||||||
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
|
|
||||||
|
|
||||||
target_key = targets_cfg.target_field
|
|
||||||
|
|
||||||
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
|
|
||||||
del step
|
|
||||||
batch_indices = batch.get("index")
|
|
||||||
if batch_indices is None:
|
|
||||||
raise KeyError("Missing 'index' in batch while building value targets.")
|
|
||||||
if not isinstance(batch_indices, torch.Tensor):
|
|
||||||
batch_indices = torch.as_tensor(batch_indices)
|
|
||||||
|
|
||||||
batch_indices_np = batch_indices.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
|
|
||||||
target_values = torch.from_numpy(value_target_lookup[batch_indices_np]).to(dtype=torch.float32)
|
|
||||||
batch[target_key] = target_values
|
|
||||||
return batch
|
|
||||||
|
|
||||||
return value_target_hook
|
|
||||||
|
|
||||||
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
|
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
|
||||||
if self.config.target_key not in batch:
|
if self.config.target_key not in batch:
|
||||||
@ -720,8 +752,8 @@ class Pistar06Policy(PreTrainedPolicy):
|
|||||||
"Make sure lerobot-value-train target hook is enabled."
|
"Make sure lerobot-value-train target hook is enabled."
|
||||||
)
|
)
|
||||||
|
|
||||||
input_ids = batch[OBS_LANGUAGE_TOKENS]
|
input_ids = batch.get(OBS_LANGUAGE_TOKENS)
|
||||||
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
|
attention_mask = batch.get(OBS_LANGUAGE_ATTENTION_MASK)
|
||||||
images = batch[PISTAR06_IMAGES_KEY]
|
images = batch[PISTAR06_IMAGES_KEY]
|
||||||
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
|
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
|
||||||
|
|
||||||
|
|||||||
@ -218,8 +218,33 @@ def make_pistar06_pre_post_processors(
|
|||||||
if not camera_features:
|
if not camera_features:
|
||||||
camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)]
|
camera_features = [k for k in (config.input_features or {}) if k.startswith(OBS_IMAGES)]
|
||||||
|
|
||||||
input_steps: list[ProcessorStep] = [
|
input_steps: list[ProcessorStep] = [RenameObservationsProcessorStep(rename_map={})]
|
||||||
RenameObservationsProcessorStep(rename_map={}),
|
if config.mode == "vision_language":
|
||||||
|
input_steps.extend(_language_steps(config, dataset_stats))
|
||||||
|
input_steps.extend(
|
||||||
|
[
|
||||||
|
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
|
||||||
|
DeviceProcessorStep(device=config.device),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
output_steps: list[ProcessorStep] = [DeviceProcessorStep(device="cpu")]
|
||||||
|
return (
|
||||||
|
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
|
||||||
|
steps=input_steps,
|
||||||
|
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
|
||||||
|
),
|
||||||
|
PolicyProcessorPipeline[PolicyAction, PolicyAction](
|
||||||
|
steps=output_steps,
|
||||||
|
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
|
||||||
|
to_transition=policy_action_to_transition,
|
||||||
|
to_output=transition_to_policy_action,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _language_steps(config: Pistar06Config, dataset_stats) -> list[ProcessorStep]:
|
||||||
|
return [
|
||||||
NormalizerProcessorStep(
|
NormalizerProcessorStep(
|
||||||
features={**(config.input_features or {}), **(config.output_features or {})},
|
features={**(config.input_features or {}), **(config.output_features or {})},
|
||||||
norm_map=config.normalization_mapping,
|
norm_map=config.normalization_mapping,
|
||||||
@ -241,23 +266,4 @@ def make_pistar06_pre_post_processors(
|
|||||||
padding="max_length",
|
padding="max_length",
|
||||||
truncation=True,
|
truncation=True,
|
||||||
),
|
),
|
||||||
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
|
|
||||||
DeviceProcessorStep(device=config.device),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
output_steps: list[ProcessorStep] = [
|
|
||||||
DeviceProcessorStep(device="cpu"),
|
|
||||||
]
|
|
||||||
|
|
||||||
return (
|
|
||||||
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
|
|
||||||
steps=input_steps,
|
|
||||||
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
|
|
||||||
),
|
|
||||||
PolicyProcessorPipeline[PolicyAction, PolicyAction](
|
|
||||||
steps=output_steps,
|
|
||||||
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
|
|
||||||
to_transition=policy_action_to_transition,
|
|
||||||
to_output=transition_to_policy_action,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|||||||
110
src/lerobot/values/rewards.py
普通文件
110
src/lerobot/values/rewards.py
普通文件
@ -0,0 +1,110 @@
|
|||||||
|
"""Success-only Monte Carlo targets with task scales fitted on training episodes."""
|
||||||
|
|
||||||
|
import math
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from lerobot.configs.value_reward import ValueRewardConfig
|
||||||
|
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, normalize_episode_success_label
|
||||||
|
|
||||||
|
|
||||||
|
def build_successful_value_targets(
|
||||||
|
dataset,
|
||||||
|
reward: ValueRewardConfig,
|
||||||
|
success_field: str,
|
||||||
|
*,
|
||||||
|
task_scales: dict[str, float] | None = None,
|
||||||
|
) -> tuple[np.ndarray, dict[str, float]]:
|
||||||
|
"""Return targets in raw frame order; reject partial, failed or unlabelled episodes.
|
||||||
|
|
||||||
|
Only episodes actually present in hf_dataset contribute to the fitted scale.
|
||||||
|
Task names, rather than dataset-local task indices, identify checkpoint scales.
|
||||||
|
Supplying scales reuses them without refitting or clipping out-of-range returns.
|
||||||
|
"""
|
||||||
|
reward.validate()
|
||||||
|
if reward.mode == "legacy":
|
||||||
|
raise ValueError("Use the legacy target builder for reward.mode=legacy.")
|
||||||
|
|
||||||
|
frames = dataset.hf_dataset.with_format(None)
|
||||||
|
episode_ids = np.asarray(frames["episode_index"], dtype=np.int64).reshape(-1)
|
||||||
|
frame_ids = np.asarray(frames["frame_index"], dtype=np.int64).reshape(-1)
|
||||||
|
if len(episode_ids) == 0:
|
||||||
|
raise ValueError("Dataset has no frames.")
|
||||||
|
episodes = dataset.meta.episodes.with_format(None)
|
||||||
|
if success_field not in episodes.column_names:
|
||||||
|
raise ValueError(f"reward.mode={reward.mode} requires explicit '{success_field}=success' labels.")
|
||||||
|
meta = episodes[:]
|
||||||
|
active_ids = set(episode_ids.tolist())
|
||||||
|
metadata = {}
|
||||||
|
for i, episode_id in enumerate(meta["episode_index"]):
|
||||||
|
episode_id = int(episode_id)
|
||||||
|
if episode_id not in active_ids:
|
||||||
|
continue
|
||||||
|
if episode_id in metadata:
|
||||||
|
raise ValueError(f"Duplicate episode metadata for episode {episode_id}.")
|
||||||
|
label = normalize_episode_success_label(meta[success_field][i])
|
||||||
|
if label != EPISODE_SUCCESS:
|
||||||
|
raise ValueError(
|
||||||
|
f"reward.mode={reward.mode} requires only explicitly successful episodes; "
|
||||||
|
f"episode {episode_id} has label {label!r}."
|
||||||
|
)
|
||||||
|
tasks = meta["tasks"][i]
|
||||||
|
if not isinstance(tasks, str):
|
||||||
|
if len(tasks) != 1:
|
||||||
|
raise ValueError(f"Episode {episode_id} must have exactly one task for reward normalization.")
|
||||||
|
tasks = tasks[0]
|
||||||
|
metadata[episode_id] = (int(meta["length"][i]), str(tasks))
|
||||||
|
if active_ids != set(metadata):
|
||||||
|
raise ValueError("Missing episode metadata for reward construction.")
|
||||||
|
|
||||||
|
interventions = None
|
||||||
|
if reward.mode == "time_takeover":
|
||||||
|
if reward.intervention_field not in frames.column_names:
|
||||||
|
raise ValueError(f"Missing intervention field '{reward.intervention_field}' for time_takeover.")
|
||||||
|
interventions = np.asarray(frames[reward.intervention_field]).reshape(-1)
|
||||||
|
if interventions.shape != episode_ids.shape or not np.isin(interventions, [0, 1]).all():
|
||||||
|
raise ValueError("Intervention flags must contain one boolean or binary 0/1 value per frame.")
|
||||||
|
interventions = interventions.astype(bool)
|
||||||
|
|
||||||
|
returns = np.empty(len(episode_ids), dtype=np.float64)
|
||||||
|
fitted_scales: dict[str, float] = {}
|
||||||
|
positions_by_task: dict[str, list[np.ndarray]] = {}
|
||||||
|
for episode_id in sorted(active_ids):
|
||||||
|
positions = np.flatnonzero(episode_ids == episode_id)
|
||||||
|
length, task = metadata[episode_id]
|
||||||
|
if (
|
||||||
|
length <= 0
|
||||||
|
or not np.array_equal(frame_ids[positions], np.arange(length))
|
||||||
|
or not np.all(np.diff(positions) == 1)
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
f"Episode {episode_id} must contain every frame exactly once, contiguously from 0 to length-1."
|
||||||
|
)
|
||||||
|
rewards = np.full(length, -1.0, dtype=np.float64)
|
||||||
|
rewards[-1] = 0.0
|
||||||
|
if interventions is not None and length > 2:
|
||||||
|
human = interventions[positions]
|
||||||
|
# Penalize the preceding autonomous frame. Entry into the terminal
|
||||||
|
# observation and initial human control are not takeover events.
|
||||||
|
takeovers = (~human[:-2]) & human[1:-1]
|
||||||
|
rewards[:-2] -= reward.takeover_cost * takeovers
|
||||||
|
episode_returns = np.cumsum(rewards[::-1])[::-1]
|
||||||
|
returns[positions] = episode_returns
|
||||||
|
# A one-frame successful episode has zero cost; use scale 1 to avoid 0/0.
|
||||||
|
fitted_scales[task] = max(fitted_scales.get(task, 1.0), float(-episode_returns[0]))
|
||||||
|
positions_by_task.setdefault(task, []).append(positions)
|
||||||
|
|
||||||
|
scales = dict(fitted_scales if task_scales is None else task_scales)
|
||||||
|
targets = np.empty(len(returns), dtype=np.float32)
|
||||||
|
for task, chunks in positions_by_task.items():
|
||||||
|
scale = scales.get(task)
|
||||||
|
if scale is None or not math.isfinite(scale) or scale <= 0:
|
||||||
|
raise ValueError(f"Missing or invalid saved reward scale for task {task!r}.")
|
||||||
|
positions = np.concatenate(chunks)
|
||||||
|
normalized = returns[positions] / scale
|
||||||
|
if np.any(normalized < -1.0) or np.any(normalized > 0.0):
|
||||||
|
raise ValueError(
|
||||||
|
f"Returns for task {task!r} exceed the saved reward scale; refusing to clip or refit."
|
||||||
|
)
|
||||||
|
targets[positions] = normalized
|
||||||
|
return targets, scales
|
||||||
在新工单中引用
屏蔽一个用户