snapshot: sync mint main from current workspace 2026-03-03 16:32:56 +0800

这个提交包含在:
Elvin
2026-03-03 16:32:56 +08:00
当前提交 908eb4bbbc
共有 677 个文件被更改,包括 167208 次插入 和 0 次删除

查看文件

@ -0,0 +1,6 @@
#!/usr/bin/env python
from .configuration_pistar06 import Pistar06Config
from .modeling_pistar06 import Pistar06Policy
__all__ = ["Pistar06Config", "Pistar06Policy"]

查看文件

@ -0,0 +1,159 @@
#!/usr/bin/env python
from dataclasses import dataclass, field
from lerobot.configs.policies import PreTrainedConfig
from lerobot.configs.types import NormalizationMode
from lerobot.optim.optimizers import AdamWConfig
from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
from lerobot.utils.constants import OBS_STATE
@PreTrainedConfig.register_subclass("pistar06")
@dataclass
class Pistar06Config(PreTrainedConfig):
"""Value model config using the Pistar06 stack (SigLIP + Gemma)."""
# Backbone components
vision_repo_id: str = "google/siglip-so400m-patch14-384"
language_repo_id: str = "google/gemma-3-270m"
vision_revision: str | None = None
language_revision: str | None = None
# Input fields
task_field: str = "task"
camera_features: list[str] = field(default_factory=list)
state_feature: str = OBS_STATE
include_state_in_prompt: bool = True
max_state_dim: int = 32
state_discretization_bins: int = 256
target_key: str = "observation.value_target"
loss_weight_key: str = "observation.value_loss_weight"
task_index_feature: str = "task_index"
# Tokenizer / model shape
tokenizer_max_length: int = 200
state_proj_dim: int = 512
fusion_hidden_dim: int = 512
fusion_num_layers: int = 2
fusion_num_heads: int = 8
# Value head (distributional)
num_bins: int = 201
bin_min: float = -1.0
bin_max: float = 0.0
# Runtime
dropout: float = 0.1
dtype: str = "float32"
freeze_vision_encoder: bool = False
freeze_language_model: bool = False
use_gradient_checkpointing: bool = False
push_to_hub: bool = False
# Training presets
optimizer_lr: float = 5e-5
optimizer_weight_decay: float = 1e-5
optimizer_grad_clip_norm: float = 10.0
scheduler_warmup_steps: int = 500
scheduler_decay_steps: int = 8_000
scheduler_decay_lr: float = 1e-6
normalization_mapping: dict[str, NormalizationMode] = field(
default_factory=lambda: {
"VISUAL": NormalizationMode.IDENTITY,
"STATE": NormalizationMode.QUANTILES,
"ACTION": NormalizationMode.IDENTITY,
}
)
def __post_init__(self) -> None:
super().__post_init__()
if not self.vision_repo_id:
raise ValueError("'value.vision_repo_id' must be non-empty.")
if not self.language_repo_id:
raise ValueError("'value.language_repo_id' must be non-empty.")
if not self.task_field:
raise ValueError("'value.task_field' must be non-empty.")
if not self.state_feature:
raise ValueError("'value.state_feature' must be non-empty.")
if not self.state_feature.startswith("observation."):
raise ValueError("'value.state_feature' must start with 'observation.'.")
if not self.target_key:
raise ValueError("'value.target_key' must be non-empty.")
if not self.loss_weight_key:
raise ValueError("'value.loss_weight_key' must be non-empty.")
if not self.loss_weight_key.startswith("observation."):
raise ValueError("'value.loss_weight_key' must start with 'observation.'.")
if self.max_state_dim <= 0:
raise ValueError("'value.max_state_dim' must be > 0.")
if self.state_discretization_bins < 2:
raise ValueError("'value.state_discretization_bins' must be >= 2.")
if self.tokenizer_max_length <= 0:
raise ValueError("'value.tokenizer_max_length' must be > 0.")
if self.state_proj_dim <= 0:
raise ValueError("'value.state_proj_dim' must be > 0.")
if self.fusion_hidden_dim <= 0:
raise ValueError("'value.fusion_hidden_dim' must be > 0.")
if self.fusion_num_layers <= 0:
raise ValueError("'value.fusion_num_layers' must be > 0.")
if self.fusion_num_heads <= 0:
raise ValueError("'value.fusion_num_heads' must be > 0.")
if self.fusion_hidden_dim % self.fusion_num_heads != 0:
raise ValueError("'value.fusion_hidden_dim' must be divisible by 'value.fusion_num_heads'.")
if self.num_bins < 2:
raise ValueError("'value.num_bins' must be >= 2.")
if self.bin_min >= self.bin_max:
raise ValueError("'value.bin_min' must be < 'value.bin_max'.")
if self.dtype not in {"float32", "bfloat16"}:
raise ValueError("'value.dtype' must be one of {'float32', 'bfloat16'}.")
if not 0.0 <= self.dropout < 1.0:
raise ValueError("'value.dropout' must be within [0, 1).")
if self.optimizer_lr <= 0:
raise ValueError("'value.optimizer_lr' must be > 0.")
if self.optimizer_weight_decay < 0:
raise ValueError("'value.optimizer_weight_decay' must be >= 0.")
if self.optimizer_grad_clip_norm < 0:
raise ValueError("'value.optimizer_grad_clip_norm' must be >= 0.")
if self.scheduler_warmup_steps < 0:
raise ValueError("'value.scheduler_warmup_steps' must be >= 0.")
if self.scheduler_decay_steps <= 0:
raise ValueError("'value.scheduler_decay_steps' must be > 0.")
if self.scheduler_decay_lr < 0:
raise ValueError("'value.scheduler_decay_lr' must be >= 0.")
def validate_features(self) -> None:
# Value model consumes observation + task text, and supervises with a scalar target key.
# The training loop injects target tensors into `target_key`.
return
def get_optimizer_preset(self) -> AdamWConfig:
return AdamWConfig(
lr=self.optimizer_lr,
weight_decay=self.optimizer_weight_decay,
grad_clip_norm=self.optimizer_grad_clip_norm,
)
def get_scheduler_preset(self):
return CosineDecayWithWarmupSchedulerConfig(
peak_lr=self.optimizer_lr,
decay_lr=self.scheduler_decay_lr,
num_warmup_steps=self.scheduler_warmup_steps,
num_decay_steps=self.scheduler_decay_steps,
)
@property
def observation_delta_indices(self) -> None:
return None
@property
def action_delta_indices(self) -> None:
return None
@property
def reward_delta_indices(self) -> None:
return None

查看文件

@ -0,0 +1,785 @@
#!/usr/bin/env python
from __future__ import annotations
import json
import logging
import os
from contextlib import nullcontext
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any
import numpy as np
import torch
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
from huggingface_hub.constants import SAFETENSORS_SINGLE_FILE
from huggingface_hub.errors import HfHubHTTPError
from safetensors.torch import save_file
from torch import Tensor, nn
from lerobot.policies.pretrained import ActionSelectKwargs, PreTrainedPolicy
from lerobot.utils.constants import OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS
from lerobot.utils.import_utils import _transformers_available
from lerobot.utils.recording_annotations import EPISODE_SUCCESS, resolve_episode_success_label
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
from lerobot.values.pistar06.processor_pistar06 import PISTAR06_IMAGE_MASK_KEY, PISTAR06_IMAGES_KEY
if TYPE_CHECKING or _transformers_available:
from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoModelForCausalLM
else:
AutoConfig = None
AutoImageProcessor = None
AutoModel = None
AutoModelForCausalLM = None
PISTAR06_SAVE_INFO = "pistar06_save_info.json"
@dataclass
class EpisodeTargetInfo:
episode_index: int
task_index: int
length: int
success: bool
def build_bin_centers(
num_bins: int,
bin_min: float,
bin_max: float,
device: torch.device | None = None,
) -> torch.Tensor:
return torch.linspace(bin_min, bin_max, num_bins, dtype=torch.float32, device=device)
def project_values_to_bins(values: torch.Tensor, bin_centers: torch.Tensor) -> torch.Tensor:
if values.ndim != 1:
raise ValueError(f"'values' must be rank-1, got shape={tuple(values.shape)}.")
if bin_centers.ndim != 1:
raise ValueError(f"'bin_centers' must be rank-1, got shape={tuple(bin_centers.shape)}.")
if bin_centers.shape[0] < 2:
raise ValueError("At least 2 bins are required.")
values = values.clamp(min=bin_centers[0], max=bin_centers[-1])
step = bin_centers[1] - bin_centers[0]
scaled = (values - bin_centers[0]) / step
low = torch.floor(scaled).long()
high = torch.clamp(low + 1, max=bin_centers.shape[0] - 1)
high_weight = (scaled - low.float()).clamp(0.0, 1.0)
low_weight = 1.0 - high_weight
target = torch.zeros(values.shape[0], bin_centers.shape[0], device=values.device, dtype=torch.float32)
target.scatter_add_(1, low.unsqueeze(1), low_weight.unsqueeze(1))
target.scatter_add_(1, high.unsqueeze(1), high_weight.unsqueeze(1))
return target
def expected_value_from_logits(logits: torch.Tensor, bin_centers: torch.Tensor) -> torch.Tensor:
probs = F.softmax(logits, dim=-1)
return (probs * bin_centers).sum(dim=-1)
def compute_normalized_value_targets(
episode_indices: np.ndarray,
frame_indices: np.ndarray,
episode_info: dict[int, EpisodeTargetInfo],
task_max_lengths: dict[int, int],
c_fail_coef: float,
*,
clip_min: float = -1.0,
clip_max: float = 0.0,
) -> np.ndarray:
if episode_indices.shape != frame_indices.shape:
raise ValueError("episode_indices and frame_indices must have the same shape.")
if c_fail_coef < 0:
raise ValueError("'c_fail_coef' must be non-negative.")
targets = np.zeros(episode_indices.shape[0], dtype=np.float32)
for i in range(episode_indices.shape[0]):
ep_idx = int(episode_indices[i])
if ep_idx not in episode_info:
raise KeyError(f"Missing episode metadata for episode_index={ep_idx}.")
ep = episode_info[ep_idx]
task_max = task_max_lengths.get(ep.task_index)
if task_max is None:
raise KeyError(f"Missing task max length for task_index={ep.task_index}.")
if task_max <= 0:
raise ValueError(f"Invalid task max length {task_max} for task_index={ep.task_index}.")
remaining_steps = ep.length - int(frame_indices[i]) - 1
c_fail = float(task_max) * c_fail_coef
g = -float(remaining_steps)
if not ep.success:
g -= c_fail
denom = float(task_max) + c_fail
g_norm = g / denom
targets[i] = np.clip(g_norm, clip_min, clip_max)
return targets
def _resolve_load_dtype(dtype_name: str) -> torch.dtype:
requested_dtype = torch.bfloat16 if dtype_name == "bfloat16" else torch.float32
if requested_dtype == torch.bfloat16 and not torch.cuda.is_available():
return torch.float32
return requested_dtype
def _freeze_module(module: nn.Module) -> None:
module.eval()
for parameter in module.parameters():
parameter.requires_grad = False
def _maybe_enable_gradient_checkpointing(module: nn.Module) -> None:
if hasattr(module, "gradient_checkpointing_enable"):
module.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
elif hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = True
def _extract_hidden_size(model: nn.Module) -> int:
config = getattr(model, "config", None)
if config is None:
raise ValueError(f"Cannot infer hidden size for model type {type(model)}: missing `.config`.")
if hasattr(config, "hidden_size"):
return int(config.hidden_size)
if hasattr(config, "text_config") and hasattr(config.text_config, "hidden_size"):
return int(config.text_config.hidden_size)
raise ValueError(f"Cannot infer hidden size for model config type {type(config)}.")
def _extract_vision_feature_size(model: nn.Module) -> int:
config = getattr(model, "config", None)
if config is None:
raise ValueError(f"Cannot infer vision feature size for model type {type(model)}: missing `.config`.")
if hasattr(config, "projection_dim"):
return int(config.projection_dim)
if hasattr(config, "vision_config") and hasattr(config.vision_config, "projection_dim"):
return int(config.vision_config.projection_dim)
if hasattr(config, "hidden_size"):
return int(config.hidden_size)
if hasattr(config, "vision_config") and hasattr(config.vision_config, "hidden_size"):
return int(config.vision_config.hidden_size)
raise ValueError(f"Cannot infer vision feature size for model config type {type(config)}.")
def _validate_loading_info(repo_id: str, model_label: str, loading_info: dict[str, list] | None) -> None:
if loading_info is None:
return
missing = loading_info.get("missing_keys", [])
unexpected = loading_info.get("unexpected_keys", [])
mismatched = loading_info.get("mismatched_keys", [])
if not missing and not unexpected and not mismatched:
return
raise RuntimeError(
f"Pretrained weights for {model_label} from '{repo_id}' did not load cleanly: "
f"missing={len(missing)} unexpected={len(unexpected)} mismatched={len(mismatched)}. "
"This usually indicates a model class/checkpoint mismatch."
)
def _resolve_image_size(image_processor: Any) -> tuple[int, int]:
size = getattr(image_processor, "size", None)
if isinstance(size, dict):
if "height" in size and "width" in size:
return int(size["height"]), int(size["width"])
if "shortest_edge" in size:
edge = int(size["shortest_edge"])
return edge, edge
if isinstance(size, int):
return int(size), int(size)
return 384, 384
def _resolve_norm_stats(image_processor: Any) -> tuple[tuple[float, float, float], tuple[float, float, float]]:
mean_raw = getattr(image_processor, "image_mean", [0.5, 0.5, 0.5])
std_raw = getattr(image_processor, "image_std", [0.5, 0.5, 0.5])
if len(mean_raw) != 3 or len(std_raw) != 3:
raise ValueError(
f"Expected RGB normalization stats of len=3, got mean={mean_raw} std={std_raw}."
)
mean = (float(mean_raw[0]), float(mean_raw[1]), float(mean_raw[2]))
std = (float(std_raw[0]), float(std_raw[1]), float(std_raw[2]))
if any(v <= 0 for v in std):
raise ValueError(f"Invalid image std values: {std}.")
return mean, std
def _load_language_model(
repo_id: str,
revision: str | None,
dtype: torch.dtype,
) -> nn.Module:
if AutoConfig is None or AutoModelForCausalLM is None or AutoModel is None:
raise ImportError("transformers is not installed. Install with `pip install 'lerobot[pi0]'`.")
model_config = AutoConfig.from_pretrained(repo_id, revision=revision)
architectures = getattr(model_config, "architectures", None) or []
prefer_causal_lm = any(isinstance(arch, str) and arch.endswith("ForCausalLM") for arch in architectures)
if prefer_causal_lm:
lm_with_head, loading_info = AutoModelForCausalLM.from_pretrained(
repo_id,
revision=revision,
torch_dtype=dtype,
output_loading_info=True,
)
_validate_loading_info(repo_id, "language_model(causal_lm)", loading_info)
if not hasattr(lm_with_head, "model"):
raise RuntimeError(
f"AutoModelForCausalLM loaded from '{repo_id}' does not expose `.model` text backbone."
)
return lm_with_head.model
language_model, loading_info = AutoModel.from_pretrained(
repo_id,
revision=revision,
torch_dtype=dtype,
output_loading_info=True,
)
_validate_loading_info(repo_id, "language_model(auto_model)", loading_info)
if not isinstance(language_model, nn.Module):
raise TypeError(
f"AutoModel loaded from '{repo_id}' returned unexpected type: {type(language_model)}."
)
return language_model
class Pistar06Model(nn.Module):
def __init__(self, cfg: Pistar06Config):
super().__init__()
if AutoModel is None or AutoImageProcessor is None:
raise ImportError("transformers is not installed. Install with `pip install 'lerobot[pi0]'`.")
self.cfg = cfg
self.model_dtype = _resolve_load_dtype(cfg.dtype)
self.vision_encoder = AutoModel.from_pretrained(
cfg.vision_repo_id,
revision=cfg.vision_revision,
torch_dtype=self.model_dtype,
)
self.language_model = _load_language_model(
repo_id=cfg.language_repo_id,
revision=cfg.language_revision,
dtype=self.model_dtype,
)
image_processor = AutoImageProcessor.from_pretrained(
cfg.vision_repo_id,
revision=cfg.vision_revision,
use_fast=True,
)
image_height, image_width = _resolve_image_size(image_processor)
image_mean, image_std = _resolve_norm_stats(image_processor)
self.image_resolution = (image_height, image_width)
self.register_buffer(
"image_mean",
torch.tensor(image_mean, dtype=torch.float32).view(1, 1, 3, 1, 1),
persistent=False,
)
self.register_buffer(
"image_std",
torch.tensor(image_std, dtype=torch.float32).view(1, 1, 3, 1, 1),
persistent=False,
)
vision_feature_size = _extract_vision_feature_size(self.vision_encoder)
language_hidden_size = _extract_hidden_size(self.language_model)
self.image_projector = nn.Sequential(
nn.Linear(vision_feature_size, cfg.fusion_hidden_dim),
nn.GELU(),
nn.Dropout(cfg.dropout),
)
self.language_projector = nn.Sequential(
nn.Linear(language_hidden_size, cfg.fusion_hidden_dim),
nn.GELU(),
nn.Dropout(cfg.dropout),
)
self.final_norm = nn.LayerNorm(cfg.fusion_hidden_dim * 2)
self.value_head = nn.Sequential(
nn.Linear(cfg.fusion_hidden_dim * 2, cfg.fusion_hidden_dim),
nn.GELU(),
nn.Dropout(cfg.dropout),
nn.Linear(cfg.fusion_hidden_dim, cfg.num_bins),
)
if cfg.use_gradient_checkpointing:
_maybe_enable_gradient_checkpointing(self.language_model)
_maybe_enable_gradient_checkpointing(self.vision_encoder)
if cfg.freeze_language_model:
_freeze_module(self.language_model)
if cfg.freeze_vision_encoder:
_freeze_module(self.vision_encoder)
def _encode_images(self, flat_images: Tensor) -> Tensor:
if hasattr(self.vision_encoder, "get_image_features"):
return self.vision_encoder.get_image_features(pixel_values=flat_images)
vision_outputs = self.vision_encoder(pixel_values=flat_images, return_dict=True)
if hasattr(vision_outputs, "pooler_output") and vision_outputs.pooler_output is not None:
return vision_outputs.pooler_output
if hasattr(vision_outputs, "last_hidden_state"):
return vision_outputs.last_hidden_state.mean(dim=1)
raise ValueError("Unsupported vision encoder output. Expected pooler_output or last_hidden_state.")
def _encode_language(self, input_ids: Tensor, attention_mask: Tensor) -> Tensor:
outputs = self.language_model(
input_ids=input_ids,
attention_mask=attention_mask,
return_dict=True,
)
hidden = getattr(outputs, "last_hidden_state", None)
if hidden is None:
raise ValueError("Language model output does not contain `last_hidden_state`.")
token_mask = attention_mask.to(dtype=hidden.dtype).unsqueeze(-1)
denom = token_mask.sum(dim=1).clamp_min(1.0)
return (hidden * token_mask).sum(dim=1) / denom
def _preprocess_images(self, images: Tensor, image_attention_mask: Tensor) -> Tensor:
if images.ndim != 5:
raise ValueError(f"'images' must have shape [B,N,C,H,W], got {tuple(images.shape)}.")
if image_attention_mask.ndim != 2:
raise ValueError(
f"'image_attention_mask' must have shape [B,N], got {tuple(image_attention_mask.shape)}."
)
bsize, num_cameras = images.shape[:2]
if image_attention_mask.shape[0] != bsize or image_attention_mask.shape[1] != num_cameras:
raise ValueError("Batch shape mismatch between images and image_attention_mask.")
if images.dtype == torch.uint8:
images = images.to(dtype=torch.float32) / 255.0
else:
images = images.to(dtype=torch.float32)
if bool(torch.max(images) > 1.0) or bool(torch.min(images) < 0.0):
images = (images / 255.0).clamp(0.0, 1.0)
flat_images = images.view(bsize * num_cameras, *images.shape[2:])
if flat_images.shape[-2:] != self.image_resolution:
flat_images = F.interpolate(
flat_images,
size=self.image_resolution,
mode="bilinear",
align_corners=False,
)
mean = self.image_mean.to(device=flat_images.device, dtype=flat_images.dtype).view(1, 3, 1, 1)
std = self.image_std.to(device=flat_images.device, dtype=flat_images.dtype).view(1, 3, 1, 1)
flat_images = (flat_images - mean) / std
flat_images = flat_images.view(
bsize,
num_cameras,
flat_images.shape[1],
flat_images.shape[2],
flat_images.shape[3],
)
camera_mask = image_attention_mask.to(device=flat_images.device, dtype=flat_images.dtype).view(
bsize, num_cameras, 1, 1, 1
)
flat_images = flat_images * camera_mask
return flat_images
def forward(
self,
input_ids: Tensor,
attention_mask: Tensor,
images: Tensor,
image_attention_mask: Tensor,
) -> Tensor:
if input_ids.ndim != 2:
raise ValueError(f"'input_ids' must have shape [B, T], got {tuple(input_ids.shape)}.")
if attention_mask.ndim != 2:
raise ValueError(f"'attention_mask' must have shape [B, T], got {tuple(attention_mask.shape)}.")
if images.ndim != 5:
raise ValueError(f"'images' must have shape [B, N, C, H, W], got {tuple(images.shape)}.")
if image_attention_mask.ndim != 2:
raise ValueError(
f"'image_attention_mask' must have shape [B, N], got {tuple(image_attention_mask.shape)}."
)
bsize = input_ids.shape[0]
if attention_mask.shape[0] != bsize:
raise ValueError("Batch size mismatch between input_ids and attention_mask.")
if images.shape[0] != bsize or image_attention_mask.shape[0] != bsize:
raise ValueError("Batch size mismatch between language and image inputs.")
if images.shape[1] == 0:
raise ValueError("At least one camera is required for Pistar06Model.")
image_attention_mask = image_attention_mask.to(dtype=torch.bool, device=images.device)
if not torch.all(image_attention_mask.any(dim=1)):
raise ValueError("Each sample must have at least one valid camera input.")
language_mask = attention_mask.to(dtype=torch.bool, device=input_ids.device)
if not torch.all(language_mask.any(dim=1)):
raise ValueError("Each sample must have at least one valid language token.")
processed_images = self._preprocess_images(images, image_attention_mask)
num_cameras = processed_images.shape[1]
flat_images = processed_images.reshape(bsize * num_cameras, *processed_images.shape[2:])
flat_images = flat_images.to(dtype=self.model_dtype)
image_context = torch.no_grad() if self.cfg.freeze_vision_encoder else nullcontext()
with image_context:
image_features = self._encode_images(flat_images)
language_context = torch.no_grad() if self.cfg.freeze_language_model else nullcontext()
with language_context:
language_features = self._encode_language(input_ids=input_ids, attention_mask=language_mask.long())
feature_dtype = torch.float32
image_features = image_features.to(dtype=feature_dtype)
language_features = language_features.to(dtype=feature_dtype)
image_tokens = self.image_projector(image_features).view(bsize, num_cameras, -1)
camera_token_mask = image_attention_mask.unsqueeze(-1).to(dtype=image_tokens.dtype)
image_tokens = image_tokens * camera_token_mask
camera_denominator = image_attention_mask.sum(dim=1, keepdim=True).to(dtype=image_tokens.dtype).clamp_min(1.0)
image_pooled = image_tokens.sum(dim=1) / camera_denominator
language_token = self.language_projector(language_features)
joint_features = torch.cat([image_pooled, language_token], dim=-1)
return self.value_head(self.final_norm(joint_features))
class Pistar06Policy(PreTrainedPolicy):
config_class = Pistar06Config
name = "pistar06"
def __init__(
self,
config: Pistar06Config,
dataset_meta=None,
**kwargs: Any,
):
del dataset_meta, kwargs
super().__init__(config)
self.config = config
self.model = Pistar06Model(config)
self.register_buffer(
"bin_centers",
build_bin_centers(config.num_bins, config.bin_min, config.bin_max),
persistent=False,
)
def _frozen_checkpoint_prefixes(self) -> list[str]:
prefixes: list[str] = []
if self.config.freeze_vision_encoder:
prefixes.append("model.vision_encoder.")
if self.config.freeze_language_model:
prefixes.append("model.language_model.")
return prefixes
def _save_pretrained(self, save_directory: Path) -> None:
self.config._save_pretrained(save_directory)
model_to_save = self.module if hasattr(self, "module") else self
state_dict = model_to_save.state_dict()
excluded_prefixes = self._frozen_checkpoint_prefixes()
if excluded_prefixes:
state_dict = {
key: tensor
for key, tensor in state_dict.items()
if not any(key.startswith(prefix) for prefix in excluded_prefixes)
}
save_file(state_dict, str(save_directory / SAFETENSORS_SINGLE_FILE))
save_info = {
"format_version": 1,
"weights_mode": "partial" if excluded_prefixes else "full",
"freeze_vision_encoder": bool(self.config.freeze_vision_encoder),
"freeze_language_model": bool(self.config.freeze_language_model),
"excluded_prefixes": excluded_prefixes,
"saved_tensor_count": len(state_dict),
}
with open(save_directory / PISTAR06_SAVE_INFO, "w", encoding="utf-8") as f:
json.dump(save_info, f, indent=2, sort_keys=True)
@classmethod
def _load_save_info(
cls,
pretrained_name_or_path: str | Path,
*,
force_download: bool = False,
resume_download: bool | None = None,
proxies: dict | None = None,
token: str | bool | None = None,
cache_dir: str | Path | None = None,
local_files_only: bool = False,
revision: str | None = None,
) -> dict[str, Any] | None:
del cls
model_id = str(pretrained_name_or_path)
save_info_path: Path | None = None
if os.path.isdir(model_id):
candidate = Path(model_id) / PISTAR06_SAVE_INFO
if candidate.is_file():
save_info_path = candidate
else:
try:
resolved = hf_hub_download(
repo_id=model_id,
filename=PISTAR06_SAVE_INFO,
revision=revision,
cache_dir=cache_dir,
force_download=force_download,
proxies=proxies,
resume_download=resume_download,
token=token,
local_files_only=local_files_only,
)
except (HfHubHTTPError, FileNotFoundError):
resolved = None
if resolved is not None:
save_info_path = Path(resolved)
if save_info_path is None:
return None
try:
with open(save_info_path, encoding="utf-8") as f:
parsed = json.load(f)
except (json.JSONDecodeError, OSError):
return None
return parsed if isinstance(parsed, dict) else None
@classmethod
def from_pretrained(
cls,
pretrained_name_or_path: str | Path,
*,
config: Pistar06Config | None = None,
force_download: bool = False,
resume_download: bool | None = None,
proxies: dict | None = None,
token: str | bool | None = None,
cache_dir: str | Path | None = None,
local_files_only: bool = False,
revision: str | None = None,
strict: bool = False,
**kwargs: Any,
) -> Pistar06Policy:
save_info = cls._load_save_info(
pretrained_name_or_path=pretrained_name_or_path,
force_download=force_download,
resume_download=resume_download,
proxies=proxies,
token=token,
cache_dir=cache_dir,
local_files_only=local_files_only,
revision=revision,
)
effective_strict = strict
if save_info is not None and save_info.get("weights_mode") == "partial":
if strict:
logging.info(
"Detected partial Pistar06 checkpoint at '%s'; "
"forcing strict=False for automatic fallback.",
pretrained_name_or_path,
)
effective_strict = False
return super().from_pretrained(
pretrained_name_or_path=pretrained_name_or_path,
config=config,
force_download=force_download,
resume_download=resume_download,
proxies=proxies,
token=token,
cache_dir=cache_dir,
local_files_only=local_files_only,
revision=revision,
strict=effective_strict,
**kwargs,
)
def get_optim_params(self):
return self.parameters()
def reset(self):
return
def predict_action_chunk(self, batch: dict[str, Tensor], **kwargs: ActionSelectKwargs) -> Tensor:
raise RuntimeError("Pistar06Policy is a value model and does not support action prediction.")
def select_action(self, batch: dict[str, Tensor], **kwargs: ActionSelectKwargs) -> Tensor:
raise RuntimeError("Pistar06Policy is a value model and does not support action selection.")
def predict_value(self, batch: dict[str, Tensor]) -> Tensor:
input_ids = batch[OBS_LANGUAGE_TOKENS]
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
images = batch[PISTAR06_IMAGES_KEY]
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
logits = self.model(
input_ids=input_ids,
attention_mask=attention_mask,
images=images,
image_attention_mask=image_attention_mask,
)
bin_centers = self.bin_centers.to(device=logits.device)
return expected_value_from_logits(logits, bin_centers)
def build_training_raw_batch_hook(self, dataset, targets_cfg):
raw_frames = dataset.hf_dataset.with_format(None)
frame_count = len(raw_frames)
if frame_count == 0:
raise ValueError("Dataset has no frames.")
episode_indices = np.asarray(raw_frames["episode_index"], dtype=np.int64)
frame_indices = np.asarray(raw_frames["frame_index"], dtype=np.int64)
absolute_indices = np.asarray(raw_frames["index"], dtype=np.int64)
episodes_ds = dataset.meta.episodes.with_format(None)
episodes = episodes_ds[:]
n_episodes = len(episodes_ds)
has_success = targets_cfg.success_field in episodes_ds.column_names
episode_info: dict[int, EpisodeTargetInfo] = {}
task_max_length: dict[int, int] = {}
for i in range(n_episodes):
ep_idx = int(episodes["episode_index"][i])
ep_length = int(episodes["length"][i])
tasks = episodes["tasks"][i]
task_name = tasks[0] if isinstance(tasks, list) else tasks
if task_name not in dataset.meta.tasks.index:
raise KeyError(f"Episode {ep_idx} references unknown task '{task_name}'.")
task_index = int(dataset.meta.tasks.loc[task_name].task_index)
explicit_success = episodes[targets_cfg.success_field][i] if has_success else None
resolved_success = resolve_episode_success_label(
explicit_success,
default_label=targets_cfg.default_success,
require_label=True,
)
ep_success = resolved_success == EPISODE_SUCCESS
episode_info[ep_idx] = EpisodeTargetInfo(
episode_index=ep_idx,
task_index=task_index,
length=ep_length,
success=ep_success,
)
task_max_length[task_index] = max(task_max_length.get(task_index, 0), ep_length)
value_targets = compute_normalized_value_targets(
episode_indices=episode_indices,
frame_indices=frame_indices,
episode_info=episode_info,
task_max_lengths=task_max_length,
c_fail_coef=targets_cfg.c_fail_coef,
clip_min=self.config.bin_min,
clip_max=self.config.bin_max,
)
max_index = int(np.max(absolute_indices))
value_target_lookup = np.zeros(max_index + 1, dtype=np.float32)
value_target_lookup[absolute_indices] = value_targets.astype(np.float32, copy=False)
target_key = targets_cfg.target_field
def value_target_hook(batch: dict[str, Any], step: int) -> dict[str, Any]:
del step
batch_indices = batch.get("index")
if batch_indices is None:
raise KeyError("Missing 'index' in batch while building value targets.")
if not isinstance(batch_indices, torch.Tensor):
batch_indices = torch.as_tensor(batch_indices)
batch_indices_np = batch_indices.detach().cpu().numpy().astype(np.int64, copy=False).reshape(-1)
target_values = torch.from_numpy(value_target_lookup[batch_indices_np]).to(dtype=torch.float32)
batch[target_key] = target_values
return batch
return value_target_hook
def forward(self, batch: dict[str, Tensor], reduction: str = "mean") -> tuple[Tensor, dict]:
if self.config.target_key not in batch:
raise KeyError(
f"Missing target key '{self.config.target_key}' in batch. "
"Make sure lerobot-value-train target hook is enabled."
)
input_ids = batch[OBS_LANGUAGE_TOKENS]
attention_mask = batch[OBS_LANGUAGE_ATTENTION_MASK]
images = batch[PISTAR06_IMAGES_KEY]
image_attention_mask = batch[PISTAR06_IMAGE_MASK_KEY]
device = next(self.model.parameters()).device
value_target = batch[self.config.target_key]
if not isinstance(value_target, Tensor):
value_target = torch.as_tensor(value_target)
value_target = value_target.to(device=device, dtype=torch.float32, non_blocking=True)
if value_target.ndim == 2 and value_target.shape[-1] == 1:
value_target = value_target.squeeze(-1)
if value_target.ndim != 1:
raise ValueError(
f"Value target must be rank-1 or [B,1], got shape={tuple(value_target.shape)} "
f"for key '{self.config.target_key}'."
)
logits = self.model(
input_ids=input_ids,
attention_mask=attention_mask,
images=images,
image_attention_mask=image_attention_mask,
)
bin_centers = self.bin_centers.to(device=device)
soft_target = project_values_to_bins(value_target, bin_centers)
log_probs = F.log_softmax(logits, dim=-1)
per_sample_loss = -(soft_target * log_probs).sum(dim=-1)
sample_weight = None
if self.config.loss_weight_key in batch:
sample_weight = batch[self.config.loss_weight_key]
if not isinstance(sample_weight, Tensor):
sample_weight = torch.as_tensor(sample_weight)
sample_weight = sample_weight.to(device=device, dtype=torch.float32, non_blocking=True)
if sample_weight.ndim == 2 and sample_weight.shape[-1] == 1:
sample_weight = sample_weight.squeeze(-1)
if sample_weight.ndim != 1:
raise ValueError(
f"Loss weight must be rank-1 or [B,1], got shape={tuple(sample_weight.shape)} "
f"for key '{self.config.loss_weight_key}'."
)
if sample_weight.shape[0] != per_sample_loss.shape[0]:
raise ValueError(
f"Loss weight batch size mismatch: expected {per_sample_loss.shape[0]}, "
f"got {sample_weight.shape[0]} for key '{self.config.loss_weight_key}'."
)
per_sample_loss = per_sample_loss * sample_weight
pred_value = expected_value_from_logits(logits, bin_centers)
value_mae = (pred_value - value_target).abs().mean()
if reduction == "none":
loss = per_sample_loss
else:
loss = per_sample_loss.mean()
loss_dict = {
"loss": float(loss.mean().detach().item()) if reduction == "none" else float(loss.detach().item()),
"value_mae": float(value_mae.detach().item()),
}
if sample_weight is not None:
loss_dict["loss_weight_mean"] = float(sample_weight.mean().detach().item())
loss_dict["loss_weight_min"] = float(sample_weight.min().detach().item())
loss_dict["loss_weight_max"] = float(sample_weight.max().detach().item())
return loss, loss_dict

查看文件

@ -0,0 +1,263 @@
#!/usr/bin/env python
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor
from lerobot.configs.types import PipelineFeatureType, PolicyFeature
from lerobot.processor import (
DeviceProcessorStep,
NormalizerProcessorStep,
PolicyAction,
PolicyProcessorPipeline,
ProcessorStep,
ProcessorStepRegistry,
RenameObservationsProcessorStep,
TokenizerProcessorStep,
)
from lerobot.processor.converters import policy_action_to_transition, transition_to_policy_action
from lerobot.processor.core import EnvTransition, TransitionKey
from lerobot.utils.constants import (
OBS_IMAGES,
OBS_STATE,
POLICY_POSTPROCESSOR_DEFAULT_NAME,
POLICY_PREPROCESSOR_DEFAULT_NAME,
)
from lerobot.values.pistar06.configuration_pistar06 import Pistar06Config
PISTAR06_IMAGES_KEY = "observation.pistar06.images"
PISTAR06_IMAGE_MASK_KEY = "observation.pistar06.image_attention_mask"
def _pad_last_dim(vector: Tensor, new_dim: int) -> Tensor:
if vector.shape[-1] >= new_dim:
return vector
return F.pad(vector, (0, new_dim - vector.shape[-1]))
@ProcessorStepRegistry.register(name="pistar06_prepare_task_prompt")
@dataclass
class Pistar06PrepareTaskPromptProcessorStep(ProcessorStep):
task_key: str = "task"
include_state_in_prompt: bool = True
state_feature: str = OBS_STATE
max_state_dim: int = 32
state_discretization_bins: int = 256
def get_config(self) -> dict[str, Any]:
return {
"task_key": self.task_key,
"include_state_in_prompt": self.include_state_in_prompt,
"state_feature": self.state_feature,
"max_state_dim": self.max_state_dim,
"state_discretization_bins": self.state_discretization_bins,
}
@staticmethod
def _clean_prompt(task: str) -> str:
return str(task).strip().replace("_", " ").replace("\n", " ").strip()
def __call__(self, transition: EnvTransition) -> EnvTransition:
transition = transition.copy()
observation = dict(transition.get(TransitionKey.OBSERVATION) or {})
complementary_data = dict(transition.get(TransitionKey.COMPLEMENTARY_DATA) or {})
if self.task_key not in complementary_data:
raise KeyError(f"Missing task field '{self.task_key}' in complementary data.")
tasks_raw = complementary_data[self.task_key]
if isinstance(tasks_raw, str):
tasks = [tasks_raw]
elif isinstance(tasks_raw, Sequence) and all(isinstance(task, str) for task in tasks_raw):
tasks = list(tasks_raw)
else:
raise TypeError(
f"Expected task field '{self.task_key}' as sequence of strings, got {type(tasks_raw)}."
)
prompts: list[str] = []
if self.include_state_in_prompt:
if self.state_feature not in observation:
raise KeyError(
f"Missing state feature '{self.state_feature}' while include_state_in_prompt=True."
)
state = observation[self.state_feature]
if not isinstance(state, Tensor):
state = torch.as_tensor(state)
if state.ndim == 1:
state = state.unsqueeze(0)
if state.ndim != 2:
raise ValueError(
f"Expected state tensor with shape [B, D], got {tuple(state.shape)} "
f"for feature '{self.state_feature}'."
)
state = state.detach().to(dtype=torch.float32, device="cpu")
state = _pad_last_dim(state, self.max_state_dim)
state_np = state.numpy()
bins = np.linspace(-1.0, 1.0, self.state_discretization_bins + 1, dtype=np.float32)[:-1]
discretized_state = np.digitize(state_np, bins=bins) - 1
if discretized_state.shape[0] != len(tasks):
raise ValueError(
f"Task count ({len(tasks)}) does not match state batch size ({discretized_state.shape[0]})."
)
for i, task in enumerate(tasks):
cleaned_task = self._clean_prompt(task)
state_str = " ".join(map(str, discretized_state[i].tolist()))
prompts.append(f"Task: {cleaned_task}, State: {state_str}\nValue: ")
else:
prompts = [f"Task: {self._clean_prompt(task)}\nValue: " for task in tasks]
complementary_data[self.task_key] = prompts
transition[TransitionKey.COMPLEMENTARY_DATA] = complementary_data
transition[TransitionKey.OBSERVATION] = observation
return transition
def transform_features(
self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
return features
@ProcessorStepRegistry.register(name="pistar06_prepare_images")
@dataclass
class Pistar06PrepareImagesProcessorStep(ProcessorStep):
camera_features: list[str]
def get_config(self) -> dict[str, Any]:
return {
"camera_features": self.camera_features,
}
@staticmethod
def _to_bchw(img_batch: Tensor) -> Tensor:
if img_batch.ndim != 4:
raise ValueError(f"Expected image batch rank 4, got shape {tuple(img_batch.shape)}.")
if img_batch.shape[1] in {1, 3}: # [B,C,H,W]
return img_batch
if img_batch.shape[-1] in {1, 3}: # [B,H,W,C]
return img_batch.permute(0, 3, 1, 2)
raise ValueError(
"Camera tensor must be channels-first or channels-last. "
f"Got camera batch with shape={tuple(img_batch.shape)}."
)
def _process_camera_batch(self, img_batch: Tensor) -> Tensor:
return self._to_bchw(img_batch).detach().to(dtype=torch.float32)
def _prepare_images(self, observation: dict[str, Any]) -> tuple[Tensor, Tensor]:
present_img_keys = [key for key in self.camera_features if key in observation]
if len(present_img_keys) == 0:
raise ValueError(
"All configured cameras are missing in the input batch. "
f"expected={self.camera_features} batch_keys={list(observation.keys())}"
)
reference_img = self._process_camera_batch(torch.as_tensor(observation[present_img_keys[0]]))
bsize = reference_img.shape[0]
image_tensors: list[Tensor] = []
image_masks: list[Tensor] = []
for key in self.camera_features:
if key in observation:
img = self._process_camera_batch(torch.as_tensor(observation[key]))
if img.shape[0] != bsize:
raise ValueError(
f"Mismatched batch size across cameras. Camera '{key}' has {img.shape[0]}, expected {bsize}."
)
if img.shape[1:] != reference_img.shape[1:]:
raise ValueError(
"Camera tensors must share the same [C,H,W] shape before model preprocessing. "
f"Camera '{key}' has {tuple(img.shape[1:])}, expected {tuple(reference_img.shape[1:])}."
)
image_tensors.append(img)
image_masks.append(torch.ones(bsize, dtype=torch.bool))
else:
image_tensors.append(torch.zeros_like(reference_img))
image_masks.append(torch.zeros(bsize, dtype=torch.bool))
images = torch.stack(image_tensors, dim=1)
masks = torch.stack(image_masks, dim=1)
return images, masks
def __call__(self, transition: EnvTransition) -> EnvTransition:
transition = transition.copy()
observation = dict(transition.get(TransitionKey.OBSERVATION) or {})
images, image_attention_mask = self._prepare_images(observation)
observation[PISTAR06_IMAGES_KEY] = images.to(dtype=torch.float32)
observation[PISTAR06_IMAGE_MASK_KEY] = image_attention_mask.to(dtype=torch.bool)
transition[TransitionKey.OBSERVATION] = observation
return transition
def transform_features(
self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
return features
def make_pistar06_pre_post_processors(
config: Pistar06Config,
dataset_stats: dict[str, dict[str, torch.Tensor]] | None = None, # noqa: ARG001
) -> tuple[
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
PolicyProcessorPipeline[PolicyAction, PolicyAction],
]:
camera_features = list(config.camera_features)
if not camera_features:
camera_features = [k for k in (config.input_features or {}).keys() if k.startswith(OBS_IMAGES)]
input_steps: list[ProcessorStep] = [
RenameObservationsProcessorStep(rename_map={}),
NormalizerProcessorStep(
features={**(config.input_features or {}), **(config.output_features or {})},
norm_map=config.normalization_mapping,
stats=dataset_stats,
normalize_observation_keys={config.state_feature},
),
Pistar06PrepareTaskPromptProcessorStep(
task_key=config.task_field,
include_state_in_prompt=config.include_state_in_prompt,
state_feature=config.state_feature,
max_state_dim=config.max_state_dim,
state_discretization_bins=config.state_discretization_bins,
),
TokenizerProcessorStep(
tokenizer_name=config.language_repo_id,
task_key=config.task_field,
max_length=config.tokenizer_max_length,
padding_side="right",
padding="max_length",
truncation=True,
),
Pistar06PrepareImagesProcessorStep(camera_features=camera_features),
DeviceProcessorStep(device=config.device),
]
output_steps: list[ProcessorStep] = [
DeviceProcessorStep(device="cpu"),
]
return (
PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
steps=input_steps,
name=POLICY_PREPROCESSOR_DEFAULT_NAME,
),
PolicyProcessorPipeline[PolicyAction, PolicyAction](
steps=output_steps,
name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
to_transition=policy_action_to_transition,
to_output=transition_to_policy_action,
),
)