chore(evo1): clean up quality issues

这个提交包含在:
javadcc_mac
2026-03-24 22:45:51 +08:00
父节点 c7e3466a23
当前提交 1fc0ad078e
共有 3 个文件被更改,包括 256 次插入 和 267 次删除

查看文件

@ -27,8 +27,6 @@ Validated in this repo / recent experiments:
- `torchvision==0.20.1` - `torchvision==0.20.1`
- `transformers==4.39.0` - `transformers==4.39.0`
- `flash-attn==2.8.3` - `flash-attn==2.8.3`
## Training Notes ## Training Notes
- `training_stage=stage1` freezes the VLM and trains the action head. - `training_stage=stage1` freezes the VLM and trains the action head.

查看文件

@ -1,7 +1,9 @@
import math import math
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F
class SinusoidalPositionalEncoding(nn.Module): class SinusoidalPositionalEncoding(nn.Module):
def __init__(self, dim: int, max_len: int = 1000): def __init__(self, dim: int, max_len: int = 1000):
super().__init__() super().__init__()
@ -10,8 +12,8 @@ class SinusoidalPositionalEncoding(nn.Module):
div_term = torch.exp(torch.arange(0, dim, 2) * -(math.log(10000.0) / dim)) div_term = torch.exp(torch.arange(0, dim, 2) * -(math.log(10000.0) / dim))
pe[:, 0::2] = torch.sin(position * div_term) pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term) pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0) pe = pe.unsqueeze(0)
self.register_buffer('pe', pe) self.register_buffer("pe", pe)
def forward(self, seq_len: int): def forward(self, seq_len: int):
if seq_len > self.pe.size(1): if seq_len > self.pe.size(1):
@ -31,6 +33,7 @@ class SinusoidalPositionalEncoding(nn.Module):
new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1) new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1)
self.pe = new_pe self.pe = new_pe
class CategorySpecificLinear(nn.Module): class CategorySpecificLinear(nn.Module):
def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1): def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1):
super().__init__() super().__init__()
@ -42,27 +45,24 @@ class CategorySpecificLinear(nn.Module):
self.bias = nn.Parameter(torch.randn(num_categories, out_dim)) self.bias = nn.Parameter(torch.randn(num_categories, out_dim))
def forward(self, x: torch.Tensor, category_id: torch.LongTensor): def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
if self.num_categories <= 1: if self.num_categories <= 1:
return self.linear(x) return self.linear(x)
orig_shape = x.shape orig_shape = x.shape
x_flat = x.reshape(-1, orig_shape[-1]) x_flat = x.reshape(-1, orig_shape[-1])
if category_id.dim() == 0: if category_id.dim() == 0:
cid = category_id.item() cid = category_id.item()
out = x_flat @ self.weight[cid] + self.bias[cid] out = x_flat @ self.weight[cid] + self.bias[cid]
else: else:
category_id = category_id.view(-1)
category_id = category_id.view(-1) weight_selected = self.weight[category_id]
weight_selected = self.weight[category_id] bias_selected = self.bias[category_id]
bias_selected = self.bias[category_id]
out = torch.bmm(x_flat.unsqueeze(1), weight_selected).squeeze(1) + bias_selected out = torch.bmm(x_flat.unsqueeze(1), weight_selected).squeeze(1) + bias_selected
out_shape = orig_shape[:-1] + (out.shape[-1],) out_shape = orig_shape[:-1] + (out.shape[-1],)
return out.view(out_shape) return out.view(out_shape)
class CategorySpecificMLP(nn.Module):
class CategorySpecificMLP(nn.Module):
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1): def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1):
super().__init__() super().__init__()
self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories) self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories)
@ -74,58 +74,53 @@ class CategorySpecificMLP(nn.Module):
out = self.fc2(out, category_id) out = self.fc2(out, category_id)
return out return out
class MultiEmbodimentActionEncoder(nn.Module):
def __init__(self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1): class MultiEmbodimentActionEncoder(nn.Module):
def __init__(
self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1
):
super().__init__() super().__init__()
self.horizon = horizon self.horizon = horizon
self.embed_dim = embed_dim self.embed_dim = embed_dim
self.num_categories = num_categories self.num_categories = num_categories
self.W1 = CategorySpecificLinear(action_dim, hidden_dim, num_categories) self.W1 = CategorySpecificLinear(action_dim, hidden_dim, num_categories)
self.W2 = CategorySpecificLinear(hidden_dim, hidden_dim, num_categories) self.W2 = CategorySpecificLinear(hidden_dim, hidden_dim, num_categories)
self.W3 = CategorySpecificLinear(hidden_dim, embed_dim, num_categories) self.W3 = CategorySpecificLinear(hidden_dim, embed_dim, num_categories)
self.pos_encoding = SinusoidalPositionalEncoding(hidden_dim, max_len=horizon) self.pos_encoding = SinusoidalPositionalEncoding(hidden_dim, max_len=horizon)
self.activation = nn.ReLU(inplace=True) self.activation = nn.ReLU(inplace=True)
def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor): def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor):
batch_size, horizon, action_dim = action_seq.shape
assert self.horizon == horizon, "Action sequence length must match horizon"
x = action_seq.reshape(batch_size * horizon, action_dim)
B, H, D = action_seq.shape
assert H == self.horizon, "Action sequence length must match horizon"
x = action_seq.reshape(B * H, D)
if category_id.dim() == 0: if category_id.dim() == 0:
cat_ids = category_id.repeat(horizon * batch_size)
cat_ids = category_id.repeat(H * B)
else: else:
cat_ids = category_id.unsqueeze(1).repeat(1, H).reshape(B * H) cat_ids = category_id.unsqueeze(1).repeat(1, horizon).reshape(batch_size * horizon)
out = self.activation(self.W1(x, cat_ids)) out = self.activation(self.W1(x, cat_ids))
pos_enc = self.pos_encoding(H).to(out.device) pos_enc = self.pos_encoding(horizon).to(out.device)
pos_enc = pos_enc.repeat(B, 1, 1).reshape(B * H, -1) pos_enc = pos_enc.repeat(batch_size, 1, 1).reshape(batch_size * horizon, -1)
out = out + pos_enc out = out + pos_enc
out = self.activation(self.W2(out, cat_ids)) out = self.activation(self.W2(out, cat_ids))
out = self.W3(out, cat_ids) out = self.W3(out, cat_ids)
out = out.view(B, H, self.embed_dim) out = out.view(batch_size, horizon, self.embed_dim)
return out return out
class BasicTransformerBlock(nn.Module):
class BasicTransformerBlock(nn.Module):
def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0): def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0):
super().__init__() super().__init__()
self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True) self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
self.norm1 = nn.LayerNorm(embed_dim) self.norm1 = nn.LayerNorm(embed_dim)
self.norm2 = nn.LayerNorm(embed_dim) self.norm2 = nn.LayerNorm(embed_dim)
self.ff = nn.Sequential( self.ff = nn.Sequential(nn.Linear(embed_dim, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, embed_dim))
nn.Linear(embed_dim, hidden_dim),
nn.GELU(),
nn.Linear(hidden_dim, embed_dim)
)
def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor): def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor):
x = self.norm1(action_tokens) x = self.norm1(action_tokens)
attn_out, _ = self.attn(x, context_tokens, context_tokens) attn_out, _ = self.attn(x, context_tokens, context_tokens)
@ -139,23 +134,25 @@ class BasicTransformerBlock(nn.Module):
x = x + ff_out x = x + ff_out
return x return x
class FlowmatchingActionHead(nn.Module):
def __init__(self, config=None, class FlowmatchingActionHead(nn.Module):
embed_dim: int = 896, def __init__(
hidden_dim: int = 1024, self,
action_dim: int = 16*7, config=None,
horizon: int = 16, embed_dim: int = 896,
per_action_dim: int = 7, hidden_dim: int = 1024,
num_heads: int = 8, action_dim: int = 16 * 7,
num_layers: int = 8, horizon: int = 16,
dropout: float = 0.0, per_action_dim: int = 7,
num_inference_timesteps: int = 20, num_heads: int = 8,
num_categories: int = 1): num_layers: int = 8,
dropout: float = 0.0,
num_inference_timesteps: int = 20,
num_categories: int = 1,
):
super().__init__() super().__init__()
if config is not None: if config is not None:
embed_dim = getattr(config, "embed_dim", embed_dim) embed_dim = getattr(config, "embed_dim", embed_dim)
hidden_dim = getattr(config, "hidden_dim", hidden_dim) hidden_dim = getattr(config, "hidden_dim", hidden_dim)
action_dim = getattr(config, "action_dim", action_dim) action_dim = getattr(config, "action_dim", action_dim)
@ -168,197 +165,202 @@ class FlowmatchingActionHead(nn.Module):
self.config = config self.config = config
else: else:
from types import SimpleNamespace from types import SimpleNamespace
self.config = SimpleNamespace(embed_dim=embed_dim, hidden_dim=hidden_dim,
action_dim=action_dim, horizon=horizon, self.config = SimpleNamespace(
num_heads=num_heads, num_layers=num_layers, embed_dim=embed_dim,
dropout=dropout, num_inference_timesteps=num_inference_timesteps, hidden_dim=hidden_dim,
num_categories=num_categories) action_dim=action_dim,
horizon=horizon,
num_heads=num_heads,
num_layers=num_layers,
dropout=dropout,
num_inference_timesteps=num_inference_timesteps,
num_categories=num_categories,
)
self.embed_dim = embed_dim self.embed_dim = embed_dim
self.horizon = horizon self.horizon = horizon
self.per_action_dim = config.per_action_dim self.per_action_dim = config.per_action_dim
self.action_dim = config.action_dim self.action_dim = config.action_dim
self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000) self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000)
self.transformer_blocks = nn.ModuleList([ self.transformer_blocks = nn.ModuleList(
BasicTransformerBlock(embed_dim=embed_dim, num_heads=num_heads, [
hidden_dim=embed_dim*4, dropout=dropout) BasicTransformerBlock(
for _ in range(num_layers) embed_dim=embed_dim, num_heads=num_heads, hidden_dim=embed_dim * 4, dropout=dropout
]) )
for _ in range(num_layers)
]
)
self.norm_out = nn.LayerNorm(embed_dim) self.norm_out = nn.LayerNorm(embed_dim)
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim) self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim)
self.mlp_head = CategorySpecificMLP(input_dim=embed_dim, hidden_dim=hidden_dim, self.mlp_head = CategorySpecificMLP(
output_dim=action_dim, num_categories=num_categories) input_dim=embed_dim, hidden_dim=hidden_dim, output_dim=action_dim, num_categories=num_categories
)
self.state_encoder = None self.state_encoder = None
if hasattr(self.config, "state_dim") and self.config.state_dim is not None: if hasattr(self.config, "state_dim") and self.config.state_dim is not None:
state_hidden = getattr(self.config, "state_hidden_dim", embed_dim) state_hidden = getattr(self.config, "state_hidden_dim", embed_dim)
self.state_encoder = CategorySpecificMLP(input_dim=self.config.state_dim, self.state_encoder = CategorySpecificMLP(
hidden_dim=state_hidden, input_dim=self.config.state_dim,
output_dim=embed_dim, hidden_dim=state_hidden,
num_categories=num_categories) output_dim=embed_dim,
num_categories=num_categories,
)
self.action_encoder = None self.action_encoder = None
if horizon > 1: if horizon > 1:
per_action_dim = getattr(self.config, "per_action_dim", None) per_action_dim = getattr(self.config, "per_action_dim", None)
if per_action_dim is None: if per_action_dim is None:
per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim
self.action_encoder = MultiEmbodimentActionEncoder(action_dim=per_action_dim, self.action_encoder = MultiEmbodimentActionEncoder(
embed_dim=embed_dim, action_dim=per_action_dim,
hidden_dim=embed_dim, embed_dim=embed_dim,
horizon=horizon, hidden_dim=embed_dim,
num_categories=num_categories) horizon=horizon,
num_categories=num_categories,
def forward(self, fused_tokens: torch.Tensor, state: torch.Tensor = None, )
actions_gt: torch.Tensor = None, embodiment_id: torch.LongTensor = None,
state_mask: torch.Tensor = None, action_mask: torch.Tensor = None):
def forward(
self,
fused_tokens: torch.Tensor,
state: torch.Tensor = None,
actions_gt: torch.Tensor = None,
embodiment_id: torch.LongTensor = None,
state_mask: torch.Tensor = None,
action_mask: torch.Tensor = None,
):
if actions_gt is None: if actions_gt is None:
return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id) return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id)
B = fused_tokens.size(0) batch_size = fused_tokens.size(0)
device = fused_tokens.device device = fused_tokens.device
if embodiment_id is None: if embodiment_id is None:
embodiment_id = torch.zeros(B, dtype=torch.long, device=device) embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id)
state_emb = state_emb.unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
t = torch.distributions.Beta(2, 2).sample((B,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
time_index = (t * 1000).long()
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
action_shape = actions_gt.shape[1]
actions_gt_seq = actions_gt
noise = torch.rand_like(actions_gt) * 2 - 1
if action_mask is not None:
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
assert action_mask.shape == noise.shape, f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
noise = noise * action_mask
if self.horizon > 1:
noise_seq = noise.view(B, self.horizon, self.per_action_dim)
else:
noise_seq = noise.unsqueeze(1)
if self.horizon > 1:
t_broadcast = t.view(B, 1, 1)
else:
t_broadcast = t.view(B, 1)
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
if self.horizon > 1 and self.action_encoder is not None:
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
else:
if not hasattr(self, "single_action_proj"):
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_intermediate_seq)
x = action_tokens
for block in self.transformer_blocks:
x = block(x, context_tokens, time_emb)
x = self.norm_out(x)
if self.horizon > 1:
x_flat = x.reshape(B, -1)
if not hasattr(self, "seq_pool_proj"):
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
return pred_velocity, noise
def get_action(self, fused_tokens: torch.Tensor, state: torch.Tensor = None, embodiment_id: torch.LongTensor = None, action_mask: torch.Tensor = None):
B = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
context_tokens = fused_tokens context_tokens = fused_tokens
if state is not None and self.state_encoder is not None: if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id)
state_emb = state_emb.unsqueeze(1)
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1) context_tokens = torch.cat([context_tokens, state_emb], dim=1)
t = (
torch.distributions.Beta(2, 2).sample((batch_size,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
)
time_index = (t * 1000).long()
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
actions_gt_seq = actions_gt
noise = torch.rand_like(actions_gt) * 2 - 1
if action_mask is not None:
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
assert action_mask.shape == noise.shape, (
f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
)
noise = noise * action_mask
if self.horizon > 1:
noise_seq = noise.view(batch_size, self.horizon, self.per_action_dim)
else:
noise_seq = noise.unsqueeze(1)
t_broadcast = t.view(batch_size, 1, 1) if self.horizon > 1 else t.view(batch_size, 1)
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
if self.horizon > 1 and self.action_encoder is not None:
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
else:
if not hasattr(self, "single_action_proj"):
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_intermediate_seq)
x = action_tokens
for block in self.transformer_blocks:
x = block(x, context_tokens, time_emb)
x = self.norm_out(x)
if self.horizon > 1:
x_flat = x.reshape(B, -1)
if not hasattr(self, "seq_pool_proj"):
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
return pred_velocity, noise
def get_action(
self,
fused_tokens: torch.Tensor,
state: torch.Tensor = None,
embodiment_id: torch.LongTensor = None,
action_mask: torch.Tensor = None,
):
batch_size = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1) context_tokens = torch.cat([context_tokens, state_emb], dim=1)
action_dim_total = getattr(self.config, "action_dim", None) action_dim_total = getattr(self.config, "action_dim", None)
if action_dim_total is None: if action_dim_total is None:
action_dim_total = self.action_dim action_dim_total = self.action_dim
if self.horizon > 1: if self.horizon > 1:
per_action_dim = getattr(self.config, "per_action_dim", action_dim_total // self.horizon) per_action_dim = getattr(self.config, "per_action_dim", action_dim_total // self.horizon)
else: else:
per_action_dim = action_dim_total per_action_dim = action_dim_total
action = (torch.rand(B, action_dim_total, device=device) * 2 - 1) action = torch.rand(batch_size, action_dim_total, device=device) * 2 - 1
if self.horizon > 1: if self.horizon > 1:
action_seq = action.view(B, self.horizon, per_action_dim) action_seq = action.view(batch_size, self.horizon, per_action_dim)
else: else:
action_seq = action.view(B, 1, per_action_dim) action_seq = action.view(batch_size, 1, per_action_dim)
action_mask = action_mask.view(B, 1, per_action_dim).repeat(1,self.horizon,1) action_mask = action_mask.view(batch_size, 1, per_action_dim).repeat(1, self.horizon, 1)
if action_mask is not None: if action_mask is not None:
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device) action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
assert action_mask.shape == action_seq.shape, f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}" assert action_mask.shape == action_seq.shape, (
f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
)
action_seq = action_seq * action_mask action_seq = action_seq * action_mask
else: else:
raise ValueError("action_mask must be provided for inference with flow matching.") raise ValueError("action_mask must be provided for inference with flow matching.")
N = int(getattr(self.config, "num_inference_timesteps", 32)) num_steps = int(getattr(self.config, "num_inference_timesteps", 32))
dt = 1.0 / N dt = 1.0 / num_steps
for i in range(N): for i in range(num_steps):
t = i / N t = i / num_steps
time_index = int(t * 1000) time_index = int(t * 1000)
time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0) time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0)
time_emb = time_emb.unsqueeze(0).repeat(B, 1) time_emb = time_emb.unsqueeze(0).repeat(batch_size, 1)
if self.horizon > 1 and self.action_encoder is not None: if self.horizon > 1 and self.action_encoder is not None:
action_seq = action_seq * action_mask action_seq = action_seq * action_mask
action_tokens = self.action_encoder(action_seq, embodiment_id) action_tokens = self.action_encoder(action_seq, embodiment_id)
else: else:
if hasattr(self, "single_action_proj"): if hasattr(self, "single_action_proj"):
action_tokens = self.single_action_proj(action_seq) action_tokens = self.single_action_proj(action_seq)
else: else:
self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device) self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_seq) action_tokens = self.single_action_proj(action_seq)
@ -372,29 +374,26 @@ class FlowmatchingActionHead(nn.Module):
if hasattr(self, "seq_pool_proj"): if hasattr(self, "seq_pool_proj"):
x_pooled = self.seq_pool_proj(x_flat) x_pooled = self.seq_pool_proj(x_flat)
else: else:
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device) self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat) x_pooled = self.seq_pool_proj(x_flat)
else: else:
x_pooled = x.squeeze(1) x_pooled = x.squeeze(1)
pred = self.mlp_head(x_pooled, embodiment_id) pred = self.mlp_head(x_pooled, embodiment_id)
action = action + dt * pred action = action + dt * pred
if self.horizon > 1: if self.horizon > 1:
action_seq = action.view(B, self.horizon, per_action_dim) action_seq = action.view(batch_size, self.horizon, per_action_dim)
else: else:
action_seq = action.view(B, 1, per_action_dim) action_seq = action.view(batch_size, 1, per_action_dim)
return action return action
@property @property
def device(self): def device(self):
return next(self.parameters()).device return next(self.parameters()).device
@property @property
def dtype(self): def dtype(self):
return next(self.parameters()).dtype return next(self.parameters()).dtype

查看文件

@ -1,29 +1,34 @@
import logging import logging
from typing import Union
import torch import torch
import torch.nn as nn import torch.nn as nn
import torchvision.transforms as T import torchvision.transforms as transforms
from PIL import Image from PIL import Image
from torchvision.transforms.functional import InterpolationMode from torchvision.transforms.functional import InterpolationMode, to_pil_image
from torchvision.transforms.functional import to_pil_image
from transformers import AutoModel, AutoTokenizer from transformers import AutoModel, AutoTokenizer
IMAGENET_MEAN = (0.485, 0.456, 0.406) IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225) IMAGENET_STD = (0.229, 0.224, 0.225)
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>" # nosec B105
IMG_START_TOKEN = "<img>" # nosec B105
IMG_END_TOKEN = "</img>" # nosec B105
# === Image Transformations === # === Image Transformations ===
def build_transform(input_size): def build_transform(input_size):
return T.Compose([ return transforms.Compose(
T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img), [
T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), transforms.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
T.ToTensor(), transforms.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD) transforms.ToTensor(),
]) transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
]
)
# === Aspect Ratio Handling === # === Aspect Ratio Handling ===
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size): def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
best_ratio_diff = float('inf') best_ratio_diff = float("inf")
best_ratio = (1, 1) best_ratio = (1, 1)
area = width * height area = width * height
for ratio in target_ratios: for ratio in target_ratios:
@ -36,15 +41,21 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_
best_ratio = ratio best_ratio = ratio
return best_ratio return best_ratio
def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False): def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False):
orig_width, orig_height = image.size orig_width, orig_height = image.size
aspect_ratio = orig_width / orig_height aspect_ratio = orig_width / orig_height
target_ratios = set( target_ratios = {
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if (i, j)
i * j <= max_num and i * j >= min_num) for n in range(min_num, max_num + 1)
for i in range(1, n + 1)
for j in range(1, n + 1)
if i * j <= max_num and i * j >= min_num
}
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
target_aspect_ratio = find_closest_aspect_ratio( target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio, target_ratios, orig_width, orig_height, image_size) aspect_ratio, target_ratios, orig_width, orig_height, image_size
)
target_width = image_size * target_aspect_ratio[0] target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1] target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1] blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
@ -55,7 +66,7 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
(i % (target_width // image_size)) * image_size, (i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size, ((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size ((i // (target_width // image_size)) + 1) * image_size,
) )
split_img = resized_img.crop(box) split_img = resized_img.crop(box)
processed_images.append(split_img) processed_images.append(split_img)
@ -65,6 +76,7 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
processed_images.append(thumbnail_img) processed_images.append(thumbnail_img)
return processed_images return processed_images
class InternVL3Embedder(nn.Module): class InternVL3Embedder(nn.Module):
def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"): def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"):
super().__init__() super().__init__()
@ -80,16 +92,16 @@ class InternVL3Embedder(nn.Module):
use_flash_attn=True, use_flash_attn=True,
low_cpu_mem_usage=True, low_cpu_mem_usage=True,
_fast_init=False, _fast_init=False,
).to(self.device) ).to(self.device)
if hasattr(self.model.language_model, 'model'): if hasattr(self.model.language_model, "model"):
layers = self.model.language_model.model.layers layers = self.model.language_model.model.layers
else: else:
layers = self.model.language_model.layers layers = self.model.language_model.layers
layers = layers[:14] layers = layers[:14]
if hasattr(self.model.language_model, 'model'): if hasattr(self.model.language_model, "model"):
self.model.language_model.model.layers = torch.nn.ModuleList(layers) self.model.language_model.model.layers = torch.nn.ModuleList(layers)
else: else:
self.model.language_model.layers = torch.nn.ModuleList(layers) self.model.language_model.layers = torch.nn.ModuleList(layers)
@ -97,15 +109,12 @@ class InternVL3Embedder(nn.Module):
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"): if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
self.model.vision_model.encoder.gradient_checkpointing = False self.model.vision_model.encoder.gradient_checkpointing = False
def _preprocess_images( def _preprocess_images(
self, self, image_tensors: list[Image.Image | torch.Tensor]
image_tensors: list[Union[Image.Image, torch.Tensor]]
) -> tuple[torch.Tensor, list[int]]: ) -> tuple[torch.Tensor, list[int]]:
pixel_values_list = [] pixel_values_list = []
for i, image in enumerate(image_tensors): for image in image_tensors:
if isinstance(image, torch.Tensor): if isinstance(image, torch.Tensor):
image = to_pil_image(image) image = to_pil_image(image)
tiles = dynamic_preprocess(image, image_size=self.image_size) tiles = dynamic_preprocess(image, image_size=self.image_size)
@ -117,21 +126,12 @@ class InternVL3Embedder(nn.Module):
return pixel_values, num_tiles_list return pixel_values, num_tiles_list
def _build_multimodal_prompt( def _build_multimodal_prompt(self, num_tiles_list: list[int], text_prompt: str) -> str:
self, prompt = ""
num_tiles_list: list[int],
text_prompt: str
) -> str:
prompt = ''
for i in range(len(num_tiles_list)): for i in range(len(num_tiles_list)):
prompt += f"Image-{i+1}: <image>\n" prompt += f"Image-{i + 1}: <image>\n"
prompt += text_prompt.strip() prompt += text_prompt.strip()
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
IMG_START_TOKEN = "<img>"
IMG_END_TOKEN = "</img>"
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
for tile_count in num_tiles_list: for tile_count in num_tiles_list:
token_count = self.model.num_image_token * tile_count token_count = self.model.num_image_token * tile_count
@ -139,15 +139,10 @@ class InternVL3Embedder(nn.Module):
prompt = prompt.replace("<image>", image_tokens, 1) prompt = prompt.replace("<image>", image_tokens, 1)
return prompt return prompt
def _prepare_and_fuse_embeddings( def _prepare_and_fuse_embeddings(
self, self, prompt: str, vit_embeds: torch.Tensor, image_mask: torch.Tensor, num_tiles_list: list[int]
prompt: str,
vit_embeds: torch.Tensor,
image_mask: torch.Tensor,
num_tiles_list: list[int]
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
true_sequence_length = untruncated_ids.shape[1] true_sequence_length = untruncated_ids.shape[1]
@ -159,26 +154,29 @@ class InternVL3Embedder(nn.Module):
prompt[:100], prompt[:100],
) )
model_inputs = self.tokenizer(prompt, return_tensors="pt", padding='max_length', truncation=True, max_length=self.max_text_length).to(self.device) model_inputs = self.tokenizer(
prompt,
return_tensors="pt",
padding="max_length",
truncation=True,
max_length=self.max_text_length,
).to(self.device)
input_ids = model_inputs["input_ids"] input_ids = model_inputs["input_ids"]
attention_mask = model_inputs["attention_mask"] attention_mask = model_inputs["attention_mask"]
img_token_mask = input_ids == self.img_context_token_id
img_token_mask = (input_ids == self.img_context_token_id)
img_token_locations = torch.where(img_token_mask)[1]
img_token_locations = torch.where(img_token_mask)[1]
input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone() input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone()
B, N, C = input_embeds.shape batch_size, seq_len, channels = input_embeds.shape
input_embeds = input_embeds.reshape(B * N, C) input_embeds = input_embeds.reshape(batch_size * seq_len, channels)
input_ids = input_ids.reshape(B * N) input_ids = input_ids.reshape(batch_size * seq_len)
selected = (input_ids == self.img_context_token_id) selected = input_ids == self.img_context_token_id
vit_embeds = vit_embeds.reshape(-1, channels).to(device=input_embeds.device, dtype=input_embeds.dtype)
vit_embeds = vit_embeds.reshape(-1, C).to(device=input_embeds.device, dtype=input_embeds.dtype)
n_token = int(selected.sum().item()) n_token = int(selected.sum().item())
if vit_embeds.shape[0] < n_token: if vit_embeds.shape[0] < n_token:
raise ValueError( raise ValueError(
@ -186,49 +184,43 @@ class InternVL3Embedder(nn.Module):
) )
input_embeds[selected] = vit_embeds[:n_token] input_embeds[selected] = vit_embeds[:n_token]
tokens_per_tile = self.model.num_image_token
tokens_per_tile = self.model.num_image_token
current_token_idx = 0 current_token_idx = 0
for i in range(len(image_mask)): for i in range(len(image_mask)):
num_tiles_for_this_image = num_tiles_list[i] num_tiles_for_this_image = num_tiles_list[i]
num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile
if not image_mask[i]: if not image_mask[i]:
start_idx = img_token_locations[current_token_idx] start_idx = img_token_locations[current_token_idx]
end_idx = start_idx + num_tokens_for_this_image end_idx = start_idx + num_tokens_for_this_image
attention_mask[0, start_idx:end_idx] = 0 attention_mask[0, start_idx:end_idx] = 0
current_token_idx += num_tokens_for_this_image current_token_idx += num_tokens_for_this_image
input_embeds = input_embeds.reshape(B, N, C) input_embeds = input_embeds.reshape(batch_size, seq_len, channels)
return input_embeds, attention_mask
return input_embeds, attention_mask
def get_fused_image_text_embedding_from_tensor_images( def get_fused_image_text_embedding_from_tensor_images(
self, self,
image_tensors: list[Union[Image.Image, torch.Tensor]], image_tensors: list[Image.Image | torch.Tensor],
image_mask: torch.Tensor, image_mask: torch.Tensor,
text_prompt: str, text_prompt: str,
return_cls_only: bool = True, return_cls_only: bool = True,
): ):
pixel_values, num_tiles_list = self._preprocess_images(image_tensors) pixel_values, num_tiles_list = self._preprocess_images(image_tensors)
if pixel_values.shape[0] == 0: if pixel_values.shape[0] == 0:
logging.warning("InternVL3 received an empty image batch after preprocessing.") logging.warning("InternVL3 received an empty image batch after preprocessing.")
vit_embeds = self.model.extract_feature(pixel_values) vit_embeds = self.model.extract_feature(pixel_values)
fused_embeds = vit_embeds fused_embeds = vit_embeds
prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt) prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt)
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(prompt, fused_embeds, image_mask, num_tiles_list) inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(
prompt, fused_embeds, image_mask, num_tiles_list
)
outputs = self.model.language_model( outputs = self.model.language_model(
inputs_embeds=inputs_embeds, inputs_embeds=inputs_embeds,