chore(evo1): clean up quality issues

这个提交包含在:
javadcc_mac
2026-03-24 22:45:51 +08:00
父节点 c7e3466a23
当前提交 1fc0ad078e
共有 3 个文件被更改,包括 256 次插入 和 267 次删除

查看文件

@ -27,8 +27,6 @@ Validated in this repo / recent experiments:
- `torchvision==0.20.1`
- `transformers==4.39.0`
- `flash-attn==2.8.3`
## Training Notes
- `training_stage=stage1` freezes the VLM and trains the action head.

查看文件

@ -1,7 +1,9 @@
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class SinusoidalPositionalEncoding(nn.Module):
def __init__(self, dim: int, max_len: int = 1000):
super().__init__()
@ -10,8 +12,8 @@ class SinusoidalPositionalEncoding(nn.Module):
div_term = torch.exp(torch.arange(0, dim, 2) * -(math.log(10000.0) / dim))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer('pe', pe)
pe = pe.unsqueeze(0)
self.register_buffer("pe", pe)
def forward(self, seq_len: int):
if seq_len > self.pe.size(1):
@ -31,6 +33,7 @@ class SinusoidalPositionalEncoding(nn.Module):
new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1)
self.pe = new_pe
class CategorySpecificLinear(nn.Module):
def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1):
super().__init__()
@ -42,27 +45,24 @@ class CategorySpecificLinear(nn.Module):
self.bias = nn.Parameter(torch.randn(num_categories, out_dim))
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
if self.num_categories <= 1:
return self.linear(x)
orig_shape = x.shape
x_flat = x.reshape(-1, orig_shape[-1])
x_flat = x.reshape(-1, orig_shape[-1])
if category_id.dim() == 0:
cid = category_id.item()
out = x_flat @ self.weight[cid] + self.bias[cid]
else:
category_id = category_id.view(-1)
weight_selected = self.weight[category_id]
bias_selected = self.bias[category_id]
category_id = category_id.view(-1)
weight_selected = self.weight[category_id]
bias_selected = self.bias[category_id]
out = torch.bmm(x_flat.unsqueeze(1), weight_selected).squeeze(1) + bias_selected
out_shape = orig_shape[:-1] + (out.shape[-1],)
return out.view(out_shape)
class CategorySpecificMLP(nn.Module):
class CategorySpecificMLP(nn.Module):
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1):
super().__init__()
self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories)
@ -74,58 +74,53 @@ class CategorySpecificMLP(nn.Module):
out = self.fc2(out, category_id)
return out
class MultiEmbodimentActionEncoder(nn.Module):
def __init__(self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1):
class MultiEmbodimentActionEncoder(nn.Module):
def __init__(
self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1
):
super().__init__()
self.horizon = horizon
self.embed_dim = embed_dim
self.num_categories = num_categories
self.W1 = CategorySpecificLinear(action_dim, hidden_dim, num_categories)
self.W2 = CategorySpecificLinear(hidden_dim, hidden_dim, num_categories)
self.W3 = CategorySpecificLinear(hidden_dim, embed_dim, num_categories)
self.pos_encoding = SinusoidalPositionalEncoding(hidden_dim, max_len=horizon)
self.activation = nn.ReLU(inplace=True)
def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor):
batch_size, horizon, action_dim = action_seq.shape
assert self.horizon == horizon, "Action sequence length must match horizon"
x = action_seq.reshape(batch_size * horizon, action_dim)
B, H, D = action_seq.shape
assert H == self.horizon, "Action sequence length must match horizon"
x = action_seq.reshape(B * H, D)
if category_id.dim() == 0:
cat_ids = category_id.repeat(H * B)
cat_ids = category_id.repeat(horizon * batch_size)
else:
cat_ids = category_id.unsqueeze(1).repeat(1, H).reshape(B * H)
out = self.activation(self.W1(x, cat_ids))
pos_enc = self.pos_encoding(H).to(out.device)
pos_enc = pos_enc.repeat(B, 1, 1).reshape(B * H, -1)
cat_ids = category_id.unsqueeze(1).repeat(1, horizon).reshape(batch_size * horizon)
out = self.activation(self.W1(x, cat_ids))
pos_enc = self.pos_encoding(horizon).to(out.device)
pos_enc = pos_enc.repeat(batch_size, 1, 1).reshape(batch_size * horizon, -1)
out = out + pos_enc
out = self.activation(self.W2(out, cat_ids))
out = self.W3(out, cat_ids)
out = out.view(B, H, self.embed_dim)
out = self.activation(self.W2(out, cat_ids))
out = self.W3(out, cat_ids)
out = out.view(batch_size, horizon, self.embed_dim)
return out
class BasicTransformerBlock(nn.Module):
class BasicTransformerBlock(nn.Module):
def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0):
super().__init__()
self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
self.norm1 = nn.LayerNorm(embed_dim)
self.norm2 = nn.LayerNorm(embed_dim)
self.ff = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.GELU(),
nn.Linear(hidden_dim, embed_dim)
)
self.ff = nn.Sequential(nn.Linear(embed_dim, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, embed_dim))
def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor):
x = self.norm1(action_tokens)
attn_out, _ = self.attn(x, context_tokens, context_tokens)
@ -139,23 +134,25 @@ class BasicTransformerBlock(nn.Module):
x = x + ff_out
return x
class FlowmatchingActionHead(nn.Module):
def __init__(self, config=None,
embed_dim: int = 896,
hidden_dim: int = 1024,
action_dim: int = 16*7,
horizon: int = 16,
per_action_dim: int = 7,
num_heads: int = 8,
num_layers: int = 8,
dropout: float = 0.0,
num_inference_timesteps: int = 20,
num_categories: int = 1):
class FlowmatchingActionHead(nn.Module):
def __init__(
self,
config=None,
embed_dim: int = 896,
hidden_dim: int = 1024,
action_dim: int = 16 * 7,
horizon: int = 16,
per_action_dim: int = 7,
num_heads: int = 8,
num_layers: int = 8,
dropout: float = 0.0,
num_inference_timesteps: int = 20,
num_categories: int = 1,
):
super().__init__()
if config is not None:
embed_dim = getattr(config, "embed_dim", embed_dim)
hidden_dim = getattr(config, "hidden_dim", hidden_dim)
action_dim = getattr(config, "action_dim", action_dim)
@ -168,197 +165,202 @@ class FlowmatchingActionHead(nn.Module):
self.config = config
else:
from types import SimpleNamespace
self.config = SimpleNamespace(embed_dim=embed_dim, hidden_dim=hidden_dim,
action_dim=action_dim, horizon=horizon,
num_heads=num_heads, num_layers=num_layers,
dropout=dropout, num_inference_timesteps=num_inference_timesteps,
num_categories=num_categories)
self.config = SimpleNamespace(
embed_dim=embed_dim,
hidden_dim=hidden_dim,
action_dim=action_dim,
horizon=horizon,
num_heads=num_heads,
num_layers=num_layers,
dropout=dropout,
num_inference_timesteps=num_inference_timesteps,
num_categories=num_categories,
)
self.embed_dim = embed_dim
self.horizon = horizon
self.per_action_dim = config.per_action_dim
self.action_dim = config.action_dim
self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000)
self.transformer_blocks = nn.ModuleList([
BasicTransformerBlock(embed_dim=embed_dim, num_heads=num_heads,
hidden_dim=embed_dim*4, dropout=dropout)
for _ in range(num_layers)
])
self.transformer_blocks = nn.ModuleList(
[
BasicTransformerBlock(
embed_dim=embed_dim, num_heads=num_heads, hidden_dim=embed_dim * 4, dropout=dropout
)
for _ in range(num_layers)
]
)
self.norm_out = nn.LayerNorm(embed_dim)
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim)
self.mlp_head = CategorySpecificMLP(input_dim=embed_dim, hidden_dim=hidden_dim,
output_dim=action_dim, num_categories=num_categories)
self.mlp_head = CategorySpecificMLP(
input_dim=embed_dim, hidden_dim=hidden_dim, output_dim=action_dim, num_categories=num_categories
)
self.state_encoder = None
if hasattr(self.config, "state_dim") and self.config.state_dim is not None:
state_hidden = getattr(self.config, "state_hidden_dim", embed_dim)
self.state_encoder = CategorySpecificMLP(input_dim=self.config.state_dim,
hidden_dim=state_hidden,
output_dim=embed_dim,
num_categories=num_categories)
self.state_encoder = CategorySpecificMLP(
input_dim=self.config.state_dim,
hidden_dim=state_hidden,
output_dim=embed_dim,
num_categories=num_categories,
)
self.action_encoder = None
if horizon > 1:
per_action_dim = getattr(self.config, "per_action_dim", None)
if per_action_dim is None:
per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim
self.action_encoder = MultiEmbodimentActionEncoder(action_dim=per_action_dim,
embed_dim=embed_dim,
hidden_dim=embed_dim,
horizon=horizon,
num_categories=num_categories)
def forward(self, fused_tokens: torch.Tensor, state: torch.Tensor = None,
actions_gt: torch.Tensor = None, embodiment_id: torch.LongTensor = None,
state_mask: torch.Tensor = None, action_mask: torch.Tensor = None):
self.action_encoder = MultiEmbodimentActionEncoder(
action_dim=per_action_dim,
embed_dim=embed_dim,
hidden_dim=embed_dim,
horizon=horizon,
num_categories=num_categories,
)
def forward(
self,
fused_tokens: torch.Tensor,
state: torch.Tensor = None,
actions_gt: torch.Tensor = None,
embodiment_id: torch.LongTensor = None,
state_mask: torch.Tensor = None,
action_mask: torch.Tensor = None,
):
if actions_gt is None:
return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id)
B = fused_tokens.size(0)
batch_size = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id)
state_emb = state_emb.unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
t = torch.distributions.Beta(2, 2).sample((B,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
time_index = (t * 1000).long()
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
action_shape = actions_gt.shape[1]
actions_gt_seq = actions_gt
noise = torch.rand_like(actions_gt) * 2 - 1
if action_mask is not None:
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
assert action_mask.shape == noise.shape, f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
noise = noise * action_mask
if self.horizon > 1:
noise_seq = noise.view(B, self.horizon, self.per_action_dim)
else:
noise_seq = noise.unsqueeze(1)
if self.horizon > 1:
t_broadcast = t.view(B, 1, 1)
else:
t_broadcast = t.view(B, 1)
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
if self.horizon > 1 and self.action_encoder is not None:
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
else:
if not hasattr(self, "single_action_proj"):
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_intermediate_seq)
x = action_tokens
for block in self.transformer_blocks:
x = block(x, context_tokens, time_emb)
x = self.norm_out(x)
if self.horizon > 1:
x_flat = x.reshape(B, -1)
if not hasattr(self, "seq_pool_proj"):
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
return pred_velocity, noise
def get_action(self, fused_tokens: torch.Tensor, state: torch.Tensor = None, embodiment_id: torch.LongTensor = None, action_mask: torch.Tensor = None):
B = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id)
state_emb = state_emb.unsqueeze(1)
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
t = (
torch.distributions.Beta(2, 2).sample((batch_size,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
)
time_index = (t * 1000).long()
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
actions_gt_seq = actions_gt
noise = torch.rand_like(actions_gt) * 2 - 1
if action_mask is not None:
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
assert action_mask.shape == noise.shape, (
f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
)
noise = noise * action_mask
if self.horizon > 1:
noise_seq = noise.view(batch_size, self.horizon, self.per_action_dim)
else:
noise_seq = noise.unsqueeze(1)
t_broadcast = t.view(batch_size, 1, 1) if self.horizon > 1 else t.view(batch_size, 1)
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
if self.horizon > 1 and self.action_encoder is not None:
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
else:
if not hasattr(self, "single_action_proj"):
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_intermediate_seq)
x = action_tokens
for block in self.transformer_blocks:
x = block(x, context_tokens, time_emb)
x = self.norm_out(x)
if self.horizon > 1:
x_flat = x.reshape(B, -1)
if not hasattr(self, "seq_pool_proj"):
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
return pred_velocity, noise
def get_action(
self,
fused_tokens: torch.Tensor,
state: torch.Tensor = None,
embodiment_id: torch.LongTensor = None,
action_mask: torch.Tensor = None,
):
batch_size = fused_tokens.size(0)
device = fused_tokens.device
if embodiment_id is None:
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
context_tokens = fused_tokens
if state is not None and self.state_encoder is not None:
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
action_dim_total = getattr(self.config, "action_dim", None)
if action_dim_total is None:
action_dim_total = self.action_dim
if self.horizon > 1:
per_action_dim = getattr(self.config, "per_action_dim", action_dim_total // self.horizon)
else:
per_action_dim = action_dim_total
action = (torch.rand(B, action_dim_total, device=device) * 2 - 1)
action = torch.rand(batch_size, action_dim_total, device=device) * 2 - 1
if self.horizon > 1:
action_seq = action.view(B, self.horizon, per_action_dim)
action_seq = action.view(batch_size, self.horizon, per_action_dim)
else:
action_seq = action.view(B, 1, per_action_dim)
action_seq = action.view(batch_size, 1, per_action_dim)
action_mask = action_mask.view(B, 1, per_action_dim).repeat(1,self.horizon,1)
action_mask = action_mask.view(batch_size, 1, per_action_dim).repeat(1, self.horizon, 1)
if action_mask is not None:
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
assert action_mask.shape == action_seq.shape, f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
assert action_mask.shape == action_seq.shape, (
f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
)
action_seq = action_seq * action_mask
else:
raise ValueError("action_mask must be provided for inference with flow matching.")
N = int(getattr(self.config, "num_inference_timesteps", 32))
dt = 1.0 / N
for i in range(N):
t = i / N
num_steps = int(getattr(self.config, "num_inference_timesteps", 32))
dt = 1.0 / num_steps
for i in range(num_steps):
t = i / num_steps
time_index = int(t * 1000)
time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0)
time_emb = time_emb.unsqueeze(0).repeat(B, 1)
time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0)
time_emb = time_emb.unsqueeze(0).repeat(batch_size, 1)
if self.horizon > 1 and self.action_encoder is not None:
action_seq = action_seq * action_mask
action_tokens = self.action_encoder(action_seq, embodiment_id)
action_tokens = self.action_encoder(action_seq, embodiment_id)
else:
if hasattr(self, "single_action_proj"):
action_tokens = self.single_action_proj(action_seq)
action_tokens = self.single_action_proj(action_seq)
else:
self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device)
action_tokens = self.single_action_proj(action_seq)
@ -372,29 +374,26 @@ class FlowmatchingActionHead(nn.Module):
if hasattr(self, "seq_pool_proj"):
x_pooled = self.seq_pool_proj(x_flat)
else:
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
x_pooled = self.seq_pool_proj(x_flat)
else:
x_pooled = x.squeeze(1)
pred = self.mlp_head(x_pooled, embodiment_id)
pred = self.mlp_head(x_pooled, embodiment_id)
action = action + dt * pred
if self.horizon > 1:
action_seq = action.view(B, self.horizon, per_action_dim)
action_seq = action.view(batch_size, self.horizon, per_action_dim)
else:
action_seq = action.view(B, 1, per_action_dim)
action_seq = action.view(batch_size, 1, per_action_dim)
return action
@property
def device(self):
return next(self.parameters()).device
@property
def dtype(self):
return next(self.parameters()).dtype

查看文件

@ -1,29 +1,34 @@
import logging
from typing import Union
import torch
import torch.nn as nn
import torchvision.transforms as T
import torchvision.transforms as transforms
from PIL import Image
from torchvision.transforms.functional import InterpolationMode
from torchvision.transforms.functional import to_pil_image
from torchvision.transforms.functional import InterpolationMode, to_pil_image
from transformers import AutoModel, AutoTokenizer
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>" # nosec B105
IMG_START_TOKEN = "<img>" # nosec B105
IMG_END_TOKEN = "</img>" # nosec B105
# === Image Transformations ===
def build_transform(input_size):
return T.Compose([
T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
T.ToTensor(),
T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
])
return transforms.Compose(
[
transforms.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
transforms.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
transforms.ToTensor(),
transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
]
)
# === Aspect Ratio Handling ===
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
best_ratio_diff = float('inf')
best_ratio_diff = float("inf")
best_ratio = (1, 1)
area = width * height
for ratio in target_ratios:
@ -36,15 +41,21 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_
best_ratio = ratio
return best_ratio
def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False):
orig_width, orig_height = image.size
aspect_ratio = orig_width / orig_height
target_ratios = set(
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
i * j <= max_num and i * j >= min_num)
target_ratios = {
(i, j)
for n in range(min_num, max_num + 1)
for i in range(1, n + 1)
for j in range(1, n + 1)
if i * j <= max_num and i * j >= min_num
}
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio, target_ratios, orig_width, orig_height, image_size)
aspect_ratio, target_ratios, orig_width, orig_height, image_size
)
target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
@ -55,7 +66,7 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
(i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size
((i // (target_width // image_size)) + 1) * image_size,
)
split_img = resized_img.crop(box)
processed_images.append(split_img)
@ -65,6 +76,7 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
processed_images.append(thumbnail_img)
return processed_images
class InternVL3Embedder(nn.Module):
def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"):
super().__init__()
@ -80,16 +92,16 @@ class InternVL3Embedder(nn.Module):
use_flash_attn=True,
low_cpu_mem_usage=True,
_fast_init=False,
).to(self.device)
if hasattr(self.model.language_model, 'model'):
).to(self.device)
if hasattr(self.model.language_model, "model"):
layers = self.model.language_model.model.layers
else:
layers = self.model.language_model.layers
layers = layers[:14]
if hasattr(self.model.language_model, 'model'):
if hasattr(self.model.language_model, "model"):
self.model.language_model.model.layers = torch.nn.ModuleList(layers)
else:
self.model.language_model.layers = torch.nn.ModuleList(layers)
@ -97,15 +109,12 @@ class InternVL3Embedder(nn.Module):
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
self.model.vision_model.encoder.gradient_checkpointing = False
def _preprocess_images(
self,
image_tensors: list[Union[Image.Image, torch.Tensor]]
self, image_tensors: list[Image.Image | torch.Tensor]
) -> tuple[torch.Tensor, list[int]]:
pixel_values_list = []
for i, image in enumerate(image_tensors):
for image in image_tensors:
if isinstance(image, torch.Tensor):
image = to_pil_image(image)
tiles = dynamic_preprocess(image, image_size=self.image_size)
@ -117,21 +126,12 @@ class InternVL3Embedder(nn.Module):
return pixel_values, num_tiles_list
def _build_multimodal_prompt(
self,
num_tiles_list: list[int],
text_prompt: str
) -> str:
prompt = ''
def _build_multimodal_prompt(self, num_tiles_list: list[int], text_prompt: str) -> str:
prompt = ""
for i in range(len(num_tiles_list)):
prompt += f"Image-{i+1}: <image>\n"
prompt += f"Image-{i + 1}: <image>\n"
prompt += text_prompt.strip()
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
IMG_START_TOKEN = "<img>"
IMG_END_TOKEN = "</img>"
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
for tile_count in num_tiles_list:
token_count = self.model.num_image_token * tile_count
@ -139,15 +139,10 @@ class InternVL3Embedder(nn.Module):
prompt = prompt.replace("<image>", image_tokens, 1)
return prompt
def _prepare_and_fuse_embeddings(
self,
prompt: str,
vit_embeds: torch.Tensor,
image_mask: torch.Tensor,
num_tiles_list: list[int]
self, prompt: str, vit_embeds: torch.Tensor, image_mask: torch.Tensor, num_tiles_list: list[int]
) -> tuple[torch.Tensor, torch.Tensor]:
untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
true_sequence_length = untruncated_ids.shape[1]
@ -159,26 +154,29 @@ class InternVL3Embedder(nn.Module):
prompt[:100],
)
model_inputs = self.tokenizer(prompt, return_tensors="pt", padding='max_length', truncation=True, max_length=self.max_text_length).to(self.device)
model_inputs = self.tokenizer(
prompt,
return_tensors="pt",
padding="max_length",
truncation=True,
max_length=self.max_text_length,
).to(self.device)
input_ids = model_inputs["input_ids"]
attention_mask = model_inputs["attention_mask"]
img_token_mask = (input_ids == self.img_context_token_id)
img_token_locations = torch.where(img_token_mask)[1]
img_token_mask = input_ids == self.img_context_token_id
img_token_locations = torch.where(img_token_mask)[1]
input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone()
B, N, C = input_embeds.shape
input_embeds = input_embeds.reshape(B * N, C)
input_ids = input_ids.reshape(B * N)
batch_size, seq_len, channels = input_embeds.shape
input_embeds = input_embeds.reshape(batch_size * seq_len, channels)
input_ids = input_ids.reshape(batch_size * seq_len)
selected = (input_ids == self.img_context_token_id)
selected = input_ids == self.img_context_token_id
vit_embeds = vit_embeds.reshape(-1, C).to(device=input_embeds.device, dtype=input_embeds.dtype)
vit_embeds = vit_embeds.reshape(-1, channels).to(device=input_embeds.device, dtype=input_embeds.dtype)
n_token = int(selected.sum().item())
if vit_embeds.shape[0] < n_token:
raise ValueError(
@ -186,49 +184,43 @@ class InternVL3Embedder(nn.Module):
)
input_embeds[selected] = vit_embeds[:n_token]
tokens_per_tile = self.model.num_image_token
tokens_per_tile = self.model.num_image_token
current_token_idx = 0
for i in range(len(image_mask)):
num_tiles_for_this_image = num_tiles_list[i]
num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile
if not image_mask[i]:
start_idx = img_token_locations[current_token_idx]
end_idx = start_idx + num_tokens_for_this_image
attention_mask[0, start_idx:end_idx] = 0
current_token_idx += num_tokens_for_this_image
input_embeds = input_embeds.reshape(B, N, C)
return input_embeds, attention_mask
input_embeds = input_embeds.reshape(batch_size, seq_len, channels)
return input_embeds, attention_mask
def get_fused_image_text_embedding_from_tensor_images(
self,
image_tensors: list[Union[Image.Image, torch.Tensor]],
image_tensors: list[Image.Image | torch.Tensor],
image_mask: torch.Tensor,
text_prompt: str,
return_cls_only: bool = True,
):
pixel_values, num_tiles_list = self._preprocess_images(image_tensors)
if pixel_values.shape[0] == 0:
logging.warning("InternVL3 received an empty image batch after preprocessing.")
vit_embeds = self.model.extract_feature(pixel_values)
fused_embeds = vit_embeds
fused_embeds = vit_embeds
prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt)
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(prompt, fused_embeds, image_mask, num_tiles_list)
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(
prompt, fused_embeds, image_mask, num_tiles_list
)
outputs = self.model.language_model(
inputs_embeds=inputs_embeds,