镜像自地址
https://github.com/MINT-SJTU/Evo-RL.git
已同步 2026-11-20 18:17:06 +00:00
chore(evo1): clean up quality issues
这个提交包含在:
@ -27,8 +27,6 @@ Validated in this repo / recent experiments:
|
|||||||
- `torchvision==0.20.1`
|
- `torchvision==0.20.1`
|
||||||
- `transformers==4.39.0`
|
- `transformers==4.39.0`
|
||||||
- `flash-attn==2.8.3`
|
- `flash-attn==2.8.3`
|
||||||
|
|
||||||
|
|
||||||
## Training Notes
|
## Training Notes
|
||||||
|
|
||||||
- `training_stage=stage1` freezes the VLM and trains the action head.
|
- `training_stage=stage1` freezes the VLM and trains the action head.
|
||||||
|
|||||||
@ -1,7 +1,9 @@
|
|||||||
import math
|
import math
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.nn.functional as F
|
|
||||||
|
|
||||||
class SinusoidalPositionalEncoding(nn.Module):
|
class SinusoidalPositionalEncoding(nn.Module):
|
||||||
def __init__(self, dim: int, max_len: int = 1000):
|
def __init__(self, dim: int, max_len: int = 1000):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@ -11,7 +13,7 @@ class SinusoidalPositionalEncoding(nn.Module):
|
|||||||
pe[:, 0::2] = torch.sin(position * div_term)
|
pe[:, 0::2] = torch.sin(position * div_term)
|
||||||
pe[:, 1::2] = torch.cos(position * div_term)
|
pe[:, 1::2] = torch.cos(position * div_term)
|
||||||
pe = pe.unsqueeze(0)
|
pe = pe.unsqueeze(0)
|
||||||
self.register_buffer('pe', pe)
|
self.register_buffer("pe", pe)
|
||||||
|
|
||||||
def forward(self, seq_len: int):
|
def forward(self, seq_len: int):
|
||||||
if seq_len > self.pe.size(1):
|
if seq_len > self.pe.size(1):
|
||||||
@ -31,6 +33,7 @@ class SinusoidalPositionalEncoding(nn.Module):
|
|||||||
new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1)
|
new_pe = torch.cat([self.pe, extra_pe.to(self.pe.device)], dim=1)
|
||||||
self.pe = new_pe
|
self.pe = new_pe
|
||||||
|
|
||||||
|
|
||||||
class CategorySpecificLinear(nn.Module):
|
class CategorySpecificLinear(nn.Module):
|
||||||
def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1):
|
def __init__(self, in_dim: int, out_dim: int, num_categories: int = 1):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@ -42,18 +45,15 @@ class CategorySpecificLinear(nn.Module):
|
|||||||
self.bias = nn.Parameter(torch.randn(num_categories, out_dim))
|
self.bias = nn.Parameter(torch.randn(num_categories, out_dim))
|
||||||
|
|
||||||
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
|
def forward(self, x: torch.Tensor, category_id: torch.LongTensor):
|
||||||
|
|
||||||
if self.num_categories <= 1:
|
if self.num_categories <= 1:
|
||||||
return self.linear(x)
|
return self.linear(x)
|
||||||
|
|
||||||
orig_shape = x.shape
|
orig_shape = x.shape
|
||||||
x_flat = x.reshape(-1, orig_shape[-1])
|
x_flat = x.reshape(-1, orig_shape[-1])
|
||||||
if category_id.dim() == 0:
|
if category_id.dim() == 0:
|
||||||
|
|
||||||
cid = category_id.item()
|
cid = category_id.item()
|
||||||
out = x_flat @ self.weight[cid] + self.bias[cid]
|
out = x_flat @ self.weight[cid] + self.bias[cid]
|
||||||
else:
|
else:
|
||||||
|
|
||||||
category_id = category_id.view(-1)
|
category_id = category_id.view(-1)
|
||||||
weight_selected = self.weight[category_id]
|
weight_selected = self.weight[category_id]
|
||||||
bias_selected = self.bias[category_id]
|
bias_selected = self.bias[category_id]
|
||||||
@ -61,8 +61,8 @@ class CategorySpecificLinear(nn.Module):
|
|||||||
out_shape = orig_shape[:-1] + (out.shape[-1],)
|
out_shape = orig_shape[:-1] + (out.shape[-1],)
|
||||||
return out.view(out_shape)
|
return out.view(out_shape)
|
||||||
|
|
||||||
class CategorySpecificMLP(nn.Module):
|
|
||||||
|
|
||||||
|
class CategorySpecificMLP(nn.Module):
|
||||||
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1):
|
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_categories: int = 1):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories)
|
self.fc1 = CategorySpecificLinear(input_dim, hidden_dim, num_categories)
|
||||||
@ -74,9 +74,11 @@ class CategorySpecificMLP(nn.Module):
|
|||||||
out = self.fc2(out, category_id)
|
out = self.fc2(out, category_id)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
class MultiEmbodimentActionEncoder(nn.Module):
|
|
||||||
|
|
||||||
def __init__(self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1):
|
class MultiEmbodimentActionEncoder(nn.Module):
|
||||||
|
def __init__(
|
||||||
|
self, action_dim: int, embed_dim: int, hidden_dim: int, horizon: int, num_categories: int = 1
|
||||||
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.horizon = horizon
|
self.horizon = horizon
|
||||||
self.embed_dim = embed_dim
|
self.embed_dim = embed_dim
|
||||||
@ -90,42 +92,35 @@ class MultiEmbodimentActionEncoder(nn.Module):
|
|||||||
self.activation = nn.ReLU(inplace=True)
|
self.activation = nn.ReLU(inplace=True)
|
||||||
|
|
||||||
def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor):
|
def forward(self, action_seq: torch.Tensor, category_id: torch.LongTensor):
|
||||||
|
batch_size, horizon, action_dim = action_seq.shape
|
||||||
|
assert self.horizon == horizon, "Action sequence length must match horizon"
|
||||||
|
|
||||||
B, H, D = action_seq.shape
|
x = action_seq.reshape(batch_size * horizon, action_dim)
|
||||||
assert H == self.horizon, "Action sequence length must match horizon"
|
|
||||||
|
|
||||||
x = action_seq.reshape(B * H, D)
|
|
||||||
|
|
||||||
if category_id.dim() == 0:
|
if category_id.dim() == 0:
|
||||||
|
cat_ids = category_id.repeat(horizon * batch_size)
|
||||||
cat_ids = category_id.repeat(H * B)
|
|
||||||
else:
|
else:
|
||||||
cat_ids = category_id.unsqueeze(1).repeat(1, H).reshape(B * H)
|
cat_ids = category_id.unsqueeze(1).repeat(1, horizon).reshape(batch_size * horizon)
|
||||||
out = self.activation(self.W1(x, cat_ids))
|
out = self.activation(self.W1(x, cat_ids))
|
||||||
|
|
||||||
pos_enc = self.pos_encoding(H).to(out.device)
|
pos_enc = self.pos_encoding(horizon).to(out.device)
|
||||||
pos_enc = pos_enc.repeat(B, 1, 1).reshape(B * H, -1)
|
pos_enc = pos_enc.repeat(batch_size, 1, 1).reshape(batch_size * horizon, -1)
|
||||||
out = out + pos_enc
|
out = out + pos_enc
|
||||||
out = self.activation(self.W2(out, cat_ids))
|
out = self.activation(self.W2(out, cat_ids))
|
||||||
out = self.W3(out, cat_ids)
|
out = self.W3(out, cat_ids)
|
||||||
out = out.view(B, H, self.embed_dim)
|
out = out.view(batch_size, horizon, self.embed_dim)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
class BasicTransformerBlock(nn.Module):
|
|
||||||
|
|
||||||
|
class BasicTransformerBlock(nn.Module):
|
||||||
def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0):
|
def __init__(self, embed_dim: int, num_heads: int, hidden_dim: int, dropout: float = 0.0):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
|
self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
|
||||||
self.norm1 = nn.LayerNorm(embed_dim)
|
self.norm1 = nn.LayerNorm(embed_dim)
|
||||||
self.norm2 = nn.LayerNorm(embed_dim)
|
self.norm2 = nn.LayerNorm(embed_dim)
|
||||||
self.ff = nn.Sequential(
|
self.ff = nn.Sequential(nn.Linear(embed_dim, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, embed_dim))
|
||||||
nn.Linear(embed_dim, hidden_dim),
|
|
||||||
nn.GELU(),
|
|
||||||
nn.Linear(hidden_dim, embed_dim)
|
|
||||||
)
|
|
||||||
|
|
||||||
def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor):
|
def forward(self, action_tokens: torch.Tensor, context_tokens: torch.Tensor, time_emb: torch.Tensor):
|
||||||
|
|
||||||
x = self.norm1(action_tokens)
|
x = self.norm1(action_tokens)
|
||||||
attn_out, _ = self.attn(x, context_tokens, context_tokens)
|
attn_out, _ = self.attn(x, context_tokens, context_tokens)
|
||||||
|
|
||||||
@ -139,23 +134,25 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
x = x + ff_out
|
x = x + ff_out
|
||||||
return x
|
return x
|
||||||
|
|
||||||
class FlowmatchingActionHead(nn.Module):
|
|
||||||
|
|
||||||
def __init__(self, config=None,
|
class FlowmatchingActionHead(nn.Module):
|
||||||
embed_dim: int = 896,
|
def __init__(
|
||||||
hidden_dim: int = 1024,
|
self,
|
||||||
action_dim: int = 16*7,
|
config=None,
|
||||||
horizon: int = 16,
|
embed_dim: int = 896,
|
||||||
per_action_dim: int = 7,
|
hidden_dim: int = 1024,
|
||||||
num_heads: int = 8,
|
action_dim: int = 16 * 7,
|
||||||
num_layers: int = 8,
|
horizon: int = 16,
|
||||||
dropout: float = 0.0,
|
per_action_dim: int = 7,
|
||||||
num_inference_timesteps: int = 20,
|
num_heads: int = 8,
|
||||||
num_categories: int = 1):
|
num_layers: int = 8,
|
||||||
|
dropout: float = 0.0,
|
||||||
|
num_inference_timesteps: int = 20,
|
||||||
|
num_categories: int = 1,
|
||||||
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
if config is not None:
|
if config is not None:
|
||||||
|
|
||||||
embed_dim = getattr(config, "embed_dim", embed_dim)
|
embed_dim = getattr(config, "embed_dim", embed_dim)
|
||||||
hidden_dim = getattr(config, "hidden_dim", hidden_dim)
|
hidden_dim = getattr(config, "hidden_dim", hidden_dim)
|
||||||
action_dim = getattr(config, "action_dim", action_dim)
|
action_dim = getattr(config, "action_dim", action_dim)
|
||||||
@ -168,113 +165,119 @@ class FlowmatchingActionHead(nn.Module):
|
|||||||
self.config = config
|
self.config = config
|
||||||
else:
|
else:
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
self.config = SimpleNamespace(embed_dim=embed_dim, hidden_dim=hidden_dim,
|
|
||||||
action_dim=action_dim, horizon=horizon,
|
self.config = SimpleNamespace(
|
||||||
num_heads=num_heads, num_layers=num_layers,
|
embed_dim=embed_dim,
|
||||||
dropout=dropout, num_inference_timesteps=num_inference_timesteps,
|
hidden_dim=hidden_dim,
|
||||||
num_categories=num_categories)
|
action_dim=action_dim,
|
||||||
|
horizon=horizon,
|
||||||
|
num_heads=num_heads,
|
||||||
|
num_layers=num_layers,
|
||||||
|
dropout=dropout,
|
||||||
|
num_inference_timesteps=num_inference_timesteps,
|
||||||
|
num_categories=num_categories,
|
||||||
|
)
|
||||||
self.embed_dim = embed_dim
|
self.embed_dim = embed_dim
|
||||||
self.horizon = horizon
|
self.horizon = horizon
|
||||||
self.per_action_dim = config.per_action_dim
|
self.per_action_dim = config.per_action_dim
|
||||||
self.action_dim = config.action_dim
|
self.action_dim = config.action_dim
|
||||||
|
|
||||||
|
|
||||||
self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000)
|
self.time_pos_enc = SinusoidalPositionalEncoding(embed_dim, max_len=1000)
|
||||||
|
|
||||||
self.transformer_blocks = nn.ModuleList([
|
self.transformer_blocks = nn.ModuleList(
|
||||||
BasicTransformerBlock(embed_dim=embed_dim, num_heads=num_heads,
|
[
|
||||||
hidden_dim=embed_dim*4, dropout=dropout)
|
BasicTransformerBlock(
|
||||||
for _ in range(num_layers)
|
embed_dim=embed_dim, num_heads=num_heads, hidden_dim=embed_dim * 4, dropout=dropout
|
||||||
])
|
)
|
||||||
|
for _ in range(num_layers)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
self.norm_out = nn.LayerNorm(embed_dim)
|
self.norm_out = nn.LayerNorm(embed_dim)
|
||||||
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim)
|
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim)
|
||||||
|
|
||||||
self.mlp_head = CategorySpecificMLP(input_dim=embed_dim, hidden_dim=hidden_dim,
|
self.mlp_head = CategorySpecificMLP(
|
||||||
output_dim=action_dim, num_categories=num_categories)
|
input_dim=embed_dim, hidden_dim=hidden_dim, output_dim=action_dim, num_categories=num_categories
|
||||||
|
)
|
||||||
|
|
||||||
self.state_encoder = None
|
self.state_encoder = None
|
||||||
if hasattr(self.config, "state_dim") and self.config.state_dim is not None:
|
if hasattr(self.config, "state_dim") and self.config.state_dim is not None:
|
||||||
|
|
||||||
state_hidden = getattr(self.config, "state_hidden_dim", embed_dim)
|
state_hidden = getattr(self.config, "state_hidden_dim", embed_dim)
|
||||||
|
|
||||||
self.state_encoder = CategorySpecificMLP(input_dim=self.config.state_dim,
|
self.state_encoder = CategorySpecificMLP(
|
||||||
hidden_dim=state_hidden,
|
input_dim=self.config.state_dim,
|
||||||
output_dim=embed_dim,
|
hidden_dim=state_hidden,
|
||||||
num_categories=num_categories)
|
output_dim=embed_dim,
|
||||||
|
num_categories=num_categories,
|
||||||
|
)
|
||||||
|
|
||||||
self.action_encoder = None
|
self.action_encoder = None
|
||||||
if horizon > 1:
|
if horizon > 1:
|
||||||
|
|
||||||
per_action_dim = getattr(self.config, "per_action_dim", None)
|
per_action_dim = getattr(self.config, "per_action_dim", None)
|
||||||
if per_action_dim is None:
|
if per_action_dim is None:
|
||||||
|
|
||||||
per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim
|
per_action_dim = action_dim // horizon if action_dim % horizon == 0 else action_dim
|
||||||
self.action_encoder = MultiEmbodimentActionEncoder(action_dim=per_action_dim,
|
self.action_encoder = MultiEmbodimentActionEncoder(
|
||||||
embed_dim=embed_dim,
|
action_dim=per_action_dim,
|
||||||
hidden_dim=embed_dim,
|
embed_dim=embed_dim,
|
||||||
horizon=horizon,
|
hidden_dim=embed_dim,
|
||||||
num_categories=num_categories)
|
horizon=horizon,
|
||||||
|
num_categories=num_categories,
|
||||||
def forward(self, fused_tokens: torch.Tensor, state: torch.Tensor = None,
|
)
|
||||||
actions_gt: torch.Tensor = None, embodiment_id: torch.LongTensor = None,
|
|
||||||
state_mask: torch.Tensor = None, action_mask: torch.Tensor = None):
|
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self,
|
||||||
|
fused_tokens: torch.Tensor,
|
||||||
|
state: torch.Tensor = None,
|
||||||
|
actions_gt: torch.Tensor = None,
|
||||||
|
embodiment_id: torch.LongTensor = None,
|
||||||
|
state_mask: torch.Tensor = None,
|
||||||
|
action_mask: torch.Tensor = None,
|
||||||
|
):
|
||||||
if actions_gt is None:
|
if actions_gt is None:
|
||||||
return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id)
|
return self.get_action(fused_tokens, state=state, embodiment_id=embodiment_id)
|
||||||
B = fused_tokens.size(0)
|
batch_size = fused_tokens.size(0)
|
||||||
device = fused_tokens.device
|
device = fused_tokens.device
|
||||||
|
|
||||||
if embodiment_id is None:
|
if embodiment_id is None:
|
||||||
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
|
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
|
||||||
|
|
||||||
context_tokens = fused_tokens
|
context_tokens = fused_tokens
|
||||||
if state is not None and self.state_encoder is not None:
|
if state is not None and self.state_encoder is not None:
|
||||||
|
|
||||||
|
|
||||||
state_emb = self.state_encoder(state, embodiment_id)
|
state_emb = self.state_encoder(state, embodiment_id)
|
||||||
state_emb = state_emb.unsqueeze(1)
|
state_emb = state_emb.unsqueeze(1)
|
||||||
|
|
||||||
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
|
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
|
||||||
|
|
||||||
t = torch.distributions.Beta(2, 2).sample((B,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
|
t = (
|
||||||
|
torch.distributions.Beta(2, 2).sample((batch_size,)).clamp(0.02, 0.98).to(device).to(dtype=self.dtype)
|
||||||
|
)
|
||||||
|
|
||||||
time_index = (t * 1000).long()
|
time_index = (t * 1000).long()
|
||||||
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
|
time_emb = self.time_pos_enc(1000)[:, time_index, :].squeeze(0)
|
||||||
|
|
||||||
action_shape = actions_gt.shape[1]
|
|
||||||
|
|
||||||
|
|
||||||
actions_gt_seq = actions_gt
|
actions_gt_seq = actions_gt
|
||||||
|
|
||||||
|
|
||||||
noise = torch.rand_like(actions_gt) * 2 - 1
|
noise = torch.rand_like(actions_gt) * 2 - 1
|
||||||
|
|
||||||
if action_mask is not None:
|
if action_mask is not None:
|
||||||
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
|
action_mask = action_mask.to(dtype=noise.dtype, device=noise.device)
|
||||||
assert action_mask.shape == noise.shape, f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
|
assert action_mask.shape == noise.shape, (
|
||||||
|
f"action_mask shape {action_mask.shape} != noise shape {noise.shape}"
|
||||||
|
)
|
||||||
noise = noise * action_mask
|
noise = noise * action_mask
|
||||||
|
|
||||||
|
|
||||||
if self.horizon > 1:
|
if self.horizon > 1:
|
||||||
noise_seq = noise.view(B, self.horizon, self.per_action_dim)
|
noise_seq = noise.view(batch_size, self.horizon, self.per_action_dim)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
noise_seq = noise.unsqueeze(1)
|
noise_seq = noise.unsqueeze(1)
|
||||||
|
|
||||||
if self.horizon > 1:
|
t_broadcast = t.view(batch_size, 1, 1) if self.horizon > 1 else t.view(batch_size, 1)
|
||||||
t_broadcast = t.view(B, 1, 1)
|
|
||||||
else:
|
|
||||||
t_broadcast = t.view(B, 1)
|
|
||||||
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
|
action_intermediate_seq = (1 - t_broadcast) * noise_seq + t_broadcast * actions_gt_seq
|
||||||
|
|
||||||
if self.horizon > 1 and self.action_encoder is not None:
|
if self.horizon > 1 and self.action_encoder is not None:
|
||||||
|
|
||||||
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
|
action_tokens = self.action_encoder(action_intermediate_seq, embodiment_id)
|
||||||
else:
|
else:
|
||||||
|
|
||||||
if not hasattr(self, "single_action_proj"):
|
if not hasattr(self, "single_action_proj"):
|
||||||
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
|
self.single_action_proj = nn.Linear(self.per_action_dim, self.embed_dim).to(device)
|
||||||
action_tokens = self.single_action_proj(action_intermediate_seq)
|
action_tokens = self.single_action_proj(action_intermediate_seq)
|
||||||
@ -286,37 +289,37 @@ class FlowmatchingActionHead(nn.Module):
|
|||||||
x = self.norm_out(x)
|
x = self.norm_out(x)
|
||||||
|
|
||||||
if self.horizon > 1:
|
if self.horizon > 1:
|
||||||
|
|
||||||
x_flat = x.reshape(B, -1)
|
x_flat = x.reshape(B, -1)
|
||||||
|
|
||||||
if not hasattr(self, "seq_pool_proj"):
|
if not hasattr(self, "seq_pool_proj"):
|
||||||
|
|
||||||
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
|
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
|
||||||
x_pooled = self.seq_pool_proj(x_flat)
|
x_pooled = self.seq_pool_proj(x_flat)
|
||||||
else:
|
else:
|
||||||
|
|
||||||
x_pooled = x.squeeze(1)
|
x_pooled = x.squeeze(1)
|
||||||
|
|
||||||
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
|
pred_velocity = self.mlp_head(x_pooled, embodiment_id)
|
||||||
|
|
||||||
return pred_velocity, noise
|
return pred_velocity, noise
|
||||||
|
|
||||||
def get_action(self, fused_tokens: torch.Tensor, state: torch.Tensor = None, embodiment_id: torch.LongTensor = None, action_mask: torch.Tensor = None):
|
def get_action(
|
||||||
|
self,
|
||||||
B = fused_tokens.size(0)
|
fused_tokens: torch.Tensor,
|
||||||
|
state: torch.Tensor = None,
|
||||||
|
embodiment_id: torch.LongTensor = None,
|
||||||
|
action_mask: torch.Tensor = None,
|
||||||
|
):
|
||||||
|
batch_size = fused_tokens.size(0)
|
||||||
device = fused_tokens.device
|
device = fused_tokens.device
|
||||||
if embodiment_id is None:
|
if embodiment_id is None:
|
||||||
embodiment_id = torch.zeros(B, dtype=torch.long, device=device)
|
embodiment_id = torch.zeros(batch_size, dtype=torch.long, device=device)
|
||||||
|
|
||||||
context_tokens = fused_tokens
|
context_tokens = fused_tokens
|
||||||
if state is not None and self.state_encoder is not None:
|
if state is not None and self.state_encoder is not None:
|
||||||
|
|
||||||
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
|
state_emb = self.state_encoder(state, embodiment_id).unsqueeze(1)
|
||||||
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
|
context_tokens = torch.cat([context_tokens, state_emb], dim=1)
|
||||||
|
|
||||||
action_dim_total = getattr(self.config, "action_dim", None)
|
action_dim_total = getattr(self.config, "action_dim", None)
|
||||||
if action_dim_total is None:
|
if action_dim_total is None:
|
||||||
|
|
||||||
action_dim_total = self.action_dim
|
action_dim_total = self.action_dim
|
||||||
|
|
||||||
if self.horizon > 1:
|
if self.horizon > 1:
|
||||||
@ -324,41 +327,40 @@ class FlowmatchingActionHead(nn.Module):
|
|||||||
else:
|
else:
|
||||||
per_action_dim = action_dim_total
|
per_action_dim = action_dim_total
|
||||||
|
|
||||||
action = (torch.rand(B, action_dim_total, device=device) * 2 - 1)
|
action = torch.rand(batch_size, action_dim_total, device=device) * 2 - 1
|
||||||
if self.horizon > 1:
|
if self.horizon > 1:
|
||||||
action_seq = action.view(B, self.horizon, per_action_dim)
|
action_seq = action.view(batch_size, self.horizon, per_action_dim)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
action_seq = action.view(B, 1, per_action_dim)
|
action_seq = action.view(batch_size, 1, per_action_dim)
|
||||||
|
|
||||||
action_mask = action_mask.view(B, 1, per_action_dim).repeat(1,self.horizon,1)
|
action_mask = action_mask.view(batch_size, 1, per_action_dim).repeat(1, self.horizon, 1)
|
||||||
|
|
||||||
if action_mask is not None:
|
if action_mask is not None:
|
||||||
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
|
action_mask = action_mask.to(dtype=action_seq.dtype, device=action_seq.device)
|
||||||
assert action_mask.shape == action_seq.shape, f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
|
assert action_mask.shape == action_seq.shape, (
|
||||||
|
f"action_mask shape {action_mask.shape} != noise shape {action_seq.shape}"
|
||||||
|
)
|
||||||
action_seq = action_seq * action_mask
|
action_seq = action_seq * action_mask
|
||||||
else:
|
else:
|
||||||
raise ValueError("action_mask must be provided for inference with flow matching.")
|
raise ValueError("action_mask must be provided for inference with flow matching.")
|
||||||
|
|
||||||
N = int(getattr(self.config, "num_inference_timesteps", 32))
|
num_steps = int(getattr(self.config, "num_inference_timesteps", 32))
|
||||||
dt = 1.0 / N
|
dt = 1.0 / num_steps
|
||||||
for i in range(N):
|
for i in range(num_steps):
|
||||||
t = i / N
|
t = i / num_steps
|
||||||
|
|
||||||
time_index = int(t * 1000)
|
time_index = int(t * 1000)
|
||||||
time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0)
|
time_emb = self.time_pos_enc(1000)[:, time_index, :].to(device).squeeze(0)
|
||||||
time_emb = time_emb.unsqueeze(0).repeat(B, 1)
|
time_emb = time_emb.unsqueeze(0).repeat(batch_size, 1)
|
||||||
|
|
||||||
|
|
||||||
if self.horizon > 1 and self.action_encoder is not None:
|
if self.horizon > 1 and self.action_encoder is not None:
|
||||||
|
|
||||||
action_seq = action_seq * action_mask
|
action_seq = action_seq * action_mask
|
||||||
action_tokens = self.action_encoder(action_seq, embodiment_id)
|
action_tokens = self.action_encoder(action_seq, embodiment_id)
|
||||||
else:
|
else:
|
||||||
if hasattr(self, "single_action_proj"):
|
if hasattr(self, "single_action_proj"):
|
||||||
action_tokens = self.single_action_proj(action_seq)
|
action_tokens = self.single_action_proj(action_seq)
|
||||||
else:
|
else:
|
||||||
|
|
||||||
self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device)
|
self.single_action_proj = nn.Linear(per_action_dim, self.embed_dim).to(device)
|
||||||
action_tokens = self.single_action_proj(action_seq)
|
action_tokens = self.single_action_proj(action_seq)
|
||||||
|
|
||||||
@ -372,7 +374,6 @@ class FlowmatchingActionHead(nn.Module):
|
|||||||
if hasattr(self, "seq_pool_proj"):
|
if hasattr(self, "seq_pool_proj"):
|
||||||
x_pooled = self.seq_pool_proj(x_flat)
|
x_pooled = self.seq_pool_proj(x_flat)
|
||||||
else:
|
else:
|
||||||
|
|
||||||
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
|
self.seq_pool_proj = nn.Linear(self.horizon * self.embed_dim, self.embed_dim).to(device)
|
||||||
x_pooled = self.seq_pool_proj(x_flat)
|
x_pooled = self.seq_pool_proj(x_flat)
|
||||||
else:
|
else:
|
||||||
@ -383,18 +384,16 @@ class FlowmatchingActionHead(nn.Module):
|
|||||||
action = action + dt * pred
|
action = action + dt * pred
|
||||||
|
|
||||||
if self.horizon > 1:
|
if self.horizon > 1:
|
||||||
action_seq = action.view(B, self.horizon, per_action_dim)
|
action_seq = action.view(batch_size, self.horizon, per_action_dim)
|
||||||
else:
|
else:
|
||||||
action_seq = action.view(B, 1, per_action_dim)
|
action_seq = action.view(batch_size, 1, per_action_dim)
|
||||||
|
|
||||||
return action
|
return action
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def device(self):
|
def device(self):
|
||||||
|
|
||||||
return next(self.parameters()).device
|
return next(self.parameters()).device
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def dtype(self):
|
def dtype(self):
|
||||||
|
|
||||||
return next(self.parameters()).dtype
|
return next(self.parameters()).dtype
|
||||||
|
|||||||
@ -1,29 +1,34 @@
|
|||||||
import logging
|
import logging
|
||||||
from typing import Union
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torchvision.transforms as T
|
import torchvision.transforms as transforms
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from torchvision.transforms.functional import InterpolationMode
|
from torchvision.transforms.functional import InterpolationMode, to_pil_image
|
||||||
from torchvision.transforms.functional import to_pil_image
|
|
||||||
from transformers import AutoModel, AutoTokenizer
|
from transformers import AutoModel, AutoTokenizer
|
||||||
|
|
||||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||||
IMAGENET_STD = (0.229, 0.224, 0.225)
|
IMAGENET_STD = (0.229, 0.224, 0.225)
|
||||||
|
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>" # nosec B105
|
||||||
|
IMG_START_TOKEN = "<img>" # nosec B105
|
||||||
|
IMG_END_TOKEN = "</img>" # nosec B105
|
||||||
|
|
||||||
|
|
||||||
# === Image Transformations ===
|
# === Image Transformations ===
|
||||||
def build_transform(input_size):
|
def build_transform(input_size):
|
||||||
return T.Compose([
|
return transforms.Compose(
|
||||||
T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
|
[
|
||||||
T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
|
transforms.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
|
||||||
T.ToTensor(),
|
transforms.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
|
||||||
T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
|
transforms.ToTensor(),
|
||||||
])
|
transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# === Aspect Ratio Handling ===
|
# === Aspect Ratio Handling ===
|
||||||
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
||||||
best_ratio_diff = float('inf')
|
best_ratio_diff = float("inf")
|
||||||
best_ratio = (1, 1)
|
best_ratio = (1, 1)
|
||||||
area = width * height
|
area = width * height
|
||||||
for ratio in target_ratios:
|
for ratio in target_ratios:
|
||||||
@ -36,15 +41,21 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_
|
|||||||
best_ratio = ratio
|
best_ratio = ratio
|
||||||
return best_ratio
|
return best_ratio
|
||||||
|
|
||||||
|
|
||||||
def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False):
|
def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnail=False):
|
||||||
orig_width, orig_height = image.size
|
orig_width, orig_height = image.size
|
||||||
aspect_ratio = orig_width / orig_height
|
aspect_ratio = orig_width / orig_height
|
||||||
target_ratios = set(
|
target_ratios = {
|
||||||
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
|
(i, j)
|
||||||
i * j <= max_num and i * j >= min_num)
|
for n in range(min_num, max_num + 1)
|
||||||
|
for i in range(1, n + 1)
|
||||||
|
for j in range(1, n + 1)
|
||||||
|
if i * j <= max_num and i * j >= min_num
|
||||||
|
}
|
||||||
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
||||||
target_aspect_ratio = find_closest_aspect_ratio(
|
target_aspect_ratio = find_closest_aspect_ratio(
|
||||||
aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
aspect_ratio, target_ratios, orig_width, orig_height, image_size
|
||||||
|
)
|
||||||
target_width = image_size * target_aspect_ratio[0]
|
target_width = image_size * target_aspect_ratio[0]
|
||||||
target_height = image_size * target_aspect_ratio[1]
|
target_height = image_size * target_aspect_ratio[1]
|
||||||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||||||
@ -55,7 +66,7 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
|
|||||||
(i % (target_width // image_size)) * image_size,
|
(i % (target_width // image_size)) * image_size,
|
||||||
(i // (target_width // image_size)) * image_size,
|
(i // (target_width // image_size)) * image_size,
|
||||||
((i % (target_width // image_size)) + 1) * image_size,
|
((i % (target_width // image_size)) + 1) * image_size,
|
||||||
((i // (target_width // image_size)) + 1) * image_size
|
((i // (target_width // image_size)) + 1) * image_size,
|
||||||
)
|
)
|
||||||
split_img = resized_img.crop(box)
|
split_img = resized_img.crop(box)
|
||||||
processed_images.append(split_img)
|
processed_images.append(split_img)
|
||||||
@ -65,6 +76,7 @@ def dynamic_preprocess(image, min_num=1, max_num=1, image_size=448, use_thumbnai
|
|||||||
processed_images.append(thumbnail_img)
|
processed_images.append(thumbnail_img)
|
||||||
return processed_images
|
return processed_images
|
||||||
|
|
||||||
|
|
||||||
class InternVL3Embedder(nn.Module):
|
class InternVL3Embedder(nn.Module):
|
||||||
def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"):
|
def __init__(self, model_name="OpenGVLab/InternVL3-1B", image_size=448, device="cuda"):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@ -82,14 +94,14 @@ class InternVL3Embedder(nn.Module):
|
|||||||
_fast_init=False,
|
_fast_init=False,
|
||||||
).to(self.device)
|
).to(self.device)
|
||||||
|
|
||||||
if hasattr(self.model.language_model, 'model'):
|
if hasattr(self.model.language_model, "model"):
|
||||||
layers = self.model.language_model.model.layers
|
layers = self.model.language_model.model.layers
|
||||||
|
|
||||||
else:
|
else:
|
||||||
layers = self.model.language_model.layers
|
layers = self.model.language_model.layers
|
||||||
layers = layers[:14]
|
layers = layers[:14]
|
||||||
|
|
||||||
if hasattr(self.model.language_model, 'model'):
|
if hasattr(self.model.language_model, "model"):
|
||||||
self.model.language_model.model.layers = torch.nn.ModuleList(layers)
|
self.model.language_model.model.layers = torch.nn.ModuleList(layers)
|
||||||
else:
|
else:
|
||||||
self.model.language_model.layers = torch.nn.ModuleList(layers)
|
self.model.language_model.layers = torch.nn.ModuleList(layers)
|
||||||
@ -98,14 +110,11 @@ class InternVL3Embedder(nn.Module):
|
|||||||
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
if hasattr(self.model, "vision_model") and hasattr(self.model.vision_model, "encoder"):
|
||||||
self.model.vision_model.encoder.gradient_checkpointing = False
|
self.model.vision_model.encoder.gradient_checkpointing = False
|
||||||
|
|
||||||
|
|
||||||
def _preprocess_images(
|
def _preprocess_images(
|
||||||
self,
|
self, image_tensors: list[Image.Image | torch.Tensor]
|
||||||
image_tensors: list[Union[Image.Image, torch.Tensor]]
|
|
||||||
) -> tuple[torch.Tensor, list[int]]:
|
) -> tuple[torch.Tensor, list[int]]:
|
||||||
|
|
||||||
pixel_values_list = []
|
pixel_values_list = []
|
||||||
for i, image in enumerate(image_tensors):
|
for image in image_tensors:
|
||||||
if isinstance(image, torch.Tensor):
|
if isinstance(image, torch.Tensor):
|
||||||
image = to_pil_image(image)
|
image = to_pil_image(image)
|
||||||
tiles = dynamic_preprocess(image, image_size=self.image_size)
|
tiles = dynamic_preprocess(image, image_size=self.image_size)
|
||||||
@ -117,21 +126,12 @@ class InternVL3Embedder(nn.Module):
|
|||||||
|
|
||||||
return pixel_values, num_tiles_list
|
return pixel_values, num_tiles_list
|
||||||
|
|
||||||
def _build_multimodal_prompt(
|
def _build_multimodal_prompt(self, num_tiles_list: list[int], text_prompt: str) -> str:
|
||||||
self,
|
prompt = ""
|
||||||
num_tiles_list: list[int],
|
|
||||||
text_prompt: str
|
|
||||||
) -> str:
|
|
||||||
|
|
||||||
prompt = ''
|
|
||||||
for i in range(len(num_tiles_list)):
|
for i in range(len(num_tiles_list)):
|
||||||
prompt += f"Image-{i+1}: <image>\n"
|
prompt += f"Image-{i + 1}: <image>\n"
|
||||||
prompt += text_prompt.strip()
|
prompt += text_prompt.strip()
|
||||||
|
|
||||||
IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
|
|
||||||
IMG_START_TOKEN = "<img>"
|
|
||||||
IMG_END_TOKEN = "</img>"
|
|
||||||
|
|
||||||
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
||||||
for tile_count in num_tiles_list:
|
for tile_count in num_tiles_list:
|
||||||
token_count = self.model.num_image_token * tile_count
|
token_count = self.model.num_image_token * tile_count
|
||||||
@ -141,13 +141,8 @@ class InternVL3Embedder(nn.Module):
|
|||||||
return prompt
|
return prompt
|
||||||
|
|
||||||
def _prepare_and_fuse_embeddings(
|
def _prepare_and_fuse_embeddings(
|
||||||
self,
|
self, prompt: str, vit_embeds: torch.Tensor, image_mask: torch.Tensor, num_tiles_list: list[int]
|
||||||
prompt: str,
|
|
||||||
vit_embeds: torch.Tensor,
|
|
||||||
image_mask: torch.Tensor,
|
|
||||||
num_tiles_list: list[int]
|
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
|
||||||
untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
|
untruncated_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
|
||||||
true_sequence_length = untruncated_ids.shape[1]
|
true_sequence_length = untruncated_ids.shape[1]
|
||||||
|
|
||||||
@ -159,26 +154,29 @@ class InternVL3Embedder(nn.Module):
|
|||||||
prompt[:100],
|
prompt[:100],
|
||||||
)
|
)
|
||||||
|
|
||||||
model_inputs = self.tokenizer(prompt, return_tensors="pt", padding='max_length', truncation=True, max_length=self.max_text_length).to(self.device)
|
model_inputs = self.tokenizer(
|
||||||
|
prompt,
|
||||||
|
return_tensors="pt",
|
||||||
|
padding="max_length",
|
||||||
|
truncation=True,
|
||||||
|
max_length=self.max_text_length,
|
||||||
|
).to(self.device)
|
||||||
input_ids = model_inputs["input_ids"]
|
input_ids = model_inputs["input_ids"]
|
||||||
attention_mask = model_inputs["attention_mask"]
|
attention_mask = model_inputs["attention_mask"]
|
||||||
|
|
||||||
|
img_token_mask = input_ids == self.img_context_token_id
|
||||||
img_token_mask = (input_ids == self.img_context_token_id)
|
|
||||||
|
|
||||||
img_token_locations = torch.where(img_token_mask)[1]
|
img_token_locations = torch.where(img_token_mask)[1]
|
||||||
|
|
||||||
|
|
||||||
input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone()
|
input_embeds = self.model.language_model.get_input_embeddings()(input_ids).clone()
|
||||||
|
|
||||||
B, N, C = input_embeds.shape
|
batch_size, seq_len, channels = input_embeds.shape
|
||||||
input_embeds = input_embeds.reshape(B * N, C)
|
input_embeds = input_embeds.reshape(batch_size * seq_len, channels)
|
||||||
input_ids = input_ids.reshape(B * N)
|
input_ids = input_ids.reshape(batch_size * seq_len)
|
||||||
|
|
||||||
selected = (input_ids == self.img_context_token_id)
|
selected = input_ids == self.img_context_token_id
|
||||||
|
|
||||||
|
vit_embeds = vit_embeds.reshape(-1, channels).to(device=input_embeds.device, dtype=input_embeds.dtype)
|
||||||
vit_embeds = vit_embeds.reshape(-1, C).to(device=input_embeds.device, dtype=input_embeds.dtype)
|
|
||||||
n_token = int(selected.sum().item())
|
n_token = int(selected.sum().item())
|
||||||
if vit_embeds.shape[0] < n_token:
|
if vit_embeds.shape[0] < n_token:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@ -186,17 +184,14 @@ class InternVL3Embedder(nn.Module):
|
|||||||
)
|
)
|
||||||
input_embeds[selected] = vit_embeds[:n_token]
|
input_embeds[selected] = vit_embeds[:n_token]
|
||||||
|
|
||||||
|
|
||||||
tokens_per_tile = self.model.num_image_token
|
tokens_per_tile = self.model.num_image_token
|
||||||
|
|
||||||
current_token_idx = 0
|
current_token_idx = 0
|
||||||
for i in range(len(image_mask)):
|
for i in range(len(image_mask)):
|
||||||
|
|
||||||
num_tiles_for_this_image = num_tiles_list[i]
|
num_tiles_for_this_image = num_tiles_list[i]
|
||||||
num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile
|
num_tokens_for_this_image = num_tiles_for_this_image * tokens_per_tile
|
||||||
|
|
||||||
if not image_mask[i]:
|
if not image_mask[i]:
|
||||||
|
|
||||||
start_idx = img_token_locations[current_token_idx]
|
start_idx = img_token_locations[current_token_idx]
|
||||||
end_idx = start_idx + num_tokens_for_this_image
|
end_idx = start_idx + num_tokens_for_this_image
|
||||||
|
|
||||||
@ -204,31 +199,28 @@ class InternVL3Embedder(nn.Module):
|
|||||||
|
|
||||||
current_token_idx += num_tokens_for_this_image
|
current_token_idx += num_tokens_for_this_image
|
||||||
|
|
||||||
input_embeds = input_embeds.reshape(B, N, C)
|
input_embeds = input_embeds.reshape(batch_size, seq_len, channels)
|
||||||
|
|
||||||
return input_embeds, attention_mask
|
return input_embeds, attention_mask
|
||||||
|
|
||||||
|
|
||||||
def get_fused_image_text_embedding_from_tensor_images(
|
def get_fused_image_text_embedding_from_tensor_images(
|
||||||
self,
|
self,
|
||||||
image_tensors: list[Union[Image.Image, torch.Tensor]],
|
image_tensors: list[Image.Image | torch.Tensor],
|
||||||
image_mask: torch.Tensor,
|
image_mask: torch.Tensor,
|
||||||
text_prompt: str,
|
text_prompt: str,
|
||||||
return_cls_only: bool = True,
|
return_cls_only: bool = True,
|
||||||
):
|
):
|
||||||
|
|
||||||
|
|
||||||
pixel_values, num_tiles_list = self._preprocess_images(image_tensors)
|
pixel_values, num_tiles_list = self._preprocess_images(image_tensors)
|
||||||
|
|
||||||
|
|
||||||
if pixel_values.shape[0] == 0:
|
if pixel_values.shape[0] == 0:
|
||||||
|
|
||||||
logging.warning("InternVL3 received an empty image batch after preprocessing.")
|
logging.warning("InternVL3 received an empty image batch after preprocessing.")
|
||||||
|
|
||||||
vit_embeds = self.model.extract_feature(pixel_values)
|
vit_embeds = self.model.extract_feature(pixel_values)
|
||||||
fused_embeds = vit_embeds
|
fused_embeds = vit_embeds
|
||||||
prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt)
|
prompt = self._build_multimodal_prompt(num_tiles_list, text_prompt)
|
||||||
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(prompt, fused_embeds, image_mask, num_tiles_list)
|
inputs_embeds, attention_mask = self._prepare_and_fuse_embeddings(
|
||||||
|
prompt, fused_embeds, image_mask, num_tiles_list
|
||||||
|
)
|
||||||
|
|
||||||
outputs = self.model.language_model(
|
outputs = self.model.language_model(
|
||||||
inputs_embeds=inputs_embeds,
|
inputs_embeds=inputs_embeds,
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户