history with channel-concat and 1 frame

这个提交包含在:
Tony Zhao
2023-11-11 00:57:32 -08:00
父节点 d89430f987
当前提交 ace063440d
共有 7 个文件被更改,包括 92 次插入 和 13 次删除

查看文件

@ -111,6 +111,22 @@ class Joiner(nn.Sequential):
return out, pos
class JoinerChannelConcat(nn.Sequential):
def __init__(self, backbone, position_embedding, conv):
super().__init__(backbone, position_embedding, conv)
def forward(self, tensor_list: NestedTensor):
tensor_list = self[2](tensor_list) # 6 to 3 channels
xs = self[0](tensor_list)
out: List[NestedTensor] = []
pos = []
for name, x in xs.items():
out.append(x)
# position encoding
pos.append(self[1](x).to(x.dtype))
return out, pos
def build_backbone(args):
position_embedding = build_position_encoding(args)
@ -120,3 +136,15 @@ def build_backbone(args):
model = Joiner(backbone, position_embedding)
model.num_channels = backbone.num_channels
return model
def build_backbone_channel_concat(args):
position_embedding = build_position_encoding(args)
train_backbone = args.lr_backbone > 0
return_interm_layers = args.masks
backbone = Backbone(args.backbone, train_backbone, return_interm_layers, args.dilation)
conv = nn.Conv2d(in_channels=6, out_channels=3, kernel_size=3, stride=1, padding=1)
model = JoinerChannelConcat(backbone, position_embedding, conv)
model.num_channels = backbone.num_channels
return model

查看文件

@ -6,7 +6,7 @@ import torch
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
from .backbone import build_backbone
from .backbone import build_backbone, build_backbone_channel_concat
from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer
import numpy as np
@ -274,8 +274,12 @@ def build(args):
# From image
backbones = []
for _ in args.camera_names:
backbone = build_backbone(args)
backbones.append(backbone)
if args.history >= 1:
backbone = build_backbone_channel_concat(args)
backbones.append(backbone)
else:
backbone = build_backbone(args)
backbones.append(backbone)
transformer = build_transformer(args)