[PHP] 基于 EasyCV 复现 DETR 和 DAB-DETR，Object Query 的正确打开方式

# 一层encoder代码如下
class TransformerEncoderLayer(nn.Module):
  def __init__(self,          d_model,          nhead,          dim_feedforward=2048,          dropout=0.1,          activation='relu',          normalize_before=False):
    super().__init__()
    self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
    # Implementation of Feedforward model
    self.linear1 = nn.Linear(d_model, dim_feedforward)
    self.dropout = nn.Dropout(dropout)
    self.linear2 = nn.Linear(dim_feedforward, d_model)
    self.norm1 = nn.LayerNorm(d_model)
    self.norm2 = nn.LayerNorm(d_model)
    self.dropout1 = nn.Dropout(dropout)
    self.dropout2 = nn.Dropout(dropout)
    self.activation = _get_activation_fn(activation)
    self.normalize_before = normalize_before
  def with_pos_embed(self, tensor, pos: Optional[Tensor]):
    return tensor if pos is None else tensor + pos
  def forward(self,         src,         src_mask: Optional[Tensor] = None,         src_key_padding_mask: Optional[Tensor] = None,         pos: Optional[Tensor] = None):
    q = k = self.with_pos_embed(src, pos)
    src2 = self.self_attn(
      q,
      k,
      value=src,
      attn_mask=src_mask,
      key_padding_mask=src_key_padding_mask)[0]
    src = src + self.dropout1(src2)
    src = self.norm1(src)
    src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
    src = src + self.dropout2(src2)
    src = self.norm2(src)
    return src

# 一层decoder代码如下
class TransformerDecoderLayer(nn.Module):
  def __init__(self,          d_model,          nhead,          dim_feedforward=2048,          dropout=0.1,          activation='relu',          normalize_before=False):
    super().__init__()
    self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
    self.multihead_attn = nn.MultiheadAttention(
      d_model, nhead, dropout=dropout)
    # Implementation of Feedforward model
    self.linear1 = nn.Linear(d_model, dim_feedforward)
    self.dropout = nn.Dropout(dropout)
    self.linear2 = nn.Linear(dim_feedforward, d_model)
    self.norm1 = nn.LayerNorm(d_model)
    self.norm2 = nn.LayerNorm(d_model)
    self.norm3 = nn.LayerNorm(d_model)
    self.dropout1 = nn.Dropout(dropout)
    self.dropout2 = nn.Dropout(dropout)
    self.dropout3 = nn.Dropout(dropout)
    self.activation = _get_activation_fn(activation)
    self.normalize_before = normalize_before
  def with_pos_embed(self, tensor, pos: Optional[Tensor]):
    return tensor if pos is None else tensor + pos
  def forward(self,          tgt,          memory,          tgt_mask: Optional[Tensor] = None,          memory_mask: Optional[Tensor] = None,          tgt_key_padding_mask: Optional[Tensor] = None,          memory_key_padding_mask: Optional[Tensor] = None,          pos: Optional[Tensor] = None,          query_pos: Optional[Tensor] = None):
    q = k = self.with_pos_embed(tgt, query_pos)
    tgt2 = self.self_attn(
      q,
      k,
      value=tgt,
      attn_mask=tgt_mask,
      key_padding_mask=tgt_key_padding_mask)[0]
    tgt = tgt + self.dropout1(tgt2)
    tgt = self.norm1(tgt)
    tgt2 = self.multihead_attn(
      query=self.with_pos_embed(tgt, query_pos),
      key=self.with_pos_embed(memory, pos),
      value=memory,
      attn_mask=memory_mask,
      key_padding_mask=memory_key_padding_mask)[0]
    tgt = tgt + self.dropout2(tgt2)
    tgt = self.norm2(tgt)
    tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
    tgt = tgt + self.dropout3(tgt2)
    tgt = self.norm3(tgt)
    return tgt

class MLP(nn.Module):
  """ Very simple multi-layer perceptron (also called FFN)"""
  def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
    super().__init__()
    self.num_layers = num_layers
    h = [hidden_dim] * (num_layers - 1)
    self.layers = nn.ModuleList(
      nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))
  def forward(self, x):
    for i, layer in enumerate(self.layers):
      x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
    return x

DETR 模型训练的主要困难是如何根据 gt 衡量预测结果(类别、位置、数量)。DETR 提出的 loss 函数可以产生 pred 和 gt 的最优双边匹配(确定 pred 和 gt 的一对一关系)，然后优化 loss。将\(y\)表示为 gt 的集合， 表示为\(N\)个预测结果的集合。假设\(N\)大于图片目标数，\(y\)可以认为是用空类(无目标)填充的大小为\(N\)的集合。搜索两个集合\(N\)个元素 的不同排列顺序，使得 loss 尽可能的小的排列顺序即为二分图最大匹配(Bipartite Matching)，公式如下：

其中 表示 pred 和 gt 关于\(σ(i)\)元素\(i\)的匹配 loss。其中二分图匹配通过匈牙利算法(Hungarian algorithm)得到。匹配 loss 同时考虑了 pred class 和 pred box 的准确性。每个 gt 的元素i可以看成\(yi=(c_i,b_i)\)，\(c_i\)表示 class label(可能是空类)\(b_i\)表示 gt box，将元素\(i\)二分图匹配指定的 pred class 表示为 ，pred box 表示为 。

# HungarianMatcher通过计算出cost_bbox,cost_class,cost_giou来一对一匹配预测框和gt框，然后返回匹配的索引对，最后通过索引对计算出loss值
# Final cost matrix
C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou
C = C.view(bs, num_queries, -1).cpu()
sizes = [len(v['boxes']) for v in targets]
indices = [
  linear_sum_assignment(c[i])
  for i, c in enumerate(C.split(sizes, -1))
]
return [(torch.as_tensor(i, dtype=torch.int64),
     torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]

# DAB-DETR将object query显示的拆分为content和pos两种属性
# 将query_embed显示的表示为xywh，表示pos属性，通过MLP学习成256维的pos特征
self.query_embed = nn.Embedding(num_queries, query_dim)
# get sine embedding for the query vector
reference_points = self.query_embed.sigmoid()
obj_center = reference_points[..., :2]
query_sine_embed = gen_sineembed_for_position(obj_center)
query_pos = self.ref_point_head(query_sine_embed)
# content_embed初始化为全0的256维特征
tgt = torch.zeros(
          self.num_queries,
          bs,
          self.embed_dims,
          device=query_embed.device)

# 通过MLP的学习，调整query_sine_embed的attn位置，进一步加快收敛速度
# modulated HW attentions
if self.modulate_hw_attn:
  refHW_cond = self.ref_anchor_head(
    output).sigmoid()  # nq, bs, 2
  query_sine_embed[..., self.d_model //
           2:] *= (refHW_cond[..., 0] /
               obj_center[..., 2]).unsqueeze(-1)
  query_sine_embed[..., :self.d_model //
           2] *= (refHW_cond[..., 1] /
              obj_center[..., 3]).unsqueeze(-1)

wget http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/small_coco_demo/small_coco_demo.tar.gz && tar -zxf small_coco_demo.tar.gz
mkdir -p data/  && mv small_coco_demo data/coco

data/coco/
├── annotations
│   ├── instances_train2017.json
│   └── instances_val2017.json
├── train2017
│   ├── 000000005802.jpg
│   ├── 000000060623.jpg
│   ├── 000000086408.jpg
│   ├── 000000118113.jpg
│   ├── 000000184613.jpg
│   ├── 000000193271.jpg
│   ├── 000000222564.jpg
│     ...
│   └── 000000574769.jpg
└── val2017
  ├── 000000006818.jpg
  ├── 000000017627.jpg
  ├── 000000037777.jpg
  ├── 000000087038.jpg
  ├── 000000174482.jpg
  ├── 000000181666.jpg
  ├── 000000184791.jpg
  ├── 000000252219.jpg
     ...
  └── 000000522713.jpg

# 查看easycv安装位置
import easycv
print(easycv.__file__)
export PythonPATH=$PYTHONPATH:root/EasyCV

单机8卡：
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -m 
torch.distributed.launch --
nproc_per_node=8 --
master_port=29500 tools/train.py 
configs/detection/dab-
detr/dab_detr_r50_8x2_50e_coco.p
y --work_dir easycv/dab_detr -
-launcher pytorch

CUDA_VISIBLE_DEVICES=0,1,2,3,4,5
,6,7 python -m 
torch.distributed.launch --
nproc_per_node=8 --
master_port=29500 tools/eval.py 
configs/detection/dab-
detr/dab_detr_r50_8x2_50e_coco.p
y easycv/dab_detr/epoch_50.pth -
-launcher pytorch --eval