1

我正在尝试构建此处定义的 Faster-rcnn 架构。有三个主要部分,特征提取器、区域提议网络(rpn)和检测/分类网络。我能够使用 Resnet 架构构建特征提取器,但我在 region-proposal-network 上苦苦挣扎。在 Faster-rcnn 论文中,他们描述了在图像上放置锚点,并计算每个锚点与最近的注释对象的交集比联合(iou)。

“RPN 进行两种不同类型的预测:二元分类和边界框回归调整。

对于训练,我们将所有锚点放入两个不同的 > 类别中。那些与地面实况对象重叠的 > 联合交点 (IoU) 大于 0.5 的那些被认为是 >“前景”,那些不与任何地面实况对象重叠或 > 与地面实况对象的 IoU 小于 0.1 的那些被认为是考虑>“背景”。

然后,我们随机抽样这些锚点,形成一个大小>256 的小批量——试图保持前景和>背景锚点之间的平衡比例。

RPN 使用为小批量选择的所有锚点来计算 > 使用二元交叉熵的分类损失。然后,它只使用那些标记为前景的小批量锚来计算回归损失。为了计算回归的目标,我们 > 使用前景锚点和最近的地面实况对象并 > 计算将锚点转换为 > 对象所需的正确 \DeltaΔ。”来源

在我的尝试中,我能够计算每个锚点的 iou,但我无法弄清楚如何获取锚点的“小批量”(保持前景和背景的平衡)并将损失反向传播到这些单个锚点。我正在使用 tensorflow(带有 tf.keras.Model 的版本 2)。我不能打电话rpn_model.fit(x,y),因为我们不适合所有的锚。我也不能打电话rpn_model.minimize(loss),因为再一次,我们只想将损失应用于 256/(x 个锚点)。

那么,如何将分类和回归损失从 Tensorflow/keras 模型反向传播到特定输出(锚的 mini_batch)?谢谢你。

更新 我花了一整天的时间来重构我的代码以使用 Tensorflow 操作,但它仍然无法正常工作!

ValueError: 变量 shape=(1, 1, 3, 1) dtype=float32> 具有None渐变。请确保 > 确保您的所有操作都定义了渐变(即 > 可微分)。没有梯度的常见操作:K.argmax、K.round、K.eval。

    def __init__(self, tf, resnet_model, image_height, image_width, in_channels, mid_channels, base_anchor_size, anchor_ratios, anchor_scales, subsample_rate):
        """
            in_channels for resnet: 512
            mid_channels: 512
            base_anchor_size: 128
            anchor_ratios: [[1,1],[1,2],[2,1]]
            anchor_scales: [1,2,3,4]
            subsample_rate: (image_size//feature_map_size), ex. (1000//32) --> 31
        """
        self.tf = tf
        self.resnet_model = resnet_model
        self.resnet_model_inputs = self.resnet_model.input
        self.resnet_model_feature_map = resnet_model.output
        self.image_height = image_height
        self.image_width = image_width
        self.in_channels = in_channels
        self.mid_channels = mid_channels
        self.base_anchor_size = base_anchor_size
        self.anchor_ratios = anchor_ratios
        self.anchor_scales = anchor_scales
        self.subsample_rate = subsample_rate


        self.anchor_boxes = self.generate_anchor_boxes(self.base_anchor_size, self.anchor_ratios, self.anchor_scales)
        self.anchor_boxes_over_image = self.tf.convert_to_tensor(self.generate_anchor_boxes_over_image(self.anchor_boxes, self.image_height, self.image_width, self.subsample_rate), dtype=self.tf.float64)
        self.total_number_of_anchor_boxes = self.tf.size(self.anchor_boxes_over_image)

        self.model = self.create_model()

    def get_model(self):
        return self.model

    def create_model(self):
        self.tf.keras.backend.set_floatx('float64')
        rpn_conv = self.tf.keras.layers.SeparableConv2D(filters=512, kernel_size=[3, 3], strides=[1, 1], padding="same", data_format="channels_last", activation="relu")(self.resnet_model.output)
        rpn_class_score = self.tf.keras.layers.SeparableConv2D(filters=24, kernel_size=[1, 1], strides=[1, 1], padding="valid", data_format="channels_last", activation="softmax")(rpn_conv)
        rpn_class_score_shape = rpn_class_score.get_shape().as_list()
        rpn_class_score = self.tf.keras.layers.Reshape(rpn_class_score_shape[1:3] + [rpn_class_score_shape[3] // 2, 2])(rpn_class_score)
        rpn_class_score = self.tf.keras.backend.cast(rpn_class_score, dtype='float64')

        rpn_bbox_pred = self.tf.keras.layers.SeparableConv2D(filters=48, kernel_size=[1, 1], strides=[1, 1], padding="valid", data_format="channels_last")(rpn_conv)
        rpn_bbox_pred_shape = rpn_bbox_pred.get_shape().as_list()
        rpn_bbox_pred = self.tf.keras.layers.Reshape(rpn_bbox_pred_shape[1:3] + [rpn_bbox_pred_shape[3] // 4, 4])(rpn_bbox_pred)
        rpn_bbox_pred = self.tf.keras.backend.cast(rpn_bbox_pred, dtype='float64')

        rpn_model = self.tf.keras.Model(inputs=self.resnet_model.input, outputs=[rpn_class_score, rpn_bbox_pred])

        self.optimizer = self.tf.keras.optimizers.Nadam(0.001)
        return rpn_model

    def get_iou(self, anchor_box_predictions, ground_truth_bounding_box, giou=False):
        ground_truth_bounding_box = self.tf.convert_to_tensor(ground_truth_bounding_box, dtype=self.tf.float64)
        """ Predicted and ground truth bounding box coordinates """
        anchor_box_predictions_center_x, anchor_box_predictions_center_y, anchor_box_predictions_width, anchor_box_predictions_height = self.tf.slice(anchor_box_predictions, [0, 0, 0, 0, 0], [-1, -1, -1, -1, 1]), self.tf.slice(anchor_box_predictions, [0, 0, 0, 0, 1], [-1, -1, -1, -1, 1]), self.tf.slice(anchor_box_predictions, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1]), self.tf.slice(anchor_box_predictions, [0, 0, 0, 0, 3], [-1, -1, -1, -1, 1])
        anchor_box_predictions_x1 = anchor_box_predictions_center_x - (anchor_box_predictions_width / 2)
        anchor_box_predictions_x2 = anchor_box_predictions_center_x + (anchor_box_predictions_width / 2)
        anchor_box_predictions_y1 = anchor_box_predictions_center_y - (anchor_box_predictions_height / 2)
        anchor_box_predictions_y2 = anchor_box_predictions_center_y + (anchor_box_predictions_height / 2)

        """ For the predicted box ensure x2>x1 and y2>y1: """
        anchor_box_predictions_x1, anchor_box_predictions_x2 = self.tf.minimum(anchor_box_predictions_x1, anchor_box_predictions_x2), self.tf.maximum(anchor_box_predictions_x1, anchor_box_predictions_x2)
        anchor_box_predictions_y1, anchor_box_predictions_y2 = self.tf.minimum(anchor_box_predictions_y1, anchor_box_predictions_y2), self.tf.maximum(anchor_box_predictions_y1, anchor_box_predictions_y2)

        """ ground truth bounding box coordinates """
        ground_truth_bounding_boxes_center_x, ground_truth_bounding_boxes_center_y, ground_truth_bounding_boxes_width, ground_truth_bounding_boxes_height = ground_truth_bounding_box[:,0:1], ground_truth_bounding_box[:,1:2], ground_truth_bounding_box[:,2:3], ground_truth_bounding_box[:,3:4]

        ground_truth_bounding_boxes_x1 = ground_truth_bounding_boxes_center_x - (ground_truth_bounding_boxes_width / 2)
        ground_truth_bounding_boxes_x2 = ground_truth_bounding_boxes_center_x + (ground_truth_bounding_boxes_width / 2)
        ground_truth_bounding_boxes_y1 = ground_truth_bounding_boxes_center_y - (ground_truth_bounding_boxes_height / 2)
        ground_truth_bounding_boxes_y2 = ground_truth_bounding_boxes_center_y + (ground_truth_bounding_boxes_height / 2)
        ground_truth_bounding_boxes_x1, ground_truth_bounding_boxes_x2, ground_truth_bounding_boxes_y1, ground_truth_bounding_boxes_y2 = self.tf.reshape(ground_truth_bounding_boxes_x1, [1,1,1,1,-1]), self.tf.reshape(ground_truth_bounding_boxes_x2, [1,1,1,1,-1]), self.tf.reshape(ground_truth_bounding_boxes_y1, [1,1,1,1,-1]), self.tf.reshape(ground_truth_bounding_boxes_y2, [1,1,1,1,-1])


        """ Get areas of boxes """
        anchor_box_predictions_area = (anchor_box_predictions_x2 - anchor_box_predictions_x1) * (anchor_box_predictions_y2 - anchor_box_predictions_y1)
        ground_truth_bounding_boxes_area = (ground_truth_bounding_boxes_x2 - ground_truth_bounding_boxes_x1) * (ground_truth_bounding_boxes_y2 - ground_truth_bounding_boxes_y1)

        """ Calculate intersection between prediction boxes and ground truth boxes """

        x1_intersection, x2_intersection = self.tf.maximum(anchor_box_predictions_x1, ground_truth_bounding_boxes_x1), self.tf.minimum(anchor_box_predictions_x2, ground_truth_bounding_boxes_x2)
        y1_intersection, y2_intersection = self.tf.maximum(anchor_box_predictions_y1, ground_truth_bounding_boxes_y1), self.tf.minimum(anchor_box_predictions_y2, ground_truth_bounding_boxes_y2)


        """ Calculate intersection area """
        intersection = self.tf.where(self.tf.logical_and(x2_intersection > x1_intersection, y2_intersection > y1_intersection), (x2_intersection - x1_intersection) * (y2_intersection - y1_intersection), 0)

        """ Find the coordinates of smallest enclosing box (union) """
        x1_coord, x2_coord = self.tf.minimum(anchor_box_predictions_x1, ground_truth_bounding_boxes_x1), self.tf.maximum(anchor_box_predictions_x2, ground_truth_bounding_boxes_x2)
        y1_coord, y2_coord = self.tf.minimum(anchor_box_predictions_y1, ground_truth_bounding_boxes_y1), self.tf.maximum(anchor_box_predictions_y2, ground_truth_bounding_boxes_y2)

        """ Get area of union box """
        union_area = (x2_coord - x1_coord) * (y2_coord - y1_coord)

        """ Intersection over union """
        iou = intersection / (anchor_box_predictions_area + ground_truth_bounding_boxes_area - intersection)
        Giou = iou - ((union_area - (anchor_box_predictions_area + ground_truth_bounding_boxes_area - intersection)) / union_area)

        """ get max iou and giou and bounding box for max iou/Giou"""
        if giou is True:
            bounding_box_true = self.tf.slice(self.tf.gather(ground_truth_bounding_box,self.tf.argmax(Giou, axis=4)),[0,0,0,0,0],[-1,-1,-1,-1,4])
        else:
            bounding_box_true = self.tf.slice(self.tf.gather(ground_truth_bounding_box,self.tf.argmax(iou, axis=4)),[0,0,0,0,0],[-1,-1,-1,-1,4])
        iou = self.tf.reduce_max(iou, axis=4)
        Giou = self.tf.reduce_max(Giou, axis=4)
        return (Giou,bounding_box_true) if giou else (iou, bounding_box_true)

    def model_loss(self, y_true, y_pred):
        mini_batch_sample_size = 256
        object_predictions = y_pred[0]
        anchor_box_change_predictions = y_pred[1]
        anchor_box_predictions = self.anchor_boxes_over_image + anchor_box_change_predictions
        anchor_boxes_giou, anchor_boxes_nearest_bounding_box = self.get_iou(anchor_box_predictions, y_true, giou=True)
        object_prediction_labels = self.tf.one_hot(self.tf.where(anchor_boxes_giou > 0.5, 1, 0),depth=2, on_value=1, off_value=0)


        object_predictions_flat = self.tf.reshape(object_predictions,[-1,2])
        object_prediction_labels_flat = self.tf.reshape(object_prediction_labels, [-1, 2])
        anchor_boxes_flat = self.tf.reshape(self.anchor_boxes_over_image,[-1, 4])
        anchor_box_predictions_flat = self.tf.reshape(anchor_box_predictions,[-1, 4])
        anchor_boxes_nearest_bounding_box_flat = self.tf.reshape(anchor_boxes_nearest_bounding_box, [-1, 4])

        """Shuffle flattened y_pred and y_true all in the same order"""
        random_indices = self.tf.random.shuffle(self.tf.range(self.tf.gather(self.tf.shape(object_predictions_flat),0)))
        object_predictions_flat = self.tf.gather(object_predictions_flat, random_indices)
        object_prediction_labels_flat = self.tf.gather(object_prediction_labels_flat, random_indices)
        anchor_boxes_flat = self.tf.gather(anchor_boxes_flat, random_indices)
        anchor_box_predictions_flat = self.tf.gather(anchor_box_predictions_flat, random_indices)
        anchor_boxes_nearest_bounding_box_flat = self.tf.gather(anchor_boxes_nearest_bounding_box_flat, random_indices)

        """ Sort in ascending order from background to foreground """
        ind_sorted = self.tf.argsort(self.tf.argmax(object_prediction_labels_flat, axis=1),axis=0)
        object_predictions_flat = self.tf.gather(object_predictions_flat, ind_sorted)
        object_prediction_labels_flat = self.tf.gather(object_prediction_labels_flat, ind_sorted)
        anchor_boxes_flat = self.tf.gather(anchor_boxes_flat, ind_sorted)
        anchor_box_predictions_flat = self.tf.gather(anchor_box_predictions_flat, ind_sorted)
        anchor_boxes_nearest_bounding_box_flat = self.tf.gather(anchor_boxes_nearest_bounding_box_flat, ind_sorted)

        """ Get 128 background anchors and 128 foreground anchors and merge them into a single batch """
        split_amount = mini_batch_sample_size // 2
        #Background
        background_object_predictions_flat = self.tf.slice(object_predictions_flat, [0,0],[split_amount,-1])
        background_object_prediction_labels_flat = self.tf.slice(object_prediction_labels_flat, [0,0], [split_amount,-1])
        background_anchor_boxes_flat = self.tf.slice(anchor_boxes_flat, [0, 0], [split_amount, -1])
        background_anchor_box_predictions_flat = self.tf.slice(anchor_box_predictions_flat, [0, 0], [split_amount, -1])
        background_anchor_boxes_nearest_bounding_box_flat = self.tf.slice(anchor_boxes_nearest_bounding_box_flat, [0,0], [split_amount,-1])

        #Foreground
        foreground_object_predictions_flat = self.tf.slice(object_predictions_flat, [split_amount,0], [-1,-1])
        foreground_object_prediction_labels_flat = self.tf.slice(object_prediction_labels_flat, [split_amount,0], [-1,-1])
        foreground_anchor_boxes_flat = self.tf.slice(anchor_boxes_flat, [split_amount,0], [-1,-1])
        foreground_anchor_box_predictions_flat = self.tf.slice(anchor_box_predictions_flat, [split_amount,0], [-1,-1])
        foreground_anchor_boxes_nearest_bounding_box_flat = self.tf.slice(anchor_boxes_nearest_bounding_box_flat, [split_amount,0], [-1,-1])


        #Merge
        final_object_predictions_flat = self.tf.concat((background_object_predictions_flat,foreground_object_predictions_flat), axis=0)
        final_object_prediction_labels_flat = self.tf.concat((background_object_prediction_labels_flat, foreground_object_prediction_labels_flat), axis=0)
        all_anchor_boxes_flat = self.tf.concat((background_anchor_boxes_flat, foreground_anchor_boxes_flat), axis=0)
        all_anchor_box_predictions_flat = self.tf.concat((background_anchor_box_predictions_flat, foreground_anchor_box_predictions_flat), axis=0)
        all_anchor_boxes_nearest_bounding_box_flat = self.tf.concat((background_anchor_boxes_nearest_bounding_box_flat, foreground_anchor_boxes_nearest_bounding_box_flat), axis=0)
        #Take only foreground anchor and the closest ground truth boxes from the mini batch for bounding box regression
        final_anchor_boxes_flat = self.tf.gather(foreground_anchor_boxes_flat, self.tf.squeeze(self.tf.where(self.tf.squeeze(self.tf.equal(self.tf.slice(foreground_object_prediction_labels_flat,[0,1],[-1,1]),1),-1)),-1))
        final_anchor_box_predictions_flat = self.tf.gather(foreground_anchor_box_predictions_flat, self.tf.squeeze(self.tf.where(self.tf.squeeze(self.tf.equal(self.tf.slice(foreground_object_prediction_labels_flat,[0,1],[-1,1]),1),-1)),-1))
        final_anchor_boxes_nearest_bounding_box_flat = self.tf.gather(foreground_anchor_boxes_nearest_bounding_box_flat, self.tf.squeeze(self.tf.where(self.tf.squeeze(self.tf.equal(self.tf.slice(foreground_object_prediction_labels_flat,[0,1],[-1,1]),1),-1)),-1))

        #losses
        anchor_boxes_loss = self.anchors_loss(final_object_prediction_labels_flat, final_object_predictions_flat, all_anchor_boxes_nearest_bounding_box_flat, all_anchor_box_predictions_flat)
        regression_loss = self.bounding_box_regression_loss(final_anchor_boxes_nearest_bounding_box_flat, final_anchor_box_predictions_flat, final_anchor_boxes_flat)
        sum_anchor_boxes_loss = self.tf.reduce_sum(anchor_boxes_loss)
        sum_regression_loss = self.tf.reduce_sum(regression_loss)
        return sum_anchor_boxes_loss + sum_regression_loss

    def train_model(self, inputs, ground_truth_bounding_boxes):
        mini_batch_sample_size = 256
        for image, bb_true in zip(list(inputs), ground_truth_bounding_boxes):
            image = image.reshape(1, self.image_height, self.image_width, 3)
            predictions = self.model.predict(image)
            loss = self.model_loss(bb_true, predictions)
            print(loss)
            gradients = self.optimizer.get_gradients(loss, self.model.trainable_variables)


    def anchors_loss(self, classification_true, classification_pred, anchor_box_true, anchor_box_pred, reg_gamma=10):
        mini_batch_size = self.tf.gather(self.tf.shape(classification_pred),0)
        classification_loss = self.logloss(classification_true, classification_pred)
        true_objects = self.tf.cast(self.tf.argmax(classification_true, -1),dtype=self.tf.float64)
        reg_loss = true_objects*self.regloss(anchor_box_true, anchor_box_pred)
        anchors_loss = (1 / mini_batch_size) * classification_loss + reg_gamma * (1 / self.total_number_of_anchor_boxes) * reg_loss
        return anchors_loss

    def regloss(self, box_true, box_predictions, alpha=1):
        error = box_predictions - box_true 
        smooth_l1 = self.tf.reduce_sum(self.tf.where(self.tf.abs(error) < 1, 0.5 * self.tf.square(error), self.tf.abs(error) - 0.5), axis=1)
        return smooth_l1


    def logloss(self, true_label, predicted, eps=1e-15):
        p = self.tf.clip_by_value(predicted, eps, 1 - eps)
        log_loss = self.tf.reduce_sum(self.tf.where(self.tf.equal(true_label, 1), -self.tf.math.log(p), -self.tf.math.log(1 - p)), axis=1)
        return log_loss

    def bounding_box_regression_loss(self, true_box, predicted_box, anchor_box):
        p_x, p_y, p_w, p_h = self.tf.slice(predicted_box, [0,0],[-1,1]), self.tf.slice(predicted_box, [0,1],[-1,1]), self.tf.slice(predicted_box, [0,2],[-1,1]), self.tf.slice(predicted_box, [0,3],[-1,1])
        t_x, t_y, t_w, t_h = self.tf.slice(true_box, [0,0],[-1,1]), self.tf.slice(true_box, [0,1],[-1,1]), self.tf.slice(true_box, [0,2],[-1,1]), self.tf.slice(true_box, [0,3],[-1,1])
        a_x, a_y, a_w, a_h = self.tf.slice(anchor_box, [0, 0], [-1, 1]), self.tf.slice(anchor_box, [0, 1], [-1, 1]), self.tf.slice(anchor_box, [0, 2], [-1, 1]), self.tf.slice(anchor_box, [0, 3], [-1, 1])

        p_x = (p_x - a_x) / a_w
        p_y = (p_y - a_y) / a_h
        p_w = self.tf.math.log(p_w / a_w)
        p_h = self.tf.math.log(p_h / a_h)

        t_x = (t_x - a_x) / a_w
        t_y = (t_y - a_y) / a_h
        t_w = self.tf.math.log(t_w / a_w)
        t_h = self.tf.math.log(t_h / a_h)

        predicted = self.tf.concat((p_x, p_y, p_w, p_h), axis=1)
        truth = self.tf.concat((t_x, t_y, t_w, t_h), axis=1)

        regression_loss = self.tf.reduce_sum(self.tf.square(predicted-truth),axis=1)
        return regression_loss

    def generate_anchor_boxes(self, base_anchor_size, anchor_ratios, anchor_scales):
        """ 
            Every anchor box is different 
            Number of anchor boxes: len(anchor_ratios)*len(anchor_scales)
        """

        anchor_boxes = []
        for scale in anchor_scales:
            for aspect_ratio in anchor_ratios:
                height = base_anchor_size*scale*aspect_ratio[0]
                width = base_anchor_size*scale*aspect_ratio[1]
                anchor_box = [width, height]
                anchor_boxes.append(anchor_box)

        return anchor_boxes

    def generate_anchor_boxes_over_image(self, anchor_boxes, image_height, image_width, subsample_rate):
        """ 
            Returns an array of all the anchor boxes, each with shape [anchor_box_center_x, anchor_box_center_y, width, height]
        """
        anchor_boxes_over_image = np.zeros(( image_height//subsample_rate, image_width//subsample_rate, len(anchor_boxes), 4))
        for row in range(anchor_boxes_over_image.shape[0]):
            for col in range(anchor_boxes_over_image.shape[1]):
                for anchor_idx in range(anchor_boxes_over_image.shape[2]):
                    anchor_boxes_over_image[row,col,anchor_idx] = [(row+1)*subsample_rate, (col+1)*subsample_rate, anchor_boxes[anchor_idx][0], anchor_boxes[anchor_idx][1]]
        return anchor_boxes_over_image

请帮忙!

4

0 回答 0