我正在尝试将空间变压器网络插入 Keras 中预制的 VGG16 模型中。这是我的代码：
from sklearn.model_selection import train_test_split
from matplotlib import pyplot as plt
from scipy.io.matlab import loadmat
import tensorflow as tf
import numpy as np
import datetime
import os
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.preprocessing import image as im
class Localization(tf.keras.layers.Layer):
def __init__(self):
super(Localization, self).__init__()
self.pool1 = tf.keras.layers.MaxPool2D()
self.conv1 = tf.keras.layers.Conv2D(20, [5, 5], activation='relu')
self.pool2 = tf.keras.layers.MaxPool2D()
self.conv2 = tf.keras.layers.Conv2D(20, [5, 5], activation='relu')
self.flatten = tf.keras.layers.Flatten()
self.fc1 = tf.keras.layers.Dense(20, activation='relu')
self.fc2 = tf.keras.layers.Dense(6, activation=None, bias_initializer=tf.keras.initializers.constant([1.0, 0.0, 0.0, 0.0, 1.0, 0.0]), kernel_initializer='zeros')
def build(self, input_shape):
print("Building Localization Network with input shape:", input_shape)
def compute_output_shape(self, input_shape):
return [None, 6]
def call(self, inputs):
x = self.conv1(inputs)
x = self.pool1(x)
x = self.conv2(x)
x = self.pool2(x)
x = self.flatten(x)
x = self.fc1(x)
theta = self.fc2(x)
theta = tf.keras.layers.Reshape((2, 3))(theta)
return theta
class BilinearInterpolation(tf.keras.layers.Layer):
def __init__(self, height=224, width=224):
super(BilinearInterpolation, self).__init__()
self.height = height
self.width = width
def compute_output_shape(self, input_shape):
return [None, self.height, self.width, 1]
def get_config(self):
return {
'height': self.height,
'width': self.width,
}
def build(self, input_shape):
print("Building Bilinear Interpolation Layer with input shape:", input_shape)
def advance_indexing(self, inputs, x, y):
shape = tf.shape(inputs)
batch_size, _, _ = shape[0], shape[1], shape[2]
batch_idx = tf.range(0, batch_size)
batch_idx = tf.reshape(batch_idx, (batch_size, 1, 1))
b = tf.tile(batch_idx, (1, self.height, self.width))
indices = tf.stack([b, y, x], 3)
return tf.gather_nd(inputs, indices)
def call(self, inputs):
images, theta = inputs
homogenous_coordinates = self.grid_generator(batch=tf.shape(images)[0])
return self.interpolate(images, homogenous_coordinates, theta)
def grid_generator(self, batch):
x = tf.linspace(-1, 1, self.width)
y = tf.linspace(-1, 1, self.height)
xx, yy = tf.meshgrid(x, y)
xx = tf.reshape(xx, (-1,))
yy = tf.reshape(yy, (-1,))
homogenous_coordinates = tf.stack([xx, yy, tf.ones_like(xx)])
homogenous_coordinates = tf.expand_dims(homogenous_coordinates, axis=0)
homogenous_coordinates = tf.tile(homogenous_coordinates, [batch, 1, 1])
homogenous_coordinates = tf.cast(homogenous_coordinates, dtype=tf.float32)
return homogenous_coordinates
def interpolate(self, images, homogenous_coordinates, theta):
with tf.name_scope("Transformation"):
transformed = tf.matmul(theta, homogenous_coordinates)
transformed = tf.transpose(transformed, perm=[0, 2, 1])
transformed = tf.reshape(transformed, [-1, self.height, self.width, 2])
x_transformed = transformed[:, :, :, 0]
y_transformed = transformed[:, :, :, 1]
x = ((x_transformed + 1.) * tf.cast(self.width, dtype=tf.float32)) * 0.5
y = ((y_transformed + 1.) * tf.cast(self.height, dtype=tf.float32)) * 0.5
with tf.name_scope("VariableCasting"):
x0 = tf.cast(tf.math.floor(x), dtype=tf.int32)
x1 = x0 + 1
y0 = tf.cast(tf.math.floor(y), dtype=tf.int32)
y1 = y0 + 1
x0 = tf.clip_by_value(x0, 0, self.width-1)
x1 = tf.clip_by_value(x1, 0, self.width-1)
y0 = tf.clip_by_value(y0, 0, self.height-1)
y1 = tf.clip_by_value(y1, 0, self.height-1)
x = tf.clip_by_value(x, 0, tf.cast(self.width, dtype=tf.float32)-1.0)
y = tf.clip_by_value(y, 0, tf.cast(self.height, dtype=tf.float32)-1)
with tf.name_scope("AdvanceIndexing"):
Ia = self.advance_indexing(images, x0, y0)
Ib = self.advance_indexing(images, x0, y1)
Ic = self.advance_indexing(images, x1, y0)
Id = self.advance_indexing(images, x1, y1)
with tf.name_scope("Interpolation"):
x0 = tf.cast(x0, dtype=tf.float32)
x1 = tf.cast(x1, dtype=tf.float32)
y0 = tf.cast(y0, dtype=tf.float32)
y1 = tf.cast(y1, dtype=tf.float32)
wa = (x1-x) * (y1-y)
wb = (x1-x) * (y-y0)
wc = (x-x0) * (y1-y)
wd = (x-x0) * (y-y0)
wa = tf.expand_dims(wa, axis=3)
wb = tf.expand_dims(wb, axis=3)
wc = tf.expand_dims(wc, axis=3)
wd = tf.expand_dims(wd, axis=3)
return tf.math.add_n([wa*Ia + wb*Ib + wc*Ic + wd*Id])
def get_model(input_shape):
model = VGG16(weights='imagenet', include_top=True)
#Dissassemble layers
my_layers = [layer for layer in model.layers]
#Insert New Layers
image = my_layers[0].output
theta = Localization()(image)
x = BilinearInterpolation(height=input_shape[0], width=input_shape[1])([image, theta])
#Stack all layers again
for i in range(3, len(my_layers)):
my_layers[i].trainable = False
x = my_layers[i](x)
return tf.keras.models.Model(inputs=my_layers[0].input, outputs=x)
model = get_model((224,224,3))
model.summary()
背景：VGG16 模型的默认输入大小为 224x224。我正在使用的 VGG16 的来源可以在https://keras.io/api/applications/vgg/找到。我将使用来自https://github.com/fastai/imagenette的 Imagenette 数据。
我正在尝试做的事情：制作一个由 1 个定位层和 1 个双线性插值层组成的空间变压器网络模块。我正在尝试将这些层插入到输入和 conv 1-1 之间的 VGG16 模型中。插入后，我将冻结所有其他权重，仅使用 Imagenette 数据训练插入的模块。
我得到的错误：
Depth of input (3) is not a multiple of input depth of filter (64) for '{{node block2_conv1/Conv2D}} = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], explicit_paddings=[], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true](Placeholder, block2_conv1/Conv2D/ReadVariableOp)' with input shapes: [?,112,112,3], [3,3,64,128].
我究竟做错了什么？