我使用 tensorflow 2.4 创建了一个连体网络。
def create_encoder_siamese(pairs, cfg):
# Based on https://keras.io/examples/vision/siamese_network/
# and https://keras.io/examples/vision/siamese_contrastive/
def euclidean_distance(vects):
x, y = vects
sum_square = tf.math.reduce_sum(tf.math.square(x - y), axis=1, keepdims=True)
return tf.math.sqrt(tf.math.maximum(sum_square, tf.keras.backend.epsilon()))
# Create the base model from the pre-trained model MobileNet V2
IMG_SHAPE = pairs.shape[2:]
print('Image shape:', IMG_SHAPE)
# img_batch = pairs[0:cfg['train']['batch_size'],0,:,:,:]
# print('batch shape:', img_batch.shape)
# pre-trained feature extraction model
base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
# freeze the feature extraction part
base_model.trainable = False
feature_batch = base_model(pairs[0:cfg['train']['batch_size'],0,:,:,:])
print('batch feature shape:', feature_batch.shape)
# add a layer to convert to 1-d vectors
global_average_layer = tf.keras.layers.GlobalAveragePooling2D()
feature_batch_average = global_average_layer(feature_batch)
print('batch average pooling shape:', feature_batch_average.shape)
# add a layer to convert to encoded vectors that we will use to measure distances
encoded_layer = tf.keras.layers.Dense(EMBEDDING_SIZE, activation='relu')
encoded_batch = encoded_layer(feature_batch_average)
print('batch encoded shape:', encoded_batch.shape)
# put all together
inputs = tf.keras.Input(shape=IMG_SHAPE)
x = base_model(inputs, training=False) # The False is necessary as it contains a BatchNormalisation layer
x = global_average_layer(x)
x = tf.keras.layers.Dropout(0.2)(x)
encoded = encoded_layer(x)
encoder = tf.keras.Model(inputs, encoded, name='encoder')
print(encoder.summary())
print()
input_1 = tf.keras.layers.Input(IMG_SHAPE)
input_2 = tf.keras.layers.Input(IMG_SHAPE)
# Link two towers
tower_1 = encoder(input_1)
tower_2 = encoder(input_2)
# Compute distance between embeddings
distance_layer = tf.keras.layers.Lambda(euclidean_distance)([tower_1, tower_2])
output_layer = tf.keras.layers.Dense(1, activation="sigmoid")(distance_layer)
model = tf.keras.Model(inputs=[input_1, input_2], outputs=output_layer)
print(model.summary())
print()
model.compile(optimizer='Adam',
loss=tfa.losses.ContrastiveLoss(margin=cfg['train']['margin']),
# loss=loss(margin=cfg['train']['margin']),
metrics='accuracy')
print()
print('Number of layers to train:', len(model.trainable_variables))
return model, encoder
我可以成功地训练并使用它来计算所需图像对之间的距离。然而,出于部署效率的目的,我想首先将所有图像转换为它们的 1D 嵌入(我可以使用编码器模型),然后使用子模型直接计算嵌入之间的距离。
但是当我尝试使用以下内容创建比较器模型时:
comparer = tf.keras.Model(inputs=[(EMBEDDING_SIZE), (EMBEDDING_SIZE)], outputs=output_layer, name='comparer')
我得到错误:
ValueError Traceback (most recent call last)
<ipython-input-22-dfe3ad813151> in <module>
88 return model, encoder, output_layer
89
---> 90 model, encoder, comparer = create_encoder_siamese(train_pairs, cfg)
<ipython-input-22-dfe3ad813151> in create_encoder_siamese(pairs, cfg)
64
65 # comparer model
---> 66 comparer = tf.keras.Model(inputs=[(EMBEDDING_SIZE), (EMBEDDING_SIZE)], outputs=output_layer, name='comparer')
67
68 # full model
/usr/local/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
515 self._self_setattr_tracking = False # pylint: disable=protected-access
516 try:
--> 517 result = method(self, *args, **kwargs)
518 finally:
519 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/functional.py in __init__(self, inputs, outputs, name, trainable, **kwargs)
118 generic_utils.validate_kwargs(kwargs, {})
119 super(Functional, self).__init__(name=name, trainable=trainable)
--> 120 self._init_graph_network(inputs, outputs)
121
122 @trackable.no_automatic_dependency_tracking
/usr/local/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
515 self._self_setattr_tracking = False # pylint: disable=protected-access
516 try:
--> 517 result = method(self, *args, **kwargs)
518 finally:
519 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/functional.py in _init_graph_network(self, inputs, outputs)
155 base_layer_utils.create_keras_history(self._nested_outputs)
156
--> 157 self._validate_graph_inputs_and_outputs()
158
159 # A Network does not create weights of its own, thus it is already
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/functional.py in _validate_graph_inputs_and_outputs(self)
680 'is redundant. '
681 'All inputs should only appear once.'
--> 682 ' Found: ' + str(self.inputs))
683
684 for x in self.inputs:
ValueError: The list of inputs passed to the model is redundant. All inputs should only appear once. Found: [64, 64]
那么我如何调整这个网络以拥有一个完整的训练模型,一个将单个图像转换为嵌入的编码器,以及一个直接比较 2 个嵌入的最终模型?
注意:我知道每个人都使用编码器将图像转换为嵌入,然后计算欧几里得距离来计算样本之间的相似性。然而,欧几里得距离和从整个模型计算的距离并不相同,并且遵循以下关系:
谁不想要一个介于 0 到 1 范围内的漂亮距离度量?