I have three Input layers in my model, and 'input3' is set as constant value. Then I input 'input3' to Embedding layer , get the result 'lookup_table' and then do some other operations.
But when I use model.summary() to observe my model and the training parameters, I find that the Input3 layer and the Embedding layer are not added to the model, and I think the parameters of Embedding layer will not be trained.
The code:
import numpy as np
from keras.models import Model
from keras.layers import*
import keras.backend as K
np_constant = np.array([[1,2,3],
[4,5,6],
[7,8,9]])
def NN():
input1 = Input(batch_shape=(None,1),name='input1',dtype='int32')
input2 = Input(batch_shape=(None,1),name='input2',dtype='int32')
# constant_tensor = K.constant(np_constant)
input3 = Input(tensor=K.constant(np_constant),batch_shape=(3,3),dtype='int32',name='constant_input_3')
embedding = Embedding(input_dim=10,output_dim=5,input_length=3)
lookup_table = embedding(input3)
lookup_table = Lambda(lambda x: K.reshape(x, (-1,15)))(lookup_table)
output1 = Lambda(lambda x: K.gather(lookup_table, K.cast(x, dtype='int32')))(input1)
output2 = Lambda(lambda x: K.gather(lookup_table, K.cast(x, dtype='int32')))(input2)
# Merge branches
output = Concatenate(axis=1)([output1, output2])
# Process merged branch
output = Dense(units=2
, activation='softmax'
)(output)
model = Model([input1, input2, input3], outputs=output)
return model
model = NN()
model.summary()
in_1 = np.array([1,2,1])
in_2 = np.array([1,0,1])
model.compile() # just for example
model.fit([in_1,in_2])
The model summary
Layer (type) Output Shape Param # Connected to
==================================================================================================
input1 (InputLayer) (None, 1) 0
__________________________________________________________________________________________________
input2 (InputLayer) (None, 1) 0
__________________________________________________________________________________________________
lambda_2 (Lambda) (None, 1, 15) 0 input1[0][0]
__________________________________________________________________________________________________
lambda_3 (Lambda) (None, 1, 15) 0 input2[0][0]
__________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, 2, 15) 0 lambda_2[0][0]
lambda_3[0][0]
__________________________________________________________________________________________________
dense_1 (Dense) (None, 2, 2) 32 concatenate_1[0][0]
==================================================================================================
Total params: 32
Trainable params: 32
Non-trainable params: 0
__________________________________________________________________________________________________
I have to feed data in the model.fit() function, and the input3 is always constant and the shape of input3 is different from input1 and input2, so I use it in this way. But I have no idea why the Input3 layer and Embedding layer are not added to the model.
I modify my original code, I define a custom function outside the model and pass a list of tensors into a Lambda layer as Anakin suggested. Here is the modified codes.
import numpy as np
from keras.models import Model
from keras.layers import*
import keras.backend as K
np_constant = np.array([[1,2,3],
[4,5,6],
[7,8,9]])
def look_up(arg):
in1 = arg[0]
in2 = arg[1]
lookup_table = arg[2]
in1 = Lambda(lambda x: K.reshape(x, (-1, )))(in1)
in2 = Lambda(lambda x: K.reshape(x, (-1, )))(in2)
output1 = Lambda(lambda x: K.gather(lookup_table, K.cast(x, dtype='int32')))(in1)
output2 = Lambda(lambda x: K.gather(lookup_table, K.cast(x, dtype='int32')))(in2)
return [output1,output2]
def NN():
input1 = Input(batch_shape=(None,1),name='input1',dtype='int32')
input2 = Input(batch_shape=(None,1),name='input2',dtype='int32')
# constant_tensor = K.constant(np_constant)
input3 = Input(tensor=K.constant(np_constant),batch_shape=(3,3),dtype='int32',name='constant_input_3')
lookup_table = Embedding(input_dim=10,output_dim=5,input_length=3)(input3)
lookup_table = Lambda(lambda x: K.reshape(x, (-1, 15)))(lookup_table)
output1 = Lambda(look_up)([input1,input2,lookup_table])[0]
output2 = Lambda(look_up)([input1,input2,lookup_table])[1]
# Merge branches
output = Concatenate(axis=1)([output1, output2])
# Process merged branch
output = Dense(units=2
, activation='softmax'
)(output)
model = Model([input1, input2, input3], outputs=output)
return model
model = NN()
model.summary()
input_1 = np.array([1,2,1])
input_2 = np.array([1,0,1])
model.compile() # just for example
model.fit([input_1,input_2])
In this way, the Embedding could be added to the model. And input3 is a constant tensor, we don't need to feed it in model.fit() function.
The model summary
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
constant_input_3 (InputLayer) (3, 3) 0
__________________________________________________________________________________________________
embedding_1 (Embedding) (3, 3, 5) 50 constant_input_3[0][0]
__________________________________________________________________________________________________
input1 (InputLayer) (None, 1) 0
__________________________________________________________________________________________________
input2 (InputLayer) (None, 1) 0
__________________________________________________________________________________________________
lambda_1 (Lambda) (3, 15) 0 embedding_1[0][0]
__________________________________________________________________________________________________
lambda_2 (Lambda) [(None, 15), (None, 0 input1[0][0]
input2[0][0]
lambda_1[0][0]
__________________________________________________________________________________________________
lambda_11 (Lambda) [(None, 15), (None, 0 input1[0][0]
input2[0][0]
lambda_1[0][0]
__________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, 30) 0 lambda_2[0][0]
lambda_11[0][1]
__________________________________________________________________________________________________
dense_1 (Dense) (None, 2) 62 concatenate_1[0][0]
==================================================================================================
Total params: 112
Trainable params: 112
Non-trainable params: 0
__________________________________________________________________________________________________