tensorflowkerashybrid

Merge two sequential models on Keras for hybrid model


I want to combine two sequential models for a hybrid model (with Keras 2.6.0). The first model is a succession of dense layer of a set of 4 parameters, and the second is a succession of 2D convolution of an image ((32,32)). The goal is to predict a curve of 128 points.

Here is an illustration of my model hybrid idea.

My actual model:

def get_model_v2(params_shape, img_shape):
    params_model = models.Sequential()
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
    params_model.add(layers.Dense(256, name='Output'))
    
    img_model = models.Sequential()
    img_model.add(layers.Input(img_shape, name='InputLayer2'))
    img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Flatten())
    
    concat = tf.keras.layers.concatenate([params_model, img_model])

    model = models.Sequential()
    model.add(layers.Input(concat, name='InputLayer3'))
    model.add(layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
    model.add(layers.Dense(128, name='Output'))
    
    model.compile(optimizer = 'adam', 
                  loss      = 'mse', 
                  metrics   = ['mae', 'mse'])
    return model


model = get_model_v2 ( (4,),  (32, 32, 1) )

My problem is when I have to combine the two models, I don't know what to use, with this "concatenate" example I have an error like: TypeError: 'NoneType' object is not subscriptable. I understand the problem, but I can't find an other solution...


Solution

  • Few issues here,

    import tensorflow.keras.layers as layers
    import tensorflow.keras.models as models
    import tensorflow.keras.regularizers as regularizers
    import tensorflow as tf
    
    def get_model_v2(params_shape, img_shape):
        params_model = models.Sequential()
        params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1', input_shape=params_shape))
        params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
        params_model.add(layers.Dense(256, name='Output'))
        
        img_model = models.Sequential()
        img_model.add(layers.Input(img_shape, name='InputLayer2'))
        img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
        img_model.add(layers.LeakyReLU(alpha=0.2))
        img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
        img_model.add(layers.LeakyReLU(alpha=0.2))
        img_model.add(layers.Flatten())
        
        param_out = params_model.outputs[0]
        img_out = img_model.outputs[0]
    
        concat_out = tf.keras.layers.concatenate([param_out, img_out])
    
        full_dense_out = layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n3')(concat_out)
        final_out = layers.Dense(128, name='Output_final')(full_dense_out)
        
        model = models.Model(inputs=[params_model.inputs, img_model.inputs], outputs=final_out)
    
        model.summary()
    
        model.compile(optimizer = 'adam', 
                      loss      = 'mse', 
                      metrics   = ['mae', 'mse'])
        return model
    
    
    model = get_model_v2 ( (4,),  (32, 32, 1) )