I'm working on a model consisting in 2 parts, as i discussed in this question: the first should take the elements of a triplet (consisting in an anchor, a positive example and a negative example, same principle adopted in FaceNet) and turn them into vectors (word2vec + lstm), while the second should take those vectors and use them to calculate the triplet loss. I started working on some code, here's what i have now:
import pandas as pd
import numpy as np
import tensorflow as tf
from nltk.tokenize import WordPunctTokenizer
from collections import Counter
from string import punctuation, ascii_lowercase
import regex as re
from tqdm import tqdm
from gensim.models import Word2Vec
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, SpatialDropout1D, Bidirectional, concatenate, Lambda
from keras.models import Model
from keras.optimizers import Adam
from keras.layers.normalization import BatchNormalization
from keras.utils import plot_model
# Constants and initial settings
path = 'Datasets/DBLP-ACM/'
tf.compat.v1.set_random_seed(1)
ALPHA = 0.2
TRIPLETS_DATA_FILE = path + 'triplets/random_triplets.csv'
MAX_SEQUENCE_LENGTH = 300
tokenizer = WordPunctTokenizer()
vocab = Counter()
# Tokenize the text
def text_to_wordlist(text, lower=False):
# Tokenize
text = tokenizer.tokenize(text)
# Optional: lower case
if lower: text = [t.lower() for t in text]
# Return a list of words
vocab.update(text)
return text
# Process data
def process_triplets(list_sentences, lower=False):
triplet_elements = []
for text in tqdm(list_sentences):
txt = text_to_wordlist(text, lower=lower)
triplet_elements.append(txt)
return triplet_elements
# Define the custom loss (Triplet Loss)
def triplet_loss(x):
anchor, positive, negative = x
pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), 1)
neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), 1)
basic_loss = tf.add(tf.subtract(pos_dist, neg_dist), ALPHA)
loss = tf.reduce_mean(tf.maximum(basic_loss, 0.0), 0)
return loss
# Build the embedding model
def build_embedding_model(): # How can i feed the input to the word2vec part?
# Inputs
wv_layer = Embedding(nb_words, WV_DIM, mask_zero=False, weights=[wv_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=False)
embedding_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = wv_layer(embedding_input)
# BiGRU (aka bidirectional gru, bidirectional LSTM)
embedded_sequences = SpatialDropout1D(0.2)(embedded_sequences)
x = Bidirectional(LSTM(64, return_sequences=False))(embedded_sequences)
x = Dropout(0.2)(x)
x = BatchNormalization()(x)
# Output
preds = Dense(1, activation='sigmoid')(x) # Just one output class (dummy)
# Build the model
model = Model(inputs=[embedding_input], outputs=preds)
model.compile(loss='mse', optimizer = "adam")
return model
# Build the entire model
def build_model():
# Inputs
anchor_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name='anchor_input')
positive_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name='positive_input')
negative_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name='negative_input')
embedding_model = build_embedding_model()
# Outputs
anchor_embedding = embedding_model(anchor_input)
positive_embedding = embedding_model(positive_input)
negative_embedding = embedding_model(negative_input)
merged_output = concatenate([anchor_embedding, positive_embedding, negative_embedding])
loss = Lambda(triplet_loss, (1,))(merged_output)
triplet_model = Model(inputs=[anchor_input, positive_input, negative_input], outputs=loss)
triplet_model.compile(loss = 'mean_absolute_error', optimizer = Adam())
return triplet_model
triplets = pd.read_csv(TRIPLETS_DATA_FILE, error_bad_lines=False, sep="|", quotechar="\"", encoding="latin_1")
list_sentences_anchor = list((triplets["anchor"].astype(str)).fillna("").values)
list_sentences_positive = list((triplets["positive"].astype(str)).fillna("").values)
list_sentences_negative = list((triplets["negative"].astype(str)).fillna("").values)
# Fill an array for anchors, one for positives and one for negatives
anchors = process_triplets(list_sentences_anchor, lower=True)
positives = process_triplets(list_sentences_positive, lower=True)
negatives = process_triplets(list_sentences_negative, lower=True)
model_anchor = Word2Vec(anchors, size=100, window=5, min_count=5, workers=16, sg=0, negative=5)
model_positive = Word2Vec(positives, size=100, window=5, min_count=5, workers=16, sg=0, negative=5)
model_negative = Word2Vec(negatives, size=100, window=5, min_count=5, workers=16, sg=0, negative=5)
word_vectors_anchor = model_anchor.wv
word_vectors_positive = model_positive.wv
word_vectors_negative = model_negative.wv
# Use the embeddings in Keras
MAX_NB_WORDS = max(len(word_vectors_anchor.vocab), len(word_vectors_positive.vocab), len(word_vectors_negative.vocab))
word_index = {t[0]: i+1 for i,t in enumerate(vocab.most_common(MAX_NB_WORDS))}
sequences = [[word_index.get(t, 0) for t in anchor] for anchor in anchors[:len(anchors)]]
# Pad
anchor_data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH, padding="pre", truncating="post")
# Create the embedding matrix
WV_DIM = 200
nb_words = min(MAX_NB_WORDS, len(word_vectors_anchor.vocab))
# Initialize the matrix with random numbers
wv_matrix = (np.random.rand(nb_words, WV_DIM) - 0.5) / 5.0
for word, i in word_index.items():
if i >= MAX_NB_WORDS: continue
try: # Words not found in embedding index will be all-zeros
embedding_vector = word_vectors_anchor[word]
wv_matrix[i] = embedding_vector
except: pass
# Build and fit the model
triplet_model = build_model()
hist = triplet_model.fit([anchor_data, anchor_data, anchor_data], 0, validation_split=0.1, epochs=50, batch_size=256, shuffle=True)
As you will surely see, there's a lot of confusion. Basically, i split the triplets in 3 different pieces, i apply word2vec on each piece and i use the result in the embedding model (i used the same result 3 times just to test if it works, and it doesn't).
The embedding model should compute a vector to be used in the second model, during the fit process, and in the triplet loss. I'm new to Keras and i'm surely doing something wrong here, since i get this error at the moment:
TypeError: Tensor objects are only iterable when eager execution is enabled. To iterate over this tensor use tf.map_fn.
This happens in the first line of the triplet loss function itself, and it's probably related to the input format. So the question is: given this code, how can i modify it to correctly accept 3 inputs, producing 3 vectors, and use those vectors in the triplet_model, during the fit?
I'll update the question if i modify the code or i get different errors.
The loss should not be a Lambda layer. Remove the Lambda layer and update your code such that:
triplet_model = Model(inputs=[anchor_input, positive_input, negative_input], outputs=merged_output)
triplet_model.compile(loss = triplet_loss, optimizer = Adam())
triplet_loss needs to be defined as:
def triplet_loss(y_true, y_pred):
anchor_vec = y_pred[:, :VECTOR_SIZE]
positive_vec = y_pred[:, VECTOR_SIZE:2*VECTOR_SIZE]
negative_vec = y_pred[:, 2*VECTOR_SIZE:]
... code ...
You should then ignore y_true.
Full example of a triple loss function that works: https://colab.research.google.com/drive/1VgOTzr_VZNHkXh2z9IiTAcEgg5qr19y0