classifyingTextMULTI.py

#written by Viktor Zenkov in 2018

#this file classifies using the Multi-input model, making predictions based on the text and hex data.

from __future__ import print_function
from __future__ import absolute_import

import sys

import numpy as np
from numpy.random import seed
seed(1)
from tensorflow import set_random_seed
set_random_seed(2)

import matplotlib.pyplot as plt
import math
import time
import datetime

from keras.preprocessing import sequence
from keras.models import Sequential, Model
from keras.layers import Input, Dense, Embedding
from keras.layers import LSTM, Dropout, Activation
from keras.layers import concatenate
from keras.utils import to_categorical, print_summary
from keras.callbacks import EarlyStopping
from sklearn.metrics import confusion_matrix
import itertools

print('sys.argv[0]: {0!r}'.format(sys.argv[0]))
print('sys.path[0]: {0!r}'.format(sys.path[0]))
 
#this plots a confusion matrix
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion Matrix', cmap=plt.cm.Blues):
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    
    print(cm)
    
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title, fontsize=30)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks,classes, rotation=45,fontsize=22)
    plt.yticks(tick_marks,classes, fontsize=22)
    
    fmt = '.2f'
    thresh = cm.max()/2.
    for i,j in itertools.product(range(cm.shape[0]),range(cm.shape[1])):
        plt.text(j, i, format(cm[i,j],fmt),horizontalalignment="center",color='white' if cm[i,j]>thresh else 'black')
        
    plt.ylabel('True label', fontsize=25)
    plt.xlabel('Predicted label', fontsize=25)


import classifyingFunctions






max_features = 5000  #this is the number of unique integers we will keep.
maxlen = 5000  # cut texts after this number of words (among top max_features most common words), used in pad_sequences (right after load data, before any modeling)
batch_size = 32

print('Loading number data...')

#num_words' default is None, which is taken to mean that all unique integers should be kept.
(x_train_N, y_train_N), (x_test_N, y_test_N) = classifyingFunctions.load_data(num_words=max_features,numOrHex=True) #all integers greater than max_features get turned into 2's, so there's max_features number of unique "words"
print(len(x_train_N), 'train sequences')
print(len(x_test_N), 'test sequences')

print('Pad sequences (samples x time)')
x_train_N = sequence.pad_sequences(x_train_N, maxlen=maxlen)
x_test_N = sequence.pad_sequences(x_test_N, maxlen=maxlen)
print('x_train_N shape:', x_train_N.shape)
print('x_test_N shape:', x_test_N.shape)

y_test_N_ld = y_test_N

#We have classes from 1 to 9, which is 9 classes, but to_categorical will make an array with spots from 0 to max_class, so we subtract 1 such that our classes are 0 to 8 and we can use 9 classes.
y_train_N -= 1
y_test_N -= 1

#to_categorical replaces each number between 0 and 8 with a 9-length array of 0's except a 1 in the place of the number.
y_train_N = to_categorical(y_train_N, 9)
y_test_N = to_categorical(y_test_N, 9)

print('y_train_N shape:', y_train_N.shape)
print('y_test_N shape:', y_test_N.shape)




#we also need the hex data
print('Loading hex data...')

(x_train_H, y_train_H), (x_test_H, y_test_H) = classifyingFunctions.load_data(num_words=max_features,numOrHex=False) 
print(len(x_train_H), 'train sequences')
print(len(x_test_H), 'test sequences')

print('Pad sequences (samples x time)')
x_train_H = sequence.pad_sequences(x_train_H, maxlen=maxlen)
x_test_H = sequence.pad_sequences(x_test_H, maxlen=maxlen)
print('x_train_H shape:', x_train_H.shape)
print('x_test_H shape:', x_test_H.shape)

y_test_H_ld = y_test_H

y_train_H -= 1
y_test_H -= 1

y_train_H = to_categorical(y_train_H, 9)
y_test_H = to_categorical(y_test_H, 9)

print('y_train_H shape:', y_train_H.shape)
print('y_test_H shape:', y_test_H.shape)




#We build up to the LSTM layer for text
main_inputNL = Input(shape=(max_features,), dtype='int32', name='main_inputNL')
embeddingNL = Embedding(output_dim=128, input_dim=max_features, input_length=maxlen)(main_inputNL)
lstm_outNL = LSTM(150, dropout=0.1, recurrent_dropout=0.1)(embeddingNL)


#and we build up to the LSTM layer for hex
main_inputHL = Input(shape=(max_features,), dtype='int32', name='main_inputHL')
embeddingHL = Embedding(output_dim=128, input_dim=max_features, input_length=maxlen)(main_inputHL)
lstm_outHL = LSTM(150, dropout=0.1, recurrent_dropout=0.1)(embeddingHL)


#We concatenate the LSTM layers
concatL = concatenate([lstm_outNL, lstm_outHL])

concatL = Dense(9, activation='softmax', name='aux_output')(concatL)

model = Model(inputs=[main_inputNL, main_inputHL], outputs=concatL)




model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

es = EarlyStopping(monitor="val_acc",min_delta=0.005,patience=50,mode='max') #meaning of this: when val_acc stops increasing by more than min_delta, we stop at current epoch.

print('Train...')
model.fit([x_train_N, x_train_H], y_train_N,
          epochs=55, batch_size=batch_size,validation_split=0.15, callbacks=[es])

model.save('model'+str(datetime.date.today())+str(math.floor(time.time()))+'.h5')

score, acc = model.evaluate([x_test_N, x_test_H], y_test_N,
                            batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)





#the rest of this is for the confusion matrix

y_probs = model.predict([x_test_N,x_test_H])

#this block of code makes a 1D array of predicted labels, which is an input to the confusion matrix
y_pred_ld = []
for i in range(0, len(y_probs)):
    probs = y_probs[i]
    predicted_index = np.argmax(probs)
    y_pred_ld.append(predicted_index)

#cnf_matrix = confusion_matrix(y_test_N_ld, y_pred_ld)
#plt.figure(figsize=(24,20))
cnf_matrix = confusion_matrix(y_test_N_ld, y_pred_ld)
print(cnf_matrix)
cnf_matrix = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]
print(cnf_matrix)

print_summary(model)


#plt.subplot(221)
#plot_confusion_matrix(cnf_matrix,classes=range(1,10),normalize=False,title="Confusion matrix")
#plt.subplot(222)
#plt.show()
#plot_confusion_matrix(cnf_matrix,classes=range(0,10),normalize=True,title="Confusion matrix")
#plt.show()
print("end of file")