I am practicing with neural network from scratch to handle XOR problem. However, the neural network output always getting close to 0.5 no matter what input it received.
Here is my code
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import pandas as pd
def generate_XOR_easy():
inputs = []
labels = []
for i in range(11):
hasil = [round(0.1*i, 1), round(0.1*i, 1)]
inputs.append(hasil)
labels.append(0)
if 0.1*i == 0.5:
continue
hasil2 = [round(0.1*i,1), round(1-0.1*i, 1)]
inputs.append(hasil2)
labels.append(1)
return np.array(inputs), np.array(labels).reshape(21,1)
def sigmoid(x):
return 1.0/(1.0+np.exp(-x))
def deriv_sig(x):
return np.multiply(x, 1.0-x)
epochs = 1000
n_input = 2
hidden_one = 3
hidden_two = 3
n_output = 1
learning_rate = 0.001
np.random.seed(10)
weigth_one = np.random.rand(n_input, hidden_one)
weight_two = np.random.rand(hidden_one, hidden_two)
weight_three = np.random.rand(hidden_two, n_output)
xor_data = generate_XOR_easy()
x_train, x_test, y_train, y_test = train_test_split(xor_data[0], xor_data[1],
test_size=0.2, random_state=4)
for epoch in range(epochs):
#feedforward
input_to_h1 = np.dot(x_train, weigth_one)
output_h1 = sigmoid(input_to_h1)
input_to_h2 = np.dot(output_h1, weight_two)
output_h2 = sigmoid(input_to_h2)
input_to_output = np.dot(output_h2, weight_three)
output_output = sigmoid(input_to_output)
#backpropagation
error = output_output - y_train
error_2_1 = error * deriv_sig(output_output)
error_1_w3 = np.dot(output_h2.T, error_2_1)
w3_error1 = np.dot(error_2_1, weight_three.T)
h2_front = w3_error1 * deriv_sig(output_h2)
h2Back_w2 = np.dot(output_h1.T, h2_front)
h2Back_h1front = np.dot(h2_front, weight_two.T)
h1front_h1back = h2Back_h1front * deriv_sig(output_h1)
h1back_w1 = np.dot(x_train.T, h1front_h1back)
weight_three -= learning_rate * error_1_w3
weight_two -= learning_rate * h2Back_w2
weigth_one -= learning_rate * h1back_w1
When I print the output every iteration, the result will go closer to 0.5 (for all input). The code is very messy since I am still new and need more practice with machine learning and deep learning code. Is there anyone know whats wrong? I really appreciate any help and suggestion. Thank you
I figured out the problem by myself and actually it is very simple. I got good result after I increase the learning rate and number of iteration. I hope my problem and solution might help other who got the same problem.