% Learn to classify using the perceptron model trained with the LMS rule
% Learning rate is set very low in this demo code to visualize the gradient
% descent
%
% Author: Aravindh Mahendran
% aravindh.mahendran@new.ox.ac.uk

clear;
close all;

%% Load and prepare the data
load('data.mat'); %loads X_positive X_negative
X = [X_positive, X_negative];
X_augmented = [X; ones(1, size(X, 2))];
Y_target = [ones(1, size(X_positive, 2)), -ones(1, size(X_negative, 2))];

%% Initialize the weights and set the hyperparameters
w = randn(3, 1) * 0.01;
eta = 0.0001;
loss = zeros(10000,1);
maxIterations = 10000;

for i=1:maxIterations % 10000 iterations of gradient descent
    Y_predicted = w' * X_augmented;
    loss(i) = sum((Y_target - Y_predicted).^2)/(2*numel(Y_predicted));
    grad = sum(bsxfun(@times, (Y_target - Y_predicted), -X_augmented), 2)/numel(Y_predicted);
    w = w - eta * grad;
    
    if(mod(i, 100) == 1)
        figure(1);
        clf;
        
        subplot(1,2,1);
        plot(loss(1:i));
        title('loss');
        xlabel('Number of Iterations');
        ylabel('Loss value');
        
        subplot(1,2,2);
        plot(X_positive(1,:), X_positive(2,:), 'bo');
        hold on;
        plot(X_negative(1,:), X_negative(2,:), 'ro');
        legend('Positive', 'Negative');
        axis([-4,4,-4,4]);
        
        x1 = -4;
        y1 = (-w(3) - w(1)*x1)/w(2);
        
        x2 = 4;
        y2 = (-w(3) - w(1)*x2)/w(2);
        
        line([x1; x2], [y1; y2]);
       
        title('data');
        
        drawnow;
        pause(1);
    end
        
end