Demo entry 6754363

abs

   

Submitted by anonymous on Jul 10, 2018 at 22:21
Language: Python 3. Code size: 9.2 kB.

import numpy as np


class Graph:
    """
    This class represents a Neural Network.
    You can append layers and units.
    Layers and Units are functions.
    The Graph can train and classify.
    """

    def __init__(self):
        self.graph = []

    def append(self, layer):
        if len(self.graph) > 0:
            self.graph[-1].next_layer = layer
            layer.prev_layer = self.graph[-1]
        self.graph.append(layer)

    def forward(self):
        layer_output = []
        for layer in self.graph:
            layer_output = layer.forward(layer_output)
        y_guess = layer_output
        return y_guess

    def backward(self):
        layer_derivative_L = np.array([])
        for layer in reversed(self.graph):
            layer_derivative_L = layer.backward(layer_derivative_L)

    def update(self):
        for layer in self.graph:
            layer.update()


class Layer:
        def __init__(self):
            pass

        def forward(self, _):
            pass

        def backward(self, _):
            pass

        def update(self):
            pass

class InputUnit(Layer):
    """
    This represents an InputUnit.
    """

    def __init__(self, x=[[0]]):
        self.next_layer = None
        x = np.array(x)
        try:
            self.x = np.reshape(x, (1, x.shape[0]))
        except(IndexError):
            self.x = np.reshape(x, (1, x.shape[1]))

    def forward(self, _):
        return self.x

    def set_input(self, x):
        try:
            self.x = np.reshape(x, (1, x.shape[0]))
        except(IndexError):
            self.x = np.reshape(x, (1, x.shape[1]))


class SigmoidUnit(Layer):
    def __init__(self):
        self.next_layer = None
        self.prev_layer = None
        self.local_grad = None
        self.global_grad = None

    def forward(self, x):
        self.pre_activation = x
        self.activation = 1 / (1 + np.exp(-x))
        return self.activation

    def backward(self, grad_nl_L):
        """
        @param: grad_nl_L: gradient of next layer with respect to Loss.
        """

        # calculate local gradient
        # ToDo:check if this is in fact the local gradient
        sig_x = self.activation
        self.local_grad = sig_x * (1 - sig_x)

        # if next layer is a loss layer:
        # this layers gradient is the loss layers gradient
        if isinstance(self.next_layer, SquareLossUnit):
            return grad_nl_L

        a = self.next_layer.grad_nl_L
        b = self.next_layer.next_layer.local_grad
        c = self.next_layer.W
        self.global_grad = np.dot(a, (b * c).T)
        return self.global_grad


class LeakyReluUnit(Layer):
        def __init__(self, alpha):
            self.next_layer = None
            self.prev_layer = None
            self.local_grad = None
            self.global_grad = None
            self.pre_activation = None
            self.activation = None
            self.alpha = alpha

        def forward(self, x):
            self.pre_activation = x
            self.activation = np.where(x > 0, x, x * self.alpha)
            return self.activation

        def backward(self, grad_nl_L):
            """
            @param: grad_nl_L: gradient of next layer with respect to Loss.
            """

            # calculate local gradient
            # ToDo:check if this is in fact the local gradient
            self.local_grad = np.ones_like(self.pre_activation)
            self.local_grad[self.pre_activation < 0] = self.alpha

            # if next layer is a loss layer:
            # this layers gradient is the loss layers gradient
            if isinstance(self.next_layer, SquareLossUnit):
                return grad_nl_L

            a = self.next_layer.grad_nl_L
            b = self.next_layer.next_layer.local_grad
            c = self.next_layer.W
            self.global_grad = np.dot(a, (b * c).T)
            return self.global_grad


class SquareLossUnit(Layer):
    def __init__(self, y_true):
        self.next_layer = None
        self.prev_layer = None
        self.loss = 0
        self.y_true = y_true
        self.x = None

    def forward(self, x):
        self.x = x
        self.loss = np.square(self.y_true - x)
        return x

    def backward(self, _):
        grad_L = 2 * (self.x - self.y_true)
        return grad_L

    def set_label(self, y_true):
        self.y_true = y_true


class FullyConnected(Layer):
    """
    This class represents a fully connected Layer
    """

    def __init__(self, W, learning_rate=1):
        self.next_layer = None
        self.prev_layer = None
        # numpyfy if not already
        self.W = np.array(W)
        self.learning_rate = learning_rate
        self.v = 0
        self.prev_v = 0
        self.cache = 0

    def forward(self, input_):
        # numpyfy if not already
        input_ = np.array(input_)
        self.input_ = input_
        # return dot product
        return np.dot(input_, self.W)

    def backward(self, grad_nl_L):
        # see below and hex03 for proof
        self.grad_nl_L = grad_nl_L
        self.global_grad = grad_nl_L * self.next_layer.local_grad * self.input_.T
        return self.global_grad

    def update(self):
        mu = 0.9
        # decay_rate = 0.99
        # eps = 1e-6
        # # Momentum update
        self.v = 0.9 * self.v - self.learning_rate * self.global_grad # integrate velocity
        self.W += self.v # integrate position

        # self.prev_v = self.v # back this up
        # self.v = mu * self.v - self.learning_rate * self.global_grad # velocity update stays the same
        # self.W += -mu * self.prev_v + (1 + mu) * self.v # position update changes form

        # self.cache = decay_rate * self.cache + (1 - decay_rate) * self.global_grad**2
        # self.W += - self.learning_rate * self.global_grad / (np.sqrt(self.cache) + eps)



        #self.W = self.W - self.global_grad * self.learning_rate


################################
#            TESTS 1           #
################################

# W = [[-1.4, -0.81],
#      [-2.2, -1.7],
#      [-0.27, -0.73]]
# W = np.array(W)
#
#
# p = [[0.75, 0.35, 0.46]]
# p = np.array(p)
#
# t = [[0, 1]]
# t = np.array(t)
#
# graph = Graph()
# graph.append(InputUnit(p))
# graph.append(FullyConnectedLayer(W))
# graph.append(SigmoidUnit())
# graph.append(SquareLossUnit(t))
#
# loss = graph.forward()
# graph.backward()
# graph.update()
# print(loss)




################################
#            TESTS 2           #
################################


# from sklearn.datasets.samples_generator import make_classification
#
# # Generate some data
# X, y_true = make_classification(n_samples=400)
#
# graph = Graph()
# graph.append(InputUnit(X[0].reshape(1,20)))
# W = np.linspace(0, 1, 20 * 5).reshape(20, 5)
# layer_W = FullyConnectedLayer(W)
# graph.append(layer_W)
# graph.append(LeakyReluUnitLevel(alpha=0.0005))
# T = np.linspace(0, 1, 5 * 1).reshape(5, 1)
# layer_T = FullyConnectedLayer(T)
# graph.append(layer_T)
# graph.append(SigmoidUnit())
# graph.append(SquareLossUnit(y_true[0]))
# loss = graph.forward()
# graph.backward()
# print(loss)

################################
#            TESTS 3           #
################################

# graph = Graph()
# x = np.array([0,1])
# x = np.array(x)
# t = [[1,1]]
# t = np.array(t)
# U = [[1.2, -1.2, -0.11],
#      [0.3, 1.1, 0.65]]
# U = np.array(U)
#
# V = [[-0.25], [-1.1], [-0.09]]
# V = np.array(V)
# W = [[-2, 0.43]]
# W = np.array(W)
#
# graph.append(InputUnit(x))
# graph.append(FullyConnected(U))
# graph.append(SigmoidUnit())
# graph.append(FullyConnected(V))
# graph.append(SigmoidUnit())
# graph.append(FullyConnected(W))
# graph.append(SigmoidUnit())
# graph.append(SquareLossUnit(t))
#
# for i in range(1):
#     guess = graph.forward()
#     a = graph.backward()
#     b = graph.update()
#     print(guess)

###########################
# Weight Derivatives test #
###########################

# def sigmoid(x):
#     return 1 / (1 + np.exp(-x))
#
#
# def d_sigmoid(x):
#     sig_x = sigmoid(x)
#     return sig_x * (1 - sig_x)
#
#
# W = [[-1.4, -0.81],
#      [-2.2, -1.7],
#      [-0.27, -0.73]]
# W = np.array(W)
# p = np.array([[0.74, 0.35, 0.46]])  # input_
# d_y = np.array([[1.75, -0.35]])  # grad_nl_L
# print(np.dot(p, W))
# d_sig_WP = d_sigmoid(np.array([[-1.96, -1.54]]))  # d_sigmoid(W @ p) #next_layer.local_grad
# print(d_sig_WP)
# print(d_y * d_sig_WP * p.T) # grad_nl_L * next_layer.local_grad * input_.T


####################
# Unit Derivatives #
####################
# d_E_p = np.dot(d_y, (d_sig_WP * W).T)  # self.next_layer.grad_nl_L @ (self.next_layer.next_layer.local_grad * self.next_layer.W).T
# print(d_E_p)
#
# print(np.dot(np.arange(1, 3).reshape(1, 2) , np.arange(1, 3).reshape(2, 1)))
#
# print((d_sig_WP * W).T)
#
# print(type(SquareLossUnit(4)))
#
# print(np.array([[1, 2, 3, 4, 5]]).shape)

This snippet took 0.01 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).