• Search by Module
• Search by Words
• Search Projects

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• brainforge-master
  • TODO
  • Readme.md
  • LICENSE.TXT
  • setup.py
  • xperiments
    • xp_evolution.py
    • xp_vae.py
    • xp_qcartpole.py
    • xp_lstm.py
    • xp_conv.py
    • xp_dnc.py
    • xp_3conv.py
    • xp_dense.py
    • xp_decoupled.py
    • xp_getout.py
    • xp_pggymin.py
    • xp_elm.py
    • xp_iris_etalon.py
    • xp_mnist.py
    • xp_sin.py
    • xp_feedbackalignment.py
    • xp_pong.py
  • tests
    • test_layers.py
    • test_numba_ops.py
    • __init__.py
    • test_numba_activations.py
  • brainforge
    • reinforcement
      • experience.py
      • qlearning.py
      • abstract_agent.py
      • policygradient.py
      • misc_algo.py
      • __init__.py
      • agentconfig.py
    • atomic
      • activation_op.py
      • recurrent_op.py
      • __init__.py
      • tensor_op.py
      • core_op.py
    • learner
      • neuroevolution.py
      • local_correlation.py
      • abstract_learner.py
      • backpropagation.py
      • __init__.py
      • extreme_learning_machine.py
      • feedback_alignment.py
    • layers
      • recurrent.py
      • fancy.py
      • abstract_layer.py
      • __init__.py
      • core.py
      • tensor.py
    • util
      • testing.py
      • operation.py
      • rnn_util.py
      • _nnutil.py
      • shame.py
      • rl_util.py
      • __init__.py
      • logging.py
      • _etalon.py
      • typing.py
      • persistance.py
    • config
      • __init__.py
      • numeric.py
      • parser.py
    • evolution
      • _evolution.py
      • __init__.py
    • llatomic
      • llactivation_op.py
      • _llrecurrent.py
      • llrecurrent_op.py
      • _llops.py
      • _llactivation.py
      • _llutil.py
      • __init__.py
      • _lllstm.py
      • lltensor_op.py
      • llcore_op.py
    • __init__.py
    • model
      • graph.py
      • abstract_model.py
      • __init__.py
      • layerstack.py
    • optimizers
      • gradient_descent.py
      • __init__.py
      • _coroutines.py
      • abstract_optimizer.py
      • adaptive_gd.py
    • metrics
      • regularizers.py
      • metrics.py
      • __init__.py
      • costs.py
    • gradientcheck
      • gradientcheck.py
      • analyze_difference.py
      • __init__.py
      • raw_gradients.py
  • .gitignore

import numpy as np

from .abstract_layer import LayerBase, NoParamMixin, FFBase
from ..atomic import Sigmoid
from ..util import rtm, scalX, zX, white
from ..config import floatX

sigmoid = Sigmoid()


class Highway(FFBase):
    """
    Neural Highway Layer based on Srivastava et al., 2015
    """

    def __init__(self, activation="tanh", **kw):
        FFBase.__init__(self, 1, activation, **kw)
        self.gates = None

    def connect(self, brain):
        self.neurons = np.prod(inshape)
        self.weights = white(self.neurons, self.neurons*3)
        self.biases = zX(self.neurons*3)
        FFBase.connect()

    def feedforward(self, X) -> np.ndarray:
        self.inputs = rtm(X)
        self.gates = self.inputs.dot(self.weights) + self.biases
        self.gates[:, :self.neurons] = self.activation.forward(self.gates[:, :self.neurons])
        self.gates[:, self.neurons:] = sigmoid.forward(self.gates[:, self.neurons:])
        h, t, c = np.split(self.gates, 3, axis=1)
        self.output = h * t + self.inputs * c
        return self.output.reshape(X.shape)

    def backpropagate(self, delta) -> np.ndarray:
        shape = delta.shape
        delta = rtm(delta)

        h, t, c = np.split(self.gates, 3, axis=1)

        dh = self.activation.backward(h) * t * delta
        dt = sigmoid.backward(t) * h * delta
        dc = sigmoid.backward(c) * self.inputs * delta
        dx = c * delta

        dgates = np.concatenate((dh, dt, dc), axis=1)
        self.nabla_w = self.inputs.T.dot(dgates)
        self.nabla_b = dgates.sum(axis=0)

        return (dgates.dot(self.weights.T) + dx).reshape(shape)

    @property
    def outshape(self):
        return self.inshape

    def __str__(self):
        return "Highway-{}".format(str(self.activation))


class DropOut(NoParamMixin, LayerBase):

    def __init__(self, dropchance):
        super().__init__()
        self.dropchance = scalX(1. - dropchance)
        self.mask = None
        self.inshape = None
        self.training = True

    def connect(self, brain):
        self.inshape = brain.outshape
        super().connect(brain)

    def feedforward(self, X: np.ndarray) -> np.ndarray:
        self.inputs = X
        self.mask = np.random.uniform(0, 1, self.inshape) < self.dropchance  # type: np.ndarray
        self.mask.astype(floatX)
        self.output = X * (self.mask if self.brain.learning else self.dropchance)
        return self.output

    def backpropagate(self, delta: np.ndarray) -> np.ndarray:
        output = delta * self.mask
        self.mask = np.ones_like(self.mask) * self.dropchance
        return output

    @property
    def outshape(self):
        return self.inshape

    def __str__(self):
        return "DropOut({})".format(self.dropchance)