add gelu activation

Author: ddbourgin

docs/numpy_ml.neural_nets.activations.rst

@@ -60,6 +60,13 @@ neural networks.
  :undoc-members:
  :inherited-members:
 
+``GELU``
+-----------
+.. autoclass:: numpy_ml.neural_nets.activations.GELU
+ :members:
+ :undoc-members:
+ :inherited-members:
+
 ``Sigmoid``
 ------------
 .. autoclass:: numpy_ml.neural_nets.activations.Sigmoid

numpy_ml/neural_nets/activations/activations.py

@@ -1,5 +1,5 @@
 """A collection of activation function objects for building neural networks"""
-
+from math import erf
 from abc import ABC, abstractmethod
 
 import numpy as np
@@ -91,7 +91,7 @@ class ReLU(ActivationBase):
 
  References
  ----------
- .. [*] Karpathy, A. "CS231n: Convolutional neural networks for visual recognition".
+ .. [*] Karpathy, A. "CS231n: Convolutional neural networks for visual recognition."
  """
 
  def __init__(self):
@@ -154,7 +154,7 @@ class LeakyReLU(ActivationBase):
  References
  ----------
  .. [*] Mass, L. M., Hannun, A. Y, & Ng, A. Y. (2013). "Rectifier
- nonlinearities improve neural network acoustic models". *Proceedings of
+ nonlinearities improve neural network acoustic models." *Proceedings of
  the 30th International Conference of Machine Learning, 30*.
  """
 
@@ -207,6 +207,100 @@ def grad2(self, x):
  return np.zeros_like(x)
 
 
+class GELU(ActivationBase):
+ def __init__(self, approximate=True):
+ r"""
+ A Gaussian error linear unit (GELU). [*]_
+
+ Notes
+ -----
+ A ReLU alternative. GELU weights inputs by their value, rather than
+ gates inputs by their sign, as in vanilla ReLUs.
+
+ References
+ ----------
+ .. [*] Hendrycks, D., & Gimpel, K. (2016). "Bridging nonlinearities and
+ stochastic regularizers with Gaussian error linear units." *CoRR*.
+
+ Parameters
+ ----------
+ approximate : bool
+ Whether to use a faster but less precise approximation to the Gauss
+ error function when calculating the unit activation and gradient.
+ Default is True.
+ """
+ self.approximate = True
+ super().__init__()
+
+ def __str__(self):
+ """Return a string representation of the activation function"""
+ return f"GELU(approximate={self.approximate})"
+
+ def fn(self, z):
+ r"""
+ Compute the GELU function on the elements of input `z`.
+
+ .. math::
+
+ \text{GELU}(z_i) = z_i P(Z \leq z_i) = z_i \Phi(z_i)
+ = z_i \cdot \frac{1}{2}(1 + \text{erf}(x/\sqrt{2}))
+ """
+ pi, sqrt, tanh = np.pi, np.sqrt, np.tanh
+
+ if self.approximate:
+ return 0.5 * z * (1 + tanh(sqrt(2 / pi) * (z + 0.044715 * z ** 3)))
+ return 0.5 * z * (1 + erf(z / sqrt(2)))
+
+ def grad(self, x):
+ r"""
+ Evaluate the first derivative of the GELU function on the elements
+ of input `x`.
+
+ .. math::
+
+ \frac{\partial \text{GELU}}{\partial x_i} =
+ \frac{1}{2} + \frac{1}{2}\left(\text{erf}(\frac{x}{\sqrt{2}}) +
+ \frac{x + \text{erf}'(\frac{x}{\sqrt{2}})}{\sqrt{2}}\right)
+
+ where :math:`\text{erf}'(x) = \frac{2}{\sqrt{\pi}} \cdot \exp\{-x^2\}`.
+ """
+ pi, exp, sqrt, tanh = np.pi, np.exp, np.sqrt, np.tanh
+
+ s = x / sqrt(2)
+ erf_prime = lambda x: (2 / sqrt(pi)) * exp(-(x ** 2)) # noqa: E731
+
+ if self.approximate:
+ approx = tanh(sqrt(2 / pi) * (x + 0.044715 * x ** 3))
+ dx = 0.5 + 0.5 * approx + ((0.5 * x * erf_prime(s)) / sqrt(2))
+ else:
+ dx = 0.5 + 0.5 * erf(s) + ((0.5 * x * erf_prime(s)) / sqrt(2))
+ return dx
+
+ def grad2(self, x):
+ r"""
+ Evaluate the second derivative of the GELU function on the elements
+ of input `x`.
+
+ .. math::
+
+ \frac{\partial^2 \text{GELU}}{\partial x_i^2} =
+ \frac{1}{2\sqrt{2}} \left\[
+ \text{erf}'(\frac{x}{\sqrt{2}}) +
+ \frac{1}{\sqrt{2}} \text{erf}''(\frac{x}{\sqrt{2}})
+ \right]
+
+ where :math:`\text{erf}'(x) = \frac{2}{\sqrt{\pi}} \cdot \exp\{-x^2\}` and
+ :math:`\text{erf}''(x) = \frac{-4x}{\sqrt{\pi}} \cdot \exp\{-x^2\}`.
+ """
+ pi, exp, sqrt = np.pi, np.exp, np.sqrt
+ s = x / sqrt(2)
+
+ erf_prime = lambda x: (2 / sqrt(pi)) * exp(-(x ** 2)) # noqa: E731
+ erf_prime2 = lambda x: -4 * x * exp(-(x ** 2)) / sqrt(pi) # noqa: E731
+ ddx = (1 / 2 * sqrt(2)) * (1 + erf_prime(s) + (erf_prime2(s) / sqrt(2)))
+ return ddx
+
+
 class Tanh(ActivationBase):
  def __init__(self):
  """A hyperbolic tangent activation function."""
@@ -305,7 +399,7 @@ def __init__(self):
 
  Notes
  -----
- :class:`Identity` is just syntactic sugar for :class:`Affine` with
+ :class:`Identity` is syntactic sugar for :class:`Affine` with
  slope = 1 and intercept = 0.
  """
  super().__init__(slope=1, intercept=0)

numpy_ml/neural_nets/initializers/initializers.py

@@ -1,11 +1,26 @@
+"""A module containing objects to instantiate various neural network components."""
 import re
 from functools import partial
-from ast import literal_eval as eval
+from ast import literal_eval as _eval
 
 import numpy as np
 
 from ..optimizers import OptimizerBase, SGD, AdaGrad, RMSProp, Adam
-from ..activations import ActivationBase, Affine, ReLU, Tanh, Sigmoid, LeakyReLU
+from ..activations import (
+ ELU,
+ GELU,
+ SELU,
+ ReLU,
+ Tanh,
+ Affine,
+ Sigmoid,
+ Identity,
+ SoftPlus,
+ LeakyReLU,
+ Exponential,
+ HardSigmoid,
+ ActivationBase,
+)
 from ..schedulers import (
  SchedulerBase,
  ConstantScheduler,
@@ -26,18 +41,20 @@
 class ActivationInitializer(object):
  def __init__(self, param=None):
  """
- A class for initializing activation functions. Valid inputs are:
- (a) __str__ representations of `ActivationBase` instances
- (b) `ActivationBase` instances
+ A class for initializing activation functions. Valid `param` values
+ are:
+ (a) ``__str__`` representations of an `ActivationBase` instance
+ (b) `ActivationBase` instance
 
  If `param` is `None`, return the identity function: f(X) = X
  """
  self.param = param
 
  def __call__(self):
+ """Initialize activation function"""
  param = self.param
  if param is None:
- act = Affine(slope=1, intercept=0)
+ act = Identity()
  elif isinstance(param, ActivationBase):
  act = param
  elif isinstance(param, str):
@@ -47,13 +64,24 @@ def __call__(self):
  return act
 
  def init_from_str(self, act_str):
+ """Initialize activation function from the `param` string"""
  act_str = act_str.lower()
  if act_str == "relu":
  act_fn = ReLU()
  elif act_str == "tanh":
  act_fn = Tanh()
+ elif act_str == "selu":
+ act_fn = SELU()
  elif act_str == "sigmoid":
  act_fn = Sigmoid()
+ elif act_str == "identity":
+ act_fn = Identity()
+ elif act_str == "hardsigmoid":
+ act_fn = HardSigmoid()
+ elif act_str == "softplus":
+ act_fn = SoftPlus()
+ elif act_str == "exponential":
+ act_fn = Exponential()
  elif "affine" in act_str:
  r = r"affine\(slope=(.*), intercept=(.*)\)"
  slope, intercept = re.match(r, act_str).groups()
@@ -62,6 +90,14 @@ def init_from_str(self, act_str):
  r = r"leaky relu\(alpha=(.*)\)"
  alpha = re.match(r, act_str).groups()[0]
  act_fn = LeakyReLU(float(alpha))
+ elif "gelu" in act_str:
+ r = r"gelu\(approximate=(.*)\)"
+ approx = re.match(r, act_str).groups()[0] == "true"
+ act_fn = GELU(approximation=approx)
+ elif "elu" in act_str:
+ r = r"elu\(alpha=(.*)\)"
+ approx = re.match(r, act_str).groups()[0]
+ act_fn = ELU(alpha=float(alpha))
  else:
  raise ValueError("Unknown activation: {}".format(act_str))
  return act_fn
@@ -70,7 +106,8 @@ def init_from_str(self, act_str):
 class SchedulerInitializer(object):
  def __init__(self, param=None, lr=None):
  """
- A class for initializing learning rate schedulers. Valid inputs are:
+ A class for initializing learning rate schedulers. Valid `param` values
+ are:
  (a) __str__ representations of `SchedulerBase` instances
  (b) `SchedulerBase` instances
  (c) Parameter dicts (e.g., as produced via the `summary` method in
@@ -86,6 +123,7 @@ def __init__(self, param=None, lr=None):
  self.param = param
 
  def __call__(self):
+ """Initialize scheduler"""
  param = self.param
  if param is None:
  scheduler = ConstantScheduler(self.lr)
@@ -98,9 +136,10 @@ def __call__(self):
  return scheduler
 
  def init_from_str(self):
+ """Initialize scheduler from the param string"""
  r = r"([a-zA-Z]*)=([^,)]*)"
  sch_str = self.param.lower()
- kwargs = dict([(i, eval(j)) for (i, j) in re.findall(r, sch_str)])
+ kwargs = {i: _eval(j) for i, j in re.findall(r, sch_str)}
 
  if "constant" in sch_str:
  scheduler = ConstantScheduler(**kwargs)
@@ -115,6 +154,7 @@ def init_from_str(self):
  return scheduler
 
  def init_from_dict(self):
+ """Initialize scheduler from the param dictionary"""
  S = self.param
  sc = S["hyperparameters"] if "hyperparameters" in S else None
 
@@ -136,7 +176,7 @@ def init_from_dict(self):
 class OptimizerInitializer(object):
  def __init__(self, param=None):
  """
- A class for initializing optimizers. Valid inputs are:
+ A class for initializing optimizers. Valid `param` values are:
  (a) __str__ representations of `OptimizerBase` instances
  (b) `OptimizerBase` instances
  (c) Parameter dicts (e.g., as produced via the `summary` method in
@@ -147,6 +187,7 @@ def __init__(self, param=None):
  self.param = param
 
  def __call__(self):
+ """Initialize the optimizer"""
  param = self.param
  if param is None:
  opt = SGD()
@@ -159,9 +200,10 @@ def __call__(self):
  return opt
 
  def init_from_str(self):
+ """Initialize optimizer from the `param` string"""
  r = r"([a-zA-Z]*)=([^,)]*)"
  opt_str = self.param.lower()
- kwargs = dict([(i, eval(j)) for (i, j) in re.findall(r, opt_str)])
+ kwargs = {i: _eval(j) for i, j in re.findall(r, opt_str)}
  if "sgd" in opt_str:
  optimizer = SGD(**kwargs)
  elif "adagrad" in opt_str:
@@ -175,12 +217,13 @@ def init_from_str(self):
  return optimizer
 
  def init_from_dict(self):
- O = self.param
- cc = O["cache"] if "cache" in O else None
- op = O["hyperparameters"] if "hyperparameters" in O else None
+ """Initialize optimizer from the `param` dictonary"""
+ D = self.param
+ cc = D["cache"] if "cache" in D else None
+ op = D["hyperparameters"] if "hyperparameters" in D else None
 
  if op is None:
- raise ValueError("Must have `hyperparemeters` key: {}".format(O))
+ raise ValueError("`param` dictionary has no `hyperparemeters` key")
 
  if op and op["id"] == "SGD":
  optimizer = SGD()
@@ -237,6 +280,7 @@ def __init__(self, act_fn_str, mode="glorot_uniform"):
  self._fn = partial(truncated_normal, mean=0, std=1)
 
  def __call__(self, weight_shape):
+ """Initialize weights according to the specified strategy"""
  if "glorot" in self.mode:
  gain = self._calc_glorot_gain()
  W = self._fn(weight_shape, gain)

numpy_ml/tests/test_nn_activations.py

@@ -200,6 +200,27 @@ def test_leakyrelu_activation(N=50):
  i += 1
 
 
+def test_gelu_activation(N=50):
+ from numpy_ml.neural_nets.activations import GELU
+
+ N = np.inf if N is None else N
+
+ i = 0
+ while i < N:
+ n_dims = np.random.randint(1, 100)
+ z = random_stochastic_matrix(1, n_dims)
+ approx = np.random.choice([True, False])
+
+ mine = GELU(approximate=False)
+ mine_approx = GELU(approximate=True)
+ gold = lambda z: F.gelu(torch.FloatTensor(z)).numpy()
+ np.testing.assert_allclose(mine.fn(z), gold(z), rtol=1e-3)
+ assert_almost_equal(mine.fn(z), mine_approx.fn(z))
+
+ print("PASSED")
+ i += 1
+
+
 #######################################################################
 # Activation Gradients #
 #######################################################################
@@ -278,6 +299,26 @@ def test_relu_grad(N=50):
  i += 1
 
 
+def test_gelu_grad(N=50):
+ from numpy_ml.neural_nets.activations import GELU
+
+ N = np.inf if N is None else N
+
+ mine = GELU(approximate=False)
+ mine_approx = GELU(approximate=True)
+ gold = torch_gradient_generator(F.gelu)
+
+ i = 0
+ while i < N:
+ n_ex = np.random.randint(1, 100)
+ n_dims = np.random.randint(1, 100)
+ z = random_tensor((n_ex, n_dims))
+ assert_almost_equal(mine.grad(z), gold(z), decimal=3)
+ assert_almost_equal(mine.grad(z), mine_approx.grad(z))
+ print("PASSED")
+ i += 1
+
+
 def test_selu_grad(N=50):
  from numpy_ml.neural_nets.activations import SELU