#! /usr/bin/python
# -*- coding: utf-8 -*-
"""A file containing various activation functions."""
import tensorflow as tf
from tensorlayer.decorators import deprecated
__all__ = [
'leaky_relu',
'leaky_relu6',
'leaky_twice_relu6',
'lrelu',
'lrelu6',
'ltrelu6',
'ramp',
'swish',
'sign',
'htanh',
'hard_tanh',
'pixel_wise_softmax',
'mish',
]
[docs]def ramp(x, v_min=0, v_max=1, name=None):
"""Ramp activation function.
Reference: [tf.clip_by_value]<https://www.tensorflow.org/api_docs/python/tf/clip_by_value>
Parameters
----------
x : Tensor
input.
v_min : float
cap input to v_min as a lower bound.
v_max : float
cap input to v_max as a upper bound.
name : str
The function name (optional).
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
return tf.clip_by_value(x, clip_value_min=v_min, clip_value_max=v_max, name=name)
# @deprecated(date="2018-09-30", instructions="This API is deprecated. Please use as `tf.nn.leaky_relu`")
[docs]def leaky_relu(x, alpha=0.2, name="leaky_relu"):
"""leaky_relu can be used through its shortcut: :func:`tl.act.lrelu`.
This function is a modified version of ReLU, introducing a nonzero gradient for negative input. Introduced by the paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x >= 0: ``f(x) = x``.
Parameters
----------
x : Tensor
Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.
alpha : float
Slope.
name : str
The function name (optional).
Examples
--------
>>> import tensorlayer as tl
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.Dense(n_units=100, act=lambda x : tl.act.lrelu(x, 0.2), name='dense')(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
"""
if not (0 < alpha <= 1):
raise ValueError("`alpha` value must be in [0, 1]`")
with tf.name_scope(name) as name_scope:
x = tf.convert_to_tensor(x, name="features")
return tf.maximum(x, alpha * x, name=name_scope)
[docs]def leaky_relu6(x, alpha=0.2, name="leaky_relu6"):
""":func:`leaky_relu6` can be used through its shortcut: :func:`tl.act.lrelu6`.
This activation function is a modified version :func:`leaky_relu` introduced by the following paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
This activation function also follows the behaviour of the activation function :func:`tf.nn.relu6` introduced by the following paper:
`Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x in [0, 6]: ``f(x) = x``.
- When x > 6: ``f(x) = 6``.
Parameters
----------
x : Tensor
Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.
alpha : float
Slope.
name : str
The function name (optional).
Examples
--------
>>> import tensorlayer as tl
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.Dense(n_units=100, act=lambda x : tl.act.leaky_relu6(x, 0.2), name='dense')(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
"""
if not isinstance(alpha, tf.Tensor) and not (0 < alpha <= 1):
raise ValueError("`alpha` value must be in [0, 1]`")
with tf.name_scope(name) as name_scope:
x = tf.convert_to_tensor(x, name="features")
return tf.minimum(tf.maximum(x, alpha * x), 6, name=name_scope)
[docs]def leaky_twice_relu6(x, alpha_low=0.2, alpha_high=0.2, name="leaky_relu6"):
""":func:`leaky_twice_relu6` can be used through its shortcut: :func:`:func:`tl.act.ltrelu6`.
This activation function is a modified version :func:`leaky_relu` introduced by the following paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
This activation function also follows the behaviour of the activation function :func:`tf.nn.relu6` introduced by the following paper:
`Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
This function push further the logic by adding `leaky` behaviour both below zero and above six.
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x in [0, 6]: ``f(x) = x``.
- When x > 6: ``f(x) = 6 + (alpha_high * (x-6))``.
Parameters
----------
x : Tensor
Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.
alpha_low : float
Slope for x < 0: ``f(x) = alpha_low * x``.
alpha_high : float
Slope for x < 6: ``f(x) = 6 (alpha_high * (x-6))``.
name : str
The function name (optional).
Examples
--------
>>> import tensorlayer as tl
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.Dense(n_units=100, act=lambda x : tl.act.leaky_twice_relu6(x, 0.2, 0.2), name='dense')(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
"""
if not isinstance(alpha_high, tf.Tensor) and not (0 < alpha_high <= 1):
raise ValueError("`alpha_high` value must be in [0, 1]`")
if not isinstance(alpha_low, tf.Tensor) and not (0 < alpha_low <= 1):
raise ValueError("`alpha_low` value must be in [0, 1]`")
with tf.name_scope(name) as name_scope:
x = tf.convert_to_tensor(x, name="features")
x_is_above_0 = tf.minimum(x, 6 * (1 - alpha_high) + alpha_high * x)
x_is_below_0 = tf.minimum(alpha_low * x, 0)
return tf.maximum(x_is_above_0, x_is_below_0, name=name_scope)
[docs]def swish(x, name='swish'):
"""Swish function.
See `Swish: a Self-Gated Activation Function <https://arxiv.org/abs/1710.05941>`__.
Parameters
----------
x : Tensor
input.
name: str
function name (optional).
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
# TODO: in this case, the beta = 1, but the beta can either be a constant or a trainable parameter
with tf.name_scope(name):
x = tf.nn.sigmoid(x) * x
return x
# @tf.RegisterGradient("QuantizeGrad")
# def _sign_grad(unused_op, grad):
# return tf.clip_by_value(grad, -1, 1)
[docs]@tf.custom_gradient
def sign(x):
"""Sign function.
Clip and binarize tensor using the straight through estimator (STE) for the gradient, usually be used for
quantizing values in `Binarized Neural Networks`: https://arxiv.org/abs/1602.02830.
Parameters
----------
x : Tensor
input.
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models, Maas et al. (2013)`
http://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf
- `BinaryNet: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1, Courbariaux et al. (2016)`
https://arxiv.org/abs/1602.02830
"""
def grad(dy):
return tf.clip_by_value(dy, -1, 1)
return tf.sign(x, name='sign'), grad
# if tf.__version__ > "1.7":
# @tf.custom_gradient
# def sign(x): # https://www.tensorflow.org/versions/master/api_docs/python/tf/custom_gradient?hl=ES#top_of_page
# """Differentiable sign function using sigmoid as the derivation function,
# see `tf.sign <https://www.tensorflow.org/api_docs/python/tf/sign>`__ and `tf.custom_gradient
# <https://www.tensorflow.org/versions/master/api_docs/python/tf/custom_gradient?hl=ES#top_of_page>`__.
#
# Parameters
# ----------
# x : Tensor
# input.
#
# Returns
# -------
# Tensor
# A ``Tensor`` in the same type as ``x``.
#
# """
# tao = tf.nn.sigmoid(x)
# def grad():
# return tao * (1 - tao)
# return tf.sign(x), grad
[docs]def hard_tanh(x, name='htanh'):
"""Hard tanh activation function.
Which is a ramp function with low bound of -1 and upper bound of 1, shortcut is `htanh`.
Parameters
----------
x : Tensor
input.
name : str
The function name (optional).
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
# with tf.variable_scope("hard_tanh"):
return tf.clip_by_value(x, -1, 1, name=name)
[docs]@deprecated(date="2018-06-30", instructions="This API will be deprecated soon as tf.nn.softmax can do the same thing")
def pixel_wise_softmax(x, name='pixel_wise_softmax'):
"""Return the softmax outputs of images, every pixels have multiple label, the sum of a pixel is 1.
Usually be used for image segmentation.
Parameters
----------
x : Tensor
input.
- For 2d image, 4D tensor (batch_size, height, weight, channel), where channel >= 2.
- For 3d image, 5D tensor (batch_size, depth, height, weight, channel), where channel >= 2.
name : str
function name (optional)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
Examples
--------
>>> outputs = pixel_wise_softmax(network.outputs)
>>> dice_loss = 1 - dice_coe(outputs, y_, epsilon=1e-5)
References
----------
- `tf.reverse <https://www.tensorflow.org/versions/master/api_docs/python/array_ops.html#reverse>`__
"""
with tf.name_scope(name):
return tf.nn.softmax(x)
[docs]def mish(x):
"""Mish activation function.
Reference: [Mish: A Self Regularized Non-Monotonic Neural Activation Function .Diganta Misra, 2019]<https://arxiv.org/abs/1908.08681>
Parameters
----------
x : Tensor
input.
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
return x * tf.math.tanh(tf.math.softplus(x))
# Alias
lrelu = leaky_relu
lrelu6 = leaky_relu6
ltrelu6 = leaky_twice_relu6
htanh = hard_tanh