|
|
|
@ -86,6 +86,10 @@ class ELU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.elu(x, self._alpha, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'alpha={}{}'.format(self._alpha, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class GELU(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -135,6 +139,10 @@ class GELU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.gelu(x, self._approximate, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'approximate={}{}'.format(self._approximate, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Hardshrink(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -179,6 +187,10 @@ class Hardshrink(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.hardshrink(x, self._threshold, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'threshold={}{}'.format(self._threshold, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Hardswish(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -225,6 +237,10 @@ class Hardswish(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.hardswish(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Tanh(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -262,6 +278,10 @@ class Tanh(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.tanh(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Hardtanh(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -304,6 +324,10 @@ class Hardtanh(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.hardtanh(x, self._min, self._max, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'min={}, max={}{}'.format(self._min, self._max, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PReLU(layers.Layer):
|
|
|
|
|
"""
|
|
|
|
@ -371,6 +395,11 @@ class PReLU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.prelu(x, self._weight)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'num_parameters={}, init={}, dtype={}{}'.format(
|
|
|
|
|
self._num_parameters, self._init, self._dtype, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ReLU(layers.Layer):
|
|
|
|
|
"""
|
|
|
|
@ -405,6 +434,10 @@ class ReLU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.relu(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ReLU6(layers.Layer):
|
|
|
|
|
"""
|
|
|
|
@ -440,6 +473,10 @@ class ReLU6(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.relu6(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SELU(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -486,6 +523,11 @@ class SELU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.selu(x, self._scale, self._alpha, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'scale={:.16f}, alpha={:.16f}{}'.format(self._scale, self._alpha,
|
|
|
|
|
name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LeakyReLU(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -530,6 +572,10 @@ class LeakyReLU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.leaky_relu(x, self._negative_slope, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'negative_slope={}{}'.format(self._negative_slope, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Sigmoid(layers.Layer):
|
|
|
|
|
"""
|
|
|
|
@ -566,6 +612,10 @@ class Sigmoid(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.sigmoid(x, self.name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self.name) if self.name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Hardsigmoid(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -613,6 +663,10 @@ class Hardsigmoid(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.hardsigmoid(x, name=self.name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self.name) if self.name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Softplus(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -653,6 +707,11 @@ class Softplus(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.softplus(x, self._beta, self._threshold, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'beta={}, threshold={}{}'.format(self._beta, self._threshold,
|
|
|
|
|
name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Softshrink(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -694,6 +753,10 @@ class Softshrink(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.softshrink(x, self._threshold, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'threshold={}{}'.format(self._threshold, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Softsign(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -729,6 +792,10 @@ class Softsign(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.softsign(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Swish(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -764,6 +831,10 @@ class Swish(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.swish(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Tanhshrink(layers.Layer):
|
|
|
|
|
"""
|
|
|
|
@ -799,6 +870,10 @@ class Tanhshrink(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.tanhshrink(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ThresholdedReLU(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -839,6 +914,10 @@ class ThresholdedReLU(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.thresholded_relu(x, self._threshold, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'threshold={}{}'.format(self._threshold, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LogSigmoid(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -874,6 +953,10 @@ class LogSigmoid(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.log_sigmoid(x, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = 'name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return name_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Softmax(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -997,6 +1080,10 @@ class Softmax(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.softmax(x, self._axis, self._dtype, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'axis={}{}'.format(self._axis, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LogSoftmax(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -1051,6 +1138,10 @@ class LogSoftmax(layers.Layer):
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.log_softmax(x, self._axis)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'axis={}{}'.format(self._axis, name_str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Maxout(layers.Layer):
|
|
|
|
|
r"""
|
|
|
|
@ -1111,3 +1202,7 @@ class Maxout(layers.Layer):
|
|
|
|
|
|
|
|
|
|
def forward(self, x):
|
|
|
|
|
return F.maxout(x, self._groups, self._axis, self._name)
|
|
|
|
|
|
|
|
|
|
def extra_repr(self):
|
|
|
|
|
name_str = ', name={}'.format(self._name) if self._name else ''
|
|
|
|
|
return 'groups={}, axis={}{}'.format(self._groups, self._axis, name_str)
|
|
|
|
|