|
|
|
|
@ -162,22 +162,27 @@ def polynomial_decay(learning_rate,
|
|
|
|
|
end_learning_rate=0.0001,
|
|
|
|
|
power=1.0,
|
|
|
|
|
cycle=False):
|
|
|
|
|
"""Applies polynomial decay to the initial learning rate.
|
|
|
|
|
"""
|
|
|
|
|
**polynomial_decay**
|
|
|
|
|
|
|
|
|
|
Applies polynomial decay to the initial learning rate.
|
|
|
|
|
|
|
|
|
|
.. code-block::python
|
|
|
|
|
|
|
|
|
|
if cycle:
|
|
|
|
|
decay_steps = decay_steps * ceil(global_step / decay_steps)
|
|
|
|
|
else:
|
|
|
|
|
global_step = min(global_step, decay_steps)
|
|
|
|
|
decayed_learning_rate = (learning_rate - end_learning_rate) *
|
|
|
|
|
(1 - global_step / decay_steps) ^ power + end_learning_rate
|
|
|
|
|
|
|
|
|
|
>>> if cycle:
|
|
|
|
|
>>> decay_steps = decay_steps * ceil(global_step / decay_steps)
|
|
|
|
|
>>> else:
|
|
|
|
|
>>> global_step = min(global_step, decay_steps)
|
|
|
|
|
>>> decayed_learning_rate = (learning_rate - end_learning_rate) *
|
|
|
|
|
>>> (1 - global_step / decay_steps) ^ power +
|
|
|
|
|
>>> end_learning_rate
|
|
|
|
|
Args:
|
|
|
|
|
learning_rate: A scalar float32 value or a Variable. This
|
|
|
|
|
learning_rate(Variable|float32): A scalar float32 value or a Variable. This
|
|
|
|
|
will be the initial learning rate during training
|
|
|
|
|
decay_steps: A Python `int32` number.
|
|
|
|
|
end_learning_rate: A Python `float` number.
|
|
|
|
|
power: A Python `float` number
|
|
|
|
|
cycle: Boolean. If set true, decay the learning rate every decay_steps.
|
|
|
|
|
decay_steps(int32): A Python `int32` number.
|
|
|
|
|
end_learning_rate(float): A Python `float` number.
|
|
|
|
|
power(float): A Python `float` number
|
|
|
|
|
cycle(bool, Default False): Boolean. If set true, decay the learning rate every decay_steps.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
The decayed learning rate
|
|
|
|
|
|