Impl optimizer (#4734)
* init parameter base class * optimize the Comments of optimizer * basic implimentation of optimizer * add test_optimizer * add no_grad_set to interface * update optimizer.py * python code can run * fix some problem * add sync_with_cpp to Python Program and Block * sync vars and ops in block from cpp * optimize code and add some comment * add more check for sync * update optimizer with return value of Backward * rm unused code * infer shape when create gradient vairiable * update test_optimizer * update test_program.py * update backward test * follow commentrevert-4814-Add_sequence_project_op
Qiao Longfei
8 years ago
committed by
GitHub
parent
3ae9aa93c4
commit
df0946ebe2
@ -0,0 +1,124 @@
|
||||
import paddle.v2.framework.framework as framework
|
||||
|
||||
__all__ = ['SGDOptimizer']
|
||||
|
||||
|
||||
class Optimizer(object):
|
||||
"""Optimizer Base class.
|
||||
|
||||
Define the common interface of an optimizer.
|
||||
User should not use this class directly, but need to use one of it's implementation.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def _append_optimize_op(self, block, param_and_grad):
|
||||
""" append optimize operator to block and return all the added optimize_op
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def create_backward_pass(self, loss, parameter_list=None, no_grad_set=None):
|
||||
"""
|
||||
create and add gradient Operators in BlockDesc to Compute gradients of `loss`
|
||||
for parameters in parameter_list
|
||||
|
||||
Args:
|
||||
loss: an variable generated by cost function.
|
||||
no_grad_set: variable that should not create gradient
|
||||
parameter_list: parameters that need to compute gradient and update to optimize the lost.
|
||||
|
||||
Returns:
|
||||
list of (parameters, gradients) pair.
|
||||
"""
|
||||
assert isinstance(loss, framework.Variable)
|
||||
param_grad_map = loss.block.program.append_backward(loss, no_grad_set or
|
||||
set())
|
||||
if parameter_list is not None:
|
||||
parameters = parameter_list
|
||||
else:
|
||||
params = loss.block.program.global_block().all_parameters()
|
||||
parameters = [param.name for param in params]
|
||||
params_and_grads = []
|
||||
for param in parameters:
|
||||
if param not in param_grad_map:
|
||||
raise Exception("param %s is not in map" % param)
|
||||
grad_info = param_grad_map[param]
|
||||
grad_block = loss.block.program.block(grad_info[1])
|
||||
if not grad_block.has_var(grad_info[0]):
|
||||
raise Exception("grad block[%d] did not have grad var %s" %
|
||||
grad_info[1], grad_info[0])
|
||||
param_var = loss.block.var(param)
|
||||
grad_var = grad_block.var(grad_info[0])
|
||||
if loss.block.has_var(grad_info[0]):
|
||||
params_and_grads.append((param_var, grad_var))
|
||||
else:
|
||||
params_and_grads.append((param_var, None))
|
||||
return params_and_grads
|
||||
|
||||
def create_optimization_pass(self, parameters_and_grads, loss):
|
||||
"""Add optimization operators to update gradients to variables.
|
||||
|
||||
Args:
|
||||
loss: the target that this optimization is for.
|
||||
parameters_and_grads: a list of (variable, gradient) pair to update.
|
||||
|
||||
Returns:
|
||||
optmization_op_list: a list of optimization operator that will update parameter using gradient.
|
||||
"""
|
||||
optimize_ops = []
|
||||
for param_and_grad in parameters_and_grads:
|
||||
if param_and_grad[1] is not None:
|
||||
optimize_op = self._append_optimize_op(loss.block,
|
||||
param_and_grad)
|
||||
optimize_ops.append(optimize_op)
|
||||
return optimize_ops
|
||||
|
||||
def minimize(self, loss, parameter_list=None, no_grad_set=None):
|
||||
"""Add operations to minimize `loss` by updating `parameter_list`.
|
||||
|
||||
This method combines interface `create_backward_pass()` and
|
||||
`create_optimization_pass()` into one.
|
||||
"""
|
||||
params_grads = self.create_backward_pass(loss, parameter_list,
|
||||
no_grad_set or set())
|
||||
optimize_ops = self.create_optimization_pass(params_grads, loss)
|
||||
return optimize_ops
|
||||
|
||||
|
||||
class SGDOptimizer(Optimizer):
|
||||
""" Simple SGD optimizer without any state.
|
||||
"""
|
||||
|
||||
def __init__(self, learning_rate):
|
||||
assert learning_rate is not None
|
||||
super(Optimizer, self).__init__()
|
||||
self.type = "sgd"
|
||||
self._learning_rate = learning_rate
|
||||
|
||||
def _append_optimize_op(self, block, param_and_grad):
|
||||
assert isinstance(block, framework.Block)
|
||||
lr_shape = [1]
|
||||
# create a var for learning_rate
|
||||
lr = block.create_var(dtype="float32", shape=lr_shape, lod_level=0)
|
||||
|
||||
# create an op to init the learning_rate
|
||||
init_op = block.append_op(
|
||||
type="fill_constant",
|
||||
outputs={"Out": lr},
|
||||
attrs={"shape": lr_shape,
|
||||
"value": self._learning_rate})
|
||||
|
||||
# create the optimize op
|
||||
sgd_op = block.append_op(
|
||||
type=self.type,
|
||||
inputs={
|
||||
"Param": param_and_grad[0],
|
||||
"Grad": param_and_grad[1],
|
||||
"LearningRate": lr
|
||||
},
|
||||
outputs={"ParamOut": param_and_grad[0]},
|
||||
attrs={"shape": [1],
|
||||
"value": self._learning_rate})
|
||||
|
||||
return sgd_op
|
||||
@ -0,0 +1,31 @@
|
||||
import unittest
|
||||
|
||||
import paddle.v2.framework.framework as framework
|
||||
import paddle.v2.framework.optimizer as optimizer
|
||||
|
||||
|
||||
class TestOptimizer(unittest.TestCase):
|
||||
def test_sgd_optimizer(self):
|
||||
program = framework.g_program
|
||||
block = program.global_block()
|
||||
mul_x = block.create_parameter(
|
||||
dtype="float32", shape=[5, 10], lod_level=0, name="mul.x")
|
||||
mul_y = block.create_var(
|
||||
dtype="float32", shape=[10, 8], lod_level=0, name="mul.y")
|
||||
mul_out = block.create_var(
|
||||
dtype="float32", shape=[5, 8], lod_level=0, name="mul.out")
|
||||
mul_op = block.append_op(
|
||||
type="mul",
|
||||
inputs={"X": mul_x,
|
||||
"Y": mul_y},
|
||||
outputs={"Out": mul_out},
|
||||
attrs={"x_num_col_dims": 1})
|
||||
sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01)
|
||||
opts = sgd_optimizer.minimize(mul_out)
|
||||
self.assertEqual(len(opts), 1)
|
||||
sgd_op = opts[0]
|
||||
self.assertEqual(sgd_op.type, "sgd")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue