From 20e579ef2ad9e3afe184ae05ea31ca4b575f810f Mon Sep 17 00:00:00 2001 From: xuezhong Date: Fri, 1 Feb 2019 03:50:46 +0000 Subject: [PATCH 1/4] add initial_accumulator_value for adagrad test=develop --- python/paddle/fluid/optimizer.py | 14 +++++++++++++- .../paddle/fluid/tests/unittests/test_optimizer.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index e0e781a322..ce5e5c4f37 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -662,7 +662,8 @@ class AdagradOptimizer(Optimizer): learning_rate, epsilon=1.0e-6, regularization=None, - name=None): + name=None, + initial_accumulator_value=0.1): assert learning_rate is not None assert epsilon is not None super(AdagradOptimizer, self).__init__( @@ -671,6 +672,7 @@ class AdagradOptimizer(Optimizer): name=name) self.type = "adagrad" self._epsilon = epsilon + self.initial_accumulator_value = initial_accumulator_value def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -683,6 +685,16 @@ class AdagradOptimizer(Optimizer): moment_acc = self._get_accumulator(self._moment_acc_str, param_and_grad[0]) + startup_block = framework.default_startup_program().global_block() + startup_block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [moment_acc]}, + attrs={ + 'dtype': moment_acc.dtype, + 'value': self.initial_accumulator_value, + 'shape': moment_acc.shape, + }) # Create the adagrad optimizer op adagrad_op = block.append_op( diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index 34c9b7e006..95ddc135b3 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -274,7 +274,7 @@ class TestAdagradOptimizer(unittest.TestCase): # Check init_program init_ops = init_program.global_block().ops - self.assertEqual(len(init_ops), 2) + self.assertEqual(len(init_ops), 3) self.assertEqual(init_ops[0].type, "fill_constant") self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) self.assertEqual(init_ops[1].type, "fill_constant") From e261b60f97e31c60a775df02a9f138e47f8d67ae Mon Sep 17 00:00:00 2001 From: xuezhong Date: Fri, 1 Feb 2019 07:27:59 +0000 Subject: [PATCH 2/4] change api spec for adagrad optimizer test=develop --- paddle/fluid/API.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index f50a38842a..03478a932c 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -427,7 +427,7 @@ paddle.fluid.optimizer.MomentumOptimizer.__init__ ArgSpec(args=['self', 'learnin paddle.fluid.optimizer.MomentumOptimizer.apply_gradients ArgSpec(args=['self', 'params_grads'], varargs=None, keywords=None, defaults=None) paddle.fluid.optimizer.MomentumOptimizer.backward ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set', 'callbacks'], varargs=None, keywords=None, defaults=(None, None, None, None)) paddle.fluid.optimizer.MomentumOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) -paddle.fluid.optimizer.AdagradOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'epsilon', 'regularization', 'name'], varargs=None, keywords=None, defaults=(1e-06, None, None)) +paddle.fluid.optimizer.AdagradOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'epsilon', 'regularization', 'name', 'initial_accumulator_value'], varargs=None, keywords=None, defaults=(1e-06, None, None, 0.1)) paddle.fluid.optimizer.AdagradOptimizer.apply_gradients ArgSpec(args=['self', 'params_grads'], varargs=None, keywords=None, defaults=None) paddle.fluid.optimizer.AdagradOptimizer.backward ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set', 'callbacks'], varargs=None, keywords=None, defaults=(None, None, None, None)) paddle.fluid.optimizer.AdagradOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) From 794b90c93ffa081c1ed0b6cce1c49f47f18160e3 Mon Sep 17 00:00:00 2001 From: xuezhong Date: Tue, 19 Feb 2019 12:03:45 +0000 Subject: [PATCH 3/4] for backward compatibility --- paddle/fluid/API.spec | 2 +- python/paddle/fluid/optimizer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index 03478a932c..a4c426a336 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -427,7 +427,7 @@ paddle.fluid.optimizer.MomentumOptimizer.__init__ ArgSpec(args=['self', 'learnin paddle.fluid.optimizer.MomentumOptimizer.apply_gradients ArgSpec(args=['self', 'params_grads'], varargs=None, keywords=None, defaults=None) paddle.fluid.optimizer.MomentumOptimizer.backward ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set', 'callbacks'], varargs=None, keywords=None, defaults=(None, None, None, None)) paddle.fluid.optimizer.MomentumOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) -paddle.fluid.optimizer.AdagradOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'epsilon', 'regularization', 'name', 'initial_accumulator_value'], varargs=None, keywords=None, defaults=(1e-06, None, None, 0.1)) +paddle.fluid.optimizer.AdagradOptimizer.__init__ ArgSpec(args=['self', 'learning_rate', 'epsilon', 'regularization', 'name', 'initial_accumulator_value'], varargs=None, keywords=None, defaults=(1e-06, None, None, 0.0)) paddle.fluid.optimizer.AdagradOptimizer.apply_gradients ArgSpec(args=['self', 'params_grads'], varargs=None, keywords=None, defaults=None) paddle.fluid.optimizer.AdagradOptimizer.backward ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set', 'callbacks'], varargs=None, keywords=None, defaults=(None, None, None, None)) paddle.fluid.optimizer.AdagradOptimizer.minimize ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None, None)) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index ce5e5c4f37..61dedbe93c 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -663,7 +663,7 @@ class AdagradOptimizer(Optimizer): epsilon=1.0e-6, regularization=None, name=None, - initial_accumulator_value=0.1): + initial_accumulator_value=0.0): assert learning_rate is not None assert epsilon is not None super(AdagradOptimizer, self).__init__( From 46fcadec185a9c4347004a4c093dbf8a36005eb2 Mon Sep 17 00:00:00 2001 From: xuezhong Date: Wed, 20 Feb 2019 17:00:48 +0000 Subject: [PATCH 4/4] add parameter description test=develop --- python/paddle/fluid/optimizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 4fb570d957..cb799b6396 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -649,6 +649,7 @@ class AdagradOptimizer(Optimizer): regularization: A Regularizer, such as fluid.regularizer.L2DecayRegularizer. name: A optional name prefix. + initial_accumulator_value (float): Initial value for moment accumulator. Examples: .. code-block:: python