From 23e38216a780387f00409f47977d9b3a2776db70 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 5 Dec 2017 13:57:19 +0800 Subject: [PATCH 01/84] add dilation --- paddle/operators/conv_transpose_cudnn_op.cc | 4 ---- paddle/operators/conv_transpose_op.cc | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/paddle/operators/conv_transpose_cudnn_op.cc b/paddle/operators/conv_transpose_cudnn_op.cc index 0192178ce3..8d5804fce5 100644 --- a/paddle/operators/conv_transpose_cudnn_op.cc +++ b/paddle/operators/conv_transpose_cudnn_op.cc @@ -22,8 +22,6 @@ class CudnnConv2DTransposeOpMaker : public Conv2DTransposeOpMaker { CudnnConv2DTransposeOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : Conv2DTransposeOpMaker(proto, op_checker) { - AddAttr>("dilations", "dilations of convolution operator.") - .SetDefault({1, 1}); AddAttr("workspace_size_MB", "workspace size for cudnn, in MB, " "workspace is a section of GPU memory which will be " @@ -39,8 +37,6 @@ class CudnnConv3DTransposeOpMaker : public Conv3DTransposeOpMaker { CudnnConv3DTransposeOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : Conv3DTransposeOpMaker(proto, op_checker) { - AddAttr>("dilations", "dilations of convolution operator.") - .SetDefault({1, 1, 1}); AddAttr("workspace_size_MB", "workspace size for cudnn, in MB, " "workspace is a section of GPU memory which will be " diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index 678b192dea..e900ad452e 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -73,6 +73,12 @@ Conv2DTransposeOpMaker::Conv2DTransposeOpMaker( AddOutput("Output", "(Tensor) The output tensor of convolution transpose operator. " "The format of output tensor is also NCHW."); + + AddAttr>("dilations", + "(vector default:{1, 1}), the " + "dilations(h_dilation, w_dilation) of convolution " + "transpose operator.") + .SetDefault({1, 1}); AddAttr>( "strides", "(vector default:{1, 1}), the strides(h_stride, w_stride) of " @@ -87,7 +93,7 @@ Conv2DTransposeOpMaker::Conv2DTransposeOpMaker( Convolution2D Transpose Operator. The convolution transpose operation calculates the output based on the input, filter -and strides, paddings, groups parameters. The size of each dimension of the +and dilations, strides, paddings, groups parameters. The size of each dimension of the parameters is checked in the infer-shape. Input(Input) and output(Output) are in NCHW format. Where N is batchsize, C is the number of channels, H is the height of the feature, and W is the width of the feature. @@ -136,6 +142,13 @@ Conv3DTransposeOpMaker::Conv3DTransposeOpMaker( "Where N is batch size, C is " "the number of channels, D is the depth of the feature, H is the " "height of the feature, and W is the width of the feature."); + + AddAttr>( + "dilations", + "(vector default:{1, 1, 1}), the " + "dilations(d_dilation,h_dilation, w_dilation) of convolution " + "transpose operator.") + .SetDefault({1, 1, 1}); AddAttr>("strides", "(vector default:{1, 1, 1}), the " "strides{d_stride, h_stride, w_stride} of " @@ -149,7 +162,7 @@ Conv3DTransposeOpMaker::Conv3DTransposeOpMaker( Convolution3D Transpose Operator. The convolution transpose operation calculates the output based on the input, filter -and strides, paddings, groups parameters. The size of each dimension of the +and dilations, strides, paddings, groups parameters. The size of each dimension of the parameters is checked in the infer-shape. Input(Input) and output(Output) are in NCDHW format. Where N is batch size, C is the number of channels, D is the depth of the feature, H is the height of the feature, From b18ca5f873b2c478b307f9110aab4812a82f67a8 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Tue, 5 Dec 2017 20:29:19 +0800 Subject: [PATCH 02/84] wip api for dist train --- python/paddle/v2/fluid/distribute_planner.py | 190 ++++++++++++++++++ .../book/test_recognize_digits_conv_dist.py | 60 ++++++ 2 files changed, 250 insertions(+) create mode 100644 python/paddle/v2/fluid/distribute_planner.py create mode 100644 python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py diff --git a/python/paddle/v2/fluid/distribute_planner.py b/python/paddle/v2/fluid/distribute_planner.py new file mode 100644 index 0000000000..86b11ac558 --- /dev/null +++ b/python/paddle/v2/fluid/distribute_planner.py @@ -0,0 +1,190 @@ +import framework +from backward import append_backward_ops +from regularizer import append_regularization_ops +import optimizer +from layer_helper import LayerHelper + +__all__ = ['SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'DecayedAdagrad'] + + +def hash_name_to_server(parameters_and_grads, pserver_endpoints): + def _hash_param(param_name, total): + return hash(param_name) % total + + param_map = dict() + grad_map = dict() + for param_and_grad in parameters_and_grads: + if param_and_grad[0].trainable is True and param_and_grad[ + 1] is not None: + server_id = _hash_param(param_and_grad[0].name, + len(pserver_endpoints)) + server_for_param = pserver_endpoints[server_id] + if param_map.has_key(server_for_param): + param_map[server_for_param].append(param_and_grad[0]) + else: + param_map[server_for_param] = [param_and_grad[0]] + + if grad_map.has_key(server_for_param): + grad_map[server_for_param].append(param_and_grad[1]) + else: + grad_map[server_for_param] = [param_and_grad[1]] + return param_map, grad_map + + +def round_robin(parameters_and_grads, pserver_endpoints): + if len(parameters_and_grads) < len(pserver_endpoints): + raise Exception("parameters is less than pservers") + + param_map = dict() + grad_map = dict() + pserver_idx = 0 + for param_and_grad in parameters_and_grads: + if param_and_grad[0].trainable is True and param_and_grad[ + 1] is not None: + + server_for_param = pserver_endpoints[pserver_idx] + if param_map.has_key(server_for_param): + param_map[server_for_param].append(param_and_grad[0]) + else: + param_map[server_for_param] = [param_and_grad[0]] + + if grad_map.has_key(server_for_param): + grad_map[server_for_param].append(param_and_grad[1]) + else: + grad_map[server_for_param] = [param_and_grad[1]] + pserver_idx += 1 + if pserver_idx > len(pserver_endpoints): + pserver_idx = 0 + return param_map, grad_map + + +def _append_sendop_for_trainer(loss, + parameters_and_grads, + pserver_endpoints, + split_method=round_robin): + assert (callable(split_method)) + param_map, grad_map = \ + split_method(parameters_and_grads, pserver_endpoints) + + for ep in pserver_endpoints: + # FIXME(typhoonzero): send to different servers can run in parrallel. + send_op = loss.block.append_op( + type="send", + inputs={"X": param_map[ep]}, + outputs={"Out": param_map[ep]}, + attrs={"endpoint": ep}) + + return send_op + + +class DistributedPlanner(optimizer.Optimizer): + def __init__(self, global_step=None, parallelism_type='dp'): + """ + parallelism_type: + dp: data parallelism + mp: model parallelism + """ + super(DistributedPlanner).__init__(self, global_step) + if parallelism_type == "mp": + raise NotImplementedError("model parallelism not implemented") + elif parallelism_type == "dp": + self.parameter_server_program_map = dict() + self.worker_program = None + else: + raise NameError("parallelism_type %s not supported" % + parallelism_type) + + def create_optimization_pass(self, + parameters_and_grads, + program, + startup_program=None): + # Create any accumulators + self.helper = LayerHelper( + self.__class__.__name__, + main_program=program, + startup_program=startup_program) + self._create_accumulators(program.global_block(), + [p[0] for p in parameters_and_grads]) + + optimize_ops = [] + for param_and_grad in parameters_and_grads: + if param_and_grad[0].trainable is True and param_and_grad[ + 1] is not None: + optimize_op = self._append_optimize_op(program.global_block(), + param_and_grad) + optimize_ops.append(optimize_op) + + # Returned list of ops can include more ops in addition + # to optimization ops + return_ops = optimize_ops + + # Get custom finish ops for subclasses + # FIXME: Need to fix this once we figure out how to handle dependencies + finish_ops = self._finish_update(program.global_block()) + if finish_ops is not None: + return_ops += finish_ops + + if self._global_step is not None: + return_ops.append( + self._increment_global_step(program.global_block())) + return return_ops + + def minimize(self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + split_method=round_robin): + """ + For distributed case, this call append backward ops and then + append sevaral send_ops at the end for each parameter server. + + Then call get_pserver_program(idx/endpoint) will return the program of + coresponding pserver program to run. + """ + params_grads = append_backward_ops(loss, parameter_list, no_grad_set) + # Add regularization if any + params_grads = append_regularization_ops(params_grads) + _append_sendop_for_trainer(loss, params_grads, self.pserver_endpoints, + split_method) + self.worker_program = loss.block.program + + optimize_sub_program = framework.Program() + optimize_ops = self.create_optimization_pass( + params_grads, optimize_sub_program, startup_program) + param_list = [] + for param_and_grad in params_grads: + if param_and_grad[0].trainable is True and param_and_grad[ + 1] is not None: + param_list.append(param_and_grad[0]) + + param_map, grad_map = \ + split_method(params_grads, self.pserver_endpoints) + + for ep in self.pserver_endpoints: + pserver_program = framework.Program() + self.parameter_server_program_map[ep] = pserver_program + pserver_program.global_block().append_op( + type="recv", + inputs={"RX": param_map[ep]}, + outputs={}, + attrs={ + "OptimizeBlock": optimize_sub_program.global_block(), + "endpoint": ep + }) + # FIXME(typhoonzero): when to use this return value? + return None + + def get_pserver_program(self, endpoint): + return self.parameter_server_program_map.get(endpoint) + + +SGD = optimizer.SGDOptimizer +Momentum = optimizer.MomentumOptimizer +Adagrad = optimizer.AdagradOptimizer +Adam = optimizer.AdamOptimizer +Adamax = optimizer.AdamaxOptimizer +DecayedAdagrad = optimizer.DecayedAdagradOptimizer + +for optcls in __all__: + eval(optcls).__base__ = DistributedPlanner diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py new file mode 100644 index 0000000000..35bf8da924 --- /dev/null +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py @@ -0,0 +1,60 @@ +from __future__ import print_function +import numpy as np +import paddle.v2 as paddle +import paddle.v2.fluid as fluid + +images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype='float32') +label = fluid.layers.data(name='label', shape=[1], dtype='int64') +conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=images, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu") +conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu") + +predict = fluid.layers.fc(input=conv_pool_2, size=10, act="softmax") +cost = fluid.layers.cross_entropy(input=predict, label=label) +avg_cost = fluid.layers.mean(x=cost) +optimizer = fluid.optimizer.Adam(learning_rate=0.01) +optimizer.minimize(avg_cost) + +accuracy = fluid.evaluator.Accuracy(input=predict, label=label) + +BATCH_SIZE = 50 +PASS_NUM = 3 +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=500), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +exe = fluid.Executor(place) +feeder = fluid.DataFeeder(feed_list=[images, label], place=place) +exe.run(fluid.default_startup_program()) + +for pass_id in range(PASS_NUM): + accuracy.reset(exe) + for data in train_reader(): + loss, acc = exe.run(fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost] + accuracy.metrics) + pass_acc = accuracy.eval(exe) + print("pass_id=" + str(pass_id) + " acc=" + str(acc) + " pass_acc=" + + str(pass_acc)) + # print loss, acc + if loss < 10.0 and pass_acc > 0.9: + # if avg cost less than 10.0 and accuracy is larger than 0.9, we think our code is good. + exit(0) + + pass_acc = accuracy.eval(exe) + print("pass_id=" + str(pass_id) + " pass_acc=" + str(pass_acc)) + +exit(1) From dd46d95fe4c3bcb21fed8264cc325361322ebd6c Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 6 Dec 2017 21:08:38 +0800 Subject: [PATCH 03/84] wip --- python/paddle/v2/fluid/distribute_planner.py | 43 ++++------- python/paddle/v2/fluid/executor.py | 75 ++++++++++++++++++++ python/paddle/v2/fluid/framework.py | 3 + 3 files changed, 92 insertions(+), 29 deletions(-) diff --git a/python/paddle/v2/fluid/distribute_planner.py b/python/paddle/v2/fluid/distribute_planner.py index 86b11ac558..2eb32b5227 100644 --- a/python/paddle/v2/fluid/distribute_planner.py +++ b/python/paddle/v2/fluid/distribute_planner.py @@ -7,55 +7,40 @@ from layer_helper import LayerHelper __all__ = ['SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'DecayedAdagrad'] -def hash_name_to_server(parameters_and_grads, pserver_endpoints): +def hash_name_to_server(parameters, pserver_endpoints): def _hash_param(param_name, total): return hash(param_name) % total param_map = dict() - grad_map = dict() - for param_and_grad in parameters_and_grads: - if param_and_grad[0].trainable is True and param_and_grad[ - 1] is not None: - server_id = _hash_param(param_and_grad[0].name, - len(pserver_endpoints)) + for param in parameters: + if param.trainable is True: + server_id = _hash_param(param.name, len(pserver_endpoints)) server_for_param = pserver_endpoints[server_id] if param_map.has_key(server_for_param): - param_map[server_for_param].append(param_and_grad[0]) + param_map[server_for_param].append(param) else: - param_map[server_for_param] = [param_and_grad[0]] + param_map[server_for_param] = [param] - if grad_map.has_key(server_for_param): - grad_map[server_for_param].append(param_and_grad[1]) - else: - grad_map[server_for_param] = [param_and_grad[1]] - return param_map, grad_map + return param_map -def round_robin(parameters_and_grads, pserver_endpoints): - if len(parameters_and_grads) < len(pserver_endpoints): - raise Exception("parameters is less than pservers") +def round_robin(parameters, pserver_endpoints): + assert (len(parameters) < len(pserver_endpoints)) param_map = dict() - grad_map = dict() pserver_idx = 0 - for param_and_grad in parameters_and_grads: - if param_and_grad[0].trainable is True and param_and_grad[ - 1] is not None: - + for param in parameters: + if param.trainable is True: server_for_param = pserver_endpoints[pserver_idx] if param_map.has_key(server_for_param): - param_map[server_for_param].append(param_and_grad[0]) + param_map[server_for_param].append(param) else: - param_map[server_for_param] = [param_and_grad[0]] + param_map[server_for_param] = [param] - if grad_map.has_key(server_for_param): - grad_map[server_for_param].append(param_and_grad[1]) - else: - grad_map[server_for_param] = [param_and_grad[1]] pserver_idx += 1 if pserver_idx > len(pserver_endpoints): pserver_idx = 0 - return param_map, grad_map + return param_map def _append_sendop_for_trainer(loss, diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index bdc82eede9..4a03e55ee0 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -1,6 +1,7 @@ import numpy as np from . import core from framework import Program, default_main_program +import distribute_planner __all__ = ['Executor', 'g_scope'] @@ -49,6 +50,80 @@ class Executor(object): self.executor = core.Executor(act_places) self.places = places + def optimize(self, optimize_ops, program=None, **kwargs): + """ + optimize the program for different runtime environment + + :param optimize_ops: op list of optimization, should be the + return value of Optimizer.minimize + :type optimize_ops: list + :param program: program to optimize, default default_main_program + :param pservers: parameter server endpoints like "m1:6174,m2:6174" + :type pservers: string + + :return: return a list of programs + """ + if program is None: + program = default_main_program() + + if kwargs.has_key("pservers"): + return self._optimize_distributed(optimize_ops, program, **kwargs) + + def _optimize_distributed(self, optimize_ops, program, **kwargs): + # remove optimize ops and add a send op to main_program + # FIXME(typhoonzero): delete_op only remove the first accurence, + # need to consider about multiple same optimize op? + for op in optimize_ops: + program.global_block().delete_op(op) + if kwargs.has_key("split_method"): + split_method = kwargs["split_method"] + else: + split_method = distribute_planner.round_robin + + assert (callable(split_method)) + pserver_endpoints = kwargs["pservers"].split(",") + params = program.global_block().all_parameters() + param_map = split_method(params, pserver_endpoints) + + for ep in pserver_endpoints: + # FIXME(typhoonzero): send to different servers can run in parrallel. + send_op = program.global_block().append_op( + type="send", + inputs={"X": param_map[ep] + }, # inputs is a list of tensors to be send + outputs={"Out": param_map[ep]}, + attrs={"endpoint": ep}) + # -------------- generate pserver program -------------- + self.parameter_server_program_map = dict() + + optimize_sub_program = Program() + optimize_ops = self.create_optimization_pass( + params_grads, optimize_sub_program, startup_program) + param_list = [] + for param in params: + if param.trainable is True: + param_list.append(param) + + param_map = split_method(params, pserver_endpoints) + + for ep in pserver_endpoints: + pserver_program = Program() + self.parameter_server_program_map[ep] = pserver_program + pserver_program.global_block().append_op( + type="recv", + inputs={"RX": param_map[ep]}, # grads to recv + outputs={}, + attrs={ + "OptimizeBlock": optimize_sub_program.global_block(), + "endpoint": ep + }) + + def get_pserver_program(self, endpoint): + pass + + def get_trainer_program(self): + return default_main_program() + def aslodtensor(self, data): def accumulate(data): if not isinstance(data, list): diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 49c6d89834..99fe94942b 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -425,6 +425,9 @@ class Block(object): self.ops.append(op) return op + def delete_op(self, op): + self.ops.remove(op) + def prepend_op(self, *args, **kwargs): op_desc = self.desc.prepend_op() op = Operator(self, op_desc, *args, **kwargs) From 71655334c61e667c6308f7100903a14ac8f099a9 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 8 Dec 2017 16:58:19 +0800 Subject: [PATCH 04/84] update --- paddle/operators/recv_op.cc | 11 +- paddle/operators/send_recv_op_test.cc | 2 +- python/paddle/v2/fluid/distribute_planner.py | 170 +++--------------- python/paddle/v2/fluid/executor.py | 52 +++--- .../book/test_recognize_digits_conv_dist.py | 45 +++-- 5 files changed, 80 insertions(+), 200 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index c69e416e10..45222f6b76 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -72,8 +72,10 @@ class RecvOp : public framework::OperatorBase { // FIXME(typhoonzero): do not copy framework::CopyFrom(t, dev_ctx.GetPlace(), dev_ctx, tensor); - auto *block = Attr("OptimizeBlock"); - auto *program = block->Program(); + std::string program_str = Attr("OptimizeProgram"); + framework::Program program_desc; + program_desc.ParseFromString(program_str); + framework::ProgramDescBind program(program_desc); framework::Executor executor(dev_ctx); // Run sub graph to get optimized tensor executor.Run(*program, &recv_scope, block->ID(), @@ -108,8 +110,9 @@ This operator will recv tensor from send_op "IP address to listen on.") .SetDefault("127.0.0.1:6164") .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); - AddAttr("OptimizeBlock", "type BlockDescBind*", - "optimize network run in server"); + AddAttr( + "OptimizeProgram", "type string", + "Serialized ProgramDesc string for recv to run."); } }; diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index ac03eb3752..c35dc8fa50 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -85,7 +85,7 @@ void StartServerNet() { paddle::framework::AttributeMap attrs; attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); - attrs.insert({"OptimizeBlock", block}); + attrs.insert({"OptimizeProgram", program.Proto()->SerializeToString()}); recv_op = paddle::framework::OpRegistry::CreateOp("recv", {{"RX", {"RX"}}}, {{"Out", {"Out"}}}, attrs); paddle::platform::CPUDeviceContext ctx(place); diff --git a/python/paddle/v2/fluid/distribute_planner.py b/python/paddle/v2/fluid/distribute_planner.py index 2eb32b5227..39e9e3d9db 100644 --- a/python/paddle/v2/fluid/distribute_planner.py +++ b/python/paddle/v2/fluid/distribute_planner.py @@ -4,172 +4,46 @@ from regularizer import append_regularization_ops import optimizer from layer_helper import LayerHelper -__all__ = ['SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'DecayedAdagrad'] +def hash_name_to_server(params_grads, pserver_endpoints): + """ + :param param_grads: + :return: a map of pserver endpoint -> + params -> [param list] + grads -> [grad list] + """ -def hash_name_to_server(parameters, pserver_endpoints): def _hash_param(param_name, total): return hash(param_name) % total - param_map = dict() - for param in parameters: - if param.trainable is True: + param_grad_map = dict() + for param, grad in params_grads: + if param.trainable is True and grad is not None: server_id = _hash_param(param.name, len(pserver_endpoints)) server_for_param = pserver_endpoints[server_id] - if param_map.has_key(server_for_param): - param_map[server_for_param].append(param) - else: - param_map[server_for_param] = [param] + if not param_grad_map.has_key(server_for_param): + param_grad_map[server_for_param] = {"params": [], "grads": []} + param_grad_map[server_for_param]["params"].append(param) + param_grad_map[server_for_param]["grads"].append(grad) - return param_map + return param_grad_map def round_robin(parameters, pserver_endpoints): assert (len(parameters) < len(pserver_endpoints)) - param_map = dict() + param_grad_map = dict() pserver_idx = 0 for param in parameters: if param.trainable is True: server_for_param = pserver_endpoints[pserver_idx] - if param_map.has_key(server_for_param): - param_map[server_for_param].append(param) - else: - param_map[server_for_param] = [param] + if not param_grad_map.has_key(server_for_param): + param_grad_map[server_for_param] = {"params": [], "grads": []} + + param_grad_map[server_for_param]["params"].append(param) + param_grad_map[server_for_param]["grads"].append(param) pserver_idx += 1 if pserver_idx > len(pserver_endpoints): pserver_idx = 0 - return param_map - - -def _append_sendop_for_trainer(loss, - parameters_and_grads, - pserver_endpoints, - split_method=round_robin): - assert (callable(split_method)) - param_map, grad_map = \ - split_method(parameters_and_grads, pserver_endpoints) - - for ep in pserver_endpoints: - # FIXME(typhoonzero): send to different servers can run in parrallel. - send_op = loss.block.append_op( - type="send", - inputs={"X": param_map[ep]}, - outputs={"Out": param_map[ep]}, - attrs={"endpoint": ep}) - - return send_op - - -class DistributedPlanner(optimizer.Optimizer): - def __init__(self, global_step=None, parallelism_type='dp'): - """ - parallelism_type: - dp: data parallelism - mp: model parallelism - """ - super(DistributedPlanner).__init__(self, global_step) - if parallelism_type == "mp": - raise NotImplementedError("model parallelism not implemented") - elif parallelism_type == "dp": - self.parameter_server_program_map = dict() - self.worker_program = None - else: - raise NameError("parallelism_type %s not supported" % - parallelism_type) - - def create_optimization_pass(self, - parameters_and_grads, - program, - startup_program=None): - # Create any accumulators - self.helper = LayerHelper( - self.__class__.__name__, - main_program=program, - startup_program=startup_program) - self._create_accumulators(program.global_block(), - [p[0] for p in parameters_and_grads]) - - optimize_ops = [] - for param_and_grad in parameters_and_grads: - if param_and_grad[0].trainable is True and param_and_grad[ - 1] is not None: - optimize_op = self._append_optimize_op(program.global_block(), - param_and_grad) - optimize_ops.append(optimize_op) - - # Returned list of ops can include more ops in addition - # to optimization ops - return_ops = optimize_ops - - # Get custom finish ops for subclasses - # FIXME: Need to fix this once we figure out how to handle dependencies - finish_ops = self._finish_update(program.global_block()) - if finish_ops is not None: - return_ops += finish_ops - - if self._global_step is not None: - return_ops.append( - self._increment_global_step(program.global_block())) - return return_ops - - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - split_method=round_robin): - """ - For distributed case, this call append backward ops and then - append sevaral send_ops at the end for each parameter server. - - Then call get_pserver_program(idx/endpoint) will return the program of - coresponding pserver program to run. - """ - params_grads = append_backward_ops(loss, parameter_list, no_grad_set) - # Add regularization if any - params_grads = append_regularization_ops(params_grads) - _append_sendop_for_trainer(loss, params_grads, self.pserver_endpoints, - split_method) - self.worker_program = loss.block.program - - optimize_sub_program = framework.Program() - optimize_ops = self.create_optimization_pass( - params_grads, optimize_sub_program, startup_program) - param_list = [] - for param_and_grad in params_grads: - if param_and_grad[0].trainable is True and param_and_grad[ - 1] is not None: - param_list.append(param_and_grad[0]) - - param_map, grad_map = \ - split_method(params_grads, self.pserver_endpoints) - - for ep in self.pserver_endpoints: - pserver_program = framework.Program() - self.parameter_server_program_map[ep] = pserver_program - pserver_program.global_block().append_op( - type="recv", - inputs={"RX": param_map[ep]}, - outputs={}, - attrs={ - "OptimizeBlock": optimize_sub_program.global_block(), - "endpoint": ep - }) - # FIXME(typhoonzero): when to use this return value? - return None - - def get_pserver_program(self, endpoint): - return self.parameter_server_program_map.get(endpoint) - - -SGD = optimizer.SGDOptimizer -Momentum = optimizer.MomentumOptimizer -Adagrad = optimizer.AdagradOptimizer -Adam = optimizer.AdamOptimizer -Adamax = optimizer.AdamaxOptimizer -DecayedAdagrad = optimizer.DecayedAdagradOptimizer - -for optcls in __all__: - eval(optcls).__base__ = DistributedPlanner + return param_grad_map diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index 4a03e55ee0..ee7497e305 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -69,7 +69,8 @@ class Executor(object): if kwargs.has_key("pservers"): return self._optimize_distributed(optimize_ops, program, **kwargs) - def _optimize_distributed(self, optimize_ops, program, **kwargs): + def _optimize_distributed(self, optimize_ops, program, params_and_grads, + **kwargs): # remove optimize ops and add a send op to main_program # FIXME(typhoonzero): delete_op only remove the first accurence, # need to consider about multiple same optimize op? @@ -83,43 +84,36 @@ class Executor(object): assert (callable(split_method)) pserver_endpoints = kwargs["pservers"].split(",") params = program.global_block().all_parameters() - param_map = split_method(params, pserver_endpoints) + self.param_grad_map = split_method(params, pserver_endpoints) for ep in pserver_endpoints: # FIXME(typhoonzero): send to different servers can run in parrallel. send_op = program.global_block().append_op( type="send", - inputs={"X": param_map[ep] + inputs={"X": self.param_grad_map[ep]["params"] }, # inputs is a list of tensors to be send - outputs={"Out": param_map[ep]}, + outputs={"Out": self.param_grad_map[ep]["params"]}, attrs={"endpoint": ep}) - # -------------- generate pserver program -------------- - self.parameter_server_program_map = dict() - - optimize_sub_program = Program() - optimize_ops = self.create_optimization_pass( - params_grads, optimize_sub_program, startup_program) - param_list = [] - for param in params: - if param.trainable is True: - param_list.append(param) - - param_map = split_method(params, pserver_endpoints) - - for ep in pserver_endpoints: - pserver_program = Program() - self.parameter_server_program_map[ep] = pserver_program - pserver_program.global_block().append_op( - type="recv", - inputs={"RX": param_map[ep]}, # grads to recv - outputs={}, - attrs={ - "OptimizeBlock": optimize_sub_program.global_block(), - "endpoint": ep - }) + # -------------- generate optimize sub program -------------- + self.optimize_sub_program = Program() + for opt_op in optimize_ops: + self.optimize_sub_program.global_block().ops.append(opt_op) def get_pserver_program(self, endpoint): - pass + pserver_program = Program() + + for param in self.param_grad_map[endpoint]["params"]: + pserver_program.global_block().create_parameter(**param.__dict__) + + pserver_program.global_block().append_op( + type="recv", + inputs={"RX": + self.param_grad_map[endpoint]["grads"]}, # grads to recv + outputs={}, + attrs={ + "OptimizeProgram": self.optimize_sub_program.to_string(), + "endpoint": endpoint + }) def get_trainer_program(self): return default_main_program() diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py index 35bf8da924..b856526114 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py @@ -37,24 +37,33 @@ train_reader = paddle.batch( place = fluid.CPUPlace() exe = fluid.Executor(place) -feeder = fluid.DataFeeder(feed_list=[images, label], place=place) -exe.run(fluid.default_startup_program()) - -for pass_id in range(PASS_NUM): - accuracy.reset(exe) - for data in train_reader(): - loss, acc = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[avg_cost] + accuracy.metrics) + +exe.optimize(pservers="127.0.0.1:6174", trainers=1) + +pserver_endpoint = os.getenv("PSERVER") +if is_pserver: + pserver_prog = exe.get_pserver_program(pserver_endpoint) + exe.run(fluid.default_startup_program()) + exe.run(pserver_prog) +else: + feeder = fluid.DataFeeder(feed_list=[images, label], place=place) + exe.run(fluid.default_startup_program()) + + for pass_id in range(PASS_NUM): + accuracy.reset(exe) + for data in train_reader(): + loss, acc = exe.run(fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost] + accuracy.metrics) + pass_acc = accuracy.eval(exe) + print("pass_id=" + str(pass_id) + " acc=" + str(acc) + " pass_acc=" + + str(pass_acc)) + # print loss, acc + if loss < 10.0 and pass_acc > 0.9: + # if avg cost less than 10.0 and accuracy is larger than 0.9, we think our code is good. + exit(0) + pass_acc = accuracy.eval(exe) - print("pass_id=" + str(pass_id) + " acc=" + str(acc) + " pass_acc=" + - str(pass_acc)) - # print loss, acc - if loss < 10.0 and pass_acc > 0.9: - # if avg cost less than 10.0 and accuracy is larger than 0.9, we think our code is good. - exit(0) - - pass_acc = accuracy.eval(exe) - print("pass_id=" + str(pass_id) + " pass_acc=" + str(pass_acc)) + print("pass_id=" + str(pass_id) + " pass_acc=" + str(pass_acc)) exit(1) From aa770198c72c115310e6075ebd403878154fbf0f Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 8 Dec 2017 17:36:46 +0800 Subject: [PATCH 05/84] add dilation in c++ code --- paddle/operators/conv_transpose_op.cc | 7 ++++++- paddle/operators/conv_transpose_op.h | 14 ++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index e900ad452e..c31a2e4a70 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -29,6 +29,7 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const { auto filter_dims = ctx->GetInputDim("Filter"); std::vector strides = ctx->Attrs().Get>("strides"); std::vector paddings = ctx->Attrs().Get>("paddings"); + std::vector dilations = ctx->Attrs().Get>("dilations"); PADDLE_ENFORCE(in_dims.size() == 4 || in_dims.size() == 5, "ConvTransposeOp intput should be 4-D or 5-D tensor."); @@ -41,14 +42,18 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const { PADDLE_ENFORCE_EQ(paddings.size(), strides.size(), "ConvTransposeOp paddings dimension and strides " "dimension should be the same."); + PADDLE_ENFORCE_EQ(paddings.size(), dilations.size(), + "ConvTransposeOp paddings dimension and dilations " + "dimension should be the same."); PADDLE_ENFORCE_EQ(in_dims[1], filter_dims[0], "In ConvTransposeOp, The input channel should be the same " "as the number of filters."); std::vector output_shape({in_dims[0], filter_dims[1]}); for (size_t i = 0; i < strides.size(); ++i) { + auto filter_extent = dilations[i] * (filter_dims[i + 2] - 1) + 1; output_shape.push_back((in_dims[i + 2] - 1) * strides[i] - 2 * paddings[i] + - filter_dims[i + 2]); + filter_extent); } ctx->SetOutputDim("Output", framework::make_ddim(output_shape)); } diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h index 1cacb770e6..65a0076d9c 100644 --- a/paddle/operators/conv_transpose_op.h +++ b/paddle/operators/conv_transpose_op.h @@ -63,6 +63,7 @@ class GemmConvTransposeKernel : public framework::OpKernel { std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); + std::vector dilations = context.Attr>("dilations"); // groups will alway be disabled in conv2dtranspose. const int batch_size = static_cast(input->dims()[0]); @@ -114,7 +115,6 @@ class GemmConvTransposeKernel : public framework::OpKernel { math::Col2ImFunctor col2im; math::Col2VolFunctor col2vol; - std::vector dilations({1, 1, 1}); // convolution transpose: gemm + col2im or col2vol (similar to conv-backward // on input) @@ -134,8 +134,7 @@ class GemmConvTransposeKernel : public framework::OpKernel { if (data_dim == 2U) { // col2im: col_matrix -> dy // from (c * k_h * k_w, h * w) to (c, o_h, o_w) - col2im(context.device_context(), col, - std::vector{dilations[0], dilations[1]}, strides, + col2im(context.device_context(), col, dilations, strides, std::vector{paddings[0], paddings[1], paddings[0], paddings[1]}, &output_batch); @@ -168,6 +167,7 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); + std::vector dilations = context.Attr>("dilations"); const int batch_size = static_cast(input->dims()[0]); @@ -221,7 +221,6 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { math::Im2ColFunctor im2col; math::Vol2ColFunctor vol2col; - std::vector dilations({1, 1, 1}); if (input_grad) { input_grad->mutable_data(context.GetPlace()); @@ -242,10 +241,9 @@ class GemmConvTransposeGradKernel : public framework::OpKernel { if (data_dim == 2U) { // im2col: dy -> col matrix // from (c, o_h, o_w) to (c * k_h * k_w, h * w) - im2col(context.device_context(), output_grad_batch, - std::vector{dilations[0], dilations[1]}, strides, - std::vector{paddings[0], paddings[1], paddings[0], - paddings[1]}, + im2col(context.device_context(), output_grad_batch, dilations, + strides, std::vector{paddings[0], paddings[1], + paddings[0], paddings[1]}, &col); } else if (data_dim == 3U) { // vol2col: dy -> col_matrix From d93bbf1b35137bece595f9ad26003904368ba845 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 8 Dec 2017 18:59:04 +0800 Subject: [PATCH 06/84] add conv_trans unit test --- .../fluid/tests/test_conv2d_transpose_op.py | 73 ++++++++++++++--- .../fluid/tests/test_conv3d_transpose_op.py | 82 ++++++++++++++++--- 2 files changed, 132 insertions(+), 23 deletions(-) diff --git a/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py b/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py index d7b1f2f2a3..d59537b924 100644 --- a/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py +++ b/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py @@ -3,14 +3,17 @@ import numpy as np from op_test import OpTest -def conv2dtranspose_forward_naive(input_, filter_, conv2dtranspose_param): +def conv2dtranspose_forward_naive(input_, filter_, attrs): in_n, in_c, in_h, in_w = input_.shape f_c, out_c, f_h, f_w = filter_.shape assert in_c == f_c - stride, pad = conv2dtranspose_param['stride'], conv2dtranspose_param['pad'] - out_h = (in_h - 1) * stride[0] + f_h - out_w = (in_w - 1) * stride[1] + f_w + stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ + 'dilations'] + d_bolck_h = dilations[0] * (f_h - 1) + 1 + d_bolck_w = dilations[1] * (f_w - 1) + 1 + out_h = (in_h - 1) * stride[0] + d_bolck_h + out_w = (in_w - 1) * stride[1] + d_bolck_w out = np.zeros((in_n, out_c, out_h, out_w)) @@ -23,9 +26,9 @@ def conv2dtranspose_forward_naive(input_, filter_, conv2dtranspose_param): for k in range(out_c): tmp_out = np.sum(input_masked * filter_[:, k, :, :], axis=0) - i1, i2 = i * stride[0], i * stride[0] + f_h - j1, j2 = j * stride[0], j * stride[0] + f_w - out[n, k, i1:i2, j1:j2] += tmp_out + i1, i2 = i * stride[0], i * stride[0] + d_bolck_h + j1, j2 = j * stride[0], j * stride[0] + d_bolck_h + out[n, k, i1:i2:dilations[0], j1:j2:dilations[1]] += tmp_out out = out[:, :, pad[0]:out_h - pad[0], pad[1]:out_w - pad[1]] return out @@ -37,11 +40,8 @@ class TestConv2dTransposeOp(OpTest): self.init_op_type() self.init_test_case() - conv2dtranspose_param = {'stride': self.stride, 'pad': self.pad} input_ = np.random.random(self.input_size).astype("float32") filter_ = np.random.random(self.filter_size).astype("float32") - output = conv2dtranspose_forward_naive( - input_, filter_, conv2dtranspose_param).astype('float32') self.inputs = {'Input': input_, 'Filter': filter_} self.attrs = { @@ -49,6 +49,10 @@ class TestConv2dTransposeOp(OpTest): 'paddings': self.pad, 'dilations': self.dilations } + + output = conv2dtranspose_forward_naive(input_, filter_, + self.attrs).astype('float32') + self.outputs = {'Output': output} def test_check_output(self): @@ -104,11 +108,60 @@ class TestWithStride(TestConv2dTransposeOp): self.filter_size = [f_c, 6, 3, 3] +class TestWithDilation(TestConv2dTransposeOp): + def init_test_case(self): + self.pad = [1, 1] + self.stride = [1, 1] + self.dilations = [2, 2] + self.input_size = [2, 3, 5, 5] # NCHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3] + + # ------------ test_cudnn ------------ class TestCudnn(TestConv2dTransposeOp): def init_op_type(self): self.op_type = "conv2d_transpose_cudnn" +class TestCudnnWithPad(TestWithPad): + def init_test_case(self): + self.pad = [1, 1] + self.stride = [1, 1] + self.dilations = [1, 1] + self.input_size = [2, 3, 5, 5] # NCHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3] + + def init_op_type(self): + self.op_type = "conv2d_transpose_cudnn" + + +class TestCudnnWithStride(TestWithStride): + def init_test_case(self): + self.pad = [1, 1] + self.stride = [2, 2] + self.dilations = [1, 1] + self.input_size = [2, 3, 5, 5] # NCHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3] + + def init_op_type(self): + self.op_type = "conv2d_transpose_cudnn" + + +# #cudnn v5 does not support dilation conv. +# class TestCudnnWithDilation(TestWithDilation): +# def init_test_case(self): +# self.pad = [1, 1] +# self.stride = [2, 2] +# self.dilations = [2, 2] +# self.input_size = [2, 3, 5, 5] # NCHW +# f_c = self.input_size[1] +# self.filter_size = [f_c, 6, 3, 3] +# +# def init_op_type(self): +# self.op_type = "conv2d_transpose_cudnn" + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py b/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py index 8fd34b87bf..a353f9b4d4 100644 --- a/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py +++ b/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py @@ -3,15 +3,20 @@ import numpy as np from op_test import OpTest -def conv3dtranspose_forward_naive(input_, filter_, conv3dtranspose_param): +def conv3dtranspose_forward_naive(input_, filter_, attrs): in_n, in_c, in_d, in_h, in_w = input_.shape f_c, out_c, f_d, f_h, f_w = filter_.shape assert in_c == f_c - stride, pad = conv3dtranspose_param['stride'], conv3dtranspose_param['pad'] - out_d = (in_d - 1) * stride[0] + f_d - out_h = (in_h - 1) * stride[1] + f_h - out_w = (in_w - 1) * stride[2] + f_w + stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ + 'dilations'] + + d_bolck_d = dilations[0] * (f_d - 1) + 1 + d_bolck_h = dilations[1] * (f_h - 1) + 1 + d_bolck_w = dilations[2] * (f_w - 1) + 1 + out_d = (in_d - 1) * stride[0] + d_bolck_d + out_h = (in_h - 1) * stride[1] + d_bolck_h + out_w = (in_w - 1) * stride[2] + d_bolck_w out = np.zeros((in_n, out_c, out_d, out_h, out_w)) for n in range(in_n): @@ -25,10 +30,11 @@ def conv3dtranspose_forward_naive(input_, filter_, conv3dtranspose_param): for k in range(out_c): tmp_out = np.sum(input_masked * filter_[:, k, :, :, :], axis=0) - d1, d2 = d * stride[0], d * stride[0] + f_d - i1, i2 = i * stride[1], i * stride[1] + f_h - j1, j2 = j * stride[2], j * stride[2] + f_w - out[n, k, d1:d2, i1:i2, j1:j2] += tmp_out + d1, d2 = d * stride[0], d * stride[0] + d_bolck_d + i1, i2 = i * stride[1], i * stride[1] + d_bolck_h + j1, j2 = j * stride[2], j * stride[2] + d_bolck_w + out[n, k, d1:d2:dilations[0], i1:i2:dilations[1], j1:j2: + dilations[2]] += tmp_out out = out[:, :, pad[0]:out_d - pad[0], pad[1]:out_h - pad[1], pad[2]:out_w - pad[2]] @@ -41,18 +47,19 @@ class TestConv3dTransposeOp(OpTest): self.init_op_type() self.init_test_case() - conv3dtranspose_param = {'stride': self.stride, 'pad': self.pad} input_ = np.random.random(self.input_size).astype("float32") filter_ = np.random.random(self.filter_size).astype("float32") - output = conv3dtranspose_forward_naive( - input_, filter_, conv3dtranspose_param).astype("float32") self.inputs = {'Input': input_, 'Filter': filter_} self.attrs = { 'strides': self.stride, 'paddings': self.pad, - # 'dilations': self.dilations + 'dilations': self.dilations } + + output = conv3dtranspose_forward_naive(input_, filter_, + self.attrs).astype("float32") + self.outputs = {'Output': output} def test_check_output(self): @@ -108,11 +115,60 @@ class TestWithStride(TestConv3dTransposeOp): self.filter_size = [f_c, 6, 3, 3, 3] +class TestWithDilation(TestConv3dTransposeOp): + def init_test_case(self): + self.pad = [1, 1, 1] + self.stride = [1, 1, 1] + self.dilations = [2, 2, 2] + self.input_size = [2, 3, 5, 5, 5] # NCDHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3, 3] + + # ------------ test_cudnn ------------ class TestCudnn(TestConv3dTransposeOp): def init_op_type(self): self.op_type = "conv3d_transpose_cudnn" +class TestCudnnWithPad(TestWithPad): + def init_test_case(self): + self.pad = [1, 1, 1] + self.stride = [1, 1, 1] + self.dilations = [1, 1, 1] + self.input_size = [2, 3, 5, 5, 5] # NCDHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3, 3] + + def init_op_type(self): + self.op_type = "conv3d_transpose_cudnn" + + +class TestCudnnWithStride(TestWithStride): + def init_test_case(self): + self.pad = [1, 1, 1] + self.stride = [2, 2, 2] + self.dilations = [1, 1, 1] + self.input_size = [2, 3, 5, 5, 5] # NCDHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3, 3] + + def init_op_type(self): + self.op_type = "conv3d_transpose_cudnn" + + +# #cudnn v5 does not support dilation conv. +# class TestCudnnWithDilation(TestWithDilation): +# def init_test_case(self): +# self.pad = [1, 1, 1] +# self.stride = [2, 2, 2] +# self.dilations = [2, 2, 2] +# self.input_size = [2, 3, 5, 5, 5] # NCDHW +# f_c = self.input_size[1] +# self.filter_size = [f_c, 6, 3, 3, 3] +# +# def init_op_type(self): +# self.op_type = "conv3d_transpose_cudnn" + if __name__ == '__main__': unittest.main() From 1c1fae607748f76032d2ff246b47314f425e29ce Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 8 Dec 2017 19:44:12 +0800 Subject: [PATCH 07/84] update recv op --- paddle/operators/recv_op.cc | 9 ++++----- paddle/operators/send_recv_op_test.cc | 5 ++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 45222f6b76..eed482c1b4 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -73,12 +73,12 @@ class RecvOp : public framework::OperatorBase { framework::CopyFrom(t, dev_ctx.GetPlace(), dev_ctx, tensor); std::string program_str = Attr("OptimizeProgram"); - framework::Program program_desc; + framework::ProgramDesc program_desc; program_desc.ParseFromString(program_str); framework::ProgramDescBind program(program_desc); framework::Executor executor(dev_ctx); // Run sub graph to get optimized tensor - executor.Run(*program, &recv_scope, block->ID(), + executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/); auto *out_var = recv_scope.FindVar("Out"); @@ -110,9 +110,8 @@ This operator will recv tensor from send_op "IP address to listen on.") .SetDefault("127.0.0.1:6164") .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); - AddAttr( - "OptimizeProgram", "type string", - "Serialized ProgramDesc string for recv to run."); + AddAttr("OptimizeProgram", "type string", + "Serialized ProgramDesc string for recv to run."); } }; diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index c35dc8fa50..3e2e2051af 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -85,7 +85,10 @@ void StartServerNet() { paddle::framework::AttributeMap attrs; attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); - attrs.insert({"OptimizeProgram", program.Proto()->SerializeToString()}); + std::string program_proto; + PADDLE_ENFORCE(program.Proto()->SerializeToString(&program_proto)); + + attrs.insert({"OptimizeProgram", program_proto}); recv_op = paddle::framework::OpRegistry::CreateOp("recv", {{"RX", {"RX"}}}, {{"Out", {"Out"}}}, attrs); paddle::platform::CPUDeviceContext ctx(place); From 5f48421cc3718f3af2c8b90cf206089f1702592d Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 8 Dec 2017 20:03:31 +0800 Subject: [PATCH 08/84] fix conv2d_transpose API (Add dilation) --- python/paddle/v2/fluid/layers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers.py b/python/paddle/v2/fluid/layers.py index 99d0ac4a1b..7c1514efad 100644 --- a/python/paddle/v2/fluid/layers.py +++ b/python/paddle/v2/fluid/layers.py @@ -1537,6 +1537,7 @@ def conv2d_transpose(input, filter_size=None, padding=None, stride=None, + dilation=None, param_attr=None, main_program=None, startup_program=None): @@ -1562,6 +1563,9 @@ def conv2d_transpose(input, stride(int|tuple): The stride size. If stride is a tuple, it must contain two integers, (stride_H, stride_W). Otherwise, the stride_H = stride_W = stride. + dilation(int|tuple): The dilation size. If dilation is a tuple, it must + contain two integers, (dilation_H, dilation_W). Otherwise, the + dilation_H = dilation_W = dilation. param_attr: Parameter Attribute. main_program(Program): the main program startup_program(Program): the startup program @@ -1586,6 +1590,11 @@ def conv2d_transpose(input, elif stride is not None: op_attr['strides'] = stride + if isinstance(dilation, int): + op_attr['dilations'] = dilation + elif stride is not None: + op_attr['dilations'] = dilation + if filter_size is None: if output_size is None: raise ValueError("output_size must be set when filter_size is None") @@ -1594,11 +1603,14 @@ def conv2d_transpose(input, padding = op_attr.get('paddings', [0, 0]) stride = op_attr.get('strides', [1, 1]) + dilation = op_attr.get('dilations', [1, 1]) h_in = input.shape[2] w_in = input.shape[3] - filter_size_h = output_size[0] - (h_in - 1) * stride[0] + 2 * padding[0] - filter_size_w = output_size[1] - (w_in - 1) * stride[1] + 2 * padding[1] + filter_size_h = (output_size[0] - (h_in - 1) * stride[0] + 2 * + padding[0] - 1) / dilation[0] + 1 + filter_size_w = (output_size[1] - (w_in - 1) * stride[1] + 2 * + padding[1] - 1) / dilation[1] + 1 filter_size = [filter_size_h, filter_size_w] elif isinstance(filter_size, int): filter_size = [filter_size, filter_size] From 308491a94a0a4f0d18d6a97e17d2c329f3023828 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 11 Dec 2017 13:06:13 +0800 Subject: [PATCH 09/84] update for simple dist train --- paddle/operators/send_op.cc | 14 +++++++------- python/paddle/v2/fluid/distribute_planner.py | 4 ++-- python/paddle/v2/fluid/executor.py | 7 ++++--- python/paddle/v2/fluid/framework.py | 3 ++- python/paddle/v2/fluid/optimizer.py | 2 +- .../tests/book/test_recognize_digits_conv_dist.py | 7 ++++--- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index a3059847f2..7cbc45e69a 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -43,13 +43,14 @@ class SendOp : public framework::OperatorBase { } void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { - auto iname = Input("X"); - auto oname = Output("Out"); + auto ins = Inputs("X"); // TODO(typhoonzero): currently it's non-blocking, // should block until server responds. - bool ret = client_->SendVariable(scope, iname, oname); - if (!ret) { - LOG(ERROR) << "send variable error"; + for (auto in : ins) { + bool ret = client_->SendVariable(scope, in, in); + if (!ret) { + LOG(ERROR) << "send variable error"; + } } } @@ -61,8 +62,7 @@ class SendOpMaker : public framework::OpProtoAndCheckerMaker { public: SendOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "(Tensor) Input tensor to be saved"); - AddOutput("Out", "(Tensor) Output fetched from server"); + AddInput("X", "(Tensor) Input tensor to be send").AsDuplicable(); AddComment(R"DOC( Recv operator diff --git a/python/paddle/v2/fluid/distribute_planner.py b/python/paddle/v2/fluid/distribute_planner.py index 39e9e3d9db..3d8df4b3c8 100644 --- a/python/paddle/v2/fluid/distribute_planner.py +++ b/python/paddle/v2/fluid/distribute_planner.py @@ -30,7 +30,7 @@ def hash_name_to_server(params_grads, pserver_endpoints): def round_robin(parameters, pserver_endpoints): - assert (len(parameters) < len(pserver_endpoints)) + assert (len(parameters) > len(pserver_endpoints)) param_grad_map = dict() pserver_idx = 0 @@ -44,6 +44,6 @@ def round_robin(parameters, pserver_endpoints): param_grad_map[server_for_param]["grads"].append(param) pserver_idx += 1 - if pserver_idx > len(pserver_endpoints): + if pserver_idx >= len(pserver_endpoints): pserver_idx = 0 return param_grad_map diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index ee7497e305..9bde9b03cc 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -50,7 +50,7 @@ class Executor(object): self.executor = core.Executor(act_places) self.places = places - def optimize(self, optimize_ops, program=None, **kwargs): + def optimize(self, optimize_ops, params_grads, program=None, **kwargs): """ optimize the program for different runtime environment @@ -67,7 +67,8 @@ class Executor(object): program = default_main_program() if kwargs.has_key("pservers"): - return self._optimize_distributed(optimize_ops, program, **kwargs) + return self._optimize_distributed(optimize_ops, program, + params_grads, **kwargs) def _optimize_distributed(self, optimize_ops, program, params_and_grads, **kwargs): @@ -92,7 +93,7 @@ class Executor(object): type="send", inputs={"X": self.param_grad_map[ep]["params"] }, # inputs is a list of tensors to be send - outputs={"Out": self.param_grad_map[ep]["params"]}, + outputs={}, attrs={"endpoint": ep}) # -------------- generate optimize sub program -------------- self.optimize_sub_program = Program() diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 99fe94942b..18d414c579 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -304,7 +304,8 @@ class Operator(object): self.desc.check_attrs() no_kernel_op_set = { 'feed', 'fetch', 'save', 'load', 'recurrent', - 'rnn_memory_helper_grad', 'conditional_block', 'while' + 'rnn_memory_helper_grad', 'conditional_block', 'while', 'send', + 'recv' } if type not in no_kernel_op_set: self.desc.infer_var_type(self.block.desc) diff --git a/python/paddle/v2/fluid/optimizer.py b/python/paddle/v2/fluid/optimizer.py index 719e3b2563..9734f2bc0f 100644 --- a/python/paddle/v2/fluid/optimizer.py +++ b/python/paddle/v2/fluid/optimizer.py @@ -202,7 +202,7 @@ class Optimizer(object): params_grads = append_regularization_ops(params_grads) optimize_ops = self.create_optimization_pass(params_grads, loss, startup_program) - return optimize_ops + return optimize_ops, params_grads class SGDOptimizer(Optimizer): diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py index b856526114..737bd9ac52 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py @@ -2,6 +2,7 @@ from __future__ import print_function import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid +import os images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') @@ -24,7 +25,7 @@ predict = fluid.layers.fc(input=conv_pool_2, size=10, act="softmax") cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) optimizer = fluid.optimizer.Adam(learning_rate=0.01) -optimizer.minimize(avg_cost) +optimize_ops, params_grads = optimizer.minimize(avg_cost) accuracy = fluid.evaluator.Accuracy(input=predict, label=label) @@ -38,10 +39,10 @@ train_reader = paddle.batch( place = fluid.CPUPlace() exe = fluid.Executor(place) -exe.optimize(pservers="127.0.0.1:6174", trainers=1) +exe.optimize(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) pserver_endpoint = os.getenv("PSERVER") -if is_pserver: +if pserver_endpoint: pserver_prog = exe.get_pserver_program(pserver_endpoint) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) From 489b9695e4fb569b984886c424ab320227b2d736 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 11 Dec 2017 21:05:28 +0800 Subject: [PATCH 10/84] wip for testing --- paddle/operators/detail/recv_impl.cc | 16 ++++--- paddle/operators/detail/send_recv.proto | 1 + paddle/operators/detail/send_recv_impl.h | 16 +++---- paddle/operators/recv_op.cc | 47 +++++++++++++++---- python/paddle/v2/fluid/executor.py | 31 ++++++++---- .../book/test_recognize_digits_conv_dist.py | 3 +- 6 files changed, 81 insertions(+), 33 deletions(-) diff --git a/paddle/operators/detail/recv_impl.cc b/paddle/operators/detail/recv_impl.cc index 89dc504522..dab3d1e14c 100644 --- a/paddle/operators/detail/recv_impl.cc +++ b/paddle/operators/detail/recv_impl.cc @@ -21,16 +21,20 @@ namespace detail { Status SendRecvServerImpl::SendVariable(ServerContext *context, const VariableMessage *in_var, VariableMessage *out_var) { - framework::LoDTensor t; - // TODO(typhoonzero): desirealize in_tensor and run pserver network. + // TODO(typhoonzero): support different variable types. std::istringstream iss(in_var->serialized()); + framework::LoDTensor t; framework::DeserializeFromStream(iss, &t); - lodtensor_queue_.Push(std::move(t)); + TensorWithName tensor_with_name = + std::make_pair(in_var->varname(), std::move(t)); + + var_recv_queue_.Push(std::move(tensor_with_name)); // Block util the sub graph is done. - t = lodtensor_return_queue_.Pop(); + auto out_tensor_with_name = var_return_queue_.Pop(); std::ostringstream oss; - // FIXME(typhoonzero): get context from op. - framework::SerializeToStream(oss, t, platform::CPUDeviceContext()); + framework::SerializeToStream(oss, out_tensor_with_name.second, + platform::CPUDeviceContext()); + std::string *varname = out_var->mutable_varname(); *varname = in_var->varname(); std::string *serialized = out_var->mutable_serialized(); diff --git a/paddle/operators/detail/send_recv.proto b/paddle/operators/detail/send_recv.proto index 07ff9d2c62..9b4058fd61 100644 --- a/paddle/operators/detail/send_recv.proto +++ b/paddle/operators/detail/send_recv.proto @@ -19,6 +19,7 @@ package sendrecv; service SendRecvService { // For parameter server round-robin like hashing, do not split tensors. // Send and recv only one tensor + // TODO(typhoonzero): add streaming API rpc SendVariable(VariableMessage) returns (VariableMessage) {} } diff --git a/paddle/operators/detail/send_recv_impl.h b/paddle/operators/detail/send_recv_impl.h index b9a5340a86..b6b9919c60 100644 --- a/paddle/operators/detail/send_recv_impl.h +++ b/paddle/operators/detail/send_recv_impl.h @@ -48,6 +48,8 @@ namespace paddle { namespace operators { namespace detail { +typedef std::pair TensorWithName; + class SendRecvServerImpl final : public SendRecvService::Service { public: explicit SendRecvServerImpl() {} @@ -55,17 +57,15 @@ class SendRecvServerImpl final : public SendRecvService::Service { Status SendVariable(ServerContext *context, const VariableMessage *in_var, VariableMessage *out_var) override; - const framework::LoDTensor Get() { return this->lodtensor_queue_.Pop(); } + const TensorWithName Get() { return this->var_recv_queue_.Pop(); } - void Push(const framework::LoDTensor &tensor) { - this->lodtensor_return_queue_.Push(tensor); - } + void Push(const TensorWithName &var) { this->var_return_queue_.Push(var); } private: - SimpleBlockQueue lodtensor_queue_; - SimpleBlockQueue lodtensor_return_queue_; - SimpleBlockQueue selected_rows_queue_; - SimpleBlockQueue selected_rows_return_queue_; + // received variable from RPC, operators fetch variable from this queue. + SimpleBlockQueue var_recv_queue_; + // calculated variable should push to this queue. + SimpleBlockQueue var_return_queue_; }; // RPCClient is a class to send tensors to pserver sub-network diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index eed482c1b4..b593c6e4f3 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -63,14 +64,32 @@ class RecvOp : public framework::OperatorBase { void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { - // blocking get one var from client. - const framework::LoDTensor &t = rpc_service_->Get(); framework::Scope &recv_scope = scope.NewScope(); + // blocking get one var from client. + const detail::TensorWithName &v = rpc_service_->Get(); + auto grad_var_name = v.first; + + // framework::Scope &recv_scope = scope.NewScope(); + auto param_list = Attr>("ParamList"); + auto grad_list = Attr>("GradList"); + auto it = std::find(grad_list.begin(), grad_list.end(), grad_var_name); + std::string param_var_name; + if (it != grad_list.end()) { + param_var_name = param_list[it - grad_list.begin()]; + } // set graph input var - auto *var = recv_scope.Var(Input("RX")); + auto input_grad = Input("RX"); + + // FIXME(typhoonzero): Find the parameter name from input grad name + // rename X -> Param + // rename RX -> Grad + auto *var = recv_scope.FindVar(input_grad); auto *tensor = var->GetMutable(); + recv_scope.Rename(param_var_name, "Param"); + recv_scope.Rename("RX", "Grad"); + // FIXME(typhoonzero): do not copy - framework::CopyFrom(t, dev_ctx.GetPlace(), dev_ctx, tensor); + framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); std::string program_str = Attr("OptimizeProgram"); framework::ProgramDesc program_desc; @@ -81,9 +100,14 @@ class RecvOp : public framework::OperatorBase { executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/); - auto *out_var = recv_scope.FindVar("Out"); - // push back - rpc_service_->Push(out_var->Get()); + auto *out_var = recv_scope.FindVar("Param"); + detail::TensorWithName out; + out.first = param_var_name; + out.second = out_var->Get(); + rpc_service_->Push(out); + // rename back the params + recv_scope.Rename("Param", param_var_name); + recv_scope.Rename("Grad", "RX"); } protected: @@ -93,13 +117,14 @@ class RecvOp : public framework::OperatorBase { // grpc send/recv service implement to register. std::shared_ptr rpc_service_; std::shared_ptr server_thread_; + framework::Scope const *recv_scope_{nullptr}; }; class RecvOpMaker : public framework::OpProtoAndCheckerMaker { public: RecvOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("RX", "(Tensor) Input tensor to be saved"); + AddInput("RX", "(Tensor) Input tensor to be optimized").AsDuplicable(); AddComment(R"DOC( Recv operator @@ -112,6 +137,12 @@ This operator will recv tensor from send_op .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); AddAttr("OptimizeProgram", "type string", "Serialized ProgramDesc string for recv to run."); + AddAttr>( + "ParamList", "type list of string", + "grad->param name mapping to find which param to optimize."); + AddAttr>( + "GradList", "type list of string", + "grad->param name mapping to find which param to optimize."); } }; diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index 9bde9b03cc..b6cfec3983 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -1,6 +1,6 @@ import numpy as np from . import core -from framework import Program, default_main_program +from framework import Program, default_main_program, Parameter, Variable import distribute_planner __all__ = ['Executor', 'g_scope'] @@ -91,7 +91,7 @@ class Executor(object): # FIXME(typhoonzero): send to different servers can run in parrallel. send_op = program.global_block().append_op( type="send", - inputs={"X": self.param_grad_map[ep]["params"] + inputs={"X": self.param_grad_map[ep]["grads"] }, # inputs is a list of tensors to be send outputs={}, attrs={"endpoint": ep}) @@ -102,9 +102,20 @@ class Executor(object): def get_pserver_program(self, endpoint): pserver_program = Program() - - for param in self.param_grad_map[endpoint]["params"]: - pserver_program.global_block().create_parameter(**param.__dict__) + for v in self.param_grad_map[endpoint]["params"]: + assert isinstance(v, Parameter) + new_p = Parameter( + block=pserver_program.global_block(), + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=v.stop_gradient, + trainable=v.trainable, + optimize_attr=v.optimize_attr, + regularizer=v.regularizer, + name=v.name) + pserver_program.global_block().vars[new_p.name] = new_p pserver_program.global_block().append_op( type="recv", @@ -112,12 +123,12 @@ class Executor(object): self.param_grad_map[endpoint]["grads"]}, # grads to recv outputs={}, attrs={ - "OptimizeProgram": self.optimize_sub_program.to_string(), - "endpoint": endpoint + "OptimizeProgram": self.optimize_sub_program.to_string(True), + "endpoint": endpoint, + "ParamList": self.param_grad_map[endpoint]["params"], + "GradList": self.param_grad_map[endpoint]["grads"] }) - - def get_trainer_program(self): - return default_main_program() + return pserver_program def aslodtensor(self, data): def accumulate(data): diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py index 737bd9ac52..1add8e4020 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py @@ -45,7 +45,8 @@ pserver_endpoint = os.getenv("PSERVER") if pserver_endpoint: pserver_prog = exe.get_pserver_program(pserver_endpoint) exe.run(fluid.default_startup_program()) - exe.run(pserver_prog) + while True: + exe.run(pserver_prog) else: feeder = fluid.DataFeeder(feed_list=[images, label], place=place) exe.run(fluid.default_startup_program()) From b4cd7f3d758e4a1f9104861dfd910afdbbbb66fe Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Tue, 12 Dec 2017 21:07:53 +0800 Subject: [PATCH 11/84] wip need ut --- paddle/operators/detail/send_impl.cc | 1 + paddle/operators/recv_op.cc | 26 ++++--- paddle/operators/send_op.cc | 1 + paddle/pybind/protobuf.cc | 6 ++ python/paddle/v2/fluid/distribute_planner.py | 8 +-- python/paddle/v2/fluid/executor.py | 72 +++++++++++++------ python/paddle/v2/fluid/framework.py | 8 +++ .../book/test_recognize_digits_conv_dist.py | 3 +- 8 files changed, 87 insertions(+), 38 deletions(-) diff --git a/paddle/operators/detail/send_impl.cc b/paddle/operators/detail/send_impl.cc index da1ddf75d2..2313255dcb 100644 --- a/paddle/operators/detail/send_impl.cc +++ b/paddle/operators/detail/send_impl.cc @@ -37,6 +37,7 @@ bool RPCClient::SendVariable(const framework::Scope& scope, msg.set_serialized(oss.str()); Status status = stub_->SendVariable(&context, msg, &out_msg); if (!status.ok()) { + LOG(ERROR) << "gRPC error: " << status.error_message(); return false; } std::istringstream iss(out_msg.serialized()); diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index b593c6e4f3..94cb39391f 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -64,12 +64,12 @@ class RecvOp : public framework::OperatorBase { void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { + // FIXME(typhoonzero): no new scopes for every run. framework::Scope &recv_scope = scope.NewScope(); // blocking get one var from client. const detail::TensorWithName &v = rpc_service_->Get(); auto grad_var_name = v.first; - // framework::Scope &recv_scope = scope.NewScope(); auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); auto it = std::find(grad_list.begin(), grad_list.end(), grad_var_name); @@ -77,16 +77,23 @@ class RecvOp : public framework::OperatorBase { if (it != grad_list.end()) { param_var_name = param_list[it - grad_list.begin()]; } - // set graph input var - auto input_grad = Input("RX"); + // find input by "grad_var_name" + // auto inputs = Inputs("RX"); // FIXME(typhoonzero): Find the parameter name from input grad name // rename X -> Param // rename RX -> Grad - auto *var = recv_scope.FindVar(input_grad); + + LOG(ERROR) << "recved grad: " << grad_var_name + << " param: " << param_var_name; + auto *var = recv_scope.Var(grad_var_name); auto *tensor = var->GetMutable(); - recv_scope.Rename(param_var_name, "Param"); - recv_scope.Rename("RX", "Grad"); + + // Param is in parent scope, put it in current scope. + auto *param_var = recv_scope.FindVar(param_var_name); + auto param_scope = recv_scope.FindScope(param_var); + param_scope->Rename(param_var_name, "Param"); + recv_scope.Rename(grad_var_name, "Grad"); // FIXME(typhoonzero): do not copy framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); @@ -100,14 +107,14 @@ class RecvOp : public framework::OperatorBase { executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/); - auto *out_var = recv_scope.FindVar("Param"); + auto *out_var = recv_scope.FindVar("ParamOut"); detail::TensorWithName out; out.first = param_var_name; out.second = out_var->Get(); rpc_service_->Push(out); // rename back the params - recv_scope.Rename("Param", param_var_name); - recv_scope.Rename("Grad", "RX"); + param_scope.Rename("Param", param_var_name); + recv_scope.Rename("Grad", grad_var_name); } protected: @@ -117,7 +124,6 @@ class RecvOp : public framework::OperatorBase { // grpc send/recv service implement to register. std::shared_ptr rpc_service_; std::shared_ptr server_thread_; - framework::Scope const *recv_scope_{nullptr}; }; class RecvOpMaker : public framework::OpProtoAndCheckerMaker { diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index 7cbc45e69a..648905743c 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -47,6 +47,7 @@ class SendOp : public framework::OperatorBase { // TODO(typhoonzero): currently it's non-blocking, // should block until server responds. for (auto in : ins) { + LOG(ERROR) << "sending grad: " << in; bool ret = client_->SendVariable(scope, in, in); if (!ret) { LOG(ERROR) << "send variable error"; diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 6c8f06cccb..6e6cafafb9 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -250,6 +250,12 @@ void BindOpDesc(py::module &m) { .def("set_attr", &OpDescBind::SetAttr) .def("attr", &OpDescBind::GetAttr) .def("set_block_attr", &OpDescBind::SetBlockAttr) + .def("set_serialized_attr", + [](OpDescBind &self, const std::string &name, + const py::bytes &seriralized) { + std::string ser(seriralized); + self.SetAttr(name, ser); + }) .def("block_attr", &OpDescBind::GetBlockAttr) .def("check_attrs", &OpDescBind::CheckAttrs) .def("infer_shape", &OpDescBind::InferShape) diff --git a/python/paddle/v2/fluid/distribute_planner.py b/python/paddle/v2/fluid/distribute_planner.py index 3d8df4b3c8..c3430b3b68 100644 --- a/python/paddle/v2/fluid/distribute_planner.py +++ b/python/paddle/v2/fluid/distribute_planner.py @@ -29,19 +29,19 @@ def hash_name_to_server(params_grads, pserver_endpoints): return param_grad_map -def round_robin(parameters, pserver_endpoints): - assert (len(parameters) > len(pserver_endpoints)) +def round_robin(params_grads, pserver_endpoints): + assert (len(params_grads) > len(pserver_endpoints)) param_grad_map = dict() pserver_idx = 0 - for param in parameters: + for param, grad in params_grads: if param.trainable is True: server_for_param = pserver_endpoints[pserver_idx] if not param_grad_map.has_key(server_for_param): param_grad_map[server_for_param] = {"params": [], "grads": []} param_grad_map[server_for_param]["params"].append(param) - param_grad_map[server_for_param]["grads"].append(param) + param_grad_map[server_for_param]["grads"].append(grad) pserver_idx += 1 if pserver_idx >= len(pserver_endpoints): diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index b6cfec3983..ba699442ce 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -70,6 +70,31 @@ class Executor(object): return self._optimize_distributed(optimize_ops, program, params_grads, **kwargs) + def _clone_param(self, block, v): + assert isinstance(v, Parameter) + new_p = Parameter( + block=block, + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=v.stop_gradient, + trainable=v.trainable, + optimize_attr=v.optimize_attr, + regularizer=v.regularizer, + name=v.name) + block.vars[new_p.name] = new_p + + def _clone_var(self, block, var): + assert isinstance(var, Variable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True) + def _optimize_distributed(self, optimize_ops, program, params_and_grads, **kwargs): # remove optimize ops and add a send op to main_program @@ -84,8 +109,7 @@ class Executor(object): assert (callable(split_method)) pserver_endpoints = kwargs["pservers"].split(",") - params = program.global_block().all_parameters() - self.param_grad_map = split_method(params, pserver_endpoints) + self.param_grad_map = split_method(params_and_grads, pserver_endpoints) for ep in pserver_endpoints: # FIXME(typhoonzero): send to different servers can run in parrallel. @@ -95,27 +119,26 @@ class Executor(object): }, # inputs is a list of tensors to be send outputs={}, attrs={"endpoint": ep}) - # -------------- generate optimize sub program -------------- - self.optimize_sub_program = Program() - for opt_op in optimize_ops: - self.optimize_sub_program.global_block().ops.append(opt_op) - def get_pserver_program(self, endpoint): + def get_pserver_program(self, endpoint, optimize_ops): pserver_program = Program() for v in self.param_grad_map[endpoint]["params"]: - assert isinstance(v, Parameter) - new_p = Parameter( - block=pserver_program.global_block(), - shape=v.shape, - dtype=v.dtype, - type=v.type, - lod_level=v.lod_level, - stop_gradient=v.stop_gradient, - trainable=v.trainable, - optimize_attr=v.optimize_attr, - regularizer=v.regularizer, - name=v.name) - pserver_program.global_block().vars[new_p.name] = new_p + self._clone_param(pserver_program.global_block(), v) + + optimize_sub_program = Program() + for opt_op in optimize_ops: + for varname, var in opt_op.inputs.iteritems(): + optimize_sub_program.global_block().create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=var.shape) + optimize_sub_program.global_block().append_op( + type=opt_op.type, + inputs=opt_op.inputs, + outputs=opt_op.outputs, + attrs=opt_op.attrs) + print("optimize program: ", optimize_sub_program) pserver_program.global_block().append_op( type="recv", @@ -123,11 +146,14 @@ class Executor(object): self.param_grad_map[endpoint]["grads"]}, # grads to recv outputs={}, attrs={ - "OptimizeProgram": self.optimize_sub_program.to_string(True), + "OptimizeProgram": optimize_sub_program.desc, "endpoint": endpoint, - "ParamList": self.param_grad_map[endpoint]["params"], - "GradList": self.param_grad_map[endpoint]["grads"] + "ParamList": + [p.name for p in self.param_grad_map[endpoint]["params"]], + "GradList": + [p.name for p in self.param_grad_map[endpoint]["grads"]] }) + pserver_program.sync_with_cpp() return pserver_program def aslodtensor(self, data): diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 18d414c579..274565b28f 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -227,6 +227,10 @@ class Operator(object): attrs=None): self.block = block self.desc = desc + # for clone a new operator + self.inputs = inputs + self.outputs = outputs + self.attrs = attrs if len(self.desc.type()) != 0: return if type is None: @@ -298,6 +302,10 @@ class Operator(object): continue if isinstance(attrs[attr_name], Block): self.desc.set_block_attr(attr_name, attrs[attr_name].desc) + elif isinstance(attrs[attr_name], core.BlockDesc) or \ + isinstance(attrs[attr_name], core.ProgramDesc): + self.desc.set_serialized_attr( + attr_name, attrs[attr_name].serialize_to_string()) else: self.desc.set_attr(attr_name, attrs[attr_name]) diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py index 1add8e4020..208002c8d6 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py @@ -43,10 +43,11 @@ exe.optimize(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) pserver_endpoint = os.getenv("PSERVER") if pserver_endpoint: - pserver_prog = exe.get_pserver_program(pserver_endpoint) + pserver_prog = exe.get_pserver_program(pserver_endpoint, optimize_ops) exe.run(fluid.default_startup_program()) while True: exe.run(pserver_prog) + print("Run pserver once end...") else: feeder = fluid.DataFeeder(feed_list=[images, label], place=place) exe.run(fluid.default_startup_program()) From 9508c72685e1eab32eb672496ba8974e8e3e0927 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 13 Dec 2017 15:35:38 +0800 Subject: [PATCH 12/84] wip: should fix variable recreate --- paddle/framework/executor.cc | 50 +++++++------- paddle/framework/executor.h | 3 +- paddle/operators/detail/recv_impl.cc | 11 ++- paddle/operators/detail/send_impl.cc | 23 +++++-- paddle/operators/detail/send_recv.proto | 4 +- paddle/operators/detail/send_recv_impl.h | 8 ++- paddle/operators/recv_op.cc | 69 ++++++++----------- paddle/operators/send_op.cc | 9 ++- python/paddle/v2/fluid/executor.py | 3 +- .../book/test_recognize_digits_conv_dist.py | 1 + 10 files changed, 103 insertions(+), 78 deletions(-) diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index 83aa927c29..cc3916e7bb 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -85,7 +85,7 @@ static void CreateTensor(Variable* var, VarDesc::VarType var_type) { } void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id, - bool create_local_scope) { + bool create_local_scope, bool create_vars) { // TODO(tonyyang-svail): // - only runs on the first device (i.e. no interdevice communication) // - will change to use multiple blocks for RNN op and Cond Op @@ -94,33 +94,35 @@ void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id, auto& device = device_contexts_[0]; Scope* local_scope = scope; - if (create_local_scope) { - local_scope = &scope->NewScope(); - for (auto& var : block.AllVars()) { - if (var->Name() == framework::kEmptyVarName) { - continue; + if (create_vars) { + if (create_local_scope) { + local_scope = &scope->NewScope(); + for (auto& var : block.AllVars()) { + if (var->Name() == framework::kEmptyVarName) { + continue; + } + + if (var->Persistable()) { + auto* ptr = scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create Variable " << var->Name() + << " global, which pointer is " << ptr; + } else { + auto* ptr = local_scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create Variable " << var->Name() + << " locally, which pointer is " << ptr; + } } - - if (var->Persistable()) { - auto* ptr = scope->Var(var->Name()); - CreateTensor(ptr, var->GetType()); - VLOG(3) << "Create Variable " << var->Name() - << " global, which pointer is " << ptr; - } else { + } else { + for (auto& var : block.AllVars()) { auto* ptr = local_scope->Var(var->Name()); CreateTensor(ptr, var->GetType()); - VLOG(3) << "Create Variable " << var->Name() - << " locally, which pointer is " << ptr; + VLOG(3) << "Create variable " << var->Name() << ", which pointer is " + << ptr; } - } - } else { - for (auto& var : block.AllVars()) { - auto* ptr = local_scope->Var(var->Name()); - CreateTensor(ptr, var->GetType()); - VLOG(3) << "Create variable " << var->Name() << ", which pointer is " - << ptr; - } - } + } // if (create_local_scope) + } // if (create_vars) for (auto& op_desc : block.AllOps()) { auto op = paddle::framework::OpRegistry::CreateOp(*op_desc); diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h index b745f4f647..28da060830 100644 --- a/paddle/framework/executor.h +++ b/paddle/framework/executor.h @@ -35,7 +35,8 @@ class Executor { * ProgramDesc * Scope */ - void Run(const ProgramDescBind&, Scope*, int, bool create_local_scope = true); + void Run(const ProgramDescBind&, Scope*, int, bool create_local_scope = true, + bool create_vars = true); private: std::vector device_contexts_; diff --git a/paddle/operators/detail/recv_impl.cc b/paddle/operators/detail/recv_impl.cc index dab3d1e14c..bc930cbb00 100644 --- a/paddle/operators/detail/recv_impl.cc +++ b/paddle/operators/detail/recv_impl.cc @@ -20,7 +20,7 @@ namespace detail { Status SendRecvServerImpl::SendVariable(ServerContext *context, const VariableMessage *in_var, - VariableMessage *out_var) { + VoidMessage *out_var) { // TODO(typhoonzero): support different variable types. std::istringstream iss(in_var->serialized()); framework::LoDTensor t; @@ -29,6 +29,12 @@ Status SendRecvServerImpl::SendVariable(ServerContext *context, std::make_pair(in_var->varname(), std::move(t)); var_recv_queue_.Push(std::move(tensor_with_name)); + return Status::OK; +} + +Status SendRecvServerImpl::GetVariable(ServerContext *context, + const VoidMessage *in_var, + VariableMessage *out_var) { // Block util the sub graph is done. auto out_tensor_with_name = var_return_queue_.Pop(); std::ostringstream oss; @@ -36,10 +42,9 @@ Status SendRecvServerImpl::SendVariable(ServerContext *context, platform::CPUDeviceContext()); std::string *varname = out_var->mutable_varname(); - *varname = in_var->varname(); + *varname = out_tensor_with_name.first; std::string *serialized = out_var->mutable_serialized(); *serialized = oss.str(); - return Status::OK; } diff --git a/paddle/operators/detail/send_impl.cc b/paddle/operators/detail/send_impl.cc index 2313255dcb..bf22d3df81 100644 --- a/paddle/operators/detail/send_impl.cc +++ b/paddle/operators/detail/send_impl.cc @@ -19,10 +19,10 @@ namespace operators { namespace detail { bool RPCClient::SendVariable(const framework::Scope& scope, - const std::string& inname, - const std::string& outname) { + const std::string& inname) { ClientContext context; - VariableMessage msg, out_msg; + VariableMessage msg; + VoidMessage out_msg; // FIXME(typhoonzero): pass device context to here. auto ctx = platform::CPUDeviceContext(); auto* var = scope.FindVar(inname); @@ -40,7 +40,22 @@ bool RPCClient::SendVariable(const framework::Scope& scope, LOG(ERROR) << "gRPC error: " << status.error_message(); return false; } - std::istringstream iss(out_msg.serialized()); + return true; +} + +bool RPCClient::GetVariable(const framework::Scope& scope) { + ClientContext context; + VariableMessage msg; + VoidMessage void_msg; + auto ctx = platform::CPUDeviceContext(); + Status status = stub_->GetVariable(&context, void_msg, &msg); + if (!status.ok()) { + LOG(ERROR) << "gRPC error: " << status.error_message(); + return false; + } + + std::istringstream iss(msg.serialized()); + auto outname = msg.varname(); framework::LoDTensor ret_tensor; framework::DeserializeFromStream(iss, &ret_tensor); auto* outvar = scope.FindVar(outname); diff --git a/paddle/operators/detail/send_recv.proto b/paddle/operators/detail/send_recv.proto index 9b4058fd61..d00c33fe42 100644 --- a/paddle/operators/detail/send_recv.proto +++ b/paddle/operators/detail/send_recv.proto @@ -20,7 +20,9 @@ service SendRecvService { // For parameter server round-robin like hashing, do not split tensors. // Send and recv only one tensor // TODO(typhoonzero): add streaming API - rpc SendVariable(VariableMessage) returns (VariableMessage) {} + rpc SendVariable(VariableMessage) returns (VoidMessage) {} + // Argument VariableMessage for GetVariable should only contain varname. + rpc GetVariable(VoidMessage) returns (VariableMessage) {} } // VariableMessage is serialized paddle variable message. diff --git a/paddle/operators/detail/send_recv_impl.h b/paddle/operators/detail/send_recv_impl.h index b6b9919c60..df01345e34 100644 --- a/paddle/operators/detail/send_recv_impl.h +++ b/paddle/operators/detail/send_recv_impl.h @@ -55,7 +55,9 @@ class SendRecvServerImpl final : public SendRecvService::Service { explicit SendRecvServerImpl() {} Status SendVariable(ServerContext *context, const VariableMessage *in_var, - VariableMessage *out_var) override; + VoidMessage *out_var) override; + Status GetVariable(ServerContext *context, const VoidMessage *in_var, + VariableMessage *out_var) override; const TensorWithName Get() { return this->var_recv_queue_.Pop(); } @@ -75,8 +77,8 @@ class RPCClient { RPCClient(std::shared_ptr channel) : stub_(SendRecvService::NewStub(channel)) {} - bool SendVariable(const framework::Scope &scope, const std::string &inname, - const std::string &outname); + bool SendVariable(const framework::Scope &scope, const std::string &inname); + bool GetVariable(const framework::Scope &scope); private: std::unique_ptr stub_; diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 94cb39391f..754338ec6b 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -66,37 +66,25 @@ class RecvOp : public framework::OperatorBase { const platform::DeviceContext &dev_ctx) const override { // FIXME(typhoonzero): no new scopes for every run. framework::Scope &recv_scope = scope.NewScope(); - // blocking get one var from client. - const detail::TensorWithName &v = rpc_service_->Get(); - auto grad_var_name = v.first; - auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); - auto it = std::find(grad_list.begin(), grad_list.end(), grad_var_name); - std::string param_var_name; - if (it != grad_list.end()) { - param_var_name = param_list[it - grad_list.begin()]; + size_t param_count = param_list.size(); + for (size_t i = 0; i < param_count; ++i) { + // blocking get one var from client. + const detail::TensorWithName &v = rpc_service_->Get(); + auto grad_var_name = v.first; + auto it = std::find(grad_list.begin(), grad_list.end(), grad_var_name); + std::string param_var_name; + if (it != grad_list.end()) { + param_var_name = param_list[it - grad_list.begin()]; + } + VLOG(10) << "recved grad: " << grad_var_name + << " updating param: " << param_var_name; + auto *var = recv_scope.Var(grad_var_name); + auto *tensor = var->GetMutable(); + // FIXME(typhoonzero): do not copy + framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); } - // find input by "grad_var_name" - // auto inputs = Inputs("RX"); - - // FIXME(typhoonzero): Find the parameter name from input grad name - // rename X -> Param - // rename RX -> Grad - - LOG(ERROR) << "recved grad: " << grad_var_name - << " param: " << param_var_name; - auto *var = recv_scope.Var(grad_var_name); - auto *tensor = var->GetMutable(); - - // Param is in parent scope, put it in current scope. - auto *param_var = recv_scope.FindVar(param_var_name); - auto param_scope = recv_scope.FindScope(param_var); - param_scope->Rename(param_var_name, "Param"); - recv_scope.Rename(grad_var_name, "Grad"); - - // FIXME(typhoonzero): do not copy - framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); std::string program_str = Attr("OptimizeProgram"); framework::ProgramDesc program_desc; @@ -104,17 +92,20 @@ class RecvOp : public framework::OperatorBase { framework::ProgramDescBind program(program_desc); framework::Executor executor(dev_ctx); // Run sub graph to get optimized tensor - executor.Run(program, &recv_scope, 0, /*global_block*/ - false /*create_local_scope*/); - - auto *out_var = recv_scope.FindVar("ParamOut"); - detail::TensorWithName out; - out.first = param_var_name; - out.second = out_var->Get(); - rpc_service_->Push(out); - // rename back the params - param_scope.Rename("Param", param_var_name); - recv_scope.Rename("Grad", grad_var_name); + try { + executor.Run(program, &recv_scope, 0, /*global_block*/ + false /*create_local_scope*/, false /*create_vars*/); + } catch (std::exception &e) { + LOG(ERROR) << "run sub program error " << e.what(); + } + + for (size_t i = 0; i < param_count; ++i) { + auto *out_var = recv_scope.FindVar(param_list[i]); + detail::TensorWithName out; + out.first = param_list[i]; + out.second = out_var->Get(); + rpc_service_->Push(out); + } } protected: diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index 648905743c..ab1ae5b31d 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -48,11 +48,18 @@ class SendOp : public framework::OperatorBase { // should block until server responds. for (auto in : ins) { LOG(ERROR) << "sending grad: " << in; - bool ret = client_->SendVariable(scope, in, in); + bool ret = client_->SendVariable(scope, in); if (!ret) { LOG(ERROR) << "send variable error"; } } + for (auto in : ins) { + LOG(ERROR) << "updating from server..."; + bool ret = client_->GetVariable(scope); + if (!ret) { + LOG(ERROR) << "GetVariable error"; + } + } } protected: diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index ba699442ce..c8c9a4ef36 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -138,7 +138,6 @@ class Executor(object): inputs=opt_op.inputs, outputs=opt_op.outputs, attrs=opt_op.attrs) - print("optimize program: ", optimize_sub_program) pserver_program.global_block().append_op( type="recv", @@ -248,7 +247,7 @@ class Executor(object): outputs={'Out': [fetch_var]}, attrs={'col': i}) - self.executor.run(program.desc, scope, 0, True) + self.executor.run(program.desc, scope, 0, True, True) outs = [ core.get_fetch_variable(scope, fetch_var_name, i) for i in xrange(len(fetch_list)) diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py index 208002c8d6..5178131ea7 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py @@ -44,6 +44,7 @@ exe.optimize(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) pserver_endpoint = os.getenv("PSERVER") if pserver_endpoint: pserver_prog = exe.get_pserver_program(pserver_endpoint, optimize_ops) + print("pserver startup: ", fluid.default_startup_program()) exe.run(fluid.default_startup_program()) while True: exe.run(pserver_prog) From 40d0fff2e55b795690ef93cb539e8c3a029b7b16 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 14 Dec 2017 12:24:25 +0800 Subject: [PATCH 13/84] single pserver workable version --- paddle/operators/recv_op.cc | 72 ++++++++++++++++-------------- python/paddle/v2/fluid/executor.py | 2 +- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 754338ec6b..a0c25a25eb 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -69,43 +69,47 @@ class RecvOp : public framework::OperatorBase { auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); size_t param_count = param_list.size(); - for (size_t i = 0; i < param_count; ++i) { - // blocking get one var from client. - const detail::TensorWithName &v = rpc_service_->Get(); - auto grad_var_name = v.first; - auto it = std::find(grad_list.begin(), grad_list.end(), grad_var_name); - std::string param_var_name; - if (it != grad_list.end()) { - param_var_name = param_list[it - grad_list.begin()]; + // TODO(typhoonzero): change this to a while_op for every cluster-batch. + while (true) { + // TODO(typhoonzero): get from multiple trainers. + for (size_t i = 0; i < param_count; ++i) { + // blocking get one var from client. + const detail::TensorWithName &v = rpc_service_->Get(); + auto grad_var_name = v.first; + auto it = std::find(grad_list.begin(), grad_list.end(), grad_var_name); + std::string param_var_name; + if (it != grad_list.end()) { + param_var_name = param_list[it - grad_list.begin()]; + } + VLOG(10) << "recved grad: " << grad_var_name + << " updating param: " << param_var_name; + auto *var = recv_scope.Var(grad_var_name); + auto *tensor = var->GetMutable(); + // FIXME(typhoonzero): do not copy + framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); } - VLOG(10) << "recved grad: " << grad_var_name - << " updating param: " << param_var_name; - auto *var = recv_scope.Var(grad_var_name); - auto *tensor = var->GetMutable(); - // FIXME(typhoonzero): do not copy - framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); - } - std::string program_str = Attr("OptimizeProgram"); - framework::ProgramDesc program_desc; - program_desc.ParseFromString(program_str); - framework::ProgramDescBind program(program_desc); - framework::Executor executor(dev_ctx); - // Run sub graph to get optimized tensor - try { - executor.Run(program, &recv_scope, 0, /*global_block*/ - false /*create_local_scope*/, false /*create_vars*/); - } catch (std::exception &e) { - LOG(ERROR) << "run sub program error " << e.what(); - } + std::string program_str = Attr("OptimizeProgram"); + framework::ProgramDesc program_desc; + program_desc.ParseFromString(program_str); + framework::ProgramDescBind program(program_desc); + framework::Executor executor(dev_ctx); + // Run sub graph to get optimized tensor + try { + executor.Run(program, &recv_scope, 0, /*global_block*/ + false /*create_local_scope*/, false /*create_vars*/); + } catch (std::exception &e) { + LOG(ERROR) << "run sub program error " << e.what(); + } - for (size_t i = 0; i < param_count; ++i) { - auto *out_var = recv_scope.FindVar(param_list[i]); - detail::TensorWithName out; - out.first = param_list[i]; - out.second = out_var->Get(); - rpc_service_->Push(out); - } + for (size_t i = 0; i < param_count; ++i) { + auto *out_var = recv_scope.FindVar(param_list[i]); + detail::TensorWithName out; + out.first = param_list[i]; + out.second = out_var->Get(); + rpc_service_->Push(out); + } + } // while(true) } protected: diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index c8c9a4ef36..4d245250e8 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -93,7 +93,7 @@ class Executor(object): dtype=var.dtype, type=var.type, lod_level=var.lod_level, - persistable=True) + persistable=var.persistable) def _optimize_distributed(self, optimize_ops, program, params_and_grads, **kwargs): From 1b20096a529bb6ce80d066fc0805c9dd8a8b9364 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 14 Dec 2017 20:25:20 +0800 Subject: [PATCH 14/84] done --- paddle/operators/recv_op.cc | 28 ++- paddle/operators/send_op.cc | 2 - python/paddle/v2/fluid/__init__.py | 3 +- python/paddle/v2/fluid/distribute_planner.py | 49 ----- .../paddle/v2/fluid/distribute_transpiler.py | 206 ++++++++++++++++++ python/paddle/v2/fluid/executor.py | 105 --------- ...y => notest_recognize_digits_conv_dist.py} | 13 +- 7 files changed, 238 insertions(+), 168 deletions(-) delete mode 100644 python/paddle/v2/fluid/distribute_planner.py create mode 100644 python/paddle/v2/fluid/distribute_transpiler.py rename python/paddle/v2/fluid/tests/book/{test_recognize_digits_conv_dist.py => notest_recognize_digits_conv_dist.py} (82%) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index a0c25a25eb..2ff6f42c94 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -62,17 +62,29 @@ class RecvOp : public framework::OperatorBase { server_thread_->join(); } + std::string GetGradVarNameForTrainer(const std::string &varname) const { + if (grads_counter_.find(varname) != grads_counter_.end()) { + grads_counter_[varname] = 0; + } + char ret[256]; + snprintf(ret, sizeof(ret), "%s.trainer_%d", varname.c_str(), + grads_counter_[varname]++); + return std::string(ret); + } + void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { // FIXME(typhoonzero): no new scopes for every run. framework::Scope &recv_scope = scope.NewScope(); auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); + auto trainer_count = Attr("Trainers"); size_t param_count = param_list.size(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. while (true) { - // TODO(typhoonzero): get from multiple trainers. - for (size_t i = 0; i < param_count; ++i) { + // Get from multiple trainers, we don't care about order in which + // the gradient arrives, just add suffix 0~n then average the gradient. + for (size_t i = 0; i < param_count * trainer_count; ++i) { // blocking get one var from client. const detail::TensorWithName &v = rpc_service_->Get(); auto grad_var_name = v.first; @@ -83,6 +95,14 @@ class RecvOp : public framework::OperatorBase { } VLOG(10) << "recved grad: " << grad_var_name << " updating param: " << param_var_name; + if (trainer_count > 1) { + auto *var = recv_scope.FindVar(grad_var_name); + if (var != nullptr) { + // must rename the var to different names to merge gradient. + grad_var_name = this->GetGradVarNameForTrainer(grad_var_name); + } + } + auto *var = recv_scope.Var(grad_var_name); auto *tensor = var->GetMutable(); // FIXME(typhoonzero): do not copy @@ -119,6 +139,7 @@ class RecvOp : public framework::OperatorBase { // grpc send/recv service implement to register. std::shared_ptr rpc_service_; std::shared_ptr server_thread_; + mutable std::unordered_map grads_counter_; }; class RecvOpMaker : public framework::OpProtoAndCheckerMaker { @@ -144,6 +165,9 @@ This operator will recv tensor from send_op AddAttr>( "GradList", "type list of string", "grad->param name mapping to find which param to optimize."); + AddAttr("Trainers", "type int", + "Number of trainers in the current cluster job") + .SetDefault(1); } }; diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index ab1ae5b31d..3fcd2144f9 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -47,14 +47,12 @@ class SendOp : public framework::OperatorBase { // TODO(typhoonzero): currently it's non-blocking, // should block until server responds. for (auto in : ins) { - LOG(ERROR) << "sending grad: " << in; bool ret = client_->SendVariable(scope, in); if (!ret) { LOG(ERROR) << "send variable error"; } } for (auto in : ins) { - LOG(ERROR) << "updating from server..."; bool ret = client_->GetVariable(scope); if (!ret) { LOG(ERROR) << "GetVariable error"; diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index 59986c9f0c..a93f936361 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -16,12 +16,13 @@ import regularizer from param_attr import ParamAttr from data_feeder import DataFeeder from core import LoDTensor, CPUPlace, GPUPlace +from distribute_transpiler import DistributeTranspiler Tensor = LoDTensor __all__ = framework.__all__ + executor.__all__ + [ 'io', 'initializer', 'layers', 'nets', 'optimizer', 'backward', 'regularizer', 'LoDTensor', 'CPUPlace', 'GPUPlace', 'Tensor', 'ParamAttr' - 'DataFeeder' + 'DataFeeder', 'DistributeTranspiler' ] diff --git a/python/paddle/v2/fluid/distribute_planner.py b/python/paddle/v2/fluid/distribute_planner.py deleted file mode 100644 index c3430b3b68..0000000000 --- a/python/paddle/v2/fluid/distribute_planner.py +++ /dev/null @@ -1,49 +0,0 @@ -import framework -from backward import append_backward_ops -from regularizer import append_regularization_ops -import optimizer -from layer_helper import LayerHelper - - -def hash_name_to_server(params_grads, pserver_endpoints): - """ - :param param_grads: - :return: a map of pserver endpoint -> - params -> [param list] - grads -> [grad list] - """ - - def _hash_param(param_name, total): - return hash(param_name) % total - - param_grad_map = dict() - for param, grad in params_grads: - if param.trainable is True and grad is not None: - server_id = _hash_param(param.name, len(pserver_endpoints)) - server_for_param = pserver_endpoints[server_id] - if not param_grad_map.has_key(server_for_param): - param_grad_map[server_for_param] = {"params": [], "grads": []} - param_grad_map[server_for_param]["params"].append(param) - param_grad_map[server_for_param]["grads"].append(grad) - - return param_grad_map - - -def round_robin(params_grads, pserver_endpoints): - assert (len(params_grads) > len(pserver_endpoints)) - - param_grad_map = dict() - pserver_idx = 0 - for param, grad in params_grads: - if param.trainable is True: - server_for_param = pserver_endpoints[pserver_idx] - if not param_grad_map.has_key(server_for_param): - param_grad_map[server_for_param] = {"params": [], "grads": []} - - param_grad_map[server_for_param]["params"].append(param) - param_grad_map[server_for_param]["grads"].append(grad) - - pserver_idx += 1 - if pserver_idx >= len(pserver_endpoints): - pserver_idx = 0 - return param_grad_map diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py new file mode 100644 index 0000000000..739b47cd28 --- /dev/null +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -0,0 +1,206 @@ +import framework +from framework import Program, default_main_program, Parameter, Variable +import optimizer +from layer_helper import LayerHelper + + +def hash_name_to_server(params_grads, pserver_endpoints): + """ + :param param_grads: + :return: a map of pserver endpoint -> + params -> [param list] + grads -> [grad list] + """ + + def _hash_param(param_name, total): + return hash(param_name) % total + + param_grad_map = dict() + for param, grad in params_grads: + if param.trainable is True and grad is not None: + server_id = _hash_param(param.name, len(pserver_endpoints)) + server_for_param = pserver_endpoints[server_id] + if not param_grad_map.has_key(server_for_param): + param_grad_map[server_for_param] = {"params": [], "grads": []} + param_grad_map[server_for_param]["params"].append(param) + param_grad_map[server_for_param]["grads"].append(grad) + + return param_grad_map + + +def round_robin(params_grads, pserver_endpoints): + assert (len(params_grads) > len(pserver_endpoints)) + + param_grad_map = dict() + pserver_idx = 0 + for param, grad in params_grads: + if param.trainable is True: + server_for_param = pserver_endpoints[pserver_idx] + if not param_grad_map.has_key(server_for_param): + param_grad_map[server_for_param] = {"params": [], "grads": []} + + param_grad_map[server_for_param]["params"].append(param) + param_grad_map[server_for_param]["grads"].append(grad) + + pserver_idx += 1 + if pserver_idx >= len(pserver_endpoints): + pserver_idx = 0 + return param_grad_map + + +class DistributeTranspiler: + def transpile(self, + optimize_ops, + params_grads, + program=None, + pservers="127.0.0.1:6174", + trainers=1, + split_method=round_robin): + """ + Transpile the program to a distributed data-parallelism programs. + + The main_program will be transform to use a remote parameter server + to do parameter optimization. And the optimization graph will be put + in to a parameter server program. + + Use different methods to split trainable varialbles to different + parameter servers. + + :param optimize_ops: op list of optimization, should be the + return value of Optimizer.minimize + :type optimize_ops: list + :param program: program to optimize, default default_main_program + :param pservers: parameter server endpoints like "m1:6174,m2:6174" + :type pservers: string + + :return: return a list of programs + """ + if program is None: + program = default_main_program() + self.trainers = trainers + self._optimize_distributed( + optimize_ops, + program, + params_grads, + pservers=pservers, + trainers=trainers, + split_method=split_method) + + def _clone_param(self, block, v): + assert isinstance(v, Parameter) + new_p = Parameter( + block=block, + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=v.stop_gradient, + trainable=v.trainable, + optimize_attr=v.optimize_attr, + regularizer=v.regularizer, + name=v.name) + block.vars[new_p.name] = new_p + + def _clone_var(self, block, var): + assert isinstance(var, Variable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=var.persistable) + + def _optimize_distributed(self, optimize_ops, program, params_and_grads, + **kwargs): + # remove optimize ops and add a send op to main_program + # FIXME(typhoonzero): delete_op only remove the first accurance, + # need to consider about multiple same optimize op? + for op in optimize_ops: + program.global_block().delete_op(op) + if kwargs.has_key("split_method"): + split_method = kwargs["split_method"] + else: + split_method = round_robin + + assert (callable(split_method)) + pserver_endpoints = kwargs["pservers"].split(",") + self.param_grad_map = split_method(params_and_grads, pserver_endpoints) + + for ep in pserver_endpoints: + # FIXME(typhoonzero): send to different servers can run in parrallel. + send_op = program.global_block().append_op( + type="send", + inputs={"X": self.param_grad_map[ep]["grads"] + }, # inputs is a list of tensors to be send + outputs={}, + attrs={"endpoint": ep}) + + def _create_var_for_trainers(self, block, var, trainers): + var_list = [] + for i in xrange(trainers): + var_each = block.create_var( + name="%s.trainer_%d" % (var.name, i), + psersistable=var.persistable, + dtype=var.dtype, + shape=var.shape) + var_list.append(var_each) + return var_list + + def get_pserver_program(self, endpoint, optimize_ops): + pserver_program = Program() + for v in self.param_grad_map[endpoint]["params"]: + self._clone_param(pserver_program.global_block(), v) + + optimize_sub_program = Program() + grad_var_names = [ + var.name for var in self.param_grad_map[endpoint]["grads"] + ] + for opt_op in optimize_ops: + for _, var in opt_op.inputs.iteritems(): + # NOTE: append operators to merge gradients from multiple + # trainers. If trainers == 1, this is not needed. + if self.trainers > 1 and var.name in grad_var_names: + vars2merge = self._create_var_for_trainers( + optimize_sub_program.global_block(), var, self.trainers) + merged_var = optimize_sub_program.global_block().create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=var.shape) + optimize_sub_program.global_block().append_op( + type="sum", + inputs={"X": vars2merge}, + outputs={"Out": merged_var}) + optimize_sub_program.global_block().append_op( + type="scale", + inputs={"X": merged_var}, + outputs={"Out": merged_var}, + attrs={"scale": 1.0 / float(self.trainers)}) + else: + optimize_sub_program.global_block().create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=var.shape) + optimize_sub_program.global_block().append_op( + type=opt_op.type, + inputs=opt_op.inputs, + outputs=opt_op.outputs, + attrs=opt_op.attrs) + pserver_program.global_block().append_op( + type="recv", + inputs={"RX": + self.param_grad_map[endpoint]["grads"]}, # grads to recv + outputs={}, + attrs={ + "OptimizeProgram": optimize_sub_program.desc, + "endpoint": endpoint, + "ParamList": + [p.name for p in self.param_grad_map[endpoint]["params"]], + "GradList": + [p.name for p in self.param_grad_map[endpoint]["grads"]], + "Trainers": self.trainers + }) + pserver_program.sync_with_cpp() + return pserver_program diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index 4d245250e8..0d02422afd 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -50,111 +50,6 @@ class Executor(object): self.executor = core.Executor(act_places) self.places = places - def optimize(self, optimize_ops, params_grads, program=None, **kwargs): - """ - optimize the program for different runtime environment - - :param optimize_ops: op list of optimization, should be the - return value of Optimizer.minimize - :type optimize_ops: list - :param program: program to optimize, default default_main_program - :param pservers: parameter server endpoints like "m1:6174,m2:6174" - :type pservers: string - - :return: return a list of programs - """ - if program is None: - program = default_main_program() - - if kwargs.has_key("pservers"): - return self._optimize_distributed(optimize_ops, program, - params_grads, **kwargs) - - def _clone_param(self, block, v): - assert isinstance(v, Parameter) - new_p = Parameter( - block=block, - shape=v.shape, - dtype=v.dtype, - type=v.type, - lod_level=v.lod_level, - stop_gradient=v.stop_gradient, - trainable=v.trainable, - optimize_attr=v.optimize_attr, - regularizer=v.regularizer, - name=v.name) - block.vars[new_p.name] = new_p - - def _clone_var(self, block, var): - assert isinstance(var, Variable) - return block.create_var( - name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=var.persistable) - - def _optimize_distributed(self, optimize_ops, program, params_and_grads, - **kwargs): - # remove optimize ops and add a send op to main_program - # FIXME(typhoonzero): delete_op only remove the first accurence, - # need to consider about multiple same optimize op? - for op in optimize_ops: - program.global_block().delete_op(op) - if kwargs.has_key("split_method"): - split_method = kwargs["split_method"] - else: - split_method = distribute_planner.round_robin - - assert (callable(split_method)) - pserver_endpoints = kwargs["pservers"].split(",") - self.param_grad_map = split_method(params_and_grads, pserver_endpoints) - - for ep in pserver_endpoints: - # FIXME(typhoonzero): send to different servers can run in parrallel. - send_op = program.global_block().append_op( - type="send", - inputs={"X": self.param_grad_map[ep]["grads"] - }, # inputs is a list of tensors to be send - outputs={}, - attrs={"endpoint": ep}) - - def get_pserver_program(self, endpoint, optimize_ops): - pserver_program = Program() - for v in self.param_grad_map[endpoint]["params"]: - self._clone_param(pserver_program.global_block(), v) - - optimize_sub_program = Program() - for opt_op in optimize_ops: - for varname, var in opt_op.inputs.iteritems(): - optimize_sub_program.global_block().create_var( - name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=var.shape) - optimize_sub_program.global_block().append_op( - type=opt_op.type, - inputs=opt_op.inputs, - outputs=opt_op.outputs, - attrs=opt_op.attrs) - - pserver_program.global_block().append_op( - type="recv", - inputs={"RX": - self.param_grad_map[endpoint]["grads"]}, # grads to recv - outputs={}, - attrs={ - "OptimizeProgram": optimize_sub_program.desc, - "endpoint": endpoint, - "ParamList": - [p.name for p in self.param_grad_map[endpoint]["params"]], - "GradList": - [p.name for p in self.param_grad_map[endpoint]["grads"]] - }) - pserver_program.sync_with_cpp() - return pserver_program - def aslodtensor(self, data): def accumulate(data): if not isinstance(data, list): diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py similarity index 82% rename from python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py rename to python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py index 5178131ea7..c7f4f2212f 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py @@ -38,17 +38,14 @@ train_reader = paddle.batch( place = fluid.CPUPlace() exe = fluid.Executor(place) - -exe.optimize(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) +t = fluid.DistributeTranspiler() +t.transpile(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) pserver_endpoint = os.getenv("PSERVER") if pserver_endpoint: - pserver_prog = exe.get_pserver_program(pserver_endpoint, optimize_ops) - print("pserver startup: ", fluid.default_startup_program()) + pserver_prog = t.get_pserver_program(pserver_endpoint, optimize_ops) exe.run(fluid.default_startup_program()) - while True: - exe.run(pserver_prog) - print("Run pserver once end...") + exe.run(pserver_prog) else: feeder = fluid.DataFeeder(feed_list=[images, label], place=place) exe.run(fluid.default_startup_program()) @@ -60,8 +57,6 @@ else: feed=feeder.feed(data), fetch_list=[avg_cost] + accuracy.metrics) pass_acc = accuracy.eval(exe) - print("pass_id=" + str(pass_id) + " acc=" + str(acc) + " pass_acc=" - + str(pass_acc)) # print loss, acc if loss < 10.0 and pass_acc > 0.9: # if avg cost less than 10.0 and accuracy is larger than 0.9, we think our code is good. From dfbc9f2e6b7ab5834fce728df686063eb728d980 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 15 Dec 2017 09:32:49 +0800 Subject: [PATCH 15/84] fix ut --- python/paddle/v2/fluid/executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index 0d02422afd..525fded85a 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -1,7 +1,6 @@ import numpy as np from . import core from framework import Program, default_main_program, Parameter, Variable -import distribute_planner __all__ = ['Executor', 'g_scope'] From e13e15d8a4b97f00111e656c5bb4fb9833796470 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 15 Dec 2017 13:39:23 +0800 Subject: [PATCH 16/84] fix ci --- python/paddle/v2/fluid/tests/test_optimizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/tests/test_optimizer.py b/python/paddle/v2/fluid/tests/test_optimizer.py index 2459dfd664..29694be58b 100644 --- a/python/paddle/v2/fluid/tests/test_optimizer.py +++ b/python/paddle/v2/fluid/tests/test_optimizer.py @@ -27,7 +27,7 @@ class TestOptimizer(unittest.TestCase): block.append_op( type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out}) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) - opts = sgd_optimizer.minimize(mean_out, init_program) + opts, _ = sgd_optimizer.minimize(mean_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "sgd") @@ -57,7 +57,7 @@ class TestOptimizer(unittest.TestCase): learning_rate = 0.01 sgd_optimizer = optimizer.SGDOptimizer( learning_rate=learning_rate, global_step=global_step) - opts = sgd_optimizer.minimize(mean_out, init_program) + opts, _ = sgd_optimizer.minimize(mean_out, init_program) self.assertEqual(len(opts), 2) sgd_op = opts[0] self.assertEqual(sgd_op.type, "sgd") From f8f80db163da76e5d0b01da54b496ee1a7236773 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 15 Dec 2017 19:24:44 +0800 Subject: [PATCH 17/84] update for multi trainer --- paddle/operators/recv_op.cc | 8 ++------ .../paddle/v2/fluid/distribute_transpiler.py | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 2ff6f42c94..07e66492e1 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -63,7 +63,7 @@ class RecvOp : public framework::OperatorBase { } std::string GetGradVarNameForTrainer(const std::string &varname) const { - if (grads_counter_.find(varname) != grads_counter_.end()) { + if (grads_counter_.find(varname) == grads_counter_.end()) { grads_counter_[varname] = 0; } char ret[256]; @@ -96,11 +96,7 @@ class RecvOp : public framework::OperatorBase { VLOG(10) << "recved grad: " << grad_var_name << " updating param: " << param_var_name; if (trainer_count > 1) { - auto *var = recv_scope.FindVar(grad_var_name); - if (var != nullptr) { - // must rename the var to different names to merge gradient. - grad_var_name = this->GetGradVarNameForTrainer(grad_var_name); - } + grad_var_name = this->GetGradVarNameForTrainer(grad_var_name); } auto *var = recv_scope.Var(grad_var_name); diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 739b47cd28..4919dce20d 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -183,11 +183,20 @@ class DistributeTranspiler: persistable=var.persistable, dtype=var.dtype, shape=var.shape) - optimize_sub_program.global_block().append_op( - type=opt_op.type, - inputs=opt_op.inputs, - outputs=opt_op.outputs, - attrs=opt_op.attrs) + + if opt_op.inputs.has_key("Grad"): + if opt_op.inputs["Grad"].name in grad_var_names: + optimize_sub_program.global_block().append_op( + type=opt_op.type, + inputs=opt_op.inputs, + outputs=opt_op.outputs, + attrs=opt_op.attrs) + else: + optimize_sub_program.global_block().append_op( + type=opt_op.type, + inputs=opt_op.inputs, + outputs=opt_op.outputs, + attrs=opt_op.attrs) pserver_program.global_block().append_op( type="recv", inputs={"RX": From 17f9be55ad525270e2ae157392955d3269f24f9e Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 15 Dec 2017 19:54:42 +0800 Subject: [PATCH 18/84] update for multi trainer --- paddle/operators/recv_op.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 07e66492e1..731e5e4756 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -95,6 +95,12 @@ class RecvOp : public framework::OperatorBase { } VLOG(10) << "recved grad: " << grad_var_name << " updating param: " << param_var_name; + auto *merged_grad = recv_scope.FindVar(grad_var_name); + if (merged_grad == nullptr) { + // create output of merged var. + recv_scope.Var(grad_var_name); + } + if (trainer_count > 1) { grad_var_name = this->GetGradVarNameForTrainer(grad_var_name); } From e6079390a930b58f5726f7182a83fd2acf61326e Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 18 Dec 2017 15:28:27 +0800 Subject: [PATCH 19/84] add example doc in transpiler --- paddle/operators/recv_op.cc | 3 ++- .../paddle/v2/fluid/distribute_transpiler.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 731e5e4756..9c3e8953bb 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -98,7 +98,8 @@ class RecvOp : public framework::OperatorBase { auto *merged_grad = recv_scope.FindVar(grad_var_name); if (merged_grad == nullptr) { // create output of merged var. - recv_scope.Var(grad_var_name); + auto merged_var = recv_scope.Var(grad_var_name); + merged_var->GetMutable(); } if (trainer_count > 1) { diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 4919dce20d..13006bfd13 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -66,6 +66,24 @@ class DistributeTranspiler: Use different methods to split trainable varialbles to different parameter servers. + Example to run: + + exe = fluid.Executor(place) + t = fluid.DistributeTranspiler() + t.transpile(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) + + pserver_endpoint = os.getenv("PSERVER") + if pserver_endpoint: + pserver_prog = t.get_pserver_program(pserver_endpoint, optimize_ops) + exe.run(fluid.default_startup_program()) + exe.run(pserver_prog) + else: + feeder = fluid.DataFeeder(feed_list=[images, label], place=place) + exe.run(fluid.default_startup_program()) + + for pass_id in range(PASS_NUM): + ... + :param optimize_ops: op list of optimization, should be the return value of Optimizer.minimize :type optimize_ops: list From 1e549563d5b06e8ae7db1edfc34ff5dd1a72ac68 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 18 Dec 2017 16:42:37 +0800 Subject: [PATCH 20/84] multi trainers --- paddle/operators/detail/recv_impl.cc | 31 +++++++++++++++++++----- paddle/operators/detail/send_impl.cc | 13 +++++----- paddle/operators/detail/send_recv.proto | 4 ++- paddle/operators/detail/send_recv_impl.h | 22 +++++++++-------- paddle/operators/recv_op.cc | 16 ++++++------ 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/paddle/operators/detail/recv_impl.cc b/paddle/operators/detail/recv_impl.cc index bc930cbb00..47decb6d7e 100644 --- a/paddle/operators/detail/recv_impl.cc +++ b/paddle/operators/detail/recv_impl.cc @@ -33,21 +33,40 @@ Status SendRecvServerImpl::SendVariable(ServerContext *context, } Status SendRecvServerImpl::GetVariable(ServerContext *context, - const VoidMessage *in_var, + const VariableMessage *in_var, VariableMessage *out_var) { - // Block util the sub graph is done. - auto out_tensor_with_name = var_return_queue_.Pop(); + std::string get_var_name = in_var->varname(); + auto *var = scope_->FindVar(get_var_name); + auto tensor = var->Get(); std::ostringstream oss; - framework::SerializeToStream(oss, out_tensor_with_name.second, - platform::CPUDeviceContext()); + framework::SerializeToStream(oss, tensor, platform::CPUDeviceContext()); std::string *varname = out_var->mutable_varname(); - *varname = out_tensor_with_name.first; + *varname = get_var_name; std::string *serialized = out_var->mutable_serialized(); *serialized = oss.str(); return Status::OK; } +Status SendRecvServerImpl::Wait(ServerContext *context, + const VoidMessage *in_var, + VoidMessage *out_var) { + std::unique_lock lock(this->mutex_); + condition_.wait(lock, [=] { return this->done_ == true; }); + return Status::OK; +} + +void SendRecvServerImpl::Start() { + std::unique_lock lock(this->mutex_); + done_ = false; +} + +void SendRecvServerImpl::Done() { + std::unique_lock lock(this->mutex_); + done_ = true; + condition_.notify_all(); +} + } // namespace detail } // namespace operators } // namespace paddle diff --git a/paddle/operators/detail/send_impl.cc b/paddle/operators/detail/send_impl.cc index bf22d3df81..7555cc63fb 100644 --- a/paddle/operators/detail/send_impl.cc +++ b/paddle/operators/detail/send_impl.cc @@ -43,19 +43,20 @@ bool RPCClient::SendVariable(const framework::Scope& scope, return true; } -bool RPCClient::GetVariable(const framework::Scope& scope) { +bool RPCClient::GetVariable(const framework::Scope& scope, + const std::string& outname) { ClientContext context; - VariableMessage msg; - VoidMessage void_msg; + VariableMessage call_msg, ret_msg; + call_msg.set_varname(outname); auto ctx = platform::CPUDeviceContext(); - Status status = stub_->GetVariable(&context, void_msg, &msg); + Status status = stub_->GetVariable(&context, call_msg, &ret_msg); if (!status.ok()) { LOG(ERROR) << "gRPC error: " << status.error_message(); return false; } - std::istringstream iss(msg.serialized()); - auto outname = msg.varname(); + std::istringstream iss(ret_msg.serialized()); + framework::LoDTensor ret_tensor; framework::DeserializeFromStream(iss, &ret_tensor); auto* outvar = scope.FindVar(outname); diff --git a/paddle/operators/detail/send_recv.proto b/paddle/operators/detail/send_recv.proto index d00c33fe42..ce72990806 100644 --- a/paddle/operators/detail/send_recv.proto +++ b/paddle/operators/detail/send_recv.proto @@ -22,7 +22,9 @@ service SendRecvService { // TODO(typhoonzero): add streaming API rpc SendVariable(VariableMessage) returns (VoidMessage) {} // Argument VariableMessage for GetVariable should only contain varname. - rpc GetVariable(VoidMessage) returns (VariableMessage) {} + rpc GetVariable(VariableMessage) returns (VariableMessage) {} + // wait for one execution of the program + rpc Wait(VoidMessage) returns (VoidMessage) {} } // VariableMessage is serialized paddle variable message. diff --git a/paddle/operators/detail/send_recv_impl.h b/paddle/operators/detail/send_recv_impl.h index df01345e34..6edbb2d834 100644 --- a/paddle/operators/detail/send_recv_impl.h +++ b/paddle/operators/detail/send_recv_impl.h @@ -20,10 +20,6 @@ #include "paddle/framework/selected_rows.h" #include "paddle/operators/detail/simple_block_queue.h" -// #include -// #include -// #include -// #include #include "paddle/operators/detail/send_recv.grpc.pb.h" #include "paddle/operators/detail/send_recv.pb.h" @@ -56,18 +52,24 @@ class SendRecvServerImpl final : public SendRecvService::Service { Status SendVariable(ServerContext *context, const VariableMessage *in_var, VoidMessage *out_var) override; - Status GetVariable(ServerContext *context, const VoidMessage *in_var, + Status GetVariable(ServerContext *context, const VariableMessage *in_var, VariableMessage *out_var) override; + Status Wait(ServerContext *context, const VoidMessage *in_var, + VoidMessage *out_var) override; + void Start(); + void Done(); + void SetScope(framework::Scope *scope) { scope_ = scope; }; const TensorWithName Get() { return this->var_recv_queue_.Pop(); } - void Push(const TensorWithName &var) { this->var_return_queue_.Push(var); } - private: // received variable from RPC, operators fetch variable from this queue. SimpleBlockQueue var_recv_queue_; - // calculated variable should push to this queue. - SimpleBlockQueue var_return_queue_; + framework::Scope *scope_; + // condition of the sub program + std::mutex mutex_; + bool done_; + std::condition_variable condition_; }; // RPCClient is a class to send tensors to pserver sub-network @@ -78,7 +80,7 @@ class RPCClient { : stub_(SendRecvService::NewStub(channel)) {} bool SendVariable(const framework::Scope &scope, const std::string &inname); - bool GetVariable(const framework::Scope &scope); + bool GetVariable(const framework::Scope &scope, const std::string &outname); private: std::unique_ptr stub_; diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 9c3e8953bb..9af8d311d9 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -76,12 +76,14 @@ class RecvOp : public framework::OperatorBase { const platform::DeviceContext &dev_ctx) const override { // FIXME(typhoonzero): no new scopes for every run. framework::Scope &recv_scope = scope.NewScope(); + rpc_service_.SetScope(&recv_scope); auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); auto trainer_count = Attr("Trainers"); size_t param_count = param_list.size(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. while (true) { + rpc_service_.Start(); // Get from multiple trainers, we don't care about order in which // the gradient arrives, just add suffix 0~n then average the gradient. for (size_t i = 0; i < param_count * trainer_count; ++i) { @@ -125,13 +127,13 @@ class RecvOp : public framework::OperatorBase { LOG(ERROR) << "run sub program error " << e.what(); } - for (size_t i = 0; i < param_count; ++i) { - auto *out_var = recv_scope.FindVar(param_list[i]); - detail::TensorWithName out; - out.first = param_list[i]; - out.second = out_var->Get(); - rpc_service_->Push(out); - } + // for (size_t i = 0; i < param_count; ++i) { + // auto *out_var = recv_scope.FindVar(param_list[i]); + // detail::TensorWithName out; + // out.first = param_list[i]; + // out.second = out_var->Get(); + // rpc_service_->Push(out); + // } } // while(true) } From 7be79231e17b677f0925397e5a0663bcdd1bfe6e Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 18 Dec 2017 20:49:00 +0800 Subject: [PATCH 21/84] wip multi-trainer --- paddle/operators/detail/send_impl.cc | 6 +++ paddle/operators/detail/send_recv_impl.h | 1 + paddle/operators/recv_op.cc | 5 ++- paddle/operators/send_op.cc | 42 ++++++++++--------- .../paddle/v2/fluid/distribute_transpiler.py | 22 ++++++---- 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/paddle/operators/detail/send_impl.cc b/paddle/operators/detail/send_impl.cc index 7555cc63fb..d7165e13db 100644 --- a/paddle/operators/detail/send_impl.cc +++ b/paddle/operators/detail/send_impl.cc @@ -66,6 +66,12 @@ bool RPCClient::GetVariable(const framework::Scope& scope, return true; } +void RPCClient::Wait() { + ClientContext context; + VoidMessage call_msg, ret_msg; + stub_->Wait(&context, call_msg, &ret_msg); +} + } // namespace detail } // namespace operators } // namespace paddle diff --git a/paddle/operators/detail/send_recv_impl.h b/paddle/operators/detail/send_recv_impl.h index 6edbb2d834..82ab3ab689 100644 --- a/paddle/operators/detail/send_recv_impl.h +++ b/paddle/operators/detail/send_recv_impl.h @@ -81,6 +81,7 @@ class RPCClient { bool SendVariable(const framework::Scope &scope, const std::string &inname); bool GetVariable(const framework::Scope &scope, const std::string &outname); + void Wait(); private: std::unique_ptr stub_; diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 9af8d311d9..6fcb544b5b 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -76,14 +76,14 @@ class RecvOp : public framework::OperatorBase { const platform::DeviceContext &dev_ctx) const override { // FIXME(typhoonzero): no new scopes for every run. framework::Scope &recv_scope = scope.NewScope(); - rpc_service_.SetScope(&recv_scope); + rpc_service_->SetScope(&recv_scope); auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); auto trainer_count = Attr("Trainers"); size_t param_count = param_list.size(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. while (true) { - rpc_service_.Start(); + rpc_service_->Start(); // Get from multiple trainers, we don't care about order in which // the gradient arrives, just add suffix 0~n then average the gradient. for (size_t i = 0; i < param_count * trainer_count; ++i) { @@ -126,6 +126,7 @@ class RecvOp : public framework::OperatorBase { } catch (std::exception &e) { LOG(ERROR) << "run sub program error " << e.what(); } + rpc_service_->Done(); // for (size_t i = 0; i < param_count; ++i) { // auto *out_var = recv_scope.FindVar(param_list[i]); diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index 3fcd2144f9..e94209ec44 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -34,34 +34,36 @@ class SendOp : public framework::OperatorBase { const framework::AttributeMap &attrs) : OperatorBase(type, inputs, outputs, attrs) { // init client when the operator is created at runtime. - if (!client_) { - std::string endpoint = Attr("endpoint"); - client_.reset(new detail::RPCClient( - grpc::CreateChannel(endpoint, grpc::InsecureChannelCredentials()))); - // TODO(typhoonzero): how to call InitVariables + std::vector endpoints = + Attr>("endpoints"); + for (auto ep : endpoints) { + client_map_[ep].reset(new detail::RPCClient( + grpc::CreateChannel(ep, grpc::InsecureChannelCredentials()))); } } void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { auto ins = Inputs("X"); - // TODO(typhoonzero): currently it's non-blocking, - // should block until server responds. - for (auto in : ins) { - bool ret = client_->SendVariable(scope, in); + std::vector epmap = Attr>("epmap"); + // TODO(typhoonzero): use async calls to send multiple variable asyncly. + for (size_t i = 0; i < ins.size(); ++i) { + bool ret = client_map_[epmap[i]]->SendVariable(scope, ins[i]); if (!ret) { - LOG(ERROR) << "send variable error"; + LOG(ERROR) << "send variable error: " << ins[i]; } } - for (auto in : ins) { - bool ret = client_->GetVariable(scope); + client_map_[0]->Wait(); // TODO(typhoonzero): support async optimization + for (size_t i = 0; i < ins.size(); ++i) { + bool ret = client_map_[epmap[i]]->GetVariable(scope, ins[i]); if (!ret) { - LOG(ERROR) << "GetVariable error"; + LOG(ERROR) << "GetVariable error: " << ins[i]; } } } protected: - std::shared_ptr client_{nullptr}; + mutable std::unordered_map> + client_map_; }; class SendOpMaker : public framework::OpProtoAndCheckerMaker { @@ -74,11 +76,13 @@ Recv operator This operator will recv tensor from send_op )DOC"); - AddAttr("endpoint", - "(string, default 127.0.0.1:6164)" - "IP address to listen on.") - .SetDefault("127.0.0.1:6164") - .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); + AddAttr>("endpoints", + "(string vector, default 127.0.0.1:6164)" + "Server endpoints to send variables to."); + AddAttr>("epmap", + "(string vector, default 127.0.0.1:6164)" + "Server endpoints in the order of input " + "variables for mapping"); } }; diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 13006bfd13..e40cdc92b5 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -145,14 +145,20 @@ class DistributeTranspiler: pserver_endpoints = kwargs["pservers"].split(",") self.param_grad_map = split_method(params_and_grads, pserver_endpoints) - for ep in pserver_endpoints: - # FIXME(typhoonzero): send to different servers can run in parrallel. - send_op = program.global_block().append_op( - type="send", - inputs={"X": self.param_grad_map[ep]["grads"] - }, # inputs is a list of tensors to be send - outputs={}, - attrs={"endpoint": ep}) + send_op_ordered_inputs = [] + epmap = [] + for ep, v in self.param_grad_map.iteritems(): + send_op_ordered_inputs.extend(v["grads"]) + for i in v: + epmap.append(ep) + + send_op = program.global_block().append_op( + type="send", + inputs={"X": send_op_ordered_inputs + }, # inputs is a list of tensors to be send + outputs={}, + attrs={"endpoints": pserver_endpoints, + "epmap": epmap}) def _create_var_for_trainers(self, block, var, trainers): var_list = [] From b1b7af400f5be0e7bcfde80e04a9ef8da0adc326 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Tue, 19 Dec 2017 14:04:24 +0800 Subject: [PATCH 22/84] support multi node --- paddle/operators/detail/recv_impl.cc | 14 +++++++++----- paddle/operators/recv_op.cc | 17 +++++------------ paddle/operators/send_op.cc | 3 ++- python/paddle/v2/fluid/distribute_transpiler.py | 3 +-- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/paddle/operators/detail/recv_impl.cc b/paddle/operators/detail/recv_impl.cc index 47decb6d7e..e984f42386 100644 --- a/paddle/operators/detail/recv_impl.cc +++ b/paddle/operators/detail/recv_impl.cc @@ -51,19 +51,23 @@ Status SendRecvServerImpl::GetVariable(ServerContext *context, Status SendRecvServerImpl::Wait(ServerContext *context, const VoidMessage *in_var, VoidMessage *out_var) { - std::unique_lock lock(this->mutex_); - condition_.wait(lock, [=] { return this->done_ == true; }); + { + std::unique_lock lock(this->mutex_); + condition_.wait(lock, [=] { return this->done_ == true; }); + } return Status::OK; } void SendRecvServerImpl::Start() { - std::unique_lock lock(this->mutex_); + std::lock_guard lock(this->mutex_); done_ = false; } void SendRecvServerImpl::Done() { - std::unique_lock lock(this->mutex_); - done_ = true; + { + std::lock_guard lock(this->mutex_); + done_ = true; + } condition_.notify_all(); } diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 6fcb544b5b..094084458e 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -14,7 +14,6 @@ #include #include -#include #include #include @@ -81,9 +80,9 @@ class RecvOp : public framework::OperatorBase { auto grad_list = Attr>("GradList"); auto trainer_count = Attr("Trainers"); size_t param_count = param_list.size(); + rpc_service_->Start(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. while (true) { - rpc_service_->Start(); // Get from multiple trainers, we don't care about order in which // the gradient arrives, just add suffix 0~n then average the gradient. for (size_t i = 0; i < param_count * trainer_count; ++i) { @@ -95,8 +94,8 @@ class RecvOp : public framework::OperatorBase { if (it != grad_list.end()) { param_var_name = param_list[it - grad_list.begin()]; } - VLOG(10) << "recved grad: " << grad_var_name - << " updating param: " << param_var_name; + VLOG(3) << "recved grad: " << grad_var_name + << " updating param: " << param_var_name; auto *merged_grad = recv_scope.FindVar(grad_var_name); if (merged_grad == nullptr) { // create output of merged var. @@ -113,6 +112,7 @@ class RecvOp : public framework::OperatorBase { // FIXME(typhoonzero): do not copy framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); } + rpc_service_->Start(); std::string program_str = Attr("OptimizeProgram"); framework::ProgramDesc program_desc; @@ -127,14 +127,7 @@ class RecvOp : public framework::OperatorBase { LOG(ERROR) << "run sub program error " << e.what(); } rpc_service_->Done(); - - // for (size_t i = 0; i < param_count; ++i) { - // auto *out_var = recv_scope.FindVar(param_list[i]); - // detail::TensorWithName out; - // out.first = param_list[i]; - // out.second = out_var->Get(); - // rpc_service_->Push(out); - // } + grads_counter_.clear(); } // while(true) } diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index e94209ec44..9eafa1655a 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -52,7 +52,8 @@ class SendOp : public framework::OperatorBase { LOG(ERROR) << "send variable error: " << ins[i]; } } - client_map_[0]->Wait(); // TODO(typhoonzero): support async optimization + // TODO(typhoonzero): support async optimization + client_map_[epmap[0]]->Wait(); for (size_t i = 0; i < ins.size(); ++i) { bool ret = client_map_[epmap[i]]->GetVariable(scope, ins[i]); if (!ret) { diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index e40cdc92b5..7dfbab4677 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -149,9 +149,8 @@ class DistributeTranspiler: epmap = [] for ep, v in self.param_grad_map.iteritems(): send_op_ordered_inputs.extend(v["grads"]) - for i in v: + for i in v["grads"]: epmap.append(ep) - send_op = program.global_block().append_op( type="send", inputs={"X": send_op_ordered_inputs From 5e04b64fa0244239e7c476ebd846d20e9958d8d6 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 19 Dec 2017 16:19:29 +0800 Subject: [PATCH 23/84] refine elementwise --- paddle/operators/elementwise_op_function.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index 7ebfc7df8c..65484f318e 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -103,10 +103,12 @@ class MidWiseTransformIterator { MidWiseTransformIterator& operator++() { ++j_; - i_ = j_ / post_; - if (UNLIKELY(i_ == n_)) { - j_ = 0; - i_ = 0; + if (UNLIKELY(j_ == post_)) { + ++i_; + if (UNLIKELY(i_ == n_)) { + j_ = 0; + i_ = 0; + } } return *this; } From 293b292e0ff3e6055dceb807c4cb57fc7bacb226 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 19 Dec 2017 17:00:55 +0800 Subject: [PATCH 24/84] refine im2col --- paddle/operators/math/im2col.cc | 39 +++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index 707ebf0596..a746c267b6 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -61,14 +61,22 @@ class Im2ColFunctor(); T* col_data = col->data(); - + int w_offset = -1; + int h_offset = 0; + int c_im = 0; for (int c = 0; c < channels_col; ++c) { - int w_offset = c % filter_width; - int h_offset = (c / filter_width) % filter_height; - int c_im = c / filter_width / filter_height; + ++w_offset; + if (UNLIKELY(w_offset == filter_width)) { + w_offset = 0; + ++h_offset; + if (UNLIKELY(h_offset == filter_height)) { + h_offset = 0; + ++c_im; + } + } for (int h = 0; h < col_height; ++h) { + int im_row_idx = h * stride[0] - padding[0] + h_offset * dilation[0]; for (int w = 0; w < col_width; ++w) { - int im_row_idx = h * stride[0] - padding[0] + h_offset * dilation[0]; int im_col_idx = w * stride[1] - padding[1] + w_offset * dilation[1]; int col_idx = (c * col_height + h) * col_width + w; int im_idx = (im_row_idx + c_im * im_height) * im_width + im_col_idx; @@ -127,19 +135,26 @@ class Col2ImFunctordata(); const T* col_data = col.data(); + int w_offset = -1; + int h_offset = 0; + int c_im = 0; for (int c = 0; c < channels_col; ++c) { - int w_offset = c % filter_width; - int h_offset = (c / filter_width) % filter_height; - int c_im = c / filter_width / filter_height; + ++w_offset; + if (UNLIKELY(w_offset == filter_width)) { + w_offset = 0; + ++h_offset; + if (UNLIKELY(h_offset == filter_height)) { + h_offset = 0; + ++c_im; + } + } for (int h = 0; h < col_height; ++h) { + int im_row_idx = h * stride[0] - padding[0] + h_offset * dilation[0]; for (int w = 0; w < col_width; ++w) { - int im_row_idx = h * stride[0] - padding[0] + h_offset * dilation[0]; int im_col_idx = w * stride[1] - padding[1] + w_offset * dilation[1]; - if ((im_row_idx) >= 0 && (im_row_idx) < im_height && (im_col_idx) >= 0 && (im_col_idx) < im_width) { - im_row_idx += c_im * im_height; - im_data[im_row_idx * im_width + im_col_idx] += + im_data[(im_row_idx + c_im * im_height) * im_width + im_col_idx] += col_data[(c * col_height + h) * col_width + w]; } } From f1ab13bd0e42005bc3cc1163ef0d8f9c6125c128 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 19 Dec 2017 19:16:14 +0800 Subject: [PATCH 25/84] refine --- paddle/operators/elementwise_op_function.h | 6 +++--- paddle/operators/math/im2col.cc | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index 65484f318e..9edfacd6df 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -105,8 +105,8 @@ class MidWiseTransformIterator { ++j_; if (UNLIKELY(j_ == post_)) { ++i_; + j_ = 0; if (UNLIKELY(i_ == n_)) { - j_ = 0; i_ = 0; } } @@ -127,10 +127,10 @@ class MidWiseTransformIterator { private: const T* ptr_; - int i_; + int64_t i_; int64_t j_; int64_t n_; - int post_; + int64_t post_; }; #ifdef __NVCC__ diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index a746c267b6..d11a6afe9b 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -66,10 +66,10 @@ class Im2ColFunctor= 0 && im_row_offset < im_height && im_col_offset >= 0 && im_col_offset < im_width) { int im_offset = From 7b0744edcf6d46a326b0bc7ec15cf9f1329cda4a Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 19 Dec 2017 19:49:43 +0800 Subject: [PATCH 26/84] refine im2col --- paddle/operators/math/im2col.cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index d11a6afe9b..50af3199f2 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -61,19 +61,10 @@ class Im2ColFunctor(); T* col_data = col->data(); - int w_offset = -1; - int h_offset = 0; - int c_im = 0; for (int c = 0; c < channels_col; ++c) { - ++w_offset; - if (w_offset == filter_width) { - w_offset = 0; - ++h_offset; - if (h_offset == filter_height) { - h_offset = 0; - ++c_im; - } - } + int w_offset = c % filter_width; + int h_offset = (c / filter_width) % filter_height; + int c_im = c / (filter_width * filter_height); for (int h = 0; h < col_height; ++h) { int im_row_idx = h * stride[0] - padding[0] + h_offset * dilation[0]; for (int w = 0; w < col_width; ++w) { From ee49f54e7fd559002c6ab92362b56c057136ca62 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 19 Dec 2017 10:56:48 -0500 Subject: [PATCH 27/84] use small samples to infer openblas for saving time. --- benchmark/paddle/image/googlenet.py | 4 +++- benchmark/paddle/image/provider.py | 3 ++- benchmark/paddle/image/resnet.py | 4 +++- benchmark/paddle/image/run_openblas_infer.sh | 11 ++++++----- benchmark/paddle/image/run_openblas_train.sh | 1 + benchmark/paddle/image/vgg.py | 4 +++- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/benchmark/paddle/image/googlenet.py b/benchmark/paddle/image/googlenet.py index 7059c13bd2..2a850ccb7f 100644 --- a/benchmark/paddle/image/googlenet.py +++ b/benchmark/paddle/image/googlenet.py @@ -7,13 +7,15 @@ num_class = 1000 batch_size = get_config_arg('batch_size', int, 128) use_gpu = get_config_arg('use_gpu', bool, True) is_infer = get_config_arg("is_infer", bool, False) +num_samples = get_config_arg('num_samples', int, 2560) args = { 'height': height, 'width': width, 'color': True, 'num_class': num_class, - 'is_infer': is_infer + 'is_infer': is_infer, + 'num_samples': num_samples } define_py_data_sources2( "train.list" if not is_infer else None, diff --git a/benchmark/paddle/image/provider.py b/benchmark/paddle/image/provider.py index 927b175994..1018ec9ce1 100644 --- a/benchmark/paddle/image/provider.py +++ b/benchmark/paddle/image/provider.py @@ -14,6 +14,7 @@ def initHook(settings, height, width, color, num_class, **kwargs): else: settings.data_size = settings.height * settings.width settings.is_infer = kwargs.get('is_infer', False) + settings.num_samples = kwargs.get('num_samples', 2560) if settings.is_infer: settings.slots = [dense_vector(settings.data_size)] else: @@ -23,7 +24,7 @@ def initHook(settings, height, width, color, num_class, **kwargs): @provider( init_hook=initHook, min_pool_size=-1, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, file_list): - for i in xrange(2560 if settings.is_infer else 1024): + for i in xrange(settings.num_samples): img = np.random.rand(1, settings.data_size).reshape(-1, 1).flatten() if settings.is_infer: yield img.astype('float32') diff --git a/benchmark/paddle/image/resnet.py b/benchmark/paddle/image/resnet.py index 4a14363ff1..2846e4763f 100644 --- a/benchmark/paddle/image/resnet.py +++ b/benchmark/paddle/image/resnet.py @@ -7,13 +7,15 @@ num_class = 1000 batch_size = get_config_arg('batch_size', int, 64) layer_num = get_config_arg("layer_num", int, 50) is_infer = get_config_arg("is_infer", bool, False) +num_samples = get_config_arg('num_samples', int, 2560) args = { 'height': height, 'width': width, 'color': True, 'num_class': num_class, - 'is_infer': is_infer + 'is_infer': is_infer, + 'num_samples': num_samples } define_py_data_sources2( "train.list" if not is_infer else None, diff --git a/benchmark/paddle/image/run_openblas_infer.sh b/benchmark/paddle/image/run_openblas_infer.sh index c1001d3a7c..83b603c170 100755 --- a/benchmark/paddle/image/run_openblas_infer.sh +++ b/benchmark/paddle/image/run_openblas_infer.sh @@ -23,24 +23,25 @@ function infer() { echo "./run_mkl_infer.sh to save the model first" exit 0 fi - log_period=$((256 / bs)) + log_period=$((32 / bs)) paddle train --job=test \ --config="${topology}.py" \ + --use_mkldnn=False \ --use_gpu=False \ --trainer_count=$thread \ --log_period=$log_period \ - --config_args="batch_size=${bs},layer_num=${layer_num},is_infer=True" \ + --config_args="batch_size=${bs},layer_num=${layer_num},is_infer=True,num_samples=256" \ --init_model_path=$models_in \ 2>&1 | tee ${log} - # calculate the last 5 logs period time of 1280 samples, + # calculate the last 5 logs period time of 160(=32*5) samples, # the time before are burning time. start=`tail ${log} -n 7 | head -n 1 | awk -F ' ' '{print $2}' | xargs` end=`tail ${log} -n 2 | head -n 1 | awk -F ' ' '{print $2}' | xargs` start_sec=`clock_to_seconds $start` end_sec=`clock_to_seconds $end` - fps=`awk 'BEGIN{printf "%.2f",(1280 / ('$end_sec' - '$start_sec'))}'` - echo "Last 1280 samples start: ${start}(${start_sec} sec), end: ${end}(${end_sec} sec;" >> ${log} + fps=`awk 'BEGIN{printf "%.2f",(160 / ('$end_sec' - '$start_sec'))}'` + echo "Last 160 samples start: ${start}(${start_sec} sec), end: ${end}(${end_sec} sec;" >> ${log} echo "FPS: $fps images/sec" 2>&1 | tee -a ${log} } diff --git a/benchmark/paddle/image/run_openblas_train.sh b/benchmark/paddle/image/run_openblas_train.sh index b9494ce119..fce6f9be4a 100755 --- a/benchmark/paddle/image/run_openblas_train.sh +++ b/benchmark/paddle/image/run_openblas_train.sh @@ -12,6 +12,7 @@ function train() { config="${topology}.py" paddle train --job=time \ --config=$config \ + --use_mkldnn=False \ --use_gpu=False \ --trainer_count=$thread \ --log_period=10 \ diff --git a/benchmark/paddle/image/vgg.py b/benchmark/paddle/image/vgg.py index 8d0a1e97a4..ca0a6798fb 100644 --- a/benchmark/paddle/image/vgg.py +++ b/benchmark/paddle/image/vgg.py @@ -7,13 +7,15 @@ num_class = 1000 batch_size = get_config_arg('batch_size', int, 64) layer_num = get_config_arg('layer_num', int, 19) is_infer = get_config_arg("is_infer", bool, False) +num_samples = get_config_arg('num_samples', int, 2560) args = { 'height': height, 'width': width, 'color': True, 'num_class': num_class, - 'is_infer': is_infer + 'is_infer': is_infer, + 'num_samples': num_samples } define_py_data_sources2( "train.list" if not is_infer else None, From cb3a74e43644edeeaa697ae6dfe1cd6c9c63a968 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 20 Dec 2017 12:37:19 +0800 Subject: [PATCH 28/84] revert im2col --- paddle/operators/math/im2col.cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index 50af3199f2..c2633b2e16 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -126,19 +126,10 @@ class Col2ImFunctordata(); const T* col_data = col.data(); - int w_offset = -1; - int h_offset = 0; - int c_im = 0; for (int c = 0; c < channels_col; ++c) { - ++w_offset; - if (w_offset == filter_width) { - w_offset = 0; - ++h_offset; - if (h_offset == filter_height) { - h_offset = 0; - ++c_im; - } - } + int w_offset = c % filter_width; + int h_offset = (c / filter_width) % filter_height; + int c_im = c / (filter_width * filter_height); for (int h = 0; h < col_height; ++h) { int im_row_idx = h * stride[0] - padding[0] + h_offset * dilation[0]; for (int w = 0; w < col_width; ++w) { From c97369b470c2eefbe5a2094af061cfcdb8e5a33a Mon Sep 17 00:00:00 2001 From: guosheng Date: Wed, 20 Dec 2017 15:29:13 +0800 Subject: [PATCH 29/84] Add python wrapper for reduce_mean --- doc/api/v2/fluid/layers.rst | 6 ++++ python/paddle/v2/fluid/layers/nn.py | 46 ++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 842f3b1800..4849a903e9 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -318,3 +318,9 @@ reduce_sum .. autofunction:: paddle.v2.fluid.layers.reduce_sum :noindex: + +reduce_mean +--------- +.. autofunction:: paddle.v2.fluid.layers.reduce_mean + :noindex: + diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 73f68466da..de5fb2451c 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -13,7 +13,7 @@ __all__ = [ 'crf_decoding', 'cos_sim', 'cross_entropy', 'square_error_cost', 'accuracy', 'chunk_eval', 'sequence_conv', 'conv2d', 'sequence_pool', 'pool2d', 'batch_norm', 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', - 'lstm_unit', 'reduce_sum' + 'lstm_unit', 'reduce_sum', 'reduce_mean' ] @@ -979,3 +979,47 @@ def reduce_sum(input, dim=None, keep_dim=False): 'reduce_all': True if dim == None else False }) return out + + +def reduce_mean(input, dim=None, keep_dim=False): + """ + Computes the mean of tensor elements over the given dimension. + + Args: + input (Variable): The input variable which is a Tensor or LoDTensor. + dim (int|None): The dimension along which the mean is computed. If + :attr:`None`, compute the mean over all elements of :attr:`input` + and return a Tensor variable with a single element, otherwise + must be in the range :math:`[-rank(input), rank(input))`. If + :math:`dim < 0`, the dimension to reduce is :math:`rank + dim`. + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension + than the :attr:`input` unless :attr:`keep_dim` is true. + + Returns: + Variable: The reduced Tensor variable. + + Examples: + .. code-block:: python + + # x is a Tensor variable with following elements: + # [[0.2, 0.3, 0.5, 0.9] + # [0.1, 0.2, 0.6, 0.7]] + # Each example is followed by the correspending output tensor. + fluid.layers.reduce_mean(x) # [0.4375] + fluid.layers.reduce_mean(x, dim=0) # [0.15, 0.25, 0.55, 0.8] + fluid.layers.reduce_mean(x, dim=-1) # [0.475, 0.4] + fluid.layers.reduce_mean(x, dim=1, keep_dim=True) # [[0.475], [0.4]] + """ + helper = LayerHelper('reduce_mean', **locals()) + out = helper.create_tmp_variable(dtype=helper.input_dtype()) + helper.append_op( + type='reduce_mean', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None else 0, + 'keep_dim': keep_dim, + 'reduce_all': True if dim == None else False + }) + return out From 22022017d9603ff9498cd1d3546dbe2719875f9d Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 20 Dec 2017 16:48:07 +0800 Subject: [PATCH 30/84] add python wrapper for sequence_pool --- python/paddle/v2/fluid/layers/nn.py | 50 +++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 73f68466da..59212e8497 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -520,9 +520,53 @@ def conv2d(input, def sequence_pool(input, pool_type, **kwargs): """ - This function add the operator for sequence pooling. - This is applied on top of the input using pool_type mentioned - in the parameters. + This function add the operator for sequence pooling. + It pools features of all time-steps of each instance, and is applied + on top of the input using pool_type mentioned in the parameters. + + It supports four pool_type: + + - average: :math:`Out[i] = \\frac{\sum_i X_i}{N}` + - sum: :math:`Out[i] = \sum_jX_{ij}` + - sqrt: :math:`Out[i] = \\frac{\sum_jX_{ij}}{\sqrt{len(X_i)}}` + - max: :math:`Out[i] = max(X_i)` + + .. code-block:: text + + x is a 1-level LoDTensor: + x.lod = [[0, 2, 5, 7]] + x.data = [1, 3, 2, 4, 6, 5, 1] + x.dims = [7, 1] + + then output is a Tensor: + out.dim = [3, 1] + with condition len(x.lod[-1]) - 1 == out.dims[0] + + for different pool_type: + average: out.data = [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 + sum : out.data = [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 + sqrt : out.data = [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), + 6.93=(2+4+6)/sqrt(3), 4.24=(5+1)/sqrt(2) + max : out.data = [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) + + Args: + input(variable): The input variable which is a LoDTensor. + pool_type (string): The pooling type of sequence_pool. + It supports average, sum, sqrt and max. + + Returns: + The sequence pooling variable which is a Tensor. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + avg_x = fluid.layers.sequence_pool(input=x, pool_type='average') + sum_x = fluid.layers.sequence_pool(input=x, pool_type='sum') + sqrt_x = fluid.layers.sequence_pool(input=x, pool_type='sqrt') + max_x = fluid.layers.sequence_pool(input=x, pool_type='max') """ helper = LayerHelper('sequence_pool', input=input, **kwargs) dtype = helper.input_dtype() From f1a9efcac5743e05fc4e1dbc53c08f69ec19d5a8 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 20 Dec 2017 17:18:21 +0800 Subject: [PATCH 31/84] add kernel hint design --- doc/design/kernel_hint_design.md | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 doc/design/kernel_hint_design.md diff --git a/doc/design/kernel_hint_design.md b/doc/design/kernel_hint_design.md new file mode 100644 index 0000000000..1ccab16844 --- /dev/null +++ b/doc/design/kernel_hint_design.md @@ -0,0 +1,54 @@ +## Problem +In PaddlePaddle's [Design](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/switch_kernel.md), one Operator may have multiple kernels. Users may have some personal preference to choose a certain type of kernel for an operator, such as `force_cpu` to use a CPU kernel, `use_cudnn` to choose a CUDNN kernel, we need to provide a way for a user to do this. + +In the current design, we use KernelType to describe one kernel. + +```cpp +struct KernelType { + Place place_; + DataType data_type_; + LayoutType layout_; +}; +``` + `place_` `data_type_` and `layout_` can come from the input tensor of the operator, `GetActualKernelType(inputs)` use inputs to infer the proper kernel key that fit the incoming data, user can not config it. + +The design also provides a virtual method `GetExpectedKernelType` that user can overload and choose the KernelType they want to use. + +so, we should send the information user defined in proto to `GetExpectedKernelType` for choosing a kernel. + +The problem is, how should we define and send the information for `GetExpectedKernelType` to use? + +## Solution +1, Do nothing, let the user add the information they want to operator‘s attribute and get them inside `GetExpectedKernelType`, this can work right. But there is a little problem that users may define many kinds of hints for the same purpose, such as `force_cpu`, `use_cpu`, `CPU` for CPU kernel, and `use_cudnn`, `force_cudnn`, `cudnn_kernel` for use of CUDNN kernel. + +2, Pre-define all the needed option and use a single attr key such as `kernel_hint` for the user, this is not so flexible if the user wants to define some more kind of hint. + + +To provide enough flexibility while avoiding confusion definition, we can predefine some options, such as `force_cpu`, `use_cudnn`, `use_mkldnn` for a user to choose. + +```cpp +const std::string kNonHint = ""; +const std::string kForceCPU = "force_cpu"; +const std::string kUseCUDNN = "use_cudnn"; +const std::string kUseMKLDNN = "use_mkldnn"; + +KernelType GetExpectedKernelTyp() { + // "kernel_hint" is a user defined attribute name + if (Attr("kernel_hint") == kForceCPU) { + return KernelType(CPUPlace, ...) + } else { + ... + } +} +``` + +In Python code + +```python +def xx_layer(..., kernel_hint=None): + layer_helper = ... + layer_helper .append_op( + type="xx", + # "kernel_hint" should be the same with the attr name in CPP + attr={"kernel_hint": kernel_hint or ""}) +``` From 6a1e31291408ad172110374c9555f6705e30b92b Mon Sep 17 00:00:00 2001 From: caoying03 Date: Wed, 20 Dec 2017 16:25:19 +0800 Subject: [PATCH 32/84] refine the doc. --- paddle/operators/mul_op.cc | 33 +++++++++---- python/paddle/v2/fluid/layers/nn.py | 73 ++++++++++++++++++----------- 2 files changed, 69 insertions(+), 37 deletions(-) diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index a4bf0711de..25944e3d13 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -73,25 +73,38 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { public: MulOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input of mul op"); - AddInput("Y", "The second input of mul op"); - AddOutput("Out", "The output of mul op"); + AddInput("X", "The first input tensor of the mul op."); + AddInput("Y", "The second input tensor of the mul op."); + AddOutput("Out", "The output tensor of the mul op."); AddAttr( "x_num_col_dims", "(int, default 1) " - R"DOC(mul_op can take tensors with more than two dimensions as input `X`, - in that case, tensors will be reshaped to a matrix. The matrix's first - dimension(column length) will be the product of tensor's last - `num_col_dims` dimensions, and the matrix's second dimension(row length) - will be the product of tensor's first `rank - num_col_dims` dimensions. + R"DOC(The mul_op can take tensors with more than two dimensions as its + inputs. If the input `X` is a tensor with more than two + dimensions, `X` will be flatten into a two-dimensional matrix + first. The flatten rule is: the first `num_col_dims` will be + flatten to form the first dimension of the matrix (height of the + matrix), and the rest `rank(X) - num_col_dims` dimensions are + flattened to form the second dimension of the matrix (width of the + matrix). As a result, height of the flattened matrix is equal to + the product of `X`'s first `x_num_col_dims` dimensions' sizes, + and width of the flattened matrix is equal to the product of `X`'s + last `rank(x) - num_col_dims` dimensions' size. + For example, suppose `X` is a 6-dimensional tensor with the shape + [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. Then, the flattened + matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. )DOC") .SetDefault(1) .EqualGreaterThan(1); AddAttr( "y_num_col_dims", "(int, default 1) " - R"DOC(mul_op can take tensors with more than two dimensions as input `Y`, - in that case, tensors will be reshaped to a matrix. Just like input `X`. + R"DOC(The mul_op can take tensors with more than two dimensions as its + inputs. If the input `Y` is a tensor with more than two + dimensions, `Y` will be flatten into a two-dimensional matrix + first. The attribute `y_num_col_dims` is used to flatten `Y` into + a two-dimensional matrix. See the comments of `x_num_col_dims` for + more details. )DOC") .SetDefault(1) .EqualGreaterThan(1); diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 2c38c23224..71dab4e66a 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -28,31 +28,52 @@ def fc(input, Fully Connected Layer. Args: - input: The input tensor to the function - size: The size of the layer - num_flatten_dims: Number of columns in input - param_attr: The parameters/weights to the FC Layer - param_initializer: Initializer used for the weight/parameter. If None, XavierInitializer() is used - bias_attr: The bias parameter for the FC layer - bias_initializer: Initializer used for the bias. If None, then ConstantInitializer() is used - act: Activation to be applied to the output of FC layer - name: Name/alias of the function - main_program: Name of the main program that calls this - startup_program: Name of the startup program - - This function can take in multiple inputs and performs the Fully Connected - function (linear transformation) on top of each of them. - So for input x, the output will be : Wx + b. Where W is the parameter, - b the bias and x is the input. - - The function also applies an activation (non-linearity) on top of the - output, if activation is passed in the input. - - All the input variables of this function are passed in as local variables - to the LayerHelper constructor. + input: The input tensor(s) to the fully connected layer. + size: The number of output units in the fully connected layer. + num_flatten_dims: The fc layer can accept an input tensor with more than + two dimensions. If this happens, the multidimensional + tensor will first be flattened into a 2-dimensional + matrix. The parameter `num_flatten_dims` determines + how the input tensor is flattened: the first + `num_flatten_dims` dimensions will be flatten to form + the first dimension of the final matrix (height of the + matrix), and the rest `rank(X) - num_col_dims` + dimensions are flattened to form the second dimension + of the final matrix (width of the matrix). For example, + suppose `X` is a 6-dimensional tensor with a shape + [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. Then, the + flattened matrix will have a shape [2 x 3 x 4, 5 x 6] + = [24, 30]. By default, `x_num_col_dims` is set to 1. + param_attr: The parameter attribute for learnable parameters/weights of + the fully connected Layer. + param_initializer: The initializer used for the weight/parameter. + If set None, XavierInitializer() will be used. + bias_attr: The parameter attribute for the bias parameter for this layer. + If set None, no bias will be added to the output units. + bias_initializer: The initializer used for the bias. If set None, + then ConstantInitializer() will be used. + act: Activation to be applied to the output of the fully connected layer. + name: Name/alias of the fully connected layer. + + The fully connected can take multiple tensor as inputs. It creates a + variable (one for each input tensor) called weights which represents a + fully connected weight matrix from each input unit to each output unit. + The fully connected layer multiplies each input tensor with its coresponding + weight to produce an output Tensor. If multiple input tensors are given, + the results of multiple multiplications will be sumed up. If bias_attr is + not None, a biases variable will be created and added to the output. + Finally, if activation is not None, it will be applied to the output as well. + + This process canbe formulated as follows: + + .. math:: + Y = \sigma({\sum_{i=0}^{N-1}W_iX_i + b}) + + where, :math:`N` is the number of input, :math:`X_i` is the input tensor, + :math`W` is the weights created by this layer, :math:`b` is the bias. """ - helper = LayerHelper('fc', **locals()) + helper = LayerHelper("fc", **locals()) dtype = helper.input_dtype() @@ -72,8 +93,8 @@ def fc(input, "Y": w, }, outputs={"Out": tmp}, - attrs={'x_num_col_dims': num_flatten_dims, - 'y_num_col_dims': 1}) + attrs={"x_num_col_dims": num_flatten_dims, + "y_num_col_dims": 1}) mul_results.append(tmp) # sum @@ -100,8 +121,6 @@ def embedding(input, size, is_sparse=False, param_attr=None, dtype='float32'): is_sparse: A flag that decleares whether the input is sparse param_attr: Parameters for this layer dtype: The type of data : float32, float_16, int etc - main_program: Name of the main program that calls this - startup_program: Name of the startup program This function can take in the input (which is a vector of IDs) and performs a lookup in the lookup_table using these IDs, to result into From 1102591595885bcd61a1041bc341e0c398bd21db Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 20 Dec 2017 17:22:38 +0800 Subject: [PATCH 33/84] add two sub title --- doc/design/kernel_hint_design.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/design/kernel_hint_design.md b/doc/design/kernel_hint_design.md index 1ccab16844..75f0e4ea72 100644 --- a/doc/design/kernel_hint_design.md +++ b/doc/design/kernel_hint_design.md @@ -19,11 +19,13 @@ so, we should send the information user defined in proto to `GetExpectedKernelTy The problem is, how should we define and send the information for `GetExpectedKernelType` to use? ## Solution + +### potential choice 1, Do nothing, let the user add the information they want to operator‘s attribute and get them inside `GetExpectedKernelType`, this can work right. But there is a little problem that users may define many kinds of hints for the same purpose, such as `force_cpu`, `use_cpu`, `CPU` for CPU kernel, and `use_cudnn`, `force_cudnn`, `cudnn_kernel` for use of CUDNN kernel. 2, Pre-define all the needed option and use a single attr key such as `kernel_hint` for the user, this is not so flexible if the user wants to define some more kind of hint. - +### final choice To provide enough flexibility while avoiding confusion definition, we can predefine some options, such as `force_cpu`, `use_cudnn`, `use_mkldnn` for a user to choose. ```cpp From c322c7bb02849bf7fa89c552088298609275989b Mon Sep 17 00:00:00 2001 From: caoying03 Date: Wed, 20 Dec 2017 17:31:04 +0800 Subject: [PATCH 34/84] some small refines. --- paddle/operators/mul_op.cc | 31 ++++++++++++++--------------- python/paddle/v2/fluid/layers/nn.py | 25 +++++++++++++---------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index 25944e3d13..cee1bb0098 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -81,18 +81,18 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { "(int, default 1) " R"DOC(The mul_op can take tensors with more than two dimensions as its inputs. If the input `X` is a tensor with more than two - dimensions, `X` will be flatten into a two-dimensional matrix - first. The flatten rule is: the first `num_col_dims` will be - flatten to form the first dimension of the matrix (height of the - matrix), and the rest `rank(X) - num_col_dims` dimensions are - flattened to form the second dimension of the matrix (width of the - matrix). As a result, height of the flattened matrix is equal to - the product of `X`'s first `x_num_col_dims` dimensions' sizes, - and width of the flattened matrix is equal to the product of `X`'s - last `rank(x) - num_col_dims` dimensions' size. - For example, suppose `X` is a 6-dimensional tensor with the shape - [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. Then, the flattened - matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. + dimensions, `X` will be flattened into a two-dimensional matrix + first. The flattening rule is: the first `num_col_dims` will be + flattened to form the first dimension of the final matrix (height + of the matrix), and the rest `rank(X) - num_col_dims` dimensions + are flattened to form the second dimension of the final matrix ( + width of the matrix). As a result, height of the flattened matrix + is equal to the product of `X`'s first `x_num_col_dims` dimensions' + sizes, and width of the flattened matrix is equal to the product + of `X`'s last `rank(x) - num_col_dims` dimensions' size. + For example, suppose `X` is a 6-dimensional tensor with the shape + [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. Then, the flattened + matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. )DOC") .SetDefault(1) .EqualGreaterThan(1); @@ -102,14 +102,13 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { R"DOC(The mul_op can take tensors with more than two dimensions as its inputs. If the input `Y` is a tensor with more than two dimensions, `Y` will be flatten into a two-dimensional matrix - first. The attribute `y_num_col_dims` is used to flatten `Y` into - a two-dimensional matrix. See the comments of `x_num_col_dims` for - more details. + first. The attribute `y_num_col_dims` determines how `Y` is + flattened. See comments of `x_num_col_dims` for more details. )DOC") .SetDefault(1) .EqualGreaterThan(1); AddComment(R"DOC( -Mul Operator. +Mul Operator. This operator is used to perform matrix multiplication for input X and Y. diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 4d8ecb5ce2..51da00f565 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -55,24 +55,27 @@ def fc(input, act: Activation to be applied to the output of the fully connected layer. name: Name/alias of the fully connected layer. - The fully connected can take multiple tensor as inputs. It creates a - variable (one for each input tensor) called weights which represents a - fully connected weight matrix from each input unit to each output unit. - The fully connected layer multiplies each input tensor with its coresponding - weight to produce an output Tensor. If multiple input tensors are given, - the results of multiple multiplications will be sumed up. If bias_attr is - not None, a biases variable will be created and added to the output. - Finally, if activation is not None, it will be applied to the output as well. - - This process canbe formulated as follows: + The fully connected layer can take multiple tensors as its inputs. It + creates a variable (one for each input tensor) called weights for each input + tensor, which represents a fully connected weight matrix from each input + unit to each output unit. The fully connected layer multiplies each input + tensor with its coresponding weight to produce an output Tensor. If + multiple input tensors are given, the results of multiple multiplications + will be sumed up. If bias_attr is not None, a biases variable will be + created and added to the output. Finally, if activation is not None, + it will be applied to the output as well. + + This process can be formulated as follows: .. math:: Y = \sigma({\sum_{i=0}^{N-1}W_iX_i + b}) where, :math:`N` is the number of input, :math:`X_i` is the input tensor, - :math`W` is the weights created by this layer, :math:`b` is the bias. + :math:`W` is the weights created by this layer, :math:`b` is the bias + created by this layer (if needed), :math:`\sigma` is the activation funtion. """ + helper = LayerHelper("fc", **locals()) dtype = helper.input_dtype() From f3cbd8d404edd956a921f0b5fd502ca3785b8e13 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 20 Dec 2017 18:47:56 +0800 Subject: [PATCH 35/84] follow comment --- doc/design/kernel_hint_design.md | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/doc/design/kernel_hint_design.md b/doc/design/kernel_hint_design.md index 75f0e4ea72..da4856bb6a 100644 --- a/doc/design/kernel_hint_design.md +++ b/doc/design/kernel_hint_design.md @@ -1,7 +1,7 @@ ## Problem -In PaddlePaddle's [Design](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/switch_kernel.md), one Operator may have multiple kernels. Users may have some personal preference to choose a certain type of kernel for an operator, such as `force_cpu` to use a CPU kernel, `use_cudnn` to choose a CUDNN kernel, we need to provide a way for a user to do this. +In PaddlePaddle's [Design](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/switch_kernel.md), one Operator may have multiple kernels. Users may have some personal preference to choose a certain type of kernel for an operator, such as `force_cpu` to choose a CPU kernel, `use_cudnn` to choose a CUDNN kernel, we need to provide a way for users to do this. -In the current design, we use KernelType to describe one kernel. +In the current design, we use KernelType to describe one kernel. ```cpp struct KernelType { @@ -10,33 +10,33 @@ struct KernelType { LayoutType layout_; }; ``` - `place_` `data_type_` and `layout_` can come from the input tensor of the operator, `GetActualKernelType(inputs)` use inputs to infer the proper kernel key that fit the incoming data, user can not config it. + `place_` `data_type_` and `layout_` can be got from the input tensors of the operator, `GetActualKernelType(inputs)` use inputs to infer the proper kernel key that fit the incoming data, but users can not directly configure it. -The design also provides a virtual method `GetExpectedKernelType` that user can overload and choose the KernelType they want to use. +The [design](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/switch_kernel.md) also provides a virtual method `GetExpectedKernelType` that user can overload and use to choose the KernelType they want to use. -so, we should send the information user defined in proto to `GetExpectedKernelType` for choosing a kernel. +So we should send the information user defined in proto to `GetExpectedKernelType` for choosing a kernel. The problem is, how should we define and send the information for `GetExpectedKernelType` to use? ## Solution -### potential choice -1, Do nothing, let the user add the information they want to operator‘s attribute and get them inside `GetExpectedKernelType`, this can work right. But there is a little problem that users may define many kinds of hints for the same purpose, such as `force_cpu`, `use_cpu`, `CPU` for CPU kernel, and `use_cudnn`, `force_cudnn`, `cudnn_kernel` for use of CUDNN kernel. +### Potential choice +1. Do nothing, let the user add the information they want to operator‘s attribute and get them inside `GetExpectedKernelType`, this can work properly. But there is a little problem that users may define many kinds of hints for the same purpose, such as `force_cpu`, `use_cpu`, `cpu_kernel` to choose CPU kernel, and `use_cudnn`, `force_cudnn`, `cudnn_kernel` to choose CUDNN kernel. -2, Pre-define all the needed option and use a single attr key such as `kernel_hint` for the user, this is not so flexible if the user wants to define some more kind of hint. +2. Pre-define all the needed option and use a single attr key such as `kernel_hint` for the user, this is not so flexible if the user wants to define some more kind of hint. -### final choice -To provide enough flexibility while avoiding confusion definition, we can predefine some options, such as `force_cpu`, `use_cudnn`, `use_mkldnn` for a user to choose. +### Final choice +To provide enough flexibility while avoiding confusion definition, we can define some global constants for these attribute names, such as `force_cpu`, `use_cudnn`, `use_mkldnn` for a user to choose. + +In C++ ```cpp -const std::string kNonHint = ""; const std::string kForceCPU = "force_cpu"; const std::string kUseCUDNN = "use_cudnn"; const std::string kUseMKLDNN = "use_mkldnn"; -KernelType GetExpectedKernelTyp() { - // "kernel_hint" is a user defined attribute name - if (Attr("kernel_hint") == kForceCPU) { +KernelType GetExpectedKernelType() { + if (Attr(kForceCPU)) { return KernelType(CPUPlace, ...) } else { ... @@ -47,10 +47,11 @@ KernelType GetExpectedKernelTyp() { In Python code ```python -def xx_layer(..., kernel_hint=None): - layer_helper = ... - layer_helper .append_op( +FORCE_CPU = core.kForceCPU() + +def xx_layer(..., force_cpu=false): + layer_helper = LayerHelper(...) + layer_helper.append_op( type="xx", - # "kernel_hint" should be the same with the attr name in CPP - attr={"kernel_hint": kernel_hint or ""}) + attr={FORCE_CPU: force_cpu}) ``` From a7bb983343a8fe9d8518a0b31388b67faf3f9320 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 20 Dec 2017 19:21:01 +0800 Subject: [PATCH 36/84] optimize indent --- doc/design/kernel_hint_design.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/design/kernel_hint_design.md b/doc/design/kernel_hint_design.md index da4856bb6a..a54b7da045 100644 --- a/doc/design/kernel_hint_design.md +++ b/doc/design/kernel_hint_design.md @@ -36,11 +36,11 @@ const std::string kUseCUDNN = "use_cudnn"; const std::string kUseMKLDNN = "use_mkldnn"; KernelType GetExpectedKernelType() { - if (Attr(kForceCPU)) { - return KernelType(CPUPlace, ...) - } else { - ... - } + if (Attr(kForceCPU)) { + return KernelType(CPUPlace, ...) + } else { + ... + } } ``` @@ -50,8 +50,8 @@ In Python code FORCE_CPU = core.kForceCPU() def xx_layer(..., force_cpu=false): - layer_helper = LayerHelper(...) - layer_helper.append_op( - type="xx", - attr={FORCE_CPU: force_cpu}) + layer_helper = LayerHelper(...) + layer_helper.append_op( + type="xx", + attr={FORCE_CPU: force_cpu}) ``` From e805cfcbf30f584abe2c0d8ae2ed06dc2fb23f98 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Wed, 20 Dec 2017 20:07:16 +0800 Subject: [PATCH 37/84] fix unit test failed --- paddle/operators/recv_op.cc | 6 ++-- paddle/operators/send_recv_op_test.cc | 29 +++++++++++-------- .../book/notest_recognize_digits_conv_dist.py | 14 +++++---- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 094084458e..7184858193 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -160,10 +160,12 @@ This operator will recv tensor from send_op "Serialized ProgramDesc string for recv to run."); AddAttr>( "ParamList", "type list of string", - "grad->param name mapping to find which param to optimize."); + "grad->param name mapping to find which param to optimize.") + .SetDefault({}); AddAttr>( "GradList", "type list of string", - "grad->param name mapping to find which param to optimize."); + "grad->param name mapping to find which param to optimize.") + .SetDefault({}); AddAttr("Trainers", "type int", "Number of trainers in the current cluster job") .SetDefault(1); diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index 3e2e2051af..1715b05c2c 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -16,12 +16,14 @@ // a RemoteOptimizer. #include +#include #include #include "gtest/gtest.h" #include "paddle/framework/op_registry.h" #include "paddle/framework/operator.h" #include "paddle/framework/program_desc.h" +#include "paddle/string/printf.h" USE_NO_KERNEL_OP(send); USE_NO_KERNEL_OP(recv); @@ -33,18 +35,21 @@ std::unique_ptr recv_op; void InitTensorsInScope(paddle::framework::Scope &scope, paddle::platform::CPUPlace &place) { paddle::platform::CPUDeviceContext ctx(place); - auto var = scope.Var("X"); - auto tensor = var->GetMutable(); - tensor->Resize({10, 10}); - float *expect = tensor->mutable_data(place); - for (int64_t i = 0; i < tensor->numel(); ++i) { - expect[i] = static_cast(i); + for (int i = 0; i < 2; ++i) { + auto var_name = paddle::string::Sprintf("x%d", i); + auto var = scope.Var(var_name); + auto tensor = var->GetMutable(); + tensor->Resize({10, 10}); + float *expect = tensor->mutable_data(place); + for (int64_t i = 0; i < tensor->numel(); ++i) { + expect[i] = static_cast(i); + } } auto out_var = scope.Var("Out"); auto out_tensor = out_var->GetMutable(); out_tensor->Resize({10, 10}); - tensor->mutable_data(place); // allocate + out_tensor->mutable_data(place); // allocate } void AddOp(const std::string &type, @@ -81,7 +86,7 @@ void StartServerNet() { paddle::framework::ProgramDescBind program; paddle::framework::BlockDescBind *block = program.MutableBlock(0); // X for server side tensors, RX for received tensers, must be of same shape. - AddOp("sum", {{"X", {"X", "RX"}}}, {{"Out", {"Out"}}}, {}, block); + AddOp("sum", {{"X", {"x0", "x1"}}}, {{"Out", {"Out"}}}, {}, block); paddle::framework::AttributeMap attrs; attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); @@ -89,8 +94,8 @@ void StartServerNet() { PADDLE_ENFORCE(program.Proto()->SerializeToString(&program_proto)); attrs.insert({"OptimizeProgram", program_proto}); - recv_op = paddle::framework::OpRegistry::CreateOp("recv", {{"RX", {"RX"}}}, - {{"Out", {"Out"}}}, attrs); + recv_op = paddle::framework::OpRegistry::CreateOp( + "recv", {{"RX", {"x0", "x1"}}}, {{"Out", {"Out"}}}, attrs); paddle::platform::CPUDeviceContext ctx(place); recv_op->Run(scope, ctx); } @@ -107,11 +112,11 @@ TEST(SendRecvOp, CPU) { attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); auto send_op = paddle::framework::OpRegistry::CreateOp( - "send", {{"X", {"X"}}}, {{"Out", {"Out"}}}, attrs); + "send", {{"X", {"x0", "x1"}}}, {{"Out", {"Out"}}}, attrs); paddle::platform::CPUDeviceContext ctx(place); send_op->Run(scope, ctx); - auto in_var = scope.Var("X"); + auto in_var = scope.Var("x0"); auto tensor = in_var->GetMutable(); float *expected = tensor->data(); diff --git a/python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py index c7f4f2212f..2680502efb 100644 --- a/python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book/notest_recognize_digits_conv_dist.py @@ -39,14 +39,16 @@ train_reader = paddle.batch( place = fluid.CPUPlace() exe = fluid.Executor(place) t = fluid.DistributeTranspiler() -t.transpile(optimize_ops, params_grads, pservers="127.0.0.1:6174", trainers=1) +pserver_endpoints = os.getenv("PSERVERS") +training_role = os.getenv("TRAINING_ROLE", + "TRAINER") # get the training role: trainer/pserver +t.transpile(optimize_ops, params_grads, pservers=pserver_endpoints, trainers=1) -pserver_endpoint = os.getenv("PSERVER") -if pserver_endpoint: - pserver_prog = t.get_pserver_program(pserver_endpoint, optimize_ops) +if training_role == "PSERVER": + pserver_prog = t.get_pserver_program(pserver_endpoints, optimize_ops) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) -else: +elif training_role == "TRAINER": feeder = fluid.DataFeeder(feed_list=[images, label], place=place) exe.run(fluid.default_startup_program()) @@ -64,5 +66,7 @@ else: pass_acc = accuracy.eval(exe) print("pass_id=" + str(pass_id) + " pass_acc=" + str(pass_acc)) +else: + print("environment var TRAINER_ROLE should be TRAINER os PSERVER") exit(1) From 5b52481058088da18c90c920bc815181badbf534 Mon Sep 17 00:00:00 2001 From: chengduo Date: Wed, 20 Dec 2017 20:34:47 +0800 Subject: [PATCH 38/84] refine accuracy_op.cu (#6774) --- paddle/operators/accuracy_op.cu | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/paddle/operators/accuracy_op.cu b/paddle/operators/accuracy_op.cu index 539a935302..dd51aad105 100644 --- a/paddle/operators/accuracy_op.cu +++ b/paddle/operators/accuracy_op.cu @@ -26,7 +26,7 @@ template __global__ void AccuracyCudaKernel(const int N, const int D, const int64_t* Xdata, const int64_t* labeldata, int* correct_data, - float* accuracy) { + float* accuracy, int* total_data) { int count = 0; __shared__ int total[BlockSize]; @@ -47,6 +47,7 @@ __global__ void AccuracyCudaKernel(const int N, const int D, if (threadIdx.x == 0) { *correct_data = result; *accuracy = static_cast(result) / static_cast(N); + *total_data = N; } } @@ -80,22 +81,11 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { if (num_samples == 0) { return; } - platform::GpuMemcpyAsync(total_data, &num_samples, sizeof(int), - cudaMemcpyHostToDevice, stream); AccuracyCudaKernel< PADDLE_CUDA_NUM_THREADS><<<1, PADDLE_CUDA_NUM_THREADS, 0, stream>>>( num_samples, infer_width, indices_data, label_data, correct_data, - accuracy_data); - - int d_num_samples, d_num_correct; - float d_accuracy; - platform::GpuMemcpyAsync(&d_num_correct, correct_data, sizeof(int), - cudaMemcpyDeviceToHost, stream); - platform::GpuMemcpyAsync(&d_num_samples, total_data, sizeof(int), - cudaMemcpyDeviceToHost, stream); - platform::GpuMemcpyAsync(&d_accuracy, accuracy_data, sizeof(float), - cudaMemcpyDeviceToHost, stream); + accuracy_data, total_data); } }; From c2b1ddb6a85c9a8f6b6f2a4d0ccde3acc863ca9b Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 20 Dec 2017 18:12:28 +0000 Subject: [PATCH 39/84] Correct the dropout_op's computation in test --- paddle/operators/dropout_op.cu | 2 +- paddle/operators/dropout_op.h | 2 +- python/paddle/v2/fluid/tests/test_dropout_op.py | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/paddle/operators/dropout_op.cu b/paddle/operators/dropout_op.cu index 10c670751d..c31d2195e9 100644 --- a/paddle/operators/dropout_op.cu +++ b/paddle/operators/dropout_op.cu @@ -71,7 +71,7 @@ class GPUDropoutKernel : public framework::OpKernel { auto M = EigenMatrix::Reshape(*mask, 1); Y.device(place) = X * M; } else { - Y.device(place) = X * dropout_prob; + Y.device(place) = X * (1.0f - dropout_prob); } } }; diff --git a/paddle/operators/dropout_op.h b/paddle/operators/dropout_op.h index 84ad39f0bb..9f6c4212d4 100644 --- a/paddle/operators/dropout_op.h +++ b/paddle/operators/dropout_op.h @@ -57,7 +57,7 @@ class CPUDropoutKernel : public framework::OpKernel { auto Y = EigenMatrix::Reshape(*y, 1); auto& place = *context.template device_context().eigen_device(); - Y.device(place) = X * dropout_prob; + Y.device(place) = X * (1.0f - dropout_prob); } } }; diff --git a/python/paddle/v2/fluid/tests/test_dropout_op.py b/python/paddle/v2/fluid/tests/test_dropout_op.py index 4f5ea836b4..2483200212 100644 --- a/python/paddle/v2/fluid/tests/test_dropout_op.py +++ b/python/paddle/v2/fluid/tests/test_dropout_op.py @@ -47,7 +47,9 @@ class TestDropoutOp4(OpTest): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} self.attrs = {'dropout_prob': 0.35, 'is_test': True} - self.outputs = {'Out': self.inputs['X'] * self.attrs['dropout_prob']} + self.outputs = { + 'Out': self.inputs['X'] * (1.0 - self.attrs['dropout_prob']) + } def test_check_output(self): self.check_output() @@ -58,7 +60,9 @@ class TestDropoutOp5(OpTest): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} self.attrs = {'dropout_prob': 0.75, 'is_test': True} - self.outputs = {'Out': self.inputs['X'] * self.attrs['dropout_prob']} + self.outputs = { + 'Out': self.inputs['X'] * (1.0 - self.attrs['dropout_prob']) + } def test_check_output(self): self.check_output() From aad8b223d63e640e68baab18def8e3131ff7802e Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Wed, 20 Dec 2017 14:14:34 -0800 Subject: [PATCH 40/84] Adding a proposal for operator documentation. (#6805) * Updating the design doc of Fluid * Organizing the operator documentation * Adding a proposed format for operator documentation * Adding more details to the format --- .../{ => op_documentation}/batch_norm_op.md | 0 .../{ => op_documentation}/name_convention.md | 0 .../{ => op_documentation}/net_op_design.md | 0 .../op_documentation/op_markdown_format.md | 64 +++++++++++++++++++ .../{ => op_documentation}/rnn_design.md | 0 5 files changed, 64 insertions(+) rename paddle/operators/{ => op_documentation}/batch_norm_op.md (100%) rename paddle/operators/{ => op_documentation}/name_convention.md (100%) rename paddle/operators/{ => op_documentation}/net_op_design.md (100%) create mode 100644 paddle/operators/op_documentation/op_markdown_format.md rename paddle/operators/{ => op_documentation}/rnn_design.md (100%) diff --git a/paddle/operators/batch_norm_op.md b/paddle/operators/op_documentation/batch_norm_op.md similarity index 100% rename from paddle/operators/batch_norm_op.md rename to paddle/operators/op_documentation/batch_norm_op.md diff --git a/paddle/operators/name_convention.md b/paddle/operators/op_documentation/name_convention.md similarity index 100% rename from paddle/operators/name_convention.md rename to paddle/operators/op_documentation/name_convention.md diff --git a/paddle/operators/net_op_design.md b/paddle/operators/op_documentation/net_op_design.md similarity index 100% rename from paddle/operators/net_op_design.md rename to paddle/operators/op_documentation/net_op_design.md diff --git a/paddle/operators/op_documentation/op_markdown_format.md b/paddle/operators/op_documentation/op_markdown_format.md new file mode 100644 index 0000000000..0ee804d592 --- /dev/null +++ b/paddle/operators/op_documentation/op_markdown_format.md @@ -0,0 +1,64 @@ +# Standard Markdown Format for Operators +The following should be the standard format for documentation for all the operators that will get rendered in the `html`: + +``` +Operator Name (In PaddlePaddle) + +Operator Name (Standard) + +Operator description. + +LaTeX equation of how the operator performs an update. + +The signature of the operator. +``` + +Each section mentioned above has been covered in further detail in the rest of the document. + +# PaddlePaddle Operator Name +This should be in all small letters, in case of multiple words, we separate them with an underscore. For example: +`array to lod tensor` should be written as `array_to_lod_tensor`. + +This naming convention should be standard across all PaddlePaddle operators. + +# Standard Operator Name +This is the standard name of the operator as used in the community. The general standard is usually: +- Standard abbreviations like `SGD` are written in all capital letters. +- Operator names that have multiple words inside a single word use `camelCase` (capitalize word boundaries inside of a word). +- Keep numbers inside a word as is, with no boundary delimiters. +- Follow the name of the operator with the keyword: `Activation Operator.` + +# Operator description +This section should contain the description of what the operator does, including the operation performed, the literature from where it comes and was introduced first, and other important details. The relevant paper/article including the hyperlink should be cited in this section. + +# LaTeX equation +This section should contain an overall equation of the update or operation that the operator performs. The variables used in the equation should follow the naming convention of operators as described [here](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/name_convention.md). Two words in the same word should be separated by an underscore (`_`). + +# The signature +This section describes the signature of the operator. A list of Inputs and Outputs, each of which have a small description of what the variable represents and the type of variable. The variable names follow the `CamelCase` naming convention. The proposed format for this is: +`Section : +VariableName : (VariableType) VariableDescription +... +... +` + + +The following example for an `sgd` operator covers the above mentioned sections as they would ideally look like in the `html`: + +``` +sgd + +SGD operator + +This operator implements one step of the stochastic gradient descent algorithm. + +param_out = param_learning_rate * grad + +Inputs: +Param : (Tensor) Input parameter +LearningRate : (Tensor) Learning rate of SGD +Grad : (Tensor) Input gradient + +Outputs: +ParamOut : (Tensor) Output parameter +``` diff --git a/paddle/operators/rnn_design.md b/paddle/operators/op_documentation/rnn_design.md similarity index 100% rename from paddle/operators/rnn_design.md rename to paddle/operators/op_documentation/rnn_design.md From c8ef45291d8a23bbcab0e292df3eb876fc7977a9 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 20 Dec 2017 17:17:01 -0800 Subject: [PATCH 41/84] Polishing the embedding layer and the fc layer documentation (#6806) * Polishing the embedding layer and the fc layer documentation * Addressing code review feedback --- python/paddle/v2/fluid/layers/nn.py | 99 +++++++++++++++++------------ 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 73f68466da..8d819de603 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -25,32 +25,48 @@ def fc(input, act=None, name=None): """ - Fully Connected Layer. + **Fully Connected Layer** + + This layer accepts multiple inputs and applies a linear transformation to each input. + If activation type is provided, the corresponding activation function is applied to the + output of the linear transformation. For each input :math:`X`, the equation is: + + .. math:: + + Out = Act(WX + b) + + In the above equation: + + * :math:`X`: Input value, a tensor with rank at least 2. + * :math:`W`: Weight, a 2-D tensor with shape [M, N]. + * :math:`b`: Bias, a 2-D tensor with shape [M, 1]. + * :math:`Act`: Activation function. + * :math:`Out`: Output value, same shape with :math:`X`. + + All the input variables are passed in as local variables to the LayerHelper + constructor. Args: - input: The input tensor to the function - size: The size of the layer - num_flatten_dims: Number of columns in input - param_attr: The parameters/weights to the FC Layer - param_initializer: Initializer used for the weight/parameter. If None, XavierInitializer() is used - bias_attr: The bias parameter for the FC layer - bias_initializer: Initializer used for the bias. If None, then ConstantInitializer() is used - act: Activation to be applied to the output of FC layer - name: Name/alias of the function - main_program: Name of the main program that calls this - startup_program: Name of the startup program - - This function can take in multiple inputs and performs the Fully Connected - function (linear transformation) on top of each of them. - So for input x, the output will be : Wx + b. Where W is the parameter, - b the bias and x is the input. - - The function also applies an activation (non-linearity) on top of the - output, if activation is passed in the input. - - All the input variables of this function are passed in as local variables - to the LayerHelper constructor. + input(Variable|list): Input tensors. Each tensor has a rank of atleast 2 + size(int): Output size + num_flatten_dims(int): Number of columns in input + param_attr(ParamAttr|list): The parameters/weights to the FC Layer + bias_attr(ParamAttr|list): Bias parameter for the FC layer + act(str): Activation type + name(str): Name/alias of the function + + Returns: + Variable: The tensor variable storing the transformation and \ + non-linearity activation result. + + Raises: + ValueError: If rank of input tensor is less than 2. + Examples: + .. code-block:: python + + data = fluid.layers.data(name='data', shape=[32, 32], dtype='float32') + fc = fluid.layers.fc(input=data, size=1000, act="tanh") """ helper = LayerHelper('fc', **locals()) @@ -91,25 +107,30 @@ def fc(input, def embedding(input, size, is_sparse=False, param_attr=None, dtype='float32'): """ - Embedding Layer. + **Embedding Layer** + + This layer is used to lookup a vector of IDs, provided by *input*, in a lookup table. + The result of this lookup is the embedding of each ID in the *input*. + + All the input variables are passed in as local variables to the LayerHelper + constructor. Args: - param_initializer: - input: The input to the function - size: The size of the layer - is_sparse: A flag that decleares whether the input is sparse - param_attr: Parameters for this layer - dtype: The type of data : float32, float_16, int etc - main_program: Name of the main program that calls this - startup_program: Name of the startup program - - This function can take in the input (which is a vector of IDs) and - performs a lookup in the lookup_table using these IDs, to result into - the embedding of each ID in the input. - - All the input variables of this function are passed in as local variables - to the LayerHelper constructor. + input(Variable): Input to the function + size(int): Output size + is_sparse(bool): Boolean flag that specifying whether the input is sparse + param_attr(ParamAttr): Parameters for this layer + dtype(np.dtype|core.DataType|str): The type of data : float32, float_16, int etc + + Returns: + Variable: The tensor variable storing the embeddings of the \ + supplied inputs. + + Examples: + .. code-block:: python + data = fluid.layers.data(name='ids', shape=[32, 32], dtype='float32') + fc = fluid.layers.embedding(input=data, size=16) """ helper = LayerHelper('embedding', **locals()) From ad9790891bf804b52cc693630608387500ed5672 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 20 Dec 2017 19:33:39 -0800 Subject: [PATCH 42/84] Polish layer documentation for fill_constant ops (#6808) --- python/paddle/v2/fluid/layers/tensor.py | 48 +++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/layers/tensor.py b/python/paddle/v2/fluid/layers/tensor.py index bda017b141..e984a6be19 100644 --- a/python/paddle/v2/fluid/layers/tensor.py +++ b/python/paddle/v2/fluid/layers/tensor.py @@ -66,9 +66,26 @@ def assign(input, output): def fill_constant(shape, dtype, value, out=None): """ - This function creates a tensor , with shape as mentioned in the input and - specified dtype and fills this up with a constant value that - comes in the input. It also sets the stop_gradient to be True. + **fill_constant** + + This function creates a tensor of specified *shape* and + *dtype*, and initializes this with a constant supplied in *value*. + + It also sets *stop_gradient* to True. + + Args: + shape(tuple|list|None): Shape of output tensor + dtype(np.dtype|core.DataType|str): Data type of output tensor + value(float): Constant value to initialize the output tensor + out(Variable): Output Variable to initialize + + Returns: + Variable: The tensor variable storing the output + + Examples: + .. code-block:: python + + data = fluid.layers.fill_constant(shape=[1], value=0, dtype='int64') """ helper = LayerHelper("fill_constant", **locals()) if out is None: @@ -90,6 +107,31 @@ def fill_constant_batch_size_like(input, value, input_dim_idx=0, output_dim_idx=0): + """ + **fill_constant_batch_size_like** + + This function creates a tensor of specified *shape*, *dtype* and batch size, + and initializes this with a constant supplied in *value*. The batch size is + obtained from the `input` tensor. + + It also sets *stop_gradient* to True. + + Args: + input(Variable): Tensor whose dimensions will be used to get batch size + shape(tuple|list|None): Shape of output tensor + dtype(np.dtype|core.DataType|str): Data type of output tensor + value(float): Constant value to initialize the output tensor + input_dim_idx(int): Index of input's batch size dimension + output_dim_idx(int): Index of output's batch size dimension + + Returns: + Variable: The tensor variable storing the output + + Examples: + .. code-block:: python + + data = fluid.layers.fill_constant(shape=[1], value=0, dtype='int64') + """ helper = LayerHelper("fill_constant_batch_size_like", **locals()) out = helper.create_tmp_variable(dtype=dtype) helper.append_op( From f04f4f9aee7c223defe060f77b8abdafd4c90357 Mon Sep 17 00:00:00 2001 From: whs Date: Thu, 21 Dec 2017 11:49:45 +0800 Subject: [PATCH 43/84] Fix equation of sequence_softmax_op. (#6810) --- paddle/operators/sequence_softmax_op.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/paddle/operators/sequence_softmax_op.cc b/paddle/operators/sequence_softmax_op.cc index fe1832a36f..b74766f012 100644 --- a/paddle/operators/sequence_softmax_op.cc +++ b/paddle/operators/sequence_softmax_op.cc @@ -50,10 +50,14 @@ input Tensor can be either [N, 1] or [N], where N is the sum of the length of all sequences. The algorithm works as follows: + for i-th sequence in a mini-batch: - $$Out(X[lod[i]:lod[i+1]], :) = - \frac{\exp(X[lod[i]:lod[i+1], :])} - {\sum(\exp(X[lod[i]:lod[i+1], :]))}$$ + +$$ +Out(X[lod[i]:lod[i+1]], :) = \ +\frac{\exp(X[lod[i]:lod[i+1], :])} \ +{\sum(\exp(X[lod[i]:lod[i+1], :]))} +$$ For example, for a mini-batch of 3 sequences with variable-length, each containing 2, 3, 2 time-steps, the lod of which is [0, 2, 5, 7], From b23982a2de4c53154f37f5e7e572e57c67c29687 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 21 Dec 2017 12:27:31 +0800 Subject: [PATCH 44/84] Add ReorderLoDTensorByRank It is useful to reorder RNN memory block. --- .../reorder_lod_tensor_by_rank_op.cc | 225 ++++++++++++++++++ python/paddle/v2/fluid/framework.py | 5 +- python/paddle/v2/fluid/layer_helper.py | 6 + python/paddle/v2/fluid/layers/control_flow.py | 25 +- python/paddle/v2/fluid/tests/__init__.py | 0 .../v2/fluid/tests/test_reorder_lod_tensor.py | 47 ++++ 6 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 paddle/operators/reorder_lod_tensor_by_rank_op.cc create mode 100644 python/paddle/v2/fluid/tests/__init__.py create mode 100644 python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py diff --git a/paddle/operators/reorder_lod_tensor_by_rank_op.cc b/paddle/operators/reorder_lod_tensor_by_rank_op.cc new file mode 100644 index 0000000000..384047428d --- /dev/null +++ b/paddle/operators/reorder_lod_tensor_by_rank_op.cc @@ -0,0 +1,225 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include +#include "paddle/framework/op_registry.h" +#include "paddle/operators/detail/safe_ref.h" + +namespace paddle { +namespace operators { + +class ReorderLoDTensorProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ReorderLoDTensorProtoMaker(OpProto *proto, OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(LoDTensor) the input lod tensor need to be reordered."); + AddInput("RankTable", + "(LoDRankTable) the rank table that input need follow"); + AddOutput("Out", "(LoDTensor) reordered lod tensor"); + AddComment(R"DOC(ReorderLoDTensorLoDRankTable + +Reorder the input X by the rank of `RankTable`. If `RankTable` is ordered by +index [3, 0, 2, 1]. Input X will reorder its sequence, the third sequence of +X will be the first sequence of Output. + +NOTE: The RankTable does not need to be calculated by X. +)DOC"); + } +}; + +class ReorderLoDTensorByRankTableBase : public framework::OperatorBase { + public: + ReorderLoDTensorByRankTableBase(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = + detail::Ref(scope.FindVar(Input("X")), + "Cannot find input lod tensor variable %s", Input("X")) + .Get(); + auto &rank_table = detail::Ref(scope.FindVar(Input("RankTable")), + "Cannot find input rank table variable %s", + Input("RankTable")) + .Get(); + auto &out = + *detail::Ref(scope.FindVar(Output("Out")), + "Cannot find output lod tensor variable %s", Output("Out")) + .GetMutable(); + + out.Resize(x.dims()); + out.mutable_data(x.place(), x.type()); + this->process(dev_ctx, x, rank_table, &out); + } + + protected: + virtual void process(const platform::DeviceContext &dev_ctx, + const framework::LoDTensor &x, + const framework::LoDRankTable &rank_table, + framework::LoDTensor *out) const = 0; + + struct AbsoluteRankTableItem { + size_t offset; // the absolute/accumulated offset. + size_t length; // the length + framework::LoD lod; + }; + + std::vector GetAbsoluteOffsetAndLengthByLoDRankTable( + const framework::LoDTensor &x) const { + std::vector absolute_table; + size_t level = 0; + size_t size = x.lod()[level].size(); + + for (size_t i = 0; i < size - 1; ++i) { + auto lod_offset = + framework::GetSubLoDAndAbsoluteOffset(x.lod(), i, i + 1, level); + + auto &offset = lod_offset.second; + + absolute_table.emplace_back(); + absolute_table.back().length = offset.second - offset.first; + absolute_table.back().offset = offset.first; + absolute_table.back().lod = lod_offset.first; + } + return absolute_table; + } + + size_t CopyTensorAndLod(const platform::DeviceContext &dev_ctx, + const AbsoluteRankTableItem &item, + const framework::LoDTensor &x, + framework::LoDTensor *out, size_t out_offset) const { + auto &out_lod = *out->mutable_lod(); + auto len = item.length; + auto x_offset = item.offset; + + if (out_lod.empty()) { + for (size_t i = 0; i < item.lod.size(); ++i) { + out_lod.push_back(std::vector({0})); + } + } + + for (size_t i = 0; i < out_lod.size(); ++i) { + auto &out_v = out_lod[i]; + auto &new_lod_v = item.lod[i]; + + for (auto &detail : new_lod_v) { + out_v.push_back(out_v.back() + detail); + } + } + + auto x_sliced = x.Slice(x_offset, x_offset + len); + auto out_sliced = out->Slice(out_offset, out_offset + len); + + framework::CopyFrom(x_sliced, out_sliced.place(), dev_ctx, &out_sliced); + out_offset += len; + return out_offset; + } +}; + +class ReorderLoDTensorByRankTableOp : public ReorderLoDTensorByRankTableBase { + public: + ReorderLoDTensorByRankTableOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ReorderLoDTensorByRankTableBase(type, inputs, outputs, attrs) {} + + protected: + void process(const platform::DeviceContext &dev_ctx, + const framework::LoDTensor &x, + const framework::LoDRankTable &rank_table, + framework::LoDTensor *out) const override { + auto absolute_table = GetAbsoluteOffsetAndLengthByLoDRankTable(x); + size_t out_offset = 0; + out->mutable_lod()->clear(); + for (auto &item : rank_table.items()) { + out_offset = this->CopyTensorAndLod(dev_ctx, absolute_table[item.index], + x, out, out_offset); + } + } +}; + +class IdentityInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + context->SetOutputDim("Out", context->GetInputDim("X")); + } +}; + +class ReorderLodTensorByRankGradOpMaker + : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("reorder_lod_tensor_by_rank_grad"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetInput("RankTable", Input("RankTable")); + return std::unique_ptr(grad_op); + } +}; + +class ReorderLoDTensorByRankGradOp : public ReorderLoDTensorByRankTableBase { + public: + ReorderLoDTensorByRankGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ReorderLoDTensorByRankTableBase(type, inputs, outputs, attrs) {} + + protected: + void process(const platform::DeviceContext &dev_ctx, + const framework::LoDTensor &x, + const framework::LoDRankTable &rank_table, + framework::LoDTensor *out) const override { + auto absolute_table = GetAbsoluteOffsetAndLengthByLoDRankTable(x); + + // offsets = enumerate([item.index for item in rank_table.items()]) + std::vector> offsets; + offsets.reserve(rank_table.items().size()); + for (size_t i = 0; i < rank_table.items().size(); ++i) { + offsets.push_back({i, rank_table.items()[i].index}); + } + + // offsets.sort(key=lambda x: x[1]) + std::sort( + offsets.begin(), offsets.end(), + [](const std::pair &a, + const std::pair &b) { return a.second < b.second; }); + + // Copy TensorAndLod + size_t out_offset = 0; + for (auto &offset : offsets) { + out_offset = this->CopyTensorAndLod(dev_ctx, absolute_table[offset.first], + x, out, out_offset); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OPERATOR(reorder_lod_tensor_by_rank, + ops::ReorderLoDTensorByRankTableOp, + ops::ReorderLodTensorByRankGradOpMaker, + ops::ReorderLoDTensorProtoMaker, ops::IdentityInferShape); +REGISTER_OPERATOR(reorder_lod_tensor_by_rank_grad, + ops::ReorderLoDTensorByRankGradOp, ops::IdentityInferShape); diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index d1b12a8f09..9ccb1f8d6e 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -389,7 +389,10 @@ class Operator(object): % (in_proto.name, len(in_args))) in_arg_names = [] for arg in in_args: - in_arg_names.append(arg.name) + if isinstance(arg, basestring): + in_arg_names.append(arg) + else: + in_arg_names.append(arg.name) self.desc.set_input(in_proto.name, in_arg_names) else: self.desc.set_input(in_proto.name, []) diff --git a/python/paddle/v2/fluid/layer_helper.py b/python/paddle/v2/fluid/layer_helper.py index 8df30ad76b..a076f26f7f 100644 --- a/python/paddle/v2/fluid/layer_helper.py +++ b/python/paddle/v2/fluid/layer_helper.py @@ -194,3 +194,9 @@ class LayerHelper(object): else: # For integer and boolean types, initialize with all zeros return Constant() + + def is_instance(self, param_name, cls): + param = self.kwargs.get(param_name, None) + if not isinstance(param, cls): + raise TypeError("The input {0} parameter of method {1} must be {2}", + param_name, self.layer_type, cls.__name__) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index dc6c0e7f51..f22dfb4c85 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -10,7 +10,7 @@ __all__ = [ 'max_sequence_len', 'topk', 'lod_tensor_to_array', 'array_to_lod_tensor', 'increment', 'array_write', 'create_array', 'less_than', 'array_read', 'shrink_memory', 'array_length', 'IfElse', 'DynamicRNN', 'ConditionalBlock', - 'StaticRNN' + 'StaticRNN', 'reorder_lod_tensor_by_rank' ] @@ -963,3 +963,26 @@ class DynamicRNN(object): if self.status != DynamicRNN.IN_RNN: raise ValueError("{0} can only be invoked inside rnn block.".format( method)) + + +def reorder_lod_tensor_by_rank(x, rank_table): + """ + + Args: + x(Variable): + rank_table(Variable): + + Returns: + + """ + helper = LayerHelper('reorder_lod_tensor_by_rank', **locals()) + helper.is_instance('x', Variable) + helper.is_instance('rank_table', Variable) + + out = helper.create_tmp_variable(dtype=x.dtype) + helper.append_op( + type='reorder_lod_tensor_by_rank', + inputs={'X': [x], + 'RankTable': [rank_table]}, + outputs={'Out': [out]}) + return out diff --git a/python/paddle/v2/fluid/tests/__init__.py b/python/paddle/v2/fluid/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py b/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py new file mode 100644 index 0000000000..8f5774835e --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py @@ -0,0 +1,47 @@ +import unittest +import paddle.v2.fluid as fluid +import numpy + + +class TestReorderLoDTensor(unittest.TestCase): + def test_reorder(self): + dat = fluid.layers.data(name='input', shape=[1], lod_level=2) + dat.stop_gradient = False + rank_dat = fluid.layers.data(name='ref', shape=[1], lod_level=1) + table = fluid.layers.lod_rank_table(rank_dat) + new_dat = fluid.layers.reorder_lod_tensor_by_rank( + x=dat, rank_table=table) + loss = fluid.layers.mean(x=new_dat) + fluid.backward.append_backward_ops(loss=loss) + + cpu = fluid.CPUPlace() + exe = fluid.Executor(cpu) + exe.run(fluid.default_startup_program()) + + ref = fluid.Tensor() + ref_lod = [0, 3, 4, 7, 8, 14] + ref.set_lod([ref_lod]) + + ref.set(numpy.random.random(size=[14, 1]).astype('float32'), cpu) + input = fluid.Tensor() + lod_level_0 = numpy.random.randint(low=1, high=5, size=5) + lod_level_0 = [0] + numpy.cumsum(lod_level_0).tolist() + lod_level_1 = numpy.random.randint(low=1, high=5, size=lod_level_0[-1]) + lod_level_1 = [0] + numpy.cumsum(lod_level_1).tolist() + + input.set_lod([lod_level_0, lod_level_1]) + input.set( + numpy.random.random(size=[lod_level_1[-1], 1]).astype('float32'), + cpu) + + ig = exe.run(fluid.default_main_program(), + feed={'input': input, + 'ref': ref}, + fetch_list=['input@GRAD'], + return_numpy=False)[0] + self.assertAlmostEqual(numpy.array(ig).sum(), 1.0, delta=0.001) + self.assertEqual(input.lod(), ig.lod()) + + +if __name__ == '__main__': + unittest.main() From ad2ab952075ac7d0ff59434b353ce5cba5d35563 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 21 Dec 2017 12:33:34 +0800 Subject: [PATCH 45/84] "small fix of Place" (#6766) --- paddle/platform/device_context.cc | 4 ++-- paddle/platform/device_context.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index 8cdc5f4340..dacee74fff 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -19,7 +19,7 @@ CPUDeviceContext::CPUDeviceContext() { eigen_device_.reset(new Eigen::DefaultDevice()); } -CPUDeviceContext::CPUDeviceContext(CPUPlace place) { +CPUDeviceContext::CPUDeviceContext(CPUPlace place) : place_(place) { eigen_device_.reset(new Eigen::DefaultDevice()); } @@ -27,7 +27,7 @@ Eigen::DefaultDevice* CPUDeviceContext::eigen_device() const { return eigen_device_.get(); } -Place CPUDeviceContext::GetPlace() const { return CPUPlace(); } +Place CPUDeviceContext::GetPlace() const { return place_; } #ifdef PADDLE_WITH_CUDA diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index 56813a1d5b..6cc0508522 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -45,6 +45,7 @@ class CPUDeviceContext : public DeviceContext { Place GetPlace() const override; private: + CPUPlace place_; std::unique_ptr eigen_device_; }; From 863661a30bd8ddb03bed6d8c07912fc8a02aae92 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 20 Dec 2017 21:46:48 -0800 Subject: [PATCH 46/84] Polishing the documentation of the less than layer (#6816) --- python/paddle/v2/fluid/layers/control_flow.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index dc6c0e7f51..7ed79968b1 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -519,6 +519,24 @@ def create_array(dtype): def less_than(x, y, cond=None, **ignored): + """ + **Less than** + + This layer returns the truth value of :math:`x < y` elementwise. + + Args: + x(Variable): First operand of *less_than* + y(Variable): Second operand of *less_than* + cond(Variable|None): Optional output variable to store the result of *less_than* + + Returns: + Variable: The tensor variable storing the output of *less_than*. + + Examples: + .. code-block:: python + + less = fluid.layers.less_than(x=label, y=limit) + """ helper = LayerHelper("less_than", **locals()) if cond is None: cond = helper.create_tmp_variable(dtype='bool') From 0295b0006699c9b7e3d4525ad67d55a778e5d32c Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Thu, 21 Dec 2017 14:47:47 +0800 Subject: [PATCH 47/84] Add libprotobuf-lite.a when install. (#6340) * Add libprotobuf-lite.a when install. * Fix protobuf.cmake * Bug fix --- cmake/external/protobuf.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index fab2af362b..ff5855052d 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -253,9 +253,9 @@ IF(NOT PROTOBUF_FOUND) IF(WITH_C_API) INSTALL(DIRECTORY ${PROTOBUF_INCLUDE_DIR} DESTINATION third_party/protobuf) IF(ANDROID) - INSTALL(FILES ${PROTOBUF_LIBRARY} DESTINATION third_party/protobuf/lib/${ANDROID_ABI}) + INSTALL(FILES ${PROTOBUF_LITE_LIBRARY} DESTINATION third_party/protobuf/lib/${ANDROID_ABI}) ELSE() - INSTALL(FILES ${PROTOBUF_LIBRARY} DESTINATION third_party/protobuf/lib) + INSTALL(FILES ${PROTOBUF_LITE_LIBRARY} DESTINATION third_party/protobuf/lib) ENDIF() ENDIF() From 091897321f2b78eb80bba5e1adee170e6c1dcfac Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 21 Dec 2017 15:25:47 +0800 Subject: [PATCH 48/84] Rename XXDescBind --> XXDesc (#6797) * Rename XXDescBind --> XXDesc * Fix Compile --- paddle/framework/backward.cc | 88 ++++++------ paddle/framework/backward.h | 2 +- paddle/framework/backward_test.cc | 126 +++++++++--------- paddle/framework/block_desc.cc | 59 ++++---- paddle/framework/block_desc.h | 43 +++--- paddle/framework/details/op_registry.h | 6 +- paddle/framework/executor.cc | 2 +- paddle/framework/executor.h | 2 +- paddle/framework/grad_op_desc_maker.h | 26 ++-- paddle/framework/op_desc.cc | 78 +++++------ paddle/framework/op_desc.h | 20 +-- paddle/framework/op_registry.cc | 4 +- paddle/framework/op_registry.h | 2 +- paddle/framework/program_desc.cc | 22 +-- paddle/framework/program_desc.h | 20 +-- paddle/framework/program_desc_test.cc | 12 +- paddle/framework/prune_test.cc | 22 +-- paddle/framework/type_defs.h | 18 ++- paddle/framework/var_desc.cc | 22 ++- paddle/framework/var_desc.h | 6 +- paddle/framework/var_type_inference.h | 3 +- paddle/framework/var_type_inference_test.cc | 7 +- paddle/operators/array_to_lod_tensor_op.cc | 6 +- paddle/operators/assign_op.cc | 6 +- paddle/operators/beam_search_decode_op.cc | 4 +- paddle/operators/cast_op.cc | 6 +- paddle/operators/conditional_block_op.cc | 12 +- paddle/operators/increment_op.cc | 6 +- paddle/operators/lod_rank_table_op.cc | 4 +- paddle/operators/lod_tensor_to_array_op.cc | 10 +- paddle/operators/lookup_table_op.cc | 4 +- paddle/operators/mean_op.cc | 6 +- paddle/operators/merge_lod_tensor_op.cc | 6 +- paddle/operators/minus_op.cc | 9 +- paddle/operators/nccl_op_test.cu.cc | 15 +-- paddle/operators/pad_op.cc | 6 +- paddle/operators/recurrent_op.cc | 13 +- paddle/operators/scale_op.cc | 6 +- paddle/operators/shrink_rnn_memory_op.cc | 6 +- paddle/operators/sign_op.cc | 6 +- .../softmax_with_cross_entropy_op.cc | 6 +- paddle/operators/split_lod_tensor_op.cc | 6 +- paddle/operators/split_op.cc | 6 +- paddle/operators/sum_op.cc | 13 +- .../operators/tensor_array_read_write_op.cc | 16 +-- paddle/operators/while_op.cc | 18 +-- paddle/pybind/protobuf.cc | 113 ++++++++-------- paddle/pybind/pybind.cc | 20 +-- 48 files changed, 447 insertions(+), 472 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index f1a577325f..76e9131638 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -42,7 +42,7 @@ static std::unordered_set& CtrlFlowOps() { static inline std::unique_ptr CreateGradOp( const OperatorBase& op, const std::unordered_set& no_grad_set, std::unordered_map* grad_to_var) { - OpDescBind op_desc; + OpDesc op_desc; op_desc.SetInputMap(op.Inputs()); op_desc.SetOutputMap(op.Outputs()); op_desc.SetType(op.Type()); @@ -53,7 +53,7 @@ static inline std::unique_ptr CreateGradOp( grad_ops.reserve(grad_descs.size()); std::transform(grad_descs.begin(), grad_descs.end(), std::back_inserter(grad_ops), - [](const std::unique_ptr& grad_desc) { + [](const std::unique_ptr& grad_desc) { return OpRegistry::CreateOp(*grad_desc); }); PADDLE_ENFORCE(!grad_ops.empty()); @@ -296,7 +296,7 @@ static std::string FwdName(const std::string& grad_name) { static void CreateGradVarInBlock( size_t grad_op_start_index, const std::unordered_map& param_name_map, - BlockDescBind* block_desc, + BlockDesc* block_desc, std::unordered_map* grad_var_record) { auto ops = block_desc->AllOps(); for (size_t op_index = grad_op_start_index; op_index < ops.size(); @@ -350,12 +350,11 @@ static void CreateGradVarInBlock( } } -std::vector> MakeOpGrad( - const OpDescBind* op_desc, std::unordered_set* no_grad_vars, +std::vector> MakeOpGrad( + const OpDesc* op_desc, std::unordered_set* no_grad_vars, std::unordered_map* grad_to_var, - const std::vector& grad_block = - std::vector()) { - std::vector> grad_op_descs; + const std::vector& grad_block = std::vector()) { + std::vector> grad_op_descs; // All input gradients of forwarding operator do not need to calculate. const std::vector& inputs = op_desc->InputArgumentNames(); if (AllGradInSet(inputs, *no_grad_vars)) { @@ -386,7 +385,7 @@ std::vector> MakeOpGrad( .Get(op_desc->Type()) .GradOpMaker()(*op_desc, *no_grad_vars, grad_to_var, grad_block); - std::list> pending_fill_zeros_ops; + std::list> pending_fill_zeros_ops; for (auto& desc : grad_op_descs) { for (const std::string& in_name : desc->InputArgumentNames()) { if (no_grad_vars->count(in_name)) { @@ -394,9 +393,9 @@ std::vector> MakeOpGrad( 0, in_name.size() - sizeof(kGradVarSuffix) / sizeof(char) + 1); std::string new_name = prefix + kZeroVarSuffix; desc->Rename(in_name, new_name); - std::unique_ptr fill_zeros_op( - new OpDescBind("fill_zeros_like", {{"X", {prefix}}}, - {{"Y", {new_name}}}, AttributeMap{})); + std::unique_ptr fill_zeros_op( + new OpDesc("fill_zeros_like", {{"X", {prefix}}}, + {{"Y", {new_name}}}, AttributeMap{})); pending_fill_zeros_ops.push_back(std::move(fill_zeros_op)); } } @@ -408,34 +407,33 @@ std::vector> MakeOpGrad( return grad_op_descs; } -static BlockDescBind* CreateStepBlock( - ProgramDescBind& program_desc, - std::unordered_set* no_grad_vars, +static BlockDesc* CreateStepBlock( + ProgramDesc& program_desc, std::unordered_set* no_grad_vars, std::unordered_map* grad_to_var, int step_block_idx); -std::vector> MakeBlockBackward( - ProgramDescBind& program_desc, int block_idx, +std::vector> MakeBlockBackward( + ProgramDesc& program_desc, int block_idx, std::unordered_set* no_grad_vars, std::unordered_map* grad_to_var) { VLOG(5) << "MakeBlockBackward"; - BlockDescBind* cur_block = program_desc.MutableBlock(block_idx); - std::vector op_descs = cur_block->AllOps(); + BlockDesc* cur_block = program_desc.MutableBlock(block_idx); + std::vector op_descs = cur_block->AllOps(); std::unordered_map> dup_out_ops; size_t grad_desc_idx = 0; - std::vector> backward_descs; + std::vector> backward_descs; for (auto it = op_descs.rbegin(); it != op_descs.rend(); ++it) { VLOG(5) << "Making backward " << (*it)->Type() << " op"; - std::vector> op_grads; + std::vector> op_grads; if ((*it)->Type() == "recurrent" || (*it)->Type() == "while") { int step_block_idx = (*it)->GetBlockAttr("sub_block"); - BlockDescBind* backward_block = CreateStepBlock( - program_desc, no_grad_vars, grad_to_var, step_block_idx); + BlockDesc* backward_block = CreateStepBlock(program_desc, no_grad_vars, + grad_to_var, step_block_idx); op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); } else if ((*it)->Type() == "conditional_block") { - BlockDescBind* backward_block = + BlockDesc* backward_block = CreateStepBlock(program_desc, no_grad_vars, grad_to_var, (*it)->GetBlockAttr("sub_block")); op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); @@ -463,14 +461,14 @@ std::vector> MakeBlockBackward( } ++grad_desc_idx; } - std::transform( - op_grads.begin(), op_grads.end(), std::back_inserter(backward_descs), - [](std::unique_ptr& ptr) { return std::move(ptr); }); + std::transform(op_grads.begin(), op_grads.end(), + std::back_inserter(backward_descs), + [](std::unique_ptr& ptr) { return std::move(ptr); }); } VLOG(5) << "Appending Sums"; // Check whether some variables are written more than once - std::list>> pending_sum_ops; + std::list>> pending_sum_ops; for (const auto& dup : dup_out_ops) { const std::string& out_name = dup.first; const std::vector dup_op = dup.second; @@ -486,18 +484,17 @@ std::vector> MakeBlockBackward( sum_op_inputs.emplace_back(new_name); next_g_name = sum_op_inputs.back(); } - std::unique_ptr sum_op( - new OpDescBind("sum", {{"X", sum_op_inputs}}, {{"Out", {out_name}}}, - AttributeMap{})); + std::unique_ptr sum_op(new OpDesc("sum", {{"X", sum_op_inputs}}, + {{"Out", {out_name}}}, + AttributeMap{})); pending_sum_ops.push_back({dup_op.back(), std::move(sum_op)}); } } - pending_sum_ops.sort( - [](const std::pair>& a, - const std::pair>& b) { - return a.first > b.first; - }); + pending_sum_ops.sort([](const std::pair>& a, + const std::pair>& b) { + return a.first > b.first; + }); for (auto& p : pending_sum_ops) { backward_descs.insert(backward_descs.begin() + p.first + 1, std::move(p.second)); @@ -508,14 +505,13 @@ std::vector> MakeBlockBackward( return backward_descs; } -static BlockDescBind* CreateStepBlock( - ProgramDescBind& program_desc, - std::unordered_set* no_grad_vars, +static BlockDesc* CreateStepBlock( + ProgramDesc& program_desc, std::unordered_set* no_grad_vars, std::unordered_map* grad_to_var, int step_block_idx) { auto backward_block_op_descs = MakeBlockBackward(program_desc, step_block_idx, no_grad_vars, grad_to_var); - BlockDescBind* backward_block = + BlockDesc* backward_block = program_desc.AppendBlock(*program_desc.MutableBlock(step_block_idx)); for (auto& ptr : backward_block_op_descs) { backward_block->AppendAllocatedOp(move(ptr)); @@ -524,7 +520,7 @@ static BlockDescBind* CreateStepBlock( } ParamGradInfoMap AppendBackward( - ProgramDescBind& program_desc, const VarDescBind& target, + ProgramDesc& program_desc, const VarDesc& target, const std::unordered_set& no_grad_vars) { std::unordered_set no_grad_var_names; no_grad_var_names.reserve(no_grad_vars.size() + 1); @@ -541,11 +537,11 @@ ParamGradInfoMap AppendBackward( PADDLE_ENFORCE(is_scalar, "target should be scalar"); VLOG(3) << "backward from loss=" << target.Name() << " data_type=" << target.GetDataType(); - std::unique_ptr fill_one_op( - new OpDescBind("fill_constant", {}, {{"Out", {fill_one_op_out}}}, - {{"shape", std::vector{1}}, - {"value", static_cast(1.0)}, - {"dtype", target.GetDataType()}})); + std::unique_ptr fill_one_op( + new OpDesc("fill_constant", {}, {{"Out", {fill_one_op_out}}}, + {{"shape", std::vector{1}}, + {"value", static_cast(1.0)}, + {"dtype", target.GetDataType()}})); // infer var type of fill_one_op fill_one_op->InferVarType(root_block); diff --git a/paddle/framework/backward.h b/paddle/framework/backward.h index 96154fa82c..2d3b75fe69 100644 --- a/paddle/framework/backward.h +++ b/paddle/framework/backward.h @@ -49,7 +49,7 @@ using ParamGradInfoMap = std::unordered_map; ParamGradInfoMap AppendBackward( - ProgramDescBind& program_desc, const VarDescBind& target, + ProgramDesc& program_desc, const VarDesc& target, const std::unordered_set& no_grad_vars); } // namespace framework diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 1099fffab3..be24846246 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -58,13 +58,13 @@ class RowWiseAddGradMaker : public SingleGradOpDescMaker { using SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto grad_op = new OpDescBind(); + std::unique_ptr Apply() const override { + auto grad_op = new OpDesc(); grad_op->SetInput(GradVarName("Out"), OutputGrad("Out")); grad_op->SetOutput(GradVarName("X"), InputGrad("X")); grad_op->SetOutput(GradVarName("b"), InputGrad("b")); grad_op->SetType("rowwise_add_grad"); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; @@ -190,11 +190,11 @@ class MinusGradOpDescMaker : public GradOpDescMakerBase { public: using GradOpDescMakerBase::GradOpDescMakerBase; - std::vector> operator()() const override { - std::vector> retv; + std::vector> operator()() const override { + std::vector> retv; auto x_g = InputGrad("X"); if (!x_g.empty()) { - auto *op_desc = new OpDescBind(); + auto *op_desc = new OpDesc(); op_desc->SetType("scale"); op_desc->SetInput("X", OutputGrad("Out")); op_desc->SetOutput("Out", x_g); @@ -204,7 +204,7 @@ class MinusGradOpDescMaker : public GradOpDescMakerBase { auto y_g = InputGrad("Y"); if (!y_g.empty()) { - auto *op_desc = new OpDescBind(); + auto *op_desc = new OpDesc(); op_desc->SetType("scale"); op_desc->SetInput("X", OutputGrad("Out")); op_desc->SetOutput("Out", y_g); @@ -505,25 +505,25 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) { } TEST(Backward, simple_single_op) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); - f::OpDescBind *op = block->AppendOp(); + f::OpDesc *op = block->AppendOp(); op->SetType("rowwise_add"); op->SetInput("X", {"x"}); op->SetInput("b", {"b"}); op->SetOutput("Out", {"out"}); - auto target = f::VarDescBind("out"); + auto target = f::VarDesc("out"); target.SetShape({1}); auto var_to_grad = AppendBackward(program, target, std::unordered_set{}); ASSERT_EQ(block->AllOps().size(), 3UL); - f::OpDescBind *fill_op = block->AllOps()[1]; + f::OpDesc *fill_op = block->AllOps()[1]; EXPECT_EQ(fill_op->Type(), "fill_constant"); - f::OpDescBind *grad_op = block->AllOps()[2]; + f::OpDesc *grad_op = block->AllOps()[2]; EXPECT_EQ(grad_op->Type(), "rowwise_add_grad"); ASSERT_EQ(grad_op->InputNames().size(), 1UL); ASSERT_EQ(grad_op->OutputNames().size(), 2UL); @@ -543,16 +543,16 @@ TEST(Backward, simple_single_op) { } TEST(Backward, default_attribute) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); - f::OpDescBind *op = block->AppendOp(); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); + f::OpDesc *op = block->AppendOp(); op->SetType("mul"); op->SetInput("X", {"x"}); op->SetInput("Y", {"y"}); op->SetOutput("Out", {"out"}); op->CheckAttrs(); - auto target = f::VarDescBind("out"); + auto target = f::VarDesc("out"); target.SetShape({1}); AppendBackward(program, target, std::unordered_set{}); @@ -560,47 +560,47 @@ TEST(Backward, default_attribute) { EXPECT_EQ(boost::get(op->GetAttr("x_num_col_dims")), 1); EXPECT_EQ(boost::get(op->GetAttr("y_num_col_dims")), 1); - f::OpDescBind *fill_op = block->AllOps()[1]; + f::OpDesc *fill_op = block->AllOps()[1]; EXPECT_EQ(fill_op->Type(), "fill_constant"); - f::OpDescBind *grad_op = block->AllOps()[2]; + f::OpDesc *grad_op = block->AllOps()[2]; ASSERT_EQ(grad_op->Type(), "mul_grad"); EXPECT_EQ(boost::get(grad_op->GetAttr("x_num_col_dims")), 1); EXPECT_EQ(boost::get(grad_op->GetAttr("y_num_col_dims")), 1); } TEST(Backward, simple_mult_op) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); - f::OpDescBind *op1 = block->AppendOp(); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); + f::OpDesc *op1 = block->AppendOp(); op1->SetType("rowwise_add"); op1->SetInput("X", {"x1"}); op1->SetInput("b", {"b1"}); op1->SetOutput("Out", {"out1"}); - f::OpDescBind *op2 = block->AppendOp(); + f::OpDesc *op2 = block->AppendOp(); op2->SetType("mul"); op2->SetInput("X", {"out1"}); op2->SetInput("Y", {"y2"}); op2->SetOutput("Out", {"out2"}); - f::OpDescBind *op3 = block->AppendOp(); + f::OpDesc *op3 = block->AppendOp(); op3->SetType("rowwise_add"); op3->SetInput("X", {"out2"}); op3->SetInput("b", {"b3"}); op3->SetOutput("Out", {"out3"}); - auto target = f::VarDescBind("out3"); + auto target = f::VarDesc("out3"); target.SetShape({1}); size_t forward_len = block->AllOps().size(); auto var_to_grad = AppendBackward(program, target, std::unordered_set{}); ASSERT_EQ(block->AllOps().size(), 6UL + 1); - f::OpDescBind *fill_op = block->AllOps()[forward_len]; + f::OpDesc *fill_op = block->AllOps()[forward_len]; EXPECT_EQ(fill_op->Type(), "fill_constant"); - f::OpDescBind *grad_op1 = block->AllOps()[6]; + f::OpDesc *grad_op1 = block->AllOps()[6]; EXPECT_EQ(grad_op1->Type(), "rowwise_add_grad"); ASSERT_EQ(grad_op1->InputNames().size(), 1UL); ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); @@ -611,7 +611,7 @@ TEST(Backward, simple_mult_op) { EXPECT_EQ(grad_op1->Output(f::GradVarName("b")), std::vector({f::GradVarName("b1")})); - f::OpDescBind *grad_op2 = block->AllOps()[5]; + f::OpDesc *grad_op2 = block->AllOps()[5]; EXPECT_EQ(grad_op2->Type(), "mul_grad"); ASSERT_EQ(grad_op2->InputNames().size(), 4UL); ASSERT_EQ(grad_op2->OutputNames().size(), 2UL); @@ -625,7 +625,7 @@ TEST(Backward, simple_mult_op) { EXPECT_EQ(grad_op2->Output(f::GradVarName("Y")), std::vector({f::GradVarName("y2")})); - f::OpDescBind *grad_op3 = block->AllOps()[4]; + f::OpDesc *grad_op3 = block->AllOps()[4]; EXPECT_EQ(grad_op3->Type(), "rowwise_add_grad"); ASSERT_EQ(grad_op3->InputNames().size(), 1UL); ASSERT_EQ(grad_op3->OutputNames().size(), 2UL); @@ -655,42 +655,42 @@ TEST(Backward, simple_mult_op) { } TEST(Backward, intermedia_var_no_grad) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); - f::OpDescBind *op1 = block->AppendOp(); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); + f::OpDesc *op1 = block->AppendOp(); op1->SetType("rowwise_add"); op1->SetInput("X", {"x1"}); op1->SetInput("b", {"b1"}); op1->SetOutput("Out", {"out1"}); - f::OpDescBind *op2 = block->AppendOp(); + f::OpDesc *op2 = block->AppendOp(); op2->SetType("mul"); op2->SetInput("X", {"x2"}); op2->SetInput("Y", {"y2"}); op2->SetOutput("Out", {"out2"}); - f::OpDescBind *op3 = block->AppendOp(); + f::OpDesc *op3 = block->AppendOp(); op3->SetType("rowwise_add"); op3->SetInput("X", {"out2"}); op3->SetInput("b", {"b3"}); op3->SetOutput("Out", {"out3"}); - f::OpDescBind *op4 = block->AppendOp(); + f::OpDesc *op4 = block->AppendOp(); op4->SetType("mul"); op4->SetInput("X", {"out1"}); op4->SetInput("Y", {"out3"}); op4->SetOutput("Out", {"out4"}); - auto target = f::VarDescBind("out4"); + auto target = f::VarDesc("out4"); target.SetShape({1}); size_t forward_len = block->AllOps().size(); auto var_to_grad = AppendBackward(program, target, {"out3"}); ASSERT_EQ(block->AllOps().size(), 7UL); - f::OpDescBind *fill_op = block->AllOps()[forward_len]; + f::OpDesc *fill_op = block->AllOps()[forward_len]; EXPECT_EQ(fill_op->Type(), "fill_constant"); - f::OpDescBind *grad_op1 = block->AllOps()[6]; + f::OpDesc *grad_op1 = block->AllOps()[6]; EXPECT_EQ(grad_op1->Type(), "rowwise_add_grad"); ASSERT_EQ(grad_op1->InputNames().size(), 1UL); ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); @@ -701,7 +701,7 @@ TEST(Backward, intermedia_var_no_grad) { EXPECT_EQ(grad_op1->Output(f::GradVarName("b")), std::vector({f::GradVarName("b1")})); - f::OpDescBind *grad_op4 = block->AllOps()[5]; + f::OpDesc *grad_op4 = block->AllOps()[5]; EXPECT_EQ(grad_op4->Type(), "mul_grad"); ASSERT_EQ(grad_op4->InputNames().size(), 4UL); ASSERT_EQ(grad_op4->OutputNames().size(), 2UL); @@ -726,32 +726,32 @@ TEST(Backward, intermedia_var_no_grad) { } TEST(Backward, var_no_grad) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); - f::OpDescBind *op1 = block->AppendOp(); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); + f::OpDesc *op1 = block->AppendOp(); op1->SetType("mult_in_out"); op1->SetInput("X", {"x1"}); op1->SetInput("H", {"h1"}); op1->SetOutput("Y", {"y1"}); op1->SetOutput("Z", {"z1"}); - f::OpDescBind *op2 = block->AppendOp(); + f::OpDesc *op2 = block->AppendOp(); op2->SetType("mult_in_out"); op2->SetInput("X", {"y1"}); op2->SetInput("H", {"z1"}); op2->SetOutput("Y", {"y2"}); op2->SetOutput("Z", {"z2"}); - auto target = f::VarDescBind("z2"); + auto target = f::VarDesc("z2"); target.SetShape({1}); size_t forward_len = block->AllOps().size(); auto var_to_grad = AppendBackward(program, target, {"z1"}); ASSERT_EQ(block->AllOps().size(), 6UL); - f::OpDescBind *fill_op = block->AllOps()[forward_len]; + f::OpDesc *fill_op = block->AllOps()[forward_len]; EXPECT_EQ(fill_op->Type(), "fill_constant"); - f::OpDescBind *grad_op2 = block->AllOps()[3]; + f::OpDesc *grad_op2 = block->AllOps()[3]; ASSERT_EQ(grad_op2->Type(), "mult_in_out_grad"); ASSERT_EQ(grad_op2->InputNames().size(), 6UL); ASSERT_EQ(grad_op2->OutputNames().size(), 2UL); @@ -767,7 +767,7 @@ TEST(Backward, var_no_grad) { std::vector({f::GradVarName("y1")})); EXPECT_EQ(grad_op2->Output(f::GradVarName("H")), std::vector()); - f::OpDescBind *fill_zero_op = block->AllOps()[4]; + f::OpDesc *fill_zero_op = block->AllOps()[4]; ASSERT_EQ(fill_zero_op->Type(), "fill_zeros_like"); ASSERT_EQ(fill_zero_op->InputNames().size(), 1UL); ASSERT_EQ(fill_zero_op->OutputNames().size(), 1UL); @@ -775,7 +775,7 @@ TEST(Backward, var_no_grad) { EXPECT_EQ(fill_zero_op->Output("Y"), std::vector({std::string("z1") + f::kZeroVarSuffix})); - f::OpDescBind *grad_op1 = block->AllOps()[5]; + f::OpDesc *grad_op1 = block->AllOps()[5]; ASSERT_EQ(grad_op1->Type(), "mult_in_out_grad"); ASSERT_EQ(grad_op1->InputNames().size(), 6UL); ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); @@ -803,37 +803,37 @@ TEST(Backward, var_no_grad) { } TEST(Backward, shared_var) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); - f::OpDescBind *op1 = block->AppendOp(); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); + f::OpDesc *op1 = block->AppendOp(); op1->SetType("rowwise_add"); op1->SetInput("X", {"x1"}); op1->SetInput("b", {"b1"}); op1->SetOutput("Out", {"out1"}); - f::OpDescBind *op2 = block->AppendOp(); + f::OpDesc *op2 = block->AppendOp(); op2->SetType("mul"); op2->SetInput("X", {"out1"}); op2->SetInput("Y", {"y2"}); op2->SetOutput("Out", {"out2"}); - f::OpDescBind *op3 = block->AppendOp(); + f::OpDesc *op3 = block->AppendOp(); op3->SetType("rowwise_add"); op3->SetInput("X", {"out1"}); op3->SetInput("b", {"b3"}); op3->SetOutput("Out", {"out3"}); - auto target = f::VarDescBind("out3"); + auto target = f::VarDesc("out3"); target.SetShape({1}); size_t forward_len = block->AllOps().size(); auto var_to_grad = AppendBackward(program, target, std::unordered_set{}); ASSERT_EQ(block->AllOps().size(), 8UL); - f::OpDescBind *fill_op = block->AllOps()[forward_len]; + f::OpDesc *fill_op = block->AllOps()[forward_len]; EXPECT_EQ(fill_op->Type(), "fill_constant"); - f::OpDescBind *grad_op3 = block->AllOps()[4]; + f::OpDesc *grad_op3 = block->AllOps()[4]; ASSERT_EQ(grad_op3->Type(), "rowwise_add_grad"); ASSERT_EQ(grad_op3->InputNames().size(), 1UL); ASSERT_EQ(grad_op3->OutputNames().size(), 2UL); @@ -844,7 +844,7 @@ TEST(Backward, shared_var) { EXPECT_EQ(grad_op3->Output(f::GradVarName("b")), std::vector({f::GradVarName("b3")})); - f::OpDescBind *grad_op4 = block->AllOps()[5]; + f::OpDesc *grad_op4 = block->AllOps()[5]; ASSERT_EQ(grad_op4->Type(), "mul_grad"); ASSERT_EQ(grad_op4->InputNames().size(), 4UL); ASSERT_EQ(grad_op4->OutputNames().size(), 2UL); @@ -858,7 +858,7 @@ TEST(Backward, shared_var) { EXPECT_EQ(grad_op4->Output(f::GradVarName("Y")), std::vector({f::GradVarName("y2")})); - f::OpDescBind *sum_op = block->AllOps()[6]; + f::OpDesc *sum_op = block->AllOps()[6]; ASSERT_EQ(sum_op->Type(), "sum"); ASSERT_EQ(sum_op->InputNames().size(), 1UL); ASSERT_EQ(sum_op->OutputNames().size(), 1UL); @@ -868,7 +868,7 @@ TEST(Backward, shared_var) { EXPECT_EQ(sum_op->Output("Out"), std::vector({f::GradVarName("out1")})); - f::OpDescBind *grad_op1 = block->AllOps()[7]; + f::OpDesc *grad_op1 = block->AllOps()[7]; ASSERT_EQ(grad_op1->Type(), "rowwise_add_grad"); ASSERT_EQ(grad_op1->InputNames().size(), 1UL); ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); @@ -895,19 +895,19 @@ TEST(Backward, shared_var) { } TEST(Backward, half_backward) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); auto *op1 = block->AppendOp(); op1->SetType("minus"); op1->SetInput("X", {"a"}); op1->SetInput("Y", {"b"}); op1->SetOutput("Out", {"out"}); - auto target = f::VarDescBind("out"); + auto target = f::VarDesc("out"); target.SetShape({1}); size_t forward_len = block->AllOps().size(); auto var_to_grad = AppendBackward(program, target, {"b"}); - f::OpDescBind *fill_op = block->AllOps()[forward_len]; + f::OpDesc *fill_op = block->AllOps()[forward_len]; EXPECT_EQ(fill_op->Type(), "fill_constant"); auto ops = block->AllOps(); ASSERT_EQ(3UL, ops.size()); diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index 6b961caebd..2d7db382a6 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -19,18 +19,18 @@ limitations under the License. */ namespace paddle { namespace framework { -VarDescBind *BlockDescBind::Var(const std::string &name) { +VarDesc *BlockDesc::Var(const std::string &name) { auto it = vars_.find(name); if (it != vars_.end()) { return it->second.get(); } need_update_ = true; - auto *var = new VarDescBind(name); + auto *var = new VarDesc(name); vars_[name].reset(var); return var; } -VarDescBind *BlockDescBind::FindVar(const std::string &name) const { +VarDesc *BlockDesc::FindVar(const std::string &name) const { auto it = vars_.find(name); if (it == vars_.end()) { return nullptr; @@ -38,11 +38,11 @@ VarDescBind *BlockDescBind::FindVar(const std::string &name) const { return it->second.get(); } -bool BlockDescBind::HasVar(const std::string &name) const { +bool BlockDesc::HasVar(const std::string &name) const { return vars_.find(name) != vars_.end(); } -VarDescBind *BlockDescBind::FindVarRecursive(const std::string &name) const { +VarDesc *BlockDesc::FindVarRecursive(const std::string &name) const { if (name == kEmptyVarName) return nullptr; auto it = vars_.find(name); @@ -53,53 +53,52 @@ VarDescBind *BlockDescBind::FindVarRecursive(const std::string &name) const { return it->second.get(); } -VarDescBind *BlockDescBind::FindRecursiveOrCreateVar( - const std::string &name_bytes) { - VarDescBind *res = FindVarRecursive(name_bytes); +VarDesc *BlockDesc::FindRecursiveOrCreateVar(const std::string &name_bytes) { + VarDesc *res = FindVarRecursive(name_bytes); if (res == nullptr) { res = Var(name_bytes); } return res; } -bool BlockDescBind::HasVarRecursive(const std::string &name) const { +bool BlockDesc::HasVarRecursive(const std::string &name) const { return FindVarRecursive(name) != nullptr; } -std::vector BlockDescBind::AllVars() const { - std::vector res; +std::vector BlockDesc::AllVars() const { + std::vector res; for (const auto &p : vars_) { res.push_back(p.second.get()); } return res; } -OpDescBind *BlockDescBind::AppendOp() { +OpDesc *BlockDesc::AppendOp() { need_update_ = true; - ops_.emplace_back(new OpDescBind()); + ops_.emplace_back(new OpDesc()); return ops_.back().get(); } -void BlockDescBind::AppendAllocatedOp(std::unique_ptr &&op_desc) { +void BlockDesc::AppendAllocatedOp(std::unique_ptr &&op_desc) { need_update_ = true; ops_.emplace_back(std::move(op_desc)); } -OpDescBind *BlockDescBind::PrependOp() { +OpDesc *BlockDesc::PrependOp() { need_update_ = true; - ops_.emplace_front(new OpDescBind()); + ops_.emplace_front(new OpDesc()); return ops_.front().get(); } -std::vector BlockDescBind::AllOps() const { - std::vector res; +std::vector BlockDesc::AllOps() const { + std::vector res; for (const auto &op : ops_) { res.push_back(op.get()); } return res; } -void BlockDescBind::Flush() { +void BlockDesc::Flush() { for (auto &op_desc : ops_) { op_desc->Flush(); } @@ -121,43 +120,43 @@ void BlockDescBind::Flush() { } } -BlockDescBind *BlockDescBind::ParentBlock() const { +BlockDesc *BlockDesc::ParentBlock() const { if (this->desc_->parent_idx() == kNoneBlockIndex) { return nullptr; } return prog_->MutableBlock(static_cast(this->desc_->parent_idx())); } -proto::BlockDesc *BlockDescBind::Proto() { +proto::BlockDesc *BlockDesc::Proto() { Flush(); return desc_; } -BlockDescBind::BlockDescBind(ProgramDescBind *prog, proto::BlockDesc *desc) +BlockDesc::BlockDesc(ProgramDesc *prog, proto::BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) { for (const proto::VarDesc &var_desc : desc_->vars()) { - vars_[var_desc.name()].reset(new VarDescBind(var_desc)); + vars_[var_desc.name()].reset(new VarDesc(var_desc)); } for (const proto::OpDesc &op_desc : desc_->ops()) { - ops_.emplace_back(new OpDescBind(op_desc, prog)); + ops_.emplace_back(new OpDesc(op_desc, prog)); } } -BlockDescBind::BlockDescBind(const BlockDescBind &other, proto::BlockDesc *desc, - ProgramDescBind *prog) +BlockDesc::BlockDesc(const BlockDesc &other, proto::BlockDesc *desc, + ProgramDesc *prog) : prog_(prog), desc_(desc) { need_update_ = true; for (auto &op : other.ops_) { - ops_.emplace_back(new OpDescBind(*op)); + ops_.emplace_back(new OpDesc(*op)); } for (auto &it : other.vars_) { - auto *var = new VarDescBind(*it.second); + auto *var = new VarDesc(*it.second); vars_[it.first].reset(var); } } -void BlockDescBind::ClearPBOps() { +void BlockDesc::ClearPBOps() { auto ops = this->desc_->mutable_ops(); while (!ops->empty()) { // we do not own the OpDesc, so release the ownership. @@ -165,7 +164,7 @@ void BlockDescBind::ClearPBOps() { } } -void BlockDescBind::ClearPBVars() { +void BlockDesc::ClearPBVars() { auto vars = this->desc_->mutable_vars(); while (!vars->empty()) { // we do not own the VarDesc, so release the ownership. diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 592fe49e07..513fc54f24 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -28,20 +28,19 @@ limitations under the License. */ namespace paddle { namespace framework { -class ProgramDescBind; +class ProgramDesc; // Each Protobuf Message, we provide a XXXBind class. In that class, we optimize // read/write speed. Only when we want the protobuf message, the local changes // will be synchronized (by `Sync` method). -class BlockDescBind { +class BlockDesc { public: - BlockDescBind(ProgramDescBind *prog, proto::BlockDesc *desc); + BlockDesc(ProgramDesc *prog, proto::BlockDesc *desc); - BlockDescBind(const BlockDescBind &other, proto::BlockDesc *desc, - ProgramDescBind *prog); + BlockDesc(const BlockDesc &other, proto::BlockDesc *desc, ProgramDesc *prog); - ~BlockDescBind() { + ~BlockDesc() { this->ClearPBVars(); this->ClearPBOps(); } @@ -50,15 +49,15 @@ class BlockDescBind { int32_t Parent() const { return desc_->parent_idx(); } - VarDescBind *Var(const std::string &name_bytes); + VarDesc *Var(const std::string &name_bytes); - VarDescBind *FindVar(const std::string &name_bytes) const; + VarDesc *FindVar(const std::string &name_bytes) const; bool HasVar(const std::string &var_name) const; - VarDescBind *FindVarRecursive(const std::string &name_bytes) const; + VarDesc *FindVarRecursive(const std::string &name_bytes) const; - VarDescBind *FindRecursiveOrCreateVar(const std::string &name_bytes); + VarDesc *FindRecursiveOrCreateVar(const std::string &name_bytes); bool HasVarRecursive(const std::string &var_name) const; @@ -70,41 +69,41 @@ class BlockDescBind { return var_names; } - std::vector AllVars() const; + std::vector AllVars() const; - BlockDescBind *ParentBlock() const; + BlockDesc *ParentBlock() const; - OpDescBind *AppendOp(); + OpDesc *AppendOp(); - void AppendAllocatedOp(std::unique_ptr &&op_desc); + void AppendAllocatedOp(std::unique_ptr &&op_desc); - OpDescBind *PrependOp(); + OpDesc *PrependOp(); - std::vector AllOps() const; + std::vector AllOps() const; size_t OpSize() const { return ops_.size(); } - OpDescBind *Op(int idx) { return ops_.at(idx).get(); } + OpDesc *Op(int idx) { return ops_.at(idx).get(); } void Flush(); proto::BlockDesc *Proto(); - ProgramDescBind *Program() { return this->prog_; } + ProgramDesc *Program() { return this->prog_; } private: void ClearPBOps(); void ClearPBVars(); private: - ProgramDescBind *prog_; // not_own + ProgramDesc *prog_; // not_own proto::BlockDesc *desc_; // not_own bool need_update_; - std::deque> ops_; - std::unordered_map> vars_; + std::deque> ops_; + std::unordered_map> vars_; - DISABLE_COPY_AND_ASSIGN(BlockDescBind); + DISABLE_COPY_AND_ASSIGN(BlockDesc); }; } // namespace framework } // namespace paddle diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index 435f0b6b78..7f5151c41d 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -106,10 +106,10 @@ template struct OpInfoFiller { void operator()(const char* op_type, OpInfo* info) const { info->grad_op_maker_ = []( - const OpDescBind& fwd_op, + const OpDesc& fwd_op, const std::unordered_set& no_grad_set, std::unordered_map* grad_to_var, - const std::vector& grad_block) { + const std::vector& grad_block) { T maker(fwd_op, no_grad_set, grad_to_var, grad_block); return maker(); }; @@ -119,7 +119,7 @@ struct OpInfoFiller { template struct OpInfoFiller { void operator()(const char* op_type, OpInfo* info) const { - info->infer_var_type_ = [](const OpDescBind& fwd_op, BlockDescBind* block) { + info->infer_var_type_ = [](const OpDesc& fwd_op, BlockDesc* block) { T inference; inference(fwd_op, block); }; diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index ea6b259c09..c4b76911a6 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -64,7 +64,7 @@ static void CreateTensor(Variable* var, proto::VarDesc::VarType var_type) { } } -void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id, +void Executor::Run(const ProgramDesc& pdesc, Scope* scope, int block_id, bool create_local_scope) { // TODO(tonyyang-svail): // - only runs on the first device (i.e. no interdevice communication) diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h index 073e04729b..1faaacfefa 100644 --- a/paddle/framework/executor.h +++ b/paddle/framework/executor.h @@ -114,7 +114,7 @@ class Executor { * ProgramDesc * Scope */ - void Run(const ProgramDescBind&, Scope*, int, bool create_local_scope = true); + void Run(const ProgramDesc&, Scope*, int, bool create_local_scope = true); private: std::vector device_contexts_; diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index 998186e339..8c47c0b0c8 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -25,18 +25,16 @@ namespace framework { class GradOpDescMakerBase { public: explicit GradOpDescMakerBase( - const OpDescBind& fwd_op, - const std::unordered_set& no_grad_set, + const OpDesc& fwd_op, const std::unordered_set& no_grad_set, std::unordered_map* grad_to_var, - const std::vector& grad_block = - std::vector()) + const std::vector& grad_block = std::vector()) : fwd_op_(fwd_op), no_grad_set_(no_grad_set), grad_to_var_(grad_to_var), grad_block_(grad_block) {} virtual ~GradOpDescMakerBase() = default; - virtual std::vector> operator()() const = 0; + virtual std::vector> operator()() const = 0; protected: std::vector InputGrad(const std::string& name, @@ -105,26 +103,26 @@ class GradOpDescMakerBase { std::string ForwardOpType() const { return this->fwd_op_.Type(); } private: - const OpDescBind& fwd_op_; + const OpDesc& fwd_op_; const std::unordered_set& no_grad_set_; std::unordered_map* grad_to_var_; protected: - std::vector grad_block_; + std::vector grad_block_; }; class SingleGradOpDescMaker : public GradOpDescMakerBase { public: using GradOpDescMakerBase::GradOpDescMakerBase; - std::vector> operator()() const { - std::vector> retv; + std::vector> operator()() const { + std::vector> retv; retv.emplace_back(this->Apply()); return retv; } protected: - virtual std::unique_ptr Apply() const = 0; + virtual std::unique_ptr Apply() const = 0; }; template @@ -133,8 +131,8 @@ class DefaultGradOpDescMaker : public SingleGradOpDescMaker { using SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - virtual std::unique_ptr Apply() const { - auto* grad = new OpDescBind(); + virtual std::unique_ptr Apply() const { + auto* grad = new OpDesc(); grad->SetType(this->GradOpType()); for (auto& input_param : this->InputNames()) { @@ -150,7 +148,7 @@ class DefaultGradOpDescMaker : public SingleGradOpDescMaker { grad->SetAttrMap(this->Attrs()); - return std::unique_ptr(grad); + return std::unique_ptr(grad); } virtual std::string GradOpType() const { @@ -161,7 +159,7 @@ class DefaultGradOpDescMaker : public SingleGradOpDescMaker { class EmptyGradOpMaker : public GradOpDescMakerBase { public: using GradOpDescMakerBase::GradOpDescMakerBase; - std::vector> operator()() const override { + std::vector> operator()() const override { return {}; } }; diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 7af5b68727..b361e64438 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -25,12 +25,11 @@ limitations under the License. */ namespace paddle { namespace framework { -class OpDescBind; -class BlockDescBind; +class OpDesc; +class BlockDesc; class CompileTimeInferShapeContext : public InferShapeContext { public: - CompileTimeInferShapeContext(const OpDescBind &op, - const BlockDescBind &block); + CompileTimeInferShapeContext(const OpDesc &op, const BlockDesc &block); bool HasInput(const std::string &name) const override; @@ -76,13 +75,12 @@ class CompileTimeInferShapeContext : public InferShapeContext { void SetDim(const std::string &name, const DDim &dim) override; - const OpDescBind &op_; - const BlockDescBind &block_; + const OpDesc &op_; + const BlockDesc &block_; }; -OpDescBind::OpDescBind(const std::string &type, const VariableNameMap &inputs, - const VariableNameMap &outputs, - const AttributeMap &attrs) { +OpDesc::OpDesc(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const AttributeMap &attrs) { desc_.set_type(type); inputs_ = inputs; outputs_ = outputs; @@ -90,7 +88,7 @@ OpDescBind::OpDescBind(const std::string &type, const VariableNameMap &inputs, need_update_ = true; } -OpDescBind::OpDescBind(const proto::OpDesc &desc, ProgramDescBind *prog) +OpDesc::OpDesc(const proto::OpDesc &desc, ProgramDesc *prog) : desc_(desc), need_update_(false) { // restore inputs_ int input_size = desc_.inputs_size(); @@ -126,20 +124,19 @@ OpDescBind::OpDescBind(const proto::OpDesc &desc, ProgramDescBind *prog) } } -proto::OpDesc *OpDescBind::Proto() { +proto::OpDesc *OpDesc::Proto() { Flush(); return &desc_; } -const std::vector &OpDescBind::Input( - const std::string &name) const { +const std::vector &OpDesc::Input(const std::string &name) const { auto it = inputs_.find(name); PADDLE_ENFORCE(it != inputs_.end(), "Input %s cannot be found in Op %s", name, Type()); return it->second; } -std::vector OpDescBind::InputArgumentNames() const { +std::vector OpDesc::InputArgumentNames() const { std::vector retv; for (auto &ipt : this->inputs_) { retv.insert(retv.end(), ipt.second.begin(), ipt.second.end()); @@ -147,21 +144,20 @@ std::vector OpDescBind::InputArgumentNames() const { return retv; } -void OpDescBind::SetInput(const std::string ¶m_name, - const std::vector &args) { +void OpDesc::SetInput(const std::string ¶m_name, + const std::vector &args) { need_update_ = true; inputs_[param_name] = args; } -const std::vector &OpDescBind::Output( - const std::string &name) const { +const std::vector &OpDesc::Output(const std::string &name) const { auto it = outputs_.find(name); PADDLE_ENFORCE(it != outputs_.end(), "Output %s cannot be found in Op %s", name, Type()); return it->second; } -std::vector OpDescBind::OutputArgumentNames() const { +std::vector OpDesc::OutputArgumentNames() const { std::vector retv; for (auto &ipt : this->outputs_) { retv.insert(retv.end(), ipt.second.begin(), ipt.second.end()); @@ -169,19 +165,19 @@ std::vector OpDescBind::OutputArgumentNames() const { return retv; } -void OpDescBind::SetOutput(const std::string ¶m_name, - const std::vector &args) { +void OpDesc::SetOutput(const std::string ¶m_name, + const std::vector &args) { need_update_ = true; this->outputs_[param_name] = args; } -proto::AttrType OpDescBind::GetAttrType(const std::string &name) const { +proto::AttrType OpDesc::GetAttrType(const std::string &name) const { auto it = attrs_.find(name); PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); return static_cast(it->second.which() - 1); } -std::vector OpDescBind::AttrNames() const { +std::vector OpDesc::AttrNames() const { std::vector retv; retv.reserve(attrs_.size()); for (auto &attr : attrs_) { @@ -190,41 +186,39 @@ std::vector OpDescBind::AttrNames() const { return retv; } -void OpDescBind::SetAttr(const std::string &name, const Attribute &v) { +void OpDesc::SetAttr(const std::string &name, const Attribute &v) { this->attrs_[name] = v; need_update_ = true; } -void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) { +void OpDesc::SetBlockAttr(const std::string &name, BlockDesc &block) { this->attrs_[name] = █ need_update_ = true; } -void OpDescBind::SetAttrMap( +void OpDesc::SetAttrMap( const std::unordered_map &attr_map) { attrs_ = attr_map; need_update_ = true; } -Attribute OpDescBind::GetAttr(const std::string &name) const { +Attribute OpDesc::GetAttr(const std::string &name) const { auto it = attrs_.find(name); PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); return it->second; } -int OpDescBind::GetBlockAttr(const std::string &name) const { +int OpDesc::GetBlockAttr(const std::string &name) const { auto it = attrs_.find(name); PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); - return boost::get(it->second)->ID(); + return boost::get(it->second)->ID(); } -const std::unordered_map &OpDescBind::GetAttrMap() - const { +const std::unordered_map &OpDesc::GetAttrMap() const { return attrs_; } -void OpDescBind::Rename(const std::string &old_name, - const std::string &new_name) { +void OpDesc::Rename(const std::string &old_name, const std::string &new_name) { for (auto &input : inputs_) { std::replace(input.second.begin(), input.second.end(), old_name, new_name); } @@ -235,8 +229,8 @@ void OpDescBind::Rename(const std::string &old_name, need_update_ = true; } -void OpDescBind::RenameOutput(const std::string &old_name, - const std::string &new_name) { +void OpDesc::RenameOutput(const std::string &old_name, + const std::string &new_name) { for (auto &output : outputs_) { std::replace(output.second.begin(), output.second.end(), old_name, new_name); @@ -244,8 +238,8 @@ void OpDescBind::RenameOutput(const std::string &old_name, need_update_ = true; } -void OpDescBind::RenameInput(const std::string &old_name, - const std::string &new_name) { +void OpDesc::RenameInput(const std::string &old_name, + const std::string &new_name) { for (auto &input : inputs_) { std::replace(input.second.begin(), input.second.end(), old_name, new_name); } @@ -278,7 +272,7 @@ struct SetAttrDescVisitor : public boost::static_visitor { void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } }; -void OpDescBind::Flush() { +void OpDesc::Flush() { if (need_update_) { this->desc_.mutable_inputs()->Clear(); for (auto &ipt : inputs_) { @@ -330,7 +324,7 @@ static void InitInferShapeFuncs() { }); } -void OpDescBind::CheckAttrs() { +void OpDesc::CheckAttrs() { PADDLE_ENFORCE(!Type().empty(), "CheckAttr() can not be called before type is setted."); auto *checker = OpInfoMap::Instance().Get(Type()).Checker(); @@ -342,7 +336,7 @@ void OpDescBind::CheckAttrs() { checker->Check(attrs_); } -void OpDescBind::InferShape(const BlockDescBind &block) const { +void OpDesc::InferShape(const BlockDesc &block) const { VLOG(3) << "CompileTime infer shape on " << Type(); InitInferShapeFuncs(); auto &infer_shape = OpInfoMap::Instance().Get(this->Type()).infer_shape_; @@ -365,7 +359,7 @@ void OpDescBind::InferShape(const BlockDescBind &block) const { infer_shape(&ctx); } -void OpDescBind::InferVarType(BlockDescBind *block) const { +void OpDesc::InferVarType(BlockDesc *block) const { auto &info = OpInfoMap::Instance().Get(this->Type()); if (info.infer_var_type_) { info.infer_var_type_(*this, block); @@ -384,7 +378,7 @@ void OpDescBind::InferVarType(BlockDescBind *block) const { } CompileTimeInferShapeContext::CompileTimeInferShapeContext( - const OpDescBind &op, const BlockDescBind &block) + const OpDesc &op, const BlockDesc &block) : op_(op), block_(block) {} bool CompileTimeInferShapeContext::HasInput(const std::string &name) const { diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 0f0f126f98..18fa02940d 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -23,17 +23,17 @@ limitations under the License. */ namespace paddle { namespace framework { -class BlockDescBind; -class ProgramDescBind; +class BlockDesc; +class ProgramDesc; -class OpDescBind { +class OpDesc { public: - OpDescBind() {} + OpDesc() {} - OpDescBind(const std::string &type, const VariableNameMap &inputs, - const VariableNameMap &outputs, const AttributeMap &attrs); + OpDesc(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const AttributeMap &attrs); - OpDescBind(const proto::OpDesc &desc, ProgramDescBind *prog); + OpDesc(const proto::OpDesc &desc, ProgramDesc *prog); proto::OpDesc *Proto(); @@ -65,7 +65,7 @@ class OpDescBind { void SetAttr(const std::string &name, const Attribute &v); - void SetBlockAttr(const std::string &name, BlockDescBind &block); + void SetBlockAttr(const std::string &name, BlockDesc &block); Attribute GetAttr(const std::string &name) const; @@ -107,9 +107,9 @@ class OpDescBind { void CheckAttrs(); - void InferShape(const BlockDescBind &block) const; + void InferShape(const BlockDesc &block) const; - void InferVarType(BlockDescBind *block) const; + void InferVarType(BlockDesc *block) const; void MarkAsTarget() { desc_.set_is_target(true); } diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index f202c0b27a..dfa151316d 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -47,7 +47,7 @@ static VariableNameMap ConvertOpDescVarsToVarNameMap( std::unique_ptr OpRegistry::CreateOp( const proto::OpDesc& op_desc) { VLOG(1) << "CreateOp directly from OpDesc is deprecated. It should only be" - "used in unit tests. Use CreateOp(const OpDescBind& op_desc) " + "used in unit tests. Use CreateOp(const OpDesc& op_desc) " "instead."; VariableNameMap inputs = ConvertOpDescVarsToVarNameMap(op_desc.inputs()); VariableNameMap outputs = ConvertOpDescVarsToVarNameMap(op_desc.outputs()); @@ -59,7 +59,7 @@ std::unique_ptr OpRegistry::CreateOp( return CreateOp(op_desc.type(), inputs, outputs, attrs); } -std::unique_ptr OpRegistry::CreateOp(const OpDescBind& op_desc) { +std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc) { return CreateOp(op_desc.Type(), op_desc.Inputs(), op_desc.Outputs(), op_desc.GetAttrMap()); } diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 7367e0e637..278550d496 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -79,7 +79,7 @@ class OpRegistry { static std::unique_ptr CreateOp(const proto::OpDesc& op_desc); - static std::unique_ptr CreateOp(const OpDescBind& op_desc); + static std::unique_ptr CreateOp(const OpDesc& op_desc); }; template diff --git a/paddle/framework/program_desc.cc b/paddle/framework/program_desc.cc index 30a265ccac..b5d9e5e385 100644 --- a/paddle/framework/program_desc.cc +++ b/paddle/framework/program_desc.cc @@ -18,49 +18,49 @@ limitations under the License. */ namespace paddle { namespace framework { -BlockDescBind *ProgramDescBind::AppendBlock(const BlockDescBind &parent) { +BlockDesc *ProgramDesc::AppendBlock(const BlockDesc &parent) { auto *b = desc_.add_blocks(); b->set_parent_idx(parent.ID()); b->set_idx(desc_.blocks_size() - 1); - blocks_.emplace_back(new BlockDescBind(this, b)); + blocks_.emplace_back(new BlockDesc(this, b)); return blocks_.back().get(); } -proto::ProgramDesc *ProgramDescBind::Proto() { +proto::ProgramDesc *ProgramDesc::Proto() { for (auto &block : blocks_) { block->Flush(); } return &desc_; } -ProgramDescBind::ProgramDescBind() { +ProgramDesc::ProgramDesc() { auto *block = desc_.mutable_blocks()->Add(); block->set_idx(kRootBlockIndex); block->set_parent_idx(kNoneBlockIndex); - blocks_.emplace_back(new BlockDescBind(this, block)); + blocks_.emplace_back(new BlockDesc(this, block)); } -ProgramDescBind::ProgramDescBind(const ProgramDescBind &o) { +ProgramDesc::ProgramDesc(const ProgramDesc &o) { desc_ = o.desc_; for (int i = 0; i < desc_.blocks_size(); ++i) { auto *block = desc_.mutable_blocks(i); - blocks_.emplace_back(new BlockDescBind(*o.blocks_[i], block, this)); + blocks_.emplace_back(new BlockDesc(*o.blocks_[i], block, this)); } } -ProgramDescBind::ProgramDescBind(const proto::ProgramDesc &desc) { +ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { desc_ = desc; for (auto &block_desc : *desc_.mutable_blocks()) { - blocks_.emplace_back(new BlockDescBind(this, &block_desc)); + blocks_.emplace_back(new BlockDesc(this, &block_desc)); } } -ProgramDescBind::ProgramDescBind(const std::string &binary_str) { +ProgramDesc::ProgramDesc(const std::string &binary_str) { PADDLE_ENFORCE(desc_.ParseFromString(binary_str), "Fail to parse program_desc from binary string."); for (auto &block_desc : *desc_.mutable_blocks()) { - blocks_.emplace_back(new BlockDescBind(this, &block_desc)); + blocks_.emplace_back(new BlockDesc(this, &block_desc)); } } diff --git a/paddle/framework/program_desc.h b/paddle/framework/program_desc.h index affec491ca..15a962bb69 100644 --- a/paddle/framework/program_desc.h +++ b/paddle/framework/program_desc.h @@ -23,23 +23,23 @@ limitations under the License. */ namespace paddle { namespace framework { -class BlockDescBind; +class BlockDesc; -class ProgramDescBind { +class ProgramDesc { public: - ProgramDescBind(); + ProgramDesc(); - explicit ProgramDescBind(const proto::ProgramDesc &desc); + explicit ProgramDesc(const proto::ProgramDesc &desc); - ProgramDescBind(const ProgramDescBind &o); + ProgramDesc(const ProgramDesc &o); - explicit ProgramDescBind(const std::string &binary_str); + explicit ProgramDesc(const std::string &binary_str); - BlockDescBind *AppendBlock(const BlockDescBind &parent); + BlockDesc *AppendBlock(const BlockDesc &parent); - BlockDescBind *MutableBlock(size_t idx) { return blocks_[idx].get(); } + BlockDesc *MutableBlock(size_t idx) { return blocks_[idx].get(); } - const BlockDescBind &Block(size_t idx) const { return *blocks_[idx]; } + const BlockDesc &Block(size_t idx) const { return *blocks_[idx]; } size_t Size() const { return blocks_.size(); } @@ -48,7 +48,7 @@ class ProgramDescBind { private: proto::ProgramDesc desc_; - std::vector> blocks_; + std::vector> blocks_; }; } // namespace framework } // namespace paddle diff --git a/paddle/framework/program_desc_test.cc b/paddle/framework/program_desc_test.cc index c4fb28f2cc..a49886f7ea 100644 --- a/paddle/framework/program_desc_test.cc +++ b/paddle/framework/program_desc_test.cc @@ -19,7 +19,7 @@ namespace paddle { namespace framework { TEST(ProgramDesc, copy_ctor) { - ProgramDescBind program; + ProgramDesc program; auto* global_block = program.MutableBlock(0); auto* x = global_block->Var("X"); x->SetType(proto::VarDesc_VarType_LOD_TENSOR); @@ -42,12 +42,12 @@ TEST(ProgramDesc, copy_ctor) { out->SetType(proto::VarDesc_VarType_LOD_TENSOR); op->SetOutput("Y", {out->Name()}); - ProgramDescBind program_copy(program); + ProgramDesc program_copy(program); auto* global_block_copy = program_copy.MutableBlock(0); ASSERT_NE(global_block, global_block_copy); - auto assert_same_var = [&](const std::string& name, VarDescBind* var_before) { + auto assert_same_var = [&](const std::string& name, VarDesc* var_before) { ASSERT_TRUE(global_block_copy->HasVar(name)); auto* copy = global_block_copy->Var(name); ASSERT_NE(copy, var_before); @@ -81,7 +81,7 @@ TEST(ProgramDesc, copy_ctor) { } TEST(ProgramDescBind, serialize_and_deserialize) { - ProgramDescBind program_origin; + ProgramDesc program_origin; auto* global_block = program_origin.MutableBlock(0); auto* x = global_block->Var("X"); x->SetType(proto::VarDesc_VarType_LOD_TENSOR); @@ -107,11 +107,11 @@ TEST(ProgramDescBind, serialize_and_deserialize) { std::string binary_str; program_origin.Proto()->SerializeToString(&binary_str); - ProgramDescBind program_restored(binary_str); + ProgramDesc program_restored(binary_str); auto* global_block_restored = program_restored.MutableBlock(0); ASSERT_NE(global_block, global_block_restored); - auto assert_same_var = [&](const std::string& name, VarDescBind* var_before) { + auto assert_same_var = [&](const std::string& name, VarDesc* var_before) { ASSERT_TRUE(global_block_restored->HasVar(name)); auto* restored = global_block_restored->Var(name); ASSERT_NE(restored, var_before); diff --git a/paddle/framework/prune_test.cc b/paddle/framework/prune_test.cc index 47fe4b0636..bdd5765943 100644 --- a/paddle/framework/prune_test.cc +++ b/paddle/framework/prune_test.cc @@ -29,7 +29,7 @@ namespace ops = paddle::operators; void AddOp(const std::string &type, const f::VariableNameMap &inputs, const f::VariableNameMap &outputs, f::AttributeMap attrs, - paddle::framework::BlockDescBind *block) { + paddle::framework::BlockDesc *block) { // insert output for (auto kv : outputs) { for (auto v : kv.second) { @@ -51,8 +51,8 @@ void AddOp(const std::string &type, const f::VariableNameMap &inputs, } TEST(Prune, one_operator) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); AddOp("one_one", {{"input", {"a"}}}, {{"output", {"b"}}}, f::AttributeMap{}, block); @@ -69,8 +69,8 @@ TEST(Prune, one_operator) { } TEST(Prune, forward) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); AddOp("one_one", {{"input", {"a"}}}, {{"output", {"b"}}}, f::AttributeMap{}, block); @@ -92,8 +92,8 @@ TEST(Prune, forward) { } TEST(Prune, multi_input_op) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); AddOp("one_one", {{"input", {"a0"}}}, {{"output", {"b0"}}}, f::AttributeMap{}, block); @@ -113,8 +113,8 @@ TEST(Prune, multi_input_op) { } TEST(Prune, multi_output_op) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); AddOp("one_two", {{"input", {"a"}}}, {{"output", {"b", "c"}}}, f::AttributeMap{}, block); @@ -132,8 +132,8 @@ TEST(Prune, multi_output_op) { } TEST(Prune, multi_target) { - f::ProgramDescBind program; - f::BlockDescBind *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); AddOp("one_two", {{"input", {"a"}}}, {{"output", {"b", "c"}}}, f::AttributeMap{}, block); diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h index baeb98c9bd..da152e8b9d 100644 --- a/paddle/framework/type_defs.h +++ b/paddle/framework/type_defs.h @@ -25,11 +25,9 @@ namespace paddle { namespace framework { class OperatorBase; -class OpDescBind; -class BlockDescBind; -class BlockDesc; +class OpDesc; class InferShapeContext; -class BlockDescBind; +class BlockDesc; using VariableNameMap = std::map>; @@ -37,7 +35,7 @@ using VariableNameMap = std::map>; using Attribute = boost::variant, std::vector, std::vector, bool, - std::vector, BlockDescBind*>; + std::vector, BlockDesc*>; using AttributeMap = std::unordered_map; @@ -45,13 +43,13 @@ using OpCreator = std::function; -using GradOpMakerFN = std::function>( - const OpDescBind&, const std::unordered_set& /*no_grad_set*/, +using GradOpMakerFN = std::function>( + const OpDesc&, const std::unordered_set& /*no_grad_set*/, std::unordered_map* /*grad_to_var*/, - const std::vector& grad_block)>; + const std::vector& grad_block)>; -using InferVarTypeFN = std::function; +using InferVarTypeFN = + std::function; using InferShapeFN = std::function; diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc index 2180827767..bd8973eeb3 100644 --- a/paddle/framework/var_desc.cc +++ b/paddle/framework/var_desc.cc @@ -18,29 +18,27 @@ limitations under the License. */ namespace paddle { namespace framework { -proto::VarDesc::VarType VarDescBind::GetType() const { return desc_.type(); } +proto::VarDesc::VarType VarDesc::GetType() const { return desc_.type(); } -void VarDescBind::SetType(proto::VarDesc::VarType type) { - desc_.set_type(type); -} +void VarDesc::SetType(proto::VarDesc::VarType type) { desc_.set_type(type); } -void VarDescBind::SetShape(const std::vector &dims) { +void VarDesc::SetShape(const std::vector &dims) { VectorToRepeated(dims, mutable_tensor_desc()->mutable_dims()); } -void VarDescBind::SetDataType(proto::DataType data_type) { +void VarDesc::SetDataType(proto::DataType data_type) { mutable_tensor_desc()->set_data_type(data_type); } -std::vector VarDescBind::Shape() const { +std::vector VarDesc::Shape() const { return RepeatedToVector(tensor_desc().dims()); } -proto::DataType VarDescBind::GetDataType() const { +proto::DataType VarDesc::GetDataType() const { return tensor_desc().data_type(); } -void VarDescBind::SetLoDLevel(int32_t lod_level) { +void VarDesc::SetLoDLevel(int32_t lod_level) { switch (desc_.type()) { case proto::VarDesc::LOD_TENSOR: desc_.mutable_lod_tensor()->set_lod_level(lod_level); @@ -54,7 +52,7 @@ void VarDescBind::SetLoDLevel(int32_t lod_level) { } } -int32_t VarDescBind::GetLodLevel() const { +int32_t VarDesc::GetLodLevel() const { switch (desc_.type()) { case proto::VarDesc::LOD_TENSOR: return desc_.lod_tensor().lod_level(); @@ -66,7 +64,7 @@ int32_t VarDescBind::GetLodLevel() const { } } -const proto::TensorDesc &VarDescBind::tensor_desc() const { +const proto::TensorDesc &VarDesc::tensor_desc() const { PADDLE_ENFORCE(desc_.has_type(), "invoke TensorDesc must after set type"); switch (desc_.type()) { case proto::VarDesc::SELECTED_ROWS: @@ -80,7 +78,7 @@ const proto::TensorDesc &VarDescBind::tensor_desc() const { } } -proto::TensorDesc *VarDescBind::mutable_tensor_desc() { +proto::TensorDesc *VarDesc::mutable_tensor_desc() { PADDLE_ENFORCE(desc_.has_type(), "invoke MutableTensorDesc must after set type"); switch (desc_.type()) { diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h index 335a864cab..4fd2abe7fb 100644 --- a/paddle/framework/var_desc.h +++ b/paddle/framework/var_desc.h @@ -53,14 +53,14 @@ inline void VectorToRepeated(const std::vector &vec, } } -class VarDescBind { +class VarDesc { public: - explicit VarDescBind(const std::string &name) { + explicit VarDesc(const std::string &name) { desc_.set_name(name); desc_.set_type(proto::VarDesc::LOD_TENSOR); } - explicit VarDescBind(const proto::VarDesc &desc) : desc_(desc) {} + explicit VarDesc(const proto::VarDesc &desc) : desc_(desc) {} proto::VarDesc *Proto() { return &desc_; } diff --git a/paddle/framework/var_type_inference.h b/paddle/framework/var_type_inference.h index 32abbeb334..1a4dca05f7 100644 --- a/paddle/framework/var_type_inference.h +++ b/paddle/framework/var_type_inference.h @@ -21,8 +21,7 @@ namespace framework { class VarTypeInference { public: virtual ~VarTypeInference() {} - virtual void operator()(const OpDescBind& op_desc, - BlockDescBind* block) const = 0; + virtual void operator()(const OpDesc& op_desc, BlockDesc* block) const = 0; }; } // namespace framework diff --git a/paddle/framework/var_type_inference_test.cc b/paddle/framework/var_type_inference_test.cc index 8b465cbc59..92f333c558 100644 --- a/paddle/framework/var_type_inference_test.cc +++ b/paddle/framework/var_type_inference_test.cc @@ -33,8 +33,7 @@ class SumOpMaker : public OpProtoAndCheckerMaker { class SumOpVarTypeInference : public VarTypeInference { public: - void operator()(const OpDescBind &op_desc, - BlockDescBind *block) const override { + void operator()(const OpDesc &op_desc, BlockDesc *block) const override { auto &inputs = op_desc.Input("X"); auto default_var_type = proto::VarDesc::SELECTED_ROWS; @@ -62,7 +61,7 @@ namespace paddle { namespace framework { TEST(InferVarType, sum_op) { - ProgramDescBind prog; + ProgramDesc prog; auto *op = prog.MutableBlock(0)->AppendOp(); op->SetType("sum"); op->SetInput("X", {"test_a", "test_b", "test_c"}); @@ -85,7 +84,7 @@ TEST(InferVarType, sum_op) { } TEST(InferVarType, sum_op_without_infer_var_type) { - ProgramDescBind prog; + ProgramDesc prog; auto *op = prog.MutableBlock(0)->AppendOp(); op->SetType("sum_without_infer_var_type"); op->SetInput("X", {"test2_a", "test2_b", "test2_c"}); diff --git a/paddle/operators/array_to_lod_tensor_op.cc b/paddle/operators/array_to_lod_tensor_op.cc index aafdb8fb24..b6ca3cad94 100644 --- a/paddle/operators/array_to_lod_tensor_op.cc +++ b/paddle/operators/array_to_lod_tensor_op.cc @@ -149,14 +149,14 @@ class ArrayToLoDTensorGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("lod_tensor_to_array"); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetInput("RankTable", Input("RankTable")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/assign_op.cc b/paddle/operators/assign_op.cc index 0d98755aa0..a914ff4ba9 100644 --- a/paddle/operators/assign_op.cc +++ b/paddle/operators/assign_op.cc @@ -121,12 +121,12 @@ class AssignGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *op = new framework::OpDesc(); op->SetType("assign"); op->SetInput("X", OutputGrad("Out")); op->SetOutput("Out", InputGrad("X")); - return std::unique_ptr(op); + return std::unique_ptr(op); } }; diff --git a/paddle/operators/beam_search_decode_op.cc b/paddle/operators/beam_search_decode_op.cc index ceb20cbe18..32756faac5 100644 --- a/paddle/operators/beam_search_decode_op.cc +++ b/paddle/operators/beam_search_decode_op.cc @@ -119,8 +119,8 @@ class BeamSearchDecodeInferShape : public framework::InferShapeBase { class BeamSearchDecodeInferVarType : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind& op_desc, - framework::BlockDescBind* block) const override { + void operator()(const framework::OpDesc& op_desc, + framework::BlockDesc* block) const override { for (auto& o : op_desc.Output("SentenceIds")) { block->Var(o)->SetType(framework::proto::VarDesc::LOD_TENSOR); } diff --git a/paddle/operators/cast_op.cc b/paddle/operators/cast_op.cc index 927a32645c..fc6da06490 100644 --- a/paddle/operators/cast_op.cc +++ b/paddle/operators/cast_op.cc @@ -52,14 +52,14 @@ class CastOpGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto grad = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto grad = new framework::OpDesc(); grad->SetType("cast"); grad->SetInput("X", OutputGrad("Out")); grad->SetOutput("Out", InputGrad("X")); grad->SetAttr("out_dtype", GetAttr("in_dtype")); grad->SetAttr("in_dtype", GetAttr("out_dtype")); - return std::unique_ptr(grad); + return std::unique_ptr(grad); } }; diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 5fe362c1b6..00048a10ca 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -65,7 +65,7 @@ class ConditionalBlockOp : public ConditionalOp { scopes->front() = &scope.NewScope(); auto &cur_scope = *scopes->front(); - auto *block = Attr("sub_block"); + auto *block = Attr("sub_block"); framework::Executor exec(dev_ctx); exec.Run(*block->Program(), &cur_scope, block->ID(), false); } @@ -86,7 +86,7 @@ class ConditionalBlockOpProtoMaker : public framework::OpProtoAndCheckerMaker { "(std::vector) The step scope of conditional block. To " "unify the conditional block, rnn and while op, the type of " "scope is std::vector"); - AddAttr( + AddAttr( "sub_block", "The step block of conditional block operator"); AddComment(R"DOC(Conditional block operator @@ -116,7 +116,7 @@ class ConditionalBlockGradOp : public ConditionalOp { auto &scopes = scope_var->Get>(); framework::Scope &cur_scope = *scopes[0]; - auto *block = Attr("sub_block"); + auto *block = Attr("sub_block"); framework::Executor exec(dev_ctx); exec.Run(*block->Program(), &cur_scope, block->ID(), false); @@ -170,8 +170,8 @@ class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto grad_op = new framework::OpDesc(); grad_op->SetType("conditional_block_grad"); grad_op->SetInput("X", Input("X")); grad_op->SetInput("Params", Input("Params")); @@ -181,7 +181,7 @@ class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); grad_op->SetOutput(framework::GradVarName("Params"), InputGrad("Params")); grad_op->SetBlockAttr("sub_block", *this->grad_block_[0]); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/increment_op.cc b/paddle/operators/increment_op.cc index 3a53ea89dc..789c92102d 100644 --- a/paddle/operators/increment_op.cc +++ b/paddle/operators/increment_op.cc @@ -93,13 +93,13 @@ class IncrementGradOpMaker : public framework::SingleGradOpDescMaker { public: using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("increment"); grad_op->SetInput("X", Output("Out")); grad_op->SetOutput("Out", Input("X")); grad_op->SetAttr("step", -boost::get(GetAttr("step"))); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/lod_rank_table_op.cc b/paddle/operators/lod_rank_table_op.cc index 46577d0c58..2d67046bfe 100644 --- a/paddle/operators/lod_rank_table_op.cc +++ b/paddle/operators/lod_rank_table_op.cc @@ -63,8 +63,8 @@ class LoDRankTableInferShape : public framework::InferShapeBase { class LoDRankTableInferVarType : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind &op_desc, - framework::BlockDescBind *block) const override { + void operator()(const framework::OpDesc &op_desc, + framework::BlockDesc *block) const override { for (auto &o : op_desc.Output("Out")) { block->FindRecursiveOrCreateVar(o)->SetType( framework::proto::VarDesc::LOD_RANK_TABLE); diff --git a/paddle/operators/lod_tensor_to_array_op.cc b/paddle/operators/lod_tensor_to_array_op.cc index 33af0e819f..643f8859f3 100644 --- a/paddle/operators/lod_tensor_to_array_op.cc +++ b/paddle/operators/lod_tensor_to_array_op.cc @@ -127,8 +127,8 @@ class LoDTensorToArrayInferShape : public framework::InferShapeBase { class LoDTensorToArrayInferVarType : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind &op_desc, - framework::BlockDescBind *block) const override { + void operator()(const framework::OpDesc &op_desc, + framework::BlockDesc *block) const override { for (auto &out_var : op_desc.Output("Out")) { block->Var(out_var)->SetType(framework::proto::VarDesc::LOD_TENSOR_ARRAY); } @@ -140,14 +140,14 @@ class LoDTensorToArrayGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("array_to_lod_tensor"); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetInput("RankTable", Input("RankTable")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc index 606b44808e..0a9defa8c5 100644 --- a/paddle/operators/lookup_table_op.cc +++ b/paddle/operators/lookup_table_op.cc @@ -108,8 +108,8 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { class LookupTableOpGradVarTypeInference : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind& op_desc, - framework::BlockDescBind* block) const override { + void operator()(const framework::OpDesc& op_desc, + framework::BlockDesc* block) const override { auto out_var_name = op_desc.Output(framework::GradVarName("W")).front(); auto attr = op_desc.GetAttr("is_sparse"); bool is_sparse = boost::get(attr); diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index e27f9eeac6..411f4d14ef 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -60,13 +60,13 @@ class MeanGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto* grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto* grad_op = new framework::OpDesc(); grad_op->SetType("mean_grad"); grad_op->SetInput("X", Input("X")); grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/merge_lod_tensor_op.cc b/paddle/operators/merge_lod_tensor_op.cc index ec76cfdf27..5edf29c3af 100644 --- a/paddle/operators/merge_lod_tensor_op.cc +++ b/paddle/operators/merge_lod_tensor_op.cc @@ -161,15 +161,15 @@ class MergeLoDTensorGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("split_lod_tensor"); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetInput("Mask", Input("Mask")); grad_op->SetOutput("OutTrue", InputGrad("InTrue")); grad_op->SetOutput("OutFalse", InputGrad("InFalse")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index eb65fededf..2e9cc9d29d 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -70,12 +70,11 @@ class MinusGradMaker : public framework::GradOpDescMakerBase { public: using framework::GradOpDescMakerBase::GradOpDescMakerBase; - std::vector> operator()() - const override { - std::vector> ops; + std::vector> operator()() const override { + std::vector> ops; auto x_g = InputGrad("X"); if (!x_g.empty()) { - auto *x_g_op = new framework::OpDescBind(); + auto *x_g_op = new framework::OpDesc(); x_g_op->SetType("scale"); x_g_op->SetInput("X", OutputGrad("Out")); x_g_op->SetOutput("Out", x_g); @@ -85,7 +84,7 @@ class MinusGradMaker : public framework::GradOpDescMakerBase { auto y_g = InputGrad("Y"); if (!y_g.empty()) { - auto *y_g_op = new framework::OpDescBind(); + auto *y_g_op = new framework::OpDesc(); y_g_op->SetType("scale"); y_g_op->SetInput("X", OutputGrad("Out")); y_g_op->SetOutput("Out", y_g); diff --git a/paddle/operators/nccl_op_test.cu.cc b/paddle/operators/nccl_op_test.cu.cc index d747cc0cf5..c1046aadaf 100644 --- a/paddle/operators/nccl_op_test.cu.cc +++ b/paddle/operators/nccl_op_test.cu.cc @@ -65,7 +65,7 @@ class NCCLTester : public ::testing::Test { } void NCCLInitOp() { - std::unique_ptr op1(new f::OpDescBind); + std::unique_ptr op1(new f::OpDesc); op1->SetType("ncclInit"); op1->SetOutput("Communicator", {"comm"}); @@ -81,10 +81,9 @@ class NCCLTester : public ::testing::Test { } template - void PerThreadProgram(int gpu_id, const f::OpDescBind &op_desc, - f::Scope *scope) { + void PerThreadProgram(int gpu_id, const f::OpDesc &op_desc, f::Scope *scope) { std::unique_lock lk(mu); - const f::OpDescBind *op1 = &op_desc; + const f::OpDesc *op1 = &op_desc; p::GPUPlace place(gpu_id); auto &ctx = dev_ctxs.at(gpu_id); @@ -125,7 +124,7 @@ class NCCLTester : public ::testing::Test { // ncclInitOp with desc TEST(NCCL, ncclInitOp) { - std::unique_ptr op_desc(new f::OpDescBind); + std::unique_ptr op_desc(new f::OpDesc); op_desc->SetType("ncclInit"); op_desc->SetOutput("Communicator", {"x1"}); @@ -145,7 +144,7 @@ TEST(NCCL, ncclInitOp) { // ncclAllReduceOp with desc TEST_F(NCCLTester, ncclAllReduceOp) { - std::unique_ptr op2(new f::OpDescBind); + std::unique_ptr op2(new f::OpDesc); op2->SetType("ncclAllReduce"); op2->SetInput("X", {"st"}); op2->SetInput("Communicator", {"comm"}); @@ -192,7 +191,7 @@ TEST_F(NCCLTester, ncclAllReduceOp) { // ncclReduceOp with desc TEST_F(NCCLTester, ncclReduceOp) { - std::unique_ptr op2(new f::OpDescBind); + std::unique_ptr op2(new f::OpDesc); const int kRoot = 0; op2->SetType("ncclReduce"); op2->SetInput("X", {"st"}); @@ -240,7 +239,7 @@ TEST_F(NCCLTester, ncclReduceOp) { // ncclBcastOp with desc TEST_F(NCCLTester, ncclBcastOp) { - std::unique_ptr op2(new f::OpDescBind); + std::unique_ptr op2(new f::OpDesc); const int kRoot = 5; op2->SetType("ncclBcast"); op2->SetInput("X", {"st"}); diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 8d2d031fcd..40f7a7eed5 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -116,14 +116,14 @@ class PadOpGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto* bind = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto* bind = new framework::OpDesc(); bind->SetInput("X", Input("X")); bind->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); bind->SetOutput(framework::GradVarName("X"), InputGrad("X")); bind->SetAttrMap(Attrs()); bind->SetType("pad_grad"); - return std::unique_ptr(bind); + return std::unique_ptr(bind); } }; diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index ca3a063553..4273c12354 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -234,7 +234,7 @@ class RecurrentOp : public RecurrentBase { auto reverse = Attr(kReverse); framework::Executor executor(dev_ctx); - auto *block = Attr(kStepBlock); + auto *block = Attr(kStepBlock); auto *program = block->Program(); for (size_t i = 0; i < seq_len; ++i) { @@ -317,7 +317,7 @@ class RecurrentGradOp : public RecurrentBase { auto reverse = Attr(kReverse); framework::Executor executor(dev_ctx); - auto *block = Attr(kStepBlock); + auto *block = Attr(kStepBlock); auto *program = block->Program(); for (size_t step_id = 0; step_id < seq_len; ++step_id) { @@ -522,8 +522,7 @@ The ex-state means the state value in the ex-timestep or the previous time step string::Sprintf( "The state variable names. [%s, %s, %s] must be the same order", kExStates, kStates, kInitStateGrads)); - AddAttr(kStepBlock, - "The step block inside RNN"); + AddAttr(kStepBlock, "The step block inside RNN"); AddAttr(kReverse, R"DOC(Calculate RNN reversely or not. By default reverse=False @@ -565,8 +564,8 @@ class RecurrentGradOpDescMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - virtual std::unique_ptr Apply() const { - auto *grad = new framework::OpDescBind(); + virtual std::unique_ptr Apply() const { + auto *grad = new framework::OpDesc(); grad->SetType("recurrent_grad"); for (auto &input_param : this->InputNames()) { grad->SetInput(input_param, this->Input(input_param)); @@ -588,7 +587,7 @@ class RecurrentGradOpDescMaker : public framework::SingleGradOpDescMaker { grad->SetAttrMap(this->Attrs()); grad->SetBlockAttr(kStepBlock, *grad_block_[0]); - return std::unique_ptr(grad); + return std::unique_ptr(grad); } }; diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index 98170c0d1b..ee39888713 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -58,13 +58,13 @@ class ScaleGradMaker : public framework::SingleGradOpDescMaker { public: using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("scale"); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttr("scale", GetAttr("scale")); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/shrink_rnn_memory_op.cc b/paddle/operators/shrink_rnn_memory_op.cc index 92dbe126bc..48194a547b 100644 --- a/paddle/operators/shrink_rnn_memory_op.cc +++ b/paddle/operators/shrink_rnn_memory_op.cc @@ -136,14 +136,14 @@ class ShrinkRNNGradOpMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *op = new framework::OpDesc(); op->SetType("shrink_rnn_memory_grad"); op->SetInput("X", Input("X")); op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); op->SetOutput(framework::GradVarName("X"), InputGrad("X")); op->SetAttrMap(Attrs()); - return std::unique_ptr(op); + return std::unique_ptr(op); } }; diff --git a/paddle/operators/sign_op.cc b/paddle/operators/sign_op.cc index b2bfce71a6..b2459fb2f5 100644 --- a/paddle/operators/sign_op.cc +++ b/paddle/operators/sign_op.cc @@ -50,13 +50,13 @@ class SignGradMaker : public framework::SingleGradOpDescMaker { public: using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("scale"); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttr("scale", 0.0f); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index bca3ff1562..d9911a6901 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -173,8 +173,8 @@ class SoftmaxGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto* grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto* grad_op = new framework::OpDesc(); grad_op->SetType("softmax_with_cross_entropy_grad"); grad_op->SetInput("Label", Input("Label")); grad_op->SetInput("Softmax", Output("Softmax")); @@ -183,7 +183,7 @@ class SoftmaxGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetInput(framework::GradVarName("Loss"), OutputGrad("Loss")); grad_op->SetOutput(framework::GradVarName("Logits"), InputGrad("Logits")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/split_lod_tensor_op.cc b/paddle/operators/split_lod_tensor_op.cc index c83b0cbad7..3542d8624f 100644 --- a/paddle/operators/split_lod_tensor_op.cc +++ b/paddle/operators/split_lod_tensor_op.cc @@ -163,8 +163,8 @@ class SplitLoDTensorArrayGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("merge_lod_tensor"); grad_op->SetInput("InTrue", OutputGrad("OutTrue")); grad_op->SetInput("InFalse", OutputGrad("OutFalse")); @@ -172,7 +172,7 @@ class SplitLoDTensorArrayGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetInput("X", Input("X")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/split_op.cc b/paddle/operators/split_op.cc index e8c5fffcd2..4dfae043cb 100644 --- a/paddle/operators/split_op.cc +++ b/paddle/operators/split_op.cc @@ -108,13 +108,13 @@ class SplitGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto op = new framework::OpDesc(); op->SetType("concat"); op->SetInput("X", OutputGrad("Out")); op->SetOutput("Out", InputGrad("X")); op->SetAttrMap(Attrs()); - return std::unique_ptr(op); + return std::unique_ptr(op); } }; diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index c56fc1f10b..36fb5bd29d 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -115,8 +115,8 @@ the LoD information with the first input. class SumOpVarTypeInference : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind& op_desc, - framework::BlockDescBind* block) const override { + void operator()(const framework::OpDesc& op_desc, + framework::BlockDesc* block) const override { auto& inputs = op_desc.Input("X"); auto var_type = framework::proto::VarDesc::SELECTED_ROWS; @@ -169,20 +169,19 @@ class SumGradMaker : public framework::GradOpDescMakerBase { public: using framework::GradOpDescMakerBase::GradOpDescMakerBase; - std::vector> operator()() - const override { + std::vector> operator()() const override { auto x_grads = InputGrad("X"); - std::vector> grad_ops; + std::vector> grad_ops; grad_ops.reserve(x_grads.size()); auto og = OutputGrad("Out"); std::transform(x_grads.begin(), x_grads.end(), std::back_inserter(grad_ops), [&og](const std::string& x_grad) { - auto* grad_op = new framework::OpDescBind(); + auto* grad_op = new framework::OpDesc(); grad_op->SetType("scale"); grad_op->SetInput("X", og); grad_op->SetOutput("Out", {x_grad}); grad_op->SetAttr("scale", 1.0f); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); }); return grad_ops; } diff --git a/paddle/operators/tensor_array_read_write_op.cc b/paddle/operators/tensor_array_read_write_op.cc index 337b7555c7..90cbc19d1b 100644 --- a/paddle/operators/tensor_array_read_write_op.cc +++ b/paddle/operators/tensor_array_read_write_op.cc @@ -96,8 +96,8 @@ class WriteToArrayInferShape : public framework::InferShapeBase { class WriteToArrayInferVarType : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind &op_desc, - framework::BlockDescBind *block) const override { + void operator()(const framework::OpDesc &op_desc, + framework::BlockDesc *block) const override { auto x_name = op_desc.Input("X")[0]; auto out_name = op_desc.Output("Out")[0]; VLOG(10) << "Set Variable " << out_name << " as LOD_TENSOR_ARRAY"; @@ -175,14 +175,14 @@ class WriteToArrayGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("read_from_array"); grad_op->SetInput("I", Input("I")); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; @@ -191,14 +191,14 @@ class ReadFromArrayGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("write_to_array"); grad_op->SetInput("I", Input("I")); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetAttrMap(Attrs()); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/while_op.cc b/paddle/operators/while_op.cc index 56a01e56d7..324c8b98c4 100644 --- a/paddle/operators/while_op.cc +++ b/paddle/operators/while_op.cc @@ -46,7 +46,7 @@ class WhileOp : public framework::OperatorBase { PADDLE_ENFORCE_EQ(cond.dims(), paddle::framework::make_ddim({1})); framework::Executor executor(dev_ctx); - auto *block = Attr(kStepBlock); + auto *block = Attr(kStepBlock); auto *program = block->Program(); auto step_scopes = @@ -82,8 +82,8 @@ class WhileOpMaker : public framework::OpProtoAndCheckerMaker { "(StepScopeVar) A vector of local scope, which size equals the " "step number of While Op. The i'th scope storages temporary " "variables generated in the i'th step."); - AddAttr(kStepBlock, - "The step block inside WhileOp"); + AddAttr(kStepBlock, + "The step block inside WhileOp"); AddComment(R"DOC( )DOC"); } @@ -99,7 +99,7 @@ class WhileGradOp : public framework::OperatorBase { void Run(const framework::Scope &scope, const platform::DeviceContext &dev_ctx) const override { framework::Executor executor(dev_ctx); - auto *block = Attr(kStepBlock); + auto *block = Attr(kStepBlock); auto *program = block->Program(); auto *step_scopes = @@ -209,8 +209,8 @@ class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad = new framework::OpDesc(); grad->SetType("while_grad"); grad->SetInput(kParameters, Input(kParameters)); @@ -279,14 +279,14 @@ class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { // while operator could be renamed. grad->SetAttr("original_output_grad", extra_inputs_list); - return std::unique_ptr(grad); + return std::unique_ptr(grad); } }; class WhileGradOpVarTypeInference : public framework::VarTypeInference { public: - void operator()(const framework::OpDescBind &op_desc, - framework::BlockDescBind *block) const override { + void operator()(const framework::OpDesc &op_desc, + framework::BlockDesc *block) const override { auto p_names = op_desc.Input(kParameters); auto pg_names = op_desc.Output(framework::GradVarName(kParameters)); diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index de26184d01..88e9cdadd8 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -108,21 +108,21 @@ static py::bytes SerializeMessage(T &self) { // Bind Methods void BindProgramDesc(py::module &m) { - py::class_(m, "ProgramDesc", "") + py::class_(m, "ProgramDesc", "") .def(py::init<>()) .def("__init__", - [](ProgramDescBind &self, const ProgramDescBind &other) { - new (&self) ProgramDescBind(other); + [](ProgramDesc &self, const ProgramDesc &other) { + new (&self) ProgramDesc(other); }) .def("__init__", - [](ProgramDescBind &self, const py::bytes &binary_str) { + [](ProgramDesc &self, const py::bytes &binary_str) { std::string str(binary_str); - new (&self) ProgramDescBind(str); + new (&self) ProgramDesc(str); }) - .def("append_block", &ProgramDescBind::AppendBlock, + .def("append_block", &ProgramDesc::AppendBlock, py::return_value_policy::reference) .def("append_backward", - [](ProgramDescBind &program_desc, const VarDescBind &target, + [](ProgramDesc &program_desc, const VarDesc &target, const std::unordered_set &no_grad_vars) { ParamGradInfoMap param_grad_map = AppendBackward(program_desc, target, no_grad_vars); @@ -138,12 +138,12 @@ void BindProgramDesc(py::module &m) { } return retv; }) - .def("block", &ProgramDescBind::MutableBlock, + .def("block", &ProgramDesc::MutableBlock, py::return_value_policy::reference) - .def("num_blocks", &ProgramDescBind::Size) - .def("serialize_to_string", SerializeMessage) + .def("num_blocks", &ProgramDesc::Size) + .def("serialize_to_string", SerializeMessage) .def("parse_from_string", - [](ProgramDescBind &program_desc, const std::string &data) { + [](ProgramDesc &program_desc, const std::string &data) { proto::ProgramDesc *desc = program_desc.Proto(); PADDLE_ENFORCE(desc->ParseFromString(data), "Fail to parse ProgramDesc from string. This could " @@ -152,35 +152,34 @@ void BindProgramDesc(py::module &m) { } void BindBlockDesc(py::module &m) { - py::class_(m, "BlockDesc", "") - .def_property_readonly("id", &BlockDescBind::ID) - .def_property_readonly("parent", &BlockDescBind::Parent) - .def("append_op", &BlockDescBind::AppendOp, + py::class_(m, "BlockDesc", "") + .def_property_readonly("id", &BlockDesc::ID) + .def_property_readonly("parent", &BlockDesc::Parent) + .def("append_op", &BlockDesc::AppendOp, py::return_value_policy::reference) - .def("prepend_op", &BlockDescBind::PrependOp, + .def("prepend_op", &BlockDesc::PrependOp, py::return_value_policy::reference) .def("var", - [](BlockDescBind &self, py::bytes byte_name) { + [](BlockDesc &self, py::bytes byte_name) { std::string name = byte_name; return self.Var(name); }, py::return_value_policy::reference) .def("has_var", - [](BlockDescBind &self, py::bytes byte_name) { + [](BlockDesc &self, py::bytes byte_name) { std::string name = byte_name; return self.HasVar(name); }) .def("find_var", - [](BlockDescBind &self, py::bytes byte_name) { + [](BlockDesc &self, py::bytes byte_name) { std::string name = byte_name; return self.FindVar(name); }, py::return_value_policy::reference) - .def("all_vars", &BlockDescBind::AllVars, - py::return_value_policy::reference) - .def("op_size", &BlockDescBind::OpSize) - .def("op", &BlockDescBind::Op, py::return_value_policy::reference) - .def("serialize_to_string", SerializeMessage); + .def("all_vars", &BlockDesc::AllVars, py::return_value_policy::reference) + .def("op_size", &BlockDesc::OpSize) + .def("op", &BlockDesc::Op, py::return_value_policy::reference) + .def("serialize_to_string", SerializeMessage); } void BindVarDsec(py::module &m) { @@ -193,25 +192,25 @@ void BindVarDsec(py::module &m) { .value("FP32", proto::DataType::FP32) .value("FP64", proto::DataType::FP64); - py::class_ var_desc(m, "VarDesc", ""); + py::class_ var_desc(m, "VarDesc", ""); var_desc .def("name", - [](const VarDescBind &self) { + [](const VarDesc &self) { py::bytes name = self.Name(); return name; }, py::return_value_policy::reference) - .def("set_shape", &VarDescBind::SetShape) - .def("set_dtype", &VarDescBind::SetDataType) - .def("shape", &VarDescBind::Shape, py::return_value_policy::reference) - .def("dtype", &VarDescBind::GetDataType) - .def("lod_level", &VarDescBind::GetLodLevel) - .def("set_lod_level", &VarDescBind::SetLoDLevel) - .def("type", &VarDescBind::GetType) - .def("set_type", &VarDescBind::SetType) - .def("serialize_to_string", SerializeMessage) - .def("persistable", &VarDescBind::Persistable) - .def("set_persistable", &VarDescBind::SetPersistable); + .def("set_shape", &VarDesc::SetShape) + .def("set_dtype", &VarDesc::SetDataType) + .def("shape", &VarDesc::Shape, py::return_value_policy::reference) + .def("dtype", &VarDesc::GetDataType) + .def("lod_level", &VarDesc::GetLodLevel) + .def("set_lod_level", &VarDesc::SetLoDLevel) + .def("type", &VarDesc::GetType) + .def("set_type", &VarDesc::SetType) + .def("serialize_to_string", SerializeMessage) + .def("persistable", &VarDesc::Persistable) + .def("set_persistable", &VarDesc::SetPersistable); py::enum_(var_desc, "VarType", "") .value("LOD_TENSOR", proto::VarDesc::LOD_TENSOR) @@ -235,26 +234,26 @@ void BindOpDesc(py::module &m) { .value("BOOLS", proto::AttrType::BOOLEANS) .value("BLOCK", proto::AttrType::BLOCK); - py::class_ op_desc(m, "OpDesc", ""); - op_desc.def("type", &OpDescBind::Type) - .def("set_type", &OpDescBind::SetType) - .def("input", &OpDescBind::Input) - .def("input_names", &OpDescBind::InputNames) - .def("set_input", &OpDescBind::SetInput) - .def("output", &OpDescBind::Output) - .def("output_names", &OpDescBind::OutputNames) - .def("set_output", &OpDescBind::SetOutput) - .def("has_attr", &OpDescBind::HasAttr) - .def("attr_type", &OpDescBind::GetAttrType) - .def("attr_names", &OpDescBind::AttrNames) - .def("set_attr", &OpDescBind::SetAttr) - .def("attr", &OpDescBind::GetAttr) - .def("set_block_attr", &OpDescBind::SetBlockAttr) - .def("block_attr", &OpDescBind::GetBlockAttr) - .def("check_attrs", &OpDescBind::CheckAttrs) - .def("infer_shape", &OpDescBind::InferShape) - .def("infer_var_type", &OpDescBind::InferVarType) - .def("serialize_to_string", SerializeMessage); + py::class_ op_desc(m, "OpDesc", ""); + op_desc.def("type", &OpDesc::Type) + .def("set_type", &OpDesc::SetType) + .def("input", &OpDesc::Input) + .def("input_names", &OpDesc::InputNames) + .def("set_input", &OpDesc::SetInput) + .def("output", &OpDesc::Output) + .def("output_names", &OpDesc::OutputNames) + .def("set_output", &OpDesc::SetOutput) + .def("has_attr", &OpDesc::HasAttr) + .def("attr_type", &OpDesc::GetAttrType) + .def("attr_names", &OpDesc::AttrNames) + .def("set_attr", &OpDesc::SetAttr) + .def("attr", &OpDesc::GetAttr) + .def("set_block_attr", &OpDesc::SetBlockAttr) + .def("block_attr", &OpDesc::GetBlockAttr) + .def("check_attrs", &OpDesc::CheckAttrs) + .def("infer_shape", &OpDesc::InferShape) + .def("infer_var_type", &OpDesc::InferVarType) + .def("serialize_to_string", SerializeMessage); } } // namespace pybind diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 31f802d4d2..2d7fe25141 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -266,36 +266,36 @@ All parameter, weight, gradient are variables in Paddle. return ret_values; }); m.def("get_grad_op_descs", - [](const OpDescBind &op_desc, + [](const OpDesc &op_desc, const std::unordered_set &no_grad_set, std::unordered_map &grad_to_var, - const std::vector &grad_sub_block) { - std::vector> grad_op_descs = + const std::vector &grad_sub_block) { + std::vector> grad_op_descs = framework::OpInfoMap::Instance() .Get(op_desc.Type()) .GradOpMaker()(op_desc, no_grad_set, &grad_to_var, grad_sub_block); - std::vector grad_op_desc_ptrs(grad_op_descs.size()); + std::vector grad_op_desc_ptrs(grad_op_descs.size()); std::transform( grad_op_descs.begin(), grad_op_descs.end(), grad_op_desc_ptrs.begin(), - [](std::unique_ptr &p) { return p.release(); }); + [](std::unique_ptr &p) { return p.release(); }); return grad_op_desc_ptrs; }); - m.def("prune", [](const ProgramDescBind &origin, + m.def("prune", [](const ProgramDesc &origin, const std::vector> &targets) { - ProgramDescBind prog_with_targets(origin); + ProgramDesc prog_with_targets(origin); for (const auto &t : targets) { prog_with_targets.MutableBlock(t[0])->Op(t[1])->MarkAsTarget(); } proto::ProgramDesc pruned_desc; Prune(*prog_with_targets.Proto(), &pruned_desc); - return new ProgramDescBind(pruned_desc); + return new ProgramDesc(pruned_desc); }); - m.def("inference_optimize", [](ProgramDescBind &origin) { + m.def("inference_optimize", [](ProgramDesc &origin) { proto::ProgramDesc pruned_desc; InferenceOptimize(*(origin.Proto()), &pruned_desc); - return new ProgramDescBind(pruned_desc); + return new ProgramDesc(pruned_desc); }); m.def_submodule( "var_names", From f56f14929833b5211324d40a47216feca423b7a0 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 21 Dec 2017 15:51:38 +0800 Subject: [PATCH 49/84] fix_output_name --- paddle/framework/backward.cc | 4 ++-- paddle/framework/backward_test.cc | 6 +++--- paddle/operators/fill_zeros_like_op.cc | 8 ++++---- paddle/operators/fill_zeros_like_op.h | 2 +- python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index faf6e60cbd..4688da07d4 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -217,7 +217,7 @@ static std::unique_ptr BackwardRecursive( // If part of input gradient of that operator is not calculated, fill // zero variables to that input gradient. net->AppendOp(OpRegistry::CreateOp("fill_zeros_like", {{"X", {prefix}}}, - {{"Y", {grad_input}}}, + {{"Out", {grad_input}}}, AttributeMap{})); } return false; @@ -396,7 +396,7 @@ std::vector> MakeOpGrad( desc->Rename(in_name, new_name); std::unique_ptr fill_zeros_op( new OpDescBind("fill_zeros_like", {{"X", {prefix}}}, - {{"Y", {new_name}}}, AttributeMap{})); + {{"Out", {new_name}}}, AttributeMap{})); pending_fill_zeros_ops.push_back(std::move(fill_zeros_op)); } } diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 9fe49881d5..6063b4bfc1 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -430,8 +430,8 @@ TEST(Backward, op_part_of_output_are_not_need) { ASSERT_EQ("fill_zeros_like", fill_zero.Type()); ASSERT_EQ(1UL, fill_zero.Inputs("X").size()); ASSERT_EQ("Z", fill_zero.Input("X")); - ASSERT_EQ(1UL, fill_zero.Outputs("Y").size()); - ASSERT_EQ(std::string("Z") + f::kZeroVarSuffix, fill_zero.Output("Y")); + ASSERT_EQ(1UL, fill_zero.Outputs("Out").size()); + ASSERT_EQ(std::string("Z") + f::kZeroVarSuffix, fill_zero.Output("Out")); auto &d_many_out = *net->ops_[1]; ASSERT_EQ("many_output_op_grad", d_many_out.Type()); @@ -772,7 +772,7 @@ TEST(Backward, var_no_grad) { ASSERT_EQ(fill_zero_op->InputNames().size(), 1UL); ASSERT_EQ(fill_zero_op->OutputNames().size(), 1UL); EXPECT_EQ(fill_zero_op->Input("X"), std::vector({"z1"})); - EXPECT_EQ(fill_zero_op->Output("Y"), + EXPECT_EQ(fill_zero_op->Output("Out"), std::vector({std::string("z1") + f::kZeroVarSuffix})); f::OpDescBind *grad_op1 = block->AllOps()[5]; diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index 720c11f5f1..45f3788e1f 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -24,9 +24,9 @@ class FillZerosLikeOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of FillZerosLikeOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Y"), - "Output(Y) of FillZerosLikeOp should not be null."); - ctx->SetOutputDim("Y", ctx->GetInputDim("X")); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of FillZerosLikeOp should not be null."); + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); ctx->ShareLoD("X", /*->*/ "Y"); } }; @@ -37,7 +37,7 @@ class FillZerosLikeOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker *op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of fill-zeros-like op."); - AddOutput("Y", "The variable will be filled up with zeros."); + AddOutput("Out", "The variable will be filled up with zeros."); AddComment(R"DOC( FillZerosLike Operator. diff --git a/paddle/operators/fill_zeros_like_op.h b/paddle/operators/fill_zeros_like_op.h index a6e2941f52..351ecf8b2f 100644 --- a/paddle/operators/fill_zeros_like_op.h +++ b/paddle/operators/fill_zeros_like_op.h @@ -23,7 +23,7 @@ template class FillZerosLikeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* out = context.Output("Y"); + auto* out = context.Output("Out"); out->mutable_data(context.GetPlace()); math::SetConstant setter; diff --git a/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py b/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py index eff8fa87d9..cd91769a22 100644 --- a/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py +++ b/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py @@ -7,7 +7,7 @@ class TestFillZerosLikeOp(OpTest): def setUp(self): self.op_type = "fill_zeros_like" self.inputs = {'X': np.random.random((219, 232)).astype("float32")} - self.outputs = {'Y': np.zeros_like(self.inputs["X"])} + self.outputs = {'Out': np.zeros_like(self.inputs["X"])} def test_check_output(self): self.check_output() From 1a3d4b0d3d037aed9cd2999bbedfcbcd7a98c58c Mon Sep 17 00:00:00 2001 From: QI JUN Date: Thu, 21 Dec 2017 16:13:08 +0800 Subject: [PATCH 50/84] add design doc on keys of operaror kernel type (#6782) * add design doc on keys of operator kernel type * follow comments --- doc/design/operator_kernel_type.md | 91 ++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 doc/design/operator_kernel_type.md diff --git a/doc/design/operator_kernel_type.md b/doc/design/operator_kernel_type.md new file mode 100644 index 0000000000..aa82e96bf7 --- /dev/null +++ b/doc/design/operator_kernel_type.md @@ -0,0 +1,91 @@ +# Design Doc: The Keys of Operator Kernel Type +## Problem +An operator can have different kernel implementations, and each operator will have a map to store the related kernels. Fluid uses `OpKernelType` as a key to identify a unique Kernel. Before an operator runs, an certain kernel must be chosen by a key of `OpKernelType`. Currently, `OpKernelType` is defined as follows: + +```cpp +struct OpKernelType { + platform::Place place_; + proto::DataType data_type_; +}; +``` +For more details, please refer to [codes](https://github.com/PaddlePaddle/Paddle/blob/2d5ec16bc8a09fb8e0f62c89b116b0cd1d333907/paddle/framework/operator.h#L348-L374) in github. + +It contains two keys, `Place` and `DataType`. And these two keys will be hashed to a unique key to represent a certain type of kernel. However, these two keys are not enough. We need a more complete representation of `OpKernelType`. + +We often implement a kernel of an operator with some computing library in certain device(place). Please remind that computing library and device are not one-to-one corresponding. A device can have a lot of computing libraries and a computing library can also support several devices. + +For example, Eigen library can support Nvidia GPU/AMD GPU/CPU. And MKLDNN library can support Intel CPU/Intel FPGA. Both `Place` and `Library` should be a key of `OpKernelType`. + +It's obvious that different DataTypes, like fp64/fp32/int8 will have different kernels. But the data layout of a Tensor will also lead to different implementation. Please refer to the batch norm operator [kernels](https://github.com/PaddlePaddle/Paddle/blob/a948fac4d0ad7e0412d373b8aabeb711c2899563/paddle/operators/batch_norm_op.cc#L180-L209). Data Layout should also be taken into consideration. + +## Solution + +There are four keys to determine a kernel type of an operator: `Place`/`Library`/`DataType`/`Layout`. + +```cpp +struct OpKernelType { + platform::Place place_; + platform::Library library_; + proto::DataType data_type_; + framework::Layout layout_; +}; +``` + +Following is the details: + +### Place + +`Place` is defined as follows: + +```cpp +typedef boost::variant Place; +``` + +`Place` is to represent the device memory where data is locating. + + +### Library + +One operator kernel is usually implemented based on one library. `Library` is defined as a enum variable: + +```cpp +enum Library { Plain, MKLDNN, CUDNN }; +``` + +We use `Plain` enumerator to represent default library. Since most operators in Fluid are implemented based on `Eigen` library, we take `Eigen` library as the `Plain` enumerator. +A library usually has a corresponding `DeviceContext` which contains some handles needed by computation. Fluid now have two default DeviceContexts in CPU and CUDA, `CPUDeviceContext` and `CUDADeviceContext`. `CPUDeviceContext` contains a Eigen library handle and `CDUADeviceContext` contains a Eigen library handle and cuBLAS handle. + +If we want to support new Library, a new enumerator need to be added to `Library` and a new corresponding `LibraryDeviceContext` will be created. + + +### DataType + + +`DataType` is defined in [framework.proto](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto). Currently, int32/int64/fp32/fp64 are supported. + +### Layout + +Actually, a Tensor is a view of a block of memory. Besides a pointer to the memory, we also have to get some other descriptions of this block of memory, such as shape(ddim), stride, and layout. + +Different layout leads to different implementation of operator kernel. There are mainly 4 principles we have to follow to support layout in our fluid framework. + +- We take layout as a data member of Tensor. Layout is actually a enum variable. If fluid is built with MKLDNN, then, the memory format in MKLDNN will be added into this enum variable too. + +- Users have to set layout for input data. And some operators like fill_constant/random, also have to set layout of generating data. Of course, we can have some default layout, like NCHW. + +- The inference of Layout is at run-time, not compile-time. + +- Every operator have to implement different kernels for different layouts. Let's take MKLDNN as an example, if we want to implement a MKLDNN convolution operator, we have to realize all the kernels for different layout, list at [here](http://01org.github.io/mkl-dnn/structmkldnn_1_1memory.html). And we will have a special macro to do registering kernels for MKLDNN operators. + +`Layout` is also defined as a enum variable: + +```cpp +enum Layout { + kNCHW, + kNHWC, +#ifdef PADDLE_WITH_MKLDNN + knChw8c + ... +#endif +}; +``` From 0895d1d3cbabe8181b5cbff2449c945e854c50bd Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 21 Dec 2017 16:24:19 +0800 Subject: [PATCH 51/84] Fix a error --- paddle/framework/backward.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index f011407f49..222aee5974 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -394,8 +394,8 @@ std::vector> MakeOpGrad( std::string new_name = prefix + kZeroVarSuffix; desc->Rename(in_name, new_name); std::unique_ptr fill_zeros_op( - new OpDescBind("fill_zeros_like", {{"X", {prefix}}}, - {{"Out", {new_name}}}, AttributeMap{})); + new OpDesc("fill_zeros_like", {{"X", {prefix}}}, + {{"Out", {new_name}}}, AttributeMap{})); pending_fill_zeros_ops.push_back(std::move(fill_zeros_op)); } } From 9189567a90eb35b9af1ef81c60e9c6db99da12be Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 21 Dec 2017 16:31:06 +0800 Subject: [PATCH 52/84] Follow comments --- .../reorder_lod_tensor_by_rank_op.cc | 21 +++++++++++++------ python/paddle/v2/fluid/layers/control_flow.py | 11 ++-------- python/paddle/v2/fluid/registry.py | 12 ++++++++--- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/paddle/operators/reorder_lod_tensor_by_rank_op.cc b/paddle/operators/reorder_lod_tensor_by_rank_op.cc index 384047428d..369bd4391c 100644 --- a/paddle/operators/reorder_lod_tensor_by_rank_op.cc +++ b/paddle/operators/reorder_lod_tensor_by_rank_op.cc @@ -19,21 +19,28 @@ namespace paddle { namespace operators { -class ReorderLoDTensorProtoMaker : public framework::OpProtoAndCheckerMaker { +class ReorderLoDTensorByRankTableOpProtoMaker + : public framework::OpProtoAndCheckerMaker { public: - ReorderLoDTensorProtoMaker(OpProto *proto, OpAttrChecker *op_checker) + ReorderLoDTensorByRankTableOpProtoMaker(OpProto *proto, + OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(LoDTensor) the input lod tensor need to be reordered."); AddInput("RankTable", "(LoDRankTable) the rank table that input need follow"); AddOutput("Out", "(LoDTensor) reordered lod tensor"); - AddComment(R"DOC(ReorderLoDTensorLoDRankTable + AddComment(R"DOC(ReorderLoDTensorByRankTable Reorder the input X by the rank of `RankTable`. If `RankTable` is ordered by index [3, 0, 2, 1]. Input X will reorder its sequence, the third sequence of X will be the first sequence of Output. NOTE: The RankTable does not need to be calculated by X. + +For example: +The X = [Seq0, Seq1, Seq2, Seq3]. The indices of RankTable are [3, 0, 2, 1]. + +The Out = [Seq3, Seq0, Seq2, Seq1] with correct LoD information. )DOC"); } }; @@ -146,8 +153,9 @@ class ReorderLoDTensorByRankTableOp : public ReorderLoDTensorByRankTableBase { size_t out_offset = 0; out->mutable_lod()->clear(); for (auto &item : rank_table.items()) { - out_offset = this->CopyTensorAndLod(dev_ctx, absolute_table[item.index], - x, out, out_offset); + PADDLE_ENFORCE_LT(item.index, absolute_table.size()); + out_offset = CopyTensorAndLod(dev_ctx, absolute_table[item.index], x, out, + out_offset); } } }; @@ -220,6 +228,7 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(reorder_lod_tensor_by_rank, ops::ReorderLoDTensorByRankTableOp, ops::ReorderLodTensorByRankGradOpMaker, - ops::ReorderLoDTensorProtoMaker, ops::IdentityInferShape); + ops::ReorderLoDTensorByRankTableOpProtoMaker, + ops::IdentityInferShape); REGISTER_OPERATOR(reorder_lod_tensor_by_rank_grad, ops::ReorderLoDTensorByRankGradOp, ops::IdentityInferShape); diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index d66c834654..f49cabfee8 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -3,6 +3,7 @@ from ..framework import Program, Variable, Operator from .. import core from tensor import assign, fill_constant import contextlib +from ..registry import autodoc __all__ = [ 'split_lod_tensor', 'merge_lod_tensor', 'BlockGuard', 'StaticRNNGuard', @@ -983,16 +984,8 @@ class DynamicRNN(object): method)) +@autodoc def reorder_lod_tensor_by_rank(x, rank_table): - """ - - Args: - x(Variable): - rank_table(Variable): - - Returns: - - """ helper = LayerHelper('reorder_lod_tensor_by_rank', **locals()) helper.is_instance('x', Variable) helper.is_instance('rank_table', Variable) diff --git a/python/paddle/v2/fluid/registry.py b/python/paddle/v2/fluid/registry.py index 6f5dd365de..7aa8290611 100644 --- a/python/paddle/v2/fluid/registry.py +++ b/python/paddle/v2/fluid/registry.py @@ -8,7 +8,7 @@ import proto.framework_pb2 as framework_pb2 from framework import OpProtoHolder, Variable, Program, Operator from paddle.v2.fluid.layer_helper import LayerHelper, unique_name -__all__ = ['deprecated', 'register_layer'] +__all__ = ['deprecated', 'register_layer', 'autodoc'] def _convert_(name): @@ -175,12 +175,18 @@ def deprecated(func_or_class): """ Wrap func with deprecated warning """ - warnings.simplefilter('always', DeprecationWarning) #turn off filter + warnings.simplefilter('always', DeprecationWarning) # turn off filter warnings.warn( "Call to deprecated function {}.".format(func.__name__), category=DeprecationWarning, stacklevel=2) - warnings.simplefilter('default', DeprecationWarning) #reset filter + warnings.simplefilter('default', DeprecationWarning) # reset filter return func(*args, **kwargs) return func_wrapper + + +def autodoc(func): + func.__doc__ = _generate_doc_string_(OpProtoHolder.instance().get_op_proto( + func.__name__)) + return func From f3cc75d8fb6321816258acf0158a1da595b1055f Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 21 Dec 2017 19:05:15 +0800 Subject: [PATCH 53/84] Fix errors --- paddle/framework/backward_test.cc | 2 +- paddle/operators/fill_zeros_like_op.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 7f55e6821d..0957646b56 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -159,7 +159,7 @@ class FillZeroOpMaker : public OpProtoAndCheckerMaker { FillZeroOpMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "x"); - AddOutput("Y", "out"); + AddOutput("Out", "out"); AddComment(""); } }; diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index 72c8a6a4f5..b4ae1de876 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -27,7 +27,7 @@ class FillZerosLikeOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of FillZerosLikeOp should not be null."); ctx->SetOutputDim("Out", ctx->GetInputDim("X")); - ctx->ShareLoD("X", /*->*/ "Y"); + ctx->ShareLoD("X", /*->*/ "Out"); } }; From 7e214b498515b50820f8535927d30879f048f6a2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 21 Dec 2017 19:45:37 +0800 Subject: [PATCH 54/84] Speed up ColwiseSum in CPU (#6834) * Remove unnecessary reshape in ColwiseSum Speed up 12s -> 10s. * Hand write ColwiseAdd in CPU --- paddle/operators/math/math_function_impl.h | 39 ++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/paddle/operators/math/math_function_impl.h b/paddle/operators/math/math_function_impl.h index 3e6d833865..aced2690bc 100644 --- a/paddle/operators/math/math_function_impl.h +++ b/paddle/operators/math/math_function_impl.h @@ -67,18 +67,45 @@ void RowwiseAdd::operator()(const DeviceContext& context, template void ColwiseSum::operator()(const DeviceContext& context, const framework::Tensor& input, - framework::Tensor* vector) { + framework::Tensor* out) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; - PADDLE_ENFORCE_EQ(vector->numel(), size); + PADDLE_ENFORCE_EQ(out->numel(), size); - auto vec = framework::EigenMatrix::From(*vector); auto in = framework::EigenMatrix::From(input); - Eigen::array shape({{1, static_cast(size)}}); - vec.reshape(shape).device(*context.eigen_device()) = - in.sum(Eigen::array({{0}})).reshape(shape); + auto vec = framework::EigenVector::Flatten(*out); + + vec.device(*context.eigen_device()) = in.sum(Eigen::array({{0}})); } +// Specialize for CPU, since Eigen implement a general reduce. However, +// colwise-sum can be easily implemented. General reduce has a huge overhead in +// CPU +template +class ColwiseSum { + public: + void operator()(const platform::CPUDeviceContext& context, + const framework::Tensor& input, framework::Tensor* out) { + auto& in_dims = input.dims(); + auto height = in_dims[0]; + auto size = in_dims[1]; + PADDLE_ENFORCE_EQ(out->numel(), size); + + T* out_buf = out->mutable_data(out->place()); + const T* in_buf = input.data(); + + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < size; ++j) { + if (i == 0) { + out_buf[j] = in_buf[i * size + j]; + } else { + out_buf[j] += in_buf[i * size + j]; + } + } + } + } +}; + } // namespace math } // namespace operators } // namespace paddle From 4658f9501efd05396b796297f81bf17de37bda9f Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 21 Dec 2017 20:07:54 +0800 Subject: [PATCH 55/84] fix delete ops --- paddle/framework/block_desc.cc | 15 +++++++++++++++ paddle/framework/block_desc.h | 2 ++ paddle/pybind/protobuf.cc | 1 + python/paddle/v2/fluid/distribute_transpiler.py | 10 +++++----- python/paddle/v2/fluid/framework.py | 12 ++++++++++-- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index 6a7a07d5cf..4707e48353 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -91,6 +91,21 @@ OpDescBind *BlockDescBind::PrependOp() { return ops_.front().get(); } +void BlockDescBind::RemoveOp(size_t s, size_t e) { + if (ops_.begin() + s == ops_.end() || ops_.begin() + e == ops_.end()) { + return; + } + need_update_ = true; + for (auto it = ops_.begin() + s; it != ops_.begin() + e; it++) { + auto names = (*it)->InputArgumentNames(); + for (auto n : names) { + // TODO(typhoonzero): delete vars if no other op use it. + VLOG(3) << "deleting var " << n; + } + } + ops_.erase(ops_.begin() + s, ops_.begin() + e); +} + std::vector BlockDescBind::AllOps() const { std::vector res; for (const auto &op : ops_) { diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 8e967e5378..51b0e75c55 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -80,6 +80,8 @@ class BlockDescBind { OpDescBind *PrependOp(); + void RemoveOp(size_t s, size_t e); + std::vector AllOps() const; size_t OpSize() const { return ops_.size(); } diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 6e6cafafb9..119cae94fb 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -159,6 +159,7 @@ void BindBlockDesc(py::module &m) { py::return_value_policy::reference) .def("prepend_op", &BlockDescBind::PrependOp, py::return_value_policy::reference) + .def("remove_op", &BlockDescBind::RemoveOp) .def("var", [](BlockDescBind &self, py::bytes byte_name) { std::string name = byte_name; diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 7dfbab4677..50364c64be 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -131,11 +131,6 @@ class DistributeTranspiler: def _optimize_distributed(self, optimize_ops, program, params_and_grads, **kwargs): - # remove optimize ops and add a send op to main_program - # FIXME(typhoonzero): delete_op only remove the first accurance, - # need to consider about multiple same optimize op? - for op in optimize_ops: - program.global_block().delete_op(op) if kwargs.has_key("split_method"): split_method = kwargs["split_method"] else: @@ -159,6 +154,10 @@ class DistributeTranspiler: attrs={"endpoints": pserver_endpoints, "epmap": epmap}) + def get_trainer_program(optimize_ops, program): + # remove optimize ops and add a send op to main_program + program.global_block().delete_ops(optimize_ops) + def _create_var_for_trainers(self, block, var, trainers): var_list = [] for i in xrange(trainers): @@ -209,6 +208,7 @@ class DistributeTranspiler: if opt_op.inputs.has_key("Grad"): if opt_op.inputs["Grad"].name in grad_var_names: + print "appending ", opt_op.type, opt_op.inputs optimize_sub_program.global_block().append_op( type=opt_op.type, inputs=opt_op.inputs, diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 7990886417..a409b2aa94 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -579,6 +579,7 @@ class Block(object): self.vars = dict() # var_name --> var self.ops = collections.deque() # operator list self.program = program + self.removed_vars = dict() def __str__(self): return self.to_string(True) @@ -635,8 +636,15 @@ class Block(object): self.ops.append(op) return op - def delete_op(self, op): - self.ops.remove(op) + def delete_ops(self, ops): + # remove from cpp + # FIXME(typhoonzero): remove only the first occuracy. + try: + start = list(self.ops).index(ops[0]) + end = list(self.ops).index(ops[-1]) + except Exception, e: + raise e + self.desc.remove_op(start, end) def prepend_op(self, *args, **kwargs): op_desc = self.desc.prepend_op() From afaa73e594fedd6c606db625d861a0175896cae8 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Thu, 21 Dec 2017 20:28:34 +0800 Subject: [PATCH 56/84] fix pip install page 404 links --- doc/getstarted/build_and_install/pip_install_cn.rst | 10 +++++----- doc/getstarted/build_and_install/pip_install_en.rst | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/getstarted/build_and_install/pip_install_cn.rst b/doc/getstarted/build_and_install/pip_install_cn.rst index b270e2c2f0..a4587f82a9 100644 --- a/doc/getstarted/build_and_install/pip_install_cn.rst +++ b/doc/getstarted/build_and_install/pip_install_cn.rst @@ -37,11 +37,11 @@ PaddlePaddle可以使用常用的Python包管理工具 :header: "版本说明", "cp27-cp27mu", "cp27-cp27m", "C-API" :widths: 1, 3, 3, 3 - "cpu_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" - "cpu_avx_openblas", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "暂无" - "cuda7.5_cudnn5_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" - "cuda8.0_cudnn5_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" - "cuda8.0_cudnn7_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cpu_avx_mkl", "`paddlepaddle-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cpu_avx_openblas", "`paddlepaddle-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "暂无" + "cuda7.5_cudnn5_avx_mkl", "`paddlepaddle_gpu-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle_gpu-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cuda8.0_cudnn5_avx_mkl", "`paddlepaddle_gpu-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle_gpu-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cuda8.0_cudnn7_avx_mkl", "`paddlepaddle_gpu-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle_gpu-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" .. _pip_dependency: diff --git a/doc/getstarted/build_and_install/pip_install_en.rst b/doc/getstarted/build_and_install/pip_install_en.rst index 70f601a11c..55e31560a0 100644 --- a/doc/getstarted/build_and_install/pip_install_en.rst +++ b/doc/getstarted/build_and_install/pip_install_en.rst @@ -40,11 +40,11 @@ If the links below shows up the login form, just click "Log in as guest" to star :header: "version", "cp27-cp27mu", "cp27-cp27m", "C-API" :widths: 1, 3, 3, 3 - "cpu_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" - "cpu_avx_openblas", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "Not Available" - "cuda7.5_cudnn5_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" - "cuda8.0_cudnn5_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" - "cuda8.0_cudnn7_avx_mkl", "`paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.10.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cpu_avx_mkl", "`paddlepaddle-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cpu_avx_openblas", "`paddlepaddle-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "Not Available" + "cuda7.5_cudnn5_avx_mkl", "`paddlepaddle_gpu-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle_gpu-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cuda8.0_cudnn5_avx_mkl", "`paddlepaddle_gpu-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle_gpu-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" + "cuda8.0_cudnn7_avx_mkl", "`paddlepaddle_gpu-0.11.0-cp27-cp27mu-linux_x86_64.whl `_", "`paddlepaddle_gpu-0.11.0-cp27-cp27m-linux_x86_64.whl `_", "`paddle.tgz `_" .. _pip_dependency: From a74db488f7bd4206d2d896668109d6101c24545a Mon Sep 17 00:00:00 2001 From: caoying03 Date: Thu, 21 Dec 2017 17:41:40 +0800 Subject: [PATCH 57/84] follow comments. --- doc/api/v2/config/layer.rst | 4 +- doc/api/v2/fluid/layers.rst | 80 ++++++++++++++--------------- doc/api/v2/fluid/nets.rst | 6 +-- doc/api/v2/fluid/optimizer.rst | 8 +-- doc/api/v2/fluid/regularizer.rst | 6 +-- paddle/operators/mul_op.cc | 51 +++++++++--------- python/paddle/v2/fluid/layers/nn.py | 7 +-- 7 files changed, 81 insertions(+), 81 deletions(-) diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index c3f9c18d06..d81481ca81 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -467,7 +467,7 @@ lambda_cost :noindex: square_error_cost --------- +----------------- .. autoclass:: paddle.v2.layer.square_error_cost :noindex: @@ -533,7 +533,7 @@ Miscs ===== dropout --------------- +-------- .. autoclass:: paddle.v2.layer.dropout :noindex: diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 842f3b1800..b25009310c 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -19,17 +19,17 @@ dynamic_lstm :noindex: data ---------- +---- .. autofunction:: paddle.v2.fluid.layers.data :noindex: mean ---------- +---- .. autofunction:: paddle.v2.fluid.layers.mean :noindex: mul ---------- +--- .. autofunction:: paddle.v2.fluid.layers.mul :noindex: @@ -45,13 +45,13 @@ elementwise_div dropout ---------- +------- .. autofunction:: paddle.v2.fluid.layers.dropout :noindex: reshape ---------- +-------- .. autofunction:: paddle.v2.fluid.layers.reshape :noindex: @@ -81,67 +81,67 @@ transpose sigmoid_cross_entropy_with_logits ---------- +--------------------------------- .. autofunction:: paddle.v2.fluid.layers.esigmoid_cross_entropy_with_logits :noindex: cast ---------- +---- .. autofunction:: paddle.v2.fluid.layers.cast :noindex: concat ---------- +------- .. autofunction:: paddle.v2.fluid.layers.concat :noindex: sums ---------- +---- .. autofunction:: paddle.v2.fluid.layers.sums :noindex: linear_chain_crf ---------- +---------------- .. autofunction:: paddle.v2.fluid.layers.linear_chain_crf :noindex: assign ---------- +------- .. autofunction:: paddle.v2.fluid.layers.embedding :noindex: split_lod_tensor ---------- +---------------- .. autofunction:: paddle.v2.fluid.layers.split_lod_tensor :noindex: merge_lod_tensor ---------- +---------------- .. autofunction:: paddle.v2.fluid.layers.merge_lod_tensor :noindex: cos_sim ---------- +-------- .. autofunction:: paddle.v2.fluid.layers.cos_sim :noindex: cross_entropy ---------- +------------- .. autofunction:: paddle.v2.fluid.layers.cross_entropy :noindex: square_error_cost ---------- +----------------- .. autofunction:: paddle.v2.fluid.layers.square_error_cost :noindex: @@ -153,68 +153,68 @@ accuracy sequence_conv ---------- +------------- .. autofunction:: paddle.v2.fluid.layers.sequence_conv :noindex: conv2d ---------- +------ .. autofunction:: paddle.v2.fluid.layers.conv2d :noindex: sequence_pool ---------- +------------- .. autofunction:: paddle.v2.fluid.layers.sequence_pool :noindex: pool2d ---------- +------ .. autofunction:: paddle.v2.fluid.layers.pool2d :noindex: batch_norm ---------- +---------- .. autofunction:: paddle.v2.fluid.layers.batch_norm :noindex: beam_search_decode ---------- +------------------ .. autofunction:: paddle.v2.fluid.layers.beam_search_decode :noindex: lod_rank_table ---------- +-------------- .. autofunction:: paddle.v2.fluid.layers.lod_rank_table :noindex: max_sequence_len ---------- +---------------- .. autofunction:: paddle.v2.fluid.layers.max_sequence_len :noindex: topk ---------- +----- .. autofunction:: paddle.v2.fluid.layers.topk :noindex: lod_tensor_to_array ---------- +------------------- .. autofunction:: paddle.v2.fluid.layers.lod_tensor_to_array :noindex: array_to_lod_tensor ---------- +------------------- .. autofunction:: paddle.v2.fluid.layers.array_to_lod_tensor :noindex: @@ -222,26 +222,26 @@ array_to_lod_tensor fill_constant ---------- +------------- .. autofunction:: paddle.v2.fluid.layers.fill_constant :noindex: fill_constant_batch_size_like ---------- +----------------------------- .. autofunction:: paddle.v2.fluid.layers.fill_constant_batch_size_like :noindex: ones ---------- +---- .. autofunction:: paddle.v2.fluid.layers.ones :noindex: zeros ---------- +----- .. autofunction:: paddle.v2.fluid.layers.zeros :noindex: @@ -253,14 +253,14 @@ increment array_write ---------- +----------- .. autofunction:: paddle.v2.fluid.layers.array_write :noindex: create_array ---------- +------------ .. autofunction:: paddle.v2.fluid.layers.create_array :noindex: @@ -272,31 +272,31 @@ less_than array_read ---------- +---------- .. autofunction:: paddle.v2.fluid.layers.array_read :noindex: shrink_memory ---------- +-------------- .. autofunction:: paddle.v2.fluid.layers.shrink_memory :noindex: array_length ---------- +------------- .. autofunction:: paddle.v2.fluid.layers.array_length :noindex: conv2d_transpose ---------- +---------------- .. autofunction:: paddle.v2.fluid.layers.conv2d_transpose :noindex: sequence_expand ---------- +--------------- .. autofunction:: paddle.v2.fluid.layers.sequence_expand :noindex: @@ -308,13 +308,13 @@ lstm_unit sequence_softmax ---------- +---------------- .. autofunction:: paddle.v2.fluid.layers.sequence_softmax :noindex: reduce_sum ---------- +---------- .. autofunction:: paddle.v2.fluid.layers.reduce_sum :noindex: diff --git a/doc/api/v2/fluid/nets.rst b/doc/api/v2/fluid/nets.rst index 2c3d075422..b792efb71f 100644 --- a/doc/api/v2/fluid/nets.rst +++ b/doc/api/v2/fluid/nets.rst @@ -3,19 +3,19 @@ Nets =========== simple_img_conv_pool ------------ +-------------------- .. autofunction:: paddle.v2.fluid.nets.simple_img_conv_pool :noindex: img_conv_group ------------ +--------------- .. autofunction:: paddle.v2.fluid.nets.img_conv_group :noindex: sequence_conv_pool ------------ +------------------ .. autofunction:: paddle.v2.fluid.nets.sequence_conv_pool :noindex: diff --git a/doc/api/v2/fluid/optimizer.rst b/doc/api/v2/fluid/optimizer.rst index 233762fcdf..19b4940f08 100644 --- a/doc/api/v2/fluid/optimizer.rst +++ b/doc/api/v2/fluid/optimizer.rst @@ -18,7 +18,7 @@ SGDOptimizer MomentumOptimizer ------------ +----------------- .. automodule:: paddle.v2.fluid.optimizer :members: MomentumOptimizer :noindex: @@ -26,14 +26,14 @@ MomentumOptimizer AdagradOptimizer ------------ +---------------- .. automodule:: paddle.v2.fluid.optimizer :members: AdagradOptimizer :noindex: AdamOptimizer ------------ +------------- .. automodule:: paddle.v2.fluid.optimizer :members: AdamOptimizer :noindex: @@ -47,7 +47,7 @@ AdamaxOptimizer DecayedAdagradOptimizer ------------ +----------------------- .. automodule:: paddle.v2.fluid.optimizer :members: DecayedAdagradOptimizer :noindex: diff --git a/doc/api/v2/fluid/regularizer.rst b/doc/api/v2/fluid/regularizer.rst index 3af2b07d2a..868e225ed3 100644 --- a/doc/api/v2/fluid/regularizer.rst +++ b/doc/api/v2/fluid/regularizer.rst @@ -3,14 +3,14 @@ Regularizer =========== WeightDecayRegularizer ------------ +---------------------- .. automodule:: paddle.v2.fluid.regularizer :members: WeightDecayRegularizer :noindex: L2DecayRegularizer ------------ +------------------ .. automodule:: paddle.v2.fluid.regularizer :members: L2DecayRegularizer :noindex: @@ -18,7 +18,7 @@ L2DecayRegularizer L1DecayRegularizer ------------ +------------------- .. automodule:: paddle.v2.fluid.regularizer :members: L1DecayRegularizer diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index cee1bb0098..599df9c3df 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -73,36 +73,35 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { public: MulOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input tensor of the mul op."); - AddInput("Y", "The second input tensor of the mul op."); - AddOutput("Out", "The output tensor of the mul op."); + AddInput("X", "(Tensor), The first input tensor of mul op."); + AddInput("Y", "(Tensor), The second input tensor of mul op."); + AddOutput("Out", "(Tensor), The output tensor of mul op."); AddAttr( "x_num_col_dims", - "(int, default 1) " - R"DOC(The mul_op can take tensors with more than two dimensions as its - inputs. If the input `X` is a tensor with more than two - dimensions, `X` will be flattened into a two-dimensional matrix - first. The flattening rule is: the first `num_col_dims` will be - flattened to form the first dimension of the final matrix (height - of the matrix), and the rest `rank(X) - num_col_dims` dimensions - are flattened to form the second dimension of the final matrix ( - width of the matrix). As a result, height of the flattened matrix - is equal to the product of `X`'s first `x_num_col_dims` dimensions' - sizes, and width of the flattened matrix is equal to the product - of `X`'s last `rank(x) - num_col_dims` dimensions' size. - For example, suppose `X` is a 6-dimensional tensor with the shape - [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. Then, the flattened - matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. + R"DOC((int, default 1), The mul_op can take tensors with more than two + dimensions as its inputs. If the input $X$ is a tensor with more + than two dimensions, $X$ will be flattened into a two-dimensional + matrix first. The flattening rule is: the first `num_col_dims` + will be flattened to form the first dimension of the final matrix + (the height of the matrix), and the rest `rank(X) - num_col_dims` + dimensions are flattened to form the second dimension of the final + matrix (the width of the matrix). As a result, height of the + flattened matrix is equal to the product of $X$'s first + `x_num_col_dims` dimensions' sizes, and width of the flattened + matrix is equal to the product of $X$'s last `rank(x) - num_col_dims` + dimensions' size. For example, suppose $X$ is a 6-dimensional + tensor with the shape [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. + Thus, the flattened matrix will have a shape [2 x 3 x 4, 5 x 6] = + [24, 30]. )DOC") .SetDefault(1) .EqualGreaterThan(1); AddAttr( "y_num_col_dims", - "(int, default 1) " - R"DOC(The mul_op can take tensors with more than two dimensions as its - inputs. If the input `Y` is a tensor with more than two - dimensions, `Y` will be flatten into a two-dimensional matrix - first. The attribute `y_num_col_dims` determines how `Y` is + R"DOC((int, default 1), The mul_op can take tensors with more than two, + dimensions as its inputs. If the input $Y$ is a tensor with more + than two dimensions, $Y$ will be flattened into a two-dimensional + matrix first. The attribute `y_num_col_dims` determines how $Y$ is flattened. See comments of `x_num_col_dims` for more details. )DOC") .SetDefault(1) @@ -110,14 +109,14 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Mul Operator. -This operator is used to perform matrix multiplication for input X and Y. +This operator is used to perform matrix multiplication for input $X$ and $Y$. The equation is: $$Out = X * Y$$ -Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD information with input `X`. +Both the input $X$ and $Y$ can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD information with input $X$. )DOC"); } diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 538a0e6f6e..ab93e57c26 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -40,7 +40,8 @@ def fc(input, This process can be formulated as follows: .. math:: - Out = Act({\sum_{i=0}^{N-1}W_iX_i + b}) + + Out = Act\left({\sum_{i=0}^{N-1}W_iX_i + b}\right) In the above equation: @@ -48,8 +49,8 @@ def fc(input, * :math:`X_i`: The input tensor. * :math:`W`: The weights created by this layer. * :math:`b`: The bias parameter created by this layer (if needed). - * :math`Act`: The activation funtion. - * :math`Out`: The output tensor. + * :math:`Act`: The activation funtion. + * :math:`Out`: The output tensor. Args: input(Variable|list): The input tensor(s) to the fully connected layer. From b848416166a6a6d0750b2b1ac112cb5e7a0b2cfa Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 21 Dec 2017 20:44:16 +0800 Subject: [PATCH 58/84] follow comments --- paddle/framework/block_desc.cc | 2 +- paddle/operators/detail/recv_impl.cc | 2 +- paddle/operators/detail/send_recv_impl.h | 2 +- paddle/operators/recv_op.cc | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index bde2ba3907..0668b08ff7 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -90,7 +90,7 @@ OpDesc *BlockDesc::PrependOp() { return ops_.front().get(); } -void BlockDescBind::RemoveOp(size_t s, size_t e) { +void BlockDesc::RemoveOp(size_t s, size_t e) { if (ops_.begin() + s == ops_.end() || ops_.begin() + e == ops_.end()) { return; } diff --git a/paddle/operators/detail/recv_impl.cc b/paddle/operators/detail/recv_impl.cc index e984f42386..517a1946a0 100644 --- a/paddle/operators/detail/recv_impl.cc +++ b/paddle/operators/detail/recv_impl.cc @@ -58,7 +58,7 @@ Status SendRecvServerImpl::Wait(ServerContext *context, return Status::OK; } -void SendRecvServerImpl::Start() { +void SendRecvServerImpl::Reset() { std::lock_guard lock(this->mutex_); done_ = false; } diff --git a/paddle/operators/detail/send_recv_impl.h b/paddle/operators/detail/send_recv_impl.h index 82ab3ab689..eec9dd38d1 100644 --- a/paddle/operators/detail/send_recv_impl.h +++ b/paddle/operators/detail/send_recv_impl.h @@ -56,7 +56,7 @@ class SendRecvServerImpl final : public SendRecvService::Service { VariableMessage *out_var) override; Status Wait(ServerContext *context, const VoidMessage *in_var, VoidMessage *out_var) override; - void Start(); + void Reset(); void Done(); void SetScope(framework::Scope *scope) { scope_ = scope; }; diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index dfb6e78529..efc9fdc46e 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -80,7 +80,7 @@ class RecvOp : public framework::OperatorBase { auto grad_list = Attr>("GradList"); auto trainer_count = Attr("Trainers"); size_t param_count = param_list.size(); - rpc_service_->Start(); + rpc_service_->Reset(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. while (true) { // Get from multiple trainers, we don't care about order in which @@ -93,6 +93,8 @@ class RecvOp : public framework::OperatorBase { std::string param_var_name; if (it != grad_list.end()) { param_var_name = param_list[it - grad_list.begin()]; + } else { + LOG(ERROR) << "grad have no paired param found!"; } VLOG(3) << "recved grad: " << grad_var_name << " updating param: " << param_var_name; @@ -112,7 +114,7 @@ class RecvOp : public framework::OperatorBase { // FIXME(typhoonzero): do not copy framework::CopyFrom(v.second, dev_ctx.GetPlace(), dev_ctx, tensor); } - rpc_service_->Start(); + rpc_service_->Reset(); std::string program_str = Attr("OptimizeProgram"); framework::ProgramDesc program_desc; From 5913e735be6301215bbc6f4400833faa77a1ad62 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 21 Dec 2017 21:23:08 +0800 Subject: [PATCH 59/84] fix compile when merge --- paddle/operators/recv_op.cc | 5 +++-- paddle/operators/send_recv_op_test.cc | 8 ++++---- paddle/pybind/protobuf.cc | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index efc9fdc46e..4e91d1151e 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -24,6 +24,7 @@ #include "paddle/framework/framework.pb.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/op_registry.h" +#include "paddle/framework/proto_desc.h" #include "paddle/operators/detail/send_recv_impl.h" #include "paddle/operators/detail/simple_block_queue.h" @@ -117,9 +118,9 @@ class RecvOp : public framework::OperatorBase { rpc_service_->Reset(); std::string program_str = Attr("OptimizeProgram"); - framework::ProgramDesc program_desc; + framework::proto::ProgramDesc program_desc; program_desc.ParseFromString(program_str); - framework::ProgramDescBind program(program_desc); + framework::ProgramDesc program(program_desc); framework::Executor executor(dev_ctx); // Run sub graph to get optimized tensor try { diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index 1715b05c2c..d899d8154c 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -56,12 +56,12 @@ void AddOp(const std::string &type, const paddle::framework::VariableNameMap &inputs, const paddle::framework::VariableNameMap &outputs, paddle::framework::AttributeMap attrs, - paddle::framework::BlockDescBind *block) { + paddle::framework::BlockDesc *block) { // insert output for (auto kv : outputs) { for (auto v : kv.second) { auto var = block->Var(v); - var->SetDataType(paddle::framework::DataType::FP32); + var->SetDataType(paddle::framework::proto::DataType::FP32); } } @@ -83,8 +83,8 @@ void StartServerNet() { InitTensorsInScope(scope, place); // sub program run in recv_op, for simple test we use sum - paddle::framework::ProgramDescBind program; - paddle::framework::BlockDescBind *block = program.MutableBlock(0); + paddle::framework::ProgramDesc program; + paddle::framework::BlockDesc *block = program.MutableBlock(0); // X for server side tensors, RX for received tensers, must be of same shape. AddOp("sum", {{"X", {"x0", "x1"}}}, {{"Out", {"Out"}}}, {}, block); diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 7fb0f072a1..f105370f22 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -159,7 +159,7 @@ void BindBlockDesc(py::module &m) { py::return_value_policy::reference) .def("prepend_op", &BlockDesc::PrependOp, py::return_value_policy::reference) - .def("remove_op", &BlockDescBind::RemoveOp) + .def("remove_op", &BlockDesc::RemoveOp) .def("var", [](BlockDesc &self, py::bytes byte_name) { std::string name = byte_name; @@ -251,7 +251,7 @@ void BindOpDesc(py::module &m) { .def("attr", &OpDesc::GetAttr) .def("set_block_attr", &OpDesc::SetBlockAttr) .def("set_serialized_attr", - [](OpDescBind &self, const std::string &name, + [](OpDesc &self, const std::string &name, const py::bytes &seriralized) { std::string ser(seriralized); self.SetAttr(name, ser); From adfe6900b94342784ef4c1913bc81529b9ced972 Mon Sep 17 00:00:00 2001 From: guosheng Date: Thu, 21 Dec 2017 21:43:24 +0800 Subject: [PATCH 60/84] Fix lr setting of param_attr in Fluid --- python/paddle/v2/fluid/param_attr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/param_attr.py b/python/paddle/v2/fluid/param_attr.py index f6f320c788..ab4561b042 100644 --- a/python/paddle/v2/fluid/param_attr.py +++ b/python/paddle/v2/fluid/param_attr.py @@ -58,7 +58,9 @@ class ParamAttr(object): def to_kwargs(self, with_initializer=False): kwargs = { 'name': self.name, - 'learning_rate': self.learning_rate, + 'optimize_attr': { + 'learning_rate': self.learning_rate + }, 'regularizer': self.regularizer, 'trainable': self.trainable, 'clip_attr': self.clip From 19939657eaf5793af62033c34c071c949c409012 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 21 Dec 2017 23:47:34 +0800 Subject: [PATCH 61/84] enable training alexnet benchmark --- benchmark/paddle/image/alexnet.py | 22 +++++++++++++++++--- benchmark/paddle/image/run_mkl_train.sh | 1 + benchmark/paddle/image/run_openblas_train.sh | 1 + 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/benchmark/paddle/image/alexnet.py b/benchmark/paddle/image/alexnet.py index 3358d43a4b..10db194485 100644 --- a/benchmark/paddle/image/alexnet.py +++ b/benchmark/paddle/image/alexnet.py @@ -6,6 +6,7 @@ height = 227 width = 227 num_class = 1000 batch_size = get_config_arg('batch_size', int, 128) +use_mkldnn = get_config_arg('use_mkldnn', bool, False) args = {'height': height, 'width': width, 'color': True, 'num_class': num_class} define_py_data_sources2( @@ -31,7 +32,12 @@ net = img_pool_layer(input=net, pool_size=3, stride=2) # conv2 net = img_conv_layer( - input=net, filter_size=5, num_filters=256, stride=1, padding=2, groups=1) + input=net, + filter_size=5, + num_filters=256, + stride=1, + padding=2, + groups=2 if use_mkldnn else 1) net = img_cmrnorm_layer(input=net, size=5, scale=0.0001, power=0.75) net = img_pool_layer(input=net, pool_size=3, stride=2) @@ -40,11 +46,21 @@ net = img_conv_layer( input=net, filter_size=3, num_filters=384, stride=1, padding=1) # conv4 net = img_conv_layer( - input=net, filter_size=3, num_filters=384, stride=1, padding=1, groups=1) + input=net, + filter_size=3, + num_filters=384, + stride=1, + padding=1, + groups=2 if use_mkldnn else 1) # conv5 net = img_conv_layer( - input=net, filter_size=3, num_filters=256, stride=1, padding=1, groups=1) + input=net, + filter_size=3, + num_filters=256, + stride=1, + padding=1, + groups=2 if use_mkldnn else 1) net = img_pool_layer(input=net, pool_size=3, stride=2) net = fc_layer( diff --git a/benchmark/paddle/image/run_mkl_train.sh b/benchmark/paddle/image/run_mkl_train.sh index 5335af5ac1..c38b3e3621 100755 --- a/benchmark/paddle/image/run_mkl_train.sh +++ b/benchmark/paddle/image/run_mkl_train.sh @@ -47,5 +47,6 @@ for use_mkldnn in True False; do train vgg 19 $batchsize $use_mkldnn train resnet 50 $batchsize $use_mkldnn train googlenet v1 $batchsize $use_mkldnn + train alexnet group2 $batchsize $use_mkldnn done done diff --git a/benchmark/paddle/image/run_openblas_train.sh b/benchmark/paddle/image/run_openblas_train.sh index b9494ce119..caea5548c3 100755 --- a/benchmark/paddle/image/run_openblas_train.sh +++ b/benchmark/paddle/image/run_openblas_train.sh @@ -36,4 +36,5 @@ for batchsize in 64 128 256; do train vgg 19 $batchsize train resnet 50 $batchsize train googlenet v1 $batchsize + train alexnet group2 $batchsize $use_mkldnn done From 86b8bdc0af01960eaded403ade4214faebe5c475 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 21 Dec 2017 23:54:38 +0800 Subject: [PATCH 62/84] enable inference alexnet benchmark --- benchmark/paddle/image/alexnet.py | 11 ++++++++++- benchmark/paddle/image/run_mkl_infer.sh | 1 + benchmark/paddle/image/run_openblas_infer.sh | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmark/paddle/image/alexnet.py b/benchmark/paddle/image/alexnet.py index 10db194485..b0beef8ca7 100644 --- a/benchmark/paddle/image/alexnet.py +++ b/benchmark/paddle/image/alexnet.py @@ -7,8 +7,17 @@ width = 227 num_class = 1000 batch_size = get_config_arg('batch_size', int, 128) use_mkldnn = get_config_arg('use_mkldnn', bool, False) +is_infer = get_config_arg("is_infer", bool, False) +num_samples = get_config_arg('num_samples', int, 2560) -args = {'height': height, 'width': width, 'color': True, 'num_class': num_class} +args = { + 'height': height, + 'width': width, + 'color': True, + 'num_class': num_class, + 'is_infer': is_infer, + 'num_samples': num_samples +} define_py_data_sources2( "train.list", None, module="provider", obj="process", args=args) diff --git a/benchmark/paddle/image/run_mkl_infer.sh b/benchmark/paddle/image/run_mkl_infer.sh index d795bcab1b..00942e32a5 100755 --- a/benchmark/paddle/image/run_mkl_infer.sh +++ b/benchmark/paddle/image/run_mkl_infer.sh @@ -79,6 +79,7 @@ fi # inference benchmark for use_mkldnn in True False; do for batchsize in 1 2 4 8 16; do + infer alexnet group2 $batchsize $use_mkldnn infer googlenet v1 $batchsize $use_mkldnn infer resnet 50 $batchsize $use_mkldnn infer vgg 19 $batchsize $use_mkldnn diff --git a/benchmark/paddle/image/run_openblas_infer.sh b/benchmark/paddle/image/run_openblas_infer.sh index c1001d3a7c..3dad42ee0d 100755 --- a/benchmark/paddle/image/run_openblas_infer.sh +++ b/benchmark/paddle/image/run_openblas_infer.sh @@ -56,6 +56,7 @@ fi # inference benchmark for batchsize in 1 2 4 8 16; do + infer alexnet group2 $batchsize $use_mkldnn infer googlenet v1 $batchsize infer resnet 50 $batchsize infer vgg 19 $batchsize From 6fc454486ccb2b653f916df979ad899e51a55ff8 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 21 Dec 2017 04:01:55 -0500 Subject: [PATCH 63/84] reduce test_period to save time when training openblas --- benchmark/paddle/image/run_openblas_train.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/paddle/image/run_openblas_train.sh b/benchmark/paddle/image/run_openblas_train.sh index fce6f9be4a..e751cd6939 100755 --- a/benchmark/paddle/image/run_openblas_train.sh +++ b/benchmark/paddle/image/run_openblas_train.sh @@ -15,8 +15,8 @@ function train() { --use_mkldnn=False \ --use_gpu=False \ --trainer_count=$thread \ - --log_period=10 \ - --test_period=100 \ + --log_period=3 \ + --test_period=30 \ --config_args=$args \ 2>&1 | tee ${log} From 81e15bcf2397be65059eeed41c575a877c13abd1 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 21 Dec 2017 11:57:32 -0500 Subject: [PATCH 64/84] reduce the training samples for infer model --- benchmark/paddle/image/run_mkl_infer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/paddle/image/run_mkl_infer.sh b/benchmark/paddle/image/run_mkl_infer.sh index d795bcab1b..9eea21793b 100755 --- a/benchmark/paddle/image/run_mkl_infer.sh +++ b/benchmark/paddle/image/run_mkl_infer.sh @@ -37,7 +37,7 @@ function infer() { --trainer_count=1 \ --num_passes=1 \ --save_dir="models/${topology}-${layer_num}" \ - --config_args="batch_size=128,layer_num=${layer_num}" \ + --config_args="batch_size=128,layer_num=${layer_num},num_samples=256" \ > /dev/null 2>&1 echo "Done" fi From ab916e54b0b7d2802c9fbb976140bd9d48e21a72 Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 21 Dec 2017 11:06:19 -0800 Subject: [PATCH 65/84] Adding layer array_length (#6817) --- python/paddle/v2/fluid/layers/control_flow.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 7ed79968b1..f544722cdd 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -585,9 +585,23 @@ def shrink_memory(x, i, table): def array_length(array): - """ - This function creates an operator to find the length of the + """This function performs the operation to find the length of the input LOD_TENSOR_ARRAY. + + Args: + array (LOD_TENSOR_ARRAY): The input array that will be used + to compute the length. + + Returns: + Variable: The length of the input LoDTensorArray. + + Examples: + .. code-block::python + + tmp = fluid.layers.zeros(shape=[10], dtype='int32') + i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) + arr = fluid.layers.array_write(tmp, i=i) + arr_len = fluid.layers.array_length(arr) """ helper = LayerHelper('array_length', **locals()) tmp = helper.create_tmp_variable(dtype='int64') From e56d03ea503724161b63b5cc16f2efef964a1e89 Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 21 Dec 2017 11:10:45 -0800 Subject: [PATCH 66/84] Writeup for array write layer (#6820) * Writeup for array write layer * Fixed the type --- python/paddle/v2/fluid/layers/control_flow.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index f544722cdd..62783bea66 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -492,9 +492,24 @@ def increment(x, value=1.0, in_place=True): def array_write(x, i, array=None): - """ - This function creates an operator to write the data out as a + """This function performs the operation to write the data out as an LOD_TENSOR_ARRAY. + + Args: + x (Variable|list): The input tensor from which the data will be read. + i (Variable|list): The subscript index in tensor array, that points the + place from which data will be read. + array (Variable|list): The data can be read into this variable if + this is assigned. + Returns: + Variable: The tensor type variable that has the data written to it. + + Examples: + .. code-block::python + + tmp = fluid.layers.zeros(shape=[10], dtype='int32') + i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) + arr = layers.array_write(tmp, i=i) """ helper = LayerHelper('array_write', **locals()) if array is None: From e473fa6bfe470f71079c87cb35917ac86738b31e Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 21 Dec 2017 12:19:34 -0800 Subject: [PATCH 67/84] Adding array read layer (#6853) --- python/paddle/v2/fluid/layers/control_flow.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 62783bea66..a54527130f 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -564,9 +564,19 @@ def less_than(x, y, cond=None, **ignored): def array_read(array, i): - """ - This function creates an operator to read the data in as a + """This function performs the operation to read the data in as an LOD_TENSOR_ARRAY. + Args: + array (Variable|list): The input tensor that will be written to an array. + i (Variable|list): The subscript index in tensor array, that points the + place where data will be written to. + Returns: + Variable: The tensor type variable that has the data written to it. + Examples: + .. code-block::python + tmp = fluid.layers.zeros(shape=[10], dtype='int32') + i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) + arr = layers.array_read(tmp, i=i) """ helper = LayerHelper('array_read', **locals()) if not isinstance( From 3528e6ede688b538efe51c52a69cc04e13855a7a Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 21 Dec 2017 13:58:40 -0800 Subject: [PATCH 68/84] Polish API docs for Fluid Assign and Concat layer (#6855) * Polish API docs for assign layer * Polishing the API docs for concat and assign layer --- python/paddle/v2/fluid/layers/tensor.py | 35 +++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers/tensor.py b/python/paddle/v2/fluid/layers/tensor.py index e984a6be19..70d800cc9c 100644 --- a/python/paddle/v2/fluid/layers/tensor.py +++ b/python/paddle/v2/fluid/layers/tensor.py @@ -27,10 +27,23 @@ def cast(x, dtype): return out -def concat(input, axis): +def concat(input, axis=0): """ - This function concats the input along the axis mentioned + **Concat** + + This function concatenates the input along the axis mentioned and returns that as the output. + + Args: + input(list): List of tensors to be concatenated + axis(int): Integer axis along which the tensors will be concatenated + + Returns: + Variable: Output variable of the concatenation + + Examples: + .. code-block:: python + out = fluid.layers.concat(input=[Efirst, Esecond, Ethird, Efourth]) """ helper = LayerHelper('concat', **locals()) out = helper.create_tmp_variable(dtype=helper.input_dtype()) @@ -55,6 +68,24 @@ def sums(input, out=None): def assign(input, output): + """ + **Assign** + + This function copies the *input* Variable to the *output* Variable. + + Args: + input(Variable): The source variable + output(Variable): The destination variable + + Returns: + Variable: The destination variable that was supplied as the *output*. + + Examples: + .. code-block:: python + out = fluid.layers.create_tensor(dtype='float32') + hidden = fluid.layers.fc(input=data, size=10) + fluid.layers.assign(hidden, out) + """ helper = LayerHelper('assign', **locals()) helper.append_op( type='scale', From 61eb085648756c0bed29acba002786354136c735 Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 21 Dec 2017 14:13:53 -0800 Subject: [PATCH 69/84] Adding documentation for the operators: lod_tensor_to_array , array_to_lod_tensor, create_array, increment (#6807) * Adding operator assignment * Adding a prototype for documentation in layers * small update to re-run Travis * Removing file from another PR * Small typo * Adding documentation for the operators: lod_tensor_to_array , array_to_lod_tensor, create_array, increment * Fixing indentation issue * Fixed datatype of input variables --- python/paddle/v2/fluid/layers/control_flow.py | 82 ++++++++++++++++--- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index a54527130f..5b7979f39f 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -440,9 +440,25 @@ def topk(input, k): def lod_tensor_to_array(x, table): - """ - This function creates an operator to convert an LOD_Tensor to - an array. + """This function performs the operation that converts an LOD_Tensor to + an array. + + Args: + x (Variable|list): The tensor that needs to be converted to an array. + table (ParamAttr|list): The variable that stores the level of lod + which is ordered by sequence length in + descending order. + + Returns: + Variable: The variable of type array that has been converted from a + tensor. + + Examples: + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[10]) + table = fluid.layers.lod_rank_table(x, level=0) + array = fluid.layers.lod_tensor_to_array(x, table) """ helper = LayerHelper("lod_tensor_to_array", **locals()) array = helper.create_variable( @@ -458,9 +474,26 @@ def lod_tensor_to_array(x, table): def array_to_lod_tensor(x, table): - """ - This function creates an operator to convert an array to a - LOD_Tensor. + """This function performs the operations that converts an array to + an LOD_Tensor. + + Args: + x (Variable|list): The array that needs to be converted to a tensor. + table (ParamAttr|list): The variable that stores the level of lod + which is ordered by sequence length in + descending order. + + Returns: + Variable: The variable of type tensor that has been converted + from an array. + + Examples: + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[10]) + table = fluid.layers.lod_rank_table(x, level=0) + array = fluid.layers.lod_tensor_to_array(x, table) + lod_tensor = fluid.layers.array_to_lod_tensor(array, table) """ helper = LayerHelper("array_to_lod_tensor", **locals()) tmp = helper.create_tmp_variable(dtype=x.dtype) @@ -473,10 +506,24 @@ def array_to_lod_tensor(x, table): def increment(x, value=1.0, in_place=True): - """ - This function creates an operator to increment each value in the input - `x` by an amount: `value` as mentioned in the input parameter. This - operation is performed in-place by default. + """This function performs an operation that increments each value in the + input :math:`x` by an amount: :math:`value` as mentioned in the input + parameter. This operation is performed in-place by default. + + Args: + x (Variable|list): The tensor that has the input values. + value (float): The amount by which the values should be incremented. + in_place (bool): If the increment should be performed in-place. + + Returns: + Variable: The tensor variable storing the transformation of + element-wise increment of each value in the input. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name='data', shape=[32, 32], dtype='float32') + data = fluid.layers.increment(x=data, value=3.0, in_place=True) """ helper = LayerHelper("increment", **locals()) if not in_place: @@ -526,6 +573,21 @@ def array_write(x, i, array=None): def create_array(dtype): + """This function creates an array of type :math:`LOD_TENSOR_ARRAY` using the + LayerHelper. + + Args: + dtype (int|float): The data type of the elements in the array. + + Returns: + Variable: The tensor variable storing the elements of data type. + + Examples: + .. code-block:: python + + data = fluid.layers.create_array(dtype='float32') + + """ helper = LayerHelper("array", **locals()) return helper.create_variable( name="{0}.out".format(helper.name), From 91911f4b5689d5313384b7894562bd02a71a7c72 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 21 Dec 2017 14:18:12 -0800 Subject: [PATCH 70/84] Fix documentation of embedding layer (#6854) --- python/paddle/v2/fluid/layers/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 1db63fbfe8..7d56e7caf1 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -117,7 +117,7 @@ def embedding(input, size, is_sparse=False, param_attr=None, dtype='float32'): Args: input(Variable): Input to the function - size(int): Output size + size(tuple|list|None): Shape of the look up table parameter is_sparse(bool): Boolean flag that specifying whether the input is sparse param_attr(ParamAttr): Parameters for this layer dtype(np.dtype|core.DataType|str): The type of data : float32, float_16, int etc From a55238590285d438d30329cfa7e80c628376fd21 Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 21 Dec 2017 14:46:15 -0800 Subject: [PATCH 71/84] Adding doc for sums layer (#6857) --- python/paddle/v2/fluid/layers/tensor.py | 35 +++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/python/paddle/v2/fluid/layers/tensor.py b/python/paddle/v2/fluid/layers/tensor.py index 70d800cc9c..e5820d24cd 100644 --- a/python/paddle/v2/fluid/layers/tensor.py +++ b/python/paddle/v2/fluid/layers/tensor.py @@ -56,9 +56,28 @@ def concat(input, axis=0): def sums(input, out=None): - """ - This function takes in the input and performs the sum operation on it - and returns that as the output. + """This function performs the sum operation on the input and returns the + result as the output. + + Args: + input (Variable|list): The input tensor that has the elements + that need to be summed up. + + Returns: + Variable: The tensor type variable that has the sum of input + written to it. + + Examples: + .. code-block::python + + tmp = fluid.layers.zeros(shape=[10], dtype='int32') + i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) + a0 = layers.array_read(array=tmp, i=i) + i = layers.increment(x=i) + a1 = layers.array_read(array=tmp, i=i) + mean_a0 = layers.mean(x=a0) + mean_a1 = layers.mean(x=a1) + a_sum = layers.sums(input=[mean_a0, mean_a1]) """ helper = LayerHelper('sum', **locals()) if out is None: @@ -99,9 +118,9 @@ def fill_constant(shape, dtype, value, out=None): """ **fill_constant** - This function creates a tensor of specified *shape* and + This function creates a tensor of specified *shape* and *dtype*, and initializes this with a constant supplied in *value*. - + It also sets *stop_gradient* to True. Args: @@ -141,9 +160,9 @@ def fill_constant_batch_size_like(input, """ **fill_constant_batch_size_like** - This function creates a tensor of specified *shape*, *dtype* and batch size, - and initializes this with a constant supplied in *value*. The batch size is - obtained from the `input` tensor. + This function creates a tensor of specified *shape*, *dtype* and batch size, + and initializes this with a constant supplied in *value*. The batch size is + obtained from the `input` tensor. It also sets *stop_gradient* to True. From 22fba722fb719ea02c95037f8b0b8f494599c754 Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 21 Dec 2017 14:58:24 -0800 Subject: [PATCH 72/84] Add doc for data layer (#6858) --- python/paddle/v2/fluid/layers/io.py | 33 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/python/paddle/v2/fluid/layers/io.py b/python/paddle/v2/fluid/layers/io.py index f4c5907f48..56c3f7b7b7 100644 --- a/python/paddle/v2/fluid/layers/io.py +++ b/python/paddle/v2/fluid/layers/io.py @@ -12,20 +12,9 @@ def data(name, type=core.VarDesc.VarType.LOD_TENSOR, stop_gradient=True): """ - Data Layer. + **Data Layer** - Args: - name: The name/alias of the function - shape: Tuple declaring the shape. - append_batch_size: Whether or not to append the data as a batch. - dtype: The type of data : float32, float_16, int etc - type: The output type. By default it is LOD_TENSOR. - lod_level(int): The LoD Level. 0 means the input data is not a sequence. - main_program: Name of the main program that calls this - startup_program: Name of the startup program - stop_gradient: A boolean that mentions whether gradient should flow. - - This function takes in input and based on whether data has + This function takes in the input and based on whether data has to be returned back as a minibatch, it creates the global variable using the helper functions. The global variables can be accessed by all the following operations and layers in the graph. @@ -33,6 +22,24 @@ def data(name, All the input variables of this function are passed in as local variables to the LayerHelper constructor. + Args: + name(str): The name/alias of the function + shape(list): Tuple declaring the shape. + append_batch_size(bool): Whether or not to append the data as a batch. + dtype(int|float): The type of data : float32, float_16, int etc + type(VarType): The output type. By default it is LOD_TENSOR. + lod_level(int): The LoD Level. 0 means the input data is not a sequence. + main_program(Program): Name of the main program that calls this + startup_program(Program): Name of the startup program + stop_gradient(bool): A boolean that mentions whether gradient should flow. + + Returns: + Variable: The global variable that gives access to the data. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name='x', shape=[784], dtype='float32') """ helper = LayerHelper('data', **locals()) shape = list(shape) From 0bfa1f7c7a07f9e7f095a506451cd2efe08212b8 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Fri, 1 Dec 2017 10:01:04 -0800 Subject: [PATCH 73/84] Enforce drop_empty_grad=false When the input of an op is duplicable. For input argument with a list of variables, drop_empty_grad is not allowed because it makes the correspondence bewteen a variable and its gradient ambiguous. Use REGISTER_OP_EX to register the op or call InputGrad(?,false) in GradOpDescMaker. --- paddle/framework/grad_op_desc_maker.h | 18 ++++++++++ paddle/framework/op_desc.h | 2 ++ paddle/framework/op_registry.h | 43 +++++++++++++++++------- paddle/operators/concat_op.cc | 4 +-- paddle/operators/conditional_block_op.cc | 5 +-- paddle/operators/recurrent_op.cc | 2 +- paddle/operators/sequence_concat_op.cc | 13 +++---- paddle/operators/sum_op.cc | 6 ++-- 8 files changed, 66 insertions(+), 27 deletions(-) diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index 8c47c0b0c8..cf411fa710 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -22,6 +22,14 @@ namespace paddle { namespace framework { +/* + This functor class is responsible for creating the gradient ops for the given + operator fwd_op. After it is called (through operator()), the pairs of + (gradient variable, corresponding input variable of fwd_op) will be added to + grad_to_var. If an input variable of fwd_op is contained in no_grad_set, its + gradient varialbe will be ignored or kEmptyVarName depending on the template + argument DropEmptyIG in the derived classes. + */ class GradOpDescMakerBase { public: explicit GradOpDescMakerBase( @@ -56,6 +64,16 @@ class GradOpDescMakerBase { if (!drop_empty_grad) { return ret_val; } + PADDLE_ENFORCE_LE(var_names.size(), 1UL, + "BUG from operator developer:" + " for input argument with a list of variables, " + " drop_empty_grad is not allowed because it makes" + " the correspondence bewteen a variable and its gradient" + " ambiguous. Use REGISTER_OP_EX to register the op" + " or call InputGrad(?,false) in GradOpDescMaker." + " Op type %s", + fwd_op_.Type()); + std::vector dropped_ret_val; dropped_ret_val.reserve(ret_val.size()); std::copy_if(ret_val.begin(), ret_val.end(), diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 18fa02940d..93d4a88f3c 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -127,7 +127,9 @@ class OpDesc { } proto::OpDesc desc_; + // input arg name => output variable names VariableNameMap inputs_; + // output arg name => output variable names VariableNameMap outputs_; AttributeMap attrs_; diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 278550d496..7f0155b61f 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -126,6 +126,14 @@ class OpKernelRegistrar : public Registrar { __test_global_namespace_##uniq_name##__>::value, \ msg) +/* + The variadic arguments should be class types derived from one of the + following classes: + OpProtoAndCheckerMaker + GradOpDescMakerBase + VarTypeInference + InferShapeBase +*/ #define REGISTER_OPERATOR(op_type, op_class, ...) \ STATIC_ASSERT_GLOBAL_NAMESPACE( \ __reg_op__##op_type, \ @@ -144,20 +152,29 @@ class OpKernelRegistrar : public Registrar { } /** - * Macro to register Operator. + * Macro to register Operator. When the input is duplicable, you should + * use REGISTER_OP_EX with deop_empty_grad=false instead. */ -#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \ - grad_op_class) \ - REGISTER_OPERATOR(grad_op_type, grad_op_class); \ - class _GradOpDescMaker_##grad_op_type##_ \ - : public ::paddle::framework::DefaultGradOpDescMaker { \ - using ::paddle::framework::DefaultGradOpDescMaker< \ - true>::DefaultGradOpDescMaker; \ - \ - protected: \ - virtual std::string GradOpType() const { return #grad_op_type; } \ - }; \ - REGISTER_OPERATOR(op_type, op_class, _GradOpDescMaker_##grad_op_type##_, \ +#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \ + grad_op_class) \ + REGISTER_OP_EX(op_type, op_class, op_maker_class, grad_op_type, \ + grad_op_class, true) + +// When an argument is duplicable, we need to use this version. +// Perhaps we can omit DropEmptyIG template parameter and +// only have one version of REGISTER_OP. +#define REGISTER_OP_EX(op_type, op_class, op_maker_class, grad_op_type, \ + grad_op_class, drop_empty_grad) \ + REGISTER_OPERATOR(grad_op_type, grad_op_class); \ + class _GradOpDescMaker_##grad_op_type##_ \ + : public ::paddle::framework::DefaultGradOpDescMaker { \ + using ::paddle::framework::DefaultGradOpDescMaker< \ + drop_empty_grad>::DefaultGradOpDescMaker; \ + \ + protected: \ + virtual std::string GradOpType() const { return #grad_op_type; } \ + }; \ + REGISTER_OPERATOR(op_type, op_class, _GradOpDescMaker_##grad_op_type##_, \ op_maker_class); #define REGISTER_OP_WITH_KERNEL(op_type, ...) \ diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc index 6151e2e73f..32b61edfd0 100644 --- a/paddle/operators/concat_op.cc +++ b/paddle/operators/concat_op.cc @@ -98,8 +98,8 @@ class ConcatOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(concat, ops::ConcatOp, ops::ConcatOpMaker, concat_grad, - ops::ConcatOpGrad) +REGISTER_OP_EX(concat, ops::ConcatOp, ops::ConcatOpMaker, concat_grad, + ops::ConcatOpGrad, false) REGISTER_OP_CPU_KERNEL(concat, ops::ConcatKernel) REGISTER_OP_CPU_KERNEL(concat_grad, diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 00048a10ca..204be7d1e5 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -178,8 +178,9 @@ class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetInput("Out", Output("Out")); grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); grad_op->SetInput("Scope", Output("Scope")); - grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); - grad_op->SetOutput(framework::GradVarName("Params"), InputGrad("Params")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X", false)); + grad_op->SetOutput(framework::GradVarName("Params"), + InputGrad("Params", false)); grad_op->SetBlockAttr("sub_block", *this->grad_block_[0]); return std::unique_ptr(grad_op); } diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 4273c12354..5981d5745d 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -570,7 +570,7 @@ class RecurrentGradOpDescMaker : public framework::SingleGradOpDescMaker { for (auto &input_param : this->InputNames()) { grad->SetInput(input_param, this->Input(input_param)); grad->SetOutput(framework::GradVarName(input_param), - this->InputGrad(input_param)); + this->InputGrad(input_param, false)); } for (auto &output_param : this->OutputNames()) { diff --git a/paddle/operators/sequence_concat_op.cc b/paddle/operators/sequence_concat_op.cc index 54e8989f25..2f0aad2003 100644 --- a/paddle/operators/sequence_concat_op.cc +++ b/paddle/operators/sequence_concat_op.cc @@ -67,12 +67,12 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { "The level should be less than the level number of inputs.") .SetDefault(0); AddComment(R"DOC( -The sequence_concat operator concatenates multiple LoDTensors. -It only supports sequence (LoD Tensor with level number is 1) +The sequence_concat operator concatenates multiple LoDTensors. +It only supports sequence (LoD Tensor with level number is 1) or a nested sequence (LoD tensor with level number is 2) as its input. - Case1: If the axis is other than 0(here, axis is 1 and level is 1), - each input should have the same LoD information and the LoD + each input should have the same LoD information and the LoD information of the output keeps the same as the input. LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) @@ -80,7 +80,7 @@ or a nested sequence (LoD tensor with level number is 2) as its input. LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) - Case2: - If the axis is 0(here, leve is 0), the inputs are concatenated along + If the axis is 0(here, leve is 0), the inputs are concatenated along time steps, the LoD information of the output need to re-compute. The LoD information of level-1 should be same. @@ -124,8 +124,9 @@ class SequenceConcatGradOp : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(sequence_concat, ops::SequenceConcatOp, ops::SequenceConcatOpMaker, - sequence_concat_grad, ops::SequenceConcatGradOp); +REGISTER_OP_EX(sequence_concat, ops::SequenceConcatOp, + ops::SequenceConcatOpMaker, sequence_concat_grad, + ops::SequenceConcatGradOp, false); REGISTER_OP_CPU_KERNEL( sequence_concat, ops::SequenceConcatOpKernel); diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 36fb5bd29d..891839bf9c 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -106,8 +106,8 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Sum operator. -This operators sums the input tensors. All the inputs can carry the -LoD (Level of Details) information. However, the output only shares +This operators sums the input tensors. All the inputs can carry the +LoD (Level of Details) information. However, the output only shares the LoD information with the first input. )DOC"); } @@ -170,7 +170,7 @@ class SumGradMaker : public framework::GradOpDescMakerBase { using framework::GradOpDescMakerBase::GradOpDescMakerBase; std::vector> operator()() const override { - auto x_grads = InputGrad("X"); + auto x_grads = InputGrad("X", false); std::vector> grad_ops; grad_ops.reserve(x_grads.size()); auto og = OutputGrad("Out"); From a785496b6904fe56754110242a8dceb8ef795221 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 22 Dec 2017 10:19:33 +0800 Subject: [PATCH 74/84] fix logical error --- benchmark/paddle/image/alexnet.py | 32 ++++++-------------- benchmark/paddle/image/run_mkl_infer.sh | 2 +- benchmark/paddle/image/run_mkl_train.sh | 2 +- benchmark/paddle/image/run_openblas_infer.sh | 2 +- benchmark/paddle/image/run_openblas_train.sh | 2 +- 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/benchmark/paddle/image/alexnet.py b/benchmark/paddle/image/alexnet.py index b0beef8ca7..77d130ae34 100644 --- a/benchmark/paddle/image/alexnet.py +++ b/benchmark/paddle/image/alexnet.py @@ -6,7 +6,7 @@ height = 227 width = 227 num_class = 1000 batch_size = get_config_arg('batch_size', int, 128) -use_mkldnn = get_config_arg('use_mkldnn', bool, False) +gp = get_config_arg('layer_num', int, 1) is_infer = get_config_arg("is_infer", bool, False) num_samples = get_config_arg('num_samples', int, 2560) @@ -41,12 +41,7 @@ net = img_pool_layer(input=net, pool_size=3, stride=2) # conv2 net = img_conv_layer( - input=net, - filter_size=5, - num_filters=256, - stride=1, - padding=2, - groups=2 if use_mkldnn else 1) + input=net, filter_size=5, num_filters=256, stride=1, padding=2, groups=gp) net = img_cmrnorm_layer(input=net, size=5, scale=0.0001, power=0.75) net = img_pool_layer(input=net, pool_size=3, stride=2) @@ -55,21 +50,11 @@ net = img_conv_layer( input=net, filter_size=3, num_filters=384, stride=1, padding=1) # conv4 net = img_conv_layer( - input=net, - filter_size=3, - num_filters=384, - stride=1, - padding=1, - groups=2 if use_mkldnn else 1) + input=net, filter_size=3, num_filters=384, stride=1, padding=1, groups=gp) # conv5 net = img_conv_layer( - input=net, - filter_size=3, - num_filters=256, - stride=1, - padding=1, - groups=2 if use_mkldnn else 1) + input=net, filter_size=3, num_filters=256, stride=1, padding=1, groups=gp) net = img_pool_layer(input=net, pool_size=3, stride=2) net = fc_layer( @@ -84,6 +69,9 @@ net = fc_layer( layer_attr=ExtraAttr(drop_rate=0.5)) net = fc_layer(input=net, size=1000, act=SoftmaxActivation()) -lab = data_layer('label', num_class) -loss = cross_entropy(input=net, label=lab) -outputs(loss) +if is_infer: + outputs(net) +else: + lab = data_layer('label', num_class) + loss = cross_entropy(input=net, label=lab) + outputs(loss) diff --git a/benchmark/paddle/image/run_mkl_infer.sh b/benchmark/paddle/image/run_mkl_infer.sh index 00942e32a5..a3b5e2db5e 100755 --- a/benchmark/paddle/image/run_mkl_infer.sh +++ b/benchmark/paddle/image/run_mkl_infer.sh @@ -79,7 +79,7 @@ fi # inference benchmark for use_mkldnn in True False; do for batchsize in 1 2 4 8 16; do - infer alexnet group2 $batchsize $use_mkldnn + infer alexnet 2 $batchsize $use_mkldnn infer googlenet v1 $batchsize $use_mkldnn infer resnet 50 $batchsize $use_mkldnn infer vgg 19 $batchsize $use_mkldnn diff --git a/benchmark/paddle/image/run_mkl_train.sh b/benchmark/paddle/image/run_mkl_train.sh index c38b3e3621..03d2d378fb 100755 --- a/benchmark/paddle/image/run_mkl_train.sh +++ b/benchmark/paddle/image/run_mkl_train.sh @@ -47,6 +47,6 @@ for use_mkldnn in True False; do train vgg 19 $batchsize $use_mkldnn train resnet 50 $batchsize $use_mkldnn train googlenet v1 $batchsize $use_mkldnn - train alexnet group2 $batchsize $use_mkldnn + train alexnet 2 $batchsize $use_mkldnn done done diff --git a/benchmark/paddle/image/run_openblas_infer.sh b/benchmark/paddle/image/run_openblas_infer.sh index 3dad42ee0d..ec9235e2c2 100755 --- a/benchmark/paddle/image/run_openblas_infer.sh +++ b/benchmark/paddle/image/run_openblas_infer.sh @@ -56,7 +56,7 @@ fi # inference benchmark for batchsize in 1 2 4 8 16; do - infer alexnet group2 $batchsize $use_mkldnn + infer alexnet 2 $batchsize $use_mkldnn infer googlenet v1 $batchsize infer resnet 50 $batchsize infer vgg 19 $batchsize diff --git a/benchmark/paddle/image/run_openblas_train.sh b/benchmark/paddle/image/run_openblas_train.sh index caea5548c3..1e007be966 100755 --- a/benchmark/paddle/image/run_openblas_train.sh +++ b/benchmark/paddle/image/run_openblas_train.sh @@ -36,5 +36,5 @@ for batchsize in 64 128 256; do train vgg 19 $batchsize train resnet 50 $batchsize train googlenet v1 $batchsize - train alexnet group2 $batchsize $use_mkldnn + train alexnet 2 $batchsize $use_mkldnn done From 025a6f3c234c07ac34d881db7f9f4dbb47be25b4 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 22 Dec 2017 10:33:02 +0800 Subject: [PATCH 75/84] unify the test reorder --- benchmark/paddle/image/run_mkl_infer.sh | 6 +++--- benchmark/paddle/image/run_openblas_infer.sh | 6 +++--- benchmark/paddle/image/run_openblas_train.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmark/paddle/image/run_mkl_infer.sh b/benchmark/paddle/image/run_mkl_infer.sh index a3b5e2db5e..c22c4deb1c 100755 --- a/benchmark/paddle/image/run_mkl_infer.sh +++ b/benchmark/paddle/image/run_mkl_infer.sh @@ -79,9 +79,9 @@ fi # inference benchmark for use_mkldnn in True False; do for batchsize in 1 2 4 8 16; do - infer alexnet 2 $batchsize $use_mkldnn - infer googlenet v1 $batchsize $use_mkldnn - infer resnet 50 $batchsize $use_mkldnn infer vgg 19 $batchsize $use_mkldnn + infer resnet 50 $batchsize $use_mkldnn + infer googlenet v1 $batchsize $use_mkldnn + infer alexnet 2 $batchsize $use_mkldnn done done diff --git a/benchmark/paddle/image/run_openblas_infer.sh b/benchmark/paddle/image/run_openblas_infer.sh index ec9235e2c2..ba9019c9de 100755 --- a/benchmark/paddle/image/run_openblas_infer.sh +++ b/benchmark/paddle/image/run_openblas_infer.sh @@ -56,8 +56,8 @@ fi # inference benchmark for batchsize in 1 2 4 8 16; do - infer alexnet 2 $batchsize $use_mkldnn - infer googlenet v1 $batchsize - infer resnet 50 $batchsize infer vgg 19 $batchsize + infer resnet 50 $batchsize + infer googlenet v1 $batchsize + infer alexnet 2 $batchsize done diff --git a/benchmark/paddle/image/run_openblas_train.sh b/benchmark/paddle/image/run_openblas_train.sh index 1e007be966..a1b5ee9da8 100755 --- a/benchmark/paddle/image/run_openblas_train.sh +++ b/benchmark/paddle/image/run_openblas_train.sh @@ -36,5 +36,5 @@ for batchsize in 64 128 256; do train vgg 19 $batchsize train resnet 50 $batchsize train googlenet v1 $batchsize - train alexnet 2 $batchsize $use_mkldnn + train alexnet 2 $batchsize done From d7a9bb6e19dd601a554cc157bb741685485cd789 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 21 Dec 2017 11:31:54 +0800 Subject: [PATCH 76/84] add python wrap for sequence_first/last_step --- python/paddle/v2/fluid/layers/nn.py | 10 +++++++++- .../v2/fluid/tests/book/test_machine_translation.py | 2 +- python/paddle/v2/fluid/tests/test_dyn_rnn.py | 5 ++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 59212e8497..ca073b2914 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -13,7 +13,7 @@ __all__ = [ 'crf_decoding', 'cos_sim', 'cross_entropy', 'square_error_cost', 'accuracy', 'chunk_eval', 'sequence_conv', 'conv2d', 'sequence_pool', 'pool2d', 'batch_norm', 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', - 'lstm_unit', 'reduce_sum' + 'lstm_unit', 'reduce_sum', 'sequence_first_step', 'sequence_last_step' ] @@ -583,6 +583,14 @@ def sequence_pool(input, pool_type, **kwargs): return pool_out +def sequence_first_step(input, **kwargs): + return sequence_pool(input=input, pool_type="first") + + +def sequence_last_step(input, **kwargs): + return sequence_pool(input=input, pool_type="last") + + def pool2d(input, pool_size, pool_type, diff --git a/python/paddle/v2/fluid/tests/book/test_machine_translation.py b/python/paddle/v2/fluid/tests/book/test_machine_translation.py index 80ffc5a544..e79864b397 100644 --- a/python/paddle/v2/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/v2/fluid/tests/book/test_machine_translation.py @@ -33,7 +33,7 @@ def encoder_decoder(): fc1 = fluid.layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) - encoder_out = layers.sequence_pool(input=lstm_hidden0, pool_type="last") + encoder_out = layers.sequence_last_step(input=lstm_hidden0) # decoder trg_language_word = layers.data( diff --git a/python/paddle/v2/fluid/tests/test_dyn_rnn.py b/python/paddle/v2/fluid/tests/test_dyn_rnn.py index 034266c26f..8090c5f478 100644 --- a/python/paddle/v2/fluid/tests/test_dyn_rnn.py +++ b/python/paddle/v2/fluid/tests/test_dyn_rnn.py @@ -63,8 +63,7 @@ class TestDynRNN(unittest.TestCase): all_timesteps = fluid.layers.array_to_lod_tensor( x=out, table=rank_table) - last = fluid.layers.sequence_pool( - input=all_timesteps, pool_type='last') + last = fluid.layers.sequence_last_step(input=all_timesteps) logits = fluid.layers.fc(input=last, size=1, act=None) loss = fluid.layers.sigmoid_cross_entropy_with_logits( x=logits, label=label) @@ -101,7 +100,7 @@ class TestDynRNN(unittest.TestCase): rnn.update_memory(mem, out_) rnn.output(out_) - last = fluid.layers.sequence_pool(input=rnn(), pool_type='last') + last = fluid.layers.sequence_last_step(input=rnn()) logits = fluid.layers.fc(input=last, size=1, act=None) label = fluid.layers.data(name='label', shape=[1], dtype='float32') loss = fluid.layers.sigmoid_cross_entropy_with_logits( From 852cd544a9332822f24961ba7e934fdea87a7c6c Mon Sep 17 00:00:00 2001 From: caoying03 Date: Fri, 22 Dec 2017 11:40:54 +0800 Subject: [PATCH 77/84] fix latex equation in fluid fc layer. --- python/paddle/v2/fluid/layers/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index a5bbf4f2bf..d21d9e4d53 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -41,7 +41,7 @@ def fc(input, .. math:: - Out = Act\left({\sum_{i=0}^{N-1}W_iX_i + b}\right) + Out = Act({\sum_{i=0}^{N-1}W_iX_i + b}) In the above equation: From 7961880ed16b10ed1fee4aca7c55500185bd37cd Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 22 Dec 2017 11:49:36 +0800 Subject: [PATCH 78/84] fix cmake require docs --- doc/getstarted/build_and_install/build_from_source_cn.rst | 2 +- doc/getstarted/build_and_install/build_from_source_en.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/getstarted/build_and_install/build_from_source_cn.rst b/doc/getstarted/build_and_install/build_from_source_cn.rst index c875c807b8..22b8b734fa 100644 --- a/doc/getstarted/build_and_install/build_from_source_cn.rst +++ b/doc/getstarted/build_and_install/build_from_source_cn.rst @@ -70,7 +70,7 @@ PaddlePaddle编译需要使用到下面的依赖（包含但不限于），其 :header: "依赖", "版本", "说明" :widths: 10, 15, 30 - "CMake", ">=3.5", "" + "CMake", ">=3.2", "" "GCC", "4.8.2", "推荐使用CentOS的devtools2" "Python", "2.7.x", "依赖libpython2.7.so" "pip", ">=9.0", "" diff --git a/doc/getstarted/build_and_install/build_from_source_en.rst b/doc/getstarted/build_and_install/build_from_source_en.rst index f194f84ce7..a885fc80d6 100644 --- a/doc/getstarted/build_and_install/build_from_source_en.rst +++ b/doc/getstarted/build_and_install/build_from_source_en.rst @@ -76,7 +76,7 @@ will be downloaded automatically. :header: "Dependency", "Version", "Description" :widths: 10, 15, 30 - "CMake", ">=3.5", "" + "CMake", ">=3.2", "" "GCC", "4.8.2", "Recommend devtools2 for CentOS" "Python", "2.7.x", "Need libpython2.7.so" "pip", ">=9.0", "" From abde3130b7ce5b8e8e3c74cd0670be2ce1e8eb6e Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Fri, 22 Dec 2017 12:35:40 +0800 Subject: [PATCH 79/84] "remove GPU Sync Interface" (#6793) * "remove GPU Sync Interface" * "fix typo" * "fix type cast error" * "fix related Copy with stream" * "fix failed tests with DevicePool" * "fix stupid removed position error" --- paddle/framework/executor.h | 10 ++++ paddle/memory/memcpy.cc | 27 ---------- paddle/operators/strided_memcpy_test.cc | 9 ++-- paddle/platform/gpu_info.cc | 11 ---- paddle/platform/gpu_info.h | 4 -- paddle/platform/transform_test.cu | 8 +-- paddle/pybind/tensor_py.h | 25 +++++---- .../v2/fluid/tests/test_batch_norm_op.py | 4 ++ .../v2/fluid/tests/test_gaussian_random_op.py | 45 ++++++++++------ .../v2/fluid/tests/test_uniform_random_op.py | 52 +++++++++++++------ 10 files changed, 104 insertions(+), 91 deletions(-) diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h index 1faaacfefa..fb861d4712 100644 --- a/paddle/framework/executor.h +++ b/paddle/framework/executor.h @@ -40,6 +40,16 @@ class DeviceContextPool { return *pool; } + const platform::DeviceContext* Borrow(const platform::Place& place) { + auto range = device_contexts_.equal_range(place); + if (range.first == range.second) { + PADDLE_THROW( + "'Place' is not supported, Please re-compile with WITH_GPU " + "option"); + } + return range.first->second; + } + std::vector Borrow( const std::vector& places) { PADDLE_ENFORCE_GT(places.size(), 0); diff --git a/paddle/memory/memcpy.cc b/paddle/memory/memcpy.cc index 1df88a6da9..5c629dc3d2 100644 --- a/paddle/memory/memcpy.cc +++ b/paddle/memory/memcpy.cc @@ -62,33 +62,6 @@ void Copy(platform::GPUPlace dst_place, } } -template <> -void Copy(platform::CPUPlace dst_place, - void* dst, - platform::GPUPlace src_place, - const void* src, size_t num) { - platform::SetDeviceId(src_place.device); - platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToHost); -} - -template <> -void Copy(platform::GPUPlace dst_place, - void* dst, - platform::CPUPlace src_place, - const void* src, size_t num) { - platform::SetDeviceId(dst_place.device); - platform::GpuMemcpySync(dst, src, num, cudaMemcpyHostToDevice); -} - -template <> -void Copy(platform::GPUPlace dst_place, - void* dst, - platform::GPUPlace src_place, - const void* src, size_t num) { - platform::SetDeviceId(dst_place.device); - platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToDevice); -} - #endif } // namespace memory diff --git a/paddle/operators/strided_memcpy_test.cc b/paddle/operators/strided_memcpy_test.cc index 68f064eaee..230cc1ab0b 100644 --- a/paddle/operators/strided_memcpy_test.cc +++ b/paddle/operators/strided_memcpy_test.cc @@ -85,8 +85,10 @@ TEST(StridedMemcpy, GPUCrop) { platform::GPUPlace gpu0(0); platform::CPUPlace cpu; + platform::CUDADeviceContext ctx(gpu0); + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); - memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src), ctx.stream()); framework::DDim src_stride({5, 1}); @@ -96,7 +98,6 @@ TEST(StridedMemcpy, GPUCrop) { framework::DDim dst_dim({2, 2}); framework::DDim dst_stride({2, 1}); - platform::CUDADeviceContext ctx(gpu0); StridedMemcpy(ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, gpu_dst); @@ -122,9 +123,10 @@ TEST(StridedMemcpy, GPUConcat) { platform::GPUPlace gpu0(0); platform::CPUPlace cpu; + platform::CUDADeviceContext ctx(gpu0); int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); - memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src), ctx.stream()); int dst[8]; int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); @@ -132,7 +134,6 @@ TEST(StridedMemcpy, GPUConcat) { framework::DDim src_stride({2, 1}); framework::DDim dst_dim({2, 2}); framework::DDim dst_stride({4, 1}); - platform::CUDADeviceContext ctx(gpu0); StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst); StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, diff --git a/paddle/platform/gpu_info.cc b/paddle/platform/gpu_info.cc index 541eca5f39..7037551d75 100644 --- a/paddle/platform/gpu_info.cc +++ b/paddle/platform/gpu_info.cc @@ -97,17 +97,6 @@ void GpuMemcpyAsync(void *dst, const void *src, size_t count, "cudaMemcpyAsync failed in paddle::platform::GpuMemcpyAsync"); } -void GpuMemcpySync(void *dst, const void *src, size_t count, - enum cudaMemcpyKind kind) { - PADDLE_ENFORCE(cudaMemcpy(dst, src, count, kind), - "cudaMemcpy failed in paddle::platform::GpuMemcpySync"); - // note: cudaMemcpy may actually be asynchronous with respect to the caller, - // block on stream 0 to make sure the copy has completed - PADDLE_ENFORCE( - cudaStreamSynchronize(0), - "cudaStreamSynchronize failed in paddle::platform::GpuMemcpySync"); -} - void GpuMemcpyPeer(void *dst, int dst_device, const void *src, int src_device, size_t count, cudaStream_t stream) { PADDLE_ENFORCE( diff --git a/paddle/platform/gpu_info.h b/paddle/platform/gpu_info.h index db961f3838..d05131fa41 100644 --- a/paddle/platform/gpu_info.h +++ b/paddle/platform/gpu_info.h @@ -52,10 +52,6 @@ size_t GpuMaxChunkSize(); void GpuMemcpyAsync(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind, cudaStream_t stream); -//! Copy memory from address src to dst synchronously. -void GpuMemcpySync(void *dst, const void *src, size_t count, - enum cudaMemcpyKind kind); - //! Copy memory from one device to another device. void GpuMemcpyPeer(void *dst, int dst_device, const void *src, int src_device, size_t count, cudaStream_t stream); diff --git a/paddle/platform/transform_test.cu b/paddle/platform/transform_test.cu index d36eac8379..464096111e 100644 --- a/paddle/platform/transform_test.cu +++ b/paddle/platform/transform_test.cu @@ -53,11 +53,11 @@ TEST(Transform, GPUUnary) { CUDADeviceContext ctx(gpu0); float cpu_buf[4] = {0.1, 0.2, 0.3, 0.4}; float* gpu_buf = static_cast(Alloc(gpu0, sizeof(float) * 4)); - Copy(gpu0, gpu_buf, CPUPlace(), cpu_buf, sizeof(cpu_buf)); + Copy(gpu0, gpu_buf, CPUPlace(), cpu_buf, sizeof(cpu_buf), ctx.stream()); Transform trans; trans(ctx, gpu_buf, gpu_buf + 4, gpu_buf, Scale(10)); ctx.Wait(); - Copy(CPUPlace(), cpu_buf, gpu0, gpu_buf, sizeof(cpu_buf)); + Copy(CPUPlace(), cpu_buf, gpu0, gpu_buf, sizeof(cpu_buf), ctx.stream()); Free(gpu0, gpu_buf); for (int i = 0; i < 4; ++i) { ASSERT_NEAR(cpu_buf[i], static_cast(i + 1), 1e-5); @@ -83,11 +83,11 @@ TEST(Transform, GPUBinary) { GPUPlace gpu0(0); CUDADeviceContext ctx(gpu0); int* gpu_buf = static_cast(Alloc(gpu0, sizeof(buf))); - Copy(gpu0, gpu_buf, CPUPlace(), buf, sizeof(buf)); + Copy(gpu0, gpu_buf, CPUPlace(), buf, sizeof(buf), ctx.stream()); Transform trans; trans(ctx, gpu_buf, gpu_buf + 4, gpu_buf, gpu_buf, Multiply()); ctx.Wait(); - Copy(CPUPlace(), buf, gpu0, gpu_buf, sizeof(buf)); + Copy(CPUPlace(), buf, gpu0, gpu_buf, sizeof(buf), ctx.stream()); Free(gpu0, gpu_buf); for (int i = 0; i < 4; ++i) { ASSERT_EQ((i + 1) * (i + 1), buf[i]); diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h index 41fa658502..268a0f2fa3 100644 --- a/paddle/pybind/tensor_py.h +++ b/paddle/pybind/tensor_py.h @@ -14,6 +14,7 @@ #pragma once #include +#include "paddle/framework/executor.h" #include "paddle/framework/tensor.h" #include "paddle/memory/memcpy.h" #include "pybind11/numpy.h" @@ -61,11 +62,15 @@ struct CastToPyBufferImpl { auto *src_ptr = static_cast(tensor.data()); auto *dst_ptr = static_cast(dst_tensor.mutable_data( tensor.dims(), platform::CPUPlace())); - // TODO(qijun): Here we use default CUDA stream to set GPU Tensor to - // a Python numpy array. It's better to manage CDUA stream unifiedly. - paddle::platform::GpuMemcpySync(dst_ptr, src_ptr, - sizeof(CUR_TYPE) * tensor.numel(), - cudaMemcpyDeviceToHost); + + framework::DeviceContextPool &pool = + framework::DeviceContextPool::Get(); + auto dev_ctx = static_cast( + pool.Borrow(tensor.place())); + + paddle::platform::GpuMemcpyAsync( + dst_ptr, src_ptr, sizeof(CUR_TYPE) * tensor.numel(), + cudaMemcpyDeviceToHost, dev_ctx->stream()); #else PADDLE_THROW("'GPUPlace' is not supported in CPU only device."); #endif @@ -132,10 +137,12 @@ void PyCUDATensorSetFromArray( self.Resize(framework::make_ddim(dims)); auto *dst = self.mutable_data(place); - // TODO(qijun): Here we use default CUDA stream to set a Python numpy - // array to a GPU Tensor. It's better to manage CDUA stream unifiedly. - paddle::platform::GpuMemcpySync(dst, array.data(), sizeof(T) * array.size(), - cudaMemcpyHostToDevice); + + framework::DeviceContextPool &pool = framework::DeviceContextPool::Get(); + auto dev_ctx = + static_cast(pool.Borrow(place)); + paddle::platform::GpuMemcpyAsync(dst, array.data(), sizeof(T) * array.size(), + cudaMemcpyHostToDevice, dev_ctx->stream()); } #endif diff --git a/python/paddle/v2/fluid/tests/test_batch_norm_op.py b/python/paddle/v2/fluid/tests/test_batch_norm_op.py index dee2febb83..ec71d391e6 100644 --- a/python/paddle/v2/fluid/tests/test_batch_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_batch_norm_op.py @@ -341,6 +341,10 @@ class TestBatchNormOp(OpTest): places = [core.CPUPlace()] if core.is_compile_gpu() and core.op_support_gpu("batch_norm"): places.append(core.GPUPlace(0)) + + core.init_devices(["CPU", "GPU:0"]) + else: + core.init_devices(["CPU"]) for place in places: for data_format in ["NCHW", "NHWC"]: test_with_place(place, data_format, [2, 3, 4, 5]) diff --git a/python/paddle/v2/fluid/tests/test_gaussian_random_op.py b/python/paddle/v2/fluid/tests/test_gaussian_random_op.py index 627ab4e235..a9d943b8b7 100644 --- a/python/paddle/v2/fluid/tests/test_gaussian_random_op.py +++ b/python/paddle/v2/fluid/tests/test_gaussian_random_op.py @@ -1,32 +1,47 @@ import unittest +import numpy + +import paddle.v2.fluid as fluid import paddle.v2.fluid.core as core from paddle.v2.fluid.op import Operator -import numpy +from paddle.v2.fluid.executor import Executor class TestGaussianRandomOp(unittest.TestCase): + def setUp(self): + self.op_type = "gaussian_random" + self.inputs = {} + self.attrs = {"shape": [1000, 784], "mean": .0, "std": 1., "seed": 10} + + self.outputs = ["Out"] + def test_cpu(self): - self.gaussian_random_test(place=core.CPUPlace()) + self.gaussian_random_test(place=fluid.CPUPlace()) def test_gpu(self): if core.is_compile_gpu(): - self.gaussian_random_test(place=core.GPUPlace(0)) + self.gaussian_random_test(place=fluid.GPUPlace(0)) def gaussian_random_test(self, place): - scope = core.Scope() - scope.var('Out').get_tensor() - - op = Operator( - "gaussian_random", - Out='Out', - shape=[1000, 784], - mean=.0, - std=1., - seed=10) context = core.DeviceContext.create(place) - op.run(scope, context) - tensor = numpy.array(scope.find_var('Out').get_tensor()) + program = fluid.Program() + block = program.global_block() + vout = block.create_var(name="Out") + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs) + + op.desc.infer_var_type(block.desc) + op.desc.infer_shape(block.desc) + + fetch_list = [] + for var_name in self.outputs: + fetch_list.append(block.var(var_name)) + + exe = Executor(place) + outs = exe.run(program, fetch_list=fetch_list) + tensor = outs[0] + self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1) self.assertAlmostEqual(numpy.std(tensor), 1., delta=0.1) diff --git a/python/paddle/v2/fluid/tests/test_uniform_random_op.py b/python/paddle/v2/fluid/tests/test_uniform_random_op.py index f736dfb2e8..00b4f19620 100644 --- a/python/paddle/v2/fluid/tests/test_uniform_random_op.py +++ b/python/paddle/v2/fluid/tests/test_uniform_random_op.py @@ -1,32 +1,50 @@ import unittest +import numpy + from paddle.v2.fluid.op import Operator import paddle.v2.fluid.core as core -import numpy +import paddle.v2.fluid as fluid class TestUniformRandomOp(unittest.TestCase): - def test_uniform_random_cpu(self): + def setUp(self): + self.op_type = "uniform_random" + self.inputs = {} + self.attrs = { + "shape": [1000, 784], + "min": -5.0, + "max": 10.0, + "seed": 10 + } + self.outputs = ["Out"] + + def test_cpu(self): self.uniform_random_test(place=core.CPUPlace()) - def test_uniform_random_gpu(self): + def test_gpu(self): if core.is_compile_gpu(): self.uniform_random_test(place=core.GPUPlace(0)) def uniform_random_test(self, place): - scope = core.Scope() - scope.var('X').get_tensor() - - op = Operator( - "uniform_random", - Out='X', - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) - - ctx = core.DeviceContext.create(place) - op.run(scope, ctx) - tensor = numpy.array(scope.find_var('X').get_tensor()) + context = core.DeviceContext.create(place) + program = fluid.Program() + block = program.global_block() + vout = block.create_var(name="Out") + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs) + + op.desc.infer_var_type(block.desc) + op.desc.infer_shape(block.desc) + + fetch_list = [] + for var_name in self.outputs: + fetch_list.append(block.var(var_name)) + + exe = fluid.Executor(place) + outs = exe.run(program, fetch_list=fetch_list) + + tensor = outs[0] + self.assertAlmostEqual(tensor.mean(), 2.5, delta=0.1) From 817cae0a5ccd4ab622e63f963cd839a4b4dbbe56 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 22 Dec 2017 12:44:47 +0800 Subject: [PATCH 80/84] update --- doc/getstarted/build_and_install/build_from_source_cn.rst | 8 ++++---- doc/getstarted/build_and_install/build_from_source_en.rst | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/getstarted/build_and_install/build_from_source_cn.rst b/doc/getstarted/build_and_install/build_from_source_cn.rst index 22b8b734fa..41ac07ca56 100644 --- a/doc/getstarted/build_and_install/build_from_source_cn.rst +++ b/doc/getstarted/build_and_install/build_from_source_cn.rst @@ -72,11 +72,11 @@ PaddlePaddle编译需要使用到下面的依赖（包含但不限于），其 "CMake", ">=3.2", "" "GCC", "4.8.2", "推荐使用CentOS的devtools2" - "Python", "2.7.x", "依赖libpython2.7.so" - "pip", ">=9.0", "" - "numpy", "", "" + "Python", "2.7.x", "依赖libpython2.7.so" + "pip", ">=9.0", "" + "numpy", "", "" "SWIG", ">=2.0", "" - "Go", ">=1.8", "可选" + "Go", ">=1.8", "可选" .. _build_options: diff --git a/doc/getstarted/build_and_install/build_from_source_en.rst b/doc/getstarted/build_and_install/build_from_source_en.rst index a885fc80d6..92211aee8c 100644 --- a/doc/getstarted/build_and_install/build_from_source_en.rst +++ b/doc/getstarted/build_and_install/build_from_source_en.rst @@ -78,11 +78,11 @@ will be downloaded automatically. "CMake", ">=3.2", "" "GCC", "4.8.2", "Recommend devtools2 for CentOS" - "Python", "2.7.x", "Need libpython2.7.so" - "pip", ">=9.0", "" - "numpy", "", "" + "Python", "2.7.x", "Need libpython2.7.so" + "pip", ">=9.0", "" + "numpy", "", "" "SWIG", ">=2.0", "" - "Go", ">=1.8", "Optional" + "Go", ">=1.8", "Optional" .. _build_options: From f3fc8de1d5a12af4657d2005f1f98a1236784f1c Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 22 Dec 2017 12:55:06 +0800 Subject: [PATCH 81/84] add doc for sequence_first/last_step --- doc/api/v2/fluid/layers.rst | 12 ++++++ python/paddle/v2/fluid/layers/nn.py | 58 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 842f3b1800..fc29795d12 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -170,6 +170,18 @@ sequence_pool :noindex: +sequence_first_step +------------------- +.. autofunction:: paddle.v2.fluid.layers.sequence_first_step + :noindex: + + +sequence_last_step +------------------ +.. autofunction:: paddle.v2.fluid.layers.sequence_last_step + :noindex: + + pool2d --------- .. autofunction:: paddle.v2.fluid.layers.pool2d diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index ca073b2914..3536a7e390 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -584,10 +584,68 @@ def sequence_pool(input, pool_type, **kwargs): def sequence_first_step(input, **kwargs): + """ + This funciton get the first step of sequence. + + .. code-block:: text + + x is a 1-level LoDTensor: + x.lod = [[0, 2, 5, 7]] + x.data = [1, 3, 2, 4, 6, 5, 1] + x.dims = [7, 1] + + then output is a Tensor: + out.dim = [3, 1] + with condition len(x.lod[-1]) - 1 == out.dims[0] + out.data = [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) + + Args: + input(variable): The input variable which is a LoDTensor. + + Returns: + The sequence's first step variable which is a Tensor. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + x_first_step = fluid.layers.sequence_first_step(input=x) + """ return sequence_pool(input=input, pool_type="first") def sequence_last_step(input, **kwargs): + """ + This funciton get the last step of sequence. + + .. code-block:: text + + x is a 1-level LoDTensor: + x.lod = [[0, 2, 5, 7]] + x.data = [1, 3, 2, 4, 6, 5, 1] + x.dims = [7, 1] + + then output is a Tensor: + out.dim = [3, 1] + with condition len(x.lod[-1]) - 1 == out.dims[0] + out.data = [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) + + Args: + input(variable): The input variable which is a LoDTensor. + + Returns: + The sequence's last step variable which is a Tensor. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + x_last_step = fluid.layers.sequence_last_step(input=x) + """ return sequence_pool(input=input, pool_type="last") From 11f4f89bc1835164ab9bacb8fe1f939a602a9971 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Fri, 22 Dec 2017 12:57:23 +0800 Subject: [PATCH 82/84] Fix Compile --- paddle/operators/reorder_lod_tensor_by_rank_op.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/operators/reorder_lod_tensor_by_rank_op.cc b/paddle/operators/reorder_lod_tensor_by_rank_op.cc index 369bd4391c..5e3079ee0c 100644 --- a/paddle/operators/reorder_lod_tensor_by_rank_op.cc +++ b/paddle/operators/reorder_lod_tensor_by_rank_op.cc @@ -173,13 +173,13 @@ class ReorderLodTensorByRankGradOpMaker using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - std::unique_ptr Apply() const override { - auto *grad_op = new framework::OpDescBind(); + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); grad_op->SetType("reorder_lod_tensor_by_rank_grad"); grad_op->SetInput("X", OutputGrad("Out")); grad_op->SetOutput("Out", InputGrad("X")); grad_op->SetInput("RankTable", Input("RankTable")); - return std::unique_ptr(grad_op); + return std::unique_ptr(grad_op); } }; From ad6d6e9cbab53a4c7221fd1fddbbaabc402a3d5f Mon Sep 17 00:00:00 2001 From: QI JUN Date: Fri, 22 Dec 2017 13:39:24 +0800 Subject: [PATCH 83/84] add library type (#6874) --- paddle/framework/library_type.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 paddle/framework/library_type.h diff --git a/paddle/framework/library_type.h b/paddle/framework/library_type.h new file mode 100644 index 0000000000..68e9cabb66 --- /dev/null +++ b/paddle/framework/library_type.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +namespace paddle { +namespace framework { + +// For more details about the design of LibraryType, Please refer to +// https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/operator_kernel_type.md#library + +enum LibraryType { kPlain = 0; kMKLDNN = 1; kCUDNN = 2; } + +} // namespace +} // framework From 6b47598103f8a7c0f76940546fbca1e1ae1baf52 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Fri, 22 Dec 2017 13:40:28 +0800 Subject: [PATCH 84/84] add data layout (#6832) * add data layout * fix ci --- paddle/framework/data_layout.h | 37 +++++++++++ paddle/operators/batch_norm_op.cc | 64 ++++++++++--------- paddle/operators/batch_norm_op.cu.cc | 35 +++++----- paddle/operators/batch_norm_op.h | 15 ----- .../v2/fluid/tests/test_batch_norm_op.py | 8 +-- 5 files changed, 92 insertions(+), 67 deletions(-) create mode 100644 paddle/framework/data_layout.h diff --git a/paddle/framework/data_layout.h b/paddle/framework/data_layout.h new file mode 100644 index 0000000000..7429de7ee3 --- /dev/null +++ b/paddle/framework/data_layout.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +namespace paddle { +namespace framework { + +enum DataLayout { + kNHWC = 0, + kNCHW = 1, + kAnyLayout = 2, +}; + +inline DataLayout StringToDataLayout(const std::string& str) { + if (str == "NHWC" || str == "nhwc") { + return DataLayout::kNHWC; + } else if (str == "NCHW" || str == "nchw") { + return DataLayout::kNCHW; + } else { + PADDLE_THROW("Unknown storage order string: %s", str); + } +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/operators/batch_norm_op.cc b/paddle/operators/batch_norm_op.cc index f545da22d7..1c14acbe11 100644 --- a/paddle/operators/batch_norm_op.cc +++ b/paddle/operators/batch_norm_op.cc @@ -13,12 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/batch_norm_op.h" +#include "paddle/framework/data_layout.h" namespace paddle { namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; +using DataLayout = framework::DataLayout; template using EigenArrayMap = @@ -60,15 +62,15 @@ class BatchNormOp : public framework::OperatorWithKernel { "Variance and VarianceOut should share the same memory"); const auto x_dims = ctx->GetInputDim("X"); - const TensorFormat tensor_format = - StringToTensorFormat(ctx->Attrs().Get("tensor_format")); + const DataLayout data_layout = framework::StringToDataLayout( + ctx->Attrs().Get("data_layout")); PADDLE_ENFORCE(x_dims.size() >= 2 && x_dims.size() <= 5, "Input X must have 2 to 5 dimensions."); const int C = - (tensor_format == TensorFormat::NCHW ? x_dims[1] - : x_dims[x_dims.size() - 1]); + (data_layout == DataLayout::kNCHW ? x_dims[1] + : x_dims[x_dims.size() - 1]); PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale").size(), 1UL); PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale")[0], C); @@ -90,7 +92,7 @@ class BatchNormOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr("is_test", "").SetDefault(false); AddAttr("momentum", "").SetDefault(0.9); AddAttr("epsilon", "").SetDefault(1e-5); - AddAttr("tensor_format", "").SetDefault("NCHW"); + AddAttr("data_layout", "").SetDefault("NCHW"); AddInput("X", "The input tensor"); AddInput("Scale", "Scale is a 1-dimensional tensor of size C " @@ -141,9 +143,9 @@ class BatchNormKernel const float epsilon = ctx.Attr("epsilon"); const float momentum = ctx.Attr("momentum"); const bool is_test = ctx.Attr("is_test"); - const std::string tensor_format_str = - ctx.Attr("tensor_format"); - const TensorFormat tensor_format = StringToTensorFormat(tensor_format_str); + const std::string data_layout_str = ctx.Attr("data_layout"); + const DataLayout data_layout = + framework::StringToDataLayout(data_layout_str); const auto *x = ctx.Input("X"); const auto &x_dims = x->dims(); @@ -151,8 +153,8 @@ class BatchNormKernel "The Input dim size should be between 2 and 5"); const int N = x_dims[0]; const int C = - (tensor_format == TensorFormat::NCHW ? x_dims[1] - : x_dims[x_dims.size() - 1]); + (data_layout == DataLayout::kNCHW ? x_dims[1] + : x_dims[x_dims.size() - 1]); const int sample_size = x->numel() / N / C; auto *y = ctx.Output("Y"); @@ -177,8 +179,8 @@ class BatchNormKernel saved_mean_e.setZero(); saved_variance_e.setZero(); - switch (tensor_format) { - case TensorFormat::NCHW: { + switch (data_layout) { + case DataLayout::kNCHW: { ConstEigenArrayMap x_arr(x->data(), sample_size, N * C); for (int nc = 0; nc < N * C; ++nc) { saved_mean_e(nc % C) += x_arr.col(nc).sum(); @@ -191,7 +193,7 @@ class BatchNormKernel saved_variance_e /= N * sample_size; break; } - case TensorFormat::NHWC: { + case DataLayout::kNHWC: { ConstEigenArrayMap x_arr(x->data(), C, N * sample_size); for (int i = 0; i < N * sample_size; ++i) { saved_mean_e += x_arr.col(i); @@ -205,7 +207,7 @@ class BatchNormKernel break; } default: - PADDLE_THROW("Unknown storage order: %s", tensor_format_str); + PADDLE_THROW("Unknown storage order: %s", data_layout_str); } EigenVectorArrayMap running_mean_arr( @@ -247,8 +249,8 @@ class BatchNormKernel Eigen::Array new_bias = bias_arr - mean_arr * inv_std * scale_arr; - switch (tensor_format) { - case TensorFormat::NCHW: { + switch (data_layout) { + case DataLayout::kNCHW: { EigenArrayMap y_arr(y->mutable_data(ctx.GetPlace()), sample_size, N * C); ConstEigenArrayMap x_arr(x->data(), sample_size, N * C); @@ -257,7 +259,7 @@ class BatchNormKernel } break; } - case TensorFormat::NHWC: { + case DataLayout::kNHWC: { EigenArrayMap(y->mutable_data(ctx.GetPlace()), C, N * sample_size) = (ConstEigenArrayMap(x->data(), C, N * sample_size).colwise() * @@ -267,7 +269,7 @@ class BatchNormKernel break; } default: - PADDLE_THROW("Unknown storage order: %d", tensor_format); + PADDLE_THROW("Unknown storage order: %d", data_layout); } } }; @@ -290,11 +292,11 @@ class BatchNormGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("Bias")), ""); const auto x_dims = ctx->GetInputDim("X"); - const TensorFormat tensor_format = - StringToTensorFormat(ctx->Attrs().Get("tensor_format")); + const DataLayout data_layout = framework::StringToDataLayout( + ctx->Attrs().Get("data_layout")); const int C = - (tensor_format == TensorFormat::NCHW ? x_dims[1] - : x_dims[x_dims.size() - 1]); + (data_layout == DataLayout::kNCHW ? x_dims[1] + : x_dims[x_dims.size() - 1]); ctx->SetOutputDim(framework::GradVarName("X"), x_dims); ctx->SetOutputDim(framework::GradVarName("Scale"), {C}); @@ -333,9 +335,9 @@ class BatchNormGradKernel const auto *saved_mean = ctx.Input("SavedMean"); // SavedVariance have been reverted in forward operator const auto *saved_inv_variance = ctx.Input("SavedVariance"); - const std::string tensor_format_str = - ctx.Attr("tensor_format"); - const TensorFormat tensor_format = StringToTensorFormat(tensor_format_str); + const std::string data_layout_str = ctx.Attr("data_layout"); + const DataLayout data_layout = + framework::StringToDataLayout(data_layout_str); // Get the size for each dimension. // NCHW [batch_size, in_channels, in_height, in_width] @@ -344,8 +346,8 @@ class BatchNormGradKernel "The Input dim size should be between 2 and 5"); const int N = x_dims[0]; const int C = - (tensor_format == TensorFormat::NCHW ? x_dims[1] - : x_dims[x_dims.size() - 1]); + (data_layout == DataLayout::kNCHW ? x_dims[1] + : x_dims[x_dims.size() - 1]); const int sample_size = x->numel() / N / C; ConstEigenVectorArrayMap scale_arr(scale->data(), C); @@ -376,8 +378,8 @@ class BatchNormGradKernel const auto scale_inv_var_nhw = scale_arr * inv_var_arr / (N * sample_size); - switch (tensor_format) { - case TensorFormat::NCHW: { + switch (data_layout) { + case DataLayout::kNCHW: { ConstEigenArrayMap x_arr(x->data(), sample_size, N * C); ConstEigenArrayMap d_y_arr(d_y->data(), sample_size, N * C); EigenArrayMap d_x_arr(d_x->mutable_data(ctx.GetPlace()), @@ -400,7 +402,7 @@ class BatchNormGradKernel } break; } - case TensorFormat::NHWC: { + case DataLayout::kNHWC: { ConstEigenArrayMap x_arr(x->data(), C, N * sample_size); ConstEigenArrayMap d_y_arr(d_y->data(), C, N * sample_size); EigenArrayMap d_x_arr(d_x->mutable_data(ctx.GetPlace()), C, @@ -425,7 +427,7 @@ class BatchNormGradKernel break; } default: - PADDLE_THROW("Unknown storage order: %s", tensor_format_str); + PADDLE_THROW("Unknown storage order: %s", data_layout_str); } } }; diff --git a/paddle/operators/batch_norm_op.cu.cc b/paddle/operators/batch_norm_op.cu.cc index c7adc3d80e..55d0736a4c 100644 --- a/paddle/operators/batch_norm_op.cu.cc +++ b/paddle/operators/batch_norm_op.cu.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/batch_norm_op.h" +#include "paddle/framework/data_layout.h" #include #include "paddle/operators/math/math_function.h" @@ -22,12 +23,12 @@ namespace paddle { namespace operators { using Tensor = framework::Tensor; +using DataLayout = framework::DataLayout; template using CudnnDataType = platform::CudnnDataType; -void ExtractNCWHD(const framework::DDim &dims, - const TensorFormat &tensor_format, int *N, int *C, int *H, - int *W, int *D) { +void ExtractNCWHD(const framework::DDim &dims, const DataLayout &data_layout, + int *N, int *C, int *H, int *W, int *D) { *N = dims[0]; if (dims.size() == 2) { *C = dims[1]; @@ -35,13 +36,13 @@ void ExtractNCWHD(const framework::DDim &dims, *W = 1; *D = 1; } else { - *C = tensor_format == TensorFormat::NCHW ? dims[1] : dims[dims.size() - 1]; - *H = tensor_format == TensorFormat::NCHW ? dims[2] : dims[1]; + *C = data_layout == DataLayout::kNCHW ? dims[1] : dims[dims.size() - 1]; + *H = data_layout == DataLayout::kNCHW ? dims[2] : dims[1]; *W = dims.size() > 3 - ? (tensor_format == TensorFormat::NCHW ? dims[3] : dims[2]) + ? (data_layout == DataLayout::kNCHW ? dims[3] : dims[2]) : 1; *D = dims.size() > 4 - ? (tensor_format == TensorFormat::NCHW ? dims[4] : dims[3]) + ? (data_layout == DataLayout::kNCHW ? dims[4] : dims[3]) : 1; } } @@ -56,9 +57,9 @@ class BatchNormKernel double epsilon = static_cast(ctx.Attr("epsilon")); const float momentum = ctx.Attr("momentum"); const bool is_test = ctx.Attr("is_test"); - const std::string tensor_format_str = - ctx.Attr("tensor_format"); - const TensorFormat tensor_format = StringToTensorFormat(tensor_format_str); + const std::string data_layout_str = ctx.Attr("data_layout"); + const DataLayout data_layout = + framework::StringToDataLayout(data_layout_str); // Get the size for each dimension. // NCHW [batch_size, in_channels, in_height, in_width] @@ -67,7 +68,7 @@ class BatchNormKernel PADDLE_ENFORCE(x_dims.size() >= 2 && x_dims.size() <= 5, "The Input dim size should be between 2 and 5"); int N, C, H, W, D; - ExtractNCWHD(x_dims, tensor_format, &N, &C, &H, &W, &D); + ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D); // ------------------- cudnn descriptors --------------------- cudnnTensorDescriptor_t data_desc_; @@ -93,7 +94,7 @@ class BatchNormKernel VLOG(1) << "Setting descriptors."; std::vector dims; std::vector strides; - if (tensor_format == TensorFormat::NCHW) { + if (data_layout == DataLayout::kNCHW) { dims = {N, C, H, W, D}; strides = {C * H * W * D, H * W * D, W * D, D, 1}; } else { @@ -180,9 +181,9 @@ class BatchNormGradKernel PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), "It must use GPUPlace."); double epsilon = static_cast(ctx.Attr("epsilon")); - const std::string tensor_format_str = - ctx.Attr("tensor_format"); - const TensorFormat tensor_format = StringToTensorFormat(tensor_format_str); + const std::string data_layout_str = ctx.Attr("data_layout"); + const DataLayout data_layout = + framework::StringToDataLayout(data_layout_str); const auto *x = ctx.Input("X"); const auto *d_y = ctx.Input(framework::GradVarName("Y")); const auto *scale = ctx.Input("Scale"); @@ -192,7 +193,7 @@ class BatchNormGradKernel PADDLE_ENFORCE(x_dims.size() >= 2 && x_dims.size() <= 5, "The Input dim size should be between 2 and 5"); int N, C, H, W, D; - ExtractNCWHD(x_dims, tensor_format, &N, &C, &H, &W, &D); + ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D); PADDLE_ENFORCE_EQ(scale->dims().size(), 1UL); PADDLE_ENFORCE_EQ(scale->dims()[0], C); @@ -219,7 +220,7 @@ class BatchNormGradKernel std::vector dims; std::vector strides; - if (tensor_format == TensorFormat::NCHW) { + if (data_layout == DataLayout::kNCHW) { dims = {N, C, H, W, D}; strides = {C * H * W * D, H * W * D, W * D, D, 1}; } else { diff --git a/paddle/operators/batch_norm_op.h b/paddle/operators/batch_norm_op.h index 8d99b68647..a817ef41fc 100644 --- a/paddle/operators/batch_norm_op.h +++ b/paddle/operators/batch_norm_op.h @@ -19,21 +19,6 @@ limitations under the License. */ namespace paddle { namespace operators { -enum TensorFormat { - NHWC = 0, - NCHW = 1, -}; - -inline TensorFormat StringToTensorFormat(const std::string& str) { - if (str == "NHWC" || str == "nhwc") { - return TensorFormat::NHWC; - } else if (str == "NCHW" || str == "nchw") { - return TensorFormat::NCHW; - } else { - PADDLE_THROW("Unknown storage order string: %s", str); - } -} - template class BatchNormKernel : public framework::OpKernel { public: diff --git a/python/paddle/v2/fluid/tests/test_batch_norm_op.py b/python/paddle/v2/fluid/tests/test_batch_norm_op.py index ec71d391e6..a9c0b1cfd3 100644 --- a/python/paddle/v2/fluid/tests/test_batch_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_batch_norm_op.py @@ -208,7 +208,7 @@ class TestBatchNormOp(OpTest): print 'python: NHWC, NCHW, backward checking passed' def test_forward_backward(self): - def test_with_place(place, tensor_format, shape): + def test_with_place(place, data_layout, shape): # attr epsilon = 0.00001 momentum = 0.9 @@ -292,7 +292,7 @@ class TestBatchNormOp(OpTest): SavedVariance="saved_variance", # attrs is_test=False, - tensor_format=tensor_format, + data_layout=data_layout, momentum=momentum, epsilon=epsilon) @@ -311,7 +311,7 @@ class TestBatchNormOp(OpTest): atol = 1e-4 self.__assert_close(variance_out_tensor, variance_out, "variance_out", atol) - print "op test forward passed: ", str(place), tensor_format + print "op test forward passed: ", str(place), data_layout # run backward batch_norm_op_grad = get_backward_op(scope, batch_norm_op, set()) @@ -336,7 +336,7 @@ class TestBatchNormOp(OpTest): self.__assert_close(x_grad_tensor, x_grad_ref, "x_grad") self.__assert_close(scale_grad_tensor, scale_grad_ref, "scale_grad") self.__assert_close(bias_grad_tensor, bias_grad_ref, "bias_grad") - print "op test backward passed: ", str(place), tensor_format + print "op test backward passed: ", str(place), data_layout places = [core.CPUPlace()] if core.is_compile_gpu() and core.op_support_gpu("batch_norm"):