From 859dba591baaac50656fe3e25ab5a50a17445fc5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 21 Sep 2017 20:13:07 -0700 Subject: [PATCH 01/30] Init commit --- paddle/pybind/pybind.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index c7009a604f..90b995decb 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -315,6 +315,23 @@ All parameter, weight, gradient are variables in Paddle. m.def("is_compile_gpu", IsCompileGPU); + py::class_(m, "ProgramDesc", "") + .def_static("instance", [] { return &GetProgramDesc(); }) + .def("append_block", [](ProgramDesc &self) { + auto desc = self.mutable_blocks()->Add(); + desc->set_idx(self.mutable_blocks()->size() - 1); + return desc; + }); + py::class_(m, "BlockDesc", "") + .def("idx", [](BlockDesc &self) { return self.idx(); }) + .def("set_parent", + [](BlockDesc &self, int32_t idx) { self.set_parent_idx(idx); }) + .def("parent", [](BlockDesc &self) { return self.parent_idx(); }); + + py::class_(m, "VarDesc", ""); + + py::class_(m, "OpDesc", ""); + return m.ptr(); } } // namespace framework From 7788b4605500d22ac31359115b3d341faa610080 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 21 Sep 2017 21:13:14 -0700 Subject: [PATCH 02/30] Expose VarDesc interface --- paddle/pybind/pybind.cc | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 90b995decb..e1f7bc8672 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -328,7 +328,31 @@ All parameter, weight, gradient are variables in Paddle. [](BlockDesc &self, int32_t idx) { self.set_parent_idx(idx); }) .def("parent", [](BlockDesc &self) { return self.parent_idx(); }); - py::class_(m, "VarDesc", ""); + py::class_(m, "VarDesc", "") + .def(py::init<>()) + .def("set_name", + [](VarDesc &self, const std::string &name) { self.set_name(name); }) + .def("set_shape", + [](VarDesc &self, const std::vector &dims) { + LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); + for (const int64_t &i : dims) { + lod_tensor_desc->add_dims(i); + } + }) + .def("set_data_type", + [](VarDesc &self, int type_id) { + LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); + lod_tensor_desc->set_data_type(static_cast(type_id)); + }) + .def("shape", [](VarDesc &self) { + const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); + int rank = lod_tensor_desc.dims_size(); + std::vector res(rank); + for (int i = 0; i < rank; ++i) { + res[i] = lod_tensor_desc.dims(i); + } + return res; + }); py::class_(m, "OpDesc", ""); From 70f398e2074b84701dc9b1b16f518ed9b9b16b62 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 21 Sep 2017 21:17:47 -0700 Subject: [PATCH 03/30] Update --- paddle/framework/attribute.cc | 3 + paddle/pybind/pybind.cc | 74 ++++++++++++++++--- .../v2/framework/tests/test_protobuf_descs.py | 16 ++++ 3 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 python/paddle/v2/framework/tests/test_protobuf_descs.py diff --git a/paddle/framework/attribute.cc b/paddle/framework/attribute.cc index 159ed03b92..0a305e8a8c 100644 --- a/paddle/framework/attribute.cc +++ b/paddle/framework/attribute.cc @@ -24,6 +24,9 @@ static ProgramDesc* g_program_desc = nullptr; ProgramDesc& GetProgramDesc() { if (g_program_desc == nullptr) { g_program_desc = new ProgramDesc(); + auto root_block = g_program_desc->mutable_blocks()->Add(); + root_block->set_idx(0); + root_block->set_parent_idx(-1); } return *g_program_desc; } diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 90b995decb..835ea85aa1 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -316,21 +316,75 @@ All parameter, weight, gradient are variables in Paddle. m.def("is_compile_gpu", IsCompileGPU); py::class_(m, "ProgramDesc", "") - .def_static("instance", [] { return &GetProgramDesc(); }) - .def("append_block", [](ProgramDesc &self) { - auto desc = self.mutable_blocks()->Add(); - desc->set_idx(self.mutable_blocks()->size() - 1); - return desc; - }); + .def_static("instance", + [] { return &GetProgramDesc(); }, + py::return_value_policy::reference) + .def("append_block", + [](ProgramDesc &self, BlockDesc &parent) { + auto desc = self.mutable_blocks()->Add(); + desc->set_idx(self.mutable_blocks()->size() - 1); + desc->set_parent_idx(parent.idx()); + return desc; + }) + .def("root_block", + [](ProgramDesc &self) { return self.mutable_blocks()[0]; }); py::class_(m, "BlockDesc", "") .def("idx", [](BlockDesc &self) { return self.idx(); }) - .def("set_parent", - [](BlockDesc &self, int32_t idx) { self.set_parent_idx(idx); }) - .def("parent", [](BlockDesc &self) { return self.parent_idx(); }); + .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) + .def("append_op", + [](BlockDesc &self) { return self.mutable_ops()->Add(); }); py::class_(m, "VarDesc", ""); - py::class_(m, "OpDesc", ""); + auto op_desc_set_var = [](OpDesc::Var *var, + const std::string ¶meter, + const std::vector &arguments) { + var->set_parameter(parameter); + auto args = var->mutable_arguments(); + args->Reserve(static_cast(arguments.size())); + for (auto &arg : arguments) { + *args->Add() = arg; + } + }; + + auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { + auto attr = desc.mutable_attrs()->Add(); + attr->set_name(name); + return attr; + }; + + py::class_(m, "OpDesc", "") + .def("type", [](OpDesc &op) { return op.type(); }) + .def("set_input", + [op_desc_set_var](OpDesc &self, + const std::string ¶meter, + const std::vector &arguments) { + auto ipt = self.mutable_inputs()->Add(); + op_desc_set_var(ipt, parameter, arguments); + }) + .def("input_names", + [](OpDesc &self) { + std::vector ret_val; + ret_val.reserve(static_cast(self.inputs().size())); + std::transform( + self.inputs().begin(), + self.inputs().end(), + std::back_inserter(ret_val), + [](const OpDesc::Var &var) { return var.parameter(); }); + return ret_val; + }) + .def("__str__", [](OpDesc &self) { return self.DebugString(); }) + .def("set_output", + [op_desc_set_var](OpDesc &self, + const std::string ¶meter, + const std::vector &arguments) { + auto opt = self.mutable_outputs()->Add(); + op_desc_set_var(opt, parameter, arguments); + }) + .def("set_attr", + [op_desc_set_attr](OpDesc &self, const std::string &name, int i) { + op_desc_set_attr(self, name)->set_i(i); + }); return m.ptr(); } diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py new file mode 100644 index 0000000000..945610ff45 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -0,0 +1,16 @@ +import unittest +import paddle.v2.framework.core as core + + +class TestProgramDesc(unittest.TestCase): + def test_instance(self): + program_desc = core.ProgramDesc.instance() + self.assertIsNotNone(program_desc) + del program_desc + program_desc = core.ProgramDesc.instance() + self.assertIsNotNone(program_desc) + del program_desc + + +if __name__ == '__main__': + unittest.main() From 332369ca5c72fbe88f2504f71285b25247cc966e Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 21 Sep 2017 21:32:42 -0700 Subject: [PATCH 04/30] Add `new_var` for BlockDesc --- paddle/pybind/pybind.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index cbb7b1cbff..fa10c8e472 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -321,18 +321,19 @@ All parameter, weight, gradient are variables in Paddle. py::return_value_policy::reference) .def("append_block", [](ProgramDesc &self, BlockDesc &parent) { - auto desc = self.mutable_blocks()->Add(); + auto desc = self.add_blocks(); desc->set_idx(self.mutable_blocks()->size() - 1); desc->set_parent_idx(parent.idx()); return desc; }) .def("root_block", [](ProgramDesc &self) { return self.mutable_blocks()[0]; }); + py::class_(m, "BlockDesc", "") .def("idx", [](BlockDesc &self) { return self.idx(); }) .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) - .def("append_op", - [](BlockDesc &self) { return self.mutable_ops()->Add(); }); + .def("append_op", [](BlockDesc &self) { return self.add_ops(); }) + .def("new_var", [](BlockDesc &self) { return self.add_vars(); }); py::class_(m, "VarDesc", "") .def(py::init<>()) @@ -372,7 +373,7 @@ All parameter, weight, gradient are variables in Paddle. }; auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { - auto attr = desc.mutable_attrs()->Add(); + auto attr = desc.add_attrs(); attr->set_name(name); return attr; }; @@ -383,7 +384,7 @@ All parameter, weight, gradient are variables in Paddle. [op_desc_set_var](OpDesc &self, const std::string ¶meter, const std::vector &arguments) { - auto ipt = self.mutable_inputs()->Add(); + auto ipt = self.add_inputs(); op_desc_set_var(ipt, parameter, arguments); }) .def("input_names", @@ -402,7 +403,7 @@ All parameter, weight, gradient are variables in Paddle. [op_desc_set_var](OpDesc &self, const std::string ¶meter, const std::vector &arguments) { - auto opt = self.mutable_outputs()->Add(); + auto opt = self.add_outputs(); op_desc_set_var(opt, parameter, arguments); }) .def("set_attr", From 618884dd69af0f2e7ea7c0527ec2ba8131ec5a07 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 21 Sep 2017 21:39:42 -0700 Subject: [PATCH 05/30] Complete unittest for ProgramDesc --- paddle/pybind/pybind.cc | 21 +++++++++++++++---- .../v2/framework/tests/test_protobuf_descs.py | 9 ++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index cbb7b1cbff..cae3671350 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -319,17 +319,30 @@ All parameter, weight, gradient are variables in Paddle. .def_static("instance", [] { return &GetProgramDesc(); }, py::return_value_policy::reference) + .def_static("__create_program_desc__", + [] { + // Only used for unit-test + auto *prog_desc = new ProgramDesc; + auto *block = prog_desc->mutable_blocks()->Add(); + block->set_idx(0); + block->set_parent_idx(-1); + return prog_desc; + }) .def("append_block", [](ProgramDesc &self, BlockDesc &parent) { - auto desc = self.mutable_blocks()->Add(); + auto desc = self.add_blocks(); desc->set_idx(self.mutable_blocks()->size() - 1); desc->set_parent_idx(parent.idx()); return desc; - }) + }, + py::return_value_policy::reference) .def("root_block", - [](ProgramDesc &self) { return self.mutable_blocks()[0]; }); + [](ProgramDesc &self) { return self.mutable_blocks()->Mutable(0); }, + py::return_value_policy::reference) + .def("__str__", [](ProgramDesc &self) { return self.DebugString(); }); + py::class_(m, "BlockDesc", "") - .def("idx", [](BlockDesc &self) { return self.idx(); }) + .def("id", [](BlockDesc &self) { return self.idx(); }) .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) .def("append_op", [](BlockDesc &self) { return self.mutable_ops()->Add(); }); diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index 945610ff45..8e94843662 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -9,8 +9,17 @@ class TestProgramDesc(unittest.TestCase): del program_desc program_desc = core.ProgramDesc.instance() self.assertIsNotNone(program_desc) + self.assertIsNotNone(program_desc.root_block()) del program_desc + def test_append_block(self): + prog_desc = core.ProgramDesc.__create_program_desc__() + self.assertIsNotNone(prog_desc) + block1 = prog_desc.append_block(prog_desc.root_block()) + block2 = prog_desc.append_block(block1) + self.assertEqual(block1.id(), block2.parent()) + self.assertEqual(prog_desc.root_block().id(), block1.parent()) + if __name__ == '__main__': unittest.main() From b154c0e51ea1853ab935bf6c86966b0092babe6a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 10:07:17 -0700 Subject: [PATCH 06/30] Update --- paddle/pybind/pybind.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index a6b583821f..543dbb739e 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -344,8 +344,12 @@ All parameter, weight, gradient are variables in Paddle. py::class_(m, "BlockDesc", "") .def("id", [](BlockDesc &self) { return self.idx(); }) .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) - .def("append_op", [](BlockDesc &self) { return self.add_ops(); }) - .def("new_var", [](BlockDesc &self) { return self.add_vars(); }); + .def("append_op", + [](BlockDesc &self) { return self.add_ops(); }, + py::return_value_policy::reference) + .def("new_var", + [](BlockDesc &self) { return self.add_vars(); }, + py::return_value_policy::reference); py::class_(m, "VarDesc", "") .def(py::init<>()) From ee547f6ac984b8880394acceb6fbec856f6a2dde Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 10:37:34 -0700 Subject: [PATCH 07/30] Add unittests --- .../v2/framework/tests/test_protobuf_descs.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index 8e94843662..71bdca8765 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -15,10 +15,25 @@ class TestProgramDesc(unittest.TestCase): def test_append_block(self): prog_desc = core.ProgramDesc.__create_program_desc__() self.assertIsNotNone(prog_desc) - block1 = prog_desc.append_block(prog_desc.root_block()) + block_root = prog_desc.root_block() + self.assertEqual(block_root.id(), 0) + block1 = prog_desc.append_block(block_root) block2 = prog_desc.append_block(block1) self.assertEqual(block1.id(), block2.parent()) - self.assertEqual(prog_desc.root_block().id(), block1.parent()) + self.assertEqual(block_root.id(), block1.parent()) + block3 = prog_desc.append_block(block_root) + self.assertEqual(block3.parent(), block_root.id()) + + +class TestVarDesc(unittest.TestCase): + def test_shape(self): + program_desc = core.ProgramDesc.instance() + block = program_desc.root_block() + var = block.new_var() + src_shape = [3, 2, 10, 8] + var.set_shape(src_shape) + res_shape = var.shape() + self.assertEqual(src_shape, res_shape) if __name__ == '__main__': From 17d93f4a04fa03517d84d20fd31829a7e02847b4 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 10:38:04 -0700 Subject: [PATCH 08/30] Add Helper for Vector2Repeated --- paddle/pybind/pybind.cc | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index a6b583821f..74f5904034 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -53,6 +53,25 @@ bool IsCompileGPU() { #endif } +template +inline std::vector RepeatedToVector( + const google::protobuf::RepeatedField &repeated_field) { + std::vector ret; + ret.reserve(repeated_field.size()); + std::copy( + repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); + return ret; +} + +template +inline void VectorToRepeated(const std::vector &vec, + RepeatedField *repeated_field) { + repeated_field->Reserve(vec.size()); + for (auto &elem : vec) { + *repeated_field->Add() = elem; + } +} + PYBIND11_PLUGIN(core) { py::module m("core", "C++ core of PaddlePaddle"); @@ -377,11 +396,7 @@ All parameter, weight, gradient are variables in Paddle. const std::string ¶meter, const std::vector &arguments) { var->set_parameter(parameter); - auto args = var->mutable_arguments(); - args->Reserve(static_cast(arguments.size())); - for (auto &arg : arguments) { - *args->Add() = arg; - } + VectorToRepeated(arguments, var->mutable_arguments()); }; auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { From e29003669ff30272070bb8513fb95c2042c305b9 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 11:23:47 -0700 Subject: [PATCH 09/30] Moving protobuf binding code to protobuf module --- paddle/pybind/CMakeLists.txt | 3 +- paddle/pybind/protobuf.cc | 136 ++++++++++++++++++++++++++++++++++ paddle/pybind/protobuf.h | 54 ++++++++++++++ paddle/pybind/pybind.cc | 140 ++--------------------------------- 4 files changed, 197 insertions(+), 136 deletions(-) create mode 100644 paddle/pybind/protobuf.cc create mode 100644 paddle/pybind/protobuf.h diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt index 4f05406c7f..a1d7483973 100644 --- a/paddle/pybind/CMakeLists.txt +++ b/paddle/pybind/CMakeLists.txt @@ -1,6 +1,7 @@ if(WITH_PYTHON) + cc_library(proto_bind SRCS protobuf.cc) cc_library(paddle_pybind SHARED SRCS pybind.cc - DEPS pybind python backward + DEPS proto_bind pybind python backward ${GLOB_OP_LIB}) endif(WITH_PYTHON) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc new file mode 100644 index 0000000000..91f4c7d7c8 --- /dev/null +++ b/paddle/pybind/protobuf.cc @@ -0,0 +1,136 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/pybind/protobuf.h" + +namespace paddle { +namespace framework { + +void bind_program_desc(py::module &m) { + py::class_(m, "ProgramDesc", "") + .def_static("instance", + [] { return &GetProgramDesc(); }, + py::return_value_policy::reference) + .def_static("__create_program_desc__", + [] { + // Only used for unit-test + auto *prog_desc = new ProgramDesc; + auto *block = prog_desc->mutable_blocks()->Add(); + block->set_idx(0); + block->set_parent_idx(-1); + return prog_desc; + }) + .def("append_block", + [](ProgramDesc &self, BlockDesc &parent) { + auto desc = self.add_blocks(); + desc->set_idx(self.mutable_blocks()->size() - 1); + desc->set_parent_idx(parent.idx()); + return desc; + }, + py::return_value_policy::reference) + .def("root_block", + [](ProgramDesc &self) { return self.mutable_blocks()->Mutable(0); }, + py::return_value_policy::reference) + .def("__str__", [](ProgramDesc &self) { return self.DebugString(); }); +} + +void bind_block_desc(py::module &m) { + py::class_(m, "BlockDesc", "") + .def("id", [](BlockDesc &self) { return self.idx(); }) + .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) + .def("append_op", + [](BlockDesc &self) { return self.add_ops(); }, + py::return_value_policy::reference) + .def("new_var", + [](BlockDesc &self) { return self.add_vars(); }, + py::return_value_policy::reference); +} + +void bind_var_dses(py::module &m) { + py::class_(m, "VarDesc", "") + .def(py::init<>()) + .def("set_name", + [](VarDesc &self, const std::string &name) { self.set_name(name); }) + .def("set_shape", + [](VarDesc &self, const std::vector &dims) { + LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); + for (const int64_t &i : dims) { + lod_tensor_desc->add_dims(i); + } + }) + .def("set_data_type", + [](VarDesc &self, int type_id) { + LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); + lod_tensor_desc->set_data_type(static_cast(type_id)); + }) + .def("shape", [](VarDesc &self) { + const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); + int rank = lod_tensor_desc.dims_size(); + std::vector res(rank); + for (int i = 0; i < rank; ++i) { + res[i] = lod_tensor_desc.dims(i); + } + return res; + }); +} + +void bind_op_desc(py::module &m) { + auto op_desc_set_var = [](OpDesc::Var *var, + const std::string ¶meter, + const std::vector &arguments) { + var->set_parameter(parameter); + VectorToRepeated(arguments, var->mutable_arguments()); + }; + + auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { + auto attr = desc.add_attrs(); + attr->set_name(name); + return attr; + }; + + py::class_(m, "OpDesc", "") + .def("type", [](OpDesc &op) { return op.type(); }) + .def("set_input", + [op_desc_set_var](OpDesc &self, + const std::string ¶meter, + const std::vector &arguments) { + auto ipt = self.add_inputs(); + op_desc_set_var(ipt, parameter, arguments); + }) + .def("input_names", + [](OpDesc &self) { + std::vector ret_val; + ret_val.reserve(static_cast(self.inputs().size())); + std::transform( + self.inputs().begin(), + self.inputs().end(), + std::back_inserter(ret_val), + [](const OpDesc::Var &var) { return var.parameter(); }); + return ret_val; + }) + .def("__str__", [](OpDesc &self) { return self.DebugString(); }) + .def("set_output", + [op_desc_set_var](OpDesc &self, + const std::string ¶meter, + const std::vector &arguments) { + auto opt = self.add_outputs(); + op_desc_set_var(opt, parameter, arguments); + }) + .def("set_attr", + [op_desc_set_attr](OpDesc &self, const std::string &name, int i) { + op_desc_set_attr(self, name)->set_i(i); + }); +} +} // namespace framework +} // namespace paddle diff --git a/paddle/pybind/protobuf.h b/paddle/pybind/protobuf.h new file mode 100644 index 0000000000..ff4813cce7 --- /dev/null +++ b/paddle/pybind/protobuf.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include "paddle/framework/op_registry.h" +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +namespace py = pybind11; + +namespace paddle { +namespace framework { + +template +inline std::vector RepeatedToVector( + const google::protobuf::RepeatedField& repeated_field) { + std::vector ret; + ret.reserve(repeated_field.size()); + std::copy( + repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); + return ret; +} + +template +inline void VectorToRepeated(const std::vector& vec, + RepeatedField* repeated_field) { + repeated_field->Reserve(vec.size()); + for (auto& elem : vec) { + *repeated_field->Add() = elem; + } +} + +void bind_program_desc(py::module& m); +void bind_block_desc(py::module& m); +void bind_var_dses(py::module& m); +void bind_op_desc(py::module& m); +} // namespace framework +} // namespace paddle diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 5ccc8c377f..10c6670e00 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -12,13 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include -#include -#include +#include "paddle/pybind/protobuf.h" #include "paddle/framework/backward.h" #include "paddle/framework/lod_tensor.h" -#include "paddle/framework/op_registry.h" #include "paddle/operators/cond_op.h" #include "paddle/operators/net_op.h" #include "paddle/operators/recurrent_op.h" @@ -27,11 +24,6 @@ limitations under the License. */ #include "paddle/pybind/pybind.h" #include "paddle/pybind/tensor_py.h" #include "paddle/string/to_string.h" -#include "pybind11/numpy.h" -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" - -namespace py = pybind11; namespace paddle { namespace framework { @@ -53,25 +45,6 @@ bool IsCompileGPU() { #endif } -template -inline std::vector RepeatedToVector( - const google::protobuf::RepeatedField &repeated_field) { - std::vector ret; - ret.reserve(repeated_field.size()); - std::copy( - repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); - return ret; -} - -template -inline void VectorToRepeated(const std::vector &vec, - RepeatedField *repeated_field) { - repeated_field->Reserve(vec.size()); - for (auto &elem : vec) { - *repeated_field->Add() = elem; - } -} - PYBIND11_PLUGIN(core) { py::module m("core", "C++ core of PaddlePaddle"); @@ -334,113 +307,10 @@ All parameter, weight, gradient are variables in Paddle. m.def("is_compile_gpu", IsCompileGPU); - py::class_(m, "ProgramDesc", "") - .def_static("instance", - [] { return &GetProgramDesc(); }, - py::return_value_policy::reference) - .def_static("__create_program_desc__", - [] { - // Only used for unit-test - auto *prog_desc = new ProgramDesc; - auto *block = prog_desc->mutable_blocks()->Add(); - block->set_idx(0); - block->set_parent_idx(-1); - return prog_desc; - }) - .def("append_block", - [](ProgramDesc &self, BlockDesc &parent) { - auto desc = self.add_blocks(); - desc->set_idx(self.mutable_blocks()->size() - 1); - desc->set_parent_idx(parent.idx()); - return desc; - }, - py::return_value_policy::reference) - .def("root_block", - [](ProgramDesc &self) { return self.mutable_blocks()->Mutable(0); }, - py::return_value_policy::reference) - .def("__str__", [](ProgramDesc &self) { return self.DebugString(); }); - - py::class_(m, "BlockDesc", "") - .def("id", [](BlockDesc &self) { return self.idx(); }) - .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) - .def("append_op", - [](BlockDesc &self) { return self.add_ops(); }, - py::return_value_policy::reference) - .def("new_var", - [](BlockDesc &self) { return self.add_vars(); }, - py::return_value_policy::reference); - - py::class_(m, "VarDesc", "") - .def(py::init<>()) - .def("set_name", - [](VarDesc &self, const std::string &name) { self.set_name(name); }) - .def("set_shape", - [](VarDesc &self, const std::vector &dims) { - LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); - for (const int64_t &i : dims) { - lod_tensor_desc->add_dims(i); - } - }) - .def("set_data_type", - [](VarDesc &self, int type_id) { - LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); - lod_tensor_desc->set_data_type(static_cast(type_id)); - }) - .def("shape", [](VarDesc &self) { - const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); - int rank = lod_tensor_desc.dims_size(); - std::vector res(rank); - for (int i = 0; i < rank; ++i) { - res[i] = lod_tensor_desc.dims(i); - } - return res; - }); - - auto op_desc_set_var = [](OpDesc::Var *var, - const std::string ¶meter, - const std::vector &arguments) { - var->set_parameter(parameter); - VectorToRepeated(arguments, var->mutable_arguments()); - }; - - auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { - auto attr = desc.add_attrs(); - attr->set_name(name); - return attr; - }; - - py::class_(m, "OpDesc", "") - .def("type", [](OpDesc &op) { return op.type(); }) - .def("set_input", - [op_desc_set_var](OpDesc &self, - const std::string ¶meter, - const std::vector &arguments) { - auto ipt = self.add_inputs(); - op_desc_set_var(ipt, parameter, arguments); - }) - .def("input_names", - [](OpDesc &self) { - std::vector ret_val; - ret_val.reserve(static_cast(self.inputs().size())); - std::transform( - self.inputs().begin(), - self.inputs().end(), - std::back_inserter(ret_val), - [](const OpDesc::Var &var) { return var.parameter(); }); - return ret_val; - }) - .def("__str__", [](OpDesc &self) { return self.DebugString(); }) - .def("set_output", - [op_desc_set_var](OpDesc &self, - const std::string ¶meter, - const std::vector &arguments) { - auto opt = self.add_outputs(); - op_desc_set_var(opt, parameter, arguments); - }) - .def("set_attr", - [op_desc_set_attr](OpDesc &self, const std::string &name, int i) { - op_desc_set_attr(self, name)->set_i(i); - }); + bind_program_desc(m); + bind_block_desc(m); + bind_var_dses(m); + bind_op_desc(m); return m.ptr(); } From 37fd8fa1b6ec75ac447a93bea990338550402baf Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 11:39:36 -0700 Subject: [PATCH 10/30] Fix typo --- paddle/pybind/protobuf.cc | 8 ++++---- paddle/pybind/protobuf.h | 8 ++++---- paddle/pybind/pybind.cc | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 91f4c7d7c8..47b3c43ebf 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -17,7 +17,7 @@ limitations under the License. */ namespace paddle { namespace framework { -void bind_program_desc(py::module &m) { +void BindProgramDesc(py::module &m) { py::class_(m, "ProgramDesc", "") .def_static("instance", [] { return &GetProgramDesc(); }, @@ -45,7 +45,7 @@ void bind_program_desc(py::module &m) { .def("__str__", [](ProgramDesc &self) { return self.DebugString(); }); } -void bind_block_desc(py::module &m) { +void BindBlockDesc(py::module &m) { py::class_(m, "BlockDesc", "") .def("id", [](BlockDesc &self) { return self.idx(); }) .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) @@ -57,7 +57,7 @@ void bind_block_desc(py::module &m) { py::return_value_policy::reference); } -void bind_var_dses(py::module &m) { +void BindVarDsec(py::module &m) { py::class_(m, "VarDesc", "") .def(py::init<>()) .def("set_name", @@ -85,7 +85,7 @@ void bind_var_dses(py::module &m) { }); } -void bind_op_desc(py::module &m) { +void BindOpDesc(py::module &m) { auto op_desc_set_var = [](OpDesc::Var *var, const std::string ¶meter, const std::vector &arguments) { diff --git a/paddle/pybind/protobuf.h b/paddle/pybind/protobuf.h index ff4813cce7..a32acfb038 100644 --- a/paddle/pybind/protobuf.h +++ b/paddle/pybind/protobuf.h @@ -46,9 +46,9 @@ inline void VectorToRepeated(const std::vector& vec, } } -void bind_program_desc(py::module& m); -void bind_block_desc(py::module& m); -void bind_var_dses(py::module& m); -void bind_op_desc(py::module& m); +void BindProgramDesc(py::module& m); +void BindBlockDesc(py::module& m); +void BindVarDsec(py::module& m); +void BindOpDesc(py::module& m); } // namespace framework } // namespace paddle diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 10c6670e00..d9dd7523bf 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -307,10 +307,10 @@ All parameter, weight, gradient are variables in Paddle. m.def("is_compile_gpu", IsCompileGPU); - bind_program_desc(m); - bind_block_desc(m); - bind_var_dses(m); - bind_op_desc(m); + BindProgramDesc(m); + BindBlockDesc(m); + BindVarDsec(m); + BindOpDesc(m); return m.ptr(); } From f5aa8b4d7ef508dcd66984ef36012eeff63a9c85 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 13:22:21 -0700 Subject: [PATCH 11/30] Update namespace of pybind/protobuf.cc and .h --- paddle/pybind/protobuf.cc | 11 +++++++++-- paddle/pybind/protobuf.h | 4 ++-- .../paddle/v2/framework/tests/test_protobuf_descs.py | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 47b3c43ebf..bfbe177e8f 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -15,9 +15,10 @@ limitations under the License. */ #include "paddle/pybind/protobuf.h" namespace paddle { -namespace framework { +namespace pybind { void BindProgramDesc(py::module &m) { + using namespace paddle::framework; // NOLINT py::class_(m, "ProgramDesc", "") .def_static("instance", [] { return &GetProgramDesc(); }, @@ -42,10 +43,14 @@ void BindProgramDesc(py::module &m) { .def("root_block", [](ProgramDesc &self) { return self.mutable_blocks()->Mutable(0); }, py::return_value_policy::reference) + .def("block", + [](ProgramDesc &self, int id) { return self.blocks(id); }, + py::return_value_policy::reference) .def("__str__", [](ProgramDesc &self) { return self.DebugString(); }); } void BindBlockDesc(py::module &m) { + using namespace paddle::framework; // NOLINT py::class_(m, "BlockDesc", "") .def("id", [](BlockDesc &self) { return self.idx(); }) .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) @@ -58,6 +63,7 @@ void BindBlockDesc(py::module &m) { } void BindVarDsec(py::module &m) { + using namespace paddle::framework; // NOLINT py::class_(m, "VarDesc", "") .def(py::init<>()) .def("set_name", @@ -86,6 +92,7 @@ void BindVarDsec(py::module &m) { } void BindOpDesc(py::module &m) { + using namespace paddle::framework; // NOLINT auto op_desc_set_var = [](OpDesc::Var *var, const std::string ¶meter, const std::vector &arguments) { @@ -132,5 +139,5 @@ void BindOpDesc(py::module &m) { op_desc_set_attr(self, name)->set_i(i); }); } -} // namespace framework +} // namespace pybind } // namespace paddle diff --git a/paddle/pybind/protobuf.h b/paddle/pybind/protobuf.h index a32acfb038..de9a008e25 100644 --- a/paddle/pybind/protobuf.h +++ b/paddle/pybind/protobuf.h @@ -25,7 +25,7 @@ limitations under the License. */ namespace py = pybind11; namespace paddle { -namespace framework { +namespace pybind { template inline std::vector RepeatedToVector( @@ -50,5 +50,5 @@ void BindProgramDesc(py::module& m); void BindBlockDesc(py::module& m); void BindVarDsec(py::module& m); void BindOpDesc(py::module& m); -} // namespace framework +} // namespace pybind } // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index 71bdca8765..d0192814ef 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -23,6 +23,7 @@ class TestProgramDesc(unittest.TestCase): self.assertEqual(block_root.id(), block1.parent()) block3 = prog_desc.append_block(block_root) self.assertEqual(block3.parent(), block_root.id()) + self.assertEqual(prog_desc.block(1).id(), 1) class TestVarDesc(unittest.TestCase): From 027fc62cb020801cef53fb0e753d3a31fb7e6f39 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 13:47:29 -0700 Subject: [PATCH 12/30] Use Vec2Repeated Repeated2Vec --- paddle/pybind/protobuf.cc | 31 +++++++++++++++++++++---------- paddle/pybind/protobuf.h | 19 ------------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index bfbe177e8f..b86185bf5b 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -17,6 +17,25 @@ limitations under the License. */ namespace paddle { namespace pybind { +template +inline std::vector RepeatedToVector( + const google::protobuf::RepeatedField &repeated_field) { + std::vector ret; + ret.reserve(repeated_field.size()); + std::copy( + repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); + return ret; +} + +template +inline void VectorToRepeated(const std::vector &vec, + RepeatedField *repeated_field) { + repeated_field->Reserve(vec.size()); + for (auto &elem : vec) { + *repeated_field->Add() = elem; + } +} + void BindProgramDesc(py::module &m) { using namespace paddle::framework; // NOLINT py::class_(m, "ProgramDesc", "") @@ -70,10 +89,7 @@ void BindVarDsec(py::module &m) { [](VarDesc &self, const std::string &name) { self.set_name(name); }) .def("set_shape", [](VarDesc &self, const std::vector &dims) { - LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); - for (const int64_t &i : dims) { - lod_tensor_desc->add_dims(i); - } + VectorToRepeated(dims, self.mutable_lod_tensor()->mutable_dims()); }) .def("set_data_type", [](VarDesc &self, int type_id) { @@ -82,12 +98,7 @@ void BindVarDsec(py::module &m) { }) .def("shape", [](VarDesc &self) { const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); - int rank = lod_tensor_desc.dims_size(); - std::vector res(rank); - for (int i = 0; i < rank; ++i) { - res[i] = lod_tensor_desc.dims(i); - } - return res; + return RepeatedToVector(lod_tensor_desc.dims()); }); } diff --git a/paddle/pybind/protobuf.h b/paddle/pybind/protobuf.h index de9a008e25..2721c128d1 100644 --- a/paddle/pybind/protobuf.h +++ b/paddle/pybind/protobuf.h @@ -27,25 +27,6 @@ namespace py = pybind11; namespace paddle { namespace pybind { -template -inline std::vector RepeatedToVector( - const google::protobuf::RepeatedField& repeated_field) { - std::vector ret; - ret.reserve(repeated_field.size()); - std::copy( - repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); - return ret; -} - -template -inline void VectorToRepeated(const std::vector& vec, - RepeatedField* repeated_field) { - repeated_field->Reserve(vec.size()); - for (auto& elem : vec) { - *repeated_field->Add() = elem; - } -} - void BindProgramDesc(py::module& m); void BindBlockDesc(py::module& m); void BindVarDsec(py::module& m); From bddb40609d604cd68f6418423147ec1ec5ec8de0 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 15:35:48 -0700 Subject: [PATCH 13/30] Buggy code --- paddle/pybind/protobuf.cc | 285 ++++++++++++------ .../v2/framework/tests/test_protobuf_descs.py | 21 +- 2 files changed, 214 insertions(+), 92 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index b86185bf5b..b4ed9c4335 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -13,10 +13,13 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/pybind/protobuf.h" +#include namespace paddle { namespace pybind { +using namespace paddle::framework; // NOLINT + template inline std::vector RepeatedToVector( const google::protobuf::RepeatedField &repeated_field) { @@ -36,45 +39,154 @@ inline void VectorToRepeated(const std::vector &vec, } } +class ProgramDescBind; +class OpDescBind; +class BlockDescBind; + +class OpDescBind { +public: + explicit OpDescBind(BlockDescBind *block) : block_(block) {} + + operator OpDesc *() { return &op_desc_; } + +private: + BlockDescBind *block_; + OpDesc op_desc_; +}; + +class BlockDescBind { +public: + BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) + : prog_(prog), desc_(desc), need_update_(false) {} + + ~BlockDescBind() { + std::cerr << "dtor " << this << "," << desc_ << std::endl; + } + + int32_t id() const { + std::cerr << "desc ptr " << desc_ << std::endl; + return desc_->idx(); + } + + int32_t Parent() const { return desc_->parent_idx(); } + + OpDescBind *AppendOp() { + need_update_ = true; + ops_.emplace_back(this); + return &ops_.back(); + } + + void Sync() { + if (need_update_) { + auto &op_field = *this->desc_->mutable_ops(); + op_field.Clear(); + op_field.Reserve(static_cast(ops_.size())); + for (auto &op_desc : ops_) { + op_field.AddAllocated(op_desc); + } + } + } + +private: + ProgramDescBind *prog_; // not_own + BlockDesc *desc_; // not_own + bool need_update_; + + std::deque ops_; +}; + +using ProgDescMap = + std::unordered_map>; +static ProgDescMap *g_bind_map = nullptr; + +class ProgramDescBind { +public: + static ProgramDescBind &Instance(ProgramDesc *prog) { + if (g_bind_map == nullptr) { + g_bind_map = new ProgDescMap(); + } + auto &map = *g_bind_map; + auto &ptr = map[prog]; + + if (ptr == nullptr) { + ptr.reset(new ProgramDescBind(prog)); + } + return *ptr; + } + + BlockDescBind *AppendBlock(BlockDescBind *parent) { + auto *b = prog_->add_blocks(); + std::cerr << "block ptr " << b << std::endl; + std::cerr << "pass ptr " << parent << std::endl; + b->set_parent_idx(parent->id()); + b->set_idx(prog_->blocks_size() - 1); + blocks_.emplace_back(this, b); + return &blocks_.back(); + } + + BlockDescBind *Root() { return &blocks_.front(); } + + BlockDescBind *Block(size_t idx) { return &blocks_[idx]; } + + std::string DebugString() { return Proto()->DebugString(); } + + size_t Size() const { return blocks_.size(); } + + ProgramDesc *Proto() { + for (auto &block : blocks_) { + block.Sync(); + } + return prog_; + } + +private: + explicit ProgramDescBind(ProgramDesc *prog) : prog_(prog) { + for (auto &block : *prog->mutable_blocks()) { + blocks_.emplace_back(this, &block); + } + } + + // Not owned + ProgramDesc *prog_; + + std::vector blocks_; +}; + void BindProgramDesc(py::module &m) { - using namespace paddle::framework; // NOLINT - py::class_(m, "ProgramDesc", "") + py::class_(m, "ProgramDesc", "") .def_static("instance", - [] { return &GetProgramDesc(); }, + []() -> ProgramDescBind * { + return &ProgramDescBind::Instance(&GetProgramDesc()); + }, py::return_value_policy::reference) .def_static("__create_program_desc__", - [] { + []() -> ProgramDescBind * { // Only used for unit-test auto *prog_desc = new ProgramDesc; auto *block = prog_desc->mutable_blocks()->Add(); block->set_idx(0); block->set_parent_idx(-1); - return prog_desc; - }) + return &ProgramDescBind::Instance(prog_desc); + }, + py::return_value_policy::reference) .def("append_block", - [](ProgramDesc &self, BlockDesc &parent) { - auto desc = self.add_blocks(); - desc->set_idx(self.mutable_blocks()->size() - 1); - desc->set_parent_idx(parent.idx()); - return desc; - }, + &ProgramDescBind::AppendBlock, py::return_value_policy::reference) .def("root_block", - [](ProgramDesc &self) { return self.mutable_blocks()->Mutable(0); }, + &ProgramDescBind::Root, py::return_value_policy::reference) - .def("block", - [](ProgramDesc &self, int id) { return self.blocks(id); }, - py::return_value_policy::reference) - .def("__str__", [](ProgramDesc &self) { return self.DebugString(); }); + .def("block", &ProgramDescBind::Block, py::return_value_policy::reference) + .def("__str__", &ProgramDescBind::DebugString) + .def("num_blocks", &ProgramDescBind::Size); } void BindBlockDesc(py::module &m) { using namespace paddle::framework; // NOLINT - py::class_(m, "BlockDesc", "") - .def("id", [](BlockDesc &self) { return self.idx(); }) - .def("parent", [](BlockDesc &self) { return self.parent_idx(); }) + py::class_(m, "BlockDesc", "") + .def_property_readonly("id", &BlockDescBind::id) + .def_property_readonly("parent", &BlockDescBind::Parent) .def("append_op", - [](BlockDesc &self) { return self.add_ops(); }, + &BlockDescBind::AppendOp, py::return_value_policy::reference) .def("new_var", [](BlockDesc &self) { return self.add_vars(); }, @@ -82,73 +194,76 @@ void BindBlockDesc(py::module &m) { } void BindVarDsec(py::module &m) { - using namespace paddle::framework; // NOLINT - py::class_(m, "VarDesc", "") - .def(py::init<>()) - .def("set_name", - [](VarDesc &self, const std::string &name) { self.set_name(name); }) - .def("set_shape", - [](VarDesc &self, const std::vector &dims) { - VectorToRepeated(dims, self.mutable_lod_tensor()->mutable_dims()); - }) - .def("set_data_type", - [](VarDesc &self, int type_id) { - LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); - lod_tensor_desc->set_data_type(static_cast(type_id)); - }) - .def("shape", [](VarDesc &self) { - const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); - return RepeatedToVector(lod_tensor_desc.dims()); - }); + py::class_(m, "VarDesc", ""); + // using namespace paddle::framework; // NOLINT + // py::class_(m, "VarDesc", "") + // .def(py::init<>()) + // .def("set_name", + // [](VarDesc &self, const std::string &name) { self.set_name(name); + // }) + // .def("set_shape", + // [](VarDesc &self, const std::vector &dims) { + // VectorToRepeated(dims, + // self.mutable_lod_tensor()->mutable_dims()); + // }) + // .def("set_data_type", + // [](VarDesc &self, int type_id) { + // LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); + // lod_tensor_desc->set_data_type(static_cast(type_id)); + // }) + // .def("shape", [](VarDesc &self) { + // const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); + // return RepeatedToVector(lod_tensor_desc.dims()); + // }); } void BindOpDesc(py::module &m) { - using namespace paddle::framework; // NOLINT - auto op_desc_set_var = [](OpDesc::Var *var, - const std::string ¶meter, - const std::vector &arguments) { - var->set_parameter(parameter); - VectorToRepeated(arguments, var->mutable_arguments()); - }; - - auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { - auto attr = desc.add_attrs(); - attr->set_name(name); - return attr; - }; - - py::class_(m, "OpDesc", "") - .def("type", [](OpDesc &op) { return op.type(); }) - .def("set_input", - [op_desc_set_var](OpDesc &self, - const std::string ¶meter, - const std::vector &arguments) { - auto ipt = self.add_inputs(); - op_desc_set_var(ipt, parameter, arguments); - }) - .def("input_names", - [](OpDesc &self) { - std::vector ret_val; - ret_val.reserve(static_cast(self.inputs().size())); - std::transform( - self.inputs().begin(), - self.inputs().end(), - std::back_inserter(ret_val), - [](const OpDesc::Var &var) { return var.parameter(); }); - return ret_val; - }) - .def("__str__", [](OpDesc &self) { return self.DebugString(); }) - .def("set_output", - [op_desc_set_var](OpDesc &self, - const std::string ¶meter, - const std::vector &arguments) { - auto opt = self.add_outputs(); - op_desc_set_var(opt, parameter, arguments); - }) - .def("set_attr", - [op_desc_set_attr](OpDesc &self, const std::string &name, int i) { - op_desc_set_attr(self, name)->set_i(i); - }); + // auto op_desc_set_var = [](OpDesc::Var *var, + // const std::string ¶meter, + // const std::vector &arguments) { + // var->set_parameter(parameter); + // VectorToRepeated(arguments, var->mutable_arguments()); + // }; + // + // auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { + // auto attr = desc.add_attrs(); + // attr->set_name(name); + // return attr; + // }; + py::class_(m, "OpDesc", ""); + + // .def("type", [](OpDesc &op) { return op.type(); }) + // .def("set_input", + // [op_desc_set_var](OpDesc &self, + // const std::string ¶meter, + // const std::vector &arguments) { + // auto ipt = self.add_inputs(); + // op_desc_set_var(ipt, parameter, arguments); + // }) + // .def("input_names", + // [](OpDesc &self) { + // std::vector ret_val; + // ret_val.reserve(static_cast(self.inputs().size())); + // std::transform( + // self.inputs().begin(), + // self.inputs().end(), + // std::back_inserter(ret_val), + // [](const OpDesc::Var &var) { return var.parameter(); }); + // return ret_val; + // }) + // .def("__str__", [](OpDesc &self) { return self.DebugString(); }) + // .def("set_output", + // [op_desc_set_var](OpDesc &self, + // const std::string ¶meter, + // const std::vector &arguments) { + // auto opt = self.add_outputs(); + // op_desc_set_var(opt, parameter, arguments); + // }) + // .def("set_attr", + // [op_desc_set_attr](OpDesc &self, const std::string &name, int i) + // { + // op_desc_set_attr(self, name)->set_i(i); + // }); } } // namespace pybind } // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index d0192814ef..b5ff2d4c36 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -9,21 +9,28 @@ class TestProgramDesc(unittest.TestCase): del program_desc program_desc = core.ProgramDesc.instance() self.assertIsNotNone(program_desc) - self.assertIsNotNone(program_desc.root_block()) + self.assertIsNotNone(program_desc.block(0)) del program_desc def test_append_block(self): prog_desc = core.ProgramDesc.__create_program_desc__() self.assertIsNotNone(prog_desc) - block_root = prog_desc.root_block() - self.assertEqual(block_root.id(), 0) + block_root = prog_desc.block(0) + self.assertIsNotNone(block_root) + print 'here' + self.assertEqual(block_root.id, 0) block1 = prog_desc.append_block(block_root) block2 = prog_desc.append_block(block1) - self.assertEqual(block1.id(), block2.parent()) - self.assertEqual(block_root.id(), block1.parent()) + self.assertIsNotNone(block1) + print 'here' + self.assertEqual(block1.id, block2.parent) + print 'here' + self.assertEqual(block_root.id, block1.parent) + print 'here' block3 = prog_desc.append_block(block_root) - self.assertEqual(block3.parent(), block_root.id()) - self.assertEqual(prog_desc.block(1).id(), 1) + self.assertEqual(block3.parent, block_root.id) + self.assertEqual(prog_desc.block(1).id, 1) + self.assertEqual(4, prog_desc.num_blocks()) class TestVarDesc(unittest.TestCase): From dc643a33523b48ace8e05dcfe0167b21d3687631 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 15:42:14 -0700 Subject: [PATCH 14/30] Hot fix unittest --- paddle/pybind/protobuf.cc | 17 ++++------------- .../v2/framework/tests/test_protobuf_descs.py | 4 ---- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index b4ed9c4335..0fb78bf7a2 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -59,14 +59,7 @@ public: BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) {} - ~BlockDescBind() { - std::cerr << "dtor " << this << "," << desc_ << std::endl; - } - - int32_t id() const { - std::cerr << "desc ptr " << desc_ << std::endl; - return desc_->idx(); - } + int32_t id() const { return desc_->idx(); } int32_t Parent() const { return desc_->parent_idx(); } @@ -114,11 +107,9 @@ public: return *ptr; } - BlockDescBind *AppendBlock(BlockDescBind *parent) { + BlockDescBind *AppendBlock(const BlockDescBind &parent) { auto *b = prog_->add_blocks(); - std::cerr << "block ptr " << b << std::endl; - std::cerr << "pass ptr " << parent << std::endl; - b->set_parent_idx(parent->id()); + b->set_parent_idx(parent.id()); b->set_idx(prog_->blocks_size() - 1); blocks_.emplace_back(this, b); return &blocks_.back(); @@ -141,6 +132,7 @@ public: private: explicit ProgramDescBind(ProgramDesc *prog) : prog_(prog) { + blocks_.reserve(100); for (auto &block : *prog->mutable_blocks()) { blocks_.emplace_back(this, &block); } @@ -181,7 +173,6 @@ void BindProgramDesc(py::module &m) { } void BindBlockDesc(py::module &m) { - using namespace paddle::framework; // NOLINT py::class_(m, "BlockDesc", "") .def_property_readonly("id", &BlockDescBind::id) .def_property_readonly("parent", &BlockDescBind::Parent) diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index b5ff2d4c36..fbe1f7152b 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -17,16 +17,12 @@ class TestProgramDesc(unittest.TestCase): self.assertIsNotNone(prog_desc) block_root = prog_desc.block(0) self.assertIsNotNone(block_root) - print 'here' self.assertEqual(block_root.id, 0) block1 = prog_desc.append_block(block_root) block2 = prog_desc.append_block(block1) self.assertIsNotNone(block1) - print 'here' self.assertEqual(block1.id, block2.parent) - print 'here' self.assertEqual(block_root.id, block1.parent) - print 'here' block3 = prog_desc.append_block(block_root) self.assertEqual(block3.parent, block_root.id) self.assertEqual(prog_desc.block(1).id, 1) From e05e27a7f71ddb6549e406f0fbc339c789373935 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 16:59:15 -0700 Subject: [PATCH 15/30] Fix bug --- paddle/pybind/protobuf.cc | 64 +++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 0fb78bf7a2..5511841c8b 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -42,15 +42,23 @@ inline void VectorToRepeated(const std::vector &vec, class ProgramDescBind; class OpDescBind; class BlockDescBind; +class VarDescBind; -class OpDescBind { +class VarDescBind { public: - explicit OpDescBind(BlockDescBind *block) : block_(block) {} + explicit VarDescBind(const std::string &name) { var_desc_.set_name(name); } + + VarDesc *Proto() { return &var_desc_; } + +private: + VarDesc var_desc_; +}; - operator OpDesc *() { return &op_desc_; } +class OpDescBind { +public: + OpDesc *Proto() { return &op_desc_; } private: - BlockDescBind *block_; OpDesc op_desc_; }; @@ -59,14 +67,28 @@ public: BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) {} + BlockDescBind(const BlockDescBind &o) = delete; + BlockDescBind &operator=(const BlockDescBind &o) = delete; + int32_t id() const { return desc_->idx(); } int32_t Parent() const { return desc_->parent_idx(); } + VarDescBind *NewVar(const std::string &name) { + need_update_ = true; + auto it = vars_.find(name); + PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name); + auto var = new VarDescBind(name); + vars_[name].reset(var); + return var; + } + + BlockDescBind *ParentBlock() const; + OpDescBind *AppendOp() { need_update_ = true; - ops_.emplace_back(this); - return &ops_.back(); + ops_.emplace_back(new OpDescBind()); + return ops_.back().get(); } void Sync() { @@ -75,8 +97,9 @@ public: op_field.Clear(); op_field.Reserve(static_cast(ops_.size())); for (auto &op_desc : ops_) { - op_field.AddAllocated(op_desc); + op_field.AddAllocated(op_desc->Proto()); } + need_update_ = false; } } @@ -85,7 +108,8 @@ private: BlockDesc *desc_; // not_own bool need_update_; - std::deque ops_; + std::deque> ops_; + std::unordered_map> vars_; }; using ProgDescMap = @@ -106,18 +130,20 @@ public: } return *ptr; } + ProgramDescBind(const ProgramDescBind &o) = delete; + ProgramDescBind &operator=(const ProgramDescBind &o) = delete; BlockDescBind *AppendBlock(const BlockDescBind &parent) { auto *b = prog_->add_blocks(); b->set_parent_idx(parent.id()); b->set_idx(prog_->blocks_size() - 1); - blocks_.emplace_back(this, b); - return &blocks_.back(); + blocks_.emplace_back(new BlockDescBind(this, b)); + return blocks_.back().get(); } - BlockDescBind *Root() { return &blocks_.front(); } + BlockDescBind *Root() { return blocks_.front().get(); } - BlockDescBind *Block(size_t idx) { return &blocks_[idx]; } + BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); } std::string DebugString() { return Proto()->DebugString(); } @@ -125,25 +151,31 @@ public: ProgramDesc *Proto() { for (auto &block : blocks_) { - block.Sync(); + block->Sync(); } return prog_; } private: explicit ProgramDescBind(ProgramDesc *prog) : prog_(prog) { - blocks_.reserve(100); for (auto &block : *prog->mutable_blocks()) { - blocks_.emplace_back(this, &block); + blocks_.emplace_back(new BlockDescBind(this, &block)); } } // Not owned ProgramDesc *prog_; - std::vector blocks_; + std::vector> blocks_; }; +BlockDescBind *BlockDescBind::ParentBlock() const { + if (this->desc_->parent_idx() == -1) { + return nullptr; + } + return prog_->Block(static_cast(this->desc_->parent_idx())); +} + void BindProgramDesc(py::module &m) { py::class_(m, "ProgramDesc", "") .def_static("instance", From eeb7c8ad795d6d7159d3659a2d41709653e2e347 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 17:34:47 -0700 Subject: [PATCH 16/30] Compelete VarDescBind --- paddle/pybind/protobuf.cc | 44 ++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 5511841c8b..126c2ce1c7 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -46,12 +46,24 @@ class VarDescBind; class VarDescBind { public: - explicit VarDescBind(const std::string &name) { var_desc_.set_name(name); } + explicit VarDescBind(const std::string &name) { desc_.set_name(name); } - VarDesc *Proto() { return &var_desc_; } + VarDesc *Proto() { return &desc_; } + + void SetShape(const vector &dims) { + VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims()); + } + + void SetDataType(int type_id) { + desc_.mutable_lod_tensor()->set_data_type(const_cast(type_id)); + } + + std::vector Shape() { + return RepeatedToVector(desc_.lod_tensor().dims()); + } private: - VarDesc var_desc_; + VarDesc desc_; }; class OpDescBind { @@ -217,27 +229,11 @@ void BindBlockDesc(py::module &m) { } void BindVarDsec(py::module &m) { - py::class_(m, "VarDesc", ""); - // using namespace paddle::framework; // NOLINT - // py::class_(m, "VarDesc", "") - // .def(py::init<>()) - // .def("set_name", - // [](VarDesc &self, const std::string &name) { self.set_name(name); - // }) - // .def("set_shape", - // [](VarDesc &self, const std::vector &dims) { - // VectorToRepeated(dims, - // self.mutable_lod_tensor()->mutable_dims()); - // }) - // .def("set_data_type", - // [](VarDesc &self, int type_id) { - // LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor(); - // lod_tensor_desc->set_data_type(static_cast(type_id)); - // }) - // .def("shape", [](VarDesc &self) { - // const LoDTensorDesc &lod_tensor_desc = self.lod_tensor(); - // return RepeatedToVector(lod_tensor_desc.dims()); - // }); + py::class_(m, "VarDesc", "") + .def(py::init<>()) + .def("set_shape", VarDescBind::SetShape) + .def("set_data_type", VarDescBind::SetDataType) + .def("shape", VarDescBind::Shape); } void BindOpDesc(py::module &m) { From ddf2448484cb6d183032e8d616ed51176dea9ded Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 17:46:48 -0700 Subject: [PATCH 17/30] Update Input/Output of Op --- paddle/pybind/protobuf.cc | 145 +++++++++++------- .../v2/framework/tests/test_protobuf_descs.py | 19 +++ 2 files changed, 112 insertions(+), 52 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 5511841c8b..67d6252af8 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -14,6 +14,7 @@ limitations under the License. */ #include "paddle/pybind/protobuf.h" #include +#include "paddle/framework/attribute.h" namespace paddle { namespace pybind { @@ -56,10 +57,90 @@ private: class OpDescBind { public: - OpDesc *Proto() { return &op_desc_; } + OpDesc *Proto() { + Sync(); + return &op_desc_; + } + + std::string Type() const { return op_desc_.type(); } + + void SetType(const std::string &type) { op_desc_.set_type(type); } + + const std::vector &Input(const std::string &name) const { + auto it = inputs_.find(name); + PADDLE_ENFORCE( + it != inputs_.end(), "Input %s cannot be found in Op %s", name, Type()); + return it->second; + } + + std::vector InputNames() const { + std::vector retv; + retv.reserve(this->inputs_.size()); + for (auto &ipt : this->inputs_) { + retv.push_back(ipt.first); + } + return retv; + } + + void SetInput(const std::string ¶m_name, + const std::vector &args) { + need_update_ = true; + inputs_[param_name] = args; + } + + const std::vector &Output(const std::string &name) const { + auto it = outputs_.find(name); + PADDLE_ENFORCE(it != outputs_.end(), + "Output %s cannot be found in Op %s", + name, + Type()); + return it->second; + } + + std::vector OutputNames() const { + std::vector retv; + retv.reserve(this->outputs_.size()); + for (auto &ipt : this->outputs_) { + retv.push_back(ipt.first); + } + return retv; + } + + void SetOutput(const std::string ¶m_name, + const std::vector &args) { + need_update_ = true; + this->outputs_[param_name] = args; + } + + std::string DebugString() { return this->Proto()->DebugString(); } + + void Sync() { + if (need_update_) { + this->op_desc_.mutable_inputs()->Clear(); + for (auto &ipt : inputs_) { + auto *input = op_desc_.add_inputs(); + input->set_parameter(ipt.first); + VectorToRepeated(ipt.second, input->mutable_arguments()); + } + + this->op_desc_.mutable_outputs()->Clear(); + for (auto &opt : outputs_) { + auto *output = op_desc_.add_outputs(); + output->set_parameter(opt.first); + VectorToRepeated(opt.second, output->mutable_arguments()); + } + + need_update_ = false; + } + } private: OpDesc op_desc_; + std::unordered_map> inputs_; + std::unordered_map> outputs_; + std::unordered_map attrs_; + + bool need_update_{false}; }; class BlockDescBind { @@ -141,8 +222,6 @@ public: return blocks_.back().get(); } - BlockDescBind *Root() { return blocks_.front().get(); } - BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); } std::string DebugString() { return Proto()->DebugString(); } @@ -196,9 +275,6 @@ void BindProgramDesc(py::module &m) { .def("append_block", &ProgramDescBind::AppendBlock, py::return_value_policy::reference) - .def("root_block", - &ProgramDescBind::Root, - py::return_value_policy::reference) .def("block", &ProgramDescBind::Block, py::return_value_policy::reference) .def("__str__", &ProgramDescBind::DebugString) .def("num_blocks", &ProgramDescBind::Size); @@ -241,52 +317,17 @@ void BindVarDsec(py::module &m) { } void BindOpDesc(py::module &m) { - // auto op_desc_set_var = [](OpDesc::Var *var, - // const std::string ¶meter, - // const std::vector &arguments) { - // var->set_parameter(parameter); - // VectorToRepeated(arguments, var->mutable_arguments()); - // }; - // - // auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) { - // auto attr = desc.add_attrs(); - // attr->set_name(name); - // return attr; - // }; - py::class_(m, "OpDesc", ""); - - // .def("type", [](OpDesc &op) { return op.type(); }) - // .def("set_input", - // [op_desc_set_var](OpDesc &self, - // const std::string ¶meter, - // const std::vector &arguments) { - // auto ipt = self.add_inputs(); - // op_desc_set_var(ipt, parameter, arguments); - // }) - // .def("input_names", - // [](OpDesc &self) { - // std::vector ret_val; - // ret_val.reserve(static_cast(self.inputs().size())); - // std::transform( - // self.inputs().begin(), - // self.inputs().end(), - // std::back_inserter(ret_val), - // [](const OpDesc::Var &var) { return var.parameter(); }); - // return ret_val; - // }) - // .def("__str__", [](OpDesc &self) { return self.DebugString(); }) - // .def("set_output", - // [op_desc_set_var](OpDesc &self, - // const std::string ¶meter, - // const std::vector &arguments) { - // auto opt = self.add_outputs(); - // op_desc_set_var(opt, parameter, arguments); - // }) - // .def("set_attr", - // [op_desc_set_attr](OpDesc &self, const std::string &name, int i) - // { - // op_desc_set_attr(self, name)->set_i(i); - // }); + py::class_(m, "OpDesc", "") + .def("type", &OpDescBind::Type) + .def("set_type", &OpDescBind::SetType) + .def("input", &OpDescBind::Input) + .def("input_names", &OpDescBind::InputNames) + .def("set_input", &OpDescBind::SetInput) + .def("output", &OpDescBind::Output) + .def("output_names", &OpDescBind::OutputNames) + .def("set_output", &OpDescBind::SetOutput) + .def("__str__", &OpDescBind::DebugString) + .def("__repr__", &OpDescBind::DebugString); } } // namespace pybind } // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index fbe1f7152b..950a936307 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -2,6 +2,25 @@ import unittest import paddle.v2.framework.core as core +class TestOpDesc(unittest.TestCase): + def test_op_desc(self): + prog = core.ProgramDesc.__create_program_desc__() + self.assertIsNotNone(prog) + block = prog.block(0) + self.assertIsNotNone(block) + op = block.append_op() + self.assertIsNotNone(op) + op.set_type("test") + self.assertEqual("test", op.type()) + op.set_input("X", ["a", "b", "c"]) + self.assertEqual(["a", "b", "c"], op.input("X")) + self.assertEqual(["X"], op.input_names()) + + op.set_output("Out", ["z"]) + self.assertEqual(['z'], op.output("Out")) + self.assertEqual(["Out"], op.output_names()) + + class TestProgramDesc(unittest.TestCase): def test_instance(self): program_desc = core.ProgramDesc.instance() From 08e99006216395ee61f3dad3047dd44316829a66 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 22 Sep 2017 18:29:25 -0700 Subject: [PATCH 18/30] Fix bugs --- paddle/pybind/protobuf.cc | 18 +++++++++--------- .../v2/framework/tests/test_protobuf_descs.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 126c2ce1c7..de6db60730 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -50,12 +50,12 @@ public: VarDesc *Proto() { return &desc_; } - void SetShape(const vector &dims) { + void SetShape(const std::vector &dims) { VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims()); } void SetDataType(int type_id) { - desc_.mutable_lod_tensor()->set_data_type(const_cast(type_id)); + desc_.mutable_lod_tensor()->set_data_type(static_cast(type_id)); } std::vector Shape() { @@ -86,7 +86,8 @@ public: int32_t Parent() const { return desc_->parent_idx(); } - VarDescBind *NewVar(const std::string &name) { + VarDescBind *NewVar(py::bytes name_bytes) { + std::string name = name_bytes; need_update_ = true; auto it = vars_.find(name); PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name); @@ -224,16 +225,15 @@ void BindBlockDesc(py::module &m) { &BlockDescBind::AppendOp, py::return_value_policy::reference) .def("new_var", - [](BlockDesc &self) { return self.add_vars(); }, + &BlockDescBind::NewVar, py::return_value_policy::reference); } void BindVarDsec(py::module &m) { - py::class_(m, "VarDesc", "") - .def(py::init<>()) - .def("set_shape", VarDescBind::SetShape) - .def("set_data_type", VarDescBind::SetDataType) - .def("shape", VarDescBind::Shape); + py::class_(m, "VarDesc", "") + .def("set_shape", &VarDescBind::SetShape) + .def("set_data_type", &VarDescBind::SetDataType) + .def("shape", &VarDescBind::Shape); } void BindOpDesc(py::module &m) { diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py index fbe1f7152b..f1074f6bb5 100644 --- a/python/paddle/v2/framework/tests/test_protobuf_descs.py +++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py @@ -33,7 +33,7 @@ class TestVarDesc(unittest.TestCase): def test_shape(self): program_desc = core.ProgramDesc.instance() block = program_desc.root_block() - var = block.new_var() + var = block.new_var('my_var') src_shape = [3, 2, 10, 8] var.set_shape(src_shape) res_shape = var.shape() From afeb01f7620a2280c946f359065589d8b0d59062 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 22 Sep 2017 18:30:05 -0700 Subject: [PATCH 19/30] Stash --- paddle/pybind/protobuf.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 67d6252af8..345bb02c86 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -114,6 +114,14 @@ public: std::string DebugString() { return this->Proto()->DebugString(); } + struct SetAttrDescVisitor : public boost::static_visitor { + explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} + OpDesc::Attr *attr_; + void operator()(int v) { attr_->set_i(v); } + void operator()(float v) { attr_->set_f(v); } + void operator()(const std::string &v) { attr_->set_s(v); } + }; + void Sync() { if (need_update_) { this->op_desc_.mutable_inputs()->Clear(); @@ -130,6 +138,13 @@ public: VectorToRepeated(opt.second, output->mutable_arguments()); } + this->op_desc_.mutable_attrs()->Clear(); + for (auto &attr : attrs_) { + auto *attr_desc = op_desc_.add_attrs(); + attr_desc->set_name(attr.first); + attr_desc->set_type(static_cast(attr.second.which() - 1)); + } + need_update_ = false; } } From fdd68fd1a17a78ead01911a8685f02ea871398a5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 25 Sep 2017 10:14:38 -0700 Subject: [PATCH 20/30] Refine Visitor --- paddle/pybind/protobuf.cc | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 23c322ac36..3b2ac68714 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -128,10 +128,24 @@ public: struct SetAttrDescVisitor : public boost::static_visitor { explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} - OpDesc::Attr *attr_; - void operator()(int v) { attr_->set_i(v); } - void operator()(float v) { attr_->set_f(v); } - void operator()(const std::string &v) { attr_->set_s(v); } + mutable OpDesc::Attr *attr_; + void operator()(int v) const { attr_->set_i(v); } + void operator()(float v) const { attr_->set_f(v); } + void operator()(const std::string &v) const { attr_->set_s(v); } + void operator()(bool b) const { attr_->set_b(b); } + + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_ints()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_floats()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_strings()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_bools()); + } }; void Sync() { From 1bfa0e130a17613c0b0e2108e7d220d4630e928a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 25 Sep 2017 13:48:13 -0700 Subject: [PATCH 21/30] Add `Prepend` for BlockDescBind --- paddle/pybind/protobuf.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index de6db60730..f119a12e98 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -104,6 +104,12 @@ public: return ops_.back().get(); } + OpDescBind *PrependOp() { + need_update_ = true; + ops_.emplace_front(new OpDescBind()); + return ops_.front().get(); + } + void Sync() { if (need_update_) { auto &op_field = *this->desc_->mutable_ops(); @@ -224,6 +230,9 @@ void BindBlockDesc(py::module &m) { .def("append_op", &BlockDescBind::AppendOp, py::return_value_policy::reference) + .def("prepend_op", + &BlockDescBind::PrependOp, + py::return_value_policy::reference) .def("new_var", &BlockDescBind::NewVar, py::return_value_policy::reference); From f9f910a33bb33fd94fd645743518a4711a7e0017 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 25 Sep 2017 16:41:18 -0700 Subject: [PATCH 22/30] Complete op --- paddle/pybind/protobuf.cc | 150 +++++++++++++++++- .../v2/framework/tests/test_protobuf_descs.py | 36 ++++- 2 files changed, 181 insertions(+), 5 deletions(-) diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 7af93cca99..673e0ab80b 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -14,8 +14,72 @@ limitations under the License. */ #include "paddle/pybind/protobuf.h" #include +#include #include "paddle/framework/attribute.h" +// Cast boost::variant for PyBind. +// Copy from +// https://github.com/pybind/pybind11/issues/576#issuecomment-269563199 +namespace pybind11 { +namespace detail { + +// Can be replaced by a generic lambda in C++14 +struct variant_caster_visitor : public boost::static_visitor { + return_value_policy policy; + handle parent; + + variant_caster_visitor(return_value_policy policy, handle parent) + : policy(policy), parent(parent) {} + + template + handle operator()(T const &src) const { + return make_caster::cast(src, policy, parent); + } +}; + +template +struct variant_caster; + +template