initial imperative

test=develop

paddle/fluid/imperative/CMakeLists.txt

@@ -1,3 +1,3 @@
-cc_library(layer SRCS layer.cc DEPS proto_desc)
+cc_library(layer SRCS layer.cc DEPS proto_desc operator)
 cc_library(tracer SRCS tracer.cc DEPS proto_desc)
 cc_library(engine SRCS engine.cc)

paddle/fluid/imperative/layer.h

@@ -14,8 +14,10 @@
 
 #pragma once
 
+#include <string>
 #include <vector>
 #include "paddle/fluid/framework/op_desc.h"
+#include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/framework/var_desc.h"
 #include "paddle/fluid/platform/enforce.h"
@@ -27,26 +29,64 @@ class OpBase;
 
 class VarBase {
  public:
-  VarBase() {}
-  virtual ~VarBase() {}
+  VarBase()
+      : pre_op_(nullptr),
+        pre_op_out_idx_(-1),
+        var_desc_(nullptr),
+        var_(nullptr),
+        grads_(nullptr) {}
+
+  virtual ~VarBase() {
+    LOG(ERROR) << "deleting var";
+    LOG(ERROR) << "done deleting var";
+  }
+
+  void ApplyGrad(framework::Scope* scope, framework::Variable* grad);
+
+  void RunBackward(framework::Scope* scope);
+
+  framework::LoDTensor& Grad();
 
   OpBase* pre_op_;
+  int pre_op_out_idx_;
+
   framework::VarDesc* var_desc_;
+  framework::Variable* var_;
+  framework::Variable* grads_;
 };
 
 class OpBase {
  public:
   OpBase()
       : input_vars_(new std::vector<VarBase*>()),
-        output_vars_(new std::vector<VarBase*>()) {}
+        output_vars_(new std::vector<VarBase*>()),
+        pre_ops_(new std::vector<OpBase*>()),
+        pre_ops_out_idx_(new std::vector<int>()),
+        op_desc_(nullptr),
+        grad_op_desc_(nullptr) {}
+
   virtual ~OpBase() {
     delete input_vars_;
     delete output_vars_;
+
+    delete pre_ops_;
+    delete pre_ops_out_idx_;
+
+    if (grad_op_desc_) delete grad_op_desc_;
+    if (grad_to_var_) delete grad_to_var_;
   }
 
+  std::vector<framework::Variable*> ApplyGrad(framework::Scope* scope);
+
   std::vector<VarBase*>* input_vars_;
   std::vector<VarBase*>* output_vars_;
+  std::vector<OpBase*>* pre_ops_;
+  std::vector<int>* pre_ops_out_idx_;
   framework::OpDesc* op_desc_;
+
+  framework::OpDesc* grad_op_desc_;
+  std::unordered_map<std::string, std::string>* grad_to_var_;
+  framework::BlockDesc* block_;
 };
 
 class Layer {
@@ -58,7 +98,7 @@ class Layer {
     return vars;
   }
 
-  virtual void Backward() { LOG(ERROR) << "backward at cpp."; }
+  virtual void Backward() { LOG(ERROR) << "To support customize"; }
 };
 
 }  // namespace imperative

paddle/fluid/imperative/tracer.h

@@ -27,6 +27,20 @@
 namespace paddle {
 namespace imperative {
 
+void CreateGradOp(const framework::OpDesc& op_desc,
+                  const std::unordered_set<std::string>& no_grad_set,
+                  const std::vector<framework::BlockDesc*>& grad_sub_block,
+                  framework::OpDesc** grad_op_desc,
+                  std::unordered_map<std::string, std::string>* grad_to_var) {
+  std::vector<std::unique_ptr<framework::OpDesc>> grad_op_descs =
+      framework::OpInfoMap::Instance()
+          .Get(op_desc.Type())
+          .GradOpMaker()(op_desc, no_grad_set, grad_to_var, grad_sub_block);
+  PADDLE_ENFORCE(grad_op_descs.size() == 1, "Only support 1 grad op now.");
+  // TODO(panyx0718): Leak?
+  *grad_op_desc = grad_op_descs[0].release();
+}
+
 class Tracer {
  public:
   Tracer() {}
@@ -44,6 +58,7 @@ class Tracer {
     for (VarBase* input : inputs) {
       const std::string vname = input->var_desc_->Name();
       framework::Variable* var = scope_->Var(vname);
+      input->var_ = var;
       if (!var->IsInitialized()) {
         framework::VarDesc* var_desc = block_->FindVar(vname);
         if (var_desc->GetType() == framework::proto::VarType::LOD_TENSOR) {
@@ -52,11 +67,17 @@ class Tracer {
           LOG(ERROR) << "tracer doesn't support yet";
         }
       }
+      if (input->pre_op_) {
+        op->pre_ops_->push_back(input->pre_op_);
+        op->pre_ops_out_idx_->push_back(input->pre_op_out_idx_);
+      } else {
+        op->pre_ops_->push_back(nullptr);
+      }
     }
 
     *op->output_vars_ = outputs;
-    for (auto output : outputs) {
-      const std::string vname = output->var_desc_->Name();
+    for (size_t i = 0; i < outputs.size(); ++i) {
+      const std::string vname = outputs[i]->var_desc_->Name();
       framework::Variable* var = scope_->Var(vname);
       if (!var->IsInitialized()) {
         framework::VarDesc* var_desc = block_->FindVar(vname);
@@ -66,9 +87,18 @@ class Tracer {
           LOG(ERROR) << "tracer doesn't support yet";
         }
       }
-      output->pre_op_ = op;
+      outputs[i]->var_ = var;
+      outputs[i]->pre_op_ = op;
+      outputs[i]->pre_op_out_idx_ = i;
     }
     op_base->Run(*scope_, platform::CPUPlace());
+
+    framework::OpDesc* grad_op_desc;
+    auto grad_to_var = new std::unordered_map<std::string, std::string>();
+    CreateGradOp(*op_desc, {}, {block_}, &grad_op_desc, grad_to_var);
+    op->grad_op_desc_ = grad_op_desc;
+    op->grad_to_var_ = grad_to_var;
+    op->block_ = block_;
   }
 
   void SetScope(framework::Scope* scope) { scope_ = scope; }

paddle/fluid/pybind/CMakeLists.txt

@@ -1,5 +1,5 @@
 
-set(PYBIND_DEPS pybind python proto_desc memory executor async_executor prune feed_fetch_method pass_builder parallel_executor profiler)
+set(PYBIND_DEPS pybind python proto_desc memory executor async_executor prune feed_fetch_method pass_builder parallel_executor profiler layer)
 set(PYBIND_SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc async_executor_py.cc imperative.cc)
 
 if(WITH_PYTHON)

paddle/fluid/pybind/imperative.h

@@ -42,6 +42,11 @@ class PyOpBase : public imperative::OpBase {
   using imperative::OpBase::OpBase;  // Inherit constructors
 };
 
+class PyVarBase : public imperative::VarBase {
+ public:
+  using imperative::VarBase::VarBase;  // Inherit constructors
+};
+
 void BindTracer(pybind11::module* m);
 
 }  // namespace pybind

paddle/fluid/pybind/pybind.cc

@@ -34,6 +34,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/reader.h"
 #include "paddle/fluid/framework/selected_rows.h"
 #include "paddle/fluid/framework/version.h"
+#include "paddle/fluid/imperative/layer.h"
 #include "paddle/fluid/memory/allocation/allocator_strategy.h"
 #include "paddle/fluid/operators/activation_op.h"
 #include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
@@ -101,8 +102,13 @@ PYBIND11_MODULE(core, m) {
 
   BindException(&m);
 
-  py::class_<imperative::VarBase>(m, "VarBase",
-                                  R"DOC()DOC")
+  py::class_<imperative::VarBase, PyVarBase>(m, "VarBase", R"DOC()DOC")
+      .def(py::init<>())
+      .def("_run_backward",
+           [](imperative::VarBase &self, framework::Scope *scope) {
+             self.RunBackward(scope);
+           })
+      .def("_grad", &imperative::VarBase::Grad)
       .def_property(
           "desc",
           [](const imperative::VarBase &self) { return self.var_desc_; },
@@ -111,13 +117,14 @@ PYBIND11_MODULE(core, m) {
           },
           py::return_value_policy::reference);
 
-  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase",
-                                           R"DOC()DOC")
+  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
       .def(py::init<>())
       .def_property(
           "desc", [](const imperative::OpBase &self) { return self.op_desc_; },
           [](imperative::OpBase &self, framework::OpDesc *op_desc) {
-            self.op_desc_ = op_desc;
+            if (op_desc) {
+              self.op_desc_ = op_desc;
+            }
           },
           py::return_value_policy::reference);
 

python/paddle/fluid/framework.py

@@ -276,6 +276,7 @@ class Variable(core.VarBase):
                  stop_gradient=False,
                  is_data=False,
                  **kwargs):
+        core.VarBase.__init__(self)
         self.block = block
         self.error_clip = error_clip
 
@@ -361,6 +362,12 @@ class Variable(core.VarBase):
         tensor = core.get_variable_tensor(scope, self.desc.name())
         return np.array(tensor)
 
+    def backward(self, scope):
+        self._run_backward(scope)
+
+    def grad(self):
+        return np.array(self._grad())
+
     def __str__(self):
         return self.to_string(True)
 
@@ -983,6 +990,7 @@ class Block(object):
         self.desc = program.desc.block(idx)
         self.vars = collections.OrderedDict()  # var_name --> var
         self.ops = list()  # operator list
+        self._op_descs = list()
         self.program = program
         self.removed_vars = collections.OrderedDict()
 
@@ -1238,13 +1246,12 @@ class Block(object):
         if _in_imperative_mode():
             op_desc = core.OpDesc()
             op = Operator(block=self, desc=op_desc, *args, **kwargs)
-            sys.stderr.write('%s %s!!!\n' % (type(op.inputs), type(op.outputs)))
             _imperative_tracer().trace(op, op.inputs, op.outputs)
-            return
-
-        op_desc = self.desc.append_op()
-        op = Operator(block=self, desc=op_desc, *args, **kwargs)
+        else:
+            op_desc = self.desc.append_op()
+            op = Operator(block=self, desc=op_desc, *args, **kwargs)
         self.ops.append(op)
+        self._op_descs.append(op_desc)
         return op
 
     def _insert_op(self, index, *args, **kwargs):

python/paddle/fluid/tests/unittests/test_imperative.py

@@ -26,6 +26,7 @@ class MyLayer(fluid.imperative.PyLayer):
 
     def forward(self, inputs):
         x = fluid.layers.relu(inputs[0])
+        self._x_for_debug = x
         return [fluid.layers.elementwise_mul(x, x)]
 
 
@@ -43,6 +44,8 @@ class TestImperative(unittest.TestCase):
             x = l(np.array([1.0, 2.0, -1.0], dtype=np.float32))[0]
             self.assertIsNotNone(x)
             sys.stderr.write("%s output: %s\n" % (x, x.numpy(scope=l._scope)))
+            x.backward(l._scope)
+            sys.stderr.write("grad %s\n" % l._x_for_debug.grad())
 
 
 if __name__ == '__main__':