m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

forward and backward

Browse Source 
test=develop
revert-15207-remove_op_handle_lock_and_fix_var
Xin Pan 6 years ago
parent 2349acea48
commit 7aad6afd49
6 changed files with 175 additions and 42 deletions
Show Stats Download Patch File Download Diff File

65
paddle/fluid/imperative/layer.cc
Unescape View File

@ -57,6 +57,7 @@ class Autograd {
if (var->stop_gradient_) {
return;
}
VLOG(3) << "start autograd";
std::deque<OpBase*> ready;
ready.push_back(var->pre_op_);
@ -122,11 +123,10 @@ framework::LoDTensor& VarBase::Grad() {
}
std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad() {
if (!grad_op_desc_) {
if (!grad_op_desc_ && backward_id_ <= 0) {
LOG(WARNING) << "op with no grad: " << op_desc_->Type();
return {};
}
VLOG(3) << "op grad " << grad_op_desc_->Type();
std::vector<std::unique_ptr<framework::Variable>> tmp_vars;
std::map<std::string, std::vector<framework::Variable*>> grad_outputs;
@ -142,23 +142,30 @@ std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad() {
}
}
framework::RuntimeContext ctx(grad_input_vars_, grad_outputs);
// No need to do compile time infer shape here.
// grad_op_desc_->InferShape(*block_);
grad_op_desc_->InferVarType(block_);
std::unique_ptr<framework::OperatorBase> opbase =
framework::OpRegistry::CreateOp(*grad_op_desc_);
framework::OperatorWithKernel* op_kernel =
dynamic_cast<framework::OperatorWithKernel*>(opbase.get());
PADDLE_ENFORCE_NOT_NULL(op_kernel, "only support op with kernel");
framework::Scope scope;
platform::CPUPlace place;
PreparedOp p = PreparedOp::Prepare(ctx, *op_kernel, place);
p.op.RuntimeInferShape(scope, place, ctx);
p.func(framework::ExecutionContext(p.op, scope, *p.dev_ctx, p.ctx));
if (backward_id_ > 0) {
VLOG(3) << "py_layer_grad";
PyLayer::ApplyGrad(backward_id_, grad_input_vars_["X@GRAD"],
&(grad_outputs["Out@GRAD"]));
} else {
VLOG(3) << "op grad " << grad_op_desc_->Type();
framework::RuntimeContext ctx(grad_input_vars_, grad_outputs);
// No need to do compile time infer shape here.
// grad_op_desc_->InferShape(*block_);
grad_op_desc_->InferVarType(block_);
std::unique_ptr<framework::OperatorBase> opbase =
framework::OpRegistry::CreateOp(*grad_op_desc_);
framework::OperatorWithKernel* op_kernel =
dynamic_cast<framework::OperatorWithKernel*>(opbase.get());
PADDLE_ENFORCE_NOT_NULL(op_kernel, "only support op with kernel");
framework::Scope scope;
platform::CPUPlace place;
PreparedOp p = PreparedOp::Prepare(ctx, *op_kernel, place);
p.op.RuntimeInferShape(scope, place, ctx);
p.func(framework::ExecutionContext(p.op, scope, *p.dev_ctx, p.ctx));
}
for (auto it : grad_output_vars_) {
auto& outputs = grad_outputs[it.first];
@ -175,6 +182,7 @@ std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad() {
void VarBase::RunBackward() {
if (!pre_op_) return;
VLOG(3) << "start backward";
auto grads_t = grads_->GetMutable<framework::LoDTensor>();
float* data = grads_t->mutable_data<float>(platform::CPUPlace());
std::fill(data, data + grads_t->numel(), 1.0);
@ -190,17 +198,30 @@ void PyLayer::RegisterFunc(int func_id, const py::object& py_func) {
}
std::vector<VarBase*> PyLayer::Apply(int func_id,
const std::vector<VarBase>& inputs) {
const std::vector<VarBase*>& inputs) {
std::vector<framework::LoDTensor> tensor_inputs;
std::vector<VarBase*> ret;
for (const VarBase& in : inputs) {
tensor_inputs.push_back(in.var_->Get<framework::LoDTensor>());
for (const VarBase* in : inputs) {
tensor_inputs.push_back(in->var_->Get<framework::LoDTensor>());
}
PADDLE_ENFORCE(py_funcs_.find(func_id) != py_funcs_.end());
CallPythonFunc(py_funcs_[func_id], tensor_inputs, &ret);
return ret;
}
void PyLayer::ApplyGrad(int func_id,
const std::vector<framework::Variable*>& inputs,
std::vector<framework::Variable*>* outputs) {
std::vector<framework::LoDTensor> tensor_inputs;
std::vector<VarBase*> ret;
for (const Variable* in : inputs) {
tensor_inputs.push_back(in->Get<framework::LoDTensor>());
}
PADDLE_ENFORCE(py_funcs_.find(func_id) != py_funcs_.end());
CallPythonFunc(py_funcs_[func_id], tensor_inputs, outputs);
}
} // namespace imperative
} // namespace paddle

35
paddle/fluid/imperative/layer.h
Unescape View File

@ -196,6 +196,35 @@ static void CallPythonFunc(const py::object& callable,
}
}
static void CallPythonFunc(const py::object& callable,
const std::vector<framework::LoDTensor>& ins,
std::vector<framework::Variable*>* outs) {
py::gil_scoped_acquire guard;
py::tuple in_args(ins.size());
for (size_t i = 0; i < ins.size(); ++i) {
in_args[i] = ins[i].IsInitialized() ? py::cast(ins[i]) : py::cast(nullptr);
}
VLOG(3) << "pyfunc in " << py::len(in_args);
// TODO(panyx0718): Who owns the returned LoDTensor.
auto ret = callable(in_args);
auto ret_tuple = py::cast<py::tuple>(ret);
size_t ret_num = py::len(ret_tuple);
VLOG(3) << "pyfunc out " << ret_num;
for (size_t i = 0; i < ret_num; ++i) {
try {
auto* py_out_tensor = py::cast<framework::LoDTensor*>(ret_tuple[i]);
PADDLE_ENFORCE_NOT_NULL(py_out_tensor,
"Output tensor %d should not be nullptr", i);
auto* tensor = (*outs)[i]->GetMutable<framework::LoDTensor>();
tensor->ShareDataWith(*py_out_tensor);
tensor->set_lod(py_out_tensor->lod());
} catch (py::cast_error&) {
PADDLE_THROW("The %d-th output must be LoDTensor", i);
}
}
}
class PyLayer {
public:
virtual ~PyLayer() {}
@ -203,7 +232,11 @@ class PyLayer {
static void RegisterFunc(int func_id, const py::object& py_func);
static std::vector<VarBase*> Apply(int func_id,
const std::vector<VarBase>& inputs);
const std::vector<VarBase*>& inputs);
static void ApplyGrad(int func_id,
const std::vector<framework::Variable*>& inputs,
std::vector<framework::Variable*>* outputs);
};
} // namespace imperative

56
paddle/fluid/imperative/tracer.h
Unescape View File

@ -131,6 +131,7 @@ class Tracer {
if (!stop_gradient) {
framework::OpDesc* grad_op_desc;
// TODO(panyx): Is this leaked?
auto grad_to_var = new std::unordered_map<std::string, std::string>();
CreateGradOp(*op_desc, {}, {block}, &grad_op_desc, grad_to_var);
op->grad_op_desc_ = grad_op_desc;
@ -143,12 +144,14 @@ class Tracer {
if (var_it == grad_to_var->end()) {
auto fwd_var_it = vars.find(grad_invar);
PADDLE_ENFORCE(fwd_var_it != vars.end());
// Forward inputs or outputs.
grad_in_vars.push_back(fwd_var_it->second->var_);
} else {
VarBase* var = vars[var_it->second];
if (!var->grads_->IsInitialized()) {
InitVar(var->var_, var->grads_);
}
// Douts.
grad_in_vars.push_back(var->grads_);
}
}
@ -172,18 +175,51 @@ class Tracer {
op->block_ = block;
}
std::vector<VarBase*> PyTrace(OpBase* op,
const std::vector<VarBase>& inputs) {
std::vector<VarBase*> outputs = PyLayer::Apply(op->forward_id_, inputs);
/*
for (const VarBase& inp : inputs) {
if (inp.pre_op_) {
op->pre_ops_[it.first].push_back(inp->pre_op_);
op->pre_ops_out_idx_[it.first].push_back(inp->pre_op_out_idx_);
std::vector<VarBase*> PyTrace(OpBase* op, const std::vector<VarBase*>& inputs,
bool stop_gradient = false) {
VLOG(3) << "py_trace";
op->input_vars_["X"] = inputs;
op->output_vars_["Out"] = PyLayer::Apply(op->forward_id_, inputs);
for (VarBase* inp : inputs) {
if (inp->pre_op_) {
op->pre_ops_["X"].push_back(inp->pre_op_);
op->pre_ops_out_idx_["X"].push_back(inp->pre_op_out_idx_);
} else {
op->pre_ops_[it.first].push_back(nullptr);
op->pre_ops_["X"].push_back(nullptr);
}
}*/
}
auto& outputs = op->output_vars_["Out"];
for (size_t i = 0; i < outputs.size(); ++i) {
VarBase* out = outputs[i];
out->stop_gradient_ = stop_gradient;
out->pre_op_ = op;
out->pre_op_out_name_ = "Out";
out->pre_op_out_idx_ = i;
}
if (!stop_gradient) {
auto& grad_input_vars = op->grad_input_vars_["X@GRAD"];
auto& grad_output_vars = op->grad_output_vars_["Out@GRAD"];
for (const VarBase* inp : inputs) {
grad_input_vars.push_back(inp->var_);
}
for (VarBase* out : outputs) {
grad_input_vars.push_back(out->var_);
}
for (VarBase* out : outputs) {
grad_input_vars.push_back(out->grads_);
if (!grad_input_vars.back()->IsInitialized()) {
InitVar(out->var_, grad_input_vars.back());
}
}
for (const VarBase* inp : inputs) {
grad_output_vars.push_back(inp->grads_);
if (!grad_output_vars.back()->IsInitialized()) {
InitVar(inp->var_, grad_output_vars.back());
}
}
}
return outputs;
}

9
paddle/fluid/pybind/pybind.cc
Unescape View File

@ -175,6 +175,13 @@ PYBIND11_MODULE(core, m) {
[](imperative::OpBase &self, int forward_id) {
self.forward_id_ = forward_id;
},
py::return_value_policy::reference)
.def_property(
"backward_id",
[](const imperative::OpBase &self) { return self.backward_id_; },
[](imperative::OpBase &self, int backward_id) {
self.backward_id_ = backward_id;
},
py::return_value_policy::reference);
py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
@ -188,7 +195,7 @@ PYBIND11_MODULE(core, m) {
.def(py::init<>())
.def_static(
"apply",
[](int func_id, const std::vector<imperative::VarBase> &inputs)
[](int func_id, const std::vector<imperative::VarBase *> &inputs)
-> std::vector<imperative::VarBase *> {
return imperative::PyLayer::Apply(func_id, inputs);
},

7
python/paddle/fluid/imperative/layers.py
Unescape View File

@ -59,22 +59,23 @@ class PyLayer(core.PyLayer):
raise NotImplementedError
@staticmethod
def backward(inputs):
def backward(douts):
raise NotImplementedError
@classmethod
def __call__(cls, inputs):
tracer = framework._imperative_tracer()
block = framework.default_main_program().current_block()
inputs = map(base.to_variable, inputs)
inputs = [x._ivar for x in inputs]
PyLayer.register_func(1, cls.forward)
PyLayer.register_func(2, cls.backward)
iop = core.OpBase()
iop.forward_id = 1
iop.backward_id = 2
block.ops.append(iop)
ivars = tracer.py_trace(iop, inputs)
ivars = tracer.py_trace(iop, inputs, False)
# ivars = core.PyLayer.apply(cls.forward, inputs)
ret = []
for ivar in ivars:

45
python/paddle/fluid/tests/unittests/test_imperative.py
Unescape View File

@ -49,8 +49,18 @@ class MyPyLayer(fluid.imperative.PyLayer):
return tuple([tensor])
@staticmethod
def backward(douts, outs):
return np.array(douts[0]) * (1 - np.square(np.array(outs[0])))
def backward(inputs):
sys.stderr.write('calling into backward: %s\n' % str(inputs))
inp, out, dout = inputs
inp = np.array(inp)
out = np.array(out)
dout = np.array(dout)
sys.stderr.write('calling into backward: %s, %s, %s\n' %
(inp, out, dout))
ret = np.array(dout) * (1 - np.square(np.array(out)))
tensor = core.LoDTensor()
tensor.set(ret, core.CPUPlace())
return tuple([tensor])
class MLP(fluid.imperative.Layer):
@ -71,20 +81,44 @@ class MLP(fluid.imperative.Layer):
class TestImperative(unittest.TestCase):
"""
def test_layer(self):
with fluid.imperative.guard():
cl = core.Layer()
cl.forward([])
l = fluid.imperative.Layer()
self.assertRaises(NotImplementedError, l.forward, [])
"""
def test_pylayer(self):
np_inp = np.ones([2, 2], np.float32)
with fluid.imperative.guard():
my_py_layer = MyPyLayer()
outs = my_py_layer([np.ones([2, 2], np.float32)])
sys.stderr.write('%s\n' % outs[0]._numpy())
# out.backward()
var_inp = fluid.imperative.base.to_variable(np_inp)
outs = my_py_layer([var_inp])
dy_out = np.sum(outs[0]._numpy())
outs[0]._backward()
dy_grad = var_inp._gradient()
with new_program_scope():
inp = fluid.layers.data(
name="inp", shape=[2, 2], append_batch_size=False)
# TODO(panyx0718): Paddle doesn't diff against data `inp`.
x1 = inp * 1
# TODO(panyx0718): If reduce_sum is skipped, the result is wrong.
x = fluid.layers.reduce_sum(fluid.layers.tanh(x1))
param_grads = fluid.backward.append_backward(
x, parameter_list=[x1.name])[0]
exe = fluid.Executor(fluid.CPUPlace())
static_out, static_grad = exe.run(
feed={inp.name: np_inp},
fetch_list=[x.name, param_grads[1].name])
self.assertTrue(np.allclose(dy_out, static_out))
self.assertTrue(np.allclose(dy_grad, static_grad))
"""
def test_layer_in_out(self):
np_inp = np.array([1.0, 2.0, -1.0], dtype=np.float32)
with fluid.imperative.guard():
@ -138,6 +172,7 @@ class TestImperative(unittest.TestCase):
self.assertTrue(np.allclose(dy_out, static_out))
self.assertTrue(np.allclose(dy_grad, static_grad))
"""
if __name__ == '__main__':

Write Preview
Loading…
Cancel
Save