Rewrite StaticRNN with Executor (#5224)
* Init commit * Make executor use ProgramDescBind * Change Attribute from BlockDesc to BlockDescBind * Since we will get the program desc in RNN, just BlockDesc is not enough. * Add DeviceContext to Executor API * Rewrite RNN * Pass Python * AddBiasOp does not care num_flatten_dims * Stash * Fix MacOS Compile * Pass RNN forward * add python test * refactor test * Make compile pass * add gradopmaker * First draft done * Polish code * add grad op maker and grad infershape * Polish code * Fix backward.cc bug * Fix infershape * Rename function * add backward test * simplify recurrent test * Update * Pass unittest * Add comments & refine test * Add comments * refactor test * Complete Unittest * fix StepScopes enforce * Remove unused unittest * no type error * Update * Make RNN Pass unittestfix-typo
Yu Yang
7 years ago
committed by
GitHub
parent
8cdb42c2b3
commit
0a32e74d13
File diff suppressed because it is too large
Load Diff
@ -1,170 +0,0 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/framework/operator.h"
|
||||
#include "paddle/operators/net_op.h"
|
||||
#include "paddle/operators/rnn/recurrent_op_utils.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
// The sequence format in RecurrentOp is Tensor<seq_len, batch_size, dim> now.
|
||||
// TODO(Superjom)
|
||||
// 1. No-padding computing for sequences with indifinite length in one batch.
|
||||
// 2. Hierarchical RNN for sequence with sub-sequence.
|
||||
// 3. Internal Memory.
|
||||
// 4. More Complex RNN architecture, such as Gated Feedback RNN.
|
||||
// Refer to: https://arxiv.org/pdf/1502.02367.pdf
|
||||
|
||||
class RecurrentAlgorithm {
|
||||
public:
|
||||
void Run(const framework::Scope& scope,
|
||||
const platform::DeviceContext& dev_ctx) const;
|
||||
|
||||
void Init(rnn::Argument* arg,
|
||||
std::unique_ptr<framework::OperatorBase>* stepnet) {
|
||||
PADDLE_ENFORCE_NOT_NULL(stepnet, "stepnet should be set before.");
|
||||
arg_ = arg;
|
||||
stepnet_ = stepnet;
|
||||
}
|
||||
|
||||
protected:
|
||||
/*
|
||||
* The step scopes will be stored in the father scope as a variable.
|
||||
*
|
||||
* NOTE the scopes are reused in both the forward and backward, so just
|
||||
* create once and expand its size if more steps need.
|
||||
*/
|
||||
void CreateScopes(const framework::Scope& scope, size_t seq_len) const;
|
||||
|
||||
const std::vector<framework::Scope*>& GetStepScopes(
|
||||
const framework::Scope& scope) const {
|
||||
return *scope.FindVar(arg_->step_scopes)
|
||||
->GetMutable<std::vector<framework::Scope*>>();
|
||||
}
|
||||
|
||||
void InitMemories(framework::Scope* step_scopes) const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<framework::OperatorBase>* stepnet_;
|
||||
rnn::Argument* arg_;
|
||||
};
|
||||
|
||||
class RecurrentGradientAlgorithm {
|
||||
/**
|
||||
* RNN's backward alogorithm.
|
||||
*
|
||||
* To accelerate the development of RecurrentGradientOp, we decouple RNN's
|
||||
* algorithm and `OperatorBase`'s implementation, the former contains the core
|
||||
* implementation of a RNN, and will keep stable even if the framework changes
|
||||
* a
|
||||
* lot, and the latter is a wrapper acts like an dapter for it to make RNN an
|
||||
* operator.
|
||||
*/
|
||||
public:
|
||||
void Init(rnn::Argument* arg,
|
||||
std::unique_ptr<framework::OperatorBase>* stepnet) {
|
||||
PADDLE_ENFORCE_NOT_NULL(stepnet, "stepnet should be set before.");
|
||||
arg_ = std::move(arg);
|
||||
stepnet_ = stepnet;
|
||||
}
|
||||
|
||||
void Run(const framework::Scope& scope,
|
||||
const platform::DeviceContext& dev_ctx) const;
|
||||
|
||||
void LinkBootMemoryGradients(framework::Scope* step_scopes) const;
|
||||
|
||||
protected:
|
||||
inline const std::vector<framework::Scope*>& GetStepScopes(
|
||||
const framework::Scope& scope) const {
|
||||
return *scope.FindVar(arg_->step_scopes)
|
||||
->GetMutable<std::vector<framework::Scope*>>();
|
||||
}
|
||||
|
||||
private:
|
||||
rnn::Argument* arg_;
|
||||
std::unique_ptr<framework::OperatorBase>* stepnet_;
|
||||
};
|
||||
|
||||
class RecurrentOp : public framework::OperatorBase {
|
||||
public:
|
||||
RecurrentOp(const std::string& type, const framework::VariableNameMap& inputs,
|
||||
const framework::VariableNameMap& outputs,
|
||||
const framework::AttributeMap& attrs);
|
||||
|
||||
RecurrentOp(const RecurrentOp& o)
|
||||
: framework::OperatorBase(
|
||||
static_cast<const framework::OperatorBase&>(o)) {
|
||||
// TODO(yuyang18): Implement copy ctor well.
|
||||
PADDLE_THROW("Not implemented");
|
||||
}
|
||||
|
||||
void Run(const framework::Scope& scope,
|
||||
const platform::DeviceContext& dev_ctx) const override {
|
||||
alg_.Run(scope, dev_ctx);
|
||||
}
|
||||
|
||||
void set_stepnet(std::unique_ptr<OperatorBase> net) {
|
||||
stepnet_ = std::move(net);
|
||||
}
|
||||
|
||||
const OperatorBase& stepnet() const { return *stepnet_; }
|
||||
|
||||
static const rnn::ArgumentName kArgName;
|
||||
|
||||
private:
|
||||
RecurrentAlgorithm alg_;
|
||||
rnn::Argument arg_;
|
||||
std::unique_ptr<OperatorBase> stepnet_;
|
||||
};
|
||||
|
||||
class RecurrentGradientOp : public framework::OperatorBase {
|
||||
public:
|
||||
RecurrentGradientOp(const std::string& type,
|
||||
const framework::VariableNameMap& inputs,
|
||||
const framework::VariableNameMap& outputs,
|
||||
const framework::AttributeMap& attrs);
|
||||
|
||||
RecurrentGradientOp(const RecurrentGradientOp& o)
|
||||
: framework::OperatorBase(
|
||||
static_cast<const framework::OperatorBase&>(o)) {
|
||||
// TODO(yuyang18): Implement Copy ctor.
|
||||
PADDLE_THROW("Not Implemented");
|
||||
}
|
||||
|
||||
void Run(const framework::Scope& scope,
|
||||
const platform::DeviceContext& dev_ctx) const override {
|
||||
alg_.Run(scope, dev_ctx);
|
||||
}
|
||||
|
||||
static const rnn::ArgumentName kArgName;
|
||||
|
||||
/*
|
||||
* set a stepnet that is created according to a RecurrentOp's stepnet.
|
||||
*/
|
||||
void set_stepnet(std::unique_ptr<OperatorBase> net) {
|
||||
stepnet_ = std::move(net);
|
||||
}
|
||||
const OperatorBase& stepnet() const { return *stepnet_; }
|
||||
|
||||
private:
|
||||
RecurrentGradientAlgorithm alg_;
|
||||
std::unique_ptr<OperatorBase> stepnet_;
|
||||
rnn::Argument arg_;
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,38 +0,0 @@
|
||||
import unittest
|
||||
from paddle.v2.framework.layers import *
|
||||
from paddle.v2.framework.framework import g_program
|
||||
|
||||
|
||||
class TestRNN(unittest.TestCase):
|
||||
def test_rnn(self):
|
||||
img = data(
|
||||
shape=[
|
||||
80, # sequence length
|
||||
22, # image height
|
||||
22
|
||||
], # image width
|
||||
data_type='float32',
|
||||
name='image')
|
||||
hidden = fc(input=img, size=100, act='sigmoid', num_flatten_dims=2)
|
||||
self.assertEqual((-1, 80, 100), hidden.shape)
|
||||
hidden = fc(input=hidden, size=100, act='sigmoid', num_flatten_dims=2)
|
||||
self.assertEqual((-1, 80, 100), hidden.shape)
|
||||
|
||||
rnn = StaticRNN()
|
||||
with rnn.step():
|
||||
hidden = rnn.step_input(hidden)
|
||||
self.assertEqual((-1, 100), hidden.shape)
|
||||
memory = rnn.memory(shape=(-1, 32), dtype='float32', init_value=0.0)
|
||||
|
||||
rnn_out = fc(input=[hidden, memory], size=32, act='sigmoid')
|
||||
self.assertEqual((-1, 32), rnn_out.shape)
|
||||
rnn.update_memory(memory, rnn_out)
|
||||
rnn.output(rnn_out)
|
||||
|
||||
out = rnn()
|
||||
self.assertEqual((-1, 80, 32), out.shape)
|
||||
print g_program
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue