From 39eb871ddfe4118293f1211b8e3427505b1cf333 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Mon, 21 May 2018 02:38:37 +0000 Subject: [PATCH 001/154] Add an interface to set the number of threads for math function, and set the default value to 1 for inference. --- cmake/external/openblas.cmake | 2 ++ paddle/fluid/inference/io.cc | 4 ++++ paddle/fluid/operators/math/blas.h | 15 +++++++++++++++ paddle/fluid/operators/math/math_function.h | 2 ++ paddle/math/MathFunctions.h | 9 ++++----- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 8af2765f58..4a49a92f2b 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -29,6 +29,8 @@ IF(NOT ${CBLAS_FOUND}) "${CBLAS_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE FILEPATH "openblas library." FORCE) + ADD_DEFINITIONS(-DPADDLE_USE_OPENBLAS) + SET(OPENBLAS_CC "${CMAKE_C_COMPILER} -Wno-unused-but-set-variable -Wno-unused-variable") SET(OPENBLAS_COMMIT "v0.2.20") diff --git a/paddle/fluid/inference/io.cc b/paddle/fluid/inference/io.cc index 65db7c7b50..6b03ac7119 100644 --- a/paddle/fluid/inference/io.cc +++ b/paddle/fluid/inference/io.cc @@ -20,16 +20,20 @@ limitations under the License. */ #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/feed_fetch_type.h" #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/pybind/pybind.h" DEFINE_string(devices, "", "The devices to be used which is joined by comma."); DEFINE_bool(init_p2p, false, "Whether to init p2p."); +DEFINE_int32(math_num_threads, 1, + "Number of threads used to run math functions."); namespace paddle { namespace inference { void Init(const std::vector argv) { framework::InitGflags(argv); + operators::math::SetNumThreads(FLAGS_math_num_threads); // init devices std::vector devices; std::string token; diff --git a/paddle/fluid/operators/math/blas.h b/paddle/fluid/operators/math/blas.h index dabde43850..b393139acd 100644 --- a/paddle/fluid/operators/math/blas.h +++ b/paddle/fluid/operators/math/blas.h @@ -20,13 +20,16 @@ #ifdef PADDLE_WITH_MKLML #include #include +#include #include #endif #ifdef PADDLE_USE_OPENBLAS #include +#ifdef LAPACK_FOUND #include #endif +#endif #ifndef LAPACK_FOUND extern "C" { @@ -46,6 +49,18 @@ namespace paddle { namespace operators { namespace math { +static void SetNumThreads(int num_threads) { +#ifdef PADDLE_USE_OPENBLAS + int real_num_threads = num_threads > 1 ? num_threads : 1; + openblas_set_num_threads(real_num_threads); +#elif defined(PADDLE_WITH_MKLML) + int real_num_threads = num_threads > 1 ? num_threads : 1; + mkl_set_num_threads(real_num_threads); +#else + PADDLE_ENFORCE(false, "To be implemented."); +#endif +} + /** * Matrix Descriptor of a memory buffer. * diff --git a/paddle/fluid/operators/math/math_function.h b/paddle/fluid/operators/math/math_function.h index d4b0e17ed4..8b296b6a07 100644 --- a/paddle/fluid/operators/math/math_function.h +++ b/paddle/fluid/operators/math/math_function.h @@ -21,8 +21,10 @@ limitations under the License. */ #ifdef PADDLE_USE_OPENBLAS #include +#ifdef LAPACK_FOUND #include #endif +#endif #ifndef LAPACK_FOUND extern "C" { diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index f3d8b1a39e..854e4baa39 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -12,8 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef MATHFUNCTIONS_H_ -#define MATHFUNCTIONS_H_ +#pragma once #ifdef PADDLE_WITH_MKLML #include @@ -21,7 +20,7 @@ limitations under the License. */ #include #endif -#if defined(PADDLE_USE_VECLIB) +#ifdef PADDLE_USE_VECLIB extern "C" { #include #include @@ -30,8 +29,10 @@ extern "C" { #ifdef PADDLE_USE_OPENBLAS #include +#ifdef LAPACK_FOUND #include #endif +#endif #ifndef LAPACK_FOUND extern "C" { @@ -126,5 +127,3 @@ template void vTanh(const int n, const T* a, T* r); } // namespace paddle - -#endif // MATHFUNCTIONS_H_ From 9cf1f351d25a2ab6a307f2c629f8e36fa33f4702 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 7 Jun 2018 10:51:54 +0800 Subject: [PATCH 002/154] refine nlp test --- paddle/fluid/inference/tests/book/test_inference_nlp.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/inference/tests/book/test_inference_nlp.cc b/paddle/fluid/inference/tests/book/test_inference_nlp.cc index a0e83a1705..def6231815 100644 --- a/paddle/fluid/inference/tests/book/test_inference_nlp.cc +++ b/paddle/fluid/inference/tests/book/test_inference_nlp.cc @@ -104,9 +104,9 @@ void ThreadRunInfer( const int tid, paddle::framework::Scope* scope, const std::vector>& jobs) { // maybe framework:ProgramDesc is not thread-safe + paddle::platform::CPUPlace place; + paddle::framework::Executor executor(place); auto& sub_scope = scope->NewScope(); - auto place = paddle::platform::CPUPlace(); - auto executor = paddle::framework::Executor(place); auto inference_program = paddle::inference::Load(&executor, scope, FLAGS_model_path); @@ -183,8 +183,8 @@ TEST(inference, nlp) { stop_ms = GetCurrentMs(); } else { // 1. Define place, executor, scope - auto place = paddle::platform::CPUPlace(); - auto executor = paddle::framework::Executor(place); + paddle::platform::CPUPlace place; + paddle::framework::Executor executor(place); // 2. Initialize the inference_program and load parameters std::unique_ptr inference_program; From f326b0117e456e3a0cc6b38bcb819b3b56bef959 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 7 Jun 2018 10:56:35 +0800 Subject: [PATCH 003/154] refine scope lock --- paddle/fluid/framework/scope.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/fluid/framework/scope.cc b/paddle/fluid/framework/scope.cc index bb2d866c82..fd23fdeab7 100644 --- a/paddle/fluid/framework/scope.cc +++ b/paddle/fluid/framework/scope.cc @@ -78,6 +78,7 @@ Variable* Scope::FindVarInternal(const std::string& name) const { } const Scope* Scope::FindScope(const Variable* var) const { + std::unique_lock lock(mutex_); for (auto& kv : vars_) { if (kv.second.get() == var) { return this; @@ -127,6 +128,7 @@ void Scope::EraseVars(const std::vector& var_names) { void Scope::Rename(const std::string& origin_name, const std::string& new_name) const { + std::unique_lock lock(mutex_); auto origin_it = vars_.find(origin_name); PADDLE_ENFORCE(origin_it != vars_.end(), "Cannot find original variable with name %s", origin_name); From b8d315fb6941908a1ade7a4b19c8276a07e3e874 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 7 Jun 2018 20:35:24 +0800 Subject: [PATCH 004/154] make scope thread safe --- paddle/fluid/framework/scope.cc | 77 ++++++++++++++++++++------------- paddle/fluid/framework/scope.h | 12 ++++- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/paddle/fluid/framework/scope.cc b/paddle/fluid/framework/scope.cc index fd23fdeab7..50f374e370 100644 --- a/paddle/fluid/framework/scope.cc +++ b/paddle/fluid/framework/scope.cc @@ -43,49 +43,29 @@ Scope& Scope::NewScope() const { } Variable* Scope::Var(const std::string& name) { - // acquire the lock when new var under this scope std::unique_lock lock(mutex_); - auto* v = FindVarLocally(name); - if (v != nullptr) return v; - - v = new Variable(); - vars_[name].reset(v); - VLOG(3) << "Create variable " << name; - v->name_ = &(vars_.find(name)->first); - return v; + return VarInternal(name); } Variable* Scope::Var(std::string* name) { - auto var_name = string::Sprintf("%p.%d", this, vars_.size()); + std::unique_lock lock(mutex_); + auto new_name = string::Sprintf("%p.%d", this, vars_.size()); if (name != nullptr) { - *name = var_name; + *name = new_name; } - return Var(var_name); + return VarInternal(new_name); } Variable* Scope::FindVar(const std::string& name) const { - // acquire the lock when find var std::unique_lock lock(mutex_); return FindVarInternal(name); } -Variable* Scope::FindVarInternal(const std::string& name) const { - auto var = FindVarLocally(name); - if (var != nullptr) { - return var; - } - return (parent_ == nullptr) ? nullptr : parent_->FindVarInternal(name); -} - const Scope* Scope::FindScope(const Variable* var) const { std::unique_lock lock(mutex_); - for (auto& kv : vars_) { - if (kv.second.get() == var) { - return this; - } - } - return (parent_ == nullptr) ? nullptr : parent_->FindScope(var); + return FindScopeInternal(var); } + void Scope::DropKids() { std::unique_lock lock(mutex_); for (Scope* s : kids_) delete s; @@ -93,6 +73,7 @@ void Scope::DropKids() { } std::vector Scope::LocalVarNames() const { + std::unique_lock lock(mutex_); std::vector known_vars; known_vars.reserve(this->vars_.size()); for (auto& p : vars_) { @@ -129,6 +110,38 @@ void Scope::EraseVars(const std::vector& var_names) { void Scope::Rename(const std::string& origin_name, const std::string& new_name) const { std::unique_lock lock(mutex_); + RenameInternal(origin_name, new_name); +} + +std::string Scope::Rename(const std::string& origin_name) const { + std::unique_lock lock(mutex_); + auto new_name = string::Sprintf("%p.%d", this, vars_.size()); + RenameInternal(origin_name, new_name); + return new_name; +} + +Variable* Scope::VarInternal(const std::string& name) { + auto* v = FindVarLocally(name); + if (v != nullptr) return v; + + v = new Variable(); + vars_[name].reset(v); + VLOG(3) << "Create variable " << name; + v->name_ = &(vars_.find(name)->first); + return v; +} + +const Scope* Scope::FindScopeInternal(const Variable* var) const { + for (auto& kv : vars_) { + if (kv.second.get() == var) { + return this; + } + } + return (parent_ == nullptr) ? nullptr : parent_->FindScope(var); +} + +void Scope::RenameInternal(const std::string& origin_name, + const std::string& new_name) const { auto origin_it = vars_.find(origin_name); PADDLE_ENFORCE(origin_it != vars_.end(), "Cannot find original variable with name %s", origin_name); @@ -139,10 +152,12 @@ void Scope::Rename(const std::string& origin_name, vars_.erase(origin_it); } -std::string Scope::Rename(const std::string& origin_name) const { - auto var_name = string::Sprintf("%p.%d", this, vars_.size()); - Rename(origin_name, var_name); - return var_name; +Variable* Scope::FindVarInternal(const std::string& name) const { + auto var = FindVarLocally(name); + if (var != nullptr) { + return var; + } + return (parent_ == nullptr) ? nullptr : parent_->FindVar(name); } Variable* Scope::FindVarLocally(const std::string& name) const { diff --git a/paddle/fluid/framework/scope.h b/paddle/fluid/framework/scope.h index 98d103d867..34687df3ab 100644 --- a/paddle/fluid/framework/scope.h +++ b/paddle/fluid/framework/scope.h @@ -85,12 +85,20 @@ class Scope { // Call Scope::NewScope for a sub-scope. explicit Scope(Scope const* parent) : parent_(parent) {} + // Called by Var. + Variable* VarInternal(const std::string& name); + + // Called by FindScope. + const Scope* FindScopeInternal(const Variable* var) const; + + // Called by Rename. + void RenameInternal(const std::string& origin_name, + const std::string& new_name) const; + // Called by FindVar recursively. - // Caller doesn't own the returned Variable. Variable* FindVarInternal(const std::string& name) const; // Called by FindVarInternal and Var. - // Caller doesn't own the returned Variable. Variable* FindVarLocally(const std::string& name) const; mutable std::unordered_map> vars_; From 5be454bf330864c7cf5c4a331ded9c0a9c211cad Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 8 Jun 2018 16:51:06 +0800 Subject: [PATCH 005/154] polish docs --- .../framework/details/fuse_vars_op_handle.cc | 2 +- paddle/fluid/operators/crf_decoding_op.cc | 16 ++--- paddle/fluid/operators/roi_pool_op.cc | 10 +++ paddle/fluid/operators/scale_op.cc | 1 + python/paddle/fluid/layers/io.py | 31 ++++++++- python/paddle/fluid/layers/nn.py | 67 ++++++++++--------- python/paddle/fluid/layers/ops.py | 1 + 7 files changed, 86 insertions(+), 42 deletions(-) diff --git a/paddle/fluid/framework/details/fuse_vars_op_handle.cc b/paddle/fluid/framework/details/fuse_vars_op_handle.cc index 32415c192f..018c9bff71 100644 --- a/paddle/fluid/framework/details/fuse_vars_op_handle.cc +++ b/paddle/fluid/framework/details/fuse_vars_op_handle.cc @@ -42,7 +42,7 @@ void FuseVarsOpHandle::RunImpl() { out_t->ShareDataWith(out_tensor->Slice(s, s + numel)); s += numel; } - this->RunAndRecordEvent([this] {}); + this->RunAndRecordEvent([] {}); } std::string FuseVarsOpHandle::Name() const { return "fuse vars"; } diff --git a/paddle/fluid/operators/crf_decoding_op.cc b/paddle/fluid/operators/crf_decoding_op.cc index 40f43936db..a8d7831122 100644 --- a/paddle/fluid/operators/crf_decoding_op.cc +++ b/paddle/fluid/operators/crf_decoding_op.cc @@ -54,20 +54,20 @@ The output of this operator changes according to whether Input(Label) is given: 1. Input(Label) is given: -This happens in training. This operator is used to co-work with the chunk_eval -operator. + This happens in training. This operator is used to co-work with the chunk_eval + operator. -When Input(Label) is given, the crf_decoding operator returns a row vector -with shape [N x 1] whose values are fixed to be 0, indicating an incorrect -prediction, or 1 indicating a tag is correctly predicted. Such an output is the -input to chunk_eval operator. + When Input(Label) is given, the crf_decoding operator returns a row vector + with shape [N x 1] whose values are fixed to be 0, indicating an incorrect + prediction, or 1 indicating a tag is correctly predicted. Such an output is the + input to chunk_eval operator. 2. Input(Label) is not given: -This is the standard decoding process. + This is the standard decoding process. The crf_decoding operator returns a row vector with shape [N x 1] whose values -range from 0 to maximum tag number - 1. Each element indicates an index of a +range from 0 to maximum tag number - 1, Each element indicates an index of a predicted tag. )DOC"); } diff --git a/paddle/fluid/operators/roi_pool_op.cc b/paddle/fluid/operators/roi_pool_op.cc index 293abb0ea4..cb8982e57f 100644 --- a/paddle/fluid/operators/roi_pool_op.cc +++ b/paddle/fluid/operators/roi_pool_op.cc @@ -141,6 +141,16 @@ class ROIPoolOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( ROIPool operator +Region of interest pooling (also known as RoI pooling) is to perform +is to perform max pooling on inputs of nonuniform sizes to obtain +fixed-size feature maps (e.g. 7*7). + +The operator has three steps: +1. Dividing each region proposal into equal-sized sections with + the pooled_width and pooled_height +2. Finding the largest value in each section +3. Copying these max values to the output buffer + ROI Pooling for Faster-RCNN. The link below is a further introduction: https://stackoverflow.com/questions/43430056/what-is-roi-layer-in-fast-rcnn )DOC"); diff --git a/paddle/fluid/operators/scale_op.cc b/paddle/fluid/operators/scale_op.cc index 4687e21e71..24c217adcb 100644 --- a/paddle/fluid/operators/scale_op.cc +++ b/paddle/fluid/operators/scale_op.cc @@ -42,6 +42,7 @@ class ScaleOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(Tensor) Output tensor of scale operator."); AddComment(R"DOC( Scale operator +Multiply the input tensor with a float scalar to scale the input tensor. $$Out = scale*X$$ )DOC"); diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index a56f3ea9db..cff074faff 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -108,10 +108,35 @@ class BlockGuardServ(BlockGuard): class ListenAndServ(object): """ - ListenAndServ class. + ListenAndServ layer. - ListenAndServ class is used to wrap listen_and_serv op to create a server - which can receive variables from clients and run a block. + ListenAndServ is used to create a rpc server bind and listen + on specific TCP port, this server will run the sub-block when + received variables from clients. + + Args: + endpoint(string): IP:port string which the server will listen on. + inputs(list): a list of variables that the server will get from clients. + fan_in(int): how many client are expected to report to this server, default: 1. + optimizer_mode(bool): whether to run the server as a parameter server, default: True. + + Examples: + .. code-block:: python + + with fluid.program_guard(main): + serv = layers.ListenAndServ( + "127.0.0.1:6170", ["X"], optimizer_mode=False) + with serv.do(): + x = layers.data( + shape=[32, 32], + dtype='float32', + name="X", + append_batch_size=False) + fluid.initializer.Constant(value=1.0)(x, main.global_block()) + layers.scale(x=x, scale=10.0, out=out_var) + + self.server_exe = fluid.Executor(place) + self.server_exe.run(main) """ def __init__(self, endpoint, inputs, fan_in=1, optimizer_mode=True): diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index ddaeb415af..a1b3503746 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -869,10 +869,17 @@ def crf_decoding(input, param_attr, label=None): return viterbi_path +@templatedoc() def cos_sim(X, Y): """ - This function performs the cosine similarity between two tensors - X and Y and returns that as the output. + ${comment} + + Args: + X(${X_type}): ${X_comment} + Y(${Y_type}): ${Y_comment} + + Returns: + A Variable contains the output of this layer. """ helper = LayerHelper('cos_sim', **locals()) out = helper.create_tmp_variable(dtype=X.dtype) @@ -1059,14 +1066,25 @@ def square_error_cost(input, label): return square_out +@templatedoc() def chunk_eval(input, label, chunk_scheme, num_chunk_types, excluded_chunk_types=None): """ - This function computes and outputs the precision, recall and - F1-score of chunk detection. + ${comment} + + Args: + input(Variable): ${Inference_comment} + label(Variable): ${Label_comment} + chunk_scheme(${chunk_scheme_type}): ${chunk_scheme_comment} + num_chunk_types(${num_chunk_types_type}): ${num_chunk_types_comment} + excluded_chunk_types(${excluded_chunk_types_type}): ${excluded_chunk_types_comment} + + Returns(typle): a tuple of variables: + (precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks) + """ helper = LayerHelper("chunk_eval", **locals()) @@ -1737,6 +1755,7 @@ def beam_search_decode(ids, scores, name=None): return sentence_ids, sentence_scores +@templatedoc() def conv2d_transpose(input, num_filters, output_size=None, @@ -1760,7 +1779,7 @@ def conv2d_transpose(input, Parameters(dilations, strides, paddings) are two elements. These two elements represent height and width, respectively. The details of convolution transpose layer, please refer to the following explanation and references - `therein `_. + `here `_. For each input :math:`X`, the equation is: @@ -1774,7 +1793,7 @@ def conv2d_transpose(input, * :math:`W`: Filter value, a tensor with MCHW format. * :math:`\\ast` : Convolution transpose operation. * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be - different. + different. Example: @@ -2781,6 +2800,7 @@ def edit_distance(input, label, normalized=True, ignored_tokens=None, def ctc_greedy_decoder(input, blank, name=None): """ This op is used to decode sequences by greedy policy by below steps: + 1. Get the indexes of max value for each row in input. a.k.a. numpy.argmax(input, axis=0). 2. For each sequence in result of step1, merge repeated tokens between two @@ -3451,8 +3471,9 @@ def one_hot(input, depth): def autoincreased_step_counter(counter_name=None, begin=1, step=1): """ - NOTE: The counter will be automatically increased by 1 every mini-batch - Return the run counter of the main program, which is started with 1. + Create an auto-increase variable + which will be automatically increased by 1 every mini-batch + Return the run counter of the main program, default is started from 1. Args: counter_name(str): The counter name, default is '@STEP_COUNTER@'. @@ -3866,34 +3887,20 @@ def label_smooth(label, return smooth_label +@templatedoc() def roi_pool(input, rois, pooled_height=1, pooled_width=1, spatial_scale=1.0): """ - Region of interest pooling (also known as RoI pooling) is to perform - is to perform max pooling on inputs of nonuniform sizes to obtain - fixed-size feature maps (e.g. 7*7). - The operator has three steps: - 1. Dividing each region proposal into equal-sized sections with - the pooled_width and pooled_height - 2. Finding the largest value in each section - 3. Copying these max values to the output buffer + ${comment} Args: - input (Variable): The input for ROI pooling. - rois (Variable): ROIs (Regions of Interest) to pool over. It should - be a 2-D one level LoTensor of shape [num_rois, 4]. - The layout is [x1, y1, x2, y2], where (x1, y1) - is the top left coordinates, and (x2, y2) is the - bottom right coordinates. The num_rois is the - total number of ROIs in this batch data. - pooled_height (integer): The pooled output height. Default: 1 - pooled_width (integer): The pooled output width. Default: 1 - spatial_scale (float): Multiplicative spatial scale factor. To - translate ROI coords from their input scale - to the scale used when pooling. Default: 1.0 + input (Variable): ${X_comment} + rois (Variable): ${ROIs_comment} + pooled_height (integer): ${pooled_height_comment} Default: 1 + pooled_width (integer): ${pooled_width_comment} Default: 1 + spatial_scale (float): ${spatial_scale_comment} Default: 1.0 Returns: - pool_out (Variable): The output is a 4-D tensor of the shape - (num_rois, channels, pooled_h, pooled_w). + pool_out (Variable): ${Out_comment}. Examples: .. code-block:: python diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 69cfde852d..24b8b86dcf 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -73,6 +73,7 @@ __all__ = [ 'sum', 'polygon_box_transform', 'shape', + 'iou_similarity', ] + __activations__ for _OP in set(__all__): From efcff3d9e50ef75e854e8945c3b829e94ddffa50 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 8 Jun 2018 17:48:14 +0800 Subject: [PATCH 006/154] polish api ref docs --- doc/fluid/api/layers.rst | 6 ++++ .../operators/detection/iou_similarity_op.cc | 7 +++-- paddle/fluid/operators/roi_pool_op.cc | 1 + python/paddle/fluid/layers/io.py | 4 +-- python/paddle/fluid/layers/nn.py | 28 +++++++++++++------ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/doc/fluid/api/layers.rst b/doc/fluid/api/layers.rst index f78e6db326..ef8ada407b 100644 --- a/doc/fluid/api/layers.rst +++ b/doc/fluid/api/layers.rst @@ -790,6 +790,12 @@ shape .. autofunction:: paddle.fluid.layers.shape :noindex: +iou_similarity +----- + +.. autofunction:: paddle.fluid.layers.iou_similarity + :noindex: + sigmoid ------- diff --git a/paddle/fluid/operators/detection/iou_similarity_op.cc b/paddle/fluid/operators/detection/iou_similarity_op.cc index 8e58605fce..b08c2cdf53 100644 --- a/paddle/fluid/operators/detection/iou_similarity_op.cc +++ b/paddle/fluid/operators/detection/iou_similarity_op.cc @@ -69,10 +69,11 @@ class IOUSimilarityOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( IOU Similarity Operator. + Computes intersection-over-union (IOU) between two box lists. - Box list 'X' should be a LoDTensor and 'Y' is a common Tensor, - boxes in 'Y' are shared by all instance of the batched inputs of X. - Given two boxes A and B, the calculation of IOU is as follows: +Box list 'X' should be a LoDTensor and 'Y' is a common Tensor, +boxes in 'Y' are shared by all instance of the batched inputs of X. +Given two boxes A and B, the calculation of IOU is as follows: $$ IOU(A, B) = diff --git a/paddle/fluid/operators/roi_pool_op.cc b/paddle/fluid/operators/roi_pool_op.cc index cb8982e57f..a6247a467a 100644 --- a/paddle/fluid/operators/roi_pool_op.cc +++ b/paddle/fluid/operators/roi_pool_op.cc @@ -146,6 +146,7 @@ is to perform max pooling on inputs of nonuniform sizes to obtain fixed-size feature maps (e.g. 7*7). The operator has three steps: + 1. Dividing each region proposal into equal-sized sections with the pooled_width and pooled_height 2. Finding the largest value in each section diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index cff074faff..ecfaf35296 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -135,8 +135,8 @@ class ListenAndServ(object): fluid.initializer.Constant(value=1.0)(x, main.global_block()) layers.scale(x=x, scale=10.0, out=out_var) - self.server_exe = fluid.Executor(place) - self.server_exe.run(main) + exe = fluid.Executor(place) + exe.run(main) """ def __init__(self, endpoint, inputs, fan_in=1, optimizer_mode=True): diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index a1b3503746..10dba6cafd 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -850,7 +850,9 @@ def crf_decoding(input, param_attr, label=None): Args: input(${emission_type}): ${emission_comment} + param_attr(ParamAttr): The parameter attribute for training. + label(${label_type}): ${label_comment} Returns: @@ -875,8 +877,8 @@ def cos_sim(X, Y): ${comment} Args: - X(${X_type}): ${X_comment} - Y(${Y_type}): ${Y_comment} + X(${x_type}): ${x_comment} + Y(${y_type}): ${x_comment} Returns: A Variable contains the output of this layer. @@ -1076,13 +1078,18 @@ def chunk_eval(input, ${comment} Args: - input(Variable): ${Inference_comment} - label(Variable): ${Label_comment} + input(Variable): ${inference_comment} + + label(Variable): ${label_comment} + chunk_scheme(${chunk_scheme_type}): ${chunk_scheme_comment} + num_chunk_types(${num_chunk_types_type}): ${num_chunk_types_comment} + excluded_chunk_types(${excluded_chunk_types_type}): ${excluded_chunk_types_comment} - Returns(typle): a tuple of variables: + Returns: + chunk_eval(tuple): a tuple of variables: (precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks) """ @@ -1755,7 +1762,6 @@ def beam_search_decode(ids, scores, name=None): return sentence_ids, sentence_scores -@templatedoc() def conv2d_transpose(input, num_filters, output_size=None, @@ -3893,14 +3899,18 @@ def roi_pool(input, rois, pooled_height=1, pooled_width=1, spatial_scale=1.0): ${comment} Args: - input (Variable): ${X_comment} - rois (Variable): ${ROIs_comment} + input (Variable): ${x_comment} + + rois (Variable): ROIs (Regions of Interest) to pool over. + pooled_height (integer): ${pooled_height_comment} Default: 1 + pooled_width (integer): ${pooled_width_comment} Default: 1 + spatial_scale (float): ${spatial_scale_comment} Default: 1.0 Returns: - pool_out (Variable): ${Out_comment}. + roi_pool (Variable): ${out_comment}. Examples: .. code-block:: python From 7cebec4b7e876c27af4de5d37101b53260c0bd49 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 10 Jun 2018 22:11:53 +0800 Subject: [PATCH 007/154] init merge_ids_op --- paddle/fluid/operators/merge_ids_op.cc | 97 ++++++++++++++++++++++++++ paddle/fluid/operators/merge_ids_op.h | 83 ++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 paddle/fluid/operators/merge_ids_op.cc create mode 100644 paddle/fluid/operators/merge_ids_op.h diff --git a/paddle/fluid/operators/merge_ids_op.cc b/paddle/fluid/operators/merge_ids_op.cc new file mode 100644 index 0000000000..939561509c --- /dev/null +++ b/paddle/fluid/operators/merge_ids_op.cc @@ -0,0 +1,97 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/merge_ids_op.h" + +namespace paddle { +namespace operators { + +class MergeIdsOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("Ids", "(LoDTensor) the input ids with shape{batch_num, 1}"); + AddInput("X", + "(LoDTensor) the input tensor with shape{batch_num, N}, N is the " + "size of embedding table") + .AsDuplicable(); + AddOutput("Out", "(LoDTensor) The merged outputs of the input tensors."); + + AddComment(R"DOC( +Merge multi LoDTensor's into one according to Ids's shard num. +The values in the input LoDTensor are lookuped from the output of splite_ids_op +Example: + Input: + Ids = [1,2,3,4,5,6] + X0 = [[0.1 0.2] # 3 + [0.2 0.3]] # 6 + X1 = [[0.3 0.4] # 1 + [0.4 0.5]] # 4 + X2 = [[0.5 0.6] # 2 + [0.6 0.7]] # 5 + + Output: + Out = [[0.3 0.4] # 1 + [0.5 0.6] # 2 + [0.1 0.2] # 3 + [0.4 0.5] # 4 + [0.6 0.7] # 5 + [0.2 0.3]] # 6 +)DOC"); + } +}; + +class MergeIdsOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Ids"), "MergeIdsOp must has input Ids."); + PADDLE_ENFORCE(ctx->HasInputs("X"), "MergeIdsOp must has input X."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "MergeIdsOp must has output Out."); + + auto ids_var_type = ctx->GetInputsVarType("Ids").front(); + auto ids_dims = ctx->GetInputDim("Ids"); + if (ids_var_type == framework::proto::VarType::LOD_TENSOR) { + PADDLE_ENFORCE_EQ(ids_dims.size(), 2); + PADDLE_ENFORCE_EQ(ids_dims[1], 1); + } + auto x_var_type = ctx->GetInputsVarType("X"); + for (auto &var_type : x_var_type) { + PADDLE_ENFORCE_EQ(var_type, framework::proto::VarType::LOD_TENSOR, + "input X only support lod tensors"); + } + ctx->ShareLoD("Ids", "Out"); + } +}; + +class MergeIdsOpInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDesc &op_desc, + framework::BlockDesc *block) const override { + auto *input_var = block->Var(op_desc.Input("Ids")[0]); + for (auto &out_var : op_desc.Output("Out")) { + block->Var(out_var)->SetType(input_var->GetType()); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(merge_ids, ops::MergeIdsOp, ops::MergeIdsOpMaker, + ops::MergeIdsOpInferVarType); +REGISTER_OP_CPU_KERNEL( + merge_ids, ops::MergeIdsOpKernel, + ops::MergeIdsOpKernel); diff --git a/paddle/fluid/operators/merge_ids_op.h b/paddle/fluid/operators/merge_ids_op.h new file mode 100644 index 0000000000..fd5b542ceb --- /dev/null +++ b/paddle/fluid/operators/merge_ids_op.h @@ -0,0 +1,83 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/tensor_util.h" +#include "paddle/fluid/operators/math/selected_rows_functor.h" + +namespace paddle { +namespace operators { + +template +class MergeIdsOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + auto place = ctx.GetPlace(); + if (!platform::is_cpu_place(place)) { + PADDLE_THROW("MergeIds do not support GPU kernel"); + } + + const auto *ids_var = ctx.InputVar("Ids"); + PADDLE_ENFORCE(ids_var->IsType(), + "only support to merge Ids of LoDTensor"); + + const auto &ids_tensor = ids_var->Get(); + const auto &ids_dims = ids_tensor.dims(); + const T *ids = ids_tensor.data(); + + auto x_tensors = ctx.MultiInput("X"); + + auto *out = ctx.Output("Out"); + + int batch_size = 0; + int embedding_size = 0; + for (auto &input : x_tensors) { + if (embedding_size == 0) { + embedding_size = input->dims()[1]; + } + PADDLE_ENFORCE_EQ(embedding_size, input->dims()[1], + "embedding size of all input should be the same"); + batch_size += input->dims()[0]; + } + PADDLE_ENFORCE_EQ( + batch_size, ids_dims[0], + "the batch size of ids and embedding value should be the same"); + + const size_t shard_num = x_tensors.size(); + + if (shard_num == 1) { + VLOG(3) << "only one shard, we can copy the data directly"; + TensorCopy(ids_tensor, place, out); + } else { + std::vector in_indexs(shard_num, 0); + auto *out_data = out->mutable_data(ids_dims, place); + // copy data from ins[shard_num] to out. + for (int i = 0; i < ids_dims[0]; ++i) { + T id = ids[i]; + size_t shard_id = static_cast(id) % shard_num; + int index = in_indexs[shard_id]; + memcpy(out_data + embedding_size * i, + x_tensors[shard_id]->data() + index * embedding_size, + sizeof(T) * embedding_size); + in_indexs[shard_id] += 1; + } + } + } +}; + +} // namespace operators +} // namespace paddle From 509cb0bc76ea3b22423e293f608d4956a63deda7 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 10 Jun 2018 23:31:41 +0800 Subject: [PATCH 008/154] add unit test, pass the unit test --- paddle/fluid/operators/merge_ids_op.cc | 12 +++++- paddle/fluid/operators/merge_ids_op.h | 23 +++++++---- .../tests/unittests/test_merge_ids_op.py | 38 +++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/test_merge_ids_op.py diff --git a/paddle/fluid/operators/merge_ids_op.cc b/paddle/fluid/operators/merge_ids_op.cc index 939561509c..bae649adec 100644 --- a/paddle/fluid/operators/merge_ids_op.cc +++ b/paddle/fluid/operators/merge_ids_op.cc @@ -73,6 +73,15 @@ class MergeIdsOp : public framework::OperatorWithKernel { } ctx->ShareLoD("Ids", "Out"); } + + private: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType( + ctx.MultiInput("X").front()->type()), + ctx.GetPlace()); + } }; class MergeIdsOpInferVarType : public framework::VarTypeInference { @@ -93,5 +102,4 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(merge_ids, ops::MergeIdsOp, ops::MergeIdsOpMaker, ops::MergeIdsOpInferVarType); REGISTER_OP_CPU_KERNEL( - merge_ids, ops::MergeIdsOpKernel, - ops::MergeIdsOpKernel); + merge_ids, ops::MergeIdsOpKernel); diff --git a/paddle/fluid/operators/merge_ids_op.h b/paddle/fluid/operators/merge_ids_op.h index fd5b542ceb..065368f8dd 100644 --- a/paddle/fluid/operators/merge_ids_op.h +++ b/paddle/fluid/operators/merge_ids_op.h @@ -30,6 +30,7 @@ class MergeIdsOpKernel : public framework::OpKernel { if (!platform::is_cpu_place(place)) { PADDLE_THROW("MergeIds do not support GPU kernel"); } + VLOG(3) << "run in MergeIdsOpKernel"; const auto *ids_var = ctx.InputVar("Ids"); PADDLE_ENFORCE(ids_var->IsType(), @@ -37,7 +38,7 @@ class MergeIdsOpKernel : public framework::OpKernel { const auto &ids_tensor = ids_var->Get(); const auto &ids_dims = ids_tensor.dims(); - const T *ids = ids_tensor.data(); + const int64_t *ids = ids_tensor.data(); auto x_tensors = ctx.MultiInput("X"); @@ -49,9 +50,11 @@ class MergeIdsOpKernel : public framework::OpKernel { if (embedding_size == 0) { embedding_size = input->dims()[1]; } - PADDLE_ENFORCE_EQ(embedding_size, input->dims()[1], - "embedding size of all input should be the same"); - batch_size += input->dims()[0]; + if (framework::product(input->dims()) != 0) { + PADDLE_ENFORCE_EQ(embedding_size, input->dims()[1], + "embedding size of all input should be the same"); + batch_size += input->dims()[0]; + } } PADDLE_ENFORCE_EQ( batch_size, ids_dims[0], @@ -61,13 +64,14 @@ class MergeIdsOpKernel : public framework::OpKernel { if (shard_num == 1) { VLOG(3) << "only one shard, we can copy the data directly"; - TensorCopy(ids_tensor, place, out); + TensorCopy(*x_tensors[0], place, out); } else { std::vector in_indexs(shard_num, 0); - auto *out_data = out->mutable_data(ids_dims, place); + auto *out_data = out->mutable_data( + framework::make_ddim({batch_size, embedding_size}), place); // copy data from ins[shard_num] to out. for (int i = 0; i < ids_dims[0]; ++i) { - T id = ids[i]; + int64_t id = ids[i]; size_t shard_id = static_cast(id) % shard_num; int index = in_indexs[shard_id]; memcpy(out_data + embedding_size * i, @@ -75,6 +79,11 @@ class MergeIdsOpKernel : public framework::OpKernel { sizeof(T) * embedding_size); in_indexs[shard_id] += 1; } + + for (int i = 0; i < shard_num; ++i) { + PADDLE_ENFORCE_EQ(in_indexs[i], x_tensors[i]->dims()[0], + "after merge, all data in x_tensor should be used"); + } } } }; diff --git a/python/paddle/fluid/tests/unittests/test_merge_ids_op.py b/python/paddle/fluid/tests/unittests/test_merge_ids_op.py new file mode 100644 index 0000000000..f209bdf30f --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_merge_ids_op.py @@ -0,0 +1,38 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest + + +class TestMergeIdsOp(OpTest): + def setUp(self): + self.op_type = "merge_ids" + ids = np.array([[0], [2], [2], [3], [5], [5], [6]]).astype('int64') + x0 = np.array([[0.1, 0.2], [0.2, 0.3], [0.3, 0.4]]).astype('float32') + x1 = np.array([]).astype('float32') + x2 = np.array([[0.4, 0.5], [0.4, 0.5], [0.5, 0.6], + [0.5, 0.6]]).astype('float32') + out = np.array([[0.1, 0.2], [0.4, 0.5], [0.4, 0.5], [0.2, 0.3], + [0.5, 0.6], [0.5, 0.6], [0.3, 0.4]]).astype('float32') + self.inputs = {'Ids': ids, "X": [('x0', x0), ('x1', x1), ('x2', x2)]} + self.outputs = {'Out': out} + + def test_check_output(self): + self.check_output() + + +if __name__ == '__main__': + unittest.main() From 9908d3cfbc4cc6589f3d71693c27a3731170c0ee Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Sun, 10 Jun 2018 23:12:44 +0200 Subject: [PATCH 009/154] MKLDNN layout: Support for convolution operator --- paddle/fluid/operators/conv_mkldnn_op.cc | 363 ++++++++++++++--------- paddle/fluid/operators/conv_op.cc | 3 +- 2 files changed, 230 insertions(+), 136 deletions(-) diff --git a/paddle/fluid/operators/conv_mkldnn_op.cc b/paddle/fluid/operators/conv_mkldnn_op.cc index 63d371310d..6b06913d1c 100644 --- a/paddle/fluid/operators/conv_mkldnn_op.cc +++ b/paddle/fluid/operators/conv_mkldnn_op.cc @@ -18,6 +18,17 @@ namespace paddle { namespace operators { +using conv_bwd_data = mkldnn::convolution_backward_data; +using conv_bwd_weights = mkldnn::convolution_backward_weights; +using conv_fwd = mkldnn::convolution_forward; +using framework::DataLayout; +using mkldnn::memory; +using mkldnn::primitive; +using mkldnn::reorder; +using mkldnn::stream; +using platform::to_void_cast; +using platform::GetMKLDNNFormat; + template class ConvMKLDNNOpKernel : public paddle::framework::OpKernel { public: @@ -25,6 +36,10 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel { PADDLE_ENFORCE(paddle::platform::is_cpu_place(ctx.GetPlace()), "It must use CPUPlace."); + // Get unique name for index + const std::string key = ctx.op().Output("Output"); + const std::string key_conv_pd = key + "@conv_pd"; + auto& dev_ctx = ctx.template device_context(); const auto& mkldnn_engine = dev_ctx.GetEngine(); @@ -33,10 +48,12 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel { auto* filter = ctx.Input("Filter"); auto* output = ctx.Output("Output"); - // Get an unique name from "argument" name of "Output" variable - // This name will be used as key when saving info into device context - const std::string key = ctx.op().Output("Output"); - const std::string key_conv_pd = key + "@conv_pd"; + PADDLE_ENFORCE(input->layout() == DataLayout::kMKLDNN && + input->format() != memory::format::format_undef, + "Wrong layout/format set for Input tensor"); + PADDLE_ENFORCE(filter->layout() == DataLayout::kMKLDNN && + filter->format() != memory::format::format_undef, + "Wrong layout/format set for Filter tensor"); std::vector strides = ctx.Attr>("strides"); std::vector paddings = ctx.Attr>("paddings"); @@ -63,60 +80,86 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel { paddle::framework::vectorize2int(filter->dims()); std::vector dst_tz = paddle::framework::vectorize2int(output->dims()); - // TODO(pzelazko-intel): support more formats - auto src_md = platform::MKLDNNMemDesc( - src_tz, mkldnn::memory::data_type::f32, mkldnn::memory::format::nchw); - auto weights_md = - platform::MKLDNNMemDesc(weights_tz, mkldnn::memory::data_type::f32, - mkldnn::memory::format::oihw); - auto dst_md = platform::MKLDNNMemDesc( - dst_tz, mkldnn::memory::data_type::f32, mkldnn::memory::format::nchw); - - auto src_memory = - mkldnn::memory({src_md, mkldnn_engine}, - reinterpret_cast(const_cast(input_data))); - auto weights_memory = - mkldnn::memory({weights_md, mkldnn_engine}, - reinterpret_cast(const_cast(filter_data))); - auto dst_memory = mkldnn::memory({dst_md, mkldnn_engine}, output_data); - - std::shared_ptr conv_pd = - ConvFwdPrimitiveDesc(src_md, weights_md, dst_md, strides, paddings, - mkldnn_engine); - - // save conv_pd into global device context to be referred in backward path - dev_ctx.SetBlob(key_conv_pd, conv_pd); + // create mkldnn memory from input tensors (data/weights) + auto user_src_memory = memory( + {{{src_tz}, memory::data_type::f32, input->format()}, mkldnn_engine}, + to_void_cast(input_data)); + auto user_weights_memory = + memory({{{weights_tz}, memory::data_type::f32, filter->format()}, + mkldnn_engine}, + to_void_cast(filter_data)); + + /* create memory descriptor for convolution without specified format + * ('any') which lets a primitive (convolution in this case) choose + * the memory format preferred for best performance + */ + auto src_md = platform::MKLDNNMemDesc(src_tz, memory::data_type::f32, + memory::format::any); + auto weights_md = platform::MKLDNNMemDesc( + weights_tz, memory::data_type::f32, memory::format::any); + auto dst_md = platform::MKLDNNMemDesc(dst_tz, memory::data_type::f32, + memory::format::any); + + // create a conv primitive descriptor and save it for usage in backward + std::shared_ptr conv_pd = ConvFwdPrimitiveDesc( + src_md, weights_md, dst_md, strides, paddings, mkldnn_engine); + + // create reorder primitive if the input format is not the preferred one + auto src_memory = user_src_memory; + primitive reorder_src; + bool is_src_reordered = false; + if (memory::primitive_desc(conv_pd->src_primitive_desc()) != + user_src_memory.get_primitive_desc()) { + src_memory = memory(conv_pd->src_primitive_desc()); + reorder_src = reorder(user_src_memory, src_memory); + is_src_reordered = true; + } + auto weights_memory = user_weights_memory; + primitive reorder_weights; + bool is_weights_reordered = false; + if (memory::primitive_desc(conv_pd->weights_primitive_desc()) != + user_weights_memory.get_primitive_desc()) { + weights_memory = memory(conv_pd->weights_primitive_desc()); + reorder_weights = reorder(user_weights_memory, weights_memory); + is_weights_reordered = true; + } + + // create memory primitive for conv dst + auto dst_memory = memory(conv_pd->dst_primitive_desc(), output_data); // create convolution op primitive - auto conv_prim = mkldnn::convolution_forward(*conv_pd, src_memory, - weights_memory, dst_memory); + auto conv_prim = conv_fwd(*conv_pd, src_memory, weights_memory, dst_memory); // push primitive to stream and wait until it's executed - std::vector pipeline{conv_prim}; - mkldnn::stream(mkldnn::stream::kind::eager).submit(pipeline).wait(); + std::vector pipeline; + if (is_src_reordered) pipeline.push_back(reorder_src); + if (is_weights_reordered) pipeline.push_back(reorder_weights); + pipeline.push_back(conv_prim); + stream(stream::kind::eager).submit(pipeline).wait(); + + // Save conv_pd/src_memory/weights_memory for backward pass + dev_ctx.SetBlob(key_conv_pd, conv_pd); + + output->set_layout(DataLayout::kMKLDNN); + output->set_format(GetMKLDNNFormat(dst_memory)); } private: - std::unique_ptr - ConvFwdPrimitiveDesc(const mkldnn::memory::desc& src, - const mkldnn::memory::desc& weights, - const mkldnn::memory::desc& dst, - const std::vector& strides, - const std::vector& paddings, - const mkldnn::engine& engine) const { - mkldnn::memory::dims stride_dims = {strides[0], strides[1]}; - mkldnn::memory::dims padding_dims = {paddings[0], paddings[1]}; - - auto conv_desc = mkldnn::convolution_forward::desc( - mkldnn::prop_kind::forward, mkldnn::convolution_direct, src, weights, - dst, stride_dims, padding_dims, padding_dims, - mkldnn::padding_kind::zero); - - auto p_conv_pd = - new mkldnn::convolution_forward::primitive_desc(conv_desc, engine); - - return std::unique_ptr( - p_conv_pd); + std::unique_ptr ConvFwdPrimitiveDesc( + const memory::desc& src, const memory::desc& weights, + const memory::desc& dst, const std::vector& strides, + const std::vector& paddings, const mkldnn::engine& engine) const { + memory::dims stride_dims = {strides[0], strides[1]}; + memory::dims padding_dims = {paddings[0], paddings[1]}; + + auto conv_desc = + conv_fwd::desc(mkldnn::prop_kind::forward, mkldnn::convolution_direct, + src, weights, dst, stride_dims, padding_dims, + padding_dims, mkldnn::padding_kind::zero); + + auto p_conv_pd = new conv_fwd::primitive_desc(conv_desc, engine); + + return std::unique_ptr(p_conv_pd); } }; @@ -139,6 +182,19 @@ class ConvMKLDNNGradOpKernel : public paddle::framework::OpKernel { Tensor* input_grad = ctx.Output(framework::GradVarName("Input")); Tensor* filter_grad = ctx.Output(framework::GradVarName("Filter")); + PADDLE_ENFORCE(input->layout() == DataLayout::kMKLDNN && + input->format() != memory::format::format_undef, + "Wrong layout/format set for Input tensor"); + PADDLE_ENFORCE(filter->layout() == DataLayout::kMKLDNN && + filter->format() != memory::format::format_undef, + "Wrong layout/format set for Filter tensor"); + PADDLE_ENFORCE(output->layout() == DataLayout::kMKLDNN && + output->format() != memory::format::format_undef, + "Wrong layout/format set for Output tensor"); + PADDLE_ENFORCE(output_grad->layout() == DataLayout::kMKLDNN && + output_grad->format() != memory::format::format_undef, + "Wrong layout/format set for output_grad tensor"); + if (!input_grad && !filter_grad) return; // Get an unique name from "argument" name of "Output" variable @@ -167,108 +223,147 @@ class ConvMKLDNNGradOpKernel : public paddle::framework::OpKernel { paddle::framework::vectorize2int(filter->dims()); std::vector dst_tz = paddle::framework::vectorize2int(output->dims()); - // TODO(pzelazko-intel): support more formats - auto src_md = platform::MKLDNNMemDesc( - src_tz, mkldnn::memory::data_type::f32, mkldnn::memory::format::nchw); - auto diff_src_md = platform::MKLDNNMemDesc( - src_tz, mkldnn::memory::data_type::f32, mkldnn::memory::format::nchw); - auto weights_md = - platform::MKLDNNMemDesc(weights_tz, mkldnn::memory::data_type::f32, - mkldnn::memory::format::oihw); - auto diff_weights_md = - platform::MKLDNNMemDesc(weights_tz, mkldnn::memory::data_type::f32, - mkldnn::memory::format::oihw); - auto diff_dst_md = platform::MKLDNNMemDesc( - dst_tz, mkldnn::memory::data_type::f32, mkldnn::memory::format::nchw); - - // create memory - auto diff_dst_memory = mkldnn::memory( - {diff_weights_md, mkldnn_engine}, - reinterpret_cast(const_cast(output_grad_data))); + // create mkldnn memory from input tensors (input/weights/output_grad) + auto user_src_memory = memory( + {{{src_tz}, memory::data_type::f32, input->format()}, mkldnn_engine}, + to_void_cast(input_data)); + auto user_weights_memory = + memory({{{weights_tz}, memory::data_type::f32, filter->format()}, + mkldnn_engine}, + to_void_cast(filter_data)); + auto user_diff_dst_memory = + memory({{{dst_tz}, memory::data_type::f32, output_grad->format()}, + mkldnn_engine}, + to_void_cast(output_grad_data)); + + /* create memory descriptor for conv backward without specified format + * ('any') which lets a primitive (conv backward in this case) choose + * the memory format preferred for best performance + */ + auto src_md = platform::MKLDNNMemDesc(src_tz, memory::data_type::f32, + memory::format::any); + auto diff_src_md = platform::MKLDNNMemDesc(src_tz, memory::data_type::f32, + memory::format::any); + auto weights_md = platform::MKLDNNMemDesc( + weights_tz, memory::data_type::f32, memory::format::any); + auto diff_weights_md = platform::MKLDNNMemDesc( + weights_tz, memory::data_type::f32, memory::format::any); + auto diff_dst_md = platform::MKLDNNMemDesc(dst_tz, memory::data_type::f32, + memory::format::any); + // Retrieve conv_pd from device context - auto conv_pd = - std::static_pointer_cast( - dev_ctx.GetBlob(key_conv_pd)); + auto conv_pd = std::static_pointer_cast( + dev_ctx.GetBlob(key_conv_pd)); PADDLE_ENFORCE(conv_pd != nullptr, "Fail to find conv_pd in device context"); // create backward conv primitive for weights if (filter_grad) { - // create primitive descriptor - mkldnn::convolution_backward_weights::primitive_desc conv_bwd_weights_pd = - ConvBwdWeightsPrimitiveDesc(src_md, diff_weights_md, diff_dst_md, - strides, paddings, *conv_pd, - mkldnn_engine); - - // create memory + // create backward convolution primitive descriptor + auto conv_bwd_weights_desc = conv_bwd_weights::desc( + mkldnn::convolution_direct, src_md, diff_weights_md, diff_dst_md, + strides, paddings, paddings, mkldnn::padding_kind::zero); + auto conv_bwd_weights_pd = conv_bwd_weights::primitive_desc( + conv_bwd_weights_desc, mkldnn_engine, *conv_pd); + + // create reorder primitive if the input format is not the preferred one + auto src_memory = user_src_memory; + primitive reorder_src; + bool is_src_reordered = false; + if (memory::primitive_desc(conv_bwd_weights_pd.src_primitive_desc()) != + user_src_memory.get_primitive_desc()) { + src_memory = memory(conv_bwd_weights_pd.src_primitive_desc()); + reorder_src = reorder(user_src_memory, src_memory); + is_src_reordered = true; + } + + auto diff_dst_memory_4filter = user_diff_dst_memory; + primitive reorder_diff_dst_4filter; + bool is_diff_dst_reordered_4filter = false; + if (memory::primitive_desc( + conv_bwd_weights_pd.diff_dst_primitive_desc()) != + user_diff_dst_memory.get_primitive_desc()) { + diff_dst_memory_4filter = + memory(conv_bwd_weights_pd.diff_dst_primitive_desc()); + reorder_diff_dst_4filter = + reorder(user_diff_dst_memory, diff_dst_memory_4filter); + is_diff_dst_reordered_4filter = true; + } + + // create mkldnn memory for output (i.e. diff weights) auto diff_weights_memory = - mkldnn::memory({diff_weights_md, mkldnn_engine}, - reinterpret_cast(filter_grad_data)); - auto src_memory = - mkldnn::memory({src_md, mkldnn_engine}, - reinterpret_cast(const_cast(input_data))); + memory(conv_bwd_weights_pd.diff_weights_primitive_desc(), + reinterpret_cast(filter_grad_data)); // create backward conv primitive for weights - auto conv_bwd_weights_prim = mkldnn::convolution_backward_weights( - conv_bwd_weights_pd, src_memory, diff_dst_memory, - diff_weights_memory); + auto conv_bwd_weights_prim = + conv_bwd_weights(conv_bwd_weights_pd, src_memory, + diff_dst_memory_4filter, diff_weights_memory); // push primitive and execute it - std::vector pipeline{conv_bwd_weights_prim}; - mkldnn::stream(mkldnn::stream::kind::eager).submit(pipeline).wait(); + std::vector pipeline; + if (is_src_reordered) pipeline.push_back(reorder_src); + if (is_diff_dst_reordered_4filter) + pipeline.push_back(reorder_diff_dst_4filter); + pipeline.push_back(conv_bwd_weights_prim); + stream(stream::kind::eager).submit(pipeline).wait(); + + filter_grad->set_layout(DataLayout::kMKLDNN); + filter_grad->set_format(GetMKLDNNFormat(diff_weights_memory)); } if (input_grad) { - // create primitive descriptor - mkldnn::convolution_backward_data::primitive_desc conv_bwd_data_pd = - ConvBwdDataPrimitiveDesc(diff_src_md, weights_md, diff_dst_md, - strides, paddings, *conv_pd, mkldnn_engine); - - // create memory - auto diff_src_memory = mkldnn::memory( - {diff_src_md, mkldnn_engine}, - reinterpret_cast(const_cast(input_grad_data))); - auto weights_memory = - mkldnn::memory({weights_md, mkldnn_engine}, - reinterpret_cast(const_cast(filter_data))); + // create backward convolution primitive descriptor + auto conv_bwd_data_desc = conv_bwd_data::desc( + mkldnn::convolution_direct, diff_src_md, weights_md, diff_dst_md, + strides, paddings, paddings, mkldnn::padding_kind::zero); + auto conv_bwd_data_pd = conv_bwd_data::primitive_desc( + conv_bwd_data_desc, mkldnn_engine, *conv_pd); + + // create reorder primitive if the input format is not the preferred one + auto weights_memory = user_weights_memory; + primitive reorder_weights; + bool is_weights_reordered = false; + if (memory::primitive_desc(conv_bwd_data_pd.weights_primitive_desc()) != + user_weights_memory.get_primitive_desc()) { + weights_memory = memory(conv_bwd_data_pd.weights_primitive_desc()); + reorder_weights = reorder(user_weights_memory, weights_memory); + is_weights_reordered = true; + } + + auto diff_dst_memory_4data = user_diff_dst_memory; + primitive reorder_diff_dst_4data; + bool is_diff_dst_reordered_4data = false; + if (memory::primitive_desc(conv_bwd_data_pd.diff_dst_primitive_desc()) != + user_diff_dst_memory.get_primitive_desc()) { + diff_dst_memory_4data = + memory(conv_bwd_data_pd.diff_dst_primitive_desc()); + reorder_diff_dst_4data = + reorder(user_diff_dst_memory, diff_dst_memory_4data); + is_diff_dst_reordered_4data = true; + } + + // create mkldnn memory for output (i.e. diff src) + auto diff_src_memory = memory(conv_bwd_data_pd.diff_src_primitive_desc(), + reinterpret_cast(input_grad_data)); // create backward conv primitive for data - auto conv_bwd_data_prim = mkldnn::convolution_backward_data( - conv_bwd_data_pd, diff_dst_memory, weights_memory, diff_src_memory); + auto conv_bwd_data_prim = + conv_bwd_data(conv_bwd_data_pd, diff_dst_memory_4data, weights_memory, + diff_src_memory); - // push primitive to stream and wait until it's executed - std::vector pipeline{conv_bwd_data_prim}; - mkldnn::stream(mkldnn::stream::kind::eager).submit(pipeline).wait(); + // push primitive and execute it + std::vector pipeline; + if (is_weights_reordered) pipeline.push_back(reorder_weights); + if (is_diff_dst_reordered_4data) + pipeline.push_back(reorder_diff_dst_4data); + pipeline.push_back(conv_bwd_data_prim); + stream(stream::kind::eager).submit(pipeline).wait(); + + input_grad->set_layout(DataLayout::kMKLDNN); + input_grad->set_format(GetMKLDNNFormat(diff_src_memory)); } } // Compute() - - private: - mkldnn::convolution_backward_weights::primitive_desc - ConvBwdWeightsPrimitiveDesc( - const mkldnn::memory::desc& src, const mkldnn::memory::desc& diff_weights, - const mkldnn::memory::desc& diff_dst, const std::vector& strides, - const std::vector& paddings, - const mkldnn::convolution_forward::primitive_desc& conv_pd, - const mkldnn::engine& engine) const { - auto conv_bwd_weights_desc = mkldnn::convolution_backward_weights::desc( - mkldnn::convolution_direct, src, diff_weights, diff_dst, strides, - paddings, paddings, mkldnn::padding_kind::zero); - return mkldnn::convolution_backward_weights::primitive_desc( - conv_bwd_weights_desc, engine, conv_pd); - } - - mkldnn::convolution_backward_data::primitive_desc ConvBwdDataPrimitiveDesc( - const mkldnn::memory::desc& diff_src, const mkldnn::memory::desc& weights, - const mkldnn::memory::desc& diff_dst, const std::vector& strides, - const std::vector& paddings, - const mkldnn::convolution_forward::primitive_desc& conv_pd, - const mkldnn::engine& engine) const { - auto conv_bwd_data_desc = mkldnn::convolution_backward_data::desc( - mkldnn::convolution_direct, diff_src, weights, diff_dst, strides, - paddings, paddings, mkldnn::padding_kind::zero); - return mkldnn::convolution_backward_data::primitive_desc(conv_bwd_data_desc, - engine, conv_pd); - } }; } // namespace operators diff --git a/paddle/fluid/operators/conv_op.cc b/paddle/fluid/operators/conv_op.cc index 850297a232..f38388256c 100644 --- a/paddle/fluid/operators/conv_op.cc +++ b/paddle/fluid/operators/conv_op.cc @@ -75,9 +75,8 @@ void ConvOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { framework::LibraryType library{framework::LibraryType::kPlain}; - - std::string data_format = ctx.Attr("data_format"); // TODO(pzelazko-intel): enable MKLDNN layout when it's ready + std::string data_format = ctx.Attr("data_format"); framework::DataLayout layout = framework::StringToDataLayout(data_format); #ifdef PADDLE_WITH_CUDA From a941786393e5b840a98215d81ba09c6cf0995ca1 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 11 Jun 2018 09:40:19 +0800 Subject: [PATCH 010/154] replace concat_op with merge_ids_op --- .../fluid/transpiler/distribute_transpiler.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index 27992df462..ed4158bc4c 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -618,7 +618,7 @@ class DistributeTranspiler: if op.type == LOOKUP_TABLE_TYPE: continue_search_lookup_table_op = True - op_index = list(all_ops).index(op) + lookup_table_op_index = list(all_ops).index(op) ids_name = op.input("Ids") out_name = op.output("Out") @@ -637,7 +637,7 @@ class DistributeTranspiler: # insert split_ids_op program.global_block().insert_op( - index=op_index, + index=lookup_table_op_index, type="split_ids", inputs={ 'Ids': [ @@ -649,7 +649,7 @@ class DistributeTranspiler: # insert prefetch_op program.global_block().insert_op( - index=op_index + 1, + index=lookup_table_op_index + 1, type="prefetch", inputs={'X': self.prefetch_input_vars}, outputs={"Out": self.prefetch_output_vars}, @@ -660,16 +660,21 @@ class DistributeTranspiler: # insert concat_op program.global_block().insert_op( - index=op_index + 2, - type="concat", - inputs={'X': self.prefetch_output_vars}, + index=lookup_table_op_index + 2, + type="merge_ids", + inputs={ + 'Ids': [ + program.global_block().vars[varname] + for varname in ids_name + ], + 'X': self.prefetch_output_vars + }, outputs={ "Out": [ program.global_block().vars[varname] for varname in out_name ] - }, - attrs={"axis": 0}) + }) # delete lookup_table_op delete_ops(program.global_block(), [op]) From 0485405b3d9d8dcf45139c63c370ca9124d26744 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 11 Jun 2018 11:04:02 +0800 Subject: [PATCH 011/154] add more debug string --- paddle/fluid/framework/executor.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index 3d68c5fb87..15af9c4090 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -317,8 +317,12 @@ void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope, } for (auto& op : ctx->ops_) { - VLOG(3) << place_ << " " << op->DebugStringEx(local_scope); + VLOG(4) << place_ << " " << op->DebugStringEx(local_scope); op->Run(*local_scope, place_); + // NOTE! Please do not delete this line, it's usefull because the debug + // string before and after op.run are different, after run the output + // will have right shape which is usefull for debug. + VLOG(3) << place_ << " " << op->DebugStringEx(local_scope); if (FLAGS_benchmark) { VLOG(2) << "Memory used after operator " + op->Type() + " running: " From e2433207d3575d46cd4a503d98b763252861e3a7 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Mon, 11 Jun 2018 14:44:23 +0800 Subject: [PATCH 012/154] update --- python/paddle/fluid/layers/nn.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 4485e91237..5f9b1db0f2 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1079,17 +1079,13 @@ def chunk_eval(input, Args: input(Variable): ${inference_comment} - label(Variable): ${label_comment} - chunk_scheme(${chunk_scheme_type}): ${chunk_scheme_comment} - num_chunk_types(${num_chunk_types_type}): ${num_chunk_types_comment} - excluded_chunk_types(${excluded_chunk_types_type}): ${excluded_chunk_types_comment} Returns: - chunk_eval(tuple): a tuple of variables: + tuple: a tuple of variables: (precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks) """ @@ -3900,13 +3896,9 @@ def roi_pool(input, rois, pooled_height=1, pooled_width=1, spatial_scale=1.0): Args: input (Variable): ${x_comment} - rois (Variable): ROIs (Regions of Interest) to pool over. - pooled_height (integer): ${pooled_height_comment} Default: 1 - pooled_width (integer): ${pooled_width_comment} Default: 1 - spatial_scale (float): ${spatial_scale_comment} Default: 1.0 Returns: From 451bc3a7bd4e4d6e9801d86b29a5097868ddb069 Mon Sep 17 00:00:00 2001 From: jshower Date: Mon, 11 Jun 2018 14:47:14 +0800 Subject: [PATCH 013/154] Correct the use of the lstm layer --- python/paddle/fluid/tests/book/test_label_semantic_roles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index bc8a1aafc8..bcdccabf00 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -76,7 +76,7 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, emb_layers.append(mark_embedding) hidden_0_layers = [ - fluid.layers.fc(input=emb, size=hidden_dim, act='tanh') + fluid.layers.fc(input=emb, size=hidden_dim) for emb in emb_layers ] @@ -94,8 +94,8 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, for i in range(1, depth): mix_hidden = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=hidden_dim, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=hidden_dim, act='tanh') + fluid.layers.fc(input=input_tmp[0], size=hidden_dim), + fluid.layers.fc(input=input_tmp[1], size=hidden_dim) ]) lstm = fluid.layers.dynamic_lstm( From 2955ff58871648a2cb151391ee82fc5ea570b8e6 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Mon, 11 Jun 2018 15:21:22 +0800 Subject: [PATCH 014/154] Polish documentation * row_conv * uniform_random * layer_norm * create_parameter * hard_shrink * ssd_loss --- paddle/fluid/operators/activation_op.cc | 13 ++--- paddle/fluid/operators/layer_norm_op.cc | 33 +++++------ paddle/fluid/operators/row_conv_op.cc | 20 ++++++- paddle/fluid/operators/uniform_random_op.cc | 20 +++---- python/paddle/fluid/layers/detection.py | 63 ++++++++++++--------- python/paddle/fluid/layers/nn.py | 63 ++++++--------------- python/paddle/fluid/layers/tensor.py | 15 ++++- 7 files changed, 115 insertions(+), 112 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 96e4c0e04c..7a567a83f9 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -276,13 +276,12 @@ class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( HardShrink Activation Operator. -$$ -out = \begin{cases} - x, \text{if } x > \lambda \\ - x, \text{if } x < -\lambda \\ - 0, \text{otherwise} - \end{cases} -$$ +.. math:: + out = \begin{cases} + x, \text{if } x > \lambda \\ + x, \text{if } x < -\lambda \\ + 0, \text{otherwise} + \end{cases} )DOC"); } diff --git a/paddle/fluid/operators/layer_norm_op.cc b/paddle/fluid/operators/layer_norm_op.cc index ab097d31e9..14ce1da2e9 100644 --- a/paddle/fluid/operators/layer_norm_op.cc +++ b/paddle/fluid/operators/layer_norm_op.cc @@ -62,36 +62,33 @@ class LayerNormOp : public framework::OperatorWithKernel { class LayerNormOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddInput("X", "(LoDTensor) The input tensor."); + AddInput("X", "The input tensor."); AddInput("Scale", - "(Tensor, optional) Scale is a 1-dimensional tensor of size " + "(optional) Scale is a 1-dimensional tensor of size " "H(`begin_norm_axis` splits the tensor(`X`) to a matrix [N,H])." "It is applied to the output.") .AsDispensable(); AddInput("Bias", - "(Tensor, optional) Bias is a 1-dimensional tensor of size " + "(optional) Bias is a 1-dimensional tensor of size " "H(`begin_norm_axis` splits the tensor(`X`) to a matrix [N,H])." "It is applied to the output.") .AsDispensable(); - AddOutput("Y", "(LoDTensor) Result after normalization."); - AddOutput("Mean", "(Tensor) Mean of the current mini batch.") - .AsIntermediate(); - AddOutput("Variance", "(Tensor) Variance of the current mini batch.") + AddOutput("Y", "Result after normalization."); + AddOutput("Mean", "Mean of the current mini batch.").AsIntermediate(); + AddOutput("Variance", "Variance of the current mini batch.") .AsIntermediate(); AddAttr("epsilon", - "(float, default 1e-5) Constant for " - "numerical stability") + "Constant for numerical stability [default 1e-5].") .SetDefault(1e-5) .AddCustomChecker([](const float &epsilon) { PADDLE_ENFORCE(epsilon >= 0.0f && epsilon <= 0.001f, "'epsilon' should be between 0.0 and 0.001."); }); AddAttr("begin_norm_axis", - "(int default:1), the " - "axis of `begin_norm_axis ... Rank(X) - 1` will be " + "the axis of `begin_norm_axis ... Rank(X) - 1` will be " "normalized. `begin_norm_axis` splits the tensor(`X`) to a " - "matrix [N,H].") + "matrix [N,H]. [default 1].") .SetDefault(1) .AddCustomChecker([](const int &begin_norm_axis) { PADDLE_ENFORCE_GT(begin_norm_axis, 0, @@ -99,10 +96,14 @@ class LayerNormOpMaker : public framework::OpProtoAndCheckerMaker { }); AddComment(R"DOC( -Layer Normalization. -Layer Norm has been implemented as discussed in the paper: -https://arxiv.org/abs/1607.06450 -... +Assume feature vectors exist on dimensions +:attr:`begin_norm_axis ... rank(input)` and calculate the moment statistics +along these dimensions for each feature vector :math:`a` with size +:math:`H`, then normalize each feature vector using the corresponding +statistics. After that, apply learnable gain and bias on the normalized +tensor to scale and shift if :attr:`scale` and :attr:`shift` are set. + +Refer to `Layer Normalization `_ )DOC"); } }; diff --git a/paddle/fluid/operators/row_conv_op.cc b/paddle/fluid/operators/row_conv_op.cc index 20f140f962..f4b540f1cb 100644 --- a/paddle/fluid/operators/row_conv_op.cc +++ b/paddle/fluid/operators/row_conv_op.cc @@ -78,18 +78,18 @@ class RowConvOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { AddInput("X", - "(LoDTensor), the input(X) is a LodTensor, which supports " + "the input(X) is a LodTensor, which supports " "variable time-length input sequences. The underlying tensor " "in this LoDTensor is a matrix with shape (T x N), where T " "is the total time steps in this mini-batch and N is the input " "data dimension."); AddInput("Filter", - "(Tensor), the input(Filter) is a learnable parameter. It " + "the input(Filter) is a learnable parameter. It " "is a 2-D tensor with shape (future_context x N), where, " "future_context is the future context length and N is the data " "dimension."); AddOutput("Out", - "(LoDTensor), the output(Out) is a LodTensor, which supports " + "the output(Out) is a LodTensor, which supports " "variable time-length input sequences. The underlying tensor " "in this LodTensor is a matrix with shape T x N, i.e., the " "same shape as X."); @@ -117,6 +117,20 @@ $$ out_{i, :} = \sum_{j=i}^{i + context} in_{j,:} \dot W_{i-j, :} $$ +In the above equation: + +* $Out_{i}$: The i-th row of output variable with shape [1, D]. + +* $\\tau$: Future context size. + +* $X_{j}$: The j-th row of input variable with shape [1, D]. + +* $W_{i-j}$: The (i-j)-th row of parameters with shape [1, D]. + +More details about row_conv please refer to +the design document +https://github.com/PaddlePaddle/Paddle/issues/2228#issuecomment-303903645 . + )DOC"); } }; diff --git a/paddle/fluid/operators/uniform_random_op.cc b/paddle/fluid/operators/uniform_random_op.cc index 137ea91cae..65525526c9 100644 --- a/paddle/fluid/operators/uniform_random_op.cc +++ b/paddle/fluid/operators/uniform_random_op.cc @@ -86,32 +86,26 @@ class UniformRandomOp : public framework::OperatorWithKernel { class UniformRandomOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddOutput("Out", "(Tensor) The output tensor of uniform random op"); + AddOutput("Out", "The output tensor of uniform random op"); AddComment(R"DOC( Uniform random operator. This operator initializes a tensor with random values sampled from a -uniform distribution. +uniform distribution. The random result is in set [min, max]. )DOC"); - AddAttr>("shape", - "(vector) The shape of the output tensor"); - AddAttr("min", - "(float, default -1.0) " - "Minimum value of uniform random") + AddAttr>("shape", "The shape of the output tensor"); + AddAttr("min", "Minimum value of uniform random. [default -1.0].") .SetDefault(-1.0f); - AddAttr("max", - "(float, default 1.0) " - "Maximun value of uniform random") + AddAttr("max", "Maximun value of uniform random. [default 1.0].") .SetDefault(1.0f); AddAttr("seed", - "(int, default 0) " "Random seed used for generating samples. " "0 means use a seed generated by the system." "Note that if seed is not 0, this operator will always " - "generate the same random numbers every time.") + "generate the same random numbers every time. [default 0].") .SetDefault(0); - AddAttr("dtype", "(int, default 5(FP32)) Output tensor data type") + AddAttr("dtype", "Output tensor data type. [default 5(FP32)].") .SetDefault(framework::proto::VarType::FP32); } }; diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 3a83db12fd..1e8dfbe521 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -373,22 +373,55 @@ def ssd_loss(location, confidence loss (or classification loss) by performing the following steps: 1. Find matched boundding box by bipartite matching algorithm. + 1.1 Compute IOU similarity between ground-truth boxes and prior boxes. + 1.2 Compute matched boundding box by bipartite matching algorithm. + 2. Compute confidence for mining hard examples + 2.1. Get the target label based on matched indices. + 2.2. Compute confidence loss. + 3. Apply hard example mining to get the negative example indices and update the matched indices. + 4. Assign classification and regression targets + 4.1. Encoded bbox according to the prior boxes. + 4.2. Assign regression targets. + 4.3. Assign classification targets. + 5. Compute the overall objective loss. + 5.1 Compute confidence loss. + 5.1 Compute localization loss. + 5.3 Compute the overall weighted loss. + >>> import paddle.fluid.layers as layers + >>> pb = layers.data( + >>> name='prior_box', + >>> shape=[10, 4], + >>> append_batch_size=False, + >>> dtype='float32') + >>> pbv = layers.data( + >>> name='prior_box_var', + >>> shape=[10, 4], + >>> append_batch_size=False, + >>> dtype='float32') + >>> loc = layers.data(name='target_box', shape=[10, 4], dtype='float32') + >>> scores = layers.data(name='scores', shape=[10, 21], dtype='float32') + >>> gt_box = layers.data( + >>> name='gt_box', shape=[4], lod_level=1, dtype='float32') + >>> gt_label = layers.data( + >>> name='gt_label', shape=[1], lod_level=1, dtype='float32') + >>> loss = layers.ssd_loss(loc, scores, gt_box, gt_label, pb, pbv) + Args: location (Variable): The location predictions are a 3D Tensor with shape [N, Np, 4], N is the batch size, Np is total number of @@ -426,34 +459,12 @@ def ssd_loss(location, mining_type is 'hard_example'. Returns: - Variable: The weighted sum of the localization loss and confidence loss, - with shape [N * Np, 1], N and Np are the same as they are - in `location`. + The weighted sum of the localization loss and confidence loss, with \ + shape [N * Np, 1], N and Np are the same as they are in `location`. Raises: - ValueError: If mining_type is 'hard_example', now only support - mining type of `max_negative`. - - Examples: - .. code-block:: python - - pb = layers.data( - name='prior_box', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - pbv = layers.data( - name='prior_box_var', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - loc = layers.data(name='target_box', shape=[10, 4], dtype='float32') - scores = layers.data(name='scores', shape=[10, 21], dtype='float32') - gt_box = layers.data( - name='gt_box', shape=[4], lod_level=1, dtype='float32') - gt_label = layers.data( - name='gt_label', shape=[1], lod_level=1, dtype='float32') - loss = layers.ssd_loss(loc, scores, gt_box, gt_label, pb, pbv) + ValueError: If mining_type is 'hard_example', now only support mining \ + type of `max_negative`. """ helper = LayerHelper('ssd_loss', **locals()) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index b9ea74fc81..ba13b344a1 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1624,6 +1624,7 @@ def batch_norm(input, return helper.append_activation(batch_norm_out) +@templatedoc() def layer_norm(input, scale=True, shift=True, @@ -1634,20 +1635,11 @@ def layer_norm(input, act=None, name=None): """ - **Layer Normalization** - - Assume feature vectors exist on dimensions - :attr:`begin_norm_axis ... rank(input)` and calculate the moment statistics - along these dimensions for each feature vector :math:`a` with size - :math:`H`, then normalize each feature vector using the corresponding - statistics. After that, apply learnable gain and bias on the normalized - tensor to scale and shift if :attr:`scale` and :attr:`shift` are set. - - Refer to `Layer Normalization `_ + ${comment} The formula is as follows: - .. math:: + .. math:: \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} a_i @@ -1655,6 +1647,11 @@ def layer_norm(input, h & = f(\\frac{g}{\\sigma}(a - \\mu) + b) + >>> import paddle.fluid as fluid + >>> data = fluid.layers.data(name='data', shape=[3, 32, 32], + >>> dtype='float32') + >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) + Args: input(Variable): The input tensor variable. scale(bool): Whether to learn the adaptive gain :math:`g` after @@ -1672,14 +1669,7 @@ def layer_norm(input, act(str): Activation to be applied to the output of layer normalizaiton. Returns: - Variable: A tensor variable with the same shape as the input. - - Examples: - .. code-block:: python - - data = fluid.layers.data( - name='data', shape=[3, 32, 32], dtype='float32') - x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) + ${y_comment} """ helper = LayerHelper('layer_norm', **locals()) dtype = helper.input_dtype() @@ -3184,29 +3174,19 @@ def im2sequence(input, filter_size=1, stride=1, padding=0, name=None): return out +@templatedoc() def row_conv(input, future_context_size, param_attr=None, act=None): - """Row Conv Operator. This layer will apply lookahead convolution to - **input**. The input variable should be a 2D LoDTensor with shape [T, D]. - Parameters with shape [future_context_size + 1, D] will be created. The math - equation of row convolution is as follows: - - .. math:: - Out_{i} = \sum_{j = i} ^ {i + \\tau} X_{j} \odot W_{i - j} - - In the above equation: + """ + ${comment} - * :math:`Out_{i}`: The i-th row of output variable with shape [1, D]. - * :math:`\\tau`: Future context size. - * :math:`X_{j}`: The j-th row of input variable with shape [1, D]. - * :math:`W_{i-j}`: The (i-j)-th row of parameters with shape [1, D]. + >>> import paddle.fluid as fluid + >>> x = fluid.layers.data(name='x', shape=[16], + >>> dtype='float32', lod_level=1) + >>> out = fluid.layers.row_conv(input=x, future_context_size=2) - More details about row_conv please refer to the paper \ - (http://www.cs.cmu.edu/~dyogatam/papers/wang+etal.iclrworkshop2016.pdf) and - the design document \ - (https://github.com/PaddlePaddle/Paddle/issues/2228#issuecomment-303903645). Args: - input (Variable): Input variable, a 2D LoDTensor with shape [T, D]. + input (${x_type}): ${x_comment}. future_context_size (int): Future context size. Please note, the shape of convolution kernel is [future_context_size + 1, D]. param_attr (ParamAttr): Attributes of parameters, including @@ -3214,14 +3194,7 @@ def row_conv(input, future_context_size, param_attr=None, act=None): act (str): Non-linear activation to be applied to output variable. Returns: - Variable: The output tensor with same shape as input tensor. - - Examples: - .. code-block:: python - - x = fluid.layers.data(name='x', shape=[16], - dtype='float32', lod_level=1) - out = fluid.layers.row_conv(input=x, future_context_size=2) + ${out_comment}. """ helper = LayerHelper('row_conv', **locals()) dtype = helper.input_dtype() diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 66db6fe13f..dbffcae86d 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -49,7 +49,18 @@ def create_parameter(shape, is_bias=False, default_initializer=None): """ - Create a parameter + Create a parameter. The parameter is a learnable variable, which can have + gradient, and can be optimized. + + NOTE: this is a very low-level API. This API is useful when you create + operator by your self. instead of using layers. + + >>> import paddle.fluid as fluid + >>> W = fluid.layers.create_parameter(shape=[784, 200], dtype='float32') + >>> data = fluid.layers.data(name="img", shape=[64, 784], + >>> append_batch_size=False) + >>> hidden = fluid.layers.matmul(x=data, y=W) + Args: shape(list[int]): shape of the parameter dtype(string): element type of the parameter @@ -61,7 +72,7 @@ def create_parameter(shape, default_initializer(Initializer): initializer for the parameter Returns: - Parameter: the created parameter + the created parameter """ helper = LayerHelper("create_parameter", **locals()) if attr is None: From 22f80cc536ef9b9545135bd4de17664bd3f2463e Mon Sep 17 00:00:00 2001 From: jshower Date: Mon, 11 Jun 2018 20:06:07 +0800 Subject: [PATCH 015/154] code sytle --- python/paddle/fluid/tests/book/test_label_semantic_roles.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index bcdccabf00..99d51ae007 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -76,8 +76,7 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, emb_layers.append(mark_embedding) hidden_0_layers = [ - fluid.layers.fc(input=emb, size=hidden_dim) - for emb in emb_layers + fluid.layers.fc(input=emb, size=hidden_dim) for emb in emb_layers ] hidden_0 = fluid.layers.sums(input=hidden_0_layers) From 8e19c324ab82feeea87cd582737ebae5bec95fb8 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 12:47:11 +0800 Subject: [PATCH 016/154] update split_lod_tensor, create_array and array_length doc --- python/paddle/fluid/layers/control_flow.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..114c1f0ed4 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -62,6 +62,8 @@ def split_lod_tensor(input, mask, level=0): The output is the true branch and the false branch with the mask applied to the input at a certain level in the tensor. + Mainly used in IfElse to split data into two parts. Related API: IfElse. + Args: input(tuple|list|None): The input tensor that contains complete lod information needed to construct the output. @@ -83,6 +85,7 @@ def split_lod_tensor(input, mask, level=0): out_true, out_false = layers.split_lod_tensor( input=x, mask=y, level=level) + """ helper = LayerHelper('split_lod_tensor', **locals()) out_true = helper.create_tmp_variable(dtype=input.dtype) @@ -887,14 +890,18 @@ def array_write(x, i, array=None): def create_array(dtype): - """This function creates an array of type :math:`LOD_TENSOR_ARRAY` using the - LayerHelper. + """ + **Create LoDTensor Array** + + This function creates an array of type :math:`LOD_TENSOR_ARRAY` using the + LayerHelper. It is mainly used to implement RNN with array_write, array_read + and While. Args: dtype (int|float): The data type of the elements in the array. Returns: - Variable: The tensor variable storing the elements of data type. + Variable: The lod_tensor_array variable storing the elements of data type. Examples: .. code-block:: python @@ -1020,9 +1027,14 @@ def shrink_memory(x, i, table): def array_length(array): - """This function performs the operation to find the length of the input + """ + **Get the length of Input LoDTensorArray** + + This function performs the operation to find the length of the input LOD_TENSOR_ARRAY. + Related API: array_read, array_write, While. + Args: array (LOD_TENSOR_ARRAY): The input array that will be used to compute the length. From 2c1e2caa7d8c225040fdc3674df72afd7313f219 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 13:35:00 +0800 Subject: [PATCH 017/154] update document --- python/paddle/fluid/layers/control_flow.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 114c1f0ed4..15f294698c 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -60,15 +60,14 @@ def split_lod_tensor(input, mask, level=0): This function takes in an input that contains the complete lod information, and takes in a mask which is used to mask certain parts of the input. The output is the true branch and the false branch with the mask applied to - the input at a certain level in the tensor. - - Mainly used in IfElse to split data into two parts. Related API: IfElse. + the input at a certain level in the tensor. Mainly used in IfElse to split + data into two parts. Args: input(tuple|list|None): The input tensor that contains complete lod information needed to construct the output. mask(list): A bool column vector which masks the input. - level(int): The specific lod level to rank. + level(int): The specific lod level to split. Returns: Variable: The true branch of tensor as per the mask applied to input. @@ -108,8 +107,9 @@ def merge_lod_tensor(in_true, in_false, x, mask, level=0): This function takes in an input :math:`x`, the True branch, the False branch and a binary :math:`mask`. Using this information, this function - merges the True and False branches of the tensor into a single Output - at a certain lod level indiacted by :math:`level`. + merges the True and False branches of the tensor into a single tensor as + output at a certain lod level indicated by :math:`level`. Used in IfElse + to merge the output if True block and False Block. Args: in_true(tuple|list|None): The True branch to be merged. @@ -117,7 +117,7 @@ def merge_lod_tensor(in_true, in_false, x, mask, level=0): x(tuple|list|None): The input tensor that contains complete lod information needed to construct the output. mask(list): A bool column vector which masks the input. - level(int): The specific lod level to rank. + level(int): The specific lod level to merge. Returns: Variable: The merged output tensor. From 4d0fd7e725b259509896f6d891965feec7effee8 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 14:50:56 +0800 Subject: [PATCH 018/154] add API reference for create_tensor --- python/paddle/fluid/layers/tensor.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 62b01d595a..6ce486d70d 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -39,6 +39,25 @@ __all__ = [ def create_tensor(dtype, name=None, persistable=False): + """ + **Create a Tensor with certain data type and name** + + Args: + dtype (string): 'float32'|'int32'|..., the data type of the + created tensor. + name (string|None): The name of the created tensor, if not set, + the name will be a random unique one. + persistable (bool): Set the persistable flag of the create tensor, + default value is False. + + Returns: + Variable: The tensor variable storing the created tensor. + + Examples: + .. code-block:: python + + tensor = fluid.layers.create_tensor(dtype='float32') + """ helper = LayerHelper("create_tensor", **locals()) return helper.create_variable( name=helper.name, dtype=dtype, persistable=persistable) From 7b54b30be5c4f22f7a96d068f75c04ac04521057 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Tue, 12 Jun 2018 16:08:26 +0800 Subject: [PATCH 019/154] follow comments --- paddle/fluid/operators/linear_chain_crf_op.cc | 2 + paddle/fluid/operators/lstm_op.cc | 28 +++---- python/paddle/fluid/layers/nn.py | 79 ++++--------------- 3 files changed, 30 insertions(+), 79 deletions(-) diff --git a/paddle/fluid/operators/linear_chain_crf_op.cc b/paddle/fluid/operators/linear_chain_crf_op.cc index a711da3627..ea1ca7f59d 100644 --- a/paddle/fluid/operators/linear_chain_crf_op.cc +++ b/paddle/fluid/operators/linear_chain_crf_op.cc @@ -84,6 +84,7 @@ CRF. Please refer to http://www.cs.columbia.edu/~mcollins/fb.pdf and http://cseweb.ucsd.edu/~elkan/250Bwinter2012/loglinearCRFs.pdf for details. Equation: + 1. Denote Input(Emission) to this operator as $x$ here. 2. The first D values of Input(Transition) to this operator are for starting weights, denoted as $a$ here. @@ -106,6 +107,7 @@ Finally, the linear chain CRF operator outputs the logarithm of the conditional likelihood of each training sample in a mini-batch. NOTE: + 1. The feature function for a CRF is made up of the emission features and the transition features. The emission feature weights are NOT computed in this operator. They MUST be computed first before this operator is called. diff --git a/paddle/fluid/operators/lstm_op.cc b/paddle/fluid/operators/lstm_op.cc index 4751e3e802..29cec6ae10 100644 --- a/paddle/fluid/operators/lstm_op.cc +++ b/paddle/fluid/operators/lstm_op.cc @@ -198,20 +198,20 @@ c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\ h_t = o_t \odot act_h(c_t) $$ -where the W terms denote weight matrices (e.g. $W_{xi}$ is the matrix -of weights from the input gate to the input), $W_{ic}, W_{fc}, W_{oc}$ -are diagonal weight matrices for peephole connections. In our implementation, -we use vectors to reprenset these diagonal weight matrices. The b terms -denote bias vectors ($b_i$ is the input gate bias vector), $\sigma$ -is the non-line activations, such as logistic sigmoid function, and -$i, f, o$ and $c$ are the input gate, forget gate, output gate, -and cell activation vectors, respectively, all of which have the same size as -the cell output activation vector $h$. - -The $\odot$ is the element-wise product of the vectors. $act_g$ and $act_h$ -are the cell input and cell output activation functions and `tanh` is usually -used for them. $\tilde{c_t}$ is also called candidate hidden state, -which is computed based on the current input and the previous hidden state. +- W terms denote weight matrices (e.g. $W_{xi}$ is the matrix + of weights from the input gate to the input), $W_{ic}, W_{fc}, W_{oc}$ + are diagonal weight matrices for peephole connections. In our implementation, + we use vectors to reprenset these diagonal weight matrices. +- The b terms denote bias vectors ($b_i$ is the input gate bias vector). +- $\sigma$ is the non-line activations, such as logistic sigmoid function. +- $i, f, o$ and $c$ are the input gate, forget gate, output gate, + and cell activation vectors, respectively, all of which have the same size as + the cell output activation vector $h$. +- The $\odot$ is the element-wise product of the vectors. +- $act_g$ and $act_h$ are the cell input and cell output activation functions + and `tanh` is usually used for them. +- $\tilde{c_t}$ is also called candidate hidden state, + which is computed based on the current input and the previous hidden state. Set `use_peepholes` False to disable peephole connection. The formula is omitted here, please refer to the paper diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 56d25e18f4..40134ed42f 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -262,6 +262,7 @@ def embedding(input, # TODO(qijun): expose H0 and C0 +@templatedoc(op_type="lstm") def dynamic_lstm(input, size, param_attr=None, @@ -274,64 +275,19 @@ def dynamic_lstm(input, dtype='float32', name=None): """ - **Dynamic LSTM Layer** - - The defalut implementation is diagonal/peephole connection - (https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: - - .. math:: - - i_t & = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) - - f_t & = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) - - \\tilde{c_t} & = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) - - o_t & = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) - - c_t & = f_t \odot c_{t-1} + i_t \odot \\tilde{c_t} - - h_t & = o_t \odot act_h(c_t) - - where the :math:`W` terms denote weight matrices (e.g. :math:`W_{xi}` is - the matrix of weights from the input gate to the input), :math:`W_{ic}, \ - W_{fc}, W_{oc}` are diagonal weight matrices for peephole connections. In - our implementation, we use vectors to reprenset these diagonal weight - matrices. The :math:`b` terms denote bias vectors (:math:`b_i` is the input - gate bias vector), :math:`\sigma` is the non-linear activations, such as - logistic sigmoid function, and :math:`i, f, o` and :math:`c` are the input - gate, forget gate, output gate, and cell activation vectors, respectively, - all of which have the same size as the cell output activation vector :math:`h`. - - The :math:`\odot` is the element-wise product of the vectors. :math:`act_g` - and :math:`act_h` are the cell input and cell output activation functions - and `tanh` is usually used for them. :math:`\\tilde{c_t}` is also called - candidate hidden state, which is computed based on the current input and - the previous hidden state. - - Set `use_peepholes` to `False` to disable peephole connection. The formula - is omitted here, please refer to the paper - http://www.bioinf.jku.at/publications/older/2604.pdf for details. - - Note that these :math:`W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}` - operations on the input :math:`x_{t}` are NOT included in this operator. - Users can choose to use fully-connect layer before LSTM layer. + ${comment} Args: - input(Variable): The input of dynamic_lstm layer, which supports - variable-time length input sequence. The underlying - tensor in this Variable is a matrix with shape - (T X 4D), where T is the total time steps in this - mini-batch, D is the hidden size. - size(int): 4 * hidden size. - param_attr(ParamAttr|None): The parameter attribute for the learnable + input (Variable): ${input_comment} + size (int): 4 * hidden size. + param_attr (ParamAttr|None): The parameter attribute for the learnable hidden-hidden weights. - Weights = {:math:`W_{ch}, W_{ih}, \ W_{fh}, W_{oh}`} - The shape is (D x 4D), where D is the hidden size. - bias_attr(ParamAttr|None): The bias attribute for the learnable bias + bias_attr (ParamAttr|None): The bias attribute for the learnable bias weights, which contains two parts, input-hidden bias weights and peephole connections weights if setting `use_peepholes` to `True`. @@ -343,21 +299,14 @@ def dynamic_lstm(input, - Biases = { :math:`b_c, b_i, b_f, b_o, W_{ic}, \ W_{fc}, W_{oc}`}. - The shape is (1 x 7D). - use_peepholes(bool): Whether to enable diagonal/peephole connections, - default `True`. - is_reverse(bool): Whether to compute reversed LSTM, default `False`. - gate_activation(str): The activation for input gate, forget gate and - output gate. Choices = ["sigmoid", "tanh", "relu", - "identity"], default "sigmoid". - cell_activation(str): The activation for cell output. Choices = ["sigmoid", - "tanh", "relu", "identity"], default "tanh". - candidate_activation(str): The activation for candidate hidden state. - Choices = ["sigmoid", "tanh", - "relu", "identity"], - default "tanh". - dtype(str): Data type. Choices = ["float32", "float64"], default "float32". - name(str|None): A name for this layer(optional). If set None, the layer - will be named automatically. + use_peepholes (bool): ${use_peepholes_comment} + is_reverse (bool): ${is_reverse_comment} + gate_activation (str): ${gate_activation_comment} + cell_activation (str): ${cell_activation_comment} + candidate_activation (str): ${candidate_activation_comment} + dtype (str): Data type. Choices = ["float32", "float64"], default "float32". + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: tuple: The hidden state, and cell state of LSTM. The shape of both \ From 6a494380e826053609dbae47241b81a774f1ed30 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 12 Jun 2018 16:12:48 +0800 Subject: [PATCH 020/154] remove mkldnn flag from gtest strdup --- paddle/testing/paddle_gtest_main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/testing/paddle_gtest_main.cc b/paddle/testing/paddle_gtest_main.cc index 507479c862..586ec48477 100644 --- a/paddle/testing/paddle_gtest_main.cc +++ b/paddle/testing/paddle_gtest_main.cc @@ -30,7 +30,7 @@ int main(int argc, char** argv) { new_argv.push_back( strdup("--tryfromenv=fraction_of_gpu_memory_to_use,use_pinned_memory")); #else - new_argv.push_back(strdup("--tryfromenv=use_pinned_memory,use_mkldnn")); + new_argv.push_back(strdup("--tryfromenv=use_pinned_memory")); #endif int new_argc = static_cast(new_argv.size()); char** new_argv_address = new_argv.data(); From 6a32f19865ac7d1a7b439b609b757f6aa08baceb Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 12 Jun 2018 16:23:33 +0800 Subject: [PATCH 021/154] fix unknown use_mkldnn --- paddle/fluid/inference/tests/book/test_inference_nlp.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/inference/tests/book/test_inference_nlp.cc b/paddle/fluid/inference/tests/book/test_inference_nlp.cc index 9dcd79c3bb..4dc576cede 100644 --- a/paddle/fluid/inference/tests/book/test_inference_nlp.cc +++ b/paddle/fluid/inference/tests/book/test_inference_nlp.cc @@ -29,6 +29,7 @@ DEFINE_string(data_file, "", "File of input index data."); DEFINE_int32(repeat, 100, "Running the inference program repeat times"); DEFINE_bool(prepare_vars, true, "Prepare variables before executor"); DEFINE_int32(num_threads, 1, "Number of threads should be used"); +DECLARE_bool(use_mkldnn); inline double GetCurrentMs() { struct timeval time; From 6602db5b3e43ef7b5b21cce0d62ef580815206fc Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 12 Jun 2018 17:09:16 +0800 Subject: [PATCH 022/154] throw warning if try to use mkldnn while not compiled --- paddle/fluid/framework/executor.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index 4a6f53cba1..571019be36 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -402,6 +402,9 @@ void Executor::EnableMKLDNN(const ProgramDesc& program) { } } } +#else + LOG(WARNING) + << "'MKLDNN' is not supported, Please re-compile with WITH_MKLDNN option"; #endif } From ff55d4c5937af9263d3dab6bfd9ee0d9b460a15b Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Tue, 12 Jun 2018 17:54:10 +0800 Subject: [PATCH 023/154] Polish documents * less_than * cumsum * multiplex * open_recordio_file --- paddle/fluid/operators/compare_op.cc | 23 +++++----- paddle/fluid/operators/cumsum_op.cc | 10 ++--- paddle/fluid/operators/multiplex_op.cc | 42 +++++++++++++------ .../reader/create_recordio_file_reader_op.cc | 10 +++-- .../operators/reader/reader_op_registry.cc | 2 +- python/paddle/fluid/layers/control_flow.py | 29 +++++++------ python/paddle/fluid/layers/io.py | 31 ++++++-------- python/paddle/fluid/layers/nn.py | 41 +++++------------- 8 files changed, 93 insertions(+), 95 deletions(-) diff --git a/paddle/fluid/operators/compare_op.cc b/paddle/fluid/operators/compare_op.cc index 3a4819f3de..11e91c5ec8 100644 --- a/paddle/fluid/operators/compare_op.cc +++ b/paddle/fluid/operators/compare_op.cc @@ -23,25 +23,22 @@ class CompareOpProtoMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { OpComment comment; - AddInput("X", - string::Sprintf("(LoDTensor) the left hand operand of %s operator", - comment.type)); - AddInput("Y", string::Sprintf( - "(LoDTensor) the right hand operand of %s operator", - comment.type)); + AddInput("X", string::Sprintf("the left hand operand of %s operator", + comment.type)); + AddInput("Y", string::Sprintf("the right hand operand of %s operator", + comment.type)); AddAttr("force_cpu", - "(bool, default false) Force fill output variable to cpu " + "Force fill output variable to cpu " "memory. Otherwise, fill output variable to the running " - "device") - .SetDefault(false); - AddOutput("Out", string::Sprintf( - "(LoDTensor) n-dim bool tensor. Each element is %s", - comment.equation)); + "device [default true].") + .SetDefault(true); + AddOutput("Out", string::Sprintf("n-dim bool tensor. Each element is %s", + comment.equation)); AddComment(string::Sprintf(R"DOC(%s Operator It operates element-wise on X and Y, and returns the Out. Each of them is a N-dim tensor. X and Y could be any type. The each element of the Out tensor is -calculated by %s +calculated by $%s$ )DOC", comment.type, comment.equation)); AddAttr("axis", diff --git a/paddle/fluid/operators/cumsum_op.cc b/paddle/fluid/operators/cumsum_op.cc index 92bb835e8f..2caa8bf2d5 100644 --- a/paddle/fluid/operators/cumsum_op.cc +++ b/paddle/fluid/operators/cumsum_op.cc @@ -33,16 +33,16 @@ class CumsumOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "Input of Cumsum operator"); AddOutput("Out", "Output of Cumsum operator"); AddAttr("axis", - "(int, default -1). The dimenstion to accumulate along. " - "-1 means the last dimenstion") + "The dimenstion to accumulate along. -1 means the last " + "dimenstion [default -1].") .SetDefault(-1) .EqualGreaterThan(-1); AddAttr("exclusive", - "bool, default false). Whether to perform exclusive cumsum") + "Whether to perform exclusive cumsum. [default false].") .SetDefault(false); AddAttr("reverse", - "bool, default false). If true, the cumsum is performed in " - "the reversed direction") + "If true, the cumsum is performed in the reversed direction. " + "[default false].") .SetDefault(false); AddComment(R"DOC( The cumulative sum of the elements along a given axis. diff --git a/paddle/fluid/operators/multiplex_op.cc b/paddle/fluid/operators/multiplex_op.cc index a4363fd25d..9db2df2a4c 100644 --- a/paddle/fluid/operators/multiplex_op.cc +++ b/paddle/fluid/operators/multiplex_op.cc @@ -62,26 +62,44 @@ class MultiplexOp : public framework::OperatorWithKernel { class MultiplexOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddInput("Ids", "The index tensor of multiplex operator."); - AddInput("X", "The candidate tensors of multiplex operator.") + AddInput("Ids", + "Tensor, index variable which is a 2-D tensor with shape " + "[M, 1] where M is the batch size."); + AddInput("X", + "A list of variables to gather from. All variables have the same " + "shape and the rank is at least 2.") .AsDuplicable(); AddOutput("Out", "The output tensor of multiplex operator."); AddComment(R"DOC( -Multiplex Operator. - -Multiplex multiple tensors according to the index provided by the index tensor. - -Ids: the index tensor. -X[0 : N - 1]: the candidate tensors for output (N >= 2). -For each index i from 0 to batchSize - 1, the output is the i-th row of the +Referring to the given index variable, this layer selects rows from the +input variables to construct a multiplex variable. Assuming that there are +:math:`m` input variables and :math:`I_i` represents the i-th input +variable and :math:`i` is in [0, :math:`m`). All input variables are +tensors with same shape [:math:`d_0`, :math:`d_1`, ..., :math:`d_R`]. +Please note that rank of the input tensor should be at least 2. Each input +variable will be treated as a 2-D matrix with shape [:math:`M`, :math:`N`] +where :math:`M` for :math:`d_0` and :math:`N` for :math:`d_1` * :math:`d_2` +* ... * :math:`d_R`. Let :math:`I_i[j]` be the j-th row of the i-th input +variable. The given index variable should be a 2-D tensor with shape +[:math:`M`, 1]. Let `ID[i]` be the i-th index value of the index variable. +Then the output variable will be a tensor with shape [:math:`d_0`, +:math:`d_1`, ..., :math:`d_R`]. If we treat the output tensor as a 2-D +matrix with shape [:math:`M`, :math:`N`] and let :math:`O[i]` be the i-th +row of the matrix, then `O[i]` is equal to :math:`I_{ID[i]}[i]`. + +* Ids: the index tensor. + +* X[0 : N - 1]: the candidate tensors for output (N >= 2). + +* For each index i from 0 to batchSize - 1, the output is the i-th row of the the (Ids[i])-th tensor. For i-th row of the output tensor: -$$y[i] = x_{k}[i]$$ +$ y[i] = x_{k}[i] $ -where `y` is the output tensor, `x_{k}` is the k-th input tensor, -and `k = Ids[i]`. +where $y$ is the output tensor, $x_{k}$ is the k-th input tensor, +and $k = Ids[i]$. )DOC"); } diff --git a/paddle/fluid/operators/reader/create_recordio_file_reader_op.cc b/paddle/fluid/operators/reader/create_recordio_file_reader_op.cc index 282ec3f36b..559827f084 100644 --- a/paddle/fluid/operators/reader/create_recordio_file_reader_op.cc +++ b/paddle/fluid/operators/reader/create_recordio_file_reader_op.cc @@ -78,11 +78,15 @@ class CreateRecordIOReaderOp : public framework::OperatorBase { class CreateRecordIOReaderOpMaker : public FileReaderMakerBase { protected: void Apply() override { - AddAttr("filename", "The filename of record io reader"); + AddAttr( + "filename", + "The filename of record file. This file will given to reader."); AddComment(R"DOC( - CreateRecordIOReader Operator +Open a recordio file and return the reader object. The returned reader object +is thread-safe. - Create a reader from a record io file +NOTE: This is a very low-level API. It is used for debugging data file or +training. Please use `open_files` instead of this API for production usage. )DOC"); } }; diff --git a/paddle/fluid/operators/reader/reader_op_registry.cc b/paddle/fluid/operators/reader/reader_op_registry.cc index 612e1f5eca..e11256a49f 100644 --- a/paddle/fluid/operators/reader/reader_op_registry.cc +++ b/paddle/fluid/operators/reader/reader_op_registry.cc @@ -54,7 +54,7 @@ std::unique_ptr CreateReaderByFileName( } void FileReaderMakerBase::Make() { - AddOutput("Out", "(ReaderHolder) The created random reader.").AsDuplicable(); + AddOutput("Out", "(ReaderHolder): The created random reader.").AsDuplicable(); AddAttr>("shape_concat", "The concat of all data's shapes."); AddAttr>( "ranks", diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..6c707a35d3 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -909,37 +909,40 @@ def create_array(dtype): dtype=dtype) -def less_than(x, y, force_cpu=True, cond=None, **ignored): +@templatedoc() +def less_than(x, y, force_cpu=None, cond=None, **ignored): """ - **Less than** + ${comment} - This layer returns the truth value of :math:`x < y` elementwise. + >>> import paddle.fluid as fluid + >>> less = fluid.layers.less_than(x=label, y=limit) Args: - x(Variable): First operand of *less_than* - y(Variable): Second operand of *less_than* - force_cpu(Bool|True): The output data will be on CPU if set true. + x(${x_type}): ${x_comment}. + y(${y_type}): ${y_comment}. + force_cpu(${force_cpu_type}): ${force_cpu_comment}. cond(Variable|None): Optional output variable to store the result of *less_than* Returns: - Variable: The tensor variable storing the output of *less_than*. - - Examples: - .. code-block:: python - - less = fluid.layers.less_than(x=label, y=limit) + ${out_comment}. """ helper = LayerHelper("less_than", **locals()) if cond is None: cond = helper.create_tmp_variable(dtype='bool') cond.stop_gradient = True + attrs = dict() + if force_cpu is not None: + attrs['force_cpu'] = force_cpu + elif force_init_on_cpu(): + attrs['force_cpu'] = force_init_on_cpu() + helper.append_op( type='less_than', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [cond]}, - attrs={'force_cpu': force_cpu or force_init_on_cpu()}) + attrs=attrs) return cond diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 9de88e2c32..13a3d5441a 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -292,6 +292,7 @@ def _copy_reader_create_op_(block, op): return new_op +@templatedoc(op_type='create_recordio_file_reader') def open_recordio_file(filename, shapes, lod_levels, @@ -299,34 +300,28 @@ def open_recordio_file(filename, pass_num=1, for_parallel=True): """ - Open a RecordIO file + ${comment} - This layer takes a RecordIO file to read from and returns a Reader Variable. - Via the Reader Variable, we can get data from the given RecordIO file. + >>> import paddle.fluid as fluid + >>> reader = fluid.layers.io.open_recordio_file( + >>> filename='./data.recordio', + >>> shapes=[(3,224,224), (1)], + >>> lod_levels=[0, 0], + >>> dtypes=['float32', 'int64']) + >>> # Via the reader, we can use 'read_file' layer to get data: + >>> image, label = fluid.layers.io.read_file(reader) Args: - filename(str): The RecordIO file's name. + filename(${filename_type}): ${filename_comment}. shapes(list): List of tuples which declaring data shapes. - lod_levels(list): List of ints which declaring data lod_level. + lod_levels(${lod_levels_type}): ${lod_levels_comment}. dtypes(list): List of strs which declaring data type. pass_num(int): Number of passes to run. for_parallel(Bool): Set it as True if you are going to run subsequent operators in parallel. Returns: - Variable: A Reader Variable via which we can get RecordIO file data. - - Examples: - .. code-block:: python - - reader = fluid.layers.io.open_recordio_file( - filename='./data.recordio', - shapes=[(3,224,224), (1)], - lod_levels=[0, 0], - dtypes=['float32', 'int64']) - - # Via the reader, we can use 'read_file' layer to get data: - image, label = fluid.layers.io.read_file(reader) + ${out_comment}. """ dtypes = [convert_np_dtype_to_dtype_(dt) for dt in dtypes] shape_concat = [] diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index ba13b344a1..1c5322288a 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3210,42 +3210,23 @@ def row_conv(input, future_context_size, param_attr=None, act=None): return helper.append_activation(out) +@templatedoc() def multiplex(inputs, index): """ - **Multiplex Layer** - - Referring to the given index variable, this layer selects rows from the - input variables to construct a multiplex variable. Assuming that there are - :math:`m` input variables and :math:`I_i` represents the i-th input - variable and :math:`i` is in [0, :math:`m`). All input variables are - tensors with same shape [:math:`d_0`, :math:`d_1`, ..., :math:`d_R`]. - Please note that rank of the input tensor should be at least 2. Each input - variable will be treated as a 2-D matrix with shape [:math:`M`, :math:`N`] - where :math:`M` for :math:`d_0` and :math:`N` for :math:`d_1` * :math:`d_2` - * ... * :math:`d_R`. Let :math:`I_i[j]` be the j-th row of the i-th input - variable. The given index variable should be a 2-D tensor with shape - [:math:`M`, 1]. Let `ID[i]` be the i-th index value of the index variable. - Then the output variable will be a tensor with shape [:math:`d_0`, - :math:`d_1`, ..., :math:`d_R`]. If we treat the output tensor as a 2-D - matrix with shape [:math:`M`, :math:`N`] and let :math:`O[i]` be the i-th - row of the matrix, then `O[i]` is equal to :math:`I_{ID[i]}[i]`. + ${comment} + + >>> import paddle.fluid as fluid + >>> x1 = fluid.layers.data(name='x1', shape=[4], dtype='float32') + >>> x2 = fluid.layers.data(name='x2', shape=[4], dtype='float32') + >>> index = fluid.layers.data(name='index', shape=[1], dtype='int32') + >>> out = fluid.layers.multiplex(inputs=[x1, x2], index=index) Args: - inputs (list): A list of variables to gather from. All variables have the - same shape and the rank is at least 2. - index (Variable): Tensor, index variable which is a 2-D tensor - with shape [M, 1] where M is the batch size. + inputs (list): ${x_comment}. + index (${ids_type}): ${ids_comment}. Returns: - Variable: Multiplex variable gathered from input variables. - - Examples: - .. code-block:: python - - x1 = fluid.layers.data(name='x1', shape=[4], dtype='float32') - x2 = fluid.layers.data(name='x2', shape=[4], dtype='float32') - index = fluid.layers.data(name='index', shape=[1], dtype='int32') - out = fluid.layers.multiplex(inputs=[x1, x2], index=index) + ${out_comment}. """ helper = LayerHelper('multiplex', **locals()) From c34945800e8d68fddd681595dc12ea4b541b1342 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 12 Jun 2018 19:32:36 +0800 Subject: [PATCH 024/154] change drop_last defalut value --- python/paddle/batch.py | 2 +- python/paddle/v2/minibatch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/batch.py b/python/paddle/batch.py index d48c54fcbb..3c6a53db3c 100644 --- a/python/paddle/batch.py +++ b/python/paddle/batch.py @@ -15,7 +15,7 @@ __all__ = ['batch'] -def batch(reader, batch_size, drop_last=False): +def batch(reader, batch_size, drop_last=True): """ Create a batched reader. diff --git a/python/paddle/v2/minibatch.py b/python/paddle/v2/minibatch.py index d48c54fcbb..3c6a53db3c 100644 --- a/python/paddle/v2/minibatch.py +++ b/python/paddle/v2/minibatch.py @@ -15,7 +15,7 @@ __all__ = ['batch'] -def batch(reader, batch_size, drop_last=False): +def batch(reader, batch_size, drop_last=True): """ Create a batched reader. From d82422997a7c21a9d440fae18c6c60c84a5bff7a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 20:07:17 +0800 Subject: [PATCH 025/154] add doc for batch norm --- python/paddle/fluid/layers/nn.py | 51 ++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 9e2c06d26f..6719a4d7ec 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1541,8 +1541,55 @@ def batch_norm(input, moving_variance_name=None, do_model_average_for_mean_and_var=False): """ - This function helps create an operator to implement - the BatchNorm layer using the configurations from the input parameters. + **Batch Normalization Layer** + + Can be used as a normalizer function for conv2d and fully_connected operations. + The required data format for this layer is one of the following: + 1. NHWC `[batch, in_height, in_width, in_channels]` + 2. NCHW `[batch, in_channels, in_height, in_width]` + + Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift + `_ for more details. + + :math:`input` is the input features over a mini-batch. + + .. math:: + + \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\ + \ mini-batch\ mean \\\\ + \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\ + \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\ + \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\ + \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\ + y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift + + Args: + input(variable): The input variable which is a LoDTensor. + act(string, default None): Activation type, linear|relu|prelu|... + is_test(bool, default False): Used for training or training. + momentum(float, default 0.9): + epsilon(float, default 1e-05): + param_attr(ParamAttr): The parameter attribute for Parameter `scale`. + bias_attr(ParamAttr): The parameter attribute for Parameter `bias`. + data_layout(string, default NCHW): NCHW|NHWC + in_place(bool, default False): Make the input and output of batch norm reuse memory. + use_mkldnn(bool, Default false): ${use_mkldnn_comment} + name(string, Default None): A name for this layer(optional). If set None, the layer + will be named automatically. + moving_mean_name(string, Default None): The name of moving_mean which store the global Mean. + moving_variance_name(string, Default None): The name of the moving_variance which store the global Variance. + do_model_average_for_mean_and_var(bool, Default False): + + Returns: + The sequence's last step variable which is a Tensor. + + Examples: + + .. code-block:: python + + hidden1 = fluid.layers.fc(input=x, size=200, param_attr='fc1.w') + hidden2 = fluid.layers.batch_norm(input=hidden1) + """ helper = LayerHelper('batch_norm', **locals()) dtype = helper.input_dtype() From f3e631cd9e59741f3d2531706080e3a94c979f35 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 20:21:18 +0800 Subject: [PATCH 026/154] small update --- python/paddle/fluid/layers/nn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 1a010ab3ac..d5db75ebea 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4078,7 +4078,7 @@ def image_resize(input, name=None, resample='BILINEAR'): """ - Resize a batch of images. + **Resize a batch of images** The input must be a tensor of the shape (num_batches, channels, in_h, in_w), and the resizing only applies on the last two dimensions(hight and width). @@ -4208,6 +4208,8 @@ def image_resize_short(input, out_short_len, resample='BILINEAR'): def gather(input, index): """ + **Gather Layer** + Output is obtained by gathering entries of the outer-most dimension of X indexed by `index` and concatenate them together. From e72eb0edec589ce6bee07ec5eb34ee7ceca2af33 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 20:23:47 +0800 Subject: [PATCH 027/154] small update --- paddle/fluid/operators/detection/polygon_box_transform_op.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/operators/detection/polygon_box_transform_op.cc b/paddle/fluid/operators/detection/polygon_box_transform_op.cc index 335e8dd470..568d50d457 100644 --- a/paddle/fluid/operators/detection/polygon_box_transform_op.cc +++ b/paddle/fluid/operators/detection/polygon_box_transform_op.cc @@ -83,11 +83,13 @@ class PolygonBoxTransformOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( PolygonBoxTransform Operator. + +PolygonBoxTransform Operator is used to transform the coordinate shift to the real coordinate. + The input is the final geometry output in detection network. We use 2*n numbers to denote the coordinate shift from n corner vertices of the polygon_box to the pixel location. As each distance offset contains two numbers (xi, yi), the geometry output contains 2*n channels. -PolygonBoxTransform Operator is used to transform the coordinate shift to the real coordinate. )DOC"); } }; From dde0a28073420134d4aae01d91511ead3d0c362a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 20:51:56 +0800 Subject: [PATCH 028/154] add doc for Switch --- python/paddle/fluid/layers/control_flow.py | 25 +++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 15f294698c..7999ee0f80 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1132,6 +1132,28 @@ class ConditionalBlock(object): class Switch(object): + """ + **Switch Class** + + Many programming languages provide `switch` as a generalization of `if-elif-else`. + Switch class works just like a `if-elif-else`. + + The Semantics: + + 1. A `switch` control-flow checks cases one-by-one. + 1. The condition of each case is a boolean value, which is a scalar. + 1. It runs the first matched case, or the default case if there is one. + 1. Once it matches a case, it runs the corresponding branch and only that branch. + + Examples: + .. code-block:: python + + with control_flow.Switch() as switch: + with switch.case(global_step == zero_var): + tensor.assign(input=one_var, output=div_res) + + """ + def __init__(self, name=None): self.helper = LayerHelper('switch', name=name) self.inside_scope = False @@ -1161,7 +1183,8 @@ class Switch(object): return ConditionalBlockGuard(cond_block) def default(self): - """create a default case for this switch + """ + create a default case for this switch """ pre_cond_num = len(self.pre_not_conditions) if pre_cond_num == 0: From d76f8a8f5d06419f3db2647fec8956444ca7c1fe Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 21:24:39 +0800 Subject: [PATCH 029/154] refine doc of polynomial_decay --- .../fluid/layers/learning_rate_scheduler.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 716cc7824e..2e5cff74c1 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -162,22 +162,27 @@ def polynomial_decay(learning_rate, end_learning_rate=0.0001, power=1.0, cycle=False): - """Applies polynomial decay to the initial learning rate. + """ + **polynomial_decay** + + Applies polynomial decay to the initial learning rate. + + .. code-block::python + + if cycle: + decay_steps = decay_steps * ceil(global_step / decay_steps) + else: + global_step = min(global_step, decay_steps) + decayed_learning_rate = (learning_rate - end_learning_rate) * + (1 - global_step / decay_steps) ^ power + end_learning_rate - >>> if cycle: - >>> decay_steps = decay_steps * ceil(global_step / decay_steps) - >>> else: - >>> global_step = min(global_step, decay_steps) - >>> decayed_learning_rate = (learning_rate - end_learning_rate) * - >>> (1 - global_step / decay_steps) ^ power + - >>> end_learning_rate Args: - learning_rate: A scalar float32 value or a Variable. This + learning_rate(Variable|float32): A scalar float32 value or a Variable. This will be the initial learning rate during training - decay_steps: A Python `int32` number. - end_learning_rate: A Python `float` number. - power: A Python `float` number - cycle: Boolean. If set true, decay the learning rate every decay_steps. + decay_steps(int32): A Python `int32` number. + end_learning_rate(float): A Python `float` number. + power(float): A Python `float` number + cycle(bool, Default False): Boolean. If set true, decay the learning rate every decay_steps. Returns: The decayed learning rate From 5d0bf8bc8f4daa3b86e478c9870297f101a9788b Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Tue, 12 Jun 2018 21:26:07 +0800 Subject: [PATCH 030/154] Add API docs. --- paddle/fluid/operators/get_places_op.cc | 2 +- python/paddle/fluid/layers/control_flow.py | 26 ++++++++++++++++++++++ python/paddle/fluid/layers/io.py | 19 ++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/operators/get_places_op.cc b/paddle/fluid/operators/get_places_op.cc index eafc364a15..db6ff78256 100644 --- a/paddle/fluid/operators/get_places_op.cc +++ b/paddle/fluid/operators/get_places_op.cc @@ -85,7 +85,7 @@ class GetPlacesOpProtoMaker : public framework::OpProtoAndCheckerMaker { .InEnum({"CUDA", "CPU", "AUTO"}) .SetDefault("AUTO"); AddComment(R"DOC( -Returns a list of places based on flags. The list will be used for parallel +Returns a list of places based on arguments. The list will be used for parallel execution. )DOC"); } diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..7e541efcfc 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1209,6 +1209,32 @@ class IfElseBlockGuard(object): class IfElse(object): + """ + if-else control flow. + + Args: + cond (Variable): condition used to compare. + name (str, default None): The name of this layer. + + Examples: + .. code-block:: python + limit = layers.fill_constant_batch_size_like( + input=label, dtype='int64', shape=[1], value=5.0) + cond = layers.less_than(x=label, y=limit) + ie = layers.IfElse(cond) + with ie.true_block(): + true_image = ie.input(image) + hidden = layers.fc(input=true_image, size=100, act='tanh') + prob = layers.fc(input=hidden, size=10, act='softmax') + ie.output(prob) + + with ie.false_block(): + false_image = ie.input(image) + hidden = layers.fc(input=false_image, size=200, act='tanh') + prob = layers.fc(input=hidden, size=10, act='softmax') + ie.output(prob) + prob = ie() + """ OUT_IF_ELSE_BLOCKS = 0 IN_IF_ELSE_TRUE_BLOCKS = 1 IN_IF_ELSE_FALSE_BLOCKS = 2 diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 9de88e2c32..df264e4f26 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -587,6 +587,25 @@ def read_file(file_obj): class Preprocessor(object): + """ + A block for data pre-processing in reader. + + Args: + reader (Variable): A reader variable. + name (str, default None): The name of the reader. + + Examples: + .. code-block:: python + preprocessor = fluid.layers.io.Preprocessor(reader=reader) + with preprocessor.block(): + img, lbl = preprocessor.inputs() + img_out = img / 2 + lbl_out = lbl + 1 + preprocessor.outputs(img_out, lbl_out) + + data_file = fluid.layers.io.double_buffer(preprocessor()) + + """ BEFORE_SUB_BLOCK = 0 IN_SUB_BLOCK = 1 AFTER_SUB_BLOCK = 2 From 9185579c45281641618e95996b3451a5a81ea215 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Tue, 12 Jun 2018 21:58:28 +0800 Subject: [PATCH 031/154] follow comments --- python/paddle/fluid/layers/control_flow.py | 1 + python/paddle/fluid/layers/io.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 7e541efcfc..2279197717 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1218,6 +1218,7 @@ class IfElse(object): Examples: .. code-block:: python + limit = layers.fill_constant_batch_size_like( input=label, dtype='int64', shape=[1], value=5.0) cond = layers.less_than(x=label, y=limit) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index df264e4f26..f3aeb6cd75 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -596,6 +596,7 @@ class Preprocessor(object): Examples: .. code-block:: python + preprocessor = fluid.layers.io.Preprocessor(reader=reader) with preprocessor.block(): img, lbl = preprocessor.inputs() From d6c8d2675cd07ae679f922fef83a0a089d04c2ee Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 12 Jun 2018 23:26:58 +0800 Subject: [PATCH 032/154] optimize code and comment --- paddle/fluid/operators/merge_ids_op.cc | 10 ++++++---- paddle/fluid/operators/merge_ids_op.h | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/operators/merge_ids_op.cc b/paddle/fluid/operators/merge_ids_op.cc index bae649adec..f3940231d7 100644 --- a/paddle/fluid/operators/merge_ids_op.cc +++ b/paddle/fluid/operators/merge_ids_op.cc @@ -21,15 +21,17 @@ class MergeIdsOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { AddInput("Ids", "(LoDTensor) the input ids with shape{batch_num, 1}"); - AddInput("X", - "(LoDTensor) the input tensor with shape{batch_num, N}, N is the " - "size of embedding table") + AddInput( + "X", + "(LoDTensors) multi input tensor with shape{batch_num, N}, N is the " + "size of embedding table") .AsDuplicable(); AddOutput("Out", "(LoDTensor) The merged outputs of the input tensors."); AddComment(R"DOC( Merge multi LoDTensor's into one according to Ids's shard num. -The values in the input LoDTensor are lookuped from the output of splite_ids_op +The values in the input LoDTensor are lookuped from the output of split_ids_op + Example: Input: Ids = [1,2,3,4,5,6] diff --git a/paddle/fluid/operators/merge_ids_op.h b/paddle/fluid/operators/merge_ids_op.h index 065368f8dd..83712a8519 100644 --- a/paddle/fluid/operators/merge_ids_op.h +++ b/paddle/fluid/operators/merge_ids_op.h @@ -47,10 +47,10 @@ class MergeIdsOpKernel : public framework::OpKernel { int batch_size = 0; int embedding_size = 0; for (auto &input : x_tensors) { - if (embedding_size == 0) { - embedding_size = input->dims()[1]; - } if (framework::product(input->dims()) != 0) { + if (embedding_size == 0) { + embedding_size = input->dims()[1]; + } PADDLE_ENFORCE_EQ(embedding_size, input->dims()[1], "embedding size of all input should be the same"); batch_size += input->dims()[0]; @@ -58,7 +58,7 @@ class MergeIdsOpKernel : public framework::OpKernel { } PADDLE_ENFORCE_EQ( batch_size, ids_dims[0], - "the batch size of ids and embedding value should be the same"); + "the batch size of ids and merged embedding value should be the same"); const size_t shard_num = x_tensors.size(); @@ -80,7 +80,7 @@ class MergeIdsOpKernel : public framework::OpKernel { in_indexs[shard_id] += 1; } - for (int i = 0; i < shard_num; ++i) { + for (size_t i = 0; i < shard_num; ++i) { PADDLE_ENFORCE_EQ(in_indexs[i], x_tensors[i]->dims()[0], "after merge, all data in x_tensor should be used"); } From 0a46de4a3c46278a387463a693fe405d10a057f6 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 12 Jun 2018 23:03:02 +0800 Subject: [PATCH 033/154] fix unit test --- .../test_image_classification_resnet.py | 5 +++-- .../image_classification/test_image_classification_vgg.py | 5 +++-- .../test_understand_sentiment_stacked_lstm.py | 7 +++++-- .../book_memory_optimization/test_memopt_fit_a_line.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py index 2df3da9cca..8e222d2690 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py @@ -96,10 +96,11 @@ def train(use_cuda, train_program, params_dirname): train_reader = paddle.batch( paddle.reader.shuffle( cifar10_small_test_set.train10(batch_size=10), buf_size=128 * 10), - batch_size=BATCH_SIZE) + batch_size=BATCH_SIZE, + drop_last=False) test_reader = paddle.batch( - paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE) + paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE, drop_last=False) def event_handler(event): if isinstance(event, fluid.EndStepEvent): diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 224cca417e..dbc7bc06c9 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -73,10 +73,11 @@ def train(use_cuda, train_program, params_dirname): train_reader = paddle.batch( paddle.reader.shuffle( cifar10_small_test_set.train10(batch_size=10), buf_size=128 * 10), - batch_size=BATCH_SIZE) + batch_size=BATCH_SIZE, + drop_last=False) test_reader = paddle.batch( - paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE) + paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE, drop_last=False) def event_handler(event): if isinstance(event, fluid.EndStepEvent): diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py index 113dda88ca..8c74be0f08 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py @@ -87,7 +87,9 @@ def train(use_cuda, train_program, params_dirname): def event_handler(event): if isinstance(event, fluid.EndEpochEvent): test_reader = paddle.batch( - paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) + paddle.dataset.imdb.test(word_dict), + batch_size=BATCH_SIZE, + drop_last=False) avg_cost, acc = trainer.test( reader=test_reader, feed_order=['words', 'label']) @@ -113,7 +115,8 @@ def train(use_cuda, train_program, params_dirname): train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.imdb.train(word_dict), buf_size=25000), - batch_size=BATCH_SIZE) + batch_size=BATCH_SIZE, + drop_last=False) trainer.train( num_epochs=1, diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index 8818cf96fa..be347cd531 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -56,7 +56,7 @@ BATCH_SIZE = 200 # fix the order of training data train_reader = paddle.batch( - paddle.dataset.uci_housing.train(), batch_size=BATCH_SIZE) + paddle.dataset.uci_housing.train(), batch_size=BATCH_SIZE, drop_last=False) # train_reader = paddle.batch( # paddle.reader.shuffle( From 26ed4b13573678507d09e466e85fe2dc8a074105 Mon Sep 17 00:00:00 2001 From: weixing02 Date: Wed, 13 Jun 2018 11:12:02 +0800 Subject: [PATCH 034/154] fix deadlink --- doc/v2/dev/contribute_to_paddle_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/v2/dev/contribute_to_paddle_cn.md b/doc/v2/dev/contribute_to_paddle_cn.md index add06e42f1..3244eedf91 100644 --- a/doc/v2/dev/contribute_to_paddle_cn.md +++ b/doc/v2/dev/contribute_to_paddle_cn.md @@ -104,7 +104,7 @@ no changes added to commit (use "git add" and/or "git commit -a") ➜ docker run -it -v $(pwd):/paddle paddle:latest-dev bash -c "cd /paddle/build && ctest" ``` -关于构建和测试的更多信息，请参见[这篇文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。 +关于构建和测试的更多信息，请参见[使用Docker安装运行](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/v2/build_and_install/docker_install_cn.rst)。 ## 提交（commit） From d76ebd7853e3f092e313059360ba451851eea624 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Wed, 13 Jun 2018 11:23:56 +0800 Subject: [PATCH 035/154] fix nccl dist train bug --- paddle/fluid/operators/gen_nccl_id_op.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paddle/fluid/operators/gen_nccl_id_op.cc b/paddle/fluid/operators/gen_nccl_id_op.cc index 111e58844c..f824eee4e7 100644 --- a/paddle/fluid/operators/gen_nccl_id_op.cc +++ b/paddle/fluid/operators/gen_nccl_id_op.cc @@ -67,6 +67,10 @@ class GenNCCLIdOp : public framework::OperatorBase { client->AsyncSendVar(ep, dev_ctx, *scope, NCCL_ID_VARNAME); } client->Wait(); + for (auto& ep : endpoint_list) { + client->AsyncSendBatchBarrier(ep); + } + client->Wait(); VLOG(3) << "sending completed..."; } From 2bf46c3272ae7e961f751a961404ede70b87d792 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 12 Jun 2018 22:06:32 +0800 Subject: [PATCH 036/154] add url of cuda9.0_cudnn7_avx_mkl --- doc/v2/build_and_install/pip_install_cn.rst | 1 + doc/v2/build_and_install/pip_install_en.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/v2/build_and_install/pip_install_cn.rst b/doc/v2/build_and_install/pip_install_cn.rst index 853bdb21bb..095da19cd4 100644 --- a/doc/v2/build_and_install/pip_install_cn.rst +++ b/doc/v2/build_and_install/pip_install_cn.rst @@ -60,6 +60,7 @@ paddlepaddle-gpu==0.11.0 使用CUDA 7.5和cuDNN 5编译的0.11.0版 "cpu_noavx_openblas", "`paddlepaddle-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle-latest-cp27-cp27m-linux_x86_64.whl `_" "cuda8.0_cudnn5_avx_mkl", "`paddlepaddle_gpu-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle_gpu-latest-cp27-cp27m-linux_x86_64.whl `__" "cuda8.0_cudnn7_avx_mkl", "`paddlepaddle_gpu-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle_gpu-latest-cp27-cp27m-linux_x86_64.whl `__" + "cuda9.0_cudnn7_avx_mkl", "`paddlepaddle_gpu-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle_gpu-latest-cp27-cp27m-linux_x86_64.whl `__" .. _pip_dependency: diff --git a/doc/v2/build_and_install/pip_install_en.rst b/doc/v2/build_and_install/pip_install_en.rst index fecf6d3712..8406e4aa1f 100644 --- a/doc/v2/build_and_install/pip_install_en.rst +++ b/doc/v2/build_and_install/pip_install_en.rst @@ -63,6 +63,7 @@ If the links below shows up the login form, just click "Log in as guest" to star "cpu_noavx_openblas", "`paddlepaddle-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle-latest-cp27-cp27m-linux_x86_64.whl `__" "cuda8.0_cudnn5_avx_mkl", "`paddlepaddle_gpu-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle_gpu-latest-cp27-cp27m-linux_x86_64.whl `__" "cuda8.0_cudnn7_avx_mkl", "`paddlepaddle_gpu-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle_gpu-latest-cp27-cp27m-linux_x86_64.whl `__" + "cuda9.0_cudnn7_avx_mkl", "`paddlepaddle_gpu-latest-cp27-cp27mu-linux_x86_64.whl `__", "`paddlepaddle_gpu-latest-cp27-cp27m-linux_x86_64.whl `__" .. _pip_dependency: From 6f7f8b4f0e9711b0b6ecb2cf512528d32e5493fe Mon Sep 17 00:00:00 2001 From: weixing02 Date: Wed, 13 Jun 2018 12:57:35 +0800 Subject: [PATCH 037/154] fix errors --- doc/v2/api/config/evaluators.rst | 2 +- python/paddle/trainer_config_helpers/attrs.py | 13 +++++++------ python/paddle/trainer_config_helpers/layers.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/v2/api/config/evaluators.rst b/doc/v2/api/config/evaluators.rst index 9ac972fb19..458d892e82 100644 --- a/doc/v2/api/config/evaluators.rst +++ b/doc/v2/api/config/evaluators.rst @@ -101,7 +101,7 @@ value_printer :noindex: Detection -===== +========== detection_map ------------- diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index e6f87ce61b..4e3beaf639 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -240,14 +240,15 @@ class ExtraLayerAttribute(object): :type error_clipping_threshold: float :param drop_rate: Dropout rate. Dropout will create a mask on layer output. The dropout rate is the zero rate of this mask. The - details of what dropout is please refer to `here - `_. + details of what dropout is please refer to `JMLRdropout + `_. :type drop_rate: float :param device: device ID of layer. device=-1, use CPU. device>=0, use GPU. - The details allocation in parallel_nn please refer to `here - `_. + The details allocation in parallel_nn please refer to `use_case + `_. :type device: int """ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index ebc31b23e0..4e67d8a846 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2556,7 +2556,7 @@ def img_conv_layer(input, the output will be obtained by concatenating the two results. The details of grouped convolution, please refer to: - `ImageNet Classification with Deep Convolutional Neural Networks + `ImageNet Classification With Deep Convolutional Neural Networks `_ The example usage is: From b9843abb613860da99cd6dc5bda502f6d595d165 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Wed, 13 Jun 2018 13:20:09 +0800 Subject: [PATCH 038/154] Polish comsum, DynamicRNN --- paddle/fluid/operators/cumsum_op.cc | 4 +- python/paddle/fluid/layers/control_flow.py | 142 +++++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/operators/cumsum_op.cc b/paddle/fluid/operators/cumsum_op.cc index 2caa8bf2d5..5302b822d6 100644 --- a/paddle/fluid/operators/cumsum_op.cc +++ b/paddle/fluid/operators/cumsum_op.cc @@ -30,8 +30,8 @@ class CumOp : public framework::OperatorWithKernel { class CumsumOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddInput("X", "Input of Cumsum operator"); - AddOutput("Out", "Output of Cumsum operator"); + AddInput("X", "Input of cumsum operator"); + AddOutput("Out", "Output of cumsum operator"); AddAttr("axis", "The dimenstion to accumulate along. -1 means the last " "dimenstion [default -1].") diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 6c707a35d3..8d28aeb2ed 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -20,6 +20,7 @@ from ..framework import Program, Variable, Operator from ..layer_helper import LayerHelper, unique_name from ..initializer import force_init_on_cpu from ops import logical_and, logical_not, logical_or +import numpy __all__ = [ 'split_lod_tensor', @@ -1314,6 +1315,39 @@ class IfElse(object): class DynamicRNN(object): + """ + Dynamic RNN. + + This RNN can process a batch of sequence data. The length of each sample + sequence can be different. This API automatically process them in batch. + + The input lod must be set. Please reference `lod_tensor` + + >>> import paddle.fluid as fluid + >>> data = fluid.layers.data(name='sentence', dtype='int64', lod_level=1) + >>> embedding = fluid.layers.embedding(input=data, size=[65535, 32], + >>> is_sparse=True) + >>> + >>> drnn = fluid.layers.DynamicRNN() + >>> with drnn.block(): + >>> word = drnn.step_input(embedding) + >>> prev = drnn.memory(shape=[200]) + >>> hidden = fluid.layers.fc(input=[word, prev], size=200, act='relu') + >>> drnn.update_memory(prev, hidden) # set prev to hidden + >>> drnn.output(hidden) + >>> + >>> # last is the last time step of rnn. It is the encoding result. + >>> last = fluid.layers.sequence_last_step(drnn()) + + The dynamic RNN will unfold sequence into timesteps. Users need to define + how to process each time step during the :code:`with` block. + + The `memory` is used staging data cross time step. The initial value of + memory can be zero or another variable. + + The dynamic RNN can mark multiple variables as its output. Use `drnn()` to + get the output sequence. + """ BEFORE_RNN = 0 IN_RNN = 1 AFTER_RNN = 2 @@ -1336,6 +1370,15 @@ class DynamicRNN(object): self.mem_link = [] def step_input(self, x): + """ + Mark a sequence as a dynamic RNN input. + Args: + x(Variable): The input sequence. + + Returns: + The current timestep in the input sequence. + + """ self._assert_in_rnn_block_("step_input") if not isinstance(x, Variable): raise TypeError( @@ -1379,6 +1422,15 @@ class DynamicRNN(object): return array_read(array=input_array, i=self.step_idx) def static_input(self, x): + """ + Mark a variable as a RNN input. The input will not be scattered into + time steps. + Args: + x(Variable): The input variable. + + Returns: + The input variable that can access in RNN. + """ self._assert_in_rnn_block_("static_input") if not isinstance(x, Variable): raise TypeError( @@ -1400,6 +1452,10 @@ class DynamicRNN(object): @contextlib.contextmanager def block(self): + """ + The block for user to define operators in RNN. See the class docstring + for more details. + """ if self.status != DynamicRNN.BEFORE_RNN: raise ValueError("rnn.block() can only be invoke once") self.step_idx = fill_constant( @@ -1426,6 +1482,9 @@ class DynamicRNN(object): x=each_array, table=self.lod_rank_table)) def __call__(self, *args, **kwargs): + """ + Get the output of RNN. This API should only be invoked after RNN.block() + """ if self.status != DynamicRNN.AFTER_RNN: raise ValueError(("Output of the dynamic RNN can only be visited " "outside the rnn block.")) @@ -1440,6 +1499,70 @@ class DynamicRNN(object): value=0.0, need_reorder=False, dtype='float32'): + """ + Create a memory variable. + + If the :code:`init` is not None, :code:`memory` will be initialized by + this variable. The :code:`need_reorder` is used to reorder the memory as + the input variable. It should be set to true when the initialized memory + depends on the input sample. + + For example, + + >>> import paddle.fluid as fluid + >>> sentence = fluid.layers.data( + >>> name='sentence', dtype='float32', shape=[32]) + >>> boot_memory = fluid.layers.data( + >>> name='boot', dtype='float32', shape=[10]) + >>> + >>> drnn = fluid.layers.DynamicRNN() + >>> with drnn.block(): + >>> word = drnn.step_input(sentence) + >>> memory = drnn.memory(init=boot_memory, need_reorder=True) + >>> hidden = fluid.layers.fc( + >>> input=[word, memory], size=10, act='tanh') + >>> drnn.update_memory(ex_mem=memory, new_mem=hidden) + >>> drnn.output(hidden) + >>> rnn_output = drnn() + + + Otherwise, if :code:`shape`, :code:`value`, :code:`dtype` are set, the + :code:`memory` will be initialized by this :code:`value`. + + For example, + + >>> import paddle.fluid as fluid + >>> sentence = fluid.layers.data( + >>> name='sentence', dtype='float32', shape=[32]) + >>> + >>> drnn = fluid.layers.DynamicRNN() + >>> with drnn.block(): + >>> word = drnn.step_input(sentence) + >>> memory = drnn.memory(shape=[10], dtype='float32', value=0) + >>> hidden = fluid.layers.fc( + >>> input=[word, memory], size=10, act='tanh') + >>> drnn.update_memory(ex_mem=memory, new_mem=hidden) + >>> drnn.output(hidden) + >>> rnn_output = drnn() + + + Args: + init(Variable|None): The initialized variable. + + shape(list|tuple): The memory shape. NOTE the shape does not contain + batch_size. + + value(float): the initalized value. + + need_reorder(bool): True if the initialized memory depends on the + input sample. + + dtype(str|numpy.dtype): The data type of the initialized memory. + + Returns: + the memory variable. + + """ self._assert_in_rnn_block_('memory') if init is not None: if not isinstance(init, Variable): @@ -1507,6 +1630,16 @@ class DynamicRNN(object): return self.memory(init=init) def update_memory(self, ex_mem, new_mem): + """ + Update the memory from ex_mem to new_mem. NOTE that the shape and data + type of :code:`ex_mem` and :code:`new_mem` must be same. + Args: + ex_mem(Variable): the memory variable. + new_mem(Variable): the plain variable generated in RNN block. + + Returns: + None + """ self._assert_in_rnn_block_('update_memory') if not isinstance(ex_mem, Variable): raise TypeError("The input arg `ex_mem` of update_memory() must " @@ -1524,6 +1657,15 @@ class DynamicRNN(object): self.mem_link.append((new_mem, mem_array)) def output(self, *outputs): + """ + mark the RNN output variables. + + Args: + outputs: The output variables. + + Returns: + None + """ self._assert_in_rnn_block_('output') parent_block = self._parent_block_() for each in outputs: From 7f32e12b7eb9cc5bfae70e4b17cb5b1d68b3e4ac Mon Sep 17 00:00:00 2001 From: weixing02 Date: Wed, 13 Jun 2018 14:22:09 +0800 Subject: [PATCH 039/154] Fix some v2 doc warnings --- doc/v2/api/config/layer.rst | 193 +++++++++--------- doc/v2/api/index_en.rst | 1 - .../paddle/trainer_config_helpers/layers.py | 4 +- 3 files changed, 96 insertions(+), 102 deletions(-) diff --git a/doc/v2/api/config/layer.rst b/doc/v2/api/config/layer.rst index 1a6496968c..5a0cfadfce 100644 --- a/doc/v2/api/config/layer.rst +++ b/doc/v2/api/config/layer.rst @@ -11,7 +11,7 @@ Data layer data ---- -.. autoclass:: paddle.v2.layer.data +.. autofunction:: paddle.v2.layer.data :noindex: Fully Connected Layers @@ -21,12 +21,12 @@ Fully Connected Layers fc -- -.. autoclass:: paddle.v2.layer.fc +.. autofunction:: paddle.v2.layer.fc :noindex: selective_fc ------------ -.. autoclass:: paddle.v2.layer.selective_fc +.. autofunction:: paddle.v2.layer.selective_fc :noindex: Conv Layers @@ -34,34 +34,34 @@ Conv Layers conv_operator ------------- -.. autoclass:: paddle.v2.layer.conv_operator +.. autofunction:: paddle.v2.layer.conv_operator :noindex: conv_projection --------------- -.. autoclass:: paddle.v2.layer.conv_projection +.. autofunction:: paddle.v2.layer.conv_projection :noindex: conv_shift ---------- -.. autoclass:: paddle.v2.layer.conv_shift +.. autofunction:: paddle.v2.layer.conv_shift :noindex: img_conv -------- -.. autoclass:: paddle.v2.layer.img_conv +.. autofunction:: paddle.v2.layer.img_conv :noindex: .. _api_v2.layer_context_projection: context_projection ------------------ -.. autoclass:: paddle.v2.layer.context_projection +.. autofunction:: paddle.v2.layer.context_projection :noindex: row_conv -------- -.. autoclass:: paddle.v2.layer.row_conv +.. autofunction:: paddle.v2.layer.row_conv :noindex: Image Pooling Layer @@ -69,27 +69,27 @@ Image Pooling Layer img_pool -------- -.. autoclass:: paddle.v2.layer.img_pool +.. autofunction:: paddle.v2.layer.img_pool :noindex: spp --- -.. autoclass:: paddle.v2.layer.spp +.. autofunction:: paddle.v2.layer.spp :noindex: maxout ------ -.. autoclass:: paddle.v2.layer.maxout +.. autofunction:: paddle.v2.layer.maxout :noindex: roi_pool -------- -.. autoclass:: paddle.v2.layer.roi_pool +.. autofunction:: paddle.v2.layer.roi_pool :noindex: pad ---- -.. autoclass:: paddle.v2.layer.pad +.. autofunction:: paddle.v2.layer.pad :noindex: Norm Layer @@ -97,27 +97,27 @@ Norm Layer img_cmrnorm ----------- -.. autoclass:: paddle.v2.layer.img_cmrnorm +.. autofunction:: paddle.v2.layer.img_cmrnorm :noindex: batch_norm ---------- -.. autoclass:: paddle.v2.layer.batch_norm +.. autofunction:: paddle.v2.layer.batch_norm :noindex: sum_to_one_norm --------------- -.. autoclass:: paddle.v2.layer.sum_to_one_norm +.. autofunction:: paddle.v2.layer.sum_to_one_norm :noindex: cross_channel_norm ------------------ -.. autoclass:: paddle.v2.layer.cross_channel_norm +.. autofunction:: paddle.v2.layer.cross_channel_norm :noindex: row_l2_norm ----------- -.. autoclass:: paddle.v2.layer.row_l2_norm +.. autofunction:: paddle.v2.layer.row_l2_norm :noindex: Recurrent Layers @@ -125,22 +125,22 @@ Recurrent Layers recurrent --------- -.. autoclass:: paddle.v2.layer.recurrent +.. autofunction:: paddle.v2.layer.recurrent :noindex: lstmemory --------- -.. autoclass:: paddle.v2.layer.lstmemory +.. autofunction:: paddle.v2.layer.lstmemory :noindex: grumemory --------- -.. autoclass:: paddle.v2.layer.grumemory +.. autofunction:: paddle.v2.layer.grumemory :noindex: gated_unit ----------- -.. autoclass:: paddle.v2.layer.gated_unit +.. autofunction:: paddle.v2.layer.gated_unit :noindex: Recurrent Layer Group @@ -148,32 +148,32 @@ Recurrent Layer Group memory ------ -.. autoclass:: paddle.v2.layer.memory +.. autofunction:: paddle.v2.layer.memory :noindex: recurrent_group --------------- -.. autoclass:: paddle.v2.layer.recurrent_group +.. autofunction:: paddle.v2.layer.recurrent_group :noindex: lstm_step --------- -.. autoclass:: paddle.v2.layer.lstm_step +.. autofunction:: paddle.v2.layer.lstm_step :noindex: gru_step -------- -.. autoclass:: paddle.v2.layer.gru_step +.. autofunction:: paddle.v2.layer.gru_step :noindex: beam_search ------------ -.. autoclass:: paddle.v2.layer.beam_search +.. autofunction:: paddle.v2.layer.beam_search :noindex: get_output ---------- -.. autoclass:: paddle.v2.layer.get_output +.. autofunction:: paddle.v2.layer.get_output :noindex: Mixed Layer @@ -183,54 +183,54 @@ Mixed Layer mixed ----- -.. autoclass:: paddle.v2.layer.mixed +.. autofunction:: paddle.v2.layer.mixed :noindex: .. _api_v2.layer_embedding: embedding --------- -.. autoclass:: paddle.v2.layer.embedding +.. autofunction:: paddle.v2.layer.embedding :noindex: scaling_projection ------------------ -.. autoclass:: paddle.v2.layer.scaling_projection +.. autofunction:: paddle.v2.layer.scaling_projection :noindex: dotmul_projection ----------------- -.. autoclass:: paddle.v2.layer.dotmul_projection +.. autofunction:: paddle.v2.layer.dotmul_projection :noindex: dotmul_operator --------------- -.. autoclass:: paddle.v2.layer.dotmul_operator +.. autofunction:: paddle.v2.layer.dotmul_operator :noindex: full_matrix_projection ---------------------- -.. autoclass:: paddle.v2.layer.full_matrix_projection +.. autofunction:: paddle.v2.layer.full_matrix_projection :noindex: identity_projection ------------------- -.. autoclass:: paddle.v2.layer.identity_projection +.. autofunction:: paddle.v2.layer.identity_projection :noindex: slice_projection ------------------- -.. autoclass:: paddle.v2.layer.slice_projection +.. autofunction:: paddle.v2.layer.slice_projection :noindex: table_projection ---------------- -.. autoclass:: paddle.v2.layer.table_projection +.. autofunction:: paddle.v2.layer.table_projection :noindex: trans_full_matrix_projection ---------------------------- -.. autoclass:: paddle.v2.layer.trans_full_matrix_projection +.. autofunction:: paddle.v2.layer.trans_full_matrix_projection :noindex: Aggregate Layers @@ -245,51 +245,46 @@ AggregateLevel pooling ------- -.. autoclass:: paddle.v2.layer.pooling +.. autofunction:: paddle.v2.layer.pooling :noindex: .. _api_v2.layer_last_seq: last_seq -------- -.. autoclass:: paddle.v2.layer.last_seq +.. autofunction:: paddle.v2.layer.last_seq :noindex: .. _api_v2.layer_first_seq: first_seq --------- -.. autoclass:: paddle.v2.layer.first_seq +.. autofunction:: paddle.v2.layer.first_seq :noindex: sub_seq --------- -.. autoclass:: paddle.v2.layer.sub_seq +.. autofunction:: paddle.v2.layer.sub_seq :noindex: concat ------ -.. autoclass:: paddle.v2.layer.concat +.. autofunction:: paddle.v2.layer.concat :noindex: seq_concat ---------- -.. autoclass:: paddle.v2.layer.seq_concat +.. autofunction:: paddle.v2.layer.seq_concat :noindex: seq_slice --------- -.. autoclass:: paddle.v2.layer.seq_slice - :noindex: - -kmax_sequence_score -------------------- -.. autoclass:: paddle.v2.layer.kmax_sequence_score +.. autofunction:: paddle.v2.layer.seq_slice :noindex: sub_nested_seq -------------- -.. autoclass:: paddle.v2.layer.sub_nested_seq +.. autofunction:: paddle.v2.layer.sub_nested_seq :noindex: Reshaping Layers @@ -297,7 +292,7 @@ Reshaping Layers block_expand ------------ -.. autoclass:: paddle.v2.layer.block_expand +.. autofunction:: paddle.v2.layer.block_expand :noindex: .. _api_v2.layer_expand: @@ -309,22 +304,22 @@ ExpandLevel expand ------ -.. autoclass:: paddle.v2.layer.expand +.. autofunction:: paddle.v2.layer.expand :noindex: repeat ------ -.. autoclass:: paddle.v2.layer.repeat +.. autofunction:: paddle.v2.layer.repeat :noindex: rotate ------ -.. autoclass:: paddle.v2.layer.rotate +.. autofunction:: paddle.v2.layer.rotate :noindex: seq_reshape ----------- -.. autoclass:: paddle.v2.layer.seq_reshape +.. autofunction:: paddle.v2.layer.seq_reshape :noindex: Math Layers @@ -332,94 +327,94 @@ Math Layers addto ----- -.. autoclass:: paddle.v2.layer.addto +.. autofunction:: paddle.v2.layer.addto :noindex: linear_comb ----------- -.. autoclass:: paddle.v2.layer.linear_comb +.. autofunction:: paddle.v2.layer.linear_comb :noindex: interpolation ------------- -.. autoclass:: paddle.v2.layer.interpolation +.. autofunction:: paddle.v2.layer.interpolation :noindex: bilinear_interp --------------- -.. autoclass:: paddle.v2.layer.bilinear_interp +.. autofunction:: paddle.v2.layer.bilinear_interp :noindex: dropout -------- -.. autoclass:: paddle.v2.layer.dropout +.. autofunction:: paddle.v2.layer.dropout :noindex: dot_prod --------- -.. autoclass:: paddle.v2.layer.dot_prod +.. autofunction:: paddle.v2.layer.dot_prod :noindex: out_prod -------- -.. autoclass:: paddle.v2.layer.out_prod +.. autofunction:: paddle.v2.layer.out_prod :noindex: power ----- -.. autoclass:: paddle.v2.layer.power +.. autofunction:: paddle.v2.layer.power :noindex: scaling ------- -.. autoclass:: paddle.v2.layer.scaling +.. autofunction:: paddle.v2.layer.scaling :noindex: clip ---- -.. autoclass:: paddle.v2.layer.clip +.. autofunction:: paddle.v2.layer.clip :noindex: resize ------ -.. autoclass:: paddle.v2.layer.resize +.. autofunction:: paddle.v2.layer.resize :noindex: slope_intercept --------------- -.. autoclass:: paddle.v2.layer.slope_intercept +.. autofunction:: paddle.v2.layer.slope_intercept :noindex: tensor ------ -.. autoclass:: paddle.v2.layer.tensor +.. autofunction:: paddle.v2.layer.tensor :noindex: .. _api_v2.layer_cos_sim: cos_sim ------- -.. autoclass:: paddle.v2.layer.cos_sim +.. autofunction:: paddle.v2.layer.cos_sim :noindex: l2_distance ----------- -.. autoclass:: paddle.v2.layer.l2_distance +.. autofunction:: paddle.v2.layer.l2_distance :noindex: trans ----- -.. autoclass:: paddle.v2.layer.trans +.. autofunction:: paddle.v2.layer.trans :noindex: scale_shift ----------- -.. autoclass:: paddle.v2.layer.scale_shift +.. autofunction:: paddle.v2.layer.scale_shift :noindex: factorization_machine --------------------- -.. autoclass:: paddle.v2.layer.factorization_machine +.. autofunction:: paddle.v2.layer.factorization_machine :noindex: Sampling Layers @@ -427,17 +422,17 @@ Sampling Layers maxid ----- -.. autoclass:: paddle.v2.layer.max_id +.. autofunction:: paddle.v2.layer.max_id :noindex: sampling_id ----------- -.. autoclass:: paddle.v2.layer.sampling_id +.. autofunction:: paddle.v2.layer.sampling_id :noindex: multiplex --------- -.. autoclass:: paddle.v2.layer.multiplex +.. autofunction:: paddle.v2.layer.multiplex :noindex: .. _api_v2.layer_costs: @@ -447,97 +442,97 @@ Cost Layers cross_entropy_cost ------------------ -.. autoclass:: paddle.v2.layer.cross_entropy_cost +.. autofunction:: paddle.v2.layer.cross_entropy_cost :noindex: cross_entropy_with_selfnorm_cost -------------------------------- -.. autoclass:: paddle.v2.layer.cross_entropy_with_selfnorm_cost +.. autofunction:: paddle.v2.layer.cross_entropy_with_selfnorm_cost :noindex: multi_binary_label_cross_entropy_cost ------------------------------------- -.. autoclass:: paddle.v2.layer.multi_binary_label_cross_entropy_cost +.. autofunction:: paddle.v2.layer.multi_binary_label_cross_entropy_cost :noindex: classification_cost ------------------- -.. autoclass:: paddle.v2.layer.classification_cost +.. autofunction:: paddle.v2.layer.classification_cost :noindex: huber_regression_cost ------------------------- -.. autoclass:: paddle.v2.layer.huber_regression_cost +.. autofunction:: paddle.v2.layer.huber_regression_cost :noindex: huber_classification_cost ------------------------- -.. autoclass:: paddle.v2.layer.huber_classification_cost +.. autofunction:: paddle.v2.layer.huber_classification_cost :noindex: lambda_cost ----------- -.. autoclass:: paddle.v2.layer.lambda_cost +.. autofunction:: paddle.v2.layer.lambda_cost :noindex: square_error_cost ----------------- -.. autoclass:: paddle.v2.layer.square_error_cost +.. autofunction:: paddle.v2.layer.square_error_cost :noindex: rank_cost --------- -.. autoclass:: paddle.v2.layer.rank_cost +.. autofunction:: paddle.v2.layer.rank_cost :noindex: sum_cost --------- -.. autoclass:: paddle.v2.layer.sum_cost +.. autofunction:: paddle.v2.layer.sum_cost :noindex: crf --- -.. autoclass:: paddle.v2.layer.crf +.. autofunction:: paddle.v2.layer.crf :noindex: crf_decoding ------------ -.. autoclass:: paddle.v2.layer.crf_decoding +.. autofunction:: paddle.v2.layer.crf_decoding :noindex: ctc --- -.. autoclass:: paddle.v2.layer.ctc +.. autofunction:: paddle.v2.layer.ctc :noindex: warp_ctc -------- -.. autoclass:: paddle.v2.layer.warp_ctc +.. autofunction:: paddle.v2.layer.warp_ctc :noindex: nce --- -.. autoclass:: paddle.v2.layer.nce +.. autofunction:: paddle.v2.layer.nce :noindex: hsigmoid --------- -.. autoclass:: paddle.v2.layer.hsigmoid +.. autofunction:: paddle.v2.layer.hsigmoid :noindex: smooth_l1_cost -------------- -.. autoclass:: paddle.v2.layer.smooth_l1_cost +.. autofunction:: paddle.v2.layer.smooth_l1_cost :noindex: multibox_loss -------------- -.. autoclass:: paddle.v2.layer.multibox_loss +.. autofunction:: paddle.v2.layer.multibox_loss :noindex: detection_output ---------------- -.. autoclass:: paddle.v2.layer.detection_output +.. autofunction:: paddle.v2.layer.detection_output :noindex: Check Layer @@ -545,7 +540,7 @@ Check Layer eos --- -.. autoclass:: paddle.v2.layer.eos +.. autofunction:: paddle.v2.layer.eos :noindex: Activation @@ -553,5 +548,5 @@ Activation prelu -------- -.. autoclass:: paddle.v2.layer.prelu +.. autofunction:: paddle.v2.layer.prelu :noindex: diff --git a/doc/v2/api/index_en.rst b/doc/v2/api/index_en.rst index b11cd449af..70c5c524aa 100644 --- a/doc/v2/api/index_en.rst +++ b/doc/v2/api/index_en.rst @@ -8,4 +8,3 @@ API model_configs.rst data.rst run_logic.rst - fluid/index.rst diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index ebc31b23e0..dddaa4dbd0 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -5678,8 +5678,8 @@ def warp_ctc_layer(input, `_ library, which is used in `Deep Speech 2: End-toEnd Speech Recognition in English and Mandarin `_, to compute Connectionist Temporal - Classification (CTC) loss. Besides, another `warp-ctc - `_ repository, which is forked from + Classification (CTC) loss. Besides, another `warp-ctc repository + `_ , which is forked from the official one, is maintained to enable more compiling options. During the building process, PaddlePaddle will clone the source codes, build and install it to :code:`third_party/install/warpctc` directory. From 7ebef493d58eaf0cbf74bf41e683c3048222b623 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 13 Jun 2018 14:30:03 +0800 Subject: [PATCH 040/154] add row_size for selected rows in DebugStringEx --- paddle/fluid/framework/operator.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index c633a2f847..d22ac66c5c 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -69,6 +69,19 @@ static DDim GetDims(const Scope& scope, const std::string& name, } } +static int GetRowSize(const Scope& scope, const std::string& name) { + Variable* var = scope.FindVar(name); + if (var == nullptr) { + return -1; + } + + if (var->IsType()) { + return var->Get().rows().size(); + } + + return -1; +} + static LoD GetLoD(const Scope& scope, const std::string& name) { Variable* var = scope.FindVar(name); auto default_lod = LoD({{}}); @@ -153,6 +166,10 @@ std::string OperatorBase::DebugStringEx(const Scope* scope) const { for (size_t i = 0; i < input.second.size(); ++i) { ss << input.second[i]; if (scope) { + int row_size = GetRowSize(*scope, input.second[i]); + if (row_size >= 0) { + ss << "[row_size=" << row_size << "]"; + } ss << "[" << GetDims(*scope, input.second[i], true) << "]"; ss << "(" << GetLoD(*scope, input.second[i]) << ")"; } @@ -173,6 +190,10 @@ std::string OperatorBase::DebugStringEx(const Scope* scope) const { for (size_t i = 0; i < output.second.size(); ++i) { ss << output.second[i]; if (scope) { + int row_size = GetRowSize(*scope, output.second[i]); + if (row_size >= 0) { + ss << "[row_size=" << row_size << "]"; + } ss << "[" << GetDims(*scope, output.second[i], true) << "]"; ss << "(" << GetLoD(*scope, output.second[i]) << ")"; } From 2e48ab623e788081b58ae91430e4355fe4c578b9 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 13 Jun 2018 15:17:45 +0800 Subject: [PATCH 041/154] add more detailed comment --- paddle/fluid/operators/merge_ids_op.cc | 55 ++++++++++++++++++-------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/paddle/fluid/operators/merge_ids_op.cc b/paddle/fluid/operators/merge_ids_op.cc index f3940231d7..59cd734367 100644 --- a/paddle/fluid/operators/merge_ids_op.cc +++ b/paddle/fluid/operators/merge_ids_op.cc @@ -30,25 +30,46 @@ class MergeIdsOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Merge multi LoDTensor's into one according to Ids's shard num. -The values in the input LoDTensor are lookuped from the output of split_ids_op + + +split_ids_op -> prefetch_op -> merge_ids_op + + +merge_ids_op should be used after split_ids_op and prefetch_op, split_ids_op + will split input Ids into multiple tensors according to Id's shard number. +prefetch_op will send them to parameter server to prefetch embedding value +back. During split, the order of ids is disordered. In merge_ids_op we use +the original Ids to restore the order of the fetched embedding value and + also pass the lod information to the merged output. + Example: - Input: - Ids = [1,2,3,4,5,6] - X0 = [[0.1 0.2] # 3 - [0.2 0.3]] # 6 - X1 = [[0.3 0.4] # 1 - [0.4 0.5]] # 4 - X2 = [[0.5 0.6] # 2 - [0.6 0.7]] # 5 - - Output: - Out = [[0.3 0.4] # 1 - [0.5 0.6] # 2 - [0.1 0.2] # 3 - [0.4 0.5] # 4 - [0.6 0.7] # 5 - [0.2 0.3]] # 6 + + Ids = [1,2,3,4,5,6] # 3 shared + +split_ids_op -> + + Id0 = [3, 6] + Id1 = [1, 4] + Id2 = [2, 5] + +prefetch_op -> + + X0 = [[0.3 0.3] # 3 + [0.6 0.6]] # 6 + X1 = [[0.1 0.1] # 1 + [0.4 0.4]] # 4 + X2 = [[0.2 0.2] # 2 + [0.5 0.5]] # 5 + +merge_ids_op -> + + Out = [[0.1 0.1] # 1 + [0.2 0.2] # 2 + [0.3 0.3] # 3 + [0.4 0.4] # 4 + [0.5 0.5] # 5 + [0.6 0.6]] # 6 )DOC"); } }; From e6f54d5aa2c637c5719add11abda07bbe82aa6c1 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 13 Jun 2018 15:20:08 +0800 Subject: [PATCH 042/154] update comment --- paddle/fluid/operators/merge_ids_op.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/merge_ids_op.cc b/paddle/fluid/operators/merge_ids_op.cc index 59cd734367..c6ec4ab047 100644 --- a/paddle/fluid/operators/merge_ids_op.cc +++ b/paddle/fluid/operators/merge_ids_op.cc @@ -49,9 +49,9 @@ Example: split_ids_op -> - Id0 = [3, 6] - Id1 = [1, 4] - Id2 = [2, 5] + Id0 = [3, 6] # id % 3 == 0 + Id1 = [1, 4] # id % 3 == 1 + Id2 = [2, 5] # id % 3 == 2 prefetch_op -> From c6c9c657e0c4d9d2c17129511f6a3ab30a190486 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 13 Jun 2018 15:35:23 +0800 Subject: [PATCH 043/154] update doc --- python/paddle/fluid/layers/control_flow.py | 36 ++++++--- .../fluid/layers/learning_rate_scheduler.py | 70 +++++++++++++----- python/paddle/fluid/layers/nn.py | 73 +++++++++++++------ python/paddle/fluid/layers/tensor.py | 42 +++++++---- 4 files changed, 155 insertions(+), 66 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..f4013b61d0 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -748,16 +748,25 @@ def max_sequence_len(rank_table): def lod_tensor_to_array(x, table): - """ Convert a LOD_TENSOR to an LOD_TENSOR_ARRAY. + """ + Convert a LoDTensor to a LoDTensorArray. + + This function split a LoDTesnor to a LoDTensorArray according to its LoD + information. LoDTensorArray is an alias of C++ std::vector in + Paddle. The generated LoDTensorArray of this function can be further read + or written by 'read_from_array()' and 'write_to_array()' operators. However, + this function is generally an internal component of Paddle 'DynamicRNN'. + Users should not use it directly. Args: - x (Variable|list): The LOD tensor to be converted to a LOD tensor array. + x (Variable|list): The LoDTensor to be converted to a LoDTensorArray. table (ParamAttr|list): The variable that stores the level of lod which is ordered by sequence length in - descending order. + descending order. It is generally generated + by 'layers.lod_rank_table()' API. Returns: - Variable: The variable of type array that has been converted from a + Variable: The LoDTensorArray that has been converted from the input tensor. Examples: @@ -1047,6 +1056,13 @@ def array_length(array): class ConditionalBlockGuard(BlockGuard): + """ + ConditionalBlockGuard is derived from BlockGuard. It is dedicated for + holding a ConditionalBlock, and helping users entering and exiting the + ConditionalBlock via Python's 'with' keyword. However, ConditionalBlockGuard + is generally an internal component of IfElse, users should not use it directly. + """ + def __init__(self, block): if not isinstance(block, ConditionalBlock): raise TypeError("block should be conditional block") @@ -1563,17 +1579,15 @@ def reorder_lod_tensor_by_rank(x, rank_table): def is_empty(x, cond=None, **ignored): """ - **Is Empty** - - This layer returns the truth value of whether the variable is empty. + Test whether an Variable is empty. Args: - x(Variable): Operand of *is_empty* - cond(Variable|None): Optional output variable to store the result - of *is_empty* + x (Variable): The Variable to be tested. + cond (Variable|None): Output parameter. Returns the test result + of given 'x'. Returns: - Variable: The tensor variable storing the output of *is_empty*. + Variable: The tensor variable storing the test result of 'x'. Raises: TypeError: If input cond is not a variable, or cond's dtype is diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 716cc7824e..9cbb559093 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -70,21 +70,40 @@ def noam_decay(d_model, warmup_steps): def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): - """Applies exponential decay to the learning rate. + """ + Applies exponential decay to the learning rate. + + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, the learning rate will be decayed by + 'decay_rate' every 'decay_steps' steps. + + >>> if staircase == True: + >>> decayed_learning_rate = learning_rate * decay_rate ^ floor(global_step / decay_steps) + >>> else: + >>> decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) - ```python - decayed_learning_rate = learning_rate * - decay_rate ^ (global_step / decay_steps) - ``` Args: - learning_rate: A scalar float32 value or a Variable. This - will be the initial learning rate during training - decay_steps: A Python `int32` number. - decay_rate: A Python `float` number. - staircase: Boolean. If set true, decay the learning rate every decay_steps. + learning_rate(Variable|float): The initial learning rate. + decay_steps(int): See the decay computation above. + decay_rate(float): The decay rate. See the decay computation above. + staircase(Boolean): If True, decay the learning rate at discrete intervals. + Default: False Returns: The decayed learning rate + + Examples: + .. code-block:: python + + base_lr = 0.1 + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=fluid.layers.exponential_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) + sgd_optimizer.minimize(avg_cost) + """ global_step = _decay_step_counter() @@ -128,22 +147,39 @@ def natural_exp_decay(learning_rate, decay_steps, decay_rate, staircase=False): def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): - """Applies inverse time decay to the initial learning rate. + """ + Applies inverse time decay to the initial learning rate. - >>> if staircase: + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, an inverse decay function will be + applied to the initial learning rate. + + >>> if staircase == True: >>> decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / decay_step)) >>> else: >>> decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / decay_step) Args: - learning_rate: A scalar float32 value or a Variable. This - will be the initial learning rate during training. - decay_steps: A Python `int32` number. - decay_rate: A Python `float` number. - staircase: Boolean. If set true, decay the learning rate every decay_steps. + learning_rate(Variable|float): The initial learning rate. + decay_steps(int): See the decay computation above. + decay_rate(float): The decay rate. See the decay computation above. + staircase(Boolean): If True, decay the learning rate at discrete intervals. + Default: False Returns: The decayed learning rate + + Examples: + .. code-block:: python + + base_lr = 0.1 + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) + sgd_optimizer.minimize(avg_cost) """ global_step = _decay_step_counter() diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index c8cbb5ef00..047c3aa2b7 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -102,14 +102,15 @@ def fc(input, """ **Fully Connected Layer** - The fully connected layer can take multiple tensors as its inputs. It - creates a variable called weights for each input tensor, which represents - a fully connected weight matrix from each input unit to each output unit. - The fully connected layer multiplies each input tensor with its coresponding - weight to produce an output Tensor. If multiple input tensors are given, - the results of multiple multiplications will be sumed up. If bias_attr is - not None, a bias variable will be created and added to the output. Finally, - if activation is not None, it will be applied to the output as well. + This function creates a fully connected layer in the network. It can take + multiple tensors as its inputs. It creates a variable called weights for + each input tensor, which represents a fully connected weight matrix from + each input unit to each output unit. The fully connected layer multiplies + each input tensor with its coresponding weight to produce an output Tensor. + If multiple input tensors are given, the results of multiple multiplications + will be sumed up. If bias_attr is not None, a bias variable will be created + and added to the output. Finally, if activation is not None, it will be applied + to the output as well. This process can be formulated as follows: @@ -878,7 +879,7 @@ def cos_sim(X, Y): Args: X (Variable): The input X. Y (Variable): The input Y. - + Returns: Variable: the output of cosine(X, Y). """ @@ -1083,7 +1084,7 @@ def chunk_eval(input, chunk_scheme (str): ${chunk_scheme_comment} num_chunk_types (int): ${num_chunk_types_comment} excluded_chunk_types (list): ${excluded_chunk_types_comment} - + Returns: tuple: tuple containing: (precision, recall, f1_score, num_infer_chunks, num_label_chunks, @@ -1143,7 +1144,7 @@ def sequence_conv(input, bias_attr (ParamAttr|None): attributes for bias param_attr (ParamAttr|None): attributes for parameter act (str): the activation type - + Returns: Variable: output of sequence_conv """ @@ -1509,6 +1510,7 @@ def sequence_last_step(input): return sequence_pool(input=input, pool_type="last") +@templatedoc() def pool2d(input, pool_size=-1, pool_type="max", @@ -1520,12 +1522,12 @@ def pool2d(input, use_mkldnn=False, name=None): """ - This function adds the operator for pooling in 2 dimensions, using the - pooling configurations mentioned in input parameters. + ${comment} Args: input (Variable): ${input_comment} - pool_size (int): ${ksize_comment} + pool_size (int): The side length of pooling windows. All pooling + windows are squares with pool_size on a side. pool_type (str): ${pooling_type_comment} pool_stride (int): stride of the pooling layer. pool_padding (int): padding size. @@ -1533,11 +1535,29 @@ def pool2d(input, use_cudnn (bool): ${use_cudnn_comment} ceil_mode (bool): ${ceil_mode_comment} use_mkldnn (bool): ${use_mkldnn_comment} - name (str): A name for this layer(optional). If set None, the layer - will be named automatically. - + name (str|None): A name for this layer(optional). If set None, the + layer will be named automatically. + Returns: Variable: output of pool2d layer. + + Raises: + ValueError: If 'pool_type' is not "max" nor "avg" + ValueError: If 'global_pooling' is False and 'pool_size' is -1 + ValueError: If 'use_cudnn' is not a bool value. + + Examples: + + .. code-block:: python + + data = fluid.layers.data( + name='data', shape=[3, 32, 32], dtype='float32') + conv2d = fluid.layers.pool2d( + input=data, + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False) """ if pool_type not in ["max", "avg"]: raise ValueError( @@ -1800,7 +1820,7 @@ def beam_search_decode(ids, scores, name=None): ids (Variable): ${ids_comment} scores (Variable): ${scores_comment} name (str): The name of this layer. It is optional. - + Returns: tuple: a tuple of two output variable: sentence_ids, sentence_scores """ @@ -2063,7 +2083,7 @@ def beam_search(pre_ids, ids, scores, beam_size, end_id, level=0): beam_size (int): ${beam_size_comment} end_id (int): ${end_id_comment} level (int): ${level_comment} - + Returns: tuple: a tuple of beam_search output variables: selected_ids, selected_scores ''' @@ -2719,7 +2739,7 @@ def topk(input, k, name=None): This operator is used to find values and indices of the k largest entries for the last dimension. - If the input is a vector (rank=1), finds the k largest entries in the vector + If the input is a vector (1-D Tensor), finds the k largest entries in the vector and outputs their values and indices as vectors. Thus values[j] is the j-th largest entry in input, and its index is indices[j]. @@ -2729,9 +2749,11 @@ def topk(input, k, name=None): Args: input(Variable): The input variable which can be a vector or Tensor with higher rank. - k(int): An integer value to specify the top k largest elements. + k(int): The number of top elements to look for along the last dimension + of input. name(str|None): A name for this layer(optional). If set None, the layer - will be named automatically. + will be named automatically. + Default: None Returns: values(Variable): The k largest elements along each last dimensional @@ -2739,13 +2761,16 @@ def topk(input, k, name=None): indices(Variable): The indices of values within the last dimension of input. + Raises: + ValueError: If k < 1 or k is not less than the last dimension of input + Examples: .. code-block:: python top5_values, top5_indices = layers.topk(input, k=5) """ shape = input.shape - if k < 1 and k >= shape[-1]: + if k < 1 or k >= shape[-1]: raise ValueError("k must be greater than 0 and less than %d." % (shape[-1])) @@ -3045,7 +3070,7 @@ def nce(input, param_attr (ParamAttr|None): attributes for parameter bias_attr (ParamAttr|None): attributes for bias num_neg_samples (int): ${num_neg_samples_comment} - + Returns: Variable: output of nce layer. """ diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 62b01d595a..e03c8ca914 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -79,20 +79,33 @@ def create_global_var(shape, force_cpu=False, name=None): """ - Create a global variable. such as global_step + Create a new variable in the global block(block 0). + Args: shape(list[int]): shape of the variable - value(float): the value of the variable - dtype(string): element type of the parameter - persistable(bool): if this variable is persistable - force_cpu(bool): force this variable to be on CPU + value(float): the value of the variable. The new created + variable will be filled with it. + dtype(string): data type of the variable + persistable(bool): if this variable is persistable. + Default: False + force_cpu(bool): force this variable to be on CPU. + Default: False + name(str|None): The name of the variable. If set to None the variable + name will be generated automatically. + Default: None Returns: Variable: the created Variable + + Examples: + .. code-block:: python + + var = fluid.create_global_var(shape=[2,3], value=1.0, dtype='float32', + persistable=True, force_cpu=True, name='new_var') """ helper = LayerHelper("global_var", **locals()) var = helper.create_global_variable( - dtype=dtype, shape=shape, persistable=persistable, name=name) + dtype=dtype, shape=shape, persistable=persistable) helper.set_variable_initializer( var, initializer=Constant( value=float(value), force_cpu=force_cpu)) @@ -152,10 +165,11 @@ def sums(input, out=None): Args: input (Variable|list): The input tensor that has the elements that need to be summed up. + out (Variable|None): Output parameter. Returns the sum result. + Default: None Returns: - Variable: The tensor type variable that has the sum of input - written to it. + Variable: the sum of input. The same as the argument 'out' Examples: .. code-block::python @@ -328,13 +342,13 @@ def argmin(x, axis=0): x(Variable): The input to compute the indices of the min elements. axis(int): Axis to compute indices along. - + Returns: Variable: The tensor variable storing the output - + Examples: .. code-block:: python - + out = fluid.layers.argmin(x=in, axis=0) out = fluid.layers.argmin(x=in, axis=-1) """ @@ -359,13 +373,13 @@ def argmax(x, axis=0): x(Variable): The input to compute the indices of the max elements. axis(int): Axis to compute indices along. - + Returns: Variable: The tensor variable storing the output - + Examples: .. code-block:: python - + out = fluid.layers.argmax(x=in, axis=0) out = fluid.layers.argmax(x=in, axis=-1) """ From 3ab32532d54af6185a9604883abd022d7a3bd6fc Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 13 Jun 2018 17:02:38 +0800 Subject: [PATCH 044/154] Add conv3d Python API --- python/paddle/fluid/layers/nn.py | 168 ++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index c8cbb5ef00..f6b4348d25 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1305,8 +1305,6 @@ def conv2d(input, conv2d = fluid.layers.conv2d( input=data, num_filters=2, filter_size=3, act="relu") """ - if stride is None: - stride = [1, 1] num_channels = input.shape[1] @@ -1369,6 +1367,172 @@ def conv2d(input, return helper.append_activation(pre_act) +def conv3d(input, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + use_mkldnn=False, + act=None, + name=None): + """ + **Convlution3D Layer** + + The convolution3D layer calculates the output based on the input, filter + and strides, paddings, dilations, groups parameters. Input(Input) and + Output(Output) are in NCHW format. Where N is batch size, C is the number of + channels, H is the height of the feature, and W is the width of the feature. + The details of convolution layer, please refer UFLDL's `convolution, + `_ . + If bias attribution and activation type are provided, bias is added to the + output of the convolution, and the corresponding activation function is + applied to the final result. + + For each input :math:`X`, the equation is: + + .. math:: + + Out = \sigma (W \\ast X + b) + + In the above equation: + + * :math:`X`: Input value, a tensor with NCHW format. + * :math:`W`: Filter value, a tensor with MCHW format. + * :math:`\\ast`: Convolution operation. + * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. + * :math:`\\sigma`: Activation function. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be + different. + + Example: + + - Input: + + Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})` + + Filter shape: :math:`(C_{out}, C_{in}, D_f, H_f, W_f)` + + - Output: + Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` + + Where + + .. math:: + + D_{out}&= \\frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (D_f - 1) + 1))}{strides[0]} + 1 \\\\ + H_{out}&= \\frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (H_f - 1) + 1))}{strides[1]} + 1 \\\\ + W_{out}&= \\frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (W_f - 1) + 1))}{strides[2]} + 1 + + Args: + input (Variable): The input image with [N, C, D, H, W] format. + num_filters(int): The number of filter. It is as same as the output + image channel. + filter_size (int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_D, filter_size_H, filter_size_W). + Otherwise, the filter will be a square. + stride (int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_D, stride_H, stride_W). Otherwise, the + stride_D = stride_H = stride_W = stride. Default: stride = 1. + padding (int|tuple): The padding size. If padding is a tuple, it must + contain two integers, (padding_D, padding_H, padding_W). Otherwise, the + padding_D = padding_H = padding_W = padding. Default: padding = 0. + dilation (int|tuple): The dilation size. If dilation is a tuple, it must + contain two integers, (dilation_D, dilation_H, dilation_W). Otherwise, the + dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1. + groups (int): The groups number of the Conv3d Layer. According to grouped + convolution in Alex Krizhevsky's Deep CNN paper: when group=2, + the first half of the filters is only connected to the first half + of the input channels, while the second half of the filters is only + connected to the second half of the input channels. Default: groups=1 + param_attr (ParamAttr): The parameters to the Conv3d Layer. Default: None + bias_attr (ParamAttr): Bias parameter for the Conv3d layer. Default: None + use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + use_mkldnn (bool): Use mkldnn kernels or not. + act (str): Activation type. Default: None + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: The tensor variable storing the convolution and \ + non-linearity activation result. + + Raises: + ValueError: If the shapes of input, filter_size, stride, padding and + groups mismatch. + + Examples: + .. code-block:: python + + data = fluid.layers.data( + name='data', shape=[3, 12, 32, 32], dtype='float32') + conv2d = fluid.layers.conv3d( + input=data, num_filters=2, filter_size=3, act="relu") + """ + + l_type = 'conv3d' + + helper = LayerHelper(l_type, **locals()) + dtype = helper.input_dtype() + + num_channels = input.shape[1] + + if groups is None: + num_filter_channels = num_channels + else: + if num_channels % groups != 0: + raise ValueError("num_channels must be divisible by groups.") + num_filter_channels = num_channels / groups + + filter_size = utils.convert_to_list(filter_size, 3, 'filter_size') + stride = utils.convert_to_list(stride, 3, 'stride') + padding = utils.convert_to_list(padding, 3, 'padding') + dilation = utils.convert_to_list(dilation, 3, 'dilation') + + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") + + input_shape = input.shape + filter_shape = [num_filters, num_filter_channels] + filter_size + + def _get_default_param_initializer(): + std = (2.0 / (filter_size[0]**3 * num_channels))**0.5 + return Normal(0.0, std, 0) + + filter_param = helper.create_parameter( + attr=helper.param_attr, + shape=filter_shape, + dtype=dtype, + default_initializer=_get_default_param_initializer()) + + pre_bias = helper.create_tmp_variable(dtype) + + helper.append_op( + type=l_type, + inputs={ + 'Input': input, + 'Filter': filter_param, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'use_mkldnn': use_mkldnn + }) + + pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=3) + + return helper.append_activation(pre_act) + + def sequence_pool(input, pool_type): """ This function add the operator for sequence pooling. From 2183d01799dcaa452c63ca04d5ac194f91f01b25 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 13 Jun 2018 17:14:00 +0800 Subject: [PATCH 045/154] Add pool3d and conv3d_trans Python API --- python/paddle/fluid/layers/nn.py | 273 +++++++++++++++++++++++++++++-- 1 file changed, 257 insertions(+), 16 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index f6b4348d25..3a4f93929f 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -39,13 +39,16 @@ __all__ = [ 'chunk_eval', 'sequence_conv', 'conv2d', + 'conv3d', 'sequence_pool', 'sequence_softmax', 'softmax', 'pool2d', + 'pool3d', 'batch_norm', 'beam_search_decode', 'conv2d_transpose', + 'conv3d_transpose', 'sequence_expand', 'lstm_unit', 'reduce_sum', @@ -1385,13 +1388,12 @@ def conv3d(input, The convolution3D layer calculates the output based on the input, filter and strides, paddings, dilations, groups parameters. Input(Input) and - Output(Output) are in NCHW format. Where N is batch size, C is the number of - channels, H is the height of the feature, and W is the width of the feature. - The details of convolution layer, please refer UFLDL's `convolution, - `_ . - If bias attribution and activation type are provided, bias is added to the - output of the convolution, and the corresponding activation function is - applied to the final result. + Output(Output) are in NCDHW format. Where N is batch size C is the number of + channels, D is the depth of the feature, H is the height of the feature, + and W is the width of the feature. Convlution3D is similar with Convlution2D + but adds one dimension(depth). If bias attribution and activation type are + provided, bias is added to the output of the convolution, and the + corresponding activation function is applied to the final result. For each input :math:`X`, the equation is: @@ -1401,8 +1403,8 @@ def conv3d(input, In the above equation: - * :math:`X`: Input value, a tensor with NCHW format. - * :math:`W`: Filter value, a tensor with MCHW format. + * :math:`X`: Input value, a tensor with NCDHW format. + * :math:`W`: Filter value, a tensor with MCDHW format. * :math:`\\ast`: Convolution operation. * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. * :math:`\\sigma`: Activation function. @@ -1433,16 +1435,16 @@ def conv3d(input, num_filters(int): The number of filter. It is as same as the output image channel. filter_size (int|tuple|None): The filter size. If filter_size is a tuple, - it must contain two integers, (filter_size_D, filter_size_H, filter_size_W). + it must contain three integers, (filter_size_D, filter_size_H, filter_size_W). Otherwise, the filter will be a square. stride (int|tuple): The stride size. If stride is a tuple, it must - contain two integers, (stride_D, stride_H, stride_W). Otherwise, the + contain three integers, (stride_D, stride_H, stride_W). Otherwise, the stride_D = stride_H = stride_W = stride. Default: stride = 1. padding (int|tuple): The padding size. If padding is a tuple, it must - contain two integers, (padding_D, padding_H, padding_W). Otherwise, the + contain three integers, (padding_D, padding_H, padding_W). Otherwise, the padding_D = padding_H = padding_W = padding. Default: padding = 0. dilation (int|tuple): The dilation size. If dilation is a tuple, it must - contain two integers, (dilation_D, dilation_H, dilation_W). Otherwise, the + contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1. groups (int): The groups number of the Conv3d Layer. According to grouped convolution in Alex Krizhevsky's Deep CNN paper: when group=2, @@ -1528,7 +1530,7 @@ def conv3d(input, 'use_mkldnn': use_mkldnn }) - pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=3) + pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) return helper.append_activation(pre_act) @@ -1720,12 +1722,84 @@ def pool2d(input, if not isinstance(use_cudnn, bool): raise ValueError("use_cudnn should be True or False") - helper = LayerHelper('pool2d', **locals()) + l_type = 'conv2d' + + helper = LayerHelper(l_type, **locals()) + dtype = helper.input_dtype() + pool_out = helper.create_tmp_variable(dtype) + + helper.append_op( + type=l_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding, + "use_cudnn": use_cudnn, + "ceil_mode": ceil_mode, + "use_mkldnn": use_mkldnn + }) + + return pool_out + + +def pool3d(input, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + use_mkldnn=False, + name=None): + """ + This function adds the operator for pooling in 3-dimensions, using the + pooling configurations mentioned in input parameters. + + Args: + input (Variable): ${input_comment} + pool_size (int): ${ksize_comment} + pool_type (str): ${pooling_type_comment} + pool_stride (int): stride of the pooling layer. + pool_padding (int): padding size. + global_pooling (bool): ${global_pooling_comment} + use_cudnn (bool): ${use_cudnn_comment} + ceil_mode (bool): ${ceil_mode_comment} + use_mkldnn (bool): ${use_mkldnn_comment} + name (str): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: output of pool3d layer. + """ + if pool_type not in ["max", "avg"]: + raise ValueError( + "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", + str(pool_type)) + + if global_pooling is False and pool_size == -1: + raise ValueError( + "When the global_pooling is False, pool_size must be passed " + "and be a valid value. Received pool_size: " + str(pool_size)) + + pool_size = utils.convert_to_list(pool_size, 3, 'pool_size') + pool_padding = utils.convert_to_list(pool_padding, 3, 'pool_padding') + pool_stride = utils.convert_to_list(pool_stride, 3, 'pool_stride') + + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") + + l_type = "pool3d" + helper = LayerHelper(l_type, **locals()) dtype = helper.input_dtype() pool_out = helper.create_tmp_variable(dtype) helper.append_op( - type="pool2d", + type=l_type, inputs={"X": input}, outputs={"Out": pool_out}, attrs={ @@ -2146,6 +2220,173 @@ def conv2d_transpose(input, return out +def conv3d_transpose(input, + num_filters, + output_size=None, + filter_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None): + """ + **Convlution3D transpose layer** + + The convolution3D transpose layer calculates the output based on the input, + filter, and dilations, strides, paddings. Input(Input) and output(Output) + are in NCDHW format. Where N is batch size, C is the number of channels, + D is the depth of the feature, H is the height of the feature, and W + is the width of the feature. Parameters(dilations, strides, paddings) are + two elements. These two elements represent height and width, respectively. + The details of convolution transpose layer, please refer to the following + explanation and references `therein `_. + + For each input :math:`X`, the equation is: + + .. math:: + + Out = W \\ast X + + In the above equation: + + * :math:`X`: Input value, a tensor with NCDHW format. + * :math:`W`: Filter value, a tensor with MCDHW format. + * :math:`\\ast` : Convolution transpose operation. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be + different. + + Example: + + - Input: + + Input shape: $(N, C_{in}, D_{in}, H_{in}, W_{in})$ + + Filter shape: $(C_{in}, C_{out}, D_f, H_f, W_f)$ + + - Output: + + Output shape: $(N, C_{out}, D_{out}, H_{out}, W_{out})$ + + Where + + .. math:: + + D_{out} &= (D_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (D_f - 1) + 1 \\\\ + H_{out} &= (H_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (H_f - 1) + 1 \\\\ + W_{out} &= (W_{in} - 1) * strides[2] - 2 * paddings[2] + dilations[2] * (W_f - 1) + 1 + + Args: + input(Variable): The input image with [N, C, D, H, W] format. + num_filters(int): The number of the filter. It is as same as the output + image channel. + output_size(int|tuple|None): The output image size. If output size is a + tuple, it must contain three integers, (image_D, image_H, image_W). This + parameter only works when filter_size is None. + filter_size(int|tuple|None): The filter size. If filter_size is a tuple, + it must contain three integers, (filter_size_D, filter_size_H, filter_size_W). + Otherwise, the filter will be a square. None if use output size to + calculate filter_size. + padding(int|tuple): The padding size. If padding is a tuple, it must + contain three integers, (padding_D, padding_H, padding_W). Otherwise, the + padding_D = padding_H = padding_W = padding. Default: padding = 0. + stride(int|tuple): The stride size. If stride is a tuple, it must + contain three integers, (stride_D, stride_H, stride_W). Otherwise, the + stride_D = stride_H = stride_W = stride. Default: stride = 1. + dilation(int|tuple): The dilation size. If dilation is a tuple, it must + contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the + dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1. + groups(int): The groups number of the Conv3d transpose layer. Inspired by + grouped convolution in Alex Krizhevsky's Deep CNN paper, in which + when group=2, the first half of the filters is only connected to the + first half of the input channels, while the second half of the + filters is only connected to the second half of the input channels. + Default: groups=1 + param_attr(ParamAttr): The parameters to the Conv3d_transpose Layer. + Default: None + bias_attr(ParamAttr): Bias parameter for the Conv3d layer. Default: None + use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + act(str): Activation type. Default: None + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: The tensor variable storing the convolution transpose result. + + Raises: + ValueError: If the shapes of input, filter_size, stride, padding and + groups mismatch. + + Examples: + .. code-block:: python + + data = fluid.layers.data( + name='data', shape=[3, 12, 32, 32], dtype='float32') + conv2d_transpose = fluid.layers.conv3d_transpose( + input=data, num_filters=2, filter_size=3) + """ + l_type = "conv3d_transpose" + helper = LayerHelper(l_type, **locals()) + if not isinstance(input, Variable): + raise TypeError("Input of conv3d_transpose must be Variable") + input_channel = input.shape[1] + + padding = utils.convert_to_list(padding, 3, 'padding') + stride = utils.convert_to_list(stride, 3, 'stride') + dilation = utils.convert_to_list(dilation, 3, 'dilation') + + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") + + if filter_size is None: + if output_size is None: + raise ValueError("output_size must be set when filter_size is None") + if isinstance(output_size, int): + output_size = [output_size, output_size] + + d_in = input.shape[2] + h_in = input.shape[3] + w_in = input.shape[4] + + filter_size_d = (output_size[0] - (d_in - 1) * stride[0] + 2 * + padding[0] - 1) / dilation[0] + 1 + filter_size_h = (output_size[1] - (h_in - 1) * stride[1] + 2 * + padding[1] - 1) / dilation[1] + 1 + filter_size_w = (output_size[2] - (w_in - 1) * stride[2] + 2 * + padding[2] - 1) / dilation[2] + 1 + filter_size = [filter_size_d, filter_size_h, filter_size_w] + else: + filter_size = utils.convert_to_list(filter_size, 3, + 'conv3d_transpose.filter_size') + + groups = 1 if groups is None else groups + filter_shape = [input_channel, num_filters / groups] + filter_size + img_filter = helper.create_parameter( + dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) + + pre_bias = helper.create_tmp_variable(dtype=input.dtype) + helper.append_op( + type=l_type, + inputs={'Input': [input], + 'Filter': [img_filter]}, + outputs={'Output': pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn + }) + + pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) + out = helper.append_activation(pre_act) + return out + + def sequence_expand(x, y, ref_level=-1, name=None): """Sequence Expand Layer. This layer will expand the input variable **x** according to specified level lod of **y**. Please note that lod level of From a7bf2b06b4b1071f050ea5d3935512918303a777 Mon Sep 17 00:00:00 2001 From: weixing02 Date: Wed, 13 Jun 2018 09:35:13 +0000 Subject: [PATCH 046/154] add apis for learning_rate_scheduler --- doc/fluid/api/gen_doc.sh | 2 +- doc/fluid/api/layers.rst | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/fluid/api/gen_doc.sh b/doc/fluid/api/gen_doc.sh index 3ee299f5ac..27f2419c06 100755 --- a/doc/fluid/api/gen_doc.sh +++ b/doc/fluid/api/gen_doc.sh @@ -1,5 +1,5 @@ #!/bin/bash -python gen_doc.py layers --submodules control_flow device io nn ops tensor detection > layers.rst +python gen_doc.py layers --submodules control_flow device io nn ops tensor detection learning_rate_scheduler > layers.rst for module in data_feeder clip metrics executor initializer io nets optimizer param_attr profiler regularizer do diff --git a/doc/fluid/api/layers.rst b/doc/fluid/api/layers.rst index ba33c0d7d6..e5ced9c04c 100644 --- a/doc/fluid/api/layers.rst +++ b/doc/fluid/api/layers.rst @@ -1041,3 +1041,42 @@ box_coder .. autofunction:: paddle.fluid.layers.box_coder :noindex: +learning_rate_scheduler +======================= + +exponential_decay +----------------- + +.. autofunction:: paddle.fluid.layers.exponential_decay + :noindex: + +natural_exp_decay +----------------- + +.. autofunction:: paddle.fluid.layers.natural_exp_decay + :noindex: + +inverse_time_decay +------------------ + +.. autofunction:: paddle.fluid.layers.inverse_time_decay + :noindex: + +polynomial_decay +---------------- + +.. autofunction:: paddle.fluid.layers.polynomial_decay + :noindex: + +piecewise_decay +--------------- + +.. autofunction:: paddle.fluid.layers.piecewise_decay + :noindex: + +noam_decay +---------- + +.. autofunction:: paddle.fluid.layers.noam_decay + :noindex: + From c1843fd2ae84fc3f88b7eb40953dbed38a682083 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Wed, 13 Jun 2018 17:49:44 +0800 Subject: [PATCH 047/154] improve --- python/paddle/fluid/layers/control_flow.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 2279197717..4db085e9f5 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1219,20 +1219,21 @@ class IfElse(object): Examples: .. code-block:: python - limit = layers.fill_constant_batch_size_like( + limit = fluid.layers.fill_constant_batch_size_like( input=label, dtype='int64', shape=[1], value=5.0) - cond = layers.less_than(x=label, y=limit) - ie = layers.IfElse(cond) + cond = fluid.layers.less_than(x=label, y=limit) + ie = fluid.layers.IfElse(cond) with ie.true_block(): true_image = ie.input(image) - hidden = layers.fc(input=true_image, size=100, act='tanh') - prob = layers.fc(input=hidden, size=10, act='softmax') + hidden = fluid.layers.fc(input=true_image, size=100, act='tanh') + prob = fluid.layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) with ie.false_block(): false_image = ie.input(image) - hidden = layers.fc(input=false_image, size=200, act='tanh') - prob = layers.fc(input=hidden, size=10, act='softmax') + hidden = fluid.layers.fc( + input=false_image, size=200, act='tanh') + prob = fluid.layers.fc(input=hidden, size=10, act='softmax') ie.output(prob) prob = ie() """ From ce6394ed73ae5f9b6ad44c0e8dca762791001f2d Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Wed, 13 Jun 2018 17:50:27 +0800 Subject: [PATCH 048/154] Polish example --- paddle/fluid/operators/row_conv_op.cc | 2 +- paddle/fluid/operators/uniform_random_op.cc | 2 -- python/paddle/fluid/layers/nn.py | 30 ++++++++++++++------- python/paddle/fluid/layers/ops.py | 21 ++++++++++++++- python/paddle/fluid/layers/tensor.py | 15 +++++------ 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/paddle/fluid/operators/row_conv_op.cc b/paddle/fluid/operators/row_conv_op.cc index f4b540f1cb..d7286111fd 100644 --- a/paddle/fluid/operators/row_conv_op.cc +++ b/paddle/fluid/operators/row_conv_op.cc @@ -114,7 +114,7 @@ and a filter ($W$) of size $context \times d$, the output sequence is convolved as: $$ -out_{i, :} = \sum_{j=i}^{i + context} in_{j,:} \dot W_{i-j, :} +out_{i, :} = \\sum_{j=i}^{i + context} in_{j,:} \\cdot W_{i-j, :} $$ In the above equation: diff --git a/paddle/fluid/operators/uniform_random_op.cc b/paddle/fluid/operators/uniform_random_op.cc index 65525526c9..edd1baa4ac 100644 --- a/paddle/fluid/operators/uniform_random_op.cc +++ b/paddle/fluid/operators/uniform_random_op.cc @@ -88,8 +88,6 @@ class UniformRandomOpMaker : public framework::OpProtoAndCheckerMaker { void Make() override { AddOutput("Out", "The output tensor of uniform random op"); AddComment(R"DOC( -Uniform random operator. - This operator initializes a tensor with random values sampled from a uniform distribution. The random result is in set [min, max]. diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index fe60d8b78d..f1241d9479 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1718,10 +1718,14 @@ def layer_norm(input, h & = f(\\frac{g}{\\sigma}(a - \\mu) + b) - >>> import paddle.fluid as fluid - >>> data = fluid.layers.data(name='data', shape=[3, 32, 32], - >>> dtype='float32') - >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) + * :math:`a`: the vector representation of the summed inputs to the neurons + in that layer. + + * :math:`H`: the number of hidden units in a layers + + * :math:`g`: the trainable scale parameter. + + * :math:`b`: the trainable bias parameter. Args: input(Variable): The input tensor variable. @@ -1742,6 +1746,12 @@ def layer_norm(input, Returns: ${y_comment} + + Examples: + + >>> data = fluid.layers.data(name='data', shape=[3, 32, 32], + >>> dtype='float32') + >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) """ helper = LayerHelper('layer_norm', **locals()) dtype = helper.input_dtype() @@ -3262,12 +3272,6 @@ def row_conv(input, future_context_size, param_attr=None, act=None): """ ${comment} - >>> import paddle.fluid as fluid - >>> x = fluid.layers.data(name='x', shape=[16], - >>> dtype='float32', lod_level=1) - >>> out = fluid.layers.row_conv(input=x, future_context_size=2) - - Args: input (${x_type}): ${x_comment}. future_context_size (int): Future context size. Please note, the shape @@ -3278,6 +3282,12 @@ def row_conv(input, future_context_size, param_attr=None, act=None): Returns: ${out_comment}. + + Examples: + >>> import paddle.fluid as fluid + >>> x = fluid.layers.data(name='x', shape=[16], + >>> dtype='float32', lod_level=1) + >>> out = fluid.layers.row_conv(input=x, future_context_size=2) """ helper = LayerHelper('row_conv', **locals()) dtype = helper.input_dtype() diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 98f169e8f0..46c6fd686e 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -64,7 +64,6 @@ __all__ = [ 'logical_or', 'logical_xor', 'logical_not', - 'uniform_random', 'uniform_random_batch_size_like', 'gaussian_random', 'gaussian_random_batch_size_like', @@ -79,3 +78,23 @@ __all__ = [ for _OP in set(__all__): globals()[_OP] = generate_layer_fn(_OP) + +__all__ += ["uniform_random"] + +_uniform_random_ = generate_layer_fn('uniform_random') + + +def uniform_random(shape, dtype=None, min=None, max=None, seed=None): + kwargs = dict() + for name in locals(): + val = locals()[name] + if val is not None: + kwargs[name] = val + return _uniform_random_(**kwargs) + +uniform_random.__doc__ = _uniform_random_.__doc__ + "\n"\ ++""" +Examples: + + >>> result = fluid.layers.uniform_random(shape=[32, 784]) +""" diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 241bbe78bd..04efc40af5 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unlessf required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and @@ -57,12 +57,6 @@ def create_parameter(shape, NOTE: this is a very low-level API. This API is useful when you create operator by your self. instead of using layers. - >>> import paddle.fluid as fluid - >>> W = fluid.layers.create_parameter(shape=[784, 200], dtype='float32') - >>> data = fluid.layers.data(name="img", shape=[64, 784], - >>> append_batch_size=False) - >>> hidden = fluid.layers.matmul(x=data, y=W) - Args: shape(list[int]): shape of the parameter dtype(string): element type of the parameter @@ -74,7 +68,12 @@ def create_parameter(shape, default_initializer(Initializer): initializer for the parameter Returns: - the created parameter + the created parameter. + + Examples: + >>> W = fluid.layers.create_parameter(shape=[784, 200], dtype='float32') + >>> data = fluid.layers.data(name="img", shape=[64, 784], append_batch_size=False) + >>> hidden = fluid.layers.matmul(x=data, y=W) """ helper = LayerHelper("create_parameter", **locals()) if attr is None: From 674327a4b179286bbe7db2e2b96f8e59c0120562 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Wed, 13 Jun 2018 18:06:59 +0800 Subject: [PATCH 049/154] Polish several API --- paddle/fluid/operators/activation_op.cc | 3 +- python/paddle/fluid/layers/detection.py | 38 ++++++++++++------------- python/paddle/fluid/layers/ops.py | 28 ++++++++++++++++-- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 5e2fa56677..89bb1e2d4f 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -271,7 +271,8 @@ class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker { void Make() override { AddInput("X", "Input of HardShrink operator"); AddOutput("Out", "Output of HardShrink operator"); - AddAttr("threshold", "The value of threshold for HardShrink") + AddAttr("threshold", + "The value of threshold for HardShrink. [default: 0.5]") .SetDefault(0.5f); AddComment(R"DOC( HardShrink Activation Operator. diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 1e8dfbe521..d5f7e420dd 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -403,25 +403,6 @@ def ssd_loss(location, 5.3 Compute the overall weighted loss. - >>> import paddle.fluid.layers as layers - >>> pb = layers.data( - >>> name='prior_box', - >>> shape=[10, 4], - >>> append_batch_size=False, - >>> dtype='float32') - >>> pbv = layers.data( - >>> name='prior_box_var', - >>> shape=[10, 4], - >>> append_batch_size=False, - >>> dtype='float32') - >>> loc = layers.data(name='target_box', shape=[10, 4], dtype='float32') - >>> scores = layers.data(name='scores', shape=[10, 21], dtype='float32') - >>> gt_box = layers.data( - >>> name='gt_box', shape=[4], lod_level=1, dtype='float32') - >>> gt_label = layers.data( - >>> name='gt_label', shape=[1], lod_level=1, dtype='float32') - >>> loss = layers.ssd_loss(loc, scores, gt_box, gt_label, pb, pbv) - Args: location (Variable): The location predictions are a 3D Tensor with shape [N, Np, 4], N is the batch size, Np is total number of @@ -465,6 +446,25 @@ def ssd_loss(location, Raises: ValueError: If mining_type is 'hard_example', now only support mining \ type of `max_negative`. + + Examples: + >>> pb = fluid.layers.data( + >>> name='prior_box', + >>> shape=[10, 4], + >>> append_batch_size=False, + >>> dtype='float32') + >>> pbv = fluid.layers.data( + >>> name='prior_box_var', + >>> shape=[10, 4], + >>> append_batch_size=False, + >>> dtype='float32') + >>> loc = fluid.layers.data(name='target_box', shape=[10, 4], dtype='float32') + >>> scores = fluid.layers.data(name='scores', shape=[10, 21], dtype='float32') + >>> gt_box = fluid.layers.data( + >>> name='gt_box', shape=[4], lod_level=1, dtype='float32') + >>> gt_label = fluid.layers.data( + >>> name='gt_label', shape=[1], lod_level=1, dtype='float32') + >>> loss = fluid.layers.ssd_loss(loc, scores, gt_box, gt_label, pb, pbv) """ helper = LayerHelper('ssd_loss', **locals()) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 46c6fd686e..f0abd3089d 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -40,7 +40,6 @@ __activations__ = [ 'relu6', 'pow', 'stanh', - 'hard_shrink', 'thresholded_relu', 'hard_sigmoid', 'swish', @@ -92,9 +91,32 @@ def uniform_random(shape, dtype=None, min=None, max=None, seed=None): kwargs[name] = val return _uniform_random_(**kwargs) -uniform_random.__doc__ = _uniform_random_.__doc__ + "\n"\ -+""" + +uniform_random.__doc__ = _uniform_random_.__doc__ + "\n" \ + + """ Examples: >>> result = fluid.layers.uniform_random(shape=[32, 784]) """ + +__all__ += ['hard_shrink'] + +_hard_shrink_ = generate_layer_fn('hard_shrink') + + +def hard_shrink(x, threshold=None): + kwargs = dict() + for name in locals(): + val = locals()[name] + if val is not None: + kwargs[name] = val + return _hard_shrink_(**kwargs) + + +hard_shrink.__doc__ = _hard_shrink_.__doc__ + "\n" \ + + """ +Examples: + + >>> data = fluid.layers.data(name="input", shape=[784]) + >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3) +""" From c58ba827bb26435628d2e65e632cb98ad760dae8 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Wed, 13 Jun 2018 21:15:16 +0800 Subject: [PATCH 050/154] update --- paddle/fluid/operators/conv_transpose_op.cc | 2 +- paddle/fluid/operators/crf_decoding_op.cc | 3 --- paddle/fluid/operators/roi_pool_op.cc | 2 ++ python/paddle/fluid/layers/io.py | 5 +++-- python/paddle/fluid/layers/nn.py | 14 ++++++-------- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/paddle/fluid/operators/conv_transpose_op.cc b/paddle/fluid/operators/conv_transpose_op.cc index 0b363f5c43..2e9e957ebd 100644 --- a/paddle/fluid/operators/conv_transpose_op.cc +++ b/paddle/fluid/operators/conv_transpose_op.cc @@ -156,7 +156,7 @@ Parameters(strides, paddings) are two elements. These two elements represent hei and width, respectively. The input(X) size and output(Out) size may be different. -Example: +For an example: Input: Input shape: $(N, C_{in}, H_{in}, W_{in})$ Filter shape: $(C_{in}, C_{out}, H_f, W_f)$ diff --git a/paddle/fluid/operators/crf_decoding_op.cc b/paddle/fluid/operators/crf_decoding_op.cc index a8d7831122..c27befe114 100644 --- a/paddle/fluid/operators/crf_decoding_op.cc +++ b/paddle/fluid/operators/crf_decoding_op.cc @@ -53,17 +53,14 @@ sequence of observed tags. The output of this operator changes according to whether Input(Label) is given: 1. Input(Label) is given: - This happens in training. This operator is used to co-work with the chunk_eval operator. - When Input(Label) is given, the crf_decoding operator returns a row vector with shape [N x 1] whose values are fixed to be 0, indicating an incorrect prediction, or 1 indicating a tag is correctly predicted. Such an output is the input to chunk_eval operator. 2. Input(Label) is not given: - This is the standard decoding process. The crf_decoding operator returns a row vector with shape [N x 1] whose values diff --git a/paddle/fluid/operators/roi_pool_op.cc b/paddle/fluid/operators/roi_pool_op.cc index a6247a467a..3dec59e247 100644 --- a/paddle/fluid/operators/roi_pool_op.cc +++ b/paddle/fluid/operators/roi_pool_op.cc @@ -149,7 +149,9 @@ The operator has three steps: 1. Dividing each region proposal into equal-sized sections with the pooled_width and pooled_height + 2. Finding the largest value in each section + 3. Copying these max values to the output buffer ROI Pooling for Faster-RCNN. The link below is a further introduction: diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 65b9c68885..76ef82ddb0 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -109,8 +109,6 @@ class BlockGuardServ(BlockGuard): class ListenAndServ(object): """ - ListenAndServ layer. - ListenAndServ is used to create a rpc server bind and listen on specific TCP port, this server will run the sub-block when received variables from clients. @@ -121,6 +119,9 @@ class ListenAndServ(object): fan_in(int): how many client are expected to report to this server, default: 1. optimizer_mode(bool): whether to run the server as a parameter server, default: True. + Returns: + None + Examples: .. code-block:: python diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 40134ed42f..4fb3ac4a92 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -806,7 +806,7 @@ def crf_decoding(input, param_attr, label=None): label(${label_type}): ${label_comment} Returns: - ${viterbi_path_comment} + Variable: ${viterbi_path_comment} """ helper = LayerHelper('crf_decoding', **locals()) transition = helper.get_parameter(param_attr.name) @@ -828,7 +828,7 @@ def cos_sim(X, Y): Args: X (Variable): ${x_comment} - Y (Variable): ${x_comment} + Y (Variable): ${y_comment} Returns: Variable: the output of cosine(X, Y). @@ -1036,9 +1036,9 @@ def chunk_eval(input, excluded_chunk_types (list): ${excluded_chunk_types_comment} Returns: - tuple: tuple containing: (precision, recall, f1_score, - num_infer_chunks, num_label_chunks, - num_correct_chunks) + tuple: tuple containing: precision, recall, f1_score, + num_infer_chunks, num_label_chunks, + num_correct_chunks """ helper = LayerHelper("chunk_eval", **locals()) @@ -3050,8 +3050,6 @@ def nce(input, def transpose(x, perm, name=None): """ - **transpose Layer** - Permute the dimensions of `input` according to `perm`. The `i`-th dimension of the returned tensor will correspond to the @@ -3918,7 +3916,7 @@ def roi_pool(input, rois, pooled_height=1, pooled_width=1, spatial_scale=1.0): spatial_scale (float): ${spatial_scale_comment} Default: 1.0 Returns: - roi_pool (Variable): ${out_comment}. + Variable: ${out_comment}. Examples: .. code-block:: python From 9b13b4c0d295544b6eaed50ad77657b20d3736b6 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 13 Jun 2018 17:32:04 +0800 Subject: [PATCH 051/154] Add doc --- doc/fluid/api/layers.rst | 19 +++++++++++++++++++ python/paddle/fluid/layers/nn.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/fluid/api/layers.rst b/doc/fluid/api/layers.rst index f78e6db326..4be56791b1 100644 --- a/doc/fluid/api/layers.rst +++ b/doc/fluid/api/layers.rst @@ -361,6 +361,12 @@ conv2d .. autofunction:: paddle.fluid.layers.conv2d :noindex: +conv3d +------ + +.. autofunction:: paddle.fluid.layers.conv3d + :noindex: + sequence_pool ------------- @@ -385,6 +391,12 @@ pool2d .. autofunction:: paddle.fluid.layers.pool2d :noindex: +pool3d +------ + +.. autofunction:: paddle.fluid.layers.pool3d + :noindex: + batch_norm ---------- @@ -403,6 +415,13 @@ conv2d_transpose .. autofunction:: paddle.fluid.layers.conv2d_transpose :noindex: +conv3d_transpose +---------------- + +.. autofunction:: paddle.fluid.layers.conv2d_transpose + :noindex: + + sequence_expand --------------- diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 3a4f93929f..888245fc9e 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1722,7 +1722,7 @@ def pool2d(input, if not isinstance(use_cudnn, bool): raise ValueError("use_cudnn should be True or False") - l_type = 'conv2d' + l_type = 'pool2d' helper = LayerHelper(l_type, **locals()) dtype = helper.input_dtype() From 76129f03314afb26acae18e7a5838612f6fb28f0 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 13 Jun 2018 22:16:23 +0800 Subject: [PATCH 052/154] update comment --- python/paddle/fluid/layers/control_flow.py | 7 +++---- python/paddle/fluid/layers/nn.py | 5 +++-- python/paddle/fluid/layers/tensor.py | 7 +++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 7999ee0f80..feac42d94e 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -893,12 +893,11 @@ def create_array(dtype): """ **Create LoDTensor Array** - This function creates an array of type :math:`LOD_TENSOR_ARRAY` using the - LayerHelper. It is mainly used to implement RNN with array_write, array_read - and While. + This function creates an array of LOD_TENSOR_ARRAY . It is mainly used to + implement RNN with array_write, array_read and While. Args: - dtype (int|float): The data type of the elements in the array. + dtype (int|float): The data type of the elements in the lod_tensor_array. Returns: Variable: The lod_tensor_array variable storing the elements of data type. diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 80452a1e8b..3f3b7e20ef 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1618,8 +1618,9 @@ def batch_norm(input, 1. NHWC `[batch, in_height, in_width, in_channels]` 2. NCHW `[batch, in_channels, in_height, in_width]` - Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift - `_ for more details. + Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing + Internal Covariate Shift `_ + for more details. :math:`input` is the input features over a mini-batch. diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 6ce486d70d..6b7f69807b 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -40,15 +40,14 @@ __all__ = [ def create_tensor(dtype, name=None, persistable=False): """ - **Create a Tensor with certain data type and name** + **Create a Tensor** Args: dtype (string): 'float32'|'int32'|..., the data type of the created tensor. - name (string|None): The name of the created tensor, if not set, + name (string, Default: None): The name of the created tensor, if not set, the name will be a random unique one. - persistable (bool): Set the persistable flag of the create tensor, - default value is False. + persistable (bool, Default: False): Set the persistable flag of the create tensor. Returns: Variable: The tensor variable storing the created tensor. From 0ac03ae723cf11b95b45eccbfc6923ece65d12ff Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 13 Jun 2018 23:03:30 +0800 Subject: [PATCH 053/154] fix --- python/paddle/fluid/layers/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index fde8a3154f..71df672fe4 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4289,7 +4289,7 @@ def random_crop(x, shape, seed=None): seed_out = helper.create_tmp_variable(dtype="int64") helper.append_op( type="random_crop", - inputs={"X": input, + inputs={"X": x, "Seed": seed}, outputs={"Out": out, "SeedOut": seed_out}, From a49ee22e319cad76cc7d0b5efd10113d302d1b8c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 13 Jun 2018 23:21:44 +0800 Subject: [PATCH 054/154] fix a bug in prefetch --- paddle/fluid/operators/detail/grpc_server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/operators/detail/grpc_server.cc b/paddle/fluid/operators/detail/grpc_server.cc index 2d34f85838..14152bded8 100644 --- a/paddle/fluid/operators/detail/grpc_server.cc +++ b/paddle/fluid/operators/detail/grpc_server.cc @@ -169,7 +169,7 @@ class RequestPrefetch final : public RequestBase { auto scope = request_->GetMutableLocalScope(); auto invar = scope->FindVar(in_var_name); - framework::Variable* outvar = scope->FindVar(out_var_name); + framework::Variable* outvar = scope->Var(out_var_name); request_handler_->Handle(in_var_name, scope, invar, &outvar, out_var_name); From d827c6e87ac5b27124b30d48d66661073cbbd27a Mon Sep 17 00:00:00 2001 From: "Yang Yang(Tony)" Date: Wed, 13 Jun 2018 11:46:01 -0700 Subject: [PATCH 055/154] Dynamic Graph first prototype (#11415) --- doc/survey/dynamic_graph.md | 2 +- paddle/contrib/CMakeLists.txt | 1 + paddle/contrib/dynamic/CMakeLists.txt | 25 ++ paddle/contrib/dynamic/README.md | 246 +++++++++++++++++ paddle/contrib/dynamic/computation_graph.png | Bin 0 -> 96637 bytes paddle/contrib/dynamic/function.h | 130 +++++++++ paddle/contrib/dynamic/tape.cc | 265 +++++++++++++++++++ paddle/contrib/dynamic/tape.h | 62 +++++ paddle/contrib/dynamic/test_tape.cc | 61 +++++ paddle/contrib/dynamic/variable.cc | 33 +++ paddle/contrib/dynamic/variable.h | 85 ++++++ paddle/fluid/framework/operator.cc | 2 + paddle/fluid/framework/scope.h | 5 +- 13 files changed, 914 insertions(+), 3 deletions(-) create mode 100644 paddle/contrib/dynamic/CMakeLists.txt create mode 100644 paddle/contrib/dynamic/README.md create mode 100644 paddle/contrib/dynamic/computation_graph.png create mode 100644 paddle/contrib/dynamic/function.h create mode 100644 paddle/contrib/dynamic/tape.cc create mode 100644 paddle/contrib/dynamic/tape.h create mode 100644 paddle/contrib/dynamic/test_tape.cc create mode 100644 paddle/contrib/dynamic/variable.cc create mode 100644 paddle/contrib/dynamic/variable.h diff --git a/doc/survey/dynamic_graph.md b/doc/survey/dynamic_graph.md index 553a9dbe15..6b80b014b1 100644 --- a/doc/survey/dynamic_graph.md +++ b/doc/survey/dynamic_graph.md @@ -171,7 +171,7 @@ Pytorch chooses immediate evaluation. It avoids ever materializing a "forward gr ## What can fluid learn from them? -TBD +Please refer to `paddle/contrib/dynamic/`. # Appendix diff --git a/paddle/contrib/CMakeLists.txt b/paddle/contrib/CMakeLists.txt index 4b19256ef4..75d0170567 100644 --- a/paddle/contrib/CMakeLists.txt +++ b/paddle/contrib/CMakeLists.txt @@ -14,3 +14,4 @@ # add_subdirectory(inference) +add_subdirectory(dynamic) diff --git a/paddle/contrib/dynamic/CMakeLists.txt b/paddle/contrib/dynamic/CMakeLists.txt new file mode 100644 index 0000000000..0acef17d6a --- /dev/null +++ b/paddle/contrib/dynamic/CMakeLists.txt @@ -0,0 +1,25 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if(APPLE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move") +endif(APPLE) + +cc_library(tape_variable SRCS variable.cc DEPS ${FLUID_CORE_MODULES}) +cc_library(tape SRCS tape.cc DEPS ${FLUID_CORE_MODULES} ${GLOB_OP_LIB} tape_variable) + +cc_test(test_tape + SRCS test_tape.cc + DEPS tape tape_variable) diff --git a/paddle/contrib/dynamic/README.md b/paddle/contrib/dynamic/README.md new file mode 100644 index 0000000000..a056877093 --- /dev/null +++ b/paddle/contrib/dynamic/README.md @@ -0,0 +1,246 @@ +# Dynamic Graph on Fluid + +PaddlePaddle Fluid is targeting the autodiff without tape, which, however, is very challenging and we are still way from there. DyNet and PyTorch provide a good design idea, the *tape*, that significantly eases the challenge. Also, DyNet provides a C++ API that is as convenient as Python but with higher efficiency and could conveniently integrate with industrial/production systems. This package, `tape`, combines the good of + +1. tape from PyTorch and DyNet +2. C++ API and core from DyNet +3. rich set of operators from PaddlePaddle + +## Overview + +We can implement Dynet-like Tape(See this survey) by wrapping Paddle Fluid's `Operator` +and `Variable`. + +The user API is straight forward since + +1. it is imperative. And it uses host language's control flow logic. +1. it avoids extra concepts such as `Scope` and `Executor`. + +All of these benefits come at the cost of just adding one line `reset_global_tape` +at every iteration. + +## Code Structure + +In short, the `Tape` contains a vector of `OpHandle`s. And an `OpHandle` contains its +`type`, the pointers to the `Variable`s, and necessary attributes. + +```c++ +class Variable { +public: + VriableHandle Grad(); // returns its gradient variable +private: + framework::VarDesc desc_; // compile time infershape, necessary for lazy execution + framework::Variable var_; // run time variable, holds data memory +}; + +using VariableHandle = shared_ptr; + +struct OpHandle { + string type_; + map> inputs_; + map> outputs_; + AttributeMap attrs_; +}; + +class Tape { +public: + void AddOp(OpHandle); // add op + void Forward(); // execute the tape_ + void Backward(); // execute the backward of the tape_ +private: + vector tape_; +}; +``` + +We uses `Function` to indicate layers. It takes care of parameter +initialization and `AddOp` to the Tape when it is called. + +```c++ +class Linear { + public: + Linear(int in_dim, int out_dim, const std::string &act) + : w_(new Variable("LinearWeight")), + b_(new Variable("LinearBias")), + act_(act) { + Tape init_tape; + + std::string initializer = "fill_constant"; + framework::AttributeMap attrs; + attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{in_dim, out_dim}; + attrs["value"] = 1.0f; + init_tape.AddOp(initializer, {}, {{"Out", {w_}}}, attrs); + + attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{out_dim}; + attrs["value"] = 1.0f; + init_tape.AddOp(initializer, {}, {{"Out", {b_}}}, attrs); + + init_tape.Forward(); + } + + VariableHandle operator()(VariableHandle input) { + VariableHandle pre_bias(new Variable("linear")); + get_global_tape().AddOp("mul", + {{"X", {input}}, {"Y", {w_}}}, + {{"Out", {pre_bias}}}, + {{"x_num_col_dims", 1}, {"y_num_col_dims", 1}}); + VariableHandle pre_act(new Variable("linear")); + get_global_tape().AddOp("elementwise_add", + {{"X", {pre_bias}}, {"Y", {b_}}}, + {{"Out", {pre_act}}}, + {{"axis", 1}}); + VariableHandle post_act(new Variable("linear")); + get_global_tape().AddOp(act_, + {{"X", {pre_act}}}, + {{"Out", {post_act}}}, + {}); + return post_act; + } + + std::vector Params() { return {w_, b_}; } + + private: + VariableHandle w_; + VariableHandle b_; + std::string act_; +}; +``` + +## User API + +```c++ +// Model function +paddle::tape::Linear linear1(3, 3, "relu"); // init weight and bias +paddle::tape::Linear linear2(3, 3, "relu"); // init weight and bias +paddle::tape::Mean mean; + +// Optimizer +paddle::tape::SGD sgd(0.001); + +// Data Feeder +paddle::tape::Fill data_feeder(...); +VariableHandle input(new paddle::tape::Variable("input")); + +for (int i = 0; i < 2; ++i) { + reset_global_tape(); + + data_feeder(input); + + auto loss = mean(linear2(linear1(input))); // compile time InferShape & InferVarType + LOG(INFO) << loss.value(); // Run forward up to loss + + // Run backward, store gradient of w at w->Grad() + get_global_tape.Backward(loss); + + // Update w + sgd(linear1.Params()); + sgd(linear2.Params()); +} +``` + +
+ +digraph G { + + subgraph cluster_0 { + node [shape=record,style=filled]; + style=filled; + color=lightgrey; + linear1 [label="{type: mul | {input | {X: before_mul1 | Y: weight1}} | {output | Out: before_bias1}}"]; + elementwise_add1 [label="{type: elementwise_add | {input | {X: before_bias1 | Y: bias1}} | {output | Out: before_act1}}"]; + relu1 [label="{type: relu | {input | {X: before_act1 }} | {output | Out: after_act1}}"]; + + linear1 -> elementwise_add1->relu1; + label = "forward tape"; + } + + linear1:before_mul1->before_mul1 + linear1:weight1->weight1 + linear1:before_bias1->before_bias1 + + elementwise_add1:bias1->bias1 + elementwise_add1:before_bias1->before_bias1 + elementwise_add1:before_act1->before_act1 + + relu1:before_act1->before_act1 + relu1:after_act1->after_act1 + + subgraph cluster_1 { + node [shape=record,style=filled]; + style=filled; + color=lightgrey; + linear1_grad [label="{type: mul_grad | {input | {X: before_mul1 | Y: weight1| Out_grad: before_bias1_grad}} | {output |{X_grad: before_mul1_grad | Y_grad: weight1_grad}}}"]; + + elementwise_add1_grad [label="{type: elementwise_add_grad | {input | Out_grad: before_act1_grad} | {output |{X_grad: before_bias1_grad | Y_grad: bias1_grad}}}"]; + + relu1_grad [label="{type: relu_grad | {input | Out_grad: after_act1_grad} | {ouput | {X_grad: before_act1_grad }}}"]; + + linear1_grad -> elementwise_add1_grad ->relu1_grad [dir=back]; + label = "backward tape"; + } + + relu1_grad:after_act1_grad->after_act1_grad + relu1_grad:before_act1_grad->before_act1_grad + + elementwise_add1_grad:before_act1_grad->before_act1_grad + elementwise_add1_grad:before_bias1_grad->before_bias1_grad + elementwise_add1_grad:bias1_grad->bias1_grad + + linear1_grad:before_mul1->before_mul1 + linear1_grad:weight1->weight1 + linear1_grad:before_bias1_grad->before_bias1_grad + linear1_grad:before_mul1_grad->before_mul1_grad + linear1_grad:weight1_grad->weight1_grad + + + subgraph cluster_2 { + node [shape=record]; + label = "Linear1"; + weight1 + bias1 + } + + weight1 -> weight1_grad [ label="Grad()", style="dashed" ]; + bias1 -> bias1_grad [ label="Grad()", style="dashed"]; + + + +} +
+ +![Image](https://github.com/tonyyang-svail/Paddle/blob/cpp_tap/paddle/contrib/dynamic/computation_graph.png) + +## Code Reuse + +We want to stay close to Paddle Fluid as much as possible. + +### Reuse All Operators + +As all Ops are registered at `OpInfoMap`, the effort of adding a new `Function` +is about 10 lines of code, similar to expose an operator to Python. + +### Reuse Compile Time InferShape and InferVarType + +Note that all the symbolic information is stored at `tape::Varaible::desc_`, instead +of `ProgramDesc.block.vars`, we create a temporary `BlockDesc` to do `InferShape` and +`InferVarType` every time we `AddOp` to the tape. + +### Reuse Operator::Run + +We use smart pointer, instead of `Scope`, to manage memory. So we create a temporary +`Scope` for every `Operator::Run()`. + +## Possible Feature + +### Release Memory on Backward + +We can release memory aggressively. During backward, we can delete the OpHandle once +we have finished its backward. Since all the variable is managed by smart pointer, the +memory is automatically released when its `ref_count` goes to 0. + +### Kernel Fusion + +As a symbolic representation of the Tape is constructed first before the actual +execution, it would be possible to perform graph optimization. One use case is kernel +fusion. diff --git a/paddle/contrib/dynamic/computation_graph.png b/paddle/contrib/dynamic/computation_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..6cf5ead735d5d18b204b079771e53d44483cf016 GIT binary patch literal 96637 zcmcG01yq&Y_a-VJk^%xsOM`Svie5?SZlt@rL6J);-5}lF4I&^?UgFXv-5}j_uHW~Y z`L8vzX3eZM<62(0Z=L<_v(Mhoe)c|uD9A}*p%bGcAt7N&NkWv6kRFhO|7U3T!4VdG z9BT06fzxX#6*M%onI-u}@b#JVTMcJrJ5y)3_l_n=X0~=VCM-@yjwU9yPUd#b`w!Yh zkdP>mq#$A{?x{O-e%{(Ht)d5ReqSDRAy+*7^k{%=%G!8d=sYT1Y-^6#MCJyj)1hMD zm|*2JPSRuDU>PbzyqHXNyfKkd@&02}2m1#YQgV|p@g8@pL3WHve&_qkr?Xsqjx7Vq zq#=~_M#onY^&EAs+whu<8V}*Ssd%S9FZb~se0GmK)}1WHB_tjJk&%98#=G1@{NnU5 zivaw1HASg~_#TV(|KRJG{GXQkc~1-+$4}@g+S*=lO|!}}pufd0ed5KfUT5*rxfU!a zi`?ei$vr*da8)oaDd!Z5*gH)z6k;;6ZoecXBn8D^Rw&?v0xwhz{6I4KVE(`Rsyq%W z`XY1ed?6r|E-7xJ8y3BZb(w1FHy=?S=<-$s?R6ur>auDp}_lfb}tAx&I6iz5M9 zEw~9d+Yzf`SEE$j=M1a(^~%_}IfWm-nwl8>Y=A(62%!~v)oz$&PC@i17lN29({BMA z3Q;c4z;Sz}7@tpzvsuzNOOkeFIgU$_k^E*3VXXM84?kpXRQ$A(>La=!{5)*3B-QsF z!jbsfXG!fPxy+vKb$79zmrydDm6um@@!&Unfg z+Wc?FWTV3O66!xpD5Yy`4&er@6zMbeeSe%sc6=Xz-+F=;}{Wb8RhUY;TU zk*phA%JenKH-^cgn4PSvqYkR%)$4Bvn89p?6mlk$Yx9bQfOK!7ctvDo$>}cb)e7 zS~gXt(V=rDk=^*4frIg63|ce2bn%jLmqwK}5Uo=2Awnr_f!k!#zOJewn=y)dRQD=F zuGT0>UCy{=XqRKi@p4)=KH(p3%03zy$HD@LPn!r8Rc!$-@c|D6%(cY_bC(*2`@h^V zd7Gs$X-7Sz32gK}i-)?MVXf3oQTG*n`BNWO#71tp6v7!`eA!^q_C|$R#w!0+sJoDq z6*JsoMRJo&?h`%v#V^u$ajQe9Z-1dKDtI%whlI!>Q0TVBH%hwPvoyta(GBDd#6_PYW1i@@pi_p9|7BM6AV>J||q6 zLw}SPG)B2m$k0C-4A!)PIHguZCI4<<>bsSmQ*2eGL+8Gg8u=%sP&-?mQ3W-t`Yot4 z9B#TBmJ~hi$}eMqr4`NfG6v)@SC=s9=NG`Z#Mt-9#yGv5e4v=P)BWh)|)#_6}yZLyvTyczDT+ncQ3 zUnQG~4xz_u4l1AZ>ivna9pWrglGn7vLefSN`aNbw>CH%sObX#}CL0=jeEjgbu&x+* z%6AFLFc_>aHKBhMo0Ok@ZoFf&A9;toQ23cmab6;afItzl2nT#X|!s{hKZIi4x7OCNQ9X%!@ZQR9XFO?bmQ^JC=S0q+X4B zTU}XMtZgPB9bK*vY%A4is8WO3wT^8vpp`!w0-=V(`ttRjel2e}v7o*GWU?0gzQ$ zG5z7(0MA7y^eOq5u+Va&SM7g1m!0m|zQi66vVQIzHB_k%G3Sp~DQ%NWd4lol@IJdEB&gHBn9B&ON04P!kx0cUd4ewk!xo6k|qo08Ld)MiOz_&Af$5e2V+tq1FCY1=rh-YAZse(5XflH;umwHBQan$ z9arz+mO`KC>bnYHi8_XoJJ$o85(&L<1*+E4?jN|D?_8Z8*qymd6SM5OV#sS*Nw_Pw z*ik-DUagqOP=G>npygaF=iiRts?tE3Q4XryETo-F3aUi$Ed-$S_MzXSIA;HXIsGCfulG zxkwGN9%_lkH=E*ShYie@bWuuB$B-5K596jQP$+VY;tL!T8mbWwZ))aiDV7UtHo-+rJGSAZumJ<1wKeur(AO$zYuK9qDw%-g9rZZuK* zv#gpD)!6vzkxPwRE{guSLRb@w?1{FEMMn12{Fct{VK;U#a~xEj! z&frXPx6gbAS>_|Vi`7IK4{?MQyyWHWuZ@iY@oWxa7z}uzxck*=`IHI7yf5eL$Jd{m z`yg@ws@FBi;nsF`k&=Vm-;}B!93e6Tb`D5!jB;=`xmU5wfM6@jzV?~&|L|vZ9YsP2 zrISuhY^^Zgu5>g!Jg0Lj=o9W52Zvvwbd`Xx@;^iI*!+pD2)_9r#+-lsh+WGe>NXy@ zkgiw=X1P9;X3KZk_+ATu>Eg1cP#44_VwBR&T?Tn32DauV?bQ#Q( zIz1&8GA!tNN06$88PEH z$=*o4qA9hxuJJIAsj>JIt#EG{_(BMvcQ)dBF8@mfR?^A_=Plm}Z43ur2DW>5xU>Qa_6X7G1~U){6Zt@(EU;9}?7`GfQgR~_H30@A%S z;Ly4q_(Fi4{Wg{W*8;l~%@I|iS+_g~?^)cKGgWEgj#@fY(u@hr?^G=I`Z|WZXcL&JU62qrBVFWG!u{ZkL*GcNBZUvHz zL9=PTkU7^FbHS5OyCaiqMq!r3F)t{m>F|vrBPDY~6H6Ewl&utgfalV?t zgrxS(>{_ct->baiCLCDNNk5JVj3ho5Qq+rx7EoBVKh%qdNW@3-IM>U z)k$Ivd|yR?xyy7;q9}ZES3pRrJE_H2-H3w`HS*&NM)>^f&Zh8G-nKLoFT0_oQzFz^ zEm-!-P$=~ac7D9WD2AEK^o2E59L1H92#hbMvR11%i8W=#KR5%1>73+!n~reqpe+99(Xf*Voj*gm^aW1tmaANKz1c(U&Dsw~vNCIT!YPtF1@-(Q%_}WNs0GDd zGQKZUTlcdl~gbLoz=DHPJxhkUlPm;LHbP7KKW4)H$N2Hv{} zfd`E1_!%G7E%ebA*=dZq+nX=jiG)>t@)MB`TbZf||l*ArNE1@>S~O#B1VQ`7j)q`3h*}^f zUG2BGmjn$KqySewZB_2ls8fOw;83F53^(U4)2v=Qf0Lb^O&rl<&OPLfKrqX%NxV)J z@kHcl_=dhwnwevK)P*vFjjJ`990Y}AgjJ3r@3?9A3cT!|%)5%@ag>7Fe|tly%}7=<+TU(SexIZO3< z%EsZO?4a&5DkW6vbhYZKDcGt5ulAs+)6v!|>E~2D6EjOS$veAkk*;RNt&NT2qg-_o zrXZpKuyv&b0nA5z{+w;ynQ`G4S>*d6?_bioI-h%1PNRWwxgy`zj3klTHnX302>ni( z-6dqPcvqG_`O1?4x8g^L0kg#SXEQCXt>U8_=x>*O`ivW7caJWp@S-yNEci%01rOtS z6H1p38dyonEh;WvKd;2{LOix3yf}7{h@Yi=8T!MZmYA`Wv^J+VB8XvV!`d#=J*F^q zM)!2=c&~YXZ>pL;rZ3jZ0Nhl-8i!X(7r=!EklH9rGPh`JZ7_w;Ze9q>s7r-KyDr=X zL9fN|gk_C$3v8EEdy7zzJ1Zx_%XIsjfYSZ@lHY27D)NxZ3=$83Lb)N3WuamOujQXP zgmDYCt7;gMkkexu^pGDmx&7>h0#sseru5umtRm)fxVL=_ns@^pSl$0pU z#HU42RcWup8j;f_%pqj&L0qWB^xG+UaJTh57F?TGE75g(@+Y7`; zKT{!22^XLexZWLF%HpPpDtt!^#2%oH6SU%4w-^7s(5N8$u=|WIDi%`F4qHxclaSg@ zZpr*I^DV+lAr953H3{GLejb7n{N<%jYtG)>^cH3^lDY`O0Fo*u%o5dsu6&`=-k6dm z4;^?g_$;C-IT}wSB6{@V)b)vtWriFzWrD&CM zZ^`Aoyn@TSM)w{aua8&qxxFj};eScEO{GwD(cZ$3SovmT~sOXT3?(epYusIm6q$K3~rt@+>HxP@G3R3*6M^nJN)bl0dCr%U|{K#0H zdAd2TV42?bu3sys%`C}yW>tW!gXSa-ugWt)a=2}SnSnm~&z zQ~0s}IFbbm2ow21ulQM>)t?Kk;FgeA`B}8VlO8!@8C)LA7W?jYOMs+7iMHroT4YR+ z#3ssxNITRV#>(#6T65syn&3-$WX@PpQvD`o5f9C;4wZ5DyYlm8ptd5BHF_pxR>hoS z&nsag8;t(zk3b~Uy|^|inmX>trE@~x&!<10#qWBVf`@qUnOSNk%)G~!>-6eqNN_*& zMOni}Rv2s3Mi!3Wc~?mo3X;ENusP4NGq0V*(W`a;+4&jM#4*A1HMQyu`8O->{x7sw-0#mP(@Ys$?9f)hVRL^y1viJIAF^4+?Rqd2$_FvpAhLfM^U25 z#4~GF8TY*<(*1r(2b_Txr4~mA#_I>U9%7bX$FVVFo!1395!{r8ihJx(lQd5+) z+FH$RS!aohHP(P-GT!;o7vjj(X9{`0HczINw4y`g8XhDrZaL>Ree>0i&W?t|@m=K( z<~pZj$|tr@E^}oP$ZrfY6($>Qn4BRQl6YYNxZn+&_v}r|>XYX!98sKAUCp zJq(K=;cL78({EaRAvrR#DR&+M>^Ajr);MNAPS5WLNylh$(vK=7Dr@-3BZ>Q5LP2=)121wiv@zwsP+zR4b3cG`5K$2-h4EKCd)KbA`v zWSD%{B09YpLGF;~)Dx4Pq(O2cci<`E_{TpLUmFur_pTIt!%9Q85vQefaR9{k)s4?I>;kdX2B0 zBV}+x*KH1qZ??jq^FG|xhCCa*Ezqt8xt05@j36K z!3jOD(*9LL-5wI@E&eesfwc{V?lovNyOzStb*fDV4}&B3Ym>7; z{bzg1EW>9l$*j(LiVE$U3mEq1WP(kzDo1De0a8b5uuo}`#B%~nqSDi|ye8+szE`B5 z)5aIL?B-jx%bNEjL@#0bw|^1<#VQ}aD(|zU97e>px3e?b=5s}JM+Us~I3}#f)1Fk~ zt&~U={hR0bA*=7?W{!nT%?OLj%k>)_P54RUO+1!+A|B$9>$G~F&?e|mg=S@`45jM> zlXduIr7FyyA|LzxYj5_rB~-dQ8a_sjU2l+Zf?rfv*tnK#RR%uxefGV83PsO5aYX(^ zeXgmin|JJInb2{<+TH@j_?uEo6@UIWA&V_sbwE${@4wJ2q3eULV)b(PrZf5H^^CiV zjA4;0Yk2NGq*u8efs`GEOCGnUGj@Kfaq54kz=G>RTqSs4eaPmuldRF_pslslRz3PVHg5u?#wmUho=O}6{BDf3LXD?<2LWH4i?5y$M@ia-_^{5=`xGThu4`MO&^t= zU~h?nx|U|_+HdM+Y_(h59pvRd_pbuAzt<%K;rHVoNbzUjpQ{hW_?lgJo0dM2kb579 zMW6~stC!VUO;oWiIeVXXQTPJuO844pWb?b|&s5mxd`Qvq=jRJeB7mqyu>r$6YJTx& zm96!7WNW-w^rBC`Y0lO==rNMP5JGEuCnqQMbFMQ72kf{$J4O^<*N45D>Hwqx_5$62 zZXiQc6fZ>RWK6k!!MC|Jq8wT6;WUok_$B`*~6+n-R z=UVp$3aY_m14+EM3iI1;Ps)uh(Lt#|tmeLfL3b!Yqt4#-(J&uz#75hFq(qs=rHRV9 zD&Hq8dA;!R8+?>~+3^l8gm!g9u8&p3K{fE#yfwhY`J4AuWsAo}fK|zhT&(c5UksAi zwJ55p!U=#$J)_i;4UCpOJ{Q_e2bIJ1BNulevVcj0HXTv4mz`_2JP!Gnyo|rMh9_&yT=uN9FC=i8SYCiGQT7^?o%r)Gbc?VSQXE?}Ol!fI{`lnzopPNFE{? zU^5KIXO2D><=3#Z#FM+a37TDGUoc+E*OAETV)+Sw#cs_-`cpK>I;XlT&M~OVc3Fxl z3Q+ajva+8F)f`9}JAFIj@FtEQb@`JkzhE^zs&ZPIDAN|)Vx?zZUoZyF0kW7YEXBwjyx$X)8Y z5T)}LA`m0Eh+FmEkJ&A{ejPtxFGpyVsYg*o9LgV`@`FU{`hDdtclvHV&&!*vjU>;l zv^SPkR{PESZNcQ8KbGvh&zcuZbg>cRJWH-+TEWXycnv?8j*iY3PZ*w|vIlha7&V>r z?qP$rg40cqq9AXm0S_g^vpY$J58rV}Gexm(k<~53x7}0b zXk2^oHS5#q8Zf*nMV~^k8K3NIuqW!=tGD3a*pftVOPu7?EBZm-Ec}Sk?;N!?_?ymi z{B7*~P@!_gCqrwyj!DkabcB(4aU+sh&D7bj8J{$kPY$B*5!BQ7216pB*PhDEr6d^cPRYtiMl`NmiA7ng6}D18y!Fqrmv|#v+rR%yJ1ZTIlXZRq+$iRE(FdmCfC+ zAU_S*9}nV{voKa!o=D8mQT=w#7a3J0D{H^K1U3EOu1cS)(_V5f%wZQ)q{IkQW4QF@ znOET3{HNFNB$4ca8N5Qwq4qhL63L?7*^?{X>glWVox zJjvqf48;2~bX_)<8N)ZD1_|9(+2ip$3_%^WCq0m@JK40Dma#z+4 zV15nC`QLxMN?J@avtK+xcd2_yclq&UX158ExiP;oXOp*Hv1{Nx`9Q9w`!LzVl~*nO zi5RZuS8ZNzUFL=a{brm+G5gaq+IN^=f{h~yNri1zTIH%x$>adCETVBLi<;^; z$>8PhUmXXBcEWx1B?Q*IF_e?nZSf;3s6wAUkz664=eZ^nCd!#c${Try&2YdKdya$% z!a}7Xp;=FI-)quR4XwCp%6UGgq)L;USXfwq!(Bnl61m!`F6qAqYD!%A7=a@d@216X zB}JA_JmunOk~WT$$6kKu81?5q5igR`2_3~NTwbGBWx?H)%MSCxQy<6hln!TLilRvf zl^$E)rlwK~BhC~P93r8+dgMeSZ`<+c*|BHjr&Zi6BQ;z=XV&m3uhbAvr@e^KO{i{4 z^(^10p)YaGA6+%)G@!M2w6wHryVp;56{qt25P1>8+@3lNDWtedDVfGLRacMX8r99`76H~@P4r(e?iwV~v z(kf$^UO@Q0k9>K%l!WEa1pXS@4?`o*_T4b)@DJTL;@cn+#xY(@{ng`Z)fqQCzkYq` zT0&q~zz0}ay*QDf2kjxTN$34qkB?PI6(ir>FjSQ|eyT*x8%^p_IEsFa5n+VSNlcYx zVfA|{C9Uq0p@7Hip6rD)r_9}=ITtQ$bFTYQfrk`%b0?~p{98?tDdf2jk)<=h#^Eci zv=k8#l9MpxzEdN3*L%`M6nlR&T-GKnQ^Vb*%bt@ZZO^4x)yQZWP;wPsXM5Y_?YAJo z_&aAYDIwt9pWXD1oV6Du((BgQUMSP<0Wx$h=aOVsKVe!YA6-rsr?!Amh1 z8@6@@W)C{<{17OO&#|t11wq;Tp=9%%43o?fUX(i!DN-glOx#?`$#cpf*xCFJ{F&7+x^CgfXi>Wh+}~ zbUDg%ca$8_Yx6!lpTTIkz1(hXZPmZ{slK(fMaQaF;nU-}XYbvW$1*G`S^+{I%dqvF z*o3ccOSVa8=IYp^{cjz^GcnVo5w!K~kcbIl99zG<{new}!%HioVZk*jmz*81m(46m zJKJHRl_vFz*xuXK82nTtES1SypX#m8eHrlNZDp{ej#zE9N@A2c?H5kf^{%#mSY0hF zB`~kKEc<;cik)s?d|BfrirdS^K9s=+^T(eT#wGK%8J3xe>1%6iGfVd<8~aUUSGmEE zu^G{${HA_L(~|{^)9Xd&17=c4C-5W3p~| z;kbY^E28>DxL9dT_(`<*sT?2YZv0Qi*mUA&gFM=b0ahiRWnbfL28=rPJTv>g?2yoJQmd8WcYE;(mAU*r zl1dif?+lvI%Is&mo6r)&o0^(>Dg3UBLB9aVDQ`sYjzm3gAj3nT++k|kInjtQrNPgb zAU?8`u3<{YTYbMkhlhs;NBHS+vS$9*J%YSpxGmNFicCG=@Op0g_QJz%HcM= zRPbnkOY|^A{(4Kj{o5=7(yO;gC@^%S$g{b9-!rfxng>m1+I$3?rtcv!^)hQbM)r3A zJo1n+XQ@{07Qp4Ex*%^L4Fo_|dwgt6==OY>;=JP_$hevSv=4T934rWJaI$!OV&d%N zr1f&ESpWJr5q+Zs8K@c-|2fv$oo^K+#ZE9ONaE(?d<{WJ9pY;A5bnA^uS=T}n4XS6 zz-FRXQzN-Dpyt}_d*g|K$12U7kQX01JC3~RVv;TuxQ4BH?@sI6)0V>aX6pABJ3mld z{t7ldZ>k=Wr}OsDk{cA?W%7(Vfp)`?`&o7?l+j(#90J&m{_M{pmp`aKS0ILN0_2WAuWRDHy91oN zgKUbM4TL!j?9ZYiDGWX?efo+-a3CDv+esGAd)7tH8Uf_`D>H%>@;67}S_8R}W{r8& z=;n7~ZlsQZS2c!6Fm%&*va*9QAdvu+&*|~;-=UcP9ZCw}H$VFxQ@1a)8}(u%L>%ye zs$m)&p(MV~xEL^lmuNb5Vez#91XE&l4=HgO$k97pTZkOrG~ zBk}6{C*{M;(|s>DDelgDKa1Y-Z)<@D1z-oLzk0CWp#>gvJizf@eW4#k6XF8;5u6CK z2oSRcRqOZ;($p5q@gk7u;L41EqzoArdXE|H{u>frYxj%Qg#GqAUw9A@PZo&xHG=Hv z<>Eupr`W;mC#7~boAQ2!z{)#FKLROCOZ`jh>gwX-2^0Fen;s+izeIzD27s!|juncv z&1A(=Zi1dxyRR^!DQS%uh*m*~HW0yrap^6>jBy_+mI%nC@%W>(zlj)<|E529;%RLl z`r+aSoZH*8cE4kogeK8_BzCudlui)1e({R(V9?IDT7$U?Ort^MKWb;{Ev=s~n%LfN zSjPSz7z01 ze-$u=1#~%}C58eL?u$+w-wLoocM!J_Af62deb=AB1_+egiER_aWHNsOFP%l8Qewb< zKALgse0928f&nJK6k&ulfuXhLFsjeJPW>$P6`*WxQ1=?xSqj@L+AFZUZ+^Zgx1Mc? za(Rw~RE-BzX>JrO?Fu-AIk0jU03)^Fxy|zTr9}|(j*q|Rb1g7NutzV~Q>jC#5VK-R zfmlP9mSAGycEY)a8o(<;6jh!CA#A1^^Ph+QTz3{g5o-VoPy;a?{r^~Wfj980XY*eB zfQ5MQPUjv{D&9ZqF$8T_O<_q(TRTGBi1M)k;x(-U&AU4^YQ7~?sG*6V_Z0L<{yM71w`@2-kvGq#*%=)W@bRx2u7mtJzx6ky@4iN!PI{(D~o(w zY7nmh*j0Lw!$$}q-XRoWgb@f;W7l?ea})jZgF3MR@Bx_rSYLHN@C0gQIv#*B15F!9 z280L`ve{wZo&`$#EPSdVl7U!duX5ki7#fsJpkHZqG8yA$w1a{7BoXR zx@p4D&=3|0@AXN!A7UzBy#;q5osQ_>Ia0?6+nPIV8NRdYjI)t@^vV6ub&T(1Tw#4~px z4SZxIe$*@ilG@r@-~*;?0pkUN3}EGn{)#U4xO5VbblpHoSA^>XBO8J3nA`4EEJ0w?b~dHVRtAX?|>Hy_|Nf(eYlt7DWNC8iAbR}#`_{lq z0jURQ^)E*fd_kRKf1y1CF~H7qt;zs%6sNvWf)|Jq2f!$)|2eLxt{x0diG%0>eD5_P z3W3+~7Y0LAloJeaAjnw1qm+de$Crq)*@06)1)7S4ZJ!Jd4*tg~03m7k0EWJn?sqfc z=d)AUdqBa9q|)dKM^Rw}G1MznF28qq3RoQoSTEtVD%l?%;@tlA*C?-47d`ayM`fL# z0PP^I9TGYkNU?3&jFKP+8u$&`41A6T`3@5w`fsX%Z3Uocf}e_bQ*!4AP9q6z$GHSA z)OHry@BSh=nYh1n;{6U%w~>s+joQB&q}O#CT!~xVxc182uzzi4QOW+0~QY|9!xoQ7-hsSHgS@z&Fm?`@k8dq^tj z(Bgvet@qUM`Boj=Wm5W!*xrc6xVVf-8tkdWus>Q% zYpa2MM7uSlX^tw4OGn5A)%{LB%Ln2dLJHh3=XQ6?=pICYoeC`-PAWf6dyQc3u$?0h zo4_YWP)4j_r|uRAum0zmEaiWCo%9=2BYUGgWb9)m5-Bi4m)=G39gr4AdtawLsroOZ z8q}u8Vej?e$>nusfMIrP6wT&=uG=&wD8%ZNGtf!YwB!|hDV*E4`E9o77tjczA{Lmy z)UzNq?eOjay*~+R5vL-)V-2?f+ey;Qor45qgZHOE zAUe-epr@vxscLVUooMl(&?lN|KPX$yY+j{-&pkulS=@+(R2GRKmbS)m=F1pRNd6KN zJ!PV9*k{}}$P*n+yc--HmrN1 zl@%}kd#){@@=6h4P(kTsx)B_4@@n04hXq8TrL)BB(Uld%wzD&D0i z0^5AF?d5qRE1;y@23cE);dAtao_R6f8psc{_1Op=7Y%JyIJQBU$`fd0QW`1}4^D9; z>42g1$|-V*xvf_e^KEfHUZr|d_#HVH7gm^;I@*8{(MRyu22I)IBZ?Tpg^a&?jGPUF z@`x1}Sn5SvWGYfdb84!xCq-)dET4F$=gjUoER@vOHe0i1^m3CD813(_7FO5sT2+Va z6K;mnbIg=#F_Q05kf^iBvER%6+X_>8vr69(cBZU!$gMfEgF2!~1|D^2|5`wZ zpSbkr*FDg*lRn@Pnnj5o3;SD+!SS`<9LX)gNIC(`i#z#(X`v3NO`ZZkJPw07fGSc| znMtJA*_;~zN(xDu4jJQ>-jJ=#-_|qhDVBx@3zs|hBhA}>RhUsW-GV(679rXtarNyD zUOAmL-iN)FjT#)$HgQX-qQO4=^Jr!(}ynIm-cJaYa_Rf0yXxNOv z_i(l6f>cph)Na4-roWZ%={hdV`?hv$h*sZ0-(g+@CDm7KF47DhOr@RPzehwEJMxBY zfBoazSYN9DvSFl{9?=Nr(RD-8)-Dq7PRCJE-{7~u*Gipw$TWJLov`nWbs|`AJO4x; zTWY$0ri`Gc6=NFr^PS_qm69uEl6#ih!~IM7JFNQ>-yf;#!YEJdG$=x?<*a3F{0j9n zOjlqpoL>VZh+qZ~B+SyIn4R;!RmX?eqzwA+$M=AD#7ADT(!V0!er;M0gDCC}^2q*)u$I*zVFh*-k7&Xk#r_8pcoUHo{RFHI>cfYs1fO9b z{RAhlK=xJJ2zZSl&?Kh9^fxQp3<6;oq|BDe%F0Ke42NzNh{V2*7?Ibt;I(=EeSgb; z1BrVu0oYNNLR8Qg(1VGsczHvtj`KoIM830oDq+>nPfjdDLlGxECR|1xUmYubs@I-$n{J zu{9~EjkwFYT`)9S^@1$726=(rrfeN54@5Fs)ZS9q*I2-Y!8orcW zE)Q7a!@p>@z)>cFl{O2pf9UVEsRmAsDJNXMif+xiiBn-i!v7AUMisLL9+_{iw-~pvzTro8n(}Qi@9Ns*`I#RIAY(m}<7g^mOA2qSEqo=xUW}*se0&#+ zD9wy_Slg;H)9}156RkaqEwf~{p(8}7JwF(CT9zQG{7*Z(01-|E7C;~%Sep8 z(gOP$BS}~JBcV_({#n;np)Wu13O6BI98;x2xKTsKgk#nM#duMRq1c$DTqY;vrFwS1 z98%>f`1mYUBKab3`Yd%NsiiBK>!T@t{+JdY)wm}RWAYr!%t>OdaNLxGm|0IhC7^+> zeum$beAuc$gBeeeHR^|@%@AQ5^EgPQHN{8-k#K=8#`LOsK#iKxg@;Z9dfyVnq;pS7 z#pb{^rU=x~e#ZcJO+^{%L)!%;&;+L{R!)0ztR2Hor6hmU+WX61MSxRHiX|$ZB%dh~ z3VIm{Wn{ok)fMZg8d@qPTB$75UQ9nC2RV>SQYQ2x=)B9Bg3Zs(P5z)ofh%r)Hg zq9#^@4uv8K3ZGaP0)^aj(|pzA6MEWw9-f+CdjewOBj(sQmX0V8?c5U&5@xtKj|jVl zx#dVytX#im@i6y%NKkCMu8Pm00)~9-Bbw2Xs5hymJwHqDL)EzQi;F@2zrUeAJu~z9 z+SlIRzURe#gOb8R3(yt@3TlXM9Y8OSOPJZ%z&&F^a;SLf8tBp9IkL)o7K=$>d8RY4 z^^K1G)=mEAg`%GSVw~XgDw;H|rY^l=lyh%xXJ{q#GIcT(>=^pz?*i~Hb71(9makhF zXwY%K%S7cz_bfGy>|agZ_ywv55sWViUL*KB;48y<#1e6d<=%jTpr!Ktq2CopT^t-% zj^Tp+Z&aSX!wl!Ht4;bZP5o@GWS6R;wS0VDp1Fq-iO4n&VytnmR3 z;)r!n`8X_}n3(VbG*bG7Clb@W=~9Edr^z!VI@1el$HH z$398FvaiOTMiyXS7@a2y{b{{ASK zwuPJ$LJ{feY4>c|wYH|F*D;oYwAqwuHl+_A_mS&o<%|=-WNq~;{2YBrB|rWs<(Dz) z$J*oO&81*Fnl%nKfm9!a`h#i($|QGTk};RdKh5~g>Rjt5$Hzh!D>1#Id^Yy>kHMC( z_s-5L*|4V(;_#r3%_~3#-yM(2FM9u=2rRh%FLT{j4)B9!G#>7Kl+*72+9R zRTj#uJA;3Vop<%pK*Kp_!R!~HEgIX#h;IqTqf~s3Z|}hgH`iZBc5(oH;s3|T0(Xl{X zp6aodguiO&Z)s4e0o_0_&69RWrmx%D-6p3^UtoQTZjnhvxvN2$i^n zj=ws;1Akg0mSo(6{q`XVH;enn2vkDx?wzNkg{S@gx`hpKN=;M>jQLYiDU!7A((MxH zc9GF^cs@?1nVLMQ(y~=CPjWt)O1`r69YCen9AlRuE$l5k{cbk?AkH%tJ4mv7@F|pz zW56)mVkaE)^bBXWY**L9&r@CN%J;UamMdF=dUGgyeZG;%BEtA?$4qo)_tlsFPdt?2 z52tVLRdZEh;%jfat3mBQxtkTtC{l9$A`7${w5+X*mwPZq0;NRqx5a}zUKt5gDrP+g zov+sRX5p@N zCIFn*L1w7kx?ept`?E~Jjxw-j7Lbw!@{h&7$aEE98u|ICg)~p~i?kaXTkm&(^gs@k| z7ij`6TcFzyNby6&1KRu{ojHas_9eXnP|{t&IBtUI$YNjO zUKbJsQVQS8RRj7YReNe-o0}RO4!e$yyd3 zB2mS}bJ*F%MM)Y(H92EHcOvtYp3~Q9;j$=}e-9`j0ezsC`**kdG4%4Tf{&3kA9weB z!!AH`P`ro4k|DWeEmU!?bQ6uPKAqDxcJ6s!9Qz}__kcW0?YsJ+(x0#SUZ`eB9(`*l zveGe5H5S=ayyHkG&B$r#wC7*Rtf^u^GkkV-*Z#Sq&F}Hg#bXT;Y29(>K+{HzA5!ng zX!^{IDWQ3Jnon7KCOQ0jAgN?b&C!jSFE4^KS7N-&OsNj?;50W~slx(%JmxCZLi&l{ zAH|g2BMW~!_c59xBwySlY{IH6{-=aXNd`G4ibW~ATEUOd$k9%B&K2%@oW3vJ8ahuW zHpR5Fl?n?{ZI2)x>6ye;vBXi9OhXb+zMVW0z4n)nD7^X5sQ^7uc=|LsYxzk)K;{#t z&6F)0C?iQs}xfKK=6;@Ez2!fC z#F@jKId$!|ziaKxG(o)rv>@Ce@!CUZjV5cM_s4D)1UwPdFg$hTB{y&q;HAs5BU8I=Yl}PIj7)|tesct2uO5Nx)S4~ zIQ)2T*$-#g`srh;;Z{I zjhN4iR~xknz|$L!dWSZm*l31t$92tJub8jN zNGB@GnV!SZtH@my%AqG>$TWcXTL8{*DVW&w#)kn7(j*xby^RCuWr!Vf_6!mnbbcF3 zFr;W>O8sq!2?KmWMhT?Z>TjMyvomtDb@g(e_Wo*)7w>u1qd=z8!a^+?SW}*>i+sD1 z*c(q|qQX}Se3>j)?3J@ekfcMY&|uaV0{Ay<)^~wb_?i@CuHv+~$+w{bdsWS2QQG*r zp)nj0;VMS$h8Z0FR!0!h4BJY291FX|k;#}Mb>BIs5TBGt>x;a&pMR4mq({toGOA$H z4j@M~64i757z2_`$Odc0Cx(iVomuI2A^EHtccz{uwo@S?`G#fm8kK@!pCs_L6C)d6l~?wgqA-IL+sLdOk#jy0cS!mP99t$bN5M7lO3+Bm zy6=+!E4(Zg{4+?<``(y?#6%}`-$n-)NbL?P3Pik{A^G~?DnDto=$$jYsm3CDb;%qI z_`RyVJ>qC5YMPR=?Atl^YcO@Lh-q+5Z=jdm=?cjc)e` zM2x)Q2BAF_#R?8FqJq`aP$kTCj#-HXtKk_hziB1+=D0*xeD3*U3!Xkzc}o7b+s9j68FVS#dy`V~iF_=XuQ}X*s z4If{92*YlC#4F4tzbr_S!?<8cLMy4ro*^Q2G-&ZoRlqoS9vVIUQhAy?VZgFAn=ZTw=T$MBHqjh=&d-=%%1eLLAsl@)QYx8P853>A5Z_eAkO zc;h_?8o9m3@3pOsj6)fyKpmy&yE`|}XBdfUR%{uof*`3-=^!PcsI!fQ7D*c7q!kb5 z74>ZfDBy%16WA{6OQiX%`A=HRasIlN(hev&$bI}(x*h{7fhRRGD6ODrnk{5(W+7?l z&dS~%K$ixAo5KGt&HnhIQ^sf|F4Doz9W1JLExU?@*lhaDM5~;esgvjZO zSC>aB`;60s^J`ftOWeTXB|ys9>(pK;pxd#HkD$dxB*|+rgbdn-d}e1ch)xK z+o4D0i8?A`y?O>N6I+God`cah@!hm_b*v;@cdc{-%|WCDNOQ|Lb`S*w0($R5;3@Y+ zChF4!)=$ItXo7_70_JL^guSEvg>ZkyBn=G*qfgMS>~z?o8r1R}_A*Shrv3NnR}Jy; zJ?sqd+~NLo`S3P#rE^Tpp^9OG^D(1eK|1r33~;MN$#HF7caB3m*`)}Jn>e^x=jM8^ z2c>pRu11Qx$fcfaA9G7(PnRQ2ajTEgKM zJ1KsUlwYDJ9VV~XPciCjHAWvKM!b32Uy9!7U0*=$l&oz!b&a=I^X0m(_M7?ilkzXl zP9Fb4k9%?gy^q4SG_RoJ$=kE+SA7-LP(45#>e`mzu#V- zIcH-irR1#Zc$M>&^P_jYDaFNzGDcDpc`c{6@RbL9u+OP|gwg3*{ey@n-Fxf+Af3Kv6XZx?tA;nee#v%@uvWV9B+7GDiA2aH!6qx`V`vk&Z6?>e_p*UxzFKcFq%Yg-1#Dcu0sQy&{0Fq=3T~B+o8P|r zO9ntFVBV`eF#!zk)Z!xlx7yolt|(Hl^1-J^T=INdTkXe^iXqtku5IbmQ-7Qg*Me*= zKc=dkRM-=MAKcJeHk%SyZSU;GM8H7Su!*BvPIRPGY)Ol>2c=uY5eN+N`gcw5f7VX8 zr-K9pi)m-XPZWF0;6s5(t~Kz~+MWq&w-9{S8rU{>eaP|qikDe#L73o$3!mE;!`u~V zeHI5L8TshfZiQQ?NC8#Mpu*rnlTu^5xn6z_uUEPtF^_U}zLn$okzy@BNw{fYlLD!s z1Arp{s=EPw3RN6^CS-}YYa3C`YApEn@#D32?1|6*6vINn>df#GK9c|vH3l|b=U1RA zZy;f4mc%OBJ6@`X(*iHoEB&Xa{I(1n15i@NS`N4pM{}1i%)C5IcJTPlu|#M|-|q$0B}* z1RLDHEAeD~rlp!CjUG93RJ)iB~W`_ z(-iuQhlPa&^D=2PbZ7CGPk!rHFMtC9FOh-R5V>??qA0*4W;Z~q^`Lm}{AuR|0&2Y$D=4b(^tr~ee(#!aqUg8&0V zxJIV>=V$iSyq1iFzh~HL+L|8FXmd$+sW#7F*VhZhOZZWVb}jBKWom05AY)m|LB%K8 z2I}F%4EsM1Wt*SFRPCEduEX6$xc2kxhD^Prv&!HgbCCyLQ%3}7L$lS0LL9S-E>kUY zBO&O5gd+qnV!R^$2NOH5#Mmo^aa{C;;8n>8NtQBe*v^J!{isMU5T|wU^%B7TqH9$z z%#8NzzJ8^zM-u+Wh)3yt;Du$6kf?v17-fv~GouX`5u>cLhI+^7pLOpjva)ZMwBLn! zBq#`!unYtS_9?Jn7+PyPj7}+J&XY#8i)zO0IE2jaw7^1iNqLHx4dszkE6<~v@7v^q z?qJ40m3iRCe5Q!XJNC@(Pwgtq#~Es>@cr`|z{q;4H$6q#}8Ge?aU8nFiJ#_!p_N6}j-J8r#9P^_e)qgMU>wP%0pIdd? zg-OLzDgO;EbZ*qf`>QzJi#)}`=`KQ46mgeR?u_>wP=nDEM$)Mzp8KFqfP_%7@Gb+!D5{XO zznAyv5r*)qt#6)o*gx74JPWB_X+}(U-CXQ4CJDbNY#2BisG!xG=M4Bgl&D6pJ>gbA z*XDe+BCe*X#J}7 zC2RFjH1EFNB}WOli-&?ko5NCK3XU5DQVuAT?G1poCX z<=u-6uFz(dMELm}QBz&5{A1YL$;T-|$kZ_pE9O9d@bubJTYo2yS(t-I2(z7YnY629 z#9nZkLik%b!rnvV0o@hLhDPdg9CDA${G6QvJrRZ?FELsqwFZJ{x=M((P_K}jDl>1M zR6Ul{*A;RXm^S9L(Wn?)6!S2KfkNM92=_!#*PQoD&5V#}B@~fwrRHp+Ly=k#+wH%) z17DCC!KTcJrhKJ`QJQy6{(#QY%857|CjN(s>YTFg*07D#EK$)q^6eaY*unKeroYAK z6fzX~_>xDWaZx0D3AWmrU`;*lx)%=Ck!}31bxWQmZb^f}%|kTG$M}Iwr1$t3%Yj>7 zPOO$ti_?1!->T$89C0;gQ{$_d7n2Zt&&_8`-DD_mkZLRoYC!}D$LLl=CaGCWeda3v zbr5%N$K=D0#mkE_q~q5SD(8~pQUpr9lSlW>Q;Lei(R&E&+VWQVCt^pGig=i*@)t0| z6p^9omADN)4;Uok@m3xSCjYLuopX0AW*?lbg@!xO^oVNVG>auCeqJ^m%Huc<&C9nd zwYTUGkF-UmGliJ_Q_t$;JaYeMJxhh@MLDZ!nU;*~BUjCkyhkUeNila@jsPDXSaSfG zn)8sVWV~L8#LRy~)u3+s7<}4?3=0Fb=e8Y3t-9y$-=tM0Y(51f(&4 z@2*|N1}|uJ`d=p6>Zfa(`SQ$Ji&rf33ivNxA^<0Zanh)d5)&X{<;xX)vI$r&SJYUw z&_>N1n%6u-(Pt}~(|g2v!xjvjJbfJ7@@sbplE}2mHG%H?c=G)b`n(Z#{n?-JTZS}% zxo`)Y3NNNDNYhQ=WH<=#*vX;Kkyfs3UJGLMKkM(z1p<2|n8=&8TJ-;@YVAm#9Ts3L z5z6UF!5oTH0_K!QTABi69EDSXoWbFH#Y?rxr2B2V3;iWX|D zhUhWE4&qk#<;LrA2#tFWy%g2HD*8KM#KA$oMfo+j;04MTuW)ljEwzM-dD zAdwD-oZ~!3Byb!VN|5n6J_s75IsoFbJsb~R`9eb-tjQ?lE(eL{cAunve zn1y5Ih^k1k{xE=Rp?f}gZ#Fbf z0uWZ@5R#W4?oY0|e$6NVCS)9nOi#ytnA^MZ?%q(!?Mc*Sza1E-tr z)Z-A>1ffn)?I;Zm;Y z1DI#vUSLhJ0zr*~wZPp{s*)@A5kEcHysf=qSp zm@QS-5p)O#PxFVGwwO=IO}8}i9YQmg&(B0=n`fdzMf7J#z%rcy(_&VhcZU1z&EJ9y zADQhxOglGsh}|FzgDtYg9<`g2ZXCX&qGkbw;%EU}Vt1ek`7;>bc#4NLafC$7_s_eJ zH}61#4*8^`zYu(C4W&Xr8f#L9au*eURa5)5soFE}2n;vWw7GX2wy;~LT-t?LcZRy+ z(6Uh-D0NTw?we7r5S=(<(t5K}xURnAjx0H4RhIBE>MtB-0}`{n#If!53(KRpmG;$D z{Pd_JZL;yoAx`Uu0Tie!p!)=++@SuFQD!xZ<_>GW2~kx7oeU+0FqsGsf?!8AC*=Sd ztvfYBt&9Da}7Oko@knLl;4uUE3zQggN+K^W4%SO8>k=Hl5M-C;Wm zSa(W7Ys*l6lE}%$XGTAp+_YJ=N9`Mtz0IWGdM)+NmCgNrhcsEoJ$}Nq>x7F~4ihrT z)N}PgrPblkjx~2IxoOu@Pf-EprA48rXS|9uD);;JTf{`FS5YcP0yH*3^>fm43z@fV z0-F&X)o?JW!$Yu-Sc|?t*ZpHIQxs zsw(Y2(()^3>|O(FZbL4f=O9YuJ0QS7%F8i8(#jBJ^@*^lrM7C~4^|h@X^E}83D{7? zcns!A#B$FzL&=?^n=>3hp0l5yAH1uCw)gIMI`kky$*iZt+DDWJtghqXk?y_$&zpJk z$ufegJhvA!3Q$uffL@>;UeI1;eb&MMvAf(_eT#;lCsEccshfO=ZrZ&Ie=G!@_~*qR zXrt3gvQ8@myhhZ4Z?q z>2u8Xs;C00()u>{KiF$2ouMmuinZ!9e*|SxO@4)z3g;|KuEk&H&`&I+vYb+|jpv@j zqaGhG5weZ)YCnMh_#AD;q8?j3Ve!8u5QiVOeH_ z4{rb*0MY*{XmyR0t3ibogEkZMdIMrRY~FuX(7m}iY+O4gA^hwt)wFf3hXw zaQGVxAH-+R0$t@eA>z!DS>HF26o?=^pj0C)A~k%DM*H%`m}Zka*Ne>=)QOdFiq~f= zz##(Qfh!qh>_7^n84vKwI7ac!k2MtL((U+;sgz1S(4{aYF%1+AG0@FoKZEQJ0e{N}dQ*iG^BFHtuxaY}^4x zVdDfq)Dn2bcYRBHy+{GQ5PuiUB~ClHnKV5&QU|5pPkQNb{G=qQ+ms#ahiqe7mGibf zzYEx2<3poL)-hlXdWeDB7)S!XAm9j^w+q_NRUN=W*1AebG3?$^Zz!6kpFK2pfLXMT zBB|iyNeO`UF9Th-2ZY zGFhQR@;C(r8|#us;PX@}^jiY}o?tEkZGj)XPSvRN5zI`gm?%+y9f2*+%RZv7!5p5j z=6@R`pv|3eZSL%|q0*g2NVePdS|DrsVFX73q(!Wg|9-KP_RiGhjWh!zL`jfey()0| z;3mOs?F`dxIVSQ8IA1Xw*E?{PoWMEgGrd!EBfJV>6N6cBpw7i8?MG->nmc&^>(a%552a%cbDiZ^eimWb2x0KHH> zmK3N78t5<*DFtd1rJ*++B#%v8->vlpi7_z*x~MM`Vx0f0YmXZ&_+0~Mb}g^(d5ZQC z;ew%`Bnfh>cE5ql5mocoV?`EjHl4vB-C?H_Pab>&sTA3^Za5w#%wtkz`O9DQy396- zS>JiFNR)SzTN)M8?C1n+k2pGYj8HJW+V8pEs}eQfU}9htW}zS8(#TaX={u9ca2#53 zk|96qBN=ivq#;3jORd*VFC01HX%gn4kGB%#@+^SNEqsXYzc=~`e@JqSViOe!k1K32 zV7y2Cx@?sqOeFt-MuZ@K;WoOlV|ky#Bi?2p%R*Fz&?lT@b{y-h0R3sIEvB>U*Dp(> zpxL818s0&Ak$x_WNCU?vboz7Jv!%Gk0NwPt?FjkCF)wp?J#=8>$F9()G}Rn%(d9_9 zWZKGdCTI?if1Sn(>biWqC4+Q^%(c`(%wPH+3~9eUKz7BOgxQg(88S)fQ+p^B3z@9? z6ZrI4%eBv#;1}Q)WabaK#Q(IFp}x%63O;oIG#ZCs#)YM)Xgtpb!{1q0KIf)gGI-l9 zCGyRCWM26QEWp7X4I(azuQF=M@!TV4k>kZ_zBCs5z!oa5>Y=)bny%YHBj_|IblrQG zU*NLJF2;L0;|;NW*;kb(aASWcefckt;PdfHyc)aBK={sLr*@0j5A(bVt=c-SzIaIr z%0U@x9%&c`^M>6>bUhdd(SPa<#quEuTVDgzb(-lU?x^eIGW~C*^-!bT_h%jXJ=PhDJ`i(l(@HEK5Dk z*jfGgx2&HyJ8o)>YD^5qrCCIq#nv3DsA{sj?ABb8@nN{|wK*z-+_>lo@nOlk>)~nFxU$@|cbZ)!f%5T)ct35|u|m?Bn&yuuToaS0hwkkOS?NS?01w zrWuO=CqvPpTA>EaN(t&Zyz7=XvLH*Hxg@zUfZ{a~b&SjjNE+iD$5(>?bl`y?vi5q| z?imLTBUvh)>)PBL4{E0TpRj|-@tK5qAI_0T-l!%5`|6T1JXx%;{{uK^MK8S0xlD+7 z4j=Q0j{lSK%q%;ue6>#~Z*MRvOK!MLH=#~2`?Q)_2@EGB&yE0c(gTBoy(dZ_E~Ok2 zC`FPp0bg~Br7?nc~#8FWtT5*a0h)fG%m-GsfhDHseZkXAAlO_!C zx1XY-r}Y1%ftV(-_ebsEDDWhweJt0t2#772%S%GWf?PYs$-#ucnI^>o`nX)b{lKD~ zA*@<^d5}(uj9Z-dz{We?_0>s8R+MaH6GcguHKdFVQK9%is&!D>gMcu2dD&|`9L5&= z!j$Jlk(q}*F6S{wbS6;9Qe_YZ~%M8pN6`2=$`dN!}r6^~k7cnL|F-p{7Tk;2*H zw2LQkTXDlwCSK1Tn70@F_;QX#nertrFX$I8AWU1aN#RID%LkEkwt#bwPp)`UOD9foaDk&1({O zB}9l`0!~D<@M_XiwPHoD`v>dUIcDlLiIkK-lI@#?ML?4wYOsJnfG86zuLt9!>|ivB zi!*apuVREM@&TWG5;y7K94U0jiN2StQ6e3)@_{lH7*84nPvV$Ih!iHiSBIKa_Xv-l zV!of$Em*^viqwCt&=O%AlWFp`bzrRh zg3xH$BWMv^)`~`jekeg8E93=zB1g;}T~WE$@5_vjkbOZ`0yHuOjEFIK(|x@JB>qyU zc>=^~+%1AG$Q$1+;!Hj#V9ynE2jDD77bORsfcLFZvf8L}3AR=9I~aYg5RR4kGjt=w z-W6*j1*@;7B|Y$vDfp`O2XO+3zxHGaZF~3@U3UrJCwTvkf_?D>h3?HWN$rIGyb@*D zl8T9ugx+!i5V!ms=ToyBIj`2dKgDG771Ahda0V;~GjbnMMFs!7$hQ%&u^Kg;`lKGg zsG`?#bj^ZipVV@(xDmka@d&>CT#{{O6#S>On>KXu%;rJt?vjm*03zNfc%7xRf39tj z{Z-WgMR(r{t<8iFtl@=T-M0NR(%ysb!`EI+zQYZX;ghCNE^(vgoXHG+Cf+o|9 zwUvHKCUT*D2&oJh%bZgU%>lr%wC=}@+HcDTqgbsT57miU0vmUAw@J|GBEl~|?W}oY zK$PGh-09ZcVue{~=<`dCAdt@7bA5t`yFKqBUOtlJ7F&6cqcbX?}* z;a&KQfW9V4{w>{)Dm;0nQ#3DwbwlR&FQPEsq9u975bv5({YjJz?JD^P!KgF&;|nim zS~oN4pj1sPFt742!UMJV8NA$!JiEP9!Z{q=*9MtUQ;G>*8d@o&IchqZ`>o%^8ni*e zcXZDBmbGohC%x<*RVwPiET8haabZ^EkjN!mPQ1+!!5+l{E?1S+L#a@XDjb-^!vLEL zD#`MBA3$+oc07AEit5V|WzWWpZT#8cw+%+?&%BgMA_(Abl%E>n5qda`IC#y{20Cgg zVW1{Fv8(iNB2V{M;IdiR0zoPuL{bCi8hG@9nAWwLPl^9UOUJ0$Y7CEO?U2g`9qlcf zD7T2Z*xBTJCv%#dS)(-$DF{Rt33g7t-5KJqmMg-aUGmS$9(BFZB-K3p#er42#MzG@ zd6Do?HfMkZZ@f&a;+cOw-c;jiO41*U^J)IwU9`iJbkJ$DzEF&`@Q#uQL-SU%H|xWw_N&cSzQDPzd*r8%!TW#BPklnwDJB9w~yP zA)ubyx!TJ*cem?nR85ML28i~T?aGvER`5#H7H!Gx`R8WmtWNs>ZZ>XQ?-Pi4fgk^? zWW(0!4nYw`VeJ_HTq5#ADR{f0dG;ebwo?On&p>DpN{ID*p90|Hmu-PEw)< zDqDMI_}d^(XA#=@7pm*iW7Duc3}A`eO$jZDZ1(H6(b zpHrg-9zKj5?EMihW#bF@Jyqhxc(!@1ZDr1bk<|=M3fo_BIV@74J_lB& z^B&{H?2e*aK!i_-#&rK{CXTp<9|97aN~)*<`y|$O`(B|G%G|~VjHKZ_nM#Gzz3D|* z1)Pw&(1flOECJcCEI|YE-oM=_`Hhnk1lyP4!&MH+LatR!W9eS`@AW+ofK=CDJ8}`$ z!P>fMCtR0^<=}<;f0r`z7Zjw#xNu)e9b|#GYxe$1qVSCyHuw_Fr^uW|-I4EF7OwGC za9FIrEbo6)p}4Uy6busxhgWRzrQ$(>DZO|Ou^Poh9MguKDW|bAAQ{urgKL#0qUht44S!&}A3+s&|CK%!;YD~_ zaGk0&wgc18*VBC4dd2Z4IzU+RSI_RRaU*-Ytsu##)g~l-8!nZ8+JUyK4UHygT z`$F66dvoO)@76sQfspj>sJgXf{Rhyr=n_Yq&jipmw?`8H$zYnV{V?yy_Y#C{y#IZy z*K8WE^$vh58{j>$BA2gQ;SSV*)|+Ep2f*2hltAtFDw|pDMJ62dL<8Fz{C=tYZYp z@3CnuF7a*HG6>9CH}fXQTKyO~>3*_&_v}SJ4Ud*xwEU3y*W8PyqSU@+yS*800>^6L zd_Gsh?=L8)3~vDhffG4E=Chw43zBmIp#zehPtMNHJXMDiKx~^*1eXnjJAj20{E2LN zN~8F_kY>uX_x%zxzQD?g+9~+=Zzu?OrFaBny4Zj-nvdSlydmPc5l4x>Mz`y2*jyO@C+f%byGS0XT00ZoP2MXl(ATA167hWeM zDVOpCrsE_pZa|VWU^t`Br+`tVHBfxgrUVX|a_WR31)|jBZ|LbOmdi2*=8LR=2(?GB znD-PWv8)HjVam`7fDK~M4_pXx)_Q<7jQkN;I6z=D^si%2f#Cuy_zMu-RndO?b_N(U zs^I0Tey6tFsf<_>a#F;x?Kdq zC-@#eUkQJnJ#mSP0q=_da#9{>6Pzr9`j{0JZ+=p_I?n&g28dnZsx)$E&JfIH#G zw;&SAT{yzZ2Us*eUqruw+NZ$9*8WTMcE6B$*{@lhIdNp$y=H`@24Bm8GZ(-ulZ~Dz z4c5xpLe19Su20^4y!(VmMnFK2mX`Jh+RNj8Z)_=t6d+zcFAq?fLL2-Jy(xf&)qv&E zwU?E|LN-v+>}fj->$L&J39Kz4V66$3>(JaJaP0J3`y^JRw4ihjxqLO2==u@t%&DiqYhri_+l+rlER#cDb6|E0pDf=!F2;cEPS9@%$5M#3yhHV_$uFNXG1P1 z^J1<4(^x)3fdS4{?6fp0$GEqJQsLsqk8>j5E*(ptRE}6CY_&l_AKf@ccnL@^SliD7 z`%_MFawB!tcbVBHegYNzt{HmS#gkG~=YqEc{S0d~nLbEDxbnC1h6Z6;3 znR0aQ^e%S&FL~*4$`+eCp2Og05RMdU-Hp7748454ZlhNf0alOdj{ zEd`P46t;+FIC{omGtBxuMK*JnTTIVS5jhYG@Sn^Ze2BCdsZ*Lp?B$w$I!VXv^3n$hFe?P)trBZivz;KZcrp~zY z{+my??!VBMWd~Q!U)g8Ad(Z*^+~B8yTn+ID>GXq^T=|>Y!>~vuQN+Ct=7w?T{<7(@ zI$iZU+gj1~#!1^d_#NjmFLx4Fq@CP$QX8Cmhd;Rl#Qse+66QMGsS>Ea_gNg(erDKp zs#OhS8Jmb_%KkM6bV9>2>kI!kBgU7?y~GPenLtY#^J2Z26bsjUH09Xl{5dTpdfSx9 z;4EoBs)?)e1W~x|0cOEuD? zQbZ{UedZ`!riOwoca9u1!3k3KwehT;-UVD!yL9{RwCw>RS$(u-Z4FISYNQhreJ!*r z8EgqtA0rq{{ugc;rQcPJ)j0!tIj6Siss@?Za3J0(w}J&rG*W z{oJXRKg=@-4?0}ldTFNl&n`qb+u;!xm$1IZbbqbHkOHdo@k?cHTlKW?^9S{>Czckv z-I^)ix_dBemQF3Y+YKwUoo{v7Pzqr3irVuNxLIXS2pA5!Kd{L~qfcCIIuOxd$8u`> z`|u5)!)_@Ez<{AiOv{M{KX~B$)ktEF-kO}wq#rn>mE1;ji&OM9C4t%c$~ z{o`YCb|P_>yE^75by>G9*9H`m?Kv{c7>eVV*)JGoyG^WZ(mVi{-}bEVCz=r= zq{>L^d7lfUk0T{6K65?1enxkEy(cM9iQe!(Af!O;q*1ydab{wAM-27as8(W2(I>(| z#@Nsj29>C*^h))7YgcdmV5W$cE&F1T@93=Je-!r>k?X#++;c2!Bu5weVeH0e3y22IsFQ8s3fD))9JUt9mrBlDW%b89s2RYXHRY*Xo@k4>|3 z0VUOLLzYr(W6ySMjg&Zul|E7F*f@uaJQc%CIhJ`0^tG%aJNyR=;F6GrQf&;%Xtor+ zBN)5&a(zN5lZ2n^?N5U=rRlMJ6PHo`0f0rgc~9p3{FrD%Fuk@qHe5iDbjb9mPFt2w zP_3-YW3ty+9=0uDkzB=LNyYeH2jg=@JXW|K}p z-w75lSLGGYv?E0DIhv2z7LB2D+`||0`!DC`=llD%trqQTP8(f+`@^vTrLBiZgx$q) zu-7Fh#-H`8iQJPj6FD&b_60;7o1;JpJ5u5T*nM?2* zM;6v!v(3a>!pK@=a?P+0qR&rmpH|Bhq|82%j0~<>b3|lm8dnOZcbXfnJ6{FLJzOEyreS| zo>CSA)+Eg}!t+TSGWOl~E2<^NdBIc?I`hHuZCBZwdaq|-?Ujri>*V$L7&3A^C<17Z zRV!ZMd&gbx17ZTK`|qGprOt?M?Na5q4M-6L*sRlLm@uZlUHSt1r>6x>jN2^W3;Rgm zXmhqg+S8C8i>4@R59VIHASgRTakhF{+ic>vr5MNkV8WVF0Bh%v_&h`I#hZ%Jh_4QB z=0_#r`(%zI2qInjNK8u1)7wfAuNAhvL_;$eN>~)xe6j0}{Zs#~k@Y4fnW3GS|mcWf@t=yR!ajNOH#nQjMWQY#Bz!8C`B4lKA)%vdX?4JB|BwAFV;- zCZ$3#a8ZK6H>lSM4C`w4!Q*3bi5CUbc-dqZmY#3>0=K6PMVd|BwEho)O~Uc}3v-NG zR2?_T9$wVg{M^>Ylvpps-zE5IY4=?VAEu)l|B=x&ogTE9cjc8<&YW}{3;(fa{Tz2G zj?{$N_u4dOlu~w+>?1t?hsrVzl{@2bK^mIKTi6zY5O@^x`Army7-y=ZP@@Z0gA=x*A}y_FC$ z{{Qf-Ed5@p?w@~yb7R&2HL^C~WFvb*>gT7vH(egv=U)PTRk`>DaZL_ud9U;`he%$);NKV>0P29FBROx`%+2DN~O&vOf8vJ0^Iu!#?lmrTx@t8Wn;XvB5prh715bjUzL!hkH(}YU;moVKW>SnF zY?@TPuizoFj?Fhl=#YNrMW+>oyWpfc6`{D7n$xVEo~14M3o(k6dXt4 znw~$G({iYOH@{ChMLU1&JgA7K%bz7;;9!mr9V|KVzG?q){X$;tURj%oM9A=Nph&xx zIikb$NZFy39$JF`+aC2XM-P+Fc}?If(}UA#xdBvHU|`&>M(*t#u8v_npGw5|C3N7_ zj@#e-P%@AgA-Xr7&YiJvVKctmXp;-fK+w5L)pk0Ig+ASn>`#~M`Kq=((Mi6SXWnL? zWmWx({3={ID}w7IE(hbEYBaYyo^7)UPusDeh47k)GQ^w;b6BzPcYK!LIQqiWs6UCp zy7{#1w|H9V@VAuwqk7VBg{?P-^cy0#yEN9HBTwHy&3}EwS2^uEv*@%@dbe8N{N}sB zn=1K_xq|13h4k(xPc~<#TM1n^PoF+k!P^%kS)%Dk@XMbPPwTN|Hcz#X>f3yAc2Dx8 z>YS~Pac64OAa&}Puw`D`{o<^t-@?%Pb>%~Yw!Bj61mB2<6#rOdvTU%7IkN5FyFf0T z1x5BY$oFqAeS2vQ7Y!CJ=l8DKH^>VK(SJ1DQTgb4xtzQ+*|r}YWz-`%`x%#&eUD&UF2PR&%Dek>X?c#r@`4T zGYn#7dSYfzXZAlW=b?r4?8cnN?Ovz-GCXm@`^c=`KG&yu?`;?P=1=dQBs|MeBC7M_ zn=V;DK28shONcVR^NUZvI{Ctc{XVYxcwZ1ZT{PwHzOBw|klLU}w^dE+kCB>I^ z`j#9)YmvK=Xn(*df^()VUn;ca&WN3zot!*Z%#Z9Ya;7q{)Aiu`wxpyart1aeFOaZn zWn*&&CVmCho<&9QNJx5Q(cyn*+yXCvn#jOlC=%r%Z8Dfw_KuDuAPD(%y_5WAud}!~ zSei=k>gB@q(IO%`QBra;@7rC)QX`zEYv4WtPnpoVyQ#vAjErBmLowliRtCw?K(ddE z!?Mv1E~I`HbAR{!iOu1}`UlcN1&N)QsFLTK$1nCQ`na7HbS*9hiygoHCo35B(?1C5 zfB3xa&@Dz!$(PE?u0VdabOfw_-qQs#Pd% z3vImCcN(mMni-#&W&%5yM5;eL+|tt1mzI`FHBq5dHNzTdWaq{;HFn#HaoPO7xJ}n}O2J=z)n|=l!v%@i9mgRz1?sfMP^yDUd!f}co z+A7zRZ?9|-NCKJHTVBb#Kd(9b{`7bQodf~o+b%FlL3(k9blSFwUhzH5xz193M8MFA zsXfaag@uum!z!$Wt8%^|7v+qRbJ-F6fwvboT2=}vU6C$Ko2^e*%^r>H z+rh8TuGRC!xJ4IRM&`U4rbRs53Hwyy%qGt69aI_nOgTgn6ZjAp8|VmzNcNsTVB7FT*uDLzpr9Zif${M3gIBk1?d>fV zzQPRC#;~Yn(@!KwjY~^PBEV0KYtb-?#j0SOkjok+P1KCSk{pc4cK8Wz0q;)%Kh-JlvI21L65|jjb7!%4#G^3z-knW z8wXYMG7j21WV~d_^mKGP?wJ#${-GeRP~4^qI2^&yKjC+$G#zoL!;6df1<8SPsiQAu z>d$#(N)%JlnX%`r>8hBx3VJWp0($S>hfhltT2Mcny^M(_lA^Bsc6SVS5P69v2#czb{u^F{+#Ki3xZ?70%<(0n ze8HnE#!$>n{z)~=_g zr{WlmN&-vNARKkQPZk}YiR1ldStE~v$uLDz)iI0YLGyjn#;Qar2}2g^a5yt#mHyzs z9^E6@$+oRGct4DNKqOIe>)!UZ0F6FK*djm+`MjTg^i{joSjipv$0rQWW{;jffBt~! zEZKl~0f(2qBjpr9-#}2&$)h=WyrkveX!|U-qQc?1xyNacqA~?jS+!I5`<_TgzRbHv zYnamj`C_m-e8j9WWPA6zDTB*uG>K7z#MN4*!DaJnTwGjG5PShDOg4|>QXR;9r{(5; z3#L4RORM|O?d`(i)3tJE-fWfA@6`dbS>729bZ=3)vyE=j)o4;k5SAnekU3o=2It-t z7zv^4fFQm{OG5+Pl;|HOgwtbVRivcgrnS6i=rWaBTJ$i@s>^w)`9j07Bx!WT%VoAqPvEn0)W*5-C7EFfP*s@smRop|6nl zwSotDf0yzGzmG|vDh-)^ehFq6y*AO-ip6Rma&RC+vyOj`Oz3d9f~s^~98Bx+WqR5$ z2&-WBLG5}Pg<2A6esR8ekpBfT`8#YF-|1$hu5K zu&RlY>P19E3h&+xo))QR_}>TW>gpiDgom|>4)yqey!;xdDW`g~_Cln|hOO#EiW&{F z3m&D5`V9DJc|?{xdoeg0<74(LEIz8cK=9Fx_1)}wb^iYi-+HwnLtXq%f}xlx$5qc=5dFgC-{l$6||vGJJwlQ8nB9phw7`=gXZA zG-@T{i-G)Z1){$yCP91TW;S@EkQjZZOB>knX6R*5P z%y^8azwb1#EMktXpMI*j#`NJ$+@(OnD-Nbp9d!@lUV6{DrSk>)tn){6?Q6U*x!=b4 z-CFW#BI;@=(sICM9zOnJKvbkp8k9EG3Jbjr|sUgYv>n`v>k~=$;-=2Or1f8 zFo!8Vp}R%}Wy3V%!UnZchD@1RcTy!r4`sv}_%&`UeKzfpm(H^+{jjfR@MeKdMV*yh z>`1NXIp@!}952?3HAO$x;x6g!l{j#G^0S6<%)q_XY7_Zg>_xH9?4r$XB?bkx$9S(DVMl2@U?kM)gvuvnTCL9~cNfn(6qQd@n-r(eD8B3XUG1S3WW> zrbDhE*)GXc)_wlUchPFzkUvpKkiy4lz%?SVOTTHzR=3d3%!cdYCIL?hjq>GzlZ)XSz?p@8Oq{uun&ev5Y~b z)76D$fH_^t($$LDlHb^bC+tSGmUs%wKwk8y4jcz=p_?| zlzgRLv+_Fc*NfLRFyRd|828_j#=)7v%%yewS*y#d%kCl8yh$$IVXo>q539IWeZnir zcbW+uz5jc?v!zu){px)=r8wJcgJd>_^rIIa-O$a6OK^4Vc8xgi-uyr?XsFQfQiyeG zR#t%~jnK(6CWR6zOMP#I#di&H7T>2~Ol|jhK*hV8Bk67@va~+?%u5aC3XZ`q=r0U= zZgo}6JYio_*Q=ZmYHN?7SFfZHOv512MJ)E!@K|jLjwdQSYHNREQj{9&#OmeCm+4g` zg{Xc!jkDP$`|6VPIa9|qJy#X78mE?zzLqKdcY3PzIVpV5@;8wD*ZA#^3E8)s4Ne>R z@~A9zRfhla7Lm`4d+wUMi`T@|!r}#sM);$HznFxd4<>tNY);>#J?!hVt@6>s2Q<84 zUt?rk{47p5gzw^Y78B*)&Nq!UdpnSy)(P8uvNV1`XR$wVK;K*;T?>JYinE?vTGVxo z#kT*xUtcSX77-1&zB#a$^4!RSdab|qJI&T^b}n!pl09!KM#+12uHm=FeW{rhng{#D zg)755Xly39qrjjDLKV<)E33D;A3~^)k$xOF{p#MAFJBT96JeLbr>zvMJ*sm0-PB%_ zDNr6cZ~9PBent4Su3^M+XHDlp`9@gdLu&Vx>NRe|Pg^z(uR9|8+*yR$;nHqL{xAbw zziRda>H6o6aH~*xhbKHI(={BD60f|sO}%~Fl!vtGiR=~^7h%Ud2^dJT*KeOc8@+e$ zC>5`>OwyY_@u@@Q6W;0?RJ<84+=HA|sS>)r6{qp>I~VVJ(K}?qFOp(n?iVCu_HD-f zbFT|yQ!C9F2zBU>?J@lD^vZv{gwhPv5p2RZVaNl0F=wn+9v@pZSuOv;&~yQ3SBmqO zH|dW^-A)gxviNV(r*pV4C)f26&y%tAr?*;NsA<03$}D=e&gbEs7??cs$M=J=5P6(I z=SvCUhmkwP`KisG4Vy84W=?GnPfM8JD+|CK};lnw~F|PUS zgEP)1HYa2Hj5RpU-;=aRNXoG7xt_32aVy;VqHE*x=gjxV9tSIBL_M#r)(Fe;N^**b z>0q$Anf&0ybGsCa34M*o1`%1_iZLD!1sj*8DW5%AzqdON7@jy&yvv)4t5zOO<yG$Oh8(W)PhPvmd=-jUqwwI$6uAy>H3PA{PCD3|!3E&u$fGx?AAS`>Xg5xv$o z{op}CNP#iAljfJVYv=7W^VV|S4w%YN@p|Yk&#Z56c#{xf?_sI;r{PP9>+UUMRU8}~ zEE2X)-1>8xFoHBbV8?Yxe+GhD6gr^+Y{aG8q)|g*=Azl>J4h?1hFB+AQ&Tb@@h~`e%0p|-oKGm8v;SG_usmzhDT=(KA7%P4y!GX3dx;$)#Wlb zejr3skcn&mja}Ymx^dR&`*iJ+*trsB^z&YgnCvbb68@=b{>;bl@dc_!G|W^g+l#_~ znK~>*7JL*7I`%sh8pSPrR4l0{hkxLKj$qP@SS@Ro)1u;U3gFemlK;#UK6LBdl?AE! znUF-Qj1lXbb}ex(uUS}3hDNTt9Est}OB%^GB>O@*d98g91T3ZV>*bota_S!yzI82n zVOMuQ`UM@UMug&(d)J@S>^#Ntr6f_&txRV}obl(5oYq#6$`{s&N&4||(auSM{EUn= zNuOIIGh0Tzcr&89yL}eh%%5F*nc8)I=I7I{Rp+33rHr%u?+O=RGgC^$_`dD_1-|2( z*RL-uOtkC!$C_(vYaf4!(PSC7Zh48&@`UxZhHqC@QMtRr$h@km>W1dMf2{aL4Bx6{x7F+_u~m7sq+{Cow=ddhM(#xE-1P1g0u36E&A`*-3}wZD&DwJhyD z5&6QUr{`72k-5Z7DGiMrN7tm@ zZ|@d)yM8B$BVA19R2E&T(zNHt9#_i50`WfM231U3eB?9P{q(8Yp>>JjXP7$v?`7wv zUdpi1(o%#%BeRFAT&>0%3Gm@f$K_s&2gjb(Qx^5bwjU1v(ixWjfhU3rJ%AY6lR8R8 z{O9OR+BuvLam+GQNYGS%U0f8sNk*QqW9UvR=bAv-rdb&Co?ARIN&D_mlbeXBxK^e! zi#>nNz!>n@juI_OY)W6Vvap>0+z!{ddF1!PC!9I2)nR5aC(06DNuyirCwv())s+_- zN2h(&tIm~1g)#~0J+y05Jb17&=$~fFE~3(vFBjWd;i%FjYlz`0(7w#w+~l-`i^;-q?a18 z97AZaWOrpXd|E|;T?ZT^&Tl|hVp#U)7!`b*Z`s&JtT?@<=7O7>+r^8Aarxl&Z{x-! zifhx1%Im`=HtRrxoLyWl1_US#?RH>4Li)XZ`}WXDGdlAAucw!%`?vD?ejVJmG8SN1 z`dU_2*4)H|-i9g*Q##`-Dk{(@?eIJ!LX#b|v{m1qP|J6^R+^faTt;0z+J~$xfBSu9 zY(*KT9^~NQKu%5`v?f6 zNIRf=Q>MQJcI1aF_9KQ>!WJLr;R(i_VG39A{?N~=vNYYFTUZ!9Ez;cf`|=-veHZ3O zXYTHJj3FI-e0&7X2T7Bg*x)`Pn6!(T`Cfkhg^kqah-NF|p>BP50lIC1xeOsJx6j=@ zz6I`QiCxosY$@HLoVhT}WDGNCy0XdXdu6ASB->NR1xS7^P7xKv-!m0=Vr`ru9iLiw zu71KJ$G6aI-yX31)eop7<{%!PX1vt!9%$ETbB7D6*dhdV=o##8Z$FNb;cK+)uTm;o z&|#0RWzq+CiB1gc07dZ&lhd#OT(2h( zXcQ;3PJY5w2v+r~8%3|OtZ5Mi7Tn7DlG@tZf`YC;wdSR^agk=&m(_)iJ;p{xKaLht zmW_BSZ7vbrxahbQrl+P>KxcGzbzNIs{pg256ePKAX+1{+*Q#E)@+W9Qa<1AB($qf2 zCJ^Ke`%dZ9o78;&n1nIez`LPp8f*%8{R!q&!R~FH@4KvRWRwEh$41zUU zX=+l9C_|dxk-=neb!x-J6t)=yr`?R}z>xJ>JV8NxQr=R=soHwb< zKXZU9Q{iGoVPPTe8xC8IFBTUB0Lg!156u5?wC1k-bF=*j0j%v?Qy=((5Ev+yW2XnnYkWn;{rj-5f{b;?P zXspI;cY$@s*y9U3DBAL%&~2ai&%D~Z9tt4=@9kJ9MfnJzt4(y`z)_>A7R|{u2w}4! z3=1}qz+xcSZH?!)lg{SfyLWFaXt#w}YT)`WKY1(UnE!*Ix>v8k^sJ-mxn6NiJ(B3X z*;snNDMKGacmLNH;Qll2d*ar$!Sp|umt87H@C`5iS)Iz)NpE$nJW{hgmg(OcukGA( zR+8M##gGNmzIJJMaS<$Q1-0k6FBt4($b(p6JG#$NM3#$d%S`fCIqP!N(}tX zXyZJ(Fo!o#lj1h@&XR26c8!-OyLScLs0d2a$nBq5!f>1OQ1eNGx)nQ*A3l?3@){(?JV+dL1|1iJq{8yMKo?zphBl0PuBxla5Wu1nA4 z+dV7aF^m4ghp~4Zz`vReJDq>R=zwUsy`ScSq&sVmoM}wdLLN zzHazs+V>JhKYTmRX?5@;?Ao|8?!VH7c~00C9>7TTp`SY9&CQ4(I2om{OYK$`7T6EE zK8LCW{S%t@Z`G;Cf$NV(JWEPTWotUFJUJ6x4@OF^(8BWaH|TES>E-(*CHs1Nd%17U z9lxm~$gNZ#0FXI&>yAB&8+eeK`Nyl{bjQXEICk%bL5zus*~zW&rmt`O`*$VBr&I}= z($1YX{%o$FEH*0|y7c>p!}`{kcE`#Qe0)Dk%;ss)qM^5M-n_X~R$gBI%~Y6~X1RCH zN!wYfPJcIY183Kl&|aPQtCP0`A&YnK-sS1|cKSnR3l35j>4QTVFI+2uFe4Q-Oo{)l z`TP9JXAt8tDK;)9<}tutU_$uF3s;YTR*b)XMf@184*;A=>T;G!*U2`}6krTjDOZ{v zdn_BSf^LteHa0i+1-S;59~BrePTpNidBK=?H4j_04?Q+DOG{NXbjZDW?9`s0<;V9t z9`1lj|Hl2_rkSyEQoG~n-5GPIuDDrD8(z6`C7uouo-^Ra$;ohsn?vqghdkfFSarni z##tx@`0#x=o@bP~XnD2d|L*A5p(6bNS()G3UJ-XGJ>1;Ndc8)P9tHNDe^j(wgfF6W zHn4B)&7WoJ*Et5l&`JV~U4N>-g${S%cQ{BL9UWJ0=m@TS9qQqZ;#Lt%`u1PBQd!fp zckkA{b;WR>>QOR?%LE(Vd-H%3O0>9}8Xsq1E;9~v_aRzd6C0bOFf=(H3&N7_leffJ zda!}98lP1E`r%r7W3uT|ac24J z5J#+_pr9*E3eiCbTN5QtGrkjkTO4_RH-r6PclJxDps$U9`mMxR1d|TK_dxQq5hffD zN4`;@USoxb_Q|)i+|;>W+PEenCwFt&CwMEN&dPbL4wFgSHTUBHFwGzk%JS=1_kh~~ zG6=vC3T(9Q9MYW9UgOF+9=-uYyF?(p1_pg-#OWr@(LYJ*AdIim4okSp# zfOr5#2s_&dFhebE>+NmCM#*@MH*aSZN5tfR@gjpw%v7-3vPqTIH|UR*Z|_(+IXAu* zmA*_r*zulao7`Y~4C@E9Z^B#QDTlT@GT6|0aT7Oa7@9XDBcs&jF$+DtvhV(^wyX#C zbwf-yUwpw}S8h44anKVu6#Rb_l{VJnt~NI-=gfW;haXgnSyhI)8pSq{>y8NQz45=j z$8n^ky~g`^tv){R8;qV-vF?Y-_3`mRZ_6We5lrmtb3;?Gg!yh#V1c4%5ZXY#BDq7dw6UR zjXN2cn7W*Q!n?fdq<(>-pT_?oye5b|lMT5F6>e$KlP8y9E#dAS#`J1>6_};m-NlX{ zz2s_l@?(H3)Bqv@Lc7Dkm7q0daQpjb6O$x^p2hF^+kbpd%QKlVPWY+iz|$K4{t3t_ zs?D#c@Imqirrc|AT1IAO`O=e@ zJLTtrM`CRszy-)rkNJbQFxt@ZaO3e^P}G==cfxnqG*K)9vuIx_$q8Bj1HXo}v^1eQ zkXz%Go9{g%l}DCY=yT$x;fsviRi%=8BR*0@6rl9JYAX z1_zVY;~tSDewmt$6_@*==Z=;ZO-#G|<=;Qh;}eJwWnHudB;tB>bcKZ&9HARU%0=_b ztW2b^V3)bt%I`h{foDDbb92+w*FRNm(caz;TK*aM>J}Dk{px-faHTS_3Z#*1VSx~R z(Bo$E!X*uETmJrGPtLB!`rZdaP(wt~7RaCia&rDqI%Ss8qHgZ)s6vT{bXVdkf*ML( z!X8XV4sJu7Fki1)1(L(_?(W1cXU{`Nl$7jv=;>`3l&P|@v-_=Uf4r^sfLh!^P0b78 zj8<5c5F}Zv!By})bdpFRhWpd?#+jKFnTn@Xj(DbMrDS9TeC}lEG!>sfKx=4d$V`)W z_b$n2mp>y*94gb8wRmLn_|6!0J|362INr2M@{+;-KLdk_hh+L4i_uHrl(4^evEQHXbcJ>3(VsLn z8f>9VrPXqcEXH+GS-lAl7LLxsAwGJ#v)l8KcHS$Df}z;9?Kb)}A@LP#Jt+^bgH_A| zS{xY6XOTVzIA*~Im%hDwjn=fY?#x+os&Px*-~Be6FKPtH&J0*6aK|4%H+Pov{?9p+ zx?y`^rO5CT3rDLv)QVD5Q%6TfF~N^=+cvjqEA$c`f_%y-DmIpv8|E275l7!3;k6O+1bRc6ddKZ+Wgrg0Zm*|T&%0DopIzs3FU?w z_ZOT8Mjx|DIY$!3HOqT(o`=T&I+Jc+)yS>WtE!kbW#@}yRlirk<1CUE<1a0ds?0~@ z)GutyOrihKb1aH(Kp^J#v#<{`x@%jF4H|dwRWM(lODmpUq_vXrnsop!^hn3=vzlBQ(y9lUZ z5>4@rm5}L^|2aj`%x>*I7~RSsr0ws#&SZ5{+O(O^ST50XziWlHS&Z1%&$O}RykSf1 zj^)?*Uq3Jv(S4-6M?&w>P9;k>vN{cPJs<3%jg>Y%qY}ASE9J)xb-T0nCqDZe=x9-D zMLaaR#8M3O9@lLLm;4e&Q=_;DUhvTrt-bZaKZG!f3NurQJ_vw4H_)!AcIJ+>G7s>L zW_EZVw`+$BnS=o{h0r@vsVPLivM=LN>)&T8e@*#iE>9S}(DmcmL+;DM*Hsw=gx-Bn zcajv={7;J4_rJ^0{PYVKnV%*ta-BR2*A9M#Q|U*NeCAnPNg|SlCAzPZuHt+6E*E?t z?NqUXhY!)^o5{1EN}dwN?B%#ou31pF-{O@ab{595Z3p2gO6uei=RTp+~^ZxJ7z`7THr zWsTK1$Pue%epO76Dd*^mz^l%wa;hoMJpwxKrT)lcsngr<%5yK){Pa?MK;_mt?D>fk zUFcf2n}b6{RP^|DwqsazV3`}&IQcLfAX-I5Me(x!Sap8g;qx1lVk}!77NzP|7?O3Cq4@0kC+w`OD~KrjW|lD4T)D)@l6VRg@pYB64a8W4q^B=SWt#n! zHuh5}4|D2wzURJUh0a2ZikBninq=|IVET&b<(-rgLf$ckMQD?LtL>vb+xdy7QLyj< zw(I`A#Fw16SD`1JfWC@s=36rEf!QuW(kaXT)nd!12HNB6Rss9>6A&tcB1 z0$K4rUTHDMi$EB7*4_@_L?2-332ST4{>k8u5cSz+2ExqK!u35r@nBcDP*-QFv$N9t z`o?eCE0;B9jqA+Bn*Lhwm{Q*>9R9Cd%_mm#&(wv3+kX1Ag*q*)h_@NfCjH$|iPOBr zt3u^cME~CUBKtZNZ<@G`{OMN$4Yqp-Pv-D=xqZL97d`9dV`|dQPosvSy;HYx%~-{) z;mkk72`?@#X4izogknA3_Qk{{$HEi*N58$ZTfv|4F}q0Q|THj`ka4j zJSD)U;~y7SD_vi>2RyZxtWso&QqwE#Y*<*`Ad-)W2aAtaSru=#z&d0y6$4Mq$;kmi zv;xFy2ZCd2^KCX&2V;|XrRA@?_U!pSUG$;u+$p$m*oQmVSvdZ*b=~_hi2O8WNsF3247Mh}O$~XZOTKYK zIa}e0haJKxyq~I|qjbpOsgePI+j}?(D|xL z>5yWYAEm~(er8KC4lvJXpSVzmq$i>w*UDA&*(lpn0FVfSa+w$z5$D3Uw5NN$e{g8% z#mxh`j~OJtbY7BlSTZ=Pm8N(lyt)SA{qGo6}ZKP z;$vg0t%`BCaWi0H5fRW(Qy&ia^-wP#w@^_$jTATh*7;bT(NQ>Qh*0YH5urt@ra*n* z`uSPo|Ni@rt@>;9qk$mAH}EjqCaZkvHaZjdi!-I94j#n4S;m~jn!-Z&-#ZkV!S{{psC{P*8?Ao3q8{&Xqs z`rX$8Xk?%9ec#Q`&C~?l*10PYnp@2F$<1ji65*YEB6dCSjO6af7;pr*cnPqghu;2O zLGvR(&G647UUOU==lLbL9c%6G?hX;$-mFf*dhB0bX2$$^qDSR>Ytv~CXwmC)?vIVH zp2vP59tKwU?R-613F(#?>GihQ@1*7=z5K@)7w>N#d-qGz#l?k+7dXJBW%qRk763x! z^eVX;u_2GwP7WcU9~`u<^O=PD+p%K@&cDC(m;Oz?{9Xiph|09`-Zs761D^`eA0oG) zpubH|Mstm?;=R}|Xg37dI41wGD^yUj$-(zgUS_r1*3f){W(5-C?7pNYN@TE}rzMgJS^by?Sc@V<@4uwM*;l!pvIZ zfdCvX{~9^9b?erM9lKpW2<+y)XCiVkpKKp|4Ky1M%=n`JlVQ;i+qKoz)e++a17V#LCq6Ft$L|hdj8gkXceU!7 z6m1*8VZ7j zqc;w)UFSZURb~D01@Qe*euFB&f5X?Tq@bGYVh@Dx6j$ptIEQ;%|%SNrR{T=qti>t-Q zedRNa3ZD1$Bx&U-aQ9}FkNUd0<0SxrML-bWzh9k-oRTtrx2!lnzZ1|Xo0;UzF#Wck z9#J0R=dPb`Ve#9yZxJ~n%oky%S?5**ta2H{Cen3Wfn^@C+39r)x)1{J^y$;)7s1Ug z2=B)w5Z;5CR%iYG)hj;6sNqsO3C1YSdim&%Pna$rMt{u#OD-^hApKmuTwsC+b6v3E zM1~8Zg1lfucD4id6AIa`%B>%W?%K+CeNP7+!GrR|&l*V9h~`d#;fiDhe$Jjfi>=J~ zuMxhKENnZznd9uXZ*R*SLRGRYs@->u(o$KoL$e?m0y~zMM^r2Y;*3!CqMf6?wbQ*?P(VOAiF{lfH|B?AwH07b z0Q2X+zt{DD;2xuqTfM`Afq|i<&*ub`Gk|*{Ogijy&$d(v_d0Q}Hc;w*KflB~cRoC{ ztc;F{@o;w!Q_0p5bdpMkA5$0Tz~ZG?IU<&ziA=&nqk{Sh#hs*lLa4S1CY>IBXo)7M zu3B2rW<_TnV@_!mg_x336Osd>DOx!qUA?_~fZ`ItYGR@fJOf}=QKhs zpC~e34A1!L)#h|L-$IZ~v^M#({(TSnWMitcZ>zdZOV#NCppBV{i530}6P=x!<~O64 zJufv}fs+W)G7)C}7r}FqvdHk2ZWtBZmhmvNwY7D37wNB7kYS0*8*&dRSh4yFURFRB@Y$#KIkV+4%i=8~}+ z4b3PrqkUu5UqotN(rD$q>YovT;*poX^zCgm&|}>T@0V_<@nL_ULDBbG;?SYgX&(gx zL_N@?t}kz=Cnl2ZcouvF0BF5Trd|ywn-_Xb&uisX9cUO^5kwAcXV}`9jVMeRXTCuJ`NpM2jI`PTg6*z zfYbTf!LGm^nXhg9>W#Z#E&idxWqVxA!zNT{;M2eeY1t%6rROfua>PEm@ZFZSEp`7% z4d4Cq#d&!!T8I?)+g)C07cG5)pp@iGjUCe+ku9GwDQT5vPn6`?E2bJBU~1DbwFnk| zq`567heK`w!Vu8k?Y?8QZ$0RbThF_T+}NubH+7JEQ8&q|)sd>A{H3YOtD<{kahj*6N8f!j zGB#e|sI>w_gc(RWb75)X{MTEeUMrTf%7^Ze*i|WiMbBDs$_yY~h>3_znf(^=Q?q9g zKLBn4wp{r3=k;<8A=V$OKz78aK2b|m{VJhiX)tP?+71_jM@%QDqF2k% z&4&HvMXH$gdZZm$^^L5oqI<-n1v7z}{mA+8l)cbrUC6#wa($}~fBaOX|Ac4p7TzTB zlf*L|8SY4E4YSd!ur!FMK3&L(G)WzVQk-x9B$ z+b0)t-G>Pu*AnTHbL$g~QJbJGCZFo9+pl=X&dTcR4dVbl+BPI5H1Y=)fBvjLmmxi# zKzvWCwz8H`$_onu(#I;SPL_0({dB%2Xe%&eHRo~VkF=x9u?>@0E0ic4Z1pWD(4yrH z+xIE&rbQlHAdc!}rxtJ9uZk=&L(7ImND3Sk7Je|>-sjJFvCv<#Lh|>7BnLp3xSdS* zQu)K^wRz}r7w-M^S8Tf%$C`I*!jz+0t2ost*)i%>urQ~0Mc9gca92)3bmD;otzsjz z=-AiesFHlBDRrMhlyzn0S0>Qqn0l<(^S?a5n@jJ%pQDLwSEdN1=5=n> zVF}ec1%-W0&hpie%3Fs$XTDBo{emNoeEvtHo#7HJ<&v?5oV;N=?kQioX9Dh|9m~y* zRHqFSHNv1*QXp}52%lSv;%{oG?Yo*XJvZL};!yhm&w zf+g!*;bkH1e9>QhcX$ZB`@xr|jxk@0(Gf)M0CY_t^^7)=+C-Kb;GY2F2}ei1{%X=V z%CeL%g!Npu+`V#TNJ(MLxnFRxUA{n1)P?t6PhVlmuTPv&R=uwaF%!@8gvBE((-Ovy zPp+}1vFi&aSsELMYJ;ttrPi8oO0?Q2>l)w7RL8e-w0VM-ceCFl?NXuQ^|@(Mzs?>mx_+Iu)3o^$yxy_F@s`kqw>zGp!{ZKzg~YOm zrenGN^!G(OAnH1$kDGoT7fN&eaB`1YwgG);dvl;)HLTa#i$LV*!5k8Yqy<}fxs>OEY8j|4^4m^9U@_(uw~HaG8%*p5FOyD$bB5`t4;i43=XUH!q%eGG~9z1yP$og48 z+>XgYF=}L6lyn60HD%oQQSv6~rSQIHdq`2NplW$GC(6jUt<0G&lu0l(!$Qot&$+W& zkRl={@y!vb$clXh-Jd1H=%w5$4<_a<_+$@H^whe@o$K1srDgQ$7 zQfg726Ek(XoiS!LSB-JK{>n8?J}Q0#M{v z(_g)z41zSdZmCVHUz{ID_IsvkMp-U>^^|dyNfDj2+A5VST1fzvf+zE8csP`7pNqs# zE&QJF3!_+qaAH$T>Mz`kwEjsgd zg5My(IOF~99rd;lg1l^9S26b8k?q+(&)Mtr*k>aFM(S%0R>hBI^X1_Nf_j2rpP>0% z*z}YuOFk9(nKn<$(eAeEfd>K(j5>u9X%&&8g;iC)tzpzaqc=O0{w&nKhGqLoWaALe zaG7fW6}SCvwQhAtR#s^5-jtM-$*HNGdlUigSob3#xsy|_Z@Gafn4lRsA+Yw9-z=B# za+M*~(y_#(z@tR)pR>J%{h9*;*%ptTcfjs|KLo2)+xgv7C;j#@hNzRQo5pMk+&c_~ z?MiKj7vpvbb{O4@ru zjWc3`NDiRL-0ACJ0Ss-+?CngAY=avu!PSbg4-O^^8Y;Y(wK)GKd^;<$l#>2G?AiSM zJCUG6R<7vvEKt?|f#xK+UEpwz0e%PN2mDF|unJj4qi(~y00xTe+t=bEQdSI?PfAem z3#QKe4*L83uXmmGW&J-UY>A$Ewj8~$``DPB%WsO&9=m3yWl*sOk{56?d1Tu$uQK%p znY9^+K*;3(hZc!v-aR5R(>+_7M_?b%>hdKZ5QBbBOk}*EZw_w}nC`H9b?sjsKp)70 z>*(px?cABxGf8J7xnypULt7nL+f!o&z6P=;m__NJ{Urj38Pb)!VeoH;n}UMy@L&?5 zmX7?3Yx(*3yw31__0-D41|H!vuLe`Cv!_Q}N9Qu2H(kN}dEQUxr3uN>)6!b|(YDi6 zWLN)WZ7S>&_;fVk6_^gvX?ph^oMIBl0>6Co%!*J~cMwIGM?Hqc@{DGtuhtO#ZYnB% zqP1m9OAB1^&=uBzM1cM0>P9CfCV-7>Oey`n`hNBdh=d(apMsW$uy%EAjc@?NU`I(R zRQpo8larGP+9ATVv(f^&12Y=&+?nv5jEwjg4?E>}0(3(HH(4tWgvCFACPYO(hMN|Sh1K|otB1sM3qWS2r-4tNXqaMsBYe8ZEbg8Ls#>93Fl=<)D5S2eY#2wZ=>rKY@KReXq# z&k>9-9Cd2Xr1aai$E@@Sz6crYJ z1?C+7M<;#_p(yaWb!4rs;P-}0Eyac;pm^g9xJQT{hKA5X^xY6y>QNJQHMKV@%DyiJ z#+hh2q=ka?rg#W6yKH3cfBucoDY5^Bfrm0OZ{7s!>>sKE zwo|%{uK~C!eEj?%bCy`w{RHEIpdvqwWB{@VHtGn#SEz|d@iH#3JPDY$>CdtRcRVW_ zTP^s#K)LNOvK8pgMMw-NX8q*#CAi!$Clg9wQ6L^g*hqR!wm3oBAXmK8JGFR#7Rm&L z(nbZz9RwJgZ7^NyO8Nf)W!VNd6B7YUS)aQ3wF|sLqD}em$^Sr1OifGs3l27sj|GDq zxI+^f8xXTq(bV73z<}OxlAZP%BnnBEY4qAeJhca5Ei%gk0|P=jH^--@j+>ay?^RL? zLLj{-aH$_Dd}4WN1X4=pP;T*Si~>7TG^CX$B_oppnASo}@y>rieKWg%>Ix2^Rl2V{ z*EI+EOueEZ5Qn4GasUrKHq5zX0@UuCXxp4Vz|Td7$Ligd{Q*BzsAhcYhwodW zh3@;m4a?&IcE7EP?cLkn^@8~}KWHV=(!MC@pvne>j0O%B_4QUD{qNef36X*xIRaXl zP%$nn=nomNQv|nfsc)-Px@YCcLo?9x0+y~0xf95+pWia^2jI3sgxv5|f6?MG%O3!) z0V-4rVmu4hD4I=RyJ`eoNzJMRIC?^wLsx*Yy5YEf(U2pXtKD^IPXG_>*8`4FI+Y*$009rTpjtpTd3k(bO#h+rk0ks)4ls5vC7`Ly|QI zFrgo(q|)bJbVrFlb;9Loedpl22e~ebi13RJm+0-Hsa@ z8#_8qUKMZP{sGZowjM?GF$N`pySe8~K_4aV=DvZ|w5A zfS4c)1f&-GH;`8Nxq!_{HCvEx+Q_?qpV1~Hg5TZv>wIow;}yU@P^HAjDti^R5%bhC zAu~f-*%}%`=LdKVz^$x|%$c$Je?w|!Ku@(~XB6d=2HWA$?5EM<$RdCBuzGCds~Lb9?* zIDj8S@-n9{4(^DKjotgX9(rS8VWBT#EE{cEt%|R`Blj{nd2%-^E1@00qEaa#yDK_s zAIw2o$r+fj4!YbrhO{7{dU~7isRzZyn=roY^z>!G%{omGT|`VV%S~9gXjcik5@I!kX~PGzAEsM-|qIqz@*$vlN33IVtEH zydoG*44Nbi=VuX;7g6J_0BfZzsy#KnxLP(^IIwi70lCcuf1kS1ckeQ=H9v?h9nG;v zO;vWFf_Nb*f1!@9#C|tph zN&1Rct;vJ=iPgxW=#`)ul9S?14GjnS!K7ghZ>v7Vb>y+SW9aNlig@gvw_ZNsodkU5qd)2($b3 z63Ha4p&MiOj9oNa-I#5!{mK5+?HRkp_*BsnON2FB5H6{gBJqW{wn;{sy}V~u^j%$a z`~8Ft@P)2wim}9~^Ifs9xT!2=>Xdm@CRRu<^(T9fylcNpjJnU$se=|FlE#I~6BG+H zcyU-TQDK`sFf;y3`BZ()^|8BAtdT?4ovY=KJoHl>&H>YcgEq=S&gS`D#+YQzYO~gC zo-qG~Zj->9!dYe|%(Zizk~qsqvRtg^6Erq`@on0+mj!u=J1DMNr*IA*`eY!N<9{IE zqaZ@#bfBA&cHX2zSJ#taRpu)d$$4@5!}ZghyYK`m;x=#s`p3p>s$>?A+$?UK5V{e` zeZr8r(WF}yxewdDlLv)43#^j3B*x30iZDqPeEK#J!KWyplhZoiL=C=FYwK?UM$-%Q zdUNlkSw^echLTMfFZ}{-A$pUz76FY{MRkTUam-f(SCp}N=N}*YZ|^yO&hAfkpmpUr z(n(EMv1FNK@!sVP6JtqLD=;iFad%Nr->W;s7j`30!BW8Jpp>)~I-D8h(ae`tS0sD% zFvKkW;9(1G%eL&UE_uYd^06%fS|SkQM^6QQPViR6J0A#OeAn3aJH4mUSF}ZA#k!di zhXSSJG?||r6FwP7uQK5rwJ90eqAZxy5<;J6T`+j=p4K1!t@aeOl{_4u5B z;GHlPFX(mrCp~idgf&Ym`RG)r94<8!Y2p==I+Rvv zxwZ!`g|_ePXZ#YMba?AyEoV`A0Xv&FUv|7xJ@F`<{rG&USLtr+2L25GolA)kqh+V{ z^1nP6IpNVKR}tkaUn{>eejn{r6raCn#G_y9lqalqCw`GVv3})GIfLBom03=mi<6sI z8a{l#EVz~Dc6!r7_+?tDbeV&Ezfr~&wEpe`avHF-D?MD>Lv9cE1@nZ6q2@qJPR?Rp zSno(eEFak?$An(7(B{(qnhUexQT|zOhYw&5a&VZr{(0A>m+x)!2S$uSbro$CIuzua z6+K%1vtUOMJme-VT&i-TnuMwG6|}<9A{VIgGf_1uS{l%t^SAZ5?|1u4TmHKqdwr6+ zJu27VG*Lt`c{JHTn3?Q$p9}@*M(^jt^gmxWyYc7_*sXS(&o{20O*ZH4vR-6H0RuI~Q=DS@5TLHXyzw$d22>+jKS72NAsu9Ffi(Czi2V_Rqo++S4L ztfD|n+yi~!IYYbOFgO=Tr9Wpy`+uL~-f#Db_ucAp4r>)LjWHfFT9Z`F{&v55tV3yU ze*pib?}Pf^QVWi>sfXUQO4GF;rQbsp|LlT+DcSSIKN(!Zf9Pz}?q(=?h+imhi-w z%w!3gZO8P&E}?yT0bc?Y#}2WyiD*aB>!s84GWRpTEiK)3di&vh&wt;mlS_MvzrCMv zmS__n_fuq>r3cY6UE4*PqrvvSx?E*5M!U&-kBe?UJ1@Uu^{IVKzuhM;g|nYBOn+gR z;wT0Qphq1V%=@LQDAPNexhc~hOQrF%Oz9~UAFxtCKOp1e%a^p}N=#7Meb0ki?JiEH z%$7Da-ZVe+rYzgqfrm8Prnzl0T+rgQ&rvzyz@j+8!cw61yNr6>=Lx0TSUG(2@EvvRH&{o_!lO+Mg$jHU7e%5&dGajW4W@0Q*7EM!&J=NO0A;ffYh~pDY0iwzTcXdsE z`DY>O2mikHC>;ZAg3`t&IKn;=p*)?AcQF)|WhYJ_B{A#WT&)#Eq>4eZ^N2pFcA*^TF%FC-O&*yq>X@1=>CACQZyRS!ieNKa@=6Bl5p# zoNvIluH`l+EG97;iI}+j{}rwN+czJTYT|k6I$xqiON1{lcdng-Ux%0afqDQah3^>s zzpvtQjW~;YngSsjTYJuq6<_-c7~|{mn!RK#aAI78C;xVCgV6Cjv+t0fu2Z!j<@Wyu zjG&0@3xK!Y+g*Rk?Y-{Q@Tz}27mYN2!ch-W=WA{#0!Z*!cPwH zbaF7qYJ9K#6qk30649FoC*6M#``>a&T{sB%Gd7t<2V@ONV zosG}3O{*^;`gZdcxYCxw)mEn8zI`KN$fbC!fZlw4Ahf0%Tv(hUHokn;4-n=F& zef=1J_h%~T&s4^TjV}^Gt35-x<#zTS{`5tp)BSBz&|geP1&4=d0sFtd z*@eG3>E!eYj!}Hjm*)q2)xynY;|zoqrf#3><{~E}go-XZ3=nlZ2w4|Ky*t3Uv7WcM zaiZ%-aAmcj**lKU-CW%q?bj%8o7tQxJ{`O{x1Ez|>5cAk^}VA{tje98RI}p7x~(!J zEQ&JO~mdjJ#m44(PlXeacV%bdqm17T~AJ!;Y%o3GLueKAdB<7Zlv z*afl91CAGyELDGWb3NN=S9mK=de?rfXn!{2}`+c;wHKUu)6W28BuWb1O}zCavN9*ZALvbkgXlw{MEZQ!uas;TY0fp8$n+OgGMsjn2cUIACfJ~ z1FPozdMjDxw|7I?^CgWBlnb6!(k3|`6OgU;lkO6U7V?%~%L)EOo?1L;afX9zYiHcs z9cLW!4)zJ&S=eV4d=aa4%aTtmHctrd&2Cg%11T2q)v_u zU8&Ufs`1CuZ!5c29Lbo(L;%i%pMAGYC|DcVZ|Ad#s6KXk_wpOgdCB_%3ns$dpXH8K zKl@TqaM7@PNVVOV?6_#CXw&`Be6EtC`%dH*)Q#Ltb!;w2v%vNP<2UV_KP?;x8yV(v z`Xx73a-&MY>lvFMMHd;2^ z>Dp~NJ7%wPQmC~UmJj&;s?UbEm&Y|r+x2Pef*yE<*9W6flWEaPNyVkEbW=(y_r z{KmjMRoT5ZksZtHDrAenxxD7`H$OiZ_((V9bs%m_$34B__g3Lo{`TY+6zoI&_u@rq zj#@o+zJzK^<-mH7O`*x=9h^AZPF5LJ<(5OVYVp zJo53r%USLCVv{j@{k}2IIgGs>dov#J;pUFD=JlI%t{q;GEsyaDx@p)s>1}d;UZF!~UoYh` zK#=JxZdl{}iZN;(N%>L_DTBO))LFGq)g{>h(~(M+sh%AJ4sxr09k`l9C*OvT0A;k3 zAgA>PhvTQ#q{~pt8S)__%e30AQsjL7p`hvWGN$ZLjvM4NhPMydMq4C&t?bt!%bPw! z56RN9bNj{XO$gso>eCue6BI-Uj}X$-BQ7fBGx@8I@X7Ym6Q_PSY9aIa>-@vkFLD%w zdK221JmCL3AUw!?9vOiIwPSxzDty2Zj!Hm@(|FOL8PUeIP{|Yy;V81X0xJTB%*lwsTK-c@3Vn{j(yB?vc#wexadLBOj_{2D+cFpu z0oBM1w1xrdFO4Pc2EOr?wy?8y5*M!$+^Pkse6HUg9m>Wm`aMTaExvCRqe5-(N)FN| z_~8+tqzTl}0MV%sWkZdXw#q{R4x$pNx;p7-1W0?#u{@6j|A^PonQ|0#6%=X4=$S(@ zJuU0w<{kKr5R`4q?h&<04q7TS8Esp{fKmz73WpIpjZQ+WB7=0K(fPAco$h zE5b`TM{D$>+gSDVbIb5V+A`6FYYP(D#omRZMaY)9+OBA^rys2P#YG5{e~+}qf~H8M z73M3xj1OTDp;3>1fvCyQFQ2FtPqq$)YN|ej-KE}fDT0s2-6>r#~$jlr! zK_yOw$njA5_eJz8RZCW?(nVRf^N*N zMXsN-iGG^#({t}b#PbgG*{E1mf@1ilqkv%YyDL7XOQ<>V6uPWk}qF&`S!4_zXC;^j>_h3;Awij=1jLg*hL@`%qgRLfbNtu*{81wRk zx}QK$hki&*>=?ZeGGcYI2Tfvtzd_Y_n^Q~at(=Y3>=Yj#OB!k0u0+2Imlnsg^5hrK zDRS#F!?cpEUj5rvuIb3AE9y)|o5TRc3XND?ns!}jC`7xy+E&^<**|0B>75XK>e7)Y zhQ!-bBNG^@FgfcJi$t(}{Teg$>KrbEbok?CKl)A;@% zODl6Wp?T`?nKuO>?}|^G#^+x#Q()>UF(WDWi>0xe&WIWs*Yf-L_{Bkb8%4MDz(?qhh9 zJ}ao21q4dp^}Lc^=jmD?7-u?H=bYk+g6{<%x|`$FU=Wc+kA~FQ%y{uR9 zBN}9W?nC^u&)a4rsJmT_YV_;<63;8;k9ZiD&_6x$9?*qV37tOJm9xZ32R~BlOi_yQUO= ztT^P4PLU!WGH$(J_gNGv4=&$$yb<-Zj#d9?bslhS6}mSjf&#a}T>qu#;|I zrgC|Gr#B#}0?t40sd4A|J~{*i{vDY6z|KQvLjX0k{TS1i_ZtoJPW|wVgz^Lfar7}q z$m9Dv31+;Ymf8EJSC1^kdklr-JM-7!%Vh0VxkJw_SH(Bn)+@fzO6Fa7-XP!h{wc5@ z9l25$_5L0h5!Ruo(Q~yV)==1azsMuqSmr_XaFf3nWyGOn>XW@c@onniCDrfjQSjd5 z?6@yKrvDio`AMK>!nvW1g~{@Zd%R}xkkFcu6G?Jc+{kHNmS7wqHI>j}7{EbEuTCRV{DcjDj*)v}Ff32gm=F*^VF z61!!dHu;@8W12i&?ead@r=GY8g8EG)5CY;DU$6*54=kHk6`lYM;jaws++FWoj}EGX z>K>)1_Rs96rA2Ma6GBoAscG^wP6R|Z`hgu=T%9vWqu6Vh?PZ~70(F_|nfAQ?qZGcd zXeLi8pAix}XRxVF=sTP10()L_Cr3MiAk zFCE<%vwTVYnY2HC9V;QF8ykJGJ$TZi&Kb}9;sW)H;>VEj%YV!ZqYFrtKd~?wYLNPW zs17QVfGs*`T)Y{=Q`-cG3||(gDBed2HYWRWDE1DiayZ_9(c^D#<};gJ&ArgGhu3pL@qryORm}2JZsm%Mlmq8Y#Y}3b09Wdw0K4*Xv;fQ zTPuCPJmwBIB94SA&v1d8~$$G#+1;fC}uOPl2l7Bb*OJ4_NS7A81Ns zf8NIYWqOpu5Jrsc_#MRS*4~Kh{GQP~ki$9T4avaFrx?Y&S^YDz=5FUyyM$*emv5^{ zUn{T%JP1&Vq}u3(f4(=hzj>9FZ%#u>Bu11F7`FG=;8YddS4N%Va_Rft=hIFXYP$C+lGGr8~PZe5a%?H zcK!*+`hoP1e5jhXa;3?mxtnhfz$O#K+op~Of`D*|W+2$CVC?#N95IR9 z4za9rS^_mBR@kU}|K?Ua6L{$4Kk+;54xzIcyV}dNgbEcewXge=njRT8`P(HMwH~qu`PGl zPk;Tk+jb@lkMcx1Y@p~Li ziR2YCz+y2VN3`TQf0=41RnEynw^=` zS;`{Gml?ru!Kz%7oUK~VCtDdgy*V>~&R(?Ft4F>x``}eZ>Xv*@UHIR&VrX$?=U4dm%kVA>JzqI|vP0jE>%)d(3;kW}LeNR>!FrNW4iDppr# zPA^ukV%>Prn8Y(Hx?s=S5E}3L<;Q2~C-(!TVGd~_i&Y>#4Z#(_ZEqSoZQ-={ZI+0w$Ur+vP<5mVN*a#`l3yN^^ui}RBvL!;?0y?HaI-12Y`^CCqbeaISg_j|EF(249J$Rtw$8DQs#tUXismk@SG2eK7qd`|+#vWQ)+Zj^ zOn2@teBpKhjTFls0t9gYcuic*=7BeKobOvOdn_#rjvFifS({;pGbbut^s?QHcr0w} zlD%R+DMtL@Z%$?#pg|X8v=Es%Z?CO&IxHOSWCO~69SHD^!8f(QbW3!k1HJ`3IP`2n zyf{Q)|BzpmRB3^M%LaB} z7wQcvYln}>egy^tfjA&Qg!D{S7a8pRkl=fZ`W5e{dvmbKDBZfZv}3-OZrEls>1guH zbp{nL%Bb3U@XzwUzJH${kA)Q2E8b;}j+j%<@FI8|JkVb4)8BoM=FtLzCkAM3f$zq37#_1+o!0=!oiYi!;i%TQ$`vfgln?+Y0lI8qLCe{9!+iz> z;bQ<_Pb{cuw#hW{0bZQUOTq?9_RZ2(IlhYr+mg-Vsd%MGgrHt(t$uenDANif?95-< zGu0W39r4{{E-OxD7=x=0{{f>`UP&N^h%f)D4N21LKN}vDYNxLooDeg~Z-k6#vP)m- zaOW`*$Vj)+sLmBHIpV-pL#(a9*aHg{3I-e81HhV4z5zQ$*!YaVtm~vCwoLOB{EQ&? zQZ$UZVj&Y4fh?tdcFqR5K4shX*>H));7gW=`oT+L?yVQrw%>zuO8_@|=-_I99uxQu z7*O}Y&jn+>z#X*`YwvDehQDVsBXS3R`GHvU78oP4r|PPkjL0EhzIdQ`=FJ8!1CjdmiC+SV zojJ|D`#CO{z9Aaate6ITgWbcyh%2UZ5yq530AO+gIHLD|ApMgQ#xJx-3D+_UWQ4$_ zI@R+U1e65<5hONyAIu`ffhQH1Ez5u5O5zOH)&eUtSgFDq3kVYYj}&srFH-Wam4+j> zZpEtugfwfd`!C{Rrr;*8)}ETKEZczD**Ao?AQewSyP-eM>`ffJZI-3@`DrAix~-E( zhaJ)6DCrC@T)p#u8R#vOb}j0!no`BEp9QY1elFs>_cqBAQPrCpuY6^K90>!4paxQDsVzg|Dk zs!=n0aEkGC#{H#Bn#Ya$Ra*bt#I9m%sY8Oh8vj@Kj!~hHxRYGX`tXZlL8N1HDCA=7 zq{=h4akp49<;!CHOg3#ALgYkl3iJ$4-tU+8{ztI>RT2`5M0{u|lr?cgTsVF>FcoxB zf%_pe*1wKEL!n$BXB^9g0@+~rdK})JlAW;$DY;>Aof_G8>>J5D0>P=Rb<+OBY3y*! zlu}hav%Ep1etJ*8@HNMzgZ{|a4})i}%1D7;nipR`;H<5N3`(k@x|t>*Wslod=klz? z{hmsoelpr@oC4R@R%swkk;#LDLLNn)Yl(zhslprh$u3@pRWN^M1Q}!#FyL4S(T~uf zw_qnC9Awc{Zap3UU;*a7KB5@eH`U=6<)7CQ6l7!)#&v+8HF zFiS%M9gtzHtN$76F@|e1-+(ZRNU35=lqKK?WbgjUU9RcCNem!w?XLER1T$E)CLoAj z0vY>i6x^jAuM*#2!d4(BpspQe*nBIs2Tq+*04*vW0jm}WQZ?QVIWCu~aG*bxa8I}X z`2WRRq(Ni+K&&63|r8OxV!vb%HQC|ZmH zv+_NvSs+W*{7__NC}cbb^9Q67KRU4KjOT~U83M28;>NE=as1ukj1|%PSU)+!KNG;JIbLn4j>28J%_aC!(cvibEU?t>C$sl*-ji8Iz%IQPg7;lrBm z77`R&YiqF~@OLR0WnEwM;So#l+q<;rzeWgBvFwy&_=z}p5Z7<*IvBKP8K05sgw;Xj zHilfE!=Ibx;q*nm0bSk6M_`g`_mj{3*KO$uEXy(EZ(kbH##i)|aY+*SvIhB-9_l(Z zqK4DCHG))t%GO`zPH@@ten0{lzQM2Kvw4$Y|0o>3VDgCT!n@c}q0O8~b|E*7%3<8J z&LH;JNYGGTe6d#3@)=0d*mLt+%8_2qUu!3koHz#$&Y-jE%3v^7>ian(5-9!%Say1a zAHbR7Zen6G{OJ)>tPLdUIRMWF;D=dVgUMruLGB3n2oyJa`4R;(1}17CPnq4H>8`pm1sKQRJ(BuB6OHh=O6f(8{af0$=MAqRrk=Ht>5 z&7TDsTY`iatr#UF!K1ae_utKj z@n;H48wy2os7Oke<;b|0u~K6;c*8n!+>l+F`V-C67bv7Wh}X!eaJxvRhO72BGV#MJ}r9HV`j zwV8&XJ@gBoj(HSyY#WH_1j9TnKe2IexWVrC0z?##GZao8c?fL#S%G^qU|7M?o}^g! zm%>2=kCW#uu#P2gBL@oIi>o0#3JTao`T6_?bUHqI06%wz$kb6-bh#iVPwh1kE3gZW>5eh`9^ z4$5fZrqYu=#x=~Srb8t9bKj{3&10s*e>4qb4Ijv?#K?_;Bk%qVA@zU9WLnrN|Z?aC8dZIuvQbMOpJ zBzI#!GE_XAnEDIJ6V0E_R__0|7@ytxyqdq&XpzUfwiaTyFaIwYq%XDLy$aE0hUahox0RkX<-8;@ zX3^%O?;`$LXsZ>Ik7tjOA$?p`zS;e0f+TZWOxyV=40p*}2KN|Xfnx924TI%>`S?75 zjqR_S#y7CkhD?!$g@^%jROIkC_vdCHD5>Xe+ek;}2E3%BBj!|i5E+3+oscI7JyZf} zR_Dcl$a57b&m{*!P@Fw1g;LnvwW4&g&(BZLsP4*V5^h%!gv#5jS_sKTSwcxTD3kNE z1T$WJ=cGK;tp2J%Y7waWQn_Z~lyn?42iVxK4`;+t)Yc7k{PK%K*wv5Bc0|!j@n9u& zOC*$@`rO)Ot$b`9#x!#I$X6MFB)W=60K%dW>`u1b_<*NcCEW}aaLGi ztxJk_zKCf%05%aguE9YKiI!`Hl58rpMC&rN(X&DV7!5wQY{uCBVOU*VTN5fg(9eKH zs}XHY!Sq+K->AoKLOw~uz?@|y6%DR@b@rB@YNvI z182VKB}hN1Yn{n?Q~;q93%{q3UV|MXXsto3l_yWM3JcyTxPMBoCy4r~=GlQ7;%zUN z#;Shj!90dX@ykL#+q`j#r;3dqm5u%QaQsg5@3KxfRm8ZyGrRhI_NfI^=njdl8}C0{ zccs0R((Z&JDWr+qUq>i@)DF@98~c?Sl!$aIyY|gv7|F;sdJhQM4}D;h1`s}r8Kr7! zQu*H%diRO9FM8fQeg_d0g~2Hgwdy~^*(V#S=XCxB?(N*7PzD_~SKjsfq14SH7WuAA z&(hVR%G|Jd{fFn_0TP=*ZP@g_axR}Sq83nnu}tL$)cAh9fYi)wGQbY<$>M*jm9-~)?iX}K8rN~D{IBg2e^oJ5@A-OU`{G_In-8h3-5R_^ z+W{JVWr07)DV-$+6bj5X10Jy3EjI0s!4`wS8h0!%tHDYr0u;Ky1lH64W3~arj&|4l zD~@SaK==L6h(q8&|I+5`sn?(Go%@EJ%f}Ic-m6lo`x9^jX7!aBv%|#16#d?h0Qf(! zO*#cxecxX9o&y;M;F)Dn!0;19n_MLr6hNxpTwhxp3*L`hXQmJPYH|G*8dUjHvu@}KAz<$>J%dOk01tG;M${E>cUgC!1EuWmpe_4Rn zM+s}PrE?;nrxQFQbN@E+%m`Y6v7YBYoy29>E-?3)iq0u@;~Pj5{tL7H0TQ_%oLXK&IK_+J**pU-i~q4~7@EF0sPCzKBXDNc1085icx~+gya> zKb5_e2~lC4JREF`CTk-j-_x|RgKw+fp*7oHCN5kPR(H&L@+_y%OzGyzYvA-N(qObB zTvIu{eS;WmOS&)yK$zcvi39WGKp-};0IW*^v1Oh`_Xw|#=J z3LDQ*78AB$xiA8x4rRFdI?=wxd_j^NXtJvS0|@# z004qMClL4a{ng{`tI3`I674a}gpdJ^3r<~P`la`sfaTrieXeU{eCC1DEa=46hO4bR zS%XxrYiz=uK!Wx2>@1`C0qYrWM#CRN`#;MCXjMA6!~nCgE`d*c-(5LDnTbbKE-mB^ z*00OU){}=P_QeeiSu*a*pO1?VglzLdy*qfmNj?AP-(pi&5#aXOh^NQGlYXDqkoI)A zLYJrP-oHMRn4AkY5W-gB1tttk5<^g8ng=RM0;&_@uk? zn^^1xTdPAw+?&jp4uddK^r@4e9rN{YSr0{#EufGW-`^ zUHA*vrE+CwL(_4z{q;D5oPWO8pF@AI$>zw(levT1wm8lU@&KJf3d8)a3R;y+dF4nNsLWR zR{j-SslrbXAY=$&f3vCq2)jDA?6VJW=D5h+tOJqJmfZ|u2ws~ z>DbnuStzF-NK#4qeBjODPFTos>9Q(Ui+S~6cPHt)hDYn z%)lD%^$J5l{`vt^;!~|&wS13hqt0R#>w?+g@2njuX8bbU#+CE#m)xmpcE|xJJm~dj zt=XiKj?wy!zRr{yx3{+!AaOi`|LKIxN4A<8tbe3IAKugvd(bB{ipzhVAxtlm+Yw1Y z4w-t_>bGW?u6Rj42}#&aIng{PtBHHIV0X@!!d$;y#X#^@ri7wSQ{`^*Xu-=D!{FJ| z9}$RaVi`=^oK|sKxddSHAzY zzI@kb(wM}!{tSA`P_5~3<1bcSMU(h9khVuIT}syR#=mDysJ-NQ(i5?35LO%js%rFF zo+3(SIJyArwaZrJ+oHL7#FV*ijGt)q$nwjmk#JQ$A}@bAA;{tXHQ^-Sj+f(aV|We% z=>>F&)FKtga`-e0ikA01gmjD4>ZGR3D^n>|((w1i4bM$?($7Jq)OGUdGzQ8Xwa6K#@AEXD50iUtH$r1+lg?z8a6^usR(x*CX#@^+x|)W6 z-g=G>qowJ>ULvUCH@1gO=*eF*{neU`!ZmYn&*RR>3By^^c%_cUPc(aA&*Oc4Vkg-o zJrzl;dsdlkf^5*Qx7#HQ)D$G+EBbj`WjGLZVPiAqd~n~Jrp&2p!xZ5y7EvLK__?*}NBF~9ZT|=y z=tnpkdW$C}Bq4Q1@QF0GRaOVdq0#gl-v@g>__1gw>vYlZ1QV5D#OJ=5vO?5luASli zgJj=|+F!PmK;&PeCj5OPCuPr~uk{0#I%l*}c%~UebvY^6q z%7vvh$G^ti*va*Vz$3p)_%l&rRH;GyAD+@sP3fKQ&aFyteyMul(R}ptRz+1!I_yt0 zXmpn~HWA6cM~tH@MC6=5i5Zhw?8&spWXlqNsKW>_5OE%6#`6`eNL`l{*cq8e{2d zgUrfT6#X3@i5c;eNUj$yhai5)aATNvnMGu$?tNn=w@eI6 zEVS%(4TSR_K;X!VUlN%3f(%R33>%DmnI#YQ-#XF@Mc)YEAv#Mm_SB2z z^BxHLjakU35i@d(An~B@qC@#>Qn!g8LmcYAR+&P83e9RvW1b|beV%cBgZWk}*x%U^ zN8o9+WV%#K7=bPWnA{&694HA5!c3+BSOE!uN9YdELHa7ttHAA?^3fX1ia-hj2N;lf z$VTM-5C^b}jnHft7^jY~7_E8vEY1xkVe8>hszj>XC$MXN(I5J}df1*nJh?lL=`%{c zr8r7nNK0V*!uf##aSYjo9>or;gVZggDZile!#@Xh+-MLXUECz65q_^mi;JMH+()lX z++EAf%#r`f;gyCC9G$A4XI(Dc{o!g>YvhyD?RR~iKYoi&<_T;A_oNUpl?inRX|XQc}!&7M}dEB(jj| zXQDN#Sm&nz;wv3Pg1(?6Rk9nMqzaU0QY{}2$1H=(Ov)4WX`d0INt6Xxe>MvA-@}Mw z)1bpm9?#&t@c|VlbPDW~assu^AhP1c-?dp$O0`Y|NmYa4c-M^Jf;JteHvMXSTp3z- zBv$87TR+UBx;8s)jZ%rBRM5q<#OmUU$!57!t<8#7;^=^+9pe^4=h<=^M_Y}5;L&-C zMcQG_jG`&U`*ZAdftw*M`DFCr5ZW3IeSL-$?MzX*;(%qd4DC}WdRwXhkLo&sbr*x5 zhadM~rAn3jS1m1FlERSfj0%7yQ}3A>#*Tkd=grz5KF>7f7VL~Qt7?Qn67lqOV}mg{a&q_54_+0*8V6wxoM&0Bt1rL6PJXbCVYd1X%je7k^X30L-s zix5~X6jZik0PG|A`5P+bWP>0CQ9eQ7Z-d7$sB?HYUZMP@<@M02lV;*Wn|aj zVRju>x|cx#`^GdRm~pBVxSeihFQb;$9r^E$yi_wK(IIMA-cvfRC^?l2evuRXP`!5= zuFNBn5ymfLN__HXtSPca%W8hhC-{H;qRS9YP%?M8n5xW}hqueg_0_)Z7E|V!^%0ke z&r$Tu*@Z7xrQ0Y-G^0?jU8mY{Vb+!|5pY{LtV*;+k03sxL?uFCdHrckL7`O@zs=!{%y)wkMli*Ws9#DQ?OLQ`dl;C>c~@Rhqo`S` ze!78zmz0Pl+*j`u=9APM;3F|n5OuCd#E|!gj_+|e`-S{%zwyRqyI`(5f~n}z*@|l?%0wQ zG<-D5%4c0CCP&rxYEoZBQf~d(^}jC_z?=*txjR9cFYrVcHmIjfQomTP_tt%VpSqH* z_BS5g>Ju(mtGsUA^xK_PTnCb`1;*kmc&GFPTSMj}N)7s{u_>VR&&dfy!8`y7?7M9> zia@$0 zQopzR{xq6j`2DP}ra}(&w4B|opwb9GBW1}=>f0M1yY**$;iK&sxVUfMnmLNw8mNGr zwIdcYkY55A+iPdwTc&nGQLIlk?|kqe%+Dk{^9+f%h+jUMw`-{h=Bu69;uD_$o^{o4xO$EIsXfUC z-5zqRNeY!b&~P3fmAmjuI)6j1jVqh)Al?O#y$>y*RsbA5q2oL)nEP0$lTXKqPt7>a zV?i09YpnrtsSGbDB{Q-?ip;mGedd%ZpBpaxkrA^MYM{^jMIY9#iFEb^PxdfnFQ zaKq|X68xg(0!u3?VsiwJ=1Vp(i|1+ncinaWuWg@D89ZJI+aOPqc?@aV~;YQug3jOfTP0Wty01t z@`j|Txs{JaCVLrGVfOa?Lrngi%)Q6$lmYcPWm7;KT{va+#g7FnOz^)0sy=W=S8JaE zVxkvJ%VA4B50nUHVNh`fgS^Z@2mtWOf|#WR@H8Vz(_@3=>{(4*14>cp>5Fs~%prA} z)M#|k`3s--^_r$_pR&w3O_>O+LIij@oF~_h!uB-7lGrVCc@oh4g`FF~fZX$c_^1Kwx~>#?Au( zkAnf7#qP6>38aonxb7WQZi+`4c zeBdFLb7y3S)WNN#rguRz%uZM)*in-0t0%3^G*kp8{~X1`mJGE6#jsInQCmdiQ|J_a zz>w89|3=>g-qf0j0b%!|>zp0*0)vrj?`CnnT-bm)urj}S^9YukUa&wi?-dhR$82Y~ z;R^Ga;kPa~&!Wj<=i$Hu>r5m5zDDf?5)VOmrcn9*@+`l+^qjwW1WA~}-Q>MhQ(v@4 z0(szA#kqMrctz!t$$LqD^f9~ zO`+SesmQX9nZ81upSRG61@!Uf*iceg9Y1r_0X+Nop>=&iyD&MFGEl&7 z78dmrT_L0L)Pqj}t|e1DGn4$#A9)J&i9@d|`^Z3~6d*`l_clz*)qkft?2j2!Ur^;I9 zxg%Nga{{vV2ruvOZ@BGjtS$?!`2IfAYhqFV{@`p~=4ct6IPqOHlHFcA3s#~GxrmmR zZIISJUv3OFoS%=Z&#J*fR}GYp@oM6|3PlqSd_<*{m3SBW-L2=b$ela86Z!1>Rqtiz zCD?k(vS%J29*W+sS_z6Ajlqh(K=*)&hQ<|8IbdZlpfdw>R6BOW<}muDNV&B4^uO;F z>=~=0aBR$8P_Oc4RIY$W?>92ZD!y^=Kn#(nnBXP*_C9=kX4(Fo2PvfApl!v7<9%t@ zG@-=pCqc${#_dsBEgFsp`RerUkLa@yLjPDdURe=lH*g&$L3ImN%%7{!N)_1PM?RXm z9G=&D;ZkzirkAKQ@Df!jrWt;6X%)dIQ%Zw}pbEpD?4-#ypBrL{@cm4Z0EWq+SZxxBBY z3&&?cKO|g;A9cNVF3~ijK7bRvSyRl{NUkA#UF*FbdC095irR0aXN%TS99_Rqtt9&U z%MWty?^tYbVHgtEp5DP3jg0fmh@tmh$g*$icgdt z4Vk7@boao;9(@dgWE?{V>cFm>)XdFCB?TMYr*O3Kh!7$Y($+DJ0`_NdDrnY$U!;Ru zTJBA3d^zDI=`Ua5xvid;?^<+zaHlVQRge9p+nS$Rm-!t?3PaYtv)T)Y)j@XSwsrKm zW+msQyKzPM=*`7L)(i61h!B*SGCqVKqKmM*Xp~b@evW{plcqn!$Ghr)^Q1sTNQN3c zrTVvGvw>Z601x+{V4mu{^Mg4BaiYMuC(u7}s!85CR~)pEOv5Ce69wdDYfsKOZui|Y z#1a{Cr`B$X<$RO?GYSbg$vZD8I)$*=(I)8apmPP2O|tdk=xV&_dsxv9;7_XZFML3F z+&RdD6)KOM{4QE8!|pka(mHUNOV-P1)^ zZg2o4ER(k+zB7g^RFg=pq$s-lkkYJ|o=p+Vo_XN(<5%btNMa(*qtv#Cc8gK#(3*uH z1@4%Q3O@>hZbz(=f~m)Xd$AB9KZ-9un7b2qJ^GKXQP_7>7Q}adzB+%2n+MKVM9H-D z^!W8tDMmiNwx%Yxr5c?*cIz+PfjwYh0;^!TQlX$f*I-x+GBjWd z(Wn>y;2_YE2?`PZ!(CJQqaGV|t$`F5m)13GP|W-LxUk!p9gud_vkUlW-ekU|#n)O|uO&8|DSNdSUcJd)shM7B zCe>0vMl`p9Cw0I40zzSzwy+ z`YWE?jgO`_yRWL}D}IEuIZoY5*kitJ;ru6pGipe~>kL{&_t-78K_U(ZXas%nf(io2 z*d16mEiWytIk)%yJK_M6Gl<=+USTtQ&$tOvIwo5H%`02va0X&r{({T>cq;oR7=}K8 zn4xsVa!^?T!V2sFTFeths#{Yb-I;#G$M`}40x@hfu@c+n9-iR0XuN-37vnyiz` zJ5}mNq{I>Czu5HN>X&Qar=9Rpio-peIQP(0KAOvKP_RC~6zN1BXFw|w6KLCu7tITV zE4J|LGzLR&$BtUmL=ZN~f*jBu;!AJ`* zE}(Jh!-prH;zL}Qpw}0K62nL;fJ7VU>vw?j2E^2Y=;5Crg$rbDzKf3jy-@gY=q0?d zzU~A1D?to0xTeuPml#Xp4Xxh$AKb_>OVup{hbWw{-+orDW7YmOeXqp*p2diw${cX*vt!t7Kt>yO-49Uil&F$Rb z_kZpz1f@AY`dI1Um5+5YIpNODA#N=0TEpo_USv1KG@nd%Gbn&)v0b^ljr`Y2uC48> zBbmOB@@0YMoA*T12Ogn5f**m^>GWJro^G<-c-^`RTiI~-kDSYAw0788$)4#1^gQ4` zQS87#;|Ze%CQOjR!|(#_-xa8&0Ue{h!ug<14CKiGfU(Gd6ap#Fpj!wR7Z*f{e*^_G zF(sD&`RLdzx@a#w0lbn(z^${B)A^MtV8I}2klPG=6d<1!oUX7!a8NpS*?CM!E<#{> z(be*U!j?w`!~=bxyZigmRH}?0AK7pPQAoH)t#ajPET$t+yb#xps9_f8 zvu=<9!hv0lQhU=Wx=rM}sa5;$nQ8Xg8U2SC`?TSPQjn673LjR&-WYthPzwpuGL>(XPi$I*#Z-I>`?g9rEE#)c ziqkX@K9nR|W@l+>mo#)XGYR*MRpANTbP6}OsBf#+%`|C5k9&|5KEa|lX_L>T+0ZXx z-_}e?HN~AiO2J=wLK!>2xK>5Jw`Nso-0kDS_*W=@2JH5V)P927F3e8euHpZcYY5z& zL1cQ2RlS^{c^&X;1adnq*U zn=m`CL)v3zQu0xpB)vzzxzWDfa1~i+4insT(J~lAkbcY(8_}Siaic}}Nwa}GDDyfxb!S%a>ih$hFcgALj+*v2fi%gX4W zlarJFI0EvR5=O_S&~iYpfW--5{6I?Nr2yL&c^2sU61AB~3&zv$BLf|#l<%~FJ$j|e zL=c@vvzUho*)Oq8xc_B<{z<_^zY_xAOH0r^20k!+3G6j#tVt>A z7izVQ*-`p0e|ir8MwXV-CzOzbAPHuE!nv-JK(fq3@EHDGT7JfTBq*;-TrU;jy`gd> z$MTIqfgL)x`nNgcWh0s3hyTD}K4lnh;>w;ZkWTWKjDf_|G^$g^Y=qgDVDgiRB4f{w z2^Ohh&P^Taw^C7Q_$PEodkye6v`N)UBL}Y%uEc)BE3GVbzB-2DdS*z%mUQ$hHOjU1 zJkz^8!c{95P8BhaWV6k6nJ}iW>t9@WFR#72(f@8v^}*mPV&Og<4HM)4V(+cPs%*b* z(M3uqg3?{mB`qz|4bm;$AYCFLh=7DhNK1Ej35t@^rG%1Ff`lL-vFGylefv85+NZAb z_gVk=>bur@o^?NWj5+6+W8k9&w^h1e*-{ZmUPk&-=@{{UlB=tD&v#7!9Z9~wEyo__ zKfmj|F6jCUE9TCMh+)f+?5E^8{l2sYil_mKGYScz&mtLsn+;b;LA9M;hExx7+qgtL^8iAIdEcPYMIJ4?K_K=eB{?Lqb0vv zY4ONq4*#?$UX~U$E1YHv-PJ7O{84qGo%6F0O+$-S$AaHzgA!xnN@pX1>PG_S52l-X zdJ3<#lDHzw^zw~eQ?^vMUDMKH8n$(wGja9bA)QsVag27C37dH&m}p=e6sK;c-?ui9 zmB9Cl)Im%_?xB9Sp564ILUQHwwu&lc!nW=C`OlcN>mgNt14{2&`K!o98|8k_p>_*S zo-xc9^lxS)Ax&Fa-;Ux#pLuHN|EwWX+5P?cvF|7H2Y8knN4Hto;_=a4CrLh03P|Tv z=jNjJ8s~zRh1hLtz*ERYw8diks%ma+sVExz`LS;x zrIPO3DHaxsx9O)tP4_9eqk39EesisIytu6S^!csK%8%w6hZPPD->{{W5>q2*9)SIL zU&=#D1R`{RMMHzqrM5L+Mvs>64#w%NTg~Ir-FOp~RzmJg2xWE_c{YXHb(?*^j%{Zv=$KSGUe1p4-nZ~YHp%;aj*)%qCKqQO&arrFi)f?Bn z5s0^gI8!=dN;D^rlge1Kt47qVeo9gW1b|r!Z1T-zQY^pQy+m|sl*vyV%_?69C~41b z-QO*sJsERee|NnhbQzIU!T!Bw(Bdw!TZL@6|25IYq4v%>TIrS9H}(zghAMcK+6xyH zsrc-KiI6rgb8Qe@i!(73NnqAwy;DeEm!!G+^0%=2^g0?M_dC%zHWy;B_UGtK#40i} zHMh{AZ+IIJd?$|ap48t!A@2L%Qo5b?v2gPtCqS=BkNSG(Jqs(8p;eL#q7?A$Z(kEd zP1PK|DCbp7SUkMTfp{d8mX^C_sQ70yay1lH62a*UwFviKzETeK~GF0>wT*So>=IM1@j<--#tnq9Z zv&#z^T=S)rSj2l|)jYGZe=0sD67vj!*t<(8`$6@F#k1sfSl4AgB|-;LQYL8Ln1=29 zl5TXMvbc`ki^7*)iHImP2wgr-E4(X0PTBOBdo3E>yfr4>&lR!_0O*>YkcOT4T%MnT z1VK6In{PY~DR^6h;9<)v!dA-YtMN?}iy!;bvy`(C6swUz8fCZE;Be?w-lLsxAN5N# zM`MmeOm_~Up-Hu~kISKdQb=ckQu+Yebm`si4qlpJqaR2AevhJS7Vwbv5JKwHLR+b{ zJ4=`gvHr%7<2XLu9zz0cdW?6b+eW^2mKd%0HhNnoU%FF|DQy2k~z$Vm~4ecQ8wk55Qy8BYUm z*%h&GbgLm06%~a~QPWI8a zaLZFP&%k5GMR2S<^54CV?iT#G>HD#n=lx;1m`S~NkClJf8@y@dlx51+uB!~NB&Hpo$ptj0l!HYojOk@^*bSL|6d6zW6<`?-7=a}9# z)EzfplrH57@f7C%;lej2M!&swv$|_|PSmY($F`3kJc8pbC!QhytNZw)M(+{w6~SK) zi+#KWjMv?{FZP|+<=ee}q#b|EpuF1^k35n*e2`w*EPwA__W*yux9VW&kqW-o&2P=) z{apM8$3N*^R=wYQH`VAmBia~U(#fs$y%!BZyDpn4V(z~^w9_Ve&o_CmY3Av|{L)&L z0?|;Mm~{n*kl|LG^zxSVC0dX8HT4Dont9v1otEC3otG==-X1uVCIF_=6#2fI&y9I^ z%-4)Wy=jCs>Hz#L{4z4kCA=!qzt}taB!bzBql7QM)0Z_s#iWcjWU{|^9{c3i)%Uu` z2N@97Lqb9xqHYb-rk~6a6xO*D*D5d;ao!A)0*W(UXyr^Y-vuefp^F zvZtN%CqWnQqXScA=#vh_byw2wT$$K^vlL%!mbnxC%dT|RrS#tHZs+w^p)wBBv>hL6 zw11bR9SA9_{&nK7d9ht`H9JMk8RR>b``3znar=+*cMeuu%9JJM@63xsXJhJN`4v4o zIyQqL&s4uR^0^492JF7+RN)!@^L71j2h@7OgE$>BeqBUIUZ9|Jz4lyK`gJc@&$^m^ zJ+q;CdVV)$(4@g(N=j7acD};tP@&Kur(pV8<{zUUSFH+%Ujk7vTT*&@j^FLP5IH7w zOq)}qx@Ug3^JKZkhv4@O@;L92g(JplR{mA&=-Li2ShE>@6!RYiXHhT19j5kYN(sI=G4^ z{00~P5BOf6PDSV4ycC(k^FM)qD!_m07gz!mS61TfJR;a4Aa(QtxemBPu=sUy`bm3L zO7pP$%LA0Q{fHRBQhuG?pm&eMilwR^Vz~sNp)?zFWz68vg^dm|8P`j9%dy^Ia(X0> zMHeP*@%g%Cr>crc>e%p!y+=QB9a@BRE&=$H5bfT=Cx512^wm7)LL$IV(_o>3SVQ)~ zawVUOrghY7f$8^u*yc`N@}H*G&KWWVa6I~{)9rT=1F3mulN;HKmcM#2dylY}&pTfc@W4XkbD|=e~^zw@HU=l7Sud zTG#_6_sOrz_w_6&%BmmW=g=Z^!ZE`8!j?PSIP-p|`rvCYK~7)%PZJK8ZLVb5aap zeh~iLvxPk&e^2|1L%0SH^d94{hdscGF>2RtkaKyM?ui+0)$s<=WMGzA%tbSvTjEk? zYU@OQGFDzx5_v4cejW1`X`5!zgppzl-Gn8t;ry)&JONT25%vg1?Cf;RR2${{=4$~} zbxK8i0ayvEx4MMSY4Ipiyf%`;iIprTt$*K6nyq*)U6%e|7;cP32A$`a?b&;C8^RtF z2LeQJT_;~{eM@oQ{P3Fl`Ssfs9IV5%C|J}tpPz1sTXIj3k?ZrO?Tx7w;aV9D5G{IX zZ#E|C)0&w{>A%#$Q>s%|P8Be$i(_mlPd|>t^;#GMKH!l^9-0D53L<+#Xc-qJ4)IbK z3u6;>F#LdL*lmI#ZO8bEH->59-8XG34bnsLJ39FTwA4zt33J6$hO8Zm>~z5mB&^za zTq)NOfXJFZJGj1WIW0ZPAHzcgDd>{np#JUGd2Zq!sfAE!(A8OfsFSpgg$SD}=)9%rJ`pAYL$)=dQhaVnxmv+k6(2V{U zC0tq*k@OysFRv|5+Ol&>xuIY^j@7cLeM6BZrSoIaWi}RNha&{!mIiY<~C1quQ( zQy{#;1${{LeEeG`_YCuNCqB|8-YI$DuQ9M)EvnBZ8vo&i%v}tdhzVTnKN0;_2Bk01 zUkMT*q`r1&V835`ZEhQgSZqvl3bOCKjeE0$CxH&%|4gTkn&^mq3KB?B5-HtU3!n%QGnblY1Rh9i|F}ej*($q8`~G50n=Hu z!d<4W+LjkVh|q~EFEq<4Ha^jFW3Jwr)qOM?MAyXXqn60?$6v0UFy#Gqbb{j1+S*#4 z2UnP!D!_6`v;#87is1=~9|>oYc-ogYgB__Ny-^`3_v${XWt~Wimb3fQ`;iUXxYI(c zWdlj52wQ_^0;A5o&&{o0?FLWyAkGlB=t`OvYK=ymHm?=S_@_2riWuNyGi$Of8I}vU z6lft#+on(w0_{5^at5{8q9^T)x#@#T9BPe!OKe3~evUA$n*ue6^#>%OA zp>2o587}AiNR=N&%6?F%HYh#h4b!#ToWy^IoKT~}$m1fy=QhX=g%`R~G@q!8c99u8 zN714M>}|!u$QC}lY;IkwRL=Jsi_B6u_F^#=HT&nEsH6F0--Oou)6(CrGGWD zyyEj0an6#Fr zkKHTz+2v)cl=pE15>GmnL}yjuXN4bD zgDjybm0GC+{<8i$x%#DVPt5C*C<-W&5>PW^S*J;CNF(id)95e4vISjgm?~3V@r>RM z4J?tC6a5_?9@IIqjiF>e%Fc?Ys++Z{y?g21R`WUIb;ZWj%T2-}f6I3!Qnw_YFO^0e zenLQEWWXuUUjFs>5#7zGj6kKd3QDK6C-3x`c(lyj-QZC5b z=J^YGg~V_z)krOyoR8x_tt6H_61TTmyh?NJa3NEvlYe{6sy_H8WoK`>P;^7oK7Z|# z0eqUr?jo5Gk1&|rknVQhnH9i`V7L|e!30ZsX=Z9_%~Qm+Wey0HV9tV6`PF1$h0fNw zKcID@t)~~!7EkwB&y~YvV63do)?rU%PiSbv= zhN^s4&8L=_9MrPdSi7U^+BAchZF^>lP6B~LXIZ4O8uWB2cG;vP5}uHM!$~l#7#LWsSStku-lDN!lHOf?sxq?U5R~Xrp270F4auzj|G7YOH zR*hO1|E0>TW!IGYeK)ROFUXNA13oZ_SUYcN2j=3(a0=6iNZVasj`WDSaSu{)zX=j8 zI_SQX%rn1)x0XQSGnIGp?k}tu@|S;MGWI0Gzk**gK?NEeEEYS|uBZFp5-p7tejU$J zo<+gGLxPQs4Vh8v5o%lDdZqBe1Ow*|@@&J-z9F-s4Vdn|dGk?_rHZn0^vpI@T++?Z zLi@BrYhHt9TyS840?v*8H|{_2WmpurQp##-?!tHRi$PY|pus^03O{nv(xF-<*L{fS zrswAa0h@!1+S!rjJX2SIL@Ha-)^Xf304EqFxw*N}qt)0bI3G^Se;vL9+Xs8& zIfylZ_|F|CrdXNoB-5&OH(|Pd96SB$DFA@@IA~>4{-+1l2q+J^_T4nq(`y1Z8_*5< zLnq1GdgnhY^1~uHPX*8#PEAhwc9>FgTfI(p1qG;)EODgXKi|{Ka}BE7H>e;}3?q{_ zw0(rMkqnth;Wn!*DZ%c4Bzdy<43K(LV816b< zp!G{`oc*?U1x#RlBu*C5qH;}vVFnHuJo(nS=8Lt#SakdZo?c#%*>}?mdH!+x2%!1# zH+OjZB~Z5hR+CpEJ!GH!F@Xn}G6*5X;~IU+Endy5LbXhAoL5d*l@c{CL;5`PmqH0m z0CJjLq{qR4)nE5}4%!_7@fR#qcR<(yN|U!EWsspGxOPI&f;AKU{2EN(`+&^@p>rIM zT8wm88nO|e;Wy5R>R|{&wP%hKAIs(26{yqziDWYbI$f#7VBl^dHK{J34IQ6~N}TOe z?xSwIaE*A#ox2HohmE=c9Vf`voEa+@e$1bjhv2tSTvk?RSRs+E&Xy!`y88O+H2Vtt z%05HGFa{dj1KhnoFjNXvHf`VX4c3D64Kx-H{`CHvVix2FF$A>FvvYDfLjerj40MMAtU>!Wp-E+Jn)aY1Pd7VT*AkQa#0@#uuvI;4}yL zz|~h-_WgTU`#9ggJkl<$=Vf>gYM``RAh@iZz~xSX`IT9?3D?02{{W_~@PdNDFt85h zr*hse-VViz;Q+DoF&sL6La4e)`tH45aDnE1P;r6!61Y|3W1+*D4z+`&O1$vv$a~ZS z<6-b2gqT9watsut)WE6{I5;~wSF7GHOm+RM9fPvp1Anic?;yXx)f+mS& zWtusUAsu0ReSN)KN-#e817w91d^eC(KKP37frmp-Hnkba>IghNfHxs+HFn4>!kGk- z>%FR`2Dt8Wva;cA_+c8po}QjaXHEDVXo5<(ul9jWj3mq$S+E8+dv_kmoTdLR@DzOO zx=k3!osnB*7Wh+^O&J&%EODXJ@hgEa&0E`>0Z>Is z2iIWsha z5fo-qI1COBv#$cja%JJ?0snT1i&&Ky`M^y7J7&M>Lx-h`1AWQj1f{@o1`pK`x^-AU z>5@+`b7iA-p^Rq-ACD#m2X5Afg<)-`{ECn2rCEG|{s4-QiU36IELwf{m%-Q6@ZSrS z8)$DZeG#-wOg8XQ7~UJ`CEyhZz2&jUc|zYFzmrA5-4Z_aqy$Xu-|R&G2pk$7v11 zr1w!?c9gfs&+E0tat^>*s+196fQY zBhW@hBBCrYvJ-OO_dF_#3t+F_)7$%yFbP^y>|&vy3fuou1zAoc3UWgB4Gf&^xIUc= z7=(g&x6bvn)$^dIpnn83??ce&$`tWpi!T5JXVizO@^W&+*|0v&Phl4}3k*Dm%JA~a z3RsbMn+QnXCPr@N1YnCha&v=J3mB0iL+yfM%s(^g?!ZZU@TT)+f@D+t8 z^1R6@Y;^SB76JujMnyQln!ZrHaGC!0?OWjMEZg6}6hQ6Yd?in<*vthFzFmMJ%!LMq z)CItqDPybMr10dwS;3ffTz&cWO)Y^jt{3+Yc#X8WuMKQY6dC>7jnp!P z#}1D`2^0|w7as<)79HJv4&tk2l1?m{V%XP)kVkenLzo=896SG4fFuAyp+N+40-)4W ziPVdz*Z~p@u9H1*U<_Ou;|%I-WaZ>;p}$|*M`9G%6f-2BzVq4I;@j47MM`P(w?z2FsJH}AlO>%e;uL|2Nv8U9QyI`@q5trxxT*63{OnI8bK0r->&w( zT{sg-S|RWle#6-<#LrJD5qJV-&f>10e$#_^c%?BJmN@KyyA#9$0E@o$dbSv&#XR~o z@sWs%YKZ4^3(TG@@j;%?a2vfmDsaQl*n%7eBO?^^o|k0M6$wuli?ITt0&sqo|`&dL86GE8=62Z2=Q52UPfhQ1C)xN1Ee5uuK!@fCvC^k1-I6WbYX`$3jhg{Jq(g2 zWdNiQT|g!81XzJoz-8Ah$FgEgV0R z0De9};*Cr}6R!L*a*#k8?L7s5S}?5hQ^<4&LP8E;WHb2J8yn9btPSc_7=q_iGPfD@ zXZHL!)SW0&Cy55X@8&-}u+KgP{uva`paTuO$P5=XYXQOm{C-xZSJi`&6R`_(4bI?u zk4@B*x9nel5%G=R-^HPClu(vaA57pwF<`u@(yy8zg5r3oYqt#Wbq*d6%r;*DZZNq( zeAg0u-z=1r1^}#en;7oFnQ_{OG)L=W2$9SJa0`lBy+9B)H8Z<{29?y_m5cLr$&uk< zT^7Qn3J(>e*(9ooZwG_+$cth))-(uZH75s>*gZTvNQ@(HXkz8W!=veGg5l8TNCp7q z2B8hdpagtBSXTX_4)|FEv*LMMJ;7ZtABX?M44nkf08Za+$YAOGhY*s)1zTRW?{Z`> zb%4`>n7{Yt_??tN+3)#zd1wfbw~0l5Y&IK7B||^BEvtF*79njX150=XY03?d3QB>< zVFPq}4qye&ls_Orn@=9b50jKR3%-~+3pO4@VCn+m0->t3>};gJJCgs5yyIrv>#2h& z+}R>tQH7fl_t|*%Xb=EZsa|NngTPxYf8|Itd}+%RB>?DwDQj=&wGu4+XtqV9MZe zWc04Nvqs$lQPmutJ^Imxh=<`QT%LRq9iZz5MjTbe#aKRccBKD_phWw+)n9L9pV+lK zIyj7PdIBTS6zEi-S_SBZbT_Ie7X}GwA%b|}{sUX8fr>}W&60^69z_|Q@DK-S^#EphIkKaPG86Yxb z&k@oZ&}Bh{(nZV@F3ZBnd4Ys?(97Z2FuS_CN*7cOn-@eT;JYo3m>)J-h9_Q`o+0K3 zLxGC8j}L3HBmFI^S+Az@+?S9F-4%jH!x(u;ZJ@~ssdSeF)&*&4tsoEh-l!Ti2e}7;i)IMtv(J&Uc>rC%W33&YB9O-zTquO_ z2qHjCQ;3fMpdqp2h9QBpoeFZgR)R_(SxnCj3c>cjyKP^zywZa}hKMC;07#g3>%5@o=5cR1Pu`j47#vtTfRTt*djt{hx20KqyX&X$7_#6)kPQl_*K6TOKh(;l z*Ai5J!<8^SJAR`@k$}Lypdv?j9>~?lupGb(LJZ{I;78#(F=8$M5JwxqrVqm>$ANvS z97tuzMTyKYWE~y2#A&>HWMVik#XOhF{PJ~5YpAGaM4c8ViM37`AybcyH32J@`YB4I}Zh9^H%o(R5*y*q2lt zgE=zodlg4SNtJ|Nl`O^TaW8<2j%4v8m4n~}4iLnYE4Q__4zJ;XF^Pi;TMRZ*cNqdx z_E!qhL95f5>9ylpG9Q|SHD_C+dejiXFv?!`j|p;b#4LykY^~Yb8-Nw?K_cVnk6F;2 zb=%P%oLDtzF*ry}EFa$`Fe1>@>~`LAs94BIXBrt88cZsBu`J_vlnZH`$rMbvPy_F+ z!*Q8hFJgqAtcziqn2?YV`yEzQ=az2^UpC9N2Ony|x4~;B8n<9_auN>9hKQq3uzqYQn)_wg) zVvlM%n=yhaX{&vfzr)&gO)c8`ArAVC9LA_&3wSkc4P+(}B_)rSHxH}5c!Byt$)HH> zc4%26ftFyRi$dfJ#Kun*tzY%py1Kg@Nj{0CM==NlRkGJX`&s_D;ovYF0RuPeb0{6B z9x%O*QFu)!jio9$rw(SbIC1@TaV5A8QvK1^p4U3je)6bt3cA`mh%*eh=v z8qnU><>l2ijBTPKUL?5crx325QgAk8=GNy+(JCn^N7*?$wckA#=i+2%mt`C%`;vi4 zYuH4`UTv1agCk-nig??t)mkTTM?I7O{L!Os0xGHt03~r7tPuGpbk*qEsn9Cc0(UY@ zot_XoCl7hOk!Q~`xuSzS3U24-(i?8Kd8ivFi|A$lO=MBaP69_A5!bGW8?JOPtwpMe zT84**yOPs=(2&z$i0CL&Q3e;Xteb~{&>3cE?R#BzekKjV zsqt~%56qi*!cU&5+U`8BH;%S@C|hD$^*!E8{L@Ek9sX8{s`>cInunQLj8$$JKI4pj zhKU~QY9t6L`JG{Zg~6mh6p0z>YX{?4UiQe(Ww8HDH7%{nPIyq+(&7jBgW(Br7=GGy z9>biwOGx>QQ2`HMiBg%Np0ItGSkv@ejt`wTu&)0tmK=6`WCCT3jb&^toh_NAY|AG6 zF79;+`?(m(JlQPVUB`fEJd{dZvKkKj$1U>^>xcbkEkIqIpGgIxv+d~$IA}wqbD$#t zdrUL@pq<0^;(2-b@NjG_9)1a@=oEc$Y|p^BmNNUCVcB=%8W$7O=|WYa77Cfi1&XW( zVz2Nyl85|H%RUtQd!>n~kz!O-#2aa{#UT)O6`U2-!kI@g9CHSxrZX&r{WB$NAMQIS z^ec^DJuNkBpQ{U3IG-B!e=;H0U9e|4}DOYjt}(jDYGW zT}WRhea(+3sbX7jHFB9A0Ct%43v&}93N|gJLutNR@oKR{v#YZiD3(NhhYRlSk zIaW+sd3ia+CeS&;F&`eL5h0t|_R!RT?Iyp!mxywm)}+R)LqF9k1H$L)@T5QqDqe$y zKUHO^5@P`(e6Avo59M#}86Y)l|2J&&-`QdE2aV9o1L)Ms$_nfgu3bpT3kgoSD)i5N z{jQQlV4r@)?l7U2WBUDe9=(iWZt;uh8n?$BgOJcV2BZvGbzC1#xZ%gF1yDBY;ds2t-5Nl8`i64-#jOknHC6Tmy$= zfL{QpVB&m-#v#a>sY{$%HAh393mcM`2))yn7o zeuyE#b27+MFrC}D8Gm={!!MX_G|=x1{fv?jEkWOP3Z%DBkXBM>TkHi|_ngTkszaf; zg6`UEcm7HV;xX;289f~xcmO+4yXZur6&HU3C?1MFEf8~mFj*|lwfBbzh33icSD!3U z5GC4Y5E|WMX1-=y20yd9E8JzYdHwM8cz40E1`MsD=XPSFqd~VOhzXgaJC(_---Or? zS&D#o11wAOz)pclM8uFQ1)=~!=wg5p6*BF94d6cYlC;nY0hi}9CJC+-r2i5orX7t$ zpfG;j!NI|K&=oShSFZmUW=sV#_>3ZgIfd<0bH5b=nuiY`K6vnK!Nt|i&d$lH9a7a% zMC{0T2f`vqoVssm|D2b{%DD>(rZf#?IDj-~`9xv{?@WMhM12kz)r!(moOX76EPv*T z6Yxsb6@3A~l^@`MkPr>zRFLrmQVJfDO>f@3foPeQf#E^fRHb3V7E%k*Clv~PXOQA% zDugF42|pF-h=j<1^#+bukV$ujgm>VHn}IDjXi< zP4AFPG-x*zBtc{yGTz*f0EE7|u4oBMGN=WdLD&YVTgbYi4{sd#qq%AlHb71q zqC$lNtyc;%nv+ljlG_%AggA>8te^fDgKLHP$k*?$K}#RP;TQRdQTuR1F~i27(b4>w z`UMZnwJdxl;%LPL$gA&ziJC%!m5Rz|2-1Nr{IV@%j#CI?1WU+JLjD{ZQsq!p zWe+TcKmKzq<#7?UX0xo#UcWx;{4ckuJq}go=k{=9@ttcJ;zdpN1D~)=a zxO|`b!*vDc3lP=y4dpD7;s&NxH+@)F$o|a@7AN!s$|UKJ*gj+^=y%wmk6?+*vK^|S za7*rcBHKSznxmy|BeR_N(yv47jEyYP(Z`|+wqNRayH?XMOB)QXhT$&I!U3_@jtA66 zkJk}p?D>kDw^Orxq6VFv+`U7T3KHX898+^Xe__Ogb+m87dCa}#0KQ3_C^YTH%D`T(=ui4x4a7uSR2&H3icb#D zxQ%RC#*#LY!u&d>#q@8JvTE8yw=BB6ZE>ISFL)uE$N+s1Hp@pg3|acGxQm|i;0)Tc zOo}8lV4Boa%4BC_NhEjQL7?;M*w9zkKU-qwPwT>BOJ<)URq_1(o16Tts4Rdtssa>!}-7wH!J!kx7OHc3lkfOg_&Rx>X6UK_|!^ zm=(eSCMDWT1lWuYsozbzf z*6PpSQ$lfW*ylTHzw};#_J9N5b}4_->F+LV)05zq6<2U#Y#t%OABh-orhX(viF)oP zo_A^>=@uCj3wNljM-d}P-fSG(yM$Ir1I@67bcXLc7uYnAH6yqR1FyJI>4dh?hEqwy zeHJ~u1UKP@4tL6$S_S5|Ocu|-^3ADI^r}{>0$$%HwW&u1(<8X2Ui_UKbDa9Itsev8 z3+UA(22P$MMjuI2FCJMyCJ4DCmlgRYS{$Okei)4;!$A8;6*qAsuq61}Sm3QL)qqf% zW%1t()!o}n`H9;}(=nJ8J<`G1>~j`<$-e^xxY{OqM+lBKPyVa(8S%b)I0`e&5?LZv z#iR^HlZY#t$wv>9p<_;eDH5SS=E{CExM|pUBJ0g-2Fyi8BHrB@re~?|vI1JKlqn7; zfQ9RMCBI?a#c+JGrOu=O51|7^8ZVhqj)G^k_CLXnWKzJSC&Yh@339N23f8 z*Kb&`a<+m(b6)Jo%e7Vn}*CK(*R#jY|D_xi`qQEU?b=uZ!brGa1u*DyQ8y z4Jxfp1_M$0XI)0F?=*FRY45zXA-wR@zRl8f;K?C_{BAD+C>Lx)BY5Z8RMqpp?;+ax zoNLR>FR3FB)MODDJ*S&$vTed3kJA2zc&O0HwuGN1=L@m=ncUS&F;0L z|6;|UNoYm>XR_#;mJzL{)kyZdxa+8GJsc#mg>`@Uxk}r6W`PCL&QEzCQQhK&dBmNd zjt0B1)%=u>>NsU;XXQs}HZQc}`WUKwXhH@tkQG{~hX8DuEk^Yh@y&^-HxH}*OA#3dvbbsrM>j*0C|K3rFsa8S_nH1!V z<_)EMpsM!N6`wx5(Kv@{$KT~PE9_Mj~n$!bYu{fVc%|cVsUm z|EzC2L8{+BZjGup^ps&>Se{|m0hb7hYmmqu1U`otGt@T82$@d^v>}wNG&w3f& z!Wpl7AiD&|ZIACOyTBGGyHA!d@N(dQI>P_FkxR1xABx$&q+RUu&6F{e*S984uA3^U zOkiN&ns`PVBjf+7MUh>UB$b1BkhVJ^G#H(v-VCZ^EhM58L*;_6=bnP1X21OPW^pU# z3`;-1qlnIDo7PMU{)_ybv*C`PiwJF*hZ^jdTv*0alq~O9U^CJ--v0i2f%bSanHST! znCtdURjZGUg+f{Dsx%QuWHZWzMI=oZv7RM6_gUZi{bTi9^%)jxk2FamuE?dK<9W4? zH$yLCEWEhXd|Fx?W(!xwOol~tXO-CnhqBQ2)Ynkm>f`pltoP{>`HfjoX*;+)bSF-d+K9mnIU@h;F5& zaSY;CD1dbfPUczQeM27k+t}j{ZyF!i3F~mI<>u=Qk_@n`@erU7)~2rD-f^$v+edin zvL|S}gej@A;(adGSuJ%iE8XI2&rwKd>@i+1YiVHb5aFk8qs`c|Ij*^^UolPo zalK4&BTc~mUHOC7M7mZjzVLI4N9_ex1+8&f%bpxB*4OzoC4cM5D@I}ScM-T5;yFz@ zvTk~YTa_J5K3o_bJHQyqN%Rm+2IWwj9#+q-q*4nJalO>kN{+Y5k)qm;lv6=-~04~<=SV9K*QOn^USFPMhd(e8mN`l zzy4gB{)hA=f9b~idUQ(n$}Gf}tDwY&I{_@6J~kwl>nD&r)V#}|q+EQ1lO@^q1HwhQ z-{!!tl*=k?U)$pTH{&zuyoVM>50fsxf6|sIX?k5}y+6ue@wt}NjUcE?3Y&ZHUDi~s zomR&6fuop;A|>W6CCkHK7RF=s`tLI4`~=91*nSXdO%6PklqyTS~+cv|`#N}aHL*ZBm(of1g)III~E*c!xjV`RL z{rko&_#=dGD^g>H&D73NfSP`^!NB9pRJ2!+MtFoaQQL37nIR}7LC2)sFfZtT+d|^n zKy0`4vn}Y?d6QW;q0~Tws8D*hJGz=9Det2*khD{;o0*HqG!!w|{ZqE*`CdFlofx-Y z>Pk#eP3^PndjaJZV{s1lwPG5CVL^WnL1n`S?E5f{|MqGsA%d}LzT7j)&ydf#V*)g`j&Tr~D!gAw2xPD23WVAgvM~`&iyz`W5Y?isNopVe-h{?$G#* zL0J@94x3zbM6jC}Mh|7M%L9`~OZlYa8ilmFdb*g`4N!u4k|O`OD*EA3Hb!7;>&{#^ zTh)5{HWHJeeZtt@m3k%2SBwK|UpnX{eroY7WHV}*zT~5Bk~A;Hx=)pj-p=!Wx$dPP zF1)bN_Br0^#yAOGp3FYRzbLUz+l<{t z`(|U`XqMU8#H|FX&F#K7t$pbX>Sa2Ir;y&C*~iXP?+mYADPMSd_P*BXZ3^nmwgjAr zv`-7m42YYe#BZ;K;rTj-BBW*}O9i^=GHy;Aii4bwNb(wDB!J+cJ^0_&ryPuPAE(fn zFPJ$Y+R?~+gajw^aFOf1e&WP7JC){jshQ$*d{xPY=CY>`^Ac0-qZ>*|5q3IhUe1SC zk5Xg~`O6Z9w(iQNViTA*G;;mg7oi~;QVQR=bZNBKzZiLeAGfbm_szYVZaGa^RbJsE zrcGn4;OnH*Q^QB|H(kW5$Syb*Zd%K?dS4I-)4jl#-Y{?zC4T0SbIGkhBPFRTCc0Fj zjjz&ixyqC3xD^vK=DRgMtufg3szUsr_we*h-jM3L?V(4saU-v!fUg&UW4aSRY0xq) z5$fWXzZCu3X{yoa49|Yx`|z;sEq*x;>Aik!=4NOYn$ckpOF~%PDII-mcLuxpCo=uVg;5DPK#$6R{^G{TJ?bICuF z6H*MN*^<&!RMgaZP*NA)2YmI{X;Om1QDJ2Ffm+#=SBkOdhcUOQW`- zNVnnjnC5b0Iwwh&b+SNU=9Fa|{);;wJ{I>K4ck2Vdp!}hICAw*_Z}qIPp^o3eHp3A zBlgYl&!@^v zGr80B>XK0W%75cGr_RO1naQB>ScAHQP|1lTPA3r-qRU|EbLXbsYFv^!g|l2xb$a0^ zwGQ$Q{Kd_iLylAITO>>r(WYXey!p?O6-p?uySuyh%HLD%j{KhC<6>N5I^{aAZ(w@) zdv+N|l$|G#ExXw>ZTeE7!E%AlUa#qtl6@wt&6;8uhu@zbedhxY#mk2?B08MkB69e_=F}P4EO3TvBcEn z@0GX#A|10zp^8^#G1&{nW(K3_+kX&P;uPc*(+i01t(%?4+fSQ?8-pISoNJy&m2wM_ zHw9sJkck`~_Db~Dh$`y=nosF$JtawB=m=Op-idjo_rOy0WgEeGTSGaA@slrAxHE?5 zuFLtSGVd)vY-A>Ft$i;Uyb;~r^VKY7!z(?)b9mZwJx9_~E z7!-Zy7~#E=mXskKtySCUHe{`gQ8P-=5Z?W3;O-=nzr#T+2^xwQroIQO{m=@b6!9)5 zc<>8>G?riQk&W#U!)hh5Y(FV~-p5bhqXl(O&OdvjdYoPNeKMl6^<-J^;Cnaazx3YkRx*Mq90jWSC>>iE_=lMXGO5m>*V1=~mgPiM7qmNKotjiz58qI$+v?6a3rssBuY|Z*VL)?po00z%W=DvL4 zxxP|d_ND*q%pbU5KV)-B27x917s?E_W4y>D*jdS6VA5Pc_W+d_QQM#uue=GWf62+k z$m*IJa<}{a@DJEq`_@wY_1K#>a)mT=Oq^ur&YRJ}(oTxMb#`Pk|A|5T{wo0+Z~HH9 zgNO0|^bc8dU)7{kQv?l7B89T7OvMCY+%_!tPae^>iT6n>RsWBssZ!omj@slPv|uqb)lr_VBNB zn^E@Q6skdq7*&X6X8D_(&+pP_U0uZZpW4wg$jN6Pn$UdUd;k-N>KYQ~?~g7u_7;wn zq}qx7EVLoVBZ&c%BYw&p7n8(Q<2Sxqo0=Y~9ZTp;Xb89RS6=W~i^_5os1(iAiauB0PMkNgngPQ)i*y=ljkELNC z#RP3px@|tqjdHeHj_r>6QX9FL0ToR_TJoEo6|qB<&sj(Ol-J zdth(#RS->rZTNH4qbqMqhdKEc>h1lKujev|@;fbBjz`Ay>gr{TX|F!=cwYIuJyaC6hamSu>?Q2`fO>gPi{eY#-8UhQOx+{grqGArO1gI+N|smnWk;F66d#s zls0^797HEc%o6vPzx&j0Hach-_y}hS>kb-jhugB>BAYy|vAJ9Hg>o&};HA2<$88h@ zVxv5n@vEC%@GUpafX+XE{u7*vS}2D*MG|V?Q$sN>OHi}EQ%aP3+rn5g>GEF4V@;>& z-}Qmf-yoT?-&`0aWzr9bfY)tJs)Xvs%y+}sE#yzGW23Y zRNl*0TAvyXWl>pm?`Y(Tu^IHZaMf~3a_5uF)fx)Jl4{Dz9$0(Qu%%7QR`?;Qb*#SG z)B8Ca=5>3wh|7)pjO@0#=*D?(`UavSi%En38^34k{WpHU|G(mQ%nvB|OPj;LS^{hJ z6)?}%72Vw}xNs^B2G+J78L-MJ2z5aG7FzB^*W!4 z^G}h74+M+K&&SG1y>3*zn@D#bLBy!{_#$isr;o+TCLQ=q=x+V2BF2#o4QrU4v-Xe{ zqb3gXrITE`hF{3Lf9aRF-t`C%;Yk5;C-bM#2{geFD+4owzOab-FY;$8qxrl@J+Y9+4iu9)Z~xSJ- zeAp>x(^)``J#==8;2Eq9F~t#6;n5cUdCrMIoM3fz4`16r$w4!%W<-oa-+L%jC;Oi7*) zZB$^3NoJxZWM26r!ZBY?<^7lW{V)*|y^|NX@Py>bY2) zKAinAY_LoBt@ekW&^%Ng4U%UAsf!XJ}j_B)&PI=iEdqEyKxqju* zPwyU{3i=e}4V;#p+zC7$^2Unf&5fcH5nY0D=vIYqo4VRmOfz$XAsm`g+vg<~8fgs(^PoEtK1OoNar- z9QilfZZa}#Pu{v+-e_J|ahp!&+T*)FZ&@HwvtBPg zHt$YEi2c^{&+Y!a&%bn;;YrSeE9Tu-Ps^`9efIYEx)*m1^VjAUEHppYpL-`@-TghE z-*3;?dwom&)~pZy^8ZfT|7*Xz;jrq#18Dt-Y4*TNNIzEm0nSD?^GACxt)KcNyN4^e z=0!~cun(8}vfKCdxwrP_v(wrc|5tVJ#c~@n`33Lav>@*OmKAlI?)lVU=3egB%<_1499 zVp5r}ukZWw?~nbzZ-;IKm;ZhEZfCts_*{A_@LxLQqT6+z$K~w z_pORuf8XH$dI8RUOE=%JuzeeE?%iF!U%v8dO+rgt59{04(^pq5URWQL-S=nP2mAlo z9Sp|V;a8?cOYi;9uH3kw;Mtc6Z0&vv1^>NSIs3ZabnW?diT8xheUk!)W5)-#A6w)9 zA3nPAtMsX-{ini1YIk1T@zwdo*$Xr0Uv{@Yh@&%cIG@eIp#pfbyPI3v-wB60L0f9~_X3ybSmgn)Kfj;# z)=bUl1aP7M56~5}z_n6wVaX!Gk=kp+zwcO;{{9|tX*zK6=EskP{kQL4iJdz?vh%F? zvERyZ>q1X=3Z1$3v@`j`y_??WOtmdr#g?Z0Pd^6jPq424_PF8_*8ar5g2zT>z>IqS z*v3Cq)|;wn z|2_?)QxOB{R8%;A{sEqbexa^1CpzK`uVq?j%Joe*uc-;HTN)Zx)0y&-xizl$<6bONsW5D5l`%K_w zzdyjuX;~7-0>w9}e}8}fKX9SEPu!!q%PxHW1X@pGQTQlwOAn|zsk0n$~^WJfGLP3*2)v5j(AF?%Yd83_#%N>gTe~DWM4f0AGpO literal 0 HcmV?d00001 diff --git a/paddle/contrib/dynamic/function.h b/paddle/contrib/dynamic/function.h new file mode 100644 index 0000000000..6434beebf4 --- /dev/null +++ b/paddle/contrib/dynamic/function.h @@ -0,0 +1,130 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "paddle/contrib/dynamic/tape.h" +#include "paddle/contrib/dynamic/variable.h" +#include "paddle/fluid/framework/type_defs.h" + +namespace paddle { +namespace dynamic { + +class Function {}; + +class Fill { + public: + Fill(const std::string &initializer, const framework::AttributeMap &attrs) + : initializer_(initializer), attrs_(attrs) {} + + void operator()(VariableHandle var) { + get_global_tape().AddOp(initializer_, {}, {{"Out", {var}}}, attrs_); + } + + private: + const std::string initializer_; + const framework::AttributeMap attrs_; +}; + +class Mean { + public: + VariableHandle operator()(VariableHandle var) { + VariableHandle out(new Variable("mean")); + get_global_tape().AddOp("mean", {{"X", {var}}}, {{"Out", {out}}}, {}); + return out; + } +}; + +class Linear { + public: + Linear(int in_dim, int out_dim, const std::string &act) + : w_(new Variable("LinearWeight")), + b_(new Variable("LinearBias")), + act_(act) { + Tape init_tape; + + std::string initializer = "fill_constant"; + framework::AttributeMap attrs; + attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{in_dim, out_dim}; + attrs["value"] = 1.0f; + init_tape.AddOp(initializer, {}, {{"Out", {w_}}}, attrs); + + attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{out_dim}; + attrs["value"] = 1.0f; + init_tape.AddOp(initializer, {}, {{"Out", {b_}}}, attrs); + + init_tape.Forward(); + } + + VariableHandle operator()(VariableHandle input) { + VariableHandle pre_bias(new Variable("linear")); + get_global_tape().AddOp("mul", + {{"X", {input}}, {"Y", {w_}}}, + {{"Out", {pre_bias}}}, + {{"x_num_col_dims", 1}, {"y_num_col_dims", 1}}); + VariableHandle pre_act(new Variable("linear")); + get_global_tape().AddOp("elementwise_add", + {{"X", {pre_bias}}, {"Y", {b_}}}, + {{"Out", {pre_act}}}, + {{"axis", 1}}); + VariableHandle post_act(new Variable("linear")); + get_global_tape().AddOp( + act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); + return post_act; + } + + std::vector Params() { return {w_, b_}; } + + private: + VariableHandle w_; + VariableHandle b_; + std::string act_; +}; + +class SGD { + public: + SGD(float learning_rate) : learning_rate_(new Variable("sgd")) { + Tape init_tape; + + std::string initializer = "fill_constant"; + framework::AttributeMap attrs; + attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{1}; + attrs["value"] = learning_rate; + init_tape.AddOp(initializer, {}, {{"Out", {learning_rate_}}}, attrs); + + init_tape.Forward(); + } + + void operator()(VariableHandle input) { + Tape temp_tape; + temp_tape.AddOp("sgd", + {{"Param", {input}}, + {"LearningRate", {learning_rate_}}, + {"Grad", {input->Grad()}}}, + {{"ParamOut", {input}}}, + {}); + temp_tape.Forward(); + input->ResetGrad(); + } + + private: + VariableHandle learning_rate_; +}; +} +} diff --git a/paddle/contrib/dynamic/tape.cc b/paddle/contrib/dynamic/tape.cc new file mode 100644 index 0000000000..fd24eabe9d --- /dev/null +++ b/paddle/contrib/dynamic/tape.cc @@ -0,0 +1,265 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/contrib/dynamic/tape.h" + +#include +#include +#include +#include +#include + +#include "paddle/fluid/framework/data_type.h" +#include "paddle/fluid/framework/dim.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/framework/scope.h" +#include "paddle/fluid/platform/place.h" +#include "paddle/fluid/pybind/pybind.h" + +namespace paddle { +namespace dynamic { + +// borrowed from +// https://stackoverflow.com/questions/874134/find-if-string-ends-with-another-string-in-c +inline bool ends_with(std::string const &value, std::string const &ending) { + if (ending.size() > value.size()) return false; + return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); +} + +std::ostream &operator<<(std::ostream &os, const framework::VarDesc &var_desc) { + os << var_desc.Name(); + os << "[" << var_desc.GetType() << "]"; + os << "[" << var_desc.GetDataType() << "]"; + os << "{"; + for (auto &i : var_desc.GetShape()) { + os << i << ","; + } + os << "}"; + return os; +} + +std::string to_string(const std::string &type, + const VariableHandleMap &in_vars, + const VariableHandleMap &out_vars, + const framework::AttributeMap &attrs) { + std::stringstream ss; + ss << type << " "; + for (auto ¶m_name : in_vars) { + for (auto &var : param_name.second) { + ss << param_name.first << ":(" << var->Desc() << ") "; + } + } + for (auto ¶m_name : out_vars) { + for (auto &var : param_name.second) { + ss << param_name.first << ":(" << var->Desc() << ") "; + } + } + return ss.str(); +} + +framework::OpDesc CreateOpDesc(const std::string &type, + const VariableHandleMap &in_vars, + const VariableHandleMap &out_vars, + const framework::AttributeMap &attrs) { + framework::VariableNameMap inputs; + for (auto ¶m_name : in_vars) { + for (auto &var : param_name.second) { + inputs[param_name.first].emplace_back(var->Name()); + } + } + framework::VariableNameMap outputs; + for (auto ¶m_name : out_vars) { + for (auto &var : param_name.second) { + outputs[param_name.first].emplace_back(var->Name()); + } + } + return framework::OpDesc(type, inputs, outputs, attrs); +} + +void InferShapeAndVarType(const std::string &type, + const VariableHandleMap &in_vars, + VariableHandleMap *out_vars, + const framework::AttributeMap &attrs) { + framework::OpDesc op_desc = CreateOpDesc(type, in_vars, *out_vars, attrs); + + // Create a temporary block for compile-time + framework::ProgramDesc program_desc; + framework::BlockDesc *block_desc = program_desc.MutableBlock(0); + PADDLE_ENFORCE(block_desc); + + for (auto ¶m_name : in_vars) { + for (auto &var : param_name.second) { + *block_desc->Var(var->Name())->Proto() = *var->MutableDesc()->Proto(); + } + } + for (auto ¶m_name : *out_vars) { + for (auto &var : param_name.second) { + *block_desc->Var(var->Name())->Proto() = *var->MutableDesc()->Proto(); + } + } + + LOG(INFO) << "- " << to_string(type, in_vars, *out_vars, attrs); + op_desc.InferShape(*block_desc); + op_desc.InferVarType(block_desc); + for (auto ¶m_name : *out_vars) { + for (auto &var : param_name.second) { + *var->MutableDesc()->Proto() = *block_desc->Var(var->Name())->Proto(); + } + } + LOG(INFO) << "+ " << to_string(type, in_vars, *out_vars, attrs); +} + +void Tape::AddOp(const std::string &type, + const VariableHandleMap &in_vars, + VariableHandleMap out_vars, + const framework::AttributeMap &attrs) { + InferShapeAndVarType(type, in_vars, &out_vars, attrs); + tape_.emplace_back(type, in_vars, out_vars, attrs); +} + +// Temporary Scope for Operator::Run() +class ScopeWrapper : public framework::Scope { + public: + ScopeWrapper(const VariableHandleMap &in_vars, + const VariableHandleMap &out_vars) { + for (auto &v : in_vars) { + for (auto &vv : v.second) { + if (!vars_.count(vv->Name())) { + vars_[vv->Name()].reset(vv->Var()); + } + } + } + for (auto &v : out_vars) { + for (auto &vv : v.second) { + if (!vars_.count(vv->Name())) { + vars_[vv->Name()].reset(vv->Var()); + } + } + } + } + + ~ScopeWrapper() { + for (auto &pair : vars_) { + pair.second.release(); + } + } +}; + +void Tape::Forward() { + LOG(INFO) << "Starting forward -------------------------"; + PADDLE_ENFORCE(!has_been_backwarded_); + while (current_position_ < tape_.size()) { + OpHandle &op = tape_[current_position_]; + + // Create Output Tensor, this is only necessary for OpWithKernel + for (auto ¶m2var : op.outputs_) { + for (auto &var : param2var.second) { + var->InitializeVariable(); + } + } + + framework::OpDesc op_desc = + CreateOpDesc(op.type_, op.inputs_, op.outputs_, op.attrs_); + ScopeWrapper scope(op.inputs_, op.outputs_); + framework::OpRegistry::CreateOp(op_desc)->Run(scope, platform::CPUPlace()); + current_position_++; + } + + LOG(INFO) << "Finishing forward -------------------------"; +} + +void Tape::Backward(VariableHandle target) { + PADDLE_ENFORCE(!has_been_backwarded_); + + Forward(); + + // TODO(tonyyang-svail): check output of last op is target + backward_tape_.reset(new Tape()); + + framework::AttributeMap attrs; + + // FIXME(tonyyang-svail): Need to infer_data_type + attrs["dtype"] = framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{1}; + attrs["value"] = 1.0f; + backward_tape_->AddOp( + "fill_constant", {}, {{"Out", {target->Grad()}}}, attrs); + + for (auto it = tape_.rbegin(); it != tape_.rend(); ++it) { + framework::OpDesc op_desc = + CreateOpDesc(it->type_, it->inputs_, it->outputs_, it->attrs_); + std::unordered_map grad_to_var; + std::vector> grad_op_descs = + framework::OpInfoMap::Instance() + .Get(op_desc.Type()) + .GradOpMaker()(op_desc, {}, &grad_to_var, {}); + + for (auto &op_desc : grad_op_descs) { + std::unordered_map name2var; + for (auto ¶m2vars : it->inputs_) { + for (auto &a : param2vars.second) { + name2var[a->Name()] = a; + } + } + for (auto ¶m2vars : it->outputs_) { + for (auto &a : param2vars.second) { + name2var[a->Name()] = a; + } + } + + VariableHandleMap in_vars; + VariableHandleMap out_vars; + std::map + loop_over{{&op_desc->Inputs(), &in_vars}, + {&op_desc->Outputs(), &out_vars}}; + for (auto &each : loop_over) { + auto &vmp = *each.first; + auto &vhm = *each.second; + for (auto &p2a : vmp) { + for (auto &argu : p2a.second) { + if (name2var.count(argu)) { + vhm[p2a.first].push_back(name2var[argu]); + } else { + PADDLE_ENFORCE(ends_with(argu, framework::kGradVarSuffix), + argu.c_str()); + std::string name = argu.substr( + 0, argu.size() - std::strlen(framework::kGradVarSuffix)); + PADDLE_ENFORCE(name2var.count(name), name.c_str()); + vhm[p2a.first].push_back(name2var[name]->Grad()); + } + } + } + } + + backward_tape_->AddOp( + op_desc->Type(), in_vars, out_vars, op_desc->GetAttrMap()); + } + + // TODO(tonyyang-svail): how to fill empty grad? + // TODO(tonyyang-svail): Sum var grad is necessary + } + + backward_tape_->Forward(); + has_been_backwarded_ = true; +} + +Tape &get_global_tape() { + static Tape T; + return T; +} + +void reset_global_tape() { get_global_tape() = Tape(); } +} +} diff --git a/paddle/contrib/dynamic/tape.h b/paddle/contrib/dynamic/tape.h new file mode 100644 index 0000000000..9467e9d543 --- /dev/null +++ b/paddle/contrib/dynamic/tape.h @@ -0,0 +1,62 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include + +#include "paddle/contrib/dynamic/variable.h" + +namespace paddle { +namespace dynamic { + +using VariableHandleMap = std::map>; + +struct OpHandle { + OpHandle(const std::string &type, + const VariableHandleMap &in_vars, + const VariableHandleMap &out_vars, + const framework::AttributeMap &attrs) + : type_(type), inputs_(in_vars), outputs_(out_vars), attrs_(attrs) {} + + std::string type_; + VariableHandleMap inputs_; + VariableHandleMap outputs_; + framework::AttributeMap attrs_; +}; + +class Tape { + public: + void AddOp(const std::string &type, + const VariableHandleMap &in_vars, + VariableHandleMap out_vars, + const framework::AttributeMap &attrs); + void Forward(); + void Backward(VariableHandle target); + + private: + bool has_been_backwarded_ = false; + size_t current_position_ = 0; + + std::vector tape_; + std::shared_ptr backward_tape_; +}; + +Tape &get_global_tape(); + +void reset_global_tape(); +} +} diff --git a/paddle/contrib/dynamic/test_tape.cc b/paddle/contrib/dynamic/test_tape.cc new file mode 100644 index 0000000000..ad8ee8c756 --- /dev/null +++ b/paddle/contrib/dynamic/test_tape.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" +#include "paddle/contrib/dynamic/function.h" + +using namespace paddle::dynamic; + +TEST(Tape, TestMLP) { + LOG(INFO) << "TestMLP"; + Linear linear1(3, 3, "relu"); + Linear linear2(3, 3, "relu"); + Mean mean; + + SGD sgd(0.001); + + std::string initializer = "fill_constant"; + paddle::framework::AttributeMap attrs; + attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; + attrs["shape"] = std::vector{3, 3}; + attrs["value"] = 1.0f; + Fill filler(initializer, attrs); + + for (int i = 0; i < 2; ++i) { + reset_global_tape(); + + VariableHandle input(new Variable("input")); + filler(input); + + auto loss = mean(linear2(linear1(input))); + + get_global_tape().Backward(loss); + + for (auto w : linear1.Params()) { + sgd(w); + } + for (auto w : linear2.Params()) { + sgd(w); + } + } +} + +int main(int argc, char** argv) { + std::vector places; + places.emplace_back(paddle::platform::CPUPlace()); + paddle::platform::DeviceContextPool::Init(places); + + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/contrib/dynamic/variable.cc b/paddle/contrib/dynamic/variable.cc new file mode 100644 index 0000000000..8eede414f5 --- /dev/null +++ b/paddle/contrib/dynamic/variable.cc @@ -0,0 +1,33 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/contrib/dynamic/variable.h" + +namespace paddle { +namespace dynamic { + +void Variable::InitializeVariable() { + LOG(INFO) << "Initialzing " << desc_.Name() << " as " << desc_.GetType(); + framework::proto::VarType::Type var_type = desc_.GetType(); + if (var_type == framework::proto::VarType::LOD_TENSOR) { + var_.GetMutable(); + } else if (var_type == framework::proto::VarType::SELECTED_ROWS) { + var_.GetMutable(); + } else { + PADDLE_THROW("Variable type %d is not in [LOD_TENSOR, SELECTED_ROWS]", + var_type); + } +} +} +} diff --git a/paddle/contrib/dynamic/variable.h b/paddle/contrib/dynamic/variable.h new file mode 100644 index 0000000000..55b5f1fda9 --- /dev/null +++ b/paddle/contrib/dynamic/variable.h @@ -0,0 +1,85 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +#include "paddle/fluid/framework/operator.h" // framework::kGradVarSuffix +#include "paddle/fluid/framework/program_desc.h" +#include "paddle/fluid/framework/variable.h" + +namespace paddle { +namespace dynamic { + +class Variable; +using VariableHandle = std::shared_ptr; + +/* + * Combination of + * framework::VarDesc desc_; + * framework::Variable var_; + */ +class Variable { + public: + Variable(const std::string pre_fix) + : desc_(pre_fix + std::to_string(count())) {} + + Variable(const std::string pre_fix, bool is_grad) + : desc_(pre_fix + (is_grad ? framework::kGradVarSuffix + : std::to_string(count()))) {} + + ~Variable() { LOG(INFO) << "Deleting " << Name(); } + + // Instantiate LoDTensor/SelectedRow + void InitializeVariable(); + + VariableHandle Grad() { + if (grad_ == nullptr) { + grad_.reset(new Variable(desc_.Name(), true)); + } + + return grad_; + } + + void ResetGrad() { grad_ = nullptr; } + + // Stochastic Gradient Descent with Momentum + // VariableHandle Momentum (); + + // void init(const std::string& initializer, + // const framework::AttributeMap& attrs); + + // void value() {}; + + const framework::VarDesc& Desc() const { return desc_; } + framework::VarDesc* MutableDesc() { return &desc_; } + + // TODO(tonyyang-svail): No need to expose name + std::string Name() const { return desc_.Name(); } + + framework::Variable* Var() { return &var_; } + + private: + int count() { + static int counter = 0; + return counter++; + } + + framework::VarDesc desc_; + framework::Variable var_; + + VariableHandle grad_; +}; +} +} diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index d22ac66c5c..122ee1dab3 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -98,6 +98,7 @@ static LoD GetLoD(const Scope& scope, const std::string& name) { } void OperatorBase::Run(const Scope& scope, const platform::Place& place) { + VLOG(10) << "- " << DebugStringEx(&scope); if (platform::is_gpu_place(place)) { #ifndef PADDLE_WITH_CUDA PADDLE_THROW("Cannot run operator on place %s", place); @@ -107,6 +108,7 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) { #endif } RunImpl(scope, place); + VLOG(10) << "+ " << DebugStringEx(&scope); } bool OperatorBase::HasInputs(const std::string& name) const { diff --git a/paddle/fluid/framework/scope.h b/paddle/fluid/framework/scope.h index 98d103d867..95b4f7c5f6 100644 --- a/paddle/fluid/framework/scope.h +++ b/paddle/fluid/framework/scope.h @@ -81,6 +81,9 @@ class Scope { // Rename variable to a new name and return the new name std::string Rename(const std::string& origin_name) const; + protected: + mutable std::unordered_map> vars_; + private: // Call Scope::NewScope for a sub-scope. explicit Scope(Scope const* parent) : parent_(parent) {} @@ -93,8 +96,6 @@ class Scope { // Caller doesn't own the returned Variable. Variable* FindVarLocally(const std::string& name) const; - mutable std::unordered_map> vars_; - // Scope in `kids_` are owned by this class. mutable std::list kids_; Scope const* parent_{nullptr}; From a59c3b73bd4695e3b8a7110cac18765e210d9bb3 Mon Sep 17 00:00:00 2001 From: "Yang Yang(Tony)" Date: Wed, 13 Jun 2018 17:01:51 -0700 Subject: [PATCH 056/154] change dynamic graph folder (#11451) * change dynamic to tape * update readme link --- paddle/contrib/CMakeLists.txt | 2 +- .../contrib/{dynamic => tape}/CMakeLists.txt | 0 paddle/contrib/{dynamic => tape}/README.md | 18 ++++++++++++------ .../{dynamic => tape}/computation_graph.png | Bin paddle/contrib/{dynamic => tape}/function.h | 6 +++--- paddle/contrib/{dynamic => tape}/tape.cc | 4 ++-- paddle/contrib/{dynamic => tape}/tape.h | 4 ++-- paddle/contrib/{dynamic => tape}/test_tape.cc | 4 ++-- paddle/contrib/{dynamic => tape}/variable.cc | 4 ++-- paddle/contrib/{dynamic => tape}/variable.h | 2 +- 10 files changed, 25 insertions(+), 19 deletions(-) rename paddle/contrib/{dynamic => tape}/CMakeLists.txt (100%) rename paddle/contrib/{dynamic => tape}/README.md (90%) rename paddle/contrib/{dynamic => tape}/computation_graph.png (100%) rename paddle/contrib/{dynamic => tape}/function.h (97%) rename paddle/contrib/{dynamic => tape}/tape.cc (99%) rename paddle/contrib/{dynamic => tape}/tape.h (96%) rename paddle/contrib/{dynamic => tape}/test_tape.cc (95%) rename paddle/contrib/{dynamic => tape}/variable.cc (94%) rename paddle/contrib/{dynamic => tape}/variable.h (99%) diff --git a/paddle/contrib/CMakeLists.txt b/paddle/contrib/CMakeLists.txt index 75d0170567..70e3a0583d 100644 --- a/paddle/contrib/CMakeLists.txt +++ b/paddle/contrib/CMakeLists.txt @@ -14,4 +14,4 @@ # add_subdirectory(inference) -add_subdirectory(dynamic) +add_subdirectory(tape) diff --git a/paddle/contrib/dynamic/CMakeLists.txt b/paddle/contrib/tape/CMakeLists.txt similarity index 100% rename from paddle/contrib/dynamic/CMakeLists.txt rename to paddle/contrib/tape/CMakeLists.txt diff --git a/paddle/contrib/dynamic/README.md b/paddle/contrib/tape/README.md similarity index 90% rename from paddle/contrib/dynamic/README.md rename to paddle/contrib/tape/README.md index a056877093..16c22a45d5 100644 --- a/paddle/contrib/dynamic/README.md +++ b/paddle/contrib/tape/README.md @@ -1,6 +1,11 @@ # Dynamic Graph on Fluid -PaddlePaddle Fluid is targeting the autodiff without tape, which, however, is very challenging and we are still way from there. DyNet and PyTorch provide a good design idea, the *tape*, that significantly eases the challenge. Also, DyNet provides a C++ API that is as convenient as Python but with higher efficiency and could conveniently integrate with industrial/production systems. This package, `tape`, combines the good of +PaddlePaddle Fluid is targeting the autodiff without tape, which, however, is very +challenging and we are still way from there. DyNet and PyTorch provide a good design +idea, the *tape*, that significantly eases the challenge. Also, DyNet provides +a C++ API that is as convenient as Python but with higher efficiency and could +conveniently integrate with industrial/production systems. This package, `tape`, +combines the good of 1. tape from PyTorch and DyNet 2. C++ API and core from DyNet @@ -8,8 +13,8 @@ PaddlePaddle Fluid is targeting the autodiff without tape, which, however, is ve ## Overview -We can implement Dynet-like Tape(See this survey) by wrapping Paddle Fluid's `Operator` -and `Variable`. +We can implement Dynet-like Tape(See this [survey](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/survey/dynamic_graph.md)) +by wrapping Paddle Fluid's `Operator` and `Variable`. The user API is straight forward since @@ -121,13 +126,14 @@ paddle::tape::SGD sgd(0.001); // Data Feeder paddle::tape::Fill data_feeder(...); VariableHandle input(new paddle::tape::Variable("input")); +VariableHandle label(new paddle::tape::Variable("label")); for (int i = 0; i < 2; ++i) { reset_global_tape(); - data_feeder(input); + data_feeder(input, label); - auto loss = mean(linear2(linear1(input))); // compile time InferShape & InferVarType + auto loss = softmax(linear2(linear1(input)), label); // compile time InferShape & InferVarType LOG(INFO) << loss.value(); // Run forward up to loss // Run backward, store gradient of w at w->Grad() @@ -209,7 +215,7 @@ digraph G { } -![Image](https://github.com/tonyyang-svail/Paddle/blob/cpp_tap/paddle/contrib/dynamic/computation_graph.png) +![Image](https://github.com/tonyyang-svail/Paddle/blob/cpp_tap/paddle/contrib/tape/computation_graph.png) ## Code Reuse diff --git a/paddle/contrib/dynamic/computation_graph.png b/paddle/contrib/tape/computation_graph.png similarity index 100% rename from paddle/contrib/dynamic/computation_graph.png rename to paddle/contrib/tape/computation_graph.png diff --git a/paddle/contrib/dynamic/function.h b/paddle/contrib/tape/function.h similarity index 97% rename from paddle/contrib/dynamic/function.h rename to paddle/contrib/tape/function.h index 6434beebf4..0584f4ec8a 100644 --- a/paddle/contrib/dynamic/function.h +++ b/paddle/contrib/tape/function.h @@ -16,12 +16,12 @@ #include -#include "paddle/contrib/dynamic/tape.h" -#include "paddle/contrib/dynamic/variable.h" +#include "paddle/contrib/tape/tape.h" +#include "paddle/contrib/tape/variable.h" #include "paddle/fluid/framework/type_defs.h" namespace paddle { -namespace dynamic { +namespace tape { class Function {}; diff --git a/paddle/contrib/dynamic/tape.cc b/paddle/contrib/tape/tape.cc similarity index 99% rename from paddle/contrib/dynamic/tape.cc rename to paddle/contrib/tape/tape.cc index fd24eabe9d..531499b6fe 100644 --- a/paddle/contrib/dynamic/tape.cc +++ b/paddle/contrib/tape/tape.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/contrib/dynamic/tape.h" +#include "paddle/contrib/tape/tape.h" #include #include @@ -29,7 +29,7 @@ #include "paddle/fluid/pybind/pybind.h" namespace paddle { -namespace dynamic { +namespace tape { // borrowed from // https://stackoverflow.com/questions/874134/find-if-string-ends-with-another-string-in-c diff --git a/paddle/contrib/dynamic/tape.h b/paddle/contrib/tape/tape.h similarity index 96% rename from paddle/contrib/dynamic/tape.h rename to paddle/contrib/tape/tape.h index 9467e9d543..9938ce9a7f 100644 --- a/paddle/contrib/dynamic/tape.h +++ b/paddle/contrib/tape/tape.h @@ -18,10 +18,10 @@ #include #include -#include "paddle/contrib/dynamic/variable.h" +#include "paddle/contrib/tape/variable.h" namespace paddle { -namespace dynamic { +namespace tape { using VariableHandleMap = std::map>; diff --git a/paddle/contrib/dynamic/test_tape.cc b/paddle/contrib/tape/test_tape.cc similarity index 95% rename from paddle/contrib/dynamic/test_tape.cc rename to paddle/contrib/tape/test_tape.cc index ad8ee8c756..e9bfd21a71 100644 --- a/paddle/contrib/dynamic/test_tape.cc +++ b/paddle/contrib/tape/test_tape.cc @@ -13,9 +13,9 @@ // limitations under the License. #include "gtest/gtest.h" -#include "paddle/contrib/dynamic/function.h" +#include "paddle/contrib/tape/function.h" -using namespace paddle::dynamic; +using namespace paddle::tape; TEST(Tape, TestMLP) { LOG(INFO) << "TestMLP"; diff --git a/paddle/contrib/dynamic/variable.cc b/paddle/contrib/tape/variable.cc similarity index 94% rename from paddle/contrib/dynamic/variable.cc rename to paddle/contrib/tape/variable.cc index 8eede414f5..5ec1612909 100644 --- a/paddle/contrib/dynamic/variable.cc +++ b/paddle/contrib/tape/variable.cc @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/contrib/dynamic/variable.h" +#include "paddle/contrib/tape/variable.h" namespace paddle { -namespace dynamic { +namespace tape { void Variable::InitializeVariable() { LOG(INFO) << "Initialzing " << desc_.Name() << " as " << desc_.GetType(); diff --git a/paddle/contrib/dynamic/variable.h b/paddle/contrib/tape/variable.h similarity index 99% rename from paddle/contrib/dynamic/variable.h rename to paddle/contrib/tape/variable.h index 55b5f1fda9..7e63aa38a7 100644 --- a/paddle/contrib/dynamic/variable.h +++ b/paddle/contrib/tape/variable.h @@ -20,7 +20,7 @@ #include "paddle/fluid/framework/variable.h" namespace paddle { -namespace dynamic { +namespace tape { class Variable; using VariableHandle = std::shared_ptr; From f790b96d6f8fc92f9b23cf852ceb7d7f72f6e677 Mon Sep 17 00:00:00 2001 From: "Yang Yang(Tony)" Date: Wed, 13 Jun 2018 17:49:36 -0700 Subject: [PATCH 057/154] make variable->Grad() a weak_ptr (#11453) * fix #11416 * make sgd check tape has been backwarded_ * add error message --- paddle/contrib/tape/function.h | 3 ++- paddle/contrib/tape/tape.h | 2 ++ paddle/contrib/tape/variable.h | 14 +++++++------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/paddle/contrib/tape/function.h b/paddle/contrib/tape/function.h index 0584f4ec8a..8c9694d9a2 100644 --- a/paddle/contrib/tape/function.h +++ b/paddle/contrib/tape/function.h @@ -112,6 +112,8 @@ class SGD { } void operator()(VariableHandle input) { + PADDLE_ENFORCE(get_global_tape().HasBeenBackwarded(), + "optimization must happen after the backward"); Tape temp_tape; temp_tape.AddOp("sgd", {{"Param", {input}}, @@ -120,7 +122,6 @@ class SGD { {{"ParamOut", {input}}}, {}); temp_tape.Forward(); - input->ResetGrad(); } private: diff --git a/paddle/contrib/tape/tape.h b/paddle/contrib/tape/tape.h index 9938ce9a7f..ed79de17a7 100644 --- a/paddle/contrib/tape/tape.h +++ b/paddle/contrib/tape/tape.h @@ -47,6 +47,8 @@ class Tape { void Forward(); void Backward(VariableHandle target); + bool HasBeenBackwarded() { return has_been_backwarded_; } + private: bool has_been_backwarded_ = false; size_t current_position_ = 0; diff --git a/paddle/contrib/tape/variable.h b/paddle/contrib/tape/variable.h index 7e63aa38a7..35c328e69c 100644 --- a/paddle/contrib/tape/variable.h +++ b/paddle/contrib/tape/variable.h @@ -45,15 +45,15 @@ class Variable { void InitializeVariable(); VariableHandle Grad() { - if (grad_ == nullptr) { - grad_.reset(new Variable(desc_.Name(), true)); + if (grad_.expired()) { + VariableHandle new_grad(new Variable(desc_.Name(), true)); + grad_ = new_grad; + return new_grad; + } else { + return VariableHandle(grad_); } - - return grad_; } - void ResetGrad() { grad_ = nullptr; } - // Stochastic Gradient Descent with Momentum // VariableHandle Momentum (); @@ -79,7 +79,7 @@ class Variable { framework::VarDesc desc_; framework::Variable var_; - VariableHandle grad_; + std::weak_ptr grad_; }; } } From d2afd2102155e535c009f75a679a744cd1aff905 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Thu, 14 Jun 2018 09:00:16 +0800 Subject: [PATCH 058/154] Remove cuptiFinalize. In cupti samples, only cuptiFlush is used. I can't find any places calling cuptiFinalize and this API can error out as not_implemented in some cuda installation. --- paddle/fluid/platform/device_tracer.cc | 1 - paddle/fluid/platform/dynload/cupti.h | 1 - 2 files changed, 2 deletions(-) diff --git a/paddle/fluid/platform/device_tracer.cc b/paddle/fluid/platform/device_tracer.cc index 1a9be044e0..d9e2afadaf 100644 --- a/paddle/fluid/platform/device_tracer.cc +++ b/paddle/fluid/platform/device_tracer.cc @@ -322,7 +322,6 @@ class DeviceTracerImpl : public DeviceTracer { DisableActivity(); dynload::cuptiUnsubscribe(subscriber_); CUPTI_CALL(dynload::cuptiGetTimestamp(&end_ns_)); - PADDLE_ENFORCE(dynload::cuptiFinalize()); enabled_ = false; } diff --git a/paddle/fluid/platform/dynload/cupti.h b/paddle/fluid/platform/dynload/cupti.h index 2ad52bc7d3..e8f4a82ef1 100644 --- a/paddle/fluid/platform/dynload/cupti.h +++ b/paddle/fluid/platform/dynload/cupti.h @@ -72,7 +72,6 @@ extern void *cupti_dso_handle; __macro(cuptiGetResultString); \ __macro(cuptiActivityGetNumDroppedRecords); \ __macro(cuptiActivityFlushAll); \ - __macro(cuptiFinalize); \ __macro(cuptiSubscribe); \ __macro(cuptiUnsubscribe); \ __macro(cuptiEnableCallback); \ From 490a07f5b6f8c461a46a045dd07025f5ce9c74ba Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 14 Jun 2018 09:22:59 +0800 Subject: [PATCH 059/154] add comment that out var of prefetch must be created in local scope --- paddle/fluid/operators/detail/grpc_server.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/operators/detail/grpc_server.cc b/paddle/fluid/operators/detail/grpc_server.cc index 14152bded8..5a87258901 100644 --- a/paddle/fluid/operators/detail/grpc_server.cc +++ b/paddle/fluid/operators/detail/grpc_server.cc @@ -169,6 +169,7 @@ class RequestPrefetch final : public RequestBase { auto scope = request_->GetMutableLocalScope(); auto invar = scope->FindVar(in_var_name); + // out var must be created in local scope! framework::Variable* outvar = scope->Var(out_var_name); request_handler_->Handle(in_var_name, scope, invar, &outvar, out_var_name); From 6fcdb240fae6d1cfd00a433cc549e55703ab8d7d Mon Sep 17 00:00:00 2001 From: whs Date: Thu, 14 Jun 2018 09:38:20 +0800 Subject: [PATCH 060/154] Add mean IOU op. (#10519) * Add mean_iou op. * Add unitest for mean iou op. * Add optional collections of confusion matrix and mean_iou. * Fix cuda kernel. * Refine code. 1. Merge computing in GPU to two kernel. 2. Use wrong array and correct array instead of confusion matrix. * Add python api and fix cuda kernel. * Fix comments. * Small fix. * Small fix. --- paddle/fluid/operators/mean_iou_op.cc | 110 ++++++++++++ paddle/fluid/operators/mean_iou_op.cu | 164 ++++++++++++++++++ paddle/fluid/operators/mean_iou_op.h | 117 +++++++++++++ python/paddle/fluid/layers/nn.py | 127 +++++++------- .../fluid/tests/unittests/test_mean_iou.py | 114 ++++++++++++ 5 files changed, 570 insertions(+), 62 deletions(-) create mode 100644 paddle/fluid/operators/mean_iou_op.cc create mode 100644 paddle/fluid/operators/mean_iou_op.cu create mode 100644 paddle/fluid/operators/mean_iou_op.h create mode 100644 python/paddle/fluid/tests/unittests/test_mean_iou.py diff --git a/paddle/fluid/operators/mean_iou_op.cc b/paddle/fluid/operators/mean_iou_op.cc new file mode 100644 index 0000000000..a60f245f53 --- /dev/null +++ b/paddle/fluid/operators/mean_iou_op.cc @@ -0,0 +1,110 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/mean_iou_op.h" + +namespace paddle { +namespace operators { + +class MeanIoUOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Predictions"), + "Input (Predictions) of MeanIoU op should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Labels"), + "Input (labels) of MeanIoU op should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("OutMeanIou"), + "Output (OutMeanIou) of MeanIoU op should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("OutWrong"), + "Output (OutWrong) of MeanIoU op should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("OutCorrect"), + "Output (OutWrong) of MeanIoU op should not be null."); + + int64_t num_classes = + static_cast(ctx->Attrs().Get("num_classes")); + + ctx->SetOutputDim("OutMeanIou", {1}); + ctx->SetOutputDim("OutWrong", {num_classes}); + ctx->SetOutputDim("OutCorrect", {num_classes}); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Predictions")->type()), + ctx.GetPlace()); + } +}; + +class MeanIoUOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("Predictions", + "(Tensor), A Tensor of prediction results for semantic labels" + " with type int32 or int64. The rank should be greater than 1."); + AddInput( + "Labels", + "(Tensor), A Tensor of ground truth labels with type int32 or int64." + "Its shape should be the same as Input(Predictions)."); + AddInput("InWrongs", + "(vector), A list of Tensor with shape " + "[num_classes]. They are used to collect wrong number among " + "batches. Empty list is also valid here.") + .AsDuplicable() + .AsDispensable(); + AddInput( + "InCorrects", + "(vector), A list of Tensor with shape " + "[num_classes]. They are used to collect correct number among batches. " + "Empty list is also valid here.") + .AsDuplicable() + .AsDispensable(); + AddInput("InMeanIou", + "(vector), A list of Tensor that Output(mean_iou) should " + "be added to. Empty list is also valid here.") + .AsDuplicable() + .AsDispensable(); + AddOutput("OutMeanIou", + "(vector), A Tensor representing the" + " mean intersection-over-union with shape [1]."); + AddOutput("OutWrong", "(Tensor), A Tensor with shape [num_classes]. "); + AddOutput("OutCorrect", "(Tensor), A Tensor with shape [num_classes]. "); + AddAttr("num_classes", "(int), The possible number of labels."); + + AddComment(R"DOC( +mean-IOU Operator. +Mean Intersection-Over-Union is a common evaluation metric for +semantic image segmentation, which first computes the IOU for each +semantic class and then computes the average over classes. +IOU is defined as follows: + IOU = true_positive / (true_positive + false_positive + false_negative). +It is based on pixel level area while "IOU Similarity Operator" +is based on area of rectangle. + +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(mean_iou, ops::MeanIoUOp, ops::MeanIoUOpMaker, + paddle::framework::EmptyGradOpMaker); +REGISTER_OP_CPU_KERNEL(mean_iou, ops::MeanIoUKernel, + ops::MeanIoUKernel, + ops::MeanIoUKernel); diff --git a/paddle/fluid/operators/mean_iou_op.cu b/paddle/fluid/operators/mean_iou_op.cu new file mode 100644 index 0000000000..83bb4dde46 --- /dev/null +++ b/paddle/fluid/operators/mean_iou_op.cu @@ -0,0 +1,164 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/math/math_function.h" +#include "paddle/fluid/operators/mean_iou_op.h" +#include "paddle/fluid/platform/cuda_primitives.h" +#include "paddle/fluid/platform/gpu_info.h" + +namespace paddle { +namespace operators { + +using platform::PADDLE_CUDA_NUM_THREADS; + +#define CUDA_1D_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +template +__global__ void CountCUDAKernel(const int num_classes, const int count, + const T* predictions, const T* labels, + int* wrong, int* correct) { + extern __shared__ int blcok_cache[]; + int* wrong_c = blcok_cache; + int* correct_c = blcok_cache + num_classes; + // init cache + for (int i = threadIdx.x; i < num_classes * 2; i += blockDim.x) { + blcok_cache[i] = 0; + } + __syncthreads(); + + T pred; + T label; + CUDA_1D_KERNEL_LOOP(i, count) { + pred = predictions[i]; + label = labels[i]; + if (pred == label) { + atomicAdd(correct_c + pred, 1); + } else { + atomicAdd(wrong_c + pred, 1); + atomicAdd(wrong_c + label, 1); + } + } + + __syncthreads(); + + for (int i = threadIdx.x; i < num_classes; i += blockDim.x) { + atomicAdd(wrong + i, wrong_c[i]); + atomicAdd(correct + i, correct_c[i]); + } +} + +__global__ void ComputeIoUCUDAKernel(const int num_classes, int* wrong, + int* correct, float* ious, float* iou) { + __shared__ int valid_count_c; + if (threadIdx.x == 0) { + valid_count_c = 0; + } + __syncthreads(); + CUDA_1D_KERNEL_LOOP(i, num_classes) { + int wrong_n = wrong[i]; + int correct_n = correct[i]; + int denominator = wrong_n + correct_n; + if (denominator > 0) { + atomicAdd(&valid_count_c, 1); + ious[i] = static_cast(correct_n) / denominator; + } else { + ious[i] = 0; + } + } + __syncthreads(); + if (threadIdx.x == 0) { + float iou_sum = 0; + for (int i = 0; i < num_classes; ++i) { + iou_sum += ious[i]; + } + iou[0] += iou_sum / valid_count_c; + } +} + +template +class MeanIoUCUDAOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto& place = *ctx.template device_context() + .eigen_device(); + // get input and output tensor + auto* predictions = ctx.Input("Predictions"); + auto* labels = ctx.Input("Labels"); + auto* out_mean_iou = ctx.Output("OutMeanIou"); + auto* out_wrong = ctx.Output("OutWrong"); + auto* out_correct = ctx.Output("OutCorrect"); + int num_classes = static_cast(ctx.Attr("num_classes")); + + // Get data ptr + const T* predictions_data = predictions->data(); + const T* labels_data = labels->data(); + int* out_wrong_data = out_wrong->mutable_data(ctx.GetPlace()); + int* out_correct_data = out_correct->mutable_data(ctx.GetPlace()); + float* out_mean_iou_data = + out_mean_iou->mutable_data(ctx.GetPlace()); + + // Get Eigen tensor + auto out_mean_iou_t = EigenTensor::From(*out_mean_iou); + auto out_wrong_t = EigenTensor::From(*out_wrong); + auto out_correct_t = EigenTensor::From(*out_correct); + + // Temporary tensor + Tensor ious; + float* ious_data = ious.mutable_data( + {static_cast(num_classes)}, ctx.GetPlace()); + auto ious_t = EigenTensor::From(ious); + + // Init out_wrong, out_correct and out_mean_iou + out_wrong_t.device(place) = out_wrong_t.constant(0); + out_correct_t.device(place) = out_correct_t.constant(0); + out_mean_iou_t.device(place) = out_mean_iou_t.constant(0.0f); + + // collect pre wrong, correct and mean_iou + auto in_mean_ious = ctx.MultiInput("InMeanIou"); + for (int i = 0; i < in_mean_ious.size(); ++i) { + out_mean_iou_t.device(place) += + EigenTensor::From(*in_mean_ious[i]); + } + auto in_wrongs = ctx.MultiInput("InWrongs"); + for (int i = 0; i < in_wrongs.size(); ++i) { + out_wrong_t.device(place) += EigenTensor::From(*in_wrongs[i]); + } + auto in_corrects = ctx.MultiInput("InCorrects"); + for (int i = 0; i < in_corrects.size(); ++i) { + out_correct_t.device(place) += EigenTensor::From(*in_corrects[i]); + } + // compute + auto stream = ctx.cuda_device_context().stream(); + int block = PADDLE_CUDA_NUM_THREADS; + int grid = (predictions->numel() + block - 1) / block; + int cache_size = (num_classes * 2 + 1) * sizeof(int); + CountCUDAKernel<<>>( + num_classes, predictions->numel(), predictions_data, labels_data, + out_wrong_data, out_correct_data); + ctx.device_context().Wait(); + ComputeIoUCUDAKernel<<<1, block, 0, stream>>>(num_classes, out_wrong_data, + out_correct_data, ious_data, + out_mean_iou_data); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(mean_iou, ops::MeanIoUCUDAOpKernel, + ops::MeanIoUCUDAOpKernel, + ops::MeanIoUCUDAOpKernel); diff --git a/paddle/fluid/operators/mean_iou_op.h b/paddle/fluid/operators/mean_iou_op.h new file mode 100644 index 0000000000..9fa00e60e0 --- /dev/null +++ b/paddle/fluid/operators/mean_iou_op.h @@ -0,0 +1,117 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { +using Tensor = framework::Tensor; + +template +using EigenTensor = framework::EigenTensor; + +template +class MeanIoUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto& place = *ctx.template device_context() + .eigen_device(); + // get input and output tensor + auto* predictions = ctx.Input("Predictions"); + auto* labels = ctx.Input("Labels"); + auto* out_mean_iou = ctx.Output("OutMeanIou"); + auto* out_wrong = ctx.Output("OutWrong"); + auto* out_correct = ctx.Output("OutCorrect"); + int num_classes = static_cast(ctx.Attr("num_classes")); + + // get data ptr + const T* predictions_data = predictions->data(); + const T* labels_data = labels->data(); + float* out_mean_iou_data = + out_mean_iou->mutable_data(ctx.GetPlace()); + int* out_wrong_data = out_wrong->mutable_data(ctx.GetPlace()); + int* out_correct_data = out_correct->mutable_data(ctx.GetPlace()); + + // get eigen tensor + auto out_mean_iou_t = EigenTensor::From(*out_mean_iou); + auto out_wrong_t = EigenTensor::From(*out_wrong); + auto out_correct_t = EigenTensor::From(*out_correct); + + // Tmp tensor + Tensor denominator; + Tensor valid_count; + Tensor iou_sum; + + // get data ptr of tmp tensor + int* denominator_data = denominator.mutable_data( + {static_cast(num_classes)}, ctx.GetPlace()); + int* valid_count_data = valid_count.mutable_data({1}, ctx.GetPlace()); + float* iou_sum_data = iou_sum.mutable_data({1}, ctx.GetPlace()); + + // get eigen tensor of tmp tensor + auto denominator_t = EigenTensor::From(denominator); + auto valid_count_t = EigenTensor::From(valid_count); + auto iou_sum_t = EigenTensor::From(iou_sum); + + // init out_wrong, out_correct and out_mean_iou + out_wrong_t = out_wrong_t.constant(0); + out_correct_t = out_correct_t.constant(0); + out_mean_iou_t = out_mean_iou_t.constant(0); + + // collect pre wrong, correct and mean_iou + auto in_mean_ious = ctx.MultiInput("InMeanIou"); + for (size_t i = 0; i < in_mean_ious.size(); ++i) { + out_mean_iou_t.device(place) += + EigenTensor::From(*in_mean_ious[i]); + } + auto in_wrongs = ctx.MultiInput("InWrongs"); + for (size_t i = 0; i < in_wrongs.size(); ++i) { + out_wrong_t.device(place) += EigenTensor::From(*in_wrongs[i]); + } + auto in_corrects = ctx.MultiInput("InCorrects"); + for (size_t i = 0; i < in_corrects.size(); ++i) { + out_correct_t.device(place) += EigenTensor::From(*in_corrects[i]); + } + + // compute + for (int64_t i = 0; i < predictions->numel(); ++i) { + if (predictions_data[i] == labels_data[i]) { + out_correct_data[predictions_data[i]] += 1; + } else { + out_wrong_data[labels_data[i]] += 1; + out_wrong_data[predictions_data[i]] += 1; + } + } + + denominator_t = out_wrong_t + out_correct_t; + valid_count_t = + (denominator_t > denominator_t.constant(0.0f)).cast().sum(); + + for (int i = 0; i < num_classes; ++i) { + if (denominator_data[i] == 0) { + denominator_data[i] = 1; + } + } + + iou_sum_t = + (out_correct_t.cast() / denominator_t.cast()).sum(); + out_mean_iou_data[0] += (iou_sum_data[0] / valid_count_data[0]); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index fde8a3154f..982340d7ee 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -25,68 +25,20 @@ import utils import random __all__ = [ - 'fc', - 'embedding', - 'dynamic_lstm', - 'dynamic_lstmp', - 'dynamic_gru', - 'gru_unit', - 'linear_chain_crf', - 'crf_decoding', - 'cos_sim', - 'cross_entropy', - 'square_error_cost', - 'chunk_eval', - 'sequence_conv', - 'conv2d', - 'sequence_pool', - 'sequence_softmax', - 'softmax', - 'pool2d', - 'batch_norm', - 'beam_search_decode', - 'conv2d_transpose', - 'sequence_expand', - 'lstm_unit', - 'reduce_sum', - 'reduce_mean', - 'reduce_max', - 'reduce_min', - 'reduce_prod', - 'sequence_first_step', - 'sequence_last_step', - 'dropout', - 'split', - 'ctc_greedy_decoder', - 'edit_distance', - 'l2_normalize', - 'matmul', - 'topk', - 'warpctc', - 'sequence_reshape', - 'transpose', - 'im2sequence', - 'nce', - 'beam_search', - 'row_conv', - 'multiplex', - 'layer_norm', - 'softmax_with_cross_entropy', - 'smooth_l1', - 'one_hot', - 'autoincreased_step_counter', - 'reshape', - 'lod_reset', - 'lrn', - 'pad', - 'label_smooth', - 'roi_pool', - 'dice_loss', - 'image_resize', - 'image_resize_short', - 'resize_bilinear', - 'gather', - 'random_crop', + 'fc', 'embedding', 'dynamic_lstm', 'dynamic_lstmp', 'dynamic_gru', + 'gru_unit', 'linear_chain_crf', 'crf_decoding', 'cos_sim', 'cross_entropy', + 'square_error_cost', 'chunk_eval', 'sequence_conv', 'conv2d', + 'sequence_pool', 'sequence_softmax', 'softmax', 'pool2d', 'batch_norm', + 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', 'lstm_unit', + 'reduce_sum', 'reduce_mean', 'reduce_max', 'reduce_min', 'reduce_prod', + 'sequence_first_step', 'sequence_last_step', 'dropout', 'split', + 'ctc_greedy_decoder', 'edit_distance', 'l2_normalize', 'matmul', 'topk', + 'warpctc', 'sequence_reshape', 'transpose', 'im2sequence', 'nce', + 'beam_search', 'row_conv', 'multiplex', 'layer_norm', + 'softmax_with_cross_entropy', 'smooth_l1', 'one_hot', + 'autoincreased_step_counter', 'reshape', 'lod_reset', 'lrn', 'pad', + 'label_smooth', 'roi_pool', 'dice_loss', 'image_resize', + 'image_resize_short', 'resize_bilinear', 'gather', 'random_crop', 'mean_iou' ] @@ -4231,6 +4183,7 @@ def gather(input, index): output (Variable): The output is a tensor with the same rank as input. Examples: + .. code-block:: python output = fluid.layers.gather(x, index) @@ -4295,3 +4248,53 @@ def random_crop(x, shape, seed=None): "SeedOut": seed_out}, attrs={"shape": shape}) return out + + +def mean_iou(input, label, num_classes): + """ + Mean Intersection-Over-Union is a common evaluation metric for + semantic image segmentation, which first computes the IOU for each + semantic class and then computes the average over classes. + IOU is defined as follows: + + .. math:: + + IOU = true_positive / (true_positive + false_positive + false_negative). + + The predictions are accumulated in a confusion matrix and mean-IOU + is then calculated from it. + + + Args: + input (Variable): A Tensor of prediction results for semantic labels with type int32 or int64. + label (Variable): A Tensor of ground truth labels with type int32 or int64. + Its shape should be the same as input. + + Returns: + mean_iou (Variable): A Tensor representing the mean intersection-over-union with shape [1]. + out_wrong(Variable): A Tensor with shape [num_classes]. The wrong numbers of each class. + out_correct(Variable): A Tensor with shape [num_classes]. The correct numbers of each class. + + + Examples: + + .. code-block:: python + + iou, wrongs, corrects = fluid.layers.mean_iou(predict, label, num_classes) + """ + helper = LayerHelper('mean_iou', **locals()) + dtype = helper.input_dtype() + out_mean_iou = helper.create_tmp_variable(dtype='float32') + out_wrong = helper.create_tmp_variable(dtype='int32') + out_correct = helper.create_tmp_variable(dtype='int32') + helper.append_op( + type="mean_iou", + inputs={"predictions": input, + "labels": label}, + outputs={ + "out_mean_iou": out_mean_iou, + "out_wrong": out_wrong, + "out_correct": out_correct + }, + attrs={"num_classes": num_classes}) + return out_mean_iou, out_wrong, out_correct diff --git a/python/paddle/fluid/tests/unittests/test_mean_iou.py b/python/paddle/fluid/tests/unittests/test_mean_iou.py new file mode 100644 index 0000000000..64d42b693b --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_mean_iou.py @@ -0,0 +1,114 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +import unittest +import numpy as np +from op_test import OpTest + + +def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, + in_mean_ious): + assert predictions.shape == labels.shape + predictions = predictions.flatten() + labels = labels.flatten() + + out_wrong = np.zeros([num_classes]).astype("int32") + for _, wrong in in_wrongs: + out_wrong += wrong + out_correct = np.zeros([num_classes]).astype("int32") + for _, correct in in_corrects: + out_correct += correct + + for pred, label in zip(predictions, labels): + if pred == label: + out_correct[pred] += 1 + else: + out_wrong[pred] += 1 + out_wrong[label] += 1 + + denominator = out_wrong + out_correct + valid_count = (denominator != 0).sum() + denominator = np.where(denominator > 0, denominator, + np.ones(denominator.shape)) + mean_iou = (out_correct / denominator).sum() / valid_count + + for _, in_mean_iou in in_mean_ious: + mean_iou += in_mean_iou + return mean_iou, out_wrong, out_correct + + +class TestMeanIOUOp(OpTest): + def setUp(self): + self.config() + self.op_type = "mean_iou" + predictions = np.random.randint(0, self.num_classes, + self.image_size).astype("int32") + labels = np.random.randint(0, self.num_classes, + self.image_size).astype("int32") + + in_wrongs = [] + for i in range(self.in_wrong_num): + in_wrongs.append(("in_wrong_%d" % i, np.random.randint( + 0, 10, [self.num_classes]).astype("int32"))) + + in_corrects = [] + for i in range(self.in_correct_num): + in_corrects.append(("in_correct_%d" % i, np.random.randint( + 0, 10, [self.num_classes]).astype("int32"))) + + in_mean_ious = [] + for i in range(self.in_mean_iou_num): + in_mean_ious.append(("in_mean_iou_%d" % i, np.random.uniform( + 0, 1, [1]).astype("float32"))) + + self.inputs = { + 'Predictions': predictions, + 'Labels': labels, + 'InWrongs': in_wrongs, + 'InCorrects': in_corrects, + 'InMeanIou': in_mean_ious + } + self.attrs = {'num_classes': long(self.num_classes)} + mean_iou, out_wrong, out_correct = compute_mean_iou( + predictions, labels, self.num_classes, in_wrongs, in_corrects, + in_mean_ious) + self.outputs = { + 'OutMeanIou': mean_iou, + 'OutWrong': out_wrong, + 'OutCorrect': out_correct + } + + def config(self): + self.num_classes = 10 + self.image_size = [128, 128] + self.in_wrong_num = 0 + self.in_correct_num = 0 + self.in_mean_iou_num = 0 + + def test_check_output(self): + self.check_output() + + +class TestCase1(TestMeanIOUOp): + def config(self): + self.num_classes = 5 + self.image_size = [100, 128] + self.in_wrong_num = 2 + self.in_correct_num = 2 + self.in_mean_iou_num = 2 + + +if __name__ == '__main__': + unittest.main() From bd2a537b054116582239779f8ccfc59da0ea2bb1 Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Thu, 14 Jun 2018 10:44:10 +0800 Subject: [PATCH 061/154] feature/anakin ci (#11330) --- CMakeLists.txt | 6 ++- cmake/external/anakin.cmake | 42 +++++++++++++++ paddle/contrib/inference/CMakeLists.txt | 53 ++++--------------- .../paddle_inference_api_anakin_engine.cc | 3 +- .../paddle_inference_api_anakin_engine.h | 3 +- ...ddle_inference_api_anakin_engine_tester.cc | 6 ++- paddle/scripts/paddle_build.sh | 3 +- 7 files changed, 66 insertions(+), 50 deletions(-) create mode 100644 cmake/external/anakin.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b35290e12f..4117f07721 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ option(EIGEN_USE_THREADS "Compile with multi-threaded Eigen" OFF) option(WITH_ARM_FP16 "Use half precision support on armv8.2-a cpu" OFF) option(WITH_FAST_BUNDLE_TEST "Bundle tests that can be run in a single process together to reduce launch overhead" OFF) option(WITH_CONTRIB "Compile the third-party contributation" OFF) +option(WITH_ANAKIN "Compile with Anakin library" OFF) option(WITH_GRPC "Use grpc as the default rpc framework" ${WITH_DISTRIBUTE}) # CMAKE_BUILD_TYPE @@ -193,7 +194,10 @@ set(EXTERNAL_LIBS if(WITH_GPU) include(cuda) include(tensorrt) -endif(WITH_GPU) + include(external/anakin) +else() + set(WITH_ANAKIN OFF CACHE STRING "Anakin is valid only when GPU is set." FORCE) +endif() if(WITH_AMD_GPU) find_package(HIP) diff --git a/cmake/external/anakin.cmake b/cmake/external/anakin.cmake new file mode 100644 index 0000000000..f1cd9c99eb --- /dev/null +++ b/cmake/external/anakin.cmake @@ -0,0 +1,42 @@ +if (NOT WITH_ANAKIN) + return() +endif() + +set(ANAKIN_INSTALL_DIR "${THIRD_PARTY_PATH}/install/anakin" CACHE PATH + "Anakin install path." FORCE) +set(ANAKIN_INCLUDE "${ANAKIN_INSTALL_DIR}" CACHE STRING "root of Anakin header files") +set(ANAKIN_LIBRARY "${ANAKIN_INSTALL_DIR}" CACHE STRING "path of Anakin library") + +set(ANAKIN_COMPILE_EXTRA_FLAGS -Wno-error=unused-variable -Wno-error=format-extra-args -Wno-error=comment -Wno-error=format -Wno-error=switch -Wno-error=return-type -Wno-error=non-virtual-dtor -Wno-reorder -Wno-error=cpp) + +set(ANAKIN_LIBRARY_URL "https://github.com/pangge/Anakin/releases/download/3.0/anakin_release_simple.tar.gz") + +# A helper function used in Anakin, currently, to use it, one need to recursively include +# nearly all the header files. +function(fetch_include_recursively root_dir) + if (IS_DIRECTORY ${root_dir}) + include_directories(${root_dir}) + endif() + + file(GLOB ALL_SUB RELATIVE ${root_dir} ${root_dir}/*) + foreach(sub ${ALL_SUB}) + if (IS_DIRECTORY ${root_dir}/${sub}) + fetch_include_recursively(${root_dir}/${sub}) + endif() + endforeach() +endfunction() + +# download library +message(STATUS "Download Anakin library from ${ANAKIN_LIBRARY_URL}") +execute_process(COMMAND bash -c "mkdir -p ${ANAKIN_INSTALL_DIR}") +execute_process(COMMAND bash -c "rm -rf ${ANAKIN_INSTALL_DIR}/*") +execute_process(COMMAND bash -c "cd ${ANAKIN_INSTALL_DIR}; wget -q ${ANAKIN_LIBRARY_URL}") +execute_process(COMMAND bash -c "mkdir -p ${ANAKIN_INSTALL_DIR}") +execute_process(COMMAND bash -c "cd ${ANAKIN_INSTALL_DIR}; tar xzf anakin_release_simple.tar.gz") + +if (WITH_ANAKIN) + message(STATUS "Anakin for inference is enabled") + message(STATUS "Anakin is set INCLUDE:${ANAKIN_INCLUDE} LIBRARY:${ANAKIN_LIBRARY}") + fetch_include_recursively(${ANAKIN_INCLUDE}) + link_directories(${ANAKIN_LIBRARY}) +endif() diff --git a/paddle/contrib/inference/CMakeLists.txt b/paddle/contrib/inference/CMakeLists.txt index f279020e93..277b0b175b 100644 --- a/paddle/contrib/inference/CMakeLists.txt +++ b/paddle/contrib/inference/CMakeLists.txt @@ -17,48 +17,9 @@ if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move") endif(APPLE) -set(ANAKIN_INCLUDE "" CACHE STRING "root of Anakin header files") -set(ANAKIN_LIBRARY "" CACHE STRING "path of Anakin library") - set(inference_deps paddle_inference_api paddle_fluid_api) -# if anakin is set enable anakin api implementation -if(ANAKIN_INCLUDE AND ANAKIN_LIBRARY) - set(ANAKIN_FOUND ON) -else() - set(ANAKIN_FOUND OFF) -endif() - -function(fetch_include_recursively root_dir) - if (IS_DIRECTORY ${root_dir}) - include_directories(${root_dir}) - endif() - - file(GLOB ALL_SUB RELATIVE ${root_dir} ${root_dir}/*) - foreach(sub ${ALL_SUB}) - if (IS_DIRECTORY ${root_dir}/${sub}) - fetch_include_recursively(${root_dir}/${sub}) - endif() - endforeach() -endfunction() - -if (ANAKIN_FOUND) - # Anakin's code style doesn't follow google c style. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=unused-variable -Wno-error=format-extra-args -Wno-error=comment -Wno-error=format -Wno-error=switch -Wno-error=return-type -Wno-error=non-virtual-dtor -Wno-reorder -Wno-error=cpp") - - message(STATUS "Anakin for inference is enabled") - message(STATUS "Anakin is set INCLUDE:${ANAKIN_INCLUDE} LIBRARY:${ANAKIN_LIBRARY}") - fetch_include_recursively(${ANAKIN_INCLUDE}) - - link_directories(${ANAKIN_LIBRARY}) - - nv_library(inference_anakin_api SHARED SRCS paddle_inference_api.cc paddle_inference_api_anakin_engine.cc) - target_link_libraries(inference_anakin_api anakin anakin_saber_common) - list(APPEND inference_deps inference_anakin_api) -endif() - - function(inference_api_test TARGET_NAME) if (WITH_TESTING) set(options "") @@ -79,7 +40,7 @@ function(inference_api_test TARGET_NAME) endfunction(inference_api_test) cc_library(paddle_inference_api - SRCS paddle_inference_api.cc paddle_inference_api_impl.cc + SRCS paddle_inference_api.cc paddle_inference_api_impl.cc DEPS ${FLUID_CORE_MODULES} ${GLOB_OP_LIB}) cc_test(test_paddle_inference_api @@ -89,9 +50,17 @@ cc_test(test_paddle_inference_api inference_api_test(test_paddle_inference_api_impl ARGS test_word2vec test_image_classification) -if (ANAKIN_FOUND) +if (WITH_ANAKIN) + # Due to Anakin do not have official library releases and the versions of protobuf and cuda do not match Paddle's, + # so anakin library will not be merged to our official inference library. To use anakin prediction API, one need to + # compile the libinference_anakin_api.a and compile with anakin.so. + nv_library(inference_anakin_api SHARED SRCS paddle_inference_api.cc paddle_inference_api_anakin_engine.cc) + target_compile_options(inference_anakin_api BEFORE PUBLIC ${ANAKIN_COMPILE_EXTRA_FLAGS}) + target_link_libraries(inference_anakin_api anakin anakin_saber_common) cc_test(inference_anakin_test SRCS paddle_inference_api_anakin_engine_tester.cc - DEPS ${inference_deps}) + ARGS --model=${ANAKIN_INSTALL_DIR}/mobilenet_v2.anakin.bin + DEPS inference_anakin_api) + target_compile_options(inference_anakin_test BEFORE PUBLIC ${ANAKIN_COMPILE_EXTRA_FLAGS}) endif() if(WITH_TESTING) diff --git a/paddle/contrib/inference/paddle_inference_api_anakin_engine.cc b/paddle/contrib/inference/paddle_inference_api_anakin_engine.cc index ea7781f691..5bafc58fa5 100644 --- a/paddle/contrib/inference/paddle_inference_api_anakin_engine.cc +++ b/paddle/contrib/inference/paddle_inference_api_anakin_engine.cc @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include "paddle/contrib/inference/paddle_inference_api_anakin_engine.h" +#include namespace paddle { diff --git a/paddle/contrib/inference/paddle_inference_api_anakin_engine.h b/paddle/contrib/inference/paddle_inference_api_anakin_engine.h index 181784cbdf..212ba41cdf 100644 --- a/paddle/contrib/inference/paddle_inference_api_anakin_engine.h +++ b/paddle/contrib/inference/paddle_inference_api_anakin_engine.h @@ -19,10 +19,9 @@ limitations under the License. */ #pragma once -// NOTE This header file do not have namespace. -//#include #include "paddle/contrib/inference/paddle_inference_api.h" +// from anakin #include "framework/core/net/net.h" #include "saber/saber_types.h" diff --git a/paddle/contrib/inference/paddle_inference_api_anakin_engine_tester.cc b/paddle/contrib/inference/paddle_inference_api_anakin_engine_tester.cc index 47b9c6fa28..1d41a5c73e 100644 --- a/paddle/contrib/inference/paddle_inference_api_anakin_engine_tester.cc +++ b/paddle/contrib/inference/paddle_inference_api_anakin_engine_tester.cc @@ -12,17 +12,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include #include #include -#include "gflags/gflags.h" #include "paddle/contrib/inference/paddle_inference_api.h" +DEFINE_string(model, "", "Directory of the inference model."); + namespace paddle { AnakinConfig GetConfig() { AnakinConfig config; - config.model_file = "./mobilenet_v2.anakin.bin"; + config.model_file = FLAGS_model; config.device = 0; config.max_batch_size = 1; return config; diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index c6eef8683d..e8b3053267 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -132,7 +132,8 @@ EOF -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake \ -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DWITH_CONTRIB=${WITH_CONTRIB:-ON} + -DWITH_CONTRIB=${WITH_CONTRIB:-ON} \ + -DWITH_ANAKIN=ON } function abort(){ From 0ae670917489d24e25e648c85df6a0f8a110f979 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 14 Jun 2018 10:49:07 +0800 Subject: [PATCH 062/154] update document --- python/paddle/fluid/layers/control_flow.py | 16 +++++++++------- .../fluid/layers/learning_rate_scheduler.py | 10 +++++----- python/paddle/fluid/layers/nn.py | 2 ++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index feac42d94e..5354582aaa 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -76,13 +76,13 @@ def split_lod_tensor(input, mask, level=0): Examples: .. code-block:: python - x = layers.data(name='x', shape=[1]) + x = fluid.layers.data(name='x', shape=[1]) x.persistable = True - y = layers.data(name='y', shape=[1]) + y = fluid.layers.data(name='y', shape=[1]) y.persistable = True - out_true, out_false = layers.split_lod_tensor( + out_true, out_false = fluid.layers.split_lod_tensor( input=x, mask=y, level=level) """ @@ -891,7 +891,7 @@ def array_write(x, i, array=None): def create_array(dtype): """ - **Create LoDTensor Array** + **Create LoDTensorArray** This function creates an array of LOD_TENSOR_ARRAY . It is mainly used to implement RNN with array_write, array_read and While. @@ -989,7 +989,8 @@ def array_read(array, i): Returns: Variable: The tensor type variable that has the data written to it. Examples: - .. code-block::python + .. code-block:: python + tmp = fluid.layers.zeros(shape=[10], dtype='int32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) arr = layers.array_read(tmp, i=i) @@ -1027,7 +1028,7 @@ def shrink_memory(x, i, table): def array_length(array): """ - **Get the length of Input LoDTensorArray** + **Get the Length of Input LoDTensorArray** This function performs the operation to find the length of the input LOD_TENSOR_ARRAY. @@ -1042,12 +1043,13 @@ def array_length(array): Variable: The length of the input LoDTensorArray. Examples: - .. code-block::python + .. code-block:: python tmp = fluid.layers.zeros(shape=[10], dtype='int32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) arr = fluid.layers.array_write(tmp, i=i) arr_len = fluid.layers.array_length(arr) + """ helper = LayerHelper('array_length', **locals()) tmp = helper.create_tmp_variable(dtype='int64') diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 2e5cff74c1..2dbc51c23f 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -163,11 +163,11 @@ def polynomial_decay(learning_rate, power=1.0, cycle=False): """ - **polynomial_decay** + **Polynomial Decay** Applies polynomial decay to the initial learning rate. - .. code-block::python + .. code-block:: python if cycle: decay_steps = decay_steps * ceil(global_step / decay_steps) @@ -180,9 +180,9 @@ def polynomial_decay(learning_rate, learning_rate(Variable|float32): A scalar float32 value or a Variable. This will be the initial learning rate during training decay_steps(int32): A Python `int32` number. - end_learning_rate(float): A Python `float` number. - power(float): A Python `float` number - cycle(bool, Default False): Boolean. If set true, decay the learning rate every decay_steps. + end_learning_rate(float, Default: 0.0001): A Python `float` number. + power(float, Default: 1.0): A Python `float` number + cycle(bool, Default: False): Boolean. If set true, decay the learning rate every decay_steps. Returns: The decayed learning rate diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 3f3b7e20ef..7c4393c4d9 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1615,7 +1615,9 @@ def batch_norm(input, Can be used as a normalizer function for conv2d and fully_connected operations. The required data format for this layer is one of the following: + 1. NHWC `[batch, in_height, in_width, in_channels]` + 2. NCHW `[batch, in_channels, in_height, in_width]` Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing From 21ecd357cffb7165813ffa65b2ab7c810eddfece Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 14 Jun 2018 11:01:07 +0800 Subject: [PATCH 063/154] little optimize --- python/paddle/fluid/layers/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 7c4393c4d9..627718f87e 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4099,7 +4099,7 @@ def image_resize(input, name=None, resample='BILINEAR'): """ - **Resize a batch of images** + **Resize a Batch of Images** The input must be a tensor of the shape (num_batches, channels, in_h, in_w), and the resizing only applies on the last two dimensions(hight and width). From 62bf672eddfdbd8c9287292c5ddae80d4eae2af4 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 14 Jun 2018 11:20:46 +0800 Subject: [PATCH 064/154] update document for Switch --- python/paddle/fluid/layers/control_flow.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 5354582aaa..db5b07558a 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1142,16 +1142,19 @@ class Switch(object): The Semantics: 1. A `switch` control-flow checks cases one-by-one. - 1. The condition of each case is a boolean value, which is a scalar. - 1. It runs the first matched case, or the default case if there is one. - 1. Once it matches a case, it runs the corresponding branch and only that branch. + + 2. The condition of each case is a boolean value, which is a scalar. + + 3. It runs the first matched case, or the default case if there is one. + + 4. Once it matches a case, it runs the corresponding branch and only that branch. Examples: .. code-block:: python - with control_flow.Switch() as switch: + with fluid.control_flow.Switch() as switch: with switch.case(global_step == zero_var): - tensor.assign(input=one_var, output=div_res) + fluid.tensor.assign(input=one_var, output=div_res) """ From 046bb5c8cb511589db68ed8769c03566bfa952b7 Mon Sep 17 00:00:00 2001 From: Qiyang Min Date: Wed, 13 Jun 2018 22:30:28 -0500 Subject: [PATCH 065/154] Fix NCCLBcast hang up bug in Parallel Executor (#11377) * 1. Create buddy allocator in each places before NcclBcast the variables 2. Check the memory usage of ALL gpus rather than the first one * 1. Make NCCLGroupGuard guards only the ncclBcast part, which avoid ncclGroupEnd blocking the exception throwing 2. NOTE the usage of NCCLGroupGuard * Remove the memory usage check of gpus * Fix code style --- paddle/fluid/framework/parallel_executor.cc | 20 +++++++++++++++----- paddle/fluid/platform/nccl_helper.h | 5 +++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index ac4d1f58a5..9406c6155d 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -145,9 +145,9 @@ void ParallelExecutor::BCastParamsToGPUs( auto &dims = main_tensor.dims(); if (paddle::platform::is_gpu_place(main_tensor.place())) { #ifdef PADDLE_WITH_CUDA + std::vector buffers; size_t numel = main_tensor.numel(); ncclDataType_t data_type = platform::ToNCCLDataType(main_tensor.type()); - platform::NCCLGroupGuard guard; for (size_t i = 0; i < member_->places_.size(); ++i) { auto place = member_->places_[i]; void *buffer; @@ -159,11 +159,21 @@ void ParallelExecutor::BCastParamsToGPUs( t->Resize(dims); buffer = t->mutable_data(place, main_tensor.type()); } - auto &nccl_ctx = member_->nccl_ctxs_->at(place); - platform::dynload::ncclBcast(buffer, numel, data_type, 0, - nccl_ctx.comm_, nccl_ctx.stream()); + buffers.push_back(buffer); } - member_->nccl_ctxs_->WaitAll(); + + PADDLE_ENFORCE_EQ(member_->places_.size(), buffers.size(), + "variables' buffer size to bcast NOT equal to places"); + { + platform::NCCLGroupGuard guard; + for (size_t i = 0; i < member_->places_.size(); ++i) { + auto &nccl_ctx = member_->nccl_ctxs_->at(member_->places_[i]); + platform::dynload::ncclBcast(buffers[i], numel, data_type, 0, + nccl_ctx.comm_, nccl_ctx.stream()); + } + member_->nccl_ctxs_->WaitAll(); + } + #else PADDLE_THROW("Not compiled with CUDA"); #endif diff --git a/paddle/fluid/platform/nccl_helper.h b/paddle/fluid/platform/nccl_helper.h index 6f8e3f22db..cc46c88fd1 100644 --- a/paddle/fluid/platform/nccl_helper.h +++ b/paddle/fluid/platform/nccl_helper.h @@ -41,6 +41,11 @@ inline ncclDataType_t ToNCCLDataType(std::type_index type) { } } +// NOTE(minqiyang): according to the ncclGroupEnd documentations: +// https://docs.nvidia.com/deeplearning/sdk/nccl-api/ncclapidoc.html, +// ncclGroupEnd will wait for all communicators to be initialized, which will +// cause blocking problem when a runtime_error was thrown, so try only guard +// NCCL actions when use it. class NCCLGroupGuard { public: static std::mutex &NCCLMutex() { From 980499faf15e869f237bfd94171f4aea079865e5 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 14 Jun 2018 13:54:23 +0800 Subject: [PATCH 066/154] fix errors --- paddle/fluid/operators/pool_op.cc | 15 ++++-- python/paddle/fluid/layers/control_flow.py | 23 +++++---- python/paddle/fluid/layers/io.py | 26 ++++++++++ .../fluid/layers/learning_rate_scheduler.py | 4 +- python/paddle/fluid/layers/nn.py | 51 ++++++++++++++----- python/paddle/fluid/layers/tensor.py | 7 +-- 6 files changed, 92 insertions(+), 34 deletions(-) diff --git a/paddle/fluid/operators/pool_op.cc b/paddle/fluid/operators/pool_op.cc index 6707cdded4..d94ddc7a53 100644 --- a/paddle/fluid/operators/pool_op.cc +++ b/paddle/fluid/operators/pool_op.cc @@ -204,8 +204,6 @@ void Pool2dOpMaker::Make() { // TODO(dzhwinter): need to registered layout transform function AddComment(R"DOC( -Pool2d Operator. - The pooling2d operation calculates the output based on the input, pooling_type and ksize, strides, paddings parameters. Input(X) and output(Out) are in NCHW format, where N is batch size, C is the @@ -215,18 +213,27 @@ These two elements represent height and width, respectively. The input(X) size and output(Out) size may be different. Example: + Input: + X shape: $(N, C, H_{in}, W_{in})$ + Output: + Out shape: $(N, C, H_{out}, W_{out})$ + For ceil_mode = false: $$ - H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 \\ + H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 + $$ + $$ W_{out} = \frac{(W_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1 $$ For ceil_mode = true: $$ - H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0] + strides[0] - 1)}{strides[0]} + 1 \\ + H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0] + strides[0] - 1)}{strides[0]} + 1 + $$ + $$ W_{out} = \frac{(W_{in} - ksize[1] + 2 * paddings[1] + strides[1] - 1)}{strides[1]} + 1 $$ diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index f4013b61d0..3ffebb960b 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -753,9 +753,9 @@ def lod_tensor_to_array(x, table): This function split a LoDTesnor to a LoDTensorArray according to its LoD information. LoDTensorArray is an alias of C++ std::vector in - Paddle. The generated LoDTensorArray of this function can be further read - or written by 'read_from_array()' and 'write_to_array()' operators. However, - this function is generally an internal component of Paddle 'DynamicRNN'. + PaddlePaddle. The generated LoDTensorArray of this function can be further read + or written by `read_from_array()` and `write_to_array()` operators. However, + this function is generally an internal component of PaddlePaddle `DynamicRNN`. Users should not use it directly. Args: @@ -763,11 +763,10 @@ def lod_tensor_to_array(x, table): table (ParamAttr|list): The variable that stores the level of lod which is ordered by sequence length in descending order. It is generally generated - by 'layers.lod_rank_table()' API. + by `layers.lod_rank_table()` API. Returns: - Variable: The LoDTensorArray that has been converted from the input - tensor. + Variable: The LoDTensorArray that has been converted from the input tensor. Examples: .. code-block:: python @@ -1579,24 +1578,26 @@ def reorder_lod_tensor_by_rank(x, rank_table): def is_empty(x, cond=None, **ignored): """ - Test whether an Variable is empty. + Test whether a Variable is empty. Args: x (Variable): The Variable to be tested. cond (Variable|None): Output parameter. Returns the test result - of given 'x'. + of given 'x'. Default: None Returns: - Variable: The tensor variable storing the test result of 'x'. + Variable: A bool scalar. True if 'x' is an empty Variable. Raises: TypeError: If input cond is not a variable, or cond's dtype is - not bool + not bool. Examples: .. code-block:: python - less = fluid.layers.is_empty(x=input) + res = fluid.layers.is_empty(x=input) + # or: + fluid.layers.is_empty(x=input, cond=res) """ helper = LayerHelper("is_empty", **locals()) if cond is None: diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 9de88e2c32..6d6cdffe27 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -572,6 +572,32 @@ def parallel(reader): def read_file(file_obj): + """ + Read data from a file object. + + A file object is also a Variable. It can be a raw file object generated by + `fluid.layers.open_files()` or a decorated one generated by + `fluid.layers.double_buffer()` and so on. + + Args: + + file_obj(Variable): The file object from where to read data. + + Returns: + Tuple[Variable]: Data read from the given file object. + + Examples: + .. code-block:: python + + data_file = fluid.layers.open_files( + filenames=['mnist.recordio'], + shapes=[(-1, 748), (-1, 1)], + lod_levels=[0, 0], + dtypes=["float32", "int64"]) + data_file = fluid.layers.double_buffer( + fluid.layers.batch(data_file, batch_size=64)) + input, label = fluid.layers.read_file(data_file) + """ helper = LayerHelper('read_file') out = [ helper.create_tmp_variable( diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 9cbb559093..0efa726331 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -90,7 +90,7 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): Default: False Returns: - The decayed learning rate + Variable: The decayed learning rate Examples: .. code-block:: python @@ -167,7 +167,7 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): Default: False Returns: - The decayed learning rate + Variable: The decayed learning rate Examples: .. code-block:: python diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 3194b72da6..030681999d 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -151,7 +151,7 @@ def fc(input, name (str, default None): The name of this layer. Returns: - A tensor variable storing the transformation result. + Variable: The transformation result. Raises: ValueError: If rank of the input tensor is less than 2. @@ -159,8 +159,7 @@ def fc(input, Examples: .. code-block:: python - data = fluid.layers.data( - name="data", shape=[32, 32], dtype="float32") + data = fluid.layers.data(name="data", shape=[32, 32], dtype="float32") fc = fluid.layers.fc(input=data, size=1000, act="tanh") """ @@ -1543,21 +1542,24 @@ def pool2d(input, ${comment} Args: - input (Variable): ${input_comment} + input (Variable): The input tensor of pooling operator. The format of + input tensor is NCHW, where N is batch size, C is the number of + channels, H is the height of the feature, and W is the width of + the feature. pool_size (int): The side length of pooling windows. All pooling windows are squares with pool_size on a side. - pool_type (str): ${pooling_type_comment} + pool_type: ${pooling_type_comment} pool_stride (int): stride of the pooling layer. pool_padding (int): padding size. - global_pooling (bool): ${global_pooling_comment} - use_cudnn (bool): ${use_cudnn_comment} - ceil_mode (bool): ${ceil_mode_comment} - use_mkldnn (bool): ${use_mkldnn_comment} + global_pooling: ${global_pooling_comment} + use_cudnn: ${use_cudnn_comment} + ceil_mode: ${ceil_mode_comment} + use_mkldnn: ${use_mkldnn_comment} name (str|None): A name for this layer(optional). If set None, the layer will be named automatically. Returns: - Variable: output of pool2d layer. + Variable: The pooling result. Raises: ValueError: If 'pool_type' is not "max" nor "avg" @@ -2764,6 +2766,27 @@ def topk(input, k, name=None): If the input is a Tensor with higher rank, this operator computes the top k entries along the last dimension. + For example: + + .. code-block:: text + + If: + input = [[5, 4, 2, 3], + [9, 7, 10, 25], + [6, 2, 10, 1]] + k = 2 + + Then: + The first output: + values = [[5, 4], + [10, 25], + [6, 10]] + + The second output: + indices = [[0, 1], + [2, 3], + [0, 2]] + Args: input(Variable): The input variable which can be a vector or Tensor with higher rank. @@ -2774,10 +2797,10 @@ def topk(input, k, name=None): Default: None Returns: - values(Variable): The k largest elements along each last dimensional - slice. - indices(Variable): The indices of values within the last dimension of - input. + Tuple[Variable]: A tuple with two elements. Each element is a Variable. + The first one is k largest elements along each last + dimensional slice. The second one is indices of values + within the last dimension of input. Raises: ValueError: If k < 1 or k is not less than the last dimension of input diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index e03c8ca914..392fa6a422 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -159,20 +159,21 @@ def concat(input, axis=0, name=None): def sums(input, out=None): - """This function performs the sum operation on the input and returns the + """ + This function performs the sum operation on the input and returns the result as the output. Args: input (Variable|list): The input tensor that has the elements that need to be summed up. - out (Variable|None): Output parameter. Returns the sum result. + out (Variable|None): Output parameter. The sum result. Default: None Returns: Variable: the sum of input. The same as the argument 'out' Examples: - .. code-block::python + .. code-block:: python tmp = fluid.layers.zeros(shape=[10], dtype='int32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) From 2f9ed97eb66250a788702e21240bc09fea93b85d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 14 Jun 2018 13:54:44 +0800 Subject: [PATCH 067/154] follow comment --- python/paddle/fluid/layers/control_flow.py | 2 -- python/paddle/fluid/layers/nn.py | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index db5b07558a..5394ac3278 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -55,8 +55,6 @@ __all__ = [ def split_lod_tensor(input, mask, level=0): """ - **split_lod_tensor** - This function takes in an input that contains the complete lod information, and takes in a mask which is used to mask certain parts of the input. The output is the true branch and the false branch with the mask applied to diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 627718f87e..d3899cd442 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1638,23 +1638,23 @@ def batch_norm(input, Args: input(variable): The input variable which is a LoDTensor. - act(string, default None): Activation type, linear|relu|prelu|... - is_test(bool, default False): Used for training or training. - momentum(float, default 0.9): - epsilon(float, default 1e-05): + act(string, Default None): Activation type, linear|relu|prelu|... + is_test(bool, Default False): Used for training or training. + momentum(float, Default 0.9): + epsilon(float, Default 1e-05): param_attr(ParamAttr): The parameter attribute for Parameter `scale`. bias_attr(ParamAttr): The parameter attribute for Parameter `bias`. data_layout(string, default NCHW): NCHW|NHWC - in_place(bool, default False): Make the input and output of batch norm reuse memory. + in_place(bool, Default False): Make the input and output of batch norm reuse memory. use_mkldnn(bool, Default false): ${use_mkldnn_comment} name(string, Default None): A name for this layer(optional). If set None, the layer will be named automatically. moving_mean_name(string, Default None): The name of moving_mean which store the global Mean. moving_variance_name(string, Default None): The name of the moving_variance which store the global Variance. - do_model_average_for_mean_and_var(bool, Default False): + do_model_average_for_mean_and_var(bool, Default False): Do model average for mean and variance or not. Returns: - The sequence's last step variable which is a Tensor. + Variable: A tensor variable which is the result after applying batch normalization on the input. Examples: From 3e58df20dfe302fb5796f1c5fee8db94dd8cff40 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 14 Jun 2018 14:11:52 +0800 Subject: [PATCH 068/154] initial with only 1 mkl/openblas threads for each pthreads --- paddle/fluid/framework/init.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paddle/fluid/framework/init.cc b/paddle/fluid/framework/init.cc index 85beae775b..a1094976f6 100644 --- a/paddle/fluid/framework/init.cc +++ b/paddle/fluid/framework/init.cc @@ -18,6 +18,7 @@ limitations under the License. */ #include "paddle/fluid/framework/init.h" #include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/operators/math/blas.h" #include "paddle/fluid/platform/device_context.h" #include "paddle/fluid/platform/place.h" #include "paddle/fluid/string/piece.h" @@ -113,6 +114,9 @@ void InitDevices(bool init_p2p, const std::vector devices) { } places.emplace_back(platform::CPUPlace()); platform::DeviceContextPool::Init(places); +#ifndef PADDLE_WITH_MKLDNN + operators::math::SetNumThreads(1); +#endif } void InitGLOG(const std::string &prog_name) { From 4c0bf774b9f8a33a3ab1b47764e1411ca0dbcf20 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 14 Jun 2018 14:33:23 +0800 Subject: [PATCH 069/154] fix compiler error in contrib/tape --- paddle/contrib/tape/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/contrib/tape/CMakeLists.txt b/paddle/contrib/tape/CMakeLists.txt index 0acef17d6a..470abf25c6 100644 --- a/paddle/contrib/tape/CMakeLists.txt +++ b/paddle/contrib/tape/CMakeLists.txt @@ -17,7 +17,7 @@ if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move") endif(APPLE) -cc_library(tape_variable SRCS variable.cc DEPS ${FLUID_CORE_MODULES}) +cc_library(tape_variable SRCS variable.cc DEPS ${FLUID_CORE_MODULES} device_context) cc_library(tape SRCS tape.cc DEPS ${FLUID_CORE_MODULES} ${GLOB_OP_LIB} tape_variable) cc_test(test_tape From 10e21b77156154d451d618306be756abadf641ab Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 14 Jun 2018 16:32:10 +0800 Subject: [PATCH 070/154] set test_parallel_executor_crf serial --- python/paddle/fluid/tests/unittests/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index ab683bc101..21182393bd 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -41,8 +41,8 @@ function(py_test_modules TARGET_NAME) endfunction() list(REMOVE_ITEM TEST_OPS test_warpctc_op) list(REMOVE_ITEM TEST_OPS test_dist_train) -#list(REMOVE_ITEM TEST_OPS test_parallel_executor_crf) -#list(REMOVE_ITEM TEST_OPS test_parallel_executor_fetch_feed) +list(REMOVE_ITEM TEST_OPS test_parallel_executor_crf) +list(REMOVE_ITEM TEST_OPS test_parallel_executor_fetch_feed) # TODO(wuyi): this test hungs on CI, will add it back later list(REMOVE_ITEM TEST_OPS test_listen_and_serv_op) foreach(TEST_OP ${TEST_OPS}) @@ -50,3 +50,5 @@ foreach(TEST_OP ${TEST_OPS}) endforeach(TEST_OP) py_test_modules(test_warpctc_op MODULES test_warpctc_op ENVS FLAGS_warpctc_dir=${WARPCTC_LIB_DIR} SERIAL) py_test_modules(test_dist_train MODULES test_dist_train SERIAL) +py_test_modules(test_parallel_executor_crf MODULES test_parallel_executor_crf SERIAL) +py_test_modules(test_parallel_executor_fetch_feed MODULES test_parallel_executor_fetch_feed SERIAL) From cbc1b7f1cecc8d5e0e14315dad43410fb0d53f23 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Thu, 14 Jun 2018 16:58:25 +0800 Subject: [PATCH 071/154] Polish documentation --- paddle/fluid/operators/activation_op.cc | 13 ++-- paddle/fluid/operators/compare_op.cc | 11 ++- paddle/fluid/operators/multiplex_op.cc | 4 +- paddle/fluid/operators/row_conv_op.cc | 2 +- python/paddle/fluid/layers/control_flow.py | 31 +++++++-- python/paddle/fluid/layers/detection.py | 81 +++++++++++++--------- python/paddle/fluid/layers/io.py | 43 +++++++++--- python/paddle/fluid/layers/ops.py | 23 +++++- 8 files changed, 144 insertions(+), 64 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 89bb1e2d4f..91a2826947 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -275,7 +275,7 @@ class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker { "The value of threshold for HardShrink. [default: 0.5]") .SetDefault(0.5f); AddComment(R"DOC( -HardShrink Activation Operator. +** HardShrink activation operator ** .. math:: out = \begin{cases} @@ -399,13 +399,12 @@ class ThresholdedReluOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( ThresholdedRelu Activation Operator. -$$ -out = \begin{cases} - x, \text{if } x > threshold \\ - 0, \text{otherwise} - \end{cases} -$$ +.. math:: + out = \begin{cases} + x, \text{if } x > threshold \\ + 0, \text{otherwise} + \end{cases} )DOC"); } }; diff --git a/paddle/fluid/operators/compare_op.cc b/paddle/fluid/operators/compare_op.cc index 11e91c5ec8..f40b1ba338 100644 --- a/paddle/fluid/operators/compare_op.cc +++ b/paddle/fluid/operators/compare_op.cc @@ -34,16 +34,15 @@ class CompareOpProtoMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(true); AddOutput("Out", string::Sprintf("n-dim bool tensor. Each element is %s", comment.equation)); - AddComment(string::Sprintf(R"DOC(%s Operator - + AddComment(string::Sprintf(R"DOC( It operates element-wise on X and Y, and returns the Out. Each of them is a N-dim tensor. X and Y could be any type. The each element of the Out tensor is calculated by $%s$ )DOC", - comment.type, comment.equation)); - AddAttr("axis", - "(int, default -1). The start dimension index " - "for broadcasting Y onto X.") + comment.equation)); + AddAttr( + "axis", + "The start dimension index for broadcasting Y onto X. [default -1]") .SetDefault(-1) .EqualGreaterThan(-1); } diff --git a/paddle/fluid/operators/multiplex_op.cc b/paddle/fluid/operators/multiplex_op.cc index 9db2df2a4c..18ad46cb5e 100644 --- a/paddle/fluid/operators/multiplex_op.cc +++ b/paddle/fluid/operators/multiplex_op.cc @@ -96,7 +96,9 @@ the (Ids[i])-th tensor. For i-th row of the output tensor: -$ y[i] = x_{k}[i] $ +$$ +y[i] = x_{k}[i] +$$ where $y$ is the output tensor, $x_{k}$ is the k-th input tensor, and $k = Ids[i]$. diff --git a/paddle/fluid/operators/row_conv_op.cc b/paddle/fluid/operators/row_conv_op.cc index d7286111fd..52c37e8c91 100644 --- a/paddle/fluid/operators/row_conv_op.cc +++ b/paddle/fluid/operators/row_conv_op.cc @@ -94,7 +94,7 @@ class RowConvOpMaker : public framework::OpProtoAndCheckerMaker { "in this LodTensor is a matrix with shape T x N, i.e., the " "same shape as X."); AddComment(R"DOC( -Row-convolution Operator. +** Row-convolution operator ** The row convolution is called lookahead convolution. This operator was introduced in the following paper for DeepSpeech2: diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 8d28aeb2ed..a618937b11 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1008,8 +1008,28 @@ def array_read(array, i): def shrink_memory(x, i, table): """ - This function creates an operator to shrink_rnn_memory using the RankTable + This function creates an operator to shrink rnn memory using the RankTable as mentioned in the input parameter. + + NOTE: This API is very low-level API. It is used by DynamicRNN only. + + Since the Dynamic RNN uses no-padding way to implement RNN. The sequence + will be sorted by order, and the length of valid memory will be shrink after + each time step. + + Args: + x(Variable): The memory object in the previous time step. + i(Variable): The step count variable. A int scalar as LoDTensor. + table(Variable): The RNNRankTable object. + + Returns: + the memory variable after shrink. + + Examples: + + Since this API is very low level API. The example is not provided. + Please reference the implementation of class DynamicRNN for detail + usage. """ helper = LayerHelper('shrink_memory', **locals()) out = helper.create_tmp_variable(dtype=x.dtype) @@ -1316,10 +1336,9 @@ class IfElse(object): class DynamicRNN(object): """ - Dynamic RNN. - - This RNN can process a batch of sequence data. The length of each sample - sequence can be different. This API automatically process them in batch. + The dynamic RNN can process a batch of sequence data. The length of each + sample sequence can be different. This API automatically process them in + batch. The input lod must be set. Please reference `lod_tensor` @@ -1500,7 +1519,7 @@ class DynamicRNN(object): need_reorder=False, dtype='float32'): """ - Create a memory variable. + Create a memory variable for dynamic rnn. If the :code:`init` is not None, :code:`memory` will be initialized by this variable. The :code:`need_reorder` is used to reorder the memory as diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index d5f7e420dd..edf528a595 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -210,53 +210,68 @@ def bipartite_match(dist_matrix, dist_threshold=None, name=None): """ - **Bipartite matchint operator** - - This operator is a greedy bipartite matching algorithm, which is used to - obtain the matching with the maximum distance based on the input + This operator implements a greedy bipartite matching algorithm, which is + used to obtain the matching with the maximum distance based on the input distance matrix. For input 2D matrix, the bipartite matching algorithm can - find the matched column for each row, also can find the matched row for - each column. And this operator only calculate matched indices from column - to row. For each instance, the number of matched indices is the number of - of columns of the input ditance matrix. - - There are two outputs to save matched indices and distance. - A simple description, this algothrim matched the best (maximum distance) + find the matched column for each row (matched means the largest distance), + also can find the matched row for each column. And this operator only + calculate matched indices from column to row. For each instance, + the number of matched indices is the column number of the input distance + matrix. + + There are two outputs, matched indices and distance. + A simple description, this algorithm matched the best (maximum distance) row entity to the column entity and the matched indices are not duplicated in each row of ColToRowMatchIndices. If the column entity is not matched any row entity, set -1 in ColToRowMatchIndices. - Please note that the input DistMat can be LoDTensor (with LoD) or Tensor. + NOTE: the input DistMat can be LoDTensor (with LoD) or Tensor. If LoDTensor with LoD, the height of ColToRowMatchIndices is batch size. If Tensor, the height of ColToRowMatchIndices is 1. + NOTE: This API is a very low level API. It is used by :code:`ssd_loss` + layer. Please consider to use :code:`ssd_loss` instead. + Args: dist_matrix(Variable): This input is a 2-D LoDTensor with shape [K, M]. It is pair-wise distance matrix between the entities represented by each row and each column. For example, assumed one entity is A with shape [K], another entity is B with shape [M]. The - dist_matirx[i][j] is the distance between A[i] and B[j]. The bigger - the distance is, the better macthing the pairs are. Please note, - This tensor can contain LoD information to represent a batch of - inputs. One instance of this batch can contain different numbers of - entities. + dist_matrix[i][j] is the distance between A[i] and B[j]. The bigger + the distance is, the better matching the pairs are. + + NOTE: This tensor can contain LoD information to represent a batch + of inputs. One instance of this batch can contain different numbers + of entities. match_type(string|None): The type of matching method, should be - 'bipartite' or 'per_prediction', 'bipartite' by defalut. + 'bipartite' or 'per_prediction'. [default 'bipartite']. dist_threshold(float|None): If `match_type` is 'per_prediction', this threshold is to determine the extra matching bboxes based - on the maximum distance, 0.5 by defalut. + on the maximum distance, 0.5 by default. Returns: - match_indices(Variable): A 2-D Tensor with shape [N, M] in int type. - N is the batch size. If match_indices[i][j] is -1, it - means B[j] does not match any entity in i-th instance. - Otherwise, it means B[j] is matched to row - match_indices[i][j] in i-th instance. The row number of - i-th instance is saved in match_indices[i][j]. - match_distance(Variable): A 2-D Tensor with shape [N, M] in float type. - N is batch size. If match_indices[i][j] is -1, - match_distance[i][j] is also -1.0. Otherwise, assumed - match_distance[i][j] = d, and the row offsets of each instance - are called LoD. Then match_distance[i][j] = dist_matrix[d+LoD[i]][j]. + tuple: a tuple with two elements is returned. The first is + matched_indices, the second is matched_distance. + + The matched_indices is a 2-D Tensor with shape [N, M] in int type. + N is the batch size. If match_indices[i][j] is -1, it + means B[j] does not match any entity in i-th instance. + Otherwise, it means B[j] is matched to row + match_indices[i][j] in i-th instance. The row number of + i-th instance is saved in match_indices[i][j]. + + The matched_distance is a 2-D Tensor with shape [N, M] in float type + . N is batch size. If match_indices[i][j] is -1, + match_distance[i][j] is also -1.0. Otherwise, assumed + match_distance[i][j] = d, and the row offsets of each instance + are called LoD. Then match_distance[i][j] = + dist_matrix[d+LoD[i]][j]. + + Examples: + + >>> x = fluid.layers.data(name='x', shape=[4], dtype='float32') + >>> y = fluid.layers.data(name='y', shape=[4], dtype='float32') + >>> iou = fluid.layers.iou_similarity(x=x, y=y) + >>> matched_indices, matched_dist = fluid.layers.bipartite_match(iou) """ helper = LayerHelper('bipartite_match', **locals()) match_indices = helper.create_tmp_variable(dtype='int32') @@ -364,7 +379,7 @@ def ssd_loss(location, normalize=True, sample_size=None): """ - **Multi-box loss layer for object dection algorithm of SSD** + **Multi-box loss layer for object detection algorithm of SSD** This layer is to compute dection loss for SSD given the location offset predictions, confidence predictions, prior boxes and ground-truth boudding @@ -372,7 +387,7 @@ def ssd_loss(location, is a weighted sum of the localization loss (or regression loss) and confidence loss (or classification loss) by performing the following steps: - 1. Find matched boundding box by bipartite matching algorithm. + 1. Find matched bounding box by bipartite matching algorithm. 1.1 Compute IOU similarity between ground-truth boxes and prior boxes. @@ -435,7 +450,7 @@ def ssd_loss(location, mining_type (str): The hard example mining type, should be 'hard_example' or 'max_negative', now only support `max_negative`. normalize (bool): Whether to normalize the SSD loss by the total number - of output locations, True by defalut. + of output locations, True by default. sample_size (int): The max sample size of negative box, used only when mining_type is 'hard_example'. diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 13a3d5441a..150be96d86 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -302,15 +302,6 @@ def open_recordio_file(filename, """ ${comment} - >>> import paddle.fluid as fluid - >>> reader = fluid.layers.io.open_recordio_file( - >>> filename='./data.recordio', - >>> shapes=[(3,224,224), (1)], - >>> lod_levels=[0, 0], - >>> dtypes=['float32', 'int64']) - >>> # Via the reader, we can use 'read_file' layer to get data: - >>> image, label = fluid.layers.io.read_file(reader) - Args: filename(${filename_type}): ${filename_comment}. shapes(list): List of tuples which declaring data shapes. @@ -322,6 +313,17 @@ def open_recordio_file(filename, Returns: ${out_comment}. + + Examples: + + >>> import paddle.fluid as fluid + >>> reader = fluid.layers.io.open_recordio_file( + >>> filename='./data.recordio', + >>> shapes=[(3,224,224), (1)], + >>> lod_levels=[0, 0], + >>> dtypes=['float32', 'int64']) + >>> # Via the reader, we can use 'read_file' layer to get data: + >>> image, label = fluid.layers.io.read_file(reader) """ dtypes = [convert_np_dtype_to_dtype_(dt) for dt in dtypes] shape_concat = [] @@ -549,6 +551,29 @@ def batch(reader, batch_size): def double_buffer(reader, place=None, name=None): + """ + Wrap a double buffer reader. The data will copy to target place with a + double buffer queue. If the target place is None, the place that executor + perform on will be used. + + Args: + reader(Variable): the reader variable need to be wrapped. + place(Place): the place of target data. Default is the sample place of + executor perform. + + name(str): Variable name. None if the user does not care. + + Returns: + wrapped reader with double buffer. + + Examples: + + >>> reader = fluid.layers.open_files(filenames=['somefile'], + >>> shapes=[[-1, 784], [-1, 1]], + >>> dtypes=['float32', 'int64']) + >>> reader = fluid.layers.double_buffer(reader) + >>> img, label = fluid.layers.read_file(reader) + """ attrs = dict() if place is not None: attrs['place'] = str(place).upper() diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index f0abd3089d..486d6f371f 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -66,7 +66,6 @@ __all__ = [ 'uniform_random_batch_size_like', 'gaussian_random', 'gaussian_random_batch_size_like', - 'cumsum', 'scatter', 'sum', 'slice', @@ -120,3 +119,25 @@ Examples: >>> data = fluid.layers.data(name="input", shape=[784]) >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3) """ + +__all__ += ['cumsum'] + +_cum_sum_ = generate_layer_fn('cumsum') + + +def cumsum(x, axis=None, exclusive=None, reverse=None): + kwargs = dict() + for name in locals(): + val = locals()[name] + if val is not None: + kwargs[name] = val + + return _cum_sum_(**kwargs) + + +cumsum.__doc__ = _cum_sum_.__doc__ + """ +Examples: + + >>> data = fluid.layers.data(name="input", shape=[32, 784]) + >>> result = fluid.layers.cumsum(data, axis=0) +""" From 055df47035fe9729f2cca9b1cd14874b1f6fe560 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Thu, 14 Jun 2018 17:31:08 +0800 Subject: [PATCH 072/154] Polish code --- paddle/fluid/operators/activation_op.cc | 9 ++++---- paddle/fluid/operators/row_conv_op.cc | 2 +- python/paddle/fluid/layers/ops.py | 29 ++++++++++++++++++++----- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 91a2826947..c73482eb12 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -275,7 +275,7 @@ class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker { "The value of threshold for HardShrink. [default: 0.5]") .SetDefault(0.5f); AddComment(R"DOC( -** HardShrink activation operator ** +:strong:`HardShrink activation operator` .. math:: out = \begin{cases} @@ -394,15 +394,16 @@ class ThresholdedReluOpMaker : public framework::OpProtoAndCheckerMaker { void Make() override { AddInput("X", "Input of ThresholdedRelu operator"); AddOutput("Out", "Output of ThresholdedRelu operator"); - AddAttr("threshold", "The threshold location of activation") + AddAttr("threshold", + "The threshold location of activation. [default 1.0].") .SetDefault(1.0f); AddComment(R"DOC( -ThresholdedRelu Activation Operator. +:strong:`ThresholdedRelu activation operator` .. math:: out = \begin{cases} - x, \text{if } x > threshold \\ + x, \text{if } x > threshold \\ 0, \text{otherwise} \end{cases} )DOC"); diff --git a/paddle/fluid/operators/row_conv_op.cc b/paddle/fluid/operators/row_conv_op.cc index 52c37e8c91..10b1b0c899 100644 --- a/paddle/fluid/operators/row_conv_op.cc +++ b/paddle/fluid/operators/row_conv_op.cc @@ -94,7 +94,7 @@ class RowConvOpMaker : public framework::OpProtoAndCheckerMaker { "in this LodTensor is a matrix with shape T x N, i.e., the " "same shape as X."); AddComment(R"DOC( -** Row-convolution operator ** +:strong:`Row-convolution operator` The row convolution is called lookahead convolution. This operator was introduced in the following paper for DeepSpeech2: diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 486d6f371f..6f404c5cc6 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -40,7 +40,6 @@ __activations__ = [ 'relu6', 'pow', 'stanh', - 'thresholded_relu', 'hard_sigmoid', 'swish', ] @@ -91,8 +90,7 @@ def uniform_random(shape, dtype=None, min=None, max=None, seed=None): return _uniform_random_(**kwargs) -uniform_random.__doc__ = _uniform_random_.__doc__ + "\n" \ - + """ +uniform_random.__doc__ = _uniform_random_.__doc__ + """ Examples: >>> result = fluid.layers.uniform_random(shape=[32, 784]) @@ -112,8 +110,7 @@ def hard_shrink(x, threshold=None): return _hard_shrink_(**kwargs) -hard_shrink.__doc__ = _hard_shrink_.__doc__ + "\n" \ - + """ +hard_shrink.__doc__ = _hard_shrink_.__doc__ + """ Examples: >>> data = fluid.layers.data(name="input", shape=[784]) @@ -141,3 +138,25 @@ Examples: >>> data = fluid.layers.data(name="input", shape=[32, 784]) >>> result = fluid.layers.cumsum(data, axis=0) """ + +__all__ += ['thresholded_relu'] + +_thresholded_relu_ = generate_layer_fn('thresholded_relu') + + +def thresholded_relu(x, threshold=None): + kwargs = dict() + for name in locals(): + val = locals()[name] + if val is not None: + kwargs[name] = val + + _thresholded_relu_(**kwargs) + + +thresholded_relu.__doc__ = _thresholded_relu_.__doc__ + """ +Examples: + + >>> data = fluid.layers.data(name="input", shape=[1]) + >>> result = fluid.layers.thresholded_relu(data, threshold=0.4) +""" From 44925eb4c2d56ede78c6a1a68aeef57cfbfe03d1 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Thu, 14 Jun 2018 17:37:20 +0800 Subject: [PATCH 073/154] fix dist ut --- paddle/fluid/operators/listen_and_serv_op.cc | 3 +- python/paddle/fluid/layers/io.py | 49 +++++++++---------- .../fluid/tests/unittests/test_dist_train.py | 29 ++++++++--- .../unittests/test_listen_and_serv_op.py | 21 ++++---- 4 files changed, 59 insertions(+), 43 deletions(-) diff --git a/paddle/fluid/operators/listen_and_serv_op.cc b/paddle/fluid/operators/listen_and_serv_op.cc index 4d12278799..57c2ce4577 100644 --- a/paddle/fluid/operators/listen_and_serv_op.cc +++ b/paddle/fluid/operators/listen_and_serv_op.cc @@ -348,7 +348,8 @@ class ListenAndServOpMaker : public framework::OpProtoAndCheckerMaker { }; void SignalHandler::StopAndExit(int signal_num) { - VLOG(3) << "Catch interrupt signal: " << signal_num << ", program will exit"; + // Do not use VLOG here for the device for printing maybe already released. + // exit will release interal allocated resoureces. exit(0); } diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 9de88e2c32..bbb8e2e9cf 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -22,9 +22,9 @@ from ..executor import global_scope from layer_function_generator import generate_layer_fn, templatedoc __all__ = [ - 'data', 'BlockGuardServ', 'ListenAndServ', 'Send', 'open_recordio_file', - 'open_files', 'read_file', 'shuffle', 'batch', 'double_buffer', - 'random_data_generator', 'Preprocessor', 'load' + 'data', 'BlockGuardServ', 'ListenAndServ', 'Send', 'Recv', + 'open_recordio_file', 'open_files', 'read_file', 'shuffle', 'batch', + 'double_buffer', 'random_data_generator', 'Preprocessor', 'load' ] @@ -177,18 +177,17 @@ class ListenAndServ(object): }) -def Send(endpoints, send_vars, get_vars=None): +def Send(endpoints, send_vars, sync=True): """ - Send layer + Send variables to the server side, and get vars from server + side when server have finished running server side program. Args: - endpoints: comma seperated IP:PORT pairs in the order + endpoints (str): comma seperated IP:PORT pairs in the order of send_vars to send - send_vars: vars to send - get_vars: vars to get from server after send completes. - - Send variables to the server side, and get vars from server - side when server have finished running server side program. + send_vars (list): variables to send to server + sync (bool): whether to wait the request finish + """ assert (type(send_vars) == list) @@ -196,40 +195,33 @@ def Send(endpoints, send_vars, get_vars=None): endpoints = list(set(epmap)) helper = LayerHelper("Send", **locals()) - if not get_vars: - get_vars = [] - for s in send_vars: - v = helper.create_tmp_variable(dtype=s.dtype, stop_gradient=True) - get_vars.append(v) rpc_op_role_name = core.op_proto_and_checker_maker.kOpRoleAttrName() helper.append_op( type="send", inputs={"X": send_vars}, - outputs={"Out": get_vars}, attrs={ "endpoints": endpoints, "epmap": epmap, rpc_op_role_name: core.op_proto_and_checker_maker.OpRole.RPC }) + if sync: + helper.append_op(type="send_barrier", attrs={"endpoints": endpoints}) - return get_vars - -def Recv(endpoints, get_vars): +def Recv(endpoints, get_vars, sync=True): """ - Recv layer + Receive variables from server side Args: - endpoints: comma seperated IP:PORT pairs in the order + endpoints (str): comma seperated IP:PORT pairs in the order of send_vars to send - send_vars: vars to send - get_vars: vars to get from server after send completes. + get_vars (list): vars to get from server after send completes. + sync (bool): whether to wait the request finish - Send variables to the server side, and get vars from server - side when server have finished running server side program. + Returns: + list: list of received variables """ - assert (type(send_vars) == list) assert (type(get_vars) == list) epmap = endpoints.split(",") @@ -242,6 +234,9 @@ def Recv(endpoints, get_vars): outputs={"Out": get_vars}, attrs={"endpoints": endpoints, "epmap": epmap}) + if sync: + helper.append_op(type="fetch_barrier", attrs={"endpoints": endpoints}) + return get_vars def monkey_patch_reader_methods(reader): diff --git a/python/paddle/fluid/tests/unittests/test_dist_train.py b/python/paddle/fluid/tests/unittests/test_dist_train.py index 2314bb2ed8..562e66b062 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_train.py @@ -16,6 +16,7 @@ import os import time import unittest from multiprocessing import Process +import signal import numpy @@ -24,9 +25,6 @@ import paddle.fluid.layers as layers class TestSendOp(unittest.TestCase): - @unittest.skip( - "This test is buggy. We cannot use time.sleep to sync processes, the connection may fail in unittest." - ) def test_send(self): # Run init_serv in a thread place = fluid.CPUPlace() @@ -35,7 +33,9 @@ class TestSendOp(unittest.TestCase): p.daemon = True p.start() - time.sleep(10) + self.ps_timeout = 5 + self._wait_ps_ready(p.pid) + with open("/tmp/paddle.%d.port" % p.pid, "r") as fn: selected_port = int(fn.readlines()[0]) self.init_client(place, selected_port) @@ -44,9 +44,23 @@ class TestSendOp(unittest.TestCase): self.assertTrue(numpy.allclose(self.local_out, self.dist_out)) # FIXME(typhoonzero): find a way to gracefully shutdown the server. - os.system("kill -9 %d" % p.pid) + os.kill(p.pid, signal.SIGKILL) p.join() + def _wait_ps_ready(self, pid): + start_left_time = self.ps_timeout + sleep_time = 0.5 + while True: + assert start_left_time >= 0, "wait ps ready failed" + time.sleep(sleep_time) + try: + # the listen_and_serv_op would touch a file which contains the listen port + # on the /tmp directory until it was ready to process all the RPC call. + os.stat("/tmp/paddle.%d.port" % pid) + return + except os.error: + start_left_time -= sleep_time + def init_serv(self, place): main = fluid.Program() @@ -84,7 +98,10 @@ class TestSendOp(unittest.TestCase): dtype="float32", persistable=False, shape=[32, 32]) - o = layers.Send("127.0.0.1:%d" % port, [x], [get_var]) + fluid.initializer.Constant(value=2.3)(get_var, main.global_block()) + layers.Send("127.0.0.1:%d" % port, [x]) + o = layers.Recv("127.0.0.1:%d" % port, [get_var]) + exe = fluid.Executor(place) self.dist_out = exe.run(main, fetch_list=o) # o is a list diff --git a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py index d1d709551c..9dec2acb1d 100644 --- a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py +++ b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py @@ -57,17 +57,18 @@ class TestListenAndServOp(OpTest): def setUp(self): self.ps_timeout = 5 self.ip = "127.0.0.1" - self.port = "6173" + self.port = "0" self.trainers = 1 - self.trainer_id = 1 + self.trainer_id = 0 def _start_pserver(self, use_cuda, sync_mode): p = Process( target=run_pserver, args=(use_cuda, sync_mode, self.ip, self.port, self.trainers, self.trainer_id)) + p.daemon = True p.start() - return p.pid + return p def _wait_ps_ready(self, pid): start_left_time = self.ps_timeout @@ -89,18 +90,20 @@ class TestListenAndServOp(OpTest): def test_handle_signal_in_serv_op(self): # run pserver on CPU in sync mode - pid = self._start_pserver(False, True) - self._wait_ps_ready(pid) + p1 = self._start_pserver(False, True) + self._wait_ps_ready(p1.pid) # raise SIGTERM to pserver - os.kill(pid, signal.SIGTERM) + os.kill(p1.pid, signal.SIGKILL) + p1.join() # run pserver on CPU in async mode - pid = self._start_pserver(False, False) - self._wait_ps_ready(pid) + p2 = self._start_pserver(False, False) + self._wait_ps_ready(p2.pid) # raise SIGTERM to pserver - os.kill(pid, signal.SIGTERM) + os.kill(p2.pid, signal.SIGKILL) + p2.join() if __name__ == '__main__': From ca743de2e09c4e966a7b647d3ce6b304fb61cdb7 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 14 Jun 2018 18:06:55 +0800 Subject: [PATCH 074/154] enable more type for splitOp and ConcatOp --- paddle/fluid/operators/concat_op.cc | 10 ++++++++-- paddle/fluid/operators/concat_op.cu.cc | 10 ++++++++-- paddle/fluid/operators/split_op.cc | 5 ++++- paddle/fluid/operators/split_op.cu.cc | 5 ++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/operators/concat_op.cc b/paddle/fluid/operators/concat_op.cc index 38337f9aa5..c724055937 100644 --- a/paddle/fluid/operators/concat_op.cc +++ b/paddle/fluid/operators/concat_op.cc @@ -107,7 +107,13 @@ REGISTER_OPERATOR(concat, ops::ConcatOp, ops::ConcatOpMaker, false> /* set false to disable empty grad */); REGISTER_OPERATOR(concat_grad, ops::ConcatOpGrad); REGISTER_OP_CPU_KERNEL( - concat, ops::ConcatKernel); + concat, ops::ConcatKernel, + ops::ConcatKernel, + ops::ConcatKernel, + ops::ConcatKernel); REGISTER_OP_CPU_KERNEL( concat_grad, - ops::ConcatGradKernel); + ops::ConcatGradKernel, + ops::ConcatGradKernel, + ops::ConcatGradKernel, + ops::ConcatGradKernel); diff --git a/paddle/fluid/operators/concat_op.cu.cc b/paddle/fluid/operators/concat_op.cu.cc index 590eca9d06..8e38e5231f 100644 --- a/paddle/fluid/operators/concat_op.cu.cc +++ b/paddle/fluid/operators/concat_op.cu.cc @@ -15,7 +15,13 @@ limitations under the License. */ #include "paddle/fluid/operators/concat_op.h" namespace ops = paddle::operators; REGISTER_OP_CUDA_KERNEL( - concat, ops::ConcatKernel); + concat, ops::ConcatKernel, + ops::ConcatKernel, + ops::ConcatKernel, + ops::ConcatKernel); REGISTER_OP_CUDA_KERNEL( concat_grad, - ops::ConcatGradKernel); + ops::ConcatGradKernel, + ops::ConcatGradKernel, + ops::ConcatGradKernel, + ops::ConcatGradKernel); diff --git a/paddle/fluid/operators/split_op.cc b/paddle/fluid/operators/split_op.cc index 5e2b2a9945..d661b276bc 100644 --- a/paddle/fluid/operators/split_op.cc +++ b/paddle/fluid/operators/split_op.cc @@ -115,4 +115,7 @@ USE_CPU_ONLY_OP(concat); REGISTER_OPERATOR(split, ops::SplitOp, ops::SplitOpMaker, ops::SplitGradMaker); REGISTER_OP_CPU_KERNEL(split, - ops::SplitOpKernel); + ops::SplitOpKernel, + ops::SplitOpKernel, + ops::SplitOpKernel, + ops::SplitOpKernel); diff --git a/paddle/fluid/operators/split_op.cu.cc b/paddle/fluid/operators/split_op.cu.cc index efa378af85..18e0904681 100644 --- a/paddle/fluid/operators/split_op.cu.cc +++ b/paddle/fluid/operators/split_op.cu.cc @@ -15,4 +15,7 @@ limitations under the License. */ #include "paddle/fluid/operators/split_op.h" namespace ops = paddle::operators; REGISTER_OP_CUDA_KERNEL( - split, ops::SplitOpKernel); + split, ops::SplitOpKernel, + ops::SplitOpKernel, + ops::SplitOpKernel, + ops::SplitOpKernel); From b3cb536402c8e3cc332c6c6da5de13a28ff78acc Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 14 Jun 2018 11:53:35 +0000 Subject: [PATCH 075/154] Fix doc of relu, log and zeros. --- python/paddle/fluid/layers/nn.py | 130 ++++++++++++++------------- python/paddle/fluid/layers/ops.py | 2 - python/paddle/fluid/layers/tensor.py | 7 +- 3 files changed, 74 insertions(+), 65 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index bd6ed0f30e..97a8af69ea 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -24,66 +24,20 @@ from tensor import concat import utils __all__ = [ - 'fc', - 'embedding', - 'dynamic_lstm', - 'dynamic_lstmp', - 'dynamic_gru', - 'gru_unit', - 'linear_chain_crf', - 'crf_decoding', - 'cos_sim', - 'cross_entropy', - 'square_error_cost', - 'chunk_eval', - 'sequence_conv', - 'conv2d', - 'sequence_pool', - 'sequence_softmax', - 'softmax', - 'pool2d', - 'batch_norm', - 'beam_search_decode', - 'conv2d_transpose', - 'sequence_expand', - 'lstm_unit', - 'reduce_sum', - 'reduce_mean', - 'reduce_max', - 'reduce_min', - 'reduce_prod', - 'sequence_first_step', - 'sequence_last_step', - 'dropout', - 'split', - 'ctc_greedy_decoder', - 'edit_distance', - 'l2_normalize', - 'matmul', - 'topk', - 'warpctc', - 'sequence_reshape', - 'transpose', - 'im2sequence', - 'nce', - 'beam_search', - 'row_conv', - 'multiplex', - 'layer_norm', - 'softmax_with_cross_entropy', - 'smooth_l1', - 'one_hot', - 'autoincreased_step_counter', - 'reshape', - 'lod_reset', - 'lrn', - 'pad', - 'label_smooth', - 'roi_pool', - 'dice_loss', - 'resize_bilinear', - 'gather', - 'random_crop', + 'fc', 'embedding', 'dynamic_lstm', 'dynamic_lstmp', 'dynamic_gru', + 'gru_unit', 'linear_chain_crf', 'crf_decoding', 'cos_sim', 'cross_entropy', + 'square_error_cost', 'chunk_eval', 'sequence_conv', 'conv2d', + 'sequence_pool', 'sequence_softmax', 'softmax', 'pool2d', 'batch_norm', + 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', 'lstm_unit', + 'reduce_sum', 'reduce_mean', 'reduce_max', 'reduce_min', 'reduce_prod', + 'sequence_first_step', 'sequence_last_step', 'dropout', 'split', + 'ctc_greedy_decoder', 'edit_distance', 'l2_normalize', 'matmul', 'topk', + 'warpctc', 'sequence_reshape', 'transpose', 'im2sequence', 'nce', + 'beam_search', 'row_conv', 'multiplex', 'layer_norm', + 'softmax_with_cross_entropy', 'smooth_l1', 'one_hot', + 'autoincreased_step_counter', 'reshape', 'lod_reset', 'lrn', 'pad', + 'label_smooth', 'roi_pool', 'dice_loss', 'resize_bilinear', 'gather', + 'random_crop', 'relu', 'log' ] @@ -4075,3 +4029,59 @@ def random_crop(input, shape, seed=1): "SeedOut": seed_out}, attrs={"shape": shape}) return out + + +def log(x): + """ + Calculates the natural log of the given input tensor, element-wise. + + .. math:: + + Out = \\ln(x) + + Args: + x (Variable): Input tensor. + + Returns: + Variable: The natural log of the input tensor computed element-wise. + + Examples: + + .. code-block:: python + + output = fluid.layers.log(x) + """ + helper = LayerHelper('log', **locals()) + dtype = helper.input_dtype() + out = helper.create_tmp_variable(dtype) + helper.append_op(type="log", inputs={"X": input}, outputs={"Out": out}) + return out + + +def relu(x): + """ + Relu takes one input data (Tensor) and produces one output data (Tensor) + where the rectified linear function, y = max(0, x), is applied to + the tensor elementwise. + + .. math:: + + Out = \\max(0, x) + + Args: + x (Variable): The input tensor. + + Returns: + Variable: The output tensor with the same shape as input. + + Examples: + + .. code-block:: python + + output = fluid.layers.relu(x) + """ + helper = LayerHelper('relu', **locals()) + dtype = helper.input_dtype() + out = helper.create_tmp_variable(dtype) + helper.append_op(type="relu", inputs={"X": input}, outputs={"Out": out}) + return out diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 60f8cbbfa7..7d3c44a389 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -17,7 +17,6 @@ __activations__ = [ 'sigmoid', 'logsigmoid', 'exp', - 'relu', 'tanh', 'tanh_shrink', 'softshrink', @@ -29,7 +28,6 @@ __activations__ = [ 'sin', 'round', 'reciprocal', - 'log', 'square', 'softplus', 'softsign', diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index be34cc81a5..c7e9813c49 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -349,11 +349,12 @@ def zeros(shape, dtype, force_cpu=False): It also sets *stop_gradient* to True. Args: - shape(tuple|list|None): Shape of output tensor - dtype(np.dtype|core.VarDesc.VarType|str): Data type of output tensor + shape(tuple|list|None): Shape of output tensor. + dtype(np.dtype|core.VarDesc.VarType|str): Data type of output tensor. + force_cpu(bool, default False): Whether to make output stay on CPU. Returns: - Variable: The tensor variable storing the output + Variable: The tensor variable storing the output. Examples: .. code-block:: python From 212651a5b8fc9a5625641775e6863f3088b80b9b Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Thu, 14 Jun 2018 19:59:42 +0800 Subject: [PATCH 076/154] bugfix/anakin-ci (#11473) --- paddle/contrib/inference/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/contrib/inference/CMakeLists.txt b/paddle/contrib/inference/CMakeLists.txt index 277b0b175b..0f56d648b1 100644 --- a/paddle/contrib/inference/CMakeLists.txt +++ b/paddle/contrib/inference/CMakeLists.txt @@ -50,7 +50,7 @@ cc_test(test_paddle_inference_api inference_api_test(test_paddle_inference_api_impl ARGS test_word2vec test_image_classification) -if (WITH_ANAKIN) +if (WITH_ANAKIN AND WITH_TESTING) # only needed in CI # Due to Anakin do not have official library releases and the versions of protobuf and cuda do not match Paddle's, # so anakin library will not be merged to our official inference library. To use anakin prediction API, one need to # compile the libinference_anakin_api.a and compile with anakin.so. From cdc06b011f23a559dc2c85c5a3c76fdda381a9ab Mon Sep 17 00:00:00 2001 From: gongweibao Date: Thu, 14 Jun 2018 20:03:50 +0800 Subject: [PATCH 077/154] Fix dependency of `tape/variable.cc` (#11472) --- paddle/contrib/tape/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/contrib/tape/CMakeLists.txt b/paddle/contrib/tape/CMakeLists.txt index 470abf25c6..5450359d85 100644 --- a/paddle/contrib/tape/CMakeLists.txt +++ b/paddle/contrib/tape/CMakeLists.txt @@ -17,7 +17,7 @@ if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move") endif(APPLE) -cc_library(tape_variable SRCS variable.cc DEPS ${FLUID_CORE_MODULES} device_context) +cc_library(tape_variable SRCS variable.cc DEPS ${FLUID_CORE_MODULES} device_context framework_proto proto_desc operator) cc_library(tape SRCS tape.cc DEPS ${FLUID_CORE_MODULES} ${GLOB_OP_LIB} tape_variable) cc_test(test_tape From 297a16986d3ed700c2d02b6a00b2012ab5bdba3b Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 14 Jun 2018 20:14:08 +0800 Subject: [PATCH 078/154] Fix doc of warpctc, array_read, edit_distance and sequence_reshape. --- python/paddle/fluid/layers/control_flow.py | 63 +- python/paddle/fluid/layers/nn.py | 742 ++++++++++++++------- 2 files changed, 522 insertions(+), 283 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index d1ea9f1485..8cd389910c 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -13,7 +13,7 @@ # limitations under the License. import contextlib -from layer_function_generator import autodoc +from layer_function_generator import autodoc, templatedoc from tensor import assign, fill_constant from .. import core from ..framework import Program, Variable, Operator @@ -721,26 +721,22 @@ def lod_rank_table(x, level=0): return table +@templatedoc() def max_sequence_len(rank_table): - """Max Sequence Len Operator. Given a LoDRankTable object, this layer - returns the max length of a batch of sequences. In fact, a LoDRankTable - object contains a list of tuples() and - the list is already sorted by sequence length in descending order, so the - operator just returns the sequence length of the first tuple element. + """ + ${comment} + + >>> import paddle.fluid as fluid + >>> x = fluid.layers.data(name='x', shape=[10], dtype='float32', + >>> lod_level=1) + >>> rank_table = layers.lod_rank_table(x=x, level=0) + >>> max_seq_len = layers.max_sequence_len(rank_table) Args: - rank_table (Variable): Input variable which is a LoDRankTable object. + rank_table(${rank_table_type}): ${rank_table_comment}. Returns: - Variable: The max length of sequence. - - Examples: - .. code-block:: python - - x = fluid.layers.data(name='x', shape=[10], - dtype='float32', lod_level=1) - rank_table = layers.lod_rank_table(x=x, level=0) - max_seq_len = layers.max_sequence_len(rank_table) + ${out_comment}. """ helper = LayerHelper("max_seqence_len", **locals()) res = helper.create_tmp_variable(dtype="int64") @@ -978,19 +974,38 @@ def equal(x, y, cond=None, **ignored): def array_read(array, i): - """This function performs the operation to read the data in as an + """ + This function performs the operation to read the data in as an LOD_TENSOR_ARRAY. + + .. code-block:: text + + Given: + + array = [0.6, 0.1, 0.3, 0.1] + + And: + + i = 2 + + Then: + + output = 0.3 + Args: - array (Variable|list): The input tensor that will be written to an array. - i (Variable|list): The subscript index in tensor array, that points the - place where data will be written to. + array (Variable|list): The input tensor that store data to be read. + i (Variable|list): The index of the data to be read from input array. + Returns: Variable: The tensor type variable that has the data written to it. + Examples: - .. code-block::python - tmp = fluid.layers.zeros(shape=[10], dtype='int32') - i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) - arr = layers.array_read(tmp, i=i) + .. code-block:: python + + tmp = fluid.layers.zeros(shape=[10], dtype='int32') + i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) + arr = layers.array_read(tmp, i=i) + """ helper = LayerHelper('array_read', **locals()) if not isinstance( diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index bd6ed0f30e..eba22f5962 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -12,78 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -All layers just related to the neural network. +All layers just related to the neural network. """ from ..layer_helper import LayerHelper from ..initializer import Normal, Constant from ..framework import Variable from ..param_attr import ParamAttr -from layer_function_generator import autodoc +from layer_function_generator import autodoc, templatedoc from tensor import concat import utils +import random __all__ = [ - 'fc', - 'embedding', - 'dynamic_lstm', - 'dynamic_lstmp', - 'dynamic_gru', - 'gru_unit', - 'linear_chain_crf', - 'crf_decoding', - 'cos_sim', - 'cross_entropy', - 'square_error_cost', - 'chunk_eval', - 'sequence_conv', - 'conv2d', - 'sequence_pool', - 'sequence_softmax', - 'softmax', - 'pool2d', - 'batch_norm', - 'beam_search_decode', - 'conv2d_transpose', - 'sequence_expand', - 'lstm_unit', - 'reduce_sum', - 'reduce_mean', - 'reduce_max', - 'reduce_min', - 'reduce_prod', - 'sequence_first_step', - 'sequence_last_step', - 'dropout', - 'split', - 'ctc_greedy_decoder', - 'edit_distance', - 'l2_normalize', - 'matmul', - 'topk', - 'warpctc', - 'sequence_reshape', - 'transpose', - 'im2sequence', - 'nce', - 'beam_search', - 'row_conv', - 'multiplex', - 'layer_norm', - 'softmax_with_cross_entropy', - 'smooth_l1', - 'one_hot', - 'autoincreased_step_counter', - 'reshape', - 'lod_reset', - 'lrn', - 'pad', - 'label_smooth', - 'roi_pool', - 'dice_loss', - 'resize_bilinear', - 'gather', - 'random_crop', + 'fc', 'embedding', 'dynamic_lstm', 'dynamic_lstmp', 'dynamic_gru', + 'gru_unit', 'linear_chain_crf', 'crf_decoding', 'cos_sim', 'cross_entropy', + 'square_error_cost', 'chunk_eval', 'sequence_conv', 'conv2d', + 'sequence_pool', 'sequence_softmax', 'softmax', 'pool2d', 'batch_norm', + 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', 'lstm_unit', + 'reduce_sum', 'reduce_mean', 'reduce_max', 'reduce_min', 'reduce_prod', + 'sequence_first_step', 'sequence_last_step', 'dropout', 'split', + 'ctc_greedy_decoder', 'edit_distance', 'l2_normalize', 'matmul', 'topk', + 'warpctc', 'sequence_reshape', 'transpose', 'im2sequence', 'nce', + 'beam_search', 'row_conv', 'multiplex', 'layer_norm', + 'softmax_with_cross_entropy', 'smooth_l1', 'one_hot', + 'autoincreased_step_counter', 'reshape', 'lod_reset', 'lrn', 'pad', + 'label_smooth', 'roi_pool', 'dice_loss', 'image_resize', + 'image_resize_short', 'resize_bilinear', 'gather', 'random_crop', 'mean_iou' ] @@ -92,7 +47,6 @@ def fc(input, num_flatten_dims=1, param_attr=None, bias_attr=None, - use_cudnn=False, use_mkldnn=False, act=None, is_test=False, @@ -219,6 +173,7 @@ def embedding(input, have two elements which indicate the size of the dictionary of embeddings and the size of each embedding vector respectively. is_sparse(bool): The flag indicating whether to use sparse update. + is_distributed (bool): Whether to run lookup table from remote parameter server. padding_idx(int|long|None): If :attr:`None`, it makes no effect to lookup. Otherwise the given :attr:`padding_idx` indicates padding the output with zeros whenever lookup encounters it in :attr:`input`. If @@ -258,9 +213,10 @@ def embedding(input, return tmp -# TODO(qijun): expose H0 and C0 def dynamic_lstm(input, size, + h_0=None, + c_0=None, param_attr=None, bias_attr=None, use_peepholes=True, @@ -321,6 +277,13 @@ def dynamic_lstm(input, (T X 4D), where T is the total time steps in this mini-batch, D is the hidden size. size(int): 4 * hidden size. + h_0(Variable): The initial hidden state is an optional input, default is zero. + This is a tensor with shape (N x D), where N is the + batch size and D is the hidden size. + c_0(Variable): The initial cell state is an optional input, default is zero. + This is a tensor with shape (N x D), where N is the + batch size. `h_0` and `c_0` can be NULL but only at the same time. + param_attr(ParamAttr|None): The parameter attribute for the learnable hidden-hidden weights. @@ -384,12 +347,20 @@ def dynamic_lstm(input, cell = helper.create_tmp_variable(dtype) batch_gate = helper.create_tmp_variable(dtype) batch_cell_pre_act = helper.create_tmp_variable(dtype) + inputs = {'Input': input, 'Weight': weight, 'Bias': bias} + batch_size = input.shape[0] + if h_0: + assert h_0.shape == (batch_size, size), \ + 'The shape of h0 should be (batch_size, %d)' % size + inputs['H0'] = h_0 + if c_0: + assert c_0.shape == (batch_size, size), \ + 'The shape of c0 should be (batch_size, %d)' % size + inputs['C0'] = c_0 helper.append_op( type='lstm', - inputs={'Input': input, - 'Weight': weight, - 'Bias': bias}, + inputs=inputs, outputs={ 'Hidden': hidden, 'Cell': cell, @@ -651,8 +622,9 @@ def dynamic_gru(input, :attr:`False`. gate_activation(str): The activation for update gate and reset gate. Choices = ["sigmoid", "tanh", "relu", "identity"], default "sigmoid". - activation(str): The activation for candidate hidden state. + candidate_activation(str): The activation for candidate hidden state. Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". + h_0 (Variable): The hidden output of the first time step. Returns: Variable: The hidden state of GRU. The shape is :math:`(T \\times D)`, \ @@ -673,11 +645,13 @@ def dynamic_gru(input, attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype) bias = helper.create_parameter( attr=helper.bias_attr, shape=[1, 3 * size], dtype=dtype, is_bias=True) + batch_size = input.shape[0] inputs = {'Input': input, 'Weight': weight, 'Bias': bias} if h_0 != None: assert h_0.shape == ( - size, size), 'The shape of h0 should be(%d, %d)' % (size, size) - inputs['h0'] = h_0 + batch_size, size + ), 'The shape of h0 should be(batch_size, %d)' % size + inputs['H0'] = h_0 hidden = helper.create_tmp_variable(dtype) batch_gate = helper.create_tmp_variable(dtype) @@ -799,7 +773,22 @@ def gru_unit(input, return updated_hidden, reset_hidden_pre, gate +@templatedoc() def linear_chain_crf(input, label, param_attr=None): + """ + Linear Chain CRF. + + ${comment} + + Args: + input(${emission_type}): ${emission_comment} + label(${label_type}): ${label_comment} + param_attr(ParamAttr): The attribute of the learnable parameter. + + Returns: + ${log_likelihood_comment} + + """ helper = LayerHelper('linear_chain_crf', **locals()) size = input.shape[1] transition = helper.create_parameter( @@ -825,7 +814,19 @@ def linear_chain_crf(input, label, param_attr=None): return log_likelihood +@templatedoc() def crf_decoding(input, param_attr, label=None): + """ + ${comment} + + Args: + input(${emission_type}): ${emission_comment} + param_attr(ParamAttr): The parameter attribute for training. + label(${label_type}): ${label_comment} + + Returns: + ${viterbi_path_comment} + """ helper = LayerHelper('crf_decoding', **locals()) transition = helper.get_parameter(param_attr.name) viterbi_path = helper.create_tmp_variable(dtype=helper.input_dtype()) @@ -843,6 +844,13 @@ def cos_sim(X, Y): """ This function performs the cosine similarity between two tensors X and Y and returns that as the output. + + Args: + X (Variable): The input X. + Y (Variable): The input Y. + + Returns: + Variable: the output of cosine(X, Y). """ helper = LayerHelper('cos_sim', **locals()) out = helper.create_tmp_variable(dtype=X.dtype) @@ -869,15 +877,15 @@ def dropout(x, dropout_prob, is_test=False, seed=None, name=None): unchanged. Args: - x(variable): The input tensor. - dropout_prob(float): Probability of setting units to zero. - is_test(bool): A flag indicating whether it is in test phrase or not. - seed(int): A Python integer used to create random seeds. If this - parameter is set to None, a random seed is used. - NOTE: If an integer seed is given, always the same output - units will be dropped. DO NOT use a fixed seed in training. - name(str|None): A name for this layer(optional). If set None, the layer - will be named automatically. + x (Variable): The input tensor. + dropout_prob (float): Probability of setting units to zero. + is_test (bool): A flag indicating whether it is in test phrase or not. + seed (int): A Python integer used to create random seeds. If this + parameter is set to None, a random seed is used. + NOTE: If an integer seed is given, always the same output + units will be dropped. DO NOT use a fixed seed in training. + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: A tensor variable. @@ -999,8 +1007,8 @@ def square_error_cost(input, label): * :math:`Out`: Output value, same shape with :math:`X`. Args: - input(Variable): Input tensor, has predictions. - label(Variable): Label tensor, has target labels. + input (Variable): Input tensor, has predictions. + label (Variable): Label tensor, has target labels. Returns: Variable: The tensor variable storing the element-wise squared error \ @@ -1029,6 +1037,7 @@ def square_error_cost(input, label): return square_out +@templatedoc() def chunk_eval(input, label, chunk_scheme, @@ -1037,6 +1046,18 @@ def chunk_eval(input, """ This function computes and outputs the precision, recall and F1-score of chunk detection. + + Args: + input (Variable): prediction output of the network. + label (Variable): label of the test data set. + chunk_scheme (str): ${chunk_scheme_comment} + num_chunk_types (int): ${num_chunk_types_comment} + excluded_chunk_types (list): ${excluded_chunk_types_comment} + + Returns: + tuple: tuple containing: (precision, recall, f1_score, + num_infer_chunks, num_label_chunks, + num_correct_chunks) """ helper = LayerHelper("chunk_eval", **locals()) @@ -1069,6 +1090,7 @@ def chunk_eval(input, num_correct_chunks) +@templatedoc() def sequence_conv(input, num_filters, filter_size=3, @@ -1081,6 +1103,19 @@ def sequence_conv(input, This function creates the op for sequence_conv, using the inputs and other convolutional configurations for the filters and stride as given in the input parameters to the function. + + Args: + input (Variable): ${x_comment} + num_filters (int): number of filters. + filter_size (int): the filter size (H and W). + filter_stride (int): stride of the filter. + padding (bool): if True, add paddings. + bias_attr (ParamAttr|None): attributes for bias + param_attr (ParamAttr|None): attributes for parameter + act (str): the activation type + + Returns: + Variable: output of sequence_conv """ # FIXME(dzh) : want to unify the argument of python layer @@ -1180,48 +1215,49 @@ def conv2d(input, - Input: - Input shape: $(N, C_{in}, H_{in}, W_{in})$ + Input shape: :math:`(N, C_{in}, H_{in}, W_{in})` - Filter shape: $(C_{out}, C_{in}, H_f, W_f)$ + Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)` - Output: - Output shape: $(N, C_{out}, H_{out}, W_{out})$ + Output shape: :math:`(N, C_{out}, H_{out}, W_{out})` Where .. math:: - H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\ - W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1 + H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\ + W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1 Args: - input(Variable): The input image with [N, C, H, W] format. - num_filters(int): The number of filter. It is as same as the output - image channel. - filter_size(int|tuple|None): The filter size. If filter_size is a tuple, - it must contain two integers, (filter_size_H, filter_size_W). - Otherwise, the filter will be a square. - stride(int|tuple): The stride size. If stride is a tuple, it must - contain two integers, (stride_H, stride_W). Otherwise, the - stride_H = stride_W = stride. Default: stride = 1. - padding(int|tuple): The padding size. If padding is a tuple, it must - contain two integers, (padding_H, padding_W). Otherwise, the - padding_H = padding_W = padding. Default: padding = 0. - dilation(int|tuple): The dilation size. If dilation is a tuple, it must - contain two integers, (dilation_H, dilation_W). Otherwise, the - dilation_H = dilation_W = dilation. Default: dilation = 1. - groups(int): The groups number of the Conv2d Layer. According to grouped - convolution in Alex Krizhevsky's Deep CNN paper: when group=2, - the first half of the filters is only connected to the first half - of the input channels, while the second half of the filters is only - connected to the second half of the input channels. Default: groups=1 - param_attr(ParamAttr): The parameters to the Conv2d Layer. Default: None - bias_attr(ParamAttr): Bias parameter for the Conv2d layer. Default: None - use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn - library is installed. Default: True - act(str): Activation type. Default: None - name(str|None): A name for this layer(optional). If set None, the layer - will be named automatically. + input (Variable): The input image with [N, C, H, W] format. + num_filters(int): The number of filter. It is as same as the output + image channel. + filter_size (int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_H, filter_size_W). + Otherwise, the filter will be a square. + stride (int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_H, stride_W). Otherwise, the + stride_H = stride_W = stride. Default: stride = 1. + padding (int|tuple): The padding size. If padding is a tuple, it must + contain two integers, (padding_H, padding_W). Otherwise, the + padding_H = padding_W = padding. Default: padding = 0. + dilation (int|tuple): The dilation size. If dilation is a tuple, it must + contain two integers, (dilation_H, dilation_W). Otherwise, the + dilation_H = dilation_W = dilation. Default: dilation = 1. + groups (int): The groups number of the Conv2d Layer. According to grouped + convolution in Alex Krizhevsky's Deep CNN paper: when group=2, + the first half of the filters is only connected to the first half + of the input channels, while the second half of the filters is only + connected to the second half of the input channels. Default: groups=1 + param_attr (ParamAttr): The parameters to the Conv2d Layer. Default: None + bias_attr (ParamAttr): Bias parameter for the Conv2d layer. Default: None + use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + use_mkldnn (bool): Use mkldnn kernels or not. + act (str): Activation type. Default: None + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: The tensor variable storing the convolution and \ @@ -1379,7 +1415,7 @@ def sequence_pool(input, pool_type): def sequence_first_step(input): """ - This funciton get the first step of sequence. + This function gets the first step of sequence. .. code-block:: text @@ -1412,7 +1448,7 @@ def sequence_first_step(input): def sequence_last_step(input): """ - This funciton get the last step of sequence. + This function gets the last step of sequence. .. code-block:: text @@ -1456,6 +1492,22 @@ def pool2d(input, """ This function adds the operator for pooling in 2 dimensions, using the pooling configurations mentioned in input parameters. + + Args: + input (Variable): ${input_comment} + pool_size (int): ${ksize_comment} + pool_type (str): ${pooling_type_comment} + pool_stride (int): stride of the pooling layer. + pool_padding (int): padding size. + global_pooling (bool): ${global_pooling_comment} + use_cudnn (bool): ${use_cudnn_comment} + ceil_mode (bool): ${ceil_mode_comment} + use_mkldnn (bool): ${use_mkldnn_comment} + name (str): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: output of pool2d layer. """ if pool_type not in ["max", "avg"]: raise ValueError( @@ -1513,6 +1565,25 @@ def batch_norm(input, """ This function helps create an operator to implement the BatchNorm layer using the configurations from the input parameters. + + Args: + input (Variable): the input variable. + act (str): activation type + is_test (bool): whether to run batch_norm as test mode. + momentum (float): momentum + epsilon (float): epsilon, default 1e-05 + param_attr (ParamAttr|None): attributes for parameter + bias_attr (ParamAttr|None): attributes for bias + data_layout (str): data layout, default NCHW + in_place (bool): if True, do not create tmp variable + use_mkldnn (bool): ${use_mkldnn_comment} + name (str): The name of this layer. It is optional. + moving_mean_name (str): The name of moving mean variable name, optional. + moving_variance_name (str): The name of moving variance name, optional. + do_model_average_for_mean_and_var (bool): + + Returns: + Variable: output of batch_norm layer. """ helper = LayerHelper('batch_norm', **locals()) dtype = helper.input_dtype() @@ -1640,6 +1711,7 @@ def layer_norm(input, bias_attr(ParamAttr|None): The parameter attribute for the learnable bias :math:`b`. act(str): Activation to be applied to the output of layer normalizaiton. + name (str): The name of this layer. It is optional. Returns: Variable: A tensor variable with the same shape as the input. @@ -1691,6 +1763,17 @@ def layer_norm(input, def beam_search_decode(ids, scores, name=None): + """ + ${beam_search_decode} + + Args: + ids (Variable): ${ids_comment} + scores (Variable): ${scores_comment} + name (str): The name of this layer. It is optional. + + Returns: + tuple: a tuple of two output variable: sentence_ids, sentence_scores + """ helper = LayerHelper('beam_search_decode', **locals()) sentence_ids = helper.create_tmp_variable(dtype=ids.dtype) sentence_scores = helper.create_tmp_variable(dtype=ids.dtype) @@ -1766,46 +1849,46 @@ def conv2d_transpose(input, W_{out} &= (W_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (W_f - 1) + 1 Args: - input(Variable): The input image with [N, C, H, W] format. - num_filters(int): The number of the filter. It is as same as the output - image channel. - output_size(int|tuple|None): The output image size. If output size is a - tuple, it must contain two integers, (image_H, image_W). This - parameter only works when filter_size is None. - filter_size(int|tuple|None): The filter size. If filter_size is a tuple, - it must contain two integers, (filter_size_H, filter_size_W). - Otherwise, the filter will be a square. None if use output size to - calculate filter_size. - padding(int|tuple): The padding size. If padding is a tuple, it must - contain two integers, (padding_H, padding_W). Otherwise, the - padding_H = padding_W = padding. Default: padding = 0. - stride(int|tuple): The stride size. If stride is a tuple, it must - contain two integers, (stride_H, stride_W). Otherwise, the - stride_H = stride_W = stride. Default: stride = 1. - dilation(int|tuple): The dilation size. If dilation is a tuple, it must - contain two integers, (dilation_H, dilation_W). Otherwise, the - dilation_H = dilation_W = dilation. Default: dilation = 1. - groups(int): The groups number of the Conv2d transpose layer. Inspired by - grouped convolution in Alex Krizhevsky's Deep CNN paper, in which - when group=2, the first half of the filters is only connected to the - first half of the input channels, while the second half of the - filters is only connected to the second half of the input channels. - Default: groups=1 - param_attr(ParamAttr): The parameters to the Conv2d_transpose Layer. - Default: None - bias_attr(ParamAttr): Bias parameter for the Conv2d layer. Default: None - use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn - library is installed. Default: True - act(str): Activation type. Default: None - name(str|None): A name for this layer(optional). If set None, the layer - will be named automatically. + input(Variable): The input image with [N, C, H, W] format. + num_filters(int): The number of the filter. It is as same as the output + image channel. + output_size(int|tuple|None): The output image size. If output size is a + tuple, it must contain two integers, (image_H, image_W). This + parameter only works when filter_size is None. + filter_size(int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_H, filter_size_W). + Otherwise, the filter will be a square. None if use output size to + calculate filter_size. + padding(int|tuple): The padding size. If padding is a tuple, it must + contain two integers, (padding_H, padding_W). Otherwise, the + padding_H = padding_W = padding. Default: padding = 0. + stride(int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_H, stride_W). Otherwise, the + stride_H = stride_W = stride. Default: stride = 1. + dilation(int|tuple): The dilation size. If dilation is a tuple, it must + contain two integers, (dilation_H, dilation_W). Otherwise, the + dilation_H = dilation_W = dilation. Default: dilation = 1. + groups(int): The groups number of the Conv2d transpose layer. Inspired by + grouped convolution in Alex Krizhevsky's Deep CNN paper, in which + when group=2, the first half of the filters is only connected to the + first half of the input channels, while the second half of the + filters is only connected to the second half of the input channels. + Default: groups=1 + param_attr(ParamAttr): The parameters to the Conv2d_transpose Layer. + Default: None + bias_attr(ParamAttr): Bias parameter for the Conv2d layer. Default: None + use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + act(str): Activation type. Default: None + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: - Variable: The tensor variable storing the convolution transpose result. + Variable: The tensor variable storing the convolution transpose result. Raises: - ValueError: If the shapes of input, filter_size, stride, padding and - groups mismatch. + ValueError: If the shapes of input, filter_size, stride, padding and + groups mismatch. Examples: .. code-block:: python @@ -1942,6 +2025,17 @@ def sequence_expand(x, y, ref_level=-1, name=None): def beam_search(pre_ids, ids, scores, beam_size, end_id, level=0): ''' This function implements the beam search algorithm. + + Args: + pre_ids (Variable): ${pre_ids_comment} + ids (Variable): ${ids_comment} + scores (Variable): ${scores_comment} + beam_size (int): ${beam_size_comment} + end_id (int): ${end_id_comment} + level (int): ${level_comment} + + Returns: + tuple: a tuple of beam_search output variables: selected_ids, selected_scores ''' helper = LayerHelper('beam_search', **locals()) score_type = scores.dtype @@ -2437,19 +2531,21 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): The l2 normalize layer normalizes `x` along dimension `axis` using an L2 norm. For a 1-D tensor (`dim` is fixed to 0), this layer computes - output = x / sqrt(max(sum(x**2), epsilon)) + .. math:: + y = \frac{x}{ \sqrt{\sum {x^2} + epsion }} For `x` with more dimensions, this layer independently normalizes each 1-D slice along dimension `axis`. Args: - x(Variable|list): The input tensor to l2_normalize layer. - axis(int): Dimension along which to normalize the input. - epsilon(float): A lower bound value for `x`'s l2 norm. sqrt(epsilon) will - be used as the divisor if the l2 norm of `x` is less than - sqrt(epsilon). - name(str|None): A name for this layer(optional). If set None, the layer - will be named automatically. + x(Variable|list): The input tensor to l2_normalize layer. + axis(int): The axis on which to apply normalization. If `axis < 0`, + the dimension to normalization is rank(X) + axis. -1 is the + last dimension. + epsilon(float): The epsilon value is used to avoid division by zero, + the defalut value is 1e-10. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: @@ -2468,46 +2564,17 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): axis = 0 helper = LayerHelper("l2_normalize", **locals()) - square = helper.create_tmp_variable(dtype=x.dtype) - helper.append_op(type="square", inputs={"X": x}, outputs={"Out": square}) - - reduced_sum = helper.create_tmp_variable(dtype=x.dtype) + out = helper.create_tmp_variable(dtype=x.dtype) + norm = helper.create_tmp_variable(dtype=x.dtype) helper.append_op( - type="reduce_sum", - inputs={"X": square}, - outputs={"Out": reduced_sum}, + type="norm", + inputs={"X": x}, + outputs={"Out": out, + "Norm": norm}, attrs={ - "dim": [1] if axis is None else [axis], - "keep_dim": True, - "reduce_all": False + "axis": 1 if axis is None else axis, + "epsilon": epsilon, }) - - # TODO(caoying) A lower bound value epsilon for the norm is needed to - # imporve the numeric stability of reciprocal. This requires a maximum_op. - rsquare = helper.create_tmp_variable(dtype=x.dtype) - helper.append_op( - type="reciprocal", inputs={"X": reduced_sum}, outputs={"Out": rsquare}) - - # TODO(caoying) the current elementwise_mul operator does not support a - # general broadcast rule which broadcasts input(Y) to have the same - # dimension with Input(X) starting from a specified dimension. So this - # exanpsion is requred. Once a general broadcast rule is spported, this - # expanding canbe removed. - rsquare_expanded = helper.create_tmp_variable(dtype=x.dtype) - expand_times = [1] * len(x.shape) - expand_times[axis] = int(x.shape[axis]) - helper.append_op( - type="expand", - inputs={"X": rsquare}, - outputs={"Out": rsquare_expanded}, - attrs={"expand_times": expand_times}) - - out = helper.create_tmp_variable(dtype=x.dtype) - helper.append_op( - type="elementwise_mul", - inputs={"X": x, - "Y": rsquare_expanded}, - outputs={"Out": out}) return out @@ -2666,8 +2733,7 @@ def topk(input, k, name=None): return values, indices -def edit_distance(input, label, normalized=True, ignored_tokens=None, - name=None): +def edit_distance(input, label, normalized=True, ignored_tokens=None): """ EditDistance operator computes the edit distances between a batch of hypothesis strings and their references. Edit distance, also called @@ -2681,26 +2747,23 @@ def edit_distance(input, label, normalized=True, ignored_tokens=None, "kitten" -> "sitten" -> "sittin" -> "sitting" - Input(Hyps) is a LoDTensor consisting of all the hypothesis strings with + The input is a LoDTensor consisting of all the hypothesis strings with the total number denoted by `batch_size`, and the separation is specified by the LoD information. And the `batch_size` reference strings are arranged - in order in the same way in the LoDTensor Input(Refs). + in order in the same way in the input LoDTensor. - Output(Out) contains the `batch_size` results and each stands for the edit + The output contains the `batch_size` results and each stands for the edit distance for a pair of strings respectively. If Attr(normalized) is true, the edit distance will be divided by the length of reference string. Args: - input(Variable): The indices for hypothesis strings. - label(Variable): The indices for reference strings. - - normalized(bool): Indicated whether to normalize the edit distance by + normalized(bool, default True): Indicated whether to normalize the edit distance by the length of reference string. - - ignored_tokens(list of int): Tokens that should be removed before + ignored_tokens(list, default None): Tokens that should be removed before calculating edit distance. + name (str): The name of this layer. It is optional. Returns: Variable: sequence-to-sequence edit distance in shape [batch_size, 1]. @@ -2710,7 +2773,6 @@ def edit_distance(input, label, normalized=True, ignored_tokens=None, x = fluid.layers.data(name='x', shape=[8], dtype='float32') y = fluid.layers.data(name='y', shape=[7], dtype='float32') - cost = fluid.layers.edit_distance(input=x,label=y) """ helper = LayerHelper("edit_distance", **locals()) @@ -2790,10 +2852,10 @@ def ctc_greedy_decoder(input, blank, name=None): where Lp is the sum of all input sequences' length and num_classes is the true number of classes. (not including the blank label). - blank(int): the blank label index of Connectionist Temporal Classification (CTC) loss, which is in thehalf-opened interval [0, num_classes + 1). + name (str): The name of this layer. It is optional. Returns: Variable: CTC greedy decode result. If all the sequences in result were @@ -2830,35 +2892,33 @@ def warpctc(input, label, blank=0, norm_by_times=False): input tensor. Args: - input(Variable): (LodTensor, default: LoDTensor), - the unscaled probabilities of variable-length sequences, + input (Variable): The unscaled probabilities of variable-length sequences, which is a 2-D Tensor with LoD information. It's shape is [Lp, num_classes + 1], where Lp is the sum of all input sequences' length and num_classes is the true number of classes. (not including the blank label). - label(Variable): (LodTensor, default: LoDTensor), the ground truth - of variable-length sequence, which is a 2-D Tensor with LoD - information. It is of the shape [Lg, 1], where Lg is th sum of - all labels' length. - blank: (int, default: 0), the blank label index of Connectionist + label (Variable): The ground truth of variable-length sequence, + which is a 2-D Tensor with LoD information. It is of the shape [Lg, 1], + where Lg is th sum of all labels' length. + blank (int, default 0): The blank label index of Connectionist Temporal Classification (CTC) loss, which is in the half-opened interval [0, num_classes + 1). - norm_by_times: (bool, default: false), whether to normalize - the gradients by the number of time-step, which is also the - sequence's length. There is no need to normalize the gradients - if warpctc layer was follewed by a mean_op. + norm_by_times(bool, default false): Whether to normalize the gradients + by the number of time-step, which is also the sequence's length. + There is no need to normalize the gradients if warpctc layer was + follewed by a mean_op. Returns: Variable: The Connectionist Temporal Classification (CTC) loss, which is a 2-D Tensor of the shape [batch_size, 1]. Examples: + .. code-block:: python - y = layers.data( - name='y', shape=[11, 8], dtype='float32', lod_level=1) - y_predict = layers.data( - name='y_predict', shape=[11, 1], dtype='float32') - cost = layers.warpctc(input=y_predict, label=y) + + label = layers.data(shape=[11, 8], dtype='float32', lod_level=1) + predict = layers.data(shape=[11, 1], dtype='float32') + cost = layers.warpctc(input=predict, label=label) """ helper = LayerHelper('warpctc', **locals()) @@ -2888,16 +2948,21 @@ def sequence_reshape(input, new_dim): x is a LoDTensor: x.lod = [[0, 2, 6]] - x.data = [[1, 2], [3, 4], - [5, 6], [7, 8], [9, 10], [11, 12]] + x.data = [[1, 2], [3, 4], + [5, 6], [7, 8], + [9, 10], [11, 12]] x.dims = [6, 2] set new_dim = 4 then out is a LoDTensor: + out.lod = [[0, 1, 3]] - out.data = [[1, 2, 3, 4], - [5, 6, 7, 8], [9, 10, 11, 12]] + + out.data = [[1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12]] + out.dims = [3, 4] Currently, only 1-level LoDTensor is supported and please make sure @@ -2905,18 +2970,18 @@ def sequence_reshape(input, new_dim): no remainder for each sequence. Args: - input (Variable): (LodTensor, default: LoDTensor), a 2-D LoDTensor - with shape being [N, M] where M for dimension. - new_dim (int): New dimension which the input LoDTensor is reshaped to. + + input (Variable): A 2-D LoDTensor with shape being [N, M] where M for dimension. + new_dim (int): New dimension that the input LoDTensor is reshaped to. Returns: + Variable: Reshaped LoDTensor according to new dimension. Examples: .. code-block:: python - x = fluid.layers.data(name='x', shape=[5, 20], - dtype='float32', lod_level=1) + x = fluid.layers.data(shape=[5, 20], dtype='float32', lod_level=1) x_reshaped = layers.sequence_reshape(input=x, new_dim=10) """ helper = LayerHelper('sequence_reshape', **locals()) @@ -2929,7 +2994,10 @@ def sequence_reshape(input, new_dim): return out -@autodoc() +# FIXME(wuyi): let docstring_checker.py understand @autodoc. +# For now, the comments in c++ use types like Tensor, but in python side +# the type is often "Variable", and arguments may vary. +@templatedoc(op_type="nce") def nce(input, label, num_total_classes, @@ -2937,6 +3005,21 @@ def nce(input, param_attr=None, bias_attr=None, num_neg_samples=None): + """ + ${comment} + + Args: + input (Variable): input variable. + label (Variable): label. + num_total_classes (int):${num_total_classes_comment} + sample_weight (int): ${sample_weight_comment} + param_attr (ParamAttr|None): attributes for parameter + bias_attr (ParamAttr|None): attributes for bias + num_neg_samples (int): ${num_neg_samples_comment} + + Returns: + Variable: output of nce layer. + """ helper = LayerHelper('nce', **locals()) assert isinstance(input, Variable) dim = input.shape[1] @@ -2994,8 +3077,9 @@ def transpose(x, perm, name=None): perm[i]-th dimension of `input`. Args: - input (Variable): (Tensor), A Tensor. - perm (list): A permutation of the dimensions of `input`. + x (Variable): The input Tensor. + perm (list): A permutation of the dimensions of `input`. + name (str): The name of this layer. It is optional. Returns: Variable: A transposed Tensor. @@ -3228,9 +3312,9 @@ def multiplex(inputs, index): row of the matrix, then `O[i]` is equal to :math:`I_{ID[i]}[i]`. Args: - inputs (list): A list of variables to gather from. All variables have the + inputs (list): A list of variables to gather from. All variables have the same shape and the rank is at least 2. - index (Variable): Tensor, index variable which is a 2-D tensor + index (Variable): Tensor, index variable which is a 2-D tensor with shape [M, 1] where M is the batch size. Returns: @@ -3429,7 +3513,8 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): begin(int): The first value of this counter. step(int): The increment step between each execution. - Returns(Variable): The global run counter. + Returns: + Variable: The global run counter. """ helper = LayerHelper('global_step_counter') if counter_name is None: @@ -3490,7 +3575,7 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=True, name=None): the corresponding dimension of x. Args: - input(variable): The input tensor. + x(variable): The input tensor. shape(list): The new shape. At most one dimension of the new shape can be -1. actual_shape(variable): An optional input. If provided, reshape @@ -3502,8 +3587,10 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=True, name=None): inplace(bool): If this flag is set true, a new output tensor is created whose data is copied from input x, otherwise the output shares data with input without copying. + name (str): The name of this layer. It is optional. - Returns(variable): The output tensor. + Returns: + Variable: The output tensor. Examples: .. code-block:: python @@ -3929,22 +4016,25 @@ def dice_loss(input, label, epsilon=0.00001): return reduce_mean(dice_score) -def resize_bilinear(input, out_shape=None, scale=None, name=None): +def image_resize(input, + out_shape=None, + scale=None, + name=None, + resample='BILINEAR'): """ - The mathematical meaning of resize bilinear layer is - Bilinear interpolation. - Bilinear interpolation is an extension of linear interpolation for - interpolating functions of two variables (e.g. H-direction and - W-direction in this layer) on a rectilinear 2D grid. + Resize a batch of images. - For details, please refer to Wikipedia: - https://en.wikipedia.org/wiki/Bilinear_interpolation + The input must be a tensor of the shape (num_batches, channels, in_h, in_w), + and the resizing only applies on the last two dimensions(hight and width). + + Supporting resample methods: + 'BILINEAR' : Bilinear interpolation Args: - input (Variable): The input tensor of resize bilinear layer, + input (Variable): The input tensor of image resize layer, This is a 4-D tensor of the shape (num_batches, channels, in_h, in_w). - out_shape(list|tuple|Variable|None): Output shape of resize bilinear + out_shape(list|tuple|Variable|None): Output shape of image resize layer, the shape is (out_h, out_w). Default: None scale(float|None): The multiplier for the input height or width. @@ -3953,6 +4043,8 @@ def resize_bilinear(input, out_shape=None, scale=None, name=None): Default: None name(str|None): A name for this layer(optional). If set None, the layer will be named automatically. + resample(str): The resample method. It can only be 'BILINEAR' currently. + Default: 'BILINEAR' Returns: out (Variable): The output is a 4-D tensor of the shape @@ -3961,8 +4053,12 @@ def resize_bilinear(input, out_shape=None, scale=None, name=None): Examples: .. code-block:: python - out = fluid.layers.resize_bilinear(input, out_shape=[12, 12]) + out = fluid.layers.image_resize(input, out_shape=[12, 12]) """ + resample_methods = {'BILINEAR': 'bilinear_interp'} + if resample not in resample_methods: + raise ValueError( + "The 'resample' of image_resize can only be 'BILINEAR' currently.") if out_shape is None and scale is None: raise ValueError("One of out_shape and scale must not be None") helper = LayerHelper('bilinear_interp', **locals()) @@ -3990,7 +4086,7 @@ def resize_bilinear(input, out_shape=None, scale=None, name=None): out = helper.create_tmp_variable(dtype) helper.append_op( - type="bilinear_interp", + type=resample_methods[resample], inputs=inputs, outputs={"Out": out}, attrs={"out_h": out_h, @@ -3998,6 +4094,62 @@ def resize_bilinear(input, out_shape=None, scale=None, name=None): return out +@templatedoc(op_type="bilinear_interp") +def resize_bilinear(input, out_shape=None, scale=None, name=None): + """ + ${comment} + + Args: + input(${x_type}): ${x_comment}. + + out_shape(${out_size_type}): ${out_size_comment}. + + scale(float|None): The multiplier for the input height or width. At + least one of out_shape or scale must be set. And out_shape has + a higher priority than scale. Default: None. + + name(str|None): The output variable name. + + Returns: + ${out_comment}. + """ + + return image_resize(input, out_shape, scale, name, 'BILINEAR') + + +def image_resize_short(input, out_short_len, resample='BILINEAR'): + """ + Resize a batch of images. The short edge of input images will be + resized to the given 'out_short_len'. The long edge of input images + will be resized proportionately to make images' length-width ratio + constant. + + Args: + input (Variable): The input tensor of image resize layer, + This is a 4-D tensor of the shape + (num_batches, channels, in_h, in_w). + out_short_len(int): The length of output images' short edge. + resample (str): resample method, default: BILINEAR. + + Returns: + out (Variable): The output is a 4-D tensor of the shape + (num_batches, channls, out_h, out_w). + """ + in_shape = input.shape + if len(in_shape) != 4: + raise ValueError( + "The rank of input must be 4 (num_batches, channels, in_h, in_w).") + hw = in_shape[2:4] + short_idx = hw.index(min(hw)) + long_idx = 1 - short_idx + out_shape = list(hw) + out_shape[short_idx] = out_short_len + out_shape[long_idx] = int( + float(out_shape[long_idx]) * (float(out_short_len) / float(hw[ + short_idx])) + 0.5) + return image_resize(input=input, out_shape=out_shape, resample=resample) + + def gather(input, index): """ Output is obtained by gathering entries of the outer-most dimension @@ -4005,7 +4157,7 @@ def gather(input, index): .. math:: - Out = X[Index] + Out = X[Index] .. code-block:: text @@ -4013,8 +4165,8 @@ def gather(input, index): Given: - X = [[1, 2], - [3, 4], + X = [[1, 2], + [3, 4], [5, 6]] Index = [1, 2] @@ -4032,6 +4184,7 @@ def gather(input, index): output (Variable): The output is a tensor with the same rank as input. Examples: + .. code-block:: python output = fluid.layers.gather(x, index) @@ -4047,10 +4200,31 @@ def gather(input, index): return out -def random_crop(input, shape, seed=1): +@templatedoc() +def random_crop(x, shape, seed=None): + """ + ${comment} + + Examples: + >>> img = fluid.layers.data("img", [3, 256, 256]) + >>> cropped_img = fluid.layers.random_crop(img, shape=[3, 224, 224]) + + Args: + x(${x_type}): ${x_comment} + shape(${shape_type}): ${shape_comment} + seed(int|${seed_type}|None): ${seed_comment} By default, the seed will + get from `random.randint(-65536, 65535)`. + + Returns: + ${out_comment} + + """ helper = LayerHelper("random_crop", **locals()) dtype = helper.input_dtype() out = helper.create_tmp_variable(dtype) + if seed is None: + seed = random.randint(-65536, 65535) + if isinstance(seed, int): seed_value = seed seed = helper.create_tmp_variable(dtype="int64") @@ -4069,9 +4243,59 @@ def random_crop(input, shape, seed=1): seed_out = helper.create_tmp_variable(dtype="int64") helper.append_op( type="random_crop", - inputs={"X": input, + inputs={"X": x, "Seed": seed}, outputs={"Out": out, "SeedOut": seed_out}, attrs={"shape": shape}) return out + + +def mean_iou(input, label, num_classes): + """ + Mean Intersection-Over-Union is a common evaluation metric for + semantic image segmentation, which first computes the IOU for each + semantic class and then computes the average over classes. + IOU is defined as follows: + + .. math:: + + IOU = true_positive / (true_positive + false_positive + false_negative). + + The predictions are accumulated in a confusion matrix and mean-IOU + is then calculated from it. + + + Args: + input (Variable): A Tensor of prediction results for semantic labels with type int32 or int64. + label (Variable): A Tensor of ground truth labels with type int32 or int64. + Its shape should be the same as input. + + Returns: + mean_iou (Variable): A Tensor representing the mean intersection-over-union with shape [1]. + out_wrong(Variable): A Tensor with shape [num_classes]. The wrong numbers of each class. + out_correct(Variable): A Tensor with shape [num_classes]. The correct numbers of each class. + + + Examples: + + .. code-block:: python + + iou, wrongs, corrects = fluid.layers.mean_iou(predict, label, num_classes) + """ + helper = LayerHelper('mean_iou', **locals()) + dtype = helper.input_dtype() + out_mean_iou = helper.create_tmp_variable(dtype='float32') + out_wrong = helper.create_tmp_variable(dtype='int32') + out_correct = helper.create_tmp_variable(dtype='int32') + helper.append_op( + type="mean_iou", + inputs={"predictions": input, + "labels": label}, + outputs={ + "out_mean_iou": out_mean_iou, + "out_wrong": out_wrong, + "out_correct": out_correct + }, + attrs={"num_classes": num_classes}) + return out_mean_iou, out_wrong, out_correct From a6e69d3cc12c922b9cfc13b653fa845050390e68 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 14 Jun 2018 20:40:02 +0800 Subject: [PATCH 079/154] Add doc for 'batch' --- python/paddle/fluid/layers/io.py | 45 ++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 6d6cdffe27..e22257ea34 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -549,6 +549,41 @@ def shuffle(reader, buffer_size): def batch(reader, batch_size): + """ + This layer is a reader decorator. It takes a reader and adds + 'batching' decoration on it. When reading with the result + decorated reader, output data will be automatically organized + to the form of batches. + + Args: + reader(Variable): The reader to be decorated with 'batching'. + batch_size(int): The batch size. + + Returns: + Variable: The reader which has been decorated with 'batching'. + + Examples: + .. code-block:: python + + raw_reader = fluid.layers.io.open_files(filenames=['./data1.recordio', + './data2.recordio'], + shapes=[(3,224,224), (1)], + lod_levels=[0, 0], + dtypes=['float32', 'int64'], + thread_num=2, + buffer_size=2) + batch_reader = fluid.layers.batch(reader=raw_reader, batch_size=5) + + # If we read data with the raw_reader: + # data = fluid.layers.read_file(raw_reader) + # We can only get data instance by instance. + # + # However, if we read data with the batch_reader: + # data = fluid.layers.read_file(batch_reader) + # Each 5 adjacent instances will be automatically combined together + # to become a batch. So what we get('data') is a batch data instead + # of an instance. + """ return __create_unshared_decorated_reader__( 'create_batch_reader', reader, {'batch_size': int(batch_size)}) @@ -571,20 +606,20 @@ def parallel(reader): {}) -def read_file(file_obj): +def read_file(reader): """ - Read data from a file object. + Execute the given reader and get data via it. - A file object is also a Variable. It can be a raw file object generated by + A reader is also a Variable. It can be a raw reader generated by `fluid.layers.open_files()` or a decorated one generated by `fluid.layers.double_buffer()` and so on. Args: - file_obj(Variable): The file object from where to read data. + reader(Variable): The reader to execute. Returns: - Tuple[Variable]: Data read from the given file object. + Tuple[Variable]: Data read via the given reader. Examples: .. code-block:: python From fbbac505104225329ff2953116acfe1db50d6eac Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Jun 2018 06:07:31 -0700 Subject: [PATCH 080/154] Fix typos and format problems in smooth_l1's doc --- python/paddle/fluid/layers/nn.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 2c1f988828..ed2e1811f6 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3411,31 +3411,30 @@ def softmax_with_cross_entropy(logits, label, soft_label=False): def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): """ - **Smooth L1 Loss Operator. ** - - This operator computes the smooth L1 loss for X and Y. - The operator takes the first dimension of X and Y as batch size. + This layer computes the smooth L1 loss for Variable `x` and `y`. + It takes the first dimension of `x` and `y` as batch size. For each instance, it computes the smooth L1 loss element by element first - and then sums all the losses. So the shape of Out is [batch_size, 1]. + and then sums all the losses. So the shape of ouput Variable is + [batch_size, 1]. Args: x (Variable): A tensor with rank at least 2. The input value of smooth L1 loss op with shape [batch_size, dim1, ..., dimN]. y (Variable): A tensor with rank at least 2. The target value of smooth - L1 loss op with same shape as x. + L1 loss op with same shape as `x`. inside_weight (Variable|None): A tensor with rank at least 2. This - input is optional and should have same shape with x. If provided, - the result of (x - y) will be multiplied by this tensor element by + input is optional and should have same shape with `x`. If provided, + the result of (`x - y`) will be multiplied by this tensor element by element. outside_weight (Variable|None): A tensor with rank at least 2. This input is optional and should have same shape with x. If provided, the out smooth L1 loss will be multiplied by this tensor element by element. - sigma (float|None): Hyper parameter of smooth L1 loss op. A float scalar - with default value 1.0. + sigma (float|None): Hyper parameter of smooth L1 loss layer. A float + scalar with default value 1.0. + Returns: - Variable: A tensor with rank be 2. The output smooth L1 loss with - shape [batch_size, 1]. + Variable: The output smooth L1 loss with shape [batch_size, 1]. Examples: .. code-block:: python @@ -3446,6 +3445,7 @@ def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): fc = fluid.layers.fc(input=data, size=100) out = fluid.layers.smooth_l1(x=fc, y=label) """ + helper = LayerHelper('smooth_l1_loss', **locals()) diff = helper.create_tmp_variable(dtype=x.dtype) loss = helper.create_tmp_variable(dtype=x.dtype) From 16a3d88a20b4c8e029efb837e6523ac651722003 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 06:25:33 -0700 Subject: [PATCH 081/154] fix typo --- paddle/fluid/operators/clip_by_norm_op.cc | 11 ++++++++++- python/paddle/fluid/layers/control_flow.py | 1 + python/paddle/fluid/layers/nn.py | 6 ++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/operators/clip_by_norm_op.cc b/paddle/fluid/operators/clip_by_norm_op.cc index c87bded034..eae86a373b 100644 --- a/paddle/fluid/operators/clip_by_norm_op.cc +++ b/paddle/fluid/operators/clip_by_norm_op.cc @@ -54,10 +54,19 @@ be linearly scaled to make the L2 norm of $Out$ equal to $max\_norm$, as shown in the following formula: $$ -Out = \frac{max\_norm * X}{norm(X)}, +Out = \\frac{max\\_norm * X}{norm(X)}, $$ where $norm(X)$ represents the L2 norm of $X$. + +Examples: + .. code-block:: python + + data = fluid.layer.data( + name='data', shape=[2, 4, 6], dtype='float32') + reshaped = fluid.layers.clip_by_norm( + x=data, max_norm=0.5) + )DOC"); } }; diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..be4dd41577 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -866,6 +866,7 @@ def array_write(x, i, array=None): Variable: The output LOD_TENSOR_ARRAY where the input tensor is written. Examples: + .. code-block::python tmp = fluid.layers.zeros(shape=[10], dtype='int32') diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 2c1f988828..2c7e04c1e6 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3159,8 +3159,6 @@ def im2sequence(input, filter_size=1, stride=1, padding=0, name=None): Examples: - As an example: - .. code-block:: text Given: @@ -3204,7 +3202,7 @@ def im2sequence(input, filter_size=1, stride=1, padding=0, name=None): output.lod = [[0, 4, 8]] - The simple usage is: + Examples: .. code-block:: python @@ -3738,7 +3736,7 @@ def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): Output(i, x, y) = Input(i, x, y) / \left( k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} - (Input(j, x, y))^2 \right)^{\beta} + (Input(j, x, y))^2\right)^{\beta} In the above equation: From bff4cec3b3ab9c1b4cf5086d006def15cc0eaa82 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Jun 2018 06:36:54 -0700 Subject: [PATCH 082/154] Format lod_reset's doc --- python/paddle/fluid/layers/nn.py | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index ed2e1811f6..378a1c33c6 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3411,8 +3411,8 @@ def softmax_with_cross_entropy(logits, label, soft_label=False): def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): """ - This layer computes the smooth L1 loss for Variable `x` and `y`. - It takes the first dimension of `x` and `y` as batch size. + This layer computes the smooth L1 loss for Variable :attr:`x` and :attr:`y`. + It takes the first dimension of :attr:`x` and :attr:`y` as batch size. For each instance, it computes the smooth L1 loss element by element first and then sums all the losses. So the shape of ouput Variable is [batch_size, 1]. @@ -3421,15 +3421,15 @@ def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): x (Variable): A tensor with rank at least 2. The input value of smooth L1 loss op with shape [batch_size, dim1, ..., dimN]. y (Variable): A tensor with rank at least 2. The target value of smooth - L1 loss op with same shape as `x`. + L1 loss op with same shape as :attr:`x`. inside_weight (Variable|None): A tensor with rank at least 2. This - input is optional and should have same shape with `x`. If provided, - the result of (`x - y`) will be multiplied by this tensor element by - element. + input is optional and should have same shape with :attr:`x`. If + provided, the result of (:attr:`x` - :attr:`y`) will be multiplied + by this tensor element by element. outside_weight (Variable|None): A tensor with rank at least 2. This - input is optional and should have same shape with x. If provided, - the out smooth L1 loss will be multiplied by this tensor element - by element. + input is optional and should have same shape with :attr:`x`. If + provided, the out smooth L1 loss will be multiplied by this tensor + element by element. sigma (float|None): Hyper parameter of smooth L1 loss layer. A float scalar with default value 1.0. @@ -3634,12 +3634,12 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=True, name=None): def lod_reset(x, y=None, target_lod=None): """ - LoD Reset Operator. Set LoD of **x** to a new one specified by **y** or - **target_lod**. When **y** provided, **y.lod** would be considered as target - LoD first, otherwise **y.data** would be considered as target LoD. If **y** - is not provided, target LoD should be specified by **target_lod**. - If target LoD is specified by **Y.data** or **target_lod**, only one level - LoD is supported. + Set LoD of :attr:`x` to a new one specified by :attr:`y` or + :attr:`target_lod`. When :attr:`y` provided, :attr:`y.lod` would be + considered as target LoD first, otherwise :attr:`y.data` would be + considered as target LoD. If :attr:`y` is not provided, target LoD should + be specified by :attr:`target_lod`. If target LoD is specified by + :attr:`Y.data` or :attr:`target_lod`, only one level LoD is supported. .. code-block:: text @@ -3692,15 +3692,16 @@ def lod_reset(x, y=None, target_lod=None): Args: x (Variable): Input variable which could be a Tensor or LodTensor. - y (Variable|None): If provided, output's LoD would be derived from y. + y (Variable|None): If provided, output's LoD would be derived + from :attr:`y`. target_lod (list|tuple|None): One level LoD which should be considered - as target LoD when y not provided. + as target LoD when :attr:`y` not provided. Returns: - Variable: Output variable with LoD specified by this operator. + Variable: Output variable with LoD specified by this layer. Raises: - ValueError: If y and target_lod are both None. + ValueError: If :attr:`y` and :attr:`target_lod` are both None. Examples: .. code-block:: python From 80ccabbfa7b9777fab2369f6a2ecb852b61b0906 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Jun 2018 06:56:17 -0700 Subject: [PATCH 083/154] Fix typos in reduce_mean's doc --- python/paddle/fluid/layers/nn.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 378a1c33c6..47dfed5468 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -2243,23 +2243,24 @@ def reduce_sum(input, dim=None, keep_dim=False, name=None): def reduce_mean(input, dim=None, keep_dim=False, name=None): """ - Computes the mean of tensor elements over the given dimension. + Computes the mean of the input tensor's elements along the given dimension. Args: input (Variable): The input variable which is a Tensor or LoDTensor. - dim (list|int|None): The dimensions along which the mean is computed. If - :attr:`None`, compute the mean over all elements of :attr:`input` - and return a Tensor variable with a single element, otherwise + dim (list|int|None): The dimension along which the mean is computed. If + `None`, compute the mean over all elements of :attr:`input` + and return a variable with a single element, otherwise it must be in the range :math:`[-rank(input), rank(input))`. If - :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`. + :math:`dim[i] < 0`, the dimension to reduce is + :math:`rank(input) + dim[i]`. keep_dim (bool): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. - name(str|None): A name for this layer(optional). If set None, the layer + name(str|None): A name for this layer(optional). If set `None`, the layer will be named automatically. Returns: - Variable: The reduced Tensor variable. + Variable: The reduced mean Variable. Examples: .. code-block:: python From acdb57a510d116d0c9f2a0d0b26083f474cb4f8a Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Jun 2018 02:30:36 +0800 Subject: [PATCH 084/154] polish doc: conv2d --- python/paddle/fluid/layers/nn.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 2c1f988828..48c6bb99bb 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1183,14 +1183,17 @@ def conv2d(input, act=None, name=None): """ - **Convlution2D Layer** - The convolution2D layer calculates the output based on the input, filter - and strides, paddings, dilations, groups parameters. Input(Input) and - Output(Output) are in NCHW format. Where N is batch size, C is the number of + and strides, paddings, dilations, groups parameters. Input and + Output are in NCHW format, where N is batch size, C is the number of channels, H is the height of the feature, and W is the width of the feature. - The details of convolution layer, please refer UFLDL's `convolution, - `_ . + Filter is in MCHW format, where M is the number of output image channels, + C is the number of input image channels, H is the height of the filter, + and W is the width of the filter. If the groups is greater than 1, + C will equal the number of input image channels divided by the groups. + Please refer to UFLDL's `convolution + `_ + for more detials. If bias attribution and activation type are provided, bias is added to the output of the convolution, and the corresponding activation function is applied to the final result. @@ -1201,15 +1204,14 @@ def conv2d(input, Out = \sigma (W \\ast X + b) - In the above equation: + Where: * :math:`X`: Input value, a tensor with NCHW format. * :math:`W`: Filter value, a tensor with MCHW format. * :math:`\\ast`: Convolution operation. * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. * :math:`\\sigma`: Activation function. - * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be - different. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. Example: @@ -1220,6 +1222,7 @@ def conv2d(input, Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)` - Output: + Output shape: :math:`(N, C_{out}, H_{out}, W_{out})` Where @@ -1231,7 +1234,7 @@ def conv2d(input, Args: input (Variable): The input image with [N, C, H, W] format. - num_filters(int): The number of filter. It is as same as the output + num_filters(int): The number of filter. It is as same as the output image channel. filter_size (int|tuple|None): The filter size. If filter_size is a tuple, it must contain two integers, (filter_size_H, filter_size_W). @@ -1254,7 +1257,8 @@ def conv2d(input, bias_attr (ParamAttr): Bias parameter for the Conv2d layer. Default: None use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn library is installed. Default: True - use_mkldnn (bool): Use mkldnn kernels or not. + use_mkldnn (bool): Use mkldnn kernels or not, it is valid only when compiled + with mkldnn library. Default: False act (str): Activation type. Default: None name (str|None): A name for this layer(optional). If set None, the layer will be named automatically. From 24fea628ccb224c3d8a4eadd37d5b23bc39ad1ce Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Jun 2018 03:03:28 +0800 Subject: [PATCH 085/154] polish doc: mean --- paddle/fluid/operators/mean_op.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/operators/mean_op.cc b/paddle/fluid/operators/mean_op.cc index 4881cff4a3..9e0bebd17c 100644 --- a/paddle/fluid/operators/mean_op.cc +++ b/paddle/fluid/operators/mean_op.cc @@ -33,12 +33,10 @@ class MeanOp : public framework::OperatorWithKernel { class MeanOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddInput("X", "The input of mean op"); - AddOutput("Out", "The output of mean op").Reuse("X"); + AddInput("X", "(Tensor) The input of mean op"); + AddOutput("Out", "(Tensor) The output of mean op").Reuse("X"); AddComment(R"DOC( -Mean Operator. - -Out is a scalar which is the mean of all elements in X. +Mean Operator calculates the mean of all elements in X. )DOC"); } From f9bebfe43092807891b25392960db6944f072d5d Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Jun 2018 04:22:33 +0800 Subject: [PATCH 086/154] polish doc: lod_rank_table, embedding --- python/paddle/fluid/layers/control_flow.py | 2 +- python/paddle/fluid/layers/nn.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..87843b0e96 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -706,7 +706,7 @@ def lod_rank_table(x, level=0): .. code-block:: python x = fluid.layers.data(name='x', shape=[10], - dtype='float32', lod_level=1) + dtype='float32', lod_level=1) out = layers.lod_rank_table(x=x, level=0) """ helper = LayerHelper("lod_rank_table", **locals()) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 48c6bb99bb..635b08635b 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -173,11 +173,11 @@ def embedding(input, have two elements which indicate the size of the dictionary of embeddings and the size of each embedding vector respectively. is_sparse(bool): The flag indicating whether to use sparse update. - is_distributed (bool): Whether to run lookup table from remote parameter server. + is_distributed(bool): Whether to run lookup table from remote parameter server. padding_idx(int|long|None): If :attr:`None`, it makes no effect to lookup. Otherwise the given :attr:`padding_idx` indicates padding the output with zeros whenever lookup encounters it in :attr:`input`. If - :math:`padding_idx < 0`, the padding_idx to use in lookup is + :math:`padding_idx < 0`, the :attr:`padding_idx` to use in lookup is :math:`size[0] + dim`. param_attr(ParamAttr): Parameters for this layer dtype(np.dtype|core.VarDesc.VarType|str): The type of data : float32, float_16, int etc From 98ab2b403efb475bf449317b139c2b99f94b49c8 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Jun 2018 04:55:06 +0800 Subject: [PATCH 087/154] polish doc: softshrink, assign, shuffle --- paddle/fluid/operators/activation_op.cc | 17 ++++++++--------- python/paddle/fluid/layers/io.py | 3 +++ python/paddle/fluid/layers/tensor.py | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index af1d85047e..4d224a1341 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -252,15 +252,14 @@ class SoftShrinkOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "Output of Softshrink operator"); AddAttr("lambda", "non-negative offset").SetDefault(0.5f); AddComment(R"DOC( -Softshrink Activation Operator. - -$$ -out = \begin{cases} - x - \lambda, \text{if } x > \lambda \\ - x + \lambda, \text{if } x < -\lambda \\ - 0, \text{otherwise} - \end{cases} -$$ +:strong:`Softshrink Activation Operator` + +.. math:: + out = \begin{cases} + x - \lambda, \text{if } x > \lambda \\ + x + \lambda, \text{if } x < -\lambda \\ + 0, \text{otherwise} + \end{cases} )DOC"); } diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 9de88e2c32..fc53cd802b 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -544,6 +544,9 @@ def __create_unshared_decorated_reader__(op_type, reader, attrs, name=None): def shuffle(reader, buffer_size): + """ + Shuffle the reader. + """ return __create_unshared_decorated_reader__( 'create_shuffle_reader', reader, {'buffer_size': int(buffer_size)}) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 62b01d595a..d4e9a19d1a 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -191,6 +191,7 @@ def assign(input, output): Examples: .. code-block:: python + out = fluid.layers.create_tensor(dtype='float32') hidden = fluid.layers.fc(input=data, size=10) fluid.layers.assign(hidden, out) From 14efb55dcff62f84599eaf08ec2490f5019f77d3 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Thu, 14 Jun 2018 18:25:30 -0700 Subject: [PATCH 088/154] add author (#11499) --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 11f227be71..8c4a113fc2 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -22,6 +22,7 @@ | jczaja | Jacek Czaja | | JiayiFeng | Jia-Yi Feng | | kbinias | Krzysztof Binias | +| kexinzhao | Ke-Xin Zhao | | kuke | Yi-Bing Liu | | lcy-seso | Ying Cao | | lipeng-unisound | Peng Li | From caf6914fadf31ac5e9c94c32e367481e750772f1 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 14 Jun 2018 20:18:27 +0800 Subject: [PATCH 089/154] add doc of sequence_softmax and parallelDo --- python/paddle/fluid/layers/control_flow.py | 5 ++-- python/paddle/fluid/layers/nn.py | 35 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 80e8ff484a..15ecc731e8 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -233,9 +233,8 @@ class BlockGuard(object): class ParallelDo(object): """ - ParallelDo class. - - ParallelDo class is used to create a ParallelDo. + ParallelDo class is used to create a ParallelDo. + It will be soon deprecated, please use ParallelExecutor instead. """ def __init__(self, places, use_nccl=False, name=None): diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 2c1f988828..04b23457d4 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1146,6 +1146,41 @@ def sequence_conv(input, def sequence_softmax(input, param_attr=None, bias_attr=None, use_cudnn=True): + """ + This function computes the softmax activation among all time-steps for each + sequence. The dimension of each time-step should be 1. Thus, the shape of + input Tensor can be either :math:`[N, 1]` or :math:`[N]`, where :math:`N` + is the sum of the length of all sequences. + + For i-th sequence in a mini-batch: + + .. math:: + + Out(X[lod[i]:lod[i+1]], :) = \\frac{\exp(X[lod[i]:lod[i+1], :])}{\sum(\exp(X[lod[i]:lod[i+1], :]))} + + For example, for a mini-batch of 3 sequences with variable-length, + each containing 2, 3, 2 time-steps, the lod of which is [0, 2, 5, 7], + then softmax will be computed among :math:`X[0:2, :]`, :math:`X[2:5, :]`, + :math:`X[5:7, :]`, and :math:`N` turns out to be 7. + + Args: + input (Variable): The input variable which is a LoDTensor. + bias_attr (ParamAttr|None): attributes for bias + param_attr (ParamAttr|None): attributes for parameter + use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn \ + library is installed. Default: True + + Returns: + Variable: output of sequence_softmax + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + x_sequence_softmax = fluid.layers.sequence_softmax(input=x) + """ helper = LayerHelper('sequence_softmax', **locals()) dtype = helper.input_dtype() softmax_out = helper.create_tmp_variable(dtype) From e42e6ea142d8f40cc0fa0a82079be20918733c8a Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Fri, 15 Jun 2018 10:30:03 +0800 Subject: [PATCH 090/154] add inference lib to release (#11482) --- cmake/inference_lib.cmake | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 236a55d332..cd44fe2542 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -39,7 +39,7 @@ function(copy TARGET) message(FATAL_ERROR "${TARGET} source numbers are not equal to destination numbers") endif() math(EXPR len "${copy_lib_SRCS_len} - 1") - + add_custom_target(${TARGET} DEPENDS ${copy_lib_DEPS}) foreach(index RANGE ${len}) list(GET copy_lib_SRCS ${index} src) @@ -155,6 +155,15 @@ copy(inference_lib DEPS paddle_fluid_shared paddle_fluid DSTS ${dst_dir}/${module} ${dst_dir}/${module} ) +if(WITH_CONTRIB) + set(contrib_dst_dir "${FLUID_INSTALL_DIR}/contrib/inference") + copy(contrib_inference_lib DEPS paddle_inference_api + SRCS ${PADDLE_SOURCE_DIR}/paddle/contrib/inference/paddle_inference_api.h + ${PADDLE_BINARY_DIR}/paddle/contrib/inference/libpaddle_inference_api.* + DSTS ${contrib_dst_dir} ${contrib_dst_dir} + ) +endif() + set(module "platform") copy(platform_lib DEPS profiler_py_proto SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/dynload/*.h ${src_dir}/${module}/details/*.h From 0b063e5e57f05de428f3fce194a8d5fe4629568c Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Jun 2018 20:26:50 -0700 Subject: [PATCH 091/154] Fix one_hot layer's doc --- python/paddle/fluid/layers/nn.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 47dfed5468..f4a8c5f37d 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3466,32 +3466,20 @@ def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): def one_hot(input, depth): """ - One Hot Operator. This operator creates the one-hot representations for input - index values. The following example will help to explain the function of this - operator. + This layer creates the one-hot representations for input indices. Args: - input(variable): A Tensor/LodTensor of indices, last dimension must be 1. - depth(scalar): an interger defining the depth of the one hot dimension. + input(Variable): Input indices, last dimension must be 1. + depth(scalar): An interger defining the depth of the one-hot dimension. Returns: - The one-hot tensor or LodTensor, same as input. + Variable: The one-hot representations of input. Examples: .. code-block:: python - - X is a LoDTensor: - X.lod = [[0, 1, 4]] - X.shape = [4, 1] - X.data = [[1], [1], [3], [0]] - set depth = 4 - Out is a LoDTensor: - Out.lod = [[0, 1, 4]] - Out.shape = [4, 4] - Out.data = [[0., 1., 0., 0.], - [0., 1., 0., 0.], - [0., 0., 0., 1.], - [1., 0., 0., 0.]] + + label = layers.data(name="label", shape=[1], dtype="float32") + one_hot_label = layers.one_hot(input=label, depth=10) """ helper = LayerHelper("one_hot", **locals()) one_hot_out = helper.create_tmp_variable(dtype='float32') From d91060d300edf3c908f25b741adc999b065887da Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 15 Jun 2018 11:38:31 +0800 Subject: [PATCH 092/154] fix errors --- paddle/fluid/operators/activation_op.cc | 2 +- paddle/fluid/operators/pool_op.cc | 8 ++++---- python/paddle/fluid/layers/nn.py | 6 +++--- python/paddle/fluid/layers/tensor.py | 3 ++- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index af1d85047e..790c012fdb 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -444,7 +444,7 @@ class SwishOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Swish Activation Operator. -$$out = \frac{x}{1 + e^{- \beta x}}$$ +$$out = \\frac{x}{1 + e^{- \beta x}}$$ )DOC"); } diff --git a/paddle/fluid/operators/pool_op.cc b/paddle/fluid/operators/pool_op.cc index d94ddc7a53..f8ad63690e 100644 --- a/paddle/fluid/operators/pool_op.cc +++ b/paddle/fluid/operators/pool_op.cc @@ -224,17 +224,17 @@ Example: For ceil_mode = false: $$ - H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 + H_{out} = \\frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 $$ $$ - W_{out} = \frac{(W_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1 + W_{out} = \\frac{(W_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1 $$ For ceil_mode = true: $$ - H_{out} = \frac{(H_{in} - ksize[0] + 2 * paddings[0] + strides[0] - 1)}{strides[0]} + 1 + H_{out} = \\frac{(H_{in} - ksize[0] + 2 * paddings[0] + strides[0] - 1)}{strides[0]} + 1 $$ $$ - W_{out} = \frac{(W_{in} - ksize[1] + 2 * paddings[1] + strides[1] - 1)}{strides[1]} + 1 + W_{out} = \\frac{(W_{in} - ksize[1] + 2 * paddings[1] + strides[1] - 1)}{strides[1]} + 1 $$ )DOC"); diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 1218766e8d..b073955e2f 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1495,9 +1495,9 @@ def pool2d(input, Args: input (Variable): The input tensor of pooling operator. The format of - input tensor is NCHW, where N is batch size, C is the number of - channels, H is the height of the feature, and W is the width of - the feature. + input tensor is NCHW, where N is batch size, C is + the number of channels, H is the height of the + feature, and W is the width of the feature. pool_size (int): The side length of pooling windows. All pooling windows are squares with pool_size on a side. pool_type: ${pooling_type_comment} diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 392fa6a422..81f42ff470 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -146,7 +146,8 @@ def concat(input, axis=0, name=None): Examples: .. code-block:: python - out = fluid.layers.concat(input=[Efirst, Esecond, Ethird, Efourth]) + + out = fluid.layers.concat(input=[Efirst, Esecond, Ethird, Efourth]) """ helper = LayerHelper('concat', **locals()) out = helper.create_tmp_variable(dtype=helper.input_dtype()) From 3380737cb7a23f5670e5fe9a7b3d4de193b7a1e0 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 15 Jun 2018 12:18:17 +0800 Subject: [PATCH 093/154] update by comment --- paddle/fluid/operators/chunk_eval_op.cc | 67 +++++++------ paddle/fluid/operators/cos_sim_op.cc | 4 +- .../operators/detection/iou_similarity_op.cc | 4 +- paddle/fluid/operators/roi_pool_op.cc | 2 +- paddle/fluid/operators/scale_op.cc | 7 +- python/paddle/fluid/layers/io.py | 2 + python/paddle/fluid/layers/nn.py | 94 ++++++++++++++++++- 7 files changed, 133 insertions(+), 47 deletions(-) diff --git a/paddle/fluid/operators/chunk_eval_op.cc b/paddle/fluid/operators/chunk_eval_op.cc index 62636bb2f9..dc43c69be0 100644 --- a/paddle/fluid/operators/chunk_eval_op.cc +++ b/paddle/fluid/operators/chunk_eval_op.cc @@ -91,32 +91,31 @@ class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker { "(int64_t). The number of chunks both in Inference and Label on the " "given mini-batch."); AddAttr("num_chunk_types", - "(int). The number of chunk type. See below for details."); - AddAttr( - "chunk_scheme", - "(string, default IOB). The labeling scheme indicating " - "how to encode the chunks. Must be IOB, IOE, IOBES or plain. See below " - "for details.") + "The number of chunk type. See the description for details."); + AddAttr("chunk_scheme", + "The labeling scheme indicating " + "how to encode the chunks. Must be IOB, IOE, IOBES or " + "plain. See the description" + "for details.") .SetDefault("IOB"); AddAttr>("excluded_chunk_types", - "(list) A list including chunk type ids " + "A list including chunk type ids " "indicating chunk types that are not counted. " - "See below for details.") + "See the description for details.") .SetDefault(std::vector{}); AddComment(R"DOC( For some basics of chunking, please refer to -‘Chunking with Support Vector Machines ’. +'Chunking with Support Vector Machines '. - -CheckEvalOp computes the precision, recall, and F1-score of chunk detection, +ChunkEvalOp computes the precision, recall, and F1-score of chunk detection, and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes. Here is a NER example of labeling for these tagging schemes: - - Li Ming works at Agricultural Bank of China in Beijing. - IO: I-PER I-PER O O I-ORG I-ORG I-ORG I-ORG O I-LOC - IOB: B-PER I-PER O O B-ORG I-ORG I-ORG I-ORG O B-LOC - IOE: I-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O E-LOC - IOBES: B-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O S-LOC + + Li Ming works at Agricultural Bank of China in Beijing. + IO I-PER I-PER O O I-ORG I-ORG I-ORG I-ORG O I-LOC + IOB B-PER I-PER O O B-ORG I-ORG I-ORG I-ORG O B-LOC + IOE I-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O E-LOC + IOBES B-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O S-LOC There are three chunk types(named entity types) including PER(person), ORG(organization) and LOC(LOCATION), and we can see that the labels have the form -. @@ -124,31 +123,31 @@ and LOC(LOCATION), and we can see that the labels have the form -("scale", - "(float, default 1.0)" - "The scaling factor of the scale operator.") + AddAttr("scale", "The scaling factor of the scale operator.") .SetDefault(1.0); } }; diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 76ef82ddb0..15be646f47 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -109,6 +109,8 @@ class BlockGuardServ(BlockGuard): class ListenAndServ(object): """ + ***ListenAndServ Layer*** + ListenAndServ is used to create a rpc server bind and listen on specific TCP port, this server will run the sub-block when received variables from clients. diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 70b7aba27d..8e4780837b 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -825,6 +825,12 @@ def crf_decoding(input, param_attr, label=None): Returns: Variable: ${viterbi_path_comment} + + Examples: + .. code-block:: python + + crf_decode = layers.crf_decoding( + input=hidden, param_attr=ParamAttr(name="crfw")) """ helper = LayerHelper('crf_decoding', **locals()) transition = helper.get_parameter(param_attr.name) @@ -1043,9 +1049,70 @@ def chunk_eval(input, num_chunk_types, excluded_chunk_types=None): """ + ***Chunk Evaluator*** + This function computes and outputs the precision, recall and F1-score of chunk detection. + For some basics of chunking, please refer to + 'Chunking with Support Vector Machines '. + + ChunkEvalOp computes the precision, recall, and F1-score of chunk detection, + and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes. + Here is a NER example of labeling for these tagging schemes: + + .. code-block:: python + + ====== ====== ====== ===== == ============ ===== ===== ===== == ========= + Li Ming works at Agricultural Bank of China in Beijing. + ====== ====== ====== ===== == ============ ===== ===== ===== == ========= + IO I-PER I-PER O O I-ORG I-ORG I-ORG I-ORG O I-LOC + IOB B-PER I-PER O O B-ORG I-ORG I-ORG I-ORG O B-LOC + IOE I-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O E-LOC + IOBES B-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O S-LOC + ====== ====== ====== ===== == ============ ===== ===== ===== == ========= + + There are three chunk types(named entity types) including PER(person), ORG(organization) + and LOC(LOCATION), and we can see that the labels have the form -. + + Since the calculations actually use label ids rather than labels, extra attention + should be paid when mapping labels to ids to make CheckEvalOp work. The key point + is that the listed equations are satisfied by ids. + + .. code-block:: python + + tag_type = label % num_tag_type + chunk_type = label / num_tag_type + + where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type` + is the num of chunk types, and `tag_type` get its value from the following table. + + .. code-block:: python + + Scheme Begin Inside End Single + plain 0 - - - + IOB 0 1 - - + IOE - 0 1 - + IOBES 0 1 2 3 + + Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG, + PER and LOC. To satisfy the above equations, the label map can be like this: + + .. code-block:: python + + B-ORG 0 + I-ORG 1 + B-PER 2 + I-PER 3 + B-LOC 4 + I-LOC 5 + O 6 + + It's not hard to verify the equations noting that the num of chunk types + is 3 and the num of tag types in IOB scheme is 2. For example, the label + id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of + I-LOC is 2, which consistent with the results from the equations. + Args: input (Variable): prediction output of the network. label (Variable): label of the test data set. @@ -1057,6 +1124,19 @@ def chunk_eval(input, tuple: tuple containing: precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks + + Examples: + .. code-block:: python + + crf = fluid.layers.linear_chain_crf( + input=hidden, label=label, param_attr=ParamAttr(name="crfw")) + crf_decode = fluid.layers.crf_decoding( + input=hidden, param_attr=ParamAttr(name="crfw")) + fluid.layers.chunk_eval( + input=crf_decode, + label=label, + chunk_scheme="IOB", + num_chunk_types=(label_dict_len - 1) / 2) """ helper = LayerHelper("chunk_eval", **locals()) @@ -1803,7 +1883,7 @@ def conv2d_transpose(input, act=None, name=None): """ - **Convlution2D transpose layer** + ***Convlution2D Transpose Layer**** The convolution2D transpose layer calculates the output based on the input, filter, and dilations, strides, paddings. Input(Input) and output(Output) @@ -1832,13 +1912,13 @@ def conv2d_transpose(input, - Input: - Input shape: $(N, C_{in}, H_{in}, W_{in})$ + Input shape: :math:`(N, C_{in}, H_{in}, W_{in})` - Filter shape: $(C_{in}, C_{out}, H_f, W_f)$ + Filter shape: :math:`(C_{in}, C_{out}, H_f, W_f)` - Output: - Output shape: $(N, C_{out}, H_{out}, W_{out})$ + Output shape: :math:`(N, C_{out}, H_{out}, W_{out})` Where @@ -3513,6 +3593,12 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): Returns: Variable: The global run counter. + + Examples: + .. code-block:: python + + global_step = fluid.layers.autoincreased_step_counter( + counter_name='@LR_DECAY_COUNTER@', begin=begin, step=1) """ helper = LayerHelper('global_step_counter') if counter_name is None: From 96e466391689f2dd0915c21f1176eae754a73a23 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Fri, 15 Jun 2018 12:22:30 +0800 Subject: [PATCH 094/154] Polish inline math and duplicable/optional in auto generated doc --- .../fluid/layers/layer_function_generator.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index cb60a3aec9..0f05ea2b08 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -44,6 +44,11 @@ def _type_to_str_(tp): return framework_pb2.AttrType.Name(tp) +_two_dollar_pattern_ = re.compile(r"\$\$([^\$]+)\$\$") +_single_dollar_pattern_ = re.compile(r"\$([^\$]+)\$") +_two_bang_pattern_ = re.compile(r"!!([^!]+)!!") + + def _generate_doc_string_(op_proto): """ Generate docstring by OpProto @@ -55,22 +60,27 @@ def _generate_doc_string_(op_proto): str: the document string """ + def escape_math(text): + return _two_bang_pattern_.sub( + r'$$\1$$', + _single_dollar_pattern_.sub( + r':math:`\1`', _two_dollar_pattern_.sub(r"!!\1!!", text))) + if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError("OpProto should be `framework_pb2.OpProto`") buf = cStringIO.StringIO() - buf.write(op_proto.comment) + buf.write(escape_math(op_proto.comment)) buf.write('\nArgs:\n') for each_input in op_proto.inputs: line_begin = ' {0}: '.format(_convert_(each_input.name)) buf.write(line_begin) - buf.write(each_input.comment) + buf.write(escape_math(each_input.comment)) buf.write('\n') - buf.write(' ' * len(line_begin)) - buf.write('Duplicable: ') - buf.write(str(each_input.duplicable)) - buf.write(' Optional: ') - buf.write(str(each_input.dispensable)) + if each_input.duplicable: + buf.write(" Duplicatable.") + if each_input.dispensable: + buf.write(" Optional.") buf.write('\n') skip_attrs = OpProtoHolder.generated_op_attr_names() @@ -83,7 +93,7 @@ def _generate_doc_string_(op_proto): buf.write(' (') buf.write(_type_to_str_(each_attr.type)) buf.write('): ') - buf.write(each_attr.comment) + buf.write(escape_math(each_attr.comment)) buf.write('\n') if len(op_proto.outputs) != 0: @@ -92,7 +102,7 @@ def _generate_doc_string_(op_proto): for each_opt in op_proto.outputs: if not each_opt.intermediate: break - buf.write(each_opt.comment) + buf.write(escape_math(each_opt.comment)) return buf.getvalue() From 1f38cbf79b8369fbfc6fa626446b9c98a7314426 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 21:26:51 -0700 Subject: [PATCH 095/154] "fix some typo" --- paddle/fluid/operators/uniform_random_batch_size_like_op.cc | 2 +- python/paddle/fluid/layers/control_flow.py | 3 +-- python/paddle/fluid/layers/nn.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/operators/uniform_random_batch_size_like_op.cc b/paddle/fluid/operators/uniform_random_batch_size_like_op.cc index 78fee77df8..192366fd44 100644 --- a/paddle/fluid/operators/uniform_random_batch_size_like_op.cc +++ b/paddle/fluid/operators/uniform_random_batch_size_like_op.cc @@ -35,7 +35,7 @@ class UniformRandomBatchSizeLikeOpMaker : public BatchSizeLikeOpMaker { protected: void Apply() override { AddComment(R"DOC( -Uniform random operator +UniformRandomBatchSizeLike operator. This operator initializes a tensor with the same batch_size as the Input tensor with random values sampled from a uniform distribution. diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 850945001c..f4e95dd363 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -866,8 +866,7 @@ def array_write(x, i, array=None): Variable: The output LOD_TENSOR_ARRAY where the input tensor is written. Examples: - - .. code-block::python + .. code-block:: python tmp = fluid.layers.zeros(shape=[10], dtype='int32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index b5745e20f1..dc1124a512 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -2935,7 +2935,7 @@ def split(input, num_or_sections, dim=-1, name=None): will be named automatically. Returns: - List: The list of segmented tensor variables. + list(Variable): The list of segmented tensor variables. Examples: .. code-block:: python From 9397a2e0c8128e2de2c8eeb8fa5d446f1a6f2f13 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 15 Jun 2018 12:38:36 +0800 Subject: [PATCH 096/154] fix format --- python/paddle/fluid/layers/io.py | 5 +---- python/paddle/fluid/layers/nn.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 15be646f47..906364f693 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -109,7 +109,7 @@ class BlockGuardServ(BlockGuard): class ListenAndServ(object): """ - ***ListenAndServ Layer*** + **ListenAndServ Layer** ListenAndServ is used to create a rpc server bind and listen on specific TCP port, this server will run the sub-block when @@ -120,9 +120,6 @@ class ListenAndServ(object): inputs(list): a list of variables that the server will get from clients. fan_in(int): how many client are expected to report to this server, default: 1. optimizer_mode(bool): whether to run the server as a parameter server, default: True. - - Returns: - None Examples: .. code-block:: python diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 8e4780837b..3704e1149a 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1049,7 +1049,7 @@ def chunk_eval(input, num_chunk_types, excluded_chunk_types=None): """ - ***Chunk Evaluator*** + **Chunk Evaluator** This function computes and outputs the precision, recall and F1-score of chunk detection. @@ -1883,7 +1883,7 @@ def conv2d_transpose(input, act=None, name=None): """ - ***Convlution2D Transpose Layer**** + **Convlution2D Transpose Layer** The convolution2D transpose layer calculates the output based on the input, filter, and dilations, strides, paddings. Input(Input) and output(Output) From dbe0fe6d181fb8e747856d93e60ae78216e6734c Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 21:44:17 -0700 Subject: [PATCH 097/154] 'fix typo' --- python/paddle/fluid/layers/nn.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index dc1124a512..3456a1bde8 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -2051,15 +2051,25 @@ def layer_norm(input, def beam_search_decode(ids, scores, name=None): """ + This layers is to pack the output of beam search layer into sentences and + associated scores. It is usually called after the beam search layer. + ${beam_search_decode} Args: ids (Variable): ${ids_comment} scores (Variable): ${scores_comment} name (str): The name of this layer. It is optional. - + Returns: - tuple: a tuple of two output variable: sentence_ids, sentence_scores + tuple(Variable): a tuple of two output variable: sentence_ids, sentence_scores + + Examples: + .. code-block:: python + ids, scores = fluid.layers.beam_search( + pre_ids, ids, scores, beam_size, end_id) + sentence_ids, sentence_scores = fluid.layers.beam_search_decode( + ids, scores) """ helper = LayerHelper('beam_search_decode', **locals()) sentence_ids = helper.create_tmp_variable(dtype=ids.dtype) From dd711c3754ddce6bf1e5ee9b52964870aaa7f944 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 21:55:21 -0700 Subject: [PATCH 098/154] "add beam search" --- python/paddle/fluid/layers/nn.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 3456a1bde8..bdbb6c47e2 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -834,11 +834,14 @@ def linear_chain_crf(input, label, param_attr=None): Args: input(${emission_type}): ${emission_comment} + input(${transition_type}): ${transition_comment} label(${label_type}): ${label_comment} param_attr(ParamAttr): The attribute of the learnable parameter. Returns: ${log_likelihood_comment} + ${transitionexps_comment} + ${emissionexps_comment} """ helper = LayerHelper('linear_chain_crf', **locals()) @@ -1170,10 +1173,6 @@ def sequence_conv(input, Variable: output of sequence_conv """ - # FIXME(dzh) : want to unify the argument of python layer - # function. So we ignore some unecessary attributes. - # such as, padding_trainable, context_start. - helper = LayerHelper('sequence_conv', **locals()) dtype = helper.input_dtype() filter_shape = [filter_size * input.shape[1], num_filters] @@ -2051,18 +2050,31 @@ def layer_norm(input, def beam_search_decode(ids, scores, name=None): """ + Beam Search Decode + This layers is to pack the output of beam search layer into sentences and associated scores. It is usually called after the beam search layer. + Typically, the output of beam search layer is a tensor of selected ids, with + a tensor of the score of each id. Beam search layer's output ids, however, + are generated directly during the tree search, and they are stacked by each + level of the search tree. Thus we need to reorganize them into sentences, + based on the score of each id. This layer takes the output of beam search + layer as input and repack them into sentences. ${beam_search_decode} Args: - ids (Variable): ${ids_comment} - scores (Variable): ${scores_comment} + ids (Variable): The selected ids, output of beam search layer. + scores (Variable): The associated scores of the ids, out put of beam + search layer. name (str): The name of this layer. It is optional. Returns: - tuple(Variable): a tuple of two output variable: sentence_ids, sentence_scores + tuple(Variable): a tuple of two output tensors: sentence_ids, sentence_scores. + sentence_ids is a tensor with shape [size, length], where size is the + beam size of beam search, and length is the length of each sentence. + Note that the length of sentences may vary. + sentence_scores is a tensor with the same shape as sentence_ids. Examples: .. code-block:: python From 6e3d168b9d3b6147efa7740023286b866d7d6e4a Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 22:00:59 -0700 Subject: [PATCH 099/154] "fix latex" --- python/paddle/fluid/layers/nn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index bdbb6c47e2..283d3c4b2a 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4210,8 +4210,8 @@ def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): .. math:: - Output(i, x, y) = Input(i, x, y) / \left( - k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} + Output(i, x, y) = Input(i, x, y) / \left( \\ + k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} \\ (Input(j, x, y))^2\right)^{\beta} In the above equation: From 541ddf7e24e263abbee3b342a69100b99ec413d7 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 22:05:36 -0700 Subject: [PATCH 100/154] squash to one pr --- python/paddle/fluid/framework.py | 31 ++++++++++++++++++++++++++++ python/paddle/fluid/layers/metric.py | 26 ++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index f6438c82ac..595f67961f 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -1034,6 +1034,37 @@ class Block(object): class Program(object): + """ + Python Program. Beneath it is a ProgramDesc, which is used for + create c++ Program. A program is a self-contained programing + language like container. It has at least one Block, when the + control flow op like conditional_block, while_op is included, + it will contains nested block. + Please reference the framework.proto for details. + + Notes: we have default_startup_program and default_main_program + by default, a pair of them will shared the parameters. + The default_startup_program only run once to initialize parameters, + default_main_program run in every minibatch and adjust the weights. + + Args: + None + + Returns: + Python Program + + Examples: + .. code-block:: python + + main_program = Program() + startup_program = Program() + with fluid.program_guard(main_program=main_program, startup_program=startup_program): + fluid.layers.data(name="x", shape=[-1, 784], dtype='float32') + fluid.layers.data(name="y", shape=[-1, 1], dtype='int32') + fluid.layers.fc(name="fc", shape=[10], dtype='float32', act="relu") + + """ + def __init__(self): self.desc = core.ProgramDesc() self.blocks = [Block(self, 0)] diff --git a/python/paddle/fluid/layers/metric.py b/python/paddle/fluid/layers/metric.py index a1c64ce277..069c060da4 100644 --- a/python/paddle/fluid/layers/metric.py +++ b/python/paddle/fluid/layers/metric.py @@ -27,8 +27,32 @@ __all__ = ['accuracy', 'auc'] def accuracy(input, label, k=1, correct=None, total=None): """ + accuracy layer. + Refer to the https://en.wikipedia.org/wiki/Precision_and_recall + This function computes the accuracy using the input and label. - The output is the top k inputs and their indices. + If the correct label occurs in top k predictions, then correct will increment by one. + Note: the dtype of accuracy is determined by input. the input and label dtype can be different. + + Args: + input(Variable): The input of accuracy layer, which is the predictions of network. + Carry LoD information is supported. + label(Variable): The label of dataset. + k(int): The top k predictions for each class will be checked. + correct(Variable): The correct predictions count. + total(Variable): The total entries count. + + Returns: + Variable: The correct rate. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name="data", shape=[-1, 32, 32], dtype="float32") + label = fluid.layers.data(name="data", shape=[-1,1], dtype="int32") + predict = fluid.layers.fc(input=data, size=10) + acc = fluid.layers.accuracy(input=predict, label=label, k=5) + """ helper = LayerHelper("accuracy", **locals()) topk_out, topk_indices = nn.topk(input, k=k) From 34ac0eb8e613868a2b6de35a62c86a4059d72335 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Fri, 15 Jun 2018 13:09:25 +0800 Subject: [PATCH 101/154] enhance memory optimization transpiler to support user defined skip_opt_set (#11372) * fix mac build error * enhance memory optimize transpiler to let users set to some skip opt set of variables --- .../memory_optimization_transpiler.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 9ff0ae6fca..8bfb554845 100644 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -157,9 +157,11 @@ class ControlFlowGraph(object): if op.type() == "fill_constant" and op.attr("force_cpu") == True: self._skip_opt.update(op.output_arg_names()) - def release_memory(self): + def release_memory(self, skip_opt_set=None): self._dataflow_analyze() self._update_skip_opt_set() + if skip_opt_set: + self._skip_opt.update(skip_opt_set) fwd_id = 0 bwd_id = 0 for i in range(self.op_size): @@ -183,7 +185,7 @@ class ControlFlowGraph(object): else: bwd_id += 1 - def memory_optimize(self, level=0): + def memory_optimize(self, skip_opt_set=None, level=0): def compare_shape(x_shape, cache_shape, opt_level): if opt_level == 0: return x_shape == cache_shape @@ -200,6 +202,9 @@ class ControlFlowGraph(object): self._dataflow_analyze() self._update_skip_opt_set() + # update skip set to meet users' demand + if skip_opt_set: + self._skip_opt.update(skip_opt_set) self.pool = [] for i in range(self.op_size): op = self._ops[i] @@ -358,7 +363,7 @@ def _get_cfgs(input_program): return cfgs -def memory_optimize(input_program, print_log=False, level=0): +def memory_optimize(input_program, skip_opt_set=None, print_log=False, level=0): """Optimize memory by reusing var memory. Note: it doesn't not support subblock nested in subblock. @@ -374,10 +379,10 @@ def memory_optimize(input_program, print_log=False, level=0): PRINT_LOG = print_log cfgs = _get_cfgs(input_program) for cfg in cfgs: - cfg.memory_optimize(level) + cfg.memory_optimize(skip_opt_set=skip_opt_set, level=level) -def release_memory(input_program): +def release_memory(input_program, skip_opt_set=None): cfgs = _get_cfgs(input_program) for cfg in cfgs: - cfg.release_memory() + cfg.release_memory(skip_opt_set=skip_opt_set) From 9de779f1cfae9daba1b38b8df829cb0e56247592 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 15 Jun 2018 13:18:33 +0800 Subject: [PATCH 102/154] update switch class --- python/paddle/fluid/layers/control_flow.py | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 209a767e73..2bc43c5ce9 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1156,16 +1156,14 @@ class ConditionalBlock(object): class Switch(object): """ - **Switch Class** - - Many programming languages provide `switch` as a generalization of `if-elif-else`. - Switch class works just like a `if-elif-else`. + Switch class works just like a `if-elif-else`. Can be used in learning rate scheduler + to modify learning rate The Semantics: 1. A `switch` control-flow checks cases one-by-one. - 2. The condition of each case is a boolean value, which is a scalar. + 2. The condition of each case is a boolean value, which is a scalar Variable. 3. It runs the first matched case, or the default case if there is one. @@ -1174,9 +1172,22 @@ class Switch(object): Examples: .. code-block:: python - with fluid.control_flow.Switch() as switch: + lr = fluid.layers.tensor.create_global_var( + shape=[1], + value=0.0, + dtype='float32', + persistable=True, + name="learning_rate") + one_var = tensor.fill_constant( + shape=[1], dtype='float32', value=1.0) + two_var = tensor.fill_constant( + shape=[1], dtype='float32', value=2.0) + + with fluid.layers.control_flow.Switch() as switch: with switch.case(global_step == zero_var): - fluid.tensor.assign(input=one_var, output=div_res) + fluid.layers.tensor.assign(input=one_var, output=lr) + with switch.default(): + fluid.layers.tensor.assign(input=two_var, output=lr) """ From 1e2acd979669b14c03a916aabc23a73b69af08f6 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 15 Jun 2018 13:46:03 +0800 Subject: [PATCH 103/154] refine ParallelDo doc --- python/paddle/fluid/layers/control_flow.py | 52 +++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 9c3b5cfda5..52276df3bf 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -234,8 +234,56 @@ class BlockGuard(object): class ParallelDo(object): """ - ParallelDo class is used to create a ParallelDo. - It will be soon deprecated, please use ParallelExecutor instead. + ParallelDo is used to represent multi-thread data parallel processing. + + Its vanilla implementation can be shown as the following (:math:`|` means + single thread and :math:`||||` means multiple threads) + + .. code-block:: text + + In the forward pass + | Split input onto different devices + | Copy parameter onto different devices + |||| Compute forward pass in parallel + | Merge output from different devices + + In the backward pass + | Split output@grad onto different devices + |||| Compute backward pass in parallel + | accumulate param@grad from different devices to the first device + | Merge input@grad from different devices +  | Copy param@grad to the place of parallel_do_op + + Examples: + + .. code-block:: python + + images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + + # ParallelDo version & Single-thread version + if thread_num > 1: + places = fluid.layers.get_places(thread_num) + pd = fluid.layers.ParallelDo(places) + with pd.do(): + images = pd.read_input(images) + label = pd.read_input(label) + predict = cnn_model(images) + cost = fluid.layers.cross_entropy(input=predict, label=label) + + avg_cost = fluid.layers.mean(x=cost) + pd.write_output(avg_cost) + + avg_cost = pd() + avg_cost = fluid.layers.mean(avg_cost) + else: + predict = cnn_model(images) + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) + + .. warning:: + + It will be soon deprecated, please use ParallelExecutor instead. """ def __init__(self, places, use_nccl=False, name=None): From 5fd142c3fd5cd673802593befd0f27a2257134f4 Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Fri, 15 Jun 2018 13:48:57 +0800 Subject: [PATCH 104/154] bugfix/trt engine op (#11487) --- .../inference/tensorrt/convert/op_converter.h | 3 +- paddle/fluid/inference/tensorrt/engine.h | 32 ++++-- paddle/fluid/operators/tensorrt_engine_op.cc | 28 ++++-- paddle/fluid/operators/tensorrt_engine_op.h | 33 ++++--- .../operators/tensorrt_engine_op_test.cc | 99 ++++++++++++++++++- 5 files changed, 158 insertions(+), 37 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h index c7a5a49dd0..6697952051 100644 --- a/paddle/fluid/inference/tensorrt/convert/op_converter.h +++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h @@ -64,7 +64,8 @@ class OpConverter { (*it)(op, scope, test_mode); } - // convert fluid block to tensorrt network + // Convert a fluid block to tensorrt network, NOTE it just convert operators, + // the INetwork's inputs and outputs should specified in some other modules. void ConvertBlock(const framework::proto::BlockDesc& block, const std::unordered_set& parameters, const framework::Scope& scope, TensorRTEngine* engine) { diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h index b60f00de9f..b06a9bbc67 100644 --- a/paddle/fluid/inference/tensorrt/engine.h +++ b/paddle/fluid/inference/tensorrt/engine.h @@ -51,11 +51,12 @@ class TensorRTEngine : public EngineBase { nvinfer1::Weights w_; }; - TensorRTEngine(int max_batch, int max_workspace, cudaStream_t* stream, + TensorRTEngine(int max_batch, int max_workspace, + cudaStream_t* stream = nullptr, nvinfer1::ILogger& logger = NaiveLogger::Global()) : max_batch_(max_batch), max_workspace_(max_workspace), - stream_(stream), + stream_(stream ? stream : &default_stream_), logger_(logger) {} virtual ~TensorRTEngine(); @@ -121,6 +122,8 @@ class TensorRTEngine : public EngineBase { // the max memory size the engine uses int max_workspace_; cudaStream_t* stream_; + // If stream_ is not set from outside, hold its own stream. + cudaStream_t default_stream_; nvinfer1::ILogger& logger_; std::vector buffers_; @@ -165,20 +168,31 @@ class TensorRTEngine : public EngineBase { */ class TRT_EngineManager { public: - TensorRTEngine* Create(int max_batch, int max_workspace, - cudaStream_t* stream) { - engines_.emplace_back(new TensorRTEngine(max_batch, max_workspace, stream)); - return engines_.back().get(); + bool HasEngine(const std::string& name) const { + return engines_.count(name) != 0; + } + + // Get an engine called `name`. + TensorRTEngine* Get(const std::string& name) const { + return engines_.at(name).get(); + } + + // Create or get an engine called `name` + TensorRTEngine* Create(int max_batch, int max_workspace, cudaStream_t* stream, + const std::string& name) { + auto* p = new TensorRTEngine(max_batch, max_workspace, stream); + engines_[name].reset(p); + return p; } void DeleteALl() { - for (auto& ptr : engines_) { - ptr.reset(nullptr); + for (auto& item : engines_) { + item.second.reset(nullptr); } } private: - std::vector> engines_; + std::unordered_map> engines_; }; } // namespace tensorrt diff --git a/paddle/fluid/operators/tensorrt_engine_op.cc b/paddle/fluid/operators/tensorrt_engine_op.cc index 4b1208c437..0ea273af9d 100644 --- a/paddle/fluid/operators/tensorrt_engine_op.cc +++ b/paddle/fluid/operators/tensorrt_engine_op.cc @@ -66,17 +66,25 @@ nvinfer1::Dims Vec2TRT_Dims(const std::vector &shape) { } // namespace template -void paddle::operators::TensorRTEngineKernel::Prepare( +void TensorRTEngineKernel::Prepare( const framework::ExecutionContext &context) const { VLOG(4) << "Prepare engine"; // Get the ProgramDesc and pass to convert. framework::proto::BlockDesc block_desc; block_desc.ParseFromString(context.Attr("subgraph")); - max_batch_ = context.Attr("max_batch"); + int max_batch = context.Attr("max_batch"); auto max_workspace = context.Attr("max_workspace"); - engine_ = Singleton::Global().Create( - max_batch_, max_workspace, &stream_); - engine_->InitNetwork(); + auto params = context.Attr>("parameters"); + std::unordered_set parameters; + for (const auto ¶m : params) { + parameters.insert(param); + } + + // TODO(Superjomn) replace this with a different stream + auto *engine = Singleton::Global().Create( + max_batch, max_workspace, nullptr /*engine hold its own stream*/, + context.Attr("engine_uniq_key")); + engine->InitNetwork(); framework::BlockDesc block(nullptr /*programdesc*/, &block_desc); // Add inputs @@ -87,24 +95,23 @@ void paddle::operators::TensorRTEngineKernel::Prepare( PADDLE_ENFORCE_EQ(var->GetType(), FluidDT::VarType_Type_LOD_TENSOR, "TensorRT engine only takes LoDTensor as input"); auto shape = var->GetShape(); - engine_->DeclareInput( + engine->DeclareInput( input, FluidDataType2TRT( var->Proto()->type().lod_tensor().tensor().data_type()), Vec2TRT_Dims(var->GetShape())); } - // TODO(Superjomn) parameters should be passed after analysised from outside. inference::Singleton::Global().ConvertBlock( - block_desc, {}, context.scope(), engine_); + block_desc, parameters, context.scope(), engine); // Add outputs VLOG(4) << "declare outputs"; for (auto &output : context.Outputs("Ys")) { VLOG(4) << "declare output " << output; - engine_->DeclareOutput(output); + engine->DeclareOutput(output); } - engine_->FreezeNetwork(); + engine->FreezeNetwork(); } class TensorRTEngineOpMaker : public framework::OpProtoAndCheckerMaker { @@ -113,6 +120,7 @@ class TensorRTEngineOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Xs", "A list of inputs.").AsDuplicable(); AddOutput("Ys", "A list of outputs").AsDuplicable(); AddAttr("subgraph", "the subgraph."); + AddAttr("engine_uniq_key", "unique key for the TRT engine."); AddAttr("max_batch", "the maximum batch size."); AddAttr("max_workspace", "the maximum batch size."); AddComment("TensorRT engine operator."); diff --git a/paddle/fluid/operators/tensorrt_engine_op.h b/paddle/fluid/operators/tensorrt_engine_op.h index 4b089601ff..8455d24ddf 100644 --- a/paddle/fluid/operators/tensorrt_engine_op.h +++ b/paddle/fluid/operators/tensorrt_engine_op.h @@ -19,10 +19,14 @@ #include "paddle/fluid/framework/operator.h" #include "paddle/fluid/inference/analysis/helper.h" #include "paddle/fluid/inference/tensorrt/engine.h" +#include "paddle/fluid/inference/tensorrt/engine.h" namespace paddle { namespace operators { +using inference::Singleton; +using inference::tensorrt::TRT_EngineManager; + class TensorRTEngineOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -47,16 +51,18 @@ template class TensorRTEngineKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - if (!engine_) { + auto engine_name = context.Attr("engine_uniq_key"); + if (!Singleton::Global().HasEngine(engine_name)) { Prepare(context); } + auto* engine = Singleton::Global().Get(engine_name); auto input_names = context.op().Inputs("Xs"); PADDLE_ENFORCE(!input_names.empty(), "should pass more than one inputs"); // Try to determine a batch_size auto& tensor0 = inference::analysis::GetFromScope( context.scope(), input_names.front()); int batch_size = tensor0.dims()[0]; - PADDLE_ENFORCE_LE(batch_size, max_batch_); + PADDLE_ENFORCE_LE(batch_size, context.Attr("max_batch")); // Convert input tensor from fluid to engine. for (const auto& x : context.Inputs("Xs")) { @@ -64,20 +70,20 @@ class TensorRTEngineKernel : public framework::OpKernel { auto& t = inference::analysis::GetFromScope( context.scope(), x); if (platform::is_cpu_place(t.place())) { - engine_->SetInputFromCPU(x, static_cast(t.data()), - t.memory_size()); + engine->SetInputFromCPU(x, static_cast(t.data()), + t.memory_size()); } else { - engine_->SetInputFromGPU(x, static_cast(t.data()), - t.memory_size()); + engine->SetInputFromGPU(x, static_cast(t.data()), + t.memory_size()); } } // Execute the engine. PADDLE_ENFORCE_GT(batch_size, 0); - engine_->Execute(batch_size); + engine->Execute(batch_size); // Convert output tensor from engine to fluid for (const auto& y : context.Outputs("Ys")) { // convert output and copy to fluid. - nvinfer1::ITensor* trt_t = engine_->GetITensor(y); + nvinfer1::ITensor* trt_t = engine->GetITensor(y); auto dims = trt_t->getDimensions(); // Use the output ITensor's dims to reshape the Fluid Tensor. std::vector ddim(dims.d, dims.d + dims.nbDims); @@ -89,27 +95,22 @@ class TensorRTEngineKernel : public framework::OpKernel { auto size = inference::analysis::AccuDims(dims.d, dims.nbDims); if (platform::is_cpu_place(fluid_t->place())) { // TODO(Superjomn) change this float to dtype size. - engine_->GetOutputInCPU( + engine->GetOutputInCPU( y, fluid_t->mutable_data(platform::CPUPlace()), size * sizeof(float)); } else { - engine_->GetOutputInGPU( + engine->GetOutputInGPU( y, fluid_t->mutable_data(platform::CUDAPlace()), size * sizeof(float)); } } - cudaStreamSynchronize(stream_); + cudaStreamSynchronize(*engine->stream()); } protected: // Build the engine. void Prepare(const framework::ExecutionContext& context) const; - - private: - mutable cudaStream_t stream_; - mutable inference::tensorrt::TensorRTEngine* engine_{nullptr}; - mutable int max_batch_{0}; }; } // namespace operators diff --git a/paddle/fluid/operators/tensorrt_engine_op_test.cc b/paddle/fluid/operators/tensorrt_engine_op_test.cc index 6f383de259..85330958cd 100644 --- a/paddle/fluid/operators/tensorrt_engine_op_test.cc +++ b/paddle/fluid/operators/tensorrt_engine_op_test.cc @@ -79,6 +79,17 @@ void SetAttr(framework::proto::OpDesc* op, const std::string& name, attr->set_type(paddle::framework::proto::AttrType::LONG); attr->set_l(data); } +template <> +void SetAttr>(framework::proto::OpDesc* op, + const std::string& name, + const std::vector& data) { + auto* attr = op->add_attrs(); + attr->set_name(name); + attr->set_type(paddle::framework::proto::AttrType::STRINGS); + for (const auto& s : data) { + attr->add_strings(s.c_str()); + } +} } // namespace @@ -123,11 +134,15 @@ TEST(TensorRTEngineOp, manual) { engine_op_desc.SetOutput("Ys", std::vector({"z0"})); SetAttr(engine_op_desc.Proto(), "subgraph", block_->SerializeAsString()); - SetAttr(engine_op_desc.Proto(), "max_batch", 30); + SetAttr(engine_op_desc.Proto(), "max_batch", 100); SetAttr(engine_op_desc.Proto(), "max_workspace", 1 << 10); + SetAttr(engine_op_desc.Proto(), "engine_uniq_key", "a_engine"); + SetAttr>(engine_op_desc.Proto(), "parameters", + std::vector({})); LOG(INFO) << "create engine op"; auto engine_op = framework::OpRegistry::CreateOp(*engine_op_desc.Proto()); + LOG(INFO) << "engine_op " << engine_op.get(); framework::Scope scope; platform::CPUPlace place; @@ -145,6 +160,88 @@ TEST(TensorRTEngineOp, manual) { engine_op->Run(scope, place); } +void Execute(int batch_size, int input_dim, int output_dim, int nlayers = 1) { + framework::ProgramDesc program; + framework::Scope scope; + platform::CPUPlace place; + platform::CPUDeviceContext ctx(place); + + auto* block_ = program.Proto()->add_blocks(); + block_->set_idx(0); + block_->set_parent_idx(-1); + + using shape_t = std::vector; + + LOG(INFO) << "create block desc"; + framework::BlockDesc block_desc(&program, block_); + + auto AddFCLayer = [&](const std::string& x_name, const std::string& y_name, + const std::string& z_name, bool x_created, + const shape_t& x_shape, const shape_t& y_shape, + const shape_t& z_shape) { + + LOG(INFO) << "create fc op"; + auto* fc = block_desc.AppendOp(); + fc->SetType("mul"); + fc->SetInput("X", std::vector({x_name})); + fc->SetInput("Y", std::vector({y_name})); + fc->SetOutput("Out", std::vector({z_name})); + + // Set inputs' variable shape in BlockDesc + if (!x_created) { + AddTensorToBlockDesc(block_, x_name, + std::vector({batch_size, input_dim, 1, 1})); + } + AddTensorToBlockDesc(block_, y_name, + std::vector({input_dim, output_dim})); + AddTensorToBlockDesc(block_, z_name, + std::vector({batch_size, output_dim})); + + // Prepare variables. + if (!x_created) { + CreateCPUTensor(&scope, x_name, std::vector(x_shape)); + } + CreateCPUTensor(&scope, y_name, std::vector(y_shape)); + CreateCPUTensor(&scope, z_name, std::vector(z_shape)); + + // It is wired, need to copy manually. + *block_->add_ops() = *fc->Proto(); + }; + + // Test with 4 layer FC + AddFCLayer("x0", "y0", "z0", false, {batch_size, input_dim}, + {input_dim, output_dim}, {batch_size, output_dim}); + AddFCLayer("z0", "y1", "z1", true, {}, {output_dim, output_dim}, + {batch_size, output_dim}); + AddFCLayer("z1", "y2", "z2", true, {}, {output_dim, output_dim}, + {batch_size, output_dim}); + AddFCLayer("z2", "y3", "z3", true, {}, {output_dim, output_dim}, + {batch_size, output_dim}); + + LOG(INFO) << "create tensorrt desc"; + framework::OpDesc engine_op_desc(nullptr); + engine_op_desc.SetType("tensorrt_engine"); + engine_op_desc.SetInput("Xs", std::vector({"x0"})); + engine_op_desc.SetOutput("Ys", std::vector({"z3"})); + + SetAttr(engine_op_desc.Proto(), "subgraph", + block_->SerializeAsString()); + SetAttr(engine_op_desc.Proto(), "max_batch", batch_size); + SetAttr(engine_op_desc.Proto(), "max_workspace", 2 << 10); + SetAttr>( + engine_op_desc.Proto(), "parameters", + std::vector({"y0", "y1", "y2", "y3"})); + SetAttr(engine_op_desc.Proto(), "engine_uniq_key", "b_engine"); + + auto engine_op = framework::OpRegistry::CreateOp(*engine_op_desc.Proto()); + + // Execute them. + engine_op->Run(scope, place); +} + +// Test with a larger FC layer. +TEST(TensorRTEngineOp, fc) { Execute(40, 256, 256); } + } // namespace operators } // namespace paddle From fd87c0e709227f45ca13562a9fb7aa0f56f3efb9 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Jun 2018 23:15:22 -0700 Subject: [PATCH 105/154] Fix cast layer's doc --- python/paddle/fluid/layers/tensor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 04efc40af5..91a3915367 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -111,8 +111,21 @@ def create_global_var(shape, def cast(x, dtype): """ - This function takes in the input with input_dtype - and casts it to the output_dtype as the output. + This layer takes in the Variable :attr:`x` with :attr:`x.dtype` and casts + it to the output with :attr:`dtype`. + + Args: + x (Variable): The input Variable for casting. + dtype(np.dtype|core.VarDesc.VarType|str): Data type of the output Variable. + + Returns: + Variable: The output Variable after casting. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name='x', shape=[13], dtype='float32') + result = fluid.layers.cast(x=data, dtype='float64') """ helper = LayerHelper('cast', **locals()) out = helper.create_tmp_variable(dtype=dtype) From e2783bb6afeb4e5b4160ff4283c18672f2f0632e Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 15 Jun 2018 14:22:21 +0800 Subject: [PATCH 106/154] update split_lod_tensor doc --- python/paddle/fluid/layers/control_flow.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 2bc43c5ce9..e261e3f63a 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -69,8 +69,10 @@ def split_lod_tensor(input, mask, level=0): level(int): The specific lod level to split. Returns: - Variable: The true branch of tensor as per the mask applied to input. - Variable: The false branch of tensor as per the mask applied to input. + tuple(Variable, Variable): + The true branch of tensor as per the mask applied to input. + + The false branch of tensor as per the mask applied to input. Examples: .. code-block:: python From f2c9c33f158890e0cfba827db3a328c317962d4c Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 23:24:32 -0700 Subject: [PATCH 107/154] "fix lrn" --- python/paddle/fluid/layers/nn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 283d3c4b2a..2d33742e73 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -839,9 +839,9 @@ def linear_chain_crf(input, label, param_attr=None): param_attr(ParamAttr): The attribute of the learnable parameter. Returns: - ${log_likelihood_comment} - ${transitionexps_comment} - ${emissionexps_comment} + output(${emission_exps_type}): ${emission_exps_comment} \n + output(${transition_exps_type}): ${transition_exps_comment} \n + output(${log_likelihood_type}): ${log_likelihood_comment} """ helper = LayerHelper('linear_chain_crf', **locals()) @@ -4210,7 +4210,7 @@ def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): .. math:: - Output(i, x, y) = Input(i, x, y) / \left( \\ + Output(i, x, y) = Input(i, x, y) / \left( \\ k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} \\ (Input(j, x, y))^2\right)^{\beta} From 1958654d6f15087c28b44759c1a8d004826f00ce Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 15 Jun 2018 14:28:17 +0800 Subject: [PATCH 108/154] refine \odot in elementwise_mul --- paddle/fluid/operators/elementwise_mul_op.cc | 2 +- .../fluid/layers/layer_function_generator.py | 28 +++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/operators/elementwise_mul_op.cc b/paddle/fluid/operators/elementwise_mul_op.cc index ba343909bb..7cd67e74de 100644 --- a/paddle/fluid/operators/elementwise_mul_op.cc +++ b/paddle/fluid/operators/elementwise_mul_op.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include "paddle/fluid/operators/elementwise_mul_op.h" #include "paddle/fluid/operators/elementwise_op.h" namespace ops = paddle::operators; -REGISTER_ELEMWISE_OP(elementwise_mul, "Mul", "Out = X \\odot\\ Y"); +REGISTER_ELEMWISE_OP(elementwise_mul, "Mul", "Out = X \\\\odot Y"); REGISTER_OP_CPU_KERNEL( elementwise_mul, ops::ElementwiseMulKernel, diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index cb60a3aec9..0f05ea2b08 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -44,6 +44,11 @@ def _type_to_str_(tp): return framework_pb2.AttrType.Name(tp) +_two_dollar_pattern_ = re.compile(r"\$\$([^\$]+)\$\$") +_single_dollar_pattern_ = re.compile(r"\$([^\$]+)\$") +_two_bang_pattern_ = re.compile(r"!!([^!]+)!!") + + def _generate_doc_string_(op_proto): """ Generate docstring by OpProto @@ -55,22 +60,27 @@ def _generate_doc_string_(op_proto): str: the document string """ + def escape_math(text): + return _two_bang_pattern_.sub( + r'$$\1$$', + _single_dollar_pattern_.sub( + r':math:`\1`', _two_dollar_pattern_.sub(r"!!\1!!", text))) + if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError("OpProto should be `framework_pb2.OpProto`") buf = cStringIO.StringIO() - buf.write(op_proto.comment) + buf.write(escape_math(op_proto.comment)) buf.write('\nArgs:\n') for each_input in op_proto.inputs: line_begin = ' {0}: '.format(_convert_(each_input.name)) buf.write(line_begin) - buf.write(each_input.comment) + buf.write(escape_math(each_input.comment)) buf.write('\n') - buf.write(' ' * len(line_begin)) - buf.write('Duplicable: ') - buf.write(str(each_input.duplicable)) - buf.write(' Optional: ') - buf.write(str(each_input.dispensable)) + if each_input.duplicable: + buf.write(" Duplicatable.") + if each_input.dispensable: + buf.write(" Optional.") buf.write('\n') skip_attrs = OpProtoHolder.generated_op_attr_names() @@ -83,7 +93,7 @@ def _generate_doc_string_(op_proto): buf.write(' (') buf.write(_type_to_str_(each_attr.type)) buf.write('): ') - buf.write(each_attr.comment) + buf.write(escape_math(each_attr.comment)) buf.write('\n') if len(op_proto.outputs) != 0: @@ -92,7 +102,7 @@ def _generate_doc_string_(op_proto): for each_opt in op_proto.outputs: if not each_opt.intermediate: break - buf.write(each_opt.comment) + buf.write(escape_math(each_opt.comment)) return buf.getvalue() From 3571df8755b5e5566360ad07fd2682f1f031454a Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Fri, 15 Jun 2018 14:35:40 +0800 Subject: [PATCH 109/154] Remove unused '\n' in comments --- python/paddle/fluid/layers/layer_function_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index 0f05ea2b08..7a95afa9a6 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -76,7 +76,6 @@ def _generate_doc_string_(op_proto): line_begin = ' {0}: '.format(_convert_(each_input.name)) buf.write(line_begin) buf.write(escape_math(each_input.comment)) - buf.write('\n') if each_input.duplicable: buf.write(" Duplicatable.") if each_input.dispensable: From 279ebdd0b2ab40172d913a3eb1d051a41f24ddb7 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Jun 2018 23:40:52 -0700 Subject: [PATCH 110/154] Fix reciprocal op's doc --- paddle/fluid/operators/activation_op.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index c73482eb12..c51e63982c 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -19,18 +19,18 @@ limitations under the License. */ namespace paddle { namespace operators { -#define REGISTER_ACTIVATION_OP_MAKER(OP_NAME, OP_COMMENT) \ - class OP_NAME##OpMaker \ - : public ::paddle::framework::OpProtoAndCheckerMaker { \ - public: \ - void Make() override { \ - AddInput("X", "Input of " #OP_NAME " operator"); \ - AddOutput("Out", "Output of " #OP_NAME " operator").Reuse("X"); \ - AddAttr("use_mkldnn", \ - "(bool, default false) Only used in mkldnn kernel") \ - .SetDefault(false); \ - AddComment(OP_COMMENT); \ - } \ +#define REGISTER_ACTIVATION_OP_MAKER(OP_NAME, OP_COMMENT) \ + class OP_NAME##OpMaker \ + : public ::paddle::framework::OpProtoAndCheckerMaker { \ + public: \ + void Make() override { \ + AddInput("X", "Input of " #OP_NAME " operator"); \ + AddOutput("Out", "Output of " #OP_NAME " operator").Reuse("X"); \ + AddAttr("use_mkldnn", \ + "(default false) Only used in mkldnn kernel") \ + .SetDefault(false); \ + AddComment(OP_COMMENT); \ + } \ } #define REGISTER_ACTIVATION_OP_GRAD_MAKER(OP_NAME, KERNEL_TYPE) \ From 0d86f13ce2455a1ae4f24d4e5550d6256530122f Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 14 Jun 2018 23:43:18 -0700 Subject: [PATCH 111/154] "fix math" --- python/paddle/fluid/layers/nn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 2d33742e73..8c8d6328eb 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4210,9 +4210,7 @@ def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): .. math:: - Output(i, x, y) = Input(i, x, y) / \left( \\ - k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} \\ - (Input(j, x, y))^2\right)^{\beta} + Output(i, x, y) = Input(i, x, y) / \\left(k + \\alpha \\sum\\limits^{\\min(C, c + n/2)}_{j = \\max(0, c - n/2)}(Input(j, x, y))^2\\right)^{\\beta} In the above equation: From 1c9fc655d0b4745f74940f99acc8421faf8656f5 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 15 Jun 2018 15:16:14 +0800 Subject: [PATCH 112/154] update --- python/paddle/fluid/layers/detection.py | 73 ++++++++++--------- .../fluid/layers/learning_rate_scheduler.py | 12 ++- python/paddle/fluid/layers/tensor.py | 8 +- 3 files changed, 48 insertions(+), 45 deletions(-) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index edf528a595..dacb31f8b6 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -603,7 +603,7 @@ def prior_box(input, offset=0.5, name=None): """ - **Prior box operator** + **Prior Box Operator** Generate prior boxes for SSD(Single Shot MultiBox Detector) algorithm. Each position of the input produce N prior boxes, N is determined by @@ -632,26 +632,30 @@ def prior_box(input, name(str): Name of the prior box op. Default: None. Returns: - boxes(Variable): the output prior boxes of PriorBox. - The layout is [H, W, num_priors, 4]. - H is the height of input, W is the width of input, - num_priors is the total - box count of each position of input. - Variances(Variable): the expanded variances of PriorBox. - The layout is [H, W, num_priors, 4]. - H is the height of input, W is the width of input - num_priors is the total - box count of each position of input + tuple: A tuple with two Variable (boxes, variances) + + boxes: the output prior boxes of PriorBox. + The layout is [H, W, num_priors, 4]. + H is the height of input, W is the width of input, + num_priors is the total + box count of each position of input. + + variances: the expanded variances of PriorBox. + The layout is [H, W, num_priors, 4]. + H is the height of input, W is the width of input + num_priors is the total + box count of each position of input Examples: .. code-block:: python - box, var = prior_box( - input=conv1, - image=images, - min_sizes=[100.], - flip=True, - clip=True) + + box, var = fluid.layers.prior_box( + input=conv1, + image=images, + min_sizes=[100.], + flip=True, + clip=True) """ helper = LayerHelper("prior_box", **locals()) dtype = helper.input_dtype() @@ -721,11 +725,9 @@ def multi_box_head(inputs, stride=1, name=None): """ - **Prior_boxes** - Generate prior boxes for SSD(Single Shot MultiBox Detector) algorithm. The details of this algorithm, please refer the - section 2.2 of SSD paper (SSD: Single Shot MultiBox Detector) + section 2.2 of SSD paper `SSD: Single Shot MultiBox Detector `_ . Args: @@ -766,24 +768,27 @@ def multi_box_head(inputs, name(str): Name of the prior box layer. Default: None. Returns: - mbox_loc(Variable): The predicted boxes' location of the inputs. - The layout is [N, H*W*Priors, 4]. where Priors - is the number of predicted boxes each position of each input. - mbox_conf(Variable): The predicted boxes' confidence of the inputs. - The layout is [N, H*W*Priors, C]. where Priors - is the number of predicted boxes each position of each input - and C is the number of Classes. - boxes(Variable): the output prior boxes of PriorBox. - The layout is [num_priors, 4]. num_priors is the total - box count of each position of inputs. - Variances(Variable): the expanded variances of PriorBox. - The layout is [num_priors, 4]. num_priors is the total - box count of each position of inputs + tuple: A tuple with four Variables. (mbox_loc, mbox_conf, boxes, variances) + + mbox_loc: The predicted boxes' location of the inputs. The layout + is [N, H*W*Priors, 4]. where Priors is the number of predicted + boxes each position of each input. + + mbox_conf: The predicted boxes' confidence of the inputs. The layout + is [N, H*W*Priors, C]. where Priors is the number of predicted boxes + each position of each input and C is the number of Classes. + + boxes: the output prior boxes of PriorBox. The layout is [num_priors, 4]. + num_priors is the total box count of each position of inputs. + + variances: the expanded variances of PriorBox. The layout is + [num_priors, 4]. num_priors is the total box count of each position of inputs Examples: .. code-block:: python - mbox_locs, mbox_confs, box, var = layers.multi_box_head( + + mbox_locs, mbox_confs, box, var = fluid.layers.multi_box_head( inputs=[conv1, conv2, conv3, conv4, conv5, conv5], image=images, num_classes=21, diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 2dbc51c23f..e76f15d838 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -163,8 +163,6 @@ def polynomial_decay(learning_rate, power=1.0, cycle=False): """ - **Polynomial Decay** - Applies polynomial decay to the initial learning rate. .. code-block:: python @@ -178,14 +176,14 @@ def polynomial_decay(learning_rate, Args: learning_rate(Variable|float32): A scalar float32 value or a Variable. This - will be the initial learning rate during training + will be the initial learning rate during training. decay_steps(int32): A Python `int32` number. - end_learning_rate(float, Default: 0.0001): A Python `float` number. - power(float, Default: 1.0): A Python `float` number - cycle(bool, Default: False): Boolean. If set true, decay the learning rate every decay_steps. + end_learning_rate(float): A Python `float` number. + power(float): A Python `float` number. + cycle(bool): If set true, decay the learning rate every decay_steps. Returns: - The decayed learning rate + Variable: The decayed learning rate """ global_step = _decay_step_counter() diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 25505e4427..978f7dde29 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -40,14 +40,14 @@ __all__ = [ def create_tensor(dtype, name=None, persistable=False): """ - **Create a Tensor** + Create an variable, which will hold a LoDTensor with data type dtype. Args: - dtype (string): 'float32'|'int32'|..., the data type of the + dtype(string): 'float32'|'int32'|..., the data type of the created tensor. - name (string, Default: None): The name of the created tensor, if not set, + name(string): The name of the created tensor, if not set, the name will be a random unique one. - persistable (bool, Default: False): Set the persistable flag of the create tensor. + persistable(bool): Set the persistable flag of the create tensor. Returns: Variable: The tensor variable storing the created tensor. From bc46c527768f03ab06701d9cd48b877ddc04957d Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Fri, 15 Jun 2018 15:17:45 +0800 Subject: [PATCH 113/154] Add doc for while op --- python/paddle/fluid/layers/control_flow.py | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 4fec682507..29713dcea9 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -607,6 +607,60 @@ class WhileGuard(BlockGuard): class While(object): + """ + while loop control flow. + + Args: + cond (Variable): condition used to compare. + name (str): The name of this layer. + + Examples: + .. code-block:: python + + # The value these d0, d1 and d2 can be fed from python. + d0 = fluid.layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32') + d1 = fluid.layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32') + d2 = fluid.layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32') + i = fluid.layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = True + init = fluid.layers.zeros(shape=[10], dtype='float32') + # Initialize mem_array from init + mem_array = fluid.layers.array_write(x=init, i=i) + # Initialize data_array from d0 + data_array = fluid.layers.array_write(x=d0, i=i) + # Set a value to data_array using d1[i]. + i = fluid.layers.increment(i) + fluid.layers.array_write(d1, i, array=data_array) + # Set a value to data_array using d2[i]. + i = fluid.layers.increment(i) + fluid.layers.array_write(d2, i, array=data_array) + # Create a idx to start the while loop. + i = fluid.layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = True + + array_len = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=3) + array_len.stop_gradient = True + # Create the while loop condition. + cond = fluid.layers.less_than(x=i, y=array_len) + + # Within the loop, perform sums. + while_op = fluid.layers.While(cond=cond) + with while_op.block(): + d = fluid.layers.array_read(array=data_array, i=i) + prev = fluid.layers.array_read(array=mem_array, i=i) + result = fluid.layers.sums(input=[d, prev]) + + i = fluid.layers.increment(x=i, in_place=True) + fluid.layers.array_write(result, i=i, array=mem_array) + fluid.layers.less_than(x=i, y=array_len, cond=cond) + + sum_result = fluid.layers.array_read(array=mem_array, i=i) + """ + BEFORE_WHILE_BLOCK = 0 IN_WHILE_BLOCK = 1 AFTER_WHILE_BLOCK = 2 From 35f64cb905a1ceff5087a1b3f785136d586008e3 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 15 Jun 2018 15:21:20 +0800 Subject: [PATCH 114/154] skip all tests in tests_parallel_executor_crf --- .../fluid/tests/unittests/test_parallel_executor_crf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index 163975555e..1ea7a6a568 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -173,6 +173,7 @@ class TestCRFModel(unittest.TestCase): pe.run(feed=feeder.feed(cur_batch), fetch_list=[avg_cost.name]))[0] + @unittest.skip(reason="CI hangs") def test_update_sparse_parameter_all_reduce(self): build_strategy = fluid.BuildStrategy() build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce @@ -181,6 +182,7 @@ class TestCRFModel(unittest.TestCase): self.check_network_convergence( is_sparse=True, build_strategy=build_strategy, use_cuda=False) + @unittest.skip(reason="CI hangs") def test_update_dense_parameter_all_reduce(self): build_strategy = fluid.BuildStrategy() build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce @@ -189,6 +191,7 @@ class TestCRFModel(unittest.TestCase): self.check_network_convergence( is_sparse=False, build_strategy=build_strategy, use_cuda=False) + @unittest.skip(reason="CI hangs") def test_update_sparse_parameter_reduce(self): build_strategy = fluid.BuildStrategy() build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce @@ -197,6 +200,7 @@ class TestCRFModel(unittest.TestCase): self.check_network_convergence( is_sparse=True, build_strategy=build_strategy, use_cuda=False) + @unittest.skip(reason="CI hangs") def test_update_dense_parameter_reduce(self): build_strategy = fluid.BuildStrategy() build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce From 6ac8383f28a572cbfcd6d2ec87a87174faa26a12 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Fri, 15 Jun 2018 00:28:02 -0700 Subject: [PATCH 115/154] "fix based comments" --- .../operators/uniform_random_batch_size_like_op.cc | 2 +- python/paddle/fluid/framework.py | 10 ++++++++-- python/paddle/fluid/layers/control_flow.py | 3 +-- python/paddle/fluid/layers/nn.py | 2 -- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/operators/uniform_random_batch_size_like_op.cc b/paddle/fluid/operators/uniform_random_batch_size_like_op.cc index 192366fd44..75d6181749 100644 --- a/paddle/fluid/operators/uniform_random_batch_size_like_op.cc +++ b/paddle/fluid/operators/uniform_random_batch_size_like_op.cc @@ -38,7 +38,7 @@ class UniformRandomBatchSizeLikeOpMaker : public BatchSizeLikeOpMaker { UniformRandomBatchSizeLike operator. This operator initializes a tensor with the same batch_size as the Input tensor - with random values sampled from a uniform distribution. +with random values sampled from a uniform distribution. )DOC"); AddAttr("min", diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 595f67961f..df0625649d 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -1130,6 +1130,8 @@ class Program(object): def clone(self, for_test=False): """Clone the Program object + Args: + for_test(bool): indicate whether clone for test. Set for_test to False when we want to clone the program for training. Set for_test to True when we want to clone the program for testing. @@ -1140,8 +1142,9 @@ class Program(object): the is_test attributes in these operators will be set to True for testing purposes, otherwise, they remain unchanged. - Returns(Program): - The cloned Program object. + Returns: + Program: The cloned Program object. + """ if for_test: p = self.inference_optimize() @@ -1259,6 +1262,7 @@ class Program(object): def copy_param_info_from(self, other): """ Copy the information of parameters from other program. + Args: other(Program): Other program @@ -1277,6 +1281,7 @@ class Program(object): def copy_data_info_from(self, other): """ Copy the information of data variables from other program. + Args: other(Program): Other program @@ -1330,6 +1335,7 @@ class Parameter(Variable): def to_string(self, throw_on_error, with_details=False): """ To debug string. + Args: throw_on_error(bool): raise exception when self is not initialized when throw_on_error is True diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index f4e95dd363..df2f9dedab 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -823,8 +823,7 @@ def increment(x, value=1.0, in_place=True): in_place (bool): If the increment should be performed in-place. Returns: - Variable: The tensor variable storing the transformation of - element-wise increment of each value in the input. + Variable: The elementwise-incremented object. Examples: .. code-block:: python diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 8c8d6328eb..4fd353171a 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -2061,8 +2061,6 @@ def beam_search_decode(ids, scores, name=None): based on the score of each id. This layer takes the output of beam search layer as input and repack them into sentences. - ${beam_search_decode} - Args: ids (Variable): The selected ids, output of beam search layer. scores (Variable): The associated scores of the ids, out put of beam From 67dc5c7f8aab84df120374d0f8671a56127f3e52 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 00:29:37 -0700 Subject: [PATCH 116/154] Polish the doc of nce layer --- paddle/fluid/operators/nce_op.cc | 6 ++++-- python/paddle/fluid/layers/nn.py | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/nce_op.cc b/paddle/fluid/operators/nce_op.cc index 06092e680a..e471f04662 100644 --- a/paddle/fluid/operators/nce_op.cc +++ b/paddle/fluid/operators/nce_op.cc @@ -128,8 +128,10 @@ class NCEOpMaker : public framework::OpProtoAndCheckerMaker { "user should avoid setting this attribute.") .SetDefault({}); AddComment(R"DOC( -Compute and return the noise-contrastive estimation training loss. -See [Noise-contrastive estimation: A new estimation principle for unnormalized statistical models](http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf). +Compute and return the noise-contrastive estimation training loss. See +`Noise-contrastive estimation: A new estimation principle for unnormalized +statistical models + `_. By default this operator uses a uniform distribution for sampling. )DOC"); } diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 27fbb0f053..0a45098bda 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3472,7 +3472,33 @@ def nce(input, num_neg_samples (int): ${num_neg_samples_comment} Returns: - Variable: output of nce layer. + Variable: The output nce loss. + + Examples: + .. code-block:: python + + window_size = 5 + words = [] + for i in xrange(window_size): + words.append(layers.data( + name='word_{0}'.format(i), shape=[1], dtype='int64')) + + dict_size = 10000 + label_word = int(window_size / 2) + 1 + + embs = [] + for i in xrange(window_size): + if i == label_word: + continue + + emb = layers.embedding(input=words[i], size=[dict_size, 32], + param_attr='emb.w', is_sparse=True) + embs.append(emb) + + embs = layers.concat(input=embs, axis=1) + loss = layers.nce(input=embs, label=words[label_word], + num_total_classes=dict_size, param_attr='nce.w', + bias_attr='nce.b') """ helper = LayerHelper('nce', **locals()) assert isinstance(input, Variable) From b00fbd962c0bff30307bcc833761e6a0e2ca075a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 15 Jun 2018 15:30:04 +0800 Subject: [PATCH 117/154] fix error --- python/paddle/fluid/layers/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index e397515ff6..b004312d2d 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -637,10 +637,10 @@ def read_file(reader): out = [ helper.create_tmp_variable( stop_gradient=True, dtype='float32') - for _ in range(len(file_obj.desc.shapes())) + for _ in range(len(reader.desc.shapes())) ] helper.append_op( - type='read', inputs={'Reader': [file_obj]}, outputs={'Out': out}) + type='read', inputs={'Reader': [reader]}, outputs={'Out': out}) if len(out) == 1: return out[0] else: From 7ad46ec03ccc1961bb1a8b22e178ce5406cfbee2 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Fri, 15 Jun 2018 00:36:19 -0700 Subject: [PATCH 118/154] "show example" --- python/paddle/fluid/layers/nn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 4fd353171a..b49560a526 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -2076,6 +2076,7 @@ def beam_search_decode(ids, scores, name=None): Examples: .. code-block:: python + ids, scores = fluid.layers.beam_search( pre_ids, ids, scores, beam_size, end_id) sentence_ids, sentence_scores = fluid.layers.beam_search_decode( From 7b82353010976533ad10df80637fd88b4d26c627 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 15 Jun 2018 15:06:57 +0800 Subject: [PATCH 119/154] fix conv3d/conv3d_trans/slice/mean_iou doc --- paddle/fluid/operators/slice_op.cc | 37 ++++++++------- python/paddle/fluid/layers/nn.py | 72 +++++++++++++++--------------- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/paddle/fluid/operators/slice_op.cc b/paddle/fluid/operators/slice_op.cc index 61bb445e8b..4bd23d5941 100644 --- a/paddle/fluid/operators/slice_op.cc +++ b/paddle/fluid/operators/slice_op.cc @@ -95,23 +95,26 @@ of that dimension. If the value passed to start or end is larger than the n (the number of elements in this dimension), it represents n. For slicing to the end of a dimension with unknown size, it is recommended to pass in INT_MAX. If axes are omitted, they are set to [0, ..., ndim-1]. - - Example 1: - Given: - data = [ [1, 2, 3, 4], [5, 6, 7, 8], ] - axes = [0, 1] - starts = [1, 0] - ends = [2, 3] - Then: - result = [ [5, 6, 7], ] - - Example 2: - Given: - data = [ [1, 2, 3, 4], [5, 6, 7, 8], ] - starts = [0, 1] - ends = [-1, 1000] - Then: - result = [ [2, 3, 4], ] +Following examples will explain how slice works: + + .. code-block:: text + + Cast1: + Given: + data = [ [1, 2, 3, 4], [5, 6, 7, 8], ] + axes = [0, 1] + starts = [1, 0] + ends = [2, 3] + Then: + result = [ [5, 6, 7], ] + + Cast2: + Given: + data = [ [1, 2, 3, 4], [5, 6, 7, 8], ] + starts = [0, 1] + ends = [-1, 1000] + Then: + result = [ [2, 3, 4], ] )DOC"); } }; diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 5e162a4ae0..d1985efc58 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1326,10 +1326,8 @@ def conv2d(input, Examples: .. code-block:: python - data = fluid.layers.data( - name='data', shape=[3, 32, 32], dtype='float32') - conv2d = fluid.layers.conv2d( - input=data, num_filters=2, filter_size=3, act="relu") + data = fluid.layers.data(name='data', shape=[3, 32, 32], dtype='float32') + conv2d = fluid.layers.conv2d(input=data, num_filters=2, filter_size=3, act="relu") """ num_channels = input.shape[1] @@ -1431,8 +1429,7 @@ def conv3d(input, * :math:`\\ast`: Convolution operation. * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. * :math:`\\sigma`: Activation function. - * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be - different. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. Example: @@ -1494,10 +1491,8 @@ def conv3d(input, Examples: .. code-block:: python - data = fluid.layers.data( - name='data', shape=[3, 12, 32, 32], dtype='float32') - conv2d = fluid.layers.conv3d( - input=data, num_filters=2, filter_size=3, act="relu") + data = fluid.layers.data(name='data', shape=[3, 12, 32, 32], dtype='float32') + conv3d = fluid.layers.conv3d(input=data, num_filters=2, filter_size=3, act="relu") """ l_type = 'conv3d' @@ -2105,32 +2100,36 @@ def conv2d_transpose(input, represent height and width, respectively. The details of convolution transpose layer, please refer to the following explanation and references `therein `_. + If bias attribution and activation type are provided, bias is added to + the output of the convolution, and the corresponding activation function + is applied to the final result. For each input :math:`X`, the equation is: .. math:: - Out = W \\ast X + Out = \sigma (W \\ast X + b) - In the above equation: + Where: * :math:`X`: Input value, a tensor with NCHW format. * :math:`W`: Filter value, a tensor with MCHW format. - * :math:`\\ast` : Convolution transpose operation. - * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be - different. + * :math:`\\ast`: Convolution operation. + * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. + * :math:`\\sigma`: Activation function. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. Example: - Input: - Input shape: $(N, C_{in}, H_{in}, W_{in})$ + Input shape: :math:`(N, C_{in}, H_{in}, W_{in})` - Filter shape: $(C_{in}, C_{out}, H_f, W_f)$ + Filter shape: :math:`(C_{in}, C_{out}, H_f, W_f)` - Output: - Output shape: $(N, C_{out}, H_{out}, W_{out})$ + Output shape: :math:`(N, C_{out}, H_{out}, W_{out})` Where @@ -2184,10 +2183,8 @@ def conv2d_transpose(input, Examples: .. code-block:: python - data = fluid.layers.data( - name='data', shape=[3, 32, 32], dtype='float32') - conv2d_transpose = fluid.layers.conv2d_transpose( - input=data, num_filters=2, filter_size=3) + data = fluid.layers.data(name='data', shape=[3, 32, 32], dtype='float32') + conv2d_transpose = fluid.layers.conv2d_transpose(input=data, num_filters=2, filter_size=3) """ helper = LayerHelper("conv2d_transpose", **locals()) if not isinstance(input, Variable): @@ -2267,32 +2264,36 @@ def conv3d_transpose(input, two elements. These two elements represent height and width, respectively. The details of convolution transpose layer, please refer to the following explanation and references `therein `_. + If bias attribution and activation type are provided, bias is added to + the output of the convolution, and the corresponding activation function + is applied to the final result. For each input :math:`X`, the equation is: .. math:: - Out = W \\ast X + Out = \sigma (W \\ast X + b) In the above equation: * :math:`X`: Input value, a tensor with NCDHW format. * :math:`W`: Filter value, a tensor with MCDHW format. - * :math:`\\ast` : Convolution transpose operation. - * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be - different. + * :math:`\\ast`: Convolution operation. + * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. + * :math:`\\sigma`: Activation function. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. Example: - Input: - Input shape: $(N, C_{in}, D_{in}, H_{in}, W_{in})$ + Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})` - Filter shape: $(C_{in}, C_{out}, D_f, H_f, W_f)$ + Filter shape: :math:`(C_{in}, C_{out}, D_f, H_f, W_f)` - Output: - Output shape: $(N, C_{out}, D_{out}, H_{out}, W_{out})$ + Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` Where @@ -2347,10 +2348,8 @@ def conv3d_transpose(input, Examples: .. code-block:: python - data = fluid.layers.data( - name='data', shape=[3, 12, 32, 32], dtype='float32') - conv2d_transpose = fluid.layers.conv3d_transpose( - input=data, num_filters=2, filter_size=3) + data = fluid.layers.data(name='data', shape=[3, 12, 32, 32], dtype='float32') + conv3d_transpose = fluid.layers.conv3d_transpose(input=data, num_filters=2, filter_size=3) """ l_type = "conv3d_transpose" helper = LayerHelper(l_type, **locals()) @@ -4680,8 +4679,8 @@ def mean_iou(input, label, num_classes): IOU is defined as follows: .. math:: - - IOU = true_positive / (true_positive + false_positive + false_negative). + + IOU = \\frac{true\_positiv}{(true\_positive + false\_positive + false\_negative)}. The predictions are accumulated in a confusion matrix and mean-IOU is then calculated from it. @@ -4689,8 +4688,9 @@ def mean_iou(input, label, num_classes): Args: input (Variable): A Tensor of prediction results for semantic labels with type int32 or int64. - label (Variable): A Tensor of ground truth labels with type int32 or int64. + label (Variable): A Tensor of ground truth labels with type int32 or int64. Its shape should be the same as input. + num_classes (int): The possible number of labels. Returns: mean_iou (Variable): A Tensor representing the mean intersection-over-union with shape [1]. From 11f31d1e828090b91d31dc75765d50c09571e764 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Fri, 15 Jun 2018 15:52:21 +0800 Subject: [PATCH 120/154] follow comments --- python/paddle/fluid/layers/control_flow.py | 51 +++++----------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 29713dcea9..128eab7721 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -617,48 +617,17 @@ class While(object): Examples: .. code-block:: python - # The value these d0, d1 and d2 can be fed from python. - d0 = fluid.layers.data( - "d0", shape=[10], append_batch_size=False, dtype='float32') - d1 = fluid.layers.data( - "d1", shape=[10], append_batch_size=False, dtype='float32') - d2 = fluid.layers.data( - "d2", shape=[10], append_batch_size=False, dtype='float32') - i = fluid.layers.zeros(shape=[1], dtype='int64') - i.stop_gradient = True - init = fluid.layers.zeros(shape=[10], dtype='float32') - # Initialize mem_array from init - mem_array = fluid.layers.array_write(x=init, i=i) - # Initialize data_array from d0 - data_array = fluid.layers.array_write(x=d0, i=i) - # Set a value to data_array using d1[i]. - i = fluid.layers.increment(i) - fluid.layers.array_write(d1, i, array=data_array) - # Set a value to data_array using d2[i]. - i = fluid.layers.increment(i) - fluid.layers.array_write(d2, i, array=data_array) - # Create a idx to start the while loop. - i = fluid.layers.zeros(shape=[1], dtype='int64') - i.stop_gradient = True - - array_len = fluid.layers.fill_constant( - shape=[1], dtype='int64', value=3) - array_len.stop_gradient = True - # Create the while loop condition. - cond = fluid.layers.less_than(x=i, y=array_len) - - # Within the loop, perform sums. - while_op = fluid.layers.While(cond=cond) - with while_op.block(): - d = fluid.layers.array_read(array=data_array, i=i) - prev = fluid.layers.array_read(array=mem_array, i=i) - result = fluid.layers.sums(input=[d, prev]) - - i = fluid.layers.increment(x=i, in_place=True) - fluid.layers.array_write(result, i=i, array=mem_array) - fluid.layers.less_than(x=i, y=array_len, cond=cond) + d0 = layers.data("d0", shape=[10], dtype='float32') + data_array = layers.array_write(x=d0, i=i) + array_len = layers.fill_constant(shape=[1],dtype='int64', value=3) - sum_result = fluid.layers.array_read(array=mem_array, i=i) + cond = layers.less_than(x=i, y=array_len) + while_op = layers.While(cond=cond) + with while_op.block(): + d = layers.array_read(array=data_array, i=i) + i = layers.increment(x=i, in_place=True) + layers.array_write(result, i=i, array=d) + layers.less_than(x=i, y=array_len, cond=cond) """ BEFORE_WHILE_BLOCK = 0 From 6ace04f655be6ea7898b5cbe61dfbdb1e16b7806 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 15 Jun 2018 16:00:07 +0800 Subject: [PATCH 121/154] update --- paddle/fluid/operators/activation_op.cc | 2 +- python/paddle/fluid/layers/nn.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index c73482eb12..8743c9500a 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -133,7 +133,7 @@ $out = \max(x, 0)$ __attribute__((unused)) constexpr char TanhDoc[] = R"DOC( Tanh Activation Operator. -$$out = \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$ +$$out = \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$ )DOC"; diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index c6c8c7c2d1..485470f281 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4475,6 +4475,7 @@ def image_resize(input, and the resizing only applies on the last two dimensions(hight and width). Supporting resample methods: + 'BILINEAR' : Bilinear interpolation Args: @@ -4494,8 +4495,8 @@ def image_resize(input, Default: 'BILINEAR' Returns: - out (Variable): The output is a 4-D tensor of the shape - (num_batches, channls, out_h, out_w). + Variable: The output is a 4-D tensor of the shape + (num_batches, channls, out_h, out_w). Examples: .. code-block:: python @@ -4579,7 +4580,7 @@ def image_resize_short(input, out_short_len, resample='BILINEAR'): resample (str): resample method, default: BILINEAR. Returns: - out (Variable): The output is a 4-D tensor of the shape + Variable: The output is a 4-D tensor of the shape (num_batches, channls, out_h, out_w). """ in_shape = input.shape From f24dec713663fd1fd5b678e90e0e7779dbd1bde0 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Fri, 15 Jun 2018 16:06:51 +0800 Subject: [PATCH 122/154] Change 'layers' to 'fluid.layers' --- python/paddle/fluid/layers/control_flow.py | 2 +- python/paddle/fluid/layers/nn.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 8cd389910c..b211bc8377 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1004,7 +1004,7 @@ def array_read(array, i): tmp = fluid.layers.zeros(shape=[10], dtype='int32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10) - arr = layers.array_read(tmp, i=i) + arr = fluid.layers.array_read(tmp, i=i) """ helper = LayerHelper('array_read', **locals()) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index eba22f5962..06fb3504a5 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -2916,9 +2916,9 @@ def warpctc(input, label, blank=0, norm_by_times=False): .. code-block:: python - label = layers.data(shape=[11, 8], dtype='float32', lod_level=1) - predict = layers.data(shape=[11, 1], dtype='float32') - cost = layers.warpctc(input=predict, label=label) + label = fluid.layers.data(shape=[11, 8], dtype='float32', lod_level=1) + predict = fluid.layers.data(shape=[11, 1], dtype='float32') + cost = fluid.layers.warpctc(input=predict, label=label) """ helper = LayerHelper('warpctc', **locals()) @@ -2982,7 +2982,7 @@ def sequence_reshape(input, new_dim): .. code-block:: python x = fluid.layers.data(shape=[5, 20], dtype='float32', lod_level=1) - x_reshaped = layers.sequence_reshape(input=x, new_dim=10) + x_reshaped = fluid.layers.sequence_reshape(input=x, new_dim=10) """ helper = LayerHelper('sequence_reshape', **locals()) out = helper.create_tmp_variable(helper.input_dtype()) From cc1239ffc97e6a5484dd323cfa926fbac9932b4e Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Fri, 15 Jun 2018 16:27:00 +0800 Subject: [PATCH 123/154] Update some doc about API reference. (#11495) * Update some doc about layers' API. * Fix format. * Fix example bug in random_data_generator. * Fix example bug in dropout. * Follow comments and some small fix for some examples. --- paddle/fluid/operators/activation_op.cc | 2 +- .../fluid/operators/detection/box_coder_op.cc | 41 ++++++---- .../gaussian_random_batch_size_like_op.cc | 9 ++- python/paddle/fluid/layers/io.py | 16 ++-- python/paddle/fluid/layers/nn.py | 75 +++++++++++-------- python/paddle/fluid/layers/tensor.py | 20 ++++- 6 files changed, 104 insertions(+), 59 deletions(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 93d1ce7179..bc03ec2f0c 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -112,7 +112,7 @@ $$out = \frac{1}{1 + e^{-x}}$$ __attribute__((unused)) constexpr char LogSigmoidDoc[] = R"DOC( Logsigmoid Activation Operator -$$out = \log \frac{1}{1 + e^{-x}}$$ +$$out = \\log \\frac{1}{1 + e^{-x}}$$ )DOC"; diff --git a/paddle/fluid/operators/detection/box_coder_op.cc b/paddle/fluid/operators/detection/box_coder_op.cc index 8c4b4321b7..d0f95f727f 100644 --- a/paddle/fluid/operators/detection/box_coder_op.cc +++ b/paddle/fluid/operators/detection/box_coder_op.cc @@ -106,23 +106,36 @@ class BoxCoderOpMaker : public framework::OpProtoAndCheckerMaker { "and M represents the number of deocded boxes."); AddComment(R"DOC( -Bounding Box Coder Operator. + +Bounding Box Coder. + Encode/Decode the target bounding box with the priorbox information. + The Encoding schema described below: -ox = (tx - px) / pw / pxv -oy = (ty - py) / ph / pyv -ow = log(abs(tw / pw)) / pwv -oh = log(abs(th / ph)) / phv + + ox = (tx - px) / pw / pxv + + oy = (ty - py) / ph / pyv + + ow = log(abs(tw / pw)) / pwv + + oh = log(abs(th / ph)) / phv + The Decoding schema described below: -ox = (pw * pxv * tx * + px) - tw / 2 -oy = (ph * pyv * ty * + py) - th / 2 -ow = exp(pwv * tw) * pw + tw / 2 -oh = exp(phv * th) * ph + th / 2 -where tx, ty, tw, th denote the target box's center coordinates, width and -height respectively. Similarly, px, py, pw, ph denote the priorbox's(anchor) -center coordinates, width and height. pxv, pyv, pwv, phv denote the variance -of the priorbox and ox, oy, ow, oh denote the encoded/decoded coordinates, -width and height. + + ox = (pw * pxv * tx * + px) - tw / 2 + + oy = (ph * pyv * ty * + py) - th / 2 + + ow = exp(pwv * tw) * pw + tw / 2 + + oh = exp(phv * th) * ph + th / 2 + +where `tx`, `ty`, `tw`, `th` denote the target box's center coordinates, width +and height respectively. Similarly, `px`, `py`, `pw`, `ph` denote the +priorbox's (anchor) center coordinates, width and height. `pxv`, `pyv`, `pwv`, +`phv` denote the variance of the priorbox and `ox`, `oy`, `ow`, `oh` denote the +encoded/decoded coordinates, width and height. )DOC"); } }; diff --git a/paddle/fluid/operators/gaussian_random_batch_size_like_op.cc b/paddle/fluid/operators/gaussian_random_batch_size_like_op.cc index 8050f61d45..4a97428148 100644 --- a/paddle/fluid/operators/gaussian_random_batch_size_like_op.cc +++ b/paddle/fluid/operators/gaussian_random_batch_size_like_op.cc @@ -36,11 +36,12 @@ class GaussianRandomBatchSizeLikeOpMaker : public BatchSizeLikeOpMaker { void Apply() override { AddAttr("mean", "(float, default 0.0) " - "mean of random tensor.") + "The mean (or center) of the gaussian distribution.") .SetDefault(.0f); AddAttr("std", "(float, default 1.0) " - "std of random tensor.") + "The standard deviation (std, or spread) of the " + "gaussian distribution.") .SetDefault(1.0f); AddAttr("seed", "(int, default 0) " @@ -55,9 +56,11 @@ class GaussianRandomBatchSizeLikeOpMaker : public BatchSizeLikeOpMaker { .SetDefault(framework::proto::VarType::FP32); AddComment(R"DOC( -GaussianRandom Operator. Used to initialize tensors with gaussian random generator. +The defalut mean of the distribution is 0. and defalut standard +deviation (std) of the distribution is 1.. Uers can set mean and std +by input arguments. )DOC"); } }; diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index aaf3ff671a..5dc18c633e 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -378,16 +378,16 @@ def random_data_generator(low, high, shapes, lod_levels, for_parallel=True): Variable: A Reader Variable from which we can get random data. Examples: - .. code-block:: python - reader = fluid.layers.io.random_data_generator( - low=0.0, - high=1.0, - shapes=[(3,224,224), (1)], - lod_levels=[0, 0]) + .. code-block:: python - # Via the reader, we can use 'read_file' layer to get data: - image, label = fluid.layers.io.read_file(reader) + reader = fluid.layers.random_data_generator( + low=0.0, + high=1.0, + shapes=[[3,224,224], [1]], + lod_levels=[0, 0]) + # Via the reader, we can use 'read_file' layer to get data: + image, label = fluid.layers.read_file(reader) """ dtypes = [core.VarDesc.VarType.FP32] * len(shapes) shape_concat = [] diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 5e162a4ae0..3816098383 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -364,8 +364,7 @@ def dynamic_lstm(input, cell_activation(str): The activation for cell output. Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". candidate_activation(str): The activation for candidate hidden state. - Choices = ["sigmoid", "tanh", - "relu", "identity"], + Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". dtype(str): Data type. Choices = ["float32", "float64"], default "float32". name(str|None): A name for this layer(optional). If set None, the layer @@ -540,27 +539,31 @@ def dynamic_lstmp(input, cell_activation(str): The activation for cell output. Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". candidate_activation(str): The activation for candidate hidden state. - Choices = ["sigmoid", "tanh", - "relu", "identity"], + Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". proj_activation(str): The activation for projection output. - Choices = ["sigmoid", "tanh", - "relu", "identity"], + Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". dtype(str): Data type. Choices = ["float32", "float64"], default "float32". name(str|None): A name for this layer(optional). If set None, the layer will be named automatically. Returns: - tuple: The projection of hidden state, and cell state of LSTMP. The \ - shape of projection is (T x P), for the cell state which is \ - (T x D), and both LoD is the same with the `input`. + tuple: A tuple of two output variable: the projection of hidden state, \ + and cell state of LSTMP. The shape of projection is (T x P), \ + for the cell state which is (T x D), and both LoD is the same \ + with the `input`. Examples: + .. code-block:: python + dict_dim, emb_dim = 128, 64 + data = fluid.layers.data(name='sequence', shape=[1], + dtype='int32', lod_level=1) + emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) hidden_dim, proj_dim = 512, 256 - fc_out = fluid.layers.fc(input=input_seq, size=hidden_dim * 4, + fc_out = fluid.layers.fc(input=emb, size=hidden_dim * 4, act=None, bias_attr=None) proj_out, _ = fluid.layers.dynamic_lstmp(input=fc_out, size=hidden_dim * 4, @@ -626,10 +629,10 @@ def dynamic_gru(input, candidate_activation='tanh', h_0=None): """ - **Dynamic GRU Layer** + **Gated Recurrent Unit (GRU) Layer** Refer to `Empirical Evaluation of Gated Recurrent Neural Networks on - Sequence Modeling `_ + Sequence Modeling `_ . The formula is as follows: @@ -676,17 +679,25 @@ def dynamic_gru(input, Choices = ["sigmoid", "tanh", "relu", "identity"], default "sigmoid". candidate_activation(str): The activation for candidate hidden state. Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". - h_0 (Variable): The hidden output of the first time step. + h_0 (Variable): This is initial hidden state. If not set, default is + zero. This is a tensor with shape (N x D), where N is the number of + total time steps of input mini-batch feature and D is the hidden + size. Returns: Variable: The hidden state of GRU. The shape is :math:`(T \\times D)`, \ - and lod is the same with the input. + and sequence length is the same with the input. Examples: + .. code-block:: python + dict_dim, emb_dim = 128, 64 + data = fluid.layers.data(name='sequence', shape=[1], + dtype='int32', lod_level=1) + emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim]) hidden_dim = 512 - x = fluid.layers.fc(input=data, size=hidden_dim * 3) + x = fluid.layers.fc(input=emb, size=hidden_dim * 3) hidden = fluid.layers.dynamic_gru(input=x, dim=hidden_dim) """ @@ -924,13 +935,13 @@ def dropout(x, dropout_prob, is_test=False, seed=None, name=None): Drop or keep each element of `x` independently. Dropout is a regularization technique for reducing overfitting by preventing neuron co-adaption during - training. The dropout operator randomly set (according to the given dropout + training. The dropout operator randomly sets (according to the given dropout probability) the outputs of some units to zero, while others are remain unchanged. Args: - x (Variable): The input tensor. - dropout_prob (float): Probability of setting units to zero. + x (Variable): The input tensor variable. + dropout_prob (float): Probability of setting units to zero. is_test (bool): A flag indicating whether it is in test phrase or not. seed (int): A Python integer used to create random seeds. If this parameter is set to None, a random seed is used. @@ -940,13 +951,14 @@ def dropout(x, dropout_prob, is_test=False, seed=None, name=None): will be named automatically. Returns: - Variable: A tensor variable. + Variable: A tensor variable is the shape with `x`. Examples: + .. code-block:: python - x = fluid.layers.data(name="data", shape=[32, 32], dtype="float32") - droped = fluid.layers.dropout(input=x, dropout_rate=0.5) + x = fluid.layers.data(name="data", shape=[32, 32], dtype="float32") + droped = fluid.layers.dropout(x, dropout_prob=0.5) """ helper = LayerHelper('dropout', **locals()) @@ -2990,32 +3002,33 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): norm. For a 1-D tensor (`dim` is fixed to 0), this layer computes .. math:: - y = \frac{x}{ \sqrt{\sum {x^2} + epsion }} + + y = \\frac{x}{ \sqrt{\sum {x^2} + epsion }} For `x` with more dimensions, this layer independently normalizes each 1-D slice along dimension `axis`. Args: x(Variable|list): The input tensor to l2_normalize layer. - axis(int): The axis on which to apply normalization. If `axis < 0`, + axis(int): The axis on which to apply normalization. If `axis < 0`, \ the dimension to normalization is rank(X) + axis. -1 is the last dimension. - epsilon(float): The epsilon value is used to avoid division by zero, + epsilon(float): The epsilon value is used to avoid division by zero, \ the defalut value is 1e-10. - name(str|None): A name for this layer(optional). If set None, the layer + name(str|None): A name for this layer(optional). If set None, the layer \ will be named automatically. - Returns: - Variable: The output tensor variable. + Variable: The output tensor variable is the same shape with `x`. Examples: + .. code-block:: python - data = fluid.layers.data(name="data", - shape=(3, 17, 13), - dtype="float32") - normed = fluid.layers.l2_normalize(x=data, axis=1) + data = fluid.layers.data(name="data", + shape=(3, 17, 13), + dtype="float32") + normed = fluid.layers.l2_normalize(x=data, axis=1) """ if len(x.shape) == 1: diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index f178248fd6..f585c88cbe 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -497,11 +497,27 @@ def save_combine(x, file_path, overwrite=True): Saves a list of variables into a single file. Args: - x(list): A list of Tensor/LoDTensor to be saved together in a single file. + x(list): A list of Tensor/LoDTensor variables to be saved together in + a single file. file_path(str): The file path where variables will be saved. - overwrite(bool): Whether or not cover the given file when it has already + overwrite(bool): Whether or not cover the given file when it has already existed. If it's set 'False' and the file is existed, a runtime error will be thrown. + + Returns: + There is no return value. + + Examples: + + .. code-block:: python + + v1 = fluid.layers.data(name="data", + shape=(4, 6), + dtype="float32") + v2 = fluid.layers.data(name="data", + shape=(6, 8, 4), + dtype="float32") + normed = fluid.layers.save_combine([v1, v2], file_path="output") """ helper = LayerHelper("save_combine", **locals()) helper.append_op( From 8f59d79d751e3174f0a6f98783fa1dbdbc279cc2 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 15 Jun 2018 16:35:53 +0800 Subject: [PATCH 124/154] update doc for sigmoid_cross_entropy_with_logits --- .../fluid/operators/sigmoid_cross_entropy_with_logits_op.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/operators/sigmoid_cross_entropy_with_logits_op.cc b/paddle/fluid/operators/sigmoid_cross_entropy_with_logits_op.cc index 135e2a6f7f..c3b0fe3209 100644 --- a/paddle/fluid/operators/sigmoid_cross_entropy_with_logits_op.cc +++ b/paddle/fluid/operators/sigmoid_cross_entropy_with_logits_op.cc @@ -113,14 +113,14 @@ The logistic loss is given as follows: $$loss = -Labels * \log(\sigma(X)) - (1 - Labels) * \log(1 - \sigma(X))$$ -We know that $$\sigma(X) = (1 / (1 + \exp(-X)))$$. By substituting this we get: +We know that $$\sigma(X) = \\frac{1}{1 + \exp(-X)}$$. By substituting this we get: $$loss = X - X * Labels + \log(1 + \exp(-X))$$ For stability and to prevent overflow of $$\exp(-X)$$ when X < 0, we reformulate the loss as follows: - $$loss = \max(X, 0) - X * Labels + \log(1 + \exp(-|X|))$$ + $$loss = \max(X, 0) - X * Labels + \log(1 + \exp(-\|X\|))$$ Both the input `X` and `Labels` can carry the LoD (Level of Details) information. However the output only shares the LoD with input `X`. From 566a94022337f227a86d4e2bcaa45dafc1cc36ae Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Fri, 15 Jun 2018 16:40:35 +0800 Subject: [PATCH 125/154] Implement a bilinear initializer for transposed convolution to do upsampling. (#11404) * Implement a bilinear initializer for transposed convolution. * Update some error message. --- python/paddle/fluid/initializer.py | 102 +++++++++++++++++- .../fluid/tests/unittests/test_initializer.py | 17 +++ 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 4e132ed261..c36ad324e7 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -15,11 +15,13 @@ import framework import numpy as np import contextlib +from framework import convert_np_dtype_to_dtype_ +from core import VarDesc __all__ = [ - 'Constant', 'Uniform', 'Normal', 'Xavier', 'force_init_on_cpu', + 'Constant', 'Uniform', 'Normal', 'Xavier', 'Bilinear', 'force_init_on_cpu', 'init_on_cpu', 'ConstantInitializer', 'UniformInitializer', - 'NormalInitializer', 'XavierInitializer' + 'NormalInitializer', 'XavierInitializer', 'BilinearInitializer' ] _force_init_on_cpu_ = False @@ -422,6 +424,101 @@ class MSRAInitializer(Initializer): return op +class BilinearInitializer(Initializer): + """Implements the bilinear initializer. + + This initializer can be used in transposed convolution operator to + act as upsampling. Users can upsample a feature map with shape of + (B, C, H, W) by any integer factor. The usage is: + + >>> factor = 2 + >>> w_attr = ParamAttr(learning_rate=0., regularizer=L2Decay(0.), + >>> initializer=Bilinear()) + >>> conv_up = fluid.layers.conv2d_transpose( + >>> input, + >>> num_filters=C, + >>> output_size=None, + >>> filter_size=2 * factor - factor % 2, + >>> padding=ceil((factor - 1) / 2.), + >>> stride=factor, + >>> groups=C, + >>> param_attr=w_attr, + >>> bias_attr=False) + + + Where, `num_filters=C` and `groups=C` means this is channel-wise tranposed + convolution. The filter shape will be (C, 1, K, K) where K is `filer_size`, + This initializer will set a (K, K) interpolation kernel for every channel + of the filter identically. The resulting shape of the output feature map + will be (B, C, factor * H, factor * W). Note that the learning rate and the + weight decay are set to 0 in order to keep coefficient values of bilinear + interpolation unchanged during training. + """ + + def __init__(self): + """Constructor for BilinearInitializer. + """ + super(BilinearInitializer, self).__init__() + + def __call__(self, var, block): + """Add biliear initialization ops for a variable + + Args: + var (Variable): Variable that needs to be initialized. + block (Block): The block in which initialization ops should + be added. + + Returns: + the initialization op + + Raises: + ValueError: If type of `var` and `block` is not right. + If the shape of `var` size is not 4 and + var.shape[2] != var.shape[3]. + """ + if not isinstance(var, framework.Variable): + raise ValueError("var must be framework.Variable.") + + if not isinstance(block, framework.Block): + raise ValueError("block must be framework.Block.") + + shape = var.shape + if len(shape) != 4: + raise ValueError("the length of shape must be 4.") + if shape[2] != shape[3]: + raise ValueError("shape[2] must be equal to shape[3].") + + weight = np.zeros(np.prod(var.shape), dtype='float32') + size = shape[3] + # factor + f = np.ceil(size / 2.) + # center + c = (2 * f - 1 - f % 2) / (2. * f) + for i in range(np.prod(shape)): + x = i % size + y = (i / size) % size + weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c)) + weight = np.reshape(weight, shape) + + if var.dtype == VarDesc.VarType.FP32: + value_name = "fp32_values" + values = [float(v) for v in weight.flat] + else: + raise ValueError("Unsupported dtype %s", input.dtype) + if np.prod(shape) > 1024 * 1024: + raise ValueError("The size of input is too big. ") + op = block.append_op( + type='assign_value', + outputs={'Out': [var]}, + attrs={ + 'dtype': var.dtype, + 'shape': list(shape), + value_name: values + }) + var.op = op + return op + + # We short the class name, since users will use the initializer with the package # name. The sample code: # @@ -436,3 +533,4 @@ Uniform = UniformInitializer Normal = NormalInitializer Xavier = XavierInitializer MSRA = MSRAInitializer +Bilinear = BilinearInitializer diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index 587e2025e1..15a72cb605 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -364,5 +364,22 @@ class TestMSRAInitializer(unittest.TestCase): self.assertEqual(init_op.attr('seed'), 134) +class TestMSRAInitializer(unittest.TestCase): + def test_bilinear_initializer(self): + """Test the bilinear initializer with supplied arguments + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[8, 1, 3, 3], + lod_level=0, + name="param", + initializer=initializer.BilinearInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'assign_value') + + if __name__ == '__main__': unittest.main() From a427e769584e9384c341bfc1fc5fface6d6bcf32 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 15 Jun 2018 16:42:16 +0800 Subject: [PATCH 126/154] skip use_mkldnn if do not use it --- paddle/testing/paddle_gtest_main.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/testing/paddle_gtest_main.cc b/paddle/testing/paddle_gtest_main.cc index 586ec48477..7772dc97f5 100644 --- a/paddle/testing/paddle_gtest_main.cc +++ b/paddle/testing/paddle_gtest_main.cc @@ -30,7 +30,8 @@ int main(int argc, char** argv) { new_argv.push_back( strdup("--tryfromenv=fraction_of_gpu_memory_to_use,use_pinned_memory")); #else - new_argv.push_back(strdup("--tryfromenv=use_pinned_memory")); + new_argv.push_back(strdup("--tryfromenv=use_pinned_memory,use_mkldnn")); + new_argv.push_back(strdup("--undefok=use_mkldnn")); #endif int new_argc = static_cast(new_argv.size()); char** new_argv_address = new_argv.data(); From 4a0f3743c34b8f75e564dffd37a656cf63f89d24 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Fri, 15 Jun 2018 17:03:36 +0800 Subject: [PATCH 127/154] Refine API doc --- python/paddle/fluid/layers/learning_rate_scheduler.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 716cc7824e..fe9b40b817 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -209,6 +209,14 @@ def polynomial_decay(learning_rate, def piecewise_decay(boundaries, values): """Applies piecewise decay to the initial learning rate. + Args: + boundaries: A list of steps numbers. + values: A list of learning rate values that will be picked during + different step boundaries. + + Returns: + The decayed learning rate. + >>> boundaries = [10000, 20000] >>> values = [1.0, 0.5, 0.1] >>> From cafdeb0a403b7a62ddebccf559489520afe5b972 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 02:24:17 -0700 Subject: [PATCH 128/154] Fix docs for detection_output & target_assign --- python/paddle/fluid/layers/detection.py | 53 ++++++++++++++++--------- python/paddle/fluid/layers/nn.py | 13 +++--- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index edf528a595..f46ca7f132 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -97,7 +97,9 @@ def detection_output(loc, nms_eta(float): The parameter for adaptive NMS. Returns: - Variable: The detection outputs is a LoDTensor with shape [No, 6]. + Variable: + + The detection outputs is a LoDTensor with shape [No, 6]. Each row has six values: [label, confidence, xmin, ymin, xmax, ymax]. `No` is the total number of detections in this mini-batch. For each instance, the offsets in first dimension are called LoD, the offset @@ -110,15 +112,15 @@ def detection_output(loc, Examples: .. code-block:: python - pb = layers.data(name='prior_box', shape=[10, 4], + pb = layers.data(name='prior_box', shape=[10, 4], append_batch_size=False, dtype='float32') - pbv = layers.data(name='prior_box_var', shape=[10, 4], + pbv = layers.data(name='prior_box_var', shape=[10, 4], append_batch_size=False, dtype='float32') - loc = layers.data(name='target_box', shape=[2, 21, 4], + loc = layers.data(name='target_box', shape=[2, 21, 4], append_batch_size=False, dtype='float32') - scores = layers.data(name='scores', shape=[2, 21, 10], + scores = layers.data(name='scores', shape=[2, 21, 10], append_batch_size=False, dtype='float32') - nmsed_outs = fluid.layers.detection_output(scores=scores, + nmsed_outs = fluid.layers.detection_output(scores=scores, loc=loc, prior_box=pb, prior_box_var=pbv) @@ -296,8 +298,6 @@ def target_assign(input, mismatch_value=None, name=None): """ - **Target assigner operator** - This operator can be, for given the target bounding boxes or labels, to assign classification and regression targets to each prediction as well as weights to prediction. The weights is used to specify which prediction would @@ -311,20 +311,24 @@ def target_assign(input, 1. Assigning all outpts based on `match_indices`: - If id = match_indices[i][j] > 0, + .. code-block:: text + + If id = match_indices[i][j] > 0, - out[i][j][0 : K] = X[lod[i] + id][j % P][0 : K] - out_weight[i][j] = 1. + out[i][j][0 : K] = X[lod[i] + id][j % P][0 : K] + out_weight[i][j] = 1. - Otherwise, + Otherwise, - out[j][j][0 : K] = {mismatch_value, mismatch_value, ...} - out_weight[i][j] = 0. + out[j][j][0 : K] = {mismatch_value, mismatch_value, ...} + out_weight[i][j] = 0. 2. Assigning out_weight based on `neg_indices` if `neg_indices` is provided: Assumed that the row offset for each instance in `neg_indices` is called neg_lod, for i-th instance and each `id` of neg_indices in this instance: + + .. code-block:: text out[i][id][0 : K] = {mismatch_value, mismatch_value, ...} out_weight[i][id] = 1.0 @@ -341,10 +345,23 @@ def target_assign(input, mismatch_value (float32): Fill this value to the mismatched location. Returns: - out (Variable): The output is a 3D Tensor with shape [N, P, K], - N and P is the same as they are in `neg_indices`, K is the - same as it in input of X. If `match_indices[i][j]`. - out_weight (Variable): The weight for output with the shape of [N, P, 1]. + tuple: + + A tuple(out, out_weight) is returned. out is a 3D Tensor with + shape [N, P, K], N and P is the same as they are in + `neg_indices`, K is the same as it in input of X. If + `match_indices[i][j]`. out_weight is the weight for output with + the shape of [N, P, 1]. + + Examples: + + .. code-block:: python + + matched_indices, matched_dist = fluid.layers.bipartite_match(iou) + gt = layers.data( + name='gt', shape=[1, 1], dtype='int32', lod_level=1) + trg, trg_weight = layers.target_assign( + gt, matched_indices, mismatch_value=0) """ helper = LayerHelper('target_assign', **locals()) out = helper.create_tmp_variable(dtype=input.dtype) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 0a45098bda..5d417daea1 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3466,7 +3466,9 @@ def nce(input, input (Variable): input variable. label (Variable): label. num_total_classes (int):${num_total_classes_comment} - sample_weight (int): ${sample_weight_comment} + sample_weight (Variable|None): A Variable of shape [batch_size, 1] + storing a weight for each sample. The default weight for each + sample is 1.0. param_attr (ParamAttr|None): attributes for parameter bias_attr (ParamAttr|None): attributes for bias num_neg_samples (int): ${num_neg_samples_comment} @@ -4638,10 +4640,6 @@ def random_crop(x, shape, seed=None): """ ${comment} - Examples: - >>> img = fluid.layers.data("img", [3, 256, 256]) - >>> cropped_img = fluid.layers.random_crop(img, shape=[3, 224, 224]) - Args: x(${x_type}): ${x_comment} shape(${shape_type}): ${shape_comment} @@ -4650,7 +4648,10 @@ def random_crop(x, shape, seed=None): Returns: ${out_comment} - + + Examples: + >>> img = fluid.layers.data("img", [3, 256, 256]) + >>> cropped_img = fluid.layers.random_crop(img, shape=[3, 224, 224]) """ helper = LayerHelper("random_crop", **locals()) dtype = helper.input_dtype() From a219f3cc19a7be055eb484a611e99e44a965541b Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Fri, 15 Jun 2018 17:36:07 +0800 Subject: [PATCH 129/154] follow comments --- .../fluid/layers/learning_rate_scheduler.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index fe9b40b817..fef1dca61b 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -209,6 +209,18 @@ def polynomial_decay(learning_rate, def piecewise_decay(boundaries, values): """Applies piecewise decay to the initial learning rate. + The algorithm can be described as the code below. + + .. code-block:: python + + boundaries = [10000, 20000] + values = [1.0, 0.5, 0.1] + if step < 10000: + learning_rate = 1.0 + elif 10000 <= step < 20000: + learning_rate = 0.5 + else: + learning_rate = 0.1 Args: boundaries: A list of steps numbers. values: A list of learning rate values that will be picked during @@ -217,15 +229,7 @@ def piecewise_decay(boundaries, values): Returns: The decayed learning rate. - >>> boundaries = [10000, 20000] - >>> values = [1.0, 0.5, 0.1] - >>> - >>> if step < 10000: - >>> learning_rate = 1.0 - >>> elif 10000 <= step < 20000: - >>> learning_rate = 0.5 - >>> else: - >>> learning_rate = 0.1 + """ if len(values) - len(boundaries) != 1: From f6daab438db0570b098963fb7f91dd01110918db Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 15 Jun 2018 17:59:04 +0800 Subject: [PATCH 130/154] fix a bug --- python/paddle/fluid/layers/tensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 09c0b2fa79..973059a2ce 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -115,7 +115,7 @@ def create_global_var(shape, """ helper = LayerHelper("global_var", **locals()) var = helper.create_global_variable( - dtype=dtype, shape=shape, persistable=persistable) + dtype=dtype, shape=shape, persistable=persistable, name=name) helper.set_variable_initializer( var, initializer=Constant( value=float(value), force_cpu=force_cpu)) From 316eb3e968b8310f38a9308e813e15902f90f771 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 03:03:28 -0700 Subject: [PATCH 131/154] Add doc for layers.auc --- python/paddle/fluid/layers/metric.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/python/paddle/fluid/layers/metric.py b/python/paddle/fluid/layers/metric.py index a1c64ce277..15d7c50bf4 100644 --- a/python/paddle/fluid/layers/metric.py +++ b/python/paddle/fluid/layers/metric.py @@ -53,6 +53,43 @@ def accuracy(input, label, k=1, correct=None, total=None): def auc(input, label, curve='ROC', num_thresholds=200): + """ + **Area Under The Curve (AUC) Layer** + + This implementation computes the AUC according to forward output and label. + It is used very widely in binary classification evaluation. + + As a note: If input label contains values other than 0 and 1, it will be + cast to bool. You can find the relevant definitions `here + `_. + + There are two types of possible curves: + 1. ROC: Receiver operating characteristic + 2. PR: Precision Recall + + Args: + input(Variable): A floating-point 2D Variable, values are in the range + [0, 1]. Each row is sorted in descending order. This + input should be the output of topk. Typically, this + Variable indicates the probability of each label. + label(Variable): A 2D int Variable indicating the label of the training + data. The height is batch size and width is always 1. + curve(str): Curve type, can be 'ROC' or 'PR'. Default 'ROC'. + num_thresholds(int): The number of thresholds to use when discretizing + the roc curve. Default 200. + + Returns: + Variable: A scalar representing the current AUC. + + Examples: + .. code-block:: python + + # network is a binary classification model and label the ground truth + prediction = network(image, is_infer=True) + auc_out=fluid.layers.auc(input=prediction, label=label) + """ + warnings.warn( "This interface not recommended, fluid.layers.auc compute the auc at every minibatch, \ but can not aggregate them and get the pass AUC, because pass \ From f3a777d8e299b9d740e06f2dc51a88b5211f789d Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 03:40:16 -0700 Subject: [PATCH 132/154] Fix the display of reciprocal's formula --- paddle/fluid/operators/activation_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index 5065244c44..92fbbc2854 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -196,7 +196,7 @@ $out = [x]$ __attribute__((unused)) constexpr char ReciprocalDoc[] = R"DOC( Reciprocal Activation Operator. -$$out = \frac{1}{x}$$ +$$out = \\frac{1}{x}$$ )DOC"; From 8c2a834ef3791170b4b9e0d29ef763866e58ad4b Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 15 Jun 2018 18:46:49 +0800 Subject: [PATCH 133/154] add doc for inference_transpiler --- .../fluid/transpiler/inference_transpiler.py | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/python/paddle/fluid/transpiler/inference_transpiler.py b/python/paddle/fluid/transpiler/inference_transpiler.py index 202aa76084..0629f2916b 100644 --- a/python/paddle/fluid/transpiler/inference_transpiler.py +++ b/python/paddle/fluid/transpiler/inference_transpiler.py @@ -19,16 +19,30 @@ from ..executor import global_scope class InferenceTranspiler: + ''' + Convert the fluid program to optimized inference program. + + There are several optimizations, only fuse batch normalization is supported now. + + Examples: + + .. code-block:: python + + # As InferenceTranspiler will modify the original program, + # please clone before use it. + inference_transpiler_program = program.clone() + t = fluid.InferenceTranspiler() + t.transpile(inference_transpiler_program, place) + ''' + def transpile(self, program, place, scope=None): ''' - Transpile the program. Support only fuse batch normalization now. - - :param program: program to transpile - :type program: Program - :param place: inference place - :type place: Place - :param scope: inference scope - :type scope: Scope or None + Run the transpiler. + + Args: + program (Program): program to transpile + place (Place): inference place + scope (Scope|None): inference Scope ''' if not isinstance(program, Program): raise TypeError("program should be as Program type") @@ -49,36 +63,43 @@ class InferenceTranspiler: can be integrated with them. Doing so will give us a forward acceleration, especially in environments like mobile or embedded. - For input X: - - Conv process: X = input * W + bias - - Batch norm process: X' = (X - mean) / std - - Scale Process: Y = a * X' + b + For input :math:`X`: + + - Conv process: :math:`X = input * W + bias` + - Batch norm process: :math:`X' = (X - mean) / std` + - Scale Process: :math:`Y = a * X' + b` After fuse into one operation: - Y = (input * W + bias - mean) / std * a + b - = input * a * W / std + ((bias - mean) / std * a + b) + .. math:: + + Y &= (input * W + bias - mean) / std * a + b \\\\ + &= input * a * W / std + ((bias - mean) / std * a + b) The operator transformation is: + - before: + - conv->batch_norm->any_other_op (bias == 0) - conv->elementwise_add->batch_norm->any_other_op (bias != 0) + - after: + - conv->elementwise_add->any_other_op The transpile stages are: + 1. insert elementwise_add op when bias == 0. 2. fuse the batch_norm's parameters to conv and elementwise_add operators. 3. remove batch_norm ops which are not used in any other ops. 4. adjust the input of any_other_op to be the output of elementwise_add operator. 5. remove unused variables. - :param program: program to transpile - :type program: Program - :param place: inference place - :type place: Place - :param scope: inference scope - :type scope: Scope + Args: + program (Program): program to transpile + place (Place): inference place + scope (Scope): inference Scope + ''' self.scope = scope self.place = place From cff5232e76ab6b424fc4453dc3cdc463f5680030 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 15 Jun 2018 17:58:30 +0800 Subject: [PATCH 134/154] remove Non-ASCII character '\xc2' --- python/paddle/fluid/layers/control_flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 532ffd754d..82f3d66ded 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -236,7 +236,7 @@ class ParallelDo(object): """ ParallelDo is used to represent multi-thread data parallel processing. - Its vanilla implementation can be shown as the following (:math:`|` means + Its vanilla implementation can be shown as the following (:math:`|` means single thread and :math:`||||` means multiple threads) .. code-block:: text @@ -252,7 +252,7 @@ class ParallelDo(object): |||| Compute backward pass in parallel | accumulate param@grad from different devices to the first device | Merge input@grad from different devices -  | Copy param@grad to the place of parallel_do_op + | Copy param@grad to the place of parallel_do_op Examples: From 23ec12cfe95bed8e0d9e7c5840451d5f50d1cab2 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 04:43:19 -0700 Subject: [PATCH 135/154] Fix the problem that metric cannot display --- doc/fluid/api/gen_doc.sh | 2 +- doc/fluid/api/initializer.rst | 14 +++ doc/fluid/api/io.rst | 36 +++++++ doc/fluid/api/layers.rst | 181 +++++++++++++++++++++++++++++----- doc/fluid/api/optimizer.rst | 7 ++ doc/fluid/api/profiler.rst | 12 +++ 6 files changed, 225 insertions(+), 27 deletions(-) diff --git a/doc/fluid/api/gen_doc.sh b/doc/fluid/api/gen_doc.sh index 27f2419c06..acc8b4aa3f 100755 --- a/doc/fluid/api/gen_doc.sh +++ b/doc/fluid/api/gen_doc.sh @@ -1,5 +1,5 @@ #!/bin/bash -python gen_doc.py layers --submodules control_flow device io nn ops tensor detection learning_rate_scheduler > layers.rst +python gen_doc.py layers --submodules control_flow device io nn ops tensor detection learning_rate_scheduler metric > layers.rst for module in data_feeder clip metrics executor initializer io nets optimizer param_attr profiler regularizer do diff --git a/doc/fluid/api/initializer.rst b/doc/fluid/api/initializer.rst index c49a98c744..57efc9823c 100644 --- a/doc/fluid/api/initializer.rst +++ b/doc/fluid/api/initializer.rst @@ -33,6 +33,13 @@ Xavier :members: :noindex: +Bilinear +-------- + +.. autoclass:: paddle.fluid.initializer.Bilinear + :members: + :noindex: + force_init_on_cpu ----------------- @@ -73,3 +80,10 @@ XavierInitializer :members: :noindex: +BilinearInitializer +------------------- + +.. autoclass:: paddle.fluid.initializer.BilinearInitializer + :members: + :noindex: + diff --git a/doc/fluid/api/io.rst b/doc/fluid/api/io.rst index dd9d88b669..21334c9eda 100644 --- a/doc/fluid/api/io.rst +++ b/doc/fluid/api/io.rst @@ -59,3 +59,39 @@ get_inference_program .. autofunction:: paddle.fluid.io.get_inference_program :noindex: +save_checkpoint +--------------- + +.. autofunction:: paddle.fluid.io.save_checkpoint + :noindex: + +load_checkpoint +--------------- + +.. autofunction:: paddle.fluid.io.load_checkpoint + :noindex: + +clean_checkpoint +---------------- + +.. autofunction:: paddle.fluid.io.clean_checkpoint + :noindex: + +load_persist_vars_without_grad +------------------------------ + +.. autofunction:: paddle.fluid.io.load_persist_vars_without_grad + :noindex: + +save_persist_vars_without_grad +------------------------------ + +.. autofunction:: paddle.fluid.io.save_persist_vars_without_grad + :noindex: + +get_latest_checkpoint_serial +---------------------------- + +.. autofunction:: paddle.fluid.io.get_latest_checkpoint_serial + :noindex: + diff --git a/doc/fluid/api/layers.rst b/doc/fluid/api/layers.rst index 8d1c9247b1..1f8f636040 100644 --- a/doc/fluid/api/layers.rst +++ b/doc/fluid/api/layers.rst @@ -181,6 +181,12 @@ Print .. autofunction:: paddle.fluid.layers.Print :noindex: +is_empty +-------- + +.. autofunction:: paddle.fluid.layers.is_empty + :noindex: + device ====== @@ -219,6 +225,12 @@ Send .. autofunction:: paddle.fluid.layers.Send :noindex: +Recv +---- + +.. autofunction:: paddle.fluid.layers.Recv + :noindex: + open_recordio_file ------------------ @@ -255,6 +267,25 @@ double_buffer .. autofunction:: paddle.fluid.layers.double_buffer :noindex: +random_data_generator +--------------------- + +.. autofunction:: paddle.fluid.layers.random_data_generator + :noindex: + +Preprocessor +------------ + +.. autoclass:: paddle.fluid.layers.Preprocessor + :members: + :noindex: + +load +---- + +.. autofunction:: paddle.fluid.layers.load + :noindex: + nn == @@ -399,10 +430,9 @@ conv2d_transpose conv3d_transpose ---------------- -.. autofunction:: paddle.fluid.layers.conv2d_transpose +.. autofunction:: paddle.fluid.layers.conv3d_transpose :noindex: - sequence_expand --------------- @@ -613,6 +643,48 @@ roi_pool .. autofunction:: paddle.fluid.layers.roi_pool :noindex: +dice_loss +--------- + +.. autofunction:: paddle.fluid.layers.dice_loss + :noindex: + +image_resize +------------ + +.. autofunction:: paddle.fluid.layers.image_resize + :noindex: + +image_resize_short +------------------ + +.. autofunction:: paddle.fluid.layers.image_resize_short + :noindex: + +resize_bilinear +--------------- + +.. autofunction:: paddle.fluid.layers.resize_bilinear + :noindex: + +gather +------ + +.. autofunction:: paddle.fluid.layers.gather + :noindex: + +random_crop +----------- + +.. autofunction:: paddle.fluid.layers.random_crop + :noindex: + +mean_iou +-------- + +.. autofunction:: paddle.fluid.layers.mean_iou + :noindex: + ops === @@ -718,12 +790,6 @@ logical_not .. autofunction:: paddle.fluid.layers.logical_not :noindex: -uniform_random --------------- - -.. autofunction:: paddle.fluid.layers.uniform_random - :noindex: - uniform_random_batch_size_like ------------------------------ @@ -742,12 +808,6 @@ gaussian_random_batch_size_like .. autofunction:: paddle.fluid.layers.gaussian_random_batch_size_like :noindex: -cumsum ------- - -.. autofunction:: paddle.fluid.layers.cumsum - :noindex: - scatter ------- @@ -760,6 +820,30 @@ sum .. autofunction:: paddle.fluid.layers.sum :noindex: +slice +----- + +.. autofunction:: paddle.fluid.layers.slice + :noindex: + +polygon_box_transform +--------------------- + +.. autofunction:: paddle.fluid.layers.polygon_box_transform + :noindex: + +shape +----- + +.. autofunction:: paddle.fluid.layers.shape + :noindex: + +maxout +------ + +.. autofunction:: paddle.fluid.layers.maxout + :noindex: + sigmoid ------- @@ -916,18 +1000,6 @@ stanh .. autofunction:: paddle.fluid.layers.stanh :noindex: -hard_shrink ------------ - -.. autofunction:: paddle.fluid.layers.hard_shrink - :noindex: - -thresholded_relu ----------------- - -.. autofunction:: paddle.fluid.layers.thresholded_relu - :noindex: - hard_sigmoid ------------ @@ -940,6 +1012,30 @@ swish .. autofunction:: paddle.fluid.layers.swish :noindex: +uniform_random +-------------- + +.. autofunction:: paddle.fluid.layers.uniform_random + :noindex: + +hard_shrink +----------- + +.. autofunction:: paddle.fluid.layers.hard_shrink + :noindex: + +cumsum +------ + +.. autofunction:: paddle.fluid.layers.cumsum + :noindex: + +thresholded_relu +---------------- + +.. autofunction:: paddle.fluid.layers.thresholded_relu + :noindex: + tensor ====== @@ -997,6 +1093,18 @@ fill_constant .. autofunction:: paddle.fluid.layers.fill_constant :noindex: +argmin +------ + +.. autofunction:: paddle.fluid.layers.argmin + :noindex: + +argmax +------ + +.. autofunction:: paddle.fluid.layers.argmax + :noindex: + ones ---- @@ -1012,6 +1120,12 @@ zeros detection ========= +prior_box +--------- + +.. autofunction:: paddle.fluid.layers.prior_box + :noindex: + multi_box_head -------------- @@ -1099,3 +1213,18 @@ noam_decay .. autofunction:: paddle.fluid.layers.noam_decay :noindex: +metric +====== + +accuracy +-------- + +.. autofunction:: paddle.fluid.layers.accuracy + :noindex: + +auc +--- + +.. autofunction:: paddle.fluid.layers.auc + :noindex: + diff --git a/doc/fluid/api/optimizer.rst b/doc/fluid/api/optimizer.rst index 79a0995fce..6ad44bb690 100644 --- a/doc/fluid/api/optimizer.rst +++ b/doc/fluid/api/optimizer.rst @@ -89,6 +89,13 @@ DecayedAdagradOptimizer :members: :noindex: +RMSPropOptimizer +---------------- + +.. autoclass:: paddle.fluid.optimizer.RMSPropOptimizer + :members: + :noindex: + Adadelta -------- diff --git a/doc/fluid/api/profiler.rst b/doc/fluid/api/profiler.rst index 74d102dcb0..39fda65863 100644 --- a/doc/fluid/api/profiler.rst +++ b/doc/fluid/api/profiler.rst @@ -23,3 +23,15 @@ profiler .. autofunction:: paddle.fluid.profiler.profiler :noindex: +start_profiler +-------------- + +.. autofunction:: paddle.fluid.profiler.start_profiler + :noindex: + +stop_profiler +------------- + +.. autofunction:: paddle.fluid.profiler.stop_profiler + :noindex: + From 68811bcb5d93a9bcbafae81c0ec866e936e3de25 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 05:08:32 -0700 Subject: [PATCH 136/154] Format the doc of layers.auc --- python/paddle/fluid/layers/metric.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/paddle/fluid/layers/metric.py b/python/paddle/fluid/layers/metric.py index 15d7c50bf4..ed2f05e5a9 100644 --- a/python/paddle/fluid/layers/metric.py +++ b/python/paddle/fluid/layers/metric.py @@ -59,14 +59,14 @@ def auc(input, label, curve='ROC', num_thresholds=200): This implementation computes the AUC according to forward output and label. It is used very widely in binary classification evaluation. - As a note: If input label contains values other than 0 and 1, it will be - cast to bool. You can find the relevant definitions `here - `_. + Note: If input label contains values other than 0 and 1, it will be cast + to `bool`. Find the relevant definitions `here `_. There are two types of possible curves: - 1. ROC: Receiver operating characteristic - 2. PR: Precision Recall + + 1. ROC: Receiver operating characteristic; + 2. PR: Precision Recall Args: input(Variable): A floating-point 2D Variable, values are in the range @@ -85,9 +85,9 @@ def auc(input, label, curve='ROC', num_thresholds=200): Examples: .. code-block:: python - # network is a binary classification model and label the ground truth - prediction = network(image, is_infer=True) - auc_out=fluid.layers.auc(input=prediction, label=label) + # network is a binary classification model and label the ground truth + prediction = network(image, is_infer=True) + auc_out=fluid.layers.auc(input=prediction, label=label) """ warnings.warn( From 2efb0e5b70b2291151a14963813f7a968cad797a Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Jun 2018 05:22:16 -0700 Subject: [PATCH 137/154] cased correction --- python/paddle/fluid/layers/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/metric.py b/python/paddle/fluid/layers/metric.py index ed2f05e5a9..0a978eaa37 100644 --- a/python/paddle/fluid/layers/metric.py +++ b/python/paddle/fluid/layers/metric.py @@ -54,7 +54,7 @@ def accuracy(input, label, k=1, correct=None, total=None): def auc(input, label, curve='ROC', num_thresholds=200): """ - **Area Under The Curve (AUC) Layer** + **Area Under the Curve (AUC) Layer** This implementation computes the AUC according to forward output and label. It is used very widely in binary classification evaluation. From dd55cc16472f9669029276e2c198bd3f2ee71b52 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Fri, 15 Jun 2018 08:31:23 -0500 Subject: [PATCH 138/154] fix warning (#11518) --- paddle/fluid/operators/crop_op.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/operators/crop_op.h b/paddle/fluid/operators/crop_op.h index 91cfbbda73..772e80bbea 100644 --- a/paddle/fluid/operators/crop_op.h +++ b/paddle/fluid/operators/crop_op.h @@ -52,7 +52,7 @@ static std::vector GetOffsets(const framework::ExecutionContext& ctx) { } else { res = ctx.Attr>("offsets"); PADDLE_ENFORCE_EQ( - rank, res.size(), + rank, static_cast(res.size()), "Offsets size should be equal to dimension size of input tensor."); } return res; From 53d1d0f0f2e0c0ab87150d4b4e8a77530b8d227c Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Fri, 15 Jun 2018 23:12:46 +0800 Subject: [PATCH 139/154] add LARS support (#10374) --- .../fluid/layers/learning_rate_scheduler.py | 41 ++++++++++++++++++- python/paddle/fluid/optimizer.py | 23 ++++++++--- .../fluid/tests/book/test_recognize_digits.py | 2 +- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index fef1dca61b..e0ac0846a6 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -25,10 +25,11 @@ import nn import ops import tensor from ..initializer import init_on_cpu +from ..framework import default_main_program, Parameter __all__ = [ 'exponential_decay', 'natural_exp_decay', 'inverse_time_decay', - 'polynomial_decay', 'piecewise_decay', 'noam_decay' + 'polynomial_decay', 'piecewise_decay', 'noam_decay', 'append_LARS' ] @@ -261,3 +262,41 @@ def piecewise_decay(boundaries, values): tensor.assign(last_value_var, lr) return lr + + +def append_LARS(params_grads, learning_rate, weight_decay): + """Applies LARS (LAYER-WISE ADAPTIVE RATE SCALING) to learning rate for + each layer. + + ```python + learning_rate *= local_gw_ratio * sqrt(sumsq(param)) + / (sqrt(sumsq(gradient))+ weight_decay * sqrt(sumsq(param))) + ``` + + Args: + learning_rate: A learning rate Variable. This + is the global learning rate for LARS. + weight_decay: A Python `float` number. + + Returns: + The decayed learning rate + """ + + def _balanced_weight(param_norm, grad_norm): + if weight_decay == 1.0: + return grad_norm + param_norm + else: + return grad_norm + weight_decay * param_norm + + for param, grad in params_grads: + param_lr = param.optimize_attr['learning_rate'] + param_norm = ops.sqrt(nn.reduce_sum(input=ops.square(param))) + grad_norm = ops.sqrt(nn.reduce_sum(input=ops.square(grad))) + if type(param_lr) == float and param_lr == 1.0: + decayed_lr = learning_rate * param_norm \ + / _balanced_weight(param_norm, grad_norm) + else: + decayed_lr = learning_rate * param_lr * param_norm \ + / _balanced_weight(param_norm, grad_norm) + # set back param local learning rate + param.optimize_attr['learning_rate'] = decayed_lr diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 115362c6bf..54fe935627 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -13,7 +13,7 @@ # limitations under the License. import re from collections import defaultdict -from paddle.fluid.framework import Program +from paddle.fluid.framework import Program, Variable import framework import layers from backward import append_backward @@ -41,7 +41,10 @@ class Optimizer(object): but need to use one of it's implementation. """ - def __init__(self, learning_rate, regularization=None): + def __init__(self, + learning_rate, + regularization=None, + LARS_weight_decay=0.0): if not isinstance(learning_rate, float) and \ not isinstance(learning_rate, framework.Variable): raise TypeError("learning rate should be float or Variable") @@ -61,6 +64,7 @@ class Optimizer(object): # {accum_name : { paramter_name : accumulator_for_parameter, ...}, ...} self._accumulators = defaultdict(lambda: dict()) self.helper = None + self._LARS_weight_decay = LARS_weight_decay def _create_global_learning_rate(self): lr = self.global_learning_rate() @@ -100,10 +104,15 @@ class Optimizer(object): # create learning rate variable for every parameter param = param_and_grad[0] param_lr = param.optimize_attr['learning_rate'] - if param_lr == 1.0: - return self.global_learning_rate() + if type(param_lr) == Variable: + # param learning rate has been updated (LARS) + print("returns updated param lr ", param_lr) + return param_lr else: - return self.global_learning_rate() * param_lr + if param_lr == 1.0: + return self.global_learning_rate() + else: + return self.global_learning_rate() * param_lr def _create_accumulators(self, block, parameters): """Create all accumulators needed by the parameters @@ -210,6 +219,10 @@ class Optimizer(object): self._create_accumulators(loss.block, [p[0] for p in parameters_and_grads]) self._create_global_learning_rate() + if self._LARS_weight_decay > 0.0: + layers.append_LARS(parameters_and_grads, + self.global_learning_rate(), + self._LARS_weight_decay) optimize_ops = [] for param_and_grad in parameters_and_grads: diff --git a/python/paddle/fluid/tests/book/test_recognize_digits.py b/python/paddle/fluid/tests/book/test_recognize_digits.py index 578b1162fb..25bcb8a641 100644 --- a/python/paddle/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/fluid/tests/book/test_recognize_digits.py @@ -94,7 +94,7 @@ def train(nn_type, test_program = fluid.default_main_program().clone(for_test=True) - optimizer = fluid.optimizer.Adam(learning_rate=0.001) + optimizer = fluid.optimizer.Adam(learning_rate=0.001, LARS_weight_decay=0.3) optimizer.minimize(avg_loss) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() From 417fcf4f43f350ee4cb8407f8e1d98da4dc0b9dd Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Fri, 15 Jun 2018 08:50:01 -0700 Subject: [PATCH 140/154] Modify Pybind LoDTensor API according to length-based LoD (#11106) * add lod_tensor util and modify pybind * refind pybind LoDTensor API and modify LoDTensor and DataFeeder test * fix test error * fix detection map op test * fix reorder_lod_tensor test * fix seq_concat_op * fix chunk evel op test * fix target assign op * fix warp ctc op * address comments step 1: reverse reset_lod op * step 2: modify op test * add warning message * remove has_valid_lod * add back has_valid_lod * address comments * add exception catching trial --- benchmark/fluid/models/machine_translation.py | 15 -- .../fluid/models/stacked_dynamic_lstm.py | 15 -- paddle/fluid/framework/lod_tensor.cc | 33 +++++ paddle/fluid/framework/lod_tensor.h | 14 ++ paddle/fluid/framework/lod_tensor_test.cc | 32 +++++ paddle/fluid/pybind/pybind.cc | 74 ++++++++-- python/paddle/fluid/data_feeder.py | 7 +- python/paddle/fluid/lod_tensor.py | 83 +---------- python/paddle/fluid/tests/test_data_feeder.py | 17 ++- python/paddle/fluid/tests/test_lod_tensor.py | 65 +++++---- .../paddle/fluid/tests/unittests/op_test.py | 14 +- .../tests/unittests/test_batch_norm_op.py | 2 +- .../unittests/test_beam_search_decode_op.py | 22 +-- .../tests/unittests/test_beam_search_op.py | 8 +- .../unittests/test_bipartite_match_op.py | 20 +-- .../tests/unittests/test_box_coder_op.py | 18 ++- .../tests/unittests/test_chunk_eval_op.py | 8 +- .../tests/unittests/test_crf_decoding_op.py | 31 +++-- .../fluid/tests/unittests/test_ctc_align.py | 12 +- .../tests/unittests/test_detection_map_op.py | 36 ++--- .../unittests/test_dynrnn_gradient_check.py | 6 +- .../unittests/test_dynrnn_static_input.py | 47 ++++--- .../tests/unittests/test_edit_distance_op.py | 44 +++--- .../tests/unittests/test_feed_fetch_method.py | 7 +- .../test_fill_constant_batch_size_like_op.py | 2 +- .../fluid/tests/unittests/test_gru_op.py | 12 +- .../tests/unittests/test_iou_similarity_op.py | 4 +- .../unittests/test_linear_chain_crf_op.py | 14 +- .../tests/unittests/test_lod_rank_table.py | 3 +- .../tests/unittests/test_lod_reset_op.py | 44 +++--- .../tests/unittests/test_lod_tensor_array.py | 8 +- .../unittests/test_lod_tensor_array_ops.py | 35 ++--- .../fluid/tests/unittests/test_lstm_op.py | 40 +++--- .../fluid/tests/unittests/test_lstmp_op.py | 34 ++--- .../unittests/test_mine_hard_examples_op.py | 4 +- .../tests/unittests/test_multiclass_nms_op.py | 4 +- .../fluid/tests/unittests/test_one_hot_op.py | 20 +-- .../fluid/tests/unittests/test_print_op.py | 4 +- .../unittests/test_reorder_lod_tensor.py | 52 ++++--- .../fluid/tests/unittests/test_roi_pool_op.py | 3 +- .../fluid/tests/unittests/test_row_conv_op.py | 15 +- .../tests/unittests/test_seq_concat_op.py | 51 +++---- .../fluid/tests/unittests/test_seq_conv.py | 57 +++++--- .../fluid/tests/unittests/test_seq_pool.py | 129 ++++++++++-------- .../tests/unittests/test_sequence_erase_op.py | 16 ++- .../tests/unittests/test_sequence_expand.py | 40 +++--- .../tests/unittests/test_sequence_reshape.py | 16 +-- .../tests/unittests/test_sequence_slice_op.py | 12 +- .../unittests/test_sequence_softmax_op.py | 13 +- .../tests/unittests/test_shrink_rnn_memory.py | 6 +- .../test_split_and_merge_lod_tensor_op.py | 13 +- .../tests/unittests/test_target_assign_op.py | 43 +++--- .../fluid/tests/unittests/test_tensor.py | 27 ++-- .../fluid/tests/unittests/test_warpctc_op.py | 35 +++-- .../unittests/test_weight_normalization.py | 8 +- .../paddle/fluid/tests/unittests/testsuite.py | 4 +- tools/codestyle/cpplint_pre_commit.hook | 2 +- 57 files changed, 765 insertions(+), 635 deletions(-) diff --git a/benchmark/fluid/models/machine_translation.py b/benchmark/fluid/models/machine_translation.py index 69541adf6b..17f6b03826 100644 --- a/benchmark/fluid/models/machine_translation.py +++ b/benchmark/fluid/models/machine_translation.py @@ -173,21 +173,6 @@ def seq_to_seq_net(embedding_dim, encoder_size, decoder_size, source_dict_dim, return avg_cost, feeding_list -def to_lodtensor(data, place): - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = np.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - lod_t = core.LoDTensor() - lod_t.set(flattened_data, place) - lod_t.set_lod([lod]) - return lod_t, lod[-1] - - def lodtensor_to_ndarray(lod_tensor): dims = lod_tensor.get_dims() ndarray = np.zeros(shape=dims).astype('float32') diff --git a/benchmark/fluid/models/stacked_dynamic_lstm.py b/benchmark/fluid/models/stacked_dynamic_lstm.py index 211869af4e..3231542a17 100644 --- a/benchmark/fluid/models/stacked_dynamic_lstm.py +++ b/benchmark/fluid/models/stacked_dynamic_lstm.py @@ -125,18 +125,3 @@ def get_model(args): batch_size=args.batch_size) return loss, inference_program, adam, train_reader, test_reader, batch_acc - - -def to_lodtensor(data, place): - seq_lens = [len(seq) for seq in data] - cur_len = 0 - lod = [cur_len] - for l in seq_lens: - cur_len += l - lod.append(cur_len) - flattened_data = numpy.concatenate(data, axis=0).astype("int64") - flattened_data = flattened_data.reshape([len(flattened_data), 1]) - res = fluid.LoDTensor() - res.set(flattened_data, place) - res.set_lod([lod]) - return res diff --git a/paddle/fluid/framework/lod_tensor.cc b/paddle/fluid/framework/lod_tensor.cc index a56674cbe2..e331c8128f 100644 --- a/paddle/fluid/framework/lod_tensor.cc +++ b/paddle/fluid/framework/lod_tensor.cc @@ -410,5 +410,38 @@ void LoDTensor::MergeLoDTensor( } } +LoD ConvertToLengthBasedLoD(const LoD &offset_lod) { + LoD length_lod; + length_lod.reserve(offset_lod.size()); + for (size_t lvl = 0; lvl < offset_lod.size(); ++lvl) { + std::vector level; + if (offset_lod[lvl].size() > 0) { + level.reserve(offset_lod[lvl].size() - 1); + } + for (size_t idx = 0; idx < offset_lod[lvl].size() - 1; ++idx) { + level.push_back(offset_lod[lvl][idx + 1] - offset_lod[lvl][idx]); + } + length_lod.push_back(level); + } + return length_lod; +} + +LoD ConvertToOffsetBasedLoD(const LoD &length_lod) { + LoD offset_lod; + offset_lod.reserve(length_lod.size()); + for (size_t lvl = 0; lvl < length_lod.size(); ++lvl) { + std::vector level; + level.reserve(length_lod[lvl].size() + 1); + size_t tmp = 0; + level.push_back(tmp); + for (size_t idx = 0; idx < length_lod[lvl].size(); ++idx) { + tmp += length_lod[lvl][idx]; + level.push_back(tmp); + } + offset_lod.push_back(level); + } + return offset_lod; +} + } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/lod_tensor.h b/paddle/fluid/framework/lod_tensor.h index 1159fee39b..4a2729373b 100644 --- a/paddle/fluid/framework/lod_tensor.h +++ b/paddle/fluid/framework/lod_tensor.h @@ -226,5 +226,19 @@ extern void WriteToRecordIO(recordio::Writer* writer, extern std::vector ReadFromRecordIO( recordio::Scanner* scanner, const platform::DeviceContext& dev_ctx); +/* + * Convert between length-based LoD and offset-based LoD. + * The implementation of LoDTensor class use offset-based LoD. + * However, we want to expose the more user-friendly length-based + * LoD to the Python side instead. + * + * Example: + * If offset_lod = [[0, 2, 3],[0, 3, 5, 9]] + * then length_lod = [[2, 1], [3, 2, 4]] + */ +LoD ConvertToLengthBasedLoD(const LoD& offset_lod); + +LoD ConvertToOffsetBasedLoD(const LoD& length_lod); + } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/lod_tensor_test.cc b/paddle/fluid/framework/lod_tensor_test.cc index 2ceffc9331..6dfe7d2d8c 100644 --- a/paddle/fluid/framework/lod_tensor_test.cc +++ b/paddle/fluid/framework/lod_tensor_test.cc @@ -228,6 +228,38 @@ TEST(LoD, CheckAbsLoD) { ASSERT_FALSE(CheckAbsLoD(abs_lod0)); } +TEST(LoD, ConvertToLengthBasedLoD) { + LoD offset_lod; + offset_lod.push_back(std::vector({0, 2})); + offset_lod.push_back(std::vector({0, 1, 3})); + offset_lod.push_back(std::vector({0, 2, 4, 5})); + + LoD length_lod = ConvertToLengthBasedLoD(offset_lod); + + LoD expected; + expected.push_back(std::vector({2})); + expected.push_back(std::vector({1, 2})); + expected.push_back(std::vector({2, 2, 1})); + + EXPECT_EQ(length_lod, expected); +} + +TEST(LoD, ConvertToOffsetBasedLoD) { + LoD length_lod; + length_lod.push_back(std::vector({2})); + length_lod.push_back(std::vector({1, 2})); + length_lod.push_back(std::vector({2, 2, 1})); + + LoD offset_lod = ConvertToOffsetBasedLoD(length_lod); + + LoD expected; + expected.push_back(std::vector({0, 2})); + expected.push_back(std::vector({0, 1, 3})); + expected.push_back(std::vector({0, 2, 4, 5})); + + EXPECT_EQ(offset_lod, expected); +} + template static void TestRecordIO() { LoDTensor tensor; diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index bd5c613f8c..74036bcb31 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -144,28 +144,74 @@ PYBIND11_PLUGIN(core) { py::class_(m, "LoDTensor") .def_buffer( [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); }) - .def( - "__init__", - [](LoDTensor &instance, const std::vector> &lod) { - LoD new_lod; - new_lod.reserve(lod.size()); - std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); - new (&instance) LoDTensor(new_lod); - }) + .def("__init__", + [](LoDTensor &instance, const std::vector> + &recursive_sequence_lengths) { + LoD new_lod; + new_lod.reserve(recursive_sequence_lengths.size()); + std::copy(recursive_sequence_lengths.begin(), + recursive_sequence_lengths.end(), + std::back_inserter(new_lod)); + LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod); + PADDLE_ENFORCE( + CheckLoD(new_offset_lod, -1), + "the provided recursive_sequence_lengths info is invalid"); + new (&instance) LoDTensor(new_offset_lod); + }) .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); }) .def("set_lod", [](LoDTensor &self, const std::vector> &lod) { + // the input lod is offset-based level-of-detail info + LOG(WARNING) + << "set_lod is deprecated and will be removed by 9.2018, " + "please switch to set_recursive_sequence_lengths."; LoD new_lod; new_lod.reserve(lod.size()); std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); + PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()), + "the provided lod info is invalid"); self.set_lod(new_lod); }) - .def("lod", [](LoDTensor &self) -> std::vector> { - auto lod = self.lod(); - std::vector> new_lod; - new_lod.reserve(lod.size()); - std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); - return new_lod; + .def("set_recursive_sequence_lengths", + [](LoDTensor &self, const std::vector> + &recursive_sequence_lengths) { + // the input recursive_sequence_lengths is length-based + // level-of-detail info + LoD new_lod; + new_lod.reserve(recursive_sequence_lengths.size()); + std::copy(recursive_sequence_lengths.begin(), + recursive_sequence_lengths.end(), + std::back_inserter(new_lod)); + LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod); + PADDLE_ENFORCE( + CheckLoD(new_offset_lod, vectorize(self.dims()).front()), + "the provided recursive_sequence_lengths info is invalid"); + self.set_lod(new_offset_lod); + }) + .def("lod", + [](LoDTensor &self) -> std::vector> { + // output the offset-based lod info + LOG(WARNING) << "lod is deprecated and will be removed by 9.2018, " + "please switch to recursive_sequence_lengths."; + LoD lod = self.lod(); + std::vector> new_lod; + new_lod.reserve(lod.size()); + std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); + return new_lod; + }) + .def("recursive_sequence_lengths", + [](LoDTensor &self) -> std::vector> { + // output the length-based lod info + LoD lod = ConvertToLengthBasedLoD(self.lod()); + std::vector> new_lod; + new_lod.reserve(lod.size()); + std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); + return new_lod; + }) + .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool { + // Check that the lod info is valid and match the outermost + // dimension of the LoDTensor data + return CheckLoD(self.lod(), vectorize(self.dims()).front()); }); py::class_(m, "SelectedRows") diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index e2013137b1..ac39600201 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -47,7 +47,7 @@ class DataToLoDTensorConverter(object): self.lod = [] for i in six.range(lod_level): - self.lod.append([0]) + self.lod.append([]) def feed(self, data): self._feed_impl_(data, self.lod, self.lod_level) @@ -56,8 +56,7 @@ class DataToLoDTensorConverter(object): if lod_level == 0: self.data.append(data) else: - cur_lod_len = len(data) - lod[0].append(lod[0][-1] + cur_lod_len) + lod[0].append(len(data)) for each_data in data: self._feed_impl_(each_data, lod[1:], lod_level - 1) @@ -66,7 +65,7 @@ class DataToLoDTensorConverter(object): t = core.LoDTensor() t.set(arr, self.place) if self.lod_level > 0: - t.set_lod(self.lod) + t.set_recursive_sequence_lengths(self.lod) return t diff --git a/python/paddle/fluid/lod_tensor.py b/python/paddle/fluid/lod_tensor.py index 9946d0a4ff..61be39c259 100644 --- a/python/paddle/fluid/lod_tensor.py +++ b/python/paddle/fluid/lod_tensor.py @@ -18,80 +18,6 @@ import numpy as np __all__ = ['create_lod_tensor', 'create_random_int_lodtensor'] -def _validate_lod(lod, tensor_height=-1): - """Check whether the input length-based lod info is valid. - - There are several things to check: - 1. lod should be a list of lists. Empty list is fine. - 2. The length of each sublist (a lod level) should be at least one. - 3. Each element in each lod level should be an integer greater than 0. - 4. The sum of one lod level should be equal to the length of the next lod level. - 5. The sum of the last lod level should be equal to the tensor height. - Bypass this check if user does not provide tensor_height as input. - - Args: - lod: the length-based lod info, e.g., [[2, 3], [2, 1, 2, 3, 4]]. - tensor_height: the outermost dimension of the tensor with which the input - lod is associated with. - - Returns: - A boolean indicating whether the input lod is valid or not. - """ - assert isinstance(lod, list), "lod should be a list" - # Empty lod is fine - if len(lod) == 0: - return True - - lod_sum = [] - for level in lod: - assert isinstance(level, list), "each item in lod should be a list" - # Each level of lod should have at least one length info - if len(level) < 1: - return False - level_sum = 0 - for lod_len in level: - # Each length in a level should be > 0 - if lod_len <= 0: - return False - level_sum += lod_len - lod_sum.append(level_sum) - - for idx, val in enumerate(lod_sum[:-1]): - # Each level's sum should be equal to - # the number of items in the next level - if val != len(lod[idx + 1]): - return False - - if tensor_height == -1: - return True - else: - # Last level's sum should be equal to the tensor height - return lod_sum[-1] == tensor_height - - -def _convert_lod(lod): - """Convert a length-based lod to a offset-based lod. - - If the length-based lod is [[2, 3], [2, 1, 2, 3, 4]], - then the offset-based lod is [[0, 2, 5], [0, 2, 3, 5, 8, 12]]. - - Args: - lod: a length-based lod info. - - Returns: - A list of lists as the offset-based lod converted to from the input lod. - """ - new_lod = [] - for level in lod: - cur_len = 0 - new_level = [cur_len] - for lod_len in level: - cur_len += lod_len - new_level.append(cur_len) - new_lod.append(new_level) - return new_lod - - def create_lod_tensor(data, lod, place): """Create a lod tensor from a numpy array, a list, or an existing lod tensor. @@ -139,11 +65,11 @@ def create_lod_tensor(data, lod, place): flattened_data = flattened_data.reshape([len(flattened_data), 1]) return create_lod_tensor(flattened_data, lod, place) elif isinstance(data, np.ndarray): - assert _validate_lod(lod, - data.shape[0]), "the provided lod info is invalid" tensor = core.LoDTensor() tensor.set(data, place) - tensor.set_lod(_convert_lod(lod)) + tensor.set_recursive_sequence_lengths(lod) + assert tensor.has_valid_recursive_sequence_lengths( + ), "the provided lod info is invalid" return tensor else: raise TypeError( @@ -181,9 +107,8 @@ def create_random_int_lodtensor(lod, base_shape, place, low, high): A fluid LoDTensor object with tensor data and lod info. """ assert isinstance(base_shape, list), "base_shape should be a list" - converted_lod = _convert_lod(lod) # append the total number of basic elements to the front of its shape - overall_shape = [converted_lod[-1][-1]] + base_shape + overall_shape = [sum(lod[-1])] + base_shape # the range of integer data elements is [low, high] data = np.random.random_integers(low, high, overall_shape).astype("int64") return create_lod_tensor(data, lod, place) diff --git a/python/paddle/fluid/tests/test_data_feeder.py b/python/paddle/fluid/tests/test_data_feeder.py index ce3ba3ebc5..30b7a634a2 100644 --- a/python/paddle/fluid/tests/test_data_feeder.py +++ b/python/paddle/fluid/tests/test_data_feeder.py @@ -22,12 +22,11 @@ class TestDataFeeder(unittest.TestCase): label = fluid.layers.data(name='label', shape=[1], dtype='int64') feeder = fluid.DataFeeder([img, label], fluid.CPUPlace()) result = feeder.feed([([0] * 784, [9]), ([1] * 784, [1])]) - print(result) self.assertEqual(result['image'].shape(), [2, 1, 28, 28]) self.assertEqual(result['label'].shape(), [2, 1]) - self.assertEqual(result['image'].lod(), []) - self.assertEqual(result['label'].lod(), []) + self.assertEqual(result['image'].recursive_sequence_lengths(), []) + self.assertEqual(result['label'].recursive_sequence_lengths(), []) def test_lod_level_1_converter(self): # lod_level = 1 @@ -42,12 +41,12 @@ class TestDataFeeder(unittest.TestCase): # label = [1] * len(data) result = feeder.feed( [([1, 2, 3], [1]), ([4, 5], [1]), ([6, 7, 8, 9], [1])]) - print(result) self.assertEqual(result['sentences'].shape(), [9, 1]) self.assertEqual(result['label'].shape(), [3, 1]) - self.assertEqual(result['sentences'].lod(), [[0, 3, 5, 9]]) - self.assertEqual(result['label'].lod(), []) + self.assertEqual(result['sentences'].recursive_sequence_lengths(), + [[3, 2, 4]]) + self.assertEqual(result['label'].recursive_sequence_lengths(), []) def test_lod_level_2_converter(self): # lod_level = 2 @@ -62,12 +61,12 @@ class TestDataFeeder(unittest.TestCase): # label = [1] * len(data) result = feeder.feed( [([[1, 2, 3], [4, 5]], [1]), ([[6, 7, 8, 9]], [1])]) - print(result) self.assertEqual(result['paragraphs'].shape(), [9, 1]) self.assertEqual(result['label'].shape(), [2, 1]) - self.assertEqual(result['paragraphs'].lod(), [[0, 2, 3], [0, 3, 5, 9]]) - self.assertEqual(result['label'].lod(), []) + self.assertEqual(result['paragraphs'].recursive_sequence_lengths(), + [[2, 1], [3, 2, 4]]) + self.assertEqual(result['label'].recursive_sequence_lengths(), []) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/test_lod_tensor.py b/python/paddle/fluid/tests/test_lod_tensor.py index 013d72f418..b7e7f5801f 100644 --- a/python/paddle/fluid/tests/test_lod_tensor.py +++ b/python/paddle/fluid/tests/test_lod_tensor.py @@ -13,44 +13,41 @@ # limitations under the License. import paddle.fluid as fluid -from paddle.fluid.lod_tensor import create_lod_tensor, create_random_int_lodtensor, _validate_lod, _convert_lod -import numpy +from paddle.fluid.lod_tensor import create_lod_tensor, create_random_int_lodtensor +import numpy as np import unittest class TestLoDTensor(unittest.TestCase): - def test_validate_lod(self): - lod = (1, 2, 1) - self.assertRaises(AssertionError, _validate_lod, lod, -1) - lod = [[1, 2], (2, 3)] - self.assertRaises(AssertionError, _validate_lod, lod, -1) - lod = [1, 2, 3] - self.assertRaises(AssertionError, _validate_lod, lod, -1) - + def test_pybind_lod(self): + tensor = fluid.LoDTensor() lod = [] - self.assertTrue(_validate_lod(lod, -1)) + tensor.set_recursive_sequence_lengths(lod) lod = [[], [1], [3]] - self.assertFalse(_validate_lod(lod, -1)) - lod = [[0], [-1], [3]] - self.assertFalse(_validate_lod(lod, -1)) + self.assertRaises(Exception, tensor.set_recursive_sequence_lengths, lod) + lod = [[0], [2], [3]] + self.assertRaises(Exception, tensor.set_recursive_sequence_lengths, lod) - # Each level's sum should be equal to the number of items in the next level - # Moreover, last level's sum should be equal to the tensor height - lod = [[2, 3], [1, 3, 1, 2, 1]] - self.assertTrue(_validate_lod(lod, tensor_height=8)) - lod = [[1, 3], [2, 1, 3]] - self.assertFalse(_validate_lod(lod, tensor_height=6)) - lod = [[1, 3], [2, 1, 3, 4]] - self.assertFalse(_validate_lod(lod, tensor_height=5)) - - def test_convert_lod(self): lod = [[1, 2, 3]] - converted_lod = [[0, 1, 3, 6]] - self.assertEqual(_convert_lod(lod), converted_lod) + tensor.set_recursive_sequence_lengths(lod) + self.assertEqual(tensor.recursive_sequence_lengths(), lod) + tensor.set(np.random.random([6, 1]), fluid.CPUPlace()) + self.assertTrue(tensor.has_valid_recursive_sequence_lengths()) + tensor.set(np.random.random([9, 1]), fluid.CPUPlace()) + self.assertFalse(tensor.has_valid_recursive_sequence_lengths()) + # Each level's sum should be equal to the number of items in the next level + # Moreover, last level's sum should be equal to the tensor height + lod = [[2, 3], [1, 3, 1, 2, 2]] + tensor.set_recursive_sequence_lengths(lod) + self.assertEqual(tensor.recursive_sequence_lengths(), lod) + tensor.set(np.random.random([8, 1]), fluid.CPUPlace()) + self.assertFalse(tensor.has_valid_recursive_sequence_lengths()) lod = [[2, 3], [1, 3, 1, 2, 1]] - converted_lod = [[0, 2, 5], [0, 1, 4, 5, 7, 8]] - self.assertEqual(_convert_lod(lod), converted_lod) + tensor.set_recursive_sequence_lengths(lod) + self.assertTrue(tensor.has_valid_recursive_sequence_lengths()) + tensor.set(np.random.random([9, 1]), fluid.CPUPlace()) + self.assertFalse(tensor.has_valid_recursive_sequence_lengths()) def test_create_lod_tensor(self): # Create LoDTensor from a list @@ -60,19 +57,19 @@ class TestLoDTensor(unittest.TestCase): self.assertRaises(AssertionError, create_lod_tensor, data, wrong_lod, fluid.CPUPlace()) tensor = create_lod_tensor(data, correct_lod, fluid.CPUPlace()) - self.assertEqual(tensor.lod(), [[0, 3, 5]]) + self.assertEqual(tensor.recursive_sequence_lengths(), correct_lod) # Create LoDTensor from numpy array - data = numpy.random.random([10, 1]) + data = np.random.random([10, 1]) lod = [[2, 1], [3, 3, 4]] tensor = create_lod_tensor(data, lod, fluid.CPUPlace()) - self.assertEqual(tensor.lod(), [[0, 2, 3], [0, 3, 6, 10]]) + self.assertEqual(tensor.recursive_sequence_lengths(), lod) # Create LoDTensor from another LoDTensor, they are differnt instances new_lod = [[2, 2, 1], [1, 2, 2, 3, 2]] new_tensor = create_lod_tensor(tensor, new_lod, fluid.CPUPlace()) - self.assertEqual(tensor.lod(), [[0, 2, 3], [0, 3, 6, 10]]) - self.assertEqual(new_tensor.lod(), [[0, 2, 4, 5], [0, 1, 3, 5, 8, 10]]) + self.assertEqual(tensor.recursive_sequence_lengths(), lod) + self.assertEqual(new_tensor.recursive_sequence_lengths(), new_lod) def test_create_random_int_lodtensor(self): # The shape of a word, commonly used in speech and NLP problem, is [1] @@ -83,7 +80,7 @@ class TestLoDTensor(unittest.TestCase): high = dict_size - 1 tensor = create_random_int_lodtensor(lod, shape, fluid.CPUPlace(), low, high) - self.assertEqual(tensor.lod(), [[0, 2, 5, 10]]) + self.assertEqual(tensor.recursive_sequence_lengths(), lod) self.assertEqual(tensor.shape(), [10, 1]) diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index 307caae4b0..e056ef9952 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -162,7 +162,7 @@ class OpTest(unittest.TestCase): tensor = core.LoDTensor() if isinstance(np_value, tuple): tensor.set(np_value[0], place) - tensor.set_lod(np_value[1]) + tensor.set_recursive_sequence_lengths(np_value[1]) else: tensor.set(np_value, place) feed_map[name] = tensor @@ -170,7 +170,8 @@ class OpTest(unittest.TestCase): tensor = core.LoDTensor() if isinstance(self.inputs[var_name], tuple): tensor.set(self.inputs[var_name][0], place) - tensor.set_lod(self.inputs[var_name][1]) + tensor.set_recursive_sequence_lengths(self.inputs[var_name][ + 1]) else: tensor.set(self.inputs[var_name], place) feed_map[var_name] = tensor @@ -293,7 +294,8 @@ class OpTest(unittest.TestCase): str(place)) if isinstance(expect, tuple): self.assertListEqual( - actual.lod(), expect[1], "Output (" + sub_out_name + + actual.recursive_sequence_lengths(), expect[1], + "Output (" + sub_out_name + ") has different lod at " + str(place)) else: idx = find_actual(out_name, fetch_list) @@ -307,8 +309,8 @@ class OpTest(unittest.TestCase): "Output (" + out_name + ") has diff at " + str(place) + str(actual_t) + "\n" + str(expect_t)) if isinstance(expect, tuple): - self.assertListEqual(actual.lod(), expect[1], - "Output (" + out_name + + self.assertListEqual(actual.recursive_sequence_lengths(), + expect[1], "Output (" + out_name + ") has different lod at " + str(place)) def _get_places(self): @@ -408,7 +410,7 @@ class OpTest(unittest.TestCase): tensor = core.LoDTensor() tensor.set(np_value, place) if lod is not None: - tensor.set_lod(lod) + tensor.set_recursive_sequence_lengths(lod) return tensor @staticmethod diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py index 4216d83653..01e5749bdb 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py @@ -128,7 +128,7 @@ def create_or_get_tensor(scope, var_name, var, place): tensor = scope.var(var_name).get_tensor() if var is not None: assert isinstance(var, np.ndarray) - tensor.set_lod([[]]) + tensor.set_recursive_sequence_lengths([]) tensor.set_dims(var.shape) tensor.set(var, place) return tensor diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py index 7976dd7c3f..4e1687477c 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py @@ -26,36 +26,36 @@ class TestBeamSearchDecodeOp(unittest.TestCase): def append_lod_tensor(self, tensor_array, lod, data): lod_tensor = core.LoDTensor() - lod_tensor.set_lod(lod) + lod_tensor.set_recursive_sequence_lengths(lod) lod_tensor.set(data, self.place) tensor_array.append(lod_tensor) def test_get_set(self): ids = self.scope.var("ids").get_lod_tensor_array() self.append_lod_tensor( - ids, [[0, 3, 6], [0, 1, 2, 3, 4, 5, 6]], + ids, [[3, 3], [1, 1, 1, 1, 1, 1]], np.array( [1, 2, 3, 4, 5, 6], dtype="int64")) self.append_lod_tensor( - ids, [[0, 3, 6], [0, 1, 1, 3, 5, 5, 6]], + ids, [[3, 3], [1, 0, 2, 2, 0, 1]], np.array( [0, 1, 2, 3, 4, 5], dtype="int64")) self.append_lod_tensor( - ids, [[0, 3, 6], [0, 0, 1, 2, 3, 4, 5]], + ids, [[3, 3], [0, 1, 1, 1, 1, 1]], np.array( [0, 1, 2, 3, 4], dtype="int64")) scores = self.scope.var("scores").get_lod_tensor_array() self.append_lod_tensor( - scores, [[0, 3, 6], [0, 1, 2, 3, 4, 5, 6]], + scores, [[3, 3], [1, 1, 1, 1, 1, 1]], np.array( [1, 2, 3, 4, 5, 6], dtype="float64")) self.append_lod_tensor( - scores, [[0, 3, 6], [0, 1, 1, 3, 5, 5, 6]], + scores, [[3, 3], [1, 0, 2, 2, 0, 1]], np.array( [0, 1, 2, 3, 4, 5], dtype="float64")) self.append_lod_tensor( - scores, [[0, 3, 6], [0, 0, 1, 2, 3, 4, 5]], + scores, [[3, 3], [0, 1, 1, 1, 1, 1]], np.array( [0, 1, 2, 3, 4], dtype="float64")) @@ -73,9 +73,11 @@ class TestBeamSearchDecodeOp(unittest.TestCase): beam_search_decode_op.run(self.scope, self.place) - expected_lod = [[0, 4, 8], [0, 1, 3, 6, 9, 10, 13, 16, 19]] - self.assertEqual(sentence_ids.lod(), expected_lod) - self.assertEqual(sentence_scores.lod(), expected_lod) + expected_lod = [[4, 4], [1, 2, 3, 3, 1, 3, 3, 3]] + self.assertEqual(sentence_ids.recursive_sequence_lengths(), + expected_lod) + self.assertEqual(sentence_scores.recursive_sequence_lengths(), + expected_lod) expected_data = np.array( [2, 1, 0, 3, 1, 0, 3, 2, 1, 5, 4, 3, 2, 4, 4, 3, 6, 5, 4], "int64") diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_op.py index bc708f3aff..5a14178c27 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_op.py @@ -48,18 +48,18 @@ class BeamSearchOpTester(unittest.TestCase): op.run(self.scope, core.CPUPlace()) selected_ids = self.scope.find_var("selected_ids").get_tensor() print 'selected_ids', np.array(selected_ids) - print 'lod', selected_ids.lod() + print 'lod', selected_ids.recursive_sequence_lengths() def _create_pre_ids(self): np_data = np.array([[1, 2, 3, 4]], dtype='int64') tensor = create_tensor(self.scope, "pre_ids", np_data) def _create_ids(self): - self.lod = [[0, 1, 4], [0, 1, 2, 3, 4]] + self.lod = [[1, 3], [1, 1, 1, 1]] np_data = np.array( [[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], dtype='int64') tensor = create_tensor(self.scope, "ids", np_data) - tensor.set_lod(self.lod) + tensor.set_recursive_sequence_lengths(self.lod) def _create_scores(self): np_data = np.array( @@ -71,7 +71,7 @@ class BeamSearchOpTester(unittest.TestCase): ], dtype='float32') tensor = create_tensor(self.scope, "scores", np_data) - tensor.set_lod(self.lod) + tensor.set_recursive_sequence_lengths(self.lod) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py index f7461ee6da..1a245fd756 100644 --- a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py +++ b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py @@ -65,23 +65,25 @@ def batch_bipartite_match(distance, lod, match_type=None, dist_threshold=None): distance (numpy.array) : The distance of two entries with shape [M, N]. lod (list of int): The offsets of each input in this batch. """ - n = len(lod) - 1 + n = len(lod) m = distance.shape[1] match_indices = -1 * np.ones((n, m), dtype=np.int) match_dist = np.zeros((n, m), dtype=np.float32) - for i in range(len(lod) - 1): - bipartite_match(distance[lod[i]:lod[i + 1], :], match_indices[i, :], - match_dist[i, :]) + cur_offset = 0 + for i in range(n): + bipartite_match(distance[cur_offset:(cur_offset + lod[i]), :], + match_indices[i, :], match_dist[i, :]) if match_type == 'per_prediction': - argmax_match(distance[lod[i]:lod[i + 1], :], match_indices[i, :], - match_dist[i, :], dist_threshold) + argmax_match(distance[cur_offset:(cur_offset + lod[i]), :], + match_indices[i, :], match_dist[i, :], dist_threshold) + cur_offset += lod[i] return match_indices, match_dist class TestBipartiteMatchOpWithLoD(OpTest): def setUp(self): self.op_type = 'bipartite_match' - lod = [[0, 5, 11, 23]] + lod = [[5, 6, 12]] dist = np.random.random((23, 217)).astype('float32') match_indices, match_dist = batch_bipartite_match(dist, lod[0]) @@ -98,7 +100,7 @@ class TestBipartiteMatchOpWithLoD(OpTest): class TestBipartiteMatchOpWithoutLoD(OpTest): def setUp(self): self.op_type = 'bipartite_match' - lod = [[0, 8]] + lod = [[8]] dist = np.random.random((8, 17)).astype('float32') match_indices, match_dist = batch_bipartite_match(dist, lod[0]) @@ -115,7 +117,7 @@ class TestBipartiteMatchOpWithoutLoD(OpTest): class TestBipartiteMatchOpWithPerPredictionType(OpTest): def setUp(self): self.op_type = 'bipartite_match' - lod = [[0, 5, 11, 23]] + lod = [[5, 6, 12]] dist = np.random.random((23, 237)).astype('float32') match_indices, match_dist = batch_bipartite_match(dist, lod[0], 'per_prediction', 0.5) diff --git a/python/paddle/fluid/tests/unittests/test_box_coder_op.py b/python/paddle/fluid/tests/unittests/test_box_coder_op.py index b4c48d85f2..4ce9a4783e 100644 --- a/python/paddle/fluid/tests/unittests/test_box_coder_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_coder_op.py @@ -81,15 +81,19 @@ def batch_box_coder(prior_box, prior_box_var, target_box, lod, code_type, n = target_box.shape[0] m = prior_box.shape[0] output_box = np.zeros((n, m, 4), dtype=np.float32) - for i in range(len(lod) - 1): + cur_offset = 0 + for i in range(len(lod)): if (code_type == "EncodeCenterSize"): - box_coder(target_box[lod[i]:lod[i + 1], :], prior_box, - prior_box_var, output_box[lod[i]:lod[i + 1], :, :], + box_coder(target_box[cur_offset:(cur_offset + lod[i]), :], + prior_box, prior_box_var, + output_box[cur_offset:(cur_offset + lod[i]), :, :], code_type, box_normalized) elif (code_type == "DecodeCenterSize"): - box_coder(target_box[lod[i]:lod[i + 1], :, :], prior_box, - prior_box_var, output_box[lod[i]:lod[i + 1], :, :], + box_coder(target_box[cur_offset:(cur_offset + lod[i]), :, :], + prior_box, prior_box_var, + output_box[cur_offset:(cur_offset + lod[i]), :, :], code_type, box_normalized) + cur_offset += lod[i] return output_box @@ -99,7 +103,7 @@ class TestBoxCoderOp(OpTest): def setUp(self): self.op_type = "box_coder" - lod = [[0, 1, 2, 3, 4, 5]] + lod = [[1, 1, 1, 1, 1]] prior_box = np.random.random((10, 4)).astype('float32') prior_box_var = np.random.random((10, 4)).astype('float32') target_box = np.random.random((5, 10, 4)).astype('float32') @@ -152,7 +156,7 @@ class TestBoxCoderOpWithLoD(OpTest): def setUp(self): self.op_type = "box_coder" - lod = [[0, 4, 12, 20]] + lod = [[4, 8, 8]] prior_box = np.random.random((10, 4)).astype('float32') prior_box_var = np.random.random((10, 4)).astype('float32') target_box = np.random.random((20, 4)).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py index 050df2801c..23932194f0 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py @@ -144,10 +144,10 @@ class TestChunkEvalOp(OpTest): starts = sorted(starts) self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = self.gen_chunks( infer, label, starts) - self.inputs = { - 'Inference': (infer, [starts]), - 'Label': (label, [starts]) - } + lod = [] + for i in range(len(starts) - 1): + lod.append(starts[i + 1] - starts[i]) + self.inputs = {'Inference': (infer, [lod]), 'Label': (label, [lod])} precision = float( self.num_correct_chunks ) / self.num_infer_chunks if self.num_infer_chunks else 0 diff --git a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py index f397f542bb..122b076c2d 100644 --- a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py @@ -22,9 +22,9 @@ from op_test import OpTest class CRFDecoding(object): def __init__(self, emission_weights, transition_weights, seq_start_positions): - assert (emission_weights.shape[0] == seq_start_positions[-1]) + assert (emission_weights.shape[0] == sum(seq_start_positions)) self.tag_num = emission_weights.shape[1] - self.seq_num = len(seq_start_positions) - 1 + self.seq_num = len(seq_start_positions) self.seq_start_positions = seq_start_positions self.x = emission_weights @@ -34,9 +34,9 @@ class CRFDecoding(object): self.w = transition_weights[2:, :] self.track = np.zeros( - (seq_start_positions[-1], self.tag_num), dtype="int64") + (sum(seq_start_positions), self.tag_num), dtype="int64") self.decoded_path = np.zeros( - (seq_start_positions[-1], 1), dtype="int64") + (sum(seq_start_positions), 1), dtype="int64") def _decode_one_sequence(self, decoded_path, x): seq_len, tag_num = x.shape @@ -71,9 +71,11 @@ class CRFDecoding(object): decoded_path[i - 1] = max_idx = track[i, max_idx] def decode(self): + cur_pos = 0 for i in range(self.seq_num): - start = self.seq_start_positions[i] - end = self.seq_start_positions[i + 1] + start = cur_pos + cur_pos += self.seq_start_positions[i] + end = cur_pos self._decode_one_sequence(self.decoded_path[start:end, :], self.x[start:end, :]) return self.decoded_path @@ -90,11 +92,13 @@ class TestCRFDecodingOp1(OpTest): TAG_NUM = 17 MAX_SEQ_LEN = 10 - lod = [[0]] + lod = [[]] + total_len = 0 for i in range(SEQ_NUM): - lod[-1].append(lod[-1][-1] + random.randint(1, MAX_SEQ_LEN)) + lod[-1].append(random.randint(1, MAX_SEQ_LEN)) + total_len += lod[-1][-1] emission = np.random.uniform(-1, 1, - [lod[-1][-1], TAG_NUM]).astype("float64") + [total_len, TAG_NUM]).astype("float64") transition = np.random.uniform(-0.5, 0.5, [TAG_NUM + 2, TAG_NUM]).astype("float64") @@ -126,7 +130,8 @@ class TestCRFDecodingOp2(OpTest): self.op_type = "crf_decoding" TAG_NUM = 5 - lod = [[0, 1, 3, 6, 10]] + lod = [[1, 2, 3, 4]] + total_len = sum(lod[-1]) transition = np.repeat( np.arange( TAG_NUM, dtype="float64").reshape(1, TAG_NUM), @@ -135,13 +140,13 @@ class TestCRFDecodingOp2(OpTest): emission = np.repeat( np.arange( TAG_NUM, dtype="float64").reshape(1, TAG_NUM), - lod[-1][-1], + total_len, axis=0) labels = np.random.randint( - low=0, high=TAG_NUM, size=(lod[-1][-1], 1), dtype="int64") + low=0, high=TAG_NUM, size=(total_len, 1), dtype="int64") predicted_labels = np.ones( - (lod[-1][-1], 1), dtype="int64") * (TAG_NUM - 1) + (total_len, 1), dtype="int64") * (TAG_NUM - 1) expected_output = (labels == predicted_labels).astype("int64") self.inputs = { diff --git a/python/paddle/fluid/tests/unittests/test_ctc_align.py b/python/paddle/fluid/tests/unittests/test_ctc_align.py index f166031a1c..131b4076f4 100644 --- a/python/paddle/fluid/tests/unittests/test_ctc_align.py +++ b/python/paddle/fluid/tests/unittests/test_ctc_align.py @@ -22,14 +22,16 @@ from test_softmax_op import stable_softmax def CTCAlign(input, lod, blank, merge_repeated): lod0 = lod[0] result = [] - for i in range(len(lod0) - 1): + cur_offset = 0 + for i in range(len(lod0)): prev_token = -1 - for j in range(lod0[i], lod0[i + 1]): + for j in range(cur_offset, cur_offset + lod0[i]): token = input[j][0] if (token != blank) and not (merge_repeated and token == prev_token): result.append(token) prev_token = token + cur_offset += lod0[i] result = np.array(result).reshape([len(result), 1]).astype("int32") if len(result) == 0: result = np.array([-1]) @@ -39,7 +41,7 @@ def CTCAlign(input, lod, blank, merge_repeated): class TestCTCAlignOp(OpTest): def config(self): self.op_type = "ctc_align" - self.input_lod = [[0, 11, 18]] + self.input_lod = [[11, 7]] self.blank = 0 self.merge_repeated = False self.input = np.array( @@ -66,7 +68,7 @@ class TestCTCAlignOp(OpTest): class TestCTCAlignOpCase1(TestCTCAlignOp): def config(self): self.op_type = "ctc_align" - self.input_lod = [[0, 11, 19]] + self.input_lod = [[11, 8]] self.blank = 0 self.merge_repeated = True self.input = np.array( @@ -77,7 +79,7 @@ class TestCTCAlignOpCase1(TestCTCAlignOp): class TestCTCAlignOpCase2(TestCTCAlignOp): def config(self): self.op_type = "ctc_align" - self.input_lod = [[0, 4]] + self.input_lod = [[4]] self.blank = 0 self.merge_repeated = True self.input = np.array([0, 0, 0, 0]).reshape([4, 1]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index f545ad155c..05d3367ad8 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -74,13 +74,13 @@ class TestDetectionMAPOp(OpTest): self.evaluate_difficult = True self.ap_type = "integral" - self.label_lod = [[0, 2, 4]] + self.label_lod = [[2, 2]] # label difficult xmin ymin xmax ymax self.label = [[1, 0, 0.1, 0.1, 0.3, 0.3], [1, 1, 0.6, 0.6, 0.8, 0.8], [2, 0, 0.3, 0.3, 0.6, 0.5], [1, 0, 0.7, 0.1, 0.9, 0.3]] # label score xmin ymin xmax ymax difficult - self.detect_lod = [[0, 3, 7]] + self.detect_lod = [[3, 4]] self.detect = [ [1, 0.3, 0.1, 0.0, 0.4, 0.3], [1, 0.7, 0.0, 0.1, 0.2, 0.3], [1, 0.9, 0.7, 0.6, 0.8, 0.8], [2, 0.8, 0.2, 0.1, 0.4, 0.4], @@ -89,7 +89,7 @@ class TestDetectionMAPOp(OpTest): ] # label score true_pos false_pos - self.tf_pos_lod = [[0, 3, 7]] + self.tf_pos_lod = [[3, 4]] self.tf_pos = [[1, 0.9, 1, 0], [1, 0.7, 1, 0], [1, 0.3, 0, 1], [1, 0.2, 1, 0], [2, 0.8, 0, 1], [2, 0.1, 1, 0], [3, 0.2, 0, 1]] @@ -112,15 +112,19 @@ class TestDetectionMAPOp(OpTest): for i, count in enumerate(class_pos_count): class_pos_count_dict[i] = count - for i in range(len(true_pos_lod[0]) - 1): - start = true_pos_lod[0][i] - end = true_pos_lod[0][i + 1] + cur_pos = 0 + for i in range(len(true_pos_lod[0])): + start = cur_pos + cur_pos += true_pos_lod[0][i] + end = cur_pos for j in range(start, end): true_pos_dict[i].append(true_pos[j]) - for i in range(len(false_pos_lod[0]) - 1): - start = false_pos_lod[0][i] - end = false_pos_lod[0][i + 1] + cur_pos = 0 + for i in range(len(false_pos_lod[0])): + start = cur_pos + cur_pos += false_pos_lod[0][i] + end = cur_pos for j in range(start, end): false_pos_dict[i].append(false_pos[j]) @@ -130,19 +134,19 @@ class TestDetectionMAPOp(OpTest): label_number = self.class_num out_class_pos_count = [] - out_true_pos_lod = [0] + out_true_pos_lod = [] out_true_pos = [] - out_false_pos_lod = [0] + out_false_pos_lod = [] out_false_pos = [] for i in range(label_number): out_class_pos_count.append([label_count[i]]) true_pos_list = true_pos[i] out_true_pos += true_pos_list - out_true_pos_lod.append(len(out_true_pos)) + out_true_pos_lod.append(len(true_pos_list)) false_pos_list = false_pos[i] out_false_pos += false_pos_list - out_false_pos_lod.append(len(out_false_pos)) + out_false_pos_lod.append(len(false_pos_list)) return out_class_pos_count, out_true_pos, [ out_true_pos_lod @@ -241,7 +245,7 @@ class TestDetectionMAPOpSkipDiff(TestDetectionMAPOp): self.evaluate_difficult = False - self.tf_pos_lod = [[0, 2, 6]] + self.tf_pos_lod = [[2, 4]] # label score true_pos false_pos self.tf_pos = [[1, 0.7, 1, 0], [1, 0.3, 0, 1], [1, 0.2, 1, 0], [2, 0.8, 0, 1], [2, 0.1, 1, 0], [3, 0.2, 0, 1]] @@ -267,9 +271,9 @@ class TestDetectionMAPOpMultiBatch(TestDetectionMAPOp): def init_test_case(self): super(TestDetectionMAPOpMultiBatch, self).init_test_case() self.class_pos_count = [0, 2, 1] - self.true_pos_lod = [[0, 0, 3, 5]] + self.true_pos_lod = [[0, 3, 2]] self.true_pos = [[0.7, 1.], [0.3, 0.], [0.2, 1.], [0.8, 0.], [0.1, 1.]] - self.false_pos_lod = [[0, 0, 3, 5]] + self.false_pos_lod = [[0, 3, 2]] self.false_pos = [[0.7, 0.], [0.3, 1.], [0.2, 0.], [0.8, 1.], [0.1, 0.]] diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py index 95af51f1b2..0f289af284 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py @@ -136,16 +136,16 @@ class BaseRNN(object): feed_dict = dict() for iname in self.inputs: - lod = [0] + lod = [] np_flatten = [] for seq_id in xrange(len(self.inputs[iname])): seq_len = len(self.inputs[iname][seq_id]) - lod.append(lod[-1] + seq_len) + lod.append(seq_len) np_flatten.extend(self.inputs[iname][seq_id]) t = fluid.Tensor() t.set(numpy.array(np_flatten), place) - t.set_lod([lod]) + t.set_recursive_sequence_lengths([lod]) feed_dict[iname] = t for pname in self.params: diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index d3f63ee2c4..92e718662d 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -39,20 +39,20 @@ class TestDyRnnStaticInput(unittest.TestCase): def prepare_x_tensor(self): self.x_tensor_dim = 10 - lod = [[0, 2, 3, 6]] - shape = [lod[0][-1], self.x_tensor_dim] + lod = [[2, 1, 3]] + shape = [sum(lod[0]), self.x_tensor_dim] self.x_tensor_data = np.random.random(shape).astype('float32') self.x_tensor = core.LoDTensor() - self.x_tensor.set_lod(lod) + self.x_tensor.set_recursive_sequence_lengths(lod) self.x_tensor.set(self.x_tensor_data, self.place) def prepare_static_input_tensor(self): self.static_input_tensor_dim = 4 - lod = [[0, 1, 3, 6]] - shape = [lod[0][-1], self.static_input_tensor_dim] + lod = [[1, 2, 3]] + shape = [sum(lod[0]), self.static_input_tensor_dim] self.static_input_data = np.random.random(shape).astype('float32') self.static_input_tensor = core.LoDTensor() - self.static_input_tensor.set_lod(lod) + self.static_input_tensor.set_recursive_sequence_lengths(lod) self.static_input_tensor.set(self.static_input_data, self.place) def fetch_value(self, var): @@ -69,7 +69,7 @@ class TestDyRnnStaticInput(unittest.TestCase): ndarray = np.zeros(shape=dims).astype('float32') for i in xrange(np.product(dims)): ndarray.ravel()[i] = lod_tensor.get_float_element(i) - return ndarray, lod_tensor.lod() + return ndarray, lod_tensor.recursive_sequence_lengths() def build_graph(self, only_forward=False): x_tensor = fluid.layers.data( @@ -131,21 +131,20 @@ class TestDyRnnStaticInput(unittest.TestCase): framework.grad_var_name('static_input_tensor')) return static_input_grad, loss - def get_seq_len_from_lod(self, lod): - return [lod[0][i + 1] - lod[0][i] for i in xrange(len(lod[0]) - 1)] - def get_expected_static_step_outs(self): - x_lod = self.x_tensor.lod() - x_seq_len = self.get_seq_len_from_lod(x_lod) + x_lod = self.x_tensor.recursive_sequence_lengths() + x_seq_len = x_lod[0] x_seq_len_sorted = sorted(x_seq_len) x_sorted_indices = np.argsort(x_seq_len)[::-1] - static_lod = self.static_input_tensor.lod() - static_sliced = [ - self.static_input_data[static_lod[0][i]:static_lod[0][i + 1]] - for i in xrange(len(static_lod[0]) - 1) - ] - static_seq_len = self.get_seq_len_from_lod(static_lod) + static_lod = self.static_input_tensor.recursive_sequence_lengths() + static_sliced = [] + cur_offset = 0 + for i in xrange(len(static_lod[0])): + static_sliced.append(self.static_input_data[cur_offset:( + cur_offset + static_lod[0][i])]) + cur_offset += static_lod[0][i] + static_seq_len = static_lod[0] static_reordered = [] for i in xrange(len(x_sorted_indices)): static_reordered.extend(static_sliced[x_sorted_indices[i]].tolist()) @@ -159,11 +158,13 @@ class TestDyRnnStaticInput(unittest.TestCase): for i in xrange(self._max_sequence_len): end = len(x_seq_len) - bisect.bisect_left(x_seq_len_sorted, i + 1) - lod = [0] + lod = [] + total_len = 0 for i in xrange(end): - lod.append(static_seq_len_reordered[i] + lod[-1]) + lod.append(static_seq_len_reordered[i]) + total_len += lod[-1] static_step_lods.append([lod]) - end = lod[-1] + end = total_len static_step_outs.append( np.array(static_reordered[:end]).astype('float32')) @@ -199,7 +200,9 @@ class TestDyRnnStaticInput(unittest.TestCase): self.static_input_tensor.set_float_element(i, origin) numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2 self.assertTrue(np.allclose(actual_gradients, numeric_gradients, 0.001)) - self.assertTrue(np.allclose(actual_lod, self.static_input_tensor.lod())) + self.assertTrue( + np.allclose(actual_lod, + self.static_input_tensor.recursive_sequence_lengths())) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py index 2957fb5058..816562621b 100644 --- a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py @@ -52,23 +52,29 @@ class TestEditDistanceOp(OpTest): def setUp(self): self.op_type = "edit_distance" normalized = False - x1 = np.array([[0, 12, 3, 5, 8, 2]]).astype("int64") - x2 = np.array([[0, 12, 4, 7, 8]]).astype("int64") + x1 = np.array([[12, 3, 5, 8, 2]]).astype("int64") + x2 = np.array([[12, 4, 7, 8]]).astype("int64") x1 = np.transpose(x1) x2 = np.transpose(x2) - x1_lod = [0, 1, 5] - x2_lod = [0, 3, 4] + x1_lod = [1, 4] + x2_lod = [3, 1] - num_strs = len(x1_lod) - 1 + num_strs = len(x1_lod) distance = np.zeros((num_strs, 1)).astype("float32") sequence_num = np.array(2).astype("int64") + + x1_offset = 0 + x2_offset = 0 for i in range(0, num_strs): distance[i] = Levenshtein( - hyp=x1[x1_lod[i]:x1_lod[i + 1]], - ref=x2[x2_lod[i]:x2_lod[i + 1]]) + hyp=x1[x1_offset:(x1_offset + x1_lod[i])], + ref=x2[x2_offset:(x2_offset + x2_lod[i])]) + x1_offset += x1_lod[i] + x2_offset += x2_lod[i] if normalized is True: - len_ref = x2_lod[i + 1] - x2_lod[i] + len_ref = x2_lod[i] distance[i] = distance[i] / len_ref + self.attrs = {'normalized': normalized} self.inputs = {'Hyps': (x1, [x1_lod]), 'Refs': (x2, [x2_lod])} self.outputs = {'Out': distance, 'SequenceNum': sequence_num} @@ -81,23 +87,29 @@ class TestEditDistanceOpNormalized(OpTest): def setUp(self): self.op_type = "edit_distance" normalized = True - x1 = np.array([[0, 10, 3, 6, 5, 8, 2]]).astype("int64") - x2 = np.array([[0, 10, 4, 6, 7, 8]]).astype("int64") + x1 = np.array([[10, 3, 6, 5, 8, 2]]).astype("int64") + x2 = np.array([[10, 4, 6, 7, 8]]).astype("int64") x1 = np.transpose(x1) x2 = np.transpose(x2) - x1_lod = [0, 1, 3, 6] - x2_lod = [0, 2, 3, 5] + x1_lod = [1, 2, 3] + x2_lod = [2, 1, 2] - num_strs = len(x1_lod) - 1 + num_strs = len(x1_lod) distance = np.zeros((num_strs, 1)).astype("float32") sequence_num = np.array(3).astype("int64") + + x1_offset = 0 + x2_offset = 0 for i in range(0, num_strs): distance[i] = Levenshtein( - hyp=x1[x1_lod[i]:x1_lod[i + 1]], - ref=x2[x2_lod[i]:x2_lod[i + 1]]) + hyp=x1[x1_offset:(x1_offset + x1_lod[i])], + ref=x2[x2_offset:(x2_offset + x2_lod[i])]) + x1_offset += x1_lod[i] + x2_offset += x2_lod[i] if normalized is True: - len_ref = x2_lod[i + 1] - x2_lod[i] + len_ref = x2_lod[i] distance[i] = distance[i] / len_ref + self.attrs = {'normalized': normalized} self.inputs = {'Hyps': (x1, [x1_lod]), 'Refs': (x2, [x2_lod])} self.outputs = {'Out': distance, 'SequenceNum': sequence_num} diff --git a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py index 9d724a6479..8b9da84311 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py +++ b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py @@ -24,17 +24,16 @@ class TestFeedFetch(unittest.TestCase): input_array = np.ones((4, 4, 6)).astype("float32") input_array[0, 0, 0] = 3 input_array[3, 3, 5] = 10 - input_tensor = core.LoDTensor([[0, 2, 4]]) + input_tensor = core.LoDTensor([[2, 2]]) input_tensor.set(input_array, place) core.set_feed_variable(scope, input_tensor, "feed", 0) output_tensor = core.get_fetch_variable(scope, "feed", 0) - output_lod = output_tensor.lod() - self.assertEqual(0, output_lod[0][0]) + output_lod = output_tensor.recursive_sequence_lengths() + self.assertEqual(2, output_lod[0][0]) self.assertEqual(2, output_lod[0][1]) - self.assertEqual(4, output_lod[0][2]) output_array = np.array(output_tensor) self.assertEqual(3, output_array[0, 0, 0]) diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py index 533d8ccfac..0c75cf33f5 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py @@ -55,7 +55,7 @@ class TestFillConstantBatchSizeLikeWithLoDTensor(OpTest): self.op_type = "fill_constant_batch_size_like" self.inputs = { 'Input': (np.random.random((31, 28)).astype("float32"), - [[0, 9, 23, 31]]) + [[9, 14, 8]]) } self.attrs = { 'value': 3.5, diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 3a13eb872a..8fbf156085 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -20,8 +20,8 @@ from test_lstm_op import identity, sigmoid, tanh, relu class TestGRUOp(OpTest): - lod = [[0, 2, 6, 9]] - batch_size = lod[0][-1] + lod = [[2, 4, 3]] + batch_size = sum(lod[0]) frame_size = 5 activate = { 'identity': identity, @@ -33,10 +33,10 @@ class TestGRUOp(OpTest): @staticmethod def seq_to_batch(lod, is_reverse): idx_in_seq_list = [] - seq_starts = lod[0] - seq_lens = [] - for i in range(len(seq_starts) - 1): - seq_lens.append(seq_starts[i + 1] - seq_starts[i]) + seq_lens = lod[0] + seq_starts = [0] + for i in range(len(seq_lens)): + seq_starts.append(seq_starts[-1] + seq_lens[i]) sorted_seqs = sorted( range(len(seq_lens)), lambda x, y: seq_lens[y] - seq_lens[x]) num_batch = seq_lens[sorted_seqs[0]] diff --git a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py index 8f62ac20a5..eff4212d91 100644 --- a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py +++ b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py @@ -58,8 +58,8 @@ class TestIOUSimilarityOpWithLoD(TestIOUSimilarityOp): def setUp(self): super(TestIOUSimilarityOpWithLoD, self).setUp() - self.boxes1_lod = [[0, 1, 2]] - self.output_lod = [[0, 1, 2]] + self.boxes1_lod = [[1, 1]] + self.output_lod = [[1, 1]] self.inputs = {'X': (self.boxes1, self.boxes1_lod), 'Y': self.boxes2} self.outputs = {'Out': (self.output, self.output_lod)} diff --git a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py index f49f7635f7..696d0ab4fa 100644 --- a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py @@ -105,11 +105,13 @@ class TestLinearChainCrfOp(OpTest): MAX_SEQ_LEN = 5 # the linear_chain_crf operator only supports sequence (LoD level = 1) - lod = [[0]] + lod = [[]] + seq_start_pos = [0] for i in range(SEQ_NUM): - lod[-1].append(lod[-1][-1] + random.randint(1, MAX_SEQ_LEN)) - emission = np.random.uniform(-1, 1, - [lod[-1][-1], TAG_NUM]).astype("float64") + lod[-1].append(random.randint(1, MAX_SEQ_LEN)) + seq_start_pos.append(seq_start_pos[-1] + lod[-1][-1]) + emission = np.random.uniform( + -1, 1, [seq_start_pos[-1], TAG_NUM]).astype("float64") emission_row_max = np.amax(emission, axis=1, keepdims=True) emission_exps = np.exp(emission - emission_row_max) @@ -118,14 +120,14 @@ class TestLinearChainCrfOp(OpTest): transition_exps = np.exp(transition) labels = np.random.randint( - low=0, high=TAG_NUM, size=(lod[-1][-1], 1), dtype="int64") + low=0, high=TAG_NUM, size=(seq_start_pos[-1], 1), dtype="int64") self.inputs = { "Emission": (emission, lod), "Transition": transition, "Label": (labels, lod) } - crf = LinearChainCrfForward(lod[0], emission, emission_row_max, + crf = LinearChainCrfForward(seq_start_pos, emission, emission_row_max, emission_exps, transition, transition_exps, labels) alpha, log_likelihood = crf.crf_forward_compute() diff --git a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py index 093eecb837..bac5e50231 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py +++ b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py @@ -30,7 +30,8 @@ class TestLoDRankTable(unittest.TestCase): tensor = core.LoDTensor() tensor.set(numpy.random.random(size=(17, 100)), cpu) - tensor.set_lod([[0, 1, 3], [0, 5, 6, 7], [0, 3, 4, 9, 10, 13, 16, 17]]) + tensor.set_recursive_sequence_lengths( + [[1, 2], [5, 1, 1], [3, 1, 5, 1, 3, 3, 1]]) exe.run(scope=scope, feed={'x': tensor}) var = scope.find_var(rank_table.name) table = var.get_lod_rank_table() diff --git a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py index 6b6d4c824a..77905c4b96 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py @@ -21,11 +21,15 @@ class TestLodResetOpByAttr(OpTest): def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float32") - lod = [[0, 3, 5, 10]] - target_lod_0 = [0, 7, 10] + lod = [[3, 2, 5]] + # target_offset_lod and target_lod are the same lod info represented + # in offset-based format and length-based format, respectively. + target_offset_lod = [0, 7, 10] + target_lod = [7, 3] self.inputs = {'X': (x, lod)} - self.attrs = {'target_lod': target_lod_0} - self.outputs = {'Out': (x, [target_lod_0])} + # The `target_lod` attribute is still based on offset + self.attrs = {'target_lod': target_offset_lod} + self.outputs = {'Out': (x, [target_lod])} def test_check_output(self): self.check_output() @@ -38,13 +42,16 @@ class TestLodResetOpByInput(OpTest): def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float32") - lod = [[0, 3, 5, 10]] - target_lod_0 = [0, 4, 7, 10] + lod = [[3, 2, 5]] + # target_offset_lod and target_lod are the same lod info represented + # in offset-based format and length-based format, respectively. + target_offset_lod = [0, 4, 7, 10] + target_lod = [4, 3, 3] self.inputs = { 'X': (x, lod), - 'Y': np.array([target_lod_0]).astype('int32') + 'Y': np.array([target_offset_lod]).astype('int32') } - self.outputs = {'Out': (x, [target_lod_0])} + self.outputs = {'Out': (x, [target_lod])} def test_check_output(self): self.check_output() @@ -57,15 +64,16 @@ class TestLodResetOpBoth(OpTest): def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float32") - lod = [[0, 3, 5, 10]] - target_lod_0_attr = [0, 7, 10] - target_lod_0_in = [0, 4, 7, 10] + lod = [[3, 2, 5]] + target_offset_lod_attr = [0, 7, 10] + target_offset_lod_in = [0, 4, 7, 10] + target_lod_in = [4, 3, 3] self.inputs = { 'X': (x, lod), - 'Y': np.array(target_lod_0_in).astype('int32') + 'Y': np.array(target_offset_lod_in).astype('int32') } - self.attrs = {'target_lod': target_lod_0_attr} - self.outputs = {'Out': (x, [target_lod_0_in])} + self.attrs = {'target_lod': target_offset_lod_attr} + self.outputs = {'Out': (x, [target_lod_in])} def test_check_output(self): self.check_output() @@ -78,11 +86,11 @@ class TestLodResetOpYIsLoDTensor(OpTest): def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float32") - lod = [[0, 3, 5, 10]] + lod = [[3, 2, 5]] y = np.random.random((10, 10)).astype("float32") - target_lod_0 = [[0, 4, 7, 10]] - self.inputs = {'X': (x, lod), 'Y': (y, target_lod_0)} - self.outputs = {'Out': (x, target_lod_0)} + target_lod = [[4, 3, 3]] + self.inputs = {'X': (x, lod), 'Y': (y, target_lod)} + self.outputs = {'Out': (x, target_lod)} def test_check_output(self): self.check_output() diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py index 63b17a5ccd..118c22fbb1 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py @@ -27,7 +27,7 @@ class TestLoDTensorArray(unittest.TestCase): for i in xrange(10): t = core.LoDTensor() t.set(numpy.array([i], dtype='float32'), cpu) - t.set_lod([[0, 1]]) + t.set_recursive_sequence_lengths([[1]]) tensor_array.append(t) self.assertEqual(10, len(tensor_array)) @@ -35,17 +35,17 @@ class TestLoDTensorArray(unittest.TestCase): for i in xrange(10): t = tensor_array[i] self.assertEqual(numpy.array(t), numpy.array([i], dtype='float32')) - self.assertEqual([[0, 1]], t.lod()) + self.assertEqual([[1]], t.recursive_sequence_lengths()) t = core.LoDTensor() t.set(numpy.array([i + 10], dtype='float32'), cpu) - t.set_lod([[0, 2]]) + t.set_recursive_sequence_lengths([[1]]) tensor_array[i] = t t = tensor_array[i] self.assertEqual( numpy.array(t), numpy.array( [i + 10], dtype='float32')) - self.assertEqual([[0, 2]], t.lod()) + self.assertEqual([[1]], t.recursive_sequence_lengths()) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py index 66a03640c1..cebe6997bb 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py @@ -29,7 +29,7 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor = core.LoDTensor() tensor.set( numpy.arange(10).reshape(10, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 3, 9, 10]]) + tensor.set_recursive_sequence_lengths([[3, 6, 1]]) expect = map(lambda x: numpy.array(x).astype('int32'), [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]]) self.main( @@ -42,7 +42,7 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor = core.LoDTensor() tensor.set( numpy.arange(10).reshape(10, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 3, 9, 9, 10]]) + tensor.set_recursive_sequence_lengths([[3, 6, 0, 1]]) expect = map(lambda x: numpy.array(x).astype('int32'), [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]]) self.main( @@ -55,7 +55,7 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor = core.LoDTensor() tensor.set( numpy.arange(20).reshape(20, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 2, 5], [0, 3, 9, 11, 17, 20]]) + tensor.set_recursive_sequence_lengths([[2, 3], [3, 6, 2, 6, 3]]) expect = [ numpy.array( @@ -65,7 +65,7 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): [17, 18, 19], dtype='int32') ] - lod = [[[0, 2, 5]], [[0, 6, 12]], [[0, 3]]] + lod = [[[2, 3]], [[6, 6]], [[3]]] self.main( tensor=tensor, expect_array=expect, @@ -77,8 +77,8 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor.set( numpy.arange(31).reshape(31, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 3, 5, 9, 11], - [0, 3, 7, 11, 11, 12, 17, 19, 21, 23, 30, 31]]) + tensor.set_recursive_sequence_lengths( + [[3, 2, 4, 2], [3, 4, 4, 0, 1, 5, 2, 2, 2, 7, 1]]) expect = [ numpy.array( @@ -88,7 +88,7 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): ], [17, 18, 3, 4, 5, 6, 11, 30], [19, 20, 7, 8, 9, 10], [21, 22]] ] - lod = [[[0, 5, 8, 8, 15]], [[0, 2, 6, 7, 8]], [[0, 2, 6]], [[0, 2]]] + lod = [[[5, 3, 0, 7]], [[2, 4, 1, 1]], [[2, 4]], [[2]]] self.main( tensor=tensor, expect_array=expect, @@ -99,8 +99,9 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor = core.LoDTensor() tensor.set( numpy.arange(50).reshape(50, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 2, 5, 6], [0, 2, 5, 6, 10, 12, 13], - [0, 3, 7, 11, 17, 21, 22, 23, 27, 31, 39, 45, 46, 50]]) + tensor.set_recursive_sequence_lengths( + [[2, 3, 1], [2, 3, 1, 4, 2, 1], + [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4]]) expect = [ numpy.array( @@ -108,8 +109,8 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): for item in [[21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], range( 22, 39) + range(7, 21), range(39, 46)] ] - lod = [[[0, 1, 3, 4], [0, 1, 4, 8, 12]], - [[0, 4, 7], [0, 1, 5, 9, 17, 21, 27, 31]], [[0, 2], [0, 6, 7]]] + lod = [[[1, 2, 1], [1, 3, 4, 4]], [[4, 3], [1, 4, 4, 8, 4, 6, 4]], + [[2], [6, 1]]] self.main( tensor=tensor, expect_array=expect, @@ -120,8 +121,9 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor = core.LoDTensor() tensor.set( numpy.arange(50).reshape(50, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 2, 5, 6], [0, 2, 5, 6, 10, 12, 13], - [0, 3, 7, 11, 17, 21, 22, 23, 27, 31, 39, 45, 46, 50]]) + tensor.set_recursive_sequence_lengths( + [[2, 3, 1], [2, 3, 1, 4, 2, 1], + [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4]]) self.main( tensor=tensor, expect_array=None, @@ -162,12 +164,13 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): exp_tensor, exp_lod = exp exp_tensor = numpy.expand_dims(exp_tensor, axis=1) self.assertTrue(numpy.allclose(exp_tensor, numpy.array(array[i]))) - self.assertEqual(exp_lod, array[i].lod()) + self.assertEqual(exp_lod, array[i].recursive_sequence_lengths()) def check_tensor_same(self, actual, expect): self.assertTrue( numpy.allclose(numpy.array(actual), numpy.array(expect))) - self.assertEqual(actual.lod(), expect.lod()) + self.assertEqual(actual.recursive_sequence_lengths(), + expect.recursive_sequence_lengths()) class TestCPULoDTensorArrayOpGrad(unittest.TestCase): @@ -188,7 +191,7 @@ class TestCPULoDTensorArrayOpGrad(unittest.TestCase): tensor = core.LoDTensor() tensor.set(numpy.arange(10).reshape(10, 1).astype('float32'), place) - tensor.set_lod([[0, 3, 9, 10]]) + tensor.set_recursive_sequence_lengths([[3, 6, 1]]) g_vars = program.global_block().var(x.name + "@GRAD") diff --git a/python/paddle/fluid/tests/unittests/test_lstm_op.py b/python/paddle/fluid/tests/unittests/test_lstm_op.py index e726f99d49..705a24bd8f 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_op.py @@ -84,15 +84,17 @@ def lstm( h = g_o * act_cell(c) return h, c - def _reverse(x, lod): + def _reverse(x, offset): y = np.zeros_like(x) - for i in range(len(lod) - 1): - b, e = lod[i], lod[i + 1] + for i in range(len(offset) - 1): + b, e = offset[i], offset[i + 1] y[b:e, :] = np.flip(x[b:e, :], 0) return y - offset = lod[0] - batch_size = len(offset) - 1 + offset = [0] + for l in lod[0]: + offset.append(offset[-1] + l) + batch_size = len(lod[0]) hidden = [] cell = [] input = _reverse(input, offset) if is_reverse else input @@ -100,7 +102,7 @@ def lstm( input = input + np.tile(w_b, (offset[-1], 1)) for i in range(batch_size): # compute one sequence - seq_len = offset[i + 1] - offset[i] + seq_len = lod[0][i] x = input[offset[i]:offset[i + 1], :] h_pre = h0[i] # 1 x D c_pre = c0[i] # 1 x D @@ -124,7 +126,7 @@ def lstm( class TestLstmOp(OpTest): def set_argument(self): - self.lod = [[0, 2, 5, 7]] + self.lod = [[2, 3, 2]] self.D = 16 self.act_gate = 'sigmoid' @@ -139,8 +141,8 @@ class TestLstmOp(OpTest): self.set_argument() self.op_type = 'lstm' - T = self.lod[0][-1] - N = len(self.lod[0]) - 1 + T = sum(self.lod[0]) + N = len(self.lod[0]) x = np.random.normal(size=(T, 4 * self.D)).astype('float64') if self.has_initial_state: @@ -186,7 +188,7 @@ class TestLstmOp(OpTest): def test_check_grad(self): # TODO(qingqing) remove folowing lines after the check_grad is refined. - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchCellPreAct'] = np.zeros( (N, self.D)).astype('float64') @@ -196,7 +198,7 @@ class TestLstmOp(OpTest): # class TestLstmOpHasInitial(TestLstmOp): # def set_argument(self): -# self.lod = [[0, 2, 5, 7]] +# self.lod = [[2, 3, 2]] # self.D = 16 # self.act_gate = 'sigmoid' @@ -209,7 +211,7 @@ class TestLstmOp(OpTest): # def test_check_grad(self): # # TODO(qingqing) remove folowing lines after the check_grad is refined. -# N = len(self.lod[0]) - 1 +# N = len(self.lod[0]) # self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') # self.outputs['BatchCellPreAct'] = np.zeros( # (N, self.D)).astype('float64') @@ -218,7 +220,7 @@ class TestLstmOp(OpTest): # max_relative_error=5e-4) # def test_check_grad_ingore_bias(self): -# N = len(self.lod[0]) - 1 +# N = len(self.lod[0]) # self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') # self.outputs['BatchCellPreAct'] = np.zeros( # (N, self.D)).astype('float64') @@ -228,7 +230,7 @@ class TestLstmOp(OpTest): # no_grad_set=set('Bias')) # def test_check_grad_ingore_weight(self): -# N = len(self.lod[0]) - 1 +# N = len(self.lod[0]) # self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') # self.outputs['BatchCellPreAct'] = np.zeros( # (N, self.D)).astype('float64') @@ -238,7 +240,7 @@ class TestLstmOp(OpTest): # no_grad_set=set('Weight')) # def test_check_grad_ingore_input(self): -# N = len(self.lod[0]) - 1 +# N = len(self.lod[0]) # self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') # self.outputs['BatchCellPreAct'] = np.zeros( # (N, self.D)).astype('float64') @@ -248,7 +250,7 @@ class TestLstmOp(OpTest): # no_grad_set=set('Input')) # def test_check_grad_ingore_h0(self): -# N = len(self.lod[0]) - 1 +# N = len(self.lod[0]) # self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') # self.outputs['BatchCellPreAct'] = np.zeros( # (N, self.D)).astype('float64') @@ -258,7 +260,7 @@ class TestLstmOp(OpTest): # no_grad_set=set('H0')) # def test_check_grad_ingore_c0(self): -# N = len(self.lod[0]) - 1 +# N = len(self.lod[0]) # self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') # self.outputs['BatchCellPreAct'] = np.zeros( # (N, self.D)).astype('float64') @@ -269,7 +271,7 @@ class TestLstmOp(OpTest): # class TestLstmOpRerverse(TestLstmOp): # def set_argument(self): -# self.lod = [[0, 2, 5, 7]] +# self.lod = [[2, 3, 2]] # self.D = 16 # self.act_gate = 'sigmoid' @@ -282,7 +284,7 @@ class TestLstmOp(OpTest): # class TestLstmOpNotUsePeepholes(TestLstmOp): # def set_argument(self): -# self.lod = [[0, 2, 5, 7]] +# self.lod = [[2, 3, 2]] # self.D = 16 # self.act_gate = 'sigmoid' diff --git a/python/paddle/fluid/tests/unittests/test_lstmp_op.py b/python/paddle/fluid/tests/unittests/test_lstmp_op.py index afff133f6c..ed2262da4b 100644 --- a/python/paddle/fluid/tests/unittests/test_lstmp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstmp_op.py @@ -64,15 +64,17 @@ def lstmp( r = act_proj(r) return r, c - def _reverse(x, lod): + def _reverse(x, offset): y = np.zeros_like(x) - for i in range(len(lod) - 1): - b, e = lod[i], lod[i + 1] + for i in range(len(offset) - 1): + b, e = offset[i], offset[i + 1] y[b:e, :] = np.flip(x[b:e, :], 0) return y - offset = lod[0] - batch_size = len(offset) - 1 + offset = [0] + for l in lod[0]: + offset.append(offset[-1] + l) + batch_size = len(lod[0]) # recurrent projection state projection = [] cell = [] @@ -81,7 +83,7 @@ def lstmp( input = input + np.tile(w_b, (offset[-1], 1)) for i in range(batch_size): # compute one sequence - seq_len = offset[i + 1] - offset[i] + seq_len = lod[0][i] x = input[offset[i]:offset[i + 1], :] r_pre = np.dot(h0[i], w_rh) # 1 x P r_pre = act_proj(r_pre) @@ -117,8 +119,8 @@ class TestLstmpOp(LstmTest.TestLstmOp): self.reset_argument() self.op_type = 'lstmp' - T = self.lod[0][-1] - N = len(self.lod[0]) - 1 + T = sum(self.lod[0]) + N = len(self.lod[0]) x = np.random.normal(size=(T, 4 * self.D)).astype('float64') if self.has_initial_state: @@ -166,7 +168,7 @@ class TestLstmpOp(LstmTest.TestLstmOp): def test_check_grad(self): # TODO(qingqing) remove folowing lines after the check_grad is refined. - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -183,7 +185,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): def test_check_grad(self): # TODO(qingqing) remove folowing lines after the check_grad is refined. - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -195,7 +197,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): max_relative_error=1e-2) def test_check_grad_ingore_bias(self): - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -207,7 +209,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): no_grad_set=set('Bias')) def test_check_grad_ingore_weight(self): - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -219,7 +221,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): no_grad_set=set('Weight')) def test_check_grad_ingore_proj_weight(self): - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -231,7 +233,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): no_grad_set=set('ProjWeight')) def test_check_grad_ingore_input(self): - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -243,7 +245,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): no_grad_set=set('Input')) def test_check_grad_ingore_h0(self): - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') @@ -255,7 +257,7 @@ class TestLstmpOpHasInitial(TestLstmpOp): no_grad_set=set('H0')) def test_check_grad_ingore_c0(self): - N = len(self.lod[0]) - 1 + N = len(self.lod[0]) self.outputs['OrderedP0'] = np.zeros((N, self.P)).astype('float64') self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') diff --git a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py index c27573c3d6..54ee85c1a7 100644 --- a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py +++ b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py @@ -70,7 +70,7 @@ class TestMineHardExamplesOp(OpTest): self.updated_match_indices = self.match_indices - self.neg_indices_lod = [[0, 1, 2]] + self.neg_indices_lod = [[1, 1]] self.neg_indices = np.array([[1], [0]]).astype('int32') @@ -92,7 +92,7 @@ class TestMineHardExamplesOpHardExample(TestMineHardExamplesOp): self.updated_match_indices = np.array([[0, -1, -1], [-1, -1, -1]]).astype('int32') - self.neg_indices_lod = [[0, 1, 3]] + self.neg_indices_lod = [[1, 2]] self.neg_indices = np.array([[2], [0], [2]]).astype('int32') diff --git a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py index 6459913c01..aacd8ae45a 100644 --- a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py @@ -135,12 +135,12 @@ def batched_multiclass_nms(boxes, scores, background, score_threshold, batch_size = scores.shape[0] det_outs = [] - lod = [0] + lod = [] for n in range(batch_size): nmsed_outs, nmsed_num = multiclass_nms(boxes[n], scores[n], background, score_threshold, nms_threshold, nms_top_k, keep_top_k) - lod.append(lod[-1] + nmsed_num) + lod.append(nmsed_num) if nmsed_num == 0: continue for c, indices in nmsed_outs.iteritems(): diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_op.py index cd78cce872..d13f2b3afd 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_op.py @@ -27,9 +27,9 @@ class TestOneHotOp(OpTest): self.op_type = 'one_hot' depth = 10 dimension = 12 - x_lod = [[0, 4, 5, 8, 11]] - x = [np.random.randint(0, depth - 1) for i in xrange(x_lod[0][-1])] - x = np.array(x).astype('int').reshape([x_lod[0][-1], 1]) + x_lod = [[4, 1, 3, 3]] + x = [np.random.randint(0, depth - 1) for i in xrange(sum(x_lod[0]))] + x = np.array(x).astype('int').reshape([sum(x_lod[0]), 1]) out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype('float32') @@ -50,9 +50,9 @@ class TestOneHotOp_default_dtype(OpTest): self.op_type = 'one_hot' depth = 10 dimension = 12 - x_lod = [[0, 4, 5, 8, 11]] - x = [np.random.randint(0, depth - 1) for i in xrange(x_lod[0][-1])] - x = np.array(x).astype('int').reshape([x_lod[0][-1], 1]) + x_lod = [[4, 1, 3, 3]] + x = [np.random.randint(0, depth - 1) for i in xrange(sum(x_lod[0]))] + x = np.array(x).astype('int').reshape([sum(x_lod[0]), 1]) out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype('float32') @@ -75,11 +75,11 @@ class TestOneHotOp_exception(OpTest): self.place = core.CPUPlace() self.dimension = 12 self.x = core.LoDTensor() - x_lod = [[0, 4, 5, 8, 11]] - data = [np.random.randint(11, 20) for i in xrange(x_lod[0][-1])] - data = np.array(data).astype('int').reshape([x_lod[0][-1], 1]) + x_lod = [[4, 1, 3, 3]] + data = [np.random.randint(11, 20) for i in xrange(sum(x_lod[0]))] + data = np.array(data).astype('int').reshape([sum(x_lod[0]), 1]) self.x.set(data, self.place) - self.x.set_lod(x_lod) + self.x.set_recursive_sequence_lengths(x_lod) def test_check_output(self): program = Program() diff --git a/python/paddle/fluid/tests/unittests/test_print_op.py b/python/paddle/fluid/tests/unittests/test_print_op.py index c75080fbb9..e01af42a58 100644 --- a/python/paddle/fluid/tests/unittests/test_print_op.py +++ b/python/paddle/fluid/tests/unittests/test_print_op.py @@ -28,7 +28,7 @@ class TestPrintOpCPU(unittest.TestCase): self.x_tensor = core.LoDTensor() tensor_np = np.random.random(size=(2, 3)).astype('float32') self.x_tensor.set(tensor_np, self.place) - self.x_tensor.set_lod([[0, 1, 1]]) + self.x_tensor.set_recursive_sequence_lengths([[1, 1]]) def build_network(self, only_forward, **kargs): x = layers.data('x', shape=[3], dtype='float32', lod_level=1) @@ -62,7 +62,7 @@ class TestPrintOpGPU(TestPrintOpCPU): self.x_tensor = core.LoDTensor() tensor_np = np.random.random(size=(2, 3)).astype('float32') self.x_tensor.set(tensor_np, self.place) - self.x_tensor.set_lod([[0, 1, 1]]) + self.x_tensor.set_recursive_sequence_lengths([[1, 1]]) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py index 76d0d2f2fe..a70321bd80 100644 --- a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py @@ -70,11 +70,10 @@ class TestReorderLoDTensor(unittest.TestCase): lod_level_i = numpy.random.randint( low=1, high=5, - size=self.num_seq if i == 0 else lod_level_i[-1]) - lod_level_i = [0] + numpy.cumsum(lod_level_i).tolist() + size=self.num_seq if i == 0 else sum(lod_level_i)).tolist() data_lod.append(lod_level_i) data_value = numpy.random.random( - size=[data_lod[-1][-1] if data_lod else self.num_seq + size=[sum(data_lod[-1]) if data_lod else self.num_seq ] + data_shape).astype('float32') self.data[data_name] = (data_value, data_lod) @@ -84,29 +83,36 @@ class TestReorderLoDTensor(unittest.TestCase): tensor = fluid.Tensor() tensor.set(self.data[desc[0]][0], place) if self.data[desc[0]][1]: - tensor.set_lod(self.data[desc[0]][1]) + tensor.set_recursive_sequence_lengths(self.data[desc[0]][1]) self.inputs[desc[0]] = tensor def reorder(self): - level = 0 + def convert_to_offset(lod): + offset_lod = [[0] for i in lod] + for i, level in enumerate(lod): + for seq_len in level: + offset_lod[i].append(offset_lod[i][-1] + seq_len) + return offset_lod + level = 0 # compute the rank_table according to ref_lod ref_lod = self.data[self.data_desc[1][0]][1][level] rank_table = [] # list of (index, length) - for i in range(len(ref_lod) - 1): - rank_table.append((i, ref_lod[i + 1] - ref_lod[i])) + for i in range(len(ref_lod)): + rank_table.append((i, ref_lod[i])) rank_table = sorted(rank_table, lambda x, y: y[1] - x[1]) # compute the input sequence info according to input_lod input_value, input_lod = self.data[self.data_desc[0][0]] + offset_lod = convert_to_offset(input_lod) input_table = [] # list of (offset, length, sub_lod) - if input_lod: - for i in range(len(input_lod[level]) - 1): + if offset_lod: + for i in range(len(offset_lod[level]) - 1): start_idx = i end_idx = i + 1 sub_lod = [] - for lod_level_i in input_lod[level:]: + for lod_level_i in offset_lod[level:]: sub_lod_i = [] for idx in range(start_idx, end_idx): sub_lod_i.append(lod_level_i[idx + 1] - lod_level_i[ @@ -132,10 +138,9 @@ class TestReorderLoDTensor(unittest.TestCase): input_seq_sub_lod = input_table[index][2] if len(output_lod) == 0: - output_lod = [[0] for i in input_seq_sub_lod] - for i, sub_lod_i in enumerate(input_seq_sub_lod): - for idx_sub in sub_lod_i: - output_lod[i].append(output_lod[i][-1] + idx_sub) + output_lod = [[] for i in input_seq_sub_lod] + for i, level in enumerate(input_seq_sub_lod): + output_lod[i].extend(level) return output_value, output_lod def test_reorder_lod_tensor(self): @@ -148,7 +153,8 @@ class TestReorderLoDTensor(unittest.TestCase): self.assertTrue( numpy.allclose( numpy.array(actual_output), expect_output, atol=0.001)) - self.assertEqual(expect_output_lod, actual_output.lod()) + self.assertEqual(expect_output_lod, + actual_output.recursive_sequence_lengths()) # check gradient expect_grad = numpy.ones_like(self.data[self.data_desc[0][0]][0]) expect_grad_lod = self.data[self.data_desc[0][0]][1] @@ -156,7 +162,8 @@ class TestReorderLoDTensor(unittest.TestCase): self.assertTrue( numpy.allclose( numpy.array(actual_grad), expect_grad, atol=0.001)) - self.assertEqual(expect_grad_lod, actual_grad.lod()) + self.assertEqual(expect_grad_lod, + actual_grad.recursive_sequence_lengths()) def test_reorder_tensor(self): self.data_desc[0][-1] = 0 # input is tensor @@ -168,7 +175,8 @@ class TestReorderLoDTensor(unittest.TestCase): self.assertTrue( numpy.allclose( numpy.array(actual_output), expect_output, atol=0.001)) - self.assertEqual(expect_output_lod, actual_output.lod()) + self.assertEqual(expect_output_lod, + actual_output.recursive_sequence_lengths()) # check gradient expect_grad = numpy.ones_like(self.data[self.data_desc[0][0]][0]) expect_grad_lod = self.data[self.data_desc[0][0]][1] @@ -176,14 +184,14 @@ class TestReorderLoDTensor(unittest.TestCase): self.assertTrue( numpy.allclose( numpy.array(actual_grad), expect_grad, atol=0.001)) - self.assertEqual(expect_grad_lod, actual_grad.lod()) + self.assertEqual(expect_grad_lod, + actual_grad.recursive_sequence_lengths()) # compare outputs between LodTensors with explicit and implicit lod # use the same data but set the input lod explicitly - input_lod = [[ - i for i in range(len(self.data[self.data_desc[0][0]][0]) + 1) - ]] - self.inputs[self.data_desc[0][0]].set_lod(input_lod) + input_lod = [[1] * len(self.data[self.data_desc[0][0]][0])] + self.inputs[self.data_desc[0][0]].set_recursive_sequence_lengths( + input_lod) # preserve the output of LodTensor with implicit lod to compare expect_output = [ numpy.array(actual_output) for actual_output in self.actual_outputs diff --git a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py index 3d754aff3a..df5684ab17 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py @@ -107,7 +107,7 @@ class TestROIPoolOp(OpTest): rois = [] self.rois_lod = [[]] for bno in range(self.batch_size): - self.rois_lod[0].append(len(rois)) + self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( 0, self.width / self.spatial_scale - self.pooled_width) @@ -121,7 +121,6 @@ class TestROIPoolOp(OpTest): roi = [bno, x1, y1, x2, y2] rois.append(roi) - self.rois_lod[0].append(len(rois)) self.rois_num = len(rois) self.rois = np.array(rois).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/test_row_conv_op.py b/python/paddle/fluid/tests/unittests/test_row_conv_op.py index 30f1efbcbc..07dcd10868 100644 --- a/python/paddle/fluid/tests/unittests/test_row_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_row_conv_op.py @@ -19,8 +19,10 @@ from op_test import OpTest def row_conv_forward(x, lod, wt): out = np.zeros_like(x) - seq_info = lod[0] - num_sequences = len(seq_info) - 1 + num_sequences = len(lod[0]) + seq_info = [0] + for seq_len in lod[0]: + seq_info.append(seq_info[-1] + seq_len) context_length = wt.shape[0] for i in range(num_sequences): # loop over number of sequences @@ -32,7 +34,6 @@ def row_conv_forward(x, lod, wt): cur_timesteps = end - start for j in range(cur_timesteps): # loop over different timesteps for k in range(context_length): - if j + k >= cur_timesteps: continue curoutput[j, :] += curinput[j + k, :] * wt[k, :] @@ -44,8 +45,8 @@ class TestRowConvOp1(OpTest): def setUp(self): self.op_type = "row_conv" - lod = [[0, 2, 5, 7]] - T = lod[0][-1] + lod = [[2, 3, 2]] + T = sum(lod[0]) D = 16 context_length = 2 @@ -75,8 +76,8 @@ class TestRowConvOp2(OpTest): def setUp(self): self.op_type = "row_conv" - lod = [[0, 20, 50, 100]] - T = lod[0][-1] + lod = [[20, 30, 50]] + T = sum(lod[0]) D = 35 context_length = 35 diff --git a/python/paddle/fluid/tests/unittests/test_seq_concat_op.py b/python/paddle/fluid/tests/unittests/test_seq_concat_op.py index 10592d127f..11ffa761a6 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_seq_concat_op.py @@ -18,14 +18,19 @@ import sys from op_test import OpTest -def to_abs_lod(lod): - if len(lod) == 0 or len(lod) == 1: - return lod +def to_abs_offset_lod(lod): + offset_lod = [[0] for i in lod] + for i, level in enumerate(lod): + for seq_len in level: + offset_lod[i].append(offset_lod[i][-1] + seq_len) + + if len(offset_lod) == 0 or len(offset_lod) == 1: + return offset_lod import copy - new_lod = copy.deepcopy(lod) - for idx, val in enumerate(lod[0]): - new_lod[0][idx] = lod[1][val] - return new_lod + new_offset_lod = copy.deepcopy(offset_lod) + for idx, val in enumerate(offset_lod[0]): + new_offset_lod[0][idx] = offset_lod[1][val] + return new_offset_lod def seq_concat(inputs, level): @@ -35,11 +40,11 @@ def seq_concat(inputs, level): x1 = inputs['X'][1][1][0] level_idx = len(lod0) - level - 1 outs = [] - for i in range(len(lod0[level_idx]) - 1): - sub_x0 = x0[to_abs_lod(lod0)[level_idx][i]:to_abs_lod(lod0)[level_idx][ - i + 1], :] - sub_x1 = x1[to_abs_lod(lod1)[level_idx][i]:to_abs_lod(lod1)[level_idx][ - i + 1], :] + for i in range(len(lod0[level_idx])): + sub_x0 = x0[to_abs_offset_lod(lod0)[level_idx][i]:to_abs_offset_lod( + lod0)[level_idx][i + 1], :] + sub_x1 = x1[to_abs_offset_lod(lod1)[level_idx][i]:to_abs_offset_lod( + lod1)[level_idx][i + 1], :] outs.append(np.concatenate((sub_x0, sub_x1), axis=0)) return np.concatenate(outs, axis=0) @@ -48,9 +53,9 @@ class TestSeqConcatOp(OpTest): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') - lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] + lod0 = [[2, 2], [1, 1, 1, 1]] x1 = np.random.random((4, 8, 3)).astype('float32') - lod1 = [[0, 2, 4], [0, 1, 2, 3, 4]] + lod1 = [[2, 2], [1, 1, 1, 1]] axis = 1 level = 1 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} @@ -72,14 +77,14 @@ class TestSeqConcatOpLevelZeroNestedSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') - lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] + lod0 = [[2, 2], [1, 1, 1, 1]] x1 = np.random.random((7, 6, 3)).astype('float32') - lod1 = [[0, 2, 4], [0, 1, 3, 5, 7]] + lod1 = [[2, 2], [1, 2, 2, 2]] axis = 0 level = 0 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - out_lod = [[0, 2, 4], [0, 2, 5, 8, 11]] + out_lod = [[2, 2], [2, 3, 3, 3]] self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} @@ -87,14 +92,14 @@ class TestSeqConcatOplevelOneNestedSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') - lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] + lod0 = [[2, 2], [1, 1, 1, 1]] x1 = np.random.random((7, 6, 3)).astype('float32') - lod1 = [[0, 3, 4], [0, 1, 3, 5, 7]] + lod1 = [[3, 1], [1, 2, 2, 2]] axis = 0 level = 1 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - out_lod = [[0, 5, 8], [0, 1, 2, 3, 5, 7, 8, 9, 11]] + out_lod = [[5, 3], [1, 1, 1, 2, 2, 1, 1, 2]] self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} @@ -102,14 +107,14 @@ class TestSeqConcatOpLevelZeroSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 3, 4)).astype('float32') - lod0 = [[0, 1, 2, 3, 4]] + lod0 = [[1, 1, 1, 1]] x1 = np.random.random((7, 3, 4)).astype('float32') - lod1 = [[0, 1, 3, 5, 7]] + lod1 = [[1, 2, 2, 2]] axis = 0 level = 0 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - out_lod = [[0, 2, 5, 8, 11]] + out_lod = [[2, 3, 3, 3]] self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} diff --git a/python/paddle/fluid/tests/unittests/test_seq_conv.py b/python/paddle/fluid/tests/unittests/test_seq_conv.py index 51dbf1f618..9701d9adef 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_conv.py +++ b/python/paddle/fluid/tests/unittests/test_seq_conv.py @@ -75,35 +75,38 @@ class TestSeqProject(OpTest): pading_data = self.pad_data out = np.zeros((self.input_size[0], self.context_length * self.input_size[1])).astype('float32') - lod = lod[0] + offset = [0] + for seq_len in lod[0]: + offset.append(offset[-1] + seq_len) begin_pad = np.max([0, -self.context_start]) - for i in range(len(lod) - 1): + for i in range(len(offset) - 1): for j in range(self.context_length): - in_begin = lod[i] + self.context_start + j - in_end = lod[i + 1] + self.context_start + j - out_begin = lod[i] - out_end = lod[i + 1] - if in_begin < lod[i]: - pad_size = np.min([lod[i] - in_begin, lod[i + 1] - lod[i]]) + in_begin = offset[i] + self.context_start + j + in_end = offset[i + 1] + self.context_start + j + out_begin = offset[i] + out_end = offset[i + 1] + if in_begin < offset[i]: + pad_size = np.min( + [offset[i] - in_begin, offset[i + 1] - offset[i]]) if self.padding_trainable: sub_w = pading_data[j:j + pad_size, :] - out[lod[i]:lod[i] + pad_size, j * self.input_size[1]:( - j + 1) * self.input_size[1]] = sub_w - out_begin = lod[i] + pad_size - in_begin = lod[i] + out[offset[i]:offset[i] + pad_size, j * self.input_size[ + 1]:(j + 1) * self.input_size[1]] = sub_w + out_begin = offset[i] + pad_size + in_begin = offset[i] - if in_end > lod[i + 1]: + if in_end > offset[i + 1]: pad_size = np.min( - [in_end - lod[i + 1], lod[i + 1] - lod[i]]) + [in_end - offset[i + 1], offset[i + 1] - offset[i]]) if self.padding_trainable: sub_w = pading_data[begin_pad + self.context_start + j - pad_size:begin_pad + self.context_start + j, :] - out[lod[i + 1] - pad_size:lod[i + 1], j * self. + out[offset[i + 1] - pad_size:offset[i + 1], j * self. input_size[1]:(j + 1) * self.input_size[1]] = sub_w - in_end = lod[i + 1] - out_end = lod[i + 1] - pad_size + in_end = offset[i + 1] + out_end = offset[i + 1] - pad_size if in_end <= in_begin: continue @@ -175,7 +178,11 @@ class TestSeqProject(OpTest): self.context_stride = 1 self.input_size = [self.input_row, 23] - self.lod = [[0, 4, 5, 8, self.input_row]] + offset_lod = [[0, 4, 5, 8, self.input_row]] + self.lod = [[]] + # convert from offset-based lod to length-based lod + for i in range(len(offset_lod[0]) - 1): + self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i]) self.output_represention = 8 # output feature size @@ -188,7 +195,11 @@ class TestSeqProjectCase1(TestSeqProject): self.context_stride = 1 self.input_size = [self.input_row, 23] - self.lod = [[0, 4, 5, 8, self.input_row]] + offset_lod = [[0, 4, 5, 8, self.input_row]] + self.lod = [[]] + # convert from offset-based lod to length-based lod + for i in range(len(offset_lod[0]) - 1): + self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i]) self.output_represention = 8 # output feature size @@ -203,8 +214,12 @@ class TestSeqProjectCase2(TestSeqProject): self.input_size = [self.input_row, 23] idx = range(self.input_size[0]) del idx[0] - self.lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + - [self.input_size[0]]] + offset_lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + + [self.input_size[0]]] + self.lod = [[]] + # convert from offset-based lod to length-based lod + for i in range(len(offset_lod[0]) - 1): + self.lod[0].append(offset_lod[0][i + 1] - offset_lod[0][i]) self.output_represention = 8 # output feature size diff --git a/python/paddle/fluid/tests/unittests/test_seq_pool.py b/python/paddle/fluid/tests/unittests/test_seq_pool.py index 2e48ef0e88..0b3659d7a6 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_pool.py +++ b/python/paddle/fluid/tests/unittests/test_seq_pool.py @@ -18,26 +18,34 @@ from op_test import OpTest class TestSeqAvgPool(OpTest): + def convert_to_offset(self, lod): + offset = [[0] for i in lod] + for i, level in enumerate(lod): + for seq_len in level: + offset[i].append(offset[i][-1] + seq_len) + return offset + def set_data(self): self.op_type = 'sequence_pool' # one level, batch size is 4 x = np.random.uniform(0.1, 1, [11, 23]).astype('float32') - lod = [[0, 4, 5, 8, 11]] + lod = [[4, 1, 3, 3]] self.inputs = {'X': (x, lod)} + offset = self.convert_to_offset(lod) out = np.zeros((4, 23)).astype('float32') self.outputs = {'Out': out} - return x, lod, out + return x, offset, out - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "AVERAGE"} - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] + for i in range(len(offset[0]) - 1): + sub_x = x[offset[0][i]:offset[0][i + 1], :] out[i] = sub_x.mean(axis=0) def setUp(self): - x, lod, out = self.set_data() - self.compute(x, lod, out) + x, offset, out = self.set_data() + self.compute(x, offset, out) def test_check_output(self): self.check_output() @@ -50,10 +58,10 @@ class TestSeqAvgPool(OpTest): class TestSeqSumPool(TestSeqAvgPool): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "SUM"} - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] + for i in range(len(offset[0]) - 1): + sub_x = x[offset[0][i]:offset[0][i + 1], :] out[i] = sub_x.sum(axis=0) @@ -61,46 +69,47 @@ class TestSeqMaxPool(TestSeqAvgPool): def set_data(self): self.op_type = 'sequence_pool' x = np.random.uniform(0.1, 1, [13, 23]).astype('float32') - lod = [[0, 4, 5, 8, 13]] - for i in range(4): - l = lod[0][i + 1] - lod[0][i] - x[lod[0][i] + np.random.randint(l), :] += 2.0 + lod = [[4, 1, 3, 5]] + offset = self.convert_to_offset(lod) + for i in range(len(offset[0]) - 1): + l = offset[0][i + 1] - offset[0][i] + x[offset[0][i] + np.random.randint(l), :] += 2.0 self.inputs = {'X': (x, lod)} out = np.zeros((4, 23)).astype('float32') self.outputs = {'Out': out} - return x, lod, out + return x, offset, out - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "MAX"} - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] + for i in range(len(offset[0]) - 1): + sub_x = x[offset[0][i]:offset[0][i + 1], :] out[i] = np.amax(sub_x, axis=0) class TestSeqSqrtPool(TestSeqAvgPool): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "SQRT"} - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] - len = lod[0][i + 1] - lod[0][i] - out[i] = sub_x.sum(axis=0) / np.sqrt(len) + for i in range(len(offset[0]) - 1): + sub_x = x[offset[0][i]:offset[0][i + 1], :] + seq_len = offset[0][i + 1] - offset[0][i] + out[i] = sub_x.sum(axis=0) / np.sqrt(seq_len) class TestSeqLastPool(TestSeqAvgPool): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "LAST"} - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] + for i in range(len(offset[0]) - 1): + sub_x = x[offset[0][i]:offset[0][i + 1], :] out[i] = sub_x[-1, :] class TestSeqFirstPool(TestSeqAvgPool): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "FIRST"} - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] + for i in range(len(offset[0]) - 1): + sub_x = x[offset[0][i]:offset[0][i + 1], :] out[i] = sub_x[0, :] @@ -109,35 +118,39 @@ class TestSeqAvgPool2D(TestSeqAvgPool): self.op_type = 'sequence_pool' # one level, batch size is 4 x = np.random.uniform(0.1, 1, [13, 3, 17]).astype('float32') - lod = [[0, 4, 5, 8, 13]] + lod = [[4, 1, 3, 5]] self.inputs = {'X': (x, lod)} + offset = self.convert_to_offset(lod) out = np.zeros((4, 3, 17)).astype('float32') self.outputs = {'Out': out} - return x, lod, out + return x, offset, out - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "AVERAGE"} - for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) + for i in range(len(offset[0]) - 1): + sub_x = np.reshape(x[offset[0][i]:offset[0][i + 1], :], + (-1, 3 * 17)) out[i] = np.reshape(sub_x.mean(axis=0), (3, 17)) class TestSeqSumPool2D(TestSeqAvgPool2D): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "SUM"} - for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) + for i in range(len(offset[0]) - 1): + sub_x = np.reshape(x[offset[0][i]:offset[0][i + 1], :], + (-1, 3 * 17)) out[i] = np.reshape(sub_x.sum(axis=0), (3, 17)) class TestSeqSqrtPool2D(TestSeqAvgPool2D): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "SQRT"} - for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) - len = lod[0][i + 1] - lod[0][i] - out[i] = np.reshape(sub_x.sum(axis=0) / np.sqrt(len), (3, 17)) + for i in range(len(offset[0]) - 1): + sub_x = np.reshape(x[offset[0][i]:offset[0][i + 1], :], + (-1, 3 * 17)) + seq_len = offset[0][i + 1] - offset[0][i] + out[i] = np.reshape(sub_x.sum(axis=0) / np.sqrt(seq_len), (3, 17)) def test_check_grad(self): # Remove MaxIndex after check_grad is refined. @@ -150,36 +163,40 @@ class TestSeqMaxPool2D(TestSeqAvgPool2D): def set_data(self): self.op_type = 'sequence_pool' x = np.random.uniform(0.1, 1, [13, 3, 11]).astype('float32') - lod = [[0, 4, 5, 8, 13]] + lod = [[4, 1, 3, 5]] self.inputs = {'X': (x, lod)} - for i in range(4): - l = lod[0][i + 1] - lod[0][i] - x[lod[0][i] + np.random.randint(l), :] += 1.0 + offset = self.convert_to_offset(lod) + for i in range(len(offset[0]) - 1): + l = offset[0][i + 1] - offset[0][i] + x[offset[0][i] + np.random.randint(l), :] += 1.0 out = np.zeros((4, 3, 11)).astype('float32') self.outputs = {'Out': out} - return x, lod, out + return x, offset, out - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "MAX"} - for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 11)) + for i in range(len(offset[0]) - 1): + sub_x = np.reshape(x[offset[0][i]:offset[0][i + 1], :], + (-1, 3 * 11)) out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11)) class TestSeqLastPool2D(TestSeqAvgPool2D): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "LAST"} - for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) + for i in range(len(offset[0]) - 1): + sub_x = np.reshape(x[offset[0][i]:offset[0][i + 1], :], + (-1, 3 * 17)) out[i] = np.reshape(sub_x[-1, :], (3, 17)) class TestSeqFirstPool2D(TestSeqAvgPool2D): - def compute(self, x, lod, out): + def compute(self, x, offset, out): self.attrs = {'pooltype': "FIRST"} - for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) + for i in range(len(offset[0]) - 1): + sub_x = np.reshape(x[offset[0][i]:offset[0][i + 1], :], + (-1, 3 * 17)) out[i] = np.reshape(sub_x[0, :], (3, 17)) diff --git a/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py b/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py index ebab77e804..8f0765277a 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py @@ -18,15 +18,17 @@ from op_test import OpTest def sequence_erase(in_seq, lod0, tokens): - new_lod0 = [0] + new_lod0 = [] out_seq = [] - for i in range(0, len(lod0) - 1): + offset = 0 + for i in range(0, len(lod0)): num_out = 0 - for dat in in_seq[lod0[i]:lod0[i + 1]]: + for dat in in_seq[offset:(offset + lod0[i])]: if dat not in tokens: out_seq.append(dat) num_out += 1 - new_lod0.append(new_lod0[-1] + num_out) + offset += lod0[i] + new_lod0.append(num_out) return np.array(out_seq).astype("int32"), new_lod0 @@ -34,7 +36,7 @@ class TestSequenceEraseOpInt32(OpTest): def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") - lod = [[0, 9, 13, 24, 30]] + lod = [[9, 4, 11, 6]] tokens = [2, 3, 5] out_seq, new_lod0 = sequence_erase(in_seq, lod[0], tokens) self.attrs = {'tokens': tokens} @@ -49,7 +51,7 @@ class TestSequenceEraseOpInt64(OpTest): def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") - lod = [[0, 9, 13, 24, 30]] + lod = [[9, 4, 11, 6]] tokens = [2, 3, 5] out_seq, new_lod0 = sequence_erase(in_seq, lod[0], tokens) self.attrs = {'tokens': tokens} @@ -64,7 +66,7 @@ class TestSequenceEraseOpEmpty(OpTest): def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") - lod = [[0, 9, 13, 24, 30]] + lod = [[9, 4, 11, 6]] tokens = [] out_seq, new_lod0 = sequence_erase(in_seq, lod[0], tokens) self.attrs = {'tokens': tokens} diff --git a/python/paddle/fluid/tests/unittests/test_sequence_expand.py b/python/paddle/fluid/tests/unittests/test_sequence_expand.py index 4c8ec1426c..0bbd31814e 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_expand.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_expand.py @@ -21,7 +21,7 @@ class TestSequenceExpand(OpTest): def set_data(self): x_data = np.random.uniform(0.1, 1, [3, 1]).astype('float32') y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float32') - y_lod = [[0, 1, 4, 8]] + y_lod = [[1, 3, 4]] self.inputs = {'X': x_data, 'Y': (y_data, y_lod)} def compute(self): @@ -37,23 +37,27 @@ class TestSequenceExpand(OpTest): out = np.zeros(shape=((0, ) + x_data.shape[1:]), dtype=x_data.dtype) if x_lod is None: - x_idx = [i for i in xrange(x_data.shape[0] + 1)] + # x_idx = [i for i in xrange(x_data.shape[0] + 1)] + x_idx = [1] * x_data.shape[0] else: x_idx = x_lod[0] - out_lod = [[0]] + out_lod = [[]] + + offset = 0 + for i in xrange(len(y_lod[ref_level])): + repeat_num = y_lod[ref_level][i] + x_len = x_idx[i] - for i in xrange(1, len(y_lod[ref_level])): - repeat_num = y_lod[ref_level][i] - y_lod[ref_level][i - 1] - x_len = x_idx[i] - x_idx[i - 1] if repeat_num > 0: - x_sub = x_data[x_idx[i - 1]:x_idx[i], :] + x_sub = x_data[offset:(offset + x_len), :] stacked_x_sub = x_sub for r in range(repeat_num - 1): stacked_x_sub = np.vstack((stacked_x_sub, x_sub)) out = np.vstack((out, stacked_x_sub)) if x_lod is not None: for j in xrange(repeat_num): - out_lod[0].append(out_lod[0][-1] + x_len) + out_lod[0].append(x_len) + offset += x_len if x_lod is None: self.outputs = {'Out': out} @@ -75,9 +79,9 @@ class TestSequenceExpand(OpTest): class TestSequenceExpandCase1(TestSequenceExpand): def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 1]).astype('float32') - x_lod = [[0, 2, 5]] + x_lod = [[2, 3]] y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float32') - y_lod = [[0, 2, 5], [0, 2, 4, 7, 10, 13]] + y_lod = [[2, 3], [2, 2, 3, 3, 3]] self.inputs = {'X': x_data, 'Y': (y_data, y_lod)} self.attrs = {'ref_level': 0} @@ -85,9 +89,9 @@ class TestSequenceExpandCase1(TestSequenceExpand): class TestSequenceExpandCase2(TestSequenceExpand): def set_data(self): x_data = np.random.uniform(0.1, 1, [1, 2, 2]).astype('float32') - x_lod = [[0, 1]] + x_lod = [[1]] y_data = np.random.uniform(0.1, 1, [2, 2, 2]).astype('float32') - y_lod = [[0, 2], [0, 2]] + y_lod = [[2], [1, 1]] self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)} self.attrs = {'ref_level': 0} @@ -95,9 +99,9 @@ class TestSequenceExpandCase2(TestSequenceExpand): class TestSequenceExpandCase3(TestSequenceExpand): def set_data(self): x_data = np.random.uniform(0.1, 1, [4, 1]).astype('float32') - x_lod = [[0, 1, 2, 3, 4]] - y_data = np.random.uniform(0.1, 1, [6, 1]).astype('float32') - y_lod = [[0, 2, 4, 4, 6]] + x_lod = [[1, 1, 1, 1]] + y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float32') + y_lod = [[2, 2, 2, 2]] self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)} @@ -105,9 +109,9 @@ class TestSequenceExpandCase4(TestSequenceExpand): def set_data(self): data = np.random.uniform(0.1, 1, [5 * 2, 1]) x_data = np.array(data).reshape([5, 2]).astype('float32') - x_lod = [[0, 2, 5]] - y_data = np.random.uniform(0.1, 1, [3, 1]).astype('float32') - y_lod = [[0, 1, 3], [0, 1, 3]] + x_lod = [[2, 3]] + y_data = np.random.uniform(0.1, 1, [5, 1]).astype('float32') + y_lod = [[2], [2, 3]] self.inputs = {'X': (x_data, x_lod), 'Y': (y_data, y_lod)} diff --git a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py index efeab56039..68f2e5eba3 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py @@ -22,7 +22,7 @@ class TestSequenceReshape(OpTest): def setUp(self): self.op_type = 'sequence_reshape' dimension = 12 - x_lod = [[0, 4, 5, 8, 11]] + x_lod = [[4, 1, 3, 3]] x = np.random.uniform(0.1, 1, [11, 24]).astype('float32') self.inputs = {'X': (x, x_lod)} @@ -34,13 +34,13 @@ class TestSequenceReshape(OpTest): def compute_output(self, x, x_lod, dimension): x_width = x.shape[1] - out_lod = [[0]] - for i in xrange(len(x_lod[0]) - 1): - seq_len = x_lod[0][i + 1] - x_lod[0][i] + out_lod = [[]] + for i in xrange(len(x_lod[0])): + seq_len = x_lod[0][i] offset = (seq_len * x_width) / dimension assert int(offset) * dimension == seq_len * x_width - out_lod[0].append(out_lod[0][-1] + int(offset)) - out = np.zeros(shape=(out_lod[0][-1], dimension)).astype('float32') + out_lod[0].append(int(offset)) + out = np.zeros(shape=(sum(out_lod[0]), dimension)).astype('float32') out.ravel()[:] = x.ravel()[:] return out, out_lod @@ -55,7 +55,7 @@ class TestSequenceReshape_reduce(TestSequenceReshape): def setUp(self): self.op_type = 'sequence_reshape' dimension = 24 - x_lod = [[0, 4, 6, 8, 12]] + x_lod = [[4, 2, 2, 4]] x = np.random.uniform(0.1, 1, [12, 12]).astype('float32') self.inputs = {'X': (x, x_lod)} @@ -70,7 +70,7 @@ class TestSequenceReshape_same(TestSequenceReshape): def setUp(self): self.op_type = 'sequence_reshape' dimension = 12 - x_lod = [[0, 4, 6, 8, 12]] + x_lod = [[4, 2, 2, 4]] x = np.random.uniform(0.1, 1, [12, 12]).astype('float32') self.inputs = {'X': (x, x_lod)} diff --git a/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py b/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py index 660b4a171d..313e485d1e 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py @@ -29,20 +29,20 @@ class TestSequenceSliceOp(OpTest): self.inputs = {'X': (x, lod), 'Offset': offset, 'Length': length} outs = [] #np.zeros((100, 3, 2)).astype('float32') - out_lod = [[0]] - out_lod_offset = 0 + out_lod = [[]] + lod_offset = 0 for i in range(len(offset)): - sub_x = x[lod[0][i] + offset[i, 0]:lod[0][i] + offset[i, 0] + + sub_x = x[lod_offset + offset[i, 0]:lod_offset + offset[i, 0] + length[i, 0], :] - out_lod_offset = out_lod_offset + len(sub_x) outs.append(sub_x) - out_lod[0].append(out_lod_offset) + out_lod[0].append(len(sub_x)) + lod_offset += lod[0][i] outs = np.concatenate(outs, axis=0) self.outputs = {'Out': (outs, out_lod)} def init_test_case(self): self.x_dim = (100, 3, 2) - self.x_lod = [[0, 20, 40, 60, 80, 100]] + self.x_lod = [[20, 20, 20, 20, 20]] self.offset = [[1], [2], [3], [4], [5]] self.length = [[10], [8], [6], [4], [2]] diff --git a/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py b/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py index d6dc99bb31..e91a69a0f8 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py @@ -26,15 +26,16 @@ class TestSequenceSoftmaxOp(OpTest): self.init_op_type() x = np.random.uniform(0.1, 1, (11, 1)).astype("float32") - lod = [[0, 4, 5, 8, 11]] + lod = [[4, 1, 3, 3]] out = np.zeros((11, 1)).astype("float32") - for i in range(4): - sub_x = x[lod[0][i]:lod[0][i + 1], :] - sub_x = sub_x.reshape(1, lod[0][i + 1] - lod[0][i]) + offset = 0 + for i in range(len(lod[0])): + sub_x = x[offset:offset + lod[0][i], :] + sub_x = sub_x.reshape(1, lod[0][i]) sub_out = stable_softmax(sub_x) - out[lod[0][i]:lod[0][i + 1], :] = sub_out.reshape( - lod[0][i + 1] - lod[0][i], 1) + out[offset:offset + lod[0][i], :] = sub_out.reshape(lod[0][i], 1) + offset += lod[0][i] self.inputs = {"X": (x, lod)} self.outputs = {"Out": out} diff --git a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py index 1d93230e7b..b779f0fb01 100644 --- a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py +++ b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py @@ -54,12 +54,12 @@ class TestShrinkRNNMemoryReferLoD(TestShrinkRNNMemoryBase): def test_refer_lod(self): cpu = core.CPUPlace() x_tensor = core.LoDTensor() - x_tensor.set_lod([[0, 2, 5, 6]]) + x_tensor.set_recursive_sequence_lengths([[2, 3, 1]]) tensor_np = np.random.random(size=(6, 100)).astype('float32') x_tensor.set(tensor_np, cpu) rank_table_tensor = core.LoDTensor() - rank_table_tensor.set_lod([[0, 1, 3, 6]]) + rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set(np.random.random(size=(6, 1)).astype('float32'), cpu) @@ -83,7 +83,7 @@ class TestShrinkRNNMemoryNoLoD(TestShrinkRNNMemoryBase): x_tensor.set(tensor_np, cpu) rank_table_tensor = core.LoDTensor() - rank_table_tensor.set_lod([[0, 1, 3, 6]]) + rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set(np.random.random(size=(6, 1)).astype('float32'), cpu) diff --git a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py index 02cc7da849..0916ed7c9f 100644 --- a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py @@ -56,7 +56,7 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): def test_split_and_merge_lod_tensor_level_0(self): tensor = core.LoDTensor() tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place()) - tensor.set_lod([[0, 3, 9, 10]]) + tensor.set_recursive_sequence_lengths([[3, 6, 1]]) mask_np = np.array([0, 1, 0]).astype('bool') mask_np = np.expand_dims(mask_np, axis=1) @@ -68,15 +68,15 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1) expect_true = core.LoDTensor() expect_true.set(expect_true_tensor, self.place()) - expect_true.set_lod([[0, 6]]) + expect_true.set_recursive_sequence_lengths([[6]]) expect_false_tensor = np.array([0, 1, 2, 9]).astype('int32') expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1) - expect_false_lod = [[0, 3, 4]] + expect_false_lod = [[3, 1]] expect_false = core.LoDTensor() expect_false.set(expect_false_tensor, self.place()) - expect_false.set_lod(expect_false_lod) + expect_false.set_recursive_sequence_lengths(expect_false_lod) self.main( tensor=tensor, @@ -126,7 +126,8 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): def check_tensor_same(self, actual, expect): self.assertTrue(np.allclose(np.array(actual), np.array(expect))) - self.assertEqual(actual.lod(), expect.lod()) + self.assertEqual(actual.recursive_sequence_lengths(), + expect.recursive_sequence_lengths()) class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): @@ -151,7 +152,7 @@ class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): tensor = core.LoDTensor() tensor.set(np.arange(10).reshape(10, 1).astype('float32'), place) - tensor.set_lod([[0, 3, 9, 10]]) + tensor.set_recursive_sequence_lengths([[3, 6, 1]]) mask_np = np.array([0, 1, 0]).astype('bool') mask_np = np.expand_dims(mask_np, axis=1) diff --git a/python/paddle/fluid/tests/unittests/test_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_target_assign_op.py index ccb41e56c5..bd20889752 100644 --- a/python/paddle/fluid/tests/unittests/test_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_target_assign_op.py @@ -22,22 +22,23 @@ def gen_match_and_neg_indices(num_prior, gt_lod, neg_lod): if len(gt_lod) != len(neg_lod): raise AssertionError("The input arguments are illegal.") - batch_size = len(gt_lod) - 1 + batch_size = len(gt_lod) match_indices = -1 * np.ones((batch_size, num_prior)).astype('int32') - neg_indices = np.zeros((neg_lod[-1], 1)).astype('int32') + neg_indices = np.zeros((sum(neg_lod), 1)).astype('int32') + offset = 0 for n in range(batch_size): - gt_num = gt_lod[n + 1] - gt_lod[n] + gt_num = gt_lod[n] ids = random.sample([i for i in range(num_prior)], gt_num) match_indices[n, ids] = [i for i in range(gt_num)] ret_ids = set([i for i in range(num_prior)]) - set(ids) - s = neg_lod[n] - e = neg_lod[n + 1] - l = e - s + l = neg_lod[n] neg_ids = random.sample(ret_ids, l) - neg_indices[s:e, :] = np.array(neg_ids).astype('int32').reshape(l, 1) + neg_indices[offset:offset + neg_lod[n], :] = np.array(neg_ids).astype( + 'int32').reshape(l, 1) + offset += neg_lod[n] return match_indices, neg_indices @@ -56,24 +57,28 @@ def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, # init weight for target label trg_label_wt = np.zeros((batch_size, num_prior, 1)).astype('float32') + gt_offset = 0 + neg_offset = 0 for i in range(batch_size): cur_indices = match_indices[i] col_ids = np.where(cur_indices > -1) col_val = cur_indices[col_ids] - gt_start = gt_lod[i] # target bbox - for v, c in zip(col_val + gt_start, col_ids[0].tolist()): + for v, c in zip(col_val + gt_offset, col_ids[0].tolist()): trg_box[i][c][:] = encoded_box[v][c][:] # weight for target bbox trg_box_wt[i][col_ids] = 1.0 - trg_label[i][col_ids] = gt_label[col_val + gt_start] + trg_label[i][col_ids] = gt_label[col_val + gt_offset] trg_label_wt[i][col_ids] = 1.0 # set target label weight to 1.0 for the negative samples if neg_indices is not None: - neg_ids = neg_indices[neg_lod[i]:neg_lod[i + 1]] + neg_ids = neg_indices[neg_offset:neg_offset + neg_lod[i]] trg_label_wt[i][neg_ids] = 1.0 + # update offset + gt_offset += gt_lod[i] + neg_offset += neg_lod[i] return trg_box, trg_box_wt, trg_label, trg_label_wt @@ -83,11 +88,11 @@ class TestTargetAssginFloatType(OpTest): self.op_type = "target_assign" num_prior = 120 num_class = 21 - gt_lod = [0, 5, 11, 23] - neg_lod = [0, 4, 7, 13] + gt_lod = [5, 6, 12] + neg_lod = [4, 3, 6] mismatch_value = 0 - batch_size = len(gt_lod) - 1 - num_gt = gt_lod[-1] + batch_size = len(gt_lod) + num_gt = sum(gt_lod) encoded_box = np.random.random((num_gt, num_prior, 4)).astype('float32') gt_label = np.random.randint( @@ -121,11 +126,11 @@ class TestTargetAssginIntType(OpTest): self.op_type = "target_assign" num_prior = 120 num_class = 21 - gt_lod = [0, 5, 11, 23] - neg_lod = [0, 4, 7, 13] + gt_lod = [5, 6, 12] + neg_lod = [4, 3, 6] mismatch_value = 0 - batch_size = len(gt_lod) - 1 - num_gt = gt_lod[-1] + batch_size = len(gt_lod) + num_gt = sum(gt_lod) encoded_box = np.random.random((num_gt, num_prior, 4)).astype('float32') gt_label = np.random.randint( diff --git a/python/paddle/fluid/tests/unittests/test_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor.py index 379081c328..f17edd3025 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor.py @@ -69,15 +69,14 @@ class TestTensor(unittest.TestCase): array[0, 0, 0] = 3 array[3, 3, 5] = 10 lod_tensor.set(array, place) - lod_tensor.set_lod([[0, 2, 4]]) + lod_tensor.set_recursive_sequence_lengths([[2, 2]]) lod_v = numpy.array(lod_tensor) self.assertTrue(numpy.alltrue(array == lod_v)) - lod = lod_tensor.lod() - self.assertEqual(0, lod[0][0]) + lod = lod_tensor.recursive_sequence_lengths() + self.assertEqual(2, lod[0][0]) self.assertEqual(2, lod[0][1]) - self.assertEqual(4, lod[0][2]) def test_float_lod_tensor(self): place = core.CPUPlace() @@ -97,21 +96,21 @@ class TestTensor(unittest.TestCase): lod_v = numpy.array(lod_tensor) self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0]) self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1]) - self.assertEqual(len(lod_tensor.lod()), 0) + self.assertEqual(len(lod_tensor.recursive_sequence_lengths()), 0) - lod_py = [[0, 2, 5], [0, 2, 4, 5]] - lod_tensor.set_lod(lod_py) - lod = lod_tensor.lod() + lod_py = [[2, 1], [1, 2, 2]] + lod_tensor.set_recursive_sequence_lengths(lod_py) + lod = lod_tensor.recursive_sequence_lengths() self.assertListEqual(lod_py, lod) def test_lod_tensor_init(self): scope = core.Scope() place = core.CPUPlace() - lod_py = [[0, 2, 5], [0, 2, 4, 5]] + lod_py = [[2, 1], [1, 2, 2]] lod_tensor = core.LoDTensor() lod_tensor.set_dims([5, 2, 3, 4]) - lod_tensor.set_lod(lod_py) + lod_tensor.set_recursive_sequence_lengths(lod_py) lod_tensor.alloc_float(place) tensor_array = numpy.array(lod_tensor) tensor_array[0, 0, 0, 0] = 1.0 @@ -121,17 +120,17 @@ class TestTensor(unittest.TestCase): lod_v = numpy.array(lod_tensor) self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0]) self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1]) - self.assertListEqual(lod_py, lod_tensor.lod()) + self.assertListEqual(lod_py, lod_tensor.recursive_sequence_lengths()) def test_lod_tensor_gpu_init(self): if not core.is_compiled_with_cuda(): return place = core.CUDAPlace(0) - lod_py = [[0, 2, 5], [0, 2, 4, 5]] + lod_py = [[2, 1], [1, 2, 2]] lod_tensor = core.LoDTensor() lod_tensor.set_dims([5, 2, 3, 4]) - lod_tensor.set_lod(lod_py) + lod_tensor.set_recursive_sequence_lengths(lod_py) lod_tensor.alloc_float(place) tensor_array = numpy.array(lod_tensor) tensor_array[0, 0, 0, 0] = 1.0 @@ -141,7 +140,7 @@ class TestTensor(unittest.TestCase): lod_v = numpy.array(lod_tensor) self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0]) self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1]) - self.assertListEqual(lod_py, lod_tensor.lod()) + self.assertListEqual(lod_py, lod_tensor.recursive_sequence_lengths()) def test_empty_tensor(self): place = core.CPUPlace() diff --git a/python/paddle/fluid/tests/unittests/test_warpctc_op.py b/python/paddle/fluid/tests/unittests/test_warpctc_op.py index ac638f7836..9f1aaee472 100644 --- a/python/paddle/fluid/tests/unittests/test_warpctc_op.py +++ b/python/paddle/fluid/tests/unittests/test_warpctc_op.py @@ -34,8 +34,8 @@ class CTCForward(object): self.level = 0 self.num_classes = softmax.shape[1] - self.batch_size = len(softmax_lod[self.level]) - 1 - assert self.batch_size == len(labels_lod[self.level]) - 1 + self.batch_size = len(softmax_lod[self.level]) + assert self.batch_size == len(labels_lod[self.level]) self.loss = np.zeros([self.batch_size, 1], dtype="float32") self.gradient = np.zeros(self.softmax.shape, dtype="float32") @@ -156,16 +156,20 @@ class CTCForward(object): return -log_prob def forward(self): + softmax_offset = 0 + labels_offset = 0 for i in range(self.batch_size): - softmax_start_i = self.softmax_lod[self.level][i] - softmax_end_i = self.softmax_lod[self.level][i + 1] - labels_start_i = self.labels_lod[self.level][i] - labels_end_i = self.labels_lod[self.level][i + 1] + softmax_start_i = softmax_offset + softmax_end_i = softmax_offset + self.softmax_lod[self.level][i] + labels_start_i = labels_offset + labels_end_i = labels_offset + self.labels_lod[self.level][i] softmax_a_sequence = self.softmax[softmax_start_i:softmax_end_i, :] labels_a_sequence = self.labels[labels_start_i:labels_end_i, :] self.loss[i] = self.forward_a_sequence(softmax_a_sequence, labels_a_sequence) + softmax_offset += self.softmax_lod[self.level][i] + labels_offset += self.labels_lod[self.level][i] return self.loss @@ -173,8 +177,8 @@ class TestWarpCTCOp(OpTest): def config(self): self.batch_size = 4 self.num_classes = 8 - self.logits_lod = [[0, 4, 5, 8, 11]] - self.labels_lod = [[0, 3, 4, 8, 12]] + self.logits_lod = [[4, 1, 3, 3]] + self.labels_lod = [[3, 1, 4, 4]] self.blank = self.num_classes - 1 self.norm_by_times = False @@ -184,11 +188,13 @@ class TestWarpCTCOp(OpTest): logits = np.random.uniform( 0.1, 1.0, - [self.logits_lod[0][-1], self.num_classes]).astype("float32") + [sum(self.logits_lod[0]), self.num_classes]).astype("float32") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank labels = np.random.randint( - 0, self.num_classes - 1, [self.labels_lod[0][-1], 1], dtype="int32") + 0, + self.num_classes - 1, [sum(self.labels_lod[0]), 1], + dtype="int32") ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, self.blank, self.norm_by_times) @@ -196,9 +202,8 @@ class TestWarpCTCOp(OpTest): max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max( - max_sequence_length, - self.logits_lod[0][i + 1] - self.logits_lod[0][i]) + max_sequence_length = max(max_sequence_length, + self.logits_lod[0][i]) self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], dtype="float32") @@ -222,8 +227,8 @@ class TestWarpCTCOpCase1(TestWarpCTCOp): def config(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 - self.logits_lod = [[0, 4, 5, 8, 11]] - self.labels_lod = [[0, 3, 4, 8, 12]] + self.logits_lod = [[4, 1, 3, 3]] + self.labels_lod = [[3, 1, 4, 4]] self.blank = 0 self.norm_by_times = False diff --git a/python/paddle/fluid/tests/unittests/test_weight_normalization.py b/python/paddle/fluid/tests/unittests/test_weight_normalization.py index 2adf917bc5..436f9b9f86 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_normalization.py +++ b/python/paddle/fluid/tests/unittests/test_weight_normalization.py @@ -76,11 +76,11 @@ class TestWeightNormalization(unittest.TestCase): lod_level_i = numpy.random.randint( low=1, high=5, - size=self.batch_size if i == 0 else lod_level_i[-1]) - lod_level_i = [0] + numpy.cumsum(lod_level_i).tolist() + size=self.batch_size + if i == 0 else sum(lod_level_i)).tolist() data_lod.append(lod_level_i) data_value = numpy.random.random( - size=[data_lod[-1][-1] if data_lod else self.batch_size + size=[sum(data_lod[-1]) if data_lod else self.batch_size ] + data_shape).astype('float32') self.data[data_name] = (data_value, data_lod) @@ -90,7 +90,7 @@ class TestWeightNormalization(unittest.TestCase): tensor = fluid.Tensor() tensor.set(self.data[desc[0]][0], place) if self.data[desc[0]][1]: - tensor.set_lod(self.data[desc[0]][1]) + tensor.set_recursive_sequence_lengths(self.data[desc[0]][1]) self.inputs[desc[0]] = tensor def weight_normalize(self): diff --git a/python/paddle/fluid/tests/unittests/testsuite.py b/python/paddle/fluid/tests/unittests/testsuite.py index 1dc94a80c9..a995ee10f2 100644 --- a/python/paddle/fluid/tests/unittests/testsuite.py +++ b/python/paddle/fluid/tests/unittests/testsuite.py @@ -22,7 +22,7 @@ def as_lodtensor(np_array, lod, place): tensor = core.LoDTensor() tensor.set(np_value, place) if lod is not None: - tensor.set_lod(lod) + tensor.set_recursive_sequence_lengths(lod) return tensor @@ -73,7 +73,7 @@ def set_input(scope, op, inputs, place): if isinstance(var, tuple) or isinstance(var, np.ndarray): tensor = scope.find_var(var_name).get_tensor() if isinstance(var, tuple): - tensor.set_lod(var[1]) + tensor.set_recursive_sequence_lengths(var[1]) var = var[0] tensor.set_dims(var.shape) tensor.set(var, place) diff --git a/tools/codestyle/cpplint_pre_commit.hook b/tools/codestyle/cpplint_pre_commit.hook index b194af76dc..a9775e10ef 100755 --- a/tools/codestyle/cpplint_pre_commit.hook +++ b/tools/codestyle/cpplint_pre_commit.hook @@ -7,7 +7,7 @@ for file in $(git diff --cached --name-status | awk '$1 != "D" {print $2}'); do if [[ $file =~ ^(paddle/api/.*|paddle/capi/.*|paddle/contrib/.*|paddle/cuda/.*|paddle/function/.*|paddle/gserver/.*|paddle/math/.*|paddle/optimizer/.*|paddle/parameter/.*|paddle/pserver/.*|paddle/trainer/.*|paddle/utils/.*) ]]; then continue; else - cpplint $file; + cpplint --filter=-readability/fn_size $file; TOTAL_ERRORS=$(expr $TOTAL_ERRORS + $?); fi done From 3a25f21eae9d175d34e3a871f1155049e615c058 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Fri, 15 Jun 2018 10:48:55 -0700 Subject: [PATCH 141/154] Modify lod tensor doc based on new LoDTensor Python API (#11253) * Modify lod_tensor.md and nn.py * Modify control_flow.py doc * undo change in lod_tensor.md --- python/paddle/fluid/layers/control_flow.py | 4 +- python/paddle/fluid/layers/nn.py | 52 +++++++++++----------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index d55a1a6f6a..8fec2f9c12 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -746,8 +746,8 @@ def lod_rank_table(x, level=0): .. code-block:: text x is a LoDTensor: - x.lod = [[0, 2, 3], - [0, 5, 6, 7]] + x.lod = [[2, 1], + [5, 1, 1]] x.data = [a, b, c, d, e, f, g] 1. set level to 0: diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 260d4afc9f..7377f7dd7d 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1621,13 +1621,13 @@ def sequence_pool(input, pool_type): .. code-block:: text x is a 1-level LoDTensor: - x.lod = [[0, 2, 5, 7]] + x.lod = [[2, 3, 2]] x.data = [1, 3, 2, 4, 6, 5, 1] x.dims = [7, 1] then output is a Tensor: out.dim = [3, 1] - with condition len(x.lod[-1]) - 1 == out.dims[0] + with condition len(x.lod[-1]) == out.dims[0] for different pool_type: average: out.data = [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 @@ -1686,13 +1686,13 @@ def sequence_first_step(input): .. code-block:: text x is a 1-level LoDTensor: - x.lod = [[0, 2, 5, 7]] + x.lod = [[2, 3, 2]] x.data = [1, 3, 2, 4, 6, 5, 1] x.dims = [7, 1] then output is a Tensor: out.dim = [3, 1] - with condition len(x.lod[-1]) - 1 == out.dims[0] + with condition len(x.lod[-1]) == out.dims[0] out.data = [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) Args: @@ -1719,13 +1719,13 @@ def sequence_last_step(input): .. code-block:: text x is a 1-level LoDTensor: - x.lod = [[0, 2, 5, 7]] + x.lod = [[2, 3, 2]] x.data = [1, 3, 2, 4, 6, 5, 1] x.dims = [7, 1] then output is a Tensor: out.dim = [3, 1] - with condition len(x.lod[-1]) - 1 == out.dims[0] + with condition len(x.lod[-1]) == out.dims[0] out.data = [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) Args: @@ -2468,18 +2468,18 @@ def sequence_expand(x, y, ref_level=-1, name=None): * Case 1 x is a LoDTensor: - x.lod = [[0, 2, 4]] + x.lod = [[2, 2]] x.data = [[a], [b], [c], [d]] x.dims = [4, 1] y is a LoDTensor: - y.lod = [[0, 2, 4], - [0, 3, 6, 7, 8]] + y.lod = [[2, 2], + [3, 3, 1, 1]] ref_level: 0 then output is a 1-level LoDTensor: - out.lod = [[0, 2, 4, 6, 8]] + out.lod = [[2, 2, 2, 2]] out.data = [[a], [b], [a], [b], [c], [d], [c], [d]] out.dims = [8, 1] @@ -2489,7 +2489,7 @@ def sequence_expand(x, y, ref_level=-1, name=None): x.dims = [3, 1] y is a LoDTensor: - y.lod = [[0, 2, 2, 5]] + y.lod = [[2, 0, 3]] ref_level: -1 @@ -3343,7 +3343,7 @@ def ctc_greedy_decoder(input, blank, name=None): [0.2, 0.2, 0.1, 0.5], [0.5, 0.1, 0.3, 0.1]] - input.lod = [[0, 4, 8]] + input.lod = [[4, 4]] Then: @@ -3351,7 +3351,7 @@ def ctc_greedy_decoder(input, blank, name=None): [1], [3]] - output.lod = [[0, 2, 3]] + output.lod = [[2, 1]] Args: @@ -3368,7 +3368,7 @@ def ctc_greedy_decoder(input, blank, name=None): Returns: Variable: CTC greedy decode result. If all the sequences in result were - empty, the result LoDTensor will be [-1] with LoD [[0]] and dims [1, 1]. + empty, the result LoDTensor will be [-1] with LoD [[]] and dims [1, 1]. Examples: .. code-block:: python @@ -3458,7 +3458,7 @@ def sequence_reshape(input, new_dim): .. code-block:: text x is a LoDTensor: - x.lod = [[0, 2, 6]] + x.lod = [[2, 4]] x.data = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]] x.dims = [6, 2] @@ -3466,7 +3466,7 @@ def sequence_reshape(input, new_dim): set new_dim = 4 then out is a LoDTensor: - out.lod = [[0, 1, 3]] + out.lod = [[1, 2]] out.data = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]] out.dims = [3, 4] @@ -3737,7 +3737,7 @@ def im2sequence(input, filter_size=1, stride=1, padding=0, name=None): output.dims = {8, 9} - output.lod = [[0, 4, 8]] + output.lod = [[4, 4]] The simple usage is: @@ -4133,47 +4133,47 @@ def lod_reset(x, y=None, target_lod=None): * Example 1: Given a 1-level LoDTensor x: - x.lod = [[ 0, 2, 5 6 ]] + x.lod = [[ 2, 3, 1 ]] x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]] x.dims = [6, 1] - target_lod: [0, 4, 6] + target_lod: [4, 2] then we get a 1-level LoDTensor: - out.lod = [[ 0, 4, 6 ]] + out.lod = [[4, 2]] out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]] out.dims = [6, 1] * Example 2: Given a 1-level LoDTensor x: - x.lod = [[ 0, 2, 5 6 ]] + x.lod = [[2, 3, 1]] x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]] x.dims = [6, 1] y is a Tensor: - y.data = [[0, 2, 6]] + y.data = [[2, 4]] y.dims = [1, 3] then we get a 1-level LoDTensor: - out.lod = [[ 0, 2, 6 ]] + out.lod = [[2, 4]] out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]] out.dims = [6, 1] * Example 3: Given a 1-level LoDTensor x: - x.lod = [[ 0, 2, 5 6 ]] + x.lod = [[2, 3, 1]] x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]] x.dims = [6, 1] y is a 2-level LoDTensor: - y.lod = [[0, 2, 4], [0, 2, 5, 6]] + y.lod = [[2, 2], [2, 2, 1, 1]] y.data = [[1.1], [2.1], [3.1], [4.1], [5.1], [6.1]] y.dims = [6, 1] then we get a 2-level LoDTensor: - out.lod = [[0, 2, 4], [0, 2, 5, 6]] + out.lod = [[2, 2], [2, 2, 1, 1]] out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]] out.dims = [6, 1] From a4ee0d0dd165cdc79beca3e0904a7adf5bf58d9c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sat, 16 Jun 2018 08:58:48 +0800 Subject: [PATCH 142/154] add reverse --- python/paddle/fluid/layers/tensor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 4c97ca40d8..18e0fedcc4 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -35,6 +35,7 @@ __all__ = [ 'argmax', 'ones', 'zeros', + 'reverse', ] From 9c128fe656e16c0be9167b97a9118dfe65649c96 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sat, 16 Jun 2018 16:22:15 +0800 Subject: [PATCH 143/154] concat support data as input --- paddle/fluid/operators/concat_op.h | 41 +++++++++++++++++---------- paddle/fluid/operators/math/concat.cc | 25 +++++++++------- paddle/fluid/operators/math/concat.h | 3 +- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/operators/concat_op.h b/paddle/fluid/operators/concat_op.h index 1b1b8bf5ed..a496301526 100644 --- a/paddle/fluid/operators/concat_op.h +++ b/paddle/fluid/operators/concat_op.h @@ -60,34 +60,45 @@ template class ConcatGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { - auto* in = ctx.Input(framework::GradVarName("Out")); + auto* out_grad = + ctx.Input(framework::GradVarName("Out")); + auto ins = ctx.MultiInput("X"); + auto out_var_names = ctx.Outputs(framework::GradVarName("X")); auto outs = ctx.MultiOutput(framework::GradVarName("X")); int64_t axis = static_cast(ctx.Attr("axis")); + // get output tensor that the name is not kEmptyVarName + std::vector outputs; + for (size_t j = 0; j < outs.size(); ++j) { + if (out_var_names[j] != framework::kEmptyVarName) { + outs[j]->mutable_data(ctx.GetPlace()); + outputs.push_back(outs[j]); + } else { + outputs.push_back(nullptr); + } + } + // Sometimes direct copies will be faster, this maybe need deeply analysis. if (axis == 0 && outs.size() < 10) { size_t input_offset = 0; - auto in_stride = framework::stride_numel(in->dims()); + const auto in_stride = framework::stride_numel(out_grad->dims()); - for (auto& out : outs) { - out->mutable_data(ctx.GetPlace()); - auto out_stride = framework::stride_numel(out->dims()); - StridedNumelCopyWithAxis(ctx.device_context(), axis, out->data(), - out_stride, in->data() + input_offset, - in_stride, out_stride[axis]); + for (size_t i = 0; i < outs.size(); ++i) { + auto out_stride = framework::stride_numel(ins[i]->dims()); + auto* out = outputs[i]; + if (out != nullptr) { + StridedNumelCopyWithAxis( + ctx.device_context(), axis, out->data(), out_stride, + out_grad->data() + input_offset, in_stride, out_stride[axis]); + } input_offset += out_stride[axis]; } } else { - std::vector outputs(outs.size()); - for (size_t j = 0; j < outs.size(); ++j) { - outs[j]->mutable_data(ctx.GetPlace()); - outputs[j] = *outs[j]; - } - auto& dev_ctx = ctx.template device_context(); paddle::operators::math::ConcatGradFunctor concat_grad_functor; - concat_grad_functor(dev_ctx, *in, static_cast(axis), &outputs); + concat_grad_functor(dev_ctx, *out_grad, ins, static_cast(axis), + &outputs); } } }; diff --git a/paddle/fluid/operators/math/concat.cc b/paddle/fluid/operators/math/concat.cc index cc69212466..c10cff9c9b 100644 --- a/paddle/fluid/operators/math/concat.cc +++ b/paddle/fluid/operators/math/concat.cc @@ -70,35 +70,40 @@ template class ConcatGradFunctor { public: void operator()(const platform::CPUDeviceContext& context, - const framework::Tensor& input, const int axis, - std::vector* outputs) { + const framework::Tensor& input, + const std::vector& ref_inputs, + const int axis, std::vector* outputs) { // TODO(zcd): Add input data validity checking - int num = outputs->size(); + size_t num = outputs->size(); int input_rows = 1; - auto dim_0 = outputs->at(0).dims(); + auto dim_0 = ref_inputs[0]->dims(); for (int i = 0; i < axis; ++i) { input_rows *= dim_0[i]; } + int input_cols = 0; std::vector output_cols(outputs->size()); - for (int i = 0; i < num; ++i) { - int t_cols = outputs->at(i).numel() / input_rows; + for (size_t i = 0; i < num; ++i) { + int t_cols = ref_inputs[i]->numel() / input_rows; input_cols += t_cols; output_cols[i] = t_cols; } auto cpu_place = boost::get(context.GetPlace()); // computation - for (int k = 0; k < input_rows; ++k) { + for (size_t k = 0; k < input_rows; ++k) { const T* src_ptr = input.data() + k * input_cols; int col_idx = 0; for (int j = 0; j < num; ++j) { int col_len = output_cols[j]; - T* dst_ptr = outputs->at(j).data() + k * col_len; - memory::Copy(cpu_place, dst_ptr, cpu_place, src_ptr + col_idx, - sizeof(T) * col_len); + auto* out_tensor = (*outputs)[j]; + if (out_tensor != nullptr) { + T* dst_ptr = out_tensor->data() + k * col_len; + memory::Copy(cpu_place, dst_ptr, cpu_place, src_ptr + col_idx, + sizeof(T) * col_len); + } col_idx += col_len; } } diff --git a/paddle/fluid/operators/math/concat.h b/paddle/fluid/operators/math/concat.h index 041ce8bf8a..9e080f2e8b 100644 --- a/paddle/fluid/operators/math/concat.h +++ b/paddle/fluid/operators/math/concat.h @@ -57,7 +57,8 @@ template class ConcatGradFunctor { public: void operator()(const DeviceContext& context, const framework::Tensor& input, - const int axis, std::vector* outputs); + const std::vector& ref_inputs, + const int axis, std::vector* outputs); }; } // namespace math From ad1ad738d89bb6b347ee0c53ef0245acb86f158d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 17 Jun 2018 10:48:34 +0800 Subject: [PATCH 144/154] add gpu support for concat --- paddle/fluid/operators/math/concat.cc | 2 +- paddle/fluid/operators/math/concat.cu | 41 ++++++++++++++++----------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/paddle/fluid/operators/math/concat.cc b/paddle/fluid/operators/math/concat.cc index c10cff9c9b..14964fc62a 100644 --- a/paddle/fluid/operators/math/concat.cc +++ b/paddle/fluid/operators/math/concat.cc @@ -98,7 +98,7 @@ class ConcatGradFunctor { int col_idx = 0; for (int j = 0; j < num; ++j) { int col_len = output_cols[j]; - auto* out_tensor = (*outputs)[j]; + auto* out_tensor = outputs->at(j); if (out_tensor != nullptr) { T* dst_ptr = out_tensor->data() + k * col_len; memory::Copy(cpu_place, dst_ptr, cpu_place, src_ptr + col_idx, diff --git a/paddle/fluid/operators/math/concat.cu b/paddle/fluid/operators/math/concat.cu index 4285d38dcd..f66baa6573 100644 --- a/paddle/fluid/operators/math/concat.cu +++ b/paddle/fluid/operators/math/concat.cu @@ -102,10 +102,12 @@ __global__ void KernelConcatGrad(const T* input_data, const int in_row, int local_col = tid_x - curr_offset; int segment_width = curr_col_offset - curr_offset; T* output_ptr = outputs_data[curr_segment]; - int tid_y = blockIdx.y * blockDim.y + threadIdx.y; - for (; tid_y < in_row; tid_y += blockDim.y * gridDim.y) - output_ptr[tid_y * segment_width + local_col] = - input_data[tid_y * in_col + tid_x]; + if (output_ptr != nullptr) { + int tid_y = blockIdx.y * blockDim.y + threadIdx.y; + for (; tid_y < in_row; tid_y += blockDim.y * gridDim.y) + output_ptr[tid_y * segment_width + local_col] = + input_data[tid_y * in_col + tid_x]; + } } } @@ -118,10 +120,12 @@ __global__ void KernelConcatGrad(const T* input_data, const int in_row, int split = tid_x / fixed_out_col; int in_offset = tid_x - split * fixed_out_col; T* output_ptr = outputs_data[split]; - int tid_y = blockIdx.y * blockDim.y + threadIdx.y; - for (; tid_y < in_row; tid_y += blockDim.y * gridDim.y) - output_ptr[tid_y * fixed_out_col + in_offset] = - input_data[tid_y * in_col + tid_x]; + if (output_ptr != nullptr) { + int tid_y = blockIdx.y * blockDim.y + threadIdx.y; + for (; tid_y < in_row; tid_y += blockDim.y * gridDim.y) + output_ptr[tid_y * fixed_out_col + in_offset] = + input_data[tid_y * in_col + tid_x]; + } } } @@ -203,17 +207,18 @@ template class ConcatGradFunctor { public: void operator()(const platform::CUDADeviceContext& context, - const framework::Tensor& input, const int axis, - std::vector* outputs) { + const framework::Tensor& input, + const std::vector& ref_inputs, + const int axis, std::vector* outputs) { // TODO(zcd): Add input data validity checking int o_num = outputs->size(); int out_row = 1; - auto dim_0 = outputs->at(0).dims(); + auto dim_0 = ref_inputs[0]->dims(); for (int i = 0; i < axis; ++i) { out_row *= dim_0[i]; } - int out_col = outputs->at(0).numel() / out_row; + int out0_col = ref_inputs[0]->numel() / out_row; int in_col = 0, in_row = out_row; bool sameShape = true; @@ -223,13 +228,17 @@ class ConcatGradFunctor { outputs_cols[0] = 0; for (int i = 0; i < o_num; ++i) { - int t_col = outputs->at(i).numel() / out_row; + int t_col = outputs->at(i)->numel() / out_row; if (sameShape) { - if (t_col != out_col) sameShape = false; + if (t_col != out0_col) sameShape = false; } in_col += t_col; outputs_cols[i + 1] = in_col; - outputs_ptr[i] = outputs->at(i).data(); + if (outputs->at(i) != nullptr) { + outputs_ptr[i] = outputs->at(i)->data(); + } else { + outputs_ptr[i] = nullptr; + } } T** dev_out_gpu_data = @@ -255,7 +264,7 @@ class ConcatGradFunctor { if (sameShape) { KernelConcatGrad<<>>( - input.data(), in_row, in_col, out_col, dev_out_gpu_data); + input.data(), in_row, in_col, out0_col, dev_out_gpu_data); } else { const int* dev_outs_col_data = outputs_cols.CUDAData(context.GetPlace()); KernelConcatGrad<<>>( From 24766a170d8199bdc8b7159ad74d6911ff2fd6cb Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Sun, 17 Jun 2018 11:39:31 +0800 Subject: [PATCH 145/154] Reformat nn.py --- python/paddle/fluid/layers/nn.py | 83 ++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 11af237b82..8afc65d60e 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -25,21 +25,74 @@ import utils import random __all__ = [ - 'fc', 'embedding', 'dynamic_lstm', 'dynamic_lstmp', 'dynamic_gru', - 'gru_unit', 'linear_chain_crf', 'crf_decoding', 'cos_sim', 'cross_entropy', - 'square_error_cost', 'chunk_eval', 'sequence_conv', 'conv2d', 'conv3d', - 'sequence_pool', 'sequence_softmax', 'softmax', 'pool2d', 'pool3d', - 'batch_norm', 'beam_search_decode', 'conv2d_transpose', 'conv3d_transpose', - 'sequence_expand', 'lstm_unit', 'reduce_sum', 'reduce_mean', 'reduce_max', - 'reduce_min', 'reduce_prod', 'sequence_first_step', 'sequence_last_step', - 'dropout', 'split', 'ctc_greedy_decoder', 'edit_distance', 'l2_normalize', - 'matmul', 'topk', 'warpctc', 'sequence_reshape', 'transpose', 'im2sequence', - 'nce', 'beam_search', 'row_conv', 'multiplex', 'layer_norm', - 'softmax_with_cross_entropy', 'smooth_l1', 'one_hot', - 'autoincreased_step_counter', 'reshape', 'lod_reset', 'lrn', 'pad', - 'label_smooth', 'roi_pool', 'dice_loss', 'image_resize', - 'image_resize_short', 'resize_bilinear', 'gather', 'random_crop', - 'mean_iou', 'relu', 'log' + 'fc', + 'embedding', + 'dynamic_lstm', + 'dynamic_lstmp', + 'dynamic_gru', + 'gru_unit', + 'linear_chain_crf', + 'crf_decoding', + 'cos_sim', + 'cross_entropy', + 'square_error_cost', + 'chunk_eval', + 'sequence_conv', + 'conv2d', + 'conv3d', + 'sequence_pool', + 'sequence_softmax', + 'softmax', + 'pool2d', + 'pool3d', + 'batch_norm', + 'beam_search_decode', + 'conv2d_transpose', + 'conv3d_transpose', + 'sequence_expand', + 'lstm_unit', + 'reduce_sum', + 'reduce_mean', + 'reduce_max', + 'reduce_min', + 'reduce_prod', + 'sequence_first_step', + 'sequence_last_step', + 'dropout', + 'split', + 'ctc_greedy_decoder', + 'edit_distance', + 'l2_normalize', + 'matmul', + 'topk', + 'warpctc', + 'sequence_reshape', + 'transpose', + 'im2sequence', + 'nce', + 'beam_search', + 'row_conv', + 'multiplex', + 'layer_norm', + 'softmax_with_cross_entropy', + 'smooth_l1', + 'one_hot', + 'autoincreased_step_counter', + 'reshape', + 'lod_reset', + 'lrn', + 'pad', + 'label_smooth', + 'roi_pool', + 'dice_loss', + 'image_resize', + 'image_resize_short', + 'resize_bilinear', + 'gather', + 'random_crop', + 'mean_iou', + 'relu', + 'log', ] From 82a4cf19608c7655a6b6394c65a10933f3b64dc0 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 17 Jun 2018 11:44:25 +0800 Subject: [PATCH 146/154] update image_resize_short and shape doc --- paddle/fluid/operators/shape_op.cc | 9 ++++++--- python/paddle/fluid/layers/nn.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/operators/shape_op.cc b/paddle/fluid/operators/shape_op.cc index c75fce7959..b44d5f8980 100644 --- a/paddle/fluid/operators/shape_op.cc +++ b/paddle/fluid/operators/shape_op.cc @@ -36,10 +36,13 @@ class ShapeOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { AddInput("Input", "(Tensor), The input tensor."); - AddOutput("Out", "(Tensor), The shape of input tensor."); + AddOutput("Out", + "(Tensor), The shape of input tensor, the data type of the shape" + " is int64_t, will be on the same device with the input Tensor."); AddComment(R"DOC( -Shape Operator. -Get the shape of input tensor. +Shape Operator + +Get the shape of input tensor. Only support CPU input Tensor now. )DOC"); } }; diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index a3b2d2b777..40e72aa488 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4650,7 +4650,7 @@ def image_resize_short(input, out_short_len, resample='BILINEAR'): Returns: Variable: The output is a 4-D tensor of the shape - (num_batches, channls, out_h, out_w). + (num_batches, channls, out_h, out_w). """ in_shape = input.shape if len(in_shape) != 4: From 811f5cca7eda2192b0c758ba1111eafbc035c0ac Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Sun, 17 Jun 2018 11:58:49 +0800 Subject: [PATCH 147/154] Hide StaticRNNMemory --- python/paddle/fluid/layers/control_flow.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index b0534b251b..8ab433e3f0 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -27,7 +27,6 @@ __all__ = [ 'merge_lod_tensor', 'BlockGuard', 'BlockGuardWithCompletion', - 'StaticRNNMemoryLink', 'WhileGuard', 'While', 'Switch', @@ -410,16 +409,17 @@ class StaticRNNMemoryLink(object): """ StaticRNNMemoryLink class. - Args: - init: the initial variable for Memory - init: Variable - pre_mem: the memory variable in previous time step - pre_mem: Variable - mem: the memory variable in current time step - mem: Variable - StaticRNNMemoryLink class is used to create a link between two memory cells of a StaticRNN. + + + NOTE: This is a internal data structure of a very low-level API. + Please use StaticRNN instead. + + Args: + init(Variable): the initial variable for Memory. + pre_mem(Variable): the memory variable in previous time step. + mem(Variable): the memory variable in current time step. """ def __init__(self, init, pre_mem, mem=None): From 962711dc3fc48af12cc317f015fbe00ce801ea64 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Sat, 16 Jun 2018 23:23:29 -0500 Subject: [PATCH 148/154] Add some paddleenforce. (#11516) --- paddle/fluid/memory/detail/system_allocator.cc | 8 +++++--- paddle/fluid/operators/detail/grpc_client.cc | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/memory/detail/system_allocator.cc b/paddle/fluid/memory/detail/system_allocator.cc index d539052916..9b1ab1e228 100644 --- a/paddle/fluid/memory/detail/system_allocator.cc +++ b/paddle/fluid/memory/detail/system_allocator.cc @@ -43,14 +43,16 @@ void* CPUAllocator::Alloc(size_t* index, size_t size) { *index = 0; // unlock memory - void* p; + void* p = nullptr; #ifdef PADDLE_WITH_MKLDNN // refer to https://github.com/01org/mkl-dnn/blob/master/include/mkldnn.hpp // memory alignment - PADDLE_ENFORCE_EQ(posix_memalign(&p, 4096ul, size), 0); + PADDLE_ENFORCE_EQ(posix_memalign(&p, 4096ul, size), 0, "Alloc %ld error!", + size); #else - PADDLE_ENFORCE_EQ(posix_memalign(&p, 32ul, size), 0); + PADDLE_ENFORCE_EQ(posix_memalign(&p, 32ul, size), 0, "Alloc %ld error!", + size); #endif PADDLE_ENFORCE(p, "Fail to allocate CPU memory: size = %d .", size); diff --git a/paddle/fluid/operators/detail/grpc_client.cc b/paddle/fluid/operators/detail/grpc_client.cc index 02ffe3651e..ea004f7cd3 100644 --- a/paddle/fluid/operators/detail/grpc_client.cc +++ b/paddle/fluid/operators/detail/grpc_client.cc @@ -245,7 +245,7 @@ void GRPCClient::Proceed() { if (c->status_.ok()) { c->Process(); } else { - LOG(ERROR) << "var: " << c->var_h_.String() + LOG(FATAL) << "var: " << c->var_h_.String() << " grpc error:" << c->status_.error_message(); } delete c; From 7a56705e4ae7229b96046be113af053eefbfaf27 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Sun, 17 Jun 2018 12:53:54 +0800 Subject: [PATCH 149/154] polish doc --- paddle/fluid/operators/lstm_op.cc | 14 ++++++------- .../fluid/layers/layer_function_generator.py | 21 +++++++------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/operators/lstm_op.cc b/paddle/fluid/operators/lstm_op.cc index 29cec6ae10..3225bf9bb6 100644 --- a/paddle/fluid/operators/lstm_op.cc +++ b/paddle/fluid/operators/lstm_op.cc @@ -184,19 +184,17 @@ Long-Short Term Memory (LSTM) Operator. The defalut implementation is diagonal/peephole connection (https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: -$$ -i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) \\ +$$ i_t = \\sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) $$ -f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) \\ +$$ f_t = \\sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) $$ -\tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) \\ +$$ \\tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) $$ -o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) \\ +$$ o_t = \\sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) $$ -c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\ +$$ c_t = f_t \\odot c_{t-1} + i_t \\odot \\tilde{c_t} $$ -h_t = o_t \odot act_h(c_t) -$$ +$$ h_t = o_t \\odot act_h(c_t) $$ - W terms denote weight matrices (e.g. $W_{xi}$ is the matrix of weights from the input gate to the input), $W_{ic}, W_{fc}, W_{oc}$ diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index 7a95afa9a6..3096389101 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -49,6 +49,13 @@ _single_dollar_pattern_ = re.compile(r"\$([^\$]+)\$") _two_bang_pattern_ = re.compile(r"!!([^!]+)!!") +def escape_math(text): + return _two_bang_pattern_.sub( + r'$$\1$$', + _single_dollar_pattern_.sub(r':math:`\1`', + _two_dollar_pattern_.sub(r"!!\1!!", text))) + + def _generate_doc_string_(op_proto): """ Generate docstring by OpProto @@ -60,12 +67,6 @@ def _generate_doc_string_(op_proto): str: the document string """ - def escape_math(text): - return _two_bang_pattern_.sub( - r'$$\1$$', - _single_dollar_pattern_.sub( - r':math:`\1`', _two_dollar_pattern_.sub(r"!!\1!!", text))) - if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError("OpProto should be `framework_pb2.OpProto`") @@ -233,9 +234,6 @@ def autodoc(comment=""): return __impl__ -_inline_math_single_dollar = re.compile(r"\$([^\$]+)\$") - - def templatedoc(op_type=None): """ Decorator of layer function. It will use the docstring from the layer @@ -253,9 +251,6 @@ def templatedoc(op_type=None): def trim_ending_dot(msg): return msg.rstrip('.') - def escape_inline_math(msg): - return _inline_math_single_dollar.sub(repl=r':math:`\1`', string=msg) - def __impl__(func): if op_type is None: op_type_name = func.__name__ @@ -269,7 +264,7 @@ def templatedoc(op_type=None): for line in comment_lines: line = line.strip() if len(line) != 0: - comment += escape_inline_math(line) + comment += escape_math(line) comment += " " elif len(comment) != 0: comment += "\n \n " From 46ae1c93c28d346d9a4c6a4bf7c9d1019216403b Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 17 Jun 2018 14:00:49 +0800 Subject: [PATCH 150/154] add doc for softmax --- python/paddle/fluid/layers/nn.py | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 6032573393..d31d12f971 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -1258,6 +1258,45 @@ def sequence_softmax(input, param_attr=None, bias_attr=None, use_cudnn=True): def softmax(input, param_attr=None, bias_attr=None, use_cudnn=True, name=None): + """ + The input of the softmax layer is a 2-D tensor with shape N x K (N is the + batch_size, K is the dimension of input feature). The output tensor has the + same shape as the input tensor. + + For each row of the input tensor, the softmax operator squashes the + K-dimensional vector of arbitrary real values to a K-dimensional vector of real + values in the range [0, 1] that add up to 1. + + It computes the exponential of the given dimension and the sum of exponential + values of all the other dimensions in the K-dimensional vector input. + Then the ratio of the exponential of the given dimension and the sum of + exponential values of all the other dimensions is the output of the softmax + operator. + + For each row :math:`i` and each column :math:`j` in Input(X), we have: + + .. math:: + + Out[i, j] = \\frac{\exp(X[i, j])}{\sum_j(exp(X[i, j])} + + Args: + input (Variable): The input variable. + bias_attr (ParamAttr): attributes for bias + param_attr (ParamAttr): attributes for parameter + use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn \ + library is installed. + + Returns: + Variable: output of softmax + + Examples: + + .. code-block:: python + + fc = fluid.layers.fc(input=x, size=10) + softmax = fluid.layers.softmax(input=fc) + + """ helper = LayerHelper('softmax', **locals()) dtype = helper.input_dtype() softmax_out = helper.create_tmp_variable(dtype) From d7345959789a22437c7065078cc3a7d457c7e70e Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Mon, 18 Jun 2018 17:16:06 +0800 Subject: [PATCH 151/154] Feature/pass manager (#11440) --- .../fluid/inference/analysis/CMakeLists.txt | 39 +++--- paddle/fluid/inference/analysis/argument.cc | 15 +++ paddle/fluid/inference/analysis/argument.h | 53 ++++++++ .../inference/analysis/data_flow_graph.cc | 15 +-- .../analysis/data_flow_graph_to_fluid_pass.cc | 77 ++++++++++++ .../analysis/data_flow_graph_to_fluid_pass.h | 59 +++++++++ .../data_flow_graph_to_fluid_pass_tester.cc | 5 +- .../analysis/dfg_graphviz_draw_pass.cc | 54 ++++++++ .../analysis/dfg_graphviz_draw_pass.h | 41 ++++--- .../analysis/dfg_graphviz_draw_pass_tester.cc | 10 +- .../analysis/fluid_to_data_flow_graph_pass.cc | 22 ++-- .../analysis/fluid_to_data_flow_graph_pass.h | 11 +- .../fluid_to_data_flow_graph_pass_tester.cc | 6 +- paddle/fluid/inference/analysis/helper.h | 1 + paddle/fluid/inference/analysis/node.cc | 3 + paddle/fluid/inference/analysis/node.h | 23 ++-- paddle/fluid/inference/analysis/pass.h | 27 ++-- .../fluid/inference/analysis/pass_manager.cc | 44 +++++++ .../fluid/inference/analysis/pass_manager.h | 116 ++++++++++++++++++ .../inference/analysis/pass_manager_tester.cc | 85 +++++++++++++ .../analysis/subgraph_splitter_tester.cc | 45 +++++-- .../analysis/tensorrt_subgraph_pass.cc | 33 +++++ .../analysis/tensorrt_subgraph_pass.h | 47 +++++++ .../analysis/tensorrt_subgraph_pass_tester.cc | 71 +++++++++++ paddle/fluid/inference/analysis/ut_helper.h | 34 +++-- .../operators/tensorrt_engine_op_test.cc | 2 +- 26 files changed, 830 insertions(+), 108 deletions(-) create mode 100644 paddle/fluid/inference/analysis/argument.cc create mode 100644 paddle/fluid/inference/analysis/argument.h create mode 100644 paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc create mode 100644 paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.h create mode 100644 paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc create mode 100644 paddle/fluid/inference/analysis/pass_manager.cc create mode 100644 paddle/fluid/inference/analysis/pass_manager.h create mode 100644 paddle/fluid/inference/analysis/pass_manager_tester.cc create mode 100644 paddle/fluid/inference/analysis/tensorrt_subgraph_pass.cc create mode 100644 paddle/fluid/inference/analysis/tensorrt_subgraph_pass.h create mode 100644 paddle/fluid/inference/analysis/tensorrt_subgraph_pass_tester.cc diff --git a/paddle/fluid/inference/analysis/CMakeLists.txt b/paddle/fluid/inference/analysis/CMakeLists.txt index 5083578444..2bb2c8135d 100644 --- a/paddle/fluid/inference/analysis/CMakeLists.txt +++ b/paddle/fluid/inference/analysis/CMakeLists.txt @@ -1,23 +1,32 @@ set(FLUID_CORE_MODULES proto_desc memory lod_tensor executor init) -cc_library(analysis SRCS dot.cc node.cc data_flow_graph.cc graph_traits.cc subgraph_splitter.cc fluid_to_data_flow_graph_pass.cc - DEPS paddle_fluid) +cc_library(analysis SRCS pass_manager.cc dot.cc node.cc data_flow_graph.cc graph_traits.cc subgraph_splitter.cc + fluid_to_data_flow_graph_pass.cc + data_flow_graph_to_fluid_pass.cc + tensorrt_subgraph_pass.cc + dfg_graphviz_draw_pass.cc + DEPS framework_proto) cc_test(test_node SRCS node_tester.cc DEPS analysis) cc_test(test_dot SRCS dot_tester.cc DEPS analysis) set(PYTHON_TESTS_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/tests) -cc_test(test_data_flow_graph SRCS data_flow_graph_tester.cc DEPS analysis ${FLUID_CORE_MODULES} paddle_fluid - ARGS --inference_model_dir=${PYTHON_TESTS_DIR}/book/word2vec.inference.model) -set_tests_properties(test_data_flow_graph PROPERTIES DEPENDS test_word2vec) +function (inference_analysis_test TARGET) + set(options "") + set(oneValueArgs "") + set(multiValueArgs SRCS) + cmake_parse_arguments(analysis_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) -cc_test(test_subgraph_splitter - SRCS subgraph_splitter_tester.cc - DEPS analysis paddle_fluid tensor - ARGS --inference_model_dir=${PYTHON_TESTS_DIR}/book/word2vec.inference.model) -set_tests_properties(test_subgraph_splitter PROPERTIES DEPENDS test_word2vec) + cc_test(${TARGET} + SRCS "${analysis_test_SRCS}" + DEPS analysis + ARGS --inference_model_dir=${PYTHON_TESTS_DIR}/book/word2vec.inference.model --fraction_of_gpu_memory_to_use=0.5) + set_tests_properties(${TARGET} PROPERTIES DEPENDS test_word2vec) +endfunction(inference_analysis_test) -cc_test(test_dfg_graphviz_draw_pass - SRCS dfg_graphviz_draw_pass_tester.cc - DEPS analysis - ARGS --inference_model_dir=${PYTHON_TESTS_DIR}/book/word2vec.inference.model) -set_tests_properties(test_dfg_graphviz_draw_pass PROPERTIES DEPENDS test_word2vec) +inference_analysis_test(test_data_flow_graph SRCS data_flow_graph_tester.cc) +inference_analysis_test(test_data_flow_graph_to_fluid_pass SRCS data_flow_graph_to_fluid_pass_tester.cc) +inference_analysis_test(test_fluid_to_data_flow_graph_pass SRCS fluid_to_data_flow_graph_pass_tester.cc) +inference_analysis_test(test_subgraph_splitter SRCS subgraph_splitter_tester.cc) +inference_analysis_test(test_dfg_graphviz_draw_pass SRCS dfg_graphviz_draw_pass_tester.cc) +#inference_analysis_test(test_tensorrt_subgraph_pass SRCS tensorrt_subgraph_pass_tester.cc) +inference_analysis_test(test_pass_manager SRCS pass_manager_tester.cc) diff --git a/paddle/fluid/inference/analysis/argument.cc b/paddle/fluid/inference/analysis/argument.cc new file mode 100644 index 0000000000..cb0263d5d9 --- /dev/null +++ b/paddle/fluid/inference/analysis/argument.cc @@ -0,0 +1,15 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/inference/analysis/argument.h" diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h new file mode 100644 index 0000000000..7d7131ed7a --- /dev/null +++ b/paddle/fluid/inference/analysis/argument.h @@ -0,0 +1,53 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * This file defines the class Argument, which is the input and output of the + * analysis module. All the fields that needed either by Passes or PassManagers + * are contained in Argument. + * + * TODO(Superjomn) Find some way better to contain the fields when it grow too + * big. + */ + +#include "paddle/fluid/framework/program_desc.h" +#include "paddle/fluid/inference/analysis/data_flow_graph.h" + +namespace paddle { +namespace inference { +namespace analysis { + +/* + * The argument definition of both Pass and PassManagers. + * + * All the fields should be registered here for clearness. + */ +struct Argument { + // The graph that process by the Passes or PassManagers. + std::unique_ptr main_dfg; + + // The original program desc. + std::unique_ptr origin_program_desc; +}; + +#define UNLIKELY(condition) __builtin_expect(static_cast(condition), 0) +#define ANALYSIS_ARGUMENT_CHECK_FIELD(field__) \ + if (!UNLIKELY(field__)) { \ + LOG(ERROR) << "field " << #field__ << " should be set."; \ + return false; \ + } + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/data_flow_graph.cc b/paddle/fluid/inference/analysis/data_flow_graph.cc index 4220451e3c..c30a7c26ce 100644 --- a/paddle/fluid/inference/analysis/data_flow_graph.cc +++ b/paddle/fluid/inference/analysis/data_flow_graph.cc @@ -14,6 +14,7 @@ limitations under the License. */ #include "paddle/fluid/inference/analysis/data_flow_graph.h" #include "paddle/fluid/inference/analysis/dot.h" +#include "paddle/fluid/inference/analysis/node.h" namespace paddle { namespace inference { @@ -57,19 +58,7 @@ std::string DataFlowGraph::DotString() const { // Add nodes for (size_t i = 0; i < nodes.size(); i++) { const Node &node = nodes.Get(i); - switch (node.type()) { - case Node::Type::kValue: - dot.AddNode(node.repr(), node.dot_attrs()); - break; - case Node::Type::kFunction: - dot.AddNode(node.repr(), node.dot_attrs()); - break; - case Node::Type::kFunctionBlock: - dot.AddNode(node.repr(), node.dot_attrs()); - break; - default: - PADDLE_THROW("unsupported Node type %d", static_cast(node.type())); - } + dot.AddNode(node.repr(), node.dot_attrs()); } // Add edges diff --git a/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc new file mode 100644 index 0000000000..f7d4cca213 --- /dev/null +++ b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.h" +#include "paddle/fluid/framework/proto_desc.h" + +namespace paddle { +namespace inference { +namespace analysis { + +bool DataFlowGraphToFluidPass::Initialize(Argument* argument) { + ANALYSIS_ARGUMENT_CHECK_FIELD(argument) + ANALYSIS_ARGUMENT_CHECK_FIELD(argument->origin_program_desc) + desc_ = argument->origin_program_desc.get(); + // Here some logic from program_desc.cc and will not add new interfaces into + // framework::ProgramDesc class, use some UT to assure the correctness. + auto* block = desc_->mutable_blocks()->Add(); + block->set_idx(framework::kRootBlockIndex); + block->set_parent_idx(framework::kNoneBlockIndex); + return true; +} + +bool DataFlowGraphToFluidPass::Finalize() { return true; } + +void DataFlowGraphToFluidPass::Run(DataFlowGraph* graph) { + auto traits = GraphTraits(graph); + for (auto it = traits.nodes().begin(); it != traits.nodes().end(); ++it) { + if (it->deleted()) continue; + switch (it->type()) { + case Node::Type::kFunction: + LOG(INFO) << "add function " << it->name(); + AddFluidOp(&(*it)); + break; + case Node::Type::kFunctionBlock: + AddEngineOp(&(*it)); + break; + default: + continue; + } + } +} + +void DataFlowGraphToFluidPass::AddFluidOp(Node* node) { + LOG(INFO) << "processing func " << node->name(); + auto* ori_op = static_cast(node->pb_desc()); + // currently only the main block is analyzed. + auto* main_block = desc_->mutable_blocks(framework::kRootBlockIndex); + auto* op = main_block->add_ops(); + LOG(INFO) << "to copy the op"; + *op = *ori_op; // copy the attributes, by default, these will not be changed + // by analysis phrase. + // The inputs and outputs of the existing ops are not changed by tensorrt + // subgraph pass. + // NOTE It might be changed by other passes in the long run. +} + +void DataFlowGraphToFluidPass::AddEngineOp(Node* node) { + // auto* ori_op = static_cast(node->extra_info()); + // auto* main_block = desc_->mutable_blocks(framework::kRootBlockIndex); + // auto* op = main_block->add_ops(); + // TODO(Superjomn) Here need to expose some arguments for default setting. +} + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.h b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.h new file mode 100644 index 0000000000..cbb05f622c --- /dev/null +++ b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +/* + * This file implements the transformation from fluid ProgramDesc to data flow + * graph. + */ + +#pragma once + +#include "paddle/fluid/framework/program_desc.h" +#include "paddle/fluid/inference/analysis/data_flow_graph.h" +#include "paddle/fluid/inference/analysis/pass.h" + +namespace paddle { +namespace inference { +namespace analysis { +class DataFlowGraphToFluidPass final : public DataFlowGraphPass { + public: + DataFlowGraphToFluidPass() = default; + + bool Initialize(Argument *argument) override; + bool Finalize() override; + + void Run(DataFlowGraph *graph) override; + + std::string repr() const override { return "DFG to fluid"; } + std::string description() const override { + return "Transform a DFG to a Fluid ProgramDesc"; + } + + Pass *CreatePrinterPass(std::ostream &os, + const std::string &banner) const override { + return nullptr; + } + + protected: + // Add a Fluid Op into the ProgramDesc. + void AddFluidOp(Node *node); + // Add a EngineOp into the ProgramDesc. + void AddEngineOp(Node *node); + + private: + framework::proto::ProgramDesc *desc_; +}; +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass_tester.cc b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass_tester.cc index dcee75cee5..d8fc5e580a 100644 --- a/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass_tester.cc +++ b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass_tester.cc @@ -27,13 +27,12 @@ namespace inference { namespace analysis { TEST_F(DFG_Tester, Test) { - framework::proto::ProgramDesc new_desc; DataFlowGraph graph; FluidToDataFlowGraphPass pass0; DataFlowGraphToFluidPass pass1; - pass0.Initialize(desc); - pass1.Initialize(&new_desc); + ASSERT_TRUE(pass0.Initialize(&argument)); + ASSERT_TRUE(pass1.Initialize(&argument)); pass0.Run(&graph); pass1.Run(&graph); diff --git a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc new file mode 100644 index 0000000000..afffb3feb0 --- /dev/null +++ b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc @@ -0,0 +1,54 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h" + +namespace paddle { +namespace inference { +namespace analysis { + +void DFG_GraphvizDrawPass::Run(DataFlowGraph *graph) { + auto content = Draw(graph); + std::ofstream file(GenDotPath()); + file.write(content.c_str(), content.size()); + file.close(); + LOG(INFO) << "draw dot to " << GenDotPath(); +} + +std::string DFG_GraphvizDrawPass::Draw(DataFlowGraph *graph) { + Dot dot; + // Add nodes + for (size_t i = 0; i < graph->nodes.size(); i++) { + const Node &node = graph->nodes.Get(i); + if (config_.display_deleted_node || !node.deleted()) { + dot.AddNode(node.repr(), node.dot_attrs()); + } + } + // Add edges + for (size_t i = 0; i < graph->nodes.size(); i++) { + const Node &node = graph->nodes.Get(i); + if (!config_.display_deleted_node && node.deleted()) continue; + for (auto &in : node.inlinks) { + if (!config_.display_deleted_node && in->deleted()) continue; + for (auto &in : node.inlinks) { + dot.AddEdge(in->repr(), node.repr(), {}); + } + } + } + return dot.Build(); +} + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h index 41d4475382..93ebff59ae 100644 --- a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h +++ b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h @@ -21,6 +21,7 @@ limitations under the License. */ #include #include +#include "paddle/fluid/inference/analysis/dot.h" #include "paddle/fluid/inference/analysis/pass.h" namespace paddle { @@ -32,35 +33,39 @@ namespace analysis { */ class DFG_GraphvizDrawPass : public DataFlowGraphPass { public: - DFG_GraphvizDrawPass(const std::string& dir, const std::string& id) - : dir_(dir), id_(id) {} - - bool Initialize() override { return Pass::Initialize(); } - void Run(DataFlowGraph* graph) override { - auto content = Draw(graph); - std::ofstream file(GenDotPath()); - file.write(content.c_str(), content.size()); - file.close(); - LOG(INFO) << "draw dot to " << GenDotPath(); - } + struct Config { + Config(const std::string &dir, const std::string &id, + bool display_deleted_node = false) + : dir(dir), id(id), display_deleted_node(display_deleted_node) {} + + // The directory to store the .dot or .png files. + const std::string dir; + // The identifier for this dot file. + const std::string id; + // Whether to display deleted nodes, default false. + const bool display_deleted_node; + }; + + DFG_GraphvizDrawPass(const Config &config) : config_(config) {} + bool Initialize(Argument *argument) override { return true; } + void Run(DataFlowGraph *graph) override; bool Finalize() override { return Pass::Finalize(); } - Pass* CreatePrinterPass(std::ostream& os, - const std::string& banner) const override { - return nullptr; + std::string repr() const override { return "DFG graphviz drawer"; } + std::string description() const override { + return "Debug a DFG by draw with graphviz"; } private: // Path of the dot file to output. std::string GenDotPath() const { - return dir_ + "/" + "graph_" + id_ + ".dot"; + return config_.dir + "/" + "graph_" + config_.id + ".dot"; } - std::string Draw(DataFlowGraph* graph) { return graph->DotString(); } + std::string Draw(DataFlowGraph *graph); - std::string dir_; - std::string id_; + Config config_; }; } // namespace analysis diff --git a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass_tester.cc b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass_tester.cc index 3fc1cc18b8..f4b5c5fd22 100644 --- a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass_tester.cc +++ b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass_tester.cc @@ -24,9 +24,10 @@ namespace inference { namespace analysis { TEST_F(DFG_Tester, dfg_graphviz_draw_pass_tester) { - auto dfg = ProgramDescToDFG(desc); - DFG_GraphvizDrawPass pass("./", "test"); - pass.Initialize(); + auto dfg = ProgramDescToDFG(*argument.origin_program_desc); + DFG_GraphvizDrawPass::Config config("./", "test"); + DFG_GraphvizDrawPass pass(config); + pass.Initialize(&argument); pass.Run(&dfg); // test content @@ -38,7 +39,8 @@ TEST_F(DFG_Tester, dfg_graphviz_draw_pass_tester) { while (std::getline(file, line)) { no++; } - ASSERT_EQ(no, 82); + // DFG is sensitive to ProgramDesc, be careful to change the existing models. + ASSERT_EQ(no, 112); } } // namespace analysis diff --git a/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.cc b/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.cc index 9f67c989cc..5f62eef528 100644 --- a/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.cc +++ b/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.cc @@ -21,19 +21,23 @@ namespace paddle { namespace inference { namespace analysis { -FluidToDataFlowGraphPass::FluidToDataFlowGraphPass() {} - -bool FluidToDataFlowGraphPass::Initialize() { return Pass::Initialize(); } - -bool FluidToDataFlowGraphPass::Initialize( - const framework::proto::ProgramDesc &desc) { - desc_ = &desc; +bool FluidToDataFlowGraphPass::Initialize(Argument *argument) { + ANALYSIS_ARGUMENT_CHECK_FIELD(argument); + ANALYSIS_ARGUMENT_CHECK_FIELD(argument->origin_program_desc); + PADDLE_ENFORCE(argument); + if (!argument->main_dfg) { + LOG(INFO) << "Init DFG"; + argument->main_dfg.reset(new DataFlowGraph); + } + desc_ = argument->origin_program_desc.get(); return true; } bool FluidToDataFlowGraphPass::Finalize() { return Pass::Finalize(); } void FluidToDataFlowGraphPass::Run(DataFlowGraph *graph) { + PADDLE_ENFORCE(graph); + PADDLE_ENFORCE(desc_); // insert vars std::unordered_map var2id; auto &main_block = desc_->blocks(framework::kRootBlockIndex); @@ -41,7 +45,7 @@ void FluidToDataFlowGraphPass::Run(DataFlowGraph *graph) { const auto &var = main_block.vars(i); auto *v = graph->nodes.Create(Node::Type::kValue); v->SetName(var.name()); - v->SetExtraInfo(const_cast(static_cast(&var))); + v->SetPbDesc(const_cast(static_cast(&var))); var2id[var.name()] = v->id(); } for (int i = 0; i < main_block.ops_size(); i++) { @@ -51,7 +55,7 @@ void FluidToDataFlowGraphPass::Run(DataFlowGraph *graph) { static_cast(o)->SetFuncType(op.type()); // Link to the original protobuf message's memory, make it easier to // generate from a data flow graph to fluid ProgramDesc. - o->SetExtraInfo(const_cast(static_cast(&op))); + o->SetPbDesc(const_cast(static_cast(&op))); // set inputs and outputs // TODO(Superjomn) make sure the InputNames is the real variable name. for (int j = 0; j < op.inputs_size(); j++) { diff --git a/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h b/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h index 33517e57be..176faf0220 100644 --- a/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h +++ b/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h @@ -34,13 +34,18 @@ namespace analysis { */ class FluidToDataFlowGraphPass final : public DataFlowGraphPass { public: - FluidToDataFlowGraphPass(); - bool Initialize() override; - bool Initialize(const framework::proto::ProgramDesc &desc) override; + FluidToDataFlowGraphPass() = default; + + bool Initialize(Argument *argument) override; bool Finalize() override; void Run(DataFlowGraph *graph) override; + std::string repr() const override { return "fluid-to-data-flow-graph"; } + std::string description() const override { + return "transform a fluid ProgramDesc to a data flow graph."; + } + Pass *CreatePrinterPass(std::ostream &os, const std::string &banner) const override; diff --git a/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass_tester.cc b/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass_tester.cc index 817d32c92c..cfbbc284e4 100644 --- a/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass_tester.cc +++ b/paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass_tester.cc @@ -23,11 +23,11 @@ namespace analysis { TEST_F(DFG_Tester, Init) { FluidToDataFlowGraphPass pass; - pass.Initialize(); - pass.Initialize(desc); + pass.Initialize(&argument); DataFlowGraph graph; pass.Run(&graph); - ASSERT_GT(graph.nodes.size(), 0); + // Analysis is sensitive to ProgramDesc, careful to change the original model. + ASSERT_EQ(graph.nodes.size(), 37); pass.Finalize(); LOG(INFO) << '\n' << graph.DotString(); } diff --git a/paddle/fluid/inference/analysis/helper.h b/paddle/fluid/inference/analysis/helper.h index 58eb0e715c..f0039e1131 100644 --- a/paddle/fluid/inference/analysis/helper.h +++ b/paddle/fluid/inference/analysis/helper.h @@ -62,6 +62,7 @@ struct DataTypeNamer { SET_TYPE(int); SET_TYPE(bool); SET_TYPE(float); + SET_TYPE(void *); } std::unordered_map inlinks; // Output links. std::vector outlinks; // A helper class to maintain the status from Pass. - // TODO(superjomn) add a checker here to ensure the T is primary. struct Attr { // NOTE T should be a primary type or a struct combined by several primary // types. // NOTE the STL containers should not use here. // Some usages - // Attr attr; - // T data; - // attr.data.assign((char*)data, sizeof(data)); + // Attr attr; + // attr.Bool() = true; bool &Bool() { return As(); } float &Float() { return As(); } int32_t &Int32() { return As(); } int64_t &Int64() { return As(); } + void *&Pointer() { return As(); } private: template @@ -130,6 +131,7 @@ class Node { size_t type_hash_{std::numeric_limits::max()}; }; + // Type checks. bool IsFunction() const { return type_ == Node::Type::kFunction; } bool IsValue() const { return type_ == Node::Type::kValue; } bool IsFunctionBlock() const { return type_ == Node::Type::kFunctionBlock; } @@ -148,9 +150,6 @@ class Node { Type type_{Type::kNone}; // Mark this node is deleted by some pass. bool deleted_{false}; - - void *extra_info_; - mutable std::unordered_map attrs_; }; diff --git a/paddle/fluid/inference/analysis/pass.h b/paddle/fluid/inference/analysis/pass.h index aa0e8667b5..65632b7491 100644 --- a/paddle/fluid/inference/analysis/pass.h +++ b/paddle/fluid/inference/analysis/pass.h @@ -19,6 +19,7 @@ limitations under the License. */ #include #include "paddle/fluid/framework/framework.pb.h" +#include "paddle/fluid/inference/analysis/argument.h" #include "paddle/fluid/inference/analysis/data_flow_graph.h" #include "paddle/fluid/inference/analysis/helper.h" #include "paddle/fluid/inference/analysis/node.h" @@ -30,19 +31,24 @@ namespace analysis { class Pass { public: Pass() = default; - virtual ~Pass() {} + virtual ~Pass() = default; // Virtual method overridden by subclasses to do only necessary initialization // before any pass is run. - virtual bool Initialize() { return false; } + // virtual bool Initialize() { return false; } // There is some passes such as FlowToDataFlowGraphPass that needs a // ProgramDesc. Here use the native ProgramDesc ProtoBuf message, so that it // only couple with the proto file. - virtual bool Initialize(const framework::proto::ProgramDesc &desc) { - return false; - } + // virtual bool Initialize(const framework::proto::ProgramDesc &desc) { return + // false; } // There are some Passes such as DataFlowGraphToFluidPass that will output a // ProgramDesc. - virtual bool Initialize(framework::proto::ProgramDesc *desc) { return false; } + // virtual bool Initialize(framework::proto::ProgramDesc *desc) { return + // false; } + + // Mutable Pass. + virtual bool Initialize(Argument *argument) { return false; } + // Readonly Pass. + virtual bool Initialize(const Argument &argument) { return false; } // Virtual method overriden by subclasses to do any necessary clean up after // all passes have run. @@ -50,7 +56,9 @@ class Pass { // Get a Pass appropriate to print the Node this pass operates on. virtual Pass *CreatePrinterPass(std::ostream &os, - const std::string &banner) const = 0; + const std::string &banner) const { + return nullptr; + } // Run on a single Node. virtual void Run(Node *x) { LOG(FATAL) << "not valid"; } @@ -60,6 +68,11 @@ class Pass { virtual void Run(FunctionBlock *x) { LOG(FATAL) << "not valid"; } // Run on a single DataFlowGraph. virtual void Run(DataFlowGraph *x) { LOG(FATAL) << "not valid"; } + + // Human-readable short representation. + virtual std::string repr() const = 0; + // Human-readable long description. + virtual std::string description() const = 0; }; // NodePass process on any Node types. diff --git a/paddle/fluid/inference/analysis/pass_manager.cc b/paddle/fluid/inference/analysis/pass_manager.cc new file mode 100644 index 0000000000..b17c0e0d72 --- /dev/null +++ b/paddle/fluid/inference/analysis/pass_manager.cc @@ -0,0 +1,44 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/analysis/pass_manager.h" +#include "paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h" + +namespace paddle { +namespace inference { +namespace analysis { + +void DfgPassManager::RunAll() { + PADDLE_ENFORCE(argument_); + for (auto& pass : data_) { + VLOG(4) << "Running pass [" << pass->repr() << "]"; + pass->Run(argument_->main_dfg.get()); + } +} + +void NodePassManager::RunAll() { + PADDLE_ENFORCE(argument_); + PADDLE_ENFORCE(argument_->main_dfg.get()); + auto trait = + GraphTraits(argument_->main_dfg.get()).nodes_in_DFS(); + for (auto& node : trait) { + for (auto& pass : data_) { + pass->Run(&node); + } + } +} + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/pass_manager.h b/paddle/fluid/inference/analysis/pass_manager.h new file mode 100644 index 0000000000..7841c4b9d0 --- /dev/null +++ b/paddle/fluid/inference/analysis/pass_manager.h @@ -0,0 +1,116 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +/* + * This file defines the logic of pass management. The analysis for inference is + * a pipeline of Passes, a PassManager is a agency that helps to manage the + * executation of the Passes. + * + * There are two modes of Passes, the first one is called NodePass and takes + * an Node as input and output; the second one is called DFGPass and takes a + * DFG(Data Flow Graph) as input and output. It is hard to put all the passes in + * the same pipeline, there are two kinds of PassManagers, both takes a DFG as + * input and output a DFG, but the Passes inside are different: + * + * 1. NodePassManager: the passes inside are all NodePasses, it can have + * different graph trivial algorithm, for example, DFS_NodePassManager will + * trigger the passes in depth first order; + * 2. DfgPassManager: the passes inside are all DfgPasses. + */ + +#pragma once + +#include +#include "paddle/fluid/framework/program_desc.h" +#include "paddle/fluid/inference/analysis/pass.h" + +namespace paddle { +namespace inference { +namespace analysis { + +/* + * PassManager is the base class for all pass managers, a pass manager has + * several Pass-es registered, and execute them in the linear order. + */ +class PassManager : public OrderedRegistry { + public: + PassManager() = default; + // Call all the passes' Initialize methods. The desc and data_flow_graph are + // globally shared, so pass them as the arguemnts for all the pass managers. + virtual bool Initialize(const Argument& argument) { return false; } + + virtual bool Initialize(Argument* argument) { + argument_ = argument; + for (auto& pass : data_) { + LOG(INFO) << "Initializing pass " << pass->repr(); + if (!pass->Initialize(argument)) { + LOG(ERROR) << "Failed to initialize pass [" << pass->repr() << "]"; + return false; + } + } + return true; + } + + // Call all the passes' Finalize methods. + virtual bool Finalize() { + for (auto& pass : data_) { + if (!pass->Finalize()) { + LOG(ERROR) << "Failed to finalize pass [" << pass->repr() << "]"; + return false; + } + } + return true; + } + + // Run all the passes. + virtual void RunAll() = 0; + + // Short identifier. + virtual std::string repr() const = 0; + // Long description. + virtual std::string description() const = 0; + + virtual ~PassManager() = default; + + protected: + Argument* argument_{nullptr}; +}; + +/* + * A pass manager that process a DFG. + */ +class DfgPassManager : public PassManager { + public: + DfgPassManager() = default; + + void RunAll() override; + + virtual ~DfgPassManager() = default; +}; + +/* + * A pass manager that process a Node each time. + */ +class NodePassManager : public PassManager { + public: + NodePassManager() = default; + + void RunAll() override; + + virtual ~NodePassManager() = default; +}; + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/pass_manager_tester.cc b/paddle/fluid/inference/analysis/pass_manager_tester.cc new file mode 100644 index 0000000000..7af6a19951 --- /dev/null +++ b/paddle/fluid/inference/analysis/pass_manager_tester.cc @@ -0,0 +1,85 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/analysis/pass_manager.h" +#include "paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.h" +#include "paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h" +#include "paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h" +#include "paddle/fluid/inference/analysis/ut_helper.h" + +#include + +namespace paddle { +namespace inference { +namespace analysis { + +class TestDfgPassManager final : public DfgPassManager { + public: + TestDfgPassManager() = default; + virtual ~TestDfgPassManager() = default; + // Short identifier. + std::string repr() const override { return "test-pass-manager"; } + // Long description. + std::string description() const override { return "test doc"; } +}; + +class TestNodePassManager final : public NodePassManager { + public: + virtual ~TestNodePassManager() = default; + + std::string repr() const override { return "test-node-pass-manager"; } + std::string description() const override { return "test doc"; } +}; + +class TestNodePass final : public NodePass { + public: + virtual ~TestNodePass() = default; + + bool Initialize(Argument* argument) override { return true; } + + void Run(Node* node) override { + LOG(INFO) << "- Processing node " << node->repr(); + } + + std::string repr() const override { return "test-node"; } + std::string description() const override { return "some doc"; } +}; + +TEST_F(DFG_Tester, DFG_pass_manager) { + TestDfgPassManager manager; + DFG_GraphvizDrawPass::Config config("./", "dfg.dot"); + + manager.Register("fluid-to-flow-graph", new FluidToDataFlowGraphPass); + manager.Register("graphviz", new DFG_GraphvizDrawPass(config)); + manager.Register("dfg-to-fluid", new DataFlowGraphToFluidPass); + + ASSERT_TRUE(manager.Initialize(&argument)); + manager.RunAll(); +} + +TEST_F(DFG_Tester, Node_pass_manager) { + // Pre-process: initialize the DFG with the ProgramDesc first. + FluidToDataFlowGraphPass pass0; + pass0.Initialize(&argument); + pass0.Run(argument.main_dfg.get()); + + TestNodePassManager manager; + manager.Register("test-node-pass", new TestNodePass); + ASSERT_TRUE(manager.Initialize(&argument)); + manager.RunAll(); +} + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/subgraph_splitter_tester.cc b/paddle/fluid/inference/analysis/subgraph_splitter_tester.cc index 0644c0db12..8134494f8b 100644 --- a/paddle/fluid/inference/analysis/subgraph_splitter_tester.cc +++ b/paddle/fluid/inference/analysis/subgraph_splitter_tester.cc @@ -19,22 +19,23 @@ namespace paddle { namespace inference { namespace analysis { +SubGraphSplitter::NodeInsideSubgraphTeller teller = [](const Node* node) { + if (node->type() != Node::Type::kFunction) return false; + const auto* func = static_cast(node); + if (func->func_type() == "elementwise_add" || func->func_type() == "relu" || + func->func_type() == "conv2d" || func->func_type() == "mul" || + func->func_type() == "sigmoid" || func->func_type() == "softmax") { + LOG(INFO) << "sub-graph marked " << node->repr(); + return true; + } + return false; +}; + TEST_F(DFG_Tester, Split) { auto desc = LoadProgramDesc(); auto dfg = ProgramDescToDFG(desc); LOG(INFO) << "spliter\n" << dfg.DotString(); - SubGraphSplitter::NodeInsideSubgraphTeller teller = [](const Node* node) { - if (node->type() != Node::Type::kFunction) return false; - const auto* func = static_cast(node); - if (func->func_type() == "elementwise_add" || func->func_type() == "relu" || - func->func_type() == "conv2d" || func->func_type() == "mul" || - func->func_type() == "sigmoid" || func->func_type() == "softmax") { - LOG(INFO) << "sub-graph marked " << node->repr(); - return true; - } - return false; - }; ASSERT_GT(dfg.nodes.size(), 5UL); auto subgraphs = SubGraphSplitter(&dfg, teller)(); @@ -62,6 +63,28 @@ TEST_F(DFG_Tester, Split) { ASSERT_EQ(subgraphs.back().size(), 6UL); } +TEST_F(DFG_Tester, Fuse) { + auto desc = LoadProgramDesc(); + auto dfg = ProgramDescToDFG(desc); + + size_t count0 = dfg.nodes.size(); + + SubGraphFuse fuse(&dfg, teller); + fuse(); + + int count1 = 0; + for (auto& node : dfg.nodes.nodes()) { + if (node->deleted()) { + LOG(INFO) << "deleted " << node->repr(); + } + count1 += node->deleted(); + } + + // At least one nodes should be deleted. + ASSERT_EQ(dfg.nodes.size(), count0 + 1); // added a new FunctionBlock + ASSERT_EQ(6UL, count1); +} + } // namespace analysis } // namespace inference } // namespace paddle diff --git a/paddle/fluid/inference/analysis/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/tensorrt_subgraph_pass.cc new file mode 100644 index 0000000000..b75df33b71 --- /dev/null +++ b/paddle/fluid/inference/analysis/tensorrt_subgraph_pass.cc @@ -0,0 +1,33 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/inference/analysis/tensorrt_subgraph_pass.h" +#include "paddle/fluid/inference/analysis/subgraph_splitter.h" + +namespace paddle { +namespace inference { +namespace analysis { + +TensorRTSubGraphPass::TensorRTSubGraphPass( + const TensorRTSubGraphPass::NodeInsideSubgraphTeller &teller) + : node_inside_subgraph_teller_(teller) {} + +void TensorRTSubGraphPass::Run(DataFlowGraph *graph) { + SubGraphFuse(graph, node_inside_subgraph_teller_); +} + +} // analysis +} // inference + +} // paddle diff --git a/paddle/fluid/inference/analysis/tensorrt_subgraph_pass.h b/paddle/fluid/inference/analysis/tensorrt_subgraph_pass.h new file mode 100644 index 0000000000..79e9e2bcc9 --- /dev/null +++ b/paddle/fluid/inference/analysis/tensorrt_subgraph_pass.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/fluid/inference/analysis/node.h" +#include "paddle/fluid/inference/analysis/pass.h" +#include "paddle/fluid/inference/analysis/subgraph_splitter.h" + +namespace paddle { +namespace inference { +namespace analysis { + +/* + * Parse the graph and replace TensorRT supported nodes with SubGraphNode + */ +class TensorRTSubGraphPass : public DataFlowGraphPass { + public: + // Tell whether to transform a sub-graph into TensorRT. + using NodeInsideSubgraphTeller = SubGraphFuse::NodeInsideSubgraphTeller; + + TensorRTSubGraphPass(const NodeInsideSubgraphTeller& teller); + + bool Initialize(Argument* argument) override { return true; } + + // This class get a sub-graph as input and determine whether to transform this + // sub-graph into TensorRT. + void Run(DataFlowGraph* graph) override; + + private: + NodeInsideSubgraphTeller node_inside_subgraph_teller_; +}; + +} // namespace analysis +} // namespace inference +} // paddle diff --git a/paddle/fluid/inference/analysis/tensorrt_subgraph_pass_tester.cc b/paddle/fluid/inference/analysis/tensorrt_subgraph_pass_tester.cc new file mode 100644 index 0000000000..d12dcf0d0f --- /dev/null +++ b/paddle/fluid/inference/analysis/tensorrt_subgraph_pass_tester.cc @@ -0,0 +1,71 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/inference/analysis/tensorrt_subgraph_pass.h" + +#include +#include +#include "paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.h" +#include "paddle/fluid/inference/analysis/ut_helper.h" + +namespace paddle { +namespace inference { +namespace analysis { + +DEFINE_string(model_dir, "", "inference test model dir"); + +TEST(TensorRTSubGraph, single_pass) { + auto desc = LoadProgramDesc(); + auto dfg = ProgramDescToDFG(desc); + + SubGraphSplitter::NodeInsideSubgraphTeller teller = [](const Node* node) { + if (node->type() != Node::Type::kFunction) return false; + const auto* func = static_cast(node); + if (func->func_type() == "elementwise_add" || func->func_type() == "relu" || + func->func_type() == "conv2d" || func->func_type() == "mul" || + func->func_type() == "sigmoid" || func->func_type() == "softmax") { + LOG(INFO) << "sub-graph marked " << node->repr(); + return true; + } + return false; + }; + + DFG_GraphvizDrawPass::Config config{"./", "test"}; + DFG_GraphvizDrawPass dfg_pass(config); + dfg_pass.Initialize(); + + DFG_GraphvizDrawPass dfg_pass1(config); + dfg_pass1.Initialize(); + + dfg_pass.Run(&dfg); + + TensorRTSubGraphPass trt_pass(std::move(teller)); + trt_pass.Initialize(); + + trt_pass.Run(&dfg); + + dfg_pass1.Run(&dfg); + + // Check the TRT op's block desc + for (auto node : dfg.nodes.nodes()) { + if (node->IsFunctionBlock()) { + } + } +} + +TEST(TensorRTSubGraph, pass_manager) {} + +} // namespace analysis +} // namespace inference +} // namespace paddle diff --git a/paddle/fluid/inference/analysis/ut_helper.h b/paddle/fluid/inference/analysis/ut_helper.h index 722fa99a48..ce1191a567 100644 --- a/paddle/fluid/inference/analysis/ut_helper.h +++ b/paddle/fluid/inference/analysis/ut_helper.h @@ -15,33 +15,46 @@ limitations under the License. */ #pragma once #include #include +#include #include #include "paddle/fluid/framework/executor.h" #include "paddle/fluid/inference/analysis/data_flow_graph.h" #include "paddle/fluid/inference/analysis/fluid_to_data_flow_graph_pass.h" #include "paddle/fluid/inference/analysis/ut_helper.h" -#include "paddle/fluid/inference/io.h" namespace paddle { namespace inference { + +// Read ProgramDesc from a __model__ file, defined in io.cc +extern void ReadBinaryFile(const std::string& filename, std::string* contents); + namespace analysis { DEFINE_string(inference_model_dir, "", "inference test model dir"); static framework::proto::ProgramDesc LoadProgramDesc( const std::string& model_dir = FLAGS_inference_model_dir) { - paddle::platform::CPUPlace place; - paddle::framework::Executor executor(place); - paddle::framework::Scope scope; - auto program = Load(&executor, &scope, model_dir); - return *program->Proto(); + std::string msg; + std::string net_file = FLAGS_inference_model_dir + "/__model__"; + std::ifstream fin(net_file, std::ios::in | std::ios::binary); + PADDLE_ENFORCE(static_cast(fin), "Cannot open file %s", net_file); + fin.seekg(0, std::ios::end); + msg.resize(fin.tellg()); + fin.seekg(0, std::ios::beg); + fin.read(&(msg.at(0)), msg.size()); + fin.close(); + framework::proto::ProgramDesc program_desc; + program_desc.ParseFromString(msg); + return program_desc; } static DataFlowGraph ProgramDescToDFG( const framework::proto::ProgramDesc& desc) { DataFlowGraph graph; FluidToDataFlowGraphPass pass; - pass.Initialize(desc); + Argument argument; + argument.origin_program_desc.reset(new framework::proto::ProgramDesc(desc)); + pass.Initialize(&argument); pass.Run(&graph); pass.Finalize(); return graph; @@ -49,9 +62,12 @@ static DataFlowGraph ProgramDescToDFG( class DFG_Tester : public ::testing::Test { protected: - void SetUp() override { desc = LoadProgramDesc(FLAGS_inference_model_dir); } + void SetUp() override { + auto desc = LoadProgramDesc(FLAGS_inference_model_dir); + argument.origin_program_desc.reset(new framework::proto::ProgramDesc(desc)); + } - framework::proto::ProgramDesc desc; + Argument argument; }; } // namespace analysis diff --git a/paddle/fluid/operators/tensorrt_engine_op_test.cc b/paddle/fluid/operators/tensorrt_engine_op_test.cc index 85330958cd..3a2fef4805 100644 --- a/paddle/fluid/operators/tensorrt_engine_op_test.cc +++ b/paddle/fluid/operators/tensorrt_engine_op_test.cc @@ -240,7 +240,7 @@ void Execute(int batch_size, int input_dim, int output_dim, int nlayers = 1) { } // Test with a larger FC layer. -TEST(TensorRTEngineOp, fc) { Execute(40, 256, 256); } +TEST(TensorRTEngineOp, fc) { Execute(40, 28, 28); } } // namespace operators } // namespace paddle From 792d3b240605d50dfad53a95f1f3283dd3fa7871 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Mon, 11 Jun 2018 10:11:24 +0200 Subject: [PATCH 152/154] MKLDNN layout: Support for activation operator --- .../fluid/operators/activation_mkldnn_op.cc | 316 +++++++++++------- paddle/fluid/operators/activation_op.cc | 29 +- 2 files changed, 212 insertions(+), 133 deletions(-) diff --git a/paddle/fluid/operators/activation_mkldnn_op.cc b/paddle/fluid/operators/activation_mkldnn_op.cc index 46ed99bcf2..137bca5e2b 100644 --- a/paddle/fluid/operators/activation_mkldnn_op.cc +++ b/paddle/fluid/operators/activation_mkldnn_op.cc @@ -12,16 +12,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "mkldnn.hpp" #include "paddle/fluid/operators/activation_op.h" -#include "paddle/fluid/operators/mkldnn_activation_op.h" #include "paddle/fluid/platform/mkldnn_helper.h" namespace paddle { namespace operators { -using paddle::framework::Tensor; -using paddle::platform::MKLDNNDeviceContext; +using framework::DataLayout; +using framework::Tensor; +using mkldnn::memory; +using mkldnn::primitive; +using mkldnn::stream; +using platform::GetMKLDNNFormat; +using platform::MKLDNNDeviceContext; +using platform::to_void_cast; namespace { std::string gethash(const mkldnn::memory::dims &operand_dims, @@ -35,188 +39,260 @@ std::string gethash(const mkldnn::memory::dims &operand_dims, }; return dim2str(operand_dims) + std::to_string(algorithm); } +} // namespace + +template +class MKLDNNActivationKernel + : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + const auto *x = ctx.Input("X"); + PADDLE_ENFORCE(x->layout() == DataLayout::kMKLDNN && + x->format() != memory::format::format_undef, + "Wrong layout/format set for Input x tensor"); + + Functor functor; + + auto attrs = functor.GetAttrs(); + for (auto &attr : attrs) { + *attr.second = ctx.Attr(attr.first); + } + functor(ctx); + } +}; -template -void eltwise_forward(const ExecContext &ctx, mkldnn::algorithm algorithm, - const T alpha = 0, const T beta = 0) { +template +class MKLDNNActivationGradKernel + : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + const auto *diff_y = ctx.Input(framework::GradVarName("Out")); + PADDLE_ENFORCE(diff_y->layout() == DataLayout::kMKLDNN && + diff_y->format() != memory::format::format_undef, + "Wrong layout/format set for Input OutGrad tensor"); + + Functor functor; + + auto attrs = functor.GetAttrs(); + for (auto &attr : attrs) { + *attr.second = ctx.Attr(attr.first); + } + functor(ctx); + } +}; + +template +void eltwise_forward(const framework::ExecutionContext &ctx, + mkldnn::algorithm algorithm, const T alpha = 0, + const T beta = 0) { PADDLE_ENFORCE(paddle::platform::is_cpu_place(ctx.GetPlace()), "It must use CPUPlace."); - auto &dev_ctx = ctx.template device_context(); const auto &mkldnn_engine = dev_ctx.GetEngine(); - // get buffers - const auto *src = ctx.template Input("X"); - const auto *src_data = src->template data(); + const auto *x = ctx.Input("X"); + auto *y = ctx.Output("Out"); - auto *dst = ctx.template Output("Out"); - T *dst_data = dst->template mutable_data(ctx.GetPlace()); + const T *x_data = x->data(); + T *y_data = y->mutable_data(ctx.GetPlace()); - // get memory dim - PADDLE_ENFORCE(src->dims().size() == 2 || src->dims().size() == 4, + PADDLE_ENFORCE(x->dims().size() == 2 || x->dims().size() == 4, "Input dim must be with 2 or 4"); - std::vector src_tz = framework::vectorize2int(src->dims()); + + std::vector src_tz = framework::vectorize2int(x->dims()); + + auto src_format = + src_tz.size() == 2 ? mkldnn::memory::format::nc : x->format(); const std::string key = gethash(src_tz, algorithm); const std::string key_src_data = key + ctx.op().Output("Out") + "@eltwise_fwd_src_data"; - const std::string key_src_mem = key + "@eltwise_fwd_src_mem"; - const std::string key_dst_mem = key + "@eltwise_fwd_dst_mem"; - const std::string key_fwd = key + "@eltwise_fwd"; + const std::string key_src_layout = + key + ctx.op().Output("Out") + "@eltwise_fwd_src_layout"; + const std::string key_with_layout = key + std::to_string(src_format); + const std::string key_src_mem = key_with_layout + "@eltwise_fwd_src_mem"; + const std::string key_dst_mem = key_with_layout + "@eltwise_fwd_dst_mem"; + const std::string key_fwd = key_with_layout + "@eltwise_fwd"; + const std::string key_fwd_pd = key_with_layout + "@eltwise_fwd_pd"; + + // save input data and layout to be referred in backward path + auto p_src_data = std::make_shared(x_data); + dev_ctx.SetBlob(key_src_data, p_src_data); + auto p_src_layout = std::make_shared(src_format); + dev_ctx.SetBlob(key_src_layout, p_src_layout); auto p_fwd = std::static_pointer_cast( dev_ctx.GetBlob(key_fwd)); - // save input data to be referred in backward path - auto p_src_data = std::make_shared(src_data); - dev_ctx.SetBlob(key_src_data, p_src_data); + std::shared_ptr dst_memory; if (p_fwd == nullptr) { - // create memory description - auto data_md = src_tz.size() == 2 - ? platform::MKLDNNMemDesc(src_tz, mkldnn::memory::f32, - mkldnn::memory::format::nc) - : platform::MKLDNNMemDesc(src_tz, mkldnn::memory::f32, - mkldnn::memory::format::nchw); - - // create memory primitives - auto p_src_mem = std::make_shared(mkldnn::memory( - {data_md, mkldnn_engine}, platform::to_void_cast(src_data))); - dev_ctx.SetBlob(key_src_mem, p_src_mem); - - auto p_dst_mem = std::make_shared(mkldnn::memory( - {data_md, mkldnn_engine}, platform::to_void_cast(dst_data))); - dev_ctx.SetBlob(key_dst_mem, p_dst_mem); - - auto fwd_desc = mkldnn::eltwise_forward::desc( - mkldnn::prop_kind::forward_training, algorithm, data_md, alpha, beta); - auto p_fwd_pd = std::make_shared( - fwd_desc, mkldnn_engine); - const std::string key_fwd_pd = key + "eltwise_fwd_pd"; - dev_ctx.SetBlob(key_fwd_pd, p_fwd_pd); - p_fwd = std::make_shared( - *p_fwd_pd, *(p_src_mem.get()), *(p_dst_mem.get())); + // create mkldnn memory for input X + auto src_md = platform::MKLDNNMemDesc( + src_tz, platform::MKLDNNGetDataType(), src_format); + auto src_memory = std::shared_ptr( + new memory({src_md, mkldnn_engine}, to_void_cast(x_data))); + // save src_memory to be referred in backward path + dev_ctx.SetBlob(key_src_mem, src_memory); + + // create primitive descriptor for activation forward and save it + auto forward_desc = mkldnn::eltwise_forward::desc( + mkldnn::prop_kind::forward_training, algorithm, + src_memory->get_primitive_desc().desc(), alpha, beta); + auto forward_pd = std::make_shared( + forward_desc, mkldnn_engine); + + // save prim desc into global device context to be referred in backward path + dev_ctx.SetBlob(key_fwd_pd, forward_pd); + + // create mkldnn memory for output y + dst_memory = + std::make_shared(forward_pd->dst_primitive_desc(), y_data); + + dev_ctx.SetBlob(key_dst_mem, dst_memory); + + // create activation primitive + p_fwd = std::make_shared(*forward_pd, *src_memory, + *dst_memory); dev_ctx.SetBlob(key_fwd, p_fwd); } else { // primitives already exist - auto p_src_mem = + auto src_memory = std::static_pointer_cast(dev_ctx.GetBlob(key_src_mem)); - PADDLE_ENFORCE(p_src_mem != nullptr, - "Fail to find eltwise p_src_mem in device context."); - auto p_dst_mem = + PADDLE_ENFORCE(src_memory != nullptr, + "Fail to find eltwise src_memory in device context."); + dst_memory = std::static_pointer_cast(dev_ctx.GetBlob(key_dst_mem)); - PADDLE_ENFORCE(p_dst_mem != nullptr, - "Fail to find eltwise p_src_mem in device context."); + PADDLE_ENFORCE(dst_memory != nullptr, + "Fail to find eltwise dst_memory in device context."); - p_src_mem->set_data_handle(platform::to_void_reinterpret_cast(src_data)); - p_dst_mem->set_data_handle(dst_data); + src_memory->set_data_handle(platform::to_void_cast(x_data)); + dst_memory->set_data_handle(y_data); } // push primitive to stream and wait until it's executed - std::vector pipeline = {*(p_fwd.get())}; - mkldnn::stream(mkldnn::stream::kind::eager).submit(pipeline).wait(); + std::vector pipeline; + pipeline.push_back(*p_fwd); + stream(stream::kind::eager).submit(pipeline).wait(); + + y->set_layout(DataLayout::kMKLDNN); + y->set_format(GetMKLDNNFormat(*dst_memory)); } -template -void eltwise_grad(const ExecContext &ctx, mkldnn::algorithm algorithm, - const T alpha = 0, const T beta = 0) { +template +void eltwise_grad(const framework::ExecutionContext &ctx, + mkldnn::algorithm algorithm, const T alpha = 0, + const T beta = 0) { auto &dev_ctx = ctx.template device_context(); const auto &mkldnn_engine = dev_ctx.GetEngine(); - // get buffers - const auto *out = ctx.template Input("Out"); - - auto *dout = ctx.template Input(framework::GradVarName("Out")); - const auto *diff_dst = dout->template data(); + const auto *diff_y = ctx.Input(framework::GradVarName("Out")); + auto *diff_x = ctx.Output(framework::GradVarName("X")); - auto *dx = - ctx.template Output(framework::GradVarName("X")); - const T *diff_src = dx->template mutable_data(ctx.GetPlace()); + const T *diff_y_data = diff_y->data(); + T *diff_x_data = diff_x->mutable_data(ctx.GetPlace()); - // get memory dim - std::vector src_tz = framework::vectorize2int(out->dims()); + std::vector diff_dst_tz = framework::vectorize2int(diff_y->dims()); - const std::string key = gethash(src_tz, algorithm); - const std::string key_diff_src_mem = key + "@eltwise_diff_src_mem"; - const std::string key_diff_dst_mem = key + "@eltwise_diff_dst_mem"; - const std::string key_grad = key + "@eltwise_grad"; + auto diff_y_format = + diff_dst_tz.size() == 2 ? mkldnn::memory::format::nc : diff_y->format(); + const std::string key = gethash(diff_dst_tz, algorithm); const std::string key_src_data = key + ctx.op().Input("Out") + "@eltwise_fwd_src_data"; + const std::string key_src_layout = + key + ctx.op().Input("Out") + "@eltwise_fwd_src_layout"; + const auto p_src_layout = + std::static_pointer_cast(dev_ctx.GetBlob(key_src_layout)); + const std::string key_src_mem = + key + std::to_string(*p_src_layout) + "@eltwise_fwd_src_mem"; + const std::string key_fwd_pd = + key + std::to_string(*p_src_layout) + "@eltwise_fwd_pd"; + const std::string key_with_layouts = + key + std::to_string(*p_src_layout) + "-" + std::to_string(diff_y_format); + const std::string key_diff_src_mem = + key_with_layouts + "@eltwise_diff_src_mem"; + const std::string key_diff_dst_mem = + key_with_layouts + "@eltwise_diff_dst_mem"; + const std::string key_grad = key_with_layouts + "@eltwise_grad"; + const auto p_src_data = std::static_pointer_cast(dev_ctx.GetBlob(key_src_data)); - const std::string key_src_mem = key + "@eltwise_fwd_src_mem"; - auto p_src_mem = + auto src_memory = std::static_pointer_cast(dev_ctx.GetBlob(key_src_mem)); - p_src_mem->set_data_handle(*p_src_data.get()); + PADDLE_ENFORCE(src_memory != nullptr, + "Fail to find src_memory in device context"); + src_memory->set_data_handle(*p_src_data.get()); + + std::shared_ptr diff_src_memory; - auto p_grad = std::static_pointer_cast( + auto p_grad = std::static_pointer_cast( dev_ctx.GetBlob(key_grad)); if (p_grad == nullptr) { - // create memory description - auto data_md = src_tz.size() == 2 - ? platform::MKLDNNMemDesc(src_tz, mkldnn::memory::f32, - mkldnn::memory::format::nc) - : platform::MKLDNNMemDesc(src_tz, mkldnn::memory::f32, - mkldnn::memory::format::nchw); - - // create memory primitives - std::shared_ptr p_diff_src_mem = - std::make_shared(mkldnn::memory( - {data_md, mkldnn_engine}, platform::to_void_cast(diff_src))); - dev_ctx.SetBlob(key_diff_src_mem, p_diff_src_mem); - std::shared_ptr p_diff_dst_mem = - std::make_shared(mkldnn::memory( - {data_md, mkldnn_engine}, platform::to_void_cast(diff_dst))); - dev_ctx.SetBlob(key_diff_dst_mem, p_diff_dst_mem); - - auto bwd_desc = mkldnn::eltwise_backward::desc(algorithm, data_md, data_md, - alpha, beta); - - const std::string key_fwd_pd = key + "eltwise_fwd_pd"; - auto *p_fwd_pd = static_cast( - dev_ctx.GetBlob(key_fwd_pd).get()); - - auto eltwise_bwd_prim_desc = mkldnn::eltwise_backward::primitive_desc( - bwd_desc, mkldnn_engine, *p_fwd_pd); - + // create mkldnn memory for input diff_y + auto diff_dst_md = platform::MKLDNNMemDesc( + diff_dst_tz, platform::MKLDNNGetDataType(), diff_y_format); + auto diff_dst_memory = std::shared_ptr( + new memory({diff_dst_md, mkldnn_engine}, to_void_cast(diff_y_data))); + dev_ctx.SetBlob(key_diff_dst_mem, diff_dst_memory); + + // retrieve eltwise primitive desc from device context + auto forward_pd = + std::static_pointer_cast( + dev_ctx.GetBlob(key_fwd_pd)); + PADDLE_ENFORCE(forward_pd != nullptr, + "Fail to find eltwise_fwd_pd in device context"); + + // ceate primitive descriptor for activation backward + auto backward_desc = mkldnn::eltwise_backward::desc( + algorithm, diff_dst_memory->get_primitive_desc().desc(), + src_memory->get_primitive_desc().desc(), alpha, beta); + auto backward_pd = mkldnn::eltwise_backward::primitive_desc( + backward_desc, mkldnn_engine, *forward_pd); + + // create mkldnn memory for output diff_src + diff_src_memory = std::make_shared( + backward_pd.diff_src_primitive_desc(), diff_x_data); + dev_ctx.SetBlob(key_diff_src_mem, diff_src_memory); + + // create activation backward primitive p_grad = std::make_shared( - eltwise_bwd_prim_desc, *static_cast(p_src_mem.get()), - *(static_cast(p_diff_dst_mem.get())), - *(static_cast(p_diff_src_mem.get()))); + backward_pd, *src_memory, *diff_dst_memory, *diff_src_memory); + dev_ctx.SetBlob(key_grad, p_grad); } else { // primitives already exist - auto p_diff_src_mem = std::static_pointer_cast( + diff_src_memory = std::static_pointer_cast( dev_ctx.GetBlob(key_diff_src_mem)); - auto p_diff_dst_mem = std::static_pointer_cast( + auto diff_dst_memory = std::static_pointer_cast( dev_ctx.GetBlob(key_diff_dst_mem)); - p_diff_src_mem->set_data_handle( - platform::to_void_reinterpret_cast(diff_src)); - p_diff_dst_mem->set_data_handle( - platform::to_void_reinterpret_cast(diff_dst)); + diff_src_memory->set_data_handle( + platform::to_void_reinterpret_cast(diff_x_data)); + diff_dst_memory->set_data_handle( + platform::to_void_reinterpret_cast(diff_y_data)); } // push primitive to stream and wait until it's executed - std::vector pipeline = {*(p_grad.get())}; - mkldnn::stream(mkldnn::stream::kind::eager).submit(pipeline).wait(); + std::vector pipeline; + pipeline.push_back(*p_grad); + stream(stream::kind::eager).submit(pipeline).wait(); + + diff_x->set_layout(DataLayout::kMKLDNN); + diff_x->set_format(GetMKLDNNFormat(*diff_src_memory)); } -} // anonymous namespace template struct MKLDNNActivationFunc : public BaseActivationFunctor { - template - void operator()(const ExecContext &ctx) const { + void operator()(const framework::ExecutionContext &ctx) const { eltwise_forward(ctx, algorithm); } }; template struct MKLDNNActivationGradFunc : public BaseActivationFunctor { - template - void operator()(const ExecContext &ctx) const { + void operator()(const framework::ExecutionContext &ctx) const { eltwise_grad(ctx, algorithm); } }; diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index a06ca7952f..b6b498a616 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -19,18 +19,20 @@ limitations under the License. */ namespace paddle { namespace operators { -#define REGISTER_ACTIVATION_OP_MAKER(OP_NAME, OP_COMMENT) \ - class OP_NAME##OpMaker \ - : public ::paddle::framework::OpProtoAndCheckerMaker { \ - public: \ - void Make() override { \ - AddInput("X", "Input of " #OP_NAME " operator"); \ - AddOutput("Out", "Output of " #OP_NAME " operator").Reuse("X"); \ - AddAttr("use_mkldnn", \ - "(default false) Only used in mkldnn kernel") \ - .SetDefault(false); \ - AddComment(OP_COMMENT); \ - } \ +using paddle::framework::Tensor; + +#define REGISTER_ACTIVATION_OP_MAKER(OP_NAME, OP_COMMENT) \ + class OP_NAME##OpMaker \ + : public ::paddle::framework::OpProtoAndCheckerMaker { \ + public: \ + void Make() override { \ + AddInput("X", "Input of " #OP_NAME " operator"); \ + AddOutput("Out", "Output of " #OP_NAME " operator").Reuse("X"); \ + AddAttr("use_mkldnn", \ + "(bool, default false) Only used in mkldnn kernel") \ + .SetDefault(false); \ + AddComment(#OP_COMMENT); \ + } \ } #define REGISTER_ACTIVATION_OP_GRAD_MAKER(OP_NAME, KERNEL_TYPE) \ @@ -58,7 +60,6 @@ framework::OpKernelType GetKernelType(const framework::ExecutionContext& ctx, const framework::OperatorWithKernel& oper, const std::string& name) { framework::LibraryType library{framework::LibraryType::kPlain}; - framework::DataLayout layout = framework::DataLayout::kAnyLayout; #ifdef PADDLE_WITH_MKLDNN auto it = oper.Attrs().find("use_mkldnn"); @@ -82,6 +83,7 @@ class ActivationOp : public framework::OperatorWithKernel { ctx->ShareLoD("X", /*->*/ "Out"); } + protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { return GetKernelType(ctx, *this, "X"); @@ -96,6 +98,7 @@ class ActivationOpGrad : public framework::OperatorWithKernel { ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("Out")); } + protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { return GetKernelType(ctx, *this, "Out"); From 4dda54aa5af6cc7f53d4ee5e5212293d9a67023b Mon Sep 17 00:00:00 2001 From: gongweibao Date: Mon, 18 Jun 2018 19:59:43 -0500 Subject: [PATCH 153/154] Fix unlikely (#11537) --- paddle/fluid/inference/analysis/argument.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h index 7d7131ed7a..f7f4e03968 100644 --- a/paddle/fluid/inference/analysis/argument.h +++ b/paddle/fluid/inference/analysis/argument.h @@ -21,6 +21,8 @@ * big. */ +#pragma once + #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/inference/analysis/data_flow_graph.h" @@ -43,7 +45,7 @@ struct Argument { #define UNLIKELY(condition) __builtin_expect(static_cast(condition), 0) #define ANALYSIS_ARGUMENT_CHECK_FIELD(field__) \ - if (!UNLIKELY(field__)) { \ + if (UNLIKELY(!(field__))) { \ LOG(ERROR) << "field " << #field__ << " should be set."; \ return false; \ } From d00a0436b1c562060b49aa2981be094d78bcbdf5 Mon Sep 17 00:00:00 2001 From: "Yang Yang(Tony)" Date: Mon, 18 Jun 2018 18:50:19 -0700 Subject: [PATCH 154/154] Remove tape (#11548) * Remove tape * remove tape in cmake * fix CI --- paddle/contrib/CMakeLists.txt | 1 - paddle/contrib/tape/CMakeLists.txt | 25 -- paddle/contrib/tape/README.md | 252 -------------------- paddle/contrib/tape/computation_graph.png | Bin 96637 -> 0 bytes paddle/contrib/tape/function.h | 131 ----------- paddle/contrib/tape/tape.cc | 265 ---------------------- paddle/contrib/tape/tape.h | 64 ------ paddle/contrib/tape/test_tape.cc | 61 ----- paddle/contrib/tape/variable.cc | 33 --- paddle/contrib/tape/variable.h | 85 ------- 10 files changed, 917 deletions(-) delete mode 100644 paddle/contrib/tape/CMakeLists.txt delete mode 100644 paddle/contrib/tape/README.md delete mode 100644 paddle/contrib/tape/computation_graph.png delete mode 100644 paddle/contrib/tape/function.h delete mode 100644 paddle/contrib/tape/tape.cc delete mode 100644 paddle/contrib/tape/tape.h delete mode 100644 paddle/contrib/tape/test_tape.cc delete mode 100644 paddle/contrib/tape/variable.cc delete mode 100644 paddle/contrib/tape/variable.h diff --git a/paddle/contrib/CMakeLists.txt b/paddle/contrib/CMakeLists.txt index 70e3a0583d..4b19256ef4 100644 --- a/paddle/contrib/CMakeLists.txt +++ b/paddle/contrib/CMakeLists.txt @@ -14,4 +14,3 @@ # add_subdirectory(inference) -add_subdirectory(tape) diff --git a/paddle/contrib/tape/CMakeLists.txt b/paddle/contrib/tape/CMakeLists.txt deleted file mode 100644 index 5450359d85..0000000000 --- a/paddle/contrib/tape/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -if(APPLE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move") -endif(APPLE) - -cc_library(tape_variable SRCS variable.cc DEPS ${FLUID_CORE_MODULES} device_context framework_proto proto_desc operator) -cc_library(tape SRCS tape.cc DEPS ${FLUID_CORE_MODULES} ${GLOB_OP_LIB} tape_variable) - -cc_test(test_tape - SRCS test_tape.cc - DEPS tape tape_variable) diff --git a/paddle/contrib/tape/README.md b/paddle/contrib/tape/README.md deleted file mode 100644 index 16c22a45d5..0000000000 --- a/paddle/contrib/tape/README.md +++ /dev/null @@ -1,252 +0,0 @@ -# Dynamic Graph on Fluid - -PaddlePaddle Fluid is targeting the autodiff without tape, which, however, is very -challenging and we are still way from there. DyNet and PyTorch provide a good design -idea, the *tape*, that significantly eases the challenge. Also, DyNet provides -a C++ API that is as convenient as Python but with higher efficiency and could -conveniently integrate with industrial/production systems. This package, `tape`, -combines the good of - -1. tape from PyTorch and DyNet -2. C++ API and core from DyNet -3. rich set of operators from PaddlePaddle - -## Overview - -We can implement Dynet-like Tape(See this [survey](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/survey/dynamic_graph.md)) -by wrapping Paddle Fluid's `Operator` and `Variable`. - -The user API is straight forward since - -1. it is imperative. And it uses host language's control flow logic. -1. it avoids extra concepts such as `Scope` and `Executor`. - -All of these benefits come at the cost of just adding one line `reset_global_tape` -at every iteration. - -## Code Structure - -In short, the `Tape` contains a vector of `OpHandle`s. And an `OpHandle` contains its -`type`, the pointers to the `Variable`s, and necessary attributes. - -```c++ -class Variable { -public: - VriableHandle Grad(); // returns its gradient variable -private: - framework::VarDesc desc_; // compile time infershape, necessary for lazy execution - framework::Variable var_; // run time variable, holds data memory -}; - -using VariableHandle = shared_ptr; - -struct OpHandle { - string type_; - map> inputs_; - map> outputs_; - AttributeMap attrs_; -}; - -class Tape { -public: - void AddOp(OpHandle); // add op - void Forward(); // execute the tape_ - void Backward(); // execute the backward of the tape_ -private: - vector tape_; -}; -``` - -We uses `Function` to indicate layers. It takes care of parameter -initialization and `AddOp` to the Tape when it is called. - -```c++ -class Linear { - public: - Linear(int in_dim, int out_dim, const std::string &act) - : w_(new Variable("LinearWeight")), - b_(new Variable("LinearBias")), - act_(act) { - Tape init_tape; - - std::string initializer = "fill_constant"; - framework::AttributeMap attrs; - attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{in_dim, out_dim}; - attrs["value"] = 1.0f; - init_tape.AddOp(initializer, {}, {{"Out", {w_}}}, attrs); - - attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{out_dim}; - attrs["value"] = 1.0f; - init_tape.AddOp(initializer, {}, {{"Out", {b_}}}, attrs); - - init_tape.Forward(); - } - - VariableHandle operator()(VariableHandle input) { - VariableHandle pre_bias(new Variable("linear")); - get_global_tape().AddOp("mul", - {{"X", {input}}, {"Y", {w_}}}, - {{"Out", {pre_bias}}}, - {{"x_num_col_dims", 1}, {"y_num_col_dims", 1}}); - VariableHandle pre_act(new Variable("linear")); - get_global_tape().AddOp("elementwise_add", - {{"X", {pre_bias}}, {"Y", {b_}}}, - {{"Out", {pre_act}}}, - {{"axis", 1}}); - VariableHandle post_act(new Variable("linear")); - get_global_tape().AddOp(act_, - {{"X", {pre_act}}}, - {{"Out", {post_act}}}, - {}); - return post_act; - } - - std::vector Params() { return {w_, b_}; } - - private: - VariableHandle w_; - VariableHandle b_; - std::string act_; -}; -``` - -## User API - -```c++ -// Model function -paddle::tape::Linear linear1(3, 3, "relu"); // init weight and bias -paddle::tape::Linear linear2(3, 3, "relu"); // init weight and bias -paddle::tape::Mean mean; - -// Optimizer -paddle::tape::SGD sgd(0.001); - -// Data Feeder -paddle::tape::Fill data_feeder(...); -VariableHandle input(new paddle::tape::Variable("input")); -VariableHandle label(new paddle::tape::Variable("label")); - -for (int i = 0; i < 2; ++i) { - reset_global_tape(); - - data_feeder(input, label); - - auto loss = softmax(linear2(linear1(input)), label); // compile time InferShape & InferVarType - LOG(INFO) << loss.value(); // Run forward up to loss - - // Run backward, store gradient of w at w->Grad() - get_global_tape.Backward(loss); - - // Update w - sgd(linear1.Params()); - sgd(linear2.Params()); -} -``` - -
- -digraph G { - - subgraph cluster_0 { - node [shape=record,style=filled]; - style=filled; - color=lightgrey; - linear1 [label="{type: mul | {input | {X: before_mul1 | Y: weight1}} | {output | Out: before_bias1}}"]; - elementwise_add1 [label="{type: elementwise_add | {input | {X: before_bias1 | Y: bias1}} | {output | Out: before_act1}}"]; - relu1 [label="{type: relu | {input | {X: before_act1 }} | {output | Out: after_act1}}"]; - - linear1 -> elementwise_add1->relu1; - label = "forward tape"; - } - - linear1:before_mul1->before_mul1 - linear1:weight1->weight1 - linear1:before_bias1->before_bias1 - - elementwise_add1:bias1->bias1 - elementwise_add1:before_bias1->before_bias1 - elementwise_add1:before_act1->before_act1 - - relu1:before_act1->before_act1 - relu1:after_act1->after_act1 - - subgraph cluster_1 { - node [shape=record,style=filled]; - style=filled; - color=lightgrey; - linear1_grad [label="{type: mul_grad | {input | {X: before_mul1 | Y: weight1| Out_grad: before_bias1_grad}} | {output |{X_grad: before_mul1_grad | Y_grad: weight1_grad}}}"]; - - elementwise_add1_grad [label="{type: elementwise_add_grad | {input | Out_grad: before_act1_grad} | {output |{X_grad: before_bias1_grad | Y_grad: bias1_grad}}}"]; - - relu1_grad [label="{type: relu_grad | {input | Out_grad: after_act1_grad} | {ouput | {X_grad: before_act1_grad }}}"]; - - linear1_grad -> elementwise_add1_grad ->relu1_grad [dir=back]; - label = "backward tape"; - } - - relu1_grad:after_act1_grad->after_act1_grad - relu1_grad:before_act1_grad->before_act1_grad - - elementwise_add1_grad:before_act1_grad->before_act1_grad - elementwise_add1_grad:before_bias1_grad->before_bias1_grad - elementwise_add1_grad:bias1_grad->bias1_grad - - linear1_grad:before_mul1->before_mul1 - linear1_grad:weight1->weight1 - linear1_grad:before_bias1_grad->before_bias1_grad - linear1_grad:before_mul1_grad->before_mul1_grad - linear1_grad:weight1_grad->weight1_grad - - - subgraph cluster_2 { - node [shape=record]; - label = "Linear1"; - weight1 - bias1 - } - - weight1 -> weight1_grad [ label="Grad()", style="dashed" ]; - bias1 -> bias1_grad [ label="Grad()", style="dashed"]; - - - -} -
- -![Image](https://github.com/tonyyang-svail/Paddle/blob/cpp_tap/paddle/contrib/tape/computation_graph.png) - -## Code Reuse - -We want to stay close to Paddle Fluid as much as possible. - -### Reuse All Operators - -As all Ops are registered at `OpInfoMap`, the effort of adding a new `Function` -is about 10 lines of code, similar to expose an operator to Python. - -### Reuse Compile Time InferShape and InferVarType - -Note that all the symbolic information is stored at `tape::Varaible::desc_`, instead -of `ProgramDesc.block.vars`, we create a temporary `BlockDesc` to do `InferShape` and -`InferVarType` every time we `AddOp` to the tape. - -### Reuse Operator::Run - -We use smart pointer, instead of `Scope`, to manage memory. So we create a temporary -`Scope` for every `Operator::Run()`. - -## Possible Feature - -### Release Memory on Backward - -We can release memory aggressively. During backward, we can delete the OpHandle once -we have finished its backward. Since all the variable is managed by smart pointer, the -memory is automatically released when its `ref_count` goes to 0. - -### Kernel Fusion - -As a symbolic representation of the Tape is constructed first before the actual -execution, it would be possible to perform graph optimization. One use case is kernel -fusion. diff --git a/paddle/contrib/tape/computation_graph.png b/paddle/contrib/tape/computation_graph.png deleted file mode 100644 index 6cf5ead735d5d18b204b079771e53d44483cf016..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96637 zcmcG01yq&Y_a-VJk^%xsOM`Svie5?SZlt@rL6J);-5}lF4I&^?UgFXv-5}j_uHW~Y z`L8vzX3eZM<62(0Z=L<_v(Mhoe)c|uD9A}*p%bGcAt7N&NkWv6kRFhO|7U3T!4VdG z9BT06fzxX#6*M%onI-u}@b#JVTMcJrJ5y)3_l_n=X0~=VCM-@yjwU9yPUd#b`w!Yh zkdP>mq#$A{?x{O-e%{(Ht)d5ReqSDRAy+*7^k{%=%G!8d=sYT1Y-^6#MCJyj)1hMD zm|*2JPSRuDU>PbzyqHXNyfKkd@&02}2m1#YQgV|p@g8@pL3WHve&_qkr?Xsqjx7Vq zq#=~_M#onY^&EAs+whu<8V}*Ssd%S9FZb~se0GmK)}1WHB_tjJk&%98#=G1@{NnU5 zivaw1HASg~_#TV(|KRJG{GXQkc~1-+$4}@g+S*=lO|!}}pufd0ed5KfUT5*rxfU!a zi`?ei$vr*da8)oaDd!Z5*gH)z6k;;6ZoecXBn8D^Rw&?v0xwhz{6I4KVE(`Rsyq%W z`XY1ed?6r|E-7xJ8y3BZb(w1FHy=?S=<-$s?R6ur>auDp}_lfb}tAx&I6iz5M9 zEw~9d+Yzf`SEE$j=M1a(^~%_}IfWm-nwl8>Y=A(62%!~v)oz$&PC@i17lN29({BMA z3Q;c4z;Sz}7@tpzvsuzNOOkeFIgU$_k^E*3VXXM84?kpXRQ$A(>La=!{5)*3B-QsF z!jbsfXG!fPxy+vKb$79zmrydDm6um@@!&Unfg z+Wc?FWTV3O66!xpD5Yy`4&er@6zMbeeSe%sc6=Xz-+F=;}{Wb8RhUY;TU zk*phA%JenKH-^cgn4PSvqYkR%)$4Bvn89p?6mlk$Yx9bQfOK!7ctvDo$>}cb)e7 zS~gXt(V=rDk=^*4frIg63|ce2bn%jLmqwK}5Uo=2Awnr_f!k!#zOJewn=y)dRQD=F zuGT0>UCy{=XqRKi@p4)=KH(p3%03zy$HD@LPn!r8Rc!$-@c|D6%(cY_bC(*2`@h^V zd7Gs$X-7Sz32gK}i-)?MVXf3oQTG*n`BNWO#71tp6v7!`eA!^q_C|$R#w!0+sJoDq z6*JsoMRJo&?h`%v#V^u$ajQe9Z-1dKDtI%whlI!>Q0TVBH%hwPvoyta(GBDd#6_PYW1i@@pi_p9|7BM6AV>J||q6 zLw}SPG)B2m$k0C-4A!)PIHguZCI4<<>bsSmQ*2eGL+8Gg8u=%sP&-?mQ3W-t`Yot4 z9B#TBmJ~hi$}eMqr4`NfG6v)@SC=s9=NG`Z#Mt-9#yGv5e4v=P)BWh)|)#_6}yZLyvTyczDT+ncQ3 zUnQG~4xz_u4l1AZ>ivna9pWrglGn7vLefSN`aNbw>CH%sObX#}CL0=jeEjgbu&x+* z%6AFLFc_>aHKBhMo0Ok@ZoFf&A9;toQ23cmab6;afItzl2nT#X|!s{hKZIi4x7OCNQ9X%!@ZQR9XFO?bmQ^JC=S0q+X4B zTU}XMtZgPB9bK*vY%A4is8WO3wT^8vpp`!w0-=V(`ttRjel2e}v7o*GWU?0gzQ$ zG5z7(0MA7y^eOq5u+Va&SM7g1m!0m|zQi66vVQIzHB_k%G3Sp~DQ%NWd4lol@IJdEB&gHBn9B&ON04P!kx0cUd4ewk!xo6k|qo08Ld)MiOz_&Af$5e2V+tq1FCY1=rh-YAZse(5XflH;umwHBQan$ z9arz+mO`KC>bnYHi8_XoJJ$o85(&L<1*+E4?jN|D?_8Z8*qymd6SM5OV#sS*Nw_Pw z*ik-DUagqOP=G>npygaF=iiRts?tE3Q4XryETo-F3aUi$Ed-$S_MzXSIA;HXIsGCfulG zxkwGN9%_lkH=E*ShYie@bWuuB$B-5K596jQP$+VY;tL!T8mbWwZ))aiDV7UtHo-+rJGSAZumJ<1wKeur(AO$zYuK9qDw%-g9rZZuK* zv#gpD)!6vzkxPwRE{guSLRb@w?1{FEMMn12{Fct{VK;U#a~xEj! z&frXPx6gbAS>_|Vi`7IK4{?MQyyWHWuZ@iY@oWxa7z}uzxck*=`IHI7yf5eL$Jd{m z`yg@ws@FBi;nsF`k&=Vm-;}B!93e6Tb`D5!jB;=`xmU5wfM6@jzV?~&|L|vZ9YsP2 zrISuhY^^Zgu5>g!Jg0Lj=o9W52Zvvwbd`Xx@;^iI*!+pD2)_9r#+-lsh+WGe>NXy@ zkgiw=X1P9;X3KZk_+ATu>Eg1cP#44_VwBR&T?Tn32DauV?bQ#Q( zIz1&8GA!tNN06$88PEH z$=*o4qA9hxuJJIAsj>JIt#EG{_(BMvcQ)dBF8@mfR?^A_=Plm}Z43ur2DW>5xU>Qa_6X7G1~U){6Zt@(EU;9}?7`GfQgR~_H30@A%S z;Ly4q_(Fi4{Wg{W*8;l~%@I|iS+_g~?^)cKGgWEgj#@fY(u@hr?^G=I`Z|WZXcL&JU62qrBVFWG!u{ZkL*GcNBZUvHz zL9=PTkU7^FbHS5OyCaiqMq!r3F)t{m>F|vrBPDY~6H6Ewl&utgfalV?t zgrxS(>{_ct->baiCLCDNNk5JVj3ho5Qq+rx7EoBVKh%qdNW@3-IM>U z)k$Ivd|yR?xyy7;q9}ZES3pRrJE_H2-H3w`HS*&NM)>^f&Zh8G-nKLoFT0_oQzFz^ zEm-!-P$=~ac7D9WD2AEK^o2E59L1H92#hbMvR11%i8W=#KR5%1>73+!n~reqpe+99(Xf*Voj*gm^aW1tmaANKz1c(U&Dsw~vNCIT!YPtF1@-(Q%_}WNs0GDd zGQKZUTlcdl~gbLoz=DHPJxhkUlPm;LHbP7KKW4)H$N2Hv{} zfd`E1_!%G7E%ebA*=dZq+nX=jiG)>t@)MB`TbZf||l*ArNE1@>S~O#B1VQ`7j)q`3h*}^f zUG2BGmjn$KqySewZB_2ls8fOw;83F53^(U4)2v=Qf0Lb^O&rl<&OPLfKrqX%NxV)J z@kHcl_=dhwnwevK)P*vFjjJ`990Y}AgjJ3r@3?9A3cT!|%)5%@ag>7Fe|tly%}7=<+TU(SexIZO3< z%EsZO?4a&5DkW6vbhYZKDcGt5ulAs+)6v!|>E~2D6EjOS$veAkk*;RNt&NT2qg-_o zrXZpKuyv&b0nA5z{+w;ynQ`G4S>*d6?_bioI-h%1PNRWwxgy`zj3klTHnX302>ni( z-6dqPcvqG_`O1?4x8g^L0kg#SXEQCXt>U8_=x>*O`ivW7caJWp@S-yNEci%01rOtS z6H1p38dyonEh;WvKd;2{LOix3yf}7{h@Yi=8T!MZmYA`Wv^J+VB8XvV!`d#=J*F^q zM)!2=c&~YXZ>pL;rZ3jZ0Nhl-8i!X(7r=!EklH9rGPh`JZ7_w;Ze9q>s7r-KyDr=X zL9fN|gk_C$3v8EEdy7zzJ1Zx_%XIsjfYSZ@lHY27D)NxZ3=$83Lb)N3WuamOujQXP zgmDYCt7;gMkkexu^pGDmx&7>h0#sseru5umtRm)fxVL=_ns@^pSl$0pU z#HU42RcWup8j;f_%pqj&L0qWB^xG+UaJTh57F?TGE75g(@+Y7`; zKT{!22^XLexZWLF%HpPpDtt!^#2%oH6SU%4w-^7s(5N8$u=|WIDi%`F4qHxclaSg@ zZpr*I^DV+lAr953H3{GLejb7n{N<%jYtG)>^cH3^lDY`O0Fo*u%o5dsu6&`=-k6dm z4;^?g_$;C-IT}wSB6{@V)b)vtWriFzWrD&CM zZ^`Aoyn@TSM)w{aua8&qxxFj};eScEO{GwD(cZ$3SovmT~sOXT3?(epYusIm6q$K3~rt@+>HxP@G3R3*6M^nJN)bl0dCr%U|{K#0H zdAd2TV42?bu3sys%`C}yW>tW!gXSa-ugWt)a=2}SnSnm~&z zQ~0s}IFbbm2ow21ulQM>)t?Kk;FgeA`B}8VlO8!@8C)LA7W?jYOMs+7iMHroT4YR+ z#3ssxNITRV#>(#6T65syn&3-$WX@PpQvD`o5f9C;4wZ5DyYlm8ptd5BHF_pxR>hoS z&nsag8;t(zk3b~Uy|^|inmX>trE@~x&!<10#qWBVf`@qUnOSNk%)G~!>-6eqNN_*& zMOni}Rv2s3Mi!3Wc~?mo3X;ENusP4NGq0V*(W`a;+4&jM#4*A1HMQyu`8O->{x7sw-0#mP(@Ys$?9f)hVRL^y1viJIAF^4+?Rqd2$_FvpAhLfM^U25 z#4~GF8TY*<(*1r(2b_Txr4~mA#_I>U9%7bX$FVVFo!1395!{r8ihJx(lQd5+) z+FH$RS!aohHP(P-GT!;o7vjj(X9{`0HczINw4y`g8XhDrZaL>Ree>0i&W?t|@m=K( z<~pZj$|tr@E^}oP$ZrfY6($>Qn4BRQl6YYNxZn+&_v}r|>XYX!98sKAUCp zJq(K=;cL78({EaRAvrR#DR&+M>^Ajr);MNAPS5WLNylh$(vK=7Dr@-3BZ>Q5LP2=)121wiv@zwsP+zR4b3cG`5K$2-h4EKCd)KbA`v zWSD%{B09YpLGF;~)Dx4Pq(O2cci<`E_{TpLUmFur_pTIt!%9Q85vQefaR9{k)s4?I>;kdX2B0 zBV}+x*KH1qZ??jq^FG|xhCCa*Ezqt8xt05@j36K z!3jOD(*9LL-5wI@E&eesfwc{V?lovNyOzStb*fDV4}&B3Ym>7; z{bzg1EW>9l$*j(LiVE$U3mEq1WP(kzDo1De0a8b5uuo}`#B%~nqSDi|ye8+szE`B5 z)5aIL?B-jx%bNEjL@#0bw|^1<#VQ}aD(|zU97e>px3e?b=5s}JM+Us~I3}#f)1Fk~ zt&~U={hR0bA*=7?W{!nT%?OLj%k>)_P54RUO+1!+A|B$9>$G~F&?e|mg=S@`45jM> zlXduIr7FyyA|LzxYj5_rB~-dQ8a_sjU2l+Zf?rfv*tnK#RR%uxefGV83PsO5aYX(^ zeXgmin|JJInb2{<+TH@j_?uEo6@UIWA&V_sbwE${@4wJ2q3eULV)b(PrZf5H^^CiV zjA4;0Yk2NGq*u8efs`GEOCGnUGj@Kfaq54kz=G>RTqSs4eaPmuldRF_pslslRz3PVHg5u?#wmUho=O}6{BDf3LXD?<2LWH4i?5y$M@ia-_^{5=`xGThu4`MO&^t= zU~h?nx|U|_+HdM+Y_(h59pvRd_pbuAzt<%K;rHVoNbzUjpQ{hW_?lgJo0dM2kb579 zMW6~stC!VUO;oWiIeVXXQTPJuO844pWb?b|&s5mxd`Qvq=jRJeB7mqyu>r$6YJTx& zm96!7WNW-w^rBC`Y0lO==rNMP5JGEuCnqQMbFMQ72kf{$J4O^<*N45D>Hwqx_5$62 zZXiQc6fZ>RWK6k!!MC|Jq8wT6;WUok_$B`*~6+n-R z=UVp$3aY_m14+EM3iI1;Ps)uh(Lt#|tmeLfL3b!Yqt4#-(J&uz#75hFq(qs=rHRV9 zD&Hq8dA;!R8+?>~+3^l8gm!g9u8&p3K{fE#yfwhY`J4AuWsAo}fK|zhT&(c5UksAi zwJ55p!U=#$J)_i;4UCpOJ{Q_e2bIJ1BNulevVcj0HXTv4mz`_2JP!Gnyo|rMh9_&yT=uN9FC=i8SYCiGQT7^?o%r)Gbc?VSQXE?}Ol!fI{`lnzopPNFE{? zU^5KIXO2D><=3#Z#FM+a37TDGUoc+E*OAETV)+Sw#cs_-`cpK>I;XlT&M~OVc3Fxl z3Q+ajva+8F)f`9}JAFIj@FtEQb@`JkzhE^zs&ZPIDAN|)Vx?zZUoZyF0kW7YEXBwjyx$X)8Y z5T)}LA`m0Eh+FmEkJ&A{ejPtxFGpyVsYg*o9LgV`@`FU{`hDdtclvHV&&!*vjU>;l zv^SPkR{PESZNcQ8KbGvh&zcuZbg>cRJWH-+TEWXycnv?8j*iY3PZ*w|vIlha7&V>r z?qP$rg40cqq9AXm0S_g^vpY$J58rV}Gexm(k<~53x7}0b zXk2^oHS5#q8Zf*nMV~^k8K3NIuqW!=tGD3a*pftVOPu7?EBZm-Ec}Sk?;N!?_?ymi z{B7*~P@!_gCqrwyj!DkabcB(4aU+sh&D7bj8J{$kPY$B*5!BQ7216pB*PhDEr6d^cPRYtiMl`NmiA7ng6}D18y!Fqrmv|#v+rR%yJ1ZTIlXZRq+$iRE(FdmCfC+ zAU_S*9}nV{voKa!o=D8mQT=w#7a3J0D{H^K1U3EOu1cS)(_V5f%wZQ)q{IkQW4QF@ znOET3{HNFNB$4ca8N5Qwq4qhL63L?7*^?{X>glWVox zJjvqf48;2~bX_)<8N)ZD1_|9(+2ip$3_%^WCq0m@JK40Dma#z+4 zV15nC`QLxMN?J@avtK+xcd2_yclq&UX158ExiP;oXOp*Hv1{Nx`9Q9w`!LzVl~*nO zi5RZuS8ZNzUFL=a{brm+G5gaq+IN^=f{h~yNri1zTIH%x$>adCETVBLi<;^; z$>8PhUmXXBcEWx1B?Q*IF_e?nZSf;3s6wAUkz664=eZ^nCd!#c${Try&2YdKdya$% z!a}7Xp;=FI-)quR4XwCp%6UGgq)L;USXfwq!(Bnl61m!`F6qAqYD!%A7=a@d@216X zB}JA_JmunOk~WT$$6kKu81?5q5igR`2_3~NTwbGBWx?H)%MSCxQy<6hln!TLilRvf zl^$E)rlwK~BhC~P93r8+dgMeSZ`<+c*|BHjr&Zi6BQ;z=XV&m3uhbAvr@e^KO{i{4 z^(^10p)YaGA6+%)G@!M2w6wHryVp;56{qt25P1>8+@3lNDWtedDVfGLRacMX8r99`76H~@P4r(e?iwV~v z(kf$^UO@Q0k9>K%l!WEa1pXS@4?`o*_T4b)@DJTL;@cn+#xY(@{ng`Z)fqQCzkYq` zT0&q~zz0}ay*QDf2kjxTN$34qkB?PI6(ir>FjSQ|eyT*x8%^p_IEsFa5n+VSNlcYx zVfA|{C9Uq0p@7Hip6rD)r_9}=ITtQ$bFTYQfrk`%b0?~p{98?tDdf2jk)<=h#^Eci zv=k8#l9MpxzEdN3*L%`M6nlR&T-GKnQ^Vb*%bt@ZZO^4x)yQZWP;wPsXM5Y_?YAJo z_&aAYDIwt9pWXD1oV6Du((BgQUMSP<0Wx$h=aOVsKVe!YA6-rsr?!Amh1 z8@6@@W)C{<{17OO&#|t11wq;Tp=9%%43o?fUX(i!DN-glOx#?`$#cpf*xCFJ{F&7+x^CgfXi>Wh+}~ zbUDg%ca$8_Yx6!lpTTIkz1(hXZPmZ{slK(fMaQaF;nU-}XYbvW$1*G`S^+{I%dqvF z*o3ccOSVa8=IYp^{cjz^GcnVo5w!K~kcbIl99zG<{new}!%HioVZk*jmz*81m(46m zJKJHRl_vFz*xuXK82nTtES1SypX#m8eHrlNZDp{ej#zE9N@A2c?H5kf^{%#mSY0hF zB`~kKEc<;cik)s?d|BfrirdS^K9s=+^T(eT#wGK%8J3xe>1%6iGfVd<8~aUUSGmEE zu^G{${HA_L(~|{^)9Xd&17=c4C-5W3p~| z;kbY^E28>DxL9dT_(`<*sT?2YZv0Qi*mUA&gFM=b0ahiRWnbfL28=rPJTv>g?2yoJQmd8WcYE;(mAU*r zl1dif?+lvI%Is&mo6r)&o0^(>Dg3UBLB9aVDQ`sYjzm3gAj3nT++k|kInjtQrNPgb zAU?8`u3<{YTYbMkhlhs;NBHS+vS$9*J%YSpxGmNFicCG=@Op0g_QJz%HcM= zRPbnkOY|^A{(4Kj{o5=7(yO;gC@^%S$g{b9-!rfxng>m1+I$3?rtcv!^)hQbM)r3A zJo1n+XQ@{07Qp4Ex*%^L4Fo_|dwgt6==OY>;=JP_$hevSv=4T934rWJaI$!OV&d%N zr1f&ESpWJr5q+Zs8K@c-|2fv$oo^K+#ZE9ONaE(?d<{WJ9pY;A5bnA^uS=T}n4XS6 zz-FRXQzN-Dpyt}_d*g|K$12U7kQX01JC3~RVv;TuxQ4BH?@sI6)0V>aX6pABJ3mld z{t7ldZ>k=Wr}OsDk{cA?W%7(Vfp)`?`&o7?l+j(#90J&m{_M{pmp`aKS0ILN0_2WAuWRDHy91oN zgKUbM4TL!j?9ZYiDGWX?efo+-a3CDv+esGAd)7tH8Uf_`D>H%>@;67}S_8R}W{r8& z=;n7~ZlsQZS2c!6Fm%&*va*9QAdvu+&*|~;-=UcP9ZCw}H$VFxQ@1a)8}(u%L>%ye zs$m)&p(MV~xEL^lmuNb5Vez#91XE&l4=HgO$k97pTZkOrG~ zBk}6{C*{M;(|s>DDelgDKa1Y-Z)<@D1z-oLzk0CWp#>gvJizf@eW4#k6XF8;5u6CK z2oSRcRqOZ;($p5q@gk7u;L41EqzoArdXE|H{u>frYxj%Qg#GqAUw9A@PZo&xHG=Hv z<>Eupr`W;mC#7~boAQ2!z{)#FKLROCOZ`jh>gwX-2^0Fen;s+izeIzD27s!|juncv z&1A(=Zi1dxyRR^!DQS%uh*m*~HW0yrap^6>jBy_+mI%nC@%W>(zlj)<|E529;%RLl z`r+aSoZH*8cE4kogeK8_BzCudlui)1e({R(V9?IDT7$U?Ort^MKWb;{Ev=s~n%LfN zSjPSz7z01 ze-$u=1#~%}C58eL?u$+w-wLoocM!J_Af62deb=AB1_+egiER_aWHNsOFP%l8Qewb< zKALgse0928f&nJK6k&ulfuXhLFsjeJPW>$P6`*WxQ1=?xSqj@L+AFZUZ+^Zgx1Mc? za(Rw~RE-BzX>JrO?Fu-AIk0jU03)^Fxy|zTr9}|(j*q|Rb1g7NutzV~Q>jC#5VK-R zfmlP9mSAGycEY)a8o(<;6jh!CA#A1^^Ph+QTz3{g5o-VoPy;a?{r^~Wfj980XY*eB zfQ5MQPUjv{D&9ZqF$8T_O<_q(TRTGBi1M)k;x(-U&AU4^YQ7~?sG*6V_Z0L<{yM71w`@2-kvGq#*%=)W@bRx2u7mtJzx6ky@4iN!PI{(D~o(w zY7nmh*j0Lw!$$}q-XRoWgb@f;W7l?ea})jZgF3MR@Bx_rSYLHN@C0gQIv#*B15F!9 z280L`ve{wZo&`$#EPSdVl7U!duX5ki7#fsJpkHZqG8yA$w1a{7BoXR zx@p4D&=3|0@AXN!A7UzBy#;q5osQ_>Ia0?6+nPIV8NRdYjI)t@^vV6ub&T(1Tw#4~px z4SZxIe$*@ilG@r@-~*;?0pkUN3}EGn{)#U4xO5VbblpHoSA^>XBO8J3nA`4EEJ0w?b~dHVRtAX?|>Hy_|Nf(eYlt7DWNC8iAbR}#`_{lq z0jURQ^)E*fd_kRKf1y1CF~H7qt;zs%6sNvWf)|Jq2f!$)|2eLxt{x0diG%0>eD5_P z3W3+~7Y0LAloJeaAjnw1qm+de$Crq)*@06)1)7S4ZJ!Jd4*tg~03m7k0EWJn?sqfc z=d)AUdqBa9q|)dKM^Rw}G1MznF28qq3RoQoSTEtVD%l?%;@tlA*C?-47d`ayM`fL# z0PP^I9TGYkNU?3&jFKP+8u$&`41A6T`3@5w`fsX%Z3Uocf}e_bQ*!4AP9q6z$GHSA z)OHry@BSh=nYh1n;{6U%w~>s+joQB&q}O#CT!~xVxc182uzzi4QOW+0~QY|9!xoQ7-hsSHgS@z&Fm?`@k8dq^tj z(Bgvet@qUM`Boj=Wm5W!*xrc6xVVf-8tkdWus>Q% zYpa2MM7uSlX^tw4OGn5A)%{LB%Ln2dLJHh3=XQ6?=pICYoeC`-PAWf6dyQc3u$?0h zo4_YWP)4j_r|uRAum0zmEaiWCo%9=2BYUGgWb9)m5-Bi4m)=G39gr4AdtawLsroOZ z8q}u8Vej?e$>nusfMIrP6wT&=uG=&wD8%ZNGtf!YwB!|hDV*E4`E9o77tjczA{Lmy z)UzNq?eOjay*~+R5vL-)V-2?f+ey;Qor45qgZHOE zAUe-epr@vxscLVUooMl(&?lN|KPX$yY+j{-&pkulS=@+(R2GRKmbS)m=F1pRNd6KN zJ!PV9*k{}}$P*n+yc--HmrN1 zl@%}kd#){@@=6h4P(kTsx)B_4@@n04hXq8TrL)BB(Uld%wzD&D0i z0^5AF?d5qRE1;y@23cE);dAtao_R6f8psc{_1Op=7Y%JyIJQBU$`fd0QW`1}4^D9; z>42g1$|-V*xvf_e^KEfHUZr|d_#HVH7gm^;I@*8{(MRyu22I)IBZ?Tpg^a&?jGPUF z@`x1}Sn5SvWGYfdb84!xCq-)dET4F$=gjUoER@vOHe0i1^m3CD813(_7FO5sT2+Va z6K;mnbIg=#F_Q05kf^iBvER%6+X_>8vr69(cBZU!$gMfEgF2!~1|D^2|5`wZ zpSbkr*FDg*lRn@Pnnj5o3;SD+!SS`<9LX)gNIC(`i#z#(X`v3NO`ZZkJPw07fGSc| znMtJA*_;~zN(xDu4jJQ>-jJ=#-_|qhDVBx@3zs|hBhA}>RhUsW-GV(679rXtarNyD zUOAmL-iN)FjT#)$HgQX-qQO4=^Jr!(}ynIm-cJaYa_Rf0yXxNOv z_i(l6f>cph)Na4-roWZ%={hdV`?hv$h*sZ0-(g+@CDm7KF47DhOr@RPzehwEJMxBY zfBoazSYN9DvSFl{9?=Nr(RD-8)-Dq7PRCJE-{7~u*Gipw$TWJLov`nWbs|`AJO4x; zTWY$0ri`Gc6=NFr^PS_qm69uEl6#ih!~IM7JFNQ>-yf;#!YEJdG$=x?<*a3F{0j9n zOjlqpoL>VZh+qZ~B+SyIn4R;!RmX?eqzwA+$M=AD#7ADT(!V0!er;M0gDCC}^2q*)u$I*zVFh*-k7&Xk#r_8pcoUHo{RFHI>cfYs1fO9b z{RAhlK=xJJ2zZSl&?Kh9^fxQp3<6;oq|BDe%F0Ke42NzNh{V2*7?Ibt;I(=EeSgb; z1BrVu0oYNNLR8Qg(1VGsczHvtj`KoIM830oDq+>nPfjdDLlGxECR|1xUmYubs@I-$n{J zu{9~EjkwFYT`)9S^@1$726=(rrfeN54@5Fs)ZS9q*I2-Y!8orcW zE)Q7a!@p>@z)>cFl{O2pf9UVEsRmAsDJNXMif+xiiBn-i!v7AUMisLL9+_{iw-~pvzTro8n(}Qi@9Ns*`I#RIAY(m}<7g^mOA2qSEqo=xUW}*se0&#+ zD9wy_Slg;H)9}156RkaqEwf~{p(8}7JwF(CT9zQG{7*Z(01-|E7C;~%Sep8 z(gOP$BS}~JBcV_({#n;np)Wu13O6BI98;x2xKTsKgk#nM#duMRq1c$DTqY;vrFwS1 z98%>f`1mYUBKab3`Yd%NsiiBK>!T@t{+JdY)wm}RWAYr!%t>OdaNLxGm|0IhC7^+> zeum$beAuc$gBeeeHR^|@%@AQ5^EgPQHN{8-k#K=8#`LOsK#iKxg@;Z9dfyVnq;pS7 z#pb{^rU=x~e#ZcJO+^{%L)!%;&;+L{R!)0ztR2Hor6hmU+WX61MSxRHiX|$ZB%dh~ z3VIm{Wn{ok)fMZg8d@qPTB$75UQ9nC2RV>SQYQ2x=)B9Bg3Zs(P5z)ofh%r)Hg zq9#^@4uv8K3ZGaP0)^aj(|pzA6MEWw9-f+CdjewOBj(sQmX0V8?c5U&5@xtKj|jVl zx#dVytX#im@i6y%NKkCMu8Pm00)~9-Bbw2Xs5hymJwHqDL)EzQi;F@2zrUeAJu~z9 z+SlIRzURe#gOb8R3(yt@3TlXM9Y8OSOPJZ%z&&F^a;SLf8tBp9IkL)o7K=$>d8RY4 z^^K1G)=mEAg`%GSVw~XgDw;H|rY^l=lyh%xXJ{q#GIcT(>=^pz?*i~Hb71(9makhF zXwY%K%S7cz_bfGy>|agZ_ywv55sWViUL*KB;48y<#1e6d<=%jTpr!Ktq2CopT^t-% zj^Tp+Z&aSX!wl!Ht4;bZP5o@GWS6R;wS0VDp1Fq-iO4n&VytnmR3 z;)r!n`8X_}n3(VbG*bG7Clb@W=~9Edr^z!VI@1el$HH z$398FvaiOTMiyXS7@a2y{b{{ASK zwuPJ$LJ{feY4>c|wYH|F*D;oYwAqwuHl+_A_mS&o<%|=-WNq~;{2YBrB|rWs<(Dz) z$J*oO&81*Fnl%nKfm9!a`h#i($|QGTk};RdKh5~g>Rjt5$Hzh!D>1#Id^Yy>kHMC( z_s-5L*|4V(;_#r3%_~3#-yM(2FM9u=2rRh%FLT{j4)B9!G#>7Kl+*72+9R zRTj#uJA;3Vop<%pK*Kp_!R!~HEgIX#h;IqTqf~s3Z|}hgH`iZBc5(oH;s3|T0(Xl{X zp6aodguiO&Z)s4e0o_0_&69RWrmx%D-6p3^UtoQTZjnhvxvN2$i^n zj=ws;1Akg0mSo(6{q`XVH;enn2vkDx?wzNkg{S@gx`hpKN=;M>jQLYiDU!7A((MxH zc9GF^cs@?1nVLMQ(y~=CPjWt)O1`r69YCen9AlRuE$l5k{cbk?AkH%tJ4mv7@F|pz zW56)mVkaE)^bBXWY**L9&r@CN%J;UamMdF=dUGgyeZG;%BEtA?$4qo)_tlsFPdt?2 z52tVLRdZEh;%jfat3mBQxtkTtC{l9$A`7${w5+X*mwPZq0;NRqx5a}zUKt5gDrP+g zov+sRX5p@N zCIFn*L1w7kx?ept`?E~Jjxw-j7Lbw!@{h&7$aEE98u|ICg)~p~i?kaXTkm&(^gs@k| z7ij`6TcFzyNby6&1KRu{ojHas_9eXnP|{t&IBtUI$YNjO zUKbJsQVQS8RRj7YReNe-o0}RO4!e$yyd3 zB2mS}bJ*F%MM)Y(H92EHcOvtYp3~Q9;j$=}e-9`j0ezsC`**kdG4%4Tf{&3kA9weB z!!AH`P`ro4k|DWeEmU!?bQ6uPKAqDxcJ6s!9Qz}__kcW0?YsJ+(x0#SUZ`eB9(`*l zveGe5H5S=ayyHkG&B$r#wC7*Rtf^u^GkkV-*Z#Sq&F}Hg#bXT;Y29(>K+{HzA5!ng zX!^{IDWQ3Jnon7KCOQ0jAgN?b&C!jSFE4^KS7N-&OsNj?;50W~slx(%JmxCZLi&l{ zAH|g2BMW~!_c59xBwySlY{IH6{-=aXNd`G4ibW~ATEUOd$k9%B&K2%@oW3vJ8ahuW zHpR5Fl?n?{ZI2)x>6ye;vBXi9OhXb+zMVW0z4n)nD7^X5sQ^7uc=|LsYxzk)K;{#t z&6F)0C?iQs}xfKK=6;@Ez2!fC z#F@jKId$!|ziaKxG(o)rv>@Ce@!CUZjV5cM_s4D)1UwPdFg$hTB{y&q;HAs5BU8I=Yl}PIj7)|tesct2uO5Nx)S4~ zIQ)2T*$-#g`srh;;Z{I zjhN4iR~xknz|$L!dWSZm*l31t$92tJub8jN zNGB@GnV!SZtH@my%AqG>$TWcXTL8{*DVW&w#)kn7(j*xby^RCuWr!Vf_6!mnbbcF3 zFr;W>O8sq!2?KmWMhT?Z>TjMyvomtDb@g(e_Wo*)7w>u1qd=z8!a^+?SW}*>i+sD1 z*c(q|qQX}Se3>j)?3J@ekfcMY&|uaV0{Ay<)^~wb_?i@CuHv+~$+w{bdsWS2QQG*r zp)nj0;VMS$h8Z0FR!0!h4BJY291FX|k;#}Mb>BIs5TBGt>x;a&pMR4mq({toGOA$H z4j@M~64i757z2_`$Odc0Cx(iVomuI2A^EHtccz{uwo@S?`G#fm8kK@!pCs_L6C)d6l~?wgqA-IL+sLdOk#jy0cS!mP99t$bN5M7lO3+Bm zy6=+!E4(Zg{4+?<``(y?#6%}`-$n-)NbL?P3Pik{A^G~?DnDto=$$jYsm3CDb;%qI z_`RyVJ>qC5YMPR=?Atl^YcO@Lh-q+5Z=jdm=?cjc)e` zM2x)Q2BAF_#R?8FqJq`aP$kTCj#-HXtKk_hziB1+=D0*xeD3*U3!Xkzc}o7b+s9j68FVS#dy`V~iF_=XuQ}X*s z4If{92*YlC#4F4tzbr_S!?<8cLMy4ro*^Q2G-&ZoRlqoS9vVIUQhAy?VZgFAn=ZTw=T$MBHqjh=&d-=%%1eLLAsl@)QYx8P853>A5Z_eAkO zc;h_?8o9m3@3pOsj6)fyKpmy&yE`|}XBdfUR%{uof*`3-=^!PcsI!fQ7D*c7q!kb5 z74>ZfDBy%16WA{6OQiX%`A=HRasIlN(hev&$bI}(x*h{7fhRRGD6ODrnk{5(W+7?l z&dS~%K$ixAo5KGt&HnhIQ^sf|F4Doz9W1JLExU?@*lhaDM5~;esgvjZO zSC>aB`;60s^J`ftOWeTXB|ys9>(pK;pxd#HkD$dxB*|+rgbdn-d}e1ch)xK z+o4D0i8?A`y?O>N6I+God`cah@!hm_b*v;@cdc{-%|WCDNOQ|Lb`S*w0($R5;3@Y+ zChF4!)=$ItXo7_70_JL^guSEvg>ZkyBn=G*qfgMS>~z?o8r1R}_A*Shrv3NnR}Jy; zJ?sqd+~NLo`S3P#rE^Tpp^9OG^D(1eK|1r33~;MN$#HF7caB3m*`)}Jn>e^x=jM8^ z2c>pRu11Qx$fcfaA9G7(PnRQ2ajTEgKM zJ1KsUlwYDJ9VV~XPciCjHAWvKM!b32Uy9!7U0*=$l&oz!b&a=I^X0m(_M7?ilkzXl zP9Fb4k9%?gy^q4SG_RoJ$=kE+SA7-LP(45#>e`mzu#V- zIcH-irR1#Zc$M>&^P_jYDaFNzGDcDpc`c{6@RbL9u+OP|gwg3*{ey@n-Fxf+Af3Kv6XZx?tA;nee#v%@uvWV9B+7GDiA2aH!6qx`V`vk&Z6?>e_p*UxzFKcFq%Yg-1#Dcu0sQy&{0Fq=3T~B+o8P|r zO9ntFVBV`eF#!zk)Z!xlx7yolt|(Hl^1-J^T=INdTkXe^iXqtku5IbmQ-7Qg*Me*= zKc=dkRM-=MAKcJeHk%SyZSU;GM8H7Su!*BvPIRPGY)Ol>2c=uY5eN+N`gcw5f7VX8 zr-K9pi)m-XPZWF0;6s5(t~Kz~+MWq&w-9{S8rU{>eaP|qikDe#L73o$3!mE;!`u~V zeHI5L8TshfZiQQ?NC8#Mpu*rnlTu^5xn6z_uUEPtF^_U}zLn$okzy@BNw{fYlLD!s z1Arp{s=EPw3RN6^CS-}YYa3C`YApEn@#D32?1|6*6vINn>df#GK9c|vH3l|b=U1RA zZy;f4mc%OBJ6@`X(*iHoEB&Xa{I(1n15i@NS`N4pM{}1i%)C5IcJTPlu|#M|-|q$0B}* z1RLDHEAeD~rlp!CjUG93RJ)iB~W`_ z(-iuQhlPa&^D=2PbZ7CGPk!rHFMtC9FOh-R5V>??qA0*4W;Z~q^`Lm}{AuR|0&2Y$D=4b(^tr~ee(#!aqUg8&0V zxJIV>=V$iSyq1iFzh~HL+L|8FXmd$+sW#7F*VhZhOZZWVb}jBKWom05AY)m|LB%K8 z2I}F%4EsM1Wt*SFRPCEduEX6$xc2kxhD^Prv&!HgbCCyLQ%3}7L$lS0LL9S-E>kUY zBO&O5gd+qnV!R^$2NOH5#Mmo^aa{C;;8n>8NtQBe*v^J!{isMU5T|wU^%B7TqH9$z z%#8NzzJ8^zM-u+Wh)3yt;Du$6kf?v17-fv~GouX`5u>cLhI+^7pLOpjva)ZMwBLn! zBq#`!unYtS_9?Jn7+PyPj7}+J&XY#8i)zO0IE2jaw7^1iNqLHx4dszkE6<~v@7v^q z?qJ40m3iRCe5Q!XJNC@(Pwgtq#~Es>@cr`|z{q;4H$6q#}8Ge?aU8nFiJ#_!p_N6}j-J8r#9P^_e)qgMU>wP%0pIdd? zg-OLzDgO;EbZ*qf`>QzJi#)}`=`KQ46mgeR?u_>wP=nDEM$)Mzp8KFqfP_%7@Gb+!D5{XO zznAyv5r*)qt#6)o*gx74JPWB_X+}(U-CXQ4CJDbNY#2BisG!xG=M4Bgl&D6pJ>gbA z*XDe+BCe*X#J}7 zC2RFjH1EFNB}WOli-&?ko5NCK3XU5DQVuAT?G1poCX z<=u-6uFz(dMELm}QBz&5{A1YL$;T-|$kZ_pE9O9d@bubJTYo2yS(t-I2(z7YnY629 z#9nZkLik%b!rnvV0o@hLhDPdg9CDA${G6QvJrRZ?FELsqwFZJ{x=M((P_K}jDl>1M zR6Ul{*A;RXm^S9L(Wn?)6!S2KfkNM92=_!#*PQoD&5V#}B@~fwrRHp+Ly=k#+wH%) z17DCC!KTcJrhKJ`QJQy6{(#QY%857|CjN(s>YTFg*07D#EK$)q^6eaY*unKeroYAK z6fzX~_>xDWaZx0D3AWmrU`;*lx)%=Ck!}31bxWQmZb^f}%|kTG$M}Iwr1$t3%Yj>7 zPOO$ti_?1!->T$89C0;gQ{$_d7n2Zt&&_8`-DD_mkZLRoYC!}D$LLl=CaGCWeda3v zbr5%N$K=D0#mkE_q~q5SD(8~pQUpr9lSlW>Q;Lei(R&E&+VWQVCt^pGig=i*@)t0| z6p^9omADN)4;Uok@m3xSCjYLuopX0AW*?lbg@!xO^oVNVG>auCeqJ^m%Huc<&C9nd zwYTUGkF-UmGliJ_Q_t$;JaYeMJxhh@MLDZ!nU;*~BUjCkyhkUeNila@jsPDXSaSfG zn)8sVWV~L8#LRy~)u3+s7<}4?3=0Fb=e8Y3t-9y$-=tM0Y(51f(&4 z@2*|N1}|uJ`d=p6>Zfa(`SQ$Ji&rf33ivNxA^<0Zanh)d5)&X{<;xX)vI$r&SJYUw z&_>N1n%6u-(Pt}~(|g2v!xjvjJbfJ7@@sbplE}2mHG%H?c=G)b`n(Z#{n?-JTZS}% zxo`)Y3NNNDNYhQ=WH<=#*vX;Kkyfs3UJGLMKkM(z1p<2|n8=&8TJ-;@YVAm#9Ts3L z5z6UF!5oTH0_K!QTABi69EDSXoWbFH#Y?rxr2B2V3;iWX|D zhUhWE4&qk#<;LrA2#tFWy%g2HD*8KM#KA$oMfo+j;04MTuW)ljEwzM-dD zAdwD-oZ~!3Byb!VN|5n6J_s75IsoFbJsb~R`9eb-tjQ?lE(eL{cAunve zn1y5Ih^k1k{xE=Rp?f}gZ#Fbf z0uWZ@5R#W4?oY0|e$6NVCS)9nOi#ytnA^MZ?%q(!?Mc*Sza1E-tr z)Z-A>1ffn)?I;Zm;Y z1DI#vUSLhJ0zr*~wZPp{s*)@A5kEcHysf=qSp zm@QS-5p)O#PxFVGwwO=IO}8}i9YQmg&(B0=n`fdzMf7J#z%rcy(_&VhcZU1z&EJ9y zADQhxOglGsh}|FzgDtYg9<`g2ZXCX&qGkbw;%EU}Vt1ek`7;>bc#4NLafC$7_s_eJ zH}61#4*8^`zYu(C4W&Xr8f#L9au*eURa5)5soFE}2n;vWw7GX2wy;~LT-t?LcZRy+ z(6Uh-D0NTw?we7r5S=(<(t5K}xURnAjx0H4RhIBE>MtB-0}`{n#If!53(KRpmG;$D z{Pd_JZL;yoAx`Uu0Tie!p!)=++@SuFQD!xZ<_>GW2~kx7oeU+0FqsGsf?!8AC*=Sd ztvfYBt&9Da}7Oko@knLl;4uUE3zQggN+K^W4%SO8>k=Hl5M-C;Wm zSa(W7Ys*l6lE}%$XGTAp+_YJ=N9`Mtz0IWGdM)+NmCgNrhcsEoJ$}Nq>x7F~4ihrT z)N}PgrPblkjx~2IxoOu@Pf-EprA48rXS|9uD);;JTf{`FS5YcP0yH*3^>fm43z@fV z0-F&X)o?JW!$Yu-Sc|?t*ZpHIQxs zsw(Y2(()^3>|O(FZbL4f=O9YuJ0QS7%F8i8(#jBJ^@*^lrM7C~4^|h@X^E}83D{7? zcns!A#B$FzL&=?^n=>3hp0l5yAH1uCw)gIMI`kky$*iZt+DDWJtghqXk?y_$&zpJk z$ufegJhvA!3Q$uffL@>;UeI1;eb&MMvAf(_eT#;lCsEccshfO=ZrZ&Ie=G!@_~*qR zXrt3gvQ8@myhhZ4Z?q z>2u8Xs;C00()u>{KiF$2ouMmuinZ!9e*|SxO@4)z3g;|KuEk&H&`&I+vYb+|jpv@j zqaGhG5weZ)YCnMh_#AD;q8?j3Ve!8u5QiVOeH_ z4{rb*0MY*{XmyR0t3ibogEkZMdIMrRY~FuX(7m}iY+O4gA^hwt)wFf3hXw zaQGVxAH-+R0$t@eA>z!DS>HF26o?=^pj0C)A~k%DM*H%`m}Zka*Ne>=)QOdFiq~f= zz##(Qfh!qh>_7^n84vKwI7ac!k2MtL((U+;sgz1S(4{aYF%1+AG0@FoKZEQJ0e{N}dQ*iG^BFHtuxaY}^4x zVdDfq)Dn2bcYRBHy+{GQ5PuiUB~ClHnKV5&QU|5pPkQNb{G=qQ+ms#ahiqe7mGibf zzYEx2<3poL)-hlXdWeDB7)S!XAm9j^w+q_NRUN=W*1AebG3?$^Zz!6kpFK2pfLXMT zBB|iyNeO`UF9Th-2ZY zGFhQR@;C(r8|#us;PX@}^jiY}o?tEkZGj)XPSvRN5zI`gm?%+y9f2*+%RZv7!5p5j z=6@R`pv|3eZSL%|q0*g2NVePdS|DrsVFX73q(!Wg|9-KP_RiGhjWh!zL`jfey()0| z;3mOs?F`dxIVSQ8IA1Xw*E?{PoWMEgGrd!EBfJV>6N6cBpw7i8?MG->nmc&^>(a%552a%cbDiZ^eimWb2x0KHH> zmK3N78t5<*DFtd1rJ*++B#%v8->vlpi7_z*x~MM`Vx0f0YmXZ&_+0~Mb}g^(d5ZQC z;ew%`Bnfh>cE5ql5mocoV?`EjHl4vB-C?H_Pab>&sTA3^Za5w#%wtkz`O9DQy396- zS>JiFNR)SzTN)M8?C1n+k2pGYj8HJW+V8pEs}eQfU}9htW}zS8(#TaX={u9ca2#53 zk|96qBN=ivq#;3jORd*VFC01HX%gn4kGB%#@+^SNEqsXYzc=~`e@JqSViOe!k1K32 zV7y2Cx@?sqOeFt-MuZ@K;WoOlV|ky#Bi?2p%R*Fz&?lT@b{y-h0R3sIEvB>U*Dp(> zpxL818s0&Ak$x_WNCU?vboz7Jv!%Gk0NwPt?FjkCF)wp?J#=8>$F9()G}Rn%(d9_9 zWZKGdCTI?if1Sn(>biWqC4+Q^%(c`(%wPH+3~9eUKz7BOgxQg(88S)fQ+p^B3z@9? z6ZrI4%eBv#;1}Q)WabaK#Q(IFp}x%63O;oIG#ZCs#)YM)Xgtpb!{1q0KIf)gGI-l9 zCGyRCWM26QEWp7X4I(azuQF=M@!TV4k>kZ_zBCs5z!oa5>Y=)bny%YHBj_|IblrQG zU*NLJF2;L0;|;NW*;kb(aASWcefckt;PdfHyc)aBK={sLr*@0j5A(bVt=c-SzIaIr z%0U@x9%&c`^M>6>bUhdd(SPa<#quEuTVDgzb(-lU?x^eIGW~C*^-!bT_h%jXJ=PhDJ`i(l(@HEK5Dk z*jfGgx2&HyJ8o)>YD^5qrCCIq#nv3DsA{sj?ABb8@nN{|wK*z-+_>lo@nOlk>)~nFxU$@|cbZ)!f%5T)ct35|u|m?Bn&yuuToaS0hwkkOS?NS?01w zrWuO=CqvPpTA>EaN(t&Zyz7=XvLH*Hxg@zUfZ{a~b&SjjNE+iD$5(>?bl`y?vi5q| z?imLTBUvh)>)PBL4{E0TpRj|-@tK5qAI_0T-l!%5`|6T1JXx%;{{uK^MK8S0xlD+7 z4j=Q0j{lSK%q%;ue6>#~Z*MRvOK!MLH=#~2`?Q)_2@EGB&yE0c(gTBoy(dZ_E~Ok2 zC`FPp0bg~Br7?nc~#8FWtT5*a0h)fG%m-GsfhDHseZkXAAlO_!C zx1XY-r}Y1%ftV(-_ebsEDDWhweJt0t2#772%S%GWf?PYs$-#ucnI^>o`nX)b{lKD~ zA*@<^d5}(uj9Z-dz{We?_0>s8R+MaH6GcguHKdFVQK9%is&!D>gMcu2dD&|`9L5&= z!j$Jlk(q}*F6S{wbS6;9Qe_YZ~%M8pN6`2=$`dN!}r6^~k7cnL|F-p{7Tk;2*H zw2LQkTXDlwCSK1Tn70@F_;QX#nertrFX$I8AWU1aN#RID%LkEkwt#bwPp)`UOD9foaDk&1({O zB}9l`0!~D<@M_XiwPHoD`v>dUIcDlLiIkK-lI@#?ML?4wYOsJnfG86zuLt9!>|ivB zi!*apuVREM@&TWG5;y7K94U0jiN2StQ6e3)@_{lH7*84nPvV$Ih!iHiSBIKa_Xv-l zV!of$Em*^viqwCt&=O%AlWFp`bzrRh zg3xH$BWMv^)`~`jekeg8E93=zB1g;}T~WE$@5_vjkbOZ`0yHuOjEFIK(|x@JB>qyU zc>=^~+%1AG$Q$1+;!Hj#V9ynE2jDD77bORsfcLFZvf8L}3AR=9I~aYg5RR4kGjt=w z-W6*j1*@;7B|Y$vDfp`O2XO+3zxHGaZF~3@U3UrJCwTvkf_?D>h3?HWN$rIGyb@*D zl8T9ugx+!i5V!ms=ToyBIj`2dKgDG771Ahda0V;~GjbnMMFs!7$hQ%&u^Kg;`lKGg zsG`?#bj^ZipVV@(xDmka@d&>CT#{{O6#S>On>KXu%;rJt?vjm*03zNfc%7xRf39tj z{Z-WgMR(r{t<8iFtl@=T-M0NR(%ysb!`EI+zQYZX;ghCNE^(vgoXHG+Cf+o|9 zwUvHKCUT*D2&oJh%bZgU%>lr%wC=}@+HcDTqgbsT57miU0vmUAw@J|GBEl~|?W}oY zK$PGh-09ZcVue{~=<`dCAdt@7bA5t`yFKqBUOtlJ7F&6cqcbX?}* z;a&KQfW9V4{w>{)Dm;0nQ#3DwbwlR&FQPEsq9u975bv5({YjJz?JD^P!KgF&;|nim zS~oN4pj1sPFt742!UMJV8NA$!JiEP9!Z{q=*9MtUQ;G>*8d@o&IchqZ`>o%^8ni*e zcXZDBmbGohC%x<*RVwPiET8haabZ^EkjN!mPQ1+!!5+l{E?1S+L#a@XDjb-^!vLEL zD#`MBA3$+oc07AEit5V|WzWWpZT#8cw+%+?&%BgMA_(Abl%E>n5qda`IC#y{20Cgg zVW1{Fv8(iNB2V{M;IdiR0zoPuL{bCi8hG@9nAWwLPl^9UOUJ0$Y7CEO?U2g`9qlcf zD7T2Z*xBTJCv%#dS)(-$DF{Rt33g7t-5KJqmMg-aUGmS$9(BFZB-K3p#er42#MzG@ zd6Do?HfMkZZ@f&a;+cOw-c;jiO41*U^J)IwU9`iJbkJ$DzEF&`@Q#uQL-SU%H|xWw_N&cSzQDPzd*r8%!TW#BPklnwDJB9w~yP zA)ubyx!TJ*cem?nR85ML28i~T?aGvER`5#H7H!Gx`R8WmtWNs>ZZ>XQ?-Pi4fgk^? zWW(0!4nYw`VeJ_HTq5#ADR{f0dG;ebwo?On&p>DpN{ID*p90|Hmu-PEw)< zDqDMI_}d^(XA#=@7pm*iW7Duc3}A`eO$jZDZ1(H6(b zpHrg-9zKj5?EMihW#bF@Jyqhxc(!@1ZDr1bk<|=M3fo_BIV@74J_lB& z^B&{H?2e*aK!i_-#&rK{CXTp<9|97aN~)*<`y|$O`(B|G%G|~VjHKZ_nM#Gzz3D|* z1)Pw&(1flOECJcCEI|YE-oM=_`Hhnk1lyP4!&MH+LatR!W9eS`@AW+ofK=CDJ8}`$ z!P>fMCtR0^<=}<;f0r`z7Zjw#xNu)e9b|#GYxe$1qVSCyHuw_Fr^uW|-I4EF7OwGC za9FIrEbo6)p}4Uy6busxhgWRzrQ$(>DZO|Ou^Poh9MguKDW|bAAQ{urgKL#0qUht44S!&}A3+s&|CK%!;YD~_ zaGk0&wgc18*VBC4dd2Z4IzU+RSI_RRaU*-Ytsu##)g~l-8!nZ8+JUyK4UHygT z`$F66dvoO)@76sQfspj>sJgXf{Rhyr=n_Yq&jipmw?`8H$zYnV{V?yy_Y#C{y#IZy z*K8WE^$vh58{j>$BA2gQ;SSV*)|+Ep2f*2hltAtFDw|pDMJ62dL<8Fz{C=tYZYp z@3CnuF7a*HG6>9CH}fXQTKyO~>3*_&_v}SJ4Ud*xwEU3y*W8PyqSU@+yS*800>^6L zd_Gsh?=L8)3~vDhffG4E=Chw43zBmIp#zehPtMNHJXMDiKx~^*1eXnjJAj20{E2LN zN~8F_kY>uX_x%zxzQD?g+9~+=Zzu?OrFaBny4Zj-nvdSlydmPc5l4x>Mz`y2*jyO@C+f%byGS0XT00ZoP2MXl(ATA167hWeM zDVOpCrsE_pZa|VWU^t`Br+`tVHBfxgrUVX|a_WR31)|jBZ|LbOmdi2*=8LR=2(?GB znD-PWv8)HjVam`7fDK~M4_pXx)_Q<7jQkN;I6z=D^si%2f#Cuy_zMu-RndO?b_N(U zs^I0Tey6tFsf<_>a#F;x?Kdq zC-@#eUkQJnJ#mSP0q=_da#9{>6Pzr9`j{0JZ+=p_I?n&g28dnZsx)$E&JfIH#G zw;&SAT{yzZ2Us*eUqruw+NZ$9*8WTMcE6B$*{@lhIdNp$y=H`@24Bm8GZ(-ulZ~Dz z4c5xpLe19Su20^4y!(VmMnFK2mX`Jh+RNj8Z)_=t6d+zcFAq?fLL2-Jy(xf&)qv&E zwU?E|LN-v+>}fj->$L&J39Kz4V66$3>(JaJaP0J3`y^JRw4ihjxqLO2==u@t%&DiqYhri_+l+rlER#cDb6|E0pDf=!F2;cEPS9@%$5M#3yhHV_$uFNXG1P1 z^J1<4(^x)3fdS4{?6fp0$GEqJQsLsqk8>j5E*(ptRE}6CY_&l_AKf@ccnL@^SliD7 z`%_MFawB!tcbVBHegYNzt{HmS#gkG~=YqEc{S0d~nLbEDxbnC1h6Z6;3 znR0aQ^e%S&FL~*4$`+eCp2Og05RMdU-Hp7748454ZlhNf0alOdj{ zEd`P46t;+FIC{omGtBxuMK*JnTTIVS5jhYG@Sn^Ze2BCdsZ*Lp?B$w$I!VXv^3n$hFe?P)trBZivz;KZcrp~zY z{+my??!VBMWd~Q!U)g8Ad(Z*^+~B8yTn+ID>GXq^T=|>Y!>~vuQN+Ct=7w?T{<7(@ zI$iZU+gj1~#!1^d_#NjmFLx4Fq@CP$QX8Cmhd;Rl#Qse+66QMGsS>Ea_gNg(erDKp zs#OhS8Jmb_%KkM6bV9>2>kI!kBgU7?y~GPenLtY#^J2Z26bsjUH09Xl{5dTpdfSx9 z;4EoBs)?)e1W~x|0cOEuD? zQbZ{UedZ`!riOwoca9u1!3k3KwehT;-UVD!yL9{RwCw>RS$(u-Z4FISYNQhreJ!*r z8EgqtA0rq{{ugc;rQcPJ)j0!tIj6Siss@?Za3J0(w}J&rG*W z{oJXRKg=@-4?0}ldTFNl&n`qb+u;!xm$1IZbbqbHkOHdo@k?cHTlKW?^9S{>Czckv z-I^)ix_dBemQF3Y+YKwUoo{v7Pzqr3irVuNxLIXS2pA5!Kd{L~qfcCIIuOxd$8u`> z`|u5)!)_@Ez<{AiOv{M{KX~B$)ktEF-kO}wq#rn>mE1;ji&OM9C4t%c$~ z{o`YCb|P_>yE^75by>G9*9H`m?Kv{c7>eVV*)JGoyG^WZ(mVi{-}bEVCz=r= zq{>L^d7lfUk0T{6K65?1enxkEy(cM9iQe!(Af!O;q*1ydab{wAM-27as8(W2(I>(| z#@Nsj29>C*^h))7YgcdmV5W$cE&F1T@93=Je-!r>k?X#++;c2!Bu5weVeH0e3y22IsFQ8s3fD))9JUt9mrBlDW%b89s2RYXHRY*Xo@k4>|3 z0VUOLLzYr(W6ySMjg&Zul|E7F*f@uaJQc%CIhJ`0^tG%aJNyR=;F6GrQf&;%Xtor+ zBN)5&a(zN5lZ2n^?N5U=rRlMJ6PHo`0f0rgc~9p3{FrD%Fuk@qHe5iDbjb9mPFt2w zP_3-YW3ty+9=0uDkzB=LNyYeH2jg=@JXW|K}p z-w75lSLGGYv?E0DIhv2z7LB2D+`||0`!DC`=llD%trqQTP8(f+`@^vTrLBiZgx$q) zu-7Fh#-H`8iQJPj6FD&b_60;7o1;JpJ5u5T*nM?2* zM;6v!v(3a>!pK@=a?P+0qR&rmpH|Bhq|82%j0~<>b3|lm8dnOZcbXfnJ6{FLJzOEyreS| zo>CSA)+Eg}!t+TSGWOl~E2<^NdBIc?I`hHuZCBZwdaq|-?Ujri>*V$L7&3A^C<17Z zRV!ZMd&gbx17ZTK`|qGprOt?M?Na5q4M-6L*sRlLm@uZlUHSt1r>6x>jN2^W3;Rgm zXmhqg+S8C8i>4@R59VIHASgRTakhF{+ic>vr5MNkV8WVF0Bh%v_&h`I#hZ%Jh_4QB z=0_#r`(%zI2qInjNK8u1)7wfAuNAhvL_;$eN>~)xe6j0}{Zs#~k@Y4fnW3GS|mcWf@t=yR!ajNOH#nQjMWQY#Bz!8C`B4lKA)%vdX?4JB|BwAFV;- zCZ$3#a8ZK6H>lSM4C`w4!Q*3bi5CUbc-dqZmY#3>0=K6PMVd|BwEho)O~Uc}3v-NG zR2?_T9$wVg{M^>Ylvpps-zE5IY4=?VAEu)l|B=x&ogTE9cjc8<&YW}{3;(fa{Tz2G zj?{$N_u4dOlu~w+>?1t?hsrVzl{@2bK^mIKTi6zY5O@^x`Army7-y=ZP@@Z0gA=x*A}y_FC$ z{{Qf-Ed5@p?w@~yb7R&2HL^C~WFvb*>gT7vH(egv=U)PTRk`>DaZL_ud9U;`he%$);NKV>0P29FBROx`%+2DN~O&vOf8vJ0^Iu!#?lmrTx@t8Wn;XvB5prh715bjUzL!hkH(}YU;moVKW>SnF zY?@TPuizoFj?Fhl=#YNrMW+>oyWpfc6`{D7n$xVEo~14M3o(k6dXt4 znw~$G({iYOH@{ChMLU1&JgA7K%bz7;;9!mr9V|KVzG?q){X$;tURj%oM9A=Nph&xx zIikb$NZFy39$JF`+aC2XM-P+Fc}?If(}UA#xdBvHU|`&>M(*t#u8v_npGw5|C3N7_ zj@#e-P%@AgA-Xr7&YiJvVKctmXp;-fK+w5L)pk0Ig+ASn>`#~M`Kq=((Mi6SXWnL? zWmWx({3={ID}w7IE(hbEYBaYyo^7)UPusDeh47k)GQ^w;b6BzPcYK!LIQqiWs6UCp zy7{#1w|H9V@VAuwqk7VBg{?P-^cy0#yEN9HBTwHy&3}EwS2^uEv*@%@dbe8N{N}sB zn=1K_xq|13h4k(xPc~<#TM1n^PoF+k!P^%kS)%Dk@XMbPPwTN|Hcz#X>f3yAc2Dx8 z>YS~Pac64OAa&}Puw`D`{o<^t-@?%Pb>%~Yw!Bj61mB2<6#rOdvTU%7IkN5FyFf0T z1x5BY$oFqAeS2vQ7Y!CJ=l8DKH^>VK(SJ1DQTgb4xtzQ+*|r}YWz-`%`x%#&eUD&UF2PR&%Dek>X?c#r@`4T zGYn#7dSYfzXZAlW=b?r4?8cnN?Ovz-GCXm@`^c=`KG&yu?`;?P=1=dQBs|MeBC7M_ zn=V;DK28shONcVR^NUZvI{Ctc{XVYxcwZ1ZT{PwHzOBw|klLU}w^dE+kCB>I^ z`j#9)YmvK=Xn(*df^()VUn;ca&WN3zot!*Z%#Z9Ya;7q{)Aiu`wxpyart1aeFOaZn zWn*&&CVmCho<&9QNJx5Q(cyn*+yXCvn#jOlC=%r%Z8Dfw_KuDuAPD(%y_5WAud}!~ zSei=k>gB@q(IO%`QBra;@7rC)QX`zEYv4WtPnpoVyQ#vAjErBmLowliRtCw?K(ddE z!?Mv1E~I`HbAR{!iOu1}`UlcN1&N)QsFLTK$1nCQ`na7HbS*9hiygoHCo35B(?1C5 zfB3xa&@Dz!$(PE?u0VdabOfw_-qQs#Pd% z3vImCcN(mMni-#&W&%5yM5;eL+|tt1mzI`FHBq5dHNzTdWaq{;HFn#HaoPO7xJ}n}O2J=z)n|=l!v%@i9mgRz1?sfMP^yDUd!f}co z+A7zRZ?9|-NCKJHTVBb#Kd(9b{`7bQodf~o+b%FlL3(k9blSFwUhzH5xz193M8MFA zsXfaag@uum!z!$Wt8%^|7v+qRbJ-F6fwvboT2=}vU6C$Ko2^e*%^r>H z+rh8TuGRC!xJ4IRM&`U4rbRs53Hwyy%qGt69aI_nOgTgn6ZjAp8|VmzNcNsTVB7FT*uDLzpr9Zif${M3gIBk1?d>fV zzQPRC#;~Yn(@!KwjY~^PBEV0KYtb-?#j0SOkjok+P1KCSk{pc4cK8Wz0q;)%Kh-JlvI21L65|jjb7!%4#G^3z-knW z8wXYMG7j21WV~d_^mKGP?wJ#${-GeRP~4^qI2^&yKjC+$G#zoL!;6df1<8SPsiQAu z>d$#(N)%JlnX%`r>8hBx3VJWp0($S>hfhltT2Mcny^M(_lA^Bsc6SVS5P69v2#czb{u^F{+#Ki3xZ?70%<(0n ze8HnE#!$>n{z)~=_g zr{WlmN&-vNARKkQPZk}YiR1ldStE~v$uLDz)iI0YLGyjn#;Qar2}2g^a5yt#mHyzs z9^E6@$+oRGct4DNKqOIe>)!UZ0F6FK*djm+`MjTg^i{joSjipv$0rQWW{;jffBt~! zEZKl~0f(2qBjpr9-#}2&$)h=WyrkveX!|U-qQc?1xyNacqA~?jS+!I5`<_TgzRbHv zYnamj`C_m-e8j9WWPA6zDTB*uG>K7z#MN4*!DaJnTwGjG5PShDOg4|>QXR;9r{(5; z3#L4RORM|O?d`(i)3tJE-fWfA@6`dbS>729bZ=3)vyE=j)o4;k5SAnekU3o=2It-t z7zv^4fFQm{OG5+Pl;|HOgwtbVRivcgrnS6i=rWaBTJ$i@s>^w)`9j07Bx!WT%VoAqPvEn0)W*5-C7EFfP*s@smRop|6nl zwSotDf0yzGzmG|vDh-)^ehFq6y*AO-ip6Rma&RC+vyOj`Oz3d9f~s^~98Bx+WqR5$ z2&-WBLG5}Pg<2A6esR8ekpBfT`8#YF-|1$hu5K zu&RlY>P19E3h&+xo))QR_}>TW>gpiDgom|>4)yqey!;xdDW`g~_Cln|hOO#EiW&{F z3m&D5`V9DJc|?{xdoeg0<74(LEIz8cK=9Fx_1)}wb^iYi-+HwnLtXq%f}xlx$5qc=5dFgC-{l$6||vGJJwlQ8nB9phw7`=gXZA zG-@T{i-G)Z1){$yCP91TW;S@EkQjZZOB>knX6R*5P z%y^8azwb1#EMktXpMI*j#`NJ$+@(OnD-Nbp9d!@lUV6{DrSk>)tn){6?Q6U*x!=b4 z-CFW#BI;@=(sICM9zOnJKvbkp8k9EG3Jbjr|sUgYv>n`v>k~=$;-=2Or1f8 zFo!8Vp}R%}Wy3V%!UnZchD@1RcTy!r4`sv}_%&`UeKzfpm(H^+{jjfR@MeKdMV*yh z>`1NXIp@!}952?3HAO$x;x6g!l{j#G^0S6<%)q_XY7_Zg>_xH9?4r$XB?bkx$9S(DVMl2@U?kM)gvuvnTCL9~cNfn(6qQd@n-r(eD8B3XUG1S3WW> zrbDhE*)GXc)_wlUchPFzkUvpKkiy4lz%?SVOTTHzR=3d3%!cdYCIL?hjq>GzlZ)XSz?p@8Oq{uun&ev5Y~b z)76D$fH_^t($$LDlHb^bC+tSGmUs%wKwk8y4jcz=p_?| zlzgRLv+_Fc*NfLRFyRd|828_j#=)7v%%yewS*y#d%kCl8yh$$IVXo>q539IWeZnir zcbW+uz5jc?v!zu){px)=r8wJcgJd>_^rIIa-O$a6OK^4Vc8xgi-uyr?XsFQfQiyeG zR#t%~jnK(6CWR6zOMP#I#di&H7T>2~Ol|jhK*hV8Bk67@va~+?%u5aC3XZ`q=r0U= zZgo}6JYio_*Q=ZmYHN?7SFfZHOv512MJ)E!@K|jLjwdQSYHNREQj{9&#OmeCm+4g` zg{Xc!jkDP$`|6VPIa9|qJy#X78mE?zzLqKdcY3PzIVpV5@;8wD*ZA#^3E8)s4Ne>R z@~A9zRfhla7Lm`4d+wUMi`T@|!r}#sM);$HznFxd4<>tNY);>#J?!hVt@6>s2Q<84 zUt?rk{47p5gzw^Y78B*)&Nq!UdpnSy)(P8uvNV1`XR$wVK;K*;T?>JYinE?vTGVxo z#kT*xUtcSX77-1&zB#a$^4!RSdab|qJI&T^b}n!pl09!KM#+12uHm=FeW{rhng{#D zg)755Xly39qrjjDLKV<)E33D;A3~^)k$xOF{p#MAFJBT96JeLbr>zvMJ*sm0-PB%_ zDNr6cZ~9PBent4Su3^M+XHDlp`9@gdLu&Vx>NRe|Pg^z(uR9|8+*yR$;nHqL{xAbw zziRda>H6o6aH~*xhbKHI(={BD60f|sO}%~Fl!vtGiR=~^7h%Ud2^dJT*KeOc8@+e$ zC>5`>OwyY_@u@@Q6W;0?RJ<84+=HA|sS>)r6{qp>I~VVJ(K}?qFOp(n?iVCu_HD-f zbFT|yQ!C9F2zBU>?J@lD^vZv{gwhPv5p2RZVaNl0F=wn+9v@pZSuOv;&~yQ3SBmqO zH|dW^-A)gxviNV(r*pV4C)f26&y%tAr?*;NsA<03$}D=e&gbEs7??cs$M=J=5P6(I z=SvCUhmkwP`KisG4Vy84W=?GnPfM8JD+|CK};lnw~F|PUS zgEP)1HYa2Hj5RpU-;=aRNXoG7xt_32aVy;VqHE*x=gjxV9tSIBL_M#r)(Fe;N^**b z>0q$Anf&0ybGsCa34M*o1`%1_iZLD!1sj*8DW5%AzqdON7@jy&yvv)4t5zOO<yG$Oh8(W)PhPvmd=-jUqwwI$6uAy>H3PA{PCD3|!3E&u$fGx?AAS`>Xg5xv$o z{op}CNP#iAljfJVYv=7W^VV|S4w%YN@p|Yk&#Z56c#{xf?_sI;r{PP9>+UUMRU8}~ zEE2X)-1>8xFoHBbV8?Yxe+GhD6gr^+Y{aG8q)|g*=Azl>J4h?1hFB+AQ&Tb@@h~`e%0p|-oKGm8v;SG_usmzhDT=(KA7%P4y!GX3dx;$)#Wlb zejr3skcn&mja}Ymx^dR&`*iJ+*trsB^z&YgnCvbb68@=b{>;bl@dc_!G|W^g+l#_~ znK~>*7JL*7I`%sh8pSPrR4l0{hkxLKj$qP@SS@Ro)1u;U3gFemlK;#UK6LBdl?AE! znUF-Qj1lXbb}ex(uUS}3hDNTt9Est}OB%^GB>O@*d98g91T3ZV>*bota_S!yzI82n zVOMuQ`UM@UMug&(d)J@S>^#Ntr6f_&txRV}obl(5oYq#6$`{s&N&4||(auSM{EUn= zNuOIIGh0Tzcr&89yL}eh%%5F*nc8)I=I7I{Rp+33rHr%u?+O=RGgC^$_`dD_1-|2( z*RL-uOtkC!$C_(vYaf4!(PSC7Zh48&@`UxZhHqC@QMtRr$h@km>W1dMf2{aL4Bx6{x7F+_u~m7sq+{Cow=ddhM(#xE-1P1g0u36E&A`*-3}wZD&DwJhyD z5&6QUr{`72k-5Z7DGiMrN7tm@ zZ|@d)yM8B$BVA19R2E&T(zNHt9#_i50`WfM231U3eB?9P{q(8Yp>>JjXP7$v?`7wv zUdpi1(o%#%BeRFAT&>0%3Gm@f$K_s&2gjb(Qx^5bwjU1v(ixWjfhU3rJ%AY6lR8R8 z{O9OR+BuvLam+GQNYGS%U0f8sNk*QqW9UvR=bAv-rdb&Co?ARIN&D_mlbeXBxK^e! zi#>nNz!>n@juI_OY)W6Vvap>0+z!{ddF1!PC!9I2)nR5aC(06DNuyirCwv())s+_- zN2h(&tIm~1g)#~0J+y05Jb17&=$~fFE~3(vFBjWd;i%FjYlz`0(7w#w+~l-`i^;-q?a18 z97AZaWOrpXd|E|;T?ZT^&Tl|hVp#U)7!`b*Z`s&JtT?@<=7O7>+r^8Aarxl&Z{x-! zifhx1%Im`=HtRrxoLyWl1_US#?RH>4Li)XZ`}WXDGdlAAucw!%`?vD?ejVJmG8SN1 z`dU_2*4)H|-i9g*Q##`-Dk{(@?eIJ!LX#b|v{m1qP|J6^R+^faTt;0z+J~$xfBSu9 zY(*KT9^~NQKu%5`v?f6 zNIRf=Q>MQJcI1aF_9KQ>!WJLr;R(i_VG39A{?N~=vNYYFTUZ!9Ez;cf`|=-veHZ3O zXYTHJj3FI-e0&7X2T7Bg*x)`Pn6!(T`Cfkhg^kqah-NF|p>BP50lIC1xeOsJx6j=@ zz6I`QiCxosY$@HLoVhT}WDGNCy0XdXdu6ASB->NR1xS7^P7xKv-!m0=Vr`ru9iLiw zu71KJ$G6aI-yX31)eop7<{%!PX1vt!9%$ETbB7D6*dhdV=o##8Z$FNb;cK+)uTm;o z&|#0RWzq+CiB1gc07dZ&lhd#OT(2h( zXcQ;3PJY5w2v+r~8%3|OtZ5Mi7Tn7DlG@tZf`YC;wdSR^agk=&m(_)iJ;p{xKaLht zmW_BSZ7vbrxahbQrl+P>KxcGzbzNIs{pg256ePKAX+1{+*Q#E)@+W9Qa<1AB($qf2 zCJ^Ke`%dZ9o78;&n1nIez`LPp8f*%8{R!q&!R~FH@4KvRWRwEh$41zUU zX=+l9C_|dxk-=neb!x-J6t)=yr`?R}z>xJ>JV8NxQr=R=soHwb< zKXZU9Q{iGoVPPTe8xC8IFBTUB0Lg!156u5?wC1k-bF=*j0j%v?Qy=((5Ev+yW2XnnYkWn;{rj-5f{b;?P zXspI;cY$@s*y9U3DBAL%&~2ai&%D~Z9tt4=@9kJ9MfnJzt4(y`z)_>A7R|{u2w}4! z3=1}qz+xcSZH?!)lg{SfyLWFaXt#w}YT)`WKY1(UnE!*Ix>v8k^sJ-mxn6NiJ(B3X z*;snNDMKGacmLNH;Qll2d*ar$!Sp|umt87H@C`5iS)Iz)NpE$nJW{hgmg(OcukGA( zR+8M##gGNmzIJJMaS<$Q1-0k6FBt4($b(p6JG#$NM3#$d%S`fCIqP!N(}tX zXyZJ(Fo!o#lj1h@&XR26c8!-OyLScLs0d2a$nBq5!f>1OQ1eNGx)nQ*A3l?3@){(?JV+dL1|1iJq{8yMKo?zphBl0PuBxla5Wu1nA4 z+dV7aF^m4ghp~4Zz`vReJDq>R=zwUsy`ScSq&sVmoM}wdLLN zzHazs+V>JhKYTmRX?5@;?Ao|8?!VH7c~00C9>7TTp`SY9&CQ4(I2om{OYK$`7T6EE zK8LCW{S%t@Z`G;Cf$NV(JWEPTWotUFJUJ6x4@OF^(8BWaH|TES>E-(*CHs1Nd%17U z9lxm~$gNZ#0FXI&>yAB&8+eeK`Nyl{bjQXEICk%bL5zus*~zW&rmt`O`*$VBr&I}= z($1YX{%o$FEH*0|y7c>p!}`{kcE`#Qe0)Dk%;ss)qM^5M-n_X~R$gBI%~Y6~X1RCH zN!wYfPJcIY183Kl&|aPQtCP0`A&YnK-sS1|cKSnR3l35j>4QTVFI+2uFe4Q-Oo{)l z`TP9JXAt8tDK;)9<}tutU_$uF3s;YTR*b)XMf@184*;A=>T;G!*U2`}6krTjDOZ{v zdn_BSf^LteHa0i+1-S;59~BrePTpNidBK=?H4j_04?Q+DOG{NXbjZDW?9`s0<;V9t z9`1lj|Hl2_rkSyEQoG~n-5GPIuDDrD8(z6`C7uouo-^Ra$;ohsn?vqghdkfFSarni z##tx@`0#x=o@bP~XnD2d|L*A5p(6bNS()G3UJ-XGJ>1;Ndc8)P9tHNDe^j(wgfF6W zHn4B)&7WoJ*Et5l&`JV~U4N>-g${S%cQ{BL9UWJ0=m@TS9qQqZ;#Lt%`u1PBQd!fp zckkA{b;WR>>QOR?%LE(Vd-H%3O0>9}8Xsq1E;9~v_aRzd6C0bOFf=(H3&N7_leffJ zda!}98lP1E`r%r7W3uT|ac24J z5J#+_pr9*E3eiCbTN5QtGrkjkTO4_RH-r6PclJxDps$U9`mMxR1d|TK_dxQq5hffD zN4`;@USoxb_Q|)i+|;>W+PEenCwFt&CwMEN&dPbL4wFgSHTUBHFwGzk%JS=1_kh~~ zG6=vC3T(9Q9MYW9UgOF+9=-uYyF?(p1_pg-#OWr@(LYJ*AdIim4okSp# zfOr5#2s_&dFhebE>+NmCM#*@MH*aSZN5tfR@gjpw%v7-3vPqTIH|UR*Z|_(+IXAu* zmA*_r*zulao7`Y~4C@E9Z^B#QDTlT@GT6|0aT7Oa7@9XDBcs&jF$+DtvhV(^wyX#C zbwf-yUwpw}S8h44anKVu6#Rb_l{VJnt~NI-=gfW;haXgnSyhI)8pSq{>y8NQz45=j z$8n^ky~g`^tv){R8;qV-vF?Y-_3`mRZ_6We5lrmtb3;?Gg!yh#V1c4%5ZXY#BDq7dw6UR zjXN2cn7W*Q!n?fdq<(>-pT_?oye5b|lMT5F6>e$KlP8y9E#dAS#`J1>6_};m-NlX{ zz2s_l@?(H3)Bqv@Lc7Dkm7q0daQpjb6O$x^p2hF^+kbpd%QKlVPWY+iz|$K4{t3t_ zs?D#c@Imqirrc|AT1IAO`O=e@ zJLTtrM`CRszy-)rkNJbQFxt@ZaO3e^P}G==cfxnqG*K)9vuIx_$q8Bj1HXo}v^1eQ zkXz%Go9{g%l}DCY=yT$x;fsviRi%=8BR*0@6rl9JYAX z1_zVY;~tSDewmt$6_@*==Z=;ZO-#G|<=;Qh;}eJwWnHudB;tB>bcKZ&9HARU%0=_b ztW2b^V3)bt%I`h{foDDbb92+w*FRNm(caz;TK*aM>J}Dk{px-faHTS_3Z#*1VSx~R z(Bo$E!X*uETmJrGPtLB!`rZdaP(wt~7RaCia&rDqI%Ss8qHgZ)s6vT{bXVdkf*ML( z!X8XV4sJu7Fki1)1(L(_?(W1cXU{`Nl$7jv=;>`3l&P|@v-_=Uf4r^sfLh!^P0b78 zj8<5c5F}Zv!By})bdpFRhWpd?#+jKFnTn@Xj(DbMrDS9TeC}lEG!>sfKx=4d$V`)W z_b$n2mp>y*94gb8wRmLn_|6!0J|362INr2M@{+;-KLdk_hh+L4i_uHrl(4^evEQHXbcJ>3(VsLn z8f>9VrPXqcEXH+GS-lAl7LLxsAwGJ#v)l8KcHS$Df}z;9?Kb)}A@LP#Jt+^bgH_A| zS{xY6XOTVzIA*~Im%hDwjn=fY?#x+os&Px*-~Be6FKPtH&J0*6aK|4%H+Pov{?9p+ zx?y`^rO5CT3rDLv)QVD5Q%6TfF~N^=+cvjqEA$c`f_%y-DmIpv8|E275l7!3;k6O+1bRc6ddKZ+Wgrg0Zm*|T&%0DopIzs3FU?w z_ZOT8Mjx|DIY$!3HOqT(o`=T&I+Jc+)yS>WtE!kbW#@}yRlirk<1CUE<1a0ds?0~@ z)GutyOrihKb1aH(Kp^J#v#<{`x@%jF4H|dwRWM(lODmpUq_vXrnsop!^hn3=vzlBQ(y9lUZ z5>4@rm5}L^|2aj`%x>*I7~RSsr0ws#&SZ5{+O(O^ST50XziWlHS&Z1%&$O}RykSf1 zj^)?*Uq3Jv(S4-6M?&w>P9;k>vN{cPJs<3%jg>Y%qY}ASE9J)xb-T0nCqDZe=x9-D zMLaaR#8M3O9@lLLm;4e&Q=_;DUhvTrt-bZaKZG!f3NurQJ_vw4H_)!AcIJ+>G7s>L zW_EZVw`+$BnS=o{h0r@vsVPLivM=LN>)&T8e@*#iE>9S}(DmcmL+;DM*Hsw=gx-Bn zcajv={7;J4_rJ^0{PYVKnV%*ta-BR2*A9M#Q|U*NeCAnPNg|SlCAzPZuHt+6E*E?t z?NqUXhY!)^o5{1EN}dwN?B%#ou31pF-{O@ab{595Z3p2gO6uei=RTp+~^ZxJ7z`7THr zWsTK1$Pue%epO76Dd*^mz^l%wa;hoMJpwxKrT)lcsngr<%5yK){Pa?MK;_mt?D>fk zUFcf2n}b6{RP^|DwqsazV3`}&IQcLfAX-I5Me(x!Sap8g;qx1lVk}!77NzP|7?O3Cq4@0kC+w`OD~KrjW|lD4T)D)@l6VRg@pYB64a8W4q^B=SWt#n! zHuh5}4|D2wzURJUh0a2ZikBninq=|IVET&b<(-rgLf$ckMQD?LtL>vb+xdy7QLyj< zw(I`A#Fw16SD`1JfWC@s=36rEf!QuW(kaXT)nd!12HNB6Rss9>6A&tcB1 z0$K4rUTHDMi$EB7*4_@_L?2-332ST4{>k8u5cSz+2ExqK!u35r@nBcDP*-QFv$N9t z`o?eCE0;B9jqA+Bn*Lhwm{Q*>9R9Cd%_mm#&(wv3+kX1Ag*q*)h_@NfCjH$|iPOBr zt3u^cME~CUBKtZNZ<@G`{OMN$4Yqp-Pv-D=xqZL97d`9dV`|dQPosvSy;HYx%~-{) z;mkk72`?@#X4izogknA3_Qk{{$HEi*N58$ZTfv|4F}q0Q|THj`ka4j zJSD)U;~y7SD_vi>2RyZxtWso&QqwE#Y*<*`Ad-)W2aAtaSru=#z&d0y6$4Mq$;kmi zv;xFy2ZCd2^KCX&2V;|XrRA@?_U!pSUG$;u+$p$m*oQmVSvdZ*b=~_hi2O8WNsF3247Mh}O$~XZOTKYK zIa}e0haJKxyq~I|qjbpOsgePI+j}?(D|xL z>5yWYAEm~(er8KC4lvJXpSVzmq$i>w*UDA&*(lpn0FVfSa+w$z5$D3Uw5NN$e{g8% z#mxh`j~OJtbY7BlSTZ=Pm8N(lyt)SA{qGo6}ZKP z;$vg0t%`BCaWi0H5fRW(Qy&ia^-wP#w@^_$jTATh*7;bT(NQ>Qh*0YH5urt@ra*n* z`uSPo|Ni@rt@>;9qk$mAH}EjqCaZkvHaZjdi!-I94j#n4S;m~jn!-Z&-#ZkV!S{{psC{P*8?Ao3q8{&Xqs z`rX$8Xk?%9ec#Q`&C~?l*10PYnp@2F$<1ji65*YEB6dCSjO6af7;pr*cnPqghu;2O zLGvR(&G647UUOU==lLbL9c%6G?hX;$-mFf*dhB0bX2$$^qDSR>Ytv~CXwmC)?vIVH zp2vP59tKwU?R-613F(#?>GihQ@1*7=z5K@)7w>N#d-qGz#l?k+7dXJBW%qRk763x! z^eVX;u_2GwP7WcU9~`u<^O=PD+p%K@&cDC(m;Oz?{9Xiph|09`-Zs761D^`eA0oG) zpubH|Mstm?;=R}|Xg37dI41wGD^yUj$-(zgUS_r1*3f){W(5-C?7pNYN@TE}rzMgJS^by?Sc@V<@4uwM*;l!pvIZ zfdCvX{~9^9b?erM9lKpW2<+y)XCiVkpKKp|4Ky1M%=n`JlVQ;i+qKoz)e++a17V#LCq6Ft$L|hdj8gkXceU!7 z6m1*8VZ7j zqc;w)UFSZURb~D01@Qe*euFB&f5X?Tq@bGYVh@Dx6j$ptIEQ;%|%SNrR{T=qti>t-Q zedRNa3ZD1$Bx&U-aQ9}FkNUd0<0SxrML-bWzh9k-oRTtrx2!lnzZ1|Xo0;UzF#Wck z9#J0R=dPb`Ve#9yZxJ~n%oky%S?5**ta2H{Cen3Wfn^@C+39r)x)1{J^y$;)7s1Ug z2=B)w5Z;5CR%iYG)hj;6sNqsO3C1YSdim&%Pna$rMt{u#OD-^hApKmuTwsC+b6v3E zM1~8Zg1lfucD4id6AIa`%B>%W?%K+CeNP7+!GrR|&l*V9h~`d#;fiDhe$Jjfi>=J~ zuMxhKENnZznd9uXZ*R*SLRGRYs@->u(o$KoL$e?m0y~zMM^r2Y;*3!CqMf6?wbQ*?P(VOAiF{lfH|B?AwH07b z0Q2X+zt{DD;2xuqTfM`Afq|i<&*ub`Gk|*{Ogijy&$d(v_d0Q}Hc;w*KflB~cRoC{ ztc;F{@o;w!Q_0p5bdpMkA5$0Tz~ZG?IU<&ziA=&nqk{Sh#hs*lLa4S1CY>IBXo)7M zu3B2rW<_TnV@_!mg_x336Osd>DOx!qUA?_~fZ`ItYGR@fJOf}=QKhs zpC~e34A1!L)#h|L-$IZ~v^M#({(TSnWMitcZ>zdZOV#NCppBV{i530}6P=x!<~O64 zJufv}fs+W)G7)C}7r}FqvdHk2ZWtBZmhmvNwY7D37wNB7kYS0*8*&dRSh4yFURFRB@Y$#KIkV+4%i=8~}+ z4b3PrqkUu5UqotN(rD$q>YovT;*poX^zCgm&|}>T@0V_<@nL_ULDBbG;?SYgX&(gx zL_N@?t}kz=Cnl2ZcouvF0BF5Trd|ywn-_Xb&uisX9cUO^5kwAcXV}`9jVMeRXTCuJ`NpM2jI`PTg6*z zfYbTf!LGm^nXhg9>W#Z#E&idxWqVxA!zNT{;M2eeY1t%6rROfua>PEm@ZFZSEp`7% z4d4Cq#d&!!T8I?)+g)C07cG5)pp@iGjUCe+ku9GwDQT5vPn6`?E2bJBU~1DbwFnk| zq`567heK`w!Vu8k?Y?8QZ$0RbThF_T+}NubH+7JEQ8&q|)sd>A{H3YOtD<{kahj*6N8f!j zGB#e|sI>w_gc(RWb75)X{MTEeUMrTf%7^Ze*i|WiMbBDs$_yY~h>3_znf(^=Q?q9g zKLBn4wp{r3=k;<8A=V$OKz78aK2b|m{VJhiX)tP?+71_jM@%QDqF2k% z&4&HvMXH$gdZZm$^^L5oqI<-n1v7z}{mA+8l)cbrUC6#wa($}~fBaOX|Ac4p7TzTB zlf*L|8SY4E4YSd!ur!FMK3&L(G)WzVQk-x9B$ z+b0)t-G>Pu*AnTHbL$g~QJbJGCZFo9+pl=X&dTcR4dVbl+BPI5H1Y=)fBvjLmmxi# zKzvWCwz8H`$_onu(#I;SPL_0({dB%2Xe%&eHRo~VkF=x9u?>@0E0ic4Z1pWD(4yrH z+xIE&rbQlHAdc!}rxtJ9uZk=&L(7ImND3Sk7Je|>-sjJFvCv<#Lh|>7BnLp3xSdS* zQu)K^wRz}r7w-M^S8Tf%$C`I*!jz+0t2ost*)i%>urQ~0Mc9gca92)3bmD;otzsjz z=-AiesFHlBDRrMhlyzn0S0>Qqn0l<(^S?a5n@jJ%pQDLwSEdN1=5=n> zVF}ec1%-W0&hpie%3Fs$XTDBo{emNoeEvtHo#7HJ<&v?5oV;N=?kQioX9Dh|9m~y* zRHqFSHNv1*QXp}52%lSv;%{oG?Yo*XJvZL};!yhm&w zf+g!*;bkH1e9>QhcX$ZB`@xr|jxk@0(Gf)M0CY_t^^7)=+C-Kb;GY2F2}ei1{%X=V z%CeL%g!Npu+`V#TNJ(MLxnFRxUA{n1)P?t6PhVlmuTPv&R=uwaF%!@8gvBE((-Ovy zPp+}1vFi&aSsELMYJ;ttrPi8oO0?Q2>l)w7RL8e-w0VM-ceCFl?NXuQ^|@(Mzs?>mx_+Iu)3o^$yxy_F@s`kqw>zGp!{ZKzg~YOm zrenGN^!G(OAnH1$kDGoT7fN&eaB`1YwgG);dvl;)HLTa#i$LV*!5k8Yqy<}fxs>OEY8j|4^4m^9U@_(uw~HaG8%*p5FOyD$bB5`t4;i43=XUH!q%eGG~9z1yP$og48 z+>XgYF=}L6lyn60HD%oQQSv6~rSQIHdq`2NplW$GC(6jUt<0G&lu0l(!$Qot&$+W& zkRl={@y!vb$clXh-Jd1H=%w5$4<_a<_+$@H^whe@o$K1srDgQ$7 zQfg726Ek(XoiS!LSB-JK{>n8?J}Q0#M{v z(_g)z41zSdZmCVHUz{ID_IsvkMp-U>^^|dyNfDj2+A5VST1fzvf+zE8csP`7pNqs# zE&QJF3!_+qaAH$T>Mz`kwEjsgd zg5My(IOF~99rd;lg1l^9S26b8k?q+(&)Mtr*k>aFM(S%0R>hBI^X1_Nf_j2rpP>0% z*z}YuOFk9(nKn<$(eAeEfd>K(j5>u9X%&&8g;iC)tzpzaqc=O0{w&nKhGqLoWaALe zaG7fW6}SCvwQhAtR#s^5-jtM-$*HNGdlUigSob3#xsy|_Z@Gafn4lRsA+Yw9-z=B# za+M*~(y_#(z@tR)pR>J%{h9*;*%ptTcfjs|KLo2)+xgv7C;j#@hNzRQo5pMk+&c_~ z?MiKj7vpvbb{O4@ru zjWc3`NDiRL-0ACJ0Ss-+?CngAY=avu!PSbg4-O^^8Y;Y(wK)GKd^;<$l#>2G?AiSM zJCUG6R<7vvEKt?|f#xK+UEpwz0e%PN2mDF|unJj4qi(~y00xTe+t=bEQdSI?PfAem z3#QKe4*L83uXmmGW&J-UY>A$Ewj8~$``DPB%WsO&9=m3yWl*sOk{56?d1Tu$uQK%p znY9^+K*;3(hZc!v-aR5R(>+_7M_?b%>hdKZ5QBbBOk}*EZw_w}nC`H9b?sjsKp)70 z>*(px?cABxGf8J7xnypULt7nL+f!o&z6P=;m__NJ{Urj38Pb)!VeoH;n}UMy@L&?5 zmX7?3Yx(*3yw31__0-D41|H!vuLe`Cv!_Q}N9Qu2H(kN}dEQUxr3uN>)6!b|(YDi6 zWLN)WZ7S>&_;fVk6_^gvX?ph^oMIBl0>6Co%!*J~cMwIGM?Hqc@{DGtuhtO#ZYnB% zqP1m9OAB1^&=uBzM1cM0>P9CfCV-7>Oey`n`hNBdh=d(apMsW$uy%EAjc@?NU`I(R zRQpo8larGP+9ATVv(f^&12Y=&+?nv5jEwjg4?E>}0(3(HH(4tWgvCFACPYO(hMN|Sh1K|otB1sM3qWS2r-4tNXqaMsBYe8ZEbg8Ls#>93Fl=<)D5S2eY#2wZ=>rKY@KReXq# z&k>9-9Cd2Xr1aai$E@@Sz6crYJ z1?C+7M<;#_p(yaWb!4rs;P-}0Eyac;pm^g9xJQT{hKA5X^xY6y>QNJQHMKV@%DyiJ z#+hh2q=ka?rg#W6yKH3cfBucoDY5^Bfrm0OZ{7s!>>sKE zwo|%{uK~C!eEj?%bCy`w{RHEIpdvqwWB{@VHtGn#SEz|d@iH#3JPDY$>CdtRcRVW_ zTP^s#K)LNOvK8pgMMw-NX8q*#CAi!$Clg9wQ6L^g*hqR!wm3oBAXmK8JGFR#7Rm&L z(nbZz9RwJgZ7^NyO8Nf)W!VNd6B7YUS)aQ3wF|sLqD}em$^Sr1OifGs3l27sj|GDq zxI+^f8xXTq(bV73z<}OxlAZP%BnnBEY4qAeJhca5Ei%gk0|P=jH^--@j+>ay?^RL? zLLj{-aH$_Dd}4WN1X4=pP;T*Si~>7TG^CX$B_oppnASo}@y>rieKWg%>Ix2^Rl2V{ z*EI+EOueEZ5Qn4GasUrKHq5zX0@UuCXxp4Vz|Td7$Ligd{Q*BzsAhcYhwodW zh3@;m4a?&IcE7EP?cLkn^@8~}KWHV=(!MC@pvne>j0O%B_4QUD{qNef36X*xIRaXl zP%$nn=nomNQv|nfsc)-Px@YCcLo?9x0+y~0xf95+pWia^2jI3sgxv5|f6?MG%O3!) z0V-4rVmu4hD4I=RyJ`eoNzJMRIC?^wLsx*Yy5YEf(U2pXtKD^IPXG_>*8`4FI+Y*$009rTpjtpTd3k(bO#h+rk0ks)4ls5vC7`Ly|QI zFrgo(q|)bJbVrFlb;9Loedpl22e~ebi13RJm+0-Hsa@ z8#_8qUKMZP{sGZowjM?GF$N`pySe8~K_4aV=DvZ|w5A zfS4c)1f&-GH;`8Nxq!_{HCvEx+Q_?qpV1~Hg5TZv>wIow;}yU@P^HAjDti^R5%bhC zAu~f-*%}%`=LdKVz^$x|%$c$Je?w|!Ku@(~XB6d=2HWA$?5EM<$RdCBuzGCds~Lb9?* zIDj8S@-n9{4(^DKjotgX9(rS8VWBT#EE{cEt%|R`Blj{nd2%-^E1@00qEaa#yDK_s zAIw2o$r+fj4!YbrhO{7{dU~7isRzZyn=roY^z>!G%{omGT|`VV%S~9gXjcik5@I!kX~PGzAEsM-|qIqz@*$vlN33IVtEH zydoG*44Nbi=VuX;7g6J_0BfZzsy#KnxLP(^IIwi70lCcuf1kS1ckeQ=H9v?h9nG;v zO;vWFf_Nb*f1!@9#C|tph zN&1Rct;vJ=iPgxW=#`)ul9S?14GjnS!K7ghZ>v7Vb>y+SW9aNlig@gvw_ZNsodkU5qd)2($b3 z63Ha4p&MiOj9oNa-I#5!{mK5+?HRkp_*BsnON2FB5H6{gBJqW{wn;{sy}V~u^j%$a z`~8Ft@P)2wim}9~^Ifs9xT!2=>Xdm@CRRu<^(T9fylcNpjJnU$se=|FlE#I~6BG+H zcyU-TQDK`sFf;y3`BZ()^|8BAtdT?4ovY=KJoHl>&H>YcgEq=S&gS`D#+YQzYO~gC zo-qG~Zj->9!dYe|%(Zizk~qsqvRtg^6Erq`@on0+mj!u=J1DMNr*IA*`eY!N<9{IE zqaZ@#bfBA&cHX2zSJ#taRpu)d$$4@5!}ZghyYK`m;x=#s`p3p>s$>?A+$?UK5V{e` zeZr8r(WF}yxewdDlLv)43#^j3B*x30iZDqPeEK#J!KWyplhZoiL=C=FYwK?UM$-%Q zdUNlkSw^echLTMfFZ}{-A$pUz76FY{MRkTUam-f(SCp}N=N}*YZ|^yO&hAfkpmpUr z(n(EMv1FNK@!sVP6JtqLD=;iFad%Nr->W;s7j`30!BW8Jpp>)~I-D8h(ae`tS0sD% zFvKkW;9(1G%eL&UE_uYd^06%fS|SkQM^6QQPViR6J0A#OeAn3aJH4mUSF}ZA#k!di zhXSSJG?||r6FwP7uQK5rwJ90eqAZxy5<;J6T`+j=p4K1!t@aeOl{_4u5B z;GHlPFX(mrCp~idgf&Ym`RG)r94<8!Y2p==I+Rvv zxwZ!`g|_ePXZ#YMba?AyEoV`A0Xv&FUv|7xJ@F`<{rG&USLtr+2L25GolA)kqh+V{ z^1nP6IpNVKR}tkaUn{>eejn{r6raCn#G_y9lqalqCw`GVv3})GIfLBom03=mi<6sI z8a{l#EVz~Dc6!r7_+?tDbeV&Ezfr~&wEpe`avHF-D?MD>Lv9cE1@nZ6q2@qJPR?Rp zSno(eEFak?$An(7(B{(qnhUexQT|zOhYw&5a&VZr{(0A>m+x)!2S$uSbro$CIuzua z6+K%1vtUOMJme-VT&i-TnuMwG6|}<9A{VIgGf_1uS{l%t^SAZ5?|1u4TmHKqdwr6+ zJu27VG*Lt`c{JHTn3?Q$p9}@*M(^jt^gmxWyYc7_*sXS(&o{20O*ZH4vR-6H0RuI~Q=DS@5TLHXyzw$d22>+jKS72NAsu9Ffi(Czi2V_Rqo++S4L ztfD|n+yi~!IYYbOFgO=Tr9Wpy`+uL~-f#Db_ucAp4r>)LjWHfFT9Z`F{&v55tV3yU ze*pib?}Pf^QVWi>sfXUQO4GF;rQbsp|LlT+DcSSIKN(!Zf9Pz}?q(=?h+imhi-w z%w!3gZO8P&E}?yT0bc?Y#}2WyiD*aB>!s84GWRpTEiK)3di&vh&wt;mlS_MvzrCMv zmS__n_fuq>r3cY6UE4*PqrvvSx?E*5M!U&-kBe?UJ1@Uu^{IVKzuhM;g|nYBOn+gR z;wT0Qphq1V%=@LQDAPNexhc~hOQrF%Oz9~UAFxtCKOp1e%a^p}N=#7Meb0ki?JiEH z%$7Da-ZVe+rYzgqfrm8Prnzl0T+rgQ&rvzyz@j+8!cw61yNr6>=Lx0TSUG(2@EvvRH&{o_!lO+Mg$jHU7e%5&dGajW4W@0Q*7EM!&J=NO0A;ffYh~pDY0iwzTcXdsE z`DY>O2mikHC>;ZAg3`t&IKn;=p*)?AcQF)|WhYJ_B{A#WT&)#Eq>4eZ^N2pFcA*^TF%FC-O&*yq>X@1=>CACQZyRS!ieNKa@=6Bl5p# zoNvIluH`l+EG97;iI}+j{}rwN+czJTYT|k6I$xqiON1{lcdng-Ux%0afqDQah3^>s zzpvtQjW~;YngSsjTYJuq6<_-c7~|{mn!RK#aAI78C;xVCgV6Cjv+t0fu2Z!j<@Wyu zjG&0@3xK!Y+g*Rk?Y-{Q@Tz}27mYN2!ch-W=WA{#0!Z*!cPwH zbaF7qYJ9K#6qk30649FoC*6M#``>a&T{sB%Gd7t<2V@ONV zosG}3O{*^;`gZdcxYCxw)mEn8zI`KN$fbC!fZlw4Ahf0%Tv(hUHokn;4-n=F& zef=1J_h%~T&s4^TjV}^Gt35-x<#zTS{`5tp)BSBz&|geP1&4=d0sFtd z*@eG3>E!eYj!}Hjm*)q2)xynY;|zoqrf#3><{~E}go-XZ3=nlZ2w4|Ky*t3Uv7WcM zaiZ%-aAmcj**lKU-CW%q?bj%8o7tQxJ{`O{x1Ez|>5cAk^}VA{tje98RI}p7x~(!J zEQ&JO~mdjJ#m44(PlXeacV%bdqm17T~AJ!;Y%o3GLueKAdB<7Zlv z*afl91CAGyELDGWb3NN=S9mK=de?rfXn!{2}`+c;wHKUu)6W28BuWb1O}zCavN9*ZALvbkgXlw{MEZQ!uas;TY0fp8$n+OgGMsjn2cUIACfJ~ z1FPozdMjDxw|7I?^CgWBlnb6!(k3|`6OgU;lkO6U7V?%~%L)EOo?1L;afX9zYiHcs z9cLW!4)zJ&S=eV4d=aa4%aTtmHctrd&2Cg%11T2q)v_u zU8&Ufs`1CuZ!5c29Lbo(L;%i%pMAGYC|DcVZ|Ad#s6KXk_wpOgdCB_%3ns$dpXH8K zKl@TqaM7@PNVVOV?6_#CXw&`Be6EtC`%dH*)Q#Ltb!;w2v%vNP<2UV_KP?;x8yV(v z`Xx73a-&MY>lvFMMHd;2^ z>Dp~NJ7%wPQmC~UmJj&;s?UbEm&Y|r+x2Pef*yE<*9W6flWEaPNyVkEbW=(y_r z{KmjMRoT5ZksZtHDrAenxxD7`H$OiZ_((V9bs%m_$34B__g3Lo{`TY+6zoI&_u@rq zj#@o+zJzK^<-mH7O`*x=9h^AZPF5LJ<(5OVYVp zJo53r%USLCVv{j@{k}2IIgGs>dov#J;pUFD=JlI%t{q;GEsyaDx@p)s>1}d;UZF!~UoYh` zK#=JxZdl{}iZN;(N%>L_DTBO))LFGq)g{>h(~(M+sh%AJ4sxr09k`l9C*OvT0A;k3 zAgA>PhvTQ#q{~pt8S)__%e30AQsjL7p`hvWGN$ZLjvM4NhPMydMq4C&t?bt!%bPw! z56RN9bNj{XO$gso>eCue6BI-Uj}X$-BQ7fBGx@8I@X7Ym6Q_PSY9aIa>-@vkFLD%w zdK221JmCL3AUw!?9vOiIwPSxzDty2Zj!Hm@(|FOL8PUeIP{|Yy;V81X0xJTB%*lwsTK-c@3Vn{j(yB?vc#wexadLBOj_{2D+cFpu z0oBM1w1xrdFO4Pc2EOr?wy?8y5*M!$+^Pkse6HUg9m>Wm`aMTaExvCRqe5-(N)FN| z_~8+tqzTl}0MV%sWkZdXw#q{R4x$pNx;p7-1W0?#u{@6j|A^PonQ|0#6%=X4=$S(@ zJuU0w<{kKr5R`4q?h&<04q7TS8Esp{fKmz73WpIpjZQ+WB7=0K(fPAco$h zE5b`TM{D$>+gSDVbIb5V+A`6FYYP(D#omRZMaY)9+OBA^rys2P#YG5{e~+}qf~H8M z73M3xj1OTDp;3>1fvCyQFQ2FtPqq$)YN|ej-KE}fDT0s2-6>r#~$jlr! zK_yOw$njA5_eJz8RZCW?(nVRf^N*N zMXsN-iGG^#({t}b#PbgG*{E1mf@1ilqkv%YyDL7XOQ<>V6uPWk}qF&`S!4_zXC;^j>_h3;Awij=1jLg*hL@`%qgRLfbNtu*{81wRk zx}QK$hki&*>=?ZeGGcYI2Tfvtzd_Y_n^Q~at(=Y3>=Yj#OB!k0u0+2Imlnsg^5hrK zDRS#F!?cpEUj5rvuIb3AE9y)|o5TRc3XND?ns!}jC`7xy+E&^<**|0B>75XK>e7)Y zhQ!-bBNG^@FgfcJi$t(}{Teg$>KrbEbok?CKl)A;@% zODl6Wp?T`?nKuO>?}|^G#^+x#Q()>UF(WDWi>0xe&WIWs*Yf-L_{Bkb8%4MDz(?qhh9 zJ}ao21q4dp^}Lc^=jmD?7-u?H=bYk+g6{<%x|`$FU=Wc+kA~FQ%y{uR9 zBN}9W?nC^u&)a4rsJmT_YV_;<63;8;k9ZiD&_6x$9?*qV37tOJm9xZ32R~BlOi_yQUO= ztT^P4PLU!WGH$(J_gNGv4=&$$yb<-Zj#d9?bslhS6}mSjf&#a}T>qu#;|I zrgC|Gr#B#}0?t40sd4A|J~{*i{vDY6z|KQvLjX0k{TS1i_ZtoJPW|wVgz^Lfar7}q z$m9Dv31+;Ymf8EJSC1^kdklr-JM-7!%Vh0VxkJw_SH(Bn)+@fzO6Fa7-XP!h{wc5@ z9l25$_5L0h5!Ruo(Q~yV)==1azsMuqSmr_XaFf3nWyGOn>XW@c@onniCDrfjQSjd5 z?6@yKrvDio`AMK>!nvW1g~{@Zd%R}xkkFcu6G?Jc+{kHNmS7wqHI>j}7{EbEuTCRV{DcjDj*)v}Ff32gm=F*^VF z61!!dHu;@8W12i&?ead@r=GY8g8EG)5CY;DU$6*54=kHk6`lYM;jaws++FWoj}EGX z>K>)1_Rs96rA2Ma6GBoAscG^wP6R|Z`hgu=T%9vWqu6Vh?PZ~70(F_|nfAQ?qZGcd zXeLi8pAix}XRxVF=sTP10()L_Cr3MiAk zFCE<%vwTVYnY2HC9V;QF8ykJGJ$TZi&Kb}9;sW)H;>VEj%YV!ZqYFrtKd~?wYLNPW zs17QVfGs*`T)Y{=Q`-cG3||(gDBed2HYWRWDE1DiayZ_9(c^D#<};gJ&ArgGhu3pL@qryORm}2JZsm%Mlmq8Y#Y}3b09Wdw0K4*Xv;fQ zTPuCPJmwBIB94SA&v1d8~$$G#+1;fC}uOPl2l7Bb*OJ4_NS7A81Ns zf8NIYWqOpu5Jrsc_#MRS*4~Kh{GQP~ki$9T4avaFrx?Y&S^YDz=5FUyyM$*emv5^{ zUn{T%JP1&Vq}u3(f4(=hzj>9FZ%#u>Bu11F7`FG=;8YddS4N%Va_Rft=hIFXYP$C+lGGr8~PZe5a%?H zcK!*+`hoP1e5jhXa;3?mxtnhfz$O#K+op~Of`D*|W+2$CVC?#N95IR9 z4za9rS^_mBR@kU}|K?Ua6L{$4Kk+;54xzIcyV}dNgbEcewXge=njRT8`P(HMwH~qu`PGl zPk;Tk+jb@lkMcx1Y@p~Li ziR2YCz+y2VN3`TQf0=41RnEynw^=` zS;`{Gml?ru!Kz%7oUK~VCtDdgy*V>~&R(?Ft4F>x``}eZ>Xv*@UHIR&VrX$?=U4dm%kVA>JzqI|vP0jE>%)d(3;kW}LeNR>!FrNW4iDppr# zPA^ukV%>Prn8Y(Hx?s=S5E}3L<;Q2~C-(!TVGd~_i&Y>#4Z#(_ZEqSoZQ-={ZI+0w$Ur+vP<5mVN*a#`l3yN^^ui}RBvL!;?0y?HaI-12Y`^CCqbeaISg_j|EF(249J$Rtw$8DQs#tUXismk@SG2eK7qd`|+#vWQ)+Zj^ zOn2@teBpKhjTFls0t9gYcuic*=7BeKobOvOdn_#rjvFifS({;pGbbut^s?QHcr0w} zlD%R+DMtL@Z%$?#pg|X8v=Es%Z?CO&IxHOSWCO~69SHD^!8f(QbW3!k1HJ`3IP`2n zyf{Q)|BzpmRB3^M%LaB} z7wQcvYln}>egy^tfjA&Qg!D{S7a8pRkl=fZ`W5e{dvmbKDBZfZv}3-OZrEls>1guH zbp{nL%Bb3U@XzwUzJH${kA)Q2E8b;}j+j%<@FI8|JkVb4)8BoM=FtLzCkAM3f$zq37#_1+o!0=!oiYi!;i%TQ$`vfgln?+Y0lI8qLCe{9!+iz> z;bQ<_Pb{cuw#hW{0bZQUOTq?9_RZ2(IlhYr+mg-Vsd%MGgrHt(t$uenDANif?95-< zGu0W39r4{{E-OxD7=x=0{{f>`UP&N^h%f)D4N21LKN}vDYNxLooDeg~Z-k6#vP)m- zaOW`*$Vj)+sLmBHIpV-pL#(a9*aHg{3I-e81HhV4z5zQ$*!YaVtm~vCwoLOB{EQ&? zQZ$UZVj&Y4fh?tdcFqR5K4shX*>H));7gW=`oT+L?yVQrw%>zuO8_@|=-_I99uxQu z7*O}Y&jn+>z#X*`YwvDehQDVsBXS3R`GHvU78oP4r|PPkjL0EhzIdQ`=FJ8!1CjdmiC+SV zojJ|D`#CO{z9Aaate6ITgWbcyh%2UZ5yq530AO+gIHLD|ApMgQ#xJx-3D+_UWQ4$_ zI@R+U1e65<5hONyAIu`ffhQH1Ez5u5O5zOH)&eUtSgFDq3kVYYj}&srFH-Wam4+j> zZpEtugfwfd`!C{Rrr;*8)}ETKEZczD**Ao?AQewSyP-eM>`ffJZI-3@`DrAix~-E( zhaJ)6DCrC@T)p#u8R#vOb}j0!no`BEp9QY1elFs>_cqBAQPrCpuY6^K90>!4paxQDsVzg|Dk zs!=n0aEkGC#{H#Bn#Ya$Ra*bt#I9m%sY8Oh8vj@Kj!~hHxRYGX`tXZlL8N1HDCA=7 zq{=h4akp49<;!CHOg3#ALgYkl3iJ$4-tU+8{ztI>RT2`5M0{u|lr?cgTsVF>FcoxB zf%_pe*1wKEL!n$BXB^9g0@+~rdK})JlAW;$DY;>Aof_G8>>J5D0>P=Rb<+OBY3y*! zlu}hav%Ep1etJ*8@HNMzgZ{|a4})i}%1D7;nipR`;H<5N3`(k@x|t>*Wslod=klz? z{hmsoelpr@oC4R@R%swkk;#LDLLNn)Yl(zhslprh$u3@pRWN^M1Q}!#FyL4S(T~uf zw_qnC9Awc{Zap3UU;*a7KB5@eH`U=6<)7CQ6l7!)#&v+8HF zFiS%M9gtzHtN$76F@|e1-+(ZRNU35=lqKK?WbgjUU9RcCNem!w?XLER1T$E)CLoAj z0vY>i6x^jAuM*#2!d4(BpspQe*nBIs2Tq+*04*vW0jm}WQZ?QVIWCu~aG*bxa8I}X z`2WRRq(Ni+K&&63|r8OxV!vb%HQC|ZmH zv+_NvSs+W*{7__NC}cbb^9Q67KRU4KjOT~U83M28;>NE=as1ukj1|%PSU)+!KNG;JIbLn4j>28J%_aC!(cvibEU?t>C$sl*-ji8Iz%IQPg7;lrBm z77`R&YiqF~@OLR0WnEwM;So#l+q<;rzeWgBvFwy&_=z}p5Z7<*IvBKP8K05sgw;Xj zHilfE!=Ibx;q*nm0bSk6M_`g`_mj{3*KO$uEXy(EZ(kbH##i)|aY+*SvIhB-9_l(Z zqK4DCHG))t%GO`zPH@@ten0{lzQM2Kvw4$Y|0o>3VDgCT!n@c}q0O8~b|E*7%3<8J z&LH;JNYGGTe6d#3@)=0d*mLt+%8_2qUu!3koHz#$&Y-jE%3v^7>ian(5-9!%Say1a zAHbR7Zen6G{OJ)>tPLdUIRMWF;D=dVgUMruLGB3n2oyJa`4R;(1}17CPnq4H>8`pm1sKQRJ(BuB6OHh=O6f(8{af0$=MAqRrk=Ht>5 z&7TDsTY`iatr#UF!K1ae_utKj z@n;H48wy2os7Oke<;b|0u~K6;c*8n!+>l+F`V-C67bv7Wh}X!eaJxvRhO72BGV#MJ}r9HV`j zwV8&XJ@gBoj(HSyY#WH_1j9TnKe2IexWVrC0z?##GZao8c?fL#S%G^qU|7M?o}^g! zm%>2=kCW#uu#P2gBL@oIi>o0#3JTao`T6_?bUHqI06%wz$kb6-bh#iVPwh1kE3gZW>5eh`9^ z4$5fZrqYu=#x=~Srb8t9bKj{3&10s*e>4qb4Ijv?#K?_;Bk%qVA@zU9WLnrN|Z?aC8dZIuvQbMOpJ zBzI#!GE_XAnEDIJ6V0E_R__0|7@ytxyqdq&XpzUfwiaTyFaIwYq%XDLy$aE0hUahox0RkX<-8;@ zX3^%O?;`$LXsZ>Ik7tjOA$?p`zS;e0f+TZWOxyV=40p*}2KN|Xfnx924TI%>`S?75 zjqR_S#y7CkhD?!$g@^%jROIkC_vdCHD5>Xe+ek;}2E3%BBj!|i5E+3+oscI7JyZf} zR_Dcl$a57b&m{*!P@Fw1g;LnvwW4&g&(BZLsP4*V5^h%!gv#5jS_sKTSwcxTD3kNE z1T$WJ=cGK;tp2J%Y7waWQn_Z~lyn?42iVxK4`;+t)Yc7k{PK%K*wv5Bc0|!j@n9u& zOC*$@`rO)Ot$b`9#x!#I$X6MFB)W=60K%dW>`u1b_<*NcCEW}aaLGi ztxJk_zKCf%05%aguE9YKiI!`Hl58rpMC&rN(X&DV7!5wQY{uCBVOU*VTN5fg(9eKH zs}XHY!Sq+K->AoKLOw~uz?@|y6%DR@b@rB@YNvI z182VKB}hN1Yn{n?Q~;q93%{q3UV|MXXsto3l_yWM3JcyTxPMBoCy4r~=GlQ7;%zUN z#;Shj!90dX@ykL#+q`j#r;3dqm5u%QaQsg5@3KxfRm8ZyGrRhI_NfI^=njdl8}C0{ zccs0R((Z&JDWr+qUq>i@)DF@98~c?Sl!$aIyY|gv7|F;sdJhQM4}D;h1`s}r8Kr7! zQu*H%diRO9FM8fQeg_d0g~2Hgwdy~^*(V#S=XCxB?(N*7PzD_~SKjsfq14SH7WuAA z&(hVR%G|Jd{fFn_0TP=*ZP@g_axR}Sq83nnu}tL$)cAh9fYi)wGQbY<$>M*jm9-~)?iX}K8rN~D{IBg2e^oJ5@A-OU`{G_In-8h3-5R_^ z+W{JVWr07)DV-$+6bj5X10Jy3EjI0s!4`wS8h0!%tHDYr0u;Ky1lH64W3~arj&|4l zD~@SaK==L6h(q8&|I+5`sn?(Go%@EJ%f}Ic-m6lo`x9^jX7!aBv%|#16#d?h0Qf(! zO*#cxecxX9o&y;M;F)Dn!0;19n_MLr6hNxpTwhxp3*L`hXQmJPYH|G*8dUjHvu@}KAz<$>J%dOk01tG;M${E>cUgC!1EuWmpe_4Rn zM+s}PrE?;nrxQFQbN@E+%m`Y6v7YBYoy29>E-?3)iq0u@;~Pj5{tL7H0TQ_%oLXK&IK_+J**pU-i~q4~7@EF0sPCzKBXDNc1085icx~+gya> zKb5_e2~lC4JREF`CTk-j-_x|RgKw+fp*7oHCN5kPR(H&L@+_y%OzGyzYvA-N(qObB zTvIu{eS;WmOS&)yK$zcvi39WGKp-};0IW*^v1Oh`_Xw|#=J z3LDQ*78AB$xiA8x4rRFdI?=wxd_j^NXtJvS0|@# z004qMClL4a{ng{`tI3`I674a}gpdJ^3r<~P`la`sfaTrieXeU{eCC1DEa=46hO4bR zS%XxrYiz=uK!Wx2>@1`C0qYrWM#CRN`#;MCXjMA6!~nCgE`d*c-(5LDnTbbKE-mB^ z*00OU){}=P_QeeiSu*a*pO1?VglzLdy*qfmNj?AP-(pi&5#aXOh^NQGlYXDqkoI)A zLYJrP-oHMRn4AkY5W-gB1tttk5<^g8ng=RM0;&_@uk? zn^^1xTdPAw+?&jp4uddK^r@4e9rN{YSr0{#EufGW-`^ zUHA*vrE+CwL(_4z{q;D5oPWO8pF@AI$>zw(levT1wm8lU@&KJf3d8)a3R;y+dF4nNsLWR zR{j-SslrbXAY=$&f3vCq2)jDA?6VJW=D5h+tOJqJmfZ|u2ws~ z>DbnuStzF-NK#4qeBjODPFTos>9Q(Ui+S~6cPHt)hDYn z%)lD%^$J5l{`vt^;!~|&wS13hqt0R#>w?+g@2njuX8bbU#+CE#m)xmpcE|xJJm~dj zt=XiKj?wy!zRr{yx3{+!AaOi`|LKIxN4A<8tbe3IAKugvd(bB{ipzhVAxtlm+Yw1Y z4w-t_>bGW?u6Rj42}#&aIng{PtBHHIV0X@!!d$;y#X#^@ri7wSQ{`^*Xu-=D!{FJ| z9}$RaVi`=^oK|sKxddSHAzY zzI@kb(wM}!{tSA`P_5~3<1bcSMU(h9khVuIT}syR#=mDysJ-NQ(i5?35LO%js%rFF zo+3(SIJyArwaZrJ+oHL7#FV*ijGt)q$nwjmk#JQ$A}@bAA;{tXHQ^-Sj+f(aV|We% z=>>F&)FKtga`-e0ikA01gmjD4>ZGR3D^n>|((w1i4bM$?($7Jq)OGUdGzQ8Xwa6K#@AEXD50iUtH$r1+lg?z8a6^usR(x*CX#@^+x|)W6 z-g=G>qowJ>ULvUCH@1gO=*eF*{neU`!ZmYn&*RR>3By^^c%_cUPc(aA&*Oc4Vkg-o zJrzl;dsdlkf^5*Qx7#HQ)D$G+EBbj`WjGLZVPiAqd~n~Jrp&2p!xZ5y7EvLK__?*}NBF~9ZT|=y z=tnpkdW$C}Bq4Q1@QF0GRaOVdq0#gl-v@g>__1gw>vYlZ1QV5D#OJ=5vO?5luASli zgJj=|+F!PmK;&PeCj5OPCuPr~uk{0#I%l*}c%~UebvY^6q z%7vvh$G^ti*va*Vz$3p)_%l&rRH;GyAD+@sP3fKQ&aFyteyMul(R}ptRz+1!I_yt0 zXmpn~HWA6cM~tH@MC6=5i5Zhw?8&spWXlqNsKW>_5OE%6#`6`eNL`l{*cq8e{2d zgUrfT6#X3@i5c;eNUj$yhai5)aATNvnMGu$?tNn=w@eI6 zEVS%(4TSR_K;X!VUlN%3f(%R33>%DmnI#YQ-#XF@Mc)YEAv#Mm_SB2z z^BxHLjakU35i@d(An~B@qC@#>Qn!g8LmcYAR+&P83e9RvW1b|beV%cBgZWk}*x%U^ zN8o9+WV%#K7=bPWnA{&694HA5!c3+BSOE!uN9YdELHa7ttHAA?^3fX1ia-hj2N;lf z$VTM-5C^b}jnHft7^jY~7_E8vEY1xkVe8>hszj>XC$MXN(I5J}df1*nJh?lL=`%{c zr8r7nNK0V*!uf##aSYjo9>or;gVZggDZile!#@Xh+-MLXUECz65q_^mi;JMH+()lX z++EAf%#r`f;gyCC9G$A4XI(Dc{o!g>YvhyD?RR~iKYoi&<_T;A_oNUpl?inRX|XQc}!&7M}dEB(jj| zXQDN#Sm&nz;wv3Pg1(?6Rk9nMqzaU0QY{}2$1H=(Ov)4WX`d0INt6Xxe>MvA-@}Mw z)1bpm9?#&t@c|VlbPDW~assu^AhP1c-?dp$O0`Y|NmYa4c-M^Jf;JteHvMXSTp3z- zBv$87TR+UBx;8s)jZ%rBRM5q<#OmUU$!57!t<8#7;^=^+9pe^4=h<=^M_Y}5;L&-C zMcQG_jG`&U`*ZAdftw*M`DFCr5ZW3IeSL-$?MzX*;(%qd4DC}WdRwXhkLo&sbr*x5 zhadM~rAn3jS1m1FlERSfj0%7yQ}3A>#*Tkd=grz5KF>7f7VL~Qt7?Qn67lqOV}mg{a&q_54_+0*8V6wxoM&0Bt1rL6PJXbCVYd1X%je7k^X30L-s zix5~X6jZik0PG|A`5P+bWP>0CQ9eQ7Z-d7$sB?HYUZMP@<@M02lV;*Wn|aj zVRju>x|cx#`^GdRm~pBVxSeihFQb;$9r^E$yi_wK(IIMA-cvfRC^?l2evuRXP`!5= zuFNBn5ymfLN__HXtSPca%W8hhC-{H;qRS9YP%?M8n5xW}hqueg_0_)Z7E|V!^%0ke z&r$Tu*@Z7xrQ0Y-G^0?jU8mY{Vb+!|5pY{LtV*;+k03sxL?uFCdHrckL7`O@zs=!{%y)wkMli*Ws9#DQ?OLQ`dl;C>c~@Rhqo`S` ze!78zmz0Pl+*j`u=9APM;3F|n5OuCd#E|!gj_+|e`-S{%zwyRqyI`(5f~n}z*@|l?%0wQ zG<-D5%4c0CCP&rxYEoZBQf~d(^}jC_z?=*txjR9cFYrVcHmIjfQomTP_tt%VpSqH* z_BS5g>Ju(mtGsUA^xK_PTnCb`1;*kmc&GFPTSMj}N)7s{u_>VR&&dfy!8`y7?7M9> zia@$0 zQopzR{xq6j`2DP}ra}(&w4B|opwb9GBW1}=>f0M1yY**$;iK&sxVUfMnmLNw8mNGr zwIdcYkY55A+iPdwTc&nGQLIlk?|kqe%+Dk{^9+f%h+jUMw`-{h=Bu69;uD_$o^{o4xO$EIsXfUC z-5zqRNeY!b&~P3fmAmjuI)6j1jVqh)Al?O#y$>y*RsbA5q2oL)nEP0$lTXKqPt7>a zV?i09YpnrtsSGbDB{Q-?ip;mGedd%ZpBpaxkrA^MYM{^jMIY9#iFEb^PxdfnFQ zaKq|X68xg(0!u3?VsiwJ=1Vp(i|1+ncinaWuWg@D89ZJI+aOPqc?@aV~;YQug3jOfTP0Wty01t z@`j|Txs{JaCVLrGVfOa?Lrngi%)Q6$lmYcPWm7;KT{va+#g7FnOz^)0sy=W=S8JaE zVxkvJ%VA4B50nUHVNh`fgS^Z@2mtWOf|#WR@H8Vz(_@3=>{(4*14>cp>5Fs~%prA} z)M#|k`3s--^_r$_pR&w3O_>O+LIij@oF~_h!uB-7lGrVCc@oh4g`FF~fZX$c_^1Kwx~>#?Au( zkAnf7#qP6>38aonxb7WQZi+`4c zeBdFLb7y3S)WNN#rguRz%uZM)*in-0t0%3^G*kp8{~X1`mJGE6#jsInQCmdiQ|J_a zz>w89|3=>g-qf0j0b%!|>zp0*0)vrj?`CnnT-bm)urj}S^9YukUa&wi?-dhR$82Y~ z;R^Ga;kPa~&!Wj<=i$Hu>r5m5zDDf?5)VOmrcn9*@+`l+^qjwW1WA~}-Q>MhQ(v@4 z0(szA#kqMrctz!t$$LqD^f9~ zO`+SesmQX9nZ81upSRG61@!Uf*iceg9Y1r_0X+Nop>=&iyD&MFGEl&7 z78dmrT_L0L)Pqj}t|e1DGn4$#A9)J&i9@d|`^Z3~6d*`l_clz*)qkft?2j2!Ur^;I9 zxg%Nga{{vV2ruvOZ@BGjtS$?!`2IfAYhqFV{@`p~=4ct6IPqOHlHFcA3s#~GxrmmR zZIISJUv3OFoS%=Z&#J*fR}GYp@oM6|3PlqSd_<*{m3SBW-L2=b$ela86Z!1>Rqtiz zCD?k(vS%J29*W+sS_z6Ajlqh(K=*)&hQ<|8IbdZlpfdw>R6BOW<}muDNV&B4^uO;F z>=~=0aBR$8P_Oc4RIY$W?>92ZD!y^=Kn#(nnBXP*_C9=kX4(Fo2PvfApl!v7<9%t@ zG@-=pCqc${#_dsBEgFsp`RerUkLa@yLjPDdURe=lH*g&$L3ImN%%7{!N)_1PM?RXm z9G=&D;ZkzirkAKQ@Df!jrWt;6X%)dIQ%Zw}pbEpD?4-#ypBrL{@cm4Z0EWq+SZxxBBY z3&&?cKO|g;A9cNVF3~ijK7bRvSyRl{NUkA#UF*FbdC095irR0aXN%TS99_Rqtt9&U z%MWty?^tYbVHgtEp5DP3jg0fmh@tmh$g*$icgdt z4Vk7@boao;9(@dgWE?{V>cFm>)XdFCB?TMYr*O3Kh!7$Y($+DJ0`_NdDrnY$U!;Ru zTJBA3d^zDI=`Ua5xvid;?^<+zaHlVQRge9p+nS$Rm-!t?3PaYtv)T)Y)j@XSwsrKm zW+msQyKzPM=*`7L)(i61h!B*SGCqVKqKmM*Xp~b@evW{plcqn!$Ghr)^Q1sTNQN3c zrTVvGvw>Z601x+{V4mu{^Mg4BaiYMuC(u7}s!85CR~)pEOv5Ce69wdDYfsKOZui|Y z#1a{Cr`B$X<$RO?GYSbg$vZD8I)$*=(I)8apmPP2O|tdk=xV&_dsxv9;7_XZFML3F z+&RdD6)KOM{4QE8!|pka(mHUNOV-P1)^ zZg2o4ER(k+zB7g^RFg=pq$s-lkkYJ|o=p+Vo_XN(<5%btNMa(*qtv#Cc8gK#(3*uH z1@4%Q3O@>hZbz(=f~m)Xd$AB9KZ-9un7b2qJ^GKXQP_7>7Q}adzB+%2n+MKVM9H-D z^!W8tDMmiNwx%Yxr5c?*cIz+PfjwYh0;^!TQlX$f*I-x+GBjWd z(Wn>y;2_YE2?`PZ!(CJQqaGV|t$`F5m)13GP|W-LxUk!p9gud_vkUlW-ekU|#n)O|uO&8|DSNdSUcJd)shM7B zCe>0vMl`p9Cw0I40zzSzwy+ z`YWE?jgO`_yRWL}D}IEuIZoY5*kitJ;ru6pGipe~>kL{&_t-78K_U(ZXas%nf(io2 z*d16mEiWytIk)%yJK_M6Gl<=+USTtQ&$tOvIwo5H%`02va0X&r{({T>cq;oR7=}K8 zn4xsVa!^?T!V2sFTFeths#{Yb-I;#G$M`}40x@hfu@c+n9-iR0XuN-37vnyiz` zJ5}mNq{I>Czu5HN>X&Qar=9Rpio-peIQP(0KAOvKP_RC~6zN1BXFw|w6KLCu7tITV zE4J|LGzLR&$BtUmL=ZN~f*jBu;!AJ`* zE}(Jh!-prH;zL}Qpw}0K62nL;fJ7VU>vw?j2E^2Y=;5Crg$rbDzKf3jy-@gY=q0?d zzU~A1D?to0xTeuPml#Xp4Xxh$AKb_>OVup{hbWw{-+orDW7YmOeXqp*p2diw${cX*vt!t7Kt>yO-49Uil&F$Rb z_kZpz1f@AY`dI1Um5+5YIpNODA#N=0TEpo_USv1KG@nd%Gbn&)v0b^ljr`Y2uC48> zBbmOB@@0YMoA*T12Ogn5f**m^>GWJro^G<-c-^`RTiI~-kDSYAw0788$)4#1^gQ4` zQS87#;|Ze%CQOjR!|(#_-xa8&0Ue{h!ug<14CKiGfU(Gd6ap#Fpj!wR7Z*f{e*^_G zF(sD&`RLdzx@a#w0lbn(z^${B)A^MtV8I}2klPG=6d<1!oUX7!a8NpS*?CM!E<#{> z(be*U!j?w`!~=bxyZigmRH}?0AK7pPQAoH)t#ajPET$t+yb#xps9_f8 zvu=<9!hv0lQhU=Wx=rM}sa5;$nQ8Xg8U2SC`?TSPQjn673LjR&-WYthPzwpuGL>(XPi$I*#Z-I>`?g9rEE#)c ziqkX@K9nR|W@l+>mo#)XGYR*MRpANTbP6}OsBf#+%`|C5k9&|5KEa|lX_L>T+0ZXx z-_}e?HN~AiO2J=wLK!>2xK>5Jw`Nso-0kDS_*W=@2JH5V)P927F3e8euHpZcYY5z& zL1cQ2RlS^{c^&X;1adnq*U zn=m`CL)v3zQu0xpB)vzzxzWDfa1~i+4insT(J~lAkbcY(8_}Siaic}}Nwa}GDDyfxb!S%a>ih$hFcgALj+*v2fi%gX4W zlarJFI0EvR5=O_S&~iYpfW--5{6I?Nr2yL&c^2sU61AB~3&zv$BLf|#l<%~FJ$j|e zL=c@vvzUho*)Oq8xc_B<{z<_^zY_xAOH0r^20k!+3G6j#tVt>A z7izVQ*-`p0e|ir8MwXV-CzOzbAPHuE!nv-JK(fq3@EHDGT7JfTBq*;-TrU;jy`gd> z$MTIqfgL)x`nNgcWh0s3hyTD}K4lnh;>w;ZkWTWKjDf_|G^$g^Y=qgDVDgiRB4f{w z2^Ohh&P^Taw^C7Q_$PEodkye6v`N)UBL}Y%uEc)BE3GVbzB-2DdS*z%mUQ$hHOjU1 zJkz^8!c{95P8BhaWV6k6nJ}iW>t9@WFR#72(f@8v^}*mPV&Og<4HM)4V(+cPs%*b* z(M3uqg3?{mB`qz|4bm;$AYCFLh=7DhNK1Ej35t@^rG%1Ff`lL-vFGylefv85+NZAb z_gVk=>bur@o^?NWj5+6+W8k9&w^h1e*-{ZmUPk&-=@{{UlB=tD&v#7!9Z9~wEyo__ zKfmj|F6jCUE9TCMh+)f+?5E^8{l2sYil_mKGYScz&mtLsn+;b;LA9M;hExx7+qgtL^8iAIdEcPYMIJ4?K_K=eB{?Lqb0vv zY4ONq4*#?$UX~U$E1YHv-PJ7O{84qGo%6F0O+$-S$AaHzgA!xnN@pX1>PG_S52l-X zdJ3<#lDHzw^zw~eQ?^vMUDMKH8n$(wGja9bA)QsVag27C37dH&m}p=e6sK;c-?ui9 zmB9Cl)Im%_?xB9Sp564ILUQHwwu&lc!nW=C`OlcN>mgNt14{2&`K!o98|8k_p>_*S zo-xc9^lxS)Ax&Fa-;Ux#pLuHN|EwWX+5P?cvF|7H2Y8knN4Hto;_=a4CrLh03P|Tv z=jNjJ8s~zRh1hLtz*ERYw8diks%ma+sVExz`LS;x zrIPO3DHaxsx9O)tP4_9eqk39EesisIytu6S^!csK%8%w6hZPPD->{{W5>q2*9)SIL zU&=#D1R`{RMMHzqrM5L+Mvs>64#w%NTg~Ir-FOp~RzmJg2xWE_c{YXHb(?*^j%{Zv=$KSGUe1p4-nZ~YHp%;aj*)%qCKqQO&arrFi)f?Bn z5s0^gI8!=dN;D^rlge1Kt47qVeo9gW1b|r!Z1T-zQY^pQy+m|sl*vyV%_?69C~41b z-QO*sJsERee|NnhbQzIU!T!Bw(Bdw!TZL@6|25IYq4v%>TIrS9H}(zghAMcK+6xyH zsrc-KiI6rgb8Qe@i!(73NnqAwy;DeEm!!G+^0%=2^g0?M_dC%zHWy;B_UGtK#40i} zHMh{AZ+IIJd?$|ap48t!A@2L%Qo5b?v2gPtCqS=BkNSG(Jqs(8p;eL#q7?A$Z(kEd zP1PK|DCbp7SUkMTfp{d8mX^C_sQ70yay1lH62a*UwFviKzETeK~GF0>wT*So>=IM1@j<--#tnq9Z zv&#z^T=S)rSj2l|)jYGZe=0sD67vj!*t<(8`$6@F#k1sfSl4AgB|-;LQYL8Ln1=29 zl5TXMvbc`ki^7*)iHImP2wgr-E4(X0PTBOBdo3E>yfr4>&lR!_0O*>YkcOT4T%MnT z1VK6In{PY~DR^6h;9<)v!dA-YtMN?}iy!;bvy`(C6swUz8fCZE;Be?w-lLsxAN5N# zM`MmeOm_~Up-Hu~kISKdQb=ckQu+Yebm`si4qlpJqaR2AevhJS7Vwbv5JKwHLR+b{ zJ4=`gvHr%7<2XLu9zz0cdW?6b+eW^2mKd%0HhNnoU%FF|DQy2k~z$Vm~4ecQ8wk55Qy8BYUm z*%h&GbgLm06%~a~QPWI8a zaLZFP&%k5GMR2S<^54CV?iT#G>HD#n=lx;1m`S~NkClJf8@y@dlx51+uB!~NB&Hpo$ptj0l!HYojOk@^*bSL|6d6zW6<`?-7=a}9# z)EzfplrH57@f7C%;lej2M!&swv$|_|PSmY($F`3kJc8pbC!QhytNZw)M(+{w6~SK) zi+#KWjMv?{FZP|+<=ee}q#b|EpuF1^k35n*e2`w*EPwA__W*yux9VW&kqW-o&2P=) z{apM8$3N*^R=wYQH`VAmBia~U(#fs$y%!BZyDpn4V(z~^w9_Ve&o_CmY3Av|{L)&L z0?|;Mm~{n*kl|LG^zxSVC0dX8HT4Dont9v1otEC3otG==-X1uVCIF_=6#2fI&y9I^ z%-4)Wy=jCs>Hz#L{4z4kCA=!qzt}taB!bzBql7QM)0Z_s#iWcjWU{|^9{c3i)%Uu` z2N@97Lqb9xqHYb-rk~6a6xO*D*D5d;ao!A)0*W(UXyr^Y-vuefp^F zvZtN%CqWnQqXScA=#vh_byw2wT$$K^vlL%!mbnxC%dT|RrS#tHZs+w^p)wBBv>hL6 zw11bR9SA9_{&nK7d9ht`H9JMk8RR>b``3znar=+*cMeuu%9JJM@63xsXJhJN`4v4o zIyQqL&s4uR^0^492JF7+RN)!@^L71j2h@7OgE$>BeqBUIUZ9|Jz4lyK`gJc@&$^m^ zJ+q;CdVV)$(4@g(N=j7acD};tP@&Kur(pV8<{zUUSFH+%Ujk7vTT*&@j^FLP5IH7w zOq)}qx@Ug3^JKZkhv4@O@;L92g(JplR{mA&=-Li2ShE>@6!RYiXHhT19j5kYN(sI=G4^ z{00~P5BOf6PDSV4ycC(k^FM)qD!_m07gz!mS61TfJR;a4Aa(QtxemBPu=sUy`bm3L zO7pP$%LA0Q{fHRBQhuG?pm&eMilwR^Vz~sNp)?zFWz68vg^dm|8P`j9%dy^Ia(X0> zMHeP*@%g%Cr>crc>e%p!y+=QB9a@BRE&=$H5bfT=Cx512^wm7)LL$IV(_o>3SVQ)~ zawVUOrghY7f$8^u*yc`N@}H*G&KWWVa6I~{)9rT=1F3mulN;HKmcM#2dylY}&pTfc@W4XkbD|=e~^zw@HU=l7Sud zTG#_6_sOrz_w_6&%BmmW=g=Z^!ZE`8!j?PSIP-p|`rvCYK~7)%PZJK8ZLVb5aap zeh~iLvxPk&e^2|1L%0SH^d94{hdscGF>2RtkaKyM?ui+0)$s<=WMGzA%tbSvTjEk? zYU@OQGFDzx5_v4cejW1`X`5!zgppzl-Gn8t;ry)&JONT25%vg1?Cf;RR2${{=4$~} zbxK8i0ayvEx4MMSY4Ipiyf%`;iIprTt$*K6nyq*)U6%e|7;cP32A$`a?b&;C8^RtF z2LeQJT_;~{eM@oQ{P3Fl`Ssfs9IV5%C|J}tpPz1sTXIj3k?ZrO?Tx7w;aV9D5G{IX zZ#E|C)0&w{>A%#$Q>s%|P8Be$i(_mlPd|>t^;#GMKH!l^9-0D53L<+#Xc-qJ4)IbK z3u6;>F#LdL*lmI#ZO8bEH->59-8XG34bnsLJ39FTwA4zt33J6$hO8Zm>~z5mB&^za zTq)NOfXJFZJGj1WIW0ZPAHzcgDd>{np#JUGd2Zq!sfAE!(A8OfsFSpgg$SD}=)9%rJ`pAYL$)=dQhaVnxmv+k6(2V{U zC0tq*k@OysFRv|5+Ol&>xuIY^j@7cLeM6BZrSoIaWi}RNha&{!mIiY<~C1quQ( zQy{#;1${{LeEeG`_YCuNCqB|8-YI$DuQ9M)EvnBZ8vo&i%v}tdhzVTnKN0;_2Bk01 zUkMT*q`r1&V835`ZEhQgSZqvl3bOCKjeE0$CxH&%|4gTkn&^mq3KB?B5-HtU3!n%QGnblY1Rh9i|F}ej*($q8`~G50n=Hu z!d<4W+LjkVh|q~EFEq<4Ha^jFW3Jwr)qOM?MAyXXqn60?$6v0UFy#Gqbb{j1+S*#4 z2UnP!D!_6`v;#87is1=~9|>oYc-ogYgB__Ny-^`3_v${XWt~Wimb3fQ`;iUXxYI(c zWdlj52wQ_^0;A5o&&{o0?FLWyAkGlB=t`OvYK=ymHm?=S_@_2riWuNyGi$Of8I}vU z6lft#+on(w0_{5^at5{8q9^T)x#@#T9BPe!OKe3~evUA$n*ue6^#>%OA zp>2o587}AiNR=N&%6?F%HYh#h4b!#ToWy^IoKT~}$m1fy=QhX=g%`R~G@q!8c99u8 zN714M>}|!u$QC}lY;IkwRL=Jsi_B6u_F^#=HT&nEsH6F0--Oou)6(CrGGWD zyyEj0an6#Fr zkKHTz+2v)cl=pE15>GmnL}yjuXN4bD zgDjybm0GC+{<8i$x%#DVPt5C*C<-W&5>PW^S*J;CNF(id)95e4vISjgm?~3V@r>RM z4J?tC6a5_?9@IIqjiF>e%Fc?Ys++Z{y?g21R`WUIb;ZWj%T2-}f6I3!Qnw_YFO^0e zenLQEWWXuUUjFs>5#7zGj6kKd3QDK6C-3x`c(lyj-QZC5b z=J^YGg~V_z)krOyoR8x_tt6H_61TTmyh?NJa3NEvlYe{6sy_H8WoK`>P;^7oK7Z|# z0eqUr?jo5Gk1&|rknVQhnH9i`V7L|e!30ZsX=Z9_%~Qm+Wey0HV9tV6`PF1$h0fNw zKcID@t)~~!7EkwB&y~YvV63do)?rU%PiSbv= zhN^s4&8L=_9MrPdSi7U^+BAchZF^>lP6B~LXIZ4O8uWB2cG;vP5}uHM!$~l#7#LWsSStku-lDN!lHOf?sxq?U5R~Xrp270F4auzj|G7YOH zR*hO1|E0>TW!IGYeK)ROFUXNA13oZ_SUYcN2j=3(a0=6iNZVasj`WDSaSu{)zX=j8 zI_SQX%rn1)x0XQSGnIGp?k}tu@|S;MGWI0Gzk**gK?NEeEEYS|uBZFp5-p7tejU$J zo<+gGLxPQs4Vh8v5o%lDdZqBe1Ow*|@@&J-z9F-s4Vdn|dGk?_rHZn0^vpI@T++?Z zLi@BrYhHt9TyS840?v*8H|{_2WmpurQp##-?!tHRi$PY|pus^03O{nv(xF-<*L{fS zrswAa0h@!1+S!rjJX2SIL@Ha-)^Xf304EqFxw*N}qt)0bI3G^Se;vL9+Xs8& zIfylZ_|F|CrdXNoB-5&OH(|Pd96SB$DFA@@IA~>4{-+1l2q+J^_T4nq(`y1Z8_*5< zLnq1GdgnhY^1~uHPX*8#PEAhwc9>FgTfI(p1qG;)EODgXKi|{Ka}BE7H>e;}3?q{_ zw0(rMkqnth;Wn!*DZ%c4Bzdy<43K(LV816b< zp!G{`oc*?U1x#RlBu*C5qH;}vVFnHuJo(nS=8Lt#SakdZo?c#%*>}?mdH!+x2%!1# zH+OjZB~Z5hR+CpEJ!GH!F@Xn}G6*5X;~IU+Endy5LbXhAoL5d*l@c{CL;5`PmqH0m z0CJjLq{qR4)nE5}4%!_7@fR#qcR<(yN|U!EWsspGxOPI&f;AKU{2EN(`+&^@p>rIM zT8wm88nO|e;Wy5R>R|{&wP%hKAIs(26{yqziDWYbI$f#7VBl^dHK{J34IQ6~N}TOe z?xSwIaE*A#ox2HohmE=c9Vf`voEa+@e$1bjhv2tSTvk?RSRs+E&Xy!`y88O+H2Vtt z%05HGFa{dj1KhnoFjNXvHf`VX4c3D64Kx-H{`CHvVix2FF$A>FvvYDfLjerj40MMAtU>!Wp-E+Jn)aY1Pd7VT*AkQa#0@#uuvI;4}yL zz|~h-_WgTU`#9ggJkl<$=Vf>gYM``RAh@iZz~xSX`IT9?3D?02{{W_~@PdNDFt85h zr*hse-VViz;Q+DoF&sL6La4e)`tH45aDnE1P;r6!61Y|3W1+*D4z+`&O1$vv$a~ZS z<6-b2gqT9watsut)WE6{I5;~wSF7GHOm+RM9fPvp1Anic?;yXx)f+mS& zWtusUAsu0ReSN)KN-#e817w91d^eC(KKP37frmp-Hnkba>IghNfHxs+HFn4>!kGk- z>%FR`2Dt8Wva;cA_+c8po}QjaXHEDVXo5<(ul9jWj3mq$S+E8+dv_kmoTdLR@DzOO zx=k3!osnB*7Wh+^O&J&%EODXJ@hgEa&0E`>0Z>Is z2iIWsha z5fo-qI1COBv#$cja%JJ?0snT1i&&Ky`M^y7J7&M>Lx-h`1AWQj1f{@o1`pK`x^-AU z>5@+`b7iA-p^Rq-ACD#m2X5Afg<)-`{ECn2rCEG|{s4-QiU36IELwf{m%-Q6@ZSrS z8)$DZeG#-wOg8XQ7~UJ`CEyhZz2&jUc|zYFzmrA5-4Z_aqy$Xu-|R&G2pk$7v11 zr1w!?c9gfs&+E0tat^>*s+196fQY zBhW@hBBCrYvJ-OO_dF_#3t+F_)7$%yFbP^y>|&vy3fuou1zAoc3UWgB4Gf&^xIUc= z7=(g&x6bvn)$^dIpnn83??ce&$`tWpi!T5JXVizO@^W&+*|0v&Phl4}3k*Dm%JA~a z3RsbMn+QnXCPr@N1YnCha&v=J3mB0iL+yfM%s(^g?!ZZU@TT)+f@D+t8 z^1R6@Y;^SB76JujMnyQln!ZrHaGC!0?OWjMEZg6}6hQ6Yd?in<*vthFzFmMJ%!LMq z)CItqDPybMr10dwS;3ffTz&cWO)Y^jt{3+Yc#X8WuMKQY6dC>7jnp!P z#}1D`2^0|w7as<)79HJv4&tk2l1?m{V%XP)kVkenLzo=896SG4fFuAyp+N+40-)4W ziPVdz*Z~p@u9H1*U<_Ou;|%I-WaZ>;p}$|*M`9G%6f-2BzVq4I;@j47MM`P(w?z2FsJH}AlO>%e;uL|2Nv8U9QyI`@q5trxxT*63{OnI8bK0r->&w( zT{sg-S|RWle#6-<#LrJD5qJV-&f>10e$#_^c%?BJmN@KyyA#9$0E@o$dbSv&#XR~o z@sWs%YKZ4^3(TG@@j;%?a2vfmDsaQl*n%7eBO?^^o|k0M6$wuli?ITt0&sqo|`&dL86GE8=62Z2=Q52UPfhQ1C)xN1Ee5uuK!@fCvC^k1-I6WbYX`$3jhg{Jq(g2 zWdNiQT|g!81XzJoz-8Ah$FgEgV0R z0De9};*Cr}6R!L*a*#k8?L7s5S}?5hQ^<4&LP8E;WHb2J8yn9btPSc_7=q_iGPfD@ zXZHL!)SW0&Cy55X@8&-}u+KgP{uva`paTuO$P5=XYXQOm{C-xZSJi`&6R`_(4bI?u zk4@B*x9nel5%G=R-^HPClu(vaA57pwF<`u@(yy8zg5r3oYqt#Wbq*d6%r;*DZZNq( zeAg0u-z=1r1^}#en;7oFnQ_{OG)L=W2$9SJa0`lBy+9B)H8Z<{29?y_m5cLr$&uk< zT^7Qn3J(>e*(9ooZwG_+$cth))-(uZH75s>*gZTvNQ@(HXkz8W!=veGg5l8TNCp7q z2B8hdpagtBSXTX_4)|FEv*LMMJ;7ZtABX?M44nkf08Za+$YAOGhY*s)1zTRW?{Z`> zb%4`>n7{Yt_??tN+3)#zd1wfbw~0l5Y&IK7B||^BEvtF*79njX150=XY03?d3QB>< zVFPq}4qye&ls_Orn@=9b50jKR3%-~+3pO4@VCn+m0->t3>};gJJCgs5yyIrv>#2h& z+}R>tQH7fl_t|*%Xb=EZsa|NngTPxYf8|Itd}+%RB>?DwDQj=&wGu4+XtqV9MZe zWc04Nvqs$lQPmutJ^Imxh=<`QT%LRq9iZz5MjTbe#aKRccBKD_phWw+)n9L9pV+lK zIyj7PdIBTS6zEi-S_SBZbT_Ie7X}GwA%b|}{sUX8fr>}W&60^69z_|Q@DK-S^#EphIkKaPG86Yxb z&k@oZ&}Bh{(nZV@F3ZBnd4Ys?(97Z2FuS_CN*7cOn-@eT;JYo3m>)J-h9_Q`o+0K3 zLxGC8j}L3HBmFI^S+Az@+?S9F-4%jH!x(u;ZJ@~ssdSeF)&*&4tsoEh-l!Ti2e}7;i)IMtv(J&Uc>rC%W33&YB9O-zTquO_ z2qHjCQ;3fMpdqp2h9QBpoeFZgR)R_(SxnCj3c>cjyKP^zywZa}hKMC;07#g3>%5@o=5cR1Pu`j47#vtTfRTt*djt{hx20KqyX&X$7_#6)kPQl_*K6TOKh(;l z*Ai5J!<8^SJAR`@k$}Lypdv?j9>~?lupGb(LJZ{I;78#(F=8$M5JwxqrVqm>$ANvS z97tuzMTyKYWE~y2#A&>HWMVik#XOhF{PJ~5YpAGaM4c8ViM37`AybcyH32J@`YB4I}Zh9^H%o(R5*y*q2lt zgE=zodlg4SNtJ|Nl`O^TaW8<2j%4v8m4n~}4iLnYE4Q__4zJ;XF^Pi;TMRZ*cNqdx z_E!qhL95f5>9ylpG9Q|SHD_C+dejiXFv?!`j|p;b#4LykY^~Yb8-Nw?K_cVnk6F;2 zb=%P%oLDtzF*ry}EFa$`Fe1>@>~`LAs94BIXBrt88cZsBu`J_vlnZH`$rMbvPy_F+ z!*Q8hFJgqAtcziqn2?YV`yEzQ=az2^UpC9N2Ony|x4~;B8n<9_auN>9hKQq3uzqYQn)_wg) zVvlM%n=yhaX{&vfzr)&gO)c8`ArAVC9LA_&3wSkc4P+(}B_)rSHxH}5c!Byt$)HH> zc4%26ftFyRi$dfJ#Kun*tzY%py1Kg@Nj{0CM==NlRkGJX`&s_D;ovYF0RuPeb0{6B z9x%O*QFu)!jio9$rw(SbIC1@TaV5A8QvK1^p4U3je)6bt3cA`mh%*eh=v z8qnU><>l2ijBTPKUL?5crx325QgAk8=GNy+(JCn^N7*?$wckA#=i+2%mt`C%`;vi4 zYuH4`UTv1agCk-nig??t)mkTTM?I7O{L!Os0xGHt03~r7tPuGpbk*qEsn9Cc0(UY@ zot_XoCl7hOk!Q~`xuSzS3U24-(i?8Kd8ivFi|A$lO=MBaP69_A5!bGW8?JOPtwpMe zT84**yOPs=(2&z$i0CL&Q3e;Xteb~{&>3cE?R#BzekKjV zsqt~%56qi*!cU&5+U`8BH;%S@C|hD$^*!E8{L@Ek9sX8{s`>cInunQLj8$$JKI4pj zhKU~QY9t6L`JG{Zg~6mh6p0z>YX{?4UiQe(Ww8HDH7%{nPIyq+(&7jBgW(Br7=GGy z9>biwOGx>QQ2`HMiBg%Np0ItGSkv@ejt`wTu&)0tmK=6`WCCT3jb&^toh_NAY|AG6 zF79;+`?(m(JlQPVUB`fEJd{dZvKkKj$1U>^>xcbkEkIqIpGgIxv+d~$IA}wqbD$#t zdrUL@pq<0^;(2-b@NjG_9)1a@=oEc$Y|p^BmNNUCVcB=%8W$7O=|WYa77Cfi1&XW( zVz2Nyl85|H%RUtQd!>n~kz!O-#2aa{#UT)O6`U2-!kI@g9CHSxrZX&r{WB$NAMQIS z^ec^DJuNkBpQ{U3IG-B!e=;H0U9e|4}DOYjt}(jDYGW zT}WRhea(+3sbX7jHFB9A0Ct%43v&}93N|gJLutNR@oKR{v#YZiD3(NhhYRlSk zIaW+sd3ia+CeS&;F&`eL5h0t|_R!RT?Iyp!mxywm)}+R)LqF9k1H$L)@T5QqDqe$y zKUHO^5@P`(e6Avo59M#}86Y)l|2J&&-`QdE2aV9o1L)Ms$_nfgu3bpT3kgoSD)i5N z{jQQlV4r@)?l7U2WBUDe9=(iWZt;uh8n?$BgOJcV2BZvGbzC1#xZ%gF1yDBY;ds2t-5Nl8`i64-#jOknHC6Tmy$= zfL{QpVB&m-#v#a>sY{$%HAh393mcM`2))yn7o zeuyE#b27+MFrC}D8Gm={!!MX_G|=x1{fv?jEkWOP3Z%DBkXBM>TkHi|_ngTkszaf; zg6`UEcm7HV;xX;289f~xcmO+4yXZur6&HU3C?1MFEf8~mFj*|lwfBbzh33icSD!3U z5GC4Y5E|WMX1-=y20yd9E8JzYdHwM8cz40E1`MsD=XPSFqd~VOhzXgaJC(_---Or? zS&D#o11wAOz)pclM8uFQ1)=~!=wg5p6*BF94d6cYlC;nY0hi}9CJC+-r2i5orX7t$ zpfG;j!NI|K&=oShSFZmUW=sV#_>3ZgIfd<0bH5b=nuiY`K6vnK!Nt|i&d$lH9a7a% zMC{0T2f`vqoVssm|D2b{%DD>(rZf#?IDj-~`9xv{?@WMhM12kz)r!(moOX76EPv*T z6Yxsb6@3A~l^@`MkPr>zRFLrmQVJfDO>f@3foPeQf#E^fRHb3V7E%k*Clv~PXOQA% zDugF42|pF-h=j<1^#+bukV$ujgm>VHn}IDjXi< zP4AFPG-x*zBtc{yGTz*f0EE7|u4oBMGN=WdLD&YVTgbYi4{sd#qq%AlHb71q zqC$lNtyc;%nv+ljlG_%AggA>8te^fDgKLHP$k*?$K}#RP;TQRdQTuR1F~i27(b4>w z`UMZnwJdxl;%LPL$gA&ziJC%!m5Rz|2-1Nr{IV@%j#CI?1WU+JLjD{ZQsq!p zWe+TcKmKzq<#7?UX0xo#UcWx;{4ckuJq}go=k{=9@ttcJ;zdpN1D~)=a zxO|`b!*vDc3lP=y4dpD7;s&NxH+@)F$o|a@7AN!s$|UKJ*gj+^=y%wmk6?+*vK^|S za7*rcBHKSznxmy|BeR_N(yv47jEyYP(Z`|+wqNRayH?XMOB)QXhT$&I!U3_@jtA66 zkJk}p?D>kDw^Orxq6VFv+`U7T3KHX898+^Xe__Ogb+m87dCa}#0KQ3_C^YTH%D`T(=ui4x4a7uSR2&H3icb#D zxQ%RC#*#LY!u&d>#q@8JvTE8yw=BB6ZE>ISFL)uE$N+s1Hp@pg3|acGxQm|i;0)Tc zOo}8lV4Boa%4BC_NhEjQL7?;M*w9zkKU-qwPwT>BOJ<)URq_1(o16Tts4Rdtssa>!}-7wH!J!kx7OHc3lkfOg_&Rx>X6UK_|!^ zm=(eSCMDWT1lWuYsozbzf z*6PpSQ$lfW*ylTHzw};#_J9N5b}4_->F+LV)05zq6<2U#Y#t%OABh-orhX(viF)oP zo_A^>=@uCj3wNljM-d}P-fSG(yM$Ir1I@67bcXLc7uYnAH6yqR1FyJI>4dh?hEqwy zeHJ~u1UKP@4tL6$S_S5|Ocu|-^3ADI^r}{>0$$%HwW&u1(<8X2Ui_UKbDa9Itsev8 z3+UA(22P$MMjuI2FCJMyCJ4DCmlgRYS{$Okei)4;!$A8;6*qAsuq61}Sm3QL)qqf% zW%1t()!o}n`H9;}(=nJ8J<`G1>~j`<$-e^xxY{OqM+lBKPyVa(8S%b)I0`e&5?LZv z#iR^HlZY#t$wv>9p<_;eDH5SS=E{CExM|pUBJ0g-2Fyi8BHrB@re~?|vI1JKlqn7; zfQ9RMCBI?a#c+JGrOu=O51|7^8ZVhqj)G^k_CLXnWKzJSC&Yh@339N23f8 z*Kb&`a<+m(b6)Jo%e7Vn}*CK(*R#jY|D_xi`qQEU?b=uZ!brGa1u*DyQ8y z4Jxfp1_M$0XI)0F?=*FRY45zXA-wR@zRl8f;K?C_{BAD+C>Lx)BY5Z8RMqpp?;+ax zoNLR>FR3FB)MODDJ*S&$vTed3kJA2zc&O0HwuGN1=L@m=ncUS&F;0L z|6;|UNoYm>XR_#;mJzL{)kyZdxa+8GJsc#mg>`@Uxk}r6W`PCL&QEzCQQhK&dBmNd zjt0B1)%=u>>NsU;XXQs}HZQc}`WUKwXhH@tkQG{~hX8DuEk^Yh@y&^-HxH}*OA#3dvbbsrM>j*0C|K3rFsa8S_nH1!V z<_)EMpsM!N6`wx5(Kv@{$KT~PE9_Mj~n$!bYu{fVc%|cVsUm z|EzC2L8{+BZjGup^ps&>Se{|m0hb7hYmmqu1U`otGt@T82$@d^v>}wNG&w3f& z!Wpl7AiD&|ZIACOyTBGGyHA!d@N(dQI>P_FkxR1xABx$&q+RUu&6F{e*S984uA3^U zOkiN&ns`PVBjf+7MUh>UB$b1BkhVJ^G#H(v-VCZ^EhM58L*;_6=bnP1X21OPW^pU# z3`;-1qlnIDo7PMU{)_ybv*C`PiwJF*hZ^jdTv*0alq~O9U^CJ--v0i2f%bSanHST! znCtdURjZGUg+f{Dsx%QuWHZWzMI=oZv7RM6_gUZi{bTi9^%)jxk2FamuE?dK<9W4? zH$yLCEWEhXd|Fx?W(!xwOol~tXO-CnhqBQ2)Ynkm>f`pltoP{>`HfjoX*;+)bSF-d+K9mnIU@h;F5& zaSY;CD1dbfPUczQeM27k+t}j{ZyF!i3F~mI<>u=Qk_@n`@erU7)~2rD-f^$v+edin zvL|S}gej@A;(adGSuJ%iE8XI2&rwKd>@i+1YiVHb5aFk8qs`c|Ij*^^UolPo zalK4&BTc~mUHOC7M7mZjzVLI4N9_ex1+8&f%bpxB*4OzoC4cM5D@I}ScM-T5;yFz@ zvTk~YTa_J5K3o_bJHQyqN%Rm+2IWwj9#+q-q*4nJalO>kN{+Y5k)qm;lv6=-~04~<=SV9K*QOn^USFPMhd(e8mN`l zzy4gB{)hA=f9b~idUQ(n$}Gf}tDwY&I{_@6J~kwl>nD&r)V#}|q+EQ1lO@^q1HwhQ z-{!!tl*=k?U)$pTH{&zuyoVM>50fsxf6|sIX?k5}y+6ue@wt}NjUcE?3Y&ZHUDi~s zomR&6fuop;A|>W6CCkHK7RF=s`tLI4`~=91*nSXdO%6PklqyTS~+cv|`#N}aHL*ZBm(of1g)III~E*c!xjV`RL z{rko&_#=dGD^g>H&D73NfSP`^!NB9pRJ2!+MtFoaQQL37nIR}7LC2)sFfZtT+d|^n zKy0`4vn}Y?d6QW;q0~Tws8D*hJGz=9Det2*khD{;o0*HqG!!w|{ZqE*`CdFlofx-Y z>Pk#eP3^PndjaJZV{s1lwPG5CVL^WnL1n`S?E5f{|MqGsA%d}LzT7j)&ydf#V*)g`j&Tr~D!gAw2xPD23WVAgvM~`&iyz`W5Y?isNopVe-h{?$G#* zL0J@94x3zbM6jC}Mh|7M%L9`~OZlYa8ilmFdb*g`4N!u4k|O`OD*EA3Hb!7;>&{#^ zTh)5{HWHJeeZtt@m3k%2SBwK|UpnX{eroY7WHV}*zT~5Bk~A;Hx=)pj-p=!Wx$dPP zF1)bN_Br0^#yAOGp3FYRzbLUz+l<{t z`(|U`XqMU8#H|FX&F#K7t$pbX>Sa2Ir;y&C*~iXP?+mYADPMSd_P*BXZ3^nmwgjAr zv`-7m42YYe#BZ;K;rTj-BBW*}O9i^=GHy;Aii4bwNb(wDB!J+cJ^0_&ryPuPAE(fn zFPJ$Y+R?~+gajw^aFOf1e&WP7JC){jshQ$*d{xPY=CY>`^Ac0-qZ>*|5q3IhUe1SC zk5Xg~`O6Z9w(iQNViTA*G;;mg7oi~;QVQR=bZNBKzZiLeAGfbm_szYVZaGa^RbJsE zrcGn4;OnH*Q^QB|H(kW5$Syb*Zd%K?dS4I-)4jl#-Y{?zC4T0SbIGkhBPFRTCc0Fj zjjz&ixyqC3xD^vK=DRgMtufg3szUsr_we*h-jM3L?V(4saU-v!fUg&UW4aSRY0xq) z5$fWXzZCu3X{yoa49|Yx`|z;sEq*x;>Aik!=4NOYn$ckpOF~%PDII-mcLuxpCo=uVg;5DPK#$6R{^G{TJ?bICuF z6H*MN*^<&!RMgaZP*NA)2YmI{X;Om1QDJ2Ffm+#=SBkOdhcUOQW`- zNVnnjnC5b0Iwwh&b+SNU=9Fa|{);;wJ{I>K4ck2Vdp!}hICAw*_Z}qIPp^o3eHp3A zBlgYl&!@^v zGr80B>XK0W%75cGr_RO1naQB>ScAHQP|1lTPA3r-qRU|EbLXbsYFv^!g|l2xb$a0^ zwGQ$Q{Kd_iLylAITO>>r(WYXey!p?O6-p?uySuyh%HLD%j{KhC<6>N5I^{aAZ(w@) zdv+N|l$|G#ExXw>ZTeE7!E%AlUa#qtl6@wt&6;8uhu@zbedhxY#mk2?B08MkB69e_=F}P4EO3TvBcEn z@0GX#A|10zp^8^#G1&{nW(K3_+kX&P;uPc*(+i01t(%?4+fSQ?8-pISoNJy&m2wM_ zHw9sJkck`~_Db~Dh$`y=nosF$JtawB=m=Op-idjo_rOy0WgEeGTSGaA@slrAxHE?5 zuFLtSGVd)vY-A>Ft$i;Uyb;~r^VKY7!z(?)b9mZwJx9_~E z7!-Zy7~#E=mXskKtySCUHe{`gQ8P-=5Z?W3;O-=nzr#T+2^xwQroIQO{m=@b6!9)5 zc<>8>G?riQk&W#U!)hh5Y(FV~-p5bhqXl(O&OdvjdYoPNeKMl6^<-J^;Cnaazx3YkRx*Mq90jWSC>>iE_=lMXGO5m>*V1=~mgPiM7qmNKotjiz58qI$+v?6a3rssBuY|Z*VL)?po00z%W=DvL4 zxxP|d_ND*q%pbU5KV)-B27x917s?E_W4y>D*jdS6VA5Pc_W+d_QQM#uue=GWf62+k z$m*IJa<}{a@DJEq`_@wY_1K#>a)mT=Oq^ur&YRJ}(oTxMb#`Pk|A|5T{wo0+Z~HH9 zgNO0|^bc8dU)7{kQv?l7B89T7OvMCY+%_!tPae^>iT6n>RsWBssZ!omj@slPv|uqb)lr_VBNB zn^E@Q6skdq7*&X6X8D_(&+pP_U0uZZpW4wg$jN6Pn$UdUd;k-N>KYQ~?~g7u_7;wn zq}qx7EVLoVBZ&c%BYw&p7n8(Q<2Sxqo0=Y~9ZTp;Xb89RS6=W~i^_5os1(iAiauB0PMkNgngPQ)i*y=ljkELNC z#RP3px@|tqjdHeHj_r>6QX9FL0ToR_TJoEo6|qB<&sj(Ol-J zdth(#RS->rZTNH4qbqMqhdKEc>h1lKujev|@;fbBjz`Ay>gr{TX|F!=cwYIuJyaC6hamSu>?Q2`fO>gPi{eY#-8UhQOx+{grqGArO1gI+N|smnWk;F66d#s zls0^797HEc%o6vPzx&j0Hach-_y}hS>kb-jhugB>BAYy|vAJ9Hg>o&};HA2<$88h@ zVxv5n@vEC%@GUpafX+XE{u7*vS}2D*MG|V?Q$sN>OHi}EQ%aP3+rn5g>GEF4V@;>& z-}Qmf-yoT?-&`0aWzr9bfY)tJs)Xvs%y+}sE#yzGW23Y zRNl*0TAvyXWl>pm?`Y(Tu^IHZaMf~3a_5uF)fx)Jl4{Dz9$0(Qu%%7QR`?;Qb*#SG z)B8Ca=5>3wh|7)pjO@0#=*D?(`UavSi%En38^34k{WpHU|G(mQ%nvB|OPj;LS^{hJ z6)?}%72Vw}xNs^B2G+J78L-MJ2z5aG7FzB^*W!4 z^G}h74+M+K&&SG1y>3*zn@D#bLBy!{_#$isr;o+TCLQ=q=x+V2BF2#o4QrU4v-Xe{ zqb3gXrITE`hF{3Lf9aRF-t`C%;Yk5;C-bM#2{geFD+4owzOab-FY;$8qxrl@J+Y9+4iu9)Z~xSJ- zeAp>x(^)``J#==8;2Eq9F~t#6;n5cUdCrMIoM3fz4`16r$w4!%W<-oa-+L%jC;Oi7*) zZB$^3NoJxZWM26r!ZBY?<^7lW{V)*|y^|NX@Py>bY2) zKAinAY_LoBt@ekW&^%Ng4U%UAsf!XJ}j_B)&PI=iEdqEyKxqju* zPwyU{3i=e}4V;#p+zC7$^2Unf&5fcH5nY0D=vIYqo4VRmOfz$XAsm`g+vg<~8fgs(^PoEtK1OoNar- z9QilfZZa}#Pu{v+-e_J|ahp!&+T*)FZ&@HwvtBPg zHt$YEi2c^{&+Y!a&%bn;;YrSeE9Tu-Ps^`9efIYEx)*m1^VjAUEHppYpL-`@-TghE z-*3;?dwom&)~pZy^8ZfT|7*Xz;jrq#18Dt-Y4*TNNIzEm0nSD?^GACxt)KcNyN4^e z=0!~cun(8}vfKCdxwrP_v(wrc|5tVJ#c~@n`33Lav>@*OmKAlI?)lVU=3egB%<_1499 zVp5r}ukZWw?~nbzZ-;IKm;ZhEZfCts_*{A_@LxLQqT6+z$K~w z_pORuf8XH$dI8RUOE=%JuzeeE?%iF!U%v8dO+rgt59{04(^pq5URWQL-S=nP2mAlo z9Sp|V;a8?cOYi;9uH3kw;Mtc6Z0&vv1^>NSIs3ZabnW?diT8xheUk!)W5)-#A6w)9 zA3nPAtMsX-{ini1YIk1T@zwdo*$Xr0Uv{@Yh@&%cIG@eIp#pfbyPI3v-wB60L0f9~_X3ybSmgn)Kfj;# z)=bUl1aP7M56~5}z_n6wVaX!Gk=kp+zwcO;{{9|tX*zK6=EskP{kQL4iJdz?vh%F? zvERyZ>q1X=3Z1$3v@`j`y_??WOtmdr#g?Z0Pd^6jPq424_PF8_*8ar5g2zT>z>IqS z*v3Cq)|;wn z|2_?)QxOB{R8%;A{sEqbexa^1CpzK`uVq?j%Joe*uc-;HTN)Zx)0y&-xizl$<6bONsW5D5l`%K_w zzdyjuX;~7-0>w9}e}8}fKX9SEPu!!q%PxHW1X@pGQTQlwOAn|zsk0n$~^WJfGLP3*2)v5j(AF?%Yd83_#%N>gTe~DWM4f0AGpO diff --git a/paddle/contrib/tape/function.h b/paddle/contrib/tape/function.h deleted file mode 100644 index 8c9694d9a2..0000000000 --- a/paddle/contrib/tape/function.h +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include "paddle/contrib/tape/tape.h" -#include "paddle/contrib/tape/variable.h" -#include "paddle/fluid/framework/type_defs.h" - -namespace paddle { -namespace tape { - -class Function {}; - -class Fill { - public: - Fill(const std::string &initializer, const framework::AttributeMap &attrs) - : initializer_(initializer), attrs_(attrs) {} - - void operator()(VariableHandle var) { - get_global_tape().AddOp(initializer_, {}, {{"Out", {var}}}, attrs_); - } - - private: - const std::string initializer_; - const framework::AttributeMap attrs_; -}; - -class Mean { - public: - VariableHandle operator()(VariableHandle var) { - VariableHandle out(new Variable("mean")); - get_global_tape().AddOp("mean", {{"X", {var}}}, {{"Out", {out}}}, {}); - return out; - } -}; - -class Linear { - public: - Linear(int in_dim, int out_dim, const std::string &act) - : w_(new Variable("LinearWeight")), - b_(new Variable("LinearBias")), - act_(act) { - Tape init_tape; - - std::string initializer = "fill_constant"; - framework::AttributeMap attrs; - attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{in_dim, out_dim}; - attrs["value"] = 1.0f; - init_tape.AddOp(initializer, {}, {{"Out", {w_}}}, attrs); - - attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{out_dim}; - attrs["value"] = 1.0f; - init_tape.AddOp(initializer, {}, {{"Out", {b_}}}, attrs); - - init_tape.Forward(); - } - - VariableHandle operator()(VariableHandle input) { - VariableHandle pre_bias(new Variable("linear")); - get_global_tape().AddOp("mul", - {{"X", {input}}, {"Y", {w_}}}, - {{"Out", {pre_bias}}}, - {{"x_num_col_dims", 1}, {"y_num_col_dims", 1}}); - VariableHandle pre_act(new Variable("linear")); - get_global_tape().AddOp("elementwise_add", - {{"X", {pre_bias}}, {"Y", {b_}}}, - {{"Out", {pre_act}}}, - {{"axis", 1}}); - VariableHandle post_act(new Variable("linear")); - get_global_tape().AddOp( - act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); - return post_act; - } - - std::vector Params() { return {w_, b_}; } - - private: - VariableHandle w_; - VariableHandle b_; - std::string act_; -}; - -class SGD { - public: - SGD(float learning_rate) : learning_rate_(new Variable("sgd")) { - Tape init_tape; - - std::string initializer = "fill_constant"; - framework::AttributeMap attrs; - attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{1}; - attrs["value"] = learning_rate; - init_tape.AddOp(initializer, {}, {{"Out", {learning_rate_}}}, attrs); - - init_tape.Forward(); - } - - void operator()(VariableHandle input) { - PADDLE_ENFORCE(get_global_tape().HasBeenBackwarded(), - "optimization must happen after the backward"); - Tape temp_tape; - temp_tape.AddOp("sgd", - {{"Param", {input}}, - {"LearningRate", {learning_rate_}}, - {"Grad", {input->Grad()}}}, - {{"ParamOut", {input}}}, - {}); - temp_tape.Forward(); - } - - private: - VariableHandle learning_rate_; -}; -} -} diff --git a/paddle/contrib/tape/tape.cc b/paddle/contrib/tape/tape.cc deleted file mode 100644 index 531499b6fe..0000000000 --- a/paddle/contrib/tape/tape.cc +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/contrib/tape/tape.h" - -#include -#include -#include -#include -#include - -#include "paddle/fluid/framework/data_type.h" -#include "paddle/fluid/framework/dim.h" -#include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/framework/operator.h" -#include "paddle/fluid/framework/scope.h" -#include "paddle/fluid/platform/place.h" -#include "paddle/fluid/pybind/pybind.h" - -namespace paddle { -namespace tape { - -// borrowed from -// https://stackoverflow.com/questions/874134/find-if-string-ends-with-another-string-in-c -inline bool ends_with(std::string const &value, std::string const &ending) { - if (ending.size() > value.size()) return false; - return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); -} - -std::ostream &operator<<(std::ostream &os, const framework::VarDesc &var_desc) { - os << var_desc.Name(); - os << "[" << var_desc.GetType() << "]"; - os << "[" << var_desc.GetDataType() << "]"; - os << "{"; - for (auto &i : var_desc.GetShape()) { - os << i << ","; - } - os << "}"; - return os; -} - -std::string to_string(const std::string &type, - const VariableHandleMap &in_vars, - const VariableHandleMap &out_vars, - const framework::AttributeMap &attrs) { - std::stringstream ss; - ss << type << " "; - for (auto ¶m_name : in_vars) { - for (auto &var : param_name.second) { - ss << param_name.first << ":(" << var->Desc() << ") "; - } - } - for (auto ¶m_name : out_vars) { - for (auto &var : param_name.second) { - ss << param_name.first << ":(" << var->Desc() << ") "; - } - } - return ss.str(); -} - -framework::OpDesc CreateOpDesc(const std::string &type, - const VariableHandleMap &in_vars, - const VariableHandleMap &out_vars, - const framework::AttributeMap &attrs) { - framework::VariableNameMap inputs; - for (auto ¶m_name : in_vars) { - for (auto &var : param_name.second) { - inputs[param_name.first].emplace_back(var->Name()); - } - } - framework::VariableNameMap outputs; - for (auto ¶m_name : out_vars) { - for (auto &var : param_name.second) { - outputs[param_name.first].emplace_back(var->Name()); - } - } - return framework::OpDesc(type, inputs, outputs, attrs); -} - -void InferShapeAndVarType(const std::string &type, - const VariableHandleMap &in_vars, - VariableHandleMap *out_vars, - const framework::AttributeMap &attrs) { - framework::OpDesc op_desc = CreateOpDesc(type, in_vars, *out_vars, attrs); - - // Create a temporary block for compile-time - framework::ProgramDesc program_desc; - framework::BlockDesc *block_desc = program_desc.MutableBlock(0); - PADDLE_ENFORCE(block_desc); - - for (auto ¶m_name : in_vars) { - for (auto &var : param_name.second) { - *block_desc->Var(var->Name())->Proto() = *var->MutableDesc()->Proto(); - } - } - for (auto ¶m_name : *out_vars) { - for (auto &var : param_name.second) { - *block_desc->Var(var->Name())->Proto() = *var->MutableDesc()->Proto(); - } - } - - LOG(INFO) << "- " << to_string(type, in_vars, *out_vars, attrs); - op_desc.InferShape(*block_desc); - op_desc.InferVarType(block_desc); - for (auto ¶m_name : *out_vars) { - for (auto &var : param_name.second) { - *var->MutableDesc()->Proto() = *block_desc->Var(var->Name())->Proto(); - } - } - LOG(INFO) << "+ " << to_string(type, in_vars, *out_vars, attrs); -} - -void Tape::AddOp(const std::string &type, - const VariableHandleMap &in_vars, - VariableHandleMap out_vars, - const framework::AttributeMap &attrs) { - InferShapeAndVarType(type, in_vars, &out_vars, attrs); - tape_.emplace_back(type, in_vars, out_vars, attrs); -} - -// Temporary Scope for Operator::Run() -class ScopeWrapper : public framework::Scope { - public: - ScopeWrapper(const VariableHandleMap &in_vars, - const VariableHandleMap &out_vars) { - for (auto &v : in_vars) { - for (auto &vv : v.second) { - if (!vars_.count(vv->Name())) { - vars_[vv->Name()].reset(vv->Var()); - } - } - } - for (auto &v : out_vars) { - for (auto &vv : v.second) { - if (!vars_.count(vv->Name())) { - vars_[vv->Name()].reset(vv->Var()); - } - } - } - } - - ~ScopeWrapper() { - for (auto &pair : vars_) { - pair.second.release(); - } - } -}; - -void Tape::Forward() { - LOG(INFO) << "Starting forward -------------------------"; - PADDLE_ENFORCE(!has_been_backwarded_); - while (current_position_ < tape_.size()) { - OpHandle &op = tape_[current_position_]; - - // Create Output Tensor, this is only necessary for OpWithKernel - for (auto ¶m2var : op.outputs_) { - for (auto &var : param2var.second) { - var->InitializeVariable(); - } - } - - framework::OpDesc op_desc = - CreateOpDesc(op.type_, op.inputs_, op.outputs_, op.attrs_); - ScopeWrapper scope(op.inputs_, op.outputs_); - framework::OpRegistry::CreateOp(op_desc)->Run(scope, platform::CPUPlace()); - current_position_++; - } - - LOG(INFO) << "Finishing forward -------------------------"; -} - -void Tape::Backward(VariableHandle target) { - PADDLE_ENFORCE(!has_been_backwarded_); - - Forward(); - - // TODO(tonyyang-svail): check output of last op is target - backward_tape_.reset(new Tape()); - - framework::AttributeMap attrs; - - // FIXME(tonyyang-svail): Need to infer_data_type - attrs["dtype"] = framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{1}; - attrs["value"] = 1.0f; - backward_tape_->AddOp( - "fill_constant", {}, {{"Out", {target->Grad()}}}, attrs); - - for (auto it = tape_.rbegin(); it != tape_.rend(); ++it) { - framework::OpDesc op_desc = - CreateOpDesc(it->type_, it->inputs_, it->outputs_, it->attrs_); - std::unordered_map grad_to_var; - std::vector> grad_op_descs = - framework::OpInfoMap::Instance() - .Get(op_desc.Type()) - .GradOpMaker()(op_desc, {}, &grad_to_var, {}); - - for (auto &op_desc : grad_op_descs) { - std::unordered_map name2var; - for (auto ¶m2vars : it->inputs_) { - for (auto &a : param2vars.second) { - name2var[a->Name()] = a; - } - } - for (auto ¶m2vars : it->outputs_) { - for (auto &a : param2vars.second) { - name2var[a->Name()] = a; - } - } - - VariableHandleMap in_vars; - VariableHandleMap out_vars; - std::map - loop_over{{&op_desc->Inputs(), &in_vars}, - {&op_desc->Outputs(), &out_vars}}; - for (auto &each : loop_over) { - auto &vmp = *each.first; - auto &vhm = *each.second; - for (auto &p2a : vmp) { - for (auto &argu : p2a.second) { - if (name2var.count(argu)) { - vhm[p2a.first].push_back(name2var[argu]); - } else { - PADDLE_ENFORCE(ends_with(argu, framework::kGradVarSuffix), - argu.c_str()); - std::string name = argu.substr( - 0, argu.size() - std::strlen(framework::kGradVarSuffix)); - PADDLE_ENFORCE(name2var.count(name), name.c_str()); - vhm[p2a.first].push_back(name2var[name]->Grad()); - } - } - } - } - - backward_tape_->AddOp( - op_desc->Type(), in_vars, out_vars, op_desc->GetAttrMap()); - } - - // TODO(tonyyang-svail): how to fill empty grad? - // TODO(tonyyang-svail): Sum var grad is necessary - } - - backward_tape_->Forward(); - has_been_backwarded_ = true; -} - -Tape &get_global_tape() { - static Tape T; - return T; -} - -void reset_global_tape() { get_global_tape() = Tape(); } -} -} diff --git a/paddle/contrib/tape/tape.h b/paddle/contrib/tape/tape.h deleted file mode 100644 index ed79de17a7..0000000000 --- a/paddle/contrib/tape/tape.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#pragma once - -#include -#include -#include -#include - -#include "paddle/contrib/tape/variable.h" - -namespace paddle { -namespace tape { - -using VariableHandleMap = std::map>; - -struct OpHandle { - OpHandle(const std::string &type, - const VariableHandleMap &in_vars, - const VariableHandleMap &out_vars, - const framework::AttributeMap &attrs) - : type_(type), inputs_(in_vars), outputs_(out_vars), attrs_(attrs) {} - - std::string type_; - VariableHandleMap inputs_; - VariableHandleMap outputs_; - framework::AttributeMap attrs_; -}; - -class Tape { - public: - void AddOp(const std::string &type, - const VariableHandleMap &in_vars, - VariableHandleMap out_vars, - const framework::AttributeMap &attrs); - void Forward(); - void Backward(VariableHandle target); - - bool HasBeenBackwarded() { return has_been_backwarded_; } - - private: - bool has_been_backwarded_ = false; - size_t current_position_ = 0; - - std::vector tape_; - std::shared_ptr backward_tape_; -}; - -Tape &get_global_tape(); - -void reset_global_tape(); -} -} diff --git a/paddle/contrib/tape/test_tape.cc b/paddle/contrib/tape/test_tape.cc deleted file mode 100644 index e9bfd21a71..0000000000 --- a/paddle/contrib/tape/test_tape.cc +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtest/gtest.h" -#include "paddle/contrib/tape/function.h" - -using namespace paddle::tape; - -TEST(Tape, TestMLP) { - LOG(INFO) << "TestMLP"; - Linear linear1(3, 3, "relu"); - Linear linear2(3, 3, "relu"); - Mean mean; - - SGD sgd(0.001); - - std::string initializer = "fill_constant"; - paddle::framework::AttributeMap attrs; - attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; - attrs["shape"] = std::vector{3, 3}; - attrs["value"] = 1.0f; - Fill filler(initializer, attrs); - - for (int i = 0; i < 2; ++i) { - reset_global_tape(); - - VariableHandle input(new Variable("input")); - filler(input); - - auto loss = mean(linear2(linear1(input))); - - get_global_tape().Backward(loss); - - for (auto w : linear1.Params()) { - sgd(w); - } - for (auto w : linear2.Params()) { - sgd(w); - } - } -} - -int main(int argc, char** argv) { - std::vector places; - places.emplace_back(paddle::platform::CPUPlace()); - paddle::platform::DeviceContextPool::Init(places); - - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/paddle/contrib/tape/variable.cc b/paddle/contrib/tape/variable.cc deleted file mode 100644 index 5ec1612909..0000000000 --- a/paddle/contrib/tape/variable.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/contrib/tape/variable.h" - -namespace paddle { -namespace tape { - -void Variable::InitializeVariable() { - LOG(INFO) << "Initialzing " << desc_.Name() << " as " << desc_.GetType(); - framework::proto::VarType::Type var_type = desc_.GetType(); - if (var_type == framework::proto::VarType::LOD_TENSOR) { - var_.GetMutable(); - } else if (var_type == framework::proto::VarType::SELECTED_ROWS) { - var_.GetMutable(); - } else { - PADDLE_THROW("Variable type %d is not in [LOD_TENSOR, SELECTED_ROWS]", - var_type); - } -} -} -} diff --git a/paddle/contrib/tape/variable.h b/paddle/contrib/tape/variable.h deleted file mode 100644 index 35c328e69c..0000000000 --- a/paddle/contrib/tape/variable.h +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#pragma once - -#include - -#include "paddle/fluid/framework/operator.h" // framework::kGradVarSuffix -#include "paddle/fluid/framework/program_desc.h" -#include "paddle/fluid/framework/variable.h" - -namespace paddle { -namespace tape { - -class Variable; -using VariableHandle = std::shared_ptr; - -/* - * Combination of - * framework::VarDesc desc_; - * framework::Variable var_; - */ -class Variable { - public: - Variable(const std::string pre_fix) - : desc_(pre_fix + std::to_string(count())) {} - - Variable(const std::string pre_fix, bool is_grad) - : desc_(pre_fix + (is_grad ? framework::kGradVarSuffix - : std::to_string(count()))) {} - - ~Variable() { LOG(INFO) << "Deleting " << Name(); } - - // Instantiate LoDTensor/SelectedRow - void InitializeVariable(); - - VariableHandle Grad() { - if (grad_.expired()) { - VariableHandle new_grad(new Variable(desc_.Name(), true)); - grad_ = new_grad; - return new_grad; - } else { - return VariableHandle(grad_); - } - } - - // Stochastic Gradient Descent with Momentum - // VariableHandle Momentum (); - - // void init(const std::string& initializer, - // const framework::AttributeMap& attrs); - - // void value() {}; - - const framework::VarDesc& Desc() const { return desc_; } - framework::VarDesc* MutableDesc() { return &desc_; } - - // TODO(tonyyang-svail): No need to expose name - std::string Name() const { return desc_.Name(); } - - framework::Variable* Var() { return &var_; } - - private: - int count() { - static int counter = 0; - return counter++; - } - - framework::VarDesc desc_; - framework::Variable var_; - - std::weak_ptr grad_; -}; -} -}