From 16ed4a92a5e441352582ff55ce784a78a2734b1a Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Mon, 27 Nov 2017 18:27:34 +0800 Subject: [PATCH 001/158] Add math function for sampling integers from: 1. uniform distribution 2. log uniform distribution --- paddle/operators/math/sampler.cc | 47 +++++++++++++++++ paddle/operators/math/sampler.h | 88 ++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 paddle/operators/math/sampler.cc create mode 100644 paddle/operators/math/sampler.h diff --git a/paddle/operators/math/sampler.cc b/paddle/operators/math/sampler.cc new file mode 100644 index 0000000000..52628c3b03 --- /dev/null +++ b/paddle/operators/math/sampler.cc @@ -0,0 +1,47 @@ +#include "sampler.h" + +namespace paddle { +namespace random { + +Sampler::~Sampler() {} + +UniformSampler::UniformSampler(int64 range) + : Sampler(range), inv_range_(1.0 / range) { + std::random_device r; + random_engine_ = std::make_shared(r()); + dist_ = std::make_shared>(0, range); +} + +int64 UniformSampler::Sample() const { return (*dist_)(*random_engine_); } + +float UniformSampler::Probability(int64 value) const { return inv_range_; } + +LogUniformSampler::LogUniformSampler(int64 range) + : Sampler(range), log_range_(log(range + 1)) { + std::random_device r; + random_engine_ = std::make_shared(r()); + dist_ = std::make_shared>(0, 1); +} + +int64 LogUniformSampler::Sample() const { + // Got Log Uniform distribution from uniform distribution by + // inverse_transform_sampling method + // More details: + // https://wanghaoshuang.github.io/2017/11/Log-uniform-distribution-sampler/ + const int64 value = + static_cast(exp((*dist_)(*random_engine_) * log_range_)) - 1; + // Mathematically, value should be <= range_, but might not be due to some + // floating point roundoff, so we mod by range_. + return value % range_; +} + +float LogUniformSampler::Probability(int64 value) const { + // Given f(x) = 1/[(x+1) * log_range_] + // The value's probability is integral of f(x) from value to (value + 1) + // More details: + // https://wanghaoshuang.github.io/2017/11/Log-uniform-distribution-sampler + return (log((value + 2.0) / (value + 1.0))) / log_range_; +} + +} // namespace random +} // namespace paddle diff --git a/paddle/operators/math/sampler.h b/paddle/operators/math/sampler.h new file mode 100644 index 0000000000..bcd7bead35 --- /dev/null +++ b/paddle/operators/math/sampler.h @@ -0,0 +1,88 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include +typedef long int64; +namespace paddle { +namespace operators { +namespace math { + +// TODO: Support for GPU + +/** +* Sample integers from [0, range). +*/ +class Sampler { + public: + explicit Sampler(int64 range) : range_(range) { /* check range > 0*/ + } + virtual ~Sampler(); + // Sample a single value + virtual int64 Sample() const = 0; + // The probability that a single call to Sample() returns the given value. + virtual float Probability(int64 value) const = 0; + + int64 range() { return range_; }; + + protected: + const int64 range_; +}; + +/** + * Sample integers from [0, range). + * And the distribution function is: + * P(x) = 1 / range + */ +class UniformSampler : public Sampler { + public: + explicit UniformSampler(int64 range); + + ~UniformSampler() override {} + + int64 Sample() const override; + + float Probability(int64 value) const override; + + private: + const float inv_range_; + std::shared_ptr random_engine_; + std::shared_ptr> dist_; +}; + +/** + * Sample integers from [0, range). + * And the distribution function is: + * P(x) = (1/ln(range+1)) * ln(1 + 1/(x + 1)) + */ +class LogUniformSampler : public Sampler { + public: + explicit LogUniformSampler(int64 range); + + ~LogUniformSampler() override {} + + int64 Sample() const override; + + float Probability(int64 value) const override; + + private: + const float log_range_; + std::shared_ptr random_engine_; + std::shared_ptr> dist_; +}; + +} // math +} // namespace operators +} // namespace paddle From 62efc896e1fb1e895414d77d4193891fdee4ca99 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 7 Dec 2017 17:02:49 +0800 Subject: [PATCH 002/158] Refine code 1. Add copyright info 2. Overload structure for customized random seed --- paddle/operators/math/sampler.cc | 31 +++++++++++++++++++++++++++---- paddle/operators/math/sampler.h | 16 ++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/paddle/operators/math/sampler.cc b/paddle/operators/math/sampler.cc index 52628c3b03..4f1cbfe31a 100644 --- a/paddle/operators/math/sampler.cc +++ b/paddle/operators/math/sampler.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + #include "sampler.h" namespace paddle { @@ -7,8 +21,13 @@ Sampler::~Sampler() {} UniformSampler::UniformSampler(int64 range) : Sampler(range), inv_range_(1.0 / range) { - std::random_device r; - random_engine_ = std::make_shared(r()); + random_engine_ = std::make_shared(seed_); + dist_ = std::make_shared>(0, range); +} + +UniformSampler::UniformSampler(int64 range, unsigned int seed) + : Sampler(range, seed), inv_range_(1.0 / range) { + random_engine_ = std::make_shared(seed_); dist_ = std::make_shared>(0, range); } @@ -18,11 +37,15 @@ float UniformSampler::Probability(int64 value) const { return inv_range_; } LogUniformSampler::LogUniformSampler(int64 range) : Sampler(range), log_range_(log(range + 1)) { - std::random_device r; - random_engine_ = std::make_shared(r()); + random_engine_ = std::make_shared(seed_); dist_ = std::make_shared>(0, 1); } +LogUniformSampler::LogUniformSampler(int64 range, unsigned int seed) + : Sampler(range, seed), log_range_(log(range + 1)) { + random_engine_ = std::make_shared(seed_); + dist_ = std::make_shared>(0, 1); +} int64 LogUniformSampler::Sample() const { // Got Log Uniform distribution from uniform distribution by // inverse_transform_sampling method diff --git a/paddle/operators/math/sampler.h b/paddle/operators/math/sampler.h index bcd7bead35..8f82089e7b 100644 --- a/paddle/operators/math/sampler.h +++ b/paddle/operators/math/sampler.h @@ -20,14 +20,21 @@ namespace paddle { namespace operators { namespace math { -// TODO: Support for GPU +// TODO(wanghaoshuang): Support for GPU /** * Sample integers from [0, range). */ class Sampler { public: - explicit Sampler(int64 range) : range_(range) { /* check range > 0*/ + explicit Sampler(int64 range) : range_(range) { + PADDLE_ENFORCE_GT(range, 0); + std::random_device r; + seed_ = r(); + } + explicit Sampler(int64 range, unsigned int seed) + : range_(range), seed_(seed) { + PADDLE_ENFORCE_GT(range, 0); } virtual ~Sampler(); // Sample a single value @@ -39,6 +46,7 @@ class Sampler { protected: const int64 range_; + unsigned int seed_; }; /** @@ -50,6 +58,8 @@ class UniformSampler : public Sampler { public: explicit UniformSampler(int64 range); + explicit UniformSampler(int64 range, unsigned int seed); + ~UniformSampler() override {} int64 Sample() const override; @@ -71,6 +81,8 @@ class LogUniformSampler : public Sampler { public: explicit LogUniformSampler(int64 range); + explicit LogUniformSampler(int64 range, unsigned int seed); + ~LogUniformSampler() override {} int64 Sample() const override; From a60b3a5d557ca5e040e8f360709d0136fd6d1645 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 14 Dec 2017 11:47:32 +0800 Subject: [PATCH 003/158] fix doc of seq_expand_op --- paddle/operators/seq_expand_op.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/paddle/operators/seq_expand_op.cc b/paddle/operators/seq_expand_op.cc index ede9754697..8051ddd702 100644 --- a/paddle/operators/seq_expand_op.cc +++ b/paddle/operators/seq_expand_op.cc @@ -59,7 +59,7 @@ This operator expands input(X) according to LOD of input(Y). Following are cases to better explain how this works: Case 1: -Given 2-level a LoDTensor input(X) +Given a 2-level LoDTensor input(X) X.lod = [[0, 2, 3], [0, 1, 3, 4]] X.data = [a, b, c, d] @@ -76,9 +76,8 @@ then we get 2-level LoDTensor Case 2: -Given a 0-level LoDTensor input(X) +Given a common Tensor input(X) X.data = [a, b, c] - X.lod = NULL X.dims = [3, 1] and input(Y) Y.lod = [[0, 2, 3, 6]] @@ -90,9 +89,8 @@ then we get 1-level LoDTensor Case 3: -Given a 0-level LoDTensor input(X) +Given a common Tensor input(X) X.data = [[a, b], [c, d], [e, f]] - X.lod = NULL X.dims = [3, 2] and input(Y) Y.lod = [[0, 2, 3, 6]] From fb62f8cb0fcf359aec6f4d4265e6d13f0d1c2b35 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 11 Jan 2018 11:52:55 +0800 Subject: [PATCH 004/158] Add python api for warp-ctc op --- python/paddle/v2/fluid/layers/nn.py | 65 +++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 48a6bee558..97056570ee 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -14,7 +14,7 @@ __all__ = [ 'chunk_eval', 'sequence_conv', 'conv2d', 'sequence_pool', 'pool2d', 'batch_norm', 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', 'lstm_unit', 'reduce_sum', 'reduce_mean', 'reduce_max', 'reduce_min', - 'sequence_first_step', 'sequence_last_step', 'dropout' + 'sequence_first_step', 'sequence_last_step', 'dropout', 'warpctc' ] @@ -248,13 +248,13 @@ def gru_unit(input, h_t & = dot((1-u_t), m_t) + dot(u_t, h_{t-1}) The inputs of gru unit includes :math:`z_t`, :math:`h_{t-1}`. In terms - of the equation above, the :math:`z_t` is split into 3 parts - - :math:`xu_t`, :math:`xr_t` and :math:`xm_t`. This means that in order to - implement a full GRU unit operator for an input, a fully + of the equation above, the :math:`z_t` is split into 3 parts - + :math:`xu_t`, :math:`xr_t` and :math:`xm_t`. This means that in order to + implement a full GRU unit operator for an input, a fully connected layer has to be applied, such that :math:`z_t = W_{fc}x_t`. - The terms :math:`u_t` and :math:`r_t` represent the update and reset gates - of the GRU cell. Unlike LSTM, GRU has one lesser gate. However, there is + The terms :math:`u_t` and :math:`r_t` represent the update and reset gates + of the GRU cell. Unlike LSTM, GRU has one lesser gate. However, there is an intermediate candidate hidden output, which is denoted by :math:`m_t`. This layer has three outputs :math:`h_t`, :math:`dot(r_t, h_{t-1})` and concatenation of :math:`u_t`, :math:`r_t` and :math:`m_t`. @@ -276,7 +276,7 @@ def gru_unit(input, .. code-block:: python # assuming we have x_t_data and prev_hidden of size=10 - x_t = fluid.layers.fc(input=x_t_data, size=30) + x_t = fluid.layers.fc(input=x_t_data, size=30) hidden_val, r_h_val, gate_val = fluid.layers.gru_unit(input=x_t, hidden = prev_hidden) @@ -1504,3 +1504,54 @@ def reduce_min(input, dim=None, keep_dim=False): 'reduce_all': True if dim == None else False }) return out + + +def warpctc(input, label, blank=0, norm_by_times=False, **kwargs): + """ + An operator integrating the open source warp-ctc library + to compute Connectionist Temporal Classification (CTC) loss. + It can be aliased as softmax with ctc, since a native softmax activation is + interated to the warp-ctc library, to to normlize values for each row of the + input tensor. + + Args: + input(Variable): (LodTensor, default: LoDTensor), + the unscaled probabilities of variable-length sequences, + which is a 2-D Tensor with LoD information. + It's shape is [Lp, num_classes + 1], where Lp is the sum of all input + sequences' length and num_classes is the true number of classes. + (not including the blank label). + label(Variable): (LodTensor, default: LoDTensor), the ground truth + of variable-length sequence, which is a 2-D Tensor with LoD + information. It is of the shape [Lg, 1], where Lg is th sum of + all labels' length. + blank: (int, default: 0), the blank label of Connectionist + Temporal Classification (CTC) loss, which is in the + half-opened interval [0, num_classes + 1). + norm_by_times: (bool, default: false), whether to + normalize the gradients by the number of time-step, + which is also the sequence's length. + + Returns: + Variable: The Connectionist Temporal Classification (CTC) loss, which is a 2-D Tensor of the shape [batch_size, 1]. + + Examples: + .. code-block:: python + + y = layers.data(name='y', shape=[11, 8], dtype='float32', lod_level=1) + y_predict = layers.data(name='y_predict', shape=[11, 1], dtype='float32') + cost = layers.warpctc(input=y_predict, label=y) + + """ + helper = LayerHelper('warpctc', **kwargs) + loss_out = helper.create_tmp_variable(dtype=input.dtype) + grad_out = helper.create_tmp_variable(dtype=input.dtype) + helper.append_op( + type='warpctc', + inputs={'Logits': [input], + 'Label': [label]}, + outputs={'WarpCTCGrad': [grad_out], + 'Loss': [loss_out]}, + attrs={'blank': blank, + 'norm_by_times': norm_by_times}) + return loss_out From 7e95793bc6db8962edc47c5b25590f5fa4a9ac44 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 11 Jan 2018 15:46:29 +0800 Subject: [PATCH 005/158] =add bim method for generating adversarail sample --- adversarial/advbox/attacks/gradientsign.py | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/adversarial/advbox/attacks/gradientsign.py b/adversarial/advbox/attacks/gradientsign.py index 15b1d176cb..33fd92e71b 100644 --- a/adversarial/advbox/attacks/gradientsign.py +++ b/adversarial/advbox/attacks/gradientsign.py @@ -36,3 +36,38 @@ class GradientSignAttack(Attack): FGSM = GradientSignAttack + + +class IteratorGradientSignAttack(Attack): + """ + This attack was originally implemented by Alexey Kurakin(Google Brain). + Paper link: https://arxiv.org/pdf/1607.02533.pdf + """ + def _apply(self, image_label, epsilons=100, steps=10): + """ + Apply the iterative gradient sign attack. + Args: + image_label(list): The image and label tuple list of one element. + epsilons(list|tuple|int): The epsilon (input variation parameter). + steps(int): The number of iterator steps. + Return: + numpy.ndarray: The adversarail sample generated by the algorithm. + """ + assert len(image_label) == 1 + pre_label = np.argmax(self.model.predict(image_label)) + gradient = self.model.gradient(image_label) + min_, max_ = self.model.bounds() + + if not isinstance(epsilons, Iterable): + epsilons = np.linspace(0, 1, num=epsilons + 1) + + for epsilon in epsilons: + adv_img = image_label[0][0].reshape(gradient.shape) + for _ in range(steps): + gradient = self.model.gradient([(adv_img, image_label[0][1])]) + gradient_sign = np.sign(gradient) * (max_ - min_) + adv_img = adv_img + epsilon * gradient_sign + adv_img = np.clip(adv_img, min_, max_) + adv_label = np.argmax(self.model.predict([(adv_img, 0)])) + if pre_label != adv_label: + return adv_img From 7a0f3fdd8d4e71ba638fa6528e3890b4d832b393 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 11 Jan 2018 16:42:11 +0800 Subject: [PATCH 006/158] modify code style --- adversarial/advbox/attacks/gradientsign.py | 1 + 1 file changed, 1 insertion(+) diff --git a/adversarial/advbox/attacks/gradientsign.py b/adversarial/advbox/attacks/gradientsign.py index 33fd92e71b..2d7a445020 100644 --- a/adversarial/advbox/attacks/gradientsign.py +++ b/adversarial/advbox/attacks/gradientsign.py @@ -43,6 +43,7 @@ class IteratorGradientSignAttack(Attack): This attack was originally implemented by Alexey Kurakin(Google Brain). Paper link: https://arxiv.org/pdf/1607.02533.pdf """ + def _apply(self, image_label, epsilons=100, steps=10): """ Apply the iterative gradient sign attack. From 8c1025d66f3574e53cc41a0df9620d53c76d3ec5 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 11 Jan 2018 09:10:29 +0000 Subject: [PATCH 007/158] first commit --- paddle/framework/lod_tensor.cc | 32 ++++++++++++++++++++++------- paddle/framework/lod_tensor_test.cc | 28 +++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 7ae94c6465..3d1b75597e 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -225,20 +225,38 @@ void DeserializeFromStream(std::istream &is, LoDTensor *tensor, std::vector LoDTensor::SplitLoDTensor( const std::vector places) const { check_memory_size(); - PADDLE_ENFORCE(lod().empty(), "Disable parallel lod for now"); PADDLE_ENFORCE(dims()[0] % places.size() == 0, "Batch size should be divided by places size"); std::vector lods; for (size_t place_idx = 0; place_idx < places.size(); ++place_idx) { - int begin = place_idx * dims()[0] / places.size(); - int end = (place_idx + 1) * dims()[0] / places.size(); + size_t batch_size = lod().empty() ? dims()[0] : NumElements(0); + size_t begin = place_idx * batch_size / places.size(); + size_t end = (place_idx + 1) * batch_size / places.size(); - auto src = Slice(begin, end); - auto &dst_place = places[place_idx]; LoDTensor dst; - framework::Copy(src, dst_place, &dst); - + if (lod().empty()) { + auto src = Slice(begin, end); + auto &dst_place = places[place_idx]; + framework::Copy(src, dst_place, &dst); + } else { + auto lod_and_offset = GetSubLoDAndAbsoluteOffset(lod(), begin, end, 0); + + auto &offset = lod_and_offset.second; + auto src = Slice(offset.first, offset.second); + auto &dst_place = places[place_idx]; + framework::Copy(src, dst_place, &dst); + + LoD my_lod; + for (auto &l : lod_and_offset.first) { + std::vector v{0}; + for (auto &ll : l) { + v.push_back(ll + v.back()); + } + my_lod.emplace_back(v); + } + dst.set_lod(my_lod); + } lods.emplace_back(dst); } diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index baad9c6f98..5ff7dca564 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -131,5 +131,33 @@ TEST(LoD, ToAbsOffset) { EXPECT_EQ(abs_lod, expected); } +TEST(LoD, SplitLoDTensor) { + LoD lod; + lod.push_back(std::vector({0, 2, 4, 5, 6})); + lod.push_back(std::vector({0, 1, 6, 8, 13, 15, 20})); + + platform::CPUPlace place; + LoDTensor lod_tensor; + lod_tensor.Resize({20, 1}); + float* dst_ptr = lod_tensor.mutable_data(place); + for (int i = 0; i < lod_tensor.numel(); ++i) { + dst_ptr[i] = i; + } + lod_tensor.set_lod(lod); + + std::vector places{platform::CPUPlace(), + platform::CPUPlace()}; + LoD lod0; + lod0.push_back(std::vector({0, 2, 4})); + lod0.push_back(std::vector({0, 1, 6, 8, 13})); + LoD lod1; + lod1.push_back(std::vector({0, 1, 2})); + lod1.push_back(std::vector({0, 2, 7})); + + auto lods = lod_tensor.SplitLoDTensor(places); + EXPECT_EQ(lods[0].lod(), lod0); + EXPECT_EQ(lods[1].lod(), lod1); +} + } // namespace framework } // namespace paddle From 208f950ccc7b5508415a5dfb720348a659189f99 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Fri, 12 Jan 2018 03:30:06 +0000 Subject: [PATCH 008/158] delete todo --- paddle/framework/lod_tensor.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 3d1b75597e..a6a445efe8 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -221,7 +221,6 @@ void DeserializeFromStream(std::istream &is, LoDTensor *tensor, DeserializeFromStream(is, static_cast(tensor), dev_ctx); } -// TODO(tonyyang-svail): make this function support LoD std::vector LoDTensor::SplitLoDTensor( const std::vector places) const { check_memory_size(); From d23ea4ef8ebc637534c5abafca995be257f83751 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 12 Jan 2018 16:47:57 +0800 Subject: [PATCH 009/158] add gradient clip by norm --- python/paddle/v2/fluid/clip.py | 12 ++++++++++++ python/paddle/v2/fluid/layers/ops.py | 1 + 2 files changed, 13 insertions(+) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index b1fd1c2b65..eb75018d77 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -77,6 +77,18 @@ class GradientClipByValue(BaseGradientClipAttr): return param, new_grad +class GradientClipByNorm(BaseGradientClipAttr): + def __init__(self, clip_norm): + self.clip_norm = clip_norm + + def process_context(self, context, p_g): + pass + + def create_operators(self, param, grad): + new_grad = layers.clip_by_norm(x=grad, max_norm=self.clip_norm) + return param, new_grad + + def append_gradient_clip_ops(param_grad): context = dict() create_op_callbacks = [] diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index d3a5b70785..884e84011d 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -16,6 +16,7 @@ __all__ = [ 'elementwise_sub', 'elementwise_mul', 'clip', + 'clip_by_norm', 'sequence_softmax', ] + __activations__ From adc26dffa9dac81bd93c88d70f0ab66fcdcc81f0 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 10:36:09 +0800 Subject: [PATCH 010/158] developing GradientClipByGlobalNorm --- python/paddle/v2/fluid/clip.py | 54 ++++++++++++++++++++++++---- python/paddle/v2/fluid/layers/ops.py | 20 ++++------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index eb75018d77..f0904e18ea 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -1,5 +1,6 @@ import functools import layers +from framework import Variable from . import core __all__ = [ @@ -44,7 +45,7 @@ def error_clip_callback(block, context): class BaseGradientClipAttr(object): - def process_context(self, context, p_g): + def process_context(self, context, param, grad): raise NotImplementedError() def create_operators(self, param, grad): @@ -52,7 +53,7 @@ class BaseGradientClipAttr(object): class NullGradientClipAttr(BaseGradientClipAttr): - def process_context(self, context, p_g): + def process_context(self, context, param, grad): pass def create_operators(self, param, grad): @@ -69,7 +70,7 @@ class GradientClipByValue(BaseGradientClipAttr): self.max = max self.min = min - def process_context(self, context, p_g): + def process_context(self, context, param, grad): pass def create_operators(self, param, grad): @@ -81,7 +82,7 @@ class GradientClipByNorm(BaseGradientClipAttr): def __init__(self, clip_norm): self.clip_norm = clip_norm - def process_context(self, context, p_g): + def process_context(self, context, param, grad): pass def create_operators(self, param, grad): @@ -89,6 +90,46 @@ class GradientClipByNorm(BaseGradientClipAttr): return param, new_grad +class GradientClipByGlobalNorm(BaseGradientClipAttr): + global_norm_var = None + clip_norm_var = None + ratio_var = None + + @classmethod + def init(cls, clip_norm): + cls.global_norm_var = layers.fill_constant( + shape=[1], dtype="float32", value=0.0) + cls.clip_norm_var = layers.fill_constant( + shape=[1], dtype="float32", value=clip_norm) + + def __init__(self): + if not (isinstance(self.__class__.global_norm_var, Variable) and + isinstance(self.__class__.clip_norm_var, Variable)): + raise ValueError( + "Class 'GradientClipByGlobalNorm' has not been properly initialized. Please call GradientClipByGlobalNorm.init() first." + ) + + def process_context(self, context, param, grad): + local_norm_var = layers.reduce_sum( + x=layers.pow(x=grad, factor=2), reduce_all=True) + layers.sums( + input=[local_norm_var, self.__class__.global_norm_var], + out=[self.__class__.global_norm_var]) + + def create_operators(self, param, grad): + if self.__class__.ratio_var is None: + self.__class__.global_norm_var = layers.sqrt( + x=self.__class__.global_norm_var) + self.__class__.ratio_var = layers.elementwise_div( + x=self.__class__.clip_norm_var, + y=layers.elementwise_max( + x=self.__class__.clip_norm_var, + y=self.__class__.global_norm_var)) + # 缺乏elementwise_max + # 没法将ratio_var送给scale_op。 + # new_grad = layers. + + def append_gradient_clip_ops(param_grad): context = dict() create_op_callbacks = [] @@ -98,10 +139,9 @@ def append_gradient_clip_ops(param_grad): clip_attr = NullGradientClipAttr() if not isinstance(clip_attr, BaseGradientClipAttr): raise TypeError( - "clip attribute should be an instance of BaseGradientClippingAttr" - ) + "clip attribute should be an instance of BaseGradientClipAttr") - clip_attr.process_context(context=context, p_g=param_grad) + clip_attr.process_context(context=context, param=p, grad=g) create_op_callbacks.append( functools.partial( clip_attr.create_operators, param=p, grad=g)) diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index 884e84011d..021b87828f 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -1,23 +1,15 @@ from ..registry import register_layer __activations__ = [ - 'abs', 'tanh', 'sigmoid', 'relu', 'sqrt', 'ceil', 'floor', 'log', 'round' + 'abs', 'tanh', 'sigmoid', 'relu', 'sqrt', 'ceil', 'floor', 'log', 'round', + 'pow' ] __all__ = [ - 'mean', - 'mul', - 'reshape', - 'scale', - 'transpose', - 'sigmoid_cross_entropy_with_logits', - 'elementwise_add', - 'elementwise_div', - 'elementwise_sub', - 'elementwise_mul', - 'clip', - 'clip_by_norm', - 'sequence_softmax', + 'mean', 'mul', 'reshape', 'scale', 'transpose', + 'sigmoid_cross_entropy_with_logits', 'elementwise_add', 'elementwise_div', + 'elementwise_sub', 'elementwise_mul', 'clip', 'clip_by_norm', + 'sequence_softmax', 'reduce_sum' ] + __activations__ for _OP in set(__all__): From 579f684661d1badf34957b8a48ffe7d713547ead Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Mon, 15 Jan 2018 15:49:46 +0800 Subject: [PATCH 011/158] Add ctc_greedy_decode_op --- paddle/operators/ctc_greedy_decode_op.cc | 88 +++++++++++ paddle/operators/ctc_greedy_decode_op.cu | 138 ++++++++++++++++++ paddle/operators/ctc_greedy_decode_op.h | 82 +++++++++++ .../v2/fluid/tests/test_ctc_greedy_decode.py | 56 +++++++ 4 files changed, 364 insertions(+) create mode 100644 paddle/operators/ctc_greedy_decode_op.cc create mode 100644 paddle/operators/ctc_greedy_decode_op.cu create mode 100644 paddle/operators/ctc_greedy_decode_op.h create mode 100644 python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py diff --git a/paddle/operators/ctc_greedy_decode_op.cc b/paddle/operators/ctc_greedy_decode_op.cc new file mode 100644 index 0000000000..3c9b705f7f --- /dev/null +++ b/paddle/operators/ctc_greedy_decode_op.cc @@ -0,0 +1,88 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/ctc_greedy_decode_op.h" + +namespace paddle { +namespace operators { + +class CTCGreedyDecodeOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input of CTCGreedyDecodeOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Output"), + "Output of CTCGreedyDecodeOp should not be null."); + + auto input_dims = ctx->GetInputDim("Input"); + + int sequence_width = + static_cast(framework::product(input_dims) / input_dims[0]); + int blank = ctx->Attrs().Get("blank"); + PADDLE_ENFORCE((blank >= 0) && (blank < sequence_width), + "The value of Attr(blank) should be in interval [0, %d).", + sequence_width); + // TODO(wanghaoshuang): it is tricky to set the wrong dimension here. + ctx->SetOutputDim("Output", {input_dims[0], 1}); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Input")->type()), + ctx.device_context()); + } +}; + +class CTCGreedyDecodeOpMaker : public framework::OpProtoAndCheckerMaker { + public: + CTCGreedyDecodeOpMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Input", + "(LodTensor, default: LoDTensor), the unscaled " + "probabilities of variable-length sequences, which is a 2-D " + "Tensor with LoD information. It's shape is " + "[Lp, num_classes + 1], where Lp is the sum of all input " + "sequences' length and num_classes is the true number of classes " + "(not including the blank label)."); + AddOutput("Output", "(Tensor, default: Tensor), the decode result "); + AddAttr("blank", + "(int, default: 0), the blank label setted in Connectionist " + "Temporal Classification (CTC) op, and it is in the " + "half-opened interval [0, num_classes + 1).") + .SetDefault(0); + AddAttr("merge_repeated", + "(bool, default: true), whether to " + "merge repeated elements between two blanks. ") + .SetDefault(true); + AddComment(R"DOC( +CTCGreedyDecoder is an implementation of the simple best path decoding +algorithm, selecting at each timestep the most likely class at each timestep. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(ctc_greedy_decode, ops::CTCGreedyDecodeOp, + ops::CTCGreedyDecodeOpMaker, + paddle::framework::EmptyGradOpMaker); +REGISTER_OP_CPU_KERNEL( + ctc_greedy_decode, + ops::CTCGreedyDecodeKernel); diff --git a/paddle/operators/ctc_greedy_decode_op.cu b/paddle/operators/ctc_greedy_decode_op.cu new file mode 100644 index 0000000000..43a78745ac --- /dev/null +++ b/paddle/operators/ctc_greedy_decode_op.cu @@ -0,0 +1,138 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/operators/ctc_greedy_decode_op.h" +#include "paddle/platform/cuda_helper.h" +#include "paddle/platform/gpu_info.h" + +namespace paddle { +namespace operators { +using platform::PADDLE_CUDA_NUM_THREADS; + +__device__ static float atomicMaxF(float* address, float val) { + int* address_as_i = (int*)address; + int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS(address_as_i, assumed, + __float_as_int(::fmaxf(val, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +} + +template +__global__ void ArgmaxCudaKernel(const size_t seq_width, const T* logits, + int* output) { + T local_max_value = 0; + int local_max_index = 0; + __shared__ T max_value; + if (threadIdx.x == 0) { + max_value = 0; + } + __syncthreads(); + + for (int i = threadIdx.x; i < seq_width; i += BlockSize) { + T value = logits[blockIdx.x * seq_width + i]; + if (value > local_max_value) { + local_max_value = value; + local_max_index = i; + } + } + + atomicMaxF(&max_value, local_max_value); + + __syncthreads(); + + if (local_max_value == max_value) { + output[blockIdx.x] = local_max_index; + } +} + +template +__global__ void MergeAndDelCudaKernel(const int64_t num_token, int* tokens, + const size_t num_seq, size_t* lod0, + const int blank, const int merge_repeated, + size_t* out_lod0, int* output) { + int ouput_idx = 0; + out_lod0[0] = 0; + + for (int i = 0; i < num_seq; ++i) { + int pre_token = -1; + for (int j = lod0[i]; j < lod0[i + 1]; ++j) { + if (tokens[j] != blank && !(merge_repeated && tokens[j] == pre_token)) { + output[ouput_idx] = tokens[j]; + ++ouput_idx; + } + pre_token = tokens[j]; + } + out_lod0[i + 1] = ouput_idx; + } +} + +template +class CTCGreedyDecodeOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use CUDAPlace."); + auto* input = ctx.Input("Input"); + auto* output = ctx.Output("Output"); + + const int64_t num_tokens = input->dims()[0]; + const size_t seq_width = input->numel() / num_tokens; + const T* logits = input->data(); + Tensor tmp; + int* tokens = tmp.mutable_data({num_tokens, 1}, ctx.GetPlace()); + // get argmax + // platform::GpuMemsetAsync(args, 0, sizeof(float), stream); + + auto stream = ctx.cuda_device_context().stream(); + ArgmaxCudaKernel<<< + num_tokens, PADDLE_CUDA_NUM_THREADS, 0, stream>>>(seq_width, logits, + tokens); + + const size_t level = 0; + auto input_lod = framework::ToAbsOffset(input->lod()); + const size_t num_seq = input_lod[level].size() - 1; + const int blank = ctx.Attr("blank"); + const int merge_repeated = + static_cast(ctx.Attr("merge_repeated")); + + thrust::device_vector dev_out_lod0(input_lod[level].size()); + size_t* dev_out_lod0_ptr = thrust::raw_pointer_cast(dev_out_lod0.data()); + + int* output_data = + output->mutable_data({num_tokens, 1}, ctx.GetPlace()); + MergeAndDelCudaKernel<<<1, 1, 0, stream>>>( + num_tokens, tokens, num_seq, input_lod[level].data(), blank, + merge_repeated, dev_out_lod0_ptr, output_data); + + thrust::host_vector host_out_lod0(dev_out_lod0.begin(), + dev_out_lod0.end()); + framework::LoD out_lod; + out_lod.push_back(host_out_lod0); + output->set_lod(out_lod); + + output->Resize({static_cast(host_out_lod0.back()), 1}); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OP_CUDA_KERNEL(ctc_greedy_decode, + paddle::operators::CTCGreedyDecodeOpCUDAKernel); diff --git a/paddle/operators/ctc_greedy_decode_op.h b/paddle/operators/ctc_greedy_decode_op.h new file mode 100644 index 0000000000..f12ea6c541 --- /dev/null +++ b/paddle/operators/ctc_greedy_decode_op.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include "paddle/framework/op_registry.h" +#include "unsupported/Eigen/CXX11/Tensor" +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; + +template +class CTCGreedyDecodeKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* input = ctx.Input("Input"); + auto* output = ctx.Output("Output"); + const size_t level = 0; + + auto input_lod = framework::ToAbsOffset(input->lod()); + auto input_dims = input->dims(); + PADDLE_ENFORCE_EQ(input_dims[0], + static_cast(input_lod[level].back()), + "The first dimension of Input(Input) should be equal to " + "the sum of all sequences' lengths."); + + const size_t num_sequences = input_lod[level].size() - 1; + const size_t sequence_width = input->numel() / input_dims[0]; + size_t blank = static_cast(ctx.Attr("blank")); + bool merge_repeated = ctx.Attr("merge_repeated"); + std::vector> pathes(num_sequences); + std::vector output_lod0(1, 0); + + const T* input_data = input->data(); + Eigen::Map< + Eigen::Matrix> + input_mat(const_cast(input_data), input->numel() / sequence_width, + sequence_width); + + size_t max_class_idx; + size_t prev_class_idx = -1; + for (size_t seq_idx = 0; seq_idx < num_sequences; ++seq_idx) { + for (size_t i = input_lod[level][seq_idx]; + i < input_lod[level][seq_idx + 1]; ++i) { + input_mat.row(i).maxCoeff(&max_class_idx); + if (max_class_idx != blank && + !(merge_repeated && max_class_idx == prev_class_idx)) { + pathes[seq_idx].push_back(max_class_idx); + } + prev_class_idx = max_class_idx; + } + output_lod0.push_back(output_lod0.back() + pathes[seq_idx].size()); + } + framework::LoD output_lod; + output_lod.push_back(output_lod0); + output->set_lod(output_lod); + int64_t num_step = static_cast(output_lod0.back()); + int* output_data = output->mutable_data({num_step, 1}, ctx.GetPlace()); + + for (int i = 0; i < num_sequences; ++i) { + memcpy(output_data + output_lod0[i], pathes[i].data(), + sizeof(int) * pathes[i].size()); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py b/python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py new file mode 100644 index 0000000000..23fceb6dcd --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py @@ -0,0 +1,56 @@ +import sys +import unittest +import numpy as np +from op_test import OpTest +from test_softmax_op import stable_softmax + + +def CTCGreedyDecode(softmax, blank, merge_repeated): + prev_token = -1 + result = [] + for token in np.argmax(softmax, axis=1): + if (token != blank) and not (merge_repeated and token == prev_token): + result.append(token) + return np.array(result).reshape([len(result), 1]) + + +class TestCTCGreedyDecodeOp(OpTest): + def config(self): + self.op_type = "ctc_greedy_decode" + self.batch_size = 4 + self.num_classes = 8 + self.input_lod = [[0, 4, 5, 8, 11]] + self.blank = 7 + self.merge_repeated = True + + def setUp(self): + self.config() + input = np.random.uniform( + 0.1, 1.0, + [self.input_lod[0][-1], self.num_classes]).astype("float32") + softmax = np.apply_along_axis(stable_softmax, 1, input) + output = CTCGreedyDecode(softmax, self.blank, self.merge_repeated) + + self.inputs = {"Input": (softmax, self.input_lod), } + self.outputs = {"Output": output} + self.attrs = { + "blank": self.blank, + "merge_repeated": self.merge_repeated + } + + def test_check_output(self): + self.check_output() + + +class TestCTCGreedyDecodeOpCase1(TestCTCGreedyDecodeOp): + def config(self): + self.op_type = "ctc_greedy_decode" + self.batch_size = 4 + self.num_classes = 1025 + self.input_lod = [[0, 4, 5, 8, 11]] + self.blank = 0 + self.merge_repeated = True + + +if __name__ == "__main__": + unittest.main() From 736842e447669ae92b8ebd580338101d6d31458d Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 16:58:15 +0800 Subject: [PATCH 012/158] wip --- paddle/operators/elementwise_max_op.cc | 45 +++++++++++ paddle/operators/elementwise_max_op.h | 107 +++++++++++++++++++++++++ paddle/operators/elementwise_min_op.cc | 45 +++++++++++ 3 files changed, 197 insertions(+) create mode 100644 paddle/operators/elementwise_max_op.cc create mode 100644 paddle/operators/elementwise_max_op.h create mode 100644 paddle/operators/elementwise_min_op.cc diff --git a/paddle/operators/elementwise_max_op.cc b/paddle/operators/elementwise_max_op.cc new file mode 100644 index 0000000000..b5c6b11ba3 --- /dev/null +++ b/paddle/operators/elementwise_max_op.cc @@ -0,0 +1,45 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/elementwise_max_op.h" +#include "paddle/operators/elementwise_op.h" + +namespace paddle { +namespace operators { +class ElementwiseMaxOpMaker : public ElementwiseOpMaker { + public: + ElementwiseMaxOpMaker(OpProto* proto, OpAttrChecker* op_checker) + : ElementwiseOpMaker(proto, op_checker) { + SetComment("Max", "Out = max(X, Y)"); + AddComment(comment_); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(elementwise_max, ops::ElementwiseOp, ops::ElementwiseMaxOpMaker, + elementwise_max_grad, ops::ElementwiseOpGrad); +REGISTER_OP_CPU_KERNEL( + elementwise_max, + ops::ElementwiseMaxKernel, + ops::ElementwiseMaxKernel, + ops::ElementwiseMaxKernel, + ops::ElementwiseMaxKernel); +REGISTER_OP_CPU_KERNEL( + elementwise_max_grad, + ops::ElementwiseMaxGradKernel, + ops::ElementwiseMaxGradKernel, + ops::ElementwiseMaxGradKernel, + ops::ElementwiseMaxGradKernel); \ No newline at end of file diff --git a/paddle/operators/elementwise_max_op.h b/paddle/operators/elementwise_max_op.h new file mode 100644 index 0000000000..5c685b75e5 --- /dev/null +++ b/paddle/operators/elementwise_max_op.h @@ -0,0 +1,107 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/operators/elementwise_op_function.h" + +namespace paddle { +namespace operators { + +template +struct MaxFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a > b ? a : b; } +}; + +template +class ElementwiseMaxKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* z = ctx.Output("Out"); + z->mutable_data(ctx.GetPlace()); + TransformFunctor, T, DeviceContext> functor( + x, y, z, ctx.template device_context(), MaxFunctor()); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input."); + + if (x_dims == y_dims) { + functor.Run(); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), + "Axis should be in range [0, x_dims)"); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + if (post == 1) { + functor.RunRowWise(n, pre); + return; + } else { + functor.RunMidWise(n, pre, post); + return; + } + } +}; + +template +struct ElementwiseSubGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto dz_e = framework::EigenVector::Flatten(*dz); + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = (x_e > y_e) * dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (y_e >= x_e) * dz_e; + } + } +}; + +template +struct ElementwiseSubOneGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto dz_e = framework::EigenVector::Flatten(*dz); + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0) * dz_e.sum(); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/elementwise_min_op.cc b/paddle/operators/elementwise_min_op.cc new file mode 100644 index 0000000000..b78846f17a --- /dev/null +++ b/paddle/operators/elementwise_min_op.cc @@ -0,0 +1,45 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/elementwise_min_op.h" +#include "paddle/operators/elementwise_op.h" + +namespace paddle { +namespace operators { +class ElementwiseMinOpMaker : public ElementwiseOpMaker { + public: + ElementwiseMinOpMaker(OpProto* proto, OpAttrChecker* op_checker) + : ElementwiseOpMaker(proto, op_checker) { + SetComment("Max", "Out = min(X, Y)"); + AddComment(comment_); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(elementwise_min, ops::ElementwiseOp, ops::ElementwiseMinOpMaker, + elementwise_min_grad, ops::ElementwiseOpGrad); +REGISTER_OP_CPU_KERNEL( + elementwise_min, + ops::ElementwiseMinKernel, + ops::ElementwiseMinKernel, + ops::ElementwiseMinKernel, + ops::ElementwiseMinKernel); +REGISTER_OP_CPU_KERNEL( + elementwise_min_grad, + ops::ElementwiseMinGradKernel, + ops::ElementwiseMinGradKernel, + ops::ElementwiseMinGradKernel, + ops::ElementwiseMinGradKernel); \ No newline at end of file From acf37ad67504ee2f4ce5f601906c1b5102ede124 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 17:53:36 +0800 Subject: [PATCH 013/158] Complete elementwise_max_op --- paddle/operators/elementwise_max_op.cc | 2 +- paddle/operators/elementwise_max_op.cu | 32 +++++++++++++ paddle/operators/elementwise_max_op.h | 63 ++++++++++++++++++++++---- 3 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 paddle/operators/elementwise_max_op.cu diff --git a/paddle/operators/elementwise_max_op.cc b/paddle/operators/elementwise_max_op.cc index b5c6b11ba3..53c27ae5be 100644 --- a/paddle/operators/elementwise_max_op.cc +++ b/paddle/operators/elementwise_max_op.cc @@ -42,4 +42,4 @@ REGISTER_OP_CPU_KERNEL( ops::ElementwiseMaxGradKernel, ops::ElementwiseMaxGradKernel, ops::ElementwiseMaxGradKernel, - ops::ElementwiseMaxGradKernel); \ No newline at end of file + ops::ElementwiseMaxGradKernel); diff --git a/paddle/operators/elementwise_max_op.cu b/paddle/operators/elementwise_max_op.cu new file mode 100644 index 0000000000..5ff4af1747 --- /dev/null +++ b/paddle/operators/elementwise_max_op.cu @@ -0,0 +1,32 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/elementwise_max_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_CUDA_KERNEL( + elementwise_max, + ops::ElementwiseMaxKernel, + ops::ElementwiseMaxKernel, + ops::ElementwiseMaxKernel, + ops::ElementwiseMaxKernel); +REGISTER_OP_CUDA_KERNEL( + elementwise_max_grad, + ops::ElementwiseMaxGradKernel, + ops::ElementwiseMaxGradKernel, + ops::ElementwiseMaxGradKernel, + ops::ElementwiseMaxGradKernel); diff --git a/paddle/operators/elementwise_max_op.h b/paddle/operators/elementwise_max_op.h index 5c685b75e5..e370aeb308 100644 --- a/paddle/operators/elementwise_max_op.h +++ b/paddle/operators/elementwise_max_op.h @@ -65,43 +65,88 @@ class ElementwiseMaxKernel : public framework::OpKernel { }; template -struct ElementwiseSubGradFunctor { +struct ElementwiseMaxGradFunctor { template void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { - auto dz_e = framework::EigenVector::Flatten(*dz); auto x_e = framework::EigenVector::Flatten(*x); auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); if (dx) { auto dx_e = framework::EigenVector::Flatten(*dx); - dx_e.device(d) = (x_e > y_e) * dz_e; + dx_e.device(d) = (x_e > y_e).template cast() * dz_e; } if (dy) { auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(d) = (y_e >= x_e) * dz_e; + dy_e.device(d) = (y_e >= x_e).template cast() * dz_e; } } }; template -struct ElementwiseSubOneGradFunctor { +struct ElementwiseMaxBroadCastGradFunctor { template - void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + typename dY, typename dZ, typename Pre, typename N> + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n)) + .broadcast(Eigen::DSizes(pre, 1)) + .reshape(Eigen::DSizes(x_e.size())); + + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = (x_e > y_e_bcast).template cast() * dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = ((y_e_bcast >= x_e).template cast() * dz_e) + .reshape(Eigen::DSizes(pre, n)) + .sum(Eigen::array{{0}}); + } + } +}; + +template +struct ElementwiseMaxBroadCast2GradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n, + Post post) { auto x_e = framework::EigenVector::Flatten(*x); auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) + .broadcast(Eigen::DSizes(pre, 1, post)) + .reshape(Eigen::DSizes(x_e.size())); if (dx) { auto dx_e = framework::EigenVector::Flatten(*dx); - dx_e.device(d) = dz_e; + dx_e.device(d) = (x_e > y_e_bcast).template cast() * dz_e; } + if (dy) { auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(d) = (-1.0) * dz_e.sum(); + dy_e.device(d) = ((y_e_bcast >= x_e).template cast() * dz_e) + .reshape(Eigen::DSizes(pre, n, post)) + .sum(Eigen::array{{0, 2}}); } } }; +template +class ElementwiseMaxGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseGradCompute, + ElementwiseMaxBroadCastGradFunctor, + ElementwiseMaxBroadCast2GradFunctor>(ctx); + } +}; + } // namespace operators } // namespace paddle From 234013a9d761190119f0f350fdde5e60ab8e3992 Mon Sep 17 00:00:00 2001 From: guosheng Date: Mon, 15 Jan 2018 18:13:01 +0800 Subject: [PATCH 014/158] Add python wrapper for matmul_op --- python/paddle/v2/fluid/layers/nn.py | 82 +++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 96a64af370..bc373d4b37 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1584,3 +1584,85 @@ def split(input, num_or_sections, dim=-1): 'axis': dim }) return outs + + +def matmul(x, y): + """ + Applies matrix multipication to two tensors. + + This operator is used to perform (batched) matrix multiplication + over the last two dimensions of the input tensors `X` and `Y`. + + If a transpose flag is specified, the last two dimensions of the + tensor are transposed. If the tensor is rank-1 of shape [D], then + for `X` it is treated as [1, D] in nontransposed form and as [D, 1] + in transposed form, whereas for `Y` it is the opposite: It is treated + as [D, 1] in nontransposed form and as [1, D] in transposed form. + + Examples without transpose: + - X: [K], Y: [K] => Out: [1] + - X: [K], Y: [K, N] => Out: [N] + - X: [B, M, K], Y: [K] => Out: [B, M] + - X: [M, K], Y: [B, K, N] => Out: [B, M, N] + - X: [B, M, K], Y: [B, K, N] => Out: [B, M, N] + + The behavior is designed to be similar to the `numpy.matmul` function. + The differences are: + - Currently only rank 1 to rank 3 input tensors are supported. + - We add `transpose_X` and `transpose_Y` flags. + + Both the input `X` and `Y` can carry the LoD (Level of Details) information, + or not. But the output only shares the LoD information with input `X`. + + Args: + x (Variable): The input variable which is a Tensor or LoDTensor. + y (Variable): If :attr:`num_or_sections` is an integer, + then the integer indicates the number of equal sized sub-tensors + that the tensor will be divided into. If :attr:`num_or_sections` + is a list of integers, the length of list indicates the number of + sub-tensors and the integers indicate the sizes of sub-tensors' + :attr:`dim` dimension orderly. + dim (int): The dimension along which to split. If :math:`dim < 0`, the + dimension to split along is :math:`rank(input) + dim`. + + Returns: + List: The list of segmented tensor variables. + + Examples: + .. code-block:: python + + # x is a Tensor variable with shape [3, 9, 5]: + x0, x1, x2 = fluid.layers.split(x, num_or_sections=3, dim=1) + x0.shape # [3, 3, 5] + x1.shape # [3, 3, 5] + x2.shape # [3, 3, 5] + x0, x1, x2 = fluid.layers.split(x, num_or_sections=[2, 3, 4], dim=1) + x0.shape # [3, 2, 5] + x1.shape # [3, 3, 5] + x2.shape # [3, 4, 5] + """ + helper = LayerHelper('split', **locals()) + input_shape = input.shape + dim = (len(input_shape) + dim) if dim < 0 else dim + if isinstance(num_or_sections, int): + assert num_or_sections > 1, 'num_or_sections must be more than 1.' + num = num_or_sections + else: + assert len(num_or_sections) < input_shape[ + dim], 'len(num_or_sections) must not be more than input.shape[dim].' + num = len(num_or_sections) + outs = [ + helper.create_tmp_variable(dtype=helper.input_dtype()) + for i in range(num) + ] + helper.append_op( + type='split', + inputs={'X': input}, + outputs={'Out': outs}, + attrs={ + 'num': num_or_sections if isinstance(num_or_sections, int) else 0, + 'sections': num_or_sections + if isinstance(num_or_sections, list) else [], + 'axis': dim + }) + return outs From f5cd9619002e2b392ac5195da0e3feef67c0c4c4 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 18:56:17 +0800 Subject: [PATCH 015/158] complete elementwise_min_op --- paddle/operators/elementwise_max_op.h | 6 +- paddle/operators/elementwise_min_op.cc | 2 +- paddle/operators/elementwise_min_op.cu | 32 ++++++ paddle/operators/elementwise_min_op.h | 152 +++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 paddle/operators/elementwise_min_op.cu create mode 100644 paddle/operators/elementwise_min_op.h diff --git a/paddle/operators/elementwise_max_op.h b/paddle/operators/elementwise_max_op.h index e370aeb308..92152f7cb6 100644 --- a/paddle/operators/elementwise_max_op.h +++ b/paddle/operators/elementwise_max_op.h @@ -79,7 +79,7 @@ struct ElementwiseMaxGradFunctor { } if (dy) { auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(d) = (y_e >= x_e).template cast() * dz_e; + dy_e.device(d) = (x_e <= y_e).template cast() * dz_e; } } }; @@ -104,7 +104,7 @@ struct ElementwiseMaxBroadCastGradFunctor { if (dy) { auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(d) = ((y_e_bcast >= x_e).template cast() * dz_e) + dy_e.device(d) = ((x_e <= y_e_bcast).template cast() * dz_e) .reshape(Eigen::DSizes(pre, n)) .sum(Eigen::array{{0}}); } @@ -131,7 +131,7 @@ struct ElementwiseMaxBroadCast2GradFunctor { if (dy) { auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(d) = ((y_e_bcast >= x_e).template cast() * dz_e) + dy_e.device(d) = ((x_e <= y_e_bcast).template cast() * dz_e) .reshape(Eigen::DSizes(pre, n, post)) .sum(Eigen::array{{0, 2}}); } diff --git a/paddle/operators/elementwise_min_op.cc b/paddle/operators/elementwise_min_op.cc index b78846f17a..99482e1bf6 100644 --- a/paddle/operators/elementwise_min_op.cc +++ b/paddle/operators/elementwise_min_op.cc @@ -42,4 +42,4 @@ REGISTER_OP_CPU_KERNEL( ops::ElementwiseMinGradKernel, ops::ElementwiseMinGradKernel, ops::ElementwiseMinGradKernel, - ops::ElementwiseMinGradKernel); \ No newline at end of file + ops::ElementwiseMinGradKernel); diff --git a/paddle/operators/elementwise_min_op.cu b/paddle/operators/elementwise_min_op.cu new file mode 100644 index 0000000000..3547e6ccb7 --- /dev/null +++ b/paddle/operators/elementwise_min_op.cu @@ -0,0 +1,32 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/elementwise_min_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_CUDA_KERNEL( + elementwise_min, + ops::ElementwiseMinKernel, + ops::ElementwiseMinKernel, + ops::ElementwiseMinKernel, + ops::ElementwiseMinKernel); +REGISTER_OP_CUDA_KERNEL( + elementwise_min_grad, + ops::ElementwiseMinGradKernel, + ops::ElementwiseMinGradKernel, + ops::ElementwiseMinGradKernel, + ops::ElementwiseMinGradKernel); diff --git a/paddle/operators/elementwise_min_op.h b/paddle/operators/elementwise_min_op.h new file mode 100644 index 0000000000..53b7f59fa0 --- /dev/null +++ b/paddle/operators/elementwise_min_op.h @@ -0,0 +1,152 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/operators/elementwise_op_function.h" + +namespace paddle { +namespace operators { + +template +struct MinFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a < b ? a : b; } +}; + +template +class ElementwiseMinKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* z = ctx.Output("Out"); + z->mutable_data(ctx.GetPlace()); + TransformFunctor, T, DeviceContext> functor( + x, y, z, ctx.template device_context(), MinFunctor()); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input."); + + if (x_dims == y_dims) { + functor.Run(); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), + "Axis should be in range [0, x_dims)"); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + if (post == 1) { + functor.RunRowWise(n, pre); + return; + } else { + functor.RunMidWise(n, pre, post); + return; + } + } +}; + +template +struct ElementwiseMinGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = (x_e < y_e).template cast() * dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (x_e >= y_e).template cast() * dz_e; + } + } +}; + +template +struct ElementwiseMinBroadCastGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n)) + .broadcast(Eigen::DSizes(pre, 1)) + .reshape(Eigen::DSizes(x_e.size())); + + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = (x_e < y_e_bcast).template cast() * dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = ((x_e >= y_e_bcast).template cast() * dz_e) + .reshape(Eigen::DSizes(pre, n)) + .sum(Eigen::array{{0}}); + } + } +}; + +template +struct ElementwiseMinBroadCast2GradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n, + Post post) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) + .broadcast(Eigen::DSizes(pre, 1, post)) + .reshape(Eigen::DSizes(x_e.size())); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = (x_e < y_e_bcast).template cast() * dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = ((x_e >= y_e_bcast).template cast() * dz_e) + .reshape(Eigen::DSizes(pre, n, post)) + .sum(Eigen::array{{0, 2}}); + } + } +}; + +template +class ElementwiseMinGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseGradCompute, + ElementwiseMinBroadCastGradFunctor, + ElementwiseMinBroadCast2GradFunctor>(ctx); + } +}; + +} // namespace operators +} // namespace paddle From 78dc93430ca8eb40e8ad4c1315dfeef6d1889c77 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 15 Jan 2018 19:22:31 +0800 Subject: [PATCH 016/158] expose use_cudnn --- python/paddle/v2/fluid/layers/nn.py | 27 +++++++++++++++++++++------ python/paddle/v2/fluid/nets.py | 19 +++++++++++++------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 99a40ce45a..df2233089c 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -660,6 +660,7 @@ def conv2d(input, groups=None, param_attr=None, bias_attr=None, + use_cudnn=False, act=None): """ **Convlution2D Layer** @@ -758,6 +759,8 @@ def conv2d(input, stride = [stride, stride] if isinstance(padding, int): padding = [padding, padding] + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") input_shape = input.shape filter_shape = [num_filters, num_filter_channels] + filter_size @@ -781,9 +784,12 @@ def conv2d(input, 'Filter': filter_param, }, outputs={"Output": pre_bias}, - attrs={'strides': stride, - 'paddings': padding, - 'groups': groups}) + attrs={ + 'strides': stride, + 'paddings': padding, + 'groups': groups, + 'use_cudnn': use_cudnn + }) pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -931,7 +937,8 @@ def pool2d(input, pool_type, pool_stride=None, pool_padding=None, - global_pooling=False): + global_pooling=False, + use_cudnn=False): """ This function adds the operator for pooling in 2 dimensions, using the pooling configurations mentioned in input parameters. @@ -950,6 +957,8 @@ def pool2d(input, pool_stride = [pool_stride, pool_stride] if isinstance(pool_padding, int): pool_padding = [pool_padding, pool_padding] + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") helper = LayerHelper('pool2d', **locals()) dtype = helper.input_dtype() @@ -964,7 +973,8 @@ def pool2d(input, "ksize": pool_size, "global_pooling": global_pooling, "strides": pool_stride, - "paddings": pool_padding + "paddings": pool_padding, + "use_cudnn": use_cudnn }) return pool_out @@ -1077,7 +1087,8 @@ def conv2d_transpose(input, padding=None, stride=None, dilation=None, - param_attr=None): + param_attr=None, + use_cudnn=False): """ The transpose of conv2d layer. @@ -1132,6 +1143,10 @@ def conv2d_transpose(input, elif dilation is not None: op_attr['dilations'] = dilation + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") + op_attr['use_cudnn'] = use_cudnn + if filter_size is None: if output_size is None: raise ValueError("output_size must be set when filter_size is None") diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index 47b550bf4d..8ac58fd733 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -13,19 +13,22 @@ def simple_img_conv_pool(input, pool_stride, act, param_attr=None, - pool_type='max'): + pool_type='max', + use_cudnn=False): conv_out = layers.conv2d( input=input, num_filters=num_filters, filter_size=filter_size, param_attr=param_attr, - act=act) + act=act, + use_cudnn=use_cudnn) pool_out = layers.pool2d( input=conv_out, pool_size=pool_size, pool_type=pool_type, - pool_stride=pool_stride) + pool_stride=pool_stride, + use_cudnn=use_cudnn) return pool_out @@ -38,8 +41,10 @@ def img_conv_group(input, param_attr=None, conv_with_batchnorm=False, conv_batchnorm_drop_rate=None, + conv_use_cudnn=False, pool_stride=1, - pool_type=None): + pool_type=None, + pool_use_cudnn=False): """ Image Convolution Group, Used for vgg net. """ @@ -70,7 +75,8 @@ def img_conv_group(input, filter_size=conv_filter_size[i], padding=conv_padding[i], param_attr=param_attr[i], - act=local_conv_act) + act=local_conv_act, + use_cudnn=conv_use_cudnn) if conv_with_batchnorm[i]: tmp = layers.batch_norm(input=tmp, act=conv_act) @@ -82,7 +88,8 @@ def img_conv_group(input, input=tmp, pool_size=pool_size, pool_type=pool_type, - pool_stride=pool_stride) + pool_stride=pool_stride, + use_cudnn=pool_use_cudnn) return pool_out From 79aa51229ad41c4633056ad0f7056bbb88648a87 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 15 Jan 2018 19:46:03 +0800 Subject: [PATCH 017/158] fix conv, pool, conv_trans to decide use cudnn or not --- paddle/operators/conv_op.cc | 2 ++ paddle/operators/conv_op.h | 1 + paddle/operators/conv_transpose_op.cc | 2 ++ paddle/operators/conv_transpose_op.h | 1 + paddle/operators/pool_op.cc | 2 ++ paddle/operators/pool_op.h | 1 + paddle/platform/dynload/cudnn.cc | 4 ++++ 7 files changed, 13 insertions(+) diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index 424eccdb7d..a6d3ebb4f5 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -70,6 +70,7 @@ void ConvOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); + use_cudnn &= platform::dynload::HasCUDNN(); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -283,6 +284,7 @@ void ConvOpGrad::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); + use_cudnn &= platform::dynload::HasCUDNN(); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_op.h b/paddle/operators/conv_op.h index 5a8933e791..2a6fad6582 100644 --- a/paddle/operators/conv_op.h +++ b/paddle/operators/conv_op.h @@ -19,6 +19,7 @@ limitations under the License. */ #include "paddle/operators/math/im2col.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/vol2col.h" +#include "paddle/platform/dynload/cudnn.h" namespace paddle { namespace operators { diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index cf4e8c0a30..53153c0296 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -61,6 +61,7 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvTransposeOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); + use_cudnn &= platform::dynload::HasCUDNN(); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -263,6 +264,7 @@ void ConvTransposeOpGrad::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvTransposeOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); + use_cudnn &= platform::dynload::HasCUDNN(); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h index a42ade41b1..f43d652fe1 100644 --- a/paddle/operators/conv_transpose_op.h +++ b/paddle/operators/conv_transpose_op.h @@ -19,6 +19,7 @@ limitations under the License. */ #include "paddle/operators/math/im2col.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/vol2col.h" +#include "paddle/platform/dynload/cudnn.h" namespace paddle { namespace operators { diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 3e567efd08..e0f9ea7aa0 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -64,6 +64,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { framework::OpKernelType PoolOp::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); + use_cudnn &= platform::dynload::HasCUDNN(); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -88,6 +89,7 @@ void PoolOpGrad::InferShape(framework::InferShapeContext *ctx) const { framework::OpKernelType PoolOpGrad::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); + use_cudnn &= platform::dynload::HasCUDNN(); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index c3d82ecbde..e0668f1360 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -18,6 +18,7 @@ limitations under the License. */ #include "paddle/framework/op_registry.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/pooling.h" +#include "paddle/platform/dynload/cudnn.h" namespace paddle { namespace operators { diff --git a/paddle/platform/dynload/cudnn.cc b/paddle/platform/dynload/cudnn.cc index 701f6240fe..fafdf5e78a 100644 --- a/paddle/platform/dynload/cudnn.cc +++ b/paddle/platform/dynload/cudnn.cc @@ -57,6 +57,10 @@ void EnforceCUDNNLoaded(const char* fn_name) { bool HasCUDNN() { return true; } #endif +#ifndef PADDLE_WITH_CUDA +bool HasCUDNN() { return false; } +#endif + } // namespace dynload } // namespace platform } // namespace paddle From 1e6e5ac64c27d105a1176f3876e4415040fae0a4 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 20:34:18 +0800 Subject: [PATCH 018/158] add unit test --- .../v2/fluid/tests/test_elementwise_max_op.py | 107 ++++++++++++++++++ .../v2/fluid/tests/test_elementwise_min_op.py | 107 ++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 python/paddle/v2/fluid/tests/test_elementwise_max_op.py create mode 100644 python/paddle/v2/fluid/tests/test_elementwise_min_op.py diff --git a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py new file mode 100644 index 0000000000..52bd123d80 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py @@ -0,0 +1,107 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestElementwiseOp(OpTest): + def setUp(self): + self.op_type = "elementwise_max" + # If x and y have the same value, the max() is not differentiable. + # So we generate test data by the following method + # to avoid them being too close to each other. + x = np.random.uniform(0.1, 1, [13, 17]).astype("float32") + sgn = np.random.choice([-1, 1], [13, 17]).astype("float32") + y = x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype("float32") + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.005) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) + + +class TestElementwiseMaxOp_Vector(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_max" + x = np.random.random((32, )).astype("float32") + sgn = np.random.choice([-1, 1], (32, )).astype("float32") + y = x + sgn * np.random.uniform(0.1, 1, (32, )).astype("float32") + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} + + +class TestElementwiseMaxOp_broadcast_0(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_max" + x = np.random.uniform(0.5, 1, (2, 3, 4)).astype(np.float32) + sgn = np.random.choice([-1, 1], (2, )).astype(np.float32) + y = x[:, 0, 0] + sgn * \ + np.random.uniform(1, 2, (2, )).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': + np.maximum(self.inputs['X'], self.inputs['Y'].reshape(2, 1, 1)) + } + + +class TestElementwiseMaxOp_broadcast_1(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_max" + x = np.random.uniform(0.5, 1, (2, 3, 4)).astype(np.float32) + sgn = np.random.choice([-1, 1], (3, )).astype(np.float32) + y = x[0, :, 0] + sgn * \ + np.random.uniform(1, 2, (3, )).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.maximum(self.inputs['X'], self.inputs['Y'].reshape(1, 3, 1)) + } + + +class TestElementwiseMaxOp_broadcast_2(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_max" + x = np.random.uniform(0.5, 1, (2, 3, 4)).astype(np.float32) + sgn = np.random.choice([-1, 1], (4, )).astype(np.float32) + y = x[0, 0, :] + sgn * \ + np.random.uniform(1, 2, (4, )).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.outputs = { + 'Out': + np.maximum(self.inputs['X'], self.inputs['Y'].reshape(1, 1, 4)) + } + + +class TestElementwiseMaxOp_broadcast_3(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_max" + x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(np.float32) + sgn = np.random.choice([-1, 1], (3, 4)).astype(np.float32) + y = x[0, :, :, 0] + sgn * \ + np.random.uniform(1, 2, (3, 4)).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.maximum(self.inputs['X'], self.inputs['Y'].reshape(1, 3, 4, 1)) + } + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py new file mode 100644 index 0000000000..5ff5a40013 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py @@ -0,0 +1,107 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestElementwiseOp(OpTest): + def setUp(self): + self.op_type = "elementwise_min" + # If x and y have the same value, the max() is not differentiable. + # So we generate test data by the following method + # to avoid them being too close to each other. + x = np.random.uniform(0.1, 1, [13, 17]).astype("float32") + sgn = np.random.choice([-1, 1], [13, 17]).astype("float32") + y = x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype("float32") + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.005) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) + + +class TestElementwiseMaxOp_Vector(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_min" + x = np.random.random((32, )).astype("float32") + sgn = np.random.choice([-1, 1], (32, )).astype("float32") + y = x + sgn * np.random.uniform(0.1, 1, (32, )).astype("float32") + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} + + +class TestElementwiseMaxOp_broadcast_0(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_min" + x = np.random.uniform(0.5, 1, (2, 3, 4)).astype(np.float32) + sgn = np.random.choice([-1, 1], (2, )).astype(np.float32) + y = x[:, 0, 0] + sgn * \ + np.random.uniform(1, 2, (2, )).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': + np.minimum(self.inputs['X'], self.inputs['Y'].reshape(2, 1, 1)) + } + + +class TestElementwiseMaxOp_broadcast_1(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_min" + x = np.random.uniform(0.5, 1, (2, 3, 4)).astype(np.float32) + sgn = np.random.choice([-1, 1], (3, )).astype(np.float32) + y = x[0, :, 0] + sgn * \ + np.random.uniform(1, 2, (3, )).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.minimum(self.inputs['X'], self.inputs['Y'].reshape(1, 3, 1)) + } + + +class TestElementwiseMaxOp_broadcast_2(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_min" + x = np.random.uniform(0.5, 1, (2, 3, 4)).astype(np.float32) + sgn = np.random.choice([-1, 1], (4, )).astype(np.float32) + y = x[0, 0, :] + sgn * \ + np.random.uniform(1, 2, (4, )).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.outputs = { + 'Out': + np.minimum(self.inputs['X'], self.inputs['Y'].reshape(1, 1, 4)) + } + + +class TestElementwiseMaxOp_broadcast_3(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_min" + x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(np.float32) + sgn = np.random.choice([-1, 1], (3, 4)).astype(np.float32) + y = x[0, :, :, 0] + sgn * \ + np.random.uniform(1, 2, (3, 4)).astype(np.float32) + self.inputs = {'X': x, 'Y': y} + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.minimum(self.inputs['X'], self.inputs['Y'].reshape(1, 3, 4, 1)) + } + + +if __name__ == '__main__': + unittest.main() From c6d7ad368931f51766030bf9dc777106b1bdce83 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 20:49:26 +0800 Subject: [PATCH 019/158] fix typo --- python/paddle/v2/fluid/tests/test_elementwise_min_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py index 5ff5a40013..eeafab5b18 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py @@ -6,7 +6,7 @@ from op_test import OpTest class TestElementwiseOp(OpTest): def setUp(self): self.op_type = "elementwise_min" - # If x and y have the same value, the max() is not differentiable. + # If x and y have the same value, the min() is not differentiable. # So we generate test data by the following method # to avoid them being too close to each other. x = np.random.uniform(0.1, 1, [13, 17]).astype("float32") From ee8e5374d8cf079e389b79aef0ebf871c7b55545 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 15 Jan 2018 20:53:29 +0800 Subject: [PATCH 020/158] add max min layer --- python/paddle/v2/fluid/layers/ops.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index 51a85dbbd3..e60daa4f9f 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -42,6 +42,8 @@ __all__ = [ 'elementwise_div', 'elementwise_sub', 'elementwise_mul', + 'elementwise_max', + 'elementwise_min', 'clip', 'sequence_softmax', ] + __activations__ From bcfb82d33e431d621317f97d3c0703d9b002a8ee Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 15 Jan 2018 20:55:48 +0800 Subject: [PATCH 021/158] dist train support split selectedrows --- .../paddle/v2/fluid/distribute_transpiler.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index d17f9815cc..00fe3e68c9 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -59,6 +59,51 @@ def split_dense_variable(var_list, return blocks +def split_selected_rows(var, + pserver_count, + min_block_size=1024, + max_block_size=1048576): + assert ((len(var.shape)) <= 1) + + split_count = pserver_count + indices = var.desc.selected_rows().dims() + var_width = reduce(lambda x, y: x * y, var.shape[1:]) + row_count = len(indices) + rows_per_block = 1 + if var_width < min_block_size: + rows_per_block = 1 + split_count = row_count + else: + rows_per_block = row_count / pserver_count + if not rows_per_block % pserver_count: + rows_per_block += 1 + split_count = row_count / rows_per_block + if not row_count % rows_per_block: + split_count += 1 + blocks = [] + for block_id in xrange(split_count): + curr_block_rows = min(rows_per_block, + row_count - (block_id * rows_per_block)) + block = VarBlock(var.name, block_id, curr_block_rows) + blocks.append(block) + return blocks + + +def split_variable(var_list, + pserver_count, + min_block_size=1024, + max_block_size=1048576): + for var in var_list: + if var.type == core.VarDesc.VarType.LOD_TENSOR: + split_dense_variable(var_list, pserver_count, min_block_size, + max_block_size) + elif var.type == core.VarDesc.VarType.SELECTED_ROWS: + split_selected_rows(var_list, pserver_count, min_block_size, + max_block_size) + else: + raise TypeError("variable must be lodtensor or selected rows") + + class DistributeTranspiler: def transpile(self, optimize_ops, From 251c6032fb6bce72751b6b32e34368a17b407e6e Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 15 Jan 2018 19:49:14 +0800 Subject: [PATCH 022/158] set use_cudnn as default --- paddle/operators/conv_op.cc | 8 ++++++-- paddle/operators/conv_op.h | 1 - paddle/operators/conv_transpose_op.cc | 8 ++++++-- paddle/operators/conv_transpose_op.h | 1 - paddle/operators/pool_op.cc | 8 ++++++-- paddle/operators/pool_op.h | 1 - paddle/platform/dynload/cudnn.cc | 4 ---- python/paddle/v2/fluid/layers/nn.py | 6 +++--- python/paddle/v2/fluid/nets.py | 6 +++--- 9 files changed, 24 insertions(+), 19 deletions(-) diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index a6d3ebb4f5..9ae3e87281 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -70,7 +70,9 @@ void ConvOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - use_cudnn &= platform::dynload::HasCUDNN(); + if (paddle::platform::is_cpu_place(ctx.GetPlace())) { + use_cudnn = false; + } framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -284,7 +286,9 @@ void ConvOpGrad::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - use_cudnn &= platform::dynload::HasCUDNN(); + if (paddle::platform::is_cpu_place(ctx.GetPlace())) { + use_cudnn = false; + } framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_op.h b/paddle/operators/conv_op.h index 2a6fad6582..5a8933e791 100644 --- a/paddle/operators/conv_op.h +++ b/paddle/operators/conv_op.h @@ -19,7 +19,6 @@ limitations under the License. */ #include "paddle/operators/math/im2col.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/vol2col.h" -#include "paddle/platform/dynload/cudnn.h" namespace paddle { namespace operators { diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index 53153c0296..46f79b1701 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -61,7 +61,9 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvTransposeOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - use_cudnn &= platform::dynload::HasCUDNN(); + if (paddle::platform::is_cpu_place(ctx.GetPlace())) { + use_cudnn = false; + } framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -264,7 +266,9 @@ void ConvTransposeOpGrad::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvTransposeOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - use_cudnn &= platform::dynload::HasCUDNN(); + if (paddle::platform::is_cpu_place(ctx.GetPlace())) { + use_cudnn = false; + } framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h index f43d652fe1..a42ade41b1 100644 --- a/paddle/operators/conv_transpose_op.h +++ b/paddle/operators/conv_transpose_op.h @@ -19,7 +19,6 @@ limitations under the License. */ #include "paddle/operators/math/im2col.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/vol2col.h" -#include "paddle/platform/dynload/cudnn.h" namespace paddle { namespace operators { diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index e0f9ea7aa0..648a1dfb56 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -64,7 +64,9 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { framework::OpKernelType PoolOp::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - use_cudnn &= platform::dynload::HasCUDNN(); + if (paddle::platform::is_cpu_place(ctx.GetPlace())) { + use_cudnn = false; + } framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -89,7 +91,9 @@ void PoolOpGrad::InferShape(framework::InferShapeContext *ctx) const { framework::OpKernelType PoolOpGrad::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - use_cudnn &= platform::dynload::HasCUDNN(); + if (paddle::platform::is_cpu_place(ctx.GetPlace())) { + use_cudnn = false; + } framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index e0668f1360..c3d82ecbde 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -18,7 +18,6 @@ limitations under the License. */ #include "paddle/framework/op_registry.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/pooling.h" -#include "paddle/platform/dynload/cudnn.h" namespace paddle { namespace operators { diff --git a/paddle/platform/dynload/cudnn.cc b/paddle/platform/dynload/cudnn.cc index fafdf5e78a..701f6240fe 100644 --- a/paddle/platform/dynload/cudnn.cc +++ b/paddle/platform/dynload/cudnn.cc @@ -57,10 +57,6 @@ void EnforceCUDNNLoaded(const char* fn_name) { bool HasCUDNN() { return true; } #endif -#ifndef PADDLE_WITH_CUDA -bool HasCUDNN() { return false; } -#endif - } // namespace dynload } // namespace platform } // namespace paddle diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index df2233089c..0cfa011036 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -660,7 +660,7 @@ def conv2d(input, groups=None, param_attr=None, bias_attr=None, - use_cudnn=False, + use_cudnn=True, act=None): """ **Convlution2D Layer** @@ -938,7 +938,7 @@ def pool2d(input, pool_stride=None, pool_padding=None, global_pooling=False, - use_cudnn=False): + use_cudnn=True): """ This function adds the operator for pooling in 2 dimensions, using the pooling configurations mentioned in input parameters. @@ -1088,7 +1088,7 @@ def conv2d_transpose(input, stride=None, dilation=None, param_attr=None, - use_cudnn=False): + use_cudnn=True): """ The transpose of conv2d layer. diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index 8ac58fd733..327d2ff2da 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -14,7 +14,7 @@ def simple_img_conv_pool(input, act, param_attr=None, pool_type='max', - use_cudnn=False): + use_cudnn=True): conv_out = layers.conv2d( input=input, num_filters=num_filters, @@ -41,10 +41,10 @@ def img_conv_group(input, param_attr=None, conv_with_batchnorm=False, conv_batchnorm_drop_rate=None, - conv_use_cudnn=False, + conv_use_cudnn=True, pool_stride=1, pool_type=None, - pool_use_cudnn=False): + pool_use_cudnn=True): """ Image Convolution Group, Used for vgg net. """ From c01bb26f1d623a9180ade5ba53316321ff13685c Mon Sep 17 00:00:00 2001 From: yangyaming Date: Mon, 15 Jan 2018 21:04:15 +0800 Subject: [PATCH 023/158] Add reorder flag for DynamicRNN's memory function. --- python/paddle/v2/fluid/layers/control_flow.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index ee97e5f4e6..182b62a3af 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1310,20 +1310,44 @@ class DynamicRNN(object): else: return self.outputs - def memory(self, init=None, shape=None, value=0.0, dtype='float32'): + def memory(self, + init=None, + shape=None, + value=0.0, + need_reorder=False, + dtype='float32'): self._assert_in_rnn_block_('memory') if init is not None: if not isinstance(init, Variable): raise TypeError( "The input arg `init` of memory() must be a Variable") parent_block = self._parent_block_() + init_tensor = init + if need_reorder == True: + if self.lod_rank_table is None: + raise ValueError( + 'If set need_reorder to True, make sure step_input be ' + 'invoked before ' + 'memory(init=init, need_reordered=True, ...).') + init_reordered = parent_block.create_var( + name=unique_name('dynamic_rnn_mem_init_reordered'), + type=core.VarDesc.VarType.LOD_TENSOR, + dtype=init.dtype) + parent_block.append_op( + type='reorder_lod_tensor_by_rank', + inputs={ + 'X': [init_tensor], + 'RankTable': [self.lod_rank_table] + }, + outputs={'Out': [init_reordered]}) + init_tensor = init_reordered mem_array = parent_block.create_var( name=unique_name('dynamic_rnn_mem_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, dtype=init.dtype) parent_block.append_op( type='write_to_array', - inputs={'X': init, + inputs={'X': init_tensor, 'I': self.zero_idx}, outputs={'Out': mem_array}) retv = array_read(array=mem_array, i=self.step_idx) From ba2f6f71ad6e8a9518d53cd9df961ee070651625 Mon Sep 17 00:00:00 2001 From: sidgoyal78 Date: Mon, 15 Jan 2018 16:32:50 -0800 Subject: [PATCH 024/158] Modify directory structure to show plots --- paddle/operators/op_documentation/batch_norm_op.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/operators/op_documentation/batch_norm_op.md b/paddle/operators/op_documentation/batch_norm_op.md index 80948adf2b..d1392619c4 100644 --- a/paddle/operators/op_documentation/batch_norm_op.md +++ b/paddle/operators/op_documentation/batch_norm_op.md @@ -66,7 +66,7 @@ As most C++ operators do, `batch_norm_op` is defined by inputs, outputs, attribu The following graph showes the training computational process of `batch_norm_op`: - + cudnn provides APIs to finish the whole series of computation, we can use them in our GPU kernel. @@ -124,7 +124,7 @@ for pass_id in range(PASS_NUM): `is_infer` is an attribute. Once an operator is created, its attributes can not be changed. It suggests us that we shall maintain two `batch_norm_op` in the model, one's `is_infer` is `True`(we call it `infer_batch_norm_op`) and the other one's is `False`(we call it `train_batch_norm_op`). They share all parameters and variables, but be placed in two different branches. That is to say, if a network contains a `batch_norm_op`, it will fork into two branches, one go through `train_batch_norm_op` and the other one go through `infer_batch_norm_op`:
- +
Just like what is shown in the above graph, the net forks before `batch_norm_op` and will never merge again. All the operators after `batch_norm_op` will duplicate. From 24144bdb2c82f696c3eba65a2ab629756570b444 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Tue, 16 Jan 2018 10:31:23 +0800 Subject: [PATCH 025/158] Add notice about norm_by_times and change 'ctc' to 'CTC' --- python/paddle/v2/fluid/layers/nn.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 97056570ee..8a9c42b430 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1508,10 +1508,11 @@ def reduce_min(input, dim=None, keep_dim=False): def warpctc(input, label, blank=0, norm_by_times=False, **kwargs): """ - An operator integrating the open source warp-ctc library + An operator integrating the open source Warp-CTC library + (https://github.com/baidu-research/warp-ctc) to compute Connectionist Temporal Classification (CTC) loss. - It can be aliased as softmax with ctc, since a native softmax activation is - interated to the warp-ctc library, to to normlize values for each row of the + It can be aliased as softmax with CTC, since a native softmax activation is + interated to the Warp-CTC library, to to normlize values for each row of the input tensor. Args: @@ -1525,12 +1526,12 @@ def warpctc(input, label, blank=0, norm_by_times=False, **kwargs): of variable-length sequence, which is a 2-D Tensor with LoD information. It is of the shape [Lg, 1], where Lg is th sum of all labels' length. - blank: (int, default: 0), the blank label of Connectionist + blank: (int, default: 0), the blank label index of Connectionist Temporal Classification (CTC) loss, which is in the half-opened interval [0, num_classes + 1). - norm_by_times: (bool, default: false), whether to - normalize the gradients by the number of time-step, - which is also the sequence's length. + norm_by_times: (bool, default: false), whether to normalize the gradients + by the number of time-step,which is also the sequence's length. + There is no need to normalize the gradients if warpctc layer was follewed by a mean_op. Returns: Variable: The Connectionist Temporal Classification (CTC) loss, which is a 2-D Tensor of the shape [batch_size, 1]. From 4031e88dc4ec6559dd78e134a2830b7251759b1a Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 15 Jan 2018 19:21:31 -0800 Subject: [PATCH 026/158] "fix the copyright hook" --- .copyright.hook | 2 +- benchmark/tensorflow/image/googlenet_multi_gpu.py | 13 +++++++++++++ doc/getstarted/concepts/src/infer.py | 13 +++++++++++++ .../v2/fluid/tests/test_dynrnn_static_input.py | 13 +++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.copyright.hook b/.copyright.hook index e28e88e266..de97ce90ac 100644 --- a/.copyright.hook +++ b/.copyright.hook @@ -90,7 +90,7 @@ def main(argv=None): retv = 0 for filename in args.filenames: first_line = io.open(filename).readline() - if "Copyright" in first_line: continue + if "COPYRIGHT" in first_line.upper() : continue original_contents = io.open(filename).read() new_contents = generate_copyright( COPYRIGHT, lang_type(filename)) + original_contents diff --git a/benchmark/tensorflow/image/googlenet_multi_gpu.py b/benchmark/tensorflow/image/googlenet_multi_gpu.py index 31466faa37..44de3800a8 100644 --- a/benchmark/tensorflow/image/googlenet_multi_gpu.py +++ b/benchmark/tensorflow/image/googlenet_multi_gpu.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. from six.moves import xrange # pylint: disable=redefined-builtin from datetime import datetime import math diff --git a/doc/getstarted/concepts/src/infer.py b/doc/getstarted/concepts/src/infer.py index 4cc58dfee0..ee71cd7a9a 100644 --- a/doc/getstarted/concepts/src/infer.py +++ b/doc/getstarted/concepts/src/infer.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import paddle.v2 as paddle import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py b/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py index 9b138a6207..d6878f0b6d 100644 --- a/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py +++ b/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import paddle.v2 as paddle import paddle.v2.fluid.core as core From f382aa773c7d91bf517a2ff9cbd958349ae862d5 Mon Sep 17 00:00:00 2001 From: gx_wind Date: Tue, 16 Jan 2018 14:13:50 +0800 Subject: [PATCH 027/158] move adversarial directory to PaddlePaddle model repo --- adversarial/README.md | 9 -- adversarial/advbox/__init__.py | 16 --- adversarial/advbox/attacks/base.py | 52 ---------- adversarial/advbox/attacks/gradientsign.py | 51 --------- adversarial/advbox/models/__init__.py | 16 --- adversarial/advbox/models/base.py | 103 ------------------- adversarial/advbox/models/paddle.py | 114 --------------------- adversarial/fluid_mnist.py | 99 ------------------ adversarial/mnist_tutorial_fgsm.py | 100 ------------------ 9 files changed, 560 deletions(-) delete mode 100644 adversarial/README.md delete mode 100644 adversarial/advbox/__init__.py delete mode 100644 adversarial/advbox/attacks/base.py delete mode 100644 adversarial/advbox/attacks/gradientsign.py delete mode 100644 adversarial/advbox/models/__init__.py delete mode 100644 adversarial/advbox/models/base.py delete mode 100644 adversarial/advbox/models/paddle.py delete mode 100644 adversarial/fluid_mnist.py delete mode 100644 adversarial/mnist_tutorial_fgsm.py diff --git a/adversarial/README.md b/adversarial/README.md deleted file mode 100644 index 51da21918a..0000000000 --- a/adversarial/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Advbox - -Advbox is a Python toolbox to create adversarial examples that fool neural networks. It requires Python and paddle. - -## How to use - -1. train a model and save it's parameters. (like fluid_mnist.py) -2. load the parameters which is trained in step1, then reconstruct the model.(like mnist_tutorial_fgsm.py) -3. use advbox to generate the adversarial sample. diff --git a/adversarial/advbox/__init__.py b/adversarial/advbox/__init__.py deleted file mode 100644 index f56f14f18d..0000000000 --- a/adversarial/advbox/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - A set of tools for generating adversarial example on paddle platform -""" diff --git a/adversarial/advbox/attacks/base.py b/adversarial/advbox/attacks/base.py deleted file mode 100644 index 000baa48f6..0000000000 --- a/adversarial/advbox/attacks/base.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -""" -The base model of the model. -""" -from abc import ABCMeta, abstractmethod - - -class Attack(object): - """ - Abstract base class for adversarial attacks. `Attack` represent an adversarial attack - which search an adversarial example. subclass should implement the _apply() method. - - Args: - model(Model): an instance of the class advbox.base.Model. - - """ - __metaclass__ = ABCMeta - - def __init__(self, model): - self.model = model - - def __call__(self, image_label): - """ - Generate the adversarial sample. - - Args: - image_label(list): The image and label tuple list with one element. - """ - adv_img = self._apply(image_label) - return adv_img - - @abstractmethod - def _apply(self, image_label): - """ - Search an adversarial example. - - Args: - image_batch(list): The image and label tuple list with one element. - """ - raise NotImplementedError diff --git a/adversarial/advbox/attacks/gradientsign.py b/adversarial/advbox/attacks/gradientsign.py deleted file mode 100644 index 77d93bd793..0000000000 --- a/adversarial/advbox/attacks/gradientsign.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -""" -This module provide the attack method for FGSM's implement. -""" -from __future__ import division -import numpy as np -from collections import Iterable -from .base import Attack - - -class GradientSignAttack(Attack): - """ - This attack was originally implemented by Goodfellow et al. (2015) with the - infinity norm (and is known as the "Fast Gradient Sign Method"). This is therefore called - the Fast Gradient Method. - Paper link: https://arxiv.org/abs/1412.6572 - """ - - def _apply(self, image_label, epsilons=1000): - assert len(image_label) == 1 - pre_label = np.argmax(self.model.predict(image_label)) - - min_, max_ = self.model.bounds() - gradient = self.model.gradient(image_label) - gradient_sign = np.sign(gradient) * (max_ - min_) - - if not isinstance(epsilons, Iterable): - epsilons = np.linspace(0, 1, num=epsilons + 1) - - for epsilon in epsilons: - adv_img = image_label[0][0].reshape( - gradient_sign.shape) + epsilon * gradient_sign - adv_img = np.clip(adv_img, min_, max_) - adv_label = np.argmax(self.model.predict([(adv_img, 0)])) - if pre_label != adv_label: - return adv_img - - -FGSM = GradientSignAttack diff --git a/adversarial/advbox/models/__init__.py b/adversarial/advbox/models/__init__.py deleted file mode 100644 index eee0f6efd4..0000000000 --- a/adversarial/advbox/models/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Paddle model for target of attack -""" diff --git a/adversarial/advbox/models/base.py b/adversarial/advbox/models/base.py deleted file mode 100644 index 084e563f7b..0000000000 --- a/adversarial/advbox/models/base.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -""" -The base model of the model. -""" -from abc import ABCMeta -import abc - -abstractmethod = abc.abstractmethod - - -class Model(object): - """ - Base class of model to provide attack. - - - Args: - bounds(tuple): The lower and upper bound for the image pixel. - channel_axis(int): The index of the axis that represents the color channel. - preprocess(tuple): Two element tuple used to preprocess the input. First - substract the first element, then divide the second element. - """ - __metaclass__ = ABCMeta - - def __init__(self, bounds, channel_axis, preprocess=None): - assert len(bounds) == 2 - assert channel_axis in [0, 1, 2, 3] - - if preprocess is None: - preprocess = (0, 1) - self._bounds = bounds - self._channel_axis = channel_axis - self._preprocess = preprocess - - def bounds(self): - """ - Return the upper and lower bounds of the model. - """ - return self._bounds - - def channel_axis(self): - """ - Return the channel axis of the model. - """ - return self._channel_axis - - def _process_input(self, input_): - res = input_ - sub, div = self._preprocess - if sub != 0: - res = input_ - sub - assert div != 0 - if div != 1: - res /= div - return res - - @abstractmethod - def predict(self, image_batch): - """ - Calculate the prediction of the image batch. - - Args: - image_batch(numpy.ndarray): image batch of shape (batch_size, height, width, channels). - - Return: - numpy.ndarray: predictions of the images with shape (batch_size, num_of_classes). - """ - raise NotImplementedError - - @abstractmethod - def num_classes(self): - """ - Determine the number of the classes - - Return: - int: the number of the classes - """ - raise NotImplementedError - - @abstractmethod - def gradient(self, image_batch): - """ - Calculate the gradient of the cross-entropy loss w.r.t the image. - - Args: - image_batch(list): The image and label tuple list. - - Return: - numpy.ndarray: gradient of the cross-entropy loss w.r.t the image with - the shape (height, width, channel). - """ - raise NotImplementedError diff --git a/adversarial/advbox/models/paddle.py b/adversarial/advbox/models/paddle.py deleted file mode 100644 index 4048b47f89..0000000000 --- a/adversarial/advbox/models/paddle.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -from __future__ import absolute_import - -import numpy as np -import paddle.v2 as paddle -import paddle.v2.fluid as fluid -from paddle.v2.fluid.framework import program_guard - -from .base import Model - - -class PaddleModel(Model): - """ - Create a PaddleModel instance. - When you need to generate a adversarial sample, you should construct an instance of PaddleModel. - - Args: - program(paddle.v2.fluid.framework.Program): The program of the model which generate the adversarial sample. - input_name(string): The name of the input. - logits_name(string): The name of the logits. - predict_name(string): The name of the predict. - cost_name(string): The name of the loss in the program. - """ - - def __init__(self, - program, - input_name, - logits_name, - predict_name, - cost_name, - bounds, - channel_axis=3, - preprocess=None): - super(PaddleModel, self).__init__( - bounds=bounds, channel_axis=channel_axis, preprocess=preprocess) - - if preprocess is None: - preprocess = (0, 1) - - self._program = program - self._place = fluid.CPUPlace() - self._exe = fluid.Executor(self._place) - - self._input_name = input_name - self._logits_name = logits_name - self._predict_name = predict_name - self._cost_name = cost_name - - # gradient - loss = self._program.block(0).var(self._cost_name) - param_grads = fluid.backward.append_backward( - loss, parameter_list=[self._input_name]) - self._gradient = dict(param_grads)[self._input_name] - - def predict(self, image_batch): - """ - Predict the label of the image_batch. - - Args: - image_batch(list): The image and label tuple list. - Return: - numpy.ndarray: predictions of the images with shape (batch_size, num_of_classes). - """ - feeder = fluid.DataFeeder( - feed_list=[self._input_name, self._logits_name], - place=self._place, - program=self._program) - predict_var = self._program.block(0).var(self._predict_name) - predict = self._exe.run(self._program, - feed=feeder.feed(image_batch), - fetch_list=[predict_var]) - return predict - - def num_classes(self): - """ - Calculate the number of classes of the output label. - - Return: - int: the number of classes - """ - predict_var = self._program.block(0).var(self._predict_name) - assert len(predict_var.shape) == 2 - return predict_var.shape[1] - - def gradient(self, image_batch): - """ - Calculate the gradient of the loss w.r.t the input. - - Args: - image_batch(list): The image and label tuple list. - Return: - list: The list of the gradient of the image. - """ - feeder = fluid.DataFeeder( - feed_list=[self._input_name, self._logits_name], - place=self._place, - program=self._program) - - grad, = self._exe.run(self._program, - feed=feeder.feed(image_batch), - fetch_list=[self._gradient]) - return grad diff --git a/adversarial/fluid_mnist.py b/adversarial/fluid_mnist.py deleted file mode 100644 index f8c7fe8d0e..0000000000 --- a/adversarial/fluid_mnist.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -""" -CNN on mnist data using fluid api of paddlepaddle -""" -import paddle.v2 as paddle -import paddle.v2.fluid as fluid - - -def mnist_cnn_model(img): - """ - Mnist cnn model - - Args: - img(Varaible): the input image to be recognized - - Returns: - Variable: the label prediction - """ - conv_pool_1 = fluid.nets.simple_img_conv_pool( - input=img, - num_filters=20, - filter_size=5, - pool_size=2, - pool_stride=2, - act='relu') - - conv_pool_2 = fluid.nets.simple_img_conv_pool( - input=conv_pool_1, - num_filters=50, - filter_size=5, - pool_size=2, - pool_stride=2, - act='relu') - - logits = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') - return logits - - -def main(): - """ - Train the cnn model on mnist datasets - """ - img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - logits = mnist_cnn_model(img) - cost = fluid.layers.cross_entropy(input=logits, label=label) - avg_cost = fluid.layers.mean(x=cost) - optimizer = fluid.optimizer.Adam(learning_rate=0.01) - optimizer.minimize(avg_cost) - - accuracy = fluid.evaluator.Accuracy(input=logits, label=label) - - BATCH_SIZE = 50 - PASS_NUM = 3 - ACC_THRESHOLD = 0.98 - LOSS_THRESHOLD = 10.0 - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=[img, label], place=place) - exe.run(fluid.default_startup_program()) - - for pass_id in range(PASS_NUM): - accuracy.reset(exe) - for data in train_reader(): - loss, acc = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[avg_cost] + accuracy.metrics) - pass_acc = accuracy.eval(exe) - print("pass_id=" + str(pass_id) + " acc=" + str(acc) + " pass_acc=" - + str(pass_acc)) - if loss < LOSS_THRESHOLD and pass_acc > ACC_THRESHOLD: - break - - pass_acc = accuracy.eval(exe) - print("pass_id=" + str(pass_id) + " pass_acc=" + str(pass_acc)) - fluid.io.save_params( - exe, dirname='./mnist', main_program=fluid.default_main_program()) - print('train mnist done') - - -if __name__ == '__main__': - main() diff --git a/adversarial/mnist_tutorial_fgsm.py b/adversarial/mnist_tutorial_fgsm.py deleted file mode 100644 index c63e030cd8..0000000000 --- a/adversarial/mnist_tutorial_fgsm.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -""" -FGSM demos on mnist using advbox tool. -""" -import paddle.v2 as paddle -import paddle.v2.fluid as fluid -import matplotlib.pyplot as plt -import numpy as np - -from advbox.models.paddle import PaddleModel -from advbox.attacks.gradientsign import GradientSignAttack - - -def cnn_model(img): - """ - Mnist cnn model - Args: - img(Varaible): the input image to be recognized - Returns: - Variable: the label prediction - """ - #conv1 = fluid.nets.conv2d() - conv_pool_1 = fluid.nets.simple_img_conv_pool( - input=img, - num_filters=20, - filter_size=5, - pool_size=2, - pool_stride=2, - act='relu') - - conv_pool_2 = fluid.nets.simple_img_conv_pool( - input=conv_pool_1, - num_filters=50, - filter_size=5, - pool_size=2, - pool_stride=2, - act='relu') - - logits = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') - return logits - - -def main(): - """ - Advbox demo which demonstrate how to use advbox. - """ - IMG_NAME = 'img' - LABEL_NAME = 'label' - - img = fluid.layers.data(name=IMG_NAME, shape=[1, 28, 28], dtype='float32') - # gradient should flow - img.stop_gradient = False - label = fluid.layers.data(name=LABEL_NAME, shape=[1], dtype='int64') - logits = cnn_model(img) - cost = fluid.layers.cross_entropy(input=logits, label=label) - avg_cost = fluid.layers.mean(x=cost) - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - - BATCH_SIZE = 1 - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) - feeder = fluid.DataFeeder( - feed_list=[IMG_NAME, LABEL_NAME], - place=place, - program=fluid.default_main_program()) - - fluid.io.load_params( - exe, "./mnist/", main_program=fluid.default_main_program()) - - # advbox demo - m = PaddleModel(fluid.default_main_program(), IMG_NAME, LABEL_NAME, - logits.name, avg_cost.name, (-1, 1)) - att = GradientSignAttack(m) - for data in train_reader(): - # fgsm attack - adv_img = att(data) - plt.imshow(n[0][0], cmap='Greys_r') - plt.show() - #np.save('adv_img', adv_img) - break - - -if __name__ == '__main__': - main() From ead7059bf93e362978c8463350ad9551db43b2b9 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 16 Jan 2018 14:35:26 +0800 Subject: [PATCH 028/158] Refine code --- paddle/operators/elementwise_max_op.h | 34 +------------------ paddle/operators/elementwise_min_op.h | 34 +------------------ paddle/operators/elementwise_op_function.h | 38 ++++++++++++++++++++++ 3 files changed, 40 insertions(+), 66 deletions(-) diff --git a/paddle/operators/elementwise_max_op.h b/paddle/operators/elementwise_max_op.h index 92152f7cb6..255728e8e6 100644 --- a/paddle/operators/elementwise_max_op.h +++ b/paddle/operators/elementwise_max_op.h @@ -28,39 +28,7 @@ template class ElementwiseMaxKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); - TransformFunctor, T, DeviceContext> functor( - x, y, z, ctx.template device_context(), MaxFunctor()); - - auto x_dims = x->dims(); - auto y_dims = y->dims(); - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input."); - - if (x_dims == y_dims) { - functor.Run(); - return; - } - - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), - "Axis should be in range [0, x_dims)"); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - if (post == 1) { - functor.RunRowWise(n, pre); - return; - } else { - functor.RunMidWise(n, pre, post); - return; - } + ElementwiseComputeEx, DeviceContext, T>(ctx); } }; diff --git a/paddle/operators/elementwise_min_op.h b/paddle/operators/elementwise_min_op.h index 53b7f59fa0..e6627a0f1b 100644 --- a/paddle/operators/elementwise_min_op.h +++ b/paddle/operators/elementwise_min_op.h @@ -28,39 +28,7 @@ template class ElementwiseMinKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); - TransformFunctor, T, DeviceContext> functor( - x, y, z, ctx.template device_context(), MinFunctor()); - - auto x_dims = x->dims(); - auto y_dims = y->dims(); - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input."); - - if (x_dims == y_dims) { - functor.Run(); - return; - } - - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), - "Axis should be in range [0, x_dims)"); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - if (post == 1) { - functor.RunRowWise(n, pre); - return; - } else { - functor.RunMidWise(n, pre, post); - return; - } + ElementwiseComputeEx, DeviceContext, T>(ctx); } }; diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index 0c75276b03..be11d5cc9d 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -356,5 +356,43 @@ void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { return; } } + +template +void ElementwiseComputeEx(const framework::ExecutionContext& ctx) { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* z = ctx.Output("Out"); + z->mutable_data(ctx.GetPlace()); + TransformFunctor functor( + x, y, z, ctx.template device_context(), Functor()); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input."); + + if (x_dims == y_dims) { + functor.Run(); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), + "Axis should be in range [0, x_dims)"); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + if (post == 1) { + functor.RunRowWise(n, pre); + return; + } else { + functor.RunMidWise(n, pre, post); + return; + } +} + } // namespace operators } // namespace paddle From f59599a3dd46def0215d441ddfb023c3ec3d2632 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 16 Jan 2018 14:39:14 +0800 Subject: [PATCH 029/158] refine elementwise_add_op --- paddle/operators/elementwise_add_op.h | 34 +-------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/paddle/operators/elementwise_add_op.h b/paddle/operators/elementwise_add_op.h index 6478e1e0c2..a8389429f2 100644 --- a/paddle/operators/elementwise_add_op.h +++ b/paddle/operators/elementwise_add_op.h @@ -28,39 +28,7 @@ template class ElementwiseAddKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); - TransformFunctor, T, DeviceContext> functor( - x, y, z, ctx.template device_context(), AddFunctor()); - - auto x_dims = x->dims(); - auto y_dims = y->dims(); - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input."); - - if (x_dims == y_dims) { - functor.Run(); - return; - } - - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), - "Axis should be in range [0, x_dims)"); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - if (post == 1) { - functor.RunRowWise(n, pre); - return; - } else { - functor.RunMidWise(n, pre, post); - return; - } + ElementwiseComputeEx, DeviceContext, T>(ctx); } }; From 0d4b8ae13364f5129b726894a18f5586714cd0e7 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 16 Jan 2018 14:42:47 +0800 Subject: [PATCH 030/158] remove v1_api_demo --- v1_api_demo/README.md | 5 - v1_api_demo/gan/.gitignore | 11 - v1_api_demo/gan/README.md | 13 - v1_api_demo/gan/data/download_cifar.sh | 18 - v1_api_demo/gan/data/get_mnist_data.sh | 17 - v1_api_demo/gan/gan_conf.py | 151 -------- v1_api_demo/gan/gan_conf_image.py | 298 --------------- v1_api_demo/gan/gan_trainer.py | 349 ------------------ v1_api_demo/mnist/.gitignore | 10 - v1_api_demo/mnist/api_train.py | 209 ----------- v1_api_demo/mnist/data/generate_list.py | 21 -- v1_api_demo/mnist/data/get_mnist_data.sh | 21 -- v1_api_demo/mnist/light_mnist.py | 79 ---- v1_api_demo/mnist/mnist_provider.py | 25 -- v1_api_demo/mnist/mnist_util.py | 30 -- v1_api_demo/mnist/train.sh | 32 -- v1_api_demo/mnist/vgg_16_mnist.py | 50 --- v1_api_demo/model_zoo/embedding/.gitignore | 2 - .../model_zoo/embedding/extract_para.py | 113 ------ .../model_zoo/embedding/paraconvert.py | 159 -------- .../model_zoo/embedding/pre_DictAndModel.sh | 32 -- v1_api_demo/model_zoo/resnet/.gitignore | 5 - v1_api_demo/model_zoo/resnet/classify.py | 312 ---------------- .../model_zoo/resnet/example/.gitignore | 1 - .../model_zoo/resnet/example/__init__.py | 13 - v1_api_demo/model_zoo/resnet/example/cat.jpg | Bin 12881 -> 0 bytes v1_api_demo/model_zoo/resnet/example/dog.jpg | Bin 71483 -> 0 bytes .../resnet/example/image_list_provider.py | 102 ----- .../model_zoo/resnet/example/test.list | 2 - .../model_zoo/resnet/extract_fea_c++.sh | 40 -- .../model_zoo/resnet/extract_fea_py.sh | 29 -- v1_api_demo/model_zoo/resnet/get_model.sh | 32 -- v1_api_demo/model_zoo/resnet/load_feature.py | 63 ---- v1_api_demo/model_zoo/resnet/net_diagram.sh | 39 -- v1_api_demo/model_zoo/resnet/predict.sh | 23 -- v1_api_demo/model_zoo/resnet/resnet.py | 271 -------------- v1_api_demo/quick_start/.gitignore | 15 - v1_api_demo/quick_start/api_predict.py | 147 -------- v1_api_demo/quick_start/api_predict.sh | 30 -- v1_api_demo/quick_start/api_train.py | 122 ------ v1_api_demo/quick_start/api_train.sh | 29 -- .../quick_start/cluster/cluster_train.sh | 45 --- v1_api_demo/quick_start/cluster/env.sh | 28 -- v1_api_demo/quick_start/cluster/pserver.sh | 26 -- v1_api_demo/quick_start/data/README.md | 9 - v1_api_demo/quick_start/data/get_data.sh | 27 -- .../data/proc_from_raw_data/get_data.sh | 79 ---- .../data/proc_from_raw_data/preprocess.py | 236 ------------ v1_api_demo/quick_start/dataprovider_bow.py | 86 ----- v1_api_demo/quick_start/dataprovider_emb.py | 52 --- v1_api_demo/quick_start/predict.sh | 32 -- v1_api_demo/quick_start/train.sh | 34 -- .../quick_start/trainer_config.bidi-lstm.py | 74 ---- v1_api_demo/quick_start/trainer_config.cnn.py | 68 ---- .../quick_start/trainer_config.db-lstm.py | 74 ---- v1_api_demo/quick_start/trainer_config.emb.py | 64 ---- v1_api_demo/quick_start/trainer_config.lr.py | 85 ----- .../quick_start/trainer_config.lstm.py | 70 ---- .../quick_start/trainer_config.resnet-lstm.py | 104 ------ v1_api_demo/sequence_tagging/data/get_data.sh | 21 -- v1_api_demo/sequence_tagging/data/test.list | 1 - v1_api_demo/sequence_tagging/data/train.list | 1 - v1_api_demo/sequence_tagging/dataprovider.py | 260 ------------- v1_api_demo/sequence_tagging/linear_crf.py | 83 ----- v1_api_demo/sequence_tagging/readme.md | 45 --- v1_api_demo/sequence_tagging/rnn_crf.py | 121 ------ v1_api_demo/sequence_tagging/train.sh | 12 - v1_api_demo/sequence_tagging/train_linear.sh | 11 - v1_api_demo/traffic_prediction/README | 7 - .../traffic_prediction/data/get_data.sh | 34 -- .../traffic_prediction/dataprovider.py | 82 ---- v1_api_demo/traffic_prediction/gen_result.py | 61 --- v1_api_demo/traffic_prediction/predict.sh | 30 -- v1_api_demo/traffic_prediction/train.sh | 27 -- .../traffic_prediction/trainer_config.py | 52 --- v1_api_demo/vae/README.md | 13 - v1_api_demo/vae/data/get_mnist_data.sh | 17 - v1_api_demo/vae/dataloader.py | 60 --- v1_api_demo/vae/vae_conf.py | 116 ------ v1_api_demo/vae/vae_train.py | 175 --------- 80 files changed, 5342 deletions(-) delete mode 100644 v1_api_demo/README.md delete mode 100644 v1_api_demo/gan/.gitignore delete mode 100644 v1_api_demo/gan/README.md delete mode 100755 v1_api_demo/gan/data/download_cifar.sh delete mode 100755 v1_api_demo/gan/data/get_mnist_data.sh delete mode 100644 v1_api_demo/gan/gan_conf.py delete mode 100644 v1_api_demo/gan/gan_conf_image.py delete mode 100644 v1_api_demo/gan/gan_trainer.py delete mode 100644 v1_api_demo/mnist/.gitignore delete mode 100644 v1_api_demo/mnist/api_train.py delete mode 100644 v1_api_demo/mnist/data/generate_list.py delete mode 100755 v1_api_demo/mnist/data/get_mnist_data.sh delete mode 100644 v1_api_demo/mnist/light_mnist.py delete mode 100644 v1_api_demo/mnist/mnist_provider.py delete mode 100644 v1_api_demo/mnist/mnist_util.py delete mode 100755 v1_api_demo/mnist/train.sh delete mode 100644 v1_api_demo/mnist/vgg_16_mnist.py delete mode 100644 v1_api_demo/model_zoo/embedding/.gitignore delete mode 100755 v1_api_demo/model_zoo/embedding/extract_para.py delete mode 100755 v1_api_demo/model_zoo/embedding/paraconvert.py delete mode 100755 v1_api_demo/model_zoo/embedding/pre_DictAndModel.sh delete mode 100644 v1_api_demo/model_zoo/resnet/.gitignore delete mode 100755 v1_api_demo/model_zoo/resnet/classify.py delete mode 100644 v1_api_demo/model_zoo/resnet/example/.gitignore delete mode 100644 v1_api_demo/model_zoo/resnet/example/__init__.py delete mode 100644 v1_api_demo/model_zoo/resnet/example/cat.jpg delete mode 100644 v1_api_demo/model_zoo/resnet/example/dog.jpg delete mode 100644 v1_api_demo/model_zoo/resnet/example/image_list_provider.py delete mode 100644 v1_api_demo/model_zoo/resnet/example/test.list delete mode 100755 v1_api_demo/model_zoo/resnet/extract_fea_c++.sh delete mode 100755 v1_api_demo/model_zoo/resnet/extract_fea_py.sh delete mode 100755 v1_api_demo/model_zoo/resnet/get_model.sh delete mode 100644 v1_api_demo/model_zoo/resnet/load_feature.py delete mode 100755 v1_api_demo/model_zoo/resnet/net_diagram.sh delete mode 100755 v1_api_demo/model_zoo/resnet/predict.sh delete mode 100644 v1_api_demo/model_zoo/resnet/resnet.py delete mode 100644 v1_api_demo/quick_start/.gitignore delete mode 100755 v1_api_demo/quick_start/api_predict.py delete mode 100755 v1_api_demo/quick_start/api_predict.sh delete mode 100644 v1_api_demo/quick_start/api_train.py delete mode 100755 v1_api_demo/quick_start/api_train.sh delete mode 100755 v1_api_demo/quick_start/cluster/cluster_train.sh delete mode 100644 v1_api_demo/quick_start/cluster/env.sh delete mode 100755 v1_api_demo/quick_start/cluster/pserver.sh delete mode 100644 v1_api_demo/quick_start/data/README.md delete mode 100755 v1_api_demo/quick_start/data/get_data.sh delete mode 100755 v1_api_demo/quick_start/data/proc_from_raw_data/get_data.sh delete mode 100755 v1_api_demo/quick_start/data/proc_from_raw_data/preprocess.py delete mode 100644 v1_api_demo/quick_start/dataprovider_bow.py delete mode 100755 v1_api_demo/quick_start/dataprovider_emb.py delete mode 100755 v1_api_demo/quick_start/predict.sh delete mode 100755 v1_api_demo/quick_start/train.sh delete mode 100644 v1_api_demo/quick_start/trainer_config.bidi-lstm.py delete mode 100644 v1_api_demo/quick_start/trainer_config.cnn.py delete mode 100644 v1_api_demo/quick_start/trainer_config.db-lstm.py delete mode 100644 v1_api_demo/quick_start/trainer_config.emb.py delete mode 100644 v1_api_demo/quick_start/trainer_config.lr.py delete mode 100644 v1_api_demo/quick_start/trainer_config.lstm.py delete mode 100644 v1_api_demo/quick_start/trainer_config.resnet-lstm.py delete mode 100755 v1_api_demo/sequence_tagging/data/get_data.sh delete mode 100644 v1_api_demo/sequence_tagging/data/test.list delete mode 100644 v1_api_demo/sequence_tagging/data/train.list delete mode 100644 v1_api_demo/sequence_tagging/dataprovider.py delete mode 100644 v1_api_demo/sequence_tagging/linear_crf.py delete mode 100644 v1_api_demo/sequence_tagging/readme.md delete mode 100644 v1_api_demo/sequence_tagging/rnn_crf.py delete mode 100755 v1_api_demo/sequence_tagging/train.sh delete mode 100755 v1_api_demo/sequence_tagging/train_linear.sh delete mode 100644 v1_api_demo/traffic_prediction/README delete mode 100755 v1_api_demo/traffic_prediction/data/get_data.sh delete mode 100644 v1_api_demo/traffic_prediction/dataprovider.py delete mode 100644 v1_api_demo/traffic_prediction/gen_result.py delete mode 100755 v1_api_demo/traffic_prediction/predict.sh delete mode 100755 v1_api_demo/traffic_prediction/train.sh delete mode 100755 v1_api_demo/traffic_prediction/trainer_config.py delete mode 100644 v1_api_demo/vae/README.md delete mode 100755 v1_api_demo/vae/data/get_mnist_data.sh delete mode 100644 v1_api_demo/vae/dataloader.py delete mode 100644 v1_api_demo/vae/vae_conf.py delete mode 100644 v1_api_demo/vae/vae_train.py diff --git a/v1_api_demo/README.md b/v1_api_demo/README.md deleted file mode 100644 index 0460a85fae..0000000000 --- a/v1_api_demo/README.md +++ /dev/null @@ -1,5 +0,0 @@ -The examples in v1_api_demo are using v1_api currently, and will be upgraded to v2_api later. -Thus, v1_api_demo is a temporary directory. We decide not to maintain it and will delete it in future. - -Please go to [PaddlePaddle/book](https://github.com/PaddlePaddle/book) and -[PaddlePaddle/models](https://github.com/PaddlePaddle/models) to learn PaddlePaddle. diff --git a/v1_api_demo/gan/.gitignore b/v1_api_demo/gan/.gitignore deleted file mode 100644 index 93a6f5080a..0000000000 --- a/v1_api_demo/gan/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -output/ -uniform_params/ -cifar_params/ -mnist_params/ -*.png -.pydevproject -.project -*.log -*.pyc -data/mnist_data/ -data/cifar-10-batches-py/ diff --git a/v1_api_demo/gan/README.md b/v1_api_demo/gan/README.md deleted file mode 100644 index 1908b534b0..0000000000 --- a/v1_api_demo/gan/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Generative Adversarial Networks (GAN) - -This demo implements GAN training described in the original GAN paper (https://arxiv.org/abs/1406.2661) and DCGAN (https://arxiv.org/abs/1511.06434). - -The general training procedures are implemented in gan_trainer.py. The neural network configurations are specified in gan_conf.py (for synthetic data) and gan_conf_image.py (for image data). - -In order to run the model, first download the corresponding data by running the shell script in ./data. -Then you can run the command below. The flag -d specifies the training data (cifar, mnist or uniform) and flag --useGpu specifies whether to use gpu for training (0 is cpu, 1 is gpu). - -$python gan_trainer.py -d cifar --use_gpu 1 - -The generated images will be stored in ./cifar_samples/ -The corresponding models will be stored in ./cifar_params/ diff --git a/v1_api_demo/gan/data/download_cifar.sh b/v1_api_demo/gan/data/download_cifar.sh deleted file mode 100755 index bbadc7c10c..0000000000 --- a/v1_api_demo/gan/data/download_cifar.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz -tar zxf cifar-10-python.tar.gz -rm cifar-10-python.tar.gz diff --git a/v1_api_demo/gan/data/get_mnist_data.sh b/v1_api_demo/gan/data/get_mnist_data.sh deleted file mode 100755 index a77c81bf5a..0000000000 --- a/v1_api_demo/gan/data/get_mnist_data.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env sh -# This script downloads the mnist data and unzips it. -set -e -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -rm -rf "$DIR/mnist_data" -mkdir "$DIR/mnist_data" -cd "$DIR/mnist_data" - -echo "Downloading..." - -for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte -do - if [ ! -e $fname ]; then - wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz - gunzip ${fname}.gz - fi -done diff --git a/v1_api_demo/gan/gan_conf.py b/v1_api_demo/gan/gan_conf.py deleted file mode 100644 index 86ac2dffe5..0000000000 --- a/v1_api_demo/gan/gan_conf.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from paddle.trainer_config_helpers import * - -mode = get_config_arg("mode", str, "generator") -assert mode in set([ - "generator", "discriminator", "generator_training", "discriminator_training" -]) - -is_generator_training = mode == "generator_training" -is_discriminator_training = mode == "discriminator_training" -is_generator = mode == "generator" -is_discriminator = mode == "discriminator" - -# The network structure below follows the ref https://arxiv.org/abs/1406.2661 -# Here we used two hidden layers and batch_norm - -print('mode=%s' % mode) -# the dim of the noise (z) as the input of the generator network -noise_dim = 10 -# the dim of the hidden layer -hidden_dim = 10 -# the dim of the generated sample -sample_dim = 2 - -settings( - batch_size=128, - learning_rate=1e-4, - learning_method=AdamOptimizer(beta1=0.5)) - - -def discriminator(sample): - """ - discriminator ouputs the probablity of a sample is from generator - or real data. - The output has two dimenstional: dimension 0 is the probablity - of the sample is from generator and dimension 1 is the probabblity - of the sample is from real data. - """ - param_attr = ParamAttr(is_static=is_generator_training) - bias_attr = ParamAttr( - is_static=is_generator_training, initial_mean=1.0, initial_std=0) - - hidden = fc_layer( - input=sample, - name="dis_hidden", - size=hidden_dim, - bias_attr=bias_attr, - param_attr=param_attr, - act=ReluActivation()) - - hidden2 = fc_layer( - input=hidden, - name="dis_hidden2", - size=hidden_dim, - bias_attr=bias_attr, - param_attr=param_attr, - act=LinearActivation()) - - hidden_bn = batch_norm_layer( - hidden2, - act=ReluActivation(), - name="dis_hidden_bn", - bias_attr=bias_attr, - param_attr=ParamAttr( - is_static=is_generator_training, initial_mean=1.0, - initial_std=0.02), - use_global_stats=False) - - return fc_layer( - input=hidden_bn, - name="dis_prob", - size=2, - bias_attr=bias_attr, - param_attr=param_attr, - act=SoftmaxActivation()) - - -def generator(noise): - """ - generator generates a sample given noise - """ - param_attr = ParamAttr(is_static=is_discriminator_training) - bias_attr = ParamAttr( - is_static=is_discriminator_training, initial_mean=1.0, initial_std=0) - - hidden = fc_layer( - input=noise, - name="gen_layer_hidden", - size=hidden_dim, - bias_attr=bias_attr, - param_attr=param_attr, - act=ReluActivation()) - - hidden2 = fc_layer( - input=hidden, - name="gen_hidden2", - size=hidden_dim, - bias_attr=bias_attr, - param_attr=param_attr, - act=LinearActivation()) - - hidden_bn = batch_norm_layer( - hidden2, - act=ReluActivation(), - name="gen_layer_hidden_bn", - bias_attr=bias_attr, - param_attr=ParamAttr( - is_static=is_discriminator_training, - initial_mean=1.0, - initial_std=0.02), - use_global_stats=False) - - return fc_layer( - input=hidden_bn, - name="gen_layer1", - size=sample_dim, - bias_attr=bias_attr, - param_attr=param_attr, - act=LinearActivation()) - - -if is_generator_training: - noise = data_layer(name="noise", size=noise_dim) - sample = generator(noise) - -if is_discriminator_training: - sample = data_layer(name="sample", size=sample_dim) - -if is_generator_training or is_discriminator_training: - label = data_layer(name="label", size=1) - prob = discriminator(sample) - cost = cross_entropy(input=prob, label=label) - classification_error_evaluator( - input=prob, label=label, name=mode + '_error') - outputs(cost) - -if is_generator: - noise = data_layer(name="noise", size=noise_dim) - outputs(generator(noise)) diff --git a/v1_api_demo/gan/gan_conf_image.py b/v1_api_demo/gan/gan_conf_image.py deleted file mode 100644 index c469227994..0000000000 --- a/v1_api_demo/gan/gan_conf_image.py +++ /dev/null @@ -1,298 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from paddle.trainer_config_helpers import * - -mode = get_config_arg("mode", str, "generator") -dataSource = get_config_arg("data", str, "mnist") -assert mode in set([ - "generator", "discriminator", "generator_training", "discriminator_training" -]) - -is_generator_training = mode == "generator_training" -is_discriminator_training = mode == "discriminator_training" -is_generator = mode == "generator" -is_discriminator = mode == "discriminator" - -# The network structure below follows the dcgan paper -# (https://arxiv.org/abs/1511.06434) - -print('mode=%s' % mode) -# the dim of the noise (z) as the input of the generator network -noise_dim = 100 -# the number of filters in the layer in generator/discriminator that is -# closet to the image -gf_dim = 64 -df_dim = 64 -if dataSource == "mnist": - sample_dim = 28 # image dim - c_dim = 1 # image color -else: - sample_dim = 32 - c_dim = 3 -s2, s4 = int(sample_dim / 2), int(sample_dim / 4), -s8, s16 = int(sample_dim / 8), int(sample_dim / 16) - -settings( - batch_size=128, - learning_rate=2e-4, - learning_method=AdamOptimizer(beta1=0.5)) - - -def conv_bn(input, - channels, - imgSize, - num_filters, - output_x, - stride, - name, - param_attr, - bias_attr, - param_attr_bn, - bn, - trans=False, - act=ReluActivation()): - """ - conv_bn is a utility function that constructs a convolution/deconv layer - with an optional batch_norm layer - - :param bn: whether to use batch_norm_layer - :type bn: bool - :param trans: whether to use conv (False) or deconv (True) - :type trans: bool - """ - - # calculate the filter_size and padding size based on the given - # imgSize and ouput size - tmp = imgSize - (output_x - 1) * stride - if tmp <= 1 or tmp > 5: - raise ValueError("conv input-output dimension does not fit") - elif tmp <= 3: - filter_size = tmp + 2 - padding = 1 - else: - filter_size = tmp - padding = 0 - - print(imgSize, output_x, stride, filter_size, padding) - - if trans: - nameApx = "_convt" - else: - nameApx = "_conv" - - if bn: - conv = img_conv_layer( - input, - filter_size=filter_size, - num_filters=num_filters, - name=name + nameApx, - num_channels=channels, - act=LinearActivation(), - groups=1, - stride=stride, - padding=padding, - bias_attr=bias_attr, - param_attr=param_attr, - shared_biases=True, - layer_attr=None, - filter_size_y=None, - stride_y=None, - padding_y=None, - trans=trans) - - conv_bn = batch_norm_layer( - conv, - act=act, - name=name + nameApx + "_bn", - bias_attr=bias_attr, - param_attr=param_attr_bn, - use_global_stats=False) - - return conv_bn - else: - conv = img_conv_layer( - input, - filter_size=filter_size, - num_filters=num_filters, - name=name + nameApx, - num_channels=channels, - act=act, - groups=1, - stride=stride, - padding=padding, - bias_attr=bias_attr, - param_attr=param_attr, - shared_biases=True, - layer_attr=None, - filter_size_y=None, - stride_y=None, - padding_y=None, - trans=trans) - return conv - - -def generator(noise): - """ - generator generates a sample given noise - """ - param_attr = ParamAttr( - is_static=is_discriminator_training, initial_mean=0.0, initial_std=0.02) - bias_attr = ParamAttr( - is_static=is_discriminator_training, initial_mean=0.0, initial_std=0.0) - - param_attr_bn = ParamAttr( - is_static=is_discriminator_training, initial_mean=1.0, initial_std=0.02) - - h1 = fc_layer( - input=noise, - name="gen_layer_h1", - size=s8 * s8 * gf_dim * 4, - bias_attr=bias_attr, - param_attr=param_attr, - act=LinearActivation()) - - h1_bn = batch_norm_layer( - h1, - act=ReluActivation(), - name="gen_layer_h1_bn", - bias_attr=bias_attr, - param_attr=param_attr_bn, - use_global_stats=False) - - h2_bn = conv_bn( - h1_bn, - channels=gf_dim * 4, - output_x=s8, - num_filters=gf_dim * 2, - imgSize=s4, - stride=2, - name="gen_layer_h2", - param_attr=param_attr, - bias_attr=bias_attr, - param_attr_bn=param_attr_bn, - bn=True, - trans=True) - - h3_bn = conv_bn( - h2_bn, - channels=gf_dim * 2, - output_x=s4, - num_filters=gf_dim, - imgSize=s2, - stride=2, - name="gen_layer_h3", - param_attr=param_attr, - bias_attr=bias_attr, - param_attr_bn=param_attr_bn, - bn=True, - trans=True) - - return conv_bn( - h3_bn, - channels=gf_dim, - output_x=s2, - num_filters=c_dim, - imgSize=sample_dim, - stride=2, - name="gen_layer_h4", - param_attr=param_attr, - bias_attr=bias_attr, - param_attr_bn=param_attr_bn, - bn=False, - trans=True, - act=TanhActivation()) - - -def discriminator(sample): - """ - discriminator ouputs the probablity of a sample is from generator - or real data. - The output has two dimenstional: dimension 0 is the probablity - of the sample is from generator and dimension 1 is the probabblity - of the sample is from real data. - """ - param_attr = ParamAttr( - is_static=is_generator_training, initial_mean=0.0, initial_std=0.02) - bias_attr = ParamAttr( - is_static=is_generator_training, initial_mean=0.0, initial_std=0.0) - - param_attr_bn = ParamAttr( - is_static=is_generator_training, initial_mean=1.0, initial_std=0.02) - - h0 = conv_bn( - sample, - channels=c_dim, - imgSize=sample_dim, - num_filters=df_dim, - output_x=s2, - stride=2, - name="dis_h0", - param_attr=param_attr, - bias_attr=bias_attr, - param_attr_bn=param_attr_bn, - bn=False) - - h1_bn = conv_bn( - h0, - channels=df_dim, - imgSize=s2, - num_filters=df_dim * 2, - output_x=s4, - stride=2, - name="dis_h1", - param_attr=param_attr, - bias_attr=bias_attr, - param_attr_bn=param_attr_bn, - bn=True) - - h2_bn = conv_bn( - h1_bn, - channels=df_dim * 2, - imgSize=s4, - num_filters=df_dim * 4, - output_x=s8, - stride=2, - name="dis_h2", - param_attr=param_attr, - bias_attr=bias_attr, - param_attr_bn=param_attr_bn, - bn=True) - - return fc_layer( - input=h2_bn, - name="dis_prob", - size=2, - bias_attr=bias_attr, - param_attr=param_attr, - act=SoftmaxActivation()) - - -if is_generator_training: - noise = data_layer(name="noise", size=noise_dim) - sample = generator(noise) - -if is_discriminator_training: - sample = data_layer(name="sample", size=sample_dim * sample_dim * c_dim) - -if is_generator_training or is_discriminator_training: - label = data_layer(name="label", size=1) - prob = discriminator(sample) - cost = cross_entropy(input=prob, label=label) - classification_error_evaluator( - input=prob, label=label, name=mode + '_error') - outputs(cost) - -if is_generator: - noise = data_layer(name="noise", size=noise_dim) - outputs(generator(noise)) diff --git a/v1_api_demo/gan/gan_trainer.py b/v1_api_demo/gan/gan_trainer.py deleted file mode 100644 index 4a26c230f7..0000000000 --- a/v1_api_demo/gan/gan_trainer.py +++ /dev/null @@ -1,349 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import random -import numpy -import cPickle -import sys, os -from PIL import Image - -from paddle.trainer.config_parser import parse_config -from paddle.trainer.config_parser import logger -import py_paddle.swig_paddle as api -import matplotlib.pyplot as plt - - -def plot2DScatter(data, outputfile): - ''' - Plot the data as a 2D scatter plot and save to outputfile - data needs to be two dimensinoal - ''' - x = data[:, 0] - y = data[:, 1] - logger.info("The mean vector is %s" % numpy.mean(data, 0)) - logger.info("The std vector is %s" % numpy.std(data, 0)) - - heatmap, xedges, yedges = numpy.histogram2d(x, y, bins=50) - extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] - - plt.clf() - plt.scatter(x, y) - plt.savefig(outputfile, bbox_inches='tight') - - -def CHECK_EQ(a, b): - assert a == b, "a=%s, b=%s" % (a, b) - - -def copy_shared_parameters(src, dst): - ''' - copy the parameters from src to dst - :param src: the source of the parameters - :type src: GradientMachine - :param dst: the destination of the parameters - :type dst: GradientMachine - ''' - src_params = [src.getParameter(i) for i in xrange(src.getParameterSize())] - src_params = dict([(p.getName(), p) for p in src_params]) - - for i in xrange(dst.getParameterSize()): - dst_param = dst.getParameter(i) - src_param = src_params.get(dst_param.getName(), None) - if src_param is None: - continue - src_value = src_param.getBuf(api.PARAMETER_VALUE) - dst_value = dst_param.getBuf(api.PARAMETER_VALUE) - CHECK_EQ(len(src_value), len(dst_value)) - dst_value.copyFrom(src_value) - dst_param.setValueUpdated() - - -def print_parameters(src): - src_params = [src.getParameter(i) for i in xrange(src.getParameterSize())] - - print "***************" - for p in src_params: - print "Name is %s" % p.getName() - print "value is %s \n" % p.getBuf(api.PARAMETER_VALUE).copyToNumpyArray( - ) - - -def load_mnist_data(imageFile): - f = open(imageFile, "rb") - f.read(16) - - # Define number of samples for train/test - if "train" in imageFile: - n = 60000 - else: - n = 10000 - - data = numpy.fromfile(f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)) - data = data / 255.0 * 2.0 - 1.0 - - f.close() - return data.astype('float32') - - -def load_cifar_data(cifar_path): - batch_size = 10000 - data = numpy.zeros((5 * batch_size, 32 * 32 * 3), dtype="float32") - for i in range(1, 6): - file = cifar_path + "/data_batch_" + str(i) - fo = open(file, 'rb') - dict = cPickle.load(fo) - fo.close() - data[(i - 1) * batch_size:(i * batch_size), :] = dict["data"] - - data = data / 255.0 * 2.0 - 1.0 - return data - - -# synthesize 2-D uniform data -def load_uniform_data(): - data = numpy.random.rand(1000000, 2).astype('float32') - return data - - -def merge(images, size): - if images.shape[1] == 28 * 28: - h, w, c = 28, 28, 1 - else: - h, w, c = 32, 32, 3 - img = numpy.zeros((h * size[0], w * size[1], c)) - for idx in xrange(size[0] * size[1]): - i = idx % size[1] - j = idx // size[1] - img[j*h:j*h+h, i*w:i*w+w, :] = \ - ((images[idx, :].reshape((h, w, c), order="F").transpose(1, 0, 2) + 1.0) / 2.0 * 255.0) - return img.astype('uint8') - - -def save_images(images, path): - merged_img = merge(images, [8, 8]) - if merged_img.shape[2] == 1: - im = Image.fromarray(numpy.squeeze(merged_img)).convert('RGB') - else: - im = Image.fromarray(merged_img, mode="RGB") - im.save(path) - - -def get_real_samples(batch_size, data_np): - return data_np[numpy.random.choice( - data_np.shape[0], batch_size, replace=False), :] - - -def get_noise(batch_size, noise_dim): - return numpy.random.normal(size=(batch_size, noise_dim)).astype('float32') - - -def get_fake_samples(generator_machine, batch_size, noise): - gen_inputs = api.Arguments.createArguments(1) - gen_inputs.setSlotValue(0, api.Matrix.createDenseFromNumpy(noise)) - gen_outputs = api.Arguments.createArguments(0) - generator_machine.forward(gen_inputs, gen_outputs, api.PASS_TEST) - fake_samples = gen_outputs.getSlotValue(0).copyToNumpyMat() - return fake_samples - - -def get_training_loss(training_machine, inputs): - outputs = api.Arguments.createArguments(0) - training_machine.forward(inputs, outputs, api.PASS_TEST) - loss = outputs.getSlotValue(0).copyToNumpyMat() - return numpy.mean(loss) - - -def prepare_discriminator_data_batch_pos(batch_size, data_np): - real_samples = get_real_samples(batch_size, data_np) - labels = numpy.ones(batch_size, dtype='int32') - inputs = api.Arguments.createArguments(2) - inputs.setSlotValue(0, api.Matrix.createDenseFromNumpy(real_samples)) - inputs.setSlotIds(1, api.IVector.createVectorFromNumpy(labels)) - return inputs - - -def prepare_discriminator_data_batch_neg(generator_machine, batch_size, noise): - fake_samples = get_fake_samples(generator_machine, batch_size, noise) - labels = numpy.zeros(batch_size, dtype='int32') - inputs = api.Arguments.createArguments(2) - inputs.setSlotValue(0, api.Matrix.createDenseFromNumpy(fake_samples)) - inputs.setSlotIds(1, api.IVector.createVectorFromNumpy(labels)) - return inputs - - -def prepare_generator_data_batch(batch_size, noise): - label = numpy.ones(batch_size, dtype='int32') - inputs = api.Arguments.createArguments(2) - inputs.setSlotValue(0, api.Matrix.createDenseFromNumpy(noise)) - inputs.setSlotIds(1, api.IVector.createVectorFromNumpy(label)) - return inputs - - -def find(iterable, cond): - for item in iterable: - if cond(item): - return item - return None - - -def get_layer_size(model_conf, layer_name): - layer_conf = find(model_conf.layers, lambda x: x.name == layer_name) - assert layer_conf is not None, "Cannot find '%s' layer" % layer_name - return layer_conf.size - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--data_source", help="mnist or cifar or uniform") - parser.add_argument( - "--use_gpu", default="1", help="1 means use gpu for training") - parser.add_argument("--gpu_id", default="0", help="the gpu_id parameter") - args = parser.parse_args() - data_source = args.data_source - use_gpu = args.use_gpu - assert data_source in ["mnist", "cifar", "uniform"] - assert use_gpu in ["0", "1"] - - if not os.path.exists("./%s_samples/" % data_source): - os.makedirs("./%s_samples/" % data_source) - - if not os.path.exists("./%s_params/" % data_source): - os.makedirs("./%s_params/" % data_source) - - api.initPaddle('--use_gpu=' + use_gpu, '--dot_period=10', - '--log_period=100', '--gpu_id=' + args.gpu_id, - '--save_dir=' + "./%s_params/" % data_source) - - if data_source == "uniform": - conf = "gan_conf.py" - num_iter = 10000 - else: - conf = "gan_conf_image.py" - num_iter = 1000 - - gen_conf = parse_config(conf, "mode=generator_training,data=" + data_source) - dis_conf = parse_config(conf, - "mode=discriminator_training,data=" + data_source) - generator_conf = parse_config(conf, "mode=generator,data=" + data_source) - batch_size = dis_conf.opt_config.batch_size - noise_dim = get_layer_size(gen_conf.model_config, "noise") - - if data_source == "mnist": - data_np = load_mnist_data("./data/mnist_data/train-images-idx3-ubyte") - elif data_source == "cifar": - data_np = load_cifar_data("./data/cifar-10-batches-py/") - else: - data_np = load_uniform_data() - - # this creates a gradient machine for discriminator - dis_training_machine = api.GradientMachine.createFromConfigProto( - dis_conf.model_config) - # this create a gradient machine for generator - gen_training_machine = api.GradientMachine.createFromConfigProto( - gen_conf.model_config) - - # generator_machine is used to generate data only, which is used for - # training discriminator - logger.info(str(generator_conf.model_config)) - generator_machine = api.GradientMachine.createFromConfigProto( - generator_conf.model_config) - - dis_trainer = api.Trainer.create(dis_conf, dis_training_machine) - - gen_trainer = api.Trainer.create(gen_conf, gen_training_machine) - - dis_trainer.startTrain() - gen_trainer.startTrain() - - # Sync parameters between networks (GradientMachine) at the beginning - copy_shared_parameters(gen_training_machine, dis_training_machine) - copy_shared_parameters(gen_training_machine, generator_machine) - - # constrain that either discriminator or generator can not be trained - # consecutively more than MAX_strike times - curr_train = "dis" - curr_strike = 0 - MAX_strike = 5 - - for train_pass in xrange(100): - dis_trainer.startTrainPass() - gen_trainer.startTrainPass() - for i in xrange(num_iter): - # Do forward pass in discriminator to get the dis_loss - noise = get_noise(batch_size, noise_dim) - data_batch_dis_pos = prepare_discriminator_data_batch_pos( - batch_size, data_np) - dis_loss_pos = get_training_loss(dis_training_machine, - data_batch_dis_pos) - - data_batch_dis_neg = prepare_discriminator_data_batch_neg( - generator_machine, batch_size, noise) - dis_loss_neg = get_training_loss(dis_training_machine, - data_batch_dis_neg) - - dis_loss = (dis_loss_pos + dis_loss_neg) / 2.0 - - # Do forward pass in generator to get the gen_loss - data_batch_gen = prepare_generator_data_batch(batch_size, noise) - gen_loss = get_training_loss(gen_training_machine, data_batch_gen) - - if i % 100 == 0: - print "d_pos_loss is %s d_neg_loss is %s" % (dis_loss_pos, - dis_loss_neg) - print "d_loss is %s g_loss is %s" % (dis_loss, gen_loss) - - # Decide which network to train based on the training history - # And the relative size of the loss - if (not (curr_train == "dis" and curr_strike == MAX_strike)) and \ - ((curr_train == "gen" and curr_strike == MAX_strike) or dis_loss > gen_loss): - if curr_train == "dis": - curr_strike += 1 - else: - curr_train = "dis" - curr_strike = 1 - dis_trainer.trainOneDataBatch(batch_size, data_batch_dis_neg) - dis_trainer.trainOneDataBatch(batch_size, data_batch_dis_pos) - copy_shared_parameters(dis_training_machine, - gen_training_machine) - - else: - if curr_train == "gen": - curr_strike += 1 - else: - curr_train = "gen" - curr_strike = 1 - gen_trainer.trainOneDataBatch(batch_size, data_batch_gen) - # TODO: add API for paddle to allow true parameter sharing between different GradientMachines - # so that we do not need to copy shared parameters. - copy_shared_parameters(gen_training_machine, - dis_training_machine) - copy_shared_parameters(gen_training_machine, generator_machine) - - dis_trainer.finishTrainPass() - gen_trainer.finishTrainPass() - # At the end of each pass, save the generated samples/images - fake_samples = get_fake_samples(generator_machine, batch_size, noise) - if data_source == "uniform": - plot2DScatter(fake_samples, "./%s_samples/train_pass%s.png" % - (data_source, train_pass)) - else: - save_images(fake_samples, "./%s_samples/train_pass%s.png" % - (data_source, train_pass)) - dis_trainer.finishTrain() - gen_trainer.finishTrain() - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/mnist/.gitignore b/v1_api_demo/mnist/.gitignore deleted file mode 100644 index 7e61d5e3a0..0000000000 --- a/v1_api_demo/mnist/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -data/raw_data -data/*.list -mnist_vgg_model -plot.png -train.log -*pyc -.ipynb_checkpoints -params.pkl -params.tar -params.tar.gz diff --git a/v1_api_demo/mnist/api_train.py b/v1_api_demo/mnist/api_train.py deleted file mode 100644 index e42c6cbb7e..0000000000 --- a/v1_api_demo/mnist/api_train.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -""" -A very basic example for how to use current Raw SWIG API to train mnist network. - -Current implementation uses Raw SWIG, which means the API call is directly \ -passed to C++ side of Paddle. - -The user api could be simpler and carefully designed. -""" -import random - -import numpy as np -import paddle.v2 as paddle_v2 -import py_paddle.swig_paddle as api -from paddle.trainer_config_helpers import * -from py_paddle import DataProviderConverter - -from mnist_util import read_from_mnist - - -def init_parameter(network): - assert isinstance(network, api.GradientMachine) - for each_param in network.getParameters(): - assert isinstance(each_param, api.Parameter) - array_size = len(each_param) - array = np.random.uniform(-1.0, 1.0, array_size).astype('float32') - each_param.getBuf(api.PARAMETER_VALUE).copyFromNumpyArray(array) - - -def generator_to_batch(generator, batch_size): - ret_val = list() - for each_item in generator: - ret_val.append(each_item) - if len(ret_val) == batch_size: - yield ret_val - ret_val = list() - if len(ret_val) != 0: - yield ret_val - - -class BatchPool(object): - def __init__(self, generator, batch_size): - self.data = list(generator) - self.batch_size = batch_size - - def __call__(self): - random.shuffle(self.data) - for offset in xrange(0, len(self.data), self.batch_size): - limit = min(offset + self.batch_size, len(self.data)) - yield self.data[offset:limit] - - -def input_order_converter(generator): - for each_item in generator: - yield each_item['pixel'], each_item['label'] - - -def main(): - api.initPaddle("-use_gpu=false", "-trainer_count=4") # use 4 cpu cores - - optimizer = paddle_v2.optimizer.Adam( - learning_rate=1e-4, - batch_size=1000, - model_average=ModelAverage(average_window=0.5), - regularization=L2Regularization(rate=0.5)) - - # Create Local Updater. Local means not run in cluster. - # For a cluster training, here we can change to createRemoteUpdater - # in future. - updater = optimizer.create_local_updater() - assert isinstance(updater, api.ParameterUpdater) - - # define network - images = paddle_v2.layer.data( - name='pixel', type=paddle_v2.data_type.dense_vector(784)) - label = paddle_v2.layer.data( - name='label', type=paddle_v2.data_type.integer_value(10)) - hidden1 = paddle_v2.layer.fc(input=images, size=200) - hidden2 = paddle_v2.layer.fc(input=hidden1, size=200) - inference = paddle_v2.layer.fc(input=hidden2, - size=10, - act=paddle_v2.activation.Softmax()) - cost = paddle_v2.layer.classification_cost(input=inference, label=label) - - # Create Simple Gradient Machine. - model_config = paddle_v2.layer.parse_network(cost) - m = api.GradientMachine.createFromConfigProto(model_config, - api.CREATE_MODE_NORMAL, - optimizer.enable_types()) - - # This type check is not useful. Only enable type hint in IDE. - # Such as PyCharm - assert isinstance(m, api.GradientMachine) - - # Initialize Parameter by numpy. - init_parameter(network=m) - - # Initialize ParameterUpdater. - updater.init(m) - - # DataProvider Converter is a utility convert Python Object to Paddle C++ - # Input. The input format is as same as Paddle's DataProvider. - converter = DataProviderConverter(input_types=[images.type, label.type]) - - train_file = './data/raw_data/train' - test_file = './data/raw_data/t10k' - - # start gradient machine. - # the gradient machine must be started before invoke forward/backward. - # not just for training, but also for inference. - m.start() - - # evaluator can print error rate, etc. It is a C++ class. - batch_evaluator = m.makeEvaluator() - test_evaluator = m.makeEvaluator() - - # Get Train Data. - # TrainData will stored in a data pool. Currently implementation is not care - # about memory, speed. Just a very naive implementation. - train_data_generator = input_order_converter(read_from_mnist(train_file)) - train_data = BatchPool(train_data_generator, 512) - - # outArgs is Neural Network forward result. Here is not useful, just passed - # to gradient_machine.forward - outArgs = api.Arguments.createArguments(0) - - for pass_id in xrange(2): # we train 2 passes. - updater.startPass() - - for batch_id, data_batch in enumerate(train_data()): - # data_batch is input images. - # here, for online learning, we could get data_batch from network. - - # Start update one batch. - pass_type = updater.startBatch(len(data_batch)) - - # Start BatchEvaluator. - # batch_evaluator can be used between start/finish. - batch_evaluator.start() - - # forwardBackward is a shortcut for forward and backward. - # It is sometimes faster than invoke forward/backward separately, - # because in GradientMachine, it may be async. - m.forwardBackward(converter(data_batch), outArgs, pass_type) - - for each_param in m.getParameters(): - updater.update(each_param) - - # Get cost. We use numpy to calculate total cost for this batch. - cost_vec = outArgs.getSlotValue(0) - cost_vec = cost_vec.copyToNumpyMat() - cost = cost_vec.sum() / len(data_batch) - - # Make evaluator works. - m.eval(batch_evaluator) - - # Print logs. - print 'Pass id', pass_id, 'Batch id', batch_id, 'with cost=', \ - cost, batch_evaluator - - batch_evaluator.finish() - # Finish batch. - # * will clear gradient. - # * ensure all values should be updated. - updater.finishBatch(cost) - - # testing stage. use test data set to test current network. - updater.apply() - test_evaluator.start() - test_data_generator = input_order_converter(read_from_mnist(test_file)) - for data_batch in generator_to_batch(test_data_generator, 512): - # in testing stage, only forward is needed. - m.forward(converter(data_batch), outArgs, api.PASS_TEST) - m.eval(test_evaluator) - - # print error rate for test data set - print 'Pass', pass_id, ' test evaluator: ', test_evaluator - test_evaluator.finish() - updater.restore() - - updater.catchUpWith() - params = m.getParameters() - for each_param in params: - assert isinstance(each_param, api.Parameter) - value = each_param.getBuf(api.PARAMETER_VALUE) - value = value.copyToNumpyArray() - - # Here, we could save parameter to every where you want - print each_param.getName(), value - - updater.finishPass() - - m.finish() - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/mnist/data/generate_list.py b/v1_api_demo/mnist/data/generate_list.py deleted file mode 100644 index 49981cc7a9..0000000000 --- a/v1_api_demo/mnist/data/generate_list.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -o = open("./" + "train.list", "w") -o.write("./data/raw_data/train" + "\n") -o.close() - -o = open("./" + "test.list", "w") -o.write("./data/raw_data/t10k" + "\n") -o.close() diff --git a/v1_api_demo/mnist/data/get_mnist_data.sh b/v1_api_demo/mnist/data/get_mnist_data.sh deleted file mode 100755 index 5a2e34026d..0000000000 --- a/v1_api_demo/mnist/data/get_mnist_data.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env sh -# This scripts downloads the mnist data and unzips it. -set -e -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -rm -rf "$DIR/raw_data" -mkdir "$DIR/raw_data" -cd "$DIR/raw_data" - -echo "Downloading..." - -for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte -do - if [ ! -e $fname ]; then - wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz - gunzip ${fname}.gz - fi -done - -cd $DIR -rm -f *.list -python generate_list.py diff --git a/v1_api_demo/mnist/light_mnist.py b/v1_api_demo/mnist/light_mnist.py deleted file mode 100644 index 3340905435..0000000000 --- a/v1_api_demo/mnist/light_mnist.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -is_predict = get_config_arg("is_predict", bool, False) - -####################Data Configuration ################## - -if not is_predict: - data_dir = './data/' - define_py_data_sources2( - train_list=data_dir + 'train.list', - test_list=data_dir + 'test.list', - module='mnist_provider', - obj='process') - -######################Algorithm Configuration ############# -settings(batch_size=50, learning_rate=0.001, learning_method=AdamOptimizer()) - -#######################Network Configuration ############# - -data_size = 1 * 28 * 28 -label_size = 10 -img = data_layer(name='pixel', size=data_size) - - -# light cnn -# A shallower cnn model: [CNN, BN, ReLU, Max-Pooling] x4 + FC x1 -# Easier to train for mnist dataset and quite efficient -# Final performance is close to deeper ones on tasks such as digital and character classification -def light_cnn(input_image, num_channels, num_classes): - def __light__(ipt, - num_filter=128, - times=1, - conv_filter_size=3, - dropouts=0, - num_channels_=None): - return img_conv_group( - input=ipt, - num_channels=num_channels_, - pool_size=2, - pool_stride=2, - conv_padding=0, - conv_num_filter=[num_filter] * times, - conv_filter_size=conv_filter_size, - conv_act=ReluActivation(), - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type=MaxPooling()) - - tmp = __light__(input_image, num_filter=128, num_channels_=num_channels) - tmp = __light__(tmp, num_filter=128) - tmp = __light__(tmp, num_filter=128) - tmp = __light__(tmp, num_filter=128, conv_filter_size=1) - - tmp = fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation()) - return tmp - - -predict = light_cnn(input_image=img, num_channels=1, num_classes=label_size) - -if not is_predict: - lbl = data_layer(name="label", size=label_size) - inputs(img, lbl) - outputs(classification_cost(input=predict, label=lbl)) -else: - outputs(predict) diff --git a/v1_api_demo/mnist/mnist_provider.py b/v1_api_demo/mnist/mnist_provider.py deleted file mode 100644 index 4192339837..0000000000 --- a/v1_api_demo/mnist/mnist_provider.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -from paddle.trainer.PyDataProvider2 import * -from mnist_util import read_from_mnist - - -# Define a py data provider -@provider( - input_types={'pixel': dense_vector(28 * 28), - 'label': integer_value(10)}, - cache=CacheType.CACHE_PASS_IN_MEM) -def process(settings, filename): # settings is not used currently. - for each in read_from_mnist(filename): - yield each diff --git a/v1_api_demo/mnist/mnist_util.py b/v1_api_demo/mnist/mnist_util.py deleted file mode 100644 index 3fd88ae7ed..0000000000 --- a/v1_api_demo/mnist/mnist_util.py +++ /dev/null @@ -1,30 +0,0 @@ -import numpy - -__all__ = ['read_from_mnist'] - - -def read_from_mnist(filename): - imgf = filename + "-images-idx3-ubyte" - labelf = filename + "-labels-idx1-ubyte" - f = open(imgf, "rb") - l = open(labelf, "rb") - - f.read(16) - l.read(8) - - # Define number of samples for train/test - if "train" in filename: - n = 60000 - else: - n = 10000 - - images = numpy.fromfile( - f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)).astype('float32') - images = images / 255.0 * 2.0 - 1.0 - labels = numpy.fromfile(l, 'ubyte', count=n).astype("int") - - for i in xrange(n): - yield {"pixel": images[i, :], 'label': labels[i]} - - f.close() - l.close() diff --git a/v1_api_demo/mnist/train.sh b/v1_api_demo/mnist/train.sh deleted file mode 100755 index ca2b1ad9eb..0000000000 --- a/v1_api_demo/mnist/train.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -config=vgg_16_mnist.py -output=./mnist_vgg_model -log=train.log - -paddle train \ ---config=$config \ ---dot_period=10 \ ---log_period=100 \ ---test_all_data_in_one_period=1 \ ---use_gpu=0 \ ---trainer_count=1 \ ---num_passes=100 \ ---save_dir=$output \ -2>&1 | tee $log -paddle usage -l $log -e $? -n "mnist_train" >/dev/null 2>&1 - -python -m paddle.utils.plotcurve -i $log > plot.png diff --git a/v1_api_demo/mnist/vgg_16_mnist.py b/v1_api_demo/mnist/vgg_16_mnist.py deleted file mode 100644 index a819b391c6..0000000000 --- a/v1_api_demo/mnist/vgg_16_mnist.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -is_predict = get_config_arg("is_predict", bool, False) - -####################Data Configuration ################## - -if not is_predict: - data_dir = './data/' - define_py_data_sources2( - train_list=data_dir + 'train.list', - test_list=data_dir + 'test.list', - module='mnist_provider', - obj='process') - -######################Algorithm Configuration ############# -settings( - batch_size=128, - learning_rate=0.1 / 128.0, - learning_method=MomentumOptimizer(0.9), - regularization=L2Regularization(0.0005 * 128)) - -#######################Network Configuration ############# - -data_size = 1 * 28 * 28 -label_size = 10 -img = data_layer(name='pixel', size=data_size) - -# small_vgg is predined in trainer_config_helpers.network -predict = small_vgg(input_image=img, num_channels=1, num_classes=label_size) - -if not is_predict: - lbl = data_layer(name="label", size=label_size) - inputs(img, lbl) - outputs(classification_cost(input=predict, label=lbl)) -else: - outputs(predict) diff --git a/v1_api_demo/model_zoo/embedding/.gitignore b/v1_api_demo/model_zoo/embedding/.gitignore deleted file mode 100644 index 908f5a3fb2..0000000000 --- a/v1_api_demo/model_zoo/embedding/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -baidu.dict -model_*.emb diff --git a/v1_api_demo/model_zoo/embedding/extract_para.py b/v1_api_demo/model_zoo/embedding/extract_para.py deleted file mode 100755 index 570b90c1f7..0000000000 --- a/v1_api_demo/model_zoo/embedding/extract_para.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/env python -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Example: - python extract_para.py --preModel PREMODEL --preDict PREDICT \ - --usrModel USRMODEL --usrDict USRDICT -d DIM - -Options: - -h, --help show this help message and exit - --preModel PREMODEL the name of pretrained embedding model - --preDict PREDICT the name of pretrained dictionary - --usrModel usrModel the name of output usr embedding model - --usrDict usrDict the name of user specified dictionary - -d DIM dimension of parameter -""" -from optparse import OptionParser -import struct - - -def get_row_index(preDict, usrDict): - """ - Get the row positions for all words in user dictionary from pre-trained dictionary. - return: a list of row positions - Example: preDict='a\nb\nc\n', usrDict='a\nc\n', then return [0,2] - """ - pos = [] - index = dict() - with open(preDict, "r") as f: - for line_index, line in enumerate(f): - word = line.strip().split()[0] - index[word] = line_index - with open(usrDict, "r") as f: - for line in f: - word = line.strip().split()[0] - pos.append(index[word]) - return pos - - -def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, - paraDim): - """ - Extract desired parameters from a pretrained embedding model based on user dictionary - """ - if paraDim not in [32, 64, 128, 256]: - raise RuntimeError("We only support 32, 64, 128, 256 dimensions now") - - fi = open(preModel, "rb") - fo = open(usrModel, "wb") - - # write filehead - rowIndex = get_row_index(preDict, usrDict) - newHead = struct.pack("iil", 0, 4, len(rowIndex) * paraDim) - fo.write(newHead) - bytes = 4 * paraDim - for i in range(0, len(rowIndex)): - # find the absolute position of input file - fi.seek(rowIndex[i] * bytes + 16, 0) - fo.write(fi.read(bytes)) - - print "extract parameters finish, total", len(rowIndex), "lines" - fi.close() - - -def main(): - """ - Main entry for running paraconvert.py - """ - usage = "usage: \n" \ - "python %prog --preModel PREMODEL --preDict PREDICT" \ - " --usrModel USRMODEL --usrDict USRDICT -d DIM" - parser = OptionParser(usage) - parser.add_option( - "--preModel", - action="store", - dest="preModel", - help="the name of pretrained embedding model") - parser.add_option( - "--preDict", - action="store", - dest="preDict", - help="the name of pretrained dictionary") - parser.add_option( - "--usrModel", - action="store", - dest="usrModel", - help="the name of output usr embedding model") - parser.add_option( - "--usrDict", - action="store", - dest="usrDict", - help="the name of user specified dictionary") - parser.add_option( - "-d", action="store", dest="dim", help="dimension of parameter") - (options, args) = parser.parse_args() - extract_parameters_by_usrDict(options.preModel, options.preDict, - options.usrModel, options.usrDict, - int(options.dim)) - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/model_zoo/embedding/paraconvert.py b/v1_api_demo/model_zoo/embedding/paraconvert.py deleted file mode 100755 index ce7a70efc4..0000000000 --- a/v1_api_demo/model_zoo/embedding/paraconvert.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/env python -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Example: - python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM - python paraconvert.py --t2b -i INPUT -o OUTPUT - -Options: - -h, --help show this help message and exit - --b2t convert parameter file of embedding model from binary to text - --t2b convert parameter file of embedding model from text to binary - -i INPUT input parameter file name - -o OUTPUT output parameter file name - -d DIM dimension of parameter -""" -from optparse import OptionParser -import struct - - -def binary2text(input, output, paraDim): - """ - Convert a binary parameter file of embedding model to be a text file. - input: the name of input binary parameter file, the format is: - 1) the first 16 bytes is filehead: - version(4 bytes): version of paddle, default = 0 - floatSize(4 bytes): sizeof(float) = 4 - paraCount(8 bytes): total number of parameter - 2) the next (paraCount * 4) bytes is parameters, each has 4 bytes - output: the name of output text parameter file, for example: - 0,4,32156096 - -0.7845433,1.1937413,-0.1704215,... - 0.0000909,0.0009465,-0.0008813,... - ... - the format is: - 1) the first line is filehead: - version=0, floatSize=4, paraCount=32156096 - 2) other lines print the paramters - a) each line prints paraDim paramters splitted by ',' - b) there is paraCount/paraDim lines (embedding words) - paraDim: dimension of parameters - """ - fi = open(input, "rb") - fo = open(output, "w") - """ - """ - version, floatSize, paraCount = struct.unpack("iil", fi.read(16)) - newHead = ','.join([str(version), str(floatSize), str(paraCount)]) - print >> fo, newHead - - bytes = 4 * int(paraDim) - format = "%df" % int(paraDim) - context = fi.read(bytes) - line = 0 - - while context: - numbers = struct.unpack(format, context) - lst = [] - for i in numbers: - lst.append('%8.7f' % i) - print >> fo, ','.join(lst) - context = fi.read(bytes) - line += 1 - fi.close() - fo.close() - print "binary2text finish, total", line, "lines" - - -def get_para_count(input): - """ - Compute the total number of embedding parameters in input text file. - input: the name of input text file - """ - numRows = 1 - paraDim = 0 - with open(input) as f: - line = f.readline() - paraDim = len(line.split(",")) - for line in f: - numRows += 1 - return numRows * paraDim - - -def text2binary(input, output, paddle_head=True): - """ - Convert a text parameter file of embedding model to be a binary file. - input: the name of input text parameter file, for example: - -0.7845433,1.1937413,-0.1704215,... - 0.0000909,0.0009465,-0.0008813,... - ... - the format is: - 1) it doesn't have filehead - 2) each line stores the same dimension of parameters, - the separator is commas ',' - output: the name of output binary parameter file, the format is: - 1) the first 16 bytes is filehead: - version(4 bytes), floatSize(4 bytes), paraCount(8 bytes) - 2) the next (paraCount * 4) bytes is parameters, each has 4 bytes - """ - fi = open(input, "r") - fo = open(output, "wb") - - newHead = struct.pack("iil", 0, 4, get_para_count(input)) - fo.write(newHead) - - count = 0 - for line in fi: - line = line.strip().split(",") - for i in range(0, len(line)): - binary_data = struct.pack("f", float(line[i])) - fo.write(binary_data) - count += 1 - fi.close() - fo.close() - print "text2binary finish, total", count, "lines" - - -def main(): - """ - Main entry for running paraconvert.py - """ - usage = "usage: \n" \ - "python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \ - "python %prog --t2b -i INPUT -o OUTPUT" - parser = OptionParser(usage) - parser.add_option( - "--b2t", - action="store_true", - help="convert parameter file of embedding model from binary to text") - parser.add_option( - "--t2b", - action="store_true", - help="convert parameter file of embedding model from text to binary") - parser.add_option( - "-i", action="store", dest="input", help="input parameter file name") - parser.add_option( - "-o", action="store", dest="output", help="output parameter file name") - parser.add_option( - "-d", action="store", dest="dim", help="dimension of parameter") - (options, args) = parser.parse_args() - if options.b2t: - binary2text(options.input, options.output, options.dim) - if options.t2b: - text2binary(options.input, options.output) - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/model_zoo/embedding/pre_DictAndModel.sh b/v1_api_demo/model_zoo/embedding/pre_DictAndModel.sh deleted file mode 100755 index f61c65a935..0000000000 --- a/v1_api_demo/model_zoo/embedding/pre_DictAndModel.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -set -x -BASE_URL='http://paddlepaddle.cdn.bcebos.com/model_zoo/embedding' - -DOWNLOAD_ITEMS=(baidu.dict model_32.emb model_64.emb model_128.emb model_256.emb) -ITEM_MD5=(fa03a12321eaab6c30a8fcc9442eaea3 - f88c8325ee6da6187f1080e8fe66c1cd - 927cf70f27f860aff1a5703ebf7f1584 - a52e43655cd25d279777ed509a1ae27b - b92c67fe9ff70fea53596080e351ac80) - -for ((i=0; i<${#ITEM_MD5[@]}; i++)) -do - FILENAME=${DOWNLOAD_ITEMS[${i}]} - REAL_MD5=`wget ${BASE_URL}/${FILENAME} -O - | tee ${FILENAME} | md5sum | cut -d ' ' -f 1` - EXPECTED_MD5=${ITEM_MD5[${i}]} - [ "${EXPECTED_MD5}" = "${REAL_MD5}" ] -done diff --git a/v1_api_demo/model_zoo/resnet/.gitignore b/v1_api_demo/model_zoo/resnet/.gitignore deleted file mode 100644 index 7a64209b62..0000000000 --- a/v1_api_demo/model_zoo/resnet/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -fea_output/ -features/ -model.list -ResNet_50.dot -ResNet_50.png diff --git a/v1_api_demo/model_zoo/resnet/classify.py b/v1_api_demo/model_zoo/resnet/classify.py deleted file mode 100755 index 6074cc1d3a..0000000000 --- a/v1_api_demo/model_zoo/resnet/classify.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import cPickle -import logging -from PIL import Image -import numpy as np -from optparse import OptionParser - -import paddle.utils.image_util as image_util - -from py_paddle import swig_paddle, DataProviderConverter -from paddle.trainer.PyDataProvider2 import dense_vector -from paddle.trainer.config_parser import parse_config - -logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') -logging.getLogger().setLevel(logging.INFO) - - -class ImageClassifier(): - def __init__(self, - train_conf, - model_dir=None, - resize_dim=256, - crop_dim=224, - use_gpu=True, - mean_file=None, - output_layer=None, - oversample=False, - is_color=True): - """ - train_conf: network configure. - model_dir: string, directory of model. - resize_dim: int, resized image size. - crop_dim: int, crop size. - mean_file: string, image mean file. - oversample: bool, oversample means multiple crops, namely five - patches (the four corner patches and the center - patch) as well as their horizontal reflections, - ten crops in all. - """ - self.train_conf = train_conf - self.model_dir = model_dir - if model_dir is None: - self.model_dir = os.path.dirname(train_conf) - - self.resize_dim = resize_dim - self.crop_dims = [crop_dim, crop_dim] - self.oversample = oversample - self.is_color = is_color - - self.output_layer = output_layer - if self.output_layer: - assert isinstance(self.output_layer, basestring) - self.output_layer = self.output_layer.split(",") - - self.transformer = image_util.ImageTransformer(is_color=is_color) - self.transformer.set_transpose((2, 0, 1)) - self.transformer.set_channel_swap((2, 1, 0)) - - self.mean_file = mean_file - if self.mean_file is not None: - mean = np.load(self.mean_file)['data_mean'] - mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel - else: - # if you use three mean value, set like: - # this three mean value is calculated from ImageNet. - self.transformer.set_mean(np.array([103.939, 116.779, 123.68])) - - conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (int(use_gpu)) - conf = parse_config(train_conf, conf_args) - swig_paddle.initPaddle("--use_gpu=%d" % (int(use_gpu))) - self.network = swig_paddle.GradientMachine.createFromConfigProto( - conf.model_config) - assert isinstance(self.network, swig_paddle.GradientMachine) - self.network.loadParameters(self.model_dir) - - data_size = 3 * self.crop_dims[0] * self.crop_dims[1] - slots = [dense_vector(data_size)] - self.converter = DataProviderConverter(slots) - - def get_data(self, img_path): - """ - 1. load image from img_path. - 2. resize or oversampling. - 3. transformer data: transpose, channel swap, sub mean. - return K x H x W ndarray. - - img_path: image path. - """ - image = image_util.load_image(img_path, self.is_color) - # Another way to extract oversampled features is that - # cropping and averaging from large feature map which is - # calculated by large size of image. - # This way reduces the computation. - if self.oversample: - # image_util.resize_image: short side is self.resize_dim - image = image_util.resize_image(image, self.resize_dim) - image = np.array(image) - input = np.zeros( - (1, image.shape[0], image.shape[1], 3), dtype=np.float32) - input[0] = image.astype(np.float32) - input = image_util.oversample(input, self.crop_dims) - else: - image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros( - (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) - input[0] = np.array(image).astype(np.float32) - - data_in = [] - for img in input: - img = self.transformer.transformer(img).flatten() - data_in.append([img.tolist()]) - # paddle input: [[[]],[[]],...], [[]] is one sample. - return data_in - - def forward(self, input_data): - """ - return output arguments which are the Outputs() in network configure. - - input_data: py_paddle input data. - call forward. - """ - in_arg = self.converter(input_data) - return self.network.forwardTest(in_arg) - - def forward(self, data, output_layer): - """ - return output arguments which are the Outputs() in network configure. - - input_data: py_paddle input data. - call forward. - """ - input = self.converter(data) - self.network.forwardTest(input) - output = self.network.getLayerOutputs(output_layer) - res = {} - if isinstance(output_layer, basestring): - output_layer = [output_layer] - for name in output_layer: - # For oversampling, average predictions across crops. - # If not, the shape of output[name]: (1, class_number), - # the mean is also applicable. - res[name] = output[name]['value'].mean(0) - - return res - - def predict(self, data_file): - """ - call forward and predicting. - - data_file: input image list. - """ - image_files = open(data_file, 'rb').readlines() - results = {} - if self.output_layer is None: - self.output_layer = ["output"] - for line in image_files: - image = line.split()[0] - data = self.get_data(image) - prob = self.forward(data, self.output_layer) - lab = np.argsort(-prob[self.output_layer[0]]) - results[image] = lab[0] - logging.info("Label of %s is: %d", image, lab[0]) - return results - - def extract(self, data_file, output_dir, batch_size=10000): - """ - extract and save features of output layers, which are - specify in Outputs() in network configure. - - data_file: file name of input data. - output_dir: saved directory of extracted features. - batch_size: sample number of one batch file. - """ - if not os.path.exists(output_dir): - os.mkdir(output_dir) - - sample_num = 0 - batch_num = 0 - image_feature = {} - image_files = open(data_file, 'rb').readlines() - for idx, line in enumerate(image_files): - image = line.split()[0] - data = self.get_data(image) - feature = self.forward(data, self.output_layer) - # save extracted features - file_name = image.split("/")[-1] - image_feature[file_name] = feature - sample_num += 1 - if sample_num == batch_size: - batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) - self.save_file(image_feature, batch_name) - logging.info('Finish batch %d', batch_num) - batch_num += 1 - sample_num = 0 - image_feature = {} - if idx % 1000 == 0: - logging.info('%d/%d, %s', idx, len(image_files), file_name) - if sample_num > 0: - batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) - self.save_file(image_feature, batch_name) - logging.info('Finish batch %d', batch_num) - logging.info('Done: make image feature batch') - - def save_file(self, data, file): - of = open(file, 'wb') - cPickle.dump(data, of, protocol=cPickle.HIGHEST_PROTOCOL) - - -def option_parser(): - """ - Main entry for predciting - """ - usage = "%prog -c config -i data_list -w model_dir [options]" - parser = OptionParser(usage="usage: %s" % usage) - parser.add_option( - "-j", - "--job", - action="store", - dest="job_type", - help="job type: predict, extract\ - predict: predicting,\ - extract: extract features") - parser.add_option( - "-c", - "--conf", - action="store", - dest="train_conf", - help="network config") - parser.add_option( - "-i", "--data", action="store", dest="data_file", help="image list") - parser.add_option( - "-w", - "--model", - action="store", - dest="model_path", - default=None, - help="model path") - parser.add_option( - "-g", - "--use_gpu", - action="store", - dest="use_gpu", - default=True, - help="Whether to use gpu mode.") - parser.add_option( - "-o", - "--output_dir", - action="store", - dest="output_dir", - default="output", - help="output path") - parser.add_option( - "-m", - "--mean", - action="store", - dest="mean", - default=None, - help="mean file.") - parser.add_option( - "-p", - "--multi_crop", - action="store_true", - dest="multi_crop", - default=False, - help="Wether to use multiple crops on image.") - parser.add_option("-l", "--output_layer", action="store", - dest="output_layer", default=None, - help="--job=extract, specify layers to extract "\ - "features, --job=predict, specify layer of " - "classification probability, output in resnet.py.") - return parser.parse_args() - - -def main(): - """ - 1. parse input arguments. - 2. predicting or extract features according job type. - """ - options, args = option_parser() - obj = ImageClassifier( - options.train_conf, - options.model_path, - use_gpu=options.use_gpu, - mean_file=options.mean, - output_layer=options.output_layer, - oversample=options.multi_crop) - if options.job_type == "predict": - obj.predict(options.data_file) - - elif options.job_type == "extract": - obj.extract(options.data_file, options.output_dir) - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/model_zoo/resnet/example/.gitignore b/v1_api_demo/model_zoo/resnet/example/.gitignore deleted file mode 100644 index 4a2b5962a6..0000000000 --- a/v1_api_demo/model_zoo/resnet/example/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*image_list_provider_copy_1.py diff --git a/v1_api_demo/model_zoo/resnet/example/__init__.py b/v1_api_demo/model_zoo/resnet/example/__init__.py deleted file mode 100644 index f662d68263..0000000000 --- a/v1_api_demo/model_zoo/resnet/example/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/v1_api_demo/model_zoo/resnet/example/cat.jpg b/v1_api_demo/model_zoo/resnet/example/cat.jpg deleted file mode 100644 index 47b01db90eddc46ff845f10bc2accaf2364c272d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12881 zcmbWdcTf{f`0yEeuTla8LbD zr~xR+$tlRmC@Cl?sHiBZX<6uKX=rFUnC~#KaC6?};pPN^c)^lFynN#PAdv6_5pgML zSy|b;LNFyLL{U;k7VF9uvd!T!e|IhNb7r;abSR>LQA>soNGZB$6 z5&azi-2JytGNS(_fd5m7h)GDv$SEkPsA>Kw;1~hKL?k4{q$Ffyq@@3}BmT_;NSVl( z?@6nW-!XEe;PYdFL}ivz@~bxWu^P|(0LwV}M^jO=u>(0c1q6kJMMP!gFi_4$CuCD)w ziwHpSzp?&}{~PT8!Nv5Ci*!#(o zU9y=$e_rDD-g^|(uaISk(zB$&sPQ1c_*YR@Q8nl(Vd*b!8N_0b-Hu7~$1ck0(#~Ij zPK+rNS@h|6S>!bo$ltU)m)&tp^8}85HcA!ske`pqQlm9LnSb)yP48Z^U(CPnJytcw&UEo4~VtxQi3grmTbubJdP(D5nrABz1{wVGxSH8Gj8)1#^BDv+rx#0{IFhADg}_A@_cK`o5CI2TE;JwK z@82)2bZD0wg=vqCR5*=fnAZ0GUeHiQgxp$xtNd&vZ)HY$rk)uGG8Rj@DBYn*+MgX* zb1+HDaIfIoPKwmBM>D^c){Y_p%`4N+=8lv^?C%7i&w)RL`HIR&FrJ*BG&BHtm1t-U zqdQjn72-IL^+%ZL_-p|qHP8wBtbvWAfYVq5uFr{?9277V*V3wtCtV0eu9RuQS7$&1|-GlgMK~$s*r3S zT20;s%|Vbl-zY!<)Q>zKeU{cF#XI>3M<+$X;I;snFDnHXLawfKPXt=YSU#NQHqR3b@WYwW^SS>L-N<8J2@>|DNcbvQk;6PjP1J6R0Y$)-z&1;|Yx(o!VS36E; zx|Mpjb1hgrex&k0;*zsYlVr9!-8|q|$&K?JOs#eo%YwLOk0W@lnR|TKD6KWx>u0Tc zhQgfDGKe!5=B1E(O2l*+(Sl?yC0>#*O}fXq`~&nChV$|`&MGDTE})FPqp* zOv64>C%{YBi9Y-5Ak7^YU!?g~bMP?BR}Q5pjh=Y5o-XrP^DBD|18+oAqY~l;(Mr!l z%OLc2ph>B>mEkJMkVyQfY7wmN&qG6?a}lnkl+Ei)N=~zrXxYF_J71?#t@&HbjH}wP zTET z>};FXMhlNr)=4>dntdJFGh4Zfn3XwZ6ZVGgsWu>H5|TWg1YuzxlXj&1w6_;B@GRdY z+oJUVD)M{M=G#ssOjhP|blXX7OLqmFsfM#%ml+g)*1ZKQCJ}BoSO{|dbXv^h9{u%P zEhJA$tMj}M^ld6!_eI${%n^H8sjUtgGLWpK%DFbJx)v$dgT=S;^YS)|OENLpShjmj z?tq^)z>Lru4tb*GdG##QvX4$xwwG09+=SUoW!Flr-+&c_`(w9P%{y8@tG7T4CNL;R z&D$&rIYvxY2V`!5dcoWl6lNalHwtl+TR*S8lcP=DjDED)R_E$rseBh?(yX&v3c%zB zv13v>I~bepwHPFs6qx!;K?k)-y|&3nF$vGkX}WSq&J46LP5{uts<#aIxld2s6E#{J zWq{vGSDDJVc|d<`(T;Aoa)}RzobQp!E0N$KI%vI*4CW}7#Vm6PD4bbG`_b>fK-zos zqB=+(u3@(#Ig%~8&nZc$5FX*_FYNN)#JDtbFGFiGTg=wAc8kfiOO;eA9GB!1AG(8< zlmc0DCgAj9)i9bPLv7MvVY640&QEDLYY&hZU;G3(zvh8^k6vJvw%0(l0HWwnUTn1H zb+%dl#6mJPnWU(|3m%VMl{7I{(pczLh0L>QK8y=V&w~kC4+UQGlm{w?t&bM`{L(}= z367Du2gCrdmzl$YRZ<)WFe?=dX2Q*gBy9@1N{h(d)lo>RinJDsVqAv3afeyRQcxaK{LM zj23D6FPEbaTdSLEB>VV?2tnmO(tjamr8eTv!0M#n50u`sHO{Jj?buZI06f2)ZGRye zXWm!&&ow?@LK=jII$AE4?pis({{k#y z8T?^)t7|jYOZ^3=`GMR>do<_KoYA*1sE%Qc_Ge`I8;(SNp%R}s8P zBeRo9;clQ;FbkDbe!j?O;eTlswJf%&T3?D1aIlxQG|raE@}j?0#hL-t3*f_ICT6g|A7` zY^WVLDefCkjizbNSneaixb#LDbpwFaIE3&rj|Xkl$jMIgaRG)@2%zDd^r<-mahjqjhfPf5 z&a~cW4TaBttft9$hC%(3Cx$<1GxalVD072y(>+rk?E=sEMw^pS==jD~=^w%)a4mw+ z&H5*OeEdMZMsLfn+sLGGJOeM(M7v)qHDi_c4!dJg$e*u6VWxilBQPvbLX9?R8tV+y zF;KP4X=>Kvsv@&AcdPKav>u#j0yr?cG*(5T+xkj}$lO9U?|}U}b@tOr6HNM`-;>UN zoEs;fD)`E@t{hvin;)i_tKZYw2s^&KHKToCSg0cC$SdlfQuR#e;}>(DRMJTqc+Rft zusD?DAZgDrP9pvjAmcx}cOa2#3)>3|g~F_+w)^Fs4B{#BXdtW(i)=EYI)+5}Lp&w5 z#`Y?uq%0l0k0z};TvDI7+>F&-1vEsMxNht*uh_<;^$$sfZf>&|dzf@VlVd&v{uH-5 ztz!4q4$Pz4I5=$T<(~kQmW^%#8SNm25KbVtWId-!340*3fe>`h%cB6ir2K=ZTJ=6Q zq^t-;hkhuC1f1W9-2XC(|9wC2RAdLR|5$V=n1YD&d~6 zN&juxK=|71!K>Q!1LkWF>s-Dh;#aBIr#B(iRF2&$d3uYl zsWSLkt6@=CIu@Wqo9}h{a2NAorMBfs{h%eu&EAiRS&`3utn+@!sCt>j{KxA`dsDEf zuji~>g}qTlqFxHwU(vD4N5=NV-&CsGj55``k-U?t#9v{_LzZ@ zZMP6BZ39=2h26m12rtrVKI1trvl@NYV4|W_h4K%fm@Ul^?bzwyb!a3{4ySCW{LDKo#|PT4ZGNWJqrFK+RBwIBiM}bhJhfC=53Ar6K4(IYRdI)|eG!M1 zWHRC4erWErvd!*j=G4KrU#PN1Y@$;Icvg@8^uD@GtRvCVhHAwwh_I$Y!k*y!Jdvf$u zSJ!YTR1-7IK@B~yXv_CjHkxkGqbpR#7|EolzCuhx4!Mi z6~g*M^GEt$fSH|8_-LlFUTgel;LgZu6v@GC&Z39a($56b;wNT$T3VD&RgQ`o2w|3i z5kqGDp@;lc(l;MzUk1RfY`iLC{L`_T#KJ_ttB8FS{zC%hV^rFgj{f4IDv5+NH zAht}!|8kI(wy6%eQb!XHt0MP@R!vT{-gy=_*y?j+bO)uO|8$41*4=Y^HXCnf#Y`(8 z`PH;R64qP6+%{vxnHa6nmE0i!)&QzyX#QvW@9Ds?%y==4^Nt79YrVjAGRVgG+&Hh2 zbj!ge;l_ongD{`c2nksxmF`*>$ zUR*M5yNWs_v0$60XFsGwGyC4qN5tfW5kF}%Q3BTKO}dQWpoePS4|+PY#NE^%er@?d zF0l=prWn%<$cv-s@I^Vd3Z;N|d@7Lqgo`>|A8SF^?;9<#RJ-w?mo~7Y0lZ?IZAh#B zndd8X`;fy?*0Q6WI>;i2X(;O66tcj)Wu&oyKwkct%jU>zn|?tAUe+AOi%SAiX)2oa zz=~i$s6<1cyEOo-w1vF`6Pv3yT;PVs)GLJwdfHJEmxa6b>5nXIf_MuH?p!oHA6`+H zwyU3MST1k>M4SKBHvBK32tA{Q1iw@2`tt5~`H7YD0P*@%ah-Ld_iBk_fkhJ>tMt22 zI;)1ucI!s)sPb3tlO~T@iw@&jRsS9nwa-eVYTgy%Zr^R6c6Ct!U#fA=|aOSvsK;Byzm|pY_Qy*5`t&PN723wZmNI z$u=rAtuUm~<>VpV4;!7XGJkD@?@U(g57}dn@)Q^An?W-0M`q8fxe;z(Bj6=hI+4t? z?)aJScHap$fqB@jmK-3!o#Uuro*jblEQ@)fv4!I5%7gr_!ngN5afH0qtHo6@As zxQyBZPx-Y;W1{r|4G)`RcS>7B-_~3Hm=|A~4me4OPVx6wItV=`8@%0wf7BHH#5%?8 z8t{wMh!|KPLsSrnE~{MsQ&KlTn-ca)1H8pLrXYVDz*!#Jrp=fYtzasZux+U3<@IMa zr0xMdPcId}!{@PTIRA@yW6FI^L|6C z1JeX`{|+x?QrB|oWS$S2pkT{noyByiHYJul>#l9d2%GfTXQyL*8#O<#{kuNDi?35B zv7pE!`j8`fw>H5gN9l@pOO9I9eHLO?;w=vbfi|Lg_py zAn9H!0x{vN^D`c>0M;fy2v#%#_2P$hYLQGHuarco7^O$5*vr=-8V@sfXNW9U$sG^| zV3W}=1@n=Pm-WUevcx^N#ptZVzkmS8sLScUfQCZ1?;h^Jf+NjQK+0}o#jGAn)IBx! zY-i(gLRQTAq^sU#A`9t1|ATV;zDh~~a#}8?N}q1(Hy`Fun?&L~`fIR{J*WKsqavKZ zM3<`?XU=HfW%WXLHCRB(_KU-XP|9v)n#gfJZ$2l)LYNS38VJV3abj#tZ%^b^iD`oU zqQ9J1)?`{t*mn#miFTJ@Hy9!CZo0LC>$#)ju@*dmgf|S+j}7Lq)>W}I&Nbh$8-Eq) zmATl!f^* zzxN|lYZy(c@yQ-jW@aV4J-wudPfgGrA?`_!hM!_%we%G@Mbw` zOOyl#B`%9rK6c@4Zq%7US*E6EeP~xjypyJkbr+`W?QO7{v0prOeX7EIs{2YSnG&aj zAJQ#+pKK=4K-?u{sBgN3yT6h;|9V{CBB|r_%>Me4qxAVh#T|L1F3fn;LiStu7raA+PWnm7iVXt!0lkhJo-jO{*0BupmAdDY8~y?m9q#5nLLdJQu|ApKc#~4xeHfNJ z=-FVaPudr2$TpIav;l4xkI=5Z_z$R| zNGxnIk)(#(mQ=X}TwebX75pj?^eCtmt~YM^iI{DVRg=ST_}W}xqmsMr(^|jw?Ue|| zW>v+1e=na+pXja_Z6Ij?IGFn-J>oY1i*9m1N^y)=@umm)-8N`(lv863TQLlCEpk{Z zV5D-0%xl;j(`X`^Gst#FeWB%Ssrcna_lTD-%iZmYoMEwzZ{1rsF`jD2lAexQKKQ}= z9C2_4f{o|bv?6C;$GW#{9{Zd|;&dYNp#n;== zU(?9yJzdm`N^}B{H4AH-AE?d93`ZbFhYClk7l*Prx1cL!X=qpXvyTO5mgX5bCan%( zzZc$UgnS_PQIXr}?Ebz1TWP|2>10g5CQjueMlWn>jD3Lt#(_rJg;=v5ucXps#Zh$u%HE3(8f^nMdidkz_jE99#r5K5}?cbDLsp!fGqt3BN)6_`u@}hkQ zYU0-ZVzu!MaR=hBj&_bybwZ~8}C$MqNL$#!_^ zioNdv+KOnDqUJB)QTD+oL4L8{%JpHTYkyX&R< zt%O@XdK->+5)-Eo@V%slV<`7SxBSq@Xl8L#lx(p-D@X6%&6#zksiWN4I=7?nq@=Ks!))ZJ`7ew*Z0uJ%4}M)iGu_g9av z7m5Q*%X39#Dj_Q7xfR%`lc5@EU-xEB`=0rFeHbghJ?TN~&7Vl6pc^Dusjpw2 z?cAuE4PUqYQ<(CKp35h~2}MvNl-9U{Gv3xyg$dd8HO_B=xv09=F%0znKKjp0?e-=DSg^Qk99uI-%B1nbGd4l z^!I#&hm1T)8`Qb;Dr8vhrCyik0n9N!1OQIVwhM4Hhfq4(bG^6`+1Vf%mgU4~zxo^k}7Pe`s3IwMBz*(p{;;1?DostD6L$V!7KLqsGJ8OqQNDhpT^nJL;jJ!Wl(KhojSJ5-*w4G+B z+QZ3ps))-?I4n{4UY7QEV+MIL@q1=m8liNv;kmr|%4=&argi?hEGwEl5K8ByS)q=x zjGKa>x>*DjB)c`$*^IOxrKw`9YHAVYgZzqAiQZX2p7di;e+dW|MqLq)+DQzrUW3cM*OA!8%uh;_;8dNp*v^i zcAQ=@W4px}>v*_D7V0xJ%#~Q4Be%i?{y;HGDlF%=0X^p2nV11TQ*CnlCcX5zV4k^P zx=Cv#IcnK>*z3w%nuCp3%SnnpbV}juV758sJ}YTM#(biCk&_eSs+2e-bCW37+>ctO zHJzWvCAr}(K38zS&13riZaZ51AUSd_UX&}qgcN^YFM2WC!mU}%0!hC}t*872SdX>mgdQj8#+{a?CW`(A zh=IN^ysUUOC>6aY>9d{cAT|P9i+C9J#W<-ZFxc^_ot~wS&R$E;gw`mD10y-|vz)58 zD%bYuqM+3sx-Qt1rhZsZ`TTtn_;;eTy9xsS5F+joC9PYW&IaF1D4!-YUE&^0Al>9b zNa&;igTs=MH5wTD?2In1h*Wf(wzuK|+wz|?&VGf@KeW48FK-@4wat>+RkbieLVq5a z)^DBq^yY65#ER?0>=Jb~DLQ;Hx$nb9b64Ii!$9Pby8c>JdUpf_`Q40aavzPYz8Qy1 zoibvjucYHhY6y)>#Eok83@x`8d7Ux;9NRDe!J4{ylOQL2qx+m=@9Hl=v=*Ubdl+66Q?mEp%Jpy8$9{;<(%e%- zx43U7t9&VzXZG@Rp^Ug-&83DbM-B9R31|!!`r; zo`xhYoTj6nRv%rbPvO^}n!YjXKy3zZS^*oC$l(-wMth6FxzC#!X?(ok>rVm9?4P;o zChLs9uXBGk^11Kx{r)FlMycPReUq5yvkep1WXI(T{@FZZ*(APd#ZUf+17|U(aerif zE!2tKfj(n#(yP5*8(!PYfiroA2 z*xDP19?;p3bW0=|srC2!e6ow%Y`c@*DnBBzV`@rW!l-imm~#{_U*Y`VExy?2{gZ@p zmfl_-99UcCvoqBGqCzeXJMbZ(zUAoAQ;@u9J}zTjLxjHLt`zr)<+Qclnw?%(!pu=w zkbsq`2=&9?^9)t1?Z>exL|n}jMy5-K9)XTN(`{@-8g}da8|qEbBMMu!F)`^w9a(je zlgo81z35{;bUV!#;0<9=v|wJ(+if=BFst>LVDb?=L(v^E%*VO6n(d)GS=8i)&MD`^1a1EAw#>nu zyG0tL_f4kn7JBGQ>r5lx7?+1B4KASesqz`i`-sgk zGu7jJs;6{xq@K*;Uh;SFuX{o-_QuS!!=M}BX~nUj}ZwjAo(Un ze0fyzbiF}E&TCRUb67Hjq{3U@Hf^Ab?-2HzE40ef{ij1vWpPiloWkYVv@t@gh995A z8~fVcx-N#FJ@YOR96~1oU+l!w^&~^q^=PYPkBUPhX{;n@(=^RKp9Q>%ox&U3j_BI1vmwTf2^Awbl)}8A{P& zo-zGCOg4J(%jC^;SBLwERs!$4Ly?$~-gZNm;SHfutUI-bkT;Q%R_iY|_TSv~WY~Gs zlmV0geAKf)m@WN*f0QquIP4r0&^PvA( z3PH&{4FFXA=y-Y`_Xyk_r{rQcjWic(*&u<>!$>^vW}WGxZLoMAmFR zh?jpzK6@Sirsh$vg+j_MZrMW1j@u2Rt(d2KND?psAYe6-<+z~=mRa|4SE`*#=udIY zM#rStgjURpS#`zg>?1R}p9#0<yw@WNFwc^Q=+asJr!@j^_jgRcA6ncvSeNp^CZRGlJ*rVf2nauA5RaZ z+;htx82nl2zg~g;V@j)=y&U<-zKLIK0HP~ zTuk1!wgJ=EpF)PWQ`qdO_Nk^}gtP^+cSVBHz!j}yY>UV=84z>}9Nx17t=zFgEQS9{ z;{QNMVWb?ClW;~|IC*U5+%$f7>7!{^M6RX1Uc?JLRmCUDy&<3Xk&xrkib>9;@dLdO z_h;xir)VUY$2JMn!xaKaf>??OzlYOb&K5g~AH_|!Hs>TAW`rI?r^3zfUu>#BB@Y5) ze{z7YKGl-Pq38>K67g3+|?lDwh-IPVot)g!D?6PGjM;#oMn zlMGsj`#&b*y_FX!sr>4czQ7`O-m|68H=;+&t?#1yg)#EQ;b3G`blJ_dX~+5l$ta;2 zR<;Gy;^Je7Ojru{k}sE0CrQywB+i3?nlej#3qPY@A0SX0KpT*UOa8Xwoos6{8G^|jRj!V*$7U!Imw8HcG2CUHGp&Efq}_jAf;*-D ze*t`yTX>V0qK-#pvKKiDo)MqWu6f%s=&g2>6xx@f+Tvwd!Y*eeVmLHpdz%ND;;Ve^ zRH~c*e$WeI2Pce2kJ;ocSVbBu@KhDQNNUYo+czHc-pLVb2ND2j1e_Zdv%1#5E4|Y` z&XPqNHTSsY4`O?1$-JefnpS>*#CynqAB9%*ukgJVbHeql2oZocOwid7j-Lid?PkR-P z+4@d!z*gth&D-(z-=PsxkWxMJ^nR?aLsMH^aD?$H-hPlMfB*J5Y z(Ae0HW3 z#`+h~Qb-Ue{^@x=1Aa?I&10TB5&D#9N;Euy=_@F;zt%iFKwv$yFK1K*wN7T;it}{) z?2AuLQ^i!dLZkI2mlwu+R2mYbf3Q_T(c`;%@%s<_NAE}}H>CmUe^n@NIH++&jb31B zNhm!co%hU+C3Z6YRY6{xYU^8M7p~G|f1O_|�L)kp1S05+XNhGI(P4E#Q}z*Z2th zy6Q46OVUZ4y`&zYiPABNj0ubFEU!dvPnZ(Lv1rov3s<0<>#tx+dzt-N_?6l!*tJPj z<_=21Y-&NSLLP4uwsto&?B`FB9rc%szLZs;hC$Danlj31G11TL@|iU+NllGwZ8VFe ziv2xHH$+yQEaxLK66Uv%ptKDfCuy3g`JZiszs&NiPLY>%t9uP$&j5!@p9u6|cvShe zSV5F+>U^8i6$MmtA8Y0;O($rXr{V{G1u ztN)ODQ@z>^x>+!1PnhE9RUX&^F-IV}tpn%V;*ZUjU-?#IW}kPf{t~eGmvMUQT?OXi z*9Y!G^gz=&fxpLB>TIyxQg1>@fKfsOpVzCi`JN#|+B}w@nob*`bthNMI%PW?9F97V z{sOut9a_{R#MLM)A6X@B%J92t+JdeZ)q{LB1?CwW4)OsOnZBVn4+D>E{5hVaT@L^B z{26t0UKvq9Dgn~v$ZTg8vqrNxZ2tGPPtf7Bssyv_7!M^)nu3GR^=8gk=#eu638Zay zSBm}%fRG;cscV+C^%JLMsc*^6bT8@2a7}%xj4nRj%KVnOhKEI~EVbl^%u(@dtszRs zpQr7<)yB(bs!lv+J#&KVgT8s!FRA{;Rn0<=xW<{vT+0F|O$ovFNDD~`)YxA@SRAd1 zf|+#D=TsV}O$~f#e}4&?e%_-CRnHH1TVu@b z1Y6rSxG7k)CmSW};u0-2xN_Ix^-a@aMb-B5@Kl)8GZ7d~*40f4V$wP;Yvrdfv3Ej^4{;uv+25qQo9O= zd#MH98M3nH>!9FDBwQ^3z00F1yGxoVqk=}Jw81C8ia(kkx0tv6q>}qMb36SmXVXIx zoFM{>BkG%abaL~WXTjc*2WH2-Z?56Q8kpF zPWu-i1qNj;R%$oCHujnP)L{62Vc)~@MT`*1N{az=%tnL$0K9DFU71P#qpc<%r8gaN zWLuq&=C{p4TpWnVE2jt?yGf`W3+|Dx@Sv3s?Q)@JPKFdu!{Q?UC8P8pdkH?3-Ir^X z-#HNAfjB9<94Ix?3QT>pxT z1VaI9@CZ>6tczsDXa8%?|0K!&``*tt3_yjDe2voo8bxPn~4cX06h|}PeKb(KyWu=ofj5BRZ3EhW; zUC;1X8-1*<%i;0hU2`S`%5My?vzv4|S3kAJ2h^4W$Qb><8t9-EOfQsw)6O@JTYEoC zdb>s3qF)SNq?3iw9@85*h?yUQb6iqik41=v{skbt`1IK;ys~z?3(x7}b*i#}kawed zN$uPdG|XcbWFdQ#?)%c>uJTz|9~-RT;j~La539h3+%)gB!7W-dg|&Z(&Z`t->#Fim z(Je=W%!_+Wk9oPt?BZD%_?=OqbyYkeT^IC~WmC?xTV(C-q%PE@k1F1;ReWG(&|qaf zoRYY7=k=8_liZ>L>ElOPD}sf)s%HBl-C)PDm4X?CBb1}tNwFh7TavS(enQIYmU9l9 zMz@PH77h2{I9E|A>6-&l*#(>SkFI3dE33zC6ya?Ug}FtTPcgy2I`p}yR$0S>Id5tY%x_0;MczdIXh8N-Ni zulqs~+I7J@xa623>Sd|L-H&)) zA~7ju13bPd{|hj`aubhG+ONS`HtvL&J_eeTw~qb=sPk7^fXqEIQO3!yN&>L=$bgkU zxt1;HkiKr>cgAVZ!y6Zs1A8NGL`c-O%XV0p85T@95?@0}Pa1%wlYP6`@9N}_9jr(^ zAWfPKCTsP`I32OfUMu4*x~d%ux&rcJ~aEREl^UX%h JXZ`*3e*j_iS4jW> diff --git a/v1_api_demo/model_zoo/resnet/example/dog.jpg b/v1_api_demo/model_zoo/resnet/example/dog.jpg deleted file mode 100644 index b9cc33cf069da5c453b97dbb7383838edd07c199..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 71483 zcmbT7WmFtd)8_|wXK)J+69{g>U4pwi1P$&GJm}yM+}&l6Ai)PG=-|OMxDz1D^S*m_ z&)HAATYbB~^zAj=jD(h|h7gUd z6`Q?_wJnVoCp!lXv!}bAvn`E+q8yE)nv5Kc7atePdm2|yb4NQ5Um6h_K@RkPs{kng z0zCYG+M6N1H6&CdBt%3cbQBb1R7`YCObm1k3@mJXTr6xnYzz!sB3wKILLd-`i9<|6 zL`Z^92qgT^P2dpTu0cdXLqb9$#KOQL{C~E8y#PE^xMKJ`1UOm%JRTeZ9^Ah{0QDOr z65M|o;C~t%{M#6jQBcv)G2S{rZ~^de2ng_q2>%)NTkpWP^8iFVB>Z=r637JV<|wqT zgj^v>g{X9rjeS6knG1Su3%5`-bRuFBQZfccCT12^9$r3v0YM=tX&G5Lc?CsHEo~iL zJ$(a9D{C9u&vy3i9-dy_KE8fo;SrJFqM~DxQ&Q8?GrnhL6&071mX%jjRy8%ZKw8_{ zJAU*J3=R!LM@GkH=jIm{VN1&^TiZLkd;156N5_{}*EhF!_YaRx|KWlIApAGh+ws4_ z{tqs^H!gTYL-=(sg5fEI2u z=tT59TMU=~LHjSV|2weI|1Yxt1@^zWRsfg?aBnXU0S_Psc-(%%wO;As6zN>hchW(9 zL`<63?)`iADbANEm=$v3n+e#N-~K5Yf0nq{X~TAeq#t}&=hS~v-97zA$!GjAf8aF*z|c>np`ui~)U#o$&% zsCme0C%Fn{GDT!X;pI7LbmNxkq{}Nl(l82P#Vz|WuU>4+UCEX1c)40?G5uvt=Vc>< zzUev#+rDPeF{YP=H#??-SfKO4(03=yByeY0=eKfBBSjr((Nlg4FHvstbfxR-Hd4b6 z1>Cq0y=nPowAO%yZ4DDyff2TpnEeUsc9H5Im&V`H+CzG`+qZ-HyL~$;V2$u)LjM4) zZiO4-=SLc-2_(SxYCB1zgeHB}WI}L@MM<)2p0u^!?^q_aGWJF&MLuh=(58)Bt1{(} zHP+Gh2~g<7?`FgGo$JAHuBn6;8+7?&-r4;~QoiOAn&)byy!=F9wwbjtyjB*DAFwoO zDv4V2-4qk+v*hz_&2GfF#WMPi$T6B0;z4EQFsHgAfMk-PF#A_>KL@lduxeEn#mH*W zhzrsgX*#~xvfUYDfqmLjuo!OSE|>IM1evw;Vi|83+x|IuF!sK$#_%KrgqQd6?+lTs zLuOrPcje(EHpkLUX<&(MVMTMw=r*LoyTf+cPV}0>Z~hl`Modis)dA4l5CbZqw#*iF zl0L2YP0*QD_G9$TU0M0Ye)y1^AO*N9&|aPa@HIk)D`7aR*#3H@Rs^NPk2F&Cx;9YY z%YMg00#nx7+;3FQA;yus$k}(GzD;oIT{!CH(WkbLt)V{WI{8R0EsU1)Lq8Jirq#76 z`Qr8=&U9;jSFELj{<>5>-44=!46Nl>_s=yS+bQOiND+60HQbXGo*jHf);}hHn?gB~ zvQ9y$m6l%}-Onkx1gDh_p2=#3B|7{PCGY=flc(DU7l~3|>!OGC3ue8Uy%)M7--k4C z28CEVDSUb&?&C(Yi2DG*=`;zmD74m4_Qe_KmS87;*P1j8S~%TDJytFC^YL^^ksUh2 z^sVz&cQ$~0aqdx180%O)x^JvkFG-rpy%Th@8{{II7F`%Tk>;>Lsr52=dQ55+ysx9y z(9xo!zcd0GbS#kH2#Of)wWj2x41>adLd5t~MIpb}49G)e{mA+jcygTk4vvhtgz$!; zWc>=gAHZcLx^(-9k+(p$m&WKOWX}2d^r&nEj_F#4eNK(DV~fW3-%`L1?Z3K+UkaVe zRE(Zr?`&{4pkQPl-an!gxuR1chB=z?+4IIK2t!l|4$R{PM<$+Jkd`uj5-9K{va!iu zP`Fn}H`M0`Qq1YtPS}tg!#tmj+;;@e?dplYIeGJg3_UH|9lXRO6T7oqmoaIUciV{t z!U#Ej3#Bd{$N~tDn*F=(i#2TRD`@A(ED2P7P#%5cQg3mT!U~)*WkW~H*{U(2r}+k2 z3-_$Nu`i4gdAF#NLU}BNPUk5az2{vbl}J{PjSny*-cBE(v7Ea}pO>1tE>K!6$U0p^ z!E&z7UeNh-+2XgFfgs`T!2{$X|JswVJXLe8m}VAq*OM5J26^}^&W#>g4*7kjxbD>P zsCid~z4~-wa{JGbV$3ZT4U8HRj9-r&J@gw=r z!b0{T)e^i3EAtT>ib=1=9Qf!knMXw6G)lGK^2f|<9EP%yHIqhpmSCxD%>?KVf3bh+ zXphBoA9BqD>avbm+0eLV{o$F;YzT7n1kMpuv5e~&%{t+Z6o4bUi3g5$o8rrzTw8T) zPthgwI_eJOjlscpwM3Dl_TECCO@b(-wTB+snOwxk%{*%8Wis9dj*553fgtAyy}jQ9 ze+rVjv{`PftZCwZ8r-P9Q*=)#wY29u^uAmvENt=l66cZLlA89n)!PinP`KT0|9Q$ck7?(7YOy0AFS2XBVEl^1VJ6`7#%GHA{dVUX ziKE%BBN1aTQ-`+Yv_U%WiV#^4fJrPu&_z}V$6&NGJ;}s5wD}B~dvzXtJ{3t#C~Kng zZ#hNSMwL){+{a%^R10aYLk?^B3L-LAYp$K(+hqiwGh4;wy9y}%+}`=H{rhX^o-vAq z8$33r0pFwJSE2E+Q0}46;tEG|D}~V+ZrtFFdHxAIME?xB+tRBXujgv+Jn8qHL3yde zyH+mtI;Q>K$0O-Is<>#qtthBi1xfPD4U(WtHx;=mEbkH>d%sL(L5i)Kj?F~gyDYUrg>!!d9P|~AQ9Wuv*ZW}4lC7AGbnNqOCvn7ZNfe?&5feMK<0w)e>i6fv=A#(4Lr(hInfou#y5$j9gJUz&`c}3OZ5#>ss5Otnp8PMz-V_S2q_OmX7ni-<0%Ac`N`9?bjeTd% zVuu9~;{y%)0Zrn|+I7}BFC}lC<0|Dz^gPcfn_eQSh&_XO;jmMT;YZhB`>h^dC?H`@ zULz*lS-TAFj<}QB9fIc?-#5xj>`n2)e*v+Q=1A*B=j(Up>y!wF?HfsX->bv>Ry#F< z@TETU*-}T~8{Y0*RJ(o{5u6m#CJ`)CSLC7DL5`FNgT-Vqj57U(7(}jpB!Gc$b?J-u zo(qEI%f>o15AEmg_TN0;Y5Vk5sm^YVv2X^mh99!zo^%@p=GTdSemZr9W#d){Zu!Ga zCBi9jS?M(&;~fbO?TF}Lq=gb*05@fC6T%Bk`@Vo*I^0wK^KDiaXM$cF76wR2eM;W9=yAdRc!gb+p1|NzW&~WU+8N|UX?ba zrmk9LG&jP3b%%zy69+xq2G0biIFzX@d)!yby5+9n`bJB_K9P7R}6{)fNLv^wy?1rZO0Bri$i7l#VQJA@#fb)RJI@$X z8lKwbc1Nzkk;SxdOt)v1qYmN$5b6`A-bYU+C5S_L%a6?E{}$(~8-Iuv zHOw$skE(N%XnDFeVOU+;chIOV`Zltush3bHvWu8=xlw4@SIaA-7kU`?SI21}>?&-? zq}@DgGB;Q_h_^+JO+TB}c}m0$%f_gTdu_3+ybi>E%9(#)d@}qYrv}U(PiN_)wk;31i1~s(~iykFx4oC}5 z8>LH^A&)hmRxGBMkv})k#Tp$_%j^?fP-x~EtK;_+{;TEh@^LN}GwEqg|C@bvoHLi5 z!giJ+%-Q4^eZ%RHUB+?L9EAQsQ6LMG`x+iaE-HO+I# z^mw6cJo0(fmm7B!sJK%mM|G2I6yJVZ!SLNm$0?d-#WhiG>(ht)`cjc7mZRIMi3br^ zX?lY{DoZG$5VyW7EsCV9QQzy|`366&{vs;RXZ_J&e!|%b|F|8s6AE`^^0@W)>sq<} zMM8Mf%yk;78>!9-!}5r7$g6cRiF93^sx6nlN?Sl?3@r`rgue$cRi?s&%fu6YP7j zudS;h+C>(iu9fin8*-5EuGiEFrJ)l0v4xKFJu?Iax zm^NJBA6B2e$8T5VE+1t75>qNG-Px5Hh4kG3DT_#*fquz;=(m(8{fdk4)+Ok58WOk( z(vtS_4EquXfEKgj$lgJ~RiEnqevmu8T_}?81tS`IUcq8cPEq&dI_tdYF>|>0e)l)o zixw#rMCPL-6Tz*{z0e147k8880-sP8r_knL2YGt7<^G4&0lIOED7f2t=6q$Vw1p;A zf%<|jy9xt_V%+g|?1NgLp+5by^;AqU#>C?8IM7Ghj2P6BdY=pQfjjwZu$=R7SCkaw z9xjIhroQzLfcMn~$ojieL<>eK&s~H)aj#zxw?Zheq8oQtN3D5P^u5?MEUR}u2;__F zs6C1Rc8FN$p;Pu`wtPNG)3p6mO?N~-jr=~Gl|@Z%z0TC|B4F|B-m^Wa5S)?Xh?9PGPYFOOU}OzZ1MA@=Q=#oa?2pF7lVyF0%x51`Dtpd?E=%@nhIaUL$5nTR+-vWb2x z@?k(g>rnuq3}mm?)X5@Ys+sr81BvA2x619((qVA0*Y#@5O2aQiMdTU7#i!5G2%qZ) z<5hy@26_{{7Qa2ywF4sRLT4vvcD1*NDQmwpdy5*Ez!neEeKc~z#efgs! z?JQ6%E}VX~oVMzv?3(q`IokPo(;rrUY#@}p=4!;=QMB8NvZtXW>UrteGPBk!`pI+m zm92vnM-u0%z9_)D)Y|HOLCoVh^sx1sWJc8J`qW6sZ{*kmFUh(#+#5s>iLv=H6VuoEj@>RKa18$*ggSM_*OhQl+p(S+rMD%xGt( zAUVfW59H%Wm8j-oQ`!h2sUa+DXs~ZRMVa}%i+x-@v=%h-HnrCN1DsSsqUEDkNhhVP zPG`S~C`%6R)GIE~LNhr(*|pOaTPXA7khR1}3U_HMWYZ*mWz8Ly(RM#Kke;Ef>DsRx zg+1Hvh9N#_>#L~pHt|Z6PuX3L$@y7g?kD*Gb(lL52l?C$bei_6sTlWL*h;2Aq_hTn zh)W1SVRzheU>3usR;3_I+#(V8=LN9iyGaGi@~;MyF{UXHRl-cD=zQFoG9AlFN{sAV z1Un=tdQV+S*GRHMn-*blkb9-=OaF&EOMDZ|!krY7?5uPL4HJ#`|rhW*5vjgh`?`x z3ER}TM3Iw8NScio=??EPm))e#RsvmC?_$7*q;B29lbFq9p%mdeEJFF!5(+9;?ei9a zOp=0OQ2l2_CI>Uun9=qqXBtd)2ew$@1skIA zyCM1HRinRDX>?(&k)7~T6LqgQMw8dNC7)S9KStkHrJgq=A4`1szHG`w+g2N(%8&)O zj~6o{A6w=+TWvI`mY{M=9EjdnSnE?~xByB?sI^-9&0*QTtIdLJO4%%n-dO7G-v9xJ zd|7xko#xFK85nxEoQS1u7+=h@e6T*L=`Dp(J?t9!>Y4pvVvQmu%Vzyyo-O`oyyblH z?LKrM)K`DWruD`Y%{jGo>UQDqNdOnKOb=#Rd<&Z0Ftip$bhC^#qE~aQ*9_GsR z{GD3y7O#X=R(1m-^!Ydj)kMZP!ZNDyq8$*Q`OI=JN@+?TA53<(*^C$VmKDX4Tt#$U z43G4=Yug&qmtVb_eXt#f?fjB4Zbl0`#ASd~Yh*7=8qhhaWpJ{i(`*@_-xT)C=oEJD`_0){f1~Q5l zpwG!byEWRwSRE8)QA)MjH6(c8#_YWwbS7Q>TB!ZOA`ZXm3)jeL+t)2t?K=vT^WN>W zPq>T`BW698Z{vTLciz3udh&)g5X$u)SB$5;`Ul%Wm4Y&CuK`QQqGOIqrYhk3UFDL$q31h2+vufc)7?J{2iZWY-hl7 z*5o>~TpPSh(ueid$cg;b=^j6&56K$s3>@*_ZVh+TSQ~J&eDc%I_QHwtyRFJl9UZBx zS;F+#@1!f7+cml3PZ|R0qm0tgl^VB-57bW{+8-?UP06X;mhHRQg8Tt-A&Zr#9}|Fc z_AB0L@6O^7SSb7c0R(F7YSCHDQ2znMUMw}oE%Dnef?7Ba?E)&v_|3wrZXI1j1S_Hs z5c7u%qu1hBfhLkTh(Vs9*1$<=S@%;l+RnERg)>s18Z(oUl`Ds=Fm23srI_xjn%?S# zm64ROqI9Wl%{8X(C`Ff2;z_6c+cR<1L-Q1war64)$4^C^$-z1Baz_3OmGpI!7THUC z3~EANRms>Y4&|3j5$6FycNWJ)5}wY@Wpb6)G*7r=15^uBMvSARdgiHDpwFWU#>5UE zxhkKJib)P)+$}FkPEQO{MY-U1O7&wtQQ22rPAZa9+Ut$?cvISIeMcItLRRh1sZRkz zaw9A?(3(J!tE4p_?G(O98G0$|e=D^elPIZjEAIxyF9_@G2D@f;6D5d@g+|GXpcSVI zesXC?$x0^~C9Ng(Apb6fAO6=Ozx?Q`<6Q>AMoKgbzTL0gIvJ0PRrnacCv6O69y+*0 zZe!VuvX9Ig*c(fA_NkAp*qR*=&J%fhqxC*BB#(;F*{+OUKMPitUh1iCuYNIhJn(jR z03l^kcnb}T{x7s*ziZVbF&Fv^R^q^!f;Z$l9rf-*9%#P0h!Opd5@o(QO8lT^9z&+A z_>iT6J+(C>qp;?e0O`hefH?5?Ix2_Uq$WA??d|tpsO>iI>JW5c47%2u!TvC*?QSiT zwPxfU-l6p*L1Ejq#Sc0nc-bFYYP|84MVG$!96hrH)Yuk6{lEy(A}pomS#v!l7YtahG#gl z=j_Ishvd_he3>)5e6<3ecVi(%Cs9GiS5A3~ zS?M@86fs2zcJ})Yri4-s!K)I^+iAtbP`S5R?#5B+`oP&i;twsDvZ^3aXuefLcWJjA z5fdANCujmGtq>>7nxHx;w~qx6YFIJ9LP!(q?bFr6h9o*gDp6mq<~+91>jWmfR-AM2 z)D|2>-h)1n92o;!;KV5fYI^IOh+VdGE~>z3PKn4>-p#(BRqR4k99cxFxeifWH%Xxiky%syCQ3p7z@UvhAwUI6#JaH6}1Hu?y3Ibd(|vv(J_#QAJsw z%)n(`x4H)cM@HPnm=KDj1B9o!v7|UK%Vni$;Z0>fzL4cbok@}7MCUg;5fJ(#Ix_qZ zXdJEtsVF4n>*u=iII;y0P3-*EmjO@^Pb-&o@HB1BaRbTPleWM1(&p`?Lm>8$KP5v7 zCbMvo;iAj?J*~^mUsj7m@zIR&#*}@pBqNBgo_|D19}GP!_1S*uV`3I2OB+w(O^JGv zJ_bTR+0=fH-gQd5QH{7hexm0`omn5eZ0-Nndd4M`kWKR<kUo z^9a`%HXH&4LNC?r?r%zA0;kfh=C4-Ncn&b;N#6B|BzpBa&R%CdY!r+oHx#3k{dvy{ z=Gufv2s4{SJIL1M9uHpJmu^XQD@7ai`J}q_0=c_x!zi9x{>Oshgc-4N_!QjqUxZRW zoDea~`dBKljtD1VHrKKNMt9Guuf?=*V0k+4pfNvc#ZEq7R{Ss0KC#%`>VhjG)n_97 zuV@OF#Oh$`hf3j(1k|^s8v#rWFTW>mx6`Q~7&5>-hM6`Cx83kh%pEZs8~f7ws!o4q zLaUDn@%^kkYZuTO_Uu-m130Ae>9U)mUf^BqTjSrro1cRTO9R{K*%jlI2@xfWgt%o- zEBjax;+rz40mESh>nTv@v};&Q08^b0znVXrw|vjc*<1~O1CKI{6fSt=zGg+Y$);ya zeNleYYke-or1LMW>9pPD-vQU&e*lKjbhIM+({yeE{PHX6DB!*c-khl(?9ZnBV8Jw5 zyA3ORIv`mB5iYN&=O5rD1_Sk`=Bm~Y9EEG2Am45xH_PaNd<&=_@iiVq9-*nCeOWwlqPl6zn zhm*cUqZ8407eR$volLprp`=?Mb!je>j)J*-E&OwK#`i)tlai&al2l0b%)$<3{9Nt+ z_)hW`lOYRTW_j5arB_B7FP|&c$6c!*)Ymfl_LiFQd$92-%bF1qm?RWG$VI#tton-4 zl9t}P&N0H)#(R`rO(@?BH&8KKc!mo73H=tNAUwK- zn3K$%)md{iv#vh^9~Z2u`X&YmN?%kvaavg+$uZ4#lR*dIL0bI>A;X}=rrsYiSkFGt z9g}5C8-_?>xHnbU0)k-_*5{|1`X)D-#HA15Rh;VOQ8q8L&#e3ymv>t_h`Hr>qDa%= z^i1xpYndXEk#NV@=7R@Q*?;{d^=g=Z0?kydVHqQ?pY0~5=|dIu; z_ztG{G|*;@S|H>|ZR7Ry>g{Qj&6X}rzb~HRhKBQv$qw_E??!lH| z629ehb-L*Y1y$CLZuk)hzwUwBYn;O_q7b5H`d^<839EH9N(T_qt$DK>=Kk>Opq6aL zEusl8{9U4#eY^@h4jHM$qlhwW1Cq80?8YsOM82xlD?*U!GxG#0od@J!qYGPC++%)J zwwCGzLkYp+K!DEoQ`5Uc~hknTWgOrhE)uTd^zkeHnr_`K{q9EJez)t0r%3> z_Uuf;*IM+cjRGg1r2hfzHgJTWv=}uzY37+x1*y{%Yj7a&OdE>3Iokf&rQ&YB73|Kf z2yGCnHN|@rx-9rRhK_lzl3I7~>t-TH&)Pocsc&>y#hzx0xF|drnqyr$#^auJvt5=$ zD^?l01$o^khxxdN;=>dEuw(O8wLXU&NnaI=72AC_ynj)k)}VJ}lQx|xB&HtaTYaupX-Qnkrg_j8 zHxy0qL*2PlAj>Upf=;M{<0v!}^$~Xa4?yd!UcYpSw780l^fY(+MF5p~JK|}q5syjV9IF6?? zBU&cFD@!pevZEkh?}Rnk6-O^T2?;jO+wNo|->i>(Hg%RQE@yZWT)S9nOrp3@fJ&}k zx%vC+W;~NYIyxEZuJNPj@sSS_eu!QE3DUl~A8-QJp!b2CWfLYORaJ8v!~kCno}L-l zn{-9pDFte2jV`A913^-aF|!*}E3+H;n089LzAV=7c%E!xzM6&L7RYeF4 zNjk$PVoOXb-e~lUJRWJM-C76p73%e>vBnSQQdDSn(7R{lI%9_I2_O`f`yP>{({zYc z7EPjaAA{eDQu z68@xI#104@+}-9SP-RvW3H{KUxa{|a7hBj3Ic*~m81lmjx19U)M+ZYY219#9CSLqm zoa7&Xv1@vVsI>BsOF?AkQJp)iS@@I^?Piqv;Z|nzE5UdmqSU0`y$b?W7^NyCpw--@ zjz|&5^4O)_Uc2%~;@W+zf2S`xR@F8QkT5Rf+;CB-ooEsWStAO^>1sT)aD4{piK5Nb z)~Oq*G;x9TlHy*tjis|%h~gnB<1{aSF3%@fc|}UJvIVOK(ymH;q2Om0|Lu0?_q2d;(M$iN? zt@P}4w9R?)SSRfm_vQg!J(G~?vWtz(q$%9^6_WJ=-Jh!qwR8D>n)zjOcI-gS7qIQN zz7N0vs>QSSyr*2-%$9DIzIz-o5=)_Sew!U9F)uR@8C$TKmJe#~k2B_mi?`8KIX#FUQ<+S_S&IYR!b#`4vV2Zh|*iKXRGT{_0FeJ#$w)XliGxEU;NMcSW~$X;kV^htxzGoULt(eBBCsqC~1~LWsuD z^>tj8*i3Zj7pALi&2S<1BFuqR*=;aK`b6G6R^Bu)SuTr`{EiJ&6vx>xvgp>;{%k!* zza}eRpYzPL?Nf6$5z=$+Zk+ZHF!P7yUvhB~TsrRZacwyQwu}1n3+? zU4-uHvprs9sav2O$Wn2;x}|ncN1UPYAunsUHS-GtmK$Ayekl_Z1r|R}M9L3${gD^E^;l8C004 z{QHS|vYU+MieY-LYaD?wd$S|8KuxxcUgPV1tqZT0CzE+wI?9-!(OZ!N4#9FjGSzoV z(7bb6NXHVHH!m5e?!N)>YhZSuN2e@JP8>)E zkn?Ix-a7VsUGa%>iYu&7@x-hg5Px2l z_;$*8H(fO+CXyk~Hdr-HZy#`aoyNPD;=b6!=F}~%yV1y|^$(kNYK}#X-r=iXXtBt( z4%O%;J@Zr8ko#l?>aAFz&ZhRqO!&%b0zM=wd(BfR1JdtgFL%FssT`8KQpehBMV#HP zT7`7pCuG9s36z_Y8Cc2Wsv4X)T3#)<&hbth(T$sD9liUS=pOyORTt?5+aC$OsD-Ij$XV6b7?ekza;o1vH!i-(zAF=&n?ezEuVqw6*Sro#>bS zJ~(=qg|if-xCpdUo$%YvOEUK@;oSApL{{s1j4C%=#k6u)&HK&NZP?Vh>G~0krt8#6m-6lV* zq$x0h?7Ls05M(~(z^4#18n<&%4i&ULffm-u+9WknWV=rTEwC65^d8wJ9A2MeeijN= z33*I76WB*u$dS97glrj0>9(XXMhvwRmL5s<55n{;s}Q6!(EQ)515gA?3bx!gnkgP4(d65(d1C486coA!SJ9%Ah_R z!}xIQ^9(vu4L4x|J0Y*Ce%=MIBTIr!#3akhQLZVU;9wEqEWCseBHEg&vgNZN(z$nY z&ISwG-&R5}-rc-M8@Zjnu6Md9olm4!Sp$w7EWxI|HfVjUW2aoUu!*#V0|S4O-MwHK zDa*-3ZVRIk28@XrJpG|BILo|TIvl0<`y`ayvzFsN$#s;pyH;Lru;+A3c}T&JlB6SB z2|;6OBdzPvEBBaY+HlY;)c2;0?Y*mR7$r+IB=5vg+$AtrDmf^X`4EMMh!E<(=oaUn zK{&H2T8O*08ce;xJNl46Oyn2XRsVh@mV|zkWlPRIW;#Rt`VRWj>{VMyH$%3$g=IIk zGyQl7PVgxOy3fKa{of}6bFrhY~6{g>xsuw{M# z4#kWp+h~cY909kmW!=nu#{mUc6mrvc3fe5~oXOK_1S`-xrv0@TqCsij3^UCaWor~7 zs;2j*-B&=^xi7)*$?Su^#TZxmdBiJIv!bgXhTJrbDqaX5bT3ur48aVXCH5);->tC! z0la%ePUd;nO~(9*KU=Ma+`~D(+CIYefA)W&UO%im6yDZD(oNHQFLjb*d9>Kmb&((K z%o~g0Kd#?85&L(^lDw@|=yvDo-1H-MfN(B<<%v!?S@nwa3EidjUAz}PmtaEvZH#N! zxv{;^Hb%9ymnT>fj+BDh5ta?DvgB#PBVWU0Q-#8k!xHAE`|v&wuRHX zwxv(Rao_}TCkFn^HQHnTot8PwnU~=H`{o}2ml5K!xVMeZE+LazAsT+ebX`ZhJsiDF z-;^8q)7#SiG`CK&m_t zdX|{wzP4L}wlA>^;{Qle0!H7j9yR%>S{1?)gr-CH-@O)dg4-DhRkBj7G*o-^=yav=wGt+ufu&b%kgRB%=6TZW0 zziEbMa@qc39htc6Y6kD`^@m>{&=pky(@e$UuN&i+pRy`_L;2gymyj<9uUDdbo0u$)0z%P)V5DghJu zgU0Fx6QW*TnPFsZsH29=ySR|7SPq_z9l+*((|P%OX#Uu@9Pl?fpw7P80DYq0Vr&B! zefw2DII_hFLK$vlqod*I4Mg}o`{Omv$F!tw~^cjUH z4Ojbd1UTsMD=^E9FG2C18Xpmi1tlHE^3jds)dpXaQaa?eE_bJ@|Mm9W?)*k-=anEs zQgd+mE+sNG-{{)Jx8fm(Cj!{3zR;{~4og@45oD@qax$zI)}kRyZLVQ3+C1}Hw=E@7 z#w}2sm)z8#*cJVYNTU&kPrR9FYTQ47Z~Kt;7vOta*1zAu3PS;&!OC zJFoPoRg?KL+F|%VfH1{E|4$x+R7dQb)RgY2yIDZ=)#a%sk0r&N1%o(N+|z8QZOpcR z?nt3K)t{YiExskjmUUT{9@PL11P(*1xvV#-)F9d!$DXkbIYe(D`S-yv>t2b3-PO>w ztEO}?j-Pp>*P08f0tyh5KA}{Pjy6dayDn2|$X6F^G02tpk?eL-4G`HiFr4%yM*WXn z)Fc14_F+rtIZ7Nw%)^O6;Qbu#*QkF0p{Mg5wrm7;V3b+!c8~btX-RWuD(49OmBVz^ z*^t$g^OZ)O%2(DJ7awD=azy~a5`70WPze*nEcH$PmV zgn8YH0PcjpQfj!;q~)gHo0@TvT?B-!4*mgPvt9QCLKw%>i1-ufBP9IaRkm7ogfWwS zc7SI+7>ivWH}}#(PIJa|h*sKPxa6Q?f&gspW8afWYmdwsm3;0_FS#=NFDQ{W9-+Ya za|$#{ElZ|_&Xn6aI;qQr`7GG7rS9KPOYugSsiWlCt8ttD{)>@~?(_y)&GP)lYXMVA z_=J96ZzJ6Q0kF?WZ$DP5VxXshzgzZFc=o=?*ZBJGxgXQavm7S4hr-4#whfbi3)r~j z`{X@}sCMvPWmlJ!lX|n z9d|2xuD8fNDQ$*<@bmo{uHot-sH_7wJ2$ngZk4|qbNSY=7b$hr`XaeRb+GaRqv{PG zw0D>pTdsuHBlL=8p=qLNS4o%SfrZj$@!$48-5r#_iR1aPC3BvL-7Oimxr0?Wm)H`| zE??6}h+zF+MFgZ$}`u7ipJA0Ql5rmI}VQ`UtRQ~{V z4nH08GX(2>EmD(H{qos_RC00{e(+@N=5e2^XS$B^=42SQiH$ zdCAl^`Wd#G_)eP%Eh2Z+t=T2ERHLd1_00Ye4c1{Vzl7hI($U$HliKJIG2EssE14wZ zXLb{k_jx7;I+yV6Hw(R!{bHh-f7w&;bfri}e$vqa&E|-+UDm=g#SHr8rOQVa-R@w= zW}6UNSAuMGgF*pYHes6cs8uAOlSXs7X8oyNsnF!4`dgTcEieq{*|!<@c|`sJXyvv{y`R~VDNXzN zbTRCmJ`{+1(_sx-&IfLp=Di{<6hOK=^%O!7_6ptfIRbIi_d6- zD;s8X*yX{{kbGFJPiC5^9H~fxC6eZfrk%6X!~VGDO#MGV>rHICb5UYpdxJ&PK@O3I z_?l}Tcg|fs`$OjTsza_HC?eZDU0A;AAK-m-Fjt_8`x!E*=^>l$rA>V{7WFDpPgNKp zFF)$Vp^vxP4iYPfebwfbOhNUCLh%#|%Hvp0EnBQuX3}WxQi}Sr%XhU|EQo~DNSE{g zt6pBU4`R6}j9gVS{-`ES^|4xwYtT*mavtw@qN@q1L+vznHd!&rsN|5uAEA(&J~72x zv2am#zI(jE^ZnwuLZ*c4S71qYIg+|_OjA`ZpF7pN%i4^=dh87!xnbhFMJnbwdi7EV zhRdJcsC76Kz=SdP<&bIM6-qSQ7bo8APGbp7@5L(~uIj)&Yk{{sSVDi#*h{@ABHKok z@YiA~?h=LUo3?>Rr9^MMoYV0j$_q|YDxYuKfnJx+`<4vjv^B|jb>aw2zhj!Yrlc)p{Y6CPc_IGT`G6nW~`W}3}v-Q9}Lw*7t>E9@gr`2;3 znD4hcC<8;CcP)Ofu)~Gb+i!KL2hqw2)@xUTaw^yJzV3j2CuKC;zsZxr5xb7qX-6g> zehLWhzBr8EDhOXeEBCvO>lCs1lj8ZGM{8Fq(30 zTJNngTPx`Q>P*&0)Uk6wnM4X)31PA|y!{Tl@`Lf$W^0>1t}RN|8>QD%go?DP4`3IT;`B7(SGytla5Z!gd5;?zZ=2yB}tTiRT_~f z{d(s?LlWm0MEm=W`608qkf0V)A=W!JnJ}WG(WV|~gCXtH7Td!x#MYJ6U+!||Fz8vU zVYnK)*Pw7}YP9bdshoO_x&bM{l z{OMXb$`pB(w@^6$c>XCPP!%a-v$c{uU?rUG3_&aQB6yJB=r=LKgSvdZN51OFQNH5H ze2t+E=Ok}r38cZ^cQIk}gFh{=kN{Lx=h ztF3+!bzJ`(!a`#VwXt50yQxnOV(^icA#5J*a+8Pp{nZ@Sq~C+pxeo>Wy%bvf=uaL$ z#+N2+(O$(~R{1%=u3{*fJQ=NQML%5a$_m`y`mAKudjI}1j^&TPj#{Qx-6ksaz=>Mg z3^r58Gv$o1UgD5z=`!NnvL1Q5X9T$*>6-j|xdwrH=Y*{rp`rmkh)DPCw&i$x@V51> zuK7Sh5S9;hf4AiI@Q*Ad|7}614Io~&jD`v_-=n<%26%vEC=mWB^zg}F3bAS3h&r_; z+*;ue{Ds&w?|}Z&O~d^5%A3PQy;0ENBsBdhYEABfgIoTv-7E}pZe4`oN^)j$sRkr+ zK3A>xBq~o%p7u>7CwDm4pCnj5M=+-GcE?{E5@T`y%%roerr47E1*c!>#%|+J)Dv|R z$hT~M@mEq*uPgzD($Dj``4I*+VQWgzy38FI8Dl$rur*KQo)JI)eF?my>|UWHA|CCH zmfQAw{W+pLN=?8MQuZ*}vfRfvyPiqTL=F=ae7X?Zl+D@mE#FlZ3zffDMLxVs*fO-E z!8G?66MTJlEK>dbvUEK?jUYC*s0yy1JDobXvMz{5f62#&Pk~SlQ&3<_9O3q8w#VSo z8+>a@Y2RGi;NZZQSdlFIm`{-c4t6(cacLTw9{JIpP%V39Tf82i-u6vXe&q`nuK)qu zbvdjV{mhYfnY};$SPScT%%`u9I#Rs;JJZF>UU(9LSdNj$ir*&HPe~!zj{t^?*TVW$ z8Ryc`72QHV&YOxiW7H%W?k3aeT4qZvQ`Z(lpqRfdI0L z*azGGUxt&f?L}daa4*~sFfX(Bt3tX?sEWDF|Oc&>H<3-s&ZuP@pHKho^x6!SF$ zdUA!*zWV;!>oWbTpekDWo`FVMLrbpte*mIDUB7b?pL56HNA;+5FNxCXGNghlbYgN9 z=8%$q@305crF^k%7OUca5=UiW3*KMdKGZ?f`jhMc$uzL)n%04NzURzXLlM0ns0XP0 zsy?S$ta-VQP2(uhpCXC+2ch`g@2yrZw5_~^1Lceu$@V9aSpGAyI_;Ay`DWk)w8(&f zarjrxe-6GT;KrupT?}NYY(M0B*RR<4>f2bhG5LtCx8ynL?Ot_fT8rhIx;ts8Rb22# z%bq0B#*5-zT*Yd(HIr#VCXhv0o{ z&b~0G+T)V@Xj{;R*_nFqaz3GlAO5=PJ`wB5W2eC-&;rJE`#r%u6F+snlw)M7VdF0{ z@M*%N(mgN6-XL9a{{X|&kOYltVw3^?*B{orqS1`{-OAwn!Qy{%r>%M={+QO+){`@C zi2)fLsQ_RQeGY$G^Zh2~1=S_Xe9t5G>+?7#^fkj8+>vQ3nkA`)sv*jcRs9}Uw`;lJ|vFdE#8@PrOb^Kr{yHX%2fSNAFY08+*v)i zR%I%WlDQp5YxF}(x44Vpw!IDZ<+b}1@w|j*5*Ud>KQGe0AMsCxZuHqAiGw`8PVxxo zGyeeBuX`z_Ds)^~^I5FpD$2(aBHVqwX*_~^R?6siCQ}(gq;PAb(mW3LAQl9ZUi>M_ zZpj`Trx@;VW;_fUXO?On&)r44>V-?XxR4^q_FiN(9KqZIA)6-s95>U zbT=Lvyo(>YBpRbWk#hjt(GV*K7Pm&c$&}+zYTQfGpj2RSf)CQJLYM<0q-R6m$Jwf; zJx4Y5r^B1ZHnIMd@;`?1x|~?`?O#@WIASd0&N>dY*@xWas?t4POR_Ef>(BgGae&#b z&6s_gr#`j8c&`R0!Ryc(?x`y+4tOiG%DhW+=LYAVmB8u}7~J6X8Lqcpc1fRc;<$Z6 zP4YK9171wz+dT-Y8PlKQKD^a`ED~|fYQ@kdJdTxJ19Z+SB#SKDLA})h$tJy*;m3mF zxQSVpB=cTDq_A6oy9D+ozJU03t4AHQYq)0sb6Zo5-llSqyQp|75pGm3Zk6I65cCmY zE>U^Ta4YE>y+$Rr+*lmfhj{Idt$fQE9dTVUib#vu=y~Rk;ps2V;th7XPlMJVGQF!q zM$sp?v{ec^bBgOU8!2wxV-b=;#&KMaQ>He0vMndWkwmMyFhy@iZ4?W)XaIWGO{e&E zj9J;uHuuAxJXS7O{dT?8wf2!LJ>`()qmFB)B@a6UU;y8C?*U~zFf?8~c zRmZrk7=9ny!zkStZ^FGwd1XegGs~-wi*||gcY?kLY91q<5W@H*Wa9$9#P}`yDd_ra zP{}r)v)rDqGKR}MK5YOQ_#JpB7vN9TUM+xE1I#q0z8A(wr%c40nKOa zHPp3E(Kd(xXT4RL=q`Dvq`wOmb6n=7uB1%BH%jPLPIpb3(q3ac)N$Ve*v)b>c#TGK z-974x_@zw52N(y|oTg{WdQ?|~j%r4|3*NYEUlJ>GxMH1o;&h2J916?YSfv}Ci|PyO zRIW8uLVj+Qz+L#!5dQ$I8T!^8-;U!~!Ekd))i#Ygo{fL5F4KmriFMW5!*B+;w(&CD z#hv3P-npG$#6}X>mmiH!JKS=oriZFU;vlQT@l75nRAZS1cqXawIt?~59^fhZ*DrJN z%EAU9-HrX1HN0xZ_PNO|8T8fnixXk_{{RZ&d`aSjbURy`@Z<5e-t6Kjw+~#^TaSoQ z&6jzVhu5WQ<;}1!GqTmS5hh*R^RJ)&Gx&}-V{ID(NaO?1R-cT1EI|YgvCgUh>RP^c z@n4RYRK1bbPcez>=~+th(G`76e-TcyGY>)f*DUjHKDg`eQ|h{ykRO{RzY4}{p@IqT z)~+zJCCcYt;LjGXhV>M*yUf;w6TyX{9Z##kxr?&+lV|cEKt zr~|+60RBdzQj}DAyV!`N$X^trfdEG0x;tc&^Ggh|{eqZ)1+W z9#!c^OLNM!4-?(^k6g1`wt=mdMDq)xMmCIbl07)Dw|`}Q6H1O4jMzgb)87>?Kn8a|4-Hr!A{cH5I;RnE<4ty-oZ7nqW z2-aB8ZkVXaY;tqgpK@#3rH5FCrO_Td=CF};-t6u4+Z2KYRRjg;p4BACS1h9~jx$=z z3aH=#O)&yaE7@2WwRU+QSk}gQLeTQOn)&|#_J+`OeP=_yitVC`1SzcA%gbyu>^^%%x*!<@GpDlFX27FA^ zaTAF~b!k^Ts+Jgx8RG_s0u$s7r1%t5{~Xki@}@oNXi#f_Um{sraAp z%UAFhiKer@xt49Rx054eNK|74pI+P^-MFqYUk*L}z?xugq+|0uxn@67Ys*qp?-?`E zoe5NGQAHW`ICXh+*=#hW(=Q+_(_CB3G0mftg{|vP(JbX zsk|NV&hJr`T_7tT2Lxk}TJ`z7Ai7S5*5M;}+4$!b^Vr->8&Hb5@8EEmN4f_5@y*J^{h9dDdi>=_^RTF%X zZ5jF1NkRCM1#uoVw;C<2w9?~nVyq8288{!274%>1OQA>NkBIj2DkRWF5u4{+;EWH? zn(L=enxrFr-HfWwl9bL} zzdmfZ#ny~K-C_1b|9r_yji@U0$g;RB5w?@qRn@?~mcoW9bt16R5GWfLZ?$#0 zE|(N6g<}ApK~B^3A0 z9u;hoGgV(%k~6S20PHG_#?7?C7A57-I!jkSjM))}x3JNK!fq z%%8+V$vYc&(z(4uQ1e)}dB=Ljb?l3cCnkCqguG2K)KLJ+manM(8(qB17Eh-^Uln*y zQlpAF{4`Rl2WZ8H(8yoSapP z^A^WoQAHHMD!hvzY>Kf(CawnyscP#BkC;^^*W*~&Zv^^gzJAxfIa|0ak`^Plt~yVQ zw-Vf!68ygAycFBm_FGrb`eygUDFH6h4n6CZ*1TYZhgkswuQlP<9~DjX88cw|6I_13 z@iXko`JV&6O*PC#NhFV~^e+^{c@P3X51}>FL9T^BO95X2S$uh$NSS0=OUM1(^IgV| z`$r=W1cmV(VtoCemH{KA{IbM_pdwFJ|L`OM`z=;c!lT1bY)_a znRCe`*AwDzi?F4;g>kzb%@O7=o-cFOb?=NzV=-ozWPX|JP0~IsTHYPPaf8k`SI7P) z_=6^?BOf;aakoFpuwd~;^@q(A?-xav!Fs`gTSNQGpX_J!}os7sY;85gGP z)gSm;7P7Vy>Nzpne>L%q=f#O{Lt^Bsf3?!EuKpm(icE4DBt6ugw4+6%Sy$|R&vEg) zPOx3FPvkfEiQ>5b02BVuI#f`$%ilUeI167A-*}T#)R~%FsW5v6BB~3yi5ZcP)aJKz zTQK&rXSI08<0ZzYJ207*Gr-MojpNy-oXFE3#Mc=sJcA{(Q0yT3eQCywVrMBfcQX7_ zzP3~3zs>8K%(?ibtlW|wb^!VkE0$=>9Am9HZ3zOmQAkhB)w%JOp?tWOLhGJ{@l{Tx zG^2&VsA96`1an*3c8ne_SCiJ8YAWm_c{?Kv85r(rJhty%kKr$ZJ}vRp+Afo#+{pv~ z0LynmvE%*YUsU`j{{Vu2M<{~l#WPEDA;C7#orpb&>VHbLCD6;B`bSgYYil`f7$kAC zvFH_YYqs#e!p|1!@<%PVhV@IB0m$2K>KmTTj1R4Ty8Jx-n7kk1D`=9(K(ewA*$mMx)

*RiFYCaLWx$w*YST()sTciTj`p2Z@H;}65DO$T1Fyb&x`I)G>=kc=ua?PK&TsgJ_G zWbw6)y}#Nm?Xd#OAj*M&ib*AZ00nrQjuUrh(@?9;1sev52+@2V>l29O5n-ic_vC+> zoAJ$fSH`?u+1qJ~&4}&%$zL3G>-hEjtJL-VckO;1*=pWF4lqar066|t<=!!hmf}?r#>DLT&j&fd=cRoEXP{eKvGQXH zyROy7YqB^mX(dwnGsu=d*Jaeizz!D@w2&xAvHF<)CbejZGcJC$JXgZ%9E>uN^{>re3;xvFgz{V2YEt9Pa0JTTNjMev zPr^?S%c^*8+DX}qGUPLN73WI7?|B*2sM?=7!T!>J2JO6cq#L;7Nbha_WR(UOj_P|i z(Eeh+P=~@6+AZLM?j(xoIcVilxgOc$^{>%wYTY#(DPrM?74r|q9|zlO3@t9AHn(aI z*)J4)j(@sNIIkBnk22dU){@Ng0V|mTSogRR9th8aqIM~&2nkyx07+)nB$y^ z8L3*$Vg1lPt%H1$5N>eok%c#>&sz9m^n7c$2C zkdxoiy^6|WDc47n(4G1OlcRs zLw}@Lx$_km{lb5{_;nTNRf?Qp-6PL~w5NTpb-pcm>!+T;MOw<+Jj*<0JL2u6_e}aOu&*s#vR8%|Fd0s*Ir8aLhdv^T((@)z?Kh zIJMJL8nIG?=F;cco-n!^j+(NfZY+QnLxxg+rF>cYLg_}@WJ_qIyt$dlK2q*rJw1he zW25ScC6X+PUviK~I3v=y4~Uw5oT|wJN41A7-9>$DaVbenpD8Rmad1b#xB4yAF|@Ns zh9@nafc6!T@CG>yr1vUo(Y`FCw_3E;R^Pd`fZ=yG-RHUd>(1e|k~xAirzaQpDpUf3CISz7K=k2*?IzG2Q7~E zR`>+L`F^z>wft^2~g_*&5|)d1iQE6#0oH;vVsEsE;&UlcL5@8uOf^1wo#DPEA52j~o+>Rvp~0A>6qw=}nK!lOACt{VSPP ziftP@QflRWQvAmGRv4~g{{BtmbMvHKnSV6akS$`*s=OzmH419~0BGZ;Y8_Np zDR%QxXo=g~(wPRh2*dTHhS$&ex{9)NxY!Stvb@{Sc8Wh0W%#u*fT6h#-nps?@k&D2 zGnFH+I0n3?T|RUt^EiOmDKwh?H9t^ zisefaTNeHWx4BQA5ThQIZi{^ZUPlFQ@gq}?Jb9!M=xVj!j<2o>FY__#O?m|X01vIz zi1L(=an_z0*=;YNc#L1N0G7m~g3!2E$@JGQP5qRZRTT5{i z2azPX0R07bQvSz&DG>RB;vDt?4ShZEYg5x8)1gLq<5d82#e3G7b);XwtGy4sTZM{` zz0~~9(!XU-8ei*cV)kMvLFVY3s2@(X?w%U`f~~G1jNjkgyAktwU?}`JHTI63q}U|o zhQd8-thakOQ9=MuwPw;Evu*p#_~!fe8}O!;qP(6&?I>OyrftOk04k^O1NLsw&armd zjjVU^$2mk3s9+R(8v1+Uzm9CQ9R}h_HibkcxZl~v_G`^Q?b0A$wKbak5cmp2OxS@rL57idVOl8 zQ?XH}Z3{47$gTiTqO>ELNe8H^3R7^wts6!_IIPs)QMe>{hwVG#+fNN>FzFY1boVcQ zrdX!~_efOw^~lKJc0Tp-g}$K&iKM@dV(7Na(is^40DC=sdSmgg(+?5&S4#0Lahv-) zdzXtLB$^d=P2m>3iF@yKNI^e`PLG$j9 zM++#m$d;$h+V-aedYzw{Geu|SDy&Lzznl-l{{YokUNZ5FF=@YLTVx~dHsZr^2abe# z6Pn5R!>8Rsx>fDFFwQJ9V+P{e|*sY^$O$EFl5ClONE;4z`x9ML_>G5DI ze>&y<7WfrD8T=x!@Vb|I)SH{8k%MlPh-TyfIKWYXfD{46bg@geEQIWe8=9r%i_M38rU1r zI@==gBeBALPjAM%NNxfST%X3Xd`b3O?JD9Mg!5r^D(Xf-A4;k6RD+9FGlZp1YDu3X z_}Mipco~H0cW$KZM?2Vlq@TpslWQ82-@?(@A-B9Ael57;p&)i6(!FQmMuP^AB&O0P zh9y5J4f5?hi1g2HocZlU#=_TJtsl5*CO(b%fT8jF$XyecP4&S zdXK~W%WEsh*<%=OndI;@`Eg9w?gqJQEKsO37?qrkIRhCLk>L$G<4@IYAQGMKf~)Tn)9sO+gl#GS6YkktKp5;iM1PhC@qcU@&ru%MuQaAPNuCuyqYdt^+<* z05~5uO=-@#(?&eEjgOyxCHxb*yK9;BjYsXW?kx7#0}j5+$4c=O@NT7Vt$8xSQK4+e z2q27dde`4c;kdj}WVZpzT*NWC4iDq&UOVwaQU1>t){)qxiws3%yp}KtY;b`0uf3rv zQ>3RI4~?%1kf`A;PmpyR)`tE>8@F!82*qb#Ub`s=wRPH$hmHNLdV--vS76A;mB(Cq zisa_E61(JeJt?Qnuv*0{T|vWjIUHv-u$r@o@%rYo;?s9G0OK`rbdsZxNc5{oq8me! z_{iY;be;rxBi5>G8iU+IK_qe3sB7BF+ej5p&7Rdp+f^02fNHIej4$;W7aNr`+PK*?voQsZG0@j%;d?Bqg*sfd9Gt9&T#b)I@l#=Xq8g8>2oCMFlYVF0^ zJ~M&rD~qv;TTprD)YY4t={&QU06z5&QhT#xqI79-61H$N-mNE>E(%~C_2+M_v~ys% zI6s|l>H5<7AnxXpoR))dOzd>)#?$5una*=wr{M34t)*>Ch2eXXUJ5nTMj4Q2>rl_) zm5Are86NeGXv#|D#!-&!`|H6!7PReh6j3~JzoQ&id^N~skL8hDK8C+LJSXDmqLe&y zZ5ipD*S7d~;#{j7VXhZFj%&$kOQG8orK)GtF{9l`Fmk8fwr#YiiUP~?dsmM5Q{o4l z@v}}C9-y8p)-=Bp+1s$7oDYTj_8HQWet!wUeCh{*e?7R7Mf!j4=JxTG6 z*kU`5m5=?Y3$tQCeKS?$)FKan!H=jkl8YC-YFu1gv*U4L)}s4liNohVT78AwOaadC zQ&ZgpWZa4cOQ#-!=aL{@eoJx!vwbsG;PE7bmN`DAv1Eyv@>c`>nwf1FpeMM`N}pzp zRN1w8s=;{Ac{{Vzq&gq8aRv$`q z-U5svbRc7(HCn>D3*7{d;>*fGXMOJ+p-&Korv8#_$U5A0~Y_!Q$l~0);@R7zVma39Ar%pFM zgYdoF_m=48+vsa<^IdH@8JXEaisAIn8d}<0FCh*(S3ToDh;d78BB2KW^fl)D+}fUx zEIEA6FXLa1j=8r?hEpe);5XAX?SBOBBC*kJV-6Q;0guAIPP%v0b&oZUNqqhl?%oOU z3hBCBDjBnnN=g@%%agK5`cZYwgXL`EufgJV2r|RyO?>fj@kR@Fc+B`=$6Cj@_}dPh zaEQ{36UI$fCZT4is!dO|1gC_#9C%(8clCxz`ZnQ=a zr>a|;N>i4S=z6m1!s;di4hZ|8)+1|wXOW?mN@IbZE1S{fxh$q4B_G|8e^FhOx)W=* z09{6~$N?)Nl21zZFv?u5K=Cn~rz4lwHDvP6)1RezMyYqG%dPpO80^&Ypz`^yx5auM zr(tr@dHds0kW^zl*EJrM=0}+sne*KAuQL&esR{Dfs#-?ro_XIroX!do^Wjv@e4@A_A~T=*8lQ+?M^thw%eJt|9Y0_j>^ zxi+@zZVMl}G0zpp^|?h^qpIn&?iTha0NoNg0thwPX*Y2u5xzmkL&b3a0NK*nJFF5q z#4*QgcB{#%Pi=zHhXkIUm72y|pyey>dNh{_7B@Eszffxz#9G`|wh}Wi364)s*13zH z5J@U+RdeZCmv)!B<-MMWZS)ub*XfA$Mf@b+! z<{YW@_WpJ2UKi0Kvav{^1Iqw<9DY@qr~D|=?XFf~xujOyLG|^oLbJ9=V?_!8#w*H# zl-qJUtJQ?kP$*5N!Fg*nt-=W*nHo70`^rHCen!0F>-K2yWzFMUTi8u+BMg4(b^vkM z9OKuD^_E2k7^{|~M9a8WY*aRp%?Zw)uxHP=zp~}!n^?A=sld+(%+uT!S9R<#0qI?y zpWsL#iq_U!Xf-SQX7dfhDJvQ6jP*X1>6(rDC@Q}@dS`+4uQ%~uihr|67V`O%K1LuT z_b25woiY5dgyfJ)SZ+^bnH4FgZwjG9?{NMTOPWkqZ>2PA6hY&R1ED@Ri!NAs?m!j=(gEHj8@Bd$&jTxBOT zjAt0eH#=Vi>i%8%h1iIs&e9J`!T7P^Ht}R%Wy;4KXLkJKKVM!cC5~MuP>eAksmiz< z8r;;~wHrn;x!gx%-oCR3iExJ_JRDsGIddcBKa1LZzMJ-24JtefbuResPf}~iA5DG< zL%q2Q1$`;wKMT5D%X4OKVfN|q$&9dTIa!c%FJ33I=10N15nY6dwO zSui?o6_0l5bcd7pSDh%uosnld4xq6{%QNga#tka#jcx#m_YYA{y72m5zVhxi_5!o# z@T`OuS9QluD>oT-A2RA(pT)O9kwYGO@me<8r=74q32t#%R{j#Zxaj=*jMdlho}p>B zLglg9MHZUWSBhs_X{9aO9zY)5>we!zYj{GzAYV{>*CU|GJQ5Ez+owwG>?K(gVfgiG z-U&}aWh)ZLa%@O03m;z8$vi>>ec&-t+*$?%=R9|=ZvOyBl1JZ?pmZQpl^HFNqgh*X zsw?eROOkARxLE_#;Pvn?An?xe|h)= z(yARw8A&Y11KzT3?EsJ>muAm6JPL(Cw~T8tnhkKrb^8^^kf zsPb6G-NsKf%Gk7J7@ixks%fg5YofC8lfkb(6&E$hjY+)|J(t10JJp29CA)R@-HQ4n z!T$geZ_)%qFn&6doY%}~73O|j z+S<&MlyoN@c&i$Qnq)!ct_C~sYtk-!DCr@}Ad|H~-UpGBbc_QIm^JK5lV<96Jo?v1 z+YluRkTbYdn`MxaHxfRa*JT%lWk_92ti&Eh+UF+Ir?kCqlYo6F_I#k{bQ)EYT-}Y} zxc>lZwe?E~?F>XJ=h03ols;@MtQ%u@0Am!lbIWiP5?P7kn!OUSH@Q606Kj-ajOX}H zMPkq5E1Onrz>Ie}=C`d9P^y!Km3@1PyL;fTx7vwfLRYvo9Iv4HyO@@qHjtqH06UuA z)4X!m3Ik;O*DrtJV{C>t+)rWFuIPFaTa|g-p&!GFi_c=;E7a|+d_xq2oDg}fYkh9n z!BRGoIOe%EOI2jU;0~iTaB09ektXEw7POB(#5+ArJC=p66Kmrqn(cJ2117bntHQ)c z20bhL4e4XeI zRja-n(&W9Fl1R$3e+yU9t)kj$wxUp@<{p*Ij|atRs5_znPjTL&c1Y5j)bqGmw8{L3 zY>$E|62m%_~~>d7ACp1ze{FAuf4cNp(jG}La+wY`xY#og0BJj2hu zL2&Hx9qdWRHD*{!CJ7|gP4wmz%NpZ$xz!!DB+>0W+rq#c=e2iM%;F4is-y9)8$i_H zdCIJSIQOpJ2VI-D;Rxq6{hXe}^TqCAYIZY8AuG6{?l?8)o;C1Rt*VCm1m-YTJK4Hd ztTv$nxxoZ~b*hy5y`(ZXn4@D15$RaU5>D);%GHmS=hVDo@bc_OCG*JvKPggKTicrN z{2%*6-6V-`e-w&O8^=oPJY%8QYr#@A4}w@R9qZ2K_;^0htfK`Ec8=n;gO^h&MMvRh z(jE@@foHBPWKOM~pzv$Axwy6v`H{F*_BHZphWEpUk0YMy1ANt(2OZR#6~N?^iLmN1Ri3^hU!l^uGdZRJ~tdelqz7GsJ2B(RJLN)>H zXpsW|!>Hs`c@J=z;33X^YbOSVPbco1J90mjQ`D}SC;>n+PZ>DHT)1GdvysL#Sx&D$ zcZ~DU_N>}!_cn@7#}nfnUhYL$BJ*NWaK%1VKT}+!xNmeMKcZy^nhjQaqMMMxAwgXoeW4+Q+U?eBY*O>^D-W z-r;0!nQXV{O(JNlueoKB+qvml)*4LDjDUTQda7;pI%O!MDVBTp`C=$NcHm;VFAB`| zm&$`A=kTp-Jq2O6+*CUsL0Gppk7aw+VMXeDpQo*M#*CxyGlkbvmn=?)#QLdC(TMQF zkih1uX*%Bg25!7{sbO1fa@Yw7Ob!?~KaFDDT1Tf$HcW(r*!8Z(HnoV<6|BhoX<#m` zq?c<*!=?sEuPbPrOh(pFJ7T?_JCwMDD`A1-70v7VBFnj>Uz^cJE9fv-)d=#)@Nu|T zeq?#1kclMu!6P`vYG|~W44{<36lz*c+!rjs4Y~Y{H_n#YRr6S6^uX(0oG}XR9v{N& z%?YhmBC0cry>Dv@M!a?Av!K*I(<^b$a^lNwg9oo{ z)jdYmJB1`xMdzOj2Ip>hsqC(qh9ouU(yXeZD7M=uCxs!NLL3GS>rh?z3MlijFb7;# zvg%)Fm<;<>{{Za)GLgqV^>n2PCvdxaoFv*Dc7tGy(V@B26tqa@KD)8SYHdVK zt^vnhE8ToA;awp#^olKkv5mY}4e>MK`&lF1TB%S5NFD3ZldTGT&D`^=$_wFI9&tE< zc`5)L7Kdv%dh1P*=bUkdqfDy*D(d)2=e-0!wb6#yJ#>(aWaSB*+7$kD;t z_bd2n@oseTuA^xf=AgM2(A|)sM&L;mO4{`_?K#_U#yVFQf8rByc3oV7oMSz!=O>H2 z6lKufag-vN)anpHb0}!eQAQ8exy@9IMQjqJ^vy|adh#Yz_x)<7tpxYWltNpl(z{`Z z=7%p)Q?!@7E{S~aDUI7IdR47j!OW{FoPp4Hr}!5@5?pV&1({AZp4Gjr#SNw7#5RN2 zQc3djGdRV{rI%9w0EVO~ZxC)+kF9rF46<$xR{(LvaJq1g+7w_xBb*BAbnS8!U5LQ2 z73S5dqWPN$!cmihc0EH!id*QijKV)pTG!Ha_MFWVoyA)uwQ(K`@qAum+sMEi^v*L~ z{{V*dNTj{FW6*($`CQBE7}n)WWNS)_Nndm2nPxIS&l@P}PAX-*jZu~u3y)g0ajDzi zTay~+-xS{rM+~}-*$Kve;|l$K&D`{P5nkNMC}r|TzG?C37W0GVCc;KddQ<5ZF9twR z-rUx0)`1(LXw`@ZsI9DFbIx@S2E@u$W6pXIdsj1i@b2f##7xM05;I=F@uh(tX$pN0 zrAMf0wrd=2La<)E)-@uMxVgI?F?r!_LrH`)hILVr-73zT;~3&Y4Dy6k!00nxm#XU4 zT7(8T&=(^cK+Spi_;)?!)Uv}0?I$N9p~a`}51B2>lj=725*FO0LCT)hnXKJKBw=CL zN$xXSZQvW|SW=9{uHo*^xccSg}(+TBdBmXU`hpwzT! zrM!;r1u--Yk&dKSZ9I8WtI+T`rb8c>8+@CI^%dV55o>)8Sy$6T(ta9v>eAy*OKXV~ zx-J+o%WayIm_yl4l`61oK}z-uz~=xz%R1M+B=C z;PvLb`i|Njq%FHUo6CSRj8bncr!}4*X|;^o-+lf1P1EW+4#d* z(|kFiu>}DO!si*SXwh%po`}k&RkT^*J|tA|rOLVv{{SyMnyq>9PFrhVkgM^HwZwSa zQ`I&7YD=lfiawFXr-fRE%TIflz}x&eBpicU-K`>Hey2kZ#0%@$*@I#~?y1fyQQ~Wx z$p@Ld?l={Trg=Jb$yWhE;eoD;$%^Sv+I2kEGSOXwoT7`iy8FShHub@BO>Z`(3tR3d z3^9|QD;CegQp+A3jAti3Yft_Vx$ZzA@Pmv2jE_ov4VBpYTgyAlZeURN>0K_7q-pbj zp(7xiZX&Rt@KQ#FLTw(EyQ5ee1w{;1Q>e{k^(yw4D>oa#dVGpp5SvfpYevsWhRrr8 z481{7&1WU#i@8D;2cgYt=)zT)Jcno;ZLVud`W*_^xahTei)s9_%@k*i@e)}34lCJy z9(a+~*d#D(#L)*m2*|8!;gukCt^2EC3~Hq9IO$mTk?sU`uFA^j@+teWIorE(!m?+8sp(xD z>b#20o-xZ;gH{)3s|I4og+S^_^{R423uNSxT2m>KUquw(Cu--pu3l56GdRRqhJKY~ zu~xwQxTmshD@i1H3PA0eO$oNgAW{kKRb*2n>JZ^U$u#)Xsbu`BZK5R3IjkiSAd~?T zr(9yI2=ECVdexj^Nj+#FZ45ae)-qhq<)>j5+&bWnDy`;>lds(w`c{->N{{DK5OqX!Db`1c`m(dZnXzEXxNj0M%?~2)mjQrO2fj* zsoY&6;jAIUD!5->zok{z^%&hl#y=J5RX))^mP$z+MObkn{{Wt9o~@~@kpf+0RpjxC z@~SUpk8&RSENS||`-O_I?_1H_wZuG~+;tV^zixPLL&wZ**c{hcqH0CcB#=nySW>HL z?rC0{lr}8t7I3L)CBakCdz$C#kDPPd zo@cm_#5p)rXp2!vkeL_;PNN;GOG>**(KFMz^sKqHm}Ix$jE)F3%-ib{TU;N(7cX#(Gyfb*K5E#yWaa?KR0Rqg4P8>skipJ_zF@1Xq1b zWjgS&j1-%8*v-;(x$Q3ehQHa*jicJS|C-37aahseQ#a9)7X8T{JeG*v8Y~070Fa>&N_g4 zRu-*p*HWp#&MORUNhQq(vE|BglWuQQZ{gp>tzz3)iaVw$$ zdRNST9EuUB!U0{t^T&Ghy-UW8qFAg1;4#28-GIiwX5B}0ZjzL5(DMC!C9c174nm=R zYeL7vP+dl3DgZo;)-JbWKB1~hexo6dRC?5QzAT>BMS?yJ;pt2MkxIG@VQ{T zgXvpFpR#I8bIiMKw8!Zs^xNlD0LDghisrmWJoD>mE0R?8$EA00+Qly2+;M}CTFKS) zzbJ+(tVcp?sxDH6+R(}~aj2VR3rn*!DvC&Oa8IpYNUkj85{1C`J!>Y?1!3uGfuD3w2vqy~__g zSGPk_^H0C9d2U$=u~CyH8g`}owse^7Cp%lXu6I%Kz22v%3lx`XHw})srL>Z3J2=zi zm5)|pDyUa=XDF?aJjzQ$CtHJ5*EJJoY(hp!`HAEYVOR;_ol*%Vx44oTE6}wNiyS z9At`?Ib@C%U;!BVRYtrs89r3cOnTRBYDO(siBfN=HOnQ;qh`_dG}$L;u9n_Gv^OI> zReKedOT@%~AUQbbE2F>r9m6SRMRXkJt!VKHR8cYZO{ThyZ3Zc0Srcg|(y=v}UrhjX z9-_0gj~Pd$TohItRB^D@T;3^NR@LP{>erLHxHAlG7edi`wupFooVR-&j-RND$kaCg za0jh#X`UXr)h19PDIZ*sRQ?h}VXR1(6G*Pz5i`KAY+3Z(JqFn$1bSCP#YwMp^dW?b zUh#vYcpl!-6{L)0bCce>P_f!X1_WrLvTuB6VQrt^C(1K|YsP$k`$XyXcQ2>exSs5t zp%nD`*AtcByw1ONCXdH&6kXiuGgxU+`PVWHi-UkOUUTq!;3bZaBI=sa7rK!?VPZ*D zKGnD2tyQ)CMds9-d$%bbI2F?By0o?uv=Oh91~|uBS|@gll$52)*{R~c9@zLwz{diI zm<%E>YVnKj6yJEB{iVE@E$*l^q{7B`dJeUkl6r}* zWqX!2pNBTHTp1z)Ec5M4pm=SeylBD71C9o2i+I9F?RQC)f9|p3u6SF-QY@`2Ddgl} zXNu*V)U1(&q?DqQxz^ojX4){r86B$Kg~}oT*dJQPzwtZE6SguGWaEm((|lKLeR^%C zC1lS4oL3ZaSE}QE3Wgy{c7fO2PR2qvW9w0`%_uvwS@K=|o@Gtpk>`wwM+{z|C~k z#6p}onWbw^JL-)QaAMjOuqxad!$JTllkZrv1}wWydJ&46Nkk|bPnX@RUS!?RHwTGy z=IK;tapAp68_R@_M?mGUs@#4VucAI5Xo;~~qo+kZ{j25w01I5{^UoZH;xNcTA&(XH zcf%-r-8w7VcO(?Vk(mR0!|BCz(rRaQTCr})o4T}!306=s?Oe6|fp--((Q5YroRXwx zu6eFr_@iX2eiglWD4aCoc4S;iaC>CcIozL=9C1#)kRjyLCFU^80Odd$zk`9+yqSB) zr5|*F#kEM!H3Y*WgTSdtV^yc!&!uowZYM-p89mQhR&lfg>C&^NxhVX$Iq8aDw5~Is z*11&fXwhKE6y(xLa5mr^oQw+1xV-@3c{r*~sQEB7j&j9_P(3JZFiEp&>iJ@e8PBaU z%Kc<^Oy}CU*{-)ofX8=S8iLZ~q=Yn$yX(zhME4Fp%-Ft-mWC8jz3zX9jgyt#MAaPN0mQX>8CewG+3YJQ@}Om{xY|3 zF|dGho}#;pomDMYFVo(*FA(^)#^T|llHAF+)lc)ULz+Cgvgs*vXpSQH#EOnB-X(Se z8(0i}7_L{ux}znj5kyftozHtPt47?#yLR`et}PTJSL` z-L~$JprY1_+L(6w$eP_%8)yTP+#2esz)n%X=Y|5>y=R^~G@)h#!CR81xse}2 zDRC^fxd(YCuS&}CozI)Qs#s?r?c4J6TLgg&9s6~IF@4aG0u9`uQF(u zg^8rk1df$cN52r^<{N&sp+AW2UQnTc9)rCKFs_8)wsiTx&b65ON zuKxgN$p{0GakHCXNS7q&x?F@y`wYo1iz9uiI6Gj%qX zD%kE}x`nj(*e2Yr2JE~1Oax`>1j!KWbc^RtOl#(nySR_bz2hzGJ z)=E09$&$aleGaZ2NgGMV;2J&-58kgZ)n&H0)a8AJ7?OGIU0s|qTUh+WmB)TfV{2X% zGT6y>jlXv%2Ax`ya!SUOrsVZ{57Kosjxh?!yBuvbS5ea1E1`5$Ix!gowH_YO#Mb-O zvpD27cmlNxO)~j%5Kllyde5G4e9vZ8YhT?qq=FmG7=4|$0;4$<9*?Jeio!`|7$=eR zthBwB7=%bVAG$crbQTxse`Iaklq6sST~Mm3O7C+GC9#vEUWUG!Mr^~L27g+&s?8PS zNQik+)Ea|b(VtI;F#M|67#&4c(Dc{0Wiqo2p65JO(Nl_WlF;UKn|DVasVtVaKXH{$ z9M-m#J3KpL!}0TO?OT2&(`{L!U_lH?X3tMySm|yptig;GUr-KBVNMZxqqVgQBSr|) zSeiKiEJt5TO$zPw*{6xe$<Io{Yh=#y*|KxE_xkm(Sx*wCt)t#7%sxZ=Hv{M72FP2Muyey6D?%_P>(^f>gh8djlqWpDz?69TMz ziuFA!OVcIOVx64`BRIu*-^9c>7g6eV!wSP>gb(LZ=$f;~rr1Ev!~!yXDO1KQE0bEM zWQ-G3oRd3W5YMKqjkUZhwr6m^Ddg7)<85B!N4<*D*Up&8gn+{*isK{8MNqnlC10%~m){oM0g2=ia!@Ys20s)$Zn&7h8K_$UeT+>hO3j+fLJ-=~oj2za;Tm zu}YU(P>MA{GXS|^SX6(!bDQX4Qlh0c`y=M&_;s)Amt*&ZG4it!o@)B|FIDjFlW}Qs z=_HSxug)vgH7!*5T^K+5-@I6N=CC|ftlVoFTF(SxM@8JhM_gBh8nLMu(v#}FPE}ph zN(UR^y(h$a2Ak&ET$Kdm?L3dAbee9v;w!fb(uRvT^BIm!VQJnbyu8rl%p9T4!NILR z2H9#?P`$*fh)E|0y?qC@bm}IxT~Vz{(v9x)BGzoRcAi3Y24gQ97k~v&@Q1?d9Xj@H zPg4E$%!Io;o(Iyp5pX8dOmQ~tzbgjyt@X8sRGJ%E$VE8F=~WCy9#fS1nJh$8*JCPO zde=|6MYSL*o~kRM)@B;MjJGh2(qkhP+39}=ygx0fyRKb_Qfra%uAIIHnA%(9#V_1b z&MUtVrAjJt*mbH|s*y(rG>fI(Tg)5GW+N(k=DkA;f^YtVa+c?YL(lMx_Z<2(?E?HDIOm%{nhVY zbK;4tuda;IN4)0?fsE6v{1I<)eww_F%3@MB6Y`VmRBz;a-A*RAk8CmEeQP{^CMt?c zOLjRGSIClPCC7(n)OR$HvLbR9rfZ|{mXenCtO)s=-aC{cv2WbMthd$HKKI) z()8D6C?*4eo|Wap5$%*?3)tDpNkv6!L9bX?=^7hLG>R|@89hyTuYqkM)_hGQ(jjYL ze*1LKYRmDzi0}0!#QTy6_zF)46?4Nn%iG(+Eu##?XFi`=nZgsqDaBbO_nA%eC;2JtlO~kG3E0^A6|rhP^6wv5PQ# zNqGW7>Qx;25t{LD7Eg6F&E-iPP5e8EL66rJ>K-xGtnH&<9LX<0vGf(9=e)K{pZq3Gdhx2b}BM|m8^7m_E(Rz}GDIIerd6KXoKk~qA=0OW)6fn6o0 zff9Vkp_KEry8i(6s~^O+*9Jc#+s#ef6$c7`8u{rdHrqYa(rF`@wzn`_n3RcJ^NtalUF6;9h+ZB7-AG8$8WezmoCD%5?PmaArbHEJ%D z>}KnlQs1+0?0Gq;EUk^I8+04M;15As8rGkEq?qNENQP6i9=z0X-@|1Akr#WPy`Ngj zQI$G=_U1`z%O{~m%G>N0goOv5=cR6Ljbq-)v(bp=vz}vjZ|4QU&V9ve*p z^_Y_HR+{b3<T#?s+D#WVf`igoyy$a&eApqZK!01<=k8{;a!Y``s!g zNXQ2reJe8P%l4%>$j^Fhzlx%g)TB#*zO=f8?C_(3%XBs7Qk0$fx(=h+8WP{Cg^zGv zF`CiR?bhQ;yjE?>GHVJA0kug}%5Y^P1RkGS>vX*$O*Yn8C2;%K8R?4lVQ5pl)~{1N zd0R_knD8Eu`mnl0Y%EMvW2vUfl2~dNQCtwBKfAY@>HGl!(KUOeLIOzHs^1bcFWpIg zs8Hht)Z(sDjOaOaN2d$ADrm8!crEX=cfHQ$#z(z!hC7>GNdPSxqhy?W^se{9ULx|m zGX=RkmG9Q8U&n4)nba#cY*bD$g(WB)^&=TI%5(bGtoO+bF)5UeODL+EgWL;s-5`f< zG3!1Vl$Dzt~bY*w|4kekj2Q#uX^CAUbR&1r*bVWZzHA_miJI2O)le% zo^klpmm2Psr`fb*sbDYz4A&i^YHIN!H&Qy9W&Mzv`L^ZJzHC%Qdl~ZIb9bt`M`@&L zcJlP+(S_ytew=PX|K8vXNZhiLXtRg4JLci35TKuH|EYPVu~& z&ck1XCpr5xht#NWVRPJk$ECW&h^{0=jmN%hi)VA zi|N;TB<1ZjTX4ATCIolCaQZSgdZW+ArHw@)i&SSLYT`Xno+PbJC4 z+P6mQnzf_r6L@PwH`icYT#>ji>CGmCphd1reXHL~x6e)S##;)!b^6z%IXO2MzvNXq zt*dTb(tJ0o_=iXPT(=_H*%#$Q)&Bs(xeYqz_U}=i5i&&#Wch8Lq;;sTd~2x1;mg|; zyf!heN6L9&>FvdNwwvO6i7w-WyCNczt?R{j^{QG?<-2URD7YoN>UtlJ;DYK1CNlVD zN&6`EAxTL%oMhfI7U?CtOi(`?OIGfKDlLLD;Ik3+rJulhWz=#@Z7RhiTrw{>BC{^; zTH5m3+CHU%XFV%}6Gp8XeD~Hvr*DQQXL)%Rou)nHl5(Jsdk>{~to}Yte(p%*c_Bsn zt&%r>KjU4e#x=Loz8naw|Anv&2at<&{pS2w>OZfxF`=w^_Cl!DN}t8$S-2~ z>KM1W70#_Um&-uS^1d>2TlX4#eiYImNa1FcMqH@m`&5^3-szXfma*aza9sZYPfF6d zGU|HmzL(`0O~hdEGlBK4D)OsA-Ws#D^48_PmpI#7=rswXlH>)?A$!-b{5A3Iv_2n` zO^(U8N58GjA_^-qhi=|}3AUP^KR=$;~MSEpM+)8Fc&Kv_?^x zm56MQT#EVI!}k|n8`A;1kIY+x$s+}M{#EpU?BSvPr%=1p(RRTcU<`Bw=DjQ|+_5yG zwZ8sGPq*ifEaR`-d!54qxCihR%j=PqC%HA$N}D&a#szZvP=C}GCOb&70E}e$rZ~&2vtLzp4``qDJ0KLBh<9} zzd8ucFfr1h)#ozCK*0mBtgR1G`%Ed9&f$^kiobBXl#n^DhVLB=Qna->$f#%aOugdUxF>r9vAizM@0@`-8L5le8W zdgRdtpsw2$zNWFYNz8HX2N~d2c9(Qk`9ro5xjFCYT19jsO|#J8fo6nA#|zrDY(m_x z$-)i*`c@n2B3P0buyesRt)wGHtL}NlbttWfa?sm`-)~XJV^4cnollq)kIJEEl?GRl z+*Mn#Gp0}t-%=}~D8!9fM_bcv9^LlDgV!DF!hAJ4LwZdCo$dN@rdt1?zpLns5~8T+T4)YjJMAu>n{j@dj` zJ*3}k%V2^@1y32{Kb>u9wo=|9jeb<=-n?|&mC@-*q;uM`$vn&zK*vliL2bGu{{U+o zfGe}U@ZnGxjP~nXwy7WXT#`uRrFL3wruuzUacwV9S?xaVELqqJ~!0cP}4=C z#-RTId;7liOF{5mxU&y&IGCPtTNyPbs#fA1%6^B^scFv0tnWE|bW(T~+ftNgFKG49 z`S?_8S*x^kaA>xZTFq|>O~a^B{c8(H(dMN{65VUph2BAv^JJmlvFy$`_N39{9!_gylGQ#t3B{{R79(S=HtFC8pQ+~m?q%%^r| z)Y!GMJpQEyK(8{_wJX2vr)4 z&2uZWlEsN!A4*7Ow9=!Apu@L9D&t@I)5R~Bl^wmPm6C4vHG^EvNaYeIgH>n(GJ@QK zE2q@et!(Z01JkWvzq!-pgv=F0;CgZ^GgG*d?@xrN-hBmJBNr>d-$NHJXS z({kZ^wQUZc!`2fnxbn&Vq*kwsZ!GnlIqjfjX&3JS$*woVdiC58{hre*?z!F56^)=@ z{hH~emO%2S-x=#&5yV236q4N5gK&?%@Uu?R=leb&k}krWY&=$OrF(hi{h{))4nW6R zpIS0&R}tUa48n{zfzXOyh?;~}T1}!xzcb)sj5Hm1N^^Ryfd2q@QZaiRPNA)N+Pp1n zGa>2+t!V1{VAn2T`%4upM&$;R!hRSsY9ecx#BCzwK<`@`pNDR3ZxZ5Th=Y8%^!KTq zHyNkVo$2$ZVx8B+@n{5*ONgTx;PIO0<<;P_x!S}44&3zs*LC7h+RPSOeUywIVgm&{ z9`*`EPO_ROAR#z($!*1U?> zP*5Hz5SI;-YVEW-j)fc+gvl%|fK&|QK9!rVS;=Rp84OIN#tLBZRbE$wV&kegWzEfg z$G_|E;h6j}aj2Bgi(^ zG0GQWjtcskwGF|uk^^rinHf{%NT;`@PBwnYbJC5{-7Wq^xz11brZ$Htu+$+IMn^_` zD3gXBqNJ94IX)(}hNeQnBnugF{{Sr5@(oQys*=JSsl;fA4)OgYhffOX3Tw z$rPKl2(OQB_2(+|t0+2m(T!@3yW(z%Y2o>!wzs(x0~|?L&l&m9grIld~>@6FDIvjtEc1B$3x7Yea$t7|6 z2amNo{{RlbZ7jRDM}S!5n%uYX-k}(V+_{a^Y!RsIT+V^usWh;*w;p7YTY^Ex20d%G z@D7D-scYYAxL_99OR6x==IkptdrI?AyVpZ2#y4r19w1AI=eM)GSB`t3xHEB)*0_HU z-pi@#?*uEokZnLZmOKjee~VU=KnO*3tX%BuJ$D*5yy*kio_ZFr>X_mO>cPF48y=$lVd2w~A+DQt-TPGmpYV-bm?J|pEeOO*{59vkCh?P4 z{hlap*5=`xYM}saKD<MgD5 z6G*bOYzYiHfDfm=MW^Zp?$p}C>|%_NL-juOrKh%&;k!R2uvGylamnMYauz-&hf(np z#}b(L^y3-#t{Rlnqt5$(Of|LcZrr=v+(wc~x(sl>gjH>OQyO-JzcRD#+N@4$mbhe& zA2v2S<0k{ws6MM{tG&IY*eK-utT`F?sl!TDsKrBmtcZKj(HkBS)nT^NZW`m38EwP8 zdMAVR_|nZKNl1o7A=Qsc^BbE5h9`T5_pnm?1@-e_qGqpPo-gamr;&wUKo=Rupn{#^H#h+G&aQ+>LN0^ zE_&9Mq5l8~{MOLl+D2PcM%aO~w4UVpfm|+L-khaX+RtOPHA~@okk7J3w?;o`a6r!N z;87m43Tv$zs8mD%sN$;Ww!hfUXPWoz6Odh(XbPGC04AyUQ$m}>mp3|tEQ=dRPV>U! zJu7@ANX{J8HhX-GF6-5dTRQ~VsxU~QXC(ClIIl?f9VVZqYnrFp;YnIDp;^G))1^_- zZEb8kJRT)OA>%!>^`?9v(QbAB02y7ADy6$kwUGzRPZ|0SmF&@e&a5QX{{ZA|QXH~T zi@QFQ(=WU`c`n(!j?#Q}pm=A-{v))% zksj94Fp;_b`06Xud>!K}JKH#}?dAd}$!nYb*#U0+QAln*|_dnXr zc=yBKhI;0$;CrtWhLSgM-)B?MsPFjT*UDy8YDW^59VuB|`FUtz8mrp!#azXc!k1e0 z^gjUTcFV)O3p21e+{`o2=UrJm4HKI$;YOLpeC9~Mg3Y;XW3 zyUkYRZQ?4Ub9$bYz-#WZ7I|E4^sl7FTGmHa8EbZNTBL$oHrz3hf-B9Pz?x<}wn)u( z{vf{z4CEc4azCYc^s|lbPw@hCUprHC*NQtI4C^Y_QN#-Tse|oZlTW!$IQFj-@ZPKC zYh^*nVT@O&X_}(k+s63o!5)XbbkmjNV<{^#?Zz|Hn$Nf#;~A^daCei(Y}Re`-dYk! z1J=1}M^e)(yo#XY=hmssaAhMLj@1>!g@*%!(xOQhJ9C^?4ke1PF78e%*fpIxZB-<vPqTjCMOH&NSlN+~KeO7bTUf6GA{L2Jw!juIbt`TA)>2 zWY?kUv5zCl{A-v137hN1YUn}@IHMI>a=qD=tk?l-m0ox?=KdqmB)znsW)Dz7>}%QX zVnb@Bxdh;v=C1TK^PW^AWSo)OqNX36opdYmBg6FH4BYS7U}ee1LE^mwK+$5;;zKFx zz#g@^bhf&G$N(&Q4_d^IS!cd|z~OoU&{reFMw7eMo!;J6RhEZRV^|#=9lbb2+ zZZ2&m!3)ukddkxDnP#|^V=P>fLf1j#2_}Z(G8+AFJfFwI<6`lTW*p* zTBIQv>9NVamg{`B*KuqooZ}#hp8LbnK?#x~St193D&D)UeX4evWb(-M>MO1Al-8Qo zkgmCIInTFR#bN2npRtbU-itCB)Q?KN;!RpxWuDhfpJlVI zAIm+D@PqWOTjHmN4uf>Jnsh7yIbegKtmt0i?@UMYjxoyDBr4UR4o0tUywa>)HOVfW zO&dKG<+Ncexm*%S@99aSMkBc!fM_BkKJ{t~g}1Y{Neeniz#eOR#uirgz8;QSW%7s_ zW&;`aHQ!A-iA}2{aLKv5TwQ9)-^6KeJVVP7cO7dtS-gUMDoEpt{a-BD_NpEc)Y?Q4 zv+%_dW zU%G3im*JhhiK$yj8xpI@2b>z^ej(agT-+3LBMZ}?Nsm~5-e9@NXcDb+b4KQpFy{nM$)~k1at53IcMp)$4-D>*Tv{2^* z1Lo(EQ20www2IP00*tZhYm!m@&A#JE$tYUKqxefzjXe7zpsRz7j{cQbTemm5t-Nxs z+m91?rryHdc0`M2@nqFg>f!wSNpolKJ*zAbv^qrsGD06gjMs#SBEDFM2v1 zEIR#+m+5edyVLkP5%jFT5qO1leM9$X303_o5?gn)vROyYH*IN`rhO|RQEoHU~fY>EhoL4oZX^Y_f5>01PcJrP_7P@Y7 z_YK(Nu&&$VABik%bxT<-r&*ej9v;hb(pI0ONbKT7XB zJ@HP@#__?YMP!gh7a(`s2Q{@fL1E=qQnxOv^Ev8XT{L7~JJw{jh7h=l-Ow{m2V>f_ zJ{e16G&9}7F_{A>!5s}vuXu?(IiW{(bg%7Y5(rFNHxZM^6|HZnNvP4lrsd{U zayZBbzu{ix;SEV3@g0zrGSia~K8I>Uwv?8!M&KZ2thVVOI-pnI=ZY(~g+$US*+pj&yl-M$=MwZPdc!a4(V6;<&3n z9(a<|#d2HS>P4)ZJ8g?_MOe~2QrB0LLkujRy=iVH=@=QTF$xo?s!7?|-o?dAm++4B z!uB2=*6gm0ybX7$%H`${$C3}eJ61o8B)qXpeLF>nn{<87S2PBULzTDX`-_Ra|e z^&r;SG+Yym`M`E)%W!@6+sM6Gh$ zZFI=g{_(~O;8!~|@LlT5YiEezd3?Cs_FVn%#+O*};@)_E%F5o~>}SjU{@yS<)((zu zHCg3-J})(*1LaaOqt?7EWa(6DiuiYv@8oIkNuBSCJRSXyq)QwZ0$J_=UqT1@%~_t} z>%$%lhfPL?2^|9OJ$e4MPr*JJ@jt|m8(8T!Hj@|w2E4mUf`_v5+ltuikM zrTAOK@8Mqt$l`z8DBl`4@Sjd=$hAKb>OLISwnJt1pDY!UHXSLl_?l~|gqnlzwnM$4 zP6yJs@Oba4QF7i)<&{ao3LOnCUss>U9xaAT*)Z2qsC;xp&p)MpfOrR4@cZ~?&q#@5 z7V*NdgdH)ASLY?A^^L~4XMd-;Hq#u1W6B>xUa#Q4i*xv6#HprgJED$GV3_ni)%4js zqbPGt>n&UJG@%Ye=Fh1-U+_ysmU#6&J{6Fxism0to@_?@qAYCt+)0YM#fV4 z^LLNCx3((={!s{#NygEDO?G}L#L)@B>}$(4tGQW>Ym(2$(!2>iW_otAHar<((!JgM zop*Ur{{T~6#*Hh%aV%}XEu40)GfAF3J4@5!LR_<*=l!hLX9KBZbv)yrt#`(iqKu|= zw-;(8Ro%x6k9y42+k2wtE8fir~CS8N6RFIb85PYo)%Q za}YZWRt57bGcz0k(9|_^NV^dHJAo~{Rcty3Y-6{jeK#EJB!(m$W1Lsao)D8hE3-bb zE(hW(>a8-+5*zr471f8Qb!2B%>vmZdhD63Ws#kXNMgiJzJBr`(jA_OPv8s0xWMHj# zdml3{V+ssL$8v5Y^#ZK4m|jLHE>j0|XCAdiX_-D;g6FTbXDGNZPUT%$g2e1w1PbKL z%ja>^IO4h;RyCRfnTFxW73cR_kJ)Z!QsaZh?kXzilVws@Z0gO%Apr0NVC!1McF{18 zZv!Nrg0q{&Pvz`b^4$T>YJ|%z`~LYFL_GO%jx+B|p6lJ*>5V%($d5{~ms`1Osj?#I zneSajm*Lo!W_bWC-Gxn}+epzWm`q-ou8u1biNV{R_^z5bbk*Lhnzfzmv_@U_$abQf zC>$uRYs415XYU-~4%q8mUCKuTmSSaZKsr_yxYFE0{#0Ob&hEmhtu*v!Gk1-TF1oRX zTa$G>ISZT?T-QA`kVAhbl7#^$DWB8Y;!@;*!+BEmKClWa-6GgV&SN&o8znx&J zLZsrJ_tfQ#ii`ns4^r@F zS_yB_iOGsAv8$dT^C?N_j9e7!D8*{Xm&2NFiRI3h&V^1)dEnGuF7YLa+8sq@mz+bs9rs#$Ju5)Z~*kGUJ`>|M{^IR+j zCEmd8VeL>yX%&>>;naDcecJO~MG8uzcT1<4GImxpbX#fk`{<^ESB~;Swlb##SF&na z+}7F@P+myVT$RJf+}D@*cTlj>HAwFDFO(r0S-JyVKB=#bavSNQP|)>^=e2qmJHnjm zRet=sb~~_Fg+}a!p>CCE@F8%L7zAVIc(DSIjuEqI!o39FwRhCz9}xG6I%Ip7_e0cd_WB;faRI*H_jqB%VC-?p6_LEnE!Sq`>=BOzy|}Mc@b`?R z@t&bPx2_WROR<*f2xJ4xm3Rkf^`pcZy!U!Ex~08^lH5vwtc3#-2TN4Y^T(2w5wIq))qk{v)id; zbr{2Bde!0Udnz%d6}F4Ljqr5i?S1v8{{V5*TkBuh`lg*amCTn(7zeO5;C~XlL96M$ zIn=+it>$o(5YHMC4{TSZ+damot=l3gZz6bQDtN{ZHP3jr#LHo<>UPmd7zS36oz!RV z9s&F-(!|NBDLd(VSi+C+uWRaguZOg`XTQ7EFH5Ed0GICqde>3m`&l)Yl`SA;)1t{c zN6kC?`&NDatEKC$5Svhu5HRx+B_BWIU3J#AZ*QfWUk^C3iIfj9z+yg?;@6sX=arNB zvCdA(8CSNy*sw-*Lkv*6h5I_U&Q~?)9ux5c#x`j1ptOX6IGM};0Hj};cHRrJi^h5=x73TQ7S0Pv^f?%<@R(;u_fMmGiTkUWoyNK0xh=0DytG%JOSwTb z6RPDx_fzOW;MOeZ7lzmSapzD1FeCfK^cCn{4cD#Zo=*)zOi?stpm|RlyMydSd1uF; z3u@X5M}2D#pQBrhJP~D#FWsO0pF>yuuSzvu3ZwnxcI__D{QUMajTd{ayZ->dIDJP| znmhHpl3l{<@5Xm7B*u5Ur%tB$Dq?p5^3`9#;&#OI>p`CxsUAEbNuno3FicHk>BvHE5tq$xw0!Pl1p?EvII&Ay>d7vM+(usrc-5_0#I_TWA_37`+P#+NMW4n#9~!LIF~;aOKv>}M--`0> zJHa=f@P}&}gpf18mk_qhWtn<&?OmUSwVhwYz8y^->I=(eRPuzV3M%o8=BAEe8dYjL z=$^e5^na<*3niA9E=ad1qAAU9)`Z$s-}Ml^E4r$-yevgH9*Ly$q_SI9pO zv}>zvLI~~$nwH3sO7`b*9B|e3rG!34fot<;5$AWUeI7}Vt%!uxS9G1z@_*Kb)Zxt= zqng*HG3}484oz_yn1x$*4xGOu! zyJ#bkT=gMhqFa`I(PZ^w(y(>QuPP#gl0{#R^(2HiZ;&^0Rka&bo=7AMyK@X<>sWG^ z9Ze4f*pcHK2X)HE$@-pa>Rmjr01=Vc*Uf$i4XJqE%GLpr(gx$Qj-=Pnx)+J9wJUu< zM1@+)>Ptq+&f>kt*EQEc#yU5-jYoMTZ$0i8Yjx?9S+@`KpOYU>^#s#2i<}U1ky9ng z7zraJbsfcasX1uHYivT!o>?RxpsZWl={j>YrOG}l<6`_1hOJA+R6{> zSk@jI4>6Rc=?EL29e%aTS~6>v+MSenWA7C_J)y~YwnE%{^K?XOMf=_-Z!Qky@qpF4*ajFaW3@g<81MQ4sCp`qc0h*~S4B=DS=J zZfo9*B9^Cg@OIamp{6==Ljr5BO6^ z(k^GcxsF@NVBA|h@x^kwb<{d{fy|y)`d!?FpwA;GkLg@MzK-%AE_Xgv#|m@aq0}{7 zn6)iF(Pw68NZP0FjB*Wn>lN*qsRd+{``M%AslJDx_}cnU40uY?$>L~abO$GZax0x* zwdr~#zmqVU-Ph($-1RkcTh}kVb9EegpO(>Sv4>!t3mye#KC=G+V~#OhWT`rXQy!pF5i7~FB`S0CQq3v+l>r3#X(7t=!T}v;X{KE>+%wF8WUpGD*T%zJxi+?ti0ti; zI2~(#$?UA;mD&$6r5vg4TzAqwj3>D>KbmF@an=u zM;eX9>%$Y3RzB5%;SU>I_@l&+ZZf4;sq8DF@h^m|-^4c28~2+}7;VThOL}Irr-pPP zDx0>;rj&WPm%R2l?Jrciu~3pIK4JbIm^CkmuFcKbpw1OVUDvHsLxv|!jcqL0Mh7Pq zq}nVx#hgx=_lR$<%qXDUG#dcjCI{~E=zXe-{2kUlAEY;9RgU%NlMfVevd0-CAsLTc@-vPq z+3`idv7S3fBN~iE0TjSwI`mWRn(xHbs-WntW~|qzai_}kcSjTP55zXsS8(atOm{kJ zPdb^SnaNnx;D9sF8PDTeQFyCU@T8Hz`s3YcQXy8h1A9r^0D2mtb*~Fp$8jHvba?em zR!K~f-A6k|Ixzrk10_fuffZ|3@a5&~7V~J5103;rklEa*hT~^JfziJMpK2T=sapEI z+7zOlmqOQr{6uwM65PUywz`D2Z5@@Nz~7IX%jdZRk6JY?U&b?PGuvqTBk7xv$8gt@ z1^fz=#bIe53MIVOV7u4vq|&WkHX=@vt1lSCa&UUq)|;m26Zn=dw%uR&h=~4UZIuO3 zV32y45>A3*}fJLs;A7fvFUJHL#k?;W~roKs@pxn zJfYtkkRC_Z+PKe&zYKLvW=s3~CXCGqk;<%t42;Er$F3@9bV&RzZBU0yI}rcSpN9BR_jTm36)z@lLnne+}Emdux55y2#CLtmny+Krl*!jN?5I zdQ~dJ4{DVo?!UwO@*Lcrs^8aUuw_3%flm7q-a^GLc8EGK5fUCuU)RWq}`#*)+ ze}?1>cdSk3IuLS$bK~6f6{-6)>DM}4{;vkNVhbFnoR(JOYai~Op!PqFS@`=k z)|-2z#SM!*jV!3-akOK1<$!BhRFa|XsYjXi*v7v~ar?|m8~AKA-A(MTrMQVfw?JPi zr@!M~Wu#eYUM<#cqttGe!pgn$r9i zV%p%hwg*(Rxf^63?C1Eu#=JUsh|0IOYSPnA%eBZ&C3ayhqjmA0Rd{rWRxLWkgm%`T z0k$#G&!%x+fndv|SjS}?K3IpWYt`TS*6x|D=sp6!x6`z13rHlmA}N*?0LTf!`ho9S zy8i%#q0#lLiJn%yf(b#mw1DNJ1ZSLcwzigG7gBBALG3P$u9y6~l_{;_k2cnId;b6n z=Tx`AzeW=vC#eUg(Dbitfuiw_k7eQ;E^Th$R+bj$W_sj$smhIQM%ye#Vg@iBgf_r4+>Nf!yF9z*4~S(X`U;ykXbmpvy}{O z6oU*Dj!}oUYHtkL+4u{?E2JZfw79vIqDhET8U@_jkGBIp_2%W-)l5zjt{W1sRtoF8 zUxuje!eTw9ir4rx@fPFZ?}_gGE2&?TV6yFo?T-tey7=S( zz>NB5*w^S4aWrF6l}W{WIqA)&kEQ&X>M22T%Gy7{o~h#B*w5ith@wOq0n*AH(VaeL zgdz3{c=}hDd`ta|v|kH&TT<5Sd_!?-ZD9*e>ntKuwR?3L{OjnO-x%9lMuO*6WexmD zoEo9yTbpke{4w!XmuAx3PjjYR!x(?!a~o~_abBft9VyyQH)~^}RADH;boqW?f!-$Y zr^DEFyD4pTeHpFI+?Q6*_pQqA1S#jHGAqP9S$TV)l(>#x^fyqCu#vNp1AFtq{44E` z*_Xu@emwo2#)UhFxV4JoOr0~56@wHw^w{b@1L<5x!T$gdcy90DKZ>;97U?i+7P=BO z#J2J%-e7(kY3c}NbCK#RA#5@^qvRsE%vD$sl7HZr|x%c+&3zdpI}M+E5dXukN9N&0E+(r z*xJ&`s(2e$Q9hrjV{s82ZIF7hWCsJN!w%K-FN&wI*7P}|FCEo|^aXcDLoAWH^#pYU z9<}zFWf;PZMHgvy?8gZECCHB`n*PtkT7ABuX#Qoj$_WvVTpq{Vau2w!GhDGVOagt< z-80vU$@oWoHH3QahA$fzI$hJ4!R~~R*!B!F{{RCO(CU%3y0r2z3KyrnbH<~sO-|mv zy-#9R8@~?+x&Cd&Xpx$0PO50hF3)>aw&|fTu7Z3MFIPNRJt=fA#hsZI!XTJmy)Ee(} z{{R+5ws6Wu6rGH7-`>5dPBfyS9TDlLO(@5wL%O`y(o16$hissb2dJzqdsU7bR)ri1 zpMhDDYxdV*DPY06VzK3c66RTwa`F>}40=^gr!(a%*u^^|Zb?=@4NI5u-*y8Iynp)X z{v7c%X#j#lye@KE`PN0Xq{m0Nw|Da%P1s_6MPfsv-$Hj7*<5uf*#@wL-Pz4X4s)p< z%=#zbE{?4}udHNAD=@c+_f_hDK0hk!ehSKNr13lx9J1SSL zqnTSPFm>JZ>H1dR!i(c&;$IWSc@z0laoYvI_^P>1Fs(xUT)!iVwc}Tm_G#gYFra~q zV>OR_M0r*G52a(;c*@&Xv5Mx}Z$8>mLcdNyAcOdWPa1hJAUcuA^{!u7zK+f@I1L(y10I#ry~@S%ImkT#t{>uUz*}o;Yj9R+&IU(0Bq;u7 zqG=@BXFei)l;3xuJ+1rd_R+knlPW8z`tmDlOt3}yfCJQe8fCtemW>U#+6n-QGuHx{ z;tgdyGTMBO%8<(`NpsXLGDsD?r5Y67TDvXFQc5Z*oh_Y|WQ?c`KU%%ytZH$BKb3d~ zjs7*5F z!5r@mI%4AF8;rn%IfRf`aW%T7%x)Y7`%r&whV;-0C@JQclzi_fZbSD_Evrt z&|22eyu~+2@=DoxU%4i{u5XIA`o@B~oGc>ADAALWjt&Sl>B_XBq>W!L$EEAK`Pe{w z1MAwnGva5C-%e>RAoEH`$I3V~-CN>FxwKOmEbed!=QY6iqr{RWgv)UlDi?9jLQg^c zYljnyQdYi(kxG<#%nuO!XuPr0V~|^;T}|^SVaft{{W<(aQt*Go+nr|e_8Y5+^y76H zM!0Tv``0z%O$Sl8vw~Ue&zo{Q((TwDf}Ny%KGaMR-Oj58tbmQ|aB=jnE}N>>yY6P( zR5{+~x@qm0VT+N)c4=Ynkg~Q^Xci>bF)lwipW=g;Y+Y*8ZvC z+r2987ru7E_2LFZgY(T@ec6I{>55pChv;fuohazhfaSGa$^!@C;xDbSRl`Cgxy zwOCV+Dvwg`o2XlOdr!8qmRROzEFLk(82xcfw~8Yc*jME|@HqVI&*apVwH+>df@Y0S zmI>asBKV^_={J$tX%NdQg1omNeLZW=oeym{?PX}LhI6Ow9j5G&2a7ee)1r|#gnZy^ z<(T>k^DFIRQn|PhPLe{>I8}|o1P@bMBV2=0*R3w@8LjWFj^>6@#z(bu;{O0t)I>U# zuWbuixH3m@xX$0giWC(XdqlLklwzW;)}~gOt+s|H({?1t#5c@38qx6|#QC)uwEJtQ zg2xULC}Q9bdd9xCv(yr4?Jkg9G2gJ)Jsz2Lsa-AZ=+ZbF8>BoF?On00?zrFGJ1NGL zV|gu4O1ZkS)UBd*X=4lXG;P!MtcEankg`o|*ER}yiF)8xEVq};<-r*1n&K~` zx$#UozL|3ukt)biGmHW=!4*G-ynkV)Xm_b}?G^3dD(wi`$~}!<(Jr)=5nYW=Lae(- zoVh0*^It%!Q<92JT=V4z1ns$F!T$gV^lyfmCElZdAbmzyP8t6IkgB>oma_O{TFUAg z=5~x+smVs{wX5Ts+n*I_u|sa~-b9JF<$s&PB^*B$09=VceSG z##HulN-IV9_cw$09Z5>E67j;$Opyj|*NVv!&e2b8RAoQs;UwAHrMw zd8qtf@gGX@y`(yYhld|P)!OA&NfzLXSqM09m-t`Mj2~RrXW*GN>%Bivxr%#ftu4?! z&7J4YlOZ@K03Lr1^}n;`PEB3Jmv}pOJbaBR?)oduTHoxK>VwL;l`;VxHxA?8uWG(G zS>cP!j$8Ql?2WP%=blHlYu0r?3;aIuC6=Y7n~3!NT4?;+X=ato2;^}d2pIbIuM*Qd z7p-`E#M)+)b!&vTxOrzhNS;VJR}4E7fDgTSc!)xAytZC?7kIs1=;{1_rD{6Ho26;- zUx?n;)FVdZvrj8yVtI%cYMg%W9IJ&H$3xqmDlZB6XGHj&;#u_jyZhY_UxRZ&4b-ZTl>lW6 zF#FjD(>~SFg`%laii-EQXH_WEoOxd7)gA}eZ!}*8&92>C$9<_AUo^oG#=weppdaUx zUnbpMTYN~BwTUk$u+|#lMtGuxvMFaIo;r1}YSCZDUIo+I$5hi|@fM9Tjj~7?B@6z{ zshp3gqr-m=^fkTIFElMu{{TU`y4@wkh03g`ImscqjIGEz-+L#9do^ z`rXEvd~WP#TUjAQn&H#sBdV_M{ZC5td0Iao>3`WTGM!46%Z4ZLGB0JsH5c(qxwx5}@IkPB7lRb6EG9ezD>E886n-cy^R{rN<)- z4n`~Ar&^6DH3?lW>r0#0!BwYKMMqBkKK0P*JQ?u6!`?AVskJ0qlt&OHoc{nRO{3-n zao)Lq8b{%;3F~)I=yu=P){GJv{F9I|!PqnYaZ^R(d!2hzyVNc0`PT63^g^Q)yvEN z4^xRn%iKqQec>Mq+seiX<(gS7mQZ)yd<^jgww11Us?S%2 zto1gLVn`SasAO3-`fy8dKU`OU{?oq)?sT6MNw3;pPiKBzUOOXf$RypCLKycT9)R|* zTh{eRylvyJi3#DicKdFh2BBuim}i#Rl^CaRNZvu~$gIn+6nKNhI&80C=NLisj1b z+Rdz27S|yBqj33i?yZz>#=TbDCb6w}j%lsKMPXnqRU7zbVZHwVEvP<*y!*yl8(4T# z!}>{%Smo1aTW!-Ez8yhk&!+H59)`NFgPtU@)R$ATl2i8Un}?cZllZrAq<$C%z9&7y zO9NBu*mbR!$u_U}XiX;LB_*Rdj~HKHY5ooH#njS=FbQn~qwWrY!$e8-*c6|9(WvZMk(1~u zx5E#HcW_N}bFH#l=t=~$Ta3Jow1N%?q3i9%b}SYaoH?}W((a|JR_nMZHzagczZ!Iw zy|=Q~-E`}zfeAW{#UTTs$0~nMr51k?Mx)~UJ2~c%$10euB$;r>86Xqf9zgonkp9zt z9D~CCE78ri+4P&Jgx0sPAxP!M20=XzHj+L4>&UgOf5d(T@gjN7K)2a6L5_B*}%Jhl0U1;jTyM9FNM3L@4D=FLX?~ui@EwcVev~*j=Pzv9(Q#@noJ4K;V0Ga6D1-wGv3;s0gz8l@0J5iQMVn%E( zN@{kORAu~4xjqZ{jc4MBQKPrI{ngrm!Q&180FL1OR-LMAHy$GK1@(kvt*y1J6PE0E zm%;p(`Bx2X@f>)=;YPFJ?Qk-9n@2ISG~*8{EQ*bs^Swh5eMdgEZ{RzfRCvDL;&S%` z$bnWy%PNF%j(Y=K&}r9jgZF3edOoZ0Iw@8C=ANd9iL}#WZe&r6FCw@ddsdgqOb_-;ER89ItPMDE|hpz-+s0EKxy=Av&t-Lovi2)O5S9P#<$yu5xQ zbor>!^1JouXVT@f?=-g<^6fM&JXIPp9Ku z9--kKKgD`1?WUKt@n(|lw>DD}KG7cUm4H1sz{ukxo}ky5ct2Kx+r}2&VQ6MI8(KLR z7;Z;Cgc|fcgd>Mk?R8?}>CT&`x^8=2!^h&CG48Cu`*dsb#y^O9e~0B&t+X9V(?ym& zLfN28l@nXJAch2X$KhP%{ruh=@f7z^N#w+4U{^mV=fAag*8U=kSJIw1er&ihKpQ3( zKg0+3Tfg9aYm$R>VeI7Z9*?(^{SH-Dbl{WNZ9`d(9cAzQI0;fwMxnrRMyRF{{R5$c&=qt;7f8*xRv~6GeG386W zRio2mVU?01&I4q2I1E1;^{;?vEPfo@D>h^@J0DU>uP5r2hUX_Z{8K2V*Q)LzPEP!j$BLTsde1wz8iRg=SKK?sNXrX)LPm`ai5ixK7Tg- zYuoga(QwE}uM5%Pjl40aMQ*cZ^8WzsK1_-?L{&mEpYIHgqPmY0{8OIFD|>A_%zKA~ zN98~U{{S?QHiCL%rh9UGR+MpcXw!0cZE0n_`y(h;<-ejj4~jl5{>8u2lTM7^>To;D z_9*hq#HJBX*C&eQv{SF@{wmUKd_{R3&FfCLFAOLIcFK#t%LEQ_j1$v6Jx+brfj5Pp zOx3UN8uMJ9EQbs=Ec{RWzU9_v$AO}x?f-Gp%RnQquct_r>$wD zOPXtU{oE<@i3cTt=lNHu_$I>B!PW}Os@Hc243dtx>K44S$MzS}Xj*tM=4*SUg62Wg zCU6P%=O2}8=-xbwS@F$=g9{drlF!FQKv@2O*Dgs(#`Bu@qo>dO&dOY>E^nde8qT)U zU%Ux31Xau9s+@h_ulU!F>8_9MX7j`Q|8vzcEUjxf7U4Dz3eJ;}PQ`caVUcFnZAs`O~3{iIX106A5p|9JuiJET^>o&Ld2@9iG z#U}{l{G<$Vit}9~O1RXoV~QBsStL7A2UGR-t#P$yNv(A3V=hiq;rE;E@aZ%k6dPz< zNogzj&`&c0#v{m89;1<;YFn$@pB8J^dM%(ZSVH^n?vw9q0AUyF&*xp0wZ*~k_)FVX zT~^O^y6#LC8NkMS6VvNl&w#Z(b`K3*&1`O7cttiA&XAZdifFdk-+FrTB~*N!fWfxcd*o~F0JJ_W{vPwbJuC+ zuHN`8=-S}cbsbL9dy+iKr<8{B`0fpAco$B$llx0wlsm_35g_e2V~(JIhplPsqgrY_ zr&V5PQgf2j;C>4P|H=ESgoESFxWqd0P(<;PP=^ zi?2niT3ZjXyRGx>nM8vHl$_%S^cA7 z;(Kon_&#lJA2qLUW7`5ckCl61dz$dsHHmb~T~u1#Ji4Bs-|4ppk{AAYsr08^)26Kz zx!p=Lr9=0RSNM4yZjbSd+C!pvaw#LUw%E;bQvkbXkF9c^DENsLt*zCy&7pllMr2!;~a1b1WUR;1Ar!qFcXsq=M zas-nH7;)TJSEbKqW-aY)WAm)7;|+)9L!O);#MeE2qRFdUL#=K%EedAp)={~Y!0My8 zt8Mn{e+@uwV+*KU5aCMnVtB~*6%vf1)FS=e(T_5cwe%?Jeh9vp-rcR7HcKm(^GV!D zC#SV*>Hh!+wH=0lkfLUM%n&OFWk1VV*c&@)d z@a*@R&i1wtLjWdc$qGLnm9VyYb=B0LV6%?yEL*^{W!pP^=xpqkOjeh>-`aF4L1MJu2du#R!HTyedpbeic9QuV|*!6MgUVq?4 z(WmkBb~c)I=AU#fj7&w#s7`R*4@?93SE^|q9k%hdp(xd@A-U5oZcXj%bIel+&6x3m z3t$D~(xSaSWT2L^X@8jUPMhUXRy?0avwsiCZ-1g{5B7^&gFJE>HslV9eQTughsQZ? zC)4h~;U3YazA$-rm$K>L&eDU!mON(|=i0VBPvZ{@d=|c%!f%H<4aLEf`BB`o^AfrF zNi5|?Mh+{p_;K-lej#|y-@_1mHt@ZLpV>{#-O}2teUczLkXHocr~^Gsddewja${0a zyzQaYcq8JJeiQgfCyIPWr>J@4NHp6x7s>P8xeMj5T#T{wQR;Z12gDx=Yf-G8A+xyB zWt?sk+8|k!da@i6K9#BC4Sv>H1(L3rrkyoQX`g7gFB?L+r%89TF*F?F5wKC{>V0A}3}#A~N(-X8H4k&fXK zSYx}zvrN9KU*#>wt$62+KWB+9b)8NYxshhm+(~+>Lo!A|VlumzBy=R#p0~uG6}N-* zD?Jy)7e>$-VtWjN5UU2-bDhKH&o$|u53|-hH)W`5Kid}%WqAyO^3{ky(h;>lLD(}K zdH^dLj+|U?qdhsQu&2!#?0O%<-3EV!Q`)|eCb-d{w_Q`qyBTPSl%WKSXQpy7#(gWB z@Q$ykXxioFo~dx}cJnl*c4BEjz)aZ#c&a#YfAcbk_a^zTNZaL2n!;dgl}yB05j`eMk=j1Pmw+Kw`-m(r7ON| z-{w;B5^25*)}K*$@2C4>f5f^=LKR?`9*lF7*QdRE&%w0u&EXdpxAyC$x)v4}2(2sR zbqlxUACKTGknj$pb>Yo&#(S%1%xu9~o+X$-=Z(vrhtr|2qBM^NS;$)Y3+ZDmxXXDq z1cVN|bHb?g>GiE}w4peA-u1mTT^l_NJ}Rsg ztJi>dAe`3y$BeZI{3U57jH)eCE%MsJL%qisZRDNA^{zYOFU3C$_~XPkejm`y>p`JD zC%n9ua@@0Ub}wZFb;mibYr(qOTfN1M=jBnkGt{ns&+A`5nYzlINKtJl+3D3as=wo* zl&9}Gb$`{)y7uxf5owll=_IVS?$ga`{4u~Dy)l9@?~&*%c<)=gxA2s4X^Z}idcob) z;DsymDfT!Yl}X`y4-V>=ekJoYB+~V0AddRZCSq3L6=NiM{`U)yujFV6_WuA4eVS;) z%WB3wg!C$MztX%(!lqd&Z)oi7*0FotcU}72#?Mq(@Ya%^0{E}+uI4y`Ca0j!IM zXL8MsIXt)@g>`=xd;@bnovT~lYJzF(q=xF+`f;(Xo0Ug4MO(f?!_UFOPGKyE4X5y zE>(W5z=d?{5j@cA=h74@NLuHEKut@dnmeGoWIId z9S=gSxE%AxO60sCW3g}TM~X{RZ6e~~c=JK|Ohoh~@JDga=jl`GwvDK*uBRjoX%x;r&uJrNe9UsdU||=xxi#@Md9YBagv7@8 z(mJPQ+x)u|H_X>A$m{+Z&nJyEZxz~H>{)JNO-99-kQJ23q!Z{sT>CXHm#=tR!j`u; zzB2yNZ35|zoeY+7XUm%#w=o#U1mmwm=~>rWl=fEW*71?PFS4}2{n)?+jDNILbM>qZ zS6eTl8!KzwKFak;F$&cNPa*@Y`H{qfB{UGC%6H^+m_G5UzLW%y+ zx{lT{$q8-5OSI)!atQo-5nivYYZiCMM4^CevYi39cq|IaFs+ zxmP>`?lX_Vx_b`hg5JIBLuf!Ls(R#=~hvie6Bpced7L3!i?Ko&sDMHms8E)eQIS% zSzZ=bSrv%ne9Cj{{{YvoMDbU}+pitzdOhvPXS=uk#Ep(h?`7qoOe@8geyto5y4!^S!^sTHl(&$pP*-UM+V9myOK>PNkEeigkw6Ia)C z%k-K{$tOvoZ@jKCfJde_u>7gEo-=J%!q%E+*-f>-p8o(Ws6hY$&JSU@*Ey=+T6lZQ zisin{l1Z^i8Tp%?$J@E~{3~31)jGb?iua}XKIFQb()U)ssj1=@y}0n@{m|X^r-`Ok z{wL~5A5og}O*6&f9}6wiNPg2L*;rN2FDC?kqP>Sr)t=(Q-6t7!+=D!wqu24suP@cU z72J5ERc$KT3z+QSnXW_H#x{(flolif;P*YNMHN~QgVpJOk+dYDoi_Bn&0ic%c$#L9 z<6RaX_?TNVtnfF=%@YhVKd?Xg5vJVyKhk_*t?9lj)8t!pm6h(I+8AVg;&IW3L+jSM zO-j^g7INvg^I2WpGJmAnLnwAw@$#1gp>BG92BYxaouyd8buNK#JX&<@3Z$0>!|fk4 z07=I@b?sf$<7?5ixqnlc&Bi>@dwwYAylLS1CA$}TOl;{e2=GWtH11DPfIvO4aniZ} z02Liy!X7DyPSU21@^}PkcN{T{tk?zOC%7F6_BHNa8`0Oo{vFpmZt|gy1oCWc%Y{d{ zVSsRY5KrcPtIIV#LMy_TmsT<9`i#JfslxpGN#q~%oblSQl2qH}Un8nA_H@&hw>2f6 z)9q13b+!X34{vvX!YAPK*r_0XmCyLY!*g89E$(JpYx`z0E3eF%9OMz$jzJme(zY(r z-d#4{NL%|t1j;g%>Zh>-(~8SkqOh38_h#-W2y0}A_p+k(+E3lv$rvXi(-p|$?kD!5 z{{RDay%nND%#woU-)(*vG@v7F^fEn(w(>utSv>Gy)Y?!U0F<~9dZ0wtJ>=~kxXW`l2hctZJ#YjWmaN41!I%eyx&pQwRW}| zRL~}xCAA_bl^J$P!yUln=Nuoxx*rbfQt5L`Wp5;=8At}+H^%S}Q|vmQ{{U5CR~uTa zp*b#R7kH$+wtDyL(AHHR?cDTRC{o&6h1=&`!6|4MF#vBQZXEO2dJLbfaQ-0i#n;21 z5H6u=kj$}3J{%@${@oE$lU@HOpK4 zf22U_g}XANI3xZ97xk<%^*Z&ICVEcWbXL9o^9!WCY|^01o9 zLG!~XRp@#Z0RI5v!>w|+@mksVKU27}mTB&^^X4u1Y~S zz_Gq1mNO-+Wc}C`vF10swX)SkMO`GYS8|h%3h=N!E-hbK78}M8Y!m>O!;j29ZQr7%NCzJb58wZ80WM7m> z#1)Ykc3?4&e!iHN9wNE%RkpEnbNi?w{{TRS9FXw8M&s$l0q(@(0CUHn&0+jb)1!}7(=BXaw~A?lO(}>1&pQ#haysJ!A5&hP;j07}cJXQf zA`mfYk|rP~dY~tts~%6}2VegH3r9hUSmF3@;7A4JA)U9S-d3u) zoUE7iw^OPUZK`osXCt6|BYzI~S{sde=y-obOu3fp3voF-ossp=Z|G}6+r|>>xUZ){tC*qUJz&dS96r0`BV5;NMaojFx@ zgLg#TMpo`q(mXL~dp3~`+*941L-&^mKJwtK_zo zWQ{ECGJftgA9$jGOA+Wl8usYoRRpP1Y0uMdQ#q;CvV*!qbq!9_;!niuN7E-?u*DIU zS(sod#-wMTb`{M0N!6|9)shPd1L^P`tWmg-oMeAL&beV^h?029m=>D;1n2 z6K|TN1~~)Nscd?EE2Po18#|p9uQhUE((KSnBCJZo!=4k;^oyJMV2VvkNwOQlh4P)9 zPs~^Nh{k^!SZhX|ci5vsw zASFtED$IWYS2~ob`?2?={{TgEwzYRC-R}PY5BMhqqS;Sf1}#uP*czMY1uX!EDSPNZ9m!v}$s<2r&}P(igJ9Ov zEb|3Kjav+Uhx4kwEtGf%Ru)#*aNXTZptYo9Eg9$mtt}4j`ukow-PAX#wlaYtb!3p| z2*SkXHC%91o_O@dcG`}-EpI%=*wBHxRn89IY<^Y8>7EZwI&rDNc`TB| z4)`Ahx#K@U%{yPxE@zeQkg@X-V+zfM$6g24xav1aKYHDaC0eL~LXMoC`M zdHk?X-i}lM04&?z(DWnm$FXUCAD_b>9*=Th3TG(~gfFbDB9~4gT}srw?9#JftYCqUbDW%gYq0R

yKh z6IQ#`^{52%$OXEf+VEgt{&mCaSLQdln&`@sD+oN4ZY3b_J;AK`b%^x&{I!xOuVo*- zxnZ_AN1y<9t)YsHsx?h%>$ws3kc&;~cAB?|;?b?v&g$ijxZy0JKfIXwxvZ@t#hQiQ zryYg0yn1YRPO(QEc-mRG17J9s9XR?O+=HQ~+5)Sp4 z5CH?%7#Qeyt<7iP=AfP&(JeH4%PSpK5v0FssXVzPX2Css1IOiDTk3FYn%(}ba^+#N z1MJa8y-r8WK4^+I861p7Xu0 z_!qCdJw1k{Ese~tAc^-XPZ;xU`u^;=CwFg~kEKE5{{R=-_;EBjR^6u62BQtrEYNwe zAc?{G*SI8f?hS1CQuj&JHT^tmdb;U)b=#en0YrdFw6IXU2qbhoagI8PJY{=rqj;yp zXl`sZ?K0k9Hfx-iQKM`t7~^leLE`|PhXjF>XwOmZ~jLWI7geA>|FSZ z@b5!|&G?hyF%^}btdd&aX?wEPSI)mLm*2{5%**N1wR0LDi!ZcutXM!Jk4$@Xl24Ft zl0ZmB4cGt)!1k>#7F^lO;oC{{_gA%{ z;?%S|%N3sM$s3;5w(mJ*OYj@tUN>F`-_&=>}9)1w#{#M z9Exp^A%JbAc8=#gjaCN`@i9IT_s?V*qElz^?TCFZem7 z!5p_5&DOE0+(yqW#mAD6MQ_By)S9ha)J@9UU;LRUbn%?6~T~%CZ_OQ(H zU0q3?fTSOEMp$II0ATTxiq^9BaBfMz$n|Fmbk{V`GVoj)joycUs9tI5WpiyINUcVl z8_P2|`L;$BF#&Uy^cCqkm6gVy;I)=WV};8s(V@U%+r0-MoM&!(iq-gU;*CGT{xgc( z#3=GO)`YxZ$#cDjLBZQWiQdpm@9W?*sw``mHP z2(Ky*raqi!hq7IDf05-%q-`w^E4I~6ljE-kqBM(lphhnRxlfUu!ve(h1U5+ib$49& zbso8Gs%jBE%sP$Ct#&-;+^r*Wz#gL{erCB(A49EJ-Ct{Q=`U?@sN?K9sdmUkAyg{o zXvpW0{8hiB{?f<5eizd1^=l>6?l~9nhf)h;f>qJ6kQeUp`BznHYEP0+mOCOS!Bk2z zb~)V_;#Q;KyZGjk++AJSe7DdN2$J6ibbyY$aDM~GHR~QH@fM}6=qq8N>WlrFTkRz= zkt}WlZ7ey!B;&WtGAqM04-t4P#hy3RukR$f&@JVV-0Ab!zn}J3C+3CMXx$mf<7glN zJJ-0w;K-rS+8r-ahwKcWXOC=(l51Ji&d8MP!g+NyaY)a1_hUP2M4cy$I_BWF0FMKF0G)Fe=7yTDI;%i ze_FBd=U2M$55%o+OZ#ljE^U`<&nEKhpcM{4=YqpMtI?(LoVrextGqg8y_DLI?j|zZ zr<=WhiClFUC!cZLajh(DF*c!BNm~19{dCmgdP?#!^%(Sz3QaDtGc1yMD-uH)B$mlI z=c&oZ9C6;S{5aJvu5PYH^lNV=;{B#Ll;>#9Fn#fZ_}8BJdd7bd_<*~Ubs~}>g``v& z{uENg4a|7SJxzA8&n1=ao2eMYP8%+&xcN71IUbJ6ZHQ&q8Kw7X3@_?c2qnQ{my6yrO9 zaya1N^%ynkc`QH&2l+Bb{w6l6Ja;5YkRsKt@R? zXasg0ed`Zg)Yo0{WHB_U3=y)y8FmVwlk3})S~~v#i09O2yVP!28;Qws0Ki{B2c=|q zGr+gLFxM|^buSES@I!BIw^q935V5y*l}N_IxRe0Fi3|wjabA5oYd6d%ieC!Re_aid za$J!s&7x|$eb%iAwU%qoF^WM3=kquiAalk5?b@?6PZnr5UL?`v@sEesP_@%zy^T}<3%3(7hhzB*41d_wVdrG2N* zX=|lLaRJk8F0DNH(O5Bm-QbWEj(v9?I#nMNBo=nl>Kdl0rO%>H6#_vVmhtb$1y^D6 z+ZgofE6m zX4UYZ~rmm7-zuWDs_Pk+iQ|oOxG+<dXhZ8!t_r_+k*rR=Cfm%gvar6&FzY*_Hh zY5xEhz6#u21XNc^A(MIB=_((Tk~)*}pM2K?;ypqwD?`#C)U^hCojMP&yzmS;gdD76 zISk3iBzHOOU6s6&Ufk-x$j}Cs%JZ!P~ihgAz9m626IL{dr@hhK|+fJ{im(R@Fn@aK7KIUGZ;`n@PX?3RE z*kezIMDnA!VrG5YhDiVb4glbFHH)G#wYi&52^m=?^KGX4thgB1bJL&)^RC=PRZ+<&j(TIQWZCL7CGm#gw!a%>9CA7zt#=j|aO=8zMRyJ-wwYYIbo{7_V|(M=3V8npAcccCwjc)6pWDLNVq@*(dAJ*5n4k3^=?u2lnm$uWfm9Z|_I%lIBg>IUM@->0O4asx7Ux zzL}*a_#=lVLUcOh-5*FzR#7RXEp-+mpKL`u_mKktW+udmV&6G?Ba&rs&v5fJH2fvX8wn zZdtR(3Ul~ZpXwUUlQgYt1M3qx!HN)e69KmqNbK7!k)CU;jn${4hTd#k_?Zv_naFTry z4APsFj>;yE7}OVQXxl0exg0U|>-DVtTUDP)vv@qje>qnkTH_q|tw|!&^xZ;4wYQKN zn?7XEmW^@3jB)8)hl^!Vd?PXlqsI1?v$b=^c;>nNpR=j#)47iOv= zX}4{$$v<_0;k}5hi!X;-jmE5Jw!eoTFBlp)dZ0$n>k9uaoJ-z+`nD`{3~O^+O%(|qZU_}8&*-# zl#$J6-|2ehnl4r}NThqCFWRS&Bt>^_Cq4Tv2W*P@_|vGAlK7sV=3#WznbW1+wvRpQ zOxyKav{M{;`;enx1MTxM?ULTrdObO0juLd|yR_6#*<_mBmXaKfSgtr6XLr;bck|65 zZKJ=tc{M|;YI9+wkyM05&p=NnpP=AZap8X&Nv6lB>F*gEiF%BaW4{D0>*V zC#z58{{RdNV#cB2>n{nUnzp5?t6kkP+r@5(L~uf&?Ew3YgB?cG>sCBJcWa?p+*xSy z8>}L-ypk-eWCVagz+CbG^cbod#+wJk3yWCfl_HV^EWz?uBhY*1uwjkz zR(~=pYl#9QoyRS_;ACzbU}v6~!Rov_;kz9(!{+)_^EJh31oK>>4-+bq%g@{jGsZas z>0V)@+RvnTiY;%>lgoqUgfRu@lqbwR$6wG_s{BRqzMW&>$r8>Nxzn|~NaeA=7%r~3 z-wflliWuRC-cySDJU$+j;XZw3ZC}gvI-uhVrnT6?)g6;rNg=sG9m!P?qI8kJk04>X zcju*b-YM1Xt^7-+Sl((;Y1aB}v{6fUBY+}iJBj4*PSwHb&2nB0(Gynqk)&EjD?P28 zOMNgcouyZH)9t?)2h>(ah;?}UNvZ1>jcE+5moBX#87~U~h}?HV0QwP-L8-#1(8F`h z-)Q=z{{Z2R6&`8X^wjUXNoQ%V$0o6%+*`W2DRk0E7ADBT%1`%tXB~0xT$Sdpp?F6| zwu46Tt*)taZo)=GCK1W|$fE;+zz4rx^;=8vJ9t{=%T?2(R?wY3%_IcwE6^BNV~hci zdxM%JUz2KWNlTup4^yA5Xa4}gA8DhkFRk9arl8{j!ZIdu zdkuhqKNkEdEk{t(d?BvH-W<5Ocp{7>(p)mDvmQ4E9W&TfE5C+wc+FwFvRN+Vl2Z3~ zQe3X_7#RUno~MD4QyE@f%Co$>K9BC|5saL*c28Y*G_?I@%Urr>qrHL!v^xw>4%P#a zmH>AAYR`wX+gSAri`AGQSlqNDBs=2(u1^D;;0k2^2hn^KR{sE0g697IbBJ_pI5LtH z4rQHzz~mEw-mL5XA&PrV9R`nMpk6w@%Ud~1?m*aG&DGf8^X*ewoUW(aqjzugUzwbw zr1@d79kgXM`-`@bRZDqeJx3WQ>5AF-ec~gk_;+5n@csOeT-v!d_fjZQaxzaNfPP>P zVOZBn@@WW=LmSN^85$OFigVi}fz#C1{{Vrt8%rCzT{0M8ht7qkAp(L=LNY$^;Cl4; zuDl%5nzvr0R!P%uuH`EoH%haYeb%6wyS8MwlF`ePo~&Jl2h`TTg)gskyIb*LuT6EO z7vBZBVjWljOJH%1a0te6$ge2zGkAg>e^&7AoYStDCMrRPA@_2i6&c0|>P>fk9FFHn z(c-tdc4r7xcP7$6z#LW;Dn^Y>wZHYLO70u(bUF>MhxLyNYkD@Z9PrH7{{Y$$$|Kxg zG3_Hb$FJvH_ltE)dz)K*HdtEMMzx7bTe;w-G8w&2Ip(XblXX74CP?nq)t4q^lq^{o z`ElxMzlbfQvAc|2DV3En>md#q3Blu^$kvtTDA4yEt!wvc`xr^{R9we7Yk#8L81RlJ8mC0F3EjYfdOSx!SPi+0)!n&w@YpdF5-a}p7y_&giBhGDsj|H~p89ggw;g`dEa~#X5#`D?h z5Sw@u#9COhmRbJ**Z{8t_4-zt_%lcFb(6leFZ?7L+;Ft3r`s_RBB9c9qHgXcO znYSD^KJE|90balD8q%bdIye5mBUMqUa!Fl&DBrdCd#Gx9MyaS+PaN95o%_!s-Q#HG zfcN0xa8Eho2R*A3O0<6z_>R&Gnbz7(K0?>Fs#Vk?F6SYXh*76!9;c4@p7`qCES~b^u(xq!IOIm8g+KSm&0N-R zMv5#3lM`)r^3qt?m&xO4$0Q%mxam~%xpaG#3FqEUw&Bn+LahG)F~%{TfZ*rapKp0+ zOWQ#A6C!-d*AWQFaCj?*Y>bY%^{yH7$~?{W)VQ}DBD5+;_PujowUSpcT3DtrD)Stf z!42*@^VYh^JUer)Y0{g>?De0Rlxrsu$&TByM_+8$J)n4m4O3A!QZvX*Tl~e&;1rJi zM*^+h_~l}0j*q9@TFB>Pt>v`4RYpGU(aFwv&v9B|oHdPD?fzb$AewJTyBn9yb>aI2 zR<$;IYJkOE4iJIBA&B(q1}df3h-B2fTc+N-pcaaSc-pCsc=}cbrQ=WSNTbs=n6DDe z<`+Bd)tyT(1A3eup68C$Vt)%C5!z2aiLB|@3_f5QP>iS0qhW{QD}G5%U%F1+zpwHo ze(kPib-oq&$eKQ{d8H#I?6+5McGGk77WO@`JJ&Vw3&g2;E7|FL!reYX7x{91Gthl& z3&ODKI!&F`-lI6$%%TZM-X9#0t)4PFR7rg`iXQw@w*~V3Losp{Ln`>}v&_<}R6LkPP zRX_uWasL1UHSJ%qF0FNSeR1LGl z60zJ#1m`Ezo;nQCMRg>jRw->`qX)dcf0^`0?Ee7p{{X{!cfl_P__hre%y^6YQtl&u z@dtKYpp1sdBz7cZb|Q}!YS-G1m8)xaHb&GwDmAH-Uf^@JKN#oN!O8KZ9^Kn@6QFxbc(^amB@pBEsFe08Zt5ke_I z7aRexXrjK(o4p9!QGWTAVWryXI+um+Eg}$0b-m_gC*~^P1CE2QYWhpV`mNYYW%Jmj z)In!P z_pAOV)UJQw9C^Iv^8^5f^vS`ZiuF`u?IkN~{{RHae+qtQMc`;-i^A}c8Zo?rUp&a6 z_hJd09&>^j6y#v>?sD$}?Sn$s;7H>U3n-i^Kz5f!W+0KqPdrgYVVOzBglzi%0FlzP zlSu9LDHeXOwS#)#?~VT10A`m+Cgb_7%l{oLpprINF->OoDs=VGBMcXQAKHz>4lR+=t1!| z5vS<>7@FOrj!UQ}^N*OsFcPxk1mFyI7|nBE4>gO+pBcfXOFKznD$8>vyI`Zq^Bu%@ zvIr+LOT5|RO6F`BokpohSB1qKh6Pdxts5Tc6pAeH94j&70l zJn!Ldr*{X5G_Nk=NW|0ZkSkz1l{^vOrFGvCE#tV>W|~FaDUc1>>=a^(E5w_=6<_&} zRVB)Gz9WJ=drMejP?01+DvpB!yo*&HR`Z$t!Lfw@ZVQ*979cpIW(> zRq@M3cIJ4*nF!rJ^%Pe`KY>rwieDU!kauVotJ$SiIOCDf{N!taURlT(s^<=yWsXSpIVUT;9l(1WbfSv%vARuL%k(;o1j8wAh91=dJiYqxqePKIve~>TYe7{38 z$1*jijWo+hPnB(M@vKp=0Y>iCT6?*q_(bT=Sw+N50qjb)ap^@Br5}ijU&Bw(^sj~y zt)-ud?qVA!8fg9JR^%}}{XqPy#6BS3TxmMQ?`JdHFqSiz&O1NmIO#?E8Zt=y_# z#eXBJ{hn`T)qF{-S>7 z8@Q6wT)&DJF-IKiQB#e)_M(dO^Izaa`WI2?-<$l*uZkCQ$!q5{@|j;ziWY6>lraK9 z{CoAPy5+QE#u{nC{mR80V{yW-lbm9TE2U{o32wa^&})jOIZ&u%>9mZF7mB~3fe(u` z)Ib$PXaEDCF|i}*nkcSnE%Qg!8Bb%Q@ukeR*7j`^lx7W*JNB$;E+h-4%eyiZVo4e9 zXrj7tjr&V^qZ$cvlxLnryhy~16Ut`V4msza&{sDsj`lZlGqX66lru3W7(7u$ME)23 zcQkrP&+#seWOQ+R_H-9b4osthPCB1jpW#$#CDo!bg1&3qKfBV1wWC~Q}|2J z^uHH5(`~MewjnPPmMR8Y>U}+`&ZXe}8sXAMu#8JBnun1IMqpP5BoIz$qPp{D;f0fn zw#4NrOA?dqFVOJ68d=5R>+>F+zS#q9!pO&rbmW?=;oGUDzSY^}W@NWjT=p3qD5AW$ zHt_0CeGZI7QgX684I5s*(`UVs+J7nj%lCpQI62749XRP;+3^AhtiB5PVk>yo7>rR# z<;I~`b3)8c2e~GSE9t0f3ZLOToKDnFE7SDLNN+qb6n4=fUq)rQ!VJpU1Li$(j@>JT zf)%py#<+g(0q;ZxcC=NgaKE{eGmT7w#KTpW-rre<0H6eyLCh~UA0Zt=X4w2*?^WMB!&9W#!AQAKl8{vz}pKjJUzpzR{vE;JY>lXJ_t z#^T2~!1trX`dzk@4wpTuG*a9tX&ZYnV}a^UKME+Wxc#H5%bA-tWXa%rNf?si{Y6Hrf$yu>%&)=+(VFL86N1rvCtZjqv{f#Q7ip*$w{L<^TWy diff --git a/v1_api_demo/model_zoo/resnet/example/image_list_provider.py b/v1_api_demo/model_zoo/resnet/example/image_list_provider.py deleted file mode 100644 index 2cd8eb8bf8..0000000000 --- a/v1_api_demo/model_zoo/resnet/example/image_list_provider.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.utils.image_util import * -from paddle.trainer.PyDataProvider2 import * - - -def hook(settings, image_size, crop_size, color, file_list, is_train, **kwargs): - """ - Description: Init with a list of data file - file_list is the name list of input files. - kwargs["load_data_args"] is the value of 'load_data_args' - which can be set in config. - Each args is separated by a column. - image_size: the crop image size. - mean_meta: the path of the meta file to store the mean image. - mean_value: can be mean value, not a file. - can not set mean_meta and mean_value at the same time. - color: 'color' means a color image. Otherwise, it means a gray image. - is_train: whether the data provider is used for training. - Data argumentation might be different for training and testing. - """ - settings.img_size = image_size - settings.crop_size = crop_size - settings.mean_img_size = settings.crop_size - settings.color = color # default is color - settings.is_train = is_train - - settings.is_swap_channel = kwargs.get('swap_channel', None) - if settings.is_swap_channel is not None: - settings.swap_channel = settings.is_swap_channel - settings.is_swap_channel = True - - if settings.color: - settings.img_input_size = settings.crop_size * settings.crop_size * 3 - else: - settings.img_input_size = settings.crop_size * settings.crop_size - - settings.file_list = file_list - settings.mean_meta = kwargs.get('mean_meta', None) - settings.mean_value = kwargs.get('mean_value', None) - # can not specify both mean_meta and mean_value. - assert not (settings.mean_meta and settings.mean_value) - if not settings.mean_meta: - settings.mean_value = kwargs.get('mean_value') - sz = settings.crop_size * settings.crop_size - settings.img_mean = np.zeros(sz * 3, dtype=np.single) - for idx, value in enumerate(settings.mean_value): - settings.img_mean[idx * sz:(idx + 1) * sz] = value - settings.img_mean = settings.img_mean.reshape(3, settings.crop_size, - settings.crop_size) - - else: - settings.img_mean = load_meta(settings.mean_meta, - settings.mean_img_size, - settings.crop_size, settings.color) - - settings.input_types = [ - dense_vector(settings.img_input_size), # image feature - integer_value(1) - ] # labels - - settings.logger.info('Image short side: %s', settings.img_size) - settings.logger.info('Crop size: %s', settings.crop_size) - settings.logger.info('Meta path: %s', settings.mean_meta) - if settings.is_swap_channel: - settings.logger.info('swap channel: %s', settings.swap_channel) - settings.logger.info('DataProvider Initialization finished') - - -@provider(init_hook=hook, should_shuffle=False) -def processData(settings, file_list): - """ - The main function for loading data. - Load the batch, iterate all the images and labels in this batch. - file_name: the batch file name. - """ - img_path, lab = file_list.strip().split(' ') - img = Image.open(img_path) - img.load() - img = img.resize((settings.img_size, settings.img_size), Image.ANTIALIAS) - img = np.array(img).astype(np.float32) - if len(img.shape) == 3: - img = np.swapaxes(img, 1, 2) - img = np.swapaxes(img, 1, 0) - # swap channel - if settings.is_swap_channel: - img = img[settings.swap_channel, :, :] - img_feat = preprocess_img(img, settings.img_mean, settings.crop_size, - settings.is_train, settings.color) - yield img_feat.tolist(), int(lab.strip()) diff --git a/v1_api_demo/model_zoo/resnet/example/test.list b/v1_api_demo/model_zoo/resnet/example/test.list deleted file mode 100644 index 30bbf630b6..0000000000 --- a/v1_api_demo/model_zoo/resnet/example/test.list +++ /dev/null @@ -1,2 +0,0 @@ -example/dog.jpg 0 -example/cat.jpg 0 diff --git a/v1_api_demo/model_zoo/resnet/extract_fea_c++.sh b/v1_api_demo/model_zoo/resnet/extract_fea_c++.sh deleted file mode 100755 index 5447aa92df..0000000000 --- a/v1_api_demo/model_zoo/resnet/extract_fea_c++.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -#set names of layer which you want to extract feature -#in Outputs() of resnet.py -#like: Outputs("res5_3_branch2c_conv", "res5_3_branch2c_bn") -layer_num=50 -configure=./resnet.py -model_path=./model/resnet_$layer_num -fea_dir=fea_output -#Output is text file. -#Each line is one sample's features. -#If you set N layer names in Outputs() -#each line contains N features sperated by ";". - -# create model list file. -model_list=./model.list -touch $model_list | echo $model_path > $model_list - -paddle train \ - --local=true \ - --job=test \ - --config=$configure \ - --model_list=$model_list \ - --use_gpu=1 \ - --predict_output_dir=$fea_dir \ - --config_args=is_test=1,layer_num=$layer_num diff --git a/v1_api_demo/model_zoo/resnet/extract_fea_py.sh b/v1_api_demo/model_zoo/resnet/extract_fea_py.sh deleted file mode 100755 index 2e87152f7f..0000000000 --- a/v1_api_demo/model_zoo/resnet/extract_fea_py.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -#Note if you use CPU mode, you need to set use_gpu=0 in classify.py. like this: -#conf_args = "is_test=0,use_gpu=1,is_predict=1" -#conf = parse_config(train_conf, conf_args) -#swig_paddle.initPaddle("--use_gpu=0") -python classify.py \ - --job=extract \ - --conf=resnet.py \ - --use_gpu=1 \ - --mean=model/mean_meta_224/mean.meta \ - --model=model/resnet_50 \ - --data=./example/test.list \ - --output_layer="res5_3_branch2c_conv,res5_3_branch2c_bn" \ - --output_dir=features diff --git a/v1_api_demo/model_zoo/resnet/get_model.sh b/v1_api_demo/model_zoo/resnet/get_model.sh deleted file mode 100755 index b33d8178ab..0000000000 --- a/v1_api_demo/model_zoo/resnet/get_model.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR - -mkdir model -cd model - -echo "Downloading ResNet models..." - -for file in resnet_50.tar.gz resnet_101.tar.gz resnet_152.tar.gz mean_meta_224.tar.gz -do - wget http://paddlepaddle.bj.bcebos.com/model_zoo/imagenet/$file - tar -xvf $file - rm $file -done - -echo "Done." diff --git a/v1_api_demo/model_zoo/resnet/load_feature.py b/v1_api_demo/model_zoo/resnet/load_feature.py deleted file mode 100644 index 5d3d0c0d30..0000000000 --- a/v1_api_demo/model_zoo/resnet/load_feature.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import cPickle -import logging - -logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') -logging.getLogger().setLevel(logging.INFO) - - -def load_feature_c(file): - """ - Load feature extracted by C++ interface. - Return a list. - file: feature file. - """ - features = [] - f = open(file, 'r') - for line in f: - sample = [] - for slot in line.strip().split(";"): - fea = [float(val) for val in slot.strip().split()] - if fea: - sample.append(fea) - features.append(sample) - f.close() - return features - - -def load_feature_py(feature_dir): - """ - Load feature extracted by python interface. - Return a dictionary. - feature_dir: directory of feature file. - """ - file_list = os.listdir(feature_dir) - file_list = [os.path.join(feature_dir, f) for f in file_list] - features = {} - for file_name in file_list: - with open(file_name, 'rb') as f: - feature = cPickle.load(f) - features.update(feature) - logging.info('Load feature file %s', file_name) - return features - - -if __name__ == '__main__': - print load_feature_py(sys.argv[1]) - #print load_feature_c(sys.argv[1]) diff --git a/v1_api_demo/model_zoo/resnet/net_diagram.sh b/v1_api_demo/model_zoo/resnet/net_diagram.sh deleted file mode 100755 index 1b06ffa44e..0000000000 --- a/v1_api_demo/model_zoo/resnet/net_diagram.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -:' -Visual deep residual network -1. Using make_model_diagram.py to generate dot file. -2. Using graphviz to convert dot file. - -Usage: -./net_diagram.sh -' - -set -e - -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR - -img_type=png -img_fileprefix=ResNet_50 -conf_filename=resnet.py -dot_filename=ResNet_50.dot -config_str="layer_num=50,data_provider=0" - -python -m paddle.utils.make_model_diagram $conf_filename $dot_filename $config_str - -# If you have installed graphviz, running like this: -# dot -Tpng -o ResNet.png ResNet.dot diff --git a/v1_api_demo/model_zoo/resnet/predict.sh b/v1_api_demo/model_zoo/resnet/predict.sh deleted file mode 100755 index 2b67b17c48..0000000000 --- a/v1_api_demo/model_zoo/resnet/predict.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -python classify.py \ - --job=predict \ - --conf=resnet.py\ - --model=model/resnet_50 \ - --multi_crop \ - --use_gpu=1 \ - --data=./example/test.list diff --git a/v1_api_demo/model_zoo/resnet/resnet.py b/v1_api_demo/model_zoo/resnet/resnet.py deleted file mode 100644 index 6fdd97fefc..0000000000 --- a/v1_api_demo/model_zoo/resnet/resnet.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * -""" -paper: https://arxiv.org/abs/1512.03385 -""" -is_test = get_config_arg("is_test", bool, False) -is_predict = get_config_arg("is_predict", bool, False) -data_provider = get_config_arg("data_provider", bool, True) -layer_num = get_config_arg("layer_num", int, 50) - -if not is_predict and data_provider: - train_list = 'train.list' if not is_test else None - # mean.meta is mean file of ImageNet dataset. - # mean.meta size : 3 x 224 x 224. - # If you use three mean value, set like: - # "mean_value:103.939,116.779,123.68;" - args = { - 'mean_meta': "model/mean_meta_224/mean.meta", - 'image_size': 224, - 'crop_size': 224, - 'color': True, - 'swap_channel:': [2, 1, 0] - } - define_py_data_sources2( - train_list, - 'example/test.list', - module="example.image_list_provider", - obj="processData", - args=args) - -batch_size = 1 -learning_rate = 0.1 / batch_size -momentum = 0.9 -weight_decay = 0.0001 * batch_size -default_momentum(momentum) -default_decay_rate(weight_decay) - -Settings( - algorithm='sgd', - batch_size=batch_size, - learning_rate=learning_rate, - - # set the appropriate parameters according your schedule - learning_method='momentum', - learning_rate_decay_a=0.5, - learning_rate_decay_b=1200000 * 10, - learning_rate_schedule="discexp", ) - - -def conv_bn_layer(name, - input, - filter_size, - num_filters, - stride, - padding, - channels=None, - active_type=ReluActivation()): - """ - A wrapper for conv layer with batch normalization layers. - Note: - conv layer has no activation. - """ - - tmp = img_conv_layer( - name=name + "_conv", - input=input, - filter_size=filter_size, - num_channels=channels, - num_filters=num_filters, - stride=stride, - padding=padding, - act=LinearActivation(), - bias_attr=False) - return batch_norm_layer( - name=name + "_bn", input=tmp, act=active_type, use_global_stats=is_test) - - -def bottleneck_block(name, input, num_filters1, num_filters2): - """ - A wrapper for bottlenect building block in ResNet. - Last conv_bn_layer has no activation. - Addto layer has activation of relu. - """ - last_name = conv_bn_layer( - name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=1, - padding=0) - last_name = conv_bn_layer( - name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - last_name = conv_bn_layer( - name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer( - name=name + "_addto", input=[input, last_name], act=ReluActivation()) - - -def mid_projection(name, input, num_filters1, num_filters2, stride=2): - """ - A wrapper for middile projection in ResNet. - projection shortcuts are used for increasing dimensions, - and other shortcuts are identity - branch1: projection shortcuts are used for increasing - dimensions, has no activation. - branch2x: bottleneck building block, shortcuts are identity. - """ - # stride = 2 - branch1 = conv_bn_layer( - name=name + '_branch1', - input=input, - filter_size=1, - num_filters=num_filters2, - stride=stride, - padding=0, - active_type=LinearActivation()) - - last_name = conv_bn_layer( - name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=stride, - padding=0) - last_name = conv_bn_layer( - name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - - last_name = conv_bn_layer( - name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer( - name=name + "_addto", input=[branch1, last_name], act=ReluActivation()) - - -def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): - """ - A wrapper for 50,101,152 layers of ResNet. - res2_num: number of blocks stacked in conv2_x - res3_num: number of blocks stacked in conv3_x - res4_num: number of blocks stacked in conv4_x - res5_num: number of blocks stacked in conv5_x - """ - # For ImageNet - # conv1: 112x112 - img = data_layer(name='input', size=224 * 224 * 3) - tmp = conv_bn_layer( - "conv1", - img, - filter_size=7, - channels=3, - num_filters=64, - stride=2, - padding=3) - tmp = img_pool_layer(name="pool1", input=tmp, pool_size=3, stride=2) - - # conv2_x: 56x56 - tmp = mid_projection( - name="res2_1", input=tmp, num_filters1=64, num_filters2=256, stride=1) - for i in xrange(2, res2_num + 1, 1): - tmp = bottleneck_block( - name="res2_" + str(i), input=tmp, num_filters1=64, num_filters2=256) - - # conv3_x: 28x28 - tmp = mid_projection( - name="res3_1", input=tmp, num_filters1=128, num_filters2=512) - for i in xrange(2, res3_num + 1, 1): - tmp = bottleneck_block( - name="res3_" + str(i), - input=tmp, - num_filters1=128, - num_filters2=512) - - # conv4_x: 14x14 - tmp = mid_projection( - name="res4_1", input=tmp, num_filters1=256, num_filters2=1024) - for i in xrange(2, res4_num + 1, 1): - tmp = bottleneck_block( - name="res4_" + str(i), - input=tmp, - num_filters1=256, - num_filters2=1024) - - # conv5_x: 7x7 - tmp = mid_projection( - name="res5_1", input=tmp, num_filters1=512, num_filters2=2048) - for i in xrange(2, res5_num + 1, 1): - tmp = bottleneck_block( - name="res5_" + str(i), - input=tmp, - num_filters1=512, - num_filters2=2048) - - tmp = img_pool_layer( - name='avgpool', - input=tmp, - pool_size=7, - stride=1, - pool_type=AvgPooling()) - - output = fc_layer( - name='output', input=tmp, size=1000, act=SoftmaxActivation()) - - if not is_predict: - classification_cost( - input=output, label=data_layer( - name='label', size=1)) - - -def res_net_50(): - deep_res_net(3, 4, 6, 3) - - -def res_net_101(): - deep_res_net(3, 4, 23, 3) - - -def res_net_152(): - deep_res_net(3, 8, 36, 3) - - -if not is_predict: - Inputs("input", "label") -else: - Inputs("input") -# Outputs("cost-softmax" if not is_predict else "output") -Outputs("res5_3_branch2c_conv", "res5_3_branch2c_bn") - -if layer_num == 50: - res_net_50() -elif layer_num == 101: - res_net_101() -elif layer_num == 152: - res_net_152() -else: - print("Wrong layer number.") diff --git a/v1_api_demo/quick_start/.gitignore b/v1_api_demo/quick_start/.gitignore deleted file mode 100644 index f71662563f..0000000000 --- a/v1_api_demo/quick_start/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -*.pyc -data/dict.txt -data/dict_all.txt -data/labels.list -data/mosesdecoder-master/ -data/reviews_Electronics_5.json.gz -data/test.list -data/test.txt -data/train.list -data/train.txt -data/pred.list -data/pred.txt -dataprovider_copy_1.py -train.log -output diff --git a/v1_api_demo/quick_start/api_predict.py b/v1_api_demo/quick_start/api_predict.py deleted file mode 100755 index 9bdffe1006..0000000000 --- a/v1_api_demo/quick_start/api_predict.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os, sys -import numpy as np -from optparse import OptionParser -from py_paddle import swig_paddle, DataProviderConverter -from paddle.trainer.PyDataProvider2 import sparse_binary_vector -from paddle.trainer.config_parser import parse_config -""" -Usage: run following command to show help message. - python api_predict.py -h -""" - - -class QuickStartPrediction(): - def __init__(self, train_conf, dict_file, model_dir=None, label_file=None): - """ - train_conf: trainer configure. - dict_file: word dictionary file name. - model_dir: directory of model. - """ - self.train_conf = train_conf - self.dict_file = dict_file - self.word_dict = {} - self.dict_dim = self.load_dict() - self.model_dir = model_dir - if model_dir is None: - self.model_dir = os.path.dirname(train_conf) - - self.label = None - if label_file is not None: - self.load_label(label_file) - - conf = parse_config(train_conf, "is_predict=1") - self.network = swig_paddle.GradientMachine.createFromConfigProto( - conf.model_config) - self.network.loadParameters(self.model_dir) - input_types = [sparse_binary_vector(self.dict_dim)] - self.converter = DataProviderConverter(input_types) - - def load_dict(self): - """ - Load dictionary from self.dict_file. - """ - for line_count, line in enumerate(open(self.dict_file, 'r')): - self.word_dict[line.strip().split('\t')[0]] = line_count - return len(self.word_dict) - - def load_label(self, label_file): - """ - Load label. - """ - self.label = {} - for v in open(label_file, 'r'): - self.label[int(v.split('\t')[1])] = v.split('\t')[0] - - def get_index(self, data): - """ - transform word into integer index according to the dictionary. - """ - words = data.strip().split() - word_slot = [self.word_dict[w] for w in words if w in self.word_dict] - return word_slot - - def batch_predict(self, data_batch): - input = self.converter(data_batch) - output = self.network.forwardTest(input) - prob = output[0]["id"].tolist() - print("predicting labels is:") - print prob - - -def option_parser(): - usage = "python predict.py -n config -w model_dir -d dictionary -i input_file " - parser = OptionParser(usage="usage: %s [options]" % usage) - parser.add_option( - "-n", - "--tconf", - action="store", - dest="train_conf", - help="network config") - parser.add_option( - "-d", - "--dict", - action="store", - dest="dict_file", - help="dictionary file") - parser.add_option( - "-b", - "--label", - action="store", - dest="label", - default=None, - help="dictionary file") - parser.add_option( - "-c", - "--batch_size", - type="int", - action="store", - dest="batch_size", - default=1, - help="the batch size for prediction") - parser.add_option( - "-w", - "--model", - action="store", - dest="model_path", - default=None, - help="model path") - return parser.parse_args() - - -def main(): - options, args = option_parser() - train_conf = options.train_conf - batch_size = options.batch_size - dict_file = options.dict_file - model_path = options.model_path - label = options.label - swig_paddle.initPaddle("--use_gpu=0") - predict = QuickStartPrediction(train_conf, dict_file, model_path, label) - - batch = [] - labels = [] - for line in sys.stdin: - [label, text] = line.split("\t") - labels.append(int(label)) - batch.append([predict.get_index(text)]) - print("labels is:") - print labels - predict.batch_predict(batch) - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/quick_start/api_predict.sh b/v1_api_demo/quick_start/api_predict.sh deleted file mode 100755 index 4d9aa9e885..0000000000 --- a/v1_api_demo/quick_start/api_predict.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -#Note the default model is pass-00002, you shold make sure the model path -#exists or change the mode path. -#only test on trainer_config.lr.py -model=output/model/pass-00001/ -config=trainer_config.lr.py -label=data/labels.list -dict=data/dict.txt -batch_size=20 -head -n$batch_size data/test.txt | python api_predict.py \ - --tconf=$config\ - --model=$model \ - --label=$label \ - --dict=$dict \ - --batch_size=$batch_size diff --git a/v1_api_demo/quick_start/api_train.py b/v1_api_demo/quick_start/api_train.py deleted file mode 100644 index 5699789daa..0000000000 --- a/v1_api_demo/quick_start/api_train.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import itertools -import random - -from paddle.trainer.config_parser import parse_config -from py_paddle import swig_paddle as api -from py_paddle import DataProviderConverter -from paddle.trainer.PyDataProvider2 \ - import integer_value, integer_value_sequence, sparse_binary_vector - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--train_data", type=str, required=False, help="train data file") - parser.add_argument("--test_data", type=str, help="test data file") - parser.add_argument( - "--config", type=str, required=True, help="config file name") - parser.add_argument("--dict_file", required=True, help="dictionary file") - parser.add_argument( - "--seq", default=1, type=int, help="whether use sequence training") - parser.add_argument( - "--use_gpu", default=0, type=int, help="whether use GPU for training") - parser.add_argument( - "--trainer_count", - default=1, - type=int, - help="Number of threads for training") - parser.add_argument( - "--num_passes", default=5, type=int, help="Number of training passes") - return parser.parse_args() - - -UNK_IDX = 0 - - -def load_data(file_name, word_dict): - with open(file_name, 'r') as f: - for line in f: - label, comment = line.strip().split('\t') - words = comment.split() - word_slot = [word_dict.get(w, UNK_IDX) for w in words] - yield word_slot, int(label) - - -def load_dict(dict_file): - word_dict = dict() - with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - return word_dict - - -def main(): - options = parse_arguments() - api.initPaddle("--use_gpu=%s" % options.use_gpu, - "--trainer_count=%s" % options.trainer_count) - - word_dict = load_dict(options.dict_file) - train_dataset = list(load_data(options.train_data, word_dict)) - if options.test_data: - test_dataset = list(load_data(options.test_data, word_dict)) - else: - test_dataset = None - - trainer_config = parse_config(options.config, - "dict_file=%s" % options.dict_file) - # No need to have data provider for trainer - trainer_config.ClearField('data_config') - trainer_config.ClearField('test_data_config') - - # create a GradientMachine from the model configuratin - model = api.GradientMachine.createFromConfigProto( - trainer_config.model_config) - # create a trainer for the gradient machine - trainer = api.Trainer.create(trainer_config, model) - - # create a data converter which converts data to PaddlePaddle - # internal format - input_types = [ - integer_value_sequence(len(word_dict)) if options.seq else - sparse_binary_vector(len(word_dict)), integer_value(2) - ] - converter = DataProviderConverter(input_types) - - batch_size = trainer_config.opt_config.batch_size - trainer.startTrain() - for train_pass in xrange(options.num_passes): - trainer.startTrainPass() - random.shuffle(train_dataset) - for pos in xrange(0, len(train_dataset), batch_size): - batch = itertools.islice(train_dataset, pos, pos + batch_size) - size = min(batch_size, len(train_dataset) - pos) - trainer.trainOneDataBatch(size, converter(batch)) - trainer.finishTrainPass() - if test_dataset: - trainer.startTestPeriod() - for pos in xrange(0, len(test_dataset), batch_size): - batch = itertools.islice(test_dataset, pos, pos + batch_size) - size = min(batch_size, len(test_dataset) - pos) - trainer.testOneDataBatch(size, converter(batch)) - trainer.finishTestPeriod() - trainer.finishTrain() - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/quick_start/api_train.sh b/v1_api_demo/quick_start/api_train.sh deleted file mode 100755 index 9b2a4e2f22..0000000000 --- a/v1_api_demo/quick_start/api_train.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -# Note: if using trainer_config.emb.py, trainer_config.cnn.py -# or trainer_config.lstm.py, you need to change --seq to --seq=1 -# because they are sequence models. -python api_train.py \ - --config=trainer_config.lr.py \ - --trainer_count=2 \ - --num_passes=15 \ - --use_gpu=0 \ - --seq=0 \ - --train_data=data/train.txt \ - --test_data=data/test.txt \ - --dict_file=data/dict.txt \ - 2>&1 | tee 'train.log' diff --git a/v1_api_demo/quick_start/cluster/cluster_train.sh b/v1_api_demo/quick_start/cluster/cluster_train.sh deleted file mode 100755 index a7b1f01064..0000000000 --- a/v1_api_demo/quick_start/cluster/cluster_train.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -# Should run pserver.sh before run this script. -bin_dir=$(cd `dirname $0`; pwd) -home_dir=$(cd "${bin_dir}/.."; pwd) -source "$bin_dir/env.sh" - -model_dir="$bin_dir/output" -log_file="$bin_dir/train.log" - -pushd "$home_dir" -cfg=trainer_config.lr.py -paddle train \ - --start_pserver=false \ - --config=$cfg \ - --save_dir=${model_dir} \ - --trainer_count=4 \ - --local=0 \ - --log_period=100 \ - --num_passes=15 \ - --use_gpu=false \ - --show_parameter_stats_period=100 \ - --test_all_data_in_one_period=1 \ - --num_gradient_servers=1 \ - --nics=`get_nics` \ - --port=7164 \ - --ports_num=1 \ - --pservers="127.0.0.1" \ - --comment="paddle_trainer" \ - 2>&1 | tee "$log_file" -popd diff --git a/v1_api_demo/quick_start/cluster/env.sh b/v1_api_demo/quick_start/cluster/env.sh deleted file mode 100644 index a404993835..0000000000 --- a/v1_api_demo/quick_start/cluster/env.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -function get_nics() { - machine=`uname -s` - local nics="" - if [ "$machine" == "Linux" ]; then - nics="lo" - elif [ "$machine" == "Darwin" ]; then - nics="lo0" - else - nics="unsupport" - fi - echo $nics -} diff --git a/v1_api_demo/quick_start/cluster/pserver.sh b/v1_api_demo/quick_start/cluster/pserver.sh deleted file mode 100755 index b187c1d9b9..0000000000 --- a/v1_api_demo/quick_start/cluster/pserver.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -bin_dir=$(cd `dirname $0`; pwd) -source "$bin_dir/env.sh" - -paddle pserver \ - --nics=`get_nics` \ - --port=7164 \ - --ports_num=1 \ - --ports_num_for_sparse=1 \ - --num_gradient_servers=1 \ - --comment="paddle_pserver" \ - 2>&1 | tee 'pserver.log' diff --git a/v1_api_demo/quick_start/data/README.md b/v1_api_demo/quick_start/data/README.md deleted file mode 100644 index 63abcf7ebf..0000000000 --- a/v1_api_demo/quick_start/data/README.md +++ /dev/null @@ -1,9 +0,0 @@ -This dataset consists of electronics product reviews associated with -binary labels (positive/negative) for sentiment classification. - -The preprocessed data can be downloaded by script `get_data.sh`. -The data was derived from reviews_Electronics_5.json.gz at - -http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_Electronics_5.json.gz - -If you want to process the raw data, you can use the script `proc_from_raw_data/get_data.sh`. diff --git a/v1_api_demo/quick_start/data/get_data.sh b/v1_api_demo/quick_start/data/get_data.sh deleted file mode 100755 index a09a18f919..0000000000 --- a/v1_api_demo/quick_start/data/get_data.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR - -# Download the preprocessed data -wget http://paddlepaddle.bj.bcebos.com/demo/quick_start_preprocessed_data/preprocessed_data.tar.gz - -# Extract package -tar zxvf preprocessed_data.tar.gz - -# Remove compressed package -rm preprocessed_data.tar.gz diff --git a/v1_api_demo/quick_start/data/proc_from_raw_data/get_data.sh b/v1_api_demo/quick_start/data/proc_from_raw_data/get_data.sh deleted file mode 100755 index d976eaebfa..0000000000 --- a/v1_api_demo/quick_start/data/proc_from_raw_data/get_data.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 1. size of pos : neg = 1:1. -# 2. size of testing set = min(25k, len(all_data) * 0.1), others is traning set. -# 3. distinct train set and test set. - -set -e - -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR - -# Download data -echo "Downloading Amazon Electronics reviews data..." -# http://jmcauley.ucsd.edu/data/amazon/ -wget http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_Electronics_5.json.gz -echo "Downloading mosesdecoder..." -# https://github.com/moses-smt/mosesdecoder -wget https://github.com/moses-smt/mosesdecoder/archive/master.zip - -unzip master.zip -rm master.zip - -################## -# Preprocess data -echo "Preprocess data..." -export LC_ALL=C -UNAME_STR=`uname` - -if [ ${UNAME_STR} == 'Linux' ]; then - SHUF_PROG='shuf' -else - SHUF_PROG='gshuf' -fi - -mkdir -p tmp -python preprocess.py -i reviews_Electronics_5.json.gz -# uniq and shuffle -cd tmp -echo 'Uniq and shuffle...' -cat pos_*|sort|uniq|${SHUF_PROG}> pos.shuffed -cat neg_*|sort|uniq|${SHUF_PROG}> neg.shuffed - -min_len=`sed -n '$=' neg.shuffed` -test_num=$((min_len/10)) -if [ $test_num -gt 12500 ];then - test_num=12500 -fi -train_num=$((min_len-test_num)) - -head -n$train_num pos.shuffed >train.pos -head -n$train_num neg.shuffed >train.neg -tail -n$test_num pos.shuffed >test.pos -tail -n$test_num neg.shuffed >test.neg - -cat train.pos train.neg | ${SHUF_PROG} >../train.txt -cat test.pos test.neg | ${SHUF_PROG} >../test.txt - -cd - -echo 'train.txt' > train.list -echo 'test.txt' > test.list - -# use 30k dict -rm -rf tmp -mv dict.txt dict_all.txt -cat dict_all.txt | head -n 30001 > dict.txt -echo 'Done.' diff --git a/v1_api_demo/quick_start/data/proc_from_raw_data/preprocess.py b/v1_api_demo/quick_start/data/proc_from_raw_data/preprocess.py deleted file mode 100755 index 5706351a21..0000000000 --- a/v1_api_demo/quick_start/data/proc_from_raw_data/preprocess.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# -*- coding: UTF-8 -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -1. Tokenize the words and punctuation -2. pos sample : rating score 5; neg sample: rating score 1-2. - -Usage: - python preprocess.py -i data_file [random seed] -""" - -import sys -import os -import operator -import gzip -from subprocess import Popen, PIPE -from optparse import OptionParser -import json -from multiprocessing import Queue -from multiprocessing import Pool -import multiprocessing - -batch_size = 5000 -word_count = {} -num_tokenize = max(1, - multiprocessing.cpu_count() - 2) # parse + tokenize + save -max_queue_size = 8 -parse_queue = Queue(maxsize=max_queue_size + num_tokenize) -tokenize_queue = Queue(maxsize=max_queue_size + num_tokenize) - - -def create_dict(data): - """ - Create dictionary based on data, and saved in data_dir/dict.txt. - The first line is unk \t -1. - data: list, input data by batch. - """ - for seq in data: - try: - for w in seq.lower().split(): - if w not in word_count: - word_count[w] = 1 - else: - word_count[w] += 1 - except: - sys.stderr.write(seq + "\tERROR\n") - - -def parse(path): - """ - Open .gz file. - """ - sys.stderr.write(path) - g = gzip.open(path, 'r') - for l in g: - yield json.loads(l) - g.close() - - -def tokenize(sentences): - """ - Use tokenizer.perl to tokenize input sentences. - tokenizer.perl is tool of Moses. - sentences : a list of input sentences. - return: a list of processed text. - """ - dir = './mosesdecoder-master/scripts/tokenizer/tokenizer.perl' - if not os.path.exists(dir): - sys.exit( - "The ./mosesdecoder-master/scripts/tokenizer/tokenizer.perl does not exists." - ) - tokenizer_cmd = [dir, '-l', 'en', '-q', '-'] - assert isinstance(sentences, list) - text = "\n".join(sentences) - tokenizer = Popen(tokenizer_cmd, stdin=PIPE, stdout=PIPE) - tok_text, _ = tokenizer.communicate(text) - toks = tok_text.split('\n')[:-1] - return toks - - -def save_data(instance, data_dir, pre_fix, batch_num): - """ - save data by batch - """ - label = ['1' if pre_fix == 'pos' else '0' for i in range(len(instance))] - lines = ['%s\t%s' % (label[i], instance[i]) for i in range(len(label))] - file_name = os.path.join(data_dir, "%s_%s.txt" % (pre_fix, batch_num)) - file(file_name, 'w').write('\n'.join(lines) + '\n') - - -def tokenize_batch(id): - """ - tokenize data by batch - """ - while True: - num_batch, instance, pre_fix = parse_queue.get() - if num_batch == -1: ### parse_queue finished - tokenize_queue.put((-1, None, None)) - sys.stderr.write("Thread %s finish\n" % (id)) - break - tokenize_instance = tokenize(instance) - tokenize_queue.put((num_batch, tokenize_instance, pre_fix)) - sys.stderr.write('.') - - -def save_batch(data_dir, num_tokenize, data_dir_dict): - """ - save data by batch - build dict.txt - """ - token_count = 0 - while True: - num_batch, instance, pre_fix = tokenize_queue.get() - if num_batch == -1: - token_count += 1 - if token_count == num_tokenize: #### tokenize finished. - break - else: - continue - save_data(instance, data_dir, pre_fix, num_batch) - create_dict(instance) ## update dict - - sys.stderr.write("save file finish\n") - f = open(data_dir_dict, 'w') - f.write('%s\t%s\n' % ('unk', '-1')) - for k, v in sorted(word_count.items(), key=operator.itemgetter(1), \ - reverse=True): - f.write('%s\t%s\n' % (k, v)) - f.close() - sys.stderr.write("build dict finish\n") - - -def parse_batch(data, num_tokenize): - """ - parse data by batch - parse -> tokenize -> save - """ - raw_txt = parse(data) - neg, pos = [], [] - count = 0 - sys.stderr.write("extract raw data\n") - for l in raw_txt: - rating = l["overall"] - text = l["reviewText"].lower() # # convert words to lower case - if rating == 5.0 and text: - pos.append(text) - if rating < 3.0 and text: - neg.append(text) - if len(pos) == batch_size or len(neg) == batch_size: - if len(pos) == batch_size: - batch = pos - pre_fix = 'pos' - else: - batch = neg - pre_fix = 'neg' - - parse_queue.put((count, batch, pre_fix)) - count += 1 - if pre_fix == 'pos': - pos = [] - else: - neg = [] - - if len(pos) > 0: - parse_queue.put((count, pos, 'pos')) - count += 1 - if len(neg) > 0: - parse_queue.put((count, neg, 'neg')) - count += 1 - for i in range(num_tokenize): - parse_queue.put((-1, None, None)) #### for tokenize's input finished - sys.stderr.write("parsing finish\n") - - -def option_parser(): - parser = OptionParser(usage="usage: python preprcoess.py "\ - "-i data_path [options]") - parser.add_option( - "-i", "--data", action="store", dest="input", help="Input data path.") - parser.add_option( - "-s", - "--seed", - action="store", - dest="seed", - default=1024, - help="Set random seed.") - return parser.parse_args() - - -def main(): - reload(sys) - sys.setdefaultencoding('utf-8') - options, args = option_parser() - data = options.input - seed = options.seed - data_dir_dict = os.path.join(os.path.dirname(data), 'dict.txt') - data_dir = os.path.join(os.path.dirname(data), 'tmp') - pool = Pool(processes=num_tokenize + 2) - pool.apply_async(parse_batch, args=(data, num_tokenize)) - for i in range(num_tokenize): - pool.apply_async(tokenize_batch, args=(str(i), )) - pool.apply_async(save_batch, args=(data_dir, num_tokenize, data_dir_dict)) - pool.close() - pool.join() - - file(os.path.join(os.path.dirname(data), 'labels.list'), - 'w').write('neg\t0\npos\t1\n') - - -if __name__ == '__main__': - main() diff --git a/v1_api_demo/quick_start/dataprovider_bow.py b/v1_api_demo/quick_start/dataprovider_bow.py deleted file mode 100644 index 2745495586..0000000000 --- a/v1_api_demo/quick_start/dataprovider_bow.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer.PyDataProvider2 import * - -# id of the word not in dictionary -UNK_IDX = 0 - - -# initializer is called by the framework during initialization. -# It allows the user to describe the data types and setup the -# necessary data structure for later use. -# `settings` is an object. initializer need to properly fill settings.input_types. -# initializer can also store other data structures needed to be used at process(). -# In this example, dictionary is stored in settings. -# `dictionay` and `kwargs` are arguments passed from trainer_config.lr.py -def initializer(settings, dictionary, **kwargs): - # Put the word dictionary into settings - settings.word_dict = dictionary - - # setting.input_types specifies what the data types the data provider - # generates. - settings.input_types = { - # The first input is a sparse_binary_vector, - # which means each dimension of the vector is either 0 or 1. It is the - # bag-of-words (BOW) representation of the texts. - 'word': sparse_binary_vector(len(dictionary)), - # The second input is an integer. It represents the category id of the - # sample. 2 means there are two labels in the dataset. - # (1 for positive and 0 for negative) - 'label': integer_value(2) - } - - -# Delaring a data provider. It has an initializer 'data_initialzer'. -# It will cache the generated data of the first pass in memory, so that -# during later pass, no on-the-fly data generation will be needed. -# `setting` is the same object used by initializer() -# `file_name` is the name of a file listed train_list or test_list file given -# to define_py_data_sources2(). See trainer_config.lr.py. -@provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) -def process(settings, file_name): - # Open the input data file. - with open(file_name, 'r') as f: - # Read each line. - for line in f: - # Each line contains the label and text of the comment, separated by \t. - label, comment = line.strip().split('\t') - - # Split the words into a list. - words = comment.split() - - # convert the words into a list of ids by looking them up in word_dict. - word_vector = [settings.word_dict.get(w, UNK_IDX) for w in words] - - # Return the features for the current comment. The first is a list - # of ids representing a 0-1 binary sparse vector of the text, - # the second is the integer id of the label. - yield {'word': word_vector, 'label': int(label)} - - -def predict_initializer(settings, dictionary, **kwargs): - settings.word_dict = dictionary - settings.input_types = {'word': sparse_binary_vector(len(dictionary))} - - -# Declaring a data provider for prediction. The difference with process -# is that label is not generated. -@provider(init_hook=predict_initializer, should_shuffle=False) -def process_predict(settings, file_name): - with open(file_name, 'r') as f: - for line in f: - comment = line.strip().split() - word_vector = [settings.word_dict.get(w, UNK_IDX) for w in comment] - yield {'word': word_vector} diff --git a/v1_api_demo/quick_start/dataprovider_emb.py b/v1_api_demo/quick_start/dataprovider_emb.py deleted file mode 100755 index ddfa3ce9b7..0000000000 --- a/v1_api_demo/quick_start/dataprovider_emb.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer.PyDataProvider2 import * - -UNK_IDX = 0 - - -def initializer(settings, dictionary, **kwargs): - settings.word_dict = dictionary - settings.input_types = { - # Define the type of the first input as sequence of integer. - # The value of the integers range from 0 to len(dictrionary)-1 - 'word': integer_value_sequence(len(dictionary)), - # Define the second input for label id - 'label': integer_value(2) - } - - -@provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) -def process(settings, file_name): - with open(file_name, 'r') as f: - for line in f: - label, comment = line.strip().split('\t') - words = comment.split() - word_slot = [settings.word_dict.get(w, UNK_IDX) for w in words] - yield {'word': word_slot, 'label': int(label)} - - -def predict_initializer(settings, dictionary, **kwargs): - settings.word_dict = dictionary - settings.input_types = {'word': integer_value_sequence(len(dictionary))} - - -@provider(init_hook=predict_initializer, should_shuffle=False) -def process_predict(settings, file_name): - with open(file_name, 'r') as f: - for line in f: - comment = line.strip().split() - word_slot = [settings.word_dict.get(w, UNK_IDX) for w in comment] - yield {'word': word_slot} diff --git a/v1_api_demo/quick_start/predict.sh b/v1_api_demo/quick_start/predict.sh deleted file mode 100755 index e47c2dd01f..0000000000 --- a/v1_api_demo/quick_start/predict.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -cfg=trainer_config.lr.py -#cfg=trainer_config.emb.py -#cfg=trainer_config.cnn.py -#cfg=trainer_config.lstm.py -model="output/pass-00003" -paddle train \ - --config=$cfg \ - --use_gpu=false \ - --job=test \ - --init_model_path=$model \ - --config_args=is_predict=1 \ - --predict_output_dir=. \ -2>&1 | tee 'predict.log' -paddle usage -l 'predict.log' -e $? -n "quick_start_predict_${cfg}" >/dev/null 2>&1 - -mv rank-00000 result.txt diff --git a/v1_api_demo/quick_start/train.sh b/v1_api_demo/quick_start/train.sh deleted file mode 100755 index 01697fed48..0000000000 --- a/v1_api_demo/quick_start/train.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -cfg=trainer_config.lr.py -#cfg=trainer_config.emb.py -#cfg=trainer_config.cnn.py -#cfg=trainer_config.lstm.py -#cfg=trainer_config.bidi-lstm.py -#cfg=trainer_config.db-lstm.py -#cfg=trainer_config.resnet-lstm.py -paddle train \ - --config=$cfg \ - --save_dir=./output \ - --trainer_count=4 \ - --log_period=100 \ - --num_passes=15 \ - --use_gpu=false \ - --show_parameter_stats_period=100 \ - --test_all_data_in_one_period=1 \ - 2>&1 | tee 'train.log' -paddle usage -l "train.log" -e $? -n "quick_start_${cfg}" >/dev/null 2>&1 diff --git a/v1_api_demo/quick_start/trainer_config.bidi-lstm.py b/v1_api_demo/quick_start/trainer_config.bidi-lstm.py deleted file mode 100644 index 3deff4aa00..0000000000 --- a/v1_api_demo/quick_start/trainer_config.bidi-lstm.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -dict_file = "./data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -bias_attr = ParamAttr(initial_std=0., l2_rate=0.) -data = data_layer(name="word", size=len(word_dict)) -emb = embedding_layer(input=data, size=128) - -bi_lstm = bidirectional_lstm(input=emb, size=128) -dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) - -output = fc_layer( - input=dropout, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) - -if is_predict: - maxid = maxid_layer(output) - outputs([maxid, output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) - outputs(cls) diff --git a/v1_api_demo/quick_start/trainer_config.cnn.py b/v1_api_demo/quick_start/trainer_config.cnn.py deleted file mode 100644 index e09e41484d..0000000000 --- a/v1_api_demo/quick_start/trainer_config.cnn.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -dict_file = "./data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -data = data_layer(name="word", size=len(word_dict)) -embedding = embedding_layer(input=data, size=128) -conv = sequence_conv_pool(input=embedding, context_len=3, hidden_size=512) -output = fc_layer(input=conv, size=2, act=SoftmaxActivation()) -if is_predict: - maxid = maxid_layer(output) - outputs([maxid, output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) - outputs(cls) diff --git a/v1_api_demo/quick_start/trainer_config.db-lstm.py b/v1_api_demo/quick_start/trainer_config.db-lstm.py deleted file mode 100644 index fba802b460..0000000000 --- a/v1_api_demo/quick_start/trainer_config.db-lstm.py +++ /dev/null @@ -1,74 +0,0 @@ -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -dict_file = "./data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -bias_attr = ParamAttr(initial_std=0., l2_rate=0.) - -data = data_layer(name="word", size=len(word_dict)) -emb = embedding_layer(input=data, size=128) - -hidden_0 = mixed_layer(size=128, input=[full_matrix_projection(input=emb)]) -lstm_0 = lstmemory(input=hidden_0, layer_attr=ExtraAttr(drop_rate=0.1)) - -input_layers = [hidden_0, lstm_0] - -for i in range(1, 8): - fc = fc_layer(input=input_layers, size=128) - lstm = lstmemory( - input=fc, - layer_attr=ExtraAttr(drop_rate=0.1), - reverse=(i % 2) == 1, ) - input_layers = [fc, lstm] - -lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) - -output = fc_layer( - input=lstm_last, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) - -if is_predict: - maxid = maxid_layer(output) - outputs([maxid, output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) - outputs(cls) diff --git a/v1_api_demo/quick_start/trainer_config.emb.py b/v1_api_demo/quick_start/trainer_config.emb.py deleted file mode 100644 index f69f98ff7f..0000000000 --- a/v1_api_demo/quick_start/trainer_config.emb.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -dict_file = "./data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, learning_rate=2e-3, learning_method=AdamOptimizer()) - -data = data_layer(name="word", size=len(word_dict)) -embedding = embedding_layer(input=data, size=128) -avg = pooling_layer(input=embedding, pooling_type=AvgPooling()) -output = fc_layer(input=avg, size=2, act=SoftmaxActivation()) -if is_predict: - maxid = maxid_layer(output) - outputs([maxid, output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) - outputs(cls) diff --git a/v1_api_demo/quick_start/trainer_config.lr.py b/v1_api_demo/quick_start/trainer_config.lr.py deleted file mode 100644 index b7b694940e..0000000000 --- a/v1_api_demo/quick_start/trainer_config.lr.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -dict_file = get_config_arg('dict_file', str, "./data/dict.txt") -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' - -# define the data sources for the model. -# We need to use different process for training and prediction. -# For training, the input data includes both word IDs and labels. -# For prediction, the input data only includs word Ids. -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_bow", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -# Define the data for text features. The size of the data layer is the number -# of words in the dictionary. -data = data_layer(name="word", size=len(word_dict)) - -# Define a fully connected layer with logistic activation. -# (also called softmax activation). -output = fc_layer(input=data, size=2, act=SoftmaxActivation()) - -if not is_predict: - # For training, we need label and cost - - # define the category id for each example. - # The size of the data layer is the number of labels. - label = data_layer(name="label", size=2) - - # Define cross-entropy classification loss and error. - cls = classification_cost(input=output, label=label) - outputs(cls) -else: - # For prediction, no label is needed. We need to output - # We need to output classification result, and class probabilities. - maxid = maxid_layer(output) - outputs([maxid, output]) diff --git a/v1_api_demo/quick_start/trainer_config.lstm.py b/v1_api_demo/quick_start/trainer_config.lstm.py deleted file mode 100644 index 8967d78807..0000000000 --- a/v1_api_demo/quick_start/trainer_config.lstm.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -dict_file = "./data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -data = data_layer(name="word", size=len(word_dict)) -emb = embedding_layer(input=data, size=128) -lstm = simple_lstm( - input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.25)) -lstm_max = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_max, size=2, act=SoftmaxActivation()) -if is_predict: - maxid = maxid_layer(output) - outputs([maxid, output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) - outputs(cls) diff --git a/v1_api_demo/quick_start/trainer_config.resnet-lstm.py b/v1_api_demo/quick_start/trainer_config.resnet-lstm.py deleted file mode 100644 index 32d0596f25..0000000000 --- a/v1_api_demo/quick_start/trainer_config.resnet-lstm.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -# edit-mode: -*- python -*- - -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This configuration is a demonstration of how to implement the stacked LSTM -with residual connections, i.e. an LSTM layer takes the sum of the hidden states -and inputs of the previous LSTM layer instead of only the hidden states. -This architecture is from: -Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V. Le, Mohammad Norouzi, -Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, -Jeff Klingner, Apurva Shah, Melvin Johnson, Xiaobing Liu, Lukasz Kaiser, -Stephan Gouws, Yoshikiyo Kato, Taku Kudo, Hideto Kazawa, Keith Stevens, -George Kurian, Nishant Patil, Wei Wang, Cliff Young, Jason Smith, Jason Riesa, -Alex Rudnick, Oriol Vinyals, Greg Corrado, Macduff Hughes, Jeffrey Dean. 2016. -Google's Neural Machine Translation System: Bridging the Gap between Human and -Machine Translation. In arXiv https://arxiv.org/pdf/1609.08144v2.pdf -Different from the architecture described in the paper, we use a stack single -direction LSTM layers as the first layer instead of bi-directional LSTM. Also, -since this is a demo code, to reduce computation time, we stacked 4 layers -instead of 8 layers. -""" - -from paddle.trainer_config_helpers import * - -dict_file = "./data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i - -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) - -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -bias_attr = ParamAttr(initial_std=0., l2_rate=0.) - -data = data_layer(name="word", size=len(word_dict)) -emb = embedding_layer(input=data, size=128) -lstm = simple_lstm(input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.1)) - -previous_input, previous_hidden_state = emb, lstm - -for i in range(3): - # The input to the current layer is the sum of the hidden state - # and input of the previous layer. - current_input = addto_layer(input=[previous_input, previous_hidden_state]) - hidden_state = simple_lstm( - input=current_input, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.1)) - previous_input, previous_hidden_state = current_input, hidden_state - -lstm = previous_hidden_state - -lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer( - input=lstm_last, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) - -if is_predict: - maxid = maxid_layer(output) - outputs([maxid, output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) - outputs(cls) diff --git a/v1_api_demo/sequence_tagging/data/get_data.sh b/v1_api_demo/sequence_tagging/data/get_data.sh deleted file mode 100755 index 0cdb394035..0000000000 --- a/v1_api_demo/sequence_tagging/data/get_data.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR - -wget http://www.cnts.ua.ac.be/conll2000/chunking/train.txt.gz -wget http://www.cnts.ua.ac.be/conll2000/chunking/test.txt.gz diff --git a/v1_api_demo/sequence_tagging/data/test.list b/v1_api_demo/sequence_tagging/data/test.list deleted file mode 100644 index 073c0a0c90..0000000000 --- a/v1_api_demo/sequence_tagging/data/test.list +++ /dev/null @@ -1 +0,0 @@ -data/test.txt.gz diff --git a/v1_api_demo/sequence_tagging/data/train.list b/v1_api_demo/sequence_tagging/data/train.list deleted file mode 100644 index 43c24d5f64..0000000000 --- a/v1_api_demo/sequence_tagging/data/train.list +++ /dev/null @@ -1 +0,0 @@ -data/train.txt.gz diff --git a/v1_api_demo/sequence_tagging/dataprovider.py b/v1_api_demo/sequence_tagging/dataprovider.py deleted file mode 100644 index bb4b4465bc..0000000000 --- a/v1_api_demo/sequence_tagging/dataprovider.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer.PyDataProvider2 import * -import gzip -import logging - -logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) -logger = logging.getLogger('paddle') -logger.setLevel(logging.INFO) - -OOV_POLICY_IGNORE = 0 -OOV_POLICY_USE = 1 -OOV_POLICY_ERROR = 2 - -num_original_columns = 3 - -# Feature combination patterns. -# [[-1,0], [0,0]] means previous token at column 0 and current token at -# column 0 are combined as one feature. -patterns = [ - [[-2, 0]], - [[-1, 0]], - [[0, 0]], - [[1, 0]], - [[2, 0]], - [[-1, 0], [0, 0]], - [[0, 0], [1, 0]], - [[-2, 1]], - [[-1, 1]], - [[0, 1]], - [[1, 1]], - [[2, 1]], - [[-2, 1], [-1, 1]], - [[-1, 1], [0, 1]], - [[0, 1], [1, 1]], - [[1, 1], [2, 1]], - [[-2, 1], [-1, 1], [0, 1]], - [[-1, 1], [0, 1], [1, 1]], - [[0, 1], [1, 1], [2, 1]], -] - -dict_label = { - 'B-ADJP': 0, - 'I-ADJP': 1, - 'B-ADVP': 2, - 'I-ADVP': 3, - 'B-CONJP': 4, - 'I-CONJP': 5, - 'B-INTJ': 6, - 'I-INTJ': 7, - 'B-LST': 8, - 'I-LST': 9, - 'B-NP': 10, - 'I-NP': 11, - 'B-PP': 12, - 'I-PP': 13, - 'B-PRT': 14, - 'I-PRT': 15, - 'B-SBAR': 16, - 'I-SBAR': 17, - 'B-UCP': 18, - 'I-UCP': 19, - 'B-VP': 20, - 'I-VP': 21, - 'O': 22 -} - - -def make_features(sequence): - length = len(sequence) - num_features = len(sequence[0]) - - def get_features(pos): - if pos < 0: - return ['#B%s' % -pos] * num_features - if pos >= length: - return ['#E%s' % (pos - length + 1)] * num_features - return sequence[pos] - - for i in xrange(length): - for pattern in patterns: - fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) - sequence[i].append(fname) - - -''' -Source file format: -Each line is for one timestep. The features are separated by space. -An empty line indicates end of a sequence. - -cutoff: a list of numbers. If count of a feature is smaller than this, - it will be ignored. -if oov_policy[i] is OOV_POLICY_USE, id 0 is reserved for OOV features of -i-th column. - -return a list of dict for each column -''' - - -def create_dictionaries(filename, cutoff, oov_policy): - def add_to_dict(sequence, dicts): - num_features = len(dicts) - for features in sequence: - l = len(features) - assert l == num_features, "Wrong number of features " + line - for i in xrange(l): - if features[i] in dicts[i]: - dicts[i][features[i]] += 1 - else: - dicts[i][features[i]] = 1 - - num_features = len(cutoff) - dicts = [] - for i in xrange(num_features): - dicts.append(dict()) - - f = gzip.open(filename, 'rb') - - sequence = [] - - for line in f: - line = line.strip() - if not line: - make_features(sequence) - add_to_dict(sequence, dicts) - sequence = [] - continue - features = line.split(' ') - sequence.append(features) - - for i in xrange(num_features): - dct = dicts[i] - n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 - todo = [] - for k, v in dct.iteritems(): - if v < cutoff[i]: - todo.append(k) - else: - dct[k] = n - n += 1 - - if oov_policy[i] == OOV_POLICY_USE: - # placeholder so that len(dct) will be the number of features - # including OOV - dct['#OOV#'] = 0 - - logger.info('column %d dict size=%d, ignored %d' % (i, n, len(todo))) - for k in todo: - del dct[k] - - f.close() - return dicts - - -def initializer(settings, **xargs): - cutoff = [3, 1, 0] - cutoff += [3] * len(patterns) - oov_policy = [OOV_POLICY_IGNORE, OOV_POLICY_ERROR, OOV_POLICY_ERROR] - oov_policy += [OOV_POLICY_IGNORE] * len(patterns) - dicts = create_dictionaries('data/train.txt.gz', cutoff, oov_policy) - dicts[2] = dict_label - settings.dicts = dicts - settings.oov_policy = oov_policy - input_types = [] - num_features = len(dicts) - for i in xrange(num_original_columns): - input_types.append(integer_sequence(len(dicts[i]))) - logger.info("slot %s size=%s" % (i, len(dicts[i]))) - if patterns: - dim = 0 - for i in xrange(num_original_columns, num_features): - dim += len(dicts[i]) - input_types.append(sparse_binary_vector_sequence(dim)) - logger.info("feature size=%s" % dim) - settings.input_types = input_types - - -''' -if oov_policy[i] == OOV_POLICY_USE, features in i-th column which are not -existed in dicts[i] will be assigned to id 0. -if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist -in dicts[i]. -''' - - -@provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) -def process(settings, filename): - input_file = filename - dicts = settings.dicts - oov_policy = settings.oov_policy - - def gen_sample(sequence): - num_features = len(dicts) - sample = [list() for i in xrange(num_original_columns)] - if patterns: - sample.append([]) - for features in sequence: - assert len(features) == num_features, \ - "Wrong number of features: " + line - for i in xrange(num_original_columns): - id = dicts[i].get(features[i], -1) - if id != -1: - sample[i].append(id) - elif oov_policy[i] == OOV_POLICY_IGNORE: - sample[i].append(0xffffffff) - elif oov_policy[i] == OOV_POLICY_ERROR: - logger.fatal("Unknown token: %s" % features[i]) - else: - sample[i].append(0) - - if patterns: - dim = 0 - vec = [] - for i in xrange(num_original_columns, num_features): - id = dicts[i].get(features[i], -1) - if id != -1: - vec.append(dim + id) - elif oov_policy[i] == OOV_POLICY_IGNORE: - pass - elif oov_policy[i] == OOV_POLICY_ERROR: - logger.fatal("Unknown token: %s" % features[i]) - else: - vec.ids.append(dim + 0) - - dim += len(dicts[i]) - sample[-1].append(vec) - return sample - - num_features = len(dicts) - f = gzip.open(input_file, 'rb') - - num_sequences = 0 - sequence = [] - for line in f: - line = line.strip() - if not line: - make_features(sequence) - yield gen_sample(sequence) - sequence = [] - num_sequences += 1 - continue - features = line.split(' ') - sequence.append(features) - - f.close() - - logger.info("num_sequences=%s" % num_sequences) diff --git a/v1_api_demo/sequence_tagging/linear_crf.py b/v1_api_demo/sequence_tagging/linear_crf.py deleted file mode 100644 index ea012ba1ae..0000000000 --- a/v1_api_demo/sequence_tagging/linear_crf.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -import math - -define_py_data_sources2( - train_list="data/train.list", - test_list="data/test.list", - module="dataprovider", - obj="process") - -batch_size = 1 -settings( - learning_method=MomentumOptimizer(), - batch_size=batch_size, - regularization=L2Regularization(batch_size * 1e-4), - model_average=ModelAverage(0.5), - learning_rate=1e-1, - learning_rate_decay_a=1e-5, - learning_rate_decay_b=0.25, ) - -num_label_types = 23 - - -def get_simd_size(size): - return int(math.ceil(float(size) / 8)) * 8 - - -# Currently, in order to use sparse_update=True, -# the size has to be aligned. -num_label_types = get_simd_size(num_label_types) - -features = data_layer(name="features", size=76328) -word = data_layer(name="word", size=6778) -pos = data_layer(name="pos", size=44) -chunk = data_layer(name="chunk", size=num_label_types) - -crf_input = fc_layer( - input=features, - size=num_label_types, - act=LinearActivation(), - bias_attr=False, - param_attr=ParamAttr( - initial_std=0, sparse_update=True)) - -crf = crf_layer( - input=crf_input, - label=chunk, - param_attr=ParamAttr( - name="crfw", initial_std=0), ) - -crf_decoding = crf_decoding_layer( - size=num_label_types, - input=crf_input, - label=chunk, - param_attr=ParamAttr(name="crfw"), ) - -sum_evaluator( - name="error", - input=crf_decoding, ) - -chunk_evaluator( - name="chunk_f1", - input=crf_decoding, - label=chunk, - chunk_scheme="IOB", - num_chunk_types=11, ) - -inputs(word, pos, chunk, features) -outputs(crf) diff --git a/v1_api_demo/sequence_tagging/readme.md b/v1_api_demo/sequence_tagging/readme.md deleted file mode 100644 index 2e17fffb83..0000000000 --- a/v1_api_demo/sequence_tagging/readme.md +++ /dev/null @@ -1,45 +0,0 @@ -# Sequence Tagging - -This demo is a sequence model for assigning tags to each token in a sentence. The task is described at CONLL2000 Text Chunking task. - -## Download data -```bash -cd demo/sequence_tagging -./data/get_data.sh -``` - -## Train model -```bash -cd demo/sequence_tagging -./train.sh -``` - -## Model description - -We provide two models. One is a linear CRF model (linear_crf.py) with is equivalent to the one at leon.bottou.org/projects/sgd. The second one is a stacked bidirectional RNN and CRF model (rnn_crf.py). -

- - - - - - - - - - - - - - - - - - - - - - -
Model nameNumber of parametersF1 score
linear_crf 1.8M 0.937
rnn_crf 960K 0.941
-
-
diff --git a/v1_api_demo/sequence_tagging/rnn_crf.py b/v1_api_demo/sequence_tagging/rnn_crf.py deleted file mode 100644 index 937a34df10..0000000000 --- a/v1_api_demo/sequence_tagging/rnn_crf.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -import math - -define_py_data_sources2( - train_list="data/train.list", - test_list="data/test.list", - module="dataprovider", - obj="process") - -batch_size = 16 -settings( - learning_method=MomentumOptimizer(), - batch_size=batch_size, - regularization=L2Regularization(batch_size * 1e-5), - model_average=ModelAverage(0.5), - learning_rate=2e-3, - learning_rate_decay_a=5e-7, - learning_rate_decay_b=0.5, ) - -word_dim = 128 -hidden_dim = 128 -with_rnn = True - -initial_std = 1 / math.sqrt(hidden_dim) -param_attr = ParamAttr(initial_std=initial_std) -cpu_layer_attr = ExtraLayerAttribute(device=-1) - -default_device(0) - -num_label_types = 23 - -features = data_layer(name="features", size=76328) -word = data_layer(name="word", size=6778) -pos = data_layer(name="pos", size=44) -chunk = data_layer( - name="chunk", size=num_label_types, layer_attr=cpu_layer_attr) - -emb = embedding_layer( - input=word, size=word_dim, param_attr=ParamAttr(initial_std=0)) - -hidden1 = mixed_layer( - size=hidden_dim, - act=STanhActivation(), - bias_attr=True, - input=[ - full_matrix_projection(emb), table_projection( - pos, param_attr=param_attr) - ]) - -if with_rnn: - rnn1 = recurrent_layer( - act=ReluActivation(), - bias_attr=True, - input=hidden1, - param_attr=ParamAttr(initial_std=0), ) - -hidden2 = mixed_layer( - size=hidden_dim, - act=STanhActivation(), - bias_attr=True, - input=[full_matrix_projection(hidden1)] + - ([full_matrix_projection( - rnn1, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) - -if with_rnn: - rnn2 = recurrent_layer( - reverse=True, - act=ReluActivation(), - bias_attr=True, - input=hidden2, - param_attr=ParamAttr(initial_std=0), ) - -crf_input = mixed_layer( - size=num_label_types, - bias_attr=False, - input=[full_matrix_projection(hidden2), ] + - ([full_matrix_projection( - rnn2, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) - -crf = crf_layer( - input=crf_input, - label=chunk, - param_attr=ParamAttr( - name="crfw", initial_std=0), - layer_attr=cpu_layer_attr, ) - -crf_decoding = crf_decoding_layer( - size=num_label_types, - input=crf_input, - label=chunk, - param_attr=ParamAttr(name="crfw"), - layer_attr=cpu_layer_attr, ) - -sum_evaluator( - name="error", - input=crf_decoding, ) - -chunk_evaluator( - name="chunk_f1", - input=crf_decoding, - label=chunk, - chunk_scheme="IOB", - num_chunk_types=11, ) - -inputs(word, pos, chunk, features) -outputs(crf) diff --git a/v1_api_demo/sequence_tagging/train.sh b/v1_api_demo/sequence_tagging/train.sh deleted file mode 100755 index 37e196c842..0000000000 --- a/v1_api_demo/sequence_tagging/train.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -paddle train \ - --config rnn_crf.py \ - --parallel_nn=1 \ - --use_gpu=1 \ - --dot_period=10 \ - --log_period=1000 \ - --test_period=0 \ - --num_passes=10 \ -2>&1 | tee 'train.log' -paddle usage -l 'train.log' -e $? -n "sequence_tagging_train" >/dev/null 2>&1 diff --git a/v1_api_demo/sequence_tagging/train_linear.sh b/v1_api_demo/sequence_tagging/train_linear.sh deleted file mode 100755 index ad6e2d8ee7..0000000000 --- a/v1_api_demo/sequence_tagging/train_linear.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -paddle train \ - --config linear_crf.py \ - --use_gpu=0 \ - --dot_period=100 \ - --log_period=10000 \ - --test_period=0 \ - --num_passes=10 -2>&1 | tee 'train_linear.log' -paddle usage -l 'train_linear.log' -e $? -n "sequence_tagging_train_linear" >/dev/null 2>&1 diff --git a/v1_api_demo/traffic_prediction/README b/v1_api_demo/traffic_prediction/README deleted file mode 100644 index 4c95188583..0000000000 --- a/v1_api_demo/traffic_prediction/README +++ /dev/null @@ -1,7 +0,0 @@ -run by: -cd ./data -sh get_data.sh -cd .. -sh train.sh -sh predict.sh - diff --git a/v1_api_demo/traffic_prediction/data/get_data.sh b/v1_api_demo/traffic_prediction/data/get_data.sh deleted file mode 100755 index f2fa548d47..0000000000 --- a/v1_api_demo/traffic_prediction/data/get_data.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -set -x - -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR - -#download the dataset -echo "Downloading traffic data..." -wget http://paddlepaddle.cdn.bcebos.com/demo/traffic/traffic_data.tar.gz - -#extract package -echo "Unzipping..." -tar -zxvf traffic_data.tar.gz - -echo "data/speeds.csv" > train.list -echo "data/speeds.csv" > test.list -echo "data/speeds.csv" > pred.list - -echo "Done." diff --git a/v1_api_demo/traffic_prediction/dataprovider.py b/v1_api_demo/traffic_prediction/dataprovider.py deleted file mode 100644 index c7883b6950..0000000000 --- a/v1_api_demo/traffic_prediction/dataprovider.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer.PyDataProvider2 import * -import sys -import numpy as np -TERM_NUM = 24 -FORECASTING_NUM = 24 -LABEL_VALUE_NUM = 4 - - -def initHook(settings, file_list, **kwargs): - """ - Init hook is invoked before process data. It will set obj.slots and store data meta. - - :param settings: global object. It will passed to process routine. - :type obj: object - :param file_list: the meta file object, which passed from trainer_config.py,but unused in this function. - :param kwargs: unused other arguments. - """ - del kwargs #unused - - settings.pool_size = sys.maxint - #Use a time seires of the past as feature. - #Dense_vector's expression form is [float,float,...,float] - settings.input_types = [dense_vector(TERM_NUM)] - #There are next FORECASTING_NUM fragments you need predict. - #Every predicted condition at time point has four states. - for i in range(FORECASTING_NUM): - settings.input_types.append(integer_value(LABEL_VALUE_NUM)) - - -@provider( - init_hook=initHook, cache=CacheType.CACHE_PASS_IN_MEM, should_shuffle=True) -def process(settings, file_name): - with open(file_name) as f: - #abandon fields name - f.next() - for row_num, line in enumerate(f): - speeds = map(int, line.rstrip('\r\n').split(",")[1:]) - # Get the max index. - end_time = len(speeds) - # Scanning and generating samples - for i in range(TERM_NUM, end_time - FORECASTING_NUM): - # For dense slot - pre_spd = map(float, speeds[i - TERM_NUM:i]) - - # Integer value need predicting, values start from 0, so every one minus 1. - fol_spd = [j - 1 for j in speeds[i:i + FORECASTING_NUM]] - - # Predicting label is missing, abandon the sample. - if -1 in fol_spd: - continue - yield [pre_spd] + fol_spd - - -def predict_initHook(settings, file_list, **kwargs): - settings.pool_size = sys.maxint - settings.input_types = [dense_vector(TERM_NUM)] - - -@provider(init_hook=predict_initHook, should_shuffle=False) -def process_predict(settings, file_name): - with open(file_name) as f: - #abandon fields name - f.next() - for row_num, line in enumerate(f): - speeds = map(int, line.rstrip('\r\n').split(",")) - end_time = len(speeds) - pre_spd = map(float, speeds[end_time - TERM_NUM:end_time]) - yield pre_spd diff --git a/v1_api_demo/traffic_prediction/gen_result.py b/v1_api_demo/traffic_prediction/gen_result.py deleted file mode 100644 index 3da70b3031..0000000000 --- a/v1_api_demo/traffic_prediction/gen_result.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -res = [] -with open('./rank-00000') as f: - for line in f: - pred = map(int, line.strip('\r\n;').split(";")) - #raw prediction range from 0 to 3 - res.append([i + 1 for i in pred]) - -file_name = open('./data/pred.list').read().strip('\r\n') - -FORECASTING_NUM = 24 -header = [ - 'id', - '201604200805', - '201604200810', - '201604200815', - '201604200820', - '201604200825', - '201604200830', - '201604200835', - '201604200840', - '201604200845', - '201604200850', - '201604200855', - '201604200900', - '201604200905', - '201604200910', - '201604200915', - '201604200920', - '201604200925', - '201604200930', - '201604200935', - '201604200940', - '201604200945', - '201604200950', - '201604200955', - '201604201000', -] -################### -## To CSV format ## -################### -with open(file_name) as f: - f.next() - print ','.join(header) - for row_num, line in enumerate(f): - fields = line.rstrip('\r\n').split(',') - linkid = fields[0] - print linkid + ',' + ','.join(map(str, res[row_num])) diff --git a/v1_api_demo/traffic_prediction/predict.sh b/v1_api_demo/traffic_prediction/predict.sh deleted file mode 100755 index 2dbd5e8805..0000000000 --- a/v1_api_demo/traffic_prediction/predict.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -cfg=trainer_config.py -# pass choice -model="output/pass-00000" -paddle train \ - --config=$cfg \ - --use_gpu=false \ - --job=test \ - --init_model_path=$model \ - --config_args=is_predict=1 \ - --predict_output_dir=. - -python gen_result.py > result.csv - -rm -rf rank-00000 diff --git a/v1_api_demo/traffic_prediction/train.sh b/v1_api_demo/traffic_prediction/train.sh deleted file mode 100755 index 48dfc5604f..0000000000 --- a/v1_api_demo/traffic_prediction/train.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -cfg=trainer_config.py -paddle train \ - --config=$cfg \ - --save_dir=./output \ - --trainer_count=4 \ - --log_period=1000 \ - --dot_period=10 \ - --num_passes=10 \ - --use_gpu=false \ - --show_parameter_stats_period=3000 \ - 2>&1 | tee 'train.log' diff --git a/v1_api_demo/traffic_prediction/trainer_config.py b/v1_api_demo/traffic_prediction/trainer_config.py deleted file mode 100755 index 52d678624a..0000000000 --- a/v1_api_demo/traffic_prediction/trainer_config.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from paddle.trainer_config_helpers import * - -################################### DATA Configuration ############################################# -is_predict = get_config_arg('is_predict', bool, False) -trn = './data/train.list' if not is_predict else None -tst = './data/test.list' if not is_predict else './data/pred.list' -process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2( - train_list=trn, test_list=tst, module="dataprovider", obj=process) -################################### Parameter Configuaration ####################################### -TERM_NUM = 24 -FORECASTING_NUM = 24 -emb_size = 16 -batch_size = 128 if not is_predict else 1 -settings( - batch_size=batch_size, - learning_rate=1e-3, - learning_method=RMSPropOptimizer()) -################################### Algorithm Configuration ######################################## - -output_label = [] - -link_encode = data_layer(name='link_encode', size=TERM_NUM) -for i in xrange(FORECASTING_NUM): - # Each task share same weight. - link_param = ParamAttr( - name='_link_vec.w', initial_max=1.0, initial_min=-1.0) - link_vec = fc_layer(input=link_encode, size=emb_size, param_attr=link_param) - score = fc_layer(input=link_vec, size=4, act=SoftmaxActivation()) - if is_predict: - maxid = maxid_layer(score) - output_label.append(maxid) - else: - # Multi-task training. - label = data_layer(name='label_%dmin' % ((i + 1) * 5), size=4) - cls = classification_cost( - input=score, name="cost_%dmin" % ((i + 1) * 5), label=label) - output_label.append(cls) -outputs(output_label) diff --git a/v1_api_demo/vae/README.md b/v1_api_demo/vae/README.md deleted file mode 100644 index e55d483b02..0000000000 --- a/v1_api_demo/vae/README.md +++ /dev/null @@ -1,13 +0,0 @@ -#Variational Autoencoder (VAE) - -This demo implements VAE training described in the original paper (https://arxiv.org/abs/1312.6114). - - -In order to run the model, first download the MNIST dataset by running the shell script in ./data. - -Then you can run the command below. The flag --useGpu specifies whether to use gpu for training (0 is cpu, 1 is gpu). - -$python vae_train.py [--use_gpu 1] - -The generated images will be stored in ./samples/ -The corresponding models will be stored in ./params/ diff --git a/v1_api_demo/vae/data/get_mnist_data.sh b/v1_api_demo/vae/data/get_mnist_data.sh deleted file mode 100755 index a77c81bf5a..0000000000 --- a/v1_api_demo/vae/data/get_mnist_data.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env sh -# This script downloads the mnist data and unzips it. -set -e -DIR="$( cd "$(dirname "$0")" ; pwd -P )" -rm -rf "$DIR/mnist_data" -mkdir "$DIR/mnist_data" -cd "$DIR/mnist_data" - -echo "Downloading..." - -for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte -do - if [ ! -e $fname ]; then - wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz - gunzip ${fname}.gz - fi -done diff --git a/v1_api_demo/vae/dataloader.py b/v1_api_demo/vae/dataloader.py deleted file mode 100644 index e9ff95d44f..0000000000 --- a/v1_api_demo/vae/dataloader.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np - - -class MNISTloader(): - def __init__(self, - data_path="./data/mnist_data/", - batch_size=60, - process='train'): - self.batch_size = batch_size - self.data_path = data_path - self._pointer = 0 - self.image_batches = np.array([]) - self.process = process - - def _extract_images(self, filename, n): - f = open(filename, 'rb') - f.read(16) - data = np.fromfile(f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)) - #Mapping data into [-1, 1] - data = data / 255. * 2. - 1 - data_batches = np.split(data, 60000 / self.batch_size, 0) - - f.close() - - return data_batches - - @property - def pointer(self): - return self._pointer - - def load_data(self): - TRAIN_IMAGES = '%s/train-images-idx3-ubyte' % self.data_path - TEST_IMAGES = '%s/t10k-images-idx3-ubyte' % self.data_path - - if self.process == 'train': - self.image_batches = self._extract_images(TRAIN_IMAGES, 60000) - else: - self.image_batches = self._extract_images(TEST_IMAGES, 10000) - - def next_batch(self): - batch = self.image_batches[self._pointer] - self._pointer = (self._pointer + 1) % (60000 / self.batch_size) - return np.array(batch) - - def reset_pointer(self): - self._pointer = 0 diff --git a/v1_api_demo/vae/vae_conf.py b/v1_api_demo/vae/vae_conf.py deleted file mode 100644 index 301dd23793..0000000000 --- a/v1_api_demo/vae/vae_conf.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * -import numpy as np - -is_generating = get_config_arg("is_generating", bool, False) - -settings(batch_size=32, learning_rate=1e-3, learning_method=AdamOptimizer()) - -X_dim = 28 * 28 -h_dim = 128 -z_dim = 100 - - -def reparameterization(mu, logvar): - eps = ParamAttr(initial_mean=0., initial_std=1) - with mixed_layer() as sigma: - sigma += dotmul_projection(layer_math.exp(logvar) * 0.5, param_attr=eps) - return mu + sigma - - -def q_func(X): - """ - xavier initialization - """ - param_attr = ParamAttr( - name='share.w', initial_mean=0., initial_std=1. / np.sqrt(X_dim / 2.)) - mu_param = ParamAttr( - name='mu.w', initial_mean=0., initial_std=1. / np.sqrt(h_dim / 2.)) - logvar_param = ParamAttr( - name='logvar.w', initial_mean=0., initial_std=1. / np.sqrt(h_dim / 2.)) - - bias_attr = ParamAttr(name='share.bias', initial_mean=0., initial_std=0.) - mu_bias = ParamAttr(name='mu.bias', initial_mean=0., initial_std=0.) - logvar_bias = ParamAttr(name='logvar.bias', initial_mean=0., initial_std=0.) - - share_layer = fc_layer( - X, - size=h_dim, - param_attr=param_attr, - bias_attr=bias_attr, - act=ReluActivation()) - - return (fc_layer( - share_layer, - size=z_dim, - param_attr=mu_param, - bias_attr=mu_bias, - act=LinearActivation()), fc_layer( - share_layer, - size=z_dim, - param_attr=logvar_param, - bias_attr=logvar_bias, - act=LinearActivation())) - - -def generator(z): - - hidden_param = ParamAttr( - name='hidden.w', initial_mean=0., initial_std=1. / np.sqrt(z_dim / 2.)) - hidden_bias = ParamAttr(name='hidden.bias', initial_mean=0., initial_std=0.) - prob_param = ParamAttr( - name='prob.w', initial_mean=0., initial_std=1. / np.sqrt(h_dim / 2.)) - prob_bias = ParamAttr(name='prob.bias', initial_mean=0., initial_std=0.) - - hidden_layer = fc_layer( - z, - size=h_dim, - act=ReluActivation(), - param_attr=hidden_param, - bias_attr=hidden_bias) - prob = fc_layer( - hidden_layer, - size=X_dim, - act=SigmoidActivation(), - param_attr=prob_param, - bias_attr=prob_bias) - - return prob - - -def reconstruct_error(prob, X): - cost = multi_binary_label_cross_entropy(input=prob, label=X) - return cost - - -def KL_loss(mu, logvar): - with mixed_layer() as mu_square: - mu_square += dotmul_operator(mu, mu, scale=1.) - - cost = 0.5 * sum_cost(layer_math.exp(logvar) + mu_square - 1. - logvar) - - return cost - - -if not is_generating: - x_batch = data_layer(name='x_batch', size=X_dim) - mu, logvar = q_func(x_batch) - z_samples = reparameterization(mu, logvar) - prob = generator(z_samples) - outputs(reconstruct_error(prob, x_batch) + KL_loss(mu, logvar)) -else: - z_samples = data_layer(name='noise', size=z_dim) - outputs(generator(z_samples)) diff --git a/v1_api_demo/vae/vae_train.py b/v1_api_demo/vae/vae_train.py deleted file mode 100644 index 1babb011c7..0000000000 --- a/v1_api_demo/vae/vae_train.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import random -import numpy as np -import cPickle -import sys, os -from PIL import Image - -from paddle.trainer.config_parser import parse_config -from paddle.trainer.config_parser import logger -import py_paddle.swig_paddle as api -import dataloader -import matplotlib.pyplot as plt - - -def plot_samples(samples): - fig = plt.figure(figsize=(4, 4)) - gs = gridspec.GridSpec(4, 4) - gs.update(wspace=0.05, hspace=0.05) - for i, sample in enumerate(samples): - plt.subplot(gs[i]) - plt.axis('off') - plt.imshow(sample.reshape(28, 28), cmap='Greys_r') - - return fig - - -def CHECK_EQ(a, b): - assert a == b, "a=%s, b=%s" % (a, b) - - -def get_fake_samples(generator_machine, batch_size, noise): - gen_inputs = api.Arguments.createArguments(1) - gen_inputs.setSlotValue(0, api.Matrix.createDenseFromNumpy(noise)) - gen_outputs = api.Arguments.createArguments(0) - generator_machine.forward(gen_inputs, gen_outputs, api.PASS_TEST) - fake_samples = gen_outputs.getSlotValue(0).copyToNumpyMat() - return fake_samples - - -def copy_shared_parameters(src, dst): - ''' - copy the parameters from src to dst - :param src: the source of the parameters - :type src: GradientMachine - :param dst: the destination of the parameters - :type dst: GradientMachine - ''' - src_params = [src.getParameter(i) for i in xrange(src.getParameterSize())] - src_params = dict([(p.getName(), p) for p in src_params]) - - for i in xrange(dst.getParameterSize()): - dst_param = dst.getParameter(i) - src_param = src_params.get(dst_param.getName(), None) - if src_param is None: - continue - src_value = src_param.getBuf(api.PARAMETER_VALUE) - dst_value = dst_param.getBuf(api.PARAMETER_VALUE) - CHECK_EQ(len(src_value), len(dst_value)) - dst_value.copyFrom(src_value) - dst_param.setValueUpdated() - - -def find(iterable, cond): - for item in iterable: - if cond(item): - return item - return None - - -def get_layer_size(model_conf, layer_name): - layer_conf = find(model_conf.layers, lambda x: x.name == layer_name) - assert layer_conf is not None, "Cannot find '%s' layer" % layer_name - return layer_conf.size - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--use_gpu", default="1", help="1 means use gpu for training") - parser.add_argument("--gpu_id", default="0", help="the gpu_id parameter") - args = parser.parse_args() - use_gpu = args.use_gpu - assert use_gpu in ["0", "1"] - - if not os.path.exists("./samples/"): - os.makedirs("./samples/") - - if not os.path.exists("./params/"): - os.makedirs("./params/") - - api.initPaddle('--use_gpu=' + use_gpu, '--dot_period=10', - '--log_period=1000', '--gpu_id=' + args.gpu_id, - '--save_dir=' + "./params/") - - conf = "vae_conf.py" - - trainer_conf = parse_config(conf, "is_generating=False") - gener_conf = parse_config(conf, "is_generating=True") - - batch_size = trainer_conf.opt_config.batch_size - - noise_dim = get_layer_size(gener_conf.model_config, "noise") - - mnist = dataloader.MNISTloader(batch_size=batch_size) - mnist.load_data() - - training_machine = api.GradientMachine.createFromConfigProto( - trainer_conf.model_config) - - generator_machine = api.GradientMachine.createFromConfigProto( - gener_conf.model_config) - - trainer = api.Trainer.create(trainer_conf, training_machine) - - trainer.startTrain() - - for train_pass in xrange(100): - trainer.startTrainPass() - mnist.reset_pointer() - i = 0 - it = 0 - while mnist.pointer != 0 or i == 0: - X = mnist.next_batch().astype('float32') - - inputs = api.Arguments.createArguments(1) - inputs.setSlotValue(0, api.Matrix.createDenseFromNumpy(X)) - - trainer.trainOneDataBatch(batch_size, inputs) - - if it % 1000 == 0: - - outputs = api.Arguments.createArguments(0) - training_machine.forward(inputs, outputs, api.PASS_TEST) - loss = np.mean(outputs.getSlotValue(0).copyToNumpyMat()) - print "\niter: {}".format(str(it).zfill(3)) - print "VAE loss: {}".format(str(loss).zfill(3)) - - #Sync parameters between networks (GradientMachine) at the beginning - copy_shared_parameters(training_machine, generator_machine) - - z_samples = np.random.randn(batch_size, - noise_dim).astype('float32') - samples = get_fake_samples(generator_machine, batch_size, - z_samples) - - #Generating the first 16 images for a picture. - figure = plot_samples(samples[:16]) - plt.savefig( - "./samples/{}_{}.png".format( - str(train_pass).zfill(3), str(i).zfill(3)), - bbox_inches='tight') - plt.close(figure) - i += 1 - it += 1 - - trainer.finishTrainPass() - trainer.finishTrain() - - -if __name__ == '__main__': - main() From 281e93bcbb3e67996f3b7a2f76df1da0071969db Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Tue, 16 Jan 2018 15:15:10 +0800 Subject: [PATCH 031/158] Remove 'top 1' from CPU and GPU kernel 1. Remove 'top 1'(or argmax) from CPU and GPU kernel 2. Add a new test case 3. Refine doc --- ...c_greedy_decode_op.cc => ctc_decode_op.cc} | 44 ++++++----- ...c_greedy_decode_op.cu => ctc_decode_op.cu} | 77 ++++--------------- ...ctc_greedy_decode_op.h => ctc_decode_op.h} | 34 ++++---- .../paddle/v2/fluid/tests/test_ctc_decode.py | 62 +++++++++++++++ .../v2/fluid/tests/test_ctc_greedy_decode.py | 56 -------------- 5 files changed, 118 insertions(+), 155 deletions(-) rename paddle/operators/{ctc_greedy_decode_op.cc => ctc_decode_op.cc} (67%) rename paddle/operators/{ctc_greedy_decode_op.cu => ctc_decode_op.cu} (60%) rename paddle/operators/{ctc_greedy_decode_op.h => ctc_decode_op.h} (75%) create mode 100644 python/paddle/v2/fluid/tests/test_ctc_decode.py delete mode 100644 python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py diff --git a/paddle/operators/ctc_greedy_decode_op.cc b/paddle/operators/ctc_decode_op.cc similarity index 67% rename from paddle/operators/ctc_greedy_decode_op.cc rename to paddle/operators/ctc_decode_op.cc index 3c9b705f7f..b290b11d1d 100644 --- a/paddle/operators/ctc_greedy_decode_op.cc +++ b/paddle/operators/ctc_decode_op.cc @@ -29,14 +29,8 @@ class CTCGreedyDecodeOp : public framework::OperatorWithKernel { auto input_dims = ctx->GetInputDim("Input"); - int sequence_width = - static_cast(framework::product(input_dims) / input_dims[0]); - int blank = ctx->Attrs().Get("blank"); - PADDLE_ENFORCE((blank >= 0) && (blank < sequence_width), - "The value of Attr(blank) should be in interval [0, %d).", - sequence_width); // TODO(wanghaoshuang): it is tricky to set the wrong dimension here. - ctx->SetOutputDim("Output", {input_dims[0], 1}); + ctx->SetOutputDim("Output", input_dims); } protected: @@ -53,25 +47,37 @@ class CTCGreedyDecodeOpMaker : public framework::OpProtoAndCheckerMaker { CTCGreedyDecodeOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Input", - "(LodTensor, default: LoDTensor), the unscaled " - "probabilities of variable-length sequences, which is a 2-D " - "Tensor with LoD information. It's shape is " - "[Lp, num_classes + 1], where Lp is the sum of all input " - "sequences' length and num_classes is the true number of classes " - "(not including the blank label)."); - AddOutput("Output", "(Tensor, default: Tensor), the decode result "); + "(LodTensor, default: LoDTensor), Its shape is " + "[Lp, 1], where Lp is the sum of all input sequences' length."); + AddOutput("Output", "(Tensor, default: Tensor), The decode result."); AddAttr("blank", "(int, default: 0), the blank label setted in Connectionist " - "Temporal Classification (CTC) op, and it is in the " - "half-opened interval [0, num_classes + 1).") + "Temporal Classification (CTC) op.") .SetDefault(0); AddAttr("merge_repeated", "(bool, default: true), whether to " "merge repeated elements between two blanks. ") .SetDefault(true); AddComment(R"DOC( -CTCGreedyDecoder is an implementation of the simple best path decoding -algorithm, selecting at each timestep the most likely class at each timestep. +CTCDecoder is used to merge repeated elements between two blanks +and then delete all blanks in sequence. + +Given: + Input.data = [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, + 6, 0, 0, 7, 7, 7, 0] + Input.dims = {18, 1} + Input.LoD = [[0, 11, 18]] + +And: + blank = 0 + merge_repeated = True + +Then: + Output.data = [1, 2, 4, 4, 5, 6, + 6, 7] + Output.dims = {8, 1} + Output.LoD = [[0, 6, 8]] + )DOC"); } }; @@ -85,4 +91,4 @@ REGISTER_OPERATOR(ctc_greedy_decode, ops::CTCGreedyDecodeOp, paddle::framework::EmptyGradOpMaker); REGISTER_OP_CPU_KERNEL( ctc_greedy_decode, - ops::CTCGreedyDecodeKernel); + ops::CTCGreedyDecodeKernel); diff --git a/paddle/operators/ctc_greedy_decode_op.cu b/paddle/operators/ctc_decode_op.cu similarity index 60% rename from paddle/operators/ctc_greedy_decode_op.cu rename to paddle/operators/ctc_decode_op.cu index 43a78745ac..e9cdad7c26 100644 --- a/paddle/operators/ctc_greedy_decode_op.cu +++ b/paddle/operators/ctc_decode_op.cu @@ -16,62 +16,20 @@ limitations under the License. */ #include #include #include "paddle/operators/ctc_greedy_decode_op.h" -#include "paddle/platform/cuda_helper.h" -#include "paddle/platform/gpu_info.h" namespace paddle { namespace operators { -using platform::PADDLE_CUDA_NUM_THREADS; - -__device__ static float atomicMaxF(float* address, float val) { - int* address_as_i = (int*)address; - int old = *address_as_i, assumed; - do { - assumed = old; - old = ::atomicCAS(address_as_i, assumed, - __float_as_int(::fmaxf(val, __int_as_float(assumed)))); - } while (assumed != old); - return __int_as_float(old); -} - -template -__global__ void ArgmaxCudaKernel(const size_t seq_width, const T* logits, - int* output) { - T local_max_value = 0; - int local_max_index = 0; - __shared__ T max_value; - if (threadIdx.x == 0) { - max_value = 0; - } - __syncthreads(); - - for (int i = threadIdx.x; i < seq_width; i += BlockSize) { - T value = logits[blockIdx.x * seq_width + i]; - if (value > local_max_value) { - local_max_value = value; - local_max_index = i; - } - } - - atomicMaxF(&max_value, local_max_value); - - __syncthreads(); - - if (local_max_value == max_value) { - output[blockIdx.x] = local_max_index; - } -} template -__global__ void MergeAndDelCudaKernel(const int64_t num_token, int* tokens, +__global__ void MergeAndDelCudaKernel(const int64_t num_token, const T* tokens, const size_t num_seq, size_t* lod0, const int blank, const int merge_repeated, - size_t* out_lod0, int* output) { + size_t* out_lod0, T* output) { int ouput_idx = 0; out_lod0[0] = 0; for (int i = 0; i < num_seq; ++i) { - int pre_token = -1; + T pre_token = -1; for (int j = lod0[i]; j < lod0[i + 1]; ++j) { if (tokens[j] != blank && !(merge_repeated && tokens[j] == pre_token)) { output[ouput_idx] = tokens[j]; @@ -89,44 +47,39 @@ class CTCGreedyDecodeOpCUDAKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), "It must use CUDAPlace."); + const size_t level = 0; auto* input = ctx.Input("Input"); auto* output = ctx.Output("Output"); + auto input_lod = framework::ToAbsOffset(input->lod()); + const T* tokens = input->data(); const int64_t num_tokens = input->dims()[0]; - const size_t seq_width = input->numel() / num_tokens; - const T* logits = input->data(); - Tensor tmp; - int* tokens = tmp.mutable_data({num_tokens, 1}, ctx.GetPlace()); - // get argmax - // platform::GpuMemsetAsync(args, 0, sizeof(float), stream); - - auto stream = ctx.cuda_device_context().stream(); - ArgmaxCudaKernel<<< - num_tokens, PADDLE_CUDA_NUM_THREADS, 0, stream>>>(seq_width, logits, - tokens); - - const size_t level = 0; - auto input_lod = framework::ToAbsOffset(input->lod()); const size_t num_seq = input_lod[level].size() - 1; + const int blank = ctx.Attr("blank"); const int merge_repeated = static_cast(ctx.Attr("merge_repeated")); + // prepare a lod to record lod information while merging elements thrust::device_vector dev_out_lod0(input_lod[level].size()); size_t* dev_out_lod0_ptr = thrust::raw_pointer_cast(dev_out_lod0.data()); - int* output_data = - output->mutable_data({num_tokens, 1}, ctx.GetPlace()); + // merge elements and delete blank + T* output_data = output->mutable_data({num_tokens, 1}, ctx.GetPlace()); + + auto stream = ctx.cuda_device_context().stream(); MergeAndDelCudaKernel<<<1, 1, 0, stream>>>( num_tokens, tokens, num_seq, input_lod[level].data(), blank, merge_repeated, dev_out_lod0_ptr, output_data); + // set output lod thrust::host_vector host_out_lod0(dev_out_lod0.begin(), dev_out_lod0.end()); framework::LoD out_lod; out_lod.push_back(host_out_lod0); output->set_lod(out_lod); + // resize output dims output->Resize({static_cast(host_out_lod0.back()), 1}); } }; @@ -135,4 +88,4 @@ class CTCGreedyDecodeOpCUDAKernel : public framework::OpKernel { } // namespace paddle REGISTER_OP_CUDA_KERNEL(ctc_greedy_decode, - paddle::operators::CTCGreedyDecodeOpCUDAKernel); + paddle::operators::CTCGreedyDecodeOpCUDAKernel); diff --git a/paddle/operators/ctc_greedy_decode_op.h b/paddle/operators/ctc_decode_op.h similarity index 75% rename from paddle/operators/ctc_greedy_decode_op.h rename to paddle/operators/ctc_decode_op.h index f12ea6c541..30bb53e157 100644 --- a/paddle/operators/ctc_greedy_decode_op.h +++ b/paddle/operators/ctc_decode_op.h @@ -16,7 +16,6 @@ limitations under the License. */ #include #include "paddle/framework/op_registry.h" -#include "unsupported/Eigen/CXX11/Tensor" namespace paddle { namespace operators { @@ -30,8 +29,9 @@ class CTCGreedyDecodeKernel : public framework::OpKernel { auto* input = ctx.Input("Input"); auto* output = ctx.Output("Output"); const size_t level = 0; - auto input_lod = framework::ToAbsOffset(input->lod()); + + // check input dims and lod auto input_dims = input->dims(); PADDLE_ENFORCE_EQ(input_dims[0], static_cast(input_lod[level].back()), @@ -39,38 +39,36 @@ class CTCGreedyDecodeKernel : public framework::OpKernel { "the sum of all sequences' lengths."); const size_t num_sequences = input_lod[level].size() - 1; - const size_t sequence_width = input->numel() / input_dims[0]; size_t blank = static_cast(ctx.Attr("blank")); bool merge_repeated = ctx.Attr("merge_repeated"); + + // merge repeated tokens and delete blank std::vector> pathes(num_sequences); std::vector output_lod0(1, 0); - const T* input_data = input->data(); - Eigen::Map< - Eigen::Matrix> - input_mat(const_cast(input_data), input->numel() / sequence_width, - sequence_width); - - size_t max_class_idx; - size_t prev_class_idx = -1; for (size_t seq_idx = 0; seq_idx < num_sequences; ++seq_idx) { + T prev_token = -1; for (size_t i = input_lod[level][seq_idx]; i < input_lod[level][seq_idx + 1]; ++i) { - input_mat.row(i).maxCoeff(&max_class_idx); - if (max_class_idx != blank && - !(merge_repeated && max_class_idx == prev_class_idx)) { - pathes[seq_idx].push_back(max_class_idx); + if (input_data[i] != blank && + !(merge_repeated && input_data[i] == prev_token)) { + pathes[seq_idx].push_back(input_data[i]); } - prev_class_idx = max_class_idx; + prev_token = input_data[i]; } output_lod0.push_back(output_lod0.back() + pathes[seq_idx].size()); } + + // set output lod framework::LoD output_lod; output_lod.push_back(output_lod0); output->set_lod(output_lod); - int64_t num_step = static_cast(output_lod0.back()); - int* output_data = output->mutable_data({num_step, 1}, ctx.GetPlace()); + // resize output dims + T* output_data = output->mutable_data( + {static_cast(output_lod0.back()), 1}, ctx.GetPlace()); + + // copy result to output for (int i = 0; i < num_sequences; ++i) { memcpy(output_data + output_lod0[i], pathes[i].data(), sizeof(int) * pathes[i].size()); diff --git a/python/paddle/v2/fluid/tests/test_ctc_decode.py b/python/paddle/v2/fluid/tests/test_ctc_decode.py new file mode 100644 index 0000000000..3b7486cfb9 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_ctc_decode.py @@ -0,0 +1,62 @@ +import sys +import unittest +import numpy as np +from op_test import OpTest +from test_softmax_op import stable_softmax + + +def CTCDecode(input, lod, blank, merge_repeated): + lod0 = lod[0] + result = [] + for i in range(len(lod0) - 1): + prev_token = -1 + for j in range(lod0[i], lod0[i + 1]): + token = input[j][0] + if (token != blank) and not (merge_repeated and + token == prev_token): + result.append(token) + prev_token = token + result = np.array(result).reshape([len(result), 1]).astype("int32") + return result + + +class TestCTCDecodeOp(OpTest): + def config(self): + self.op_type = "ctc_greedy_decode" + self.input_lod = [[0, 11, 18]] + self.blank = 0 + self.merge_repeated = False + self.input = np.array( + [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0]).reshape( + [18, 1]).astype("int32") + + def setUp(self): + self.config() + output = CTCDecode(self.input, self.input_lod, self.blank, + self.merge_repeated) + + self.inputs = {"Input": (self.input, self.input_lod), } + self.outputs = {"Output": output} + self.attrs = { + "blank": self.blank, + "merge_repeated": self.merge_repeated + } + + def test_check_output(self): + self.check_output() + pass + + +class TestCTCDecodeOpCase1(TestCTCDecodeOp): + def config(self): + self.op_type = "ctc_greedy_decode" + self.input_lod = [[0, 11, 18]] + self.blank = 0 + self.merge_repeated = True + self.input = np.array( + [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0]).reshape( + [18, 1]).astype("int32") + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py b/python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py deleted file mode 100644 index 23fceb6dcd..0000000000 --- a/python/paddle/v2/fluid/tests/test_ctc_greedy_decode.py +++ /dev/null @@ -1,56 +0,0 @@ -import sys -import unittest -import numpy as np -from op_test import OpTest -from test_softmax_op import stable_softmax - - -def CTCGreedyDecode(softmax, blank, merge_repeated): - prev_token = -1 - result = [] - for token in np.argmax(softmax, axis=1): - if (token != blank) and not (merge_repeated and token == prev_token): - result.append(token) - return np.array(result).reshape([len(result), 1]) - - -class TestCTCGreedyDecodeOp(OpTest): - def config(self): - self.op_type = "ctc_greedy_decode" - self.batch_size = 4 - self.num_classes = 8 - self.input_lod = [[0, 4, 5, 8, 11]] - self.blank = 7 - self.merge_repeated = True - - def setUp(self): - self.config() - input = np.random.uniform( - 0.1, 1.0, - [self.input_lod[0][-1], self.num_classes]).astype("float32") - softmax = np.apply_along_axis(stable_softmax, 1, input) - output = CTCGreedyDecode(softmax, self.blank, self.merge_repeated) - - self.inputs = {"Input": (softmax, self.input_lod), } - self.outputs = {"Output": output} - self.attrs = { - "blank": self.blank, - "merge_repeated": self.merge_repeated - } - - def test_check_output(self): - self.check_output() - - -class TestCTCGreedyDecodeOpCase1(TestCTCGreedyDecodeOp): - def config(self): - self.op_type = "ctc_greedy_decode" - self.batch_size = 4 - self.num_classes = 1025 - self.input_lod = [[0, 4, 5, 8, 11]] - self.blank = 0 - self.merge_repeated = True - - -if __name__ == "__main__": - unittest.main() From 10dd632659012374f827ae0208c05b0eb5c17fb6 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Tue, 16 Jan 2018 15:56:52 +0800 Subject: [PATCH 032/158] Rename 'ctc_greedy_decode' to 'ctc_decode' --- paddle/operators/ctc_decode_op.cc | 18 ++++++++---------- paddle/operators/ctc_decode_op.cu | 8 ++++---- paddle/operators/ctc_decode_op.h | 2 +- .../paddle/v2/fluid/tests/test_ctc_decode.py | 4 ++-- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/paddle/operators/ctc_decode_op.cc b/paddle/operators/ctc_decode_op.cc index b290b11d1d..480c9ae133 100644 --- a/paddle/operators/ctc_decode_op.cc +++ b/paddle/operators/ctc_decode_op.cc @@ -12,20 +12,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/ctc_greedy_decode_op.h" +#include "paddle/operators/ctc_decode_op.h" namespace paddle { namespace operators { -class CTCGreedyDecodeOp : public framework::OperatorWithKernel { +class CTCDecodeOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Input"), - "Input of CTCGreedyDecodeOp should not be null."); + "Input of CTCDecodeOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Output"), - "Output of CTCGreedyDecodeOp should not be null."); + "Output of CTCDecodeOp should not be null."); auto input_dims = ctx->GetInputDim("Input"); @@ -42,9 +42,9 @@ class CTCGreedyDecodeOp : public framework::OperatorWithKernel { } }; -class CTCGreedyDecodeOpMaker : public framework::OpProtoAndCheckerMaker { +class CTCDecodeOpMaker : public framework::OpProtoAndCheckerMaker { public: - CTCGreedyDecodeOpMaker(OpProto* proto, OpAttrChecker* op_checker) + CTCDecodeOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Input", "(LodTensor, default: LoDTensor), Its shape is " @@ -86,9 +86,7 @@ Then: } // namespace paddle namespace ops = paddle::operators; -REGISTER_OPERATOR(ctc_greedy_decode, ops::CTCGreedyDecodeOp, - ops::CTCGreedyDecodeOpMaker, +REGISTER_OPERATOR(ctc_decode, ops::CTCDecodeOp, ops::CTCDecodeOpMaker, paddle::framework::EmptyGradOpMaker); REGISTER_OP_CPU_KERNEL( - ctc_greedy_decode, - ops::CTCGreedyDecodeKernel); + ctc_decode, ops::CTCDecodeKernel); diff --git a/paddle/operators/ctc_decode_op.cu b/paddle/operators/ctc_decode_op.cu index e9cdad7c26..b10db100f7 100644 --- a/paddle/operators/ctc_decode_op.cu +++ b/paddle/operators/ctc_decode_op.cu @@ -15,7 +15,7 @@ limitations under the License. */ #include #include #include -#include "paddle/operators/ctc_greedy_decode_op.h" +#include "paddle/operators/ctc_decode_op.h" namespace paddle { namespace operators { @@ -42,7 +42,7 @@ __global__ void MergeAndDelCudaKernel(const int64_t num_token, const T* tokens, } template -class CTCGreedyDecodeOpCUDAKernel : public framework::OpKernel { +class CTCDecodeOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), @@ -87,5 +87,5 @@ class CTCGreedyDecodeOpCUDAKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_CUDA_KERNEL(ctc_greedy_decode, - paddle::operators::CTCGreedyDecodeOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(ctc_decode, + paddle::operators::CTCDecodeOpCUDAKernel); diff --git a/paddle/operators/ctc_decode_op.h b/paddle/operators/ctc_decode_op.h index 30bb53e157..bc8dfab9f6 100644 --- a/paddle/operators/ctc_decode_op.h +++ b/paddle/operators/ctc_decode_op.h @@ -23,7 +23,7 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; template -class CTCGreedyDecodeKernel : public framework::OpKernel { +class CTCDecodeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { auto* input = ctx.Input("Input"); diff --git a/python/paddle/v2/fluid/tests/test_ctc_decode.py b/python/paddle/v2/fluid/tests/test_ctc_decode.py index 3b7486cfb9..6e798a8465 100644 --- a/python/paddle/v2/fluid/tests/test_ctc_decode.py +++ b/python/paddle/v2/fluid/tests/test_ctc_decode.py @@ -22,7 +22,7 @@ def CTCDecode(input, lod, blank, merge_repeated): class TestCTCDecodeOp(OpTest): def config(self): - self.op_type = "ctc_greedy_decode" + self.op_type = "ctc_decode" self.input_lod = [[0, 11, 18]] self.blank = 0 self.merge_repeated = False @@ -49,7 +49,7 @@ class TestCTCDecodeOp(OpTest): class TestCTCDecodeOpCase1(TestCTCDecodeOp): def config(self): - self.op_type = "ctc_greedy_decode" + self.op_type = "ctc_decode" self.input_lod = [[0, 11, 18]] self.blank = 0 self.merge_repeated = True From 6497bff901e20615411b0095efb791dfab36acbf Mon Sep 17 00:00:00 2001 From: caoying03 Date: Tue, 16 Jan 2018 19:47:15 +0800 Subject: [PATCH 033/158] add python wrapper for l2 normalize. --- doc/api/v2/fluid/layers.rst | 5 + paddle/operators/clip_op.cc | 4 +- paddle/operators/elementwise_op.h | 32 ++-- paddle/operators/expand_op.cc | 18 +-- .../trainer_config_helpers/evaluators.py | 23 ++- python/paddle/v2/fluid/framework.py | 25 ++-- python/paddle/v2/fluid/layers/io.py | 4 +- python/paddle/v2/fluid/layers/nn.py | 140 +++++++++++++++--- .../fluid/tests/test_normalization_wrapper.py | 95 ++++++++++++ 9 files changed, 278 insertions(+), 68 deletions(-) create mode 100644 python/paddle/v2/fluid/tests/test_normalization_wrapper.py diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 62c154e65d..290d409879 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -493,3 +493,8 @@ swish ------ .. autofunction:: paddle.v2.fluid.layers.swish :noindex: + +l2_normalize +------------ +.. autofunction:: paddle.v2.fluid.layers.l2_normalize + :noindex: diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc index 573bb9c7df..7adb74eab7 100644 --- a/paddle/operators/clip_op.cc +++ b/paddle/operators/clip_op.cc @@ -51,8 +51,8 @@ class ClipOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Clip Operator. -The clip operator limits the value of given input within an interval. The interval is -specified with arguments 'min' and 'max': +The clip operator limits the value of given input within an interval. The +interval is specified with arguments 'min' and 'max': $$ Out = \min(\max(X, min), max) diff --git a/paddle/operators/elementwise_op.h b/paddle/operators/elementwise_op.h index a342595b54..1a0131d8b9 100644 --- a/paddle/operators/elementwise_op.h +++ b/paddle/operators/elementwise_op.h @@ -26,9 +26,9 @@ class ElementwiseOp : public framework::OperatorWithKernel { using Tensor = framework::Tensor; void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("X"), - "Input(X) of elementwise op should not be null"); + "Input(X) of elementwise op should not be null."); PADDLE_ENFORCE(ctx->HasInput("Y"), - "Input(Y) of elementwise op should not be null"); + "Input(Y) of elementwise op should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of elementwise op should not be null."); @@ -45,12 +45,12 @@ class ElementwiseOpMaker : public framework::OpProtoAndCheckerMaker { public: ElementwiseOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "(Tensor) The first input tensor of elementwise op"); - AddInput("Y", "(Tensor) The second input tensor of elementwise op"); - AddOutput("Out", "The output of elementwise op"); + AddInput("X", "(Tensor), The first input tensor of elementwise op."); + AddInput("Y", "(Tensor), The second input tensor of elementwise op."); + AddOutput("Out", "The output of elementwise op."); AddAttr("axis", - "(int, default -1) The starting dimension index " - "for broadcasting Y onto X") + "(int, default -1). The start dimension index " + "for broadcasting Y onto X.") .SetDefault(-1) .EqualGreaterThan(-1); comment_ = R"DOC( @@ -58,19 +58,18 @@ Limited Elementwise {name} Operator. The equation is: -.. math:: - {equation} +$${equation}$$ -X is a tensor of any dimension and the dimensions of tensor Y must be smaller than -or equal to the dimensions of X. +$X$ is a tensor of any dimension and the dimensions of tensor $Y$ must be +smaller than or equal to the dimensions of $X$. There are two cases for this operator: -1. The shape of Y is same with X; -2. The shape of Y is a subset of X. +1. The shape of $Y$ is same with $X$; +2. The shape of $Y$ is a subset of $X$. For case 2: -Y will be broadcasted to match the shape of X and axis should be -the starting dimension index for broadcasting Y onto X. +$Y$ will be broadcasted to match the shape of $X$ and axis should be +set to index of the start dimension to broadcast $Y$ onto $X$. For example .. code-block:: python @@ -81,7 +80,8 @@ For example shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 -Either of the inputs X and Y or none can carry the LoD (Level of Details) information. However, the output only shares the LoD information with input X. +Either of the inputs $X$ and $Y$ or none can carry the LoD (Level of Details) +information. However, the output only shares the LoD information with input $X$. )DOC"; AddComment(comment_); diff --git a/paddle/operators/expand_op.cc b/paddle/operators/expand_op.cc index 08fa91ed72..043c93654d 100644 --- a/paddle/operators/expand_op.cc +++ b/paddle/operators/expand_op.cc @@ -58,21 +58,21 @@ class ExpandOpMaker : public framework::OpProtoAndCheckerMaker { ExpandOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", - "(Tensor, default Tensor) A tensor with rank in [1, 6]." - "X is the input tensor to be expanded."); + "(Tensor, default Tensor). A tensor with rank in [1, 6]." + "X is the input to be expanded."); AddOutput("Out", - "(Tensor, default Tensor) A tensor with rank in [1, 6]." - "The rank of Output(Out) is same as Input(X) except that each " - "dimension size of Output(Out) is equal to corresponding " - "dimension size of Input(X) multiplying corresponding value of " - "Attr(expand_times)."); + "(Tensor, default Tensor). A tensor with rank in [1, 6]." + "The rank of Output(Out) have the same with Input(X). " + "After expanding, size of each dimension of Output(Out) is equal " + "to size of the corresponding dimension of Input(X) multiplying " + "the corresponding value given by Attr(expand_times)."); AddAttr>("expand_times", "Expand times number for each dimension."); AddComment(R"DOC( Expand operator tiles the input by given times number. You should set times number for each dimension by providing attribute 'expand_times'. The rank of X -should be in [1, 6]. Please notice that size of 'expand_times' must be same with -X's rank. Following is a using case: +should be in [1, 6]. Please note that size of 'expand_times' must be the same +with X's rank. Following is a using case: Input(X) is a 3-D tensor with shape [2, 3, 1]: diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index 95797fba8f..0eeaf7eabb 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -16,13 +16,22 @@ from paddle.trainer.config_parser import * from default_decorators import * __all__ = [ - "evaluator_base", "classification_error_evaluator", "auc_evaluator", - "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator", - "chunk_evaluator", "sum_evaluator", "column_sum_evaluator", - "value_printer_evaluator", "gradient_printer_evaluator", - "maxid_printer_evaluator", "maxframe_printer_evaluator", - "seqtext_printer_evaluator", "classification_error_printer_evaluator", - "detection_map_evaluator" + "evaluator_base", + "classification_error_evaluator", + "auc_evaluator", + "pnpair_evaluator", + "precision_recall_evaluator", + "ctc_error_evaluator", + "chunk_evaluator", + "sum_evaluator", + "column_sum_evaluator", + "value_printer_evaluator", + "gradient_printer_evaluator", + "maxid_printer_evaluator", + "maxframe_printer_evaluator", + "seqtext_printer_evaluator", + "classification_error_printer_evaluator", + "detection_map_evaluator", ] diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 8042febfed..4f8366b640 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -116,8 +116,8 @@ def _debug_string_(proto, throw_on_error=True): """ error_fields = list() if not proto.IsInitialized(error_fields) and throw_on_error: - raise ValueError("{0} are not initialized\nThe message is {1}".format( - error_fields, proto)) + raise ValueError("{0} are not initialized.\nThe message is {1}:\n". + format(error_fields, proto)) return proto.__str__() @@ -374,12 +374,13 @@ class Operator(object): >>> outputs={"Out": [var1]}) Args: - block(Block): The block has the current operator - desc(core.OpDesc): The protobuf description + block(Block): The block has the current operator. + desc(core.OpDesc): The protobuf description. type(str): The type of operator. inputs(dict): The input dictionary. Key is the input parameter name. Value is a list of variables. - outputs(dict): The output dictionary. Has same format with inputs + outputs(dict): The output dictionary which has the same format with + inputs. attrs(dict): The attributes dictionary. Key is attribute name. Value is the attribute value. The attribute type should be as same as the type registered in C++ @@ -436,10 +437,11 @@ class Operator(object): for m in proto.outputs: need.add(m.name) if not given == need: - raise ValueError( - "Incorrect setting for output(s) of operator \"%s\". Need: [%s] Given: [%s]" - % (type, ", ".join(str(e) for e in need), ", ".join( - str(e) for e in given))) + raise ValueError(("Incorrect setting for output(s) of " + "operator \"%s\". Need: [%s] Given: [%s]") % + (type, ", ".join(str(e) + for e in need), ", ".join( + str(e) for e in given))) for out_proto in proto.outputs: out_args = outputs[out_proto.name] @@ -818,9 +820,8 @@ class Program(object): if isinstance(t, Variable): t = t.op else: - raise ValueError( - "All targets of prune() can only be Variable or Operator." - ) + raise ValueError(("All targets of prune() can only be " + "Variable or Operator.")) targets_idx.append([t.block.idx, t.idx]) res = Program() diff --git a/python/paddle/v2/fluid/layers/io.py b/python/paddle/v2/fluid/layers/io.py index 6177f0b4d7..a43e0ee4de 100644 --- a/python/paddle/v2/fluid/layers/io.py +++ b/python/paddle/v2/fluid/layers/io.py @@ -28,9 +28,9 @@ def data(name, **Data Layer** This function takes in the input and based on whether data has - to be returned back as a minibatch, it creates the global variable using + to be returned back as a minibatch, it creates the global variable by using the helper functions. The global variables can be accessed by all the - following operations and layers in the graph. + following operators in the graph. All the input variables of this function are passed in as local variables to the LayerHelper constructor. diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 4e8fd407c9..cfa60d2924 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -50,6 +50,7 @@ __all__ = [ 'sequence_last_step', 'dropout', 'split', + 'l2_normalize', ] @@ -945,7 +946,8 @@ def pool2d(input, pool_type, pool_stride=None, pool_padding=None, - global_pooling=False): + global_pooling=False, + name=None): """ This function adds the operator for pooling in 2 dimensions, using the pooling configurations mentioned in input parameters. @@ -991,7 +993,8 @@ def batch_norm(input, epsilon=1e-05, param_attr=None, bias_attr=None, - data_layout='NCHW'): + data_layout='NCHW', + name=None): """ This function helps create an operator to implement the BatchNorm layer using the configurations from the input parameters. @@ -1067,7 +1070,7 @@ def batch_norm(input, return helper.append_activation(batch_norm_out) -def beam_search_decode(ids, scores): +def beam_search_decode(ids, scores, name=None): helper = LayerHelper('beam_search_decode', **locals()) sentence_ids = helper.create_tmp_variable(dtype=ids.dtype) sentence_scores = helper.create_tmp_variable(dtype=ids.dtype) @@ -1091,7 +1094,8 @@ def conv2d_transpose(input, padding=None, stride=None, dilation=None, - param_attr=None): + param_attr=None, + name=None): """ The transpose of conv2d layer. @@ -1118,8 +1122,8 @@ def conv2d_transpose(input, contain two integers, (dilation_H, dilation_W). Otherwise, the dilation_H = dilation_W = dilation. param_attr: Parameter Attribute. - main_program(Program): the main program - startup_program(Program): the startup program + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: Output image. @@ -1183,7 +1187,7 @@ def conv2d_transpose(input, return out -def sequence_expand(x, y): +def sequence_expand(x, y, name=None): """Sequence Expand Layer. This layer will expand the input variable **x** according to LoD information of **y**. And the following examples will explain how sequence_expand works: @@ -1227,6 +1231,8 @@ def sequence_expand(x, y): Args: x (Variable): The input variable which is a Tensor or LoDTensor. y (Variable): The input variable which is a LoDTensor. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: The expanded variable which is a LoDTensor. @@ -1253,7 +1259,8 @@ def lstm_unit(x_t, cell_t_prev, forget_bias=0.0, param_attr=None, - bias_attr=None): + bias_attr=None, + name=None): """Lstm unit layer. The equation of a lstm step is: .. math:: @@ -1300,6 +1307,8 @@ def lstm_unit(x_t, initializer, name etc. bias_attr (ParamAttr): The attributes of bias weights, if not False, bias weights will be created and be set to default value. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: tuple: The hidden value and cell value of lstm unit. @@ -1365,7 +1374,7 @@ def lstm_unit(x_t, return h, c -def reduce_sum(input, dim=None, keep_dim=False): +def reduce_sum(input, dim=None, keep_dim=False, name=None): """ Computes the sum of tensor elements over the given dimension. @@ -1379,6 +1388,8 @@ def reduce_sum(input, dim=None, keep_dim=False): keep_dim (bool): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: The reduced Tensor variable. @@ -1409,7 +1420,7 @@ def reduce_sum(input, dim=None, keep_dim=False): return out -def reduce_mean(input, dim=None, keep_dim=False): +def reduce_mean(input, dim=None, keep_dim=False, name=None): """ Computes the mean of tensor elements over the given dimension. @@ -1423,6 +1434,8 @@ def reduce_mean(input, dim=None, keep_dim=False): keep_dim (bool): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: The reduced Tensor variable. @@ -1453,7 +1466,7 @@ def reduce_mean(input, dim=None, keep_dim=False): return out -def reduce_max(input, dim=None, keep_dim=False): +def reduce_max(input, dim=None, keep_dim=False, name=None): """ Computes the maximum of tensor elements over the given dimension. @@ -1467,6 +1480,8 @@ def reduce_max(input, dim=None, keep_dim=False): keep_dim (bool): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: The reduced Tensor variable. @@ -1497,7 +1512,7 @@ def reduce_max(input, dim=None, keep_dim=False): return out -def reduce_min(input, dim=None, keep_dim=False): +def reduce_min(input, dim=None, keep_dim=False, name=None): """ Computes the minimum of tensor elements over the given dimension. @@ -1511,6 +1526,8 @@ def reduce_min(input, dim=None, keep_dim=False): keep_dim (bool): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: Variable: The reduced Tensor variable. @@ -1541,20 +1558,22 @@ def reduce_min(input, dim=None, keep_dim=False): return out -def split(input, num_or_sections, dim=-1): +def split(input, num_or_sections, dim=-1, name=None): """ - Splits the tensor into multiple sub-tensors. + Split the input tensor into multiple sub-tensors. Args: input (Variable): The input variable which is a Tensor or LoDTensor. - num_or_sections (int|list): If :attr:`num_or_sections` is an integer, - then the integer indicates the number of equal sized sub-tensors - that the tensor will be divided into. If :attr:`num_or_sections` - is a list of integers, the length of list indicates the number of - sub-tensors and the integers indicate the sizes of sub-tensors' + num_or_sections (int|list): If :attr:`num_or_sections` is an integer, + then the integer indicates the number of equal sized sub-tensors + that the tensor will be divided into. If :attr:`num_or_sections` + is a list of integers, the length of list indicates the number of + sub-tensors and the integers indicate the sizes of sub-tensors' :attr:`dim` dimension orderly. - dim (int): The dimension along which to split. If :math:`dim < 0`, the + dim (int): The dimension along which to split. If :math:`dim < 0`, the dimension to split along is :math:`rank(input) + dim`. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: List: The list of segmented tensor variables. @@ -1597,3 +1616,84 @@ def split(input, num_or_sections, dim=-1): 'axis': dim }) return outs + + +def l2_normalize(x, axis, epsilon=1e-12, name=None): + """ + **L2 normalize Layer** + + The l2 normalize layer normalizes `x` along dimension `axis` using an L2 + norm. For a 1-D tensor (`dim` is fixed to 0), this layer computes + + output = x / sqrt(max(sum(x**2), epsilon)) + + For `x` with more dimensions, this layer independently normalizes each 1-D + slice along dimension `axis`. + + Args: + x(Variable|list): The input tensor to l2_normalize layer. + axis(int): Dimension along which to normalize the input. + epsilon(float): A lower bound value for `x`'s l2 norm. sqrt(epsilon) will + be used as the divisor if the l2 norm of `x` is less than + sqrt(epsilon). + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + + Returns: + Variable: The output tensor variable. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name="data", + shape=(3, 17, 13), + dtype="float32") + fc = fluid.layers.l2_normalize(x=data, axis=1) + """ + + if len(x.shape) == 1: axis = 0 + + helper = LayerHelper("l2_normalize", **locals()) + + square = helper.create_tmp_variable(dtype=x.dtype) + helper.append_op(type="square", inputs={"X": x}, outputs={"Out": square}) + + reduced_sum = helper.create_tmp_variable(dtype=x.dtype) + helper.append_op( + type="reduce_sum", + inputs={"X": square}, + outputs={"Out": reduced_sum}, + attrs={ + "dim": 1 if axis is None else axis, + "keep_dim": True, + "reduce_all": False + }) + + # TODO(caoying) A lower bound value epsilon for the norm is needed to + # imporve the numeric stability of reciprocal. This requires a maximum_op. + rsquare = helper.create_tmp_variable(dtype=x.dtype) + helper.append_op( + type="reciprocal", inputs={"X": reduced_sum}, outputs={"Out": rsquare}) + + # TODO(caoying) the current elementwise_mul operator does not support a + # general broadcast rule which broadcasts input(Y) to have the same + # dimension with Input(X) starting from a specified dimension. So this + # exanpsion is requred. Once a general broadcast relu is spported, this + # expanding canbe removed. + rsquare_expanded = helper.create_tmp_variable(dtype=x.dtype) + expand_times = [1] * len(x.shape) + expand_times[axis] = int(x.shape[axis]) + helper.append_op( + type="expand", + inputs={"X": rsquare}, + outputs={"Out": rsquare_expanded}, + attrs={"expand_times": expand_times}) + + out = helper.create_tmp_variable(dtype=x.dtype) + helper.append_op( + type="elementwise_mul", + inputs={"X": x, + "Y": rsquare_expanded}, + outputs={"Out": out}) + return out diff --git a/python/paddle/v2/fluid/tests/test_normalization_wrapper.py b/python/paddle/v2/fluid/tests/test_normalization_wrapper.py new file mode 100644 index 0000000000..caff63011d --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_normalization_wrapper.py @@ -0,0 +1,95 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. +import unittest +import paddle.v2.fluid as fluid +import paddle.v2.fluid.core as core +import numpy as np + + +class TestNormalization(unittest.TestCase): + data_desc = {"name": "input", "shape": (2, 3, 7)} + + def gen_random_input(self): + """Generate random input data. + """ + self.data = np.random.random( + size=self.data_desc["shape"]).astype("float32") + + def set_program(self, axis, epsilon): + """Build the test program. + """ + data = fluid.layers.data( + name=self.data_desc["name"], + shape=self.data_desc["shape"], + dtype="float32", + append_batch_size=False) + data.stop_gradient = False + l2_norm = fluid.layers.l2_normalize(x=data, axis=axis, epsilon=epsilon) + out = fluid.layers.reduce_sum(l2_norm, dim=None) + + fluid.backward.append_backward(loss=out) + self.fetch_list = [l2_norm] + + def run_program(self): + """Run the test program. + """ + places = [core.CPUPlace()] + if core.is_compile_gpu(): + places.append(core.CUDAPlace(0)) + + for place in places: + self.set_inputs(place) + exe = fluid.Executor(place) + + output = exe.run(fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=True) + self.op_output = output + + def set_inputs(self, place): + """Set the randomly generated data to the test program. + """ + self.inputs = {} + tensor = fluid.Tensor() + tensor.set(self.data, place) + self.inputs[self.data_desc["name"]] = tensor + + def l2_normalize(self, data, axis, epsilon): + """ Compute the groundtruth. + """ + output = data * np.reciprocal( + np.sum(np.square(data), axis=axis, keepdims=True)) + return output + + def test_l2_normalize(self): + """ Test the python wrapper for l2_normalize. + """ + axis = 1 + #TODO(caoying) epsilon is not supported due to lack of a maximum_op. + epsilon = 1e-6 + + self.gen_random_input() + + self.set_program(axis, epsilon) + self.run_program() + + expect_output = self.l2_normalize(self.data, axis, epsilon) + + # check output + self.assertTrue(np.allclose(self.op_output, expect_output, atol=0.001)) + + +if __name__ == '__main__': + unittest.main() From aab4cfeb65646692caf89248fff5582fc641dccf Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 16 Jan 2018 15:37:15 +0000 Subject: [PATCH 034/158] Add doc for dynamic_lstm python api --- paddle/operators/lstm_op.cc | 2 +- python/paddle/v2/fluid/layers/nn.py | 100 ++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/paddle/operators/lstm_op.cc b/paddle/operators/lstm_op.cc index 3b90b64b4e..afb095a04e 100644 --- a/paddle/operators/lstm_op.cc +++ b/paddle/operators/lstm_op.cc @@ -117,7 +117,7 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("C0", "(Tensor, optional) the initial cell state is an optional " "input. This is a tensor with shape (N x D), where N is the " - "batch size. `H0` and `C0` can be NULL but only at the same time") + "batch size. `H0` and `C0` can be NULL but only at the same time.") .AsDispensable(); AddInput("Weight", "(Tensor) the learnable hidden-hidden weights." diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 4e8fd407c9..7759ce6af6 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -227,6 +227,106 @@ def dynamic_lstm(input, cell_activation='tanh', candidate_activation='tanh', dtype='float32'): + """ + **Dynamic LSTM Layer** + + The defalut implementation is diagonal/peephole connection + (https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: + + .. math: + + i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) \\ + + f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) \\ + + \tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) \\ + + o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) \\ + + c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\ + + h_t = o_t \odot act_h(c_t) + + where the W terms denote weight matrices (e.g. $W_{xi}$ is the matrix + of weights from the input gate to the input), $W_{ic}, W_{fc}, W_{oc}$ + are diagonal weight matrices for peephole connections. In our implementation, + we use vectors to reprenset these diagonal weight matrices. The b terms + denote bias vectors ($b_i$ is the input gate bias vector), $\sigma$ + is the non-line activations, such as logistic sigmoid function, and + $i, f, o$ and $c$ are the input gate, forget gate, output gate, + and cell activation vectors, respectively, all of which have the same size as + the cell output activation vector $h$. + + The $\odot$ is the element-wise product of the vectors. $act_g$ and $act_h$ + are the cell input and cell output activation functions and `tanh` is usually + used for them. $\tilde{c_t}$ is also called candidate hidden state, + which is computed based on the current input and the previous hidden state. + + Set `use_peepholes` False to disable peephole connection. The formula + is omitted here, please refer to the paper + http://www.bioinf.jku.at/publications/older/2604.pdf for details. + + Note that these $W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}$ + operations on the input $x_{t}$ are NOT included in this operator. + Users can choose to use fully-connect operator before LSTM operator. + + Args: +def dynamic_lstm(input, + size, + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + dtype='float32'): + input(Variable): The input of dynamic_lstm layer, which support + variable-time length input sequence. The underlying tensor in + this Variable is a matrix with shape (T X 4D), where T is the + total time steps in this mini-batch, D is the hidden size. + size(int): The size of input. + param_attr(ParamAttr): The parameter attribute for the learnable + hidden-hidden weights. + - The shape is (D x 4D), where D is the hidden size. + - param_attr = {W_ch, W_ih, W_fh, W_oh} + bias_attr(ParamAttr): The bias attribute for the learnable bias + weights, which contains two parts: input-hidden bias weight + and peephole connections weight if setting `use_peepholes` to True. + 1. `use_peepholes = False` + - The shape is (1 x 4D). + - Bias = {b_c, b_i, b_f, b_o}. + 2. `use_peepholes = True` + - The shape is (1 x 7D). + - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}. + use_peepholes(bool, defalut: True): whether to enable diagonal/peephole + connections. + is_reverse(bool, defalut: False): whether to compute reversed LSTM. + gate_activation(string, choices: "sigmoid", "tanh", "relu", "identity", + default: "sigmoid"): The activation for input gate, forget gate and + output gate. + cell_activation(string, choices: "sigmoid", "tanh", "relu", "identity", + default: "tanh"): The activation for cell output. + candidate_activation(string, choices: "sigmoid", "tanh", "relu", + "identity", default: "tanh"): The activation for candidate hidden + state. + dtype(string, ) + + Returns: + hidden(Variable): the hidden state of LSTM layer. The shape is (T x D), + and lod is the same with the `input`. + cell(Variable): the cell state of LSTM layer. The shape is (T x D), and + lod is the same with the `input`. + + Example: + .. code-block:: python + + hidden_dim = 512 + forward_proj = fluid.layers.fc(input=input_seq, size=hidden_dim * 4, + act='tanh', bias_attr=True) + forward, _ = fluid.layers.dynamic_lstm( + input=forward_proj, size=hidden_dim * 4, use_peepholes=False) + """ helper = LayerHelper('lstm', **locals()) size = size / 4 weight = helper.create_parameter( From adcfde3eab274b76029f3efb13ca9b3627273e7a Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Wed, 17 Jan 2018 10:08:41 +0800 Subject: [PATCH 035/158] Modify unitest --- python/paddle/v2/fluid/tests/test_ctc_decode.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/tests/test_ctc_decode.py b/python/paddle/v2/fluid/tests/test_ctc_decode.py index 6e798a8465..1efacab4b3 100644 --- a/python/paddle/v2/fluid/tests/test_ctc_decode.py +++ b/python/paddle/v2/fluid/tests/test_ctc_decode.py @@ -50,12 +50,12 @@ class TestCTCDecodeOp(OpTest): class TestCTCDecodeOpCase1(TestCTCDecodeOp): def config(self): self.op_type = "ctc_decode" - self.input_lod = [[0, 11, 18]] + self.input_lod = [[0, 11, 19]] self.blank = 0 self.merge_repeated = True self.input = np.array( - [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0]).reshape( - [18, 1]).astype("int32") + [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0, 0]).reshape( + [19, 1]).astype("int32") if __name__ == "__main__": From 24f528a1a56e099fc9ebad146614855e3481531e Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 17 Jan 2018 10:47:20 +0800 Subject: [PATCH 036/158] follow comments --- paddle/operators/conv_op.cc | 4 +--- paddle/operators/conv_transpose_op.cc | 4 +--- paddle/operators/pool_op.cc | 4 +--- python/paddle/v2/fluid/nets.py | 7 +++---- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index 9ae3e87281..63e4018555 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -70,9 +70,7 @@ void ConvOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - if (paddle::platform::is_cpu_place(ctx.GetPlace())) { - use_cudnn = false; - } + use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index 46f79b1701..4145638c74 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -61,9 +61,7 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvTransposeOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - if (paddle::platform::is_cpu_place(ctx.GetPlace())) { - use_cudnn = false; - } + use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 648a1dfb56..ebe7d9a0a5 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -64,9 +64,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { framework::OpKernelType PoolOp::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - if (paddle::platform::is_cpu_place(ctx.GetPlace())) { - use_cudnn = false; - } + use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index 327d2ff2da..440467e0ab 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -41,10 +41,9 @@ def img_conv_group(input, param_attr=None, conv_with_batchnorm=False, conv_batchnorm_drop_rate=None, - conv_use_cudnn=True, pool_stride=1, pool_type=None, - pool_use_cudnn=True): + use_cudnn=True): """ Image Convolution Group, Used for vgg net. """ @@ -76,7 +75,7 @@ def img_conv_group(input, padding=conv_padding[i], param_attr=param_attr[i], act=local_conv_act, - use_cudnn=conv_use_cudnn) + use_cudnn=use_cudnn) if conv_with_batchnorm[i]: tmp = layers.batch_norm(input=tmp, act=conv_act) @@ -89,7 +88,7 @@ def img_conv_group(input, pool_size=pool_size, pool_type=pool_type, pool_stride=pool_stride, - use_cudnn=pool_use_cudnn) + use_cudnn=use_cudnn) return pool_out From 3aa5886bdd9654b002084b02c09d4698d3d376fc Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 12:31:01 +0800 Subject: [PATCH 037/158] update_error_clip_doc --- doc/design/error_clip.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/doc/design/error_clip.md b/doc/design/error_clip.md index 8e845462cc..58aa73b8cd 100644 --- a/doc/design/error_clip.md +++ b/doc/design/error_clip.md @@ -46,12 +46,12 @@ class ErrorClipByValue(BaseErrorClipAttr): self.min = min def append_clip_op(self, block, grad_name): - block.append_op( - type="clip", - inputs={"X": grad_name}, - outputs={"Out": grad_name}, - attrs={"min": self.min, - "max": self.max}) + clip_op_desc = block.desc.append_op() + clip_op_desc.set_type("clip") + clip_op_desc.set_input("X", [grad_name]) + clip_op_desc.set_output("Out", [grad_name]) + clip_op_desc.set_attr("min", self.min) + clip_op_desc.set_attr("max", self.max) ``` The `BaseErrorClipAttr` have one main member functions: `append_clip_op(self, block, grad_name)`. @@ -80,6 +80,11 @@ def error_clip_callback(block, context): op_desc.output_arg_names()): fwd_var = block.var_recursive(grad_to_var[grad_n]) error_clip = getattr(fwd_var, "error_clip", None) + if not (error_clip is None or isinstance(error_clip, + BaseErrorClipAttr)): + raise TypeError( + "Variable's error_clip should be an instance of BaseErrorClipAttr or None." + ) if error_clip is not None: error_clip.append_clip_op(block, grad_n) ``` From e35cee59e8e15e72ed3d913d7c1c721623afb9f4 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 17 Jan 2018 13:15:07 +0800 Subject: [PATCH 038/158] Fix a typo --- doc/howto/usage/capi/organization_of_the_inputs_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/usage/capi/organization_of_the_inputs_cn.md b/doc/howto/usage/capi/organization_of_the_inputs_cn.md index 563ec5ca21..a889ae4ffa 100644 --- a/doc/howto/usage/capi/organization_of_the_inputs_cn.md +++ b/doc/howto/usage/capi/organization_of_the_inputs_cn.md @@ -19,7 +19,7 @@ ### 基本使用概念 -- 在PaddlePaddle内部,神经网络中一个计算层的输入/输出被组织为一个 `Argument` 结构体,如果神经网络有多个输入或者多个输入,每一个输入/输入都会对应有自己的`Argument`。 +- 在PaddlePaddle内部,神经网络中一个计算层的输入/输出被组织为一个 `Argument` 结构体,如果神经网络有多个输入或者多个输出,每一个输入/输出都会对应有自己的`Argument`。 - `Argument` 并不真正“存储”数据,而是将输入/输出信息有机地组织在一起。 - 在`Argument`内部由`IVector`(对应着上文提到的一维整型数组)和`Matrix`(对应着上文提到的二维浮点型矩阵)来实际存储数据;由 `Sequence Start Positions` (下文详细解释) 来描述输入/输出的序列信息。 From 5d226743eb7ec82bdac2c20d4b34ce83a493955e Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 17 Jan 2018 14:54:48 +0800 Subject: [PATCH 039/158] fix mkldnn concat dimension issue for rnn --- paddle/gserver/layers/MKLDNNConcatLayer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/gserver/layers/MKLDNNConcatLayer.cpp b/paddle/gserver/layers/MKLDNNConcatLayer.cpp index 44bb0883b8..520ccc1a99 100644 --- a/paddle/gserver/layers/MKLDNNConcatLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConcatLayer.cpp @@ -43,7 +43,7 @@ void MKLDNNConcatLayer::reshape( channels_[0] = ic; oc = ic; for (size_t i = 1; i < inputLayers_.size(); i++) { - int batchsize, height, witdh; + int batchsize = 0, height = 0, witdh = 0; reshapeInput(batchsize, height, witdh, i); CHECK_EQ(bs, batchsize); CHECK_EQ(ih, height); @@ -84,6 +84,7 @@ void MKLDNNConcatLayer::resetFwdBuffers(std::vector& inputs, bool has8c = false, has16c = false, hasnc = false; for (size_t i = 0; i < inputs.size(); i++) { resetInValue(inputs[i], nullptr, i, channels_[i]); + inputs[i]->downSpatial(); CHECK(inputs[i]); auto dm = inputs[i]->getDims(); // inputs format can be different, but ndims must equal From ed7e74abf2f3a41238e5813d1e8fde998e924ef8 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 17 Jan 2018 14:58:49 +0800 Subject: [PATCH 040/158] follow comments and refine python doc --- paddle/operators/conv_op.cc | 4 +--- paddle/operators/conv_transpose_op.cc | 4 +--- paddle/operators/pool_op.cc | 4 +--- python/paddle/v2/fluid/layers/nn.py | 6 ++++-- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index 63e4018555..0e8dddd7f1 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -284,9 +284,7 @@ void ConvOpGrad::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - if (paddle::platform::is_cpu_place(ctx.GetPlace())) { - use_cudnn = false; - } + use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index 4145638c74..f71838c2aa 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -264,9 +264,7 @@ void ConvTransposeOpGrad::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType ConvTransposeOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - if (paddle::platform::is_cpu_place(ctx.GetPlace())) { - use_cudnn = false; - } + use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index ebe7d9a0a5..a450279451 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -89,9 +89,7 @@ void PoolOpGrad::InferShape(framework::InferShapeContext *ctx) const { framework::OpKernelType PoolOpGrad::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); - if (paddle::platform::is_cpu_place(ctx.GetPlace())) { - use_cudnn = false; - } + use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 0cfa011036..251a1535d8 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -724,6 +724,8 @@ def conv2d(input, connected to the second half of the input channels. Default: groups=1 param_attr(ParamAttr): The parameters to the Conv2d Layer. Default: None bias_attr(ParamAttr): Bias parameter for the Conv2d layer. Default: None + use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True act(str): Activation type. Default: None Returns: @@ -1115,8 +1117,8 @@ def conv2d_transpose(input, contain two integers, (dilation_H, dilation_W). Otherwise, the dilation_H = dilation_W = dilation. param_attr: Parameter Attribute. - main_program(Program): the main program - startup_program(Program): the startup program + use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True Returns: Variable: Output image. From 14f6fa346bfd57205dd465f8d2a602bd107af1c4 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 15:27:16 +0800 Subject: [PATCH 041/158] make elementwise op support scalar as input Y --- paddle/operators/elementwise_div_op.h | 7 ++++- paddle/operators/elementwise_mul_op.h | 7 ++++- paddle/operators/elementwise_op_function.h | 14 ++++++++++ paddle/operators/elementwise_sub_op.h | 7 ++++- .../v2/fluid/tests/test_elementwise_add_op.py | 26 +++++++++++++------ .../v2/fluid/tests/test_elementwise_div_op.py | 26 +++++++++++++------ .../v2/fluid/tests/test_elementwise_max_op.py | 22 ++++++++++++++++ .../v2/fluid/tests/test_elementwise_min_op.py | 22 ++++++++++++++++ .../v2/fluid/tests/test_elementwise_mul_op.py | 26 +++++++++++++------ .../v2/fluid/tests/test_elementwise_sub_op.py | 26 +++++++++++++------ 10 files changed, 148 insertions(+), 35 deletions(-) diff --git a/paddle/operators/elementwise_div_op.h b/paddle/operators/elementwise_div_op.h index 7783875e24..ef26cb6c91 100644 --- a/paddle/operators/elementwise_div_op.h +++ b/paddle/operators/elementwise_div_op.h @@ -19,11 +19,16 @@ limitations under the License. */ namespace paddle { namespace operators { +template +struct DivFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a / b; } +}; + template class ElementwiseDivKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseCompute(ctx); + ElementwiseComputeEx, DeviceContext, T>(ctx); } }; diff --git a/paddle/operators/elementwise_mul_op.h b/paddle/operators/elementwise_mul_op.h index 0e6559eacc..4b86b00b5a 100644 --- a/paddle/operators/elementwise_mul_op.h +++ b/paddle/operators/elementwise_mul_op.h @@ -18,11 +18,16 @@ limitations under the License. */ namespace paddle { namespace operators { +template +struct MulFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a * b; } +}; + template class ElementwiseMulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseCompute(ctx); + ElementwiseComputeEx, DeviceContext, T>(ctx); } }; diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index be11d5cc9d..db5d30c1af 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -340,6 +340,13 @@ void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { return; } + if (y_dims.size() == 1 && y_dims[0] == 1) { + // y is a scalar + auto extended_dims = framework::vectorize(x_dims); + extended_dims.push_back(1); + x_dims = framework::make_ddim(extended_dims); + } + int axis = ctx.Attr("axis"); axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); @@ -378,6 +385,13 @@ void ElementwiseComputeEx(const framework::ExecutionContext& ctx) { return; } + if (y_dims.size() == 1 && y_dims[0] == 1) { + // y is a scalar + auto extended_dims = framework::vectorize(x_dims); + extended_dims.push_back(1); + x_dims = framework::make_ddim(extended_dims); + } + int axis = ctx.Attr("axis"); axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), diff --git a/paddle/operators/elementwise_sub_op.h b/paddle/operators/elementwise_sub_op.h index 347e92f87c..a2aca79302 100644 --- a/paddle/operators/elementwise_sub_op.h +++ b/paddle/operators/elementwise_sub_op.h @@ -18,11 +18,16 @@ limitations under the License. */ namespace paddle { namespace operators { +template +struct SubFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a - b; } +}; + template class ElementwiseSubKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - ElementwiseCompute(ctx); + ElementwiseComputeEx, DeviceContext, T>(ctx); } }; diff --git a/python/paddle/v2/fluid/tests/test_elementwise_add_op.py b/python/paddle/v2/fluid/tests/test_elementwise_add_op.py index 1e88231877..3564772fb5 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_add_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_add_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np from op_test import OpTest @@ -40,6 +40,16 @@ class TestElementwiseOp(OpTest): ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) +class TestElementwiseAddOp_scalar(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(1).astype(np.float32) + } + self.outputs = {'Out': self.inputs['X'] + self.inputs['Y']} + + class TestElementwiseAddOp_Vector(TestElementwiseOp): def setUp(self): self.op_type = "elementwise_add" diff --git a/python/paddle/v2/fluid/tests/test_elementwise_div_op.py b/python/paddle/v2/fluid/tests/test_elementwise_div_op.py index fbabc79be2..77b113af76 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_div_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_div_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np from op_test import OpTest @@ -45,6 +45,16 @@ class ElementwiseDivOp(OpTest): ['X'], 'Out', max_relative_error=0.05, no_grad_set=set('Y')) +class TestElementwiseDivOp_scalar(ElementwiseDivOp): + def setUp(self): + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 4]).astype(np.float32), + 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float32) + } + self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} + + class TestElementwiseDivOp_Vector(ElementwiseDivOp): def setUp(self): self.op_type = "elementwise_div" diff --git a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py index 52bd123d80..9526f0199b 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest @@ -30,6 +43,15 @@ class TestElementwiseOp(OpTest): ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) +class TestElementwiseMaxOp_scalar(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_max" + x = np.random.random_integers(-5, 5, [2, 3, 4]).astype("float32") + y = np.array([0.5]).astype("float32") + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} + + class TestElementwiseMaxOp_Vector(TestElementwiseOp): def setUp(self): self.op_type = "elementwise_max" diff --git a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py index eeafab5b18..b900728233 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest @@ -30,6 +43,15 @@ class TestElementwiseOp(OpTest): ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) +class TestElementwiseMinOp_scalar(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_min" + x = np.random.random_integers(-5, 5, [2, 3, 4]).astype("float32") + y = np.array([0.5]).astype("float32") + self.inputs = {'X': x, 'Y': y} + self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} + + class TestElementwiseMaxOp_Vector(TestElementwiseOp): def setUp(self): self.op_type = "elementwise_min" diff --git a/python/paddle/v2/fluid/tests/test_elementwise_mul_op.py b/python/paddle/v2/fluid/tests/test_elementwise_mul_op.py index ef3a829abc..12dfa6599c 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_mul_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_mul_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np from op_test import OpTest @@ -38,6 +38,16 @@ class ElementwiseMulOp(OpTest): self.check_grad(['X'], 'Out', no_grad_set=set('Y')) +class TestElementwiseMulOp_scalar(ElementwiseMulOp): + def setUp(self): + self.op_type = "elementwise_mul" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(1).astype(np.float32) + } + self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + + class TestElementwiseMulOp_Vector(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" diff --git a/python/paddle/v2/fluid/tests/test_elementwise_sub_op.py b/python/paddle/v2/fluid/tests/test_elementwise_sub_op.py index db24db7b30..cf53d85bba 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_sub_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_sub_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np from op_test import OpTest @@ -40,6 +40,16 @@ class TestElementwiseOp(OpTest): ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) +class TestElementwiseSubOp_scalar(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(1).astype(np.float32) + } + self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + + class TestElementwiseSubOp_Vector(TestElementwiseOp): def setUp(self): self.op_type = "elementwise_sub" From c9641a03dc232862f9f8015f39fc11eb30d81693 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 17 Jan 2018 15:18:49 +0800 Subject: [PATCH 042/158] refine code --- paddle/operators/conv_op.cc | 13 +++++++++++++ paddle/operators/conv_transpose_op.cc | 12 ++++++++++++ paddle/operators/pool_op.cc | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index 0e8dddd7f1..d6882b275b 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -71,6 +71,12 @@ framework::OpKernelType ConvOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); +#ifdef PADDLE_WITH_CUDA + if (platform::is_gpu_place(ctx.GetPlace())) { + auto& dev_ctx = ctx.template device_context(); + use_cudnn &= dev_ctx.cudnn_handle() != nullptr; + } +#endif framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -285,6 +291,13 @@ framework::OpKernelType ConvOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); +#ifdef PADDLE_WITH_CUDA + if (platform::is_gpu_place(ctx.GetPlace())) { + auto& dev_ctx = ctx.template device_context(); + use_cudnn &= dev_ctx.cudnn_handle() != nullptr; + } +#endif + framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc index f71838c2aa..a2382a7e42 100644 --- a/paddle/operators/conv_transpose_op.cc +++ b/paddle/operators/conv_transpose_op.cc @@ -62,6 +62,12 @@ framework::OpKernelType ConvTransposeOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); +#ifdef PADDLE_WITH_CUDA + if (platform::is_gpu_place(ctx.GetPlace())) { + auto& dev_ctx = ctx.template device_context(); + use_cudnn &= dev_ctx.cudnn_handle() != nullptr; + } +#endif framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -265,6 +271,12 @@ framework::OpKernelType ConvTransposeOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); +#ifdef PADDLE_WITH_CUDA + if (platform::is_gpu_place(ctx.GetPlace())) { + auto& dev_ctx = ctx.template device_context(); + use_cudnn &= dev_ctx.cudnn_handle() != nullptr; + } +#endif framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index a450279451..b97333bb1a 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -65,6 +65,12 @@ framework::OpKernelType PoolOp::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); +#ifdef PADDLE_WITH_CUDA + if (platform::is_gpu_place(ctx.GetPlace())) { + auto &dev_ctx = ctx.template device_context(); + use_cudnn &= dev_ctx.cudnn_handle() != nullptr; + } +#endif framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; @@ -90,6 +96,12 @@ framework::OpKernelType PoolOpGrad::GetExpectedKernelType( const framework::ExecutionContext &ctx) const { bool use_cudnn = ctx.Attr("use_cudnn"); use_cudnn &= platform::is_gpu_place(ctx.GetPlace()); +#ifdef PADDLE_WITH_CUDA + if (platform::is_gpu_place(ctx.GetPlace())) { + auto &dev_ctx = ctx.template device_context(); + use_cudnn &= dev_ctx.cudnn_handle() != nullptr; + } +#endif framework::LibraryType library_; if (use_cudnn) { library_ = framework::LibraryType::kCUDNN; From 7d3b2e4b03d1ec3387e766b2821e24532ea95afd Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 17 Jan 2018 00:06:00 -0800 Subject: [PATCH 043/158] Fix a bug in sequence_erase_op --- paddle/operators/sequence_erase_op.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/sequence_erase_op.cu b/paddle/operators/sequence_erase_op.cu index 5da8eba3e1..daf5b29863 100644 --- a/paddle/operators/sequence_erase_op.cu +++ b/paddle/operators/sequence_erase_op.cu @@ -55,7 +55,7 @@ __global__ void SetOutput(const T* in_dat, const int in_len, const int* num_erased, T* out_dat) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < in_len) { - if (in_dat[index] != in_dat[index + 1]) { + if (num_erased[index] == num_erased[index + 1]) { out_dat[index - num_erased[index]] = in_dat[index]; } } From 7150289b5cad76d3347a268b54c31e13a0e49f42 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Wed, 17 Jan 2018 16:34:38 +0800 Subject: [PATCH 044/158] Refine CPU kernel 1. Allocate memory for output before compute. 2. Rename 'ctc_decode' to 'ctc_align' --- .../{ctc_decode_op.cc => ctc_align_op.cc} | 20 +++++++++---------- .../{ctc_decode_op.cu => ctc_align_op.cu} | 8 ++++---- .../{ctc_decode_op.h => ctc_align_op.h} | 19 +++++++----------- .../{test_ctc_decode.py => test_ctc_align.py} | 14 ++++++------- 4 files changed, 28 insertions(+), 33 deletions(-) rename paddle/operators/{ctc_decode_op.cc => ctc_align_op.cc} (78%) rename paddle/operators/{ctc_decode_op.cu => ctc_align_op.cu} (93%) rename paddle/operators/{ctc_decode_op.h => ctc_align_op.h} (80%) rename python/paddle/v2/fluid/tests/{test_ctc_decode.py => test_ctc_align.py} (82%) diff --git a/paddle/operators/ctc_decode_op.cc b/paddle/operators/ctc_align_op.cc similarity index 78% rename from paddle/operators/ctc_decode_op.cc rename to paddle/operators/ctc_align_op.cc index 480c9ae133..3fa8d2af74 100644 --- a/paddle/operators/ctc_decode_op.cc +++ b/paddle/operators/ctc_align_op.cc @@ -12,20 +12,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/ctc_decode_op.h" +#include "paddle/operators/ctc_align_op.h" namespace paddle { namespace operators { -class CTCDecodeOp : public framework::OperatorWithKernel { +class CTCAlignOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Input"), - "Input of CTCDecodeOp should not be null."); + "Input of CTCAlignOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Output"), - "Output of CTCDecodeOp should not be null."); + "Output of CTCAlignOp should not be null."); auto input_dims = ctx->GetInputDim("Input"); @@ -42,14 +42,14 @@ class CTCDecodeOp : public framework::OperatorWithKernel { } }; -class CTCDecodeOpMaker : public framework::OpProtoAndCheckerMaker { +class CTCAlignOpMaker : public framework::OpProtoAndCheckerMaker { public: - CTCDecodeOpMaker(OpProto* proto, OpAttrChecker* op_checker) + CTCAlignOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Input", "(LodTensor, default: LoDTensor), Its shape is " "[Lp, 1], where Lp is the sum of all input sequences' length."); - AddOutput("Output", "(Tensor, default: Tensor), The decode result."); + AddOutput("Output", "(Tensor, default: Tensor), The align result."); AddAttr("blank", "(int, default: 0), the blank label setted in Connectionist " "Temporal Classification (CTC) op.") @@ -59,7 +59,7 @@ class CTCDecodeOpMaker : public framework::OpProtoAndCheckerMaker { "merge repeated elements between two blanks. ") .SetDefault(true); AddComment(R"DOC( -CTCDecoder is used to merge repeated elements between two blanks +CTCAlign op is used to merge repeated elements between two blanks and then delete all blanks in sequence. Given: @@ -86,7 +86,7 @@ Then: } // namespace paddle namespace ops = paddle::operators; -REGISTER_OPERATOR(ctc_decode, ops::CTCDecodeOp, ops::CTCDecodeOpMaker, +REGISTER_OPERATOR(ctc_align, ops::CTCAlignOp, ops::CTCAlignOpMaker, paddle::framework::EmptyGradOpMaker); REGISTER_OP_CPU_KERNEL( - ctc_decode, ops::CTCDecodeKernel); + ctc_align, ops::CTCAlignKernel); diff --git a/paddle/operators/ctc_decode_op.cu b/paddle/operators/ctc_align_op.cu similarity index 93% rename from paddle/operators/ctc_decode_op.cu rename to paddle/operators/ctc_align_op.cu index b10db100f7..99e716e989 100644 --- a/paddle/operators/ctc_decode_op.cu +++ b/paddle/operators/ctc_align_op.cu @@ -15,7 +15,7 @@ limitations under the License. */ #include #include #include -#include "paddle/operators/ctc_decode_op.h" +#include "paddle/operators/ctc_align_op.h" namespace paddle { namespace operators { @@ -42,7 +42,7 @@ __global__ void MergeAndDelCudaKernel(const int64_t num_token, const T* tokens, } template -class CTCDecodeOpCUDAKernel : public framework::OpKernel { +class CTCAlignOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), @@ -87,5 +87,5 @@ class CTCDecodeOpCUDAKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_CUDA_KERNEL(ctc_decode, - paddle::operators::CTCDecodeOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(ctc_align, + paddle::operators::CTCAlignOpCUDAKernel); diff --git a/paddle/operators/ctc_decode_op.h b/paddle/operators/ctc_align_op.h similarity index 80% rename from paddle/operators/ctc_decode_op.h rename to paddle/operators/ctc_align_op.h index bc8dfab9f6..589413feb3 100644 --- a/paddle/operators/ctc_decode_op.h +++ b/paddle/operators/ctc_align_op.h @@ -23,7 +23,7 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; template -class CTCDecodeKernel : public framework::OpKernel { +class CTCAlignKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { auto* input = ctx.Input("Input"); @@ -43,7 +43,8 @@ class CTCDecodeKernel : public framework::OpKernel { bool merge_repeated = ctx.Attr("merge_repeated"); // merge repeated tokens and delete blank - std::vector> pathes(num_sequences); + T* output_data = output->mutable_data(ctx.GetPlace()); + size_t output_idx = 0; std::vector output_lod0(1, 0); const T* input_data = input->data(); for (size_t seq_idx = 0; seq_idx < num_sequences; ++seq_idx) { @@ -52,11 +53,12 @@ class CTCDecodeKernel : public framework::OpKernel { i < input_lod[level][seq_idx + 1]; ++i) { if (input_data[i] != blank && !(merge_repeated && input_data[i] == prev_token)) { - pathes[seq_idx].push_back(input_data[i]); + output_data[output_idx] = input_data[i]; + ++output_idx; } prev_token = input_data[i]; } - output_lod0.push_back(output_lod0.back() + pathes[seq_idx].size()); + output_lod0.push_back(output_idx); } // set output lod @@ -65,14 +67,7 @@ class CTCDecodeKernel : public framework::OpKernel { output->set_lod(output_lod); // resize output dims - T* output_data = output->mutable_data( - {static_cast(output_lod0.back()), 1}, ctx.GetPlace()); - - // copy result to output - for (int i = 0; i < num_sequences; ++i) { - memcpy(output_data + output_lod0[i], pathes[i].data(), - sizeof(int) * pathes[i].size()); - } + output->Resize({static_cast(output_lod0.back()), 1}); } }; diff --git a/python/paddle/v2/fluid/tests/test_ctc_decode.py b/python/paddle/v2/fluid/tests/test_ctc_align.py similarity index 82% rename from python/paddle/v2/fluid/tests/test_ctc_decode.py rename to python/paddle/v2/fluid/tests/test_ctc_align.py index 1efacab4b3..96f45890ee 100644 --- a/python/paddle/v2/fluid/tests/test_ctc_decode.py +++ b/python/paddle/v2/fluid/tests/test_ctc_align.py @@ -5,7 +5,7 @@ from op_test import OpTest from test_softmax_op import stable_softmax -def CTCDecode(input, lod, blank, merge_repeated): +def CTCAlign(input, lod, blank, merge_repeated): lod0 = lod[0] result = [] for i in range(len(lod0) - 1): @@ -20,9 +20,9 @@ def CTCDecode(input, lod, blank, merge_repeated): return result -class TestCTCDecodeOp(OpTest): +class TestCTCAlignOp(OpTest): def config(self): - self.op_type = "ctc_decode" + self.op_type = "ctc_align" self.input_lod = [[0, 11, 18]] self.blank = 0 self.merge_repeated = False @@ -32,8 +32,8 @@ class TestCTCDecodeOp(OpTest): def setUp(self): self.config() - output = CTCDecode(self.input, self.input_lod, self.blank, - self.merge_repeated) + output = CTCAlign(self.input, self.input_lod, self.blank, + self.merge_repeated) self.inputs = {"Input": (self.input, self.input_lod), } self.outputs = {"Output": output} @@ -47,9 +47,9 @@ class TestCTCDecodeOp(OpTest): pass -class TestCTCDecodeOpCase1(TestCTCDecodeOp): +class TestCTCAlignOpCase1(TestCTCAlignOp): def config(self): - self.op_type = "ctc_decode" + self.op_type = "ctc_align" self.input_lod = [[0, 11, 19]] self.blank = 0 self.merge_repeated = True From 9bcb2d268e4cee2e9f67d806288d7c71182266b3 Mon Sep 17 00:00:00 2001 From: guosheng Date: Wed, 17 Jan 2018 17:25:00 +0800 Subject: [PATCH 045/158] Add python wrapper for matmul_op and dot_product_attention --- doc/api/v2/fluid/layers.rst | 6 + doc/api/v2/fluid/nets.rst | 6 + python/paddle/v2/fluid/layers/nn.py | 115 ++++++++---------- python/paddle/v2/fluid/nets.py | 53 ++++++++ .../paddle/v2/fluid/tests/test_matmul_op.py | 8 +- 5 files changed, 121 insertions(+), 67 deletions(-) diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 62c154e65d..6e1d6719ac 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -364,6 +364,12 @@ split .. autofunction:: paddle.v2.fluid.layers.split :noindex: + +matmul +------ +.. autofunction:: paddle.v2.fluid.layers.matmul + :noindex: + logsigmoid ---------- .. autofunction:: paddle.v2.fluid.layers.logsigmoid diff --git a/doc/api/v2/fluid/nets.rst b/doc/api/v2/fluid/nets.rst index cca0dcdf08..f6b1cb4ba1 100644 --- a/doc/api/v2/fluid/nets.rst +++ b/doc/api/v2/fluid/nets.rst @@ -25,3 +25,9 @@ glu .. autofunction:: paddle.v2.fluid.nets.glu :noindex: + +dot_product_attention +--------------------- +.. autofunction:: paddle.v2.fluid.nets.dot_product_attention + :noindex: + diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index bc373d4b37..68499f1feb 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -37,6 +37,7 @@ __all__ = [ 'sequence_last_step', 'dropout', 'split', + 'matmul', ] @@ -1586,83 +1587,71 @@ def split(input, num_or_sections, dim=-1): return outs -def matmul(x, y): +def matmul(x, y, transpose_x=False, transpose_y=False, name=None): """ - Applies matrix multipication to two tensors. + Applies matrix multipication to two tensors. Currently only rank 1 to rank + 3 input tensors are supported. - This operator is used to perform (batched) matrix multiplication - over the last two dimensions of the input tensors `X` and `Y`. + The actual behavior depends on the shapes of :math:`x`, :math:`y` and the + flag values of :attr:`transpose_x`, :attr:`transpose_y`. Specifically: - If a transpose flag is specified, the last two dimensions of the - tensor are transposed. If the tensor is rank-1 of shape [D], then - for `X` it is treated as [1, D] in nontransposed form and as [D, 1] - in transposed form, whereas for `Y` it is the opposite: It is treated - as [D, 1] in nontransposed form and as [1, D] in transposed form. + - If a transpose flag is specified, the last two dimensions of the tensor + are transposed. If the tensor is rank-1 of shape :math:`[D]`, then for + :math:`x` it is treated as :math:`[1, D]` in nontransposed form and as + :math:`[D, 1]` in transposed form, whereas for :math:`y` it is the + opposite: It is treated as :math:`[D, 1]` in nontransposed form and as + :math:`[1, D]` in transposed form. - Examples without transpose: - - X: [K], Y: [K] => Out: [1] - - X: [K], Y: [K, N] => Out: [N] - - X: [B, M, K], Y: [K] => Out: [B, M] - - X: [M, K], Y: [B, K, N] => Out: [B, M, N] - - X: [B, M, K], Y: [B, K, N] => Out: [B, M, N] + - After transpose, the two tensors are 2-D or 3-D and matrix multipication + performs in the following way. - The behavior is designed to be similar to the `numpy.matmul` function. - The differences are: - - Currently only rank 1 to rank 3 input tensors are supported. - - We add `transpose_X` and `transpose_Y` flags. + - If both are 2-D, they are multiplied like conventional matrices. + - If either is 3-D, it is treated as a stack of matrices residing in the + last two dimensions and a batched matrix multiply supporting broadcast + applies on the two tensors. - Both the input `X` and `Y` can carry the LoD (Level of Details) information, - or not. But the output only shares the LoD information with input `X`. + Also note that if the raw tensor :math:`x` or :math:`y` is rank-1 and + nontransposed, the prepended or appended dimension :math:`1` will be + removed after matrix multipication. Args: x (Variable): The input variable which is a Tensor or LoDTensor. - y (Variable): If :attr:`num_or_sections` is an integer, - then the integer indicates the number of equal sized sub-tensors - that the tensor will be divided into. If :attr:`num_or_sections` - is a list of integers, the length of list indicates the number of - sub-tensors and the integers indicate the sizes of sub-tensors' - :attr:`dim` dimension orderly. - dim (int): The dimension along which to split. If :math:`dim < 0`, the - dimension to split along is :math:`rank(input) + dim`. + y (Variable): The input variable which is a Tensor or LoDTensor. + transpose_x (bool): Whether to transpose :math:`x` before multiplication. + transpose_y (bool): Whether to transpose :math:`y` before multiplication. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Returns: - List: The list of segmented tensor variables. + Variable: The product Tensor variable. Examples: .. code-block:: python - # x is a Tensor variable with shape [3, 9, 5]: - x0, x1, x2 = fluid.layers.split(x, num_or_sections=3, dim=1) - x0.shape # [3, 3, 5] - x1.shape # [3, 3, 5] - x2.shape # [3, 3, 5] - x0, x1, x2 = fluid.layers.split(x, num_or_sections=[2, 3, 4], dim=1) - x0.shape # [3, 2, 5] - x1.shape # [3, 3, 5] - x2.shape # [3, 4, 5] + # Examples to clarify shapes of the inputs and output + # x: [B, M, K], y: [B, K, N] + fluid.layers.matmul(x, y) # out: [B, M, N] + # x: [B, M, K], y: [K, N] + fluid.layers.matmul(x, y) # out: [B, M, N] + # x: [B, M, K], y: [K] + fluid.layers.matmul(x, y) # out: [B, M] + # x: [M, K], y: [K, N] + fluid.layers.matmul(x, y) # out: [M, N] + # x: [K], y: [K] + fluid.layers.matmul(x, y) # out: [1] + # x: [M], y: [N] + fluid.layers.matmul(x, y, True, True) # out: [M, N] """ - helper = LayerHelper('split', **locals()) - input_shape = input.shape - dim = (len(input_shape) + dim) if dim < 0 else dim - if isinstance(num_or_sections, int): - assert num_or_sections > 1, 'num_or_sections must be more than 1.' - num = num_or_sections - else: - assert len(num_or_sections) < input_shape[ - dim], 'len(num_or_sections) must not be more than input.shape[dim].' - num = len(num_or_sections) - outs = [ - helper.create_tmp_variable(dtype=helper.input_dtype()) - for i in range(num) - ] + helper = LayerHelper('matmul', **locals()) + assert max( + len(x.shape), len(y.shape) + ) <= 3, 'Currently only rank 1 to rank 3 input tensors are supported.' + out = helper.create_tmp_variable(dtype=helper.input_dtype()) helper.append_op( - type='split', - inputs={'X': input}, - outputs={'Out': outs}, - attrs={ - 'num': num_or_sections if isinstance(num_or_sections, int) else 0, - 'sections': num_or_sections - if isinstance(num_or_sections, list) else [], - 'axis': dim - }) - return outs + type='matmul', + inputs={'X': x, + 'Y': y}, + outputs={'Out': out}, + attrs={'transpose_X': transpose_x, + 'transpose_Y': transpose_y}) + return out diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index d515429216..18bef45d66 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -4,6 +4,7 @@ __all__ = [ "simple_img_conv_pool", "sequence_conv_pool", "glu", + "", ] @@ -135,3 +136,55 @@ def glu(input, dim=-1): a, b = layers.split(input, num_or_sections=2, dim=dim) out = layers.elementwise_mul(x=a, y=b) return out + + +def dot_product_attention(querys, keys, values): + """ + The dot-product attention. + + Attention mechanism can be seen as mapping a query and a set of key-value + pairs to an output. The output is computed as a weighted sum of the values, + where the weight assigned to each value is computed by a compatibility + function (dot-product here) of the query with the corresponding key. + + The dot-product attention can be implemented through (batch) matrix + multipication as follows: + + .. math:: + + Attention(Q, K, V)= softmax(QK^\mathrm{T})V + + Refer to `Attention Is All You Need + `_. + + Note that batch data containing sequences with different lengths is not + supported by this because of the (batch) matrix multipication. + + Args: + query (Variable): The input variable which is a Tensor or LoDTensor. + key (Variable): The input variable which is a Tensor or LoDTensor. + value (Variable): The input variable which is a Tensor or LoDTensor. + + Returns: + tuple: The Tensor variables representing the output and attention scores. + + Examples: + .. code-block:: python + + # Suppose q, k, v are tensor variables with the following shape: + # q: [3, 5, 9], k: [3, 6, 9], v: [3, 6, 10] + out, attn_scores = fluid.nets.dot_product_attention(q, k, v) + out.shape # [3, 5, 10] + attn_scores.shape # [3, 5, 6] + """ + assert keys.shape[-2] == values.shape[ + -2], 'The shapes of keys and values mismatch.' + assert querys.shape[-1] == keys.shape[ + -1], 'The shapes of querys and keys mismatch.' + product = layers.matmul(x=querys, y=keys, transpose_y=True) + attn_scores = layers.reshape( + x=layers.reshape( + x=product, shape=[-1, product.shape[-1]], act='softmax'), + shape=product.shape) + out = layers.matmul(attn_scores, values) + return out, attn_scores diff --git a/python/paddle/v2/fluid/tests/test_matmul_op.py b/python/paddle/v2/fluid/tests/test_matmul_op.py index d51572c8ab..3ad714ad67 100644 --- a/python/paddle/v2/fluid/tests/test_matmul_op.py +++ b/python/paddle/v2/fluid/tests/test_matmul_op.py @@ -83,18 +83,18 @@ class Generator(object): self.outputs = {'Out': Out} def test_check_output(self): - self.check_output(atol=1e-2) + self.check_output(atol=1e-3) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=1e-3) def test_check_grad_ignore_x(self): self.check_grad( - ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) + ['Y'], 'Out', max_relative_error=1e-3, no_grad_set=set("X")) def test_check_grad_ignore_y(self): self.check_grad( - ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) + ['X'], 'Out', max_relative_error=1e-3, no_grad_set=set('Y')) # Generate test cases for all possibilities From f189ad74426cf0970bd05016d4a2827ea6c1ea00 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 17:27:54 +0800 Subject: [PATCH 046/158] refine the defination of class GradientClipByGlobalNorm --- python/paddle/v2/fluid/clip.py | 47 +++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index f0904e18ea..fcdd4c29e4 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -93,41 +93,48 @@ class GradientClipByNorm(BaseGradientClipAttr): class GradientClipByGlobalNorm(BaseGradientClipAttr): global_norm_var = None clip_norm_var = None - ratio_var = None + scale_var = None @classmethod def init(cls, clip_norm): + if not (isinstance(clip_norm, int) or isinstance(clip_norm, float)): + raise TypeError("The 'clip_norm' must be a value of int or float") + cls.global_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=0.0) cls.clip_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=clip_norm) - def __init__(self): - if not (isinstance(self.__class__.global_norm_var, Variable) and - isinstance(self.__class__.clip_norm_var, Variable)): + @classmethod + def check_init(cls): + if not (isinstance(cls.global_norm_var, Variable) and + isinstance(cls.clip_norm_var, Variable)): raise ValueError( - "Class 'GradientClipByGlobalNorm' has not been properly initialized. Please call GradientClipByGlobalNorm.init() first." - ) + "Class 'GradientClipByGlobalNorm' has not been properly initialized. \ + Please call GradientClipByGlobalNorm.init() first.") + + @classmethod + def process_context(cls, context, param, grad): + cls.check_init() - def process_context(self, context, param, grad): local_norm_var = layers.reduce_sum( x=layers.pow(x=grad, factor=2), reduce_all=True) layers.sums( - input=[local_norm_var, self.__class__.global_norm_var], - out=[self.__class__.global_norm_var]) + input=[local_norm_var, cls.global_norm_var], + out=[cls.global_norm_var]) - def create_operators(self, param, grad): - if self.__class__.ratio_var is None: - self.__class__.global_norm_var = layers.sqrt( - x=self.__class__.global_norm_var) - self.__class__.ratio_var = layers.elementwise_div( - x=self.__class__.clip_norm_var, + @classmethod + def create_operators(cls, param, grad): + cls.check_init() + + if cls.scale_var is None: + cls.global_norm_var = layers.sqrt(x=cls.global_norm_var) + cls.scale_var = layers.elementwise_div( + x=cls.clip_norm_var, y=layers.elementwise_max( - x=self.__class__.clip_norm_var, - y=self.__class__.global_norm_var)) - # 缺乏elementwise_max - # 没法将ratio_var送给scale_op。 - # new_grad = layers. + x=cls.clip_norm_var, y=cls.global_norm_var)) + new_grad = layers.elementwise_mul(x=grad, y=cls.scale_var) + return param, new_grad def append_gradient_clip_ops(param_grad): From db959d63509e65bd3591b15ab33d96103449e836 Mon Sep 17 00:00:00 2001 From: guosheng Date: Wed, 17 Jan 2018 17:49:33 +0800 Subject: [PATCH 047/158] Add dot_product_attention to nets.__all__ --- python/paddle/v2/fluid/nets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index dc11afbc47..ee6f70b899 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -17,7 +17,7 @@ __all__ = [ "simple_img_conv_pool", "sequence_conv_pool", "glu", - "", + "dot_product_attention", ] From d1d614b9f8ea054692c119fa107db2deb8963a40 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 17 Jan 2018 01:51:59 -0800 Subject: [PATCH 048/158] Refine the GPU kernel for sequence_erase_op --- paddle/operators/sequence_erase_op.cu | 69 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/paddle/operators/sequence_erase_op.cu b/paddle/operators/sequence_erase_op.cu index daf5b29863..5c0576dc5e 100644 --- a/paddle/operators/sequence_erase_op.cu +++ b/paddle/operators/sequence_erase_op.cu @@ -42,8 +42,8 @@ __global__ void LabelErasedIdx(const T* in_dat, const int in_len, } template -__global__ void GetOutLod(const T* num_erased, const int* in_lod, - const int lod_len, int* out_lod0) { +__global__ void GetOutLod(const T* num_erased, const size_t* in_lod, + const int lod_len, size_t* out_lod0) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < lod_len) { out_lod0[index] = in_lod[index] - num_erased[in_lod[index]]; @@ -61,6 +61,26 @@ __global__ void SetOutput(const T* in_dat, const int in_len, } } +template +thrust::device_vector set_device_vector(Vector& vector) { + thrust::host_vector host_vec(vector.size()); + for (size_t i = 0; i < vector.size(); ++i) { + host_vec[i] = vector[i]; + } + thrust::device_vector dev_vec = host_vec; + return dev_vec; +} + +template +std::vector get_std_vector(thrust::device_vector& dev_vec) { + thrust::host_vector host_vec = dev_vec; + std::vector std_vec(host_vec.size(), 0); + for (size_t i = 0; i < host_vec.size(); ++i) { + std_vec[i] = host_vec[i]; + } + return std_vec; +} + template class SequenceEraseOpCUDAKernel : public framework::OpKernel { public: @@ -73,52 +93,45 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel { PADDLE_ENFORCE_EQ(lod[0].back(), (size_t)in->numel(), "The actual size mismatches with the LoD information."); auto tokens = ctx.Attr>("tokens"); - auto tokens_len = tokens.size(); auto in_len = in->numel(); auto in_dat = in->data(); - auto lod0 = lod[0]; + // Copy tokens to GPU + thrust::device_vector dev_tokens = + set_device_vector>(tokens); + T* dev_tokens_ptr = thrust::raw_pointer_cast(dev_tokens.data()); - thrust::host_vector host_tokens(tokens_len); - for (size_t i = 0; i < tokens.size(); ++i) { - host_tokens[i] = tokens[i]; - } - thrust::device_vector dev_tokens = host_tokens; + // Count number of elements to be erased thrust::device_vector num_erased(in_len + 1); - - T* dev_tokens_ptr = thrust::raw_pointer_cast(dev_tokens.data()); int* num_erased_ptr = thrust::raw_pointer_cast(num_erased.data()); - auto stream = ctx.cuda_device_context().stream(); LabelErasedIdx<<<(in_len - 1) / PADDLE_CUDA_NUM_THREADS + 1, PADDLE_CUDA_NUM_THREADS, 0, stream>>>( - in_dat, in_len, dev_tokens_ptr, tokens_len, num_erased_ptr); + in_dat, in_len, dev_tokens_ptr, tokens.size(), num_erased_ptr); thrust::inclusive_scan(num_erased.begin() + 1, num_erased.end(), num_erased.begin() + 1); - // Calc LoD + // Copy LoD to GPU + auto lod0 = lod[0]; auto lod_len = lod0.size(); - thrust::host_vector host_lod(lod_len); - for (size_t i = 0; i < lod_len; ++i) { - host_lod[i] = lod0[i]; - } - thrust::device_vector dev_in_lod = host_lod; - thrust::device_vector dev_out_lod(lod_len); - int* dev_in_lod_ptr = thrust::raw_pointer_cast(dev_in_lod.data()); - int* dev_out_lod_ptr = thrust::raw_pointer_cast(dev_out_lod.data()); + thrust::device_vector dev_in_lod = + set_device_vector>(lod0); + size_t* dev_in_lod_ptr = thrust::raw_pointer_cast(dev_in_lod.data()); + + // Calc output LoD + thrust::device_vector dev_out_lod(lod_len); + size_t* dev_out_lod_ptr = thrust::raw_pointer_cast(dev_out_lod.data()); GetOutLod<<<(lod_len - 1) / PADDLE_CUDA_NUM_THREADS + 1, PADDLE_CUDA_NUM_THREADS, 0, stream>>>( num_erased_ptr, dev_in_lod_ptr, lod_len, dev_out_lod_ptr); - thrust::host_vector host_out_lod = dev_out_lod; - std::vector out_lod0(lod_len, 0); - for (size_t i = 0; i < lod_len; i++) { - out_lod0[i] = host_out_lod[i]; - } + + // Set LoD for output + std::vector out_lod0 = get_std_vector(dev_out_lod); framework::LoD out_lod; out_lod.push_back(out_lod0); out->set_lod(out_lod); // Set output - out->Resize({out_lod0.back(), 1}); + out->Resize({static_cast(out_lod0.back()), 1}); auto out_dat = out->mutable_data(ctx.GetPlace()); SetOutput<<<(in_len - 1) / PADDLE_CUDA_NUM_THREADS + 1, PADDLE_CUDA_NUM_THREADS, 0, stream>>>(in_dat, in_len, From 5ae0c97faf78a02a3721298ccda9d6a0edbb7860 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 17 Jan 2018 01:56:28 -0800 Subject: [PATCH 049/158] Add unit test case for no tokens to be erased --- .../v2/fluid/tests/test_sequence_erase_op.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/paddle/v2/fluid/tests/test_sequence_erase_op.py b/python/paddle/v2/fluid/tests/test_sequence_erase_op.py index 650984009a..d8aa4f7e94 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_erase_op.py +++ b/python/paddle/v2/fluid/tests/test_sequence_erase_op.py @@ -44,5 +44,20 @@ class TestSequenceEraseOp(OpTest): self.check_output() +class TestSequenceEraseOpEmpty(OpTest): + def setUp(self): + self.op_type = "sequence_erase" + in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") + lod = [[0, 9, 13, 24, 30]] + tokens = [] + out_seq, new_lod0 = sequence_erase(in_seq, lod[0], tokens) + self.attrs = {'tokens': tokens} + self.inputs = {'X': (in_seq, lod)} + self.outputs = {'Out': (out_seq, [new_lod0])} + + def test_check_output(self): + self.check_output() + + if __name__ == '__main__': unittest.main() From 02ea349101662e5ad5199dac47b48f1835eda361 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 17 Jan 2018 18:02:45 +0800 Subject: [PATCH 050/158] enhance dist train performance --- paddle/operators/detail/grpc_client.cc | 5 +- paddle/operators/detail/grpc_client.h | 2 +- paddle/operators/recv_op.cc | 66 ++++++++----------- paddle/operators/send_op.cc | 6 +- .../paddle/v2/fluid/distribute_transpiler.py | 15 ++++- .../notest_recognize_digits_conv_dist.py | 17 ++--- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/paddle/operators/detail/grpc_client.cc b/paddle/operators/detail/grpc_client.cc index 5a4db2d7e6..521760228b 100644 --- a/paddle/operators/detail/grpc_client.cc +++ b/paddle/operators/detail/grpc_client.cc @@ -63,9 +63,6 @@ bool RPCClient::AsyncGetVariable(const std::string& ep, sendrecv::VariableMessage req; req.set_varname(var_name); - auto* var = scope.FindVar(var_name); - SerializeToMessage(var_name, var, ctx, &req); - // varhandle VarHandle var_h; var_h.ep = ep; @@ -87,7 +84,7 @@ bool RPCClient::AsyncGetVariable(const std::string& ep, return true; } -bool RPCClient::wait() { +bool RPCClient::Wait() { bool ok = true; while (true) { diff --git a/paddle/operators/detail/grpc_client.h b/paddle/operators/detail/grpc_client.h index d27b5ced9e..a62e70a253 100644 --- a/paddle/operators/detail/grpc_client.h +++ b/paddle/operators/detail/grpc_client.h @@ -130,7 +130,7 @@ class RPCClient { const framework::Scope& scope, const std::string& var_name, int64_t time_out = 600 * 1000); - bool wait(); + bool Wait(); private: bool Proceed(); diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 55b33343af..dea7db391c 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -27,6 +27,7 @@ limitations under the License. */ #include "paddle/operators/detail/grpc_server.h" #include "paddle/operators/detail/sendrecvop_utils.h" #include "paddle/operators/detail/simple_block_queue.h" +#include "paddle/string/printf.h" #define LISTEN_TERMINATE_MESSAGE "TERMINATE@RECV" @@ -77,35 +78,37 @@ class RecvOp : public framework::OperatorBase { if (grads_counter_.find(varname) == grads_counter_.end()) { grads_counter_[varname] = 0; } - char ret[256]; - snprintf(ret, sizeof(ret), "%s.trainer_%d", varname.c_str(), - grads_counter_[varname]++); - return std::string(ret); + return string::Sprintf("%s.trainer_%d", varname, grads_counter_[varname]++); } void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { - // FIXME(typhoonzero): no new scopes for every run. + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(dev_place); framework::Scope &recv_scope = scope.NewScope(); rpc_service_->SetScope(&recv_scope); auto param_list = Attr>("ParamList"); auto grad_list = Attr>("GradList"); - auto trainer_count = Attr("Trainers"); + auto fan_in = Attr("Fanin"); size_t param_count = param_list.size(); + std::string program_str = Attr("OptimizeProgram"); + framework::proto::ProgramDesc program_desc; + program_desc.ParseFromString(program_str); + framework::ProgramDesc program(program_desc); + framework::Executor executor(dev_place); + rpc_service_->Reset(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. bool exit_flag = false; while (!exit_flag) { - // TODO(gognwb): simply this loop. - // Get from multiple trainers, we don't care about order in which - // the gradient arrives, just add suffix 0~n then average the gradient. - for (size_t i = 0; i < param_count * trainer_count; ++i) { - // blocking get one var from client. + // Get from multiple trainers, we don't care about the order in which + // the gradients arrives, just add suffix 0~n and merge the gradient. + for (size_t i = 0; i < param_count * fan_in; ++i) { const detail::MessageWithName &v = rpc_service_->Get(); auto grad_var_name = v.first; if (grad_var_name == LISTEN_TERMINATE_MESSAGE) { - VLOG(4) << "received LISTEN_TERMINATE_MESSAGE and RunOp.Run() exit"; + LOG(INFO) << "received terminate message and exit"; exit_flag = true; break; } @@ -114,44 +117,27 @@ class RecvOp : public framework::OperatorBase { if (it != grad_list.end()) { param_var_name = param_list[it - grad_list.begin()]; } else { - LOG(ERROR) << "grad have no paired param found!\"" << grad_var_name - << "\""; + LOG(ERROR) << "grad have no paired param:" << grad_var_name; } VLOG(3) << "recved grad: " << grad_var_name << " updating param: " << param_var_name; - - auto *merged_grad = recv_scope.FindVar(grad_var_name); - if (merged_grad == nullptr) { - auto *ptr = recv_scope.Var(grad_var_name); - CreateTensorFromMessageType(ptr, v.second.type()); - VLOG(3) << "Create Variable " << grad_var_name - << " on recv scope, which pointer is " << ptr << " type is " - << v.second.type(); + // Assume grad_var_name must appear in global scope. + std::string grad_var_name_trainer; + if (fan_in > 1) { + grad_var_name_trainer = this->GetGradVarNameForTrainer(grad_var_name); } - - if (trainer_count > 1) { - grad_var_name = this->GetGradVarNameForTrainer(grad_var_name); + auto *var = recv_scope.FindVar(grad_var_name_trainer); + if (var == nullptr) { + LOG(ERROR) << "can not find server side var: " + << grad_var_name_trainer; + PADDLE_THROW("can not find server side var"); } - - auto *var = recv_scope.Var(grad_var_name); - platform::DeviceContextPool &pool = - platform::DeviceContextPool::Instance(); - auto &dev_ctx = *pool.Get(dev_place); detail::DeserializeFromMessage(v.second, dev_ctx, var); } - if (exit_flag) { break; } - rpc_service_->Reset(); - - std::string program_str = Attr("OptimizeProgram"); - framework::proto::ProgramDesc program_desc; - program_desc.ParseFromString(program_str); - framework::ProgramDesc program(program_desc); - framework::Executor executor(dev_place); - // Run sub graph to get optimized tensor try { executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/, false /*create_vars*/); @@ -195,7 +181,7 @@ This operator will recv tensor from send_op "GradList", "type list of string", "grad->param name mapping to find which param to optimize.") .SetDefault({}); - AddAttr("Trainers", "type int", + AddAttr("Fanin", "type int", "Number of trainers in the current cluster job") .SetDefault(1); } diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index 4d145250bd..d65153c1fd 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -41,14 +41,16 @@ class SendOp : public framework::OperatorBase { // FIXME(gongwb): DeviceContext? auto ctx = platform::CPUDeviceContext(); for (size_t i = 0; i < ins.size(); i++) { + VLOG(3) << "sending " << ins[i]; client_.AsyncSendVariable(epmap[i], ctx, scope, ins[i]); } + client_.Wait(); for (size_t i = 0; i < outs.size(); i++) { + VLOG(3) << "getting " << outs[i]; client_.AsyncGetVariable(epmap[i], ctx, scope, outs[i]); } - - client_.wait(); + client_.Wait(); } private: diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 00fe3e68c9..9876296a37 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -452,6 +452,19 @@ class DistributeTranspiler: pserver_program = Program() for v in self.param_grad_ep_mapping[endpoint]["params"]: self._clone_var(pserver_program.global_block(), v) + for v in self.param_grad_ep_mapping[endpoint]["grads"]: + # create vars for each trainer in global scope, so + # we don't need to create them when grad arrives. + pserver_program.global_block().create_var( + name=v.name, persistable=True, dtype=v.dtype, shape=v.shape) + for trainer_id in xrange(self.trainers): + print("create variable for program: %s.trainer_%d" % + (v.name, trainer_id)) + pserver_program.global_block().create_var( + name="%s.trainer_%d" % (v.name, trainer_id), + persistable=True, + dtype=v.dtype, + shape=v.shape) # step6 optimize_sub_program = Program() for idx, opt_op in enumerate(optimize_ops): @@ -481,7 +494,7 @@ class DistributeTranspiler: p.name for p in self.param_grad_ep_mapping[endpoint]["grads"] ], - "Trainers": self.trainers + "Fanin": self.trainers }) pserver_program.sync_with_cpp() return pserver_program diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py index 20b4a8b34c..e563e0ddc5 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py @@ -39,26 +39,27 @@ train_reader = paddle.batch( place = fluid.CPUPlace() exe = fluid.Executor(place) -t = fluid.DistributeTranspiler() -# all parameter server endpoints list for spliting parameters -pserver_endpoints = os.getenv("PSERVERS") -# server endpoint for current node -current_endpoint = os.getenv("SERVER_ENDPOINT") -# run as trainer or parameter server +pserver_endpoints = os.getenv("PSERVERS") # all pserver endpoints +trainers = int(os.getenv("TRAINERS")) # total trainer count +current_endpoint = os.getenv("SERVER_ENDPOINT") # current pserver endpoint training_role = os.getenv("TRAINING_ROLE", "TRAINER") # get the training role: trainer/pserver -t.transpile(optimize_ops, params_grads, pservers=pserver_endpoints, trainers=2) +t = fluid.DistributeTranspiler() +t.transpile( + optimize_ops, params_grads, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) - exe.run(fluid.default_startup_program()) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": trainer_prog = t.get_trainer_program() feeder = fluid.DataFeeder(feed_list=[images, label], place=place) + # TODO(typhoonzero): change trainer startup program to fetch parameters from pserver exe.run(fluid.default_startup_program()) for pass_id in range(PASS_NUM): From 7a2aa486cc40dcd6c77b5cf717ee275600986570 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 17 Jan 2018 02:41:05 -0800 Subject: [PATCH 051/158] Unify data type in sequence_erase_op --- paddle/operators/sequence_erase_op.cu | 29 +++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/paddle/operators/sequence_erase_op.cu b/paddle/operators/sequence_erase_op.cu index 5c0576dc5e..c1e8bc2090 100644 --- a/paddle/operators/sequence_erase_op.cu +++ b/paddle/operators/sequence_erase_op.cu @@ -23,13 +23,13 @@ using platform::PADDLE_CUDA_NUM_THREADS; using LoDTensor = framework::LoDTensor; template -__global__ void LabelErasedIdx(const T* in_dat, const int in_len, - const T* tokens, const int tokens_len, - int* num_erased) { +__global__ void LabelErasedIdx(const T* in_dat, const int64_t in_len, + const int* tokens, const size_t tokens_len, + size_t* num_erased) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < in_len) { int erased = 0; - for (int i = 0; i < tokens_len; ++i) { + for (size_t i = 0; i < tokens_len; ++i) { if (in_dat[index] == tokens[i]) { erased = 1; } @@ -41,9 +41,8 @@ __global__ void LabelErasedIdx(const T* in_dat, const int in_len, } } -template -__global__ void GetOutLod(const T* num_erased, const size_t* in_lod, - const int lod_len, size_t* out_lod0) { +__global__ void GetOutLod(const size_t* num_erased, const size_t* in_lod, + const size_t lod_len, size_t* out_lod0) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < lod_len) { out_lod0[index] = in_lod[index] - num_erased[in_lod[index]]; @@ -51,8 +50,8 @@ __global__ void GetOutLod(const T* num_erased, const size_t* in_lod, } template -__global__ void SetOutput(const T* in_dat, const int in_len, - const int* num_erased, T* out_dat) { +__global__ void SetOutput(const T* in_dat, const int64_t in_len, + const size_t* num_erased, T* out_dat) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < in_len) { if (num_erased[index] == num_erased[index + 1]) { @@ -92,17 +91,17 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel { PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now."); PADDLE_ENFORCE_EQ(lod[0].back(), (size_t)in->numel(), "The actual size mismatches with the LoD information."); - auto tokens = ctx.Attr>("tokens"); + auto tokens = ctx.Attr>("tokens"); auto in_len = in->numel(); auto in_dat = in->data(); // Copy tokens to GPU - thrust::device_vector dev_tokens = - set_device_vector>(tokens); - T* dev_tokens_ptr = thrust::raw_pointer_cast(dev_tokens.data()); + thrust::device_vector dev_tokens = + set_device_vector>(tokens); + int* dev_tokens_ptr = thrust::raw_pointer_cast(dev_tokens.data()); // Count number of elements to be erased - thrust::device_vector num_erased(in_len + 1); - int* num_erased_ptr = thrust::raw_pointer_cast(num_erased.data()); + thrust::device_vector num_erased(in_len + 1); + size_t* num_erased_ptr = thrust::raw_pointer_cast(num_erased.data()); auto stream = ctx.cuda_device_context().stream(); LabelErasedIdx<<<(in_len - 1) / PADDLE_CUDA_NUM_THREADS + 1, PADDLE_CUDA_NUM_THREADS, 0, stream>>>( From 4cb6e72b85fef0205a3d3ebfd136e11c009e39f6 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 18:43:54 +0800 Subject: [PATCH 052/158] refine code details --- python/paddle/v2/fluid/clip.py | 18 +++++++-------- python/paddle/v2/fluid/framework.py | 2 +- python/paddle/v2/fluid/param_attr.py | 22 +++++++++---------- .../tests/book/test_recognize_digits_mlp.py | 18 +++++++-------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index d8240dc155..f7917fc142 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import functools import layers from framework import Variable @@ -162,7 +162,7 @@ def append_gradient_clip_ops(param_grad): context = dict() create_op_callbacks = [] for p, g in param_grad: - clip_attr = getattr(p, 'clip_attr', NullGradientClipAttr()) + clip_attr = getattr(p, 'gradient_clip_attr', NullGradientClipAttr()) if clip_attr is None: clip_attr = NullGradientClipAttr() if not isinstance(clip_attr, BaseGradientClipAttr): diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 8042febfed..9128a0eebe 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -946,7 +946,7 @@ class Parameter(Variable): self.regularizer = kwargs.get('regularizer', None) - self.clip_attr = kwargs.get('clip_attr', None) + self.gradient_clip_attr = kwargs.get('gradient_clip_attr', None) # program is a global instance. diff --git a/python/paddle/v2/fluid/param_attr.py b/python/paddle/v2/fluid/param_attr.py index 3af0190590..8c8de0d104 100644 --- a/python/paddle/v2/fluid/param_attr.py +++ b/python/paddle/v2/fluid/param_attr.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from initializer import Initializer, Xavier, Constant from regularizer import WeightDecayRegularizer @@ -24,13 +24,13 @@ class ParamAttr(object): learning_rate=1.0, regularizer=None, trainable=True, - clip=None): + gradient_clip=None): self.name = name self.initializer = initializer self.learning_rate = learning_rate self.regularizer = regularizer self.trainable = trainable - self.clip = clip + self.gradient_clip = gradient_clip def set_default_initializer(self, initializer): if initializer is None: @@ -76,7 +76,7 @@ class ParamAttr(object): }, 'regularizer': self.regularizer, 'trainable': self.trainable, - 'clip_attr': self.clip + 'gradient_clip_attr': self.gradient_clip } if with_initializer: kwargs['initializer'] = self.initializer diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py index 02da2fcc85..e614e5e3f1 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from __future__ import print_function import numpy as np import paddle.v2 as paddle @@ -26,7 +26,7 @@ hidden1 = fluid.layers.fc(input=image, act='relu', param_attr=fluid.ParamAttr( regularizer=regularizer, - clip=fluid.clip.ClipByValue(10))) + gradient_clip=fluid.clip.ClipByValue(10))) hidden2 = fluid.layers.fc(input=hidden1, size=64, From f1a889720a27f8c780730baa3ed2175d91a2233a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 18:52:00 +0800 Subject: [PATCH 053/158] fix copyright --- paddle/gserver/tests/sequence_recurrent_group.py | 13 +++++++++++++ .../paddle/v2/fluid/tests/test_edit_distance_op.py | 13 +++++++++++++ .../v2/fluid/tests/test_elementwise_max_op.py | 13 +++++++++++++ .../v2/fluid/tests/test_elementwise_min_op.py | 13 +++++++++++++ 4 files changed, 52 insertions(+) diff --git a/paddle/gserver/tests/sequence_recurrent_group.py b/paddle/gserver/tests/sequence_recurrent_group.py index a1d54542e3..1343f2956f 100644 --- a/paddle/gserver/tests/sequence_recurrent_group.py +++ b/paddle/gserver/tests/sequence_recurrent_group.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. #!/usr/bin/env python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # diff --git a/python/paddle/v2/fluid/tests/test_edit_distance_op.py b/python/paddle/v2/fluid/tests/test_edit_distance_op.py index 38e87728b3..cf118df634 100644 --- a/python/paddle/v2/fluid/tests/test_edit_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_edit_distance_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py index 52bd123d80..3dfab4dd2f 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py index eeafab5b18..8422a9cdae 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest From 117918a545694cf2258dfaa497c8ebc471fbc9f1 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 18:56:38 +0800 Subject: [PATCH 054/158] Fix copyright infomation --- paddle/gserver/tests/sequence_recurrent_group.py | 13 +++++++++++++ .../paddle/v2/fluid/tests/test_edit_distance_op.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/paddle/gserver/tests/sequence_recurrent_group.py b/paddle/gserver/tests/sequence_recurrent_group.py index a1d54542e3..1343f2956f 100644 --- a/paddle/gserver/tests/sequence_recurrent_group.py +++ b/paddle/gserver/tests/sequence_recurrent_group.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. #!/usr/bin/env python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # diff --git a/python/paddle/v2/fluid/tests/test_edit_distance_op.py b/python/paddle/v2/fluid/tests/test_edit_distance_op.py index 38e87728b3..cf118df634 100644 --- a/python/paddle/v2/fluid/tests/test_edit_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_edit_distance_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest From 3f5c77cc4b8eeb21a7056852f83e54b6728355fb Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 19:03:56 +0800 Subject: [PATCH 055/158] fix copyright information --- paddle/gserver/tests/sequence_recurrent_group.py | 13 +++++++++++++ .../paddle/v2/fluid/tests/test_edit_distance_op.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/paddle/gserver/tests/sequence_recurrent_group.py b/paddle/gserver/tests/sequence_recurrent_group.py index a1d54542e3..1343f2956f 100644 --- a/paddle/gserver/tests/sequence_recurrent_group.py +++ b/paddle/gserver/tests/sequence_recurrent_group.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. #!/usr/bin/env python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # diff --git a/python/paddle/v2/fluid/tests/test_edit_distance_op.py b/python/paddle/v2/fluid/tests/test_edit_distance_op.py index 38e87728b3..cf118df634 100644 --- a/python/paddle/v2/fluid/tests/test_edit_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_edit_distance_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest From 6ebfade465be5526939b52b0d251486298c4c734 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 19:14:05 +0800 Subject: [PATCH 056/158] fix copyright information --- paddle/gserver/tests/sequence_recurrent_group.py | 13 +++++++++++++ .../paddle/v2/fluid/tests/test_edit_distance_op.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/paddle/gserver/tests/sequence_recurrent_group.py b/paddle/gserver/tests/sequence_recurrent_group.py index a1d54542e3..1343f2956f 100644 --- a/paddle/gserver/tests/sequence_recurrent_group.py +++ b/paddle/gserver/tests/sequence_recurrent_group.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. #!/usr/bin/env python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # diff --git a/python/paddle/v2/fluid/tests/test_edit_distance_op.py b/python/paddle/v2/fluid/tests/test_edit_distance_op.py index 38e87728b3..cf118df634 100644 --- a/python/paddle/v2/fluid/tests/test_edit_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_edit_distance_op.py @@ -1,3 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import unittest import numpy as np from op_test import OpTest From 6f71f89ded879025ba1f0742cf54c0723705681e Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Wed, 17 Jan 2018 20:12:22 +0800 Subject: [PATCH 057/158] change DEVICE_TYPE in op_registry to LIBRARY_TYPE (#7588) --- paddle/framework/op_registry.h | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index d75c0233e8..5de9ae559c 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -177,16 +177,16 @@ class OpKernelRegistrar : public Registrar { /** * Macro to register OperatorKernel. */ -#define REGISTER_OP_KERNEL(op_type, DEVICE_TYPE, place_class, ...) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __reg_op_kernel_##op_type##_##DEVICE_TYPE##__, \ - "REGISTER_OP_KERNEL must be called in global namespace"); \ - static ::paddle::framework::OpKernelRegistrar \ - __op_kernel_registrar_##op_type##_##DEVICE_TYPE##__(#op_type, \ - #DEVICE_TYPE); \ - int TouchOpKernelRegistrar_##op_type##_##DEVICE_TYPE() { \ - __op_kernel_registrar_##op_type##_##DEVICE_TYPE##__.Touch(); \ - return 0; \ +#define REGISTER_OP_KERNEL(op_type, LIBRARY_TYPE, place_class, ...) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __reg_op_kernel_##op_type##_##LIBRARY_TYPE##__, \ + "REGISTER_OP_KERNEL must be called in global namespace"); \ + static ::paddle::framework::OpKernelRegistrar \ + __op_kernel_registrar_##op_type##_##LIBRARY_TYPE##__(#op_type, \ + #LIBRARY_TYPE); \ + int TouchOpKernelRegistrar_##op_type##_##LIBRARY_TYPE() { \ + __op_kernel_registrar_##op_type##_##LIBRARY_TYPE##__.Touch(); \ + return 0; \ } #define REGISTER_OP_CUDA_KERNEL(op_type, ...) \ @@ -208,14 +208,14 @@ class OpKernelRegistrar : public Registrar { static int use_op_itself_##op_type##_ __attribute__((unused)) = \ TouchOpRegistrar_##op_type() -#define USE_OP_DEVICE_KERNEL(op_type, DEVICE_TYPE) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __use_op_kernel_##op_type##_##DEVICE_TYPE##__, \ - "USE_OP_DEVICE_KERNEL must be in global namespace"); \ - extern int TouchOpKernelRegistrar_##op_type##_##DEVICE_TYPE(); \ - static int use_op_kernel_##op_type##_##DEVICE_TYPE##_ \ - __attribute__((unused)) = \ - TouchOpKernelRegistrar_##op_type##_##DEVICE_TYPE() +#define USE_OP_DEVICE_KERNEL(op_type, LIBRARY_TYPE) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __use_op_kernel_##op_type##_##LIBRARY_TYPE##__, \ + "USE_OP_DEVICE_KERNEL must be in global namespace"); \ + extern int TouchOpKernelRegistrar_##op_type##_##LIBRARY_TYPE(); \ + static int use_op_kernel_##op_type##_##LIBRARY_TYPE##_ \ + __attribute__((unused)) = \ + TouchOpKernelRegistrar_##op_type##_##LIBRARY_TYPE() // TODO(fengjiayi): The following macros // seems ugly, do we have better method? From 1dac173b518faeb8f31c321a61fa287b8de4246e Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 20:15:03 +0800 Subject: [PATCH 058/158] add API for clip_by_global_norm --- python/paddle/v2/fluid/clip.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index f7917fc142..d1e6987e01 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -13,7 +13,7 @@ # limitations under the License. import functools import layers -from framework import Variable +import framework from . import core __all__ = [ @@ -128,8 +128,8 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): @classmethod def check_init(cls): - if not (isinstance(cls.global_norm_var, Variable) and - isinstance(cls.clip_norm_var, Variable)): + if not (isinstance(cls.global_norm_var, framework.Variable) and + isinstance(cls.clip_norm_var, framework.Variable)): raise ValueError( "Class 'GradientClipByGlobalNorm' has not been properly initialized. \ Please call GradientClipByGlobalNorm.init() first.") @@ -158,6 +158,23 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): return param, new_grad +def gradient_clip_by_global_norm(clip_norm, param_list=None, program=None): + if program is None: + program = framework.default_main_program() + if param_list is None: + param_list = program.block(0).all_parameters() + if all(isinstance(elem, basestring) for elem in param_list): + param_list = [program.block(0).var(elem) for elem in param_list] + if not all(isinstance(elem, framework.Parameter) for elem in param_list): + raise TypeError( + "'param_list' should be a list of Parameter or basestring(parameter's name)." + ) + + GradientClipByGlobalNorm.init(clip_norm) + for param in param_list: + param.gradient_clip_attr = GradientClipByGlobalNorm() + + def append_gradient_clip_ops(param_grad): context = dict() create_op_callbacks = [] From 958d07bee3343288f9813693b5a85150a5131cdd Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 20:21:05 +0800 Subject: [PATCH 059/158] fix a error --- python/paddle/v2/fluid/framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 9128a0eebe..91fdb5fa7e 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -777,7 +777,7 @@ class Block(object): trainable=p.trainable, optimize_attr=p.optimize_attr, regularizer=p.regularizer, - clip_attr=p.clip_attr, + gradient_clip_attr=p.gradient_clip_attr, error_clip=p.error_clip, name=v.name) self.vars[new_p.name] = new_p From a247972ddad05490a7b72911521bff0b48cf2d1c Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 17 Jan 2018 20:31:05 +0800 Subject: [PATCH 060/158] fix a error --- python/paddle/v2/fluid/clip.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index d1e6987e01..7a36df0dab 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -134,8 +134,8 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): "Class 'GradientClipByGlobalNorm' has not been properly initialized. \ Please call GradientClipByGlobalNorm.init() first.") - @classmethod - def process_context(cls, context, param, grad): + def process_context(self, context, param, grad): + cls = self.__class__ cls.check_init() local_norm_var = layers.reduce_sum( @@ -144,8 +144,8 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): input=[local_norm_var, cls.global_norm_var], out=[cls.global_norm_var]) - @classmethod - def create_operators(cls, param, grad): + def create_operators(self, param, grad): + cls = self.__class__ cls.check_init() if cls.scale_var is None: From 8f9480cc2cbc02bd96d237e8b52706e53d86e217 Mon Sep 17 00:00:00 2001 From: ying Date: Wed, 17 Jan 2018 20:14:02 +0800 Subject: [PATCH 061/158] fix copyright. --- paddle/gserver/tests/img_conv_cudnn.py | 16 +------------- paddle/gserver/tests/img_conv_exconv.py | 16 +------------- paddle/gserver/tests/pyDataProvider.py | 21 +++++++++--------- paddle/gserver/tests/rnn_data_provider.py | 21 +++++++++--------- paddle/gserver/tests/sequenceGen.py | 21 +++++++++--------- ...ence_nest_rnn_multi_unequalength_inputs.py | 22 +++++++++---------- paddle/gserver/tests/sequence_recurrent.py | 16 +------------- .../gserver/tests/sequence_recurrent_group.py | 22 +++++++++---------- .../tests/sequence_rnn_matched_inputs.py | 16 +------------- .../tests/sequence_rnn_mixed_inputs.py | 16 +------------- .../sequence_rnn_multi_unequalength_inputs.py | 16 +------------- paddle/gserver/tests/test_PyDataProvider2.py | 21 +++++++++--------- 12 files changed, 66 insertions(+), 158 deletions(-) diff --git a/paddle/gserver/tests/img_conv_cudnn.py b/paddle/gserver/tests/img_conv_cudnn.py index e424261bda..0ea6d6bae6 100644 --- a/paddle/gserver/tests/img_conv_cudnn.py +++ b/paddle/gserver/tests/img_conv_cudnn.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. @@ -11,20 +11,6 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -#edit-mode: -*- python -*- -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/paddle/gserver/tests/img_conv_exconv.py b/paddle/gserver/tests/img_conv_exconv.py index 3b59cd80b8..c618cdab27 100644 --- a/paddle/gserver/tests/img_conv_exconv.py +++ b/paddle/gserver/tests/img_conv_exconv.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. @@ -11,20 +11,6 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -#edit-mode: -*- python -*- -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/paddle/gserver/tests/pyDataProvider.py b/paddle/gserver/tests/pyDataProvider.py index 7235a23943..d2ad5888b5 100644 --- a/paddle/gserver/tests/pyDataProvider.py +++ b/paddle/gserver/tests/pyDataProvider.py @@ -1,17 +1,16 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import numpy import struct import traceback diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 913365a5a4..063a4127e5 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -1,17 +1,16 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. from paddle.trainer.PyDataProvider2 import * # Note that each config should has an independent provider diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index fd725727c0..04a1732d61 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -1,17 +1,16 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import os import sys diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.py b/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.py index 7303d08804..aeaaa221f9 100644 --- a/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.py +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.py @@ -1,18 +1,16 @@ -# edit-mode: -*- python -*- -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. from paddle.trainer_config_helpers import * ######################## data source ################################ diff --git a/paddle/gserver/tests/sequence_recurrent.py b/paddle/gserver/tests/sequence_recurrent.py index d20d3331ae..8786a5465d 100644 --- a/paddle/gserver/tests/sequence_recurrent.py +++ b/paddle/gserver/tests/sequence_recurrent.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. @@ -11,20 +11,6 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -#!/usr/bin/env python -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/paddle/gserver/tests/sequence_recurrent_group.py b/paddle/gserver/tests/sequence_recurrent_group.py index a1d54542e3..da182942c9 100644 --- a/paddle/gserver/tests/sequence_recurrent_group.py +++ b/paddle/gserver/tests/sequence_recurrent_group.py @@ -1,18 +1,16 @@ -#!/usr/bin/env python -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. from paddle.trainer_config_helpers import * ######################## data source ################################ diff --git a/paddle/gserver/tests/sequence_rnn_matched_inputs.py b/paddle/gserver/tests/sequence_rnn_matched_inputs.py index b156fa9baa..0c55f2cf9d 100644 --- a/paddle/gserver/tests/sequence_rnn_matched_inputs.py +++ b/paddle/gserver/tests/sequence_rnn_matched_inputs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. @@ -11,20 +11,6 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -# edit-mode: -*- python -*- -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/paddle/gserver/tests/sequence_rnn_mixed_inputs.py b/paddle/gserver/tests/sequence_rnn_mixed_inputs.py index c2c98aae5f..22b376b91a 100644 --- a/paddle/gserver/tests/sequence_rnn_mixed_inputs.py +++ b/paddle/gserver/tests/sequence_rnn_mixed_inputs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. @@ -11,20 +11,6 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -# edit-mode: -*- python -*- -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.py b/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.py index c812c6ced2..3ce87490bb 100644 --- a/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.py +++ b/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # #Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. @@ -11,20 +11,6 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -#edit-mode: -*- python -*- -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/paddle/gserver/tests/test_PyDataProvider2.py b/paddle/gserver/tests/test_PyDataProvider2.py index 0d0fe476ff..044aede98e 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.py +++ b/paddle/gserver/tests/test_PyDataProvider2.py @@ -1,17 +1,16 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import random from paddle.trainer.PyDataProvider2 import * From ac73900942ab9d81f4181162230a195a016f985e Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Wed, 17 Jan 2018 22:00:47 +0800 Subject: [PATCH 062/158] enhance/add lod check (#7439) --- paddle/framework/lod_tensor.cc | 59 ++++++++++++++++++++++ paddle/framework/lod_tensor.h | 32 ++++++++++++ paddle/framework/lod_tensor_test.cc | 78 ++++++++++++++++++----------- 3 files changed, 139 insertions(+), 30 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 87a57d0951..9a115bb609 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -135,6 +135,65 @@ bool operator==(const LoD &a, const LoD &b) { return true; } +bool CheckLoD(const LoD &in, int tensor_height) { + if (in.empty()) return true; + for (const auto &level : in) { + // check: there should be more than 2 offsets existing in each level. + if (level.size() < 2) return false; + // check: the first offset(the begin offset) of each level should be 0. + if (level.front() != 0) return false; + // check: all the offsets in a level should be ascending(no same items + // allows). + if (!std::is_sorted(level.begin(), level.begin(), [](size_t a, size_t b) { + if (a < b) return true; + return false; + })) { + LOG(INFO) << "ascending error"; + return false; + } + } + // check: the lowest level's last offset should equals `tensor_height` if + // tensor_height>0. + if (tensor_height > 0 && (size_t)tensor_height != in.back().back()) + return false; + + // check: the higher level's last offset should equals the lower level's + // size-1. + // NOTE LoD store the levels from top to bottom, so the higher level goes + // first. + for (size_t level = 0; level < in.size() - 1; level++) { + if (in[level].back() != in[level + 1].size() - 1) return false; + } + return true; +} + +bool CheckAbsLoD(const LoD &in, int tensor_height) { + if (in.empty()) return true; + for (const auto &level : in) { + // check: all the offsets in a level should be ascending(no same items + // allows). + if (!std::is_sorted(level.begin(), level.begin(), [](size_t a, size_t b) { + if (a < b) return true; + return false; + })) { + return false; + } + + // check: there should be more than 2 offsets existing in each level. + if (level.size() < 2) return false; + + // check: the first offset of each level should be 0, and the last should be + // the same(the height of underlying tensor). + if (level.front() != 0) return false; + if (tensor_height < 0) { + tensor_height = level.back(); + } else if ((size_t)tensor_height != level.back()) { + return false; + } + } + return true; +} + using LoDAndOffset = std::pair>; LoDAndOffset GetSubLoDAndAbsoluteOffset(const LoD &lod, size_t start_idx, size_t end_idx, size_t start_level) { diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index 88ea78f268..9d1294fdeb 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -71,6 +71,38 @@ LoD ToAbsOffset(const LoD& in); bool operator==(const LoD& a, const LoD& b); +/* + * Check whether this lod's format is valid. + * + * ATTENTION: + * - Empty lod is treated as valid. + * + * It will check two things: + * + * 1. all the offsets in a level should be ascending(no same items allows). + * 2. there should be more than 2 offsets existing in each level. + * 3. the higher level's last offset should equals the lower level's size-1. + * 4. the first offset(the begin offset) of each level should be 0. + * 5. the lowest level's last offset should equals `tensor_height` if + * tensor_height>0. + */ + +bool CheckLoD(const LoD& in, int tensor_height = -1); +/* + * Check whether this absolute lod's format is valid. + * + * ATTENTION: + * - Empty lod is treated as valid. + * + * It will check two things: + * 1. all the offsets in a level should be ascending(no same items allows) + * 2. there should be more than 2 offsets existing in each level. + * 3. the first offset of each level should be 0, and the last should be the + * same(the height of underlying tensor) or `tensor_height` if + * tensor_height>0. + */ +bool CheckAbsLoD(const LoD& in, int tensor_height = -1); + /* * LoDTensor (Level of details Tensor) * see https://en.wikipedia.org/wiki/Level_of_details for reference. diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 19ae7815cc..f718960387 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -37,36 +37,6 @@ namespace framework { const int kLodTensorSize = 20 * 128; -class LoDTensorTester : public ::testing::Test { - public: - virtual void SetUp() override { - // tensor's batch_size: 30 - // 3 levels - // 0 10 20 - // 0 5 10 15 20 - // 0 2 5 7 10 12 15 20 - LoD lod; - lod.push_back(std::vector{0, 2, 3}); - lod.push_back(std::vector{0, 2, 5, 8}); - lod.push_back(std::vector{0, 2, 5, 7, 10, 12, 15, 17, 20}); - - ASSERT_EQ(lod.size(), 3UL); - - lod_tensor_.Resize({20 /*batch size*/, 128 /*dim*/}); - // malloc memory - float* dst_ptr = lod_tensor_.mutable_data(place); - for (int i = 0; i < kLodTensorSize; ++i) { - dst_ptr[i] = i; - } - - lod_tensor_.set_lod(lod); - } - - protected: - platform::CPUPlace place; - LoDTensor lod_tensor_; -}; - TEST(LodExpand, test) { LoD lod{{0, 2}}; LoDTensor tensor; @@ -144,5 +114,53 @@ TEST(LoD, ToAbsOffset) { EXPECT_EQ(abs_lod, expected); } +TEST(LoD, CheckLoD) { + LoD relative_lod; + relative_lod.push_back(std::vector({0, 2})); + relative_lod.push_back(std::vector({0, 1, 3})); + relative_lod.push_back(std::vector({0, 2, 4, 5})); + + // check compatible + ASSERT_TRUE(CheckLoD(relative_lod)); + relative_lod[1].back()++; + ASSERT_FALSE(CheckLoD(relative_lod)); + relative_lod[1].back()--; // recover it + + // check empty + LoD empty_lod; + ASSERT_TRUE(CheckLoD(empty_lod)); + + // check less than 2 offsets in a level + LoD some_lod0; + some_lod0.push_back(std::vector({0})); + ASSERT_FALSE(CheckLoD(some_lod0)); + + // check with underlying tensor storage. + ASSERT_TRUE(CheckLoD(relative_lod, 5)); + ASSERT_FALSE(CheckLoD(relative_lod, 9)); +} + +TEST(LoD, CheckAbsLoD) { + LoD relative_lod; + relative_lod.push_back(std::vector({0, 2})); + relative_lod.push_back(std::vector({0, 1, 3})); + relative_lod.push_back(std::vector({0, 2, 4, 5})); + + auto abs_lod = ToAbsOffset(relative_lod); + + ASSERT_TRUE(CheckAbsLoD(abs_lod)); + + // check less than 2 offsets in a level. + + // check the last item should be compatible with tensor height. + abs_lod.back().back()++; + ASSERT_FALSE(CheckAbsLoD(abs_lod)); + abs_lod.back().back()--; // restore + + // check less than 2 offsets in a lod. + LoD abs_lod0; + abs_lod0.push_back(std::vector({0})); + ASSERT_FALSE(CheckAbsLoD(abs_lod0)); +} } // namespace framework } // namespace paddle From 1e31315088e46121247d8c8700e144a359d3da4d Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 17 Jan 2018 09:59:41 -0800 Subject: [PATCH 063/158] Adding a Code of Conduct for Paddle open source project (#7579) * Adding a Code of Conduct for our open source project * Adding code of conduct in traditional chinese --- CODE_OF_CONDUCT.md | 46 +++++++++++++++++++++++++++++++++++++++ CODE_OF_CONDUCT_cn.md | 50 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CODE_OF_CONDUCT_cn.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..54131b48ec --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at paddle-dev@baidu.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CODE_OF_CONDUCT_cn.md b/CODE_OF_CONDUCT_cn.md new file mode 100644 index 0000000000..c3a22f29cc --- /dev/null +++ b/CODE_OF_CONDUCT_cn.md @@ -0,0 +1,50 @@ +# 貢獻者公約 + +## 我們的承諾 + +為了促進一個開放透明且受歡迎的環境,我們作為貢獻者和維護者保證,無論年齡、種族、民族、性別認同和表達、體型、殘疾、經驗水平、國籍、個人表現、宗教或性別取向,在我們的專案以及社群的參與者都有不被騷擾的體驗。 + +## 我們的準則 + +舉例來說有助於創造正面環境的行為包括: +* 使用歡迎和包容性語言 +* 尊重不同的觀點和經驗 +* 優雅地接受建設性批評 +* 關注在對於社群最好的事情上 +* 對其他社群成員的表現友善 + +舉例來說身為參與者不能接受的行為包括: +* 使用與性有關的言語或是圖像,以及不受歡迎的性騷擾 +* 酸民/反串/釣魚行為或進行侮辱/貶損的評論,人身攻擊及政治攻擊 +* 公開或私下的騷擾 +* 未經許可地發布他人的個人資料,例如住址或是電子地址 +* 其他可以被合理地認定為不恰當或者違反職業操守的行為 + +## 我們的責任 + +專案維護者有責任為"可接受的行為"準則做出詮釋,以及對已發生的不被接受的行為採取恰當且公平的糾正措施。 + +專案維護者有權力及責任去刪除、編輯、拒絕與本行為準則有所違背的評論(comments)、提交(commits)、程式碼、wiki 編輯、問題(issues)和其他貢獻,以及專案維護者可暫時或永久性的禁止任何他們認為有不適當、威脅、冒犯、有害行為的貢獻者。 + +## 使用範圍 + +當一個人代表該專案或是其社群時,本行為準則適用於其專案平台和公共平台。 + +代表專案或是社群的情況,舉例來說包括使用官方專案的電子郵件地址、通過官方的社群媒體帳號發布或線上或線下事件中擔任指定代表。 + +該專案的呈現方式可由其專案維護者進行進一步的定義及解釋。 + +## 強制執行 + +可以透過paddle-dev@baidu.com,來聯繫專案團隊來報告濫用、騷擾或其他不被接受的行為。 + +任何維護團隊認為有必要且適合的所有投訴都將進行審查及調查,並做出相對應的回應。專案小組有對事件回報者有保密的義務。具體執行的方針近一步細節可能會單獨公佈。 + +沒有真誠的遵守或是執行本行為準則的專案維護人員,可能會因專案領導人或是其他成員的決定,暫時或是永久的取消其身份。 + +## 來源 + +本行為準則改編自[貢獻者公約][首頁],版本 1.4 +可在此觀看https://www.contributor-covenant.org/zh-tw/version/1/4/code-of-conduct.html + +[首頁]: https://www.contributor-covenant.org From b870e042564427301f7ed2facdbb38fe2c7b8003 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Thu, 18 Jan 2018 03:55:48 +0800 Subject: [PATCH 064/158] clean lod_tensor_test (#7618) --- paddle/framework/lod_tensor_test.cc | 40 ++++++++++------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index f718960387..9c7ad6c7b4 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -1,28 +1,16 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #include "paddle/framework/lod_tensor.h" @@ -35,8 +23,6 @@ namespace paddle { namespace framework { -const int kLodTensorSize = 20 * 128; - TEST(LodExpand, test) { LoD lod{{0, 2}}; LoDTensor tensor; From c5067a6af00ccb9398b07259b773848b58c242e7 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 18 Jan 2018 03:56:47 +0800 Subject: [PATCH 065/158] "fix hook" (#7616) * "add hook" * "small fix" --- .copyright.hook | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.copyright.hook b/.copyright.hook index de97ce90ac..2446e27248 100644 --- a/.copyright.hook +++ b/.copyright.hook @@ -49,12 +49,12 @@ def generate_copyright(template, lang='C'): LANG_COMMENT_MARK = "//" lines = template.split(NEW_LINE_MARK) - ans = LANG_COMMENT_MARK + COPYRIGHT_HEADER + NEW_LINE_MARK + ans = LANG_COMMENT_MARK + " " + COPYRIGHT_HEADER + NEW_LINE_MARK for lino, line in enumerate(lines): if lino == 0 or lino == 1 or lino == len(lines) - 1: continue - ans += LANG_COMMENT_MARK + line + NEW_LINE_MARK + ans += LANG_COMMENT_MARK + " " + line + NEW_LINE_MARK - return ans + return ans + "\n" def lang_type(filename): From d4d143796fa5af7b74aa2801b4e38704f9c095b7 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 17 Jan 2018 13:14:55 -0800 Subject: [PATCH 066/158] Fix build error --- paddle/operators/detail/grpc_server.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/paddle/operators/detail/grpc_server.cc b/paddle/operators/detail/grpc_server.cc index c0b94746a0..5e35d0cec8 100644 --- a/paddle/operators/detail/grpc_server.cc +++ b/paddle/operators/detail/grpc_server.cc @@ -36,7 +36,10 @@ class RequestBase { CallStatus Status() { return status_; } void SetStatus(CallStatus status) { status_ = status; } - virtual std::string GetReqName() { assert(false); } + virtual std::string GetReqName() { + assert(false); + return ""; + } protected: grpc::ServerContext ctx_; From b8a17987ec92d413a403a00117ce495d22ddd13f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 18 Jan 2018 06:37:51 +0800 Subject: [PATCH 067/158] Feature/parallel for bug fix (#7474) * Fix ParallelDo not support empty input gradient * Polish ParallelDo and fix several bugs * Fix CI * Fix CI --- paddle/framework/lod_tensor.cc | 46 +++--- paddle/operators/parallel_do_op.cc | 135 +++++++++--------- paddle/string/to_string.h | 11 ++ .../paddle/v2/fluid/tests/test_parallel_op.py | 28 ++-- 4 files changed, 128 insertions(+), 92 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 9a115bb609..3e239e9911 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -291,23 +291,32 @@ std::vector LoDTensor::SplitLoDTensor( const std::vector places) const { check_memory_size(); PADDLE_ENFORCE(lod().empty(), "Disable parallel lod for now"); - PADDLE_ENFORCE(dims()[0] % places.size() == 0, - "Batch size should be divided by places size"); - - std::vector lods; - for (size_t place_idx = 0; place_idx < places.size(); ++place_idx) { - int begin = place_idx * dims()[0] / places.size(); - int end = (place_idx + 1) * dims()[0] / places.size(); + size_t result_size = std::min(static_cast(dims()[0]), places.size()); + size_t remainder = dims()[0] % places.size(); + + std::vector results; + results.reserve(result_size); + + int step_width = static_cast(dims()[0] / result_size); + for (size_t i = 0; i < result_size; ++i) { + int begin = static_cast(i * step_width); + int end = static_cast((i + 1) * step_width); + if (i + 1 == places.size()) { // last + end += remainder; + } auto src = Slice(begin, end); - auto &dst_place = places[place_idx]; + auto &dst_place = places[i]; LoDTensor dst; - framework::Copy(src, dst_place, &dst); - - lods.emplace_back(dst); + if (!(dst_place == place())) { + framework::Copy(src, dst_place, &dst); + } else { // It is no need to copy if src_place and dst_place are same. + dst.ShareDataWith(src); + } + results.emplace_back(dst); } - return lods; + return results; } // TODO(tonyyang-svail): make this function support LoD @@ -318,12 +327,17 @@ void LoDTensor::MergeLoDTensor( framework::DDim new_dim = lod_tensors[0]->dims(); std::type_index new_type = lod_tensors[0]->type(); auto new_layout = lod_tensors[0]->layout(); + int64_t new_height = 0; for (auto *lod : lod_tensors) { - PADDLE_ENFORCE(new_dim == lod->dims()); - PADDLE_ENFORCE(new_type == lod->type()); - PADDLE_ENFORCE(new_layout == lod->layout()); + new_height += lod->dims()[0]; + for (int i = 1; i < new_dim.size(); ++i) { + PADDLE_ENFORCE_EQ(new_dim[i], lod->dims()[i]); + } + + PADDLE_ENFORCE_EQ(new_type, lod->type()); + PADDLE_ENFORCE_EQ(new_layout, lod->layout()); } - new_dim[0] *= lod_tensors.size(); + new_dim[0] = new_height; Resize(new_dim); set_layout(new_layout); diff --git a/paddle/operators/parallel_do_op.cc b/paddle/operators/parallel_do_op.cc index e1bec0421e..c2561fa2bf 100644 --- a/paddle/operators/parallel_do_op.cc +++ b/paddle/operators/parallel_do_op.cc @@ -30,16 +30,13 @@ static constexpr char kParallelScopes[] = "parallel_scopes"; static constexpr char kParallelBlock[] = "sub_block"; -// using ParallelScopeVar = std::vector; using LoDTensor = framework::LoDTensor; -using OperatorBase = framework::OperatorBase; -void SplitTensorAndMoveTensorToScopes( - const framework::Scope &scope, - const std::vector &sub_scopes, +static void SplitTensorAndMoveTensorToScopes( + const framework::Scope &scope, std::vector *sub_scopes, const std::vector &places, const std::vector &names) { - PADDLE_ENFORCE_EQ(sub_scopes.size(), places.size()); + size_t num_sub_scopes = 0; for (auto &argu : names) { auto *var = scope.FindVar(argu); const auto &tensor = var->Get(); @@ -48,9 +45,21 @@ void SplitTensorAndMoveTensorToScopes( for (auto &lod : lod_tensors) { VLOG(3) << lod.dims(); } + if (num_sub_scopes == 0) { + num_sub_scopes = lod_tensors.size(); + } else { + PADDLE_ENFORCE_EQ(num_sub_scopes, lod_tensors.size()); + } + PADDLE_ENFORCE_NE(num_sub_scopes, 0); + if (sub_scopes->size() == 0) { + sub_scopes->reserve(num_sub_scopes); + for (size_t i = 0; i < num_sub_scopes; ++i) { + sub_scopes->emplace_back(&scope.NewScope()); + } + } - for (size_t i = 0; i < sub_scopes.size(); ++i) { - *sub_scopes[i]->Var(argu)->GetMutable() = lod_tensors[i]; + for (size_t i = 0; i < lod_tensors.size(); ++i) { + *(*sub_scopes)[i]->Var(argu)->GetMutable() = lod_tensors[i]; } } } @@ -70,7 +79,7 @@ class ParallelDoOp : public framework::OperatorBase { const framework::VariableNameMap &inputs, const framework::VariableNameMap &outputs, const framework::AttributeMap &attrs) - : OperatorBase(type, inputs, outputs, attrs) {} + : framework::OperatorBase(type, inputs, outputs, attrs) {} void Run(const framework::Scope &scope, const platform::Place &place) const override { @@ -85,19 +94,17 @@ class ParallelDoOp : public framework::OperatorBase { auto &sub_scopes = *scope.FindVar(Output(kParallelScopes)) ->GetMutable>(); - for (size_t place_idx = 0; place_idx < places.size(); ++place_idx) { - sub_scopes.push_back(&scope.NewScope()); - } // split input - SplitTensorAndMoveTensorToScopes(scope, sub_scopes, places, + SplitTensorAndMoveTensorToScopes(scope, &sub_scopes, places, Inputs(kInputs)); + // copy parameter for (auto ¶m : Inputs(kParameters)) { PADDLE_ENFORCE(scope.FindVar(param)->IsType(), "Only support parameter type as LoDTensor"); auto &src = scope.FindVar(param)->Get(); - for (size_t i = 0; i < places.size(); ++i) { + for (size_t i = 0; i < sub_scopes.size(); ++i) { auto &place = places[i]; auto *sub_scope = sub_scopes[i]; auto *dst = sub_scope->Var(param)->GetMutable(); @@ -108,9 +115,7 @@ class ParallelDoOp : public framework::OperatorBase { std::vector> workers; workers.reserve(places.size()); - for (size_t place_idx = 0; place_idx < places.size(); ++place_idx) { - VLOG(3) << "Run " << place_idx; - + for (size_t place_idx = 0; place_idx < sub_scopes.size(); ++place_idx) { auto &place = places[place_idx]; auto *cur_scope = sub_scopes[place_idx]; @@ -157,21 +162,16 @@ ParallelDo Operator. } }; -class ParallelDoGradOp : public OperatorBase { +class ParallelDoGradOp : public framework::OperatorBase { public: ParallelDoGradOp(const std::string &type, const framework::VariableNameMap &inputs, const framework::VariableNameMap &outputs, const framework::AttributeMap &attrs) - : OperatorBase(type, inputs, outputs, attrs) {} + : framework::OperatorBase(type, inputs, outputs, attrs) {} void Run(const framework::Scope &scope, const platform::Place &place) const override { - // // get device context from pool - // platform::DeviceContextPool &pool = - // platform::DeviceContextPool::Instance(); - // auto &dev_ctx = *pool.Get(place); - auto *block = Attr(kParallelBlock); auto *program = block->Program(); @@ -181,26 +181,16 @@ class ParallelDoGradOp : public OperatorBase { auto &places = scope.FindVar(Input(kPlaces))->Get(); // feed output@grad - SplitTensorAndMoveTensorToScopes(scope, sub_scopes, places, - Inputs(framework::GradVarName(kOutputs))); + SplitTensorAndMoveTensorToScopes( + scope, const_cast *>(&sub_scopes), + places, Inputs(framework::GradVarName(kOutputs))); WaitOnPlaces(places); - // for debugging - for (auto &s : Inputs(framework::GradVarName(kOutputs))) { - VLOG(3) << s; - VLOG(3) << scope.FindVar(s)->Get(); - for (auto *sub_scope : sub_scopes) { - VLOG(3) << sub_scope->FindVar(s)->Get(); - } - } - // exe run std::vector> workers; - for (size_t place_idx = 0; place_idx < places.size(); ++place_idx) { - VLOG(3) << "Run " << place_idx; - - auto &place = places[place_idx]; - auto *cur_scope = sub_scopes[place_idx]; + for (size_t i = 0; i < sub_scopes.size(); ++i) { + auto &place = places[i]; + auto *cur_scope = sub_scopes[i]; // execute workers.emplace_back(framework::Async([program, cur_scope, place, block] { @@ -216,33 +206,38 @@ class ParallelDoGradOp : public OperatorBase { // merge grad for (auto &s : Outputs(framework::GradVarName(kParameters))) { - VLOG(3) << "merge grad " << s; - - auto &t = sub_scopes[0]->FindVar(s)->Get(); - VLOG(3) << t; - - std::string s_buf = s + "@BUF"; - auto *t_buf = sub_scopes[0]->Var(s_buf)->GetMutable(); - - for (size_t place_idx = 1; place_idx < places.size(); ++place_idx) { - auto &tt = sub_scopes[place_idx]->FindVar(s)->Get(); - VLOG(3) << place_idx; - VLOG(3) << tt; - framework::Copy(tt, places[0], t_buf); + auto &result = sub_scopes[0]->FindVar(s)->Get(); + std::string tmp_name; + auto *tmp = sub_scopes[0]->Var(&tmp_name)->GetMutable(); + + for (size_t i = 1; i < sub_scopes.size(); ++i) { + auto &tensor_to_merge = sub_scopes[i]->FindVar(s)->Get(); + if (!(places[i] == places[0])) { + framework::Copy(tensor_to_merge, places[0], tmp); + } else { + tmp->ShareDataWith(tensor_to_merge); + } auto sum_op = framework::OpRegistry::CreateOp( - "sum", {{"X", {s, s_buf}}}, {{"Out", {s}}}, + "sum", {{"X", {s, tmp_name}}}, {{"Out", {s}}}, framework::AttributeMap{}); sum_op->Run(*sub_scopes[0], places[0]); WaitOnPlaces(places); } - VLOG(3) << t; - framework::Copy(t, place, scope.FindVar(s)->GetMutable()); + VLOG(3) << result; + framework::Copy(result, place, scope.FindVar(s)->GetMutable()); } } }; +std::ostream &operator<<(std::ostream &sout, + const std::vector &strs) { + std::copy(strs.begin(), strs.end(), + std::ostream_iterator(sout, ",")); + return sout; +} + class ParallelDoGradOpDescMaker : public framework::SingleGradOpDescMaker { public: using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; @@ -283,18 +278,30 @@ class ParallelDoGradOpShapeInference : public framework::InferShapeBase { void operator()(framework::InferShapeContext *ctx) const override { std::vector input{kParameters, kInputs}; std::vector output{kOutputs}; - for (auto &s : input) { - PADDLE_ENFORCE(ctx->HasInputs(s)); - PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(s)), - "Cannot find the gradient variable %s", - framework::GradVarName(s)); - } + + PADDLE_ENFORCE(ctx->HasInputs(kParameters)); + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kParameters))); + PADDLE_ENFORCE(ctx->HasInput(kInputs)); + for (auto &s : output) { PADDLE_ENFORCE(ctx->HasInputs(s)); } - for (auto &s : input) { - ctx->SetOutputsDim(framework::GradVarName(s), ctx->GetInputsDim(s)); + + ctx->SetOutputsDim(framework::GradVarName(kParameters), + ctx->GetInputsDim(kParameters)); + + auto i_dims = ctx->GetInputsDim(kInputs); + auto ig_names = ctx->Outputs(framework::GradVarName(kInputs)); + + for (size_t i = 0; i < ig_names.size(); ++i) { + auto &ig_name = ig_names[i]; + if (ig_name == framework::kEmptyVarName) { + continue; + } + + ctx->SetDims({ig_name}, {i_dims[i]}); } + if (ctx->HasInputs(kParameters)) { PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kParameters))); ctx->SetOutputsDim(framework::GradVarName(kParameters), diff --git a/paddle/string/to_string.h b/paddle/string/to_string.h index 3b3bcc69a4..178edc1895 100644 --- a/paddle/string/to_string.h +++ b/paddle/string/to_string.h @@ -15,9 +15,15 @@ limitations under the License. */ #pragma once #include #include +#include namespace paddle { namespace string { +inline std::ostream& operator<<(std::ostream& s, const std::type_index& t) { + s << t.name(); + return s; +} + template inline std::string to_string(T v) { std::ostringstream sout; @@ -25,6 +31,11 @@ inline std::string to_string(T v) { return sout.str(); } +template <> +inline std::string to_string(std::type_index t) { + return t.name(); +} + // Faster std::string/const char* type template <> inline std::string to_string(std::string v) { diff --git a/python/paddle/v2/fluid/tests/test_parallel_op.py b/python/paddle/v2/fluid/tests/test_parallel_op.py index 3c190477d1..45196ef6fe 100644 --- a/python/paddle/v2/fluid/tests/test_parallel_op.py +++ b/python/paddle/v2/fluid/tests/test_parallel_op.py @@ -151,24 +151,28 @@ class BaseParallelForTest(unittest.TestCase): class ParallelOpTest(BaseParallelForTest): - def test_simple_fc(self): - def __network__(): - x = fluid.layers.data(shape=[784], dtype='float32', name='img') - # FIXME: This is a bug of parallel.do - x.stop_gradient = False - x = yield x - hidden = fluid.layers.fc(input=x, size=200, param_attr='fc1.w') - loss = fluid.layers.mean(x=hidden) - yield loss + @staticmethod + def __network__(): + x = fluid.layers.data(shape=[784], dtype='float32', name='img') + x = yield x + hidden = fluid.layers.fc(input=x, size=200, param_attr='fc1.w') + loss = fluid.layers.mean(x=hidden) + yield loss + def test_simple_fc(self): self.run_test( - callback=__network__, + callback=ParallelOpTest.__network__, feed={ - 'img': - numpy.random.random(size=(128 * 3, 784)).astype('float32') + 'img': numpy.random.random(size=(51, 784)).astype('float32') }, fetch='fc1.w@GRAD') + def test_fc_with_tiny_data(self): + self.run_test( + callback=ParallelOpTest.__network__, + feed={'img': numpy.random.random(size=(1, 784)).astype('float32')}, + fetch='fc1.w@GRAD') + if __name__ == '__main__': unittest.main() From 151f579c23c75789726c07b522a45ea6d78670e2 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 17 Jan 2018 14:48:52 -0800 Subject: [PATCH 068/158] Recv OP: use BlockDesc* instread of ProgramDesc proto as Attribute --- paddle/operators/recv_op.cc | 14 +++++++------- paddle/operators/send_recv_op_test.cc | 5 +---- python/paddle/v2/fluid/distribute_transpiler.py | 2 +- .../v2/fluid/distribute_transpiler_simple.py | 2 +- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index f9ed751682..ba0d805110 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -33,6 +33,8 @@ limitations under the License. */ namespace paddle { namespace operators { +constexpr char kOptimizeBlock[] = "OptimizeBlock"; + void RunServer(std::shared_ptr service) { service->RunSyncUpdate(); VLOG(4) << "RunServer thread end"; @@ -150,14 +152,12 @@ class RecvOp : public framework::OperatorBase { rpc_service_->Reset(); - std::string program_str = Attr("OptimizeProgram"); - framework::proto::ProgramDesc program_desc; - program_desc.ParseFromString(program_str); - framework::ProgramDesc program(program_desc); + auto *block = Attr(kOptimizeBlock); + auto *program = block->Program(); framework::Executor executor(dev_place); // Run sub graph to get optimized tensor try { - executor.Run(program, &recv_scope, 0, /*global_block*/ + executor.Run(*program, &recv_scope, block->ID(), /*global_block*/ false /*create_local_scope*/, false /*create_vars*/); } catch (std::exception &e) { LOG(ERROR) << "run sub program error " << e.what(); @@ -189,8 +189,8 @@ This operator will recv tensor from send_op "IP address to listen on.") .SetDefault("127.0.0.1:6164") .AddCustomChecker([](const std::string &ip) { return !ip.empty(); }); - AddAttr("OptimizeProgram", "type string", - "Serialized ProgramDesc string for recv to run."); + AddAttr( + kOptimizeBlock, "Serialized ProgramDesc string for recv to run."); AddAttr>( "ParamList", "type list of string", "grad->param name mapping to find which param to optimize.") diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index ea09169479..045a0f5434 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -130,10 +130,7 @@ void StartServerNet(bool is_sparse) { attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); attrs.insert({"ParamList", std::vector({"Out"})}); attrs.insert({"GradList", std::vector({"x1"})}); - std::string program_proto; - PADDLE_ENFORCE(program.Proto()->SerializeToString(&program_proto)); - - attrs.insert({"OptimizeProgram", program_proto}); + attrs.insert({"OptimizeBlock", block}); recv_op = f::OpRegistry::CreateOp("recv", {{"RX", {"x1"}}}, {}, attrs); recv_op->Run(scope, place); } diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 06a7b6fb02..e1b7e341d0 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -439,7 +439,7 @@ class DistributeTranspiler: }, # grads to recv outputs={}, attrs={ - "OptimizeProgram": optimize_sub_program.desc, + "OptimizeBlock": optimize_sub_program.global_block(), "endpoint": endpoint, "ParamList": [ p.name diff --git a/python/paddle/v2/fluid/distribute_transpiler_simple.py b/python/paddle/v2/fluid/distribute_transpiler_simple.py index bd88f02bde..56ffb56b12 100644 --- a/python/paddle/v2/fluid/distribute_transpiler_simple.py +++ b/python/paddle/v2/fluid/distribute_transpiler_simple.py @@ -243,7 +243,7 @@ class SimpleDistributeTranspiler: self.param_grad_map[endpoint]["grads"]}, # grads to recv outputs={}, attrs={ - "OptimizeProgram": optimize_sub_program.desc, + "OptimizeBlock": optimize_sub_program.global_block(), "endpoint": endpoint, "ParamList": [p.name for p in self.param_grad_map[endpoint]["params"]], From 484af6c7e052e02f9a1001a953c685d13a15977a Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 17 Jan 2018 14:56:30 -0800 Subject: [PATCH 069/158] Fixing the rendering for Reduce operators in operator documentation --- paddle/operators/reduce_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc index 172d28bb3b..09b7091358 100644 --- a/paddle/operators/reduce_op.cc +++ b/paddle/operators/reduce_op.cc @@ -129,7 +129,7 @@ If reduce_all is true, just reduce along all dimensions and output a scalar. } void SetComment(std::string name, std::string op) { - Replace(comment_, "{ReduceOP}", name); + Replace(comment_, "{ReduceOp}", name); Replace(comment_, "{reduce}", op); } }; From f6dfccb6e8d51785f75c2f632831d058d020217d Mon Sep 17 00:00:00 2001 From: Darcy Date: Wed, 17 Jan 2018 15:14:10 -0800 Subject: [PATCH 070/158] Adding fluid distributed training guide doc (#7619) * init check in for fluid dist train doc * gramma update * minor tweaks * update following comments --- .../usage/cluster/fluid_cluster_train_en.md | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 doc/howto/usage/cluster/fluid_cluster_train_en.md diff --git a/doc/howto/usage/cluster/fluid_cluster_train_en.md b/doc/howto/usage/cluster/fluid_cluster_train_en.md new file mode 100644 index 0000000000..419eac51aa --- /dev/null +++ b/doc/howto/usage/cluster/fluid_cluster_train_en.md @@ -0,0 +1,138 @@ +# Fluid Distributed Training + +## Introduction + +In this article, we'll explain how to config and run distributed training jobs with PaddlePaddle Fluid in a bare metal cluster. + +## Preparations + +### Get your cluster ready + +Prepare your computer nodes in the cluster. Nodes in this cluster can be of any specification that runs PaddlePaddle, and with a unique IP address assigned to it. Make sure they can communicate with each other. + +### Have PaddlePaddle installed + +PaddlePaddle must be installed on all nodes. If you have GPU cards on your nodes, be sure to properly install drivers and CUDA libraries. + +PaddlePaddle build and installation guide can be found from [here](http://www.paddlepaddle.org/docs/develop/documentation/en/getstarted/build_and_install/index_en.html). + +### Update training script + +#### Non-cluster training script + +Let's take [Deep Learning 101](http://www.paddlepaddle.org/docs/develop/book/01.fit_a_line/index.html)'s first chapter: "fit a line" as an example. + +This demo's non-cluster version with fluid API is as follows: + +``` python +import paddle.v2 as paddle +import paddle.v2.fluid as fluid + +x = fluid.layers.data(name='x', shape=[13], dtype='float32') +y_predict = fluid.layers.fc(input=x, size=1, act=None) +y = fluid.layers.data(name='y', shape=[1], dtype='float32') + +cost = fluid.layers.square_error_cost(input=y_predict, label=y) +avg_cost = fluid.layers.mean(x=cost) + +sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) +sgd_optimizer.minimize(avg_cost) + +BATCH_SIZE = 20 + +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.uci_housing.train(), buf_size=500), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) +exe = fluid.Executor(place) + +exe.run(fluid.default_startup_program()) + +PASS_NUM = 100 +for pass_id in range(PASS_NUM): + fluid.io.save_persistables(exe, "./fit_a_line.model/") + fluid.io.load_persistables(exe, "./fit_a_line.model/") + for data in train_reader(): + avg_loss_value, = exe.run(fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost]) + + if avg_loss_value[0] < 10.0: + exit(0) # if avg cost less than 10.0, we think our code is good. +exit(1) +``` + +We created a simple fully connected neural networks training program and handed it to the fluid executor to run for 100 passes. + +Now let's try to convert it to a distributed version to run in a cluster. + +#### Introducing parameter server + +As you see from the non-cluster version of training script, there is only one role in it: the trainer, who does the computing as well as holding parameters. In cluster training, since multi-trainers are working on the same task, they need one centralized place to hold and distribute parameters. This centralized place is called the Parameter Server in PaddlePaddle. + +![parameter server architect](src/trainer.png) + +Parameter Server in fluid does not only hold parameters but is also assigned with a part of the program. Trainers communicate with parameter servers via send/receive OPs. For more tech detail, please refer to this [document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/dist_refactor/distributed_architecture.md). + +Now we need to create program for both trainers and parameter servers, the question is how? + +#### Slice the program + +Fluid provides a tool called "Distribute Transpiler" to automatically convert the non-cluster program into cluster program. + +The idea behind this tool is to find optimize OPs and gradient parameters, slice the program into 2 pieces and connect them with send/receive OP. + +Optimize OPs and gradient parameters can be found from the return values of optimizer's minimize function. + +To put them together: + +``` python +... #define the program, cost, and create sgd optimizer + +optimize_ops, params_grads = sgd_optimizer.minimize(avg_cost) #get optimize OPs and gradient parameters + +t = fluid.DistributeTranspiler() # create transpiler instance +# slice the program into 2 pieces with optimizer_ops and gradient parameters list, as well as pserver_endpoints, which is a comma separated list of [IP:PORT] and number of trainers +t.transpile(optimize_ops, params_grads, pservers=pserver_endpoints, trainers=2) + +... #create executor + +# in pserver, run this +exe.run(fluid.default_startup_program()) +#current_endpoint here means current pserver IP:PORT you wish to run on +exe.run(t.get_pserver_program(current_endpoint, optimize_ops)) + +# in trainer, run this +... # define data reader +exe.run(fluid.default_startup_program()) +for pass_id in range(100): + for data in train_reader(): + exe.run(t.get_trainer_program()) + + +``` + +### E2E demo + +Please find the complete demo from [here](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py). In parameter server node run this in the command line: + +``` bash +PSERVERS=192.168.1.2:6174 SERVER_ENDPOINT=192.168.1.2:6174 TRAINING_ROLE=PSERVER python notest_dist_fit_a_line.py +``` + +*please note we assume that your parameter server runs at 192.168.1.2:6174* + +Wait until the prompt `Server listening on 192.168.1.2:6174` + +Then in 2 of your trainer node run this: + +``` bash +PSERVERS=192.168.1.2:6174 SERVER_ENDPOINT=192.168.1.2:6174 TRAINING_ROLE=TRAINER python notest_dist_fit_a_line.py +``` + +*the reason you need to run this command twice in 2 nodes is: in the script we set the trainer count to be 2. You can change this setting on line 50* + +Now you have 2 trainers and 1 parameter server up and running. From dc168ed00a71b7db3149454eee1619cf02ab85ed Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Wed, 17 Jan 2018 15:23:37 -0800 Subject: [PATCH 071/158] modify programDesc based on feed and fetch names --- paddle/inference/example.cc | 18 +++--------------- paddle/inference/inference.cc | 27 +++++++++++++++++++++++++++ paddle/inference/inference.h | 1 + python/paddle/v2/fluid/io.py | 22 ++++++++++++++++++++++ 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/paddle/inference/example.cc b/paddle/inference/example.cc index 9711b20e6f..0c18b45624 100644 --- a/paddle/inference/example.cc +++ b/paddle/inference/example.cc @@ -18,33 +18,21 @@ limitations under the License. */ #include "paddle/inference/inference.h" DEFINE_string(dirname, "", "Directory of the inference model."); -DEFINE_string(feed_var_names, "", "Names of feeding variables"); -DEFINE_string(fetch_var_names, "", "Names of fetching variables"); int main(int argc, char** argv) { google::ParseCommandLineFlags(&argc, &argv, true); - if (FLAGS_dirname.empty() || FLAGS_feed_var_names.empty() || - FLAGS_fetch_var_names.empty()) { + if (FLAGS_dirname.empty()) { // Example: // ./example --dirname=recognize_digits_mlp.inference.model - // --feed_var_names="x" - // --fetch_var_names="fc_2.tmp_2" - std::cout << "Usage: ./example --dirname=path/to/your/model " - "--feed_var_names=x --fetch_var_names=y" - << std::endl; + std::cout << "Usage: ./example --dirname=path/to/your/model" << std::endl; exit(1); } std::cout << "FLAGS_dirname: " << FLAGS_dirname << std::endl; - std::cout << "FLAGS_feed_var_names: " << FLAGS_feed_var_names << std::endl; - std::cout << "FLAGS_fetch_var_names: " << FLAGS_fetch_var_names << std::endl; - std::string dirname = FLAGS_dirname; - std::vector feed_var_names = {FLAGS_feed_var_names}; - std::vector fetch_var_names = {FLAGS_fetch_var_names}; paddle::InferenceEngine* engine = new paddle::InferenceEngine(); - engine->LoadInferenceModel(dirname, feed_var_names, fetch_var_names); + engine->LoadInferenceModel(dirname); paddle::framework::LoDTensor input; srand(time(0)); diff --git a/paddle/inference/inference.cc b/paddle/inference/inference.cc index 37b8b20ddf..1f6e510b8b 100644 --- a/paddle/inference/inference.cc +++ b/paddle/inference/inference.cc @@ -25,6 +25,33 @@ limitations under the License. */ namespace paddle { +void InferenceEngine::LoadInferenceModel(const std::string& dirname) { + std::string model_filename = dirname + "/__model__.dat"; + LOG(INFO) << "loading model from " << model_filename; + std::ifstream inputfs(model_filename, std::ios::in | std::ios::binary); + std::string program_desc_str; + inputfs.seekg(0, std::ios::end); + program_desc_str.resize(inputfs.tellg()); + inputfs.seekg(0, std::ios::beg); + LOG(INFO) << "program_desc_str's size: " << program_desc_str.size(); + inputfs.read(&program_desc_str[0], program_desc_str.size()); + inputfs.close(); + + program_ = new framework::ProgramDesc(program_desc_str); + GenerateLoadProgram(dirname); + + framework::BlockDesc* global_block = program_->MutableBlock(0); + feed_var_names_.clear(); + fetch_var_names_.clear(); + for (auto* op : global_block->AllOps()) { + if (op->Type() == "feed") { + feed_var_names_.insert(feed_var_names_.begin(), op->Output("Out")[0]); + } else if (op->Type() == "fetch") { + fetch_var_names_.push_back(op->Input("X")[0]); + } + } +} + void InferenceEngine::LoadInferenceModel( const std::string& dirname, const std::vector& feed_var_names, diff --git a/paddle/inference/inference.h b/paddle/inference/inference.h index a3f3ef4b44..7fc09cb9e5 100644 --- a/paddle/inference/inference.h +++ b/paddle/inference/inference.h @@ -28,6 +28,7 @@ public: delete load_program_; } + void LoadInferenceModel(const std::string& dirname); void LoadInferenceModel(const std::string& dirname, const std::vector& feed_var_names, const std::vector& fetch_var_names); diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 499df05e59..516b8471df 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -243,6 +243,28 @@ def save_inference_model(dirname, # Save only programDesc of inference_program in binary format # in another file: __model__.dat + global_block = inference_program.global_block() + feed_var = global_blok.create_var( + name='feed', type=core.VarDesc.VarType.FEED_MINIBATCH, persistable=True) + + for i, name in enumerated(feeded_var_names): + out = global_block.var(name) + global_block.prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}) + + fetch_var = global_block.create_var( + name='fetch', type=core.VarDesc.VarType.FETCH_LIST, persistable=True) + + for i, name in enumerated(fetch_var_names): + global_block.append_op( + type='fetch', + inputs={'X': [var]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}) + with open(model_file_name + ".dat", "wb") as fp: fp.write(inference_program.desc.serialize_to_string()) From 28b240bbcf3743d201ae80b508d6d120b6109497 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 17 Jan 2018 23:49:44 +0000 Subject: [PATCH 072/158] delete todo in MergeLoDTensor --- paddle/framework/lod_tensor.cc | 30 ++++++++++++++++------- paddle/framework/lod_tensor_test.cc | 37 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index a6a445efe8..b179e270e1 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -262,24 +262,38 @@ std::vector LoDTensor::SplitLoDTensor( return lods; } -// TODO(tonyyang-svail): make this function support LoD void LoDTensor::MergeLoDTensor( const std::vector &lod_tensors, platform::Place dst_place) { PADDLE_ENFORCE(!lod_tensors.empty()); + framework::DDim new_dim = lod_tensors[0]->dims(); std::type_index new_type = lod_tensors[0]->type(); - auto new_layout = lod_tensors[0]->layout(); - for (auto *lod : lod_tensors) { - PADDLE_ENFORCE(new_dim == lod->dims()); - PADDLE_ENFORCE(new_type == lod->type()); - PADDLE_ENFORCE(new_layout == lod->layout()); + framework::DataLayout new_layout = lod_tensors[0]->layout(); + LoD new_lod = lod_tensors[0]->lod(); + for (size_t i = 1; i < lod_tensors.size(); ++i) { + auto *t = lod_tensors[i]; + PADDLE_ENFORCE_EQ(new_type.hash_code(), t->type().hash_code()); + PADDLE_ENFORCE_EQ(new_layout, t->layout()); + + PADDLE_ENFORCE_EQ(framework::product(new_dim) / new_dim[0], + framework::product(t->dims()) / t->dims()[0]); + new_dim[0] += t->dims()[0]; + + auto &lod = t->lod(); + for (size_t j = 0; j < lod.size(); ++j) { + auto &sub_lod = new_lod[j]; + auto &offset = sub_lod.back(); + for (size_t k = 1; k < lod[j].size(); ++k) { + sub_lod.push_back(lod[j][k] + offset); + } + } } - new_dim[0] *= lod_tensors.size(); Resize(new_dim); set_layout(new_layout); - + set_lod(new_lod); mutable_data(dst_place, new_type); + int begin = 0; for (auto *src : lod_tensors) { int end = begin + src->dims()[0]; diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 5ff7dca564..ed66a98ebd 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -159,5 +159,42 @@ TEST(LoD, SplitLoDTensor) { EXPECT_EQ(lods[1].lod(), lod1); } +TEST(LoD, MergeLoDTensor) { + LoD lod; + lod.push_back(std::vector({0, 2, 4, 5, 6})); + lod.push_back(std::vector({0, 1, 6, 8, 13, 15, 20})); + + platform::CPUPlace place; + + LoDTensor lod_tensor0; + LoD lod0; + lod0.push_back(std::vector({0, 2, 4})); + lod0.push_back(std::vector({0, 1, 6, 8, 13})); + lod_tensor0.set_lod(lod0); + + lod_tensor0.Resize({13, 1}); + float* dst_ptr = lod_tensor0.mutable_data(place); + for (int i = 0; i < lod_tensor0.numel(); ++i) { + dst_ptr[i] = i; + } + + LoDTensor lod_tensor1; + LoD lod1; + lod1.push_back(std::vector({0, 1, 2})); + lod1.push_back(std::vector({0, 2, 7})); + lod_tensor1.set_lod(lod1); + lod_tensor1.Resize({7, 1}); + dst_ptr = lod_tensor1.mutable_data(place); + for (int i = 0; i < lod_tensor1.numel(); ++i) { + dst_ptr[i] = i; + } + + std::vector lods{&lod_tensor0, &lod_tensor1}; + + LoDTensor lod_tensor; + lod_tensor.MergeLoDTensor(lods, place); + EXPECT_EQ(lod_tensor.lod(), lod); +} + } // namespace framework } // namespace paddle From d002f60af9668f7b934f9e4616d96003a3e4f755 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 18 Jan 2018 00:23:46 +0000 Subject: [PATCH 073/158] merge develop --- paddle/framework/lod_tensor.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 3f2c2e64eb..b29f528f3f 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -289,14 +289,15 @@ void DeserializeFromStream(std::istream &is, LoDTensor *tensor, std::vector LoDTensor::SplitLoDTensor( const std::vector places) const { check_memory_size(); - PADDLE_ENFORCE(lod().empty(), "Disable parallel lod for now"); - size_t result_size = std::min(static_cast(dims()[0]), places.size()); - size_t remainder = dims()[0] % places.size(); + int batch_size = + lod().empty() ? dims()[0] : static_cast(lod()[0].size()) - 1; + size_t result_size = std::min(static_cast(batch_size), places.size()); + size_t remainder = batch_size % places.size(); std::vector results; results.reserve(result_size); - int step_width = static_cast(dims()[0] / result_size); + int step_width = static_cast(batch_size / result_size); for (size_t i = 0; i < result_size; ++i) { int begin = static_cast(i * step_width); int end = static_cast((i + 1) * step_width); @@ -307,14 +308,14 @@ std::vector LoDTensor::SplitLoDTensor( LoDTensor dst; if (lod().empty()) { auto src = Slice(begin, end); - auto &dst_place = places[place_idx]; + auto &dst_place = places[i]; framework::Copy(src, dst_place, &dst); } else { auto lod_and_offset = GetSubLoDAndAbsoluteOffset(lod(), begin, end, 0); auto &offset = lod_and_offset.second; auto src = Slice(offset.first, offset.second); - auto &dst_place = places[place_idx]; + auto &dst_place = places[i]; framework::Copy(src, dst_place, &dst); LoD my_lod; @@ -327,7 +328,7 @@ std::vector LoDTensor::SplitLoDTensor( } dst.set_lod(my_lod); } - lods.emplace_back(dst); + results.emplace_back(dst); } return results; From ed3e5717ba97dfca2348a6f29a3180b2c591bb3a Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Wed, 17 Jan 2018 16:56:19 -0800 Subject: [PATCH 074/158] fix bug --- paddle/inference/inference.cc | 2 +- python/paddle/v2/fluid/io.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paddle/inference/inference.cc b/paddle/inference/inference.cc index 1f6e510b8b..4a8bc21b1c 100644 --- a/paddle/inference/inference.cc +++ b/paddle/inference/inference.cc @@ -89,7 +89,7 @@ void InferenceEngine::LoadInferenceModel( } bool InferenceEngine::IsParameter(const framework::VarDesc* var) { - if (var->Persistable()) { + if (var->Persistable() && var->Name() != "feed" && var->Name() != "fetch") { // There are many unreachable variables in the program for (size_t i = 0; i < program_->Size(); ++i) { const framework::BlockDesc& block = program_->Block(i); diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 516b8471df..ad4837c3f7 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -15,6 +15,7 @@ import os import cPickle as pickle from paddle.v2.fluid.framework import Program, Parameter, default_main_program, Variable +from . import core __all__ = [ 'save_vars', @@ -244,10 +245,10 @@ def save_inference_model(dirname, # Save only programDesc of inference_program in binary format # in another file: __model__.dat global_block = inference_program.global_block() - feed_var = global_blok.create_var( + feed_var = global_block.create_var( name='feed', type=core.VarDesc.VarType.FEED_MINIBATCH, persistable=True) - for i, name in enumerated(feeded_var_names): + for i, name in enumerate(feeded_var_names): out = global_block.var(name) global_block.prepend_op( type='feed', @@ -258,10 +259,10 @@ def save_inference_model(dirname, fetch_var = global_block.create_var( name='fetch', type=core.VarDesc.VarType.FETCH_LIST, persistable=True) - for i, name in enumerated(fetch_var_names): + for i, name in enumerate(fetch_var_names): global_block.append_op( type='fetch', - inputs={'X': [var]}, + inputs={'X': [name]}, outputs={'Out': [fetch_var]}, attrs={'col': i}) From e4695457571ea6bb80b2ebbaadc2fa0551d83af7 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 18 Jan 2018 09:14:47 +0800 Subject: [PATCH 075/158] Add Copyright to test_ctc_align.py --- python/paddle/v2/fluid/tests/test_ctc_align.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/paddle/v2/fluid/tests/test_ctc_align.py b/python/paddle/v2/fluid/tests/test_ctc_align.py index 96f45890ee..5a7c16997c 100644 --- a/python/paddle/v2/fluid/tests/test_ctc_align.py +++ b/python/paddle/v2/fluid/tests/test_ctc_align.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + import sys import unittest import numpy as np From e1f475ad1e2fdfbc1628ab063fdaaa037468c44e Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Wed, 17 Jan 2018 17:15:42 -0800 Subject: [PATCH 076/158] refine code --- python/paddle/v2/fluid/io.py | 50 +++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index ad4837c3f7..1b17afbe3f 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -192,6 +192,33 @@ def get_inference_program(target_vars, main_program=None): return inference_program +def prepend_feed_ops(inference_program, feeded_var_names): + global_block = inference_program.global_block() + feed_var = global_block.create_var( + name='feed', type=core.VarDesc.VarType.FEED_MINIBATCH, persistable=True) + + for i, name in enumerate(feeded_var_names): + out = global_block.var(name) + global_block.prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}) + + +def append_fetch_ops(inference_program, fetch_var_names): + global_block = inference_program.global_block() + fetch_var = global_block.create_var( + name='fetch', type=core.VarDesc.VarType.FETCH_LIST, persistable=True) + + for i, name in enumerate(fetch_var_names): + global_block.append_op( + type='fetch', + inputs={'X': [name]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}) + + def save_inference_model(dirname, feeded_var_names, target_vars, @@ -244,27 +271,8 @@ def save_inference_model(dirname, # Save only programDesc of inference_program in binary format # in another file: __model__.dat - global_block = inference_program.global_block() - feed_var = global_block.create_var( - name='feed', type=core.VarDesc.VarType.FEED_MINIBATCH, persistable=True) - - for i, name in enumerate(feeded_var_names): - out = global_block.var(name) - global_block.prepend_op( - type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) - - fetch_var = global_block.create_var( - name='fetch', type=core.VarDesc.VarType.FETCH_LIST, persistable=True) - - for i, name in enumerate(fetch_var_names): - global_block.append_op( - type='fetch', - inputs={'X': [name]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + prepend_feed_ops(inference_program, feeded_var_names) + append_fetch_ops(inference_program, fetch_var_names) with open(model_file_name + ".dat", "wb") as fp: fp.write(inference_program.desc.serialize_to_string()) From 6edbf1387cd97e36749d0c4ccf7b8bee18ebc5b9 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Wed, 17 Jan 2018 17:45:05 -0800 Subject: [PATCH 077/158] remove ptools --- paddle/inference/CMakeLists.txt | 21 --------------------- paddle/inference/inference.cc | 11 +---------- python/paddle/v2/fluid/io.py | 7 +++---- 3 files changed, 4 insertions(+), 35 deletions(-) diff --git a/paddle/inference/CMakeLists.txt b/paddle/inference/CMakeLists.txt index 8437b2b219..02ca8a45a8 100644 --- a/paddle/inference/CMakeLists.txt +++ b/paddle/inference/CMakeLists.txt @@ -8,27 +8,6 @@ cc_library(paddle_fluid_api # Merge all modules into a simgle static library cc_library(paddle_fluid DEPS paddle_fluid_api ${FLUID_CORE_MODULES}) -# ptools -# just for testing, we may need to change the storing format for inference_model -# and move the dependent of pickle. -# download from http://www.picklingtools.com/ -# build in the C++ sub-directory, using command -# make -f Makefile.Linux libptools.so -set(PTOOLS_LIB) -set(PTOOLS_ROOT $ENV{PTOOLS_ROOT} CACHE PATH "Folder contains PicklingTools") -find_path(PTOOLS_INC_DIR chooseser.h PATHS ${PTOOLS_ROOT}/C++) -find_library(PTOOLS_SHARED_LIB NAMES ptools PATHS ${PTOOLS_ROOT}/C++) -if(PTOOLS_INC_DIR AND PTOOLS_SHARED_LIB) - add_definitions(-DPADDLE_USE_PTOOLS) - set(PTOOLS_LIB ptools) - message(STATUS "Found PicklingTools: ${PTOOLS_SHARED_LIB}") - add_library(${PTOOLS_LIB} SHARED IMPORTED GLOBAL) - set_property(TARGET ${PTOOLS_LIB} PROPERTY IMPORTED_LOCATION ${PTOOLS_SHARED_LIB}) - include_directories(${PTOOLS_ROOT}/C++) - include_directories(${PTOOLS_ROOT}/C++/opencontainers_1_8_5/include) - add_definitions(-DOC_NEW_STYLE_INCLUDES) # used in ptools -endif() - add_executable(example example.cc) if(APPLE) set(OPTIONAL_LINK_FLAGS) diff --git a/paddle/inference/inference.cc b/paddle/inference/inference.cc index 4a8bc21b1c..4900177880 100644 --- a/paddle/inference/inference.cc +++ b/paddle/inference/inference.cc @@ -56,15 +56,6 @@ void InferenceEngine::LoadInferenceModel( const std::string& dirname, const std::vector& feed_var_names, const std::vector& fetch_var_names) { -#ifdef PADDLE_USE_PTOOLS - std::string model_filename = dirname + "/__model__"; - LOG(INFO) << "Using PicklingTools, loading model from " << model_filename; - Val v; - LoadValFromFile(model_filename.c_str(), v, SERIALIZE_P0); - std::string program_desc_str = v["program_desc_str"]; - LOG(INFO) << "program_desc_str's size: " << program_desc_str.size(); -// PicklingTools cannot parse the vector of strings correctly. -#else std::string model_filename = dirname + "/__model__.dat"; LOG(INFO) << "loading model from " << model_filename; std::ifstream inputfs(model_filename, std::ios::in | std::ios::binary); @@ -75,7 +66,7 @@ void InferenceEngine::LoadInferenceModel( LOG(INFO) << "program_desc_str's size: " << program_desc_str.size(); inputfs.read(&program_desc_str[0], program_desc_str.size()); inputfs.close(); -#endif + program_ = new framework::ProgramDesc(program_desc_str); GenerateLoadProgram(dirname); diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 1b17afbe3f..57f051985c 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -198,11 +198,10 @@ def prepend_feed_ops(inference_program, feeded_var_names): name='feed', type=core.VarDesc.VarType.FEED_MINIBATCH, persistable=True) for i, name in enumerate(feeded_var_names): - out = global_block.var(name) global_block.prepend_op( type='feed', inputs={'X': [feed_var]}, - outputs={'Out': [out]}, + outputs={'Out': [name]}, attrs={'col': i}) @@ -269,11 +268,11 @@ def save_inference_model(dirname, "fetch_var_names": fetch_var_names }, f, -1) - # Save only programDesc of inference_program in binary format - # in another file: __model__.dat prepend_feed_ops(inference_program, feeded_var_names) append_fetch_ops(inference_program, fetch_var_names) + # Save only programDesc of inference_program in binary format + # in another file: __model__.dat with open(model_file_name + ".dat", "wb") as fp: fp.write(inference_program.desc.serialize_to_string()) From 07e87ff185647aa7351e3bfbe6b2c03f40b34102 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Thu, 18 Jan 2018 10:17:59 +0800 Subject: [PATCH 078/158] Fix sequence_padding compile warning --- paddle/operators/math/sequence_padding.cc | 38 ++++++++++++----------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/paddle/operators/math/sequence_padding.cc b/paddle/operators/math/sequence_padding.cc index fd66455eae..2e69aa47eb 100644 --- a/paddle/operators/math/sequence_padding.cc +++ b/paddle/operators/math/sequence_padding.cc @@ -32,7 +32,8 @@ class PaddingLoDTensorFunctor { framework::LoD abs_offset_lod = framework::ToAbsOffset(lod); auto seq_dims = seq.dims(); - PADDLE_ENFORCE_EQ(seq_dims[0], abs_offset_lod[level].back(), + PADDLE_ENFORCE_EQ(seq_dims[0], + static_cast(abs_offset_lod[level].back()), "The first dimension of LoDTensor seq should be " "equal to the sum of all sequences's length."); @@ -41,32 +42,32 @@ class PaddingLoDTensorFunctor { "The input padding should be a 3-D Tensor of shape " "[max_sequence_length, num_sequences, sequence_width]."); - const size_t max_sequence_length = MaximumSequenceLength(lod, level); + const int64_t max_sequence_length = MaximumSequenceLength(lod, level); PADDLE_ENFORCE_EQ(padding_dims[0], max_sequence_length, "The first dimension of Tensor padding should be the " "maximum length of all sequences in LoDTensor seq."); - const size_t num_sequences = abs_offset_lod[level].size() - 1; + const int64_t num_sequences = abs_offset_lod[level].size() - 1; PADDLE_ENFORCE_EQ(padding_dims[1], num_sequences, "The second dimension of Tensor padding should be the " "number of sequences in LoDTensor seq."); - const size_t sequence_width = seq.numel() / seq_dims[0]; + const int64_t sequence_width = seq.numel() / seq_dims[0]; PADDLE_ENFORCE_EQ(padding_dims[2], sequence_width, "The third dimension of Tensor padding should be the " "width of sequence in LoDTensor seq."); const T* seq_data = seq.data(); T* padding_data = padding.data(); - for (size_t i = 0; i < max_sequence_length; ++i) { - for (size_t j = 0; j < num_sequences; ++j) { - size_t start_pos = abs_offset_lod[level][j]; - size_t sequence_length = abs_offset_lod[level][j + 1] - start_pos; + for (int64_t i = 0; i < max_sequence_length; ++i) { + for (int64_t j = 0; j < num_sequences; ++j) { + int64_t start_pos = abs_offset_lod[level][j]; + int64_t sequence_length = abs_offset_lod[level][j + 1] - start_pos; if (i < sequence_length) { // i > 0 => sequence_length > 0 T scale = norm_by_times ? (1.0f / static_cast(sequence_length)) : 1.0f; - for (size_t k = 0; k < sequence_width; ++k) { + for (int64_t k = 0; k < sequence_width; ++k) { padding_data[(i * num_sequences + j) * sequence_width + k] = seq_data[(start_pos + i) * sequence_width + k] * scale; } @@ -93,7 +94,8 @@ class UnpaddingLoDTensorFunctor { framework::LoD abs_offset_lod = framework::ToAbsOffset(lod); auto seq_dims = seq.dims(); - PADDLE_ENFORCE_EQ(seq_dims[0], abs_offset_lod[level].back(), + PADDLE_ENFORCE_EQ(seq_dims[0], + static_cast(abs_offset_lod[level].back()), "The first dimension of LoDTensor seq should be " "equal to the sum of all sequences's length."); @@ -102,31 +104,31 @@ class UnpaddingLoDTensorFunctor { "The input padding should be a 3-D Tensor of shape " "[max_sequnece_length, num_sequences, sequence_width]."); - const size_t max_sequence_length = MaximumSequenceLength(lod, level); + const int64_t max_sequence_length = MaximumSequenceLength(lod, level); PADDLE_ENFORCE_EQ(padding_dims[0], max_sequence_length, "The first dimension of Tensor padding should be " "the maximum length of all sequences in LoDTensor seq."); - const size_t num_sequences = abs_offset_lod[level].size() - 1; + const int64_t num_sequences = abs_offset_lod[level].size() - 1; PADDLE_ENFORCE_EQ(padding_dims[1], num_sequences, "The second dimension of Tensor padding should be " "the number of sequences in LoDTensor seq."); - const size_t sequence_width = seq.numel() / seq_dims[0]; + const int64_t sequence_width = seq.numel() / seq_dims[0]; PADDLE_ENFORCE_EQ(padding_dims[2], sequence_width, "The third dimension of Tensor padding should be the " "width of sequence in LoDTensor seq."); const T* padding_data = padding.data(); T* seq_data = seq.data(); - for (size_t i = 0; i < num_sequences; ++i) { - size_t start_pos = abs_offset_lod[level][i]; - size_t sequence_length = abs_offset_lod[level][i + 1] - start_pos; - for (size_t j = 0; j < sequence_length; ++j) { + for (int64_t i = 0; i < num_sequences; ++i) { + int64_t start_pos = abs_offset_lod[level][i]; + int64_t sequence_length = abs_offset_lod[level][i + 1] - start_pos; + for (int64_t j = 0; j < sequence_length; ++j) { // sequence_width > j > 0 T scale = norm_by_times ? (1.0f / static_cast(sequence_length)) : 1.0f; - for (size_t k = 0; k < sequence_width; ++k) { + for (int64_t k = 0; k < sequence_width; ++k) { seq_data[(start_pos + j) * sequence_width + k] = padding_data[(j * num_sequences + i) * sequence_width + k] * scale; From 856f650a9be1e59201f561d4784e166405e99a90 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Wed, 17 Jan 2018 18:25:00 -0800 Subject: [PATCH 079/158] fix bug --- python/paddle/v2/fluid/io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 57f051985c..e7a06a0714 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -198,10 +198,11 @@ def prepend_feed_ops(inference_program, feeded_var_names): name='feed', type=core.VarDesc.VarType.FEED_MINIBATCH, persistable=True) for i, name in enumerate(feeded_var_names): + out = global_block.var(name) global_block.prepend_op( type='feed', inputs={'X': [feed_var]}, - outputs={'Out': [name]}, + outputs={'Out': [out]}, attrs={'col': i}) From f050390754b480698f37739c508db413726acf04 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 17 Jan 2018 19:20:11 -0800 Subject: [PATCH 080/158] Polish the doc of dynamic_lstm --- python/paddle/v2/fluid/layers/nn.py | 129 +++++++++++++--------------- 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 7759ce6af6..faaa68c918 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -233,99 +233,94 @@ def dynamic_lstm(input, The defalut implementation is diagonal/peephole connection (https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: - .. math: - - i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) \\ + .. math:: + + i_t & = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) - f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) \\ + f_t & = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) - \tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) \\ + \\tilde{c_t} & = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) - o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) \\ + o_t & = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) - c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\ + c_t & = f_t \odot c_{t-1} + i_t \odot \\tilde{c_t} - h_t = o_t \odot act_h(c_t) + h_t & = o_t \odot act_h(c_t) - where the W terms denote weight matrices (e.g. $W_{xi}$ is the matrix - of weights from the input gate to the input), $W_{ic}, W_{fc}, W_{oc}$ + where the :math:`W` terms denote weight matrices (e.g. :math:`W_{xi}` is the matrix + of weights from the input gate to the input), :math:`W_{ic}, W_{fc}, W_{oc}` are diagonal weight matrices for peephole connections. In our implementation, - we use vectors to reprenset these diagonal weight matrices. The b terms - denote bias vectors ($b_i$ is the input gate bias vector), $\sigma$ + we use vectors to reprenset these diagonal weight matrices. The :math:`b` terms + denote bias vectors (:math:`b_i` is the input gate bias vector), :math:`\sigma` is the non-line activations, such as logistic sigmoid function, and - $i, f, o$ and $c$ are the input gate, forget gate, output gate, + :math:`i, f, o` and :math:`c` are the input gate, forget gate, output gate, and cell activation vectors, respectively, all of which have the same size as - the cell output activation vector $h$. + the cell output activation vector :math:`h`. - The $\odot$ is the element-wise product of the vectors. $act_g$ and $act_h$ + The :math:`\odot` is the element-wise product of the vectors. :math:`act_g` and :math:`act_h` are the cell input and cell output activation functions and `tanh` is usually - used for them. $\tilde{c_t}$ is also called candidate hidden state, + used for them. :math:`\\tilde{c_t}` is also called candidate hidden state, which is computed based on the current input and the previous hidden state. Set `use_peepholes` False to disable peephole connection. The formula is omitted here, please refer to the paper http://www.bioinf.jku.at/publications/older/2604.pdf for details. - Note that these $W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}$ - operations on the input $x_{t}$ are NOT included in this operator. - Users can choose to use fully-connect operator before LSTM operator. + Note that these :math:`W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}` + operations on the input :math:`x_{t}` are NOT included in this operator. + Users can choose to use fully-connect layer before LSTM layer. Args: -def dynamic_lstm(input, - size, - param_attr=None, - bias_attr=None, - use_peepholes=True, - is_reverse=False, - gate_activation='sigmoid', - cell_activation='tanh', - candidate_activation='tanh', - dtype='float32'): - input(Variable): The input of dynamic_lstm layer, which support - variable-time length input sequence. The underlying tensor in - this Variable is a matrix with shape (T X 4D), where T is the - total time steps in this mini-batch, D is the hidden size. - size(int): The size of input. + input(Variable): The input of dynamic_lstm layer, which supports + variable-time length input sequence. The underlying + tensor in this Variable is a matrix with shape + (T X 4D), where T is the total time steps in this + mini-batch, D is the hidden size. + size(int): 4 * hidden size. param_attr(ParamAttr): The parameter attribute for the learnable - hidden-hidden weights. - - The shape is (D x 4D), where D is the hidden size. - - param_attr = {W_ch, W_ih, W_fh, W_oh} + hidden-hidden weights. + + - The shape is (D x 4D), where D is the hidden + size. + - Weights = {:math:`W_{ch}, W_{ih}, \ + W_{fh}, W_{oh}`} bias_attr(ParamAttr): The bias attribute for the learnable bias - weights, which contains two parts: input-hidden bias weight - and peephole connections weight if setting `use_peepholes` to True. - 1. `use_peepholes = False` - - The shape is (1 x 4D). - - Bias = {b_c, b_i, b_f, b_o}. - 2. `use_peepholes = True` - - The shape is (1 x 7D). - - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}. - use_peepholes(bool, defalut: True): whether to enable diagonal/peephole - connections. - is_reverse(bool, defalut: False): whether to compute reversed LSTM. - gate_activation(string, choices: "sigmoid", "tanh", "relu", "identity", - default: "sigmoid"): The activation for input gate, forget gate and - output gate. - cell_activation(string, choices: "sigmoid", "tanh", "relu", "identity", - default: "tanh"): The activation for cell output. - candidate_activation(string, choices: "sigmoid", "tanh", "relu", - "identity", default: "tanh"): The activation for candidate hidden - state. - dtype(string, ) + weights, which contains two parts, input-hidden + bias weights and peephole connections weights if + setting `use_peepholes` to `True`. + + 1. `use_peepholes = False` + - The shape is (1 x 4D). + - Biases = {:math:`b_c, b_i, b_f, b_o`}. + 2. `use_peepholes = True` + - The shape is (1 x 7D). + - Biases = { :math:`b_c, b_i, b_f, b_o, W_{ic}, \ + W_{fc}, W_{oc}`}. + use_peepholes(bool): Whether to enable diagonal/peephole connections, + default `True`. + is_reverse(bool): Whether to compute reversed LSTM, default `False`. + gate_activation(str): The activation for input gate, forget gate and + output gate. Choices = ["sigmoid", "tanh", "relu", + "identity"], default "sigmoid". + cell_activation(str): The activation for cell output. Choices = ["sigmoid", + "tanh", "relu", "identity"], default "tanh". + candidate_activation(str): The activation for candidate hidden state. + Choices = ["sigmoid", "tanh", "relu", "identity"], + default "tanh". + dtype(str): Data type. Choices = ["float32", "float64"], default "float32". Returns: - hidden(Variable): the hidden state of LSTM layer. The shape is (T x D), - and lod is the same with the `input`. - cell(Variable): the cell state of LSTM layer. The shape is (T x D), and - lod is the same with the `input`. + tuple: The hidden state, and cell state of LSTM. The shape of both \ + is (T x D), and lod is the same with the `input`. - Example: + Examples: .. code-block:: python - hidden_dim = 512 - forward_proj = fluid.layers.fc(input=input_seq, size=hidden_dim * 4, - act='tanh', bias_attr=True) - forward, _ = fluid.layers.dynamic_lstm( - input=forward_proj, size=hidden_dim * 4, use_peepholes=False) + hidden_dim = 512 + forward_proj = fluid.layers.fc(input=input_seq, size=hidden_dim * 4, + act='tanh', bias_attr=True) + forward, _ = fluid.layers.dynamic_lstm( + input=forward_proj, size=hidden_dim * 4, use_peepholes=False) """ helper = LayerHelper('lstm', **locals()) size = size / 4 From 9bd9d8b5ca96ff442a7ba3a3df0564e414c11af5 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Thu, 18 Jan 2018 11:29:09 +0800 Subject: [PATCH 081/158] Add sequence_reshape_op. --- paddle/operators/sequence_reshape_op.cc | 78 +++++++++++++++ paddle/operators/sequence_reshape_op.h | 127 ++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 paddle/operators/sequence_reshape_op.cc create mode 100644 paddle/operators/sequence_reshape_op.h diff --git a/paddle/operators/sequence_reshape_op.cc b/paddle/operators/sequence_reshape_op.cc new file mode 100644 index 0000000000..31a970354f --- /dev/null +++ b/paddle/operators/sequence_reshape_op.cc @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/sequence_reshape_op.h" + +namespace paddle { +namespace operators { + +class SequenceReshapeOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SequenceReshapeOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SequenceReshapeOp should not be null."); + auto x_dims = ctx->GetInputDim("X"); + PADDLE_ENFORCE_EQ(x_dims.size(), 2U, "Rank of Input(X) should be 2."); + int dimension = ctx->Attrs().Get("dimension"); + ctx->SetOutputDim("Out", {{x_dims[0], static_cast(dimension)}}); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class SequenceReshapeOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SequenceReshapeOpMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddOutput("Out", ""); + AddAttr("dimension", ""); + AddAttr("is_padding", "Default padding zero."); + AddComment(R"DOC()DOC"); + } +}; + +class SequenceReshapeGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE( + ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) of SequenceReshapeGradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Out"), + "Input(Out) of SequenceReshapeGradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SequenceReshapeGradOp should not be null."); + + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ framework::GradVarName("X")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(sequence_reshape, ops::SequenceReshapeOp, + ops::SequenceReshapeOpMaker); +REGISTER_OPERATOR(sequence_reshape_grad, ops::SequenceReshapeGradOp); +REGISTER_OP_CPU_KERNEL( + sequence_reshape, + ops::SequenceReshapeKernel); +REGISTER_OP_CPU_KERNEL( + sequence_reshape_grad, + ops::SequenceReshapeGradKernel); diff --git a/paddle/operators/sequence_reshape_op.h b/paddle/operators/sequence_reshape_op.h new file mode 100644 index 0000000000..bc7694b6b1 --- /dev/null +++ b/paddle/operators/sequence_reshape_op.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using LoDTensor = framework::LoDTensor; +template +class SequenceReshapeKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in = context.Input("X"); + auto* out = context.Output("Out"); + int out_width = context.Attr("dimension"); + bool whether_padding = context.Attr("whether_padding"); + + const T* p_in_data = in->data(); + T* p_out_data = out->mutable_data(context.GetPlace()); + + // compute shape for output + auto in_dims = in->dims(); + int64_t in_width = in_dims[1]; + auto& in_lod = in->lod(); + + PADDLE_ENFORCE_EQ(in_lod.size(), 1UL, + "Only support one level sequence now."); + PADDLE_ENFORCE_GE( + in_dims[0], + /* batch size = */ static_cast(in_lod[0].size() - 1), + "The 1st dimension of Input(X) must be equal or larger than batch " + "size."); + + auto in_lod_l0 = in_lod[0]; + int seq_num = in_lod_l0.size() - 1; + + auto& out_lod = *out->mutable_lod(); + out_lod.push_back(std::vector({0})); + size_t offset = 0; + for (int i = 0; i < seq_num; ++i) { + size_t seq_len = in_lod_l0[i + 1] - in_lod_l0[i]; + if (whether_padding) { + offset += std::ceil((float)(seq_len * in_width) / out_width); + } else { + offset += (seq_len * in_width) / out_width; + } + out_lod[0].push_back(offset); + } + + out->Resize({{static_cast(out_lod[0].back()), out_width}}); + math::set_constant(context.device_context(), out, 0.0f); + + for (int i = 0; i < seq_num; ++i) { + size_t in_offset = in_lod_l0[i] * in_width; + size_t out_offset = out_lod[0][i] * out_width; + size_t bytes = sizeof(T) * (in_lod_l0[i + 1] - in_lod_l0[i]) * in_width; + if (platform::is_cpu_place(context.GetPlace())) { + std::memcpy(p_out_data + out_offset, p_in_data + in_offset, bytes); + } else { +#ifdef PADDLE_WITH_CUDA + auto& dev_ctx = context.template device_context(); + memory::Copy(boost::get(context.GetPlace()), + p_out_data + out_offset, + boost::get(context.GetPlace()), + p_in_data + in_offset, bytes, dev_ctx.stream()); +#endif + } + } + } +}; + +template +class SequenceReshapeGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x_tensor_ptr = context.Input("X"); + auto* out_tensor_ptr = context.Input("Out"); + auto* out_grad_tensor_ptr = + context.Input(framework::GradVarName("Out")); + auto* x_grad_tensor_ptr = + context.Output(framework::GradVarName("X")); + + T* p_x_grad_data = x_grad_tensor_ptr->mutable_data(context.GetPlace()); + const T* p_out_grad_data = out_grad_tensor_ptr->data(); + + auto& x_lod = x_tensor_ptr->lod(); + int seq_num = x_lod[0].size() - 1; + int x_width = x_tensor_ptr->dims()[1]; + auto& out_lod = out_tensor_ptr->lod(); + int out_width = out_tensor_ptr->dims()[1]; + + for (int i = 0; i < seq_num; ++i) { + size_t src_offset = out_lod[0][i] * out_width; + size_t dst_offset = x_lod[0][i] * x_width; + size_t bytes = sizeof(T) * (x_lod[0][i + 1] - x_lod[0][i]) * x_width; + if (platform::is_cpu_place(context.GetPlace())) { + std::memcpy(p_x_grad_data + dst_offset, p_out_grad_data + src_offset, + bytes); + } else { +#ifdef PADDLE_WITH_CUDA + auto& dev_ctx = context.template device_context(); + memory::Copy(boost::get(context.GetPlace()), + p_x_grad_data + dst_offset, + boost::get(context.GetPlace()), + p_out_grad_data + src_offset, bytes, dev_ctx.stream()); +#endif + } + } + } +}; + +} // namespace operators +} // namespace paddle From 3b0eff6196d1adc01a48eb87c06c59c40504704e Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 17 Jan 2018 20:13:15 -0800 Subject: [PATCH 082/158] Format the writing in doc of dynamic_lstm --- python/paddle/v2/fluid/layers/nn.py | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index d9fe1ca0ac..ebcc914f60 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -249,22 +249,23 @@ def dynamic_lstm(input, h_t & = o_t \odot act_h(c_t) - where the :math:`W` terms denote weight matrices (e.g. :math:`W_{xi}` is the matrix - of weights from the input gate to the input), :math:`W_{ic}, W_{fc}, W_{oc}` - are diagonal weight matrices for peephole connections. In our implementation, - we use vectors to reprenset these diagonal weight matrices. The :math:`b` terms - denote bias vectors (:math:`b_i` is the input gate bias vector), :math:`\sigma` - is the non-line activations, such as logistic sigmoid function, and - :math:`i, f, o` and :math:`c` are the input gate, forget gate, output gate, - and cell activation vectors, respectively, all of which have the same size as - the cell output activation vector :math:`h`. - - The :math:`\odot` is the element-wise product of the vectors. :math:`act_g` and :math:`act_h` - are the cell input and cell output activation functions and `tanh` is usually - used for them. :math:`\\tilde{c_t}` is also called candidate hidden state, - which is computed based on the current input and the previous hidden state. - - Set `use_peepholes` False to disable peephole connection. The formula + where the :math:`W` terms denote weight matrices (e.g. :math:`W_{xi}` is + the matrix of weights from the input gate to the input), :math:`W_{ic}, \ + W_{fc}, W_{oc}` are diagonal weight matrices for peephole connections. In + our implementation, we use vectors to reprenset these diagonal weight + matrices. The :math:`b` terms denote bias vectors (:math:`b_i` is the input + gate bias vector), :math:`\sigma` is the non-line activations, such as + logistic sigmoid function, and :math:`i, f, o` and :math:`c` are the input + gate, forget gate, output gate, and cell activation vectors, respectively, + all of which have the same size as the cell output activation vector :math:`h`. + + The :math:`\odot` is the element-wise product of the vectors. :math:`act_g` + and :math:`act_h` are the cell input and cell output activation functions + and `tanh` is usually used for them. :math:`\\tilde{c_t}` is also called + candidate hidden state, which is computed based on the current input and + the previous hidden state. + + Set `use_peepholes` to `False` to disable peephole connection. The formula is omitted here, please refer to the paper http://www.bioinf.jku.at/publications/older/2604.pdf for details. From 83bce37680c79bdee7559d7562afa5235218b2c7 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 18 Jan 2018 14:03:10 +0800 Subject: [PATCH 083/158] change CODE_OF_CONDUCT_cn.md from Traditional Chinese to Simplified Chinese --- CODE_OF_CONDUCT_cn.md | 62 +++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/CODE_OF_CONDUCT_cn.md b/CODE_OF_CONDUCT_cn.md index c3a22f29cc..2be794f1f3 100644 --- a/CODE_OF_CONDUCT_cn.md +++ b/CODE_OF_CONDUCT_cn.md @@ -1,50 +1,50 @@ -# 貢獻者公約 +# 参与者公约 -## 我們的承諾 +## 我们的保证 -為了促進一個開放透明且受歡迎的環境,我們作為貢獻者和維護者保證,無論年齡、種族、民族、性別認同和表達、體型、殘疾、經驗水平、國籍、個人表現、宗教或性別取向,在我們的專案以及社群的參與者都有不被騷擾的體驗。 +为了促进一个开放透明且友好的环境,我们作为贡献者和维护者保证:无论年龄、种族、民族、性别认同和表达(方式)、体型、身体健全与否、经验水平、国籍、个人表现、宗教或性别取向,参与者在我们项目和社区中都免于骚扰。 -## 我們的準則 +## 我们的标准 -舉例來說有助於創造正面環境的行為包括: -* 使用歡迎和包容性語言 -* 尊重不同的觀點和經驗 -* 優雅地接受建設性批評 -* 關注在對於社群最好的事情上 -* 對其他社群成員的表現友善 +有助于创造正面环境的行为包括但不限于: +* 使用友好和包容性语言 +* 尊重不同的观点和经历 +* 耐心地接受建设性批评 +* 关注对社区最有利的事情 +* 友善对待其他社区成员 -舉例來說身為參與者不能接受的行為包括: -* 使用與性有關的言語或是圖像,以及不受歡迎的性騷擾 -* 酸民/反串/釣魚行為或進行侮辱/貶損的評論,人身攻擊及政治攻擊 -* 公開或私下的騷擾 -* 未經許可地發布他人的個人資料,例如住址或是電子地址 -* 其他可以被合理地認定為不恰當或者違反職業操守的行為 +身为参与者不能接受的行为包括但不限于: +* 使用与性有关的言语或是图像,以及不受欢迎的性骚扰 +* 捣乱/煽动/造谣的行为或进行侮辱/贬损的评论,人身攻击及政治攻击 +* 公开或私下的骚扰 +* 未经许可地发布他人的个人资料,例如住址或是电子地址 +* 其他可以被合理地认定为不恰当或者违反职业操守的行为 -## 我們的責任 +## 我们的责任 -專案維護者有責任為"可接受的行為"準則做出詮釋,以及對已發生的不被接受的行為採取恰當且公平的糾正措施。 +项目维护者有责任为「可接受的行为」标准做出诠释,以及对已发生的不被接受的行为采取恰当且公平的纠正措施。 -專案維護者有權力及責任去刪除、編輯、拒絕與本行為準則有所違背的評論(comments)、提交(commits)、程式碼、wiki 編輯、問題(issues)和其他貢獻,以及專案維護者可暫時或永久性的禁止任何他們認為有不適當、威脅、冒犯、有害行為的貢獻者。 +项目维护者有权利及责任去删除、编辑、拒绝与本行为标准有所违背的评论(comments)、提交(commits)、代码、wiki 编辑、问题(issues)和其他贡献,以及项目维护者可暂时或永久性的禁止任何他们认为有不适当、威胁、冒犯、有害行为的贡献者。 -## 使用範圍 +## 使用范围 -當一個人代表該專案或是其社群時,本行為準則適用於其專案平台和公共平台。 +当一个人代表该项目或是其社区时,本行为标准适用于其项目平台和公共平台。 -代表專案或是社群的情況,舉例來說包括使用官方專案的電子郵件地址、通過官方的社群媒體帳號發布或線上或線下事件中擔任指定代表。 +代表项目或是社区的情况,举例来说包括使用官方项目的电子邮件地址、通过官方的社区媒体账号发布或线上或线下事件中担任指定代表。 -該專案的呈現方式可由其專案維護者進行進一步的定義及解釋。 +该项目的呈现方式可由其项目维护者进行进一步的定义及解释。 -## 強制執行 +## 强制执行 -可以透過paddle-dev@baidu.com,來聯繫專案團隊來報告濫用、騷擾或其他不被接受的行為。 +可以通过paddle-dev@baidu.com,来联系项目团队来举报滥用、骚扰或其他不被接受的行为。 -任何維護團隊認為有必要且適合的所有投訴都將進行審查及調查,並做出相對應的回應。專案小組有對事件回報者有保密的義務。具體執行的方針近一步細節可能會單獨公佈。 +任何维护团队认为有必要且适合的所有投诉都将进行审查及调查,并做出相对应的回应。项目小组有对事件回报者有保密的义务。具体执行的方针近一步细节可能会单独公布。 -沒有真誠的遵守或是執行本行為準則的專案維護人員,可能會因專案領導人或是其他成員的決定,暫時或是永久的取消其身份。 +没有切实地遵守或是执行本行为标准的项目维护人员,可能会因项目领导人或是其他成员的决定,暂时或是永久地取消其参与资格。 -## 來源 +## 来源 -本行為準則改編自[貢獻者公約][首頁],版本 1.4 -可在此觀看https://www.contributor-covenant.org/zh-tw/version/1/4/code-of-conduct.html +本行为标准改编自[贡献者公约][主页],版本 1.4 +可在此观看https://www.contributor-covenant.org/zh-cn/version/1/4/code-of-conduct.html -[首頁]: https://www.contributor-covenant.org +[主页]: https://www.contributor-covenant.org From 3388e52da12b6f525e34feadb05d4a712bd1d54e Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Thu, 18 Jan 2018 14:32:03 +0800 Subject: [PATCH 084/158] Bugfix/beamsearch op (#7611) --- paddle/operators/CMakeLists.txt | 3 +- paddle/operators/beam_search_op.cc | 23 +++++-- paddle/operators/beam_search_op.h | 21 ++++-- paddle/operators/beam_search_op_test.cc | 86 +++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 13 deletions(-) create mode 100644 paddle/operators/beam_search_op_test.cc diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 2412ebd82a..6745a8da17 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -178,14 +178,13 @@ foreach(src ${GENERAL_OPS}) endforeach() file(APPEND ${pybind_file} "USE_OP(less_than);\nUSE_OP(logical_and);\nUSE_NO_KERNEL_OP(read_from_array);\n") - set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") - cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) cc_test(beam_search_decode_op_test SRCS beam_search_decode_op_test.cc DEPS lod_tensor) +cc_test(beam_search_op_test SRCS beam_search_op_test.cc DEPS lod_tensor beam_search_op) cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) if(WITH_GPU) cc_test(nccl_op_test SRCS nccl_op_test.cu.cc DEPS nccl_op gpu_info device_context) diff --git a/paddle/operators/beam_search_op.cc b/paddle/operators/beam_search_op.cc index ed2e7b738a..4c71d66d22 100644 --- a/paddle/operators/beam_search_op.cc +++ b/paddle/operators/beam_search_op.cc @@ -29,7 +29,7 @@ void BeamSearch::operator()(const framework::LoDTensor &pre_ids, PruneEndidCandidates(pre_ids, &selected_items); // calculate the output tensor's height size_t num_instances = std::accumulate( - std::begin(items), std::end(items), 0, + std::begin(selected_items), std::end(selected_items), 0, [](size_t a, std::vector &b) { return a + b.size(); }); // the output tensor shape should be [num_instances, 1] auto dims = framework::make_ddim( @@ -48,12 +48,20 @@ void BeamSearch::operator()(const framework::LoDTensor &pre_ids, size_t low_offset = 0; for (auto &items : selected_items) { low_level.push_back(low_offset); + sort(items.begin(), items.end(), [](const Item &a, const Item &b) { + if (a.offset < b.offset) { + return true; + } + return a.id < b.id; + }); for (auto &item : items) { ids_data[low_offset] = item.id; scores_data[low_offset] = item.score; low_offset++; } } + low_level.push_back(low_offset); + // fill lod auto abs_lod = framework::ToAbsOffset(ids_->lod()); auto &high_level = abs_lod[lod_level_]; @@ -64,16 +72,21 @@ void BeamSearch::operator()(const framework::LoDTensor &pre_ids, selected_scores->set_lod(lod); } -void BeamSearch::PruneEndidCandidates(const framework::LoDTensor &pre_ids, - std::vector> *items) { +int BeamSearch::PruneEndidCandidates(const framework::LoDTensor &pre_ids, + std::vector> *items) { auto *pre_ids_data = pre_ids.data(); + int res = 0; for (size_t offset = 0; offset < items->size(); offset++) { auto prefix_id = pre_ids_data[offset]; if (prefix_id == end_id_) { items->at(offset).clear(); + } else { + res++; } } + + return res; } std::vector> BeamSearch::ToMap( @@ -121,11 +134,7 @@ bool BeamSearch::NextItemSet(std::vector *items) { auto ids = *ids_; auto scores = *scores_; - auto source_abs_two_level_lod = framework::SliceInLevel( - ids.lod(), lod_level_, sent_offset_, sent_offset_ + 1); - source_abs_two_level_lod = framework::ToAbsOffset(source_abs_two_level_lod); auto abs_lod = framework::ToAbsOffset(ids.lod()); - PADDLE_ENFORCE_GE(source_abs_two_level_lod.size(), 2UL); auto *ids_data = ids.data(); auto *scores_data = scores.data(); diff --git a/paddle/operators/beam_search_op.h b/paddle/operators/beam_search_op.h index 08b551ef9b..45d14d68fe 100644 --- a/paddle/operators/beam_search_op.h +++ b/paddle/operators/beam_search_op.h @@ -73,7 +73,15 @@ namespace operators { * second level: * [0, 2, 4] * - * tensor's data + * id tensor's data + * [[ + * 4, + * 1, + * 3, + * 8, + * ]] + * + * score tensor's data * [[ * 0.5, * 0.3, @@ -137,16 +145,21 @@ class BeamSearch { Item() {} Item(size_t offset, size_t id, float score) : offset(offset), id(id), score(score) {} - // offset in the lod_level_+1 + // offset in the higher lod level. size_t offset; + // // prefix id in the lower lod level. + // size_t prefix; // the candidate id id_t id; // the corresponding score score_t score; }; - void PruneEndidCandidates(const framework::LoDTensor& pre_ids, - std::vector>* items); + /* + * Delete all the records that follows the end token. + */ + int PruneEndidCandidates(const framework::LoDTensor& pre_ids, + std::vector>* items); /* * Transform the items into a map whose key is offset, value is the items. diff --git a/paddle/operators/beam_search_op_test.cc b/paddle/operators/beam_search_op_test.cc new file mode 100644 index 0000000000..d4beb64a85 --- /dev/null +++ b/paddle/operators/beam_search_op_test.cc @@ -0,0 +1,86 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/beam_search_op.h" + +#include +#include + +namespace paddle { +namespace test { + +using std::vector; +using framework::LoDTensor; +using framework::LoD; +using operators::BeamSearch; +using paddle::platform::CPUPlace; +using std::cout; +using std::endl; + +void CreateInput(LoDTensor* ids, LoDTensor* scores) { + LoD lod; + vector level0({0, 1, 4}); + vector level1({0, 1, 2, 3, 4}); + lod.push_back(level0); + lod.push_back(level1); + ids->set_lod(lod); + scores->set_lod(lod); + + auto dims = framework::make_ddim(vector({4, 3})); + ids->Resize(dims); + scores->Resize(dims); + CPUPlace place; + + auto* ids_data = ids->mutable_data(place); + auto* scores_data = scores->mutable_data(place); + vector _ids({4, 2, 5, 2, 1, 3, 3, 5, 2, 8, 2, 1}); + vector _scores( + {0.5, 0.3, 0.2, 0.6, 0.3, 0.1, 0.9, 0.5, 0.1, 0.7, 0.5, 0.1}); + + for (int i = 0; i < 12; i++) { + ids_data[i] = _ids[i]; + scores_data[i] = _scores[i]; + } +} + +TEST(beam_search_op, run) { + CPUPlace place; + LoDTensor ids, scores; + CreateInput(&ids, &scores); + + LoDTensor pre_ids; + pre_ids.Resize(framework::make_ddim(vector(4, 1))); + for (int i = 0; i < 4; i++) { + pre_ids.mutable_data(place)[i] = i + 1; + } + + BeamSearch beamsearch(ids, scores, (int64_t)0, (int64_t)2, 0); + LoDTensor sids, sscores; + beamsearch(pre_ids, &sids, &sscores); + + LOG(INFO) << "score: " << sscores << endl; + + ASSERT_EQ(sids.lod(), sscores.lod()); + + vector tids({2, 4, 3, 8}); + vector tscores({0.3, 0.5, 0.9, 0.7}); + + for (int i = 0; i < 4; i++) { + ASSERT_EQ(tids[i], sids.data()[i]); + ASSERT_EQ(tscores[i], sscores.data()[i]); + } +} + +} // namespace test +} // namespace paddle From 2ce5c9d45e395e21dadeb4249c891973e99ee82c Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Thu, 18 Jan 2018 14:55:47 +0800 Subject: [PATCH 085/158] fix gpu implement --- paddle/operators/math/sequence_padding.cu | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/paddle/operators/math/sequence_padding.cu b/paddle/operators/math/sequence_padding.cu index e4be178f81..a38df26f59 100644 --- a/paddle/operators/math/sequence_padding.cu +++ b/paddle/operators/math/sequence_padding.cu @@ -71,7 +71,8 @@ class PaddingLoDTensorFunctor { framework::LoD abs_offset_lod = framework::ToAbsOffset(lod); auto seq_dims = seq.dims(); - PADDLE_ENFORCE_EQ(seq_dims[0], abs_offset_lod[level].back(), + PADDLE_ENFORCE_EQ(seq_dims[0], + static_cast(abs_offset_lod[level].back()), "The first dimension of LoDTensor seq should be " "equal to the sum of all sequences's length."); @@ -80,17 +81,17 @@ class PaddingLoDTensorFunctor { "The input padding should be a 3-D Tensor of shape " "[max_sequence_length, num_sequences, sequence_width]."); - size_t max_sequence_length = MaximumSequenceLength(lod, level); + int64_t max_sequence_length = MaximumSequenceLength(lod, level); PADDLE_ENFORCE_EQ(padding_dims[0], max_sequence_length, "The first dimension of Tensor padding should be the " "maximum length of all sequences in LoDTensor seq."); - const size_t num_sequences = abs_offset_lod[level].size() - 1; + const int64_t num_sequences = abs_offset_lod[level].size() - 1; PADDLE_ENFORCE_EQ(padding_dims[1], num_sequences, "The second dimension of Tensor padding should be the " "number of sequences in LoDTensor seq."); - const size_t sequence_width = seq.numel() / seq_dims[0]; + const int64_t sequence_width = seq.numel() / seq_dims[0]; PADDLE_ENFORCE_EQ(padding_dims[2], sequence_width, "The third dimension of Tensor padding should be the " "width of sequence in LoDTensor seq."); @@ -101,7 +102,7 @@ class PaddingLoDTensorFunctor { return; } - const size_t kBlockSize = 512; + const int64_t kBlockSize = 512; /* At least use 32 threads to copy sequence_width elements, * and at least 8 elements for each thread. @@ -143,7 +144,8 @@ class UnpaddingLoDTensorFunctor { framework::LoD abs_offset_lod = framework::ToAbsOffset(lod); auto seq_dims = seq.dims(); - PADDLE_ENFORCE_EQ(seq_dims[0], abs_offset_lod[level].back(), + PADDLE_ENFORCE_EQ(seq_dims[0], + static_cast(abs_offset_lod[level].back()), "The first dimension of LoDTensor seq should be " "equal to the sum of all sequences's length."); @@ -152,17 +154,17 @@ class UnpaddingLoDTensorFunctor { "The input padding should be a 3-D Tensor of shape " "[max_sequnece_length, num_sequences, sequence_width]."); - size_t max_sequence_length = MaximumSequenceLength(lod, level); + int64_t max_sequence_length = MaximumSequenceLength(lod, level); PADDLE_ENFORCE_EQ(padding_dims[0], max_sequence_length, "The first dimension of Tensor padding should be " "the maximum length of all sequences in LoDTensor seq."); - const size_t num_sequences = abs_offset_lod[level].size() - 1; + const int64_t num_sequences = abs_offset_lod[level].size() - 1; PADDLE_ENFORCE_EQ(padding_dims[1], num_sequences, "The second dimension of Tensor padding should be " "the number of sequences in LoDTensor seq."); - const size_t sequence_width = seq.numel() / seq_dims[0]; + const int64_t sequence_width = seq.numel() / seq_dims[0]; PADDLE_ENFORCE_EQ(padding_dims[2], sequence_width, "The third dimension of Tensor padding should be the " "width of sequence in LoDTensor seq."); @@ -173,7 +175,7 @@ class UnpaddingLoDTensorFunctor { return; } - const size_t kBlockSize = 512; + const int64_t kBlockSize = 512; /* At least use 32 threads to copy sequence_width elements, * and at least 8 elements for each thread. From 77cf21e53ec9f4cdc3881f03ca01b171459f8f18 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 18 Jan 2018 14:56:47 +0800 Subject: [PATCH 086/158] Change input data type to int64_t --- paddle/operators/edit_distance_op.cc | 26 +++++++++++++------------- paddle/operators/edit_distance_op.cu | 4 ++-- paddle/operators/edit_distance_op.h | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/paddle/operators/edit_distance_op.cc b/paddle/operators/edit_distance_op.cc index e383f07fa9..62a1fcebe7 100644 --- a/paddle/operators/edit_distance_op.cc +++ b/paddle/operators/edit_distance_op.cc @@ -49,10 +49,10 @@ class EditDistanceOpMaker : public framework::OpProtoAndCheckerMaker { EditDistanceOpMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Hyps", - "(2-D LoDTensor, 2nd dim. equal to 1) " + "(2-D LoDTensor, 2nd dim. equal to 1) " "The indices for hypothesis strings."); AddInput("Refs", - "(2-D LoDTensor, 2nd dim. equal to 1) " + "(2-D LoDTensor, 2nd dim. equal to 1) " "The indices for reference strings."); AddAttr("normalized", "(bool, default false) Indicated whether to normalize " @@ -66,22 +66,22 @@ class EditDistanceOpMaker : public framework::OpProtoAndCheckerMaker { EditDistance operator computes the edit distances between a batch of hypothesis strings and their references. -Edit distance, also called Levenshtein distance, measures how dissimilar two strings -are by counting the minimum number of operations to transform one string into anthor. -Here the operations include insertion, deletion, and substitution. For example, -given hypothesis string A = "kitten" and reference B = "sitting", the edit distance -is 3 for A will be transformed into B at least after two substitutions and one +Edit distance, also called Levenshtein distance, measures how dissimilar two strings +are by counting the minimum number of operations to transform one string into anthor. +Here the operations include insertion, deletion, and substitution. For example, +given hypothesis string A = "kitten" and reference B = "sitting", the edit distance +is 3 for A will be transformed into B at least after two substitutions and one insertion: - + "kitten" -> "sitten" -> "sittin" -> "sitting" -Input(Hyps) is a LoDTensor consisting of all the hypothesis strings with the total -number denoted by `batch_size`, and the separation is specified by the LoD information. -And the `batch_size` reference strings are arranged in order in the same way in the +Input(Hyps) is a LoDTensor consisting of all the hypothesis strings with the total +number denoted by `batch_size`, and the separation is specified by the LoD information. +And the `batch_size` reference strings are arranged in order in the same way in the LoDTensor Input(Refs). -Output(Out) contains the `batch_size` results and each stands for the edit stance -for a pair of strings respectively. If Attr(normalized) is true, the edit distance +Output(Out) contains the `batch_size` results and each stands for the edit stance +for a pair of strings respectively. If Attr(normalized) is true, the edit distance will be divided by the length of reference string. )DOC"); } diff --git a/paddle/operators/edit_distance_op.cu b/paddle/operators/edit_distance_op.cu index cf5ebc5c38..c165bbae7f 100644 --- a/paddle/operators/edit_distance_op.cu +++ b/paddle/operators/edit_distance_op.cu @@ -113,8 +113,8 @@ class EditDistanceGPUKernel : public framework::OpKernel { dist_t.Resize({m + 1, n + 1}); dist_t.mutable_data(ctx.GetPlace()); auto dist = dist_t.data(); - auto x1 = x1_t->data() + hyp_lod[num]; - auto x2 = x2_t->data() + ref_lod[num]; + auto x1 = x1_t->data() + hyp_lod[num]; + auto x2 = x2_t->data() + ref_lod[num]; FillFirstColumn<<<1 + m / PADDLE_CUDA_NUM_THREADS, PADDLE_CUDA_NUM_THREADS, 0, stream>>>(dist, m, n); diff --git a/paddle/operators/edit_distance_op.h b/paddle/operators/edit_distance_op.h index 537e70281a..4c5a29813c 100644 --- a/paddle/operators/edit_distance_op.h +++ b/paddle/operators/edit_distance_op.h @@ -60,8 +60,8 @@ class EditDistanceKernel : public framework::OpKernel { dist_t.Resize({m + 1, n + 1}); dist_t.mutable_data(ctx.GetPlace()); auto dist = dist_t.data(); - auto x1 = x1_t->data() + hyp_lod[num]; - auto x2 = x2_t->data() + ref_lod[num]; + auto x1 = x1_t->data() + hyp_lod[num]; + auto x2 = x2_t->data() + ref_lod[num]; for (int64_t i = 0; i < m + 1; ++i) { dist[i * (n + 1)] = i; } From 30f69beefdd7c3dd47194a051632e79dcb210969 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 18 Jan 2018 15:17:23 +0800 Subject: [PATCH 087/158] remove unused v1_api_tutorials --- doc/v1_api_tutorials/README.md | 5 - .../embedding_model/index_cn.md | 139 ----- .../embedding_model/index_en.md | 140 ----- .../embedding_model/neural-n-gram-model.png | Bin 68482 -> 0 bytes doc/v1_api_tutorials/gan/gan.png | Bin 17810 -> 0 bytes doc/v1_api_tutorials/gan/index_en.md | 137 ----- doc/v1_api_tutorials/gan/mnist_sample.png | Bin 28721 -> 0 bytes doc/v1_api_tutorials/gan/uniform_sample.png | Bin 24880 -> 0 bytes .../imagenet_model/resnet_block.jpg | Bin 22422 -> 0 bytes .../imagenet_model/resnet_model_cn.md | 284 --------- .../imagenet_model/resnet_model_en.md | 284 --------- doc/v1_api_tutorials/quick_start/index_cn.rst | 397 ------------- doc/v1_api_tutorials/quick_start/index_en.md | 562 ------------------ .../quick_start/src/NetContinuous_cn.jpg | Bin 35863 -> 0 bytes .../quick_start/src/NetContinuous_en.png | Bin 54256 -> 0 bytes .../quick_start/src/NetConv_cn.jpg | Bin 44038 -> 0 bytes .../quick_start/src/NetConv_en.png | Bin 59083 -> 0 bytes .../quick_start/src/NetLR_cn.jpg | Bin 30320 -> 0 bytes .../quick_start/src/NetLR_en.png | Bin 49488 -> 0 bytes .../quick_start/src/NetRNN_cn.jpg | Bin 46358 -> 0 bytes .../quick_start/src/NetRNN_en.png | Bin 57109 -> 0 bytes .../quick_start/src/PipelineNetwork_cn.jpg | Bin 9559 -> 0 bytes .../quick_start/src/PipelineNetwork_en.jpg | Bin 7466 -> 0 bytes .../quick_start/src/PipelineTest_cn.jpg | Bin 9408 -> 0 bytes .../quick_start/src/PipelineTest_en.png | Bin 8656 -> 0 bytes .../quick_start/src/PipelineTrain_cn.jpg | Bin 9210 -> 0 bytes .../quick_start/src/PipelineTrain_en.png | Bin 8791 -> 0 bytes .../quick_start/src/Pipeline_cn.jpg | Bin 14244 -> 0 bytes .../quick_start/src/Pipeline_en.jpg | Bin 11633 -> 0 bytes 29 files changed, 1948 deletions(-) delete mode 100644 doc/v1_api_tutorials/README.md delete mode 100644 doc/v1_api_tutorials/embedding_model/index_cn.md delete mode 100644 doc/v1_api_tutorials/embedding_model/index_en.md delete mode 100644 doc/v1_api_tutorials/embedding_model/neural-n-gram-model.png delete mode 100644 doc/v1_api_tutorials/gan/gan.png delete mode 100644 doc/v1_api_tutorials/gan/index_en.md delete mode 100644 doc/v1_api_tutorials/gan/mnist_sample.png delete mode 100644 doc/v1_api_tutorials/gan/uniform_sample.png delete mode 100644 doc/v1_api_tutorials/imagenet_model/resnet_block.jpg delete mode 100644 doc/v1_api_tutorials/imagenet_model/resnet_model_cn.md delete mode 100644 doc/v1_api_tutorials/imagenet_model/resnet_model_en.md delete mode 100644 doc/v1_api_tutorials/quick_start/index_cn.rst delete mode 100644 doc/v1_api_tutorials/quick_start/index_en.md delete mode 100755 doc/v1_api_tutorials/quick_start/src/NetContinuous_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/NetContinuous_en.png delete mode 100755 doc/v1_api_tutorials/quick_start/src/NetConv_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/NetConv_en.png delete mode 100755 doc/v1_api_tutorials/quick_start/src/NetLR_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/NetLR_en.png delete mode 100755 doc/v1_api_tutorials/quick_start/src/NetRNN_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/NetRNN_en.png delete mode 100755 doc/v1_api_tutorials/quick_start/src/PipelineNetwork_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/PipelineNetwork_en.jpg delete mode 100755 doc/v1_api_tutorials/quick_start/src/PipelineTest_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/PipelineTest_en.png delete mode 100755 doc/v1_api_tutorials/quick_start/src/PipelineTrain_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/PipelineTrain_en.png delete mode 100755 doc/v1_api_tutorials/quick_start/src/Pipeline_cn.jpg delete mode 100644 doc/v1_api_tutorials/quick_start/src/Pipeline_en.jpg diff --git a/doc/v1_api_tutorials/README.md b/doc/v1_api_tutorials/README.md deleted file mode 100644 index 071b8da61f..0000000000 --- a/doc/v1_api_tutorials/README.md +++ /dev/null @@ -1,5 +0,0 @@ -The tutorials in v1_api_tutorials are using v1_api currently, and will be upgraded to v2_api later. -Thus, v1_api_tutorials is a temporary directory. We decide not to maintain it and will delete it in future. - -Please go to [PaddlePaddle/book](https://github.com/PaddlePaddle/book) and -[PaddlePaddle/models](https://github.com/PaddlePaddle/models) to learn PaddlePaddle. diff --git a/doc/v1_api_tutorials/embedding_model/index_cn.md b/doc/v1_api_tutorials/embedding_model/index_cn.md deleted file mode 100644 index 2b4a79fbbf..0000000000 --- a/doc/v1_api_tutorials/embedding_model/index_cn.md +++ /dev/null @@ -1,139 +0,0 @@ -# 中文词向量模型的使用 # ----------- -本文档介绍如何在PaddlePaddle平台上,使用预训练的标准格式词向量模型。 - -在此感谢 @lipeng 提出的代码需求,并给出的相关模型格式的定义。 - -## 介绍 ### -### 中文字典 ### -我们的字典使用内部的分词工具对百度知道和百度百科的语料进行分词后产生。分词风格如下: "《红楼梦》"将被分为 "《","红楼梦","》",和 "《红楼梦》"。字典采用UTF8编码,输出有2列:词本身和词频。字典共包含 3206326个词和4个特殊标记: - - ``: 分词序列的开始 - - ``: 分词序列的结束 - - `PALCEHOLDER_JUST_IGNORE_THE_EMBEDDING`: 占位符,没有实际意义 - - ``: 未知词 - -### 中文词向量的预训练模型 ### -遵循文章 [A Neural Probabilistic Language Model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)中介绍的方法,模型采用 n-gram 语言模型,结构如下图:6元上下文作为输入层->全连接层->softmax层 。对应于字典,我们预训练得到4种不同维度的词向量,分别为:32维、64维、128维和256维。 -
![](./neural-n-gram-model.png)
-
Figure 1. neural-n-gram-model
- -### 下载和数据抽取 ### -运行以下的命令下载和获取我们的字典和预训练模型: - - cd $PADDLE_ROOT/demo/model_zoo/embedding - ./pre_DictAndModel.sh - -## 中文短语改写的例子 ## -以下示范如何使用预训练的中文字典和词向量进行短语改写。 - -### 数据的准备和预处理 ### -首先,运行以下的命令下载数据集。该数据集(utf8编码)包含20个训练样例,5个测试样例和2个生成式样例。 - - cd $PADDLE_ROOT/demo/seqToseq/data - ./paraphrase_data.sh - -第二步,将数据处理成规范格式,在训练数集上训练生成词向量字典(数据将保存在 `$PADDLE_SOURCE_ROOT/demo/seqToseq/data/pre-paraphrase`): - - cd $PADDLE_ROOT/demo/seqToseq/ - python preprocess.py -i data/paraphrase [--mergeDict] - -- 其中,如果使用`--mergeDict`选项,源语言短语和目标语言短语的字典将被合并(源语言和目标语言共享相同的编码字典)。本实例中,源语言和目标语言都是相同的语言,因此可以使用该选项。 - - -### 使用用户指定的词向量字典 ### -使用如下命令,从预训练模型中,根据用户指定的字典,抽取对应的词向量构成新的词表: - cd $PADDLE_ROOT/demo/model_zoo/embedding - python extract_para.py --preModel PREMODEL --preDict PREDICT --usrModel USRMODEL--usrDict USRDICT -d DIM - -- `--preModel PREMODEL`: 预训练词向量字典模型的路径 -- `--preDict PREDICT`: 预训练模型使用的字典的路径 -- `--usrModel USRMODEL`: 抽取出的新词表的保存路径 -- `--usrDict USRDICT`: 用户指定新的字典的路径,用于构成新的词表 -- `-d DIM`: 参数(词向量)的维度 - -此处,你也可以简单的运行以下的命令: - - cd $PADDLE_ROOT/demo/seqToseq/data/ - ./paraphrase_model.sh - -运行成功以后,你将会看到以下的模型结构: - - paraphrase_model - |--- _source_language_embedding - |--- _target_language_embedding - -### 在PaddlePaddle平台训练模型 ### -首先,配置模型文件,配置如下(可以参考保存在 `demo/seqToseq/paraphrase/train.conf`的配置): - - from seqToseq_net import * - is_generating = False - - ################## Data Definition ##################### - train_conf = seq_to_seq_data(data_dir = "./data/pre-paraphrase", - job_mode = job_mode) - - ############## Algorithm Configuration ################## - settings( - learning_method = AdamOptimizer(), - batch_size = 50, - learning_rate = 5e-4) - - ################# Network configure ##################### - gru_encoder_decoder(train_conf, is_generating, word_vector_dim = 32) - -这个配置与`demo/seqToseq/translation/train.conf` 基本相同 - -然后,使用以下命令进行模型训练: - - cd $PADDLE_SOURCE_ROOT/demo/seqToseq/paraphrase - ./train.sh - -其中,`train.sh` 与`demo/seqToseq/translation/train.sh` 基本相同,只有2个配置不一样: - -- `--init_model_path`: 初始化模型的路径配置为`data/paraphrase_modeldata/paraphrase_model` -- `--load_missing_parameter_strategy`:如果参数模型文件缺失,除词向量模型外的参数将使用正态分布随机初始化 - -如果用户想要了解详细的数据集的格式、模型的结构和训练过程,请查看 [Text generation Tutorial](../text_generation/index_cn.md). - -## 可选功能 ## -### 观测词向量 -PaddlePaddle 平台为想观测词向量的用户提供了将二进制词向量模型转换为文本模型的功能: - - cd $PADDLE_ROOT/demo/model_zoo/embedding - python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM - -- `-i INPUT`: 输入的(二进制)词向量模型名称 -- `-o OUTPUT`: 输出的文本模型名称 -- `-d DIM`: (词向量)参数维度 - -运行完以上命令,用户可以在输出的文本模型中看到: - - 0,4,32156096 - -0.7845433,1.1937413,-0.1704215,0.4154715,0.9566584,-0.5558153,-0.2503305, ...... - 0.0000909,0.0009465,-0.0008813,-0.0008428,0.0007879,0.0000183,0.0001984, ...... - ...... - -- 其中,第一行是`PaddlePaddle` 输出文件的格式说明,包含3个属性:: - - `PaddlePaddle`的版本号,本例中为0 - - 浮点数占用的字节数,本例中为4 - - 总计的参数个数,本例中为32,156,096 -- 其余行是(词向量)参数行(假设词向量维度为32) - - 每行打印32个参数以','分隔 - - 共有32,156,096/32 = 1,004,877行,也就是说,模型共包含1,004,877个被向量化的词 - -### 词向量模型的修正 -`PaddlePaddle` 为想修正词向量模型的用户提供了将文本词向量模型转换为二进制模型的命令: - - cd $PADDLE_ROOT/demo/model_zoo/embedding - python paraconvert.py --t2b -i INPUT -o OUTPUT - -- `-i INPUT`: 输入的文本词向量模型名称 -- `-o OUTPUT`: 输出的二进制词向量模型名称 - -请注意,输入的文本格式如下: - - -0.7845433,1.1937413,-0.1704215,0.4154715,0.9566584,-0.5558153,-0.2503305, ...... - 0.0000909,0.0009465,-0.0008813,-0.0008428,0.0007879,0.0000183,0.0001984, ...... - ...... -- 输入文本中没有头部(格式说明)行 -- (输入文本)每行存储一个词,以逗号','分隔 diff --git a/doc/v1_api_tutorials/embedding_model/index_en.md b/doc/v1_api_tutorials/embedding_model/index_en.md deleted file mode 100644 index 9525f64f9b..0000000000 --- a/doc/v1_api_tutorials/embedding_model/index_en.md +++ /dev/null @@ -1,140 +0,0 @@ -# Chinese Word Embedding Model Tutorial # ----------- -This tutorial is to guide you through the process of using a Pretrained Chinese Word Embedding Model in the PaddlePaddle standard format. - -We thank @lipeng for the pull request that defined the model schemas and pretrained the models. - -## Introduction ### -### Chinese Word Dictionary ### -Our Chinese-word dictionary is created on Baidu ZhiDao and Baidu Baike by using in-house word segmentor. For example, the participle of "《红楼梦》" is "《","红楼梦","》",and "《红楼梦》". Our dictionary (using UTF-8 format) has has two columns: word and its frequency. The total word count is 3206326, including 4 special token: - - ``: the start of a sequence - - ``: the end of a sequence - - `PALCEHOLDER_JUST_IGNORE_THE_EMBEDDING`: a placeholder, just ignore it and its embedding - - ``: a word not included in dictionary - -### Pretrained Chinese Word Embedding Model ### -Inspired by paper [A Neural Probabilistic Language Model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf), our model architecture (**Embedding joint of six words->FullyConnect->SoftMax**) is as following graph. And for our dictionary, we pretrain four models with different word vector dimenstions, i.e 32, 64, 128, 256. -
![](./neural-n-gram-model.png)
-
Figure 1. neural-n-gram-model
- -### Download and Extract ### -To download and extract our dictionary and pretrained model, run the following commands. - - cd $PADDLE_ROOT/demo/model_zoo/embedding - ./pre_DictAndModel.sh - -## Chinese Paraphrasing Example ## -We provide a paraphrasing task to show the usage of pretrained Chinese Word Dictionary and Embedding Model. - -### Data Preparation and Preprocess ### - -First, run the following commands to download and extract the in-house dataset. The dataset (using UTF-8 format) has 20 training samples, 5 testing samples and 2 generating samples. - - cd $PADDLE_ROOT/demo/seqToseq/data - ./paraphrase_data.sh - -Second, preprocess data and build dictionary on train data by running the following commands, and the preprocessed dataset is stored in `$PADDLE_SOURCE_ROOT/demo/seqToseq/data/pre-paraphrase`: - - cd $PADDLE_ROOT/demo/seqToseq/ - python preprocess.py -i data/paraphrase [--mergeDict] - -- `--mergeDict`: if using this option, the source and target dictionary are merged, i.e, two dictionaries have the same context. Here, as source and target data are all chinese words, this option can be used. - -### User Specified Embedding Model ### -The general command of extracting desired parameters from the pretrained embedding model based on user dictionary is: - - cd $PADDLE_ROOT/demo/model_zoo/embedding - python extract_para.py --preModel PREMODEL --preDict PREDICT --usrModel USRMODEL--usrDict USRDICT -d DIM - -- `--preModel PREMODEL`: the name of pretrained embedding model -- `--preDict PREDICT`: the name of pretrained dictionary -- `--usrModel USRMODEL`: the name of extracted embedding model -- `--usrDict USRDICT`: the name of user specified dictionary -- `-d DIM`: dimension of parameter - -Here, you can simply run the command: - - cd $PADDLE_ROOT/demo/seqToseq/data/ - ./paraphrase_model.sh - -And you will see following embedding model structure: - - paraphrase_model - |--- _source_language_embedding - |--- _target_language_embedding - -### Training Model in PaddlePaddle ### -First, create a model config file, see example `demo/seqToseq/paraphrase/train.conf`: - - from seqToseq_net import * - is_generating = False - - ################## Data Definition ##################### - train_conf = seq_to_seq_data(data_dir = "./data/pre-paraphrase", - job_mode = job_mode) - - ############## Algorithm Configuration ################## - settings( - learning_method = AdamOptimizer(), - batch_size = 50, - learning_rate = 5e-4) - - ################# Network configure ##################### - gru_encoder_decoder(train_conf, is_generating, word_vector_dim = 32) - -This config is almost the same as `demo/seqToseq/translation/train.conf`. - -Then, train the model by running the command: - - cd $PADDLE_SOURCE_ROOT/demo/seqToseq/paraphrase - ./train.sh - -where `train.sh` is almost the same as `demo/seqToseq/translation/train.sh`, the only difference is following two command arguments: - -- `--init_model_path`: path of the initialization model, here is `data/paraphrase_model` -- `--load_missing_parameter_strategy`: operations when model file is missing, here use a normal distibution to initialize the other parameters except for the embedding layer - -For users who want to understand the dataset format, model architecture and training procedure in detail, please refer to [Text generation Tutorial](../text_generation/index_en.md). - -## Optional Function ## -### Embedding Parameters Observation -For users who want to observe the embedding parameters, this function can convert a PaddlePaddle binary embedding model to a text model by running the command: - - cd $PADDLE_ROOT/demo/model_zoo/embedding - python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM - -- `-i INPUT`: the name of input binary embedding model -- `-o OUTPUT`: the name of output text embedding model -- `-d DIM`: the dimension of parameter - -You will see parameters like this in output text model: - - 0,4,32156096 - -0.7845433,1.1937413,-0.1704215,0.4154715,0.9566584,-0.5558153,-0.2503305, ...... - 0.0000909,0.0009465,-0.0008813,-0.0008428,0.0007879,0.0000183,0.0001984, ...... - ...... - -- 1st line is **PaddlePaddle format file head**, it has 3 attributes: - - version of PaddlePaddle, here is 0 - - sizeof(float), here is 4 - - total number of parameter, here is 32156096 -- Other lines print the paramters (assume `` = 32) - - each line print 32 paramters splitted by ',' - - there is 32156096/32 = 1004877 lines, meaning there is 1004877 embedding words - -### Embedding Parameters Revision -For users who want to revise the embedding parameters, this function can convert a revised text embedding model to a PaddlePaddle binary model by running the command: - - cd $PADDLE_ROOT/demo/model_zoo/embedding - python paraconvert.py --t2b -i INPUT -o OUTPUT - -- `-i INPUT`: the name of input text embedding model. -- `-o OUTPUT`: the name of output binary embedding model - -Note that the format of input text model is as follows: - - -0.7845433,1.1937413,-0.1704215,0.4154715,0.9566584,-0.5558153,-0.2503305, ...... - 0.0000909,0.0009465,-0.0008813,-0.0008428,0.0007879,0.0000183,0.0001984, ...... - ...... -- there is no file header in 1st line -- each line stores parameters for one word, the separator is commas ',' diff --git a/doc/v1_api_tutorials/embedding_model/neural-n-gram-model.png b/doc/v1_api_tutorials/embedding_model/neural-n-gram-model.png deleted file mode 100644 index f70b765b3fd69816345a79fc59adfea46008dbfd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 68482 zcmZ^qRX`hGu<(&UXp2+ap+Ipj4#A~G3#B*(ifeGUV#SL)|Ke`NCAb%N3+@m+crN|E zm-})bR?o@q+1c5d$(i3osHwm@^3+2GpF`=IZ5Cm`_)C#)y2{Q zjP%LXflc$9!+S-UPt2J+n#M29H2<2V!Qa2RSUK2hTG@R^;^E@{Ak4)t%q>8}Eg;Mz zD9k1B@UT+<(wFvs_oey2Iy<{SUIu|+SCo;|^e{Yr0_N&#yK=--Ew_55FpRXrgS?S# zO{2JRQT{_Wr1`NOfA;ahaM+s@=e8(r%|j7`{mQx;8MPk`9vW#oVvx)>RLfR&bd%QX z{tSgzw2uq%y4ypvwSVm`$nTAf3OyXx-ZW}M`DH1-h?RH6tF^+(rt+$b_w zwydgTgZc{v$)EHVFY;b_s(OCevM#u&wTG&QeQ5e;U%Pr>VLmYPzD&LzXkHGadzJi) zPNHvGofrvoy`rL`&*}>hb2p;uUo&JN(6)<_biZT9(h1Cl-fhZpCzkrbSShO%gzs_x1Bf z!G@NlTnNda9_6`*F$h99hKJoX;#kkn59dS{j@)fh)J&@+fq+t)VT3)HFa_@hv_ z`Ktpe(yF+RK4gPoKNWa zgBr;8Uw?S6C{kR<`B>O-k-Mnz-M#J+`MVz|u~i0L&RlJ!SCV z&ON3zLveaQ`tJbf{rUN#ojK?6abtAro14cG)VW~j{eKO)EWyC6$Wm-MAbV2VA=wwS zUG3F4m<3j4pi+><(8eM0Sep9Ymii<($-*KWqy4n>~dD`Cg$q)ayL0y7f+dkav|KvIRIu+C?NL4YW~;v3 zraMlmm)GSI1*#uS+Ye;n8keckhL%vh-*5_^<)*9;3MWO8mg!Qy&x&C6#QeqmGHt~1Nama5tjWA}RGd~Bbb%~*Hp>#b zfU0F&i9a|B2X)_KGsyR)#py8$_OPN#F3g4F^qLm{Dspuz@A=0cl~n%2LNy4iqPL4TaLckqJF;T{aPmn1k% z#Our&eo=dOLvG)$?}DrT>DF2FLEt0kR8m4F>3mY9vEy+H$2nIs-U z8c~YQ@rjAHLnFNxZ5MmhCM<0H?ra$4LZ->0lm}y2wDvF(W5)DgW6*N|+~1-Q)f*Vw zP;KTP`^5W&aO~@nX1m3vm>Sw{1l-_y2(=ohF*d;}K~W@3#gUll#KweTn~WN}(%7ct zC2W?|AtfTO@{e07S;nsVvCFzYw=hQM@gY@U=W-y-yvhB;pMS_Tg zKKn7clOm^65#b@#jceYwmXs*M*Zdw%-1h$qTWW%{#Ha!zKAxsHU$uey8c$)P;k!nM1+envP^^l1fpQe$BPDqJFvX0>?24P*WTV9 zpfHQ^2jh#*1Y-c+nXrgR0vZ18;9oh=J3+bnE(7<~@8kmZe{g)(QCo6gQGS1YsAE-q zHVNGJvfNAcv)RHd0RGSI-tg1yMvNwa>Kyl*U*!(|{{EX1f4%Rr_wlBtrsy2)?d)cK zMejtKA20@eLU%4Qc`a1%JrZ_0>+Uanxin|B&Af840hbZhv6|k-a^5r%aO3XZwzt&M zD(lPS4Jj?xmGX8-)Ev!cZ*6D=O0xclu@x|$CK?clE9BC0sRvK2tgx-}NLCjWjd$GC zS1phG-F$x7;q+&oFlew%L;49nr{VHFu`#NnM%)fG*e!n$x!X?oxU%5^gCd@w%ZO(v zVtLTDQoni1zIlg8WG~a%g&O&)VcEV8^3V#YgZaX|-Forl7g}78Y@hD0-KG?o7B9yo z9E|RUwQc7s4cx9K<&WBp9yf_kY6ht4nw|GvbLDuS3O&p9IU8g zpEvZb@P)TRXEz;hH^P=(=M5~1GQmHIuu*u9I$^@X!nV(kcPRVyHuHzg(%3>W(91Q9 zUiDe^>xtB69rd06?VJr@|b&T64ludS`U^Y{_a;&Px7c=L)Wljh~im!0kD4tay- zx9wx~R`h1U)D_W#BVmhph9NjAZ|17jg?eXLwv4fCAq%F@?2){Tz1hY=e=0_DRudAD^D~@&@;f`#)dyC9!HJ{kfB- zzE{K3uj@cnAnFn$f;{eU65p}m1nA)z>-n5@0kz~rk#1!ji~3Z34?o|(4k%6l`fs(b zmqxC;T~lgOFu>4&74N)|w)0mc5SJ43t>Sk>s*Lc9Y{+dSWMVHno63N*W&bxF5zfcU`+Ts!P~-iVVU&1=tc#nksU#v;e}i|cI$UP1Xf&USYs0M zG?eqGs0gjntyjt^k!E!FAE0zsn8jb+c>hX`;WB-`O>U>RleoeE=6F%V^l{GNDS63t zh#_U+VGSD;P?x2&)`*muwBT7rm*SIO+>Pwl}g{H-EUdSnCbX1 z1*XnFoVJd0bljRCyK0h8?8|lLXTu*a#~VL#w)5RZAHM03Aj7^B#W_r+57(%dy6)p1 zph)nDDmQG`h1V!gUcPI{xj%eS^w0eru|C`MEMYW;r(?uN;;!^+>{fJQafMf8rw1+x z;XmDbPj<_+Kgp>K9t!P><4S(F^_>jZV}#7)rr;ZTnhUvofJ8pZ7Wjv z!yh0Br@)i7b_2KL^MN~7Ya665LVAd9FOsvd&!-kaJc;kzrj;8PkUsbpsI(CCJ^0L&+MJa`fx$5{GYhSC6%;AFNNRR0<(lD%{bV>>BAlq*SEbw>;0PbtExbgl_4)Eq~ zyX=MEuKk)^hy53tgw8B(&|WB}a}rr|9RjXN!qXZu%biAZ=+aU=Pb1^Yf10V(2>!J3 z3ipRUx@5bCu!Z=DH-_dm_*JnF)44?-?a0Y8^e)>Kx%~eATnsGrz_D82V0HlZ40wCy zDv^D++t^_Wd4`JTA2Paj`5N6F&YxWIj@5?BHFeZ*IChQT{W)iY`QFCMUyt>~3J!N+ z8|}V1OYVH~7HPFxzR7aikWJYaKzSB}uQ~i>w{yE-F8W4GQ#rSg<-TevPvQAToY#(j z#M09Ci@}|Mug;t}WKH~GaeQ%dfWo5Ur0&YkA$D<0%=@I#f6w%0Pi?i@XDw%-CBV9K zs{s2%QmDqjgR#IU=yIX|0@R_(ryYpxn~C};`w{_mIR%_}*rV}>^{#(I(i~9OPy(rJ zDmbr!d1z_gBF>EvBM4K}(s^r%s4#50LA0x+CFCpFYIXTdXBshOA8v5LS=oE>mA6&f zDt!kbHY?<6Rzm87v)V>>{)kuvpcC!ncd>N{k5ODIpV~PbpTk77S|rpP`0cWg zKf}#zDe~#?gS~EhWNlPWZKToTRqGb>NRtW#;#G0evv5E&rE^>tWEp3XPw(~ms{54* zu;XFh2%&v0?VslL9TRgq@-C3Vhv%ZN+t^Thib0%V{t@(FQ)43m{M_3^;c-jit)B2w zB&#;1uD;X$@Fp6u!PmCE4)60f?V!iD%hT4M@>IT>=izxVY}tOhq(^%`f02RUW-a#*(i4oL0-k%c^N8(Lgg#Q6mPM?A$Zc#n`iz(0Mj3JU1 z`l4G=(=&bzl<4i7%h2Z=I!w>L#`Y7tz6Th@2r}9F8|&n2t*zC0UCx8*?4C5_?i)`} zAI@VV4iUBQW6dVwX`AWl&Ikdu_$3TE_^W-?>i)PS64`olJSXCl4df*D%xq#BUKlv< z%!KDB=?jJgpjwKk+;Bc5Ty;YgRUga3YdC|@(cd(Y8}vH2#E`WcA{mzEILrIL7NTu1 zVwF$UJ7?2*^i8Dj(2=V01wddk!6DcY$ON zGdVt;yPM9^lfDT*YgQwN#?N_W;p0InP%`Csj_T3pLXEPS}Qc+{MB0bCKED|E4z_{Kk4Z-}e}< zKLgjh)CUG7Zr@IOOO11xiOQh=@=chG`wlQId>0mV1+vfT|4R~@g*6|;mwc@qhab8k zVTQ%>pH!2~jo}Xh#$dvYK`O>cNnyWPbLG!Cakpswsd(7MGb83$K9A@^)Q-({oORbulfBu7K+`wbvWSUQ>us9)t(Tb!!bVM%_mGSUak!89^p8J=@^zy|u z{gP$bod|Z4-LN1v;#>dvvDBg-hY^H2>5#lD@=UDkk=S)BpaDEEnk4PQWpiY2T1jL) zx_+Cp<7*gXBxEHRVclg%3|{v-$|$F%B1Ze#_LBytFG2d6kz2|NX}5pw!$B&!p6>bk z)!jH_4A2@hXF3V41~-_Wv2DF^TY@|-Lza`Z8y1TYH?Z=;|L_Rv=kC8T+jd?~@JK_W z^og2?U)z!_uG0oW-gt=n4e)rYHLQi~-A3s~z^f?^$j`_j7I$qIypTPZaZUlu-X1o= z-x2%0)^;ude%^R1dQ|L|8&%lgwkl8mA34kUGNAtq%XO6W1a(rd{)C+pO2@F-g@|L`uh%=Z#V~wXsnYo;|vx|nG+;DHi={m zk>_D^eCH-0*CGMOzFC%;3{M6&#pi9iZ-?sXrk^}O5rMQQYC;@>YF%+WWWN0x~ zzv#Sz1hrKd`JX9+CCSbWgCv&MW)^<%EJ#5`sznNg{4$QqzcaOIu8_lPm3z{S<3v8@)ODm zUC~zYe;pZUBh5t^9JKgC+UkGPX>|LQlD_uQewjqGY)Otjuym>)eJTWQFR1+<_*Bdp z2y9LXL@xYrN8o?W;FuG9f+(p!w{GnDuf-;AqEK6}HT9o)nZvzX zS$_#vjO#J%QI3K*Mc4GHuOssrEbK_UL~Y=Q6-}3~Saql}eQuPjKN9SD#EnVJ!bM!l zwL_YIgC|gv=4PdO30Sn_Jz3DVW_8d#{Nhz#4=?&5J(IlpLv8fDn1g5yI7#roQMc)_ zV66EhDnvgOeVN`AdRmnwQOm`3P@+AI_Oj0d0q3@FX2c(&~Ci$Sz+C~8cd1P+lBjI#~KmrT@*nOhLbFTi&hO@pBjLM6<4J7 z#POl}8JVRPTd-d7sq`RkM*=pWnw^s#_dq{0aJP7v2BWnlX6%kiR4T6%MCoEtJlmsM zA#b3JWDKnBG7v1_2eQKzGDU1!RuHhwX;?#VDcUd+PZdCYUDk_L1mIY;e1 z7hHn-X;qeum4V+jOR!pxV-cIw<))Lb1NbnRt{2uq*Ts+!A4BA|{`x1RrdOTHCk#$& zbmkNDF!j@1k;oR0rZ4S%)yBD?3Dyu z;~To}_8;t-gBC?|B?_r@Mp;OLvJH<1Fp^~7VD+K4f0%%fhwioCr1{?_a%b0xAB_ph z)-@NVh7sVnG9WFO?Ic!`X1->WiQk8mWn() zLq2}rhSQ;klSsrFu-27NnsvnIp<6?2UT*1`{a!B4X9Hsh%(=i0fbiG$V z0kj;$U=<4fSFbA-!qS5_Dz2bd#eA3Oog3viI@{DLI)dl3w8%T!2W@|c-<(CN%PUnd zAk3QRHpc_$RyX$hBQoiN9Yua@nQs{v+37q~`)`;9ldhl^EBQirDu6$zpe-P zYaRQdHpuK*e8RU(>)S`oHV`0?CM?q?_AY(mYVga|kh12>KE)s3=JKP{M)OaRN^qrQ(Bku~b((zj&n=c_F z9|BXI$leVGE-{iuk<(TX=30yiH0WJ|8m}wcS3Az;AwW}_&!H+$c=!ObA^uP3js#a6 zXR7)&2eYHoyex{c0_m*dGxLqQo04;=7aeNdshxz3?}akM>_)BPmF>Bl;wpkJB<0k$ zY*i9_hl8VsQRPyzrI=ZI2h7>P(=s6b!I7%pubQ?~Nj*-;#Gq0?1FObiql>VoYwcd{ z`A(@AF1BrMJ8)w&uD1BtbMab7&J~CO8!W1ofoULzr_q_Ivn@u)j-Hu_;<8=(8L`aA z6`~sbCFkNM#tmWV%boBfpu1WMiCsf=dK|R;g?C*cG04>9e@SC&C)0Cp0(lq}Xy`Vm z9BIjc%y*m}AnX;6h1ummeJ4p5BiO_5zRuH&Kr*k{+4od8Wl9Yx)vy!&9BD4Ov-e$j zPLx6F9k5IiE2w9dRJ&!IVib@4TTHPEn}%ol*&+>Zrq779QwVCv*v-6`^`n%&S2Ag? zoAw`)-C9e|Om!gT-pKxhM2(2`QDC>(`1i*xXm}LZ2vfJcS&`vY4v~Ujn57<}E;k;0 zz{TT&)5-MpMw^#d^?etXQGx`M@TnD8$x99Nc|ery)!wHwtd);vEoT_)j9v1&NF`G% zmY>%`Hx7CA_5Kaw7!onpdaoa94pOKb%l&AsHqB@cRyUA?3wceU(bKrTGI?9bnNQi} zJ(?mzNp1KRzpg)vpy}W(Yc*B8KjN}V!h$tMh;Vs{iv&&o&FaCRMTiVf)!eZr(5scH zAsffnn(nDnV`cxC<`^%((xLFP`PI_dIQB5w1XX(&+0W4oNnMvQ^CtQXX50J*mHg+= zX%;5GVO8S9f4jCI<_Cef3~*2aZB#5v+C?_dQtAV12rd;m=CveH3nV;y4V7y|$7Z7U+02n_V@hpW z48-+QZc%go_CGTAA54FfXous)4PhmO#k7`8zQ;;YnV_Udz}-wuGp7R z21tgoG*+8BMKFljvj4h*Ihv6*(xfe%sdrG;JS9DdO&duCN8A|c9%u_Zsk)qrg45)G zvt#^`LFeaaLsdYXz%X#2Y$)HjF{xNtZ}Uzu@Efuz&&Di8Q7$3^+JJ+@ZN8DPb_v~@ z3c@##Mt>@}N}iH9i-ywnyP6AUl`}1^&hBAnQiX5o3W@c>6i>OogP5LgQ#e%?`P=RKae=`%ywp~ zuIvN_Ybe|e`HVg)W_(0RB(;?ii^AXH?IHe>jB?4%q$Vb|R5^HVC9s8Qp+5Es8}5Av z&jWK23XNYN?Yp7^mRTc@0o@l-GXT>j|=%dKw4?&$zM>R}X)$7U|~R9S$SP z74f64-{@#8@~V;%_0%eN0R_EjP(?^Qm#dn^{(IG*nw(T$2t>XGo~$2 ziPfFhWHdiMtscwjIO_!#RLy9l(RxTyWlzQ`icX?!gKqN&jkFE1nH{`T*%0JZ(+Z4H z1Z3Q21RVY5Pec#-m{1posB1G_mHrbRUDKcGh6h>R4b{8brvJFwbI5v!CWr+jvPQe2 zKg$ctkwXEv=9bTY1)o$=jP@Ik(;_olK3WRphs<^pW$prV#VorE{XJzQ624UooT^~r zvO)WNc&>1Bw7UJQukt%EPVZz3P`)#=YSw2@7g+xNYrHs#LH0<(5F;$rIJJ*dY4MG= z%F#7aSWmp>$Z;p;`XQo2KwMzSyRh|99ZGE+YCs|B)~3xAI;_f_mt5AsVkERn2LDz` zKg;ULxf*xn*TSvddTc!4?;ZaRLM`#M*}+urtpYc0KRM-GYL6AQ$}&!LUZ)?V@#uEu z8Dq|R#W!*gGgIEYf1+ibj<@g5_9T%OA(-}@@WC&14;nR%05~z6ZXOjApT%C$G4T)7 zkuh{KfrQ2IftMko9~>-yn9879^3JNOhN82G=}qG#>!c9}YiFzCYO$`2i_x!m4mAth za)pJ2jtixFU*vrwIyr=Yn_Q)Suqef_rrWCGItB?$zRCN5WQ)A{-#~8qiKJID0$$*- zJohYjPjx0M(}jdasy%d?h^KkOB)8NyOjd{5|15U(4Ko zC>vMCcHazVey8U`K$2(-$QL$63K*~j_X?k`^gd-2`{m7(IXV5D9VykQ2nO>#HjhXc zQ6eG;k#!PyCl6YG@Bu4y?CNel30Qf_&WI57mnl{KgfHp zy_!M>^UP1KeLjHyQj8z6B`H)mv;S~(bXp_OzQN=D)JGg32()w4%kaHLg*C`tiu}nE zEmHjCp+e=s=M|B*Pbba2^hEz8zUUV@u9}bN*MMn@JjE_N?>EyIE?6n7Gmb45RGXDp zD4F@a1hQ7Rz!LW{s=Sh{{yN+FPEJxcT#MPPWeko_l9^HI#3vz1mfuXGfBV_pN0}z@ zS8Cd0D}q7E}dsxjo@+$ zF&`cs6~H15Wl`3+$xzE&c(S(IAe+IcVC$xj*MR&{jU&m#8YL%3;im65Wf4Bk4HFD+izDkU$oz4w8dZv{R^`|fs|h#u}E=iB+N*ZvTj zPw$lBlWxnO#91cv#y)R~l$(MZCI^FEMN6*)SA5vu5*({+T@UQp@pN@c)kCCyb+(Fk z&>_9TUs%)!A+yjIU}};oobRy3Ov$5!SWIZAd_;r!tLjzs?4IFYF|Jt-$E|~p`>Cuj z{#}ZJ!Aj9BPxb3HYaGqPK(+{3>qwDA8EHd&q&_>p^^dx_K{#O=C=+tO{@bwT)b=(O z72}!`kKAImD20W{55A-1E^3jRy)e`g!wr#iCoI@&D*}WI?~>i}^BN&1k*eA*e9sAv z@g#K_g}}4x-Wr==D#UXO+pZ7G@G*GxXluEDA@`2E>a;vZ1}gHF{S~C~0%_4HWrMl$Tur{?f-DC_Jvg-<{#7| zC|u@s#CH>L;Z-F6Zd9Wl;1e!|O}wT96ln3#MA>0cQ{D2=ip-w-*Y}T`L;YFFF>)!F zhc%4eeXYXqiqB&VAlUEkAyk#m*nC(F{VIUM zd(?06?bg@1&Z2%7R_?Cvx%P3-+L~dYgQ;m=rPKMhj8uQ$7UF0p%#pIw zITkL+VW^LUy(79UD>EB_Z^`K$)pXw6AXx|9yaMPW!8<~NF`*G!`Oe)WW8y$NIwXPg zbGvuJ#y7tDUrpLg+v(IA8+ox37l}L0LS)D`P^1av$a!v+-6j`ene8A|x`e6l2U;&z z?P`2MnV7U^{B!tFkOH8EEk&_X@G$Y!A2Bhpp$CkafBAfS|1+RJjeUZ#TswR7-*Rgj zGL$^YXt*q@{ugaZ1f!+7*!y8vbm*W5bqRg4!O-PqLe69U(-RcY-l?;3jC5EW6Jhu8 z7y79DFeoo>@-dmsDK=&On?}Tuzw76%4jbQ>p%@rxL-{`8gsT%VW&Sd4F6@@ra=#O4 zeY322K9bZlZpjr%Vdn1MSl`g_+U3>SKcwQQ3bGa>es4J&@?FiJ+_eS3pS{zmi#=NN zk&Yxe*cGsTVZ+AsW>|+Bb}L34i!s~=-U!boQTLUsHf*%dYJWO$P$eQ-8|f?d8vTT$ z>HK~46GR(I<`vMog<)35^7{l9)W^2(iDtaH<|W|g zDY;V{^L(34Pb1-8ps;clw3(jC8M|K-cdZ)JUt<}b&&e+t*SR{m_N`ge?K3j)FU$lf zYOU+)NkLu7*ui>Re8kVJXDshj;IJ}JiOx&?z18Ql2o9jK`1I`T_d4^xJcluQy1Gi> zP~5?5dp0UI`ep0}_tTZ;o4*cz)0LhhdgT)HMj8WbHH?wper@L~4Np&^YQ4;?c-8Cf z-g?uFc?5#3-Wn&Dm79Uc5hGJf!`GbvNtBC1<&5K|tr!A%pZ^CrtzC1J@OV5PL7vN8 zU0fDk*rST^74Ga6K4*8n4}6E0#E@HAk?TL7PbRd&mi8nNo)5POxKryevn27F0yfWoDD$nLZao zOXAF}j-Lu0%EncG13{^?c}u9gbEv_6fpqTH^LWqUP{K1UKN{`rdYrvXcoD*@u8HwcV>psj}^w#KMX+D|H^aUYm!J(;y z>2M;sox`3_VZR<0sOHXJ^0F6|SSP4R#HxZDKQ@!J4NG7i)ocN^hi_w_Meiwn$sfRM zPhZ_GFI%PGL&y=wF(UZ@zsyzSNyig0& zjY7`5??S~7zpF|-xoAaLqxqTZY-`>+T@5H~3n84v+l~#clL4g1_HQ1RsXOA&4ip=exb-e~Id&r#g%Ah8G3U z%;Y@Ea*b#`EG_#rZo3Q=KSN%wF%A0%vcGC1xi)HqqpOC0U8#Cns#)E01^X@mC3ymGu><=macf_+PG^%G$JE%AdX+=4- zVjlRksq(2GhGX|6RdgQ=E=xPxpjbm?Wl}rp{x#PT!)eG28zaR=t~6MkB538#AGXuK zC-nGk$IAo8W?Fx>ZsXBEQ~8`p`i8Pd=`%^jP;lkY5;lST$hdkOphwiAk=;V32W0hr4D@R*V+^jul(v4E_SnT zdJ_gg0B??9-jI`}8s0>Qn^{i(z8JB45|P7jdbFPa)l%etM9}7t!KKLD(@UbPL-6no zXPX*SHs&gn+&=DGB~{7CNYufBK)AT$i?W^CjKvEK7=P=uZ@(kb6Z8+H^xanPQZBG; zTyd#8)oE}^+quvD%=C47^(2`3(Rj1$6Yj19UgE1>oB7IF6p-mRfH8QyoLb0h?c>&W zBh{)7Q{S?I(SK08MIp4zp<)J=^ro3jv9ReSi{fa{N5qBp?vsIy!h-Yos{68eF{!oG z)Nkm01++GMjjDwCiqg<$T3H|>!ya(P?Zr{bR;;p)_Z0kMpP3zA1(^VPFx+M&r}KV` zZ?EnDAPXu5fRB8Dl36gJ7q-q0P4E(V7YY&^D)@}n?M;@kdL8X$cUW2ShDrVtD6~tX z<(TBL8&+Z%0kgQ%9PoKqh9;jr;uhPH<)SPG*JjiVsul1o=4P?PtF{3P14@~mFHoO` zlqmWfG3}@3zy+%c$E#h>%kKf}|Gl%~*jj+v|G5aOcVZP>r8VXLqDX(`q)fSZPWBLt zI1js7Mm)I#`di&#wNONlM0F#G-)YooK-b7W=X40?Z8=Uy$Ew)s&(z50K7d@%W=h8u zL*O2h`1z|~_t367DcA3mCE~dpsT9n!Brd9^Y4eEvurJUB<1lK^AH%V*#dh~tS6BB% zYQ#a#R|lgi-|ale_t*;0m5!?&gofvg71S3db~8=x&$w>x-<2YDy(L$#Dt7tD_k`SN z0KEa#tzXBYbcNjEzS;HnsQ-P6e}h2#o%IM|TrURPZ@-7S_C|;DqA|vA9C%M5rgmfm z+`Q`z)V+JESzD?XB1n*5{NY{w?*G8CSJ4t;CP@7at&;y59 z|1gA9QwoND%Q13t(nEb;J-p+*++g=P4v3uD#o`l2WYr*TCbgr2}0jKcu}+{)s(o52<(OdD@}QOrjE5inNbeXge4UC>oE%#6NGvr2onKLOi*`5)@jvB{7%q-5h2rWGnrcJ$}$p zj8FG|3ubFQZP__`??GMN>lUEZ_Trr#JL%4(nzUXm4vj;R?!^7T4Di!O0QyGVJ-e`9_}lzwsvSY&fkF zZ4{T(_VAt0euj4uU0<(!c2qn3h0@ND-$7$36jw;I1B=;GcnLwCwxw>5<27_{-LLm&4+qJf=~xJ+irCxU%upfTY>CbM@ z!p_8?`=&v&1`PwJt6`5MjPqjpu*Kqo0Nia6Z+)-FB!&a->NtDaZQ>S-rr1x=7P($d&g-Hg+Fi#A+?(meR4-gW$ zt~`q+QP%-5%N|`B@YbL)Gw;E{I%i?T*K1Db_g10S2mEW|EoP@Prkh+v>SM<%*hyD_ zls8>x5W5Ck_6v0l$33t3I8Hh1$YkYL$#)YE2l4W1GNhJLJZnCxSwtj|oEv$sQCYm8 zoinkvS_(309cu9bn5PlF_};BRJ?MQA>G!M4vYpCjWB^5maCJf@8bgRo1#C*Hic@&N@rbx0}_x)9Fco?sf6DSBti)3wF(^r^R3 z-440AP6xW;^oE&UEa~oA!Fp|+w&2hGolJ~NywA{<@v8D=05Jg8GK68UV=fN=NHv=K z9%IJ!_aNuw#DYm=3N=eREIHc7$#m`hGCQ)lmzR+(-PHL6+_M$NTK4G5)kiPnXL(FO^>iHKO!R!%b z-JrBsH5jTql*G$MvfS_e*JV`3TEb_(D8OKFr=q?>r#X%=IuzNM#0B-M9J(Cehk8q= zK@Jxk&{(L8wwKY>6hVgg+RgaY69ZQIAV!lDZwjnVw|e+)%{~J)&hK%I{A9rBIiEj> zA?&Y~%K+ME>tC2$m(g1KR7?R;=Qk`|u{C*{b~?s|V^u^P3W+FpPel?BcHW-GG1vTK z^->1pQ6S$r^q-X>wis(Vndn{3sftCZUXGt^O`^MSz5zhvAN+p9;5?brlu&+Aoo@gW z55-a*sn^u|pl|sJqc6ZpqVGITo`EK-7WinmrCp268c1GmoKDnnz%d`_Om|&{%Cv%! z+(?ZV68#tJ3joAvZBq+$c#2;=E0Cx|FFq&+I~ltT4a=i5OY$nkIH~-1cw+jm9xe92 zgxW?kEmFWPaxoKnkjhon^OkqR?5>Vcj}N)!S=%eD5JSA{T#Dx*?$6&hOmgX$FfS>< z)ESnPZFmG)Xdb}Z7pK0LY>JpT`Zk^|HaE3&oQwG?Ua{`4TO`1_QTszD#>#&K+dw#IhtRI6H)1%dyT`EjD1X? zED}6W91x>?*3N=W=gBLX?s|OrDm0qwh*VpLv58B~0_8YAuIGg~hHaZrJ!()hTHz5^ z?3x|)3G=avzX?-@R&Zd~i94KcE1PrH^w2wZsmyeLdP6^SJLNSoed$IxI{-*lHdufM zI_y-woG07QeMR}Gj{~wY29Ub5VHbje`lB+3G`e<7Hj6TX7l2!BKM8tztIq6E3PA*? zsLB&i?-*LIZY}^>Mgb3)-4T5HRvwOVOcxYVjd2>AQ?G9_NpdQ@(#kYV{BWbwfafc& ze&3bKokt;GK~(vkfAkJhJK7IWJ<&zqf}N~_r^HM_>_=W@nkqA^*FXDDs?zQ+#SLt@G&%hA#oEn%xU_68)*}v8xb2 zmL=An(=Q&FMbAG_L-edc%4!Jy? z`0JOH>m`f0_jW=6=LC~B5L9?YOU&u$*4(=|KeFzA?xGr9TuORCY4#jcc@Jt+39@gZ za|+!0JrJyseFMTo=0gSlx}oQAAM)s{$tm_U(YQQa_iuVVnm}Uu*L%z=Vv1Jx=SDl= zyOTVX)`tLZ52E#~4JLry0803O*Xvig_awbn3-j}mNNH;li>x5Lgv^dVTg`Gb)We)b zZ6~>G?ntKk2Wbj%yY863iYxMpD!f}2{9|ET%muF{eK9z*lTT=P2~Tot5z3o)PE*R1 z17vn?UXjx|o7uom@lN5Ttxcq5AEP)XJ+Tu??LG0UxeMiATzH{9*+Jn0zE7urV0q{D`K8WQldsO+D zSEGAfzRC{Wv+QIQ3LYee*bERgah_ zRa0xgceMB)6v!RN%>zJ5p`*ku9PAk_$LzO0QD=@X^|;j*^o_}m47fN41?pajP9x<YV{CeKE3)LiyP!0sKeSDCDu6KQ_{LI@ZzKu~ITrl_ zHjm6Y&hsJt_p6^YfZUCe6pR%SQ9QlI59Hf#R#gU7nJcR@ z>ZU2h*f4@F9M{|IK&+`)C5SNC@fH2NoUpvN-~Rw(K%Bqy-^+BoWjbHBXQb0}O`{j+ zHBsRzu69^X)+6&OmXAKu_Z6<}-}rrrE;{O@$8jC4eUtrthz`A6r#{KgW2FxdaiX;1 z^v35>&~}K8l3j`RP0U1wiBPCz1mv>cCOMa?kEcCQ4au%Z4eoGX=}2{v{#UVxC^Pxi z*4f|v``2wt4neI*Vl+_OB0_IuNUI{G0L=s__}cOs6B*Spae6}duIq~_M!o!zGhr`ABk1HAh&K zQX`WTKohQkx5wKnJ$(rAj_FkuEy!h2zR>VBN8i+iT~E}%7`Zsw6TN0DU#TUj*zckH zF+88oDZ(+R3wSM!-6*HQSh}lq%4sxGN;KRNfVo&I@Taa(bf+ROoe-6)DF}s zBDJPAXHPwRn_d5(e}?dE& zTBy0*c6H_~hpb-R+c(*fhc1`P60^}YDxDWM;Y1onB<*1tl@15NK}?r=yc^)(ggPL1 zgd*d(MH03*s>8l;1>`aAMC#1CmNsy%&9-ZMr_DxYwt@T(o7K^5gZM2muw`7OvKiY( zBs{4D8nNpJxn}Mkhr5wLq!r(>jK11_By?)F-a;EvclO%67IJQgjzm9D{1!k=!m1>_ zHNd5T>J7XbsUL7cCQ%>{fChcTL{raN%Cx4j*1D16l*yYo zoHYVOVrvSbEE|F9qDImU!Bv^xLPt7B4DiLZfU~Z(!`5xrImFIcqLnz6y+teTt+ z_UysVEr4z~g*VkUTOY=!xuwy%_Pk<$`H4TYX5~Qf!BNWEu-bbH0%r2(5f|oCDygbC zCI{D5U=9pG(v4-J8>7dxiDlrR6M_RU7dYWSA`XJ5qg6xH6iGXN2};dytpX5wS@1vVI)#kjd>|_3 zyTKhJ(_8_ZZX2AJ@eK%=`5kI(XHU^S|LvdI4fj4_&u{OsZ9wVHJ-h9KQw{@2&exVj z%_xh-aiS2=BWNaI%kRhyiBk)c7TiufLuol#Xb8gh%H59}_5z{$a9#US&c1ow{dVhZ z49?#Ts1;~Rthl+-~x#V-5*!BqA zRo*q}IlSZUM>pH;cRz|gs<&fTEwWb3>L!+pydrhX*Dy@^WBERg7Bl!Ey%9J>zy|f! z{4YoVPp#i-cdkaiZ`^IC9=}qc;=qG&WHd1L(`!?yOBjkb8s9Gg3PtYb zqe@~B;u*>*(l!YOBqzyXAiS;LBVQn5aC;iX8Wx{@J6&HjZ+q1Pa~$ai(51N*NQTaZ zCTEg-$ceVG4#FVx$vN!US**dNgU)UM6R`p8M-izM2OHZhU*BZAz!5+4*i+WCcdxY| zvs_*84m)z$ENdr_;riB|JgnN4Mq(O$Kf7YbIKT0EtiQ0m9 zHiey&sdui0_*$%+W!=3@i3xCSX}0#32E+f>f*m`V<*rC}od%&8o-0sD({85kj&<-W z7pVc{_b}wFpsI%3X4Tij_zAX)Dg`iw_Pw9oW4(2At!4gl=y`LtWjA!2s9zuS&`L5y z_Ka-gL{2`dARZbu!QEjcs7HMo5p{`S5I7Ogo_KPj)i*D&mX0O%^s_J7t-rWWfSpIB z71tI$X(nk-Kq&EoO6-fv^E4wz^h(X>Z>C z^Q+&nAN~9;Yr(jQ`z`i+E@Tp#*wz4x)JX$`Ne9V(e6b&O^?NDIaQB=%*z-96-jj}8 zVTI1^*2Ij5sKfeAn+*&!Xaqov!K&x3-da$wU@{>W{^o+`+YIyFaRb%>8;OYUs=>>B z*3c?2kFl%E_muN2ap09OA880|YzNO`g+ zKq_L=<(PgH^_Vxi$u79)L~zN?*c*@epmWyOy%WIfI?A4IaP`4MK`IQD&)mGs-p1U# z5>x~CR_O}y9Cgnj9cKf!*lNHteZ759Wi+Y$AKiGL?by=|v3DOb&tekBt7q;57Pd#Qox#=OOH^A)!xSLy%i@6U>1HCv%i%Q28S(X^`9#l4ZNfkcE z2p{9Vt*T>o2POx%%;5=L!cY1&{GM?kAjikaO(vzL2HXwiMh=`zeG_&u0ngKkXSO5l z@;82Pmp#93ldV`X-`;)orFP!&^K3SV`lQ2`0nA){<%F(d?ohh5QLJWRo*&d_S)2$J zuC>r_YeQK4N6t)p5dwg~gxL>&y4rT{?z9!l=hRu=WPVb0H*IIZ$IM5Uz57$vI2}xlPiv{~+!IDF4&az`{*WPa1v;9R|xn!1Y zUAM+AIrlW?bi*R#&wBYdwawPpude@$fet>7%qdKEv=%2b&8>fNuRXOM9CO=j>+S8b zlaD>zu6fgiws>x{bJXG8YNr;VaaBynEA2 z+wAFQ)>%KK2dXcQJ?1Do<3xB(psZ%*AL;J{NYf9TeZ)YYNQ1smJ1-r%SN~g^r4rC1 zEsK1hI09XEv^7{CIOey!;Z!?fD7rIG5w+PK zUfXTo{=v@y)QxuJAq(y8Z#vr+0YhscLbs6eSrpxSaVaRbFx}0V#*(T6cg`b+#EH)O z49!o@=?~{KOze7y?{!dt6zc}84;AZ%JKEh3J!`kzvD%tiI_x(-c%`*N=~@eavbjKI zzNH8~b{$k7Jk(xtV4Curd!8IUzIV7Qr>S59tf>TDfO0Jjpa6(&aKvl2n*F~oUS~(H zTwuTXo(rrE_k%gJ27#iQ?C%NFeOyzMPj&M)yz9-?50kTpTR)Kj!G!-AJ$|8YC1Ftr;GqXfTy>77dOFzosjau$IM}d!8 z;Rfp&gh&rizW?!U_S2u<4xMkk9R*N+%av!_9Ds5GT%u^!5ls__Q%2kR;N;)p0hJ)W zocl^Nzj~baJ$$2N*LZ~lhqgaBnOz?p# z7uPBCf7$dCH=1%^kqaxo#tas|^%L~ebS*%giUaV;Mjg9$bU}>H+49*9){f6K$Pt+3 zt-v|0d?Eyv@n5|YgF=-H4aw}DdzFw)zh4Zsz*f+v>~aX*s&Xe(usNli87wm;5k(}-6KeJ^w?2qZAO z@wNxyiIKD8jylZFI(daSRfCQfAtwkhU}AnuUT|;j`sY`bqyIR;{OU(}KV?6slhiF* z4)WZ|g(Ys?x>fC6v1G15xjq=Fs4tK296J;ag-qYE+0_r!7f@fUExp1}(Gbf8g703r za5mI8FvRYm7YC&Q=R)amK$Qk}gmz7LdUYjFjI&Fx+TSs7r?HRR?XWZ%Cc8rmkR&WD z8IV%QLG#{bRJ##fjJ3lqIeP`v3rlP@48*^D>?P=&0nSiI`9bl5){EGFH8CKg_rEXM z&KYME85#}(I5}&2uEQJ*PZ2CuxS#zp5C6R9?U&lAr=4tjyZh{OU%S>Gd~zH7dr&5o z+~S(49^>oKUkq-c@e=Rx9ZrJWP^NDW6|Y_<*aM^Czyymu@E;2jql_#r7lHsWv5Jh~ z;{?N3VrVdV82PprKT|}!_Q#LdUH3d;t%%%u{rM-`d#*Xt8t_{q{7KsZ2G=h9MxWsv+^`r*%cu^T0v}rQiaquGW@|!(-^CZ4 zCC?AYr%+{ZYJ$s#>Z1mp6OG)eNfBcHe&wHt^+<9dyNjOJqx|YzwYCJsgk`iHuX#ub z1$e8lF3-I1qIkJO4>?ps!BA}vIVX4On~4rz{&4S&zBAE@3`RtGeq&*RSB95JmY9nt z7lXXG^^>tk<=~z2^fNCw+$n0%LCxdWQg4(fo5b(SuO4LKffgDUJ{vrC=+5Tfxn!Vg z@N`k`3`#k8wgKqMC_K8;%|Z3gJM%=jv)yvX1NOoz=p8KLu7IFRA|(-b2_!_*HBWzv zS%_gt9{%s6;`M^yU^USH7s36?P(1*TXGEzMVGON>`m6@b0}X?3y5eMe-6iK^Bi1JS z`n9*(W9z$YFJ9d%iH?25{)GyL>N^z7w$nAx|BG@<`ggk6s;jHVID4Q9%#mfys4RrT zBsQ@?0AaX5(K_xNuIIcEe+hynIFq|zAjLO+d@lfTgTiSqz3@!C@a)5b=ni){I`8-P zS_{<<%=8? z$dZNdj+kY4J@lA$5fl-5*Ux+^q?s6&7a0XF-dAmh|KxoHW%)SsU6hd64>eebVN_-? z6oY-3DgbLW42{sYsLrOkjn%AP$KC@~U@q$`2bm}k(y|W&j9iC29|1MWIm~DgYam#tQSFOg zz0FoX{sLT+YV3pWzsAlxb-72kF^iWp0?wqaoQlVpl|pqa zQ?~?`eh}Pm9eka7;K$pCr@wvu1JFn2VQgDu=bye*fYbu@1}A17+tPD_Q_q>q;N-34 zqWRc}h;0zpNKR0Ilh$+lZr61#STG+uTL++Eo5g9xtP&_8yT#LZ{?3B3%JqIqQ-cNr zZIF8A%x<$Ki|1P}7AoqQqv|xxsutBNq3u%djPZz=oMN+c+G40J>-p{(rA{H5k@U|(%^Z5VxZL!dX)5PQ11AuAk5|p z9Dtfvswhwg8n4Cq_vdUrUhu|?SJ)e0f2KI&yB~NGM#(kSiNCerLbbtQlmP?_H3Fa; zX%~sMh=h%Brb4>yY}?DbG1E_D?BQ3{ZB<~7#6vpgwB}Xw(YIToPK_>IwFnF#DAUK3 ze!iqV0`n`KMf>V^Zm{+6ja{;Efqn43*Vys};7>up^$=O*4o4P=g@zfc(E7JZ97!=D=s<5;29@h-eydq94zhiNSvd` z#VOK|$qGbHtHM}fQLb`0Q9Z+z8?fbXdj?O)H)s+|9VGHY|SFMG2gp7Dl zUC%se8i>^i z4;a4lg<>3MLjPH#U<9jhMS*!*(y*A)GjRSqgd}5JHoml3ndk)0(eIclst}#wsZxj2 zF#Gq>2SPG}7Pv5OQPq|0AJvtBN;*o_4(#6s&;`eZI%jglZ+zWJwru$lEwcANw#l&e z<2LT*&EZcM4Q(#Kh|Qp)qSo0R9(kXvbq?SdM()}!rXxi1#hu;77~qFGNKpjbMi?iX za)WlssSE6PKK!dT7i0IpQ)}(tzjC9!+zpjoSkI~RPSO)do}@9bpfGNpi4=k|Qxo^( z0qfOL!m9(AGk8`yuxaztjXm8THRvBaU8g1?IpzlGO}kOvsA3<3HC5Jx3yQ2M;{k>w@Vry@1vdvq!BY#Kg z!@w%hT`q=DWb2oO{o(5!z*eYOVOj~ZIq)eciy5Fr4o5%i4-KG|dgyIiKq_aQc9Qz* zz6YPMod86yLtq@JT7Zg75Ug?JnPi|2fE;QqV2sPTzbmeN8YnzV(&XQCx0JDz-{Gsp z$ggqLWf$6Nk~y%G#Y4RqC>)QEG2Rq%F4Rgxqq8I>#aT2hIr zri6o|yn_Ik(^*~-e?=78sWtR|4W+E*IAN)-hy33t_o+Z+AiZu_`D9$ftvBAg-hO!d z8l>**v^QLGw*A_bud(?c?k0i&ivnlZgsd&};-a8sQp6;erX5tA6IFmIow zPZH5>uy-|!Coz#Gs_kuV-N4+E+;f(=>s|J$*u})QvwL7 zL6*^PgV3FF@e7{nz|Av{Hx|@J zjOD4z+wFJXd9^KTZ?kQ$blSgu`6q^Cg|-z0#gs|(DB2!NSAG=3H=4yrgu9nM7eiw(uWChc73f0Uwpc%?7kXi+;f@sDc->A+lDi&QZMm+UcEK4Z+A+&pkcwFo~(I=pO+O!H{YUg&mvi0UN)4_g)yb>TM3xJ8_W_*BtH- zislKwztV$o38r+SNeuR!9yLt>W7ZezgmkHS4D7aDoeGU+kGEp+a*q^0v_XLukC5mn zzAPX^)=)cKUq+^Iq`vmli zS1g9FbFcl^H}0^Pb`_OVx&}$Q3Rr-tC(ncCn6Qa`?k^ATiJ_e(9T5{EP3iC-G=&`` z!0h8uaUvw5W#nc`o6HF)k12px9>F27X^89Ngkgdl0%1Lq^8kpybL~BL?>!IL0$BIo zdih!Q`m>fHv>IY4b^+nRaB7y$Jhs#sYDx#c=MlGvqIi$zoSX$=;^5^_2Q`SeUXY@$ ze)uEjpLJkhJ9cz-TUTFR9tX(pB+Za-c@;fhCULA3fa4CnR&Q2a~;#Q`M>>Qth?w?xXo5Ip2KQpJWGSgL5V$2J~krjco zLkUDUk+0-hhRwnisIdjxAu%UGgKdR7+}FSJ3ws6$UzW_DZSQ>3>+JNS7ii~+0R)b4 z0#kR(BkByrL8`zJjUFz%8p#TuUobiQ@Y7`rY>UQ_sk4 z>B>tH@=c#q(mUETz^KG1%2G{}AZKcdnJQI+XCQS8eQfg99lOAGHrYI|s8L?1gCjFN z0Fp&&DrQ#f!!W@WNAa!0x5dL?3fhssa6a-WiA5Dunc}-LM6Qz!$^}_knw05H>UW zk)!u2`@dY^y{6u|*)EQJbCH`X)prQ;q1ovRTq=(-vjmeg8-N#FB8cw z1;eP1%KlOgl}XYUh_(&bl7mTc`mh%tle7#VHRW^L;iG)*&9)9n);ngk*)^{_%T8I@ zj-1Mv#NZ5?K(Wn;pkunuqBv~_0&!<@>IY{_Z-o+v)|6bdXyBteb+LBNrKeNMx!ggy zeNj(K77H9Ha<1Qg-)hvg&rUz-SUY;fTo@h6t+1@H&}fo!+7UU)%{Ygo^{0H+PLzVl z7_rttML~-^%ip+VJN&&72EF1Cl}x?8RNUI!GlyP#5QiH=Xcv)l>!bqcpBxg2}6vBVT-&c z!=Z)E^O<;Txl1S2BTK~xOK)WLW*MDxR)otAeG=Fa=Asht&{LV*FZ5(O_(AUQ(Uf4end5fd^<-aJWdX(KO8(0 z4v3a=d_&mirRYl43x2-i{%62NZAPTnBIqKI7e~wl`~AI0)E?U8Qese&&_yrdT{E1^ zWXGS3rK%hFH0BGz$CtM56lZtXq4V`tPe$*YSZR)liB}*2p;$x`oRR@p`sg;ne_O^n z_ex+&krgXeU{TB4I=DP)Ii%r>oU!)ha#>r+r9x*;g>*4oO9!~kRUl5iUUY=I!WdXT z*k`Nex7)?%pKp7+dhEwPzYl5y7*TOgJzXz@+KXkg2th0t`v|fw3x+eTfxj-BGSwqD z~~A#Om8rfj-?vW8d;5`9Jr%A?PbiPOq~_T&N2F%Ma1O9--@5J~j7L9#biYxuMJ zPQ;v;5Zhq6uVV--Zr!3dp{7BvMvhVX%Q9;XAf*YT~XXo|2Q?FvZ~} zaLxS8-6=eod1==sL>JkXOvN2WSRY0h26ZDznyEbHJpny7L_{VZ;uL!s+o9Zs{o?Ad z`^6hx2X`juFSw`NAQqFBwnmAaHB4TQGBv?jLdd2Qc=B|zPMT3k3n0%=Aln7kmEG9s zxoy^Ln}e?M{p!#@s1uZbIULH;Ih9B3+k{lm`lgL$((g+0*20oj+IQZAGm6hl5(eHS?>~*yLlL@lhrrOE>IYH~=6k)HEUQ zMV;O8%NLX@U=Olwz2@{2?N_fl)sX5z1j*e-#D56kn2VVi=A1&ZyH~`|41ZSSU>Q(r zeGvB;52C5p(RpW{oFq&r5BEJ<>+u~uF@AE>?buVV!A?E-cx>c~-AN(?ET?}Ljk!e? zy${4CD{#W4)8v>6qGlO^qofb4Ec^Is`)=%Wh*vLOyigOKTV91;R&-$M36rVMB>pT> zPsNEF%8MBQIZkBK=$pZAm@~WG+S}W0`_4Veb)wuEVPO$Hs3if;%KFNUf|ICl_R&yI z62*sCMmt3ly=pLa^s+g6Nt0c4-Wj$RUKw{i@R;EO17>h|q&g(jSR^B+H>$?`1rEX~ zy(Mdxvk`%r;7%w`GRiG3YV_xt=sT+U1$xZ&S{rp{FrZ#^>N0!#)t6aEYl}Vk%zFFU z_a1<>8T9saS>zMSD2N_aq##dv@Puq3QMwEBaf*8NfG_5t0On*Aq`TrLHp|rq1Khzm z0y5KchJ!{Z>hx;4nJEbn`s6(HcAx*@gZA_LkR1+wn^#J_eD*Bh@8GbwsL`NM=Z=XTV`c(x9-A>H&EZ#JI?j8io{b@!*}M`n&6q=WQMI zx3jUc@oO(QR@@ZRM|#=Z!Gaf7GH8;evn(CvlEq-MBpeWm>Kf)^W)H7jw^0io6Y}uw zo+O_2^XNroO7oACK1%Ts<%Tb$UJA>eq!liL;zm6y+lr;_SY)9df-ZF14g}zM1dLcj z>cJg?Tml%w+bXEzq5 zb3Wy;cKek#zt%e7e)HH<&)Jv0ew*#a>tXRg@*{4yi~dBVXmjBdtfRLE^w)H+B*&nf z9hv+glO~FTpn!u0m{ZzIn?#Pe0h6x)f-N#|BzR8H(8Dcz20&!}P#188-ws0h@(=H` z^_zFt&ON*BSFgFu&O2qP@=Y{DF1M(4(i|>cW2P3@1 zT>voPc5AoY^Dx}da+nBLDEU3?m0=RGF687;Vd-1wEw<7?b3e;_MK9I&>-74$Gxp4xgBTj{E3wBMH{=yFb0vy0N{*$tRp( zM=S;>4fR3Xg(LEHNp`m+A4A|yvwFlhQo*NrQUe!V%0iDI-7yl;cO&<610w!VN5Kj+ zjd^g4!gVvo*-5=yWjkl*{$BbbIlSebZI0i&5!O{1OvAcj`2y^~NPcZR_P= zPOjd)G=`STQp;X7@n!_C1h1)6mD>S=+aFX?c`VQjkReE}oNIPI@;2|CtokJ*D$F)IlOH! zRDIzaH^C=(1N?&B&|-ObqImBjD}a z3oZ#@w|H)cwL#R5+?F;8&7kH~%&E8-CC!LyWu|Z~3IU+#c~KWO7-hWxaZ5JqZSh~#r}Q12N8kbG%tD0vDi4Z*`9fRJwo3L zGPuQnsF2-Lc`t@0ObYJ}{WR3r=1W4kl;tWGE?2&X3iFZko;s>QkaT~rI|nbNK}5h7 ztrp(~c!i!X1r?_7VQZQO=k(nZ=xq{R?)fH*Srvqiy6!@1_+TuP^_xA*6A z4|re>EKV}zBTQ*Nfv~B&VlD!5OL?sRU_NXR`b6e)c<8x3_StXTXq&d}w6l&o%>L-N z-eRkevU3(za%#qCMHWsHhB2YJZ4~v=3JM6^v-3BTI|k**%2$au7x$wrtB^XBzvHQP<7f9wn}41Q&6oo>ElvF9Y03q*YSgsTuwna+(AYl)DA4 zgk`k+7B7N}Zj2AfYq?>q#0yy3Eo z(a)Xs?_d0el2}pl<6eQ}^C=l7YOKidC$A;QW@t9yY%<>^q13Vclp2EP1bT_-aAhx$ zrw>!&f{nq1qPk@^7LEnroZtJ-3+;?kkFl=Ke*5}2e{2suwcU21T2!+17kg0#XdCJ- z>hp(lD}6P)&rQwDoE}OYz0{zeD+fnc3%G8TpjXcT5PX@3PMKt$;=(2g%3)H7%LyaFCXv~x z1z^-+w-YWPKfmoR5OL9tKkg_yVMVJ)_rZOxChTyXfKZsvrAn4EeRAZMB5;r1%Ol>Q zco)7maFx5e`rz6(fczLu)<*3GyGbrlo+dLI;wGyJs#vKC;uHBixqQ(dEgb9-<_c^@ z{pQV?3mZhyUfJ0r?+jni06O#;4>k#xHL}Mf$9;qzO-9SfS85ey1`vV3cb-xb`u8*@ zIYMZ85ukX-eJ>_zkp6z;8wB0fDm`t6(^ud4%-|Ul2vw3e6D}wIsCqbYiKiwXfsaw` zLLCd~RcVm`w&05q9An5#cZQvK+|ky#x7&Vp(;c?vnJtP~%)y}Drx<-F`Ut?UW*{yn zi31y$GlUk2CaW`Z=LxdBWG~@=34hBTfZm<=ud^TB_zUYsgx|&3LHhk~eGPVxh5-w- z(m+laEJ`DCO%R9~@;^Y=pd>;GV9(H5XGg~jYq4J0t6M*)j&QUTq06t@t?UGfJ0E`0 zo`sraF*Xvt^t{vMI>k+18?h5N0mW_R1ifrHZrl|m6q?jBjkL=4*O(ho;l?PLkb7c% zckS&|pD$T3*Y(Q*!yFERT46e)HOiVCxli>i0mYd;cA`e_sKbu}k<{Ds>t4vjzsJXy^;LApA z!Z1juY%PLL6D7-tC#gs)S3-~hD0v6f%VBZwGGvyPR44!a&=X9Y$ltjkF}ZtB<^>}k z?&dy*zjEbCcF9F&Dxb^^Kfl9ny!&bNa|;&B+1d?-=@8skKglGW)UA?gqkJBXwU-aH z`}QMXj;nd4E=*VY5;80ETTVs64BaUKk#7*X2ZV6@{mKw{Np+-tby%rE`*U zUXTl=#}ti)a70S`w7c@C-3$t6sv*59P2@@tfkHKS=JA*6PM4 zWD&!wH|;d+nEZs&WnO&l8Ma~`b~&a<%Ov-l01RQPaKy5Q_}c1+GEsJt6dO5xa;0#7 zfkl3|`flmMg)&w#>^AWbh#)r|3};1YA@pkoP7}!lox_n^amW(*<@Vc#jV~oznrcM3 zpbI&IN{nA6eYKCL4U^#$BpD*kNJbd5sfs!5kQO`r)Dvy@?mc$n&38(j!FnsrYlui4 zx{yRRc;xvOhsZ;z{XafQzaj2fDH(#y1LKA`cOcz~%ESl!|`(I{kH|`cIOm_k+jxF@;rXaXIK# znzj5c4ZBFdSu_^lILKA>o^T&|WjRp5NDe@@72kj3#z*akzd&}lCClyIZ@I!QJ!P4- z;`>JGIpAhQ?v)~A@RR~^HE369!e0YWGuh2KsvQE*M3IiQOlgVSa6EmxcWu8DL`mnC zC8Odg&N>iuD7O!VjYvnk}=^@KioLMHJ@{svK>CZ@nw7Lxn3FBsB3OyA|xhd(p=!- z^GZ2j=ux4r2#a(wkJ5L=_j~U#&j|me8zFSU$_o={FZ8E;zwshUOJr`iC3iF4)Qsh# zuFz-aoVMJ4PMf2i_{@l!OyibPFbndoS7X+s-6_zsXz|z6lU>}Mz(sOkJfVS#y}4bw;+$%iYyMJ z0t5nF^PR9=0Qg zg4pEAGX&_cpSShN$VPW4hK2jugPP%QIapC-je!JiL7n{TCtQN>i??{5*<1wc(34~r zlF~CTgzxTxf@S!P$8>=5@Rcte&mfhNUV<1z#jyz^JCv6hw=yYsGx?VI1f-*&=1Du>+SQWK#ZT^WeN?p9d3FW|~y z5>pqtrdtY?tZ{b2q_7`8IK#kPPVA|%BGH8D4^AR@L=>6WH?03PuWU;<^J?S|cEpZTw!SPvXN51HL+Z@%ahJL9MhWf`M|IEGPY{U=BJyJQ@L zPH<>PraT9U$GPd4-HjY?$lL%9cmVtI5heP;wbKh@H$L8d*ZnpJVXhaQc>?O~IN_W* zCxhGtMo%lc4+)Mn+Vs*EaZ<|{FOrnuVe7tIvM+XF>L($8GfRGadn+oseDMP7gG~N1 zf=xoIilDN}f^r0bq!s(Hchgs8nE|?NQeAZ3nJciDA#EZ>yXEdDJ@lRp!2oz_F6K-m z>M>^pjoV}Z=y$|GE6^y$%_n zGmdJtx4hwen?JV|duDF3@BHlkkl_z^6MzB4y|DdsozN6Np&nm+tRY9s%C z@Fv@z*_ly74W(Jth9{IDriay^Mu%ED;Un6K+Xv-9d<76j4p!-Bw&d;eU%d|BBXJEt z{C9ujDm(QE7^NK7B;dxd+fg0c$4t4|8%x1Co#y0xoP%z_A~HyKK5h{N@pCWg_QRjv zX)kWtU`MZ7W@n#tsMegw8;esBOp_fQ$Ap?t{YQUDx<^!pT}r=I8h?&lwZd!2ya<}M zu9GZ{M%zzF4_$cJisf>p+_ZVCi}*|&kMGGnvyWBNv56$3ci1HEh~eQLq(i$0B|gv) zNa0uBd1cpvcD9SEbSUsR632IW`1yLg~ zAq{odQHQbPP=7Rl9=MR4a%6|S=UrDJjV|(QJh$Hdl;fh> zz+FRRJ!_T2{UaOp*jIn>GiCY+{K4y?`&qi6BLEyGzc?~CQfBiV@9Sz-dLMg_w6Jp7 zEHIe$wsG^zh^(aFFkB=-N;0@{#Iyi9O_X54J_UxrkOZ|D$KG?mkQ+5yItMC^Q%i8-XjeD;9#^V7XnPTV8PmmSJRvegVn&r zjRu7K*Su`syzT~YK?8Q#C9kpf{mO+l4+dba+ENJs1RFHSQvhrwgy9C3>=Er-KDpD@ zYE#WXzIoKW2=}+GSW$lXlb;I+&cXhDOXgteft#kco+p1{<pY--#ML z0j8OpGV$H%#tT#>FfR#rXMg>LC)@JnOYNC;8|?lk;oFSc6`Ufn$z;SMV-oOF<1(Hx z$w+D0eO;1XDFey@0BjSYD?QPdt1jDs(#+{hhU2~OyxfjC>M#T*?XiFP{I_k*hP`r8 zt;0QZ4~*(NM+0tw6=#?Kay(_nX2baGGly}>GYpkoQ%Z@pGrCoaN$nGHTkZevMb<7cmYdoG^w^<`q z%O449fmY3I~NfsLsxy)5ZR#JE(2BBApE2~4+t0jGW-}@H}3a$ z%{VX*?V;epqk(mjA-@$tTs$-enQ99f1`1#c_Ul$~$5^jy30{Ntp9 zYoS>L1~#tRvQ9_(91|suHin~jVcf!(JaV5q&#lmwZ9eNj;op|H| zTfS%k)B}Ck<>o21pg^f6rhgU9JEs{Hi4(1R>J=fKk00f83LJfZt{Z|AjDnB~`ym}t z{m976+Cn$Ho_cLAD*o=bo^K}|dlYusF4|9Sx!dk}e6uIr$BsBmQ9#cqrQQW7OTDB> zp^4Uf|9@^~DGvP;MJn|%_W)C39!+IeSI85?p}7b|d7mWZH}9?;+v&N@GuwKhiyj+P|}kQ{>Ho` zRBfbBW$<9%OrN;PLe@o7>W7fHp502%E_2kfX?IgPtIR=9$jCW^ z#JO)MYnTQ~{BexLp~Q>fPJZj5#ynz}D)nK!qYl2jd2)>K#@LHd`_eaVfC#$I+FBaz zo!7k4PCp9nTI6~#m^qqvnYy=5- zkg5r0`or(FkY*c-{nn1ZuX^3l2%0(5T3gzYI^j{f@%AT$4835|@>FF|3ZoxC7MiTU z7$Y3A?aAie7+2iE3Z?R(*H!32EKtK^Ffqn&{xRTc0w#LdKjNH$0rS`i*e%Z1| z$Q{H(*7Uk&!^Ro%H+}G_9rnQL$C1LbU~k24$n(H%_9FmKK(N1bl>@^xrOuzQ&M{_? zz&s^mKj}i^7MMf5K&Ppal z36kmr=2f1sktGAlngpfe%dh0&EaNaeju~viFl`2NLDGfKa~{!`Qio77VLKp2wqR;r zao!4h=arWr&vT>w@3BRIY$pEX`H$hb^)a?Si~ zQ4f;(JzH>zS7cHxhEnG(sZ1uHAa)y<}g99dAh`!U)=MoUH8MAB?_H!%CYvow_Sj+ zPp0gozb0~5AzODJj34zFaPIz5P~KSIA0+Y{okZdG3)BZZql%pUH;lrq<-n(27cO33UOK5K$H=WBx&6na`awONKP{%nA=F}X=XEIg z|MfTCFF*N4aG?XZzZO~PfYE=0M9d<{|+)u`R)?svcfReMncz zeF5`1`Z^b^858~DQ|8-;-+d*c7-p8tPvsdRT=|tIvVz4#a%VGYZVEhoY{PZfxvb3ul?1o$J0tZlQZ@lVtcK+GRZ4Mr5V=#as=qW-Y6BcxxIlAc7 zyOvqTD8U__=K|nGc5lk&n~P}K`75svc_x4pfXr3{ZlxI$p9{=Qciv+gHos(-z%_2} zZ0CX*R?f^}wdC%#=bF&k4~o)3g!WX7kwck&z$@pfvY%Ox4VzxnLU$-an@b7GJ2?Th z{YZ@wjX_cu1*U*Lx679<)+E=W8M>Bkj^5#A63Ud(&cw%14t*dK#=okb+T|4f?=bw4 zbe7JmwbM^J9xiJ|sLg(P?ZAhY^ zIm=YKb@UjaRLEQ#tTh%xs##K_2CBoD6y5)?*P&{j|j}e(_J9cKu;z2IbxKsNEN%-|z3ZKCv)7!uOpY9bh@Is=BYeiA(D)<*PvD|cAon0`Ry=|t zBfE7DwJ7797sK99MI;F^mdCRB0k6$dw~SxNqug@uTHA&QGlIZ5ryr##DCU)Dl7|3u zAvz=|BseQ->cuHI44-@^j!$J($N1Wtl1QTC4Bjnrp+$|hVOL}-nHJA?b&K@N3|yez z(3|g4k!u=DMq`{^Kl$5ZWvq{2;Ba_Gx85+WL@-d7@cc& zI;uF&%Ni5|j{NJ&^rvnM{Sqb1h?%}Sxx3SmT_IHWgR?(;QN6wEEw9CPRVUa>FTY~n zfhzOCwR?pRIjBHoq-9~yNy%8QG4--Uw^XgAuT5)R@?}q+lI{s##QKkCaf$6vvy%tKANvUh<*X)H z+DTaZIMFxlg2C^ehY-bAYp*}=Y+DG*Yvzu@@O3VLOHf}b#E*BosGV4zglVpC7QuzL zlmcz(c*gPd_(?x~@k+NB@;-?WT6$Fw45Sdbk1?Gi+ToUcUvzfCBO?b;n>V-3S`?Il zfDPsi#&^Ehx>1+U7AP5Mg0O#_qbjGl@#9!*?{I`)xj$l?Sr_`mGWilgVAYaVM3eT} zmX}}l@DfSj z00!sT`0n)=oF<9%-qp|9mQHt0%_G1=KGJ+3?iCf*ptOE*jNB_Ir8UQa=*k+K=ywP1 zm^2mqo28~>cWT0s7zY-@FY2^p>m6v!h|u+=(sW ze)QAZ?1@cz`j6&IyvLNw5qVB^xk8t_$T*df{Tz>$slDa3`XpKYyq2JYpI0mI$ zON?S0Ycw14@AA-66GX+KMDG{8nvT4mM4{Z}iCCBt;~lu~&{N|L0+ z$)1iN6XuZPWcyfsWQGN;EyKJR{qR_5Q8sDwnD4S zO^&DO{>rRG=udm@IXAK~t!V)n>CZZ8ISl0|V(YJ*-G1LQuyug$nvp7$hCnt+<4hn` z0pZ9ldfsOx;2hgC>@daw!-{b8FBY@HgJ^T08le z!>}j}*w?>vo!#`nM(f9K12EPO5cF}-+om3L$!5v+ADYFgri%&dHFP%b3#uG#oKdS% z4yhb~vek|$h3J2)Gbx?i7do9=k|X}zqxbk@`n0UDlk77uIt?^bho6_0B_I z1d#+2pdUSX@3Z~(U*EsY_F!j;la5$s|Ltw(*?g2sLtjl1x>$&tF&Au_{K(-kTjwG3 zCEFupM)5gXh(&hg5lor;BbP_x1a)M7jl<}XRoE&RPatos3}EzNa4Cc&u;!VUwe?ai z--BEj$6Fg-E>Vs_0(2ST2AcSrn(+|49XS4UjFLKJ=TG@4%g9nXu5q0GTj#lBhQ4I? z>~gMvVAxE&$jjgmW_|NIS-dboQP%on;rEahz05x7_oH-TdH-YEJ>4PX#5U zCaKCgsB)sQ(dRlLQWI;E}-a!eKDvvzX0O|rKShX!qWPZ-f5AV+kXb|p+iB_@l zp}r`tc#-6o4Z=@vT%7YPmbze`dFWS*b#!gxorrELK>wPnvu?cVmPfYP^|$>}`sPc| zJK3(e^9kODPJ-vRNJ@DjK z8cK~zTyNLjgk01p8dD9@I5Ib?W%TkU-UKe0 z?II1!c8~?+v;o=HRf6at&6~iejr7RaDI)=F4tz!&RShG=MB?*e{a*)o4Lbi#tn++~ zdwO9aUV{B5{8a`ey8^R`|+|c0I%1CT6&8mdja1Y^BI{UDhDxcX2iDuWH zZfgg(JRb?KIn~JFNaLZQVY@2x9dYlA2@Tk=4^<-P2+*mq(V30VpRunMWJ2*BiT)R1 zkpaiwh5Qp+cI=g&vb?)pHIZ2wLV1x4|z;3zY4%>ueGyIkN_(_8V-~91v+X0Otod=ll zkRC!DtH=_R(|xuku{9~*7U?EyK1>e zI5qXR$faCh!h6(-K^Z(;cNz0sG0(DpKWbEy+7RoQLT<`UWns7!U}E#r6hnbavQ%&~#j-LJKjCe&oOAlT zm*U2j>{)TV;+J9Uqe&}rm}zjl%Ly^yhek5KLy{r>+O>19b$54T!I>r149n3@6~Qk( z5q~8OU?SnHmCo5sk0mfVY{ion4Ee*SaaxWSk>!u$Bbjk5TPR|E61@>7J9Ym{7cI12 zBxYK->1EGuhBM3aC2gZ(I*AGlL{6Fv&txSPJ(J}ECeKrZ(M~0->F!tVbLP?B`B4)D_x+R4pT`oGdz{>!f#F`4E!w* z4IWzf__E+d6}fcLr>DN<7!1k=1r-|o`@!Cj0q8(}T<%76!O`vZvG-qNv!I50;Hh=? zr5o9Xj3nG@?pcrDQLz2zsI89RS@z&sC73K6PAMy3D@!^SkWfC!av@MXkLbQE+D* zy~LWp$uwbO)fNS4&~nV^ISdSw89^>n@-o^RC(|3lTFE28m3L1HD3SI^P|h8)n>?D3 z_{t%r8Suxq{1W0~qn&c<@pi<@c0*QDaLZWfL1`heSrGB5uStDA_Y?LTAybX{wp&I4%-EZh0S+0@dzb2wWEGiRru*SY-zm~vCvyV z{*q2A41_5_Z>49UmsHwk9CyP~{51Tpn_Jtl#TEANBeCPRdU{_-%*0KMDWMJ%_03UD zfG_&Nl19%A*VV^>TSiERdJJJX(-vk)%SAtKD&V=rqUa(rCr~S?GFc{qrbl2F@p%AB z=0SZ26A~mG){Fb#Id=mc9)@xwBpjB1@(CwO15*!+IIG1j<=D9Wk3P*IFG&C|bzxYb z>8|BvI!&>C`W$j{q#!oj2&M}Q?+E-+&>ix<5f!}R{7j~RH0?S3*cv=VX&blH3!ITe z=hzOAYmYs@X%i-Q04UIP4f>4fGJPC;R}Ofg2g`sj5wUHNIcLxW6InURA+H4!T7@?c zOPAq?k|@jIt{9(T8P3x|`eITd8EL&>_Gdi}U_q~4hdoZj8|CuIqfu)M@Y7YVy&RI; z9Q*1wZ*h8u)YhNt*DfJlFzN>E3L8sse)yYRf^x2sZA-jW_+z*y3d)gQB10DnsMu`H zWx9$c9mp~BIp#X&)P}*$Bb;O?4PeY5fm4D$Vz${IefUklljXK{{VVpjp9j7`BmncQ zxy=8lM=yYhV7?cUzQ|u{M4tok^h}q;9Z2_>o7^QdhkKOCn)ny!c(gH`$nOr!3CH2q z-Q);==OJ@*M_v-5@}~xidm3ryTFF7lZ2=a-vrfDWFkpiFLF^6z(-z8leC0=zH=QHDpBu|fd>C(?2e5hZ{q$h*E< z%STw6JrIKi9+G~v<~lIDm6wx@!$uJEKBz;)Aq{6cKnb_pez#oeE;#2jTZ(y34VzTS zxY91h``NdVo=KEBEfA~PKM*Gw|(pS zn{55oZb`8C*;P_u-j{@MC)h{4a%PBNULeCFH&)U$>VrQ-+tYZsmObb;*&(iGk2)(? z$2WthmT%cG&#|OsaZ1x2tWwD0!q3DW5vwI)#IiU#dZ$ zaN_rXV&Gt6Ke_pSILFr@ec@6&@3^Jb&VB*`v1|wbK1@uAm-X}n0JSh#INsBOgbmaM z%I706KyC+cExHr83wH7yAK`F13!F`YicFmfH4;}JNF}aQ><6Ky=S|%ViL?kPzD#ZL2#0+SEO$U1~_C?z;L+}p)X6kkA|JF0&)Qc zSaGpU+`f2c2&o-;C|z`jq28R+O2{7c*$)3ST#}9)RO+ZEL#NWWSBid02cR!XDa^yd z5(&)5AaPC~KdI4pX6J%Q30+$ddO5ris$J8NHY{%N4gPP_l1hirRi zQC=uD$QRy#+~-Yo)EAOAc!Iuht-&tWHI%-hgpI|A9uGB;U>V0yi?Oc5OQp`EG!#ZW zj(h^nkw|=a0^`H_`))l+R0#VOMA%Ft9#l)5Bh2eenysb53JXCalbkRV8}!tI1!w@a zffcjN-hRbdcHU_xTG!5Q`{I|cw}+qJj@&zthDrM1F0{b9i0I=Jq+xDmAtQD;nLnBb zQ;qT<6T)$B7(Q}pBEwqp40#)Uktze?;6=N+i_T&^!mhR4F<}5K{UAkL0PXhW)U$9ou#jMwZy(bwPcgxvyvUWx`MpLBw~=bB4okYJy1zq=mfQm%ncw!n0k zP+L$G)|yIU3kzv&h_{Re+R5~SxHd{fAON2_SRsirIOu{SJd+r~WtwRQA=1lX2iiJL zVh{{hY(0;O)&row`>_}7v1gt~#;hj00!N?1oHH1G|6|4&`roWj7mTOW|53Zwv}B4A(|k^lzqU9K~!;F-21LvcyjkBo*Zt z>B_CXpMcw2lfCH;m)W^zELVu7#C7xqJD`q)#j-F6T%Tl=AaGDuja8hJL);HLi^`!G zZ#jX+lS~x7M@fJn$?$NriShbQK%8LidnzE~#3rHQ{oH++?POgUu~?5yRjI{&0lvLI zyzK$BvCS^M_TCM6XE#GDuBmh6t!5}oj3TG2Jx!KG*~H*#X)(!*GU z+Nu5t43|!H{}fp6&^Oc>bCMz#zqlIbxA}f7t^trs^w|V(oaF3Ww2S<%*< zcEX7#+Tn*T3h>|?<4S~8T{)3sjtu`n%S;k!6>5*ME@07@ftP(wP~n|SmLwwLQ3hlz zJiLQrB?wTYqN|;vih|=;;JD%2d>8|td|pSTiNJd_COA2!EKgdP##USX)U$f~DaRcJ z&J4d(MHAi|<(0?{`a;A$^SKgz`nBK&?wGQOq)c((9M@9t3XM%Jm6PMULDs!nT)L%> zwU#@o6MzLMB0FA2zGP}Cx}FW7}-q=zcfbCG+$W=GKBA%X-y1h$zvYtC(71%1fb9;js0 zUz}@VDliZzTl<%2K60LNv8c0NE{IU;a6Eak2Fq8>gG=r#+q`W%c68ZfwN34|Wa(@% z5E5u22#0IzuSmlTGKy;q@vt8B635RCDLx)kdDLKQ^I-B~@saxq@jp^{*CF@upxt`g zJ=WMh2jIz7g;9ELSc!fFahl6ZAwz83w+?8Ff!|?|KC|9hX3en$^XA*}N33v>1d(vH z07o?z1Jnd=xf9lYCMmB4K+!TTkqYA0aYwGeE+5D$2kx^z=E0a$e~6dS3OIA}tA5%p#4Z zApuY=^oGYBw$zSthq~7fxUqho}Au2k34OAkvFyv zq8ueK>8<550@x&DL8*@Ftl&aHkzIynfJ`r}viCgpv^9e!Pg=Rm=5-*ICfqjr;on@3 zv1a=SJ_*MZY%f&7DZxG%33}i(qBEB-ng^Zdp<1MwQXsJ%ghhEQlRD-q$ws>lz`ci& zvuofY_7cMVd+`Pc`2rgU3#Sdv)JCc2Xa>X2&_Q*hZd{ZLpvg7Qy=X7P17!|88qS3W zQ6thM6fr(UFdUrgZWaf1uz$p$(3ax;$DCW-x33=dhWTyH$e4JXENZ0Da&S(@IgFow z4ojPWneTGRQCbG75+xxIfT6&+)wN(upR}$%H^vR%CZ_qNDoz^~LrU$3N|bZD1rjg+ zu4|cX`TBPIAfDb%+lwus8W5P! zhXG_61$@^+B4^%Wiq{|r-#!M9=R0j+&t`l38!oV4x$*)*5Qj2aYAX@-*zlUAxR+`_ z2DuAuc+30#KU;ppNswD+0X(tYI;~ym9}Zpsm|z{(#8g%*3?W%5aLHZRmu%s(6;|l$ zwOy}l!A=ZVWblpw3@9J32Mfir+Xqp*r3ESzSS9K)p#F%;NE++nT(8bt+!BLH*o9N`#nap>YY35o3eAO0UUXXz^IYnY8OCCBWx z0avHEj+W!fI&oeQ0I1U^=@uJWz>qcDoOugu@9v%22sDTG*29UJe$0cx=)@I(ppnCb zazh^I0_D_`s{x1?_HMCvU3R)%ec1()60{Qqb^KT`GU>6r|D7jJk0HQJeJBB!v&5N) zKGfyopZquEP{26iq}qa=;Ic^&L$Sz6G#B^E4GJKV6A%ErdGu}T>;>xIJQg9Q11JFc zxJ_7{!hX5qGqv$t5U6HypkhuK?s(FOt}Jl7r-2^r{`}$hT_GQ7+G!Z&sg;V{UwPXq z5j4Il2W9~m%45FrfBDbv*rRJVA`8=E`24rXfQ`|RghbUCT@{=0zIxCbb;5(FTrU?8 z+*2g;QP*k<7cK<L**Osd8COTJvXd*s2PTbk@Y>SL-h38Pq$^IeF_?Y0H5 zH+fZo3(YoZqPn`y)pzWZV2o8ixGjZ}24)E8w9)?PFaFK8 z?!k2HSd4K+ZfS9>WK`T6jN?Es?T3d{Gj>PGVT(E1x*DL0Shy679qPaNrH!t_q%(m^ zV25#Bs7{Lk;pjUsXicC;daZHXS{vc9vTG~Gb+>)#U;hkLg*mNf#)de<4DFDwYd~07 zq@s}O-RJHq>;wQ7zSyw6V1Mz?U$8mLj1j%gZT^x;0QoR!Y2pQQJb^9o7Nx_@ML?- zWv98XDY>^1NP2ADM2?b4T<|^+A&l?#-j99SjymlkaKG5@1{_lX3sfpL1~HkWD&-PI z@EQ;yh6nPF3_>t)QcXdwstq0*bo$Hnbz1`nR=Zw zwrzUSKJxx6?TjOqNQ1}r)?)&TlL$mDLgd5};KegLzEQv?_3`A+$gYN=d)`O?$KTnK zqfWN2h6ONK`tB+oLhS|-5i3B4Kt=U{Xo4IUN0>TRw7IumTVq-?R!_Ep;j`ROlewiC znaS@6%=EfoJ_B;f{H_JI1R8RBURi5zJnIm<=JjVQ>aKwX9?~Bt3B4Br_wCcVGBBrN zGJ!eh{wR8b_PSBn|NQjVurN@KBw52mBw*HnpGKNhjzJA}&R|{T&H#yxMoNMg zRxd7Ay#!jkyFvcW5ZpnFT&M=2l$O^D|7Kr_Lk%vMloSYg=*mV6c<+wQcGQAq`|lrk zttK)zBo)`Lh$~Kpk<}S-RI|X$@3g$22pXIJ<;y>@^*g{d&PHZDI>ga;HzaBy_)+qx z!+18~0ZOLW2V9H+=c#o5R9CUy7?(BxSPcLx4|FfowTV~eLCMywD@@cx&e9hnrtED; ziETB#*1vtN{nh{aZ}N_X?X0d;p#Ff?STHL6#0Kb*I@xXL zumAIRU$v(BM<7EUMzJ1SXGzT`YYpm+#RH2%KjvdI=Hj4IH~?R$gGc+h%~m()wqy1z zmw>exM4LpMGbs*n;gaVSB>2HxkAS(M&}lENU1Oj9o8QwOqjKk+Vqk`_6=Pq-&$w?g zfd_zCkAdKNPHX)hm@@MK(HpYVfL?IHG`#*;PCb5JF9S&YM!X#_pd4M6a0R)f~ zS|MIQH0-B(f+yZl3xcPEMIHpx2UmnHi2gaSUft-#UYzu(lyM3`E$;~o99?5*?01ia zp=TDNhX0!c0N-0o%m{6qXa&@X9N-{a`N%^u*-#xtGfB!+VYpFKoE7Tb4X(JKKCX?- zNdT)gq8A&R=$DLk)1r;sp7g?yxFo|2dKhgXlkF=u03C(r(3ozN*(2vl7pt16)52tO zu7MFy;M$m-OHV~LhG;8ux{8K}>*UCpc<>875!lu)ynGidAbT+cJ>&?nP|{$V1MzYk zSQqBGWms!J>a3I88C||GM*V14Eyjp8fjqe=Ou_+7P-a%6E?v=@RAZ0}mH~w>PqafI z+OsDI(GVal5O=j%O`?x<&uyx2Bb#A?KKPgyGJ;d5$pt`*0SSTj z7|*=NBZsoO!ENn@gjv9c&`iA|jVTdD88Y&LrN6_(DB0?Fe7#5}A!~4c+PAgGrz(Ak~VE z+NY4A*`ULoutT+woGHAf$%wb4+zhW-%JIZQpqZ=fBnK{=z2BARa zqzGxY*s3ele$u7jCN64J;r3(k>H`gPdbVJ`(rJM+aH>I>y;Pg*U>xSPI#P-PE<*15 zBoU1b#VX7zxu&Q`mWV2ADYY#$XkmqcXw1gagF&lOfE60^deLA`l3pap=npHp)<*_c z$jC<{8Lj(FgGmt+fFY}E)T#!i(HFWYI}+hZ#zd~#Ot0k0yt(a-fKa+fHv+m|(!ZBc z4|S!kwg3Pgz&?KT&n7VuLZ!W8{R!wCYxE}#G@djxmP@e_-T_R|#P$as(Kljs6cU+5 zDpvEOhCjn`MSu=H6F>Tb9(=L+EWJRWs$m-v!lL^lIpn|ra!g2&c>)>UC1Vn$1gdlK zcAa6~Y3_lTsGsA)?c*3YaC|4RSMG3}u~eIcgf$7LQOY&CkH#1VI8Nr!7p@-g#!Gn; z>gVIdJ`PEk0z=}u`C!-;Kj?t#Dhcw+!62zQkk(zIJny4_x7z71RTwm-6V!|S#*NzP zbu&m28^6n}OYR!=Or@Q%k#RWyL`hD|Fy9BDrRHq_E);YDjf<7no6HT`js?O!aOjs# zk1BHcKAxJZfx+V6vOQoCX2P)|^s+jG&nU0zQ_?+EHc@&2tEZj_YCo^Ta8g}qyX6hR zM@d3q4C5Ucve&q}BQN_qq*YbF`<#;DRjOFdIcI6eDD-0-=$VoPl>~N7v`P0~oOH&M z$}3<%-0fYG1-?_7G15O4v_wNQw&5GsU5Y!Pf2TXCxY*_{Sz$79(B%f?t|{~+BzWY5 zn~}IaO@IJeT6qx1j&ooT+-MEf6}rArSciVKwzeJPL@{C>@O&N~1lV(##XYs)c4~0e zhva4Hq(kAWtS+=T6zY@*Q-&qI(&h$@7#~w0lT! z!b`A<+(x)6a!KP9d7-e3MnaB1I8r9}C3-y0LJ>bV8AZSWITCOM%+$rR5yT5=O=le@ zZ6i1kXxSth^Q=RkkQ<=~H&0i3I7qn{g1owuMMH}fP7*egRnhH=XOgd&0Hui*+~p#w zYSN7Xm@dJq0yST;MqIFg`iZX+LzXyfQDdcv9`S zK@OW^D0Qe!T-?!6fhXN06xoD3 zaOGz$ehj>!Lp`sM)ctRD4x|`&BNUZzQHo3kNQRuWTpuZMidHyyEFUz0IZ*`emwcRNv4>ijET7CCLYy zedJQONTW>UY6$J1;a)(|;-rI;W=g^1a;e5S>b#pgR`v zF)pU66@B{^>K0{KkyQCDlZ z(O$Z-UZ3mN#7H<-Ge=*7tNke)X~u3Qvj@OK?_ zp`Jy620@%POsWnYEcA7B8KS<9E3fvq4y)v#^ryHN_fT&sZKd|V%W_bjmqGxUfYAgx zDT;fvUdMv;T|7v9)G<;Q4xOqjvmg+l4o^;cZGEHF+=VFi8F#s6Khane6&4(HX36nO(TbER10S|#RQFKk1 z0eU$k{88FWbzl6b1;tN7T#_Fg-_!mOzoU(yfgXeW4wYcyB`vH14o*nlb%x_i3rmct z^u>gr#)wGL03+NeZJ^%a8eylLyN@nyTFAH_uz&FuO@HO8CR&dVm^E&3(GGn|?-tQj zf)f6g!6HBz))@k{Z_Px*9>JA5Wd>=u%7yJCEy_a}-;?eLT)sFu8_9S9(mm;2AQsq5 z1?Yu^otY(rOyEPJ^~Zf{;#mS@WvK*`do;NC6uH+xQ)0{l$Lz*OmjFUQgU`Jn{krQe z%b*RzcN21k!}Bux!sYWinYWw);XC$$`kFm}_tIDzW=LQ*oI#mZV2%SSt9O_=k?^Gs z3|tlgCvve)(zNb!F>@Z9mFHGZ9@fo!DLyi65u!1QZpNxd4Yag0&{fp|xS5EEZbbq% zaVznCBoJza++gNI8-?a{l1}=s2w&TcKP5!AjUEjsv;p_gV?!!}(1z$XDgIfi z10TDX50_&o{UI?)`fQ&7YF%7F0{D!5Y|{n7pGn*4tz_O_0&?u*j1FCTP#4KP+{f?b zoN!}y5Z9ybJ*?SWbA0?+cb7sqj%rKb9>vrI5R`Q!Gj4G`(T28TZ0K-mov2cex=;d@ zXf)0-%|nipFI-%lqBEE_st|WiBch-6`i`!00hBnm!?Ugqq7gaJGRKT`pDfBA@{)4N z-oA=$EMb6RCy)}HI@6JK`&Fb5r~Y6UpwAOs08t0JU{W>5YjQ$@pkICb-YLaBW+=^& zj+4!#!rWH-g8|oP9p)_huR(KOA$Fk#l_j+?4(F)1p?ZNh!vu+52gwY|6ya`h*@=J# z!tz^77Z-jn`6Y6g05cLMLaeC{<{WiK(KR=617l=c;EH`x2X2#{#NRcseMNU|kAqbd zX+~|`SZ;`7Swb5GLVu^YcW{g`55OP*TB|9ai9X-7wlKL8x4B{v1Kl?1ZpM>=6+EfR zaYjWaKkKBoMMeaaN=cJDt#~xH@SAwAlLq@L^2I3`mo&M2Mg+A&^{Opz|%a@~Hj6+^N1P2i{Bz zBm2ztij+cemHyo#xTq#^=}ilyBtI|)N^%C`~kpoe)4absG=m` zdUBOYojDh2jx(#G1v@L%y1jsHAHTda7C`cQ1k8~mEq%>BHWmyz7JjwY_=z*CV_?tsCRi7G`mWAKBTo;oTSr)ky$>TP|lkoT8@gjYz zb3AIF+AH{XLyjhgbM5hOw`RMy3=h6fDv?kQ$Jh0+k(2WK^}F8Tb^{$@CXYxCJi5Uzk%neMY3=ppdR(!Ykq z!{r=_8|3j4`pbf9lo%v|4$n`Hfp-0tC|9LkF7y=fey3rc^Cr$=>a;Z$k_P-#GA`M0 zJey#2iSRO_q=xy(tdN|C8HtLv0;x5^LLNP})bE_q0=DGWbdXu`Fn8JSks@_}W+M8Z zk;i31G}N;M02~_x(!on6j&FRkV^$FP)J95%-Y)m_3Qi6jHdH|L8|HTMEb9*U7o(GvhD(u;z_Rs zy1H>)rIH+OSqyJw#9$7hGxoPu005)2xCv_w-SFg7O3saEE8f+P?>cY~o*96$9ggly zcFl27{C@%mV{7PecV%@&0wOFjTts*l=*Ua25%{7;j#AzkOy0uXcxfo2g&TEY+vVSh zcJ+ct=sv`-MY<@_e^U7v|h`dL7q8DxObX#rtrS^*Ilv}wcxulNSb55myvMrCh23{XwZXSF~XeW)66aJiwLOTy#>@h>f ztWuM+y5(enC)1<-FG(BdXY@bo!a1sJj_|Btnt&WUqj>t?IOSD^F_Nyk4Veb%BG!S= zEpTV0=N0F>yR6Dfh~+A25uZidK|1}k4WYjX&Qe}bGLqB~_TVJuOJ^OGW|>ySMN_mE zRReA)9V!_Fjj^P0m!$ZPqEeDVTZ|XvFt>PeJalUQ`8<=?7#Sv5U}^B|=XfyEnDddz z)#&#O+8}+nhYiP)?FF5XPUS7-`DAK2SIHpOx1k*G_i`#6D>E?~YH3+_Wj~evQRz*e zJ$EAGi|KwNJ&V|PxK`nhzV5y19DXP99wN~o9n9&lPNNuBBkCjx01U}(Xl}7SY^chP zZ{QBgeTc1Rr%~);M3pQgbi%|NX<)#WlT=E7CPzUz{dk^7i$gor8XC;R%ap}NFe ziiRXy8vCf?p}5IZpZHCYlf1e}$>oeb(2SH5+>lEX!8-RX!p#^?)C*C6ES@l7MjVd= z<3_H5R%rGa(-scqdzHSmcAy(=zzGXJ(>byp6mO|&kq16<_ttVNFD8|uwZ(A)c9BPbHU+5(=$y}fiu(A3hMZ&J0>F%2q#?yQC_*shG?iR_q$7iA;){X@ z5L36h*AUaI0zjCpkK8c9k-2@DBO-X0Fkw1#LU5guzLo3*`eC5Z3%Uc{GOscF05meU zW0q3Vwl!fwGe0of8C7>NxI{)FmO*ZkDJi+Q(-`M%(K;!|8i30L^ND9CzgsAr+oNuE z0DaErM$N6r<;A%zX@DK=fn73oVqBREgn6eqznYujB8vIg(#$#9sWBz%>z+9@Xw&;4 z1a;u~RwS$!-QqqrHIIl_k`8yKPJ@KX;VWe);_wQdWnU>hB#=k~@d#3q{{ZqYasH)^^UB~Ei)a5FEX4EmGS=mSO z#>2gRyeQFs<;?Xsbse zAUxkv$8r!%+JGGvaBXS8*@&%*29ax^7C{^6)Q+}Rb&mUJq!W15R*rB3JC^|hiU!5_ zv>{5b8FkI=-D<7HF0{AD+Ccyv;Pcy9U$mbmzvt%;)?MBds5iDy-~{eyX+=&9Y`FxE znGr&*&{MZT&&kvKs%W#&7d+>{d4De2JiE5n+Htm^><081LwQxVpy}}@q-v*(Ey?A- zKqW!=Oj!L_~K$FUuC>p&;BJd!S4q*V7Kxn_F8NabC$VKQgv;m$stt=n+v|_Azwjrvl9VhrU zosCF3(}c!#cVY57P9@xuOEGRp9?Rjn`HWr|-9*NKOWW-B7UdJ?c1mqnG)Qahpl6N& z$78nIhQCR#1RB)29rv~3{#n>ox*NGb=C`$5GsDvX(6cbE4Tx}@17ZfBg?>W2aJHdu zc(!6Z+W;CYy9Hy|iuVj+CzQza5T2PIh0FkbaL0Q@ZaF?jRg8lg+x+%sV2|f>E)b;ASFJcj8=zl~n`oAOhB0l$ z*#f3z7BGu>KpF;nY(Dme>p+ArnJIO98Hr9+6d)L3cuI7sSwu(jIL8xxbG5}g-;X-FxauNg4FVI~Hx}{z z=)?lJ4jSorf8=~s>!3^JW&!meW_ofIz;t>OR}AmMl>k$HRX8z&Y%9Hbq*p}MCvg{ z+&QgFz-BNFxo9T%Ck<0FLK`VX<>(>?6&!P;x3*$T!KH~S11KxUgmQpDdJGekZik}( zXwfH4gv!~2)J%k;i ziwzwvx>GtM2QBAprX1kX6#|NRGhxffDPbB7kVN{6Ev%Ak8IcjG~y^dhNIeN!9u3T(sf z?t87R4S_V^@a0&badSNL-F-;3zL#JVL;*=%D1(7wg|1!78u^v~_bK;u;aG+-saTl_ zdyNc_dNUzcA#CXhEf!)XR;nt+&w4A#nPz5&&(u8 z4hXV z+<}qNTME|%AXAUt-8*PowrsKEk39@D48jCJE5IQPocu;rP?*RlpWrMGk!l-ErtZ$| zcE$;-urhPi0^t(`I3e*;>ET$LFex0GUG&l~!SLu^5;<)C9gKGDb`*Qx>gTPwV>Wgo z0aqgj{%GK!LDuzx!g7k_#5qQafgbw#8x~lGYCpSntsQgp3Cfv54w>AD2$+jBfSE~f z2viC{3-c!pyG5Z;r!1b+W{0g<7?^%nV<{b{jFsp&-v*^JFjGC_>$ln}e{*z$jA}~I z(_oLSebIW5bSux?5U58jy=;X3pf|otfJ_}bb0!40H|DH{D9+4k&F%B-=9_P~%PzUV z`a5?)qGKB&8bkc&a~?d;*G6`M^$4J#`e88P>}45&Su>pjLi>X!XfcVdk3LV_SS(jL=1n0WermxZD%7rs?S3|r}BU* z5#-Ol1sl(msvvz5)^SE9Q3M7dGQON!IFc!d!dH+!0^jHphi6nUu(Lcy-3^GJ|Mu;g zupfNKh2jwy?8?Y=@>x8h>xmmOICdnQU}V7rXa!-_Nww76BxvX+X|twZq+vcsPF*=qjn7*O`6C1qIqpe`~4re=+nN~wG{c%mh9pUaHw5tOnxc8R(s!x{6vnK_&M zsS2mz*xsLP2u6(gz-gm^O@hopFHY*TOPxE1_9WlRI=T}B5{}`Y&wk;%cG9V5*(K+! zQX74rByX#RDO_0^%g>Psgj6fqCstBj$pxOttjM?p8XfC6j^#c{jPdkXE9mAxIPN|J zP>>-G1}@DkPIOnK^CKG*9NziG9`a`c@1fA{38~WIq$L+<QXpBTIX-E_9Xy;wE6RXf8kH3iRrF!kL$JdAPqz1~fWw;YyX_?B| zOJ5Uk8*(PhDCbIixdFdK!sIv)~|gr7NHex1=Z5C$x(`#-w}o z3=M;vveKY|`==4s`vLnk*j?BLjS}Ns_dliun0bzAtgOKwP({md?vA3~=L*8$xB)aH zJ-hVJ<&5k+C$eX?<91i09p!zKb-R=Z!2C1ZxPod$RDfehaO8G)q*L}e>6Cuk?sLU| z$q>2h78!&4u)9zWF$oX+@^Q6;DYYXL<{W8^E$G4bzL_iM1m`P3oJ>fQ-ifiGibifW zR0l*_j1=!Jb{Dyj<0N*1R<*!^iqgp>y7vW*aP-M$E^`H}QFR7-;}|H@q)rdNkEh37 zq>K^A^q~jW+OFN*_VhFBY&UK5tP3%XhHqwobZUD)@`#8}QL zPpMaivuwWhCjg*+d$EX;V1HrdtKI|2Km z6Bi4wPpohB%k2dlx&DPN+wjs}o4e$2yYIo(*2%p&V-?t$0WQ>ec zrG9XnW~1;c#PuNahO6ZUxu_oXg_(|!; z@E*Is}d_FA0qx{PEJ@|-EL>e)Qy<`(j}!FqUM(j$i6XZ9Jjb|rEMDhuh^cXO?rDecGLzj_p@5hpeN>?X68oH z6a#u_)TQ1V-`6Q~quK3uV{z?g}hW-C|pIV+R@lb3c5M z?|bxV+qIXADEzY(v%$Q~z63cuyY$5x^374I^lt)<28K2ngDKKAmoZ%_F{LEp-=!0y z(mf77v2z7N=n-zx@aUtDigQs$EBrto-PMminCN{Hjv&*a{_gVkj!v|byJMX1lNA|T zLj~X9M<067cJ0`yy#wit&jDTi=;PLdZCWB>h}bA8gk?7Eu}bY7+RwvH#*y1ydA#B? z?v#`{;s-f+dZZqA5^*xbc&>iNwr|@4L-1VdLc;s`ix%30zkJxXzS1e~h&wCSK|dtv z#(5PL?!{)vjbBU&yD@V6O8dqQpfOsGb#otQ=WsbLYy)g(wt4ejEVNI`J4d^i1EijR z{&{L1jp8%k^3P7#6xZ@dvA$v zfBJ6v$Aig+>)wFfbI(08T=yVNL~F-f+uPl3t5>fU8FC9b8vn_u#=IsWzm^J%uP0yKFm6bJ+@>{hpc<_njuFS69l=^DK30b=`l0oodrwZ%$7z&iZvm( zG}_pQiAzafHzML5+_=|%@{2p|!gEft*BraTmNyq|Q9DFx_^9hS^2ix3NeT9&8)KNz z`))i--<0H5$+*{Lr#tDl6V|L3Ug@?qn|kbl)oW}Kf)lQO-8r@x3gbl`E7M*lSegcuFUyO%AQ2bVQJdbn z@o=^Kraz*bKyXbwKsZX6n5~FObEO|pvj_nv*GU6Z1d;AEbPwKdM+MO z`pJUwzG+G2w~WMZWete!P}Ux~vIAo{8yo9xx2200f{yEK{wxL~rR__`G3K+rC1?r2SSwp5SdV&VR9i~^b*Nb015V^nl2YGp(fqTBkq_9U)Gw3tWTJpFWi zqs#sOkeePk=MX|Y2p`HE6wNetw}8W7;7VWDUZ-mep{J>ZWI#p#0d^!hdm6eJG!#@> zGz2%It-aXQo0KVsLX3SWl?Gao8jq8B<{O_>W0>iruo|kOln;o;Ktv5Pf{~G#Xa>J) zp+O;pVmIS&sV!)@oTM^gGQBgUl;zf5cgP_(NcB10?D!HztbzL=P{~D1j{TlHBOV7D zK+BT>)u%MToC^eI<`@`6q#_l|Z2w?TwySNitzp2+$$l~eIn~elRz3hLD#)2H0N_kF zw?PCRa5CgDf$6qHZvZVw_>!Epa~;u;9#te+GMiG8n^3c}T6O9a`x(_onpA!g8UrzL zTRno3(hgnn=iAag9)5^s*TyQlnqCOc|G z6sz=+Tv5P4j82PEOrRyJ)bJ1r#3?s%?>JN2gr%!*M~%qc9}Rd>v3 zLUUK#YHA9qIH4jY2h2&YIG4;LswSc+Afq0|Kjk2R+cp&OdSy&#_U(g*hU5BNvf*wepjEWJ4T-lHdQ2&Rsz<6ZqD0`%o|#27>z1|;+A#Vn zMp)pZJ%=ulY=siyL)D13dL9tEJ-YJ&7h6hA?3;KU{oT`Z$K-%H)`?bV>SQ=!gC$hS zYJ#4I+!Q;6mQQk)B6zvoI6_6@$y$+TQj8c&S4RViuP>>Zf_W4uC4}`zla~(niKBgH z{h2{YsicYGFbKxULhFRirA8Uh)JE!z3CQkAQV-6Vu4dF*IM(7v>Li1A4Hx3_LkrP< zwi-o~tfJYK6M-9%df-qz*iIU$V~djAl$EzSU1hUwXk($FeQ{5DBF%D;N}D=)>Kb9Td8$X}q!qEGOmYt5zFif=tV!slJ#-+bF_Jn` zRS_mIrT0kmv70?hhS)LxYQH0fQcfeSUfi;iUJ1@uD3exLS(rJFexJ%jsD}AMJ8~0U zhr2)MDHCv(XdCKH|8mlzk{<=S9l<)EcsT5LV_F%{B0vwhmfgLgB^GA4=LDe&V66TYGduQT4H(g;eB|yylO*U5^bzTn z)i+->rfBULLop+hfedWc-OL7Wc*Xfm7DIpw5VN>pN*XG?gs6qg7^xe;)$KAyCHW zC;p{p0^H~1E+^Uz@D_4qsKxtbsMO0=tyE#{c9olM5Z%~)aVEDvlf*Tv7|a82p# zNLHzgAuNJdozkIsbX9sdXL+C-|FZ6IaRC~#$dqu+u4&Kiyf9-#kd5LBb&R5{fF}sC zQH??OF%L*UE?n+195OUXLXvvQy*XIa%qcKN@69C0gpCZ+UCi-E88=EioWv2BBYhK% zl`lJND>eGt_K!O%F+MT!FB-?Fo}yGM3e`fqe;kNnI<0gGkkE7ZYnSv5}0}><6`#HUjT4od*C|C*8^lYvCm= z`yAwSrMgk7sX&l%qg-7xis!Pti~Dr6rWBY{GWE$^gV5G|jvLodw5PZP_bT=k{^n+! zWWC5pbFkyey-!;|u9uY=$f8jT5mWUc-v?XWgph0+j&i{-RLMaw)464rCUzih55UNp zCdO5nsk=D@3nYHb(R!_k8GxDU3WOxrAyGt;?NK}15Xy{%KdtSaNsgiS5QhuW#YUgs z2*tkbz4i|z$Ym5cGXZTSzPYSCL##1bxvNKSFA~~>Kns^^P;Ime8BlCb+zjOec$NAXmtf_7GH@BU*z*O-1bdorl+g zQetb)lB?ve+>_Gn2HJB(1x5w!)YgBzrA78;2m_n&J^UqXQ^r~=g44sV!CPp2sm4+I zfe`&jI@OxQsH1QUk^ZLHs6BaL&Qvff00?ZEkh{6vxQTfm6R2au(g7yb##?jXDD!=u zcY(+LoHwGBSW1V8VY%EBY!H)`85cVH2e5MqbanOInVGwelP;Kl!8xudCQ{{bARGmT zg(=AoCNhAy4uH(@9K=}mW8;GyBIn3$A>A#;6ydH|X_*fNN!rpQ!(F;gcc`d1V|}5B zFG4aQs~*1Do(8=ENtro*{q#R?WLvRa6%scgy|#9fmRntgrtD98%Gjr|PT8OtktP1& z#snNHY}|sav;bJxfrm6pTCHPVaSWAbL5!OBArLrYL+39pw_W&WADtkXVhkCF%XT&* z4R3b=pKDsM0fIl>iR^N0D1DnL3A(}8-9aOzMxadAxR8&1x*9)uV6Id%S$DC+1#;YW zqeC8B+htq!<}Kea%YJ(I6Sj6c#3+RA7F)n=0jNjkGN!Bf6Dc?4a>@PmVHb$Lx)yE5 z`sAj(J@wKatDUpl?s)WB!xo2@$3$oH{5YVL5T>WoC(@RYpJDqNno)O9X94XjV(WKo zYiR4Xo89~P^VTqDiQV(qdTdxafL+4LeNgA@e!)>$aM zk6oRywu*~H>S=rQiFE)f?DD>4r>%ab7Xa>qD4+&s zSGU=ncddiqFweH{%-h}fZWUk^u${W6IYOQr#2qw%xvIrL*Wg;3EPsmXI-z2`KT8#Z z?%pS}G#CfsP!Y_u?fRSVwocsA9XvE9 zG#)ek5I6dyv4E>9==Aw5{r3GEZ?oO~h`Vo_XHTu)V%PrUPTLJ61AV*56i)X5U?0td zAU}M^IYyO=Z`^#8oQ7}dY4*xq>;!(}-L`T2ZX0ZxWiNN;?I*Y0Z!ch*W#!YL2ZusE zDJXbUozp|GG4LmHYMM^_hn9VSiEAXY)o3RR8mfEHhU;#)#hzTd(Hh$3VpD(~`{7S- zv9%jI!*1c^s>-=F@7w*2(TLs{YGX!6EOiC`x%u`r_T~PhY*=aqlBYn*Wl1 zlOZfbV)(d~?%^@h({Qx+ixg31@P&5wgB$F2ZmQcd&jzsn=snnt=V!M);Oa#NzcHo) z>K&>X-4lu8QBc`XB{bU0rttSk0&~{1Q=wL9%tMbngB=8FtZnujh`hDdK6jBl^TH(lpK8Ar~=U%tvQ3YUvzFzllJyTC24e)1`6 z!d6QKN_N=Tl`{Ta*x#@Rk7{q%VEkj9v-E3BvU3^wYg9-m;;oO#} zsR=tl+MF;8#FoxBUx<$dm7@3%w*j?NEgZWF~c1FgYma^ z`p`Ax3&|q?ej+SHhM+**BHN3K=Rm|=Xb5JPC&buk&NzvT&TG7iR&WWLnG^o zw%_^VM}0W2^1*haAc1vnS05ImdKf)BVbDdUxq>VRR)>sh-Thct#We?}If^*(yJ@Gu z$-f}RufgqJ>1&zRw9rol>wt2KQ;*Elwlt-%KVnwFK)7CkNUQshYn~Qlkgg8x;r3ET ztiWDUy}PZw*lCMe5sYkZ(oQg}y5S!~5D1y5WBLWUCCiNKA6AoW) zO@&UY&+W2>EzqHNzM@?cmM>Y9Y=<7XL|<()8Ry3Ew&}WM_?6!i| z!EGY*W@;N6>9?eHK_ZK)Lta}kW=Dc{bGtU%yv6}LbPiM(dtR~Si{_bj< z-)?Wd>|#4|X`3dya+d{Bi=|CxV=K2a4(UYRjJ9I;5^e%8dv?1uwzS!fox82CbC)e@ zYp^RXIonP<3V=n?ip2+YfikvM8{yNo=a^+rBKaH0JsYaL7Ilk_Ks$2YiqZc zw`{f|cK&F_Mzbd$bEv)ktV5-JEOL7(u-LVDjt*9AH&cctz#QDm(%zm(zfV=GJtuKJ z!@;n443I0vSdwE#;yjzbU>SCf?z5LSZM4~KwRXulr`ajTtpH=e1sVMU?ztAvmPFK% zhASA{rmDtcEhjOs%B{i|NA8cT=5aP3 z`{a2dRL*IZD48fKr-D{~Cdbcv!)q6@gDvAXana5Fy-YejuLFs>*%8&a|na36!Z z&bHv6yMMZuGJ&dQwR>Xy9tLf~msBeC5m2BL z8^)wl1ZM_bz>=TCz|iQ~4at7n_I^c+Em~NwO#$lBC)kozOhxMd?NyTB@V*4*GQJ|= z9D{*5sY?<>FolpQ;wB<9GT0-)w73T-!JWXsLxT^G^c8yE`mKHb z=Uex1?YsBpOX%ZdzTA6{?7eI++iUMLbWmofx)X_kCdK2cKgrwGjE9O|@@wN)e!Ex9 zeUcs&v%LbG{X9+BzvaV-GT4%=z7I9DGl`$f7MM(GX#lSI4s(S{{;4SDR8qrx!ORTV ztBC$Zu5r>KW{7yphoNMDlN&OV``e{a%*C$Ug(dICPw!rw>$Fe+lotEIr*vYluhcZ< zW&C9O_(AM>Vl}<^PD~7uZzYe1!ZOT#vc+F~$>Iib)2J^Lo^KS>XN=MYu-?OXL8nop zwThWJf&z|Wr5`4VZn*Myjh0`HTcJhCzm?=e)@*4O-Tf^Skv6T3!`7c%v=Z4*eo=mG^^Z2j4%o(FWScGQ_K@ic+=SJj}Ic)6^r;KVxRw+1n@>!X!3eICZd zqnK+IlLjo}jJWxG?htLC;4++@&q)};0j$!V5e!;_Uh|IOslAPOGQIV~OAAeH7lWT? zCK|;I7H(vVJh2&4yhG)Sc~=}>rxpaUm&7E#Ga;dy! zxLE3Flc4_oKd*D0sF+&I;>N%F2HjxZuxpG=JNlYhG65%bfk5~&WDje{k5z3K+D`8S zOPdQz&IGw(U#pvh*2{?OHClsm%PE^7o+M1>kzbxfF9vX>k(gZAZl_ZWK|6)7H_w5K z=15XWnQskgz%sm<2pnwXf4Nnmc&digU>F{}i-Am8bb_q%%i-DT4s)K)F>UJ+ff?9A zCbO={;J25G^?BEx*r=H=D45y_Y(0CXNL^F}N(qhR5T&TVD=@IZoH0CoUo~emOiq6T znucmZgYFe`?TGWiSFXdcu;F{(TMvY7NvoF^6-v!%^;6QyVs)QVPHEE=1z?TAGY#mp zPeC)KoN}90uc~L#+HphTh)fBJDY6MH9a9^i?nU~JHz__Cy7wzjBJ5RdSgkgl5t$96*HL~%*$w5vz&4~sdmsy0p*u<1Hk}1TSq}F7aRAe@hCvX zx7bG`B`|boYCkFKiOQ*JQ03z+^K?zxJFs|PN76kQA4g5hSJ*+t3V-S9kK8KHtrvge zA&ub>S;*F%^HPjGdCNjGaSjVhXoy#m-hT1JATpCS`#q@Ge}Sar1kEwGJ1}aMXnHdo zFYZj5%c~DuJ+U1mUI_4*VQ%+22bEI_wc4?nrg_$*W!_RwLH~&&t%J1Drg2 z9Wj-W-btAht&dkTK8#h*p|co@RE|eUDNVzqULWp#DLb`2P%_)KVit3gEiLxyb%GU> z1BP3ALoJ~M7i?gf5DYHfBbQTF<<=~rJWz=DUydZ0!DII9TYBBOFB~BnD*Ow)Mb_RA z_BSaPc=b7uH7UCO!_u24GDiE^)}!FPk+bz3r6>X}?S; zkXCZ0b}`C@W~Ux@u z*cQt~(O9vDxlNcS@NL2kg=2q~Z}iu#7LHV6T=JHGlCRAw%7D^Wi!q+LpH_aR*`QM`TDDGvL~Jxm5rEp zgopI6z4lu6^wUqqs6c&OT_*p0#s7(8bJ6y$kFqRu6pIcEzQ3O-6!1tT;7XVJbTZ^gvoM*U<_(7z0a zJMQ?e?9Dgd$`)RD5jT{;^}&Z9Wur$wmyH-PuPssjK70G^w~g;lWn;#?lP$2oe9p3e z7Zdf&B@pp4X476L2EVhim{cqJ75sYh3wTHtgp1yVDV!zqEpZIWG)5!dmjTNsy6Cmn zU(1FLoi3YZnrR$|ao>HHjU0JbHvjzd+oC6yo&PhtYvkS8n77Ad!-vmh%dbrrp@C53 z_5g7^bVF{%w^v)2*h4?eFB1PtK>Fdg}ax4?ghcUU=aJzh~~b=gv);`}4HJp=bE6 zbA0C>U9Vo_J{z><{oEU2xt2 z+_-W1jyvvXdS_66>#etX|G);UT&o^cm}x^ebB*7EUv#&9Df}MVga^GiL{Hj!;)y46 zQ((CXtT)Xy;8=X|#q$+cT+z<~+i9nr=JeQm@4ahyD8BUD?az!bI)=-3g$;2ebmpr@x~iYYwlZc!3CY~!c#nq|uEM%Gurg%-n6OmC)B7lTD3Q~38v_!Pd4Pp`uU3=}dOJTn1s;ivo7$Oe(zxwK{ z^Eu|2qa0+=D#9#!=v)+lDqUp2q_jkL2{*V`m3t@nsXeu|>#n=zGtWG;&moJ-t|NZy#sivB$G`>|OBwjRwM~WoU0N!D|MT>90{kBH&SKDiczfo2qoPYlF zpM8LG4qATn(MOKcl1nb>-9Y#E-h0m#_{SfATy|Y=0LODx1qmGmrwpAMQA8(*Z8BIDW{yWOkohJ^2hzO!~MM|$5R`?$xdqe zZQiBB_|{u*mGGW>?zx$HSso$*1G|%P#IVlyEG`o6F-)(*%JZD#UY0dOfBF~hoD;p; z4F&7nM;pLFyU@+*HqxWk@$MJ#QfBW0tvh~(m&(c-1 zIvO!0BDx7*o{@a(JMGc0(ii@AW_G7D+2ncQ%BK#lnIY_3xYQ5k(oal>(T89s^~i6F zbTjiLy?s-he$JxX%nVbu_10T^Z}5*)64!uN<4XJxeE_1E5q zMG7vkyYIexW?q%U`|`^#XBet2w%EeyAt96!=KeVNWQgi1SI$WR^E`Be1_;?T*IbjC z2jDa;_l@lWNf}DfbhzbQEfrNNA?63*;TKW{`%Ly zT8?$N%P!~6I_oT#W$_){BoC4?8E-iYlIPez@*u+?_`%O2j?%Dej!I*@760mu#V8Ye z2X1tMMYY8?yMwYDZ@e+vZMWUB-~RTu*{iR<>dlNAHOln^=Q!e+lzhZ<5!QNef;;V@ zoatZsOEl|7c3UcD0VInD0uX{v3;q1l@~tk=qeqXhQsG4(dE}8U?A$M* zmo=eXzKfAjDU0b*0ZEbY3=BW^*kiNpw%abd?6S*TfpMNc+KEK&j{wAbHE`yfb56&L z0d)A`hi3;Ld~n7&`s6s`h$FH;{_&5cao`+-kY|8jla@L!FFFc#^*gxUfB*fikQre7 z-f5?uvJXG}uqU_yi@ZbNqkR+>IHDLCuz1j`uDYrZEC%?s*It_~y6B=dNwk2|Sbsb# zFUa@bE+)8SfW#L?Jo=1Am-e}TpMCc6LB7;Cr zJ@&8(vW3e>ue|a~$8o8pmg<{0w5uo|DQR$)C(eUWcKIz2mh<2XemE@4F1u{D!wx%S z>#Va*W~r0xfd?Ka`5Pmuc4W5E;HMq_rkid$|D*qBpM5ra_uY40Vc}#zPa?htAAB%7 z=%9njafrMMW{p!RE?|-p(x-SNzDqe<-q&&5cH3>Ug%(;UJLjBp9HnT)+QBj26Y-Tt z4?i)!=p~#c;Ib5>>m3fFc8)*%;SWAZ$br6!&Ytv=^pWtUZAFv%j55i0`i8DV92;%4 zk>4-Js=xT@{+5cl8n(y-5K=SFIHL<7#AgRlgaOYH#8Fn3v-ETP<~|fR1CMiDX8_`D z(hq~b8peX41_MAVoTRN~mRTlSWRXQOn{3MtJn%qQ{4qo-XW~1hP9a1*PG7)+CAJ)F zW|NtYC-ivy@y8wR_19nDdqO~M!Z7>gFMsJmVUweoO~Sah0;5sZ9F3SK;o}npvdSu} z_^dhJE2#zgkpOjpS~*C>Q$B5ez~OrR_1CitFTBtV&`dMUOr=EJMM}QnW|GZUbz|Wg+zS+;Mw%Tf$ z!o`D@B8Si5h4TUbEGO$ql}Z-imRBqzr1TnX!W*uG2mHV7w%f8xFTK>2;Za8&mHp{Y ze<~f6s1!SqVHsK4k>iF_fzv{&4Sj?hLtkVR8e4AJdCq~6JmG{BGRv)dg!|%)FM4{* zCiBb6ih2%!&Z&QRN@PVEQ;sUvT6x zS5*t@--OHYG=cW^fsa{VV@ypI1J|y<{`%aMSiZ|HyZE0ebBEJ}-zU>}mOr&Eq~mW* zLYOy|TN=eHUrZ_Z1d1jYG_lf>fh}&J9nqHwH)iL}3&<@5EMWw8&YKY}&oSF=9&~OY zqEGPdyYIf<6BD4O0Lq!~W|KYOEWDwSd04r5aJeb_5^AnPFWM3A{lPiPt@=U^f!Xqs zxhY9+4%(Wd;S+xFR_DyC%x&=H_uhN2!^NZ$?eLvRV994a`Q@7W)MTITM`o#-Fk@7M z4sgUt>{hr&8md3&L>u(E?z-#dv&=F}Zn@R6sg+k=*=09QUem8dNj{NzqnE&C z(@i%mVFG5GKXCa*-nQCmtNb7T_(yKeh~F#zK#z!Dq%+?|U%`cpthCZfW#vsPoSI#4 zt13ecXn=^={8pKY%Hy~f#mvmU4h?#qd+DW@Tp&u8Jtz#2uK4>VjWI!yp+;X{jRpoP> zgNKyuQ3Vqfn(&N9q+Q(O;F9cuFN4_T1j_#T0-hEn_&HUDNRG}w|NPu?v##uD2N)2Z z&G!HR3uZ|~K~&+r4xA0}L=nPQKB<7<(;|wp1k&=_P7C^?B(If`ZiZ)7F^UHo2mDr9 zYitLD1gzjDC8M@f3Tjb;KVsfze#jw*lytiD&O1vyX`6N!tH>{7ks}%*bc01eIc18v zRuAracs9c?>J^^F$Usw^2N`fnXShQ|C6SI;(ZqN<=9pug1{OKx76sQ-1zD8jyaG?@ zA&X%1-~H})Zoq^OpH#J2>Z_*YQGGUZI_u%xwPLnV-I=ReMF=8qRThnVMBoc9xWF07 z0k9T6xU$o4?nCk7Js}`t$BuQ+ngI(TQ9}csJE{ zqxU8+M;Ie;ecifb0)I=;eL_h57^_vu~> z@<7QY28l>T^`Y{N=(Fl2Hlsr$&rI^EdfiyJE3Yr|*mA>VB!JSw!Qc?@!39GKFZlyU z=m4K-FY*i?TVB$YyG<~b-~+1$e5bq>ZK7AH@rR7*yKv(-@W}%_<&;x$aOFVm7h7yG zkNQM+?n7?jCC`B?{I%3bjYmwo;ArUte;#;b?9^5#I7V3&pX39v=)DyD~?HHw&IcDWT4DOn= zh<-D%p!}NhD*9BumwU)PagfimnpgoY3DNe?|1deovu26Pi3fg@7a|A7AKyvplmAgk z3X^V{ZB}X4eDlqh*`V+~l$)!<+%m6I=LX>94;VBv9cN+H2XJU6n{&)~13x)F@?dw} zbyv38W}B6zJj~{jqlPx*Q~9J1@>Jlg+~o~7+~E0L+BAmxb-rtIrW2gvGr|v=kuRfP za+1Q4>&ngYUGxS9%|csl}hxd+_z|^ zb~QQL6!&&5maK5~Q`RMVcCr-F+t76^lssh87I?md<~QARlUrL^!)R^vUAbytCS)M@ z#W`}4SY=ija?n?|UPqNKF|6mBEOhkZV)}%zU#sP?sK5BdFFcn>eiAQ2w5jG2Ij`u0 zeh3c<|G|g!2I&wM4JiCJofI6hD)|CWQfVVcjx2N3F~uVf ziTgRPw(zR(Kov!pa`(tU2Oe2HTwCE-PhP9-2&R}D6^}Kx_+;fSksgOl#Ck5B)FU2c zJnDIVc$G?H{OVV~@<>rM<~wPINaOzERlPsTsK%ve&Zp!#@<8WFB`S9gZOl1xJw!en z@1LVyI#iLCU{M5VRUaPdF^?3-iM~gAHHzZYW?Oq@pxL99ir;ljGs(Ipiy`ZvaxC^; zpi%=XO9cRyi`jlk-mvCaX_a1Kfz&SNWBoFo1~cj`=rD!sR+-sjETpU^u<7|uEHlr@ za}i9OqYst92`?jPIkU~Z;E9((xfKWA)g_l)lG$J`JuTrRPej!ioYw*ko1}5?n;B%< zWTxAcd|6xDZRLqPVtLHK0Di(ZT6f{fb$QN|TLCZpVs ztAA)=%FDeRN)c^vmz6m3*ttitA(}PfulSCXpP`J9 zY9M3|JUNhM+QhS>EuIw}^_waryfB+s^9mH|q-`F;y1(0NoaYY3YV-uJtU9?oImktt@Bj@mv`=fkRWtfT`N)YLT z$PZkJa&02epB1iApS9v`*V}$dQK_C&%2R}dC^`U8WioLwfv$_eS|lRDiG!+&kQ9pq zk3I-hR4m_ZSq^~#Ea)75Rpy_5ebQ8h=dpSa}J@<0t*eMIJoFDDw%GD zJK`AW&G)FVD34M>zXV)HLBz#Mn`Eg zU=s}Dm(oC7gN9PdRoI}H91X4di-uJ9OJ=%tC?c-WNb_C%im(G4FsR>16Rydiif57& zrK$8>9D_;`)k~|<7j0@R$XV1_84$%Ri=>>2h!BAi!Ww`8h(IdO7xOVupd~!jfy@A* zeF&x%iBjy*WA4T3xX45q8VDN)yM#ajs@xYiYUz0N8YQSvK%=k=zevNX@Ixos?)HcS zD7IBqnu3*q2aTf#EpJ5kKz9@j{Ef<3#<{8p#Y6Rvf}nljC*#xzzD6{NxJ5ozagK_k zDl?L24Q#F8i;+NdE@XhzX7rRK&%h$%imXs;i*o0@ltEPR-SD^(A7h))o4mT@4IJPJ zxMMV`iY*$JXyAAT`M|*N2`+NlkOkzLqZbYn?TDvRgtS|YexhOJe*KnXBiv<-6ul^# zt2b8Fhjf69t0D^Imi)X{vdUa{IpEa8M>h}OQEoM5eIcQ z_*Ffhsux*p8$pbr9N%Ryc$R^oMF7fRl3NY)ko%8QN;O*gnjom9^=k8&-1Vk9RW1U%9m;-zquK2!S| zgHr4|j{?T1lJX%67Omx+@QG|_#dJH-LQhf|^$1)~nd_XCQM6=@m{heaQ88(N>e+f$ zs2w77BOrRV3YUJ@3&~@D5$1aIX=D`DMyvLFfmZkzUi)#rLI#fKd*v;+!~ck1Jp)=DUsWDP;~RO;eNt|E zCdzXwudyB2yS+#Jt0%klQWVwxWT2}`n|9^MwWH^%?T$CaEalsX_^lL9D}qM_(M&P7 zYG-_0uPU!j`shU)o%Gx6nJUfe@wUp>{@~G{dwY#rr|oqMb2aRi=h-Pd-8@&1zwM6G zMzYaJ9wr8*9BI`lA0z*&v{yvZ39jQ+F?S;*KR^331ttax#E`fOPqS;Cz+P>y*X?%# zQ@79jC*aU74cawg?X=hS0+V(cn-KTJFe-*40|m>3=yRZO>!hD{=`o@C)-G<%yq!)w zp;rAy&#him*y{QIUazJhT0LLoTdVv3)2}zA?2*e|c~~2MBWyBLDyZ07*qoM6N<$f~vbN@Bjb+ diff --git a/doc/v1_api_tutorials/gan/gan.png b/doc/v1_api_tutorials/gan/gan.png deleted file mode 100644 index 0eafd7cb49b545f412f8e775804bcd0b22c42454..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17810 zcmeIaWl&sC^o9w9B!eWl1$TD~?!hHE!9BQJ2qAcIcPB`Ypuyb<8rk4`2Ae#=Rc{JH_ zi=jt*e9RIF88nfqGMRb0cx>3KzvCa~84lm!)hp3#5L$iPd2|4(0< zP;&J1&sCL!9$nqHEr|Hb-vnm;L(7RD73h+CQT-IhcSo&_RPrfx7t{JeC#Ja^7~hpe zprS)%D;b}5X|Hbg%I0N< z-&mErKdjEltEqL;@+;(>)~YRU8H_t;=EUrxxkuMkXrp8zDBhKnifvXyzQZ?6_Ueh**3 z(f>@Er}d3n!v-A|txp~rjD>COLr(Q`^gF~SD zq*0XLY^62^V^BAQGPk^?-qVL@Ad8ekjiE%Lq|U5^OEQz!l?Ohn#Huo z*%0^8hvtnKSppUQQ~6wuvxA7j7U&IIjt=kZ%e$TClvaDbk6e)E81mqKwdi)xTKj83 z|M3*ZZQiCf+0kT}OeT`QVOrNMo!sGB>EyK~BSr4hiXY0tq=x>%m&Y4(r^Xtq>EX#D zwE;%0%C4bwzG;`;iQm9PhCzfnU@+uJV+**K#bwx+50iH3p?4jyX=p`oQ8pE%^Q9_u`df^QQ|oSxnP~MmM`EIxF-pRPTghT@*aVNoTAMnr1Fzs21|9aRd? zz38l>O*HUwCwDH;$m?b^afprLO%21IeWI>w#f4a=*QJ)zI2-ovvxt3um%S+$LOxe# zAW^LkE=Nm?E{6*|>vQq-5KS3&oTHFk3h1?KQK9NooQ%Inv{CX8Gd;x{h^mm<$O&a~ z#GsP1-%fYSsMqH~v7NIY#M00)R5yN$vz{Pzwl$=pzSS2^uKV=(zzFlr*sl3Po!g`j z^=hxI)vP`~-kX{2yq?95&SkM&<)K(Z~^G~ichLgxv%!=c~PC=&5-Q|JGq^L#9Ne_O_M=lvQ2m&gu zo4Ci?yG8f)a9M&*OiUr>@yDn_m4dlA42ItVmouIDS-D8ly+fhZ4PXlRgitsVk2Q7O zMP{t4h8Do$&@fxf<#0NzDHH|6b|Xik5pO2Q{9~Y>?1PbSK;)T8t zhvib0se-X9N&%y+7rpogo(ZH9fDPz;@x@Dd1c`tA!{ua{n8#} zdC8P3yx~w$5jFrqQ)BuI0*To`fo&BHZQLy?W>eQ3t-?bq1e)^|qMFtQRrrRzQY#fM zw(wZB`~68LcvywO5ehWV79<-(Tg2oLsof~)Tn=`$u`kf^)s%)??~Qb0ZaVgb zMl(vf5jRp+*ia2(&;5#oG!p&q{$gz?XEM@t_I;#56dBxfdC5o3nin`wIiaKY^HZC9 z8z-Z*gMMo&k0U=>`%0|(pVFjJMMd5iRUxR#u%~jkF#V_hzEmW6T7POa!ZH3$Z66gZ z?U41@L_Jjhh2arXzQ&n}EbDP9{0ki{cr&&jv{eU>$P z)Yu@Ee7fwvH)mx5hk$*4ptkJoVd~AvKC9z6mGji*ICm<%rZs6o35cWp7Dd)$@T^Hs*)==c9Z*Zypk zZYVaRPS9(@1-sUVEh@!~eZjkZ_9`kKSD#zCt4enZR zw(z0;s&U!-X>szW_VfXeC(7OPzuzMp)PH`MT^?5yve`bGD2#==fI_bvr|n_e`|A^l zd3_j)e+xT@qTG76vXA~9a?lT8B`$^Tbsf5~dQ7HDwBGMc{gmBr+{sT>ZnABCaGiGp zw@e((c&TnWRB7A=6dqiyCr<9XB^ zT3P1*4(Q6ofyTN-Rvkjt%#^O%VxP9hXr$01u1E0zrXx~7rEOPjW1i*^7_U`fB(cHxe!j;A8aEqXEFU!<3DP?~ zTujfq*`>cDmy8G#yg%-QNQ5pp4lue7*E?;$TY?M!bh&R~`o2F7kKeuAEAy>`(Z4-)Vu~6y#&BpJ z_<fG%w7OQUo!#V7(_%8g7scC_Hs`}c@b0qBCh=;r7CtFh`Ux%IP zvP>KA^>A)$_Dt?2cVkZnDY9-`3p&&v?gD3)^$<(}pM3e$2B=r|y4@+@(%ZobX4L|r zalf_paKF`UvSEm8e$Zoc?Q+hF8UM^=AKkD7^#rjFegv48#mN$_@fru)4dCC(ova&n z@?Bdpzuf-r5ku=yj(+y~?TX@6z{wz~Sm4ZC%JpQBA22u|8)tm0MOEn?2Ch@kuH|Nn za(KExk@)Tr+&1|&?1A;asZ#HE8s?78fhz7|fxA^gwyjY;-7S&#D8G6o5lsAK+>}K@ zx|y7BC3)ZWjB%ukB%w>(8FoJ3QPY`DnO<|rAaT#!)8oq|L%wA{(Q{ihH!yg!9j8IV ztn0EPl3@8#HmGtaeTDzy7KX-VcD-nf;caD#SY_DGf0Z@rlY4hVnSfB{@fV2I>TYCvkS`Qg>{g6q5u;d+%||MZsYC;IWpVvP|G+1ci&$Gd!I zJXurxUZ>%@7|CRr7&MRpo204AfYEA$pz5q|%g^CbwhM&qWeYz{pd&TZ@0B*DWj22N z3C(dB>O~-RA_zpze?-i9KH(HnK5t7AcX-|&@&GKlH8LAynj*Rbq^yQAp=X9SKkK(r z^7*Odk_^_<@oj#+N0_swmI`)Ig!94mezaKM~=ra_dco>z#qA~IDSea`=UGzGOih>VTq86G|9j{UT+ueOd_pEB-O21URU*EuX)4~*%c4Z!CuEMbRSC(xQj5( zg#YbM4~o$JA77K#g4Zh>FAGhoG3wSr=og|$#n`6AWZ22cuz<-%_1S=L0r?tI!m477 z5d72!E5=&jIKr?LIP!m|rR|It;CNAe3*(?Kgc)_c*qe5`X++@4x$&^UDtV~? z2M2TGK3i|rC9oaoG?hQ1V76qmuVF!2R(rUasP78)MQY~_{9ZZcS@OJ(EG}*6_X0va zB?;VC>U8!o4=l$(eAd&Ki%AXJ;WVB@Xo9P4T_KTB7NuR#xE{f`&)opTZ(F_FF_#PM zk2uto1NXE~P&oQjkC@a*CTKVXr?!2$=%#;F+xjpBqPvP0N6?{s6W#eTG{|&9@|64ZVnLi8=d>8h z6?J-YJg-rcsfI*#+zD3(>BQpO*0tNz%$h}P%Os1wX&!^dr0C2j?I~NEvUhMT;}T1i zMLlmedh$FkcCiG2%c*~lH%Z3o!&tk(b}GZ+aPyh|uRbD~oxExD+}ZX>BuJ!7{xW;t z$o>KuvE-LW^9|j$Xm7M&J?{sX#w9R(I5+j#qk;IgxHsa6(V3o_IYj)^53+6>x_xL@ z?dI3P_x#;|{ZIsH-rFuUd5qR?XW(<%E%Bdb`1}RoJL1d3g$Dh4?*p085yvOse!hcG z=GOP)`SRa~xR?3g0C#AEbPBZyf^~GK!B}B_-qKtzX5-Mv-)T1X$0|Sb>0{O` z(-Ww&q^Ot7G9SsX-Jhwb*QlT?MT_kl|{_XxWW-S@mNa^{@IrhoYQ?6v*0p65?W znYKpX)H851R9373uh^R}2aNRV!-`&_s!DB}>Ua;tr<)P)e9I@O@rT9~LXhs-7NomY zTCi&NP#7E|PWPkT?8T?A3xd6tp+EtUn`|ra9dOJma*@Q;puFMetF~KS^4PReuhxj+ z;#k9~df2Y5U;4CHuc9b0`HEO@p=LE-(sHuMRE(Aaqs@=UX-jS!EARUkO^=BKXnebb zCw;!!`t)Fj?{pqAL+HNZ=>;-w5y*0l2DoE38u#HeNWv3!MFK5kdVk3=%?@#qd>C2j8==>(7@k$mYLa`D(R6FZq@CI~CpzCJas5F%o8P_Gd-U8|b8!M7t659OV~v(C;eGk_q?38y&S`neJf4 z6QV~7zfQJ-Oo->pC9i#`vEGq-JwowdF`i$a?zS{*MB3C*!KT;#A!-b$Ze|gjIwg9k z0}v3+a_sd#;2aauso{y&7FQeoLXH|-aK{wTL1AE-(|YEQ z2NG)l{X=~OnM8+cPTB)*LB0Fcj4=w|2*`LX>HX|&&YvO1AM7A0V_*$@$?roe!HskB zK+;(M^tfL&Qo^{?Y7ysVzw@G5+}X-SW-1mR<`E>oHK{Z=#IQ?`N!VydO>0%B@amK? zskJ&pOiiz3#Yu=ES-71YBVp`pSWy*1`8lM9MOf>Ra(_^~2f;4u0JF_!mRR`icl=1} zftv9&9K+r2)YGCK#-?O6J9TjHqANp?Mhi5X3GjQQ$FW@QuR=ATS)SWK0^?omSnlhv zcruaf7nGTW0`b*Cdomb#Z6p`ZQ23R!T!shEPA}hm=>%#{&%CEE6@O{WbeV8JCd;f< zRmTl4_ZdXp_$fJGu6x`Gy39=O2MmR`?KiCHYJ!OhQ9|(Uqm2)j^NcJn1I|I+fk$X7 z@r@`_$o9k4a_zg-8f0(M87dVkKjqW+KW#hw5vF?Wy93X%2~&jWW04nytQqmm!`CP4gvJWuXlB|; zLcgzLD6Sv|(rXF1#H=mfE+o3FK~Aj_Z)!oG`~Leo?DRo?DF(Tc)^Y=me(ptOq~!{N zXe8#QKaa-MF`@A-Jt2%ViWz%hl4tzu%uJ+X@jtYQRZV&WD*jw^f^Bf_D`2MQcoPPB zBoQiqaQgZdM6&CnPqVD~W$kfo_Y6sLBBbP|_32Is zDWzRTTq)~Igb9MqJrQmhvT6A~P`OBS+_IenrvMnYGfw~Y%2oE8_quqkZex5MdVg@&-@BHwOK(@|8q$qF^*KAP{?i2`NDh<;1zm49Fa2>0t zzcD09+dIASMPe!!xgZhuMKkQdCrM9s2$c(o|FlvAu`{2X^(oq$`vO-PUjIju9YM(C z5UKsex#CjPj^sJtsoz>)4l$egkRoF3U$HMU=R$E16r@{|L?&Hj6zrwzoAf`4f3G%* z7QjZ${z0-}kXqY4_bt*@>irM_hR(0Lz!To%D1%|fOl?pOy1S9DXdyO!Z!`DHp(!~C zSt$%{b=Pz-*lMLY%kKTlUBA=sM+YG9C*Sp)1=;NMyl#E<<=HuNMzTl}hRiaZ ztk7}Ob}eQP`HmCKRvjvbcpK_^vO_zgY&wr1(?slEekOIQ*T+0|xSJq}jb#Qh{dUKK zwkXbyWvNtIr-}EHT7d{fG6$)KWxhYSDPF3;j>gUWe;aJo%O0cZY?6?b^VztP=wR+TK zhGFUzN`l-z)N7mFDYa>=3K`S-JE0+wRnm5s>(M~19M0#+f0;iR`L z9*;B#8qJjRc9u;JUScV~bqHaRp1hk-F*h;#S&OZ*+)1JHv}HH{*CZ`y!ai3OG1i4z zAq_V>$IK33+R<>?w|SR*C1oY{0MfL1}FXuA1x%=~whtkvS|)_W@29BC1-?&@R2# z&oG2)ha)bg8mn>MWi#b1636j)r3ddX8ASxXCQRAk+M4{L*HW48JJF~e&dO|w`*JDO za745w{>nE-W5qG6QaWx+W+aN*0Oks-l;p{E_h^yW7bRUdKqy1N8^Q#q^E~MmnJTES zh@~CN4Hq5V_9c#w_c~`LSE;_vONxlzp?C0x4f;gf7^N^eGIcG-y#fZYKx4d~=H)}rarymmN@`4z@i@U04KVa!Gwr3=DxI#Z9hokxCEbe?4)dn@*mHxw?kfUh(2Z0oFdHKwsUL6& zk>YTV&pR`Mm+;NRkn8J|rMTwPW4Gz#>!~X^Je+#vV>4a{`?t4b7jkIj@{pa_y3?Pc zWW~u&OU}-1`XTkvN21&PqQKwMxnHe3eb%-`)X}VxvQl}5@{IoIWg)qcLM+`DU*)8q zS9^T7c?|K(6BKMGmG0+PKZ)H!?vq3;*%OF*Z!7SCA?BZ{*73+a-{At z3PX{9G@?BoXRgnCE~r{B1G6d*Pp9&Hv`#XML(iSoASJ^#lG~}~r7T8q_>mxLyMmYy zGLu}*`m5u0=bDko8!bo6vM||Z35vEJfn=t3EaIAVo7JV>=b@o{-QF+V^=7b3Sm%bW z3@lE~23fumm&v6gBH>uQsb%VLTevxQiRez36u2Q+`1bu>c;(TT&w?&Q%8M1@@QV%z zOh*Q*EKO_Ac+d+t*01oDzef+h+N+O(ACahG91>lN9k7u$vM6;vXYi|NQbW^0OFWk@ z(aeCqMt!xp_!CXQ(95y6)SuV|b)8^4@{IqLRYA+^qyFEi~-b5Bu#m4fmLhdKLzXheqxti2lL2blv z)@%V8L8>-8*(b>}Mx3KT<>lpp*W`7K@Ur%cr^CqXam1>)1)l>7^8(VF7?WOC0)luB4)ih0xeG8KGiMa*gFjFHQ*r!u^5uEnC1sL^0r&! z5==wn&@;9k%$|=jp5G;Yvr+ZoB4nQsh$Zo7kA?BDE=d!*cS=;S3-;fvO_91N)biu8 zi`Ms);msvHW*NwZROrq9KIP*#!SfS-n{npPO2hkF_KIaX*rRA7^@QqFAm=aTENR;> z#GaKplm)kZU4Pc=^R(2}kT452oR~z$TlA&Z{uRP{O-gUU7G6GMWEUoCi87Tl`FZA8 zN!Yod_6L0=W{13sBcDo_`0LD9Gd<*4+A?v8js5SNRPsVOr(qx`=ON z@%W4pX!5u184zP>Il}JPv0fk`<;psk6^vk?Y9+ntRE_@wD5r`%U{IY$^^(Sk1oV<}&Im2npwyS|P0XK20h9VEi zp8c#ki9A_?6GD?hkRAxew+h#!@t$Yl2yGLo4^95618*E|7D5LK`7nwra6OKlmJtkA zTgn;Fh4=|HKV<8l7)hYwIMLK!4LWqqArrxDt>(D%yzCE@-e@2AiiJ~|)m0quy!S#> zfAuPc=*PRvtfwx<;d30>TY^nd;tlo(D#6-B~+^=t1;I^*(zbQSGq9I*qI0-^Xc+nsB68FM@xU|v0!ysANj;i3LNbt$j&@*-hu}-o0 zA7XR;oqNCzvvaedbr=S~EV#GtZt46A2x3N=s59DPOy@Ozf7T`*{9>T7BqM9%#qjO( zD=KNnXK&H7gDZ`Z!?T|UVu2HISpHij9(}4D5b#caMbDAhmrv-6Q2C3UGK!^@x5J_6 z2%T6Il;4oB=@YD^I0H3cBgOe8sUNV9;2-F`e)w8t{leAAI~pY}r0-RF*@P_;ox@Sc z*Gd~9_Jetf-3zal9x;{TSrg3*Zmvz-BZ^{V(>1i+U-3BdU)E1d^WFw-5Yc=>CyuL4 zkyy#zcoBaWM%|9|kl51UwZ((0KZr*j^eFk1m1*=Bo-}}Pe&@`c(A~_b6>nt;+QFyZ3wN(~NK^{lSDxZ|xrt`i(3PMiNwu&bu&c%*8 zgA=oaN=jrVZU3spDV#(cPbEWWOXV!ngl&?tm_Q))pv`~R-RVb&Rba)BqL8FkgtyK+KSt%f@k4G?29qnd}@)8p6$pav&+)Gqi}!wO#`8Vpl&<3 z(cnkenInS(j{8m$nva3Q+V9Tq@9>&vG*dMjb-r_CGwev6l-MKpu1grd5c#Z0KYg88 z4{=eW{i^er)zRex4~E1N2e7pS#{2ySj2H6vEJI#%kW$7E6bOqEm63u zLCKO2!E$Ht8VCC~4jk(YVR^ErkuVS33LUm$?R_=x0_o82gUo5x z)9UH+=;Rrw;iyGZ8g+htu9yntu64JpHQErq>(p5?vi26NO)lgmH8l{dCCAwtM%S+U z;81T=AN?iruk8kuIsBO;-o-h(Ql~EcG9{yhWieJQMT0($mN8p4P1b{+>v@2z0lKYL zz9?&J##hcUwZOg;opqh9IE>7#elodDxPhPA?{b-h5OXF%bg#1w9f~5i6ctHE;be1J zz15X@;N7#kUJcApo;lz}QuL>Mv-F2(l?1Q$ckM0t_^-a@L==h87|7A!=f1&T(CSND}qjU-^`z`;rSwBhXViESj zG{fBcrDl2!3uVKV==T99@=Lggt3Ile+(v4plH0F61n|uN;NNCi#K= zG*X!C$ZyE0HJjS8t8yhTG(mDythy{ z4RB!u)hHoI`96L-0lzcFjl@^OXAG8=DQeZIXtFu4;p`y&t9hZ~@FG$~VqNsi2^0>b zut=~xnx9FF5(??yH{nS?4vhTdOtjFv4R=stkgj}hh{g7evL}x@N zH1R2lD&C4kAi)341fi*d5~{Q$M5)Nq5FBj#Jdb+)B5JH;w=rigSQrkjZuWj{cndCe z0|(a8#&|KHY5Yd*F22}CDW%SA9no!bSlvr9d!sXLveL3brrx}`%%Y(ZTHD;9f?erkpbA;N~n3t>aOC?dXc%Q5nFp#vjKIMxRt(aEf7l%&x`8c4NG`z3vH019LNQ1O+$qZpjSrdLIF%)B( zqK8|Uk>x&+DZPT{$k%%Hhki9iJSIl<>&+y5YgZy}iT{AwJZ*atV z`TxtWHYq{MzmQpBLkoccV-(e5@Q%=f4llh?lY@^zGl12{zBEl2hhmwVfE_H+Y2j^z zsPg(m#f0mmNN*;aXH>YLNK;b*%1a7V>DhVc<1u~GNXtKtl5vA-Sg|6Yg!jEa392|+ zeF6CL>nodva^x{k@Iu5b5ysa61f?lmTAN8}36e}m-`HS5V_pL`r;mo7NB3OoylrFr%AX2j z$vYzb`$X0e&(*Vk32|pYF@GV0#1Tg<(t9Xas(8A3kpy3W<6$LBUcG^W9B=~F>-ty%G z?C4)fIV>H@VlgN(rle7)i%*s~1Mn(7LK6qoXCFNY0KUN(Mx12rKR*GsuW>}^X~EZX z$?Hm2GMAxU*g*ugtY*%VGH$ia=h@;fi}8iQWcHzlQ1nH>^2k<8Rn3#7%fp`rfvJ03 zpr8EJhp-Pu>)lE>jy{8k5daG>2`RtHWeP6lST?%sZN1^PwO&t%rB-BeK%TNts@cB= z)tyDqj$-U10tNJ;wek{vCOtSm3ILsWgIiKn5!HvEjOF7WlHM7rV1#!O1UsKDc1_1c zqq5-NC)6&wrZ%3B$u1tRcIs{qr`sq{bzD=70T$yf(&aY?2AK^Wembv95k8l75@FEh z2LLeemBo;3yr+JzMjzj{`xxY6k*6m@4YcV8iw`Kj4(WUcj0TMZgA;WHjYIaDtFCc4 z=K^J-juEWj;QWH~xV()V`W3?+!tN|6CEI*L45?7e6m;d^)@}J*k1z1)msTtkIg3x` zb1StG`^`2IYe5H(M)>J(wk-fBO6>UnsDb$gR-T9U7hlmOCcIB&ewu$zbVX2@5a`xm zGkrHjobNvY`UFB)aMv5Zo0s4yjJAJC`O7~VXf7I1KH#^qq7CH{j@#9^Ss-OK-Q9FG@5YB}}RUVD%&ZUOKu#Sb^Oi-xt6$>nBRGr~#m!PB(`?Y{{Y|;YHuH zUXK}Vs%5$eLP;@h(|%0tPy7A!WgNw4U>Oa3S1&H=eh0;l4(sMpui$L~(C-Ec!>MYn zlWH0dg#rvRL$}G2#9qJdK=nyx-yC1dS87n<1xbH1kTLbZcqIk?6LB|ivJHT~Y3g#g zTJ~;b#0j(~n5Lf8Lr-Qk4%7|$T$j6pG*+fBq z_SmHM(e$&Z+uh;W_&dO(m3pDfH_danr0E+C_Mc3bXtCcREsK!~QCZB})~yCkIOx&N zab>~?y2xD}F3RRT0QOf?>k(8(H4uQJ%ueHhSRjK=Ld}3A8A-@DHEamj>tZMZ zCeNh9#MuV`glTUSNe1OWpBp~TTp*U*n0vTY(e^eQ zpx$*__=-{?T@X|bzc5_Z4rt@ZrAP74n##>q-!xD$t>H!JKL^096M$@<#>e!)kByYZ zC>`dG14Is8XoWUVb_U^+w9owSpX|Z|3h%3_LW$-CDiD}5ZOkaK7b+=|sXh2RKBbE+ zxv#$}M)3wQC+)BvvcUn4p4K8zlm@wv6b1;*;*a;=7eJ@dAX(?F*lO4F?#^G4PzExt z&}}v!yWsVv)IJeTOd|lAZq4FfwC9A&Dy8gLSsCR8QvMbtL7kQakRcTq zeFX{emuth_^RRZ;ow06)HG|RJmQdkRD6K!q_x)3_FfF`&YEGMN{U)|H!<*13gE#)0 z&Gt?}EEloEW$MGZ=K9q6gJqR04e>XVsL(OV=hrjWdtY?p)>J6VXpL4H?jm3xfs(30 zsQI14hr2NPw~#%*%B=!%_){NnFA6SDPQHg9gPb4CPs?ZU&;G2Q`tXk;0HMc z9fCkyn~E^dVb_%|MIN+&U5bIzBjk61hh6j<(JfYCtz zhJa_OT(xr;M3o1O9-U+1sJL4MCNl|(W!$LC|og< z-M+CrS)1(Ly10Dt@!I%{bXIAGP7rXkl;L!$p!9a(HZjM?D4X6lW)1zI@;ung514f) zR`Lnry=0ygBy0vv9u{61_#&vtV0BF8i5sXvP$!%~rxmnjKEhB9xSRPK3(9={_er2j zC|RNqouCVM4cyBc!~AD*D&-EOF1k9J-vIF5>I$WBZC;0}6lh0=$)Zb5w0L=jwlyDj zAT7?CXWH}v#qN6yGKSL6R{%L<-I8c)n0;_!~?U$L`? zFZgnAu_%2`v9ry>P))^z6_o@Xr0Oj8CX3Bglu+RuMUPQ3XK{#AyOb%93xJ&{9A7co zd=n>eucOHAer$lhC(d)ay>0opvt%$EXuQVffZ=45V}n2xg4CaW+x`7(=xKiOsE3Gf zf|=d|m6|gAqa=*!H!O{mj$BOjk4j0OBP9n;c?6PzSbOU_HgFfJ95!(ls%bPCmc}rt zdgJq1W)N@PNWDbavSq$ub83Bg2%?a{2@8Yg!^Dh#&_g;0=^dyo!$HmV(k`q>^&HBTwje=-OVKndi~@d!|3)N52bseH zXovluz8h_wTQJpb-z6ilUx8N$Ko4!QF}L_6n$Ih)nKmetTMGIad`UrThcW!G46}$u zR5faDNVyqhDBC*Ss{tc5wIR(W#K}VtEje#Joi_j(4pme z77`EwFsOsod^xwO^gd7^;9`Gv9CSoEOn!*esk?@*+|s%=M5xzw>q3*<5RF%TiT1-4 z#MQ|!NUAnyy{8LRsNH4^gEnR2i<*fvm!Ley-L}w^MX6CR1d|W;IUJTJiqc(p zTMlST0Qe+LK=KqZfHl`6gdW|w%?5EVFhLKIqOgq0qA)!Fzt9Z)wt~@nMA#k;gs>Pe zr+^Vg`Suk}6Ht_kmc=`Sl_iH+`{K9R-`a@b&*PL70V~-jl~q)*Xi^e|_)VA-x>Tr+ z8%qAI2jNHPMYek8zri8(!DBb8drAEH&%@*0k;Kd=PAzyxsL?xIK$;YXm!T9ku>&mr z0-kwVwu8n68rlyt=(T=+=InviC;k;n5l~-*Zk?~VP3VfCKykbrSgSFxd*=ZnE1>z( z9^BY1sREv^!7rkBOF=tI4IRQM@J{t#9&WW-K0JbA;)k4j5U`J20VO)|2t;;T;P&tL zO;M-`W&pYofB;^%vOV3W-`0^aArT~Hk>WAx&pnaTlq4;_{3mDjmZtm zP=+SKAR=+^EI(e>qGN)-AEH|ORnh4>2jh7_4m!CFFxJ)qOlw;@5@XKfiQP5L}W*G8k z#sLlx(HK{Agnon^3PxG7MC>uRh%`4 z`y}PBK*3O|8$m1EROtxb>-3VqUO3rQ6T!5X(9IbD22$}*=wHR<9EEo~zN}cSSiPup zf1~Mczbeubnm5Hz$~yajJR2QY{eMOk1`g_@kdTAq9#jv_SBRsbF+|U|J#1O>J z_8iUeJiz500@vdhRt;SlY+z*!4oXpqlSr%pfU8<+3IT&^<_QQ6NwpGt|H}dNfPl)7 zCtc@J3D?&>sL)ucq70a8S8&q@{VY*VN%MZZslglg#g8yeOK*jezz`~}n96maSYTLE z1xCp1yUs>$2{H5fjE#sd0QvfV9RsDAS^`z|~1B3OzVq zfy}$EI}}G-)hY_g@nWh1#7uX&#cL>pA|~EJtfUFrT&fD->A;!uD3A{K-WU*Qd|*4T zFz)3gR@z`{I_?PDaAjRXQhT#k7O9b z3X^7wu)>(aZ-aE6F)+UvSWQXp{M(*d9YCc zHNe($-@3G>a%^fBr!J0`_1}V%`=9??@C}Vb(h6W%42y*Af`erY`6v9o*|?n)txVD= zQViaMti3-XfePFwevQj-@P)*QewE{)ph*ImE2TO!wuoU&b|15!`XRxD(Q3A9h%hhs zZwMC%|2kNqA|~u3j382&^q-6PpP=(!2oVKg;QxOKj4bpaEDHBT{E60A0b}Po5BR4h OAQ@K>FBLWL{a*m;J>miY diff --git a/doc/v1_api_tutorials/gan/index_en.md b/doc/v1_api_tutorials/gan/index_en.md deleted file mode 100644 index ac9ed37b22..0000000000 --- a/doc/v1_api_tutorials/gan/index_en.md +++ /dev/null @@ -1,137 +0,0 @@ -# Generative Adversarial Networks (GAN) - -This demo implements GAN training described in the original [GAN paper](https://arxiv.org/abs/1406.2661) and deep convolutional generative adversarial networks [DCGAN paper](https://arxiv.org/abs/1511.06434). - -The high-level structure of GAN is shown in Figure. 1 below. It is composed of two major parts: a generator and a discriminator, both of which are based on neural networks. The generator takes in some kind of noise with a known distribution and transforms it into an image. The discriminator takes in an image and determines whether it is artificially generated by the generator or a real image. So the generator and the discriminator are in a competitive game in which generator is trying to generate image to look as real as possible to fool the discriminator, while the discriminator is trying to distinguish between real and fake images. - -
![](./gan.png)
-

- Figure 1. GAN-Model-Structure - figure credit -

- -The generator and discriminator take turn to be trained using SGD. The objective function of the generator is for its generated images being classified as real by the discriminator, and the objective function of the discriminator is to correctly classify real and fake images. When the GAN model is trained to converge to the equilibrium state, the generator will transform the given noise distribution to the distribution of real images, and the discriminator will not be able to distinguish between real and fake images at all. - -## Implementation of GAN Model Structure -Since GAN model involves multiple neural networks, it requires to use paddle python API. So the code walk-through below can also partially serve as an introduction to the usage of Paddle Python API. - -There are three networks defined in gan_conf.py, namely **generator_training**, **discriminator_training** and **generator**. The relationship to the model structure we defined above is that **discriminator_training** is the discriminator, **generator** is the generator, and the **generator_training** combined the generator and discriminator since training generator would require the discriminator to provide loss function. This relationship is described in the following code: -```python -if is_generator_training: - noise = data_layer(name="noise", size=noise_dim) - sample = generator(noise) - -if is_discriminator_training: - sample = data_layer(name="sample", size=sample_dim) - -if is_generator_training or is_discriminator_training: - label = data_layer(name="label", size=1) - prob = discriminator(sample) - cost = cross_entropy(input=prob, label=label) - classification_error_evaluator( - input=prob, label=label, name=mode + '_error') - outputs(cost) - -if is_generator: - noise = data_layer(name="noise", size=noise_dim) - outputs(generator(noise)) -``` - -In order to train the networks defined in gan_conf.py, one first needs to initialize a Paddle environment, parse the config, create GradientMachine from the config and create trainer from GradientMachine as done in the code chunk below: -```python -import py_paddle.swig_paddle as api -# init paddle environment -api.initPaddle('--use_gpu=' + use_gpu, '--dot_period=10', - '--log_period=100', '--gpu_id=' + args.gpu_id, - '--save_dir=' + "./%s_params/" % data_source) - -# Parse config -gen_conf = parse_config(conf, "mode=generator_training,data=" + data_source) -dis_conf = parse_config(conf, "mode=discriminator_training,data=" + data_source) -generator_conf = parse_config(conf, "mode=generator,data=" + data_source) - -# Create GradientMachine -dis_training_machine = api.GradientMachine.createFromConfigProto( -dis_conf.model_config) -gen_training_machine = api.GradientMachine.createFromConfigProto( -gen_conf.model_config) -generator_machine = api.GradientMachine.createFromConfigProto( -generator_conf.model_config) - -# Create trainer -dis_trainer = api.Trainer.create(dis_conf, dis_training_machine) -gen_trainer = api.Trainer.create(gen_conf, gen_training_machine) -``` - -In order to balance the strength between generator and discriminator, we schedule to train whichever one is performing worse by comparing their loss function value. The loss function value can be calculated by a forward pass through the GradientMachine. -```python -def get_training_loss(training_machine, inputs): - outputs = api.Arguments.createArguments(0) - training_machine.forward(inputs, outputs, api.PASS_TEST) - loss = outputs.getSlotValue(0).copyToNumpyMat() - return numpy.mean(loss) -``` - -After training one network, one needs to sync the new parameters to the other networks. The code below demonstrates one example of such use case: -```python -# Train the gen_training -gen_trainer.trainOneDataBatch(batch_size, data_batch_gen) - -# Copy the parameters from gen_training to dis_training and generator -copy_shared_parameters(gen_training_machine, -dis_training_machine) -copy_shared_parameters(gen_training_machine, generator_machine) -``` - - -## A Toy Example -With the infrastructure explained above, we can now walk you through a toy example of generating two dimensional uniform distribution using 10 dimensional Gaussian noise. - -The Gaussian noises are generated using the code below: -```python -def get_noise(batch_size, noise_dim): - return numpy.random.normal(size=(batch_size, noise_dim)).astype('float32') -``` - -The real samples (2-D uniform) are generated using the code below: -```python -# synthesize 2-D uniform data in gan_trainer.py:114 -def load_uniform_data(): - data = numpy.random.rand(1000000, 2).astype('float32') - return data -``` - -The generator and discriminator network are built using fully-connected layer and batch_norm layer, and are defined in gan_conf.py. - -To train the GAN model, one can use the command below. The flag -d specifies the training data (cifar, mnist or uniform) and flag --useGpu specifies whether to use gpu for training (0 is cpu, 1 is gpu). -```bash -$python gan_trainer.py -d uniform --useGpu 1 -``` -The generated samples can be found in ./uniform_samples/ and one example is shown below as Figure 2. One can see that it roughly recovers the 2D uniform distribution. - -
![](./uniform_sample.png)
-

- Figure 2. Uniform Sample -

- -## MNIST Example -### Data preparation -To download the MNIST data, one can use the following commands: -```bash -$cd data/ -$./get_mnist_data.sh -``` - -### Model description -Following the DC-Gan paper (https://arxiv.org/abs/1511.06434), we use convolution/convolution-transpose layer in the discriminator/generator network to better deal with images. The details of the network structures are defined in gan_conf_image.py. - -### Training the model -To train the GAN model on mnist data, one can use the following command: -```bash -$python gan_trainer.py -d mnist --useGpu 1 -``` -The generated sample images can be found at ./mnist_samples/ and one example is shown below as Figure 3. -
![](./mnist_sample.png)
-

- Figure 3. MNIST Sample -

diff --git a/doc/v1_api_tutorials/gan/mnist_sample.png b/doc/v1_api_tutorials/gan/mnist_sample.png deleted file mode 100644 index f9c7bf7ddd7f148eac4fe347e9c38afaa8876760..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28721 zcmXV22RP7s+b$s?gsf~rLXu>Zy_1z>m64q?Lo%{MD3T;8BZO=jWhU8VCp)C9kcjwh z=l#xgz3077=cNC5e$R8S=O3o6sdA8%iFD7NJqOiP6?O6dW4r$&A;90`wEmUu+2gpb zrYNW9`E35ZhY9WFKQdJp-Qx>C%jsx;7%|Q%@HqJKYU)QiV}XJfA8kcHzR{t%R#mAm!H>6P_ut`kLb&&yp~%4eS6Y2RG( z_V)ISt7&V?EtDDDA7Z{lNhs$ab7G~MVsE4KdT3>3Wv;B2meybAwIdf<$xW%0bT*^& ziH0?&WvK}3^Y`u}_Vn`7Ung>ObljwVawd9MOPJi%*;!8KQR8H9rp~@?9aixuDaj9= zeSvu7L{Hv@?R*iCpmtj1P0G~*3f<}FX|Y#%aXnBMBoi=y~eZ_{Iywgq9GIP*Ws3ahAeN}X6Q1&& zm0j(#nXtz5vx7QKI}|-{u1E ztj!J9>(GX$jQadZnjhHHYQ3^HKa_Nfl!B6ymtEX;UFY?s@srm+=06a-z5FdDg)@dbH!k&Z-qlv3ipt95b9YMde0a59$nhgp<#2<5bC50a98DR<>v z`Pk`n>lTZQ?>qhM-dQ)@tR7p@AHThZHta>25{4wiqhx&kKH^pjPvHv>@Ed5q_17;_ zIW;-iG>+g9;l^^yiOtp7*hi1j`hVh=)zf2Irj(SFL=xtrWwvcQU&tMPKH1C@E6aW_ zEKG-wR?xUKO~#=ovn$ze`xtc?LsdSD=bbyprwBOgN<~Y zB-FIDqJG=nUQ7vX)}hnq0@&~TSr8Nuz)g9*zxq=99scppsGp2CUSECa42spNnKoDd zyv`%_^neS!(n0bZsu#0+t&0rv`iI&*iv}Hp8E$?p>q=9j2_4mEr}F;QKu;HO^z2z? z9%nN%GdVfAwx_3A9$z~2#@PmcsJ=oj=6KO?hHZ>aRpCBU zyIK>^EJmEDVmNd2Yo<=Br2E{!n=6(1`T6~Mru8*hy+0m2pb9k|Xl$Hmjyp~lVTGfG zXN%t~BqT&}K8u7TAT1-K{thW2fe;O?+yfjtrbuNkovUR|av9xSG^?}S87xxn-v-7G zaVbA(vrf>^uXMlBC%^7pWt1y@XU!o&ZE|t3+MS}4^kuzTw=F-#Cap@+6kN1X{ z5509fAt+d4l%4L5azj4b8#Aqb42+Bk@$r|f11ieOa!ksXarZ<-M8a3m5G)2o$m^F^ zR@$sxm6bziu85)f-5dkd-gM7C85llX*U`~oCOq(Tg)U<1*JPt+7w5~DFV)jOTNH@e zcHT{8j%X0^-SYG*uGx6e|I}u$xoiB>d?wjk&eAe1G4Y6^aBxV7eaV-vUrqXscb6t{Dc|>8 zn`0bkwl0zo5^6v8P}W;|2ggfT%46YclLbf8+1rK1#l@MK_t;4SYh|evH8tB&l_BJe z1XTfe;rWiEs4z6&4`DPX$$t(O&7cjY>DpOXSWJG5G;CS@^=ofnlSP50jZL0WiF#_+ zqsNam6GYPM7KcW+otTqPUJJ>MU|suL>9JTy;(L6bj7;^TN9gJwqgYyWXrm@^2>AH; z2J+48zm!~$KVj-S`}t_DzLdoWlBgIvGX^o6c4qB#$s{2Gf%}!MI;qNx)tuDICa zLY5u{oAg;pNuw-1UD|YQzA^{b8()f#9XrOM_PKrdRqE2(+RdTT`7-B8CQ;i5v9Yc{ z`vg^ZGb17*So%Y?`3PN~rDWo9xt1#_P<=!95SE<(Qj)#AiDzOX^bW(xqF}(H!0F3q zV>FwiNCJN0(zNS|6L|vz29e4|mQB)EZQ$e zb@_7tTgTBl-iHq#&VAq{*)4nA`e?N&%lLVc01dQQ-uMVG2*Hfmv3~X|%Pq^KH90A7A+P%k%6=ZQ%ad@{wY@i?m8ow|*$H z)!|AyZc@&7XGG)V|NK4K7(Mg(tQ2s`gO(`@i6^)pEh<8_7_ax+Xc48phsop+O!m{;IQ6J3+&`#Asz@Wsv3S{N(afi_>GP>h(i}b_4k@HF#P&I!;uJ6B5Yz z`c^erTCh&-k!;fUQ0I;T&~qW5ej?=|bi=J3y%W#ezJy5{gZN5?n|T)z=fm%PxyH0g zXyx})Tk0tK-dxGn=Bsd@Z{FgSmX^kU>&7idg#umq^~(ulS07xzHxi1c(6vtXu!8H{U`*9{7 ze|FUfIziQ#)~2QdBqTuF;!)9zr>0SJbP)|^!VaTV!yHH4FkFZO$Gf#;rh0OAlyM{9 zmyR-F+@OEuM31aYw{fWz=I3|n{GNRvvA2sNN>VOZejM#e{rve_*|fJ@5e+H*KUbEQ zF`i}~)QB($o9z+*j2ipaIVSS;z=pT9=!m6=1V`~cxd(X84CEo-zJ0@GqbGiL5NnxQ z24rtblKP{d>Pr=-2$X46X9;wD?(A8EmbuUA5lxnoPGff!rbAQTAEKtV5KA=t+9CIJ z8IYi;uy8C>M_YTT-_)WJ%CTaUvY`NJmcIv zaoqL&><3kuA(clv55i8`%n)S#8csT-YvjQu)o zGbtQ5LdOEk2r}LlJ-uqeN9!wkTxNnxdV*_PYl7|aqWnvzt`sG_4X^joxQXM(ucHvz zmZgQL!#;ohygjSGMgLUPZh56cW@dVNdTJ`}(W6W~{$Tls&!0;>4CGUm^JdHD8kKNU ztG&HBJm+^X<(j;L!tqUl5OW|B^Zr8^nD?U_x^z502jvjiYgjk_n_bVzTQ$0OJzk4a)i|x8USUCIoZoeJ%ZBe8M zV5ej~OF~9gFHGL(%-l#s;2#~W1vJKb&I{zmy8T%bh$VUbyZ7%wi!ft81_cp@DN@tW zT)gmy;2rTnrmf%orrc`HQ!blo_Q}3~vlKaXvU}YZR58XisR$#OeQA}BQmdh<-3anS zmEhi!Y(3dYIB7{kDOkL|cfLGj(@xGP5)>f{!1L*rIV**}VK$E8esYr1!KpEq(6Lvb zy&>0M)csVaRr=YVr$rSc|8h(umWt4BwkTTjAV890Jammf}owY@5#QL_aX~V%(9S6C29|s zsl7da%j0qJDw_`0EIqGppAHRm1O^745)t7G508jAUY~n$xni(0J=23AK$xDO(@uI-n`lW=1L&P2?2q$$>u~+I}2e3)fjS0N_rAP zilf&a*-=JK#`0>lx3>dSywuJg*O~bB_3KxQYuDPWPX!TyxT6jBf~A9U=u5Rw$q~xE z(n`K53Z`+bH?xue?i z*90muf}JF467K`mV=dP+xv(&jXIhPU@@dKi(^DitH74lng2&=W!-o$!SKmu=aG0Z) zfGefzI_sPVJ>XUoDWI4)IEKQHZJE-~KF!N(^QG8sb*B2?+L4@euo%fN!e`FBT^Oze zp0WqG1=1_?+mV)|0=S=Oh{%cak(Irx!E=g%oIlPU<%tKx7k>4f=R@u$Q2sT!P4TG4 z#zy1ZzSUJX21howWXI}%`suoiksmP>FEKb6Up>lBQs_7;9i9>s6T{D}_fnf=BbHk& zido|GOJ#!WL`BX+5e@aBlsiAN^9H8O%^BQRzv~nj>F_qwCw z;@!J@{AD}A`f_q|Vq&PXdz%{^>V)T}TH8_=uN#J#>5a*O0=GrH1=naOUreVPDaW6JA7Qf_37!K_+f-{ z%n~j^*QNiwyS-9uG!OQxATPf#@;+OSUo8f#8&XE=8Evo$hd}}GbZMf7f__)o=E5I-~|+@-N8}uCE3i zUNzmb6fv~0NKxeMnsqbE)z8(RnVK3da}Fen)B}|USg+Z-7ZKqa5Us>cMN3C#ZEgLT zHuU!G+xzxYxUIT*|6R2UPiZz5e)HzdZbi4X1;-f~7<>v)a)68g$hru%3xWq9T||k| ze4$m#Ggiv+pFhbeie9{U@y>jb>TV8X8cHV7dr3hp@h24cV-z_PPG5hVoqau!_#jxy zorSLwW*Y&50A0^H<`0FKtEs7(9l}-MK}v~>5A^nyfR@Y;y`>~4cj$jJ3FKr7$LJzZlHY?4kT!>DdP8flK4kD=X2tZKC&b9;=xAB?rKKe&SJ#!_ zzd=gG+H4LSuL80Xpg1TV)!oGnaRG3txuc`nc`}41f{!G)4s~gm+t<_6)BO4C6XW+_ zLfL~3efA}#TQ#jGty>vh?D*gT)cEg<7c_RC_z*TRVIj`a{Vz*5i&^ADu)H~-V|{)7 zhPQ9d;z=B}OVw0-}~tzdOhzO7hi? zTkYeA5BXB`m1SknY})NhLPA4N803%P_ngiyY?Is8`mgnM&!HZPr*9M4@43#j zJ3HDlj)_a#_wW@Bp1t)$k7xAH@^Zkti^4}VcxaaVjzlVJr=u7srb@Qy@#kOG);9Y` zX}AC9g^TP|%x68Rm=sr%jRhV98Gu)uzVW5UQe@UmGKx8&6nN#KQSSEk=Hk55wD9HL z)7Sg-_-JXB4l)Sem6DN(Zme_~C+JKqEK@P4a69JXRFL#kIrZeRV|KuuvScs1X9p;~ zr(~%t$LoU87E`-~H@5FHo=Vf51rmn(@TK&IF{EkfmEm5O2zSa1!G+0P9SC>k23fEG)Qa$U-w{l4V5 zVUDIzidT5fOEg|=AHlwTY%;#&n^{E;#O%Flnwr)kaxLT|`0aMec4bG^R83d*3k?LEot-Jp zq<}(=iSeoxW@dgmR&nt4KutO=OG5Odr=H%UUVEQk6GT!>b5N2WcMv3sScMNdY(sOD zC}nK?1$FiAWWXK0{WZP4zlSS5xYeqB|2=JbrDq*L8e9kX4$N?8syVJZL(37bVP0|T zTdYyeE+?&Iz11$R2bHHlRKg#Bdt6uoIE@_T3Wc&Tt26%S#68D zeedKy9@@QUL?cVDd8u?_rxy2xavP%r(D2*duqh!XSl-z3*<5?beS5ldB+EcgOlX^iz$Ec8x6r(v7?QOp)A{q~ zO{=_Tb6$tW?XAy0e))|y?eERS>LT$vz?Jhnu{Q?tYlns^G1x)v0wR|4s{V$jcFj#r zuB^^pTxxPXwxGibm61QL*}la0ckktP{$~VxDsG~+i!#{&F?##Rr?nT^bnyQaWo`r# zslMJX6wwfqkkIt})WgPlXKzqg$)t7%F%=~xiI2Xh`8f5X&$1v=4Jr|fr@j3;Jd6nl zj{=jccES)77%qWRsHHGNt96kgr(vPxfyPcF^7^s6S$h0QS9#xo0p&|U%4jODyeKQX z$LT9$q_RpLHE59E=KR}7j~)S54`^P5!T|C=1#oGU+vgzWn$^=hvw~-HP=^_f;o>U| zXM>})SrA!3xeKoaySZ7HRaVOQZb8&Asq#)}n1n8X!3Ux6mgE(4^YMlV`lOf_XMO&< zW!_)wy=;1N>%fZDV6O3*0Co(~T;sQM*|&MrMoMkFUmfg8?XoWs_+h=Yv32a?t6c#b zEvUS_T$^XuRun8B80RafHmY&1OaPDl3se||P>Mr@FZ+LeBxb(YyK6%QfZi6vjnXjm=_%PElch?-6=-)8 zyqg6AwxUTe^5J%*815TrymOv0>6A?rnp}d$p+kpEzLk}giN6&fj&2OpOm zk0(P9=&sM=3;%1H_+rOV>%_`*>FX!vImhFzJKw6u`Mi`AWeH>+iA50Zz2_vFVk-en#h+fU+>A>yW!mX_XU6AuaW1MSRDUyFt==H zoOGWvNY@<%ME0xT(5%@noxsoCyh(Bg4$1{p)fasB=s7!X#?lkj;|mL#jdrozqigcSd`kx?!@8ZwngZN9H>9g}sl1cZcqK?ec7Al%gq zw=zk(WnRte|M?r`^bh?~T>W?;jsO&Qh@!#rIwmG2+S*RrTN{+u_sLON{^-dnJ$g^% z)T#1T-j~{CUcc?3y+3>fou{~XH9N0rg7s7|`zN?PYP$wY;nb6{d-}OX33yda#c&7u zPAtsMmU}E-{h|@z36YUh&LC%~)T)Ipu$Dc(1;9lI^kn8ae~|q6V)c&DLsj>Mud_2V zq2>diAfxZ^cx)_BX(mbTH4eKddL9fNMoLmr5)1yKrqlr zjZs}ghXUAvYdj&!y1TmrKrCSV*}95`+4f0jj`9t}f?T%>Zj$<0uOjG6I%hlF>?PvcYfIrRv1UEG~xxI7AgmTce zo;-O1E$iRLOfpm;YdB|OQV&EYrl*OGV^WQBx12ahKI4tWa9`qwEuPxEL#Wh+pCPa6 zFQ}H3`UJkL{7dcZUK(D_oaL6awqq5RR)7GhQ=%>G+}z(F@Re?{on^(q{0Vl-t!9sY zU~<*K0GOMR)XmwssiR|;%l;1L8xyCnlR--#|K&*%0M2~k+_@{c`j}|R<;>E_d9WAH zUJYOevNkp`d6t;Ci3{$BPp@|%JY`prO3+v{wB8|uPcwGerKijeHrxD83SWg?HxD1@ z4cUt9y?)~HLnJGGbBPF7CGZN$O)ym#*| zX5vm6Kyv_km72?%Xx=7K* z4Vp`F@Zc>E4;w*dDq32?L}k2sy!QkxFG4veNw~`=4bblJc%UT>l{sI#diA*mPoS;9 zfC9k5wQJV^LJf+w`Dh8)e!yvXq&|fb3kVEkv7hOX*&%Pb0oVqGWNCT%wSaSfW1~u5 zKMmoy?6wmSubr;Om>tw<8XgU&o_&%I1H!_kuSHMX70oX$eiTuCQ0@UP@3r3R%|DAp z`nT}+3_kUj+MVZ#l)Auyq<#MC{bvom%E*YdAy4S$uECjL513auYP zm`)E~ceVx}o0!9sE?e)7<@9Sqr=9d3)k4;15ov|tWmjOpO&yk`k&Q!A?XyX@762L* zw?nXU3=)OYwBul5@NKRekQd4FrFRuHM?eY8IS+9=CUW8g4PxH!a1e?fM4Jy_(z2iQ zst-9BWuy)H=Ye`U4h&@21y~6n>h>ilD#&;TNJ$HG--b-CLxgCvUcoIH6w@em;jPAS zXZ8Oyv@|d&cOml3%zie=`H)0JW1y4zaqfGTAEcrPx!W>_T8bnmKL&1<$2zzfdsl#m1w9rx%&ON#`F`4I4^IyYyr0*i%9{$ z<`@=1qtd0l1;R|%T!qdj>v1%;$-v~(-XWQMQhIv&jEoFec(3K-{3q7|(GL>ut2`dE z{crO~jm8e%SMUFtsng}++j#uw2fO^=!OB+_k@^pgEkTo@#p8T(Ff z(QaC|w@l5<#6NmO_&j)SZVuEMy1Uzldxy|Owm@kfsUKZ3+iTu}HhKJ+B&rsofMwy~ zwb}0dwh!k=Z+|&-8rENU%IEIx+VS2(EAYpxE?b@pPYDejI3L=AJ8-LwRC+|~UnZyQ zAqi-Ge(s1v2F_n>lU8$%Ki&=<9bMh&Mwpf0QnTR$-c!(rH2|Ayr7w9Oh!4!RAL(6FjYE}u>h zKuCac3PAewceDOcNWkpk;tc$I_4FkJH>4R@St&l8$YBZE4L;avFBv{Rcc;%*l;F2e zCn3h*utfn{vdC8vjXo_{w^y&Sn(&;=p=D!Yvj2HVf$GEX@Ovl)&J@Mgy@W;}d!$0^qWTGa~sCjuC+dW{-LhdCA zDD(WqnK-~jUay~bHFr0M(b~}wdb7kpU_T|RNwMw2`iZ$ZWMrOkF(0U)B7_q``OUJB zj$nw!V5mkvr+XHu9kcy=rgrB6oyRCc2&8zzcbqS_UUMZ8H~xY$!$cl}IauH{z(FEn z`H3oq+y5V<+yhk0)RMzR~@ArVJ zk52SwZH=SYg98QEB!o1jlPRB9LQBF~KtD<^F4p6G zI0y3wbpk_CH%nAV=q6OvIr&&&@*|?QRKfxIRnL`lR`GaXtgj8)VEPXYNl8foX7xrz zNhNp4xK6iX#yISn6Oe>!dHtY=KDK6vnzXWztzBgnb6_B=yTn698l0c(l5^tTWSJr- z2gwA?j!V89|7NDxNnGHyh_D#z>V7VFouLynk~O2hK8_>|TFHK%?}7U%UF>Xt5^Bw0 za4S=#fi3E0JsKX|Gtr|>d&|`|8}b<(kD4Tb*rw@@hz1UXK|v(??hHE#0a2Un_upPZFLl9*5gZvy<02tYwf1*4)Vjt%G98rFkH8LahisPhGS|kfkwILBAYDqP6`%DZv4Tm2`eL?H?|+PDzmY% z6&R44hoY1%>#o^mC{(RakVQ1;nwY#phXU0?Z$fBIcvf~(iX%LQMbZskQ9$fEPLRaf zB?E&e1_J0I=x4r;+$YTnz7u2vyw3*^q^LDBD6|Yy z2sEA-B@*DN-Fg3TQ}~zkCgbM@H%XQV`JcUsX{gP%?hjO&yy1oWq>~*B44VC-QZh@P zX1&;Ue)rZFq8vLrI}gt^X6n(SM>Xkg%;dR0W+yWuS+8P=Ja7C@v(-D7LU6*ZPFTpZ zW9ZO1(Wt2HO|iGHCvWf2aX#`pFO|KIt>QNEL%Uv%zF85L8+SuPRnUdYRl58>sE9KH zy8!GGjQEk<9WA|)SX4^F+*rM}eE#04G(Ppb{vA}VW{1-Zh0P`5O;eC8Xu%ixXkq@s zBL<8w|JM@Cf2$}W;yg^q{Rf$f!M*8ET;=BF{rTG~u+}V5u$_q^WPY#+0uVoKI7I%; zmwPoYEYJ|hJ;11WCIi)GWd{0_O;V4Hc^%=sfxIR4e}Q)+B3d9LySTcNk&^c17y?4R zd-rbp*NwMlA-1KarrwZ9YqfUO)TF5NJ}C>uW88^YwCI19VRrAUSFc?3TyTiIm%oLf zSw=<%7b$6LcR~R>5b&VM@=B%q6z5k+M`-TWMU2H&x>;wvR{0{jhj^2{Q}p;b8iH@x z+y1U=YrBCbReJQ(vGC9KB{<#y6S&kJDr^4ml-&oBr6N1Uv+RbwdM2q*l$Y>p#A;_2&RuG1Q-nE{_e#pqUZ)Fhtbk zhs@okzsP06d6D{?hHS`gjte6S>f-HR6T#Wp$A6Y`l87Mn2kx}N4P3ecjD*xBfF3)6 zQEHcXe>T;TBM_O&oxUSeYs&fXYo$kC>$wbMd*xqxE{*vsWW7JjN~xrS`TOb9Cqx>w zpApKXrKht3%ce8i8Hi5#FYm*%*Ee9*tZsHQ}Mdhlw`wG`Lqbwvfxo>a`z} z@DI3-bG!QI&)X9(MgG{x%E~%BpOmA5N(zkUA9=rH=IxE4$0Og9?p$+U8bb;}$K0B& zo{deH~YS(GL~* z!WV+#tsjpnE2Ri8)S4b51jcSeyIl1;mlJ|wMs`g9SSgN+*7LJnKYtq<8Upr1aTvwH z!V@sRF_>q6$xsVb*EHo4!;N2>ic;DIp zBC2DJ!fMx`-cwz|1Cu)i;|#InQNNw{044c$#SVkZ7}xoW$7o4;rakfUVMA+XBqILu zk0WiYJ>eAZ>C?A>4qHtT_5(pmO38#aj>q5Cf0BcPgdoqe}13Rlejo5Q#WmGD!m?)0t+Ri!Oqo?1c<^qKuiFofe2Z{(GNI_ zwSN&iVU%>U1&FaDQ_Rv6a~SC4@LwCBn!03S;tlhabg0RThMbg?pO24SO&96Su0M#) zfqDqn2&;h0o7_IQ{#v&CMP=zGr*-%S8vAIJ^r;yhc+@QDrD+) z_w1b3F+w@Ce3ckW4UPNrdT%gSy}i$Be}>4H83Z94jmdZ_ssIX+*FDYbw8g4&-(QeJ7f|YdvpJiQ{Mw|ABtXEhkZ{=n={}p8>(~2oATEMzXF7ab+3-%+ zO?dqHBc}{g<1a|K4SkJi`q@dW<_8BQAS<5z{DnZN>qPYyby>+{p)1YW`{9X9A%+Mi z%>GMpRuUI`h`Dys5%PH`p(-0?TY>3_MgM1Ox=Wkp7c2|A5aFN_Df7hkkbHDSU8kKV&5S zI2cmS!3ZG-9oA)cnRY880kSN#KIb+}R z9qwIe?2*P_Nb-VyVX7rR>cI5CNz%{$4dizCW4hqP*4`+`o73rErR7lPSG2P_uP#68TKDk8I70 z5MVjCY8=E)t^N!N4)#9-#XzrVt?#X)Df{OF0|A8Gybq39248GP&w6sUSRPLe{Xe@` zJtj>zi;yx1nVI$w=jW z4l-kV%%z9n9_@D08L#&vQO8i%(6)!QHP4o8mY&F<&6O)xb{#rb z`Rv|>p|^R>$;i~JC@MnDcvV(r@KPJ%Z6vcqq2JE76Rn`5`YFNycGz|MZahp(974Hb z*CZ`1?Md4-oY1(-dk%l8t+Bv_fBgJ8lEVjy6`%$p{)(u4Ke#>49XfWoi7$o#7$<44 zJo=u$Y>lw=VWlp355SoS^IO~4RNr19{&b#I+S?%gn38b@*2u{_F;nz!r%{s-V%QLgJOjWDCZiYm_E<#+Mr*htr>UuFzFd7> z-Q9OVzvGZdLHBx*sB)iSue+d=o0XMSoT-DEPFzA6ywUw_db+yQ^zR4 z$kKcLU$_DnYiKp|_wQd&*0ICK*x6S`A=D#PlA98xgV`gdZvaL&W zPGY-$_ZUOSg;^wc$xpmTi-fYC5VW@@6B%@x@5OxKSeIDySWoiolJHowP6ci&v2|X{Alea?1LCul;)zx72~Nta*5MX6L%U zy(J02CXPIG487OQ2I(ytok%C_OSZys#Fo=^d(s`e2#gO_Dfb)@u#9eyK)n2~s1+!_ zNWnlWGHjVe^905WKCyqRUqFVuogLuCucfHt?>zimT|g&t#JyZ2GyXw!)bpd z1nzaqyPIaKn+iNtsC;o}9c!EFcG#24e-OAyDq!d=cgRbXQbOQt+^~Vy6y+P&q!51EZBMoS2A65M+^^Bu>4d^LP;ZC&B|ti<|8JErG8Un{#2fi(p#P?$VA8wpaR0(@x-L5TugE7E9~=y4HB^HK%@6V#f91? zgB1{6-I%L+m-%QBfc$|J%b)clNa^KM9;FW3SN;j9c9`=o?r`WT=Jj6}W%{z8Vw^=J zL4cDpfGe81rlsZhv14J}YJDY#_af>RBv%2Xf~m~K#Z~($B0AdUs;3$st+dxF$;Ts{ zqwKyN{-0^YV242x1?8H7z3|4Q9C5U}G#!0B%2eN=uR}0L8Ef3+LJR>zwXVJ%TLEQg zXMnhFK)O&#E<6?8Sg2ft{)3^r_QURMfNpYg-KD>gl^-U+iOy z>2|i#AtEHoKn`5K{yUACnbeJ zdqqTIqNW1`wQ}`O7#4oukbpV>y$lzL@DCgmNDwv`)S}o`<>dqP_}@E?*WujG2`h#M z2LrplMGCO+_k2cBShDeY|2&0H zr6mC11L~?m2zl7Od#YMmew%AIz>@H^%E_wSDv zot|BMG)VF6vo&uN9rxuydJ@LgR;>@3XQjF`9qu#FlMXEoS*zvWWatict_@Yb|7&EA z^wU3Ee}x6Ne@X9LW~P>|Yg)GrEKGPQ9`)x(?{zFvtquR)NtC4uI&T>uh=cJ;GwH9~ zFA4qq&u&lNW&bn-U7_vF=o1Ydv(IoLCJZ*?q@X4BJ@ZUn$N1kpqIsq=Qlri(#5bUZ z-Ez$?vb}**#UZ=4>!+?dAcBbM8?E%96hF(pjEvxi4GmDf$Wfb+!iStGEKNjYhzJSu#jYXqg9&jM`%#Dqra!r} z`hEPnc8TNY^Edld8WZ;(CZ$PgwyKk5MfEqA< z3z`h77=c+0q4Hf3t2G)IAgY^y!*#Zyu?}ZpFIxj_AzD~)%2A92aJ~;VQ zf%Hd?82d)MWNIOFP1tM&cD3ef+h{HYz4zX2Oh;_LvaB4now*I60mB^&A?LgvwYOn8 zB(cH5xp?eUbfa;;*74RpqlgV^Hacp7hnrdfnYH&5I+mXg;n8>j*RRLtl(K%THbx_@#y{uZf?Sq7_}I; zo43vp%0YENQ?H#~`}2pAloS96;sP@CF3nuT6wD+W9?9Oz2O3{dAYAyW2P(a!#{#*I zJp2`_0wUPI3IcQrS3yS7z!8ZGx0It<+tOCXq^i7rH|xj~O5?n5-4;I*)-VCi2ei5k z!Ucnqgp_omu7Z!2^v~ib<;&lQ5dHe|2QaNdk#iQ@YlFh-avnBReDWi|Xz6BCBcnq^ z`VU|Zf)T6)`3g-)bcU)(l$>I?h1IS6d07DQPOch-PctiqMHPKn7x(L?U8t_PmN~vZN0C28?4|>cA-BVhI#N)&*be!DyM&xcuLG$Epot-4~o(T!x>JYZT@q z7q1TF+}j+2$Ao;GIW8y4`WSLZY<^pZKK;UuYnb@}76`EQ3T5lYq=gRc@1>rRHmR-IHVvP_ukLELmLx*aGl5zp4k=C=vVv^C{cfpgM zPEJcWhvr2h2^xr}Awr>xi;Ot%V_^YyA_aqhdzdJi)XeOxnwItuv;O*tPSK#y(4x#f zqT7LkgM+vk*h}IRA1~_Y9IC7rNV;>0!|ccT^v~c`!nEQ}V>KT>D5iE9n3(K_udozu z&fvuL#fTkiFkf(@Kww>6Tny&Yu+q5arvw%$x<)@!kr3i1=Xc2w`(MpNsuBCQb8g1C zI(i@6G@vu{Y#vYe1S{2#`%X1j; z&?;R@XtHAilaE*Bejuw%*#|UXa6Yi@3=9>?-M^SJDQ>*%vNag;#B$hP01Vps%M#I3M)+6e}w$ycGbZX8>J@nb+9u;j*cI@E7GImr+D_`0!C)UNHD7 zv=FFwNQk0C?{3e76d<6q>)9U;y_JZ8aHGTsn~r74(kQrJzXq}JrL3$B?q(?_WhAn9 zY-)TS^Qv|2j0^#GQn?)fx z(0!ABdhiWjxR;mL_OC`Z{p?FZ@kLda{(o&T8+V14W@n)?HgJHFGE_5zuh~)r*tZR{ z3@2e~>h5)>n-^wT4w6=OZ;_x8%ENvaC@r!ybR-+tNT71XP6PyulwS zv}w=xD^61tT~ZSx$1Fi@sTAh6a3i|x4VXl66zZqO35Vpgb zsrYUMyt5)i6VQwXp!#8j;tohXK4lQmK;VnGCwF`c@Z_iT5}c`g0sq>Nl4s4rh*lR3 zGQ3Xj@`5lfz72kA=(Y*G1}yRl4gs>G*$E z*$$No8Wgl{1RoEQWzYXDG0>;;f{&k<_dndjLR|?#w8ZG#^*#Y1!89F^g}OR<+X1wm zIogEMx?~wY&6sb{Xr84f-dUSF(8yI_&}uZDrWT7&fSga3PWmj&8r@hg?6CU1OE;_d z>}{rVQdlXibepZy+=i@>tAAXMs&#hoIP{?-Vh;I76^XFgUt;8gZRx(^3GpZer3)dU z)%>^Wp%&B(U!eK`m2;pZym8W;L4W$$A*6+9F9S|88@{n!+VCEe;~}Zj48G65Yjwo0 z!w7IT@cD9CP@6#ri9A^HX2N%0hi=&a7nnehy~WG?LXza+s3!6Q0`oPEjT!m*53VX1 z<%;1qCeOYAQxD-%bwCcw95k#HeY_l7Bh z*PEK3PsI=fpW=feLF`RK%)E6Wh*lrJ{0U=Mg@)gnfEZl?jRj@t>FH+?y9&4Wgo{{O z7Z%-sCNQRuQ$oiqGCMCo>F2jgK#&)T7;gxhx6gZ^;ReN*P!1k#6q}4RJG&VU4^kAx zltIdIf|MjA+Wc|KsfXqm$C9>i{ZBeSoMdhT$2!lARh*}>vDc~}soWlDXT**UJoT{y zme*`0-anyoZB3G**KWeZX0JZN za@zj(BWS2wd+v<+yg*mRLQ_UJ9LWsHfq=O^U)L8`!q{k^@u14(!s zBa|1p31tg?iHx70_^ZI&IP3c+Cw89k!(!}{aI)9ePPltXC8^d}NBm$&^&^tLsp7EI z<=~UNKF}YvSMC9zykv{xq5xK|u)<#3Hi$?3I`uYA-bR@S@;-~9Y`zA-AfaOWY~2<| zSZ=dkFDo0`0K?4>M?C7Vdlo4IMZw{;nvXUX1ONM^2&^S!Xu6z~X_X)iAhkJL zb5~^Q-m_rzONDa93}Ib~>|g|Hj&6LdotT|$A*KC zkA&Z4*GP_IF<}t91Ih+ll3UFsD^De`-6^KYlIpR!SR!!3dqmlzQsDySLtYL~!I|9L z%VJ1KP>X2HReY-fjv|!HUBNWF_iCkk+sjohDSRq{kG2n=-;oO>#*$X5W(Gv@-JA-N zj@bTD(%Cioh1!rLL8d_@A%}c%pNEtjH}!>NX%+&tljO)`Q9W22IM&m?t6W1&2C9%u zx{38D<|8<%YL`dg>OrZUMt;lxS2TAMz)>IBC2neDamYWBgCItnJ-9U5)qYT$4FNPP zEQFezEHLm!j8L`hI6uEd1@Zq@_7Hm}+gL=ELaN6W$$><}xDFhid(qLm=-1a*n$`xP zTiCh}((!c#y1)8k@W}jUOOn@ zXq_-A+cTUPAMdfV?IRQ#4fIf_st~|;#rkW1{~t8}{K-7SqL5&PHu&1m&Isj@x&o=> zcim}mIEHax=-?pG6I1lXiSmm-;vB21ITcwX0f3J9c91dM8yIZ`23z%%GU9)qPo0F# z`)Lp#!vX9AVt+1qi(m9ONLxm?o#@l$*A<2Cl@%4caYN3B`S-jEM(Gb85hJ|Wh%LKy zO8_z|q86hwXRi0n0m!PSyMWL=2=bj9IN4{+8=eBOF-_&IM$H9ShmW#$QT6R>Z<|Bg zkuB4u27v*JnolIPvN(dhS#0&a{9+Tg5))#S%PQmR%~+KbZh8mwja%(Yi2u0w^Tb3c zmv5>I?KjBlVKC!g1O@~k^>ah(#@Dik?DxQ>mV+GQ40nSpeY0+OH`i3w>}K+I)!Xg# z7E#S-;)i(nDt*e)H-m$N3FTm;aSICzb8x(@3sRhx)e)7EVRO0A(ztbvp%mpC+mxow z*PQ%#?wwR;xm?Z_SPiqoIP))SS90bl5m|Crf->9 z*^SCjmg3kfafFyd9JjJ<#}_P2GP-y7J~Uw`0`EOXJ_G6;-tfnU2INHSA!Z_Miy;3e zWV9H_AAH61^IypOAq_^xoA#jFI8d99n`8p=YVWsP#D6NiH$*H2giW)6kT9r@9t~zW ztYcNX|G;=*|r1HBcHAMFRi}3v;<~SBgRWb zGA69xLGNUREJ?RmvIc)#d$LURk4J}LFkPug>$RWV^27MTm!~iVLKd5t#iMYVs6^-x zK4j+ni*H;UR*Qjlo4a(kDayAE<{t!gBFFe%dwg5wRj{J6avcTH%Q-BfbEx1uTf1Mu zfPt9UMjC8^KN0Ktcu`TM`6wc^zE!$o@h!)%6aVM52!}D+(~;LhN>$R)14L&uT?A%O zjVX4Ob))4A2Z4Fud#J={7~ed4Wn5fYdB@qgpuyrcBxWE>fLRzm^#Q?`r1yQ*fCryx z{B{B1<{A!oG&2v5kjXtjUF*{JA}Z$JN(?rF|uVOk?4>Lse}rZkac7f84)3s9U&uSv}5Z+~Yl7>k}Fj1i4tqk3)Rgp+WO5PA#ZFW<#VbX-`DZ+eHa^2pEWHvm7ua>7KS=Dq}&H8&!O>h2#3)E zai@t54wHwv|35WsV#>-^GI3iNzL<7S9y{f0@us4nC0oi6@K3cv57U~4&508`!mJBz zY3UlPK&n#)K{eyil3m#rmWIJE1I-A5LH}ohh zlL3b|ApQ+fSq?#!5^PeiwgQv~`}K7-gc8|r=Ux(Q@;E#q;_6+G5;lU!HoKdy$dC9H z^r9p;KqpNHI-oiQa)$LkLeZYQAbl3-UF(sWC=)Ad3)qrl7!SQ-)9}%u0TK ze(2x995in2e9K1^w=^SjI=v1+Cql~ZqnI`l_B!))KS^V%tFPa_d-rZ}@f)fAF>QpH zxLv}IAQ5m=0M~85*F5>4(dl{8yRVqKetejK&~n$|mNyJg<_GcsuiI3#qbo$@CW5;< zGY|SNePMzh`@=U#;fV+k)1*}ti;aQVKfelUMI($LMnX{LoVh4q;ed1@E^|*$ij*6K zZ?LpAH|OGV;)8?7{Dh*x*ZHXoO$HV+2YEw@7~BB-%s0To6RwBwDp)x2!jgI+v{>sq z_Xws$bq$SEIY*O>B}3w4ACSKQR)K>=;o2A*A6aDAt{(s+;3#mre3^djG9VU!QXBs} zFP{jdfC@TDq(wgh=KR;HsB&lbXzA$a*sZ;7YO+SH!W#ySOpWYCGT%`d?u6~8o1zKB zq&zmy5Tg=QUGlT$s7tP{g}@CHjV)F>zo+32gY!u{2V_G~-hr8yRtc60%8sA9Sa()H zg%VDZM;fL1P3Bv1bX*Xde(>T& z;%g+%xQ%&XRlE_Vj(iA7NpDY2*JF=BC}F3w&X3=48evy^5i&4Im$gRM{P`E>vM?UF zR1H9MC=FD(cwnPMwgMscx|C{4#TwP!eH2n;b)UvFZeq;Em&Z-KRgr>Z!>qsN55onP z#6G`HY@i|{RG21w8;Fg|<+(NL6Yc{vMhLP2p8yHo%EkySvjrsWU=OgF`b^~7>>za>2L0 z@2HhrcqV!H;VO^}K%`&~;H^Ww;uBAq4{~Gzzj-7lN$G-hYEYBnK>&I1eVvaKCvzLb ztd89~MR~vMh5?cPXl~Dw9w&Nw`UhS!=8`gwo|1p(STpC@R4+WlwQ+1PV0=^XabmgRQtgu!dg!KbQg{+)8G z1o(Fj@^c?(l%dj^NxaKavpwjEcJHHE%;HEhqqq4D?tXP?B#N!xGM({Dm3ZZ0AI(0c zo+u|HsS*z9WE0Zs06O~5^XVGHr*8eOmb6muFqviP^-Y;0g{83vpZ~bIk^Mzb_KTu} zFRt(VE7(7w3Wo+noc=P;I4xu!m?WOe6n-~8{~%7-`P}K?w(p_u?yW_qpUrQH9PV!G z*<)bwlsnYtquS$`>wiaF7XpU!=Mt8U*pHfuK<1|PM4swtriR|zV0>fi#+Kq%lgdV;EC|-3FV_V0A-ig9rBpMn*qS*T6QQAFyeCNr_xp zSpfocDz^tC{1cYiEj))Y<@EpvdSD?BJ>?20f|V7_@}RWLQ%zkyS0;uFf$T!W>kH*B zcJvE$DQ9JQwnSq@qRBj_tRh}T^!pA`(N}G4Q7_xjl;H7cCKQYi;I!BbDlmnmp`l?d zg-qP*(6>2eW@d{2ze;3F)+2nzI^Pclzsze)ktU`X$j}8kJC8t-UlFH>9t)q;#N917 zQ&2qY4t=7fIeK^~NxUqI{|V-bxj-&Cxdz#fT_Q$Ycy2!?g+yq>Wbj}4-NeEPw zy%!)K&`ET52D?{VD*|-lNzW9XcGcgJPvE5iyr`gX8ErN1S`6x_l%yn9FMu4VE?it( z?SzIMxu#}X_XW=cA+&NGSCp7x@v4!#!93Fbv?ZMr4O4oYnt5C8%eYa<`R!fd9ZxLH zVEYKkJH&qSi6Z|#SjHXUf{xs=fjQ?${V>H3vL0vUnwWJY1~*`@l3551!fO85c@mAP zF>!~C49muiMQe33!CfgkjHgE{j_xP~<_m~ij>muQt(0rX?EHMX&Q{U2aeOn}I{VLs zYk>w55OO@~cG|m;^2&DTNWne?fFS8La$V^I29$cZS->`E(G3vxh>QGD4fnClLX-fA z9R6!82B<}c&Upf#L}hpuKRq^f6;lz*IVp0iR@ojsGr;Sm1-Vln3jC2H$Lw7~mBV8~ z53K?hdY)KdZZ(#a<@~Rw%J+Kq=tfQ`ci^T-KD2`S z3T##6T8pEiq@*M)tjz=mGcunE5u4I+$cJevvVK zf^a&PNXFvfJfVnG6l~%udpcpD1QW@i)#y-+a$Mx)&yRr-z+!m)7GdE_{;y>fWMF&w z@A^J6DgF+!O9L=#0VdO8fAGrIHg<)?$Cm)ZmF3P+JMFFJp&R}(VPLb!IlAyvm(cTl z1kF4*H;2ioe#|x;Wj>GV*02_3AuKoAB{8ygH5rW&I61Mi?i9b#qX&baiSTgdu&O*=#*FWxQQ^T+8 z-rfYS*WlK85&qEAmscr%s}x6D*v2N>MLqH}L`lpj#;oNJt^}%O_=0bV%viUcHiXGp z!Vs|`;ipxbb)>n_O>6oTBqiUZ4g&ng-Rex<4*wV8lf!j3>2(ls6MK;KXIdq80mvVW zc6b@fjZIIT8bqE?pa?-3S~ub`#nuk6(<*g}E@^A;BmcjP$3l!O(%T3^(*Bd_ zx}&SXy70y6_Ir?j3f#jK47~?;=Vp}+P{RkF_$*pQf%>}!?jRz&g*}i>M&?8m+5B zLzxmhrjHz^12V3(EduV#I4^D(K~oLt0Nw~#9>#MM+($oIE6{TsNlz0QRRHQX-bDa6 z5l6(oil50TnD0roc`d06yzyXY6#S!tgEtlKmtfQN^mruPRdgbmgLJFQFB$vC*S zgG)q7*4crG?dfr|K7CKmvObsi<}nMNgQYhxU|Y$>qbwZl?Z?1~DW_&`ocx_Pas&-y z!;KsSl>V8K9e9CFxh4Qs`7;COG3KxLH&2{ikvS8OAWK41+2fR79H|J8(OIw)(qZdU z4T3e4(KH;ZDl63@fAE8cb*04=2tvTBeGwFB@6Ckj=BVZF+b3?LTG!NQ zE)y0{2s8uzAHF=SE3xH2ulYAl&ym>hsbi4vHZsq`6410_j#uQo)1F=BwTe~CQ?dy^ z0K4Z2dN2q2Ab6v24O;Ear*TlOYFTXKncAw?0che@y*n1L4I2pXUQGmUefuIZ@3-&Y zH@U2yQQs{s9pK-cXBE4+I0>=8AYLT}dL{2AAMhB=f$&f9SSA8GK=xKaK|#Ge$n!S* zv3E}|s0rWY0i(wT$R)OKrEBr9Tj+sB}Rk&~Rj zJzW5LUqE2Vg3ES)&LLu$wlbh}jJ>W9o-I&DiN#8>_FkkR>tW zEj~wx{NG^?IN!vQ-r2is(IYLt~B z5G}SpvZTXNPEKwBokqXnkv!%4FS*y33(1^V9O8^JpSxpZ0F|u2_rKE8ec!+BL zqCz$PZniQwZ35%9bQnZ|6`CSM;HHiEJy?>c%XR8RX5~?tje)L4Cs?@j3T(>POW9@~ zv4U}+BB!j(b8htQz}P&(7J&aV?RQqXUoNp1YD<=jQ8ltJP`}_JR4q-G3={#`o+T}_ zdv{d5wKmBC@j#2HPr zW2JVx6JnR3<5Df>ic1_L0c^clC4#lC5b7P|(0U_hiY#x1hf|$dIp{l9gueP^`r7D)89K&E=cQoQ#zLUx?cmWqYCBwLV+B)<4L; zpR4l@QD6a7sgIs{{eAeNzl}ndL>k9m-SqE{&uY{Xj!(Gh^rrg6K7Zuz zDp16K?QbdOc2$+wx7}#H)6*{cH;PFNqRfb?Al#_H$6U+4%kUp2Q_u_*^0Cb1JtZf$ck$uVMAK*{0*DYri?w&Kj#Q^B2&lHUObHf;* zP!^ws<&T~qsPmvpop#bNN;X5AY!Cw>edI&=Z{0bUphLQd*(Teq|DC1e*uE~XJ#Q_?z6`xBIyyQbfiJgb%8?`$IRZ2m ziG&N$b!E_@L!{E<#0>*O>N-n-(9XdW({a2RsTsE3-W5ifV$8!33bLKHA+v!?-&*M2 zI5wA@oLpNAb4W7g15isyECiF6BucjpyU)MoF@!!K(_3zXoQTM}C*#mGPE1VTH!5EG zjOK=>sX6r%Xj;>6NgYnmkceFdtS2Lt?c3yvO8aEXIKhPg0*2$?m&G)%eb-2@I`-iSM#@IpZozSU6Ssl{c4uL9v^ zWMnM9d0K`()pQAO4ZMDVB6xw{LsomopycW|OKkJVXA9M}<3kJ%#d8b`@Lj{j17;CXQ(S@qIy1GyTUx4!v`ryL- z{?vTS2PXRvg6|QfX-0?{Tt6-ihwr8uAa3&-^b{FhHPeJo_JYt60!edsPkDza;FhuN zXy7B4{udGI8$Lcv>PrQ(fS3L3Gn}08U}5QizP__!5+0MHmx06cf3W@V7S@1yOKuqX zLV^D4VQeh@cm{ae;rxt%1&+)k<j9Pz)CddsRok!VqKV^ zW@HM0GLprn(+jEGZL>^dJdyf)u*1@Q zC|T5Cm}AglgS)!iP9CSjVBy3>=9<=vNf$c??JLD=PJ);I8VZ{09mqrL(}GJ_<;2C) zjTHil2o7Q40FuhBt+%lr->PTUAFdB26P_N-34v6=sy|%xS&?Li26cKTUBfJz#Ere{ z6}C*bVkkVGZ+T0%_D#(VR>h=azfeelBqJIG9A4`ob;c$Nq;tW^jXb~`FsIRcArp|Y z8GiFBE%ZXX1nf~4gm)@&eFH=&6Kybx9&sYtW4*OUo7(yhc z0h=JUn{H*=b_DtI!pP01tS9kKwOmca+}ttFRb0& z<)*N{trs|uyx!=Gr+2UiY57H;S(-qdL>1qwR zHeZdbNM9KKJdWD{OdN9hjrxEe0PJwnwE|^& zziP+5UJahwBypxlO%cG+^rX?+3!=P(GAX$6?d4dzZTG2R{R^Fte{Q`gO=PHXSy)CC z!mAJ8I0Ps|(Nr$Ik>O$HK!ZbvKH+dGl2F|2E|5S77Jken`nlq2E*y5_Y|C)U*zMy- zB2wfCv=_t$#wfe=7)NHrjgN$VfwbxJRZb!RbJAI5g?qr!^3qr0II{S9`!CuVO75eL zK+bh;Yz(AeIQiZ8c4QWVGsoskBB4!66oG8G-9tIFIPp6MIQ7gGGdmJ1eu1n+W0QLk zi8=8Y-+k67%xf5%U7OpU_MItoA8SHEghnu`{?gjg2nRI`b&7Z=1}nV)@Jt5OG#Mv7 zp^Z>3K$@49jmoa7;n1;z0K2qE9;3 zD*XaiByDXe<88fTDJLiAN=-#3ii89Vh71ZI*|Y?0 zw|e>D8ls?pKoFuLC@QEZp%lnL|NEQ?1tqjN+6IN{KcAt1BuGcu|950QJg{|SiWdS4 z9QA%MbB&?}yv|3M~|BA)G3eqcV+e@{#TmJk(b z606jxKoj(RN_JT9lwGJbASWiypvBP#dTTZc^ua=dLBT!u-sXf_;EDRzB#=Vum>pHC7+Ec|*h35UfXQzRW<8ZY!D^Zy(X8z4TB;2Dmg9$6jJ zI(~C7Nh=UdgC4%j-4f-aGXh~WIhTjZi zs-B)a4HhvkP_K5Z(TXzR>8I7IGVoXo`Ai1wQ^0A8c!Zkm7UJB`S7rCcQi>815*9t@ zUp5}0Mh?uYi+z246^oJ|D1X?@Qax=3BP$dod)B&NtSi30zH<3{{e2?Ibjn@SIUfB+ z?Yc9JNT=UgPn(R^Q(b4+RoPA4Xz@$RQbkTc`5DQm!4zt(-7PwaO3`*@A*Ab+K$XpE zf`B|@C>om{I6D67!uRQ~s^w(v8JNpe+-$yD2Ts$ueQQKD27!=oPPN!o+zMFI_N^h+ zm~^LHBEhf43_6X2swhW66I%n}nA#DpebM9@G-|~WkWhB)vg1}V!2A{0bw5Sb8ucg% zdf#biWtA!A6h1%RRAnX#p5SqPma$)ME~rp1n^|ly`_yEx+ppJ&&1|?>D(kkM=@Hvm z$gbzL;#Ht(VUe*oXk2PK?6rOIZD|({3?`|yr0W#lUdt66Va~)vTX9t{Rj8oL9E?P- zx)XQTHKrV*)M+&5u$`lk%i(KjU+eaN>ptp!o)%PCWR-FX~TYvDzku zh}T^7D=mJe%_XZsDs~tH?Rs^`3H1XALXeaMPFA^zw37!hP`xZJz}(1q%o#D?lJJ=eL|02mmM3eS$bjeuF&7QcY9s^!!(D+&2TAETN4B`W+oWLvb@v zBZCDSC=r>|r{R&V+l(V?zx+F2KF!>N0{Q0(>cicNgGPtDY$mH&kjNexupDh*Yuw=% zaJ~3IHt&h`a`Rp?tC{6JI|acZR$!?j{$A4}(XE7lg$VuR6-D8SFftccn0=d3^MeA{nknpeo zt_W9&M8gySL)=Hw!HxPZ{`v_lcD_1jzD6(prs$K3LR*E)!R}ny?!7Tvo9hM|C-EY zuk-Z!uaAhlqXP;vboU%L;R*_f>RRC6Is8OvrFsEnoaGCLCv{faR`s=>9Q5pk3BUQ8 z$kKozFNXGJ^Y#=FOaI;6u6eDZi85#|{$>xXXCoqW+4t9%`=f{-nsH%BXM1x_F%mv6 zPbqX%8$LHspC~mZd%JtWn?@D2C9N{&m5n%<*%=!zKGbwj5N|vEf71lck$B9olSN&o!B9R29lpbpoIJsSGZgdH(iL zw3#F^9^=NpyR#xlC^!<{OyXE#;q_0AxVG8rUEylsNRc$^RKk@iCRAc6j^?&Clrdj# zO>YL==G3CwvCO6u8BWg&{LB)ds>4D@=!?FtCQ1d&@0u_Q#Xi zEaWQHO5jT5)8ag(In9O<{_%2oDSx@}beH$s{Rxr&h^DsHAKbtrBZ5r4J|LD)e^?@I zACfZ+K3?~YSo^#7hgB)N6^Xdr?}bjqk |6?(2(hcJAWnD<)m>0t*BjdFSGC!8od zdA<%QE03tCD66UKzdL)Mqo&SR@7L$YBOnc_E>p|ozjO}%Mz@IloWuV^s>!9?PW67q zD(}xQtZ}nZf95ogh9E+mdXUb=PT*TPgSW3 zZG_H-p_t6bI*Tgl~3i)Td#ZGZu`89C3TxDD|RB<1J!_#c2pFU8^8fBHs2%uypIo1IzbO}9-d&fN zov&(>H|wk<)2b7>IHi35Y!h@-gn#*&VVgzuWUe9-b4p#e^-`-+wMe)|?}vcYGVr7o z!=iT=b$S2b|Lh};4ve2WcU08%;WD0O!`_+ZaVT2LRGKe-rfd=d$}j%)UO4)sT|6As zlu5#PB<}VSg-EJQ>1C&1HI^5HU+HB5&a#DqrNV};#cn~<@pT|D7h@4ArNL~3?!8id z`$CJ)?V!PC`MbPWF-tQLE0&Gux9eE3>^F`p(wv;|-d}0TtRro#Wdu{vCtpi^|2<73 zwL0zKJquzUgXcKLjQkm0h|KuP2+*$84hZ19=SePqF&AAi97wz<`IaIRty(E^$J?V5 z5L9C@Xsh&bl4P-7ZWC$RF;jD%u61w&H6;AKBOi|qh_nhbu+s~k1oGNAVJktZ#d@ZA zEWIvQ^=J1#2luCcU))hP7mc=lHfO{l5j-rISEV2yF@sytAcW9=r?XPe>U8-qxLPZ9)$(_93VNH%s#~GtUc1PNv!!lA6 zm-Q+J_wm`>caR5vQzh}wIcrJL4(ejF4W(l4v*22*)1RwBMVZ7&&}^xNbb&AGjQ!Bw zEdf=s1~J(vk}UC!F%A*iJw93 z_+;~?yQ9>ziR2w9N^*v^4dajNT^;2L8SzsL__g%{W$4s#+fza|pVtKLE*izJ7b`rv zHXDW({Silk2*m;Y`fT2KV48VDPuq6P=&iQ?7_`y&@3Nhu5@Itpy{woEYg z_4#c4IxB$ihNIx|Zo;(Dg{(DKfzQj?UD*E<67f26Q#diD zl73jFR$Uac$6r-J0o=ND^Nk;0_+Rc;XsXBJt%IlyI|q|B4Cag?@|& zmo=ABw{TMUOAY!&a|0@~p;)JW zWe|8lvb?52Hx=V7QMJqMgwJ9;6TU5XtXHvn_k=~N^+&j6>R{B0BT<^cin+vJrPP?< zvu--?`|A>8d8xTt*R6iDZOM@N>#guxlTrgP8v$1Z_TD6M5Kq+GaC1zg-|Mra^loL?(L~1jd;W z3;%ej^B!GW7NtS#M&%!Ba2+RtKu)Q6_%Ug&8GlO8=r^nFjeFJTO@dN*g`BwfNUaJq zX+A3#JZU-H5Eg0NNNiMH`Rm<^gpvmu-5iZsbeZJ zd9(ZDb!6Q#R|x+ZR&F#FBaG6pgd=-+qm}YrLQDf6YP@dqVFdQELUv^50a_}6T(^^) znvn~q_I~+L|3p)&BFy2vp)=BceGWA?^^Rmwu}f@!A)RSmOd~(B7x?b5mRPD_6NbSS zdi0B#ndEei>x@iNMJA2u<3t&idOVT%&joIG$}~DmBjio_KV{@J&q`46*BNOT%A&v> zrDWM|)N@g8Gm9|9_nqPA^%6s&cFb+VnrrTMxC_@ zKK$K~XRs(-xbv0Eei_S(u}KnFQiN{or_*F=B2DDzgiI4SuOnA7xvg7KYc!3K7k8R7t= z$4TUWs$vF;N-vGJ#y!(g@(OrHfNpWbf-FqP4y~TT1n>y3T@MQ&0br z0a#>3O7B`EX}nO&2#(gkrs9-u#e&Ns4L49N0*4Bud*KGb&LNO5- zgrNbubdi(rQ*=!xj6P(sSx^anbxoeg6OJTlq*S2FiFilMy6KL ztFthqsOaSCWQe1&E^xDFxZ6syi1s>x2?W1JIPZfC$d4Zn=NX~Y^h`hc z#r@s&7DRwTUBCn;%o%EoEsVt2806#oO|9!|D5)D$Ezx%V)MYo=g^eQa9<@>`BXytq|e>+$5i<-lJc@jEROq&}H za;@E0UcHn-`hhyGnN$+RQ2-b?L41Wgjl)tS)}tRE04BcoR=N>ls8grK&7fL2K;B#1upi?Uq+bz_p_V~RVHcbjO zg$mo?e}we?hg6iwS(os`gNWuIQM2{5z_>P}(JQs}UdisL%WMd2=SLR2_D=_NGN+Sgz2TvQ}O008&eWW#jEJXUWOJf~rWb#nbh#i=#o%{C{sRK6`3{TNCAW)y$6 zmtp{mh#1pwF_scN#YJojwq>5HE@xI9p;!5=8GAGuD<)?6q3Seo|7+v*I5(jHE?VMR zAb68TnpUG6rF)=&OZ`u?`1SR5|4H;0$2YysuC9#-{~r!(*1u92wTFyKfR!_o)Z~vs z+ZXzL2T)mww#(kn4-G4rEoH0;LcU0HS*epBMha^sqawUC3d9i8dJ{No;9^_^O{@PU zvlL(}Bwm>jcppr(JggXN5YMs95v#VnU@w+<4}fl`D1GqVP>Ay$=!PYb$s}5pWWP~y zSMvPb9%9$+`U;W5>oQl*hLdxZkbu&^AF_xyeDQR7>1g_5Y&Tt;fJUtp6TX!axeE}J z#}@!a=NlXjfC7~YwUW8XjBCbIK| ze3g1*eaLbIyb@-LWiLKdHheo=Z4>@2HfYItke_877E3|uZQq|j~CwU35 zu$5?IS=@DM0Ohp@PzBR;0BT_{LNkN7fOCi^4I!8M8!GUT3}Z%oXkh`_IW$9zs|YS|Em5uBShNEp2^;?G+}9L% z?EZAf5%Re@a?`gH2>U*rx*klJUuq(@u#IOe#AI^*TeavE@_+qcSznC9T>EP{IglW_ zw=B<3!et);dYX-H3y2w0UEb8ABh-OrFwU$kR+G6&J^rlbBudeV@w2<58(VO4TG%e6 z1Cmk!li!|SUXBi@i+JGuX3I#piaByfhotmhYY77nCW%FHuL1?>$(eEve4Y z{|vyOx%H;(XkXsYOE_)wP^m@kw4%v4h-M;(B_l}X~o4dO}m=mh6 zcb}2nSJTQJP5k2%4M3boKrTXH(RyaXW!Z!|)#TcAP{B-d(r>a%G8Rgp5cnbuq)q|SFOnoC z{StHE?Ujq`9c(rvxv@9(@eer*5Xjw^=Z$n%~Upu39=amgr*5(Hf5>BH$%7R~+CCCr(~_k{02 zuybW5OTQ`gU}61@Or_lwad0mi96$7Hu4+Ml&{5AqP1b-um4;$r`>b&qpnIa4l8ckc zhQeC~(xZrk=F&R8{|cC5QSofuVLxy-4`eliHu?0zsDGGey{XjR4j)LZd`_~f3yomJ z6cp<2db()n0X&5O8<}Ct_l8;O`>(XMts8~-IoK5ey+S^1pDJaJ*VOmzMWyxIft1+j zw(E22&J12M!;!M!*AUZBgvia&fnGnP&z+G197>;dM?dnkJ_FE-XI(6TL5FYb8#Mu1 zV@2FI?biVLk=r$f2Hr62Q1dvi7_y=9EUl|0Y1F(05voOa;51jJHTsV*Y^2O(D?gZL z3&e}cKkLB9Ee``c=9k^mtB;8nDllZPx~R)t-bGx6GyHInWy*OX)!i{aX#}trAR1>j z{J*&H$b}&9Wrq+k6qRGh?T zSgq9n`DloqM(0Sa9H(y-KlcXEmVsf2CvgQBzI6bZ@O&n4rM-a(lvEK*(=7z1RSu z9m*UcThJ~`>DH>5?EAO27@dbvG zfxuG<<9H+X{!j&_!!pzC-bN)@(s-eO`XF_x{gB zd?dligwhgWC@8^#BuWD_wSP}1cL~X6!j0_Hh)>5=L`a`$YU|%E|r#hW}LQJVt?95 ziG5aT-beee&L)%1b7idQo3=e9zssg+I4-R+=*b+Iao6UEd&7q-gEBiR0mgrl&X@g` zih<@#=)YNY`3IlF&)H+(8h1Usw94SzL~VdUfQ~rg>wlP zrvnKhLd036l1nqS`#d^_9BNf4%6qqD~g-dP{zFH(O1*E4r>O z@w)62;P`2Q;N!1u__pb;CGBa00Mz6K4T2CSVR1VKrN4bu@A1)$h-Z@YTUv%O&rCj&_wzTLHY<jNBCz;ldr|_v}>p%PRo_3c>@BMn?^P{%ZX~qEb&{G`Rb#lmIqM z-QrYqE;xuuf)+vchRV2er~H)k2iBX%OB+mh7c4@jvU2WcRpuosrPB-#9etiFw&b{X z;9N9%X|DPFd`;%QDv@4(t=1RR;*{PqJml!9-tJ1vT1we|i}xe*7hnrGF@3b)5$%0> zpO-fRFbpaf5YQ1U+oOL@eNCd%sy%krFICQqn&L;)aM7vNpS3wufqmJ&mQ8>V!~=3V zk>CrNk2xp`i!so83Hk79XMB*zKRZP6-DlE@1nf{dpeq#x5eSg*7`U zGo$*jfC~j580#+dl*Lc~cb*tAh4zNkd4!?_4pVnFi4q<}1aKOcDj*K};*T&$DwDZ_ zHfOrO)*{Pg);qn>w;o_-^F@OX0qvd$&P}5sF?4RH0f#SHaCxHWPNFN$C2KuwfP&=` zij3YhUsb#R;k~ z98DAz0N(-|VZl)@PpxCK-idpcz#^IVJHtf`X~J{Ng3)3u`H!P>3X=g= z!vWhYv42d7^Ij&9IlU37gIO5Bhma7AkBjnYOf}W|k91!IFk+K9?dEBA<&tR%PFLG* zY6kuu|B=t&ut~zwMEK)zdn}Au=v)qWEIF4Pw!M+~Tgb=LNH&`XlZB?()0gB>}523HZq zY1Hil5Y0aAPCMf$H!uM2I({z%fM1AW^f+l1rD-}-Vq#(g8fz^Urz^4xv?@u2p+4%_ zso63Q1J>sAVVOiSO!J=jIS~Hl7FNI^Xn2{FQZW;{V-=?NkF);~cku*M5r1|l=>Z9g z^X`wz3inr^$rk5l>?L4clhV@C3hZoe zQW0BoY~bRL#hDJUowK&q)(LSsK|Ne-K%78g?$h`O&uPa6A!pNmo1-|-#=?S9dMs|c z)?|h3G_UQQoJ18mn3<4*mR4WS(gJvtIOk-?GdcfoS^P>Z0Wc{}rSVdPM*0;nqG`a` zG>4a?#yfAo~?R!sLpOOcyr$363~@Gt~e^VbVt$S4$u6kGpt^B63vX^#4Ev zhlGbuResW&G9$7zxjon(n*E$_TwD5l?*DLE@btQ$_vSmIM_)q~s?q7`wA}rYqBRYu zNg8{}Olw&lJTqm=%0M3Yv{KKDd^bds`K^U^B2(ia9Iz%;?F%=h0XCfJ1kxUo3_axQ zt4KR{Cqc1p9;X=L_^mCIpFS)2^so0eJ5te2kSpfRRe%=j`>+*Wk20qcZ*{6Fmx(b0Ff0myx{r8!I2~;Jr&7{vB<>UIJ|VLAf>VaD zzfg<)N@~0NStVW8bdj_)KXP?4z=Yh9)tik#7o!qG%wEWP%gP9os1!-Lovs>fd-LNl zr`1`XXqr)hU5Cs8^An_I-VxwdK(bwb#zn-r45knJ^Mdot^g|lA!8j8GkbgsMR=(S7 z6w9CjdL81QDlKKG6JSimfWBGPqgAO9>3+5>*_pM?<&#SR&}9^m!fyg_X|_H2N;zV* zU)JURP6~$;%C4JPLqQ7_AmZ1GtQ)DksQOFD-RTgK_tXS6icf6#Uj*(QP>kpN3$Ix# z@#T2u$rPBq?@s61-RwTm>c|1YH7(#Is(Z_$R?eldH|GLDNR#B#kaB#+DU0O^=lkHs zdk(sF0@=hDZJlHx-1qi>LNH3XYBhVP9oaIj>J<`#{d`7tW+npo+|bN{&`!3whnWU& zaFAFG9fgFBgjQN#Y41qtop#{#_i!MYRfN;Gy+}tZ=1RbjweL)!onxwMc0_M7EdlAS z;GEb`6OPF4mpf_dI7^2auj}1#m_yQb-+ver-b=33z$6d?ct=0HC1#fbG?rEuBR*&4 z$Zb%!ub#wEwH%1Wp6reg8gvE(_(nqZqXKo1TCp*={(UkaQ%?QPFe|{g=m-zgmJ;;5 z{F2FK&yhqNH4x53Et9SWu%pE{Fk8d1O&Ayys(rTtih2I)kRB{yvsQU)Auh_Q?fyna zCi$AR`U;ft^L;r#J^)5EHh}>KleoVzm6!XGz$s^rs>4G=)rO)<97Q%j4Kvr9-CD>w z&c{oif%Pe^Q1|#4B3NFu3Aw7cMtfB8_hN(a^m4);d@PyW;5LQs-5*VI`JQI@d~pEV zM>q}6DdR9zYF0$zaO6pl+gQy(<52C4glKuH!8ED_{b)74$V}-TPar>;=d#k#EIxfe zO>HlRW%fHoTh?zUd{w5rq+kWhGY0W_r_U9e)3N4-ytVeA#m-V`>`-!N;W@m&E|s7u?ENHdbiE`R>&4L3cZYXW@z$zVHyW2NIxF@&Q(vxCDSSN*r-x;NEYMEQi#-U=Kw{+?sqk$b$RWr0A%bkMm={@$nI{n zirwECFwdbRmwR|15kDoHnM2{mr$1F`6=ox0b3eX#e|>%^FeI|s8W?@g&uqPSNy-Vm z*!Q9nWUFeve4dVw6SVs)@?-Wd?e6wr4ITxrv0u}>ANp5Kijt^&+3V|90^&_JlWEzH zVoR(ay&f)!Fd4LisFWQPi=;3m>i5LLHuciyfI^1i42LfkvaVV9>CkKtg2VD5btInG z$zUDqb8QTB_0vWhcIy!J-ES0&?;l}B!>*VoG>yVyztIhSWQ`ZRuw1@&IFa-F-2ozg z$)sQ_4&1|B&!5i;0pr)ckklEAJHtWLmcama`6w!)U#Y4jE5inWyrt`Ruomr&!k0YI zobN8ZlDg%)eI)xfXTVf-TL-9~S=YEZ7)^k=YBZF6wQN#r_e2k8>)3_7Eo~3}Knz-6 zAi3v_-*s=yM)!L)f3EA*Zh0de^NuLLOEK^5-T!D}j>dWK=a@cs*eb&f*sx;iIo)emlOu%3~5xVfNa zrfx2u8|Y<;X`+P07CO6Pby`B54<53C)baTN6~LyP?q$3Qu66O1F^hJL_R9##tUDve z@Ok9ChYdKgxP(0Qm;lv`&C5JvO2MU|K&^E8hrGIdUlt=Ny0pywY@cP}rCO+3!br73 zEi!>!Zo^QADM%{Ji40I!{*gHk%5^d@&r;_#Yf9q@buzDY+;jTvU_R>{ueSA|^Ijs% z(5e-O{#DSnnV*Cx1!FxvJ2 z=O!xR0d|avdwz_b{guYpw?fQeBIaN0e^JUW6#G-f(eXo}$sj3l-k_BE2C2X=+$3>h z46>jAE0P33ZzRu+oWfA}XfnbfT~#Th>^sk^q_53vGjJv8r_E&L!LMmdIc0d9QRdx{_ObZ)BEiC;p3rq)9`-`L1(O;AL=A368;|Qwu%?~XG-KpraJ{}S?7w&?YG`0AH8C<_9t>6 zmFt~2Y}8-~P$nfcjuVH``vVBREHy^GKFKq=;5i;Eyc^|$l`HayVRBa^zk)bFxba~! z>JBHUDQ5J;MWon#nlp&#x{D^=V?mp3vC{$o|68Q(AKkbPIjESy_Dfr_cSX!tZ8A?- z{7+>|L8C?Vbhj@*>U}i&KisaKvHYGNydvbfz_cqjWP80l z+2o_Yrq9Fsqv9gbnic=Sybofg!1xq4HHVNJpDj2N-TNub}hn>y7kADxR{rCW+ zM=6&)h(tFtqzeM*{#b%f#D!kabNX}jCTesB?TsyKg!8o?7f6>Vw?*e`D||7fC~Q2 z2tpE-#@&CLg=QdXgT(Ivbu2Ph8o;G^9skZUJXe{Ow}yroiOY$Lo?><^;HQU7S}z*? z=I@c9&Gv=_Ia6W|SWGb-aDQWh6|%Sz)k?-lEK;I}8FNb4SuyOt=8@GsW+p zeYf~xHrt+-Tm~C=q5VN$Te_JwUkEEH8a?2gn;DM9uK*}r4v+Jdo`-kGAW-(t0OoZw z3pu>^a?A1M3CeciT<`0{B>`LEc_EeJvbs!!fJ@B5Ksfhl=pU9BCIddjvlZm9vVBRw z(T2Z$>kGY3*%GEorSeS-aIB&F-Yr?p)fovt9hK#k`@f~5`}#iarL|hIQ8OZrw+FrX!3OcnZf&0}Lao7ctBqkw}^r;%dVU?u{U+?e`YA`+ci(cfL1@SiBnd1gX!@0cAy8A0$J1soUCXRq?rq*_9uim=EpUKs@Y5`05``=r zpVu{s+aXK%*Sl5uGUZ>-_%@B4w8c|XD(N3Ojeec%?MX5No(l2TR7O3ruQwF8?3z+n z0KUP>1Hq!`MYzZm85NZT zIQ}A4Eo!#w}*WR)J0;+KpH zcrbzd9_LfNfgg{uzI68m)i49?8*@x0yP*crYh29$E+K#YB<{)CqK>HucuXV4Jr$BX z3=>Tt*by+hld` z#C6NW+(foZ^l`{F{_8!_xhl`0`XY8Qrn`)K znbOdmKhB?buQYPy^hHlk``mCGe22%yu!*xDpgeNgFV+WYR&NGNMHR*{D0KTg&fxi# z#FA`cfp}0V(iY5O#$i$07|H~^c(5@|4v53}If@$iJ}Vzg;Ep68V380C05Xv{3p4KK zPnAqO&dMX>W3dS$kmtw#c^2;|s!+g`Nc9Q+8tDLNM)*0RuDQ1b%oB?ZMs)uqFz>~2&NwDRB?i5JqE3sN+=wrBfZehNrQYc z4QkCctzl=YfxDz$XwqF!06R*#dKqbGP6D5W^$?s;-fMEHf~+{usu8{uxjQp+ zq@gpEO2y1yB~@RDnkRY%wSq**r#-Ek=}87j$k!aQ(@PqW3u7~(+;6jwbKX3(fQi9u z1TE6n`4Z~+?+#I)j5)#%ZK&^`QnnXE=%W(SjvHrUe+(69v7&EdBQK6f@K~3bUr}G) zO0-uF6cm&zf@?2r#}&;uB-e^IVPlxHbyzW28k4^8_lSKO=z?&f+=fX%&y8Z6nFYZy z1FiuDcZWPk6A~`a_OL~h=I%2W1Q?+CeiA)irv!8~cDs8vj0cxJ`@X$0y6*nm$R(?- z-Ngtkt~x@oEEFeEA}}I&T^Pt~%ug-x*bodJI7y$h0ZEGX8iFc6(TiVn-Uz2>f8A!N zkLofn&dUKZTO&FT<63{eVW{4WdYi`u6BDmTIR6bY(7z<;4;V|;R}?Zi(5y&myM8|J z#P1yS1$VW2@+G>T3xHNQ%F+_Zz&uESfC@VGbbIRBpE&I?J5zFhbo@!yV2dZjtc>El z^?_iXjDjM9PRlR8(Q-U~(4=#FLE7v2fs~02y9DB+v&u*uHp9HwfC~P>%=7kZ_G&fHYg0~1(zUBD!X1~8u_!Jh|{5$i?mrXDwC+dKO^m|D# zXr821^4$4@!5Ia>p$7a+1&jGe@8eM_zkb<8d=>xxx$Wv=K7tGR@`f#G!(v{jRRsm) z8q(Wnk%cnki)7I8J$rv#uaW*v22?S0MfLZqYdWQ#*C$~E07`)&qsmne#5iS)=G&m z4Y%W5sTM&`y}sxj4Itx_idb=D$G_ff1qChU@WJAF4K1?pJkT-~U-#8jTexz1^~aFp zqk{lroDx+(9@o>~M`9+o^Qjy*)AgN^1DcPl^yCic%tZ7Is#5d6TbkdUJ}W2*+|l-) zvUULd8UYb_bMPjLff!eAa^7e!%~bSF=>=bW025GGwI&M zT8vyWPE-}*X^G{8^Tj%%=XOib&VVSNizUAAUsdweQEy8nlQohQ8Vc?JhY3J zDY_wDx7Cq}O#5F2ww1@(vMs}YsNOi()9b3uY!^xvM}YPA))vrJ7#j61rWouZ(Jn;4 zbLKapr;zhi7h*9k&ph6o^3qGvLq2`AZI&Z{F^KCunjI;Z{Wbw$N((;s&!j2LZ{6?Y z{93d~5Qw(i?WEC|OxPCc>)Q)|gz0*3mx3f-m)(MUtF+cf8~HFYB?D48SlI7bKSE%? zZO6?x2^=tr1lKw0mkN4I^=XW1kri7@0VdL%R` za8V`|D>VCCy3IMo`euK0k(RTJNN~bLv`Lv=ku*Rr0WPW|Facx#JQre;P*|&00FW>p zKJ-?tq|B9$V*rFy=mDgu+Gw2tzx4gZhE#6Ry#if<-(Zpq;!_6v)}S%Kvfh0C%^zg< zMJ?GeSeE2eH&`T!5OmlNrltzu{*?EHx9-!sop4DRmPG?tiExwBEvwOd)Io?;lB|pb!0m>*-%ok1 zo63$%2Fp8T0r2TKrkyLMfss`9yE)dmR-YGJ&`)Z-QI%$5lt;?FN27U^-DUv2w5@beX3 z2^1T|$6Iek)|@cZfFO`X@~x+xHJYe;#CX%Rb!=C-sL4_>1}~g_d)JQu69BWb`(2i> zKeQUTa`8}f*>nJeN5!WqOLsMAhaYi--ejakO2v*7N*CS#;O$p`Zm9umZ;*&NNV8@+ z?pJToR>dR!-nbr^znGxB&1Nt{6f&6*9I^6)FyH0~uwYT@wk9A?z*m?9Ef#?)6#ud9 zU^Ob8;A3Ga09|s~Qbu5S$R{b1m^k8%cnx&PE5g{NV(+(CM7eKa%uq(+7V3sntr^6H z8Wn8EdIHpr)D9a;AO7f2yu5gMzxje5X_LvNb=T;~E!C0}n{q(~RURNFqwa_`B{YdV z91!yFVlY7`qf{!hK2K;5X7mHU5jjne{C3)3ro!_F^lv%QWyOFw+1T!`6an`cCEp3U zM)x)?nm1wV-93p1db+g>xBOh`hY4S2x1hx8PH)f-w>LKAkqBLM7yfuoR=9Dqtscer zo?)RFVOnzd+^~MTmf4KN_0USfMf&fNXotp-Y^nf?j8PELf#7yha!~Nbd9~x^ChF7G zhzcJ2Fib%R5`QwbGHIzSDHJp`+38nU&s6h~Ef1+x%{aWhHpi|0-GhU&`Rl!318zJ& z)$4t$vVnPnf&l{a6f5M0zcAwJ6;m0nEi_H0(fXMEOz~*B)uhSaKU(4_)|`=Xl(Wos z0H+Svz7p<>=?uDkGW;I4VE)rlBg1DghM|wuQ9S=UgMIeLhF_HtM+p0Sz{d)N$DRPl zvVjD|dA~2eeE+#mg}`o1uXin@@3om`0&Smw=BN)an3+HyGj_QgCejSuP{uolPOn)l z5@Pke2@PQF;{hX?^km-6_@2Lf2HRU$5wqTRhP6(w*^Fzjy=dClSaV zAFJ(J^*otJ^hr}wd42BMT1t|Mw>Q>eaQ zitg0?k#H_UU#SdvoWYPlV?zm#)2TY+-EEZd%E}5pGS$D;H$81yNv(Ik8Q@k6I*^^F z@xQyhEde0e3_yT6;Wl9lOltCia7ecWK;P~4PP zH$N>TlZv;i8gH5-e8;_obUVR%=^y7@?c|l)|DEkPZEDFM*1-uIkZ5m}pTYJR2<=_v zq*G_unrcb}0_=t2fJ|EarTa1L>F!KkIZvoXnbt{&!do8b>Q31iiLY9_*yyR-jG@t0 zi^bzqd#3?*SX9K|Fp|1Hdn)t*O?mVX5FTqx7n^-A2~7nbj#Y*hxNDsEf6M;3*#$!b zHm466GeypE^=ai5$sN2! zY|O$MfGdt(S%gPKEWYS53b^8}y!udmTsF1X@X>7d;PFPuDZH)`zH`KXuOYX#wiE*&VnIY&l&w@erZu#=jF?f6 z@h8Ee5?c;T?9x!mB&^b%4-9<#)09iV3#a!-=(|*v7IS#g7Pci*v3-QRdf6)^b9K@~ z59JLVsf~^((5*A8r6Fn`ZwGPYBvhXF$zOp@=<8pU+R*_gscms1pJ`-%wG);&ME1uH zl1*B$3y)4Gb zMNl#-d9LRz9_Y=yFN`tuM%ldp>?x>I=8Y~a%%zmrSXfaV2$)-VMwx6DW05%nHd*FC zBQAtOo>c;5AupjIA7<(FM$IX4oZe3M>>=ARlfIIJ_|CB8S_#l-hrZuJ;K^8YnXJ>~S2PfwRk)`)N6*7gP;eK8%}hFDThF%5&=jpA9E^5m0VQek-dT|w^MBeq zuc)TFwp}ZNASwb`O06zRQpP(XU`0s_*c2asZrD!upeN)?eB z2p~QD6TkNxtmgGt}9bvQqv+2qXf)D>61V`Fu1+&%t6wQ?JFovFyw#r!(`W3Pv4k(&F>(SRW+y-N)j+M zWXNvi*AtgV!-xI#F ztuM7^e-g>aT)ZNZ@@u5&vH97NVxfl<2}Xn3<+0_Pl-UNI@7lq)XdI%PNvexSrmJX+ zybjAt*B?dVQb~@9!AY`uYqYMg^D3j({JLr*wM;eXND9NN$Zd zxZ%nrS!X!r7t$49z&l>Jdt;y?Ct6C(xI;fpsFe6fPxbV0u;4{riJOb+hbO5@py;JV zW1lw_+zh?@`TJ(~S7pW&oohBM4@wEH#xpV5&azquG)$3`3P^JrIL3f{( zj`|mb$?t(zBg=Wm&>`Zxj;Pk+!1&F&u5@TzowC34E>l3jV`R1KV`u|y6iaT2_V8WU zyj9)&gIg0uiOW919`1DiM)AsF+ z7dSa4Cja{`F;Byqh6*-jDH4=#d&4@2~qg`F>C53^{4@a6LcV zGV^-b4|vIzt=rc)R;b!&zBiHsW+@8rr^qg{oS4n)W8L(#)fVqJ@C|mYGqU3!&5o{S zIcY;7RQLdclIf27@!d~^PXfAuabU#wTvecmdn0~LE^J2xAIi!d4zZeeXvxlKb!j3k zG*MLT=zu|Scm^^87#?=LFV=REZ{Kj8*Vde#AFcqi1GE19Z%>Uo>`|f+Pq~H7>0HCe zs%FPErGn>93C@sMvN~Y2r_vcM(yte*D?Wt2=t3(Evap%~UzG`}AotCnw89PYcQAOp z;R~8)%E_qCw1e-RorG?&+|Aa9WG~=!GVlSiYHA3X1pNAV${vG`Wb3Tn$w|I!HKF1g z47*r{T!D|*w@m|L7cNVO^`Y#hs$f3OA12)|hA>U#1qNTa+tdp(jaKYgmjO2r`X#(0 zP129qaf7bqN0J}u3e2+Xfz?zYqLF4~^bvg2nXG@${};VggQca@;kK#Dd0v$jsrmdj zXn)4+)x-S!(X(TNLKmxE+it<-UiM$I;6910QZhJ(3MvaZvvxIYTI{pq!BjYUCI zu})~&)#PLz)}xX=byo~gPRu{yO=WP*xYZSI8}3}W@Uv_>RTdX{s)F6wdj8H9Shy$A zoFa>Ta5ZggfAPq{h87*4nY}#|C5Of>XDADS7qfdXr3;Y(^O84yYg?OrnGShs*nDgyByku@fuOkZm1vIJWqz*A!^2rm02EYGo;W;n0 zBlHxW4BOwKZnvcRdkY^#GRc7aB z4K{`g%B;5fB!<+4f@FlKm%+DbkFJ-tg}5#5k;j^ws53}@v#BXr?!5T)P<6B5;q%Ow zsS-Y@@pI1G045Hvgyoi9DW^%~e-kzPjKwje;5p{)s{jRTpIY#(wVcp=7EwkCw4P42 z`R72VFT!1bxhG|n4zyM}*{@H`Q@*foS9>*lw35sipniR9fvCii-D(nNp^3Ek>G^+< zcXy&DQ%=L#AD*r*0DR^sjw*Ejh1DmfThZtE#=mLq-_J?lOue>TE$j2#Vyfcv%*dqh zn@^R(kgb{VP=`&e9y6%r`_Il%^su9btsFV0U7Z;s-RPZBy?WpCR@!PIJ4Dpr2K@dY zvfgneArck*(I+L+hk})9fxDxh+F~;+F_z_2UYXG0N@;$UsEhp22Wn~I0#KP9VfV4g zesj?LSb$EbMAE%G>?9~o)aCxlL@BLh@A;|QLX|a{cV_3HyKV@o!Owo@ZUptQ$m!Em zVY|Tb^So>;vZ0)qc9YedjKZqm?v%&EKKo=|W14a{iWs!htpG(@QSZ7$s2}|IUDep; z8$Z7+qu;ijiREriACGJy$(B{xviS=vqrd4l`GVHc~3EawkO$z za#Yp|KuCqf7Xf%GW7=b{kegpPUSv6rUw4!~D6iu)H20@dLNQK&my1vGgFJE zLp)wC+I(OT2@?}8E|k8u(O)wW#j@q>?{DSuW3|}6e`1(8XEG%IL^TX|2dP54oQ8Mwl zWNCJ2reH=q+ls+X|NF;Hn(`6uH8;^N8)Kxt!U&pdjoFuKfM5_CfxcpiYo~Ya) z_FPkIc-mJUB8%Pr* zFz$eJ<^SwSl##}3nKS?i0tLYJqDCW7Sd87b@UMAjW9xQ-6HGr)*S82W}8)PoGb7^T}I%k$^l66H8oKG+{}DV zeZ$Ejqz`JPtyv0)S=#?Kl}Cj8gI@m8uJOC)5Pzak#$}w8{FL8wS?Q(LpqoTI@fm3X z7J)E=>6m+4*$%_x1ON4+2ZPPVK&;lgrHeq|wR*0c#3}7vbY1;ksZJH!wtAnm`?=5i zc)=_6$e*?Wqh>EbG4RruWq4^briZyMU=aqIXCTb(Gh032Fws{D)7PKOvLu6?s3DMU zbNg0u#lMk)3!Gq+hZ4frg9A;EfM(a_Sm_zI> z_86WY}FHZU-z?7Qp1Yp-a9RuoHHf4cDa^P^R;+CacBR!`^$ z5ldr(0BAn(tfiKInO{UbHN1Irbhj_X}}z*)oLfOP11`Hl31R!E_M>m+%5%OoS`l@gYEh zmT|EDuU&H)%Y{?rJ&+44l(<&%QNJukx7Jn#K&~u*z7!jL6^f)4R+WpcwOd?VT*m;X zY7SUZ$!qEPOo&|8#~`}za$#D09U4T*|DG!q{*ieHgK7IYD`T7({NE2rWTddnBDa*7 zAA&IjK3m|%2X@Q)a0?o(%j5x2Q1u2wmv8<$xs2j?$9{Syt4oWNmBg25HMu-`im6)vv1~0*M679-Tc4fjF%BSGkwAZ4`Tmb z$l%NOPv!!Zl*=T!|9{Z`7nc|CNWhU@S&~`6#h}8PM6lj{-IxFc>WPaf;!*fKyL- zY;4{VNRS}z<$7{{IBx;q9(DI3y;|-_2JtlP>frNvzYB2yo4p5nCusA z-4E2#OF`Q&26FY^9{w{^WA%wWjM=>=Ns)g5poTo5iE*htV2N1+9&|Hu;Db)1AWvTg zfoR!$F(OW8VW4meLM=Jvg9kFjivzPjaX@&+UJ{8!ss#QnbppH)yT!;SPh(|%Nq{&_ zJd?4**Jhh*OfLfAnMbGX^H6?D{vHZpTYZpT%ErUPlU~qs6WVq!IT%cEz;(^mn9T_! z$vE|v|J7?61V9SOC}oO?5D*ZkXFmG94@B~}^0@)n@XsFDAS z1~eZlb%+{W7H)`mA60yI*BRzBGh+uZQZ6tRtA7upR54y!st0~dMt~M-oRg;-6_|i@ zO}E?W&C^&lVAOu8&3NST)%U#Nq|dZFE~vG&_0~n2A!|Ek2V`E2fK)9Ih-V_yz5go6 z`50@uR22gdCp23RkCxEa8q0B|bC2C6Ii+N7dXOTO4l?t=l41~+bQRa?2VgVMlt9WI z!)Db}Ny>fucAgqu_3`!|Bkbb@ajKC>S2LKMQC|BmP5%Bwn$^%maxMPURvn1LbO8!lTBlKW+WNU=sFK5Cv_v+q_S*sN424V5ZV^laRbP#dZMr68IMzZ+*jl+;4LMbwURi z{mx=N54Tl72jh`M^YY--tu7ebe`nG_e}c<)LukcuuuTw9S`USMv?aVw_91yniApZV zX7KA@peVkj9^yXhkS?sk->Hm=$XrG$KAHYb|Ce>^X;kAY`J_Q@8iz$0pI1_7r=B#y z;+ct@h+qBb1q?BJ2Km29%XqyJy;)q zNuRTim`W2cA7XMTn_7hA0|lo0cec-meVNK@z0v! zAR~a`tHP$~YE)o2ig=q10$?QoJQKuI21SB=1P-vmYv7Jo8T0JszY!;VfN4iM^nn!N zs*oJKX!4b0q_28~a>NJ~k95s#^c~7)nJ==R5*T!o((~I(sIO0!m(z(lD+DZu1@fHH z_J8!JU}pi2*K=ZSxnk^MpW38j%bNVkT_iC5pGoHC~r7gK&<5#>3pbOeBTm`Tpp};6i0#v(LCD=ZqN1v(ojq#*j0)z`Y*5HN!`F zhP76E+GU&m^jRYy9Ai=@AGkbPRN0Adp2aQm(7Y-l?k!s#`6zvjAbD7m%3+TGXq%>v@@ybe$B+t6bc=9~AUp7!c%0MlU~wRT<414Dt=Fri zoVwNQx(Gm=Wn0fwE1uIzrq?Z75T&}y1Y}s3pj*lF4dJH~u4yW+m6(C{9l< z@A3#txDGL_^n;`yu9SNiP1&g|UH+iRdv`prD@x#qTrL%AdkBWiXQQ$y8_Ib)gWh&C zC^|)xfh*r>Gih!iwQa=v{9sbReaCQVHNMSft=|6K4WTVP;?8c9H$`1}z~^M?e2<;` zG6my=9w@l%%vp0G_Z8`MevVh{92mNBK&|+E?`|r!@0SOIloz=9`c~B>(5K6GUH(ep z>4skH{Y?db@GI3XgyuLBvY-Y-dhZf&>DQ?pyxW>A&q5R^tc06}MZo+DaJ1(^vpcN` zZHbrUYsE5?#2~My5#i%rZ>(F)hTrOFA)h1qC4vH_>aORI_Vl&k@itiqRhYichSMT- zhTPT4J-p|^rk(r9V4Y%Ykgi1g*vsx1=C_{AQ!m_u+Mu;vL3SK_%=LLYSF21ZKRwdn zMDN^W!Y2E>IiYpKOFnVh9Lsgu7%5NWvfJ&nC)av}*g(V{pq0; zK}0lSk_R8>D~I@Jr-s;-Axz(0LL`uw+IQHAf>Io5{p-yZz$eS)3`0`EJdC@Bgf3Q? zdXr_j?L5I~V5ILi+T+uOV^=Vg7XuS4xztB^qyueZg%p*P1(1hADwRe4l%aV`AF(wd zFM+RYoL4}9>SjOr>%jhp=m)hF-v-;6-QC^tYJo9}{$x2?5>XW-V=qWaNZmW}q5md2 zaJdz=8lQWQW-sgdCq)GOgXJpmRNB+$4`04H)yBenQ8BMel$S--|(~<-d z-x0rMJ0!8E>EfDT^>Y&Z8L!DY5zqcea6txU?nl5waIizdCgJihe3I&7Ak*rMVfC$) zMBUFemp^ACFI&P@yOVjc)~`Ns-PBGL#6>g?aS~M4CIE%N?oN7paiu*|$GuAf1D7~k z4J1^`xXHaq)7`Wzm%WfzxMO5(J;(2v!e3XwF~wEr6yyxyP}Vkr+8ac-G7Lhz?APiS z@=6y7$_DQGh2Ty$coGd81)Fmj*5N2Q8)EYC}9Uk%S?x_2mcLqWbUL;E=EgP$M z!W4ZNEAEo@{uzK_*wvan$(f0H&gNbhaN_2gl|Jpj0&cq26Av7W%~!T*(u*FYnj9be zDLXA!vsc$THcd4QP_wU*vuVAkbr;8CD(*vwB0hfGAe&w--TQR>xt@~P}9{b0@F1F>Eo4j0;O_v!rA{zyX9!z&<9k^J{&@}PLXo+VrckGk(ok&4)b3VdE z;b`&W+)&ntiBl2s!JAo=sG@Kccm1h|KCJe)M`DtkZa;;hnnzLv?)eGn>0k?XVg7EV zu`1f$Y#rE($!FWozHR39!(EDOPTf2?^@n5FLfvo zN0-tn=d!ILM)j|VYq$)7{r~TO2!Oj|GF;DiLzjCN{Hwtg NMLAX3vZscD{{un^Hgf;~ diff --git a/doc/v1_api_tutorials/imagenet_model/resnet_block.jpg b/doc/v1_api_tutorials/imagenet_model/resnet_block.jpg deleted file mode 100644 index e16bd3c624030c4c09b358a015b491141b42d8f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22422 zcmce-2S8KH);7FD=q>aPp@UTENDGR9h)7YofHdi#6ahh^h;&3iK|nz1RisM|9RZOJ z0@8wwrU?p02+6b^LY!4Ct zNBbBLfb{n^V4pt=co(oQ;OK7PU+5F?`YC~RPe}e~4as@}{X+&0!0!R#HlS+a?ic76 z;O=*oUqR+LpnA%{i1es;u>6!oeu@lJyoqzkKrQL|EviRbxKj~CG_93^j*f$gnX#V1 zIbCqF06=@@oQJO;2?GH5_y+o$>7U}avbN!;cn!{x3LppM0Km~X;HuFDy^BYE8tCfq z2Y`)!_W$R{-ptQ=0F%;ZO!)c#tp3*oM(3;kf#3p|fXx+LTmzgzISQ1$g9EP~)nh=J z#mVbPCOMiTi$6F(Q06$2-F}m$ezy5d);^M5e0*KNHb-N=>f+;aB=>@H?DZg5P==lZ z<>>1kuEC%@17V34*AQ@=0HEV{HPqk3%{`D`-1!*4+=&ya{AXN)yE!`KKd2FbhNho;36x_olsVmmX`sS|IhM&{PNFM|7YRo+5XnztLe{Y24Q>kkGg;S z_8)bA`Jhhif@^dCA9YTd0MK+7066CVQ78Na0ANu7&^Y=}^I<#E7x%!xt7@{c!NI{Y z9@{^@=)NB8@4?f6f-UU3Za4&*;tRA)bLzaW49fUAzquKdz} zRpS5nhyOIJe;S9Bnd=o-e^+1dRu-U^dHA}4kK5P9BhbUom*2zpzk0&|;m`hQ3`gQ` zdJPK9g#>`bK?YzLWdWeuuK-eJ8UT7J7yJhD=W{zpWdR($d6w{xzv(?FgWv!B{QrHB zB!mAX3Gi^^KdRO-yU6bxEunfB`H3C%^{?0pfr(AP=Yj>VP(&3!DXv zfs24OU=KJ0?tl;A4+I0@z-=HJNC55w4}mNo4=4o6fM-B0&;+yrT|hrD42%KOzyh!W zyazS`46p|rfSWS~gbu)n${;n6W=I!g5Hb##gRDZhdvS_k&vSP9ZvOcnDviD?rwP z$g9aa$;Zjp$ag6yC^#u3DKsgJDI6&RDWWMdDJm&Y6cZHhDR7iDlyFLUNR18!iRBBYlR4!DZRLN9@R4r7aRPU({s2Qons5PlCQhQS0rhY{I zjJl8dE%jF#8X7^G6EvnY9yGUT9??9fc}cTM^MjU&R)SWC_7ZIn?LFEO+D_Uz+OKqU zbfR?HbT)K>boc1W=z8du=y3GR^fL4Y^e*&B`fU0}`U(2aFe;b`>?F({b_13UtAmZg zwiu`wL>Y7#92p`QvKg8gW*K%FnHl96jTwCy6B)}H2N^#xQ80-z=`y)6MKKjH^)Rh5 zLz#t`b(o!*?=TlK_c6a`A!iX|F<|jzNnojBdCh`hWnxuiwO|cl&1P+9U15W=iLx26 zd9x+6)w0d7?X&Z;pJaDsk7IwvKEeK-gPTK}!<8eRqncxyW1o|sQbCHXb zOM=UkE0`;ntB-4on}u7Q+nGC_yOw*NhlEFh$BZY8r-0`b&n_h^mOYNQTIO$ahgOQA^Pn(I(MPVjN-y zVmHJp#NLY2i6g|XiRX(?9V0)cdd%xs&ap8G5(y;Wl0ao9LaGhGN}_% zzEb&8v(mKEC#A1TS4gkQu*sa0iIi!Q`7A3eYa^Q?J0MFqu5{e{c;WE{IVQQYa*=W^ zay#-8^3L+v@>2?Q3VI4ig%=7~MM*_h#azWVO3X?|N-;{^N{7lS%KpmFl-E^+RW7SM zR+&|0Q8iIbRDF4Z?8K=P$P;ZReyAy{1*+AmeNmTI_fjuc|D++N;i^%nv8E}g>8SZs zb480^>$27ptrdg-!V!^=Kx+$WyJ#0{e>{2Yr02=Xlb=r=KjnX_;na5>HJzI}ou{Fv z^-ss2eyz)-Yp$E6yQC+m=cZSohtXHk57X~FLw3gSOv;&A16~7XgE9lmS(URl&-NP9 z8eTBWGDM$~IOl(^#fZe{tWlcL+w&slea|-;1I7l%X~v5tVkXy2T20AKO-!>*KbXmx zg_{jrV7}mRq5Q($#ZwoPFD{sin+KV9Tfi(XS(I6Pw>)i`YPn)1Yjx9V*qY1Q!}^5{ zg^h(xvCXcnuI)qH_m`9|MPHh+6SE7k8?@)L_qK0$fH^oi)LkaKYop?9Mwj5N$StUFvF{C4>2 z&678uMnEE*BD#@6$SCCdTl%+3Z&Tm)x;+wkJTf)%+a1e0Em6Etx1!eW>ff!1rjPcI zo{dqD$%`e8^@x2Prx2GJM~HWcA4)i$ke+}~bWR*fl1qA&M7($9-s@!LM(fsrR@`CFH9||oB2Z~gS%8I#) z6N(Q@d`nhJ%}RUARLaWAdCHS3NGgIV)+;Ynj#Ztmdhty9+0$ya>V)UO^PuM&HJ58< zY7J|<>s0HW)r-_WZeVVRZ-g|4He#CGn^s;}zZh>m+uYrv-qO%2(^}Fd(3aiK+Me7& z-4WG6MBPB`c3$n=>T>H^>vrf~=&|aV>NV|s-DlYMvR|*iXW--j>ZRt(wn4SQ<{{Og zreWpb##c(O8b*{x8eS{EZX8t^eKB@otYutdyki0}(KUH`vVY28YIyqm^!SYV%$wOu zvny|0-h7(#p2N;xpFdc*^_J{y{363*`V!Al{_?Tq>J{ac_SMs?uh19J3v146o9_bN z;onDnp!<;i5&p64ll-T)b-nem4cm?Pn^!jvwxYKgw;z8N|6Kn?`^zhg73MwGA4}Xx z{L1;Y4Jr9Gd$gYWS;PF(2^)gJ@hA` z*%5$&+yuV|NEykP1msSVGn+b62wr26zn$`wQs`7o7pvJgMp(hgKaz@?jh%y&OGH#m z{FsEIlCp~G2{oP5x_bI&49;G-Xl`K%x;M@)u2<+XM74UJ7Nnp?Vidi(kZUJedTOioSD%)XhMUq!FId;j6% zr}Yi&&ew0dd*5+C_K)TTBDLS9_2TXP*RGcc|k~mk0#DYN+uvj&UDh0 z!toljp!{u0mQyKDYr3d}6wENJPX6Q6Y{H7GBG{v;{hZmqHnGV6Xl8#->>u-50CWJ7 zp9M-n0wse&p=9J_U?HajJrQzpN@~iVh5GkG`?Js=71*DJ2zCMi`yeGHr2zkDprN8+ z__qsj5%egzi1Pp~6olH0P(}a&9OCoi#ex3^7nC^mH}`65T7xC-`ay7KdoGQeiM?pX z!lDBknUA^dC?)w0pMSwmQW6rqI>fW0OhrlpD>H%Uvi#E(VU9O#>%i3q~Rb=G`Z>KjMCoJIzWEC1QkFhQjjRw%7lCSq#k*3FN);l>!} zwUTbxJILCS&f(;Eg3NQw%W1qoepQ9TE1|FRbXsqQSaOSkpGD-?`)VnMJK}P%(B53$ z!yEz3+kN&sKJ|0H)w_)?_dSEviprOJ$TAyT8J;wzih8^?cy)_1*32Ecbwl!Ad3#Q}YCfL#R*@+8b8E>m z71&{)P->Aq{!kG^=4KX7?qQ zNxTxewPD7&CUw~?re{BD-sIlCaJovG$(gK0bMwxW#7BChaxayB5CLqRJ?;lKc~1q@ ztr-Nj?K**H+PmT9HeModA;7yuv!S;!G!;Vxif+~qaAse<#u4!Tht)eEqC--$)RjL{ zp7N8fwE1%)kj#zT|B(06yc`hEj|Yh3zr3K(kFfdLy^rT{h90HXZ;Z-~z0k{~hP*j7 zac<3lSgl9Hw0fa}vFa%e=TZep^gfsma=|gpT7pDioC=#qAP=!50vS^g6U#g2>yr<7 z@Ki*g%p${{2x#*`hyYC*TK143kCO`A5&#L>Y4k-rlV8nfCtD1X8>b5s2c}!CY@TkO;ZctD*&; zaGvmK(6l5l(zH(UIL4+dHgu)StH0Fld^-S8AIj(7(t_h?hyWFPr)<;`H_mnyb*n?_ zSpM8XPf&m_fu(_cHl;UZGz1G{(3%r-U7EXcLZfE$5|5I?C=yzt)AIW$8}(<70j$g*P(yAFBsZy^v7Y$2n)M4&cf z>nkhqIIzevrlq1QH&EJ|Xp{5WX_Xh^iF*55@6@}y)ngJE*!%#& z$ZbvRFfLbs2uudazS-7x;W?gYj9TU8ni&)Ky;6;u(fQn*=s~`gD44i?=$NU}wR1Z8dmnAsCYjhpltL`i$9@Lc)5k>H7H; zEYh)9HMAY85lq|KzoftlO|>w03Y7|d{qbKJ>ops(t=lUjTt;v`KNei%K_`KC%6U3) zl={*oTjdn50Mg$R7_(%)SjV8fmk+UM88Zt8gMLRL1e(v^&!^9}WA<2X9l2jfep4BxaKkNl+jH-Cz0~@bv|M zv&>&SJiHve=#m~7lC`%;UF*LkA;{lde!CN(g*fnp4D1PG^b<8S(S88fi3d7|yE9iJ zcVlO>C+p^7;f?TMOMlLyRPxjb!$)JU`~CQh-f_a@9*V$9kv)odH6ri?l}H3ItypkB zkQOHb(H^)UB5)E3zWS@d=xTzE2}OG!R}a&vW&sxQ(Myo{)Hdkem9SBq`7B#-)AyFV zJ-4_vhms!Ur@@+svvb}Wx~20A`XVEf!-^f3J8UTMYH43wsxErFf!d*Np4eWC7vKJ}rR;YQR;QqT*ay)wcmhM_ zw?PSvqx0E>c=#OzCTR-!qp;ww75}%J4IAU6M!T2U?AGl89BNg)V)6OgB=Vi5CF}T1 zk$?ba!R{D$N5PjgYa)p{C#WTNWA*}YuECgq6~9OV*M_D4vL}~l4wL<|GlOI1Tx$}pZy73 z#5^j=e#p*q*cwZpqS11*Ej_&s>9on77Y^aC&>d6rxjEu!p`{?3>94HcjLA7ZjuoZw^ zCKTBe9qHTFBoPqa1UJH((E;TCL(hl5nO6vL@(^DQVBz>2m7Y=1l)4gu5;(mjYW7x0 zGU0so>POHX%Y*hfIYu11{0US}$(ID#kP5U)OyD;2Ih^H_fy&J#fvJM$)}LpbOFW*3 zni)yIUybtHnRa?p&S^OGsWm#lzS*7~JK0Y-h7rxq_q0Jq`D!??dJPZMVR_r{wl!rX zb>wLwzPJ#?kYA=?_CI*Cns+ zpCbbE$9oi|bbi#@)?abG7nCyR?9FzyB>5aJ4x2yOGs7n{PJnytgiH=-an#l~tBGWt zeFZm78>(kMm(rqF86b4e&e0;xcZ>?x+GQb+Q|N|sS*|8`FJAe+nin^FezHw=)kx=j zTjRa27d!5`D{*!VUainf&lWPEF%ll+0#aYTc)94eQW8T@ju=AFhx=eEj4%(DG_V%Y zGiry$_4nU$gd4c3_t3p>ZoYn5iAKC2YZ;PH#KzpkgG(XUZ3{ZUW(_;?D4m4_jog#bonm}nHRDeFCz77 z0Aj9ZAT~yg-^O{*YoZ1g4biABV@OC^k3Gu^452J^jiXvIZ1MQUJ}R4KMsEJil9aLL zH{|-scX8AVt178_TCIENBk+Sz*_dd_3-rSQI@34%WiU{S$P7|S2;l`Mbp zlJ;uY_b0GN5gddu6q}aFCLafuEv-1j7fFqKb}?6)OH*eUtCXH_m9whBdcvAbOy5|> z@;390{xz?comJOSTu65#!%jG38x7`T*S@z})$4{TkLy|Qd;1b?eD0M<_5j?R0RdNE zeh@hU5xnj=M2>Urgj1o%3EX%AocikW-TB-E?42$x!3mM6hQ09*(eq1mYM*X)eA~Pt zM>f(vov&efDy>z%MgRS+M-KY9hZxztV$9|zvrkKc;MNfPZn{I@jaL)9jsfvr$$ZFs)3;8a`lL+jczseGg#XE&kAWrT?U+W8 z5`i}#Ly17?vvv@(*h4@gaaOi7?||nEVwZKcV)@h00jXHo*?miRkHO#9E$0wnOAbsD zX?g3TW&MybB@^m=)xu0qoByi=oG>y!+kYKfGT=jlj%fBlS2KhlZE6fvx1$`SB!O9n;U$zCN6~P^cbPU0>1J2FHBtx2MG}_dD=fH{5;KaiZfnYJ4Tt!`WoU zMziYPa1%pKtRjZ_QSXby=>DxI06Kz|Fnt})u5lqhguBO{vpJ!c@no=Uf{%4+0Hgf}fs(OP05S;MB|ZWs^LWc{`%)wcIG zwoFRPrA%>>Q-&Jj@{&8m-1KoDrex0-mx0L}m`C){z!?|3xV|T(J7RfdTyvxwsNUAk z%-Z7VDAGqs<()n|873mw7jm4_McK@)*IpTk5A@jzLzfWwk)QM{esLY-Si zUE-I;)}$97ZfQ26Ld(MsqJ$=9_~wGdhBqwt98*a*u%R;$!~%V^+wNnoqmyc$-_j|)}xS} zUnMKT^}30WQ)|97r=?#uUI$~ihKznjkiZ46QeoZ^xN*(FS_Mk4t2C!nZfI^FKDG$H zcVEX(F}S9-yS6_`iu8m;Z?z(@JYRxF#Uocuya?<#{b%JNi8wtX(3>&87;PGOC^0`_ z;QZoNv{UNx*sRFW5_^o zJxCp~uJG}I9#Jta=Ex(ugZ#FJR{Oq>fzcxeC&P6>3#HJ7+P{Y71znQ=(56j+Htms? zL`b|W6n_(kT(fgHJL4B<6|R7;(z5=R#BMgzmR-@hNbN54P;ntFBl(q~M|@rBg_`Jq zzMBYL00e%DO(5_i4FQ24r7;Nno+aZv{6OGG2?9Tc+#}#;_d7>0u?zw~MiBT3Xn??P zs0FcF)ByrNArSbT3IT!N!=NwUOek(4dkn*m0Ezw)Ac;BxBod!MK$3n0NVJoW00|Wc zNX%zJJ^`+%33jUVioT|qJznhk~tF6cs*i-8fvRH-vLc5lQ< z@6#t$Z?j{%`)9Z%rD^)v_Vzy2^sz*cV9yE)(n^#kfw?psz*RDy*@D2AL<0oAQDPwQwbcJ*=Iw-d00aAF`f$u&Xa0TvXqWu!^aKB9=9|l( zp1^>^%+#+0)}X%k+jl=_9+sf}*f$t@N^OiCgZ6BR0Hddh#e{)RV2TlYCT*EuWo%W$ zZ9#A>KS^WoK3?!2N_mNSd!iwr78y;>f(}c>3}VL%!HL*9hlLTcORAAiNaDr+yyy}A zC|EcYD|t5I6fSACN}6dHq<58kQ+ z$sWJ1Je;w|GT!O;m2mko=SGNK1~FRhBnsCD#|$pljvg|i$dRj>B}i=i@9U6;a?RTp zil7C7YuI@ru*QPy5sMf`eAB~&PdjQahY0jb{#*x84bC5?BR?e_fgY-D(7GgAh{YjX zjpuH_=r`)RFkxNegS?enl^yWOJCc-xq>SVz$vd(t;jGA)HI>NOoB=|DC-6gPMSlx* zptuQ#68Jplo4Oq+mzM2u!3Mc#^i2xmRO_PD*P@}q*KgG+$ca2kH8`QPvHaJOFCK!U zdbdpAM82uX2SEvo`PN0qk1%XP7rd^idvso?i;v#;HUU0cpUfB%V3VV0C)BNGH2wZV zkyntDfNLm!@-SoPcd~>L9;<`i#xC2m<%rI7d(z|qo0u*-rpYTPc(ly7W({7)!wT1m zAr+DOnF>ay*VGSo5^whukIYp&rQWwb22{aDeM16YV3x(?6jcl!og0s4&P`P{%- zOm4O!+n&vxm(hEn;SE+*Rc&o=&)^oCPsNBq$J-p)leE~wt#LKjeK;)1R_{)inxe%S zKc8)~_e&Yq%8WI*-#IzWr}YQ7kf}c}ysxd8eAiwa8{E5`$R~zn%i8D5z_NEO@lAMt zK9@2mbwSGJetWXh^OSSsA9&I)(nqYTsscL*@}2{FkbUD~dq~mAD|vRJ%}%KAiC0$g zo%+q2l2$3EMdpne!}7<{g*^nmt|g3Up9_2OdU-1dZ5-Xffk}?6S4YsSXTcL9M5<#y z3pbAkhVjaCKhjVt(`-pdapR%3QT@#5^n)8W$e(IEqD3Du!Y4SXgoy5dV=a4pt_*$c zY_bU0P@5@OlU&r-2~~V)rE{&wW4%nBJ>C3N&0Cw^`~9zqY$4~r-_c?X_rw{Ogfn9k zBPd>Ac|S%bCvFA#i1zPw$vT77<|er6%x^F0`tX~Q_&KN^X9^s84#0zPaJ0;AEXl_{ zwk)pm$*a6hKJ;H->uUQxiROGX5bZPA7}TYDVw5LfX?7SSAiYDvO@wx{QNDTt_?l<P={w5 z&K?=3xe|iGOf?a3&bvnhb|t34clW0oMx6eYz3xPK_<_EnKIkigzT8xW;ZI-j*Zr4T zyn2+|gP=BeV2$^lB+&Le(as;}{%W`6h2F4ANU*)l+Ga9e@00N(Vdzq9=Ow-OFMvb( ze3TYm{lFCOge$AU!Movf;hF^=YNK(AJ`o=+)LOx1EA-^!n|eCwdr`Xe^eiK%^)w5| zT$50SNoCcy;H_P(3P{f~7$WN0{#8w%2V34cR zAsME+Ch^^@(dyxYe2MbUJMg&W25Zn|3s`gOmiFphV#40VV(6=?Y&`8@FH#LndH7Fi zHoQ_WP#BQlbauHWz*OekK%Jruub#3Tjez2cDmNoEaY-wN3ATZC7?FE%(uL*lkw+VB zej1UNYB(&k+1+`9MuOk~fqx^9O^XW0jrMKwqbq)jPU0Ij&E>w@&GD`|&c!uRUBQPG zn=ZC=OVb~;3ZVI^M82DekVn4BM8gkNjDLFC2UZ|*uEe_>`5?WBec#;Q{rNxRRFb_~ z3b>|$d9v?4M1U^*MC+vDwqbuRf7;p9FXlDbrje8a6H2@WnLaD6wO6itiD;ke1bSB< zTOjd5c?~LuEC|p@aF-H+uLgf`3V*HF{~xw=pg?w=371@3MDCemMO!fom!|49y}G5c z2Gx9~(yOj{_*Q0IJ3C(cIq7oBdkKF9j%t7KO$pd5CIa~}E2DU!AUMH8BE4V*HScUW z`MEyl!ay!nOI>^1N-n1cXKAegOT78`Z7f{i!F~NAARK%nNDDF5$$`sr(X;(|3`!&V z=GlG0Mlv~MyHk_;ZpGHgb+!YvW;IrKqlYD0?h&HgaL`77Cl<;524RXA{R6&>HD`=0u zT=}|fctnL!e%uDBXY9mUSwEoY3zP)4$_kc30qa#`v@< z2f_e?mk4Z_ms_I7jsngU#F>9>ZyEWru({WSbH&In_c+kwJzOiao|bwpabjr|Y~~Bo zizoGFw`FB6jD#|O@0PUe20~p1P3GWUcs^Xo>X#%GU7&1y1WRLk{e*@N^N0`MQ;NCI zr#*!9o7aW`j^g+H_lI<#k>b#aigzQyE3$oIutfXgxGqdJTC@1JN|JQof_jy@={S9n z7uWiDn}CLMnyF1%m&4O2I!;)IXK1TKyl&X=gx;Ghn{WGRx?JtMlnSQ;DiPu*l`(HE}G#y z_K*l9Yys)|X}e^7=Ge8?n+w(r@3u92!ka|{sh^aS(e@sO>H^^fX!zXsfdMpoEZzb0 zbpRXU+ta^)C9ilhebVF9-kMstzYg!n-GO;0+t93I)De5f#u&Ig`X%2)(BMyi2ulZ( z(FL-D$a+KrYL#VTTXDJJhrRy1)-Xr%tDUC7y6Xx}Va~-yH++7E!oaXsLoyil>Nf_t zIuI)ruWv4I{*5>MQ-S`-TA;WU4sL0w2^cZLfcog6ZqUZ}{U1Ghw@_&(X9uTtHmJV3z{1rau~3TH|;qr z-P5%IFxT#MRWpH#U$KlEHo)D*s`n8@u;i_)yy_-(m>+dGBM>sqEPXLe)D_u%r*(QH zK(vqMec^pQh&N!%51j$spm7vahYW5zf<2FJ!QJM|?bR2_1GTSe2mRc_4f~wQvx3VW z*lnqt-Lf`Y_I&(ujSp5Ht}vlRg?Ga#6wUQi&g0U3Je6h%0`~QhqrGPKw&#T>uTUwz z8xFp@{$5{xfJL38PvCO-j0W$jUsMOljD`{#NwC-LN(@&k=;S8^9rWX^EnlN(Cl-fJg?Ftb)+OpYn1^#8mMU&v(3@3SW;H^OGJ3T1 z*Mx?xZdUjKxVz43Wc{->)H5yNHAC4{a<4A6r923x?9KlHOSn>>eG$6_r)@Juf1$$p zp~I`LJ+n5%wDz&j%Gnutr)Q{e1&z)LSPAEH+*iqF*?2}%N{YUvhMd2HX2ext)cZQ9 za9+=hWRg8ryiiPDS>4C)S=MKqPhC4-XBPkT3d)t~6-iR#(2#jJZsg`5X7>(`qURj_ z)_QvQE*PVgbjU_AYKR?B>^NK!=Q98vhfRt`Lf5J79ch|Vq zU5=&dxjCi{^BLO!`!is;@LvGdU|8!{a^zS1jgFweYh%9m5rHIobzI)cnO*K=>ZK5` z9yOoFr=dj->-XQS#j`NZpaBT>;6wQ41yl@M>iKPu5~!&l>bs20?#CJn#zv zp^sNeKEmp{X`op*SzpFS{EpSV;2>5{5CgILQxK~k!Z9-l5UaBpgKn4(h}HLxusU{t zKrQKWph}nq`GI3@De_lQY|IbssP%!r?ik`)1@6Rb5rI^ixo2;2vb2&KO+;t`<)l5Ka} ztXCa)+$}neVJzxg(R6IzktsJz-s%T?sibdn{|GqsDyd=m;q&tx458#L)c-Zw`5Q~~ z-|;U0%)uPVpy!>20zGd!5MO1kemPV*`J3x4h6Fut@MWt}=vm(PjRZY!Ef{Vg9|tm< zhz26ib5m&bOB{1|uq7WoR&2fjUfUOXjWvJS!@GO_*jHHA1m&!}Q2En<`bNqXkW)(n z!zdIn+g}Jw188pUYA#$Dn(g-Loo9n3{rqhcyPLXg@gCbOQPybdYl6Mbw9edTDU^1D z`()?7=;N~2s_F4YG2qj?t6{uqs6MQz_65v^BJ~Q#Dpm6Z+s8|Ow_@!p-_r!4C$_+041ibU=f)BZ5tjx@~v{v_b`Gry13er?v{;-lIB7hiT z_k^b2Fs8x7$28Q@NCsU0=g%K4r`jwF*SYixWrhq$YdmQDOIMYB7iH%QzXrFG|G3JY z(+MZX3mg_jDB|w)I?#vf+zdd&m~C($KFmwI>1k1doPHJGvT0&lFq>M1t!yT5WwcVt z$8_B+`H5ras6QcGlX-F3dUZHvFk)leZ1v5U*#@uE(h=34IAK`4$$0Zo41ewR!RbtD zZSSv-srQXRLW&{=#IU$9w*<2 z2eH4%m?O58xhjc(;Uv#u^5Pr5(0N8cUQAz^*bO`8J^xL#LBS}DwBnkVbbZR1SI4<{ zl41ky8vvVHWFcRBJ5;9##}-23-}^da1`@05!+@In(Cb%itcOxkZ}|$m4`{A^yOK+; zx!)GE^05m^hi9JH-B#$yiY_RJGi`j`3~u5o+`Oid(IltCGTLsc8bIk!sd~lR@p}3G z?tilDf*Gy#kASnnt5IJ6OH)m;^8%s#lg-3|3(*IDHef=$vqA!kq89`-6Up>=wmmqQ-`K zq8NnrNF^mnj%#Zv;TS7Jmb)XkaA*{nhKguf*y^GE>+ffA$ZF2JBS}x|6GymMQwm}Z zP7f%i=W|}U?GPCnWNeFZ=tVFR=7<12-XUy-`$sq@=Bsu5^7#p+yYZ)IWL)$wl%FXW zNj%AwG?S8Y(8Zr>^5!C4k=WMoTD$XE?E8;#T0#)?mpEsmn(19lm2(F?7W*=JAE3@9 zr)JICV1h>S*_ombZ{L4@uKYIZ$E~3JH|tA^n3$d~j3X<{Ezdn^}*>1C4a7T;vmILfypGnjYN6;?NzJ zCp)OZE&Z6A`Vg@M%_gi^JRb36!^Scu`eD9ko}ZftE~&URM64}un_XU!k*-S7E2U}_nM?mNM%AS-t|zqijq;grVW4K09D#n zyUD!@AIO8_bb4D{za%thYJP4D3_e9oO;zZv-EV9bV9d2G)ke2@QbT*XAa z@tjKU9+&am4i3*;Ekn@aM9@x42WhJ7b6F*u1x`NGnsjyxDm^SpH4F=7Q`ng=yvHDk zF!Kr5J-mZ*gYjJQP-)~N9}D!VF!G!=?|7(K>T>fGM^0{-^_gQkqj{P4z62>k_JUYh zj`#V{Uf=r!M!+fK3&03C03D-F|0C%7?+|zYktxH%5((1Sz1ugzeF}uyzTZaF&9Ei0 z!V`t9)81F6E=N)ZCh419u?>;g%FM|cG+{mt$f=64=KUKa!{0Y`Q1MJh)knmTTH4I^ z!AaIJqZ-QN?E_KOXv^FWy73$jP;r2)RAOoTi7Nuy!nbW4F!v&+mcN<&3^w={5P=a9 zFq{?+S{gF4i3rg}!f=EHmZBTMxDmmHWxq?}WvHAHB=UID*2ZnyR3qhnA1Uq5_NX-1 zY`zy;UOkwBT+trvfGb$BQ!siS_p!O98S@Z-T!+P3YUd0=m^%K6OAS9{rqVt{XD@rI z^5gRk?3YMiY2O0(q9ptgX~dmWSqY29oTMPVVM5W5#GF|aLwo1dMRlm)?xKy1EuYUk zGaEfw*WMDA5GE~aaKSt8ioV#8McS*xd~Bm|XZ5tr53!y+n3i%lABHX_`kOHv$J~?4 zf@#k>Ntw}oBeloJhh&9QG`qL=F}?LWr{4FOal%vk=TX=K2h3Z7!dQo5_=%}-glDbd zd{R^Nvk!GiPTYmEY9WvMpHfH}UR4{TnEqxt7{QKJ>6hiXWbCn0os>t1KdD(8ldg&A z3AXg(V!rkIIW99kGGwsAMzk7>>TM|^#H@}c=F#uk=^Lv1ZdYl@g=*|=(aT9)!#%16 zeXbB{V88sx!`DZ6`t6JVj0CBGyvK*s{)I>4cznO-m5j$Wm@POek=- zX)jtzDSEr^u*2FsE&VRu>a*;Z?^_2X^0h5D_sGsej$rwpv9f<7a{+Obja|)vOUA4X z%!5vNn*x@5bu{kVC~U(XKB;t5;>@m{ppOmJ%5?GboJw|;Zlotx80f-o z{>+>HMfKpnYUA(D?*Bs59g5y|+r45iroNzwBrrRWd&0eD0;Pq(pg@$qX@cM0QiWG> zXe(+}Ap8fyUk5|9_-GiF91zG=;Bi8>mVpn1U3|tFg;lvGg z<@=3u&(Lo>+L0EPf;?t0woZ!yd1l#CXb-}>I{Jo80hpzL%M2>rs6rNneC@iwFqd@dTJfqQUwVz_ntHR~vEP1dH6+7A=Gn)*12+dEXAjhS=`tERQk@&- zyD}t$Q^RvghqRp}D{x8ALQ2pPnT6{%M4$`CXWLWYrDoqu)!2bat-G|&Zo1f6jk3M! z>>&i{gv7rizwq*Xn_;2x8j@>cNeMg9>Bk##x50?X7$$Oe`+HPw|Lg8|Y1gF-?0l?U zT7HDDbI!${HkT+Mg+4rd5y9}*Yq@41uDU}2S71J)v`iJ^ANnDEV#7#nz@$CNLPxqU zspgFENx=__L6-}7(o&J9!7~mQ^G-TYfPQ>jP~I(eL|tP$CJ${9~mm^KNNQ-ScPCC%||4b5`ykyGMI4m8~v zGx3P8>?E6Kz7P(&r~Hby&}7 z4F>HA_H&07202O@DG0@O4!?{AG-3VhC~A!$tYLQ^cepF&OZQcrjv=lB9uwFO*PpH_ z;@%A7j%=DwdMA-M)4#EF<(}fPkMep&V{9hs$y~S$`@?e3dxZ5`EYoTUxo07BigDHF zTF3pu`Dkmo14Jt1H*6B0u9+TBu{gtcr(Dk5wbK-1Dw*zmxKri79OLa zl#eSm1~xhb#+9n6YzMC-(bY*;VvTs_Qm!#t$y*OZ%awImKh6u>PEdVYakcf9aGWHvCxUU-yMYdA zhXtX$UmTpV0m--yTRq5bzkbk_X(woQuJM|VbA?TJ%Fy|BEB*GDWZAdK5TqeOy#z+A zN;KY}GF%l`wgy=#b-mLwll8i6Wl;KCLl(TQZeZ`G1(j&VpfuC4|4vH*1HFMT^qa+z zMfsQBE099g0J&%}BQSBIca$6q{SWj8{*uIMf$aW|TEk4^DSx*p zjQBIB{%6X^A7=%jU)}Z&51(~#;i+SK5cF+s#hc6yo@&+6Z8~&)MfLtR+;-d!_3bSQ zuR;s7LgJ?zpP0!Hri0-QpKu;r*%Q1E&f{tLMNE6nZgd)&BR9-Fh(ou1EXl>aN5k_q zaI^J(w7~re`UVUF&q}dUeFyY7v@qB6kmwv(kCy0oLq&j}Oik(h8@KbQ6rV3=Zx&nK z@$L;^pt{1yqWyXZ_Nk<26G5?_OEDIRjQP6oHpK5-ZRDtNn6*Vn#E5sKc=ws83ZR_) zLL@nbsS>r>k5GrIC^k^~JnB$xlK1ax?PeJgx^rv*$RwYbbjgMNeSG z84{QluY_3#9otgW)D#|949pa6Zh7%gfSf%`>A0=T>C`g^4>;By-iLmZIob>UlDQQ6 zrF#@)tZP2@jY2`>#Gs*qxzOD>e?9-Z+yL)#^tbv)+=g+|dHw@D@sN;5&p7ZeUejv> z2jqX#)h|g;Yf6Mk^*Ky#c&m9U?KPG*D8C4_aIAmzNg^r-lIZ#T`0Yyx8DvyH!oSFTGiP z%TXQdAz4edVZ1W^lGjQ%_PsYQ+AV>38gh}657k#A$qd-!lZ0C{pC&I6j^mKFShsFt z@U&i!8D+ywH=U`<%E#YqvlBdoVsrKN9p75s2|wmH&)tup(~v{^QG<-Xi<&th#~X7h zoV)b%?bD9u=n<igZ<`0!)0F^Y#? z)|gkAg@!3-lLp)2!Ef2I1<`sBT~i7VG<(j9I4t&K8{ZNVKF#-Mpr!c+dbvb~>^ShjPTJdX z(&;``u~7tT%7k2WQ2g;;CdPnzkfnuaP#s)wN04d23X?f-mU>o-37U=U5zMGu^dOgr zYh&u^*;NbolZ=n`p7?aGX8{MkN4obv!c)Kg`oDx;|IeEKO>)uZf8^!=zIlS_um8Mx zf--c?VZLYOWA_&ZT!0ZaqGB~$Y}Pm2e5K+<^;+^^Yx)wB$zx2z--vBCXlL;OxtOp2 z3st3WAMa>DR+3|EY~4FZ$13Z&#R4P;+(=FdKkWFH$_lhL1v^=NS9-r(-vnka2!YuP zB*+c(qFFF`!6BND9fRyTl-v`UA)Ek@J-gs)R@?P0)O<^Lw@OUvr_s~JY&PCXx@F-hy)1a9t6Q!?9G^-QJ4*Q4&12fn~iz?rA|ZkSN@fb_dh%N zQ=Sn1t_Z>0Lt*^7kZc-Z2WFcH#;u|$-+IJW|D=;wX1cWerx2&tS6(z+47&LEh4Iqb zbp}9N3O2@o+grYLm?S-EGb@nS*V5Dj+YWUMyL;f>_3i10#n7;}?hdcZ9da+$PAKi# zgxL1TjZ|J5D6}r@5ReE`VZ%muNHTSzJk$e}1x)gX5L~i(wog8jZGWfklI7r>be(_tusjSFzZSx{WyiULTx@$EH6lN~(aniGmuVy7m;<%;G! zTF5$EC#xK5BUPoInOmQp!IqQ7Yd>321KwQWd^bk%fj2KnG28D=!fIp2wMxB~_Sxk6 z98D($SiMp&pEOKSB%sy*bfUGf?q(P3{<6Il3aUjd7AuRl>Hx;bo63iIz6USQT}OGm z$U2saJ|Ubjy!5(GFEv{D{zi4q5aq7gZ)f5DI1~3beXy_^7dB~7*u+qYmrdkWHR_s_4uJ7IU! zKpvI}6ggF6j#MgXxlWJbhxU*^k;QWBYJ@tct|;RC;B!6xr~r_nmK_I{RgF z&&zJ>U~SR;WtcS`obHo#HJ{mg;Yt;r->a+YzZRzj7uqk(UDC6dD^Q`VGZ5B>aymDB z7boNH`2Tk0A45(2B|GgJ#~<@QtbSCwtPwc*D)@%?YTv?z^_i}jf9EHQ&hpJWU71(& zdE<=fY(`wG-OBQEfir~mo7Fl$EI)FO{aTIfnzOdkoBrsptSE6T7N~r$dgXm?{<6K( z3Lda$=HA%E@xiC1X<>nbRsGG^kMcYJssB(9-2L%r@6z_p8q2j=v)GpGsBsV7)cPhue0V=P`+-M$CS;JPc6z*+*Ef)Ogx@zPw*}(l4}N#5Y!y7jB78gMu*qbbqJ`7U+=T?SrdbE96#dTLR%ev`z1&{1Wcj9T zf`9bARW~HammZkyy!}f5{LcxGJiU5n*9BOAj<8euJGV{+xW~i2?~ndR{YNI^)9v&= z{P`!p?ceE(xBV}#33^=id)wyhleendeir$>5OoaUk_1lWt-rba`1O4nAD@12{HNsl z@GkeG(y42=?aWr2ps#l+I$HJepV!>;w;Y+UW!j6bLmp|j!iyfvW3X%d;ri(J`;K@n z8}Y|)gCyR*ukl_4tgAmd?{JmbKJDU}SF<~F7tU#%fg)`d+m>DM@T^Ooxc$rg zZxz>pi*#2%=x??O{+OG)_+#4E<*v2WvP*RHMe4K8Y;=FR+cMl^rkGzrj3IDKxcj%J zzw`bu{`UM~bo{`5!K6vP_WoCOwnqF}#9X# z?J0q|N8Y7MJel}@>toP_^PyVn$MX-?OZ-v4(!}ynZm#YgeT7p;OZM;CHly+Ssky** z=mQhR&*FcCmw)U0(0`mCctXR6^{pjy!%q3P&C+XM7=P#fL6NfU<`W`yg1FuLGPzet zY${(Ru&!NZg5BTs6YLY?Z~prHE$8oS;PT1;46$3i3^ryA?JWA3N7` n;mF2Ak&;t>Rt5n#3G9dOet>l`k!gM`>kDDsY--cY|8D{S>XFvd diff --git a/doc/v1_api_tutorials/imagenet_model/resnet_model_cn.md b/doc/v1_api_tutorials/imagenet_model/resnet_model_cn.md deleted file mode 100644 index 82ec9d70b3..0000000000 --- a/doc/v1_api_tutorials/imagenet_model/resnet_model_cn.md +++ /dev/null @@ -1,284 +0,0 @@ -# Model Zoo - ImageNet # - -[ImageNet](http://www.image-net.org/) 是通用物体分类领域一个众所周知的数据库。本教程提供了一个用于ImageNet上的卷积分类网络模型。 - -## ResNet 介绍 - -论文 [Deep Residual Learning for Image Recognition](http://arxiv.org/abs/1512.03385) 中提出的ResNet网络结构在2015年ImageNet大规模视觉识别竞赛(ILSVRC 2015)的分类任务中赢得了第一名。他们提出残差学习的框架来简化网络的训练,所构建网络结构的的深度比之前使用的网络有大幅度的提高。下图展示的是基于残差的连接方式。左图构造网络模块的方式被用于34层的网络中,而右图的瓶颈连接模块用于50层,101层和152层的网络结构中。 - -
![resnet_block](./resnet_block.jpg)
-
图 1. ResNet 网络模块
- -本教程中我们给出了三个ResNet模型,这些模型都是由原作者提供的模型转换过来的。我们使用PaddlePaddle在ILSVRC的验证集共50,000幅图像上测试了模型的分类错误率,其中输入图像的颜色通道顺序为**BGR**,保持宽高比缩放到短边为256,只截取中心方形的图像区域。分类错误率和模型大小由下表给出。 -
- ----- - - - - - - - - - - - - - - - - - - - - - - - - - - -
ResNetTop-1Model Size
ResNet-5024.9%99M
ResNet-10123.7%173M
ResNet-15223.2%234M
-
- -## ResNet 模型 - -50层,101层和152层的网络配置文件可参照```demo/model_zoo/resnet/resnet.py```。你也可以通过在命令行参数中增加一个参数如```--config_args=layer_num=50```来指定网络层的数目。 - -### 网络可视化 - -你可以通过执行下面的命令来得到ResNet网络的结构可视化图。该脚本会生成一个dot文件,然后可以转换为图片。需要安装graphviz来转换dot文件为图片。 - -``` -cd demo/model_zoo/resnet -./net_diagram.sh -``` - -### 模型下载 - -``` -cd demo/model_zoo/resnet -./get_model.sh -``` -你可以执行上述命令来下载所有的模型和均值文件,如果下载成功,这些文件将会被保存在```demo/model_zoo/resnet/model```路径下。 - -``` -mean_meta_224 resnet_101 resnet_152 resnet_50 -``` - * resnet_50: 50层网络模型。 - * resnet_101: 101层网络模型。 - * resnet_152: 152层网络模型。 - * mean\_meta\_224: 均值图像文件,图像大小为3 x 224 x 224,颜色通道顺序为**BGR**。你也可以使用这三个值: 103.939, 116.779, 123.68。 - -### 参数信息 - -* **卷积层权重** - - 由于每个卷积层后面连接的是batch normalization层,因此该层中没有偏置(bias)参数,并且只有一个权重。 - 形状: `(Co, ky, kx, Ci)` - * Co: 输出特征图的通道数目 - * ky: 滤波器核在垂直方向上的尺寸 - * kx: 滤波器核在水平方向上的尺寸 - * Ci: 输入特征图的通道数目 - - 二维矩阵: (Co * ky * kx, Ci), 行优先次序存储。 - -* **全连接层权重** - - 二维矩阵: (输入层尺寸, 本层尺寸), 行优先次序存储。 - -* **[Batch Normalization]() 层权重** - -本层有四个参数,实际上只有.w0和.wbias是需要学习的参数,另外两个分别是滑动均值和方差。在测试阶段它们将会被加载到模型中。下表展示了batch normalization层的参数。 -
- ----- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
参数名尺寸含义
_res2_1_branch1_bn.w0256gamma, 缩放参数
_res2_1_branch1_bn.w1256特征图均值
_res2_1_branch1_bn.w2256特征图方差
_res2_1_branch1_bn.wbias256beta, 偏置参数
-
- -### 参数读取 - -使用者可以使用下面的Python脚本来读取参数值: - -``` -import sys -import numpy as np - -def load(file_name): - with open(file_name, 'rb') as f: - f.read(16) # skip header for float type. - return np.fromfile(f, dtype=np.float32) - -if __name__=='__main__': - weight = load(sys.argv[1]) -``` - -或者直接使用下面的shell命令: - -``` -od -j 16 -f _res2_1_branch1_bn.w0 -``` - -## 特征提取 - -我们提供了C++和Python接口来提取特征。下面的例子使用了`demo/model_zoo/resnet/example`中的数据,详细地展示了整个特征提取的过程。 - -### C++接口 - -首先,在配置文件中的`define_py_data_sources2`里指定图像数据列表,具体请参照示例`demo/model_zoo/resnet/resnet.py`。 - -``` - train_list = 'train.list' if not is_test else None - # mean.meta is mean file of ImageNet dataset. - # mean.meta size : 3 x 224 x 224. - # If you use three mean value, set like: - # "mean_value:103.939,116.779,123.68;" - args={ - 'mean_meta': "model/mean_meta_224/mean.meta", - 'image_size': 224, 'crop_size': 224, - 'color': True,'swap_channel:': [2, 1, 0]} - define_py_data_sources2(train_list, - 'example/test.list', - module="example.image_list_provider", - obj="processData", - args=args) -``` - -第二步,在`resnet.py`文件中指定要提取特征的网络层的名字。例如, - -``` -Outputs("res5_3_branch2c_conv", "res5_3_branch2c_bn") -``` - -第三步,在`extract_fea_c++.sh`文件中指定模型路径和输出的目录,然后执行下面的命令。 - -``` -cd demo/model_zoo/resnet -./extract_fea_c++.sh -``` - -如果执行成功,特征将会存到`fea_output/rank-00000`文件中,如下所示。同时你可以使用`load_feature.py`文件中的`load_feature_c`接口来加载该文件。 - -``` --0.115318 -0.108358 ... -0.087884;-1.27664 ... -1.11516 -2.59123; --0.126383 -0.116248 ... -0.00534909;-1.42593 ... -1.04501 -1.40769; -``` - -* 每行存储的是一个样本的特征。其中,第一行存的是图像`example/dog.jpg`的特征,第二行存的是图像`example/cat.jpg`的特征。 -* 不同层的特征由分号`;`隔开,并且它们的顺序与`Outputs()`中指定的层顺序一致。这里,左边是`res5_3_branch2c_conv`层的特征,右边是`res5_3_branch2c_bn`层特征。 - -### Python接口 - -示例`demo/model_zoo/resnet/classify.py`中展示了如何使用Python来提取特征。下面的例子同样使用了`./example/test.list`中的数据。执行的命令如下: - -``` -cd demo/model_zoo/resnet -./extract_fea_py.sh -``` - -extract_fea_py.sh: - -``` -python classify.py \ - --job=extract \ - --conf=resnet.py\ - --use_gpu=1 \ - --mean=model/mean_meta_224/mean.meta \ - --model=model/resnet_50 \ - --data=./example/test.list \ - --output_layer="res5_3_branch2c_conv,res5_3_branch2c_bn" \ - --output_dir=features - -``` -* \--job=extract: 指定工作模式来提取特征。 -* \--conf=resnet.py: 网络配置文件。 -* \--use_gpu=1: 指定是否使用GPU。 -* \--model=model/resnet_50: 模型路径。 -* \--data=./example/test.list: 数据列表。 -* \--output_layer="xxx,xxx": 指定提取特征的层。 -* \--output_dir=features: 输出目录。 - -如果运行成功,你将会看到特征存储在`features/batch_0`文件中,该文件是由cPickle产生的。你可以使用`load_feature.py`中的`load_feature_py`接口来打开该文件,它将返回如下的字典: - -``` -{ -'cat.jpg': {'res5_3_branch2c_conv': array([[-0.12638293, -0.116248 , -0.11883899, ..., -0.00895038, 0.01994277, -0.00534909]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.42593431, -1.28918779, -1.32414699, ..., -1.45933616, -1.04501402, -1.40769434]], dtype=float32)}, -'dog.jpg': {'res5_3_branch2c_conv': array([[-0.11531784, -0.10835785, -0.08809858, ...,0.0055237, 0.01505112, -0.08788397]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.27663755, -1.18272924, -0.90937918, ..., -1.25178063, -1.11515927, -2.59122872]], dtype=float32)} -} -``` - -仔细观察,这些特征值与上述使用C++接口提取的结果是一致的。 - -## 预测 - -`classify.py`文件也可以用于对样本进行预测。我们提供了一个示例脚本`predict.sh`,它使用50层的ResNet模型来对`example/test.list`中的数据进行预测。 - -``` -cd demo/model_zoo/resnet -./predict.sh -``` - -predict.sh调用了`classify.py`: - -``` -python classify.py \ - --job=predict \ - --conf=resnet.py\ - --multi_crop \ - --model=model/resnet_50 \ - --use_gpu=1 \ - --data=./example/test.list -``` -* \--job=extract: 指定工作模型进行预测。 -* \--conf=resnet.py: 网络配置文件。network configure. -* \--multi_crop: 使用10个裁剪图像块,预测概率取平均。 -* \--use_gpu=1: 指定是否使用GPU。 -* \--model=model/resnet_50: 模型路径。 -* \--data=./example/test.list: 数据列表。 - -如果运行成功,你将会看到如下结果,其中156和285是这些图像的分类标签。 - -``` -Label of example/dog.jpg is: 156 -Label of example/cat.jpg is: 282 -``` diff --git a/doc/v1_api_tutorials/imagenet_model/resnet_model_en.md b/doc/v1_api_tutorials/imagenet_model/resnet_model_en.md deleted file mode 100644 index 478ad06193..0000000000 --- a/doc/v1_api_tutorials/imagenet_model/resnet_model_en.md +++ /dev/null @@ -1,284 +0,0 @@ -# Model Zoo - ImageNet # - -[ImageNet](http://www.image-net.org/) is a popular dataset for generic object classification. This tutorial provides convolutional neural network(CNN) models for ImageNet. - -## ResNet Introduction - -ResNets from paper [Deep Residual Learning for Image Recognition](http://arxiv.org/abs/1512.03385) won the 1st place on the ILSVRC 2015 classification task. They present residual learning framework to ease the training of networks that are substantially deeper than those used previously. The residual connections are shown in following figure. The left building block is used in network of 34 layers and the right bottleneck building block is used in network of 50, 101, 152 layers . - -
![resnet_block](./resnet_block.jpg)
-
Figure 1. ResNet Block
- -We present three ResNet models, which are converted from the models provided by the authors . The classfication errors tested in PaddlePaddle on 50,000 ILSVRC validation set with input images channel order of **BGR** by single scale with the shorter side of 256 and single crop as following table. -
- ----- - - - - - - - - - - - - - - - - - - - - - - - - - - -
ResNetTop-1Model Size
ResNet-5024.9%99M
ResNet-10123.7%173M
ResNet-15223.2%234M
-
- -## ResNet Model - -See ```demo/model_zoo/resnet/resnet.py```. This config contains network of 50, 101 and 152 layers. You can specify layer number by adding argument like ```--config_args=layer_num=50``` in command line arguments. - -### Network Visualization - -You can get a diagram of ResNet network by running the following commands. The script generates dot file and then converts dot file to PNG file, which needs to install graphviz to convert. - -``` -cd demo/model_zoo/resnet -./net_diagram.sh -``` - -### Model Download - -``` -cd demo/model_zoo/resnet -./get_model.sh -``` -You can run above command to download all models and mean file and save them in ```demo/model_zoo/resnet/model``` if downloading successfully. - -``` -mean_meta_224 resnet_101 resnet_152 resnet_50 -``` - * resnet_50: model of 50 layers. - * resnet_101: model of 101 layers. - * resnet_152: model of 152 layers. - * mean\_meta\_224: mean file with 3 x 224 x 224 size in **BGR** order. You also can use three mean values: 103.939, 116.779, 123.68. - -### Parameter Info - -* **Convolution Layer Weight** - - As batch normalization layer is connected after each convolution layer, there is no parameter of bias and only one weight in this layer. - shape: `(Co, ky, kx, Ci)` - * Co: channle number of output feature map. - * ky: filter size in vertical direction. - * kx: filter size in horizontal direction. - * Ci: channle number of input feature map. - - 2-Dim matrix: (Co * ky * kx, Ci), saved in row-major order. - -* **Fully connected Layer Weight** - - 2-Dim matrix: (input layer size, this layer size), saved in row-major order. - -* **[Batch Normalization]() Layer Weight** - -There are four parameters in this layer. In fact, only .w0 and .wbias are the learned parameters. The other two are therunning mean and variance respectively. They will be loaded in testing. Following table shows parameters of a batch normzalization layer. -
- ----- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Parameter NameNumberMeaning
_res2_1_branch1_bn.w0256gamma, scale parameter
_res2_1_branch1_bn.w1256mean value of feature map
_res2_1_branch1_bn.w2256variance of feature map
_res2_1_branch1_bn.wbias256beta, shift parameter
-
- -### Parameter Observation - -Users who want to observe the parameters can use Python to read: - -``` -import sys -import numpy as np - -def load(file_name): - with open(file_name, 'rb') as f: - f.read(16) # skip header for float type. - return np.fromfile(f, dtype=np.float32) - -if __name__=='__main__': - weight = load(sys.argv[1]) -``` - -or simply use following shell command: - -``` -od -j 16 -f _res2_1_branch1_bn.w0 -``` - -## Feature Extraction - -We provide both C++ and Python interfaces to extract features. The following examples use data in `demo/model_zoo/resnet/example` to show the extracting process in detail. - -### C++ Interface - -First, specify image data list in `define_py_data_sources2` in the config, see example `demo/model_zoo/resnet/resnet.py`. - -``` - train_list = 'train.list' if not is_test else None - # mean.meta is mean file of ImageNet dataset. - # mean.meta size : 3 x 224 x 224. - # If you use three mean value, set like: - # "mean_value:103.939,116.779,123.68;" - args={ - 'mean_meta': "model/mean_meta_224/mean.meta", - 'image_size': 224, 'crop_size': 224, - 'color': True,'swap_channel:': [2, 1, 0]} - define_py_data_sources2(train_list, - 'example/test.list', - module="example.image_list_provider", - obj="processData", - args=args) -``` - -Second, specify layers to extract features in `Outputs()` of `resnet.py`. For example, - -``` -Outputs("res5_3_branch2c_conv", "res5_3_branch2c_bn") -``` - -Third, specify model path and output directory in `extract_fea_c++.sh`, and then run the following commands. - -``` -cd demo/model_zoo/resnet -./extract_fea_c++.sh -``` - -If successful, features are saved in `fea_output/rank-00000` as follows. And you can use `load_feature_c` interface in `load_feature.py ` to load such a file. - -``` --0.115318 -0.108358 ... -0.087884;-1.27664 ... -1.11516 -2.59123; --0.126383 -0.116248 ... -0.00534909;-1.42593 ... -1.04501 -1.40769; -``` - -* Each line stores features of a sample. Here, the first line stores features of `example/dog.jpg` and second line stores features of `example/cat.jpg`. -* Features of different layers are splitted by `;`, and their order is consistent with the layer order in `Outputs()`. Here, the left features are `res5_3_branch2c_conv` layer and right features are `res5_3_branch2c_bn` layer. - -### Python Interface - -`demo/model_zoo/resnet/classify.py` is an example to show how to use Python to extract features. Following example still uses data of `./example/test.list`. Command is as follows: - -``` -cd demo/model_zoo/resnet -./extract_fea_py.sh -``` - -extract_fea_py.sh: - -``` -python classify.py \ - --job=extract \ - --conf=resnet.py\ - --use_gpu=1 \ - --mean=model/mean_meta_224/mean.meta \ - --model=model/resnet_50 \ - --data=./example/test.list \ - --output_layer="res5_3_branch2c_conv,res5_3_branch2c_bn" \ - --output_dir=features - -``` -* \--job=extract: specify job mode to extract feature. -* \--conf=resnet.py: network configure. -* \--use_gpu=1: speficy GPU mode. -* \--model=model/resnet_5: model path. -* \--data=./example/test.list: data list. -* \--output_layer="xxx,xxx": specify layers to extract features. -* \--output_dir=features: output diretcoty. - -If run successfully, you will see features saved in `features/batch_0`, this file is produced with cPickle. You can use `load_feature_py` interface in `load_feature.py` to open the file, and it returns a dictionary as follows: - -``` -{ -'cat.jpg': {'res5_3_branch2c_conv': array([[-0.12638293, -0.116248 , -0.11883899, ..., -0.00895038, 0.01994277, -0.00534909]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.42593431, -1.28918779, -1.32414699, ..., -1.45933616, -1.04501402, -1.40769434]], dtype=float32)}, -'dog.jpg': {'res5_3_branch2c_conv': array([[-0.11531784, -0.10835785, -0.08809858, ...,0.0055237, 0.01505112, -0.08788397]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.27663755, -1.18272924, -0.90937918, ..., -1.25178063, -1.11515927, -2.59122872]], dtype=float32)} -} -``` - -Observed carefully, these feature values are consistent with the above results extracted by C++ interface. - -## Prediction - -`classify.py` also can be used to predict. We provide an example script `predict.sh` to predict data in `example/test.list` using a ResNet model with 50 layers. - -``` -cd demo/model_zoo/resnet -./predict.sh -``` - -predict.sh calls the `classify.py`: - -``` -python classify.py \ - --job=predict \ - --conf=resnet.py\ - --multi_crop \ - --model=model/resnet_50 \ - --use_gpu=1 \ - --data=./example/test.list -``` -* \--job=extract: speficy job mode to predict. -* \--conf=resnet.py: network configure. -* \--multi_crop: use 10 crops and average predicting probability. -* \--use_gpu=1: speficy GPU mode. -* \--model=model/resnet_50: model path. -* \--data=./example/test.list: data list. - -If run successfully, you will see following results, where 156 and 285 are labels of the images. - -``` -Label of example/dog.jpg is: 156 -Label of example/cat.jpg is: 282 -``` diff --git a/doc/v1_api_tutorials/quick_start/index_cn.rst b/doc/v1_api_tutorials/quick_start/index_cn.rst deleted file mode 100644 index d565fcf95e..0000000000 --- a/doc/v1_api_tutorials/quick_start/index_cn.rst +++ /dev/null @@ -1,397 +0,0 @@ -============= -快速入门教程 -============= - -我们将以 `文本分类问题 `_ 为例, -介绍PaddlePaddle的基本使用方法。 - -安装 -==== - -请参考 :ref:`install_steps` 安装PaddlePaddle。 - -使用概述 -======== - -**文本分类问题**:对于给定的一条文本,我们从提前给定的类别集合中选择其所属类别。 - -比如, 在购物网站上,通过查看买家对某个产品的评价反馈, 评估该产品的质量。 - -- 这个显示器很棒! (好评) -- 用了两个月之后这个显示器屏幕碎了。(差评) - -使用PaddlePaddle, 每一个任务流程都可以被划分为如下五个步骤。 - - .. image:: src/Pipeline_cn.jpg - :align: center - :scale: 80% - -1. 数据格式准备 - - 本例每行保存一条样本,类别Id和文本信息用 ``Tab`` 间隔,文本中的单词用空格分隔(如果不切词,则字与字之间用空格分隔),例如:``类别Id '\t' 这 个 显 示 器 很 棒 !`` -2. 向系统传送数据 - - PaddlePaddle可以执行用户的python脚本程序来读取各种格式的数据文件。 - - 本例的所有字符都将转换为连续整数表示的Id传给模型。 -3. 描述网络结构和优化算法 - - 本例由易到难展示4种不同的文本分类网络配置:逻辑回归模型,词向量模型,卷积模型,时序模型。 - - 常用优化算法包括Momentum, RMSProp,AdaDelta,AdaGrad,Adam,Adamax等,本例采用Adam优化方法,加了L2正则和梯度截断。 -4. 训练模型 -5. 应用模型 - -数据格式准备 ------------- - -接下来我们将展示如何用PaddlePaddle训练一个文本分类模型,将 `Amazon电子产品评论数据 `_ 分为好评(正样本)和差评(负样本)两种类别。 -`源代码 `_ 的 ``demo/quick_start`` 目录里提供了该数据的下载脚本和预处理脚本,你只需要在命令行输入以下命令,就能够很方便的完成数据下载和相应的预处理工作。 - -.. code-block:: bash - - cd demo/quick_start - ./data/get_data.sh - ./preprocess.sh - -数据预处理完成之后,通过配置类似于 ``dataprovider_*.py`` 的数据读取脚本和类似于 ``trainer_config.*.py`` 的训练模型脚本,PaddlePaddle将以设置参数的方式来设置 -相应的数据读取脚本和训练模型脚本。接下来,我们将对这两个步骤给出了详细的解释,你也可以先跳过本文的解释环节,直接进入训练模型章节, 使用 ``sh train.sh`` 开始训练模型, -查看`train.sh`内容,通过 **自底向上法** (bottom-up approach)来帮助你理解PaddlePaddle的内部运行机制。 - - -向系统传送数据 -============== - -Python脚本读取数据 ------------------- - -`DataProvider` 是PaddlePaddle负责提供数据的模块,主要职责在于将训练数据传入内存或者显存,让模型能够得到训练更新,其包括两个函数: - -* initializer:PaddlePaddle会在调用读取数据的Python脚本之前,先调用initializer函数。在下面例子里,我们在initialzier函数里初始化词表,并且在随后的读取数据过程中填充词表。 -* process:PaddlePaddle调用process函数来读取数据。每次读取一条数据后,process函数会用yield语句输出这条数据,从而能够被PaddlePaddle 捕获 (harvest)。 - -``dataprovider_bow.py`` 文件给出了完整例子: - -.. literalinclude:: ../../../demo/quick_start/dataprovider_bow.py - :language: python - :lines: 21-70 - :linenos: - :emphasize-lines: 8,33 - -详细内容请参见 :ref:`api_dataprovider` 。 - -配置中的数据加载定义 --------------------- - -在模型配置中通过 ``define_py_data_sources2`` 接口来加载数据: - -.. literalinclude:: ../../../demo/quick_start/trainer_config.emb.py - :language: python - :lines: 19-35 - :linenos: - :emphasize-lines: 12 - - -以下是对上述数据加载的解释: - -- data/train.list,data/test.list: 指定训练数据和测试数据 -- module="dataprovider_bow": 处理数据的Python脚本文件 -- obj="process": 指定生成数据的函数 -- args={"dictionary": word_dict}: 额外的参数,这里指定词典 - -更详细数据格式和用例请参考 :ref:`api_pydataprovider2` 。 - -模型网络结构 -============ - -本小节我们将介绍模型网络结构。 - - .. image:: src/PipelineNetwork_cn.jpg - :align: center - :scale: 80% - - -我们将以最基本的逻辑回归网络作为起点,并逐渐展示更加深入的功能。更详细的网络配置连接请参考 :ref:`api_trainer_config_helpers_layers` 。 -所有配置都能在 `源代码 `_ 的 ``demo/quick_start`` 目录下找到。 - -逻辑回归模型 ------------- - -具体流程如下: - - .. image:: src/NetLR_cn.jpg - :align: center - :scale: 80% - -- 获取利用 `one-hot vector `_ 表示的每个单词,维度是词典大小 - - .. code-block:: python - - word = data_layer(name="word", size=word_dim) - -- 获取该条样本类别Id,维度是类别个数。 - - .. code-block:: python - - label = data_layer(name="label", size=label_dim) - -- 利用逻辑回归模型对该向量进行分类,同时会计算分类准确率 - - .. code-block:: python - - # Define a fully connected layer with logistic activation (also called softmax activation). - output = fc_layer(input=word, - size=label_dim, - act_type=SoftmaxActivation()) - # Define cross-entropy classification loss and error. - classification_cost(input=output, label=label) - - - - input: 除去data层,每个层都有一个或多个input,多个input以list方式输入 - - size: 该层神经元个数 - - act_type: 激活函数类型 - -**效果总结**:我们将在后面介绍训练和预测流程的脚本。在此为方便对比不同网络结构,我们总结了各个网络的复杂度和效果。 - - ===================== =============================== ================= - 网络名称 参数数量 错误率 - ===================== =============================== ================= - 逻辑回归 252 KB 8.652 % - ===================== =============================== ================= - -词向量模型 ----------- - -embedding模型需要稍微改变提供数据的Python脚本,即 ``dataprovider_emb.py``,词向量模型、 -卷积模型、时序模型均使用该脚本。其中文本输入类型定义为整数时序类型integer_value_sequence。 - -.. code-block:: python - - def initializer(settings, dictionary, **kwargs): - settings.word_dict = dictionary - settings.input_types = [ - # Define the type of the first input as sequence of integer. - # The value of the integers range from 0 to len(dictrionary)-1 - integer_value_sequence(len(dictionary)), - # Define the second input for label id - integer_value(2)] - - @provider(init_hook=initializer) - def process(settings, file_name): - ... - # omitted, it is same as the data provider for LR model - -该模型依然使用逻辑回归分类网络的框架, 只是将句子用连续向量表示替换为用稀疏向量表示, 即对第三步进行替换。句子表示的计算更新为两步: - -.. image:: src/NetContinuous_cn.jpg - :align: center - :scale: 80% - -- 利用单词Id查找该单词对应的连续向量(维度为word_dim), 输入N个单词,输出为N个word_dim维度向量 - - .. code-block:: python - - emb = embedding_layer(input=word, size=word_dim) - -- 将该句话包含的所有单词向量求平均, 得到句子的表示 - - .. code-block:: python - - avg = pooling_layer(input=emb, pooling_type=AvgPooling()) - -其它部分和逻辑回归网络结构一致。 - -**效果总结:** - - ===================== =============================== ================== - 网络名称 参数数量 错误率 - ===================== =============================== ================== - 词向量模型 15 MB 8.484 % - ===================== =============================== ================== - -卷积模型 ------------ - -卷积网络是一种特殊的从词向量表示到句子表示的方法, 也就是将词向量模型进一步演化为三个新步骤。 - -.. image:: src/NetConv_cn.jpg - :align: center - :scale: 80% - -文本卷积分可为三个步骤: - -1. 首先,从每个单词左右两端分别获取k个相邻的单词, 拼接成一个新的向量; - -2. 其次,对该向量进行非线性变换(例如Sigmoid变换), 使其转变为维度为hidden_dim的新向量; - -3. 最后,对整个新向量集合的每一个维度取最大值来表示最后的句子。 - -这三个步骤可配置为: - -.. code-block:: python - - text_conv = sequence_conv_pool(input=emb, - context_start=k, - context_len=2 * k + 1) - -**效果总结:** - - ===================== =============================== ======================== - 网络名称 参数数量 错误率 - ===================== =============================== ======================== - 卷积模型 16 MB 5.628 % - ===================== =============================== ======================== - -时序模型 ----------- - -.. image:: src/NetRNN_cn.jpg - :align: center - :scale: 80% - -时序模型,也称为RNN模型, 包括简单的 `RNN模型 `_, `GRU模型 `_ 和 `LSTM模型 `_ 等等。 - -- GRU模型配置: - - .. code-block:: python - - gru = simple_gru(input=emb, size=gru_size) - - -- LSTM模型配置: - - .. code-block:: python - - lstm = simple_lstm(input=emb, size=lstm_size) - -本次试验,我们采用单层LSTM模型,并使用了Dropout,**效果总结:** - - ===================== =============================== ========================= - 网络名称 参数数量 错误率 - ===================== =============================== ========================= - 时序模型 16 MB 4.812 % - ===================== =============================== ========================= - -优化算法 -========= - -`优化算法 `_ 包括 -Momentum, RMSProp,AdaDelta,AdaGrad,ADAM,Adamax等,这里采用Adam优化方法,同时使用了L2正则(L2 Regularization)和梯度截断(Gradient Clipping)。 - -.. code-block:: python - - settings(batch_size=128, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) - -训练模型 -========= - -在数据加载和网络配置完成之后, 我们就可以训练模型了。 - -.. image:: src/PipelineTrain_cn.jpg - :align: center - :scale: 80% - -训练模型,我们只需要运行 ``train.sh`` 训练脚本: - - .. code-block:: bash - - ./train.sh - -``train.sh`` 中包含了训练模型的基本命令。训练时所需设置的主要参数如下: - - .. code-block:: bash - - paddle train \ - --config=trainer_config.py \ - --log_period=20 \ - --save_dir=./output \ - --num_passes=15 \ - --use_gpu=false - -这里只简单介绍了单机训练,如何进行分布式训练,请参考 :ref:`cluster_train` 。 - -预测 -===== - -当模型训练好了之后,我们就可以进行预测了。 - -.. image:: src/PipelineTest_cn.jpg - :align: center - :scale: 80% - -之前配置文件中 ``test.list`` 指定的数据将会被测试,这里直接通过预测脚本 ``predict.sh`` 进行预测, -更详细的说明,请参考 :ref:`api_swig_py_paddle` 。 - - .. code-block:: bash - - model="output/pass-00003" - paddle train \ - --config=trainer_config.lstm.py \ - --use_gpu=false \ - --job=test \ - --init_model_path=$model \ - --config_args=is_predict=1 \ - --predict_output_dir=. \ - - mv rank-00000 result.txt - -这里以 ``output/pass-00003`` 为例进行预测,用户可以根据训练日志,选择测试结果最好的模型来预测。 - -预测结果以文本的形式保存在 ``result.txt`` 中,一行为一个样本,格式如下: - - .. code-block:: bash - - 预测ID;ID为0的概率 ID为1的概率 - 预测ID;ID为0的概率 ID为1的概率 - -总体效果总结 -============== - -在 ``/demo/quick_start`` 目录下,能够找到这里使用的所有数据, 网络配置, 训练脚本等等。 -对于Amazon-Elec测试集(25k), 如下表格,展示了上述网络模型的训练效果: - - ===================== =============================== ============= ================================== - 网络名称 参数数量 错误率 配置文件 - ===================== =============================== ============= ================================== - 逻辑回归模型 252 KB 8.652% trainer_config.lr.py - 词向量模型 15 MB 8.484% trainer_config.emb.py - 卷积模型 16 MB 5.628% trainer_config.cnn.py - 时序模型 16 MB 4.812% trainer_config.lstm.py - ===================== =============================== ============= ================================== - - -附录 -===== - -命令行参数 ----------- - -* \--config:网络配置 -* \--save_dir:模型存储路径 -* \--log_period:每隔多少batch打印一次日志 -* \--num_passes:训练轮次,一个pass表示过一遍所有训练样本 -* \--config_args:命令指定的参数会传入网络配置中。 -* \--init_model_path:指定初始化模型路径,可用在测试或训练时指定初始化模型。 - -默认一个pass保存一次模型,也可以通过saving_period_by_batches设置每隔多少batch保存一次模型。 -可以通过show_parameter_stats_period设置打印参数信息等。 -其他参数请参考 命令行参数文档(链接待补充)。 - -输出日志 ---------- - -.. code-block:: bash - - TrainerInternal.cpp:160] Batch=20 samples=2560 AvgCost=0.628761 CurrentCost=0.628761 Eval: classification_error_evaluator=0.304297 CurrentEval: classification_error_evaluator=0.304297 - -模型训练会看到类似上面这样的日志信息,详细的参数解释,请参考如下表格: - - =========================================== ============================================================== - 名称 解释 - =========================================== ============================================================== - Batch=20 表示过了20个batch - samples=2560 表示过了2560个样本 - AvgCost 每个pass的第0个batch到当前batch所有样本的平均cost - CurrentCost 当前log_period个batch所有样本的平均cost - Eval: classification_error_evaluator 每个pass的第0个batch到当前batch所有样本的平均分类错误率 - CurrentEval: classification_error_evaluator 当前log_period个batch所有样本的平均分类错误率 - =========================================== ============================================================== diff --git a/doc/v1_api_tutorials/quick_start/index_en.md b/doc/v1_api_tutorials/quick_start/index_en.md deleted file mode 100644 index ca110431cf..0000000000 --- a/doc/v1_api_tutorials/quick_start/index_en.md +++ /dev/null @@ -1,562 +0,0 @@ -# Quick Start - -This tutorial will teach the basics of deep learning (DL), including how to implement many different models in PaddlePaddle. You will learn how to: - - Prepare data into the standardized format that PaddlePaddle accepts. - - Write data providers that read data into PaddlePaddle. - - Configure neural networks in PaddlePaddle layer by layer. - - Train models. - - Perform inference with trained models. - - -## Install - -To get started, please install PaddlePaddle on your computer. Throughout this tutorial, you will learn by implementing different DL models for text classification. - -To install PaddlePaddle, please follow the instructions here: Build and Install. - -## Overview -For the first step, you will use PaddlePaddle to build a **text classification** system. For example, suppose you run an e-commence website, and you want to analyze the sentiment of user reviews to evaluate product quality. - -For example, given the input - -``` -This monitor is fantastic. -``` - -Your classifier should output “positive”, since this text snippet shows that the user is satisfied with the product. Given this input: - -``` -The monitor breaks down two months after purchase. -``` - -the classifier should output “negative“. - -To build your text classification system, your code will need to perform five steps: -
![](./src/Pipeline_en.jpg)
- - - Preprocess data into a standardized format. - - Provide data to the learning model. - - Specify the neural network structure. - - Train the model. - - Inference (make prediction on test examples). - - -1. Preprocess data into standardized format - - In the text classification example, you will start with a text file with one training example per line. Each line contains category id (in machine learning, often denoted the target y), followed by the input text (often denoted x); these two elements are separated by a Tab. For example: ```positive [tab] This monitor is fantastic```. You will preprocess this raw data into a format that Paddle can use. - -2. Provide data to the learning model. - - You can write data providers in Python. For any required data preprocessing step, you can add the preprocessing code to the PyDataProvider Python file. - - In our text classification example, every word or character will be converted into an integer id, specified in a dictionary file. It perform a dictionary lookup in PyDataProvider to get the id. -3. Specify neural network structure. (From easy to hard, we provide 4 kinds of network configurations) - - A logistic regression model. - - A word embedding model. - - A convolutional neural network model. - - A sequential recurrent neural network model. - - You will also learn different learning algorithms. -4. Training model. -5. Inference. - -## Preprocess data into standardized format -In this example, you are going to use [Amazon electronic product review dataset](http://jmcauley.ucsd.edu/data/amazon/) to build a bunch of deep neural network models for text classification. Each text in this dataset is a product review. This dataset has two categories: “positive” and “negative”. Positive means the reviewer likes the product, while negative means the reviewer does not like the product. - -`demo/quick_start` in the [source code](https://github.com/PaddlePaddle/Paddle) provides script for downloading the preprocessed data as shown below. (If you want to process the raw data, you can use the script `demo/quick_start/data/proc_from_raw_data/get_data.sh`). - -```bash -cd demo/quick_start -./data/get_data.sh -``` - -## Transfer Data to Model -### Write Data Provider with Python -The following `dataprovider_bow.py` gives a complete example of writing data provider with Python. It includes the following parts: - -* initalizer: define the additional meta-data of the data provider and the types of the input data. -* process: Each `yield` returns a data sample. In this case, it return the text representation and category id. The order of features in the returned result needs to be consistent with the definition of the input types in `initalizer`. - -```python -from paddle.trainer.PyDataProvider2 import * - -# id of the word not in dictionary -UNK_IDX = 0 - -# initializer is called by the framework during initialization. -# It allows the user to describe the data types and setup the -# necessary data structure for later use. -# `settings` is an object. initializer need to properly fill settings.input_types. -# initializer can also store other data structures needed to be used at process(). -# In this example, dictionary is stored in settings. -# `dictionay` and `kwargs` are arguments passed from trainer_config.lr.py -def initializer(settings, dictionary, **kwargs): - # Put the word dictionary into settings - settings.word_dict = dictionary - - # setting.input_types specifies what the data types the data provider - # generates. - settings.input_types = [ - # The first input is a sparse_binary_vector, - # which means each dimension of the vector is either 0 or 1. It is the - # bag-of-words (BOW) representation of the texts. - sparse_binary_vector(len(dictionary)), - # The second input is an integer. It represents the category id of the - # sample. 2 means there are two labels in the dataset. - # (1 for positive and 0 for negative) - integer_value(2)] - -# Delaring a data provider. It has an initializer 'data_initialzer'. -# It will cache the generated data of the first pass in memory, so that -# during later pass, no on-the-fly data generation will be needed. -# `setting` is the same object used by initializer() -# `file_name` is the name of a file listed train_list or test_list file given -# to define_py_data_sources2(). See trainer_config.lr.py. -@provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) -def process(settings, file_name): - # Open the input data file. - with open(file_name, 'r') as f: - # Read each line. - for line in f: - # Each line contains the label and text of the comment, separated by \t. - label, comment = line.strip().split('\t') - - # Split the words into a list. - words = comment.split() - - # convert the words into a list of ids by looking them up in word_dict. - word_vector = [settings.word_dict.get(w, UNK_IDX) for w in words] - - # Return the features for the current comment. The first is a list - # of ids representing a 0-1 binary sparse vector of the text, - # the second is the integer id of the label. - yield word_vector, int(label) -``` - -### Define Python Data Provider in Configuration files. -You need to add a data provider definition `define_py_data_sources2` in our network configuration. This definition specifies: - -- The path of the training and testing data (`data/train.list`, `data/test.list`). -- The location of the data provider file (`dataprovider_bow`). -- The function to call to get data. (`process`). -- Additional arguments or data. Here it passes the path of word dictionary. - -```python -from paddle.trainer_config_helpers import * - -file = "data/dict.txt" -word_dict = dict() -with open(dict_file, 'r') as f: - for i, line in enumerate(f): - w = line.strip().split()[0] - word_dict[w] = i -# define the data sources for the model. -# We need to use different process for training and prediction. -# For training, the input data includes both word IDs and labels. -# For prediction, the input data only includs word Ids. -define_py_data_sources2(train_list='data/train.list', - test_list='data/test.list', - module="dataprovider_bow", - obj="process", - args={"dictionary": word_dict}) -``` -You can refer to the following link for more detailed examples and data formats: PyDataProvider2. - -## Network Architecture -We will describe four kinds of network architectures in this section. -
![](./src/PipelineNetwork_en.jpg)
- -First, you will build a logistic regression model. Later, you will also get chance to build other more powerful network architectures. -For more detailed documentation, you could refer to: layer documentation. All configuration files are in `demo/quick_start` directory. - -### Logistic Regression -The architecture is illustrated in the following picture: -
![](./src/NetLR_en.png)
- -- You need define the data for text features. The size of the data layer is the number of words in the dictionary. - -```python -word = data_layer(name="word", size=voc_dim) -``` - -- You also need to define the category id for each example. The size of the data layer is the number of labels. - -```python -label = data_layer(name="label", size=label_dim) -``` - -- It uses logistic regression model to classify the vector, and it will output the classification error during training. - - Each layer has an *input* argument that specifies its input layer. Some layers can have multiple input layers. You can use a list of the input layers as input in that case. - - *size* for each layer means the number of neurons of the layer. - - *act_type* means activation function applied to the output of each neuron independently. - - Some layers can have additional special inputs. For example, `classification_cost` needs ground truth label as input to compute classification loss and error. -```python -# Define a fully connected layer with logistic activation (also called softmax activation). -output = fc_layer(input=word, - size=label_dim, - act_type=SoftmaxActivation()) -# Define cross-entropy classification loss and error. -classification_cost(input=output, label=label) -``` - -Performance summary: You can refer to the training and testing scripts later. In order to compare different network architectures, the model complexity and test classification error are listed in the following table: - - -
- - - - - - - - - - - - - - - - - -
Network nameNumber of parametersTest error
Logistic regression252 KB8.652%
- -
- -### Word Embedding Model -In order to use the word embedding model, you need to change the data provider a little bit to make the input words as a sequence of word IDs. The revised data provider `dataprovider_emb.py` is listed below. You only need to change initializer() for the type of the first input. It is changed from sparse_binary_vector to sequence of intergers. process() remains the same. This data provider can also be used for later sequence models. - -```python -def initializer(settings, dictionary, **kwargs): - # Put the word dictionary into settings - settings.word_dict = dictionary - settings.input_types = [ - # Define the type of the first input as a sequence of integers. - integer_value_sequence(len(dictionary)), - # Define the second input for label id - integer_value(2)] - -@provider(init_hook=initializer) -def process(settings, file_name): - ... - # omitted, it is same as the data provider for LR model -``` - -This model is very similar to the framework of logistic regression, but it uses word embedding vectors instead of a sparse vectors to represent words. -
![](./src/NetContinuous_en.png)
- -- It can look up the dense word embedding vector in the dictionary (its words embedding vector is `word_dim`). The input is a sequence of N words, the output is N word_dim dimensional vectors. - -```python -emb = embedding_layer(input=word, dim=word_dim) -``` - -- It averages all the word embedding in a sentence to get its sentence representation. - -```python -avg = pooling_layer(input=emb, pooling_type=AvgPooling()) -``` - -The other parts of the model are the same as logistic regression network. - -The performance is summarized in the following table: - - -
- - - - - - - - - - - - - - - - - -
Network nameNumber of parametersTest error
Word embedding model15 MB8.484%
-
-
- -### Convolutional Neural Network Model -Convolutional neural network converts a sequence of word embeddings into a sentence representation using temporal convolutions. You will transform the fully connected layer of the word embedding model to 3 new sub-steps. -
![](./src/NetConv_en.png)
- - -Text convolution has 3 steps: -1. Get K nearest neighbor context of each word in a sentence, stack them into a 2D vector representation. -2. Apply temporal convolution to this representation to produce a new hidden_dim dimensional vector. -3. Apply max-pooling to the new vectors at all the time steps in a sentence to get a sentence representation. - -```python -# context_len means convolution kernel size. -# context_start means the start of the convolution. It can be negative. In that case, zero padding is applied. -text_conv = sequence_conv_pool(input=emb, - context_start=k, - context_len=2 * k + 1) -``` - -The performance is summarized in the following table: - - -
- - - - - - - - - - - - - - - - - -
Network nameNumber of parametersTest error
Convolutional model16 MB5.628%
-
- -### Recurrent Model -
![](./src/NetRNN_en.png)
- -You can use Recurrent neural network as our time sequence model, including simple RNN model, GRU model, and LSTM model。 - -- GRU model can be specified via: - -```python -gru = simple_gru(input=emb, size=gru_size) -``` - -- LSTM model can be specified via: - -```python -lstm = simple_lstm(input=emb, size=lstm_size) -``` - -You can use single layer LSTM model with Dropout for our text classification problem. The performance is summarized in the following table: - - -
- - - - - - - - - - - - - - - - - -
Network nameNumber of parametersTest error
Recurrent model16 MB4.812%
- -
- -## Optimization Algorithm -Optimization algorithms include Momentum, RMSProp, AdaDelta, AdaGrad, Adam, and Adamax. You can use Adam optimization method here, with L2 regularization and gradient clipping, because Adam has been proved to work very well for training recurrent neural network. - -```python -settings(batch_size=128, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) -``` - -## Training Model -After completing data preparation and network architecture specification, you will run the training script. -
![](./src/PipelineTrain_en.png)
- -Training script: our training script is in `train.sh` file. The training arguments are listed below: - -```bash -paddle train \ ---config=trainer_config.py \ ---log_period=20 \ ---save_dir=./output \ ---num_passes=15 \ ---use_gpu=false -``` - -We do not provide examples on how to train on clusters here. If you want to train on clusters, please follow the distributed training documentation or other demos for more details. - -## Inference -You can use the trained model to perform prediction on the dataset with no labels. You can also evaluate the model on dataset with labels to obtain its test accuracy. -
![](./src/PipelineTest_en.png)
- -The test script is listed below. PaddlePaddle can evaluate a model on the data with labels specified in `test.list`. - -```bash -paddle train \ ---config=trainer_config.lstm.py \ ---use_gpu=false \ ---job=test \ ---init_model_path=./output/pass-0000x -``` - -We will give an example of performing prediction using Recurrent model on a dataset with no labels. You can refer to Python Prediction API tutorial,or other demo for the prediction process using Python. You can also use the following script for inference or evaluation. - -inference script (predict.sh): - -```bash -model="output/pass-00003" -paddle train \ - --config=trainer_config.lstm.py \ - --use_gpu=false \ - --job=test \ - --init_model_path=$model \ - --config_args=is_predict=1 \ - --predict_output_dir=. \ - -mv rank-00000 result.txt -``` -User can choose the best model base on the training log instead of model `output/pass-00003`. There are several differences between training and inference network configurations. -- You do not need labels during inference. -- Outputs need to be specified to the classification probability layer (the output of softmax layer), or the id of maximum probability (`max_id` layer). An example to output the id and probability is given in the code snippet. -- batch_size = 1. -- You need to specify the location of `test_list` in the test data. - -The results in `result.txt` is as follows, each line is one sample. - -``` -predicted_label_id;probability_of_label_0 probability_of_label_1 # the first sample -predicted_label_id;probability_of_label_0 probability_of_label_1 # the second sample -``` - - -```python -is_predict = get_config_arg('is_predict', bool, False) -trn = 'data/train.list' if not is_predict else None -tst = 'data/test.list' if not is_predict else 'data/pred.list' -obj = 'process' if not is_predict else 'process_pre' -batch_size = 128 if not is_predict else 1 -if is_predict: - maxid = maxid_layer(output) - outputs([maxid,output]) -else: - label = data_layer(name="label", size=2) - cls = classification_cost(input=output, label=label) outputs(cls) -``` - -## Summary -The scripts of data downloading, network configurations, and training scrips are in `/demo/quick_start`. The following table summarizes the performance of our network architecture on Amazon-Elec dataset(25k): - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Network nameNumber of parametersError rateConfiguration file name
Logistic regression model(BOW) 252KB 8.652%trainer_config.lr.py
Word embedding 15MB 8.484%trainer_config.emb.py
Convolution model 16MB 5.628%trainer_config.cnn.py
Time sequence model 16MB 4.812%trainer_config.lstm.py
-
-
- -## Appendix -### Command Line Argument - -* \--config:network architecture path. -* \--save_dir:model save directory. -* \--log_period:the logging period per batch. -* \--num_passes:number of training passes. One pass means the training would go over the whole training dataset once. -* \--config_args:Other configuration arguments. -* \--init_model_path:The path of the initial model parameter. - -By default, the trainer will save model every pass. You can also specify `saving_period_by_batches` to set the frequency of batch saving. You can use `show_parameter_stats_period` to print the statistics of the parameters, which are very useful for tuning parameters. Other command line arguments can be found in command line argument documentation。 - -### Log - -``` -TrainerInternal.cpp:160] Batch=20 samples=2560 AvgCost=0.628761 CurrentCost=0.628761 Eval: classification_error_evaluator=0.304297 CurrentEval: classification_error_evaluator=0.304297 -``` -During model training, you will see the log like the examples above: -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameExplanation
Batch=20 You have trained 20 batches.
samples=2560 You have trained 2560 examples.
AvgCost The average cost from the first batch to the current batch.
CurrentCost the average cost of the last log_period batches
Eval: classification_error_evaluator The average classification error from the first batch to the current batch.
CurrentEval: classification_error_evaluator The average error rate of the last log_period batches
-
-
diff --git a/doc/v1_api_tutorials/quick_start/src/NetContinuous_cn.jpg b/doc/v1_api_tutorials/quick_start/src/NetContinuous_cn.jpg deleted file mode 100755 index b18e452a481232f47c453fc30521f8c78ad2da0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35863 zcmeFZ2V7HKmp2-UpdupD5r_y#6OmpeBGLt(QUZ~Zko^4l+lTBF+0PFJIXT%WN(xHKpV#R#RA)|~K6{#y@+|e)vsCAR z59OKjG}PzL|GfV>$j|OSdjUV^PE(%#+2TL3gQoc7d3P7y^4I$dkZ zD<1Ts&*C#rbKNfe%%Ing<`%Pg9CGF?BNHKgh6 zhDOFFre?Nw_708@oSZ$MczOHy`uRT(eep6ZJR&k7@pV#i%A2>TS=l+cdHEj-3d_nX zDyyn%YU`R?THD$?I=i|D28V`6M!)|Uo1L3qSX^3OSzW{I?C$L!9Ab}-f6j{pME2*j z{yww6%!?M77b(C4a>}3cA|dqwE;3s3Q=B3c7Zr6Wtv%?jh(0?_e>*<2^z#`mF+DVc z&Ex*FjNIb0JeZ$T`(tMRv5AHJKbqO!C-%?t8UvjpBLSF4Mhk*~@PILp2W@WCZ0{-R(m zGS`cTfq}`DX(8^EUNrJ*9qP+GY*uX*x_1#HML2(z2x5i&{C@gY)~pH{`*)`j6hrESsq;8j`0kAG7{bx4*plFOM6|2{|(}UgGrk zpMCUBeCqL8Jiz%~%Xg*Hc35-#0BVJ2&K<|5MG*h(N#aK#qu!^BOa)}mlvKJ4+tjE# zxgR|^7?ukIW)%JV6#m7Gvf^CzYXVMtOHX6 zp7Rt}~ z1s^cMi6Hxu5KXLg5R&`BXU$h%NycR1@Ar|sXI^FtSPWlD8-*G9GZ!nH40}9x5)4*b zCW2-Sou+p&ABdnC{eB|oY>_q*1dmZAf=Dwe;-V|!6TEK9ll0*NGdtFe@bK@V4ShXc_B52BPUsoBEm`o#3V1)~D zb{NVsVT6iu(#^Z1Tr$6T!{1Y9%@?|G2k%IW`?8Gsx5=xHMLg6w#zMAdfGwH)bxW|) zAyRGOp2i~cjm|acypO)d2N!~z>!$85L;k@aGeetOC3!EnV{)rIQlB+F*bx<#vokDY zq&jyHLW3|i?@AXpqYq}rKF@7&k?4mA+aZKTxaXPS7w4BZ4g4AW2>> z-}FAKhklE-4AtFf8odq_t3pmKP}9!8(wGYPN>fW*YrC6Nbas^+U-XDX$o+hlRdt1(7WQ_h$qt6Oq~ z+zvsA@@4W-K|6B7_2eEP#Qb8UMl1UWe1{@=UqTK1Nbj*U^=T z&DR-PZ4F~>by{RIlB2hO7`5c^j@O{1dkR)CIvIGBXP4_tT5`mc)HHKvC5FBMqE}Th$F>Hd5A}v{%fA#c@?3_hj7vfE}DO z1WUgMYc3o_Ezj!`L7(zV@*+pqm|+VlFmohsWAPsdZq>F#kEH~U)ACEs)B2Z%x+$c| zV?U)Pv#U@=McAm@`ek;!iTqS%XJ_Xa9T`_J>+W_(BBIp{0E2%7E!D!6X|8(6{M*I01wlAy@*8JL~@|GK}gg8xH~S z)%jw-jtKht)g(}0u$+MUV{-X{{PFPSGmQ(cat7XCZ;K^Mf9#T90-c56A|VI6QiPW- zU~BEr$5K9F06%dfrW&5cFhUtO&5|E=03=;05e~TOw$fCVx+2k}Kkgvh@hMMKS}gy7Cz!lbw5D@Y+|KKe4r*2p;$f{Gw3?tmJ{&QOY zX1m~Rz3S2j$)>i^UkMfgX>JFf4|;S6#^?UqNP3CS7k*hGnRI*Z1JRL2Y>Q!|~kFJ$s3#?FnW9s^>+$2FG8r zv{WG#5J@-czFTaN=HY&1yg;w3?8Ek1nf@NhFBfI+)FmP)p%w!sw73vKS;|i;dwl`u zUT%e*P$LNu3nvqz}j7nHg528@BIoWKV|PlV}ztk79nO|MPe#3D<4 zVOq6A5+@%vuelh!bL(BE zug_d7;aq(+>;7h2+wca**->}o=GM1vYhT-17ei&Ta|gYA-4#NXea+K9rus}vF}A+t zs@@q;c4yX+gzGfNp-~o524!{8=T+9DH2EVFt8~gf*J^}Z;%bMK_oBjJ&ZrlxeJ>k? zOOnD2qzK$2Bfi>j_R8e2IXN}EH{u~q!pn6S(4;_nyS>r%Pf8{=Vy;vzP)u&$CRdS_ zlftPwSp=fE&aZyNeKYngMI!j!0hjwhKm?EJ_I8eGj_94}s^oJBaBA{Wu>OG~hNrE% z-I90K=*2uj`=DhuD0df|xVdi4(-BjmRB>x8{!Fr3c80o}utVf+*R8h30&n+xDv=_dlabUQg33&BsnxTGF2PslEDltI zi3@``3PKpLU{jPk#U)E66P*iFEd@WEQeLe{(QNX*BC z#kp^zoT=2r5tOY=$4hZ6-^7bFzq^ZXimJJ#4y+rEaMy^uakH*|A;cNp(fFdL=i%Ke zGqU$88%7Z@ClrhGNL>t~&UuOvUGmPU0M<-ZFlE+zV_0HZRKlQ!Y{5MIZXC)^Q(jN) zHmB{#3I{2w7N3JtE`eFQRCzPS*;1{IA!Dwyc5g{rpE{MC=(frHlqtn$ET*1i%7?bB zX=xXF#uwVb1S@xM9@j+B%%v+_4KqW3Ii8&C3kS5e?)f4Yp4kjYHOn0p`b2+u3(gjh zK-$M{a5cC+6l;BAL+E>?gIMt=)h6)NGNq;)iy0lh z4baC_Rilk(kej^o_^b0;?w6eC);KtaW`>_c6-AkAc0Mt89IU{I_FdFsW7ND~9t2<5 z&6r=Bu&UFEfu8CMCxTpQo=@=*nrlX{d6byB9MYNLv$_Xbo#>R@T_Kkkc~h4HDHg}( z6uVy-&Jsb+M9@>KE9k>d*|WW*Cvz#Wl_+|XwG3v8sU9uXpuOXv6^Ghkk z89EBHa}fyxpwi2!>hf?tdzNSS49h4j9;+d?*_{qq-NVE&^g7qf>=d4q^o3GLTq3zQ zUvz)L4%c8T3ae9Io($I-sWjU1L^5MIStw?PKFh+AW=Zs6> zNO^bJMDslr+fdb4p?u)_?)?qLB39&!>v`xW-Zy%hf-o{-3N!Vlnjqg9+hQG{O3xbc;k(xC^Cs8=uI0gvw0DNtO8ZSqzH9X!`PfAMcFK zUTzGCuutrvw2}5+@$6jhvj`aOpwZRdE*JBSw5YNi-7k%wt!-I5PLHA9>$t!#xixu5 zFyXs(MrG#AYIMf6k)fbmT}DuWT&ew$wlDX*grgc7@iocW=+c5}Qgk9L`=!xClARFW zdzl6_%JjfGt@yAyG!|G{=&%t%4{6B->UFwv*kYd}5l$-6b2F(ws*m0dAH?g>{rC{lxjzh;0uZO5 z$L{BFXE9dqQx(!m#!Xk55+;8@C&X_|+^X7PQmT@=#(a+9NRb@Z+(_+zcCksw_Zm(I zYdWVxo!k9Gp)q%2b^THQQ4~q?)o2-um`{vfMa9-+N3-9k%WymY zWHvls0ChG2mAgM~Sr3Nb?hI8>L-d;>wZKM6Pc8|aA7wPV9wjfyA80f-C6S62A8>u7 z&V8R$S|Wc23z-FBnNAe@GB*Ys2fSR}%+-amO`VoW9`L=|%7BlCAdFCKOaM;90O$gw@| z1tT4Euy*Npem#Ss3cVNGxuf+#vPA`rMiJSO!-4j;voUh*8JC~8m>uLpo>jQ;OPC+Y zyL>R8uo4`L`pkA;>`{})Oj&d_$5a^F%p5N_r`qNs=T}KJo43YDW$#qrJjz7-3940{ z6mO*?60cc*(SR*6opmtnzzA*Cn6<`_@b%Of@}jw5`q{NmOY(?SiJ|OMlewMvErDkz zh%l6%mLl5t^Atb2seXRbrB(8nd8k48!{?jnj7)kBG;EBI#aNz)OjsNcr1JtJO+63G=Ow zupwI#7=!pvJ7-2rEp^3b$aBal>m|>uT0KYW%)-M7^y|mb`bsW7*)Dn#n-jb1(=PRq zD_1VtD-X^(`4oO|OtvxZxtGm?6uj2s@ud8eY-wOYy{VIr_d(>*@w=l8>z75(JaR_G zjC}mZBsGgh1N^y0jkh9q;Nv<3t+=-7{Y@r9Tx_Fh@CuS258(8|cbp{=WYrH_!LF}i zqX^0-a6p>yJQ*}Ju+C8`A_&S!1m$uKiqjmCn8HTMiJ))F=7p)0xTnRa^$^&~6~rh) zbH*wx2@;WB-3G`gZ3%2W2X-vtQeYKQ>mEe}?Qj7AJE}$m?b(d#HJ|OgdYwOT`az>!d2T2PEXL zPLo%Ei>#jMHJ2UdHhU3IVZ56i(B>%v+3EKIr2N3jP}kQH9i z%CQ(jKS?F%ie0l_uU=NeMn=ZjlV_L<_Aor-yc_+-xMjzqdTF!9qxX7RwizlcP~K+n zwPlBwn}dg)VHiZx++C-=8YwT~X^Z&mlg~{ao;Nknsp!Pbr^K-kn7>2zOuGgx_x*e8 z2%W8NEL46%bHu?$Dy#$jah{!mX&|-PVEJ`wKWg}rkNt8+|f&J`3_HUrM@81+T4ujvZ)aYo_UFH=VE$TE$` zO6dec)H*jP5&qPkQ&-UR!B>W^+b6HtX{PyZc3)<0E*IgI`RZfKUaB9HE%5rt7XM&$ z&FXxG)6i1NejzL~0#F~$buXg9Zpr)z2~ILHx2LUB`bqcw&YgXwI+{9&$q}S(iehL@ zv?!>j5)`;N5GQaq;e$dtiBTn9^iXe9yhFK~(Yu->pwxz&()OG!v&`gryFfq;KT=-F z>Cwp@Mv-#i!wi*6o_l1F!_i95_Rvk~ZEw_Ve6c%=tKGp-dT=H~`P$M5^Va)m##t|( z=jAWR-6+H^*mlh}Ohda9C| z@f3XuRWpTq7A&|jw@>S*PAXP{+HVLx6ryD5cHdc1xmx(z_S85^_d!p;rDz1rT!91VTJl34BBlDjpWYmf^AuU1IC$+k!1T8M3K`|7A47 zMJN-jUkam&$3@{D{7K7=B`X-PkX(6L|E10UVp>1=3*|Q0b!|*aE49G5R@^5J&N87O zS2-=QJG!JA#h%Ysjn(;j)w_U|__9|2IR`$`R|GKjC~@j6U)EC@XxQ{RPLT+*b|Qif z%+`wHp=8cg#)suj$C&ci1Miv#y$R|s-5$M`hy&qRySthH3vQ_hb$t+*cv1)m5Rp$&#iJ!XJ7GCc9 zC^SMk4D2o9Q&w%Yx&P5P9*FC8VVqzGJiUNoi?XV>GQ^nASk=z;MHpELpbw+tUU*Sy ziLb_L$FXA*#Q0yqr*2Sv6>Ilh6LoohGuO zrcW4d1#~LAPE!gIBqV$yObx`aUlZ1*aUTxO3U&a^v;&O*G5i8U$Cnd9b02^(lMKe3 z5G6tcHLLUiwng@dJ{=Ks%S+)HimQN>p{Ms)bhe6!pzgT)SwG*u{PX?5KxZyN2m*0f zeK-SZr5udoen0aFu@G6(IADV&_5{zbbj`C!ET5vKqGV8-y^w(sK3mmh^rk{uG%Ub z&cR&YzaJNswSBkgRw7_ToJJG}e7fJVq&3{CZ-%Vc)iDD{uT4Y-nXR%hE7`1`*Ad-82>tKy#AC4$6F3K!VXT2O@!=-!+*+@M2D2D z*P;X^_rTbs@MLyYy3^Nao29J5fC8^-yAyUxhVH)z9T**hO(eWbCW1cO!R8Hr5y6(V zJ%H+s;8kGND2zw*UV%{VY(;up@fd=8`-9+#qE35{4H~ z4P;+JI*Fk6N_qC##92#bvx>RJ=M^!RwbCMFRudz`yK~JjiO(a|w|hn=(>4r-rSH8q z7)o5bmt4l;{&^Q5Rv~2Bxvs;&2a$piuW#_?~$6p9nO%4yYEm-U$%R z>5AX?hwUCfo&eS@Sy5K44ps%HP6R1w!e+`KK%(i_QI017W)3xBVLR6UH#&i!M}m)G z-vO6=Wo>Q!ZNK~_zx?$9XQ?sd_@ayFLr8xz8};XUHZRKR;K~7{q7EesUstc$X4AW% z>!^z!*X6sEr@F4A5HGSK1wStT)KR})T(z~D(yZ$2Trm`;@2Z|ERQ6fobO$!mLz(EF z;dDE8;BDb0wsXAU(}u%S!zNMYDr59g(;h?c%$XoT_@{%CV`&M#-MjNe;Z9?e?_teT zK&oUEuu)M?01WC9K~w=hQ>~Ut1bjXb#9#$`w!-&mj{!O4bo4gJAwPZvjIIWMFH6QcA9j}28Hz5F)|JJ>z4}J3# zGe=0=hrwN%;-Q_JWCX?oAd8|<7luF4`Kbwh?YoPkQBoIYgw0&j{}lsFHk@> z;o9<8IWByIz8-e~Mf1^F}znS*m-u+)s`)UWl(u?Ng0FE>CBLK?? ztj#`au=W1iBaUal&VK%LXS@Envzb(J*Qp_dpvpg*TmIHOHA58;j2VE}%j5qf8T~)| z>-^IHZ<0YDr1H1D4o24iitp&)Psw=l*8xW1GyZ39+&l%=&0hkW7A*Kv{zky>y}g#P z93jy(imHIvg^f7-{Yn0>uUEwh;iUm<;r|?G3IC~G`QOahUj*`hU!wj>`>!+qk4V%{ ziTsD_e*9mD27jgfS1kFD(EeNh|7|q*W97#Fqe%AKyZ`Ia;6Jfv5GJ_01Kainjsf1b z&zABim(Gx3kP-mQduOj1;Mz+?*KNnL-rC>mHhdi&a$Yg3M)(DRD?x5Bud^-d6=N8$ z9P?v?0NTXZ?^zAU73nEm>wxS;BW7k#YC?g99=XDMf@4~AF+0tI4O7KWAwd8HU* zNX?j^DtOj+GGuykpN(O8yy?jddN&(*l)H@rGtivAER9EPD! z2NFSxvw)%gD=BeU4e_{RDkmtG5k7Cie{vRg^>rYM=E?1V;LV>N)>0Eka4(pga3xRy zy&OJ=4ai;{Y2vSnTjae^M=Bj|zv4G01Sj)BSeEb8nu`H~_?)h0{G?!)y#35x(J|A;GFrHeLpDuuu;EPRRD-Mq>!*gN#mNau*LFV@mXsxY z;h4Jn^oI1Z42e>VC%<^MMr@= z{MX?-wJHt@PaP9CjWv>_FNerJA2MbLI-$f}9&*iKktfR*P|MXxwvT96bucCeXO`&C zTr&o?4?)c zHgjQIvS{c;OYFsv(2)@F146I3k_QDud8>bF1>a<1@>N*V$9_pbG4Ln z^3I4p0F3RX8SKZbNIua2#fx1&kiv2pGGUN~nwOIW&l@=AZak6S1niG2F ziY|GR4WrI}Pi24mUPl@BA%-v@J3sCVGxMqV_9Wo5!$=c*wDgHv_KwWX55w~VWXcf{ zh@QOq?C#=*@%9nWH( zVwXZIvld5*PuyZHbQ+j^3u`kc9Z=BjNJDBZrNR}%iRy%uE=ARC)RdS0SBo2%_t_O} z!AHA5{l;*dC_ZyfNUVdUpCZ&DVD6Pm-5x8IRbSqitq_}v^yHVFtGeDHSBxFGX)kIn z0}0NFgA7`9eu#|%Rbx~6$X>vinNcAeV9;Us>s`Dqc3jK5+;=YaT-2l7VKXBm`&&tS zR}u5(mb~p_kgzGil%|nQVd49BcUw!xnHjY@WXRLrd;+m8?6ukZ(&>Mb=g(frVC{dh z`S@%1*U5*)nTfZD^!RKdAxUOVHqEofmvRCVpCzc#8}?*h`)re>Rv_46IGx<(GVDeW zte<#NPRT~r{8+s}f(L2OJvDh?ZfZ6@4TO?hXu`Y+oc)#FL-M>5tz#3@-;dcCM~=Oa z<489#de0uI)ZNZb(>$HY7jZpPD^vHLtTvs+s;b6yFdlb=?opkvQ!~0yeZOw5?U6UZ zOXkL%uta%}RWg*}7TbmAAdWbF4C-khKPG=eHY3}fG73^o_`y~eGjbThV#mMO>;RkG ze)ux>GRY;MbFPI`lo(ixa0L9Kc4JOem{jx?zrFb@4KTO(`wq8F4WlkF30+cu`F$pj z?EY>s2Y7vI*dPinf$$2F@c*%&(&Ac-%*-DVp&~EPxVTWc?da~uU4eQqsc|^@eOH*l z*V~;E=YmlUzdRTyQh4<(?0uNJklT&a%B|cOg7xcSW;Z_Hf@cqnPA2B#6Usx=BI(yx|`AP@1tS&Dug(eB4-lr z{@}f~7~$HRU?Rx&idrix@MeF5)Lf9UMd_gHW~Hkumu64+XMO40izFV@EAk{rg(b12o3W$*kjmt!S27z5#Bdi4?iJCAs++bPP1yFBN<3 z@rJ+sx|X4t5aG;W8BPa6b-ITtDmo|Y9dFV2e3(o57+;CFuW-96!4Ys&Lh&fTb=L#} zc+hCnN)Qys`-cj|^R5pA5eZgU5QJb7NIAp@`*ufR4UT*Ez2CwI#~CaN1i1ELXgE+6 z@YCxJ6>f*_DIf`S2SgCg5P}2{_`MT;4{&kr2i*?ClPhf%09A?XF$^8BW~O=JV=&O* z%n9tLOM4q<4+Lh`01X6j{~1--2X6PMAx(e{aE;~z;8ML04;qFXk(FlEZ2ybKeHv5AvbVS~`(K+vW~neKcp4z;`sWg;jsmye#0keR+((5> zB|-@2p#czXKi?iXkNRuVzb?S>KJ1tRJ`MO>$^Xfv0$U-0K=9xRV4>`MI(|WOd^&7W zAE#~B;FR(p?Fn}-3!P7sZ$iJ`%d=K#26)&6=!geTal|82a+LUks#RB_w zjKb=S|BPt|`J?b;I@`e7^pLI5LsqVUy>tk-HXfA7{iWew+4Vo1{f`R4|9?2UM35hB zaGaSYUH5MdmWrLTQ0LJ0P`J9;7jFFz=05b~}ns4FLE_p&$s`=9`1Q+|eF-`Yn!ts`#N3=_9)>@PAr6< zEeFYQdtvkIml?KmgV}FnMhyh?USHDIz}0=I$wzmao%96ILP43TQ9vF(T*^U9>;$ z#w$RBS-VUA@B zWAl~_?`4W)@K|4)3g;4z0kiQqmc=%8Oc|^)A$k)n22GgDmp?B@ckT|4Q}0)y+j@G@ zi1J8pr%CaR}Z{|XWGvK#}!)4GR44krO~fJqmRRH^_RZc81)9ppx_>LCnm2RY(O z$L?NIJkeV@PBV4#_SX)Z#j6|y^hD`M29(o}aCkhmE*JOcA$gi~f6z^Z<;A7sZcsXgzBRchYGlW&Rf#!n?Q!4D!cKF<~ zRgHOCSBZPVn!4q^D=`CK+6)bE8CVGHJ{5yXZk6gGv^_QSmt1}f|4*JXUxUavwoLFDljbYo$k&z$Bv!+p)DARCNL zsPZG%-0E1}i&TCL<{rC@%-^}yOYOB2XOa%;J?HT*eP|*mjQydvum85iU?qR6E5m!I z%b~ZwaqGz<+D8)D-z)5K8cBa%l5meXtlRHOJ(+>FYIaGq zn$#JAng@AL?)HD6+sO1%{=i+Fw^X&+`CLxT|OR3Vb#4pwT zo1EmT50PIF7Ep?P&t5I)%g}1pRp;w+=Pns{;%*o(Q}JzMV5kGV-v+)G8uOlh~m|7gghIF#6Q5*00DR)83TCsf2MHw-jn^A zKgoJ}?vKn%RwU@Zy$M+gcD(Pi4l{7V?(88@x^vTg3^HcUfhlUv$G*N zm_+Z(ArY~a-@b&gI? z02Pdm1c%ImUmr?3aeS>nInVn)IGFp(5u0bLD!_@E-;Uh;di(~sxATt#@nv%j>Bg@C z!Jg6U?A|63Is>A&(!IS!xhAfO3eErh=3 zglm6iul^h)LV||kf$qAbu264D;-hNWcXh3O;>ovY#gr?SzeLmWF}u!Py(Kym%XDCY zNhgS)+tOL>l6QT5l&e$EYz?_f84p;9KeM^-QO2i{eECA}qxWpBpKu4fBDv@0V%ewP zHhWuct=1#C6H)o8sBhL~$7S8la}2cxvp!?pS>?N~IgNpF=2_1M^KLfsAu}uEGVTHl zp`h1I{wMd0oMiDLgw;G$0Aey~*)Ix^N5-;g|@OAO&VM$j-;gZHXgB;q#?foYq~7-0K># zv=#%p(Kg8uSChu;-My#n67c5KhBG&O{e2Cxd$mOq7o#4(M%Fvk^VlVK36+co0~J{3 z42dAafcX>xJqCt}gv>^w){udZ0sALIn4pSud%yRA(DE532$&?f1bdjPE#xh12H1sM z%RxQ|ZUYT+bC)aymglxN$qx^!b}U<{OGYy=ibwX+(q?N7G*wF%SU&JD<0 zO5`6vTeruIcE;A6?z@zK&^*;xf5+0KzUKk_`kwtMA5t{}U0oH*pdzEXY|%J4ShZ0lk8{?D z_nlRrw#lBcrNolAmHC=w)PZ?f+8lWkYuwy)NA-~P;QY|S_#EFCj}~quy`MYROUKx| z{A{7N-v{MSorszSs(f1*dIBWwVH_jp`CfwL!sXb9l zd%ag%<}-4g{PUYrM~ZwD5UxI?ZLS_y;Z8Zdi%~8iEa!HnHLs2PJ-p>}l}Tb0C)B?* zNZ`bYsxJ<|kA+fAOsp=+t-HoKu^D|3+OmDw{9PLC5-~^~`k?H_nMrXD4wDG|BsspG zq?4VdGF?kIM8GD3V1v|WI7?f=n({a@;m{U`Xi8l}WUnq+045k9%Y}`Y9w6}Ti#pBF7==>-mDB8ICtl%o@BO2Nbm}UI1x3j-+zoi6bQxqDW291WY&exEMR7_ot z^!>b9E1xuRQ$@v@<{408&t@ft5q>J~=;8DM8`m6s2kl&i4%as=dlS6n$vn%hHa1zN zs-LW|)$-hm+66cOel#z9Lbpc*;raYW$<*xuT|8@%P@tfG-C-~)G^4jsU!&KwZXwrboP?+LRTm@hjS^`gc@e^7Bqy^v~Kwki9 zK|j$qn``9h*%a-H6Z?UV1d@gLA)4LDNTja|;kFXK1uqC^lDQq7%f8|MGUS~}+rGnu z?5PSa4m0@UY)FX{pg>26pxrLmb`r2?wUG}KP7ftt2A$!~0I{WFWay@dpa3FBZ15mq z=OpEb0aA)UEhi8`(}DzgnC&>tksay~Sj`uv6>r^zY*Vi_=#?x=b6tswNci!1N0oVX zw`l^dk+gB-S6N*|1c{~>^7f4O!{W_PIK~Nn!uTE3tR_ANht)RE=?%JRVwEo?RJjLT zP{zmoeS};>JD`^THncvX(#5 z`g5wzOeq#BK3unu`4pzm>z#KkyRzBpWw zI&3Lu>Ef^vWPG|m_nM_f<^9}^S$^)}QOS6%{OoFpjlDW!wrD$q zwvCl;;>FH+g|~p-{7uZTZ2`{dA9Cd*v8WAs@D2)h7K)4CpC0$^JxU>vfp-)RZ(|;H8_4GvIwCTmvO*GFL!`#?tVlr-B^w4;3vk0}?*rdd;UP<>frz2fCpL zEyfxIJ?P7n9?{E7(N-`G@T>sBUaYf|G$Zne<-+4H4nrCa?C_Ac+M+iAwm25D@F6?-4|e2H%_ zF5*UB8S{G(ZmD!}f-bJH%><451g7$x)@4i;-aLiTG_Ac6RIF@0pJx76KKjG1cMkVP z58q_8-80iE=>eS5ocmaYW}_?%s*J8>)M0FD9XcfBc0Dt8XLM7n~jAfmadDtt7R~obXczyb^)^G2q=d5H97eA zsx3ZE6Pzu!pL$^T(xgl2L^I-hMX=sB$M{Q^i7?V=;~|*!O!XoROON^j47}OkxGRjK zfGqeLf70Q=zMO|*q(bCvg({~boferEzpt4dR6ZWv@|j@ zo&Rf(-Lkf5Qp%n2?|>C|oV|4W7h4qg@`>q97L2AL)^oP;YDbIrL3OYo#x0*CDo3dA zIGpFz#lcQW9$FQ(eZdSdt8fyx^wHNiZ=kYt6Rk>U2b92G{Q9zQIfFS(l3_tu%C)pm zee)8(c50+}&gQba(u%J!l_qVf{tMq-W&W=G@`cd6#2q(*#Wb4lJx29arV^-V6w`Gt zMwYjGy?R2jL)AgAPETAuphPo3a3aQd!|hloRv&<}cERymt{T ztcuRdsWTNA_OI_ePHqKSR7oD7Cxj-N#462YTFb`NA;L@r2#i zW%g@sj<3<^9K1c|+}aiI!Hvzz&Y7zC#KVL_*UHn#m!huRBKJBdZEdQb3kTXwv+lcy zW0+pPO%{K7D4&O7Tdk;r%gFE11EV;}Z5{ z79jv-hB~D>M(nxl9(bIJRTySFesb@c`FjTk$b;9LHn^Fmq@zipnJ)?`!wb~&S$6V$ z^2N-cV^H(9C)e)>o^V!1O=;tXZwBxXk)C3N=$zQT4OhM zeS;5A6wr_eTFJZjUp^v{B!X^;^s#hqsco9&AN*={VcSfxg0s}Obs`|N_CV^Uyp=?p z2x3vOSB}$Ze@~H=FUbS3=O$(5--&$moNWPar&xJwk6~@`$c?e_y5FGFBr?*H1MajZ zk8^O0%+^m}pfl2$wbUc5$!ijriwK2>9l|*NIpEwgJ@|Xk<@ve84$=TOjJRF!t0vTb>VrdupZqX$MMea9okypY;hm1S4scZ2g`V@Zp7j-7?-06*?5B@WH2_)HGq|f zpx5EN_(UcmXs~&-!Qr>!1do3zP7pW)pwV?BrVO69*-jx$ByX86rIUJA`drshtZ_tW z#jjhLW9qF^ZetE#Mt^iFZ?39q94>Ail{$X-YW1s?nw_`r&RW}}{tJ|r!@dm~Er2Pw zl`=Jezk6ek-D&4~FTM-)ZRP%m{F!3kTw$A`K;8hv*Ur+egHDGxAB_zSdM@7Jeg(>o zle7pr_;Re%3?w|UG|gA1*snGZ-We=e81{X4 z#7RTP3`auBl)}Kikl398ck9ZJHzu9sT<0ytt7+JTIa25vyB-WXgnlJG)q`8h+zfn- z$x&b)@;ZcTpnbwCV<_*&>8D%x-miJd^hW8N-bCf+vE^j}Nw#9QM<7J9?DLi&H-mw4 zAR~HpVC}GRGGXQZ$I3?3u)!1xS&ztw9J8#f-IQeAjLrA)ba(Z0@$hh}-lSC(sQ+AJ z@AE;%B38vC-dM0Nn58-17iBcKaUI9$ice^#*3l|OkFOrOaCwqP2Y+knu{+o8edAHE zv0ky@H${*a!=`m%-K!-RXcCITZ}@(;pz!EE{T`BPP!GteyCGTSv>sP$&fb*vQ+GqM z=*V>0EPkd=2i4#3J1TDrcq>DR@Ri8N!Iz^^8#+LcbT4yV0JZ&{2dK6&Cd7+5L$`bZ zb6guvNFf`RuYoKV!w=Vqpt5W`@a~At$vzSEL;nGC_$K@iK1)ztni@SR+F~!GIa=q% z162SUAAV5c_7e$s*nz*q7>!;*EmexZm-5&F75q?q(k0;AA~MQg(?~*h znleY<$&CpzreMFHdP|Gd>+2AOuwIeTpVM-F~`LV#zRn zlfTCGiI-kMmvmowZJ!z9-6Hy#>yuIa2lm}#F9aS_Q26h!n<;>uxX}AgmBZsW6?T(4 zIJ$j{WMi_Y$B$e*#97EMzBzg}6MX^Js-IP1B4osUEj}yMGXK=A)n1SO77d0=JA0e? zGu}q6?N8pmxI}KWcK;pw$Ia&{Za=+wjCR%v_T5`CatW>KymXJz za!2aWKvaqah_eni3Ts-5%G^pYwK6>yJwTfPBT+AW@NMm*Y?Y1Q*(GJesT+!*i_)uF z_da~w3_|QvhUcsU-+oeW%A87FymtB7eWTQ&co7@Pwh6~Y@&u7+x^=VTJfu)4L61Iu zlH(S+2Y^5iZpeqvOsERnXSt}y{%NCiAG}}_)p$kjjRO_;$1drk9F?Ou z<_<_p?4+QqiP^`TS#~heX3eI+pfI1uYjxuJ-I0YM>Z~qN^PX!i(s4bU_T-Q0ljUhX zEsvMHUp2HI-%|~gAF-X?%+$`|a*jO-)&Ra6u)``>1~OUXezc0E zm%l~klt8Lx!S+h@*5^%&_Rc?c>Awk_P96W}T^RGv+fFO15$uRx#=_z|Heq#5v;S7BxIY@b3V?1QYrOw_>%Wkb>>t4K zbpm*S;~z;$z%NqL7NypoUhK;KH0}+vBJ<%<&GG14tQ!?G%DwT<_=Jv7kMDUAZY_|> z9N9g~T&RZ-9Deyjdw-8(oYH-@%OartL!>AgipyaFavh}V*1w zE*4XvJBlta5G_%orvX=aPJ^oRE%=;9HwF*or`cPY*vEW6HkuTnodTra*`pG6X6%Kp ziY%Q=m?hJ=P5Jo~q069!J;AIXm=Q;Mg@9y6#K#@<=kv8r+~=H{AEi@_)#$zOiwxu` z+~zy=yq!Gm;w*i`f3Db9X z_bYIt`(uQqxTDuxA+s6#{j^fwlm)%L>qeN*6t+m3By>0!jVnaRXkEE^_ipc}k_{8_ z);4Mm!?p3RzBM(=L;bqVF{T_US`R+|C=P-IO`8)xk<2HTL#}$xAwYWgk0XomyrWeo zCPOmYvN5cgvgXoS*rRWrH5w~qk7wtj70mJ=GE{{(3koLHX&&ki>qG-E4ZB3DT*mr0 zTpLx-KpiN@zQ8tR{BRT75JCZb(Z$y*cC10h9q&`XQa_Onz?q zF-Z~vyRpa%>m)t`W}b2n>n3t@*Q^8av3}6Zq4sNr=pwI{Nb=2kQ?nu;qeuGBO0Ku0 z;!5BR&!2*e~WZa8h>NIiPP|{GQYMAAP%%ZNKZT_19IQTG^y+UT&Y_gzRUpM~ z?T|c%WH@H~k7Bxg8$ATzEi;?|IkBC}WpEXcegC1LbOh`sYwy;XJB(qz}P*R z;F_lEupfw7F{1D`?8v|>@~^rk%pN_s9zfKA=&%>TdvtV7Fk;64{3d|9rP|YTx?2ET ztf%cnIq{UdZ5tTurYFD|Wpf#|`fs!Ifiyvg_(i5%`y7R<05XtE+P-)GtWAOf07B9_ zj9B{T6M%PMB&{7ljnunQ>WA~Ck-O^D_uN$J@NXZB!1qm$5{LYZ zJ!V+qezb{)-IKI_YD+A>x>diDf9abn$MIM?(eK{}0p{D=%Bed}nN8Owpv zSG$R#c1F*6?2~dz@0FP>q;z;Kh*7K^yS|2ZrVmbae3*X7moU>a**)#OQ$J@e@hQfh zH_a*j`FTG&xDpd)QWG2|Tup1Q6Ozj{%C@yewD5yIbd+68%MJZ|Pp?9NwsIrFe1at8w5>aot*}6!GQ)M;r zv7Yc5HF0&@gZdSoJ`D+Xn!|F>4O{b!MjbQC#r2iA)JpVpkn_?%802h!X%s`bbAgOf zS6_WPqNAfio7L=k(tS`KOsOie##`J!J`BF*#&leH zWHJy@_ezVmwDMi*HA8nX2C~TQ&7-;1JiK%U%rkHLI+QvZA(qlI8mN-16{fa>;f5GG zXNr?`(Ov<+9n@k$7Sjx+v+fHR4ESjK1+F0)e0J2{;uL<`AbJ9u(n8X5%7RiEGtdT_ za70kiTYnQQ9guW+f#W{K&;z0GYUq&3yMN)qJFkL9W-*EEzsPK0hfKHo=C{Ks09+lM zl=JDoNT?%=JBZL9M{@FDeGvnavJ7}1u%t3Ry14#r2U-EXD4`IwaNh+(Ns{)5%_ro1 zm?4C#R}U-$wKJug3%l;uDgz#@d2)nU@-}|iBQNLk0b&bbuZWCkx6tTE?SYIbI!(p~ zQfu?%!k>^jG z7z=3xb2eD7m=FpDN2jrIurVzmhAbvlC0;gIJf6{Ve0gTT;2IgbGy8MB>=?{tFa-XB zOJ=Of%Di{pGL&o7JLi!L|D&pbmE5h_0EU)D_e=_EuF0s@S>Tcv!=5Qcl70Yairn9m zRG|u08JmMMmb1BJ>5C97O!vi=Fz% zi?te<6CRaWY#%fkAk zj=>4&Flo4ZVE7*(7LZ)k$iq8I1w&achsW-hM;^#7(3DatQZ9yiC8k(p*Uu*;KUF%15Wic{f8lN;G?@@L!cvG{%feJlL!BCD78OM^85Qazb|)TSwaa< zEQ$RG91C3e2_^@C8DO(fV?ZFE=LDdTU?=g8@Ms!b`8=~wISbp)C&BMmi0J#X*W5tc{ATQR4=`;PUu$~FRFJR z-rupDFblmtvSbjK68C01C4L@2tH@QS8IiG}^UK_qvOX{r4WG&qJ5O0R^uArf!JJZbbV@Gl9M0pn0`&%BV z{UxHLc8vVNk|lIb2245=u-*(r&asJ=Zc$N(tdXo&8?7Gr_>`h4$74HhH=+o89!K4F ztkM_=ZXYVdphK95we)LV0Wx%~ap!_;Do(RiuSh+hj2UpHi9^(7<>O_r1vs_N`o6|Y zWP3FK;O1h&Jg1cR$#9riQuv1yZh@)>&ig{aA1Ql;19W{E*bZy0#h@w-pXs|_EkL6(7h`Uutl{@mZS>fi!K_fwoyWJaqDFttCb;;GleebEu`F%G`agAx$XKhyQ)Oq0D;DE zv@+=8$--E335HE3#-4x>iHdQ6_l0PnHS24C7Mzv%ph=gu^!mJnC8BN4@Ivc5n}W_f z&&2)TY9SFz{pa+Lr=ML|u5)@-IC6xFPy`lA<85>lizD7If$P0*E#9`){V=5AP=Rg^ zj`-1r-lcN7@a3t5mqD@4#CLlC|o1d*(~bMj2}MJpkym zOFXoej5#)F_ppHYA%zj+hAhv!*Sg7PzRI5{FOY)%5ObIEWI6i9f!svqOd`dy_v7M$ z_Q5!lQ`y|ybfiNg!mXwZwgl6s*9kaQva<(Z!8iyu-1PmtH;!}eJepHc1S4Te2)MwWL!`{?-d;hZf$NP04; zN8c~ScfhLl{Q!t(>2~m7K%|gqMu*yiWD9`~bC8u)viVnwh7AWmB&_E?{u9Ri8_G>) z@|HA6x-=%GDAT6IkL!3nBDH-MO20u5X`fy9h!B*dUfnE8ER^LXUmLb-oWckfy|v{J z|1_a*ff&-39LqM>wq9Sex;@ZSDfE`cQrvscP0XoMTXJaUjYVvAPFAH_#u^5v2YadC z&Y*`ECUK$eDh(8^2Wn6G86&4TgrC+?@I>AK9IpSQy!0_T&(BD{yYyUkKK(@XuL(Y) z+c%K`AfWXy!VQ}O@215>z+W_zNIX7Y0;vCdz;P2htnptiIe`)>Mit!i2x0Eg^s(D? z1+C^0)lKwv9xr;uQo7Gv7!;4IMR?}Y5)N^it;`emVi%r#NjDdXU(SI@$^|5L*Vg6S zOq06Mkto6eyUpxb8ObVpZ1KgS(kMI|dF^`(?(zqSt*J;vgZ2N6K$vP=XfV0eA zdt2lG}Xb=`AEaGnKi>7 z?9ct?nNa^rmsXb2^ZmUin4l+x`V5dJu~KX}pBRE=*u{7b?^;0f2V964e2I z|1J0a9N_PFLIA;a{xU(}9h{)+sw3MxwEc=m7L~cz|Ciu({--dmhmAa4Acd{LW?#W} zb~8M0>WxvG0Ie(3fK(b6=UhL7_Q|8rA0#e(O-7-Mx!;RW?s4mm511K?>c;w|k*QFv z#4ER^u0QzBlK8dd=oK~gu`3Hp^L*-5*SMW&?&9L0$)E%4jb3MsD{zGsQHaK9LxnkC zg@z!X-}{l;;@$BXccA};M4aq(%AHP`uc?zG`zdd-+`6q+lYi9CZ68qs8NIV4)ffdJ zB7EH$Hfz&0U`0U12XZ-b!i$-KPx(4K{CM*h*|%XS>3_F3&K_7!?l$l8Z@#u(jWS(luECC95bv zwx(w_tzzrGVyVg0$V$DeF?X3;EBS(M0pIjgU`Tc?gWC>bpdjR(lS`P+ZA=2F4eSIT zU%7w!vGF=xJL;8mA5b>ho+z9d@@A+@xJv#WkI;QamlpNwezz}sC1Jc|dhr)ROwJpp zU8PkwEH=m}X)^uT*52|)Z52{LW8dwbA6n2z*o2{*O(F5XK_hAV%&gWlZE>eSW zW%pGqB3+5+CxuOzkj@wV?pi{nngoQCTtUEm4-uh26ep5{paO(!E&y?xU@%Y$sz$uE z(kgb$e|+^6$OjRP{4pCkz`9Wqgw1yU4nd9nA~V@R0r4l`>miE*qCnpNj^^Pvw*vex zZUxH|QX7V9LWVwA5!c(rxM_Svz&_#3wPe}7b0=)O+^oSugnP56w_E3TXHUN-Am*v; zcT3$7q6{Z;WHOcy2UR&aI&YoLZVe2mMN~{w zEESoT09GK9NSx@-1{(QZqDDt@m-d8v^5bEmclo+l7EKry#At#q;X-ob5^-a#GMTA;dvd-7=W7ZW z)CzSh(F#x0d_)K2KA*%Ckqc*(Zs5P$s#qh0G?8dUr2EkUMD68gCadBRz+c05`J#wK z#a+^s5a3nR|La$gh%4Gm#5r|ZbJY$%C6dqOGc|(+JO$&z!wT|Khj>oYcxI}=b5Tn4 zumSK7)HX2l^Ch1YOtN}}^ztxZ14sST}ygVGJ6><=G2-FRnh z(Ih6otSj+byyZuw=gAa-)108~&m(izZ=eQ#ammnu4`Z|aui78wuq7Ahnd`ZN4FQ3` z2-E{SwfkKZz*>ZM=WP>6SorzmbLb61CwCN#uDUPO|J%T74TJU3)(X7El(tSV8Pl$&ZUx-RJ0A*Wv^%v$M1J7U60A{t5m< zKU!@+o@gZNXv;q96xSJhR+Wo<@W#5YfzNAA!ZI>fG!AHF()2ykMNTbKN87sFTBzG? zDs97Siqr-s0v(MAk|B{W02q6UgZ9DZrcoEbtJ*Lkw+oDzqzO>7V+rEGB7_1y%n6V+ zT8sfAqocu-q!)lNnq7nKgadA@382j{mH?p>TcB#UU;TszfUkDioXid!umU^}Afr4u5^@+@J^m27|WO=c+*Nrflr@B>Pn-_EjcBKaynXl_geIoV3 z$c001PWZcVN9FR?B_SLefq1^p&;iRxn5ypuk!z{M<`1^8XDbSJbx?e+y_&{b7xe~M z@7n$($&!6&POGMNwpEzdFnFUnB;f6*60|URlxr$D5jQjiLc=d^s zn`@tz`6 zB3KXm%mD%Ggtpm}U9}--0_Hd0Sb9XK-F0QgrO2H5m+5n1jo zITob3pNoG$*?}7UIKl=gN|74SSWkd8Uk{#R#gCm{3cO0X0ys_GJ_CQJLJGQLJ>qj0c0h{$CirljhqJg` zZQ>X~cyE;4#jPLXa2spBhsA6aSul|@c5M5ZDtT2Nt(=q#?9gG9tpwd>#H~Y-O5Zu+FEeJwSNJp(^U&on;bDUyyK^%TN5_^$tUWl6XDE?sWc0|i9}YDxT;sL8yKb8I7{|v z;T?TfPIREXvOWMTp&_?Nx5Bx$R_q+ET>Xkl!i|AcH>Jn7NBn5m657yBR|)BmGP7?q zqa?1>t9r49Lr zvARi@(eHlI=uvO53YN2Q+Zk($-i0w(nXY5V71SO}VaRQ&b2jzOy7{Bss+T^eE~?#^ zOb)1&%;*PbTPOGi1Q!f*Btk|)d z;C>BKzm^)SWLTV>CCwF& z5Rg}!T;YqG@j+$ISi_jO|{166DKA6>y14&7sN!Q+|S$#vWLz=EiA?zbH(3T zv6q;UKSY0%V0nL2b$u@VdIgnza^TdHI;v2RbN$T>$?ifzwSAmI;T;9;PifE8?ISFn zR$fux=RM9`$CjNRrn(VjlyskLwX69uolrBA!rnFdzM!9GEiKBo^&z02JJ8nmDR1Tn zT>;UxGMvI~zDSZwBvf!o0XPhp9#DG-0hN~NF4PiV1ANthQ}Fd8kS-B^Q~rx=s~XnY z(*kP*%=|0F?>q*=i;NqI>?$8gvxtgm>zlYd-gaaX0GG*q z{IOS}UB;Rk1J{6mJGs00v1`9fhD7hqAx7Rg0irt~iuaS`y(!iJB4H&tz%1(r%(7N9 z)RE{{8zf+K|4FQ3yYvUMO4}bq;J#*D?~|r(B<&5IN`m2UF}U?#Jrzcsd)TLJXbZ~ohhisF|6(F2Hoa`}U| z^-+&b(;Gd#+$Ck0$GEublLg;<;Cl*hM;de7I^-~U7kp`D79Nh;q;mX4_5nZEjbT*a zS-6dx?3`$d2q{`qqRCQi%+ilNIcp^}X=cyLhuvJPd(+vz<1~JOxHWm_ddY^^iv{lU z3Y-b>pJ}jTUCvO5fn;&wcTV%F9O+^{1VSv-dhh8$$Uz9ZIXq8IxG1b=K_}HnPcD89 z5D`%gpn^}>`t_6_>wp`!VXDMPh|l*QcA+ht5$-O;Xcp;g#=Tl;znf(AmTb-xEBvx- z$??143yj?7SiKbWuL&%z+R|y!D=0{g@&tOeu#xv`eZ{aF!m(z6)EyvF-F_UGBxj+* z{ibINd!6jscilV9y#C9TbJR#C!bY21P3%Vt%c*s?VbAG4x>z=gSm`LO`Ush=a=#ky z(|PO@y9$MDcRuvsib*Y0wn+Wlz~t604Nu=Ss*ACu7tl`BzA_+3BvpOg_7Rz?c{OZ;-}%J3l*Py918SH?fHB5 zYZ^ZtjNF{ZmJHpuWOmd~cFE>Q);G7&adO-0vr#E)3OZ<_E00yjrDeHZ%5lx_a0gO& z5#a9MheNmt92Bj{pp z_<~x5Shc1+$ThTeeVt@`j*2`ZR`%R;U-GKQTo&Y*%ww>tM&_e8$$=2BcEfe;I)R#o zpX(NRNbb`n3ZgAK5y1JifcM@D?e9FRUgWMt15k>@%Arj`plF%iLM*yKj|xAUj0xp>)K~(gUIQ&gEzCZn{MC)Y;Gv#HOdA* zWLK4dRZ;AeaF2uOVXO8yZW4nZhWHsdxVp@wm<{t(Oy7a+IIWCS{0veLC9$J`wYzu; z{g0GR0;!u23s?l)&w*@J7LdkTiue!1Aiz~Zu>M!T|Gm%3gmIKS=}=1>3v|1qdT}%3zi_csr;2gRkZ3?K2l*s3gv(K<81S zNc9pXqbYu`b_Nn{Sp0Z|WmrF5YHxWmRkYX#HDv`9i;pRIy#N2Fv5>T2UpuZsc&cV1d`8_X|_k;kh7_SXi4G0ZxhrS~8z(O#=0>`6Raf4sw!n z?Oz9w6ereE5WqdHhZA3I!5rqn4S=U}9rzYF#9~7QH|oo>KAKNpYzVPt1G=$ET{TU0 z8C*Shm54fR#W|H}_LXIQ@u~%=pi&er`C%hn2%fiKDJm)%{b*3nsZm)3@qEC0F3LEG zlJm3WWgRL-kc`^thicfRNlj)r4%9iSXQ|8gt@!MCXGJg1)uf*wGPAeIy8403l?t4A zlmU+KJ5o9aXEN`eQ)U&rQRgOgzCKy~0pzlLK}PhWr1uf$1jt|5NGS_3*&08~&iqAI zWaNFw$9ZJoT3hz7Jyf!w2)cEE9X6=pK7X1PKgB*OY4VE(J|Mmt+#T*UM^4jttH zqX`&NDuH-7W%a!775Fl&_pVm-S~Rn5Z@MGp8Kh%n?Qke-@RcisEc3MQu8Pdcl6Q=C z`6C{bhxf-%1ea4rqz5JEBW*+nly1W52q7C~&c zjHaTQELgAQx}d|A#K3I`E_qspgHU{0mJ?0qvVbjf-u2gIeO=T!Z*||CZ#J6qxKq-s z`%?p06N|Zxsoo)+byC`V1$#AH%ceb-D&p+&op-k7%s*6{tPIQ?Pg-muoRIeO{oEG* z?~iFUTmzhj8U>3shE|Q^3oR`>gln(HN~nqb4Er{J$NtQ%PiCeRiwx`Fi`DV+?KX4( z{7P+ftMUE^jLJbQbmNswx16l73};Cg6)65)*)*!{#e66ep&A^fV1w8=wrPe%GuPKf z2cv5G4&G)z%CYjXjj@~=jg=fMsHk;T-mTaCH7zqS?N?HAA|*Af-ps2!e0!V z2;zqqW?}KLP3vLUUZ%z_h5!XKav#0NT!w8oxRXTdFwv;XO1z;1O|12}$fQ|CwPtg- zTGrZ;>b_^|S8x`$+Db2XYRXsSeD#pHH<{B`c{M{7K{c4CjSUk23I;g2X9&d(sIzdY@2-WBf-8c zEtd{An)AVwN6o&#LJi_p`s{({JC*J&4_T>TZ{(;%qrzu`(4By7$_}t$G7OZz%hAuk zp(=s!cNJ7es!p2heg>T18IRw2Y>unghnk^%xr&P4!+lbK=6-#CFmep|d@%PU8(%sp z1#Q+Oi7O~d4Cel{>NnYhE;Z|Q6t`(Cxxpl?*!fJc87V}t#iKhgm4^{e8@U(2QB5-V z>p$|D)8w`zya$RW*O^zv+3VenW9~jbi}-%;4(Bg2nfctOq7WKKQnWP&o;aRtms9M>3YG?96hN&K z5)PdUxSq>1p4y&GKkPt{z7`k0b7#}-QsaWLlt@ozt?>ScQVn*wdW2guq8IP|Zmr(0 zL3chOQ)>EfC(wCcE-_3lNNsML2|c$zwD%^lVZu6(jIA<;@AM%hubQ=}WE=#W$&&-%QLw6-6kKmB z9%9T*-K3-w%bU{Y={IMRHopbo=;p1E#@CIHziKAWB_lgS5ruNLVsC?A_DQ^g#-R_w zzYI%#yixtylHdlJuZ@xRqqYIfo2XkAsNx*5<}%<3sr z_yTBRBs1bDWvL;4+vgN3S+{O2HM~zp8GOeAPERO$Q0d=GUDD+}Sz_yRtgwAg3YzTS4IV?JF*!bc2!xTqVVpgeap- zPC05tA$F+G@Dn?l$}@6W1FEl@RF5WI$|TU|@Ds51KCQJ{Phz!#bq^Ld2)6N6(yRByWeSo zz2GISPL_uGl6o#XZqy`9N!s73h3jle+xCQVGFW^T=&6Syw#MI9b_p ztEJazD`7zj5!`wEQvutkaUh@KUiDr>Lqq+~o=6jY#)%yEo(&r>SKo$;!JdAvwWpWg z@+MOl43E7I#ZZuLEuevP>n!|OkI4{RQ&9JdEYk!J9HOECuy3hoq6t1d2KXxEC||*8%M}J(@%MpFyI&B=UZWAm4poh|ATaB1`M=sk9qnq zNwt{m<%2UuYPc88jO(WrZ!90bKjy*rBLAc%;psFCu^IeT>jI z;Ku$F1r9muZ>zF-?{~8Z6`)VjpYWjz0z8s>cP{{7>%Z_sfBSubq5#;t@H_S?{#QTm z7ixk3dGNFU;)4H^W7MJSRukW-hvZRv3@UEHB0qAsjZT7N9!O87Qjz$1D7e0yBYB%- z#guRbae51?cbbQQfDV^L+oI@x#`d`p|r_zpx$ zgtd@=P69Q@TtnJJy#+F$;D2rklr=d`Ez%KG5U8(W%iz|;f7^-?SPp6a+nImc6v)pw zBm+dW`Q*tU3Fw&jx6StnPE+J0dZ_!h=20RJ=+N`$*83P)xX1rp=5I6gx6S{XwEz3k z{Dq6z1G^{^0Hz`U%r0<0|P^n5EoVi1A}A&0|N(#gZWz{?nn#uHv@N66cYrip29o* zTYxeZkP!d_tBXZ=(}(_BhPM+}cLW1NbNc54-!dU|0|T26mJk+DcGJ1ghSpOSTkf*7 zAHx6gt4ztY_+xSNZ(8rU`X>nn&k0fwWupjcEpPcwd3Wf;%R>meB>vRBCBNWJcRlJ( z=TznSy4HEa<^Fh_K8F$&wtS9i8Ny#riV`kb?CBG7QD^TvI{Agn(-rGlw@inG#Rt8# zTAEu&_w0V#qZ0*u81-X1aZ|_(G9l%R+3y`Gc1c!-5UuKON2 zp%EZ!l_~}w`zp>Y3srDno%*^iqX`F|Opa)nDzJCXQyEO(?v)yjDHnJ07JgNEP}gSu zJR?e-& zK-gzdaRm@smPh(6BSWAjn+YWwWpmVP=a6zgXkTzx^Ucujv>eEwA5naIyfz&qXL?ut zS$QAd0rj67h(a@z%zZ5mtQhMSxUoD3#yjO=sc2p{x zBLA9}xfVOvd2HkJE2rKnTk6*-831nS?8)6>$2m@#>?m^9(HN1AX|{~i(`HS#p5+ag zru1uu%dN-T&NWL%ixA*3_n0s@VN&KST%#FU2Z1<8&n=d=Bl*8%?#L^AXl!D8EK z!|h#hy~3N4*1{maoZD-&9N{~e{gZwrduQ}4msc(8?3rYR(J1H9v4h**jZPeKq|LYM zef{vmWj;lKh{$bd%w>-c}aq4))yl!=;6E7FynmR6IQ zr9ttHy7KkX-!D+|R#qtaN1sN#O4-H4OZTk%J~^{zqwqjJhL3P*wIE4@s6>(1)EC_W zSt~v$lEVgaR?|vh%W=%pAask~<9#yL>+K;fEc$Hf{8hst_{A2pVd}(Nn=NINoQvd; zQNHGrrb~nWOwu}%zm$Y&b(M3j*s{9F)QB#iet18Aqf=|N|8ac2Q>Xt}Jx597P7N&$ z_Nx1rj@uCv==lxcVcnrd$)9J^EllA6fv#DS; zWcEyFZ!wO=slo?|laK&kCD@#@%JfsO7=_CBT`=7f#H9#2Dvs1DJz5ttx;h*vIvHal zSLC#fzdqV-Dx7FD!Y5}2ta5)1^TZGHT!}Y>JJ`(l_rmgzk<6HjM@^q6dS10kG;g{I zJwJ?3laaD0!A3(-pnV<3p~MF~Zny|Fgh#ZtKoh3}MNzApE<}%|ZnmKA`N?%+$-@-!VW?RK6t6hFi85PGxQh!jh)qd19)d zqkF5>P|y>j>P=|Df_X|$%Fs-;$v^v(KY?h5+QeQ%`g_3r&1vZMXj;!loTKbd*@$C| z@a=0?328Etp6EA;kMBH^a}eWOWNneTq@Gue!Cp^OxC!ZpXa&tS8`H){Mv@*c1u4bN zSD~8>zfk?a6v93NvCG(?ZTM&F(_i-e%vg(mIz5N0O!PgnkbQ~vXtW?`+{E}2!NEfa zRf2G}E6c{cTn??iTUR`PZ~M-zOxfR4H##V*TChmoDmvk@KOoCdip$=KaHv%?FOM`n zY4ynEd|1u}hu^(zUSS2poItz_Jx`{{ER)OeWezU~0U>*u{2S7z*J_xM-P!cs0*Ck- zO=d(*p6&iIQld{>Ms9BE8DVb93;Z^HfTr0NRQdZQhMm3G*|0BppLq{`r_QWf`SYoI zIsSc!usJGXk*nLpCHTJ%;t3+0(-*V>e=-nJ`l{PJz{m&0?G8$(B>QQa&qe_N$Ur=W zb0pwuSUNa3;`*M10H-0%!AHfXlSUIM#MRZ)4~}1xq>GA62NDvi9>Y0&s&n=0VmNrd zR_lTGy`EyKZQ5F~HrL#yqo7hb7+6C)MY*QNN9|)Jq%@s~Deeunu?hDd^mpI60*TgH znz2@X_<(TcpZ^vV* zx^Uu}k5dTB4(r)|A^M@y^ev^!DP*F@5mx=rOuXXDpE^Z{=_%&VEA8*ED({m6iVoMr ziqE_EieBSi74J{+U$ox`ZHo_507iX0^Rk>jj2&Jlm&M)Hi!}^0*b~6fqEvOgMA>z1*>E;!#hTa5{@TUSP)RhO zw+~!(&#?0&yX7j5k)18?gC*RW1WJN#xrbF}&GPEs%5#Fu4$D=`=cdKvx2P?9^S9ex zhtR9;nY%>%#gaZ}XBLM64RQ(kfu0i6S~XEC9;D9)1FrXK4>e|2RycKwcF&ioxraO6 zV&cbCLzlbf+j`Qy)rrt@mya>3g`H z#cnWGbYDm6l$g~E5GX8t3*lN_|LC~G3R-p_(eFu214B_k2g4hFC%;P`ux6~RG_n&U z_MYYjJC0=cG(F`&cWUUmoZIU z-bn38_u*hoKVm>Z$?hjp9#)4PsuG8u$YC|-M5iuCv~%}&$AhWsN*2DK%451er$xxn zRF)efmTN+wP4OiN+et_4#m91fO7tfiC$#m^E4r;@*S_`rR#RUv!;u#FQ8oS_yTAm+ zg%Q3Bv5I09R~W;mPWX8?J{5Woi5X3C#YzTg>t~kb`U3AT5ELvj;2R%%;PW%|+>CszTUOyga48{dTe}*8&kZ>U_RZl96?O!9+Z(AuvIMGb zMK0^9Cd=W1SW$~rj=p`*<8-U5)XuUL>3w=sCXqwOTcEmY4}$Zlw$S*`?~01dEH)uM z5tKaV!Ezc*@ZCRm>Z;r;tR4ujkuB@LXAE9a6rlu>zF1-Mf-{|8R_k3`gP@ zkC|M25*5avp}ye?i&*T~9LWtIVH!a*L~u077ukVz58UstAD!2QxZT=U1!I|e$z@yI z(QXhWY>G@j9zCL7(`fp^-D}2+H94+<`A_7NI$Z=R`7CuXY2s87r{faR>5Ql}7Qm!@ zSKHsY!l2I}(or*=3$-dAR`I<>F%mA?iF+lpEDR*`oF^r-ovyG$Jh~0v?&x~l1CtIj zhg?d^QH@H-s*+9GSI^HYJqH1gYmznV+jvIq>qCp#Y z{|4U&V~G@ZOW%z~ZyY<$q3cs&qY;e+Yi#0v5YQl-5LwfyQ{8c-xN4Y->To^qD49yH zeVB4y6SugGO;wuh+?`Xuin%_Pew+))v}ta-@Z9NsS)Ja#&G314T*zfcuH)!r0MO#3 zgR9Fc0t!E$WF9si!A@scRJT0O*`9CaV-ME1f&SRpw4T?#ab7p~({mkix^3`})fJZV z2wwrr!P*_8s-&MujtuQuE?d@e=Dj|#(bZCUnVfrp12vob?yRaZ0W;A1p;D3y>hhrMVT}FBsyysF;BP*dP@qI`+ z6CzGxlW$deNu0AtGlxXW8*09{%8_evA>fQ2iLd3!EHuRdw4&6gy>h#kWTUn3?=^(D z?H-MzRKC?IFlJY@mejC8K&`PUi2Xj2YNVj)?%djo9a?(oK>9{#*l1;86{+9i`LFlt z%QcO}YR4pntqtN$KBV|6hew{}4x;}Bv7*Fv@~+_jVtaZg@QRy`-M3q|c1t5jz(>U%?Twk{lB1yN?_ zKkhI&MH?bOL5Qm`YQ%6F*>ZjyupGXP)2GhOovAG~swCrgxQPks)$RCnB7f)EHfkxY z);6ncq$;1)^RyzuDdR)Za6y5W+rD4Ss4qG?@A_JwY)wX>p)F4tbY-S6*x3s3sWB#B z4!iGbJf$Y}=T%$u3SJcl^4H%4FNMP2**ByV*K~DA*?qjL1SF#H1@BI(`CbD}m1c{6 zh@w)VTN1R!`XdQU>BacJ>KC32hT8>jx8aasu@5tE6i02SiT&#h@x%bdg&$6geP=?f z(E5?m!8UdHs4Ggs4y|E8R}?kMka3Q=n5oN+)%Ut@`~~8P^5ds0yP%>U%XF8rMXI7s zUNb#SD4};i6z(h5o`J>TF=0<)Rr9)n_9P80O%dL)_w&&3T$Ls=8TN{BKws@QEG*&_ zaw$Rh99^wk1cVL=o-jo?T%PvGp(>hq3S5r%s48YRI698^gIC+`ICl~96)}xTY|qP7 zbpI96Scei+y0AcEVpKJw0(fUlwT9ryL~Xc94mujRl1HP12-}nDF#G4+CzW+sFKrn5 zCXa;BYLl6WR~Ww$C)|P#C1=g~$o5gHtgs3qHH}-N-j{*>G-!Ye3WAF$Dy<>F9aUV{ z6xkV};$?@6oCW9nvOK$(4TxYLnRj9gV2*_!By#KuCIc=b?MJ(}U8SmiC#?2oV*4d9 zsdH4EMc)^3JboFo@;!FU4COR?_3YzWkfia$c`TX1-FU#`Usq3W1ksFn=}kt42IrqR zQsd}+OUny{^O2eF1B?m8LMTBIE^nizQ3mceJ6*10ggx&huY?Y*XcRwsNBqDm zc08JX|A{fBHQcMKtF6AfSe5#++Yj_SDY1sPVSl4Xx52RW`BVJ%(9NNisLnnu#WLL3Lh>~$P(V7MHt2IlE_A2=Vcm z=T1we6z}=b(FjgRw2pcN8@Rte1la7wK`JuAPa7fI2uv>%50SS@^bsb2@@a6l2Y92# zfe|rdsL!C)$G1)Ib~{Rg5s|6enMX>$X#92S_CIp_zC)?9TmF4l9j$q`+B40^e_;2p zEZIvSY691OS}KOZ3{DgwVK<6f9BypZ%o&d5@=1*8{P=~*?cAu`>0N~u-|21R#K1qP z91!(qy0v(}qep$stRBII& zgeLuIbJJ8WzV;l_TaS98Oh#c(5*<)=KwSGF4Rt7&&5R2E$|nKl|9F6~>3Lw$e)zg; zp&}&&r_irc)6E+ZWnvs*p`+9Np(C#t)i@j`)%Q6|GgqqX>rCzW*4JT7P)Rqiq$Bt8 za7``*dhPSPmZzQc4<(>(3Ul5iYuN>6o;nkF|3QsQu{vF)RTYhW(d_=%LtnKKGg>%J z*cLZPVb8k%Bcq*jmN0#AwMyfm=oy1P=+%w$xk;_Z!v-j zDNw%`JOtx{Jvi_d!#d7qW{oBHxQLoYSOMKX2&TPb0+a#Un8!ZAU$;IpjqY0ss{)ZI zeb-(Kx;HwKn;`?w=?rEQvD!&Q_U{GzMkn(~JY)~mch9+Z&D&a+csk$`4;S(?BKMB1 zC3MI{zugn&01M9x(m*RQCBYfqbsK3-z>4~E;m3qKQL zuz-Jz zyS|@9JUqq~)~-pfgq~lAn7bU#WtPXS#oIckUU0|GuYKsDwZHs5^SKGP@aP5xpn$A}ot(m#Q%0#)uQbJ#G0`C`$RqGmWHkDFRHo{8d21$#9 zmT>a+(oJJGLL7FK^eV+lPO7gseFkC<7lv#et$O5^^W}bI5SVGguLKppf0iNaLxbcM z{f~mq$ssIbC`&`zqX5)nA{-f=1QB29M|H4!bh&FaYj7b;PeC4l_I@VJ(rmD(Z_$wP z8D;N(s=7%x&b3@ynk`Tl7T~+1t-^_t{P7vkKHFa#7pvCacxPWMb}}0_3@>{D%ZVf+ z^yj%;{OFi7$t<_r9GRM$Eq~EX<>wv3=2Q*i)RtN3#q^kh;zld^S%F}9mM?oPev1SA6Svk56ZP|$B}ZZ>n^MmWM$bzCbxB{Xo@VwU@3E=TMB>)3uP8lo%y61gcYTqo zwP@3vr3=n|noV-mcY44eWlOtY2em9z$7-zAMD8@Z#3pksHtpdIC;5gT5YECHL`wsa z7L{359}F<@R~dpE9ffB+%YOP`iYV9KqZGHf#PaT=2S6iW4VAm}yN!L*ic^4vd&k!h zjC^a_{&@=woX>LLR9#Li7G`72npwVje4T9Zg!Arf&2Q%<-09vU%EobdQ3W>@j^oBb zdedDa8ay?kRCNS$0Bp*-3`mDamc9eFvI3*F|K9f2)kvL)%gHtvA&nCGbWQ)4jmiKF z7Z)cD0=1A*LR2I$HI&x4S#UN#PW?+`A~m zg?9wN4P6!ag0;7H!(b!Z-lsiZCze9111CZWo5?_76I(MY1z3xbB^&S!a#kR}y;6}R*kE6IFp_hb+GDtk;icNsH~P{Emy_PT za$KdQ%CnYk0`&B|Xf$ZTFIzx>r~)QyHjf4y$VW|O1v?hYn49oAm@^{7$o6q@0a~Rd z5StrT9gXLmeWBoY{9L;Hh~6zyMHE9bbx{S5A*6b21X}}%l0JE1a^NeZ4s3Pv$H&0# z6x2A~)p1|R8Z1en%0Y-mcv7#%_;*#hiH!8vfNr7qnS$O+U7Y!OIaImprbUS3Z=dHt z1Q+dJUhp=l45^~<$27^m*-qp@ z*@7p~B;@>9T`*LO!`aWTSDC>bG)wcGH0Nt$n^4nAzky#i2O8cG;<^ z{~(@mnw2DDeB(x(lMl`FEKxFvFf>7sYP6amr~7c<`@2mjeSo)avE-=OTah-FS6!Gr z$P!vmKRabM!4^@Jxuwom=K2b|G@b& zC}w`a>^ZfD=1?%W?Nsc+Q`}~;yD3A6p;6<884qUgBO&y?@()ZZ$haP`Oimh^U#@=3 zQb!a_4T;7?qSDiLa0n##SO#c!Z+rl#H7Ox0dd-19iGpyn=3%B@#tR zltiQ+;^tz4@VcEz2@FS9vc)1N&{Y)BmR;mbgElV#Tp*Z7F;jwC$c{vS^i3<=_y_2B zl7(tjkFpF(MY|Q%D#teOlBcK6_XOE7a14d^80%h%~wb zHp8k)Dp)4pgR0M>ART#pW8vDWI+I>2H7{CaxtY_DO;iv-DwJoYj*oMu2UvRIv4Ton zal3ZgOLzhYO}pt@*>zRe(0;9yiMr1l&$4ABE1D6kI7!brKYPq3*$C9|z8em8y!%m- zs_NRf^1iQFXQEDTP)U9*V+^3zLOo_ST2?dcGJ&+BI&v`tO!e zq2}LRhunCa(+hur)oI!2YGY|KXdgUTHD1)7#$c*szkP@^e3iy_1q&$EC7Q|?GS<*g z=>6R1UhYD$QLjzlBtu=9p=FBcV4aT;R0Sm9w_nX_M-@Rp4Vv3kkt=9OsSQU}he&e^ z$OUFy?LiZpPeiR|bqld~IdKLmPUAYNI@yAeY~tGgOn0wyfFP=i(|n39H;%>R0Py5p zqjVQKGi*`es&5X8?TNUKaF4(MA|?(b9)Xj zue@gWSfE+J?n|dcfHq$XAW4iyCg@Ns$Q!IOjr0#RO#Ox|>5!0i8cP;Yo#f;dbk7ta zojYrOqI%DrY6nOmq7Y!mME%~xLI09N_UhN9_ZcY{ zB5ejcW-qBUl@79KN6qb31Gl;7=VcAc_*LQ1(Iy&K0*UF0E(pfHkBRq&X{4eVbY)kF z(~vg<@y$2pshE`)h(L>;8!Qs|?DgAtm_wW-52+3(a)Ymq6vTzqZvTN?p>mAT94se` z@TE%o352ww%reBpE_oqAZk0E_N0k&T{Dg zK?!6Z(njEPNix;oHyctS>EJd@Wb92EzoYAl#Ttdhe6Z97RW)$Dw2wJNRWf_3HYaU& zKzsibVnJ&NQZS)lD*5#<7u{#x!(RCuBgTDd$_z>Jkm5r>^`f z;mCn4+aa^B*vGRPhyNDA$QS5wsK`B0O5e~4Ik*$C5Jnl*s3$Cd%Wwa&H+3V^o)y!x zPvj~#LjtpC)OIl9p$>LTpz$Ww0#?BTPr!$;f+{taZB{#;yJyWg!SaEQ6Sd-`_aPoq zXN)X=7lsj?BY^*{FE@-&K0z>m{T10828l+lm4GFN8R0wI)r1w$gY`{cUq-_u4bv_D zwL+9xmWNZOTo4;(d(#8+NMnH-ivF8hfatjTz4l;(MMitSq&B0eU_r2$4ko z0b_=A_5rXs$53^$5CP*>x`@@W(r;+xk*M4?n0zDOfU4-s@+Ikh0>+7k-yn0m)f7=T z$w!$t7Y%Hp6&_5#7zOqai^ap|b`+@^Qw#9oijdNWUfH&fD&(XSkwozR%!tZ!Y)qnL z`rzlUZ*U~@+Xl>v>-{AOjEl%H57p%iTUb-n(7yG?}l@ z0L4#BfIl&<&_PvI+z{W*qLll(`!xGp32n3+EoQ7k?5W{WOys$ha6Ok-;C(*IcKgzh zO;iPCaB}t!VUnWh^`fZ7>7qvOF^5k}n2)wZK7E1nX5cbF;=$=*@xy1tj)wrC$fWj*1YEq=C7(g zh)PFaoFsw{WsS-D-X-^A>)Xvb!&qx|q`A?tPNsj*vHGjnYLP$w+WOh9Qvc{emb)P>wUaGDxDoIW@Si2_Uw z>77Kt;NsiskSn;u?xbI5EeLN+bkCEYGnh`2Ya7xPciJ_ES#&%emvo*}LvlGBo`}fq zrpWhjK6Wb=-(~BUimoGIK}?;KMZ-llX2W^>-r+o2bzH8HQk>yiqE*IP8q`rsJm09N zZ4X6hmVM6M#iC}UsfmfP`In=Rmy~rQ(4l^A+7{wIjHoY?lW;aVJi9|8?nc1^JDATBfHmt(69}fn+ ztuSpi%)Nv|GM?I;8W7& zIPWk=HxCPjVi0BVsYO(+Lo|>!i?8vA<`~~ysXh1L7TnZy+X)^o3~-|`FE<>t=oZfh zS-q8iK`Vx&$TOzuo<2s|h7bFZX%SV6N;|En&zVLq_Yq1tUOm_NydATlzX10$babZ= zP14u?Jl)b={&4m;{5DtsP|#kxQ%@aMFQf@U>Y+`Wd%PzTGmbJoSTBYa$Bd(K?Zhn4 z!AtME(FvGn<7k}pE-2fmD6`{Uwl!X{36S=-)ExO(m zKVw$3c|Q*zsTlyl^``pCcI_F)CM3rkyyK&j$ys=5b?7i@i&u^yPG7e!;3z$2?XV$B z12Z`X1J-i5ac!>v8{4A8EXMN4#!Z-2^Wu^P@#w@zk{Cg&l8z5Z{Zt1#4snA6w|vf*3JZCu$#sEMb?IT7oe9kQ7< z)%~%-otpM%b$$oav?`E@AAkq}(?+ewW>!;jI_Sfy(*f=)hFT=)yYnZ_mHh1(7fm#I zmx@-IlE2r$O8;OTITrY6xSjZ?M_2qQS8l;HRc3nlQNe849Q?~CD8sV-_+H)Ivp@Ak z9lNKH9IiE5TazbDVS*m9rS?yms{JR&rUmd~-`A^jd5AL}M9A?nat`|Qjx-=Ra31vO z0qZlgJsjY&iEF{P3cH}87kv455;NPe3OqlXB(yWdX_J9=#-RL?5M(9@-lF>=EfM+5 zyr)n0R0FkNXOv!~BFojJEHO7DS&=Vg&V@exEzGY01kC&5u@hY$Wl*NA%!M?>AE896 zFFm||zl;FsK2R}jZhi~V&v~GDGv|u83)ysU9oKZ9Ffo(dd66+4F6~W^J-`Y{7&#;EsQm&%YJjKgq)IR9_P{?R1*xTsi z-LtLS|E~f3ml~iE9)eoh0D}1}HNH%3n}}ok*)QO}@24$Qx*n*sV%KD_@0iG}K|+p>TF~W;|7i<8X}sIR2Ni3{Fc&j9%Qme%$sE zn&P_u(RQ1HYo~=z$74gTt`qD1d>U%I(Y<537X*@|2Y#}s5OfYB3BI32%Gq6Ao&iM* zZZ?!+M@DkHZKI$Ke0!cQNZsF)kMBIUTo-Kl3}qfioKtnF{wfE`7nVu$(AJ=NhtrsR zM;3bzZX&1R6?$TlI^RU#buz_sH;bH6!k#xRLNH8BiR%QapxLW584-Sk@6PS}^VO8X zo?-OLP;s4eEbC?t_aa4u*H*UdT(%IrwIJ+dgj|=sgj>#>szxOx&2p~f-PoE4^*v7P z4S;DRTYo#+V^a&NWRgN;{>F4q9{c!pLzo-R!11(&yln46PCN( zYb}!lQb6H1zd$d)&;)jV_h^Fe8JbA0vJ6YMu8^nv(%m36p3Y|AeG0}RaBKh)DR;l- zDRcOwaN@(P*kT?uQmUv%5*0=5>W_s5Bv*VAJ!KDJh<00<0)uRYn zgAR*SI|&&Pw2!}`fe$n~ckFZBDyL88(keZuMnx-6gUr2Wg`dnKGG@rlccvnVYl4o& zC&@SOk=**EPpRB}6*qhMRt3Gc+#iXMY(}B+A010oB)?<=w#^$(;LV>v*Rh;eJR-|# zPVwd~8%mULT!Zo>9!b3Jw+q9zgN=FP^XSmg$Z1~U>(1xj%F4=R9n0mzxm2*%Jc2$i zADYiE6>>(yfTdIh-glJkI+qkx3rh>*M|}2VS_WQNsC`Y&^Mk^}C(^H>9ow2mf1=txo%LXVr&#Rx~^jns*U`fu;RaWZu9@`|c>XVq(fX|6-#g39tpCXtX3 ztaIScH4e!PGi%QGpv1$x&&!L4!*My|QowTRU&2ecL^^{=j`s+yu|~jvpzV|X`F)vC zK>p2P720&Hf?=K~F5PDAzM;tUWC#(o-aKGJ&k_=@Hq zJSZ?zS!(B}0%cNoC|sLP=?@~2pB>(Kd0$dBKruEpw>kPPKqIf)7x`rk(c-SUKKUFL zahX+jx@m*JQD}@`1r%w^Qxt^fQ>Nip&}ipZ#yCVBdyllS(Dkt4yYaeA?1>_Q0pX!; zy&mfMv@Gas@lLN|RXOS!otR`3P(CK+&kP)yz%N`jnyJ*h#J1;qhT9PCA#wcBS&@5b z97rB%AyBmVx+#NalBaJB%D{HFIAuD4Ti*90!gjeD!WExA)EwK0Y)-yOoiZCt5*U{i zlCN*!Re0h1mU4w9q@SSe7wubTWbMw)iy#pzB}O8p90FDPSrWvLFs+~Cz^Hq4=|jJ# zcqFymF-~SZ!$pL1Tl^5q5$DjN^_#C975dYm__%4G%ywC-=oq7LXuBhF;q8XInne`> zU&Y%Y^T&x{YNAe@4u^Ccud9rpK9{n-sRRm?Mrw~YG#r~2aA`yMWrJ7lg-4!29Ty*r zBmsp5zCS-*X}_;o0}*-pmueLg+dH9ET#o8Hw3IS?&aP|t3>T*>_Xh!kdL{ui^y_=B z)6Yu)=6rlptC2+TAF=+v$2%=^x%7C)lT96Y+d1D81qE&XHc_938-Uii5D(D{f@wKRPA| zgJ^SMcRh;!ne6ipJ954MbW!-{0`f4GuxH+r@3}wo=9EKs%=R zpuVMegmj{ZFX9{$tZw5p5X<$YD}+Ak@;8HP!$)ry9oH*E_{0Y~bT7Y^I7%9_t~z+b zwI3Cu(wFMvh1R94p~ea6?_dr=X2~n}_J|dT1AJM+VXJd$!D==5qBh=IlZi-=><%`V z_<RUxg>egxemEh&2&w;KmgSJ@Kl*CKv}^Z$s1d z+>`@66=Bf>lYQ|Lkap%TqbQ5csgcE7=C2d7vZpoOw1GOthvG|rT|Lu}e%>cna5^9L zgoXk27xld3v$Gh?;(KXrB+gc%r1aHTik_G^ zfmeqYb;^7Ow#d8U=4O>Mi;aa`r@)Hp@P6_rj4bfr9p(-<(K z%`+6N*xkJoaN6tpCV>MB1>(QT$w-kfQK614U!*qpm`$UWVt0x=qk7cFTqB8vAq1O# zLRz3h;-oi82O6WD19r9}i=<+EsG`dnH^7{VXoCwAx(@9V8aB-nI_5bO>bQV1nsfo- zJ$u4&Z^EVI_3=NHh{9tCr?2;RT8${SI#=n-?`9TJ50W<09Pr*AY0a@e$@dg|{EbU; zi>lh$w3=0BEjA=oGMa(AP{}o}$1=!~8$%hU@!#=2ndC1B=07O1%H{(+^N1VWwb0p#jnZdL) z>~;s%9Nc4WpZ(l1EnTrSVmUu-kKi>8rBhIjG+Tw?{4;>{62=f@o}I z`6XlrHXAb1*=4&nMVej3cLyl^n$j!;bMAWVZDzL@)b z!6vlt2lSwVCBd5WkN7PPnUpOK*}0yM-1j=48jMv#(G&U6M&P-yr2hSWgeWa)jnF=c z*6JQ0%+JeQc4Q2#wd!}2d}h8+|MJ|QM~;N9$#6xt14BDvPkE0`Rt1M>|KETz?-$ldVuqs6z8@6G*q|N5Y?Mr@4U^J24r&E|s5k^#zq8FD-%banLU zYm%Jn&p?YMCj>_t6N$!x5pTW|jb<>ML6noqYea~LCJnX`$bDqB;xzx~=*2R0!iv)n z0a1z#`; z;YcQ3nJ1n4LxlzgH6o=Tj6^#{KHvQm0E)J*VUpZ?1s-$ds#=J4Q?c=JQ?qUJ5&zoe zM$96{jqW=Ktx6P|>_U=LX>1^)hs9`Ekic#53HVG~Se)z12{0?p8oudF+#4^44df!0UX$c{qIhVOJ$LIb1W z4M`ZY$O+eZO-=4QuLT5i+@Xa@wZY*0pVCvGa`gd!jF`R$X;ZC`AuJ%Wv(Wi^OX!_I z)?{v>*e*dZv)f`Hvt4o@vr!ZA9CQluO@j0CI=?N`Sqmug5!!n)tk}gQz%L6gH-P}H zxEm67?0RG>C3Iq1yZwmD{T062AiTUF^C5ess^KH##eQ~!vmU5%GM(4AR`UU=(1-o5 zY8l(KAnaF1xUxXWKDjp3iV1bcVz!pLiG-P|ZvhCGeO4jy?WBokvXycIGDO?(6SQ8y z1>E7o?4}wNsxZUj4Ev;vY!jMI1hBunopfO~Z9KOls>8Wvf(Dp--_18-L)3j=y=vq&`395Z&5x{tB2E zHUHz8q<(a^Q@>b76c7wuj~a{%0F14*o{DY2)f6u=Q@Hi^@T7AlG>;wC@AN4Zy#-gq zgDM0ZTsws4cB+9MxX_^+u>l!EMK@KiMo00%%=>ZD>7J)wiKugd^ykT)@3sQT57ml~ zxZgBVtt0hb6%yKZc~2nQgFKC_o$`v1)}hqr0u&uy^A%qPaT4mbJ{PQ+zqC0RLk{nu ziu-BjKo=u3FO_#_1X%qT*&06ovPa9Zw5ARbs1o{fRhP&`PGV7vV?01CCUv#+PO0AL zKkR4TCCVSY`7=3Y4V z3e@sylRR#^M`pQy40&9k$8Jy-7b6s@UU%O-9+aaE5?s!_Kg5k!w@JVtt83?GKP@V& zj>>8xr@pLlah?Z^&pfil>AdKo45%LFQ|ct;ba^uDsVut`taSc-?tL>Mb~*V+Rp-0* zFIC;leYspQQ14~L2?h27=ejM+P}P zed~giEtD($+QGO#J3cwAYBGBH`Ytq^JN3C4Q#eV1>HRAV8;?ER`_60)-^CMQL8$hD zq*dZ)f}-cB*UA1n4#1tkwS<>ZncqS;!pmfsklLe|0w`@Pz6CaOr@UBlrd54CAI!p`7@0$KSYt7fRUh@XzyUb!>< zcIJpO&+39hiL6%<{;%LDguVjTz(PUCVTpkrUE19#Y<9tY-7^JS3hU&_I2HG>hZ_C2 zW3&tooTY3ea>GgdqN^I@WIxmD7OW9z@{f1=v+X;e<7HWS;rZu>a81oLyi`dyjUftF z2zKV**oTw|SNha34x5Cx0h^Q!Yq`K*B`P$0UE05STZNdHw^Z75V?Qd5Dty1FFWkNV zPM)~+(HX6%S~;ym5-GU;t48crT~b8%hdcA+%-RQCy)19GXcGk3Cui_uT$lR-$hK=! z3gCWd#LA3+tF^)2>`Tlm{uZw9oq}UWyzaanec0s6m*~8N8#DRR4<-$B?fWIbZnF4z zlythYaw(jW(4GGu-R9|ErTzXpSnx9#UJmcTo6@xabR?)$m*`o2HWhXbzLihH*sN({6D1(-Di8YAxyD-tHt8T11j zTgo~f=Z2Hg5A%=PBloJhbn~So=;$gB{(&m-|HRJ>!bI|Dv151oj`*C6D?gAE!QusU zYq_;O)rRZA)HBGB+@48I!|Kz<-!gQG%5q&=Menv90%_y9UZ~Z~91bXq7-%zCq<*0W zMkYGJ9*GYDRnpZycgY!=t7{}uRu$a|}(I+rDCG&lrbcyL&_LvVL@cPB`Y;O-LK z-8}?%cMlpoxVyV^SN?PM+4hh7a9{5egVF1=?&{gqRkLQ-8$H_*Z!tZ)Jg5B{%1~2O zBx{yv)j=$U!HpgpMkIPld9rP(nw<+2Ad0bw@ba4_XW(B=es`L^x`5?9q4F~#)UbgW zP|x*Tq^)D4$M#h@zzm&K(`cPfQtG)wrg7BcqnEQ$T!=$JE@haLS`L2&S^oF;i^o86 z#;`<9Bdu*-twm7@jXX+({ZZjCtrlCT^-5SFVRlBX@U|B=m1J2fcvMEfv|B;$ZKL{n znlmJ=MAc`kDm_%4?H`8Cnl4R_{AchQg_v4k>gX7xOf+E`ne*tN`fYMFx)1>naUx+U z(oR?pF0Kj)-rG%5Vvyqr3t8Mzfn)+!4+W@+>W4`HyE82$!dR%kSok3|llb+)e!9Kv zzzs;P%0VutA%WzYec$P(s~7Ke~%{@%&kGVjL27=bOry)lcf)y-oQVw3H+doD(T& zVF6=Vg{S@y@Pt*>2FKcnn}8eB{R+zHC=a}rsxr=k) zwTInU^$8v&WjvysxAp&2U5|td{D@82@#-?gCzP*lE89xidm8APBd2JlN=VM?x$k`1 zEb_NmR3k^xFU``H+S$QblyfM9!Nj)pCM;;jC&}vZ*4mx`09?{G8ev|LA+aG6f`H)7 zl!vM94~lVfF4mEEHGevOj_!cz2D_z|Kdb)#TSISP3L2`~8dy-3!avLKCyRa?t#Z?e z*}s~qYCuhznvf@+e{fd*SCS8+oq^~qRLjmR|7z0E1T~e>`a5?JgKnU)l9(u3lT(#jxfS7o09gzV%8gsJA07 zaicbln2z<-ITKSfrx&9Y((eNE8f*9MtBhiOx`^zu%kmcBlrV4 zc4-m_iEWYzf(AQ69wH9qV$ ztjBItpXKur0AU>pE6*M^b`^MjY5gEx@5Vz(Wz~w2*W=;j?PE8eE^vh=Ws+n5!jZ2?LGiYBJ zVkiMM#RM&O=MdS7*>E1KoG!5p-dn#yS^IGPe8J@>49d0nLKsma`nQuW_$Ug86PXI_ z_w#F=yzWFBOfiKiq7#03gIZxEzMR7lZkrkFep9pT=2r!-C=L|TVTM(5j$Dva%(FM* z^(JBlN*?$5s4jRAzp#}p%a@vdh9$3vX0+Iw>j)ZYpwD|NVL@mZWVF1$xYewTgreGT z{*tOErLQWZ7}fXRox3|lZGVh`*JbH7*}FLiDlFxS z((LvvYg^k`!rW^=C)G{4Pt|BWw~RFtl?%v?`S0Rc4i>QJLOx=i9IDb zl;#S>AU9Q7`LJVG*iO^alCW{*|28rFFvJ8TNBzIMyFCr2Dq$gnvrEc|&^(Jc8x;!> z`OCPszh^OZiAgz*aQ-U1k50KH{8!D5m<(9TP|5KM&B}jG>@E}yJg^ZtGbO|S?Ee0c z&j9^L733S6lJJ54sO%rY9-!ZZGU5DBRO63(`XvZ;NytbcCOggj<#Ycd^AXXPg7Dvv z!=HZyq=*fGjxR{X)^dC3=pHj&Giv)=u2EFsaSJJYyQxVa;Z^;&9EV3#8 zx;}>!_P&h(?p}Q~YYHW7$UZ(rfs%UmhPb46IM+^v1T}dyW=MnOU+N~p2AZ%1eqGxp z&_FDH9E%Q9{Enf!>2Rwwd85MsL#mR`F^>iXKMyxAXxp+VE(k|v5xE!s)dO5;;Buq! z;pqNqsWxs0ap{S3w6JF5w$Pz)L8Q!`HG1EV3>vykw{-d?q-z-)0{->x25_)3Lw&gI zEmIJD2WjaQm}zg8BS0n$Rzb~opt6g83B8D>l;{7ZkN?ug(xd`>y(u0I;2I_^SSGYL zm_R1!sG@FFbFWlwt>S8WAcL5JmixD3_kW?c0HitNMV!O>Sxp*rDqE#i9%BJ*IbCeu zXz91)%kMe&Ui5C&c35cif3wryV`Xj)ImGL!R|y#liuw9476MZHC1l{>WSl0K2emu` zk01Z_s45Xil%qff+pC|62Zf5W)y5QSjIc1c*=i20~vC2)c|ocMn|tx z1M34wv;&a183xT!p!v?n10iAMuQTpH_Jc_xK(ZC|r7({i__2ZZCqP+gHm9EkN5S}i zOx!;$+V57(S7ZQ;mlzb8zgCx)!B$K8-hu)v^t zsgeV7ze3-8-~f$t88fvhj}QLf)Bbm!`I(6OW*KjMCEcK8rY_ajthrlw{Pd3-VS{H9 zfSIWV6N|M`BK}`{5GdXS0lo!mY#fsracW`~E_sPsd-_lF%=`eRltf9b&vk$C!S7(E z#sK-By<30)4rETtq&0Q_jQ?-lA>ntwp@`|vf&1eV{y#v`dg{kwKx)!bC1;t^)(x=` zjAt1CoDu|BK?53vKRRj|%S;7H!UtDPhQD{07J{EM8Iv3l8m?i1D}5$)Rts~i0qbid zVzsHKO*NQJCcL5VbLysv6GpMYyVr3PxY-Y>7Wh>s-~T?^pc)4(z?SO^$wBHS{#(ht z7%$zbCTYJi6O2gFDm0}iirv)9=Jp*{@g5_btz(-=$O$RH21Op2LLzUl^6_twC7}wr zn9;}4@v*+CpxwFde!wTEse}d49zRM$<+M$il+SJRG5i*v9#OoVB5p;N$B;C?ge7K9 z`LA^tKrCZ3uXo1 zHT+q`BwbH|7m&|H^oC#FKS`Wdu>%)fNS|Bjo3p#!|874qHL!I!pTR%ayQv3O^=kZgglKX1Dn0LL&=j< zw{YaA{MUPwV`MDBvpT|l?hQ1uin)IvgkO?aj4gfNfJN-r?GyXlfi4b2 ziIUHBJE%EUsSJ&+=md7@9A6_Zh@?~BqKggV;CdR&nB=JVyKdk8)vkaz;MM|0RiTl= zeJl^sUr;c&#*XN5N^L{NY=q+k4OcXAATtcg3y@cJJXc_dSK+pJKGnHeN|*T_{P4Bh z`>P9K085MNr$)hql-JJt?f8`LLuu;Z^N_&qSv^vhoRqc@`AS z{lIEDnRSDIVyn3z;m`6$e*vH0>P>7wv)CqB6r806^5OGFJ9dtWFC*b(X;y}IC-#5+dTj77 zkc(gYu-7wT{IET$U0=spwr>mm`inXykbu}Cwq0xHzc95757eZF#LvC-w_*Rifmy@` z?Lw^)npVdD)ue_5YN83r&b0o^JO5|LWE}x=r`VvEZT>zNemh(=A)uzj-idLxzfI)d z&LW!_$fv@Euxb34TjG}}+$AykW8>2&+<(fe3E+|%0E;5Q8khbFxWQdNKt7k*&bjvE zFM@=!yW((HH54O#fk6I$8X}mm0V<@LUC7F?!u6v_oA>j&i^B(4nU5+Ah+Ca;=CZ#o z(QuY2IJS`C#vfWaU22X0X~=TF-A7ZzMt<2tL9ritoa#cggKQ&Qn|@G5hCPi+j#o@d zyq9hl1*UXI^zfY_JLHQBG-T2E7+=%J{xH#zyx^0|{s|BWEFxei@_ZvgS<_cQWgt zq^Ygz#&t8*#k5B2YAcVan!?Sk^uo)nr^1okM#O=Bm*9IC?YCuz*S;`FVvQ zIfE`bR>G?ueXUy9G9J59oW$ysEG`Qk98=dqupa(@PAE;#yDhC$M+$$@`7{tR!g6dH zoMEpXJAQGmv=S+TI%fye<^h9k%xd4_f+hJSPMwR7`Z(TR+YM7w>3PQ=6@*053bteO z`;)UNfe7PM|y4@vHx_XhGI=VaqT6aM!UHha<+8t^fJo#)* zl2(qkdbQ4wGrjt9aPlq}Kv(r?{%!mRJAst_5J0LYwO4Z5yga7qhDVMobn>NWJk8Pi71H<{reVAy3-+e4p4=b#J2A ztkRJ-?M69k0ED!~3&u3rqQ|>BMEv9O>OrRe-~mV&5l@8&w#^&l7jzQc%#Wi);}|Ph zK643u!sjS5!9R^%&vlVDsbvu~vgGP9u0LV{gdDhvhATIQ1yrj`7SSV+&gbaL_ZAoPs9LZHM=TF8> zb$=p!M3NycwX3lJ05wChUa}`cd&M(Eg;Dy(Q&6|oXvAm}PN*1}k}k=H=37{$IH3vc z2*B1`-c>WGaIiW1u$hz_)AuE?~5#)*Nwz_OfG;UctqU)3wiw?FEuCr zH~!*4oD$W4$$1NotKKEw6#rmiON2%h?GVGpMIH3C0{oSJ7U}z}Po|_w3k9c+5EnJTA?BiIv#O8vGLPHq2LJ9v-y=iPk{5Aq`hjTlVBz!C+f+!F%^Y8E&V6S@ zVgbLxIWV>KXL>Q!&c|7;Pe={?ph!|DZs5ujkd5TuT$ zET*bq4~!H@g>Cj9_G!q-I24RT{nv=iO@BWs5v*Z^0SzoXRFhm}#LHLHC=CJ!{9S&1 zp;ivdYyP%>0hAgBvP77&KAcg24BE5cF~Zg4(I*4~$PLBK03V8$5GFz2tY{#ZBdIcY zK8%0cx@TK&j4~`}loDTwwkCBEJ$6up&a6kXFl9FpP_Qo!qbq$Kv6O{hs8Pne8Qvm+ zG@9{N=^>jgI6aF7Eqs-mvib2ugY4QuFzrOKxo>Fl!m5{i{x_rlG2t9#5W*EHBf2Q@ zmBpqc5hj5)l2Rt?-rH0kNG%u$X;(SpMVLJo(8P5}WoGuJN07ga=qtt>2Uz(Bgu!SI z7l3Ugr!#ctuO{wy(dFneBC3z%+o=E73Di*hof@FDch-~s{gA>K9oNi;&``;JcG)Jm zN~eO2V zA)+}Zm8^V6U$r&wZ*x-s&yTIKug~_Fs7Pqt8%V^uYOxkx>=@>1DK(=ytQAMvgIbKi3~c0Ye20bE-Djy!VnG2Pnf$4y+fl&noO0Af@(1(XNY6p>N6J zWLu|(n?7%N*qUw9KuS3xWgdsuglq2km}?kz)QeBl7Y%cspt?VPM_;8>9M15p*9W`NEAmLn4#QAa9iKld(OtcY>M1VdIB$;F#pBgZfASlCzXGrMQ~k8ar` z?nxxD95Jmrp-8KGf!xP3*^qp#GbbK-u~2P`{D)K*Npb^(c3z|X6TWmJdAaCrr2KaTUfu~;^=h8Th2u9?qkupio9yV=C^t$oYrP~qC)17Lw!f3y1aZ{5~99&-q z<^ggER9E7Lt4e+fMx&`R1C~#mV~NT<_z1Vps4fzG zC8mW^eu;XnEz^Rti+v#h;M8P_%q>FzTi93$7(?AHfaIX)TgzW!A)u%M#rwCaG8@3^ zt37)T7YYi08Iw}9tcjQ(JynKpMi3N#4L^9VWLym^$(=&A z$OQ9ipv~dc7r};sps$64hy`iWR3eAii)i)+u)0qPhaUxeH>*nZm~gUW4T4LCS7(<$ z7iM!qNXK|=Bq`TB?)~cgg8pd*s;sn!ue<_wLS7O*+Sv?pg8HP4bP?^k^uYp6a(NJJ zNkfkzp&ubpU*n*e1J<6?0?CU7dbst#)Cb%HK12OZX8U*jviM`hgTz|fU|rWqFi)Br zL>f95oUn|ov7NE}=3@nMrCduLs+;Q!UbpePXHHXpXjQ2Rc@V+t>A8&Y>_VSz!T5x@ zG=1oPEB&50uo(8+yYIKFt^Lz&B>IKwyUX1+910jaz>wAtM+7npi8k;cW8#<(&><23 z5@#%YD!$N^AV*z9x?mgoWi^t!3xF~J6DCt9@VeyNXqMy8_5)Z#5qL{&4X~wD#26}j7ZgWgxMUmKfb<7An^c)W zmd*TmJy-mjaB5rdBY?OW($H>)rXj)B4=42vBI+w(EB$?_Wd|r}s)@4WvCA^dh_u$b zYz*Y}67*A?a)cR38x6n|toTd=D-@o({?W}HDxt!slC|8hm(LfDDJ@ibkk_BSz8$5j z9t`v($qD+X_7r+<>7*n@-iNJHRyEagq#$d$1lA-U+b4g0Az|jQZ8JN+Ofc3(VmYyc zQBWWrT4VW?a8K?@D`%D~!p1wBNyybxs|X#dJ2u?!&f}`B4qdX1RNsNn03T}Tj8epP z{o>kr@8c!Y<^-?fGfNpO@jzAR!1InxRR(&nvUE>q}!_6SOqWjZ0nnd=dn<_uHEBfURXTy10V z!Fcodl*hOxLTS$!?SM8FJ7+wOWwGFqb6?nUF2Xh0d+|#lyial@&U}9MW2&Izq?)T( zpFx4Iz2Y7ZD(v&{04p0AdF4SQ7AT45os=2d_eL%HZvNk5GBQDK zs($RgW$Qk8?C1xgMjL^od+#PJw_7i}ZEs=C;O4k}7P+SK}K2_6f?Qo|B& zT9^U^v}>AVz5qXNl7DF_-_?2_Uyyzby%TvV{(3io@is(J*9l~m!+sg`VZ$Q-Y49Sh zJ~YDNOu$dmVZ6R?*gC8TSiPbzvsRcRs}4_XvETB7^Ry!{IcLh;m4Q|kJ_UG`Fb|PY z3*Z!3MI-INk%aOky zO~KR%|D2vd8+bcq%H+-mr16U$4tNrQg09_Sap@8*qy>_8ofNTh=L+VrrE!+kQT*{* z22-_)ba$ANPy0%xIZ z0nT5@lI>z3!wS|_We486$>dp`yk`j9PQfjCx$R0LaJ}`}e0qW~zfg3o1WS85<{4hS zMv)|}D{-xbmpzZuJLCbB(H8nO2Y>DYa(E-QpX>^KSu4n#U^Q z1e*4|^}KEpUNz7J2o4#jRhZb?SN+aXv}r;~@`KL_{xArF{1h1nI?o^oK}`TUcO5Cn zF!Gnu2f0VJ_Hr#@=%t804!--PJR^%;Ld?iCrx z80?h48n?*58e`tCGq0q978@MDmbjnhF|c{Mct)j?3Xuxm_NDFK#|JfCe7=y@r?M3I zDcrce6Nxrer?{MqAG`vle(Y+%`^zL z%A@$)>E{WB{j?X~##jwVUUh_kEjd&i$*KzOeq3`jS!$!Ap~CE=IA~EQm7H)nyzrQNvE3w5Y$< z)$&yo&+P$M4mZ^^p8Un;0${4QXNe`5=Fq^cd+E{&>C^j~6`1xNJu!^!tO3TaJofv8 zlIPj!X%Xrc?&{%ZxD$JAT1?_*c4H)1$Mt^Ue*Q<(CZ(JtsI21ib#5}_6J00cJcokV zBevvHjMm&%&iq7D9UjFWYwc=>gPu|KNNQ^tSpc3_5VsnCxq^vVtKNI_Q{3zIX$(6h z_~e>Dgl%$JBobzE#pv^p8c0lbopuznN@u2?2QV?iWCf9 zY+EA*}>ADfb?gMFb34 zdDW9;JfTf!kc#BW!jt)LEGmORsl4x5J`K3P#Xq-Ks#-^b_+1~q?VN=Ng#Jukl5N3{ zt}Ch1n8X;l8`;{Vdce{g#coS%lVwjp>+t30fQ=wbU(2YNI_XOWI;`-r-p7AXC1o7u z_71)=nW_rLK zvCG{E*{676pjLp0ycw8GT&z~CjRN!tnejd>C>}!2K0-dX)%6t9{f@Sc#(wyXqu5#k z0dk6a1LEfAHpKRcF9PNAx^v)lFdYvOl^Z9ySr<98&jgL5p_!;woEmiX)ca%v+1hLs zj8J4I&CUFxnIkQO!&tdrAD4$2MQB!FY;Gai=Mu^gktanbl_88+Hb^5_Yr28f4lC}@ z`YW)VVs-~DF~3u;1(h|`X`6H?x8pRkM-^vZ*o;d#U`IV`&}Fd?}M5O`QAXPI!=2Zr8ic z$pRCzvw&l&^unZ+0yq(~3_&L*|KbtHEE#<$5iQnSI_u?Q4&k&g`Avh~6g~seF;`*4 zY!`Y~e#vLxq=b8JWI~G)rE{Lf=-n67sb#7-*U6+#E4goi%=2qYnKq0k_f39Q49y~J zl9)H9#%y}z(FQ1;^--Yg)jspwvg8P~IY9`ChLM)f;5-isRlwETY}!;L^Uxyi92Yt` zQcZx_fGJAN9$k<7au4%DPqOy=SA{W;I!5{xR|S=DEu|$MXG+7_pQemf?nzHNDI6<~ z?!y~$MK15AiHWe=+)UDIR*b!8xb)!z@UoM$Fn&i z2*5>D>Gv7_h@}J_;Fug)KA0ewey1c_leQ*5N|UClY>LuXpPSb|zvaBxPc|~p@L+5x z&IaKJ4E52pl*eLfNkL#U7EykUDIheJJD%|$JnV9f1z#>>q+)QEc74~RX0qLsJPE{h5|&a425@X)%IJ86)YUGbBeH8^qgWEU`os;5roz@Tx0@( zsyqc-E4o*AtCkSe@xbEeF$TxSK_NmvGNQ@gBceTqrt{xfKEJ3CfxTr}2PNaAC{c1Y zneVjA->6{G_DP~Qs_8bUgRH7PcqTXzrm>VA`23un-OnHW@Z!Q#T}PpX4)((g zO>-UWii?BS+Qjof>7qMQOO@xN?WT_nf~J1q##$xtqe}+&#wJDJ;RK?x zchAHGK&XK|#P3Q*I)%1*Em{2Xc)^V1-7gXH&PN}C&@&GI)XL$7ojF~&1@_<#%w2*C=`MURc@g_u`y0q-vX63K_Q1W9gFpzYNbyD_>T(*`>_B!ZZ8X8E z4wu4xrsnCMPV2f0IpbzT5)6sb1`+Ap+eat>luHO}tWX$3q?lbV zAT)Uu4jFoWx{J5_Y;-lz}blJ5!)RtIjk4=>g;x`aRO+?4F@UKomr7w0Z5@siEw#p~8TmLgBh*CHBLv8486tpg9YN4j8taU%tpqHK8?hBIs~ zK@wCTD*bZLAbSw^ezU5^tIr*sK9&56knf$k@-e*PNgK0 zn`>;qWN11%yp!J{!zo$`I2d~Zu|_9_6lZE;h)l|uAWPo2Hmi0SX5H@IPNq?= zwh_R~&W|!fveaXhfIlq7$DXB4Fdeu#*wK#PnK=cSQLkk3Bpc3NGLWWw3v{ZfY%c;JtHl!nQ2tIy#6u-C2#Fnj605j`JBKRyM?p~s<#jO3SzO* zXO=9=7FQs^o8fFjJo&oCFJC9bl&+WY6O*O__5-M!C2YT)q9yPSQSZ#fYfkQyh&00g zU|J#KNsZA2xX-B#`y*Uk;SJHO2z=D89#;D?G#ciJ?v#-CEm!5HA7b-tN#0%eGX9$5 z4sCet=)Fp6v45c3up(oUL&o^0m}p4Nyw-egXq-*F{o&xzFXJV z${nqQ{pdVyPtx_J{|#g)LWct{)Xht0biCspFqhY~bE~st{z4Tk(Dv>997jL#_Lj3Y zhkMRICl6UAasPPCgsCA*$*0tAQE<>Ll<6~(oi`mayNlP4w<&HTex-+)nf7~63Zz|sA!}cA|L{u{*`>Cb-G+8L znmXoy0<&vpg9GJG3H3AdfjiTYKcCXg&I|KN%ISj1YDq;x|J@ncbTm!EFRW{Dsz>;{ zeS?(6Em|-z^O}wKXrk?$TFah9?FSHSebCn*;T&-b(uZM}ZA&z>y^V5TztL{sX}zQR zR7<~4r@F=qZFOP8yme2{e3y0uL`W@r+JjFGnE>Z~S$O89uN^-kN38O6$XCA@pXG`k z=O`&1ZMOW@@?dvLc980I(o(gWL0VX;|7~r8tCM1PZC)2kD?^+Afg;uGfI4QyhWRR^ z;_!CFp*}krZ~yiRczq?=m`%fWmR3c}4z7`ZNa4lcl<(7}#VXN~^+;`E0WqxO#eLv* zai?{hMqfj(ZD(wT$>S0y!_TT;M7XAGmXQ|He4xbM0cFy7oLFnRYoTE)e~NS#IM!cM z8N;Yk8g8?qRg}4aSo2eqny(F&IhGhcivzIbg`U96QgvE^C^`WS*G=;AN`)Ui&bZKd zMmwff)_@bIgsSp@OVjhNg83zoo*Cj2v@1uXo^UeeOZb&s`TkdrmXD0zXfM;oB?*Wo z8{ClhC|yb_P;Q!Yr+~k=jxYq=ZZzC%lrn>OVV~^RT%STIq3jXj z*4{~6JWv=`p>jJk4;7MY!SjgUw(+1ex$&Z%nGX*U-N+X-r#=$!Jxe&2Z9vga7F2?B zA^BlGcYXo{ZN4{JU-9&Ole|e<^`?jg!U^jJEFhqLB}TgjY_Ll1@6|G`cvjz;kdc_$ z>Ol67vQNDv3>KVsjTGMQ-3fZ;^7vtd^y8J7lz5(TIbn?~g7VLM;|V|RY*j{F$W20% z`aV8qwhwOr_B+0-yqu$-f{BhseFE{~&itUcfrWP2k#kJzxKPBocHcqTd^6EcI+xCT znDj!3O2cE+n`n}A(%IaKPwG`?I0}cKDG;kJ?U3j_wv061bvQq{y?cD1d2-uhE?<*m zCm;DqO#Ks^Vh_{JdTk(YF!z^FX(3qvCbC8Z{*2HREBp`4P&8~Zrmu=nKVjwGbl=~0 zKpNlrnv^wEt9V}~40z2MXO+I*nhvjQh=90h4k_`@gDg`g7jM9iR(O0KdMyMF+_X4B zAG!#wzfBSP$(UI;Ww(SqF*}fb8rhm%-1HsgV(bzTl28MuQ#c4O_OpA2h9C2esovTk zxh)+Jce3*s)v%^H@Lmjvxq&b6KDJeQSs_*MMM@l1Wmn>qK0=P`oK;kz>q(59FQ2;u z>S&hPtZqCgFT1&TxXXCbo7(CJ8I7Qx6hk1X-}W5rZ*`>f!bqLSpB>azjIQQ*iLbPC zS63B`4>JNXi;++Gx9TD0GGtihRuSW=+1?!9g3VeW(|c1vJN(nQj>Fx_%dYH(&up|z z`KZxx^q@WIPq?Pr`D91%V-T|}u6FifYG>V}FPSg~6sJ=|%o!g}O$8l(_>!WLJ>wIy z8`0nH9On;l7IR9`_O;(rdp=?XJTh=V+J(?A&z!aHNV3YezRS3Ll57)WQGI_K$Uoc- z%`|DNOWWgC)e;#(4(M2Wi%O{T!Wkz^&VnrD!>@EQjjTHigy5i{`iJY z;F!E^<4~h@`n-}`O>iYhD@>q)s+O^=%McvCxkR*+4$^q2tO^I!U&0-@?|a#4^5~kb zE?yF^8<3t6#fCt?76~c4%*q_NFFVBXIXVnYLz+j}20)p*0NGQEPs5#jJSjjSki-bq z#=&P7a2-Q%8>7%*Ey|f{Ee>@Cwq6$acx0{7KrKIgw*bLE$7-O{#(`1XXH4&&Z_;P)SraD zrVECvj4F!Cz0zZacIF^&qgs8Ovjv2d-P|dL6PLS4c|Y*Cc|4@d^K|vVH!k!9oCZ3f z+C&^WAGMG`7e~(AaH1VK5YO%Tzv+^$MEPK{_hvM|(LHJoOyUD!ENtjkP)VsF13@a?${7XXtw@*=3y|xyV9M&4oa5Nv%7q zTuoz?{k!~E*~aR+*Iv~vSwsWKYNm{L5bdb?C>9E(h;RXwNAG4a?RzdUZu`k=U>U4w zZJ=H&{Rm1+#mIYz_-Uy-db~A^QQ+gF;1WCP-&ji+LPUOE0{oIP(;<8qR$0vQ^rNgC z1BE3GcqY#|HB#^lrZzqbF5ng21jC!yFzAjRZ5(#JKLhhjib5pAT5#8R>!}?)FiNS0 z<_mZZ!jw_FlkkZq{!Udut0YH~I0?P!Vamwk4Xg8!h#N`CXF6mVM&E|evmWP@sOG%o zB`6*J&UBI+zM62_2PLFJvCib<)3Gy8%q7@-H9>J_zEsM!pOv)vBL`WpW8sB?u_pVe zaw-HnL(7mQbLg>uAd-0h1c%_jXQqa*FNhiO=A1=sCC%@80v{OOm*zq^IYM4QU9I~| z&l9$+Jzjd)hm_Pv&WlB+8h%DhkZ}B>99uYx`s=Pql03F}`7*7|R>XKI4!dluhrMLTe@8!Ko>$>06Vngl{f}3ca$)d9qCV?0Y%k#;!F+KWQ}F4qjyK z>b1|yH2+;^>nuUh{bQ^$0Q)Q=8+cN03_L$Man(O!yxaL^Isl?Jr-4q-00i{M#;YxM zb$@^z1EEug7pw0~v<8m8AvyV&I%^ky)q8Tr79Yd7Tx2>{%9+4v{bavs)RZWYAfZU# z+M=UTuZZ|q5p-owUvy+H?of1RYT{=yB~}k33x)b9h)0E&*_TE30U5iRb6y_AS0kIz&J?Dl+1SNgNE8g&Eq%n0b(Bg){3U3h#in8_SgdCWb{*^4vRf@?OZ?WP(h8SCZkMzb^0;xWu9 zZltj4&1b4`*wDHtJPeaoZJzk2q@JzMInlFq18KI#95RGPs_a zILTA`>`BSU)yK7xdDdF|kjvl`G{LUy8YR8)@d3*|_9#F&NIMeq!Pd4*XXUF-0K0%o z2!bj7oxCf+6AW^w&?iivUqkJz;#~%v7jI#o=CTrHmtffz{)W0X^r?^gN&VvOrn+El zM%hzehlb!$-x&OaQg#R9LjEULAKdc|-zJtjDcblUwe&$%L7;R|Z~w0L@S=gO?u=v- z%SN8C8=cRV(#=JM>z!20?Jv8|1>|s=x}=N#c%|S@9JGxy76i;eH;z33;yr^8%mqu@ zbxH4GwYHkbH{q*wus?rj4gg=+6}k> z;v9VjYdp56YH?E@n#xzY7(QX%n^N6)VPs2sLE|-&K`6IODQ?>*rE|Qg8mhLz-D_y+ zT#dIJf)BU{|0+n#UbJM~AHr@J`KtM>3v-Y{^K(bk=&}Z~TJ!os&~-}f7xt-*E=uKP zl9tc+P7jqZLOz9Z8sOxJB?8_BLU7`WU;S%_-Q;ZX=?b;YD=Mn!Cf3r$KU zq9h$Q#T?^}e$$VW+*HB?q=SYi=HiyH(0u_)rIKPlvJq`QajVv>*?R~wYP-SsebxMP!9gE7qYK?Zvgm$)?>;_(kkeyu=KVzZk0SRX;e z2zBm)s6Hogz%pMlB9G}=PpUfUSnB+RijJ*OG^Y#WKE3po(JUBwf-1Hq*(UZ-KG(0u z1q!yRu(Q%~ntQZhLFZv`7Z*O#_PKTQ=>p^UGb}GSJ%76z_++5RU?+nd+sD`lSqV)# zbp&|YvIB@Y7uGG%!Nm;6$l>p&{CkAZ_G>XWTfQkeu2@l)>d3IDEs0$^b0emY`w&;c z2;11G_oO#2^)K^wWw&A0Nc0n7eVWIo@JYVPl^liMa1;8W*D7V-C=ABlK}yVSx3tKN z(&d4zi`M^HT!%ynQMR$D!#zJ6!o(s8NnTEJ01;xvRty!b3(bx{ zG}hNyU@#F*=CD8B0!GENkMz0ceT?aa1XYKXtqCrT)ZUnV~dDf2LK8i4h z{Cc~9g6$1=74(v}9xqym(=NA|+2CPilUImhLCNbj>Kuvj6>n$Fg^7^ysrvzp@9=RV z#&vZSuIq<*C!MJv@{zufc8@z|U#nE}m9fqJIyf8;r%+ow9~Q-YVyjwX+0>LT;Xr~3 z(i*<;I1pi8h{>Ma=qcDRL+vLQ6Y92Y&WHv%deAF@SFGK(V5-a0xQ4Xiw>W5w7UE4M z_^cM}e(FXTswVZC7{(A|{b-@d{yV(Th#U{%tQ*^5ah(L130%Ql(faWcZjxn8!=R$Q zz;WFs4RlBv@Yn)vEM2p!yH>>&H0kVCJxm^t%%Z$v8l7>ovQ9Sju!=Qki$T=Zp!Fw8 zS{j{CENUxK%2e}Fuc}f6Z6Z5TXzv;Qzy?SV3;2EHhnANQxXzL~?tNTC*y2I)m-!|f zr;xtdK}hYjXxP3Cje=vmkw9o|&=Ex;#7HdpC?06Xw#+ec?w9IuuQN8AZ|C4+Nc#oC z(D5mw=4Mhk2i-GJdG(O0(*L#fuM+p{&YO79jot z@Xb#)FpXJxG-}N1$~o(erhUm)No(i)Wl@A;3AGg%V1qEJp)?}P&dhJ5J0l7nAJ@=S zl9HCRz{!(!n9oSw^bQ=5zieQXM@oLk&@q!ZJQh?5FGq{&t362S=MuL(`TqdTKr+95 zjV@IGMT*)8XXf)Sd+mIXyXVD=b+qC9KP1{?Eoh`$&&^F;bZF8q9xs7+&ldlb&s}r9 zs>bqR^Bx9*mi%qRddm zS)vuku_ecVwHIbzwyI`A@-oBKoxrA&8=1Cgnq&g*sxtQS+BnA@V^Iwi_{m(hcH|MwQ2Uxb9BZ zco>qTxLh+>U3|cj#E931oU1#UMs)MvkZsI?lx_A^zvEm8`PFw~v3@RUEpKCVwf@P> zGxPB7Vl$W3P3mr*3%UHSUL-8dn*ELKKbC9am5%jxL6HPrIZS*re_j|1Lb)4;Bn>;= zAd%HJkn|_<{hu$PW{aS1%1Etc@}KL`ihxCVVy!fcOBjiAX!TGxY%QFN&Adtv-48jI z={RN%+yM*3hghmeY)#d5nPUw3@aOkN;vBj)B9R2fK8!g^;QC5y3~|wQkn@*ndN7}H zZKj(TH=!znEKxu?KzPkjE|3Eyotuq(uyvKJEIQhT3L*2_fNQX^VXbE9FfI)EY`>`hPg9W%y(w;j~ ze1AMI*bYgrtB`sdHFsKE2A2y;3;^b+qW{}cBJ-!}Q0DD&mgMgaS4GB@#v7B4#9idL z|B8FzP%gBNmHN%bKBsKM3d-25*m3$!aLl=W$6UB+{nLaGI)}lPn>DZE;g!+_l~yaTPe|Wn~W9vYo{KPiHx-`Z7@bmhv2!=;IuQ~N zC3pDOxVOar>to`rnBg8rxr3r9wCgRfUcV>4e?NlTRJ8R-Oe11TnSWG<;?Lgnn1peT zkFh3&NSMFkWGYRkiwD`Zj zUHrd1o;)N5WHdDMFVyroM6J@Hk||KCbdw&ZzScNDkvs=cTd?lWx1s1OMoRjPHBGiI z5g^VrKRyN5?x`AEojLqKCAifn!9gOMjjVRAd*Yj@PdHifMx5`a5l3A@#v0a+ep{2F zWAGuh#Ln}+b!pj?pX)~zpMlFEAA|!?y{8hG5^RcIN3jDG$b5c_f z-^9c>F1irMHbt>aH7>Dq7iayB69oOGSKC>&T-vxeP}7Hvp{HV2?08%s8Vve?drJNl zZ}mL)Sk<5*PK147ErGiF6eck(Fi;)*5{)E=mBF!_i)KOtOiPnG-p6GvwsCF@;Dqe> zC~!?aSIZ~QTew+{4&DyLxJyhvx%r}u#$;BH$L1j}-&&XvD|$nvc>jUhBdjv2Wupcb zKha*pB-(u~@sE82N?xaZEioNjF53I?7LpibOVc@xa?sq2lFen?v8SY5XXIlO1lKIi zy=o=`pnza#qp?hC-eNtFrXt`jPysFRclOE@Mh-4GD!WHrdq}_f2@l%^BVGYB&UVJag zb5KbDt4*Y$MtWWIo%nNmO7R2xXze0T2CU0oY`(!=dIguk4f#H@1UL%6QsP3G9gSwKVIfsfmHZg?=D@BeG2`lXzdYS^7W0x zv=8of*A+ux5fTG8kO=&=)f$r76$>^q1JPHW+BBU`7i!hw8h`yfMp9ya)vmh%KE`C{ zE&9uy2=#Q=bnoQB)Q{`C=f$*;G|f-_1`{Mr^D&Jc1O4G@p#IC4U{co-vtnb@na)~B zVh~&p;R$bkrUuokAjPc71t*7Hwrj1cwRHdVL@tFce2hTB<=95i@1|zcbB?r=fL#~8N`;B zQA=eNKiKT~knzr)HskV0ZEH*}7UTvhKiScB z@1DqrJ5mNbI7d^1m%elcTxt8DXzz{U?*g~jSeDuJp-3c!_qG^`G3v&E^t>2CTF?S|<8J)4!SNZ9lnGJahiQ8c*{zHi;o@WQQLjPXA32H*;=^yOXutcy-h(E#^*k zKM0B{qT(ba{&(zm+@+<_wI#mbHkD8On{h@=nXdI7VlDB!I6;s3C97OArSRWx!$S7k5i)f`qG6KWe~9$B=xePzRCFV<*unzS{gl@cGupi| zSxDKNmrC^y&r7!xpLP?WFx@v}qELW(-d#JZC8p;^-^57FULY|)FC;O~pN0fFr+~!V zpc&D4{f3F(!oBPrxU9|Bq$Huab7jCo^QGj;!;!u3Yw2;)%difi2s;)H7I%$Q&Xf6% z9|nS0DLpSA32V&=U1PePjMVGQ4B1{a1lxeb(D%WbA<*&&a5KlCdoI2VT4Tl;2KRTpF}ILd#l8k${ z)}mWlF9tr~k%3!D!x=wH@Yt7d9x67kXnan~BV<=>3k1kJAPTJ2q7}{QY6$lR57MT) zKdxM=Y&Mn4M48n157)2T0qIy&4k4G6P0AxuUpslIC=RWs(`^Vcl+8ns^%0QGlc5nu zATZ^gnGDz1LmJNeLxP7sfFN&J5=}1KLK>3m9+XYU)S^F7)=**7^d}~x9=eC|hjN?n z<7&9e#rNI)l6gge1irdWQ+;cuLTJQ8+g~hoS0G`-X2@Pf&z??LQ!Kd#za4tRb(lFf zSm&-4euT*w!jRp+K9G$68;&|zhr!x6+AVj0$BnFah%{U}N5XsDqC}@P6aFkOx(vu? zTnP*1R)6)JOh2x-%shLT7C8;hm<$pz-c6;=QUplmz(V8k_$(%#U}iab5c^moC?2Dw z?(yge_b%M)a4$rKtH!o1?4kb2K!3ce4DPRD;nzF%SC=p!l2$O;1e?R85+Qh`FmZ|2 zDRkWed4HxE&s=L~Eq-3{_EjK1MOx6^uj$8|>$1AMs3phoDMn&qewAu|Uy8;mIdE;? zUhoCe0MK@BF#^(PN{)@1J9^Abvm8HM?5TA~_|Xho&^Gx#zfSYf`$s(|l7opNgpDRr zB?kAp7imY|L;BH8V8y{A-Fd)X2jT>)c71XY{M zEHkN|s`@{MzS(lg4Q3NrTtqHuePO}B*N>NUq*c~9m`uoJJyOPk>;(6JLV}0CAx=+J zlSjCM#_Ozfv$?TZ5UN!;$%~FQ>p>PCq;9e$6OegvA4z-XOxM+sNhPY`I=0?HP~dz5 z&G-Ll%3=@QD%*lEx5qr8o5sZVU^#(Y>BBqz9Jw@z9A?IYZ28|jQPN-7A9dT{PL+lC ztQP5`d#xwI!(Nx*LC{NLh5*1BpG_hc$~W~!xSy>{Fu;z6dr5RPFQa1Y8mFgsbKd={)F z^QC^`cOXRw*k5ePb%_mK+Uh4EQ(KpOEhJoc*aL{M0g0hkDZEbnu;xapYxFv-z11*f zqy(q^s-7xZ7Y=(ax7Jo2;%K&8=(4{&SftpoNwCvxCStLTmevd4=2+9&A&qG=Y}T9k z2+cYy9(>%agUJRMi0lTxvr*#_M)r&-1GzdHHcc@>npXe&XG_{gSGz7g^g8orNqDp4 zrT)qiiEME?-h5~+Sgf>J+2EQB`tls-E~;_4b$@dPU(Qz#*NDQpRQ1iHFfSuEXl02E zHgaJU7p(dDd987x6;gjG7PAeHR_&Nte_Eo&GgLuKsBR#LG zEvA~H!l=#GA9tKtu(xrZWjs-9-@h>_h`w>yN5gAiQ5JAQUU zjLYG08w*Iy@2^P|X}V9DDDb+6 zuG2o$9m-sbf%r8t`}`p?|GquM*KIITHgiitkiuHujS2>h8HS}MZNNr2z5=zlc9uC8 z^+UfVC_!ReOU@36pDE~ynoHN?J{eMc=fRrkjMqvVvV|6_rc0U^d*iHi_h`RZm^PHf z<=jcNVb2n8#SIry(Jgr4G%0)W3>2n1MM_^jORBzm%q>LO+LKjVJvPmbaCc!f>t;0u zqroh!MYy;rreEi*KV|N9Bc%KT_}_CQ)4`i+7OBV%LgTaQ2x&O}GYReYxH~D(b*N1i z?xln50nsXb=`0P%D~0@)K7T5Ts+}lx<5ABho@>52V}y4gf|>&hp1cmkYH=! zl8ymM;ly0>)TNkgQLu0kQUr?+7U=RfFOl-MFT*5TtrgJcUb_o^BSojyKC-q%1T+Zy zM+9t_xjYs15h5EMD-9=qE#aN7hqe+i5OiZ(EYkGx3JbpPr7xZi`NG_YK+*CK zu2*?kE*V?FF0{>QR8yj$9!Lo5(Ja#vPsIV#x#D=uWC(etcCz^1-KFS`?X})WM{3S0 zTdTY|IejFw+ij@D@D0*3<1@MqX@ZlsX{@bQJL*j-#T?fFUw*f4ODi{KmHO}|-5co7 z3#X$%<}oNXId0{ZJxl32*V`zHYDQwpLe}kI3YtzMgmWa`Ph?^Au!Ixd)6k|T zNbsa@CA_F&^UCc;2^7cAc(iulX1tAobc{`pd1^Znk>9|+!_NPeKx^3{NH_R#R~e21 z3mCffe#d(?xfZQNv3uw>6G>Jq&{vBb;XaA?@}hRcf|t(L z6n;?zXf=`$_6^Sy|A}iCX-+J8^kA8H-=6pgeZh|xMc=YE*%28jc32pE9CI7ezBcKP z{+n(>dJle2j8eo^50g!3%d=4__$auI!CZaho8t7|pi%3X6i)$ptNHmuST(-Tf^Ff_ z84|&7x8NJ)BqnQdxs4aGT4X$8uPL*xLDXoTUMRz{qF3>mBzDb(W8DM=$b9@e{-)1w zBY{bfTgtizrhhF;2n~_H?|R5{zk>t@tgpVi`tHPv#V(Yagz01vnOjuO+k4V$)%!DL z=+OTm=4E4v4A@*6PW@Jb`#vI0;gFbI!Y#5TJHTvv2tcj>^L=T6`vSFs!5M#|%J?Hv zeEJsiZUHuqY2&=qcwVfyTD*8+oO~}PYYF4O$ahU#78_+SVV)%> z@#|{=xcD6HZi1-{sD-b6(G6QmaKiT>A^GlHoG_oe)+hIPx?a@$_>3MODw?F{dvV`P zq&&^p{FsD-u!(bZlQo0$J%18$OGZ~T7GK`rNU6VU8sh(cmGD;Q=-jU{Uo9zX&AArF zE;M^AG}}LbM9*jpd~Q6IF0rw(U3`shG8Tz)tLtBrbvAZU22hhY)%}hA0cvZIJT*-F zO*f+P!j$gX9cwD%T$d?7u{oGlm6%*EvM*3!imlc0`>RW$2io|3Wd6ek!L0~I8gZ>a zpS5+4Odqt7J4eR%6|p@+<2_x8j?MkNZ*0#JpZlAGb5b4SDiuO|-X#r3J%w5{qa?id zgW|BrCoO=JN{|?LRdTzUwb8-S5G1C}neKYu3C zc$@(yfgmO?T4<_KHJ)x&I`LtGil8u2HC$&Lm_2LZB2)GCLlT^fyxp9Tu!#@O_yd}h zceN6ArV1{U;nL}-K>r-B`M3n9{-Q)d7f(!h5v1a)K_R2cFYm_&;%m(m!OtnU zwST;Wdpx4r84bXS#i=w*{!tqK`9iBy*FqCs{o~WfHIB;7n8+F?{eX?`G|lN<3-e~f zl%LgQE(Euq>X9!?<<}1)rQa*Ar3MpmJp`6nDR_JhCUIu5@Zm+v+Hij@g2ems`6HD@ zs&%jW_EBsE-dBxeoZ!{`_8K5SMU-A7*ePM%qG{UcwYG!~dIlTEz1^ju zP0~j$-;1d0BgUimr9VpV%YT*LSB;YF;Rj(HYSo=9QeC3fkgD%6KJd%shAV=CS(V>B zEH%Hq;*Mhi5oY!0I>=DfHxGdvp&Gnun(+e};>1<+>#G>+N05vDV>dBE%%z6DwG9)# zR#~R2m{gd!j|4V(ygLB40;RWAw9{@j#5(@`D9qx!oibnfGrYXMk}1vYhD_o}{v%wWxpvd$C6 zf@^T~56`&kEaX;qX-()E@9FE(2lD^5^!>l7(*LevNguqqZZ0X?4HIK2s#s=+eDJyZOh!n5hWNh5)53M=kOYd_tjcDvFmHH1e}*hNc4Q36T82r@kdDu%+Eo@$Yidw znWXhxtsP^4+jQi_Gigp`W5V0kxHXHuF}+^KCArAzLCVI#3%CZ+<2Mv_3|~(=~KK zEqUl9_&e^1zd<4(OTI4YV7oUdYb}5IY%I2~fe~*cwZ9<`_z@RN*4n$u{6`M~SpiXC zt-K7c2gntKQ z!PCcM()w97+YQJSPQKvp*&7?%(J~*!lIkb^gzI7BP!)tQZXsO`e+VY(^?(=dAB#s# z8@Q%)Kl450EZ<$@6nNc8MM!eC*&imxQ)SLoJHgxuQ{y4`V8Z@I7Cd*7glEsd1{crF z^||+|((95Rb)z}=nsp_u|GG+sLbK;#(LikccE?EJem98&H=^Zfi=1q%WxEP!=K{qX#>+XQ> z;e!wN#(Bt%A1(tqk7p~!`;)W%aZ(7={PN1x^2i{MT_nXa=7rijv300qotWfn%p4W? z_BO8(< zT~~UZ`-#lEb7u+77^^|#4hnTMExwx8fyA4>TpT<>L&ju)^VJcaiu1M*mppxkcK^X!kL7*=a; zT>ITV7i&PZ%*PzwFm)6Z34o1R+Pig;E^yzV+ei_L)bUs^Fm9Lzg&6LUK6Eqba@c*i zf0tMmqUOugVkwrd#e7@U_HyZD<{}Be!Qvb9YKx~>rqZk&*^#QheNqaKxL2|k=)-bFW<&#S?wfr``J|Mw?S^33sg=WrZ3Gv29;VLM3o zlU~tE?Xxf04EvF7K*~XsE0A>x`qsR?uD}`&*P)isqBl*ky!!4EJpKbWu)dv}%HYdK zijQf&ford;5X_ERDIGnNR%^n}J#eIKwpxr>D6^4twdPHib?umPboK>nK*5f<-c@&m zt1A-Sw4e;8jwJ?QaT8yUSb_zQA1ZUO`Sw7&*yV5pT4C_0(J6oPMpz*ZkbVz`rPsAT zD48jPU%!VWgXUYDADlH-dR_L7^t|+KOpFMW#=@y-f9Z}DBhI}pLjfcBD3;<}vfX}n z&yxX9M__VX7k=m0s&}D`%-#c^s)4R%z?@8 zBB}oRO=$GEvBWBgJn7L|Y!30l_POyF$y|R2#Iej(?PT{;-^2pG9wuY2^t|A0>3Za2 zszqf{qk)f?%7AAakdxCTIDI@8-jS+x_2u_ct!VFSzl9&=&92s73y8yK7i_)!oh#MS z(&OxxWzh4rAYdR-Qzv1waEm?%31Vo@-}wA=ShqgGCZM-8V8cL5REG+FUDnIiir;fQ zLPg_b2FT3x6T8TaGkS<82c~ug@-dkjTp$V#yhFO5@wA$T>tI#OggpQEJe(`O|Jq8r6x zPFbh+z2^mQpdi*;cyAbS&do%cS{ZA8`cTpat&c)qqa_D^xw@dJrvSITcz+u6N6-=p zqFpjw&EtKp`wr&~#X|oLDjJW|_?d9=TqSM&??H;;zH1@f;|3trP(CatF>b|S=}?!Z zRgD(i1<#%!#kcRE7PZWcM@SZ`KYLJUmN5z*Z1Vfxzd-sw4mXCr!;nh!KDBVDRhIJr zh)%)&|0jL#_!A1A-q0SSLNqLb0woExB$M;Cy_9x>U2lD+r?l}#vZ+C+o zU@oH7uK@%fC z%#A(H1%XC&>zZF)kb(nlMG)+ zo%E>2IaFZIjQX>1c3#}qLOo} zZ(7$EymGF(F!sLU3(Uv!vDWW_cjQZ0t)Exf(u7_hFx`%N6iR_tupYzu2M-PhULzeQ z&bL*h$u7Di1h@wHyAM&BObPGHRYHSpb4T?WQ(IEt>4;kx7Y=mlkQl=XT`EbVFAVU^LM*MHJ9|) zr5gp!^kuJKgi5Z5VL?T(F0K*5J<AH7*w2=Y1`0=F8pm#E5$Mi(&;wR_lb&@{_bdg{lG#Wri>Lt%b@Tq<5U z8&;g(^*QN6##DZsFq!R&{+M36&e4R)JtuwTVQFFfg}?hC4fJxbk`yYh?3+`Mg!2cvw|-*pII9n;93~zPC0#)l6<<1u>x@3 zHmB-h-KZIJdCWp54!W52UPCQ^e)N&nJj$WQaQ^gAl+dpY}#UzSN!mJ8s^w{JS=PDLl8*49p>S^T>bzB zAl-e#MgQo$n1f=3e7~|WKT=sq8B|wW9Lic8^oGz4fre?pI?7Crb)``5pk7M9b#)%i zTzfaYR;UPWwU~2t?neeG5`1RG(wGZat)h&Lu(dZ?MLD_qrHaS4$U>MrFr*ZT;R2Ge z-d@uC<|%N0f%$jT=UBL?!N`Qx1SVU~A@mIr|W@MliW>fmH&%Ok%79ZZI zF$He0VoZRjjcTkmp73}4CNSUAB}ja+sO(E@k=M@}tB@y; zu%>KJ7?y&CH*b4X<^_4p-42;a=+hd5Ut>?=_|rN=vZ94RjV>(4r7>Lm$A53sR576a z#< zR9nm}J8njAj3-?gK?>4W+X~v_lO%w1;ziRmMv5~6ioxdkr!m(F6PmAP`}NM&68n{om!9igSi6nOB!woyj_fuW*ink zU6vqQU5|SPGI_Mdh%9*aEVwXUqU&bQb6-bRron1qE536F5M{V9Hcc_l`3%+N zt8|~ML9(&F$Jc!hB3Y)3((RNtr0a<D-A6{JuV%I_>r4o0jh&$ z?+?vx!I*;F9nO+&r{bKGUz0-k(569q*_g2;wg$Ts=%O1`FWt@RIa$Iq4Xu1F%-SSx z1qaZfszZV@4)s92!&0tej=MBN47U%7=7axTu<<`m3xZ{d2cOV>m1Rm&l*<=nV+MJ!eKvr+tw z^DY9mscGk}XhrOLGF%!@#Ta8x*Glrcpm=S9jQ(~Z-D7DRSa zy1XvtDjg@e31iKB1X%GLe7Awc1>x;L4+h4dd1FOWl{m+enZ%VoT0vbsDe};N$P_Gl z+)rp?Ac$UCF9g1``g?rs)%Pm)Br2k27d~k8AM!*zjkrk5s8;^ECK<{=K{6_c{n@-U@g+xYWw8EC$9 zu%P-1(q#VKyTUrNy%a-JIQObyTH#eKAo0u=+;Hgakez}`N0Kv6^dMmj0-SgEZt!D2 z0WKbE=#*CV@lDWZ?uiU7_qYPfjRjK!!aayCZbf*eig&JZ=SAX#3EPKEQW@*+C6U@{ zXs~vYdDpEk74KgUQ#exip=c7*^+kigRkfb6Ewg6Az=G0_*=;)98%f@c4N}n!8%fc1 z>&cv}hiVoa`ibjUX=+7xqo$iVwaUa;f@VO8jR$3h!;aX8qMJ9Bq8m4oa0PPL58G8^ zqqrIL<@xaWCNlr79iQ-ZKz4lKoSdI9Ol@2$)?fpQMi}-k8oak1XiR23$ud2`Q)lE^}&TvJYJw@i;FkI%}vzruMHB?IA z(&Z8kSkb5{M=VMS(lXDzW^HV^#=^yAZ4l!j8sp;1sk`x_d@}ghL2V|+CsI3JeDfAE z`|{OQxkIb?yqh=IIg_=9D6?sU8C<=dWURKfRDN{3%)4nL$mj@InFnZgJ54`}ji=7R zlr^0jIp@P7&N-h%8+{S*61o-@-L$D@81rG?<@~?k;r)@o8+g&BPyx435RHYckBy!_Sk@NNQ^M-Bo8h#Q2(l!`GP&txd^0;1d;yK{A-7OA}l zKMC)z8JB=7VvCAy+!)wyh>T+oz?J(LSnM*S?6nJZEiJ}c%Ec`QDP}bQ|E^tBCdzPL zxDtg2;V#SAQ_5rMV~1$UY<~fh0j#d;W@4iomL?ETO9_>hBnGJe0g|594|5tmcIsd-@$(+kq!@PKeDuM)acL4GjtqY-u zVK1pc7Dl>@$Im7CPluauaOOnJwOe7&F&r-8Pr^lRF!me=YP@RVJH?y3&tPkB5=yjn zDTxvAopt`D*R*Q?XvW12{GZ>97vF<`(E*zEW0BN!%j$r-7tl4TSh9xgjKv8x7qEHq zLBlNGYD|p%G^HG~CV2v|WHh)F8aHk^TO6hS;Y@*8JMulmyqt~m2f>RV18zMh!1w<| zchO{()2wt{e^)nuJ0`UZ_&_s0Know?dq48QGm}{wh=mt9(3#zfx$pzf(B$uZon*i= z<3+$eYqW47r;83v=b3P8%Ygd^18fpw|zA7U>$22MdokJ705^GY>g)h%==M1H#!~YCcjIeJCDZLH1$3HkFnxLq#lb-2tMxA zI7PO=Mkl*_yBD0)F4L#JXpH#ABDE&|nK5ifb+cMLrQ))LaXG)I3qi0%-46ZT2B}*F z3sy1yCD46UZ2JCDO(SDf_@-#w3Swe7CQ5n{&p@qSCSs@DEOe7$UAbR3YLuU>;rk-z zJYsCXApBU|DQDFDv8W*N4u0g$mm@DUU4qvMu4)<2ybi~n3_)H09 z=7#@6rmj_&Yon~`bur$IOJZ7CzSo?q91m(2cFx0iah!LB3m`HoVr)5=dXUaBeU0s*sDWl01c0(kmo!z%FlL3s z3G%}6X5B8$z6U}>m&KfIj?uaalvl=cIT03Z?cY?0DzCy$S;`suo>%e=Zua3-BxIgFGyqyq8otuw%#SL}trxx9l15G^e zp(`-s!1%0beTU-jt#ocrUws?Z+>?lNKFCKLQ3jG#Fgq0ILVs7hdu!s~8b63vIs#Y}tc%b$c)@(Lfrl^!m2jO=U zN35jfPh;JXwe|A{HHm=Jbv21>csS|@wS&wxl^Dt()?#nbUuvDr+4i``J;_STpj{w& z5a4_OS+Dx^e#ju+@-BlVGW@6(mBExqv8*E!FN8)dTijooDPHk*s#ZqhOQ2DtA8br> z=l+d+YpM>vEL;+{5GU6XWA&-i^pQlniQ(Jbc2Oe#x%CXwQ{)Q!GM1HP19Wq>jx=9;lV3+D?^XnHa|A z44s4GsabWt#CXw_Ha0K0^H=N`7T}A;jf9cCa6VEG`%vei*A=7Seu_+Zl#{q-oAVGe zD{{W!+!}9f(9ud0TNCwrUR@P| zv3{w0BljPyHAGpUQ;WNgaFdwuIUuj7>7_AdoJ$k;0kOS`5_Qh?JUSMm+hW!jOR>Bh zZ>JsFDhODL%5H`AU#JfvVHSf)duJLa-E=LO|@w~XZ7W)_L zXKbd7^=~1uaQlmA)BEFZ{da5;RW~2^dv`O+=UNn}#(wiV23IHexNBduU{g%%Lz8w4 zpA{48xZnATo!t@7BIeXqC;qsB{H}~Pl3yjr7-J>4iQ^HI7s>!-g^Q#X)8fAqWQpLZ z98>nfwMvxYf(g|OJ&JDK zPG+2xjkvDYk<{Q=1b!de>I9}E%Fi;NNnK)Mb3^=Koi7vL?bw*&xuL>l zOD(Ulc}ry|J{NMsPB}>=FpJ0<UnY>ISy_^TdD6jVtoqvL1eU=dtD--<)HX)vslI zpD2rMvIZ`_KTbR=rp|?J@a&6qWqIYIJ^I+R(<`zESV@^#nbSG-HD>7Jn%r3EYh|w5 zV&@_tVBu85nq{dlSGU;Hca{smk{vJmK}7{!Xc$RV7LS_7%NEHu?C@&KMgWnMo=G2 zy)7Yo(VRXlKJT?UiP72-*jPk+F>u*I-IH0!?lE!kJnyYN)-=(^To79OR-N+tn1D28 zCev4PVnP5Ht70b&VWc)755dk%GytiupY)x^1eo@xm84bU)R;-o%`DYcjXvv453N4; za9N4=+ZY@3wz~TY+=nul1;t8$#V7#R+`sX=)u-hq9mS3!9*435k6@=X0|AhP5DF)1 zqx)aM!1mivg5W#7*VD}e(_hz*{cvFh&t^P|9g-Rd&^@&7?+04tHHElc7-t zd#$TIaZ|8U1{v36ts^1e#hBJ|exVFfuc+&B%gSIItd^K58GOI`F8UBrRHm;R`%x_W zMn@X1&iyRRZc{)03si@np`JX^ew%1YpRfhioM!MdRtV^QJ$4>t*d zwT^*6tZH}t=yziT3Uavi0r)$KSq$Z}`)cB36ktyDNG%p~w;oDcT4OrK@F(+AaGt6A z`xAFqS!;O}Wc+I{v4k=&y5?T)1`_0o>v0%q+ZQV!naqG%-~Jl`SYg6d#ZvbdNeq*> zK<0++=f>wGL!%6?ftq(rRs$KV$GZ_OpJOtZ3}e$$eV`0h&zJhY-gjk?xv19Rdc5Sg zwiUf?8;}@u4cf-Y&;!IlicAm8OUxP1qH3D8$y&-Izy&&tVq)d*Tn$oy#0hXC3$K5q zn^~sad?pr!Rjn>*xf0s+6a*F#gm&XC&QUPGWgteDn11_D^jnTo&WmzUhkl$6c&4sY?r zyt!a`XbH;iCgF7t6c1|cuwY}wyO_Ia0c={$zPT5*zUCuBnF#^XrJ}*5c{KE1JWBpNXuS26Rt<&QJbZUo95Mmy4p^_TI5cc zY%p$`$G>VGiWwfSWYvQLc+tVzNOZNG6S4}m$PHKiBB+59-SBASDn~VCOy-dV)1(yX zf3@G-G*S*t1HSS&x8)VM;IRf!Zg*4y|3=DnGY0Y;K^=yTP-7{FlV*FVyqg0Aox{=B zEzc28jf3~eFBKpEpUS37WXvMd92ArG7|xMU-c_JJOr!*33OAbQ8aqq0knUv5pQgFR zp0mC|BHLU9#SYRIgS^y@eg}n+Z`3+xn#$NVk}U>xn_Y-m2C;GE7*>9AJCq@>x~{2# zGzo8a3H%0=pgv3@7%GDo!0jAm1W-Gte#{3T)7Od@b?Qt8+e~U&_C}2|*c4J815n{J_TfP8Xl6)&*kp6df7)+S!LRCRLz}2Bu!X3NV1qhC6{ir>x;WcTg(dsVL z_GbG+HHg!z!re3{0U&Y9LT~mOa-R}O>${e=;w1C18w5mZ12JbVktV2xZ<%BPtU0T+ z!GWu#YJ}avXsnQ0a*yorVoCSsxx5Xt_MTK=LEY2Z4r_iqE588oB9g%R;zPK|SxD>U z7Iw?h*IRDz9Ted=8~}hhe7{rDFEJFeoAeEMT<1i%9$o;D;H{ss{KLORlMRNalU-SQ zH!!-ez&nUd{q4L>5Fc!e#9#sj@e%O%7Bv) z`YX>499Uj@SN8yL%GN)Sl1ilD5*k55!q{*VcbXJgB0fL)57#w~TdWMU+b7t9X(P~k zo!Rv0NFf>fBBSlvMLA^v9&XktV-UPSw?Pz6KpRk6fI`KN|^Q zGkp>d)!_Y7Dn7&z;=I!4(R6M`3mIS)2j2?(?4+T~jdqh2o_1eM${o?Q-8O2r&$QBy zD+i+o4u6sujD>elA$04D50`An{3rMWvV+kj;?tdcZ8_0vtJ%J{udrT`(i&~bA+&zL+$l<-cq_zFZsSEL~;bXnQ z_2m}MVeFObggmN6infc2ur|AG!&k=6yyDmZxt+(|Gae&?CLRq3FZ`L#M`{~6HpawP zB_j{0=s1p0XQYj8o^lS9;8s76`8gEzc`~}{zyRg%>ppb@1hZRm*~Z+3M`XIAWGLi| zK4iltySD*UP0Q2_c)!V!ZxTlK+_Bj6TC!MppP-Ph{1$aw$-utx6(|P19P;&X`j>FR zoKI_AaQO$*DzKBoGm{q+$W4A}ZIwCG10QS?h1P;a`)maQuG)Hhn`wZN1Xq$)Lb zRdm10F!s6^L4rv>2=DpOQi4gv4$lo>dpU(irS03X(`HOaF-7Onz80edvZQqMTGwml z+DveNeG7{tx;OuYt~!@HnASi}RNi>aWpzX8_GaEy(SX}N_O%Dr9T|12>xC7xaS^Z) z(v|7CjwXouUa9Vzb5Yp=y{&I zV9Wr~`_gWKOT%A8&Yx^WRK|Uq^k=M$i!5G0IXnzRG2e0jA(tKMY!MtY$qs@|aeVd! zSghAPeN513d1FgGHpd=x{=Qv*SZ3}az|R7RY1AQ`-$#=GqgG8rSsWiKTius;*t;ej zv!-SfiUZ3##dssZs!ZYa`sL%?kpfAr#>0+O4`T%d4Z=*L9>sn$>pp#{DCLn8EFI{) zHaEV{EnqYHBT?OUq?^`&m=^gA-CdiMUWs#MF6D^YXcSgDDyBS$QQ&ipFCKL5{|XAF zAI-^z_lF4wnMB!XD-4PYY-@QYz*J%Dcjnj-OI*jtdV7vNz?J4)Bovz0B|5-^z*NMO zKc+206l4!lH^-oo5lR(Z!XjSAf<^f6%FmrP;V7>M@5C&wem5(S_%WOZpME0}*;srS z3Mb5<9la&#$9@cg^Lx5l1DQ5?f_>C@I+EJRY1pG2P~rvpGn|@q|A=AUyiBN3 z=uQG11B>{he}o!aGGPnR#rA6(r>3|OeJWOAdtC9r%aMt#R& zvdi=@JQi_M4V?jE@NVIIrUL_GrG3p zr`@58Xl5w^Uy|d_YL;6{XUt-Xp~s4Ol9kb5?rQQ!nW;y!X{WN1T&&r^>};(D^EZ*H(a(WwHt4rj z+w+d^OUwKH?8RNpqLyXu9xArL2ROP~xK0&gW`}F37?hh6K%Dd0&45Zn_8x%;5tI1_ z@h8sZ6ueJ{%j$4lkY%q70y~uL>m$gWH3_!g1D>i}Aqb`sxZy;5h@K!=QhKxgZ2jan z0(Ab#R!LO&K7klGaN4-U3@ZYsByC%TFMHBSBlz42 fKh($u%bn2mL@CNGzX#XS(UyU(u}-Cy{o}s@@Lxvd diff --git a/doc/v1_api_tutorials/quick_start/src/NetConv_cn.jpg b/doc/v1_api_tutorials/quick_start/src/NetConv_cn.jpg deleted file mode 100755 index 0f5ebfa52ff2abcfc63464d48f00e406d4cbfe86..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44038 zcmeFZ2V7Izwl5q+L_nnXCQ_ve0wOJ<0s=y$h8`8zGy&>~1)s5G7u zUU#4teHHofGRM6yjWn8lyPUV4Ire@PLZ~x%%=oo!+`eR;rApGB_ z_4k?mWnL7(ye+=D1KlruMg){c98R`yZOw-zWCxyv9LS@bQ4f!>0g&LFe~(!LR1g{fXFR)Q`MZ&;q*Tj{n`c5Q)~5f}UTjRl|Y&gOP2~vq6}*jA`eAJ2=pR zM^U#Y@u|mPlHJ;857#uR-5dj3-Gyz zYp?@K)NSllZXAda{OjAz5+!TdGWEHCq2&=XzJK7V!*C#hp5JAT99ad{@$45z(_TTURgj$=O*{A_Fl2Cpodn7Bzq*guZA%&y)p_74K?%pmwTK4oj&BR1ZkJ;hIq@N5wmy1 zwp5M4cEh^UE|r7k2`myK{H4iG+x&G!rme_5Z8>HfXtV)&Dwvm{>8PfB0FM-k(MdFh zE=OcGG^B^{FGIX}(?dY}I~UxACR*hf0#EmR|gE&rg+cA8F(pX<{HVm;R z>^0loHGLeYP8b_%JykViTM(rsu-n9rmpM7>@#KIKU~uUn~s!W8<@ zWOg32srYY=z|!hu!+YB@aG+NZrJn}vw`Z{8%L>na0$KrE-aofqn$EwJ{A@q-A?BE( zJ+AFY7Ia(`5YSlWLcg|Zx`k36t3ba8@8*^6oS+BcyEj&U8Q750J+SS1BSK*C<9Oqz z$J(cC-b49G9kv$|SS62>Nx#wYAQ+B%m$W~-E0scLRa@ra30NL!V9D~;aiFv?B^>BN zY6M!xzzu`32BWDD>(29^Lr&4)omoKWzSuj)zbJ?-uC)^_C}7a@qPW-iH1)guVZpuW zN6YXz9jlprR1OX_tMx+$ej2di;9Py?F{KA(W;%~Z{Xj~Al>d%yPB_Vxp+RyPU3Td^ zx8ClyC07%+@NYacuk>;a@o@!U{AF2UYGuVPu>wt~sa-q?Mrr7%#y3wxIGjQq6IojaA` zK;I2QpPTZii$c|5XEyMU?z;1NDni}BtnEU`{+#szyeV%0xw4>v12tq9WjfQav`)Ha zT?~I?W*c1fa&p>#8nI3fUsQw}A~2iF$p2vCGg=i&d;|CBnaPbVfDL@=2BDwrgZm%$ z;zjyjkZ)Xc7}PGHW!SoSJETQ$vBwQdqnm;Qff*yP1hQ;@uJDGZtO1ezVyQkw5&Pp_ zdYKQhw+>qk($+etYS>qqlm z9;T_>9aqENd1)(=`bU?V(qa{)&^h^?WykM-elSlfD|y9k_1dvS^h~e;mbd3D?rCYN zJBA-w1f7Rr{7Yxfh$e8LTl%IqW1cE%zSjJ$@{yXpZm2o;fK*Y8hJt4{-aHOeHdLGV zP3}U!?}&%3MIkzU`s>c-K&de2^Qa5AZ?K%KRMLPkBJlypRlx;e467Pe3|Xdx2?djY zHyulXCF4XqG4ZR{WiX+md;iRYJ2A;97-r~IF|h1IU2;X84nIiGqjE~Q#tQB^j)`Ey zfdWN$E6xdO%>fPEiEN8Nn+lV_zZBjP;)A>)YRNP+6y3WuDwsD?*mu&GoK=EEO zv*~cD7#oe%G_N6Ib7F&k=kug&R_|E0zFO^C7)nlVqexMEHgu1lcTrzDo-c#s=zDg` za;}q9NkzWQ0G7V12`~?|sCX>d@47v-RuBi0w`YLQ?%AUd_Xp*!Stcrlh@BJf=ne>$ z73v!o9GqJ(H#%VqX6Ah+- zj|#2eKo9Dk_;w}x_FzeHpwAE3PDx+PeWv#ovZsIgAw5eSs+zAEfW$AZ7p>>H;nbqs z*_Qcyx?EipIqgYfitbkmYLZH7TfVJoMf`a^+p3Y~iyl@22a1%|!GRFU@B=EC(i_(d zo<$spqv0b)?-bEZ@5_dEFwM1eb?TJMScY75{6hEWlD<5+(C`m+>GL;LSBhc5fv$f7 zcAd!!9BB26LDzSvtN;y>*4r0`9G>sce@M??_L7?^eWmbOjl;ni2pY>J7?^;Cm@Q1-#x6C^YWIG|RuYjG5wPU~~*iMw& z8v#w_R1#A=Q_4Gzx7kfQaUdDuP5zkOUQ7%d^|>whK)v%&{3 zTA3dJV_CXF%$xr5=S%JhnjK(aTQ&~#v<3${XSFFY@vH3hW7`w3h95B@aiG5C8u3;B zH9idFbkkiM99VbZ)4-rC2J})sQl9YR^2glg%nCjvB6zwAoTpcYf6MMnD}lF#Sr`pq zucD=;!mH-NRPO5gt$S0_o?eSl0IWLZQ9()_S(G5cfr^P-wMGQ_pF~by0FCZFMY~Sk=~M%;@P+42%#% z7SsT%V3x`b^5)62tY>7U8!kE&yzf$NbqRY4yE5KrZ$YeFce!F0Q~WhmGsl7~%A%oh zATC~9R7Upc0y5+@C}LAkMm!%)_ZSrrIdIr*uUS`CS^eEE>p?|n5adkoTVJGUHc^pr zG9tmn%TAjo;l;a&^N4yrWmI%qb-KG(i@7)HtnJl-2XCAAd_wfMi;{D(eM;dp@7Hpv z`AJhc;6lgWYGUle1XD_|+};<4(?(0B5wDDnZBWa_Jqb|<$JBN2a0NT^NS^tqV&vGTBkD2 zB<`$*)X947JOIx4_D%oMld)=dhh^@Wn>Q}r)H2|_)#J;7ag1(6?&{b|D;Q2)=g-*f zrj_Q8O^McuS}@>d;+>=pWLk?F8wQKKCtWA5bkVAr6{fVtfi5C{3*+0lH*{e88xEv} z4zthPh#wiJnQzZomx>5kC>^#4NSq^S<4WB%d`QI2?sL004T-sqSU{BlAm1z_|a4o%SbUF9tCP>4{JZ56J8}Fo5)Vc z1#qD0cpOMk{4wfldyaM6@Y=PqGqucMmhp%EQXkADMYb5YK7%^tQr2;xU)rnXpgplG zx;ALUlW=$Kti1P5j34pYPdYh|nXg=x8VOSujk}us43TNSX6C8Goe`SLy;V@S;S}Nz z@zK22E7%wft&F3O5(}-=jb!Nx8@ou&z-+tLZI3>ce(%yn_>d>+vTM%uXCD_M%bz?Q z%qI}^Z|J%*lyd9QkeFfBZ*NsTr>?^fwFS7a~V#Y=qw>eY(w){c502 z{WlxadzCD!haQgo4>QzFM)I*JCoe=NwX+htI5DK-ic8CY~oFDU*l|D|w zR;{}}L+EU+!^X|U$eyDFFWDdJ%l6PrO%IF4%lS%U##u}jAFkZqUt~4SP}wigKRxo8 zpV8^8gJ$e~atT#rCueF*D#iEZK3W`Ti?W^HcuHpOohkd5L01{w(wv@K*)G%PWyplk z^~N?U*)98M1rp&CKIqI58RYyjRNp8i4HGN9b4Hkrea&A`xC^_21GNAyEAd%7hJTNE zsT6#2!vwHkMXtzG5~!9@c@Pk6@V9^udqw~k%vuD_B8-A>6^k`vXy8rAgPatV1DalM z`D%$%h8Y`s7rXwpBF@MlZUuahuw0#z$}03jK^na!={yTXp(ebfL`pK+ZmUL*tv6(b zJaes#bkp>nvbvA~M^a^-VV$Mo{WQSg+%!)dzN$ys9^FaL;*e@B?J=W{R=)D?*1c&< zqqbHuj5oOl^PXg3?mYu2zw4y*AJ(St6;pwCVPq>))S~qH{mo?3y;!MGo^Bt^% zq%<`0TgRenYNQf)#*+Z323!@9kE^CIGS-Rp>al>BlhizFZnx6*U zs%Nmw^9N(L{nBO+s=K@HykSbZ3lV_maQbpr_Xc*ee_z-eqU|Kf3$h)cyE+lfAJA?D zlGgQ3yIOR;>a)nEp6y%OOIv_9>qdd<~EsA&esCcx*PU|mBL1q6$xs!fx&~%1# zM1qV=Gv&c@se$k1)wX9=Jt-d!J+T54CVo}+!SFrO0r)15J*L~m*d`VYXsT;dx3NvG zIMBBdUoe_d@-u9QTq=(@y++-s9S5>bJE8Zbg^w;-0T73>NkbihJ0ONyfTu~;)O*HU z2tFX>2 zq5{(au^n=>?s5~qYuG{M;c5}WcKK>cCnZ-{MH18Jltg&)qq{FwmV03f_5pHS-?1E> z`Dh_op|-Cp%?_5A*=r0LNh>Geljc| zX8BxY*;92Kr8lxl3~eGchtJMOQdn(xD5YGwvjCv!VhV0X9(?UWoNSogq-u=whswI zn+B}vRmx+f@Hr6|u9}pEfQXIsTO9`Vj=f)>M_*L=gRP$oq0!%qy*AC)~c*fiTIu7aQ!^a2^j|oR*=(fu6hp zA280K0X&1Vw*_`|r3&z^h4t8qcQ8rqet;UGAjJBcTtqH9k)0k3pYst`keyKVV(WoG zM>%0tCt|_=gsKd7dH_hgy!c-OD3f{~v?yTH7awBh-ulyd3bcrZg`B9^%%G?RC8uBsBn zU6<8Kn<3_Su}Tqaq2R86<>`0-SefJHQW=wX;NXLns{L`{lHN2v+Q?Vm+}X9+7OgRf zI$Q0bg4@Qn!#%?{C>Zp1o}EeZhdNZ`h(9A`UqXph?AhY6li&TYxaTI#mOCzj?KWkk zwASz{uRuch6Qw%Exr!mShe$(a*>4iIa1W}i?yORf%s}U}euij5P;Rj6;sqk_&=*pR zo)R&ry4Tz^=C9j`FJwJ5X(o(3C@7>;u7Z%S$3q<&A}8Nu1ZzTW@x7|LqY+UzFkTUK zp{!Bk#d7rejH=76mLbhj=!5DRXTK2MqJ@|xnOlIZA#4R3Le6Pz@&WAx0gO|pJPI;A zfCDKcu8({ulA&wdkH^MzqriaCsxbW{4Ex;h1G4=2I`RJ%Y5wL+(}-4updwzH4jvru z(!CLGwswQwkRc-vSBTPm8k&62Y6EXlo!>d-p611Ykdg2(AF+PBb=4i%vb%Q5hiJ&Q zg_?pBcsB}NgAGQ8fOi&Q04l;+FN6c7+s{XwozS5N9X$|r@C5I!-fj4z^B@$M4UzC$ z-g5012qf) z(!qhv!gxn!WE%PHPJG%V4%D&s z0$_940=Vhx=wB%D8+e3*h1PSSMXhRS3fCkO74s_G=;$DjYSgF$_mW%;lP z$IV725p4;IDu!pCalM|&X#17~015%j&1^9kg%BpuE#%Y4=7T(lL!Fz2pYCR97OeB- za!-u=z_$kS)2kq8MT{~IbYB(zqkRUJ0!az~-amsDh``vu3wyCXsyl!S#BF})3Ew{g zJL5nn`2TS)%Ts{kD}(iD_}>(oeT*5w-4&(bz|&x#xzusB3ivZ29flstf;j4Wn!f zhc739RsXRjzwp?~#}2-~4)X?|h@6|GVa_9_z}UO%t3_HrFT@Y&wr@#05UaqFZ~(F` zR{=gb6s~8(y!ZLv-)G}@Cu!>sC#fIabbB6h$_M~y1d%vUn1ai!8cZ{BW(#}m%t&Hi z01F+Zl!?8=?5{V^H<+*1sAY|HIOpu4Dd z*c&p614VV?KyhJdI1mj?BJbIY(eXoCR;|zz1BVxDOw#-6LdScutIcG=O~8&oW%D|5 zXJ!vWVOKbjRHK#%04ol-gEGXv6#)WK#a{3;zqb5ooP68susT8;0{a z4usAF|3tmgd3+$?!G49iz_8kGNNPfi;SXq5bmb3dHf_(I_1~ab$J2@i`{3&7cqelg z)Ei}64=Q<4HX;Mk)vr47>?IvM+2X5d#I=`^q`oYcnt#Lek!o{a*EC~_~ z&5j-w&hDJhi0z#218PelWEemn&me?ozov1?r>Hv*KPrULor&Hm#d|d~KqLI?#w@aS z4~YzUnkUF!7*(&3>}RPi~~{$W^ll0{;mwNkh}V9=)Ob1YwOMR*#&e~ z^M7MMyt%X!Ub!2m=^57U}qnEpW%c?k?DjvoPGv&$ecZpXWYnv&j87x`TX8p*&C8aMTlIj{X6v~ zS{(;$Xv1+g%;`GY@fH}^0)J^;d6W@hEtduPCij0v+t0GOFJuYWG_#l4^m7+I2{D3O z(pVoCva_Q!j1Jpw#lgmuK(Hh zUn2dd&i_le{^#TW71Dndm_PgeOQiqQ`ClO&{a%IhX&@S zpVC*plI{5Jh5TWbz!kv^Q@z|n+0V=&{BdZv1=?YDFV%8VmkWzv%mEDUFf$yoZ{0c& zFA%=kscg(#(PA@Pi!%NiB{2~Ytdz}wH5(SRa__||PTj}Wz~%L`v%H*3{Y;eN!NI4m zdWi6)6(Jw+qagi&e>%t84_@WZ!rr#RiZA+d?P~fC!b{`d$gmYnnvmgp%awdiJXWo! z^l~UNnP!h-zxDA-A0PgnpPZfJZN}}n^8hS)(|cIZDOnZoVp{H0G>ioH_Dcyulb)F3 z#p`5`Y>DJ#!_a~Z2K^h{u~SnBvqE@TRufvq*{X}dM8zr8yzBC(^vswH*Tteq16wY0 zFXNgC01JDa@euH9%K`ShvijksX=DP`9zZ=>1K=Jo?w`9IJw*$Da6f*qdoXj=xx9L$ zP^f4&=9*rNVW%y_J%hV$HB&8lAdWko%v4IvY#=NV8ll^7dkt+&Q)2B}JbBAxtPC;6 z!fA=^<@La4&}+X#{3>w5icMOg#1UTPk#@V}NZSKp|BEN?PgKJa4|y8 z37ioa5(p6fQ-<+i{l=kKf^76ig8SJK94)xJe6l4hN7F8b{$g31#3N$#HXmJ}%~^OY z!ckSsD>A{*<^n!(|8B%)*{)XGz+_fh^Axx6b+-`V(xyJ~d5(*>DQ@_5<2mY>UE`*a;cnfcM5H4l{}>Sa^+eqNzU9(Ide0?f{IqS^qGxn2 zh10cRh4`wP;hGU^*%`j~^%1eoYpldyzPPb*wZ;_7%aD%s`MVIN$}>A&?vTWRO32A3$Oogv#Wcek^Py@+W;fl}`B?kp z^cB@rP*4Y}HbH6b0jPSuiTW`q7*w$-v|m0BBVJrnzY5jJIHXGkX{DzzH(%o1K8|h3&cFKrG0O z_(1qUp!M<1OXIt}V;fiq{zApM<8CkZUAM{d9Rp0f0NMsVvZ;hk%kj-h@OFl8^Z?=h zBTYG1bnC3~{T84q?I=JzD=g-w(g%}{5a$}z+%ne?A8;?{BOQk&HD(8CsrQR7lf~fG z?)8_^c%{8zSkdaY3PYcyEP`zpoa0C@Q3u16zeq2Nv~lXHP54cS;s@|J(M|2RnNSSu z$Uv+LI;|0TdgI*|_6}7g)VIXc`0$dwyDutnQc7GgO)d(Xvmqta7_t}GiegG}oU_|8 z({ys+Yi8za=gA4@9SOa{TSWMd=gn!eXLT~*_VDnX%)pK9wtHez#P5&t0uosd!K5Ya zVIZk+J7Y36ltnXyy4C(efy9x4?ea%>k^IM!P7jwW0=p9eolMjk*K`vt_+Bb2eyYim$&?x1=wBm53=r#gF18xaKltk@ z(^6f#EycA|Dy<#Ro+O~kx~)j(Ixl+nn#6SmGkB?qggA$5)S8=3Vj;+->r8D&@ z&nvw<3S$z-u0x9beKzSsvI1MCD~NC;>A@CMw=Y_Em!^B^D7&DQD=x*{;#0$y3B!(x z_$b?OeG`vI8t)SpuS@7-%E%Ksqx-NI+<)s;9kM%2yD~c2K9}OyZT7jcGOlf7bN>>1 zA6JM{$^twlHnOKJEPiQKc$JMD%i?Z74|#KNIOMzmp;|L}B=!1uJFhhm{}LYZ@?ivkrH~(z^isI%J6D#?P>z$N+#T=uVv^&{@6p69XEaG*8p*9~ zCqul_G`ovx^Rm=_8|~%9IU*ESZBSF5=rdexQ_n&1#$8}zuFnDkGZln#68FE4>W zPh5INnf>%UQ8}3fNpK$88n{?N4QIUkv;8cSm=6OrAuw^?dc=vT+q7DV8_oW41N=ld zjR{ChJp-zU`ZZv8;iTT3yCe68%Ob{^7PPG7*s(3)vc97a)5g9SdlVjN1a?JMAn}WA z#>4crx?kK1HL5$YEQHT>`+SO@_ekxz)#VTw69eURajQVFV{(xV$MXFlKw&%yD&m|B zfQ??CBY`Ma6~Jv^p~zJq7=~x-2MTshwJ`+XNsMqGFjmi-XV4P<{ej$i8s^n#pOG7e z-51Efe>{Zmrd9k7z`Yi3gB{2ruvAAl5Jm@#0MNP*R_6f1bMnOJUeNi)``fudf`KKB zrXAiyw;+7V1~ec#gJbz3(DyJv<>Y;!tYr>t|09Av1a1#V!Sw*5BtVt}=(7)Re1^a$ z_+Qd1cm7@DzmhFi2gAO~J_e$crB{c?s`cS?=R~tRMTaHh?!>Z3|gI{goC{Q6;`pCAONIt;Oqws z)^Hj?c6vEzN(%>iC(Xzo4lK?uY-QUGKzCt-fOTo|o%sR0{&1xnk;6M2pg*#UUdg}+V>UcJud0>R$hv@DYTOT)h|*Z*DF ze{~c5e|=?x8hu6DF%f@P0sl9t8#F&vR&%@U>I=S=KV*W zz}^e@eA6oo8%;#7z~}1-Q?r}@rURZ{8PCj9iE8dm;G8Z!a#O{cxV=3KGtp7g3WJ(3 z7nYXhR+qp{OXH{imAz?r{-?ZNssF#5^N(Hp|Fg~c(3@^S0N(MRDY25oTSkPzrDP12 zatwp^1MN5XM-93hbH)kEM|cBXf^Rw zAR#-Hx7w5BuX0+sj@%kkMTsK(wAc8)=X^rgQkTMy4ouOE)W9JP!pj;8)F9x(<4EAxX?(2(07KXT3fdTXq@iDywE#Y0av?hf3~g~pw=92NT2mcGFMu4R zTAazkJ7+MihkLbvH=jubt?~nHz`)*jchlb1(|P{}e^XC5Ac%QbT^QI?33Pqr8z|`Kj{l zdgLlfdEv|=l7&@KgH<^?8y{=?U!a=;HqE6|A0ttZXX+230?WRbVGb|eL~-4 z__5)qk+{?8_Nrj@GA@RD%wI)9eEU-lB80Z>^B_Q=*=ir`Ns!@vuU|mKc23WEX_P~% zba!1U5?N17I#F=z-gjlsw{=ACN;*=l-aJHgPWg6A9aYRrmW4qzzLy$47_m!S-_aT< z<4<@pxO1JvdW)LsK^6zUU6#Hw)ELp#G3#(UjlQUiKTU3B7TRJHd=x{a?28cngu15V zB(d~(JrpEKKpjeB1ewEu!cWs)uP6fmupf-_iO6`nS@6-KdlWr({-RaL%Q`#|D};x; zv{@xZ05(h~8ey)UZ6hd~Qfk1j*G{3BWdttvE;4bsG2vC0?PF3s^YK7Pfq`{s9=1E| z%fq&>P=5^vnsJ@M7_$K%0I^lNAc_!sRt~s>8#CCEkK=H(5a3;0_W+OSMQj6Q79H?C zr3&JcvsooJHs9MT2n?q#@>GRyI{Qq=eg0^$In$dO<+QHp_ZbJWoSrZ=$al43b7%@b zKf6F`44g5bSqGEKNaszV{Z3gRt-gGNWxN`y;Zxq6BD17u$1xtWlvCyegb*2ji(>6Uw~nGnNQ~JGlHvAtg;|W7 zYmJpq`2k3jK!^}HvT?4)mKgYZrbhup|F;Ygu-TvdEjI(;1HY$qH11yd-`>O_jTOF_ z1U?UN?md?@9I>3W2OG>|Wq<-G3HTZDTH5K0ilRR3$>~VKeXJr4vJvn|RdPICkc-Aj zfLqS@?2joxKY6_RUpNV%tf~zi094o=x3C?N0J+*eVddQotnzUX5XUgs0K&nM^|$Z$ z|LOM#5qL*C7O07tNLUv>p#!6UdeKiq)EXv5t|Lpuesq4&;4q(*d`Fu`17YX;%y_mH9WH7PciH z#TUi`CtK+Hl*Js&J;*?!><#T>OC}qsXLCI@Iq2->``Uv~3`KIk9ZTqb5{;TLkdO(8 z=foa-m~Jq&ddDsIC@r~hSuF3j!_R$4sZc88f|dL|H{I=&-jA(IhgM5s{9h-Fv`Thd zNa+$wkROr2$%MW%nSUelpZ_nvc9(vJ9J!e4?vHJB1D|rO$+5sYWp{z&O#XuKg9k3% z7+;k6%qtAL?AeLh+{u<*feV!PQCy(qkzwyUhou5!jYBv=8|D?0xg5y7Xs4~7+z&;| zxqDugRTh|dMwZ~kDMCbd;Z&3;H&=<+hJ76nK{^=-=0Ui2R;X(!-42O__i7W3c&=u z1Qf=|z%bA1s{K2};iK-tOK>zb@>|+gfyKTs<`Q_(dm$IwTP+kl&r~O5|8}O7-*|LF z!R^Jhp{%Nh>F+8Pg=vy_dt-Sn`bq4ESiFW?Ox$=^J#%RYM{$|{^{^+EbtwjGmrPqL>lcv&z#-+snTgRHhp7ccl#{jNKVuQkV}3r48gDcqxa zCXV~g!(KY2#Ti$zD_$Y|Txd8ZUr36@DflgHN%gkaq%P$++LTKh9>F~G2eHh8ij@|Tr9$YZeuO7 z8PsZoJK>RJN8(7#9_(xh-V}YtI)U{NKHo#mLAC83D?WVs&LMPA3&K*#v=}bxyM0Vj zv;VP;$QWynX;iUSK&Ar@5b{X2;*ahTn>Nqkd4C%z6FXSGPeIT^uq3Ua(r*BT) z(zE#bv0kzT=&GekG!&?ij2_MGcfxmogdH*X%NBCw5Bh(T&RKf+ z_49S}$&|OD`0u0+IQDb0hHP!kZ-@_-3Q6U6nGL+0)S`<3t$wg7LOum@4Oj zv+HNOc^MwA_B#ua&0hG<;>++7Acq;6hz6iDSTz_MS>_F67|!hsY-&;$$=8b0b+zN8 ziY+RGIw(&VntVIOl4q+yD1u|Ov()Lr`mJtDDjs=C-;h9L9Y0t&;^$KI6~;JlUWH&d zQTqV?26(iAd^k{E9jXgpxjPcw>g7WC3U;knhHejuA%J}RUou-uHFgTm$V6Ao8f)pDXE-A^cPOkqZ3nTmgISb~ zHkSRFH(borim8aWm>k&T_A;aH6iFZH4|K1D%kV~++Oix6H8nbse-0tZ=f<{{?SEV3 zjpkn0r%AlmqFm+lPTPBNf!>scP;IO?)~kO+=X|Bu4I{O-tyQS|iEbh{V_ zBx=FC_zzHlRko(t7z^~8i}-S?_yagv`7vLU3SMo}3!p;EJ;)R^%K1(tWF9L#4i53Q z$(BoU$cHz{TI9@FmLD6`7StPcx>@*$aekx8xO=@yf$|BT;q1E_HBX++osZpf!fzR~ zigK+SvO6zDTub|QpLQ$lsiOJQ<2IdmPf8-i#s%@_z#|=pqNFUdaigz*T^KG>-|&*l zHxvw?e3ajODSIoW-#$K$X^JPCuf)^Xd9=9(tu$vSOfj`q>}`cUKx7;G6rZjgE{C@@ zUAd!gUS%UsoZ(@IH|-QIxm;E&awg#pZ>zYiVTok*72Kd4vprMMckh{Yt(Z!wX=-v^ zv1;?2J%uKDKkD~Z_3%QaX;oHKMF4dwuU*Go%Bn`}`;U6QHTr&572YA9r0nRi{d9$D z3qRKKYnm@wx~Op;9L`2vv}NELRwVZEY{TQ4_giwZu4}qmlfEtkFJt-8v5L-{$%*JZ z9`oM3+5#+@?5UxNw^_c{hxLJ6F9lUggPpPRQsZ?G+u`Gz5t~}QnAn|g7+R|p0N@WV zWt=fvA4#;?lX;t?3vKh-h=R*fbOlvs>uq81SIZ%Ukd;?wMa@!7deXj?@v&{m zyd;8Dl`L^6`<~2~W3#9+Cq+>!4;~(SexX0J)f5+`d|X@q*4yj<*3bp$32`KmWNK%DKIF?KK6!lY29 zA}(~2dF8T1N}^_zX36TfEzHHM5Rx-@MVzn7Kh`pr_nwn-{-*`np%ESDlycXS{?#w` zXSC|O{Kgnl^niHtm5!rRj4MiHTNljbcV#6swTuTNq6=O|csl}EeX1~4#AH%R14eNk zOkl6IP&H{fZ6_q~o;^kEKSzQ6ViT!p(tCXp5Fa*H{;n z|3dYo_0376{0Pbx_6k1&-6$)ghKmkit2GfQNfuG1KAtT1+bt27+l#RMdX=_vmwyZF z*H%+%v8K|he)FX)R4?Bng0wc^6U9^SvGA>R>R`!R%oU{flFijgD>&M-R0 z4JH0*a5**_enrZ#8a6Urr3&`+)?BI^AJC0G&yFoE?qAcN+-ZG=Dz#R5ZC#{SWTS7NZa3ZAbF`Y0RDB0b z=-X8dj}UW14*1^0xOus6Zx;kdw*)g0*=}=I#5|cepj2#Nef{LPQ0t5aKJ=%HUFiGO zgtDHMgl?tfbTV#unth~i^NQ=U@m1g~QpIU!ugyZ>;ak;?q1H!k0>v=f zM$nwAe)R+XA~e=dEQpI8k6d^zPSh+x9?Te;jChvDPI#uVw;tSRW+E<~&)8xo5{r$$ zrE4n`U$$AQR>N8PPviDg2O#`vY<_;OIt{>`KNTwHl)-0H1BfjFrXxiosF;k9kq3uE-mY(Tx z0O^-FxD%A0( zrfkfdq>gf2s_L<6uO*y^qL6J!qA7^)c>1V&wBA4lywt#`<&&TQ6{n`Wm2I;}P~a6W zzYkk9NNZC1?p#5?)x-z&*N4m^9@EBA8X6ff*?2_hBRFTWPhLqVm8OYTNb~V^s5j3G z43m&*Q87B$s&uIxyxkw_EF@E2Dsc~Scaw;TvNw)hViTnHOdZJgCf)5p%-M7UXU7ym z=DRdvm#alf5{k@iYo?ZXJf2a=OP7@C$jwe@x_dRS((T&6L6jMbHbVyN4@umgZyU@| z2t>B*RpT?x$&-o&xqfeVC1TgUN;rGzJ>grI6xB!lr562mTMIybTbSLq?QX04du{s<^I5EZ zv*E9?DXWP65Yx=paXr$ZipfgyX{z>~sI2wW3nZb?FqULYKnNo=OxvHNFHpKgP+Aiv zw?(wC`NHCz-MH4`Xy%K?6n!zxLA8C)@I0Ns1n=$_5IFO4^bGbYYBx?p# zvct|}nE-e-I?UVZS+2vGH+6#x5BWLUmm8##zq zhMAqPDsh^!-nBnZIxLJJ;}n^$4DK?!&^35tQ`K=#~=`;WI28@N){y=#mV*Z68t8oiOx^QhLtu?4hhvn#$V6s6gXj z3a#+scW`oQAY}2^>IZ6?qD^~Wd@jEjd4tJ}d9Fn1FAq}!ncjW;DCp%@W%2sa8OwH)XVB`l#>rPQ=j*=mIrcJHY{W#s=9@P-`}*v zCrZbxY_Rm0inC> z2`gssvdA1PXo6s-gJ|Y)l%NOn3_}vL27Bh z(7sF)y02wEbCj=1+&=9AcJ0WA<6NLfC3B9jsJf`y>RIab*Lk_!##R}J!wUsFCP#|utLnvO~ghsLrgu(97uq24 zvHG%7r{RAJ>QBOLOSAgB(1wF;&WDb#{^Lf}UjI{qt=IuxDM&ls01|Yc`jkqr`xtPC zEaDhCsaNKAt4itpTJpO-5Z#P)CGaayjQilRb$&^tRazL*G}WWoc(u)5f`UE_+j(s~ z+d<$a^)|b@URbT!DT5K2y$aAc1FLSc@Cm$f4D8{42}HpBsk->Cqup^1Q1>!)f79K- zT3h6W%1jOH)HPwd?A`{fc%aPChM{PSy|zx7tHMt?);#6$h0PF;CpGbs%ytZ|ux{_A zv%Bu)+an7_NiJjMKmT6^ui*8cs@Is2UJI)9u$#P6GH z@{Q+x-ut=l`=v1plCoK~ahK$MHMwgR*IrDR5>Ve`jTk5rlZ#fLkJA;{!bTSzJJnZ| zdJ%NUP>!DXt6^q(6)TYzV`hdX%{}ux2C*LV>7YDyYV=*U>!N1KF9Z2ro!XfGL)rP= z6!)@VH#qK@L0!6KDqiSN!pJ9%)a-X+&u%#xHiv$KUzR*q&P|J-9Ju-l_5in>Z2L02 z6S!#Bl@j_YmY-O9)z>p%WM3A$cM0vtylY^(XxyDQTeZUtN~2Fv%p8#{c%Yw%qaD@t zq8!T#`+h4qN-oDs(gZUGw=c~~)0fNdVmMl{E?{y#{i|Xd)A&{iGbLpbXiTB z<8AO+t%TNf<7iL0{VLgWEls*)ig^_b5XbwNpWd0R<$m0;yxr53U%Ee5lM!WGoayGe zTrQR}!KS3Hde5_Q@>1GuI#d10GCi3i2}GhiaK<;?l&ZJ}$5Wt}`8SJM+6QbnaGp1UnmeG4Aip}EjoOmi ztshNo*s4^im!NxB`Ur+gDjF3x0}cBt!So8h9qM?;eMnNPKbKpfIhZulIKqY4iO3Z1(3mx69zTI~9edT$DpTh!bCJet78GqZUC&7>@i(xQ1hw~j3h?uc+RRV}3&1zbZ~DE~^*Lup7`DX>bq8Ve7-#N7w-^}x z0EV96fBGdjFQXhU@iOOu?Q3G_Suo=zZs;Ni^L+IuR&DD8==YK(el*Vk^g>(dy5^7o z6YO{cx@_eOsD4xS05u*JLxR72-SsW?{Q}^j&a0;}*Vvsuu{eNVFv5)LEJ0<={uX$k zUyh6kU8_x{5KWSighlVEty)KO_QZ88e{^2^xJG>4Iqm|(@snKzkz%F-aOw|x5?|)o zLXlCnudwYO^19TbQIGzn5RUQpQ?o-(p%Y#{h zqH8vMLeZIJfuKF@ap;LHxWm~)-7qdhKtEg<@sQ8Vo#UX_z>o`JG(R(-QibpW3+HHO z63~#UYJa^bW1ni^Wf|hIz)Wtp*u{98H9oYt(U!yf=|fk2v=Tk}Xl@Mc_eVURhUYG= zaH)5LqH!QeO7-VBmGLEX|tFXS2u%(M6ol zWRMr#Nw*hnuhClaN?jDY*Zrv#+Z=yMG)=o>;<`k>FFLI?|d&8hhjR~XLZKJDZS=WvEXi9swiU~ zdGx(Ufrv(lL!`8BL$zC|xb@(`F$<`$ge6hvhwJ#rpKTBH=fXYV>U^oT{-OmoG#nj_ z1XQCrV<$;1NuF^lu2myV4K`HCuI=6%V$?mxoETCQ(x})cmQ2187s+Qaqb#CRQwRCw zWBbBi{Q7DbZ#5@%ug>HSI6GE)RNs3CU3_?@QrQ(p>yeoJ9cq=1FU_dm+K?$b$-~bl zv}+l$sfip)ieepQ;%0JuAp5C#eu%kTzh`TP6~8-mtYag&TZ-=CQMchtiKr2;9U@!o zF%`kfBraXP`GcHAg{IoCml+kZZbEMkpNQj@_;FMH z6x_Y=u=)BU$XeC0XR6U;zhDVUH$6SOE2^ha(>238kzMVqeE6j=tfzN_o>sa|ZwrpW zXG5x~Da4e>d$+?65@NnQIyC(dZT>WtnEx!Q7k!?Jxe`YHh$ijsX;E5oe*BJu zgUDRN@OnPIS_V$sEvBUT?(q+AHK$1PA^Zr+NUcbRbn%_+1cws0 z#EJb8;h$KcVSYfOpLGee5tbtiHRO`PaSiOu07pR@)DAI;EruI@Dmmaw315xa={Pfb zNh@>Ymvcw!y3YKK;yz>^TI~SZY+gK{&bg9b#rO^ZB&U>(qV|h){iBJlvA9J;GM}=m zXg95uw92%GtkAC+GvEk76*-69ehQkoT8hp()RsHsLMvClb@OaUn^wpgE_9U|#fuzb zdL#be$?(ebgMj@xI)wUJEQ;9wP*m}dBKeT*gt~gmVX2kE(7vyJQg3Ot%DavTTNzJe zG-@fd7r!5?cN8=u%6Posc(pfm?@DfAh@%Z4)`z6%;ozb8)X}xk^^ei?F?pdyruq?% zNyl}SDU%|*h*=}vR!>9v2I#%@3FrG;^1-@8`%nsEg@BB9Gra{Ex3szpvJPR16m6$D z9qNfaG{g)jHIOmx-LP@yOTJaAWW#L0OQYWy#}X~~j=`^aQ~dHA!HpCI$^vN=AaWp6 zu4W{(B`B;-ric}a-4QIRE@F*qsYvxseBb0K^YS$h0ZHdpQPk5_1X%#au^Ae92?tg4 z{n5Ck;ZE`|IXR_NB~8!H1WnT8_1}#(8uCg-Lr-D6+MAaxDhRY;K(c4&&fY@PMFuDz zGYi5JG9$OWM3hDDJ(#HJ3Y6SV8qr|1Aqmm1<4>&PDwSWl!GxuR|F{U)1CyX00u1xh zObpf{^f2WFh9HQ!I1s%)v&M`934F#ebn$iL%MOEL@~rUO7ETq9{l&v`yAD6qM5X!V zuQ0>IY3pR9h~g0-^~2{!hS0j65#LuS+abgY>{T1%XKiQ<`1UqOcTOGK^~QlFW)MR* zxmSYLU4fD5r?VIwo&iq6+wbCL>~1`LLa>?nh=WIrg^2`@xkb0Ez_L9>i}=+>c9|^q zvY#A}@#Bv>J2?dbx>RDUYsG5^647c?Dmpdx`3MJ|-f(N3juC>l*-HfuRu2oCINZst zpRoCg=J>Ftfa}HYYsy3RMn3%4%GxiM3cyJDe6w^*B4YbI)Ds`Y0BEyVCwO8sm0}V4 zQ@LN;;g0e)@gD!Dd=Fqf zd|i~kxZp+E>tvWZFW*03%KCjD@x6yeN5iP!4di8oi*?gWW?DJ=)gKY7i~!UJt+wuY zl*zixXWc(lRb^NJ$xl@^k#@7$@iky~KE&HdbIuoIA9Yf5P!`KL+PV-s{+p zeRz?pYa3EbKkBcmW$r*^gz6@R)x$|l(?w+2KS8cDed$}?Cs+s|j>6+!l@7G@X%P2^ zsR;Tyo(2CA%!TOXR~dIVeqyx_S>3%e$kgy4EH}puvBw4}p+v*D{EU+NjmJ7C93fZM zdq7^Dj2nV7KWjieUM^#p+mQfubB76)$mP~Id}o34CZ&01zF5V2mkumo+%l|40czGw9N=nQzpb;Bh|~BJYi2-g zB7oV;yE@!*d=D*Z&Ko>|KI}iFe`>>@9m4JQ!Zobjn=0DV8~cMk@l%w0(Qa=i!m6}o z);2{4Va~g1sjZ`fJ<{hJIA z+x$V;a&1^@I@ybIDygH~uosash8`Tj9OI{S^O-%4ZzR;pc&}r+C8K1H_3p_Pt6Ez5 zT06c-*tez-Th@PRvQ~Q%Gcq9Q-87$eEhS7F2%7qR+4^U+Eqm>U8`+-*9IEYNEh!G<)=qD2}dpRJ@kodw)TdF-L|C z#1->?6qsW!`q~Fv2(>nbj(EcksuZW-Vh{2Yg2pMOrgZs8h91Io%p>x%VcUZzre?6Z zJh%y5e6yR{cwP2>-C_V?MzOp&TOdx>;t$U^yc0YMkmnMLjK( z`zi~l3g;Ioz+{JjEN&?qdOJUiTFNdd?k)rM$?RGj%e>-I3`^@BhEsmDl%Pa!ap;nqs*OBM?eGQ217oBU3i}ohYLi1G=!lfKP3HsnrO&}IM-nupVyUW`m5)D_mKfPM0+3_ z;rwO_5ec__yGM183>}ZmEh*^u4=jhrsDnP}FOi<8|8t5%tV1`=xVQQ->=g8ZyZ+bEM z^gWN|16Q0^i}#P^yR5?RP`$w81KBdZonpMd*Q^5AO=x5r*z$fYc?0K@im9fTypq;u zta8MEz~iwN*vmq-xdlSd{<*k+|8sE|v2TnS^o&00@*XE?w`%?GTM zg_(v{B?mjs>?%(-^XJbCBnTRM1z5Ar_>``{p`&vf=AMTAncC!BJ^eFptVgQ7#E&k1 zd_N&Y8Pj9J{}W3c-Iq+ceiT^1MV_YZ<7@3+cP(}#HHBedL26`PFcx3@#44a�Y8v z6ZrN$)I%}~28lrJX>)klr;nF6)=s5w`*{A@|3bi&V!Ut!(i%w4wcapji!T|2m;4ip zUwn$plHREXigbfsJOQeRxOWQ<<5!1U4%Lp8TE15+omitKm#>Vr2EkyDKxnn}cx+~U z858Z3odWo+D-tT~IR6oCS+cG;7xfbIQv-HQx+bE%`)RWdTJd8?@l76vY@=-1N_@#s z;k1pIM_pF|a<}3lU)5B#(K3c0C z)4%a8CzJn1<83TNbC1%&QJG&f<;X39?*pt`XR~T`bSxtYbP5juP2JpQrpT|J0h~wD zL)>epn4ee*3Z@>;0MiMrfMUB~NKyrI7Fo12dUypm(1R5qpa9UnrOaQheF)5lUdr9q z{yiV~!he7``4YQd%2aJ|mt=&ihjB}($D2I#(5e@NEN-Plg?2q9uaB357c@t+Qyb9V zuY7pMgX3RijN5MZqpo8?5NVpZ5))P%vuZ|#ik_6b8fcOC^g^H~7Ro!R9*~nf!j3%r z@q##Eq&}jS_#PnxSzMu(ri0~G)50|0p>Sot3w5o=QFS2TXS z-?7v9;Gp)M+JI#vUbNj=Bkt+2)Vuvwi=xlwXRFaY(CR)dHcgKM7} zp;CNTWiq|WEZ*6Nab=5rTF@@PZ4YBD@X*28>~hYql0{V)@+MUkPE14g_S`g=?HbFF zySl!xKzi~;baTUA=D&MGWM1NL|LzT8YO`WCU9~KexFmK8;eVH;y|`e^mY(#<^L~={ z-18$lUCH<4CRw>0x57qKac_YitndQ0$K;g~oB?H&|8fk80YrtH7dXDB**RF>ubY!K z&7to>fJ;yY0clWhfK=~y8{Ir5`?Jl7X`^#bpEKc{4l0_$Mt9pLvpw1DCfI?{7^e37 zc^OLSB=#rPV-s$Gb_xI8bn~xGDLl9htte<8a(o`RKh+jMokH-jE4JA;or*N$>l$NQ ztTwch2o2O%v!PkgMPh(O3-(z_Tj{&PH~fhO2fFR3Di(RBlb-1Fm!A5|P!M()KBAXj z9Ii`}46CXe3=hhZc^gU%I3-fKE>m9P(NJCkl{~brcNu~hN&ECV!H<`}Ya~>!w^LMx zE{SrYx}J|!#Be>eKepPD_VJB&C*5Q$_nsvxOQT3vIw=Mg^3b3HddohMwVa7OPS(Ae z8n}&(XO^3n_KonA&gH{z z6?XHxrz-g}_e8Ad{`Q45T6B;=`Ue#BK^(ysi;sHY8NBmgBPpoR&Usk4Vuq9f5<;-Ikuuti6V8~8p83!A9 z6xxO@=)`f0$(CD`stHEtT2c2tpKwG-xUmIK zeMfdVl>N86N_6X9*z`)>9BFKPddYxD7t6UnEfEg&MBmH!gY>x9+L4jqA1ux7h@i>* z)(Xr`9FSH4`T)?#1UfeWs)8vCyAO90GfTH0E7=}#1%Dviq|3BBFQDt2ckLON1g2yL zzR0M-MPCH{j7( zFoff>=W;4h0VD4Y+nonoj|+G70RDwL|K$bvC|4wcvrOlxGUNSOekEo!bb#{a;|IY(I7D?UPW`}1 zX+h)erF6pyJ4?G-`DG8~mG~v;r!B3=dbA2fv*R4cwiX+hjFm8l7`MR~O=G_d+Cz8) zM~@GM5-__W3}6?0@fgxfdg%AfEa)=NB0AZC2Ow`Z{h^j7%D4&0+5Z7t-|hXKzxumU z+p5>>M10E(r6LNj-9`bi`A9Tu%k?`9ndFF3Dfp|Xo${AL{5j|d3J9)&(Tu&pbXRHV z#sZ*jm-llA7OD)OAn>jVdSt@oW!`JIL`=P9z!8JW`vz!Z`u|cV$wz90i>?T%#Ot~ylOz8bH|L?&lw4R z@6H%1BYcPx1Eok@E);4ibho)1%$|_JEb07lZ^7UZVEHjHkgLV8V4D%pgpviuR%Qo4 zOZdlId>j!diW+9qD&bHYk5io)Gd@1gd3}QB_Z6uD$zq1{g8H-Yb#)H3)cViguLZ6r zj!mhYSAmtb-1INUNZ&3br3z)}*v*>M26|*Cy>1)EqKn_cVm&LNY)S6(_WKE5)Orps#3$f+qLo%FN2h z6Ii1X$(qZn@LEy2qL-o0n(eDGcl@3sBWKcQ4UM-N?bjyb`brFFtq}#`gIW)b618&0 zxHHyBGir`MMdL&Z*!1M`TiV}mKJeXcDVwe-8G-QkOm@f>+%eogv-dW#_h@O#y<{*y z2ujcvtCqu7~UpvKTD@w4Gk2U z*V}wnJ|$#iU7e~}su1srKbsJncjFb;;$s{Uw%=Brr1ncn7`48mrI*&mg}$)7R{(P5 z=elq^26eG--tqn61Z2~<(7C5WR@&ToVdq(TQvFj9v!MR7n*EGLf$i7ik1o~HiVq;$ zhZi&FH7RqRotk7Q0{J6eHzQ7>Tzfj2XB`l{@h{wKie%D<=KHnX-}bnj?%sCcSgDQ5 zHF|R2(CmF2(`Y-5>@vN!-HQ)JZFL==S6P>J2$9@n^xRvIPMKD+?r=ctMi}zM3x+sr z0=Aw$A~rmXL2-)p?wPFaK}T}UF2O{;Pqs}Zul!XA87pz4Er3JjW_#`J#yS`W1im!C zNhAscGntEt4Lh4cNs89>0rrgASLin3NhPk`6h1(!SqEM7(l_7o&j6iMUf>5Pe95L} zI%4VK!)brJak9Ftb__(F3?Mgxf-Wa65#lPbzGcADDR?yVAx7{VhKrH?V=2ld1EPUc zDh2~R{c`*Qc!?_T5&(6-r%;o81nWc(Ua+7SGcQQjd?KF5wPERJs?|L3BG%7Dejwgf z&QMWodYSCu^q3Q4p@}dl1@d#^W19Rji#|8?3;f;_P=6yDEL@;OSa%o{FE(uCp`f;j z4Kmz@^c0Col&TuZU1D$x@i;XjwOh+h&`%>kB)EkLUAr2Hzx?#;XbZgQp4IA8maI!X zmo)7v^GMc-G0Z3R?y(CAAA9dx-BYh}9=*Qt)&bY}Zb5l*m4$rWJvpw&N#Yh{1S`|5 zWryB=PLCs9ur?#MRl8IeQ|rl1Wp6Z<(UGR<(kpIT6V`ducDml4XUk#>-;*&_A8Vd7 zrPoQWb~O>*Yu+1CxfABL`t9>2Mg}a$Yt1%vn&oW0ukne*)xtS#+-YZW>K>2H1HQHoB5BSC-vfkq8q3uaYg`9)@>d6wg1;6OLBGkUpQ|4g z$;RrszmRFW7=Gj$6ev$bZq{@jVC@#bzgs1aC66lvEY=xpLD*}_m@CF*m5ugmK%Y^8 zAwOY&_g(kK?7^B_kEDJFUkjyvfExI(&p;h9BEWF&L934gIbY?|Ul~Q@KZKBfY7D(M5f-jjk5B-avw{zl z=%dhqGtvPiIE=Ir$-HFaX6#|OUv)V3u158q2H2P0bj(P+<9aFXp%!@bjX3VqWOLE- zW>;A_sA@3;A;ngL2FU21R@ptg)d)}7#`mHg-){D>hdiZb)xDu!ifh)on0olk_T{mn zxz!yBmx6&Iss=-4H9gyhO+qIgwb5rXfBH?U?1CQ02up+)papm z?E}s$?{k5-tvD3DCWVI40cR6HMlc{#!J#k05FpUp9tVoy#Aa7qFjF%$H8Do<*j{u+ zt?Cfm)=2l|+?2V#0XY@AsPW0(T{5_#v7yD9KNd5>W%MwbZJ>xlqu=`$u>s1pxc9Rk z|KS6P?u_V|RYsEE7_9RW^KiAA7>rf!$M%n$YIxZh{N_zO4Yfqq`gGPCoD(h-PRDBS~>)qkZADAP_hOzL&_UDPb7T>jUY}4?i zm!fQY&-@PG^?5Yaq#D%*Zf__pD3MzTeCz5_JNlxeE8a%c?G`|b+Vx#R@Qn>TM-i?1 z8bNM(l}x!e330Ab-bi^m_d#$(jCz=Jz?MrRSuk;}#+KYI@fsn{vnc3j7P>saJ~Q~} zs8HCtr>}@Z!mwu`=^#^;tfiGdVtZ!lKnl7-Okj*U+w7t_56_fE=I1~h{U!-s!EMTw z@@Iy4Iw@XmD6XZ*^E8>5(+0+gOjUb%zf#5N-8*-LI@o{jb~_3;To)m#q{MRI5uaZa zKQNOvO{3z3>emLTf4u5o9Yfyy(-W8i8n zhkkWExT9%7z12FJy9czCg^PIv>s>tE2B4O{dRx+~-v>18&jDR(wsLDh{JJE&8o3Tw z`=<)O%IO`Ous#9rwLzyp+C{fXra z{ZOFU`vPkZ_laD<)NSkB0Fbz;DiG1Q!toyijvQ717WAPQDvK^EOaUz*yr(01}c-^(p z`Y+O~YN1IGe#J&naEi}$(&9i0*-=cTz?NwJ>V=;_oXhK@e?+eyeac8gT}pThi~Vbna0k*=15YK`&nJI{yWXmHmUYBf7i(@ zxK$@jabPHBZ0Kv3y!&i+EP5g46Z^cLz)*wNcH#ZzoJ(lkIJO(*kd> zlJk|c;I^j3w!;?!zE|Sldx-eO4O~x<;c&^F{_MDEeo&^Yp>7t91HT8l)kFn6RVnLD zNZMSfB+f%F@|NswsS@rRuk#_m)FRxSz?Z*zZmwfW)>LIzi4rfWUeTzsG#CNC4HVu`9=}j%J7tU*OY}8(mLyGW_br3Vs)Q`$-Jyv zvjOT2y~NJP$}`@jy)sx#T7c&^?#+lx!Hz#zQh=KQV8KlV;EYrtGAX(R8KnI}BK^z{ zRl!akFcEG>g_L%$KvIm$0c{ReRA-=faVLPVF*cF;y0TZ^`OK(Lo#TPz)@xlG?K4eHTf{Ea0NCQxrX}twS`tIL$Oxco%gZ1%YXQLU z0?rYSMgd2jgc9EYkUTaJV1)*$$gaZl=b=wQNDBa=U{Ocd!up8Ol*E{4%O|QLaQyOD@2eEtA-aKp}idHp|7Bz$IK?FhHAKq4AZ+iP3OkV`;>+phNx?2D? znR4~a-o1Td;51oEa=lDJTR|I>21!F{i&>w=FRwk~^R)WJpNLvymwApoepOwNV$5wOyb7+B0cmFcFoR_ z_Yo}wYJ-XJ0)`i{8^1%`zhLhF;2)XI;?eyo03PW>ci;cZHF@{@d7uBlCUoado6sz| z`?_)eyZ($8?Jzqs`oC^n&Qk4OMmj%QBwT~S2f$j~q?r#BM5p&1N!fIz0{hZCr7Iu# zzOH~Sm&#BwUSy&l16sNFrmA*Vyg8ak|*zf!GwlnH>S02ff?V+iOpo)f^&zCuv~jOf^c%cfN5 zcV2ZgB?$BE#Fl&b_m71F)b!;{G@&7AxdcP`ue-_TIhepgpiBq&*Z}Vbc6GeG4crE9 z0svMv~}lPTl9mrxH2(>KrY70f9fd${075J6dx2FzBLXO zUov|iK!M7rf_8|nNK+a?0xVS|yOI<`pI2hn6E0aqx=uDqoFIRE6$|64X3wLX5Sm&@ zRY(o%)XUNwE5oJcixR?Y#N%@j%x(!4Q9C#k1WwM(KaU#{3KUz>ocI)jMeJ&PA6Apu ziN|ie*o&bcd>2S@ZA5iW$JkNcz41|sMTOGvk)wkR+eNkGy8&mf+y#n+{3LXR3p!q! z?B9PG;1(70uipt^4u5SI1+)KkkC^@0Nanv9Ddw-8`2kwgP+! zj)gxO@$YwkYf{j^H)$G>wHy2K0kkpT0)(QfhIRrS@|WAdR1`q1c=v)iMS>r!T0()}5g1A7)i5t>@&?7XLUwGj7buj1^s zzm$ppJ}Zo&8&G5~28S1LZvwDX*w!C+-iy2Jnm`R1h+Rkv&;<*CV}f1H{+p40aLI@U zhWgw03ctqv3c&Y4wgcOl?mSGTCF2P>5K(b}SrSM5x27#Bq63bZ{(ErD0X)UN0FLN) zU!!KT7t5lWB_vn#u@(*cpx3FzXn3$oRsGL-iMBiC@aQORL!O(S}yV7 zE3LxP$~7SU9jdu&${X+g^!#&vx4<+cO&)21?MWO zC%D8ST;iu6cGn%LZRl1?^+bv4l``bMs&gPy*_f%6h~T8tG|gK7Pon+*CsCd|5jDLk z850V^lhQ0R9i6OJ;_h(K>*t>vxTaCf{u`f%3sx5`{ zjzn!46n}2sWJKU1TICzT?b{V^r>MnQQ%*8bdZx6%6!!&&Z5tH~l@hcxf{q1JhaWUR zkbb6M!W-b4{K9D*nXrs+!>2py0k+?t^{L&T_04pR?s>(vVyjamGV54$kZlq-&1R7- zOMoFy4uNf%0_zD9habb22RNZjcd%rq63uF^Bq-X*kB+Y+n zDX*W1<^Iw%dY;HT8R7Zb;U|`r86_h^TCa`$lZ&zqGMwQwl8kE>wm<*QZm%iAjsuGl z-#~xRM!O)mP{PT$pSL8bZ?;bJMc%^|n+_Kb7p46Zdw#))!PY54}?K{xFtLk{a zeqf!Zb*}0p?>C~|}jMNYE z-EEOEFTs=?%X_ zhfNGHE@}QcF6T`HL$)AWGDQj(lyfnRGvgSm^QukIEO=Qa9C@&uws=oUF#iY2w>#rb z674s>y_ZI=?$n7aad!e=C`Oigfu1j=T@l_R2xIxcH_99`yB(VbCc1(``#$~JZ#3rs zN$W?*=gDU0TrWmtIO`<;3f%rcUH@5roZtiF{-l&Ma6LlT#$ zMkw3EmbeX-7c(!+xjCb&iShX>mI<;P{IS`@{H7O|<=WK9B+e zD~pX;N7AKkZ+HzYn@jjtDJ>q2Zgz3bVwD#Ru6snjYMID_jz-nWCyvW8L{A7`aAn(+ zf90gdy5Yp#pQ8msYVcw+MWXO`>#jk(zQf!DIyMu7&WSsD4}H~)fX<_u_TdMd6vqn1 z$J$y-q505t0Q3gl=uPIGkq8B;x z7ir(mZj4v0^}YWx%{$0P@wXneg&$=GW#reDejJ{~fKiOx<{3q}KYeXHQ^dXv-F)Zn zI#XDn_btCpnTZg4CGGa{bSgfJ%xRO%cy8Wg8Ll+0fq8p}MfZ*O3)l^nhy3sTGz*N; zGVt}9{S)Rw;XI`xHd>7bBMBG-d+-8G!ATsAGpg{WovAPiWGh_=QP0}te zx>HVN_Snaj^;0yH!GjQz40AhBbA6ONzCaRM6secxf2h*am`j5jnrkg((!zg$Zg&ae z<;A{u@qoH#siYZ8=^GP?orw))qi44n>26~rgJ?>>LEOyjUQ(Y8At~QjA8AeE!Rv+H z4QiGs>}A~_#gM9R@4s$6H7;76sJ~8F09-&}_ZpSsip1v8608`wiv0nKG+-p91FbT4 z5`LPAjdFi5F4XCI)&#w)WKSnp9LyS=q}YFgRTO^r;G@0q&1ntvYh+D9mm{e)WSQum z;T2zldEWMbdDfZ=U97x=QY=fXslI-HkFds-`Igh*V+^(-w4)l;kZ+U0Hp(Y7|LSbz zh?ww+bN=jrkcZaAd6YBXXQ>W2C&QBv@sPQj{mVhou(np(G4z8&P_J_*kdC8@PX`p> zYokK!X@>Fnv*CocnV98eYFCje z2TM%M#XCE11g9reDep}A+aV@*vpmnBlg*$*^<_+yxBs%gfb^@KPK5SHG&c}}2_rwV zR%XewU^6N9wqgk{5{H=5w1JQ@)&PBv5-1Frx?iz>@)JiM{5IxJj8fIsp?MlwJ1!)S zu(#__^iqI$jf)@S60U#br=EfQ=v7xfKZo=~dh zhR4EWQ%$uY9pV6>hueybIJz#!5RTk`Q>4I`O2AF*1Wq{w1v5|wEhQrqRt~VNmsgPK z`%%Qucm0B4`jRl#ga#x^N_eBqn!(L(msy;;CRN0}GIp1JH^WtHWXYA8dEY%MIro796B zZ#3e0k$F2;AwVWI8?cJrb|L~@ltM0Qwu{C-GYoCAY7MJ7<<-`BYnABNq1=_)el)7X zIa9wq@*A$|su`_~DJxPmT4%7oCSB3 zoz>ye(PF3L0l&ug4m{??RUiP&N%f@!eN;GZS_A9-x(G1yG^v!>~IWA8=sOUID}@ zh{qRHcnSDvvuIM(7egLC%NqS#2&d0v=6GJOezcjcy*)4SWA#O99I zQ#Hwl9AUih<&6|xq3hQh!=s;(tjQ1_&RIQmQG_B@`DQS|ma6+A&VDtN%8wa-n(o;t zL;K!XKgf2{BY<(RR!fwd)(Gn;XOGh(Esewn{B|3P=WqhK;`Wzq>$CLtH7^}`2@!uh z;5GuiHMvvdC{+?y`lVm%M|LugNB+(^@pP0>Oz0xPDYXM)zgk+A-!j#tu9cG3Z9BcP zwnBx^Z);VHlty6S!NnAVrJt#)Y2I_)BKgsE`(rct3ybRY=&t}vg=nO32*@)+uHj9M zRZ%r6Dg3$|IW^aIVFyx1%k`}y`qr8#dH5BV|IHPQ*23L}E5&ra7f~W*@$n3Oj2ti5 z!Li>L4}lhKre9hN%3waUFr|Wjj!}qs79nlai72>$2a*YO9m>zr$jjZ8(V~42wo$b& z#kN8Z?YT_u8Um@t@tSr}6}w5(Y_|@Gr`W2tLC4}EC*k2-?uTN5rUt!lI4vCFJy9OxjyB4a zQ7y9uLYGDXCS$f|4c+)|>xO}9s=gJ8DFMfxHkXK@Dw~-z@exwA{iq-D%9TIAE)p7) ztIDC)Xdkf5Tl_N0@fAh42t{IZf-_6KlBl@Wqq{vUvu>H8jr5bB?1dMD-6SOoUn!(4 z?tS>i`#fh?n-iQ_eyo7ypouZWhU4{w;33? z)Q;WMiLAwAG(-!)a$-|FQG6nzo<{fz?C>CFP>Lav&xPis;Pi}HL; zl}?UegT)JJK-q;0tZr4bFzSDTN;#1u{`Xws$Z_U)i87Q=hGaK@YxHE6_%5ocgTe)3 zXj)icR8ljj^;=$l6%%dj8>WzN8!a>Sj59G9)yx%+>-XTFSeL47J`J@5#8N~SfL3WT zB|j-j_$wqw%q30L+T6NcwalCMTy;9Xv0%cT8d&YS99Q|VtZBw$iv=MPCM8$dr8kAj zb3RDFbV)Vfmm!pM*Z&T&`hSZ){sjuW(YFh`V=tUZHgAjESzsP>ICvRC8?(22Oz4Ui zbPG5Or5=|>7#IEU;4;+@z$k*Cn4TsnO;fkZG}hd;<)+pH&my?J!M1{EcMlhrFKew4 zFJ(qAHAe?OAN$J{7=7(%8j;!UmJl7nj36Ou6q`KPe=QMy@?!Ko(>Qos@ln0NWVbW?(BTfI7#jkf0xaj9D)Gw30#F zHZcHE(+pT+AA{*qzP{X^IrE$l+^RPZbFMz7dGszBCdQ5^NsF^_n`$X7$%~Oi0jM0S zYlqnxPSq~p!oyOU)Bp`U3J)McJqyH>ppRYJENONh2TXbH5??Q5qQGl0VA4FBGSQ1f zcgQ)sy6JrwqM+I9l??@+$3+#10J$fLfDhpZ0eMwduo?MK#wSQXE5lTG%TTi?+omrq zNA1FmqxI;O9dVWq7C3sQcY0K<{Zy(|U|0~z^~TWts-$Oe3mYolb7F051S=C5=)SY{X&&Z5W!pzua270Od@aCmL(y}UiU+y{i@RM(CmcpB_y)F~D z<9e6&Y3_ z+SoIxsR>^BUiyi30_dh3^ACzWcX}YaP-`L?Y4a`;rM%t4?$3Cx{Aao|p9aJ0`Dobi zT%Ja);qZuKi>+ZOzYbhU&ddrKPIMaYGO^oH zw8lW9;)%S5g|AIYaTj#1sKF&^q~s#6(>cBTx6>zopVuu;D_ag4&4$QJP7YrC(}-m?(nJq)E>#CJ-+C5CeuMSH zXDo?hIK6iv_kzMluSLYA2d1YLBt>T+@BI_2>?4G`n~~sDOiJ7lC%zB`H)t4VArUb4 zdds_4^1WaX9v6wA-mVmK=~_bEDhFW2z&GX?s@Y!jstC!KbM_sQ+nyARN5pji?ydx& zu@M8D1BK>-&s8N-0(O#$+z%#U$xHSr<~z#*#X{Qvt9qtUIz3B%!MAmDWbsA-iD*0H z?7iPchijlEI7r>K)uDc)>HQYZY4;EZy?Vk2E|T%NO(=#=o}}mJ_B7Yrp%ximr`~2 ze-ZbCw?TCKsC z@OC?lHXN=-EBASdi$Z8cx(DdK zOvKa!ZgDlaWBw*5{vN8^<~|p%DZ~^E(^}TDuNDqf&&CdF5xfpDpC8L%7XYq|R>N<% z&BVQ?01hTH5wl2>z&K;rlemt$yPz(p+HfTmgURTGh};FIt&S5bUp|h(X{K3!3yh#( z$ZsUdUO01%jwi>FQ|VE8G#}IG#fu8`K$p*(Y(1BD`ZE~G#UzE5&C-0goT1b*fNPyZ z<@iCVJM^B74>TCz$X?S7n}|EmYUUP*FviS0tG^;=Q!SD{J3-gg%tPeV#Po>$PyHqr^s@J^TUEcfbv$rExk`k>Y;6$ zvFWtOYH6vCC3*Lo<5bY|^FPX|Yy2D(SM}z~3vb13d-cIY0Pf6N3m-?rC83e%GsDcN zX4yO$M(yYU|I5buc0u_NDg%WCr$`n5>w&lBQ?Bj~d`n&f5{_C;u@U7X zSyW6{(F*3KcfR_G9qL78dsQipR>Mg2Wf>ZX&xnR#VFA4mNC9zWb&l1{{j|E7@K<@C z($eCGUVd76d|%vz+`Z&_ihJ6+de-cS@HW11Y82QiuVtOqBzK`{I)Kuj*c=(tFA^I4QF5j&5W3AjZqi*HwB=4$~d7fhObp%U37O(nPos63u>LbSG zgtwXjp!xJDc>}0*Ovny~b%Kv0VoE>mVFr=ZfOQAneFUI{Y2zt=GIyu>DY$)ZVn+Z5q3l8j(Y8MzM}J-@98?Xr0pYmbK!d ztYGeGo#=;~KQ#L6EC)zKW*V3neq)O0a*S!TFN+lB6*1y?r9j3AB#LpaNRJ- zvlA>(b%X#k*uAm&UdMifP4fEpE?=b^*TFGF+h-XJ6QyXgqCuKPxuiv-B(Z%5+VrCVN9L6Up7qw((=@>*U0)nTRT{^& zwNOEx5O3(Jkr|OnfzYH#0Q0zq$%6_7z*h4lUpGErQ-A)W$SuockHb=Kfiz1z?)j35 zy7h$g>uJ+|Y5pAblCX}YgoI7Y0$;=sRap(K$v{bss+-*1?f>J_}7D^}vq=__1kg~xf-mT|8c<-EhXYSj1 zxDR_~&)%~i_CM?Y*T2^Hed{Z@1;-btZad!o9ip36!)v%FD*Hq!a=qa9oN~h^-6jo? ze`1J9!7PkfetCa2LKMkqL!XhoRb`OEWn=WNMf`h1K7;1pZo(fXFMjWsG44v}cL>%I zUrg-N#cWE0!1h}xM^z~s)J`m~QK+UItrK8({sJ{&27>1Q<>LKM%CMV1lwq!ar3?dD zAs{U0QV!ai#CMjaV}w`D^|)&D;5xwTF9`y_wCF$m926g)cj{e=I91>>beI*cel>3WjcP$O&KHNgm1H zN-f?Mw?Jf7R=;?YpbxkQ0A%BXd~b{pS8i_PB)oXo*_Z6_#|{hk8G<`XWj5(nU?#oP zERx4;eo!CR<#VkzV`e$z5PhuXL2+%s8JYeQbv?Th_>Xu;H_0ui1qt`=6$|B!j zRW^v1@?~BEidfyJgFXxly|!?glhs3^Ydo28bh>aG+*rfddZCDH&3}nI7I31-VLoSu zf#{Af2(M4VX+`+&5Ck}gPW%}j>>Al^RmAFtLtzJ0h6TNAvKQS;ip{yo;x`oqeBXu# zx4U%a%}Gmq=q8X9T2n0>_@d<{q(2>=%RIP#D>RU55(UyW25plkUe6d>v?H?K6sn<) zdOc}VA*rM=zKkAA7(GgScI!-Dg1g8{i$*UJd=`NBuYb~}+>JJ1EA<~k6@5l4GrLN} zqNZ$xTbEzDTFc({jZWUCS}}h_r-tWIt8r3Tp+C!fieSf=(O?3pNe)iY-r2U>v}Q*% zwkg#$=L|eb>7ApW&0fbM`z-bD$!TwO{XBn5U1YU`@7l0)HBYllf{Ef`@4lF`)QMRe zmi1U$i(Lr0ZH!?U`qd3^8kfVe`>;HiiF2Sf%uI<{Nj8LT;uQJ85&?<=f%w&8cq+#- zj>9+j)2OhV$O(AL*-rc6HrUvTeSp?f+G&ngiorm0?4$O|y*YOJN;XWH&u&Dwo!owW zy?CUM-c%phB8A>l59l-BExYu!Xa>tx7T&JCF1V?xBkd|ABLG_*%7>)^647*c(?I3& zcZgQ?GTWHlf!*!kM9|j+2{hUvOKg`qtOC{^$l|9Myi$6*9IG=gy-V{wU8E#ZrCek2 z;>9{bd2veagZ5W@b_(HM#;d`_fswigU`gJgFzSs7$68Z)j$CdV67E5|1rk$ zf5~l)_`YAl^BOa_2{KExNjQ+Vb*vo|@1~%*ZzsUh@Y~risP_FUDS8`M9eA;R1H^k6 zL48}kdD8d;wnOpTQh?qpDoc4&l|c3Nmh=#JK8_RUIJb6~5lKbdfl4AT@I?PHP^Vp1NufcaI$k+r-w~@AH&W;E&Q$=&_-0T*Qj1h(QXkeyd7@SqM6bd%Fm8?HHXg1iYe*b$THt=BR!L*P>@>L$Ix3WO=qZ)4zCba$lGi(B{J2nd$qZ0 zmZT5AEM4pyt}ORVhsKY~-JhN4j>K}_6TuxIglV&BVrfDNbG(5-5ZnViZgu}4UjArl ztX?;sMd`1>^tc54u>FJ^2mG3n3QQX!%L+UZA*fr7y%+G$oohj6j$T|CJBRkjEO+8wMjKnUDK-(5L+y;}JvTFN!09Zr%QaB3yccCK zzbd!d*Q>5w9j(2rLZq5uY+-0EXJ#C~p1zsw3o;_yrfz_}Rm{zg;1oOb{zq1L0s+k! zDYk;AG5HY_KQWdeI?(hqP_>jYP3SutYN5<`|LU_9i|LKd@Btb8Q(naNW!MF71xVGq zD3WmPF&S?;zOf6W1}ZuiMw|vZa)$~_avGs=>rcu&J7S?bpXHxRv4MF&uB3#0u_(@# zXQ?$~1+62tknaLww*q~Tny3)}%5k-&LxvZX{5%4)b*XBRz}AvXv-aBVB+V4-MV@r7Jl+U)cW#HONgdNhuqWX&M zM`QI$H;K`YS2bIow=GIeqdmd46+pgY0v@(2QzlnZB4}+1(xyGaaB34hSS@TZ-2U>E zg7t;ZUU~5i{WHOsO%dBw*oAmFKDP^OLHGO=t^U(a?C)@|;14ot%I^5ScD)y`;p6x4 PB~PKGoul&k@5BE9MjV9- diff --git a/doc/v1_api_tutorials/quick_start/src/NetConv_en.png b/doc/v1_api_tutorials/quick_start/src/NetConv_en.png deleted file mode 100644 index ad618d1d6f8f4839f566f5f5cb5db37a4b7d9093..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59083 zcmc$`WmH^Ex2}x^2ogNFy99T4ch?4jy9IX-1b27W;O^eI2W_0-PUCz$-#Pnz_Sxh7 zKR5a>Vt`9W@)6MH~F92m)ni>P}5 zP7UCUFc+5u8@>tL#srYj6nwuhIPC&&b_jYegDfrty)Qm0^mLIwWzpil1m4!KaK3}H zSCC05P!klSdB9y*f3n+L4m&K)*XO%;eclg!z$0UbgJ@_Y+y2$KPDyIiCC7pd2%9 z=F}ZH5Q4+brdGM2XuBG#=;*KsTiu>JarrIloN6O6Ce#N41w77N@LU%bq%MWH(6Ib- zIR=jeIu{*=zva+g9SKDSN5^Iq*b+M2@kk$j5-8-dn2YZn-I!Z(Eqto*d*zprCl0=& zm0R~{R5|=3VuuN7C`^Hd-IuFV>i1g|7WetiOey8mphfgj=|rNA^XM3}_Urphb^Z^q zv<-MEopeS8txTo)F=!fixz^oNIQ0XrmFZlhvSK$*MQgRGKDzB^w6_FB+>s|4w9W(7 zETnl^!v+)#D`c%(l`L1|n;^MnRJ{Sa*bbXtSSl%rB8eE+Qi)TGnobDyN_9=%qxmPg zwL{sU>ySEb1Go&9-(8XDOTjQmPU{O-0*Ri$aQZ#PGF!wXI}Wl>4QC&3X--OrJ@nG| zPy*ZPA^uWnOQ~2xyLKeJ^z`=uo*UeGSDGoTEp4wU?6zh}Ssv4mU_FUy84Pa<^Ljm% zhN}v|O1ySN$~kLV-J-$gqR?pGj`(r?OhRB&IGk-N)x-m{UZEPH^d^pl;cw{C?A_$| zr&bTi%$qlfoc%pcfkf?$tE*sA;SvEy`83vj$k@6+^450xL54z~PUkSz022NW<$S2{ zLw9yQfduLudPsfO4*yyZ4I_0D^_#I?>B&-6E>=K5hhxQ{MeOnd?YlC~+QA9{vcV|T zHWq$?!j#hJwWz>=P~p!Avt=PGDmKt6+}guR$7Tec#HUlCshKl}^OGVUt~|c(F7F@f zHANVb=mA@~i_~WJ6n4?|^7iktK^$VesM_{0QLmD+G5exVO*Fdi*T#u(x0FXMCXuro|P?(1cu=;);EbA6aVld+jLa3U~V zU6b?xGL-XK1@C;HSX#w~vVVk%n8u-x_~g&T>BvPCe)`JzvG^nLk(uuG3VgcV6^}Y^ zK~um8)=0Pu==et zp%3RqYqYLNnVT;mU{KKg5qi6f-g_j#y?I?OajgBFZrOwFwyQ!TtTy;90DU1yHe64G z!QOTS*oj}M`H4pGwR9-feRoF_Ye$VteY3iW4JJ>zL%9*d#m~_8ZGCQKhcPP(#g9Hq z(?u=I1X$EzR3TktmSo)0DZF0a_8Vqs6*rAj|KIsx=>Y%3!;BB2_U_jIJ>H;YLtWCL zyXJ+*xYN|s{Fr}sO~AO{p?-NvZ|50w3mUgpZ?u@qB6aA#7cES_F}rK)?5V`qyeeV* zf}&y)4Kwad$k$Cy4nca|ce#fsoT5ur5#05(yl&tlpk=@rS93zUh=s_KBN5J*?zv;= zYjK4F`a;$-s@#0~ zy%W9{93;{#Lf8QHoBlsy3CY>j%az{z$=-nQ<`BJ$HO*X%Negg+od&z{}shel$G;a%Bn9&x!BK1uX_ z05|ikRm6Y|ZstSXpr1ef2x$_RYW{ZTmB0TMc_d1|kJFcdU!OMJ{m|L`?%%R)@p1sH zdf(NYx`aSCHQ0cnQI4n0v)->#y`Mt{$Bx20C00dc-O|1&8${ZOJ^W5-OXCBE0i~-D zdXme7&P{juvB;`sZ>Z&LuPxFpP$m?gerY^HDv?2dg6*An2o z!bnzY`O!_g@=fr>?Y|ioY0{bPC`z1r&;>1w8d3Rnz}dF z*QQjR%e9>kRh=&{iL541>@)*B8EXRfrFGYb7d=O9x&GJPJ`bCE=rt7yV34PQ`7VJ* zYyRhsh!CHfq0(awCE6MFG=IFC?3erT=?cf^t@6Tzw`Jnp{Wfai#ASdVzEUM@Mr2QAAvur1_qsG!z*_Gr%ug~S&7e$&*d z+)Nna!51j+6fc(<(4PS8Q%t%78Af81qf@t((h1nF5Gcj>^HjiNy^*7bU5BKkfx}xH zM)H|xj11r+F&OW!oy*n9*Y%+}^O8Py<{a;b?M(99wV0N`5BMd1V{s^GRiC0qFzyoN zm0ICN0!;Mi*CC9V3uo-X}?h`x1;i?S7~r+{(KLab1SMXJ-G^xxY_=M1`PGGTUJ& z|8Tu?^&9ibQt`jGXh9-xo@Oms`6rlXOIgQcQpNdj@;$l#(g} zso$3L6l&V{gq#X@L)OobKW7NO469?#$Lbr|RG^rLVAiwDkM)s&<+phZpJfbEb<+JO` z&vIXicmkhs=}F~;?kG@Z<(z_9QV^#Eb*A|DEqQMC3ild!QBp$@v40Xr^5MVSBxGVI zF$ukkyMg@F9`OKRB#+h)qdqAj>|#!jICL~;=u>GQJ)N?1t79M#Ptj3;UgTLjI&zP^ zsr*7;&D7m(BitdPEA}RQ|3TpppJ#)yfV%AWbJzum-xN4Nc^}E zstxI0WMuoiiP=?kT_tyy*0Mjftno{r$S`he&V$yP2tg71%ukca|K`5~mxTZMjChhU zF9&c${)%rotTH)XFCsKczd(Y?Ag7);h7p4LSbfj4YF6JqsPxZ6_pOQ3aW2yVT0TJi z$A?U&{R2J$GjBH;eS`{q+@?*~`Wt)iF}>O@nA&=5x;OeKr^w3ijr znS6R*cusL%^!A2rYLk`?Jpn|1$7T`67`ix4W^06Q2@OgDnp6re7$ar1N^h?g6)`6* zV=kuaMJscKQc9DH=HtMr1Qt+q^1%I?Mhx%0Y1{cyM{#GT+0Rg2F_JSpI|xXxr1BSL zgM9iV2R{CsiT&zYmQws(m-;-P{rmgjq0aLpXAtaG11UpHK%wVanvx_x#a1?> zNet%!vp>WES|qpGPXvykQUX|l(KIHma$D@_Ytw>dW`*L9d5Xl24qTe~K?yFbSI_~D zhLU|DSAenJ&9(NO{`I`Ykh-tqEET-zqo9e$t}Uc62)P7g1!tXz^CVd`ud;TqKSrzX7}4zw zE6e~WX7j%zO*hA-6U3v+295BTmT_a6CKQz~ zuk2gUt)LxzVQ`KOI*PS{q~4#=HH+~E{L4EZ^8WTqC(}qWa@-htBj3SSdsWlD#d`}! z0BgZSf9vkaZ`XvZ4V~I+t4`rL8s^-&upgr9d{ zz*s7gkF4Az%#Lj|WsU?q&#e)Ut1)#^#tMNm_@5>!uUEZu$8w$t;8`0b$`U2kQIr5s z2#!5((E9;mF~WlPde>VROf@-hxRl9t`;* zmYs}aGk***BcD6lT$qQ^c0NUVBe>JqcHU5Q&uOe6IhS{5vczC-owWa`h@Fe8M`R~kXfdv_k03x>?xf^^Tx)vD_qQITgTLr zS!|Lt7irto(kBC4nmV7|?x$LpuOeFu9ox+J&J>##*Q28Iy%4Ns6c6Q@67%Ubu4qzj zN+BkBWdBy3l@q;U$l`5>@R!g=8wLZrN4iV*&KJjaMS{|jj=ohr@trH%kk@Z#5kdwk zmfE@mOQ775hF^RbiMCq)9~apbFZHtxUE@UFCm@jj39^nuu!3o!LUWwTiG9H8Itfqk z2osS~uilv*rvLjHHZ2qVkUIQg@!hTa#!7irJqI=KB{caWA@#x5^73M0AerB^H#UA&7)RTnR=*?z-& z)E8y_n@x2?4T0+=fb~4JPPWQ?5>*=y>PrN$e%qj^6z6fpevIKlEpSfSl0x#Pwmth` zI>A>TcXV5U!U;Fo4jDFI3qSgFu1fEeES0a$v4W0+*3D`*&Zu@FM>00gRO|aT`)gu| zwO@k46*kS;Tk?`xm^FHB+diW8R6MF^bRy5jDK!UdYSjV(ehe0!Z_6>~v4DG@cCPSS%e)D*pqTp90??{@2=(XGs~SQskA%6QpuTxd z4*92+*pvzo$foXuqja}k=^}29i#Q|!No>lw5hhBIapl6M>F>l9>tTVb+4-Uk89w!x zMIC|fYW{dQ1c#yR9?&6&5mM(2q_s@GJHnnla(CwpN^s4A>YF7>K z*Nt7uD{54SyuO9_fy1f2m(Muk<124dXcMWd-^Idi<4)K0KZ{vynp{8^6%Y(sUTP)^ zSGSf)-wSSS6=4jACIDzt-@#vX++U-G?VEquHlI7w%_jMeY`AJfgF%ho#;=cf z&;p_5W)BK|xA1X`hrr2Hi{9RPGhT}%KAn1*>{+#(u9U|Ww_?HuIMB5UG7CIERZ+c> z;{QS!G&^Q7efx$tO}WdRNz?%Q+{Ynxeo&W6tuy?{_pBH_0%sP9G_RiaLeAX1y~YF_ z(U81R`3t?f^G6FL;?pD=y9VFS8O00#04pZQof=oZS`J&zb1wqxYt*E4aL14!hGY98 z))H`fdN8)lJSy5Pl~>{ny}u#k>~1zxyM|M79^QX8237>J37{Nkr z5~Qj!amB*rEEgy1`~orObjQh1VcW=n%B(>W1F%_oebm>N@yC$o_|SZUWrlVpkfmJq<<=d$Ror=J`u&h(X3 zhZ&?>4HlC#s=Ae=jrp<903f_dv%U8Q7Z89;6lA*jPvb~xJL%m4Q;l# zJy5LF^LG4dp5w9I50&)wS!+q(8}k;go}U~B!#6D7R0&5&v4{FI{)jrWf46+jZ8#v@ zwUcjEj*9$mBFe~A0?<((IKa~Ox3_6^VW5#eIN~Ef>7tR{3D_a-9J|h1rHXMyi$Kf0 zOregwrLzdoXm5up&Q!j@!PlZ&Q2)W}MZTg%&@_yzcoBXTP_!%d1(9mNV|Xs}^S6PrWU4A$sIXpEw7-`?PP9B{SxPRvP>T-593m`R zx=(eyA0Dt6O0L=Yz?5wvE7GkZ9*rN~EM2c)6=_R7@70eO)93g35^8$>9@10At3d_v z{dBi>X6EJzPTHhejt7xgXy#pSAzCaX?_#I-wR5N1a(;d7a-Tb)!t8->W_{PhVVefZ zR*7!c+V7$clAJkb10=NbCOlxMYmDwiiET^8sB7e}CU>)rzbk6Rr-jcH+(u=$UFOcL zgr;NVzev{G_mW*G`HyTuUNmKX8yb$S9r=uH+#_jxsr2)f`?}uI@`LXMQiepm^)#BE zLPdiy{p6H!L&-@Y#1cgW(mK=uKNU4y$7WdVZ3tuG|`g25F2xhE7By8{^%&)IS-=dcnk z7|DcFaxY3a*m)t=;$M@Ls^vcqQ1D1tV%|THK1Iy?D`3hA>wfk>ggcV4?<R*dtzWG)JzEuKJ-iPxr* zl4zw}VX5@Ez!n)F8I+9IiXuS~uNfq5!e+)7ZKSJ}jSv`f;kEN5HhifIbU4GMg7#?4 z0hRf%iD&M2Rg6IanOYBq5}~)YYjgsH3O(|ue<+( zTi*a&-fJDDL$!%O!i%4gZlzvyYh#m7!h-wwje!rED+Hbkqxt)}uS>`5k#2nHRQM&N zGZpZIh!!*|K1l<=(3Ty~n+9!Ii z7tDwkHW6pe4Y?rb<+zHvT;K~|b2zx`H^6phFNG#r4yx^~{43M2JI%%vs0@=OIjkto zhs#D6xod?X_Gf)W-6JA}CPXpgiY%Z@I5JlN1*DXiH3<&J_8~ccB@D@TP;rsB-R?op ze*4AH>Ozh4$q&`vEoJbld}@C<%7uE{h{9J*pl-c-@h?<=O;Tk=^lP@?8knNnuV}-@ zarD;$5{z&(AjY2}8|tA@IB~d(B15SNtmCNdH&n+d!0ngsIh5zc(sfRoP4ZOlrgCrD z;`vpb!T|VG5Q}Cp0wt5kLBwRCI{|)y6gt~54JaaIIHYdKOq!qnj>8Palc7ciT{+c< zhTSwVs_~L|UuagHp#*p7;d@+*0sYzUF^|=Vk6mjVCk$_BeKu!f75P7E1`=ek068Qc zCxon@*+0<`-fvJ>&P5u^vwi(>JJw|=QT6V%$vAXQdohBtPk_gcTizSrBhNYdV^>Ds zcgeWgo_YHlx9+s{s7oAB>3a!U`}JXJCxUZ_ zK}CoPB~w-*P&JpnJ2B^yAKYOL)xXB2Cei&c^uEBSMG?2KU#0n_#cxEOk;Vy^zPfPp z$5I5C9IFh}q{cGLq8!&T7hOGa^x(;TUi`4XmfTYZs%|Jg9AlDuKY34%T3iaeCQ3|b zshQht4Wp`{qhR;3!M*)W4}DR-aQO_Q=E#RJucekl6ofCi82I@@<`m4OUcQKcyA?n} z9Wn(?hpUVW&VMPo4zFtGU=USqJsPXI_JGl53BIY^%YN7pLNYV#1LsMw{7B$0n9dmX zFBNR^55GPa<93JGwDAX%K72x8PPRprsJ|`t4<2Yth3VXX0YoqP_ECyJsTP^FVM&kB zFX-Hz-dC)N@R4cDx`nq8~OMs{4loe zwh8|w5q;=NladfV*Bly<{IQz}(L`H+I%Sl@^1OSWv#rx@E)YJELIa;fW4AL`8Gj_r z@+$Hx;Bya5bYny~*Yg@TntfW3Li9D2n;QO7n$RVEAkX# zb`K{HhAI;g&T6@@y#8m~i?X1K|S~|9m$^JQ*^lu51FpC?tMF zBos ztrf5QuKSN3cZLZ`-NGkY)O8HaLRas~t@_uq1YNNkZgJek8}v7@Xm0RWqDdAf9))bt zht`Y;0r~m!t)U!im|f(|xhMMDE_C_By~<*fvs7+z>EL)=<~XX+-eHMo)MHoXE$?#3 z4Pzh|T+2f$9u?rZLl-Pr>n`>!-jq&h|I;$T2+NF#Y)rClb#r|Gh{;6d#?Mknl2B)5 zV`zX4s(YP!o*vAObeK*hi85|zf8?dL**6P6_WmZBEXg1x;;Fm-j{G?9ZCzyjh8pEK z38YkM>!<4`16a%##dz>v281fob&XIwTWe@|i}l zkB9?F7kpZMHh{mH>%a7=JT@utWJ-O2l(ZLQ$5yT z__%O;G1&g9ZxUTBrX@^Q$i0Muu9DKmF3B4DNarl0@7EE=m_KuAGKG~VpFtUd)$Hzw z^bi+{-04BHwyF z6Fr^h%j4p-%*sfMWtw*dOL!KR=#KJg1+_ozRe_Go8dTdJc2m_|pM6ZWrpH{Y>fu&Q z=MxRrjGE+;&LBQU+fwC2{;h##8n=vZTmSzL2!t@LZ8Dgzl~vtKtJF{tj~JAKM;Lo# z{cC9lDZ;r|w1(2(-}Yd=OTa%znAO!<4*$F)DEz5rkiD_wb$1up8#p*~EvBp7zRHz> z)b2P0HMTKyf`u1W-*6QNkeFj~8Q*FQC|;JiST8bESi$xA%H|hKs8^jEj5iPzM>qvu zHsSNCeQt#>B!iqsN8`G1+kcd+b|{ru^GrfCH&G|{0KPVGfc|v@Mis)!y-*jk^I|cx z^W@6l&MIM6sVe%FOrrBKsDY>>BgGn~^po?#$DR1{9_c;>z0rQSFIfJm$rOE8GXv3I z{|ZW{&ujp3w1KfnL45Aw@cfv}GyI$zxYQMecqgoV|;S(E5g&UY)qDNle zlA6LnOL#qMD_v)XbAojch?_VJ%I3qp&Tyh$Uby`|(zH0&N!ap~w{PiY5E zj?a26F`O)}53}m$`rYOvr6$X~Lqa!UYWaRcOmQX#dUM=*yNnngF2@S@UW|fVfE0Mp z7~Tw|dQqqI_W81xpQc!e|6o3~f3dyy9aMQ|WkG(7L@ep2i|sK9-e#qUM&SbHG=h6s z?L`~+K6gSg#Lk=JaUC&3IS%ZJ{RJVE^C3_sPzc!yCY7sP$g|3Glmu|Y zQ%2F$jt1LWn;0_w$@XQiYe-uzDHmYmjjftxkRunI5Mzz4DF~oaUw966RRuUXoo}cz+l#Fr+xn>{F2zhNHj5lpL+8P45ov!x zeJn{QcZcEmLyrEJ1qbAsxPCSj9;Y^o$8g^ifr?Qv0gsPZ@-Z|Wzu!N8JC}X z7d|mRgpT7gOmL3R_QI4+o=I>ZpKR=1Eor0|m<5U7a$3ee&ApCF^;*8+@?Ut%S~7R$ zTQWd?Ws+QBSyvja&=dCDj4c#@xX#5He@f^gdXJ{W8TkYk+m9ps0y{)As>7aPQ9;97Kt0eR87J2HdcB z>^@U8^64v)6FwO{vpu=79C`DSM!00&#?p$p@975+HtOrX$xvCvD<30pg!FA-aiIi} z{VS*`{66{2ZC!%M^Y{}jt&Hwh8`srhxg3J}i*r`TViD*4rWWgR#--+|@#(MxKq&9Z z(M_kBxtWZlq_AvcDuI;RMT0=Dg=;f@PS?XNG1?OBSEDe_9kVk@<3&j;ZVf?UV1KlT zx4n~x#3!U`#oTxf{MlwlA;P*TPJ!v$V@H=$)>6(R3DV$NrF7TKrJeVJL1fvp2d&lK zHKUTcA(dZCp`{U<)Ixv4PHY&;#bV=))Ch!etb*>g(a3<46}8jHRs;$+?ba8r%_l=S zu7R0F&0Hws?G_D#YI5odMrkZylxVAO9AQs28@R`|H{EKC1?`giPu;an)K&tuVa2NB zElW4v8-kA)L#mI77%Cv`6!Y8rg6;eywXs^ZbmR@Gu90uM-FW1M$b=@Y&6&YD*=y-& zmg{Tz$YhvMHb*v0g7ixK&OI++;4ia&haOoPb^82@&n5@;nTgrEck{g!qu`^gm4>3S zGG&zfIU)Z|f)%G<3Z}u9yhR!rYFhaTp423aDe^-;j*%>kDc|aguLh{3 ziDTV1G39av#jHU;A*X0-)uw%pns`9`u-DBvS2N%1hy9qp#w4~SH{r=aKWgLs1lV%| zr8#oKP176|-kWp+KhV1|$Kyw%2c}_FsU^N^=Y~%3(pOBom^MmGyR(+~NH@6{W6#4P za>Cjy^SgCrUr=dx#>VYbs|-v2o4eVxeW_#Da@I^5%sq5R)ENLN z@IFMiL%q2Tey;GPzae&9B%F|PR$AQe;oJt9T8Tp_Y179e-^JOvSZbPoZ0gFn^-mBy z*RI2c_g*~WfTy|#scXB@h>98o?^^}hGL9Z-K(aRQJ99X5C z%6k-uckf1KJlHqQ3LG=I6mQYqPsnBD9|?^6X&{qi-=(7y^Y6T-x$Un@6O1qeXeAZ= zV9;Ht%3fN{k}&odp_T_RN1?k*OF@|!2jbraapILbRAz?>s7ic_Sqogk+R9ZHcxn;f z`@nHrz)|tr*XfoFYMl-%09mXtacfK%=hEl&AL-;VS>qWdX-ax}+r!$l9T7)Do~%Qc zo>2mUS?3Q>AyaJr39M{)^vH(_9FlQ=AE>_jJn`kCTnLW3+5R6?04CI#>Pm?-x9_Tu zvgCWp{JTqCl&-t4=fPDtTK^1CPLqb)L1(-*y^= zxSzvkA6}6C?Ssmw%JLrm9C_HJrBCyGQCI47w<5I)hqbOG=%sbPW+%1cHjljb^xh#K ziHK;EwDvT7Jq=3KaC)0gw-`UE?aq58^7tF?+)Kn~!+!YS)B|*qBA}2!#IER9kl@qT zpd2A`3y2ErBgN10+n6!mKH*`YiMPt5@vqC*>a`>`2uL0)Nk=atFV?z(rLBFF-SnUbyBrw!8R&F~9h* zuVY7s`(H}#h*=R%;ubb_p2!l(7NM1ZGoq?Zy(UG7@MX-jW5QqA;F^g=m$g>rZXQV0 zweSTB)JGj_9QG%ZK5J@oB?WEFIMD8g0$GruXql+5IWbHlB;LEvN83(1!VX!o!Knwh zOnB6tm!q-N+sH1f?CcqbqL3}kNHENY))2-9x_gOB>?tE6a`Nwe<8Tr+?bq)lBblaR zbS|EIY61KtpiH9E3`rhZk#^`pW|CkpB59xp@)<^CmE1l)g>I5kiBOyYmtrX2icEni zXVOED`WM0jBhf%};wCJsgdCN}Wda?dfdz!o$L~1`z(l`<;?##z<*@i#d34ILhD_u)NQ!p-8rjOwKuF)^Jq6!o%)ucG$-kep!_Z`=Y1d2D0(@+&&sH9r?B# zd^3P`V6x1WLfEpZOcu|59vs0;=DLg@-jTzn_-s_b>g<$nUqUliev&cnw1j@VUh~G{N*GnE%zeAvORpUdW|6V!`qs>A#KD~h#&`g zX)XYk@@2TxiXxkJ{v1i3xSe(IqmGRdy}?FEJ#ttcbR~G{z0v_Xm#ZashoV@=-8*9E z4GuI1ak8{n4t{V|zRKd+6E!)kzmzXChSxPmBOP#531RKvA1RE{&Dr;P!s4`o zm43nHd6C|Bzn%Q_Po8vSyzY6_XUWR0mkTq*srQjyR#q+QCU5Fy!g4bN7qM8Em#xPB z3oXy2qguKHIk9FcDsYm0pQK{#4|eoqUle|I`_@;Y*S7-tH~Q($mkQgi*O+6^5B*AR z|4Hv6-uvP`SKo7A;`%RJt*l6&&*aeg3WxH$_U043xPylC>$eu*<}+c@tKjoO;q~pf zgPf#h3EhU5e^dYHtBivzT~rSEd?2i@vr$~|<(fLKtBY*y{5KmCqK5NZI`tT>MGkc= z+48G%)X`BYD82$H>j?#xf(~f9kAetEyA#bfii=MotyYB9_Uh}cL~dI|CC~`yMAZpV zCnwMxa7gs5sNfcEGoq!WoBr$s}9R6?aUEmwdA_gJHyQnMD)a;sm#lIMvGScb+vd?0tJ2Wd@PfrN@H~I zI!ZC_!0P1t^q9QOu}^O|?Xu}?Vo?<2uy?+rNBFo}Y2Wk^I=X~tqM*QN9LHfBNR7K9 zu-B!^UXISeAsTcux{&$Ok~mZbPt)Lp6U6P!e$V_AUefx98a#|}tZmLncUlI9xT+MMuw6vrgW3&oc^s!PRugmD#FM4jm{M^?R-l@B0 zq+&aLcwPcFH1e~#uiVqQ%@l)8w!`WxSihKUj=M^*z}@QA6c1b92AiFprYZS>#gt^G z0bL&f3OqQ$=RM!Ei(-r1BqAI&rYI}+&X)88i)> z1$|hcWeNn&V*IK5aIy5?GH3Z2c7uP0C;2={ol^z6Doa-2%T8Y6;DzIC2aHB%z^P0PHC@-?cr&s>3FA#02=GN1hq zN_CQ!<9lY{c@-VI*RFRidBj?Ax3tOuC8JYf`4ER8r{n_l*i6SO!}MEEBZ^@CD2F

EH7<7hN z@w#F-LHs7ia{cY}h27)lZxWUoE&a$lmVR1i5 znO>6K(Jvm?##oDW|L`sU0)dQBXR<3)5i8<*l>blyn-7=tv`EyZishBL@JUg|16rwC zIo^1C#{0iQb#6+^*=yM?nYFt*fGvi;jI(K#^82^P^+{S;o5`5ukJqE0Ie(afmuv#8X((WccvS6TnWgW>(ZR-9wuW09B@cD3zyo=ng_9PiNzCtIz~#) zgLDWor5ml@itx{pZoxzmj=?74%0YSw7u@m z3vqO8O5tY;s{kz$z~Xtr$KUFj+gGmGifY*_;ru#RpU_I43kOHKc6_OV(K;Q&+;`z8 zla(1oP4i9r$aB)}Di6+v-#w(Te%Prql=&fcgmhQaAZOHu$Hf#bWkC1|j+%cKNYYKJog(nZCaPY|u@z`wf`-~mUW zF#)o7%)(oW@;k=)vwBq3GV$ALV(W!0fvQg(`u7aI4LAJEaVFwxjK@;E5b_Dt60Jni zdyJThgu%(+sgu#Sch!MmLXSx`*BZiu4f4jU5Q{4i;KtOle@(f2WNE#!b{g+z{phUA z82kCu16sXQA!;@BCf>mb2LWul562G8?rRUodlqjGv23Wb)}-beBJEsb(p(NJ`wRn(A6opjSn@`g4#U=JboAlPC1 zEqNa+qm~e^kO%8x_1pf2qVLGejkqdHv?_tKTli^_URtaV>*I2%A`@s+V`axuR zm6bS!nRi{~0>6!EpyQ z07G~kF!Bb1Z*BOS$>IcQS@xhS@d4)=?)HM@yfC+Ejjz3a+omdNL!r#^raN#GTcCnSV6mb4O^0WIBSGp+Q?aUxIb7=F5;MAsfFs1 z=uIJ^OFi4x5T5qSLyN;!dp}QU6fXt6*prAt4%dFh;g-IDIV${&X4phv)kNULQQU98 zCz|H@C(!8EpMmcfi+V|ApE8s@zadg zeUf4Q?L49b5*Qgi$=B|`lIeZ`U{;gs6^NMWM&XFuihcy|NITfkiSr|=*4as26p=0x z`(C;!jI!VEWDIv6?ke(%O?5rUdhUl>?6nSn6F*Mja*Um~??qCzo4OqxQWt>H`SX-L zB~fnj>$~grAMpU+tw#k-qT@jRzl;V>NCFZECp+$;LQ3t;J2EPnHQMTT z_!65XnwD8zOD-gw$=$5%R?toZsCrb0fcCeW%&+`^}zMr?(G7I*_r9Jk%c;o;k5!;V}ads&B~&w`qCCR;;}k7?t1j zk`2s=i68}SrYrw>ip(|Z%S(Io{!-m`KIcDBYv4qD`U}Rsd~>UqLadz_0aNHBm*A*6 zIjOqp#C5oQVy`Q|H;{JYS8R(WHY8m|6lb2Z{>`?2F%lt3!OehCy(J*fR50x`i1fj!h6s9P zOa6()+yRg{Cs|Cedk+rp4AjZ37tsN`-$bQ#)&Mah<{unzeE)>!Iw#2+UOq_>6HxQf zn6!U#ZYGs_>TB3@A~njcb|I$H`PXI{{jZExIZArIc_zLZF5!woJ3ep}=^p|IGqX^- zxF-A)o5KTB8CQxD(pjVn#&9l=It4sV`L?z;_}Q=N943NE6Pxg~O1*r5{Iio)wrewk@x9*uqoax&Bj?FY zg-!U^(`3!$7b*Xei9pzQUXMgh7sQM;-d&4or7Bs@PoD-=zjUS@f0C!uw=?gh_aI?# z{rdhaH>pwDujF87!-d`Sdv~72!tzbz5~uVyzvW(-x9?6HMZXPFJChF{I+^vXFoi1$ zvYYhsPpQau%m=|Y42|P;1%epe*6q;f@grIa$ZbS2I(Rghq*z?Vh#Ni$z1Tn#!7h`E zmCl<`6O=ZYj=rD^Ya~su;gx>9L;*ElQLo0^eSvH-svqq0?UN=FA{!?tg3fU#Oi zvCd4sx3DJ#YN{ozXHntL=V zi5`>xh|mAwAzV@aE_~!H2mklBe^<&3`UfwtfnKJY`)}KS>zs`Tsn*PQce^%;|KGL* z|GA+N<{$h5redk||Mqm@Qpw(=Gev<{x10a#!@mUv(5`(8thlp+Xa5ShV9~r3gLc`( z^3}RDI=Yll=?nqCU2N*6d3JS5dKH%F%pTM3=yfh^uYF_EL=V?dFQK$KC?a|Q)e;73 zO2FYY!dl~B4wt=a|2G~BEZ-oiB_yNq1pLI9 z5G`+Qm1dax8g6M~jRYl_yQKAzIh-VePpWva9nC{uJ<=+{M)4sdN0C!7nnkdhRd0}O z!LYfWGTh_fwCPoXr0#4EWxrGHe|9zi1O)!c$y=l*HNc)YDi`3?XoNWWw!UChb6K5v8q22VLwIZ9fPGn~rz)2vMXGdlPbX&{9#@7&l z|AAwM&oOVZ3m8)5HN9C^L6sgH1i35FtPSYC#;ot8qrbaH{ah4va6Q#K4bmnbJTxQ) zx;;eS481(Ews%!(t(=$pa-V3x7=3j4_3OV#zSpXjd(jVGK}xdm-|W+b6(S`@e=Ax3 zl3+CC0-LVdoP;g%Rcvu=5NUvk042z5xrLnZOTF_d5)^ZQkNa7Fni8lYf#;ny4fZRA z?WEHq;q}|x(gz5YNaoMI>%J#g`R9fovr?}@8&rdx3~G%?Z4Sc0be7E9_K%=?Eg?JshkN%$r9@uMnp!Q4fA9jI@~=GYjppn?8qom-AU)t z!9U*6|78AuohILCFweR@J?Z-=b?W!30RGql9RMaxFqNaJf3T2$hWY9Usutj~pY7Z1 z`47+kQ&Zrtw-ZmW!DgHO$-w^GA?rXpSow=H8~4{w*Zw#-G@1mrApXM~c~yf*r?s4m zBTivYX}UV%NT##JKivNh<5jM}Ro*(&ZX}Gv>b+r>xhB3QW1D2uHcB&4fY?HkB^m89 zL-|l{7V~4%)wO))jrpGjnP7qPnyBzbc@2D1-W(f7kYeKVd5tZaGdAJQmXI*vV4_l} zX85tqmARvyf}7q=@-I)~Z?S<3VFTvBgjIP+D2LTEHj3aD?rIA5XmfSFKux!NZHSXB zIuju)4Hi1F;$H*aVgvWjB7Pep3lltQtY{AznjLcO)YcuZ#Erd{F{N4uD|%=pzEUJF z4F9f)Zc#RPZR~eP_wVB~w}LBZS(8Z_gL^FWy?Y10IlgNXrB;WN7*hC+C-23|y7{ky z5dWOk-zW1Gk|Nfx>%>pE%+pq)Y2SotK8s0tF*;DuYp?6`_vj{tYDR6Z#);a%Khg{c zZ@6~)*`?+J)4H;_Wkngbq?B~>%UpZ!U8xXWMUc3UDe;L5BPE;LpFZxtD^8j;o`rrb z`9{15IR9z9Fr^hZV|&V5rH8%+s4J1_++T9O3UBJNOD$F!h6CgC&2hMSJLWC669Hzb1u#smQ`r0ljLO`A_edBS6pvHLZ>2AU@$hDR@yu|rODhLti7xP1?Cuv+3#X^S{-?~||j;bDYbDW!U zIUHq6ap_M+YUcjqUYe3`z)4Q;rKseRC`AC*EE7-iF2ASsJXeYT8+Z zUj9Q!hrKAA;uf$b7ebFrixnGH76~qS$gqrgy%jzRDn3Kk5u-W@=Ohl5Hc+Ph?|bw= zCRqWNzZm_p!n_dYR>oV(eS4!UL_5!+y9CC0G}AO{_S}W0*{Go3En1HJihfH9{~|B}F7&T?S%wJm8)t_~ z4JjqoKR|(OSZ}DP5(g(AXA-jXSKQxAGmD>+YFM38ck2H5RzIUjAP(k;lg2ApLfrz~ z!YiQL=NuxbENuO zxxW~9W<)6(M=l69owx=EXVTh{~gf;QhCrsP8R`q)UQ-IAfMbfz|hy zq8A(f@8Bcd1u`4kg~5ec9yVADo+e|+6M1aIT|8506JG(i${vJwHBgTz(8`VSUdnu! z>p}A^@wRh+AjyQ(HYC^D>}x|Ue7x0 z;6hgab)|vjvFo@-Yir2(uIW|w?pmMB@cKO#ObGQphqCEv|3-C)*KE`2#zpIo3RwaOAvw(T7mh1U4(K#^8eVGNCC);?&SUq)6B9zi+i~sKdK{_yumsx)<5bVB2e(IEm7tP z{U1&f7a|MpFF$C~c=^v#xP}m-maYmk0B2^|pC}Ets1?oz(r&EYL&&jjx5G z_4}pTKP2!q;V-9vCd;ka<`3tH<|4h_6Pp%W%;0D(@2A2E9}R zh(g?o@e}oF5*g*xa2%MgE(WiJIwlF-er6#(`>Eil_t#}877)x9%EJrMy{=}imY>P_=PaDfgKuo!c(kQ)B62E(5*Grfa z#~?bMO|9DALcJQ}AyPfJ050XK-a5+E)ym-OBU$o#FVYhwS@|Y0Na7uR@8}_MXm%$t ziTFu++~_p+V1=KOYackM>uNWsX~!R@F5<uCJ}60)|jwk>($U;tG%Ua z!hH8jF~0G!ZtA7dQ`I3f!n1ef?X&i5%CyejdUK3Okr#!Ws!++PL$BykFE)E|#O@LM{uYP~d zS#pCgZ30tXVE6tDwQY<+k^OvEgEHN+F+vR1xY|(7#sni~2xkg1MwocPyoH_AZbN;+ zeef`qXF_YiGB|r6txeuYyE7{MwT9%Q?q%ReySKR8opd26xVe*yxW{f;a#ucQcq4S5 z!q~ffq|x%^_GB@dhO%k@q^5C0x%t^cnkVT$z>y9FQLrZsv(q!c{R4WKt+T&`Er~UF zC$BN6^wm3v(k+!n1tB|aWRoWXNu&jVOgCqv@ze8uXC{*j5mUGmlo(%mdLJJ_yy5(x zG++2DQm*-KSij@GwPEslr(JXH`nZ+e2`0MNjR^vogsL)K&q5Q*?VF7C*wnjlycoh>k20!t5W7$+tnWDX;0FmC699z| zLYzwXQ|?N)lL_(cj|zjh)p3cW1=?S=ZvD-gA*48xPO=s19Ak^K1hVfk-?XGxhL{gs zcRA8rhuOXW$77kAbwbdt49`nGG%nI`5)e{Wg_8%x)jN)}C7zrk7jDHLml5q`0QZ&j zL;r2C&!|BKXly-g+YGwd^26u6{%mL!6HqkdX=D}t>Hyae8BLo_t??=5y-73iz%Tk` zqQEF^J4_M10kIgIaUqU-N-A!OIE@=nw(%hPq02;8{LeuHVr9Y@S0Ev8sg{bX7a9cps}dsq+;_oK9%gN8JjqY~b1Ek~=zC)q3R z$v;=`o+KJb#@jEYx;o>OgwVN~NjMoENM_>B4n6-BG}2_xo93>om*yh{+;KP~mt!09 zyGZ8%$zST;-kMXn>yv+(|ov!GVAIy=|eh>Vqw6ne8DEk-`r3r>W8J_{Y}=xCT%G6)!Yy zcT{Y#HYf0dRLp{b=5uYU6Dxix?W#Stw=q=VK}GS59{tjo(d`;JHWgDq z?&P-4^m%Jwn{wIhnM$9ITic-Rnr=GX@`*{2b>{CGai=59%#KGBPj*4dT?Dsr zjd`(4&|d9{0zN1lRKKF{4E+kZ&Ox={fyS}?f;&e3<$M2(TFoNxb@WU*XQ}3-1Tw#@8A7RR`G%W9vef0Kbceda0PgO%@66Dv)P~j4|cj|KV_TQ}eH^Tp3p>9EH==UstKYMjnwXxUBa zjMcxc{tLGL-vxnu5DAaETnFfnJ)@)8un6jT*4dQw5xHJkiSAsTx|`kxeA|UrJ!SNl ziO0`88QWNVbZmcRWBnVsIrsuc$-Asl=h$;nRkCK}v?dakg7fA6ndl;$`-GSC^=l9I zGX*HuV~aGAY|^ZtfiLI(HJwK^3HoIRho7EOh6m=fy5tvKjY=0M7+b?5^6N>Z#aNMk z6ikUWV^(=B>b%QgaZh3Q!ni4P%M5Oxk1zjsO@I666P;)ZCd<;ZauQx;kUgUUwC=RD zyL!CuW~Y6yX(ABn(Ti%&++BN8re#5{N9<6J9=cu!&^DzFpl5?4<5}aqo*wlFaL}La zOGjS8v3e~fj2-y}ax_*k7k%_PFgS*YZj+xMi-tm)GS=Xs)y5tog^O~>oV!!zMJ3u> zHawdj@d_X9d$hDugk$eXD@+$I$fC8Iw%qmI4O| zl08d(0ydccK0wCkzwR$k975YcKc#cFOz4+O?7N#!OwFVYW*vLpmJus;NBsK<)#cdL+pNI^^QxB&3}7IZrVRD z3bO85XhCBue@6*;XY)$3^|sA=@p4mlzxNz6v0cfWtaa|N?0s*J;&4GyFP%;KSPs3_ z{6h7?8Yc*$j9RWh$ZY=Kyd|(N1R8UUb)C_nzR0>*a`6_^mL)kvsJ!BJ*r_Htb}{)x zdHU#r1TB}dk7un~UF+=kmDd>jLkE{c|~x z)E2FOQyz4sXaj}MKv2FdKcp}LRvHXi88ym(EK2}Z*iS0{r+(gjKR*yq7SMH4afhPg zE0GDRolA0;$zkFsyLqbrc_MRJagjFda8RuG8O9{AdRHZ7t6{&F?pez%n-vwbC$fP5 zP!;|tL4Nd0qHhuhRaxhpLIG!)41kdyBCd#aD&DndirjwJq)73Y#Ek^caAQ+vQp#hs z^l(4K##lEMW>O4#f>`vy0PFLoCPsFxfN_#{c(Jt;`P^Ya5%?^pEY3V9H$0$YI2u`O z_>W`oe?;>W+7Y4eb-*BR^tp&}SISJ|7uqa8Sq+)OM3UM#TJB-@jc;)0bD+n;&khj) zfDfSU(kRi}@)?`=k;LZHhzQ};q6HuY3vE5UKwiAvKt~70(V(N#XtoCF6#wk99C9(s za&NZ#7w!0?g!w_n7#nWD00^8pga)>^f;L)xUX0NXGi2l-!=l$Nu(es5p#$QgR4p%xFpL>7P9( zA8@7W2hj@$5^X$_|8t+CLqPWcRaq5xrwCGUc4Kyc`x{O-@}D;h?9HK2Z7q22OVA26 z+wr({Wj&!Mv;ag`7RqCv^l*U!p{;zB;7wCXV@m3_ExfI_3jx9t6qUI$>ZH0*0obaSlsa-LEsm!sP!oh_6X1^TD^n zJP=aKOtOy~lrv;pbtf4VhmP0CG2XI72hMD=OicB!O$QhLN45?)?@nXOD61ZAvnOG~ zoi|pmdPheSJBr_6V_ZdA*DjJAMl4AcA+K6%Md3J(s4@un01IXD;y^es$>rdvgIkLx`d1=yo__X;+&d(fqknTM5dk=jCVb2`+po=J zIE}nqLOORW7HZo0$&-qZt}L@~mwxO()732Dd>NOv9x=VW!B|v!O5qX7SZovu`6o9A zPt?Qfa2IdVjOx{QlRp#wlW*9f1sXK-xOHKJ>EyAjom+cLsMdVF!zz3W;__-Q4y`b{ zf0W1H_t8k^y=k7C)3nYTx1Mo|7;0q!l}vg(WYy~M==5`LqXaYDBy7-rpLq==6xUK) z%4VBw#U^0&mk_PgKy%?I=3^=AGgtv9XlTM9HKNn(PnDU@96SaE4$Vl33%8%l(>RjI zb}W@AT|80llmJ|*W2q?UnwDdA)Q%r{z(#@qe$YsYG{(#|b8=`KP|Q-aL!r4%v)zHQ zRU5kw@3q6QRjfdgdMEt=B3yKK(q#H?ojJjYHqnasOWw+YB1XTl``BCObUW9~Y^}QP z-ZUm#psC+nG@uR$ur@b#h}4vj{IeojB13`sc>1P|Z|_ zC#J2pBk$N>Wx&k{Ux+z387_n{-W=#P#6v_=5E){W5HwBM~$--3YFzSz_w)~Q+SxsY% z32WlH5Q`1b_57Zr!*>BQ(Hy{&&Fq$aJ}oP|rqz&*Gj&C>kHbF)SoZmJ@bB(FUkdw; z@=X)cYMrel;$>t5f+*{fOWoMN{BTf~NjnrE*qX~|!E&Nn9oXP!fl#*9EK6&t!Jj#7 zQD7K~XbU`d*n1&+<=5?fz0Mp+ysk)6XE-K2a=(p1Cy{@4Cyo|kO-R2l#EUv0hH`3=y{_$q>^cIt{4&0{v@R>wDjC|V|kwx_!((Ta9 z58C;ae=FnhJBMRm1W;+u9aNak07~21mbrJ70PN^CBp$cz=Uq`rh(j6lzTCQK=^cE2 zEOg0FV_rF`UyR6W-UcZxxJP${Ba^owZ3z$zNL3A6_caH z&U7!#?hzNh&9nTz$ou4^^TqAM_g7=!w_7p^N|d1PShk>J>#!Iul{{{3L<&_@T$zI! zZ4a%Y>N2&~U!UPyomeo7T&pJT_5I|&ph6FMFY5Ffs7KYZMf)w?QKEm37g4QhIG1_E z?L(FqvL=EX(Y>=&d~{-1VQQ7$1PqL!Z-rvTScP-7DrHx&MW-X%p+i`Hn%4QyQ}jmr zo$t}xE-tJvQKVsGV^yAlGaDk9eLh`@g1Cun0c{EHIRC1^x(MGxU~sV#n-!rZ2P; zZMDvjvWRGx!b$)Rxxxp9&<;HWs4 zY(Lp~)T_l$$a0hBuzY=T3Y`0b3U7*h+$%&F*~zN9@BnEy87h(Pa$W81utYFGL^rf8 zt5do-qjiA=ag8Ni;u<(^L~7}SH1A$^yF421xKyae(s#L?N0MKcr3RV7so96QGs;|A z`o7<4al(24*N(?wZDV|H-4r0vJ6(TNV9Ey0JVgQK z&bx1m2jt^JCfSi;QbxJjRsB@RwyBQ`}-OAQl>q&3(X!@x{JREP!Jalms!HiRjg#)TIGH=0?3lQUXs zb|~u%*BRddWkemM_N73kUr)V$JiUY*t@P(Q4mIXbyFlKNkgegNUgYVI7QiC4%!l1( z%t%*T%;R6d(^SvnxS*0fAHrmWcQ7)gm9UZqve~7)G7x*QHATZGN83DS3NT8EW?y0$?F- z*7W?KugDSE@cQgo);0q$L#AJ5#m2Ms;(rFR+=I0&a!#4#9Oasym8Z8#14R)-V9MXKOVzxneMTB zemq8>2tq4&%>`L(kbQXkA|IJvSF+<9_r~H`ErHLio>RlC$>&v7??S-X*=Lgg(35h# zzzXS|v$L-v2CJRgO&U4{kBKBbCw!PN>3a+-*9u%=UxH4TWxO28LRm&Sa0w5ps&}k? z5`#ckK%|vWfKB|;dk81rNq!27IvM$|PWGHOl@90jXD z&RUwf%=;l~n;mqziXvAuK7;t{j7_15#HE5OwV^MTa1gVAjS_PhAAv|(LUz0P`}LJ) z;qePMTCU$+FpzSRLO11p!e-hMYR{g9+3>Ui;9=-`j)@hJ$j!u=K?u5`ZBZf=foUGG zlAC4Ko4Y@`!-@gJA;#Y%m97e&Htsa)m+*SUlokGhx}rc%^MRLiNxw1I|v)&k>#O8FdA1PtDteo9HG27$0B{gv>9XrJq%Jqh0m)c2kFqbM) zds3UqG?F(pAGRpIzno73IAO?k&(+vy+NVZ`5<0}+XgcpIF!#P(d2nVBQRuj|LM5+T z7YLBj^dWgNm_DXFm7GId6hgUwf7~{rbKeJ5ru#7V6T($PlDyalwYS;xY7TFRLVLc` z3`HP(D0E@r-bwbTdYtwBfxj}CRpbu>!{H0P(?czE8hgI9_>hOZE}iFJqHBH71Tzbd z63$C^ai*%m3MQ*NCZZ$O!`eTgcY{UL+9c1U_q?0Ct*|hCb?pw_FNfc6A>ZBt=azsF zu>^W4K1$asc;}z_%L&E8Iy1-_Pl1$Oi2d#(jP4=EEQ9j~K- zAx&S6K8gy(oaO0$*7WzxxvC0!j1s_gKdR<;@sqT4tl#jnU{+M6{#ijB7Tj%Q5k>r( z4)Sa!j2jho;RYnX7Kr>2+w9!p&Y7@kE}B#+CL9@oD+$eH`-mBS#1>}4rgH`^+SWV z6b5pqhEAd3F@w@`7w7v_flaX`sT&6pX~Tm#*;_u46(#SWzV~gK3HTn*h!LJQ1R$o& zm&idY*+k-yv19G@t-+w2VQz^R;>)>=U!`NGaHxd!SdJSi6{Qy!Qpd}@jwSB-9 z3!mH1kFSV^8e-7iCe(3PjHq3zup)3xSXT>AFrhWh{By$#kXIHo|AWQ7=^niquG4iZ_q!aOf#*R*2U1%1tJe3c{$QZfys5Q zYnVGdWR&a+ibGW7`utgPa9!gcKI}d2U(0>DV)?{6h^C z>^cXYV|3*>HPY-(A^g|c6y>RR!1N!yh@7pCKGU=p8y&fSm5O!4sg;ql^Ym{Nj!HPR zaKqWPMGV1=bzGaYP4a1dJHsbO_O z-KjdK0N^XW*SpC79IZBGA#(N0ZE}88vuOdX3i`3UsLO(2U5s=g8pU8t;_LnpDMR2* z6!w;TyXR&~<{LE&6}mz0oo`zQHLh6t^BSqYUokus)*KfKgtl>jFk8SeX(`_}?vp=QXc2-Oa?bCqeAqfCzx25rK zj}45uaiJBrDdX07-9l$@JFS{P2a1i^hb>&1-Mv=pLh1Eku|@R zCfkXtyqfUW#M|-??@^6m>+qRlweH=TI&6er1nS{$`&Z)eVP+jOF9XER0u!=r#v99s zzFItB`;Ymi=Mc zfVr^q>lEhV=3Y@An0H5IsI5@ooIDd>5l3yKh25AR+66$clsH0 z!nIfO-PE~5M7G7)am}Ek2DG5#`2&|A-Fv;wnVn`!&QPn%>P)}d@$PAIVAEMJ1M|Zp zS)g;mn$-zLmmI5Q=x#d?GOVYbh-BeU#pnf^AC8slVj_uquy=7JaYIch){v zk^;;;Rd>uR?2O^_dT>|PqIYojq-3S6cnP>ju+12!`H!hatymAi@AT!Y&gPYCdh%&) zdql=hHOkr=lj{fAhL21PgG_q-k9CF>aboToP84-#I#mfv@t-Pq>t!|6lUeIDz0+|S z5lda3J`)Mxpj7k^$54r7?yece;44w0O04&GVgTjs;C_RLgkY~l7= zDJrV}V*|W7t)fdOmz^zf29t2~Eu?(T1$ipv--t622{?Vjz@3aril>RQBGVAOu~eyB zbYh!tg2=wCOiou6uULyyn}}I;0p4x?;EToXy=&op*;|6;4B}ok7xVTfP@KQ{8ei@0 z8r-}AUHiawjx)bCf@?pHRdaVA*+gBE%{!S*I$HAA4*$RG#&c%Rxpucn89J|9Bsz}= zudtMSV>HG<1zUzTUp0i&WqHFw$Bc@B4X)jF9`fD_ZL&$|w|8xDFdtRm_3nEdKu!^N zq9rEx+rFp9{e4yS`QUaT+By!OV|t>G8$%?-Y$bIUw{4jX8Ccq;bTqI4Ul=8d_5DT< zLXi-b8hs^xA&(j}4v=a<(=m$13qH?Su`>`>`k_AN$I5SM(f%mmU~5-ZjTnXJ!9PKH z!s0^#p(sgPpMddhy%1qzuQ~`h&B|TAK6%;ud+g(QDwkuvCTnG$l28css5?bi{B5L& zB%B&Ud`)`vJ|5_vb0o&6H%}6Sp6qVfH`H|;cVcv*m?iRwK9~Gyo~u3p!8b9?IT3}E z8kA6hWwpBRrvh~0V~@)8{cT%VfKUkCm+=_&RKBE=ob_aNCG?^xrPtAX07E8>H|-#o z#GcX3`+YYl*)k9Frf*fJ48pzNk$$Aa8L(Bdby1uIFB?YyF?{E41oB9jVrLV4V?`Fd zw1L8gS49%DcI#50jOZT4h%%2$oJzH|@Lp@>u#uOahpeXDnFF;^1fLa_Y+pg`CR#G; zE-`IKKxHd~$Ft}h?eG~3-4r<~&xANsoaUhD8jQ^7yl#Y&!c18;Qx!uSex+0a9&l;{ zSyzVsM$$mH8T1g9-|L+&Fi+`z(D=|@oi4z8&dm#=;ZMGpJj{S}n>`dKDJZ zE}N1Ur47+UGgNx5?n!cw%QL)#t^nE7tb&Agoz)W~Gq|<7K1NO_Wj(lZJ)7*xb{R$P zdqnSiA@CVH+PJ(baL1+|Rf;mIrcmCQ8U2Mjff=gkpfD4$13Btn$DB$J!vyMJ?E<)?94}ezoN0|pA8u9EA8muGpQ*5; zRX8r<2L)6PZuSev%Mm!SPeHb6Y+K6PyO(m@(Ds~nVc`He_p5XdP<1w|?Bfj0W*XovAw6PzJ{d1XzFXl%V&UftByr*BW zVE7!hwpMk2BQ2^h)O^JEmeiK=Lg~%)tm%>SkZxSsQdLCkkpXoIAF%KdTN>}StoKX@ z6?mcZeki7w4+Dkr)M8VR>1m6P&gTiA?(MGT#CKT7?$r62X5~RJcBGxpe7fJ<#j-{ApUM6T&ha$lg5-fsV%U_- z+2+1}>x}w^HQ&KsApL9of+kCQH)oxpD&8G%up}=e%y!+f)Z&ecL2e&gwLTSNmoTKK z*RM;EME|~=BM)(yuUY&~HEDZI;Emk?^%l?PDpPL`$v-+kfZ$Wz_SV@4iRV>VNeEP` zGIQG+CBvV6(qGSp6rA)w1chcueZIM_(#-kDqR)6C3?D8eNbh$4J}6y}y6|k3OcV|_ z$M-H9kij4PYDM?-OibtXDnsY-i_-1uS%A1l6O)SQcxLeS*^B}Su+;2tovbprPW_&P)!Vg{+n2BtjtzBFoIhI_qjMXr{k-I^FSpLh8E z<)g%I5UTMhS;%T#J!s0mr_%+cUzO5^!r?9v;*ka*37D|!cR=qf=5`$XoLjka;LQPw4$95Fe z=V{7?J116DJg-Ew=P8n`iN^<#7ZN!udh{C#CAgr1WUU@mTnO}Sz6hQPZwI;!$1em_ zx_v9w;^Ddt>LoPZ!7NRSa18CZo4e!P4imB^Z}(91ThtWDQ7Q+*%5(rODA?-IvlEuQvPRYGr4s59Db6Rw7laM9 z5@2&;0_{o$F?8MM&`_MT;1Si}g@y&c7`)rH*=x+LCek)B!ZqFK?UN3_Q8t)snuk_K(mq)vIF3;`^EKY^Ry=fk4LyGqE3|xG z{<4xuAJ^XgDY$hVdD>w_W{|M1F!AlW1+M9r!wv=WMqTjKeq=rGYSxqNWfOTx`Mk#2 z=A*Dm)tPHAYW&Yd#q_ZvF<)FLKvJR7fWQl$7TFOHov zgE&{MXJk{$Ant&1_M)Kj>mDws)mLEVOPAm8ATj?T!-g@b`%tCMCO~AFFH{boYKBrN>; z)5*3<8z!x=S~k{FPGQ%Xu=sfbUqzdU+(5kp4yg|7-ki4r2HGfCQ`Fo)QacyW*<5%| zIv3@a0~w=r_glc`@j&4Pn&{?-a34d=9nZvfMjW+gAL$#(2GM z#X=28+K9_raaioEYX!)c1FBx403`HFZ4e@?U7Y(cj+O0IwZ)1Zi-I@g6Ssx1h@$3& z&H`;4$}ZjK6?Os^V*)k=a1y#I6Fojq+u(MK62Ao(Isl3nyER*j($kB@*)3B@)c1-Yp-zt1UB_j4jTPPj;Az%4HSSKZH|| z%j8&_RX1AMQwx4yiR`Z?(9xUSxo*aK1L~wzlm4Y+XskBIQ!)N}ZcR(^FqoYCt|QrZIq(3W`#DvG4-~X!H zTT9<}py>9wfaSf7;7n8x$H|}}&Na3=U%6_c%g)#AbN@_SWo;KFH2k8$Ffy(Lj7yBI3^n>LN(=GJp+=6Zr*oW7D+U4_p&4up8T ztUN=*oAR^jX1FSOd2p8lccx_kJC=?2B9V@AczLzS&Wr9op0R;qLv+X=#-8B;9+o8V zt@gB$EbDI75YR1IP@h>Yx#tNa(sQ=fV4#zK3e95hjskAuV2^Zame4kQ7_cTTJQN}Tgk`C;F#IUcY? zW_b(8C@%-k-SE#n_3opLZGxuip(V^1An?wa&sb|Mg2jn+&G-o+olqQ*_}tomP`99- zrUQbU0w<8PGWc-3DORJsR5;^QEcPnD0X_y`K=n0^D;oYZ?&b@9qLxzHu|pf9p1(UU z06LU`Why|Pov<9f{yohrn&Dzcl*>iy}GuhI`keD#&g); zj*iYkrTs)`v1p&uac2y!(1D803|q)|KIGi^yU@w_8EnO^&^soB(UZBRIfXn>JUKq1 zHq&Pd*M!CWJK^k;&GYDesT3WcKeGbZy1~OGp<;Ab>+s3z`TK=y( z=X^qkrnYvi#1`5#otE%)y^J!goYCJe1&@$&NO9j9>Z3+Qi5vqi=RZQvDo@Kog-S`TWnN-qgEM+s3*! zn!osqw~GkoVb{6k29X@XI}-w4B7LNaJc*apl5l_;MNi=MWFpN3dGB*#IX^|lE~X$s zD=rpl8ickV=>j~Z_`A9wS7CBl4u+ZElFJ0W>PXh|SInsOpWl@Cv|FQn^sb*Z(z8<= zm)N8<*{%Wvt2fHCsGf(97Ameqg7!(rl}CIM431SUH#^E}IXj(m?;L^0;@2i~k5AnW z3e=8vE1+(P(->oUh_i8Mgw<7hLF3KnL6x7PuBT4p$Ds6VQMIj13N~Kor{zx}_UFv9 z4ZvPL`x15C3;m$j@$TAZe%ewszBTE#AE}QI9%!GKkiEg>RpZ^!rNCbD-7ad{jm`@! z;J$6{$j)|s3;MJfv;=oXAKP5p=&B0&UVmJ0TXKm zJ<|fpLJFkbdW1bHvquHIiBXDZ?VTEn(?~_|?C2nrMt7`YeCrzDTDt}T9%?g?U9Va! zN-Xsh)D5dQa2P)z4eqS4Hp-m})OjG**7NOC_c(8i5yaYX+<~2uNzR_TN+n^=4@CJ_ zH5y5_%hs1g9ZHPdo|crTyZaxks744*XF!eadPQ=UQH{Z2<#BhoUBYUy6DIfNO!5cr z%mhN^<#Wqv3B`+TL`~F{uuiuR_#;Vfik>QD+K1=Nzv@xieqgUhei(d$rp8!9f1`|5 zy9V=V{qza*+q=t_j{f=W;0>@C!T3N?l_=bkT*dOH>rXs}j(gwXbDuWrw|4^%;O07? zxA1~u#i<-mPAJW?S}(dfrQz=mx@3u7Pk8B`HsI-=ep&1QlceV%ZTlkZtV}3?Pg_JI zZ`7pcf$>o|+Ld<-VKOfOncN)G<0JIS*zy3v?~OyC$VXz`qN=)_4q2YAZ}3CG%l@?e z^&H5=$F{l$xbU!V4`5bsm!%txcmCe|=ffBcz!r&6Yv{!eQ{3@PTUd0ty^I&cokPZ? zhSFM>nH#Y|!}J+ae~uJRGs{+4x}RQ~fuUsnm%e;KU4>G0jUNajB#=kT?Dk-i<{94h z(COZ8>2NJ7Z)ePcx?Z?)WzcxW1Vy0}ag5jSg>TZm+9hM%a%2J}MfX|Yn>A^^D-B)- zA4)qLEH+P}+TG0YUlS2{f9F6yzWu_zGsu6T%A=g>+AjZ`&<=RB8xPNjU}o_qMMYxl z=PGB0%p2-hDDI)_p+*heKXVhCGM@}s{y<-t+U#~AGce~>kxA12qn6H^X(x)0BwUzK zk-d<%8G|7;hX&mihH->cZ}bv!n0&CRSV4#_C^xvr>1zjZrR!!$8+ECO9u zaZI`0@nxGUwR}GUU3KMr8cYrra-J_aEy$r>1DRR)G)~D)Ut@`R!O?`OJI_F4{t%1o z4S989Y6n^!ioZpLnV1)*FRhvK9GQ=ncUOzU*1_FC_w=~R!i!|~qdFI(pV6FVb0_DF z^c`WR4+W*~GDemBg7Ug>8)>HK54)IeqqVbXJ?C$tyJ-ViF4b2_qdb0? zwJnoo{xgu={!PqJU)Hfa(=Qh@$`-CA-s>O4sVS$vLr1S(A|Wxsly7L{cz#Druros z$;-Q*W0U-t1S2(2+TM2owQQr0@XejwhEdgMO%St}J06~BaJ`VkzAQh++(W>-Z64Iz z!D?Z<-O4xR`N{Bw>dC@>T=kC2`H3>tf`yjNK{xRntqH>&BegINx2cv=t@CQOqiJ~g zBzwPbkn^wDP9*3XEV-me0x>Cwv;9nD)Og#_l|nK~o1#h5uMdEMCk1SAJIC^MDo5pp zg?HGOQ8+}7m4{iTK4l)?Z+Pr}_J7}wDB1HjeW0G(+>1b$;z}dng@E;_KZ2tvYaRJ$ zkmChz0Jt_O$-0CrOj>6T)h~nY*hh4FZGDn}B8DhNkXd9;ETNHv2gb{Gb6O0fUhtN> znML?gt2b7W&C!Bc2ViuµvK7g4_3Zf|OK@a2(Um7T`X?tWnrqh2sDKf~$`;66% z7%8O@`=B}qE53L}Mj}{1_I_k20bJ(&IJhFou)Do5K@b+_gq6vv{Re2WkVF_}-_L>N z)zI&fRtX8P=`~ksL^Q6L(-0=9+1s3{W2Foy9nDG%y85ppBVv8$ys^&Q$;ZLjx;jyT z$sO_v)elJa^^qywh8-S&<%@Oz3CcFn=r4}Bx9M(x$A!peV#BP)-j5TMj?KxRZ0|=@ zBi+?N;Vv?lE?l2375B5pjjcQRmF&mY9s|z2QKar4IPAz0SsRfQkx25nIw9ZJv5F7& zmu+aIyB?v?Dqlt*wRvstXS)LxL0>LzP2L5+5Dog(mU zzUYakt~}5@59D&!dV1tfd`Qp;Mw^eBasFSzh&E=Qp+p3j)a_Dwn!eZTuDd6GKWGV~ z6?EijVRD0I{yzZKKq|j|1aj1E-y0=k*cPpCga&poq_oWc-)@@TLe}1;uCaJ$B}(Q? zq~LET=9_3*RlH<2JXRPx-iZTmo{dTgZE%g|ed2kUd+oMrCx~x&j6@E2RE>B_q{nyC z)XyaN))^o!`dc^nx_0t88+Dk(cDVuXp&sxSI7oqNd3oXUT{-S8DR~2tYrr@Y&ve&4 zu0+hNm#0dsi{HEohFQorj_tPZb&wwzA1Tw#Y3P?>xhyM5o6hy1WF<`DB8!iXvE*>s^La-XXI*3fP|aSX%fxwsno8SZ&`KNyosCjy^OJ)iyJI{Z6%l2XrzVp8kg*)#S?S`MwL@5crvVX^lSo}sE+ zQKP(c8BQ?nV;I;pZ>j`8z`N=%GQDGipVU9}UA5msl2~*x$CO?389r6Vvqa1i{ zieDfVZBv(Tpp^k84q@m{1@|4Q856M`u8`>HtKpkm(y*eO2e^fy{osrx)U7<$reBU8 zSGC^ZMOMoz#0Q$+c@J+LPX{KK=P=Jjg{$T~9EU|N6-Hh&C<>PL&-3u75~nfLyq*U# zNvc*O7!VLzvbbT@KlS`(TA7=UR-?~k*rsrCWByz6qn=1SbP1G|sJ`H5sXp>`N%V!a zl>{}w*#F1dl6k`bC|6gAl+SHXQnVy4*r?E&n=xd4&*M|eLdCG~j6v&pkqb34z=am) zbRVo%!c3I9@OK-Sl#>LM>1<{1tWmUiMV5hMgaka_AzX(qKwgWS1MNZ4u2-07wM@P^ z6&qN?qX#`M)n|>9*ofV{`@(AEoBOMT?%!2HPwxc}0=xJ{=75-|6?vE2~E30*SzYlJf2`pD8on`_Q0LLs)~24n6klk6t7E zu9jckGr#Atcy9PY@-dCSuEmxfB1AdNMa}gncpx%{ZYJ3qAEq7$veSEpL!6fEoBi?c zHT~$g@^n1hd^nInC9mP4jj@D5+&A~y!!@st`E@l`X&M2Q^%ci!%7@oDPVfFa&H?_Z z|K^Ji@>u-x1zM|BqdIuq7H$sL?ViIVa>CmZIqoBg_g~L@*+VwyMaaBul!TDa?G#Lc z0>v9>rB@a3s{*$3ZW*l#Y9^GU?2Qh=9A_hS0*irH4`HEl-Vf z-tTBSH;|)Rq)6WU1Mk%R@tCA6s?4QQ9%yWar;8g?Z$2THi~WvG6aUTgas9F{?w5Jj z9)Ww9tLqm%P(6;gUwR(>I4otgHd?rLjqE4+D71gd9LLT3;W4#kr-fyszV0>h3eS3C z*p3puV2VU`g<{=xuySE&Wsx5q!8>Us@qc+E-n*3VRv_KuW)=q7SP?wT*bh{VeHoEI zzt<=~tZQ8U>EpVQjnALqMbvPks?Q$R98W%KLi>p|saG4!>V|pN^1ydLoyyE_s5< zb(8BA)}XAeo^0`XXszcrb#IN97o^8Uuu*P}>b{ceB+pMNquke~QX*^Zk5k0IHHH`VRePVz{VuOtYoDt7UA;E;z#5^5@zHU_ z@41iW`MhrOdJ1>VcyS)z`?`PTzB{>Z)_ryII*i1qZCI~+@4O<%lx9VHu{9<6t`7Y4 zHd5f8Tl(jhr^Wx{)7ZQGB(V)okl1?1c#*MukJ3pwYm8*xig(pZ=X+yrWjSBkQq1+c z61Cd>`5}#V;CQQ1-ZoB-Ax#s3(+Ezc1&<#q)6ZN*rk}E^6x_dm@*R=`$HWyf^P(X# zg8VU+j~ZBhEgeKP0&){){@FO%t~ z4u`SvO1w`$k=aO{n0dx9nQ{6s>{~Vf`7lU>8EpaiF9dco&mXMUnS0%ay0*=|c^mB~ zmBJF4_M^-=2iN`R_L_ar3V01Q>yoXp#58Mwc5(|Mfb}Bq=~WW==xT(5U%BMxy(=a7 zCag9KCQE$K<_I4jiKQ1J1M4R@1c6Vkk6k;7k?>Vuw@L@z`M zP_fR-f`R5u2|d1(1mA(@M;H1u$qO(H^co`mp1H`M=SCMBR4B6a{3vNE?P@cAQ42l^ zW|%$^7yt`2lwS2b7tX|s;bJMccW-!#PF9aNH;61_R{YjgQVFkA#rMz1ygPP+rM671 z{Uz_;po(ev=l8?oiOq67jS&pxpZyOZ$WisS_c;1dq_JEpJx_dIUwkYHNP|q-*N>qF z`{7dXz`+Pfe@vAGj;k6km!elLRAs63qg!Rc{Rc|frwD_DVJM7{>x!RWKvf0=6oA;T z2;)ywEYuBd_2jR;{vHHV?N<+Z;m-_rAM>0n!1vKPAiu#TI6CV$9CI+jl0n*oe&eMm z!;+B&&WX!0j>o+yopd&OSQyQp52If4BRLG!#v`%rXo;NtlT;u5j=1prtD=;JF!LXu zkc=mG5&zdHotN~Gq;p71_C-s2Ldw6qU-JmN>~WQJKk^~zwh!j*q!%TR9sxYVzMLLX z^1&^#;K4&>!NZ3m+T(dOY-p+l2sw+v+d;C$;@2<3ydH$ei|4=>(pRb=OADVk0py#L zYLxNvuOEeibgmkL=sl18`?C~64h!zx7xQ|NZakNm3(O5`>lPx?gz~}b(i6|~A}}UZ zB5yN16;Vl$kCOLql#=&uPy=BU-toxHao#h6a}t$Bx}hq#Z(mvX!-I-wK~O9g#PCNx?&h>IRmTF~So*(r2^^&oH0Sc=0U!elS>|zg+TOG|nE^Md&e8 zn7SQ!E5eC$wDLs^X%7v{5^%!$+tAvx7=ty2%!s$>Y{(7AhiU2?FVdh2>n%09an z^1Gibc;Yy$VHZfrJBVz8thKZv(j<9#>r4rLg;bYND;Q_eCWLjIN;Um9^nKbc+%Y<$ z!5n+LjNA4AU&s-S`~2@97=U-wR9h|Mn+m&T{y&z0;73TiASXZ?9SO|>JDEloMmbxi zJ~F(YWOdb3fANU+uzt`8%SgYH^CESy=47A@T)dQZ?WSyJ2*eCl;N3#DDl#J&N6{qN5WrEd4PaLfJ!4Llrte(ff zE1V`}PD}a)z}K*ylEFk!2c26EdM7hi+Z;-DHz?s_)tb%_)mTA=WUmYBEzXNf`w14= zEkSVivc5dRzkeWeZ`lr;-uEEV8TkHHDB>5P49kTQL#}$FtWXx9_G#(ccWLfx2^5tu zJj*1iS)##9RWa^EO*PE}Zv&MkDmOGzF;#(H11h`4FI^6We>(EHXM*(Ra)b)+E)JBr zD9XfCe)A;4)VGrgy!?t^ zz6v4ir|afF(0vG4Wj|5{%|-d7h0mNOvAKVNH=rN9{Xb}$1H=BSr+$aWW*_-CmGKiF8;qiUF4Nd+ra8{q+x5- zh#L5I(YzsyoY*iAG0=0UdZsuhrSILQ28!7HNl--3gR**@&KqT_>`Rp2xqfr0`u$xf zCu5}W+4GU(1+O^LRLVblODd7}zzrMwE&cFj@HxOR1m&t6O3J)j_SJLTaH&+l=rQ-^ ztyPiM4PL52QLchwR{7nFnomyae{Ak#7)f5o-2JNT#7evr^h*>*8!F&#q+DgE;;V-> zl_3WHfLWKUFO}avt@l(t>2r7}_LcJQz(IlZh^n6<`|n;O^KV1S5WKaOzrHOCA3X}` z22)#Sy(a-GNI7#3Yd~g>!1q{xf?wPM*y;%`bm6nr;M&9Jh*V?9G5%iu?W0oq zG1A6hSd1bSs@`?!iw9v)>L&}II$lcNzE(Z$l)LiJp44YTSn{fw92rY}?3*etOlQFw zlUCBqmRg^z-}E4$BzYN9@scZ*k2K=*m@5@ez5i{YaXpAk`vK4EpWp@BN8e3(+nuK5bCrm!^T9aGbb*TRK0#FHD&Q)*PjWUn zLi$~b+;pT0lz$DKpnHaNJKz?{-Q;lTefEdouG-AIXNHc~^58QqfNPsJO%VXE%3haC z&*PufeEn|w-XQtAT@GG=fnIrL5L@&8M5h!D`+?qF5Iu9BM(Fgt=sVpXWsllJ_f?fY zy{vmjD&@gJeK6Jyo6k$Jf+on6?G*%ko49OMOsY@%MygMPSLu`=mi(OhqlC|%Ak}A# zmH03)y))!o!;0{N4}<(F8=K4B?|?_Moe-k63D7&hAgaR>-RmZa9Qn9Jj(BA855GJ9 z6^ZV4v$)v^Z`MpW{((qT?VPy1Q-dl|#=(!j>N1mA|~H)>?X}<3+{N z?cm#`&v~EW93K>@PeE8S0{{M0dY$>MhHEd89*K*n^_b~&-QZqKJu|%DWYvP`%-^MI z{3jUWw$kUquaQ^&G13w?KtB0z2s!=R%caAuWvsHPzUaE{a~(o~9nJsD#!E48XL!BK z6H1apo&?rdEqVJI+#7OwQH!@^kqA;MIoQ?Ds#cF$`hn z8smj8N(3RegVqHWo?sYmMHdY-0ul0MLf=kOh_9Izq?~2^&Po>Mw7i)wCy_4Cn?)wm#T?OwqJ))86 zKPk@Sy70s6Q2Z7mjC$lFWcMw|^U38=60C#1gA)0l9dvewjJq9;rU=If-p@Lb6VKQf3&!%KX39 z7-OBWXRW&%o;N(dTx;M_q9KHy9YzayB8@a7cxieb^SEwmd!P5E;+rUQwR9;{TXMHP z9v++ueMX~D$aA+kUWr`mBPQe2)016>&`m~B#7iJcAkXuG;I;Uz|EpJ)7d<1K5EgFsiw*c zl_MB7taUO3XrN-|X(dt!mi7UOFxT_odpAku9cxP9i<>pqo_k*Ihod`Phcu|qB#u2n zTa%;esXwqhJB(gGNFgFkn>s+?yIH6Gv1Z*AVU9 z_8F$f>Be~V?eG$Clz}>xSLS3olDo}Gu)eN}m(whiAA}->5`$E-LxV=a8%2iRI(I>5 zyXIfxrz&`A>oj_rxw+P`s5KGtn#^y{0MjeqVGQ+F@*MP&36DKDssSit+_W>4U8?{v zcbC_NXGVuvhRaDX#l9+rs#vKc`^W?^LvhOvSi4HWa3dJP#WOl6zQ z!+jthBcLcF-GcK5k3rVD$l;&VlD&Mv#A@$hxX5k?GgP^|Rm=;55V_ z4@*WGq&7WuZwjO2T#~7t^1=LB&Baa%DFp!01F?Bil&gZJ3`v&83K~qUh?gR2g7=Pl zWl%~E$^lWIp`fooaAKxY(#?13wM1)RvSl$o0Z!PLX;4T~EEh z4j$zciB;ZKCxR`wFE}SY$FdLu;o238$c*YvHHmLi*Ns`*N=pe20HiZD+)4I~t{}F_U`@rCxar+t|^1Kk_o-j0O3Y`N&#segL{y%ts zAy=D*PVRx*iV{fgGS9Yc3IHSFu=j{FVvi)Eb)r_3aE*~M*xMApiV|{o zSJL?0`#&FPbsi<5PV!O2;h13Lm8S2JR{-8~A9Rweb@xS7)g*+vUkay10FH#MTj$-= zLPDBeWt)I2aX|8uBI;Aw%;{<1(c<^DLqgj?IfEwMH_CdE z5JZ(DZfYSPzOgSz#{bp@vDFFU3m4-zES%^(w&8ISIqgS@Z+WICM?5#34GTAwVSl%k zB#t`f;osiX7e}Bk%KwenR+S@q79%qzYv~eCkxjm7SwoXvOU{cD#o+rSi>tmoIbIT? z@MV+qO6%r1$+}rDd7^Hr5zg8bdE#YZu+T$!CJ9zDms43jdH;OND|QUSA};i#CKdJ6 zi(3h2$P`+Oq%r07T?|8Ni1f&=EL}#QgR(NG!W;g7(*3~Ov`h?JY|bATF76eI7+;oA z)NR$AQ{Gn!kD9GVIw#26h@hGvHYIqe;**MO@S&_w#>3l5=t+=}EyOxQ48ZCiTlWYF zpE^e3JN*~)!}HT=ooV3SXlx*9RelD%gh)GJ_%_L%IB)!u3T)D%?=l`QSp~y^BB2{L z&2#sZV9hb-ywzA@7pM&=jI^P7@LS>`9Ea_PUQnqv1Px!_F@}3 zPodU`3XVwtX1)VxRMis6$}_!oxT2kD0#GlNAkwT@_67-(2XbO6!#us4)|j4 zi$x$Y0)q;E?{hws{#TBZu8305=PbEaG$c!@b!?ey&(fxMiFJwVWJJrRa!o1y;ATya z>3--{66}Sjl-4L2))aX`#r!d(kImgMoF6i>!O+n^-RyYP&#q(njzsDE31*wYd{;3;tjZiEmVcxqN;ehL{~ znFn0!(4#rr$MuGpAEnZU9Byqe4RY||jst3Ku(iL&%pw^dNLl=|oT&s}*_32wSYv#MI-)f+(`3r zZbA<(*ZT#NV3#9!uaXkp4cI@Ge~z^hYX`~sdfd>fgzA3HL@1SvAru zn0wx3&uc&|c%9^I4)$i1zt(-fHC|r$kfbqXrZ!~s?7}6h7~r{Q0RFy3s7RV{M#p5hvT6MYsxZD;jM7lgn>E%V_xK-$D|tM zukK2l>U+t70jYJRdYL2E1DLNJ#vX-fMwugb78lWO36PQ{K&F{7WFzchkita$4AD)R zLN;_wa4TS6^70MX<6tepej#tG6ObykJEBy^V83!TNX__e7^?Z!c_4g^7nUFu>0V7g z%U)-9*wXNR(o)HY;-dYQ>`qLZL3BXZT`z^`RvC2>8>YHIdg3KpW@nOVQ{mzZX;j*HSN}IB+H79?5S?2tA51D_* z-uUbT0+y|0_ElgfCIL+CT6`<|uA6avtwV{8SLypVr`!fh?+pu2Dd_6?(UKU<)q|%i z{Kt1F4Kz0CMQFx=(tRqd=>AC{c0&G_ItQ{4{=38FQho6(Nvs1B7zO98BTg^wSDmlSqE%_-dg&lg{Evo734%D^C< zD%DhTdw3z0zJC))xpszU10keXuxFk%7>mp~*fq7X<#{UO|Q2yyHo3+_Et{yl+p#IHtp@RpLl%_-{X^Gf+8HYYpI4cH25Jx!+r#r6g^&)nJ$AQy6g24zVJ_! zciJf_GbXwzJa2bZon8|{3PKG!@Lm{Ktn-j zWo)Wod?~p144HlPIvTw}B~H^L;5qGry!=4VK2r4574q+i-N3AV7Cf;~l6JjB4Ppr# zLq>KlcbQmWf*Hk0N~oq(ACq0T$>nG6I^MelJ^xpdkVZ%d!YQ!V3Y&8w^f+o7n}1&%1ez=sRaav z`;;mu7jv&W9P4!Jlq{l=t{8RS=iRWk%)4nXnRoqOD512Ulza#xL6jwJiN-xw#dlAE zVftF{{RHD}90ZJ2zkefS!)}WTX`^spK-plVV{Li1gfE?sRJo&dpVro+IaP_p`abCX z0#qvidv_|01^cid48j9$o+W+G`AAdI$THsdg0E!Yt&?S?JO2h>$ZHz0$hE5HF^}Q9 z*)s6v3F;XSpxiC{&fWS%S?PZ>@Y^)$fp=C9>^lbDJQI1<(?ACMA6efM0VgtaMYIZ3_nX&qrH7S?-9=GXCM>mAQc0YY2(8^f^F3I*>w?}?#S%+2Gb zR*%BZJSS;^RYDVvhafOnHOOrfEVJxdvGimY`Gwj9Kc6!R8qOqTBZyyg%dV zc@qe0K`QT3z3@qIGJi|TPy))y{9Cq#(lr3=iBGCG)C;NP1}3FI@jT{YGOsmxZfy@O z=P()0S^dxVGVjKHVA(%ax`2UEy$u?s&N{gvEellk*+bI%jF;fy8Ue=G?bMUjbtjat zNqI0hHlCv)%vf}4!j$lDhW+w;9h#?vwMK{@^J|L0WX(K0AB>pQ7a{zv`zrck_5b<| z!hnwg!zYG28qD3BKv_i%T(XZR2U;TzE|da4nD0X`o~RDXZin20_2%HkYi~A1cxX+J zMv%k!Auj14GDM93(Q*K#nl1#-cfkBct$`MJINNARU{92w%FFu*8{rUZ10p3RY zU}$(0H7sd~_IK%z(5rRynqJ*;yaY!fEd*;?WcDAL!^QgYfnKZXeT9+N#Z6KOo>WEz zG&e`uhn(e^=k@c#^RcdkZ@?N>hIJ7oo%}!C55{`rjAJ>AZFZc*w!6%e8P;R1gQJ_B zhe(?p#5V=wt8``HEy=LHmWYy|?2!ZtYiwl7*UH+>=Zm@h!GTa(xwb<#7^x6L8984L z3>6@-3V=aX=LC@j#5HSG5LM;()o}f4^y9}yg$5GJBcl)KabomD#;RL+ggd-vT$gf6 zd8ATJF9N;VeC9d-ED6O3h72guJe(!%s!^_s9tSG$%)?W*SIvows{HUe!G0qp6U2nf zrDZ5QQz_Wu@XANPUd}&hND87}eE_61Tv~Xni`?$4HFnl@kKXp`f8mKAwhi7#-4|2t zn)0~Tdk*fOJPLEK#d?f&J-*>F5;@>ejfkNPgdP|zfnPt7L|1r-_xPX0*E$f3e%B>7 zzpW-Otapv^G94fbsQ@eqM_Ffnq(;@?+2VdBj%PUx?8xRDK>jkzsp+$fy-?-|)D6e8 z$vKTzfJXw}CWcn}Ay-kXOmC`owXv9&JIj38+JGgFB=0G{vjo zaB(*m?Ajc?egwwK61+RwOH(~0xu0BY%wD8IJuI-vqv}SyG<3ra^^-cj z&hLzrK}6Z?OE-XQI8f{h)jPkU8=r|Vbf<#*4pskHY=8E&-_tx%Y}&f+b}7Or)|LhL8~`TL8zpD6BURBa ze*IDnO-x|Uf&=?Wm))<#`5?}LfX%*SZRvKLMoLL3nTkcPlV4IV4vik%6mwIcwUVJwaPt$P zl7;&SaHESqsi4qU!TNzz+UZrIvQ4X9w}bD%=6Vivs*n;Jd0uW{06C(vM1{-;BM?I^ z^+iOb7{`)KDF)IRAJ{2bKY8Abd)Fx2x&a3n7D*tup{+nzRp}@9N$=B;-#l~+tzFBF z8B3*5(V?fE8!HV($BT%e(duPPD&ov{iQ{?l0Rv@&B(YSFAuXhm$MUPM+r+hB* zCNU<~NoF2X10JxJ+}vpHb22SMHw>xGDK;)%o;F#mlAhw!BSRyJmbs$I4KMiG=cS@? zY6y;a7_N~lv%}|~vc&V1;Hch)elez$QBB7{%)Hpq#eY*TZ!&q1#u+Mo{I}MarlEzK zGVhtx#?acD`$~ox>%Gz-YY(n_EWM( z9P~LNk3mwdYwoMobJYylsQ1*RDNn?j3droSp7EDy6bJ`6FapaCfIc05tK?R_1L-Os84gHa9sZh6w{ z_@}U69JOd|Nh#aEzOJR0GS}MGdpF_3G=(CRf{Y+{w)YutD{&o5DiuC=ne;jPMGXmN zBx)|wpID-<{3{sNk+Ffh?Logux>cni*hd%$r02K)n9 zGw?nnIb@f8Zjd>bjzC|>VDEgDzIV&NeNsx^zX3`(Jgj}XN**G}_@VP@K5AHJpkjs( zp~5Ba+=V@lBi-ROWeKPhS!p9AVyTBLud>a~YCEEq)_J>EwwnuXy3besim6hD(qP$e z5U{F4%HWgkOD#`-%~7_~7Dk>Lu<{H+A!aUF*(Z15?4RWl%#qSJuh6`kY_QOhsgEznY_R3gdQEoE zSixi{*q7s_awBTj_Qy-+Dz!dy;h{&99)+BZ50wDC42&2^kXH)UL{45Y>h)H9^%5+{ zdrQ{?F4C80;gct;M@}t#c$XHx3eOLeubd5c!a}$}OShyfkjo=1%Q{v=ez@t4Kuq>2yA;NZheFF$-4EqLr0cw*0z`S3E&yK`SHt5f{a`5Iouda?^? zac6#C(%Vf%cL74l=UsPz%)4VRDSn$q7UqR#B9HDqtv6fnzk?ADyeH)A0u7JhHR<78 z`1DCK_s0EX&hW-(LwRRNsg3pagx-yM1RVxZ^;XcN>1cV;5O~+P4~2v4Wb;1=0$b zZlK)9JTXaQ-etwXf(ox$JF+gQW2`HXzSY-*T6cY zQ74+G)<@X! zweWuTlA;%{*OUW|+`)J%et2Fz^0`|csD`BC*Dv>~LNq6}uaV1rLxIEz;k^)7NO_W zdre}O5IoQ<$exYR_0TFCfusZBpH<8`gTly2T!gP@thyz_dM^_{yo8lMzodJQ{M|2u zLccTWpJLIQ1MehR^3=w<90YTs-Dssk3MHM9JF?Ic@OX(c;BAl36I-6GG@- zpYuLf&v(wohd`+Z!4Y1Bth?F;Bq$ETCaKtUKO8Ko2ya!71v!GC%u%7PhOr=H*eG~b z@5J#JB46t)9iK7=lFbo+<L~dU3lSem|7?zQ7S)P=;%_z(T}DIrV9-fVJ59d{JP2 z3m!ZOp8r*4)h@DBHp%5OpLRxkzmBF`ftU+7HgiV%%(1tJjMf`JG*$}kX> zfBQIyvc_QEjzvA>7vbTalH>@1EaOE{f_!p%wR#?Zvqo@ed7vb7Fvq97H)I7)y@Vri zlmr?B&-c>^F+^zS#PJ%=OgX9e3dWzUPrx_|C2PbMs<8Ju{u%X#ugIpdqHOlb`&f~K zv1_*r1XO0o9<1p-)(nAVBS3|l#$Q`!mW@@bAE6N+KFQzteCc)Co6?tc#Lxaf6>Uw+ zQ)d3wZOf9E6mgy3*SaeZz1k$%j4uF*CZZy{?0gaS6}KZte=DSF{Nj~nqY@0r=sd{Y z;6TaW;cRce$Fxe8)`VBO3-V*>4PL@Hssv>sr61oe3m!TIO8wyoeTDZt$%^i!t$6p$ zy^n4oz5l3DE;MkY^uYeQ`=NJh>PrIqXDYo_NDHM$ol%}$5TV)g*eAR_FNgsB$o-|w zqU-LL>t0-yR7T!mFUAn{D(>MkeK_Yy$=muAIF%q5h$t`r4tsnh`*gu)F$nwmXQVs! z!5PE0h7q=0y_)v)j48D)+km2-OY0(bxurC#nbR}kj0P$}ZW z$ZN2mvpd7aE9@#pwt z4nk&TUA{VqwVp;OuICxmhU@ro9Sue25`;WSt|SQ(gC+AyYI0@O%TWm{+-1l$+el{r z-#Su-qVcrMdB^g@d>ri{`JHzeE_7sxjKeC-#Z&9qL2**nW9a!t-%OZu>U55vls z+s*K?R9~HStIBsfWGaZ40mJ=iP1V}N8as>mLk+>M`y-Sed1egPX6t^yZIX$9XK!?{ zXV`~Vn)0V*TL!KvUH80NjW0PUm*7@|RUC?O6|BPaM3Y0H{L2UN;wjd?VhI0ccvEC5 z2z$^d?^z?Z$BW`_k5~vVvjo z0nZpTTo!`7bIvvEY4j(Vq;t1F3!Dyr%A6~ju!aorES z6`TMF1J~%9`A3Q%&X!108zbf&}k5cNskBvu3Qb?K$4b2?b=*>ZY%h8f(CaxFobV5N=Dy2WjV zq}i!$5P|e7+t348^EL;iXZWy3{{2^p(*zGK35!^+ByXn+q~9fDP~`j*c!WRG{N$8X zjSm*{Xq8BT9!DrdzC02pBttK$+$M9&Xd$HalHn}&;L@->2p3wNZIqKvY^IbeJLUDmn0NSsAo7m z$SZhhB~VM&OXU|{jwt(TgOb$RL!uWIFN6FY&QS|2bHe;&F4Ry+)XOE#{5wPKQZE!L zt2F$iYEBaGRC254DB^gq2CH#}jYM=XhLBsJ3iW`?!023sNQ&S}!?ipV8_PMd#DX$N ztK5=MPlZLm0*nA15_%x#l&8Z zP6F}=o5Vc9C6G_WJb>?x0q?|w8vnUDg;y2E{F9{-bfQpUh@#a28Pla=c+%-ykmfTC} zewiCnI+s6gw5%pm91z;>2eF=BhmENby5~+^cQHRi3(k&pZ3whH z0_jz@lS-hH?jYoa%66b@hLpi07>AdZ`ObW9nRBfKQ24GI%x+o&E2)I~;G8T%Zf)6T zcY)a)p)+N1hr@Zo^BttBjSEFD9ts3{XDn_h3TlxFDo)Qe^dEXSap<$`}S#9djem#Al%RF-uypm!SHM0mXEZ7=)cK9q3!Nn#~1 z2+;ePov&nAk?G@-o_K20U|K{zOT4&D%2C&rOu<}CwIPf$&~G*MxR-r; zm#%-=>mk3r_I)e9rfJta77B%Dou%4b+rq0MVl^+(6lQh*Oul8v?piVjmfLAc48*cr z^k(x#zvJmyh?QzkI=YS9CT`79?A}wV~M}snumxBZi zgjQixYH(2z#PegsbsvRaC2C(%w{EGiG!|$(&JbwZ z2&7TjrpLJ&N;RIbBvBPhJ!A~q61gc+sr>Od^>}wX00qwHOpS&*~s zG=IsLnGN|y=^o4r8SD%0-4`tR*DEP7L#Ki$2NdW&1Vkl&X>NJJJ^O;Jc4#@+5EEo}#V|<4yg}&@Ly0QX;r+<=B^{1hq5^!NRXKR+p*0x9vs9MrQ~Z0)~Jg zUQ-WWlx5ZhB zi2H|H+e59Mv*ig;t^*)odjWgm+rS!syjLe)OU`0%iG8l;bi)7XTSCrkqP$R3=CVfa z3`75cD7pG4D!3etvc4$!Icy8ft0h+v!{YV+K=kSaQEQv~RuZ@&Cq8fVIUq{j1ROR` zqW1TGh!p$|oJl96&mAz|J4&{>z7jv7*7Ov-zp&Hj<&a1yGzK3$|DXka_j$tVT*1)9Y zjTS$x`NyU?FuaDRh$+bf7f!8v3*Lbvt!$&jk|Y)wkt&V+i)wT1q!td1l;CBR{%T}5 zdf198{l$4tJ{EUV>Njo*)-JO#4@nfVZjFGSwN#33M63^@)6Xjm=$s zV9$FgRS4M~xQ4CrrA+}D@aaqlF;I1)oQYFUS|1^}@?&s{XiwY>r|IxQs09C#rT6Bp z`!aU-a&vIOU5DhsP{OrlnUX7gBF^4W&E{B*aEZ}j%cZmuV{;_B3bCnL$XQ~%+otgi z=wptCH406~biLG&;F)06vIgQ*`2*aAHe*I=8C6cv2B7o{l!9yj)ejg`OX1pRw=0)e ziL{g!V`V~iU+3d1aTRVs0;wiSCzev8)Kv4Hj1aA8tMv4E#M`vw9oaa`8dRi^+%+L!fPLPVKG<%bOJKd@g(rd82DjbHrsvM5x;-sHSP(fxK!HJ zQUAFMxMg;aO>D$<sPGFr=noDJd&*jxHF+d86E)rtzl2l29f9kdnid2CxxU z#k%;4USuFCd^sG)lga*5n$VatvSchS4m$AyHRg!EJsX$SsITZ;|ADHg_=W<3b{1?* z*$BQ7M`J1RyI4LDf99nBFhO7U8PkRui-gabe-9(u(FRp-LhkR>Vg{y160xfhELbE_ z-0upxNmYpla)xRNA2#sIK&)POuB?39E#HCfRBsx+eLIVC(}cBuM|DhG&361)>G)BS zQP?eHuEY9>i^)ly77!T}nwC!TV^O$qA=}~ctvas-?QC|5fYtl{KM3J3Y4K5~5Iv_3 zu0iSxJbTVoYO6Vk3!@qJk08#jFM<@T7^}z6j?5qM_M^E!;PYl8e>?O!JLU5iEbLte zk4ZrCveeD9iDa;!r|V_Ny)vsP$s}p%vKJ7|HG`R0y=7c1M;wJ$ zUv3cDx`a=yCjdi#n@koO;KkPvc2W1ZifV~@{0I#uAx#( zAdZwiJ^PhhvSBTQzy|3NB$sCZ5k9Z?x0ws$Yn`0}h+1v< zw#aH13ABS*hAK9Qlj!A_*TFO=Ln+lWt<g@0pv;GtO<3m}T~X7-W~D)2a@4lfSZxJV@X z=>vK~qh5A7Aw8JJ1#L|g5*n?SBAysu`nZi`(SI{sgoZ}}<$SvyvtMp-uWN4KKHjEWjo-{;puHq?wBV@1k8S`QD=*pql=p|bq-@f zU$1T|5+ZQp|4xRwg=Q*e3ITd^P$vjXeh>u&mHngs#n)=jc=#(2|Lu0YH1`s_Z zr^S?GNm%CLq9TVdAMJL|xXzb315}|y--Lx0eB5=5lPKO}jEXe8lQ$zeqdA&wnjRKq ziFq^5Ez-}Q{0Sy){XKu~$MYp7YTu_2)vaV)>T% z1}2<&toGrb4U9Q^-{6z!?ka*^rb+;ZV=KpfQRA(2^-oX{IHz$j&mVeW#H#*AC>yKi zjTGpucUfjg6Dgdu4mnm0$ogHZJ?+xV-{w)w;W^lt3))hyslic*TqFax3KzR6eWKs9 zKTs~TTc6ZqoRbh!v;MW$BzoXUlSvuee99AZ6q9B-8QZ||Sjq@q;VM>@WTAsmw`;bs z%o$fikww4GnQZskPs8d=(=uOL^O}Y07Yp%V^_)JIX{i_TmH`=HsRXKeMnXAp(gnIn z)ntbe6YGepws#-XI242~VyEYj%}T~$q)!U$7*iy4KF}fDTZRuWQngu|tZ#uK4ZCEo z3bRjmucjy3b(`3TZ5BzbelAI2+EO&Mf<$ahX$G+~T1G#g#M2Jj586=%6pkMiEuQd3 znqZSQ{T67x&8G+uf}4l(vS3naEF9#8G9k0Oriyn>JWc(Zu7jTUO?c%tHSfQWtsd~q z$AGf46`xFZj6sC~fuRG<8zo0sb-2Ot%GR}|6vm(2Mc>W4L?;}q#FNxZAFjsMnhz4& z7!`&lFH0NclzJn*RLI@d+*;eoby{IEd6kOuQRlgostveQ-ifVck&#pE;xqZ&wT9tbH7GXt zY;)o`BE$|t50X}^YQV+XVahunrkABBE1$27eqyK_|6!w~0c)LDmo%|muM^0Px`+wZ z7f3&vwp_)S6ARjY8%-y$7Xr`p_oS4)WnPhL|qc9+k-0kH=8 z95XlXmC7X(uzxZ%j;#O4XB#A}g>g6CyP)g!DqgbLaE%P~FZPbX1{<-tNhl|xX$?a# z-2TH-0APGC`gNM?2R9gi#te;5y8cxNeZ&z&w+r10&k+%U@T!W4v4u+Q`*Br^Pf>R6 zH<#%F586^FsdYsf+@VBW=FdMM8joncVm8 zdOq15N5(sEFv!=&_!tIMMn35;6wDCMl_l=tr>D02#p;b6k<|_3XYtR&7EW(}=S3>$ zj+`xD8NotOO%#(Mqf6W~6kM2mMzN1G>7iBxh<3_QMb-c|jrwV+WZLdGB5XbV>dtVe zTuc-@TmRBWlE(085?4O?(x<9!iCGwm>T{G4h2|XrL(CEy{`1 z8@k6LYQj zbvA`V^_Hs}$7e2QI?Ev|UaOjNb;OJ!}ErAGLM=0eLX@q`FSs+6}9Do;O+F`o)+@lq($xOROzR5n#_L(45tK z{^Udko~gJbR!I3VZcH6zMJ6ftI(VHwVsnbED6unmojksOC7J`%{pQ8iPxLrczXrhn ze2dl~(T{0f54W5kEG#Rgrehm#flX{>&(SW#xWKzh)AY=M7N$+^&m$dqvE)qtxdRox z{81{!dg&>raraig1kx65$Wl4rV0U4jU7 z+UCUfvAZ&p_eDX^Xis2QP%fUKLLf_6S&vLT8uFDZw6h>ksPp+qHF-s4Mdz& zx7N!c7DlNbzM16yGL#*X`y{$y1Owy5qOLZFGcZf6N3qS~XrY1)*m%e67TUfZK56#7f2mReIiXW?izk*8s^ zn@@7HC1>{1ZJ~O;{+%ayQ!=QdbW631`Ir> zRSWAgeYD+a|2mWzjF?H}rnCHGUGUQGm!Xnn$K#HDE?=FGaz6Nrtv|MKWKU|J3p!6R zGB@JcdMf}$I@i6F=~%F?tea1^ptb}o1ZNJlczb3crH}KE+JNT16)~ZTzn&&BhJKI{ z6s0YCeK?*MxW)^+o#z=+Zd#BLh;koCvcGc4K`s5ZZT0n-_YPYP_8)T#ym=|i_0a2oqS-H)$aLjiS5sj6hL3+>I76HvRMY zb=Bc_)bHCti@cmz)5B>t&6Pt7iuQVJQd$QmLFeJ#%uYQ1 zYHF{%&_8AT*)~d3A-3Kn+SWel-Z>Vu^Ekm5t$z1Z9@G6}q zeGhcvLN+;uk9c@yNnr=>gUQII^1VLU5eRbYA*bul&4C#9~#o^gX{gv8E0y~WD490xr` z#2uLObg4xIA>OP%lGHK=8GU>Q>Zqv}c(w-IrjsX$Km3jebXWX2r!OE9@o|-%yLDAC zP-8I;V4A|Ut0$2UIOey)(b3*ywAjjE{&iQO(Z$LazsQMb*=H5&;a!$Ps2sKr`1=*> z6&9YyPDr~9BW^#jih3SKo1=@H!MOZFTyU`WB`Yv=l>G@;$V=Dygj?EI4s-|*W#yIy zqRD%&*SclNkxUMGVkvF6&On^+Kvg|3Vrj4XtB=txF7QMRJSj}G9v<;#&s%2-A3lfg92U3~QZ@xb)uI}%xhQViE5v9jy?MHUsJfkQI6}K$JOw!KtS?z4`&Dq%Kdf3Q8iC*24 z{CT7tFgm~)ys26}D9rsqM8+?Xw^a3+ui#V&7Huj0tmRI>bQIO~HEk_n$AOTk6!CN7 zV)WfyDrig{1slV8S_vF&;c(ig9$XF#QGEkHiR0Y#R6*2SCUB-gnvtheGU1d8qEYR*=kNU z2fIGBj7D{Bm--ZEmb)TE&XQ!jX{0(*V=R<9rpA6H>~u2Qu3G zuzb4PT$9@MnEo^>D_(WcMXjXPP6WSse%a%MUvNo;PYfs1f$!uSJpA4T4VskLG zUIno%l386pE*n(eCYuaBR{g=LQLkq7V8B!4(aZb^grqofxi&`kmNE=pZ-Z(`H7T#C z=}Y*I7s_;qp*}@GE`XfB=3($B5skrV%M~ zfxuM_#H${ZobjYM24B9->6Q(%-O)D|M+5kJ4aDIOCa_5hqjiLGH%h1tEg)MjhDI~s zv}>lcWf_aa$T`kS0;7uqi`75ZGNUzsPvp=~M&YglbMXqN#Tjp3MR%$sSW+4%coWfP zF%6}S6IX;M;|DpKIp{Udcn^{Bvuk~<;>BAP^KJwsm50Do}qm6Wym2?mLh1XqL52aA^$xCum8&Q};O`5rFnK~8NrkKAmieKCN{?5b694#&!f6J( z3zn*QDQcmaw^338wr8k3MUosp{3>Tr`rv6){Jl6M!hz|$6fy2Q{|HM`P zEkqnSN0L&vWWN0W`7!w+a~rp3PaoW;qbIlzPgtX*WD9rI?>Q%r)w2wW=J*HR@Ly}6 z+Eco1ZF2?ChQKo;llzL7)C#8?39mv7D0*@%t2dT7{t9-w@2fcO@!Sz+8>WmC{sbh;e$sPyX*JOwE7$KSR7D`uJ*1sm4)QIeGpv;mwy{Rt70OV z>oK|r7rdBUe}B*N8MTN~A(_4`!V+fyvQXZml#|gSO=ZcWH61!+nswh(zbBZezXop! zPD|X0-?(z2`v3PkartDaZHd4BoyEd9-J=ktJIbHR!3iWc9vU*3E&W>_+uQ4B+Md{ z0o{|7H^OpZgpdNja<0sW1agLX4dAm}$D09h_&Wl4Md9AF`>8O_swdib75u_N;+Nc3 z)FW`9weHbD+t~srR00uSw)}sk@1!=?2;Nk%tqWrvzW@O%pujXLlmuB19dIoLe za&3#F-^8df<^T5RAA3IxOu4Dp!GLhDX+n!%aeXenk_lI&CLZp`YtOh3CwX{A z9tfD-e_M1!*sfHbS^G==0C- zcK~|Oq&9O~ex3G!zX;jekM|VGf5V%cB($9*eEh{7%;&gc_<7ahJlyb$2^1l<-h^@Y Ns4MFzRll%#_dhKvR388U diff --git a/doc/v1_api_tutorials/quick_start/src/NetLR_cn.jpg b/doc/v1_api_tutorials/quick_start/src/NetLR_cn.jpg deleted file mode 100755 index ee65d1f4127a347e991b4de5e5208347b2b0f51f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30320 zcmeEu2Ut_vwr&&=1O)-h`MOEx0X1O%j)C`b`0(xpr9O?nNzBhsZ4 zLazxmgpj=DKKq`d`<(mEz3;sH-uvEnBkOZ5GUr^Q%sIyx{}>becl-kAsNFpHx8dP2d$R{8;4-iXu;R4Xw8~7e{fr60ohLFrfDh*Sj zn-16Ryp2dBW_wuHLajN3WWW2&(f1Mw4J{o#0|zG;_bnb_5m7O5iF>k-**Up+`CrN_ zDyyn%YU^6t+B-VCx_f$uM@GlSC%#Wk!50xr%PXsE>l>)O{e#1!WAq8;2Vdtv1b<=c zublmfFA9LK^MDR65dGlm+<8~vBA~cHcthwSrHlrVsRPx`J8y}vJ&Z^zYq`XBR})G7 z%yEc>hFuuWf%?JP@0|U|81wyK;_R=C{f)0F&}D*ifbs|^Kw!|B_9&JUcGc%L`tXoh z#prySw%&CW?}vJt!h3O{pApqr`go8(TNd=2O8Y!oJP~WX8%TsT9(>Pw#0WW~rnRMA3MbSIDdDVx^&_Bg#dXL5B}xoO#)$_pfPa15%FWu&cV#UNGzl@v1pHFQpE}X@+2~uPovpPNt;LTR#v_@zeDj#H~TBg zqp|ye07oIevha5trH9*U)p!!Sh|i-n@9CahXi&w2{NBSlYw@7>EQI0B4Aj{<+17Jx z!8=m&!?y?X`}+doWx5AEVjRG%;H8ItxK@i5muLJ~7F|b0cy4Zb@QL5r z!(i^u(NrLHv2Q^7oK1e;Wh;`GJoPeNj6uw$pl#!DtWcak zY>giwh-CqIgng;TzH8#bgTC+QLf3?JbFEEod>LV%baJ-2kRm&?6{#)17sWWA3ODYH zT@$;ExY=}{v&RvG=AI}~=C9_h#&LR^I3$|onsLa7>T_o?vg1K@;R1M2z#U}u8DWDt zz-JGvGaPLuNDBRC*a<;PCSq-JAX9?PcMRTlg-O!=CUlYx*`Bub2RA08JMSM9g+dnP z1Jm8)!B=NCrd}H`6;LVoSU}1ZIn5P=99qb}zJ29BBV_!sXHSU}p++{noxbyxer2+q zK{ILWXkK@ciRf1bp>6bThB)83J9tpb(mcW#7SzP=X4XpUXt>dI^V{p`peK?M#v4Kp zQF-*2wLe`I`l@bc1Hn-mQf0M4j|w1VMm)m0ERH9dMAvWVekEJ%3t@4+>|Dh5acR$% z=XTxvGZxnb_QV1yugMiKveSeH5Bgjk0zzN!=@$!;I?z$mc)>Zgc zwlyk^wZ1BZ#XCO7!T{pAnJ?`+(nLRnWjxE1WX!QLh^^<(F3+*1H0tp5IVN5aCKR_5 z=K??BU2~>8-(bH8#-V>fspE@?d3(zHO^ zH7cjT6Xs*HN3z(Ul*tK0oZ2X@)GA&~8Qu=oQBUmebSoaO1@@SwJQ+Obw@y50D7{hg z_0tTx#-z9v_wP!My_$x(ux_G9&|pX?vgY24aX8b z#kO1v&`9C#jrz-`krL;b3pMb3>=@di{&Z0kMj{>#gce{e;RT8 zAfb?Az9=tJ$l)eE&7C%fC$H3`c)kns(a>*S02e)@*|XsP^`+szbAh!6qpuwn2!0U$ z{=#iuqdXMBp(oXvM>KM=a6O^1xG-$IyzNzxOIfHN$Xh&5&xp-MBYZ&&LK!m+eed_{ zfL|n?7gW^A-87H(h7nKn7anvQEC*fqC}s1FHwj&U2LY>43lA!`!GqHH%|lwAgkGxY z;IcSjfIEm}8BjR7*!sLUOXIhq;9~#WgXK{U?!rl?;Xw}27!{LfpKA_MUo94A_6*%P zm(yKCk2~+=5D|_tSo|&y&kcZygH&KPF3AJCS8)U$wAN9a=RpFGVQ_Y!)MoSKz$9xF ziJKz2w+qh53~|&t$#@W$Ask02#fr?Y*gZl2Q08V{dL%< zZf0z_8BRxBdFDU^yDNbQ{d*(!ZNHdS7Ak5-Hjms^9Gf}6bVR?Y6H@QaMo^`8tB{7i z-tjtS{N}`p#ob?J{<9KChqMnbgZFND+rjqBV_8Wmw(y{mAAGIPlRURT`r5m*-xAIx zEi-$w%9OCa8HZNKgKmbOH4Fd>N$C2nl@%0N9$K7r(mpwsgxp(o9S28;rFoD9c%85j zF+r8%fm`UI?E2XiSH(qRGOA`DqDmC&yRBWj9T5V_)A*yA(hGs6Bh|{^|5y{=zt8Pt zIuiFW9gx9iA^m9Q4A)c+lP0UzU=>PxE}kvN$}~ z6c37^`wMITZo~Qd(V{H_j#xd!U058;%5-C?+R$!GfqtGI-`$Z2@=a|#C_btBr;&eI zL0FHNhU!Se!d#1>i*9&O5WzRZ$XsCojg)xWZ>58l*D+(x4e@<=&`&cV9MI28KZPOsK+o0*Sv}c$xrBN zya^7mp#BoYOU*fBh|@kHyryB?bz4L38tA9%+`el7Fu{wx0I`-%0n&^x=65n-#DgMh zQLMN&U|FThy{%h_-x!)VYE@nE4DM;(}XjzmbUO1o+z(z3Kit%J&h6G4W)%qK`qrUc4QBF=2)6A zqszd40^6O9o(>MqoqbK>S0UvUOOMBUb#fY|YcqVZ-#CSFR_hoa{6_5K}HIi`V{8dUagj^7KRG1lG#wRB>Y> zLH(^=Sb>d=K#JXA;;C7-OlH@rchK}%%A%5v=jl3))#V5_{wzOUg_W$@)abK#tdfH* zDozQlo#wWw)mVy#Sb5>tPd&aXIUsC4ZeJD6d}toYy&7rwVKV(=qGy@CwQ=G$af-h! z^2MT+q&w6d-bkvPYFG=a2~$yZ#DjEUV+68G>@As!a&Ka8J{@ zMNFUWPWcXB;gvxV;%YNGUti%FK4KiW|2UE{yP|Ha^1Z<27mS~5;VO#Hgmz6G_ok_c zYsjMc?L{^rQ}&$(mBkr0b_vz~4ZJx9))1uchHGs3V-Z!?{!ncJ&0l* zt3i5)bwqDUcA!>Z3Y*mL2HVO0i0`eKsse;cz?b?kIvE-B%;|dh>x9d*M;UgVuF35$ zOKKY?T1{wC9rIh$$g1H@S#%Eh`?T?fyVBLJPK%ddqq6x^J#H4^+npK|{_Jz%w8XEP zNwT-@1a5bs(xmBpzQf9As5U)V-bBIP+sUouQ1_3EuAik+H*J}*~%LzDF{bfWx)mysq9(}R1hW%xer>|hi z&PS1{*Y_v-XT^i{BFFmr@3DKE^4C_?L|nS8Q5AWPzKgjB_d14ut3YJwE6jPO6hoz< zvm#@p-Zgk~^)s-5ljCsV2HOSi(I>zre2w*p>I>j)Y#MySu|KxKUd99mbC)DYEu;r? z6Lf>k?N2Uw0h@K%>JvQZ+q3-n3jeVJxB2Fw4G)%hj^Z?L*X+s7Nb5)$>Gyn7=9?;l zkvAuAx(kV)ED>;t6Ur2Te|4vI0^0}Q)vbnOZL&9Bp~L#r{o42hD^#?y<71o&u9a3) zEA(HqY{=8s5$}Iw|I9dS0LJQw zRjtonY~5yVW9>w?ErmJ!!Ej|_)hO0I*Q!oB%PeCJsrTI~!a8m$VVy{sachU(JoI_x zM`4GnjJ^F^r|LZ$uqyI*hp@oZ3tp_Ynd=jnrkI5f9QCj|u_Ajr)o2p_3quM{j6z~8 zPyI9cfbci&W}Ph?91m(wCg>V*3x-}qQ*{`iE`PL+o4Z?EDLNEGqQm=o*sJEtIpfJl z&9~aiudZFOpHf+PQGhyLXdI>Lk;-|`s1i%}dQ;V7ga4+=`}2U;K*B~#Qgh>RneEB@ z$s}78_fljS5rtZ0oMde9u(5k}q*Zmgm6K&r|DaBl6FuV9A+PuzXYolohI^YPK}9FA zssyeO;I^;R`0W|G#Z(|1!OSz+aF@Z_CCVl=&sg2+$grO;j^EsSxx4rI{gh?jK#e2; z-tSGK`m-CbFRJf6#u|g}Y}B3q;AS*PQGKuRW2n)W(Cy`qtjnN}uR!AY5Gt@~DDM+w zb?A#BK|4lW6j>;LXl3-2OB8dY;`JC)ZC>*xc786tLdK-k>kR8ySER1~D9qv%dU+kw zTsJOCrMG+KkyRwiUYV0l9e$RC_CSSs*nTPt}>pNjuVd z;A_)QB4;*wUU1++;h!I;Xr07U*doNIjf?$>kvUt2DDQ&?t0GfZ1ZS0KMvv%LoNF41 zZRc>BE?wtsA(II5HZWz2IY*%@9z-5F?wR3(EP2O4(euiDKcvHF*6&8#>YV&G9wh2C z@B)1ou9F5A^lM`O=G};gZm? zKdW-d(t@wK;>#Pxuu!aYYaRH!7wb5VWo=&}+XiDKmJq$`ov$JgHo~`Gs(t=Cr@8<) zLUHkt#EqLqWiO++T2uWotA02>xA}I1uQ@pzvh8K-L(^xmvqk+6=WbE>-Qc^Zf(if! zDo~iGnmM^g# zg{jj`#bsH6k%(pqjKg73@F0pM);XDMW5DP}t$+Gud6y+K@uC9G^-AWg{ygCvTK?j& z`x>K~!ZDg++Eu}mE?u^ypW5F(d^6b7B_(J4MK;IHuzgNxZ;pOP@j0X74O*hGRDY9j z_*fE6=*q?V8t0IPD2bC6P<< zt(f8>6?trwT0fPL?*lIJ)!{64g6}=q_ht;`_>Ql#JI1ny@~ly_cX6o+GFqCxlyO5)%Vtpar?YGbyBT%=kOZ=qRs6m8&L9{;xXe3 zx7cP$<1f`-m-wo#h&d>-zq01aKs}T82x?KosYI$?+lF8w=XWIcsx%E~8w<7Ry=Q~2 z8Yq~^;n7joN;mPX9Snx;sULzdD=-Y@g#0~{RcRi;HwBLZGa&(RGF40h#L!d+&?xa& zsk_(CbXP(*Sd`sQc8gtDa$GJRc}@|aGgRq@X#F*oNTw9>(Rq?l0XV zcOxP1o-|O3AC-`Zdz7^sA9@~4x~G)4YrWINQkyF?ltB(~v^%X`<|;5C^)#Jc@)@g+94QkKd! zkwLc)%^K(rPikWg^CARn5!Kvp%t-p%lU5~WLtHx4L__kmmctJ-AbD>0hH|tLAme$s z0M0CThx`qi+un70*%hPjE?=o5pmJB!e!Fy=M~2nJM!oZ6p(ut=mr~U+dyB6yS4CMe z?~d|k%qv2JiG#jb6{8Vmni>768TXxzppVQULLtAcx|Y!8?FeT#psr-KZC};v?1#;{ z)7cJwyNn0Tn~V)IP*orrr_M-^${Nfoqo!9JFz<8*3ljCr>ioP6{YPWTzFwoKq3f3k z=kklp$XoZDbdxKBv>j!4Q@qq#>X+v`-M;=CRn+>r_Jm4ZEy~Gwfa#KDY{_|w8!x=S z=0sGOF?y@nIA0tkpldEFtzfda1?x+rOUTyg)ge?HWr*!~*&^=x=p1UKA3sdwtq;7G?S zJ*CVvXmyThO6WqeoA0LrF^cjhx3~@?^vQK8ddJNvXXDwtjh-GgA;Mnv( z$QZ(dE(v~sttFx}a5y=fu%mlPBlMG#AQ-@MB1SZVSD-+G;%}pcqm{gj@t{ZZ9jwQD zf~bt&iD`>>KO4k`CH&Vlu1O6~-Mb-!2bta%JY%U$`;};s4~U}sosg@-1-7DKkXt}^ zL&f(euf7wgtCf|;P*=SBmvs~xt*D0tP{$$D5H(2JD4i&>-*4daqmL^Ct!=&MpS z00}uQR-e(>V=G<)5L5n%Q3=I|R?}D(H~p%_v3okMQC0<`>h^5w`wVoFCY*Xf&(f}q zzP2tJOOH|;otiJVC|wNE4F`r!LKopda-Z-ZIGY6?)Bz~-F9QSn{JT(K0#Ta;9u;%p z-fVauz)q|>&f)-%;&U(58G?K2Nk~`13IH<_C_~%_Az&1L0)oO28yLd%aJ6kXUC&M$Xs;T|p7~!1kQ{4c9%>>t8$nTTvAyB$fg2od>SV(QrJpB$a9IxV>CdeG zupw`w8Jatkp`rSa#`YG6j}^Eg{mAC?FRqie(R`2>vvcld#D=Ir{K$%;q#rKq%Z~AM zxF#UJ@nY=r7YJHHPvPw~>{b|)aC+wGY;i(Uqc^Z_2r|=aW#KSN3!Hou9^~7N2LZVZ zd8`s1B&z~l_yR`dVF2^JI@*MW?wNa=q+-v)XTi7!8%QbcVl5m6j1u~7cpl9cj$u+<05)K%GCD!z~{*Ceq+hg1W0LyslM1UspDiQ*jE6vH22;=w&^oJ$YL3 zoo&zb(N`C4`W)R1>gN&d#}f7*029h)ga=J10;ccTgk~e5xm#Ns6dALoIe9WC6A>La zT~DaT63>!a^gQ(nItsieoQkDUz=?E--E&?RAz~k%niZVQ$WF5Q8uDx1=ekC|Qv z;~s7Nlc_zi=>()`G*1CT+Y@o<0o^|Wzs7@32ny0`)qh!U8NVzl%*?+nKGXjjoj{Ni zUPtJ6Cct^$*w~C8l4Nwx*&MbOn<|-JwsCq19*SkU(o)alOjPYtIb0&6PDIb$GhndY zl0|u(}@7xY<8)>$RZ@=jam{Yu$-azopKaeOCL)T~fbu{jK zv#H~FYUZgo!VBGkcN8Qpu|4vWAjwxGz~Dq=&o?r-aqks-f4#KGnYrabgyrA9kMnjW zKRrtQvDH@tIheyy@a_~8rM-$~@eLuD(LH4Fd4BdHCt~fFub)C&=YUMi1Q2(?tNZgcSdX{C==@*17Yrh$2SffA0K`7;gNP;a|k{H->+Z{|Cc=?tuGAzQ1<($Ljs1|3BdN zzf!xuygTF9x$z&N{g>JCFVp_BzeVnt|GFLF z&$R!{`G17=U;6*ca{pa&f7y@z1H0KT@BUwy@4xi_mudf5a>=wEjiU1BJ~xK2Y%?GZ z%uGCieYN@MuY2*cVdq3!yYH$tJSy~v>w7uh!a9TZA)t-jOt%1`u`a zCl(WBeM?@g_sDkII5u!{0c}HETCs_+QBqD{(Rg>Qh-OMnO*5=`LJ6X3IJypoa}Pt; z__Lr320!43Xe-tOnqe4{AKY0(&gjz*A(SNWgrj!>Sp8H|Hdi#6sYSHoWuJ}1>%kjV z7q2SVo!xO%V0Hiu7YWfq_0cGUrx#Bc{nF^_2$vX5=y?OWH_}e^-PCDL>?EP|j@wRj zpk%n$i9?D7y?AfTwei&URAXIi{g!iAz@16n{4pq!*b4|1;edz!Gb!O{W#MoPsTBlG zlZw#0bxMNejP`<&zkcZHv;89yUuk0T8DPDDyXhs3Tn$=8duFVUH{GfVUg;(MM(IL% zBSMRm?fmk^SDC0(Sp#YQf#@+Y`OrC7drEH)0B1IBdoLP^#X8oWT36{nE{M6>;7r`X zH3wket*IE2_21vBfaPFNI3uhm!IG+@N<`2(r;=D2v_j^fdoETW*zLwhiSrGPqUcEZ zJ_}NHcDZUeyX9E$d(i^h>!R{J^MT+OVBZwZiQ)=0j%mGWc=}E!%L*Hl>z%E6_9v;g zPl~QKf6Uf4ka&GdTS`cUBpt;>XN`nC{=8XJ-}o=uS;{HtJT>Za$AxVbzkT0U+BDaP=KM` zT15(Wu-r?O8ELqbHrH^^h@rXRO<`=LEZ=@}e({&cewMk%Z*GgPN}Q7ob-E>-p&V8R zrSyjG&}+htns|1K?6K#W!8=^>>sQ?ZAJ@j4g=>vuMtmK6R4e~N`i(`@wytuF_;ufV zexth7-ls&^>!Y?Q^pXS_Jc?QBvF7hP6<+9GU`;F5THpgBP!gP=TRq&rGsm;Y62n|n zRr1+rPHAPP#X^E-{Og%vAGCfu#Si46M|P{Dk6!&sSO#5wc~(a{++QA(2q zn;8ZffsAc>ko>3Zzile@CagqT5VE?gi_v7UPuUvZw{l4dIRe1FrUi5My;{VHtl279Y{=z9yX9AL_27MxE2=-;yC}frx#BFL zdl47#pf6@zxD=YkRq6!{D>hRGy!8q8;~A0Vqa}N`ON$3VaxkoU&tS7i*x?3i-U4f7 zoL?m?otF;UiY1XbxiB<$fVio4B&|`5mUZ%Yxb2b>G3BH5OJ41Kshe>&8KLXp=EzCJ zS*CMW6%tpmYxQ~U*mt1F>pq>s(X7If)AE;m=FE#9s3?goKsQ()+DOxsP8>3G?(EH4 zq&}<@t5VYsj1US?bPc(BX>a+XQ^Y=lo+N+H6gp*9PepIyX+veeSDQDA76F_w2(_Bl zis2N{>1vU?h$ETTqk>v{4#7y@@Ofy%Movm@=n8rlO_`W9 zcgMdstGdmX80L6MOy~_rqGvt07s;9P*-cAfNKzzyeQG&5Cb@1>UzXE9H}gryH7HNT zaEE_uv6$is7XkZ6?t(aZecukA2u+yHp6;5*G_D8q=wK64Z8GzMdo|WxWHsVr`Qy}jQr5V} zp@z@vBVN?VuyEEe{=@J~YqUrA82H9%OO2|M5UMq57iW|>`pG^E+Y0ZDX*3u|6!{ z6bzWwhsdfIKBKv$8O_mSGxv3FPsz$>s4ix=zHG&W2e~yzj-FTyzbccBHhZcHRO!l) zMHx`5rLfwSklhXpi_<*VhWFRKceq+H{2DNP@U>5 zZeyFd4Sp!?H^GV4g-^fDdjHy!9$Z#JmS$-nH9+0M9{)v6zmIY{k-bz$;yOd(Bfvhm z*|%egYt6E!r>&=^Ql8zXmHPpW85;?( zZTMovaAo+Ggl;Q?ySsFIoXNBJ8QZC)bBDN?&1V+eh}&8rMhW>9hFGdyb99Oi4Mi+1 zExnT5J$2pcF0u)W3JoJmBqSspJD350yQfebPaN(lj2`+814FA%1AwhIMt!0Hs8l0uTlev1D58Wd&lXIB=bJJg`e(LQ3J0nov1;Xz>f{kn7wVSNY>bM76 zmc{q~)DYTAix9-H0w&^j?!!xfyU$i<0Frm|oA<-OvkS62c|hWmIh4BV&rSc1_d`Go z2UoyNfV;x?pB9vKBLw$0=NP~Vmfs$dupUs5*8$XDcM;ZQ#)E#9DHu))%ms2lE$_}? z0G_Y#lZK?zWO2SXj-LW}Bw1%L0`}*o*c(2;LP$G=B2%k>fE1WRDCT17ygC5K6OD31 zzde%PNX5SWK4j$j=azq`t^d~Se{2c-|D@Rgtb2~zU`rF>_+b#%p1HkFE!DvzLg}-i ze@sSb62bm!4jcKlR+Bq*rM_B-N-aL?C35oVh35ok8?6$R3fe=MhLcEj4^PnvEOsZp zOr*>^B#HdP<^LDm zZws6#7u_M+^adW(+d9#(e&2!I%7chnVtc zZFza#b{M?yM}@#6!$tD5V_U%J58**qEEb`doAXzpyY==sfs9j=K)?rHs7uAZhfL~) z?*RAR&^-kS)QYIzQP$9*=$GNGgJkkHOz75e98Jp0;}^FIB@v*u7nX=vf^$4A*)xgm zd;)G-i>dF~(6MzEA&16LmG`>`A6{1ld(yLw>%s^EXSA?ADo~6Cbvd=74)LXcJ^78LDEWePX9!C!53eH zVmW7TnmRjAJ0`THW*)H^ony|Rio)PP#_G`2cD+s2^x&!7dV+Th_Supg+*3Pcw>|D& z!A2Ynrem>#ICx=N#hxjhoo0`x3bDc`RH_*DtT&mfg?HO!wF@(m3de)U?tR}v9?sR~ zSZf7M@je)qA@={QnCWa7TEu?;f%n?OLI=No>v_gq=glaQ<7QY;T%?_gtjLlhadlc~ zD}exo{w2-)^B?<(TZ9K#5q25y(|Y$J=}TUZL$@UnlvSB5B6p=vl;l4Z(7cS)ql#2> zR4Rg7F3#O#8V5q4K+WB3SiMcb82g0NEz^7zl3bOTdP9Tgt;(j@WDGx}O`q&qVYu}j zse*rRCWQZOCN$1(doTljjPC|?k1OgdDhRiWy9_+{uuLi5Y@iuTo!;!^nrOke;=-Y* z0x@u9+y}2nejJJ0ijw+_ez9#@NXm4o!<3X7*{#&`I0iT0whk6HR9C1$kC)-%$Ypyo zbFO&%jH!pjVUv*L?H+;VkBS#8BAr_#%=Vdh+FDh$QY91LrAKN+&M6sJ4J)}%mB$P;AqGH@g_uqsOnVT6rj4GlzJBzUhigh00lJ%K6;mg)n@2?0=TGC>SDrW7Xj*#0aWaJ#N4B?0)fO<)8R^H@dv zFRW0nv;6kIml=ZX=F>-B#S2cY1gj5zdb%&D=mXyMSZdgJWAl*Yo z;;>K;MvX~Kgmvm31(2FA9>fmFYW@6R9D&vd(vYKm@Vg3?xj&x=4!Qog2ndEl$bWkN zx8MKq{{ZM3p!dJO68@)14?!Gy3cA?q)`e@rWxnd$ z#JusP|0a~&+CX(D*#)kiY~jRr!B8a$(d&9PLn{8A zA2?n;iPLyg&tAi)Mxw+Crz6;rXXWZgtfqa^bZ;-k?C&Y4^P5A};&8V2r`Igr6r4pc zsU;=tLZsO2O{#FlUSUl?tk#-(%FhL9|K2^T9TduX+IBKQekx6lE;ryBMgSUmQKTM( zV;_~IPg5wId{ix3r`kHR;j&33Ijj$QG(a1xbw=L}7I*crxHRcnpS{x% zsfJ6j2BvsG-$vi9f*_q`E&q8Np5VcOs>s7VDZ5}IwNFO}=}iIS((bp@gSqK6WBk=+f(RLmA0)ITaB`peTKt3O?e zKB}NIg<~AY;Rb~d`tGI;Rfal_8*6EfXTDQ=ZF^6Y_<5Mio4eVL1 zWR@5!osVck%}Kw>%`ons9c{0u3eV?@1(s$wQ4ab8wCVM~*!77ey@q>T2$YDBV{|Q# zxSW%cyil+KdxEarcH^{nq{`t-5#2W}@O8LmxQKYKdMtmhX47`|R!pQJdsV$H8aQmN zd#1e`20?3e0Hs6R0FJ_g2jw@S`T(@k4R*rPSm!sKaY{~z1p@H%aVIfnl_5AFR)5ol z4NpFtpR89#?grVuE=c_VtDu^no9=PFELztd_j0InD5GqzaGuWUiPMBD{pz&_F!HV< z>h*Z)B$NtfW!Ptvm-V*zYZ@QT(Kdzivt9mlulk{A>X}PyXtj;0v!e4G4BLzDLx$dg zZD7kteeK{3(UP1BMWJh1nIEUSbA4~or;_BZE=zzfFGuO2PvM)%^0tC1EDUvH6`r1k zi9wV$J(8IZt2S8@E$KphhvNA1@0jgE#Pl@+#5Q{G84Cs*WDk4H^~D&UoUj}+d29CrA~27{T#lJq_jRQpQ;9DcV!qgLOi>Q)?tb;~8xnY8=HyzG5LG z!inTz9x+)PsU$WAMKhSn${AAX%Ju~B>?pBV{$k?icl*5g%59g+&dq!>nt&jy6%y6a zfyf@NI2~UJFF_}#oKL$&RZ^J_2A`{I!x%|_TktOCdQ3@?-rdag`kR~kGY++%mQO;2n4&E7e)*|_vD ztLdHAlU;EWNgLcgK?t5w9pFJ{+#VAIO6DhF)NrH7JZU9?7b73ur3^G`DG%ruAK6Nx zVp8r4wzpl2PP#Ej+Z!FF6+Dz#VO5{wy!<-h@CIO2sWIy(dOw|&1hK@l`Fu;I;X;cc zI}uOI%8eED>S_X2hqviIR*N|`b;p#-G%pA3(9+WnxL&stClFbJR02TKn%A|X0roE=fhT2CO$7|!qW6XrPM#1gDt%Gw+9l8rFTOXv|;FHC76PFCKxj310EEj zTP@xrLOs^1-@Mn|8_}6H`DuZhSxMR#CWUKnH8_+=+?ZLK-nTn`c?#PuG(VfTR>wii z-VmwrMd1UqjT$;;f?ft{CnO=*XW@SuxYSJ@Qh}-td|BzuBkP`HeY>cmVv~_OR+L`n z+#7MWh=>uEifvbuGUPyf&1eb#>r?aJ0u1AI*e|Pr`mWybzWe0MOVZ#31L=n0>J{*I zlKtIq(SjU4`xBP?z+Vsmf+B5CL>%rK3W^H;^Arlq#lA$E#4Neb2_W+s3xc+GPg-pTmJaTh8YeY`u8093UEBJ*>i z%@3*93rcSF;6dAMDfVF>Dw0!w@c1=0(yjdHCAtwlNM?iMXhP-qHCWvJzMc@4UT~I? zZIVx`mlXk>;CpkGZjU^h*xt;Y3~uVl4tMHV4v@y=AeOTDeh#21A=2j_c^dR_*>T6y$N`s|4z35~JuD;q0Z8hwF7 zhBeZE^^AUaN4&j0g6QfF7JhU|237=Vl6KKQcSDrCU}I|ZBWd99Rt&b1Lf-98Ju2$h zk+|g~Zmg_D%HKWEPuCTDjp`}u;dP+OE{EiVMYvr%faF@9QrFbe5JR=WIGte|?bk)l zMTupHMr`uYWzIZfzH#O6GNW{jrsPCZ!e%DQhu4j1sE+s|KGOT#_Of*+?xVvhT(eYn zr0o{Ryomc{-$GDt=R2lH@9_eASSb<;2nBfy2ofitwg|Bm2>LR#`(&^aTHX|~a1}NM zrUhF*?j7o4ovgI7i9}+bR$l=5}kIcGHXqPlT{~~ zGg4SmxO*ovZ18zL)sj=vDzO_^@DA8gop{~`52C~|F*5t?D%97E3w*AsQ8BbqOM#XP zOeT}I>+O6O1;~HqSSg@-7M6S{G5irOMxVLiUY_Y20F4!w$zWerU+g zV*5gcnM;S&KGI1r!VC({F*4}9sRJtX3@syc^xsh-*cY+)VnS|zeT%AZQ+2XN7PV{k zTxouSoRR+qQOx4h)Rg)1$UY`aEMy@`mrFz}{Ph@zojHq5?~!8_cH)ay-}ouB(P1p% z(->QKqgr=Uo7{Oj?Yd>3)OYu~TCNbtqhgAE=jp(fw+?wrzm`pJ3f@s*_)-|z@WE<1 z{f6^pl~-!g$JA!6co4{oAL0GTz!FwL+rj^0spY7QIxx)E?)l!GjDe|-QRRFYE<%xn z7H$X?6#09YwY$K<+6!E_I=OM8jb{<`No7r4!2Lc*#URsr&b5!_+1g{pM&hqM*DtNCxde zVO6@$K(C=^7nKk1orYa$*C~oEu@~dIH0nQBl&hu@IEz*gm2;mz zDp|Z&$C%3gH=rVPBRu|RN1rTa6$96j4;vD<*hb;3F)6D`y`Xo%$KAmi)sE`Duow|h ztk~3?J$k_Q%I`aC^HX3$=ztvRHc3F?>a=(eU5PE|>Mfi7<%orCm&AQXK%94Xz*S|* z?2&sRx@&H~dc@s_;fsTl0bMQSRuslwhUgPv%r?eqJi-tiEWx6ruitZBru8OUy%;i| zs;3z&%OdBdyDbLyw5RZrNA6kko0Ek*Tk&TdOw8R55&d=Syb+1!Thh0X?fI=iu6k+by4fRk11j>oOue(M zQn-dn1}EET#800(tq(mYvwIu==`3BV^25F_h}1-fGJqzNP+KgNyc4-3&5o6e!%+@iy)c3`r;nkL0yDf} z&`J6peHQ(Z__v(iTdkL!BE2InjuCv*Er?sLF*vAQRn2i}y7k(_JLlUzDn5lLR88@z zsy^7?S>-erCHSjX^#uB?(C>#QaA~`~eJBF`7Bn~_vOTKndc;C{0>Kdq;uM#0Q=L#C zPu2{W_(ADQ<6c@wLxgi++b;EHPDdT&{kDJrR;+MljC-S0O)0^a=qZ2?gY}@x~aE`!O=04C#kI7Hn--XPW{+q&(yKY z@M@ofH@BxU-KP$DZL59d9`>ai@wvW7Q^AdeB}E_paU6S41ArZ@{QtC`W9^T6j%PsS zy_>;tFw&bGr+Gc~=rIHjQYSS*C1A4yPv-=2F!cklMhCPW4&59DKgs1BbGbn&^7-K$Gq$7X7!F2YkPMHPzN?K`(Uj7huZ81A%W50Xr69P5;m8qKc*XM82x zZ$wNY`UiajlxTbF;JZmx5*5Xraj(l)FOmfoUc>3--m=w#k047d!d!0zCN~KmGW!#S#P_b&U zy3*o)=|)^6BinCuM%<((SIkgXu}?-{Q%=1*9v}C)j_mKDGO{|RU^9}8GP|m8_@(ZG zwuGzE>N26D2Ocz4{)pDY3t1JOF=Hdm8K%l8>w9ZEn;Hn|xi zCI+NwqGq0a_AM>-I13wN7|bPd-nOW?efZmC}t}r8S^sHM|m}$HPC~s+$L{E zB&Ko0EmTJ(F~{t#TD{&Q&4-TE+C8eDE;Uvyl!>H%_F_Z^zO5`0JyjhRo)tY-_!6aF z9ecSGCwn3ngM_+4ooM?{KiP`2IqmGTzVT3oxSgtuRt!)nH3p0~w>jr-SBJhHo{@aj z;9;b8%#op;KqXF0E4ZH@Sc&TE{D?Nwn3(hCZJE0{F3fyKO3lFm?z>25VtQFf#c_hj zCN(MdyJJ~x^Z6>h4e~38wOY!9P;8<~!Q8QnN}6x$IFOCk*7CqL?NoX2TDgp`R(!Xq zx|cjV&{(-#)X|YWUm?c?9Dswz%za*Vl%VgFGb)inTad^to*Al_)+7C^k5uSHq(r;g z>~m%dL;ER(PG!mYKi<9dy`|;ESh=)B=EAvA-O7Oc4cYi29~~_HsktL1j~)|#2GAayTOY3d z3=Du#EVX5~qsu;tcdGG^859_MM(re=4vqOInfZ70YG^7)4b2VC2fj$18Ty6?*?K($ zlHi8T5VZarP?(S%cWSdpdqy&M9*Cg79s@_&M85w83-yUPWSBpo0M`LIeO_sz(}&?_ z1b}nQ1w`Fn!GG=~4E^)C^_TxB%NK#I-Gn-WaXPG1*Pv@GtUwv!$=z_=6xtsTdRYAi zhAan=_hnVqX~3b3iqQ1pol|CTsy%Hs{c~!l(_9*~e`_v4!9Y04e$= z3EqR^9`6s*&%4!~w#Ka&p`GA|*)hvBb;($7?-$qQPO4A(vf1RHzHz;1bd^lb%{9c! z43SFNvhc|{(NG0;Ne;1n<)FxRH#2r)J)~IlqNkymq#GC22{+kAmX~oQ1tl+k8!ZYE zyDj5HaJ5q+Ls$N~FBx&yh4WIxM7h+B{LN1BleV>;}Qr}=Ol-Hu1<&K zqawPEB392i1LYx;jiwK}-de^!U4Bjpq|7QLrKpCijD~rE#109t`=NDD5rc=}n*35~ zUdjEly!03!0=I-^a~Z1~t7F>g*bd<6t!Xfq(@UeP8tm73OP4y0t+BZ-oS&dl6_2IZ9Owo zgj^mhR1rvo*Q9$|gf7a3b1E3wx`}$vyE3&dN|9*8$oRUl`jOeE3_0S@EVUpqo4%Ii@(dHzsc{!8j(W8o<3|4F z+S%YP`6B8o_OGXG8>7bRIf}4Hl5x#iZq=Plf~czh(cX1NHMMPP6crQ^P*6Hbl`cw= zA`uHn7m!{>R6-0$4LvH-OQbg;BGQ}EYmg=eL`dkNgeKL{OCTYl z@Pje--eYC0xxP8q{JuF?rc6$dq)Zx}d7=(Bs}U7*r~DR4Nz91Z{GnK(oh{bUlt)qX zs38k%JE0${BsEDAH!v^lVT&tj$VV$3+qqlqdkE?nJM@*qb(-}dWrJ|{4#A(zS0;Ee z8=2Rdl8{+w;js6zHom`WA-pR_&CtSD`qm*mCe?F7Z1Vgm0aL+E4Wd&~&war1v08N~ zSyF{4>y&06qfYFb(+Zb9dNws3Ir-YfRdewwBDX_${9Q>VY+!i=+%L2zdcp6}%MW_7 z)^$c^qF}r911h;HtG#rARmCRu%Ftqa?|W7p?83iLua7tarn=qi_~=17 zW8v+6;|bN!+{F|q8OBaX$jZ4GOsK?FG+SETV6BRFzF^yXJAq#`_WkP&jk;;4z)Gu2 ztCXt8^y8TsmXMAD<&2!J(s`(5{-LKG_YPmbk)`}-*G2v6q|0|59y$@}@%(|Z$^;A; zBKH>NQ@6`*+ zQsB$$!wPkj?EN(GiMCkC{z5dl=28joyNEE|wIffw*@1+}pQr;OiA=xo9Ad=HxVl@ODr;QnuhbZ{ zAY^m>4#q8ZuFP%Sl%Js|qZqPnFbEK!k|PZIHEY>v&}<0J$N3M1`~gab_dPV`n=};d zzG-BzvfUE0-$6q!q)f4;U*v|^eiCNC)O~8QqVfZqMLVS%Kq3X25_Gx0c8aT7`&erwP5j3x~6Di&= z`+DIbo$$lX2W;|*2E=QVK8Ta491Wl^rBw|GvjkHKfh&I5g2v`SuN#vro>04jL}X*j zzLij|GK3Pw+naa5IaZxckPNkDz?Bv>)_l-LDLb&|wXBNY(xLl)@9myvg&T9YG3mJipP$;XQfc_$^RtIy-q z7t;y&Qr&^}ECY!GHKvoGa_2G8icE8X#{)~)9mO$@CKSh@dv3jPPWxjo**6awqt?eC zh@BJvb|h+oO5+Oc`+VYOgC>qB(=1=+RZ3_PI3+MLV!Jz2pc!|QYjkRW;aa#x=P@8^ zaTaUw7=6}C9dTj;{%E_u&{tqm6!u}Q@WVo_w|!NCmseI+Hj-Z|AGB=i%BO(lZfX=` zLZiEtc-p(e#AJ9v7NFpXt(6$_eq@v z=BOVQefi=>c*(IviU1KmMY~CG6c^kuZ7a&H#hThxH6%;S@N0?32-cdI=C~W))psel z&=kK~fLzxLbdZjXo9R{GsvsAShi7LG`6l8F(?t5&BY0DwZnah(<|tGx(sG{r{D(Ew zNNDatqDO6D;Eb)kL{XOM{WJzuHG?3UKEvhy>BiFR4{~8p8J_}iTgpnaw@o>=ToG*h z;^Lb~F~YlOXHNC%cTo>!((1TqKVRKu`Z(M0{gi<1Mb*-!JuqCYx3+Ix$`f{@JWmrmvfIHaw~)~G81ju6=x4JXBh10|&{Evwk`fg| z*K5X)6T2R1Q*bWN)Mx8 z+4QfTU5o7}$1O00p&JMZTf=9352DiwRf>(LW7}M(C21D+ps$dh60xIg@5b&3M2&Zh zpvTp>?n)@smy{O(FDB^tDt|Nt7)-zVQJ2=a`2goL=2r|tCn zR9e9ZCB5Er-|>?mX8RP!G{HnauHyK)T^`$hrzd0e>+JB$cOI&xetCFu*2uOi;mLVPPPGf4jZ4DNxI>4fB? z-GS0w9$$Oa+R%N`zTv|X6?B?1quFw2cIl-2q7G9?-)ue(M*2#wk)k+bJd(b=R}sMu{ed;oO`?u%H&(G?v?DdlfiY^`f(R z-%E;WExtF;5F=FCjNPqp?gPe9$4v+wp{UM%T`g%$ z!yA?0mylNM6VII7ZlH-Q^2td8a<2@)srvaU&pw_BpzkUmy~b}!F7IamanY(WHBdA% z1=^P_^0uEB?VrFS0EW*RzQb~*YicEc=77oyNlD8$|E180`={=sB(IGO>Dg6K1nG+# zP;SV9wuXWB!MP65U;crVf=);&1yA2}z|&G>J>gSPY465~0qPZ)S)ksp@cZ^Yw4y$)++1c6JQQDg_*pk|#@7FJvX(vf!^v~zLu z8(pxcjgQkA_A$2*Rs!?cAqpFoh^F)N^P)o~{c_*i6oeHRp&c)mX=s zt%t06|!r(g>JmYV2hy#z}jMfG!jixH_)txnJ9k3QJ16rDZYhM0lHBiT>5-w z>snbDnt1$Sq3)YJ#qSK~>I$Vpstz?*Cr#x|IS++&8Teu%&QH`7_RoyYj>URtbnMm5yF92)r4tj^ zQa<+b&7y>m`KykOCd&&?xVAcHOhl|!3k81> zYE?Lv2`OH1)_9tD?re9%1O8RH!yoOQ$0)dShpLJOTTla8(uw8M*tDLoCUE?*_K<1g6OYg? z?dPRvo1#8_dc)WwQ3eg&#{QHIm?c7`%s$1G0cJ z)?_Xo%3ZILfBN;gfaW2@ml?>A?g55G<fI=$*lA7Y{1SfWRTmQ?i zrUGO8^Rtp$h>J^s1;AAwz*)V6fwm8I9x*~Q%@80_F9Y`VrJ2y29gvXetb1IAFs!FF zVucP|XNx;i6H47k1sz-F%TiD`GaCK=#ZMibJ&3Vm&qjE-1Ei8*BD~e`727* z1qz%zMxLyeefjoa?x+rD+(N-^>*U1|Xs2c+{#FczKL(&m8#Y>gB}`AO;lp7H$R>uU zJ?mDIHb7-q^$1`_t`AOp2=A)>5TZG!<`x}gdDG?+&GWv90IRkeF*)Za5E^S6U*5_m za16;`RhPMlhIC$R;7Ui7>AKFZO zCKt4troxkt&avhbUZ%Y?D78kn{h25ObhmD)>RdCT;_$#awZ-xE86a8|ISlG8e@D7QVH| z;HNN&zd1*zi=&%37rYtnxd{)=(Qb^|6Ww*>!_KZda$jMk-EMqHx*Ean5W>Xr7RSbB z?kcA?frxZZ^^&m;Wz@|YMsu|)I4tbs9Vz3ZQ5RI!1A8b0-P^s(lweA$j~f10 z7CUfhBBEJpCW<#vBE{#pxh^Tnf>4uO$rc9qbWiziUFj9H#LdGWXE=>9@V|D?e#T#0O5`;?BE>mDFx9NihYd#n z?7NeaSGQZ2c&$4EDu%$j94DUpCsHtfcTAekoi{AXT6-OKD9k%6MIMaPoG6L4mOh;UikK{cOkcO7 z#4w2@Sg*=Wig(SJskPBXzZvu2atjX^&BM|K*e5i1(L$NCJtfTkXAP$b=$rucLnQ2C{t%8=*mmI5j%QpOQ9 zvu-wdm{x3r@xFX|dSZ!PeUanmPbJLX+#f9(wirTJNb^fer0}XapPk1DG8J~Kgq3`2 z4cVIJq*>EL6He)bWPuY!^k>IA#!kG86(*HjP<*k)dd=ndRtENki+#xlv`g;8p*xfJ zg+ygXd^+@Du^t%9s<#+XzHe82v0gzUCdqATw_?mMwwZ}qTE>_)nAa`Y0rJAdbemt* zg~mw~EalXo%r&_dWueu!SN$?8hapT$OyJgu#ixP7(sc9Ra}e3`{L~A5w)LZUv7Ds) zLIL?Weef4g*v!0h+4^!hvFqdWNKXx%r(;8V6KcvzqxZc;on*wR0~J z2+bR8>$UV(?fVBTH_o+#HCL2`+L0BHzwfS{*u&pNt&PPxRLW)#U178CLh_7Jwcv4aV{d=FX0Z2!(~k# zIH#d4Jgu5nb`vMy`h#Y}Tsoe^R?MMI2>#bQN+~-GKjK9DpRpr=uKKPF5CZChW@_X6 zL?B9=JD~Dtt!|Xq_HQ3D`bHp3rva_JxAl+Yq1J`8t8Ew*C z{~ikWT4i#!gBk3(KhLmL+dKTYc}0EgxBgLoEyU?dqZe*{WkobJ?S|43Y@n>HAE)9+ zKM&iQq|eM?jj?uYyq)$Mr;$o?5@3yix>7TNQVn4N*xUZ9yp7>;r~M4G%MtPyXxBf< zHN=nfZ^g!?I-)J-V|xBgv4$lP&@XU5{7s1eK2XLy3)G?%ur}(zzO3ZDx^7PT)bIgk z7n^MRpvrlRCd(zDg*}vKZ4u6PjtqGlX|HoUn?)F{$edfyxEj6)+E@alRK~xz!nD={ z+WlFR!y2c}T-$4}%ph&!Szr_lwo*&-KCQ27wDBN`AY@>zC)FJa<&k%55SdD`eh11bGiQ{2Kn= zh4+^iy1D1&g@&?jTLKl0KYyv(KfEA6Jx0S%`j1nirsVVHzccf%jOM>H^ULn|M50m(+eR{6Fln=-U7Q diff --git a/doc/v1_api_tutorials/quick_start/src/NetLR_en.png b/doc/v1_api_tutorials/quick_start/src/NetLR_en.png deleted file mode 100644 index 9d514bf1b18a0c330f98c28785e5d008f409fc1d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 49488 zcmZ5{b95%rw`Gz}I_g**+w9o3ZQHhO+vXQ_?0iW&wr$(yEE`+V!*(yf55CJwc>J1yC@qItf)FehDeR?*%dRi4$qgAcG|PmIecWzu8Q0oz zAJfChY>q=Gg+mOr5iY**Qwl8v8dSZe1>%5UWxuj{al_7%d zD!oM+H>2r!vZDLFxRcSHD%Jn-QRh}eT^7`(LYEAM37^Qnh=2`O{~1Y792QlZd@g+> zX~Y;CT4xvRrh^Jb#5^}aMu0vgoOT;UPb*cjm|VKLN_`bdecqLio%H?c){Z!~;9$FX zS5e?|U2Y_w2Y`l=tb!fe@P~9QrGOSNaAE7npOELP_DhE;nKAxl0ytc1zF|LS9JE)Q z=5iBrFpJ6RRx*RxbTPp&5n=S+clFlBr?-?;`IX^zlOV%`8A)146p&YM_ftH)@sf(c zEw{e-BBb$X6_tL1-jp38A(H!%+PXwQji&{j71`AlGY1?8M(j@J>0^rp_;NAV`u~m* z*QcPtGe=IJ^`T<4Efy!37*EX-2@02KJkBeUhom4`eMseRDWIH4N*Qv#8|r8$m4GvZ zJ^f|x3DE098uCW6vV73{j2F0l3_xdwzff>iOCfE7Zk>Mo8_yqvEDr=4Q=RXP%wtvP zr5T@XRmdgF@q@t$>QjQ&3Dk^6#e+cqf29})6egT<$+(K)|MxJ%e|uv7rKpjDA`|?- zB_NrYAN0>bW;7bv|7-rf4;Uom2d#5xy#5ek0z&>A~p5vV( zDJ?DSKwj{I9UYi03C&K15h=G-XRBv>B(h12PS8N?%_?`^KsnuoKx+Cd89o1i5UGv2e>qPA!>oS7%kaNJ}jQSkYG!G-jP2CqfdsBnGZYuK-;D%wBAB5SZ$E97!(suB#;aubr;@zTq_ z6rybwx$XcJKOaz#b3eSgjK=tYa-)16a#YmWo^o}B4B@K;? z&gb)_+o9(=cRWj^tm;3CFN5&tgX;NFA6jb^a>(OnG&%HslAJoPt>BKUwbytmd|f0_ zc-#`Be4S03buW-xv=>vIDy!#yV=%S7KN92lHJiinvz!2EH1z7)0#LDF6NOU<0@W~A z^X4)eZn7_6@9;K8s*o1h&|^m%)Q@t=ibg5_8A<*6dx|7D}*-=*ee{rp#G)fxx;RDA4A?}3ZWN3Kv@L@6DpV4axW@qOPu7JL2InCO7 zhf`VP1?drGxJmZe)m+zZyKP@D8JlJiUIS;yz)&}yT3%0U#8r7TN0d0NTVHCi2(czv zu;Yefv14hehYevo7NzdA+giMYdb!KQQ9V5^m=q-)I<&11IZ3{5Au2yl1u8$UA44(8 z<|p2EA2~}B?;NOCGSy?C8$f#Q@`gpv;IKjYHCfS&CR@KXeAI3sP^rn9!}JBa}{V*nVDjt zkbKy*FY**b#r!HWu3?8r>Zi;GOb&dpts1vi(?Ha-1<& zd^`YaJVOjQ_P!P>rHrvjvGamr(i?*t&c$Iae++f`vR8X_MqyjJ))H+u^?9)LC>Vqrvyw2jp z%uGjL_lQr9^DVxdk=$&|9l2no`_tD(L0i#D$Lxg` z*H}V7;Fe*D`fLpbi+$+&phF01dpq@3Qz4t}9_@}lYv808aW?oS9Rs%nx3dgcRPr>< zHt~^jm}Co?e38r!?RLk%)|mHgZBp1h&#V6I9zQX-xkyci-F%u7{&yI@hH9h&RlsM* zfgkgQ#Gw)jvmr6d^NtR-vLvvl?P8ZFtZZx)rsF(9U9p#^$6N$v;nZJ2VcwlhO>-X( zyQpcC4B<8d1=juIE=v63SQ#%{^!CGlxTc-ebPj{dpzY7jGVSMgxh6(U@`}{g*PjJ9 zM@{mmO!9oVk4#J?oS);ofP$SNL0*QQ?XOJ{dd0;@pnNh<6vI^L`Q1cp9G%hgCB^)# zCjDq{N#OItSCcFM1Mx} z5WE8iRTfa(`7yXFug95*gV8b$Aedsd@sNirKCaKMogDjWe^o2$fMdM-o!L0 zc@SqOp&3^&d$B2a;<}`5;WVw{THTpLpBzodCD5TnT#$<`zE*UVkZmJ(m)jTP3G)}NL@{vx(Tk2~q1#0Y5zt3l8iDmO8Q9p!If2M@F=%>m`FNrtR~>Imu>KwYad5Ro$iXd-1yml{&4)3NaikJ zhNuBhj<>h_fn&7^5-8-uL-?IP1OVL;2{5iR<-}=v?xmeCt5UbqDD=Z)cHU5I)#2w2 zV~fo_8&CN{2f<2H^^3fB>rn@#T=>bKKOiO2Y0+7Ug!*r1-)u<;BYt=Tyt z#Xx;#Dt|nuzmUXxtMmZdF)+X@$i}4U{_sNQ(Q8qmvIt*RU>V<$1@Wo(% zy8g-`Aml`>_@%9y_+luep!=H2EkfH+a~in9Ds=v{+k9?Y8r#lOuvTZGoQseB38h7$dnDN!Ni!5%!2|K|b0o$>A+&c=6 zINa1yvtOKr*XqaJoNVrk1&v`v)5>#9O9(Fizu6@yFJ<&s#t*`B~xr=1lO6camO^>S3l9-p5Ugke5Bni3v z1?ceR!E%bUB7{w)pTBUxXg8r1byZcSt{9RJlk&R1G&jAhir*iXq&f7zHjIyQS`xkR z9*M|#-M4%gy|ZLYhLAc|wmL4`YH@t;>!xX%lZ@oJRV)ii-rs70B;@2en>a6qI^@sy zMf9XxQM!|7Q5c;9Czby)LLxN@;`b_Ov}d{o(}qm0>C5 z#yomS)kcKmc*)3fJ)7@!JK}^18ei)O6j7G;VmWm;J@!;y4AxOG>)!B6aZLyVSO~Y1 zY5g%zaGrmdX8c8`Kk*6ET8|$Vq}i@Rl+$zLQn?%{e0vJU-(^yKey>18B|L{)cBojZ zBzDg#d?x*W@x1k7UlPe0UQf=62-kIk2`RHi4;e2u6`2L4{1bQf!q-D#!$%pkM z518K;TfccCykL&*ISkfGTU2 z7rXJI>m*`8xDNqSk&*a6DF8tP zRHG(nUFl#-b@PVooeoq zP287CO)~ZJ(f!bw!!1oU8Rza0lx++$kK`_tZ17p(?w#h)3(MQ;A)03V^;cRi9UB84 zv1Eo`*e5C9m$)#nacyU4qGHDc|2uu#Mlx>#D}>R2fB!|_b}9-@40A&Kv;_hB$2>6Q zQ0j871qj-H^7yvDDTbHIh3~F>*?MKr_Ovr(++~2hjYA`8*d<<6^=%WMeDWiT5fyZ$L#{QigUozxn8|-H+7F>J?xN4xkrPA~W zv!?M*5#;E*RoL+RS=h*X7F&~e8#=*HA9*KaI&XVBjpt!dbc^}0kK|6CFN81C%((m@ z2j-)9c*d)**RgkO*7(2a z^M6Zg|6=9N{){ia>{#%P1Zp+M&pvT~2}cQ6>=Bm!84bKD;G?G4fi5xtwAZ1ow`7e2 zO(83Co);v!*%2)0z{rkI$Usu%VhGYPdLkcb;KeTdOFs19BWIGH>cW{q+6Kp#XM~Q{~0pk-dV`9AOTKI zf(zt*l94R84DW|IsyEV@wZJbAsnBX*UDH(xMqUsS; z+yGQ$W;mc!K?)9j)1#&9Z@9}r?wV~fUYDm-82n28gR9o6ncIeEjxn+0$>Q1tg=Hon zLFUOH`m~nsi=b!|)5L;}v4QbmbFE8w+w0QFe}TN)Re^SQcA%+;4U!menlO|@v!v}C z;QcUQ=4$gD&5%I-T)m))h>9&$3x{bjM07bm91LHM>RCC(SMxq;(6=s%Eyp_)AI7j# z&%5_Z^rQ|Erx2EAM0h_ZR`vZx*7xnxa}Ndfo-t<)ai|ksrojwBYYxk)lr1H++#uh9 za^H)36-iOupikVC$Fg<$g2|8bx99*a$wyX5t)be_GstM%scg*&M4!LX(Xc2dU2F-l z7{1Vie=yomh6v0j!xH~kOG$*8CX%*6Kd(9qK^W1SpuHOofGrn$9F_uB=AepQwklHX zFKGLhN;@oSa{4TmMf_7l-kJuMtmc+Ep2gVCXA$!jk`i>mD?~qEP%Xp}6I6e|k{UR$_#I~7OxqJP%wnNqfF>8B@CkXM ztBcc2)q28R_0FB4$m1U#-{aGs)_s_3W_$Sc-uhocol}$iWd!#|bY7ntoJ`p&BWyth zPe;ObO{@Afan!k2B{ zt|O$CC;uSQS{d_i?4>g3u;lPM%CT~4P$QboQr;3V-pT!CRjv52~kWhKPceHO;yW5&r*<^p*Mu6(jdEQQ%kkfuW7>LmeUIohflFtGd>IVm5NM8)r0h zY^W84OnVbO)?wS_l^X}ao!=>n#{F+@8?2GH;MtoYa(%Lr$Xabed zQPi)~(Dg zGj6{3QwpYZmt$8J38C2S!YmgthkPP>4+Dz(%Wq&=Dc!b0)1~4Sb1;#w?j26PEix>A zOT96V{Rw=VO)d@Cza;E?=we*)bz4uM8nHkXWW-p)$6NIL2+yj43C`#9n5ve!+r^Yb zrC(fVrn!D2=vp?g2J~yp!I@&$wS0pk!F+yc8FODOHu*o(Fqw_ZA;M>p(e4f>ds%?V zqmeKxK3Crz+W}8@|G{OF7#-x_y0Ew=6tp}et!5_srsn%`LB3)*W>c(mM8{zAa9JCe z`-&9%a$)xJHg$sz=duABWb!*$x~s;L1%ob~esA#IaA~eHTod@rpa_3qpZe@&XN%22*9e>( zuR~P4b{mIuIVflTQ~>zQ@=H{b5E#nF$$qZ0(t-KJ7cr3$SlXzW&K2F{jdloCA|kRJ zQA&^aTo*mLVOwI;P%%qVdgl_7{0uu^P;Sk7;GHQv*%b^rwVL}ks9}WmjR1=@ztpUK zgMlJdeM(r30Zj?3oI4D$yyYEu^1zCG^eNi%t%^ehpc0?ZgGRJIq{^H+4KKa<$hpgr zu#D*5DkqQFlnT?LumD3aKYK*pKy8s=30`d2pOg*p!zzHSJ8d4g0Zf8_e90S1& z9UsP0<+D-=2t+a7iRqXNu%d^z?6}6u#uoWAf67j9vK-69y-V)Iou2)Lmkxhhe!n1c zSH0h|`_9V!G4{>rX4iY>mm&F7&^Q;A**SNJ=fELfZHWH$wjo0UpIaoncIOUJ|w%bfy+AH>*YY!I|ddYnmeJ9P0#-K8{f4 zg5M14ig;BQ;s5SCz-+@PDC|Rw3;Nx;UB>Yqc^G&MjSFiP0Q2c92I7EoAdM<|5&X}y z?^Tw&c*76xB}Gm%stgK`ZvO@;*V*P97bpeVDZo{ffU1;P2-PpSr95$;Z@8-{l*N`~ zGkJv5uK1Q8W4p#ZIBPn=p9Q9M!w^dlx)3ate0u?-oEbzK ztuWg16#9crfqni?C0vjzt=*oi-PTZ|~5d_9EZh|b5ywRV_;cEWi zgKfHz3DYzP>%(hR@}c*wcf=lA4Vdw=q^Jz3vKT~H!Zg)D`*kb~l>Z>Q8tiu4;X4}_ zmNnjgW%gSi$3gjB^yS;YZrR`{z;(ygAyFFo9F83ZSN z7KVk%u^CE*Gs(}TA9HZUIUVfj#tY7gw!b?>pjP<c9Gm-+wTf2e918MW5<5FEp~9t z6#r#*C=Zoyo`@ZELtvrtcV<;?$bWG=G`9?m-m3HX)*RwA+lb6zmdL-4fkiKaFJ46I zIPY;eB43_SHgkdQQRaSe52WnoZZxL`yc7&-A+8EhTkjAFqu{V#?IFyKeCQOFM)+Fn zTu%8O^U11RtaWeTuQqXSu51BHO>|>)zJm)LiH?~gCT5y$@wp=Yv6CkAaw3|cP zC|L`-6Ye`iXO>#})7-c9aQ-7@u*W##stb*2vJ#wH^hLJ2a$nOb5}TP^lR#XR%6r|Y z1SuO5naOjod%%2h)0w&ju9k{J9WGA?LtbY*l3%dHHzv7=4E=bGLZ}D;6M7Qpa&7TE zRd=Y2wBJO$arthfpy8c#0&ky>Y5Ds2o>9bn|5nCi_h0`_`1Jr#u9Y#Vc@ZpJl|2ckCxCpC0b)bbZ`gHBO$mC7mtT3H$1V1xeVdjQ%`30Ai>6~c~6BApa zDlt^3eP;$}7Hbl(L!}~nY|6& z#d5HG00T^(h}wj>iuoKbB<)KG&8|MJUUrdii{^Rd(;7K4*&7s((>7jn49 zu+KOd&@voNQRdd4L5>EwHV=w$^)SIre^sTd6}8paB&zJTwew0rXFm%lgHJtEedVue z7uPADK+!FDABcMbk8;ZZ3L@_FYlHz|BKUQ%NS7DpF}LWExVb3)k`@$)o9RZlRO zsdt=vV0=qXIXv)iLIi+^QQ;-sE{H0oHC>rsAs7IY5V4bvq%bE*Q!**ZW3v0wOeM#D z1{CZ)YtY0^4e!Bj=(JB`{!!&tovuDRD>zY$dac3@7437)1$QsflW+S^n!yR3poNqb zw5nWHb4_|r5tTR{mU-bc>Ua*S11i%vm0?j&A3l;D_2E@YMwtqoTF(Gw%64A+9!G3Rf?QJE}BKb?RzS#dUY6~8Qfi`iu!{YN?9rJ zBK&8NkRX9dO@Lm6!C2qC18z<$fT=HITKt&l)1+jjY8NqdlLZntwb3i>FfsrSg$ZKc zC1x~R==VwH5+UxTQo2tWl%O{-P}hNpO^fUhQRRlMbl#xfpjy# zNc|^xd?_J2NJU2D#6qpy9RP4;RhU;`S$1(+XVUokMEV$I!umA|ZH4}U{;`QMD&XU) zrA?pS{L`SYT~lNT2^P8v?3?GV1c?g!zhrV17AQUuhvg&}9OK^PyG&vDdWC>-Arv@r zPIx0Cmi<6sn4t<>z@z}PC`*D9B2E9ekV;lj)JyAPRc@*6NxQi(9~za8aGEnx{&A|f>+V4>$B(wm*Y!5`nc{v&d zL}e7d53LZ$7d#0r%f;qy%leGh`ebuYMuJ)dZOFp{!g6FDZY;2a=;CT3g z_8jkQcRn`4uom`0K0}?Un%|;3!*3>;#m+o)v6#%W<2v>&0e;dm5;EgLJ@Y)#t?BH7 zz!=1}V09VVT1y@mnsWRCrQqT18_$!UQJI;GDQHh|?%W`_kXRE-EK0WDN%x=cb{aD% zj<{;5?<8sYQn6;#Ck!AVLZsCp*ugX+`6kw%nBLrGrZgoZR8xlw%iqZM$S7@ejoVG!J>L??Wg9m?5fuxv+ruCpAfM3E#78WO z`HUzlRjf+QD}-p7%^nZ?yhzqfPJ z87!Xvd!T`WKRuOxvvueq^9>#Dh$^~U6cLmOMvsfd%EI+&yB>*&fw(Z5>>QZUz>a32 zpC^KD4CfAiv2jXxk}Ga4H?6owtemr?!rAn_V@XU~6OHA#nfI)qLat2<*|SVInw|-P zP?TCG0rP{!pbsFOQ>c>V**z*p1jsH%STHZ{d(!S`9|~yuQloRXw#>itQXT&D-!R{Q z`sI2~ei9@mz9w`j^Kw}G@+6kwgI@S)fYLpbM2b9ZZZDyzX^7WfZmID%{9*jea0khG zCn)Exr=6=-Ge0!gHS%=wIU3(apgI8*yS^Mj?OqNlY~!MGt7+7bwG89uP}`GcCE(tZ(5uo&f&DItEOhb>78 z(^Z(5IR5R1Pf(`i_H9!3Nr6=mj9$a{@bPmYo(o+>)&=W@MsGw;A1H>R7o?99te$?9 zQyRaV($nd)v)?m4OuxB2vrGG~Tf}sS)OnnZ`rMCyhQOpDTGNJ0RI!WK<`0?c-B<8e zE{SRWf}A>-n)9sQ?5Jhg7jT{#%AN(gDbi_*TX+yy04dT@p~khdlM&i^7*mf#VPO*G zO0O{MthQ{dZUp962M(vChdst-fK`@G(f)Hgm4e#QC0-JpfxLh9#D}yg&Ex_;VlLnf z)kb{oCr{RW*7~@O7-BnW(zBSTqS(Mq27Bim5=S5}9Ct29Ul+wy$Xk2}g{X6f@!|XM z<}Qi^;aS8AN2BRsM3QZMq*oyj%yhbPXGfO(Qpl$a5N=#Tbu0EQRPtrm=tv$++riw+=fJvnHD=)D|sK zQ;OD*SW>Gsh-fWY%!t@G*Se6{ji>HMo#c#;^=QY+?;Zq&t=EQ`WV zfIk9eEB6_U2I*}6=yx>J;H_!yN^r-DLL$j~+%tA$8KX*nQPV=R5QSEnvkA zFak*+*y}6EDO#1FiT{BWOh(9AY=QTnjy9Z%2!Fw?lWY~}W$w?s<56f|zV$|^49>Z| zo^F4f69j%DUbJ)7SpyxW$?V{v&E9A<{dCXd)9r7WXn5D`Yw9A=W9b9$(!#mj9?@5@ zK!}hbPz=!sx0fLOzyx)>h|wsg z3GDBn6<}|?qj9{{eD+0Lg_#jYc^CG{u$j@DF>BJ9PsZ%a+$5&^_1ZMih>H^Z&)sj9 zw~l6Kx?7D%hZm63N{#Q8FS!JfRX?AlI!*`U3Yw9rtEX9pFV#5c;_x53q75pZZC9^*4 zQew^g2r{zpVxfN^tq*l7)EX+B%cuFh6({>TcPPht%)ycZg`wp>1V(u~0EOgT!WfzW}ml??~2^;<``nui!_ar%`+hua~0iWu& zg-00@m9H5G?c}gS;V+>F!3tTLm)lm_5|#$KBLzX1)2{+DZ*&SiyzKCr0pr)cEiN^5 zX(G*E2>>J~O|h#@c09v$#O9WnmJSRIOh2OSwdaZ|M17v=XiW$w{Kj~QfX%xJRTZ!w zX+QQ5z|qoHrhrA)->Hi5xzTth_q}*#*>#D7&A0&ySy)V0g`RKvK0Hl_Af6UJDNmT8 z!jjg;ZaV#K`~?cj=tWSfXd@inB3`EN8@Skr(u6b1_i3C z?IwLgeKZ`n)||MTHkD9_G5KFQqJb$w#pit_SLuPYIPo2(p^?HbT-qqSbt!b78$ZPH z!4)?XJsv8E7Hc|@shrR|`uIREw*-h5J0&-phbV3Pxzg4}1x3bNNy^tP4aNKWnMWVg zL(Ye8uS4s3rbk@xUzyB(qCsqrN;F9JgmlFjh%9?%Bn7!7@Y2k<;1=>< zx!*FQjC|Cb`waGm9Vb>Z?svGIBs96q%dT+go_T)K%|Cl2fKMK({eegrotcrMn-6y> zOWd4Kl*=b8o-HD=kFa}RCx`K0-f>$p_$k-CPB3QM>2?=*ljC~ZKb~x8pxequltI}5 zPd#qPQN<^8Jxmj?aGw&_GQqXQyg@wplht0n;DU9p69OMB;Shm;k2xlt87g!%32e_z zcOU-40_)2_7gzb4A7+aT+>XM2C2c%y%1w4(GgbtnJIt@uN{N=5Rpu>`iOF((GI-q1 zydAt&(kFCp-0sKwxF)c^a7;5Fj(#)kR2??JjQf*JYN0phgm1hYUN9daOKZgaqg4QE>fuj91dsYEZ%yX|XI z%U+uf{BJn7^VUWENc<*aqn+A{#Tnee-qTjy*|AR}-Bs5^`+4`jLl>8}AH4A|7xf3x zNwLXebnJ6A9UG=X#eQ+kj5m4Ej4r3fnB6lVFK@Mc($7kGEG!H4UwCU{Ome(59Bmx+ z<~v|3PGz(Q9kt!bLXWrxQ(yOc2$Jxp`%iECD`j{oHmrCQLT@ff)vUh>i1mCbazVOE zbN+Yii`tFdW?t~~JG8jYUztO**~C-REWt5%MCWl0pP$U5&7u3yY~SF~V`N*E1CU3j z?W#>S7!^)8qXHfCz`cH@*gnxDKALqJl+svk&R;HSJrY#c=DV}&SH#6eY$vz zws`h^T4c^Uu&+1duR7XU`~rIv?XRGqni>n=~H#pVEiw<7Kw4d@~qR(y~o4u;irckbxO5KX!70`4Q2#e;kPr(pV~h zN3`u)XiEmArGLddnwaC{fC(qflU;%^QIW*dr1|oP$S+)52un{Sg@K>> zPDGRj5T=f^nPF&P1u0#}nW?Us{C$bcu+#c8OMZ-2AcC<^J*RCI0DA4#Xr8qkJ-2^| z1)A``F|TjfOEH>%6eiqaI`ZS@O+ACltod8M16_YpdWP-E!ln{1j&dz*DxF2vvgO>w z{d8~Sc$b4PeED{62YWDrLOzN~jz^ip>6Wmx`KC?Reai_IzLeE0i{=NQeZT+rv|nmP zpJrv(tz_Hz!RzsHaP%!#xMEJ4MXcem88v?9kxjO5)nr{aXKFh<Ciu|`xeoptA!JAl#K+0;h8LT?T?-vq;}x>4?5FQQd8EM(Nl)(F`=6sR;p`JqkQEO zIGIk==kx5?SC41&%E}Hh}sH{oA_|!b0f1De765r`{c%K}u@7+91U-w!%ph{+LUh*dJ0@ zxXte&2l|OFw7MYyHdD^4WfSF}dXZ0A#T{Kar)tz9uByr{%3$ilTd_jWspgH+_b(CW zo-K$@p536(6|T6hnq3Y95<=#RT^F^Vjf|x__HtA1d9`*q%_I+hLK3-y=N^Tb8M%n; z8mLbn;@{-&UwIWZuIy1gTlOr0PvPWN_Wc-hgr=|u_mQV9=zucKy(5pDipU_=GTJTy zd@@?u?Y#7xvwNU*BMScmPNrTL1>~i^8d=!3d(OzQAVMo5-fynoy#)|h#J~c@V9gbw z2<59ew!?~++*sVfy}hKvg@DtAft9X;574mAT2prjvrq}>e-IxOfy@h$MRFPRvMmL= zKAyy$1)9nvtCgAd=>&wUwTM()k6|7c49}PkPu^4cv2kLYEDaiPDF{hscL<7GWHH$l zy(G&e|508OK>SzeE+;Ff?RxJluiXyK`X$P&kkBv*RNi#IAkE-03e9k^RJhQTacO#g zU2=vJG8?93fK^#a`i_2*ifJmf(s(gd_GqIxB`_ryw9Pf1UVVmX`d#Gx$Vb6c8$+z_| zZ5M-;2Qyz?MC7fU!H>BJA*$%QgOqCh+WfsWGG<+X1knFRUMn?N-m^-HX}w9 z;xv9mr`yLcclS?*mHPxc!_|Mv_b@vmqzOJtJ^U-$SL@#r!kPW^Pe9x@QFh?$-TJ8&NW)bn`E$8g426){~Tt}ehM=Km4G_rC)Byk4JoqxZJ z8t1GF8gDBOo1e_t_F>yKyy}Q}zAS#-qFrI!u?P>2I1BZx$un1=C}={;#EJT{H&nMv z(O3+y-IS1nfG4WSqw zwTgPu851DY;gy+w24JZb$F;-Eaj=ZC|NFi2DVNznuuH_1%GGjl#c4`Xh#(<*hg`A) zMNc@~rtxbCL9*J?xKaNG5MV8MW=*DJ`>f2V^V#m)`RtOO&>fnW9*PiFp`j0=uobD> zu1MAWoM&v)S+-BCGMHEckowu8AQNeMR^P&_*{=Om;PwHln+9*zMf3u>VWR>?@a|p~ z1x#=y=LWl*_eIWN(?aK|&l4`dgfS-dT>@Tmv5?P0IXu8fe>R#gzxIEleRfX~BG}P> zWf66I0~q*H&@%`teyKA2KAJpsT_&0abV%`}bk3wOdUacA?tX%NY4$?zH8zD__Nz6k zbe?)KAS-TMH%}{AtAn(N7S2p#7)L3~FA*L)ySU0|;eXOVuUbhLLZ~bDq&gpwZ;LBk z=8Mn8r?xHii+>4o32%10pZT zov>?pW6SrC8T*<*NAwuEysaBq`Ix;vFnSA5kkvuN(_%i)>D)J5ilyFcR_z97k}>78 z1~Zpl?K@ci>i@7B%)u^X$gly9$p zB2bn4nD4bu9*j%w-OJTi^BzmVRgCHb6Bl74s#%7Z3KLC#ZI)cVW?n_nW`$KABk{(otJUCFQqOl545V}Hg?{}_E~k0r zrL(VlbJ+%es&f&*GDrb6^Ef7(WC0C5Yw35cjMtBgRu;~B6OgJZr?^Vt-G!9gs%#8t zi#oNs3@d#zkUuZTzS%eN{HbQBk|}0Pt93T_|Lj;YC5}%7Gjw&8#|uSVt}`;@_&A6C zo-%T2d$PPbvOIU*n00HqQ0uoH(i!akyb)~#P*4)B6+kF!Y}OtbNGFmgp&ko|)RpC7 zxr=(WQsayx2zxZ-9-jnz@%?TyZhMLlMspCjVNmA8_G=*V6S^Mxh6}R} z51ao?KZ%EHTn%FG-ZOS&Gl@FYJykgNADn zcpZUiWsvgZ>D_oP>y*>#gVHUT)mq=tL$IeXO_0HH0&ms zsrUgqdK*uK@vBs(_;;Yu4E53Iw^mWOczz+HgJ(vD-(-g=B2kbnnaDFQ zmX>iAeQKL0X*%5QRFw?85JT3ZBGV-?WviLl7*PGW&w}wPl2;M)4w}aE##zJuEHxFY z$bEkRStJtdK?wF?^Y;)DjKzi<#z6NOX)oQGW};<>j`SOb^_mW2n5_ZU7({rc=JPpp zxMDlYxYAe84Q{>&Pp=n356N_;si2O`vW9mx%g(0Z6bnCM=;$GF9|e(zC5gN+)`EH7 zrfG_;U`Bc4Zvk%%>YEZwKC(}Zce~E#>=8Y}zT+p;(u$>9i*EWZ>UT^Rp&k+bObW0~ z^Vw$#qGHZ3Z%Yf3#^?Ai*&jnt!P0QKMhfyr|CBZO=GxEVV5|wDvNGv@D9EzxRzwlV z<{}shZd|C@DNUR)-(00m2M^}1oQ8v+*qRgwlY3K4k_bBtG_cS0k`ZFrM_pihGyLzM z5r8g1aPYkgE|g3zmu}1FF6G>7y1;P;4PzwZNC)|iLu|5p+tI2VyevpiKtoK9k?4{|0|?KJ53EV+i{T zE_JTUOK!wwPW1CC(WwM=(P2TwAU$G{z${tbp=GQ@i#zouUEjaF zT~*F!7Ao-5k3r7|tXE_)0lo}q6DV-_*C+L6L+jdUtp2mM*NlX==rR>|@>RaxjTdRE z1yMmVFOc=d%$y7%nNBUHnrkr5cu;kzzkdx&UAUuzjLlSo6*Q*5BH%Bx`LD)qI`BVl z5eL+b>j9=i8QUG!^a|dky;{2i+vMJ*M;51q!apAf30qH^aQV9EzSNWyomo<0&Sv2CmJJodQ}J7 z_v#v9f!Or9dgRcViy{YUG*#e7p3ZXSW?5Z;B~e`mlYjb{FW~Ojk#D=4NE5)%g(gVx zhWa0_wU_`DUz%a#|41>>GB9_;;X0&Lf1-yeO&$^yQ)%2oM_u$tMIFO@^Kj2tHBdj8%Gq@!#Tb~p- zc3YFi=^gtQu?3=@Tb%##9F6CO5U^hC58Wuy7@p!lCB8*3YkqI9{E;`}?BTEvS^qEm z;^1=4HtnpoeVI8A?V5g`fWxlyGZH7pYSUrs$L$e5it$+tV5BLm&N{f%A_VavtkJ2eYQCp9RM(|v9b(J zrm0Y(#VS`Aj~Dz7%Gy4@*v84VfjclL8NGeGdBC@7JdCyG@Cv_Vk2FvGuqWqy?v@^3 zbhB@c)Mq$#_62!AL2Cp#WV^IGdW=ezbQXs|UwpS~Z^&bd@UM7beqEp!Ag>UZ0p5Mx zk(Nm&P3&u`eVtSSVR4Ea@>5ENlEq%q{-bo{rrfTdl=dp^DWj{>NFF)v&qPaRD&17D z-c&`1%a8YuRIljQU*@j_I9PdosemOvoQ5i=)c?iSSB6E|t!)bqEieozAq|2c9Rk8o z(%m2((jnd5BHi8HNOyN5-Q6t>-?;aq@7Cx0!*MeU?pbkloXh2FXa1Os^^OXB?_@LT zJqoeqo-6UihbM;~=2%bdyHYDMxv7Nugl_)QX$!d1{FV7m@*sIqbB6m^27g%CV<^XX zD2Iev1l$X|a4S-Yj-!DiA}E1Z=6h?#)QLPovlPKOp$Iz$$YP@C0r#gRU9eVY8}#g2 zu~BYVic6$npP@58>?FOIkVbyYxItZq`nZN%uuJ!NuiH52+gda9gL5EG)4ifMg@1av zR>LwDn6Vj6BG~z})Z6{(Cb#I@4-TxToafBe_1D*{4W(8Z2`}cgGL=8yc_2i}QA$Or z+bs_Oc20HOzdd8aGh6e$pKkQa&L%%Yz9-!tPNC&IXiO}cfD3;*0VQ#MikZ^yj|v<| zp(uF{hZ+s#4Xf5E{Xo^>*kY(2Hf>qFq>7z|yfZoLquF%yBCGKFF_BIpVA`+%*GysL z&gmjoc=nV{Tm`P*QULt$V}EQMVLRWb@K(jm-x~JC#}UAY!?Rb#g@_5kSXjL?3@t<~ z-Go5P%2q@ET^_Bl2lIwCk&lo9MYnaDT5HbySHV@3CaCmgsoqtv3d=mf_d(~Y!PfyB zrmmvvN+exJqpC2S7$!Neg(X`$Ho2~$5-;LP+i4WNMzacYDotNiWvGX(>9WIg!e$O3 zv0y-Q?Z|==(}Cz|9y4dReEkRkbAT;te?peRkBX9| zJ9!6LZJM_N`Dt|;L`(ACcWe%jX>hxD zoEaJI;RE+YaP2y1&y$@DEO1ZphW(-;Y9U*FHy0{4VE>lG{j;TnZ^5T+Ka? z)}@_$?c4Ev;?ZRs6aB>g08WDVv!5{{%C*=nT31!NPY335vd}+Z3*ZiPVG*eERfdxw zNHNg>X&M!@*Ue&A>O@KyBwiB3+xq#uJ#@y6Mv4yUmkxLfA%OgOyP|i$s7R$N`k$~2=JVxSe~*l& z{e)$BEvfMZpqT1G=R-D6LnYCQ4MyR@0vQTJKWg!7m<^R?JHAp8PD_U;`osd6Irt3i zgjgx%Zg!^b?5>Jrqt#l!({GXd&GWv6ru#z(s!0@paU7cQEO8f}iY@~pgH4{YinfEO zH!+685^{fF_Zffkzjbl@@}6#lT+A3PZ`uwMCc+8=ZDQir%#Qp;yU4m!%0-9t zBi9x4Vp@l^(-#GWF3ouPF+2ak3DG^*;N#KUEIjquq5!%}?V7H`dV%oEOstgxuPmNH zPZF^{6VUJiC?-Dpu1W!{m$|db)JCIhPsPyFGQ+kIQs-oYWJv8Srg@jg3DtmSlv6m$ zckuy)CJj?=W6Z6AGvFTpez*#lJNbf3@l{A0PL z92us|d{~FBQ_v*>O*Zd>+4`#pWMm!Fw?uRxVNvJ;7MIIHBdStfo6K*Z#|;Ck92fmL zerKYXNpcSaS<1fFv_*2WyYqz2cS++gYlA*NWRHjc7qaGEGWn?W^KgSvQBDQ&HSs%M zmwKxX?y0n)c^d3W?oD*}W7^Kt#p>Nck2|4HA(%BbD9dsZQY}_ThOUUEx#s(UEoPBx(?YbB~(MWv8Q~|DbFLa*r1DT5W#t zh8&B>;Vb(S14}d_lb*DNS}l}UNl1e!C=f>Q0PFs$BIUhKR*7+$KV1qt%P8C z6Nc}x|H2~)govsS&;tX%OUQmPe!)*X5@Xu#+kcIaLPlnUwH#Ic1XqCq39~@NpImX5 z@-Hxv-{32TWRMX&1R6TAU)_2CT`d?o|7X2k!8e@!e~p9)Xl0)h`I%&bN^Hs2YS-v( zbx2v@{{04I{?_AjkY#g4iq!c&>rU6go|E%hlD1!V- z#sEdQ2m4Y+UizZLj+uv3;_M{TovxHd4s4@9UsM$B6=4kT>(q9ed-u5RB{Jx*@W~$0 z12Lj9b~~d*3#ZYf3{fP$OxAo16g738)#s7_vp~oitDvE&_wxsHD^mMC0(0_gq1W!$ zsNo4!Itvm4q7*(DD$Y?iz!E0>NjqsQ zpcCdmor<)6A60QM4K=FsTIQAeuVCh{74H8iION zPYulZ6f`L>qSfB8miUx=ExH{r5|YI=8T0szda^-u3Y7lkiUZRi#;gf_;ie`$V^23V9#I#=6sKxoNZC9dy|MC zw`0O@*F58I-V5*vbyd&sM)jyOQYqj<+5dC; zWY3@_NQjK3(Nh6r7wyfUxa)A3mO-4_rYfE-1ST1qbO}8X8E8+SQ@Ez+a@_~f7AI9` zBY3UIWwC;h$LFbcTE?Ocs~Bb~ii!UtBGNEHh6!8&o;Iga4r2E}6m@vywzsa?9f2c> z&v;kqyPValu(-3nMYvykN0c970UrqrgK|Yq_BjRk0%ZTMi~GNVB#aS|@B!>olkR>- z*&CaCO8XGeFhq3?%wk^49aR^PHuf>r= z__!nPR#g`lDWT8*EYJg>%HjC|8_4oGFo){|a~P7M2AfktuwP)(!<&t03#@sAcLoMy zwR*T?#>e48GUz^Z31F~15D1jehSw&>0kz!ZFi2?_+k{7#W^W3JkDMha z&d-YnP!W}I@xJ==)P}^;0s);{uEI;nf6VHqAkpfA*uW)mXraG_%s&sFio6hgTWDy& zP4+jF7sWk;>BMAOL;GX1{-YJ(rNO3{LxcYk*q)dWC$CoL1DDs|Gf;(r*!sm8@i$?A zzpYFMBJPw11AciJzcm)PkO*SyD=;$=M*RIYMR7<&L(t1p_&?YfvLr*ypSHd+(;e>R zUL*V8D`F;tOyZ3ox>pL12dNV8%K55;=6z#yeczh*1s7pLx0~W>N}&VZtAB~`z@NL9 z#{&jiLrhlM_+Zk-Xmrp6Z$51tNpQ+LnS!U6v>A>k;@?Ob-#?KOybfbkDEJW3bM>$2)7Meiq|m;~y1L zFLZ|5qVk!Q2N9YSAm~cnqDk%MrcmHUNNz4s2be1+H@j+9yf>>CzDyH@>=eUr@^V>xx6-#+#7a*lG2pNA0i5y3@h!noz>r=Mp?3;glfd(HUX`NB{Un>P%_Ixqac6b@zrjxHK0p9OfQ&KcD5SU{M0v=#tFWo*2g1fpehjsNW@@pTh&In{fMw>+2uU_R1fEKgirafL|uAI~5m)+&K4vT)pH*28-J*P5?{7zjbCdBMT9FLLgWy zYrU>iOf^)S@oD%_P7UlMu58_gW93vt2bwxm`(?@*l0aKRowh}jA3h?JRrK|N79ojS z(WKyn0B#F@=n%=)p;NJLmi>T+?=c|m0B9Zl=eZuW)X;0;HDX6GJwe`SEXJ~`kTO@< zi!hvNUDfh|(|_ zPt=eW{UAA&FRe_=B+&W7ub6#i_m6{hwO)ka`)TkJ;p)}AzPq(T7{RAhSYL{qyQw6b zbfstZHY1zNryGW@r19Ep++wbeViY#1mCcIqv%cIv-xuxHQ|OkFCi ze-SQ&7L9-=MMYI2PI`*-mk44Z2cu75qWQ#&R@pQQAm((QVemx~gg=_KBZKEw%8Oi;FLydoHsH7VS4M}?BrGVlM(2mwP&2ttek zxp0Ur@xT5GjK>RF<$Ic|d|BSm5Zo zt7}?*r04GU%zi%g~vT-lO3m|T^Eu$@0`s?B^1wWwXoHlAJ3TAbUavO9iWIj#U( zOKffY&2^X2;>YWYKUNsV2f$SaGo$mnODCoK&zG9(2SV^1F}7FKc3PH|65SL?oD&?h8t}Ni_0))9emAgay=OrXE&R z6CFAwOf;C;9&fZ^AwFmRj|B2$AzIl>l2>`Xi)uz39ugYcJ3Jj!nq!Jw(klnC>LLO? z5+mR2vXzjV9~YCmj9N=eM^VaM$~M21pdg~Aa@7_iBT2U(4qJ)aPG`9Y^d@30;Jx^? z8N8gK!N|h*`HH&KbwL;h%_z7tpGt^{FE12qc+gPngX*dRXftvd#jX5@Lc!LMPUji? zNk=J=AqPY##b_EQRK}L z@oTBK*??{?(npn+)ngDmcF0|8<81=EB}<5)WIg+PwwRr?ht8K(p|xhytf~Bj6RJh; zv@|^bW2kF|Kt#buQWvgxKtu|})a^FQFaQCY{6B|3@lbgMSNSz|KHFj#ZR*v;eCo2be|MNAgx!+R6tB1BHJH$uRKGUhz0aiLRi3g)i4ZKmXCi(e_WB0|{Tg zglx53PQJ3t-y?<}|8od*rT_92sQ4ih?Ptb(Bl<@`0J7sDVn3H~wmJ2eHy});57{p9 z_#7Rdtl7 zJX*Sx$_&Wk6KO>fZeeZ=}0r|I$-}Z z+H4YXk8@+l54wKj{-XBf^wU*0#b_wbay>$ofXGOA>vb&IgYP=|!wq`Z)0Hzx zx<0Pby6$oYNBg>Oq~U=TBiPJ8FWA&y}^- zI91~)%&aSP$fSk9Jyb;ifHOBYC3V}?@M*IsehXCUH6n$`$@#uug^kPpNzYSx5i8`B z3PZk9{-#D|PS3eAf$BN{v<^437catBXTkd?H*Z0m9EG9#h_YI7=O^g6I6uL%-Y+Qc z1(J^N0tHMGn{JCB=icfL96hZcJ6)?%8|b#z>OKZ}yv+Z)>tHo;GG%`iJv-CQ>Ce50xORUu6JZlZ>)nlQHkn0E!R-VYAWV(r$a) zsUE3P`3*ZiT{qh0nxyH#7wp6VP+O@X;)y2IV$w>5>yrCJ-H-Dy}{ti#I+iuC8rclBI zw{z-45BhH&kMIN6@y3rTM=P$F~2 zl+i%S@1UC@_ZXlosFC2k+73DH_XWbg(I%#sa$(s#VJnKBYdXT{+ZSE6+zfl%mVu~< zXC!pIzhn?3!Zo?vi<~2(7hI{k`=+tcaLeCkg?UUPF__7aXM(j1a*$cLP^M?CG+Y^v z9g4s@`JUJM*ONQ^!@0n_TkWaSEaK5qa!o|859&@21uPmp(F2|rkwJD+~IYae8E-<6=b5}@i}ktX*+XjVnZsLfu%(-Ar@ve2Kehk)I{~SplLt9_1Z1ku@C0Ni#20#F4jw}&a>c~ zqCl|+c!naiN<)4>B#U81qk3nP00)z+dMgu^?t~KP#tCC|OWW|Y@aZiiK>(WT^A=*iTfDUvJWaZg&u^I4;6Z#*TyL&CLYfb~$CjG?rk6>Nf^=F{N)R zOWqLArlD%hU$rS$fQPPjKdAZr_}F9rTFgr~HFhO2cje^-naiM1fMq=zNo@6tb6LFQ zIk(OMRXUkIiu{-aXP*pkB$(s2nT1pnSb!C`I&c2iI zhk9@Jed1Ni8f1E21*UEBh-#${d9fa|uh)o=F|8Y+29L{%e;pMTrX{;-K}~X8%DIQa&sEOujP&5H8M!V9=-t6bD{f~J0|*7+S4aaRhRQe zEcbyK!c$yjLI%=~NW6k6*)-p(2JX;$g{^OxHQN;Q@ynVS5BA-#(;V>K&)?+-ibAZF zw{|6qMdApqE}4Lq2EyQhn>qjY{DN|Xvi>syW-Wd-GA$@o3Dj=Z^I|q9ZXoV!E1#F$ zfUX5ulU~*v$*f@S6TO9&QIY2*a0brVW0VpNtW)CHW9C(Ddgh*|{`lOPuR<$UNS=m>lTRH{ z#U_RI#6zg1u1Nm&MYC#Dyiu^l;>`6|s-cDZSD>YBVk^G`-)cco)NczW8{=8f8Is}& zxnvjPZIEDlw6N1NJZBk5cqUS7{s z@$!asF}}R~BRhXaHWF6oCaZeZk4~vBBh+Ov^GQ~9YqGKA1uehp?RyF%R8#Y_wEZhx zy5(rF8(UGOGf_O($A-KBkqj&~JB5#?7wpl3pe`Kyf;TH;FbFLiLGVL8qKD0)+N8q8 zFt|#>ri@TyjZvRJldTi8M@zm8S$ts#3hyB^ncye?Y1B_@o1JbG-vY4{`j)LuE36=X zSA|xDywtV0C{NwQoCIg#gyZ}Ccb96FiRQ-6yIF9tUYN{7ZgyXNQaI(kbTvE!sly0c3Nj`rb+XzieObu075VY^$x$wrmGLpRE;u8Hw<1G* znv8XpjBG2o--o&xw^l~w?!->6*sw!4FRvmNRw+f&_$|KZZ@_hW{RGRLQ7&m$)=1O6 zQKNckn~Csz&wHmza~f~pCGbc5Ypy!H3Qv>m_T|x<6$=q(W9*Hmk3?VB19)t>H?oM7%*Y5exCWU`aBMNKgvT6yO6P7HG`2I{YL*Q@gp{ zDv|N9E&}s%OY6&~#ZmNV!!~K_^QA>)@h!s7blqp>I6WtrE6N%o2_m1B*OdJ%RL0J8 z`L&YZhlsmBKP8j7j8ozdh>Xii$RJx^!5fI)E>vf+(%S@_ZvUj$t4S$SV}BR zkJ3&H%`pstt%qY>a@`Z#LlV9T9T7VRHj-rOw_&EgTfe#_U@u(u6ax+-!Jab>x* zrAdEz;)2i|!By3@)>^498~g5?)2EwPtS}X_`|-;2nCPuJ9`0lE2yUU}_&$B{UpF5t zVh)h6D$9!~Ht+}DbjQgHk$v49-q)ewA#Wb!n(svtzn8yvZyG62l1KjD8|?|AIN6_A zOGRDzJY~PoZgFtzCjEX*;J3Wtlo>T@h`pAf(Z)X zIgV?$8#g*9iF3@K7uNMpj%Zgf!sJkw=#|to=$Scquw(ir8Oqiu>>HOvN|w1=7*)>e z13@Ip$~uHA*kqt-k-+t8m@3P5(jAJ!D5LAZ4|4d5j2#HZsD{J+di_h(M{!i!g6|ZZEfNh zd&nHVlEQM%0Wzu&p%|Tw#>MRJGKtNX*r~-#{Je9V4?a!( zp$s@`Ut@N=qW%g8Nl;wc+Lc+-$OcM=T%E8A_0iEiVG z{V-mSdb;tr7H^qG+ijdR)}S2B8)t*+g&pSp!vYIt$Uh|c!kk}Av45vZWLnZhmKgI# zI+f&cQ=kgYZGZS#noc?b1`Tb1hxyqc!KW=@&x0R85?E`RnzXHNLY@XQu`%T78hEp> zQc7Rs2T(J_yo0N$iqW`rEVC*+B7%-3l<Q;A8`tL!Gjm4ha;;43Jn;BuloiMY+~eN*|zkQo$Vq0P-^U86AobC8QZF?38TBdQtU-gv4> zY0S(@S>p?0g6J+E@Cq(f%J)1uNLxBIP-O8<{muqbA-Ub6GRpTEJ%NGjNef+dz0T)R zd+ibHte5jP@lqbumvkBew*m@*v==gJ`?|Z0Cb}=pil3}#YGzWTW2pzXw9IT!wy@@B zY*GCf2vBi8{phlRF)BAp(OS$jD!{FTr1g94<#ky}OC_K~1UyWFKR_oA#_DHMTJIOx zEzT4bhP<_v(#y;>pD#8ZLZ3^pf>y-TKqN>_b2ZusY0g0h%E9v#{wwlO9R4%k_^qD9 zfo^l7xG!gl$tG~IPXARy1|H3}vzSwYd0^kqSfCETW`WoYxLl3?qkbq8D72vycgvi{saahm|x`X`%hNycsz{_Q?rRKf)Li>Z z7dCWY6wlFzhaxs7uv}HkxZBKa;K5UnPHL&yJ zPhuyB@8G;td+mevR=4TUKg^2OoS?;6fdv+2E zql3LH)_9suUxih3l#m|qQs+-cac|R!PO(~(Tq5iMgwA#^+z?Ya1+f;I320zMn@$b8 z#Nf1a(ReP!o670Gg%A_m;Y_m{T;2x4%t=_#WJ(6|9k@?cj`5zsl8q_@MoW|XN#dO! zZe?#K)T%`i7%Tzfs+(eca5XBAf-PS$2`D+rjg{T*&ChjBlO{t{Sn#G8@Dh>s2l;i+ zbJ)5W9+`=+ITFPhNa?ybSH%YwRhSMbtqP5^57X_1L+=L2n|h@C<_{73maky5KJ(tf zh1_D#@2I;h&)f&So4yn8u}S%qP*94MWlrsI0gShPuqy*nU&)FPg@79}Etk=-mF1NO zXw!7nMnZX{lDlP6;VzMN-qtwpBs@EQX)UMyGg;!f0f~E_^2`YiDKWwlufASdhamRP zcLE`0K9W`pK70W09Kjds)=9t>+9_AtOj%*Yi4ESe% z9!$(pf9>|5^QAACT$aA8J4qFaJvz9|aDWxPwqH}@+=qkaOq8bqJHbCP`}(kutU@b@ zawq@0&K@e{-Y|mNPvKFk8oqQ%RMEQ2)%c|psyYlfk5@ueoilG;PG98v#MkcV%{ zUp6u*jx=a_`Nw|9*j1f~*jyA&(}qt&Rnnpn6cZyi@m|cDn9a;uL_-zMWm<}=hFkW` zTI?;&=ZiuH(r|{26t2Z8=|Swzev&V<6;j7t${xE$gF@wNxv_d!kCLy=OsB}I7}6&CtIi6+TGe2s7gSzF7gS!dfy^llH9LpbF6$C?mm)jK7HFNDX+ zc-~W(IQ#bPYjC=E@8i-{y&S(l@biSndIwH#M9Kj5^6iywH6w4D9dRitH$lB3Lhp}b zAP7n|SHZ!zwb9u?w{Umh@fX8W{Khws&U-4yCxbgRUoBe^s)rrB{P^qa$>9gQeb36l z@!ou|eMHvypAjlyqL+{u^2czG8T&T}E@b#$>jY*-F4*z`HXek4c>PhSs7#< z9B!0?Q?cWQYy?liQccLUHRud7t{#EfRZ)%BnKVw<;K2}jBQLlk|tIHGgc# zwH-?IzBfWx?KUO%^PjSDk@}(I%5Gz&MagvGbdrVB#t^i4U+$Hm0O|rTv@5f5xrHEJ z(1^KzmjRfB{wc+I(Lfo$nL0e;-Hc_O`qpxjwYY8o%+l3}@M*(zJDI7H8hecUc@iyj zX7+jZStu%*DkExePe6RT!qYhnmJ10?D1Lz8&LKTPDqQ+0W<$zboQIhW2=vt}3N!l- z7jvH|*YjHW5LYCxjd(jV$Sn(0Pe(>7C#`L=zWY<+x46s6K`@^~1l|PeA0a2skg&=! zVqZ4OZ7pw3MJhsvECoAwcawNPfEz2T3^-V2h>!IwtQn(xEF)VD0*r4MZ?z zxCcWwm*0~LmTCkrut*wz`r2{bh}k~dMIjvxj-z&2C7Jg?9(h2*;vC_RqxS9FYnush z96*2T-Zhi(4p!IxYgFR{PLVx#8|*$$HAAMgDe`0S^`>p1(Cl=m$PG3s@dt2rh-0#v z7*8$I`ZXL?Pq*0(YAyHCx0BP3tlLnTnHRT8)Y=49Dd(zzqEvo2qVQ0>KPfW9qIFq=&wo58 zrFzf``{~+LwdnB1?Tt zIYf{Lkd=fX7(l{#pCAr8g9_GydjO5o9!|cmqSQB!?Bm4%f_s}?%enpWn=?&vi%wrq zJm>S^?ohMJ?kZTpU^4sI;Ov1-9?`8Zf^56*Eue1pLx(jakgAEUDYU+5&%|;}wf`dJ z)cSI40Z()I{eyYurJEP)#&h^wW&FqM##{t>Kb%B};VZly+|0LF!n`Rwq&XwdZ%o;B z{LyZ)CdW{|t3|a9d$PKvuaambJ^Al2UDw@UH zE?BAIm>!)kn$Th?s(tgjF1t2ZtPyGNs)#IZU$x>WNw_=A*}#|g%(HML-=$|=Ep@Uw z&J&(z*)%*_?0#O9nB<>K1J+qFEHfPlx$_|&0%S|PA%rTxK9{bIk?mUQap1)lU~ z#XkBSh2xhweQ`admFHsVFIlXW-=nDZur{cB*>}h0-O*bSaeW`|s`pau^ux!Z-BKMD zbGByr9(Bu!guu-5lg1#GPP@XaSYCKK*g#c*xBvld*u9PnX6JQ&v+`V-!S?z{6~x2U zXK|SsKINo)?d`M@-$G|^s?!GXpAqJYz6J$tX|=-9=WEf{^`!$zYGSltPe1~3b-t8w2i!J>>?QEyCqf?jo4i_7UPRv7L91*9~zvD{a%cp1R` zzB?MMh*t}f&Ff9yw~*D};wrkO-jHI1XN!jG2Ugu?^V$Do=0LPJE;jB|;(sN88$}j9 z$~=gjOGaX3Vn)5Y=pV15_Wkxwr(D@{U-CO9Qx)kahs*obcXC9NZv{N?ucgW%$k(8< zAp#nFSMo|S`c0($?Rv+tk4Pd@lSZzl(+E)C_u$g=TWW2~PYbM5VC~edTVKgqPNNu* z9Cd6%4|o)5JL{eV4K`@&s$9%yvqC!C{lUU`fZ}^j#_ftF9L^ElOe`#bEex9?PmfLG z#Dx;P_ttUVv^*W}rHQ;4VO&As{)b@$p_vBKNqu#MOB@|Oy|i@>11}B8C9gPlQfs*E zS@kc_IlYGSTZl?9dwrhLy0J!KJh4Kuf^B3!gI) z`xD{qO=>dwA9;>Ai<|2egfs zl-!hafb2QGOO|m0p=1FD?_Gf)lLn4>owHHjtnK}h?c5;qTykXJm5HEuhnu$5g_s%I%A#O#@TZ>y$GYiJ+92A}8M7H$9G#l{tP)Zca3I;h1Npw;F z;!_O^kAH85$_xSiTj+`CFSa$lDiTnGic21WgB)?4^)q~QGv%eos|g~f3u4^)k9SOQ zg|}!TX0)eWlmXZ680aEp*6tI=@?UrX?AMA$CrBR_Ew;wSGR#8>Yz%8%Fwv~6%a~M{ zB_J7Q+3}PmOxKrfSY>y`TgD>~}+ripy ztRK&QG{53uYX#iHGr?#&F(_Iz8E}i{z<6x6Pl2(h19bc~r|mtzutwBuvqaR4AV<`6 zsEop+Xqa-;ypX0=zJ)+~nCU~V#7z@X_6W)RDjQVAj=RMZHb0vp_z<;zIXHJOjG#xC zn<6reQzU%`9OYIcZth3?qJA+2yP@1IX2p?*15cXli)oR|Mo5bszmvjS!1}lx!;o>eOH>K!wBWth$ z9SAWd%Fm^Y7Om|$q|Q+5k+u%-W~>x167nkLWOR3P2R|||p2AhETNDJG(vBCd2`+qpu^Ntlm^hjv#=H*;hD zuCuR`Lb^ct>9uMw?t$Qy{>GBtfyqEA`p@78whJCa+=g};2wj6Q;^XsfrSjz5x}6$2 z>If7WthYpf+=#sD%I&7KP0QaSnl{5vG_Ms5lID%(%q+ub8;aKpAqUnLy(oeb`tbrD#R{FLM)eS8-GqXDmBSF;iTai zf|^C^IE8i}^7Hx`)GuJs=yzAo@4&E-~)mgC_Y0d*f zXsbY>IHd5j!>fm;_JJtg)vjcatBOleWKT2c{@H8gP8zw zf!DXL<)k-uF^=|!Wbfz+N0y8=0mHSL1}R@mlH*CAUQNs*Ie8|lzn25=A<>QF8mN8tVreqTGjJ9f9 zwhR32n%yfB7Rtsql08s6!Ny9vkt^3qWakO5Ku&}=4rs~KQx}L5S|3W*8SLlf=*`2J zY7Mb&ueBN1vr^zED^kr4@}%>06?!thhJ4X>*2Wgp`|<7ur|_`+ot&wz(9jYK&d{J7 z!*D49V#pF(L}ec_)_a%lH_!ElyB2{nMu2gtP2uj9MRB}5Yomn`8CQ_{3x2$LD}9oZ z{nk!3@5v2wdk$IM3s2Z{w|KiS&8>UFElWU}h^D>sU`@R2{h1@Lj5%oFMk}QmRW7)1A}lucavE{uk^Gh7OxBo_P6uL>VRCYJVOlGzv8K!nQWWN=W6jumPO7T0Vp*x;jHLiKSIEV0h=Uca~ue;ezDdYJs-++;1oNUv< zhPMLt5Z-KEwL5EX1U3A*EkU1;owUR>q$rrTovAzS2Qf0m?yd=`2j&veH{sIsIYi`I z;u!@Ae(rByZ0$#*4bl14T{w})G=FqG$etYyt@SvuxZJCO2+H7?hpAyIA<04|YnzzC zn9-}N4zfd(MiYhNuc!Fk4o3kJ5r<*p`se5jxveb^@)k7y^)JGP=^^42ZQV%g>&kI*sG%oGU_^6?2CB2w6761`4P(-XWX@eF@W2t3HRv`$Ou+C$9prKaJ%49r zzd&lKKGR{XZQQ-)k108pw;%p_3o3E#><7EV*;R9Q<@fwI#`a&ggAxrX6SU^H>*by^ zba5-ihA2PQTw9g9bO*Qe3`nwRtl{bIwd-?_m!E@5F-yHwrDh*=uL6r#_e=77XdRag z*7ipBX)53D=$SCL&E?+aFrG%YDb=GP!gNOa79g10w(hOSOL$_&L3KuG?GNj}CzecJ z+b2PHBA`1;&1wk6Jsdh03Yafn`HU+dFm>vLjfp$N@QOguoQ8Nve_F*!5%l zEw)Z3e<>c1X6S@d=SXO3A?eH#c8W-{5yDzHK5?3H*1E&{{U+>${9wrJQqIB!Rw=1o zL`*@_E>sVUD+9~`q`?muKwayM=_`g0 zQ}?nx$sy?KZ8}`LE;+hwTS8ynNu9o&$Xe~nN({txfy>FUUGB1ez31>nlZiXW=3up+ zRkQnZ(42F-i?=Sil*_BHnz={npu*H!mF7&n`EH7oog;Qt?E;i{pfgeMB&WyJrqq*FxLZ;Ratpgd1Jjm;tf|a-{d*} zd*W{Q*G7aIUTG|HG*g9|>h-GECoWkCL{jxn5LzRBO7x5a$rujC}L^ zve)DLc>_VuyF%+j9HDBtB=R6YIF88)Q%F(JI!vJ=5Xf)%Npv6!MSl)uaAGi2b(GxW zo_gvh78;Qmyw; zGTSSVKG&K>4*Y@qrHlG8XlpCNjECDEZpgPXEe#{mxE7T4jarwn-voFwpy96LQodsj zaKQF`dJw_lvH{9UCF?4BV=i-35=inQvR}Vx_%`#x>&bkqaNGIa7j1bQQ>2y+IgRq` zSC}o$6yz2fD5-jey+mX+v&URb;6Z{zkU($` zu0a}tyGw9)cMTTY-5r9vTX5IL-5PJ4>2Ktpy5F6lPEp-Jz3>)?=A6Bsy;dNUGT~*p z`+ULd-8I-+*{%20ONOSQMAfeY12xD}L(qpRq((28fO6M{nFgsKGV}aLLn6DU?>=}+ zdWP>k>}3y$KBQ;(Jhp2rP%X*#^2H2FE{m2#e+D1>n{}eI|gANm?6gE$g;nQBQHs92h88u)Tji)L>9vRp6i6P}MSo zTR}@+EH-Bdi75+UEVfK`^spfvxvSmG^5*nwILj`Cmf>Mkqn2wH_%Azp$qPWw92hk4 z4LgpR+h-M1OwVUjO!R#KS(zV^ApQ2WKKZ7~DNR9cX>t`0wiEVm4J$g#GA=~9P=?gN zzqV?V!5U(;MqcT|JEAgsZeyqF*-_cJc7O1FtLf`RsU~B2WpXzl^MaIl?O3C;ixaKf zF002B{*q6^kyQ3vpgGXjDvc+SJaWUsiw)+ufKEf0hXG)79z{o+qu_={fWR4}w zM*c8H2;GBw<~vc~j~^)^u_PaG9%7Qg%h&Nn0)-cJH@3^TisQ`^)z^wjux@paZs$R_ zPLewvYxJ;pd`YYQVR?uu{*H7m$o$QEB#J4E}j?OpyTjzyIDNgn~( zsaan6>Zzkn!%t@_(Z6EN!AiqM)Sw?um0?591NZB zF53gK5Z!sKj}jZ=#d>8|79_0h!P30b#x5my&D~Y>M=Fd&LW1=L_eWxn5Xh@nYM!}- zhH}Ohmg>*aEB1MP)+4EDKVscV&^(pUVq!H#(0rlo&ERl-jd>HCWEO3&P~6&lCU`4D z{nH@NhH&5Vk;9W6g(i|xfpI#F(Mc2dm*pfm=U2)^-c9X$J-M<#4*cFty@ah2^$5Q= zp;~PJh1j7Qf5#Tr=C9OkD`h*bM@)HBpyVySL)tJ5%5fm4{dxzHTfD;Xy`Cp`%KQXLR+C!#w%hl<6(j}hhh3%AS02FxR$xC`Ee=IXfR@rPS>4091??&ittq7$z;;zis;^RJ~D>tH^(Z z1HWRGmp>a$DbfFtqSt}%VH-EzLsc93O>p)`ZU*1fR=rH{M53(KBoH|)kc7k1SGgfi z^IT0tI7Cd}g%vO4P2od*fN_+=3dm%4PRGt^2N)i)IvFEZ-+YVH{w1x6Q?S8_d9o z((9nRA$mNIB!NP3`l&EKZu5rfr zUHS?()5f8@*d=gUjCM5Cs3^==%3O-eiNdp1AL`o$ff2*)J0jiXd0V%*d17!K&nd~5 zE|e&<)U;|#kj~6&%FD)qY2*=b+3gol(+v^_fUSc!-`y0tdQ?CxoSz9{yh*LLjNQI^ zmxj9Wf1hfmdabJ(=*HCQw;o^O3}@N+CvbQRUna085m)Kzj%eUL%;k{~`ej3o-o)ZB zKJM)**?W{T@2T$d%%;mz4e5J;1vLB88cu)-F`Wkp;ej%n1|H-5vnph0mUngxj;|qr z?#SuSQ5w!a;e!mJf^tbQ;1YX)JC>mK;0*&Gu)W5t)#4YabD)6JDEe`V)w_cFeK z{_-m`l!h+=+%j2MdQG_U8IybiGd1cy63Z#tY- zM>R1xJB2jA?$9{;D17keAb~DMhfx5d^OgGRp`_kZ#!#CdgNr0TM|aEvq!S+J;bv>s z8cj#VC1Pg9*z7O9`wq6cQXgP|aCP}RHU*TN7VUOEd?&rI|E8o%W%0yb$w@f^t~fSh zLjYN}3TOR+d0}W!QCZ0#Lr|XLFAIfA>paezz5i{+Aa8L8A|oFZcazpOLq%1Z+b&EY z&xN!J%5&20S*&)&D(**~62(b1GtCn$971wE`|O(uLjKhl zU%x*CDM;xhFSYEnze{HbR3{toeNkY^RxMb08^CA8DBlN2TxL|F-ACWC;fq-O2=ou9 zYbBC(r(i)jSxJ|X*%P_7__B(eBh%|M41iiStBzef;@hHRIQ=st`nI;g;0}? zE84_pha0DcS|0(4Z5>a`oV!OmiXgSAu5`ih(bp4u!*r~dGp(!7MH0?Oz1YAlq+5(= zNA>z~>Lkdhou*l37e+qKAf)Lp3sH0MKC!i>O}Y>i=WPfS0x7`cGj~Ig1p%7x4FPqt zyMsg*b#GR>>cejQ))#&qh{?|}QtS58-rQ8{w7&90&2?_b<4V6a+tK;rvZ&0S*~gdq zh;FVYqdvmM#Hx5ZZ%*5#e{wEM;c@0WT)}spJ(QGM~DcZk-`&zrLexeOfS_chU?Z z8iTpyJn`NrNd{S8@#q-+h>6YMjfpUZGo}#?8{(2btEE2Fm6H*&B7VuZ7x_`Y>;(@> zR%mu*`Ruhhlo#DS;K7}JQ)&A{|DL#`#RI|{Cx4J|U!7u!_98}I5TimxzHpgqJ6b@c ziu%^v@9-Xkp$gKN6t~+oJ8m0ZB0#qKU9|YoYO(hysKxXpe&1S7WdCdGmpo?7TzFAs z$cV^*O*cARKgtzj|NDg8Skg4NoAf0RHGTjQi%@<-sIvb9VatF+k#BHyjHGVHN&8<} zDOP6#%W^9{^+xBmi-j*IwIA&NqISBW2-XwtStbQr@r=Y0j9ib4-f4_gm2wAn2$fW&_`-gpx!d?>{uCaAl6ok+gyQeX z{shD{KlxiQ91x@AwP|UjYratToI6>THR*xJHG)OwvAa5$=f{chkvAyl$KKHebi5Mt zqDt(O0aU_h)i}JD89ruAS&^k{F#_*1kDAj;uS+IA2<^$ z111^*WV}u#uddV=zMRq;Ju}x`1^ONzq4L%FdDc8GyZ5$)7@Am|GDGZI4_8S9zirD=!otMV>1F16Dk5$nt&)|RFMobW~E z*d;4N(YZ`OISFU)j~a{3?G+XT64{e`Of}}c{0jPzx#wpHf+B0bXwK8r&Lz{i)trN} z@cf7xFIHE3lleZT_Q@Ot*6ymz72q@xsf>cPnq@#WmTwM*oRkf~lvNaJ&4W zC9H?_y1H=qGI(HWVC$oCN4GCH8Hz{f-Cw&9C(zIxnrp{N;3CYmHl(6c-FafGiq#q| z|Dg7%YtAhM*i4pAE?;;ue`j(*$raW|#<=p@^7JO$fxV8EYC2bb>>wP@bz(SysC zH;Cja=Q!bk9I~cao!R5vPLjw|1d%3Ln#Iu5aG8Szw()6Z)yzK&$m+c9A-9?_r)Y_q zhv-9Be9{y1WGCK?zx3J_On2J9rBugWc4hzy&5Pxb_VtBgtk8r>6k@+FhC~3JN~sif z+xt?cl0BNyQ)Z_?DVq4;lofom`23W~ffy;gvmA`!JHNp@F&!Tgd+sHJPPa?_ zRcf05aQ-ePv?=mSs7=mPFWkZX&ESDlSUV6Xb`HJ)txpae1wMn5Kha`{y@%}z#;-HA z*vY54Y#XFSfP3=IegqHpdUqhQ#YMRlf8_V99q+P7>LR0$d~?f0&iEmh^wEfTvW;N( zZ0FA80*xrcxb4FBgo*={%qTJ3u4lz{!gBbIkiJss%l$66K$EDQSih2jUT?n~7oz-g z<9ux_$_dd>2W^ngOn~^E-kaTY^u^1q-wCnMbau(T`9m!lDJN@|7Z6Vg-ieSTBmKr= z+c@m+_zi;5I(ocEvkh$CwQhy{UEvHH_=JLB*KHJnG(3p@Ixb{({8FEt=m00WLAnEe%rr|?mWHagX73LW)l-A4}=B#Jhbg`f;`xNO`G=A2wLTuXM zhuT*4MpOx;NTI_vL#0p;&&8tDpsVaJ-uEUr;&(5rt$w30QHmso^;5m+TUl0xjxS#B zQ#!vqOGMWEa>jY8+8Jf#t`jay#rE6DmDLs3|<|BO8ppdgCrJ ze?{4;AMlx9=YR|%6+#+x(od^Mw?By*t11eA`yX|RsUlTCip)oisI^GiUp|R(H30<{ zf{#TN-QaokZ<(+y^^k4W!AjRN=Lb?3lo-D($s;#A7RQ-yTm^_t>E|y0bguzE#B>!C zj{YLcPU^_Q8iBnZq9y|d;J+ark%OgE0Gq)mo&msq#CEd6sGX8+E${OBj& zd`Qw(6yp|faZf9acT%P$rW@?abEkOhx!=wrw5CSi<*RCG;0H7vuCFs*UBvtA(Vi6z zoPX^YXelq82jZQDC;D8G!!4#5f_;8Fw|?LPj-)9+^(s%hdz#6|B$LG!uq!e{VOkFXZ{xpls2 zkFMGa(fPb1Bg?-HaT$4+f79nt{wvg}ze?i4P8n$fr5{j(m39`==Fbh0Rz|(?$zMZw z+fUXrBLG>ycJn zVZRr_GKZ*`K-|?fBkO_Em52-er`+mN;v1R`p?d1~EdVRiXQd~K4MrX&vvziCcADRt zXf8ZBkD0Fi_Ky{XU}BmqW?#8Bz3L@EN_<>O-v>Jzz;cCz6KCVftGBEmQjKmji9%xx zCp%$kCFP^t!lsaPyRmKjnMowl=jFY0+0qzzoeE!`HWzYtX{Gg(2hEU zx8o76P)czD@}I&Q0xjr6_NJepFm;E2$PVzX*%H8J%#J?o{)LttrzG&`MPydq$sVbM zs#yado!Vvq)LBi&qrM+Wa%cj2kbFl_I&R%k5im-0E)pw_RLMmP2_ZwjKc zp&Gcb>8QznF>$;329Y3_Qu?is3^H zUVDAs5@M^C=4L1nuI0jC2~FMAX0$%GdlX%hXDsei20K&s=YF-W*IotR!4ftDh*>ro6?@K zc`VzbLG@<_32T(8J!}-gC~)Fck{gqU(u=uS{0WLa1*MNzc=Zd*a%iq+U(RE_RoD>E z8EnAa5-%Qt;$fbPXVskX%APrQhBP;LF9dddAZ=lCi3*?`!w76F)i|42f+q znVRHEeN9+aBXZ$dVn|#l=I>1mCJn5c8O%Hh(nVP9WBFhcXx4VTdZ=6TB(Zwt1uHAk z96VG#4L7?k7t3;Pk}HQlO5^}7Y77oCeyN+izeL2sYcH^XMC3k*LD9*N>`L{5P$b>EM(XfttQP9JStZLvf$Gl|RiqC`6LX z{X#o3<_c|q==83}<&jCWSzM2}V#-|TaugbhzOCV-=hyLOE~n+}<&T{RyHbV3MEn8c z-Wkv3wB$$SmtZ6Y=rCl2K!Zn!X(Pyv`N~RI~eNTQW34pVv;1}$v#JB3n>3J z_-Wvj)9QKDeO1)>-F{KXxVW74bTNjOpVs4950TDIITdfKD2K>P-_ih&3 zhrmlWOj1;%V#kNKBU*?Ouvx^v1IQKP?3B9ORK`mK5`*~)fI6scgEqN>Qd87!{nIRU zy{iNe4ki+T1YQ6oH-f<|P05hn0Q|U&Awp;te+dkFu?QKSuU&ue8D^?nwiW%hL5T~$ z5Dnv#;M63ymoa1%uNJ6=Q zaR+xvO%s-41ggoMS1;Ua`p`BorSA?l1XO?QnU|f=fSA`~Rg8>O_CanqTyz&=7X=J!=Sy<9xWDFUg_mrK$To~1oK0(&a|EBv0o8*ch`3*t$VU)`oE8zdS?}Y)7=R+29rw zoM!!P_YA}kGl;sJ;ev(5`<)-w4$hqeTU9@5O2j=n4$&C0`>5wJ zP8U~L9J|cP_&Gb#UBLc)3vuu}b}!-xSV?a?@K^vX$cXQ*@TeNCTF5c12!IJ<1roVH7`Dx zu9^mz2Wrf37-y)H$a_s>f32EQ%SN`)RkXn9q9#0^2s|6$GKK=+m+)3w9sam)L|2yCoj&68ySoowMef~7uV6g$Yrh~!qBvpiUNl@h$a}kWhn+R~N zMIroVy>8@Pkz6&L#ZKjw(QmMD9iSqTQJTz?HTf#srt^w*Pu?h|U-TdyEP^yQK2}v3J zmNCF#@Q>W$^FTQ9BYP354y#2{)_&kiUsOC*QqM`Dyp8%|u{FgOaR$u*KzbgssY_uHU!=%t+MhJ734V|vcR(HWG zA4zNcJjc8%VCR&dL};47`0gZR@Zg5Us&j5A7@<>fQ;Z~azkqDeYui)<_JzV$;d6SZ z^&B09hRA~d-j7$1gfy%2h8sf!A02EdU@hw*YOWhVZg$Ea?QI?L$B^*X77w+Zesm&# z&hjSj0OxeHUWMsS%7!U=4A__*@Qx(^EGM(k`n`NR_UBZ2KYHX8X7F)?o)v}rgk;Oc zdvEM19^8cIN7O@f_lt>+@IhYwT~s+Avt!UaDVMRN-9+=^Ti6BSY1&S6O~X7!c|`Lg zVwsL5zO1Bws#DR~Tswn%y4J@%BUmXPM3)VvW~?=`UjCAsGz4F|=ND+m3;!5&w=Gqf zu$r9=0 zHf#`yXm1W4fFaF_hC|JVy_x?x%T!T=?BV~PB7%ogbjECndZ$sS0=;2xRYQdXlxjaMHsm5szWpj<2=3u0dn)Y~Hq z&*d0ND?BtGm^mpE65+XT>PF^`ovdFI0UY3ITZ z1GFYjlH$_4JqfW-4r4aT@RnZ=_DDk*L29rG;_V8B`W9YWj#EZYWn{r|4Q~7XhkKPI zaso^~{g#uK?8^VCmvPIv2B!$W6an?zy&OZ=Sv%FP(YWY$Y9%Z;^-0kypa4nB6xsaT z1$yzPt_IYHzT!#ivb4K)o9PCDGuu?RA5DdGgx=-YZ(){w$GEq>P(!>g2{J-$WPw1q zheJ|2&2T1DtY0%zuPmB35PIgsW(`mgvlw%s_Mys(1>uVH;TnrsI3xy{fd@drV|lA| zr~h~J{kwKzxu*7gvKN4wEAcinKMv_fp0RFYq81uwv>2)MoXUxlNPRBR4;peK{zC{B zByBz^+IJMjtYPI5M%z4@oY#>qaMZ-c(OedKJ7h^c$8e!M(dV%NzH$1?(EVEgW<_^1 zA4=TZB4LzS7h>c*5p@AK_EVqT>P5xdET-*(XwS{J#qLg>9{>vprTl{0cse<#SLSIv zqNLjL2+6&whbteA(w58mc37vK#nh>d9{1pH+_RI;{CBJxtG8TV{jcq^MNJGew#fZe zH%Lo7xA$0UEN*uw_Y@Ys)sa;CV>g16^ZRC{EozNZlh+W;DuoR`xPBsU>`C_wv7yT{MHo5<$ul-SOxp49F{ zUPYAM6OQ%1wKZW@%s+`XAX(~vf+GuLpwlsxxe?i!0yIsFH;Fs=vaXua_^V~=Qd*hv z=MQO<-RFXZI&O6l+r!e^KQLQO41NsBiMe<|#8REUA+j&Ju9-5~h8>oE%TpIkZAa>o zamuYaHQBzNxE;s)qcKp9p}1Z+KF-eyZKxEP%vcN8!5dLZ_B&;OeNNU?1valcqbBf& z-bqgIF2{Z_lML)Pso}iCBcF!JOr;sTq-!`a|eZm-$Pqb`3pwK;~3VVR&BqzPo~QdjZz z%kqE_VALjK6Kr)HWA%7MJAh%>ac~A2IPQwuUp1B%fU!l7x60Y3UP5<0fxQ_6P3Tyw z{<@REMYqxMFwBSei6bLBqIk0>qOkkqJ_gX=YUuKmcjv_~G_k=F%js=DchM(0A zTib}uU5ts1B{mt$Iin64t1HNxVjbj(Kv(=x^jIf#yo;_SxN44S?=_t^M<5{|dIE;z_xMlZ^$Z<48u#*##GI8dUIzAR6} zlym;fhmgXh{bB0J7qAYFf# z&9hOHkW6zC%`DCsLHj~~29HtVqMPc$Wu_*HI0;brYm4v z?A|tJK2?qiklV3{wAN97rMUgX@CW-Mbg)puLJreb*T?o|lg-{`%3xUds}W9t527-W zjy7z_04GMW6bD6wMZeoOJ=P+m%I*6&#@mLeppE}!@C zgugk_-rSA=MA(*QeNzwvTwWD9Ej&ed-ne1pO0cnBPqi9 zh)|lyK^SG*{s`jM4H_`QPhCTRxpcpAv+l%Xr&u^$Lr{FtEj>JC=lE?uQn!vN@GYsP z7wk&kykWV}8JE$a2{-WFw|IHSp>8GOiLTIvEUcp)ro9ICwcX5N$SK=8(~)V zr(?<(=WjWXcKZbIZJHS(tuv8Od*Ht^)2Om-L80aJJgpYhbw5hSo*wDi*eOL`UGUb& z8DhkXPs<>se0T2nRK?E-O0yw!8LPB-)5xJCLIy@8cNLxx6$neJbU1 ziGDV?Bb*5AcCI6qr9k?1*AOhX(Z&c4wPZR}t7s+F(t52w8ctn}t~xir%3^suvhDbe zZT;6F(oF)vJO0|Wa~Vdw^4=6+*(|!dv{CtSJ{UcGL2fLc!bx%EVJqVvohz9UNt+So zi0g`Yy&-nh6I6^RcAtGe^e3f1sc0XkrzJEaln8KVk!&lQHG7>-VsoOYaO{j`g$n3j zaF1PifRN;*PKf5}a^!xb`|H9g4cxqQZWJ&&{M3ajJ66v}wnzYW;I1S2Dxu3BF>rMM zeY4qm*Gv(s#^v_|c#B6r&;C@%N^>G;g=P#_R>lJ=DB)D%T8?D=QDQvOvxJF7 z-&OM&{jKHSp{+o4T2SiPbt60_`$uPUj$rH~sg~O3Q;kr24L++%?qn?X0OQu5oL7!H z-{vy6*Sx5B~X6t|G#5#Bc;#eS$7)8k-`pU~zxg zvx=TrDdjeXBxhcioET1Z7vRtd?RB}=NeJ3Ke0^q4n!4U9|MjX&%Nu-JO{M1QE~)#Vn3F(@<&sIj}# zC-X`#b}uho?8zMzDhL(52Q0KGYN%WFsdkJZUZCZbUr`M};NH77c)D)#scDp=wAppH zH_vr*A>?R;=~>iw$mXf`soaSjURfbrOWtZfx`DMBNmkH78cK;!4Q>$nU}Q#7Z?ZSi z<6DC`N?fhp99?apV1UN$oKYcY2C1?Q=lFKGW#Xd8A>*k^`yCncn2*mDv8i}dV*?e*u2C=yw!xG*ghF=1xU zCk^(;J)` zquqyzDh2QZsmV4Dip*!x;F1%mbC0d^L)dLze>&o9Y|k)Gg=#vuji{tYjgL)0v9~J0*lI46>Q9AK_>h%-+cg^tTBd{(x3PmzqF>m3e?tPAeSiZIp)>_b^a=&NH z{BkVc6Wd}DK~C0OA~nnK9szn zn7NZQFGoaqc3G*s(Avx`XsZtmSEZ*s)!N&$vlY2+N>aD=Vwq}LiVSLKaX=G|K`_;~ zal;XyO$W#f$MPz&Q*HFmj3Cxzu%Ar+DT#G)(LW!qwbC=2ASAE7K=JmdFZ6LtdZa4P z{gPfTv$SHBO6ojeHS)Fc19YT#RPinR>xO4L%&)XNw}g>w@UIMzrZcCjg(I{gnH238 zm&y+>x214?-s0Z-HRNz1Tx0M08w;R(V-*RD0iiZ}5}<`a=d->Gp1_pj1;gqmVu%1K3? zJ4)F!KD?7Kc1b9cQ{eY>;YG^PJXpsM=~rb8+-RBZMXu%L@N{xXUU2NmRg_t(Xdf+< zf0ehihjH1(JMn!_oHF#}9VKc$`7+_Kw$(v&7T9J}=`mZIx7!4wwVjfyLyh3zSU5?* zp4o>3eEqDzolNm%RYQ*>&G;MclQ2+Yvi{rhkRvIa*my7G*EH=fr!1KO%NZ(6$bj(6 zG~Dk%o-(rdSGU>?%Bgoxv+rEPfaSWx(n^4{Lp>iQ{vBpzeX~m0D9&|;LZ2NMG;7Q_ z4IP?6fnqX2j2ow?5_?uYGXXA2ZGxew0HA`>fG_dHt(0bnxVdcZ{-1Yhaw^I(`!q1X z>E>+>Tl0Lzd(Z00IlObV!&AKHR7OzmJm^PbNZX{XXfF%;pB4ClA{2t-oKN2~#CFTm^KXj!3dX$R%KNksvl9OGb{@2$+o<>2z zi@U=8_r+2EpW?+xWQCxSjsI-q|2{yYeEx6S`ThT~Hv%v(+nLh-wRgzV23Ht*q9-3g z^)FOS_tiaz|APOUPr#|rsH{wb@X2@-qL?MRv;8_ z-+%8*VP!fe4lg%??)kszARCNeERD+l(%JLjC{x)1#Klmj20;C_SF0 z5es|}>o)keK?b6NBY{&vP|;NQDaz{`y?h( z7Xn5R{FmiMxPmg5O;2Y-c}4zb&%jPUG~gSln##i8z<9z4agwXYTYLSpO|`)ojQ{4l zWeoJEz4Hi=4ubA>wuGH9JTRIte4$98BOmJ4(NJQ<<-{thj_%WRsQZ0%Q~3S=xUl~3 zA3*>WHRck0xx$e!&o~l16`z0x{%=#wg*SOumhVEZiq~a5ijkg1)poK-wzR4_KKMjVW1#y(h@(!t3?e0 F{|6GMn8N@7 diff --git a/doc/v1_api_tutorials/quick_start/src/NetRNN_cn.jpg b/doc/v1_api_tutorials/quick_start/src/NetRNN_cn.jpg deleted file mode 100755 index f8bc081827f3a50af80495b4561813c5efe9c9dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46358 zcmeFZ2Ut_v_AeSlKtVc42bHE&kt!`46%g2f)X)(D=^(um5h+roD=pFlq(pirBE2J3 zdha#VKtl4C`<#2Wd!KXe+3&Xh_r7j=j88yBNJRAGzH*i1>Xj?RSBQv+Nr{O`t^pU( zRWfqYYh*v3e+=@Y`;T70&$TN=SAMkke|O=&fGDrv`CSUb$GZi(M2Uw_iHB$zpcM1P80UC@1V<+1XMSLd$@H3ZJSBbCF(K9e|a&h0j!y_UpCN6Pb^5G+S1;xio%39hw zx)42m19J<@=T_D>wytjO9-dy_K0&X8LqfyegvZ4vBqk-ld!Le(os;`9FTdbZS$Rce zRdr2m-Pe}Zw)T$BuI{1Xk?i%ZKZt842Uo5+L1qvMlP)Y>rH%&etU98a^H{dH9qdFzDjpJ~(g@735H8>yuHq zcwYdf*BJKhSU>hbQ7eeh3xCk97=hB1gj`&1P{o0Q-XPlJ=R+|OjIfK4JsjxByQI&D z_}qIW&Lg7qaHK%tiE4Hf4iwCu3msHxokK~aVyyr8(XK2jdrEuPX8Y#%CFsKA4bvkk zq&Sw03kPBZ|MBB?xuUI1rP{(@XnDen9~83gJQhNr_v#9JzKlHUWZsJtifw1uevTXb z{Hj+f4n#x^jbi)bM-N}M>G^qN3Ij0#2L-(amE^abPoCsvWE~7R5V4{%4)h`U4PxI} z9QEx_pU15}Z_fl(OY}?hkwq~u1sX+0Mq2p%<=&S6pbzm!g0w65!~JAY@cH{9J1RzC z$1&Ym_nHyQ_bg&UycKD#yS$Ah=52@rZCPd<==&GMIsd0@O&3+ABWSEZf=-GtWHlz| z%a^Qh-c@zK+gahD!@Wyh0#j|W3?b(S{&IQmVa|q0_@V6Qx*ZsvVkvBZC>w^@$MiMZ z!3}*Js8J9bX**Lp>QEG?#kc?UCSK0;y!X>1ZW#h^dTIWvW(#-POuBb6LIf%GA!#?c zEoPE?n}Mm-$%78GXX8MD>WV)NI^@J)!;>4G_Y|}SwtjeFyE0oynD*>2=P~A#vLmVe zL_EM~X%&W96Lf>P#rGP0bRKtN_QHnUwrHmMq zj)5lzV+%&n95-GR+N+>7B)=$%Eo*SRS5(BH=S%sZ*(_sF?zrf|?60fP z1s$8YLu5Vs@JxyJ7pf0&1GTj4;QYG`cp1`d=-ymja4t?aAvZt<8F zYVJGoh!xYf5Hc)lL>ri*GR*N~1EqmHUevSnx1>D-_|c=&5dKUYs7o11e&Je)0}UEP z+M9E$-Giu^qV1rWUb>5)lm+?#t{ub4f9LuL`t{Q=Vr@wS2l|p%lH*3p(l+gpdpSDH z!r@Kr%jwymS@usLF z)kzlh)>m7RG$_8p{5n=%5|v-rReAdE=NEIgu~Jm;*KM4N#m~Jlz~1ghCz(}bcwu-E zC6Gl^Oi;xf`sx%8B&=_KH^EFn^R?#lnqO7*bt5ggh9yf9G~|8q@VfleBWJ3j?3(nJ z>PM*VNLTZ;J(jizaOVa{mcW5}{C`uq)Fm9~?24Bs_G>AW-PquWcC|fNvU1b+@$48T z_WPE0F#T;Qr?Y>{=eU-<)A2R%0lQ%FIM%~u?swV!X?3`6jlGb7)Vvf*kc(Q9HCq-e zEwFc|7i<{dvR*kzYhTb~iIfbA4xP;2^fH`EB;S&VFnynf1N}Sk?zERX*!j~I#eYqv zUbFahUU=SO-hG|az;hz{z3kB#$+eSL`EA#-b2?v6EgOBjF z16avX*&+kyr=~0V4tfavrGuz|KP7qqO+}uD8h8*eNS1}~#N*LwI z;iAxHB1Ta~&)Vv9QZnj8ya2XQKNAO5Ibbfy4soDIl?=sWWQFS6Jvflkvz|DwIwc%P z1bUR?z1BiLq27Y!MY%3qXzuAE1M5dBdhBX@eMY)dm*41r7XH#Dt$T_QN)AS>^DhZv zSXHs2;}{YgD2{i54UP952U>q!(%w+SMz#{y@>MgFsiq`DG4IVlIjw=MXii-G)~XjE z@}IklKwB8R46(0f@8Lkr%X3R6&GK$IP@%pl1{E`Hc7b>g9XRvfM&dvyb3qd5pkccy zO8Pf%8gg&H`huXr67a@di1tQS9H`yLBX!Y`<(BW1b)~BEeang}PZG}Vw(8#7TaEzsx1-`K3nJs69NbNwpuT` z&bz^bQ9s}I(gue#UivsZB7*_Fl#7)k%3RGXuxFO{ClSKamFGCWHugNP3sw$okFqiv z#*(3=B%^B=!PH)Ahi#B#Pb_zx{xPqV)3i4QKyxAIl8MozS&l;37p|K_97HvA}}<-k8&f43y90Q*fbn)cmB z0Szx{dM8xiw6{LNDT+V69Lwc*X)O8V5wbEWH0UUuz4i25XF6$8EuzF^tJ=jxRaouG zruc)I6)oQKD;YOwEHHWW9md}KwgYW!fR_x$K>Qa-HSN@|qFnVt)kluE){h}WdyWD{ z5B%R}ta=9al^MMne;bXd$5(t>vi?ENeBz;S#7U?VF%J8e#Wx6tTC~tay?6a+9=T?FT)N5skGn+ z^03Jhm~8e!jev=v6y8FgSkI#o!mROl(FTnOyI5uw$Dn6SK_bMLBM8^;Ruu`B9_`S$ zXVv}ssy2oW+VuqGiUXNn0<3dYNyeXlQ6;oJs3rL>Tr z+SimVfZph|^K>_IVlT%_3xWi&J+@HQ!=mu=yJ;;r%E*(Wl{$t>Y-Sm1hWLhN$Di;r zx}rO2C*Gx%Q^$63WF%xz4z7L!{Ob-?2e0vr^uk;7o24Tj(z+G-{la ze0IkK@L(k#h;tH%mQhtGkY4b%f{**h0sm_&1Z5FKLU+nUS}?TGuZW}kbmb%30YAAq zv2%twJ12L?q4pAv*ibHce2}1QqoVRUgMwKi&R zBggnrcrzbJ6gSTsstSlEjp2!oxcX;GZ@hE#Gfx6#yx%039%WxJtVVefzLSPm@)q7D zJ%I0d#%7>cr^ny(&}uoVm_VwF)KH^2LmED=iqQ9$Q(o)F%?z-@oU76D4jgYyJ4veM zUoOc6(q_*`q`v19d};myIoHSNeJI|YS;KHM=2DTd*S&$x#IktGd2nq;r!k#@={jxt%xmEjFXj{)A;yU3@c_-MD64}L_GP_@r?3}4^vU2l!kRTnER@=ck%s&#q;Js=&1ZgV?f`rM7}5>bGqx-lb;{px`O z^^OOCQB)G2P4_4yKi$r%S947Of=JA3X&E54(L=%S^%E z=Jv;+NAHQ^)YM$lMt4kGs>kZaC*3+1uZ+Pg@htRym3TJ6pkU-WOoIavpd7bUqK0;F zvq#g9@jffN$`R?%;y0ym&wH^!S%jy9($jwih%k43Mhk}%nRxAsaUfFjm@)X|6lC1t zrkL&XO1tNG^~#D{2x7y@(-!Iu`#mTF!#C)ZDp+p}=9Ue|vAj}9lCKcTznHt6cI{A? zD8#X%@!1+$^jXy_U%j0%;|Z_N{k`8idgsI$c9mpLAYUIU!c_gm?JrF<)Q-N36;b3^ z8?AP^x@$NVuBX)2)nn}Iu;)hmt2U#uz@+Eg%%~Nr`QUezhqcehXMuIzca3CO_UNS}F%|h2hipot-dFJMmv2lI(22Sh1i!s=Q z2rxc)IUJ>k{o(;WzsXx#A~kP?i8a0QBQXL3^+%i@q9ynjV=xqCIT8u&D8O=`1^f9T z38jivmP+cWz;ABwBGnbELk{>KI;4M=R82h#kV__rSP9(?Pdf;BM*dxAX|iKGHg^jL zqHt%UG`36GuW3NaCx_p9HM`v6bgimt@R!(o7KMf0_n+Sox~5G*;LR)_SR9_GCZe-F zb#(Of zcF86>nRKg*rKwL45kshqO>HxqT&TsyesI?tjs%q#D~KtJY9kK{D{fC_5tr_vU;`J& zgF+%tUr_>*da>=)2m=|1$qpT(?(pW|s`33EsJ zX?&dduCj1J*D|)h=~I>GJFh)rsPoxbfIW_FyEBF25fyJmZo+Mz)Hi~3-!o|IDjN4! zBF89toXj{YoE&J9_Uw`E+B90!!Csw4q(zyb5Af(qm=>yEV&vG`;Y_Ghlgc5!;!0mt z@n1i`8ajqwi!~4(39;g*@g)%*n|WvN{b*MzL`LKqh`0V$&zf9+&e5u4_n3$3RB^GL zJz)H+Cc09WG7aO>WV?+WOBUfnr;VqMJ&)oZ2+pu68{CFxNo#dri2+n<*v|q7f+_%h zTMYix}H!5Nqhj5_bCqVSo zSl;z>UiRbpU*u*i1TO-phB4;z^v5zX7=lHT{mJyvZgu=t_E?L=cjnwOLI!jDaj1Ii zo9b}z-m>YEsj5h7T^xG+EQ6NYnqwIKepth30K?HFfCFVYEykeF=zd_lR|O*w>#3-0 zELI*X%A8@hXS!;7?+^_;V-cn81;RX)#l3Sb>}?zf5&I)o{_O>bzr8?_I|Ud$^nawa zn2-YRdmv&-Zksz&#Q^vm^ww`+I49Ecf*MK#o$ExjN2`mT!Vk$T*G|C)lJ@_Jb`ckE zcq8)ju_d2rkYB-1fVV1C<6sj|{aC3miXD@}Xi_8nPt#Cg56%6M>zN z>c`km#WQVf!yTnsHbj{NK&D)hQlS+eH*sv-D9`Q7k+GG%+Ys~7`U5inOApxpfTdTk z6wuBCT&OU=MEuwN>+oy?{>tRA-n$CDI;5{174$N4%V+asnUGx2Wg9SxF9u`R z^L4aPIl-no{u^flRG+=6;f{8pWcVkgsuP$g+HG^aL<>uapn?t#&HZR@4G7e!6?P<2 zjU{0R#5uPGBmyu^G@yH3@BK|+n2VShF!q=K)J?V;e0D6@#D;nIf4+|j_>3p{9|7Q4 z;5@of&=B^daRfUj@;OZLT(a#YnJ;rd;!t$LzD8V-t8Tqho~a7FHS%H1Jq^z?6}bE= zZI3ndK(pZ?4rH_gJ-S}L#!L0g}CJ3?@Vo;m*FcDsV=R}Dxx zVt(9I^YdZ2R%Cif@%UNRxetr-5e)=zF<0i^fY<*v8-EtBsUZw=o;*j6{S^uUFch%) zsX7kyRklvvy@WBfj*}r z4n<)5QK)!qND=_fSwNRGjJ>EKBOj~L&@kC?(uN*z;>qP0PBq0FV-n$&Ji{-vClP5- zC1K^exoTPYo7_%!0!kIl^8jLitJjs z>hcSFCi_?c8!0MY+AC)X+YDG3?{2L;U6&^=YEu#?Dus$h=4x(BbfVYcb;;1%<%LL>*afL}gTfzFqK0d(yr^|w?1U9f_M!+`3SqJeqoWHF}ghJF=aR6l1l1VXG>90;&u zKQpR0wuS@A=AO3TK=I`Nr5%|9fOY$p^z_2F3qT|pbN3IikNSkMjpJv!ezVj_f?jgw z46f<8JGyKWvDbo{S?oMpQf};HLmezQs3N>teNT2~0>>Fd3TpeSr9UjjRaNZ;LVm<({-A5W?BZL4qsW5;8g=eE7)Hu*D?y~2mmgyGF&X1_P!<&$lS?A39iDzQzj{DvrucNm{_Z( z>rtnf8rPT#Dc|k)o2}%jDg2gvPvf;C{Q1(Aw#f}CWB12^yE-8k`QfyFzImil6Dq>G z*mIoodxY;kLA=QqO@B^G+`MaLuMmVZ|4!~|VhrHDGhQk44Sl6V(_ze6&^M=|m5lca z;178!wJE1({nUnvne$VWI`^okz`}t#bBnVX@A`lo27J^9MtwTmRFf&J-_*wRD~7{j z*TMVMKxy=N1iUv1MQTI8&H|OZ@h3nWVuJ&&+SnBt*VH%DTo6u@`_vqgDJv zXKyUghiv>ojbp|Dv*VNu#Sedy6%bwhvAYR0aQ*g|BN~4N*gOB{d9mw%d;Px^^}G-9 zU4V3re@FW7)AV0L`tPs*XD#_>(k}q1SN=olzxVx1r2o|Ue~Hw8fBjz}{f`Rsd*8oA z`cIwz3hB_F_3d9N%5{LhpAli+}L@*hb5gV=y&`V-%O zS?UgdU*-Rb^rL@R2|()eK#a3C_1{e5IP9O4U4Rt)-;~{$ZNEkYsLlV~#<>408xsV? zudu(rolX(+bMJpa{2$-`4KOl)=>IRM|Cs={zd-)a()-V%|HHKZZ?KTR&-{N>co{?+ zslao*j78>CuLGiy0@232%_p00uRcw>pABw}iZv0{6>)gJCRa~w__%wb=MJ}h(6`Ws zm`$rd-BZ?&Z`-6Mt;#g?1Fj?aWI2Z)eR~<1O~vH`%<~gp)+0zN$Knoq**#6jn+`wS zQG-25bSLG?)Q#u(^`Eg3k^ZW#w0TRR_q?i~4A?w~`Jka6n(OO!D^b+ECx1`u%i86@ z^la`yye{qyeH>^%!G`UK7Kk^P06)oi0H#9S0o#HnRa#le#~lTU0-MLdB6SM%4;nRO=XRG7l`IoCAGx={w&Fn7ffGE?JqpxN!#Jq%F#y|>f#NPS|gilmT={qvO?I*{#qvJ%N zRteI9kwxZd!N0UPRK%pG>eqtDr3X?c&^*%Pm5n*4!`So_n{?C7CgYxgolleFsGC)Y zqr*DuqdkX1>rG!yg&1)s8>!uK4En&biLNIRY`0^3Fgv~+T&0c;@UY$mqUPH}(Em1jY=Fjr zkFF0P_W8l>4XKY&_xGhBRo|Qt3y>)T8%E4Hg~2ABPz200GG;kyO{$&eh=tO9%Pxr&c zJtDjF$dvxfc~Z-Yn#wzBmWWCV{Ttp_mpkz+xd#PYT-1vbkP2KS_voeFY00f&eA(Qb zUWGLkOV`umZ3^?Od~KlO_ZAvNr8$sZJY~u3M&A-NZN#!tY&BK6hHNG_i!!Q?m2YP? z4W5Cr!f*eV-5(C+Y49rdPjwvxTiS!ESfcMA0>N7NbEg7D?N zYq~X7)jM~;_J6bMtzQD4F}ahU@8CdX;eEw|3_wCYl@A?vx=7G<@YF&9-;I8gN5aZd~Q{Gq7tJA4N(tL199Aq*Xf(#9%C+wK>cp+**aKQy~l@egx0?o_;{TM8(Us)9mmb z?`wQ*#O`T5{L02@26x4nMI~gE&*6D_bX`78Fgt4xGK{Sbxy%8;P^ z&FUnNF4F$m7uB9+ckb6Dnnlx^$B!t|*tHYVK=3=MTm8)%jn$x-_(Z=wL`k)$m2!Vd z<~9vpmEU&sr8Rz0Z8M&E?B4(6Zik4zJl8Q-q^<4q|jy=@(LY2ceC1OAjIC$e80c; zd6}M(k&3zdiFXjidR!ZOKN8Wc+_`41&~tD^cSIfrC2J5za<}^UY}UEwy*Yw&u{`kI z`Z9ixmpdi4D6anTxBRY2S)FGsctN1K*FGIB&@i9iwXg=hIwOO<7;M%gZ z4GYw40*;qTJL7r8SqWShUU+#e$yD&jcQw|=*(IwO^BYg8%5E8p&QXT7N;76lD9H)* z52R{u&xUu2j9cY3=Dlg2?}1&hWt=;C9+PQ%rx2BdZZQygHR+y2$H=$ivtO^=dzQhQ z-`#}i9%3%JD`R>B8P5AwxyPb6^WfxEH{QBbmHJ|R#*I{h!oT@)0PVUn=HC91WPg0V zV)h2oc>S(SjV!B862m+bX$VfM}Z^^yLVJj3r$dt?8q{Rda5 zl{K_Hu1cI~{DKbx-IZLkCzIoo@fy+8RYk-{7$Z`2jpI~4lzI=rm7mzmv+?0GD|!*lv0x3&&EQ06>sVz?)-WE6NcmxCsD}_++U! zMZoK~*xc2zzieBVNMR`u4A8-I1WFy)9PtA}pj#3=XM+IX4;qT)fsGde`-nxb=TOXN z+02L7S2s_!fr=$rd-xJ!fdrX+K@R-=^#$U$eEVhqQ0>n=hVH|F-5HDd=f}`6JV^2Ge9<)0~LGXW5tUxv{eIMBhD^I-DxE1@%50Olvf$Q%7n zZ2|Usequj+3y{P*Bh(*^)nj{-3HW#l^e5;6k-5C{+@G8NiFbb=#09?k9snNn;BV6W zbs@#T0dak>xi|t+ho}AzLN6{q+%5QlsYcOu{7IfMs0SEeJL5xOLKeVI07j$n^W2c< z1M%yJF#l2T8GdP2&E7vX1|mt*NW{9oDTW*H6{t_Lji%VZyi*{>SqVHQW;>!JuLG#R z;UNNH`Eyh7pQr2p&g_3y|Nl?S?8A8k{SsF39|qvxtpH1T@0eCzD5Dqe*R>}x?WIOF z^%YeI%Y*!1V?}$P#J5-l-UMqH7SF>O2*F5`paHFg9~R(qh=ZmMr;^!PZ=92XuM}Wi z1f|+8+TlDO3)&0eWV(|E+B5oy(n*!Qzp^@uvA=6+NSu%QbOW3zh$kq!=)*^ga=>PQa{SOW2r_t$E7)QW(TSV z8p=WzRkKNImvkjqLR58`ST{%AtW=wG{f*pBPquSW?;*`RZi&{~T9B%612Ec02t)c~ zV*&o_{|4d519;xsT-b$&`s(fMpPozjOtuEMTI?@WW`%*ec{O0AOXkqGizbJVFicck z=moI5YC`^bM&hXYf=SpKto1@&_#y+byoZ(YIWqyG`Jt<#~i@ ziz$hc&L4+W^BNp8`eYb(&RDZv$sNI|aG;QKq~ZmoQyBPk25{H_m?a7e2d_u?$>Bgv zI|x+WFxn&(s3XkR!7y*wCiG%}JaxVkdZ0+X#30H~ev*c)YRt19yFE5(l1i3p|2fAe zDc4!#JW@#dp5)`$ie%gMZRBUP6vI`#yS{?2VU(4sDvJ#%kT8r~v8u$lsGTaWO_iZk zOqm1u?uCVz2HT5~kY&%W-|Use}sD8}Mh@Rjayt7Hmu=_I|eYj3ub_2)~-88Jk$GS(H;)GQP6>LLzk_5@L`z z0CU~vT@-CaD__k)K;CP4z2&J*b;);E05M+*S0FCo-p7GD{MEm@=A=NI1>fAI-!U(H zgm#fux(0Q=B+~Ya$;+4I-0|@SX1Quff?Fi-70dFpDw8t$C;4fM%)_!ZOV}o4A%c<# zJB_o$O>kqU!E?G>kukPNL6jYsI_4~Ejsm(@?}QbI0Y1|}zg_Pf|;kBLDr9gbtEcug-jX`Qq#J~cS|+L{f?5_{Di z-;vXU0=tiji*@QnVrmu?113&YohTK!_s5pbEo&t+T?W@o!bU5Qh~zLs_BAso;xXw}A@@ z!*O6Eh2Mo;FA%`86VJ^9QC+UKjO5&bKsFGRCkp+Y2i{@nFaHa<(2xh2`#k`TF8MB! zqY0*gI6658<&Fb!pj&`6=2|6i*oATk^P_?Hzi%-94zUcBtR0~{y~lKz2p9_JI&2$$ z8?MT#37Ff!U>c2IsBGpAXvzUA+J$MO`RSRqEn>!p#mfO!;M)#85Zh!s29gp>+EfFR zKib{RynXm1SM|k!%<$WNv=$*hYI{S#jJ5$|696|MQ7%1e!f($he<;xY&C*B47=Uhn zNYjYj1fZhG7!3u*{12DH@5#tBu>Lsx(|Jr|;R6(Qv)M;9Vz+<+pZ~GS`}aW>|8Y?0 z&(jHQ3aDHLSHNR9_ANgdZXQfFFI?y@srjmmbzdZ{lC0;X+&7dCeaS#EZ@zs1cOUTI zUo#%@(^|;qMDz6rUl0z{Q>w}&+sLvCpr$QB-)NEYp{Z9`Wb%cGsThFB{ zTrC-M9X*pO^i*3LTs%0hI7p#gafIBNIIRnb=+mDYoCA*gt`B7V*eLhkzdsy(xaJgQ zdY~>182+dAunVdq7#U(Y0|mu4Cn3&nNv>kSUS)e7YW2Q4T9ha5@eu*lBfutdxOQu)kydtQ)0w#whTU7l+dmH2EUrkS2NgVVd>f{2k znJ_*MBfrI78>syS#3aDD0UqGAe}r(i@FySP0S;wcn5!xBV|E-(k?z%2u86F_q40AN zzkLmP8W3>faRD0xZ>oULw_&{+3O_v?={1gLj)zQ~CpQ2n7`r2|a*Pot8v1#;HeLf* zg_(!gdgyjmx!ot&*G9@d@SF3Dhf7hOZBT|L@o9|oesq-|Z=SU-#U|^makJN|ULXD5 z@jspUDxBAEy64)=2?5G!OdBe%?m*p{X{ zKwyYXSJh9(gwK+(_-qHV&LaRkI76p|B%MtbZ`+%sE!lR+s;{B29h%P3cx=Q@7v7g( z)gAvzj>)@C_$#=4H+88n;$bdE{UUp%yr}|h4;kK!GcZh)QZ9L3p)lRu{E{S+aE+aC zyn|E<4PE@06We#$L>K-+KmR=mQ*ESV()umavKb<%z^?dHv!|td&nG__fln+)nc5I^ zT3==ky1PqK7X|z(#sjEX(6<{PDJ@ zs~b)oou@oHtwp@81>5lf+nzR34r_?ycYVO9!-B_f%I`6erYz5#Na)+MMKU*wpIHu@2yAG6$x&3 z(bWvGVG$-7-_|;ZX&020PI=|1sLX@zKK>oY-QdyV3bD60y_UoEsMW*xUh)-GEjxlf zYwcn|L+~Q74XFpe=8~plBO;Vpqd0G_@#|ZMLjLsSJ(QhnT51)kRscg|7mDLUF2sl~{e-q7a?2Y*k(>^V! zVXAZy`U(p`4S_0{0dlk*1EvRn`C`sk>HN`*n@Jw+#^g(EZS^wBvQPaXx}h4Cx-29u zD>uxx4&9E|?Xd%F)S^(9b%XxoMQeTsjQ;$NvMQ4<`fe*7*$0}MNDqCQHD;Hqpi8`y zNK>#M5n$xckO*x1Z{B?>Uh6#CAtWz2UgOgWgW~%dgo~zH--MFHZgI9vT5TmHsz->r zHD^UtK_i(*(**NubV#+W!2UdE99H!0F*!{wYZgKWEc7SS`jhnq9Tge~WVn-L{5X@mjOYwSGtObXfv$)JrHdE` z97sADduF;{_(}XWJICEliFVsNdC}9P2Z{_}Pv78~v%#gyB{+~c_Gzwo*4#H}tSOpx z>hJv(PeiN0qrVNM3AHu!4tuqD=^#D-(|nR_B41h9Gb@(|xB1kBUlNIMDi!8cog)rO zMpU_~r{uBnPTRgtXUmAwZeVj1+xL^X!@o07&NAYXW2r}ZZ&$6Y9j}wchw^MhbF-?d zEFoocl8!}ZSqB^|0$1X#Mj;LvPogFSo1w96CoW=sJT;A{P zdezJOPu^aynzX-TVPiC_pt`hK^fOGuYuRt_EyeZ&0L9Fs# zrxBv+v>12PLV*|g*OcWZif?{`4wtbv#vIJ`1#{z^>!kF=omr{lmeP=&3ypzwjECTf z9B8|$pBu2HZH5h~VN5m&Ms8^@x0so6(~lPP@Jfl(UhN(?D6GlZDTY7yhhEY8m>1UC z&7+w|(!L>*n#S%c!;+8+8A4|{qiN%^!E{)OI3jK=mDd@7!_h!nXrZGfs1-om(x#Ya zjdej^;BWO=NzP2M1cKPd=dhh3t3yEyA$ZwuZc*^1q+oF6l(E>>;@l3^cyj!en|h*m z*7TXo?@w-6Xutf_`u2qw?4bqNCC9&0f^R%+UMw6zljBDcR>$s?z7?`)F z67x67w1T! z@Su;|Yu$|-cAv0^6H-_fx6?agg<2##qdM+%+`AnE=tleG{tLb>uZ|PwzF2XA3ChkG zRft^DNEz&sKB1mg@eAk2yl1p0;ns{;d+=4R-H@UCGm{A$3NOp6#i*)u7%Pl%t4ldl zXUqa)IPz)DI6d-UcG%ivMTcRkHKT|H*76aj3Payw6J1jQ zi~91W6sqUM-#$9An-LxZpR@0M zMlbR7G+w=&{S=w9(BjKeGp;&R<{)5sXFJ{w!tLG&(EB1 zPSkG{sIxbl!LaWaCx`7GO~{I1d?3TcO%wiZ=ZA{sW`xW|Wz&@(f;AedL3{T*m|YEK z=`T{H!Apjz`4y%y;Oob4?5i$v`5LU8qzrfaXBNz)Tq~AjUikC&HF4RmUz@coa)1 zj59vDIBs|=D<4L*@|M?Q7c9B)MLE$=Ia|_0P1R3_x@>LKL>EewalsFly{A(!=x#I8 zn3Y?>@3}o?AaE>QwdMOx@#M}LR{69;5OZe{fZu(BG%lh=eVNR~L!0hlbgM?qFSkzi z7hcb`f}PutC!{7LG33gp8bI6x+m>dN=UmO>7|(7Tmi7JNG`h~)`N2ANqMKfb?LDBX zh>Bg1Q6Q%csR_e%GYoqa9LZc?GpY66QM#jl)k0Y zTU;}tYrDEZJeIzY?GVy*AG)AUhXc{W9a(O}!>}#QP~IP3mI9m@U^Bm{jQsHh1ur`= zF_ZwL0tB$18j*lwCiX(0+Mam@o!AWf2`{QjDc7Nzku#2>^o{~L9dDjdOf_FBR1zt9 zdP$=F$hZTQM)sb4J$|TxX;JaD`8d>u&qxe*S@@w14_Ef0=+%OmM|zX(d^!ysA+ryL zuY7K@@9;P#ew4-#Z5WcNb}x@`D$bM!IFk&R&zjBVRj-s`TUX^SL&Mg;l%&h$&@->_|KNZY?7hOaEVRjjUwX!Qxa$QB(S9$0cv>z-Uw z8a^f!o47TMS>PMqWVKbS8%dUM5~_%LVhj?3(hO`$S&xO-CH z#;s`AbFtKGQk3(7D1;9pT2%c_+{tRq*G;W&h8Y%wu$%#Rq)5$eAm{_68XG)4bsu#t zK8ku>bxLK}eb?dqD*IFN2X}qP3NFGCo4<$Gx(G|btES`x5r>QZjw*Ie6kF>jgi=Bd zZqQ06I^V=oELv;pqq5Lx??C;FC~ znaGHt=`|nleg-G-bs9Q2kdE(CGL{AjMTP_7DSZ@39*+!Ka_PRW#T~X7D|c#o!@}1p z<&;r@nQ;gE^=~?si?tH`M~E*?J>L=Or@DX$XFP2&Wi&P1GH8mhyk45SrSbKKgQ=S5 zL1bLJgg$#bu{zRkI!;{t6WOZ#1yXO4>)U@O`bd;fRhVBELp#M(_G(V|vuGVq8urpkMK<3Mll%hDw5}&zjk|NBqH3X`rG-$Y`H4sPTqR-T^O2-xH`QCFl(uDx z`6Mb=`AY;yD;v7_#^aoZF<(;IC)HD?(i3SdxXqB)?Fd2&-b6`MLtmigU>#tizs}Iip4!k4C+R+6>&z zI@NG9-sCw6sm`dEr`UGMD0=dAcEDlMs%pb$G`gZ(G;yYKa;ke!*(}k5@^Q^{R7Kj| z0UmD#)=I}3)1YKKD8Y6TT>zaOiH&Mel7YvoN#k~y$Lq5K3E@t0hdT%Mni5a@i$2!B zRp6ay897M+QtAZ+;kPxN@W^RjA8ZxP5u6Z3OPYdlMU8x2+!}DAoI1;QNKgpcZn1&P zC3p6_6jsMHrp0*o0RSl*R%H#?G=ruuh8`tae}Jbx&7Rc+PBXglRAVB9Vk&^GIe9mH zCLRl8tJKj)z>a(|0DqAIOasF7B2mOG3!!!borQ*;v8AI;IZw$Zh%m?T*bC^1wtW!R zrwa!PK%jXXXf9xamU3l0xuVKN$1uRf)n89_|D(#hrYP57=Q zDX~{LTvB31y;q{5`r`L}lPK@LM6u=i9pW`+{AdLQXCvAI^kWH6$^4NMSFLTA(;Yi^ z^CF(fjTMLDW+nD2%2E#2+JRiRS4s~BTEhxD^dE!6h@t6n1sl-GuF|y{J;O_n6Qat< zYVI>IFU9IwMRrl&_Kb{ex!EE(RD0VjPfc@cBlj4(>OR)`HRl;(e#)G!rG0hxi}CZT zjvOB9Vph+l9#U33bsip{UQKT8r56d%CS2Y2*0}99d&W$iFi&o@G+QB3EGm*p6G$TQ zYkFT~oDz5}3$c%33{*JFbWcz=ktT9|bzfp2V3aIaz0g+_DFYa-FS&2>j*~SoyiUVN zVT^Q{iYsL_?bKy6E>94FS@V-iK@F^!m;hUUD^r-NVJurbuR+^t`IYZH%ky{64&%qy zc{XkCws@4cJ`WC^y?XPKy6SA+TtPk@118-me88$lIpvnhbYXQPePV0Scce(hYv@JL zxG}_7q*R^qfsXcSNyWGGE$1;$xUbJQ>1K6ZDfQoMxys+H>3fh1 zQM4CN-?`6zSP<-~@j448j5uQ!&0`Q*%DRWqq}v7y@zk39(zUe_Yqa|}-C2hsbp}z@ zSU_FFaUjVx;H*uODM`E9p67+V^AdMYcTq;Vzv53?`mgZ#jDyLQY0!(x=61P6+#gZ z1VTre2%+~L0YQ2%(m{F`2)#t4DM;_V_Zn(|B;LXIJ?Ffq{J!`6Zu{InqLV^q=E*b9 z?7j9{YXgoe*PU(|B4+fmq9&UeJIj0WE;&hwebMboCKV)E zC|zl?!={UAJHt?;3UwcG6oGH{Mna4AUz!rQe}tZu8t`S*@yf5dT?!1?8B+xSf?Ym89=J=#^V|gS$DE4~3{pi7SISmi~A% za!moOW5nmQJe>RxASM1_OK`Y^Q#-BE(HIrGIu9aVJCn$TTN-6)x;!!CF6NpsvWnyQ zzR7?{peDT+yvryon#9@dvfDhlh7n!ty=1m9u&}^MDvMz;KiGJM90xBz?@it-8~bJ= zp`-lZEdxn+2LT0cc;?=a$)K14K!-7L$=EsS78(6kXXbm(G1*hz{o(mar|Hk=?*3OKFtj#bZLQc;xMG{FwD*R#)@IW&epqxP6#OQlhxdo4HWxwL3fTs8Xr*b_Xj z^pT^~y2Nl`d;Ewht%tOmQE4@0FrrA#m$iN2u?8NmcaZ*d%)|bMdjNoAW>z)&s)ys@ zpdp#7EuZ=^YS9ovTl-?|iR$GsYQ_VNSz`YBX}G$Aq$S2cBC=?rDBrbU#;lmq-D@t4 zR+{lP$z&4VvV_4v13?G!R(K#nv@iuMv!|Gm7wURgc&>eq)oc3QqJ{t}keT&Sn0&Am zz>R$_mq;TYx#;M7)9QHVi_6{0j^*%z9?c%(p!`Cfwz|*kOK!fX@tlx7@Udr+Q@AI|4W>Gd zJOOM%&sdWq$&cz2{w{rqSP|9Sk6BqO%bpHO+Q)S(1FL#>m_y|%ALwNU`Gl5K!sOi~ zbS=kSK~~R2bR_8_>weG`P-`jzwYVTp*VQdl(S`*3o+qw84;G&|M3b8Dmgo?j^~k4_ zHsOJH1cU~^5$d$#d^x9)$VEBvOo!7^F9arK#^ks@2rKqfB5kBrDP&Wb;p!bN7`O2F zvF8O?-i+wqk>Dxu%S1cBuC~4_W8x{Vo2D2E`VsCYvsHpmVieUin(KB5-YyM*;7qJ9 z!RJlgf8li|u7FM9r)+5eqV5n6m|Z~h%DmV0X0Oeg_dd#fG!paLkx%Xhz>hvnpIvZm z&<@Qe8L){Z{E;^}d@zmMxtA3`71{9%&+?*XmZAMsAFVk9FxGOyfxYM-vbzuiau3m2 zoa(5-Zz&no;V(R8AeNQ9bq-t(`xUET1OTG+iAew$)xIeRTZtmZ`TgV2z*S8T;C;n& z+!&5s_~~oi-$kwq0o&{$Ik*m>^0AX`Uk4wOku||F3IF|30D`t^>=mUQAbag>Ie!+8 z#c$aKzKq%S7v7})P2`&T-W}X+XCx*+3pauG`-S(c`aN>5q7h)2DL4sTNbck$G68c! zmJ^^TyljBYfZ)960ZH)1DMWf0j9`>3(wg?ENk*tBu zPoe#nb_BTj4Pkv>i3g~(0HDN-#Mg0Il>0JJ2}Vm-@?5HliL3h{mt+$4BzRePb!iUO z2|0nS-Kp@=&dYT%aNTYp%zoGG;!Gki!oVKc+@W#Rr5J(w|Qbwr$G53FHkliQIqk8&*g0y)$C)4`qx_f@&WyFC?n#JD*hd z0o9dOd18(i1>y|u4$I3bX7%g2@d}j^NrQ;cTk~XoA@BOv-Q@MUM1R-{kFQQoAwuM9 z=Nb;^NEW#$;e;gt?cdYOBgET0y3QeMkx84ICxaSh&NM?3#uMQ_dh&*-Z55N6>Tyvm zqHo~c7;N^qq3n%%xvn*EMUNkSc#*zNZ^q{BJMlD*ZiLKa`j6W6{ zg96F#F!=e8MbJqQ=x76!|Gz4R>MH?-bdd1>PBoN$VV82o>n8}oT)haJ<~n)32M)8? z+CEAue5AEtd3&Q@ODNlA$hg(W=zugdb&lP^4Dbn{K(nwgbc|$*&8?qwLWb_Tl*&MO zzOczfPSMfCpGz;|+`9oXNg~<;Lb$M~m$fJK>D#U7c#iej@gKGFu>}!Mc2?@n6;?eH zAhVAP$&7j5?rw?Y3Q6;<_(>7Bvq}ppCLbxw398r0ZJd}U{=#c%mADS?!vf>+`LCY$mYUiLU@Or*3<-IZRKHtNDVNtaUMwI279kwY7*qI$M!C<(|u62%6AI zSuVly=%_Q22?s$NePT|5r70M@jpca5q`!Ce720?0d(RjBOz@1>(za>CPj>~e$FM-` z=iRr_bMS!Y3hek-JnpdIv>`foaikpsEyJZ8Qi5Hslj907TF2w<;<#DSV$vyO;QmHU zkORZBf|V0+FhM3#=}l>&wSH`BZkmhsaXFfHF4S#hQ3hQMH_4g`Bs<1K)WpmCKp7E{ z9C4u>y0YXQ55JK~=-DS@-H&6@)Ie!Kkr)KT^y6A&{`^;mCgp==5@n94hbPpcWc9|n z8#>#jtJ5A=+`bxMaH!UvVS5}1h6~Ps+da+FChq((xT7y>7#_z^(le)&y1P@C^!zA# zW~R$|I~4*xn~lpq!0Kfm<2b?c_47wg*1DKp&>-A0?L7^5P^G8SXuH$Rjfn-xdxu@7 zd1Q!y=N#Jcos+Rp1{=HxJVw8}!@ux826zBCXU17C5bIbY(cNzP$9^4{lO)_ZAN<`E zyw?@ETcP`So@3;CU+aKg3{$A4_nIEYaozpt=%_r1WDB9Auv52l zPJyt%vJ14T2A!&J8?_?X3{3sL_S?L!=8YMej;;$r+y@K?`Wq<}*r(`%9%b?$n|}ZI z+S&zE1z@GTx&(E&C*^S7Qt#(MW&yzR=5dg`tnbG~umQC<>(!ZjnQDWHiJ~&QMv<3p zbeKC&Wa3bm!obA|fE5Kz!=}7G9i0YTJGala!Ao#B1|M<$(hHdlG7CSS&9-&>|~l)jZ(aGc3LyK z!S;xASp?L;>-+wa7|0?U!t?m|ng#VsTS89o8QoQivWe$2fsKzBzpflq$SFy!<2w>B ziWWNa)2wYty%o-p=`K)`8fq~poBOt;QsJFz9lZHP6Vdle^b&~mzV(x@;dzfyI;LS@ z7@03UFRHlY%PbWq17L8tD=*J_NVH?=q;qQcmd)U5pv>2a>Y1` zaFl>HcncQD1$cyY?zmJk+>ie04Dq#4sgU_;{6X4w-=6-ag^t$*=t^Z8jrJx@ZfgLy zV$y@?mzB8Q#;}>T&L`APr!Ri+KuuEpM6{L@yg0g zorqQy{s=HEsoiRV}vE5ThG#JgbCpo z#-6~Xx5`Kinqli)j@7sGDX@|A)rD`H-bKN$_gJ8DxQ>OC_B7W*qR%7D65RL)ihME%~;I)pz<$L z^4KT3R-(mt=bg8M_g>E{CP~DaqU7WoDFpEq%4!BnG|XOk<30;txeR$~U;UG#^KtE0 zs=iskm}>S2MZx3_Kqz6d9zqA|0ZNjQ=_9& zH@)WhW3-@7A4YuP9=vrVQ(?5hwz#`QXMM8z9zPvh8$SKcO_DQX()7_>(yNrv@nvC4 zF(iQ!foxTJwR_d{q?hFerZ=kFp5_xq2j!@usAN(utI4z$0W1z{sV;wA9iUMd_lgk@ zd=&T__?CkL+pfNE%j2o_x|t+%@12#hWG>@Cx3W-g!%W5{Clc659v#iTeNhf|7}b0> z%N>WII8=<5t=rnxNMZi^(rMN{My^$wO|Dz0F`Jq>aZoLDHe+Q3ShtkHK7%hvQU&B? z3CKv!Eca!}cd}d|k+WCA-Qscd5B6T;K5l|B7nb&m{OiwO)!<`2fZ@yhN4#{ZbOaRy zq(F80=!g24hO>{Y36PhZ`j~ql+$$Go5E7>UzKMC{C{zaXLc{t9PS!QQVqoUcFNM$h z#YBmPwnau(cokoqSJR6Wn*;E1!OH%}pir7YJ!0B>BF+MLwLftRp7N9jlioQgUo|6X z{?1`Tc*gRND;$hq0f6L-OT3op>)5I4F;X)tEtZ-a+vCs?Vt1r8-XN~tuqnB{`SkQ) zfnfX2xe!Kn$aV{5_9|LAjd+%gBZ$6AI|+>?k}Umoiq*Gxm+)(;_%uiODul6~C_ z4|382N@9L_${dxqht$>MP%dEemCPp?IevNL+lSh$Dx0}x9j|IZ+}rk z3u3j$8piIV#=I>VnKZn~^|K>6V-v-D?1N3j-A5Z9cXHo!5ml^j zilf##1{1#=G%?lGkGveEbM4A1SeQkeJ*)2*o&|Ee>s$4V3h&Y)F-9Cb0!NPxmp z9BTKKlvKo&YHRjy7h69zAUOplxDgis9^~STyMp&mC11y0?_q5e zM~F<~l3nRxzwpS&(T|ic;UgXZ_8$sR2dVRbYuG0cLm5a+c2AOZt89Jr=TA?(eYaGf zM$v!F0WgA#3{wq6HvIj#LQ~M;9eLoE4LstLHv^x9)o3BnB4A893hw*yHTnl?RS%u( znl-bg#dyZ&b#D(_eE62VuwD{8LBAo>I|Yrnk$&`g-N!S4Rr9Evs2&*Ii|&>p3abqWT%*<&lX!_K;78p2&wjveG9)F|ai2I-Tmh%6^feDvq29UK3oDj~` z7WgSZ6z~Rk%9FHzY&KpkTlw-ii1q!&MX!42*Q0Pz$(2iB4nIN?Xk zcbYp`Km~B&fdo89uBEN%Ul_5W)Bwz{Shxd&29kAO;F|61mB;D>tg=&Oj;;1yG259*+fmSMIx|h4GW0$LDqh8mu;oC0=_3~z5 zv{d7|G>0~}DvzB+9_>}>LeY)|jkC?9LAy9Tonxhcb%#*GUUm5G4pGX7;@%qHOOxB9 zbE}O1CeOON%SF-C^|C%lo^^-WRe&v}EvTe#;Fiyecip}|>=CHjzI*ucD0N?xY!8XO zjsT9z?AW-o8QhG}w;~C{7z#x};|Qd%Wbf60=5I)Pqcy&Qfvl=?`;D#Q^ZG`2n*Mo` z8>-VHmAa+J5BhayPZG`G+V9*)F^p$2zwkbl08ym38IJE?hH=FKIW{=AQk^5|44VU! zbYJ~xC^IA86GGUlW=A$BlaHMD$C1PP!F|XJ1H*;FMq5Q&W2_%1d>#0`PBp)y-Psg7 z8~^eX{dERZo4S%_TCZw%-D}mHTzS#lm-e(JWRw-45D4!D<38LV#sS1^wuaH;kLVj2 zUY!!R;}=X@3XSHI(n#frHzsuMP|$MPbuY_I2!?)oKUc%n^-9`t z`ePev`$~Ulx4C}(&pC;85=1#EpMXoOl@gE)P#UY&Ii@!?gpxX?*6fJ0=T{FJR)?GC zDR1%UXe1kDOUShEj&$gfJV|!V+OiY=%^TE>AB#B7+>%|_`Gq&Z4~~S->F?cW%2qm^ zy+}+J1<91QyUWShLAeB|jK@0%f7@P~i<(%rUP%PN_vElHX`4dPJS@@U5^o z+jo-vJc>xz@jH@OpYuk02yQDgiKj_euF))q0y&n>)adkj&UjFGe2Hb zck&yu6?4$GhQfLC(v*R+1~~dY0b`h5!=S$fx_X0H6RNqL_~O@ znkw5czh8C+UakXPwk)iS#u|CTI)UO7gUIaBPCPDF^@tt*y)GW>QFy9C3-l`}a5a#k ze0Ax6^Km878RN(C3vaU=&Lc(hmu*|a6TOdmw5A(+|0H)Ci4Os|XqH3Z&1zgHyFgt& zhP|I|5S^9>gk8oLDvztS`Uu&MX3))Fc%y`X0^PxJ;;;xD&_oK{`@QEpm88yJYr@~P zEC+kvjri8=du=kZ#zb0LT5>~Vl+;wt(Z(_VH8duZF+L@1s~llw#HyOqFDtn2U_BMV z#Rz1MK~dzDtb|zm=U7U(1oMc0Q!FVzG)&)xnjk57OQLsLJm2iJ1{Cs z@a^S)Q??U*#3=z3)w^kk<3d;ou9IGtM$NuxpgLXg37dqau3Lv@Ic&!Xj&$nVwsbh767WC&kg z9$+Q2=2rsiVjcizrZf-#^Y4F}CaATqQsk>g=bFnhZ<}$sp7&R{dEIGD(nWe@Qv z+@X1{i44)`d9I{8EjOU&DvUK@8V$kkLThWxxQww=*8}h=R-|7M(%p)BdacQJhywvL zYoH~G*ShyH(9()8$kh%&ciM5fA2_odMAlxt>Y+^4n3(w0e`Ah~Rf&0~ASp zUo^g(Is}NED~6(4nrW;G?LTY45;+lC3RzHwAGLm#oiYRO%c!hB`O;FP+t$!KtB-`| zzw@HsvoohrC2z}|g6FWUSCO$K8;ku|e&H8r;p3&^wo;KFJJ9OhS4eLHt1N9^cGo&l zvQBWQxcR#ll9kS!;u>OGb7Z)T6+Nv1OuDg_6q?H2*OnOrsSp}(6Zeih6Qez>28iwm2% zEYS-(561X`0ihO2ge&>N9sD<658P}Ib6q2T;q?Lm5i$*Hj&(}T{@yr>h9{g_%0Lp% zXMF(y5uiu_=<6y(z(SC!0K}Bwvmw_n@LnLiL;`OIW=?dS>d)iedPL%^docm~;!@1A zZ}2#jan!xl>wy#~0DtJXph~N< zDbu0?^5lxeqV0rRWFMAYSdn$StewJq9sV+RVcjD|T%f_naI;ML6EEJ^YWxy{*JJ9M z>Vict1TFM%cp^mgRuBrd_XX zQ0uc3)fd%z4B}>4*R$d#Es&oOE=fApb282T&?;iP^vV+sV*eb6pjbKUt3fzFdKtj= ziQ4tNpr(s!i*9ic)r%OMuO7}0TNs#8QN0oR@G}8-4auUf_h9L~?#movm#PXWiYHsI zG+5~0qt4WXtPwvfTFpla@nyeA+oXT7upwiKjBr02aokgmm9J&rE!7Rq42dzD;BlH% z5-@x8mG08^y8NqY-8S>5vBwlognG<{9AU7)IKGJZH!`$}LMLlrnj>8emd=UV6}H#5 zEAr2V)rOHvAeQsoQ?-om1$AFQZ#-7-3O6@1Tnj3lK1cG2fbSfQC%Dn#KO?)+j0MCTWjIQAC6*I{a@_uT56j{zCTg{c%9F`Uy5oo_0?Y?jsJ4{e_JxU3N9Loj}^JzY~h%Kc5MOr9! za9l2o{p8`9iX4jvbv820ze4zZ(2EldF+j0rDmMT^Im(h zl=+I&qS)T& zj^dZjT6d#A^FKb#;;HzMBK&%JB}Au~iG`h?ayrpXR2$+Bg16ZDGjPUH+zQI4NuDZ? z3SE5ErYp=>l58=&TTppaJpd?J3OCeM%x^1ELY;3vd3-k!&u@@GjEE{?_3pCLl|vzh zcKCeJh420g?FBOIi6szS48U77&`G!eWGg9rvT_wjK})GuhG;5tfrxGcaG5E^#_E@H z<4Az8pV74{3B{s`E}YXPRdp{V*U>@4T6XhZR@J3^o0`^{LEQNc++|}qV>&37hokdM z#C!;fFv257eG+%j5h;ef5C|$f;653jO6P!YzHT@dQ;^KEnc|brPc-J{W!v7YgAsbA z^8mUam1Ty+7jw*Acj|Y&IjNPTL2(7QamBU*haZOBs0Omcu(Q&Zd&K9WGH6y5o2Zf4 z=u0c^%(_I`x3SrgGnyu*c0Eg&pSJpk?_@ruUZO021IG+ni;K|Io_x;S=ybZj&>AU2E=^>LclCv{ zm4AcvH~m|Op?4HVJ@}2zC0xE4*XvGn7f(t$95C)O29;AA8yc~=sxlTN&yg6`0qxNo z8~2>z%}jDUuAv%>D}BV&H|wSoDeZ{gcR#R_pY9hkOcsM$R)5xUDxi=?JZ}v``!#3-d*u5HAEwE--=lY`9 zF6r7h@q5ABbUVArC#9*FODHXUe*s{H58b0(xLPDKJvpPtFxB!nVf@)mk008EQPCRlA}{T#hgaTvwmkP&l$YM)&9=E|kXV=N3{Oou4@MPMF3n()w5G|M zL%!DjaDUXyWa+xjKEWzz!cu5fUHW>8cOOT!2-7eHj%JUVk}??G+Qs%Gub&gb}7!%-}xRVXQXr^}s!yr7r#0#yld zESqC;H5?o)Y_SZFn11jV#dA*D$}+iEP4qkWJ(&`1Hexlhef_~eQBL8YdHqheaZHls z#1FBVj%W6)-5dl(`Sc9A5a{1vL5n3;li)7?_$*!8!ftV|F6-rPB3H;b_0#(#Z&C?22N1}r zZF^iD_) zIT4`oL-dQ4c|W=GR+Ey4J^|?L--wyC#UX4 z&nzN*>=M{)2T|Lvd!3DJ=`wrB6KtplJWIcRU3IqZ4Au@N)@FC1T5#D8U~L=@U`RgN z96cyzq6sT~;Q_ zD_H9VOd^!Y83geOavPmDRr-7^kt|b-A8SstEyYHS((rDw01 zEFMAtZwC@kAU&xy$VFgRE^OO^IblAI-QTxYBb9YAK^{`;9NPb0dcIj~!~j6o!lwfz z9`B`rk63|>tcwP4hq`|1$X4lDVW(oo?W7VGwEZqoyZR*Mke3aSfBq=1(&;F~q_Er# zG4sQ-;~tQ5?uG-72{wCaI0}Fpwcj3&hIqgMudk0E02nU{;8?}+kM(N;L`W5!TN?Ta z921#)T&chG5l0N%EgssOF;^^jvMj`=ek?Yxqc4QzncZ>zeyk-HN72H_JJJ3`YvoO| zU-KHJPiD04*ERMqAl8`|19DKq8Gucet!3#%bV4wY%N`t5sFYAijVPH)DW)&CY_uNY z6_p*xH%?AZDp!WquT#d2toLaFwl}YT{?E1m|7;t;IEY+gH3cN~(WgO={<6&;e5dYE zWN#9za`I1T0OFrT_?5q%aaI6l902kyWcpoh?aFN`vB7JvxF z4U%}_00E#=|LI})r&q+a|8yXKdu_4O`cz1X>=)ixbpd=`JxSNcv-z+D3=nmqKLFDO z8|;>gi#4kpW0?ZU9oX%LAC-uLJXA z)EH+N^w;f$ww*@O27%t8#(#>){>3NM@xmumI2Q$e^*M;`-+Cqe>u(LAo##H~{J$}} z)R_z;X*Xg9OH^1K*CT|0aW|PtJ2l{5)|6}H+ggh6h9?wcs1@c_??3p%Pb2M|Z58LD zaH@=r1#oRKW=y`PVf+C%(2mIcI`TVi754%N$bg*eHQWRsauC7Uz~wvO)ep#6kVZD9 z3+T8+|E+ZYci$q?KLYdFac9uMH~r4=t*-FXdG2M%2rK*v&}%7pObrJG+9Cb1-yz^* z;ERvNpK}5Gmp@fL3ex}Cbc8cHrh-Ref#mK{b|OKwg_wo!5GqmfdohC$ppjm&3s&sb zxX?3S<+8cNc#O|D)FNLaem@3BnX5vMPSodVPhQ^3!IJ-ZEC6K&3hc$YGy0rs@nSEk zf1X@|*Z<~fpf?^4U<;xY2wah(HVGuP0Aa{vf28l{c4Z#|>0bfk!Pcb7-F`h*o%hz8Ktv{^nE?_yU_$qdM4 z6^}~RhBNM%XfynLH|S_Szah=;G|$*%S;&z0*}IKugI?pmND%*DNDlu8@R@MY1IVM9 zsyjeOn?Ss?Rv)xg7`hjaiJPT^kCsFO-D7sx%S(A#aaWKXz@1c4=CDE^un?yNAmpiM zDW*45cNnefmhcb1=cLApCv#rQ#(N{l#gqEMH1DWn<6t?LIkn2^JfTfmFtKi6;}@QQ zW5m5ig9{rDv-#qQLfu@z-+whcEOYC^A`GI@&rZ>{| zD^mV)51j;{a7 zP{A8G2UZouES;0~^D`F-CX4mWYLzPSgYF{17M|XIRtfm*X_S8xp;#TXQW_ zIA^Pkm}hh_gq1Rw)!X z8k$K?Hk0HUrLF|mf&fpDZ+U26RZDh-M495LI(qI#c=+{9>bZUMswCqBb7)kN`cm=Cb2*L2U-dpM9^^BIW{xn| zIy9P4Tvd}I55fPgn^kXoB!q{GZnKdtHdSy&npAEtkL@hwZZmlnPwx!C^I1L~1s*X; z@)17IFnHM7S?^2k8yViJ%7e~%kUu3=#zOBVMW`lDcB6+>MtE_e@LkzGcBnu5r<8ls zad>OF=J$STRFR2ayL9)kP(&@})#jaZBbOJv-0@58GB4A64TTA)QrL#G2_GR+wsw(g zW-2LB?Y-jY^-Ii;Kl*cqT-}kO zKl;9RXz3+`{XRJ{)L5G~(@T`3H=^H~dcRufW8yo+;89YNsI>3bDBYqjnUTg69Y_)M`!+hO(Menm?ioP*++TiON14f|Dcb(yU4hCw=I6G??r;J17O5+D!m zA(Q&*y1pa6#yCr-8MKb&V2)GgL>ou#L1w7Ti+5cdA6Io;V^OaBZm~S-#k7#v>TXZk z<*30pkwU-}Qcxn7KU*=C_C#@Kqeozv>s_O2YS`#eIYR@=+J6B(Zjb?)r&Zd@KO_!5 zm3I+Y-G1m$>%%wEr1;XE(Ie$!tRCb!yaUI)=j`i6>Jr=*>0+elBTmms@#x5!tSW6) zkx*vvRvWE_88xNQsf>j`Lt<~uUVH#hD5c%dzr}|Ykg0ShG%V{ogyo!bOoS!-O}MO( zJoT6Nj}SB1GkPq!a+Uc-9{Am~GX(wP{Gm&x28~tj&5FSirt0e9E#|V+iDl*Jm9SU2 zc2Up1bnYx`-={fDUZvrmRBp_Eq$hA}ZhpIAv{>A`p$}uhVedX4$dB&+_nXjUE z!z~BA=+~UJOc0}9RDD}J*@5&hc=_}D-km-_Y98J2jZ7w4XnKSG;gQ%Lj!GXlCuTDj zJfYC+xVm83pue@LMiRiCd2q?Zq;pb+Nae}ceNPw=^4!!w7eYS18nmL=>6yDlS3o%J z19^6d&Uff~a!^*mNdEN^GxL(zZC92-fFJUt!p$aEE;(OUJhjwa>SS;M)=vqZ*K~p5 z{I-FF2hR{-Ex>*tVZHD5=xWfBic{@`NXA5PKt(eg#xil_yi23puKw{<1e+Jk9>UIG1;NPZk zznQ`RVf+8vxQ75VH!fBhswaF3#IXGisT#(=@Cs{6dbb%s8|gq{fj#Kof*=;QTmoqo z*!B>>9g7mT7$}O>$ilUOv4Z3{IB4Sy@JHs$u@hOSXZgg}Y~-(geZ2)PewFXgdrI^i zw}*2a3~YI(IvWL#?D8w+S7sV6?Y`#&F90cRD(U$oA~!w`#E^S;yYc~)EaxHgr^!GP ziU9(DPCp#OW3pb#tfgJtX&m;MS3GL&Yt&mzUlu z)H{ES+~<7<1b$Kq`$9?<=1s5`b+a*Os-d0*8wJ4 zb8dSp`LjPCB7)e>w9KZTwF7i1W#r`JUm;mW*BO%@=0_ z5swvPf5sa=qk8*RO69h4bR30IItyCS4Vu6q1ut^vi$g}PLvBDea09!~wtIu_SJu`9 z%0zw+dE#gD@~54^rOin7BJR;wAHG1JQ;#*ryua&+7s2ZrnP$FvRH(Vb56jPvn+jm? zSC|Ycc)#ua)lg$DEOS>CM^T+Pan2RlifUzud${_O+Hf8fCOM6MGoor6+f>%Y@94ao z-EBU%RzSDsj!fs?z`n8rg+Fy!5o45*Ep6OAS;w$#ur<^I42_=`XqX zzy>lW?Y@6EWCD8JIvnu40o6vPim>^9_ue^JwK)r0Eb2*0VjJ{(Y6mVu5%6-Hx zN-#vRC`(TJspN1oiCO6}k0Ns@Z$$B+P=`>_FFXnljris>)%Z~yHR{FV=YF($%}cR? z-?H9SOHGeOP*GYLKE;c=YK)dO8ENA6Q1)jJwD3sat5-|K3w=m#wvx)I{t|a+qdG_4 zA!d&g-3uwl^@DBFf=KJ~?!s z!`Vg0yn8KZlC<_vd&m!^57IPH{zVfXmc!$%sSbNIZI`mVWJ4bct*I%?GM4+mR<;F3 zn5&1xRJjnMm*8R(sw44QPVu}h&qUo54E#5ag^D%AEbk}Rm$Y5Y(C0NwfJm4ms<((Z zj2zcbnlyEDPB~#pMiRVqNg#-%ev9hq#@qRgG1Zs}rA=BmRk!>6yCWICZgtZ{ssxUz z4ysuo8j9^#`=~;EspGY#;Oxz#PnlDx_$bSkIJJ%mL^j@SyfWsK^efRJsX|xgpzI6U z!Jgo`nKZNcZLQt5vxes4Lru%0TBER)`Px~Is(mibg&yq{%g8*nDLMZ|hx%eEe(nH4 zr|Nhdf92j#oxOsQ=>jLjWPp~^J*MIpo}7uM;ErSHxbGWjO=oMGg$ma8S3Zps)7oB| zva{2%TbToAd}m`JTtu>pPFl`QQ~g2b%9T@fld7ZAy&1^MWEtB!WZMuxdfbb;g=+)i zNGx+yUD}m<;tjM}Rgsht>*1!iCLj;%>Ylx#Bq=>$oqYe?j!<}sW*KfSO?1xd@tosX zDTcxMlm*wgRI=3${W=VvieEB_XE!qA-t9P$HvQUr`oN0-uv)t4dN-g^SWp6;d&4ki z>fL;op=H7e0!JSO)fHfQSuW_sAPhiL8&E#ovBv$v)5h2XPnIrlWP;THh#>z3RhG?# ze^=?n)w3K>D*s~>dYjvE&GEnXL<0D7>N5W!Liuatasi3Me^{bAux{(%Gp>btw*LU6 zba%qNxlw<_3ml^tozAWT;lj;LnJ~2$QffuXLp}sFYry>H!RM_VD@}AcUjPxMzrNHI zSS|@iF{v*L-wnrA+9Lp&SdvmEY!wIwx6d$VZQ#wJHRL>J6>jO%$#WSTJ0g-=OTgIb zI>MD;Lv&PJ8YsJ*i-3Lt_2_-j@rRLq{qWPI;~AC;M3shNNC!p874npHPu2(*|Mn3w zwnr~)?e{tKxtDSLAAt~(oh31EYIkPsY27>Nwuu@&zL5335L>v8ZXe`hB&m4H- zJOi1PD5}s9lLsGKvx9F&Mn<_Me#^w~QxhwXHk)2w$34gOr4)Tef(}Hu(ejp_7eoz! z(*yse4O{2N0tm@SiBk}KybovQxry;RI)$@g;6KNPJzgIrOE^a+xC@8G z>1T)qS+>&%`$Sw~J0^AuFcSL-$%Sis!url_Ur z0*R)yJD1X4@2Us}7R}hQOC_ip=(HU?Iu`1F^F}%B5w0tDdeTb_mOVoY=JKxBov z{AU*mFo{cj^IPXM#V;dPS%`*y=cw-MGMAtsJWO_iWRg9eIU!I|+lcvtJ48cvALhL@ zr6y!Wb;So2JJ*W`+%xOhdN$`M*PB_tIjw}mE;?3d^9@fYx(ifU97v+H7j5YF zvqWCBRW+R5=}3`dD~~Aq<B6hHT>&NN1C^#+1m+ zMY$41>l<>r&gWf7=r;d>n8e9(H98~LL#1NRc!qHAp-UL9ztrHpOPE6T9hR$@27>!M z#G}Ljo~hm9#g49?+Z5hD4ENI~Jsh(tVXRN!1QK5+)5PZqjBmW}{T1Ws6k^zjJfaB;-&Av$ zM7zP4d4afGC--erp}Z~th?nA2#fC6Y^pv7fOE^C0JOL=G1{8s)(eew`+q4#9;tOaD z?_BA%mm3o)Rgun2D;WjtcLgIDO^9hIt|?VZ6Hskas>yqQ89ae4u=U(CT&7HmUjCr1 zJG^za?yzi|`j9Nu@A|9630c|7mD*g!G*ejdNR_!LOrxmyvk&0_t;Gb<&Yv>#!~ZJ#(8R(3 ztO3k$QhLC>F$|B=M-%h?;lU3C7k_PMdeIOQ^tXnX;l&1kzGcbl2xo9)j{L!mxtjP# z-SB_f6=dYgFT8_&FtEZsJ_Bm*@wm&~B)@(Cfk5`J#{{g4G=LQ4k4B}Ln;P)bQxH(0 z^wcW7;0j)P>u)Qi9Na46IP`;3D=dvK%&9=iE^Nyak02Tv6eD!sN9g`>VXpcusIO9g zj;V;fFJe%ypjnDi{R03;{O(Nf*Z=99G65^8Tj}tzQlP;6HbFP&kg8(?yl*Q)fDJm! z#l@M$jseaOR=OwvxTFOF5<@r5@aE?j;sao|L9<*l{)ZaNrGI(v|9Nu%e>kDPogBlz zRdY%Nzv=I5d)WTM)2ma$1=o~xHcaKBDSqMIQ^E=}40OXoL3=={XV4a?Q#Uen|D#$h z&Q>(h7TZ)|nJe!KyPe@q+hu8fz}9UO7`hdpxI!Emb?ut#c0$bJJ>xB;-b_)=GnE0h z-7^f?Lv&`k(zztA|2FtWiS;huE}>Z>3x<(qOFin7$#Agzvef_4-j#<#xwn7i6ppP$ z$TnUPhl8>X(WEkxaN;m;!;nxWYu2$&BBBuDbS6tkj%93(VQdo_b1XxQU5q{1qJ}2M z{C?j~owrl(@4e3Z`{RB8I)BagdY(N%vdQZkXZ+i=wcExh#I z>J8q#`=9C`rq~N|oU>cfS$7s z9s#(L119VvFPqgj6*oWio&`7@VU10acP!Fh;qsZp!SszDR2wJmn*24&p;|ai4&JV- z<;B$%VNG!j)85YWhsEtf+xk#~MV~~J(l3<29V=VQ!(-*!%pEcMMN~vag;EY%=!vUhMP894rHv4$6-lrR^dD zL=cloL0UZsKl%3A$VGB>^%K&TachCBDejDwQvdzn%L=v;Cw~?T9N)?oK7LdfvRF;z zJ~dDh(b@hDeV8V7TKsSKR?vsc3pCQWyc9c}pUVDtN-(9Xpz@}&=U7>!S7}|4vE_jW z&Dn&svvC(-iFU`1gJD?MXF_3epiIpTdOaE;U=Us&N8E2xY@7E(ryhfCgX+hk!(nh% z3`a8i#Gq46(s`72sq0Uc@z0N)P&?)A{_x~`#UN!NpPCmZ@YtAaNv-bqk;WXMG$|Sy z?d*&_&~YQ%%`6Y7|8ZTKG`tf1O{u7I(z1KY|S5kr`I9xp_$wqN;t3pW?Jn_Vh>AU=K;zl{yA(3(GO!jIam7B z|G=CpLmInA-af3Df1fP^drH>%5T0GZ{aJYS$MgTSP;EymC_D?U+peYOmAG*fQ%%CH z_%U8i`X0JxTK7zfi^?X->xl^OVo?EcN75%U$E6B5Pku0Y7#d0>rHzJIpiv7vf{? znP)kgep`e~LWZ&6*6eFKV{V3MSy2M8 z+{1ZIV1x^VpLM&5OcX6HVCA(yR#V;gu;hl?f@eLNA9SZ1Qs+|LraQwl*AHnkS+nlV z^}M54Z^j-O^4h5pP{wd73$WUm=>9gNKu|!p`Wd|;&o&p~*`Uy0N9?Yr9o1|(n-G#- zUr6_AQz)@*b4$v=qE%&HBUlu?v9~B5mYvV?;8pdIn3KF)`=VsZb}y0g;|oSGw98jN z0EdY2s@K~xV!=Cps4MT2QPpgLT*lz7L=zdy`oS@}{5Lh^_T=B=@)6Q7Jx;B=b0xmR zk{5lRg)jQnv8UILzF{q$e_d*duqEHU5$Y+s~0+jL>8y ztr>f3+tqlk;F+Nvive$Bx+I~G(&(LUU2mgL6NGDZAO@wv9d890!`$cWk`6K!2R5ZP zco03Ro+}S+;^23i#xb2Y%&JK(ZTk1*6s!qvpJ#jZb0p{WIr*x~8_CWA0lH~(go@bx z!Kd%nQw)wbcOn7Olz`nM#;vIDe|FhvYSXKcO?PByl<%Cytk;T}82XXq?ZaUu(e@KT*{j#x9IRw}{2OjH z^&f8&?Y|hbo7-D}uPgj&xNh^v<=9|DTKdmU=m7tsRP$Z6a4<#tc?;Ly1u|ULvW}uK z_UR%ODtl6T#)5T9HsTxl8@~5L&=mD$TDMgfQftR827Ap0961Hh7COb$J?o= z-UEXfrHp-Jw2gNpz*bQ(&Iv%4d|<6?TUf%Ivmd};sPn*tBU;H!12mast_ZqFzBbuu zUzluJV6wlwW9CqL zGeBhYZ+9vGc>v?MRw=UP$qDU91`cd|XX#H+XsB@ds5AZ9lB_X?&DzIeDqX-rLd)NT#iH*&6iTG)Mz(m{Dy!-1lHCf9R|$0 z{;5F;=%t|R6n0bYcUH#+N$%b|!bbLtg#hfPKsILw(1&EOagYGQ%;bTvzXdq-m$21= z2%v_!s%t!)Eg~rX0|otx9n>M=qlS_Dz-N(pP@TQ=gD(MDbbxTKPlJj(5SwD~k|WVg zfiR650Wy;j$Y7%&K(Nvu1uypere`;j)^*6+yFpCN1$N7?v73Xj00#=N-V?wFoJrFA zr^_2{m(+txeD1^=%mdtwAa@K~*9U^YssB@uHfg&!%vsCz`v^YFlzYLHR!v>i4LR(H zFV*cG6|epj7K3~lTgEn-B8!`&)$bTwxh74S(TT1iKKRD0QqskHYJ{&tNLVVv>v$?$ z<%!xP)M@PqWlE*JBqzc~K_quPVFZjud8J$FHJVYwQA+dBD(_Qj#buIkzMlC-h|~}V z+M_D8<3+%FvS^w9Ija2IEWDERekp1_mr&1>FKykj!46BOcKa>WC$U3C z5Se$156IUiE`9GU%I-E^6N{Z(lexb^_j%-MS1Z$6UyXde;?K{Jn>|_42A3*{>Vv%M zLsZy=!4?)Eq3F4$iI1^qVZr6yFDbciy|ZU0#5P z?xVO4T49ax5&r-Vuh`QI5TTPx6C32YL`vAHL`@A%)6tdnO4>n3Pa&)VgtO<|`AbzkZ^X~5(f`oClj_4oWa{7p%TL6Z+OP6BEm%6CVs5U^wZ1vmp*kvX zMe=F)j3(-GQXX>tTCi=o$+vq`BRe2|eH`sHdt^*9GE?LiKb05x6AXTn$pk3%?z??# cI&|Lh;fAyl9;ect>Kl|iD*j9Hq5m2FUsbh%tpET3 diff --git a/doc/v1_api_tutorials/quick_start/src/NetRNN_en.png b/doc/v1_api_tutorials/quick_start/src/NetRNN_en.png deleted file mode 100644 index 180f273d32ea59dc8ececa69c08e249f79f9d4f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57109 zcmZU)WmH`;v@MKtaCdiiclY8@+_gY)IUL;Gy|_zpDeh3*-Q6kf`f=Zt@5zrnGP0AA zWGC4xS#!-9p`s*%1dk671_p*CCo8E21_tQ?1_q76>f#XIOrXE=aE`J%E?{8juK)3XZ(5Q7!N4{v`LXVV%Z!)j#-cWo)>r&vMK0j8OAE zc2fOtKb*{VxLEhP6PRpH!jT$MX26t@BnzC+Yk7K9Gshf2L#heKz|HoGmCm{$IfDom*5%@7(C zFmDWJiGM?=%o~#Z*(z!2$D&k=eU?)%A|Miq=f{WO3lS%v{rm>i@xpV@LzvxLzr*8U*KITmm!NU)Ivjp4fbqKek5AaTymjB$?$fNB^osZ< zSsbY+&LD)0i!!GM%O+apZSwN=xK@HR~hOe*R8YLd>0+B%-?=e~`yFnOfU_j;O z3Q)vTxo|&@7l3=esSg(+N_7J2T+LzMAbTP*_^C2G?vO%lllqFc5^N@nZeXItX~n=m zK!%R&F?<5oI$Z1ST1K<+xYnNOD0iIkEuey2 zR|EySYk@EhfP;%al&*FLlmy^Mn@&lE39=gX9Nz5iNxf<~1LVdNsIsb1vnKw)!z4WX zk>M@n<)$PC57fUtw*PVptk-IP1&JP~rRfij=C0)D2wcMqjGRJ;7&jauuEol!LKRRo zPzfGc0hG}M$C$QT&*m;D+=M(-E_g$^+`rc}Hcdu|0*S@{L|OG}qXs(jV2>KWdR~;R zldbdYz2)-L#B-m*K-t)DqC#jhC%4CpyQCG|7LcXqnAmHap9Z46;=<4g!K&RE*lg5} zgI4(0&nrvN{8{a*k|Er(#$^Xh7gMGDn1~&aK@&D=2 z81S`d>KBPIW%pM9$4LIyxI|$|zgmU$&xuyxkEQVS6BQ)(3Lfd1d$A^fD#+K3NM?yY z9>tuMT9}vB%$Kwo8-y?WGvRV--zkm#Nw>o%BG*`5(7w^OQS+Z)BZ?!Fw`L7vSff>6 z-$2DA^iC2te{>7ieaRRTX~2q`3?~woR$s5Ce#Ji)q)_J+YURz4FXtd%Wapb1!+5i# zh5mf29>MtQPoJ%1p=tz-Efs!+t*c#8g)F(BH)%q7Zg_AI!}$=hEE7~gx>1&uiEwu? zc2!Nvbffas#u~bz4RUCdWy{g{XD5<9%x$GzwIqU7uPgW|3>nFE4jfX!kpU>b3qxLmd-~xKab(5s!C+Uo%Pye+N zsYKRW!0tX$a@kPxuJ3vHym(J8nRsz$2`=%vZ2xk>LJ0X?mef^cz8%vuM7Ji6iT&=S zq2JniR8Z9vWDqf;#o~W2ctCZqXtT=2ah2C&)zZP?S#>-GG9g2Q)nvRVsbVgLpYP3# z7yC|HF6v>6saRm-M)F9~{&-Sc!s%%RvicPv(va)j3-X861gbR~1g7I&u0_^3t8A!- zbbr1@=y&PDgwV@-58}97JEbkI0S;3IC+Ao0#Em8q;0`|AE1MJtJXPFQAx3kYXbCXX zfjOs_Kor}EeJL$E<~>-?`_*0im2}&DKeEW8eN^M1fa*4{WZvhm%5otVLK)aZhwL$5F#2njwX&W@YVyY|?GbB)*=9Fc$|F56D~=}+yk z8be*?yAmf3f5}8lvRw7WJplw z*(1naLhf)+ILTGOCCk3sR$bX^>i6^gBRPGL|soszO(iAXRH1G6P$`}uhj)qaT{1}e1+yk?`&(Ur^ z1RN@62d5OKx=C4t9+Qc^j(#qU9pD&{N&%ibK1LPx$EHNk{J<71?{=$vV8m4ZYEo_iu1wk=SavMRe+LdhkID0QaM|4ho{OfHlnJMF zcy$U6ZfCXtz`H*xHTsRIN2KfMfx`9qtV=(d9QzNg-&CYTUv zR^AauuRMpE+&Pc*=b{BteqyRFSiIRrck5VF^W9Ofs<>!W&l@gbZriAsGRnkFPkdkT{ zxksEguW|n>#X$}^SnhvJfqYmNgTL{)8H?LHNXzw`=IDCcH}gc<$-z1}xcTtf+YjVW z4%CX-XB0+S>3Az>IJn!3>$!h>Y2Mu>=Ob9k=0ETNKCiFm$1D<((-67UGU~Q~dHT&A z1XG|S20d;{i07jUhMwHaJvr>RF?>+8+jX*G_Lo}h)`*PlN+A&Ri5Uz|CJYgG$S4WE z7>`U$a3mC)Z5A`TEm+eD{#5Csq%i8RJ*V9aIE%1a_&F#ym0-PKdfzDZJBmzWDXMf! zcb5*AJjW!noc02$esIl-3kT&sw%;Wv7r$Qa0L28THJ9M9V^IsDU! z5z5i;7f@?UuA_e@_i$OPLm5D#Ih$e%xO&R;nAjK)05$@bh#8c+n-mR|XLODO3aE!W z3&li*x;3g%K_?>;O4KO=f_t)zoO4u9y@Gvj#)UkwFi?l~AU4>}pv9rL<=$)zR$*%Z z%MnrxBw*+cQCp_t>|fM#OGK!L^eA=@Fuvq36v<)cgL(^izyPZywV>zLJFt0XPkmA;@`k4D=Lyh zZW$=!vExG!4i#_%hsnZ7qTYmO2z#Vd<_&$0IPe~KU~evHa>s~1A8|W;97e22y5oF} z%cNy#^=gDW52CKS$1+KLnSRIW8>EPEhnjTV9)>xyi7T?So#VG|C;rx~YqSJZB0|ip zs%g=Uoh}$NUOn}2A`BwgI|jEcRMaXR#2BJ!uB+Vm6Jueur8(!%P{^=kC1upWo0uc_ z38B&Blr!tFf1^?jARiK}EXL&^*gy~xxH}Cfq9RTme_TsSXl8G=JNiQfao6%(o0rf* zgLu#Z9|}HMS-HsyV~j@a1dr)Akybe44+92zeR7zTBFwnh9i(>j`ayxHrQ&Us@)CcU? z-%LAZs~qRX4QH3dCGN0G`wETW) z*&lK-9WT2L^<&GO=Iunyse=d%MU9QR4Zn8onvuvi(-NFPX3PAs5*sqmgrbw)s#9>`?}%sUYgSPh*0RtTzuq6)@SJncRRGYQ(U2F`?vOC4hhp<@F?VeuO zGR|5|_g{F{kpF;C$(~J4>JIQ$JSS&nMn9eiY?8?SYG*Yv21`#mOPsybP~?zhZgWWY zGYYK$diA8$QS`PF0KeB%vxIGrU;C%|n+KsB;cpeA%kvdzAq3<2^kQ%^VCK{Kp(=|i@_rCC4teQnI%>ztwclUq!<@A-h1x9i}~FqYVus9@O^Nqpk0_~N+W8iLSauhOLjonMa=QPbccD3pnaV}# z2`6M!xVWW4BJ6S*ihN1o^YV}c4*5cMR24_>qOs3zYL|@W`YTgrQK5T&adFHSP|irl zlt@t1i5ZGKK9$%mzel@vSfA20JkCe*A*UZRM6uEssc4Hkt~(!)LhE=EatAutg-WhmKU@vU zWn&grFjcB-3*lEkEtut)G`zlWG#sLdaW&QL zKiM91+lrohT}=rFomtYrXAG$yNFa!8G6I%X&1{M+7KH(uPUTp|YZFFLeJE=I+KNT&L254g`8^2MA82B8kXQt^q=v5%fD z3Dl{jFAEpOU%82a=4(#{n@>@loY1oP2fQGIK~z~RsI~}7OeqZ|+1r`2_&R8`-Ii0E z-(&e4A_i zi1Wh3cAkS$I3xR8qGPeXJR9DKh_h(H=Dx@*n-3~aP&-+zD(mq3Nhn)MO^Z5OFOQd1 zW^iKht-Eb{#Vutj4;w!PT8|#qP(M8D&PCk4bo}~b62h9J<|QV{oZ$P}%7>}AnLeUz z)l|nPA4VvYI~i|}CK5PQ^t0hhxg?CG06;YP6Qb7aSDl}*7NnFMB?BGQN;Nz7=?CFe zjq?GpLv^IBVbK}KSvkV=4jPSuzY9=8bpyGQZrODaS3#Vx%0@cLG8I7SYk1Jwz!1K5#7wWCuLRGIUbX-Dy~c8=C+Xv zNPCZU0p9|0g3Ufc28XdmWt*IpdGL($=8twNmF^4AO(agqs)75+wWJDc!|?}X(edX) zYM;Q+=+s`2{SLyq;LfAD+LngaluC@3+_Br-{?vEOTI>R`0 z+;p6GJ^HfSJppL#xawE&ZO0}&Hc?RYGCE$^1yL(lrSl0R^+5;s!`%dQLs^a$8d?iv zHO!PaM$bk-aIJ)f$##v33jNI!Ft-cYAQGC1@JG0*Oq%8zAm|2UZ%wSjEp#<|e2-xt4(}hU7&j=y z$f7(|A}&wbqlW+D-Ehr_k<6;O?F`xpy8LA<L8qoKC@xvhJ!IZ{N$8^ax5!jVtH^%5o{$SVS-@xZF z9s2I;M-2Cv1$Bi}&DE}k>7r)NB5a%ZCzZmg1}-iu%#UKSOvJjm4!#fbGS(C(c;^jD zN%p%j1$#DoKWf3eHILy!L`5K;AuMW=sN7@Ut3}l<`8Kwdj5X;47K`8=pF%D`c9$*ZeCNDD%yTPXsyII^#t`SwgnvA0K%mIb7dNMI z=Z*D>l%obRNVEuGD-z-MZ1Te{6V7K&3>CLe9$ zrETgJ%KQrvb~IqmIPSeCrH)38+*kT>CJz-#zxe9f1Na7+@(|^{4Y-NKGaNsY!B$ph zA(Du}E4et~5|6V&u1~!aMUug9*<~uY7elL$oz_VF&d18j8Qt0i<1-~K5{`I7_7b`; zjz}~aI75z;n=0rZY}OA*v0~<|n+N475X|eYI4CIn_Imf8h@(zWd+`+EzQSrOiaQ`8 z5L0@3i;|Hos+W8*e9nY|sjiv=UQg&L{41)Vv^v*ObDaK%WHA5n5iqaxjQe*OAFi^# z)jU@Cm}Fx~@l)3ZZ5OYcKDp7&J#GB{A(1u5_Q2{f0w;9jd?L<1y?bFxD8>p#$Ntvr z&H?Lki#))ju=r!DMx&RJ_wBRD_aRR0^))Q9b0I=4tYhnx#3*@vgBjbed*MFoWkp18 zVyb2rG>DGNCe1!X*gBvT;zj3oNqvo?uf@Jd#Gq*;!hGNGR{^ZaxPeieq;w=voXI;sy~Bl-oHL>AxTb~% zC$0O-?1IMGR_gcjC!)@g2MQEQjDIOHH1a(rSFRA4*E&{pEw^Yo~I#Po~vYEWC=8!X_uNU!{2> zD&h_C6IT_zpo&_%O(zeQoP38q8qo2$<~amYFYm3ZSQaBkQhwJipj|=_P?aC?6s^U` zc|l9&P2RRHC=U#^m0|lEDs5U0O`1!e6*@t1kDni`P6sWNOrMmX9h(-5mNRDE>P2El zwh1e)mXa4aAPTTudNWV`G!B}@&lW)QoO=59x=huo8|`Zu07kpBY1dT+2+*En8oM75 zyuDi=(i=gr8Y#$hPpweCNT^^TN$m4NNex-5+$x)!U1Cm=T5=#e*sJbmLmAsWdrTdj zA^{s&(x>xzL#Ok)&Z3&ax-G~2WcIr^g_A%l0?&K$!;xG0{h2U(RMT@LEcEB4RIBJN zB&z7D%ctlnM`-2|16?wQRBy$L#}tvETumFPMsEgEX5+ReOBbYT+V4O)H1gf+42Fd zc?$j0NpkZEi)CsXLlCTec-Cmu&V3kpIO^JQpBz1o_bU>5gB zXuPse>}I*Px-wRO>%BH$NF^C6hqJyOB+~0TwhJ3(@bmNI8evVYW` zKeDeDt7zV9UKL&pAAT3rw6@MGW|O8E2~`ZuPdpjY$CnmYCa@hj6@XJx$zl#TLE-L9v1!H&W;f;E3C4{lMjLiKY;I( zi-d;Dg}Pymi_MlTwW4s z4}YM>P%1NVF6$euIAv27vt8IU8#J|K-SnMV9@G< zB|{1Adnr62Qk4d_Q>d@D8srhHMq+e3+DE2mDlbotQ6^|k z-SYY|bfh}|m~3;H#!prqLs)o?!FC?mYl(HU`Fs63kX5@HTKyADm&VABiw6+L2xAN|}3ffN^M<4BKM%JqMWZOtnb zR0+R;Y4&_+Tg$7Rfiw*z6pf|8Z#QP z^9aC?QYFvGU2^|L6EpQ;d6TZ^=KJRS*g?3HPZ=y;Wr0gOn@9WdQWe0?5)LBn=rXtW z4u1meZqzO^p?yU)^@QEqfBern#Gp>ErbLJ6C(?YxF_hoHu<_QFkhvU|we(`3nT!Ot zp#ICN;mt?_%{vlek)_V(4F$WQpL~k@CM~WP&B{Vh*O$YfxyudG{_P+j2|Glqk5g$n z1AmkGuBWg0P33AnD(%XFX#pJpQs3|nX>4#h2~b>}Uk5`mHZv_Dm$mySA4ei$cc**v zh%4YuAXHU7Q?*EkDEC3;As>hF0SmoK)B8h2OvkHx=OgfzCIaUq%Ywy#oCC%6PW|PM z>ZKFIN-g&`Y84YgVk1!`MpwO9ectR*S+#BC1T16=jVoQvvf~#yZ!sqzr+7R=Me+; z$weVhHRVy2M%ck4m_|Fe=a}DmBA*#Q5v_PI=;XN^qU=I0Y-I zmKVymr-Yi#{KTau(I3%t5&aeE-grJ)>m-2427uS+sy^8=2D%jx)aHw~gr)@-L1OME zty?#hUJ)87A#r6(2*P+WIGKd+*pKVfr3yj`epwXB6-bLTs03N-(=hapR~fhfK`Oxr z#NepGlg^5<_3*24*7Q$uO1=#68`Na&nRrRyrv9Lfr$fhUcG6J%u%Mt;d>-q%`U)c& z`^aVbW15I;Cw@Y=6A`PaG`7s6%`cEMI6aMQ?pyzV+b8LhM8Sc&MEWt8U2Y(d zU`zF3arlS#g;a@)q#$fe#yuOY*p`lpM|y5rT5U>`cRG}5zwvh z1J*rXtt1%02Pjpq_x^#gHDI@G!@3(zSYM}_A5;|jnM!?CPBr<9EFK@AqD_rJ7!o>c z>ivWO z1Bo%4F*_bt=;Tg}nFNcmI>~$yN}Y-e93^r^fX_Yya$<*~pGJadMa4liL%27Th$%O$ zRgICDK5&@A75Fc`_5uVTli$G&Fy3UfkEN0xE@>iy-OkZA^I$vO6`R2dwgHuGKSZS5 zMi}{>sC>s4-`#N^@e(wspkeX};14q4twa~HJ~{gbX@OEivA30Myn~obv#bpt3k?pJ zV|1dv^mWB)hQK(?S9=}f-LJXV{PyPa9U5-bOukI*cJD~-X1~dvet)$+5~nh=)vRW^ z<4@a`UjX_&b@(@XY!*a3M$aBOoDebcK-%?TwPM1;{JhvWnCgMU5T3dtxG9Rrv$x#f z+EBf-9DwU4zXxJ-4}q4Ney}xSF6P}aobvPN&q*Cx%&xgqf??gWN}A@4+uBrY?KaiA zOtyDrI1_L~i-iSz_^5_hVg>R6gcuq#eovLLN!SCfx^*%3mw)3v+S%%635j0J$zbK8^%tVH$%y$K=Q8PIP54YEG!OyZz75T+{43Vg~~us+?z2J#9+K5hNS&yF9tSEBqU zD7kW)}b z8k^M{V~Ft``Irr;EM9!O-WE%GM2087>x32xW%d^HG9WbPf2P=C)#WhGG?1`+)@?a! z3-8SGhu?$A6H?%CS?R1*0ikFlZp)8?poT;LVZ)7pj)lSC2MV!GoPv`onaz3%Cny1f zMA26)^t^E)wElCMsL_`Qi*1py>MXQW#<$2Xh^2^X3D%43w|qgt+)YFc{47M=Jl1$8)WGGTX9 z@xb>)NdGl>h}yRU7G_CuQ0aAWOHkzlf0KaTOoKIzeAV&ELqBtXUNI$n+Lo!<{P>-v z3Rtu%p%idktNk_`NfahEIFyE@Y#*3EYI28_Tza?g=R5U;AFVs600p{dDH+Up=gYT! zhllRCzXYbfyotr5Yn|V1wOAIQA5IEMW*j`&>(=do1C_m@A*p_9ViLnf9nM}xhFvNL z)|dJ0=3rn6G!ww?8OpptF0!w%K_6PkAmz`|d)zH{Uav;&KEj$Wj(6{8vIr+}Vmd_g zC^`BL<$f`nx)t;rpS{4~O?5$Sz9bI#ha*T}TXO_M%BIU!IzO^It~b+e?i3v4q@_zh zi>IeH2W1iu30uDJQuPfc8NsH1%n9qxwqGKaZMoURLS;8*6 z+n>;;vSTlwvo~=*h(wq-QC`CxqGK+Ib_&niuhZ9s!{rDIV}JR(Z%|Tz#X;Fpc%rTj z^zHr{K>6K*jyr(wg5F1qPxx``vfq}6>$vjfXWmpM1L?_n7aZJ6j2)`Q8m+4;m`0Yj zvm&zH0iB1-BI&fFOprh=X}utMob-UAtO&Kk9uf0hz@OkFkMwd zzL>t79LK<5v{?L{`>I&3KqqMo@pM@+VrjM=z^T0yZXX1TA`k{1FOB6J`;|diJ8&JP zd^7(ID7v9ERRim(fE*0Rjh%%&^+L=YR2KSaqlA8_*7!F@%9PE@=Odj)A&l(WiycEc z-FA?#$4u003mBVcgc*mdEYl6hjK&GGcSU48HPJ3xfjvT9*-N!YY(gxIO(*UzSc<3& z7ECt*(FRVu&IA+HbCSb`NtLBUCzup<@C08|Eploqvmd5xQf#4MHhf1JxmhM_pT?}y zWBX46qO1WFf3dU7AY-87)5#D?DlD;|lJo7&{z6)A5@_IH2rI9xjsN*M>{5d*^Lfej z`8cS#xDkr&wqhQ-HcV!Jh@>k)Ss18QqMTU}soA0GbO$1Oc6#%;S-{Dam-5WW>6h;1 zgk2hA@yCMp!Rnpk*|8d!Dy+-R^?N}hZkBo>!lH#o6p0;j6ZY1!e7m=zph_}R3=8CX zpZf9iA@mHlSB(DG_Wl0Il`t+Itnq6r;l0PEp^C%VsOH^-U3M*^PD?^N^s%&D?1tbM z;6ovIzd^av%F?)e}`%V=`9(d(8=k5*W@G7ZL~>;t@CcS*`ci$bHfB4W<>CvAyoNB?G&tWbTe0G22evHrsd z9%P(V=@@3D1rioE9gzCJq2a^eGUKS95?GVd8>9AcXcAia26^Lb>kE^}&s(W1b`5Ri zKjr)4AbeSMpY`FE9zuo*gpfp{5V9VjLkz(NZDEo_oZDS;mGnk!%tI$AAGHzN7U4rl zbNCl2ADViW#N)uBWBoX;iMh8uiHN@V{2|gMI>Egt=^k1E>(AA#zp)+pCgJ?GcCa;%dujg$|g!NO*&aA>;5vK*; zh6axPo2bcDh7T9+(L<>HkW!K~iN_7iA?Jg^qfutPxKtXF)-SGRJ?Fq}IOIdTkH^~g zSdkmi75j18*0VonO(gV|ogk~R5UX6qH3x3_*a2kA1Z3=NkC^6zZni%D3If|jJ&@OH9Pit^BvZ{o3Golj3Mh) z#%(>Ol;#2@>l@fL!G-??Q<$GWWR6D25K*@OY>&`)y5{>u5nmZU8~bbA-Qb%s;PhJ`RPDQC5Cto2Tp*)Xim% zuPsF`-4azzA!>{*pN^2@_~NdAag<-|q|7H`gLrzTM+y^Hk@9P_*O5&Ek7zLlS>Xkn zUWCS~G9Ex5HA@c4LF&%^uVhMN4?KntC3LY8;ofhU0;YV@f zUxCt5OpN()Z~Q^KA%yiPt^FIn#w3T|LuEi03F#`ePE`!2R)@b+3kT&0_Vx(%L-K%4 zNn31BoD;DR-me5GZ69{jtyp*``XJFS&Kcec^H&1pu+1aor&~2oLDrH~u1oMOJk3e> zwSXL7Pvo(|Pp|7cSTvX|G};2n-tDYCZNc;{e&2WtTDT#wco$aAsbBv?^bG*G${#Qj z4bnMuUeMz;41TX49oDqy4no!f82KIW1+xS~;1O`bDW9I59c=1)VH}() zu*9De4rods)3Pv#85;5>kKps&SKeCmX#hE=iG5xyGPLj$U0Ax%;c>*d@_8qNo9$J( z!i~0_ysrGFMkbaF$Z~z&$X=#WUoX3qb3V2xoQHhkX9ebYJBg{*H68BR#a3tcD-I2`($UiowD8Jj@2Y> zedyA{61oOY^Leg1eI~|jdg)Q`ZX*1)-A{6$87)Lvr}*^R2S?&)T8u#3HYbfDL?0|H zei8*x5oF=)W6__ppAgJXQN|g^2C$Q;sgkpSrO<<{OmiZD&(zt*l%jd!0PrOw?;uAd zy4+{7es1-hbx5i>2+w43wo_@gH$@?~N`wL1+CNyz_a?E|RDnE{eU5U`VXaWBEO7_g ztkuJI!7hKh_4q{;o=x+50y<;iLt(k92~qKQdAH_J%t;9K#Q!nU;~?MMb@(Lc$JUCM z!dlOGN@-e}4RB;X)is|?mmibPr}1)3mCY2A@`q8sD-_}s4BOXJ^ zV>*=MWhfVM3d#5H#F8+75hS58McX6IMz6 z^-zfV?h>z}y_fuy)!ad|zwySw+NhWt>t_Go^!qo+0HxA(Hiw_5$>ld^H6kO1uY^#S zDZ_ca66T#q(qmudjVVLNUR(%EFw`=~@ifEvjjmMIUMc5}gjZHxe)K>qm~|7l_9^#8 z$3q*{rpt6_LR#kbDj$j+Pd=+|?plAn`qT&~>Kc_@@tN{D$Yc?b@ck7<(rIA`g0y}Q zQ#*C7a5PVZ}S?1j)jqVe3%(4rKPz>Bvy--MO@1c%)JGM4e2clXmi9wXG5Kgq&~ zJR2DeHK;mY?^2Eo{fyYzIWk^mP~A^VK3h`ReFh_t(Tf-lYgqT$zXUkl<;%pt9xW`P{Zu{%i#BAY>KLj}EA?O9!@6%9KP#=OL3BY37mwbEGj zIv(_OPHmLbFY2C92t*-w!?}P4vrlyqM1S&$Ds$#uy$@91~Q(nPbL7G(giizivBSDPG+D-o# zgHEg8_K=r2hYPc?G&WvNp}o?ugP(Qt8E~KTD)PcI%!%V^MLY0*X6eO~`6GbND$)tw zCEf8KAJS4$F-^lC&*2;019AB`3$rDAQLq8dz_6T>pcy7NQ-yjuPpxi%93_3t=`JJg(~_0~@hZ{0+$J|_ajwYH{ zu{~`}_J5PfAbq~RH^W6f$M5v#gOQ~{$zx#U1K}VKn^f}=7W!WYXJMYYJaT$wh4+ zMIFEU1@o$f#pH7JPp~-}T9D%8Pw+lE5bck~ou5|^L22+!IgQq?jO!DcfvCJPacidn zQ4XDu0|GW8*MaDp?H8M)jpg_v61a4z>iH7B1ZQdqs*XR$4rr~vfBH$_mt!z-DtB@~`ocCr1qiEmLG^fh|fmyG)n%+;SK zJ6oIgoX?6XI&w=Iyx9B^kS>!|m@ZNHut46&pVOim5E`LvwrN?2{aAUD<+vsit3$k~ z$kd`u$@LizgAocO*O=?|eyFUE(F4&rg z$lLu7FZ+e=&g)YJQYO`{><rlE@j{;p-F zfiX754VgW!(!zeCxy@H1N*qHF*I$>`ET3K~=hx0S|MWx8@(O)&sH!W=fBu+~{yDUW z4Bvy7PPoRi8H$HXgKEd8rI6~1v|@AOfn5pouBd9B^b&hE3LB0?!nS!OgwCbf!)cM< z3kB?VeN=u~62cy+z$scIc#3%S^&e$<`%rRz zX!P5ojs99j$2xerRP4=u;yGOEUBiObqmDsA!^BzSBPhi3I3`;Jn(6x8H^#pwZspW`)*5t&g z5y-Q%>jQ6Gi;guMHt#X};f2TvkSIdpB4HnAfPJ`X=I3YV#FP}KB;Qxf+kI{4fM;LC z5j$LjPK@&;3-l|YlsUw=N$J$d=xDG$6^;~WPJr^^Y)*=#XC(2+f~grkx2Y!It`Y30 zSREUY^T1C1%wMy47wf7da#3YKN+VQUc#;4ZPtZu&c`9kS=&gT|xCFKiJUCeb8oQR# ztL%@BCICY}zZkE$3GX-Z7!eUSJjQDd_{(2wtPPxx)<2Zt!Zko~)A1t!asCmda*|+c z>6B;G()m22(BMF^gm-qH_=ypc0#DDEjrroPC&U*Wln$Y6^xAAfzH znwRw>L{M-~ac75xQ|v;B`<6N8;!SCd8QrTZGqMU1rznFM;3dwyPeMGUSvZm4vdsK^ z^2xFpmHru`Y>_W1#UQb2A3vplW6KlY^5ZQMBD*2tEI?U33jQE%P zeG_Lk%tH5yxRQppx|$ZNf6+br4o@1#8 zt!6Fe-M+>ck`!oTH6qF?=qdRsdEB_25rluX?2RAZ#4Pg~doMBkl7@)AWvM@II7aEr zhviChDYksxL)Kk4+-SX64TL}MBv$Mj?&JN!@b%+IV3CL`-n*X8lm0^tKhqE6vhUk{0V9JfV9%+PeC-lkGNKKYG-AE84RM-Crz$t%Y49Eiwwk z^Rj$9nK_Ew?GvVCbcx;N1HnWGE=^3-XrQNP&cGfyflYN9mw6%Q+A4`W>aw2H8ozD! z*Fo_BZERz-b359=F9}!?yY6gj)AP%jHSS~!O4saL8zUyO z$o+Ev!2wRr%!iW5D@5ztDuKo5W%t+#4RHuRP{dlOT^Ccp%4;VE#r&43dB(ddDV87C zq@;#RuqE4LO`RCD3%xLwp5y@-W&D;QDA(=xy60eVm(HegY?xMvvQfH|m3iqc%FKI4 z$#*^Pvftgj2`_bplPtO{`r&oE;d}>I%FSw54`T=?=zEmqOAcSu%>a(+X(dcNCfdE_e52#BBCCw6-`?1w$M|u zc{i`k@DF7%T=K5)xedOO>Wg3wR?mYe)+4VEbAji4j3!IqhHTe*Kfds<0Xql*V8W zrZr?|PF~GzOH*2VutG;CNAE4WZ~q>M^9*WM0FIr%J3S?+94HDMSB356lbhzn^VI1) zH2TP|x}L=<#CH>U6XXch=z%8K1tI8vw{)0$P@qHkTq9!q%y9QjG2bnj+R#WX za-h?uDVb5N?X%d=T20bT7DIx1&`2bx#Tmy?iFKm&+X8}MepHhT_5AYI)J$Qw{p)(N z@*725=9M9eCWXFUul`L`$T_h@Ih+1Vq9N59LRel70|!&BIGB+Ci&63-gr+`E2Bj0{ zBW5X^$LbI0v8+*ZNyGM-7VH0G>zd>0eEA#a2|IAmY+>}!)3uUua%UQvjlm=VR7ojK{QBFiRR#&{R$aQF?00M?@*?u2akxu zGqIpG)rA8n9TZEPa8ZQ;pPR~R=5T|<<0^DX!`cT72A5CtC>2P$(>V;HbtCu8h4`0F z0eWaCC``*RTp`EJXYZWUGjl2tXyOpB461bSP`YzbIkaOM&hH@~ak=$#t`Fv9$`>$E zn(JKQO7>WR(~i^5(J~xxx`@=!CC!#lVQss|B`d3g^&ySTc_6gS$#C=v&f{HNygOsw zq|MmH=T{Gpo%`7)*ft^AGVN4^z|G4 ztZ!oEtrGaN&HEof1EThTzJ7@tGJ~z7MaT-TEM@_GHOP$HyjtmSul@w}PwO}Nvrb=` zXaqX8%oi@q-RxQ%tgvb8hkXIDt?WHm&-#q2%`nk%#a;NF3a{7LXt#P-IFuj{qvjLzc(6X_r)1CP7s>q*K zu*;KFgq5Z4N+)Xt0HYjAvwKsMJ1*5P7{D@wK1fe&npA+Rv;)GL7V`UzF8Ce3Laf;R2T1vDmGr=QOOqui(fpv6ly}SewND zP=pFVG#K5Z8K%)?d+jnmsvXCj!O<0`w~PN;;7X7VM{1 z9GRwGun<@2GH}PR@64M|ND-}La;kH31=>S$>Z$f-_g1rSFFH)J@Ot(6l&o%Iwqg2R z+}_m)7DPot|e*y=Y@{RV9FS|CBQY4jxMci097RS8Hg8VC>a4a}JPd z<9}M6|BpSff(o=(Hs=}Pf}sQo>O-J#+;k+kFL@^gM&|=HoKAMq=JExrDeT98DASk` zl+#(wqYy?|veJw(sC3JKNEU%eP>Y@#Y@u&6{_EFLo_}$H_gJWxf}%EdDog!xD+o;K zS&K{+M;K>(XzT2R+u$HQSUCrmOS8(qpD`p_7o3E}|JOhNpnmVMFy5zWQhSS*uh;)x z&;S0)AqL3R-0jbme1BLw|HRmsU;uE98K$;F&7ZO0e_(J>1OphA;z9MR>~CIM0}Fu4 zP!CAJG5<4G_xF2w#R&n7D$b3D9s8Tt0$4D%Iex=X1p6OgAAbxR84dZ}CLuba7u@hS zuO0r07GmhM@Mq`aKc@v)rZh?^mq_2R(0y>w`H$?l0Ik z@0^LfWL!D|BkD(WO&E7*F`y;X>_sU}ea@#E#Z%Z13R+p3a3IvplULI1zy4-T&4iH{ z9}2BQDs_b@-6n2`8CU2Y7Ce;h<7dN_jWPz6PBS``X8m(&vT)OyD#Am{1_Hoo#;zOU zoHZTFew~>xC>Dj=$95Ir4kBkYpuxd&GmO2Vx*Q5erCL4;>hc&}@+`gh4;l>M4OO3v z_kMQ(rjW7=q16!+z6l5<5g~v;Qk?RssV{+=raBq(Kw&lmx(u#nBHu#eKllERledEa z-NbFBmtXiEF~>ot#W3ww+H?~qe=1ISn1g}$ty}3>c{OIfD7hl$H}ZS`7?NBDNv}}W^5UdMyfHK*yfY;Wt58Tgb;>nKc}{=*1oSry5HH+8gF_a5dMEmZn>bXnuso~3 z6Ar!N6OA=mTHVsL+x4~oca+Zw%7ljijVnQMeH2B2ISM=dHOfXMw55qn=m)JH^xGG` z@1w5l4!W zkOg=&yHd#z5E?@ER{p8X5{<hk^6o})~dGyOG-Mx{{$RC%PRsbV-Ju|!Ud=QpR00Zb$8 z#jb|K=?HV+Wfp8BVl)z60731#BWBa0rz%tGLmzC;pt*bRs@jLk$)OTJZFW`uPo(75GfAHiu_G5ry#3r zGUK&yxzkMLn=hx;17@H&B^NJt=r4SO0c`QW(X=rXDU1#+f`$JOXrrPCbe9%g{(WwY zH2gO|FAVm|cF~Y~Kt9tV+9_+ICC*j4USviQkQsfdkoz6Ag&}3!g}Z2gK_`nKW)04Y zHWrs6z_hvuXHA8D30`UXWe7VgP|*Ojkn@qgA)MENBq+3R3nppVa5MkWJ%IMtkHFUX zno8!X1>4L{r(HpF-_WVg`tIjTf5Y4pCWt}kEC>OF60yyPiy3}MF(w3wsk>~8ynj~U z{$)^QBopzzg!NGVun+3;dy;Pah>oHnAtWWrIWlCX7s_t+j(rt_?91@D<;KqOxdDEL z49;7y-$i$qbqrA9joyC_#SRo0slaaGpp9f`<=Pn)@g|$*a=B1`ZycqN=6$|?8*6PY z;YMUJld6O{Xu-Fw1)*veaVE=1?oVXP4Uqb#YhGbvzpQ)45M(DDm-A-XpD&f|GhZ&@ zbV`M!(h9H|VW!K4hhw`_ehW#OPh|MR?oi!FA|hv7|M9#E{5&c#Ognd zB24LqwQeHaEX`P|itQVr0Ag|xv0j%TYP=MB-Yk*=$yMs;YoYRl)@He7WlenKqHizS=Wd zR4tPgsGZdw|A(481QhN?h+yk|J%tm+3yQhbe>jJi5V&YLPH{#|j_tA(Bfs%OFZE&I zCd-MIn5&ED_Lod#7jjU43#q}SEAfjv9WBm*HJcXhqM!hqA|kJeVNr8`KdR82%j&1x z!=?Q73g3QOjOd9+2=ViC^hgP{U^0`kc}36IUyOV&$O0P`$Z{=flFPmL?pVwle6iv- z@hMAfp_E7Qpl95{d0m|Y>wEWs*5Www%d>$dS93UEf>BAA!$Hggz-n&t*O+_VR9RH4>kMb33aLA<~ z!=$M16prOz)ZT2}z7@hjkjN{Mn-rUqIJ@ec#0z^xpW(bgh8Us{Zlj5}{*_4j%U2O~ z232g-kW&C^uTpj6;e;+7gE6Fqj#zmMzDM*iaaN=9DOYgAE2NDo2}6kcc5^$!DL&pp zv6*^{lxD4-kI@eIUvRShQ%EL>#`lqCPDY-gA4=qdyQ%?2<`@?@7)7~=CH6?5`>jIz z=8pwj=kLXv@t{}F%6S_Z$FUon5OU($pTEr|pJn8fF)hlMTc7TbvPt;}RPy7CL0vRww6P!En?^tJ!^}}Kx`Xbka zzW7r*Ja)vf+4f3uXrVwf18lv2KN3SQ@pY%{!d>m8TKYC6nMbq92)pKfH{D3&FK77A zh4S_jPKfW6eTFbs`HeR1Ks5eY7|%lVw}Sm^sC{676*&ino3-NeAD;fHPXWP(5C8Nz z|9C6|=BIaw?iAzz^XClyp^d-We6juO{eM60P5TpvPHUZDk>+pn_wUDsQb+${`+q#$ z^bRlyd+$Kw`hQrae+^3EIGhZETORajVn}T>=-#T#m%HS5Dy0w74sR!19E%4&wrm&B z$Co+fEYI^K63nu9FWk?H92Su4@mP;sM#%tH-h00Q_1F9V+>#0TpZ(N(_l!BuFUh&j zt;r;W$oqH0w9bO^1&ISRRvk`5=K(i%6#WKu|OEC;!b;}KUSK9U0 z3nTW;dyvhZb{+PS=>8+eb7+Qd3fN;84?QtqYruK)x$EC52>!cxEqEZZkh+2z;h`d* z%?l=+Dh#P%q&E?j;IkJhk`HJH?w9YQxotFp157!<@TFbKM%vwIlsa6}FQc8Vz=H(& zi#mje^q^2jqeUP1$(^|`b`Vf3Blui&NXAo1nBT9(ujaRw^$Q&p_pCYEUl{BjJ%sb% z?2X0C4t^zf@VL19K$y1NR08-8%O^3bOpK4q4E~tlnM-D zjTV!rK7R9yOiU7#Wr*hldEv{}7pXazZ>j<=>5r5sEx?7T*J&s`o^^i1AtdU7XW$6Y zY5iPaU{=h9Pe)6D-UK8>k*?3u$eCFH9J_yz><$4h-yomhdnX88)K4X_-f{dLogbjf zD+B%JmWNSV(C`;p<2cCa;&gZ@+q8>CADfybj5idW?(5=4XdDu>MYySZWzs2s@U)P6 zEA@-M;wFgo_B)FIxJSs*ZHFOoT~aW7Wj+cm-=lY8(20F)e4l%sDUi8dO6NZ2qGC$s zxjY!xjHP6hjR{7$p$n52i(~pX5v{-jLBmA_vBr@gj|yss(5;}fSbTztcIp;&wGl@b zb4sn0_uOW+KI}}?D^-KEv)qB#ON8gN%4BO7e`ygL%2B(%H;HDk`FvpecqDr&9hIcB z>Q!}{1{z>VXg}Q+DRg%nd{!T|tTh(bdv{9Zt;CX0B3D7Fm>{~N62Y(aO}zchjqb*l zbf63S?qFLlF??oOg${lS1ptxyH}_OXf{Jof7<(HmGiV-$I|+3iosotw1ekZ#AbXRO zQ%#lO5k*fb97ebd$2Oy_&%eYrE9eC8EE5>KV+%mVTOo?m;VgJ?T=nTh_KOw~DnqzD z4oPRn)JxpTxpT%7%UHvbf4c*vXnA{g0r!D~Z7B^{=-l|i=muFvT1K^a0`KvBtHDR$ zEdCL5Yol?g9snLHSJ(>H=)aSBqD-h&-meq-M zafd+&2C+iBy4oE@ch?YLxO-~Ta{p9PB-YuMCP3nTLX*3<@kW-pxp25~WN(NG+~zf%X23H@e~^@Mf#+u>x~Y9^-r_TFc~pgE3yF7~sv-dv zit@ZNxb?~}!`&mSd(V6|;~VHvDC!VD+c1{8vuq!~XSBuir>Dn`3^d$$BzoPvT1&H6 zb~9e1;dBa#5E>*>6oTc>WQQE03P^Bc$Fk$TFD?FtdM-TnA&%umBhEv{Qp;DrU)tIu zCr=WUwIACmB3HjZz-q)8?rp^xrFm9{h-3Pf(`Co{iM2BOH}v(p`x-&;uG#)w0wrfPmtGok7tLgzj^ZybvmK+ zGf-5_Db|eeAN!*NaewxwZBUBWzS8oiBK%7U+o1xYVMmlQQ>Wd(ET@V2r_Mg=xBO)J zx4imeWI#|!;k0QeMsm@eRl-HzgvadHDN!IA*V&-Qk??!Q@_YKgPf(oBfG4zqRtX-n%BolNEw60b)#%1mP+$t2SV13|)I{MmZSR;We05CBjRJfjlqlyRVT#17ekt?{(P zfeben3Ys|w(0M3awk!6j5MLetz{&jcOyZD?<8Af?`3boK#f|4@H~rR!ygPJmZ#{e8 z57f1k&@ylU5WotbTzwSZo%03imB>ayOt0gB!o8z!c8i%>}ZZk5~5?w(y*Fzs@{E0Ur>?MT?siRa27XOxsadGh_3s z`FhdnSb5LWqMc>p5j-m}dF4#}c}7)C$4uqI=&nN!@`y31H#YShyv-5e%@5=h4!I%^ z4=H}zS7ezy1ns6s!Al5}|(MGEb~-B)~j2yTKX zdju3R)l_znnkYL;&@44J7>PH-B1&0$O{EjL#r&xFt+rS%|qHbhuR|u$6`v^hTGW`%O zM*Nz8=4pgFVmXqb1_c2OJ)6C7TrqM_f>B4NI|O^;7B-Xi66iO(DgsKKgbTo(bQ8~` zn7BEfhN3^ezo^J45OFD|xC!JYzv*v_;B4N=o3DL!uVVbK9Dx_e=uKzHG7HK4`Mv<|6#T0qHxF#?Ae_6 zr!<)m`YC)uIR{W0;Q$fxH~i?V<=WRvhrfRCKZfA_@-sy?_6n+C_M8X!-noVqAdwS( zS=Rrv@**@qz$`+m9nJkBmtegee(tFQ=o0PkLgpF-04b?KVD|Ehdj$v)hW`|W;krw? zzerbrcSZFxGeBifs`DSI^wW=7MEW@&q_54t3y+%n0|dIcf60y6FM=|Fm*ZzPiLo@9YbZz&l9UhVVxQLH=4~ID?HfsZXz5hvQw=d>_(D^R^eR zO`yFCKEb^wA(SiE$--S@;M;WM3~17t!KSBA_?fw6Vg7r@KldWZ-;2@)x7?|{L2c@6 zL9}IUnFtuHT&!FIUxZU(E$*6QF219|a|J1Tm_Jn0eyj;Isz_FA?F{rWF_tllZhM%Dz$E*aFmiFzFJ;5`ZxPbd*1Hmf!R6emeiHQR>RS0A{py>x6g@4W&7q;yL} z0CnO}-4UGM_wGz5J;-qr9kSW45}!t-<^8J2{H_dpc^0NK^6D<(&Aa5xi~~0PA+0+n z$79-mQoSfHaP1G|)kDqTYh3BH(V98P4Yt`N z;f~-e8bC^+swl&WQ~6UDX~>@v?0*8PBLqn|5#+3AkK}waiWS2n_M4kZ@aKC&sbJ{C zPOY-|iSUGtHB_q8iSYe8NMk6!MH?|!1J?`cM}c5RLT|~7o;X%NOX0^73j6#wNEsc~ z55p^(yP6-zwd&o&j{E+(R8=TYer9B&ft__J#X?*Bl$f&BLSI63H~|bDsV;0>kO+xU zam^m-6L?5g$kEu6RmdAIxQ#^om5WEgw>+Vf?|TOwW{%vNJ@?HN4#TO)04puMXd2_#`nmy?NANJpHYPY*);_c20Iuh zceBi%M)A}RT}1WImQWGg>K~q07B2y{31db<|Jo+rVnnMhdqOS4rU8;RxD z^QQE5Wle7ySnMH}T#oi!_Uw_7M>TftJSmS{Pg5bZtfrF8ndhq6Se!B4kuHw4vWCtH zRbZ0fwCxK;c_YDXP|Pl}NRCe^Qi#OXL>U~D>tPXjEZIRL++i#(KdcxXFLmk8%12DQ z&y-o9vDByefh2o`#XWz8AZ87#jB7Uf4O+1T1v<(gDCx6010pWXgGJAA^JmEp?X`Eiz-JMp3|DhlVSb~l85 z`VoGoCLE|q!L^neaoWOKh{8RY&9t~ZXOo5fZ2<62+Td0X#}ieo7zRm{{cXRiPrc~F z35Vr$Bd9-T(!zAAL~PqC(Ots{B^c>(Ym?V0ZD$dlsnSs1geH9$S4JF zcZ3&X_b4$f{3$L4GlQ5a$QGkbsHaxi{1ow9vRV%evuiY6 z2!cLeIvKh9jsgTVZ_wooS}!QMYkXmdSGRkz;@pD(zz%0ZThAs*;+Q}kLLypQ4HJf; zbdW~%Bl`-yim9XP``sVI9V4|V#J{DL&piK#ZK8|lpb`0`9S9QZHh$o|v<8vEmWG|* z%jT|8(t2n&6YTXPFF@3I#AINBO?s!`Ry&;)9HQoQ zwfqcu?Ox<#0xxHmRc{$ff?K61E%ZiC&6)V4<1=_l~slNiWr=k z!ht40;p95KIaQmLNimH)QebBlZZuFykuJl`k&dJrchZwYL0Q_BP;q=5aVF6SjcX>S z(ZTq3=1bKx-t=D2I0GZYubWu-@EANP&ae6^rHqB2Q@nrAI-9?U5E-|&8uYG7gG}Ak z-0k8V0MdUfH)vf66Dx^_vz?m9dF1c$17 zt0+4(nfGr`^sFzBNv>)Gm<1P`(hg=6;k+YVc%8&ut7r#5M7oxAXbG@V2f}B=(KMJr zBR5}~pH}!s>v za7#_j#MT#l^_;qrFuX*Nt`cw7q74epI1zH+N5m3B5t-t+=pNB0xLn6P5@|>6cZ7XR zSQP10z)UflR;ia*UxCTE*2PVPT|F@RfsFaV?0vck^X{~1ve-)-f>_PnKwNyMM!~XP zluy10YV%!r-&w`{Ty4DiPby^89Yz>Sbx?NM@ln>C>gW$R$+}Mj$5m^gC zW+P*74Acgck3>#CeId^c$4pVDEo?0fQB+?r1VhhT*T>DL2kTa#`%_{p!60a^vyp7` zPVz?xDdlMsfx^`!NJOJG$;MFEY6=q`Y5OrJ*{$3zGM?w*`te zg%W@*71w0K6HJB^U|k|JSp!_ia|{+-(gf1-XfE~PeM~1$v)Hqx4rK1w%3gy-UU?&+ zw|F-17?;#H|6Fn^Qx3e}11;q)dFJ^x9Q2E1=i)XNJ(06-&a0oyh5=@iJnjY>h2tb| zofA(R^sG;@V&!{PRmM*|>X-f6A;ID2+>{`i4U&=KH-$TG&#*cIL3}qdbP@~6bK(WY z0Z#jVnLc-C=@nM^@y%f+JErbk`j}2Go_;90G5#bmKAO5cyU~@e+R-0PWiEEE!<@Cg z^5>rVZRVlN3-AG8E^{86^GUwld(vvBEmq8&c*OZRHnm8F&J~?sAK4;19m_^ZLT;*h zwXn64e9i_TF>7t@;W5@Chk?vJu=$8n$U&vf%`ig5Subi>IV0^MQ{>^McKG%T9RX`! z?Neb4uZiAu-yEr{YiF9xOu=o01>LgtjOS-Pqb}avu0 z_W5J|6{aa(#vJ;k*l78g=t2V(ag8o?JelwZL(vnk>Ws=gUFlk>dwdu-*VjkYsna#6 zIdA-MvG)M*M3)cN>G#m!-K_AvQ@ipU;~}$JV?V`r6##&7T>f&i5&8KET_xTJz5KR| zhj{U`U@}&(efiTS3sj4BSu=}J*8JXWjE}pempAwLY~C!_B0O8Mp1DtV3Al88m9>h5 z@g&m@RfcWu?Q)@&he5rIxoMqE|29v)jd#vk6|{{C{`b$HhZios4z4_KQpl9rctc+}K=6FjAP$v?Vv z$GRfu^Lqo>=FviFhUxE&2k&Ln(O^NVbDmo@au1*OGGz5edB=oKOmrmdCP>Zo3G#uu z6j+6x5xx@PZddpn@kcklT13|$dG^eC!l^j%>uFs9ImQ=)nz7nYlY`Dlx=D!ZaN><4 zI$O{a2e7b8`atlUpov&y=jm6oO!bmMIvDA&Vq7wlnh$lgV-ia-O-Z)C`Z9ojJx3_diMN~PMe=p}j`qtI%Qhn_7VFrW|~SP^Y}%=f~%mUgczB`7fSpniwp z>~^O04Gaox_%YJJ{FZKT33C?2qK4GZah5H2E1ZySNP}L~<--d(DJEydd~C}a*SoG1 z%%T05W8o>7FROT)PoN<_fD3B6knwe9zhP(5(a0QCiv7@!274LLJ@K5&VkZ0Uy92)Hry%@g z3b+GsXICOHZ=VJV2Xk;&SGy7Co!-YDcj-M8P=oOv^jIJz(Vp1j0ArxD`@o)Rrw|us zcVFcMpYBupY&pqF36{l^MZDHKCNwKrXPC#rk?oOtP0sfA-72Pfq3PNKhPY0mfoh30 zUCY6>t*Y+?%YY(fua?%wZ?j6lpXwv7XKJA)3BJpu>7*g6(RY@LPlU`;??~HbhkOY| z*Z@mv2>#IA!bK~xil*AV*ug&;@>HRqj0{AnXd>__8UU;}O;4N;?DIHtS0NjDnue1oWw z??z{}ag~#&rxqBqXZc37=Y%ayIq_{ zQU;vy){lGHO5PTQPR_m0m+2rI2hb``hxkob(ofIwcI$5OuW166dmoBd_41pb8=JPG z+gJ)Wpq>&F++)Zfowm<{8;9)&IjJewIn-IHXPbQ2Q=5;3%0@sxM>;~M)t_<=j`shk zE&R$oo99Tc^07YhEScvPn%r44xe3eP047}koh_x0OlR!}vFixZ>7t?%fp#paNh$0q z^-g#aQQ?8oy|BLXBYQI;C{=$8m6m}#^D;PE@O7JsUitjXTS0kdVr&J60=ZE<_skZRp7?EM24gjwg`Th z8Q)ci);rxl{Z5q9_QgS(Q@}ht!KF29m@E{OKf2ufwKHjya=il!T(xT(3;}@LV zU-p3E;6wDlIUNv=rgNqj`Gr2rDAR!;1y_u&Spd_mv%DapLACqi^z1ym^G@qm^FvVE z93cU}5?jI*n~Br=t=xrn-7Ef*=1 zi*vm$R??h#OjNbxvA6m{^UT5oNkQ^UYhy@(H870iz!tx5vo({F10x4f6yp!F>Xc2H!<_P2I0| zk5VjzRUYim75Zkr3yp0bFDlEQxXg+Uv+KVNlp2rW8K~v8rj4T!FVpXD&d#l|tCKhf z9nLEsH;qUCQ7O}EUItGX)6pS!2;{II=YOLfDn-(&m)2!`F!6rG9Ui+R?naD#0B4Or zAdOddy}2ZRC3*VKu>E z_c|f6V})#t%p-7@P>3_cX#*F&m35q$nHCHXwS0lzJ}C&%v$Dc^bc?ao;tYYJW@z>X z7UF!gM_Ad7Ec@O{!6ZdL4Oa)^8hw`VqW<6n_xnBL>iYh%U^7Qtd$;th!>DaTF}<+y z+`9c{4xW0H7RT=p@#^7=8=1DRb=`*Yh^J*d#O%}hPsy*wJKA$YE5k-g38{L)y~*lq zg~~xvSqxu#vy>rK^11L|zb={L(nJBSclb(czgSq+9syO$2>o7Nb70MQe0cE)bo}wf4jPp&mIBv_~r@H*?=M41vF^T~G1EihDPB;v-Z5TX!##n)n2RJXr%C39+M7m2EaK~Yd_$dsQ$cIIMBK(E<5D^5AgPTqNuHB(5KGO0 zpJHbOl?U-2D_zz@f;_i8Ki8ALJjPVzl^fuaQ|or@6!H|e;x>+CW?U<-t@pX|=>HP!2mQr#!$RahT1{jO zfCqKKUSflQI=4oaKG}K3A9dqRy`W8|dvgoEZ8B=xk zzsS)3Bc!eje_18fl^fBI_kAX61Vd;WL;Pw^TWIpHlc{2hfnKC2&5!MEfIL19ex|>I zmXSGe7aEsK4t3LK6$+O_lmT`n`RnUK`HKU0DlpHK0^+g)n>Q-Ab^OBA%SH+(D*uwy zWgNu)nIRnX+A2qjSpT^*I%b~sqAsR4<%b23s`ESnyGrXRT`r#tsie0q*SjeHdOUn- zQVar>qQ)Ge%--otO@6u0a#Ntg-?taeBtIN+yv+kT@j=LO9kr>0g~<08r} z*CNd%908+ry7xO@me6^Ln32+Y2n$NqysnXvq+B`ArSltU2MxM6cWN(UROStpxWm#{Jw?#=-Kr zG(P}>6%t`W$mpOqbi_UJft6N=%dV!#xy592^hSKO4P9cbIz7JSMLI@Qh=5vKYRPq6$Uc9$ILgI$bvS3rlumq3_)%)l!eyF#yOQ9w9Y&v_3#q3+`jM|BrLB2*d9%EY@_GHf$oJ@R42bdjo!^47E^!5Z9V>Rpc`! zt+ZZ_mgKolwFf>nP91GO<&0yFi=ECBZ74k{>>yhZG`&sW!ev)O)ZTI?a%r-G3oT=3 zN#TFG;DE4!*~xSgL^33XsERb;P1Bpala;lpV-tGRc`Y=ssNw1Q$e1xQ>RqWo?0Hs9 z;&>;~vu4SBV7vNdYOCT6#J7@4!CGgVc_G173+KCr1t`P!z!wEHqHc!1;g9lsrT6<5 z)|cQ1YYcNs09fhU#O+fj@a@K=@30V<=4uR@@MRE0B}4l6D>430qRQ&h(43ywK0mLZ zkpCQKy2F_DiPrrjwRTnMr34Q#Nv&KjX@d|&Gantnv;5&jAb07@cOw+x96+(7%~sc> zc)u2n?bs1nwah8SRhhO-k*9|gY9pj6&M2E_WNf=*HFbjtRif*GDRLoYRFc@);PdcY z^g{k!!!EPA-_U)&UN=i6|!t|RMrz<&C*KM>3C_PCpA2eDWKrDkZ z;Y#e2tfYLvq#v5(5f}mM5 zvBtc@Awn||vIF6Ymv(F9F_(@OUX$ZQT(37?24UR1Qz|Q((6^1!(C`gngo<8@)tf9^ zfn+D|6CFp56YXv2679vkshi$-bBack{EIC592wq{;C9ZYp9a>xT*o%?aL*>y$42d@ zt!V5%HzQ=FIn1EM0yd^Yq5YyT$x+n};UaN^9Uq zJ97F_-_Rxm{LDEr7ViCu54K~kuuB|u!{a+R5j}D#0+u$gw-pfV%USuGlbHDlb_zR| zum8(?vVR0t-2Q;!{z8XG=1q2od=?QQAT|(5-lBKCTkGn(N2SRik|kT$;;xL!!yUC~ zGeA*$KVuBdr%%9yg;DH@liz>C`V2m}_JC6OaG_B6e5e;Zt0b{tmaz3D?wb9;GrE4s zSr)c92aN(O?re=LF&iOPyML9=I}=+#wJ=815b9f!-T*xbGDZuKK}BbSBBbjU1{C0` z@UEa}=!DLFwKiFzgd$I8xMfguf+W+(?Gx*y@Jf#{Q1bX{MT_B;6E_OhbIRicFj2z> zResb^#MMSP1ENo+B%f0&H&0rjzoi-?Z`u?1PCDsa?lQ|iOn#1T>vM>1-(#yd7Z7&6 z7MzoX2K%Uv-s!Ft(CUQt?NL^87)|T|TZQzTvF1tPsUOT_N(TALy?^an7=A9|{_+rx z4lH6}jzCnXVwbIttlz{Y4^p@qg3&0(HrBK(_-SH#ea2eCSzp!sYyaY!f!hv^wixbwZm&jmfuTMv3O$Pf z*attk2BS@lXwLym)9(em$ZjhmV$se0VpC5&BA*)*SAZP-xH^ZH34j$Gz3Q=7KgY7o zNidQ*3Vb+@R{;jfGo*f&dR}XzPHS&Z=qM%=U2Me;R)FlLCS=Vz!8sKf$GOEsuC>;Q zddD4X&yD?Z8iMxlq)o=Pu`#@QZ8f-ZWwT=I1CvLI2NX7kM6T85(+TOy2|l3+FPkU;8GRF@YwQ$qyHrt{YmcLBtkQ| z19^d6Ji`qsXjfBHu;u3nEdgl6*The^41+5NU*U3!9xsHHULpB(2Og)6deE)b?f})F zh)S$P^c^fY+$BA+stXGCQkyNA-W{{~asJa|IsjnA^5x0s*tXs9rtdRSue@BA*8#Z^ zKDGJ%^jh!CJe;+<*Pnfb`RLEG=^S&KFxu~aEH{ZtfoVT?Lehx(GQ;XYtKO`G`dva4 zD_ZD;sUe_`bx8!Z!)dCtTCM{8f3JzSDe-0)^e^fSi17zUqhK)+S#j?6=QXo%UNDO4 z;ZJ&rLGZW=$(r2AC;(p`$$||LdqQ#~7!x_=f@rsg9Ws~=(iXa}dx2-Oxe;HbFtFAN z!99-|l4!$iXzd8gDKSuN*GNSCF(@pvMN0XIY!CfY`*Ukq5BRf=7G-&M`(YBq^I zvaypJ5J;59hjmrj)vgGLZS(MV>EGO@P7?15B7(EuM0g%U@l&;(og+3kKvlT$%bM`*%1tQxi$2!vi~MF2i;0 z%2U^#*97-_+i+_>chm3D3t^ODraTp6VK99I+Y`jsw6VQ1*lARj8sR|r2HwRLu+7ar zM^979CWKTre2x*YWULJoq?lDPMtNi=k*}elJ0a|TSV~ay zq5{45(f2a8_MtuJOzli!l{=pbQ(^)8{FqPq!1TcN@(tTNQXw6g9G|!_FJsU>8>6H$0L(zr!4Z^R%Ue>$>r?2-4 zWU2(r^pIhdraj{XK+JW+W}*6}8(XY>yTB5jw#W8+hoS2BUIz^#r25NlLbUaY>;;b} zEGHc#0`vq?N|)ZSfoEu3r>K8qdKrg|y44qYXja?)&@Z=4)uOtrvT4x};+l=#EZu)dkOVVhY@y059gj{b1|)@c4(8V~TB7eO z_0vGLU+Z(Ucx2&Ho4{)f3rKgv>B`Fu&g|e}$9$5S-+ZWTTNT4Rwb9l*y>`t)h0!WI zlY=UT>1`HT48r~NDo@Ah5=rxg&#Vk9^ML#=TBZnyoNYSdgEdI3#Zmbw8_{zC<=+Mr z7W+~;nM{!4)|h=Sg$xL>RZOuW>r-0y^HDA)&h`Z|4Oqcx_ufB? zqJ9kR^8vyZja^lso3Yx)%j3DBgIcbOa?1B513r_<|7fc3o|P?_N;0Tfym+5_dbr{U zwG&>ee4oX577nBQZX>}U#A zatfE+iwDYmiz0u)LUip#nJMOHYSyYqjy=uKn+()RQF^K!CaD6d ze)(tw730Trulc+f3JeNahwfroUQ^zpCgEf(yjZ|K^rYvFHm8*z+j!KqBCLkjHn>(5 z)w;=3asgMHwYo6DqVTvTp>R2oQ_YF#P}?ptP9(=nNbL1&ypkN?HnOHqW=`ZF_Yy z_I0I#rM7qkLWmNWuX)?wmUqhj8wyu&MVp{oCW_y#)a+dvzsrt_SdqCH0z8J37=zsP z{A59}gJ1k6h4YI|$(MOfY-bVmh5=cV9uF1xEQTr`4%N9KzR&IFy8^MfQAkvU%UM?6 z1#K&2m~RF<68MOhiSQ)x--`-Wo*s2_oyI=;rbnF@H8%`dqluFcEOxy0q_Ov>RJAY7 z;P)f;Q^yhw;J60F4^cR8prJf)qeIWAU=7KXt9>Vg>^tUz7M~>z)EiF~WrsK$cAIT8Yx*#`ycykeTYw7eScuqndMvo zx+l}OMhjNo^@xR(-EV3a-)v*;A#7z?x+f;O2}}BMvJ!fjs`!?x{+{b_Y*JsNnA9g^l7?au-? z${uZn0@JLIDxGhWN}H~DpomVMuIbNyIT3#7-$J#-nUb6<^frZzUZbn1tuy65ee|I| zGGk_0som_MmN8WdZ8{FH+G8aCKLFK0D!@%wor-5;5Ck;$--y zuV5f9fn#|bH?BmB7IV%R*Uh+ih7YLb7sIx}`{0gQtyC$b*wM2z_oAV?A#iN9w}fEz z#!9`MrXCf4wsx@bpsI%R|0i+1uB_z;L{7 z=xWhV@y2?P3vb^E9`Xs|sRCQu?RXzJua@DZRskEzn*#yK_uj?go$xFS26&J7o?zjL z^jTek``j;)A;^2-`vjKK^z&%VWzEWL8F>9n6`I?%;oYYD(QId>5YN@?eD*e z6zqSa1ctc`bDg&F{!jvj#wzBJI%;8PS5?S;fByJH7C(5Da7hmBa+L(PzZ{<4<*!S` zoMbATQ{IU$itmF<#Itx>qqQl#5g3rSyA(owKlNgGe|S^^&mX6@g@S`_L?p-ID$FLe zK_H?hz8Ibi?oH8}sPs7DKFL~l5A`^(hU#Ka$=6N!QjL@8t8Iz3;Z-*!fO`$bsZBk5 zv3g&cDiHGoGU}3{a5R`Ct`k_3Lxq2-5XSnPX(PA7zh_Bk{xldmdq}@4CnB!g^|EPb zWo)lvZL>yul49wJ zb7395qQ{Ow8HTsSgG<+5K3k%2CO9iF#Hx{!Mjgx@-}r~b_ZjNv!Z;`>6i){OJ$Iqq zZYO9Lvt{3{wyE3&a2jl;-cimlk@T|3z2fe>o7chra-=Oh8b07haLxK!ro>Yd zlIs7%J(sO`uUqoZ(}TVEjI%}$1(^_m18-?!5$cRJgU%&svYe2j4+Q#~B+#-lBd{;mf=l)VYq3**K^Mvtg+ceu{0eiwhIg3+S8#=?m7 zv-CdWStyd*r~qCw;SC%MVIT7=W8~H_QjFH;&Sw!SoQeGQ4AhR}sz zxN1=DuKZ2v5rS7Y>qmsB@1RAhtABU_dB}*2!8=e5wrU>?i5bYVV*VAqHVnP1{q1cn zl0^p4=S#MvclVqXqJzkZ`?UvZA}Wbr_@fN7Q*+ z{0D^jSXe!jxoBP=3Bfbt|K?s571~U>YuI2YU&O)|6J?o(T>jX7>AFjqCE;FFA&VKC z*&heokfR1UMul$@4#db);h7S8;5} z$J2e%RT7&e!^LyLnhGsl!5Xx@ua{QtYHq-~l*2u{nVs^MsaGtJ+`vT({U) z8X}`uLaZL&ZpvLV-y1uuWe7A^k(= zQOQ{}S$rQ|A-+kk$2>3S2Mc;+@E8fNK34qiU#fdWXY|~)4AG^IZb%kyZZEF2t24Gl z<&I%-HMc;Spw$kFs)rQf%L_N51&vBCtZ3d};dOz*fOV#H!KGpZ!7#Y}6%u&&JXL~$T zhbjHHDs2>OQJab0+q&65YnXPABOZXzn}yK1qm(^0cgM5T3sCmv)e!!tgFW?T&7moN z_Dn5UT>8@a@cOO+FUe6_vnp@D>+#*cwWu-sqIaU+D&3C&Hp;Ws_$+)Ye`rD^e;PUrd4TAbklCvE`hLS97FK>STL%DgUC|?lBV-%lXz)6)4r{MdAhZlvN6HY+SpLSfl7HZhQt`pfTAQ`^+3%~u zSM%fRFf2jY;CmE|Nd5u;Rd4?MtJVS2^CIbU{)fl~Unq<32HpMcr%B%K7cVg)#FaaQ zd(yx}8hSqW!Y59|GkyctcuCSxt8C#d+d>dR^o0ug`1pO&^O(mFrLc>Z6e8myL+;q< zrA`|*3Z4YKpXh(#?WiwH&)s#?r1ZJ7r0lJGrOzpkfN*L<6sF!9<%PgB42v{9G}JFL zNAD>;4}Spj{zoZBXcv`R8W*Dcp3mUGYbAU0qmu5SNL21TRUxT*5QdYgrVPNb2^Kf@ zs>g_m3jRO)7xBM%lsLsOdTK+7uuH6dZOF%5nS)QBIdh+K}H zPJJ0{u4`%;jsaI9;s+k~@^@~Ql9z4;3D6Trty)E+fqVr+uxI|EYeb)aza&LZovem) z^@O5EbztbaShF72TEoRJT_Dxpzu>M{xK8#qN28?G8VGs62z%$*lDi9J2pE*+Up)$8 zzCEQ6N@vhm)!3NRev>s>%3bHrxobbnyV;fCX@G#8c6mQZzi~+OKi3bC^lN%a+QYEc z&i_p!i1pW|^;`ntne(0a--FeimXYq*TI{WD;Rcbam}aL-#yKcRdpqhY2-HE%3S!lWWk#P;w(Sa9Ei&>MY0C#LtgEYL>p?SZdh9<0$ zAl&f}?uSq%(0*eREJU7V)z^scp@gW4|(dy}| zhV_!xk9vgf(Q-+Fmx6fx>|?eEp=d{(5rK z;3eBy%HF*JdF9hl_w`&Ed@qVxpYWvSP?f)nJUV!ai(foj=UVU6Uz5TAR}JOyT+G{F z;gx$y=U>8PQLl_gq$7UL$l z+5VEX#KMAJQ_W;axsd2HrGVdY|&7vewtn`9&%} zdsvmV{#X7Y1Frg^fwYj}!*19%_ZqAljCx5}M?6@cgYeqrAmkvDe zG=4^^%E70w9XyczJ4~@l=A$*`E}3X{slMFBCA|5+CA9GgE&Gf)86i$*N_hQ4+W z7zBk)Sev8j@@{U-LYD%y``%1Vt<;_HveX~<3_j0E-C5sA-I)m6TGr>Ruchw%pQZMa z83@BX0A4%JBgVjD-mq*5ey=<1;aIYH3MjzXi)tsniRAxXK#*~=q^-KK^gQ-|Dwr}f zt!~CIs3rAp$v^x)E#~VVG*GI4q(B_d;`QOuddY!zj_33~_fI`<(S2a{O+HQHi@{V0xeVE{XwPCRN`&DPhQ;eL!`df9SoEv-A1b zTo>sEsRB6^K-T!o-}_pG2EKssn0;00V_mHL18zp>AOuwi_C1e$7#ruQ9sBOWiXGYT zFsVD~Rm`KOoBka8l+>O42Idy#P<(!!^_A3}^PSXQ^oP_RjqsanW0 z%_X=KJc63*(LRmyBa0O9nAY{6nP(_u?X*dF{!q>t)*YYP5E?T!*bg2xc*LGOK}ue^ z7{f;xEpu29E=j>EW5XSx$bSZMunCg4_myg#YjT|>F-0ke!I0Ni^w{}22MRDRC`j>{ zdvmiTeXX5Awo!t9QQoH4a9HI_rGEYt&A&=Rsi6QuLZisv%7h1+P3KZmX%9;*13DHK z$<7AB2J^1cFh{A7GJKTb#gyf0Sk7qrSE}5(8<0{ow$H;@?sb4PBuww8H@amHCcllB z4DSw#cQZGt{(wiN?(ml&*l&UGIkJg#d@Z$h0?jUyhya%C`h6de`a_?P`a_;<-skWq zu{S;`!L2StS_`~ST$f^@@lI6@_P@RH{ihP%01T+X)KTsjp;i6kYm%|%CK`6lP)Q&5 zih8h~hKj*0fO`ll+cykx8-TzH>rNUBNZgwkeZsoHnF!JJA)k_-vh0lxT%uGpa8u~A z6yN>yA5q}r@3*?-?(lDT=wSRsEyDmp$IIWiLW&U$QTrDdlJRPCZDoDxD3qhhV6&<7g0=qPckS4}Mf@1qpL!PSDjl9fdW!S-{mL|D^TT_Y*t=ODVtC4wmn&Sj{TAAgP<(9|5%R~Z$*zX+%H#s2suPYI3*5P$I zmzwyDIs`14yGdl=M(+F?yop4hxJ7#m7vIlMi2t#@#9Is^jhLYz(q}EHKk+rGI~z(x zZx9-_qx__dgBd;W40k}t?nhu#zDyk0zQ}Ay#^;6x<)`rOy+GV^q$*NQCd#OgT_)bh zvOr@1Xw}&ziN6x0qQ;Ml^*S+MdY$qjLYcosF6G}4vVa$AY`WfF7!8zsfbmlB^Qi$H z|DorKjrqa&K?92?9$M{km#w(bi$}$WV~y9Z#5Kuq=w#)fzSOYpeY;I$jR&g|K<@`?&^5?pyH%gee)C zuNU*+B`SKcdyf+`@t@4KoA;m9QG(@Jw@>D_>-xmpbvJz6gS}95%=J2GFdAiUb~HSV z0^+($5CQWYj9R@UW8GafG`uUc4p8n~SL-r2p00rE%HBg_a`VdewNmFppyNE}VzZ); zGF0Iu8vlZ%J+O@g#$DigTlhkOT{Hv_VBHBHOL$Y%EhD>Cr*Fv)1x$8-sRytV#C+1w zCv1FRb!+NFFO{=+iuj*H&7c46h5V>LvB*P+t*S*%)D}{IB#H1WNa3HX?-RwDol1S+&e>8`keNv^g8Y->2>TA>S^<+7Y+&(HzI62>X{>3 z9P?J=1OO1+bu%MrN)s!#)-0okBOzo*Nq6`+Zl*XdQX2s&$LrGj#Ah{ss`p9HO4=Gb zEFt}gG2Xev2{G*^PF|owKIW4$?jzX{9AGDGMXRtG}>=a=4(-;aX9_e69h@R5m2?dD;m6Oo+7lDraJqfP`%W^etvYnWV8|guKCrB2+IaK?{=b9(!K`8XV8FTw28d-$483W3-{arMe!(6=L6HPgJr21;=Vwp6AO1n>wQFwYa;G!r5ipPf zyaVMhEUkNh)E)Jfgm%8dEs=ozQH=%d+B#y-NR;0?;`$UFNov;tqH7YLEsfutFW%2@ zfhQQHen2u8MMxv-GSg^Kzu!YrfBZ)h>AMa%52)-l$8wUy0j{Vk>_;q5;mPz%-mYgN z;^TJdb>eN3zwhO$@KyitqLu^F^@UM8e|`?O)!r~Pg7^ff2F%}PeaM<$-_~`EYjO2Y zuV^X)jiS-2dQDdd75@#wGUTaiC9OCHz(}J6+_D?BNK;@Lw1OkP&gn(dA{_HUoSks20BnBQa1!WQz1;BpBP!zQ)(rW># zd+sM?Z-NC9?|tPrPatpVHqd_$RpqY^n-}x3n1EhC{p(mhLc#{rR32w+^W%^7RW?y? z9&_u`AfN|Ovy-)x98&2wxsff|L|v~Q>R48CgZ>Ki%zsn5YEu}K)w#0El40(?tw14u z3Tkfth4+Tk?U0pL+ezyFGY4Uq*TQp_A1kZSj+;7_X&AkfG9LzzC-Q1Ah0<0- z-bZGEdbpTdNpCLedHMRSf!vUyB3^G?X^h>U6`*=uyUZae8we zfqReedrtrY89aA5r}*hpk&k+Y%)4|o5XcdjH+BW$wbbG9o`OzzQ88T{kHhY z6J_DG8-f`V9y}h3buO^V7GMw_1Fzm2dYyTfZGfD>_jG+?h$d?mlT0Ook^|YBf%P-l zWp9#$c&mExUWry!8gFM!CK)Wb!JQ$j z?uEYKk|(vgS~f}fzUX%_Aim6qACLxY=h9=xO8~S4>y{i#>olJ>}sZhauip*TOh z>8TQgAQc&cM054_1($4gD39QK|7zXj`L2;svQc?~R*bo&r#SFt1s6<}%1_?}Aq5Cv z*24HPM?&y+)J~d=M1VyRu2SeqfNG9*;QoZ*cf!u`Or1s~} zr2Op%rRJB9z!12<<~BvZyjeT(3wQys@zu?RfWM~ZWo3bFjG-+RAH0Gb0+7$3W*I@N zoAncjTRsL0;r2RT_+D}I3zz%><&oD06XOT>!(xiOzA=X=feMRclSPvZXI|t!)J*+M zYJU0_-|r6V<{Db+qjvHqQt`<+c#x*yejv!N%$#o_$i$ddT;_LBwv4R09l|WPaTyAi z*C0T7Pb5?YAG}Z}0#UD>bSHw6sCm5Ujh2&yk)4SlZL!(WUp# z3r`hET}U-fFCX*n{5U6V$SBQ`@xpVLF=`iB>#xyuOT(-1taxl1!Ymnv<;l%e0g8-* zv~Z-27!xZp4aE=t^Y`nA``Zkp2Tx%RE|LP|q-KJ~oFT(i-+UlxtB+RFg=Fu6T$?Jt zdQ_?09mqbx-LJ&YWzQ#-jMNJZ!y4Wh;=SBDW;85-t=0GaqsYm_wm57fC2L4Z%}ee< z*vF^{(+Hf7e4{qsKep0sg7|7q1zpvWQO3aNXBa6g44poIkuW#Ue|nB2TDa~o3phn)N(z!l!N~IRD6GY z2J8?)u;y;ME=4v!30KF2;kuM~UtF8!j2G`8AS>b8wBBxbch+o?4Bo=qTK3^8<0AAF zid6=};{yZNara$i9fq=nyjo9>T#RoGW%}j%Ke5D=4r<9#4UVJQ{|aw$tSWURRj*3RSI^$2^iFK>WUefb2|Brr{Hb0Q^) zj-OAM)0Lm2o-f{U2h7#Mu~?gyBZDvNLA6_*I|qf(7vH}hgoJO9(=kvdL1@u@b?cH~ zfI?2ZxS$p-jD-ZTC}%^Wwz6>nd=@a&{0}}Kh9fcQ8m*z2c+9dCn{3Px(4wB)P7;sj zISt>jFBZ9s^`(qGZoY}e3MUGsyp9%0$5fyPh1O8bA#m${EV_SxOt@9u!m}vcsha}t zDQ1D9;-|^-%pXF{BGw5>!$j-}w{jE$TP57n)ewq%9DFC3fPfXRuji|8S{28JO(M2g zQ*nvort!)v5`301BU)O-TnlmH{{1f?$LCk%(xv+)xW(CS(Q-U(&rEGXRG zMa)^3Zoze^)kK_P3Cj^!l5=)AOLAfDZnnjSg@B=~%sX#0$=-fX>2cUSvCy{GZ7_^o zFaPvjDS7e+P?6s+nFtM7Qg|5HVy-Q|d#nodea?Cpir{y!XrC*2dme)_64$x4@=~<5 zxQIFCi~pr#z>fJi%4KXR!Tlc5JSi=-+XVG~G&iT?iuRz^(fU4VVxpugMa(r%YT+bFdkpVX zeVJPuas7XRc@8w!3=dvPD2!rL_5S#H+ZQqCgJND=toXFMmg8BoMkedn?DO#4rpGPWVpuYck0O;AI54%bmMwZL7W) zex>uW2qi$Oe)vrKU-}jV#I=&HyewS`q52+{LU8kQlF_29EBW_$NxOT?Y5j>#MG`JO0C#AO?gh;X|j{!fh#3?XJo7nX_1+38%ja5`J!*b`;E2APq?+FfF*f5?C<`M_IK z#GGN7e5bvWUKYt605Yd--S>kYsU})9043;K7BT;KYSg)-K|Jof&^2aB(}+CIg}AI0 zUkBprNxXdU1MnpJrAi3@lr_fXoQCK%Vpx_`FcFoL0xyW z>=M=`uC@H0b1K%K8fR$W0o!o#lNUmPJQd}CE@~HM-W5%^PFD&+Hz+(nPgbV%f-*PY z#@Uj;|K$*Zf=cHsRMr&upwvzI7@$83=B)QMEK5Vs@WAW;o`BGxuk~S9!E&R85zFQ^ zytK3^g^Fh4xL;y1{0QH|L#>}PL8?&1unGkeY3b68Dn z_0y}k#)}ZHCdFu;U2%XGs|fPOC~#N*^paGA`c?B$&}yfAqSs(w%7VxIGLkCH{S}+l zi@46S2(kLy72L4U)&B93)~gCZNYmr7c%_fnOmcTR8;fYnGeZh#2v^k;UdNbTLhc1Z zvtk92xu{qF{2HzdrZZ4mSB-yO@4vmm_7o9X*7tuL3{j6E0eGr}#@-0x3v3jcfZsZ$ zXGJeS%fF{F0Sr!f;WI!fo&jd2%+ZIy>IBO?7XvM%%Od2U*@st*8%u+*qW>m2hWm_l z<6=n~Y%E%glix!zX=J_RJVGB%MxNW(Qn@djW1iS|nqS3zC$WD$7v4EAi|Uy0;o_Bn zBE>zAM_$*K$EiS0%cTn7#9RBrIAfcqJIAqMa(w(Q_I<H+fwVVhDXNETh?SJZfc2-|7`5h;utjQf#YDE#`t{T z98ByxKF&Ci!qV4;u*OIVhM9f-#w$D;!2Q%XIBRjjLL7HkSd19(P+4o zH%^Vyc*OajogZ{hC8+01r5ExeX#ilK)sUAco2ii-&z1d9M&cesnI$+ErI8R~SN#iQ zx1U~vG7n`sUZk9|7{c``L9Jg6IVghpOm97Np(2$i69U<$4iNw*G!f@%lhWn)~q%tGsAZv_P{`iWLS+$pZw5*w;UOmWl;nBw-b!8XV zQLqffE(K=WKZn3~K5=bIID$%;u15)M#QJY2RQfX>Ex`3HzRs|(xC2E)FL}HhmhPTM z@)>t;b+4O9T;ztDHyqE%EUruOb(HHN|DDhu%^dJ8+B3RHfvTX&6+y z<=naMPS+Bx8an09n1G=kGHCJKFiKy(2p+y;H0FQt1IJ6r8%WUPf`Nsm0&H`||2tla zo;V4tVW;8uvvKS}m_wKZm;elwW9~uGqemiS;T%m13zg5s7*FAS9RJxk z|4=Fa@JQP9w@ZmG0(jjD^Kdp5z(@;CDc*Sa0EI5~Kx6<Ov%>w9y?%vWmj(g%{cyd}VT9?`&-fm7)UbiUI>y4-46EZNTKw2C zQiu<|!Er@3wD>P6dHOWm)4mwbSyKG`nQotP14py4Yrli2;5n#F>M$gM7Q2oX!kcsJ zn#84??n{p5Cbab5_@2ZU5yLnvA*Np9Hl~xecy0hTr1uhuAA^T zewWa{Mj0e3%u7cmH1yGU3=Njh=3tT{$0%tJ4FvSm>D)@Sbty%F>abzB-W-M*_d4NO z>2v1Wh){SNgg-C9fQ6#w4a#qa4X{&Pm@*2LBg!>}_nitEQ2!0%__*hAD<>dK=X*ECmC$)Au@8&`NbZaCfS%3OECS}Lg5 z@lUD2G$|p|DTHFXDR+9$8d6y=SA*9EL0L)A*}J@DqABx?&4U# z^4x`*k1_C;pJmV;zaeC6HRSHxuGTPR7liCz`^P8J=ie{Gt8*(tuPSu0b*a?bX905; zD&*Cru$M(06Br|(9W8ye@Srk0A_A*U)%e%3=pU>bMF>iD)psu<`F2eyeEewOISX_1 zHINJJ>3Yc+nTAndv{3$VMXO1J*505atuBERcuaSmT{tx{xx^p-5_aO45@a*x9)J`Rd21ASCAzeHb&;AuC zo!_hXrUtnWrLSHli=jl;!#m6Zq6=@`QI!KGozK5^2dRVS+VN#d4d!U|x6i}y01qaU zmr}KLF$-F%!rXnvMI|*=A_%$<>dn7XJR9tJUhI?HH~H}(_r~>D+H-3*j_`spT(jia zGc}@Qz;$1u5dBn*AgDmzb}c+#)jvFo5Y8&dDeuXUhl9%A%YCU5ij=?qNEv(|WRyn= z!5p|tla%m`FkzHa5e&$M5C04M*DJ0OnYKp;wy2WAjaKdT*rVL%M+Gljgm7|%eh;|n zBh&#M1y9^J5=0(f2wuOkH}2DL^Z_?~CH*gZ6S=;tNzN`8NY2)W={O1w{;%}A5ON^H zJ2f4K(K9Hb{moNQ;9ipKZBLM)PnSWSLhZ{rv$1cUrzL3^RTQa#%!tACzxs2?hFgQ{ z0r}A2eMqby+H8#Nz%GLTgH@8Ua#5zzRXh8#gAwWqP7Js5*%#fsFHiJwF>vBgY@V$e_e8ZNm(1xPc> z-@nD}k4g|MPfD0Vg%JytkMyu{QX5??p{z2b0_$TYia;`Kg|(}gNF1&VB6MZ0=JCPN z0al{pl+1w^Pz`NI;<18fZ3zO{5E$;W@b&{`!GHIV`B&|WdQk{h)cgQET7=T|I_7cE z)ISA<9${+`E|}NW`}A>=Hhfep+;*m$JINFh8ht236Bo>1$k0%>KSx5OH~Ioloap8ZWl_Qvp)p62I{n+FTOk6%so&V3--SW<9Jijm{^Pi zW`@;7$(Co$5HJJ`fmA_2bH$n8$J}@Gu(tHEa8IDJMB{M!nmcJ;E=hILNA00(xlHW{ z4BHeDLtqws<}`Tou7#1ir;=Fsdk%*jHBAa(B&`1FWyoafHywUW`#kp0I@ATLpZT2{ zpI!TL+=v|esgWk}zq=_Y-kt-Yu$`w&iD|2D8LLB9DYYnCPem{jY>WP0BQ&o!d-J0q zYGJR#*r+d(bc4CYZq!#S3Jmg5Eoz*BY!z}O2+m;4qzvRAFc1+SQ(_-SGi-MRk-X64!l|Vo=?FuU8~B6 zQ8P#L?Z7h0mplYd39UJ#dCuB&C#|pJzSJ}a(Q1QZnT$(8mkG7Z%VUe>oN!~{0>>~q zt^;AvlQRaLvDQ?^(&5?T`DEAYaqxdMgrMx@OEi2_tvRH0#s(9@IU$e``1+0n6B(G9 zVnu>=$-;Oeps8&jdr_Kc#0tm9$%A6KE|s4>1nTI`L5==^G9zk&XpAgj$?-T}V~P_k zm52+D6=8J42wXxK&9`HUO+&5&$Ah?e3hY!oI24X$WoAZKJQwqX8${o8UXgy6zAXc; zeoHrm*wj%4+&>hQ3oe|dQy_iS zE!2{ohM%-(#zUDKJr*^VAV&(vWvsOejz=vjFiml8FeI5o1gsZa_0{8GP()57JiUH0 z&&7)ocg1IH_hbkd0){}hL4f%IbdkBGbQp!A&NH%la>r8Q2u2*?|Mitnpbi3o-F3PL zl)ZM9EV_Lc4WsOFK)wyEhiE$7tIh(Q@scvIrZwZkjVKAd=W&)y<+|3K&>y+VQ`N(=7DqA&H_-T~o ze#&#TRvN!2Lo3r~c&G|M{7)<$C`L-;_GiX}Zu5$C(;IC!<<70Aq*{l5=+(-Ar6yca z11i!-RH>q+DFWF!Z}&^3_ceM3FEf=k=CLt+jb4sOC8$tAK%fjr zaVeeGAuTExD+Q;KjG*_K?@RBq@PQX5dkcimkzyB&cDZ|AE(5QIMQh|{SlGVD0s$`? zJUC49re~%1IUlLlgCUzL+{cwKy^q)}Ev-gzk)>iuG7GRARek+1#vPRb*G|P4KGeLg zB?}cTTs)qO6iH=VhE8pI4`9^8Kbe!K$}IY>MOdLX$18r&xatDvu2zkIcGC&HyxS&3 zRLWVtAONavTJjog7@EW6T8;Uf^%E}+-Be7xcF4Lv%oj!Myrz!E7){YlHtJ2~!ZV$- z(>c=P@LS+j+bzc6L^}XCA`ofB*vXcR+kKu?=G?iqltr}38{{}$rdK^j_cGLRL8$~! z2FQFs)>Dq02S4b83l2sYY0O|?Q^XK31PlQ~pfe&6U+1CpP@zK<5Nj7wULY@5aNa~$ zu_K|}jBhpC!w0We1bhQIJDef?Z1r_Ufjq}P=Y9->Bg&zGCxB@t zOZFe#m0bHel}TklbVDThM@?72uVnV+_e?i%3r{cI8iXeI3+#b`J}iX=8HYRpAPA$3 zM-muKMF1ZGqgrUV)^Z35<+`*GHOjouE6EfU4+yKsrx2Vs1*JG3H#O|pF}c&-Sv%vy zwlNz5Ek>Z-lsjD)$eu-}EefI#KIuYd*jV*1U@!tRQr0G@y9B2cJ!pj>fGE6YZ<%-X z1_&4US%q>XWr6HlHtuR!aP3A~4~{Q(*5*gK$+hfTX_wv71nFz;h{dh|IZAiS;#;?o zqK6KV`B$thd^_y$a zSTfOR-@F8`6W8uoKaEUB6xgxI`JlXo7S7T_tYo7Ct|OTX0?5hkB2FjXL-aO0?)24m(ZL&Sd=wui=z>Ch9bTw_{n02ea=9B6G{Pu zQPLn>u>i;Kf$xVmxamC^nUV{J8`h6v$XcJXpVc^R5}>57GZw@o$Y0X@JD07F;h$Ms z9-~SS!z}3?)Vw5kFS1aymr7Cn%pXw2eizN1qCj16z)dL3dMOr-9#F)ls8vdjMIT)7 zGS=G$mG4nhS;Gv1stl&Dy|c1uGJZX8pKI}Ng!3Usjto@=2i>ORFCm27F~lzaV6ZfT zIE9ueFaDjk&ovk~!thWhiy?pkP@-pSFj~XxNcqY-Hd)&qr%BQb7tG!Hbm?=-OS-`M zPzNt#!;Q6|WxO6y=IBk~iA4XppqZsgUxSze7o0s^=3KC;%sFFSnSJVpQVK%>pId~C z;s{k~s2v5$W~GBxV_$&7JQ3a$BT0u$Xiv_NI`Q zeIdDtmR7;I`d#!jLPif%vONFb4Zxdl8KSs~VDLW}WipNg9|H0RA?LBCT>2V2YF%J% zc;$Y5K8T{);FX8N4*yq+g8xTlTSRMJ`O9J%%6I^bTKu@U4Y7~C78y;9+s z+87LhBqPvnMa*^Fc(G~aQAohlPv+aHIM@EMQ1iKWps3Gs49qPB*TF4OJknb>p;YS&G%#pqGWk zVkvM_sbbTb8(Lq6*NN2}iiaeSZzpyS8vm`5Hzf90h@2Mm#5m(cf%W@XF-NzCSv&?D ztdmSUWDZ@dh7rU59Mn5=?-Rc>KT(TUqZi~KvL2O=J?5>`x@ubMsEIm9E0{nnEn?o% z`PflK%*kNNxubNqvCt0A2NrWCgBQa{sCWg2Y^b;@Lo0r!t+IuZ^^mlO-nI-V@x;9P z=a)6ymr3fO`IAA0v$`tzn()A|TqJDQ!kec>qcO)B0?JL;4K?vg&D75&3}uh`ef+Kl zF^mA4BQYGs?op43*JB6{zY7fCSi@{|oR}wRROa!nFjut>WGb0p zJ1aw(gc^8`dmizCZj3dkkytwJMr_{4fhgu$^p(7#y-m%7mKHH@@mxxUBIYd}v>o4K z1X^3fyrmav*COUE9h@DXR0KM85pyCzIfS)P6T=j_=ffzevJmALEh9nFE#~9L41nYm z&@_T7E#;$LI2a054r93r4{~ujZHj5Xv|3`_6*flBRV6W9kBFb-C{skNZ z>IHP8k+?_dF-${=%UmQ+;dL?2cuiMVo{9BK*^8xww6?3uTCVaD#~70txi_k;#}uCU z_{gG8(mBdPw4gEqjMvscoy;j$#Jr_9V8=HbfetQW9?t`6_ITX`@q)U= z1!MVfae|6?fkGZbcRLk34Z-AM1pzds_-JS*6&t>k>_dy9k}jF623~e74AG?vXKNo| zj%55f4`Y8Z1y0XpjS(tlT>Roi6nPxSNos$DwNcns#ZIro825@s3Z(n}lA?*KE zF~)PDw0L7|jQl;Jn4?>O5%;8@SS>6H-f^5*>`}*!{#YPVh4Pq3j^PdbE@7eKzuob+ z!fThXS+wT#7S4$FJDg#`43n>Q2-VUYT=UEQpdbaObIH6%Cut55HJLmLLhO? zY~?;@n077%TCq(w?*`j&E^!}0GzW9_;~$j+G<*Z!sZ4-1o?(pqJ+7!xPEff93Xc|c zj-|0^F?vRZP`-&Nf=k~dWix))WbHv$wsDQh=!?hTr>kHxl_e%a#q!|v?};)Oz96znpDPksgQL16Te4SJ7!v zTV9Cba$d%ztye9+X5tAe`_ixM`OO%-DmSt9Ny|?(Q7~0`X5w z0gros6O?=1AYvPn_RUQ}t&T1@4PMn)pN)>?@tU6iMLc#bcQZ))z)Q-M)qmn@_uHLi)_l5ib!zrHpu>H?%sm+ZTa(%2*0UN<-})a z*Qc;+0Wey*bz^8}_l@wd20WYqm=}|WA6Lh*u@n;`2twWXm8(qF!;N22U9`|5u4A{gh?M|pmaY? zt|-2{5PlW;LrJ|}&f)Kr90`P!uQpsK5tO}LDX6h3`Ekw|1etNn$m4aJ4+^ZaJmw>* z6QJc7bRd(0w45>B(fE2eRQS6_-AZnmq4YA@F>bj(&PccMP=Y? zjxUc-CM{_++bHv!+1ki%*(VaC3?^URaOD&Yi0AU)61B`@(lz9O2+8tuv6j@$@Y)yn zu8b7bg%LN}OH?Md(uYn=1_g=Yby+~fiDD28NvJOWQJmafSYv)vY_%$(I`w)J;3#;) zFzj*QIa`#`nKSc)Va}K%FupyFjZ8wz4?*feQx2{c+SQrYVa+}wy^ni{Mk#LoC^GVwWxThs@1NC$0}qlrUUtVYTJKH%gbLVz2(l z*y`L%AgHVTicU-nv-3EnJ@W}}7}l(P%3O?+RD`V@ZjBKF(6 zzWZiz!5F7(<<|*gsvp(bn9VDWb4@Lp?60l|NJOm2y8s{5^{6Mqlmvh4kLvgoY zvx?1nr9N@39Om((COEqd>+>fEkqFx$3;U}Po!*iltv}+QD(zziCsM#v#9)JtB$kS{U6pYC$adtaPyOzxJgZ5=J8Qw_MHhh# z0pPfa6z?L}JE5u6v1cn(+rbf#fiF-6XW>NPU1BUp0rJ#`IsT* z*!SNfH%v8o-+~{NxqA~lAn)kT8*$OIE7qWKvG*!`m51RyBU*q&BT`2HCYWwx;lMOU<6hLF?JcM;=>A=r z<(O&}faANVeN!5e7%@2p*(qyeVLV_PTMHQ6h658F>~hBH_8o+ zkQU+y=AP2fA}nLOpK@QxR{ZIf8^TrZa;U-8+P0v7Ci5=adoB+Tm|^} zMs%YcK=Z~OS8gKX_stz4yDF0YXvXqSfU_$M+TF8p8S1$KY!uy#3@%zz_TL*xQ<8|t z9z`e4AKZaF@GVB6gIbDmK3lYfq|Brd0gsSn!e`&M+Md#L+dyZWGm=gxc*K+}U-)fv z;q&B=$@={1fM4V&mH9liguZInW-#<$rUm-_O1ZzWnFjQzS}4M{c~gA?avF&W(Krdn zFCXlhcZn38rG5E|a=#O*)fE>;yDA=RFwt`5X=C$n$2*1d(PVfJPl=)XbyY*}cN|4F-4ZFh|}3 z?3|r|Es4B<8H+r1zb2H^D|-X|Kf-E{_?{Pn`IFn<>5L zO7}0agie4x8J-{KMu|%e1Pj(~{eYuO`Ace?mRqDlDH`@?>h#w?yXI@aI-qJTh0Bj; zj%*p1Cas#Q1PWVXok-5$%OnYcG4<8{&>ewiS+faWkpT zfE)6h=pf3cz`GJ+a|?36Tb02!Rkwx#mUda}LxNyWvL>!uk6(~n{(bQxG&4%#p+Up# z^18Fyw0;WMPVYJhFc-|8bzGItbWxfDYLnxHuGOw9dtu4paSVWQ_Tjrp_9?FV87W_P z4I!du720=KHW#XeFWQ%c5BJY?>F7z{Y8&}iSwhd;IX3Tc_ph4k09eaq0bixeiU!sl z*K&R{vE_XT8A}xbuh78EP`HgfQ!d#;cvHSR~;xfmZgGBK|V@p1dg^-evxu;-bzUDop1en)qkCTNkY&wtfgSBoZ`-e}yv9lhVq2XsdsBC?qpn95re$16uS29qh! zAr!`2avv7XjqP*!g&yo9+KrbTblr9jXcdVA6yCJHg%3Ck)pqX~eRjbFv)XEn_a93I zSgwUZYo~@I7rAAM2pD)Gznjc|B%b?h1Qr3B`Y&z4n>KNRg{X>>m!Ep6M#<|^d*hS> zE?e%;pC6enoLIg9jq>nS(C*~yz3~HF%}w2|yzJ~Cen}BHyYwQH_xSU2dA&FZcz>~F z)EviZ0S%8mn`0E*YkiPQ8f+r3g9Z7rp`yb02i9Re_thfQTbXVmId-kO{KdD$Jp8$h zH#d{REF)h6CMRTJT+lFo^CH8&a+i2Q2hW%o&LwKFYl4s27sv8CRR z653qBNjK&2w{KzVB3aePu4z220d*HSl{$M=^^9MV6M<(wGBTzBm`@PkQa#81BsxnI zy&?}#C(VfjL2;`&g!t+raBRTkgw{v26&5nvUM9Co()1>bfyY;3d-6w>sHW{n(simU zg|aP=OZEkBcg`L?dQX1_@_N4%GNgZ2(Vk-3D6D%!z% z#j@u*y3eucJ&_?ZTs@EaI4d&-Dx}bJMWW-d(TSW-G-OE50Zs6`pisSOZg#ckg zVjrDJY1Ix4n<6Nf#6i@cj^&o3_inqo@B-&@8>rTc4lU%4>bflDBHGQUYYxaVbZ=j` znfVCjZcF8RDG_r#*_EJq^fgg}mpQz85T9%|Pgog8`5Y-%zK@^|gJSn>#Z z!_gkX_j=o1St3a?i`;oobxpN4*yU;-Z;#vI_9Upc@2_Z`>r2zJW&>`0r(q6$|Als*Gt>@W7tD6XnJz546hQgnD3dQeyuHZuNw}qyb{hW?6;ZH=TPn`b1FoAH?Jz zc-~TGU;5|_4 zwGB4>e%G{Tr;spMqEb3bzsRG-`ASDrp1vvlZ;bwH_f@!N5(8(Ut`9t|T1W>}*mS^z zJCL*pqS5QFPkk>v?Z7?Yh{}(@wXvgS7=&uF0;_l3`diDtLtMiXq{R^>R)85y**~xV z0(Kx&JCNB3lDlEo!bq1EIH`n{;)Z)N%+}*LOCPT45@pI;$NNoADfg~k%atGx7LB5W zk=dlL+bVkp(M>UK?Yh9e#K;oUh1#@Iev|9A=G>~kTW*10HGL*#rEg_Paeiv*DNl|6 zC5JYt?Zq{}ZOLXc#MpjqL@KYjSynq4FAYk$nFWM3(B8p7#KUj9zQ0a5E^yY;n7*@# zVc!jb2(5p-#9NM7vDbAgmUxMZD9%4GRTkySe8G{sL}CbtVT3FUK$K| z!i*2y>fK)0<8BBFqyI9#9C~t{7P)M;*%VAKBtg#&3|v{}E65_G_q<@~d1qWz)hEaa z+vTVlk?!l)xb7oeHVuJIYV4R z&A%ch&2#HDSyK(jqo8!%1>a%ti%FFW21+lMSUcCTRNPB_d*?@Y$ucz#axGh=OYS=tM0yv2 z&^st46saNkH|HGBH($AP+svK+%-of|l0B2P)?3zopY@iFAH&ZA=T()V$^ZcY0e}EM z0Dc0v3lI_zoLzr5A`+srMoLUfL_$VNMt0W8DX1vO$tlUn$S7$jDXFNzMn-XtmWKM= z+5OofXVcGSffqG78Tr|W|GkE90_eyAX~KLWf@=UF9RU#?0sb?<4gdrsAh%}>|Jeu# ziHJ!^L6#_~zyX!#LH3D=2tj5^h>5|`FTwu-VmcD~t2Y%$88pnut~oP``p3Q_=Tt0f zX3`wQaEU#0380{4X1Q?j68Ci;UcMXR5|UEVGPmyDQ&LvB4^`EAqOGH=r*B|xVQKZ; z8fN4A!p+^o)5|+BC^#haby#>@{M!UX;%`aGS=l+c@AE$77knzOsH{R(*VMMOes25H z-qG1LH2ihs+vwOhdTxGUacOyFb?y81&hFmN{e#1!<1@Jk0HQy{`b)BZk&6zLixAuc zVzM*22napEPDDpca`h%Dy@CdrnKQ#RQGaqq#n^Xc%@mwsni!^ME`yZJT;g-w+h?Nv zF4_N@U;+O}lKmyvKjcCK)IvtDsDl zS(357iU(w6@IVDO_Cx-1|LGAP*veK(fgG&hfyOAqrai`|-jtB?Z0tJZZ!-iw-hDI% zw^lpSc$k-PGclE2aXEfNG(G0#opiI@hKtpBz;Bp?y)ov8iN!mc^u8@0wp6P$2+Xpw zQ*Im&pdhDv*1yzR=*jjLRKOnUOar@+3ih3>ykE*;`%|f)&IdI7S9x)LF}oje1Brj% zA)H34X(moDsgi5_S8t@P_O@e5Yqq78R9U+ord_bpt<2yL7nOUy&G6{ZC#x#E*hbjQ zf>mW%D(Sk>VlV;0ZGsJS4f~<(OlT|43V{cj)cSD}+IRq_sS<(}SKx?YfZ>5-hEdf~ z%|+af`pN2OIl@xUd(6_HJYm9C#WJ@(Y1!&xl#O1fYI0PZVd2)UG7DDXpmpo`k|`b_ z5QZF%uwzDNXi=*$IF`5`4=iZc?dQW+axxAEvo|#HKtB|+m2Hd%#;S$4@W8AZ9*C7j zRhF4ffxLh#aGAkdO?~g+fp@uh07Jiv2ck6a0L@Fx2`(h@Pj(GaHiCT8S zb}t^Z8ThkV6bcXc3*+uV{{L$K&GcrBk@~#BKmRC6a5FT&@trMoqgpk6d*#P-<5~ zYGDa7UHFvx9G{s;SA|h(GMg~nx@A{Zl5TIt321#Y$}y7gt^PAxLxS+xw8u1Zn#R8J zoM%4>mB93$pg16sRzYR|1w1YEZ#aZ8{kt~T#p}ehN2|8kyM`xcg6qoq!o3bDSNW5} zmr~!M+aG%Ll0TZ5Y>LMzI^qFi8pDD(c*jFL@Z(*f+i;?<=ueq)nz>aaHy@_h*J@nO zM?>WCJ|vSX(i-V2tml3v!WK4!a5L(06ljh7`kNCU!VVY&Nau)XYP1#uR-~JLga^)x zsj!DG#=(T1Miax5t#zPmf_@?vQLl6polJ${l(Qn8Mr`V>k{Hjst;|ve2@_EEkm5)d zip%O0Rd4jdjHCqvIiB30+@LP^Q(k9SWAj?|?kjbiJ7wE?To>O}UKYH`6AyC~lF;lK z1LFN|5j-|IOz&&gK#Vj_?#eS(hpJbn>3Z<*Do4t&{w$Gd{{abYBr-vR*laUn6K{>A zn-sSfAJ16-+_lr_o@EigdtL^e8OK8oNR`bImS%CJN7}*b*>3aV__VlTDTs#Ct2Vr@ zrO}W!_KBF7^NxQFnep-H0(2G9K1;By#$CP2s2z4*Z@jzv>Qxn@$c6m*vU=KlJ=;5!$V-Be`^=K7 zRr{DnUr4l#%#N~IcO8ETPr1jPZ?xnJ_2#+Kgr?M~F;nmMxZGIz`TsJ3^&NH=jKYodo z{3I*U`&qM>QRB;ySdpRB>43MR!loWUXMuOy`=q&-RQh%G;p$~j#>7jn%>7rG2_10!_TZ*;T8z%O#XkG*<<;{_hbqnXckaKvi+cBJ=Yt?Q$F4{8dR)o}uiq`_AXNdi$jD{>6O z$)696Z|7sCDm&9#Y~K$T^$gv65bwVjKwJV9Qe!b|`Z9krH9oA!q|-xi(SUnoBWP$^ zq9ofxPE1KCm;W03_1k4-VL^tHYB5v(!OGKmfXDHxT>C<_z4-zS*1mL4{0D z$!F*M$tLO0c_ltgcX%tlt@q}p#}>fo)@mwu9PFu`RhHt}I;PY(fhUoq{oQe-UaLyL zlLkBxlW|C+LUz20tK{Co`htFgjWlFba&T{RYl+HyBcNa^8jZS4HP^&cycoPa<+J9NrCIx zZ(mAdUgVXBSXNuH&{A6mu!u4z-(`ghiHdf4;{kHP;pDINb!wOSO1SLH48IK3G)rq= z3Js05B5PLD`iR+HM`yd;d*jH8sIk==_fhi&0Wjsyj`IH#o|wGY%Zn zirkf*(~3vaY4nHOQeL`@ysv`k)9VBWy@65{=C=}-5X^25)i2v?{NE3*)~n&+S9>l!~4m3Sg(#xL{|pUM#K(52_&I% z!RBMbduawV4xJA6shvdSH>!qeZnQlTCA-MV7W#qfY5KJ-tvb}9`uoX=W_w33A7VOm zgbY_?tN0}Q4KA910wZkB8zMHp@HMQErf&NDNb%)Iv6|A6QlBR6EGCQUjO-(+dC?*+ zmp$YFnq=NAKgxU`JirdYTtaFK!xvU?L`D~5FpbaMO|ho2%a95rHp1jG9w`lrRVr)k_P zXnR~cy?B&N>^;?jbIXEhnbeYxJkhR^x?&%&&Y0;$%p=I_Pg5otbJLABJcpU4enawSFHUu>GX+)d{oi+ z_*g4rRl&eG`Yu&SxxXbcrpNJT^t2W9?FZkvu9r6uF1NcBSq`En7}*G9+ILH32T-sr z63>)bJP?TWGRvQy@_gPtD#@J8xRyI5ez0TG!HFcCR9O*Hvy9*%58s^7P@|q9^&Ex< zwy?>J?rq%C-zuuCDb7U>S6+c`Nl>k7UdTjx)J9Xh8tNHn4(=jYD%lNSSrR_vsR}Dl zIAOsOBx(&5E!BP;+AExZmR9;iX+2CHfOR3!MUurLK3;Q3*g9v3_!HZRuWydB6w8o; zc2CT+pRW>q9`w_{Pxh{zG{uDHMDoWa_J}*TeKo#erQDvwGi|&bLoA0&ik3wUCKb37 zKCf}XZ)#LOy6i_m;+j4^{v>Ur2mMh)iSe`RO!*cz#McWCm^02C6T7V;;M*9~7F=}~ zGJb%&dq)nm!Enb@!drXYki$dM15~cByE7i(l<_V-z0NbTWV$145`za+Ez!JL4GcY$ zXOD+;<9NWRJScgU<8f@~+}9TB8?pKK-Tl)=ZnoQnmw9+F<@pOy z4X1zzNRmh8?(n|`cO^H~YLO5^Z@Pc(s?+QPxK zUM#lVmpayg|F>l?Cax&-^|3iU`Y1oAm*>c(E;gEfYfLL z)&bH4Mh$tAv#teZ0lCJ8)bx)QGnAC9APJVJ&sT2n^EsFp zenldm>o4B2AJC&T_cUL^*17gMNpn~PkQQ&$)FG{R{Eh6LnxmJRyD&|W$17Ns-z4h_ z^gLXLn63o)(q-*qr_Hg|+pkhMgVl21Bn^!*MkP_lAZGwMLiYF;K0cKJe9HESe$!rSd1TB8&gYIqxe>WW)Q zGJGe=8lJygR}1iFh5z33;J9)O(||7I&)+ci1@%QACP<+A@>4K>a#acsl!O$IKp~6Q z@xaf4k=nKH#*5ymMBO9;!9$b^+DD%9RbH4fQ2RTw=!V7@`V&N>yCd>6$L@lJ2Xx4V|56SIjjLPxldqy-#CEuPd zB_7osXec_!Fhg^P?G*pkI(y%o%R!UHqqm%$Ta7H}eZ@AlE@t}mq~SwGRPE$2RG^a+ zB1f?E^qi9j8I_@DHX}jMG+nGO(+8NQO*E5rM45%V-RYcv0K`-w%}yg6Xd9H1aLe($-U^L*97h$w@0ls}=5z4t z2Sk+?)b;++3SWGN*X0<@p65=YRm*MYOufFWny6Z`T(?VdNmYJ_WDVA4O(ihcdrWpM zgT`S?oW@7xt3-FbnX^|l>5-YvzmxWz#_pO?VWbRtPB!dOUh}>@_e+3_rUoJ~D?a#qe*a2#L8%*K7 zzl*Dd6#H(dA7mu$G2(%##CNa4c}w4^^m7!Ti~YZ@=OC|5wu?+~BnwOEs}tL=x`>U& znCn=-7X$s7U_)DHGkD&l@yz5}@;e$h@u(pGbmOF$c<8cx6UU7pYUNBjlDAWdt;wl4 z+J<4PZ`Z18(q%ZpcMRzgqTUvSajIy?`pJDdGK-N_e(0bf&$^Kqz^?XnZ+v6xqD5JN zXO2~jYmbq9VYCxr4q#q#bB61?rz138F?!B2)u>`LZpwk8oM6Iz(#QLtaBM#EnV(37 zFok-KCv!XWQgB;wZ`#KoV}tYLPj6O}lIZ!+9kIo)p_RrlMa&G-Dvqx7L9o5W9c8BK zu~JmGKqk@GfJfhmNsH|?C!L+?jpe$o(@i znbJB7SYi%ooU9-%vY+t4JT^x3G#9mKfSEastYQbVvG=APx}yHwhb;90&DJ@z2z9r; z7e2$i#{O0H?9>Wcdc}nmHhql-^P#2bdj>Hip05x6?%8AKeBtGe>-V@eFGChidS*iK zfELoRx5isG1!h3e*ei;jlJlX53qX+<`4b>s3Oq|4C%pj_9qHlnUZt(H16nWwtxunKyKX#>!k*UY&4%4AT!6DDAPS|Yk z<@QU z=uIzCuz~<$=>SSE6D3-(;1ne`VE;2!BT{n!nr?U^;R}3yMbW=NwPCJ_nc9ChU#v=D zR)yIQc=3?`{>RFS@S(nMRPO2S#42>L6Fr>1Jq1j3O5O>Kk1KRYHyJgQZn7S%pMdb3 z!3G~|lfwC5@wUvs`U;=S4ApnDHR*l~i~~pGhGe zfQJ3_(#1xMX5)dOB=_)JgC=3izaPXp-(GyEoMPMg;aby`!2R<;fMlOYibIZlYyKA*a*wG}{A++wplH zH}KjRp`WhByiQNnAJCy4EzKXt{|q!bnO=A^{nW&?nz^&>xm<}lgNc3*A~2`IaEf8+ z(G{4apV(0?y{ycyu7+7RP%iXvqaKxYr04d$xJ&+mTCUA>&Po$3?(KYCkl|7M;~@Lp zVA58Ow@y}$w7c?raCNvNoc4+py!ke3`C+pq7<%#Hej1qs4TgusMBh8JwvIR(`2;O+CwgIllY2tW|0f}PKhss*d^FNA^1skYw zH%+F*9Jh7Mew-vS7Yv^8s=u@0>!`DNzceSEk1Lj}Yv7%o;l^z(S{@$3TE*GGHk+p) ztl$_h>(lIsTCm)T_zk3@UL1O+Bbpp>VLnQ7)9y{#jsj6C01#H4d7*!Hz8O4Fge7^w z)c@;dUgdE3h^7Hg_?_yt?}#w)b1DclYQ%~ z7+c+=nQ7LyV9ffm`PZyh2RKhKl?<+~@yGPmxQrd)Y6ut#%Yi9*lvT{Hg|fnL7n`oY zJzcbLN(W^BMqQ<{5WBT?P8HtKID5K<-HykdCjVoVaNhk&c-rY)r}`1gB>S02qVSn= zD9FV;SXNyhRAT#9x{Iop;ll&hPC~jF%SEp9xke8r+qv&5kJEKmvj#3D^A>TdoA}hF zD3_!nbhyrFz)$vJ@V__>=ZF6;q4_@(QTfH{KhX&MrIug3@@At)a2^Mt+>k|6r@yrF z7pqE)J_e{h9OnOpaw&ePW!NfQzvDWe3z;>z`JH{X2p78#)y=!ge5}Yect?dL`p1pTQLmXyzjE0E-gSL}2?ouh#yP zTuW3d3##C`5GR3R{b}+2AK&pn4`fc&a|;)KZ{|?d5Wf8a-tu;P=9DoEg?U>Ao@lFC zweZUAj6M|R0(|E>WL(;G2@iDU;>_>>rB*w9PZK=IS~Mr{Ktwi1z z8R51IkTm$L2p(9xd%{D7MIDKw{-*?EHhiD&mSRcSv9IA4b8xVDcs=nb6w9Z61rH3~ z2f>p<%@HRQbEugviwCk|rEv6!8LTm*#%XCJ&JCN$$0fNJ{V_;O*UO-$K=X+-~pv;j#zF%)M4$vC`v;Iyu|4i-$JLM}TV=#9-LNc9u2kB90k}_U$r%6zG zz97d=#|$ge(LPk$R+Yegjx&)nZ?l(CH$lgxSo_dZ)4{u?~w|5pIQ1DZ6LSeYq*fS86^PMRk|e)bG1JxDDO((kNzV)XRr9pi6d z_5v=XbltJT2Ua5+q7el3W}R6&M6mM3RW)%m@ew;vgU(NY03YWQijg z$x$RH$&w_eId9^~1M~$N2UJ&7)Kvf;9sp{% zAAlynV`UeM`v7Qa0zLo$2_VE{0z|kPjs?i!G5@Hm;PC*0<92)i0xSUG?>tv=?<2xV zJMQz_n;;SIcW#`VMEt*T<3CR%_*)%_A9aJ4fTWhWjf0K7xy_?9qUVHxq>{QO;gNJ4 z9oNquSE+hgg9k!DKH>ah(pTShl)RvifSLq|<3A_BV+Z)ucm&jVP#u8dv>?JersHn7 z0uP^nkcb#YLP~Z5$52EC@bL%;@CgZsh>nKA^Td4zgw#YdznoVjrq#X!W4EOf@eWNQ z;ZVw}roYy;$|-u+&WDtYf$`L7rn6k!JiL5j;uj<&rKB%iR#s6}Q@^5f{l-mQJ$h=_@BL`*__L?mP+$3*r!kss5^ zBcl945KaglP6i<%Aq@AYIzf7Z>iKbpE%6*&0_GAtG(WreGjVPyFl8y^S zG>3PG@FA|0D#Hg}?!W&F*g`{Jn)(@Ln(g~8>g#?~61LNf>1=G*(YBzsRAdf)!6#jj zVC&^mD(vFYYpu4>aQCbxH%)E?cN*LE_z1h84fz7zZ8O+b+|{OxY4v$k+=pAqA>W!k zzo0C7zXf?+kALZU&Fw}r54JU9B;c=S>&Ri$SzAKC&b~D_pFTE*uqbG~&>2YYQBPj6 z3`^R`S*XZGKp=Fb-gs*Y0?C>JA+W1O>Tll6^J96v(xcnYrJYjlak3s-wb>AYKutpc z*6+{+0(dY8FsdF1n4{*5Kg+LDq1GJBl|3o95-(^eYDHYW&ljc62mK zncRSOjtARVgW}k)-8Y~;Xj$g}=Sp;#Lx9eMV3I<^DZ^UTFmbyQD^k0iJiAHGh=QO>wd2`6(laO-0DJ>NTWW-i@J9k zXPd)`!|Wie&Brzf_&ujQJh9h^#izuc3)#`@gg__eEo%G>1SS`epBDzV2APMXZo0ca z(dP&yF?FzwFy-(eks{8>I1G7$RoxEQD(2pe8oq=2r*^zm;34NO9z*lKPVw`R4wK6Ruh`AtJu%=D3SeVsJ}CA*Pm_wuuJu~xq<(S zQ$vR`nUFt%{w@$}y3HUfH7O%ctKa5oM&3!nWzNd2h0b|QLqfRMfBB8xgjKs73j{LP zV2A$aAP}aTR=wTPU1h@T`{cY`3Tuuh10%iEod|uKJ5_&kcYYKu1P0m;Gj#HKcdxl= z<3dB1CuLtG!3jR-D8rucBYuv;=gwcx#*;5{I1$`qSqFjIoj$kkd15cQCPQ>6ZnTsA z8vY}NxM6TPq@d+5NFx7cN@{&btd_a3e%P?blw#y1O37r+6SpIK`3=`s6x4ZG@$B$+ zcEr`=e3=jl0gAnNwCREJoR*=khSf|D!*^Pj6x%0iRpzXAI#P`jy55BM>?mC6)(wUh zqL3Ft-5HV?G6aXz$zAphW9wPtwkgL2E=)Y;cmzW0Z}{rw7q6y!TUceOAt#5it0PZM z^f{i3GOHI2*@gQBpLA`StR!f^n4CRgXz*pIvTxXj$7DLX>}21xt+5qT+q>9qOHl(+ zb8VxN@FM41!tGPlQx^kZ7<#`PE0G1H>GoE7>Q+CSa=5`@mR130UP{xxum;7nH^;Ky zQX6n@qn{Rb@v*2dv$C>qz}||Ay3kY0Iq{lWnSc{_}DHY2VmhDkDp(9V^Y= zl}-Kbi&y4pmYqzs2{~FDdVB4vNYc;?^~^`+>C!yLh9;O zaqZ4fBmhap^y%_!%)-18qXCj z<#4!RYQJE5i>t&Fp~21e7qmpFY?c1$lyOtd)uNspU`>YKc z4ys@*Bguu)lm+?uYAp@Fz7|fM<&YfjguUy=Cu#c1wDyi$Kj9W8zIkw2x45z=Cyl2c zU%gvxRNSyRGnujWg%<5HvD5>m%7_v_u@f4GSCsl0X1D#nU1IvgV)1HFGiT^!tiWyi zc2zpOC3Su4OD?3i02FQ(7uPI9d?qvzVM)^EdHcwnCyl$Q5-w!?K`bx!2fl zWPjRboa1|Lg!q6_H1g746YXrp(n+Vpm|IV}KMT-H4rVaBOMgR^Gt32CkNC8+j=lWO z8C#Ko%aauDuwa`3OQch(zoGO3f2z@m%M0UhyVSlK1;X&h{B7|NxKg>sT9rM~Z}4>K z!LrwYc;d;lN9kh27=F3&D`>Uq%^dY2nK+xvYl5*7HntD;R0pYVj^C1Qx*UD2If(8( z5HmUBw5Y(gqTj#Q5T5puDefDC|F285YI)0zw;pmx=AK*Q!q}LlRW86}F5-5HD0X@UO2@CYi z;>hJw%f3-la0^be5ZtfRXpxSr=2Ijd7F9btZ0q}BLIw6KmcH)Nq@m)*l9R!S8J8YS z^bFU&La9sI-SKIk%$p4&zQ>AAUBB1+jUDss*Rb2?(yuMTZ}N$LYE-PQzUZ#|dcm0q z9qpJkj~Y6AH}HWOrL#zlUULsA2QAUA#8dB6EG*u^Yr*|d{?>>(h3v&(uBp`9L+(U>@mM+#}dFoJB~z zd^@{Jo#aLf6)}R-8(A1%M&T?k`wh$LacNbk0TEC}>F?kk=<$oIZKjL==(tjwO)}1S zf+LknMGeI4=*bzq*=J6ooZ*>sM2|3(SjL@}x+++|LkG>MrM*kocD38{$f8t4;vxe- z>aBF$C-H_fS7-6KwmN6B8t!5;PW4ln)F6A$G{?#f;a{kEsm^V)S>Gb&Mn#FU&ZnA7 zge3RER2ogS)VJtj(B;V%Om3Egl?MzLuE|mhY}35ejy`u!FG-6L&}p$QQhJq*IzeIe$(qtRn*Hltg`StZ-en>MT*6E|AIwSA zuQ%o8H?GI(&R;8&a}s_)bMoXPRqd-O1Fp{o;jvcZ6dLtL+&yBsC+CgMHiKpl_Yh>4@jiz$n_k?5 z-|&WjZAg-35Km4|8yQdXV-Xwi!xL9u4Oucp6cv8Nzos9`3`?q|vbu6Q`9mPL_h+?K zt>)%S{JdYOR0*j&`_E2hrdV}QMk=WHzF$Qe7I#cV;`3t>ySvP9y2*Q*0O&&8y-_lYfE4sA0*nFaK$6Pk=#bCokp_y{(VVs zQk`=Hzv8v}=OWT^HEiE3CvwvE#sPnoe7R2=~y>V&M(heSr`NSbMX%{$*tnvzW9Y37-&J%t}(G)9p-rg`qSs<$<1CEUNFShy*GRZ6LmnLOTj?4f z({G2gVDhZ&Ee;*EjCs?1_lziCi`~vA7>jp@@vm%_aoLs<$jAt{hg_xQg~gvuf*BfP ztj~o>)&gP;K;Sm61uM`tVzqv^Xr|CxbgIb3)UhWxIM9B!gSClcmUmip`lK3f6AYZB zu--v?BGfC2x>d5i8WdIJpLm^}WL@Q1`!-(3x6qCjP$9I0!9`@ycZ!?ah75!C>y{70 zEu;d?)VK5^u^FyEDbhMzuVKe=dUWtlF#lzcU z?p|(dFNk>vx-)Y{R^^hijECPAEIcx7Jo~2b{j&r)(*%y0W3BW=oFu^`s~&TZ9K?f{ zyUjdj`P>NgVeHa%tHxF=r@NF{O$XHV4>xOE#P(>-Q*18_brW5yRfy=c@Nb(*rhVjK z%}joLh&BWLR&a8mUu?M^6P-`D+T8}ur<+{4C+ zx$*)#(R24vWUROeeeY7O23UE`6A1?;r_u!JwD8m)OX5cc0VTZg;O;JEa0WQg75?TN zr#GkFPzNDie7I~5Kcunh<8Msq-58N+^y0>LmIw8>?lX*u_etDkv;7b}#r}Tc$p|tK zC4?QJEqEH2RRaOs38}bg+59bHN8LjT0*VHReF(sJaAy%xp#b?gehUboaHmYBJ&dVb z>u!hz+iVM}FduTtsY-v12=_+z1g)^a` zm>5rd21Rk8IW-1fFea^HSseYP_do`fY!T1ZbD&{kCZK+?J!xVnPGpo;AW;qWDnD?Y z*`wv0i1t%?7gUYLL=|SRIAI$rzm^JtC*ifO9b4}*4j~Y7U?t%nw{jnu%Z!|zje2@u zYCVbag#a^ieY|XGRHH`9&gAH9P5LA?eMDUHh_VIt3CB7$QCThfFpX>vm-Bm5?aIkE z`$prF;>;^#$en^~eM_@Ns|<;pnDers2-M`|yb9_E&*|CY0bK)eUo~$DSJg63gOFM& z!NV-GFkwUj{EU|vW;JZb2J7>ihH0DnTPAyTh<0~DXPq6Jamr_*E<$zdpC*~*nRk(~ zx=~ZD-5c2>SI(c^58-ky(HBfLli*?`OKhgE$mf43&*qZTQ5z#B$GH=oQ9piBx4v6{ z#A;BkIOg?yncKXktdp*EMdarNe}if5nS7YuSG9_^RtnEOS@nrT4T@_$+*#{Wv+(C) zE&(-@@;sJKcg{bra&T$v5%aw1`3jpyJC$%EgtT@a;o`1~U>#l1vht#QVnH)J*1++Z zX>{TC{Hg2D&s}I};Uy8VpZQKIKDiAqlnAt`qIP*m!Q4QDm>l7rNC?W6RmhXHdS+;U z^?mX8w$H>Ir9rm5Bwrq8JR}j{yX~Pmk*p}4mc-w)y|%X3jjB{9x&JO5HiD^Q4K>{WKR+Qew!7tpYamS!W(OkSIP@|!>M<)XCh^^ zo`(2-(%XXP$uSP^_^(f3{FV<4vIeY>t%Ig1BV+SzW$nxN12qtMgVAW6^(?X!*kc%d z1(svAC8xhn;0s%Wc@x%Q<&{x2;>OAS!QY~MNXnb!jUOH-+8nMqFXUv?+6)p?zK|x2Ya5luC448|43Vz4rtfo zLfTsySKOM$_!X90rGB`?tC#NnbkahvF04tf;LToGTXoavak9C&MEoG0rTt z-W{b9+^_`3%MzV^Ocl7bxynDYKP37N3u=P_6r2=u4y6JTFMUjg~mRZKfv-kYm*elQfYV@CN zqeb?Wk4zK?OOBov*7l+%LTAflSRRQWWOeP8ZwbGDR^hOVS!Z`C{e(mwgwA4C-^t=` z3>Aii4hYd9J2&I$A&_Vtg~eAj@>90{G=ts7-uLR-GPaZ5#?oi_q2`!PA@Dd9w_uU= z)L=F!@^M#{YKxtjDdBC0D+jfMaxKfMxPxj<;cD(7!2`@HjkNp((nS~RU2MEV%IZc> zXXlE+x$7%;)dA0@^|7A^R6XEFkm1h37g}()ryRLT$!LVcHfEjDd|#-P&HYaU&fzW^ zEhdcjhzC&PoQ`wXwT~6OvjjKY=IB8EG$8zEc$H*9QnF*C5NN%a@~!Ht>&l-7O!*mK zf0DK;_v`(!-GD;RsA2S#mYNIF59G~yB4zXf1@n0#7`d}*UWnd=z|TQcqbF@Wl3V{; zm06pgd*`1;WX4h8n(S>F;Vw_xA#fYjgF0}AK(6W*>`(#%ji|lNE^XCb3#WtH#Xfhe zs>~s7XRir?wJ~fZ{0n?z3<9hWSnO9;hHIw{tyAXSSlMM+`O$d;^+Tc|J)NgYlufH# hr7q^HJ?4+TT|bF*xURj)WQY;|uR{NE7D7P-{{uuDx77du diff --git a/doc/v1_api_tutorials/quick_start/src/PipelineTest_cn.jpg b/doc/v1_api_tutorials/quick_start/src/PipelineTest_cn.jpg deleted file mode 100755 index 01715db886d811a776c9e7d47f32ba8b1ffe6230..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9408 zcmeHsXH-BqDamH0s@kgNTvZ51j$jd5|ki02gxECiISTf zB_o;cyKrwkXY6zCd*|JI$GGRmtL`YFuZ)mL+(N6|9?iGqy041j@w0mOkH z06hWR1u!u%F1~*jENrZcf&+nIVdLW9;$B>M_=Na)cm#O3xCEC82na8M1s9))`0^#9 zi~2<;7vdLE;P(<9F5X3p|F?o}1W54!G0bc%3}yh66a$MC1KkQR000IyINFN=|5-3F zu^`wu;E)Ii!3O0d;PA1qFu}oMLm*&lZ}2$)A;l(R;=heUu4;_S>X|+^Gq-qR zY32OX<(aFSyT|hvF9QN!1qDY%$Hd0Pzez|;&&bSr{~f>o z_Kwbh!J*-iZ=+-5vvczci%ZKZt7|*Ed;32Q4v&scE^uK0SbxC!jqESDNI_hf;2c13 zFK}UCx`G!LDFmB|ABXI=Dz333IkSKt9`sJ+yW%E%7D2Ub3KOS(0!mh)S+<=EXg`tt zYry>fOUQl$`x~xt;1U)FIC)s401P;3*Kdf8{!FU0j~6?RQ9iK^7~~EoKUWZf&a_GmWvMx`J`U<9SwNDSMOhB zAM6&YY&mB~VRDK)o-UjO@39eT9JqR>CO=MrS(=r1i1wWyqk*mW8Nakv3HLyZbseWo zyQhs4ACs#8y8o+T-c!||YMMQNiv}Voe%<~%c?M7%InD5IBTehBp4UzM&KV_amup14 zpLT>-V53E<7AO#HAwxy6VbM7k;lHHy2&+>b4ZIJC{{`4K5)H_mff)13ir#2yUFlV# zc-?wRv&-@RpV-DP+u{9Rp8j9v_&2Nm*R57vR3MMUP(}7!NQf{&*O&M@J1UWu7x67E z8+9kD9SvmqAlkphcby{5o_WL@c8!nOI!-L3Kn)!Zs@5a6`p|&4uzva{Nw92 zMs1-m(ZDeaE{gVa77d(VzJUgOUDM92!jG)C0@1*}P$ddu8nKf*eX7y0H=umF(tnVH zSk6p4?9Vu*!ahd>XZVO+rvKm5|A?ND|0RtpH^NAT`hsXZkWUhmjKuuJ^@Z;?jIBFA;c3Q3TfbXX zMgvTuXrLek*@rkZT+cg>tnehR&U(`2D{N$jw`jd5d^Kk|D>6Vx$`yENQ~>+V+k&rM zVi3E96WLS``qGd3M+lkPA)bX@o9|4XK?~#hU7;GKB2Y31Tw@g@x5R-K)x>_(+SCJR%rF;1ztACS-Vy-o4YSuurLI;D z-;44N8@ag{Q^phGPw5i9_=t}2-Tc*D{ZOazaa7w_O5FzBP)@tcc$Ya(P?Uj#?yGZn zLgSi@I|DQ1$vmr2eUA?g zYL2wKzqwAE$l4fuJyzoRF7#%S;5`smUCnLwC~DjAzXEV0YTiz_wZZJ1p&X2#)j zG7j#?K|Q8N!JA5oEgjj4)(4)FygDzGn1;+F=Si%tL$>_iLP&$so^6a``9(?N8W6v- zVed~;Rh%$2rr$WkZvH|)!Z{IiTB}1cQCx{DmQ_EpvA=$_@z~Wm;ceoaSVm~3gWX<>Y7@=>!$7WWd~i9e@R z<2G^7hVVMMty!j3qJ*BKh(BQxjFY48J-k4Mp;TgzI8sW0` zsA{S7%z88s4zii-4Upur4{7)IoiCoG-Bvyy^PpNu4?clu1U_JG0OQa#3}Kgv?ZmHRqUqVOLgdc&#vKXSZw{cMwswTR~_GhZHwL zwz1v6`3kL+H4v?YW*a^HNXu6tO6Xu-QB@kJZ*(0p_NcZo)^D7|HWrI`%HP(iQb6Yn zvR62$S3Oj2z?PMu(IZsv6&KRekW!rME6=To!{0?hja5{=qZylzjOcM?oxbmi{ivua z%}Qz}GbY3@DvZzlSV=i^o$_QcvZ}&TPdH9aIf1ZwZDqnhp$cN`cU4?FZLKDMB+#;; z&tACBd1585hv4c1ttXq>8{}d%xbB!wh$ex0D_=et&#UxP8G}5w`x%c*6x})D+@iK_ z5uF<~x(p7~Tr9ATUI!c>Q(t`9kg|h;$Mxj=6~X(fpIaZuS5d*tMlwl&c#hYnQ}h3<J+TPbo?)=OZQ;}uJr%|Lv*%11S=H6)hP21G| z{ncTKIR68EUm91I(EQytoMD5!Dg9@}P>f(^5d|5sVHOT2i1OA4VXJ#uHP1?yLG%%|(EtzoRhcaPZlswcjw^MhX#)V<5@+^DP z8`s%yy82D|Cxw=l8htC6oxj`hF?dm;tWp$QFAGwk{=-ieN#m&W8{U z0(_hO#vgq)LsYcqo#pw%{I<%>FWY&_ zQ5`qxVChSE%>D;ZjYKksh-+-5m2VV6Cprb<7E9#QG|J-+6~cPw4m|T8>?&85rj;}M zQM5kjxMKBzl(*eC)g7|1@DuJL0s0s)qwb6xTEXo9#vR39$;(@)R0 zO0lI5r$>z_<{4wPPGq>DTI|-eAt5Bk?d!F|{a-Oav%~(Aj%SvD3t@QQF|Ga|^!)e# z<8suGJ*iMcjo0fkRk%0P-V=`WU>pjfciFeIbUU0~gtSJ!7Oa}Z`e!N*I+HhE35jl8 zv$ASCdX5l@pYb+dqbQ4+(0Qm}QvIR}T5IA&;cM1DA2~_JfhE08nTHV0sd{N=+ifB@ z8q4vxFYe|{-1sOK>-+~s4c<9@>N2`~r2?qV7@!y(I*k%&+J;Y+hrZ^FLvKEZuhqP(+*NGHXpOg z5yX=$*2|W_+NO)!$C)zsd#eYlYSbjnb287ci_~cza+urrm2p{6=gd~gcfELuxX~#! zf@?~Y>-aVNZhN8B4sM{`N3Zp2`nBCcdv)`N5V8b8n}=$~BhbxUZl zMqo@F*-4rxHb7;w}U8X1|`IEj0C8v0<`(Y6*-U9Cwib2yO;^OrnWm3-|l>T`>hW~A)!9693V0c z6x9uvetVij=2)?-0?+smc}R!g&WlgaupQ)pf4HB zb_Sb0!m2Gk-2RAfSANxeuVp03X7iFZ?aOyHR3dpiecT zj{j7A{bcwsM3_)$d#e=&tvtJ5Frn^k-Z|K+7sD{xvcaSKVx&09O_j(by5mWIr@Q?O z!==aH?CSaCO>?YVY2zZZj#^qbM^;(r#OyMMeE3>i92YRPSVX!sEm)Rz=hYsJ2kMQ` zesJFG!To&V8dNUn)Y-Y9>^hq-ZRwG0sQ<34TBPF8L_k;CbiqLSSgGr@+rp`&AjNAH z4M2tx3ZzJnf@PgH4^IdM$&;gJ!U>a64fD~_M(~l2sfWD^y-jeaI*BSAiV=7!hz9a@ z0?)%d0(S#~VdjV*K744PxK9P8(r-l9%CI#oiMZ!2qK#Tr9{gdE1v`6zMAguiYcinj zfJYaD%la8Njx5%+jO!!`5_=O}Jk{;)-w|OfcDt8JVP?ijjNzwpm&8F@tiZb-I&Sb} zVAa`hO<-kL=^&EuVOgm})AJxbbL=knjP52@`zwo1vBUF1+L`Xlu!19PwH*fEf_={( zy=m*b6~mFaz3<|XqIyj<@Qi_wDI~I5&HyRW?G-B|MV2_xs)}s!U_q2BL6J3+gYw*6 zEHDX--Th@Oz#+Y&ScU_OKp77_j^l}BWYN9Jtf~*&XWgLDyz9u?D|YqHah2Hm^DkB< zH8;AYZlL-DmROC7;Z~~YJQI;qr9%#PF^&(KZoSEXVuYr<*{#cN%cNw&vpcXqzbq43 z+NF*tg#cIc7|rN>H;Dw1&~;wOsNSlmh+5|O3#w_v_~(W9s!rytJu$P;o969s711jg zdiyH4j6Spm$RpQQT9K?m1LW#>ebRY?t!P}1w>2qwo_)-M!pwcXMU9lVMdsW+)j3B# z2C}W|UX?QAyhTcn`{C5A$Axqj>>+?Jk@#}ut*0iw98Ycuu24oyWG%KHRE+fN-@TH@ z(9S}AxmttYMC5st_98W5uls33fV!czsJ4;SpoUT|$b^_XK9m*Uy)?n=SqdqHY~!jv z|03xP#YmXtC9Vt*GAfa%*yFFRs+ziV4N7m*$Mf}N5?~K_eR=QJ%k?0b;~$tYY?bA_ zU$qJKNh-bG1F;1YT)B=0YI*ORLsgBW!&7f-mMdcA>VB%GT)HW9MTsgg)0IvcH{^U% zGF10%$RJH;Wpk}!iol^#bxlLfK>0OYUfZQ=OT!hD)Wa{IKw;vAoUI? zmsD<|4TyFAc-PUz9a^#OdBe9aW=+FAG9$P4r9{c`qvVvlWUUP-hM(=sF}KS)T{|xg zTSY~6fLh0=MN7S*=N*j%aZS?ezIDlDPwRxUcKZ;XKVNL6fa+D_=$8EtJO>2M;bXzGa(_{8oS)1tJ+R( zh9hlr8#fA!@5!?bYIoeBs9~;{r#4d|B3kSiZ}c6`3Fg2`lOn1a$b5ZDe=^WZ=msyN zJdC@xDK6ItuA@U8Zai`a4EbSDfI(M={YcbGA&me3NC~@O{)QI68KLU% z{_Q-YRlx)igqa~-QLU(Z=0qP6(#CzFv~RNQW_#r=@r*Yq#xdb>N`4iotzlsB%p0#q zDj4|uwT^yVIbc6UO6ks~^7qkLRyUoA{0Trn4pI}^;-Kksi>xuZ&vtgCbcg#SL_4~f zy{xi)H&SEfE?aBTYd#a_1Al(M&`*19DNu|^v)Q~*(WVo_XxltP-HIez?BbfrFkS&; zS_q8$)tacOB$2pBzi0q!d@H4EzhMt`MOo@l;Q4yjQU}&1_i>v_b>(mTXiv^c$*J(4BE!p{3lrjQ zM{;!b(GV!e>LoLWh%F4%F;ED zQ|4&kfzt3PP?#1uVQ}d~O`#Y)Zm%1itSD&%M>lPahlKH@J|DbJGOyCNHfQ&csW&My zjWhWiw#-<>`Ie_A;5B_po|UMoqGP36r1=wCrT%+IObNK#TQfyhDCvs9Q$XiWL{k0T?`ef(DLuw`I@6XzfuW z1^)Hn9B9BX@MD8uYYz;`w#v{BQpkNG60m2in4|TX8?tkSO=%QIM)D&!o)V$~GD}fC zH7hDTPb-q~{8lBxQt6WB0b_$#m>LUMQDQey{6{pF^Ha(#4r?V79dF!l+w^4moj zSP6`GKu6}5W&=sGS}6~4@M2wiEbTU!;7{mk(_66zatSJy@ZC&c(dI~_H>Gs zsaNZ-_p{cQ1{IWni_U`wTS z8WNUi_?BXsidQ$=%j8m;WdYZxYN~ga#CUzv2waR_R@8IH z>kkVlsjKfOzhcm~|L%QFiKn>}T2eZ=kw51zzx;T@)))gZO}ZqJbf;kW>IPK(j@|5k zx}}@O?1W$J@2LLtdQljdc*|KoVOcHdJ5=yV%ai1BGuQ8+mgP8~W&Fu1il&UO)?*stI8 zaBFQaA4T7rw#!@fTjqC#BV&B{B5zmv;q3J$@iZFfV+V5jCK7mV}5~#XR-Yvv(G?-M$fKybK=bdeB@gF<|(^cM0r^w?4Fe|gt|GNUG zA1MU0Dv~L%1tSNaLzFx0{6y&2ud3`%X<`fIW2#Api|#=>cR%ebcpC)`w?N>1YB;Ts{9lgnQUlb*X- za?9uY&`)LXUW^v1ip%Nqo%QQtmuQdJ``@bh+ZZVff2tt--wWmcRmt+;9L^!Zx%~Sn zbi5L`r^sY_W%9@QSUQ(`dIVpXO;}p3kgD#kb)Ty@%Dn#D14S;|T~^Qsc0gaS{OvOw z`9F=-J^tUo-2dCoHv7|w^}(eeOwm0nqBzVZwEQBLNpi}QkCrjs*y)(sbE4NJzNw;2IlC_44~R} z#C)w^Gjr5p2aEih6!Vr`T7{O`*-SQ?HC@E?H17gvNXD$EGSoZ9?@>Mj@FPj90r4$h; zo?s5tw>HErFxm^gt3B{Qi|QD$Yl(P+XpY%IoBAs@o3;${Bhu4RT2Kzu|w5SqyI!V2?JjAN5(GOy=`lR zuaC=LrzmShYoxA>5gJX)73a`l@u4nld+vh)M<L4;jfgZzhjci^&<&z=mw%Nl1=J4KZ{`NXXFL&7ASM z-v2qz`{i8c!+r02tsQH}TKnE>{dUxw*NXT!lsG6TDEP`sa+)Y8Xr;(+5iAVkn~!?k z8JVDhH5Fx0YDTGdkObhpw7N73N_{-;y~PtGjqR#r07gL}`SABaUACd~MnPe)R+f|2 z_BK8Eg6*xdkvXuk#Psncb|qB<7Do-2^Y{EW6PBqJ`V~5RnhNzznidsJQ=ZgfQ=YpY za(AY-%4xiw_L6zM)mA|z7xrq5WBW)`|AL8&!7BUcDqMt?`zivQlnvipz7ikXi16#W zKy+^TBhH*zOK+pCn=B(4%(0%MMFjsRNx;DBoixW22FU&(;sxNL{&&J?4xa{e|9?Ds zCn8?=EMZj(biRL=B!cqgX_2D+cTFNCOn_zU4lfRstm=z5TAf}lgM)TX>Z)Efxf2)vdr?s_9D}~yhCgNMZl^jqy zT{{$zeF>>>Z^ZwZeA5?#=E2Dh;eCpcYCHLyz8J|@trqpEj_ zRLNRBE`~aRll2Ogn2#aE0ECrSr%lEMFmHApK>Y#s?(W6j103ZZD;5OF@w{x-uQQR7 zXpKEL$Y&&$vOFcgAL$3x?0=)bBy2w582*Xm`CE0U0a%~QgXl}6seu8VMJm1?NA$m= zafKwUXnxhzUes zNLf%tX1HMOI2Vrr{R`7d(aX&Jq{;k%+Nm-)Js@>is*(YP?_s+z9)2-rM*cX9-f(+5%%kdY(FGQjeJ=K4d;A3$ z3WKG3?9C1?xv^XuY7_~V6j^1y6Q8e51&)UL5gA=E0LS`t1kW=?T7A17P zP>~P_Pmt~Kp(CLT%6}VhI>@?F-Ljm7PbCbMC6s_G?~HyCdvLK;C+E^%#daQuquFTP z$|c2O2ax@_nby+qQRlKEc_HR7%cUV_M=tO%U9Pq9Z47j_>AO%M_&GUPm|KlwZxf$VJ~Hh?~} zCz3GTdN^%x6PA6&`&$-`;fqPzhs0>^Hl?nUSQjX8D~IuH6PA4VyL}khtv#XicLQ;$ zNcl#d2Mf&Tm;~omDQyX3$RP|bG&x8B2NV}Dj&VX{@E9yR)Clso73HeB_wH1 z2;faGJfRk}O&H0Oge)^@FsSXz4HP~x>F}=SGHirquTI2h67mmqi_l@chXg+C4zfF= zu;zLf%!O9)IOKr>#JjZx0&7h>i#j@%vhN5b?_CjM4>GR^MYriSK)_m1ovrRfzl=UE zE^cu}#gOjw->o<#MiHs^{XF&R<47_TONuKJpK3GeYtmbZG2qC9t(2I@E>mA$pQ+zx z4cmKEAW~gw=NvT&@j0@zcUMV*G009;fTM*p`1TU1DfT~}ZiQ~8DqS7^UL@U+v#WEo z#{awjL*)Pq;y`te0(({8FHx4uo|_-V3q1sYj$J}w=OF@si&&B6GS-0mI8T!Oc+p)p zmpH}ta)6dH(}v*Z+@?_RF2v*Aa_v(~Tnm#<-zL`5hr6q*-e!lnn&sYFP)SNp{8O)x zh>`Pzg^5B%KRWGuZPJ@TVLHv(2IT6?$>;yL$wJ!uAR2ig8o3^*7@orF=bXD#w9aUr z{c7~%%skgI11<)kU@@IAvoIda2kIYMZk#nK>|safcs*W2oB;pS{7knEbh1*O5@z*H zA(w#zFhcb4-RZdXbDbTX>UZ)UeP<=}-l|HhR)%cPaSwI1c<8Keoi4QtP> zjL8?4u5w2lY(};8&5b1*mP2`-b@k9LmH9Mmm9$&-;b>@LbFG%tifmt>vso;@)+IoT zi3kmhm$>}(CaRHYH45EI;(6Fn9W z&@9}6g`AA1+G(G^AMq-_gEOkN8EFLD`IE<+xvhxtvdO=s16#Wfzvub2PYKi z4yNSovdS3aUKN18Jazcl!@F_0=pq?(AR}HLotElH!i`F#u6A$yO~oImAz_TIzfR1k zaNW?>pAtTfag=-?F`McOQ0C)4)F|&t7Y-~o{#=!1l4PW{c1UNTD;}`Hs52*d+8e+$ zvP@j;xIR}VAsz9*4VCwsnOiA4vXOb(Ki-S9*JMtp6p4@D==_|g8 zSGaumX>{J}w4hGwr0H4hVN}e^q{qnFC$bCmB53vcagH3739&|=DiCMB9i9CQdGpSm z`osYG;n?J8Vkg2Z6z5I4p!<5rTT*|i+oz*_4te7!wm50#m+A<`Df1Pk?*VWTe5iU7rUlgtog$*(M@hV>?$#ICRtRv0o$6qd!Jf zuJsnkqOwqPP`prGTs<_>)epNE2`DcsD+>$@iIF0R7d!m!8>$36)RK)PNEH};)srXt zZm6V)^39>!05w?cL6&(BJgLlj)66!|MiX>@lG}kn$6a?yZ#G>TKpUZxJ>0`B-CKDu6#x}~8oRmgRR%6ocvx)s39s-F=3X*ChB}pXQfH~r${`xiI?NK>b~?oQ zaG%C)V#sqImT8_4PVP!5g;c2dgcEkLAzZlR? zvIuAFF}6w|()=^<+AW6ITfaI2$dknq(-+ku@04bpUwjKZZjmcNG(+N8I=UuNEodUnj%NoZJbE z!_g*;JFXdnoPY1?obGloo8o@T41!gV_*gAGG)(*+X!opKC#62(GGMzqSTUgWU8OaE zILES<;im@qoyhD2n1AImXq2*n@N2bs92si}GYk=pljr(O=IynbnMfXZVtS4GK2l^p zoT{;N9PV#TJc~=oxJb>h#E@NtcH?K@?xgCOo@C3|*_A``zhBRLE3?5Ah9kti4vi^z z&4V>uVc`+N#FGj>LSgUh*EeH)F6swp&$9c)jyl$rUtk&%bI(RBY|tzap?zYzpumN! z=_x?qui!QzC3sqt>wmnU!ZkS8iF`F0k`L3_m#)n88f!_+s^Wuvj)W&tJ;V?`X|?A( zA;_OxMID;ofn#F77z3#BYo((mz=&JB02q-bTM^@2e zO5a!bu^@m$<(BWI=a}EB6v8r@oe%8wn7Iy%@ zL|iw;Kn^i>7HdmmB)l3&^3E!jt_$g)_KKoG;JvYpYf_EG^>`|_Qd$vc?Zw?u7xS3p z!$&c%XyXrS@~6?+EWYW%*@67ho{lT*@3_UsZoZ^LfsC(HFb}&laszhxYBM`GKf5Up z^Q@RWw!SxWwYqfnIseM;zLkeyYI)q$ZGpg{r(oLCDMWnzde(A2(#fYv}728lZcT|DMsxX$Vl9=!@%5Y|~Ap z@wkpP4)AVy>21+LV|UL}DiSyiJ}2eOQ4r$pk&;i##VFB=>b;R1{q9ArA11iixj`Zp zY%%N(OAbn3eRWy!ZB%slsCr4{=ONIE`0fKGxaCk{si)VG9Xwh`_cFjN&=q#!J`ATD ze4TFRyxu?d@a8D<_M?`zV?wD}4kwQUqH&8&W6WE-x&25IapI{cS1FG9=llI`P!dNA zWM#u!OLoHzclEpFJwQaRBjjJE*0z)0aNbkgU@>gt<2EYe zaxQt-$l!iQhj`&Kn#ZXs={-lh?smHFP@6CsPBVo)`|+J}lmXpw>ksbEb4|!w+1F7j z8zK^C_XYQQ*1BX6)VF6F&{M|n61yDB(`BY?$~a= zgJPaPC`MG3`S*3SP-a4PIp@L&a~i%@239DpRT2yx=@8To&baNZTHsZ;+ds}MMN>H+@&rO43)_rXd&u9p$hm_GZ5>IWK zT$%5Q4SPW^NL`X}yoP_jF=Uk05VvblHnmzjG|%OK&%C^8`sZ?Tq_jA?{b+@E=RN=9 z!rq3SmYtPg z>Oi%+n9N{|PtCW)mZ2NyeBn8=azEt^rdR-gq#k0M`_EeX)Kr|$cXWA7>NH#7-MCs3 z&YVygSCjK2IZzz*$sSBd#%QWU{p^ra`n^x8dJ_h{k?l+r7wfSCdND~;p7Wj!(&yvH z4q=-M%8+)E@s56O>Kr z4~}4Vcz7TG>M>&x!UDb6;VSoa^bLd`4)7ved1waeaLT=`HNbPH7W&=LNx*=Xo6=J! zl5+Ie=>kFRLWJF&C#!Np3=gv(Q&Lg`JKq;mb&2MtBxudt+DQ>piBNGN=JJVz4PjfC zYms!mbXp@>VVH!=oB=BQb@XgyJrlGOn1-*F>M&&F?evt@VYiINC# zZUzzD$~k;Sa6Q)8YDG0~w0}h>kui&*(F~gK}Wh8S<_T=iUAHNeWIlWwm4t zM-`pTV;v=)hm-W9TVyvm@R~?*afGIoNqr(wk#+}m4<)C(<0S++S;uMtM@tk6ScdM2 zsm=5qlsLAmoQhe|)LS2^#28Z;^zCZe4@L7L_7@y#FF@6slT-R~T5ra-Knr5wlr9t? zoAEc&-+(u_C#T6lU2mN`1_M|g#>wOLKI-Vq$Z^y%zUSHAX#lu#c~VXs{uWbmOK+f!!+85%KG z`|x(>Xt>f`BrvM$DNrzfhoT3B zMbgQVat6o)L`hYcjwDogPP$txZ0BqGJcu-wqG}u)I+tI^K3}RmN6UaG12(XMQ+iGTX|KmPMYlv^L@q7n1I8 z;&Ugf9ZYaqSKl5QHy5PLwVms1Hhxcbek?W)nAdN85P_uU{oW33@%2pbtQkN00@<;C zHmT2=O3Hm!Ep5s8Vn-i2iAiIF<$EoGEMcxB(dgf*`6HJ~pqzVs~? zhO|nXv~)uz9p=jJ-QcU8_>M4J4eSNWu13u6*dYD_@deG9pMO?V>Ttb}<vt#9tmh+~xq`;;b&Dr|+-X*VJvg&&&@nZu6PfDpGZU=0wK-M>J z%<@>+s00&t9ni=|B6bz*=FCLfSFA@@vX-Mx#?x;Lcd!4zd_@)d$?-N-Y&`Luvnps} z$P*&^>Bt7^9Q2acY4JR((hRR?Hf>}4KiY}tmo(LDwthR0~N-T^tt1wl1oN3f3CR!K8R6>oX|ylY0ZWh zfrj3v8wUHKF(-8-rXPScDUP^5+XEM1oTX0+2VPX6dmOeOfieZ{G&>Q%^~rPB-B8kV z%$Oqq1F`NZW>Gy_|LGQrf*5LP$52YpV{Hik%VL9Ck4ob=$T&IY@Xt<*n(&Q6cK6~; z!^YU_0QRAamGRtXK`&W3hM(8v5>fli=;+;Clv3-epmF)I;qK{9>#I>}@qG6Gu3gaJ z?NL2qbjVc{`5`d`lN?xW#R+*aq}~SYys=wsZu=0#G*av75~O*+4wS`RAcuOE=KrDJ zLWWU2P?vpIj;04Uo_U}uU6<>AuY1HHSMT#!Xa%#Dj#LioRb%p#HT(OU#Z>3h&s}{B z@M{eU;K00!3%%?x{`7;r5#j;1J7643+}4;)8@yvSvg!>M2bRZ!jqu)#rX)_y=9XQo zcmrM}63(fsCQS8@_1l+FhA5p*{Dg<$Q_f+j-8xRfW0-_;Y_5JhjkgYipXA4}7 zRo`u-J~5;u;2tD{cqjf80M=$am0n^f_mjw)Kwmz5rC^Dr$)*e{EFTcUIuK_M-=-qv zp^(GME#!8Vf*r&^5BK+Ob+ouqx@a2DI$+p4+Zwy#wzWMy5d&{^*-^vz(s{A_RTCi3Qh=OAg408NFLg;MKrOgheZspvQ_ z@|UoXQk9d6yiHsdCRZpMt6Ih?I5?h8a(UR5*GS1`bjc<)78%fn_3N|%sn95bQ9cqU z4V2*XZAdXHn-^s5GrUhw70lk9iY+0NiNZ()C+K?6$S{Uql91rK_0H)D#EsJ(oWvgS zK1{MKs0K1_lfF#bRPB$XdC3MRu6FXz+g+h`#C%zP`Bn8PB4}uwFw4LnpP_Hnbu;t* zZPyR9hPY^%cj&$RQsVsXIJRp(2{IVopXvoQDOyK_*&~A!wMfzb!V$?>QXTX`L~>Z{ zbV|$=V$+h;mQg;mXk{$Up~4Jb@B437F$i13P~PYU46b7b-$tWn-Hyo8GvH?&(D@%r z=S78$&}8+zdYc!ufc=f=yu3T}K9+^-U>^wtSl^!SbOkkbWDW7CW$X6;6Sw~G7q1bm zj)?6;k}`}aKD_{q%eafZIltr%3?2+K6cV}*W!>^B=Ih{*U6e*s*%CL2Qq&Y%BW(S( z)P@ee__;+HPtpqtN0CRDFPQI?Qv&E;v3_C?F!tFP+%9{*xwSSOHQY=J0aXvaZWD&fqJ^H4@F!_xp={} zzqqzvJ)Z2_+ka41B3h*6wfcsnzmi$7q<+$8TPitZ8cnCUBiDCNRi$ zA1^jv`s)PSMU<#z{uz+{%n;ZT zTqa1YPxTjCk(2qMsVpDLDS)e}LI$U=)7H#n#~idFQwACG>bxoN&mWNn_sHn7qf1AR zp+@y7bZr&rcWXrJ-8d;pAGA86$^VWc)d{bt2;V&Ge zD~~dn#%2i2$jIf7N=lYJe}(CzHA!xk29{Q@*rc9OcIY&}OIh>6;_(PuSDj{cVzyUo zD)gTU96z8Bq;C8=u_!9sI0#c$Gx9j82LFXGYyl%xc@x79$pXmcXQh68vT(?E4nQh~ zR0+K~Tu@7Qa?|2p7`PP1UoQHyiIRUKFkU=BAX!pMOFd6PuP1o`$}0V z6^-xAqGoj8f38{-r<4lTmhKbNRj$4x04Jyn!5Q&1GvBf#A^$7V?WfU2FCf2)sVfKZ zP#m8r6PLlFl%=i98!NfMz)hKLQ8iG5N>%|Bv%Kh8bw%o&YDJOW6~ScZCvxp+8#2>* z!FhcyMd4H(Xq);5|9yfu#jJ>;Jz9zY1CW{g%ZhO?fRXBB-&O&bO05j`L<-rMG}rFd zb7he6ZSn3BT6x^TjY44+8rLvOw$V3L_e##C2gcTCa0d(i8z}!tLft7U&}19Bv$Qdj zL{~|+UYyCp9+#)`ACx@uMQJ;>(x&V?fhtA%m{~@wbi2L=Rqq05BxBhjHcR{Q;O=}I z4GSKhP7G4An&Ho3%y^OL*=cV#oRhL->sL0*A0A$>JyOhyHP6K*>VvaRuJq#C$rWjE|5`tak3SITx^^(N V#m2Em3>Zh4UBB zuFsU5J%08S_&HBVKzJtcUpp`#0a8Lh68k+477Kt)iiJapg=qzt000XQ^!Ci*p9dB; z4lW)(=n@ezC{TV0bRP!?8*~;A7Z;TF1%C%{N%1Z-3n}1}K}-l(oXCa!qtXaj6+bjl z=nQPLiI_SEoFk&7qNbr`=iubx<`ESWmyndYrgTI3ri!YX`W;<8=-qqz_sz^LEUm0< zY+YR4+&w(Ko&*L3hdg^83XhI?6&n}-`b|Q5#=Fd{?DsjjC8cHM6_r)hHO(!pZS5VM zUEPC2!y{it$HpfRbMp&}OUooEW-YT35*musL{Xq7g z0~YXqh3rpYzi~|f=W(#W&ch)EZ&&mnb>ODpT@8{w^G*BmikYRFu+!ZQUvl% zD$Jnw-{i8 zFP+wVR%jFkI2_1x zg`DYrb`XfZq5G>bnWSmIBu9IvKpq}UaYE=1UBwnAq4UorDkReY_$}>GY z^*WAN;O)f#*7N9Rvk@osqBExj24nQ1cBbl4INOCOW#ZwJTtwGcKKi}21o$b zJAwhCD`6ywwxX{6yQo*#s@(&F5qyIQg@*xlZwjKs z`u*fqY{9O0(1;c@veVd4gYqZ=PlOLPc_qMD7;LT2 z!ak!MF@V|AOjKGSeGgB|K+Cn^w}lDQ({{Tm4{?MG?q15#g_LSr*f%`dZ*t+iinYjw z0fv+`ezApS2?mIW=s%*ZeaJyZyph(sFtBF2Birr+=Z^h{nQ-^+8F4y;!}5PGDgT4p zzh4G=l~u35-po=X_*9M8wUzQsSrP+us>>Y?9YG9=p|3JLGOp$Jyt%375YmVWnnBml zmg^az6~Pc2UbtVFUx`S`>3Qu)qDWVZ)^r&Y>r4hUTG<$V5UFpZxD<9S>sUX!7rqw$ODCPIzVM z5{ESNJhos1f@{PH7w+;Rkg3~pA@+`fMVv`K5)SqP_#DsHcZr!4?W{1Iv@!HL$|$ViI)gn zPeMmL&PODOT70^0#Zoo%Tzt^eY?gm9toMHGK_bazZ7nYCRrJ(()J0E>6QfDbYMt=ysNN*=jiCmC;6KV#dKi%T^rdke49*HGm>F* z|I;A%Vq}>hy`Q3?IUO~Ar}da|c-cu2>7>3blrp-M?KoPxH~_#F)#F=No5kKKIvM2J zGRO`eH$k$_5teP0ik7s%7^=!^yvRPmL~BJJKd=+2#p&pLvdtr-HL$|3~oUdlzEetx}=*K56BZzA+fqNUQEVqJ~9TZTzD;05`1k=BZz zKN+2huU?m|C|vJ6njlG?T{y@u!n3VtVBXfuv#EI$N6C9x#okef^d>8o`?_ZjcA`@V zllP8T=$zd*L6p#Grc6?OeR6DW#x3Zo8Uy9b?R%LqvA~s3bLS^7-P)~j3K%(tCU3;+ zoET`Bki1#yJyL$r;~eeqNqpVHc)WcuTZY?F3~4V$T_D~g*1N6gLhr=v+(p^998PfD ziX-CzNJ-{SOgF?ME|BQg*HnUxvYS~otj8)M+R4h?rKREH7ydS!H@0>ld9+<Xv+stOn{E{y4LJUbqB)RFk881{+uE#Wk&Avn{s^ioF6p=5DA4_3ZHg%>F#n0@3*&!AML*wznF+BPNEeo4v$pGVHkODEf zE_WEe>We3LXt83H4(DBg>~-?cO(I>9Bg?VOVM%5emx*Xz^8|RCN6HAR6lmTeGE+f@~(lSJaskPZRz=;!iMVO5;cdPz+Bfrh0b zlQj2*Set!1A5vX>2^r4wYdKH&ZK4hXKx?Z9X1}~E7q@Yu48;nND-%&IElS}H!c&XP z?RR|ZC+jwASt~;+?Anv(%QvpP^tC&bzpNc$?Cxyt&WEpW{?%z{GkoDQPJFw&U|L0b zfyjO0{^C5saPycABa0)2)vQN=3a&G+mesDACG&GV{$QO}|53nMyLj!P9impv0_n27 zIhCD+(`3O{Ldq9bLI-`Le*lOq^Dzc}U$-mW4jz8eJgs5IUm1C5DZ=K|trkV7hV}KP zQyCX)xQsRSNMmTe!00XO!t?|gb22kFD_0MrT4A4jNw)M-tZ7Ju*`06K-d+2WL@{0M zf;;y5}p<_e>o9k7k{u5D~Q%I7Jpb7bOe5t4l=rTwA5GWp$iUTyYD z&_E4ux1{d!sVr(;R-ve7>EN4KXu_$z(+NmUYcD{y0KS&b$srS3ch)*6a}XY}nfFHZ zUVV6RYK#FwEieF>RoF>>+rs%PNS>UV8;ly>8M&NJlG4b;9R3ooLM#!!+1zFG?CeergM>4+`%F(Z^j^aj>J(+!+ZeK-A+v z{ibBHLvLI#c^>hGrH?e0y2zLnFLVWv`gYqU-suAN@lrRnX3KOJt{u5^%aS4Q^GMm* z{Cn5ZMIK+RUXet|lH^YK@NX+=+gpa$$h?9NO1;)}5IynTP_yHnv%Plk02wi7$gsav>FLs3QpM`2BE!zt4xJAN+v*+!jXuBY?7Zbj)AQOlvhRl0&ND~zX$8lr0SxKWFq z9LhH=eLJxr)h^n%k<*Ai2?Wz~X&3gXne8y7DL>TyCahGoMCmkOcNfh9+deUr5|?UTf%4-p%?SqF6Q{Uu<4_BTfA)-p4=*{JYJV& zI(<`?^AH2vsjPv)A6`NCe<&MbM9WxymsZN}vB&$U%HcU1Zauj8PPGTe5}r2Rs-05C z`|6;d4E1Om0|b(_e9JZHvhbZVyg8{}KD4O!V62eSzffF*)Yxnn5w2#=xyhzAHJ`MB z({{}Y9=%akuc0+O*u1!2sF~TV#h)#o|G8n%z3Q%_?rc~KOT!&xIjFT2MpCw+q9@C| zb0P4IG1R*Q@s3t4GSaS_`~$i0P&WnN|y<*{+4!vWru z(?oDN)!&W*4kT8Mx@lMb)-v0N0j8|hn0hz5!Lo$OH^I*RT0tZdtfNHbm8;;PNUR^`4z}# zSsclgkK9NdcDC2B4&h@*MXsAREEV~Q&`Cj0A%2OQeX-d|x_9Ev|2FE#U(E%FV&_=qa zMm)FP&M%4un>?*i2`VgHY)qb9Pv~3A@v3XDA@&vKZ|qNM8X|p9AHvfuoL-&8NE|iC zsm~VeiD;~w!hihIa_4ec@Dnb_>6eEe7iMo0WR9&CuS!rw-ROPT3@hy=iYiBy#V~@a z+UL$Pwte7)#M{9Dgd1P4d41`?0Nin6W4E!C$$y-9E*OBB3yiz=0vA}y`eP|V#M^4r znv%1wG5aia6az$pJvgca&CLLRSHl2nX?YLYTo)V9pKAHEcs`*h_my0&Xi%{JB7WtA zb)4c=1w17SC2Zky1h^M@@t#uXi^p)hVQ)|4h}`qdSnqS)Y&QtGOhX;t`Irqy@~Z^> zK7;Je&7Fr)%dfBX2RYU$%+e*%?%my6t_$5L6nxtGeCWnF6pwu4qM78>RUDImS$kn+ zgp_?bvM8;D z$g*vOR`0aQgq4XeZBl=}eq@ZTSmj3dVa4g~OHv&h^!cH6(XI_zZUIE?Uc+aBOq&}^ zrtJoSyJ3q{k*Op}#rSr2JTp8~FRG#h<4#&84qm)plc+q_`j8E?XsN=hE%%QzH?~;r zwJ-5wcty`OU9}kgH7`{*XHKk9f}fC5XS{?3`$4bgScR4oI{#wQ-bC3CXc=kN&lJkw z(+->2m6es@U#S;K=fo7k#`OrCX6d74^*+)sWA7ns8omfPOzckkOlzO0i`kJ!y*lEh zY&U&sapdyZeHp&yaTcZC?TBY`2&zHa;qDlbX#;~e)gX20@r8Kn+gjLL9S_I1?!QWG zqYDP~8o(Do3X<^s1~?WthjoAD!}LsRF{`}G+33AJ*?^Rjl6}DewJIVB_%W`}F=gH= z)arJbr^qF>o1=Rym$Buhqc62LzxN3zAPBPrlve@acZjg zHnB)d6@K%k-W7%ypDi?NXNYZSBLdtmaQND4ym-->=`Uzep)JJfROYD1Y#0y|N9XL- zVX)~)?nsUmOc2nQTxsErH>u2CMn(DnFs(WV*GvC`0B z`ESf5l-i^lG=}i!J2#`^r$rA4uvBWvsvbd==QlT5{Xf#{MO?6^y2f`nWN0{P*azcEh6c3R2da|Z zt~GgGYJY=k&+)|oOmf?#&`P4bpO*$S@!Jhn?nWr1$jQGk z=h6-&76ahc4IyWkt>1yVcn78DlJhbyXhQ}IiC77c~jRjl7mzDNhU9zT=(q-rpq z?DFZ+1ElMOXSlB-Zw_K_gMY*4{m4~ZVFOj_I07jNTvsNK2G7ACsyKR@2W`+~gZp{D z>c_)2#Y&GHu8`)Js?eKU_29+pgmls{yiaT%AKeg4tE~!LuZWq5isJ2BsuQd#-Tbz& z-?8IHQBVh|yNxGEw6fd+O}^7Ts1@jdJ_|$p&|P4SiQp0<-k7ET8ufW%5m|drTA!^fryuXe!V6 zX4+wwz6{SIwVHvV`4Gg$LiE=;4!E*D*^(k;4FBOx=WEbY@-iIwAR5OSIzdkv|M)aZ zceDVd$yXfssKjXUNO3xV`^Yr-N+n#f>yfhkvO^7ZZjX4cFPn2@5 z-CkG5!>)H-PbzDIH@3zWqB-`02uE#6;#sbrCJ8s`M;=p(-Lj9L;G4>%*cY%`o50sWV%He%ZLa}e(2$)}1XS)&3wl7G85*YHK z*uiP1MJEcRVeJ(3QXSTK8^nJISe5K`bi!3Rz0;d)lvuXm9&BXo9+i@j9u0Yb<3Yrg zo;wGAz3{s$cOroSI#R+gz`Emb<14YxV2dQvz8oC}7@Ns~aoP3%TPDH$&zZ#1j?wkA z32`W*J?|5o9s}|-m`?n0kxInKH;}bM72>#lrRIb@zE~RDkxOH7Q>S_|D*4>zbzfpZ z9|^`x{)Aqo`}ba%KeO&_BjjF^aD(dFrvm!qZ|5m8&&DJ^8B$oy)oa~+*ulcj_O{W) zmY!9`y)3xX#W~?_^RV{$J-iqic7;4%L|CPrvw*oq4yTAg&HG8$ED@ULIi0&7C*&7G zq#2~lSJQnYvVK({gKpB`2z&qMXf#0oAbEcIgm>386EoW05~Bjk`Ds!1#~jxPOH z_*+n!-1~lGZ_9tx@*lwdw0uKHMYJG>e7pm-C#pg|P|U(zauGI?z?B~>YxQZex~Sa7 znJr&qSqrxKGUagN{hk#DNY;VPRHVa4Cq~eg7V9}M77}sT(IV&RiS1dvclp(sTzZ7v z-J{puJVu>?eJN-!Fjsmy0d6P$A;y()_9$a_{O*-M^?^HrKgHl&2j#Fg&q^{;&wuOt zQ!X7YQ(&}Y^50=r7=-cuUAWBy4(`&3qb2L~bmzCDeDB5|Ljspx>JX+UaNsJ7tDK66 z2_{KlfVea9bhy(}R<#L7Fehz9g43;k^sfj+{zmryH@LGt`p?Al4^%ScKV!-g$DL_< z>y9~C0ueDNYKsV=G9q);l${`63u9c<7)Fd-siuTkPqL{a6clwCjbxs@pB+#pf0d)F zH#-r%&!C}~wbJLEzk~r0s>pdXoP891`2gHwF=X0)=L2Tn%9pn=z&URiS^&=5c;Zcm z>H*hX5l8e)V@T~qlm*%Z0}!cnFzrJGFu-;^SW}!^!2rRm^AX2f@i3&uB)FgB3Q;*Y zkT{A*<1p=#tov~8qJ=TQ)xSx>nxl3)PRTVIQS4xzhOFVr_-&5hK$h$XMgx2OS0TBd zUeEmWnjCUkABF}?K7Jjjz!Ro(sK%F#y?+(@J2WozS8$OBy7kcvt#BFMV_(X*dYHNT_^V-Rx9Cq*Go7&9DYHrh+wa#mez zU8WKROXH%I^_1^LTK>04YBfA-ZDUe?K@eu()Fji47k8hxM-j%j@k~ ztS|p9_7AVnj+EcsMA-ZdrNIBuQO;kWM}hILf)EA3i)i(d;uodH;?fDhrgPIU*j95r ze(Dp9gAseEnqKs!^>w9b(u-*4GnjOj#_?~~GuQ~NhN-QwgZawktwfnIS>n6o!|-d# zxEyc#U&);*Nk8K(fm?~VqJgPqjPL9EE50SpZqJe>rXPklqcz}_`4wgit>)E29e-SP z;IXsIi@-VK6UKFTC`KZ4jFy3ptI#1_)r3>NU8hVyrG&{Ulnp3fQ%&x>y9)Nb827Gl zQ`CZT0K8L$FerbQmd+it?`oGPXK@&)g8b6wmv3|?Hcu;BtZ8L+lV@Tkyq9KNn)Ewd zBu5!T5@NXR?UtGIeoIcgIU~aVA!v0NocH}2|^Yj};laqDHG}oZ_v11awU$*r`rbCvr1hIJYpHEJY zn3V?v;~DKo>@9nBRKR}Gsd6%hIW6qWPQ?)s(D2O zMh+rs@*1y>MYBYm>osZTnBE=l${5ss)HQ#}Rp-3QGU>9+wwG9!Zv1I8N$;ub=u2jq z)mqIhkfx;b3Hq3c1MU-yh+qK30!tzn-ZDovGw3V5v5 zrN$a1_WEDx*SM*47%E_m+Og{X|HI`&asPkWDN|CW$BFd+e03p;6_U|C%di!g*i0qt#v z@41aHd?zD({ezrlf$uI&YxmnOh>Dk&*CTs$FaMM7YpeU)OI6#0ORLdwfj5Yuao8Z|?&>(@{_gtD5r>4~V&Urd>QNo=VB17hRA9WdrR7$y zWGl2Hzl&+e#M#fU6Ow+!7;urq$5%~}J-fSW^YCwIhq#0KhV9$0>zgN0u`7i^+D8Ev zlaC;z*~D|;rjQ=Ih~~(LtRt^l*B<>u-koG%;t=~N+;*R!D^JUD-6AsLoC7(7#>G)~Q+ z`K&!o?+LGN{mn=Cp~%>CdW^%WOpPqjrbHFAw^1jY?4C_f+Dbg#C`PX3iStQWiwW^P zx`!-rkFo`$h^-LbWCfsv(<1)~v0QctCeA2oYVAk!Fb;DKrk}TAo#x1tX;PVq2n*8* z%z#Gtn#*e2s1eBP-fNp60$`ej{k*ax?4-m4J%Fy~!m83Mxe5NRdnca&%SXkxNRwNr zowGCdY?wxTM}8Y%`3G$oO#+nIMq;n~dW4$jD*;~@gyJJUV6py9Xo1pjPRqXfj}kA9 zjQo)ZHC+GVs<6*#Nqlsl(|+hJ5X4i4;$;Iz=Nbpr;Y0FTUr<9=-*vMYE1q=VztEm~ zwjfhF5vL<$^Zhh7!-oMrbxmq8l+wEPVJrniCA3j6`Z;np#YI8)P1)XaD+-L56V8&< z2g{+jKi5CW(crA>UqW&G$j30as8n=#E#n=mUrs1UiGu=3Un9)hGhQM^4|YI@m6eq{ zg$2FY-TwSn=bi2F!jplvHX)Lp-hiB6Yu!MgyjHTPY)y-q5J0W~H1OS8eNyafR!`ga z;jCP{6Ocz3k_*wpwl}D*HEW-)6E3whTEd6xVvG#r9bta?f)y#wLs-2!=gx%V(tF>+ z0o8nVG+%R08*66Q!1M`dPz?bw_zrFja9cEq9uB0}Ouz>POqarY*7S@7a2rms+=n9r zH~r=+Uk0*y$ut`b3vNU+F>w_5H_k3B1dYa-?FVu~&wM6M5AHv|O>f$Wv%UQjEd^%Ek z;lfnK9=`62uz;%BC zmA$MA&iYjjq@@+^HM9u0Bx6plMQVDhxy8-AXLyL=u5elA{+m@K>C0?x8$UA=sGlm3 z^!{}U2r?2Uvfrk|!eS=7#jgBSE>)fz5o^0YQ|=U2CN)mMLr>dwRNFa`fmejLBj4Qd zx-{0#(+ylJf_M|enw&U6Y+r-QO%6kJ0R^-s$uSDxnHd)yA~vzJ5dFiVy*~XD*q`VJ z+ED#y4og*GdP}vYNwsgjBFB*nUj8tt3XO~0baTy={MaFGuXlc>Wz8@xx-2jHG-scs z`ewcPo>g{eB0VdGc6mn2)n*de=~~ypI}j(%@FB=xq_qc5eRjB;XtkGcy*f4|M|@opgR#%_H=f&sUlG{Sa}py1OVn>qBx z{E4vW>JK|z&#k*2dN|TQel zip10;UJGG{%%yVRbrN4*Inrzvy;0I(mh*%~MSK>AhstY@*17@hVbSxdtelZn%kR43>LbkL)3uhVd*TWHZh2smPYRX8IeyRj716O&~)g)ym z4-`N5P*L_+l}S){b9i2h#q=?HFTgx1MmMv0ePUig8DPlRe2om-{5NCC-7LHGJi2om zqBy}`)?zJbD!ipfp9^!J+{8`S5(wS=cOa-YHCt`Ah&SN z22O%Nul$kX#^?J3&fsMLv_RlKD!s>cXCtxH(7<5$Tl{#V<+VJGP>F7Wi>nZ59?4t# zM{)06u@nCq^yz|E!q94@ZLZs0+g5_c2#F_M)EU_>RRU+h)J2w061RS)V%X6~Yn)aOP=4=VuC*<$EbbLJHoddr-ipuRX{3XisgSY^4+0 z!UC*JQ=EVhVDy~F$>}rtpUU$&n#~}RwCAST8K3+jMx`RSWO7fJ=a<=!-$m;C|2m$7 zt!+FrV;l*3*y|lxp@E9`i^H;J``toq1V3|aduTO zQm}GLq@C~b04E2zM-DLs%ulW=%PnA)&@K#w&!&DdNY+Q7?L?mCLK69(TWJ&|P`{~& z@GE@H5+p{6U|prt#XmB1R{*1bn8+2Ek!!j=dP8pe*0vP5&XMPtyr3WeFHS@JvXwm& ziA>?pzUna!E-wS04KNb$un?f%Oy=O`;n3ts`?Vxe`@oc`eJ|UyC1I{MqtqV-9Os`s z0P>lTrKCz`#KR;7^;6YlB^kGhq-`V$@;^>rhL-4FA>ilI@?WebW0uHay1SJCK9T7=_q#$_~>D2IECir^NF0ERK~&U^7d*R18z`F%365NUD9y;kn)b zi3K8j>b#UAe6W3NTtnmsg1$VWzS*-poUH))6DMf|7%STN8pP~riP=w@d{8XBA$t*Y z&enUA_6!c!ZQmuZNGe$MEm(28K6W13b9+HO1V zd5N<8L3Lj0Pus&oiJxzcSH@=9JJ%)YE{E`Erp*Smchh^@34Vqn$bh?@*aC}v>UT>? zL0HrAN2jzmL^yOC--RYG5U7>;H6j5OIgy$NLVuW38NVx^KG~&hBf~i|(H#er7WR$e zi9f<(2jL#t%AC_Y%m=IJ-V4lnmIb;=?U!d83x~ImYw6nI8w#rK{mF?m4CpsPdxSS1 zPzF;!7^LH{P<^z_`&*tM+=VMF6~P5s#VLMUYhBp=v1C&twwDuU@1{_E>mA#Qh2+c@ z2DmBIea;#{$y>C})1O21%_-prJSfb=LP6klIAJ+r|Iqw9TUSFPB)Vszr${zQYgLHe zLrQQch)U{B&o@|p22DWVYzC?Cb|~G(<`>b7E}c(2bBGE8Q@3Q^0%TeHiP{*9l+zrm zw)>16%CmZ8cxfn^f*60eF>n7>tYMA(ngLfiK6mrbYjyeki|cG-qI;8$2@xVLvx=3= z%|qq38&j%*)R0-4+g(LJJbG&NP$cr+`!QOG=az(WRRPDA7~_pHcVJ%!_U>m_86BBq z9%=PPPQJ}okIh_ZY6Tek!i9U>E5%D5iz!SDbH!VqDO31-r8M*jo@HeiC3IoPL;~D{ z@n&qBoo2O#QrK1N!qLvzr_%gKfWjHj6c>r7%nJ7~E11lork}6I=DSBc{O>@(^wAUH z?AG7higy)?xwnoCZf?)|&P^c9F^_<0(wZDbc>6-b?6gyM!>EMk<1Kl=23_MM2B(2D zGzP2h=f`R?XWVfvmmz5I4GR(3(QWsH1|41TJ}vz#ElbvUy3 zZr-rOvg~&@Z@XUaSPbE-rHbi}OKm|!W|I4Q-S1R!A#hq}V1vFvjq%9}wx*+rDlGQs zE-#vKUyA%Jx8$ICyoew>Uk{w}j+l$pS#1n8ufkIs%R%(6up7Id_Q86Tq7)Lnf5w&W zMwr>ZXB2Ts>tT8(;a(q&iVJHW5<^%m4!OUhN2*@s*y4U@^6JBMa*WCQL ziD+jH^mVF7(SXj4cJ}z}ywh;F7`WSRd(o!&QnKqO$5BT?A;4{#IVT=x-q?$mtHAlB`$HX`+fp!N+_kL9X!JT@LU%m4N}r;^j0N&hn7hXQl6EA!3$Bg`Y-{Ms~@X74}#7N2YOIH^2`X0Rd9iu)sSP%DzMsRGe3Tf_RD#n4K`5bjB#(0R{~AtR-C z#HbkiMNIOe*s9bRc#@eTfq}&~Ur@crb@kHypeqacTa;QIe|$f%RgQc{)=H%7IaZb+ zClGAnuO4|pDSp|+CFr|?+m2igcfX>cFe99&FsnXUrMeT~b<7E4;&G9va4kAE zai*gixNu`-)1lLF)p&yQ984cW1hmRPLKElWC`L=?sbyc(veaGh(SmRFtwACOq~^r} z5XRn@8i}I%0eXL^nTarIeVIvaIPCWYt@17^&a`9Pk}smSKjCaXK3kDrJ0PH9^WNDZ ztd1`0V+Nf~`FxL|u(r#6bIf?lo>$6>*V8`Ap_bLIZVGk-Q&KSESub!je&KO9sXz6Y z-ptq7%#d+?=ypG*uDdQdhm?L&2n@J>Q2YG3#r!(0Ze)2b@y1nW@Oi*)h1WE{U6f{~ zPi0CeCTP{vZ~??%=G&{8BQWM@wjc{vns53%nZ+WxDkT;)A-psw#Fu`>nr<_v%B981}$ZH6Tc*x5quEYKh{SHpGY}6UuBPr)w z{VrUh3SK%{WBPS9{`NMMH}nCASG zSp%A?U2Ab`98d$@MIaZ4fvT&oiXKH`DZE_Md0QFkiu=TVHkF2rQOCntxcBtc8F3xQ zh~W`K<>4pdQSLBx=C)eKZi}(mXg>3Tuy@x2{7Hrg~C)cGPWcgM4z zt#`mJwr}g!Uw95S0s%X=e2!u@V0G)3&J(Qz@P^aX@J5Jl}rp2cYmqMsY z@0lW+(38O8o1iVhu-L>1hma=u#l?v9!dX&1^Xhh|B|&W?wNR3_4pKM1z(K-&h%Dlx zR}n|iiL2};ExWK+9TVf+=%$M{Z4_|hE7Qa*88dCV8cptu#B1K|=iB=4CujQBAh}*g z^YAm_c_En-h=oiW6PX&;x~&;x>a^CVVG~z?xA)aa9*;t;^X(c>`Ax^h3}Ge z#z>d%Il_yoXyZ=9)j^@p+-vS-PfsPKwIwFA9qBMFPq_m>AS+c(T**FoKmYxjuS_Z( z4gvY8X@TVXS`wb2w;(EM@)2|`2cz9$<=8G%mJ|~Pms5I4QOTrh^TdI08XvDz%c-yL zKGkDfWZ~x2$_ZbAHr`4XGEuL9{}C;)bWmM-U%1ZyM7AqLLmt0XfiMJv8C5q9er>9Q z!WkfvSWIrDI$p@gk`0$paS^DZn6c{+GnV;wm<<_uG(7*SePt0Vk4z~fI#DDlpL3NfP0>@Y6faBNp9q&?PpLowIe zI2#MsYQ028-X?vPlv4(U(@rshfC}N)1w^NLnS43X)p3v?=Fkn7tkCFAy=$g82_K@^IGT&N2q+Dl92k)r@g14n>TLWyC|)m)CQpjf@%G{%Fil#%e#tIG z*(h^aTsKY49FQ8;H|_6A5wA^b1{_$neTRdf0H}*8#MEcPxYNI=1&TzH9u?fh`a$0OrkI%3h#12xYg!!{5LyKj{)m%>iwINNR6U6 ztROsAfSiuig0k>MdAcQ|P=7$OowZ006AXPg6tSclcK$04)V-J*=0U-(dAc`~hDSqu z4&WOt4YX5)#pufZ{$?$QS2K^vD*la~bVZ|`)WEf9&D>2}bzYKEcNNC`lAgB*TJ^px zbi+F5sZGnTLr7BS-EjS_#6dk5JIxAPYDZ(Ef{e3MI(;X}XeJ=d-5)b&&B{@3V<^!{ zq~R!Bj_Lj;=`#}|+%6(5i4lwZk@5SN!`H+*aQo7o<(F9GBOvt9Q^!8IK9(k?85G}+ zg!m09lApiV!IZL!|An`(UCVBf8@i(KtAQDG=3)W(qk#(3x5Hj1+R9I#s9?*+58Kg{ke*t`~JVVuOd;4*Q41G-MJ$l4YFsf z0XT3~b4PY?X;G-PB0q7fAgZmGr^ji{NK3cmF$+!HM|wM3xh+=l9{A#`s9G|EUzNE- z?szSW-_+O{PW^Y&sNJTy?oY3Lbkn}mQby*YK>1x}H~n+e7G5#J)tWQ^>G_DNLCrU4 zA3vNQt{NH4L2vx&-8C;_w4-^#^XZI#Y|{I!D@9r_p_}yP7dJx(oc#A~qcFZUb( diff --git a/doc/v1_api_tutorials/quick_start/src/Pipeline_cn.jpg b/doc/v1_api_tutorials/quick_start/src/Pipeline_cn.jpg deleted file mode 100755 index d5d99253ea528ebd2d8af6cddc6a9f00e20de8cd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14244 zcmeHtcT`jBw(p_|f`AkOL5dIr1w@ddfJQ+;KtMox$+pm&qDUteddmg`6qQI5>7azr zBVC&GUSdK=kc1i_%)!ZigyZM=Pa;1b|M?X7Il{us^7D@W*+KgZaIpXv=<^xq!~uFPItDH}S_>cw z0CY^?YJV>HuLm7H10&NRa7nCe;0>sw;PM$5=)uJ@F*1U8hk(}sMlL4qlL}W4@tE9Z z7JtC2_%tz-MMAT@@tA2p0doG1&$GjHEkW;>w5ZT zx6Capt*mYC-g9(vc5!uk_{jIMpMOB$^RV!U$f)R;q~sSVscA1?rDtX5JPd=UpM&E{8${uK12lki zIDiJUeV_pm4Wtknpk+V@x00#*N)an|qBDQd8MOR9 z3`+w(r_lg$(TTCrAE1gsJtdv>=f`Ith-ce^-g26IY6>}T%%60MYHPVi4>8yHwzFoT zVLw|4rFSuN3q);NTU{}Cuuz1`-hao)6b*U$xrFCi5Fc@|MN<#5QU+g2f4!3f(!2#< zm2{>7+1mao_jyO|l_h$ophj(#GHAg0$-KDau9?%h-N`|&nNnos^=2ISUjz7W=9c>( zSm3f(?xyW7Vu%KC<@|0yqi9O(Gb#co-(u}m z+2>K@woORan_u6FNtLfnREv__v60PGr{7m|GjyN@!Qt=C80VkfOan9nmjzeskiR)s zuWV=8c3CINS*CI%q01oPT=)4iL**PP%0iHgf&+5@KVg=m@SU@42OuZi5Ajy|FRkrP60HecWEs7N?%`G&?Azp`r{lK;m~e&i+$8N89{i@t6M^VU)o* za_1WcLj#`5Q?G5nQRT2{(0xI;h1~uJ!!-Hb!+%F-Tg)Fh#qR`_;C~|r`@1-G4)f1u zEB{&K`!~h^S=RU8mSbT)LA6y_3UBdRZXvQjCDW5J<5nDK$SR6-90dU)!^Uf-G@w@x zx}I&<{@Fqf4;SUuld|{1Yn*-pvwQj%BSw()Z4vvAe&VA!Z0Gd_ z!r51AB~~9s^pCrGOHI|aD;66ZT{biMsJBvD zPaeJdKh9`Nwf>gUO7ZVh${b)wY#@Ti$^YNEF!WE>T-m7-}yCAAd(tt0K z|EBf#waWjNykZ>=k@9ka4F;WR`%pSVa^K)*b?B{PxD^$JZ*5#E>{CUqLn+%1!UVZk zQZt?olH}_{=M?^P_2dT}EG5d<>do;ibYqR?iwktpbVqlqb3)pqo&MMpt zYD+we#8rM23|}io?4(RR=6fH&cn805WAA?c1)-^D1?N-m6wo=*S9!BjhZ`7(p=mgw zdR6Q6A5dYt9w{14T+zlEYF%TcjU9oCZzDAk>ih3bFP}8+8v}!zvg~oDV~K#Ir=e=t!sYma(4@ zjZn7l4oXDc(H;09rRUizBbA84F=3GnD4TZB{WLb&N%nyp5gP;g$#x~1_ic#o$pt?n zpV*HV@DLB~}JO8^zRl_Lt%p=Jm=_4%Z);Z&XpBjdA)J?1t z&e1^_Qt;jhp(2F5*wQVTFrwi|UT#$URG@PhD#S#N+&c3#7llgINNH?a-4?(%JfM_{ zVMv!~0OK2?Xk?uw4LH0T9sB@Jz*tlcEk-O*$qnxxSwk>Ybyf0nSLre72TqR=K&z*a zx-OeBk-t4~w^{Ufu+_maYOH&qjel3Onwl0g-aF$5%MZu^q+tT08x)l!XWu>nVd%6F zGU2|^CuRDzY5sCd1!)@BMFYYD^Ac{?Rl_-4`rK6clP`sL^GwDbJ{7ZOGh;RI3$epn zkyn~yPK=?=!+~tmSFz6PZ>E#Jf>Rb1(t6;VqqTDpZNOA#(jhxQ##Curvam730@$qmxN#1+wWd)MMAa4oEi?4jy{=uDk=0F>jkT7WAWz*^2s4hDic6c^k zB1uTS%b%&^(We-;r$c2vNL%Zf)(^IjK122v#kibbzX+cQj5xuyRb(6`%By4?-j!Kd z?I)Q#1Ow(B(m|cM41sF2jE3KWNYZMBSxtuqPO{ zP;vK}ljRK}d^MkHhlA}t#!kD~UrFso-|&o1iE=qs*BX5Ym+Ix?zPL>0@y!bDui=jh z)>C<%FD_v70r{ro;glzcp!y;b;_-%r9NM|Z!S;a7E%1D>;DB$3GUs8e z|FhX9krN@NKU(LihBbxNRO996U4<7C^W4)A^EZya)l8M`U7!KnWZe)vyRO3^Rq*>| zNkV0Q0Nzzel%Mq6tZc|bp6m)&F0Om7U*YeuklJZp!3E)OWw6l|usZp&d$8%vYO?65 zVo_^mq(8kYFvhc*>=k+}isyKYif|J5y7CMC4&___hifF)=%ZE^A_?KNG54GvH zt_N6%{$1DO7-lX&6wEf3rFWO%U(#W?w1r1J@M7=$_k-_gfDP8nur$-TSF9j#G0HAz1VCm1o~JMY>X*uHX%m$&Hu zq0Dw#cI|8SD?a9zY#qmQVgy**HR!K&d7nUx8{?zvltIM(`rMv)O_bYrotw1Y^l^p9 zcB1FQifY!zra|hh1UtQX8{1Y^Tz4sZK?Pi46C&c|mhHA$SrQb!zQ&I;P_Gr>hH<;` zxTzVIRI@H;@D6XwZ^H_ExoTxUg`Ee!KGXJ;}QX@)bHURUvCz?ai?Ps~R$R17jvm4z_8n7Jh}v z8CR-ATFdZH6#~wL-9q~BvIL#+kS~BShDcqp!nyU~)3H>v@ z9Uyt~k_;XZet$(3Q;uG3%(AoX_+YiN8>l=y4>IaGW3?*w+Cr7zWMY!#j&nV#(o9}X zqBFd~N%nFIi4FhZvV;NN0Kx8NBU_p1jvODD(JgXQ8#g!zDYO5)>~oS!pc3LAUg^#4 zCG?zL$CAS=Yp_9{WND2eylq8BZMoMPi_p;kmgymW8M(@;8;9m3>$+afJ$?#FxuxQ8 zA;-5f`{NWrK4Ggg+TnPlb-!2o?v1Kcm&`Yf5Cer6%jj{&nB3$*m2qBPU%JFs)>h

QOH86#{YS?rlb&HUB&5_?ud_EcjPr zOcpjfVEOrRa)o${h>kd!!gjGl zzmNgNS`M>*>8zNKn$@MxU>hcv#o?orJZhBkz^F38fT}OmVhk1u6*Qp#P}QdKt{_I9 za$0yCb}$b`&KVqlU6e1?ILyKEBIuS>Nq9~ABE}WI$FHGE1MVtEQp2|U&}P>6us_ai zr!5^XJ(#PP*pJrcwamKz>hc+4Tk^NmL4Nc4gr4of-qr6+j!y2i$WyN59M;2f^S0d- zH63hhT3ev|1rw#YjOVMCC%`f$WdVcIo-RS+;m=k1(Pm@gjk6s&84C>2p8dfJc>1qz zL|F0}7lB-l=_h@AXXk7%7p8!xVS%d$emPP5-NIVU$DS=^=VdZHk? zE~Wr}zd-b1L8WCYj78teuv-!-4gW$=b1!{2f$MGY*0KZq(&*O&3)|=)69Ff! zGF`+{StMRS&b50(YMyq6SW}AKDQ_hgCp@X`+)D8m6R+tPa4g8ZeRo8~$baHtoaOls zf^KnlBFxRxhS{O6|I9?b2N9Eve|>X$~F$|g~DEaRauN7ba{)CcnoR4 zvp{Pa(4@eW)&z^#zccCLGfryglwjO(@{Mr6mc%ypEyK3`E=#rT`xrf`Uq7Wznd!Du zh3kmxO|dlKh#OVBVTkNWJT!A)awWcKe0;2#x3Zvbd_s#Y;?q+{G@{#kJ7LC2@5P5; zTxZCcRG&+on*6&7le{8y7uz>W)%!56>rDQyztMo_Ug?V~Du>AcIi(@MJ= z4qqhD^i#TvXAK-<#aLo~Oqv)R!5;D-)O+3}a&dHPP0f0}2vuF2iylM?>#ZxZEtv|u zL;KYv91b1m?rRM1q?<3nobCDI3@-oYX5t1;v{gruVsa zq9=+}iq8cF;LxtC5)n$b++&A+-OJJ}N6UHKx|{91#L&|J#QHjOX8WPn4pIjy-vvcu zFJB5a86m6sI>X<&1$sv~4we)xDB4V2l}u?_hQJj}s*Hr59A7xH)x~7rW2+2w8z8M^A`S;#5TUg-ofvn2qDaMM|6*e2HrybP&EFbT=LeGdlv3?b0ezW^J>-D}8wjCrfG@r1!^MJS;Re@Ly zkk?Sh)tlY24KZz9m-Fi8GrKOHq@1c~^6HA$^)ps?--PW{4OBOTN|6GB&3&^}KK+uO zl9HQ3CThI4xZv%S&ZaYSKdeWVJ4^abcSoF$VkMG$OJ<#AWn#na@p86{53Cg)bk1K4 zrJ|^JY~1Po8N^n}(txJ!!Ti*m@BM9yyu zwSEOw-}k%sN28SOnXe9O+&=ZmFL(KYu$FBSTa-i$o8c9?CTpunyBiH=*FIoeD{t4_ zy;zz_J%Sa&u!q4|XGo8wah13I&bXFq71e6K*SnMO)m4Qt-8ZhRI*)ko~v%`Ki-aB3w1eV$(AiNvcS?3+37$5l20DWBt zd+HwN0eDI8E)8()-Q!Zi9&r5DIO3sfZkPD&yaD^?KVhJ39zl&$?uTNbHeFzY<^iGz zBc1jd-q1Aw{qVv>J5ZMed^z1~eCaV8lB%96I??Tu)`IL+rL{BbW>roevqvY|Avq5^ z*q%PnKE+~znS)F-4SwCe#JQnzJi@xNr zkdW|h^dmpwA;hH^RC=?YQ1gBC(T+16JQ6Zu8}rdG-XLD8GF&*`EZ^KgG;0s#rX{LCZ5fX$eR&B&`UGDQWH2q<3q;4x@%YFur@0RS2KX~T`;gWwbx* zuKf_)HQr(Xk6$}ENEKT?|6TBg8S>1)j;F2m#$3=Nbvich$4`k*$}ddEgquv>yji`I zfWWUo9f*f&b8YOROUdW`gcI^OQ&&7sCC$jZ(6qkd(X3&Z0%0G!@XM8f(ih-1Nuo~x z@>ABui;6B8Y>zKWJ-EzF*pq$O0m-QUzS22xc3IJ>+VPjHm^0C=(pgU~VxB@bRLNIB ztEd5+jEN372T_6)RR>HzMR)jN6mFR*nfXt!-3jE*IyISR$!C364ZgEWebFmGDT2-+ z^C%H(@GVC#18RB0d$3pU4tC-BD9}|^cW@U~=CSd1{C27BUyO{DyD#$Cle$dr(k+tmb2`TNqF0^nZzl_^5s1zCL?f=-BL+-c>dn z#TXaA+e|TN2~}wz1^EWolnp<_-fWS$5_qKub0+& zEprBf-mn^a1n9P`KcSu@)gPR3$8u=s&aSDOjA$6=PP^FTMd?Pgmeqf(;yjWwk^A{t zt-JH>MV+prNNzT7*^rlG15`TL5?r82!}2{T%0}-{eEt}g?<0@lhiTY~%I-jKs)x9Y z7V0ijW&SmR>un*IV`L+{1H8%ShgYcGp=SvaU@tfj9i2%$7ipf}*DhRlj&G8QZ|`Usxzc;@V&Zx@^NF9Z}a(&q4^-Ktc}XDim^#^|GG62$~M zAPJx8Lk`6-yxfFE&;SNhZzS^LdRT*~HECd7e?;Saj+Luai<%eG6H-)({d%MMDE|=H@ZTD_OB(3mQCzlz8PHOI%tDHZnt=lvqeA{XC;e@F7X*S0(j}5lU@6jV)h1((Wi>fWv~Q2CR^KB7 zVNRoROw6F;M;qPwBI>=HkYd>=u6t7p(CL%cXC=ExG$0y&ds?>z!=<5BB6^f``u1?J zH)njYcpV{Zbo5*YAJY3?<))E5j{Dj**?9{ku_~bi#n9NSNZyv5k#@ENb+Tv`=_Fxi zQM+bHson_x-EP=7EzM|9$Pi^*iS6~9*JUtN&_;Y6?~bp^_fq}E&#XKX7LpTFw zE{GHjVzU)=%h2!u+$!ie1Kr{N1R`}-CP^6Fp2D^mAFwvkfGjuI1n8=*`OqDnzg%tz zMUxQ{-$f~sqE`*5*9O3R3v)j6HN`!|Q01-s{>Xm1X9&aP0H(YCyG>Za8~;1sb(1cM zxL?v&dA1^F^cgnU!*fJY;CPCt7!44yd_Q+z|84qdm-9_|l(0wa?oFYxgoICcP67Yt zpF=h7AzRg)ii>+f0`Tq#X>FM&%V{&@g>SX zLy1O>2)3c}C&6~AU(|Z=*(@uM_c{s6_2_4K=^rIrCn8cA+>*{6kubWt%_{wI$x?mA ze>Vc^SWV`dMI8DL_H)u_W}5?@^6my{R5m=?kDb=dZRWap-D*HUsVa*h-&|`n!0Y?B zyeD4NW7aF>*p+OXG&ABcYVAUC8G2)6;h}0ug#u%~tm+YlF-v0|X+62H@p&vO4RD?| z28S?K3vtGg{UnT4-*zd7%dg9Yr7ou~%*Wh{lQB5qcNn60$`3ddU)2mG980!rg0gGq zGJlSjX{td-;rf+pStKf6n-d(m@DKTue4JbR4^F&?4Q}+F7^A?}d0Bu3e zmmAS=XM5+*=n-SWV#|bRq-;gk`P+G}4dR5k46#rBVxJGGOB|bjLn!>_&8bw)8q;7S z{$Y@Gnvm9ltQW{yU$S}TCHjKvG&kN4GHI@|rkAH~C<=L-|WvJmH^5djsWkpVeIpz9Ozdz25*GNAHMp z`>`X5lm+i*c>W}MVX9uZr=pk<>-5UgE8|i$G!-%#m(q!>dzZX17qZaEIA^RarZI|( zTqUYL4dw7RYeG|HM}Exry-(jXsqPUU4>Q5x(#&XwW+%chfT{)vOKs8UE_wXHt( zTeB-h5Zk&VA~v*z^5&d$f!(c)IWA&+<_CW9Ym)1LX?I&lRhv(4tja0}K8AJSf+#Cl za5$KUAajl26h9mPP_wQ-J3(}fqSDkd^Cf2WE1&uYFS))3yy9Jt$Mo-p88)1;^#xdKh1(vKn@wvE&)@4U@#2GQ9eSDO#cS_d;ykPXNE%}f!RqZ)>U z!Dgb%h|PiPb*HCI?@|qX4sxPTw7oKaP^I%~NoD`5blYhcNUt0vy#a88GA9uRj(f-Z6m|r2S)Ul z2|+s{G~m}>1a$SzYZ`DRA$bSWtkxLfWLZoyKgjEK%OK1q)|_ zDQ5F+a5fJWOa`sQAJc-_dJ**C=W{fGjXd5D79aZ4hLS-`L;=t$Kg)39y`OkomI+oN z@`wAHJ*u&c-YP5>->lUjD&rH7eQeDXXjS2?`$Kh5llPe;A??QV-gafjTXi4nI(VuwwUwdVS>c+;THEoL9npfCy-EpR`x*|}yTUl-HPkd5wpp@JWemxeh_h=}H@?i3`*u-IqR~r0@mjg1>o?=) zYVc-+u>W(LLHX@+{%ARmlQjbiHS&3_58OVsdEbf5vE*ro=&7G%lN!H$M>XQ{_Yawp zH9UWj`?DD7cl6bmC&-QS{=3xXVi4t1YxEbWbYQ=wXBN!G(bO0ld=GT006vM?H&xqM z-~9rg#O@o29vH#DM1$?sF`ssD4pMLlxsEJ;h^bg4N2PmHB4y!Qw@ad_d;237hj+>@>z8-KKVXUH)}vim0hFqkQhGu)r9$l@)P(HbC)mjt*H1Kg$E_`G4|ONYp$G~1Z{Cag|B zR{zsL(JzBVtDeQz-ehX;PCCH3@Szdj~cb z6%TcnJVZE9_C;=yd5Pmv)U(&H;Kn_i_{hYLq|wf!!;x~@8n$<5cl*XnC2RQOf?Eld zmFs!kP@KO1IyL6^o8e6_h*%15G=}h^k_O@zkZ|YeD$ut9W!K_48Dt9WD zY|m>#DUGB0foBjO%oseI@VuxdxN|BtHKamU%FJys$ig{PAjdU zxH!#g1U;3|s`JLBP7zTxIOsslI~U4IT1|rE?)CISH_rUDj(<>iz33kbh;IMilE%wF zFY6z)s4Y4{>Rp79jzIU0Qaiwwlswf?ev|W0X7OhwN@pLyj*~$rql^4GyZf(W&y@|w z$ddIWzXwHAShr8#(KT~KL#1{jzNh*pO*RV8txxir*o}4f9y5*0yf_X0(duh&YD`qg zOv5epnCA&wzFsAF!HzZX_Yzf`Q%6i3dUJZ6L)gix^u23B4wuF{x!+NKETSI53Aw?K z;R`v2y%hc5yykg3UfOKuR<0wanIBVdJL>bzVyL^R+CJvL7G(+YZ-#>S-HiXgQ`|of zL<4`e@vQ$4`ra}(A;T@<>_xs;{ql{Ar&mVb@38avfbO?(1(4HK-x%`%HiGWj7MyDck9#kgu6KPO!hw_s!=60 zpnIXW5hk1j$ItzN-2aGKGDFm{HX86dYQ2H}>v+yTuYiBiNbnN*r^tgPnGxP^;#w<< zyp=C**Y-GUu01rSWXJ)YnW6Yk91q^ag-B;rmj@)ALaFK(l^}QTU?A~jd!Zt6l;R63 zb=bXRvIXg`Y6}=T{_m%!chqc=OW?cRL_o43P|4-rk|1}Oetv6x9`7s=~ zhv$B#5Aw>fi4SFIz&O5-p5J@Q;&!iHT!_q&MjjQ6eOM(EyUPQ<1)asZkJtL)iFM0O PaT5CJ9~~NWY2*I~P}htv diff --git a/doc/v1_api_tutorials/quick_start/src/Pipeline_en.jpg b/doc/v1_api_tutorials/quick_start/src/Pipeline_en.jpg deleted file mode 100644 index 21a7a7bb6a1af746120e6f4f51f797b6aaafb9d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11633 zcmeHtc|4SD`}buFNp`YMh3qLyD$7J6BvJO53NeIR$Oug(*-Z#Vc0#sf%R2TYTbArg z#*%HAu?}W=uezV-x$o}x_uTjM{Qi00=l8w`pPB13*LfYs@jZ_3c^>C^UMFRk@)cm! z)zZ-dsHgzoGWZ8jrh&(r2>4w9(ANhf0RUhCXsP%BI`9f?0;o~(|KnPVN*tj1@jEpD z1i}H@zqPpvet#EG+mB~{{iaEw`de#IPYU(lTT_Rp(ER-xQ2hQF1q+h9qB z%Tf80r+~8>I{LKVwS)5GTJ*;y>wsX`2nzT>yZD&#)e1r55v2}bV*nJX!)d6F1JrC( zG;CCqCIA9@K}YpNemn+#P*Ky+(jA~@U_8hKHYjBUsHtdZsA*~F=)Rkw@&oS!v}|V{3QU-U06D?&0~s3*r6H|5?EEz@XresOXn5v2m|n$ET%d zWWLSH&UyEtxTLhK{9{FBLt|5OOKV&Er@sDyFM~tFBcs!pnc2DdZwre{`1Os=t?eDc z?%sD?U_5_?1%CdD><_ruKwQ+cv^2Ez-*Hh^Z|BtL;Blp9LMFJFmP%_ zrWVyR3drhLjks0rdFcYiXYn> zmV3zmX~m)yDik_vJcwNWG(!P2OWl+BC_tpUE(IvO4Ba%#Zo?@^8opd=Slyo3UUyjN zc8kgR5(fYDX(DbZ2s(WdA4fWvysOGX0XVdYqLXqIAoV&05KE%~7^k$f8q`Xs7;z8g zi{2IfMDA@y_O$M36C=FH>}$Np&p8wzx)+I8KsH#-$nJ4L*B{D~PgF{ZlRF!bJ()Wl z6d;NPD_}|_B0Z38-;jT0>^ECS|5HufC}jx}66te>TpOr9Jv*jZ?n5d&(}Lmrg)_Rn zp!Q>O5PJQ@5@Z{Uj8-OGTcSbNEi|LlZ;k@A^M2_%Z5SGry}6Fs6-s_L#Vo5=JusqD z4?h&m)ve_1XvvFxW12LEm}>Tl?X?<1So&U1t5h0G=QZJ_r*rSCf^Z|-a`Py_P)sX1 z)bX2&A_X`;A437GUmzEXtcA3nq#A`~necd(t(8Y>7r&X2x}JNaKPIM1_6zGz=|d6p zs_iI%Mi$vXlL9miX=?gFMqU~0Zr1K1*R^9v=^+##8MTVq6EcqfV~_?2&;-t#;96q z4#K9rEYxLZ|DJPrllWH&v1ltlHz*r&R(}ABjU$Ka_C;KO_Hu?t$8rCA-;vLazmv`* zYPdKDSyz4d(aNB;?7U09d-~f1q@U{Bo>OBQBE}n>avL9o0{g1ot=B-w+GsB17>XRM zxLzT2IMFfM*tpM>Xhbr2+wJBl-f%LZ*k#HenTjNHD?m48mmvfOD4`NMZF3RKH@S!c zJocvmq4}dy3&WC>=UDZpoR9`fr^a9{wF7!b}vV%lpZQ?rmy zmODvV4Q`^D9pVFnwmP0TywB4kypU z++cs|M1hYsA^j_a6;9^<;$d}1s>6V9l}{xyH>Og8Htr;S$2wPtm^1`Vr<Bgf2u0&}E0@5u#bd_iS*6eM)DVo2aRa;G zePu%u>KR#kCvdw=YBz4&^5?*!eh&NcgAnMC#NhtLyf~!qLl}P`zh|%^`>|-chi!xq z-|f_K_U>bK`3sBZpLu!!FD{4;7MD)(nLAj-!&z*+eeUCew?~^q8n~sRH)sYPJ~V+f zCT|IWd4JzLwXU0+4GvcNuzea7NnCNZw?n+zW1)04cS}{bBEZ-`z?Ssw&d{Gl*s{bl z|5waOuUklIg@BT4;mLyu7rcB)+v{L%1SLtFe=|kP3BsiYvX)$s|dz&wfxgW|YaV>e(%B*o-KqSi*!pot`7dr00lkqc!7?v1eZlW^G9B0V{jtCvtr*0i!GR~JAxEltl& z7w+;X6z*+;qqb(qR3Lg{jx76Ger=|;f9ss8J!a_frYw*8SIhm8Df5Jt{j{faJ1-tV-d)1 zn|b%)g*VqaPjl7)5sJS;2%MHyJLcc2v;jAXE7QxoIgUVDyl2n0|5$Y@!O5%qxz1_1 z9?lR`C+Wv&$#OUf@O-5fLX^TM2PjNYFUd`ohr7-UaO(+L=y~>^-14c&E98=S?5Dwi zO=N1mhusNGETyDS2<{X&Nyv5tC&2QSMewp$8t)ICZ2W&DRSb8*ZPc z6tb##4(st;r$G4C;)zf7SyqZnk_V!9>P5m_0#RN3--6FhkRi17O-b?ffyllE>Q|M6tvmH zJh=ottP)#OGLTPf!zwz>qu0}`scoN%c*4KL+;EYT!@f|S==aD+8sktcC|06;Z?*b# zGPe_JO>IJ!!P})l>#K?O^1?cY%JpdFCMTZ98qVrh7^iC5*-C2~`agwUmTFZ%Mf-vyYjnZ7qau! zBzfA;1r!7y=Ds%BDuM52beq$i@dS7BUy1ry<_; zi6^hCWlK4JiAgjfU$k;D?qSXNplL{KomwjGgYE!K*@OMY?y;wEp9|ae3v)kB2iK7! zjzdk`kV7ugDhABs%&z&4%eHZ`sB=V;V)}<3ll#jo!YM#~qLHd(!?IBlKZ*Nmy`8JfM5CtjyU{L*%STNkKedE4 z9glwei_~!Rhl*KEtE7Ohg~7G-gy%z_3D=GGH3wsDcpemARLqx@B zeQh+izP=Igw5To4Sm0)Ntel%Y!|f&~)ukLXV-nNkz;qhmesEA#hq_i7FGB%>d`&SR z4By+hWPe4U8d+_e=4vE!=>U&~;&!RG+xhXlYnsQKefa8%-kYBhc6k%zf9fnQ0Yydd zZ-O0ClU`&=apc+K^b!*MqL6!5A>347hmV&Ez0Vc18FjNbXy#bMW7~Ujt+8~)p1h6> zq$`EegeUmqX>hHxUu6m98=taJTnxC4P{`NV=8nR^P8=bQS$f;Bx1Zp$krs&h2z%|0 z4{WW^TVl7g^p(fG&AmHn5}q+mc(kjK?}C{(I%(_mlr{X2g-7f{$Xz8}yO^8D!k<2V zlut_847aqv#RWQ-_(&9H`Ztr=DBG-Z3C};D7LGZrt#!ezM?>mP?dbucXs#I{dS<98 z0d?51_V@r+(B|{K*rnBxlg`~wY>P!vv2`@kW4*J_cqU(6uK|Q)GYC$takYmC0jUmU z7Byw4Y^O&I2?pGJE`@0U;ibxMIW-wdQK|#CkB614OQ=7#l}uTcO5BDV+jG7tlhq`{ zP61lWDFBP~(aMhl3fae#);%7RqQ<+KG&wqdQE65GC{9nqD|{3Zq~iY7S-VkYMvz{= z{7WisI~pTpby2I|w%^mw{>lJtytI>Q-qRkBWu}|*mm`j}3bu_KC(NB4MK>b(GLxH7 zdpa4nH#q&*C;v?CY6!K$ZdMCEllun>VV(#6jZ;Y z)CT$r0iLsL0e$8nzQqhP!?mQ(^t*XF9f-~)JY-@%>!9Q$&BV3RcQcgNMT1)!!y%H- z_vDS~f?{}LS)!e5)<{;Is}r9l-)z~y1um_tI(33NT{rFSi{x1pi2c-WOMe;!mKrDk zw@~iJIj~a1rK>)yE;Inw&}+@G?HRDI>Oai%woax1!c!q=EI8G>6i*=Yl0^-`s*CMa zD-1#g8W3Uo_aZ)oh!`Up#+u8B-ngVm z_xg1ZKoxbB&Z}~=Dh=VJmu!g_N_EYI4~|_aSGu3Us$KE4U{~arsq_V2l_)yLzK08n zc; ztLZ6R4MZpbHY3YUfLQgb=y~UKVf3cxmd;oVMymPendOt;FLQ_t@x{!I4|*xFjY}~k z4U0#L+)24aUAP!qk-wU`SayiQKr7gA8$aMg6E+Ey1$(>C-lS)d2! zqdB;ePt?yCN623rb*IF5=qwMBo9Ui$AK30_FLL^`5t5(iLJnfiv-U(Z=mePqfEV3&@sSnU?9LZW=z4bnR zww~9VT5Fwql;H*ULJ67y(b1S(5?hfw<**mi(^#Q$(S|vXJJKL%?YaM3%TY%TXS#ws zjKkcMhzIIQYa$cn9#CIbeX{j5nzzNfwLoHk^vdT#T8PxeQ)waMG)&Vydni`q@SU|? zQR4yR?At=plUDLezEa#bl6DYu<+K+u>fI7_{kkc+4kVv0v$q0t*50(2e0gh`0<_g8 zzaw@hfkZP~rw$QMVLA1Me1>=gN6vCfTb@+=tT3U`)#*NVbE)h_7e~Z9eD#x;`)u)> zIQO=~gley>VZD7Fq6}On-Tg!~=4n$}Vw0dUExrpO6g?wodVxSUy(WnG)?(2$pE{Fh zBz59r^7TO7qT=D5M@H>lv*p9sPg-2F*KDH<`M!NS`mN{gD!lz|;FQ<-?nwLW5r>_) z&+5qRVEz2+D9Ns~&kjU?((qENQ+H1J%Bs%?mJbI$-}tgBzXORjjD{jRa?+=dJz%Ix zY%{EQCEh;!;(R=DsaH*!s4<|%OOQ8Du!?eO{Xhui7;ioLs#8J7bf328Q;Y-d>~GtRwt-Mc)XKo&Ge+X zZuZTS>J6mmN5>GM7+BanPpnV#kWtr(ryG-S)A*MsT?*$C87s=(qnK$~TYSs4X>2|e zFw#xz>)(^L&+O^t6HhCVKPaB|_}ml&vTY87-UrKmuf9pG1hOp{+|(U|kOX2(|LJ1V zmn=or8X&>RhN?s*VyM_6v|Fp0(-3w6L8X89hS93VyRaw(aCkn^yt0S*R2IGb486yg z!dQaUHW*acx3auk8M4>1Es`{ep*O!1FTp9N!=EGh>65w%?bE}Ww$s1eHac+mcr7OZ#u7}_|4IR#3LWSt8s`w)b)%=R zoPU}7R!6`{&(l>Xfm6C0UqMBi_shqLx;r;MlqZ`Zk};xR-qzq-FMW|MP`T}c3$1_H zcwEgn7DL+Ns)%tOJjZiSvhe)T@VaQX;W})dUru2$)Hu7ao5_{iSECvivw5DSsfr1! zWQ>;$P}o;aWVva;IyrnI9q}F}9e3Z{GMhbBD_9T$mnABjM6wk(dtgww6DZ}Ezu}GCK;?!7UZ<;u)$H$u2d$O9oym^Gs z*1<^;yi)ib{DX|51tfb3yo@@==7vYgPk7d`e;8OaXjtK@bEjrO3)v!9A!$uBZB z{d7!uJi+?4!fB{^TEHt5Jw)&#H*zO;)bN|ct8-}Jso5DeoT8inN*sP_8#$4dOWkWO0l|&0&=R^}VA-E!6ehQ$@LJW|l z0BqJfUmU@jxsT#l=A;0x{k}PCxB%Pn|v@}%JIkh$e;W(&LyH&S@N!v z?RFAN|H6CiU!C{=Z~AV^f#)G#nz%?FK`sn{8#l84$c1AR;6*05vnv&cPYxq0?qqYc^Z zXS|L;w!3mW6re%`qaMwE>`B)$J-X`4%X621`Sh6W#j9nQQlrP5w90_-7KtSVb1*X8 zVM7_dmlX4tX)&$&k(ZgK{{e~lcS-f7i3q-;iB!5q0m@)|isInrap>2nJqF~g1d5R9eQ*Yus4@r*|$*Dl=}pEtr;eN0W$#;EQ zOHl4}=uvYq@TXuE37pYXV>F@oYL#WatJVARcm}3ni>wi>;pE&*7=86?QNt*Mkue$| zegpcPd;-6Q+LC=Xb=Y%_b?K2Hx>uox15Oq#FOC&S`gRS6?SAdb7<)~Grqle=Zg{O- z9Itp2q#8OcikpIAEe-Z50EEzoY|xpm{bT4Yu8&rxUj$T^NPn(aOVe*^~IY6*Gl_ zeBnC*GQQ?Hj@uif1g2~3uvbZ!v(WhCW|ItMS|O1Jz(E@t$FDEPqG4Xh2JPuLvE+f` z_9J3Fs25N^7(vHZ2^^#T?&4nF{%O)3v2UueRXIc@9DQ?JH4nyn7SEg6`(EkC;DmB9i_)v5Scs** z{^o?L$J-=jsYGi%C(nyA>)9LTG)pJJ$1IuJg?D28{5#`QFtCLJ|vKrDtEnlq@W)_TF zYu1Ogs?k*Av;$5OT}2Cm^Wmd0r1{c(r_j`3PvbMSSB`H|E{xK@9Ns-{Di%i{H4sS$Ji^a61xKdhW>RlA(Kcpg58kQ zQtUkz8wFeBd9R=8?U)`I?^P^17;2f|@t4&9&U}LslpFdmasc5^Zsb9>OBSS408vHc z%3Jvz{*)9Jqu32k(0`FH8u*7Gbn$ezZElAzIiIr4dUMxuK&zoIuyy)`bG{6<2I6n49 zGJrYEe-VO0yq4@NlttK)e#YuTS|3Z48phwoA^OaH@KdbmJ9e;tiNY<9+ z0!KX*ImEFPuSdS5Z$KWV01LBuYqU4VtP#b{N@t=n@X+B0m8a~^20y76@Whln`u!w( z0|8dugFoidYJrC+z_i?>XyWWWL$l<64oGn`>672upYPk)#9XRN-HO|0Q1iYpHpyc9H_Xrl38niq26g$gs4cpeEo zZ`fvf&0|t>r|5FQ1=_G_otgOUce_bqq1NqcJsWDMK#NCx6GO8|N*`&ITt@-kQUJml zc!&bE!m{T|0U{{C)^b`}g<85^`Mf9TVd3{44=s$E8goOMs2C>eaT0b}Sn;Q$i*9o4v z*$Y28^~&%6d-Q*IjFykD)$uVCv7UZ#XLorUSQfLz=bhcB5eeUV3GumH7U+k-W>Cs)JHDcqaTF0t(5?CQhaD^nAduSZ{dFZqS z!j$}^3`SrSe8|Ol-y03m^_7qIh+o^r)b9;y6Qv9ef~%o3;JFkk;X;kH2m_@Z^ahK= zzHEh%&rb%w9}a;}!}bo0Ais)uF5EEu;plBF#BN8p2l+dLkl(|rIT&(IWonWFbgHJW z)XsXZ{bVrx_xOfV4YkD*_NR6OOZ?)-BXm0&&YZsoweOErHVc;dARfabmecS;-gvE= zIo# z&!a`D{yw_k@rU1XxhX0BtDmQ~Q6KjIf;|FBu4~A7p?{M+f;L}kWc3bL+Fl!FJpJ+F z!PY84`EuT7vkF{k=?aSzqFqJKZ#L@GC%2xXZ}^rR@~pWJ-|Rt}I2O6<#W*jZ|B6#( ze~Yv1|Lbw4qi-xcY Date: Thu, 18 Jan 2018 15:30:03 +0800 Subject: [PATCH 088/158] add 4-d for matmul_op --- paddle/operators/math/matmul.h | 36 +++++++++++++--- paddle/operators/matmul_op.cc | 42 +++++++++++++++---- paddle/operators/matmul_op.h | 18 ++++++-- .../paddle/v2/fluid/tests/test_matmul_op.py | 17 ++++++++ 4 files changed, 97 insertions(+), 16 deletions(-) diff --git a/paddle/operators/math/matmul.h b/paddle/operators/math/matmul.h index 7048e11e6f..98a98e4b99 100644 --- a/paddle/operators/math/matmul.h +++ b/paddle/operators/math/matmul.h @@ -41,10 +41,26 @@ class MatMulFunctor { "Input tensor a must be at least 1-dimensional."); PADDLE_ENFORCE_GE(dim_b.size(), 1, "Input tensor b must be at least 1-dimensional."); - PADDLE_ENFORCE_LE(dim_a.size(), 3, - "Input tensor a must be at most 3-dimensional."); - PADDLE_ENFORCE_LE(dim_b.size(), 3, - "Input tensor b must be at most 3-dimensional."); + PADDLE_ENFORCE_LE(dim_a.size(), 4, + "Input tensor a must be at most 4-dimensional."); + PADDLE_ENFORCE_LE(dim_b.size(), 4, + "Input tensor b must be at most 4-dimensional."); + + std::vector out_dim; + int64_t batch_count = 1; + if (dim_a.size() > 3) { + PADDLE_ENFORCE(dim_b.size() > 3, + "The dimensions of X and Y must be the same, and both of " + "them should be 4-dimensional."); + for (int j = 0; j < dim_a.size() - 2; ++j) { + PADDLE_ENFORCE( + dim_b[j] == dim_a[j], + "The dimensions of X and Y must be the same, and both of " + "them should be 4-dimensional."); + out_dim.push_back(dim_a[j]); + batch_count *= dim_a[j]; + } + } int M = 0, N = 0, kA = 0, kB = 0, batchCountA = 0, batchCountB = 0, strideA = 0, strideB = 0; @@ -67,7 +83,11 @@ class MatMulFunctor { strideA = M * kA; break; default: - assert(false); + batchCountA = batch_count; + size_t mat_s = dim_a.size() - 2; + M = trans_a ? dim_a[mat_s + 1] : dim_a[mat_s]; + kA = trans_a ? dim_a[mat_s] : dim_a[mat_s + 1]; + strideA = M * kA; } switch (dim_b.size()) { @@ -88,7 +108,11 @@ class MatMulFunctor { strideB = kB * N; break; default: - assert(false); + batchCountB = batch_count; + size_t mat_s = dim_b.size() - 2; + kB = trans_b ? dim_b[mat_s + 1] : dim_b[mat_s]; + N = trans_b ? dim_b[mat_s] : dim_b[mat_s + 1]; + strideB = kB * N; } PADDLE_ENFORCE_EQ( diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index fd65d894d5..155346db41 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/matmul_op.h" +#include namespace paddle { namespace operators { @@ -41,10 +42,26 @@ class MatMulOp : public framework::OperatorWithKernel { "Input tensor X must be at least 1-dimensional."); PADDLE_ENFORCE_GE(dim_y.size(), 1, "Input tensor Y must be at least 1-dimensional."); - PADDLE_ENFORCE_LE(dim_x.size(), 3, - "Input tensor X must be at most 3-dimensional."); - PADDLE_ENFORCE_LE(dim_y.size(), 3, - "Input tensor Y must be at most 3-dimensional."); + PADDLE_ENFORCE_LE(dim_x.size(), 4, + "Input tensor X must be at most 4-dimensional."); + PADDLE_ENFORCE_LE(dim_y.size(), 4, + "Input tensor Y must be at most 4-dimensional."); + + std::vector out_dim; + int64_t batch_count = 1; + if (dim_x.size() > 3) { + PADDLE_ENFORCE(dim_y.size() == dim_x.size(), + "The dimensions of X and Y must be the same, and both of " + "them should be 4-dimensional."); + for (int j = 0; j < dim_x.size() - 2; ++j) { + PADDLE_ENFORCE( + dim_y[j] == dim_x[j], + "The dimensions of X and Y must be the same, and both of " + "them should be 4-dimensional."); + out_dim.push_back(dim_x[j]); + batch_count *= dim_x[j]; + } + } int M = 0, N = 0, KX = 0, KY = 0, batchCountX = 0, batchCountY = 0; bool remove_initial_dim = false, remove_final_dim = false; @@ -70,7 +87,11 @@ class MatMulOp : public framework::OperatorWithKernel { KX = transpose_x ? dim_x[1] : dim_x[2]; break; default: - assert(false); + batchCountX = batch_count; + size_t mat_s = dim_x.size() - 2; + M = transpose_x ? dim_x[mat_s + 1] : dim_x[mat_s]; + KX = transpose_x ? dim_x[mat_s] : dim_x[mat_s + 1]; + break; } switch (dim_y.size()) { @@ -94,7 +115,10 @@ class MatMulOp : public framework::OperatorWithKernel { N = transpose_y ? dim_y[1] : dim_y[2]; break; default: - assert(false); + batchCountY = batch_count; + size_t mat_s = dim_y.size() - 2; + KY = transpose_y ? dim_y[mat_s + 1] : dim_y[mat_s]; + N = transpose_y ? dim_y[mat_s] : dim_y[mat_s + 1]; } PADDLE_ENFORCE_EQ( @@ -110,7 +134,11 @@ class MatMulOp : public framework::OperatorWithKernel { std::vector dim_out; if (batchCount) { - dim_out.push_back(batchCount); + if (dim_x.size() > 3) { + dim_out.insert(dim_out.begin(), out_dim.begin(), out_dim.end()); + } else { + dim_out.push_back(batchCount); + } } if (!remove_initial_dim) { dim_out.push_back(M); diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 78adc64f76..11266db4b9 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -149,7 +149,10 @@ class MatMulGradKernel : public framework::OpKernel { M = transpose_x ? x_dims[2] : x_dims[1]; break; default: - assert(false); + batchCountX = accumulate(x_dims.begin(), x_dims.end() - 2, 1, + std::multiplies()); + size_t mat_s = x_dims.size() - 2; + M = transpose_x ? x_dims[mat_s + 1] : x_dims[mat_s]; } switch (y_dims.size()) { @@ -161,7 +164,10 @@ class MatMulGradKernel : public framework::OpKernel { N = transpose_y ? y_dims[1] : y_dims[2]; break; default: - assert(false); + batchCountY = accumulate(y_dims.begin(), y_dims.end() - 2, 1, + std::multiplies()); + size_t mat_s = y_dims.size() - 2; + N = transpose_y ? y_dims[mat_s] : y_dims[mat_s + 1]; } if (batchCountX && batchCountY) { PADDLE_ENFORCE_EQ( @@ -172,7 +178,13 @@ class MatMulGradKernel : public framework::OpKernel { int batchCount = std::max(batchCountX, batchCountY); std::vector dout_dims = {M, N}; if (batchCount) { - dout_dims.insert(dout_dims.begin(), batchCount); + if (x_dims.size() > 3) { + dout_dims.insert(dout_dims.begin(), x_dims.begin(), x_dims.end() - 2); + } else if (y_dims.size() > 3) { + dout_dims.insert(dout_dims.begin(), y_dims.begin(), y_dims.end() - 2); + } else { + dout_dims.insert(dout_dims.begin(), batchCount); + } } Tensor X = Reshape(x, make_ddim(x_dims)); Tensor Y = Reshape(y, make_ddim(y_dims)); diff --git a/python/paddle/v2/fluid/tests/test_matmul_op.py b/python/paddle/v2/fluid/tests/test_matmul_op.py index f7dc4e0532..0548a9bbfe 100644 --- a/python/paddle/v2/fluid/tests/test_matmul_op.py +++ b/python/paddle/v2/fluid/tests/test_matmul_op.py @@ -128,5 +128,22 @@ for dim_X in [1, 2, 3]: }) globals()[test_name] = test_class +# Test case 4-dim +dim_X = 4 +dim_Y = 4 +transpose_X = False +transpose_Y = False +test_name = ('TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim_X, dim_Y, transpose_X, transpose_Y)) + +shape_X = [2, 2, 2, 3] +shape_Y = [2, 2, 3, 4] +test_class = type(test_name, (Generator, OpTest), { + 'shape_X': shape_X, + 'shape_Y': shape_Y, + 'transpose_X': transpose_X, + 'transpose_Y': transpose_Y, +}) + if __name__ == "__main__": unittest.main() From 578d60bf4adf8659a28d9157fa5c5511b17900b1 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 18 Jan 2018 17:52:40 +0800 Subject: [PATCH 089/158] code refine --- paddle/operators/math/matmul.h | 14 +++++--------- paddle/operators/matmul_op.cc | 15 +++++---------- paddle/operators/matmul_op.h | 14 ++++++++------ 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/paddle/operators/math/matmul.h b/paddle/operators/math/matmul.h index 98a98e4b99..8a63d204cb 100644 --- a/paddle/operators/math/matmul.h +++ b/paddle/operators/math/matmul.h @@ -41,22 +41,18 @@ class MatMulFunctor { "Input tensor a must be at least 1-dimensional."); PADDLE_ENFORCE_GE(dim_b.size(), 1, "Input tensor b must be at least 1-dimensional."); - PADDLE_ENFORCE_LE(dim_a.size(), 4, - "Input tensor a must be at most 4-dimensional."); - PADDLE_ENFORCE_LE(dim_b.size(), 4, - "Input tensor b must be at most 4-dimensional."); std::vector out_dim; int64_t batch_count = 1; if (dim_a.size() > 3) { PADDLE_ENFORCE(dim_b.size() > 3, "The dimensions of X and Y must be the same, and both of " - "them should be 4-dimensional."); + "them should be %d-dimensional.", + dim_b.size()); for (int j = 0; j < dim_a.size() - 2; ++j) { - PADDLE_ENFORCE( - dim_b[j] == dim_a[j], - "The dimensions of X and Y must be the same, and both of " - "them should be 4-dimensional."); + PADDLE_ENFORCE(dim_b[j] == dim_a[j], + "The dimensions of X[%d] and Y[%d] must be the same.", j, + j); out_dim.push_back(dim_a[j]); batch_count *= dim_a[j]; } diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index 155346db41..6ced0ef6c0 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/matmul_op.h" -#include namespace paddle { namespace operators { @@ -42,22 +41,18 @@ class MatMulOp : public framework::OperatorWithKernel { "Input tensor X must be at least 1-dimensional."); PADDLE_ENFORCE_GE(dim_y.size(), 1, "Input tensor Y must be at least 1-dimensional."); - PADDLE_ENFORCE_LE(dim_x.size(), 4, - "Input tensor X must be at most 4-dimensional."); - PADDLE_ENFORCE_LE(dim_y.size(), 4, - "Input tensor Y must be at most 4-dimensional."); std::vector out_dim; int64_t batch_count = 1; if (dim_x.size() > 3) { PADDLE_ENFORCE(dim_y.size() == dim_x.size(), "The dimensions of X and Y must be the same, and both of " - "them should be 4-dimensional."); + "them should be %d-dimensional.", + dim_x.size()); for (int j = 0; j < dim_x.size() - 2; ++j) { - PADDLE_ENFORCE( - dim_y[j] == dim_x[j], - "The dimensions of X and Y must be the same, and both of " - "them should be 4-dimensional."); + PADDLE_ENFORCE(dim_y[j] == dim_x[j], + "The dimensions of X[%d] and Y[%d] must be the same.", j, + j); out_dim.push_back(dim_x[j]); batch_count *= dim_x[j]; } diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 11266db4b9..9f06791f7b 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -137,6 +137,12 @@ class MatMulGradKernel : public framework::OpKernel { y_dims.push_back(1); } + int batch_count = 0; + // + if (x_dims.size() > 3) { + batch_count = accumulate(x_dims.begin(), x_dims.end() - 2, 1, + std::multiplies()); + } // Fix the dOut dimensions. int M = 0, N = 0, batchCountX = 0, batchCountY = 0; @@ -149,8 +155,7 @@ class MatMulGradKernel : public framework::OpKernel { M = transpose_x ? x_dims[2] : x_dims[1]; break; default: - batchCountX = accumulate(x_dims.begin(), x_dims.end() - 2, 1, - std::multiplies()); + batchCountX = batch_count; size_t mat_s = x_dims.size() - 2; M = transpose_x ? x_dims[mat_s + 1] : x_dims[mat_s]; } @@ -164,8 +169,7 @@ class MatMulGradKernel : public framework::OpKernel { N = transpose_y ? y_dims[1] : y_dims[2]; break; default: - batchCountY = accumulate(y_dims.begin(), y_dims.end() - 2, 1, - std::multiplies()); + batchCountY = batch_count; size_t mat_s = y_dims.size() - 2; N = transpose_y ? y_dims[mat_s] : y_dims[mat_s + 1]; } @@ -180,8 +184,6 @@ class MatMulGradKernel : public framework::OpKernel { if (batchCount) { if (x_dims.size() > 3) { dout_dims.insert(dout_dims.begin(), x_dims.begin(), x_dims.end() - 2); - } else if (y_dims.size() > 3) { - dout_dims.insert(dout_dims.begin(), y_dims.begin(), y_dims.end() - 2); } else { dout_dims.insert(dout_dims.begin(), batchCount); } From ae19d2ea1ecd28db7f5704da4cb07c59e038e195 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 18 Jan 2018 18:27:32 +0800 Subject: [PATCH 090/158] fix comm issues --- paddle/operators/detail/grpc_server.cc | 47 +++++++++++++++----------- paddle/operators/detail/grpc_server.h | 15 ++++---- paddle/operators/recv_op.cc | 15 +++++--- 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/paddle/operators/detail/grpc_server.cc b/paddle/operators/detail/grpc_server.cc index c0b94746a0..42d3cc5758 100644 --- a/paddle/operators/detail/grpc_server.cc +++ b/paddle/operators/detail/grpc_server.cc @@ -36,7 +36,10 @@ class RequestBase { CallStatus Status() { return status_; } void SetStatus(CallStatus status) { status_ = status; } - virtual std::string GetReqName() { assert(false); } + virtual std::string GetReqName() { + assert(false); + return ""; + } protected: grpc::ServerContext ctx_; @@ -80,11 +83,13 @@ class RequestGet final : public RequestBase { public: explicit RequestGet(sendrecv::SendRecvService::AsyncService* service, grpc::ServerCompletionQueue* cq, framework::Scope* scope, - const platform::DeviceContext* dev_ctx) + const platform::DeviceContext* dev_ctx, + SimpleBlockQueue* queue) : RequestBase(service, cq), responder_(&ctx_), scope_(scope), - dev_ctx_(dev_ctx) { + dev_ctx_(dev_ctx), + queue_(queue) { service_->RequestGetVariable(&ctx_, &request_, &responder_, cq_, cq_, this); } @@ -100,6 +105,7 @@ class RequestGet final : public RequestBase { // TODO(gongwb): check var's info. responder_.Finish(reply_, grpc::Status::OK, this); status_ = FINISH; + queue_->Push('c'); } protected: @@ -108,8 +114,15 @@ class RequestGet final : public RequestBase { ServerAsyncResponseWriter responder_; framework::Scope* scope_; const platform::DeviceContext* dev_ctx_; + SimpleBlockQueue* queue_; }; +void AsyncGRPCServer::WaitClientGet(int count) { + for (int i = 0; i < count; ++i) { + var_get_queue_.Pop(); + } +} + void AsyncGRPCServer::RunSyncUpdate() { grpc::ServerBuilder builder; builder.AddListeningPort(address_, grpc::InsecureServerCredentials()); @@ -170,7 +183,8 @@ void AsyncGRPCServer::TryToRegisterNewGetOne() { if (is_shut_down_) { return; } - RequestGet* get = new RequestGet(&service_, cq_get_.get(), scope_, dev_ctx_); + RequestGet* get = new RequestGet(&service_, cq_get_.get(), scope_, dev_ctx_, + &var_get_queue_); VLOG(4) << "create Requestget status:" << get->Status(); } @@ -188,9 +202,8 @@ void AsyncGRPCServer::HandleRequest(bool wait, grpc::ServerCompletionQueue* cq, } PADDLE_ENFORCE(tag); - if (wait && !done_) { - Wait(); - } + if (cq_name == "cq_get") WaitCond(2); + if (cq_name == "cq_send") WaitCond(0); RequestBase* base = (RequestBase*)tag; // reference: @@ -222,22 +235,18 @@ void AsyncGRPCServer::HandleRequest(bool wait, grpc::ServerCompletionQueue* cq, } } -void AsyncGRPCServer::Wait() { - std::unique_lock lock(this->mutex_); - condition_.wait(lock, [=] { return this->done_ == true; }); -} - -void AsyncGRPCServer::Reset() { - std::lock_guard lock(this->mutex_); - done_ = false; +void AsyncGRPCServer::WaitCond(int cond) { + std::unique_lock lock(this->barrier_mutex_); + barrier_condition_.wait(lock, + [=] { return this->barrier_cond_step_ == cond; }); } -void AsyncGRPCServer::Done() { +void AsyncGRPCServer::SetCond(int cond) { { - std::lock_guard lock(this->mutex_); - done_ = true; + std::lock_guard lock(this->barrier_mutex_); + barrier_cond_step_ = cond; } - condition_.notify_all(); + barrier_condition_.notify_all(); } } // namespace detail diff --git a/paddle/operators/detail/grpc_server.h b/paddle/operators/detail/grpc_server.h index 2c078b7777..5c7be5f5bd 100644 --- a/paddle/operators/detail/grpc_server.h +++ b/paddle/operators/detail/grpc_server.h @@ -41,9 +41,12 @@ class AsyncGRPCServer final : public sendrecv::SendRecvService::Service { void RunSyncUpdate(); - void Reset(); - + // functions to sync server barrier status. + void WaitStart(); + void WaitDone(); + void Start(); void Done(); + void WaitClientGet(int count); void SetScope(framework::Scope *scope) { scope_ = scope; } @@ -56,7 +59,6 @@ class AsyncGRPCServer final : public sendrecv::SendRecvService::Service { void ShutDown(); protected: - void Wait(); void HandleRequest(bool wait, grpc::ServerCompletionQueue *cq, std::string cq_name, std::function TryToRegisterNewOne); @@ -78,11 +80,12 @@ class AsyncGRPCServer final : public sendrecv::SendRecvService::Service { const platform::DeviceContext *dev_ctx_; // received variable from RPC, operators fetch variable from this queue. SimpleBlockQueue var_recv_queue_; + SimpleBlockQueue var_get_queue_; // condition of the sub program - std::mutex mutex_; - volatile mutable bool done_; - std::condition_variable condition_; + std::mutex barrier_mutex_; + mutable int barrier_cond_step_; + std::condition_variable barrier_condition_; std::unique_ptr t_send_; std::unique_ptr t_get_; diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index b77d150dcc..2ecd56671f 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -34,6 +34,10 @@ limitations under the License. */ namespace paddle { namespace operators { +constexpr int kCondStart = 0; +constexpr int kCondRunning = 1; +constexpr int kCondDone = 2; + void RunServer(std::shared_ptr service) { service->RunSyncUpdate(); VLOG(4) << "RunServer thread end"; @@ -101,12 +105,14 @@ class RecvOp : public framework::OperatorBase { framework::ProgramDesc program(program_desc); framework::Executor executor(dev_place); - rpc_service_->Reset(); + // rpc_service_->Reset(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. bool exit_flag = false; while (!exit_flag) { // Get from multiple trainers, we don't care about the order in which // the gradients arrives, just add suffix 0~n and merge the gradient. + rpc_service_->SetCond(kCondStart); + VLOG(3) << "================ start get from service ==========="; for (size_t i = 0; i < param_count * fan_in; ++i) { const detail::MessageWithName &v = rpc_service_->Get(); auto grad_var_name = v.first; @@ -139,15 +145,16 @@ class RecvOp : public framework::OperatorBase { if (exit_flag) { break; } - rpc_service_->Reset(); + // rpc_service_->Reset(); try { executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/, false /*create_vars*/); } catch (std::exception &e) { LOG(ERROR) << "run sub program error " << e.what(); } - - rpc_service_->Done(); + VLOG(3) << "================ run sub program end ==========="; + rpc_service_->SetCond(kCondDone); + rpc_service_->WaitClientGet(param_count * fan_in); grads_counter_.clear(); } // while(true) } From 6e04e580d2c1b9e09cd48237fabda8e39a4fdb2f Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Thu, 18 Jan 2018 19:23:28 +0800 Subject: [PATCH 091/158] Change input data type to int64_t in unitest and GPU kernel --- paddle/operators/edit_distance_op.cu | 4 ++-- python/paddle/v2/fluid/tests/test_edit_distance_op.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/operators/edit_distance_op.cu b/paddle/operators/edit_distance_op.cu index c165bbae7f..338fd79bcc 100644 --- a/paddle/operators/edit_distance_op.cu +++ b/paddle/operators/edit_distance_op.cu @@ -39,8 +39,8 @@ __global__ void FillFirstColumn(T* dist, const int M, const int N) { } template -__global__ void Levenshtein(T* dist, const int* x1, const int* x2, const int M, - const int N, const int start) { +__global__ void Levenshtein(T* dist, const int64_t* x1, const int64_t* x2, + const int M, const int N, const int start) { int idx = blockDim.x * blockIdx.x + threadIdx.x; int offset = N; int index = start + idx * offset; diff --git a/python/paddle/v2/fluid/tests/test_edit_distance_op.py b/python/paddle/v2/fluid/tests/test_edit_distance_op.py index cf118df634..5f5634e297 100644 --- a/python/paddle/v2/fluid/tests/test_edit_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_edit_distance_op.py @@ -51,8 +51,8 @@ class TestEditDistanceOp(OpTest): def setUp(self): self.op_type = "edit_distance" normalized = False - x1 = np.array([[0, 12, 3, 5, 8, 2]]).astype("int32") - x2 = np.array([[0, 12, 4, 7, 8]]).astype("int32") + x1 = np.array([[0, 12, 3, 5, 8, 2]]).astype("int64") + x2 = np.array([[0, 12, 4, 7, 8]]).astype("int64") x1 = np.transpose(x1) x2 = np.transpose(x2) x1_lod = [0, 1, 5] @@ -79,8 +79,8 @@ class TestEditDistanceOpNormalized(OpTest): def setUp(self): self.op_type = "edit_distance" normalized = True - x1 = np.array([[0, 10, 3, 6, 5, 8, 2]]).astype("int32") - x2 = np.array([[0, 10, 4, 6, 7, 8]]).astype("int32") + x1 = np.array([[0, 10, 3, 6, 5, 8, 2]]).astype("int64") + x2 = np.array([[0, 10, 4, 6, 7, 8]]).astype("int64") x1 = np.transpose(x1) x2 = np.transpose(x2) x1_lod = [0, 1, 3, 6] From 5f4d9130f01833dfef44dac2eadb7089accbe0ba Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 18 Jan 2018 19:27:20 +0800 Subject: [PATCH 092/158] merge codes --- paddle/operators/detail/grpc_server.cc | 5 +++-- paddle/operators/detail/grpc_server.h | 6 ++---- paddle/operators/recv_op.cc | 15 +++++---------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/paddle/operators/detail/grpc_server.cc b/paddle/operators/detail/grpc_server.cc index 42d3cc5758..3ddcd839bd 100644 --- a/paddle/operators/detail/grpc_server.cc +++ b/paddle/operators/detail/grpc_server.cc @@ -162,7 +162,6 @@ void AsyncGRPCServer::ShutdownQueue() { } // This URL explains why shutdown is complicate: -// https://stackoverflow.com/questions/35708348/grpc-what-is-the-recommended-way-to-shut-down-an-asynchronous-server-in-c void AsyncGRPCServer::ShutDown() { server_->Shutdown(); ShutdownQueue(); @@ -188,6 +187,7 @@ void AsyncGRPCServer::TryToRegisterNewGetOne() { VLOG(4) << "create Requestget status:" << get->Status(); } +// FIXME(typhoonzero): remove wait argument and change cq_name to enum. void AsyncGRPCServer::HandleRequest(bool wait, grpc::ServerCompletionQueue* cq, std::string cq_name, std::function TryToRegisterNewOne) { @@ -202,7 +202,8 @@ void AsyncGRPCServer::HandleRequest(bool wait, grpc::ServerCompletionQueue* cq, } PADDLE_ENFORCE(tag); - if (cq_name == "cq_get") WaitCond(2); + // FIXME(typhoonzero): de-couple the barriers with recv_op + if (cq_name == "cq_get") WaitCond(1); if (cq_name == "cq_send") WaitCond(0); RequestBase* base = (RequestBase*)tag; diff --git a/paddle/operators/detail/grpc_server.h b/paddle/operators/detail/grpc_server.h index 5c7be5f5bd..1ca9086c74 100644 --- a/paddle/operators/detail/grpc_server.h +++ b/paddle/operators/detail/grpc_server.h @@ -42,10 +42,8 @@ class AsyncGRPCServer final : public sendrecv::SendRecvService::Service { void RunSyncUpdate(); // functions to sync server barrier status. - void WaitStart(); - void WaitDone(); - void Start(); - void Done(); + void WaitCond(int cond); + void SetCond(int cond); void WaitClientGet(int count); void SetScope(framework::Scope *scope) { scope_ = scope; } diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 2ecd56671f..8d1479bdd6 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -105,15 +105,14 @@ class RecvOp : public framework::OperatorBase { framework::ProgramDesc program(program_desc); framework::Executor executor(dev_place); - // rpc_service_->Reset(); // TODO(typhoonzero): change this to a while_op for every cluster-batch. bool exit_flag = false; + int64_t barrier_size = param_count * fan_in; while (!exit_flag) { // Get from multiple trainers, we don't care about the order in which // the gradients arrives, just add suffix 0~n and merge the gradient. - rpc_service_->SetCond(kCondStart); - VLOG(3) << "================ start get from service ==========="; - for (size_t i = 0; i < param_count * fan_in; ++i) { + rpc_service_->SetCond(0); + for (size_t i = 0; i < barrier_size; ++i) { const detail::MessageWithName &v = rpc_service_->Get(); auto grad_var_name = v.first; if (grad_var_name == LISTEN_TERMINATE_MESSAGE) { @@ -130,8 +129,6 @@ class RecvOp : public framework::OperatorBase { } VLOG(3) << "recved grad: " << grad_var_name << " updating param: " << param_var_name; - // Assume grad_var_name must appear in global scope. - std::string grad_var_name_trainer; if (fan_in > 1) { grad_var_name = this->GetGradVarNameForTrainer(grad_var_name); } @@ -145,16 +142,14 @@ class RecvOp : public framework::OperatorBase { if (exit_flag) { break; } - // rpc_service_->Reset(); try { executor.Run(program, &recv_scope, 0, /*global_block*/ false /*create_local_scope*/, false /*create_vars*/); } catch (std::exception &e) { LOG(ERROR) << "run sub program error " << e.what(); } - VLOG(3) << "================ run sub program end ==========="; - rpc_service_->SetCond(kCondDone); - rpc_service_->WaitClientGet(param_count * fan_in); + rpc_service_->SetCond(1); + rpc_service_->WaitClientGet(barrier_size); grads_counter_.clear(); } // while(true) } From 30529e314e7e9bdce78aa0adf9667da3fe9977cb Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 18 Jan 2018 20:02:26 +0800 Subject: [PATCH 093/158] delete debug transpiler code --- .../paddle/v2/fluid/distribute_transpiler.py | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 3cba015fc5..13d2bb8325 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -72,51 +72,6 @@ def split_dense_variable(var_list, return blocks -def split_selected_rows(var, - pserver_count, - min_block_size=1024, - max_block_size=1048576): - assert ((len(var.shape)) <= 1) - - split_count = pserver_count - indices = var.desc.selected_rows().dims() - var_width = reduce(lambda x, y: x * y, var.shape[1:]) - row_count = len(indices) - rows_per_block = 1 - if var_width < min_block_size: - rows_per_block = 1 - split_count = row_count - else: - rows_per_block = row_count / pserver_count - if not rows_per_block % pserver_count: - rows_per_block += 1 - split_count = row_count / rows_per_block - if not row_count % rows_per_block: - split_count += 1 - blocks = [] - for block_id in xrange(split_count): - curr_block_rows = min(rows_per_block, - row_count - (block_id * rows_per_block)) - block = VarBlock(var.name, block_id, curr_block_rows) - blocks.append(block) - return blocks - - -def split_variable(var_list, - pserver_count, - min_block_size=1024, - max_block_size=1048576): - for var in var_list: - if var.type == core.VarDesc.VarType.LOD_TENSOR: - split_dense_variable(var_list, pserver_count, min_block_size, - max_block_size) - elif var.type == core.VarDesc.VarType.SELECTED_ROWS: - split_selected_rows(var_list, pserver_count, min_block_size, - max_block_size) - else: - raise TypeError("variable must be lodtensor or selected rows") - - class DistributeTranspiler: def transpile(self, optimize_ops, From bea41444d78faba95a0e7f7fda79edf62afbe7cf Mon Sep 17 00:00:00 2001 From: yangyaming Date: Thu, 18 Jan 2018 20:16:03 +0800 Subject: [PATCH 094/158] Refine the implementation and add unit test. --- paddle/operators/sequence_reshape_op.cc | 64 +++++++++++++--- paddle/operators/sequence_reshape_op.cu | 23 ++++++ paddle/operators/sequence_reshape_op.h | 67 ++++++++++------ .../v2/fluid/tests/test_sequence_reshape.py | 76 +++++++++++++++++++ 4 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 paddle/operators/sequence_reshape_op.cu create mode 100644 python/paddle/v2/fluid/tests/test_sequence_reshape.py diff --git a/paddle/operators/sequence_reshape_op.cc b/paddle/operators/sequence_reshape_op.cc index 31a970354f..308de59c64 100644 --- a/paddle/operators/sequence_reshape_op.cc +++ b/paddle/operators/sequence_reshape_op.cc @@ -27,9 +27,8 @@ class SequenceReshapeOp : public framework::OperatorWithKernel { "Output(Out) of SequenceReshapeOp should not be null."); auto x_dims = ctx->GetInputDim("X"); PADDLE_ENFORCE_EQ(x_dims.size(), 2U, "Rank of Input(X) should be 2."); - int dimension = ctx->Attrs().Get("dimension"); - ctx->SetOutputDim("Out", {{x_dims[0], static_cast(dimension)}}); - ctx->ShareLoD("X", /*->*/ "Out"); + int dimension = ctx->Attrs().Get("new_dim"); + ctx->SetOutputDim("Out", {x_dims[0], static_cast(dimension)}); } }; @@ -37,11 +36,41 @@ class SequenceReshapeOpMaker : public framework::OpProtoAndCheckerMaker { public: SequenceReshapeOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", ""); - AddOutput("Out", ""); - AddAttr("dimension", ""); - AddAttr("is_padding", "Default padding zero."); - AddComment(R"DOC()DOC"); + AddInput("X", + "(LoDTensor, default LoDTensor) A 2-D LoDTensor with shape " + "being [N, M]."); + AddOutput("Out", + "(LoDTensor, default LoDTensor) A 2-D LoDTensor with " + "shape [T, new_dim] where T is calculated based on X.lod, M and " + "new_dim."); + AddAttr("new_dim", "Sequence dimension of the output LoDTensor."); + AddComment(R"DOC( +Sequence Reshape Operator. + +This operator will rearrange the input sequences. The new dimension is set by +attribute and length of each sequence may change longer or shorter which is +decided by original length, original dimension and new dimension. The following +example will help to illustrate the function of this operator: + +x is a LoDTensor: + x.lod = [[0, 2, 6]] + x.data = [[0.1, 0.2], [0.3, 0.4], + [0.5, 0.6], [0.7, 0.8], [0.9, 1.0], [1.1, 1.2]] + x.dims = [6, 2] + +set new_dim = 4 + +then out is a LoDTensor: + out.lod = [[0, 1, 3]] + out.data = [[0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]] + out.dims = [3, 4] + +Currently, only 1-level LoDTensor is supported and please make sure (original +length * original dimension) can be divided by new_dim with no remainder for +each sequence. + +)DOC"); } }; @@ -63,12 +92,29 @@ class SequenceReshapeGradOp : public framework::OperatorWithKernel { } }; +class SequenceReshapeGradOpMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto* op_desc_ptr = new framework::OpDesc(); + op_desc_ptr->SetType("sequence_reshape_grad"); + op_desc_ptr->SetInput("X", Input("X")); + op_desc_ptr->SetInput("Out", Output("Out")); + op_desc_ptr->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op_desc_ptr->SetOutput(framework::GradVarName("X"), InputGrad("X")); + op_desc_ptr->SetAttrMap(Attrs()); + return std::unique_ptr(op_desc_ptr); + } +}; + } // namespace operators } // namespace paddle namespace ops = paddle::operators; REGISTER_OPERATOR(sequence_reshape, ops::SequenceReshapeOp, - ops::SequenceReshapeOpMaker); + ops::SequenceReshapeOpMaker, ops::SequenceReshapeGradOpMaker); REGISTER_OPERATOR(sequence_reshape_grad, ops::SequenceReshapeGradOp); REGISTER_OP_CPU_KERNEL( sequence_reshape, diff --git a/paddle/operators/sequence_reshape_op.cu b/paddle/operators/sequence_reshape_op.cu new file mode 100644 index 0000000000..dc620ef522 --- /dev/null +++ b/paddle/operators/sequence_reshape_op.cu @@ -0,0 +1,23 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/sequence_reshape_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL( + sequence_reshape, + ops::SequenceReshapeKernel); +REGISTER_OP_CUDA_KERNEL( + sequence_reshape_grad, + ops::SequenceReshapeGradKernel); diff --git a/paddle/operators/sequence_reshape_op.h b/paddle/operators/sequence_reshape_op.h index bc7694b6b1..8e302a364b 100644 --- a/paddle/operators/sequence_reshape_op.h +++ b/paddle/operators/sequence_reshape_op.h @@ -26,53 +26,63 @@ class SequenceReshapeKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& context) const override { auto* in = context.Input("X"); auto* out = context.Output("Out"); - int out_width = context.Attr("dimension"); - bool whether_padding = context.Attr("whether_padding"); + int out_width = context.Attr("new_dim"); const T* p_in_data = in->data(); - T* p_out_data = out->mutable_data(context.GetPlace()); - // compute shape for output auto in_dims = in->dims(); int64_t in_width = in_dims[1]; auto& in_lod = in->lod(); PADDLE_ENFORCE_EQ(in_lod.size(), 1UL, "Only support one level sequence now."); - PADDLE_ENFORCE_GE( - in_dims[0], - /* batch size = */ static_cast(in_lod[0].size() - 1), - "The 1st dimension of Input(X) must be equal or larger than batch " - "size."); + PADDLE_ENFORCE_EQ( + in_dims[0], in_lod[0].back(), + "Inconsistent size between X.shape[0] and X.lod()[0].back()."); auto in_lod_l0 = in_lod[0]; int seq_num = in_lod_l0.size() - 1; auto& out_lod = *out->mutable_lod(); - out_lod.push_back(std::vector({0})); - size_t offset = 0; + out_lod.resize(1); + out_lod[0].clear(); + out_lod[0].push_back(0); for (int i = 0; i < seq_num; ++i) { size_t seq_len = in_lod_l0[i + 1] - in_lod_l0[i]; - if (whether_padding) { - offset += std::ceil((float)(seq_len * in_width) / out_width); - } else { - offset += (seq_len * in_width) / out_width; - } - out_lod[0].push_back(offset); + size_t offset = 0; + offset = (seq_len * in_width) / out_width; + PADDLE_ENFORCE_EQ(offset * out_width, seq_len * in_width, + "Please make sure (sequence_length * dimension) can be " + "divided by new_dim with no remainder for each " + "sequence. The %dth sequence is invalid.", + i + 1); + PADDLE_ENFORCE_GT(offset, 0, + "Illegal operation, length of the %dth sequence become " + "to 0 after reshaped.", + i + 1); + out_lod[0].push_back(out_lod[0].back() + offset); } - out->Resize({{static_cast(out_lod[0].back()), out_width}}); + out->mutable_data(context.GetPlace()); + out->Resize({static_cast(out_lod[0].back()), out_width}); + T* p_out_data = out->mutable_data(context.GetPlace()); math::set_constant(context.device_context(), out, 0.0f); for (int i = 0; i < seq_num; ++i) { size_t in_offset = in_lod_l0[i] * in_width; size_t out_offset = out_lod[0][i] * out_width; - size_t bytes = sizeof(T) * (in_lod_l0[i + 1] - in_lod_l0[i]) * in_width; + size_t in_count = (in_lod_l0[i + 1] - in_lod_l0[i]) * in_width; + size_t out_count = (out_lod[0][i + 1] - out_lod[0][i]) * out_width; + size_t bytes = sizeof(T) * std::min(in_count, out_count); if (platform::is_cpu_place(context.GetPlace())) { - std::memcpy(p_out_data + out_offset, p_in_data + in_offset, bytes); + memory::Copy(boost::get(context.GetPlace()), + p_out_data + out_offset, + boost::get(context.GetPlace()), + p_in_data + in_offset, bytes); } else { #ifdef PADDLE_WITH_CUDA - auto& dev_ctx = context.template device_context(); + auto& dev_ctx = + context.template device_context(); memory::Copy(boost::get(context.GetPlace()), p_out_data + out_offset, boost::get(context.GetPlace()), @@ -103,16 +113,23 @@ class SequenceReshapeGradKernel : public framework::OpKernel { auto& out_lod = out_tensor_ptr->lod(); int out_width = out_tensor_ptr->dims()[1]; + math::set_constant(context.device_context(), x_grad_tensor_ptr, 0.0f); + for (int i = 0; i < seq_num; ++i) { size_t src_offset = out_lod[0][i] * out_width; size_t dst_offset = x_lod[0][i] * x_width; - size_t bytes = sizeof(T) * (x_lod[0][i + 1] - x_lod[0][i]) * x_width; + size_t src_count = (out_lod[0][i + 1] - out_lod[0][i]) * out_width; + size_t dst_count = (x_lod[0][i + 1] - x_lod[0][i]) * x_width; + size_t bytes = sizeof(T) * std::min(src_count, dst_count); if (platform::is_cpu_place(context.GetPlace())) { - std::memcpy(p_x_grad_data + dst_offset, p_out_grad_data + src_offset, - bytes); + memory::Copy(boost::get(context.GetPlace()), + p_x_grad_data + dst_offset, + boost::get(context.GetPlace()), + p_out_grad_data + src_offset, bytes); } else { #ifdef PADDLE_WITH_CUDA - auto& dev_ctx = context.template device_context(); + auto& dev_ctx = + context.template device_context(); memory::Copy(boost::get(context.GetPlace()), p_x_grad_data + dst_offset, boost::get(context.GetPlace()), diff --git a/python/paddle/v2/fluid/tests/test_sequence_reshape.py b/python/paddle/v2/fluid/tests/test_sequence_reshape.py new file mode 100644 index 0000000000..91ff275821 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_sequence_reshape.py @@ -0,0 +1,76 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. +import unittest +import numpy as np +import math +from op_test import OpTest + + +class TestSequenceReshape(OpTest): + def setUp(self): + self.op_type = 'sequence_reshape' + dimension = 12 + x_lod = [[0, 4, 5, 8, 11]] + x = np.random.uniform(0.1, 1, [11, 24]).astype('float32') + + self.inputs = {'X': (x, x_lod)} + self.attrs = {'new_dim': dimension} + + out, out_lod = self.compute_output(x, x_lod, dimension) + + self.outputs = {'Out': (out, out_lod)} + + def compute_output(self, x, x_lod, dimension): + x_width = x.shape[1] + out_lod = [[0]] + for i in xrange(len(x_lod[0]) - 1): + seq_len = x_lod[0][i + 1] - x_lod[0][i] + offset = (seq_len * x_width) / dimension + assert int(offset) * dimension == seq_len * x_width + out_lod[0].append(out_lod[0][-1] + int(offset)) + out = np.zeros(shape=(out_lod[0][-1], dimension)).astype('float32') + for i in xrange(len(x_lod[0]) - 1): + x_offset = x_lod[0][i] * x_width + out_offset = out_lod[0][i] * dimension + out_count = (out_lod[0][i + 1] - out_lod[0][i]) * dimension + x_count = (x_lod[0][i + 1] - x_lod[0][i]) * x_width + count = min(out_count, x_count) + out.ravel()[out_offset:out_offset + count] = x.ravel()[ + x_offset:x_offset + count] + return out, out_lod + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out") + + +class TestSequenceReshape_reduce(TestSequenceReshape): + def setUp(self): + self.op_type = 'sequence_reshape' + dimension = 24 + x_lod = [[0, 4, 6, 8, 12]] + x = np.random.uniform(0.1, 1, [12, 12]).astype('float32') + + self.inputs = {'X': (x, x_lod)} + self.attrs = {'new_dim': dimension} + + out, out_lod = self.compute_output(x, x_lod, dimension) + + self.outputs = {'Out': (out, out_lod)} + + +if __name__ == '__main__': + unittest.main() From fc581bc5f20e3dd3a2a518a1eb5120abf89a3a52 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Thu, 18 Jan 2018 20:27:43 +0800 Subject: [PATCH 095/158] Change the CopyRight. --- paddle/operators/sequence_reshape_op.cc | 2 +- paddle/operators/sequence_reshape_op.cu | 2 +- paddle/operators/sequence_reshape_op.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/operators/sequence_reshape_op.cc b/paddle/operators/sequence_reshape_op.cc index 308de59c64..ddedbc3bc6 100644 --- a/paddle/operators/sequence_reshape_op.cc +++ b/paddle/operators/sequence_reshape_op.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/operators/sequence_reshape_op.cu b/paddle/operators/sequence_reshape_op.cu index dc620ef522..9ba0e34e27 100644 --- a/paddle/operators/sequence_reshape_op.cu +++ b/paddle/operators/sequence_reshape_op.cu @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/operators/sequence_reshape_op.h b/paddle/operators/sequence_reshape_op.h index 8e302a364b..7c2215f772 100644 --- a/paddle/operators/sequence_reshape_op.h +++ b/paddle/operators/sequence_reshape_op.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 259858b41bce74e079503b33cf93ed4e48da9fdb Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 18 Jan 2018 20:42:07 +0800 Subject: [PATCH 096/158] modify doc --- paddle/operators/math/matmul.h | 1 + paddle/operators/matmul_op.cc | 8 +++++++- paddle/operators/matmul_op.h | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/paddle/operators/math/matmul.h b/paddle/operators/math/matmul.h index 8a63d204cb..ca41201e12 100644 --- a/paddle/operators/math/matmul.h +++ b/paddle/operators/math/matmul.h @@ -49,6 +49,7 @@ class MatMulFunctor { "The dimensions of X and Y must be the same, and both of " "them should be %d-dimensional.", dim_b.size()); + // The previous Rank-2 dimensions are accumulated on the batch_count. for (int j = 0; j < dim_a.size() - 2; ++j) { PADDLE_ENFORCE(dim_b[j] == dim_a[j], "The dimensions of X[%d] and Y[%d] must be the same.", j, diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index 6ced0ef6c0..1707ed7e7d 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -49,6 +49,8 @@ class MatMulOp : public framework::OperatorWithKernel { "The dimensions of X and Y must be the same, and both of " "them should be %d-dimensional.", dim_x.size()); + + // The previous Rank-2 dimensions are accumulated on the batch_count. for (int j = 0; j < dim_x.size() - 2; ++j) { PADDLE_ENFORCE(dim_y[j] == dim_x[j], "The dimensions of X[%d] and Y[%d] must be the same.", j, @@ -185,10 +187,14 @@ Examples without transpose: - X: [B, M, K], Y: [K] => Out: [B, M] - X: [M, K], Y: [B, K, N] => Out: [B, M, N] - X: [B, M, K], Y: [B, K, N] => Out: [B, M, N] +- X: [B, ..., M, K], Y: [B, ..., K, N] => Out: [B, ..., M, N] The behavior is designed to be similar to the `numpy.matmul` function. The differences are: -- Currently only rank 1 to rank 3 input tensors are supported. +- When the rank of the input is greater than 3, the rank of X and + Y must be equal, and the former rank-2 dimensions are equal. +- When the rank of the input data is less than or equal to 3, it + is similar to the `numpy.matmul` function. - We add `transpose_X` and `transpose_Y` flags. Both the input `X` and `Y` can carry the LoD (Level of Details) information, diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 9f06791f7b..cf60234295 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -138,7 +138,7 @@ class MatMulGradKernel : public framework::OpKernel { } int batch_count = 0; - // + // The previous Rank-2 dimensions are accumulated on the batch_count. if (x_dims.size() > 3) { batch_count = accumulate(x_dims.begin(), x_dims.end() - 2, 1, std::multiplies()); From 8e02870ce2780c662802caf8ce0c5fd34191d25b Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 18 Jan 2018 20:47:07 +0800 Subject: [PATCH 097/158] modify doc --- paddle/operators/math/sequence_padding_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/operators/math/sequence_padding_test.cc b/paddle/operators/math/sequence_padding_test.cc index 9799bcd65d..3e504f4a15 100644 --- a/paddle/operators/math/sequence_padding_test.cc +++ b/paddle/operators/math/sequence_padding_test.cc @@ -31,7 +31,7 @@ void TestSequencePadding(const paddle::framework::LoD& lod, cpu_seq.set_lod(lod); cpu_seq.mutable_data(seq_dims, paddle::platform::CPUPlace()); - for (size_t i = 0; i < cpu_seq.numel(); ++i) { + for (int64_t i = 0; i < cpu_seq.numel(); ++i) { cpu_seq.data()[i] = static_cast(i); } @@ -69,7 +69,7 @@ void TestSequencePadding(const paddle::framework::LoD& lod, EXPECT_EQ(cpu_seq.numel(), cpu_seq_back.numel()); EXPECT_EQ(cpu_seq.dims(), cpu_seq_back.dims()); - for (size_t i = 0; i < cpu_seq.numel(); ++i) { + for (int64_t i = 0; i < cpu_seq.numel(); ++i) { EXPECT_EQ(cpu_seq.data()[i], cpu_seq_back.data()[i]); } From ef56e6839a9f084f27a2857f18eaed24f317b943 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 18 Jan 2018 05:02:15 -0800 Subject: [PATCH 098/158] Correct the usage of fc in the example of dynamic_lstm's doc --- python/paddle/v2/fluid/layers/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index ebcc914f60..7cf3735395 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -321,7 +321,7 @@ def dynamic_lstm(input, hidden_dim = 512 forward_proj = fluid.layers.fc(input=input_seq, size=hidden_dim * 4, - act='tanh', bias_attr=True) + act=None, bias_attr=None) forward, _ = fluid.layers.dynamic_lstm( input=forward_proj, size=hidden_dim * 4, use_peepholes=False) """ From 773f2f735c235afcc6ea40ddc2af23fe7a69a2e9 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 18 Jan 2018 21:06:51 +0800 Subject: [PATCH 099/158] fix errors --- python/paddle/v2/fluid/clip.py | 5 +++-- python/paddle/v2/fluid/layers/ops.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index 7a36df0dab..d4f025a4af 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -138,8 +138,7 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): cls = self.__class__ cls.check_init() - local_norm_var = layers.reduce_sum( - x=layers.pow(x=grad, factor=2), reduce_all=True) + local_norm_var = layers.reduce_sum(input=layers.pow(x=grad, factor=2.0)) layers.sums( input=[local_norm_var, cls.global_norm_var], out=[cls.global_norm_var]) @@ -154,6 +153,8 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): x=cls.clip_norm_var, y=layers.elementwise_max( x=cls.clip_norm_var, y=cls.global_norm_var)) + assert cls.scale_var.shape == (1L, ) + new_grad = layers.elementwise_mul(x=grad, y=cls.scale_var) return param, new_grad diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index dd3197fc00..a2055c5d7b 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -48,7 +48,7 @@ __all__ = [ 'mean', 'mul', 'reshape', 'scale', 'transpose', 'sigmoid_cross_entropy_with_logits', 'elementwise_add', 'elementwise_div', 'elementwise_sub', 'elementwise_mul', 'elementwise_max', 'elementwise_min', - 'clip', 'clip_by_norm', 'sequence_softmax', 'reduce_sum' + 'clip', 'clip_by_norm', 'sequence_softmax' ] + __activations__ for _OP in set(__all__): From 84de7e7f03565aa9d9394312c58b366c55154083 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 Jan 2018 00:12:05 +0800 Subject: [PATCH 100/158] make auto-registry layers supporting specified output --- python/paddle/v2/fluid/registry.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/python/paddle/v2/fluid/registry.py b/python/paddle/v2/fluid/registry.py index 6c0c3a3518..ff10542d40 100644 --- a/python/paddle/v2/fluid/registry.py +++ b/python/paddle/v2/fluid/registry.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import re import cStringIO import warnings @@ -167,13 +167,18 @@ def register_layer(op_type): inputs[ipt.name] = val outputs = dict() - out = helper.create_tmp_variable(dtype=dtype) - outputs[o_name] = [out] + out = kwargs.pop(_convert_(o_name), []) + if out: + out_var = out[0] if (isinstance(out, list) or + isinstance(out, tuple)) else out + else: + out_var = helper.create_tmp_variable(dtype=dtype) + outputs[o_name] = [out_var] for name in intermediate_output_names: outputs[name] = [helper.create_tmp_variable(dtype=dtype)] helper.append_op( type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs) - return helper.append_activation(out) + return helper.append_activation(out_var) func.__name__ = op_type func.__doc__ = _generate_doc_string_(op_proto) From 7905e36741231ebaf717bd1da2bb47f7208531e1 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 18 Jan 2018 10:09:32 -0800 Subject: [PATCH 101/158] Implement Book chapter 02 in distributed framework (#7629) --- .../notest_recognize_digits_mlp_dist.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py new file mode 100644 index 0000000000..6de3e4da74 --- /dev/null +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py @@ -0,0 +1,87 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. +from __future__ import print_function +import numpy as np +import paddle.v2 as paddle +import paddle.v2.fluid as fluid +import os + +BATCH_SIZE = 128 +PASS_NUM = 100 + +images = fluid.layers.data(name='x', shape=[784], dtype='float32') + +# TODO(aroraabhinav) Add regularization and error clipping after +# Issue 7432(https://github.com/PaddlePaddle/Paddle/issues/7432) is resolved. +hidden1 = fluid.layers.fc(input=images, size=128, act='relu') +hidden2 = fluid.layers.fc(input=hidden1, size=64, act='relu') +predict = fluid.layers.fc(input=hidden2, size=10, act='softmax') + +label = fluid.layers.data(name='y', shape=[1], dtype='int64') + +cost = fluid.layers.cross_entropy(input=predict, label=label) +avg_cost = fluid.layers.mean(x=cost) + +optimizer = fluid.optimizer.Momentum(learning_rate=0.001, momentum=0.9) +optimize_ops, params_grads = optimizer.minimize(avg_cost) + +accuracy = fluid.evaluator.Accuracy(input=predict, label=label) + +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=8192), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +exe = fluid.Executor(place) + +t = fluid.DistributeTranspiler() +# all parameter server endpoints list for spliting parameters +pserver_endpoints = os.getenv("PSERVERS") +# server endpoint for current node +current_endpoint = os.getenv("SERVER_ENDPOINT") +# run as trainer or parameter server +training_role = os.getenv("TRAINING_ROLE", + "TRAINER") # get the training role: trainer/pserver +t.transpile(optimize_ops, params_grads, pservers=pserver_endpoints, trainers=2) + +if training_role == "PSERVER": + if not current_endpoint: + print("need env SERVER_ENDPOINT") + exit(1) + pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + exe.run(fluid.default_startup_program()) + exe.run(pserver_prog) +elif training_role == "TRAINER": + trainer_prog = t.get_trainer_program() + feeder = fluid.DataFeeder(feed_list=[images, label], place=place) + exe.run(fluid.default_startup_program()) + + for pass_id in range(PASS_NUM): + accuracy.reset(exe) + batch_id = 0 + for data in train_reader(): + loss, acc = exe.run(trainer_prog, + feed=feeder.feed(data), + fetch_list=[avg_cost] + accuracy.metrics) + pass_acc = accuracy.eval(exe) + if batch_id % 100 == 0: + print("batch_id %d, loss: %f, acc: %f" % + (batch_id, loss, pass_acc)) + batch_id += 1 + + pass_acc = accuracy.eval(exe) + print("pass_id=" + str(pass_id) + " pass_acc=" + str(pass_acc)) +else: + print("environment var TRAINER_ROLE should be TRAINER os PSERVER") From 030c0b91fb272b5899cdd53797aa5b902f1454e1 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 18 Jan 2018 15:46:32 -0800 Subject: [PATCH 102/158] Do not send to optimize_ops to distribute transpiler again --- python/paddle/v2/fluid/distribute_transpiler.py | 8 ++++---- .../fluid/tests/book_distribute/notest_dist_fit_a_line.py | 2 +- .../book_distribute/notest_dist_label_semantic_roles.py | 2 +- .../fluid/tests/book_distribute/notest_dist_word2vec.py | 2 +- .../book_distribute/notest_recognize_digits_conv_dist.py | 2 +- .../book_distribute/notest_recognize_digits_mlp_dist.py | 2 +- .../notest_understand_sentiment_conv_dist.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 06a7b6fb02..9591bd92af 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -407,7 +407,7 @@ class DistributeTranspiler: outputs=opt_op.outputs, attrs=opt_op.attrs) - def get_pserver_program(self, endpoint, optimize_ops): + def get_pserver_program(self, endpoint): """ get pserver side program by endpoint @@ -422,9 +422,9 @@ class DistributeTranspiler: self._clone_var(pserver_program.global_block(), v) # step6 optimize_sub_program = Program() - for idx, opt_op in enumerate(optimize_ops): - is_op_on_pserver = self._is_op_on_pserver(endpoint, optimize_ops, - idx) + for idx, opt_op in enumerate(self.optimize_ops): + is_op_on_pserver = self._is_op_on_pserver(endpoint, + self.optimize_ops, idx) if not is_op_on_pserver: continue if opt_op.inputs.has_key("Grad"): diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py index b886071f94..881cca2b7d 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py @@ -53,7 +53,7 @@ if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) - pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + pserver_prog = t.get_pserver_program(current_endpoint) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) else: diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py index 2b5a098ff2..a9730415e6 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py @@ -197,7 +197,7 @@ def main(): if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) - pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + pserver_prog = t.get_pserver_program(current_endpoint) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) elif training_role == "TRAINER": diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py index dc04af5b7b..65b81778d4 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py @@ -87,7 +87,7 @@ if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) - pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + pserver_prog = t.get_pserver_program(current_endpoint) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) elif training_role == "TRAINER": diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py index 27512c4f78..c0a3a36505 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py @@ -66,7 +66,7 @@ if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) - pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + pserver_prog = t.get_pserver_program(current_endpoint) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) elif training_role == "TRAINER": diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py index 6de3e4da74..6cd3e84f35 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py @@ -60,7 +60,7 @@ if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) - pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + pserver_prog = t.get_pserver_program(current_endpoint) exe.run(fluid.default_startup_program()) exe.run(pserver_prog) elif training_role == "TRAINER": diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py index 74f20f3f4c..9e0a1a52a4 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py @@ -98,7 +98,7 @@ def main(): if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) - pserver_prog = t.get_pserver_program(current_endpoint, optimize_ops) + pserver_prog = t.get_pserver_program(current_endpoint) exe.run(pserver_prog) elif training_role == "TRAINER": trainer_prog = t.get_trainer_program() From 0adbc598d9fcade0a06ecbf55f456570dc096b5e Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 18 Jan 2018 16:22:15 -0800 Subject: [PATCH 103/158] doc update with PR 7667 --- doc/howto/usage/cluster/fluid_cluster_train_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/usage/cluster/fluid_cluster_train_en.md b/doc/howto/usage/cluster/fluid_cluster_train_en.md index 419eac51aa..09f7f0e70b 100644 --- a/doc/howto/usage/cluster/fluid_cluster_train_en.md +++ b/doc/howto/usage/cluster/fluid_cluster_train_en.md @@ -103,7 +103,7 @@ t.transpile(optimize_ops, params_grads, pservers=pserver_endpoints, trainers=2) # in pserver, run this exe.run(fluid.default_startup_program()) #current_endpoint here means current pserver IP:PORT you wish to run on -exe.run(t.get_pserver_program(current_endpoint, optimize_ops)) +exe.run(t.get_pserver_program(current_endpoint)) # in trainer, run this ... # define data reader From 7e4cbfe44164e5ca5de32f9a32ccfca0a88dfcca Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Fri, 19 Jan 2018 08:25:30 +0800 Subject: [PATCH 104/158] update switch kernel documentation (#7597) * update switch_kernel.md * update the demo code * update --- doc/design/switch_kernel.md | 101 ++++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 34 deletions(-) diff --git a/doc/design/switch_kernel.md b/doc/design/switch_kernel.md index 1846e5d9f9..9719e031c7 100644 --- a/doc/design/switch_kernel.md +++ b/doc/design/switch_kernel.md @@ -1,21 +1,24 @@ ## Background -Every operator has many kernels because there are multiple data types, places, data layout that Fluid supports. We use the `KernelType` to describe kernel types that operators can hold. +Every operator has many kernels because there are multiple data types, places, data layout, library type that Fluid supports. We use the `OpKernelType ` to describe kernel types that operators can hold. -The `KernelType` is as follows. +The `OpKernelType ` is as follows: -``` -struct KernelType { +```cpp +struct OpKernelType { Place place_; DataType data_type_; - LayoutType layout_; + DataLayout data_layout_; + LibraryType library_type_; }; ``` -The `place_` is a descriptor of the device and the computational library, e.g., `MKLDNNPlace`, `CUDAPlace`. +- The `place_` is a descriptor of the device, e.g., CPUPlace, CUDAPlace. -The `data_type_` is the data type that this kernel performs on, e.g., `FP32`, `INT64`. Note that one kernel may have inputs with different data types. However, it will be a major `data_type`. For example, the `cross_entropy` takes `int64` as it label, and `double`/`float` as its input logit and output cost. The major `data_type` of `cross_entropy` is `float`/`double`. +- The `data_type_` is the data type that this kernel performs on, e.g., `FP32`, `INT64`. Note that one kernel may have inputs with different data types. However, it will be a major `data_type`. For example, the `cross_entropy` takes `int64` as it label, and `double`/`float` as its input logit and output cost. The major `data_type` of `cross_entropy` is `float` or `double`. -The `layout` is useful for some computational library. One example is that MKLDNN uses many kinds of layout, such as `nChw8c`. Each kind of layout will invoke the different kernel. +- The `data_layout_ ` is useful for some computational library. One example is that MKLDNN uses many kinds of layout, such as `nChw8c`. Each kind of layout will invoke the different kernel. + +- The `library_type_` describes the computational library, e.g., `MKLDNN`, `CUDNN`. ## Problem @@ -25,42 +28,72 @@ We register a kernel for every operator and every kernel type ideally. However, 2. Some operators will take too many memory. It is better to force them into CPU. However, the rest of operators in this neural network will be performed on GPU, i.e., model parallel problem. 3. Some layout and place are particular. One example is that MKLDNN uses `nChw8` and there is no other library uses `nChw8c`. -Problems under these situations are similar. We can formalise this problem as follow. +Take one situation to give a detailed explanation, if we have two Operators: OP1 and OP2, OP1 has one output `op1_to_op2`, and `op1_to_op2` is the input of OP2. + +If OP1 and OP2 run on the same place(for example CPUPlace), then `op1_2_op2` can be used directly by OP2. + +``` +OP1(CPUPlace) + | + op1_2_op2 + | +OP2(CPUPlace) +``` + +If OP1 and OP2 run one different place, then OP2 cannot `use op1_2_op2` directly. + +Problems under these situations are similar. We can formalize this problem as follow. We register kernels with types $KT = \{kt_1, kt_2, kt_3, ...\}$ for one operator. The inputs of this operator should be run on kernel type $kt_{?}$, which the $kt_{?} \notin KT$. How to cast the input of this operator from $kt_{?}$ to any of kernel type in $KT$. -## Solution +## Solution: data transform -It is clearly that transforming inputs of an operator toadapt another kernel type is not related to the particular operator. So we should register these transformation methods as global methods. +It is clear that transforming inputs of an operator to adapt another kernel type is not related to the particular operator. So we should register these transformation methods as global methods. -We can infer a kernel type from the inputs of an operators. We let this kernel type as `actual kernel type`, which means this kernel type is the actually kernel type that operator should be performed. +We can infer kernel type for each input of an operator. We let this kernel type as `actual kernel type for var`, which means this kernel type is the kernel type that can process this input variable. We can get a kernel type by 1) The configuration of operator description. (Users may want to force use `MKL` for `conv` operator). 2) The place of the current executor. (Executor is running on GPU). This kernel type is what we expect the operator will be performed on. We let this kernel type as `expect kernel type`. -We transform the input data from `actual` to `expect` if the expect kernel type is not as same as actual kernel type. +We transform the input data from `actual` to `expect` if the actual kernel type is not as same as expect kernel type. -The algorithm is described as follow +The algorithm is described as following ```cpp -using DataTransformationFN = std::function; -using KernelTypePair = std::pair; - -map g_data_transformation_; - -void OpWithKernel::Run() { - vec inputs = ... - auto actual_kernel_type = GetActualKernelType(inputs); - - // The expected kernel type is related to actual kernel type. - // For the most operators, the expected kernel type is as same as - // actual kernel type. - // - // So we pass `actual_kernel_type` as a parameter of - // GetExpectedKernelType - auto expect_kernel_type = GetExpectedKernelType(actual_kernel_type); - - auto trans = g_data_transformation_[{actual_kernel_type, expect_kernel_type}]; - - kernel.run(trans(inputs)); +void OperatorWithKernel::Run( + const Scope& scope, + const platform::Place& place) const { + ExecutionContext ctx(...); + auto expected_kernel_key = this->GetExpectedKernelType(ctx); + + Scope& new_scope = scope.NewScope(); + + for (auto& var_name : this->Inputs()) { + auto* tensor_in = GetTensor(var_name); + auto kernel_type_for_var = this->GetKernelTypeForVar(...); + if (kernel_type_for_var.place_ != expected_kernel_key.place_) { + auto* trans_var = new_scope.Var(var_name); + auto* out = DataTransform(expected_kernel_key, + kernel_type_for_var, + *tensor_in); + CopyVariableWithTensor(...); + } + } + + auto kernel = kernels.find(expected_kernel_key); + kernel->Compute(ExecutionContext(...)); } ``` + +then the actual process for the multi-device above will be: + +``` +OP1(CPUPlace) + | +op1_2_op2(on CPU) + | +[transform](from CPU to GPU) + | +op1_2_op2(on GPU) + | +OP2(CUDAPlace) +``` From 95d6dce354052e41e571ec9f1a199c30fc3dd1fc Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 18 Jan 2018 17:17:18 -0800 Subject: [PATCH 105/158] update doc and dist test due to API change https://github.com/PaddlePaddle/Paddle/pull/7619#pullrequestreview-89656079 --- doc/howto/usage/cluster/fluid_cluster_train_en.md | 6 ++++-- .../fluid/tests/book_distribute/notest_dist_fit_a_line.py | 3 ++- .../book_distribute/notest_dist_label_semantic_roles.py | 3 ++- .../v2/fluid/tests/book_distribute/notest_dist_word2vec.py | 3 ++- .../book_distribute/notest_recognize_digits_conv_dist.py | 3 ++- .../book_distribute/notest_recognize_digits_mlp_dist.py | 3 ++- .../notest_understand_sentiment_conv_dist.py | 5 +++-- 7 files changed, 17 insertions(+), 9 deletions(-) diff --git a/doc/howto/usage/cluster/fluid_cluster_train_en.md b/doc/howto/usage/cluster/fluid_cluster_train_en.md index 09f7f0e70b..a64004a7c4 100644 --- a/doc/howto/usage/cluster/fluid_cluster_train_en.md +++ b/doc/howto/usage/cluster/fluid_cluster_train_en.md @@ -101,9 +101,11 @@ t.transpile(optimize_ops, params_grads, pservers=pserver_endpoints, trainers=2) ... #create executor # in pserver, run this -exe.run(fluid.default_startup_program()) #current_endpoint here means current pserver IP:PORT you wish to run on -exe.run(t.get_pserver_program(current_endpoint)) +pserver_prog = t.get_pserver_program(current_endpoint) +pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) +exe.run(pserver_startup) +exe.run(pserver_prog) # in trainer, run this ... # define data reader diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py index 881cca2b7d..00e0b22c64 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py @@ -54,7 +54,8 @@ if training_role == "PSERVER": print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) - exe.run(fluid.default_startup_program()) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) else: trainer_prog = t.get_trainer_program() diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py index a9730415e6..adc7ae8adf 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py @@ -198,7 +198,8 @@ def main(): print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) - exe.run(fluid.default_startup_program()) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": trainer_prog = t.get_trainer_program() diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py index 65b81778d4..27ef2cad1d 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py @@ -88,7 +88,8 @@ if training_role == "PSERVER": print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) - exe.run(fluid.default_startup_program()) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": feeder = fluid.DataFeeder( diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py index c0a3a36505..8af228e7bf 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py @@ -67,7 +67,8 @@ if training_role == "PSERVER": print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) - exe.run(fluid.default_startup_program()) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": trainer_prog = t.get_trainer_program() diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py index 6cd3e84f35..9cfb4ab8c4 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py @@ -61,7 +61,8 @@ if training_role == "PSERVER": print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) - exe.run(fluid.default_startup_program()) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": trainer_prog = t.get_trainer_program() diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py index 9e0a1a52a4..840afb6376 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py @@ -92,15 +92,16 @@ def main(): t.transpile( optimize_ops, params_grads, pservers=pserver_endpoints, trainers=2) - exe.run(fluid.default_startup_program()) - if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": + exe.run(fluid.default_startup_program()) trainer_prog = t.get_trainer_program() feeder = fluid.DataFeeder(feed_list=[data, label], place=place) From c79d530ad387399ea2cbb97d7f9eb771d694711f Mon Sep 17 00:00:00 2001 From: Yancey Date: Fri, 19 Jan 2018 10:22:29 +0800 Subject: [PATCH 106/158] Add split selected rows op (#7604) * add split selected rows op * update comment * add grad check * registry cuda kernel * fix ci failed --- paddle/operators/split_selected_rows_op.cc | 114 ++++++++++++++++ paddle/operators/split_selected_rows_op.cu | 19 +++ paddle/operators/split_selected_rows_op.h | 58 ++++++++ .../tests/test_split_selected_rows_op.py | 128 ++++++++++++++++++ 4 files changed, 319 insertions(+) create mode 100644 paddle/operators/split_selected_rows_op.cc create mode 100644 paddle/operators/split_selected_rows_op.cu create mode 100644 paddle/operators/split_selected_rows_op.h create mode 100644 python/paddle/v2/fluid/tests/test_split_selected_rows_op.py diff --git a/paddle/operators/split_selected_rows_op.cc b/paddle/operators/split_selected_rows_op.cc new file mode 100644 index 0000000000..d9a023987b --- /dev/null +++ b/paddle/operators/split_selected_rows_op.cc @@ -0,0 +1,114 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/split_selected_rows_op.h" + +namespace paddle { +namespace operators { + +class SplitSelectedRowsOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SplitSelectedRowsOpMaker(OpProto *proto, OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The input SelectedRows."); + AddOutput("Out", "The outputs of input SelectedRows.").AsDuplicable(); + AddAttr>("rows_sections", "Rows section for output.") + .SetDefault(std::vector({})); + AddAttr>("height_sections", + "Height for each output SelectedRows.") + .SetDefault(std::vector({})); + + AddComment(R"DOC( +Split a SelectedRows with a specified rows section. +height_sections is only needed when need to split the dims of the original tensor. + +Example: + Input: + X.rows = {0, 7, 5} + X.height = 12 + Attr: + rows_sections = {1, 2} + height_sections = {} + Out: + out0.rows = {0} + out0.height = 12 + out1.rows = {7, 5} + out2.height = 12 + +)DOC"); + } +}; + +class SplitSelectedRowsOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "SplitSelectedRowsOp must has input X."); + PADDLE_ENFORCE(ctx->HasOutputs("Out"), + "SplitSelectedRowsOp must has output Out."); + + std::vector height_sections = + ctx->Attrs().Get>("height_sections"); + std::vector rows_sections = + ctx->Attrs().Get>("rows_sections"); + PADDLE_ENFORCE_EQ( + rows_sections.size(), ctx->Outputs("Out").size(), + "The size of rows section should be the same with Outputs size."); + int64_t n = ctx->Outputs("Out").size(); + + std::vector outs_dims; + outs_dims.reserve(n); + + // make output dims + for (int64_t i = 0; i < n; ++i) { + auto dims = ctx->GetInputDim("X"); + if (height_sections.size()) { + PADDLE_ENFORCE_EQ( + height_sections.size(), static_cast(n), + "The size of height section should be the same with height" + " section size."); + dims[0] = height_sections[i]; + } + outs_dims.push_back(dims); + } + ctx->SetOutputsDim("Out", outs_dims); + } +}; + +class SplitSelectedRowsGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDesc(); + grad_op->SetType("sum"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(split_selected_rows, ops::SplitSelectedRowsOp, + ops::SplitSelectedRowsOpMaker, + ops::SplitSelectedRowsGradMaker); +REGISTER_OP_CPU_KERNEL( + split_selected_rows, + ops::SplitSelectedRowsOpKernel); diff --git a/paddle/operators/split_selected_rows_op.cu b/paddle/operators/split_selected_rows_op.cu new file mode 100644 index 0000000000..983285480f --- /dev/null +++ b/paddle/operators/split_selected_rows_op.cu @@ -0,0 +1,19 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/split_selected_rows_op.h" +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL( + split_selected_rows, + ops::SplitSelectedRowsOpKernel); diff --git a/paddle/operators/split_selected_rows_op.h b/paddle/operators/split_selected_rows_op.h new file mode 100644 index 0000000000..1cae53f1af --- /dev/null +++ b/paddle/operators/split_selected_rows_op.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class SplitSelectedRowsOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* x = ctx.Input("X"); + auto outs = ctx.MultiOutput("Out"); + + auto rows_sections = ctx.Attr>("rows_sections"); + auto height_sections = ctx.Attr>("height_sections"); + + int64_t n = outs.size(); + int offset = 0; + + for (int64_t i = 0; i < n; ++i) { + framework::Vector out_rows; + for (int64_t j = 0; j < rows_sections[i]; ++j) { + out_rows.push_back(x->rows()[offset + j]); + } + + auto& out = outs[i]; + auto x_dims = x->GetCompleteDims(); + x_dims[0] = rows_sections[i]; + out->mutable_value()->mutable_data(x_dims, ctx.GetPlace()); + framework::Copy(x->value().Slice(offset, rows_sections[i] + offset), + x->place(), ctx.device_context(), out->mutable_value()); + outs[i]->set_rows(out_rows); + if (height_sections.size()) { + outs[i]->set_height(height_sections[i]); + } + offset += rows_sections[i]; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py b/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py new file mode 100644 index 0000000000..a6cc4f6c6d --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py @@ -0,0 +1,128 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. +import unittest +import paddle.v2.fluid.core as core +import numpy as np +from paddle.v2.fluid.op import Operator + + +class TestSpliteSelectedRows(unittest.TestCase): + def get_places(self): + places = [core.CPUPlace()] + if core.is_compile_gpu(): + places.append(core.CUDAPlace(0)) + return places + + def test_check_output(self): + for place in self.get_places(): + self.check_with_place(place) + + def test_check_grad(self): + for place in self.get_places(): + self.check_grad_with_place(place) + + def check_with_place(self, place): + scope = core.Scope() + rows = [0, 5, 7, 4] + height = 10 + row_numel = 2 + + # initialize input variable X + x = scope.var('X').get_selected_rows() + x.set_rows(rows) + x.set_height(height) + np_array = np.ones((len(rows), row_numel)).astype("float32") + np_array[0, 0] = 2.0 + np_array[2, 1] = 4.0 + x_tensor = x.get_tensor() + x_tensor.set(np_array, place) + + rows_sections = [2, 2] + height_sections = [] + + # initialize output variables [out0, out1] + out0 = scope.var('out0').get_selected_rows() + out1 = scope.var('out1').get_selected_rows() + + # expected output selected rows + expected_out0_rows = [0, 5] + expected_out1_rows = [7, 4] + expected_height = height + + op = Operator( + "split_selected_rows", + X="X", + Out=["out0", "out1"], + rows_sections=rows_sections, + height_sections=height_sections) + + op.run(scope, place) + + self.assertEqual(out0.rows(), expected_out0_rows) + self.assertEqual(out1.rows(), expected_out1_rows) + + self.assertEqual(out0.height(), expected_height) + self.assertEqual(out1.height(), expected_height) + + self.assertAlmostEqual(2.0, np.array(out0.get_tensor())[0, 0]) + self.assertAlmostEqual(4.0, np.array(out1.get_tensor())[0, 1]) + + def check_grad_with_place(self, place): + scope = core.Scope() + height = 10 + row_numel = 2 + + # attr + rows_sections = [2, 2] + height_sections = [] + + # initialize input variable X + out0_grad = scope.var("out0@GRAD").get_selected_rows() + rows0 = [0, 5] + out0_grad.set_rows(rows0) + out0_grad.set_height(height) + out0_grad_tensor = out0_grad.get_tensor() + np_array = np.ones((len(rows0), row_numel)).astype("float32") + np_array[0, 0] = 2.0 + out0_grad_tensor.set(np_array, place) + + out1_grad = scope.var("out1@GRAD").get_selected_rows() + rows1 = [7, 5] + out1_grad.set_rows(rows1) + out1_grad.set_height(height) + out1_grad_tensor = out1_grad.get_tensor() + np_array = np.ones((len(rows1), row_numel)).astype("float32") + np_array[0, 1] = 4.0 + out1_grad_tensor.set(np_array, place) + + x_grad = scope.var("X@GRAD").get_selected_rows() + + grad_op = Operator( + "sum", + X=["out0@GRAD", "out1@GRAD"], + Out="X@GRAD", + rows_sections=rows_sections, + height_sections=height_sections) + + grad_op.run(scope, place) + + self.assertEqual(x_grad.rows(), rows0 + rows1) + self.assertEqual(x_grad.height(), height) + + self.assertAlmostEqual(2.0, np.array(x_grad.get_tensor())[0, 0]) + self.assertAlmostEqual(4.0, np.array(x_grad.get_tensor())[2, 1]) + + +if __name__ == "__main__": + unittest.main() From 4020451a7b6a42e3a86bafeb683b2c20297560ab Mon Sep 17 00:00:00 2001 From: caoying03 Date: Fri, 19 Jan 2018 10:12:11 +0800 Subject: [PATCH 107/158] delete memory copy from linear_chain_crf_op. --- paddle/operators/linear_chain_crf_op.cc | 4 +- paddle/operators/linear_chain_crf_op.h | 226 ++---------------------- 2 files changed, 20 insertions(+), 210 deletions(-) diff --git a/paddle/operators/linear_chain_crf_op.cc b/paddle/operators/linear_chain_crf_op.cc index 975e394c78..e24bf622b7 100644 --- a/paddle/operators/linear_chain_crf_op.cc +++ b/paddle/operators/linear_chain_crf_op.cc @@ -187,7 +187,7 @@ class LinearChainCRFOp : public framework::OperatorWithKernel { const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("Emission")->type()), - ctx.device_context()); + platform::CPUPlace()); } }; @@ -248,7 +248,7 @@ class LinearChainCRFGradOp : public framework::OperatorWithKernel { framework::ToDataType( ctx.Input(framework::GradVarName("LogLikelihood")) ->type()), - ctx.device_context()); + platform::CPUPlace()); } }; diff --git a/paddle/operators/linear_chain_crf_op.h b/paddle/operators/linear_chain_crf_op.h index f502ebefde..afc197a1c3 100644 --- a/paddle/operators/linear_chain_crf_op.h +++ b/paddle/operators/linear_chain_crf_op.h @@ -65,57 +65,14 @@ class LinearChainCRFOpKernel : public framework::OpKernel { const size_t level = 0; const size_t seq_num = in_lod[level].size() - 1; - // These local variables hold the inputs and outputs, garanteeing them on - // CPU memory, to provide a consistent reference. - // TODO(caoying) Fix this by moving all these local variables into the - // class's data members once we can profile the whole training process. - LoDTensor* emission_weights = nullptr; - LoDTensor emission_weight_tensor; - Tensor* transition_weights = nullptr; - Tensor transition_weight_tensor; - LoDTensor* label = nullptr; - LoDTensor label_tensor; - - Tensor* emission_exps = nullptr; - Tensor emission_exps_tensor; - Tensor* transition_exps = nullptr; - Tensor transition_exps_tensor; - Tensor* alpha = nullptr; - Tensor alpha_tensor; - Tensor* ll = nullptr; - Tensor ll_tensor; - - if (platform::is_gpu_place(ctx.GetPlace())) { - emission_weights = &emission_weight_tensor; - transition_weights = &transition_weight_tensor; - label = &label_tensor; - - CopyInputsToCpuMemory( - ctx.device_context(), *ctx.Input("Emission"), - *ctx.Input("Transition"), *ctx.Input("Label"), - emission_weights, transition_weights, label); - - emission_exps = &emission_exps_tensor; - emission_exps->Resize(emission_weights->dims()); - - transition_exps = &transition_exps_tensor; - transition_exps->Resize(transition_weights->dims()); - - alpha = &alpha_tensor; - alpha->Resize(ctx.Output("Alpha")->dims()); - - ll = &ll_tensor; - } else { - emission_weights = - const_cast(ctx.Input("Emission")); - transition_weights = const_cast(ctx.Input("Transition")); - label = const_cast(ctx.Input("Label")); - - emission_exps = ctx.Output("EmissionExps"); - transition_exps = ctx.Output("TransitionExps"); - alpha = ctx.Output("Alpha"); - ll = ctx.Output("LogLikelihood"); - } + const LoDTensor* emission_weights = ctx.Input("Emission"); + const Tensor* transition_weights = ctx.Input("Transition"); + const LoDTensor* label = ctx.Input("Label"); + + Tensor* emission_exps = ctx.Output("EmissionExps"); + Tensor* transition_exps = ctx.Output("TransitionExps"); + Tensor* alpha = ctx.Output("Alpha"); + Tensor* ll = ctx.Output("LogLikelihood"); // Because the computation codes only runs on CPU, here the memory for all // the outputs is FIXED to be allocated on the CPU memory. @@ -173,61 +130,9 @@ class LinearChainCRFOpKernel : public framework::OpKernel { one_seq, one_seq_row_max, one_seq_exps, *transition_weights, *transition_exps, one_seq_label, &one_seq_alpha); } - - if (platform::is_gpu_place(ctx.GetPlace())) { - CopyOutputsToGpuMemory( - ctx.device_context(), *emission_exps, *transition_exps, *alpha, *ll, - ctx.Output("EmissionExps"), - ctx.Output("TransitionExps"), ctx.Output("Alpha"), - ctx.Output("LogLikelihood")); - } }; private: - void CopyInputsToCpuMemory(const platform::DeviceContext& ctx, - const LoDTensor& emission_weights_src, - const Tensor& transition_weights_src, - const LoDTensor& label_src, - LoDTensor* emission_weights_dst, - Tensor* transition_weights_dst, - LoDTensor* label_dst) const { - // Copy the inputs from GPU memory to CPU memory if this operators runs on - // GPU device. - auto copyLoDTensor = [](const platform::DeviceContext& ctx, - const LoDTensor& src, LoDTensor* dst) { - dst->mutable_data(src.dims(), platform::CPUPlace()); - framework::Copy(src, platform::CPUPlace(), ctx, dst); - }; - - copyLoDTensor(ctx, emission_weights_src, emission_weights_dst); - copyLoDTensor(ctx, label_src, label_dst); - - transition_weights_dst->mutable_data(transition_weights_src.dims(), - platform::CPUPlace()); - framework::Copy(transition_weights_src, platform::CPUPlace(), ctx, - transition_weights_dst); - } - - void CopyOutputsToGpuMemory(const platform::DeviceContext& ctx, - const Tensor& emission_exps_src, - const Tensor& transition_exps_src, - const Tensor& alpha_src, const Tensor& ll_src, - Tensor* emission_exps_dst, - Tensor* transition_exps_dst, Tensor* alpha_dst, - Tensor* ll_dst) const { - // Copy the forward results from CPU memory to GPU memory if this - // operators runs on GPU device. - auto copyTensor = [](const platform::DeviceContext& ctx, const Tensor& src, - Tensor* dst) { - dst->mutable_data(platform::CUDAPlace()); - framework::Copy(src, platform::CUDAPlace(), ctx, dst); - }; - copyTensor(ctx, emission_exps_src, emission_exps_dst); - copyTensor(ctx, transition_exps_src, transition_exps_dst); - copyTensor(ctx, alpha_src, alpha_dst); - copyTensor(ctx, ll_src, ll_dst); - } - T ForwardOneSequence(const Tensor& emission, const Tensor& emission_row_max, const Tensor& emission_exps, const Tensor& trans_weights, const Tensor& trans_weight_exps, const Tensor& label, @@ -296,63 +201,17 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { auto lod = ctx.Input("Label")->lod(); PADDLE_ENFORCE(lod.size(), "Input(Label) must be a sequence."); - // These local variables hold the inputs and outputs, garanteeing them on - // CPU memory, to provide a consistent reference. - // TODO(caoying) Fix this by moving all these local variables into the - // class's data members once we can profile the training process, or - // implementing a real GPU kernel for CRF. - Tensor* label = nullptr; - Tensor label_tensor; - Tensor* emission_exps = nullptr; - Tensor emission_exps_tensor; - Tensor* transition_exps = nullptr; - Tensor transition_exps_tensor; - Tensor* alpha = nullptr; - Tensor alpha_tensor; - Tensor ll_grad_tensor; - T* ll_grad = nullptr; - - Tensor* emission_grad = nullptr; - Tensor emission_grad_tensor; - Tensor* transition_grad = nullptr; - Tensor transition_grad_tensor; - - if (platform::is_gpu_place(ctx.GetPlace())) { - label = &label_tensor; - emission_exps = &emission_exps_tensor; - transition_exps = &transition_exps_tensor; - alpha = &alpha_tensor; - CopyInputsToCpuMemory( - ctx.device_context(), *ctx.Input("Label"), - *ctx.Input("EmissionExps"), - *ctx.Input("TransitionExps"), *ctx.Input("Alpha"), - *ctx.Input(framework::GradVarName("LogLikelihood")), label, - emission_exps, transition_exps, alpha, &ll_grad_tensor); - ll_grad = ll_grad_tensor.data(); - - if (ctx.Output(framework::GradVarName("Emission"))) { - emission_grad = &emission_grad_tensor; - emission_grad->Resize(emission_exps->dims()); - } + const Tensor* label = ctx.Input("Label"); + const Tensor* emission_exps = ctx.Input("EmissionExps"); + const Tensor* transition_exps = ctx.Input("TransitionExps"); + const Tensor* alpha = ctx.Input("Alpha"); + const T* ll_grad = + ctx.Input(framework::GradVarName("LogLikelihood"))->data(); - if (ctx.Output(framework::GradVarName("Transition"))) { - transition_grad = &transition_grad_tensor; - transition_grad->Resize(transition_exps->dims()); - } - } else { - label = const_cast(ctx.Input("Label")); - emission_exps = const_cast(ctx.Input("EmissionExps")); - transition_exps = - const_cast(ctx.Input("TransitionExps")); - alpha = const_cast(ctx.Input("Alpha")); - ll_grad = const_cast( - ctx.Input(framework::GradVarName("LogLikelihood"))) - ->data(); - - emission_grad = ctx.Output(framework::GradVarName("Emission")); - transition_grad = - ctx.Output(framework::GradVarName("Transition")); - } + Tensor* emission_grad = + ctx.Output(framework::GradVarName("Emission")); + Tensor* transition_grad = + ctx.Output(framework::GradVarName("Transition")); // TODO(caoying) Fix this constraint. When the Input(Emission) is from the // data reader operator, it can have no gradients. @@ -389,58 +248,9 @@ class LinearChainCRFGradOpKernel : public framework::OpKernel { one_seq_emission_exps, *transition_exps, one_seq_alpha, one_seq_label, &one_seq_beta, transition_grad, &one_seq_emission_grad); } - - if (platform::is_gpu_place(ctx.GetPlace())) { - CopyOutputsToGpuMemory( - ctx.device_context(), emission_grad, transition_grad, - ctx.Output(framework::GradVarName("Emission")), - ctx.Output(framework::GradVarName("Transition"))); - } }; private: - void CopyInputsToCpuMemory(const platform::DeviceContext& ctx, - const LoDTensor& label_src, - const Tensor& emission_exps_src, - const Tensor& transition_exps_src, - const Tensor& alpha_src, const Tensor& ll_grad_src, - Tensor* label_dst, Tensor* emission_exps_dst, - Tensor* transition_exps_dst, Tensor* alpha_dst, - Tensor* ll_grad_dst) const { - // Copy the inputs from GPU memory to CPU memory when this operators runs on - // GPU device. - label_dst->mutable_data(label_src.dims(), platform::CPUPlace()); - framework::Copy(label_src, platform::CPUPlace(), ctx, label_dst); - - auto copyTensor = [](const platform::DeviceContext& ctx, const Tensor& src, - Tensor* dst) { - dst->mutable_data(src.dims(), platform::CPUPlace()); - framework::Copy(src, platform::CPUPlace(), ctx, dst); - }; - copyTensor(ctx, emission_exps_src, emission_exps_dst); - copyTensor(ctx, transition_exps_src, transition_exps_dst); - copyTensor(ctx, alpha_src, alpha_dst); - copyTensor(ctx, ll_grad_src, ll_grad_dst); - } - - void CopyOutputsToGpuMemory(const platform::DeviceContext& ctx, - const Tensor* emission_grad_src, - const Tensor* transition_grad_src, - Tensor* emission_grad_dst, - Tensor* transition_grad_dst) const { - // Copy the backward results from CPU memory to GPU - // memory if this operators runs on GPU device. - auto copyTensor = [](const platform::DeviceContext& ctx, const Tensor* src, - Tensor* dst) { - if (src && dst) { - dst->mutable_data(platform::CUDAPlace()); - framework::Copy(*src, platform::CUDAPlace(), ctx, dst); - } - }; - copyTensor(ctx, emission_grad_src, emission_grad_dst); - copyTensor(ctx, transition_grad_src, transition_grad_dst); - } - void BackwardOneSequence(const platform::CPUDeviceContext& ctx, const T ll_grad, const Tensor& emission_exps, const Tensor& transition_exps, const Tensor& alpha, From 07cf302e6e1556f7be987f11181d9bbdd96254c7 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 18 Jan 2018 18:36:43 -0800 Subject: [PATCH 108/158] first commit --- paddle/operators/parallel_do_op.cc | 10 +++++++++- python/paddle/v2/fluid/tests/test_parallel_op.py | 7 ++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/paddle/operators/parallel_do_op.cc b/paddle/operators/parallel_do_op.cc index c2561fa2bf..a00458ea06 100644 --- a/paddle/operators/parallel_do_op.cc +++ b/paddle/operators/parallel_do_op.cc @@ -64,6 +64,12 @@ static void SplitTensorAndMoveTensorToScopes( } } +void WaitOnPlace(const platform::Place place) { + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(place); + dev_ctx.Wait(); +} + void WaitOnPlaces(const std::vector places) { platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); @@ -214,6 +220,7 @@ class ParallelDoGradOp : public framework::OperatorBase { auto &tensor_to_merge = sub_scopes[i]->FindVar(s)->Get(); if (!(places[i] == places[0])) { framework::Copy(tensor_to_merge, places[0], tmp); + WaitOnPlace(places[0]); } else { tmp->ShareDataWith(tensor_to_merge); } @@ -222,12 +229,13 @@ class ParallelDoGradOp : public framework::OperatorBase { "sum", {{"X", {s, tmp_name}}}, {{"Out", {s}}}, framework::AttributeMap{}); sum_op->Run(*sub_scopes[0], places[0]); - WaitOnPlaces(places); + WaitOnPlace(places[0]); } VLOG(3) << result; framework::Copy(result, place, scope.FindVar(s)->GetMutable()); } + WaitOnPlaces(places); } }; diff --git a/python/paddle/v2/fluid/tests/test_parallel_op.py b/python/paddle/v2/fluid/tests/test_parallel_op.py index 45196ef6fe..d36f7d07ac 100644 --- a/python/paddle/v2/fluid/tests/test_parallel_op.py +++ b/python/paddle/v2/fluid/tests/test_parallel_op.py @@ -15,9 +15,6 @@ import unittest import paddle.v2.fluid as fluid import numpy -import sys -# TODO(dzhwinter): get places op check need to be enhanced. -sys.exit(0) class BaseParallelForTest(unittest.TestCase): @@ -165,13 +162,13 @@ class ParallelOpTest(BaseParallelForTest): feed={ 'img': numpy.random.random(size=(51, 784)).astype('float32') }, - fetch='fc1.w@GRAD') + fetch=['fc1.w@GRAD']) def test_fc_with_tiny_data(self): self.run_test( callback=ParallelOpTest.__network__, feed={'img': numpy.random.random(size=(1, 784)).astype('float32')}, - fetch='fc1.w@GRAD') + fetch=['fc1.w@GRAD']) if __name__ == '__main__': From 9f50195b9d6346e08a59e6656f55b0d22efb3d81 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 19 Jan 2018 11:15:55 +0800 Subject: [PATCH 109/158] update using cifar10 --- benchmark/cluster/v2/Dockerfile | 8 ++++---- benchmark/cluster/v2/reader.py | 2 +- benchmark/cluster/v2/vgg16.py | 22 +++++++++++++++------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/benchmark/cluster/v2/Dockerfile b/benchmark/cluster/v2/Dockerfile index 32e68b6150..ac85b1a7d0 100644 --- a/benchmark/cluster/v2/Dockerfile +++ b/benchmark/cluster/v2/Dockerfile @@ -1,5 +1,5 @@ FROM paddlepaddle/paddlecloud-job -RUN mkdir -p /workspace && mkdir -p /root/.cache/paddle/dataset/flowers/ -ADD vgg16.py reader.py /workspace/ -COPY 102flowers.tgz imagelabels.mat setid.mat /root/.cache/paddle/dataset/flowers/ - +RUN mkdir -p /workspace +ADD reader.py /workspace/ +RUN python /workspace/reader.py +ADD vgg16.py /workspace/ diff --git a/benchmark/cluster/v2/reader.py b/benchmark/cluster/v2/reader.py index 060bf2bda2..16ac2dbcef 100644 --- a/benchmark/cluster/v2/reader.py +++ b/benchmark/cluster/v2/reader.py @@ -67,4 +67,4 @@ if __name__ == '__main__': # print len(im[0]) #for im in train_reader('test.list'): # print len(im[0]) - paddle.dataset.flowers.train() + paddle.dataset.cifar.train10() diff --git a/benchmark/cluster/v2/vgg16.py b/benchmark/cluster/v2/vgg16.py index 8644a547b3..9189493276 100644 --- a/benchmark/cluster/v2/vgg16.py +++ b/benchmark/cluster/v2/vgg16.py @@ -14,13 +14,15 @@ import gzip -import paddle.v2.dataset.flowers as flowers +import paddle.v2.dataset.cifar as cifar import paddle.v2 as paddle import reader +import time -DATA_DIM = 3 * 224 * 224 # Use 3 * 331 * 331 or 3 * 299 * 299 for Inception-ResNet-v2. -CLASS_DIM = 102 +DATA_DIM = 3 * 32 * 32 +CLASS_DIM = 10 BATCH_SIZE = 128 +ts = 0 def vgg(input, nums, class_dim): @@ -74,6 +76,7 @@ def vgg19(input, class_dim): def main(): + global ts paddle.init(use_gpu=False, trainer_count=1) image = paddle.layer.data( name="image", type=paddle.data_type.dense_vector(DATA_DIM)) @@ -100,13 +103,13 @@ def main(): train_reader = paddle.batch( paddle.reader.shuffle( - flowers.train(), + cifar.train10(), # To use other data, replace the above line with: # reader.train_reader('train.list'), buf_size=1000), batch_size=BATCH_SIZE) test_reader = paddle.batch( - flowers.valid(), + cifar.test10(), # To use other data, replace the above line with: # reader.test_reader('val.list'), batch_size=BATCH_SIZE) @@ -120,10 +123,14 @@ def main(): # End batch and end pass event handler def event_handler(event): + global ts + if isinstance(event, paddle.event.BeginIteration): + ts = time.time() if isinstance(event, paddle.event.EndIteration): if event.batch_id % 1 == 0: - print "\nPass %d, Batch %d, Cost %f, %s" % ( - event.pass_id, event.batch_id, event.cost, event.metrics) + print "\nPass %d, Batch %d, Cost %f, %s, spent: %f" % ( + event.pass_id, event.batch_id, event.cost, event.metrics, + time.time() - ts) if isinstance(event, paddle.event.EndPass): with gzip.open('params_pass_%d.tar.gz' % event.pass_id, 'w') as f: trainer.save_parameter_to_tar(f) @@ -137,3 +144,4 @@ def main(): if __name__ == '__main__': main() + From 6089b50c4b01f507e0fe7200a68a1972fd1505c0 Mon Sep 17 00:00:00 2001 From: wanghaoshuang Date: Fri, 19 Jan 2018 11:24:04 +0800 Subject: [PATCH 110/158] Registry int64_t kernels --- paddle/operators/ctc_align_op.cc | 3 ++- paddle/operators/ctc_align_op.cu | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/operators/ctc_align_op.cc b/paddle/operators/ctc_align_op.cc index 3fa8d2af74..eeecbd3212 100644 --- a/paddle/operators/ctc_align_op.cc +++ b/paddle/operators/ctc_align_op.cc @@ -89,4 +89,5 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(ctc_align, ops::CTCAlignOp, ops::CTCAlignOpMaker, paddle::framework::EmptyGradOpMaker); REGISTER_OP_CPU_KERNEL( - ctc_align, ops::CTCAlignKernel); + ctc_align, ops::CTCAlignKernel, + ops::CTCAlignKernel); diff --git a/paddle/operators/ctc_align_op.cu b/paddle/operators/ctc_align_op.cu index 99e716e989..45635f1674 100644 --- a/paddle/operators/ctc_align_op.cu +++ b/paddle/operators/ctc_align_op.cu @@ -87,5 +87,5 @@ class CTCAlignOpCUDAKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_CUDA_KERNEL(ctc_align, - paddle::operators::CTCAlignOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(ctc_align, paddle::operators::CTCAlignOpCUDAKernel, + paddle::operators::CTCAlignOpCUDAKernel); From 7869a05ffccc6f98271bce1236f4bbed53a50f48 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Fri, 19 Jan 2018 11:47:23 +0800 Subject: [PATCH 111/158] update new_op doc (#7672) * update new_op doc * follow comment --- doc/howto/dev/new_op_en.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index 7175d8370d..da8b1bdd10 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -4,7 +4,8 @@ - [Implementing C++ Types](#implementing-c-types) - [Defining ProtoMaker](#defining-protomaker) - [Defining Operator](#defining-operator) - - [Registering Operator](#registering-operator) + - [Defining OpKernel](#defining-opkernel) + - [Registering Operator and OpKernel](#registering-operator-and-opkernel) - [Compilation](#compilation) - [Python Binding](#python-binding) - [Unit Tests](#unit-tests) @@ -16,12 +17,13 @@ Here are the base types needed. For details, please refer to the design docs. -- `framework::OperatorBase`: Operator (Op)base class. -- `framework::OpKernel`: Base class for Op computation. -- `framework::OperatorWithKernel`: Inherited from OperatorBase, describing an operator with computation. - `class OpProtoAndCheckerMaker`: Describes an Operator's input, output, attributes and description, mainly used to interface with Python API. +- `framework::OperatorBase`: Operator (Op)base class. +- `framework::OpKernel`: Base class for Op computation kernel. +- `framework::OperatorWithKernel`: Inherited from OperatorBase, describing an operator with computation kernels. + -An operator can be differentiated by whether in has kernel methods. An operator with kernel inherits from `OperatorWithKernel` while the ones without inherit from `OperatorBase`. This tutorial focuses on implementing operators with kernels. In short, an operator includes the following information: +Operators can be categorized into two groups: operator with kernel(s) and operator without kernel(s). An operator with kernel(s) inherits from `OperatorWithKernel` while the one without kernel(s) inherits from `OperatorBase`. This tutorial focuses on implementing operators with kernels. In short, an operator includes the following information: Information | Where is it defined @@ -32,7 +34,7 @@ Kernel implementation | The kernel methods shared between CPU and CUDA are Registering the Op | Ops are registered in `.cc` files; For Kernel registration, `.cc` files contain the CPU implementation, while `.cu` files contain the CUDA implementation. -New Operator implementations are added to the list [paddle/operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators), with file names in the format `*_op.h` (if applicable), `*_op.cc`, `*_op.cu` (if applicable).** The system will use the naming scheme to automatically build operators and their corresponding Python extensions. ** +New Operator implementations are added to the list [paddle/operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators), with file names in the format `*_op.h` (if applicable), `*_op.cc`, `*_op.cu` (if applicable).** The system will use the naming scheme to automatically build operators and their corresponding Python extensions.** Let's take matrix multiplication operator, [MulOp](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/mul_op.cc), as an example to introduce the writing of an Operator with Kernel. @@ -156,7 +158,8 @@ Usually `OpProtoMaker` and `Op`'s type definitions are written in `.cc` files, w - `typename T` denotes data type, such as `float` or `double`. `MulKernel` types need to rewrite the interface for `Compute`. -- `Compute` takes one input variable `const framework::ExecutionContext& context`. + +- `Compute` takes one input parameter: `const framework::ExecutionContext& context`. - Compared with `InferShapeContext`, `ExecutionContext` includes device types, and can similarly extract input, output, and attribute variables. - `Compute` implements the computation logics of an `OpKernel`. @@ -177,7 +180,7 @@ Usually `OpProtoMaker` and `Op`'s type definitions are written in `.cc` files, w }; ``` -Note that **different devices (CPU, CUDA)share an Op definition; whether or not they share the same `OpKernel` depends on whether `Compute` calls functions that support both devices.** +Note that **different devices (CPU, CUDA)share one Op definition; whether or not they share the same `OpKernel` depends on whether `Compute` calls functions can support both devices.** `MulOp`'s CPU and CUDA share the same `Kernel`. A non-sharing `OpKernel` example can be seen in [`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43). @@ -188,13 +191,14 @@ This concludes the forward implementation of an operator. Next its operation and The definition of its corresponding backward operator, if applicable, is similar to that of an forward operator. **Note that a backward operator does not include a `ProtoMaker`**. -### Registering Operator +### Registering Operator and OpKernel - In `.cc` files, register forward and backward operator classes and the CPU kernel. ```cpp namespace ops = paddle::operators; REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad, ops::MulOpGrad); + REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); REGISTER_OP_CPU_KERNEL(mul_grad, ops::MulGradKernel); @@ -204,6 +208,7 @@ The definition of its corresponding backward operator, if applicable, is similar - `REGISTER_OP` registers the `ops::MulOp` class, type named `mul`, its type `ProtoMaker` is `ops::MulOpMaker`, registering `ops::MulOpGrad` as `mul_grad`. - `REGISTER_OP_WITHOUT_GRADIENT` registers an operator without gradient. + - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulGradKernel`. @@ -225,6 +230,7 @@ The definition of its corresponding backward operator, if applicable, is similar Run the following commands to compile. ``` +# maybe you need to rerun cmake make mul_op ``` From 8809d43ad72c7e587ee07a3cf8cfd93dd27417fc Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 18 Jan 2018 19:52:40 -0800 Subject: [PATCH 112/158] Remove unnecessary dtype conversion & register int64 kernels --- paddle/operators/sequence_erase_op.cc | 3 +- paddle/operators/sequence_erase_op.cu | 41 ++++--------------- .../v2/fluid/tests/test_sequence_erase_op.py | 17 +++++++- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/paddle/operators/sequence_erase_op.cc b/paddle/operators/sequence_erase_op.cc index d17b268623..aa0c00aa6f 100644 --- a/paddle/operators/sequence_erase_op.cc +++ b/paddle/operators/sequence_erase_op.cc @@ -86,4 +86,5 @@ REGISTER_OP_WITHOUT_GRADIENT(sequence_erase, ops::SequenceEraseOp, ops::SequenceEraseOpMaker); REGISTER_OP_CPU_KERNEL( sequence_erase, - ops::SequenceEraseKernel); + ops::SequenceEraseKernel, + ops::SequenceEraseKernel); diff --git a/paddle/operators/sequence_erase_op.cu b/paddle/operators/sequence_erase_op.cu index c1e8bc2090..f1e3b96acd 100644 --- a/paddle/operators/sequence_erase_op.cu +++ b/paddle/operators/sequence_erase_op.cu @@ -28,16 +28,12 @@ __global__ void LabelErasedIdx(const T* in_dat, const int64_t in_len, size_t* num_erased) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < in_len) { - int erased = 0; for (size_t i = 0; i < tokens_len; ++i) { if (in_dat[index] == tokens[i]) { - erased = 1; + num_erased[index + 1] = 1; + break; } } - num_erased[index + 1] = erased; - if (index == 0) { - num_erased[0] = 0; - } } } @@ -60,26 +56,6 @@ __global__ void SetOutput(const T* in_dat, const int64_t in_len, } } -template -thrust::device_vector set_device_vector(Vector& vector) { - thrust::host_vector host_vec(vector.size()); - for (size_t i = 0; i < vector.size(); ++i) { - host_vec[i] = vector[i]; - } - thrust::device_vector dev_vec = host_vec; - return dev_vec; -} - -template -std::vector get_std_vector(thrust::device_vector& dev_vec) { - thrust::host_vector host_vec = dev_vec; - std::vector std_vec(host_vec.size(), 0); - for (size_t i = 0; i < host_vec.size(); ++i) { - std_vec[i] = host_vec[i]; - } - return std_vec; -} - template class SequenceEraseOpCUDAKernel : public framework::OpKernel { public: @@ -95,12 +71,11 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel { auto in_len = in->numel(); auto in_dat = in->data(); // Copy tokens to GPU - thrust::device_vector dev_tokens = - set_device_vector>(tokens); + thrust::device_vector dev_tokens(tokens.begin(), tokens.end()); int* dev_tokens_ptr = thrust::raw_pointer_cast(dev_tokens.data()); // Count number of elements to be erased - thrust::device_vector num_erased(in_len + 1); + thrust::device_vector num_erased(in_len + 1, 0); size_t* num_erased_ptr = thrust::raw_pointer_cast(num_erased.data()); auto stream = ctx.cuda_device_context().stream(); LabelErasedIdx<<<(in_len - 1) / PADDLE_CUDA_NUM_THREADS + 1, @@ -112,8 +87,7 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel { // Copy LoD to GPU auto lod0 = lod[0]; auto lod_len = lod0.size(); - thrust::device_vector dev_in_lod = - set_device_vector>(lod0); + thrust::device_vector dev_in_lod = lod0; size_t* dev_in_lod_ptr = thrust::raw_pointer_cast(dev_in_lod.data()); // Calc output LoD @@ -124,7 +98,7 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel { num_erased_ptr, dev_in_lod_ptr, lod_len, dev_out_lod_ptr); // Set LoD for output - std::vector out_lod0 = get_std_vector(dev_out_lod); + thrust::host_vector out_lod0 = dev_out_lod; framework::LoD out_lod; out_lod.push_back(out_lod0); out->set_lod(out_lod); @@ -142,4 +116,5 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel { } // namespace paddle REGISTER_OP_CUDA_KERNEL(sequence_erase, - paddle::operators::SequenceEraseOpCUDAKernel); + paddle::operators::SequenceEraseOpCUDAKernel, + paddle::operators::SequenceEraseOpCUDAKernel); diff --git a/python/paddle/v2/fluid/tests/test_sequence_erase_op.py b/python/paddle/v2/fluid/tests/test_sequence_erase_op.py index d8aa4f7e94..4cc2613cf9 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_erase_op.py +++ b/python/paddle/v2/fluid/tests/test_sequence_erase_op.py @@ -29,7 +29,7 @@ def sequence_erase(in_seq, lod0, tokens): return np.array(out_seq).astype("int32"), new_lod0 -class TestSequenceEraseOp(OpTest): +class TestSequenceEraseOpInt32(OpTest): def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") @@ -44,6 +44,21 @@ class TestSequenceEraseOp(OpTest): self.check_output() +class TestSequenceEraseOpInt64(OpTest): + def setUp(self): + self.op_type = "sequence_erase" + in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") + lod = [[0, 9, 13, 24, 30]] + tokens = [2, 3, 5] + out_seq, new_lod0 = sequence_erase(in_seq, lod[0], tokens) + self.attrs = {'tokens': tokens} + self.inputs = {'X': (in_seq, lod)} + self.outputs = {'Out': (out_seq, [new_lod0])} + + def test_check_output(self): + self.check_output() + + class TestSequenceEraseOpEmpty(OpTest): def setUp(self): self.op_type = "sequence_erase" From 9c0b29014cd38c36f6a599b7c6477000db30917d Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Fri, 19 Jan 2018 12:29:41 +0800 Subject: [PATCH 113/158] Make compare_op reuse elemwise_op_funcs --- paddle/operators/compare_op.cc | 11 +++++----- paddle/operators/compare_op.cu | 4 ---- paddle/operators/compare_op.h | 22 ++----------------- paddle/operators/elementwise_op_function.h | 12 +++++----- .../paddle/v2/fluid/tests/test_compare_op.py | 2 -- 5 files changed, 14 insertions(+), 37 deletions(-) diff --git a/paddle/operators/compare_op.cc b/paddle/operators/compare_op.cc index daa2c193b4..930c295a9c 100644 --- a/paddle/operators/compare_op.cc +++ b/paddle/operators/compare_op.cc @@ -39,6 +39,11 @@ N-dim tensor. X and Y could be any type. The each element of the Out tensor is calculated by %s )DOC", comment.type, comment.equation)); + AddAttr("axis", + "(int, default -1). The start dimension index " + "for broadcasting Y onto X.") + .SetDefault(-1) + .EqualGreaterThan(-1); } }; @@ -95,11 +100,5 @@ REGISTER_LOGICAL_OP(less_than, "Out = X < Y"); REGISTER_LOGICAL_KERNEL(less_than, CPU, paddle::operators::LessThanFunctor); REGISTER_LOGICAL_OP(less_equal, "Out = X <= Y"); REGISTER_LOGICAL_KERNEL(less_equal, CPU, paddle::operators::LessEqualFunctor); -REGISTER_LOGICAL_OP(greater_than, "Out = X > Y"); -REGISTER_LOGICAL_KERNEL(greater_than, CPU, - paddle::operators::GreaterThanFunctor); -REGISTER_LOGICAL_OP(greater_equal, "Out = X >= Y"); -REGISTER_LOGICAL_KERNEL(greater_equal, CPU, - paddle::operators::GreaterEqualFunctor); REGISTER_LOGICAL_OP(equal, "Out = X == Y"); REGISTER_LOGICAL_KERNEL(equal, CPU, paddle::operators::EqualFunctor); diff --git a/paddle/operators/compare_op.cu b/paddle/operators/compare_op.cu index 26049271be..f625824dbc 100644 --- a/paddle/operators/compare_op.cu +++ b/paddle/operators/compare_op.cu @@ -16,8 +16,4 @@ limitations under the License. */ REGISTER_LOGICAL_KERNEL(less_than, CUDA, paddle::operators::LessThanFunctor); REGISTER_LOGICAL_KERNEL(less_equal, CUDA, paddle::operators::LessEqualFunctor); -REGISTER_LOGICAL_KERNEL(greater_than, CUDA, - paddle::operators::GreaterThanFunctor); -REGISTER_LOGICAL_KERNEL(greater_equal, CUDA, - paddle::operators::GreaterEqualFunctor); REGISTER_LOGICAL_KERNEL(equal, CUDA, paddle::operators::EqualFunctor); diff --git a/paddle/operators/compare_op.h b/paddle/operators/compare_op.h index 567e89c0a7..15e9cfcaab 100644 --- a/paddle/operators/compare_op.h +++ b/paddle/operators/compare_op.h @@ -16,6 +16,7 @@ limitations under the License. */ #include #include #include "paddle/framework/op_registry.h" +#include "paddle/operators/elementwise_op_function.h" #include "paddle/platform/transform.h" namespace paddle { @@ -33,18 +34,6 @@ struct LessEqualFunctor { HOSTDEVICE bool operator()(const T& a, const T& b) const { return a <= b; } }; -template -struct GreaterThanFunctor { - using ELEM_TYPE = T; - HOSTDEVICE bool operator()(const T& a, const T& b) const { return a > b; } -}; - -template -struct GreaterEqualFunctor { - using ELEM_TYPE = T; - HOSTDEVICE bool operator()(const T& a, const T& b) const { return a >= b; } -}; - template struct EqualFunctor { using ELEM_TYPE = T; @@ -65,14 +54,7 @@ class CompareOpKernel public: void Compute(const framework::ExecutionContext& context) const override { using T = typename Functor::ELEM_TYPE; - auto* x = context.Input("X"); - auto* y = context.Input("Y"); - auto* out = context.Output("Out"); - Functor binary_func; - platform::Transform trans; - trans(context.template device_context(), x->data(), - x->data() + x->numel(), y->data(), - out->mutable_data(context.GetPlace()), binary_func); + ElementwiseComputeEx(context); } }; diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index db5d30c1af..e6f3e39ece 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -176,14 +176,15 @@ class MidWiseTransformIterator }; #endif -template +template class TransformFunctor { public: TransformFunctor(const framework::Tensor* x, const framework::Tensor* y, framework::Tensor* z, const DeviceContext& ctx, Functor func) : x_(x->data()), y_(y->data()), - z_(z->mutable_data(ctx.GetPlace())), + z_(z->mutable_data(ctx.GetPlace())), nx_(x->numel()), ctx_(ctx), func_(func) {} @@ -208,7 +209,7 @@ class TransformFunctor { private: const T* x_; const T* y_; - T* z_; + OutType* z_; int64_t nx_; const DeviceContext& ctx_; Functor func_; @@ -364,7 +365,8 @@ void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { } } -template +template void ElementwiseComputeEx(const framework::ExecutionContext& ctx) { using Tensor = framework::Tensor; @@ -372,7 +374,7 @@ void ElementwiseComputeEx(const framework::ExecutionContext& ctx) { auto* y = ctx.Input("Y"); auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); - TransformFunctor functor( + TransformFunctor functor( x, y, z, ctx.template device_context(), Functor()); auto x_dims = x->dims(); diff --git a/python/paddle/v2/fluid/tests/test_compare_op.py b/python/paddle/v2/fluid/tests/test_compare_op.py index fbf8921e40..00e781c616 100644 --- a/python/paddle/v2/fluid/tests/test_compare_op.py +++ b/python/paddle/v2/fluid/tests/test_compare_op.py @@ -37,8 +37,6 @@ def create_test_class(op_type, typename, callback): for _type_name in {'float32', 'float64', 'int32', 'int64'}: create_test_class('less_than', _type_name, lambda _a, _b: _a < _b) create_test_class('less_equal', _type_name, lambda _a, _b: _a <= _b) - create_test_class('greater_than', _type_name, lambda _a, _b: _a > _b) - create_test_class('greater_equal', _type_name, lambda _a, _b: _a >= _b) create_test_class('equal', _type_name, lambda _a, _b: _a == _b) if __name__ == '__main__': From 08cb472ab90495d536a91b63135930c2397974b6 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 19 Jan 2018 12:42:54 +0800 Subject: [PATCH 114/158] Simplify the implementation. --- paddle/operators/sequence_reshape_op.cc | 30 +++-- paddle/operators/sequence_reshape_op.cu | 11 +- paddle/operators/sequence_reshape_op.h | 107 ++++-------------- .../v2/fluid/tests/test_sequence_reshape.py | 24 ++-- 4 files changed, 68 insertions(+), 104 deletions(-) diff --git a/paddle/operators/sequence_reshape_op.cc b/paddle/operators/sequence_reshape_op.cc index ddedbc3bc6..884c49276c 100644 --- a/paddle/operators/sequence_reshape_op.cc +++ b/paddle/operators/sequence_reshape_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/sequence_reshape_op.h" +#include "paddle/framework/ddim.h" namespace paddle { namespace operators { @@ -26,9 +27,11 @@ class SequenceReshapeOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of SequenceReshapeOp should not be null."); auto x_dims = ctx->GetInputDim("X"); + auto x_numel = product(x_dims); PADDLE_ENFORCE_EQ(x_dims.size(), 2U, "Rank of Input(X) should be 2."); - int dimension = ctx->Attrs().Get("new_dim"); - ctx->SetOutputDim("Out", {x_dims[0], static_cast(dimension)}); + int new_dim = ctx->Attrs().Get("new_dim"); + ctx->SetOutputDim("Out", + {x_numel / new_dim, static_cast(new_dim)}); } }; @@ -54,16 +57,16 @@ example will help to illustrate the function of this operator: x is a LoDTensor: x.lod = [[0, 2, 6]] - x.data = [[0.1, 0.2], [0.3, 0.4], - [0.5, 0.6], [0.7, 0.8], [0.9, 1.0], [1.1, 1.2]] + x.data = [[1, 2], [3, 4], + [5, 6], [7, 8], [9, 10], [11, 12]] x.dims = [6, 2] set new_dim = 4 then out is a LoDTensor: - out.lod = [[0, 1, 3]] - out.data = [[0.1, 0.2, 0.3, 0.4], - [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]] + out.lod = [[0, 1, 3]] + out.data = [[1, 2, 3, 4], + [5, 6, 7, 8], [9, 10, 11, 12]] out.dims = [3, 4] Currently, only 1-level LoDTensor is supported and please make sure (original @@ -82,8 +85,6 @@ class SequenceReshapeGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE( ctx->HasInput(framework::GradVarName("Out")), "Input(Out@GRAD) of SequenceReshapeGradOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("Out"), - "Input(Out) of SequenceReshapeGradOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of SequenceReshapeGradOp should not be null."); @@ -101,7 +102,6 @@ class SequenceReshapeGradOpMaker : public framework::SingleGradOpDescMaker { auto* op_desc_ptr = new framework::OpDesc(); op_desc_ptr->SetType("sequence_reshape_grad"); op_desc_ptr->SetInput("X", Input("X")); - op_desc_ptr->SetInput("Out", Output("Out")); op_desc_ptr->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); op_desc_ptr->SetOutput(framework::GradVarName("X"), InputGrad("X")); op_desc_ptr->SetAttrMap(Attrs()); @@ -118,7 +118,13 @@ REGISTER_OPERATOR(sequence_reshape, ops::SequenceReshapeOp, REGISTER_OPERATOR(sequence_reshape_grad, ops::SequenceReshapeGradOp); REGISTER_OP_CPU_KERNEL( sequence_reshape, - ops::SequenceReshapeKernel); + ops::SequenceReshapeKernel, + ops::SequenceReshapeKernel, + ops::SequenceReshapeKernel, + ops::SequenceReshapeKernel); REGISTER_OP_CPU_KERNEL( sequence_reshape_grad, - ops::SequenceReshapeGradKernel); + ops::SequenceReshapeGradKernel, + ops::SequenceReshapeGradKernel, + ops::SequenceReshapeGradKernel, + ops::SequenceReshapeGradKernel); diff --git a/paddle/operators/sequence_reshape_op.cu b/paddle/operators/sequence_reshape_op.cu index 9ba0e34e27..d9c2f7e9a4 100644 --- a/paddle/operators/sequence_reshape_op.cu +++ b/paddle/operators/sequence_reshape_op.cu @@ -17,7 +17,14 @@ limitations under the License. */ namespace ops = paddle::operators; REGISTER_OP_CUDA_KERNEL( sequence_reshape, - ops::SequenceReshapeKernel); + ops::SequenceReshapeKernel, + ops::SequenceReshapeKernel, + ops::SequenceReshapeKernel, + ops::SequenceReshapeKernel); REGISTER_OP_CUDA_KERNEL( sequence_reshape_grad, - ops::SequenceReshapeGradKernel); + ops::SequenceReshapeGradKernel, + ops::SequenceReshapeGradKernel, + ops::SequenceReshapeGradKernel, + ops::SequenceReshapeGradKernel); diff --git a/paddle/operators/sequence_reshape_op.h b/paddle/operators/sequence_reshape_op.h index 7c2215f772..623904ec7c 100644 --- a/paddle/operators/sequence_reshape_op.h +++ b/paddle/operators/sequence_reshape_op.h @@ -28,8 +28,6 @@ class SequenceReshapeKernel : public framework::OpKernel { auto* out = context.Output("Out"); int out_width = context.Attr("new_dim"); - const T* p_in_data = in->data(); - auto in_dims = in->dims(); int64_t in_width = in_dims[1]; auto& in_lod = in->lod(); @@ -43,53 +41,29 @@ class SequenceReshapeKernel : public framework::OpKernel { auto in_lod_l0 = in_lod[0]; int seq_num = in_lod_l0.size() - 1; - auto& out_lod = *out->mutable_lod(); - out_lod.resize(1); - out_lod[0].clear(); - out_lod[0].push_back(0); - for (int i = 0; i < seq_num; ++i) { - size_t seq_len = in_lod_l0[i + 1] - in_lod_l0[i]; - size_t offset = 0; - offset = (seq_len * in_width) / out_width; - PADDLE_ENFORCE_EQ(offset * out_width, seq_len * in_width, - "Please make sure (sequence_length * dimension) can be " - "divided by new_dim with no remainder for each " - "sequence. The %dth sequence is invalid.", - i + 1); - PADDLE_ENFORCE_GT(offset, 0, - "Illegal operation, length of the %dth sequence become " - "to 0 after reshaped.", - i + 1); - out_lod[0].push_back(out_lod[0].back() + offset); + if (in_width == out_width) { + out->set_lod(in->lod()); + } else { + auto& out_lod = *out->mutable_lod(); + out_lod.resize(1); + out_lod[0].clear(); + out_lod[0].push_back(0); + for (int i = 0; i < seq_num; ++i) { + size_t seq_len = in_lod_l0[i + 1] - in_lod_l0[i]; + size_t offset = 0; + offset = (seq_len * in_width) / out_width; + PADDLE_ENFORCE_EQ(offset * out_width, seq_len * in_width, + "Please make sure (sequence_length * dimension) can " + "be divided by new_dim with no remainder for each " + "sequence. The %dth sequence is invalid.", + i + 1); + out_lod[0].push_back(out_lod[0].back() + offset); + } } out->mutable_data(context.GetPlace()); - out->Resize({static_cast(out_lod[0].back()), out_width}); - T* p_out_data = out->mutable_data(context.GetPlace()); - math::set_constant(context.device_context(), out, 0.0f); - - for (int i = 0; i < seq_num; ++i) { - size_t in_offset = in_lod_l0[i] * in_width; - size_t out_offset = out_lod[0][i] * out_width; - size_t in_count = (in_lod_l0[i + 1] - in_lod_l0[i]) * in_width; - size_t out_count = (out_lod[0][i + 1] - out_lod[0][i]) * out_width; - size_t bytes = sizeof(T) * std::min(in_count, out_count); - if (platform::is_cpu_place(context.GetPlace())) { - memory::Copy(boost::get(context.GetPlace()), - p_out_data + out_offset, - boost::get(context.GetPlace()), - p_in_data + in_offset, bytes); - } else { -#ifdef PADDLE_WITH_CUDA - auto& dev_ctx = - context.template device_context(); - memory::Copy(boost::get(context.GetPlace()), - p_out_data + out_offset, - boost::get(context.GetPlace()), - p_in_data + in_offset, bytes, dev_ctx.stream()); -#endif - } - } + framework::Copy(*in, context.GetPlace(), out); + out->Resize({static_cast(out->lod()[0].back()), out_width}); } }; @@ -98,45 +72,14 @@ class SequenceReshapeGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* x_tensor_ptr = context.Input("X"); - auto* out_tensor_ptr = context.Input("Out"); - auto* out_grad_tensor_ptr = + auto* outg_tensor_ptr = context.Input(framework::GradVarName("Out")); - auto* x_grad_tensor_ptr = + auto* xg_tensor_ptr = context.Output(framework::GradVarName("X")); - T* p_x_grad_data = x_grad_tensor_ptr->mutable_data(context.GetPlace()); - const T* p_out_grad_data = out_grad_tensor_ptr->data(); - - auto& x_lod = x_tensor_ptr->lod(); - int seq_num = x_lod[0].size() - 1; - int x_width = x_tensor_ptr->dims()[1]; - auto& out_lod = out_tensor_ptr->lod(); - int out_width = out_tensor_ptr->dims()[1]; - - math::set_constant(context.device_context(), x_grad_tensor_ptr, 0.0f); - - for (int i = 0; i < seq_num; ++i) { - size_t src_offset = out_lod[0][i] * out_width; - size_t dst_offset = x_lod[0][i] * x_width; - size_t src_count = (out_lod[0][i + 1] - out_lod[0][i]) * out_width; - size_t dst_count = (x_lod[0][i + 1] - x_lod[0][i]) * x_width; - size_t bytes = sizeof(T) * std::min(src_count, dst_count); - if (platform::is_cpu_place(context.GetPlace())) { - memory::Copy(boost::get(context.GetPlace()), - p_x_grad_data + dst_offset, - boost::get(context.GetPlace()), - p_out_grad_data + src_offset, bytes); - } else { -#ifdef PADDLE_WITH_CUDA - auto& dev_ctx = - context.template device_context(); - memory::Copy(boost::get(context.GetPlace()), - p_x_grad_data + dst_offset, - boost::get(context.GetPlace()), - p_out_grad_data + src_offset, bytes, dev_ctx.stream()); -#endif - } - } + xg_tensor_ptr->mutable_data(context.GetPlace()); + framework::Copy(*outg_tensor_ptr, context.GetPlace(), xg_tensor_ptr); + xg_tensor_ptr->Resize(x_tensor_ptr->dims()); } }; diff --git a/python/paddle/v2/fluid/tests/test_sequence_reshape.py b/python/paddle/v2/fluid/tests/test_sequence_reshape.py index 91ff275821..857b15237a 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_reshape.py +++ b/python/paddle/v2/fluid/tests/test_sequence_reshape.py @@ -40,14 +40,7 @@ class TestSequenceReshape(OpTest): assert int(offset) * dimension == seq_len * x_width out_lod[0].append(out_lod[0][-1] + int(offset)) out = np.zeros(shape=(out_lod[0][-1], dimension)).astype('float32') - for i in xrange(len(x_lod[0]) - 1): - x_offset = x_lod[0][i] * x_width - out_offset = out_lod[0][i] * dimension - out_count = (out_lod[0][i + 1] - out_lod[0][i]) * dimension - x_count = (x_lod[0][i + 1] - x_lod[0][i]) * x_width - count = min(out_count, x_count) - out.ravel()[out_offset:out_offset + count] = x.ravel()[ - x_offset:x_offset + count] + out.ravel()[:] = x.ravel()[:] return out, out_lod def test_check_output(self): @@ -72,5 +65,20 @@ class TestSequenceReshape_reduce(TestSequenceReshape): self.outputs = {'Out': (out, out_lod)} +class TestSequenceReshape_same(TestSequenceReshape): + def setUp(self): + self.op_type = 'sequence_reshape' + dimension = 12 + x_lod = [[0, 4, 6, 8, 12]] + x = np.random.uniform(0.1, 1, [12, 12]).astype('float32') + + self.inputs = {'X': (x, x_lod)} + self.attrs = {'new_dim': dimension} + + out, out_lod = self.compute_output(x, x_lod, dimension) + + self.outputs = {'Out': (out, out_lod)} + + if __name__ == '__main__': unittest.main() From fccab36e1a3fbee42faad525ab2efb429ffd45ff Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Fri, 19 Jan 2018 13:15:50 +0800 Subject: [PATCH 115/158] "fix python encode error" (#7613) --- .copyright.hook | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.copyright.hook b/.copyright.hook index 2446e27248..6bfe58cebc 100644 --- a/.copyright.hook +++ b/.copyright.hook @@ -77,10 +77,13 @@ def lang_type(filename): elif filename.endswith(".proto"): return "C" else: - print("Unsupported filetype") + print("Unsupported filetype %s", filename) exit(0) +PYTHON_ENCODE = re.compile("^[ \t\v]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)") + + def main(argv=None): parser = argparse.ArgumentParser( description='Checker for copyright declaration.') @@ -89,8 +92,15 @@ def main(argv=None): retv = 0 for filename in args.filenames: - first_line = io.open(filename).readline() - if "COPYRIGHT" in first_line.upper() : continue + fd = io.open(filename) + first_line = fd.readline() + if "COPYRIGHT" in first_line.upper(): continue + if filename.endswith(".py"): + second_line = fd.readline() + if first_line.startswith("#!") or PYTHON_ENCODE.match( + second_line) != None or PYTHON_ENCODE.match( + first_line) != None: + continue original_contents = io.open(filename).read() new_contents = generate_copyright( COPYRIGHT, lang_type(filename)) + original_contents From 9e17c46c351cbff14e07811182e20b8e0fe559a3 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Fri, 19 Jan 2018 13:19:00 +0800 Subject: [PATCH 116/158] add new_op_kernel_en doc (#7681) * add new_op_kernel_en.md --- doc/howto/dev/new_op_kernel_en.md | 121 ++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 doc/howto/dev/new_op_kernel_en.md diff --git a/doc/howto/dev/new_op_kernel_en.md b/doc/howto/dev/new_op_kernel_en.md new file mode 100644 index 0000000000..123df0a7ee --- /dev/null +++ b/doc/howto/dev/new_op_kernel_en.md @@ -0,0 +1,121 @@ +## Add Kernels for a New Device + +### Background + +PaddlePaddle Fluid have hundreds of operators. Each operator could have one or more kernels. A kernel is an implementation of the operator for a certain device, which could be a hardware device, e.g., the CUDA GPU, or a library that utilizes a device, e.g., Intel MKL that makes full use of the Xeon CPU. + +[This document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/new_op_en.md) explains how to add an operator, and its kernels. The kernels of an operator are indexed by a C++ type [`OpKernelType`](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/operator_kernel_type.md). An operator chooses the right kernel at runtime. This choosing mechanism is described [here](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/switch_kernel.md). + +### Write Kernels for A New Device + +#### Add A New Device + + For some historical reaons, we misuse the word *library* for *device*. For example, we call the deivce type by *library type*. An example is the header file [`library_type.h`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/library_type.h#L24). We will correct this ASAP. + +To register a new device, we need to add an enum value to `LibraryType`: + +``` +enum class LibraryType { + kPlain = 0, + kMKLDNN = 1, + kCUDNN = 2, +}; +``` + + +#### Add A New [Place](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/place.h#L53) + +If you have a new kind of Device, firstly you need to add a new kind of [`Place`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/place.h#L53). For example `CUDAPlace`: + +```cpp +struct CUDAPlace { + CUDAPlace() : CUDAPlace(0) {} + explicit CUDAPlace(int d) : device(d) {} + + inline int GetDeviceId() const { return device; } + // needed for variant equality comparison + inline bool operator==(const CUDAPlace &o) const { + return device == o.device; + } + inline bool operator!=(const CUDAPlace &o) const { return !(*this == o); } + + int device; +}; + +typedef boost::variant Place; +``` + +#### Add [device context]((https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/device_context.h#L37)) +After a new kind of Device is added, you should add a corresponding [DeviceContext](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/platform/device_context.h#L37) for it. + +```cpp +class DeviceContext { + public: + virtual ~DeviceContext() {} + virtual Place GetPlace() const = 0; + + virtual void Wait() const {} +}; +``` + +#### Implement new [OpKernel](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L351) for your Device. + +A detailed documentation can be found in [`new_op_and_kernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/new_op_en.md) + +```cpp +class OpKernelBase { + public: + /** + * ExecutionContext is the only parameter of Kernel Run function. + * Run will get input/output variables, state such as momentum and + * device resource such as CUDA stream, cublas handle, etc. from + * ExecutionContext. User should construct it before run the Operator. + */ + + virtual void Compute(const ExecutionContext& context) const = 0; + + virtual ~OpKernelBase() = default; +}; + +template +class OpKernel : public OpKernelBase { + public: + using ELEMENT_TYPE = T; +}; +``` + + +#### Register the OpKernel to framework + +After writing the components described above, we should register the kernel to the framework. + +We use `REGISTER_OP_KERNEL` to do the registration. + +```cpp +REGISTER_OP_KERNEL( + op_type, + library_type, + place_type, + kernel0, kernel1, ...) +``` + +kernel0, kernel1 are kernels that have the same `op_type`, `library_type`, `place_type` but different `data_types`. + +take [`conv2d`]((https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/conv_cudnn_op.cu.cc#L318)) as an example: + + ```cpp + REGISTER_OP_KERNEL(conv2d, CPU, paddle::platform::CPUPlace, + paddle::operators::GemmConvKernel, + paddle::operators::GemmConvKernel); + + REGISTER_OP_KERNEL(conv2d, CUDNN, ::paddle::platform::CUDAPlace, + paddle::operators::CUDNNConvOpKernel, + paddle::operators::CUDNNConvOpKernel); + ``` + +In the code above: + + - `conv2d` is the type/name of the operator + - `CUDNN/CPU` is `library` + - `paddle::platform::CUDAPlace/CPUPlace` is `place` + - template parameter `float/double` on `CUDNNConvOpKernel` is `data_type`. From 0071b5f7978e25143be00f3eeb87c6edbf6b17e3 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Fri, 19 Jan 2018 13:20:18 +0800 Subject: [PATCH 117/158] complete data layout transform (#7440) * add data layout transform and optimize the implementation of data_transform --- paddle/framework/CMakeLists.txt | 7 ++- .../framework/data_device_transform_test.cu | 1 + paddle/framework/data_layout_transform.cc | 51 +++++++++++-------- paddle/framework/data_layout_transform.h | 12 ++--- .../framework/data_layout_transform_test.cc | 44 ++++++++++++++++ paddle/framework/data_transform.cc | 31 +++++++++-- paddle/framework/op_kernel_type.h | 7 ++- 7 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 paddle/framework/data_layout_transform_test.cc diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 597ea959f2..8e5a956061 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -33,8 +33,13 @@ cc_library(scope SRCS scope.cc DEPS glog threadpool) cc_test(scope_test SRCS scope_test.cc DEPS scope) cc_library(data_device_transform SRCS data_device_transform.cc DEPS tensor) +nv_test(data_device_transform_test SRCS data_device_transform_test.cu + DEPS operator op_registry init math_function) + cc_library(data_type_transform SRCS data_type_transform.cc DEPS tensor) + cc_library(data_layout_transform SRCS data_layout_transform.cc DEPS tensor math_function) +cc_test(data_layout_transform_test SRCS data_layout_transform_test.cc DEPS data_layout_transform) cc_library(data_transform SRCS data_transform.cc DEPS math_function tensor framework_proto selected_rows data_device_transform data_type_transform data_layout_transform) @@ -82,5 +87,3 @@ cc_test(init_test SRCS init_test.cc DEPS init) cc_test(op_kernel_type_test SRCS op_kernel_type_test.cc DEPS place device_context framework_proto) cc_test(cow_ptr_tests SRCS details/cow_ptr_test.cc) -nv_test(data_device_transform_test SRCS data_device_transform_test.cu - DEPS operator op_registry init math_function) diff --git a/paddle/framework/data_device_transform_test.cu b/paddle/framework/data_device_transform_test.cu index 5d89f5546f..efc05b3106 100644 --- a/paddle/framework/data_device_transform_test.cu +++ b/paddle/framework/data_device_transform_test.cu @@ -150,6 +150,7 @@ TEST(Operator, CPUtoGPU) { // get output auto* output2 = scope.Var("OUT2"); gpu_op->Run(scope, cuda_place); + VLOG(3) << "after gpu_op run"; // auto* output2_ptr = output2->Get().data(); DeviceContextPool& pool = DeviceContextPool::Instance(); diff --git a/paddle/framework/data_layout_transform.cc b/paddle/framework/data_layout_transform.cc index 96794cae97..1059bd9761 100644 --- a/paddle/framework/data_layout_transform.cc +++ b/paddle/framework/data_layout_transform.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,12 +14,23 @@ limitations under the License. */ #include "paddle/framework/data_layout_transform.h" -#include "paddle/framework/tensor.h" #include "paddle/operators/math/math_function.h" namespace paddle { namespace framework { +std::vector GetAxis(const DataLayout& from, const DataLayout& to) { + PADDLE_ENFORCE_NE(from, to, + "layout transform should transform different layout"); + if (from == DataLayout::kNCHW && to == DataLayout::kNHWC) { + return {0, 2, 3, 1}; + } else if (from == DataLayout::kNHWC && to == DataLayout::kNCHW) { + return {0, 3, 1, 2}; + } else { + PADDLE_THROW("unsupported transform"); + } +} + struct CastDataLayout { CastDataLayout(const platform::DeviceContext* ctx, const std::vector& axis, const framework::Tensor& in, @@ -44,38 +55,36 @@ struct CastDataLayout { } }; -void TransDataLayout(const std::vector& axis, - const platform::DeviceContext* ctx, - const KernelTypePair& kernel_pair, const Variable& in, - Variable* out) { - PADDLE_ENFORCE(in.IsType(), "Only support Tensor transform!."); +void TransDataLayout(const OpKernelType& kernel_type_for_var, + const OpKernelType& expected_kernel_type, const Tensor& in, + Tensor* out) { PADDLE_ENFORCE( - platform::places_are_same_class(kernel_pair.first.place_, - kernel_pair.second.place_), + platform::places_are_same_class(kernel_type_for_var.place_, + expected_kernel_type.place_), "TransDataLayout only support DataLayout transform on same place!"); - PADDLE_ENFORCE(kernel_pair.first.data_type_ == kernel_pair.second.data_type_, - "TransDataLayout only support Datatype are same!"); - auto src = in.Get(); - auto* dst = out->GetMutable(); - PADDLE_ENFORCE(arity(src.dims()) == 4, "Input Arity Only Suppport 4!"); + PADDLE_ENFORCE(arity(in.dims()) == 4, "Input Arity only support 4!"); + + auto& pool = platform::DeviceContextPool::Instance(); - auto src_dim = src.dims(); + auto src_dim = in.dims(); std::vector dst_dim; + auto axis = GetAxis(kernel_type_for_var.data_layout_, + expected_kernel_type.data_layout_); dst_dim.resize(axis.size()); for (size_t i = 0; i < axis.size(); i++) { dst_dim[i] = src_dim[axis[i]]; } - dst->Resize(make_ddim(dst_dim)); - auto place = kernel_pair.second.place_; - dst->mutable_data(place, src.type()); + out->Resize(make_ddim(dst_dim)); + out->mutable_data(expected_kernel_type.place_, in.type()); - auto src_type = kernel_pair.first.data_type_; - framework::VisitDataType(src_type, CastDataLayout(ctx, axis, src, dst)); + framework::VisitDataType( + framework::ToDataType(in.type()), + CastDataLayout(pool.Get(expected_kernel_type.place_), axis, in, out)); - dst->set_layout(kernel_pair.second.data_layout_); + out->set_layout(expected_kernel_type.data_layout_); } } // namespace framework diff --git a/paddle/framework/data_layout_transform.h b/paddle/framework/data_layout_transform.h index befae1f636..ec87257d70 100644 --- a/paddle/framework/data_layout_transform.h +++ b/paddle/framework/data_layout_transform.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,17 +15,17 @@ limitations under the License. */ #pragma once #include "paddle/framework/op_kernel_type.h" +#include "paddle/framework/tensor.h" #include "paddle/framework/variable.h" namespace paddle { namespace framework { -using KernelTypePair = std::pair; +std::vector GetAxis(const DataLayout& from, const DataLayout& to); -void TransDataLayout(const std::vector& axis, - const platform::DeviceContext* ctx, - const KernelTypePair& kernel_pair, const Variable& in, - Variable* out); +void TransDataLayout(const OpKernelType& kernel_type_for_var, + const OpKernelType& expected_kernel_type, const Tensor& in, + Tensor* out); } // namespace framework } // namespace paddle diff --git a/paddle/framework/data_layout_transform_test.cc b/paddle/framework/data_layout_transform_test.cc new file mode 100644 index 0000000000..2eb99fa04a --- /dev/null +++ b/paddle/framework/data_layout_transform_test.cc @@ -0,0 +1,44 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/data_layout_transform.h" + +#include "gtest/gtest.h" +#include "paddle/platform/device_context.h" + +TEST(DataTransform, DataLayoutFunction) { + using namespace paddle::framework; + using namespace paddle::platform; + + auto place = CPUPlace(); + Tensor in = Tensor(); + Tensor out = Tensor(); + in.mutable_data(make_ddim({2, 3, 1, 2}), place); + in.set_layout(DataLayout::kNHWC); + + auto kernel_nhwc = OpKernelType(proto::DataType::FP32, place, + DataLayout::kNHWC, LibraryType::kPlain); + auto kernel_ncwh = OpKernelType(proto::DataType::FP32, place, + DataLayout::kNCHW, LibraryType::kPlain); + + TransDataLayout(kernel_nhwc, kernel_ncwh, in, &out); + + EXPECT_TRUE(out.layout() == DataLayout::kNCHW); + EXPECT_TRUE(out.dims() == make_ddim({2, 2, 3, 1})); + + TransDataLayout(kernel_ncwh, kernel_nhwc, in, &out); + + EXPECT_TRUE(in.layout() == DataLayout::kNHWC); + EXPECT_TRUE(in.dims() == make_ddim({2, 3, 1, 2})); +} \ No newline at end of file diff --git a/paddle/framework/data_transform.cc b/paddle/framework/data_transform.cc index d826f0edac..e28b2e015d 100644 --- a/paddle/framework/data_transform.cc +++ b/paddle/framework/data_transform.cc @@ -15,18 +15,43 @@ limitations under the License. */ #include "paddle/framework/data_transform.h" #include "paddle/framework/data_device_transform.h" +#include "paddle/framework/data_layout_transform.h" namespace paddle { namespace framework { +static void PassTensorData(Tensor* from, Tensor* to) { + to->ShareDataWith(*from); + *from = Tensor(); +} + void DataTransform(const OpKernelType& expected_kernel_type, const OpKernelType& kernel_type_for_var, - const Tensor& input_tensor, Tensor* out) { + const Tensor& input_tensor, Tensor* output_tensor) { + bool transformed = false; + Tensor in; + in.ShareDataWith(input_tensor); + Tensor out; + + // do layout transform + if (NeedTransformLayout(expected_kernel_type.data_layout_, + kernel_type_for_var.data_layout_)) { + TransDataLayout(kernel_type_for_var, expected_kernel_type, in, &out); + transformed = true; + PassTensorData(&out, &in); + } + + // do device transform if (!platform::is_same_place(kernel_type_for_var.place_, expected_kernel_type.place_)) { - DeviceTransform(input_tensor, expected_kernel_type.place_, out); + DeviceTransform(in, expected_kernel_type.place_, &out); + transformed = true; + PassTensorData(&out, &in); } - PADDLE_ENFORCE_NOT_NULL(out, "out should not be null"); + + PADDLE_ENFORCE(transformed, "no transform is done, please check!"); + // get output data + output_tensor->ShareDataWith(in); } void CopyVariableWithTensor(const Variable& in_var, const Tensor& tensor, diff --git a/paddle/framework/op_kernel_type.h b/paddle/framework/op_kernel_type.h index 312bd5f892..44adb94d2a 100644 --- a/paddle/framework/op_kernel_type.h +++ b/paddle/framework/op_kernel_type.h @@ -85,9 +85,14 @@ inline std::string KernelTypeToString(const OpKernelType& kernel_key) { return stream.str(); } +inline bool NeedTransformLayout(const DataLayout& l, const DataLayout& r) { + return l != DataLayout::kAnyLayout && r != DataLayout::kAnyLayout && l != r; +} + inline bool TransFromNeeded(const OpKernelType& l, const OpKernelType& r) { return (!platform::places_are_same_class(l.place_, r.place_)) || - (l.data_type_ != r.data_type_) || (l.data_layout_ != r.data_layout_); + (l.data_type_ != r.data_type_) || + NeedTransformLayout(l.data_layout_, r.data_layout_); } } // namespace framework From 541b42e6fb7a5f4adaaad96251659e3bc9591b9d Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 19 Jan 2018 13:56:42 +0800 Subject: [PATCH 118/158] fix style --- benchmark/cluster/v2/trainer.yaml | 2 +- benchmark/cluster/v2/vgg16.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark/cluster/v2/trainer.yaml b/benchmark/cluster/v2/trainer.yaml index 33c95df365..75fffc64b0 100644 --- a/benchmark/cluster/v2/trainer.yaml +++ b/benchmark/cluster/v2/trainer.yaml @@ -28,7 +28,7 @@ spec: - name: TOPOLOGY value: "" - name: ENTRY - value: "cd /workspace && python /workspace/vgg16.py" + value: "cd /workspace && MKL_NUM_THREADS=1 python /workspace/vgg16.py" - name: TRAINER_PACKAGE value: "/workspace" - name: PADDLE_INIT_PORT diff --git a/benchmark/cluster/v2/vgg16.py b/benchmark/cluster/v2/vgg16.py index 9189493276..59e3997d78 100644 --- a/benchmark/cluster/v2/vgg16.py +++ b/benchmark/cluster/v2/vgg16.py @@ -144,4 +144,3 @@ def main(): if __name__ == '__main__': main() - From b48fedc30bc9565e1a4a8a6942c3e1afb49d2a0d Mon Sep 17 00:00:00 2001 From: peterzhang2029 Date: Fri, 19 Jan 2018 14:21:03 +0800 Subject: [PATCH 119/158] init --- paddle/operators/scale_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index f634ebe9a2..c0e614743a 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -48,7 +48,7 @@ Scale operator $$Out = scale*X$$ )DOC"); AddAttr("scale", - "(float, default 0)" + "(float, default 1.0)" "The scaling factor of the scale operator.") .SetDefault(1.0); } From 9f731a608b66fc10924fcdc02a47f329e29819ad Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Fri, 19 Jan 2018 14:39:45 +0800 Subject: [PATCH 120/158] Add math operator patches Users can use `a+b`, `a*10`. --- python/paddle/v2/fluid/__init__.py | 2 + python/paddle/v2/fluid/layers/__init__.py | 3 + .../paddle/v2/fluid/layers/math_op_patch.py | 161 ++++++++++++++++ .../v2/fluid/tests/test_math_op_patch.py | 181 ++++++++++++++++++ 4 files changed, 347 insertions(+) create mode 100644 python/paddle/v2/fluid/layers/math_op_patch.py create mode 100644 python/paddle/v2/fluid/tests/test_math_op_patch.py diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index 8c29ee741c..c190f68a86 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -36,6 +36,7 @@ import clip from memory_optimization_transpiler import memory_optimize Tensor = LoDTensor + __all__ = framework.__all__ + executor.__all__ + [ 'io', 'initializer', @@ -93,4 +94,5 @@ def __bootstrap__(): core.init_devices() +layers.monkey_patch_variable() __bootstrap__() diff --git a/python/paddle/v2/fluid/layers/__init__.py b/python/paddle/v2/fluid/layers/__init__.py index c190af3329..b578adc894 100644 --- a/python/paddle/v2/fluid/layers/__init__.py +++ b/python/paddle/v2/fluid/layers/__init__.py @@ -23,6 +23,8 @@ import control_flow from control_flow import * import device from device import * +import math_op_patch +from math_op_patch import * __all__ = [] __all__ += nn.__all__ @@ -31,3 +33,4 @@ __all__ += tensor.__all__ __all__ += control_flow.__all__ __all__ += ops.__all__ __all__ += device.__all__ +__all__ += math_op_patch.__all__ diff --git a/python/paddle/v2/fluid/layers/math_op_patch.py b/python/paddle/v2/fluid/layers/math_op_patch.py new file mode 100644 index 0000000000..5efc116db2 --- /dev/null +++ b/python/paddle/v2/fluid/layers/math_op_patch.py @@ -0,0 +1,161 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..framework import Variable, unique_name +from ..registry import OpProtoHolder + +__all__ = ['monkey_patch_variable'] + + +def monkey_patch_variable(): + def new_name(): + return unique_name("tmp") + + def safe_get_dtype(var): + try: + dtype = var.dtype + except: + raise ValueError("Cannot get data type from %s", var.name) + return dtype + + def create_scalar(block, value, dtype): + value = float(value) + tmp_name = new_name() + var = block.create_var(name=tmp_name, shape=[1], dtype=dtype) + block.append_op( + type="fill", + outputs={"Out": [var]}, + attrs={"value": [value], + "shape": [1], + "dtype": dtype}) + return var + + def create_tensor(block, value, dtype, shape): + value = float(value) + tmp_name = new_name() + var = block.create_var(name=tmp_name, shape=shape, dtype=dtype) + block.append_op( + type="fill_constant", + outputs={'Out': [var]}, + attrs={'dtype': var.dtype, + 'shape': shape, + 'value': value}) + return var + + def create_tensor_with_batchsize(ref_var, value, dtype): + assert isinstance(ref_var, Variable) + value = float(value) + tmp_name = new_name() + var = ref_var.block.create_var(name=tmp_name, dtype=dtype) + ref_var.block.append_op( + type='fill_constant_batch_size_like', + outputs={'Out': [var]}, + inputs={'Input': [ref_var]}, + attrs={'shape': ref_var.shape, + 'value': value}) + return var + + def astype(self, dtype): + """ + Cast a variable to data type. + NOTE: The variable must be a Tensor + Args: + self(Variable): The source variable + dtype: The target dtype + + Returns: + Variable with new dtype + """ + tmp_name = new_name() + out = self.block.create_var(name=tmp_name, dtype=dtype) + self.block.append_op( + type="cast", + inputs={"X": [self]}, + outputs={"Out": [out]}, + attrs={"in_dtype": self.dtype, + "out_dtype": out.dtype}) + return out + + def _elemwise_method_creator_(method_name, op_type, reverse=False): + def __impl__(self, other_var): + lhs_dtype = safe_get_dtype(self) + + if not isinstance(other_var, Variable): + if reverse: + has_batch_size = False + for elem in self.shape: + if elem < 0: + has_batch_size = True + break + if not has_batch_size: + other_var = create_tensor( + self.block, + other_var, + dtype=lhs_dtype, + shape=self.shape) + else: + other_var = create_tensor_with_batchsize( + self, other_var, lhs_dtype) + else: + # add fill_op to self.block + other_var = create_scalar( + self.block, value=other_var, dtype=lhs_dtype) + + rhs_dtype = safe_get_dtype(other_var) + if lhs_dtype != rhs_dtype: + other_var = astype(other_var, lhs_dtype) + if reverse: + tmp = self + self = other_var + other_var = tmp + + tmp_name = new_name() + out = self.block.create_var(name=tmp_name, dtype=lhs_dtype) + self.block.append_op( + type=op_type, + inputs={'X': [self], + 'Y': [other_var]}, + outputs={'Out': out}) + return out + + comment = OpProtoHolder.instance().get_op_proto(op_type).comment + + __impl__.__doc__ = """ + {0} + Args: + self(Variable): left hand variable + other_var(Variable|float|int): right hand variable + + Returns: + Variable + """.format(comment) + __impl__.__name__ = method_name + return __impl__ + + # inject methods + for method_name, op_type, reverse in ( + ("__add__", "elementwise_add", False), + # a+b == b+a. Do not need to reverse explicitly + ("__radd__", "elementwise_add", False), + ("__sub__", "elementwise_sub", False), + ("__rsub__", "elementwise_sub", True), + ("__mul__", "elementwise_mul", False), + # a*b == b*a. Do not need to reverse explicitly + ("__rmul__", "elementwise_mul", False), + ("__div__", "elementwise_div", False), + ("__rdiv__", "elementwise_div", True)): + setattr(Variable, method_name, + _elemwise_method_creator_(method_name, op_type, reverse)) + + Variable.astype = astype diff --git a/python/paddle/v2/fluid/tests/test_math_op_patch.py b/python/paddle/v2/fluid/tests/test_math_op_patch.py new file mode 100644 index 0000000000..2e77639a4c --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_math_op_patch.py @@ -0,0 +1,181 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import decorators +import paddle.v2.fluid as fluid +import numpy + + +class TestMathOpPatches(unittest.TestCase): + @decorators.prog_scope() + def test_add_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = a + 10 + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(a_np + 10, b_np)) + + @decorators.prog_scope() + def test_radd_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = 10 + a + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(a_np + 10, b_np)) + + @decorators.prog_scope() + def test_sub_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = a - 10 + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(a_np - 10, b_np)) + + @decorators.prog_scope() + def test_radd_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = 10 - a + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(10 - a_np, b_np)) + + @decorators.prog_scope() + def test_mul_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = a * 10 + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(a_np * 10, b_np)) + + @decorators.prog_scope() + def test_rmul_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = 10 * a + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(10 * a_np, b_np)) + + @decorators.prog_scope() + def test_div_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = a / 10 + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(a_np / 10, b_np)) + + @decorators.prog_scope() + def test_rdiv_scalar(self): + a = fluid.layers.data(name="a", shape=[1]) + b = 10 / a + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + 1e-2 + + b_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b]) + self.assertTrue(numpy.allclose(10 / a_np, b_np)) + + @decorators.prog_scope() + def test_div_two_tensor(self): + a = fluid.layers.data(name="a", shape=[1]) + b = fluid.layers.data(name="b", shape=[1]) + c = a / b + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = numpy.random.random(size=[10, 1]).astype('float32') + 1e-2 + c_np = exe.run(fluid.default_main_program(), + feed={"a": a_np, + 'b': b_np}, + fetch_list=[c]) + self.assertTrue(numpy.allclose(a_np / b_np, c_np)) + + @decorators.prog_scope() + def test_mul_two_tensor(self): + a = fluid.layers.data(name="a", shape=[1]) + b = fluid.layers.data(name="b", shape=[1]) + c = a * b + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = numpy.random.random(size=[10, 1]).astype('float32') + c_np = exe.run(fluid.default_main_program(), + feed={"a": a_np, + 'b': b_np}, + fetch_list=[c]) + self.assertTrue(numpy.allclose(a_np * b_np, c_np)) + + @decorators.prog_scope() + def test_add_two_tensor(self): + a = fluid.layers.data(name="a", shape=[1]) + b = fluid.layers.data(name="b", shape=[1]) + c = a + b + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = numpy.random.random(size=[10, 1]).astype('float32') + c_np = exe.run(fluid.default_main_program(), + feed={"a": a_np, + 'b': b_np}, + fetch_list=[c]) + self.assertTrue(numpy.allclose(a_np + b_np, c_np)) + + @decorators.prog_scope() + def test_sub_two_tensor(self): + a = fluid.layers.data(name="a", shape=[1]) + b = fluid.layers.data(name="b", shape=[1]) + c = a - b + place = fluid.CPUPlace() + exe = fluid.Executor(place) + a_np = numpy.random.random(size=[10, 1]).astype('float32') + b_np = numpy.random.random(size=[10, 1]).astype('float32') + c_np = exe.run(fluid.default_main_program(), + feed={"a": a_np, + 'b': b_np}, + fetch_list=[c]) + self.assertTrue(numpy.allclose(a_np - b_np, c_np)) + + +if __name__ == '__main__': + unittest.main() From 50ac67fc63682f7a1f458857cb382f83b1368530 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Fri, 19 Jan 2018 14:59:36 +0800 Subject: [PATCH 121/158] Bugfix/check if kernel for type exist (#7657) * check if kernel if found for kernel type * do kernel check before data transform --- paddle/framework/operator.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index 84c010df7c..831b1e2a1e 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -485,9 +485,15 @@ void OperatorWithKernel::Run(const Scope& scope, // } auto expected_kernel_key = this->GetExpectedKernelType(ctx); - VLOG(3) << "expected_kernel_key:" << expected_kernel_key; + auto kernel_iter = kernels.find(expected_kernel_key); + if (kernel_iter == kernels.end()) { + PADDLE_THROW("op %s does not have kernel for %s", type_, + KernelTypeToString(expected_kernel_key)); + } + + // do data transform Scope& new_scope = scope.NewScope(); for (auto& var_name_item : this->Inputs()) { @@ -520,8 +526,6 @@ void OperatorWithKernel::Run(const Scope& scope, } } - auto kernel_iter = kernels.find(expected_kernel_key); - auto* new_dev_ctx = pool.Get(expected_kernel_key.place_); kernel_iter->second->Compute( ExecutionContext(*this, new_scope, *new_dev_ctx)); From 42b0748ab4f797902cadad4b5278a4cb9fdea9bd Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 Jan 2018 15:12:39 +0800 Subject: [PATCH 122/158] add unittest --- python/paddle/v2/fluid/clip.py | 14 +++- .../{test_clip.py => test_error_clip.py} | 0 .../v2/fluid/tests/test_gradient_clip.py | 82 +++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) rename python/paddle/v2/fluid/tests/{test_clip.py => test_error_clip.py} (100%) create mode 100644 python/paddle/v2/fluid/tests/test_gradient_clip.py diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index d4f025a4af..f6ff83924f 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -113,6 +113,7 @@ class GradientClipByNorm(BaseGradientClipAttr): class GradientClipByGlobalNorm(BaseGradientClipAttr): global_norm_var = None + local_norm_var = None clip_norm_var = None scale_var = None @@ -123,12 +124,18 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): cls.global_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=0.0) + cls.local_norm_var = framework.default_main_program().current_block( + ).create_var( + name=framework.unique_name("local_norm"), + dtype="float32", + persistable=False) cls.clip_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=clip_norm) @classmethod def check_init(cls): if not (isinstance(cls.global_norm_var, framework.Variable) and + isinstance(cls.local_norm_var, framework.Variable) and isinstance(cls.clip_norm_var, framework.Variable)): raise ValueError( "Class 'GradientClipByGlobalNorm' has not been properly initialized. \ @@ -138,9 +145,10 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): cls = self.__class__ cls.check_init() - local_norm_var = layers.reduce_sum(input=layers.pow(x=grad, factor=2.0)) + cls.local_norm_var = layers.reduce_sum( + input=layers.pow(x=grad, factor=2.0)) layers.sums( - input=[local_norm_var, cls.global_norm_var], + input=[cls.local_norm_var, cls.global_norm_var], out=[cls.global_norm_var]) def create_operators(self, param, grad): @@ -148,7 +156,7 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): cls.check_init() if cls.scale_var is None: - cls.global_norm_var = layers.sqrt(x=cls.global_norm_var) + layers.sqrt(x=cls.global_norm_var, out=cls.global_norm_var) cls.scale_var = layers.elementwise_div( x=cls.clip_norm_var, y=layers.elementwise_max( diff --git a/python/paddle/v2/fluid/tests/test_clip.py b/python/paddle/v2/fluid/tests/test_error_clip.py similarity index 100% rename from python/paddle/v2/fluid/tests/test_clip.py rename to python/paddle/v2/fluid/tests/test_error_clip.py diff --git a/python/paddle/v2/fluid/tests/test_gradient_clip.py b/python/paddle/v2/fluid/tests/test_gradient_clip.py new file mode 100644 index 0000000000..4fb7f0b2cb --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_gradient_clip.py @@ -0,0 +1,82 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import paddle.v2 as paddle +import paddle.v2.fluid as fluid + + +def _get_global_param_norm_(params_grads): + res = fluid.layers.fill_constant(shape=[1], dtype="float32", value=0.0) + for _, grad in params_grads: + norm_var = fluid.layers.reduce_sum( + input=fluid.layers.pow(x=grad, factor=2.0)) + fluid.layers.sums(input=[norm_var, res], out=[res]) + fluid.layers.sqrt(x=res, out=res) + return res + + +BATCH_SIZE = 128 +CLIP = 0.5 +prog = fluid.framework.Program() + +with fluid.program_guard(main_program=prog): + image = fluid.layers.data(name='x', shape=[784], dtype='float32') + + hidden1 = fluid.layers.fc(input=image, size=128, act='relu') + hidden2 = fluid.layers.fc(input=hidden1, size=64, act='relu') + predict = fluid.layers.fc(input=hidden2, size=10, act='softmax') + + label = fluid.layers.data(name='y', shape=[1], dtype='int64') + + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) + +prog_clip = prog.clone() + +avg_cost_clip = prog_clip.block(0).var(avg_cost.name) + +p_g = fluid.backward.append_backward(loss=avg_cost) +p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip) + +with fluid.program_guard(main_program=prog): + gloabl_norm = _get_global_param_norm_(p_g) + +with fluid.program_guard(main_program=prog_clip): + fluid.clip.gradient_clip_by_global_norm(clip_norm=CLIP) + p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip) + gloabl_norm_clip = _get_global_param_norm_(p_g_clip) + +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=8192), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +exe = fluid.Executor(place) +feeder = fluid.DataFeeder(feed_list=[image, label], place=place) +exe.run(fluid.default_startup_program()) + +count = 0 +for data in train_reader(): + count += 1 + if count > 5: + break + out, = exe.run(prog, feed=feeder.feed(data), fetch_list=[gloabl_norm]) + out_clip, = exe.run(prog_clip, + feed=feeder.feed(data), + fetch_list=[gloabl_norm_clip]) + + if not np.allclose(out_clip, np.minimum(out, np.array([CLIP]))): + exit(1) +exit(0) From 408a6b8bb2af4f8f075680bb361daad329ad6eca Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 Jan 2018 15:17:35 +0800 Subject: [PATCH 123/158] tiny fix --- python/paddle/v2/fluid/clip.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index f6ff83924f..9800ad7c5d 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -124,11 +124,11 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): cls.global_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=0.0) - cls.local_norm_var = framework.default_main_program().current_block( - ).create_var( - name=framework.unique_name("local_norm"), - dtype="float32", - persistable=False) + cls.local_norm_var = framework.default_main_program().block( + 0).create_var( + name=framework.unique_name("local_norm"), + dtype="float32", + persistable=False) cls.clip_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=clip_norm) From 538f1ad28f766c0e47ef4eef2ec59e187ba30f8e Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 Jan 2018 15:24:14 +0800 Subject: [PATCH 124/158] tiny fix --- python/paddle/v2/fluid/clip.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index 9800ad7c5d..d97cd9ecc9 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -124,11 +124,7 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): cls.global_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=0.0) - cls.local_norm_var = framework.default_main_program().block( - 0).create_var( - name=framework.unique_name("local_norm"), - dtype="float32", - persistable=False) + cls.local_norm_var = layers.create_tensor(dtype="float32") cls.clip_norm_var = layers.fill_constant( shape=[1], dtype="float32", value=clip_norm) From b07ca1de1fa53a6cafa177b474bd6d79b3c1aef6 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 19 Jan 2018 16:03:24 +0800 Subject: [PATCH 126/158] resize before computing LoD. --- paddle/operators/sequence_reshape_op.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/paddle/operators/sequence_reshape_op.h b/paddle/operators/sequence_reshape_op.h index 623904ec7c..dd9b611250 100644 --- a/paddle/operators/sequence_reshape_op.h +++ b/paddle/operators/sequence_reshape_op.h @@ -46,8 +46,8 @@ class SequenceReshapeKernel : public framework::OpKernel { } else { auto& out_lod = *out->mutable_lod(); out_lod.resize(1); - out_lod[0].clear(); - out_lod[0].push_back(0); + out_lod[0].resize(seq_num + 1); + out_lod[0][0] = 0; for (int i = 0; i < seq_num; ++i) { size_t seq_len = in_lod_l0[i + 1] - in_lod_l0[i]; size_t offset = 0; @@ -57,11 +57,10 @@ class SequenceReshapeKernel : public framework::OpKernel { "be divided by new_dim with no remainder for each " "sequence. The %dth sequence is invalid.", i + 1); - out_lod[0].push_back(out_lod[0].back() + offset); + out_lod[0][i + 1] = out_lod[0][i] + offset; } } - out->mutable_data(context.GetPlace()); framework::Copy(*in, context.GetPlace(), out); out->Resize({static_cast(out->lod()[0].back()), out_width}); } From b4685675df63d7331e8215c36fff66eacde5da7e Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 19 Jan 2018 16:39:56 +0800 Subject: [PATCH 127/158] refine openblas install lib when CAPI --- cmake/external/openblas.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 0e79c0cc79..4012a164be 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -100,6 +100,11 @@ IF(NOT ${CBLAS_FOUND}) \"${CBLAS_INSTALL_DIR}/lib -> ${CMAKE_INSTALL_PREFIX}/${TMP_INSTALL_DIR}\" )" ) + INSTALL(CODE "execute_process( + COMMAND rm -r ${CMAKE_INSTALL_PREFIX}/${TMP_INSTALL_DIR}/cmake + ${CMAKE_INSTALL_PREFIX}/${TMP_INSTALL_DIR}/pkgconfig + )" + ) ENDIF() ENDIF(NOT ${CBLAS_FOUND}) From 95b896ce72ee603509fd65d3b2954210aa937c45 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 19 Jan 2018 16:42:51 +0800 Subject: [PATCH 128/158] update python interface --- python/paddle/v2/fluid/layers/nn.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 65e731b6a1..6f58dedb1e 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1702,10 +1702,11 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): def matmul(x, y, transpose_x=False, transpose_y=False, name=None): """ - Applies matrix multipication to two tensors. Currently only rank 1 to rank - 3 input tensors are supported. + Applies matrix multiplication to two tensors. Currently, the input + tensors' rank can be any, but when the rank of anyone inputs is + bigger than 3, this two inputs' rank should be equal. - The actual behavior depends on the shapes of :math:`x`, :math:`y` and the + The actual behavior depends on the shapes of :math:`x`, :math:`y` and the flag values of :attr:`transpose_x`, :attr:`transpose_y`. Specifically: - If a transpose flag is specified, the last two dimensions of the tensor @@ -1715,17 +1716,17 @@ def matmul(x, y, transpose_x=False, transpose_y=False, name=None): opposite: It is treated as :math:`[D, 1]` in nontransposed form and as :math:`[1, D]` in transposed form. - - After transpose, the two tensors are 2-D or 3-D and matrix multipication + - After transpose, the two tensors are 2-D or n-D and matrix multiplication performs in the following way. - If both are 2-D, they are multiplied like conventional matrices. - - If either is 3-D, it is treated as a stack of matrices residing in the + - If either is n-D, it is treated as a stack of matrices residing in the last two dimensions and a batched matrix multiply supporting broadcast applies on the two tensors. Also note that if the raw tensor :math:`x` or :math:`y` is rank-1 and nontransposed, the prepended or appended dimension :math:`1` will be - removed after matrix multipication. + removed after matrix multiplication. Args: x (Variable): The input variable which is a Tensor or LoDTensor. @@ -1742,6 +1743,8 @@ def matmul(x, y, transpose_x=False, transpose_y=False, name=None): .. code-block:: python # Examples to clarify shapes of the inputs and output + # x: [B, ..., M, K], y: [B, ..., K, N] + fluid.layers.matmul(x, y) # out: [B, ..., M, N] # x: [B, M, K], y: [B, K, N] fluid.layers.matmul(x, y) # out: [B, M, N] # x: [B, M, K], y: [K, N] @@ -1757,9 +1760,9 @@ def matmul(x, y, transpose_x=False, transpose_y=False, name=None): fluid.layers.matmul(x, y, True, True) # out: [M, N] """ helper = LayerHelper('matmul', **locals()) - assert max( - len(x.shape), len(y.shape) - ) <= 3, 'Currently only rank 1 to rank 3 input tensors are supported.' + assert max(len(x.shape), len(y.shape)) <= 3 or len(x.shape) == len( + y. + shape), 'Inputs\' rank should be equal or their rank should be less 4.' out = helper.create_tmp_variable(dtype=helper.input_dtype()) helper.append_op( type='matmul', From 22662ae4245a26d51b1c0857939934041e5bee56 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Fri, 19 Jan 2018 16:52:42 +0800 Subject: [PATCH 129/158] Move paddle.v2.fluid.registery to layers * registery is only used by layers * Rename it to layer_function_generator --- python/paddle/v2/fluid/layers/control_flow.py | 13 ++++---- python/paddle/v2/fluid/layers/device.py | 6 ++-- .../layer_function_generator.py} | 29 ++++++++++------- python/paddle/v2/fluid/layers/ops.py | 4 +-- python/paddle/v2/fluid/tests/test_registry.py | 32 ++++++++----------- 5 files changed, 43 insertions(+), 41 deletions(-) rename python/paddle/v2/fluid/{registry.py => layers/layer_function_generator.py} (93%) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index e72b22c83f..f333c3c26e 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -11,12 +11,13 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -from ..layer_helper import LayerHelper, unique_name -from ..framework import Program, Variable, Operator -from .. import core -from tensor import assign, fill_constant import contextlib -from ..registry import autodoc + +from layer_function_generator import autodoc +from tensor import assign, fill_constant +from .. import core +from ..framework import Program, Variable, Operator +from ..layer_helper import LayerHelper, unique_name __all__ = [ 'split_lod_tensor', 'merge_lod_tensor', 'BlockGuard', @@ -1457,7 +1458,7 @@ class DynamicRNN(object): method)) -@autodoc +@autodoc() def reorder_lod_tensor_by_rank(x, rank_table): helper = LayerHelper('reorder_lod_tensor_by_rank', **locals()) helper.is_instance('x', Variable) diff --git a/python/paddle/v2/fluid/layers/device.py b/python/paddle/v2/fluid/layers/device.py index ef74b2b2f0..b586a213ca 100644 --- a/python/paddle/v2/fluid/layers/device.py +++ b/python/paddle/v2/fluid/layers/device.py @@ -15,14 +15,14 @@ All util layers. """ -from ..layer_helper import LayerHelper +from layer_function_generator import autodoc from ..framework import unique_name -from ..registry import autodoc +from ..layer_helper import LayerHelper __all__ = ['get_places'] -@autodoc +@autodoc() def get_places(device_count=None, device_type=None): helper = LayerHelper('get_places', **locals()) out_places = helper.create_variable(name=unique_name(helper.name + ".out")) diff --git a/python/paddle/v2/fluid/registry.py b/python/paddle/v2/fluid/layers/layer_function_generator.py similarity index 93% rename from python/paddle/v2/fluid/registry.py rename to python/paddle/v2/fluid/layers/layer_function_generator.py index 6c0c3a3518..0c4cda86a3 100644 --- a/python/paddle/v2/fluid/registry.py +++ b/python/paddle/v2/fluid/layers/layer_function_generator.py @@ -11,19 +11,21 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -import re import cStringIO -import warnings import functools -import inspect +import re +import warnings -import proto.framework_pb2 as framework_pb2 -from framework import OpProtoHolder, Variable, Program, Operator -from paddle.v2.fluid.layer_helper import LayerHelper, unique_name +from .. import proto + +framework_pb2 = proto.framework_pb2 + +from ..framework import OpProtoHolder, Variable +from ..layer_helper import LayerHelper __all__ = [ 'deprecated', - 'register_layer', + 'generate_layer_fn', 'autodoc', ] @@ -96,7 +98,7 @@ def _generate_doc_string_(op_proto): return buf.getvalue() -def register_layer(op_type): +def generate_layer_fn(op_type): """Register the Python layer for an Operator. Args: @@ -202,7 +204,10 @@ def deprecated(func_or_class): return func_wrapper -def autodoc(func): - func.__doc__ = _generate_doc_string_(OpProtoHolder.instance().get_op_proto( - func.__name__)) - return func +def autodoc(comment=""): + def __impl__(func): + func.__doc__ = _generate_doc_string_(OpProtoHolder.instance( + ).get_op_proto(func.__name__)) + comment + return func + + return __impl__ diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index 21945edf08..24f6b7294e 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -11,7 +11,7 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. -from ..registry import register_layer +from layer_function_generator import generate_layer_fn __activations__ = [ 'sigmoid', @@ -62,4 +62,4 @@ __all__ = [ ] + __activations__ for _OP in set(__all__): - globals()[_OP] = register_layer(_OP) + globals()[_OP] = generate_layer_fn(_OP) diff --git a/python/paddle/v2/fluid/tests/test_registry.py b/python/paddle/v2/fluid/tests/test_registry.py index dba1189630..b1749a76f0 100644 --- a/python/paddle/v2/fluid/tests/test_registry.py +++ b/python/paddle/v2/fluid/tests/test_registry.py @@ -1,35 +1,31 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest -import warnings import paddle.v2.fluid as fluid -import paddle.v2.fluid.framework as framework -import paddle.v2.fluid.layers as layers -import paddle.v2.fluid.registry as registry +import numpy as np +import decorators class TestRegistry(unittest.TestCase): + @decorators.prog_scope() def test_registry_layer(self): - self.layer_type = "mean" - program = framework.Program() - x = fluid.layers.data(name='X', shape=[10, 10], dtype='float32') - output = layers.mean(x) + output = fluid.layers.mean(x=x) + place = fluid.CPUPlace() exe = fluid.Executor(place) - X = np.random.random((10, 10)).astype("float32") - mean_out = exe.run(program, feed={"X": X}, fetch_list=[output]) + mean_out = exe.run(feed={"X": X}, fetch_list=[output]) self.assertAlmostEqual(np.mean(X), mean_out) From 9a97c7f745b0014504f9c52897b72fd58147e70a Mon Sep 17 00:00:00 2001 From: ying Date: Wed, 17 Jan 2018 20:09:44 +0800 Subject: [PATCH 130/158] add wmt16 into dataset. --- python/paddle/v2/dataset/__init__.py | 16 +- python/paddle/v2/dataset/common.py | 21 +- python/paddle/v2/dataset/tests/wmt16_test.py | 66 ++++ python/paddle/v2/dataset/wmt14.py | 18 +- python/paddle/v2/dataset/wmt16.py | 348 ++++++++++++++++++ python/paddle/v2/fluid/layers/control_flow.py | 33 +- 6 files changed, 482 insertions(+), 20 deletions(-) create mode 100644 python/paddle/v2/dataset/tests/wmt16_test.py create mode 100644 python/paddle/v2/dataset/wmt16.py diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py index 90830515c1..c1acbecd9c 100644 --- a/python/paddle/v2/dataset/__init__.py +++ b/python/paddle/v2/dataset/__init__.py @@ -24,11 +24,23 @@ import conll05 import uci_housing import sentiment import wmt14 +import wmt16 import mq2007 import flowers import voc2012 __all__ = [ - 'mnist', 'imikolov', 'imdb', 'cifar', 'movielens', 'conll05', 'sentiment' - 'uci_housing', 'wmt14', 'mq2007', 'flowers', 'voc2012' + 'mnist', + 'imikolov', + 'imdb', + 'cifar', + 'movielens', + 'conll05', + 'sentiment' + 'uci_housing', + 'wmt14', + 'wmt16', + 'mq2007', + 'flowers', + 'voc2012', ] diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py index fab8a68b0b..9aba35a648 100644 --- a/python/paddle/v2/dataset/common.py +++ b/python/paddle/v2/dataset/common.py @@ -25,8 +25,12 @@ import glob import cPickle as pickle __all__ = [ - 'DATA_HOME', 'download', 'md5file', 'split', 'cluster_files_reader', - 'convert' + 'DATA_HOME', + 'download', + 'md5file', + 'split', + 'cluster_files_reader', + 'convert', ] DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset') @@ -58,12 +62,15 @@ def md5file(fname): return hash_md5.hexdigest() -def download(url, module_name, md5sum): +def download(url, module_name, md5sum, save_name=None): dirname = os.path.join(DATA_HOME, module_name) if not os.path.exists(dirname): os.makedirs(dirname) - filename = os.path.join(dirname, url.split('/')[-1]) + filename = os.path.join(dirname, + url.split('/')[-1] + if save_name is None else save_name) + retry = 0 retry_limit = 3 while not (os.path.exists(filename) and md5file(filename) == md5sum): @@ -196,9 +203,11 @@ def convert(output_path, reader, line_count, name_prefix): Convert data from reader to recordio format files. :param output_path: directory in which output files will be saved. - :param reader: a data reader, from which the convert program will read data instances. + :param reader: a data reader, from which the convert program will read + data instances. :param name_prefix: the name prefix of generated files. - :param max_lines_to_shuffle: the max lines numbers to shuffle before writing. + :param max_lines_to_shuffle: the max lines numbers to shuffle before + writing. """ assert line_count >= 1 diff --git a/python/paddle/v2/dataset/tests/wmt16_test.py b/python/paddle/v2/dataset/tests/wmt16_test.py new file mode 100644 index 0000000000..cef6c3216e --- /dev/null +++ b/python/paddle/v2/dataset/tests/wmt16_test.py @@ -0,0 +1,66 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.v2.dataset.wmt16 +import unittest + + +class TestWMT16(unittest.TestCase): + def checkout_one_sample(self, sample): + # train data has 3 field: source language word indices, + # target language word indices, and target next word indices. + self.assertEqual(len(sample), 3) + + # test start mark and end mark in source word indices. + self.assertEqual(sample[0][0], 0) + self.assertEqual(sample[0][-1], 1) + + # test start mask in target word indices + self.assertEqual(sample[1][0], 0) + + # test en mask in target next word indices + self.assertEqual(sample[2][-1], 1) + + def test_train(self): + for idx, sample in enumerate( + paddle.v2.dataset.wmt16.train( + src_dict_size=100000, trg_dict_size=100000)()): + if idx >= 10: break + self.checkout_one_sample(sample) + + def test_test(self): + for idx, sample in enumerate( + paddle.v2.dataset.wmt16.test( + src_dict_size=1000, trg_dict_size=1000)()): + if idx >= 10: break + self.checkout_one_sample(sample) + + def test_val(self): + for idx, sample in enumerate( + paddle.v2.dataset.wmt16.validation( + src_dict_size=1000, trg_dict_size=1000)()): + if idx >= 10: break + self.checkout_one_sample(sample) + + def test_get_dict(self): + dict_size = 1000 + word_dict = paddle.v2.dataset.wmt16.get_dict("en", dict_size, True) + self.assertEqual(len(word_dict), dict_size) + self.assertEqual(word_dict[0], "") + self.assertEqual(word_dict[1], "") + self.assertEqual(word_dict[2], "") + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/dataset/wmt14.py b/python/paddle/v2/dataset/wmt14.py index 95a35d97ce..1e54a4999b 100644 --- a/python/paddle/v2/dataset/wmt14.py +++ b/python/paddle/v2/dataset/wmt14.py @@ -25,12 +25,20 @@ import gzip import paddle.v2.dataset.common from paddle.v2.parameters import Parameters -__all__ = ['train', 'test', 'build_dict', 'convert'] - -URL_DEV_TEST = 'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz' +__all__ = [ + 'train', + 'test', + 'get_dict', + 'convert', +] + +URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' + 'cslm_joint_paper/data/dev+test.tgz') MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' -# this is a small set of data for test. The original data is too large and will be add later. -URL_TRAIN = 'http://paddlepaddle.cdn.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz' +# this is a small set of data for test. The original data is too large and +# will be add later. +URL_TRAIN = ('http://paddlepaddle.cdn.bcebos.com/demo/' + 'wmt_shrinked_data/wmt14.tgz') MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' # BLEU of this trained model is 26.92 URL_MODEL = 'http://paddlepaddle.bj.bcebos.com/demo/wmt_14/wmt14_model.tar.gz' diff --git a/python/paddle/v2/dataset/wmt16.py b/python/paddle/v2/dataset/wmt16.py new file mode 100644 index 0000000000..a1899f20b5 --- /dev/null +++ b/python/paddle/v2/dataset/wmt16.py @@ -0,0 +1,348 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +ACL2016 Multimodal Machine Translation. Please see this websit for more details: +http://www.statmt.org/wmt16/multimodal-task.html#task1 + +If you use the dataset created for your task, please cite the following paper: +Multi30K: Multilingual English-German Image Descriptions. + +@article{elliott-EtAl:2016:VL16, + author = {{Elliott}, D. and {Frank}, S. and {Sima"an}, K. and {Specia}, L.}, + title = {Multi30K: Multilingual English-German Image Descriptions}, + booktitle = {Proceedings of the 6th Workshop on Vision and Language}, + year = {2016}, + pages = {70--74}, + year = 2016 +} +""" + +import os +import tarfile +import gzip +from collections import defaultdict + +import paddle.v2.dataset.common + +__all__ = [ + "train", + "test", + "validation", + "convert", + "fetch", + "get_dict", +] + +DATA_URL = ("http://cloud.dlnel.org/filepub/" + "?uuid=46a0808e-ddd8-427c-bacd-0dbc6d045fed") +DATA_MD5 = "0c38be43600334966403524a40dcd81e" + +TOTAL_EN_WORDS = 11250 +TOTAL_DE_WORDS = 19220 + +START_MARK = "" +END_MARK = "" +UNK_MARK = "" + + +def __build_dict__(tar_file, dict_size, save_path, lang): + word_dict = defaultdict(int) + with tarfile.open(tar_file, mode="r") as f: + for line in f.extractfile("wmt16/train"): + line_split = line.strip().split("\t") + if len(line_split) != 2: continue + sen = line_split[0] if lang == "en" else line_split[1] + for w in sen.split(): + word_dict[w] += 1 + + with open(save_path, "w") as fout: + fout.write("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)) + for idx, word in enumerate( + sorted( + word_dict.iteritems(), key=lambda x: x[1], reverse=True)): + if idx + 3 == dict_size: break + fout.write("%s\n" % (word[0])) + + +def __load_dict__(tar_file, dict_size, lang, reverse=False): + dict_path = os.path.join(paddle.v2.dataset.common.DATA_HOME, + "wmt16/%s_%d.dict" % (lang, dict_size)) + if not os.path.exists(dict_path) or ( + len(open(dict_path, "r").readlines()) != dict_size): + __build_dict__(tar_file, dict_size, dict_path, lang) + + word_dict = {} + with open(dict_path, "r") as fdict: + for idx, line in enumerate(fdict): + if reverse: + word_dict[idx] = line.strip() + else: + word_dict[line.strip()] = idx + return word_dict + + +def __get_dict_size__(src_dict_size, trg_dict_size, src_lang): + src_dict_size = min(src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else + TOTAL_DE_WORDS)) + trg_dict_size = min(trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else + TOTAL_ENG_WORDS)) + return src_dict_size, trg_dict_size + + +def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): + def reader(): + src_dict = __load_dict__(tar_file, src_dict_size, src_lang) + trg_dict = __load_dict__(tar_file, trg_dict_size, + ("de" if src_lang == "en" else "en")) + + # the indice for start mark, end mark, and unk are the same in source + # language and target language. Here uses the source language + # dictionary to determine their indices. + start_id = src_dict[START_MARK] + end_id = src_dict[END_MARK] + unk_id = src_dict[UNK_MARK] + + src_col = 0 if src_lang == "en" else 1 + trg_col = 1 - src_col + + with tarfile.open(tar_file, mode="r") as f: + for line in f.extractfile(file_name): + line_split = line.strip().split("\t") + if len(line_split) != 2: + continue + src_words = line_split[src_col].split() + src_ids = [start_id] + [ + src_dict.get(w, unk_id) for w in src_words + ] + [end_id] + + trg_words = line_split[trg_col].split() + trg_ids = [trg_dict.get(w, unk_id) for w in trg_words] + + trg_ids_next = trg_ids + [end_id] + trg_ids = [start_id] + trg_ids + + yield src_ids, trg_ids, trg_ids_next + + return reader + + +def train(src_dict_size, trg_dict_size, src_lang="en"): + """ + WMT16 train set reader. + + This function returns the reader for train data. Each sample the reader + returns is made up of three fields: the source language word index sequence, + target language word index sequence and next word index sequence. + + + NOTE: + The original like for training data is: + http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz + + paddle.dataset.wmt16 provides a tokenized version of the original dataset by + using moses's tokenization script: + https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl + + Args: + src_dict_size(int): Size of the source language dictionary. Three + special tokens will be added into the dictionary: + for start mark, for end mark, and for + unknown word. + trg_dict_size(int): Size of the target language dictionary. Three + special tokens will be added into the dictionary: + for start mark, for end mark, and for + unknown word. + src_lang(string): A string indicating which language is the source + language. Available options are: "en" for English + and "de" for Germany. + + Returns: + callable: The train reader. + """ + + assert (src_lang in ["en", "de"], ("An error language type. Only support: " + "en (for English); de(for Germany)")) + src_dict_size, trg_dict_size = __get_dict_size__(src_dict_size, + trg_dict_size, src_lang) + + return reader_creator( + tar_file=paddle.v2.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, + "wmt16.tar.gz"), + file_name="wmt16/train", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang) + + +def test(src_dict_size, trg_dict_size, src_lang="en"): + """ + WMT16 test set reader. + + This function returns the reader for test data. Each sample the reader + returns is made up of three fields: the source language word index sequence, + target language word index sequence and next word index sequence. + + NOTE: + The original like for test data is: + http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz + + paddle.dataset.wmt16 provides a tokenized version of the original dataset by + using moses's tokenization script: + https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl + + Args: + src_dict_size(int): Size of the source language dictionary. Three + special tokens will be added into the dictionary: + for start mark, for end mark, and for + unknown word. + trg_dict_size(int): Size of the target language dictionary. Three + special tokens will be added into the dictionary: + for start mark, for end mark, and for + unknown word. + src_lang(string): A string indicating which language is the source + language. Available options are: "en" for English + and "de" for Germany. + + Returns: + callable: The test reader. + """ + + assert (src_lang in ["en", "de"], + ("An error language type. " + "Only support: en (for English); de(for Germany)")) + + src_dict_size, trg_dict_size = __get_dict_size__(src_dict_size, + trg_dict_size, src_lang) + + return reader_creator( + tar_file=paddle.v2.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, + "wmt16.tar.gz"), + file_name="wmt16/test", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang) + + +def validation(src_dict_size, trg_dict_size, src_lang="en"): + """ + WMT16 validation set reader. + + This function returns the reader for validation data. Each sample the reader + returns is made up of three fields: the source language word index sequence, + target language word index sequence and next word index sequence. + + NOTE: + The original like for validation data is: + http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz + + paddle.dataset.wmt16 provides a tokenized version of the original dataset by + using moses's tokenization script: + https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl + + Args: + src_dict_size(int): Size of the source language dictionary. Three + special tokens will be added into the dictionary: + for start mark, for end mark, and for + unknown word. + trg_dict_size(int): Size of the target language dictionary. Three + special tokens will be added into the dictionary: + for start mark, for end mark, and for + unknown word. + src_lang(string): A string indicating which language is the source + language. Available options are: "en" for English + and "de" for Germany. + + Returns: + callable: The validation reader. + """ + assert (src_lang in ["en", "de"], + ("An error language type. " + "Only support: en (for English); de(for Germany)")) + src_dict_size, trg_dict_size = __get_dict_size__(src_dict_size, + trg_dict_size, src_lang) + + return reader_creator( + tar_file=paddle.v2.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, + "wmt16.tar.gz"), + file_name="wmt16/val", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang) + + +def get_dict(lang, dict_size, reverse=False): + """ + return the word dictionary for the specified language. + + Args: + lang(string): A string indicating which language is the source + language. Available options are: "en" for English + and "de" for Germany. + dict_size(int): Size of the specified language dictionary. + reverse(bool): If reverse is set to False, the returned python + dictionary will use word as key and use index as value. + If reverse is set to True, the returned python + dictionary will use index as key and word as value. + + Returns: + dict: The word dictionary for the specific language. + """ + + if lang == "en": dict_size = min(dict_size, TOTAL_EN_WORDS) + else: dict_size = min(dict_size, TOTAL_DE_WORDS) + + dict_path = os.path.join(paddle.v2.dataset.common.DATA_HOME, + "wmt16/%s_%d.dict" % (lang, dict_size)) + assert (os.path.exists(dict_path), "Word dictionary does not exist. " + "Please invoke paddle.dataset.wmt16.train/test/validation " + "first to build the dictionary.") + tar_file = os.path.join(paddle.v2.dataset.common.DATA_HOME, "wmt16.tar.gz") + return __load_dict__(tar_file, dict_size, lang, reverse) + + +def fetch(): + """download the entire dataset. + """ + paddle.v4.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, + "wmt16.tar.gz") + + +def convert(path, src_dict_size, trg_dict_size, src_lang): + """Converts dataset to recordio format. + """ + + paddle.v2.dataset.common.convert( + path, + train( + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang), + 1000, + "wmt16_train") + paddle.v2.dataset.common.convert( + path, + test( + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang), + 1000, + "wmt16_test") + paddle.v2.dataset.common.convert( + path, + validation( + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang), + 1000, + "wmt16_validation") diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index e72b22c83f..b2183ebda1 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -19,13 +19,32 @@ import contextlib from ..registry import autodoc __all__ = [ - 'split_lod_tensor', 'merge_lod_tensor', 'BlockGuard', - 'BlockGuardWithCompletion', 'StaticRNNMemoryLink', 'WhileGuard', 'While', - 'lod_rank_table', 'max_sequence_len', 'topk', 'lod_tensor_to_array', - 'array_to_lod_tensor', 'increment', 'array_write', 'create_array', - 'less_than', 'array_read', 'shrink_memory', 'array_length', 'IfElse', - 'DynamicRNN', 'ConditionalBlock', 'StaticRNN', 'reorder_lod_tensor_by_rank', - 'ParallelDo', 'Print' + 'split_lod_tensor', + 'merge_lod_tensor', + 'BlockGuard', + 'BlockGuardWithCompletion', + 'StaticRNNMemoryLink', + 'WhileGuard', + 'While', + 'lod_rank_table', + 'max_sequence_len', + 'topk', + 'lod_tensor_to_array', + 'array_to_lod_tensor', + 'increment', + 'array_write', + 'create_array', + 'less_than', + 'array_read', + 'shrink_memory', + 'array_length', + 'IfElse', + 'DynamicRNN', + 'ConditionalBlock', + 'StaticRNN', + 'reorder_lod_tensor_by_rank', + 'ParallelDo', + 'Print', ] From 811c4ee401b2116d4379c818950f7eeb26afef0b Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 19 Jan 2018 17:57:07 +0800 Subject: [PATCH 131/158] Add python wrapper for sequence_reshape. --- doc/api/v2/fluid/layers.rst | 5 + python/paddle/v2/fluid/layers/nn.py | 128 ++++++++++++++------ python/paddle/v2/fluid/tests/test_layers.py | 8 ++ 3 files changed, 104 insertions(+), 37 deletions(-) diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 0eb531cf02..986026e0b9 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -504,3 +504,8 @@ l2_normalize ------------ .. autofunction:: paddle.v2.fluid.layers.l2_normalize :noindex: + +sequence_reshape +---------------- +.. autofunction:: paddle.v2.fluid.layers.sequence_reshape + :noindex: diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 7d61d111a8..cbd2cfec59 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -28,7 +28,7 @@ __all__ = [ 'batch_norm', 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', 'lstm_unit', 'reduce_sum', 'reduce_mean', 'reduce_max', 'reduce_min', 'sequence_first_step', 'sequence_last_step', 'dropout', 'split', - 'l2_normalize', 'matmul', 'warpctc' + 'l2_normalize', 'matmul', 'warpctc', 'sequence_reshape' ] @@ -213,33 +213,33 @@ def dynamic_lstm(input, (https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: .. math:: - - i_t & = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) - f_t & = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) + i_t & = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) - \\tilde{c_t} & = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) + f_t & = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) - o_t & = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) + \\tilde{c_t} & = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) - c_t & = f_t \odot c_{t-1} + i_t \odot \\tilde{c_t} + o_t & = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) + + c_t & = f_t \odot c_{t-1} + i_t \odot \\tilde{c_t} h_t & = o_t \odot act_h(c_t) - where the :math:`W` terms denote weight matrices (e.g. :math:`W_{xi}` is + where the :math:`W` terms denote weight matrices (e.g. :math:`W_{xi}` is the matrix of weights from the input gate to the input), :math:`W_{ic}, \ - W_{fc}, W_{oc}` are diagonal weight matrices for peephole connections. In - our implementation, we use vectors to reprenset these diagonal weight - matrices. The :math:`b` terms denote bias vectors (:math:`b_i` is the input - gate bias vector), :math:`\sigma` is the non-line activations, such as - logistic sigmoid function, and :math:`i, f, o` and :math:`c` are the input - gate, forget gate, output gate, and cell activation vectors, respectively, + W_{fc}, W_{oc}` are diagonal weight matrices for peephole connections. In + our implementation, we use vectors to reprenset these diagonal weight + matrices. The :math:`b` terms denote bias vectors (:math:`b_i` is the input + gate bias vector), :math:`\sigma` is the non-line activations, such as + logistic sigmoid function, and :math:`i, f, o` and :math:`c` are the input + gate, forget gate, output gate, and cell activation vectors, respectively, all of which have the same size as the cell output activation vector :math:`h`. - The :math:`\odot` is the element-wise product of the vectors. :math:`act_g` - and :math:`act_h` are the cell input and cell output activation functions - and `tanh` is usually used for them. :math:`\\tilde{c_t}` is also called - candidate hidden state, which is computed based on the current input and + The :math:`\odot` is the element-wise product of the vectors. :math:`act_g` + and :math:`act_h` are the cell input and cell output activation functions + and `tanh` is usually used for them. :math:`\\tilde{c_t}` is also called + candidate hidden state, which is computed based on the current input and the previous hidden state. Set `use_peepholes` to `False` to disable peephole connection. The formula @@ -251,38 +251,38 @@ def dynamic_lstm(input, Users can choose to use fully-connect layer before LSTM layer. Args: - input(Variable): The input of dynamic_lstm layer, which supports - variable-time length input sequence. The underlying - tensor in this Variable is a matrix with shape - (T X 4D), where T is the total time steps in this + input(Variable): The input of dynamic_lstm layer, which supports + variable-time length input sequence. The underlying + tensor in this Variable is a matrix with shape + (T X 4D), where T is the total time steps in this mini-batch, D is the hidden size. size(int): 4 * hidden size. - param_attr(ParamAttr): The parameter attribute for the learnable - hidden-hidden weights. + param_attr(ParamAttr): The parameter attribute for the learnable + hidden-hidden weights. - - The shape is (D x 4D), where D is the hidden - size. + - The shape is (D x 4D), where D is the hidden + size. - Weights = {:math:`W_{ch}, W_{ih}, \ W_{fh}, W_{oh}`} bias_attr(ParamAttr): The bias attribute for the learnable bias - weights, which contains two parts, input-hidden - bias weights and peephole connections weights if - setting `use_peepholes` to `True`. + weights, which contains two parts, input-hidden + bias weights and peephole connections weights if + setting `use_peepholes` to `True`. - 1. `use_peepholes = False` - - The shape is (1 x 4D). + 1. `use_peepholes = False` + - The shape is (1 x 4D). - Biases = {:math:`b_c, b_i, b_f, b_o`}. - 2. `use_peepholes = True` - - The shape is (1 x 7D). + 2. `use_peepholes = True` + - The shape is (1 x 7D). - Biases = { :math:`b_c, b_i, b_f, b_o, W_{ic}, \ W_{fc}, W_{oc}`}. - use_peepholes(bool): Whether to enable diagonal/peephole connections, + use_peepholes(bool): Whether to enable diagonal/peephole connections, default `True`. is_reverse(bool): Whether to compute reversed LSTM, default `False`. - gate_activation(str): The activation for input gate, forget gate and - output gate. Choices = ["sigmoid", "tanh", "relu", + gate_activation(str): The activation for input gate, forget gate and + output gate. Choices = ["sigmoid", "tanh", "relu", "identity"], default "sigmoid". - cell_activation(str): The activation for cell output. Choices = ["sigmoid", + cell_activation(str): The activation for cell output. Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh". candidate_activation(str): The activation for candidate hidden state. Choices = ["sigmoid", "tanh", "relu", "identity"], @@ -1914,3 +1914,57 @@ def warpctc(input, label, blank=0, norm_by_times=False, **kwargs): attrs={'blank': blank, 'norm_by_times': norm_by_times}) return loss_out + + +def sequence_reshape(input, new_dim): + """ + **Sequence Reshape Layer** + + This layer will rearrange the input sequences. The new dimension is set by + user. Length of each sequence is computed according to original length, + original dimension and new dimension. The following example will help to + illustrate the function of this layer: + + .. code-block:: text + + x is a LoDTensor: + x.lod = [[0, 2, 6]] + x.data = [[1, 2], [3, 4], + [5, 6], [7, 8], [9, 10], [11, 12]] + x.dims = [6, 2] + + set new_dim = 4 + + then out is a LoDTensor: + out.lod = [[0, 1, 3]] + out.data = [[1, 2, 3, 4], + [5, 6, 7, 8], [9, 10, 11, 12]] + out.dims = [3, 4] + + Currently, only 1-level LoDTensor is supported and please make sure + (original length * original dimension) can be divided by new dimension with + no remainder for each sequence. + + Args: + input (Variable): (LodTensor, default: LoDTensor), a 2-D LoDTensor + with shape being [N, M] where M for dimension. + new_dim (int): New dimension which the input LoDTensor is reshaped to. + + Returns: + Variable: Reshaped LoDTensor according to new dimension. + + Examples: + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[5, 20], + dtype='float32', lod_level=1) + x_reshaped = layers.sequence_reshape(input=x, new_dim=10) + """ + helper = LayerHelper('sequence_reshape', **locals()) + out = helper.create_tmp_variable(helper.input_dtype()) + helper.append_op( + type='sequence_reshape', + inputs={'X': [input]}, + outputs={'Out': [out]}, + attrs={'new_dim': new_dim}) + return out diff --git a/python/paddle/v2/fluid/tests/test_layers.py b/python/paddle/v2/fluid/tests/test_layers.py index a4e155b534..b366e5ba36 100644 --- a/python/paddle/v2/fluid/tests/test_layers.py +++ b/python/paddle/v2/fluid/tests/test_layers.py @@ -216,6 +216,14 @@ class TestBook(unittest.TestCase): self.assertIsNotNone(x) print(str(program)) + def test_sequence_reshape(self): + program = Program() + with program_guard(program): + x = layers.data(name='x', shape=[8], dtype='float32', lod_level=1) + out = layers.sequence_reshape(input=x, new_dim=16) + self.assertIsNotNone(out) + print(str(program)) + if __name__ == '__main__': unittest.main() From d3905fbc1e53dcb8ef5481860e44e9ab4a704e5d Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 19 Jan 2018 19:02:02 +0800 Subject: [PATCH 132/158] add fluid vgg16 dist test --- benchmark/cluster/vgg16/fluid/Dockerfile | 12 + benchmark/cluster/vgg16/fluid/k8s_tools.py | 78 ++++++ benchmark/cluster/vgg16/fluid/paddle_k8s | 200 ++++++++++++++ benchmark/cluster/vgg16/fluid/pserver.yaml | 72 +++++ benchmark/cluster/vgg16/fluid/reader.py | 2 + benchmark/cluster/vgg16/fluid/trainer.yaml | 69 +++++ benchmark/cluster/vgg16/fluid/vgg16.py | 248 ++++++++++++++++++ benchmark/cluster/{ => vgg16}/v2/Dockerfile | 2 + benchmark/cluster/{ => vgg16}/v2/pserver.yaml | 0 benchmark/cluster/{ => vgg16}/v2/reader.py | 0 benchmark/cluster/{ => vgg16}/v2/trainer.yaml | 0 benchmark/cluster/{ => vgg16}/v2/vgg16.py | 0 12 files changed, 683 insertions(+) create mode 100644 benchmark/cluster/vgg16/fluid/Dockerfile create mode 100644 benchmark/cluster/vgg16/fluid/k8s_tools.py create mode 100755 benchmark/cluster/vgg16/fluid/paddle_k8s create mode 100644 benchmark/cluster/vgg16/fluid/pserver.yaml create mode 100644 benchmark/cluster/vgg16/fluid/reader.py create mode 100644 benchmark/cluster/vgg16/fluid/trainer.yaml create mode 100644 benchmark/cluster/vgg16/fluid/vgg16.py rename benchmark/cluster/{ => vgg16}/v2/Dockerfile (81%) rename benchmark/cluster/{ => vgg16}/v2/pserver.yaml (100%) rename benchmark/cluster/{ => vgg16}/v2/reader.py (100%) rename benchmark/cluster/{ => vgg16}/v2/trainer.yaml (100%) rename benchmark/cluster/{ => vgg16}/v2/vgg16.py (100%) diff --git a/benchmark/cluster/vgg16/fluid/Dockerfile b/benchmark/cluster/vgg16/fluid/Dockerfile new file mode 100644 index 0000000000..77cd17f2b9 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/Dockerfile @@ -0,0 +1,12 @@ +#FROM paddlepaddle/paddlecloud-job +#RUN mkdir -p /workspace +#ADD reader.py /workspace/ +#RUN python /workspace/reader.py +FROM python:2.7.14 +ADD *.whl / +RUN pip install /*.whl && rm -f /*.whl +ADD paddle_k8s /usr/bin +ADD k8s_tools.py /root +RUN pip install -U kubernetes opencv-python && apt-get update -y && apt-get install -y iputils-ping libgtk2.0-dev + +ADD vgg16.py /workspace/ diff --git a/benchmark/cluster/vgg16/fluid/k8s_tools.py b/benchmark/cluster/vgg16/fluid/k8s_tools.py new file mode 100644 index 0000000000..8a64dbd361 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/k8s_tools.py @@ -0,0 +1,78 @@ +#!/bin/env python +import os +import sys +import time +import socket +from kubernetes import client, config +PADDLE_JOB_NAME = os.getenv("PADDLE_JOB_NAME") +NAMESPACE = os.getenv("NAMESPACE") +PORT = os.getenv("PSERVER_PORT") +if os.getenv("KUBERNETES_SERVICE_HOST", None): + config.load_incluster_config() +else: + config.load_kube_config() +v1 = client.CoreV1Api() + + +def fetch_pods_info(label_selector): + api_response = v1.list_namespaced_pod( + namespace=NAMESPACE, pretty=True, label_selector=label_selector) + pod_list = [] + for item in api_response.items: + pod_list.append((item.status.phase, item.status.pod_ip)) + return pod_list + + +def wait_pods_running(label_selector, desired): + print "label selector: %s, desired: %s" % (label_selector, desired) + while True: + count = count_pods_by_phase(label_selector, 'Running') + # NOTE: pods may be scaled. + if count >= int(desired): + break + print 'current cnt: %d sleep for 5 seconds...' % count + time.sleep(5) + +def count_pods_by_phase(label_selector, phase): + pod_list = fetch_pods_info(label_selector) + filtered_pod_list = filter(lambda x: x[0] == phase, pod_list) + return len(filtered_pod_list) + + +def fetch_pserver_ips(): + label_selector = "paddle-job-pserver=%s" % PADDLE_JOB_NAME + pod_list = fetch_pods_info(label_selector) + pserver_ips = [item[1] for item in pod_list] + return ",".join(pserver_ips) + +def fetch_master_ip(): + label_selector = "paddle-job-master=%s" % PADDLE_JOB_NAME + pod_list = fetch_pods_info(label_selector) + master_ips = [item[1] for item in pod_list] + return master_ips[0] + +def fetch_trainer_id(): + label_selector = "paddle-job=%s" % PADDLE_JOB_NAME + pod_list = fetch_pods_info(label_selector) + trainer_ips = [item[1] for item in pod_list] + trainer_ips.sort() + local_ip = socket.gethostbyname(socket.gethostname()) + for i in xrange(len(trainer_ips)): + if trainer_ips[i] == local_ip: + return i + return None + + +if __name__ == "__main__": + command = sys.argv[1] + if command == "fetch_pserver_ips": + print fetch_pserver_ips() + elif command == "fetch_trainer_id": + print fetch_trainer_id() + elif command == "fetch_master_ip": + print fetch_master_ip() + elif command == "count_pods_by_phase": + print count_pods_by_phase(sys.argv[2], sys.argv[3]) + elif command == "wait_pods_running": + wait_pods_running(sys.argv[2], sys.argv[3]) + diff --git a/benchmark/cluster/vgg16/fluid/paddle_k8s b/benchmark/cluster/vgg16/fluid/paddle_k8s new file mode 100755 index 0000000000..8f1c5db717 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/paddle_k8s @@ -0,0 +1,200 @@ +#!/bin/bash +start_pserver() { + stdbuf -oL paddle pserver \ + --use_gpu=0 \ + --port=$PADDLE_INIT_PORT \ + --ports_num=$PADDLE_INIT_PORTS_NUM \ + --ports_num_for_sparse=$PADDLE_INIT_PORTS_NUM_FOR_SPARSE \ + --nics=$PADDLE_INIT_NICS \ + --comment=paddle_process_k8s \ + --num_gradient_servers=$PADDLE_INIT_NUM_GRADIENT_SERVERS +} + +start_new_pserver() { + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-master=${PADDLE_JOB_NAME} 1 + export MASTER_IP=$(python /root/k8s_tools.py fetch_master_ip) + stdbuf -oL /usr/bin/pserver \ + -port=$PADDLE_INIT_PORT \ + -num-pservers=$PSERVERS \ + -log-level=debug \ + -etcd-endpoint=http://$MASTER_IP:2379 +} + +start_master() { + stdbuf -oL /usr/bin/master \ + -port=8080 \ + -chunk-per-task=1\ + -task-timout-dur=16s\ + -endpoints=http://127.0.0.1:2379 +} + +check_failed_cnt() { + max_failed=$1 + failed_count=$(python /root/k8s_tools.py count_pods_by_phase paddle-job=${PADDLE_JOB_NAME} Failed) + if [ $failed_count -gt $max_failed ]; then + stdbuf -oL echo "Failed trainer count beyond the threadhold: "$max_failed + echo "Failed trainer count beyond the threshold: " $max_failed > /dev/termination-log + exit 0 + fi +} + +check_trainer_ret() { + ret=$1 + stdbuf -oL echo "job returned $ret...setting pod return message..." + stdbuf -oL echo "===============================" + + if [ $ret -eq 136 ] ; then + echo "Error Arithmetic Operation(Floating Point Exception)" > /dev/termination-log + elif [ $ret -eq 139 ] ; then + echo "Segmentation Fault" > /dev/termination-log + elif [ $ret -eq 1 ] ; then + echo "General Error" > /dev/termination-log + elif [ $ret -eq 134 ] ; then + echo "Program Abort" > /dev/termination-log + fi + stdbuf -oL echo "termination log wroted..." + exit $ret +} + +start_fluid_process() { + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-pserver=${PADDLE_JOB_NAME} ${PSERVERS} + if [ "${TRAINING_ROLE}" == "TRAINER" ]; then + check_failed_cnt ${TRAINERS} + sleep 5 + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-master=${PADDLE_JOB_NAME} 1 + export PADDLE_INIT_TRAINER_ID=$(python /root/k8s_tools.py fetch_trainer_id) + fi + export PADDLE_INIT_PSERVERS=$(python /root/k8s_tools.py fetch_pserver_ips) + stdbuf -oL sh -c "${ENTRY}" + check_trainer_ret $? +} + +start_new_trainer() { + # FIXME(Yancey1989): use command-line interface to configure the max failed count + check_failed_cnt ${TRAINERS} + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-pserver=${PADDLE_JOB_NAME} ${PSERVERS} + sleep 5 + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-master=${PADDLE_JOB_NAME} 1 + export MASTER_IP=$(python /root/k8s_tools.py fetch_master_ip) + export ETCD_IP="$MASTER_IP" + + # NOTE: $TRAINER_PACKAGE may be large, do not copy + export PYTHONPATH=$TRAINER_PACKAGE:$PYTHONPATH + cd $TRAINER_PACKAGE + + stdbuf -oL echo "Starting training job: " $TRAINER_PACKAGE, "num_gradient_servers:" \ + $PADDLE_INIT_NUM_GRADIENT_SERVERS, "version: " $1 + + stdbuf -oL sh -c "${ENTRY}" + check_trainer_ret $? +} + +start_trainer() { + # paddle v1 and V2 distributed training does not allow any trainer failed. + check_failed_cnt 0 + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-pserver=${PADDLE_JOB_NAME} ${PSERVERS} + stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job=${PADDLE_JOB_NAME} ${TRAINERS} + + export PADDLE_INIT_PSERVERS=$(python /root/k8s_tools.py fetch_pserver_ips) + export PADDLE_INIT_TRAINER_ID=$(python /root/k8s_tools.py fetch_trainer_id) + stdbuf -oL echo $PADDLE_INIT_TRAINER_ID > /trainer_id + # FIXME: /trainer_count = PADDLE_INIT_NUM_GRADIENT_SERVERS + stdbuf -oL echo $PADDLE_INIT_NUM_GRADIENT_SERVERS > /trainer_count + + # NOTE: $TRAINER_PACKAGE may be large, do not copy + export PYTHONPATH=$TRAINER_PACKAGE:$PYTHONPATH + cd $TRAINER_PACKAGE + + stdbuf -oL echo "Starting training job: " $TRAINER_PACKAGE, "num_gradient_servers:" \ + $PADDLE_INIT_NUM_GRADIENT_SERVERS, "trainer_id: " $PADDLE_INIT_TRAINER_ID, \ + "version: " $1 + + # FIXME: If we use the new PServer by Golang, add Kubernetes healthz + # to wait PServer process get ready.Now only sleep 20 seconds. + sleep 20 + + case "$1" in + "v1") + FILE_COUNT=$(wc -l $TRAIN_LIST | awk '{print $1}') + if [ $FILE_COUNT -le $PADDLE_INIT_NUM_GRADIENT_SERVERS ]; then + echo "file count less than trainers" + check_trainer_ret 0 + fi + let lines_per_node="$FILE_COUNT / ($PADDLE_INIT_NUM_GRADIENT_SERVERS + 1)" + echo "spliting file to" $lines_per_node + cp $TRAIN_LIST / + cd / + split -l $lines_per_node -d -a 3 $TRAIN_LIST train.list + CURRENT_LIST=$(printf "train.list%03d" $PADDLE_INIT_TRAINER_ID) + # always use /train.list for paddle v1 for each node. + echo "File for current node ${CURRENT_LIST}" + sleep 10 + cp $CURRENT_LIST train.list + + cd $TRAINER_PACKAGE + + stdbuf -oL paddle train \ + --port=$PADDLE_INIT_PORT \ + --nics=$PADDLE_INIT_NICS \ + --ports_num=$PADDLE_INIT_PORTS_NUM \ + --ports_num_for_sparse=$PADDLE_INIT_PORTS_NUM_FOR_SPARSE \ + --num_passes=$PADDLE_INIT_NUM_PASSES \ + --trainer_count=$PADDLE_INIT_TRAINER_COUNT \ + --saving_period=1 \ + --log_period=20 \ + --local=0 \ + --rdma_tcp=tcp \ + --config=$TOPOLOGY \ + --use_gpu=$PADDLE_INIT_USE_GPU \ + --trainer_id=$PADDLE_INIT_TRAINER_ID \ + --save_dir=$OUTPUT \ + --pservers=$PADDLE_INIT_PSERVERS \ + --num_gradient_servers=$PADDLE_INIT_NUM_GRADIENT_SERVERS + # paddle v1 API does not allow any trainer failed. + check_trainer_ret $? + ;; + "v2") + stdbuf -oL sh -c "${ENTRY}" + # paddle v2 API does not allow any trainer failed. + check_trainer_ret $? + ;; + *) + ;; + esac +} + +usage() { + echo "usage: paddle_k8s []:" + echo " start_trainer [v1|v2] Start a trainer process with v1 or v2 API" + echo " start_pserver Start a pserver process" + echo " start_new_pserver Start a new pserver process" + echo " start_new_trainer Start a new triner process" +} + +case "$1" in + start_pserver) + start_pserver + ;; + start_trainer) + start_trainer $2 + ;; + start_new_trainer) + start_new_trainer + ;; + start_new_pserver) + start_new_pserver + ;; + start_master) + start_master + ;; + start_fluid) + start_fluid_process + ;; + --help) + usage + ;; + *) + usage + ;; +esac + diff --git a/benchmark/cluster/vgg16/fluid/pserver.yaml b/benchmark/cluster/vgg16/fluid/pserver.yaml new file mode 100644 index 0000000000..47d2380d2e --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/pserver.yaml @@ -0,0 +1,72 @@ +apiVersion: extensions/v1beta1 +kind: ReplicaSet +metadata: + name: vgg16job-pserver +spec: + replicas: 10 + template: + metadata: + labels: + paddle-job-pserver: vgg16job + spec: + hostNetwork: true + imagePullSecrets: + - name: job-registry-secret + containers: + - name: pserver + image: "registry.baidu.com/paddlepaddle/rawjob:vgg16_fluid" + imagePullPolicy: Always + ports: + - name: jobport-30236 + containerPort: 30236 + env: + - name: PADDLE_JOB_NAME + value: vgg16job + - name: MKL_NUM_THREADS + value: "1" + - name: TRAINING_ROLE + value: "PSERVER" + - name: TRAINERS + value: "20" + - name: PSERVERS + value: "10" + - name: TOPOLOGY + value: "" + - name: ENTRY + value: "MKL_NUM_THREADS=1 python /workspace/vgg16.py --local 0" + - name: TRAINER_PACKAGE + value: "/workspace" + - name: PADDLE_INIT_PORT + value: "30236" + - name: PADDLE_INIT_NICS + value: "xgbe0" + - name: PADDLE_INIT_TRAINER_COUNT + value: "1" + - name: PADDLE_INIT_PORTS_NUM + value: "1" + - name: PADDLE_INIT_PORTS_NUM_FOR_SPARSE + value: "1" + - name: PADDLE_INIT_NUM_GRADIENT_SERVERS + value: "20" + - name: PADDLE_INIT_NUM_PASSES + value: "1" + - name: PADDLE_INIT_USE_GPU + value: "0" + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: "metadata.namespace" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: "status.podIP" + command: ["paddle_k8s", "start_fluid"] + resources: + requests: + memory: 10Gi + cpu: 4 + limits: + memory: 10Gi + cpu: 4 diff --git a/benchmark/cluster/vgg16/fluid/reader.py b/benchmark/cluster/vgg16/fluid/reader.py new file mode 100644 index 0000000000..c5161ddea2 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/reader.py @@ -0,0 +1,2 @@ +import paddle.v2 as paddle +paddle.dataset.cifar.train10() diff --git a/benchmark/cluster/vgg16/fluid/trainer.yaml b/benchmark/cluster/vgg16/fluid/trainer.yaml new file mode 100644 index 0000000000..bada190764 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/trainer.yaml @@ -0,0 +1,69 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: vgg16job-trainer +spec: + parallelism: 20 + completions: 20 + template: + metadata: + labels: + paddle-job: vgg16job + spec: + imagePullSecrets: + - name: job-registry-secret + hostNetwork: true + containers: + - name: trainer + image: "registry.baidu.com/paddlepaddle/rawjob:vgg16_fluid" + imagePullPolicy: Always + command: ["paddle_k8s", "start_trainer", "v2"] + env: + - name: PADDLE_JOB_NAME + value: vgg16job + - name: TRAINING_ROLE + value: "TRAINER" + - name: TRAINERS + value: "20" + - name: PSERVERS + value: "10" + - name: TOPOLOGY + value: "" + - name: ENTRY + value: "cd /workspace && MKL_NUM_THREADS=1 python /workspace/vgg16.py" + - name: TRAINER_PACKAGE + value: "/workspace" + - name: PADDLE_INIT_PORT + value: "30236" + - name: PADDLE_INIT_NICS + value: "xgbe0" + - name: PADDLE_INIT_TRAINER_COUNT + value: "1" + - name: PADDLE_INIT_PORTS_NUM + value: "1" + - name: PADDLE_INIT_PORTS_NUM_FOR_SPARSE + value: "1" + - name: PADDLE_INIT_NUM_GRADIENT_SERVERS + value: "20" + - name: PADDLE_INIT_NUM_PASSES + value: "1" + - name: PADDLE_INIT_USE_GPU + value: "0" + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: "metadata.namespace" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: "status.podIP" + resources: + requests: + memory: 40Gi + cpu: 2 + limits: + memory: 40Gi + cpu: 2 + restartPolicy: Never diff --git a/benchmark/cluster/vgg16/fluid/vgg16.py b/benchmark/cluster/vgg16/fluid/vgg16.py new file mode 100644 index 0000000000..0595a28784 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/vgg16.py @@ -0,0 +1,248 @@ +"""VGG16 benchmark in Fluid""" +from __future__ import print_function + +import sys +import time +import numpy as np +import paddle.v2 as paddle +import paddle.v2.fluid as fluid +import paddle.v2.fluid.core as core +import argparse +import functools +import os + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + '--batch_size', type=int, default=128, help="Batch size for training.") +parser.add_argument( + '--learning_rate', + type=float, + default=1e-3, + help="Learning rate for training.") +parser.add_argument('--num_passes', type=int, default=50, help="No. of passes.") +parser.add_argument( + '--device', + type=str, + default='CPU', + choices=['CPU', 'GPU'], + help="The device type.") +parser.add_argument( + '--data_format', + type=str, + default='NCHW', + choices=['NCHW', 'NHWC'], + help='The data order, now only support NCHW.') +parser.add_argument( + '--data_set', + type=str, + default='cifar10', + choices=['cifar10', 'flowers'], + help='Optional dataset for benchmark.') +parser.add_argument( + '--local', + type=str2bool, + default=True, + help='Whether to run as local mode.') +args = parser.parse_args() + + +def vgg16_bn_drop(input): + def conv_block(input, num_filter, groups, dropouts): + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max') + + conv1 = conv_block(input, 64, 2, [0.3, 0]) + conv2 = conv_block(conv1, 128, 2, [0.4, 0]) + conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0]) + conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0]) + conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0]) + + drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5) + fc1 = fluid.layers.fc(input=drop, size=512, act=None) + bn = fluid.layers.batch_norm(input=fc1, act='relu') + drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5) + fc2 = fluid.layers.fc(input=drop2, size=512, act=None) + return fc2 + + +def main(): + if args.data_set == "cifar10": + classdim = 10 + if args.data_format == 'NCHW': + data_shape = [3, 32, 32] + else: + data_shape = [32, 32, 3] + else: + classdim = 102 + if args.data_format == 'NCHW': + data_shape = [3, 224, 224] + else: + data_shape = [224, 224, 3] + + # Input data + images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + + # Train program + net = vgg16_bn_drop(images) + predict = fluid.layers.fc(input=net, size=classdim, act='softmax') + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) + + # Evaluator + accuracy = fluid.evaluator.Accuracy(input=predict, label=label) + + # inference program + inference_program = fluid.default_main_program().clone() + with fluid.program_guard(inference_program): + test_target = accuracy.metrics + accuracy.states + inference_program = fluid.io.get_inference_program(test_target) + + # Optimization + optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) + optimize_ops, params_grads = optimizer.minimize(avg_cost) + + # Initialize executor + place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0) + exe = fluid.Executor(place) + + + # test + def test(exe): + accuracy.reset(exe) + for batch_id, data in enumerate(test_reader()): + img_data = np.array(map(lambda x: x[0].reshape(data_shape), + data)).astype("float32") + y_data = np.array(map(lambda x: x[1], data)).astype("int64") + y_data = y_data.reshape([-1, 1]) + + exe.run(inference_program, + feed={"pixel": img_data, + "label": y_data}) + + return accuracy.eval(exe) + + def train_loop(exe, trainer_prog): + iters = 0 + for pass_id in range(args.num_passes): + # train + start_time = time.time() + num_samples = 0 + accuracy.reset(exe) + for batch_id, data in enumerate(train_reader()): + img_data = np.array(map(lambda x: x[0].reshape(data_shape), + data)).astype("float32") + y_data = np.array(map(lambda x: x[1], data)).astype("int64") + y_data = y_data.reshape([-1, 1]) + + loss, acc = exe.run(trainer_prog, + feed={"pixel": img_data, + "label": y_data}, + fetch_list=[avg_cost] + accuracy.metrics) + iters += 1 + num_samples += len(data) + print( + "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f" % + (pass_id, iters, loss, acc) + ) # The accuracy is the accumulation of batches, but not the current batch. + + pass_elapsed = time.time() - start_time + pass_train_acc = accuracy.eval(exe) + pass_test_acc = test(exe) + print( + "Pass = %d, Training performance = %f imgs/s, Train accuracy = %f, Test accuracy = %f\n" + % (pass_id, num_samples / pass_elapsed, pass_train_acc, + pass_test_acc)) + + if args.local: + # Parameter initialization + exe.run(fluid.default_startup_program()) + + # data reader + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.cifar.train10() + if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), + buf_size=5120), + batch_size=args.batch_size) + test_reader = paddle.batch( + paddle.dataset.cifar.test10() + if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), + batch_size=args.batch_size) + train_loop(exe, fluid.default_main_program()) + else: + pserver_ips = os.getenv("PADDLE_INIT_PSERVERS") # all pserver endpoints + eplist = [] + for ip in pserver_ips.split(","): + eplist.append(':'.join([ip, "6174"])) + pserver_endpoints = ",".join(eplist) + print("pserver endpoints: ", pserver_endpoints) + trainers = int(os.getenv("TRAINERS")) # total trainer count + current_endpoint = os.getenv("POD_IP") + ":6174" # current pserver endpoint + training_role = os.getenv("TRAINING_ROLE", + "TRAINER") # get the training role: trainer/pserver + t = fluid.DistributeTranspiler() + t.transpile( + optimize_ops, params_grads, pservers=pserver_endpoints, trainers=trainers) + + if training_role == "PSERVER": + if not current_endpoint: + print("need env SERVER_ENDPOINT") + exit(1) + pserver_prog = t.get_pserver_program(current_endpoint) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + print("starting server side startup") + exe.run(pserver_startup) + print("starting parameter server...") + exe.run(pserver_prog) + elif training_role == "TRAINER": + # Parameter initialization + exe.run(fluid.default_startup_program()) + + # data reader + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.cifar.train10() + if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), + buf_size=5120), + batch_size=args.batch_size) + test_reader = paddle.batch( + paddle.dataset.cifar.test10() + if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), + batch_size=args.batch_size) + + trainer_prog = t.get_trainer_program() + feeder = fluid.DataFeeder(feed_list=[images, label], place=place) + # TODO(typhoonzero): change trainer startup program to fetch parameters from pserver + exe.run(fluid.default_startup_program()) + train_loop(exe, trainer_prog) + else: + print("environment var TRAINER_ROLE should be TRAINER os PSERVER") + + +def print_arguments(): + print('----------- Configuration Arguments -----------') + for arg, value in sorted(vars(args).iteritems()): + print('%s: %s' % (arg, value)) + print('------------------------------------------------') + + +if __name__ == "__main__": + print_arguments() + main() diff --git a/benchmark/cluster/v2/Dockerfile b/benchmark/cluster/vgg16/v2/Dockerfile similarity index 81% rename from benchmark/cluster/v2/Dockerfile rename to benchmark/cluster/vgg16/v2/Dockerfile index ac85b1a7d0..5f129a8e32 100644 --- a/benchmark/cluster/v2/Dockerfile +++ b/benchmark/cluster/vgg16/v2/Dockerfile @@ -3,3 +3,5 @@ RUN mkdir -p /workspace ADD reader.py /workspace/ RUN python /workspace/reader.py ADD vgg16.py /workspace/ + +ADD vgg16_fluid.py /workspace diff --git a/benchmark/cluster/v2/pserver.yaml b/benchmark/cluster/vgg16/v2/pserver.yaml similarity index 100% rename from benchmark/cluster/v2/pserver.yaml rename to benchmark/cluster/vgg16/v2/pserver.yaml diff --git a/benchmark/cluster/v2/reader.py b/benchmark/cluster/vgg16/v2/reader.py similarity index 100% rename from benchmark/cluster/v2/reader.py rename to benchmark/cluster/vgg16/v2/reader.py diff --git a/benchmark/cluster/v2/trainer.yaml b/benchmark/cluster/vgg16/v2/trainer.yaml similarity index 100% rename from benchmark/cluster/v2/trainer.yaml rename to benchmark/cluster/vgg16/v2/trainer.yaml diff --git a/benchmark/cluster/v2/vgg16.py b/benchmark/cluster/vgg16/v2/vgg16.py similarity index 100% rename from benchmark/cluster/v2/vgg16.py rename to benchmark/cluster/vgg16/v2/vgg16.py From 19c554f9e4ef5c96e47f65efd44e2524417e38d7 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 19 Jan 2018 19:19:35 +0800 Subject: [PATCH 133/158] update --- python/paddle/v2/fluid/clip.py | 82 +++++++++---------- .../v2/fluid/tests/test_gradient_clip.py | 44 +++++----- 2 files changed, 59 insertions(+), 67 deletions(-) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index d97cd9ecc9..fb0907c9f4 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -112,58 +112,52 @@ class GradientClipByNorm(BaseGradientClipAttr): class GradientClipByGlobalNorm(BaseGradientClipAttr): - global_norm_var = None - local_norm_var = None - clip_norm_var = None - scale_var = None - - @classmethod - def init(cls, clip_norm): - if not (isinstance(clip_norm, int) or isinstance(clip_norm, float)): - raise TypeError("The 'clip_norm' must be a value of int or float") - - cls.global_norm_var = layers.fill_constant( - shape=[1], dtype="float32", value=0.0) - cls.local_norm_var = layers.create_tensor(dtype="float32") - cls.clip_norm_var = layers.fill_constant( - shape=[1], dtype="float32", value=clip_norm) - - @classmethod - def check_init(cls): - if not (isinstance(cls.global_norm_var, framework.Variable) and - isinstance(cls.local_norm_var, framework.Variable) and - isinstance(cls.clip_norm_var, framework.Variable)): - raise ValueError( - "Class 'GradientClipByGlobalNorm' has not been properly initialized. \ - Please call GradientClipByGlobalNorm.init() first.") + def __init__(self, clip_norm, group_name="default_group"): + if not isinstance(group_name, basestring): + raise TypeError("'group_name' must be a basestring.") + + self.clip_norm = clip_norm + self.group_name = group_name def process_context(self, context, param, grad): - cls = self.__class__ - cls.check_init() + if self.group_name not in context: + context[self.group_name] = [] + context[self.group_name + "_clip_value"] = self.clip_norm + context[self.group_name + "_clip"] = layers.fill_constant( + shape=[1], dtype="float32", value=self.clip_norm) + else: + if not self.clip_norm == context[self.group_name + "_clip_value"]: + raise ValueError( + "All parameters' 'clip_norm' of a same group should be the same" + ) - cls.local_norm_var = layers.reduce_sum( - input=layers.pow(x=grad, factor=2.0)) - layers.sums( - input=[cls.local_norm_var, cls.global_norm_var], - out=[cls.global_norm_var]) + local_norm_var = layers.reduce_sum(input=layers.pow(x=grad, factor=2.0)) + context[self.group_name].append(local_norm_var) - def create_operators(self, param, grad): - cls = self.__class__ - cls.check_init() + self.context = context - if cls.scale_var is None: - layers.sqrt(x=cls.global_norm_var, out=cls.global_norm_var) - cls.scale_var = layers.elementwise_div( - x=cls.clip_norm_var, + def create_operators(self, param, grad): + group_scale_name = self.group_name + "_scale" + if group_scale_name not in self.context: + group_norm_var = layers.sums(input=self.context[self.group_name]) + layers.sqrt(x=group_norm_var, out=group_norm_var) + clip_var = self.context[self.group_name + "_clip"] + group_scale_var = layers.elementwise_div( + x=clip_var, y=layers.elementwise_max( - x=cls.clip_norm_var, y=cls.global_norm_var)) - assert cls.scale_var.shape == (1L, ) + x=clip_var, y=group_norm_var)) + assert group_scale_var.shape == (1L, ) + self.context[group_scale_name] = group_scale_var - new_grad = layers.elementwise_mul(x=grad, y=cls.scale_var) + new_grad = layers.elementwise_mul( + x=grad, y=self.context[group_scale_name]) return param, new_grad -def gradient_clip_by_global_norm(clip_norm, param_list=None, program=None): +def gradient_clip_by_global_norm(clip_norm, + param_list=None, + group_name="default_group", + program=None): if program is None: program = framework.default_main_program() if param_list is None: @@ -175,9 +169,9 @@ def gradient_clip_by_global_norm(clip_norm, param_list=None, program=None): "'param_list' should be a list of Parameter or basestring(parameter's name)." ) - GradientClipByGlobalNorm.init(clip_norm) for param in param_list: - param.gradient_clip_attr = GradientClipByGlobalNorm() + param.gradient_clip_attr = GradientClipByGlobalNorm(clip_norm, + group_name) def append_gradient_clip_ops(param_grad): diff --git a/python/paddle/v2/fluid/tests/test_gradient_clip.py b/python/paddle/v2/fluid/tests/test_gradient_clip.py index 4fb7f0b2cb..75c5fd9892 100644 --- a/python/paddle/v2/fluid/tests/test_gradient_clip.py +++ b/python/paddle/v2/fluid/tests/test_gradient_clip.py @@ -15,21 +15,10 @@ import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid - -def _get_global_param_norm_(params_grads): - res = fluid.layers.fill_constant(shape=[1], dtype="float32", value=0.0) - for _, grad in params_grads: - norm_var = fluid.layers.reduce_sum( - input=fluid.layers.pow(x=grad, factor=2.0)) - fluid.layers.sums(input=[norm_var, res], out=[res]) - fluid.layers.sqrt(x=res, out=res) - return res - - BATCH_SIZE = 128 -CLIP = 0.5 -prog = fluid.framework.Program() +CLIP = 1 +prog = fluid.framework.Program() with fluid.program_guard(main_program=prog): image = fluid.layers.data(name='x', shape=[784], dtype='float32') @@ -49,13 +38,12 @@ avg_cost_clip = prog_clip.block(0).var(avg_cost.name) p_g = fluid.backward.append_backward(loss=avg_cost) p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip) -with fluid.program_guard(main_program=prog): - gloabl_norm = _get_global_param_norm_(p_g) - with fluid.program_guard(main_program=prog_clip): fluid.clip.gradient_clip_by_global_norm(clip_norm=CLIP) p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip) - gloabl_norm_clip = _get_global_param_norm_(p_g_clip) + +grad_list = [elem[1] for elem in p_g] +grad_clip_list = [elem[1] for elem in p_g_clip] train_reader = paddle.batch( paddle.reader.shuffle( @@ -72,11 +60,21 @@ for data in train_reader(): count += 1 if count > 5: break - out, = exe.run(prog, feed=feeder.feed(data), fetch_list=[gloabl_norm]) - out_clip, = exe.run(prog_clip, - feed=feeder.feed(data), - fetch_list=[gloabl_norm_clip]) - - if not np.allclose(out_clip, np.minimum(out, np.array([CLIP]))): + out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list) + out_clip = exe.run(prog_clip, + feed=feeder.feed(data), + fetch_list=grad_clip_list) + global_norm = 0 + for v in out[1:]: + global_norm += np.sum(np.power(v, 2)) + global_norm = np.sqrt(global_norm) + + global_norm_clip = 0 + for v in out_clip[1:]: + global_norm_clip += np.sum(np.power(v, 2)) + global_norm_clip = np.sqrt(global_norm_clip) + + if not np.isclose( + a=global_norm_clip, b=np.minimum(global_norm, CLIP), rtol=5e-3): exit(1) exit(0) From 0468422d06eb2094824141002dfb63f0dc03c513 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 19 Jan 2018 19:32:24 +0800 Subject: [PATCH 134/158] follow comments --- paddle/operators/math/matmul.h | 11 ++++++----- paddle/operators/matmul_op.cc | 22 ++++++++++++---------- paddle/operators/matmul_op.h | 3 ++- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/paddle/operators/math/matmul.h b/paddle/operators/math/matmul.h index ca41201e12..88341412fb 100644 --- a/paddle/operators/math/matmul.h +++ b/paddle/operators/math/matmul.h @@ -45,15 +45,16 @@ class MatMulFunctor { std::vector out_dim; int64_t batch_count = 1; if (dim_a.size() > 3) { - PADDLE_ENFORCE(dim_b.size() > 3, + PADDLE_ENFORCE(dim_b.size() == dim_a.size(), "The dimensions of X and Y must be the same, and both of " "them should be %d-dimensional.", dim_b.size()); - // The previous Rank-2 dimensions are accumulated on the batch_count. + // The front rank-2 dimensions are accumulated on the batch_count, and the + // last two dimensions are used for matrix multiplication. for (int j = 0; j < dim_a.size() - 2; ++j) { - PADDLE_ENFORCE(dim_b[j] == dim_a[j], - "The dimensions of X[%d] and Y[%d] must be the same.", j, - j); + PADDLE_ENFORCE_EQ(dim_b[j], dim_a[j], + "The %d-th dimension of X and Y must be the same.", + j); out_dim.push_back(dim_a[j]); batch_count *= dim_a[j]; } diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index 1707ed7e7d..d395dfd81b 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -45,16 +45,18 @@ class MatMulOp : public framework::OperatorWithKernel { std::vector out_dim; int64_t batch_count = 1; if (dim_x.size() > 3) { - PADDLE_ENFORCE(dim_y.size() == dim_x.size(), - "The dimensions of X and Y must be the same, and both of " - "them should be %d-dimensional.", - dim_x.size()); + PADDLE_ENFORCE_EQ( + dim_y.size(), dim_x.size(), + "The dimensions of X and Y must be the same, and both of " + "them should be %d-dimensional.", + dim_x.size()); - // The previous Rank-2 dimensions are accumulated on the batch_count. + // The front rank-2 dimensions are accumulated on the batch_count, and the + // last two dimensions are used for matrix multiplication. for (int j = 0; j < dim_x.size() - 2; ++j) { - PADDLE_ENFORCE(dim_y[j] == dim_x[j], - "The dimensions of X[%d] and Y[%d] must be the same.", j, - j); + PADDLE_ENFORCE_EQ(dim_y[j], dim_x[j], + "The %d-th dimension of X and Y must be the same.", + j); out_dim.push_back(dim_x[j]); batch_count *= dim_x[j]; } @@ -191,10 +193,10 @@ Examples without transpose: The behavior is designed to be similar to the `numpy.matmul` function. The differences are: -- When the rank of the input is greater than 3, the rank of X and - Y must be equal, and the former rank-2 dimensions are equal. - When the rank of the input data is less than or equal to 3, it is similar to the `numpy.matmul` function. +- When the rank of the input is greater than 3, the rank of X and + Y must be equal, and the front `rank - 2` dimensions must be equal. - We add `transpose_X` and `transpose_Y` flags. Both the input `X` and `Y` can carry the LoD (Level of Details) information, diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index cf60234295..4935db839c 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -138,7 +138,8 @@ class MatMulGradKernel : public framework::OpKernel { } int batch_count = 0; - // The previous Rank-2 dimensions are accumulated on the batch_count. + // The front rank-2 dimensions are accumulated on the batch_count, and the + // last two dimensions are used for matrix multiplication. if (x_dims.size() > 3) { batch_count = accumulate(x_dims.begin(), x_dims.end() - 2, 1, std::multiplies()); From 2ac46d538da103929e58c0500ca0baca391f39bc Mon Sep 17 00:00:00 2001 From: gongweibao Date: Fri, 19 Jan 2018 20:28:13 +0800 Subject: [PATCH 135/158] Add distribution implement of image classification. (#7687) Add distribution implement of image classification --- .../notest_dist_image_classification.py | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 python/paddle/v2/fluid/tests/book_distribute/notest_dist_image_classification.py diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_image_classification.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_image_classification.py new file mode 100644 index 0000000000..218dea31e1 --- /dev/null +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_image_classification.py @@ -0,0 +1,173 @@ +#Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +from __future__ import print_function + +import sys + +import paddle.v2 as paddle +import paddle.v2.fluid as fluid +import os +import sys + +TRAINERS = 5 +BATCH_SIZE = 128 +PASS_NUM = 100 + + +def resnet_cifar10(input, depth=32): + def conv_bn_layer(input, ch_out, filter_size, stride, padding, act='relu'): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=False) + return fluid.layers.batch_norm(input=tmp, act=act) + + def shortcut(input, ch_in, ch_out, stride): + if ch_in != ch_out: + return conv_bn_layer(input, ch_out, 1, stride, 0, None) + else: + return input + + def basicblock(input, ch_in, ch_out, stride): + tmp = conv_bn_layer(input, ch_out, 3, stride, 1) + tmp = conv_bn_layer(tmp, ch_out, 3, 1, 1, act=None) + short = shortcut(input, ch_in, ch_out, stride) + return fluid.layers.elementwise_add(x=tmp, y=short, act='relu') + + def layer_warp(block_func, input, ch_in, ch_out, count, stride): + tmp = block_func(input, ch_in, ch_out, stride) + for i in range(1, count): + tmp = block_func(tmp, ch_out, ch_out, 1) + return tmp + + assert (depth - 2) % 6 == 0 + n = (depth - 2) / 6 + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1) + res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) + res2 = layer_warp(basicblock, res1, 16, 32, n, 2) + res3 = layer_warp(basicblock, res2, 32, 64, n, 2) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1) + return pool + + +def vgg16_bn_drop(input): + def conv_block(input, num_filter, groups, dropouts): + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max') + + conv1 = conv_block(input, 64, 2, [0.3, 0]) + conv2 = conv_block(conv1, 128, 2, [0.4, 0]) + conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0]) + conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0]) + conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0]) + + drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5) + fc1 = fluid.layers.fc(input=drop, size=512, act=None) + bn = fluid.layers.batch_norm(input=fc1, act='relu') + drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5) + fc2 = fluid.layers.fc(input=drop2, size=512, act=None) + return fc2 + + +classdim = 10 +data_shape = [3, 32, 32] + +images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32') +label = fluid.layers.data(name='label', shape=[1], dtype='int64') + +net_type = "vgg" +if len(sys.argv) >= 2: + net_type = sys.argv[1] + +if net_type == "vgg": + print("train vgg net") + net = vgg16_bn_drop(images) +elif net_type == "resnet": + print("train resnet") + net = resnet_cifar10(images, 32) +else: + raise ValueError("%s network is not supported" % net_type) + +predict = fluid.layers.fc(input=net, size=classdim, act='softmax') +cost = fluid.layers.cross_entropy(input=predict, label=label) +avg_cost = fluid.layers.mean(x=cost) + +optimizer = fluid.optimizer.Adam(learning_rate=0.001) +optimize_ops, params_grads = optimizer.minimize(avg_cost) + +accuracy = fluid.evaluator.Accuracy(input=predict, label=label) + +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.cifar.train10(), buf_size=128 * 10), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +exe = fluid.Executor(place) + +t = fluid.DistributeTranspiler() +# all parameter server endpoints list for spliting parameters +pserver_endpoints = os.getenv("PSERVERS") +# server endpoint for current node +current_endpoint = os.getenv("SERVER_ENDPOINT") +# run as trainer or parameter server +training_role = os.getenv("TRAINING_ROLE", + "TRAINER") # get the training role: trainer/pserver +t.transpile( + optimize_ops, params_grads, pservers=pserver_endpoints, trainers=TRAINERS) + +if training_role == "PSERVER": + if not current_endpoint: + print("need env SERVER_ENDPOINT") + exit(1) + print("start pserver at:", current_endpoint) + pserver_prog = t.get_pserver_program(current_endpoint) + pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + exe.run(pserver_startup) + exe.run(pserver_prog) + print("pserver run end") +elif training_role == "TRAINER": + print("start trainer") + trainer_prog = t.get_trainer_program() + feeder = fluid.DataFeeder(place=place, feed_list=[images, label]) + exe.run(fluid.default_startup_program()) + for pass_id in range(PASS_NUM): + accuracy.reset(exe) + for data in train_reader(): + loss, acc = exe.run(trainer_prog, + feed=feeder.feed(data), + fetch_list=[avg_cost] + accuracy.metrics) + pass_acc = accuracy.eval(exe) + print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str( + pass_acc)) + # this model is slow, so if we can train two mini batch, we think it works properly. + print("trainer run end") +else: + print("environment var TRAINER_ROLE should be TRAINER os PSERVER") +exit(1) From ce93eea88f25fca141f880fb146047b2620d9466 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 19 Jan 2018 19:59:04 +0800 Subject: [PATCH 136/158] refine unit test --- .../paddle/v2/fluid/tests/test_matmul_op.py | 72 ++++++++++++------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/python/paddle/v2/fluid/tests/test_matmul_op.py b/python/paddle/v2/fluid/tests/test_matmul_op.py index 0548a9bbfe..7151b1f5c8 100644 --- a/python/paddle/v2/fluid/tests/test_matmul_op.py +++ b/python/paddle/v2/fluid/tests/test_matmul_op.py @@ -59,19 +59,18 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): X = X.reshape((X.size, 1)) elif X.ndim == 2: X = X.T - elif X.ndim == 3: - X = np.transpose(X, (0, 2, 1)) else: - raise ValueError('X must have between 1 and 3 dimensions') + dim = [i for i in range(len(X.shape))] + dim[-1], dim[len(X.shape) - 2] = dim[len(X.shape) - 2], dim[-1] + X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: Y = Y.reshape((1, Y.size)) - elif Y.ndim == 2: - Y = Y.T - elif Y.ndim == 3: - Y = np.transpose(Y, (0, 2, 1)) else: - raise ValueError('Y must have between 1 and 3 dimensions') + dim = [i for i in range(len(Y.shape))] + dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] + Y = np.transpose(Y, tuple(dim)) + Out = np.matmul(X, Y) if not Out.shape: # We do not support 0-dimensional Tensors (scalars). So where @@ -120,30 +119,49 @@ for dim_X in [1, 2, 3]: dim_X, dim_Y, transpose_X, transpose_Y)) shape_X, shape_Y = generate_compatible_shapes( dim_X, dim_Y, transpose_X, transpose_Y) - test_class = type(test_name, (Generator, OpTest), { + globals()[test_name] = type(test_name, (Generator, OpTest), { 'shape_X': shape_X, 'shape_Y': shape_Y, 'transpose_X': transpose_X, 'transpose_Y': transpose_Y, }) - globals()[test_name] = test_class - -# Test case 4-dim -dim_X = 4 -dim_Y = 4 -transpose_X = False -transpose_Y = False -test_name = ('TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim_X, dim_Y, transpose_X, transpose_Y)) - -shape_X = [2, 2, 2, 3] -shape_Y = [2, 2, 3, 4] -test_class = type(test_name, (Generator, OpTest), { - 'shape_X': shape_X, - 'shape_Y': shape_Y, - 'transpose_X': transpose_X, - 'transpose_Y': transpose_Y, -}) + + +def generate_compatible_shapes(dim, transpose_X, transpose_Y): + M = 2 + N = 4 + K = 3 + shape_X = [2 for _ in range(dim - 2)] + shape_Y = [2 for _ in range(dim - 2)] + + if transpose_X: + shape_X = shape_X + [K, M] + else: + shape_X = shape_X + [M, K] + + if transpose_Y: + shape_Y = shape_Y + [N, K] + else: + shape_Y = shape_Y + [K, N] + + return shape_X, shape_Y + + +# Test case n-dim +for dim in [4]: + for transpose_X in [False, True]: + for transpose_Y in [False, True]: + test_name = ( + 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim, dim, transpose_X, transpose_Y)) + shape_X, shape_Y = generate_compatible_shapes(dim, transpose_X, + transpose_Y) + globals()[test_name] = type(test_name, (Generator, OpTest), { + 'shape_X': shape_X, + 'shape_Y': shape_Y, + 'transpose_X': transpose_X, + 'transpose_Y': transpose_Y, + }) if __name__ == "__main__": unittest.main() From e22039f82b9612199a3f8d268a0e365f00a4fac5 Mon Sep 17 00:00:00 2001 From: wangmeng28 Date: Fri, 19 Jan 2018 21:25:17 +0800 Subject: [PATCH 137/158] Fix priorbox layer when multiple values given in min_size --- paddle/gserver/layers/PriorBox.cpp | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/paddle/gserver/layers/PriorBox.cpp b/paddle/gserver/layers/PriorBox.cpp index 331bc7672e..7018a9eb2f 100644 --- a/paddle/gserver/layers/PriorBox.cpp +++ b/paddle/gserver/layers/PriorBox.cpp @@ -65,14 +65,15 @@ bool PriorBoxLayer::init(const LayerMap& layerMap, std::copy(pbConf.aspect_ratio().begin(), pbConf.aspect_ratio().end(), std::back_inserter(tmp)); - // flip - int inputRatioLength = tmp.size(); - for (int index = 0; index < inputRatioLength; index++) { - aspectRatio_.push_back(tmp[index]); - aspectRatio_.push_back(1 / tmp[index]); + // flip aspect ratios + for (int index = 0; index < tmp.size(); index++) { + real ar = tmp[index]; + if (fabs(ar - 1.) < 1e-6) continue; + aspectRatio_.push_back(ar); + aspectRatio_.push_back(1. / ar); } - numPriors_ = aspectRatio_.size(); - if (maxSize_.size() > 0) numPriors_++; + numPriors_ = aspectRatio_.size() * minSize_.size(); + if (maxSize_.size() > 0) numPriors_ += maxSize_.size() * minSize_.size(); return true; } @@ -99,13 +100,12 @@ void PriorBoxLayer::forward(PassType passType) { for (int w = 0; w < layerWidth; ++w) { real centerX = (w + 0.5) * stepW; real centerY = (h + 0.5) * stepH; - real minSize = 0; for (size_t s = 0; s < minSize_.size(); s++) { - // first prior. - minSize = minSize_[s]; + // square prior with size minSize + real minSize = minSize_[s]; real boxWidth = minSize; real boxHeight = minSize; - // xmin, ymin, xmax, ymax. + // xmin, ymin, xmax, ymax tmpPtr[idx++] = (centerX - boxWidth / 2.) / imageWidth; tmpPtr[idx++] = (centerY - boxHeight / 2.) / imageHeight; tmpPtr[idx++] = (centerX + boxWidth / 2.) / imageWidth; @@ -115,7 +115,7 @@ void PriorBoxLayer::forward(PassType passType) { if (maxSize_.size() > 0) { CHECK_EQ(minSize_.size(), maxSize_.size()); - // second prior. + // square prior with size sqrt(minSize * maxSize) for (size_t s = 0; s < maxSize_.size(); s++) { real maxSize = maxSize_[s]; boxWidth = boxHeight = sqrt(minSize * maxSize); @@ -127,19 +127,19 @@ void PriorBoxLayer::forward(PassType passType) { for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; } } - } - // rest of priors. - for (size_t r = 0; r < aspectRatio_.size(); r++) { - real ar = aspectRatio_[r]; - if (fabs(ar - 1.) < 1e-6) continue; - real boxWidth = minSize * sqrt(ar); - real boxHeight = minSize / sqrt(ar); - tmpPtr[idx++] = (centerX - boxWidth / 2.) / imageWidth; - tmpPtr[idx++] = (centerY - boxHeight / 2.) / imageHeight; - tmpPtr[idx++] = (centerX + boxWidth / 2.) / imageWidth; - tmpPtr[idx++] = (centerY + boxHeight / 2.) / imageHeight; - // set the variance. - for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; + + // priors with different aspect ratios + for (size_t r = 0; r < aspectRatio_.size(); r++) { + real ar = aspectRatio_[r]; + boxWidth = minSize * sqrt(ar); + boxHeight = minSize / sqrt(ar); + tmpPtr[idx++] = (centerX - boxWidth / 2.) / imageWidth; + tmpPtr[idx++] = (centerY - boxHeight / 2.) / imageHeight; + tmpPtr[idx++] = (centerX + boxWidth / 2.) / imageWidth; + tmpPtr[idx++] = (centerY + boxHeight / 2.) / imageHeight; + // set the variance. + for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; + } } } } From 871b0e68543cf5fffed63410a8fe6a28bf367813 Mon Sep 17 00:00:00 2001 From: wangmeng28 Date: Sat, 20 Jan 2018 00:31:29 +0800 Subject: [PATCH 138/158] Simplify priorbox layer --- paddle/gserver/layers/PriorBox.cpp | 44 +++++++++++++----------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/paddle/gserver/layers/PriorBox.cpp b/paddle/gserver/layers/PriorBox.cpp index 7018a9eb2f..337b9ba7bc 100644 --- a/paddle/gserver/layers/PriorBox.cpp +++ b/paddle/gserver/layers/PriorBox.cpp @@ -65,6 +65,9 @@ bool PriorBoxLayer::init(const LayerMap& layerMap, std::copy(pbConf.aspect_ratio().begin(), pbConf.aspect_ratio().end(), std::back_inserter(tmp)); + + if (maxSize_.size() > 0) CHECK_EQ(minSize_.size(), maxSize_.size()); + // flip aspect ratios for (int index = 0; index < tmp.size(); index++) { real ar = tmp[index]; @@ -72,8 +75,9 @@ bool PriorBoxLayer::init(const LayerMap& layerMap, aspectRatio_.push_back(ar); aspectRatio_.push_back(1. / ar); } - numPriors_ = aspectRatio_.size() * minSize_.size(); - if (maxSize_.size() > 0) numPriors_ += maxSize_.size() * minSize_.size(); + + numPriors_ = aspectRatio_.size() * minSize_.size() + maxSize_.size(); + return true; } @@ -101,32 +105,9 @@ void PriorBoxLayer::forward(PassType passType) { real centerX = (w + 0.5) * stepW; real centerY = (h + 0.5) * stepH; for (size_t s = 0; s < minSize_.size(); s++) { - // square prior with size minSize real minSize = minSize_[s]; real boxWidth = minSize; real boxHeight = minSize; - // xmin, ymin, xmax, ymax - tmpPtr[idx++] = (centerX - boxWidth / 2.) / imageWidth; - tmpPtr[idx++] = (centerY - boxHeight / 2.) / imageHeight; - tmpPtr[idx++] = (centerX + boxWidth / 2.) / imageWidth; - tmpPtr[idx++] = (centerY + boxHeight / 2.) / imageHeight; - // set the variance. - for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; - - if (maxSize_.size() > 0) { - CHECK_EQ(minSize_.size(), maxSize_.size()); - // square prior with size sqrt(minSize * maxSize) - for (size_t s = 0; s < maxSize_.size(); s++) { - real maxSize = maxSize_[s]; - boxWidth = boxHeight = sqrt(minSize * maxSize); - tmpPtr[idx++] = (centerX - boxWidth / 2.) / imageWidth; - tmpPtr[idx++] = (centerY - boxHeight / 2.) / imageHeight; - tmpPtr[idx++] = (centerX + boxWidth / 2.) / imageWidth; - tmpPtr[idx++] = (centerY + boxHeight / 2.) / imageHeight; - // set the variance. - for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; - } - } // priors with different aspect ratios for (size_t r = 0; r < aspectRatio_.size(); r++) { @@ -140,9 +121,22 @@ void PriorBoxLayer::forward(PassType passType) { // set the variance. for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; } + + if (maxSize_.size() > 0) { + // square prior with size sqrt(minSize * maxSize) + real maxSize = maxSize_[s]; + boxWidth = boxHeight = sqrt(minSize * maxSize); + tmpPtr[idx++] = (centerX - boxWidth / 2.) / imageWidth; + tmpPtr[idx++] = (centerY - boxHeight / 2.) / imageHeight; + tmpPtr[idx++] = (centerX + boxWidth / 2.) / imageWidth; + tmpPtr[idx++] = (centerY + boxHeight / 2.) / imageHeight; + // set the variance. + for (int t = 0; t < 4; t++) tmpPtr[idx++] = variance_[t]; + } } } } + // clip the prior's coordidate such that it is within [0, 1] for (int d = 0; d < dim * 2; ++d) if ((d % 8) < 4) From 84568d38e84edd75c83ce10ae6658517046d3ba1 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 19 Jan 2018 14:15:13 -0800 Subject: [PATCH 139/158] remove unused vars which block the build --- paddle/operators/recv_op.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index f9f1c134d7..e007d71fbc 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -35,9 +35,6 @@ namespace paddle { namespace operators { constexpr char kOptimizeBlock[] = "OptimizeBlock"; -constexpr int kCondStart = 0; -constexpr int kCondRunning = 1; -constexpr int kCondDone = 2; void RunServer(std::shared_ptr service) { service->RunSyncUpdate(); From 782ddc5f59631319a201d04cf88b142ab87e7e14 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Sun, 21 Jan 2018 10:03:56 +0800 Subject: [PATCH 140/158] follow comments --- paddle/operators/math/matmul.h | 2 +- paddle/operators/matmul_op.cc | 4 ++-- paddle/operators/matmul_op.h | 2 +- python/paddle/v2/fluid/tests/test_matmul_op.py | 9 +++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/paddle/operators/math/matmul.h b/paddle/operators/math/matmul.h index 88341412fb..ae7f1fe9be 100644 --- a/paddle/operators/math/matmul.h +++ b/paddle/operators/math/matmul.h @@ -49,7 +49,7 @@ class MatMulFunctor { "The dimensions of X and Y must be the same, and both of " "them should be %d-dimensional.", dim_b.size()); - // The front rank-2 dimensions are accumulated on the batch_count, and the + // The first rank-2 dimensions are accumulated on the batch_count, and the // last two dimensions are used for matrix multiplication. for (int j = 0; j < dim_a.size() - 2; ++j) { PADDLE_ENFORCE_EQ(dim_b[j], dim_a[j], diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index d395dfd81b..3336978c8d 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -51,7 +51,7 @@ class MatMulOp : public framework::OperatorWithKernel { "them should be %d-dimensional.", dim_x.size()); - // The front rank-2 dimensions are accumulated on the batch_count, and the + // The first rank-2 dimensions are accumulated on the batch_count, and the // last two dimensions are used for matrix multiplication. for (int j = 0; j < dim_x.size() - 2; ++j) { PADDLE_ENFORCE_EQ(dim_y[j], dim_x[j], @@ -196,7 +196,7 @@ The differences are: - When the rank of the input data is less than or equal to 3, it is similar to the `numpy.matmul` function. - When the rank of the input is greater than 3, the rank of X and - Y must be equal, and the front `rank - 2` dimensions must be equal. + Y must be equal, and the first `rank - 2` dimensions must be equal. - We add `transpose_X` and `transpose_Y` flags. Both the input `X` and `Y` can carry the LoD (Level of Details) information, diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 4935db839c..fe6a97465f 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -138,7 +138,7 @@ class MatMulGradKernel : public framework::OpKernel { } int batch_count = 0; - // The front rank-2 dimensions are accumulated on the batch_count, and the + // The first rank-2 dimensions are accumulated on the batch_count, and the // last two dimensions are used for matrix multiplication. if (x_dims.size() > 3) { batch_count = accumulate(x_dims.begin(), x_dims.end() - 2, 1, diff --git a/python/paddle/v2/fluid/tests/test_matmul_op.py b/python/paddle/v2/fluid/tests/test_matmul_op.py index 7151b1f5c8..2dba25e17d 100644 --- a/python/paddle/v2/fluid/tests/test_matmul_op.py +++ b/python/paddle/v2/fluid/tests/test_matmul_op.py @@ -127,6 +127,7 @@ for dim_X in [1, 2, 3]: }) +# Test case n-dim def generate_compatible_shapes(dim, transpose_X, transpose_Y): M = 2 N = 4 @@ -135,14 +136,14 @@ def generate_compatible_shapes(dim, transpose_X, transpose_Y): shape_Y = [2 for _ in range(dim - 2)] if transpose_X: - shape_X = shape_X + [K, M] + shape_X += [K, M] else: - shape_X = shape_X + [M, K] + shape_X += [M, K] if transpose_Y: - shape_Y = shape_Y + [N, K] + shape_Y += [N, K] else: - shape_Y = shape_Y + [K, N] + shape_Y += [K, N] return shape_X, shape_Y From d0a9393668e0e6e49cd7355e91cc2c4ea59db622 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Sun, 21 Jan 2018 16:31:57 +0800 Subject: [PATCH 141/158] Split SelectedRows to multiple pservers --- paddle/operators/recv_op.cc | 2 +- paddle/operators/split_selected_rows_op.cc | 21 ++---- paddle/operators/split_selected_rows_op.h | 68 +++++++++++++------ .../paddle/v2/fluid/distribute_transpiler.py | 32 ++++++--- .../tests/test_split_selected_rows_op.py | 41 +++++------ 5 files changed, 101 insertions(+), 63 deletions(-) diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 8d1479bdd6..93a7e84488 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -107,7 +107,7 @@ class RecvOp : public framework::OperatorBase { // TODO(typhoonzero): change this to a while_op for every cluster-batch. bool exit_flag = false; - int64_t barrier_size = param_count * fan_in; + size_t barrier_size = param_count * fan_in; while (!exit_flag) { // Get from multiple trainers, we don't care about the order in which // the gradients arrives, just add suffix 0~n and merge the gradient. diff --git a/paddle/operators/split_selected_rows_op.cc b/paddle/operators/split_selected_rows_op.cc index d9a023987b..0515ea13aa 100644 --- a/paddle/operators/split_selected_rows_op.cc +++ b/paddle/operators/split_selected_rows_op.cc @@ -23,8 +23,6 @@ class SplitSelectedRowsOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input SelectedRows."); AddOutput("Out", "The outputs of input SelectedRows.").AsDuplicable(); - AddAttr>("rows_sections", "Rows section for output.") - .SetDefault(std::vector({})); AddAttr>("height_sections", "Height for each output SelectedRows.") .SetDefault(std::vector({})); @@ -35,16 +33,16 @@ height_sections is only needed when need to split the dims of the original tenso Example: Input: - X.rows = {0, 7, 5} + X.rows = {7, 5} X.height = 12 Attr: - rows_sections = {1, 2} - height_sections = {} + height_sections = {4, 8} Out: - out0.rows = {0} - out0.height = 12 - out1.rows = {7, 5} - out2.height = 12 + out0.rows = {} + out0.height = 4 + + out1.rows = {5, 7} + out2.height = 8 )DOC"); } @@ -61,11 +59,6 @@ class SplitSelectedRowsOp : public framework::OperatorWithKernel { std::vector height_sections = ctx->Attrs().Get>("height_sections"); - std::vector rows_sections = - ctx->Attrs().Get>("rows_sections"); - PADDLE_ENFORCE_EQ( - rows_sections.size(), ctx->Outputs("Out").size(), - "The size of rows section should be the same with Outputs size."); int64_t n = ctx->Outputs("Out").size(); std::vector outs_dims; diff --git a/paddle/operators/split_selected_rows_op.h b/paddle/operators/split_selected_rows_op.h index 1cae53f1af..12e64e2901 100644 --- a/paddle/operators/split_selected_rows_op.h +++ b/paddle/operators/split_selected_rows_op.h @@ -16,40 +16,70 @@ limitations under the License. */ #include #include "paddle/framework/op_registry.h" +#include "paddle/operators/math/selected_rows_functor.h" namespace paddle { namespace operators { +static int FindOutIdx(int row, const std::vector& height_sections) { + int offset = 0; + for (size_t i = 0; i < height_sections.size(); ++i) { + if (row >= offset && row < (offset + height_sections[i])) { + return i; + } + offset += height_sections[i]; + } + return -1; +} + template class SplitSelectedRowsOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { auto* x = ctx.Input("X"); auto outs = ctx.MultiOutput("Out"); - - auto rows_sections = ctx.Attr>("rows_sections"); auto height_sections = ctx.Attr>("height_sections"); - int64_t n = outs.size(); - int offset = 0; + auto x_rows = x->rows(); + std::vector> outs_rows_idx; + outs_rows_idx.resize(outs.size()); - for (int64_t i = 0; i < n; ++i) { - framework::Vector out_rows; - for (int64_t j = 0; j < rows_sections[i]; ++j) { - out_rows.push_back(x->rows()[offset + j]); - } + auto row_numel = x->value().numel() / x->value().dims()[0]; + auto src = x->value().data(); + + for (size_t i = 0; i < x_rows.size(); ++i) { + int out_idx = FindOutIdx(x_rows[i], height_sections); + outs_rows_idx[out_idx].push_back(i); + } + auto place = ctx.GetPlace(); - auto& out = outs[i]; - auto x_dims = x->GetCompleteDims(); - x_dims[0] = rows_sections[i]; - out->mutable_value()->mutable_data(x_dims, ctx.GetPlace()); - framework::Copy(x->value().Slice(offset, rows_sections[i] + offset), - x->place(), ctx.device_context(), out->mutable_value()); - outs[i]->set_rows(out_rows); - if (height_sections.size()) { - outs[i]->set_height(height_sections[i]); + for (size_t i = 0; i < outs_rows_idx.size(); ++i) { + auto rows_idx = outs_rows_idx[i]; + if (rows_idx.size() > 0) { + auto dims = x->GetCompleteDims(); + dims[0] = rows_idx.size(); + outs[i]->mutable_value()->mutable_data(dims, x->place()); + for (auto idx : rows_idx) { + outs[i]->mutable_rows()->push_back(x_rows[idx]); + } + auto dst = outs[i]->mutable_value()->mutable_data(ctx.GetPlace()); + for (size_t j = 0; j < rows_idx.size(); j++) { + if (platform::is_cpu_place(place)) { + memory::Copy(platform::CPUPlace(), dst + j * row_numel, + platform::CPUPlace(), src + rows_idx[j] * row_numel, + sizeof(T) * row_numel); + } else { +#ifdef PADDLE_WITH_CUDA + auto stream = ctx.cuda_device_context().stream(); + memory::Copy(platform::CUDAPlace(), dst + j * row_numel, + platform::CUDAPlace(), src + rows_idx[j] * row_numel, + sizeof(T) * row_numel, stream); +#else + PADDLE_THROW("Paddle is not compiled with GPU"); +#endif + } + } } - offset += rows_sections[i]; } } }; diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index bd957f88de..5d42725e80 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -18,6 +18,7 @@ import optimizer from layer_helper import LayerHelper from distributed_spliter import * import math +from . import core class VarBlock: @@ -216,15 +217,28 @@ class DistributeTranspiler: if len(splited_vars) <= 1: continue orig_var = program.global_block().vars[varname] - sections = [] - for v in splited_vars: - sections.append(v.shape[0]) - program.global_block().append_op( - type="split", - inputs={"X": orig_var}, - outputs={"Out": splited_vars}, - attrs={"sections": sections} # assume split evenly - ) + if orig_var == core.VarDesc.VarType.SELECTED_ROWS: + height_sections = [] + for v in splited_vars: + height_sections.append(v.shape[0]) + program.global_block().append_op( + type="split_selected_rows", + inputs={"X": orig_var}, + outputs={"Out": splited_vars}, + attrs={"height_sections": height_sections}) + elif orig_var == core.VarDesc.VarType.LOD_TENSOR: + sections = [] + for v in splited_vars: + sections.append(v.shape[0]) + program.global_block().append_op( + type="split", + inputs={"X": orig_var}, + outputs={"Out": splited_vars}, + attrs={"sections": sections} # assume split evenly + ) + else: + AssertionError("Variable type should be in set " + "[LOD_TENSOR, SELECTED_ROWS]") return var_mapping def get_trainer_program(self): diff --git a/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py b/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py index a6cc4f6c6d..7a18dd8ad6 100644 --- a/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py +++ b/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py @@ -34,8 +34,8 @@ class TestSpliteSelectedRows(unittest.TestCase): def check_with_place(self, place): scope = core.Scope() - rows = [0, 5, 7, 4] - height = 10 + rows = [0, 5, 7, 4, 20] + height = 20 row_numel = 2 # initialize input variable X @@ -45,38 +45,41 @@ class TestSpliteSelectedRows(unittest.TestCase): np_array = np.ones((len(rows), row_numel)).astype("float32") np_array[0, 0] = 2.0 np_array[2, 1] = 4.0 + np_array[4, 1] = 8.0 x_tensor = x.get_tensor() x_tensor.set(np_array, place) - rows_sections = [2, 2] - height_sections = [] + height_sections = [5, 5, 5, 5, 3] # initialize output variables [out0, out1] - out0 = scope.var('out0').get_selected_rows() - out1 = scope.var('out1').get_selected_rows() + outs_name = ["out%d" % i for i in xrange(len(height_sections))] + outs = [ + scope.var(var_name).get_selected_rows() for var_name in outs_name + ] # expected output selected rows - expected_out0_rows = [0, 5] - expected_out1_rows = [7, 4] - expected_height = height + expected_out0_rows = [0, 4] + expected_out1_rows = [5, 7] + expected_out4_rows = [20] op = Operator( "split_selected_rows", X="X", - Out=["out0", "out1"], - rows_sections=rows_sections, + Out=outs_name, height_sections=height_sections) op.run(scope, place) - self.assertEqual(out0.rows(), expected_out0_rows) - self.assertEqual(out1.rows(), expected_out1_rows) + self.assertEqual(outs[0].rows(), expected_out0_rows) + self.assertEqual(outs[1].rows(), expected_out1_rows) + self.assertEqual(outs[4].rows(), expected_out4_rows) - self.assertEqual(out0.height(), expected_height) - self.assertEqual(out1.height(), expected_height) + self.assertEqual(outs[0].height(), height_sections[0]) + self.assertEqual(outs[4].height(), height_sections[4]) - self.assertAlmostEqual(2.0, np.array(out0.get_tensor())[0, 0]) - self.assertAlmostEqual(4.0, np.array(out1.get_tensor())[0, 1]) + self.assertAlmostEqual(2.0, np.array(outs[0].get_tensor())[0, 0]) + self.assertAlmostEqual(4.0, np.array(outs[1].get_tensor())[1, 1]) + self.assertAlmostEqual(8.0, np.array(outs[4].get_tensor())[0, 1]) def check_grad_with_place(self, place): scope = core.Scope() @@ -84,8 +87,7 @@ class TestSpliteSelectedRows(unittest.TestCase): row_numel = 2 # attr - rows_sections = [2, 2] - height_sections = [] + height_sections = [5, 5] # initialize input variable X out0_grad = scope.var("out0@GRAD").get_selected_rows() @@ -112,7 +114,6 @@ class TestSpliteSelectedRows(unittest.TestCase): "sum", X=["out0@GRAD", "out1@GRAD"], Out="X@GRAD", - rows_sections=rows_sections, height_sections=height_sections) grad_op.run(scope, place) From e983cc90fcee4e5b73bce9d4853b85aac4661e3a Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Sun, 21 Jan 2018 22:19:25 +0800 Subject: [PATCH 142/158] "fix decode bug" (#7711) * "fix decode bug" * "follow commnet" * "fix error" * "fix hook bug" * fix based comment * fix copyright * fix based on comment --- .copyright.hook | 27 +-- benchmark/paddle/image/alexnet.py | 21 +-- benchmark/paddle/image/googlenet.py | 13 -- benchmark/paddle/image/provider.py | 21 ++- benchmark/paddle/image/resnet.py | 13 -- .../paddle/image/smallnet_mnist_cifar.py | 13 -- benchmark/paddle/image/vgg.py | 13 -- benchmark/paddle/rnn/imdb.py | 14 ++ benchmark/paddle/rnn/provider.py | 21 ++- benchmark/paddle/rnn/rnn.py | 13 -- benchmark/tensorflow/image/alexnet.py | 21 ++- .../tensorflow/image/alexnet_multi_gpu.py | 21 ++- benchmark/tensorflow/image/googlenet.py | 21 ++- .../tensorflow/image/googlenet_multi_gpu.py | 21 ++- .../tensorflow/image/smallnet_mnist_cifar.py | 21 ++- benchmark/tensorflow/rnn/reader.py | 21 ++- benchmark/tensorflow/rnn/rnn.py | 13 -- benchmark/tensorflow/rnn/rnn_multi_gpu.py | 13 -- cmake/make_resource.py | 21 ++- doc/api/v1/data_provider/src/mnist_config.py | 21 ++- .../data_provider/src/mnist_provider.dict.py | 21 ++- .../data_provider/src/sentimental_config.py | 21 ++- .../data_provider/src/sentimental_provider.py | 21 ++- doc/faq/local/src/reduce_min_pool_size.py | 22 ++- doc/faq/local/src/word2vec_config.py | 21 ++- doc/faq/local/src/word2vec_dataprovider.py | 21 ++- doc/getstarted/concepts/src/infer.py | 21 ++- doc/getstarted/concepts/src/train.py | 21 ++- .../cluster/src/k8s_train/start_paddle.py | 13 -- .../cluster/src/word2vec/api_train_v2.py | 21 ++- .../src/word2vec/api_train_v2_cluster.py | 21 ++- .../usage/cluster/src/word2vec/prepare.py | 14 ++ go/pserver/client/c/test/test_mnist.py | 21 ++- go/pserver/client/c/test/test_train.py | 21 ++- paddle/api/test/testTrainConfig.py | 21 ++- .../examples/model_inference/dense/main.c | 14 ++ .../model_inference/dense/merge_v2_model.py | 21 ++- .../model_inference/dense/mnist_v2.py | 21 ++- .../model_inference/dense/trainer_config.py | 38 +--- .../model_inference/multi_thread/main.c | 14 ++ .../model_inference/multi_thread/main_gpu.c | 14 ++ .../examples/model_inference/sequence/main.c | 14 ++ .../sequence/trainer_config.py | 21 ++- .../model_inference/sparse_binary/main.c | 14 ++ paddle/capi/tests/test_predict_network.py | 21 ++- paddle/framework/data_layout_transform.cc | 26 +-- paddle/framework/data_layout_transform.h | 26 +-- .../framework/data_layout_transform_test.cc | 26 +-- paddle/framework/eigen_test.cc | 12 -- paddle/framework/lod_tensor_test.cc | 26 +-- paddle/framework/lod_tensor_test.cu | 16 +- paddle/framework/tensor_test.cc | 15 -- paddle/framework/tensor_util_test.cc | 12 -- paddle/framework/variable.h | 12 -- paddle/framework/variable_test.cc | 14 ++ paddle/operators/assign_value_op.cc | 26 +-- paddle/operators/assign_value_op.h | 26 +-- paddle/operators/net_op.cc | 15 -- paddle/operators/sequence_reshape_op.cc | 26 +-- paddle/operators/sequence_reshape_op.h | 26 +-- paddle/optimizer/parameter_optimizer_test.cc | 15 -- paddle/optimizer/serialization.h | 14 ++ paddle/optimizer/serialization_test.cc | 15 -- paddle/platform/assert.h | 14 ++ paddle/scripts/cluster_train/paddle.py | 178 ------------------ paddle/scripts/cpplint.py | 13 -- paddle/string/piece.cc | 15 -- paddle/string/piece.h | 15 -- paddle/string/piece_test.cc | 15 -- paddle/string/printf.h | 15 -- paddle/string/tinyformat/tinyformat.h | 5 +- paddle/trainer/tests/picojson.h | 14 ++ .../tests/simple_sparse_neural_network.py | 21 ++- .../tests/simple_sparse_neural_network_dp.py | 21 ++- paddle/utils/enable_virtualenv.py | 21 ++- .../tests/configs/img_layers.py | 21 ++- .../tests/configs/img_trans_layers.py | 21 ++- .../tests/configs/last_first_seq.py | 21 ++- .../tests/configs/layer_activations.py | 20 +- .../tests/configs/math_ops.py | 21 ++- .../tests/configs/projections.py | 20 +- .../tests/configs/shared_fc.py | 21 ++- .../tests/configs/shared_gru.py | 21 ++- .../tests/configs/shared_lstm.py | 21 ++- .../tests/configs/simple_rnn_layers.py | 21 ++- .../tests/configs/test_BatchNorm3D.py | 21 ++- .../tests/configs/test_bi_grumemory.py | 21 ++- .../tests/configs/test_bilinear_interp.py | 21 ++- .../tests/configs/test_clip_layer.py | 21 ++- .../test_config_parser_for_non_file_config.py | 13 -- .../tests/configs/test_conv3d_layer.py | 21 ++- .../tests/configs/test_cost_layers.py | 21 ++- .../configs/test_cost_layers_with_weight.py | 21 ++- .../tests/configs/test_crop.py | 21 ++- .../configs/test_cross_entropy_over_beam.py | 13 -- .../tests/configs/test_deconv3d_layer.py | 21 ++- .../configs/test_detection_output_layer.py | 21 ++- .../tests/configs/test_dot_prod_layer.py | 21 ++- .../tests/configs/test_expand_layer.py | 21 ++- .../configs/test_factorization_machine.py | 21 ++- .../tests/configs/test_fc.py | 21 ++- .../tests/configs/test_gated_unit_layer.py | 21 ++- .../tests/configs/test_grumemory_layer.py | 21 ++- .../tests/configs/test_hsigmoid.py | 21 ++- .../configs/test_kmax_seq_socre_layer.py | 13 -- .../tests/configs/test_l2_distance_layer.py | 21 ++- .../tests/configs/test_lstmemory_layer.py | 21 ++- .../tests/configs/test_maxout.py | 21 ++- .../tests/configs/test_multibox_loss_layer.py | 21 ++- .../tests/configs/test_multiplex_layer.py | 21 ++- .../tests/configs/test_ntm_layers.py | 21 ++- .../tests/configs/test_pad.py | 21 ++- .../tests/configs/test_pooling3D_layer.py | 21 ++- .../tests/configs/test_prelu_layer.py | 21 ++- .../tests/configs/test_print_layer.py | 21 ++- .../tests/configs/test_recursive_topology.py | 21 ++- .../tests/configs/test_repeat_layer.py | 21 ++- .../tests/configs/test_resize_layer.py | 21 ++- .../tests/configs/test_rnn_group.py | 21 ++- .../tests/configs/test_roi_pool_layer.py | 21 ++- .../tests/configs/test_row_conv.py | 21 ++- .../tests/configs/test_row_l2_norm_layer.py | 21 ++- .../tests/configs/test_scale_shift_layer.py | 21 ++- .../configs/test_scale_sub_region_layer.py | 21 ++- .../tests/configs/test_seq_concat_reshape.py | 21 ++- .../tests/configs/test_seq_slice_layer.py | 13 -- .../tests/configs/test_sequence_pooling.py | 21 ++- .../tests/configs/test_smooth_l1.py | 21 ++- .../tests/configs/test_split_datasource.py | 21 ++- .../tests/configs/test_spp_layer.py | 21 ++- .../test_sub_nested_seq_select_layer.py | 13 -- .../tests/configs/unused_layers.py | 21 ++- .../tests/configs/util_layers.py | 21 ++- .../tests/test_reset_hook.py | 3 +- python/paddle/utils/image_multiproc.py | 21 ++- python/paddle/utils/plotcurve.py | 13 -- python/paddle/v2/dataset/sentiment.py | 13 -- .../paddle/v2/dataset/tests/imikolov_test.py | 21 ++- .../paddle/v2/dataset/tests/test_sentiment.py | 13 -- python/paddle/v2/event.py | 20 +- python/paddle/v2/fluid/__init__.py | 21 ++- python/paddle/v2/fluid/backward.py | 21 ++- python/paddle/v2/fluid/clip.py | 21 ++- python/paddle/v2/fluid/data_feeder.py | 21 ++- python/paddle/v2/fluid/default_scope_funcs.py | 20 +- .../paddle/v2/fluid/distribute_transpiler.py | 21 ++- .../v2/fluid/distribute_transpiler_simple.py | 21 ++- python/paddle/v2/fluid/distributed_spliter.py | 22 ++- python/paddle/v2/fluid/evaluator.py | 21 ++- python/paddle/v2/fluid/executor.py | 21 ++- python/paddle/v2/fluid/framework.py | 21 ++- python/paddle/v2/fluid/initializer.py | 21 ++- python/paddle/v2/fluid/io.py | 21 ++- python/paddle/v2/fluid/layer_helper.py | 21 ++- python/paddle/v2/fluid/layers/__init__.py | 21 ++- python/paddle/v2/fluid/layers/control_flow.py | 21 ++- python/paddle/v2/fluid/layers/device.py | 20 +- python/paddle/v2/fluid/layers/io.py | 21 ++- python/paddle/v2/fluid/layers/nn.py | 20 +- python/paddle/v2/fluid/layers/ops.py | 21 ++- python/paddle/v2/fluid/layers/tensor.py | 21 ++- .../fluid/memory_optimization_transpiler.py | 21 ++- python/paddle/v2/fluid/net_drawer.py | 21 ++- python/paddle/v2/fluid/nets.py | 21 ++- python/paddle/v2/fluid/op.py | 21 ++- python/paddle/v2/fluid/optimizer.py | 21 ++- python/paddle/v2/fluid/param_attr.py | 21 ++- python/paddle/v2/fluid/profiler.py | 21 ++- python/paddle/v2/fluid/regularizer.py | 21 ++- python/paddle/v2/fluid/tests/__init__.py | 20 +- .../v2/fluid/tests/book/test_fit_a_line.py | 21 ++- .../book/test_image_classification_train.py | 21 ++- .../tests/book/test_label_semantic_roles.py | 21 ++- .../tests/book/test_machine_translation.py | 21 ++- .../tests/book/test_recognize_digits_conv.py | 21 ++- .../tests/book/test_recognize_digits_mlp.py | 21 ++- .../tests/book/test_recommender_system.py | 21 ++- .../book/test_understand_sentiment_conv.py | 21 ++- .../test_understand_sentiment_dynamic_lstm.py | 21 ++- .../book/test_understand_sentiment_lstm.py | 21 ++- .../v2/fluid/tests/book/test_word2vec.py | 21 ++- .../book_distribute/notest_dist_fit_a_line.py | 21 ++- .../notest_dist_label_semantic_roles.py | 21 ++- .../book_distribute/notest_dist_word2vec.py | 21 ++- .../notest_recognize_digits_conv_dist.py | 21 ++- .../notest_recognize_digits_mlp_dist.py | 21 ++- .../notest_understand_sentiment_conv_dist.py | 21 ++- .../tests/book_distribute/test_split_var.py | 21 ++- python/paddle/v2/fluid/tests/decorators.py | 21 ++- python/paddle/v2/fluid/tests/demo/fc_gan.py | 21 ++- python/paddle/v2/fluid/tests/op_test.py | 21 ++- .../paddle/v2/fluid/tests/test_accuracy_op.py | 21 ++- .../v2/fluid/tests/test_activation_op.py | 21 ++- .../paddle/v2/fluid/tests/test_adadelta_op.py | 21 ++- .../paddle/v2/fluid/tests/test_adagrad_op.py | 21 ++- python/paddle/v2/fluid/tests/test_adam_op.py | 21 ++- .../paddle/v2/fluid/tests/test_adamax_op.py | 21 ++- .../fluid/tests/test_array_read_write_op.py | 21 ++- .../paddle/v2/fluid/tests/test_assign_op.py | 21 ++- .../v2/fluid/tests/test_assign_value_op.py | 21 ++- python/paddle/v2/fluid/tests/test_auc_op.py | 21 ++- .../v2/fluid/tests/test_batch_norm_op.py | 21 ++- .../fluid/tests/test_beam_search_decode_op.py | 21 ++- .../v2/fluid/tests/test_beam_search_op.py | 21 ++- .../tests/test_bilinear_tensor_product_op.py | 21 ++- .../v2/fluid/tests/test_calc_gradient.py | 21 ++- python/paddle/v2/fluid/tests/test_cast_op.py | 21 ++- .../v2/fluid/tests/test_chunk_eval_op.py | 21 ++- python/paddle/v2/fluid/tests/test_clip.py | 21 ++- .../v2/fluid/tests/test_clip_by_norm_op.py | 21 ++- python/paddle/v2/fluid/tests/test_clip_op.py | 21 ++- .../paddle/v2/fluid/tests/test_compare_op.py | 21 ++- .../paddle/v2/fluid/tests/test_concat_op.py | 21 ++- python/paddle/v2/fluid/tests/test_cond_op.py | 21 ++- .../v2/fluid/tests/test_conditional_block.py | 21 ++- .../paddle/v2/fluid/tests/test_const_value.py | 21 ++- .../paddle/v2/fluid/tests/test_conv2d_op.py | 21 ++- .../fluid/tests/test_conv2d_transpose_op.py | 21 ++- .../paddle/v2/fluid/tests/test_conv3d_op.py | 21 ++- .../fluid/tests/test_conv3d_transpose_op.py | 21 ++- .../v2/fluid/tests/test_conv_shift_op.py | 21 ++- .../paddle/v2/fluid/tests/test_cos_sim_op.py | 21 ++- .../fluid/tests/test_create_op_doc_string.py | 21 ++- .../v2/fluid/tests/test_crf_decoding_op.py | 21 ++- python/paddle/v2/fluid/tests/test_crop_op.py | 21 ++- .../v2/fluid/tests/test_cross_entropy_op.py | 21 ++- .../paddle/v2/fluid/tests/test_ctc_align.py | 20 +- .../paddle/v2/fluid/tests/test_data_feeder.py | 21 ++- .../v2/fluid/tests/test_decayed_adagrad_op.py | 21 ++- .../fluid/tests/test_default_scope_funcs.py | 21 ++- .../fluid/tests/test_detection_output_op.py | 21 ++- .../paddle/v2/fluid/tests/test_dropout_op.py | 21 ++- python/paddle/v2/fluid/tests/test_dyn_rnn.py | 21 ++- .../fluid/tests/test_dynrnn_gradient_check.py | 21 ++- .../fluid/tests/test_dynrnn_static_input.py | 21 ++- .../v2/fluid/tests/test_edit_distance_op.py | 21 ++- .../v2/fluid/tests/test_elementwise_max_op.py | 21 ++- .../v2/fluid/tests/test_elementwise_min_op.py | 21 ++- .../paddle/v2/fluid/tests/test_exception.py | 21 ++- .../v2/fluid/tests/test_executor_and_mul.py | 21 ++- .../paddle/v2/fluid/tests/test_expand_op.py | 21 ++- .../v2/fluid/tests/test_feed_fetch_method.py | 21 ++- .../test_fill_constant_batch_size_like_op.py | 21 ++- .../v2/fluid/tests/test_fill_constant_op.py | 21 ++- python/paddle/v2/fluid/tests/test_fill_op.py | 21 ++- .../v2/fluid/tests/test_fill_zeros_like_op.py | 21 ++- .../fluid/tests/test_framework_debug_str.py | 21 ++- python/paddle/v2/fluid/tests/test_ftrl_op.py | 21 ++- .../paddle/v2/fluid/tests/test_gather_op.py | 21 ++- .../v2/fluid/tests/test_gaussian_random_op.py | 21 ++- .../v2/fluid/tests/test_get_places_op.py | 21 ++- python/paddle/v2/fluid/tests/test_gru_op.py | 21 ++- .../paddle/v2/fluid/tests/test_gru_unit_op.py | 21 ++- .../v2/fluid/tests/test_hinge_loss_op.py | 21 ++- .../v2/fluid/tests/test_huber_loss_op.py | 21 ++- .../tests/test_image_classification_layer.py | 21 ++- .../paddle/v2/fluid/tests/test_infer_shape.py | 21 ++- .../v2/fluid/tests/test_inference_model_io.py | 21 ++- .../paddle/v2/fluid/tests/test_initializer.py | 21 ++- .../paddle/v2/fluid/tests/test_is_empty_op.py | 21 ++- .../paddle/v2/fluid/tests/test_l1_norm_op.py | 21 ++- python/paddle/v2/fluid/tests/test_layers.py | 21 ++- .../fluid/tests/test_linear_chain_crf_op.py | 21 ++- .../fluid/tests/test_lod_array_length_op.py | 21 ++- .../v2/fluid/tests/test_lod_rank_table.py | 21 ++- .../v2/fluid/tests/test_lod_reset_op.py | 21 ++- .../v2/fluid/tests/test_lod_tensor_array.py | 21 ++- .../fluid/tests/test_lod_tensor_array_ops.py | 21 ++- .../paddle/v2/fluid/tests/test_log_loss_op.py | 21 ++- .../paddle/v2/fluid/tests/test_logical_op.py | 21 ++- .../v2/fluid/tests/test_lookup_table_op.py | 21 ++- python/paddle/v2/fluid/tests/test_lrn_op.py | 21 ++- python/paddle/v2/fluid/tests/test_lstm_op.py | 21 ++- .../v2/fluid/tests/test_lstm_unit_op.py | 21 ++- .../fluid/tests/test_margin_rank_loss_op.py | 21 ++- .../paddle/v2/fluid/tests/test_matmul_op.py | 21 ++- .../paddle/v2/fluid/tests/test_maxout_op.py | 21 ++- python/paddle/v2/fluid/tests/test_mean_op.py | 21 ++- .../test_memory_optimization_transpiler.py | 21 ++- python/paddle/v2/fluid/tests/test_minus_op.py | 21 ++- .../v2/fluid/tests/test_mnist_if_else_op.py | 21 ++- .../tests/test_modified_huber_loss_op.py | 21 ++- .../paddle/v2/fluid/tests/test_momentum_op.py | 21 ++- python/paddle/v2/fluid/tests/test_mul_op.py | 21 ++- .../v2/fluid/tests/test_multiplex_op.py | 21 ++- python/paddle/v2/fluid/tests/test_nce.py | 21 ++- python/paddle/v2/fluid/tests/test_net.py | 21 ++- python/paddle/v2/fluid/tests/test_norm_op.py | 21 ++- .../fluid/tests/test_normalization_wrapper.py | 21 ++- .../v2/fluid/tests/test_op_support_gpu.py | 21 ++- python/paddle/v2/fluid/tests/test_operator.py | 21 ++- .../v2/fluid/tests/test_operator_desc.py | 21 ++- .../paddle/v2/fluid/tests/test_optimizer.py | 21 ++- python/paddle/v2/fluid/tests/test_pad_op.py | 21 ++- .../paddle/v2/fluid/tests/test_parallel_op.py | 21 ++- .../paddle/v2/fluid/tests/test_parameter.py | 21 ++- .../paddle/v2/fluid/tests/test_pool2d_op.py | 21 ++- .../paddle/v2/fluid/tests/test_pool3d_op.py | 21 ++- .../paddle/v2/fluid/tests/test_pool_max_op.py | 21 ++- .../tests/test_positive_negative_pair_op.py | 21 ++- .../fluid/tests/test_precision_recall_op.py | 21 ++- python/paddle/v2/fluid/tests/test_prelu_op.py | 21 ++- python/paddle/v2/fluid/tests/test_print_op.py | 21 ++- python/paddle/v2/fluid/tests/test_profiler.py | 21 ++- python/paddle/v2/fluid/tests/test_program.py | 21 ++- python/paddle/v2/fluid/tests/test_protobuf.py | 21 ++- .../v2/fluid/tests/test_protobuf_descs.py | 21 ++- .../fluid/tests/test_proximal_adagrad_op.py | 21 ++- .../v2/fluid/tests/test_proximal_gd_op.py | 21 ++- .../v2/fluid/tests/test_rank_loss_op.py | 21 ++- .../v2/fluid/tests/test_recurrent_op.py | 21 ++- .../paddle/v2/fluid/tests/test_reduce_op.py | 21 ++- python/paddle/v2/fluid/tests/test_registry.py | 21 ++- .../paddle/v2/fluid/tests/test_regularizer.py | 21 ++- .../v2/fluid/tests/test_reorder_lod_tensor.py | 21 ++- .../paddle/v2/fluid/tests/test_reshape_op.py | 21 ++- .../paddle/v2/fluid/tests/test_rmsprop_op.py | 21 ++- .../fluid/tests/test_rnn_memory_helper_op.py | 21 ++- .../paddle/v2/fluid/tests/test_roi_pool_op.py | 21 ++- .../paddle/v2/fluid/tests/test_row_conv_op.py | 21 ++- python/paddle/v2/fluid/tests/test_scale_op.py | 21 ++- .../paddle/v2/fluid/tests/test_scatter_op.py | 21 ++- python/paddle/v2/fluid/tests/test_scope.py | 21 ++- .../v2/fluid/tests/test_selected_rows.py | 21 ++- .../v2/fluid/tests/test_seq_concat_op.py | 21 ++- python/paddle/v2/fluid/tests/test_seq_conv.py | 21 ++- python/paddle/v2/fluid/tests/test_seq_pool.py | 21 ++- .../v2/fluid/tests/test_sequence_erase_op.py | 21 ++- .../v2/fluid/tests/test_sequence_expand.py | 21 ++- .../v2/fluid/tests/test_sequence_reshape.py | 21 ++- .../v2/fluid/tests/test_sequence_slice_op.py | 21 ++- .../fluid/tests/test_sequence_softmax_op.py | 21 ++- python/paddle/v2/fluid/tests/test_sgd_op.py | 21 ++- .../v2/fluid/tests/test_shrink_rnn_memory.py | 21 ++- ...st_sigmoid_cross_entropy_with_logits_op.py | 21 ++- python/paddle/v2/fluid/tests/test_sign_op.py | 21 ++- .../v2/fluid/tests/test_smooth_l1_loss_op.py | 21 ++- .../paddle/v2/fluid/tests/test_softmax_op.py | 21 ++- .../test_softmax_with_cross_entropy_op.py | 21 ++- .../test_split_and_merge_lod_tensor_op.py | 21 ++- python/paddle/v2/fluid/tests/test_split_op.py | 21 ++- .../tests/test_split_selected_rows_op.py | 21 ++- python/paddle/v2/fluid/tests/test_spp_op.py | 21 ++- .../tests/test_squared_l2_distance_op.py | 21 ++- .../v2/fluid/tests/test_squared_l2_norm_op.py | 21 ++- python/paddle/v2/fluid/tests/test_sum_op.py | 21 ++- python/paddle/v2/fluid/tests/test_tensor.py | 21 ++- python/paddle/v2/fluid/tests/test_top_k_op.py | 21 ++- .../v2/fluid/tests/test_transpose_op.py | 21 ++- .../v2/fluid/tests/test_uniform_random_op.py | 21 ++- .../paddle/v2/fluid/tests/test_unpool_op.py | 21 ++- python/paddle/v2/fluid/tests/test_variable.py | 21 ++- .../paddle/v2/fluid/tests/test_warpctc_op.py | 21 ++- python/paddle/v2/fluid/tests/test_while_op.py | 21 ++- python/paddle/v2/image.py | 20 +- python/paddle/v2/inference.py | 21 ++- python/paddle/v2/master/__init__.py | 21 ++- python/paddle/v2/master/client.py | 21 ++- python/paddle/v2/reader/tests/__init__.py | 20 +- python/paddle/v2/reader/tests/creator_test.py | 14 ++ .../paddle/v2/reader/tests/decorator_test.py | 3 +- python/paddle/v2/tests/test_image.py | 3 +- python/paddle/v2/tests/test_layer.py | 3 +- python/paddle/v2/tests/test_op.py | 3 +- .../paddle/v2/tests/test_paramconf_order.py | 14 ++ python/paddle/v2/tests/test_parameters.py | 21 ++- python/paddle/v2/tests/test_rnn_layer.py | 3 +- python/paddle/v2/tests/test_topology.py | 3 +- python/paddle/v2/trainer.py | 20 +- .../build_scripts/manylinux1-check.py | 21 ++- .../build_scripts/python-tag-abi-tag.py | 14 ++ tools/manylinux1/build_scripts/ssl-check.py | 21 ++- 372 files changed, 3745 insertions(+), 3822 deletions(-) diff --git a/.copyright.hook b/.copyright.hook index 6bfe58cebc..dc1b096a0a 100644 --- a/.copyright.hook +++ b/.copyright.hook @@ -49,10 +49,15 @@ def generate_copyright(template, lang='C'): LANG_COMMENT_MARK = "//" lines = template.split(NEW_LINE_MARK) - ans = LANG_COMMENT_MARK + " " + COPYRIGHT_HEADER + NEW_LINE_MARK + BLANK = " " + ans = LANG_COMMENT_MARK + BLANK + COPYRIGHT_HEADER + NEW_LINE_MARK for lino, line in enumerate(lines): if lino == 0 or lino == 1 or lino == len(lines) - 1: continue - ans += LANG_COMMENT_MARK + " " + line + NEW_LINE_MARK + if len(line) == 0: + BLANK = "" + else: + BLANK = " " + ans += LANG_COMMENT_MARK + BLANK + line + NEW_LINE_MARK return ans + "\n" @@ -62,6 +67,8 @@ def lang_type(filename): return "Python" elif filename.endswith(".h"): return "C" + elif filename.endswith(".c"): + return "C" elif filename.endswith(".hpp"): return "C" elif filename.endswith(".cc"): @@ -92,16 +99,14 @@ def main(argv=None): retv = 0 for filename in args.filenames: - fd = io.open(filename) + fd = io.open(filename, encoding="utf-8") first_line = fd.readline() - if "COPYRIGHT" in first_line.upper(): continue - if filename.endswith(".py"): - second_line = fd.readline() - if first_line.startswith("#!") or PYTHON_ENCODE.match( - second_line) != None or PYTHON_ENCODE.match( - first_line) != None: - continue - original_contents = io.open(filename).read() + second_line = fd.readline() + if "COPYRIGHT (C)" in first_line.upper(): continue + if first_line.startswith("#!") or PYTHON_ENCODE.match( + second_line) != None or PYTHON_ENCODE.match(first_line) != None: + continue + original_contents = io.open(filename, encoding="utf-8").read() new_contents = generate_copyright( COPYRIGHT, lang_type(filename)) + original_contents print('Auto Insert Copyright Header {}'.format(filename)) diff --git a/benchmark/paddle/image/alexnet.py b/benchmark/paddle/image/alexnet.py index 07f478d8fa..7029608187 100644 --- a/benchmark/paddle/image/alexnet.py +++ b/benchmark/paddle/image/alexnet.py @@ -1,17 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -#!/usr/bin/env python +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from paddle.trainer_config_helpers import * diff --git a/benchmark/paddle/image/googlenet.py b/benchmark/paddle/image/googlenet.py index 3241be9c5f..2a850ccb7f 100644 --- a/benchmark/paddle/image/googlenet.py +++ b/benchmark/paddle/image/googlenet.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from paddle.trainer_config_helpers import * diff --git a/benchmark/paddle/image/provider.py b/benchmark/paddle/image/provider.py index 220c4bee35..21e0d381aa 100644 --- a/benchmark/paddle/image/provider.py +++ b/benchmark/paddle/image/provider.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import io, os import random import numpy as np diff --git a/benchmark/paddle/image/resnet.py b/benchmark/paddle/image/resnet.py index acc6d31d4b..2846e4763f 100644 --- a/benchmark/paddle/image/resnet.py +++ b/benchmark/paddle/image/resnet.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from paddle.trainer_config_helpers import * diff --git a/benchmark/paddle/image/smallnet_mnist_cifar.py b/benchmark/paddle/image/smallnet_mnist_cifar.py index 64a5da3220..58879c454f 100644 --- a/benchmark/paddle/image/smallnet_mnist_cifar.py +++ b/benchmark/paddle/image/smallnet_mnist_cifar.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from paddle.trainer_config_helpers import * diff --git a/benchmark/paddle/image/vgg.py b/benchmark/paddle/image/vgg.py index a357207a62..ca0a6798fb 100644 --- a/benchmark/paddle/image/vgg.py +++ b/benchmark/paddle/image/vgg.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from paddle.trainer_config_helpers import * diff --git a/benchmark/paddle/rnn/imdb.py b/benchmark/paddle/rnn/imdb.py index fc4ed4025f..c3b5faa19a 100755 --- a/benchmark/paddle/rnn/imdb.py +++ b/benchmark/paddle/rnn/imdb.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import six.moves.cPickle as pickle import gzip diff --git a/benchmark/paddle/rnn/provider.py b/benchmark/paddle/rnn/provider.py index c03df3a002..f35cd5b079 100644 --- a/benchmark/paddle/rnn/provider.py +++ b/benchmark/paddle/rnn/provider.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import io, os import random import numpy as np diff --git a/benchmark/paddle/rnn/rnn.py b/benchmark/paddle/rnn/rnn.py index 97005f2c35..83eb3e5654 100755 --- a/benchmark/paddle/rnn/rnn.py +++ b/benchmark/paddle/rnn/rnn.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from paddle.trainer_config_helpers import * diff --git a/benchmark/tensorflow/image/alexnet.py b/benchmark/tensorflow/image/alexnet.py index edf462e6a1..a37d7e7c62 100644 --- a/benchmark/tensorflow/image/alexnet.py +++ b/benchmark/tensorflow/image/alexnet.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from six.moves import xrange # pylint: disable=redefined-builtin from datetime import datetime import math diff --git a/benchmark/tensorflow/image/alexnet_multi_gpu.py b/benchmark/tensorflow/image/alexnet_multi_gpu.py index 90b8f16bca..2ebab8fb60 100644 --- a/benchmark/tensorflow/image/alexnet_multi_gpu.py +++ b/benchmark/tensorflow/image/alexnet_multi_gpu.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from six.moves import xrange # pylint: disable=redefined-builtin from datetime import datetime import math diff --git a/benchmark/tensorflow/image/googlenet.py b/benchmark/tensorflow/image/googlenet.py index 55431eceb3..1202cbb171 100644 --- a/benchmark/tensorflow/image/googlenet.py +++ b/benchmark/tensorflow/image/googlenet.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from six.moves import xrange from datetime import datetime import math diff --git a/benchmark/tensorflow/image/googlenet_multi_gpu.py b/benchmark/tensorflow/image/googlenet_multi_gpu.py index 44de3800a8..f06437eb6c 100644 --- a/benchmark/tensorflow/image/googlenet_multi_gpu.py +++ b/benchmark/tensorflow/image/googlenet_multi_gpu.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from six.moves import xrange # pylint: disable=redefined-builtin from datetime import datetime import math diff --git a/benchmark/tensorflow/image/smallnet_mnist_cifar.py b/benchmark/tensorflow/image/smallnet_mnist_cifar.py index 0858b5f9c9..558c68575f 100644 --- a/benchmark/tensorflow/image/smallnet_mnist_cifar.py +++ b/benchmark/tensorflow/image/smallnet_mnist_cifar.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from six.moves import xrange # pylint: disable=redefined-builtin from datetime import datetime import math diff --git a/benchmark/tensorflow/rnn/reader.py b/benchmark/tensorflow/rnn/reader.py index 710940c9ae..9660d3c22b 100755 --- a/benchmark/tensorflow/rnn/reader.py +++ b/benchmark/tensorflow/rnn/reader.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os.path import io import numpy as np diff --git a/benchmark/tensorflow/rnn/rnn.py b/benchmark/tensorflow/rnn/rnn.py index 507481b9cc..f288083e13 100755 --- a/benchmark/tensorflow/rnn/rnn.py +++ b/benchmark/tensorflow/rnn/rnn.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from six.moves import xrange # pylint: disable=redefined-builtin import math diff --git a/benchmark/tensorflow/rnn/rnn_multi_gpu.py b/benchmark/tensorflow/rnn/rnn_multi_gpu.py index f24cbaef62..eabee4fa8f 100755 --- a/benchmark/tensorflow/rnn/rnn_multi_gpu.py +++ b/benchmark/tensorflow/rnn/rnn_multi_gpu.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python from six.moves import xrange # pylint: disable=redefined-builtin import re diff --git a/cmake/make_resource.py b/cmake/make_resource.py index d71e82eca2..4f9f5546b9 100644 --- a/cmake/make_resource.py +++ b/cmake/make_resource.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re import sys diff --git a/doc/api/v1/data_provider/src/mnist_config.py b/doc/api/v1/data_provider/src/mnist_config.py index 427e0465a6..d2af9d849e 100644 --- a/doc/api/v1/data_provider/src/mnist_config.py +++ b/doc/api/v1/data_provider/src/mnist_config.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * define_py_data_sources2( diff --git a/doc/api/v1/data_provider/src/mnist_provider.dict.py b/doc/api/v1/data_provider/src/mnist_provider.dict.py index 3fbb783e2f..284f7dadb0 100644 --- a/doc/api/v1/data_provider/src/mnist_provider.dict.py +++ b/doc/api/v1/data_provider/src/mnist_provider.dict.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer.PyDataProvider2 import * diff --git a/doc/api/v1/data_provider/src/sentimental_config.py b/doc/api/v1/data_provider/src/sentimental_config.py index edbf3cf140..56adde13b9 100644 --- a/doc/api/v1/data_provider/src/sentimental_config.py +++ b/doc/api/v1/data_provider/src/sentimental_config.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * dictionary = dict() diff --git a/doc/api/v1/data_provider/src/sentimental_provider.py b/doc/api/v1/data_provider/src/sentimental_provider.py index 03ad1fe7d8..59a2b6f7f5 100644 --- a/doc/api/v1/data_provider/src/sentimental_provider.py +++ b/doc/api/v1/data_provider/src/sentimental_provider.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer.PyDataProvider2 import * diff --git a/doc/faq/local/src/reduce_min_pool_size.py b/doc/faq/local/src/reduce_min_pool_size.py index 96073633d2..9efdb5707a 100644 --- a/doc/faq/local/src/reduce_min_pool_size.py +++ b/doc/faq/local/src/reduce_min_pool_size.py @@ -1,16 +1,18 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + @provider(min_pool_size=0, ...) def process(settings, filename): os.system('shuf %s > %s.shuf' % (filename, filename)) # shuffle before. diff --git a/doc/faq/local/src/word2vec_config.py b/doc/faq/local/src/word2vec_config.py index 03619b2628..b4fcf0960e 100644 --- a/doc/faq/local/src/word2vec_config.py +++ b/doc/faq/local/src/word2vec_config.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ... # the settings and define data provider is omitted. DICT_DIM = 3000 # dictionary dimension. word_ids = data_layer('word_ids', size=DICT_DIM) diff --git a/doc/faq/local/src/word2vec_dataprovider.py b/doc/faq/local/src/word2vec_dataprovider.py index a439a8f52e..3b6273b057 100644 --- a/doc/faq/local/src/word2vec_dataprovider.py +++ b/doc/faq/local/src/word2vec_dataprovider.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + DICT_DIM = 3000 diff --git a/doc/getstarted/concepts/src/infer.py b/doc/getstarted/concepts/src/infer.py index ee71cd7a9a..a1b60388c4 100644 --- a/doc/getstarted/concepts/src/infer.py +++ b/doc/getstarted/concepts/src/infer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2 as paddle import numpy as np diff --git a/doc/getstarted/concepts/src/train.py b/doc/getstarted/concepts/src/train.py index d9c0c66b8a..0e5bdb57bc 100644 --- a/doc/getstarted/concepts/src/train.py +++ b/doc/getstarted/concepts/src/train.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2 as paddle import numpy as np diff --git a/doc/howto/usage/cluster/src/k8s_train/start_paddle.py b/doc/howto/usage/cluster/src/k8s_train/start_paddle.py index 1774f8b640..935c12bb67 100755 --- a/doc/howto/usage/cluster/src/k8s_train/start_paddle.py +++ b/doc/howto/usage/cluster/src/k8s_train/start_paddle.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # diff --git a/doc/howto/usage/cluster/src/word2vec/api_train_v2.py b/doc/howto/usage/cluster/src/word2vec/api_train_v2.py index d449e02023..9a65f14628 100644 --- a/doc/howto/usage/cluster/src/word2vec/api_train_v2.py +++ b/doc/howto/usage/cluster/src/word2vec/api_train_v2.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import gzip import math diff --git a/doc/howto/usage/cluster/src/word2vec/api_train_v2_cluster.py b/doc/howto/usage/cluster/src/word2vec/api_train_v2_cluster.py index a5dd347f0b..2afce9a66e 100644 --- a/doc/howto/usage/cluster/src/word2vec/api_train_v2_cluster.py +++ b/doc/howto/usage/cluster/src/word2vec/api_train_v2_cluster.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import os import paddle.v2 as paddle diff --git a/doc/howto/usage/cluster/src/word2vec/prepare.py b/doc/howto/usage/cluster/src/word2vec/prepare.py index 24f5c5b26d..ade01c378e 100644 --- a/doc/howto/usage/cluster/src/word2vec/prepare.py +++ b/doc/howto/usage/cluster/src/word2vec/prepare.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2 as paddle import tarfile import os diff --git a/go/pserver/client/c/test/test_mnist.py b/go/pserver/client/c/test/test_mnist.py index 7b50a10afc..821d9adfcb 100644 --- a/go/pserver/client/c/test/test_mnist.py +++ b/go/pserver/client/c/test/test_mnist.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2 as paddle import gzip diff --git a/go/pserver/client/c/test/test_train.py b/go/pserver/client/c/test/test_train.py index 7ef0fca496..445a8d3aa4 100644 --- a/go/pserver/client/c/test/test_train.py +++ b/go/pserver/client/c/test/test_train.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2 as paddle import paddle.v2.dataset.uci_housing as uci_housing import paddle.v2.master as master diff --git a/paddle/api/test/testTrainConfig.py b/paddle/api/test/testTrainConfig.py index ab9a83e4a3..1a1283e116 100644 --- a/paddle/api/test/testTrainConfig.py +++ b/paddle/api/test/testTrainConfig.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=100, learning_method=AdamOptimizer()) diff --git a/paddle/capi/examples/model_inference/dense/main.c b/paddle/capi/examples/model_inference/dense/main.c index 376cd46fb0..f795bfe11d 100644 --- a/paddle/capi/examples/model_inference/dense/main.c +++ b/paddle/capi/examples/model_inference/dense/main.c @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include diff --git a/paddle/capi/examples/model_inference/dense/merge_v2_model.py b/paddle/capi/examples/model_inference/dense/merge_v2_model.py index 760a485a53..7aeb482903 100644 --- a/paddle/capi/examples/model_inference/dense/merge_v2_model.py +++ b/paddle/capi/examples/model_inference/dense/merge_v2_model.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.utils.merge_model import merge_v2_model from mnist_v2 import network diff --git a/paddle/capi/examples/model_inference/dense/mnist_v2.py b/paddle/capi/examples/model_inference/dense/mnist_v2.py index 174436bd1d..183eecfdf2 100644 --- a/paddle/capi/examples/model_inference/dense/mnist_v2.py +++ b/paddle/capi/examples/model_inference/dense/mnist_v2.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import sys import gzip diff --git a/paddle/capi/examples/model_inference/dense/trainer_config.py b/paddle/capi/examples/model_inference/dense/trainer_config.py index fbf0890357..b94a21a7e4 100644 --- a/paddle/capi/examples/model_inference/dense/trainer_config.py +++ b/paddle/capi/examples/model_inference/dense/trainer_config.py @@ -1,31 +1,13 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. -from paddle.trainer_config_helpers import * - -img = data_layer(name='pixel', size=784) - -hidden = fc_layer( - input=img, - size=200, - param_attr=ParamAttr(name='hidden.w'), - bias_attr=ParamAttr(name='hidden.b')) - -prob = fc_layer( - input=hidden, - size=10, - act=SoftmaxActivation(), - param_attr=ParamAttr(name='prob.w'), - bias_attr=ParamAttr(name='prob.b')) - -outputs(prob) +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/paddle/capi/examples/model_inference/multi_thread/main.c b/paddle/capi/examples/model_inference/multi_thread/main.c index d7675cd80a..eecb9138e7 100644 --- a/paddle/capi/examples/model_inference/multi_thread/main.c +++ b/paddle/capi/examples/model_inference/multi_thread/main.c @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include #include diff --git a/paddle/capi/examples/model_inference/multi_thread/main_gpu.c b/paddle/capi/examples/model_inference/multi_thread/main_gpu.c index 6fd376e0d1..85bb456584 100644 --- a/paddle/capi/examples/model_inference/multi_thread/main_gpu.c +++ b/paddle/capi/examples/model_inference/multi_thread/main_gpu.c @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include #include diff --git a/paddle/capi/examples/model_inference/sequence/main.c b/paddle/capi/examples/model_inference/sequence/main.c index 50bc0c9201..80937c830d 100644 --- a/paddle/capi/examples/model_inference/sequence/main.c +++ b/paddle/capi/examples/model_inference/sequence/main.c @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include #include "../common/common.h" diff --git a/paddle/capi/examples/model_inference/sequence/trainer_config.py b/paddle/capi/examples/model_inference/sequence/trainer_config.py index c1326bb955..889f8acdfd 100644 --- a/paddle/capi/examples/model_inference/sequence/trainer_config.py +++ b/paddle/capi/examples/model_inference/sequence/trainer_config.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * WORD_DIM = 3000 diff --git a/paddle/capi/examples/model_inference/sparse_binary/main.c b/paddle/capi/examples/model_inference/sparse_binary/main.c index 029b94ee63..efec010a91 100644 --- a/paddle/capi/examples/model_inference/sparse_binary/main.c +++ b/paddle/capi/examples/model_inference/sparse_binary/main.c @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include diff --git a/paddle/capi/tests/test_predict_network.py b/paddle/capi/tests/test_predict_network.py index 46a985d476..6560417b2a 100644 --- a/paddle/capi/tests/test_predict_network.py +++ b/paddle/capi/tests/test_predict_network.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=100) diff --git a/paddle/framework/data_layout_transform.cc b/paddle/framework/data_layout_transform.cc index 1059bd9761..9d0a6d5ea3 100644 --- a/paddle/framework/data_layout_transform.cc +++ b/paddle/framework/data_layout_transform.cc @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "paddle/framework/data_layout_transform.h" diff --git a/paddle/framework/data_layout_transform.h b/paddle/framework/data_layout_transform.h index ec87257d70..368f7fc989 100644 --- a/paddle/framework/data_layout_transform.h +++ b/paddle/framework/data_layout_transform.h @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #pragma once diff --git a/paddle/framework/data_layout_transform_test.cc b/paddle/framework/data_layout_transform_test.cc index 2eb99fa04a..093e8d4d34 100644 --- a/paddle/framework/data_layout_transform_test.cc +++ b/paddle/framework/data_layout_transform_test.cc @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "paddle/framework/data_layout_transform.h" diff --git a/paddle/framework/eigen_test.cc b/paddle/framework/eigen_test.cc index c6f77ecfab..9e368a522c 100644 --- a/paddle/framework/eigen_test.cc +++ b/paddle/framework/eigen_test.cc @@ -11,18 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "paddle/framework/eigen.h" #include diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 2c4d2e10e0..4d172c43c7 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "paddle/framework/lod_tensor.h" diff --git a/paddle/framework/lod_tensor_test.cu b/paddle/framework/lod_tensor_test.cu index 0f46e9b1e3..1e253a2f6f 100644 --- a/paddle/framework/lod_tensor_test.cu +++ b/paddle/framework/lod_tensor_test.cu @@ -1,28 +1,16 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include #include diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc index c04cd38697..9a387526ac 100644 --- a/paddle/framework/tensor_test.cc +++ b/paddle/framework/tensor_test.cc @@ -11,18 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "paddle/framework/tensor.h" #include @@ -60,9 +48,6 @@ TEST(Tensor, DataAssert) { ASSERT_TRUE(caught); } -/* following tests are not available at present - because Memory::Alloc() and Memory::Free() have not been ready. -*/ TEST(Tensor, MutableData) { { framework::Tensor src_tensor; diff --git a/paddle/framework/tensor_util_test.cc b/paddle/framework/tensor_util_test.cc index f541927c0e..906b0b5656 100644 --- a/paddle/framework/tensor_util_test.cc +++ b/paddle/framework/tensor_util_test.cc @@ -11,18 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "paddle/framework/tensor_util.h" #include diff --git a/paddle/framework/variable.h b/paddle/framework/variable.h index 03992c8608..3b7ec0a2a9 100644 --- a/paddle/framework/variable.h +++ b/paddle/framework/variable.h @@ -11,18 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #pragma once #include diff --git a/paddle/framework/variable_test.cc b/paddle/framework/variable_test.cc index aea03bcf57..e4732d9718 100644 --- a/paddle/framework/variable_test.cc +++ b/paddle/framework/variable_test.cc @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/paddle/operators/assign_value_op.cc b/paddle/operators/assign_value_op.cc index d5671c1183..8e3a530489 100644 --- a/paddle/operators/assign_value_op.cc +++ b/paddle/operators/assign_value_op.cc @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "paddle/operators/assign_value_op.h" diff --git a/paddle/operators/assign_value_op.h b/paddle/operators/assign_value_op.h index db2e430779..ec98c53513 100644 --- a/paddle/operators/assign_value_op.h +++ b/paddle/operators/assign_value_op.h @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #pragma once diff --git a/paddle/operators/net_op.cc b/paddle/operators/net_op.cc index f12074a5f2..000e029840 100644 --- a/paddle/operators/net_op.cc +++ b/paddle/operators/net_op.cc @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "paddle/operators/net_op.h" #include diff --git a/paddle/operators/sequence_reshape_op.cc b/paddle/operators/sequence_reshape_op.cc index 884c49276c..57cca13105 100644 --- a/paddle/operators/sequence_reshape_op.cc +++ b/paddle/operators/sequence_reshape_op.cc @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "paddle/operators/sequence_reshape_op.h" #include "paddle/framework/ddim.h" diff --git a/paddle/operators/sequence_reshape_op.h b/paddle/operators/sequence_reshape_op.h index dd9b611250..c6f528ab8a 100644 --- a/paddle/operators/sequence_reshape_op.h +++ b/paddle/operators/sequence_reshape_op.h @@ -1,16 +1,16 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #pragma once #include "paddle/framework/op_registry.h" diff --git a/paddle/optimizer/parameter_optimizer_test.cc b/paddle/optimizer/parameter_optimizer_test.cc index 795d2de1d6..2bcfca55cc 100644 --- a/paddle/optimizer/parameter_optimizer_test.cc +++ b/paddle/optimizer/parameter_optimizer_test.cc @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "parameter_optimizer.h" #include diff --git a/paddle/optimizer/serialization.h b/paddle/optimizer/serialization.h index 92fbf65cc6..98548ddb7a 100644 --- a/paddle/optimizer/serialization.h +++ b/paddle/optimizer/serialization.h @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include diff --git a/paddle/optimizer/serialization_test.cc b/paddle/optimizer/serialization_test.cc index 0f1b14eec1..25a8f5d351 100644 --- a/paddle/optimizer/serialization_test.cc +++ b/paddle/optimizer/serialization_test.cc @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "serialization.h" #include "gtest/gtest.h" diff --git a/paddle/platform/assert.h b/paddle/platform/assert.h index 70d3bf7506..d813b9529b 100644 --- a/paddle/platform/assert.h +++ b/paddle/platform/assert.h @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #define STRINGIFY(x) #x diff --git a/paddle/scripts/cluster_train/paddle.py b/paddle/scripts/cluster_train/paddle.py index e44bb4505b..ba313ac6a1 100644 --- a/paddle/scripts/cluster_train/paddle.py +++ b/paddle/scripts/cluster_train/paddle.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # @@ -93,168 +80,3 @@ def job_prepare(jobdir, data=None): #create job dir run('rm ' + jobdir + ' -fr && ' + 'mkdir -p ' + jobdir) #push data and paddle bin - put(data + "/*", jobdir) - run("mkdir -p " + log) - run('rm -fr ' + log + "/*") - - def set_nodefile(nodeid): - ''' - create nodefile for later usage - ''' - run('echo ' + str(nodeid) + ' > ' + jobdir + '/nodefile') - - execute(job_create_workspace, jobdir, data, hosts=conf.HOSTS) - for i in xrange(len(conf.HOSTS)): - execute(set_nodefile, i, hosts=conf.HOSTS[i]) - #clean rubbish caused by exception - with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) - - -def job_pserver(jobdir, pids=None): - ''' - start all pservers - ''' - pargs = " --num_gradient_servers=" + str(len(conf.HOSTS)) - pargs += (" --nics=" + conf.PADDLE_NIC) - pargs += " --port=" + str(conf.PADDLE_PORT) - pargs += " --ports_num=" + str(conf.PADDLE_PORTS_NUM) - #always start sparse pserver by default - pargs += " --ports_num_for_sparse=" + str(conf.PADDLE_PORTS_NUM_FOR_SPARSE) - pargs += " --comment=" + "paddle_process_by_paddle" - - def start_pserver(jobdir, pargs): - ''' - start pserver process with fabric executor - ''' - with prefix('export LD_LIBRARY_PATH=' + \ - conf.LD_LIBRARY_PATH + \ - ':$LD_LIBRARY_PATH'): - program = 'paddle pserver' - run('cd ' + jobdir + '; ' + \ - 'GLOG_logtostderr=0 GLOG_log_dir="./log" ' + \ - 'nohup ' + \ - program + " " + pargs + ' > ./log/server.log 2>&1 < /dev/null & ', - pty=False) - - execute(start_pserver, jobdir, pargs, hosts=conf.HOSTS) - - -def job_trainer(jobdir, train_args_dict, pids=None): - ''' - start paddle trainer - ''' - args = " --num_gradient_servers=" + str(len(conf.HOSTS)) - args += " --nics=" + conf.PADDLE_NIC - args += " --port=" + str(conf.PADDLE_PORT) - args += " --ports_num=" + str(conf.PADDLE_PORTS_NUM) - args += " --comment=" + "paddle_process_by_paddle" - ip_string = "" - for i in xrange(len(conf.HOSTS)): - host = conf.HOSTS[i] - left = host.find("@") - right = host.find(':') - left = 0 if left == -1 else left + 1 - right = len(host) if right == -1 else right - ip_string += (socket.gethostbyname(host[left:right]) + ",") - ip_string = ip_string.rstrip(",") - args += " --pservers=" + ip_string - - args_ext = "" - for key, value in train_args_dict.items(): - args_ext += (' --' + key + '=' + value) - args += " " + args_ext - - def start_trainer(jobdir, args): - ''' - start trainer process with fabric executor - ''' - with prefix('export LD_LIBRARY_PATH=' + \ - conf.LD_LIBRARY_PATH + \ - ':$LD_LIBRARY_PATH'): - program = 'paddle train' - run('cd ' + jobdir + '; ' + \ - 'GLOG_logtostderr=0 ' - 'GLOG_log_dir="./log" ' - 'nohup ' + \ - program + " " + args + " > ./log/train.log 2>&1 < /dev/null & ", - pty=False) - - for i in xrange(len(conf.HOSTS)): - train_args = copy.deepcopy(args) - train_args += " --trainer_id=" + str(i) - execute(start_trainer, jobdir, train_args, hosts=conf.HOSTS[i]) - - -def job_all(job_package, jobdir=None, train_args_dict=None): - ''' - param job_package - param train_args_dict - ''' - if jobdir is None: - timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime()) - jobdir = conf.ROOT_DIR + "/JOB" + timestamp - job_prepare(jobdir, job_package) - job_pserver(jobdir) - time.sleep(5) #wait until pservers completely start - job_trainer(jobdir, train_args_dict) - job_clean() - - -def job_clean(): - ''' - if starting job failed from paddle internal, the framework always - is launched successfully since these process are daemon processes. - so this job_clean can alway clean job rubbish process with ctrl+c. - ''' - - def signal_handler(signal, frame): - ''' - SIGINT handler - ''' - - def kill_process(): - run("ps aux \ - | grep paddle_process_by_paddle \ - | grep -v grep \ - | awk '{print $2}' \ - | xargs kill > /dev/null 2>&1") - - with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) - - signal.signal(signal.SIGINT, signal_handler) - signal.pause() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - prog="paddle.py", description='simple tool for cluster training') - parser.add_argument( - '-j', - '--job_workspace', - required=False, - default=None, - help='job workspace') - parser.add_argument( - '-p', - '--job_dispatch_package', - required=False, - default=None, - help='job package for dispatching to all other nodes') - - args, train_args_list = parser.parse_known_args() - train_args = refine_unknown_args(train_args_list) - train_args_dict = dict(zip(train_args[:-1:2], train_args[1::2])) - - if args.job_workspace is not None: - #if assigned workspace, do not need to dispatch data, - #so job_local_package should be None - assert args.job_dispatch_package is None - job_all(None, args.job_workspace, train_args_dict) - elif args.job_dispatch_package is not None: - assert args.job_workspace is None - assert os.path.isdir(args.job_dispatch_package) - job_all(args.job_dispatch_package, None, train_args_dict) - else: - print "--job_workspace or --job_dispatch_package should be set" diff --git a/paddle/scripts/cpplint.py b/paddle/scripts/cpplint.py index d0cbb070c4..dff4339ea3 100644 --- a/paddle/scripts/cpplint.py +++ b/paddle/scripts/cpplint.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python # # Copyright (c) 2009 Google Inc. All rights reserved. diff --git a/paddle/string/piece.cc b/paddle/string/piece.cc index 2a553e2832..330ca5f015 100644 --- a/paddle/string/piece.cc +++ b/paddle/string/piece.cc @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "paddle/string/piece.h" diff --git a/paddle/string/piece.h b/paddle/string/piece.h index fc95263379..f2bb6b2c76 100644 --- a/paddle/string/piece.h +++ b/paddle/string/piece.h @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #pragma once diff --git a/paddle/string/piece_test.cc b/paddle/string/piece_test.cc index fb8b972988..250f26d61f 100644 --- a/paddle/string/piece_test.cc +++ b/paddle/string/piece_test.cc @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ #include "paddle/string/piece.h" diff --git a/paddle/string/printf.h b/paddle/string/printf.h index 70d2511531..03809d2209 100644 --- a/paddle/string/printf.h +++ b/paddle/string/printf.h @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ // Compared with std::stringstream, there are primary purpose of // string::Printf: diff --git a/paddle/string/tinyformat/tinyformat.h b/paddle/string/tinyformat/tinyformat.h index 092c04c315..d1a2c47f1a 100644 --- a/paddle/string/tinyformat/tinyformat.h +++ b/paddle/string/tinyformat/tinyformat.h @@ -1,16 +1,17 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // tinyformat.h // Copyright (C) 2011, Chris Foster [chris42f (at) gmail (d0t) com] // diff --git a/paddle/trainer/tests/picojson.h b/paddle/trainer/tests/picojson.h index 4aa64961d0..eaa8b9baf6 100644 --- a/paddle/trainer/tests/picojson.h +++ b/paddle/trainer/tests/picojson.h @@ -1,3 +1,17 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* * Copyright 2009-2010 Cybozu Labs, Inc. * Copyright 2011-2014 Kazuho Oku diff --git a/paddle/trainer/tests/simple_sparse_neural_network.py b/paddle/trainer/tests/simple_sparse_neural_network.py index ba554d5872..970fb466dc 100644 --- a/paddle/trainer/tests/simple_sparse_neural_network.py +++ b/paddle/trainer/tests/simple_sparse_neural_network.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=17, learning_method=AdaGradOptimizer(), learning_rate=1e-4) diff --git a/paddle/trainer/tests/simple_sparse_neural_network_dp.py b/paddle/trainer/tests/simple_sparse_neural_network_dp.py index 44e96873f0..49043c9175 100644 --- a/paddle/trainer/tests/simple_sparse_neural_network_dp.py +++ b/paddle/trainer/tests/simple_sparse_neural_network_dp.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer.PyDataProvider2 import provider, integer_sequence, integer_value import random diff --git a/paddle/utils/enable_virtualenv.py b/paddle/utils/enable_virtualenv.py index 29f8deb324..4e998381e9 100644 --- a/paddle/utils/enable_virtualenv.py +++ b/paddle/utils/enable_virtualenv.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py index c944a96042..93b505a602 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-3, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py index 27b11ffdfc..745f060fa5 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-3, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py index 6a90051827..b6fc8f70f9 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py +++ b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py index 06115d62e7..6edc03bba0 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py +++ b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ''' Test all activations. ''' diff --git a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py index f5e90fdd89..59a71e1cd1 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py +++ b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.py b/python/paddle/trainer_config_helpers/tests/configs/projections.py index c683d378ca..96f06b4018 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/projections.py +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ''' Test mixed layer, projections and operators. ''' diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py index bf90d1762c..69a0a5b8ff 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_gru.py b/python/paddle/trainer_config_helpers/tests/configs/shared_gru.py index 7cfab83855..97b41fb372 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/shared_gru.py +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_gru.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py index 8a425c7062..4e653dedb9 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py index 8ee213a493..dc418325f8 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-4) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_BatchNorm3D.py b/python/paddle/trainer_config_helpers/tests/configs/test_BatchNorm3D.py index cbd3c3e97f..5b98e3fb34 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_BatchNorm3D.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_BatchNorm3D.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-4) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py b/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py index bed9154fe3..f3abdfe1ae 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-4) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py index 7e1da753f5..4eb9f207e0 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_clip_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_clip_layer.py index 0a719b0735..24564c105f 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_clip_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_clip_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='input', size=300) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_config_parser_for_non_file_config.py b/python/paddle/trainer_config_helpers/tests/configs/test_config_parser_for_non_file_config.py index 7003872700..9b791a0222 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_config_parser_for_non_file_config.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_config_parser_for_non_file_config.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. # diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_conv3d_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_conv3d_layer.py index fb2cacd443..35087c4228 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_conv3d_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_conv3d_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py index a8b5c860ef..b076b89106 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py index eba2e1e483..fa7a1abe9a 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_crop.py b/python/paddle/trainer_config_helpers/tests/configs/test_crop.py index 870388faf7..569d747857 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_crop.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_crop.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cross_entropy_over_beam.py b/python/paddle/trainer_config_helpers/tests/configs/test_cross_entropy_over_beam.py index 253244dcd4..4a5bdf1181 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cross_entropy_over_beam.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cross_entropy_over_beam.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python #coding=utf-8 diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_deconv3d_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_deconv3d_layer.py index db950093b3..4f27d99873 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_deconv3d_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_deconv3d_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py index d304a29859..d37954222e 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_dot_prod_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_dot_prod_layer.py index 2e5dde2da2..63ba0a72b9 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_dot_prod_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_dot_prod_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * vec1 = data_layer(name='vector1', size=10) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py index 345fb2b6ab..9892bca05d 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_factorization_machine.py b/python/paddle/trainer_config_helpers/tests/configs/test_factorization_machine.py index 3a489a39da..6fb773d9f7 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_factorization_machine.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_factorization_machine.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='data', size=1024) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_fc.py b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py index 90b0e37270..4dd37d0242 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_fc.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_gated_unit_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_gated_unit_layer.py index 2bd4ab2da4..082646b9d3 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_gated_unit_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_gated_unit_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='input', size=256) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py index 451909ee18..f5271b8280 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-4) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py index 3ebe40aadc..ad86d7d5bd 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_kmax_seq_socre_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_kmax_seq_socre_layer.py index c762467feb..171da10f75 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_kmax_seq_socre_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_kmax_seq_socre_layer.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python #coding=utf-8 from paddle.trainer_config_helpers import * diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_l2_distance_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_l2_distance_layer.py index 58bf3de104..1796e1c6b6 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_l2_distance_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_l2_distance_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * outputs( diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py index 8d570706df..7484818ab2 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py index 3b6117d297..22788be2e9 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py index 083d064367..0dcccc49e4 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py index 9c14455841..046d38741e 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py index 046698fb4e..d81128c77c 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_pad.py b/python/paddle/trainer_config_helpers/tests/configs/test_pad.py index 1046db2f09..44b0b34d5a 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_pad.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_pad.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_pooling3D_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_pooling3D_layer.py index 37805d4376..e257e735ad 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_pooling3D_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_pooling3D_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=100, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_prelu_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_prelu_layer.py index 10d759f6d9..098e2397ec 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_prelu_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_prelu_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='input', size=300, height=10, width=10) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py index 22e0ce3e5a..714d8893e9 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_recursive_topology.py b/python/paddle/trainer_config_helpers/tests/configs/test_recursive_topology.py index d1d97f1c5e..188a3d2320 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_recursive_topology.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_recursive_topology.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_repeat_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_repeat_layer.py index 6818b91f96..93b673afee 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_repeat_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_repeat_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py index ce8a22ebb1..3a202974e3 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='input', size=300) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py index 79dad5e250..91074b8fdf 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py index 264341f899..f0a37f7e99 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='data', size=3 * 14 * 14, height=14, width=14) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_row_conv.py b/python/paddle/trainer_config_helpers/tests/configs/test_row_conv.py index 342a5029a8..68b1a991f3 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_row_conv.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_row_conv.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_row_l2_norm_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_row_l2_norm_layer.py index 9521fa6c47..c25393f580 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_row_l2_norm_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_row_l2_norm_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='input', size=300) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py index 698d19d037..3691e8daea 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='data', size=100) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py index 22fb25d0f2..426afcf3a0 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_seq_concat_reshape.py b/python/paddle/trainer_config_helpers/tests/configs/test_seq_concat_reshape.py index 1883ed9d4e..7296081857 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_seq_concat_reshape.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_seq_concat_reshape.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_seq_slice_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_seq_slice_layer.py index 12d7f1f33b..510ad32208 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_seq_slice_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_seq_slice_layer.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python #coding=utf-8 from paddle.trainer_config_helpers import * diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py index 8cf5fd70e3..d13a5a8429 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_smooth_l1.py b/python/paddle/trainer_config_helpers/tests/configs/test_smooth_l1.py index 7188d82a53..42225b8505 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_smooth_l1.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_smooth_l1.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * data = data_layer(name='input', size=300) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py b/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py index a628272196..7ebdf7408d 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * define_py_data_sources2( diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py index 58c1675e6b..1f19ea77ad 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=100, learning_rate=1e-5) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sub_nested_seq_select_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_sub_nested_seq_select_layer.py index 64d1d7b6ee..6d1c3175ba 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_sub_nested_seq_select_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sub_nested_seq_select_layer.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/env python #coding=utf-8 from paddle.trainer_config_helpers import * diff --git a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py index 6294cb04ef..8581ba60ab 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(batch_size=1000, learning_rate=1e-4) diff --git a/python/paddle/trainer_config_helpers/tests/configs/util_layers.py b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py index 89b881b361..a66c9515c7 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/util_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.trainer_config_helpers import * settings(learning_rate=1e-4, batch_size=1000) diff --git a/python/paddle/trainer_config_helpers/tests/test_reset_hook.py b/python/paddle/trainer_config_helpers/tests/test_reset_hook.py index 0423babdb7..81186dedd2 100644 --- a/python/paddle/trainer_config_helpers/tests/test_reset_hook.py +++ b/python/paddle/trainer_config_helpers/tests/test_reset_hook.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import unittest from paddle.trainer.config_parser import parse_config diff --git a/python/paddle/utils/image_multiproc.py b/python/paddle/utils/image_multiproc.py index 1acf40df58..fdbefef9ff 100644 --- a/python/paddle/utils/image_multiproc.py +++ b/python/paddle/utils/image_multiproc.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os, sys import numpy as np from PIL import Image diff --git a/python/paddle/utils/plotcurve.py b/python/paddle/utils/plotcurve.py index 27a69b6a5c..27bd8157d3 100644 --- a/python/paddle/utils/plotcurve.py +++ b/python/paddle/utils/plotcurve.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. #!/usr/bin/python # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved # diff --git a/python/paddle/v2/dataset/sentiment.py b/python/paddle/v2/dataset/sentiment.py index 7174413018..b0b9757c1a 100644 --- a/python/paddle/v2/dataset/sentiment.py +++ b/python/paddle/v2/dataset/sentiment.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. # /usr/bin/env python # -*- coding:utf-8 -*- diff --git a/python/paddle/v2/dataset/tests/imikolov_test.py b/python/paddle/v2/dataset/tests/imikolov_test.py index 9b3ab72feb..eed1458244 100644 --- a/python/paddle/v2/dataset/tests/imikolov_test.py +++ b/python/paddle/v2/dataset/tests/imikolov_test.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.dataset.imikolov import unittest diff --git a/python/paddle/v2/dataset/tests/test_sentiment.py b/python/paddle/v2/dataset/tests/test_sentiment.py index f107948801..4074052907 100644 --- a/python/paddle/v2/dataset/tests/test_sentiment.py +++ b/python/paddle/v2/dataset/tests/test_sentiment.py @@ -1,16 +1,3 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. # /usr/bin/env python # -*- coding:utf-8 -*- diff --git a/python/paddle/v2/event.py b/python/paddle/v2/event.py index f322bffe13..01067ef426 100644 --- a/python/paddle/v2/event.py +++ b/python/paddle/v2/event.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Testing and training events. diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index 8c29ee741c..4ae4165ba4 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function # import all class inside framework into fluid module import framework diff --git a/python/paddle/v2/fluid/backward.py b/python/paddle/v2/fluid/backward.py index 27cf637c48..ae81d68baf 100644 --- a/python/paddle/v2/fluid/backward.py +++ b/python/paddle/v2/fluid/backward.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.v2.fluid import framework as framework from . import core import collections diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index e4d9ed599e..5241f4843c 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import functools import layers from . import core diff --git a/python/paddle/v2/fluid/data_feeder.py b/python/paddle/v2/fluid/data_feeder.py index bfdd00e3ef..a3b22a8633 100644 --- a/python/paddle/v2/fluid/data_feeder.py +++ b/python/paddle/v2/fluid/data_feeder.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import core import numpy diff --git a/python/paddle/v2/fluid/default_scope_funcs.py b/python/paddle/v2/fluid/default_scope_funcs.py index 2218bb140a..a27280208b 100644 --- a/python/paddle/v2/fluid/default_scope_funcs.py +++ b/python/paddle/v2/fluid/default_scope_funcs.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Default scope function. diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 02a0e4cd26..66c5d46fa2 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import framework from framework import Program, default_main_program, Parameter, Variable diff --git a/python/paddle/v2/fluid/distribute_transpiler_simple.py b/python/paddle/v2/fluid/distribute_transpiler_simple.py index 56ffb56b12..73d9bed1ae 100644 --- a/python/paddle/v2/fluid/distribute_transpiler_simple.py +++ b/python/paddle/v2/fluid/distribute_transpiler_simple.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import framework from framework import Program, default_main_program, Parameter, Variable import optimizer diff --git a/python/paddle/v2/fluid/distributed_spliter.py b/python/paddle/v2/fluid/distributed_spliter.py index e647f760e9..8cf0b06786 100644 --- a/python/paddle/v2/fluid/distributed_spliter.py +++ b/python/paddle/v2/fluid/distributed_spliter.py @@ -1,16 +1,18 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + def hash_name(varlist, pserver_endpoints): """ hash variable names to several endpoints. diff --git a/python/paddle/v2/fluid/evaluator.py b/python/paddle/v2/fluid/evaluator.py index adf174a07d..396d56fc8b 100644 --- a/python/paddle/v2/fluid/evaluator.py +++ b/python/paddle/v2/fluid/evaluator.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import layers diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index a99c5157b2..9d5ed9571a 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import contextlib from framework import Program, default_main_program diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 4f8366b640..f876665458 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import collections import contextlib diff --git a/python/paddle/v2/fluid/initializer.py b/python/paddle/v2/fluid/initializer.py index 2e8cfa3177..b9c0d12ad6 100644 --- a/python/paddle/v2/fluid/initializer.py +++ b/python/paddle/v2/fluid/initializer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import framework import numpy as np diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index e7a06a0714..376d6013a3 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import cPickle as pickle diff --git a/python/paddle/v2/fluid/layer_helper.py b/python/paddle/v2/fluid/layer_helper.py index 191d2349b5..0b0064ade9 100644 --- a/python/paddle/v2/fluid/layer_helper.py +++ b/python/paddle/v2/fluid/layer_helper.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy import itertools diff --git a/python/paddle/v2/fluid/layers/__init__.py b/python/paddle/v2/fluid/layers/__init__.py index c190af3329..cc8a1b1ce5 100644 --- a/python/paddle/v2/fluid/layers/__init__.py +++ b/python/paddle/v2/fluid/layers/__init__.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import ops from ops import * import nn diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index e72b22c83f..2f1188c542 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from ..layer_helper import LayerHelper, unique_name from ..framework import Program, Variable, Operator from .. import core diff --git a/python/paddle/v2/fluid/layers/device.py b/python/paddle/v2/fluid/layers/device.py index ef74b2b2f0..736813d1b1 100644 --- a/python/paddle/v2/fluid/layers/device.py +++ b/python/paddle/v2/fluid/layers/device.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ All util layers. """ diff --git a/python/paddle/v2/fluid/layers/io.py b/python/paddle/v2/fluid/layers/io.py index a43e0ee4de..9af00e7de5 100644 --- a/python/paddle/v2/fluid/layers/io.py +++ b/python/paddle/v2/fluid/layers/io.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .. import core from ..layer_helper import LayerHelper diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 78d162c097..b1db16a83e 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ All layers just related to the neural network. """ diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index 21945edf08..7716052a5c 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from ..registry import register_layer __activations__ = [ diff --git a/python/paddle/v2/fluid/layers/tensor.py b/python/paddle/v2/fluid/layers/tensor.py index 255b9d4678..6e7d09459c 100644 --- a/python/paddle/v2/fluid/layers/tensor.py +++ b/python/paddle/v2/fluid/layers/tensor.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from ..layer_helper import LayerHelper from ..param_attr import ParamAttr from ..framework import convert_np_dtype_to_dtype_ diff --git a/python/paddle/v2/fluid/memory_optimization_transpiler.py b/python/paddle/v2/fluid/memory_optimization_transpiler.py index 89ffe26ed1..1b4b647559 100644 --- a/python/paddle/v2/fluid/memory_optimization_transpiler.py +++ b/python/paddle/v2/fluid/memory_optimization_transpiler.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import defaultdict import framework from framework import Program, default_main_program, Parameter, Variable diff --git a/python/paddle/v2/fluid/net_drawer.py b/python/paddle/v2/fluid/net_drawer.py index 7448975b59..9b126f5197 100644 --- a/python/paddle/v2/fluid/net_drawer.py +++ b/python/paddle/v2/fluid/net_drawer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import json import logging diff --git a/python/paddle/v2/fluid/nets.py b/python/paddle/v2/fluid/nets.py index c53fbd280f..a30e646d8c 100644 --- a/python/paddle/v2/fluid/nets.py +++ b/python/paddle/v2/fluid/nets.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import layers __all__ = [ diff --git a/python/paddle/v2/fluid/op.py b/python/paddle/v2/fluid/op.py index 4bc0f79c64..f368e0c2d8 100644 --- a/python/paddle/v2/fluid/op.py +++ b/python/paddle/v2/fluid/op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core import paddle.v2.fluid.proto.framework_pb2 as framework_pb2 diff --git a/python/paddle/v2/fluid/optimizer.py b/python/paddle/v2/fluid/optimizer.py index 8bd62ef0c0..0c3533b892 100644 --- a/python/paddle/v2/fluid/optimizer.py +++ b/python/paddle/v2/fluid/optimizer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import defaultdict import framework diff --git a/python/paddle/v2/fluid/param_attr.py b/python/paddle/v2/fluid/param_attr.py index 3af0190590..26e9111f6f 100644 --- a/python/paddle/v2/fluid/param_attr.py +++ b/python/paddle/v2/fluid/param_attr.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from initializer import Initializer, Xavier, Constant from regularizer import WeightDecayRegularizer diff --git a/python/paddle/v2/fluid/profiler.py b/python/paddle/v2/fluid/profiler.py index f049498b9f..29e0d54a3a 100644 --- a/python/paddle/v2/fluid/profiler.py +++ b/python/paddle/v2/fluid/profiler.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core from contextlib import contextmanager import os diff --git a/python/paddle/v2/fluid/regularizer.py b/python/paddle/v2/fluid/regularizer.py index e53dee98fd..c2f28eecfd 100644 --- a/python/paddle/v2/fluid/regularizer.py +++ b/python/paddle/v2/fluid/regularizer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import framework __all__ = [ diff --git a/python/paddle/v2/fluid/tests/__init__.py b/python/paddle/v2/fluid/tests/__init__.py index 2619c1c0e9..b94a21a7e4 100644 --- a/python/paddle/v2/fluid/tests/__init__.py +++ b/python/paddle/v2/fluid/tests/__init__.py @@ -1,13 +1,13 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/paddle/v2/fluid/tests/book/test_fit_a_line.py b/python/paddle/v2/fluid/tests/book/test_fit_a_line.py index 904df66dc1..462669c262 100644 --- a/python/paddle/v2/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book/test_fit_a_line.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book/test_image_classification_train.py b/python/paddle/v2/fluid/tests/book/test_image_classification_train.py index a06486aa08..30582a21d0 100644 --- a/python/paddle/v2/fluid/tests/book/test_image_classification_train.py +++ b/python/paddle/v2/fluid/tests/book/test_image_classification_train.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import sys diff --git a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py index 42971da0f0..be92c48ac5 100644 --- a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import numpy as np diff --git a/python/paddle/v2/fluid/tests/book/test_machine_translation.py b/python/paddle/v2/fluid/tests/book/test_machine_translation.py index deeb6b1bad..53ae200a23 100644 --- a/python/paddle/v2/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/v2/fluid/tests/book/test_machine_translation.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv.py index 1d5defbed3..4710d16c24 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_conv.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py index 02da2fcc85..be22e97054 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/book/test_recommender_system.py b/python/paddle/v2/fluid/tests/book/test_recommender_system.py index 47e2afcd83..d4a694e572 100644 --- a/python/paddle/v2/fluid/tests/book/test_recommender_system.py +++ b/python/paddle/v2/fluid/tests/book/test_recommender_system.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_conv.py b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_conv.py index b44d2b41e3..df27399dd2 100644 --- a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_conv.py +++ b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_conv.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_dynamic_lstm.py b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_dynamic_lstm.py index 5a139c1dcd..529223eba8 100644 --- a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_dynamic_lstm.py +++ b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_dynamic_lstm.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py index fab8a82f85..6181914241 100644 --- a/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py +++ b/python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book/test_word2vec.py b/python/paddle/v2/fluid/tests/book/test_word2vec.py index 3d4bbccd33..8cf54846fe 100644 --- a/python/paddle/v2/fluid/tests/book/test_word2vec.py +++ b/python/paddle/v2/fluid/tests/book/test_word2vec.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py index 00e0b22c64..52c7ecdeb3 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_fit_a_line.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py index adc7ae8adf..08bb67b0a1 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_label_semantic_roles.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import numpy as np diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py index 27ef2cad1d..04b3113690 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_dist_word2vec.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py index ee5cd07fb3..f18ca05c78 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_conv_dist.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py index 9cfb4ab8c4..7733248cb4 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_recognize_digits_mlp_dist.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py b/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py index 840afb6376..49f26d6b69 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py +++ b/python/paddle/v2/fluid/tests/book_distribute/notest_understand_sentiment_conv_dist.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import os import numpy as np diff --git a/python/paddle/v2/fluid/tests/book_distribute/test_split_var.py b/python/paddle/v2/fluid/tests/book_distribute/test_split_var.py index f979f642d8..4a50049bf2 100644 --- a/python/paddle/v2/fluid/tests/book_distribute/test_split_var.py +++ b/python/paddle/v2/fluid/tests/book_distribute/test_split_var.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import unittest from paddle.v2.fluid.distribute_transpiler import split_dense_variable diff --git a/python/paddle/v2/fluid/tests/decorators.py b/python/paddle/v2/fluid/tests/decorators.py index 3b314a15e1..0a8a2ccc4d 100644 --- a/python/paddle/v2/fluid/tests/decorators.py +++ b/python/paddle/v2/fluid/tests/decorators.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid as fluid __all__ = ['many_times', 'prog_scope'] diff --git a/python/paddle/v2/fluid/tests/demo/fc_gan.py b/python/paddle/v2/fluid/tests/demo/fc_gan.py index 5f9e8f9507..0652c8134d 100644 --- a/python/paddle/v2/fluid/tests/demo/fc_gan.py +++ b/python/paddle/v2/fluid/tests/demo/fc_gan.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import errno import math import os diff --git a/python/paddle/v2/fluid/tests/op_test.py b/python/paddle/v2/fluid/tests/op_test.py index c3b2220e6e..56f54de86f 100644 --- a/python/paddle/v2/fluid/tests/op_test.py +++ b/python/paddle/v2/fluid/tests/op_test.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import random diff --git a/python/paddle/v2/fluid/tests/test_accuracy_op.py b/python/paddle/v2/fluid/tests/test_accuracy_op.py index a20abac8a0..ac3f3bdff4 100644 --- a/python/paddle/v2/fluid/tests/test_accuracy_op.py +++ b/python/paddle/v2/fluid/tests/test_accuracy_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_activation_op.py b/python/paddle/v2/fluid/tests/test_activation_op.py index a6a6eb9d63..18605e6065 100644 --- a/python/paddle/v2/fluid/tests/test_activation_op.py +++ b/python/paddle/v2/fluid/tests/test_activation_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_adadelta_op.py b/python/paddle/v2/fluid/tests/test_adadelta_op.py index 8de6a1f9a9..949318d007 100644 --- a/python/paddle/v2/fluid/tests/test_adadelta_op.py +++ b/python/paddle/v2/fluid/tests/test_adadelta_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_adagrad_op.py b/python/paddle/v2/fluid/tests/test_adagrad_op.py index 30ed092d48..86b0567ce1 100644 --- a/python/paddle/v2/fluid/tests/test_adagrad_op.py +++ b/python/paddle/v2/fluid/tests/test_adagrad_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_adam_op.py b/python/paddle/v2/fluid/tests/test_adam_op.py index 32d00cf702..10580adca7 100644 --- a/python/paddle/v2/fluid/tests/test_adam_op.py +++ b/python/paddle/v2/fluid/tests/test_adam_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_adamax_op.py b/python/paddle/v2/fluid/tests/test_adamax_op.py index 35b2bc47ed..e285c454f0 100644 --- a/python/paddle/v2/fluid/tests/test_adamax_op.py +++ b/python/paddle/v2/fluid/tests/test_adamax_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_array_read_write_op.py b/python/paddle/v2/fluid/tests/test_array_read_write_op.py index 8775cd4f9f..a32c24486e 100644 --- a/python/paddle/v2/fluid/tests/test_array_read_write_op.py +++ b/python/paddle/v2/fluid/tests/test_array_read_write_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core import paddle.v2.fluid.layers as layers diff --git a/python/paddle/v2/fluid/tests/test_assign_op.py b/python/paddle/v2/fluid/tests/test_assign_op.py index 4ac173c96b..fbbfe0d02c 100644 --- a/python/paddle/v2/fluid/tests/test_assign_op.py +++ b/python/paddle/v2/fluid/tests/test_assign_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import op_test import numpy import unittest diff --git a/python/paddle/v2/fluid/tests/test_assign_value_op.py b/python/paddle/v2/fluid/tests/test_assign_value_op.py index f4e2ff9bde..93970f863b 100644 --- a/python/paddle/v2/fluid/tests/test_assign_value_op.py +++ b/python/paddle/v2/fluid/tests/test_assign_value_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid as fluid import paddle.v2.fluid.layers as layers import op_test diff --git a/python/paddle/v2/fluid/tests/test_auc_op.py b/python/paddle/v2/fluid/tests/test_auc_op.py index aa74d224d5..5e4caedf5d 100644 --- a/python/paddle/v2/fluid/tests/test_auc_op.py +++ b/python/paddle/v2/fluid/tests/test_auc_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_batch_norm_op.py b/python/paddle/v2/fluid/tests/test_batch_norm_op.py index fe82b7d7f3..371bd42678 100644 --- a/python/paddle/v2/fluid/tests/test_batch_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_batch_norm_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_beam_search_decode_op.py b/python/paddle/v2/fluid/tests/test_beam_search_decode_op.py index 9ef6e08cc1..3674784985 100644 --- a/python/paddle/v2/fluid/tests/test_beam_search_decode_op.py +++ b/python/paddle/v2/fluid/tests/test_beam_search_decode_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_beam_search_op.py b/python/paddle/v2/fluid/tests/test_beam_search_op.py index f31c737ba6..4da463df26 100644 --- a/python/paddle/v2/fluid/tests/test_beam_search_op.py +++ b/python/paddle/v2/fluid/tests/test_beam_search_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from paddle.v2.fluid.op import Operator, DynamicRecurrentOp import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_bilinear_tensor_product_op.py b/python/paddle/v2/fluid/tests/test_bilinear_tensor_product_op.py index aed1bf4d3a..4b03f512c2 100644 --- a/python/paddle/v2/fluid/tests/test_bilinear_tensor_product_op.py +++ b/python/paddle/v2/fluid/tests/test_bilinear_tensor_product_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_calc_gradient.py b/python/paddle/v2/fluid/tests/test_calc_gradient.py index b99eeb09cd..c773e81768 100644 --- a/python/paddle/v2/fluid/tests/test_calc_gradient.py +++ b/python/paddle/v2/fluid/tests/test_calc_gradient.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/test_cast_op.py b/python/paddle/v2/fluid/tests/test_cast_op.py index 3795b96dbf..327b246ed8 100644 --- a/python/paddle/v2/fluid/tests/test_cast_op.py +++ b/python/paddle/v2/fluid/tests/test_cast_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import op_test import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_chunk_eval_op.py b/python/paddle/v2/fluid/tests/test_chunk_eval_op.py index 59ef2bbb2f..5c3efe9baa 100644 --- a/python/paddle/v2/fluid/tests/test_chunk_eval_op.py +++ b/python/paddle/v2/fluid/tests/test_chunk_eval_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_clip.py b/python/paddle/v2/fluid/tests/test_clip.py index 63353a1096..6f7718f4d8 100644 --- a/python/paddle/v2/fluid/tests/test_clip.py +++ b/python/paddle/v2/fluid/tests/test_clip.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/test_clip_by_norm_op.py b/python/paddle/v2/fluid/tests/test_clip_by_norm_op.py index 5147e75046..b30f321c79 100644 --- a/python/paddle/v2/fluid/tests/test_clip_by_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_clip_by_norm_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_clip_op.py b/python/paddle/v2/fluid/tests/test_clip_op.py index 3338dc61b3..ef0b75e286 100644 --- a/python/paddle/v2/fluid/tests/test_clip_op.py +++ b/python/paddle/v2/fluid/tests/test_clip_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_compare_op.py b/python/paddle/v2/fluid/tests/test_compare_op.py index fbf8921e40..08ef90b10e 100644 --- a/python/paddle/v2/fluid/tests/test_compare_op.py +++ b/python/paddle/v2/fluid/tests/test_compare_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import op_test import unittest import numpy diff --git a/python/paddle/v2/fluid/tests/test_concat_op.py b/python/paddle/v2/fluid/tests/test_concat_op.py index 3e413e1540..ea0a95ebec 100644 --- a/python/paddle/v2/fluid/tests/test_concat_op.py +++ b/python/paddle/v2/fluid/tests/test_concat_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_cond_op.py b/python/paddle/v2/fluid/tests/test_cond_op.py index 5312fa51a2..4b7ca0963e 100644 --- a/python/paddle/v2/fluid/tests/test_cond_op.py +++ b/python/paddle/v2/fluid/tests/test_cond_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import paddle.v2.fluid.core as core import unittest diff --git a/python/paddle/v2/fluid/tests/test_conditional_block.py b/python/paddle/v2/fluid/tests/test_conditional_block.py index 965e7d39c8..5ee729cfee 100644 --- a/python/paddle/v2/fluid/tests/test_conditional_block.py +++ b/python/paddle/v2/fluid/tests/test_conditional_block.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.layers as layers import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_const_value.py b/python/paddle/v2/fluid/tests/test_const_value.py index 190bfa779b..d5b7cfded1 100644 --- a/python/paddle/v2/fluid/tests/test_const_value.py +++ b/python/paddle/v2/fluid/tests/test_const_value.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.framework as framework diff --git a/python/paddle/v2/fluid/tests/test_conv2d_op.py b/python/paddle/v2/fluid/tests/test_conv2d_op.py index 8b03a3ae16..24de74d730 100644 --- a/python/paddle/v2/fluid/tests/test_conv2d_op.py +++ b/python/paddle/v2/fluid/tests/test_conv2d_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py b/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py index b7b86c58fb..0c76e222c9 100644 --- a/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py +++ b/python/paddle/v2/fluid/tests/test_conv2d_transpose_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_conv3d_op.py b/python/paddle/v2/fluid/tests/test_conv3d_op.py index 5b0397cc69..8121e32865 100644 --- a/python/paddle/v2/fluid/tests/test_conv3d_op.py +++ b/python/paddle/v2/fluid/tests/test_conv3d_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py b/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py index b08969062a..4934c5a34e 100644 --- a/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py +++ b/python/paddle/v2/fluid/tests/test_conv3d_transpose_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_conv_shift_op.py b/python/paddle/v2/fluid/tests/test_conv_shift_op.py index 14b2640e24..7029d5a2eb 100644 --- a/python/paddle/v2/fluid/tests/test_conv_shift_op.py +++ b/python/paddle/v2/fluid/tests/test_conv_shift_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_cos_sim_op.py b/python/paddle/v2/fluid/tests/test_cos_sim_op.py index f6e5e2cbe9..33db12ba9c 100644 --- a/python/paddle/v2/fluid/tests/test_cos_sim_op.py +++ b/python/paddle/v2/fluid/tests/test_cos_sim_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_create_op_doc_string.py b/python/paddle/v2/fluid/tests/test_create_op_doc_string.py index 6c92264221..2b7951ecea 100644 --- a/python/paddle/v2/fluid/tests/test_create_op_doc_string.py +++ b/python/paddle/v2/fluid/tests/test_create_op_doc_string.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.layers as layers diff --git a/python/paddle/v2/fluid/tests/test_crf_decoding_op.py b/python/paddle/v2/fluid/tests/test_crf_decoding_op.py index 40e80a824a..f819387cdc 100644 --- a/python/paddle/v2/fluid/tests/test_crf_decoding_op.py +++ b/python/paddle/v2/fluid/tests/test_crf_decoding_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import random import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_crop_op.py b/python/paddle/v2/fluid/tests/test_crop_op.py index a0b2fc954d..36bf176168 100644 --- a/python/paddle/v2/fluid/tests/test_crop_op.py +++ b/python/paddle/v2/fluid/tests/test_crop_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_cross_entropy_op.py b/python/paddle/v2/fluid/tests/test_cross_entropy_op.py index f05e6b2356..ae8e9be6de 100644 --- a/python/paddle/v2/fluid/tests/test_cross_entropy_op.py +++ b/python/paddle/v2/fluid/tests/test_cross_entropy_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest, randomize_probability diff --git a/python/paddle/v2/fluid/tests/test_ctc_align.py b/python/paddle/v2/fluid/tests/test_ctc_align.py index 5a7c16997c..773c69d1ad 100644 --- a/python/paddle/v2/fluid/tests/test_ctc_align.py +++ b/python/paddle/v2/fluid/tests/test_ctc_align.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import sys import unittest diff --git a/python/paddle/v2/fluid/tests/test_data_feeder.py b/python/paddle/v2/fluid/tests/test_data_feeder.py index 5574766f8f..f967221015 100644 --- a/python/paddle/v2/fluid/tests/test_data_feeder.py +++ b/python/paddle/v2/fluid/tests/test_data_feeder.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/test_decayed_adagrad_op.py b/python/paddle/v2/fluid/tests/test_decayed_adagrad_op.py index 5e745a2843..78d4e3608e 100644 --- a/python/paddle/v2/fluid/tests/test_decayed_adagrad_op.py +++ b/python/paddle/v2/fluid/tests/test_decayed_adagrad_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_default_scope_funcs.py b/python/paddle/v2/fluid/tests/test_default_scope_funcs.py index 7a62168be9..5ff52f6d6b 100644 --- a/python/paddle/v2/fluid/tests/test_default_scope_funcs.py +++ b/python/paddle/v2/fluid/tests/test_default_scope_funcs.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.v2.fluid.default_scope_funcs import * import unittest diff --git a/python/paddle/v2/fluid/tests/test_detection_output_op.py b/python/paddle/v2/fluid/tests/test_detection_output_op.py index 147a43628c..4a9cd474b8 100644 --- a/python/paddle/v2/fluid/tests/test_detection_output_op.py +++ b/python/paddle/v2/fluid/tests/test_detection_output_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_dropout_op.py b/python/paddle/v2/fluid/tests/test_dropout_op.py index f401050dcc..107b9567dc 100644 --- a/python/paddle/v2/fluid/tests/test_dropout_op.py +++ b/python/paddle/v2/fluid/tests/test_dropout_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_dyn_rnn.py b/python/paddle/v2/fluid/tests/test_dyn_rnn.py index a946fea58d..2ac926c63c 100644 --- a/python/paddle/v2/fluid/tests/test_dyn_rnn.py +++ b/python/paddle/v2/fluid/tests/test_dyn_rnn.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid as fluid import paddle.v2 as paddle import unittest diff --git a/python/paddle/v2/fluid/tests/test_dynrnn_gradient_check.py b/python/paddle/v2/fluid/tests/test_dynrnn_gradient_check.py index 95cc80739d..dd608432df 100644 --- a/python/paddle/v2/fluid/tests/test_dynrnn_gradient_check.py +++ b/python/paddle/v2/fluid/tests/test_dynrnn_gradient_check.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy import random import collections diff --git a/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py b/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py index d6878f0b6d..d14923b6b3 100644 --- a/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py +++ b/python/paddle/v2/fluid/tests/test_dynrnn_static_input.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2 as paddle import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_edit_distance_op.py b/python/paddle/v2/fluid/tests/test_edit_distance_op.py index 5f5634e297..11cb85a151 100644 --- a/python/paddle/v2/fluid/tests/test_edit_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_edit_distance_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py index 9526f0199b..0fc15693b1 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_max_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_max_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py index b900728233..51584d6980 100644 --- a/python/paddle/v2/fluid/tests/test_elementwise_min_op.py +++ b/python/paddle/v2/fluid/tests/test_elementwise_min_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_exception.py b/python/paddle/v2/fluid/tests/test_exception.py index 98c4cbe3f2..cd57ca586b 100644 --- a/python/paddle/v2/fluid/tests/test_exception.py +++ b/python/paddle/v2/fluid/tests/test_exception.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core import unittest diff --git a/python/paddle/v2/fluid/tests/test_executor_and_mul.py b/python/paddle/v2/fluid/tests/test_executor_and_mul.py index e8baf631e5..44f93be6cb 100644 --- a/python/paddle/v2/fluid/tests/test_executor_and_mul.py +++ b/python/paddle/v2/fluid/tests/test_executor_and_mul.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy diff --git a/python/paddle/v2/fluid/tests/test_expand_op.py b/python/paddle/v2/fluid/tests/test_expand_op.py index 0524f2041f..b1a1cbc0fa 100644 --- a/python/paddle/v2/fluid/tests/test_expand_op.py +++ b/python/paddle/v2/fluid/tests/test_expand_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_feed_fetch_method.py b/python/paddle/v2/fluid/tests/test_feed_fetch_method.py index 718311517d..827a7590ff 100644 --- a/python/paddle/v2/fluid/tests/test_feed_fetch_method.py +++ b/python/paddle/v2/fluid/tests/test_feed_fetch_method.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_fill_constant_batch_size_like_op.py b/python/paddle/v2/fluid/tests/test_fill_constant_batch_size_like_op.py index 0adc487c04..f34a1ceb23 100644 --- a/python/paddle/v2/fluid/tests/test_fill_constant_batch_size_like_op.py +++ b/python/paddle/v2/fluid/tests/test_fill_constant_batch_size_like_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_fill_constant_op.py b/python/paddle/v2/fluid/tests/test_fill_constant_op.py index 50d4ccb3bd..a05fa39729 100644 --- a/python/paddle/v2/fluid/tests/test_fill_constant_op.py +++ b/python/paddle/v2/fluid/tests/test_fill_constant_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_fill_op.py b/python/paddle/v2/fluid/tests/test_fill_op.py index 42b06ec87c..901546f6f8 100644 --- a/python/paddle/v2/fluid/tests/test_fill_op.py +++ b/python/paddle/v2/fluid/tests/test_fill_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py b/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py index a28bed9697..b7f0b96647 100644 --- a/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py +++ b/python/paddle/v2/fluid/tests/test_fill_zeros_like_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_framework_debug_str.py b/python/paddle/v2/fluid/tests/test_framework_debug_str.py index 6c82e67220..f8fcfb2249 100644 --- a/python/paddle/v2/fluid/tests/test_framework_debug_str.py +++ b/python/paddle/v2/fluid/tests/test_framework_debug_str.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest from paddle.v2.fluid.framework import Program diff --git a/python/paddle/v2/fluid/tests/test_ftrl_op.py b/python/paddle/v2/fluid/tests/test_ftrl_op.py index 599233efd9..895337de0f 100644 --- a/python/paddle/v2/fluid/tests/test_ftrl_op.py +++ b/python/paddle/v2/fluid/tests/test_ftrl_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_gather_op.py b/python/paddle/v2/fluid/tests/test_gather_op.py index 95093f9b84..7675636797 100644 --- a/python/paddle/v2/fluid/tests/test_gather_op.py +++ b/python/paddle/v2/fluid/tests/test_gather_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_gaussian_random_op.py b/python/paddle/v2/fluid/tests/test_gaussian_random_op.py index bf4785211e..82842534d4 100644 --- a/python/paddle/v2/fluid/tests/test_gaussian_random_op.py +++ b/python/paddle/v2/fluid/tests/test_gaussian_random_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy diff --git a/python/paddle/v2/fluid/tests/test_get_places_op.py b/python/paddle/v2/fluid/tests/test_get_places_op.py index b44011fb76..68698c5f4a 100644 --- a/python/paddle/v2/fluid/tests/test_get_places_op.py +++ b/python/paddle/v2/fluid/tests/test_get_places_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid as fluid import decorators import unittest diff --git a/python/paddle/v2/fluid/tests/test_gru_op.py b/python/paddle/v2/fluid/tests/test_gru_op.py index a6647d1bf2..69cfd6c481 100644 --- a/python/paddle/v2/fluid/tests/test_gru_op.py +++ b/python/paddle/v2/fluid/tests/test_gru_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import math diff --git a/python/paddle/v2/fluid/tests/test_gru_unit_op.py b/python/paddle/v2/fluid/tests/test_gru_unit_op.py index 53f10c32c7..71f13c4513 100644 --- a/python/paddle/v2/fluid/tests/test_gru_unit_op.py +++ b/python/paddle/v2/fluid/tests/test_gru_unit_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_hinge_loss_op.py b/python/paddle/v2/fluid/tests/test_hinge_loss_op.py index dc7774d01c..71ff47316e 100644 --- a/python/paddle/v2/fluid/tests/test_hinge_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_hinge_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_huber_loss_op.py b/python/paddle/v2/fluid/tests/test_huber_loss_op.py index 18a48bb18c..e4560af778 100644 --- a/python/paddle/v2/fluid/tests/test_huber_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_huber_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_image_classification_layer.py b/python/paddle/v2/fluid/tests/test_image_classification_layer.py index 9d676e8759..c64cfed5f5 100644 --- a/python/paddle/v2/fluid/tests/test_image_classification_layer.py +++ b/python/paddle/v2/fluid/tests/test_image_classification_layer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/test_infer_shape.py b/python/paddle/v2/fluid/tests/test_infer_shape.py index 0c2a6f1423..521096388a 100644 --- a/python/paddle/v2/fluid/tests/test_infer_shape.py +++ b/python/paddle/v2/fluid/tests/test_infer_shape.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_inference_model_io.py b/python/paddle/v2/fluid/tests/test_inference_model_io.py index c5cad2166b..adf428aa84 100644 --- a/python/paddle/v2/fluid/tests/test_inference_model_io.py +++ b/python/paddle/v2/fluid/tests/test_inference_model_io.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_initializer.py b/python/paddle/v2/fluid/tests/test_initializer.py index fa3c2afeed..67746b4d7d 100644 --- a/python/paddle/v2/fluid/tests/test_initializer.py +++ b/python/paddle/v2/fluid/tests/test_initializer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import unittest diff --git a/python/paddle/v2/fluid/tests/test_is_empty_op.py b/python/paddle/v2/fluid/tests/test_is_empty_op.py index d6876a885f..7c17e3d57a 100644 --- a/python/paddle/v2/fluid/tests/test_is_empty_op.py +++ b/python/paddle/v2/fluid/tests/test_is_empty_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from paddle.v2.fluid.op import Operator diff --git a/python/paddle/v2/fluid/tests/test_l1_norm_op.py b/python/paddle/v2/fluid/tests/test_l1_norm_op.py index 92484c49f0..bbc2087846 100644 --- a/python/paddle/v2/fluid/tests/test_l1_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_l1_norm_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import unittest from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_layers.py b/python/paddle/v2/fluid/tests/test_layers.py index b366e5ba36..709abd6c6a 100644 --- a/python/paddle/v2/fluid/tests/test_layers.py +++ b/python/paddle/v2/fluid/tests/test_layers.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import unittest diff --git a/python/paddle/v2/fluid/tests/test_linear_chain_crf_op.py b/python/paddle/v2/fluid/tests/test_linear_chain_crf_op.py index cd917dff7f..cbfd9d5e5b 100644 --- a/python/paddle/v2/fluid/tests/test_linear_chain_crf_op.py +++ b/python/paddle/v2/fluid/tests/test_linear_chain_crf_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import random import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_lod_array_length_op.py b/python/paddle/v2/fluid/tests/test_lod_array_length_op.py index f80136cb0d..eff28368f1 100644 --- a/python/paddle/v2/fluid/tests/test_lod_array_length_op.py +++ b/python/paddle/v2/fluid/tests/test_lod_array_length_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.layers as layers from paddle.v2.fluid.executor import Executor diff --git a/python/paddle/v2/fluid/tests/test_lod_rank_table.py b/python/paddle/v2/fluid/tests/test_lod_rank_table.py index 673605d79c..eb0392e8bf 100644 --- a/python/paddle/v2/fluid/tests/test_lod_rank_table.py +++ b/python/paddle/v2/fluid/tests/test_lod_rank_table.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.v2.fluid.layers import lod_rank_table, data from paddle.v2.fluid.executor import Executor import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_lod_reset_op.py b/python/paddle/v2/fluid/tests/test_lod_reset_op.py index d799dbfa21..4ee360403e 100644 --- a/python/paddle/v2/fluid/tests/test_lod_reset_op.py +++ b/python/paddle/v2/fluid/tests/test_lod_reset_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_lod_tensor_array.py b/python/paddle/v2/fluid/tests/test_lod_tensor_array.py index c593b1e061..0f3ac3c03d 100644 --- a/python/paddle/v2/fluid/tests/test_lod_tensor_array.py +++ b/python/paddle/v2/fluid/tests/test_lod_tensor_array.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core import numpy diff --git a/python/paddle/v2/fluid/tests/test_lod_tensor_array_ops.py b/python/paddle/v2/fluid/tests/test_lod_tensor_array_ops.py index 5887f9799a..c2d04db99b 100644 --- a/python/paddle/v2/fluid/tests/test_lod_tensor_array_ops.py +++ b/python/paddle/v2/fluid/tests/test_lod_tensor_array_ops.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core import numpy diff --git a/python/paddle/v2/fluid/tests/test_log_loss_op.py b/python/paddle/v2/fluid/tests/test_log_loss_op.py index fde99bfaa1..338355d0c4 100644 --- a/python/paddle/v2/fluid/tests/test_log_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_log_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_logical_op.py b/python/paddle/v2/fluid/tests/test_logical_op.py index 8c9e8de739..dd67dc561b 100644 --- a/python/paddle/v2/fluid/tests/test_logical_op.py +++ b/python/paddle/v2/fluid/tests/test_logical_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import op_test import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_lookup_table_op.py b/python/paddle/v2/fluid/tests/test_lookup_table_op.py index 1ff6b305bc..d5255ba31f 100644 --- a/python/paddle/v2/fluid/tests/test_lookup_table_op.py +++ b/python/paddle/v2/fluid/tests/test_lookup_table_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_lrn_op.py b/python/paddle/v2/fluid/tests/test_lrn_op.py index 051704617e..a841dcf79f 100644 --- a/python/paddle/v2/fluid/tests/test_lrn_op.py +++ b/python/paddle/v2/fluid/tests/test_lrn_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_lstm_op.py b/python/paddle/v2/fluid/tests/test_lstm_op.py index 76ea8def7c..d9fa01e247 100644 --- a/python/paddle/v2/fluid/tests/test_lstm_op.py +++ b/python/paddle/v2/fluid/tests/test_lstm_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_lstm_unit_op.py b/python/paddle/v2/fluid/tests/test_lstm_unit_op.py index c97c1e72aa..d6348ea0ec 100644 --- a/python/paddle/v2/fluid/tests/test_lstm_unit_op.py +++ b/python/paddle/v2/fluid/tests/test_lstm_unit_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_margin_rank_loss_op.py b/python/paddle/v2/fluid/tests/test_margin_rank_loss_op.py index 3d8c1d19f9..694ce20712 100644 --- a/python/paddle/v2/fluid/tests/test_margin_rank_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_margin_rank_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_matmul_op.py b/python/paddle/v2/fluid/tests/test_matmul_op.py index 2dba25e17d..5138af38f4 100644 --- a/python/paddle/v2/fluid/tests/test_matmul_op.py +++ b/python/paddle/v2/fluid/tests/test_matmul_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_maxout_op.py b/python/paddle/v2/fluid/tests/test_maxout_op.py index ed8c0d2b67..5cd7fbde84 100644 --- a/python/paddle/v2/fluid/tests/test_maxout_op.py +++ b/python/paddle/v2/fluid/tests/test_maxout_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_mean_op.py b/python/paddle/v2/fluid/tests/test_mean_op.py index f9d7d6921e..81e8421635 100644 --- a/python/paddle/v2/fluid/tests/test_mean_op.py +++ b/python/paddle/v2/fluid/tests/test_mean_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_memory_optimization_transpiler.py b/python/paddle/v2/fluid/tests/test_memory_optimization_transpiler.py index 76f3c4eb64..2e9ed78ffd 100644 --- a/python/paddle/v2/fluid/tests/test_memory_optimization_transpiler.py +++ b/python/paddle/v2/fluid/tests/test_memory_optimization_transpiler.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import unittest diff --git a/python/paddle/v2/fluid/tests/test_minus_op.py b/python/paddle/v2/fluid/tests/test_minus_op.py index 99c0d9056a..aee909f56c 100644 --- a/python/paddle/v2/fluid/tests/test_minus_op.py +++ b/python/paddle/v2/fluid/tests/test_minus_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_mnist_if_else_op.py b/python/paddle/v2/fluid/tests/test_mnist_if_else_op.py index 18e3991b94..3288a0f007 100644 --- a/python/paddle/v2/fluid/tests/test_mnist_if_else_op.py +++ b/python/paddle/v2/fluid/tests/test_mnist_if_else_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.layers as layers from paddle.v2.fluid.framework import Program, program_guard, default_main_program, default_startup_program from paddle.v2.fluid.executor import Executor diff --git a/python/paddle/v2/fluid/tests/test_modified_huber_loss_op.py b/python/paddle/v2/fluid/tests/test_modified_huber_loss_op.py index 40955283e6..eb3873b9ea 100644 --- a/python/paddle/v2/fluid/tests/test_modified_huber_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_modified_huber_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_momentum_op.py b/python/paddle/v2/fluid/tests/test_momentum_op.py index 8008a5586f..048eaae06b 100644 --- a/python/paddle/v2/fluid/tests/test_momentum_op.py +++ b/python/paddle/v2/fluid/tests/test_momentum_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_mul_op.py b/python/paddle/v2/fluid/tests/test_mul_op.py index 3033b8ef70..83715f0e27 100644 --- a/python/paddle/v2/fluid/tests/test_mul_op.py +++ b/python/paddle/v2/fluid/tests/test_mul_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_multiplex_op.py b/python/paddle/v2/fluid/tests/test_multiplex_op.py index 5746ab391e..a06aef94a5 100644 --- a/python/paddle/v2/fluid/tests/test_multiplex_op.py +++ b/python/paddle/v2/fluid/tests/test_multiplex_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_nce.py b/python/paddle/v2/fluid/tests/test_nce.py index ce66a7c6b3..3ae727a573 100644 --- a/python/paddle/v2/fluid/tests/test_nce.py +++ b/python/paddle/v2/fluid/tests/test_nce.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_net.py b/python/paddle/v2/fluid/tests/test_net.py index cc78cb4a56..69d95d4f70 100644 --- a/python/paddle/v2/fluid/tests/test_net.py +++ b/python/paddle/v2/fluid/tests/test_net.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core from paddle.v2.fluid.op import Operator import unittest diff --git a/python/paddle/v2/fluid/tests/test_norm_op.py b/python/paddle/v2/fluid/tests/test_norm_op.py index b053522d72..dd1cd5a31c 100644 --- a/python/paddle/v2/fluid/tests/test_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_norm_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_normalization_wrapper.py b/python/paddle/v2/fluid/tests/test_normalization_wrapper.py index caff63011d..57f14f6b9c 100644 --- a/python/paddle/v2/fluid/tests/test_normalization_wrapper.py +++ b/python/paddle/v2/fluid/tests/test_normalization_wrapper.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid as fluid import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_op_support_gpu.py b/python/paddle/v2/fluid/tests/test_op_support_gpu.py index 741686a874..3493981812 100644 --- a/python/paddle/v2/fluid/tests/test_op_support_gpu.py +++ b/python/paddle/v2/fluid/tests/test_op_support_gpu.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_operator.py b/python/paddle/v2/fluid/tests/test_operator.py index e75ee41149..b82cf580e8 100644 --- a/python/paddle/v2/fluid/tests/test_operator.py +++ b/python/paddle/v2/fluid/tests/test_operator.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.op as op diff --git a/python/paddle/v2/fluid/tests/test_operator_desc.py b/python/paddle/v2/fluid/tests/test_operator_desc.py index ed18fafe33..2c8665ffa2 100644 --- a/python/paddle/v2/fluid/tests/test_operator_desc.py +++ b/python/paddle/v2/fluid/tests/test_operator_desc.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_optimizer.py b/python/paddle/v2/fluid/tests/test_optimizer.py index dbec3a5944..480ee70915 100644 --- a/python/paddle/v2/fluid/tests/test_optimizer.py +++ b/python/paddle/v2/fluid/tests/test_optimizer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.framework as framework diff --git a/python/paddle/v2/fluid/tests/test_pad_op.py b/python/paddle/v2/fluid/tests/test_pad_op.py index 1036b6bcad..0bd4800055 100644 --- a/python/paddle/v2/fluid/tests/test_pad_op.py +++ b/python/paddle/v2/fluid/tests/test_pad_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_parallel_op.py b/python/paddle/v2/fluid/tests/test_parallel_op.py index d36f7d07ac..dfde492c7c 100644 --- a/python/paddle/v2/fluid/tests/test_parallel_op.py +++ b/python/paddle/v2/fluid/tests/test_parallel_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/test_parameter.py b/python/paddle/v2/fluid/tests/test_parameter.py index e0db318345..dfecdf939b 100644 --- a/python/paddle/v2/fluid/tests/test_parameter.py +++ b/python/paddle/v2/fluid/tests/test_parameter.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest from paddle.v2.fluid.framework import default_main_program import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_pool2d_op.py b/python/paddle/v2/fluid/tests/test_pool2d_op.py index ac8b24e7ad..2f43be8a0f 100644 --- a/python/paddle/v2/fluid/tests/test_pool2d_op.py +++ b/python/paddle/v2/fluid/tests/test_pool2d_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_pool3d_op.py b/python/paddle/v2/fluid/tests/test_pool3d_op.py index 54b8df8465..c93711e051 100644 --- a/python/paddle/v2/fluid/tests/test_pool3d_op.py +++ b/python/paddle/v2/fluid/tests/test_pool3d_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_pool_max_op.py b/python/paddle/v2/fluid/tests/test_pool_max_op.py index c4ec0e50cc..330ad24bd4 100644 --- a/python/paddle/v2/fluid/tests/test_pool_max_op.py +++ b/python/paddle/v2/fluid/tests/test_pool_max_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_positive_negative_pair_op.py b/python/paddle/v2/fluid/tests/test_positive_negative_pair_op.py index b75f7152ef..9b5e544655 100644 --- a/python/paddle/v2/fluid/tests/test_positive_negative_pair_op.py +++ b/python/paddle/v2/fluid/tests/test_positive_negative_pair_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import itertools import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_precision_recall_op.py b/python/paddle/v2/fluid/tests/test_precision_recall_op.py index 87c7fcb4b5..188b7af559 100644 --- a/python/paddle/v2/fluid/tests/test_precision_recall_op.py +++ b/python/paddle/v2/fluid/tests/test_precision_recall_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_prelu_op.py b/python/paddle/v2/fluid/tests/test_prelu_op.py index 38bd260bc9..848036234c 100644 --- a/python/paddle/v2/fluid/tests/test_prelu_op.py +++ b/python/paddle/v2/fluid/tests/test_prelu_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_print_op.py b/python/paddle/v2/fluid/tests/test_print_op.py index 4e42863af4..3177700dfa 100644 --- a/python/paddle/v2/fluid/tests/test_print_op.py +++ b/python/paddle/v2/fluid/tests/test_print_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core from paddle.v2.fluid.executor import Executor diff --git a/python/paddle/v2/fluid/tests/test_profiler.py b/python/paddle/v2/fluid/tests/test_profiler.py index 4b439a16aa..abf8881b67 100644 --- a/python/paddle/v2/fluid/tests/test_profiler.py +++ b/python/paddle/v2/fluid/tests/test_profiler.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/test_program.py b/python/paddle/v2/fluid/tests/test_program.py index bcaeede93e..9967da1593 100644 --- a/python/paddle/v2/fluid/tests/test_program.py +++ b/python/paddle/v2/fluid/tests/test_program.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import unittest diff --git a/python/paddle/v2/fluid/tests/test_protobuf.py b/python/paddle/v2/fluid/tests/test_protobuf.py index 5f0646d036..48e6dedc58 100644 --- a/python/paddle/v2/fluid/tests/test_protobuf.py +++ b/python/paddle/v2/fluid/tests/test_protobuf.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.proto.framework_pb2 as framework_pb2 import unittest diff --git a/python/paddle/v2/fluid/tests/test_protobuf_descs.py b/python/paddle/v2/fluid/tests/test_protobuf_descs.py index 24638dc0e8..9034b2f4ef 100644 --- a/python/paddle/v2/fluid/tests/test_protobuf_descs.py +++ b/python/paddle/v2/fluid/tests/test_protobuf_descs.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_proximal_adagrad_op.py b/python/paddle/v2/fluid/tests/test_proximal_adagrad_op.py index c197d850f9..744d71bdcf 100644 --- a/python/paddle/v2/fluid/tests/test_proximal_adagrad_op.py +++ b/python/paddle/v2/fluid/tests/test_proximal_adagrad_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_proximal_gd_op.py b/python/paddle/v2/fluid/tests/test_proximal_gd_op.py index 1545255825..96540cf6cf 100644 --- a/python/paddle/v2/fluid/tests/test_proximal_gd_op.py +++ b/python/paddle/v2/fluid/tests/test_proximal_gd_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_rank_loss_op.py b/python/paddle/v2/fluid/tests/test_rank_loss_op.py index b4ba7920cd..f31a2c2681 100644 --- a/python/paddle/v2/fluid/tests/test_rank_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_rank_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_recurrent_op.py b/python/paddle/v2/fluid/tests/test_recurrent_op.py index bcc3457aa3..6d59e199e2 100644 --- a/python/paddle/v2/fluid/tests/test_recurrent_op.py +++ b/python/paddle/v2/fluid/tests/test_recurrent_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.layers as layers diff --git a/python/paddle/v2/fluid/tests/test_reduce_op.py b/python/paddle/v2/fluid/tests/test_reduce_op.py index 57ee307ba6..1a4af39fb9 100644 --- a/python/paddle/v2/fluid/tests/test_reduce_op.py +++ b/python/paddle/v2/fluid/tests/test_reduce_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_registry.py b/python/paddle/v2/fluid/tests/test_registry.py index dba1189630..6435e7e243 100644 --- a/python/paddle/v2/fluid/tests/test_registry.py +++ b/python/paddle/v2/fluid/tests/test_registry.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import warnings diff --git a/python/paddle/v2/fluid/tests/test_regularizer.py b/python/paddle/v2/fluid/tests/test_regularizer.py index 9eaae1904a..b33817fa41 100644 --- a/python/paddle/v2/fluid/tests/test_regularizer.py +++ b/python/paddle/v2/fluid/tests/test_regularizer.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.framework as framework diff --git a/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py b/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py index 0bcdfafcf4..74cd6de9e6 100644 --- a/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py +++ b/python/paddle/v2/fluid/tests/test_reorder_lod_tensor.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid as fluid import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_reshape_op.py b/python/paddle/v2/fluid/tests/test_reshape_op.py index d6e6797043..2cc0b36460 100644 --- a/python/paddle/v2/fluid/tests/test_reshape_op.py +++ b/python/paddle/v2/fluid/tests/test_reshape_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_rmsprop_op.py b/python/paddle/v2/fluid/tests/test_rmsprop_op.py index 27a1ea2137..b6d7c69800 100644 --- a/python/paddle/v2/fluid/tests/test_rmsprop_op.py +++ b/python/paddle/v2/fluid/tests/test_rmsprop_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_rnn_memory_helper_op.py b/python/paddle/v2/fluid/tests/test_rnn_memory_helper_op.py index 378d7f8523..82b54bbd1a 100644 --- a/python/paddle/v2/fluid/tests/test_rnn_memory_helper_op.py +++ b/python/paddle/v2/fluid/tests/test_rnn_memory_helper_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest from paddle.v2.fluid.framework import Program diff --git a/python/paddle/v2/fluid/tests/test_roi_pool_op.py b/python/paddle/v2/fluid/tests/test_roi_pool_op.py index 6d7a698b09..af48848dcd 100644 --- a/python/paddle/v2/fluid/tests/test_roi_pool_op.py +++ b/python/paddle/v2/fluid/tests/test_roi_pool_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import math diff --git a/python/paddle/v2/fluid/tests/test_row_conv_op.py b/python/paddle/v2/fluid/tests/test_row_conv_op.py index 1234d289cb..580b08f75e 100644 --- a/python/paddle/v2/fluid/tests/test_row_conv_op.py +++ b/python/paddle/v2/fluid/tests/test_row_conv_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_scale_op.py b/python/paddle/v2/fluid/tests/test_scale_op.py index 9847d3d361..95cd935dda 100644 --- a/python/paddle/v2/fluid/tests/test_scale_op.py +++ b/python/paddle/v2/fluid/tests/test_scale_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_scatter_op.py b/python/paddle/v2/fluid/tests/test_scatter_op.py index b6c4162f6f..f2936e19ae 100644 --- a/python/paddle/v2/fluid/tests/test_scatter_op.py +++ b/python/paddle/v2/fluid/tests/test_scatter_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_scope.py b/python/paddle/v2/fluid/tests/test_scope.py index adaaf16906..566a11abbe 100644 --- a/python/paddle/v2/fluid/tests/test_scope.py +++ b/python/paddle/v2/fluid/tests/test_scope.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core import unittest diff --git a/python/paddle/v2/fluid/tests/test_selected_rows.py b/python/paddle/v2/fluid/tests/test_selected_rows.py index 3179a3caae..65ddf1f8f5 100644 --- a/python/paddle/v2/fluid/tests/test_selected_rows.py +++ b/python/paddle/v2/fluid/tests/test_selected_rows.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_seq_concat_op.py b/python/paddle/v2/fluid/tests/test_seq_concat_op.py index 1f026fd76e..ba2bb075e6 100644 --- a/python/paddle/v2/fluid/tests/test_seq_concat_op.py +++ b/python/paddle/v2/fluid/tests/test_seq_concat_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import sys diff --git a/python/paddle/v2/fluid/tests/test_seq_conv.py b/python/paddle/v2/fluid/tests/test_seq_conv.py index c7e5085194..674a2e1694 100644 --- a/python/paddle/v2/fluid/tests/test_seq_conv.py +++ b/python/paddle/v2/fluid/tests/test_seq_conv.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import random diff --git a/python/paddle/v2/fluid/tests/test_seq_pool.py b/python/paddle/v2/fluid/tests/test_seq_pool.py index bb15495373..9dd6b2a087 100644 --- a/python/paddle/v2/fluid/tests/test_seq_pool.py +++ b/python/paddle/v2/fluid/tests/test_seq_pool.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_sequence_erase_op.py b/python/paddle/v2/fluid/tests/test_sequence_erase_op.py index 4cc2613cf9..4823836ba9 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_erase_op.py +++ b/python/paddle/v2/fluid/tests/test_sequence_erase_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_sequence_expand.py b/python/paddle/v2/fluid/tests/test_sequence_expand.py index aacdabf295..6fc045125f 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_expand.py +++ b/python/paddle/v2/fluid/tests/test_sequence_expand.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_sequence_reshape.py b/python/paddle/v2/fluid/tests/test_sequence_reshape.py index 857b15237a..06d5af8f5e 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_reshape.py +++ b/python/paddle/v2/fluid/tests/test_sequence_reshape.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import math diff --git a/python/paddle/v2/fluid/tests/test_sequence_slice_op.py b/python/paddle/v2/fluid/tests/test_sequence_slice_op.py index 94062431f0..bf1f21bcde 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_slice_op.py +++ b/python/paddle/v2/fluid/tests/test_sequence_slice_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import sys diff --git a/python/paddle/v2/fluid/tests/test_sequence_softmax_op.py b/python/paddle/v2/fluid/tests/test_sequence_softmax_op.py index 8170e4d7f1..5bd780f6b5 100644 --- a/python/paddle/v2/fluid/tests/test_sequence_softmax_op.py +++ b/python/paddle/v2/fluid/tests/test_sequence_softmax_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_sgd_op.py b/python/paddle/v2/fluid/tests/test_sgd_op.py index 4a71fb30a9..f87927968b 100644 --- a/python/paddle/v2/fluid/tests/test_sgd_op.py +++ b/python/paddle/v2/fluid/tests/test_sgd_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_shrink_rnn_memory.py b/python/paddle/v2/fluid/tests/test_shrink_rnn_memory.py index 1825a5258f..4578211bac 100644 --- a/python/paddle/v2/fluid/tests/test_shrink_rnn_memory.py +++ b/python/paddle/v2/fluid/tests/test_shrink_rnn_memory.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core from paddle.v2.fluid.executor import Executor diff --git a/python/paddle/v2/fluid/tests/test_sigmoid_cross_entropy_with_logits_op.py b/python/paddle/v2/fluid/tests/test_sigmoid_cross_entropy_with_logits_op.py index 132502c9cb..f88fa62119 100644 --- a/python/paddle/v2/fluid/tests/test_sigmoid_cross_entropy_with_logits_op.py +++ b/python/paddle/v2/fluid/tests/test_sigmoid_cross_entropy_with_logits_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np from op_test import OpTest from scipy.special import logit diff --git a/python/paddle/v2/fluid/tests/test_sign_op.py b/python/paddle/v2/fluid/tests/test_sign_op.py index f649cb9e7c..c1dfa7f45d 100644 --- a/python/paddle/v2/fluid/tests/test_sign_op.py +++ b/python/paddle/v2/fluid/tests/test_sign_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_smooth_l1_loss_op.py b/python/paddle/v2/fluid/tests/test_smooth_l1_loss_op.py index 1052eaa8b0..5a388bb7b3 100644 --- a/python/paddle/v2/fluid/tests/test_smooth_l1_loss_op.py +++ b/python/paddle/v2/fluid/tests/test_smooth_l1_loss_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_softmax_op.py b/python/paddle/v2/fluid/tests/test_softmax_op.py index d03e50b2f1..cf43e676c5 100644 --- a/python/paddle/v2/fluid/tests/test_softmax_op.py +++ b/python/paddle/v2/fluid/tests/test_softmax_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_softmax_with_cross_entropy_op.py b/python/paddle/v2/fluid/tests/test_softmax_with_cross_entropy_op.py index 330467081b..626f34f0e0 100644 --- a/python/paddle/v2/fluid/tests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/v2/fluid/tests/test_softmax_with_cross_entropy_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_split_and_merge_lod_tensor_op.py b/python/paddle/v2/fluid/tests/test_split_and_merge_lod_tensor_op.py index 4e90404eca..bc541298ed 100644 --- a/python/paddle/v2/fluid/tests/test_split_and_merge_lod_tensor_op.py +++ b/python/paddle/v2/fluid/tests/test_split_and_merge_lod_tensor_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_split_op.py b/python/paddle/v2/fluid/tests/test_split_op.py index 000c300446..b80b64c41b 100644 --- a/python/paddle/v2/fluid/tests/test_split_op.py +++ b/python/paddle/v2/fluid/tests/test_split_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py b/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py index a6cc4f6c6d..fb44e10ebf 100644 --- a/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py +++ b/python/paddle/v2/fluid/tests/test_split_selected_rows_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.core as core import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_spp_op.py b/python/paddle/v2/fluid/tests/test_spp_op.py index f09bb94b44..e912b56de5 100644 --- a/python/paddle/v2/fluid/tests/test_spp_op.py +++ b/python/paddle/v2/fluid/tests/test_spp_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_squared_l2_distance_op.py b/python/paddle/v2/fluid/tests/test_squared_l2_distance_op.py index 7b80d81d72..8171207cd9 100644 --- a/python/paddle/v2/fluid/tests/test_squared_l2_distance_op.py +++ b/python/paddle/v2/fluid/tests/test_squared_l2_distance_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_squared_l2_norm_op.py b/python/paddle/v2/fluid/tests/test_squared_l2_norm_op.py index 80994f5937..b7575cb4d2 100644 --- a/python/paddle/v2/fluid/tests/test_squared_l2_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_squared_l2_norm_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import unittest from numpy import linalg as LA diff --git a/python/paddle/v2/fluid/tests/test_sum_op.py b/python/paddle/v2/fluid/tests/test_sum_op.py index 366708ac83..0a15a9485d 100644 --- a/python/paddle/v2/fluid/tests/test_sum_op.py +++ b/python/paddle/v2/fluid/tests/test_sum_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_tensor.py b/python/paddle/v2/fluid/tests/test_tensor.py index 62a48b206c..d5cc235f58 100644 --- a/python/paddle/v2/fluid/tests/test_tensor.py +++ b/python/paddle/v2/fluid/tests/test_tensor.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2.fluid.core as core import unittest import numpy diff --git a/python/paddle/v2/fluid/tests/test_top_k_op.py b/python/paddle/v2/fluid/tests/test_top_k_op.py index 86968dba14..a50faf0fff 100644 --- a/python/paddle/v2/fluid/tests/test_top_k_op.py +++ b/python/paddle/v2/fluid/tests/test_top_k_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_transpose_op.py b/python/paddle/v2/fluid/tests/test_transpose_op.py index ff2541f450..a16de1416f 100644 --- a/python/paddle/v2/fluid/tests/test_transpose_op.py +++ b/python/paddle/v2/fluid/tests/test_transpose_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_uniform_random_op.py b/python/paddle/v2/fluid/tests/test_uniform_random_op.py index 332ac4f07f..b2a39f975e 100644 --- a/python/paddle/v2/fluid/tests/test_uniform_random_op.py +++ b/python/paddle/v2/fluid/tests/test_uniform_random_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy diff --git a/python/paddle/v2/fluid/tests/test_unpool_op.py b/python/paddle/v2/fluid/tests/test_unpool_op.py index 988c0c7506..3dd43f9ba4 100644 --- a/python/paddle/v2/fluid/tests/test_unpool_op.py +++ b/python/paddle/v2/fluid/tests/test_unpool_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/v2/fluid/tests/test_variable.py b/python/paddle/v2/fluid/tests/test_variable.py index 199fd4a8c2..9f9748ca4e 100644 --- a/python/paddle/v2/fluid/tests/test_variable.py +++ b/python/paddle/v2/fluid/tests/test_variable.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest from paddle.v2.fluid.framework import default_main_program, Program, convert_np_dtype_to_dtype_ import paddle.v2.fluid.core as core diff --git a/python/paddle/v2/fluid/tests/test_warpctc_op.py b/python/paddle/v2/fluid/tests/test_warpctc_op.py index 9f565676c5..55d1c73262 100644 --- a/python/paddle/v2/fluid/tests/test_warpctc_op.py +++ b/python/paddle/v2/fluid/tests/test_warpctc_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys import unittest import numpy as np diff --git a/python/paddle/v2/fluid/tests/test_while_op.py b/python/paddle/v2/fluid/tests/test_while_op.py index 72de0a0361..9f5e1b668c 100644 --- a/python/paddle/v2/fluid/tests/test_while_op.py +++ b/python/paddle/v2/fluid/tests/test_while_op.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import paddle.v2.fluid.layers as layers from paddle.v2.fluid.executor import Executor diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py index a6fa0cecb8..1429d6b1e0 100644 --- a/python/paddle/v2/image.py +++ b/python/paddle/v2/image.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ This file contains some common interfaces for image preprocess. Many users are confused about the image layout. We introduce diff --git a/python/paddle/v2/inference.py b/python/paddle/v2/inference.py index 39d1bfff0c..78bf9807da 100644 --- a/python/paddle/v2/inference.py +++ b/python/paddle/v2/inference.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy import collections import topology diff --git a/python/paddle/v2/master/__init__.py b/python/paddle/v2/master/__init__.py index 09daaaa75e..494e4baf20 100644 --- a/python/paddle/v2/master/__init__.py +++ b/python/paddle/v2/master/__init__.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from client import * __all__ = ['client'] diff --git a/python/paddle/v2/master/client.py b/python/paddle/v2/master/client.py index b874c2f349..b3c790e39d 100644 --- a/python/paddle/v2/master/client.py +++ b/python/paddle/v2/master/client.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import ctypes import os diff --git a/python/paddle/v2/reader/tests/__init__.py b/python/paddle/v2/reader/tests/__init__.py index 2619c1c0e9..b94a21a7e4 100644 --- a/python/paddle/v2/reader/tests/__init__.py +++ b/python/paddle/v2/reader/tests/__init__.py @@ -1,13 +1,13 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/paddle/v2/reader/tests/creator_test.py b/python/paddle/v2/reader/tests/creator_test.py index cf190aa664..ac6cd4e9b6 100644 --- a/python/paddle/v2/reader/tests/creator_test.py +++ b/python/paddle/v2/reader/tests/creator_test.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Copyright PaddlePaddle contributors. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/paddle/v2/reader/tests/decorator_test.py b/python/paddle/v2/reader/tests/decorator_test.py index 4ba71969df..e41e9c78a0 100644 --- a/python/paddle/v2/reader/tests/decorator_test.py +++ b/python/paddle/v2/reader/tests/decorator_test.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import time import unittest diff --git a/python/paddle/v2/tests/test_image.py b/python/paddle/v2/tests/test_image.py index b2d773510d..2b0444bb03 100644 --- a/python/paddle/v2/tests/test_image.py +++ b/python/paddle/v2/tests/test_image.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import unittest import numpy as np diff --git a/python/paddle/v2/tests/test_layer.py b/python/paddle/v2/tests/test_layer.py index de932ad715..710e8135f2 100644 --- a/python/paddle/v2/tests/test_layer.py +++ b/python/paddle/v2/tests/test_layer.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import unittest import paddle.v2.activation as activation diff --git a/python/paddle/v2/tests/test_op.py b/python/paddle/v2/tests/test_op.py index 69acccddf4..dd04cc4ab6 100644 --- a/python/paddle/v2/tests/test_op.py +++ b/python/paddle/v2/tests/test_op.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import unittest import paddle.v2.data_type as data_type diff --git a/python/paddle/v2/tests/test_paramconf_order.py b/python/paddle/v2/tests/test_paramconf_order.py index 41fea64122..33c240b8f5 100644 --- a/python/paddle/v2/tests/test_paramconf_order.py +++ b/python/paddle/v2/tests/test_paramconf_order.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Copyright PaddlePaddle contributors. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/paddle/v2/tests/test_parameters.py b/python/paddle/v2/tests/test_parameters.py index ab6863620f..1fe1f09b9d 100644 --- a/python/paddle/v2/tests/test_parameters.py +++ b/python/paddle/v2/tests/test_parameters.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import sys diff --git a/python/paddle/v2/tests/test_rnn_layer.py b/python/paddle/v2/tests/test_rnn_layer.py index 192b0ee678..7920e342e1 100644 --- a/python/paddle/v2/tests/test_rnn_layer.py +++ b/python/paddle/v2/tests/test_rnn_layer.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import difflib import unittest diff --git a/python/paddle/v2/tests/test_topology.py b/python/paddle/v2/tests/test_topology.py index 7fd2ee82fd..11b4154eed 100644 --- a/python/paddle/v2/tests/test_topology.py +++ b/python/paddle/v2/tests/test_topology.py @@ -1,4 +1,4 @@ -# Copyright PaddlePaddle contributors. All Rights Reserved +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import unittest import paddle.v2.layer as layer import paddle.v2.topology as topology diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 1a70a7203b..a0060bf227 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -1,16 +1,16 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Module Trainer """ diff --git a/tools/manylinux1/build_scripts/manylinux1-check.py b/tools/manylinux1/build_scripts/manylinux1-check.py index e4bde065a2..a27eab1c77 100644 --- a/tools/manylinux1/build_scripts/manylinux1-check.py +++ b/tools/manylinux1/build_scripts/manylinux1-check.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Logic copied from PEP 513 diff --git a/tools/manylinux1/build_scripts/python-tag-abi-tag.py b/tools/manylinux1/build_scripts/python-tag-abi-tag.py index 301fbf07a4..cd2573314c 100644 --- a/tools/manylinux1/build_scripts/python-tag-abi-tag.py +++ b/tools/manylinux1/build_scripts/python-tag-abi-tag.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Utility script to print the python tag + the abi tag for a Python # See PEP 425 for exactly what these are, but an example would be: # cp27-cp27mu diff --git a/tools/manylinux1/build_scripts/ssl-check.py b/tools/manylinux1/build_scripts/ssl-check.py index 900185cef1..34a3116207 100644 --- a/tools/manylinux1/build_scripts/ssl-check.py +++ b/tools/manylinux1/build_scripts/ssl-check.py @@ -1,16 +1,17 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # cf. https://github.com/pypa/manylinux/issues/53 GOOD_SSL = "https://google.com" From eee62648cf075e6520405d6ae18b039a357d40e8 Mon Sep 17 00:00:00 2001 From: ranqiu92 <31875089+ranqiu92@users.noreply.github.com> Date: Sun, 21 Jan 2018 22:30:44 +0800 Subject: [PATCH 143/158] Refine test_label_semantic_roles of Fluid (#7193) --- python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py index be92c48ac5..1a342bf1fb 100644 --- a/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/v2/fluid/tests/book/test_label_semantic_roles.py @@ -34,7 +34,7 @@ mix_hidden_lr = 1e-3 IS_SPARSE = True PASS_NUM = 10 -BATCH_SIZE = 20 +BATCH_SIZE = 10 embedding_name = 'emb' From 2dc5c69ecc0d83e20101d2767a94f48e19a07ce0 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 21 Jan 2018 06:42:41 -0800 Subject: [PATCH 144/158] Add Fluid Compiler design doc (#7178) * Add fluid_compiler.md * Paragraphing --- doc/design/fluid.md | 10 +--- doc/design/fluid_compiler.md | 110 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 doc/design/fluid_compiler.md diff --git a/doc/design/fluid.md b/doc/design/fluid.md index 585dc8ef39..2acc168007 100644 --- a/doc/design/fluid.md +++ b/doc/design/fluid.md @@ -105,18 +105,10 @@ There are two ways to execute a Fluid program. When a program is executed, it c There is a C++ class [`Executor`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/executor.h), which runs a `ProgramDesc`, similar to how an interpreter runs a Python program. -Fluid is moving towards the direction of a compiler, which is explain in more detail later in this article. +Fluid is moving towards the direction of a compiler, which is explain in [fluid_compiler.md](fluid_compiler.md). ## Backward Compatibility of Fluid Given all the advantages from the removal of the concept of a *model*, hardware manufacturers might still prefer the existence of the concept of a model, so it would be easier for them to support multiple frameworks all at once and could run a trained model during inference. For example, Nervana, a startup company acquired by Intel, has been working on an XPU that reads the models in the format known as [n-graph](https://github.com/NervanaSystems/ngraph). Similarly, [Movidius](https://www.movidius.com/) is producing a mobile deep learning chip that reads and runs graphs of operators. The well-known [ONNX](https://github.com/onnx/onnx) is also a file format of graphs of operators. For Fluid, we can write a converter that extracts the parts in the `ProgramDesc` protobuf message, converts them into a graph of operators, and exports the graph into the ONNX or n-graph format. - -## Towards a Deep Learning Language and the Compiler - -We can change the `if-then-else` and loop structure a little bit in the above Fluid example programs, to make it into a new programming language, different than Python. - -Even if we do not invent a new language, as long as we get the `ProgramDesc` message filled in, we can write a transpiler, which translates each invocation to an operator, into a C++ call to a kernel function of that operator. For example, a transpiler that weaves the CUDA kernels outputs an NVIDIA-friendly C++ program, which can be built using `nvcc`. Another transpiler could generate MKL-friendly code that should be built using `icc` from Intel. More interestingly, we can translate a Fluid program into its distributed version of two `ProgramDesc` messages, one for running on the trainer process, and the other one for the parameter server. For more details of the last example, the [concurrent programming design](concurrent_programming.md) document would be a good pointer. The following figure explains the proposed two-stage process: - -![](fluid-compiler.png) diff --git a/doc/design/fluid_compiler.md b/doc/design/fluid_compiler.md new file mode 100644 index 0000000000..2a6beafc52 --- /dev/null +++ b/doc/design/fluid_compiler.md @@ -0,0 +1,110 @@ +# PaddlePaddle Fluid: Towards a Compiled Programming Language + +As described in [fluid.md](fluid.md), when a Fluid application program +runs, it generates a `ProgramDesc` protobuf message as an intermediate +representation of itself. The C++ class `Executor` can run this +protobuf message as an interpreter. This article describes the Fluid +compiler. + +![](fluid-compiler.png) + +## ProgramDesc + +Before we go deeper into the idea of compiled language, let us take a +look at a simple example Fluid application. + +```python +import "fluid" + +func paddlepaddle() { + X = fluid.read(...) + W = fluid.Tensor(...) + Y = fluid.mult(X, W) +} +``` + +This program consists of a [block](block.md) of three operators -- +`read`, `assign`, and `mult`. Its `ProgramDesc` message looks like +the following + +```protobuf +message ProgramDesc { + block[0] = Block { + vars = [X, W, Y], + ops = [ + read(output = X) + assign(input = ..., output = W) + mult(input = {X, W}, output = Y) + ], + } +} +``` + +## Transpilers + +We can write a transpiler program that takes a `ProgramDesc`, e.g., +the above one, and outputs another `ProgramDesc`. Let us take some +examples: + +1. *Memory optimization transpiler*: We can write a transpiler that + inserts some `FreeMemoryOp`s in the above example `ProgramDesc` so + to free memory early, before the end of an iteration, so to keep a + small memory footprint. + +1. *Distributed training transpiler*: We can write a transpiler that + converts a`ProgramDesc` into its distributed version of two + `ProgramDesc`s -- one for running by the trainer processes and the + other for the parameter server. + +In the rest of this article, we talk about a special kind of +transpiler, *Native code generator*, which takes a `ProgramDesc` and +generates a `.cu` (or `.cc`) file, which could be built by C++ +compilers (gcc, nvcc, icc) into binaries. + +## Native Code Generator + +For the above example, the native code generator transpiler, say, the +CUDA code generator, should generate a `main` function: + +```c++ +void main() { + auto X = fluid_cuda_read(...); + auto W = fluid_cuda_create_tensor(...); + auto Y = fluid_cuda_mult(X, W); +} +``` + +and the definitions of functions `fluid_cuda_read`, +`fluid_cuda_create_tensor`, and `fluid_cuda_mult`. Please be aware +that each function could just define a C++ instance of an operator and +run it. For example + +```c++ +paddle::Tensor fluid_cuda_read(...) { + paddle::Tensor t; + paddle::operator::Read r(&t, ...); + r.Run(); + return t; +} +``` + +For computational operators that have multiple *kernels*, each for a +specific hardware platform, for example, the `mult` operator, the +generated code should call its CUDA kernel: + +```c++ +paddle::Tensor fluid_cuda_mult(const paddle::Tensor& a, + const paddle::Tensor& b) { + paddle::Tensor t; + paddle::operator::Mult m(a, b, ...); + Mult.Run(cuda_context); +} +``` + +where `cuda_context` could be a global variable of type +`paddle::CUDADeviceContext`. + +## Multi-Block Code Generation + +Most Fluid application programs may have more than one blocks. To +execute them, we need to trace [scopes](scope.md). From cd775a130a318d7cd00178ae7cfad5127e33d4be Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Sun, 21 Jan 2018 06:44:38 -0800 Subject: [PATCH 145/158] Fixing typo in Send op (#7625) --- paddle/operators/send_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/send_op.cc b/paddle/operators/send_op.cc index a38d8b7a6f..807533a6c6 100644 --- a/paddle/operators/send_op.cc +++ b/paddle/operators/send_op.cc @@ -66,7 +66,7 @@ class SendOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(Tensor) Output tensor to get from server") .AsDuplicable(); AddComment(R"DOC( -Recv operator +Send operator This operator will send tensor to recv_op. )DOC"); From 02add30ce1744e9cfa66862acd66045c4a89aff4 Mon Sep 17 00:00:00 2001 From: ranqiu92 <31875089+ranqiu92@users.noreply.github.com> Date: Sun, 21 Jan 2018 22:48:07 +0800 Subject: [PATCH 146/158] Update faq (#7584) --- doc/faq/model/index_cn.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/faq/model/index_cn.rst b/doc/faq/model/index_cn.rst index b47bbe05bd..6947948bc7 100644 --- a/doc/faq/model/index_cn.rst +++ b/doc/faq/model/index_cn.rst @@ -67,3 +67,14 @@ * 不同于上述介绍的recurrent layer , :code:`paddle.networks.lstmemory_unit` 定义了LSTM单元在一个时间步内的计算过程,它并不是一个完整的recurrent layer,也不能接收序列数据作为输入; * :code:`paddle.networks.lstmemory_unit` 只能在recurrent_group中作为step function使用; + +5. PaddlePaddle的softmax能否指定计算的维度 +----------------------------------------- + +PaddlePaddle的softmax不能指定计算维度,只能按行计算。 +在图像任务中,对于NCHW,如果需要在C维度计算softmax,可以先使用 :code:`paddle.layer.switch_order` 改变维度顺序,即将NCHW转换成NHWC,再做一定的reshape,最后计算softmax。 + +6. PaddlePaddle是否支持维数可变的数据输入 +------------------------------------------ + +PaddlePaddle提供的 :code:`paddle.data_type.dense_array` 支持维数可变的数据输入。在使用时,将对应数据层的维数设置成一个大于输入数据维数的值用于占位即可。 From 85671b8acbefe538845896148425450498890b7d Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Sun, 21 Jan 2018 22:50:22 +0800 Subject: [PATCH 147/158] Data type transform (#7653) * init complete data layout transform * can compile * test passed * optimize code * fix while_grad_op first step loss lod problem * optimize in out ptr for transform * add check * update copyright * clean code * add NeedTransformLayout * add comment * change the interface of data_type_transform * init data_type_transform_test * complete data_type_transform_test * add TransDataType to data_transform --- paddle/framework/CMakeLists.txt | 1 + paddle/framework/data_device_transform.cc | 2 +- paddle/framework/data_device_transform.h | 2 +- paddle/framework/data_transform.cc | 11 +++- paddle/framework/data_type_transform.cc | 42 ++++++---------- paddle/framework/data_type_transform.h | 7 +-- paddle/framework/data_type_transform_test.cc | 53 ++++++++++++++++++++ 7 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 paddle/framework/data_type_transform_test.cc diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 8e5a956061..afb55bdaaa 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -37,6 +37,7 @@ nv_test(data_device_transform_test SRCS data_device_transform_test.cu DEPS operator op_registry init math_function) cc_library(data_type_transform SRCS data_type_transform.cc DEPS tensor) +cc_test(data_type_transform_test SRCS data_type_transform_test.cc DEPS data_type_transform) cc_library(data_layout_transform SRCS data_layout_transform.cc DEPS tensor math_function) cc_test(data_layout_transform_test SRCS data_layout_transform_test.cc DEPS data_layout_transform) diff --git a/paddle/framework/data_device_transform.cc b/paddle/framework/data_device_transform.cc index d38d87927f..5daf5a4e0a 100644 --- a/paddle/framework/data_device_transform.cc +++ b/paddle/framework/data_device_transform.cc @@ -31,7 +31,7 @@ static const platform::DeviceContext* GetDeviceContext( } } -void DeviceTransform(const Tensor& in, const platform::Place& dst_place, +void TransDataDevice(const Tensor& in, const platform::Place& dst_place, Tensor* out) { VLOG(3) << "DeviceTransform in, src_place " << in.place() << " dst_place: " << dst_place; diff --git a/paddle/framework/data_device_transform.h b/paddle/framework/data_device_transform.h index b21ed0be34..39750a85f2 100644 --- a/paddle/framework/data_device_transform.h +++ b/paddle/framework/data_device_transform.h @@ -21,7 +21,7 @@ limitations under the License. */ namespace paddle { namespace framework { -void DeviceTransform(const Tensor& in, const platform::Place& dst_place, +void TransDataDevice(const Tensor& in, const platform::Place& dst_place, Tensor* out); } // namespace framework diff --git a/paddle/framework/data_transform.cc b/paddle/framework/data_transform.cc index e28b2e015d..b6fd46401f 100644 --- a/paddle/framework/data_transform.cc +++ b/paddle/framework/data_transform.cc @@ -16,6 +16,7 @@ limitations under the License. */ #include "paddle/framework/data_device_transform.h" #include "paddle/framework/data_layout_transform.h" +#include "paddle/framework/data_type_transform.h" namespace paddle { namespace framework { @@ -41,15 +42,21 @@ void DataTransform(const OpKernelType& expected_kernel_type, PassTensorData(&out, &in); } + if (expected_kernel_type.data_type_ != kernel_type_for_var.data_type_) { + TransDataType(kernel_type_for_var, expected_kernel_type, in, &out); + transformed = true; + PassTensorData(&out, &in); + } + // do device transform if (!platform::is_same_place(kernel_type_for_var.place_, expected_kernel_type.place_)) { - DeviceTransform(in, expected_kernel_type.place_, &out); + TransDataDevice(in, expected_kernel_type.place_, &out); transformed = true; PassTensorData(&out, &in); } - PADDLE_ENFORCE(transformed, "no transform is done, please check!"); + PADDLE_ENFORCE(transformed, "No transform is applied, please check!"); // get output data output_tensor->ShareDataWith(in); } diff --git a/paddle/framework/data_type_transform.cc b/paddle/framework/data_type_transform.cc index 63373232e9..7df1cc6b75 100644 --- a/paddle/framework/data_type_transform.cc +++ b/paddle/framework/data_type_transform.cc @@ -38,14 +38,11 @@ struct CastDataType { template void operator()() { - auto place = ctx_->GetPlace(); - auto* in_begin = in_.data(); - auto numel = in_.numel(); - auto* in_end = in_begin + numel; - auto* out_begin = out_->mutable_data(place); + auto* in_end = in_begin + in_.numel(); + auto* out_begin = out_->mutable_data(in_.place()); - if (platform::is_cpu_place(place)) { + if (platform::is_cpu_place(in_.place())) { platform::Transform trans; auto* context = static_cast(ctx_); trans(*context, in_begin, in_end, out_begin, @@ -57,38 +54,31 @@ struct CastDataType { } }; -void TransDataType(const platform::DeviceContext* ctx, - const KernelTypePair& kernel_pair, const Variable& in, - Variable* out) { - PADDLE_ENFORCE(in.IsType(), "Only Support Tensor transform!."); - PADDLE_ENFORCE( - platform::places_are_same_class(kernel_pair.first.place_, - kernel_pair.second.place_), - "TransDataType Only Support DataType transform on same place!"); - - auto src = in.Get(); - auto* dst = out->GetMutable(); +void TransDataType(const OpKernelType& kernel_type_for_var, + const OpKernelType& expected_kernel_type, const Tensor& in, + Tensor* out) { + platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); - auto dims = src.dims(); - dst->Resize(dims); - auto dst_type = kernel_pair.second.data_type_; - auto src_type = kernel_pair.first.data_type_; + out->Resize(in.dims()); + auto src_type = kernel_type_for_var.data_type_; + auto dst_type = expected_kernel_type.data_type_; + auto ctx = pool.Get(in.place()); switch (src_type) { case proto::DataType::FP32: - framework::VisitDataType(dst_type, CastDataType(src, dst, ctx)); + framework::VisitDataType(dst_type, CastDataType(in, out, ctx)); break; case proto::DataType::FP64: - framework::VisitDataType(dst_type, CastDataType(src, dst, ctx)); + framework::VisitDataType(dst_type, CastDataType(in, out, ctx)); break; case proto::DataType::INT32: - framework::VisitDataType(dst_type, CastDataType(src, dst, ctx)); + framework::VisitDataType(dst_type, CastDataType(in, out, ctx)); break; case proto::DataType::INT64: - framework::VisitDataType(dst_type, CastDataType(src, dst, ctx)); + framework::VisitDataType(dst_type, CastDataType(in, out, ctx)); break; case proto::DataType::BOOL: - framework::VisitDataType(dst_type, CastDataType(src, dst, ctx)); + framework::VisitDataType(dst_type, CastDataType(in, out, ctx)); break; default: PADDLE_THROW("Not support type %d", src_type); diff --git a/paddle/framework/data_type_transform.h b/paddle/framework/data_type_transform.h index 8ec9074225..067c0c2a5b 100644 --- a/paddle/framework/data_type_transform.h +++ b/paddle/framework/data_type_transform.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include "paddle/framework/op_kernel_type.h" +#include "paddle/framework/tensor.h" #include "paddle/framework/variable.h" #include "paddle/platform/device_context.h" @@ -23,9 +24,9 @@ namespace framework { using KernelTypePair = std::pair; -void TransDataType(const platform::DeviceContext* ctx, - const KernelTypePair& kernel_pair, const Variable& in, - Variable* out); +void TransDataType(const OpKernelType& kernel_type_for_var, + const OpKernelType& expected_kernel_type, const Tensor& in, + Tensor* out); } // namespace framework } // namespace paddle diff --git a/paddle/framework/data_type_transform_test.cc b/paddle/framework/data_type_transform_test.cc new file mode 100644 index 0000000000..89d32f5283 --- /dev/null +++ b/paddle/framework/data_type_transform_test.cc @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/data_type_transform.h" + +#include "gtest/gtest.h" + +TEST(DataTypeTransform, CPUTransform) { + using namespace paddle::framework; + using namespace paddle::platform; + + auto place = CPUPlace(); + + Tensor in; + Tensor out; + + float* ptr = in.mutable_data(make_ddim({2, 3}), place); + int data_number = 2 * 3; + + for (int i = 0; i < data_number; ++i) { + ptr[i] = i / 3; + } + + auto kernel_fp32 = OpKernelType(proto::DataType::FP32, place, + DataLayout::kAnyLayout, LibraryType::kPlain); + auto kernel_fp64 = OpKernelType(proto::DataType::FP64, place, + DataLayout::kAnyLayout, LibraryType::kPlain); + auto kernel_int32 = OpKernelType(proto::DataType::INT32, place, + DataLayout::kAnyLayout, LibraryType::kPlain); + + TransDataType(kernel_fp32, kernel_fp64, in, &out); + double* out_data_double = out.data(); + for (int i = 0; i < data_number; ++i) { + ASSERT_EQ(out_data_double[i], static_cast(i / 3)); + } + + TransDataType(kernel_fp32, kernel_int32, in, &out); + int* out_data_int = out.data(); + for (int i = 0; i < data_number; ++i) { + ASSERT_EQ(out_data_int[i], static_cast(i / 3)); + } +} From 2f344e7f0f85f65ce0f5867ac5985d5576d07395 Mon Sep 17 00:00:00 2001 From: ying Date: Mon, 22 Jan 2018 10:22:34 +0800 Subject: [PATCH 148/158] fix name convention. --- python/paddle/v2/dataset/wmt14.py | 12 ++++++------ python/paddle/v2/dataset/wmt16.py | 32 +++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/python/paddle/v2/dataset/wmt14.py b/python/paddle/v2/dataset/wmt14.py index 1e54a4999b..5104e29051 100644 --- a/python/paddle/v2/dataset/wmt14.py +++ b/python/paddle/v2/dataset/wmt14.py @@ -50,8 +50,8 @@ UNK = "" UNK_IDX = 2 -def __read_to_dict__(tar_file, dict_size): - def __to_dict__(fd, size): +def __read_to_dict(tar_file, dict_size): + def __to_dict(fd, size): out_dict = dict() for line_count, line in enumerate(fd): if line_count < size: @@ -66,19 +66,19 @@ def __read_to_dict__(tar_file, dict_size): if each_item.name.endswith("src.dict") ] assert len(names) == 1 - src_dict = __to_dict__(f.extractfile(names[0]), dict_size) + src_dict = __to_dict(f.extractfile(names[0]), dict_size) names = [ each_item.name for each_item in f if each_item.name.endswith("trg.dict") ] assert len(names) == 1 - trg_dict = __to_dict__(f.extractfile(names[0]), dict_size) + trg_dict = __to_dict(f.extractfile(names[0]), dict_size) return src_dict, trg_dict def reader_creator(tar_file, file_name, dict_size): def reader(): - src_dict, trg_dict = __read_to_dict__(tar_file, dict_size) + src_dict, trg_dict = __read_to_dict(tar_file, dict_size) with tarfile.open(tar_file, mode='r') as f: names = [ each_item.name for each_item in f @@ -160,7 +160,7 @@ def get_dict(dict_size, reverse=True): # if reverse = False, return dict = {'a':'001', 'b':'002', ...} # else reverse = true, return dict = {'001':'a', '002':'b', ...} tar_file = paddle.v2.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) - src_dict, trg_dict = __read_to_dict__(tar_file, dict_size) + src_dict, trg_dict = __read_to_dict(tar_file, dict_size) if reverse: src_dict = {v: k for k, v in src_dict.items()} trg_dict = {v: k for k, v in trg_dict.items()} diff --git a/python/paddle/v2/dataset/wmt16.py b/python/paddle/v2/dataset/wmt16.py index a1899f20b5..bbc28a2da9 100644 --- a/python/paddle/v2/dataset/wmt16.py +++ b/python/paddle/v2/dataset/wmt16.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -ACL2016 Multimodal Machine Translation. Please see this websit for more details: -http://www.statmt.org/wmt16/multimodal-task.html#task1 +ACL2016 Multimodal Machine Translation. Please see this website for more +details: http://www.statmt.org/wmt16/multimodal-task.html#task1 If you use the dataset created for your task, please cite the following paper: Multi30K: Multilingual English-German Image Descriptions. @@ -56,7 +56,7 @@ END_MARK = "" UNK_MARK = "" -def __build_dict__(tar_file, dict_size, save_path, lang): +def __build_dict(tar_file, dict_size, save_path, lang): word_dict = defaultdict(int) with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile("wmt16/train"): @@ -75,12 +75,12 @@ def __build_dict__(tar_file, dict_size, save_path, lang): fout.write("%s\n" % (word[0])) -def __load_dict__(tar_file, dict_size, lang, reverse=False): +def __load_dict(tar_file, dict_size, lang, reverse=False): dict_path = os.path.join(paddle.v2.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size)) if not os.path.exists(dict_path) or ( len(open(dict_path, "r").readlines()) != dict_size): - __build_dict__(tar_file, dict_size, dict_path, lang) + __build_dict(tar_file, dict_size, dict_path, lang) word_dict = {} with open(dict_path, "r") as fdict: @@ -92,7 +92,7 @@ def __load_dict__(tar_file, dict_size, lang, reverse=False): return word_dict -def __get_dict_size__(src_dict_size, trg_dict_size, src_lang): +def __get_dict_size(src_dict_size, trg_dict_size, src_lang): src_dict_size = min(src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else TOTAL_DE_WORDS)) trg_dict_size = min(trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else @@ -102,9 +102,9 @@ def __get_dict_size__(src_dict_size, trg_dict_size, src_lang): def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): def reader(): - src_dict = __load_dict__(tar_file, src_dict_size, src_lang) - trg_dict = __load_dict__(tar_file, trg_dict_size, - ("de" if src_lang == "en" else "en")) + src_dict = __load_dict(tar_file, src_dict_size, src_lang) + trg_dict = __load_dict(tar_file, trg_dict_size, + ("de" if src_lang == "en" else "en")) # the indice for start mark, end mark, and unk are the same in source # language and target language. Here uses the source language @@ -173,8 +173,8 @@ def train(src_dict_size, trg_dict_size, src_lang="en"): assert (src_lang in ["en", "de"], ("An error language type. Only support: " "en (for English); de(for Germany)")) - src_dict_size, trg_dict_size = __get_dict_size__(src_dict_size, - trg_dict_size, src_lang) + src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, + src_lang) return reader_creator( tar_file=paddle.v2.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, @@ -222,8 +222,8 @@ def test(src_dict_size, trg_dict_size, src_lang="en"): ("An error language type. " "Only support: en (for English); de(for Germany)")) - src_dict_size, trg_dict_size = __get_dict_size__(src_dict_size, - trg_dict_size, src_lang) + src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, + src_lang) return reader_creator( tar_file=paddle.v2.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, @@ -269,8 +269,8 @@ def validation(src_dict_size, trg_dict_size, src_lang="en"): assert (src_lang in ["en", "de"], ("An error language type. " "Only support: en (for English); de(for Germany)")) - src_dict_size, trg_dict_size = __get_dict_size__(src_dict_size, - trg_dict_size, src_lang) + src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, + src_lang) return reader_creator( tar_file=paddle.v2.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, @@ -308,7 +308,7 @@ def get_dict(lang, dict_size, reverse=False): "Please invoke paddle.dataset.wmt16.train/test/validation " "first to build the dictionary.") tar_file = os.path.join(paddle.v2.dataset.common.DATA_HOME, "wmt16.tar.gz") - return __load_dict__(tar_file, dict_size, lang, reverse) + return __load_dict(tar_file, dict_size, lang, reverse) def fetch(): From 87b424e866b742b0173389ace4abfd6e66caa1ad Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Mon, 22 Jan 2018 11:12:42 +0800 Subject: [PATCH 149/158] Follow comments --- .../paddle/v2/fluid/layers/math_op_patch.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/python/paddle/v2/fluid/layers/math_op_patch.py b/python/paddle/v2/fluid/layers/math_op_patch.py index 5efc116db2..11197b70a3 100644 --- a/python/paddle/v2/fluid/layers/math_op_patch.py +++ b/python/paddle/v2/fluid/layers/math_op_patch.py @@ -19,7 +19,7 @@ __all__ = ['monkey_patch_variable'] def monkey_patch_variable(): - def new_name(): + def unique_tmp_name(): return unique_name("tmp") def safe_get_dtype(var): @@ -29,21 +29,9 @@ def monkey_patch_variable(): raise ValueError("Cannot get data type from %s", var.name) return dtype - def create_scalar(block, value, dtype): - value = float(value) - tmp_name = new_name() - var = block.create_var(name=tmp_name, shape=[1], dtype=dtype) - block.append_op( - type="fill", - outputs={"Out": [var]}, - attrs={"value": [value], - "shape": [1], - "dtype": dtype}) - return var - def create_tensor(block, value, dtype, shape): value = float(value) - tmp_name = new_name() + tmp_name = unique_tmp_name() var = block.create_var(name=tmp_name, shape=shape, dtype=dtype) block.append_op( type="fill_constant", @@ -53,10 +41,13 @@ def monkey_patch_variable(): 'value': value}) return var + def create_scalar(block, value, dtype): + return create_tensor(block, value, dtype, shape=[1]) + def create_tensor_with_batchsize(ref_var, value, dtype): assert isinstance(ref_var, Variable) value = float(value) - tmp_name = new_name() + tmp_name = unique_tmp_name() var = ref_var.block.create_var(name=tmp_name, dtype=dtype) ref_var.block.append_op( type='fill_constant_batch_size_like', @@ -68,7 +59,7 @@ def monkey_patch_variable(): def astype(self, dtype): """ - Cast a variable to data type. + Cast a variable to a specified data type. NOTE: The variable must be a Tensor Args: self(Variable): The source variable @@ -77,7 +68,7 @@ def monkey_patch_variable(): Returns: Variable with new dtype """ - tmp_name = new_name() + tmp_name = unique_tmp_name() out = self.block.create_var(name=tmp_name, dtype=dtype) self.block.append_op( type="cast", @@ -120,7 +111,7 @@ def monkey_patch_variable(): self = other_var other_var = tmp - tmp_name = new_name() + tmp_name = unique_tmp_name() out = self.block.create_var(name=tmp_name, dtype=lhs_dtype) self.block.append_op( type=op_type, From 2024489bb85175ff77e27910b50f032e0b485325 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Mon, 22 Jan 2018 11:59:23 +0800 Subject: [PATCH 150/158] Fix CI --- paddle/operators/compare_op.h | 2 +- paddle/operators/elementwise_op_function.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/operators/compare_op.h b/paddle/operators/compare_op.h index 15e9cfcaab..9c655d6c0d 100644 --- a/paddle/operators/compare_op.h +++ b/paddle/operators/compare_op.h @@ -54,7 +54,7 @@ class CompareOpKernel public: void Compute(const framework::ExecutionContext& context) const override { using T = typename Functor::ELEM_TYPE; - ElementwiseComputeEx(context); + ElementwiseComputeEx(context); } }; diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index e6f3e39ece..d749b8e875 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -373,7 +373,7 @@ void ElementwiseComputeEx(const framework::ExecutionContext& ctx) { auto* x = ctx.Input("X"); auto* y = ctx.Input("Y"); auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); + z->mutable_data(ctx.GetPlace()); TransformFunctor functor( x, y, z, ctx.template device_context(), Functor()); From d3d855fbc81149bea6050c3518b041583999424f Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Mon, 22 Jan 2018 12:22:27 +0800 Subject: [PATCH 151/158] Fix unittest --- python/paddle/v2/fluid/tests/test_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/tests/test_registry.py b/python/paddle/v2/fluid/tests/test_registry.py index 4fddb91268..44e50ca55a 100644 --- a/python/paddle/v2/fluid/tests/test_registry.py +++ b/python/paddle/v2/fluid/tests/test_registry.py @@ -28,4 +28,4 @@ class TestRegistry(unittest.TestCase): exe = fluid.Executor(place) X = np.random.random((10, 10)).astype("float32") mean_out = exe.run(feed={"X": X}, fetch_list=[output]) - self.assertAlmostEqual(np.mean(X), mean_out) + self.assertAlmostEqual(np.mean(X), mean_out[0]) From 5c26f60875a3ce9fcc86c74dcf6884aa4a01c966 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Mon, 22 Jan 2018 12:33:14 +0800 Subject: [PATCH 152/158] Fix license --- python/paddle/v2/fluid/layers/control_flow.py | 10 +++++----- python/paddle/v2/fluid/layers/ops.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 67c3b2e9ac..a7fc4f6539 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -6,11 +6,11 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import contextlib from layer_function_generator import autodoc diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index 603de1f600..19dc0fdeee 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -6,11 +6,11 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from layer_function_generator import generate_layer_fn __activations__ = [ From e8adcaf27855e452dea5b2deaddb830363cd3964 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 22 Jan 2018 12:50:45 +0800 Subject: [PATCH 153/158] update --- python/paddle/v2/fluid/clip.py | 1 + python/paddle/v2/fluid/layers/ops.py | 1 + python/paddle/v2/fluid/param_attr.py | 1 + python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py | 1 + python/paddle/v2/fluid/tests/test_gradient_clip.py | 1 + 5 files changed, 5 insertions(+) diff --git a/python/paddle/v2/fluid/clip.py b/python/paddle/v2/fluid/clip.py index 777a39e105..386df9823d 100644 --- a/python/paddle/v2/fluid/clip.py +++ b/python/paddle/v2/fluid/clip.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import functools import layers import framework diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index 7e52dc4c34..d296076162 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from ..registry import register_layer __activations__ = [ diff --git a/python/paddle/v2/fluid/param_attr.py b/python/paddle/v2/fluid/param_attr.py index 17fcb262ef..dcca8b6c54 100644 --- a/python/paddle/v2/fluid/param_attr.py +++ b/python/paddle/v2/fluid/param_attr.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from initializer import Initializer, Xavier, Constant from regularizer import WeightDecayRegularizer diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py index 2fde3707da..8776a65bf8 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from __future__ import print_function import numpy as np import paddle.v2 as paddle diff --git a/python/paddle/v2/fluid/tests/test_gradient_clip.py b/python/paddle/v2/fluid/tests/test_gradient_clip.py index 75c5fd9892..4e6e6a1ef6 100644 --- a/python/paddle/v2/fluid/tests/test_gradient_clip.py +++ b/python/paddle/v2/fluid/tests/test_gradient_clip.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid From a6da470b11dd803045dd1be6c99069a2def48198 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Mon, 22 Jan 2018 14:21:44 +0800 Subject: [PATCH 154/158] add memory optimization transpiler demo (#7443) * add memory optimization transpiler demo * add memory benchmark compile option * add gflags instead of macro * refine code --- paddle/framework/executor.cc | 11 ++ paddle/framework/scope.cc | 12 +- python/paddle/v2/fluid/__init__.py | 4 +- python/paddle/v2/fluid/tests/CMakeLists.txt | 1 + .../book_memory_optimization/CMakeLists.txt | 11 ++ .../test_memopt_fit_a_line.py | 44 ++++++ .../test_memopt_image_classification_train.py | 133 ++++++++++++++++++ 7 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 python/paddle/v2/fluid/tests/book_memory_optimization/CMakeLists.txt create mode 100644 python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py create mode 100644 python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index c0418c9266..1382bfca19 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -23,6 +23,7 @@ limitations under the License. */ #include "paddle/framework/op_registry.h" #include "paddle/platform/place.h" +DECLARE_bool(do_memory_benchmark); DEFINE_bool(check_nan_inf, false, "Checking whether operator produce NAN/INF or not. It will be " "extremely slow so please use this flag wisely."); @@ -117,6 +118,10 @@ void Executor::Run(const ProgramDesc& pdesc, Scope* scope, int block_id, auto op = paddle::framework::OpRegistry::CreateOp(*op_desc); VLOG(3) << op->DebugStringEx(local_scope); op->Run(*local_scope, place_); + if (FLAGS_do_memory_benchmark) { + VLOG(2) << "Memory used after operator " + op->Type() + " running: " + << memory::memory_usage(place_); + } if (FLAGS_check_nan_inf) { for (auto& vname : op->OutputVars(true)) { auto* var = local_scope->FindVar(vname); @@ -130,6 +135,12 @@ void Executor::Run(const ProgramDesc& pdesc, Scope* scope, int block_id, if (create_vars && create_local_scope) { scope->DeleteScope(local_scope); } + if (FLAGS_do_memory_benchmark) { + VLOG(2) << "-------------------------------------------------------"; + VLOG(2) << "Memory used after deleting local scope: " + << memory::memory_usage(place_); + VLOG(2) << "-------------------------------------------------------"; + } } } // namespace framework diff --git a/paddle/framework/scope.cc b/paddle/framework/scope.cc index 2bd0ac8f5a..a67ff91009 100644 --- a/paddle/framework/scope.cc +++ b/paddle/framework/scope.cc @@ -20,6 +20,10 @@ limitations under the License. */ #include "paddle/framework/threadpool.h" #include "paddle/string/printf.h" +DEFINE_bool(do_memory_benchmark, false, + "Doing memory benchmark. It will make deleting scope synchronized, " + "and add some memory usage logs"); + namespace paddle { namespace framework { @@ -88,8 +92,12 @@ void Scope::DeleteScope(Scope* scope) { auto it = std::find(this->kids_.begin(), this->kids_.end(), scope); PADDLE_ENFORCE(it != this->kids_.end(), "Cannot find %p as kid scope", scope); this->kids_.erase(it); - // Make delete async. - Async([scope] { delete scope; }); + // When making memory benchmark on Fluid, we have to delete scope sync. + if (FLAGS_do_memory_benchmark) { + delete scope; + } else { + Async([scope] { delete scope; }); + } } void Scope::Rename(const std::string& origin_name, diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index e91eaa4f35..1f041c7459 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -86,7 +86,9 @@ def __bootstrap__(): os.environ['OMP_NUM_THREADS'] = str(num_threads) - read_env_flags = ['use_pinned_memory', 'check_nan_inf'] + read_env_flags = [ + 'use_pinned_memory', 'check_nan_inf', 'do_memory_benchmark' + ] if core.is_compile_gpu(): read_env_flags += ['fraction_of_gpu_memory_to_use', 'op_sync'] core.init_gflags([sys.argv[0]] + diff --git a/python/paddle/v2/fluid/tests/CMakeLists.txt b/python/paddle/v2/fluid/tests/CMakeLists.txt index 9a0240cbf6..8305316082 100644 --- a/python/paddle/v2/fluid/tests/CMakeLists.txt +++ b/python/paddle/v2/fluid/tests/CMakeLists.txt @@ -6,3 +6,4 @@ endforeach() add_subdirectory(book) add_subdirectory(book_distribute) +add_subdirectory(book_memory_optimization) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/CMakeLists.txt b/python/paddle/v2/fluid/tests/book_memory_optimization/CMakeLists.txt new file mode 100644 index 0000000000..213af5d27f --- /dev/null +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") +string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") + +list(REMOVE_ITEM TEST_OPS test_memopt_image_classification_train) +py_test(test_memopt_image_classification_train_resnet SRCS test_memopt_image_classification_train.py ARGS resnet) +py_test(test_memopt_image_classification_train_vgg SRCS test_memopt_image_classification_train.py ARGS vgg) + +# default test +foreach(src ${TEST_OPS}) + py_test(${src} SRCS ${src}.py) +endforeach() diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py new file mode 100644 index 0000000000..6206fcc4be --- /dev/null +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -0,0 +1,44 @@ +import numpy as np +import paddle.v2 as paddle +import paddle.v2.fluid as fluid + +x = fluid.layers.data(name='x', shape=[13], dtype='float32') + +y_predict = fluid.layers.fc(input=x, size=1, act=None) + +y = fluid.layers.data(name='y', shape=[1], dtype='float32') + +cost = fluid.layers.square_error_cost(input=y_predict, label=y) +avg_cost = fluid.layers.mean(x=cost) + +sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.1) +sgd_optimizer.minimize(avg_cost) + +# memopt_program = fluid.default_main_program() +memopt_program = fluid.memory_optimize(fluid.default_main_program()) + +BATCH_SIZE = 200 + +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.uci_housing.train(), buf_size=500), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) +exe = fluid.Executor(place) + +exe.run(fluid.default_startup_program()) + +PASS_NUM = 100 +for pass_id in range(PASS_NUM): + fluid.io.save_persistables(exe, "./fit_a_line.model/") + fluid.io.load_persistables(exe, "./fit_a_line.model/") + for data in train_reader(): + avg_loss_value, = exe.run(memopt_program, + feed=feeder.feed(data), + fetch_list=[avg_cost]) + + if avg_loss_value[0] < 10.0: + exit(0) # if avg cost less than 10.0, we think our code is good. +exit(1) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py new file mode 100644 index 0000000000..cc37f773c4 --- /dev/null +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -0,0 +1,133 @@ +from __future__ import print_function + +import sys + +import paddle.v2 as paddle +import paddle.v2.fluid as fluid + + +def resnet_cifar10(input, depth=32): + def conv_bn_layer(input, ch_out, filter_size, stride, padding, act='relu'): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=False) + return fluid.layers.batch_norm(input=tmp, act=act) + + def shortcut(input, ch_in, ch_out, stride): + if ch_in != ch_out: + return conv_bn_layer(input, ch_out, 1, stride, 0, None) + else: + return input + + def basicblock(input, ch_in, ch_out, stride): + tmp = conv_bn_layer(input, ch_out, 3, stride, 1) + tmp = conv_bn_layer(tmp, ch_out, 3, 1, 1, act=None) + short = shortcut(input, ch_in, ch_out, stride) + return fluid.layers.elementwise_add(x=tmp, y=short, act='relu') + + def layer_warp(block_func, input, ch_in, ch_out, count, stride): + tmp = block_func(input, ch_in, ch_out, stride) + for i in range(1, count): + tmp = block_func(tmp, ch_out, ch_out, 1) + return tmp + + assert (depth - 2) % 6 == 0 + n = (depth - 2) / 6 + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1) + res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) + res2 = layer_warp(basicblock, res1, 16, 32, n, 2) + res3 = layer_warp(basicblock, res2, 32, 64, n, 2) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1) + return pool + + +def vgg16_bn_drop(input): + def conv_block(input, num_filter, groups, dropouts): + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max') + + conv1 = conv_block(input, 64, 2, [0.3, 0]) + conv2 = conv_block(conv1, 128, 2, [0.4, 0]) + conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0]) + conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0]) + conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0]) + + drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5) + fc1 = fluid.layers.fc(input=drop, size=512, act=None) + bn = fluid.layers.batch_norm(input=fc1, act='relu') + drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5) + fc2 = fluid.layers.fc(input=drop2, size=512, act=None) + return fc2 + + +classdim = 10 +data_shape = [3, 32, 32] + +images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32') +label = fluid.layers.data(name='label', shape=[1], dtype='int64') + +net_type = "vgg" +if len(sys.argv) >= 2: + net_type = sys.argv[1] + +if net_type == "vgg": + print("train vgg net") + net = vgg16_bn_drop(images) +elif net_type == "resnet": + print("train resnet") + net = resnet_cifar10(images, 32) +else: + raise ValueError("%s network is not supported" % net_type) + +predict = fluid.layers.fc(input=net, size=classdim, act='softmax') +cost = fluid.layers.cross_entropy(input=predict, label=label) +avg_cost = fluid.layers.mean(x=cost) + +optimizer = fluid.optimizer.Adam(learning_rate=0.001) +opts = optimizer.minimize(avg_cost) + +accuracy = fluid.evaluator.Accuracy(input=predict, label=label) + +# memopt_program = fluid.default_main_program() +memopt_program = fluid.memory_optimize(fluid.default_main_program()) + +BATCH_SIZE = 128 +PASS_NUM = 1 + +train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.cifar.train10(), buf_size=128 * 10), + batch_size=BATCH_SIZE) + +place = fluid.CPUPlace() +exe = fluid.Executor(place) +feeder = fluid.DataFeeder(place=place, feed_list=[images, label]) +exe.run(fluid.default_startup_program()) + +for pass_id in range(PASS_NUM): + accuracy.reset(exe) + for data in train_reader(): + loss, acc = exe.run(memopt_program, + feed=feeder.feed(data), + fetch_list=[avg_cost] + accuracy.metrics) + pass_acc = accuracy.eval(exe) + print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str( + pass_acc)) + # this model is slow, so if we can train two mini batch, we think it works properly. + exit(0) +exit(1) From cb34f6a230bf51cc6cb0b8b2ef93b3e13ed3f516 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 22 Jan 2018 14:45:06 +0800 Subject: [PATCH 155/158] update fluid vgg16 and add readme --- benchmark/cluster/vgg16/fluid/Dockerfile | 7 +++++-- benchmark/cluster/vgg16/fluid/README.md | 15 +++++++++++++++ benchmark/cluster/vgg16/fluid/paddle_k8s | 1 - benchmark/cluster/vgg16/fluid/pserver.yaml | 2 +- benchmark/cluster/vgg16/fluid/trainer.yaml | 4 ++-- benchmark/cluster/vgg16/fluid/vgg16.py | 6 ++++-- 6 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 benchmark/cluster/vgg16/fluid/README.md diff --git a/benchmark/cluster/vgg16/fluid/Dockerfile b/benchmark/cluster/vgg16/fluid/Dockerfile index 77cd17f2b9..711076b09e 100644 --- a/benchmark/cluster/vgg16/fluid/Dockerfile +++ b/benchmark/cluster/vgg16/fluid/Dockerfile @@ -3,10 +3,13 @@ #ADD reader.py /workspace/ #RUN python /workspace/reader.py FROM python:2.7.14 -ADD *.whl / -RUN pip install /*.whl && rm -f /*.whl ADD paddle_k8s /usr/bin ADD k8s_tools.py /root RUN pip install -U kubernetes opencv-python && apt-get update -y && apt-get install -y iputils-ping libgtk2.0-dev +ADD *.whl / +RUN pip install /*.whl && rm -f /*.whl +ENV LD_LIBRARY_PATH=/usr/local/lib +ADD reader.py /workspace/ +RUN python /workspace/reader.py ADD vgg16.py /workspace/ diff --git a/benchmark/cluster/vgg16/fluid/README.md b/benchmark/cluster/vgg16/fluid/README.md new file mode 100644 index 0000000000..63a460f7a6 --- /dev/null +++ b/benchmark/cluster/vgg16/fluid/README.md @@ -0,0 +1,15 @@ +# Fluid distributed training perf test + +## Steps to get started + +1. You must re-compile PaddlePaddle and enable `-DWITH_DISTRIBUTE` to build PaddlePaddle with distributed support. +1. When the build finishes, copy the output `whl` package located under `build/python/dist` to current directory. +1. Run `docker build -t [image:tag] .` to build the docker image and run `docker push [image:tag]` to push the image to reponsitory so kubernetes can find it. +1. Run `kubectl create -f pserver.yaml && kubectl create -f trainer.yaml` to start the job on your kubernetes cluster (you must configure the `kubectl` client before this step). +1. Run `kubectl get po` to get running pods, and run `kubectl logs [podID]` to fetch the pod log of pservers and trainers. + +Check the logs for the distributed training progress and analyze the performance. + +## Enable verbos logs + +Edit `pserver.yaml` and `trainer.yaml` and add an environment variable `GLOG_v=3` to see what happend in detail. \ No newline at end of file diff --git a/benchmark/cluster/vgg16/fluid/paddle_k8s b/benchmark/cluster/vgg16/fluid/paddle_k8s index 8f1c5db717..af5f35b3ec 100755 --- a/benchmark/cluster/vgg16/fluid/paddle_k8s +++ b/benchmark/cluster/vgg16/fluid/paddle_k8s @@ -61,7 +61,6 @@ start_fluid_process() { if [ "${TRAINING_ROLE}" == "TRAINER" ]; then check_failed_cnt ${TRAINERS} sleep 5 - stdbuf -oL python /root/k8s_tools.py wait_pods_running paddle-job-master=${PADDLE_JOB_NAME} 1 export PADDLE_INIT_TRAINER_ID=$(python /root/k8s_tools.py fetch_trainer_id) fi export PADDLE_INIT_PSERVERS=$(python /root/k8s_tools.py fetch_pserver_ips) diff --git a/benchmark/cluster/vgg16/fluid/pserver.yaml b/benchmark/cluster/vgg16/fluid/pserver.yaml index 47d2380d2e..e1a58260af 100644 --- a/benchmark/cluster/vgg16/fluid/pserver.yaml +++ b/benchmark/cluster/vgg16/fluid/pserver.yaml @@ -33,7 +33,7 @@ spec: - name: TOPOLOGY value: "" - name: ENTRY - value: "MKL_NUM_THREADS=1 python /workspace/vgg16.py --local 0" + value: "LD_LIBRARY_PATH=/usr/local/lib MKL_NUM_THREADS=1 python /workspace/vgg16.py --local 0" - name: TRAINER_PACKAGE value: "/workspace" - name: PADDLE_INIT_PORT diff --git a/benchmark/cluster/vgg16/fluid/trainer.yaml b/benchmark/cluster/vgg16/fluid/trainer.yaml index bada190764..c8e26d4b51 100644 --- a/benchmark/cluster/vgg16/fluid/trainer.yaml +++ b/benchmark/cluster/vgg16/fluid/trainer.yaml @@ -17,7 +17,7 @@ spec: - name: trainer image: "registry.baidu.com/paddlepaddle/rawjob:vgg16_fluid" imagePullPolicy: Always - command: ["paddle_k8s", "start_trainer", "v2"] + command: ["paddle_k8s", "start_fluid"] env: - name: PADDLE_JOB_NAME value: vgg16job @@ -30,7 +30,7 @@ spec: - name: TOPOLOGY value: "" - name: ENTRY - value: "cd /workspace && MKL_NUM_THREADS=1 python /workspace/vgg16.py" + value: "cd /workspace && LD_LIBRARY_PATH=/usr/local/lib MKL_NUM_THREADS=1 python /workspace/vgg16.py --local 0" - name: TRAINER_PACKAGE value: "/workspace" - name: PADDLE_INIT_PORT diff --git a/benchmark/cluster/vgg16/fluid/vgg16.py b/benchmark/cluster/vgg16/fluid/vgg16.py index 0595a28784..a973f9d2a6 100644 --- a/benchmark/cluster/vgg16/fluid/vgg16.py +++ b/benchmark/cluster/vgg16/fluid/vgg16.py @@ -140,12 +140,14 @@ def main(): def train_loop(exe, trainer_prog): iters = 0 + ts = time.time() for pass_id in range(args.num_passes): # train start_time = time.time() num_samples = 0 accuracy.reset(exe) for batch_id, data in enumerate(train_reader()): + ts = time.time() img_data = np.array(map(lambda x: x[0].reshape(data_shape), data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") @@ -158,8 +160,8 @@ def main(): iters += 1 num_samples += len(data) print( - "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f" % - (pass_id, iters, loss, acc) + "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, spent %f" % + (pass_id, iters, loss, acc, time.time() - ts) ) # The accuracy is the accumulation of batches, but not the current batch. pass_elapsed = time.time() - start_time From 1c4968ee8b89b52e78d46eff6e6f2d6bb60f3136 Mon Sep 17 00:00:00 2001 From: ying Date: Mon, 22 Jan 2018 15:34:13 +0800 Subject: [PATCH 156/158] fix copyright --- .../test_memopt_fit_a_line.py | 14 ++++++++++++++ .../test_memopt_image_classification_train.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index 6206fcc4be..cf054bb0fe 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py index cc37f773c4..42b3cb81ce 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import sys From b38452dffaa766311450fd79fb0432b63899545d Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 22 Jan 2018 17:01:01 +0800 Subject: [PATCH 157/158] fix styles --- benchmark/cluster/vgg16/fluid/README.md | 3 +- benchmark/cluster/vgg16/fluid/k8s_tools.py | 18 ++++++- benchmark/cluster/vgg16/fluid/reader.py | 14 ++++++ benchmark/cluster/vgg16/fluid/vgg16.py | 58 +++++++++++++++------- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/benchmark/cluster/vgg16/fluid/README.md b/benchmark/cluster/vgg16/fluid/README.md index 63a460f7a6..02b17dceb9 100644 --- a/benchmark/cluster/vgg16/fluid/README.md +++ b/benchmark/cluster/vgg16/fluid/README.md @@ -12,4 +12,5 @@ Check the logs for the distributed training progress and analyze the performance ## Enable verbos logs -Edit `pserver.yaml` and `trainer.yaml` and add an environment variable `GLOG_v=3` to see what happend in detail. \ No newline at end of file +Edit `pserver.yaml` and `trainer.yaml` and add an environment variable `GLOG_v=3` to see what happend in detail. + diff --git a/benchmark/cluster/vgg16/fluid/k8s_tools.py b/benchmark/cluster/vgg16/fluid/k8s_tools.py index 8a64dbd361..4bee96a7a8 100644 --- a/benchmark/cluster/vgg16/fluid/k8s_tools.py +++ b/benchmark/cluster/vgg16/fluid/k8s_tools.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/env python import os import sys @@ -33,6 +47,7 @@ def wait_pods_running(label_selector, desired): print 'current cnt: %d sleep for 5 seconds...' % count time.sleep(5) + def count_pods_by_phase(label_selector, phase): pod_list = fetch_pods_info(label_selector) filtered_pod_list = filter(lambda x: x[0] == phase, pod_list) @@ -45,12 +60,14 @@ def fetch_pserver_ips(): pserver_ips = [item[1] for item in pod_list] return ",".join(pserver_ips) + def fetch_master_ip(): label_selector = "paddle-job-master=%s" % PADDLE_JOB_NAME pod_list = fetch_pods_info(label_selector) master_ips = [item[1] for item in pod_list] return master_ips[0] + def fetch_trainer_id(): label_selector = "paddle-job=%s" % PADDLE_JOB_NAME pod_list = fetch_pods_info(label_selector) @@ -75,4 +92,3 @@ if __name__ == "__main__": print count_pods_by_phase(sys.argv[2], sys.argv[3]) elif command == "wait_pods_running": wait_pods_running(sys.argv[2], sys.argv[3]) - diff --git a/benchmark/cluster/vgg16/fluid/reader.py b/benchmark/cluster/vgg16/fluid/reader.py index c5161ddea2..3e20f830fc 100644 --- a/benchmark/cluster/vgg16/fluid/reader.py +++ b/benchmark/cluster/vgg16/fluid/reader.py @@ -1,2 +1,16 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.v2 as paddle paddle.dataset.cifar.train10() diff --git a/benchmark/cluster/vgg16/fluid/vgg16.py b/benchmark/cluster/vgg16/fluid/vgg16.py index a973f9d2a6..3c7b5bf2f1 100644 --- a/benchmark/cluster/vgg16/fluid/vgg16.py +++ b/benchmark/cluster/vgg16/fluid/vgg16.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """VGG16 benchmark in Fluid""" from __future__ import print_function @@ -11,6 +25,7 @@ import argparse import functools import os + def str2bool(v): if v.lower() in ('yes', 'true', 't', 'y', '1'): return True @@ -19,6 +34,7 @@ def str2bool(v): else: raise argparse.ArgumentTypeError('Boolean value expected.') + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--batch_size', type=int, default=128, help="Batch size for training.") @@ -122,7 +138,6 @@ def main(): place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0) exe = fluid.Executor(place) - # test def test(exe): accuracy.reset(exe) @@ -148,20 +163,21 @@ def main(): accuracy.reset(exe) for batch_id, data in enumerate(train_reader()): ts = time.time() - img_data = np.array(map(lambda x: x[0].reshape(data_shape), - data)).astype("float32") + img_data = np.array( + map(lambda x: x[0].reshape(data_shape), data)).astype( + "float32") y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape([-1, 1]) loss, acc = exe.run(trainer_prog, feed={"pixel": img_data, - "label": y_data}, + "label": y_data}, fetch_list=[avg_cost] + accuracy.metrics) iters += 1 num_samples += len(data) print( - "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, spent %f" % - (pass_id, iters, loss, acc, time.time() - ts) + "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, spent %f" + % (pass_id, iters, loss, acc, time.time() - ts) ) # The accuracy is the accumulation of batches, but not the current batch. pass_elapsed = time.time() - start_time @@ -170,7 +186,7 @@ def main(): print( "Pass = %d, Training performance = %f imgs/s, Train accuracy = %f, Test accuracy = %f\n" % (pass_id, num_samples / pass_elapsed, pass_train_acc, - pass_test_acc)) + pass_test_acc)) if args.local: # Parameter initialization @@ -179,8 +195,8 @@ def main(): # data reader train_reader = paddle.batch( paddle.reader.shuffle( - paddle.dataset.cifar.train10() - if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), + paddle.dataset.cifar.train10() if args.data_set == 'cifar10' + else paddle.dataset.flowers.train(), buf_size=5120), batch_size=args.batch_size) test_reader = paddle.batch( @@ -196,19 +212,25 @@ def main(): pserver_endpoints = ",".join(eplist) print("pserver endpoints: ", pserver_endpoints) trainers = int(os.getenv("TRAINERS")) # total trainer count - current_endpoint = os.getenv("POD_IP") + ":6174" # current pserver endpoint - training_role = os.getenv("TRAINING_ROLE", - "TRAINER") # get the training role: trainer/pserver + current_endpoint = os.getenv( + "POD_IP") + ":6174" # current pserver endpoint + training_role = os.getenv( + "TRAINING_ROLE", + "TRAINER") # get the training role: trainer/pserver t = fluid.DistributeTranspiler() t.transpile( - optimize_ops, params_grads, pservers=pserver_endpoints, trainers=trainers) + optimize_ops, + params_grads, + pservers=pserver_endpoints, + trainers=trainers) if training_role == "PSERVER": if not current_endpoint: print("need env SERVER_ENDPOINT") exit(1) pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) + pserver_startup = t.get_startup_program(current_endpoint, + pserver_prog) print("starting server side startup") exe.run(pserver_startup) print("starting parameter server...") @@ -220,13 +242,13 @@ def main(): # data reader train_reader = paddle.batch( paddle.reader.shuffle( - paddle.dataset.cifar.train10() - if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), + paddle.dataset.cifar.train10() if args.data_set == 'cifar10' + else paddle.dataset.flowers.train(), buf_size=5120), batch_size=args.batch_size) test_reader = paddle.batch( - paddle.dataset.cifar.test10() - if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), + paddle.dataset.cifar.test10() if args.data_set == 'cifar10' else + paddle.dataset.flowers.test(), batch_size=args.batch_size) trainer_prog = t.get_trainer_program() From 900e911f4223e654c20a68a2db1404dadccfb953 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Mon, 22 Jan 2018 18:54:54 +0800 Subject: [PATCH 158/158] fix style check --- benchmark/cluster/vgg16/fluid/README.md | 1 - benchmark/cluster/vgg16/fluid/vgg16.py | 1 - benchmark/cluster/vgg16/v2/pserver.yaml | 6 +++--- benchmark/cluster/vgg16/v2/trainer.yaml | 6 +++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/benchmark/cluster/vgg16/fluid/README.md b/benchmark/cluster/vgg16/fluid/README.md index 02b17dceb9..71a3a934d2 100644 --- a/benchmark/cluster/vgg16/fluid/README.md +++ b/benchmark/cluster/vgg16/fluid/README.md @@ -13,4 +13,3 @@ Check the logs for the distributed training progress and analyze the performance ## Enable verbos logs Edit `pserver.yaml` and `trainer.yaml` and add an environment variable `GLOG_v=3` to see what happend in detail. - diff --git a/benchmark/cluster/vgg16/fluid/vgg16.py b/benchmark/cluster/vgg16/fluid/vgg16.py index 3c7b5bf2f1..88d6d79cc0 100644 --- a/benchmark/cluster/vgg16/fluid/vgg16.py +++ b/benchmark/cluster/vgg16/fluid/vgg16.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """VGG16 benchmark in Fluid""" from __future__ import print_function diff --git a/benchmark/cluster/vgg16/v2/pserver.yaml b/benchmark/cluster/vgg16/v2/pserver.yaml index ed1671bbbd..943675e147 100644 --- a/benchmark/cluster/vgg16/v2/pserver.yaml +++ b/benchmark/cluster/vgg16/v2/pserver.yaml @@ -1,13 +1,13 @@ apiVersion: extensions/v1beta1 kind: ReplicaSet metadata: - name: vgg16job-pserver + name: vgg16v2job-pserver spec: replicas: 10 template: metadata: labels: - paddle-job-pserver: vgg16job + paddle-job-pserver: vgg16v2job spec: hostNetwork: true imagePullSecrets: @@ -21,7 +21,7 @@ spec: containerPort: 30236 env: - name: PADDLE_JOB_NAME - value: vgg16job + value: vgg16v2job - name: TRAINERS value: "20" - name: PSERVERS diff --git a/benchmark/cluster/vgg16/v2/trainer.yaml b/benchmark/cluster/vgg16/v2/trainer.yaml index 75fffc64b0..200b6dc304 100644 --- a/benchmark/cluster/vgg16/v2/trainer.yaml +++ b/benchmark/cluster/vgg16/v2/trainer.yaml @@ -1,14 +1,14 @@ apiVersion: batch/v1 kind: Job metadata: - name: vgg16job-trainer + name: vgg16v2job-trainer spec: parallelism: 20 completions: 20 template: metadata: labels: - paddle-job: vgg16job + paddle-job: vgg16v2job spec: imagePullSecrets: - name: job-registry-secret @@ -20,7 +20,7 @@ spec: command: ["paddle_k8s", "start_trainer", "v2"] env: - name: PADDLE_JOB_NAME - value: vgg16job + value: vgg16v2job - name: TRAINERS value: "20" - name: PSERVERS