From e9c61b67e25f37324131bd8f54afc0b45d992c6d Mon Sep 17 00:00:00 2001 From: Peng Li Date: Wed, 30 Nov 2016 21:55:29 +0800 Subject: [PATCH 01/62] Fix bug in processing instance weight and coeff in CRFLayer The instance weight and coeff are only mulipled to the gradient with respect to input. --- paddle/gserver/layers/CRFDecodingLayer.cpp | 2 +- paddle/gserver/layers/CRFLayer.cpp | 29 ++- paddle/gserver/layers/CRFLayer.h | 1 + paddle/gserver/layers/LinearChainCRF.cpp | 72 ++++---- paddle/gserver/layers/LinearChainCRF.h | 26 ++- paddle/gserver/tests/CMakeLists.txt | 9 + paddle/gserver/tests/test_CRFLayerGrad.cpp | 178 +++++++++++++++++++ paddle/gserver/tests/test_LayerGrad.cpp | 21 --- paddle/gserver/tests/test_LinearChainCRF.cpp | 2 +- 9 files changed, 260 insertions(+), 80 deletions(-) create mode 100644 paddle/gserver/tests/test_CRFLayerGrad.cpp diff --git a/paddle/gserver/layers/CRFDecodingLayer.cpp b/paddle/gserver/layers/CRFDecodingLayer.cpp index 8986741dc3..3c66f4191a 100644 --- a/paddle/gserver/layers/CRFDecodingLayer.cpp +++ b/paddle/gserver/layers/CRFDecodingLayer.cpp @@ -24,7 +24,7 @@ bool CRFDecodingLayer::init(const LayerMap& layerMap, return false; } crf_.reset(new LinearChainCRF( - numClasses_, parameter_->getBuf(PARAMETER_VALUE)->getData(), nullptr)); + numClasses_, parameter_->getBuf(PARAMETER_VALUE)->getData())); return true; } diff --git a/paddle/gserver/layers/CRFLayer.cpp b/paddle/gserver/layers/CRFLayer.cpp index ed4f864ba9..fa434cda00 100644 --- a/paddle/gserver/layers/CRFLayer.cpp +++ b/paddle/gserver/layers/CRFLayer.cpp @@ -42,6 +42,8 @@ bool CRFLayer::init(const LayerMap& layerMap, CHECK_EQ(parameters_[0]->getSize(), numClasses_ * (numClasses_ + 2)); parameter_ = parameters_[0]; + weight_.reset( + new Weight(numClasses_ + 2, numClasses_, parameter_)); // We don't need sequenceStartPositions because each sample of output_ is // for the cost of one sequence. @@ -69,11 +71,7 @@ void CRFLayer::forward(PassType passType) { for (size_t i = 0; i < numSequences; ++i) { if (i >= crfs_.size()) { - crfs_.emplace_back(numClasses_, - parameter_->getBuf(PARAMETER_VALUE)->getData(), - parameter_->getBuf(PARAMETER_GRADIENT) - ? parameter_->getBuf(PARAMETER_GRADIENT)->getData() - : nullptr); + crfs_.emplace_back(numClasses_, weight_->getW()->getData()); } output_.value->getData()[i] = crfs_[i].forward(output.value->getData() + numClasses_ * starts[i], @@ -94,21 +92,22 @@ void CRFLayer::backward(const UpdateCallback& callback) { int numSequences = label.sequenceStartPositions->getSize() - 1; for (int i = 0; i < numSequences; ++i) { + bool needWGrad = weight_->getWGrad() ? true : false; crfs_[i].backward(output.value->getData() + numClasses_ * starts[i], - output.grad->getData() + numClasses_ * starts[i], label.ids->getData() + starts[i], - starts[i + 1] - starts[i]); - if (weightLayer_) { - real weight = getInputValue(*weightLayer_)->getElement(i, 0); - MatrixPtr grad = output.grad->subRowMatrix(starts[i], starts[i + 1]); - grad->mulScalar(weight); + starts[i + 1] - starts[i], needWGrad); + real instanceWeight = weightLayer_ ? + getInputValue(*weightLayer_)->getElement(i, 0) : real(1.0f); + instanceWeight *= coeff_; + + MatrixPtr grad = output.grad->subRowMatrix(starts[i], starts[i + 1]); + grad->add(*crfs_[i].getXGrad(), real(1.0f), instanceWeight); + if (needWGrad) { + weight_->getWGrad()->add(*crfs_[i].getWGrad(), real(1.0f), + instanceWeight); } } - if (coeff_ != real(1.0f)) { - output.grad->mulScalar(coeff_); - } - parameter_->incUpdate(callback); } diff --git a/paddle/gserver/layers/CRFLayer.h b/paddle/gserver/layers/CRFLayer.h index 21c7fc61e1..ae024b8fa3 100644 --- a/paddle/gserver/layers/CRFLayer.h +++ b/paddle/gserver/layers/CRFLayer.h @@ -38,6 +38,7 @@ protected: ParameterPtr parameter_; std::vector crfs_; LayerPtr weightLayer_; // weight for each sequence + std::unique_ptr weight_; // parameters real coeff_; // weight for the layer }; diff --git a/paddle/gserver/layers/LinearChainCRF.cpp b/paddle/gserver/layers/LinearChainCRF.cpp index 2b3a50b2e2..f9e4bb83d4 100644 --- a/paddle/gserver/layers/LinearChainCRF.cpp +++ b/paddle/gserver/layers/LinearChainCRF.cpp @@ -17,18 +17,12 @@ limitations under the License. */ namespace paddle { -LinearChainCRF::LinearChainCRF(int numClasses, real* para, real* grad) +LinearChainCRF::LinearChainCRF(int numClasses, real* para) : numClasses_(numClasses) { a_ = Matrix::create(para, 1, numClasses_); b_ = Matrix::create(para + numClasses_, 1, numClasses_); w_ = Matrix::create(para + 2 * numClasses_, numClasses_, numClasses_); - if (grad) { - da_ = Matrix::create(grad, 1, numClasses_); - db_ = Matrix::create(grad + numClasses_, 1, numClasses_); - dw_ = Matrix::create(grad + 2 * numClasses_, numClasses_, numClasses_); - } - ones_ = Matrix::create(1, numClasses_); ones_->one(); @@ -107,19 +101,24 @@ real LinearChainCRF::forward(real* x, int* s, int length) { return -ll; } -void LinearChainCRF::backward(real* x, real* dx, int* s, int length) { +void LinearChainCRF::backward(real* x, int* s, int length, bool needWGrad) { MatrixPtr matX = Matrix::create(x, length, numClasses_); - MatrixPtr matDX = Matrix::create(dx, length, numClasses_); - MatrixPtr matGrad = Matrix::create(length, numClasses_); + Matrix::resizeOrCreate(matGrad_, length, numClasses_); Matrix::resizeOrCreate(beta_, length, numClasses_); real* b = b_->getData(); - real* dw = dw_ ? dw_->getData() : nullptr; + if (needWGrad) { + Matrix::resizeOrCreate(matWGrad_, numClasses_ + 2, numClasses_); + matWGrad_->zeroMem(); + da_ = matWGrad_->subRowMatrix(0, 1); + db_ = matWGrad_->subRowMatrix(1, 2); + dw_ = matWGrad_->subRowMatrix(2, numClasses_ + 2); + } real* alpha = alpha_->getData(); real* beta = beta_->getData(); real* expW = expW_->getData(); real* expX = expX_->getData(); - real* grad = matGrad->getData(); + real* grad = matGrad_->getData(); for (int i = 0; i < numClasses_; ++i) { beta[(length - 1) * numClasses_ + i] = exp(b[i]); @@ -140,39 +139,38 @@ void LinearChainCRF::backward(real* x, real* dx, int* s, int length) { normalizeL1(beta + k * numClasses_, numClasses_); } - matGrad->dotMul(*alpha_, *beta_); - matGrad->rowNormalizeL1(*matGrad); + matGrad_->dotMul(*alpha_, *beta_); + matGrad_->rowNormalizeL1(*matGrad_); for (int k = 0; k < length; ++k) { grad[k * numClasses_ + s[k]] -= (real)1; } - matDX->add(*matGrad); - if (da_) { - da_->add(*matGrad->subMatrix(/* startRow= */ 0, /* numRows= */ 1)); - } - if (db_) { - db_->add(*matGrad->subMatrix(/* startRow= */ length - 1, 1)); - } - beta_->dotMul(*beta_, *expX_); - beta_->rowNormalizeL1(*beta_); + if (needWGrad) { + da_->add(*matGrad_->subMatrix(/* startRow= */ 0, /* numRows= */ 1)); + db_->add(*matGrad_->subMatrix(/* startRow= */ length - 1, 1)); - for (int k = 1; dw && k < length; ++k) { - real sum = 0; - for (int i = 0; i < numClasses_; ++i) { - for (int j = 0; j < numClasses_; ++j) { - sum += expW[i * numClasses_ + j] * alpha[(k - 1) * numClasses_ + i] * - beta[k * numClasses_ + j]; + beta_->dotMul(*beta_, *expX_); + beta_->rowNormalizeL1(*beta_); + + real* dw = dw_->getData(); + for (int k = 1; k < length; ++k) { + real sum = 0; + for (int i = 0; i < numClasses_; ++i) { + for (int j = 0; j < numClasses_; ++j) { + sum += expW[i * numClasses_ + j] * alpha[(k - 1) * numClasses_ + i] * + beta[k * numClasses_ + j]; + } } - } - sum = 1 / sum; - for (int i = 0; i < numClasses_; ++i) { - for (int j = 0; j < numClasses_; ++j) { - dw[i * numClasses_ + j] += sum * expW[i * numClasses_ + j] * - alpha[(k - 1) * numClasses_ + i] * - beta[k * numClasses_ + j]; + sum = 1 / sum; + for (int i = 0; i < numClasses_; ++i) { + for (int j = 0; j < numClasses_; ++j) { + dw[i * numClasses_ + j] += sum * expW[i * numClasses_ + j] * + alpha[(k - 1) * numClasses_ + i] * + beta[k * numClasses_ + j]; + } } + dw[s[k - 1] * numClasses_ + s[k]] -= (real)1; } - dw[s[k - 1] * numClasses_ + s[k]] -= (real)1; } } diff --git a/paddle/gserver/layers/LinearChainCRF.h b/paddle/gserver/layers/LinearChainCRF.h index 6368f2b9de..f58ac6b581 100644 --- a/paddle/gserver/layers/LinearChainCRF.h +++ b/paddle/gserver/layers/LinearChainCRF.h @@ -21,7 +21,7 @@ namespace paddle { class LinearChainCRF { public: /** - * The size of para and grad must be \f$(numClasses + 2) * numClasses\f$. + * The size of para must be \f$(numClasses + 2) * numClasses\f$. * The first numClasses values of para are for starting weights (\f$a\f$). * The next numClasses values of para are for ending weights (\f$b\f$), * The remaning values are for transition weights (\f$w\f$). @@ -34,7 +34,7 @@ public: * all possible * sequences is \f$1\f$, and \f$x\f$ is the input feature to the CRF. */ - LinearChainCRF(int numClasses, real* para, real* grad); + LinearChainCRF(int numClasses, real* para); /** * Calculate the negative log likelihood of s given x. @@ -45,29 +45,45 @@ public: /** * Calculate the gradient with respect to x, a, b, and w. - * The gradient of x will be stored in dx. * backward() can only be called after a corresponding call to forward() with * the same x, s and length. - * @note The gradient is added to dx and grad (provided at constructor). + * The gradient with respect to a, b, and w will not be calculated if + * needWGrad is false. + * @note Please call getWGrad() and getXGrad() to get the gradient with + * respect to (a, b, w) and x respectively. */ - void backward(real* x, real* dx, int* s, int length); + void backward(real* x, int* s, int length, bool needWGrad); /** * Find the most probable sequence given x. The result will be stored in s. */ void decode(real* x, int* s, int length); + /* + * Return the gradient with respect to (a, b, w). It can only be called after + * a corresponding call to backward(). + */ + MatrixPtr getWGrad() { return matWGrad_; } + + /* + * Return the gradient with respect to x. It can only be called after a + * corresponding call to backward(). + */ + MatrixPtr getXGrad() { return matGrad_; } + protected: int numClasses_; MatrixPtr a_; MatrixPtr b_; MatrixPtr w_; + MatrixPtr matWGrad_; MatrixPtr da_; MatrixPtr db_; MatrixPtr dw_; MatrixPtr ones_; MatrixPtr expX_; + MatrixPtr matGrad_; MatrixPtr alpha_; MatrixPtr beta_; MatrixPtr maxX_; diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 0651d0b473..1525960ffd 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -20,6 +20,15 @@ add_unittest_without_exec(test_LayerGrad add_test(NAME test_LayerGrad COMMAND test_LayerGrad) +################ test_CRFLayerGrad #################### +add_unittest_without_exec(test_CRFLayerGrad + test_CRFLayerGrad.cpp + LayerGradUtil.cpp + TestUtil.cpp) +add_test(NAME test_CRFLayerGrad + COMMAND test_CRFLayerGrad) + + add_unittest_without_exec(test_ActivationGrad test_ActivationGrad.cpp LayerGradUtil.cpp diff --git a/paddle/gserver/tests/test_CRFLayerGrad.cpp b/paddle/gserver/tests/test_CRFLayerGrad.cpp new file mode 100644 index 0000000000..bc1d5f3061 --- /dev/null +++ b/paddle/gserver/tests/test_CRFLayerGrad.cpp @@ -0,0 +1,178 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "paddle/gserver/layers/DataLayer.h" +#include "paddle/trainer/Trainer.h" +#include "paddle/gserver/layers/LinearChainCRF.h" +#include "ModelConfig.pb.h" + +#include "TestUtil.h" +#include "LayerGradUtil.h" + +using namespace paddle; // NOLINT + +P_DECLARE_bool(use_gpu); +P_DECLARE_int32(gpu_id); +P_DECLARE_double(checkgrad_eps); +P_DECLARE_bool(thread_local_rand_use_global_seed); +P_DECLARE_bool(prev_batch_state); + +static inline bool getNextSequence(std::vector& seq, int numClasses) { + for (auto& v : seq) { + if (++v < numClasses) { + return true; + } + v = 0; + } + return false; +} + +// log(exp(x) + exp(y)) +static inline real logSum(real x, real y) { + real maxValue = std::max(x, y); + if (std::isinf(maxValue)) { + return -std::numeric_limits::infinity(); + } else { + return maxValue + log(exp(x - maxValue) + exp(y - maxValue)); + } +} + +static inline std::vector genRandLabels(int numClasses, int length) { + std::vector labels(length); + for (int i = 0; i < length; ++i) { + labels[i] = rand() % numClasses; // NOLINT + } + return labels; +} + +TEST(CRFLayer, cost) { + const int numClasses = 4; + CpuVector para(numClasses * (numClasses + 2)); + real* a = para.getData(); + real* b = para.getData() + numClasses; + real* w = para.getData() + 2 * numClasses; + LinearChainCRF crf(4, para.getData()); + for (int length : {1, 2, 3, 10}) { + for (int tries = 0; tries < 10; ++tries) { + CpuMatrix x(length, numClasses); + x.randomizeUniform(); + para.randnorm(0, 2); + + std::vector goldenLabels = genRandLabels(numClasses, length); + + real cost = crf.forward(x.getData(), goldenLabels.data(), length); + + real logZ = -std::numeric_limits::infinity(); + real logNominator = -std::numeric_limits::infinity(); + std::vector testResult(length, 0); + do { + real score = a[testResult.front()]; + score += x.getElement(0, testResult.front()); + for (int k = 1; k < length; ++k) { + score += x.getElement(k, testResult[k]) + + w[numClasses * testResult[k - 1] + testResult[k]]; + } + score += b[testResult.back()]; + logZ = logSum(logZ, score); + + if (goldenLabels == testResult) { + logNominator = score; + } + } while (getNextSequence(testResult, numClasses)); + + real trueCost = -logNominator + logZ; + + real diff = fabs(trueCost - cost); + diff /= fabs(cost) < fabs(trueCost) ? fabs(cost) : fabs(trueCost); + VLOG(1) << "cost=" << cost << " trueCost=" << trueCost + << " diff=" << diff << std::endl; + if (typeid(real) == typeid(double)) { // NOLINT + EXPECT_LE(diff, 1e-10); + } else { + EXPECT_LE(diff, 5e-3); + } + } + } +} + +inline real epsilon() { + return typeid(real) == typeid(double) ? 1e-10 : 0.05; +} + +TestConfig initTestConfig(size_t numClasses, bool withWeight) { + TestConfig config; + config.layerConfig.set_type("crf"); + config.layerConfig.set_size(numClasses); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", + numClasses, numClasses * (numClasses + 2)}); + config.layerConfig.add_inputs(); + config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_label", + numClasses, 0}); + config.layerConfig.add_inputs(); + + if (withWeight) { + config.inputDefs.push_back({INPUT_DENSE_DIM_DATA, "layer_weight", + 1, 0}); + config.layerConfig.add_inputs(); + } + + return config; +} + +TEST(Layer, CRFLayer) { + size_t numClasses = 10; + for (int tries = 0; tries < 5; ++tries) { + TestConfig config = initTestConfig(numClasses, /* withWeight= */ false); + for (int length : {1, 3, 100}) { + // Not support GPU now + testLayerGrad(config, + "crf", + length, + /* trans= */ false, + /* useGpu= */ false, + /* useWeight= */ false, + epsilon()); + } + } +} + +TEST(Layer, CRFLayerUseWeight) { + size_t numClasses = 10; + for (int tries = 0; tries < 5; ++tries) { + TestConfig config = initTestConfig(numClasses, /* withWeight= */ true); + for (int length : {1, 3, 100}) { + // Not support GPU now + testLayerGrad(config, + "crf", + length, + /* trans= */ false, + /* useGpu= */ false, + /* useWeight= */ false, + epsilon()); + } + } +} + +int main(int argc, char** argv) { + initMain(argc, argv); + hl_start(); + hl_init(FLAGS_gpu_id); + FLAGS_thread_local_rand_use_global_seed = true; + srand(1); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 374ae57dd3..2aa86cc65c 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -270,27 +270,6 @@ TEST(Layer, AddtoLayer) { } } -TEST(Layer, CRFLayer) { - TestConfig config; - config.layerConfig.set_type("crf"); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 120}); - config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // Not support GPU now - testLayerGrad(config, - "crf", - 100, - /* trans */ false, - /* useGpu */ false, - false /*useWeight*/, - 0.03 /*epsilon*/); -} - TEST(Layer, CTCLayer) { TestConfig config; config.layerConfig.set_type("ctc"); diff --git a/paddle/gserver/tests/test_LinearChainCRF.cpp b/paddle/gserver/tests/test_LinearChainCRF.cpp index 913d6ed751..82361fa551 100644 --- a/paddle/gserver/tests/test_LinearChainCRF.cpp +++ b/paddle/gserver/tests/test_LinearChainCRF.cpp @@ -36,7 +36,7 @@ TEST(LinearChainCRF, decoding) { real* a = para.getData(); real* b = para.getData() + numClasses; real* w = para.getData() + 2 * numClasses; - LinearChainCRF crf(4, para.getData(), nullptr); + LinearChainCRF crf(4, para.getData()); for (int length : {1, 2, 3, 10}) { for (int tries = 0; tries < 10; ++tries) { CpuMatrix x(length, numClasses); From d60d34ef3f7fdcf54126e7a8342b50b35bae7f47 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Thu, 1 Dec 2016 13:38:42 +0800 Subject: [PATCH 02/62] Fix bug in python/paddle/trainer/config_parser.py The dims for crf and crf_decoding layers are wrong, i.e. height and width are swapped. --- python/paddle/trainer/config_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 9db42bf172..08b54bf0e1 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2960,7 +2960,7 @@ class CRFLayer(LayerBase): super(CRFLayer, self).__init__(name, 'crf', size, inputs, device=device) config_assert(2 <= len(self.inputs) <= 3, 'CRFLayer must have 2 or 3 inputs') - self.create_input_parameter(0, size * (size + 2), [size, size + 2]) + self.create_input_parameter(0, size * (size + 2), [size + 2, size]) self.config.coeff = coeff @@ -2982,7 +2982,7 @@ class CRFDecodingLayer(LayerBase): config_assert( len(self.inputs) <= 2, 'CRFDecodingLayer cannot have more than 2 inputs') - self.create_input_parameter(0, size * (size + 2), [size, size + 2]) + self.create_input_parameter(0, size * (size + 2), [size + 2, size]) @config_layer('ctc') From 749456bd600c623ed327420949bab677656ae1f8 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Thu, 1 Dec 2016 19:20:46 +0800 Subject: [PATCH 03/62] Fix test_layerHelpers bug The values for ____crf_layer_0__.w0 in python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr is not correct due to swapped height and width in config_parser --- .../tests/configs/protostr/test_cost_layers.protostr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr index f6045fe1f6..2c16ebcf81 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr @@ -225,9 +225,9 @@ parameters { name: "___crf_layer_0__.w0" size: 24 initial_mean: 0.0 - initial_std: 0.5 - dims: 4 + initial_std: 0.408248290464 dims: 6 + dims: 4 initial_strategy: 0 initial_smart: true } From 62eaa1fbabaf2ae751bc40d6be5c8cb3f7c2f291 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Thu, 1 Dec 2016 20:38:33 +0800 Subject: [PATCH 04/62] Move one line with identical value across loops out of the loop. --- paddle/gserver/layers/CRFLayer.cpp | 2 +- paddle/gserver/tests/test_CRFLayerGrad.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/layers/CRFLayer.cpp b/paddle/gserver/layers/CRFLayer.cpp index fa434cda00..b9fb7e54bd 100644 --- a/paddle/gserver/layers/CRFLayer.cpp +++ b/paddle/gserver/layers/CRFLayer.cpp @@ -91,8 +91,8 @@ void CRFLayer::backward(const UpdateCallback& callback) { const int* starts = label.sequenceStartPositions->getData(false); int numSequences = label.sequenceStartPositions->getSize() - 1; + bool needWGrad = weight_->getWGrad() ? true : false; for (int i = 0; i < numSequences; ++i) { - bool needWGrad = weight_->getWGrad() ? true : false; crfs_[i].backward(output.value->getData() + numClasses_ * starts[i], label.ids->getData() + starts[i], starts[i + 1] - starts[i], needWGrad); diff --git a/paddle/gserver/tests/test_CRFLayerGrad.cpp b/paddle/gserver/tests/test_CRFLayerGrad.cpp index bc1d5f3061..ad5149306c 100644 --- a/paddle/gserver/tests/test_CRFLayerGrad.cpp +++ b/paddle/gserver/tests/test_CRFLayerGrad.cpp @@ -108,7 +108,7 @@ TEST(CRFLayer, cost) { } inline real epsilon() { - return typeid(real) == typeid(double) ? 1e-10 : 0.05; + return typeid(real) == typeid(double) ? 1e-10 : 0.06; } TestConfig initTestConfig(size_t numClasses, bool withWeight) { From 35279398966bdd05fa56c5c4a017cc10f5224c99 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Wed, 21 Dec 2016 20:54:31 +0800 Subject: [PATCH 05/62] Fix merge and style error --- paddle/gserver/layers/CRFLayer.cpp | 15 ++++++----- paddle/gserver/layers/CRFLayer.h | 4 +-- paddle/gserver/tests/test_CRFLayerGrad.cpp | 30 ++++++++++------------ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/paddle/gserver/layers/CRFLayer.cpp b/paddle/gserver/layers/CRFLayer.cpp index 16cc738488..0b54442009 100644 --- a/paddle/gserver/layers/CRFLayer.cpp +++ b/paddle/gserver/layers/CRFLayer.cpp @@ -42,8 +42,7 @@ bool CRFLayer::init(const LayerMap& layerMap, CHECK_EQ(parameters_[0]->getSize(), numClasses_ * (numClasses_ + 2)); parameter_ = parameters_[0]; - weight_.reset( - new Weight(numClasses_ + 2, numClasses_, parameter_)); + weight_.reset(new Weight(numClasses_ + 2, numClasses_, parameter_)); // We don't need sequenceStartPositions because each sample of output_ is // for the cost of one sequence. @@ -95,16 +94,18 @@ void CRFLayer::backward(const UpdateCallback& callback) { for (int i = 0; i < numSequences; ++i) { crfs_[i].backward(output.value->getData() + numClasses_ * starts[i], label.ids->getData() + starts[i], - starts[i + 1] - starts[i], needWGrad); - real instanceWeight = weightLayer_ ? - getInputValue(*weightLayer_)->getElement(i, 0) : real(1.0f); + starts[i + 1] - starts[i], + needWGrad); + real instanceWeight = weightLayer_ + ? getInputValue(*weightLayer_)->getElement(i, 0) + : real(1.0f); instanceWeight *= coeff_; MatrixPtr grad = output.grad->subRowMatrix(starts[i], starts[i + 1]); grad->add(*crfs_[i].getXGrad(), real(1.0f), instanceWeight); if (needWGrad) { - weight_->getWGrad()->add(*crfs_[i].getWGrad(), real(1.0f), - instanceWeight); + weight_->getWGrad()->add( + *crfs_[i].getWGrad(), real(1.0f), instanceWeight); } } diff --git a/paddle/gserver/layers/CRFLayer.h b/paddle/gserver/layers/CRFLayer.h index 000c48e2d5..3c7192913f 100644 --- a/paddle/gserver/layers/CRFLayer.h +++ b/paddle/gserver/layers/CRFLayer.h @@ -37,9 +37,9 @@ protected: size_t numClasses_; ParameterPtr parameter_; std::vector crfs_; - LayerPtr weightLayer_; // weight for each sequence + LayerPtr weightLayer_; // weight for each sequence std::unique_ptr weight_; // parameters - real coeff_; // weight for the layer + real coeff_; // weight for the layer }; } // namespace paddle diff --git a/paddle/gserver/tests/test_CRFLayerGrad.cpp b/paddle/gserver/tests/test_CRFLayerGrad.cpp index ad5149306c..6985977aed 100644 --- a/paddle/gserver/tests/test_CRFLayerGrad.cpp +++ b/paddle/gserver/tests/test_CRFLayerGrad.cpp @@ -21,13 +21,10 @@ limitations under the License. */ #include "TestUtil.h" #include "LayerGradUtil.h" -using namespace paddle; // NOLINT +using namespace paddle; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_double(checkgrad_eps); -P_DECLARE_bool(thread_local_rand_use_global_seed); -P_DECLARE_bool(prev_batch_state); +DECLARE_int32(gpu_id); +DECLARE_bool(thread_local_rand_use_global_seed); static inline bool getNextSequence(std::vector& seq, int numClasses) { for (auto& v : seq) { @@ -96,8 +93,8 @@ TEST(CRFLayer, cost) { real diff = fabs(trueCost - cost); diff /= fabs(cost) < fabs(trueCost) ? fabs(cost) : fabs(trueCost); - VLOG(1) << "cost=" << cost << " trueCost=" << trueCost - << " diff=" << diff << std::endl; + VLOG(1) << "cost=" << cost << " trueCost=" << trueCost << " diff=" << diff + << std::endl; if (typeid(real) == typeid(double)) { // NOLINT EXPECT_LE(diff, 1e-10); } else { @@ -107,9 +104,7 @@ TEST(CRFLayer, cost) { } } -inline real epsilon() { - return typeid(real) == typeid(double) ? 1e-10 : 0.06; -} +inline real epsilon() { return typeid(real) == typeid(double) ? 1e-10 : 0.06; } TestConfig initTestConfig(size_t numClasses, bool withWeight) { TestConfig config; @@ -117,16 +112,17 @@ TestConfig initTestConfig(size_t numClasses, bool withWeight) { config.layerConfig.set_size(numClasses); config.biasSize = 0; - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", - numClasses, numClasses * (numClasses + 2)}); + config.inputDefs.push_back({INPUT_SEQUENCE_DATA, + "layer_0", + numClasses, + numClasses * (numClasses + 2)}); config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_label", - numClasses, 0}); + config.inputDefs.push_back( + {INPUT_SEQUENCE_LABEL, "layer_label", numClasses, 0}); config.layerConfig.add_inputs(); if (withWeight) { - config.inputDefs.push_back({INPUT_DENSE_DIM_DATA, "layer_weight", - 1, 0}); + config.inputDefs.push_back({INPUT_DENSE_DIM_DATA, "layer_weight", 1, 0}); config.layerConfig.add_inputs(); } From 5c3b8f6bd18bfadfa23e2a4eb4821bf383a3fd77 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Wed, 21 Dec 2016 22:01:15 +0800 Subject: [PATCH 06/62] fix pre-commit style problem --- paddle/gserver/tests/test_CRFLayerGrad.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/gserver/tests/test_CRFLayerGrad.cpp b/paddle/gserver/tests/test_CRFLayerGrad.cpp index 6985977aed..c267dcd9ab 100644 --- a/paddle/gserver/tests/test_CRFLayerGrad.cpp +++ b/paddle/gserver/tests/test_CRFLayerGrad.cpp @@ -13,13 +13,13 @@ See the License for the specific language governing permissions and limitations under the License. */ #include +#include "ModelConfig.pb.h" #include "paddle/gserver/layers/DataLayer.h" -#include "paddle/trainer/Trainer.h" #include "paddle/gserver/layers/LinearChainCRF.h" -#include "ModelConfig.pb.h" +#include "paddle/trainer/Trainer.h" -#include "TestUtil.h" #include "LayerGradUtil.h" +#include "TestUtil.h" using namespace paddle; // NOLINT From e022c065d7728bd768addf7d723e6527d87b9311 Mon Sep 17 00:00:00 2001 From: caoying03 Date: Wed, 28 Dec 2016 10:08:09 +0800 Subject: [PATCH 07/62] enable dropout in average and max layer. --- python/paddle/trainer/config_parser.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 2eb7b17a0b..8397659fff 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2305,9 +2305,10 @@ class MaxLayer(LayerBase): active_type='linear', device=None, bias=False, - output_max_index=None): + output_max_index=None, + **xargs): super(MaxLayer, self).__init__( - name, 'max', 0, inputs=inputs, device=device) + name, 'max', 0, inputs=inputs, device=device, **xargs) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') self.config.trans_type = trans_type self.config.active_type = active_type @@ -2609,14 +2610,16 @@ class AverageLayer(LayerBase): trans_type='non-seq', active_type='linear', device=None, - bias=False): + bias=False, + **xargs): super(AverageLayer, self).__init__( name, 'average', 0, inputs=inputs, device=device, - active_type=active_type) + active_type=active_type, + **xargs) self.config.average_strategy = average_strategy self.config.trans_type = trans_type config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') @@ -3490,7 +3493,7 @@ def parse_config(config_file, config_arg_str): def parse_config_and_serialize(config_file, config_arg_str): try: config = parse_config(config_file, config_arg_str) - #logger.info(config) + # logger.info(config) return config.SerializeToString() except: traceback.print_exc() From ce939b30ba3f6dc50ca49fab56b907990d7c1de2 Mon Sep 17 00:00:00 2001 From: caoying03 Date: Wed, 28 Dec 2016 15:21:43 +0800 Subject: [PATCH 08/62] enable dropout rate in several computation layers. --- python/paddle/trainer/config_parser.py | 74 +++++++++++++++----------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index bdd0e001fe..18cbd44f49 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1803,9 +1803,9 @@ class ConvTransLayer(ConvTransLayerBase): @config_layer('norm') class NormLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(NormLayer, self).__init__( - name, 'norm', 0, inputs=inputs, device=device) + name, 'norm', 0, inputs=inputs, device=device, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) norm_conf = self.config.inputs[input_index].norm_conf @@ -1817,9 +1817,9 @@ class NormLayer(LayerBase): @config_layer('pool') class PoolLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(PoolLayer, self).__init__( - name, 'pool', 0, inputs=inputs, device=device) + name, 'pool', 0, inputs=inputs, device=device, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) pool_conf = self.config.inputs[input_index].pool_conf @@ -1927,9 +1927,9 @@ class BatchNormLayer(LayerBase): @config_layer('trans') class TransLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(TransLayer, self).__init__( - name, 'trans', 0, inputs=inputs, device=device) + name, 'trans', 0, inputs=inputs, device=device, **xargs) config_assert( len(self.inputs) == 1, 'TransLayer must have one and only one input') @@ -1938,9 +1938,9 @@ class TransLayer(LayerBase): @config_layer('resize') class ResizeLayer(LayerBase): - def __init__(self, name, size, inputs, device=None): + def __init__(self, name, size, inputs, device=None, **xargs): super(ResizeLayer, self).__init__( - name, 'resize', size=size, inputs=inputs, device=device) + name, 'resize', size=size, inputs=inputs, device=device, **xargs) config_assert( len(self.inputs) == 1, 'ResizeLayer must have one and only one input') @@ -2270,9 +2270,10 @@ class ExpandLayer(LayerBase): inputs, trans_type='non-seq', device=None, - bias=False): + bias=False, + **xargs): super(ExpandLayer, self).__init__( - name, 'expand', 0, inputs=inputs, device=device) + name, 'expand', 0, inputs=inputs, device=device, **xargs) config_assert( len(self.inputs) == 2, 'ExpandLayer takes 2 and only 2 inputs') self.config.trans_type = trans_type @@ -2356,14 +2357,16 @@ class SequenceLastInstanceLayer(LayerBase): active_type='linear', trans_type='non-seq', device=None, - bias=False): + bias=False, + **xargs): super(SequenceLastInstanceLayer, self).__init__( name, 'seqlastins', 0, inputs=inputs, device=device, - active_type=active_type) + active_type=active_type, + **xargs) config_assert( len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') self.config.trans_type = trans_type @@ -2400,14 +2403,16 @@ class SequenceConcatLayer(LayerBase): inputs, active_type='linear', device=None, - bias=False): + bias=False, + **xargs): super(SequenceConcatLayer, self).__init__( name, 'seqconcat', 0, inputs=inputs, device=device, - active_type=active_type) + active_type=active_type, + **xargs) config_assert( len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') for input_index in xrange(len(self.inputs)): @@ -2424,14 +2429,16 @@ class SequenceReshapeLayer(LayerBase): inputs, active_type='linear', device=None, - bias=False): + bias=False, + **xargs): super(SequenceReshapeLayer, self).__init__( name, 'seqreshape', size, inputs=inputs, device=device, - active_type=active_type) + active_type=active_type, + **xargs) config_assert( len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') self.set_layer_size(size) @@ -2445,14 +2452,16 @@ class SubSequenceLayer(LayerBase): inputs, active_type='linear', device=None, - bias=False): + bias=False, + **xargs): super(SubSequenceLayer, self).__init__( name, 'subseq', 0, inputs=inputs, device=device, - active_type=active_type) + active_type=active_type, + **xargs) config_assert(len(inputs) == 3, 'SubSequenceLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) size = input_layer0.size @@ -2462,9 +2471,9 @@ class SubSequenceLayer(LayerBase): @config_layer('out_prod') class OuterProdLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(OuterProdLayer, self).__init__( - name, 'out_prod', 0, inputs=inputs, device=device) + name, 'out_prod', 0, inputs=inputs, device=device, **xargs) config_assert(len(inputs) == 2, 'OuterProdLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) @@ -2473,9 +2482,9 @@ class OuterProdLayer(LayerBase): @config_layer('power') class PowerLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(PowerLayer, self).__init__( - name, 'power', 0, inputs=inputs, device=device) + name, 'power', 0, inputs=inputs, device=device, **xargs) config_assert(len(inputs) == 2, 'PowerLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) @@ -2486,9 +2495,10 @@ class PowerLayer(LayerBase): @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__(self, name, inputs, slope=1.0, intercept=0.0, device=None): + def __init__(self, name, inputs, slope=1.0, intercept=0.0, + device=None, **xargs): super(SlopeInterceptLayer, self).__init__( - name, 'slope_intercept', 0, inputs=inputs, device=device) + name, 'slope_intercept', 0, inputs=inputs, device=device, **xargs) self.config.slope = slope self.config.intercept = intercept config_assert(len(inputs) == 1, 'SlopeInterceptLayer must have 1 input') @@ -2498,9 +2508,9 @@ class SlopeInterceptLayer(LayerBase): @config_layer('scaling') class ScalingLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(ScalingLayer, self).__init__( - name, 'scaling', 0, inputs=inputs, device=device) + name, 'scaling', 0, inputs=inputs, device=device, **xargs) config_assert(len(inputs) == 2, 'ScalingLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) @@ -2511,9 +2521,9 @@ class ScalingLayer(LayerBase): @config_layer('conv_shift') class ConvShiftLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(ConvShiftLayer, self).__init__( - name, 'conv_shift', 0, inputs=inputs, device=device) + name, 'conv_shift', 0, inputs=inputs, device=device, **xargs) config_assert(len(inputs) == 2, 'ConvShiftLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) @@ -2521,9 +2531,9 @@ class ConvShiftLayer(LayerBase): @config_layer('convex_comb') class ConvexCombinationLayer(LayerBase): - def __init__(self, name, size, inputs, device=None): + def __init__(self, name, size, inputs, device=None, **xargs): super(ConvexCombinationLayer, self).__init__( - name, 'convex_comb', size, inputs=inputs, device=device) + name, 'convex_comb', size, inputs=inputs, device=device, **xargs) config_assert( len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') config_assert( @@ -2562,9 +2572,9 @@ class BilinearInterpLayer(LayerBase): @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): - def __init__(self, name, inputs, device=None): + def __init__(self, name, inputs, device=None, **xargs): super(SumToOneNormLayer, self).__init__( - name, 'sum_to_one_norm', 0, inputs=inputs, device=device) + name, 'sum_to_one_norm', 0, inputs=inputs, device=device, **xargs) config_assert( len(self.inputs) == 1, 'SumToOneNormLayer must have 1 input') input_layer0 = self.get_input_layer(0) From 08a817e3fbb76c1d6dc003b14f8935eb54671558 Mon Sep 17 00:00:00 2001 From: caoying03 Date: Wed, 28 Dec 2016 16:12:02 +0800 Subject: [PATCH 09/62] delete unnecessary parameters and modifications for some mathmatical layers. --- python/paddle/trainer/config_parser.py | 127 +++++++++---------------- 1 file changed, 45 insertions(+), 82 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 18cbd44f49..e0a43f6b6e 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1803,9 +1803,8 @@ class ConvTransLayer(ConvTransLayerBase): @config_layer('norm') class NormLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): - super(NormLayer, self).__init__( - name, 'norm', 0, inputs=inputs, device=device, **xargs) + def __init__(self, name, inputs, **xargs): + super(NormLayer, self).__init__(name, 'norm', 0, inputs=inputs, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) norm_conf = self.config.inputs[input_index].norm_conf @@ -1817,9 +1816,8 @@ class NormLayer(LayerBase): @config_layer('pool') class PoolLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): - super(PoolLayer, self).__init__( - name, 'pool', 0, inputs=inputs, device=device, **xargs) + def __init__(self, name, inputs, **xargs): + super(PoolLayer, self).__init__(name, 'pool', 0, inputs=inputs, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) pool_conf = self.config.inputs[input_index].pool_conf @@ -1851,7 +1849,6 @@ class BatchNormLayer(LayerBase): inputs, active_type="linear", bias=True, - device=None, use_global_stats=True, moving_average_fraction=0.9, batch_norm_type=None, @@ -1893,7 +1890,6 @@ class BatchNormLayer(LayerBase): 0, active_type=active_type, inputs=inputs, - device=device, **xargs) if use_global_stats is not None: @@ -1927,9 +1923,9 @@ class BatchNormLayer(LayerBase): @config_layer('trans') class TransLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): + def __init__(self, name, inputs, **xargs): super(TransLayer, self).__init__( - name, 'trans', 0, inputs=inputs, device=device, **xargs) + name, 'trans', 0, inputs=inputs, **xargs) config_assert( len(self.inputs) == 1, 'TransLayer must have one and only one input') @@ -1938,9 +1934,9 @@ class TransLayer(LayerBase): @config_layer('resize') class ResizeLayer(LayerBase): - def __init__(self, name, size, inputs, device=None, **xargs): + def __init__(self, name, size, inputs, **xargs): super(ResizeLayer, self).__init__( - name, 'resize', size=size, inputs=inputs, device=device, **xargs) + name, 'resize', size=size, inputs=inputs, **xargs) config_assert( len(self.inputs) == 1, 'ResizeLayer must have one and only one input') @@ -2265,15 +2261,9 @@ def Generator( @config_layer('expand') class ExpandLayer(LayerBase): - def __init__(self, - name, - inputs, - trans_type='non-seq', - device=None, - bias=False, - **xargs): + def __init__(self, name, inputs, trans_type='non-seq', bias=False, **xargs): super(ExpandLayer, self).__init__( - name, 'expand', 0, inputs=inputs, device=device, **xargs) + name, 'expand', 0, inputs=inputs, **xargs) config_assert( len(self.inputs) == 2, 'ExpandLayer takes 2 and only 2 inputs') self.config.trans_type = trans_type @@ -2304,12 +2294,10 @@ class MaxLayer(LayerBase): inputs, trans_type='non-seq', active_type='linear', - device=None, bias=False, output_max_index=None, **xargs): - super(MaxLayer, self).__init__( - name, 'max', 0, inputs=inputs, device=device, **xargs) + super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, **xargs) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') self.config.trans_type = trans_type self.config.active_type = active_type @@ -2356,7 +2344,6 @@ class SequenceLastInstanceLayer(LayerBase): inputs, active_type='linear', trans_type='non-seq', - device=None, bias=False, **xargs): super(SequenceLastInstanceLayer, self).__init__( @@ -2364,7 +2351,6 @@ class SequenceLastInstanceLayer(LayerBase): 'seqlastins', 0, inputs=inputs, - device=device, active_type=active_type, **xargs) config_assert( @@ -2378,39 +2364,32 @@ class SequenceLastInstanceLayer(LayerBase): @config_layer('seqfirstins') class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): - def __init__( - self, - name, - inputs, - active_type='linear', - trans_type='non-seq', - device=None, - bias=False, ): + def __init__(self, + name, + inputs, + active_type='linear', + trans_type='non-seq', + bias=False, + **xargs): super(SequenceFirstInstanceLayer, self).__init__( name, inputs=inputs, active_type=active_type, device=device, - bias=bias) + bias=bias, + **xargs) self.config.trans_type = trans_type self.config.select_first = True @config_layer('seqconcat') class SequenceConcatLayer(LayerBase): - def __init__(self, - name, - inputs, - active_type='linear', - device=None, - bias=False, - **xargs): + def __init__(self, name, inputs, active_type='linear', bias=False, **xargs): super(SequenceConcatLayer, self).__init__( name, 'seqconcat', 0, inputs=inputs, - device=device, active_type=active_type, **xargs) config_assert( @@ -2428,7 +2407,6 @@ class SequenceReshapeLayer(LayerBase): size, inputs, active_type='linear', - device=None, bias=False, **xargs): super(SequenceReshapeLayer, self).__init__( @@ -2436,7 +2414,6 @@ class SequenceReshapeLayer(LayerBase): 'seqreshape', size, inputs=inputs, - device=device, active_type=active_type, **xargs) config_assert( @@ -2447,21 +2424,9 @@ class SequenceReshapeLayer(LayerBase): @config_layer('subseq') class SubSequenceLayer(LayerBase): - def __init__(self, - name, - inputs, - active_type='linear', - device=None, - bias=False, - **xargs): + def __init__(self, name, inputs, active_type='linear', bias=False, **xargs): super(SubSequenceLayer, self).__init__( - name, - 'subseq', - 0, - inputs=inputs, - device=device, - active_type=active_type, - **xargs) + name, 'subseq', 0, inputs=inputs, active_type=active_type, **xargs) config_assert(len(inputs) == 3, 'SubSequenceLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) size = input_layer0.size @@ -2471,9 +2436,9 @@ class SubSequenceLayer(LayerBase): @config_layer('out_prod') class OuterProdLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): + def __init__(self, name, inputs, device=None): super(OuterProdLayer, self).__init__( - name, 'out_prod', 0, inputs=inputs, device=device, **xargs) + name, 'out_prod', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'OuterProdLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) @@ -2482,9 +2447,9 @@ class OuterProdLayer(LayerBase): @config_layer('power') class PowerLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): + def __init__(self, name, inputs, device=None): super(PowerLayer, self).__init__( - name, 'power', 0, inputs=inputs, device=device, **xargs) + name, 'power', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'PowerLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) @@ -2495,8 +2460,13 @@ class PowerLayer(LayerBase): @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__(self, name, inputs, slope=1.0, intercept=0.0, - device=None, **xargs): + def __init__(self, + name, + inputs, + slope=1.0, + intercept=0.0, + device=None, + **xargs): super(SlopeInterceptLayer, self).__init__( name, 'slope_intercept', 0, inputs=inputs, device=device, **xargs) self.config.slope = slope @@ -2508,9 +2478,9 @@ class SlopeInterceptLayer(LayerBase): @config_layer('scaling') class ScalingLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): + def __init__(self, name, inputs, device=None): super(ScalingLayer, self).__init__( - name, 'scaling', 0, inputs=inputs, device=device, **xargs) + name, 'scaling', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ScalingLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) @@ -2521,9 +2491,9 @@ class ScalingLayer(LayerBase): @config_layer('conv_shift') class ConvShiftLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): + def __init__(self, name, inputs, device=None): super(ConvShiftLayer, self).__init__( - name, 'conv_shift', 0, inputs=inputs, device=device, **xargs) + name, 'conv_shift', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ConvShiftLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) @@ -2531,9 +2501,9 @@ class ConvShiftLayer(LayerBase): @config_layer('convex_comb') class ConvexCombinationLayer(LayerBase): - def __init__(self, name, size, inputs, device=None, **xargs): + def __init__(self, name, size, inputs, device=None): super(ConvexCombinationLayer, self).__init__( - name, 'convex_comb', size, inputs=inputs, device=device, **xargs) + name, 'convex_comb', size, inputs=inputs, device=device) config_assert( len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') config_assert( @@ -2572,9 +2542,9 @@ class BilinearInterpLayer(LayerBase): @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): - def __init__(self, name, inputs, device=None, **xargs): + def __init__(self, name, inputs, device=None): super(SumToOneNormLayer, self).__init__( - name, 'sum_to_one_norm', 0, inputs=inputs, device=device, **xargs) + name, 'sum_to_one_norm', 0, inputs=inputs, device=device) config_assert( len(self.inputs) == 1, 'SumToOneNormLayer must have 1 input') input_layer0 = self.get_input_layer(0) @@ -2619,17 +2589,10 @@ class AverageLayer(LayerBase): average_strategy='average', trans_type='non-seq', active_type='linear', - device=None, bias=False, **xargs): super(AverageLayer, self).__init__( - name, - 'average', - 0, - inputs=inputs, - device=device, - active_type=active_type, - **xargs) + name, 'average', 0, inputs=inputs, active_type=active_type, **xargs) self.config.average_strategy = average_strategy self.config.trans_type = trans_type config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') @@ -2653,9 +2616,9 @@ class CosSimLayer(LayerBase): @config_layer('tensor') class TensorLayer(LayerBase): - def __init__(self, name, size, inputs, device=None, bias=True, **xargs): + def __init__(self, name, size, inputs, bias=True, **xargs): super(TensorLayer, self).__init__( - name, 'tensor', size, inputs=inputs, device=device, **xargs) + name, 'tensor', size, inputs=inputs, **xargs) config_assert(len(self.inputs) == 2, 'TensorLayer must have 2 inputs') config_assert(size > 0, 'size must be positive') config_assert(inputs[1].parameter_name == None, From cbbad4202ae10e0ef9851861c77eab820bf6cce3 Mon Sep 17 00:00:00 2001 From: caoying03 Date: Wed, 28 Dec 2016 16:20:59 +0800 Subject: [PATCH 10/62] delete the modification of SlopeInterceptLayer. --- python/paddle/trainer/config_parser.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index e0a43f6b6e..b88853ea00 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2460,15 +2460,9 @@ class PowerLayer(LayerBase): @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__(self, - name, - inputs, - slope=1.0, - intercept=0.0, - device=None, - **xargs): + def __init__(self, name, inputs, slope=1.0, intercept=0.0, device=None): super(SlopeInterceptLayer, self).__init__( - name, 'slope_intercept', 0, inputs=inputs, device=device, **xargs) + name, 'slope_intercept', 0, inputs=inputs, device=device) self.config.slope = slope self.config.intercept = intercept config_assert(len(inputs) == 1, 'SlopeInterceptLayer must have 1 input') From 7d551dd4845ae71792d8c3b77a096c0d90314959 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Fri, 10 Feb 2017 16:37:59 -0800 Subject: [PATCH 11/62] Make it possible to postpone setting the layer name for a memory. The reason for adding the function is exemplified in the following hypothetical code: mem = memory(name=None, size=256) hidden = fc_layer(input=mem) state = hidden + x mem.set_input(state) In the above code segment, it would be very annoying if we require the user to provide the name at memory() call because the layer name of state is automatically generated and is not easy to set it. Change-Id: I918bf1d3d5c26addd88a6f7021e98b3e0e9df494 --- python/paddle/trainer/config_parser.py | 40 +++++-- .../default_decorators.py | 6 +- .../paddle/trainer_config_helpers/layers.py | 82 +++++++++++--- .../configs/protostr/test_rnn_group.protostr | 107 ++++++++++++++++++ .../tests/configs/test_rnn_group.py | 14 ++- 5 files changed, 216 insertions(+), 33 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index d403a6029a..575e110741 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2219,7 +2219,10 @@ def Link( # memory for recurrent layer group. # *name* and *size* are actual layer's name and size. -# will return name of the memory, +# If *name* is None, need to provide *memory_name* and need to use +# SetMemoryInput() later to specify the layer which this memory remembers. +# +# return the name of the memory, # use this name if you assign the memory as other layer's input # # boot frame of memory is zeroed by default, @@ -2231,15 +2234,18 @@ def Link( # can only be initailized by a *boot_layer* which is a sequence. # @config_func -def Memory( - name, - size, - is_sequence=False, - boot_layer=None, - boot_bias=False, - boot_bias_active_type="", - boot_with_const_id=None, ): - agent_name = name + "+delay1" +def Memory(name, + size, + is_sequence=False, + boot_layer=None, + boot_bias=False, + boot_bias_active_type="", + boot_with_const_id=None, + memory_name=None): + if not memory_name: + config_assert(name is not None, "name needs cannot be None") + memory_name = name + "+delay1" + agent_name = memory_name if is_sequence: agent_layer = SequenceAgentLayer(agent_name, size) else: @@ -2247,7 +2253,8 @@ def Memory( config_assert(g_current_submodel.is_recurrent_layer_group, 'Memory should be used in recurrent layer group only') memory = g_current_submodel.memories.add() - memory.layer_name = MakeLayerNameInSubmodel(name) + if name is not None: + memory.layer_name = MakeLayerNameInSubmodel(name) memory.link_name = MakeLayerNameInSubmodel(agent_name) memory.is_sequence = is_sequence options = sum((boot_layer is not None, bool(boot_bias), @@ -2271,6 +2278,17 @@ def Memory( return agent_name +@config_func +def SetMemoryInput(memory_name, layer_name): + memory_name = MakeLayerNameInSubmodel(memory_name) + layer_name = MakeLayerNameInSubmodel(layer_name) + for mem in g_current_submodel.memories: + if mem.link_name == memory_name: + mem.layer_name = layer_name + return + logger.fatal("Nonexistent memory name: " + memory_name) + + # Generator for recurrent layer group, to use it: # 1. define a id layer as output of layer group # 2. define a memory of this id layer, and assign a boot id(begin of sequence) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index ad3efcbf36..b7463a022a 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -93,13 +93,13 @@ def reset_hook(): register_parse_config_hook(reset_hook) -def wrap_name_default(name_prefix=None): +def wrap_name_default(name_prefix=None, name_param="name"): """ Decorator to set "name" arguments default to "{name_prefix}_{invoke_count}". .. code:: python - @default_name("some_name") + @wrap_name_default("some_name") def func(name=None): print name # name will never be None. If name is not set, # name will be "some_name_%d" @@ -111,7 +111,7 @@ def wrap_name_default(name_prefix=None): """ factory = DefaultNameFactory(name_prefix) _name_factories.append(factory) - return wrap_param_default(["name"], factory) + return wrap_param_default([name_param], factory) def wrap_param_attr_default(param_names=None, default_factory=None): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 1fdc4c4623..4087f3051e 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -280,6 +280,14 @@ class LayerOutput(object): """ assert False, "this method should not be invoked" + def set_input(self, input): + """ + Set the input for a memory layer. Can only be used for memory layer + """ + assert isinstance(input, LayerOutput) + assert self.layer_type == LayerType.MEMORY + SetMemoryInput(self.name, input.name) + ERROR_CLIPPING = 'error_clipping_threshold' DROPOUT = 'drop_rate' @@ -2570,8 +2578,10 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): size=sz) +@wrap_name_default("memory", "memory_name") def memory(name, size, + memory_name=None, is_seq=False, boot_layer=None, boot_bias=None, @@ -2593,14 +2603,32 @@ def memory(name, If boot_layer is not null, the memory is just the boot_layer's output. Set :code:`is_seq` is true boot layer is sequence. - The same name layer in recurrent group will set memory on each time step. - :param name: memory's name. + .. code-block:: python + + mem = memory(size=256, name='state') + state = fc_layer(input=mem, size=256, name='state') + + If you do not want to specify the name, you can equivalently use set_input() + to specify the layer needs to be remembered as the following: + + .. code-block:: python + mem = memory(size=256) + state = fc_layer(input=mem, size=256) + mem.set_input(mem) + + + :param name: the name of the layer which this memory remembers. + If name is None, user should call set_input() to specify the + name of the layer which this memory remembers. :type name: basestring :param size: size of memory. :type size: int + :param memory_name: the name of the memory. + It is ignored when name is provided. + :type memory_name: basestring :param is_seq: is sequence for boot_layer :type is_seq: bool :param boot_layer: boot layer of memory. @@ -2622,13 +2650,21 @@ def memory(name, boot_bias = ParamAttr.to_bias(boot_bias) assert boot_layer is None or isinstance(boot_layer, LayerOutput) + if name is not None: + memory_name = None - agent_name = Memory(name, size, is_seq, boot_layer.name - if boot_layer is not None else None, boot_bias, - boot_bias_active_type.name, boot_with_const_id) + memory_name = Memory( + name, + size, + is_sequence=is_seq, + boot_layer=boot_layer.name if boot_layer is not None else None, + boot_bias=boot_bias, + boot_bias_active_type=boot_bias_active_type.name, + boot_with_const_id=boot_with_const_id, + memory_name=memory_name) lout = LayerOutput( - name=agent_name, + name=memory_name, size=size, layer_type=LayerType.MEMORY, parents=[boot_layer] if boot_layer is not None else None) @@ -2754,8 +2790,8 @@ def gru_step_layer(input, :param name: :param gate_act: :param bias_attr: - :param param_attr: the parameter_attribute for transforming the output_mem - from previous step. + :param param_attr: the parameter_attribute for transforming the output_mem + from previous step. :param layer_attr: :return: LayerOutput object. :rtype: LayerOutput @@ -2766,10 +2802,10 @@ def gru_step_layer(input, Layer( name=name, type=LayerType.GRU_STEP_LAYER, - # The parameter here is for transforming the output_mem. The input has - # already been transformed outside this module so it does not need - # parameter associated with it. - # The parameter here is instead grouped with input is due to + # The parameter here is for transforming the output_mem. The input has + # already been transformed outside this module so it does not need + # parameter associated with it. + # The parameter here is instead grouped with input is due to # backward model compatibility. inputs=[Input(input.name, **param_attr.attr), output_mem.name], bias=ParamAttr.to_bias(bias_attr), @@ -3376,7 +3412,7 @@ def __cost_input__(input, label, weight=None): ipts = [Input(input.name), Input(label.name)] parents = [input, label] if weight is not None: - assert weight.layer_type == LayerType.DATA + assert weight.size == 1 ipts.append(Input(weight.name)) parents.append(weight) return ipts, parents @@ -4740,7 +4776,12 @@ def lambda_cost(input, @wrap_name_default() @layer_support() -def cross_entropy(input, label, name=None, coeff=1.0, layer_attr=None): +def cross_entropy(input, + label, + name=None, + coeff=1.0, + weight=None, + layer_attr=None): """ A loss layer for multi class entropy. @@ -4755,22 +4796,27 @@ def cross_entropy(input, label, name=None, coeff=1.0, layer_attr=None): :type input: LayerOutput. :param name: The name of this layers. It is not necessary. :type name: None|basestring. - :param coeff: The coefficient affects the gradient in the backward. + :param coeff: The cost is multiplied with coeff. + The coefficient affects the gradient in the backward. :type coeff: float. + :param weight: The cost of each sample is multiplied with each weight. + The weight should be a layer with size=1. Note that gradient + will not be calculated for weight. + :type weight: LayerOutout :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput. """ + ipts, parents = __cost_input__(input, label, weight) Layer( name=name, type=LayerType.CROSS_ENTROPY, - inputs=[input.name, label.name], + inputs=ipts, coeff=coeff, **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput( - name, LayerType.CROSS_ENTROPY, parents=[input, label], size=1) + return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=parents, size=1) @wrap_name_default() diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr index 3e9d28416e..a0fb729e06 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr @@ -331,6 +331,54 @@ layers { } trans_type: "non-seq" } +layers { + name: "__recurrent_group_3__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "seq_input@__recurrent_group_3__" + type: "scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "__memory_6__@__recurrent_group_3__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "__fc_layer_0__@__recurrent_group_3__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "seq_input@__recurrent_group_3__" + input_parameter_name: "___fc_layer_0__@__recurrent_group_3__.w0" + } + inputs { + input_layer_name: "__memory_6__@__recurrent_group_3__" + input_parameter_name: "___fc_layer_0__@__recurrent_group_3__.w1" + } + bias_parameter_name: "___fc_layer_0__@__recurrent_group_3__.wbias" +} +layers { + name: "__fc_layer_0__" + type: "gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__last_seq_4__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__fc_layer_0__" + } + trans_type: "non-seq" +} parameters { name: "___mixed_0__.w0" size: 40000 @@ -481,6 +529,36 @@ parameters { initial_strategy: 0 initial_smart: false } +parameters { + name: "___fc_layer_0__@__recurrent_group_3__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__@__recurrent_group_3__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__@__recurrent_group_3__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} input_layer_names: "seq_input" input_layer_names: "sub_seq_input" output_layer_names: "__last_seq_0__" @@ -488,6 +566,7 @@ output_layer_names: "__first_seq_0__" output_layer_names: "__last_seq_1__" output_layer_names: "__last_seq_2__" output_layer_names: "__last_seq_3__" +output_layer_names: "__last_seq_4__" sub_models { name: "root" layer_names: "seq_input" @@ -510,6 +589,9 @@ sub_models { layer_names: "__gru_group_0___recurrent_group" layer_names: "__gru_group_0__" layer_names: "__last_seq_3__" + layer_names: "__recurrent_group_3__" + layer_names: "__fc_layer_0__" + layer_names: "__last_seq_4__" input_layer_names: "seq_input" input_layer_names: "sub_seq_input" output_layer_names: "__last_seq_0__" @@ -517,6 +599,7 @@ sub_models { output_layer_names: "__last_seq_1__" output_layer_names: "__last_seq_2__" output_layer_names: "__last_seq_3__" + output_layer_names: "__last_seq_4__" is_recurrent_layer_group: false } sub_models { @@ -647,4 +730,28 @@ sub_models { } target_inlinkid: -1 } +sub_models { + name: "__recurrent_group_3__" + layer_names: "seq_input@__recurrent_group_3__" + layer_names: "__memory_6__@__recurrent_group_3__" + layer_names: "__fc_layer_0__@__recurrent_group_3__" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__fc_layer_0__@__recurrent_group_3__" + link_name: "__memory_6__@__recurrent_group_3__" + is_sequence: false + } + in_links { + layer_name: "seq_input" + link_name: "seq_input@__recurrent_group_3__" + has_subseq: false + } + out_links { + layer_name: "__fc_layer_0__@__recurrent_group_3__" + link_name: "__fc_layer_0__" + has_subseq: false + } + target_inlinkid: -1 +} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py index 60b4849d69..91010759e4 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py @@ -16,6 +16,16 @@ def generate_rnn_simple(name): return rnn_simple +def generate_rnn_simple_no_name(): + def rnn_simple(s): + m = memory(name=None, size=200) + fc = fc_layer(input=[s, m], size=200) + m.set_input(fc) + return fc + + return rnn_simple + + with mixed_layer() as lstm_param: # test lstm unit, rnn group lstm_param += full_matrix_projection(input=seq, size=100 * 4) @@ -33,4 +43,6 @@ outputs( last_seq(input=lstmemory_group( input=lstm_param, size=100)), last_seq(input=gru_group( - input=gru_param, size=100))) + input=gru_param, size=100)), + last_seq(input=recurrent_group( + step=generate_rnn_simple_no_name(), input=seq)), ) From 0f8cc82c8114e93ff4eefcc186aaab8be094d5f5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 9 Mar 2017 14:42:07 +0800 Subject: [PATCH 12/62] Fix ccache refernece. --- cmake/ccache.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index 968d41801d..900f59d4cb 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -1,9 +1,9 @@ # Use ccache if found ccache program -find_program(CCACHE_FOUND ccache) +find_program(CCACHE_PATH ccache) -if(CCACHE_FOUND) +if(CCACHE_PATH) message(STATUS "Ccache is founded, use ccache to speed up compile.") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) -endif(CCACHE_FOUND) \ No newline at end of file + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_PATH}) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_PATH}) +endif(CCACHE_PATH) From 2affe815a159d9b4ef44f0eb73003fcca8ac7f0a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 9 Mar 2017 15:02:40 +0800 Subject: [PATCH 13/62] Fix font issue in doc --- doc_theme/static/css/override.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc_theme/static/css/override.css b/doc_theme/static/css/override.css index 438a87848a..09ecff688b 100644 --- a/doc_theme/static/css/override.css +++ b/doc_theme/static/css/override.css @@ -1,3 +1,6 @@ +* { + font-family:"Roboto","Lato","proxima-nova","Helvetica Neue",Arial,sans-serif; +} body { padding-top: 80px; background-image: none !important; From 06c2437b3aa84de9fc31790cf15057ce1a44edaa Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 9 Mar 2017 15:47:44 +0800 Subject: [PATCH 14/62] Specify book develop branch --- .gitmodules | 1 + book | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index c614602cb8..832698b8bf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "book"] path = book url = https://github.com/PaddlePaddle/book.git + branch = develop \ No newline at end of file diff --git a/book b/book index 22ed2a01ae..6e3875eb62 160000 --- a/book +++ b/book @@ -1 +1 @@ -Subproject commit 22ed2a01aee872f055b5f5f212428f481cefc10d +Subproject commit 6e3875eb62533de1f2c1088a477719eb57b9732c From 70f380753e14ac7b46607d0ad6aa0e48064cce56 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 9 Mar 2017 16:30:01 +0800 Subject: [PATCH 15/62] modify api doc catalog --- doc/api/index_cn.rst | 23 +- doc/api/index_en.rst | 19 +- doc/api/v2/config/activation.rst | 101 +++++++ doc/api/v2/config/attr.rst | 6 + doc/api/v2/config/layer.rst | 487 +++++++++++++++++++++++++++++++ doc/api/v2/config/networks.rst | 117 ++++++++ doc/api/v2/config/optimizer.rst | 47 +++ doc/api/v2/config/pooling.rst | 46 +++ doc/api/v2/data.rst | 43 ++- doc/api/v2/model_configs.rst | 58 +--- doc/api/v2/run_logic.rst | 27 +- 11 files changed, 859 insertions(+), 115 deletions(-) create mode 100644 doc/api/v2/config/activation.rst create mode 100644 doc/api/v2/config/attr.rst create mode 100644 doc/api/v2/config/layer.rst create mode 100644 doc/api/v2/config/networks.rst create mode 100644 doc/api/v2/config/optimizer.rst create mode 100644 doc/api/v2/config/pooling.rst diff --git a/doc/api/index_cn.rst b/doc/api/index_cn.rst index fca981221e..9be0b370ee 100644 --- a/doc/api/index_cn.rst +++ b/doc/api/index_cn.rst @@ -1,26 +1,9 @@ API === -模型配置 API ------------- - .. toctree:: :maxdepth: 1 - v2/model_configs.rst - -数据 API --------- - -.. toctree:: - :maxdepth: 1 - - v2/data.rst - -训练 API --------- - -.. toctree:: - :maxdepth: 1 - - v2/run_logic.rst \ No newline at end of file + 模型配置 + 数据访问 + 训练与应用 diff --git a/doc/api/index_en.rst b/doc/api/index_en.rst index f0ad0fb2ae..25c1dd00b9 100644 --- a/doc/api/index_en.rst +++ b/doc/api/index_en.rst @@ -1,26 +1,9 @@ API === -Model Config API ----------------- - .. toctree:: :maxdepth: 1 v2/model_configs.rst - -Data API --------- - -.. toctree:: - :maxdepth: 1 - v2/data.rst - -Train API ---------- - -.. toctree:: - :maxdepth: 1 - - v2/run_logic.rst \ No newline at end of file + v2/run_logic.rst diff --git a/doc/api/v2/config/activation.rst b/doc/api/v2/config/activation.rst new file mode 100644 index 0000000000..eca3ce03bc --- /dev/null +++ b/doc/api/v2/config/activation.rst @@ -0,0 +1,101 @@ +=========== +Activation +=========== + +Abs +=== + +.. automodule:: paddle.v2.activation + :members: Abs + :noindex: + +Exp +=== + +.. automodule:: paddle.v2.activation + :members: Exp + :noindex: + +Identity +======== + +.. automodule:: paddle.v2.activation + :members: Identity + :noindex: + +Linear +====== + +.. automodule:: paddle.v2.activation + :members: Linear + :noindex: + +Log +=== + +.. automodule:: paddle.v2.activation + :members: Log + :noindex: + +Square +====== + +.. automodule:: paddle.v2.activation + :members: Square + :noindex: + +Sigmoid +======= + +.. automodule:: paddle.v2.activation + :members: Sigmoid + :noindex: + +Softmax +======= + +.. automodule:: paddle.v2.activation + :members: Softmax + :noindex: + +SequenceSoftmax +=============== + +.. automodule:: paddle.v2.activation + :members: SequenceSoftmax + :noindex: + +Relu +==== + +.. automodule:: paddle.v2.activation + :members: Relu + :noindex: + +BRelu +===== + +.. automodule:: paddle.v2.activation + :members: BRelu + :noindex: + +SoftRelu +======== + +.. automodule:: paddle.v2.activation + :members: SoftRelu + :noindex: + +Tanh +==== + +.. automodule:: paddle.v2.activation + :members: Tanh + :noindex: + +STanh +===== + +.. automodule:: paddle.v2.activation + :members: STanh + :noindex: diff --git a/doc/api/v2/config/attr.rst b/doc/api/v2/config/attr.rst new file mode 100644 index 0000000000..a93f41b867 --- /dev/null +++ b/doc/api/v2/config/attr.rst @@ -0,0 +1,6 @@ +Parameter Attribute +=================== + +.. automodule:: paddle.v2.attr + :members: + :noindex: diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst new file mode 100644 index 0000000000..db33a20487 --- /dev/null +++ b/doc/api/v2/config/layer.rst @@ -0,0 +1,487 @@ +.. _api_v2.layer: + +====== +Layers +====== + +Data layer +=========== + +.. _api_v2.layer_data: + +data +---- +.. automodule:: paddle.v2.layer + :members: data + :noindex: + +Fully Connected Layers +====================== + +.. _api_v2.layer_fc: + +fc +-- +.. automodule:: paddle.v2.layer + :members: fc + :noindex: + +selective_fc +------------ +.. automodule:: paddle.v2.layer + :members: selective_fc + :noindex: + +Conv Layers +=========== + +conv_operator +------------- +.. automodule:: paddle.v2.layer + :members: conv_operator + :noindex: + +conv_projection +--------------- +.. automodule:: paddle.v2.layer + :members: conv_projection + :noindex: + +conv_shift +---------- +.. automodule:: paddle.v2.layer + :members: conv_shift + :noindex: + +img_conv +-------- +.. automodule:: paddle.v2.layer + :members: img_conv + :noindex: + +.. _api_v2.layer_context_projection: + +context_projection +------------------ +.. automodule:: paddle.v2.layer + :members: context_projection + :noindex: + +Image Pooling Layer +=================== + +img_pool +-------- +.. automodule:: paddle.v2.layer + :members: img_pool + :noindex: + +spp +--- +.. automodule:: paddle.v2.layer + :members: spp + :noindex: + +maxout +------ +.. automodule:: paddle.v2.layer + :members: maxout + :noindex: + +Norm Layer +========== + +img_cmrnorm +----------- +.. automodule:: paddle.v2.layer + :members: img_cmrnorm + :noindex: + +batch_norm +---------- +.. automodule:: paddle.v2.layer + :members: batch_norm + :noindex: + +sum_to_one_norm +--------------- +.. automodule:: paddle.v2.layer + :members: sum_to_one_norm + :noindex: + +Recurrent Layers +================ + +recurrent +--------- +.. automodule:: paddle.v2.layer + :members: recurrent + :noindex: + +lstmemory +--------- +.. automodule:: paddle.v2.layer + :members: lstmemory + :noindex: + +grumemory +--------- +.. automodule:: paddle.v2.layer + :members: grumemory + :noindex: + +Recurrent Layer Group +===================== + +memory +------ +.. automodule:: paddle.v2.layer + :members: memory + :noindex: + +recurrent_group +--------------- +.. automodule:: paddle.v2.layer + :members: recurrent_group + :noindex: + +lstm_step +--------- +.. automodule:: paddle.v2.layer + :members: lstm_step + :noindex: + +gru_step +-------- +.. automodule:: paddle.v2.layer + :members: gru_step + :noindex: + +beam_search +------------ +.. automodule:: paddle.v2.layer + :members: beam_search + :noindex: + +get_output +---------- +.. automodule:: paddle.v2.layer + :members: get_output + :noindex: + +Mixed Layer +=========== + +.. _api_v2.layer_mixed: + +mixed +----- +.. automodule:: paddle.v2.layer + :members: mixed + :noindex: + +.. _api_v2.layer_embedding: + +embedding +--------- +.. automodule:: paddle.v2.layer + :members: embedding + :noindex: + +scaling_projection +------------------ +.. automodule:: paddle.v2.layer + :members: scaling_projection + :noindex: + +dotmul_projection +----------------- +.. automodule:: paddle.v2.layer + :members: dotmul_projection + :noindex: + +dotmul_operator +--------------- +.. automodule:: paddle.v2.layer + :members: dotmul_operator + :noindex: + +full_matrix_projection +---------------------- +.. automodule:: paddle.v2.layer + :members: full_matrix_projection + :noindex: + +identity_projection +------------------- +.. automodule:: paddle.v2.layer + :members: identity_projection + :noindex: + + +table_projection +---------------- +.. automodule:: paddle.v2.layer + :members: table_projection + :noindex: + +trans_full_matrix_projection +---------------------------- +.. automodule:: paddle.v2.layer + :members: trans_full_matrix_projection + :noindex: + +Aggregate Layers +================ + +.. _api_v2.layer_pooling: + +pooling +------- +.. automodule:: paddle.v2.layer + :members: pooling + :noindex: + +.. _api_v2.layer_last_seq: + +last_seq +-------- +.. automodule:: paddle.v2.layer + :members: last_seq + :noindex: + +.. _api_v2.layer_first_seq: + +first_seq +--------- +.. automodule:: paddle.v2.layer + :members: first_seq + :noindex: + +concat +------ +.. automodule:: paddle.v2.layer + :members: concat + :noindex: + +seq_concat +---------- +.. automodule:: paddle.v2.layer + :members: seq_concat + :noindex: + +Reshaping Layers +================ + +block_expand +------------ +.. automodule:: paddle.v2.layer + :members: block_expand + :noindex: + +.. _api_v2.layer_expand: + +expand +------ +.. automodule:: paddle.v2.layer + :members: expand + :noindex: + +repeat +------ +.. automodule:: paddle.v2.layer + :members: repeat + :noindex: + +rotate +------ +.. automodule:: paddle.v2.layer + :members: rotate + :noindex: + +seq_reshape +----------- +.. automodule:: paddle.v2.layer + :members: seq_reshape + :noindex: + +Math Layers +=========== + +addto +----- +.. automodule:: paddle.v2.layer + :members: addto + :noindex: + +linear_comb +----------- +.. automodule:: paddle.v2.layer + :members: linear_comb + :noindex: + +interpolation +------------- +.. automodule:: paddle.v2.layer + :members: interpolation + :noindex: + +bilinear_interp +--------------- +.. automodule:: paddle.v2.layer + :members: bilinear_interp + :noindex: + +power +----- +.. automodule:: paddle.v2.layer + :members: power + :noindex: + +scaling +------- +.. automodule:: paddle.v2.layer + :members: scaling + :noindex: + +slope_intercept +--------------- +.. automodule:: paddle.v2.layer + :members: slope_intercept + :noindex: + +tensor +------ +.. automodule:: paddle.v2.layer + :members: tensor + :noindex: + +.. _api_v2.layer_cos_sim: + +cos_sim +------- +.. automodule:: paddle.v2.layer + :members: cos_sim + :noindex: + +trans +----- +.. automodule:: paddle.v2.layer + :members: trans + :noindex: + +Sampling Layers +=============== + +maxid +----- +.. automodule:: paddle.v2.layer + :members: maxid + :noindex: + +sampling_id +----------- +.. automodule:: paddle.v2.layer + :members: sampling_id + :noindex: + +Slicing and Joining Layers +========================== + +pad +---- +.. automodule:: paddle.v2.layer + :members: pad + :noindex: + +.. _api_v2.layer_costs: + +Cost Layers +=========== + +cross_entropy_cost +------------------ +.. automodule:: paddle.v2.layer + :members: cross_entropy_cost + :noindex: + +cross_entropy_with_selfnorm_cost +-------------------------------- +.. automodule:: paddle.v2.layer + :members: cross_entropy_with_selfnorm_cost + :noindex: + +multi_binary_label_cross_entropy_cost +------------------------------------- +.. automodule:: paddle.v2.layer + :members: multi_binary_label_cross_entropy_cost + :noindex: + +huber_cost +---------- +.. automodule:: paddle.v2.layer + :members: huber_cost + :noindex: + +lambda_cost +----------- +.. automodule:: paddle.v2.layer + :members: lambda_cost + :noindex: + +rank_cost +--------- +.. automodule:: paddle.v2.layer + :members: rank_cost + :noindex: + +sum_cost +--------- +.. automodule:: paddle.v2.layer + :members: sum_cost + :noindex: + +crf +--- +.. automodule:: paddle.v2.layer + :members: crf + :noindex: + +crf_decoding +------------ +.. automodule:: paddle.v2.layer + :members: crf_decoding + :noindex: + +ctc +--- +.. automodule:: paddle.v2.layer + :members: ctc + :noindex: + +warp_ctc +-------- +.. automodule:: paddle.v2.layer + :members: warp_ctc + :noindex: + +nce +--- +.. automodule:: paddle.v2.layer + :members: nce + :noindex: + +hsigmoid +--------- +.. automodule:: paddle.v2.layer + :members: hsigmoid + :noindex: + +Check Layer +============ + +eos +--- +.. automodule:: paddle.v2.layer + :members: eos + :noindex: diff --git a/doc/api/v2/config/networks.rst b/doc/api/v2/config/networks.rst new file mode 100644 index 0000000000..6f209bc95b --- /dev/null +++ b/doc/api/v2/config/networks.rst @@ -0,0 +1,117 @@ +======== +Networks +======== + +The v2.networks module contains pieces of neural network that combine multiple layers. + +NLP +=== + +sequence_conv_pool +------------------ +.. automodule:: paddle.v2.networks + :members: sequence_conv_pool + :noindex: + +.. _api_trainer_config_helpers_network_text_conv_pool: + +text_conv_pool +-------------- +.. automodule:: paddle.v2.networks + :members: text_conv_pool + :noindex: + +Images +====== + +img_conv_bn_pool +---------------- +.. automodule:: paddle.v2.networks + :members: img_conv_bn_pool + :noindex: + +img_conv_group +-------------- +.. automodule:: paddle.v2.networks + :members: img_conv_group + :noindex: + +.. _api_trainer_config_helpers_network_simple_img_conv_pool: + +simple_img_conv_pool +-------------------- +.. automodule:: paddle.v2.networks + :members: simple_img_conv_pool + :noindex: + +vgg_16_network +--------------- +.. automodule:: paddle.v2.networks + :members: vgg_16_network + :noindex: + +Recurrent +========= + +LSTM +---- + +lstmemory_unit +`````````````` +.. automodule:: paddle.v2.networks + :members: lstmemory_unit + :noindex: + +lstmemory_group +``````````````` +.. automodule:: paddle.v2.networks + :members: lstmemory_group + :noindex: + +simple_lstm +``````````` +.. automodule:: paddle.v2.networks + :members: simple_lstm + :noindex: + +bidirectional_lstm +`````````````````` +.. automodule:: paddle.v2.networks + :members: bidirectional_lstm + :noindex: + +GRU +--- + +gru_unit +```````` +.. automodule:: paddle.v2.networks + :members: gru_unit + :noindex: + +gru_group +````````` +.. automodule:: paddle.v2.networks + :members: gru_group + :noindex: + +simple_gru +`````````` +.. automodule:: paddle.v2.networks + :members: simple_gru + :noindex: + +simple_attention +---------------- +.. automodule:: paddle.v2.networks + :members: simple_attention + :noindex: + +Miscs +===== + +dropout_layer +-------------- +.. automodule:: paddle.v2.networks + :members: dropout_layer + :noindex: diff --git a/doc/api/v2/config/optimizer.rst b/doc/api/v2/config/optimizer.rst new file mode 100644 index 0000000000..ec6ba0aa46 --- /dev/null +++ b/doc/api/v2/config/optimizer.rst @@ -0,0 +1,47 @@ +.. _api_v2.optimizer: + +========== +Optimizer +========== + +Momentum +======== +.. automodule:: paddle.v2.optimizer + :members: Momentum + :noindex: + +Adam +==== +.. automodule:: paddle.v2.optimizer + :members: Adam + :noindex: + +Adamax +====== +.. automodule:: paddle.v2.optimizer + :members: Adamax + :noindex: + +AdaGrad +======= +.. automodule:: paddle.v2.optimizer + :members: AdaGrad + :noindex: + +DecayedAdaGrad +============== +.. automodule:: paddle.v2.optimizer + :members: DecayedAdaGrad + :noindex: + +AdaDelta +======== +.. automodule:: paddle.v2.optimizer + :members: AdaDelta + :noindex: + +RMSProp +======= +.. automodule:: paddle.v2.optimizer + :members: RMSProp + :noindex: diff --git a/doc/api/v2/config/pooling.rst b/doc/api/v2/config/pooling.rst new file mode 100644 index 0000000000..d26b365c92 --- /dev/null +++ b/doc/api/v2/config/pooling.rst @@ -0,0 +1,46 @@ +======= +Pooling +======= + +BasePool +======== +.. automodule:: paddle.v2.pooling + :members: BasePool + :noindex: + +Avg +=== +.. automodule:: paddle.v2.pooling + :members: Avg + :noindex: + +Max +=== +.. automodule:: paddle.v2.pooling + :members: Max + :noindex: + +Sum +=== +.. automodule:: paddle.v2.pooling + :members: Sum + :noindex: + +SquareRootN +=========== +.. automodule:: paddle.v2.pooling + :members: SquareRootN + :noindex: + +CudnnAvg +======== +.. automodule:: paddle.v2.pooling + :members: CudnnAvg + :noindex: + +CudnnMax +======== +.. automodule:: paddle.v2.pooling + :members: CudnnMax + :noindex: + diff --git a/doc/api/v2/data.rst b/doc/api/v2/data.rst index 1c0a202a8c..b042320bc2 100644 --- a/doc/api/v2/data.rst +++ b/doc/api/v2/data.rst @@ -1,52 +1,53 @@ -================ -Data Related API -================ +======== +Datasets +======== -######### DataTypes -######### +========= .. automodule:: paddle.v2.data_type :members: + :noindex: -########## DataFeeder -########## +========== .. automodule:: paddle.v2.data_feeder :members: + :noindex: -###### Reader -###### +====== .. automodule:: paddle.v2.reader :members: + :noindex: .. automodule:: paddle.v2.reader.creator :members: + :noindex: -######### minibatch -######### +========= .. automodule:: paddle.v2.minibatch :members: + :noindex: -####### Dataset -####### +======= .. automodule:: paddle.v2.dataset :members: - + :noindex: mnist +++++ .. automodule:: paddle.v2.dataset.mnist :members: + :noindex: cifar @@ -54,40 +55,54 @@ cifar .. automodule:: paddle.v2.dataset.cifar :members: + :noindex: conll05 +++++++ .. automodule:: paddle.v2.dataset.conll05 :members: + :noindex: imdb ++++ .. automodule:: paddle.v2.dataset.imdb :members: + :noindex: imikolov ++++++++ .. automodule:: paddle.v2.dataset.imikolov :members: + :noindex: movielens +++++++++ .. automodule:: paddle.v2.dataset.movielens :members: + :noindex: sentiment +++++++++ .. automodule:: paddle.v2.dataset.sentiment :members: + :noindex: uci_housing +++++++++++ .. automodule:: paddle.v2.dataset.uci_housing :members: + :noindex: + +wmt14 ++++++ + +.. automodule:: paddle.v2.dataset.uci_housing + :members: + :noindex: diff --git a/doc/api/v2/model_configs.rst b/doc/api/v2/model_configs.rst index e9cd3d5bf7..a5fae7e29e 100644 --- a/doc/api/v2/model_configs.rst +++ b/doc/api/v2/model_configs.rst @@ -1,46 +1,12 @@ -######################### -Configuration Related API -######################### - -====== -Layers -====== - -.. automodule:: paddle.v2.layer - :members: - - -========== -Attributes -========== - -.. automodule:: paddle.v2.attr - :members: - -=========== -Activations -=========== - -.. automodule:: paddle.v2.activation - :members: - -======== -Poolings -======== - -.. automodule:: paddle.v2.pooling - :members: - -======== -Networks -======== - -.. automodule:: paddle.v2.networks - :members: - -========== -Optimizers -========== - -.. automodule:: paddle.v2.optimizer - :members: +Model Configuration +=================== + +.. toctree:: + :maxdepth: 1 + + config/activation.rst + config/layer.rst + config/optimizer.rst + config/pooling.rst + config/networks.rst + config/attr.rst diff --git a/doc/api/v2/run_logic.rst b/doc/api/v2/run_logic.rst index 0f807873ff..94921e1a7b 100644 --- a/doc/api/v2/run_logic.rst +++ b/doc/api/v2/run_logic.rst @@ -1,34 +1,27 @@ -########### -Trainer API -########### +====================== +Training and Inference +====================== - -========== Parameters ========== .. automodule:: paddle.v2.parameters - :members: + :noindex: - -======= Trainer ======= -.. automodule:: paddle.v2.trainer - :members: +.. automodule:: paddle.v2.trainer + :noindex: - -===== Event ===== -.. automodule:: paddle.v2.event - :members: +.. automodule:: paddle.v2.event + :noindex: - -========= Inference ========= -.. autofunction:: paddle.v2.infer \ No newline at end of file +.. autofunction:: paddle.v2.infer + :noindex: From 7ff9764fc093466f64e6d6830c547e6d6af1c92c Mon Sep 17 00:00:00 2001 From: gaoyuan Date: Thu, 9 Mar 2017 17:19:44 +0800 Subject: [PATCH 16/62] smooth_l1_loss_layer --- paddle/gserver/layers/CostLayer.cpp | 59 +++++++++++++++++++++++++ paddle/gserver/layers/CostLayer.h | 23 ++++++++++ paddle/gserver/tests/test_LayerGrad.cpp | 14 ++++++ paddle/math/Matrix.cpp | 49 ++++++++++++++++++++ paddle/math/Matrix.h | 11 +++++ 5 files changed, 156 insertions(+) diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 998b8d7d30..e2a4153bad 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -192,6 +192,65 @@ void SumOfSquaresCostLayer::backwardImp(Matrix& output, outputG.sumOfSquaresBp(output, *label.value); } +// +// class SmoothL1CostLayer +// + +REGISTER_LAYER(smooth_l1, SmoothL1CostLayer); + +bool SmoothL1CostLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + return CostLayer::init(layerMap, parameterMap); +} + +void SmoothL1CostLayer::forwardImp(Matrix& output, + Argument& label, + Matrix& target) { + MatrixPtr targetCpu, labelCpu, outputCpu; + if (useGpu_) { + Matrix::resizeOrCreate( + targetCpu, target.getHeight(), target.getWidth(), false, false); + Matrix::resizeOrCreate( + outputCpu, output.getHeight(), output.getWidth(), false, false); + Matrix::resizeOrCreate(labelCpu, + label.value->getHeight(), + label.value->getWidth(), + false, + false); + targetCpu->copyFrom(target); + outputCpu->copyFrom(output); + labelCpu->copyFrom(*label.value); + targetCpu->smoothL1(*outputCpu, *(labelCpu)); + target.copyFrom(*targetCpu); + } else { + target.smoothL1(output, *label.value); + } +} + +void SmoothL1CostLayer::backwardImp(Matrix& output, + Argument& label, + Matrix& outputG) { + MatrixPtr outputGCpu, labelCpu, outputCpu; + if (useGpu_) { + Matrix::resizeOrCreate( + outputGCpu, outputG.getHeight(), outputG.getWidth(), false, false); + Matrix::resizeOrCreate( + outputCpu, output.getHeight(), output.getWidth(), false, false); + Matrix::resizeOrCreate(labelCpu, + label.value->getHeight(), + label.value->getWidth(), + false, + false); + outputGCpu->copyFrom(outputG); + outputCpu->copyFrom(output); + labelCpu->copyFrom(*label.value); + outputGCpu->smoothL1Bp(*outputCpu, *labelCpu); + outputG.copyFrom(*outputGCpu); + } else { + outputG.smoothL1Bp(output, *label.value); + } +} + // // class RankingCost // diff --git a/paddle/gserver/layers/CostLayer.h b/paddle/gserver/layers/CostLayer.h index b3045e0b31..2f85fd3eca 100644 --- a/paddle/gserver/layers/CostLayer.h +++ b/paddle/gserver/layers/CostLayer.h @@ -159,6 +159,29 @@ public: Matrix& outputGrad) override; }; +/** + * This cost layer compute smooth L1 loss for real-valued regression + * tasks. + * \f[ + * L = + * y / -1 < y < 1 / + * sign(y) / otherwise / + * \f] + */ +class SmoothL1CostLayer : public CostLayer { +public: + explicit SmoothL1CostLayer(const LayerConfig& config) : CostLayer(config) {} + + bool init(const LayerMap& layerMap, + const ParameterMap& parameterMap) override; + + void forwardImp(Matrix& output, Argument& label, Matrix& cost) override; + + void backwardImp(Matrix& outputValue, + Argument& label, + Matrix& outputGrad) override; +}; + /** * A cost layer for learning to rank (LTR) task. This layer contains at leat * three inputs. diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 14d9db5247..28a0609733 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -1623,6 +1623,20 @@ TEST(Layer, PadLayer) { } } +TEST(Layer, smooth_l1) { + TestConfig config; + config.layerConfig.set_type("smooth_l1"); + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); + config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 1, 0}); + config.layerConfig.add_inputs(); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "smooth_l1", 100, false, useGpu, false, 2.0); + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 07450bfb0e..9eead5b62c 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -3590,6 +3590,55 @@ void CpuMatrix::sumOfSquaresBp(Matrix& output, Matrix& label) { } } +void CpuMatrix::smoothL1(Matrix& output, Matrix& label) { + CHECK(output.useGpu_ == false && label.useGpu_ == false) + << "Matrix type are not equal"; + + size_t numSamples = getHeight(); + size_t dim = output.getWidth(); + CHECK_EQ(label.getHeight(), numSamples); + CHECK_EQ(output.getHeight(), numSamples); + CHECK_EQ(label.getWidth(), dim); + CHECK_EQ(getWidth(), (size_t)1); + real* out = output.getData(); + real* cost = getData(); + real* lbl = label.getData(); + + for (size_t i = 0; i < numSamples; ++i, out += dim, cost += dim, lbl += dim) { + for (size_t j = 0; j < dim; ++j) { + cost[j] = std::fabs(out[j] - lbl[j]); + if (cost[j] < 1.0) + cost[j] = 0.5 * cost[j] * cost[j]; + else + cost[j] = cost[j] - 0.5; + } + } +} + +void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) { + CHECK(output.useGpu_ == false && label.useGpu_ == false) + << "Matrix type are not equal"; + + size_t numSamples = getHeight(); + size_t dim = output.getWidth(); + CHECK_EQ(label.getHeight(), numSamples); + CHECK_EQ(output.getHeight(), numSamples); + CHECK_EQ(label.getWidth(), dim); + CHECK_EQ(getWidth(), (size_t)1); + real* out = output.getData(); + real* cost = getData(); + real* lbl = label.getData(); + + // f'(x) = x if |x| < 1 + // = sign(x) otherwise + for (size_t i = 0; i < numSamples; ++i, out += dim, cost += dim, lbl += dim) { + for (size_t j = 0; j < dim; ++j) { + cost[j] = out[j] - lbl[j]; + if (std::fabs(cost[j]) >= 1) cost[j] = (0 < cost[j]) - (cost[j] < 0); + } + } +} + void CpuMatrix::tanh(Matrix& output) { CHECK(isContiguous()); CHECK(output.isContiguous()); diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index d0ba2e93fe..dbdb629614 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -783,6 +783,14 @@ public: LOG(FATAL) << "Not implemented"; } + virtual void smoothL1(Matrix& output, Matrix& label) { + LOG(FATAL) << "Not implemented"; + } + + virtual void smoothL1Bp(Matrix& outputV, Matrix& label) { + LOG(FATAL) << "Not implemented"; + } + virtual void tanh(Matrix& output) { LOG(FATAL) << "Not implemented"; } virtual void tanhDerivative(Matrix& output) { @@ -1720,6 +1728,9 @@ public: /// gradient of sumOfSquares. void sumOfSquaresBp(Matrix& outputV, Matrix& label); + void smoothL1(Matrix& output, Matrix& label); + void smoothL1Bp(Matrix& output, Matrix& label); + void tanh(Matrix& output); void tanhDerivative(Matrix& output); From 58d5be1ab663de2cc53acae4e518d0e233f3eaaa Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 9 Mar 2017 09:09:30 -0800 Subject: [PATCH 17/62] update docker tag, add translate for jupyter notebook --- .../build_and_install/docker_install_cn.rst | 32 +++++++++++--- .../build_and_install/docker_install_en.rst | 42 +++++++++---------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/doc/getstarted/build_and_install/docker_install_cn.rst b/doc/getstarted/build_and_install/docker_install_cn.rst index 78f518cfe4..cf7dddd073 100644 --- a/doc/getstarted/build_and_install/docker_install_cn.rst +++ b/doc/getstarted/build_and_install/docker_install_cn.rst @@ -56,6 +56,26 @@ PaddlePaddle目前唯一官方支持的运行的方式是Docker容器。因为Do cd /paddle/build ctest +4. 在Docker容器中运行PaddlePaddle书籍 + + Jupyter Notebook是一个开源的web程序，大家可以通过它制作和分享带有代码、公式、图表、文字的交互式文档。用户可以通过网页浏览文档。 + + PaddlePaddle书籍是为用户和开发者制作的一个交互式的Jupyter Nodebook。 + 如果您想要更深入了解deep learning，PaddlePaddle书籍一定是您最好的选择。 + + 当您进入容器内之后，只用运行以下命令： + + .. code-block:: bash + + jupyter notebook + + 然后在浏览器中输入以下网址： + + .. code-block:: text + + http://localhost:8888/ + + 就这么简单，享受您的旅程！ 纯CPU和GPU的docker镜像 ---------------------- @@ -64,20 +84,20 @@ PaddlePaddle目前唯一官方支持的运行的方式是Docker容器。因为Do .. code-block:: bash - docker build -t paddle:cpu -f paddle/scripts/docker/Dockerfile . - docker build -t paddle:gpu -f paddle/scripts/docker/Dockerfile.gpu . + docker build -t paddle:cpu -f paddle/scripts/docker/Dockerfile --build-arg BUILD_AND_INSTALL=ON . + docker build -t paddle:gpu -f paddle/scripts/docker/Dockerfile.gpu --build-arg BUILD_AND_INSTALL=ON . 以交互容器方式运行纯CPU的镜像： .. code-block:: bash - docker run -it --rm paddledev/paddle:cpu-latest /bin/bash + docker run -it --rm paddledev/paddle:0.10.0rc1-cpu /bin/bash 或者，可以以后台进程方式运行容器： .. code-block:: bash - docker run -d -p 2202:22 paddledev/paddle:cpu-latest + docker run -d -p 2202:22 paddledev/paddle:0.10.0rc1-cpu 然后用密码 :code:`root` SSH进入容器： @@ -94,7 +114,7 @@ SSH方式的一个优点是我们可以从多个终端进入容器。比如， export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:gpu-latest + docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:0.10.0rc1-gpu 非AVX镜像 @@ -128,7 +148,7 @@ Paddle的Docker镜像带有一个通过 `woboq code browser .. code-block:: bash - docker run -d --name paddle-cpu-doc paddle:cpu + docker run -d --name paddle-cpu-doc paddle:0.10.0rc1-cpu docker run -d --volumes-from paddle-cpu-doc -p 8088:80 nginx 接着我们就能够打开浏览器在 http://localhost:8088/paddle/ 浏览代码。 diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index a92201c618..a4f62b2835 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -84,27 +84,27 @@ Windows -- in a consistent way. 4. Run PaddlePaddle Book under Docker Container - The Jupyter Notebook is an open-source web application that allows - you to create and share documents that contain live code, equations, - visualizations and explanatory text in a single browser. + The Jupyter Notebook is an open-source web application that allows + you to create and share documents that contain live code, equations, + visualizations and explanatory text in a single browser. - PaddlePaddle Book is an interactive Jupyter Notebook for users and developers. - We already exposed port 8888 for this book. If you want to - dig deeper into deep learning, PaddlePaddle Book definitely is your best choice. + PaddlePaddle Book is an interactive Jupyter Notebook for users and developers. + We already exposed port 8888 for this book. If you want to + dig deeper into deep learning, PaddlePaddle Book definitely is your best choice. - Once you are inside the container, simply issue the command: + Once you are inside the container, simply issue the command: - .. code-block:: bash - - jupyter notebook - - Then, you would back and paste the address into the local browser: + .. code-block:: bash + + jupyter notebook - .. code-block:: text + Then, you would back and paste the address into the local browser: + + .. code-block:: text - http://localhost:8888/ + http://localhost:8888/ - That's all. Enjoy your journey! + That's all. Enjoy your journey! CPU-only and GPU Images ----------------------- @@ -116,21 +116,21 @@ automatically runs the following commands: .. code-block:: bash - docker build -t paddle:cpu -f paddle/scripts/docker/Dockerfile . - docker build -t paddle:gpu -f paddle/scripts/docker/Dockerfile.gpu . + docker build -t paddle:cpu -f paddle/scripts/docker/Dockerfile --build-arg BUILD_AND_INSTALL=ON . + docker build -t paddle:gpu -f paddle/scripts/docker/Dockerfile.gpu --build-arg BUILD_AND_INSTALL=ON . To run the CPU-only image as an interactive container: .. code-block:: bash - docker run -it --rm paddledev/paddle:cpu-latest /bin/bash + docker run -it --rm paddledev/paddle:0.10.0rc1-cpu /bin/bash or, we can run it as a daemon container .. code-block:: bash - docker run -d -p 2202:22 paddledev/paddle:cpu-latest + docker run -d -p 2202:22 paddledev/paddle:0.10.0rc1-cpu and SSH to this container using password :code:`root`: @@ -152,7 +152,7 @@ to install CUDA driver and let Docker knows about it: export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:gpu-latest + docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:0.10.0rc1-gpu Non-AVX Images @@ -194,7 +194,7 @@ container: .. code-block:: bash - docker run -d --name paddle-cpu-doc paddle:cpu + docker run -d --name paddle-cpu-doc paddle:0.10.0rc1-cpu docker run -d --volumes-from paddle-cpu-doc -p 8088:80 nginx From 00f88d44495f9529bdf77b06cd2cef33d64cb63b Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 9 Mar 2017 10:31:33 -0800 Subject: [PATCH 18/62] reduce docker image size by removing /build in or docker build --- paddle/scripts/docker/Dockerfile | 1 + paddle/scripts/docker/Dockerfile.gpu | 1 + paddle/scripts/docker/build.sh | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/paddle/scripts/docker/Dockerfile b/paddle/scripts/docker/Dockerfile index 6435923c89..48af9e5b5f 100644 --- a/paddle/scripts/docker/Dockerfile +++ b/paddle/scripts/docker/Dockerfile @@ -18,6 +18,7 @@ ENV WITH_GPU=OFF ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} +ENV DOCKER_BUILD=TRUE ENV HOME /root diff --git a/paddle/scripts/docker/Dockerfile.gpu b/paddle/scripts/docker/Dockerfile.gpu index 06e53a0ef3..a687d490a3 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/paddle/scripts/docker/Dockerfile.gpu @@ -18,6 +18,7 @@ ENV WITH_GPU=ON ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} +ENV DOCKER_BUILD=TRUE ENV HOME /root diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index b2e6416c3d..668b6e6b84 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -57,6 +57,12 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl pip install /usr/local/opt/paddle/share/wheels/paddle*.whl paddle version + + if [[ ${DOCKER_BUILD:-FALSE} == 'TRUE' ]]; then + # reduce docker image size + rm -rf /paddle/build + rm -rf /usr/local/opt/paddle/share/wheels/ + fi fi trap : 0 From a6eec0228c9176095b6112e314763edd95a254a0 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 10 Mar 2017 17:09:19 +0800 Subject: [PATCH 19/62] thinner docker --- paddle/scripts/docker/README.md | 9 +++ paddle/scripts/docker/buildall.sh | 27 +++++++++ .../docker/buildimage/Dockerfile.build | 59 +++++++++++++++++++ paddle/scripts/docker/paddle-core/Dockerfile | 38 ++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 paddle/scripts/docker/README.md create mode 100644 paddle/scripts/docker/buildall.sh create mode 100644 paddle/scripts/docker/buildimage/Dockerfile.build create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md new file mode 100644 index 0000000000..167b6be8ee --- /dev/null +++ b/paddle/scripts/docker/README.md @@ -0,0 +1,9 @@ +# Build docker image + +We use a docker environment to build paddle binaries and put it into a runtime image `paddle-core` for uses of most cases + +***Notice***: do **not** run in this directory, run under the top level of this project like: + +``` +sh paddle/scripts/docker/buildall.sh +``` diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh new file mode 100644 index 0000000000..2d7de80965 --- /dev/null +++ b/paddle/scripts/docker/buildall.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +BINARIES_DIR=paddle/scripts/docker/buildimage/binaries + +function build_in_docker() { + docker build . -t paddle-build-env -f paddle/scripts/docker/buildimage/Dockerfile.build + BUILDER=$(docker run -d paddle-build-env) + docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh" + mkdir -p $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR + docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR + docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR + + docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR + + docker stop $BUILDER && docker rm $BUILDER +} + +function build_paddle_core() { + docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile + +} + +build_in_docker +build_paddle_core diff --git a/paddle/scripts/docker/buildimage/Dockerfile.build b/paddle/scripts/docker/buildimage/Dockerfile.build new file mode 100644 index 0000000000..03e6ac792f --- /dev/null +++ b/paddle/scripts/docker/buildimage/Dockerfile.build @@ -0,0 +1,59 @@ +# A image for building paddle binaries +FROM ubuntu:14.04 +MAINTAINER PaddlePaddle Authors + +ARG DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR +RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' + +# ENV variables +ARG BUILD_WOBOQ +ARG BUILD_AND_INSTALL +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} +ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y git python-pip python-dev openssh-server bison && \ + apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ + apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ + apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ + apt-get install -y automake locales clang-format-3.8 && \ + apt-get clean -y + +# git credential to skip password typing +RUN git config --global credential.helper store + +# Fix locales to en_US.UTF-8 +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 + +RUN pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' && \ + pip install -U wheel pillow BeautifulSoup && \ + pip install -U docopt PyYAML sphinx && \ + pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ + pip install -U pre-commit 'requests==2.9.2' jupyter + +RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ + cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ + cd .. && rm -rf cmake-3.4.1 + +COPY . /paddle/ +RUN cd /paddle/ && git submodule update --init --recursive +RUN /paddle/paddle/scripts/docker/build.sh + +VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] + + +# FIXME: wait a long time is OK +CMD ["sleep", "3600"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile new file mode 100644 index 0000000000..708e4fe801 --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile @@ -0,0 +1,38 @@ +FROM python:2.7.13-slim +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy + +ADD paddle/scripts/docker/buildimage/binaries/paddle paddle/scripts/docker/buildimage/binaries/paddle_trainer \ +paddle/scripts/docker/buildimage/binaries/paddle_pserver_main paddle/scripts/docker/buildimage/binaries/paddle_merge_model \ +paddle/scripts/docker/buildimage/binaries/paddle_usage /usr/local/opt/paddle/bin/ +ADD paddle/scripts/docker/buildimage/binaries/wheels/py_paddle*linux*.whl paddle/scripts/docker/buildimage/binaries/wheels/paddle*.whl \ + /usr/local/opt/paddle/share/wheels/ +RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ + pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \ + rm /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ + rm /usr/local/opt/paddle/share/wheels/paddle*.whl +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] From b07d95d3fb52067d301a20999ffa9872684eb6d5 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Sun, 12 Mar 2017 21:57:03 +0800 Subject: [PATCH 20/62] add README.md --- paddle/scripts/docker/README.md | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 paddle/scripts/docker/README.md diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md new file mode 100644 index 0000000000..8d2df44cd0 --- /dev/null +++ b/paddle/scripts/docker/README.md @@ -0,0 +1,44 @@ +因为我们不提供非Ubuntu的bulid支持，所以如果用户用其他操作系统，比如CoreOS、CentOS、MacOS X、Windows，开发都得在docker里。所以需要能build本地修改后的代码。 + +我们可能需要两个 Docker images： + +1. development image：不包括源码，但是包括开发环境（预先安装好各种工具），也就是说Dockerfile.dev里既不需要 COPY 也不需要 RUN git clone。虽然这个image和源码无关，但是不同版本的源码需要依赖不同的第三方库，所以这个image的tag里还是要包含git branch/tag name，比如叫做 `paddlepaddle/paddle:dev-0.10.0rc1`，这里的0.10.0.rc1是一个branch name，其中rc是release candidate的意思。正是发布之后就成了master branch里的一个tag，叫做0.10.0。 + +1. production image： 不包括编译环境，也不包括源码，只包括build好的libpaddle.so和必要的Python packages，用于在Kubernetes机群上跑应用的image。比如叫做 `paddlepaddle/paddle:0.10.0rc1`。 + +从1.生成2.的过程如下： + +1. 在本机（host）上开发。假设源码位于 `~/work/paddle`。 + +1. 用dev image build 我们的源码： + ```bash + docker run -it -p 2022:22 -v $PWD:/paddle paddlepaddle/paddle:dev-0.10.0rc1 /paddle/build.sh + ``` + 注意，这里的 `-v ` 参数把host上的源码目录里的内容映射到了container里的`/paddle` 目录；而container里的 `/paddle/build.sh` 就是源码目录里的 `build.sh`。上述命令调用了本地源码中的 bulid.sh 来build了本地源码，结果在container里的 `/paddle/build` 目录里，也就是本地的源码目录里的 `build` 子目录。 + +1. 我们希望上述 `build.sh` 脚本在 `build` 子目录里生成一个Dockerfile，使得我们可以运行： + ```bash + docker build -t paddle ./build + ``` + 来生成我们的production image。 + +1. 有了这个production image之后，我们可能会希望docker push 到dockerhub.com的我们自己的名下，然后可以用来启动本地或者远程（Kubernetes）jobs： + + ```bash + docker tag paddle yiwang/paddle:did-some-change + docker push + paddlectl run yiwang/paddle:did-some-change /paddle/demo/mnist/train.py + ``` + + 其中 paddlectl 应该是我们自己写的一个脚本，调用kubectl来在Kubernetes机群上启动一个job的。 + +同时: + +1. 我们希望最好可以在docker hub上自动跑起来。如果不行，需要把我们的CI服务器设置一下来自动打包。 + + +曾经的讨论背景： +["PR 1599"](https://github.com/PaddlePaddle/Paddle/pull/1599) +["PR 1598"](https://github.com/PaddlePaddle/Paddle/pull/1598) + + From 824b193fecc5a48cd43c8cf8d49254f46d5fb957 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Sun, 12 Mar 2017 22:00:01 +0800 Subject: [PATCH 21/62] add READM.md --- paddle/scripts/docker/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 8d2df44cd0..80a1177c6d 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -32,9 +32,7 @@ 其中 paddlectl 应该是我们自己写的一个脚本，调用kubectl来在Kubernetes机群上启动一个job的。 -同时: -1. 我们希望最好可以在docker hub上自动跑起来。如果不行，需要把我们的CI服务器设置一下来自动打包。 曾经的讨论背景： From 289d2e2da6a5d62d8aacc61906596f1e65b42cb6 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Sun, 12 Mar 2017 22:04:17 +0800 Subject: [PATCH 22/62] del some --- paddle/scripts/docker/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 80a1177c6d..1181ea31f2 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -34,7 +34,6 @@ - 曾经的讨论背景： ["PR 1599"](https://github.com/PaddlePaddle/Paddle/pull/1599) ["PR 1598"](https://github.com/PaddlePaddle/Paddle/pull/1598) From b7a5d27efdf3843627d18d769ce3bd4293b87329 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Mon, 13 Mar 2017 09:26:07 +0800 Subject: [PATCH 23/62] precommit check --- paddle/scripts/docker/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 1181ea31f2..c5cde167f8 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -33,7 +33,6 @@ 其中 paddlectl 应该是我们自己写的一个脚本，调用kubectl来在Kubernetes机群上启动一个job的。 - 曾经的讨论背景： ["PR 1599"](https://github.com/PaddlePaddle/Paddle/pull/1599) ["PR 1598"](https://github.com/PaddlePaddle/Paddle/pull/1598) From 815b4913e39482bafef792721d1f0901793a1861 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Mon, 13 Mar 2017 09:51:12 +0800 Subject: [PATCH 24/62] pre-commit check --- paddle/scripts/docker/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index c5cde167f8..dd4a1d30d5 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -36,5 +36,3 @@ 曾经的讨论背景： ["PR 1599"](https://github.com/PaddlePaddle/Paddle/pull/1599) ["PR 1598"](https://github.com/PaddlePaddle/Paddle/pull/1598) - - From 9201a4c29ff01a27c58753bf7e4879f2561cd5be Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 13 Mar 2017 10:06:41 +0800 Subject: [PATCH 25/62] Change the order of docker doc session --- .../build_and_install/docker_install_cn.rst | 155 +++++++------- .../build_and_install/docker_install_en.rst | 191 +++++++++--------- 2 files changed, 171 insertions(+), 175 deletions(-) diff --git a/doc/getstarted/build_and_install/docker_install_cn.rst b/doc/getstarted/build_and_install/docker_install_cn.rst index cf7dddd073..13b256e485 100644 --- a/doc/getstarted/build_and_install/docker_install_cn.rst +++ b/doc/getstarted/build_and_install/docker_install_cn.rst @@ -4,138 +4,137 @@ PaddlePaddle的Docker容器使用方式 PaddlePaddle目前唯一官方支持的运行的方式是Docker容器。因为Docker能在所有主要操作系统（包括Linux，Mac OS X和Windows）上运行。 请注意，您需要更改 `Dockers设置 `_ 才能充分利用Mac OS X和Windows上的硬件资源。 -通过Docker容器开发PaddlePaddle ------------------------------- +纯CPU和GPU的docker镜像 +---------------------- -开发人员可以在Docker中开发PaddlePaddle。这样开发人员可以以一致的方式在不同的平台上工作 - Linux，Mac OS X和Windows。 +对于每一个PaddlePaddle版本，我们都会发布两个Docker镜像：纯CPU的和GPU的。 +我们通过设置 `dockerhub.com `_ 自动生成最新的docker镜像： +`paddledev/paddle:0.10.0rc1-cpu` 和 `paddledev/paddle:0.10.0rc1-gpu`。 -1. 将开发环境构建为Docker镜像 - - .. code-block:: bash +以交互容器方式运行纯CPU的镜像： - git clone --recursive https://github.com/PaddlePaddle/Paddle - cd Paddle - docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile . +.. code-block:: bash + docker run -it --rm paddledev/paddle:0.10.0rc1-cpu /bin/bash - 请注意，默认情况下，:code:`docker build` 不会将源码导入到镜像中并编译它。如果我们想这样做，需要设置一个参数： +或者，可以以后台进程方式运行容器： - .. code-block:: bash +.. code-block:: bash - docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile --build-arg BUILD_AND_INSTALL=ON . + docker run -d -p 2202:22 -p 8888:8888 paddledev/paddle:0.10.0rc1-cpu +然后用密码 :code:`root` SSH进入容器： -2. 运行开发环境 +.. code-block:: bash - 当我们编译好了 :code:`paddle:dev`， 我们可以在docker容器里做开发，源代码可以通过挂载本地文件来被载入Docker的开发环境里面： - - .. code-block:: bash + ssh -p 2202 root@localhost - docker run -d -p 2202:22 -v $PWD:/paddle paddle:dev +SSH方式的一个优点是我们可以从多个终端进入容器。比如，一个终端运行vi，另一个终端运行Python。另一个好处是我们可以把PaddlePaddle容器运行在远程服务器上，并在笔记本上通过SSH与其连接。 - 以上代码会启动一个带有PaddlePaddle开发环境的docker容器，源代码会被挂载到 :code:`/paddle` 。 - 请注意， :code:`paddle:dev` 的默认入口是 :code:`sshd` 。以上的 :code:`docker run` 命令其实会启动一个在2202端口监听的SSHD服务器。这样，我们就能SSH进入我们的开发容器了： - - .. code-block:: bash +以上方法在GPU镜像里也能用－只是请不要忘记按装CUDA驱动，以及告诉Docker： - ssh root@localhost -p 2202 +.. code-block:: bash -3. 在Docker开发环境中编译与安装PaddlPaddle代码 + export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" + export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') + docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:0.10.0rc1-gpu - 当在容器里面的时候，可以用脚本 :code:`paddle/scripts/docker/build.sh` 来编译、安装与测试PaddlePaddle： - - .. code-block:: bash - - /paddle/paddle/scripts/docker/build.sh - 以上指令会在 :code:`/paddle/build` 中编译PaddlePaddle。通过以下指令可以运行单元测试： - - .. code-block:: bash +运行PaddlePaddle书籍 +--------------------- - cd /paddle/build - ctest +Jupyter Notebook是一个开源的web程序，大家可以通过它制作和分享带有代码、公式、图表、文字的交互式文档。用户可以通过网页浏览文档。 -4. 在Docker容器中运行PaddlePaddle书籍 +PaddlePaddle书籍是为用户和开发者制作的一个交互式的Jupyter Nodebook。 +如果您想要更深入了解deep learning，PaddlePaddle书籍一定是您最好的选择。 - Jupyter Notebook是一个开源的web程序，大家可以通过它制作和分享带有代码、公式、图表、文字的交互式文档。用户可以通过网页浏览文档。 +当您进入容器内之后，只用运行以下命令： - PaddlePaddle书籍是为用户和开发者制作的一个交互式的Jupyter Nodebook。 - 如果您想要更深入了解deep learning，PaddlePaddle书籍一定是您最好的选择。 - - 当您进入容器内之后，只用运行以下命令： +.. code-block:: bash + + jupyter notebook - .. code-block:: bash - - jupyter notebook +然后在浏览器中输入以下网址： + +.. code-block:: text - 然后在浏览器中输入以下网址： - - .. code-block:: text + http://localhost:8888/ - http://localhost:8888/ +就这么简单，享受您的旅程！ - 就这么简单，享受您的旅程！ -纯CPU和GPU的docker镜像 ----------------------- +非AVX镜像 +--------- -对于每一个PaddlePaddle版本，我们都会发布两个Docker镜像：纯CPU的和GPU的。我们通过设置 `dockerhub.com `_ 自动运行以下两个命令： +纯CPU镜像以及GPU镜像都会用到AVX指令集，但是2008年之前生产的旧电脑不支持AVX。以下指令能检查Linux电脑是否支持AVX： .. code-block:: bash - docker build -t paddle:cpu -f paddle/scripts/docker/Dockerfile --build-arg BUILD_AND_INSTALL=ON . - docker build -t paddle:gpu -f paddle/scripts/docker/Dockerfile.gpu --build-arg BUILD_AND_INSTALL=ON . + if cat /proc/cpuinfo | grep -i avx; then echo Yes; else echo No; fi -以交互容器方式运行纯CPU的镜像： +如果输出是No，我们就需要手动编译一个非AVX版本的镜像： .. code-block:: bash - docker run -it --rm paddledev/paddle:0.10.0rc1-cpu /bin/bash + cd ~ + git clone https://github.com/PaddlePaddle/Paddle.git + cd Paddle + docker build --build-arg WITH_AVX=OFF -t paddle:cpu-noavx -f paddle/scripts/docker/Dockerfile . + docker build --build-arg WITH_AVX=OFF -t paddle:gpu-noavx -f paddle/scripts/docker/Dockerfile.gpu . -或者，可以以后台进程方式运行容器： -.. code-block:: bash +通过Docker容器开发PaddlePaddle +------------------------------ - docker run -d -p 2202:22 paddledev/paddle:0.10.0rc1-cpu +开发人员可以在Docker中开发PaddlePaddle。这样开发人员可以以一致的方式在不同的平台上工作 - Linux，Mac OS X和Windows。 -然后用密码 :code:`root` SSH进入容器： +1. 将开发环境构建为Docker镜像 + + .. code-block:: bash -.. code-block:: bash + git clone --recursive https://github.com/PaddlePaddle/Paddle + cd Paddle + docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile . - ssh -p 2202 root@localhost -SSH方式的一个优点是我们可以从多个终端进入容器。比如，一个终端运行vi，另一个终端运行Python。另一个好处是我们可以把PaddlePaddle容器运行在远程服务器上，并在笔记本上通过SSH与其连接。 + 请注意，默认情况下，:code:`docker build` 不会将源码导入到镜像中并编译它。如果我们想这样做，需要设置一个参数： + .. code-block:: bash -以上方法在GPU镜像里也能用－只是请不要忘记按装CUDA驱动，以及告诉Docker： + docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile --build-arg BUILD_AND_INSTALL=ON . -.. code-block:: bash - export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" - export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:0.10.0rc1-gpu +2. 运行开发环境 + 当我们编译好了 :code:`paddle:dev`， 我们可以在docker容器里做开发，源代码可以通过挂载本地文件来被载入Docker的开发环境里面： + + .. code-block:: bash -非AVX镜像 ---------- + docker run -d -p 2202:22 -v $PWD:/paddle paddle:dev -纯CPU镜像以及GPU镜像都会用到AVX指令集，但是2008年之前生产的旧电脑不支持AVX。以下指令能检查Linux电脑是否支持AVX： + 以上代码会启动一个带有PaddlePaddle开发环境的docker容器，源代码会被挂载到 :code:`/paddle` 。 + 请注意， :code:`paddle:dev` 的默认入口是 :code:`sshd` 。以上的 :code:`docker run` 命令其实会启动一个在2202端口监听的SSHD服务器。这样，我们就能SSH进入我们的开发容器了： + + .. code-block:: bash -.. code-block:: bash + ssh root@localhost -p 2202 - if cat /proc/cpuinfo | grep -i avx; then echo Yes; else echo No; fi +3. 在Docker开发环境中编译与安装PaddlPaddle代码 -如果输出是No，我们就需要手动编译一个非AVX版本的镜像： + 当在容器里面的时候，可以用脚本 :code:`paddle/scripts/docker/build.sh` 来编译、安装与测试PaddlePaddle： + + .. code-block:: bash + + /paddle/paddle/scripts/docker/build.sh -.. code-block:: bash + 以上指令会在 :code:`/paddle/build` 中编译PaddlePaddle。通过以下指令可以运行单元测试： + + .. code-block:: bash - cd ~ - git clone https://github.com/PaddlePaddle/Paddle.git - cd Paddle - docker build --build-arg WITH_AVX=OFF -t paddle:cpu-noavx -f paddle/scripts/docker/Dockerfile . - docker build --build-arg WITH_AVX=OFF -t paddle:gpu-noavx -f paddle/scripts/docker/Dockerfile.gpu . + cd /paddle/build + ctest 文档 diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index a4f62b2835..ddb7a2952b 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -9,6 +9,100 @@ Please be aware that you will need to change `Dockers settings of your hardware resource on Mac OS X and Windows. +CPU-only and GPU Images +------------------------ + +For each version of PaddlePaddle, we release 2 Docker images, a +CPU-only one and a CUDA GPU one. We do so by configuring +`dockerhub.com `_ +automatically generate the latest docker images `paddledev/paddle:0.10.0rc1-cpu` +and `paddledev/paddle:0.10.0rc1-gpu`. + +To run the CPU-only image as an interactive container: + +.. code-block:: bash + + docker run -it --rm paddledev/paddle:0.10.0rc1-cpu /bin/bash + +or, we can run it as a daemon container + +.. code-block:: bash + + docker run -d -p 2202:22 -p 8888:8888 paddledev/paddle:0.10.0rc1-cpu + +and SSH to this container using password :code:`root`: + +.. code-block:: bash + + ssh -p 2202 root@localhost + +An advantage of using SSH is that we can connect to PaddlePaddle from +more than one terminals. For example, one terminal running vi and +another one running Python interpreter. Another advantage is that we +can run the PaddlePaddle container on a remote server and SSH to it +from a laptop. + +Above methods work with the GPU image too -- just please don't forget +to install CUDA driver and let Docker knows about it: + +.. code-block:: bash + + export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" + export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') + docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:0.10.0rc1-gpu + + +PaddlePaddle Book +------------------ + +The Jupyter Notebook is an open-source web application that allows +you to create and share documents that contain live code, equations, +visualizations and explanatory text in a single browser. + +PaddlePaddle Book is an interactive Jupyter Notebook for users and developers. +We already exposed port 8888 for this book. If you want to +dig deeper into deep learning, PaddlePaddle Book definitely is your best choice. + +Once you are inside the container, simply issue the command: + +.. code-block:: bash + + jupyter notebook + +Then, you would back and paste the address into the local browser: + +.. code-block:: text + + http://localhost:8888/ + +That's all. Enjoy your journey! + + +Non-AVX Images +-------------- + +Please be aware that the CPU-only and the GPU images both use the AVX +instruction set, but old computers produced before 2008 do not support +AVX. The following command checks if your Linux computer supports +AVX: + +.. code-block:: bash + + if cat /proc/cpuinfo | grep -i avx; then echo Yes; else echo No; fi + + +If it doesn't, we will need to build non-AVX images manually from +source code: + +.. code-block:: bash + + cd ~ + git clone https://github.com/PaddlePaddle/Paddle.git + cd Paddle + docker build --build-arg WITH_AVX=OFF -t paddle:cpu-noavx -f paddle/scripts/docker/Dockerfile . + docker build --build-arg WITH_AVX=OFF -t paddle:gpu-noavx -f paddle/scripts/docker/Dockerfile.gpu . + + Development Using Docker ------------------------ @@ -82,103 +176,6 @@ Windows -- in a consistent way. cd /paddle/build ctest -4. Run PaddlePaddle Book under Docker Container - - The Jupyter Notebook is an open-source web application that allows - you to create and share documents that contain live code, equations, - visualizations and explanatory text in a single browser. - - PaddlePaddle Book is an interactive Jupyter Notebook for users and developers. - We already exposed port 8888 for this book. If you want to - dig deeper into deep learning, PaddlePaddle Book definitely is your best choice. - - Once you are inside the container, simply issue the command: - - .. code-block:: bash - - jupyter notebook - - Then, you would back and paste the address into the local browser: - - .. code-block:: text - - http://localhost:8888/ - - That's all. Enjoy your journey! - -CPU-only and GPU Images ------------------------ - -For each version of PaddlePaddle, we release 2 Docker images, a -CPU-only one and a CUDA GPU one. We do so by configuring -`dockerhub.com `_ -automatically runs the following commands: - -.. code-block:: bash - - docker build -t paddle:cpu -f paddle/scripts/docker/Dockerfile --build-arg BUILD_AND_INSTALL=ON . - docker build -t paddle:gpu -f paddle/scripts/docker/Dockerfile.gpu --build-arg BUILD_AND_INSTALL=ON . - - -To run the CPU-only image as an interactive container: - -.. code-block:: bash - - docker run -it --rm paddledev/paddle:0.10.0rc1-cpu /bin/bash - -or, we can run it as a daemon container - -.. code-block:: bash - - docker run -d -p 2202:22 paddledev/paddle:0.10.0rc1-cpu - -and SSH to this container using password :code:`root`: - -.. code-block:: bash - - ssh -p 2202 root@localhost - -An advantage of using SSH is that we can connect to PaddlePaddle from -more than one terminals. For example, one terminal running vi and -another one running Python interpreter. Another advantage is that we -can run the PaddlePaddle container on a remote server and SSH to it -from a laptop. - - -Above methods work with the GPU image too -- just please don't forget -to install CUDA driver and let Docker knows about it: - -.. code-block:: bash - - export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" - export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:0.10.0rc1-gpu - - -Non-AVX Images --------------- - -Please be aware that the CPU-only and the GPU images both use the AVX -instruction set, but old computers produced before 2008 do not support -AVX. The following command checks if your Linux computer supports -AVX: - -.. code-block:: bash - - if cat /proc/cpuinfo | grep -i avx; then echo Yes; else echo No; fi - - -If it doesn't, we will need to build non-AVX images manually from -source code: - -.. code-block:: bash - - cd ~ - git clone https://github.com/PaddlePaddle/Paddle.git - cd Paddle - docker build --build-arg WITH_AVX=OFF -t paddle:cpu-noavx -f paddle/scripts/docker/Dockerfile . - docker build --build-arg WITH_AVX=OFF -t paddle:gpu-noavx -f paddle/scripts/docker/Dockerfile.gpu . - Documentation ------------- From 2e0f46251afaed0f7146b923d979865adcb08ca9 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 13 Mar 2017 15:45:39 +0800 Subject: [PATCH 26/62] follow comment to revise subtitles --- doc/getstarted/build_and_install/docker_install_cn.rst | 4 ++-- doc/getstarted/build_and_install/docker_install_en.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/getstarted/build_and_install/docker_install_cn.rst b/doc/getstarted/build_and_install/docker_install_cn.rst index 13b256e485..af889ec9d1 100644 --- a/doc/getstarted/build_and_install/docker_install_cn.rst +++ b/doc/getstarted/build_and_install/docker_install_cn.rst @@ -4,8 +4,8 @@ PaddlePaddle的Docker容器使用方式 PaddlePaddle目前唯一官方支持的运行的方式是Docker容器。因为Docker能在所有主要操作系统（包括Linux，Mac OS X和Windows）上运行。 请注意，您需要更改 `Dockers设置 `_ 才能充分利用Mac OS X和Windows上的硬件资源。 -纯CPU和GPU的docker镜像 ----------------------- +纯CPU和GPU的docker镜像使用说明 +------------------------------ 对于每一个PaddlePaddle版本，我们都会发布两个Docker镜像：纯CPU的和GPU的。 我们通过设置 `dockerhub.com `_ 自动生成最新的docker镜像： diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index ddb7a2952b..606746597a 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -9,8 +9,8 @@ Please be aware that you will need to change `Dockers settings of your hardware resource on Mac OS X and Windows. -CPU-only and GPU Images ------------------------- +Usage of CPU-only and GPU Images +---------------------------------- For each version of PaddlePaddle, we release 2 Docker images, a CPU-only one and a CUDA GPU one. We do so by configuring From 4375a64ea1bfb9a47295d038f5eac14a7f9f195b Mon Sep 17 00:00:00 2001 From: Tao Luo Date: Mon, 13 Mar 2017 17:49:34 +0800 Subject: [PATCH 27/62] fix bug to pass travis --- python/paddle/trainer/config_parser.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 04ea135c1c..4d15210fc3 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2412,12 +2412,7 @@ class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): bias=False, **xargs): super(SequenceFirstInstanceLayer, self).__init__( - name, - inputs=inputs, - active_type=active_type, - device=device, - bias=bias, - **xargs) + name, inputs=inputs, active_type=active_type, bias=bias, **xargs) self.config.trans_type = trans_type self.config.select_first = True From bd41a8c482c6c1df03d118a6c7cb174a44b5d5ed Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 13 Mar 2017 18:06:30 +0800 Subject: [PATCH 28/62] Fix cmake find protobuf problem --- cmake/external/protobuf.cmake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 1575d8e9f5..446a7532c5 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -16,6 +16,14 @@ INCLUDE(ExternalProject) FIND_PACKAGE(Protobuf 3.1) +IF(PROTOBUF_FOUND) + EXEC_PROGRAM(${PROTOBUF_PROTOC_EXECUTABLE} ARGS --version OUTPUT_VARIABLE PROTOBUF_VERSION) + STRING(REGEX MATCH "[0-9]+.[0-9]+" PROTOBUF_VERSION "${PROTOBUF_VERSION}") + IF (${PROTOBUF_VERSION} VERSION_LESS "3.1.0") + SET(PROTOBUF_FOUND OFF) + ENDIF() +ENDIF(PROTOBUF_FOUND) + IF(NOT PROTOBUF_FOUND) SET(PROTOBUF_SOURCES_DIR ${THIRD_PARTY_PATH}/protobuf) SET(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/protobuf) From f66fd44fecc8b6fe1880aea96d266998a35f37a0 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Mon, 13 Mar 2017 19:00:45 +0800 Subject: [PATCH 29/62] do not use docker cp --- .gitignore | 4 ++- paddle/scripts/docker/buildall.sh | 27 +++++++++++-------- paddle/scripts/docker/paddle-core/Dockerfile | 12 ++++++--- .../Dockerfile} | 7 ----- 4 files changed, 27 insertions(+), 23 deletions(-) rename paddle/scripts/docker/{buildimage/Dockerfile.build => paddle-dev/Dockerfile} (90%) diff --git a/.gitignore b/.gitignore index 6aae076a49..3b407e6310 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ *.DS_Store -build/ +build/paddle/math +build/paddle/utils +build/paddle/gserver *.user .vscode diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh index 2d7de80965..c10720ab62 100644 --- a/paddle/scripts/docker/buildall.sh +++ b/paddle/scripts/docker/buildall.sh @@ -1,27 +1,32 @@ #!/bin/bash BINARIES_DIR=paddle/scripts/docker/buildimage/binaries +BUILD_DIR=$PWD/build function build_in_docker() { - docker build . -t paddle-build-env -f paddle/scripts/docker/buildimage/Dockerfile.build - BUILDER=$(docker run -d paddle-build-env) + if [ ! -d $BUILD_DIR ]; then + mkdir -p $BUILD_DIR + fi + docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile + # FIXME: need to wait a signal not sleeping + BUILDER=$(docker run -d -v ${PWD}:/paddle paddle-build-env sleep 3600) + # TODO(typhoonzero): docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh" mkdir -p $BINARIES_DIR - docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR - docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR - docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR - docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR - docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR - - docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR + # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR + # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR + # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR + # docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR + # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR + # + # docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR docker stop $BUILDER && docker rm $BUILDER } function build_paddle_core() { docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile - } build_in_docker -build_paddle_core +#build_paddle_core diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile index 708e4fe801..628f581776 100644 --- a/paddle/scripts/docker/paddle-core/Dockerfile +++ b/paddle/scripts/docker/paddle-core/Dockerfile @@ -24,10 +24,14 @@ RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/ pip install -U 'protobuf==3.1.0' RUN pip install numpy -ADD paddle/scripts/docker/buildimage/binaries/paddle paddle/scripts/docker/buildimage/binaries/paddle_trainer \ -paddle/scripts/docker/buildimage/binaries/paddle_pserver_main paddle/scripts/docker/buildimage/binaries/paddle_merge_model \ -paddle/scripts/docker/buildimage/binaries/paddle_usage /usr/local/opt/paddle/bin/ -ADD paddle/scripts/docker/buildimage/binaries/wheels/py_paddle*linux*.whl paddle/scripts/docker/buildimage/binaries/wheels/paddle*.whl \ +ADD build/paddle/trainer/paddle_trainer \ + build/paddle/pserver/paddle_pserver_main \ + build/paddle/trainer/paddle_merge_model \ + /usr/local/opt/paddle/bin/ +ADD build/paddle/scripts/usage.sh /usr/local/opt/paddle/bin/paddle_usage +ADD paddle/scripts/submit_local.sh.in /usr/bin/paddle + +ADD paddle/dist/py_paddle*linux*.whl build/python/dist/paddle*.whl \ /usr/local/opt/paddle/share/wheels/ RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \ diff --git a/paddle/scripts/docker/buildimage/Dockerfile.build b/paddle/scripts/docker/paddle-dev/Dockerfile similarity index 90% rename from paddle/scripts/docker/buildimage/Dockerfile.build rename to paddle/scripts/docker/paddle-dev/Dockerfile index 03e6ac792f..1aee44e21b 100644 --- a/paddle/scripts/docker/buildimage/Dockerfile.build +++ b/paddle/scripts/docker/paddle-dev/Dockerfile @@ -48,12 +48,5 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh - -VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] - - # FIXME: wait a long time is OK CMD ["sleep", "3600"] From e80a3cfb546f7a89c6ae4bc81aa008690fd7629c Mon Sep 17 00:00:00 2001 From: Peng Li Date: Mon, 13 Mar 2017 19:30:02 +0800 Subject: [PATCH 30/62] merge with latest code --- paddle/gserver/tests/CMakeLists.txt | 3 +-- paddle/gserver/tests/test_CRFLayerGrad.cpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 6b1f2a62b9..3c4128b5b8 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -21,8 +21,7 @@ add_test(NAME test_LayerGrad ################ test_CRFLayerGrad #################### add_unittest_without_exec(test_CRFLayerGrad test_CRFLayerGrad.cpp - LayerGradUtil.cpp - TestUtil.cpp) + LayerGradUtil.cpp) add_test(NAME test_CRFLayerGrad COMMAND test_CRFLayerGrad) diff --git a/paddle/gserver/tests/test_CRFLayerGrad.cpp b/paddle/gserver/tests/test_CRFLayerGrad.cpp index c267dcd9ab..df14449291 100644 --- a/paddle/gserver/tests/test_CRFLayerGrad.cpp +++ b/paddle/gserver/tests/test_CRFLayerGrad.cpp @@ -19,7 +19,7 @@ limitations under the License. */ #include "paddle/trainer/Trainer.h" #include "LayerGradUtil.h" -#include "TestUtil.h" +#include "paddle/testing/TestUtil.h" using namespace paddle; // NOLINT From 25a525bb092a09ef23df1ca20462cd3ee18434bf Mon Sep 17 00:00:00 2001 From: gaoyuan Date: Tue, 14 Mar 2017 11:09:49 +0800 Subject: [PATCH 31/62] Fix annotation --- paddle/gserver/layers/CostLayer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/layers/CostLayer.h b/paddle/gserver/layers/CostLayer.h index 2f85fd3eca..c2b7803710 100644 --- a/paddle/gserver/layers/CostLayer.h +++ b/paddle/gserver/layers/CostLayer.h @@ -164,8 +164,8 @@ public: * tasks. * \f[ * L = - * y / -1 < y < 1 / - * sign(y) / otherwise / + * output - label / -1 < (output - label) < 1 / + * sign(output - label) / otherwise / * \f] */ class SmoothL1CostLayer : public CostLayer { From 1b269dbe0628d88181bb0a079da4e02bb3735d2a Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Tue, 14 Mar 2017 15:09:49 +0800 Subject: [PATCH 32/62] use deb to build paddle-core image --- paddle/scripts/deb/build_scripts/build.sh | 18 +++++++--- paddle/scripts/docker/buildall.sh | 27 ++++++++------- paddle/scripts/docker/paddle-core/Dockerfile | 16 ++------- .../scripts/docker/paddle-core/Dockerfile.gpu | 32 ++++++++++++++++++ .../docker/paddle-core/Dockerfile.gpunoavx | 33 +++++++++++++++++++ .../docker/paddle-core/Dockerfile.noavx | 32 ++++++++++++++++++ paddle/scripts/docker/paddle-dev/Dockerfile | 7 ++-- 7 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpu create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx create mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.noavx diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh index d13dea5148..51cf7f1b9b 100755 --- a/paddle/scripts/deb/build_scripts/build.sh +++ b/paddle/scripts/deb/build_scripts/build.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +ARCH=$(uname -i) apt-get update apt-get install -y dh-make cd ~ @@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu mkdir -p ~/dist/cpu-noavx mkdir -p ~/dist/gpu-noavx cd paddle -mkdir build + +# clean build dir and third_party dir cache +rm -rf build third_party +mkdir -p build cd build -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` +# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything? cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/cpu rm -rf * -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ +ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu rm -rf * -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF +rm -f /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/cpu-noavx rm -rf * -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ +ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu-noavx diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh index c10720ab62..9c1a985754 100644 --- a/paddle/scripts/docker/buildall.sh +++ b/paddle/scripts/docker/buildall.sh @@ -1,32 +1,31 @@ #!/bin/bash -BINARIES_DIR=paddle/scripts/docker/buildimage/binaries BUILD_DIR=$PWD/build +DEB_DIST_DIR=$PWD/dist +VERSION=latest function build_in_docker() { if [ ! -d $BUILD_DIR ]; then mkdir -p $BUILD_DIR fi + if [ ! -d $DEB_DIST_DIR ]; then + mkdir -p $DEB_DIST_DIR + fi docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile # FIXME: need to wait a signal not sleeping - BUILDER=$(docker run -d -v ${PWD}:/paddle paddle-build-env sleep 3600) - # TODO(typhoonzero): - docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh" - mkdir -p $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR - # docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR - # docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR - # - # docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR + BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600) + # NOTICE: build deb files for real paddle image + docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh" docker stop $BUILDER && docker rm $BUILDER } function build_paddle_core() { - docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile + docker build . -t paddle-core:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile + docker build . -t paddle-core:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu + docker build . -t paddle-core:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx + docker build . -t paddle-core:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx } build_in_docker -#build_paddle_core +build_paddle_core diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile index 628f581776..eade296a40 100644 --- a/paddle/scripts/docker/paddle-core/Dockerfile +++ b/paddle/scripts/docker/paddle-core/Dockerfile @@ -23,20 +23,10 @@ RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/ pip install --upgrade pip && \ pip install -U 'protobuf==3.1.0' RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb -ADD build/paddle/trainer/paddle_trainer \ - build/paddle/pserver/paddle_pserver_main \ - build/paddle/trainer/paddle_merge_model \ - /usr/local/opt/paddle/bin/ -ADD build/paddle/scripts/usage.sh /usr/local/opt/paddle/bin/paddle_usage -ADD paddle/scripts/submit_local.sh.in /usr/bin/paddle - -ADD paddle/dist/py_paddle*linux*.whl build/python/dist/paddle*.whl \ - /usr/local/opt/paddle/share/wheels/ -RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ - pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \ - rm /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \ - rm /usr/local/opt/paddle/share/wheels/paddle*.whl ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" # default command shows the paddle version and exit CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpu b/paddle/scripts/docker/paddle-core/Dockerfile.gpu new file mode 100644 index 0000000000..89386b0379 --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile.gpu @@ -0,0 +1,32 @@ +FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y python python-pip libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx b/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx new file mode 100644 index 0000000000..beaeb5c51e --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx @@ -0,0 +1,33 @@ +FROM python:2.7.13-slim +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb + + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.noavx b/paddle/scripts/docker/paddle-core/Dockerfile.noavx new file mode 100644 index 0000000000..853dd7703a --- /dev/null +++ b/paddle/scripts/docker/paddle-core/Dockerfile.noavx @@ -0,0 +1,32 @@ +FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 +MAINTAINER PaddlePaddle Authors + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=OFF +ENV WITH_AVX=${WITH_AVX:-ON} +ENV WITH_DOC=${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y python python-pip libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-dev/Dockerfile b/paddle/scripts/docker/paddle-dev/Dockerfile index 1aee44e21b..11465a4ed8 100644 --- a/paddle/scripts/docker/paddle-dev/Dockerfile +++ b/paddle/scripts/docker/paddle-dev/Dockerfile @@ -1,5 +1,6 @@ # A image for building paddle binaries -FROM ubuntu:14.04 +# Use cuda devel base image for both cpu and gpu environment +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Authors ARG DEBIAN_FRONTEND=noninteractive @@ -22,8 +23,7 @@ ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} ENV HOME /root -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y git python-pip python-dev openssh-server bison && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ @@ -48,5 +48,6 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 +RUN apt-get install -y swig # FIXME: wait a long time is OK CMD ["sleep", "3600"] From baf890acf6119739b95740b3ce89bd10641ebf59 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 14 Mar 2017 16:14:37 +0800 Subject: [PATCH 33/62] Add doc in chinese. Why plain C is better for Paddle to use as a multi-language interface. --- .../multi_language_interface/why_plain_c.md | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 doc/design/multi_language_interface/why_plain_c.md diff --git a/doc/design/multi_language_interface/why_plain_c.md b/doc/design/multi_language_interface/why_plain_c.md new file mode 100644 index 0000000000..c375db13b6 --- /dev/null +++ b/doc/design/multi_language_interface/why_plain_c.md @@ -0,0 +1,88 @@ +# Paddle多语言接口实现 +## 背景 + +Paddle需要一个多语言接口，这个接口需要做到: + +* 有标准的，良好的文档 + * 例如Python可以使用[Sphinx](http://www.sphinx-doc.org/en/stable/)生成API文档，golang可以使用[GoDoc](https://godoc.org/golang.org/x/tools/cmd/godoc)生成文档。这都需要这个接口按照约定俗成的规则来注释完备。 +* 不同语言的接口适应不同语言的特性 + * 例如Java与Python的错误处理是直接扔出来Exception，而对于golang错误处理应该使用返回值。 + +我们可以将计算相关的代码使用C/C++来完成，而将逻辑结构相关的代码交由其他更高级的语言完成。不同开发语言具有不同的优势领域。使用多种语言混合开发Paddle可以增加Paddle的开发效率。 + + +## 基本要求 + +Paddle的多语言接口实现包括一下几个方面: + +* 我们使用动态库来分发Paddle。在这个动态库中不嵌入任何其他语言的解释器，也不使用其他动态库。 +* 这个动态库使用C99标准的头文件导出一些函数，不使用/导出C++符号。 +* 不导出Paddle内部的结构体、类，仅仅使用`void*`指针作为类型的句柄(handler)。 +* 不使用SWIG这种代码生成器，而是手写多语言绑定。 + + +## 原因 + +### 使用动态库来分发Paddle + +* Paddle的链接方式比较复杂(使用了--whole-archieve)，如果使用了静态库分发，那么要求使用Paddle库的人按照Paddle的要求链接Paddle，这通常比较复杂。 +* 编译型语言，例如C/C++使用静态库和动态库难度差不多。但是含有解释器的语言，例如python或者java，基本上只能够调用动态库。 + +### 动态库中不嵌入任何其他语言的解释器 + +* 目前Paddle的进程模型是C++内部驱动Python解释器进行模型配置解析和数据读取 +* 我们最终的动态库中不嵌入Python或者其他任何语言的解释器。模型配置解析，数据读取均交由其他语言完成 + +现阶段Paddle有一个问题是，Paddle内嵌的Python解释器和外部使用的Python如果版本不同，会直接报错退出。 + +### Paddle动态库中，不引用其他动态库 + +* 即这个动态库是不依赖于其他任何文件的，可以在任何机器上执行的。 + +### 这个动态库使用C99标准的头文件导出一些函数，不使用/导出C++符号 + +* 由于C++编译器没有[名字修饰](https://en.wikipedia.org/wiki/Name_mangling#C.2B.2B)的规范，不同版本的编译器之间，对于同一段C++代码生成的符号可能不一致。而多语言接口需要直接读取生成的二进制(动态库)，需要有稳定的导出符号。 +* C语言是有导出符号的标准的，并且在常见的平台上，都是ABI调用标准的。 +* 大多数语言都支持使用C语言API + +### 不导出Paddle内部的结构体、类，仅仅使用`void*`指针作为类型的句柄(handler) + +* Paddle内部的类为C++书写，直接导出到C的接口比较困难。 +* 在C-API中使用`void*`来表示Paddle内部类。再在每一个API中自己检查类型。 + +```C +// in matrix.cpp +struct PaddleMatrix { + int type; + paddle::MatrixPtr mat; +}; + +// in Paddle.h +typedef void* PD_Matrix; +extern "C" PD_Error getShape(PD_Matrix, uint64_t* height, uint64_t* width); +``` + +### 不使用SWIG这种代码生成器，而是手写多语言绑定 + +* [SWIG](http://www.swig.org/)是一个多语言接口的代码生成器。他的目标是使用C/C++写代码，SWIG直接读取C/C++的头文件，生成各种语言的绑定代码。 + * 对于多语言接口，SWIG需要写一个interface文件。这个文件具有独特的语法，学习成本高。且增加一个第三方语言，就需要对这个第三方语言增加一些定义。有的时候，interface文件的写法非常[tricky](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/api/Paddle.swig#L36)。社区贡献代码学习成本高。 + * SWIG暴露的接口保留了C++的接口样式，很难保证多语言代码风格的一致性。(函数命名，错误处理) + * 对于大多数语言，直接使用C语言的.h并不困难。例如Python的[cffi](https://cffi.readthedocs.io/en/latest/overview.html#simple-example-abi-level-in-line)或者[Cython](http://cython.org/), golang的[cgo](https://golang.org/cmd/cgo/)。 + * swig支持的语言或者解释器有局限。例如对于Python，使用SWIG只支持CPython解释器，而不支持PyPy解释器。 + + +## 原因列表 + +| 结论 | 对比 | 原因 | +|---| --- | --- | +| 使用动态库 | 不实用静态库 | 解释型语言只能调用动态库，Paddle静态库链接复杂 | +| 不嵌入其他语言解释器 | 不嵌入Python解释器 | Paddle C++目前嵌入Python解释器，会导致不同版本Python在一个进程里的bug | +| 不引用其他动态库 | | Paddle一个动态库可以在任何Linux系统上运行 | +| 使用C99做接口 | 不使用C++做接口 | C有标准的ABI，C99是目前C最广泛的使用标准(而不是C11，和C89) | +| 使用void*作为类句柄 | 不显示的写每个类具体包含什么| 实现简单，并且让接口脱离实现细节 | +| 手写多语言绑定 | 不使用SWIG | 写SWIG很tricky，社区参与难。SWIG生成的代码不能保证多语言代码风格的一致性 | + + +## 简单实现 + +TBD From 8c4176a261599fe848c51e636b22e30cf870f111 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 14 Mar 2017 16:22:11 +0800 Subject: [PATCH 34/62] Typo --- doc/design/multi_language_interface/why_plain_c.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/multi_language_interface/why_plain_c.md b/doc/design/multi_language_interface/why_plain_c.md index c375db13b6..13000a285f 100644 --- a/doc/design/multi_language_interface/why_plain_c.md +++ b/doc/design/multi_language_interface/why_plain_c.md @@ -75,7 +75,7 @@ extern "C" PD_Error getShape(PD_Matrix, uint64_t* height, uint64_t* width); | 结论 | 对比 | 原因 | |---| --- | --- | -| 使用动态库 | 不实用静态库 | 解释型语言只能调用动态库，Paddle静态库链接复杂 | +| 使用动态库 | 不使用静态库 | 解释型语言只能调用动态库，Paddle静态库链接复杂 | | 不嵌入其他语言解释器 | 不嵌入Python解释器 | Paddle C++目前嵌入Python解释器，会导致不同版本Python在一个进程里的bug | | 不引用其他动态库 | | Paddle一个动态库可以在任何Linux系统上运行 | | 使用C99做接口 | 不使用C++做接口 | C有标准的ABI，C99是目前C最广泛的使用标准(而不是C11，和C89) | From 684c9910ebdc1964a57daaa2202ed6afab157159 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 14 Mar 2017 17:10:04 +0800 Subject: [PATCH 35/62] Add FPE to FAQ. --- doc/faq/index_cn.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 6d5367177d..df5e172252 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -286,3 +286,16 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 .. code-block:: bash paddle train --use_gpu=true --trainer_count=2 --gpu_id=2 + + +12. 训练过程中出现 :code:`Floating point exception`, 训练因此退出怎么办? +------------------------------------------------------------------------ + +Paddle二进制在运行时捕获了浮点数异常，只要出现浮点数异常(即训练过程中出现NaN或者Inf)，立刻退出。浮点异常通常的原因是浮点数溢出、除零等问题。 +主要原因包括两个方面: + +* 训练过程中参数或者训练过程中的梯度尺度过大，导致参数累加，乘除等时候，导致了浮点数溢出。 +* 模型一直不收敛，发散到了一个数值特别大的地方。 +* 训练数据有问题，导致参数收敛到了一些奇异的情况。或者输入数据尺度过大，有些特征的取值达到数百万，这时进行矩阵乘法运算就可能导致浮点数溢出。 + +主要的解决办法是减小学习律或者对数据进行归一化处理。 From 7a81327c28dd864dca3fdc711587c2c717717c99 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 14 Mar 2017 16:13:49 +0800 Subject: [PATCH 36/62] remove compile option "with_metric_learning" --- cmake/util.cmake | 12 ------------ doc/howto/usage/cmd_parameter/arguments_cn.md | 10 ---------- doc/howto/usage/cmd_parameter/arguments_en.md | 10 ---------- .../usage/cmd_parameter/detail_introduction_cn.md | 9 --------- .../usage/cmd_parameter/detail_introduction_en.md | 9 --------- .../gradientmachines/MultiGradientMachine.cpp | 7 ------- paddle/gserver/layers/Layer.cpp | 3 +-- paddle/pserver/BaseClient.h | 3 --- paddle/pserver/ParameterServer2.cpp | 8 ++------ paddle/trainer/Trainer.h | 8 -------- paddle/utils/Flags.cpp | 1 - paddle/utils/Flags.h | 1 - paddle/utils/GlobalConstants.h | 5 ----- 13 files changed, 3 insertions(+), 83 deletions(-) diff --git a/cmake/util.cmake b/cmake/util.cmake index 24ad5c815c..3640e4651f 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -71,21 +71,10 @@ function(link_paddle_exe TARGET_NAME) generate_rdma_links() endif() - if(WITH_METRIC) - if(WITH_GPU) - set(METRIC_LIBS paddle_metric_learning paddle_dserver_lib metric metric_cpu) - else() - set(METRIC_LIBS paddle_metric_learning paddle_dserver_lib metric_cpu) - endif() - else() - set(METRIC_LIBS "") - endif() - target_circle_link_libraries(${TARGET_NAME} ARCHIVE_START paddle_gserver paddle_function - ${METRIC_LIBS} ARCHIVE_END paddle_pserver paddle_trainer_lib @@ -95,7 +84,6 @@ function(link_paddle_exe TARGET_NAME) paddle_parameter paddle_proto paddle_cuda - ${METRIC_LIBS} ${EXTERNAL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS} diff --git a/doc/howto/usage/cmd_parameter/arguments_cn.md b/doc/howto/usage/cmd_parameter/arguments_cn.md index 2e2a2fcc54..f7aa525054 100644 --- a/doc/howto/usage/cmd_parameter/arguments_cn.md +++ b/doc/howto/usage/cmd_parameter/arguments_cn.md @@ -228,16 +228,6 @@ √√ - -度量学习(metric learning)external -√√√√ - - - -data_server_port -√√ - - 参数服务器(PServer)start_pserver √√ diff --git a/doc/howto/usage/cmd_parameter/arguments_en.md b/doc/howto/usage/cmd_parameter/arguments_en.md index e5546f0ddc..d1963067bd 100644 --- a/doc/howto/usage/cmd_parameter/arguments_en.md +++ b/doc/howto/usage/cmd_parameter/arguments_en.md @@ -228,16 +228,6 @@ It looks like there are a lot of arguments. However, most of them are for develo √√ - -metric learningexternal -√√√√ - - - -data_server_port -√√ - - PServerstart_pserver √√ diff --git a/doc/howto/usage/cmd_parameter/detail_introduction_cn.md b/doc/howto/usage/cmd_parameter/detail_introduction_cn.md index 3b573a324d..b4625ba68c 100644 --- a/doc/howto/usage/cmd_parameter/detail_introduction_cn.md +++ b/doc/howto/usage/cmd_parameter/detail_introduction_cn.md @@ -180,15 +180,6 @@  - 用户可以自定义beam search的方法，编译成动态库，供PaddlePaddle加载。 该参数用于指定动态库路径. - 类型: string (默认: "", null). -## 度量学习(Metric Learning) -* `--external` - - 指示是否使用外部机器进行度量学习. - - 类型: bool (默认: 0). - -* `--data_server_port` - - 数据服务器(data server)的监听端口，主要用在度量学习中. - - 类型: int32 (默认: 21134). - ## 数据支持(DataProvider) * `--memory_threshold_on_load_data` diff --git a/doc/howto/usage/cmd_parameter/detail_introduction_en.md b/doc/howto/usage/cmd_parameter/detail_introduction_en.md index 33b7ec0d51..b681ebc81a 100644 --- a/doc/howto/usage/cmd_parameter/detail_introduction_en.md +++ b/doc/howto/usage/cmd_parameter/detail_introduction_en.md @@ -184,15 +184,6 @@ - Specify shared dynamic library. It can be defined out of paddle by user. - type: string (default: "", null). -## Metric Learning -* `--external` - - Whether to use external machine for metric learning. - - type: bool (default: 0). - -* `--data_server_port` - - Listening port for dserver (data server), dserver is mainly used in metric learning. - - type: int32 (default: 21134). - ## DataProvider * `--memory_threshold_on_load_data` diff --git a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp index 4654d02064..6ae60102b3 100644 --- a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp @@ -24,9 +24,6 @@ limitations under the License. */ DEFINE_bool(allow_only_one_model_on_one_gpu, true, "If true, do not allow multiple models on one GPU device"); -#ifdef PADDLE_METRIC_LEARNING -DECLARE_bool(external); -#endif namespace paddle { @@ -45,11 +42,7 @@ MultiGradientMachine::MultiGradientMachine(const ModelConfig& config, trainerBarrier_(FLAGS_trainer_count), allBarrier_(FLAGS_trainer_count + 1), inArgsCopied_(false) { -#ifdef PADDLE_METRIC_LEARNING - isPassGrad_ = FLAGS_external; -#else isPassGrad_ = false; -#endif numThreads_ = FLAGS_trainer_count; if (useGpu) { //! TODO(yuyang18): When useGpu=false && paddle is not compiled with gpu, diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index f76d41ad3e..125aaf947f 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -381,8 +381,7 @@ void Layer::backwardActivation() { void Layer::forwardDropOut() { auto& outV = getOutputValue(); - if (passType_ == PASS_TRAIN || passType_ == PASS_METRIC_TRAIN || - passType_ == PASS_METRIC_TRAIN_WITH_NOERROR) { + if (passType_ == PASS_TRAIN) { // new dropOutMask_ if dropOutMask_ is null ptr Matrix::resizeOrCreate(dropOutMask_, outV->getHeight(), diff --git a/paddle/pserver/BaseClient.h b/paddle/pserver/BaseClient.h index 11d7a147bf..667bc451d1 100644 --- a/paddle/pserver/BaseClient.h +++ b/paddle/pserver/BaseClient.h @@ -30,9 +30,6 @@ namespace paddle { * the first solution arms with sendThreads_/recvThreads_ and sendJobQueue_/ * recvJobQueue_. the second solution use some shared thread pool to manage * connections. - * In addition to pserver, metric learning also uses network to exchange - * features within multi-machines, so this class just abstracts some basic - * threads and queue buffer creation for them */ class BaseClient { protected: diff --git a/paddle/pserver/ParameterServer2.cpp b/paddle/pserver/ParameterServer2.cpp index 856fa0ad1a..877cbb86ec 100644 --- a/paddle/pserver/ParameterServer2.cpp +++ b/paddle/pserver/ParameterServer2.cpp @@ -367,11 +367,8 @@ void ParameterServer2::addGradient(const SendParameterRequest& request, std::vector* outputBuffers) { VLOG(1) << "pserver: addGradient"; -/// forwardbackward delta from all trainers -/// indicate the fluctuation caused by forwardbackward. -#ifndef PADDLE_METRIC_LEARNING - // @TODO(yanfei): - // add support tuning forwardbackward balance for metric learning + // forwardbackward delta from all trainers + // indicate the fluctuation caused by forwardbackward. if (!numPassFinishClients_) { REGISTER_BARRIER_DELTA_SERVER_SET( *statSet_, @@ -381,7 +378,6 @@ void ParameterServer2::addGradient(const SendParameterRequest& request, request.forwardbackward_time(), isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); } -#endif { /// approximately pure network overhead diff --git a/paddle/trainer/Trainer.h b/paddle/trainer/Trainer.h index c8ee4726c2..fac589d1d7 100644 --- a/paddle/trainer/Trainer.h +++ b/paddle/trainer/Trainer.h @@ -30,10 +30,6 @@ limitations under the License. */ #include "TrainerConfigHelper.h" #include "TrainerInternal.h" -#ifdef PADDLE_METRIC_LEARNING -#include "paddle/internals/metric_learning/MetricTrainer.h" -#endif - DECLARE_int32(num_passes); namespace paddle { @@ -201,12 +197,8 @@ protected: // parameter util std::unique_ptr paramUtil_; -#ifdef PADDLE_METRIC_LEARNING - MetricTrainer trainerInternal_; -#else // trainer Internal TrainerInternal trainerInternal_; -#endif }; } // namespace paddle diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index e8f31bc811..320f671ed9 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -30,7 +30,6 @@ DEFINE_bool(parallel_nn, DEFINE_int32(trainer_count, 1, "Defined how many trainers to train"); DEFINE_int32(gpu_id, 0, "Which gpu core to use"); DEFINE_int32(port, 20134, "Listening port for pserver"); -DEFINE_int32(data_server_port, 21134, "Listening port for dserver"); DEFINE_int32(ports_num, 1, "Number of ports for sending dense parameter," diff --git a/paddle/utils/Flags.h b/paddle/utils/Flags.h index 3e72f8356d..dc4faef833 100644 --- a/paddle/utils/Flags.h +++ b/paddle/utils/Flags.h @@ -19,7 +19,6 @@ limitations under the License. */ DECLARE_bool(parallel_nn); DECLARE_int32(async_count); DECLARE_int32(port); -DECLARE_int32(data_server_port); DECLARE_bool(use_gpu); DECLARE_int32(gpu_id); DECLARE_int32(trainer_count); diff --git a/paddle/utils/GlobalConstants.h b/paddle/utils/GlobalConstants.h index 707346f2c7..0ec1c28dfb 100644 --- a/paddle/utils/GlobalConstants.h +++ b/paddle/utils/GlobalConstants.h @@ -23,11 +23,6 @@ enum PassType { PASS_TEST, // Test pass PASS_GC, // Gradient Check pass PASS_METRIC, // pass for generate template output with no drop rate. - // pass for metric learning training with metric learning error, only used - // when we are doing KNN evaluation. - PASS_METRIC_TRAIN, - PASS_METRIC_TRAIN_WITH_NOERROR, // Pass for metric learning training - // with no evaluation. }; enum ParameterType { From 74da070e5ca0e9fb82298320f88a437cbc06ade8 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 14 Mar 2017 17:30:55 +0800 Subject: [PATCH 37/62] Speed up dense converter. --- paddle/py_paddle/dataprovider_converter.py | 31 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index c009b05cde..f1ed57f13f 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -16,6 +16,7 @@ import paddle.trainer.PyDataProvider2 as dp2 import collections import swig_paddle import numpy +import itertools __all__ = ['DataProviderConverter'] @@ -36,6 +37,12 @@ class IScanner(object): self.data_in_gpu = swig_paddle.isUsingGpu( ) and swig_paddle.getTrainerCount() == 1 + def pre_scan(self, dat): + pass + + def finish_pre_scan(self, argument): + pass + def scan(self, dat): pass @@ -51,12 +58,19 @@ class DenseScanner(IScanner): def __init__(self, input_type, pos): IScanner.__init__(self, input_type, pos) self.__mat__ = None + self.__height__ = 0 + + def pre_scan(self, dat): + self.__height__ += 1 + + def finish_pre_scan(self, argument): + self.__mat__ = numpy.ndarray( + shape=(self.__height__, self.input_type.dim), dtype=numpy.float32) + self.__height__ = 0 def scan(self, dat): - if self.__mat__ is None: - self.__mat__ = numpy.array([dat], dtype='float32') - else: - self.__mat__ = numpy.append(self.__mat__, [dat], axis=0) + self.__mat__[self.__height__] = dat + self.__height__ += 1 def finish_scan(self, argument): assert isinstance(argument, swig_paddle.Arguments) @@ -163,7 +177,14 @@ class DataProviderConverter(object): ] for each_sample in dat: - for each_step, scanner in zip(each_sample, scanners): + for each_step, scanner in itertools.izip(each_sample, scanners): + scanner.pre_scan(each_step) + + for scanner in scanners: + scanner.finish_pre_scan(argument) + + for each_sample in dat: + for each_step, scanner in itertools.izip(each_sample, scanners): scanner.scan(each_step) for scanner in scanners: From 952f8c139d0c46092cb33a697a275488f83e538f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 15 Mar 2017 10:48:50 +0800 Subject: [PATCH 38/62] Follow comments --- .../multi_language_interface/why_plain_c.md | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/doc/design/multi_language_interface/why_plain_c.md b/doc/design/multi_language_interface/why_plain_c.md index 13000a285f..a780d99bc4 100644 --- a/doc/design/multi_language_interface/why_plain_c.md +++ b/doc/design/multi_language_interface/why_plain_c.md @@ -8,9 +8,6 @@ Paddle需要一个多语言接口，这个接口需要做到: * 不同语言的接口适应不同语言的特性 * 例如Java与Python的错误处理是直接扔出来Exception，而对于golang错误处理应该使用返回值。 -我们可以将计算相关的代码使用C/C++来完成，而将逻辑结构相关的代码交由其他更高级的语言完成。不同开发语言具有不同的优势领域。使用多种语言混合开发Paddle可以增加Paddle的开发效率。 - - ## 基本要求 Paddle的多语言接口实现包括一下几个方面: @@ -25,8 +22,10 @@ Paddle的多语言接口实现包括一下几个方面: ### 使用动态库来分发Paddle -* Paddle的链接方式比较复杂(使用了--whole-archieve)，如果使用了静态库分发，那么要求使用Paddle库的人按照Paddle的要求链接Paddle，这通常比较复杂。 -* 编译型语言，例如C/C++使用静态库和动态库难度差不多。但是含有解释器的语言，例如python或者java，基本上只能够调用动态库。 +* Paddle的链接方式比较复杂 + * Paddle链接静态库使用了GCC的--whole-archieve参数，它要求使用Paddle静态库的二进制，在链接参数中指定`--whole-archieve paddle_xxx_lib --no-whole-archive`。且这个链接参数是GCC独有的。对于clang或者msvc，参数会不同。这增加了用户使用Paddle静态库的难度。 +* 编译型语言，例如C/C++使用静态库和动态库难度差不多。但是含有解释器的语言，例如[Python](http://stackoverflow.com/questions/19560594/how-to-import-static-library-in-python)或者[Java](http://stackoverflow.com/questions/24493337/linking-static-library-with-jni)，调用动态库远比调用静态库方便。 + * 解释性语言实际运行的二进制是解释器本身，如果调用静态库只能将静态库与解释器链接。例如对于Java来说，便是将静态库加入JVM中。这对于通常的Java的开发者来说，是不常见的做法。 ### 动态库中不嵌入任何其他语言的解释器 @@ -44,6 +43,8 @@ Paddle的多语言接口实现包括一下几个方面: * 由于C++编译器没有[名字修饰](https://en.wikipedia.org/wiki/Name_mangling#C.2B.2B)的规范，不同版本的编译器之间，对于同一段C++代码生成的符号可能不一致。而多语言接口需要直接读取生成的二进制(动态库)，需要有稳定的导出符号。 * C语言是有导出符号的标准的，并且在常见的平台上，都是ABI调用标准的。 * 大多数语言都支持使用C语言API +* 使用C99而不使用C89，是因为C99支持[Fixed-width integer types](https://en.wikipedia.org/wiki/C_data_types#Fixed-width_integer_types)和[Boolean type](https://en.wikipedia.org/wiki/C_data_types#Boolean_type)。 +* 使用C99而不使用C11的原因是，[C11](https://en.wikipedia.org/wiki/C11_(C_standard_revision))并没有Paddle特别需要的特性，且C99相对于C11使用更加广泛。 ### 不导出Paddle内部的结构体、类，仅仅使用`void*`指针作为类型的句柄(handler) @@ -51,15 +52,24 @@ Paddle的多语言接口实现包括一下几个方面: * 在C-API中使用`void*`来表示Paddle内部类。再在每一个API中自己检查类型。 ```C + +// in Paddle.h +typedef void* paddle_matrix; + +extern "C" paddle_error getShape(paddle_matrix mat, uint64_t* height, uint64_t* width); + + // in matrix.cpp struct PaddleMatrix { int type; paddle::MatrixPtr mat; }; -// in Paddle.h -typedef void* PD_Matrix; -extern "C" PD_Error getShape(PD_Matrix, uint64_t* height, uint64_t* width); +paddle_error get_shape(paddle_matrix m, uint64_t* height, uint64_t* width) { + PaddleMatrix* realMat = (PaddleMatrix*)(m); + ... +} + ``` ### 不使用SWIG这种代码生成器，而是手写多语言绑定 @@ -67,8 +77,10 @@ extern "C" PD_Error getShape(PD_Matrix, uint64_t* height, uint64_t* width); * [SWIG](http://www.swig.org/)是一个多语言接口的代码生成器。他的目标是使用C/C++写代码，SWIG直接读取C/C++的头文件，生成各种语言的绑定代码。 * 对于多语言接口，SWIG需要写一个interface文件。这个文件具有独特的语法，学习成本高。且增加一个第三方语言，就需要对这个第三方语言增加一些定义。有的时候，interface文件的写法非常[tricky](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/api/Paddle.swig#L36)。社区贡献代码学习成本高。 * SWIG暴露的接口保留了C++的接口样式，很难保证多语言代码风格的一致性。(函数命名，错误处理) + * 因为SWIG在第三方语言中暴露的函数名，类名和C++中完全一致。C++的命名风格并不能适应其他第三方语言。如果使用SWIG我们需要将在interface文件里，将大量的`SomeCppClass`重命名成`some_python_class`，或者`SomeGoTypes`。 + * 对于不同语言，错误处理的方式也不尽相同。例如对于Java或者Python，最常见的错误处理方式是Exception，而对于Golang，错误处理方式是返回值。而SWIG只能简单的暴露C++接口，无法做到对于各种语言错误处理方式的适配。 * 对于大多数语言，直接使用C语言的.h并不困难。例如Python的[cffi](https://cffi.readthedocs.io/en/latest/overview.html#simple-example-abi-level-in-line)或者[Cython](http://cython.org/), golang的[cgo](https://golang.org/cmd/cgo/)。 - * swig支持的语言或者解释器有局限。例如对于Python，使用SWIG只支持CPython解释器，而不支持PyPy解释器。 + * SWIG支持的语言或者解释器有局限。例如对于Python，使用SWIG只支持CPython解释器，而不支持PyPy解释器。 ## 原因列表 @@ -78,9 +90,9 @@ extern "C" PD_Error getShape(PD_Matrix, uint64_t* height, uint64_t* width); | 使用动态库 | 不使用静态库 | 解释型语言只能调用动态库，Paddle静态库链接复杂 | | 不嵌入其他语言解释器 | 不嵌入Python解释器 | Paddle C++目前嵌入Python解释器，会导致不同版本Python在一个进程里的bug | | 不引用其他动态库 | | Paddle一个动态库可以在任何Linux系统上运行 | -| 使用C99做接口 | 不使用C++做接口 | C有标准的ABI，C99是目前C最广泛的使用标准(而不是C11，和C89) | +| 使用C99做接口 | 不使用C++做接口 | C有标准的ABI，C99是目前C最广泛的使用标准，且C99支持bool类型和定长整数(uint64_t等)类型 | | 使用void*作为类句柄 | 不显示的写每个类具体包含什么| 实现简单，并且让接口脱离实现细节 | -| 手写多语言绑定 | 不使用SWIG | 写SWIG很tricky，社区参与难。SWIG生成的代码不能保证多语言代码风格的一致性 | +| 手写多语言绑定 | 不使用SWIG | 使用SWIG需要多语言绑定的开发人员熟练掌握SWIG配置，社区参与困难。SWIG生成的代码不能保证多语言代码风格的一致性 | ## 简单实现 From db045acb219774fe001eab0c45818a596bc3354a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=9B=8A?= Date: Wed, 15 Mar 2017 17:35:49 -0700 Subject: [PATCH 39/62] Improve the design doc of Docker build --- paddle/scripts/docker/README.md | 152 +++++++++++++++++++++++++++----- 1 file changed, 130 insertions(+), 22 deletions(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index dd4a1d30d5..722380a0e1 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -1,38 +1,146 @@ -因为我们不提供非Ubuntu的bulid支持，所以如果用户用其他操作系统，比如CoreOS、CentOS、MacOS X、Windows，开发都得在docker里。所以需要能build本地修改后的代码。 +We need to complete the initial draft https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/README.md. -我们可能需要两个 Docker images： +I am recording some ideas here, and we should file a PR later. -1. development image：不包括源码，但是包括开发环境（预先安装好各种工具），也就是说Dockerfile.dev里既不需要 COPY 也不需要 RUN git clone。虽然这个image和源码无关，但是不同版本的源码需要依赖不同的第三方库，所以这个image的tag里还是要包含git branch/tag name，比如叫做 `paddlepaddle/paddle:dev-0.10.0rc1`，这里的0.10.0.rc1是一个branch name，其中rc是release candidate的意思。正是发布之后就成了master branch里的一个tag，叫做0.10.0。 +## Current Status -1. production image： 不包括编译环境，也不包括源码，只包括build好的libpaddle.so和必要的Python packages，用于在Kubernetes机群上跑应用的image。比如叫做 `paddlepaddle/paddle:0.10.0rc1`。 +Currently, we have four sets of Dockefiles: -从1.生成2.的过程如下： +1. Kubernetes examples: -1. 在本机（host）上开发。假设源码位于 `~/work/paddle`。 + ``` + doc/howto/usage/k8s/src/Dockerfile -- based on released image but add start.sh + doc/howto/usage/k8s/src/k8s_data/Dockerfile -- contains only get_data.sh + doc/howto/usage/k8s/src/k8s_train/Dockerfile -- this duplicates with the first one. + ``` + +1. Generate .deb packages: + + ``` + paddle/scripts/deb/build_scripts/Dockerfile -- significantly overlaps with the `docker` directory + ``` + +1. In the `docker` directory: + + ``` + paddle/scripts/docker/Dockerfile + paddle/scripts/docker/Dockerfile.gpu + ``` + +1. Document building + + ``` + paddle/scripts/tools/build_docs/Dockerfile -- a subset of above two sets. + ``` + +## Goal + +We want two Docker images for each version of PaddlePaddle: + +1. `paddle:-dev` + + This a development image contains only the development tools. This standardizes the building tools and procedure. Users include: + + - developers -- no longer need to install development tools on the host, and can build their current work on the host (development computer). + - release engineers -- use this to build the official release from certain branch/tag on Github.com. + - document writers / Website developers -- Our documents are in the source repo in the form of .md/.rst files and comments in source code. We need tools to extract the information, typeset, and generate Web pages. + + So the development image must contain not only source code building tools, but also documentation tools: + + - gcc/clang + - nvcc + - Python + - sphinx + - woboq + - sshd + + where `sshd` makes it easy for developers to have multiple terminals connecting into the container. + +1. `paddle:` + + This is the production image, generated using the development image. This image might have multiple variants: + + - GPU/AVX `paddle:-gpu` + - GPU/no-AVX `paddle:-gpu-noavx` + - no-GPU/AVX `paddle:` + - no-GPU/no-AVX `paddle:-noavx` + + We'd like to give users choices of GPU and no-GPU, because the GPU version image is much larger than then the no-GPU version. + + We'd like to give users choices of AVX and no-AVX, because some cloud providers don't provide AVX-enabled VMs. + +## Dockerfile + +To realize above goals, we need only one Dockerfile for the development image. We can put it in the root source directory. + +Let us go over our daily development procedure to show how developers can use this file. + +1. Check out the source code -1. 用dev image build 我们的源码： ```bash - docker run -it -p 2022:22 -v $PWD:/paddle paddlepaddle/paddle:dev-0.10.0rc1 /paddle/build.sh - ``` - 注意，这里的 `-v ` 参数把host上的源码目录里的内容映射到了container里的`/paddle` 目录；而container里的 `/paddle/build.sh` 就是源码目录里的 `build.sh`。上述命令调用了本地源码中的 bulid.sh 来build了本地源码，结果在container里的 `/paddle/build` 目录里，也就是本地的源码目录里的 `build` 子目录。 + git clone https://github.com/PaddlePaddle/Paddle paddle + ``` + +1. Do something -1. 我们希望上述 `build.sh` 脚本在 `build` 子目录里生成一个Dockerfile，使得我们可以运行： ```bash - docker build -t paddle ./build + cd paddle + git checkout -b my_work + Edit some files ``` - 来生成我们的production image。 - -1. 有了这个production image之后，我们可能会希望docker push 到dockerhub.com的我们自己的名下，然后可以用来启动本地或者远程（Kubernetes）jobs： + +1. Build/update the development image (if not yet) ```bash - docker tag paddle yiwang/paddle:did-some-change - docker push - paddlectl run yiwang/paddle:did-some-change /paddle/demo/mnist/train.py + docker build -t paddle:dev . # Suppose that the Dockerfile is in the root source directory. + ``` + +1. Build the source code + + ```bash + docker run -v $PWD:/paddle -e "GPU=OFF" -e "AVX=ON" -e "TEST=ON" paddle:dev + ``` + + This command maps the source directory on the host into `/paddle` in the container. + + Please be aware that the default entrypoint of `paddle:dev` is a shell script file `build.sh`, which builds the source code, and outputs to `/paddle/build` in the container, which is actually `$PWD/build` on the host. + + `build.sh` doesn't only build binaries, but also generates a `$PWD/build/Dockerfile` file, which can be used to build the production image. We will talk about it later. + +1. Run on the host (Not recommended) + + If the host computer happens to have all dependent libraries and Python runtimes installed, we can now run/test the built program. But the recommended way is to running in a production image. + +1. Run in the development container + + `build.sh` generates binary files and invokes `make install`. So we can run the built program within the development container. This is convenient for developers. + +1. Build a production image + + On the host, we can use the `$PWD/build/Dockerfile` to generate a production image. + + ```bash + docker build -t paddle --build-arg "BOOK=ON" -f build/Dockerfile . ``` - 其中 paddlectl 应该是我们自己写的一个脚本，调用kubectl来在Kubernetes机群上启动一个job的。 +1. Run the Paddle Book + + Once we have the production image, we can run [Paddle Book](http://book.paddlepaddle.org/) chapters in Jupyter Notebooks (if we chose to build them) + ```bash + docker run -it paddle + ``` + + Note that the default entrypoint of the production image starts Jupyter server, if we chose to build Paddle Book. + +1. Run on Kubernetes + + We can push the production image to a DockerHub server, so developers can run distributed training jobs on the Kuberentes cluster: + + ```bash + docker tag paddle me/paddle + docker push + kubectl ... + ``` -曾经的讨论背景： -["PR 1599"](https://github.com/PaddlePaddle/Paddle/pull/1599) -["PR 1598"](https://github.com/PaddlePaddle/Paddle/pull/1598) + For end users, we will provide more convinient tools to run distributed jobs. From 12a3af19fbebb04b4b26d3aa0ca474bcfabdc672 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Thu, 16 Mar 2017 09:55:54 +0800 Subject: [PATCH 40/62] update .gitignore --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3b407e6310..6aae076a49 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ *.DS_Store -build/paddle/math -build/paddle/utils -build/paddle/gserver +build/ *.user .vscode From 3298d3075d25376e931c81ea599618e7520010b7 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Thu, 16 Mar 2017 09:56:36 +0800 Subject: [PATCH 41/62] production image name paddle not paddle-core --- paddle/scripts/docker/buildall.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh index 9c1a985754..63e5301ad9 100644 --- a/paddle/scripts/docker/buildall.sh +++ b/paddle/scripts/docker/buildall.sh @@ -21,10 +21,10 @@ function build_in_docker() { } function build_paddle_core() { - docker build . -t paddle-core:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile - docker build . -t paddle-core:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu - docker build . -t paddle-core:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx - docker build . -t paddle-core:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx + docker build . -t paddle:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile + docker build . -t paddle:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu + docker build . -t paddle:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx + docker build . -t paddle:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx } build_in_docker From 42cb7a201c8ec1ca362d4d67c88530dc3c149da2 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Thu, 16 Mar 2017 17:39:44 -0700 Subject: [PATCH 42/62] Update Docker build design doc to incorporate comments --- paddle/scripts/docker/README.md | 165 +++++++++--------- ...paddle-development-environment-gpu.graffle | Bin 0 -> 2912 bytes .../paddle-development-environment-gpu.png | Bin 0 -> 91803 bytes .../paddle-development-environment.graffle | Bin 0 -> 2691 bytes .../doc/paddle-development-environment.png | Bin 0 -> 67311 bytes 5 files changed, 87 insertions(+), 78 deletions(-) create mode 100644 paddle/scripts/docker/doc/paddle-development-environment-gpu.graffle create mode 100644 paddle/scripts/docker/doc/paddle-development-environment-gpu.png create mode 100644 paddle/scripts/docker/doc/paddle-development-environment.graffle create mode 100644 paddle/scripts/docker/doc/paddle-development-environment.png diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 722380a0e1..fc38fd7fa6 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -1,39 +1,21 @@ -We need to complete the initial draft https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/README.md. +# Building PaddlePaddle -I am recording some ideas here, and we should file a PR later. +## Goals -## Current Status +We want the building procedure generates Docker images, so we can run PaddlePaddle applications on Kubernetes clusters. -Currently, we have four sets of Dockefiles: +We want it generates .deb packages, so that enterprises without Docker support can run PaddlePaddle applications as well. -1. Kubernetes examples: +We want to minimize the size of generated Docker images and .deb packages so to ease the deployment cost. - ``` - doc/howto/usage/k8s/src/Dockerfile -- based on released image but add start.sh - doc/howto/usage/k8s/src/k8s_data/Dockerfile -- contains only get_data.sh - doc/howto/usage/k8s/src/k8s_train/Dockerfile -- this duplicates with the first one. - ``` +We want to encapsulate building tools and dependencies in a *development* Docker image so to ease the tools installation for developers. -1. Generate .deb packages: +We want developers can use whatever editing tools (emacs, vim, Eclipse, Jupyter Notebook), so the development Docker image contains only building tools, not editing tools, and developers are supposed to git clone source code into their development computers, instead of the container running the development Docker image. - ``` - paddle/scripts/deb/build_scripts/Dockerfile -- significantly overlaps with the `docker` directory - ``` +We want the procedure and tools work also with testing, continuous integration, and releasing. -1. In the `docker` directory: - ``` - paddle/scripts/docker/Dockerfile - paddle/scripts/docker/Dockerfile.gpu - ``` - -1. Document building - - ``` - paddle/scripts/tools/build_docs/Dockerfile -- a subset of above two sets. - ``` - -## Goal +## Docker Images We want two Docker images for each version of PaddlePaddle: @@ -45,7 +27,9 @@ We want two Docker images for each version of PaddlePaddle: - release engineers -- use this to build the official release from certain branch/tag on Github.com. - document writers / Website developers -- Our documents are in the source repo in the form of .md/.rst files and comments in source code. We need tools to extract the information, typeset, and generate Web pages. - So the development image must contain not only source code building tools, but also documentation tools: + Of course developers can install building tools on their development computers. But different version of PaddlePaddle might require different set/version of building tools. Also, it makes collaborative debugging eaiser if all developers use a unified development environment. + + The development image should include the following tools: - gcc/clang - nvcc @@ -54,7 +38,7 @@ We want two Docker images for each version of PaddlePaddle: - woboq - sshd - where `sshd` makes it easy for developers to have multiple terminals connecting into the container. + where `sshd` makes it easy for developers to have multiple terminals connecting into the container. `docker exec` works too, but if the container is running on a remote machine, it would be easier to ssh directly into the container than ssh to the box and run `docker exec`. 1. `paddle:` @@ -65,82 +49,107 @@ We want two Docker images for each version of PaddlePaddle: - no-GPU/AVX `paddle:` - no-GPU/no-AVX `paddle:-noavx` - We'd like to give users choices of GPU and no-GPU, because the GPU version image is much larger than then the no-GPU version. + We'd like to give users the choice between GPU and no-GPU, because the GPU version image is much larger than then the no-GPU version. + + We'd like to give users the choice between AVX and no-AVX, because some cloud providers don't provide AVX-enabled VMs. + + +## Development Environment + +Here we describe how to use above two images. We start from considering our daily development environment. + +Developers work on a computer, which is usually a laptop or desktop: + +![](doc/paddle-development-environment.png) + +or, they might rely on a more sophisticated box (like with GPUs): + +![](doc/paddle-development-environment-gpu.png) + +A basic principle is that source code lies on the development computer (host), so that editing tools like Eclipse can parse the source code and support auto-completion. + - We'd like to give users choices of AVX and no-AVX, because some cloud providers don't provide AVX-enabled VMs. +## Usages -## Dockerfile +### Build the Development Docker Image -To realize above goals, we need only one Dockerfile for the development image. We can put it in the root source directory. +The following commands check out the source code on the development computer (host) and build the development image `paddle:dev`: -Let us go over our daily development procedure to show how developers can use this file. +```bash +git clone https://github.com/PaddlePaddle/Paddle paddle +cd paddle +docker build -t paddle:dev . +``` -1. Check out the source code +The `docker build` command assumes that `Dockerfile` is in the root source tree. This is reasonable because this Dockerfile is this only on in our repo in this design. - ```bash - git clone https://github.com/PaddlePaddle/Paddle paddle - ``` -1. Do something +### Build PaddlePaddle from Source Code - ```bash - cd paddle - git checkout -b my_work - Edit some files - ``` +Given the development image `paddle:dev`, the following command builds PaddlePaddle from the source tree on the development computer (host): -1. Build/update the development image (if not yet) +```bash +docker run -v $PWD:/paddle -e "GPU=OFF" -e "AVX=ON" -e "TEST=ON" paddle:dev +``` - ```bash - docker build -t paddle:dev . # Suppose that the Dockerfile is in the root source directory. - ``` +This command mounts the source directory on the host into `/paddle` in the container, so the default entrypoint of `paddle:dev`, `build.sh`, would build the source code with possible local changes. When it writes to `/paddle/build` in the container, it actually writes to `$PWD/build` on the host. -1. Build the source code +`build.sh` builds the following: - ```bash - docker run -v $PWD:/paddle -e "GPU=OFF" -e "AVX=ON" -e "TEST=ON" paddle:dev - ``` +- PaddlePaddle binaries, +- `$PWD/build/paddle-.deb` for production installation, and +- `$PWD/build/Dockerfile`, which builds the production Docker image. - This command maps the source directory on the host into `/paddle` in the container. - Please be aware that the default entrypoint of `paddle:dev` is a shell script file `build.sh`, which builds the source code, and outputs to `/paddle/build` in the container, which is actually `$PWD/build` on the host. +### Build the Production Docker Image - `build.sh` doesn't only build binaries, but also generates a `$PWD/build/Dockerfile` file, which can be used to build the production image. We will talk about it later. +The following command builds the production image: -1. Run on the host (Not recommended) +```bash +docker build -t paddle -f build/Dockerfile . +``` - If the host computer happens to have all dependent libraries and Python runtimes installed, we can now run/test the built program. But the recommended way is to running in a production image. +This production image is minimal -- it includes binary `paddle`, the share library `libpaddle.so`, and Python runtime. -1. Run in the development container +### Run PaddlePaddle Applications - `build.sh` generates binary files and invokes `make install`. So we can run the built program within the development container. This is convenient for developers. +Again the development happens on the host. Suppoose that we have a simple application program in `a.py`, we can test and run it using the production image: -1. Build a production image +```bash +docker run -it -v $PWD:/work paddle /work/a.py +``` - On the host, we can use the `$PWD/build/Dockerfile` to generate a production image. +But this works only if all dependencies of `a.py` are in the production image. If this is not the case, we need to build a new Docker image from the production image and with more dependencies installs. - ```bash - docker build -t paddle --build-arg "BOOK=ON" -f build/Dockerfile . - ``` +### Build and Run PaddlePaddle Appications -1. Run the Paddle Book +We need a Dockerfile in https://github.com/paddlepaddle/book that builds Docker image `paddlepaddle/book:`, basing on the PaddlePaddle production image: - Once we have the production image, we can run [Paddle Book](http://book.paddlepaddle.org/) chapters in Jupyter Notebooks (if we chose to build them) +``` +FROM paddlepaddle/paddle: +RUN pip install -U matplotlib jupyter ... +COPY . /book +EXPOSE 8080 +CMD ["jupyter"] +``` - ```bash - docker run -it paddle - ``` +The book image is an example of PaddlePaddle application image. We can build it - Note that the default entrypoint of the production image starts Jupyter server, if we chose to build Paddle Book. +```bash +git clone https://github.com/paddlepaddle/book +cd book +docker build -t book . +``` -1. Run on Kubernetes +### Build and Run Distributed Applications - We can push the production image to a DockerHub server, so developers can run distributed training jobs on the Kuberentes cluster: +In our [API design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/api.md#distributed-training), we proposed an API that starts a distributed training job on a cluster. This API need to build a PaddlePaddle application into a Docekr image as above, and calls kubectl to run it on the cluster. This API might need to generate a Dockerfile look like above and call `docker build`. - ```bash - docker tag paddle me/paddle - docker push - kubectl ... - ``` +Of course, we can manually build an application image and launch the job using the kubectl tool: - For end users, we will provide more convinient tools to run distributed jobs. +```bash +docker build -f some/Dockerfile -t myapp . +docker tag myapp me/myapp +docker push +kubectl ... +``` diff --git a/paddle/scripts/docker/doc/paddle-development-environment-gpu.graffle b/paddle/scripts/docker/doc/paddle-development-environment-gpu.graffle new file mode 100644 index 0000000000000000000000000000000000000000..4629f9b9da7ababdafa0b964db18a98a819c6a9e GIT binary patch literal 2912 zcmV-m3!n5KiwFP!000030PS5{bK*D_exCdaZ=Uv&21&m6WHOaocQTt!GJ$lqyQtby zYysLBi~%$>l{gz^nE(#2+P~Q-VXxpmhgzO@Rv>40Gk>K zJ?7J}_siyG``sI7^Vgk??Y|wI?YBRjA8NyZMnXHk+&ekm*EZiYo4dndK$^{i_JMYO za@=ZbGLL5S@N`q#>r7jZ|p(!rzp|B)R9R zBd_|@6FVC}ZQqcGon24RTXKRQ2tN*ea=+bFS7jm%1?dsK1Gbx!(}{{Kn|fHtEI%eW zPn9s|Se^W|9SJUPc}Lc;!3H7iaW)z@&g9v9j=SA}Sli99NJbi3ZUgDE`pC8{3uQ#M zo0F`QDvSi%_5U}D#I$I^N2EDP-^1RGDl+tCal=0KrefoJAB!o8Xp%Oq#@DZ4O9PKv z+E>ffw=`(j_Tyu&zDi*~z)@MZ#R8fZT2MJYNaZJuin8sSh_%E+KvKy`m6Jr7CA&#e zGs#K%0mpY~wbF{6o@J{e8+6(dle6?BX~KwnW^6FWfY`_3EsoAO?a_27OxiMT%4|=j z8ZG(r-K1N_kPa8N1|YBbb-(bS=+ z_KpUm{V*)4V2OvKY&-}Lm^T`bP!xeWZ8ubExbfhP6qi&0J3kr=2+BYRA+up_X*!gU zb}i(%26AizS-QfNu2MiAR8^=-Vb#Y&QiZEYyNEWq7iGOqhph_ffZdfvFGv)R$D(uw zv}wB;dDteBCp3HlnAvx%q9W5@>Uun4LBrN{;5a65Vz2;R5JduAw7G-XqN3 z*jTf-v?gCdGt)Bp^O%SG*v}55a!VAMcor2`kblab+BM8V%btep#fame^Tksa1nu<& z3A@;nybm@u;=7AcY`D-ip=$u$HEm#nnuBW&t~vOx#=$daQZdkM zl9Zir9Cq34K35VIeLn4@!qAG@BI8F*fd#XqRJ`vag8fBV(tMh-swvv3py+cZFJ&8U z>Re=oxB=w!BnS0l;eOL}pxg^fcZ)ro<~9$Nu9reoKkRaztH3mW>Ffn$lAJAQJkyaL z_8xMIT^vMl5F}*~OoPP>c#5!Oh6~TH`mCX|jD2GPIWY9tgmKlW`3feCbe|RWfnsqo-j(kXB)&PU4gHq)i5?k z)Ys(FY`Xy&iQRBZ2T$7#s!`4Z>=6}PtAJm=C6{426K-kHbrj1%<-^IZ)$v?4W$24j z#(JT$&v;1q1@`GEUqp;_C+C#aU|F}p{;aj$CVW6c9H_;uHN($DNLiQCBW*xiM+lg< z4iSX5!lS9n6#IJ)oUAH%t;0nFC$N!gFd?+8I&e}4PU^tPw-`9-Mp|cs2(M8` zP*xQ|fz~o16u4{%Mbg8QA(S}X(m@?S`PKs{$XHz= zGg{4$<6*u`h#4&%Vn#-6$lJ5{F-}L8^TYDv3o_%&bE30CN`$VHiz!=99aFAAnpww` z>zHyKQ(i8n><#b@>1-t5L+twjnfx((`de^J!!?J#4A*?^$TKqP$a5WeUSs6>&hF=Z8HNx)$_VyUN(T1F9 zfA`5P3D{8KOY_)ZI1+^aW8I6f26)|oWhqWy2P{_*Vv(I_n(@P6j%`Xm+5(1UBzBJV zoj-{H%@r-ELu1heb;((_r8};vmeY~_V?Bw0mWl7oqsf=htQXYl8FuLfb)R`Rgll{h zYHx1q74?;^sOw9t7(wGjSB!q-6?O1^ucNPW%?RnnD_f;{ZI!OvDplb&1L?0AZgXmP zzjpW6=qeBlNFPln$gM+tyicQseRm9ww`US1s2( z!ZbN+|Gsa^-8EPJ7fB1Ecx4nMBjsfsC9`&v>iKrn^KHO99B2_6agRv<$DcXxF8k*J z-FBvb9?*gH!WgOlI9CS~))`EYGvnx9yJq*=9Tk1;{rTnE|G=}Qxp6cP+VTv%^rKDR z&_Cksypx*WlSz7S{Jz|+ zKD&#QNNIcRLYyv<4)HK*vpMARJ6yjXQJ+L7Di-P&H&o6Iv5fRb8qw?Wmd%A?VqfGO z_efOQaJjtS^6@|l?n_1!gwL_B^}h)jwpo7T=&8$+xGQQi_vqQJo7Ig!7gIns7Ipts_~JMyJxeVHQ;1z*+|SDa=RfNWVZjF6=PAU9_1eU zK*NK~2H6w$JY&J6vMnN!gmdovgg`+-K8UM%81}L;;z?b6CQ6kF)vn{pK|LX1PxMO# zOxPK@4nR6mk6PpkjP)0&ptxg0I+uJNY_f_sv(Qrp+XvhWiC9EFlIUsm2@!>XKSjj{ z=F{%@wFS>@^p4&F_!b(~8@(qPk$HD|KV^c=_vFb~|6sFD(n7^i=zc$lm=4GY7q zT#)kS{K|s%cEun5(Qm=0D+tJc_JYsHM{L*leed_<)?t6Qe|8B>czyL-;Pv+9Z~yEP ze(ql%eGGPG`u?HWZlmA6e7-sbyT@0DyR9!$=G}d8xBvd>>YtFV_%qF;xa6t)&!xG_ zb0O7=?8{44)AuYuw58c@es_q9Qq(xOAnHjqQZYEchmN>VHq1-*M4Js?BKtgn#vO4M z5(VNJ6aNLl{#h72Jdf@Rb~o>PG^FTUFVTp}w@pQvh?19m&I{A%PYk1a%%lON-+6CTDJZ4L}%+S)SUUs7_^4xmR z^nE*N`o$E*~P zZ3j4cz=lnQ-zU6hPkf&tUG46SfdyH^Rb;Xm%E*v{_=+H@} zdv~=D9fE{_zl(H#fKR57=MTVthuyUm|2$OE%{C8yp?`SK$onm5cLriw7>2*4KTUAA-9N9g_5cgCCu(JuJ90?}6T#{}NY~b2=75@D=_)Uu2&cov&90v3D z_P*|Y^SXZt55P=g+^|>0xj4zu$y(|97`Q2VvAJFyZS$u>ZUpJSs{36n@9W`JtP& zyE}Nlw20)dJOA?8|9;QE?$@+)@o)j7;AU^7g!Hg>0}p#xP+KQ`0@sRH^vVgl0Bq!S5DXG z@Q-o&F@OBt?VtUmXzloooRbO6Mtz}|?;`fX$}g8+QOdc?ec4FxPZ`Jgp=I0cOP7oj zFRKJD7*hff3Gw=G`a~&YanCkMa;IcRr(}gBS-NycvZr&rTYSZDwmYpS)lXgd%wZY` z9Rru_qeK7q{%{`xVLq>Uelm?5@c9ub5gKg4bX$aF?yZGjJn1h()4a#Ts>@|J6+(Ml z)G%NjKbui-f$`PTW|1M2xMke#awO^M#coTH$TNQ#&)^sc^NL+DN6}fi%zh1-#-qmN z^fcLmZx>7I^8`z3EIkA>zyDdlJQVzfiFHx-%V)lf zRQ20aPdxh7#Hz-GQ=F@fMyz6``essGCfp9*JmX02hA$Sa?aj)_u)BS_1WmFRSxsu% zv-Vvyo&WIefCqQ8D(y~JZc zOA-HA_O##dhd!NjQheonUP_*xzESRvG;j;7abC?uS&=hft!Q&F(rhhjt;kkNaR28s znbh?<{lEBVs(0#q+B{rEw=+6u94>yW#BCvgGRP#^!YO{TA@b3)s`iT7u@_FglMtES$u{;(++;VKjXlo z7LQJzk@?+^-Z!Ley~=PlIF$-5engIYjY5|I6wz5ux{_#^HTEa-alAuSALo~NK@V|q zy5F(@G`w!Cv&bM;J%=F7B7jhMI_}gjbFdm4Gl(IDk5aO%es~EiTq!E>iZu2rBcX`&icfe1BZ! zClB(D-w!L!l#R(CO6A6gquBN2iA~&QZU?NXVXV8F5F?@f%Ahe!GMj0k7&`%PoDnH4 z@#;6wwJE!rI6XuG(6h&*^>yi}~)zP~TVEjI`^x zt!a-AC^nuE&$0bUF^>iJ^J%Wtk{3_zK3MzrY|~LTb>CmA^sNt>XK;|7Kg(GkG=@pd zD`_Z~`V$5FA9Ji_D%I+9xcCz;_VcDVy_DXp?a2kBGn9)rxYpFGa4~nN>0sU3tjK@F zK0o*8sYRi%{ndt9snW5HA%{uT_AkG!4x#rnR_Ew;z;e^_!i$bN-CX+eGkSOJ=ZDiS z{zgH8T_QvA-+HV#$ggfJWl;8PQKsfxGTY<~iV24>6tQfCJ;%XFNelJR=lN z&-f~QP?+&kL>qPb)_P+DDE%BTvyI1Hums?_}7fyh}kVdEG*VK(J1eG*^rqZK3@cTJG@zQP53O-EbGof#{Li6 z`TEIy;~%da5>- zE|PL~QPDBwUhlD4jQPGpp$^2{(k~KPZD=3hG56+~`mmD~!yGNlM~W-GSn)WKy7eEm1Hrt^UG3#i1#=GHMI+Ezb zE0|K;PL4gEv?>##=o)D>mUwRQ?puC^8>4}{OC6TIS=QIJD@A?VBqm;D;WaBW!~HXM z$7U1tMS8`w$3~c;);7u0UL~9dKc88z(oHlhdzPwDy#%UJpW_9TMo1WLoI~d$|BzR^ za&UVC+hm&F20zN_|L|DFJ;smWvR$?^yI-|i!E!IXa$ny@bU!@P=(hB%c#pulq5>U{ zt^Jot4LR4&7eAgxL2@kXUswjQKWsj{K7_yLW}PSVo2}tE7Ny2Dw}2@vS;0}Q`9Z_# zXO}>7N+n+3SZ%vzdnVlEhM_ z&g*l*nXkRpOx7Qtt){GUQhFWwbDOf}V)YJg6eWx`s=fsBZKlXh+ zy-}hsuyZGZn5^SDYNi@HsL*BqE`-(|Y`tr7eoJGiNkhN2^$SfL55wc@y=+8I_R|K1 z4$<0;9N#Ie2SFD)%EsI}@a77f@J3Z*r!sl_ll{n)p1_TYj-t72KDcwbQiSv)ZYjd~ z9=r3y5h&0ixOYt8ldLPcIS!3b>{z+A?HB7edwkUw1&{`gco}VpHacIuLI-=@#*zbA z?e~{>9m-!_adQJRYf8H7MG5r-;4lW<@>>gZY*SrZb5JZM59Mm39r~iK^WDu5@7n*v zv-b1(;)&(K8V6$QBc;k5hoKNOLi#}QDE@^_xs%b)U>QA?Vziz*p`mpa?`$Is-I$CJ zEE&FCzb13b;?GEysqw2_m3SSDry9>+cW3rxA<^K>ITo93Vv>f~xJQ4E#WxH)a{!^J zvQHV`u|t_Nx4EvpW_2@6cinv8`bw)Y-lp$@z>ZnO;GGvX{dK&9kuU1GpYH1xy3TE} zPO1L3`5`tiAyI57ty<+)_}czTP*5vYvW3*kNaD|(->ECynYA~_pNp?mjxK(Tw(p|w zJgi(mV-i32k1I0=aL9A3W)0L5i2;+3inso~$y^hy(1(hlWCa&`JM%FtnLA#@8!tAkMN(0w1Hv|#aq^22T z?Xbg7=JCbGO_co=K2;Zg#(-U?iZ0Q4dksIt&S#B4Z5utgw8Fti+8!CoUb%mz4LU^K zI^?%~@TfWJmv8^@%HTtI5aHtFJ-*b3@8dOlg;%GZahf#+*tb6&?fl_Mxrr}!)Z@w( zA7yezlzxfZs4C6%TWJ(COFVlx zNem#mdEnAjdN0_OHg2G85-79jen{U1oeTuWGd8zRPf)_Q{P0(Q;VKq0pQ$I?RExBD zIFD{dS6cYTZe9u!k+|P{R9Z+t_T1qte#N!^v-_tTr%t-OdTR>%jR*^VeLPCdw)dP8 zgeykoV9VqC3&q6dkFRahUr#fB=umuqB{Z>#)GoMq_~qP(87e5OU%~k$m#m~+joqTW zYLnz`2{X)3_jN#hto&sA^qhWJK6}2Y*Vo+22NC6LU})#(vSZbOuQL;HuUz{I*2gcj zqAt1mu^Q!6h1b8L$t0JZ9Ro+7AO=!Vit|V1Ud!1>M^4>y1f754eD_O1-4$qvb+kZ; z2BWUTT;^ltXJdZLwYCTpF+r#DeMEDgo`I#rQtMf{Ttirhv{t@GASo{QRBw`& z$OWSxSOBUTO?O3R7Jr*zRK7?<5d)(n{-W%-E5TjK(U7crV&vZDJ z&hoAh4^k=OQm@#gKiP==;_h<2OYOe>YwN_{X3k2Lbr~ z2`XL$*m~7o@ms|G&RH4CwlscdQTh6K2;$2fl{M7(baojSGU&K!N7iar2{M2vr1+Q-aB@-gCV#sSzyer2iP{r?yYSBwe;bm2 z#_DfF^3N*y+mQUTO8zz^|E!X~4axsMtHdy8MSLP->OCOroZjv%U{3Szu727`G)4-X zGey=CTeIKu7fq0QH)i=el7bpLI z!$V>QZFIwsMkQW1;+lk*D zKBV<72zRM^RXY4F+yx0?JMw!2|7kkcKcoK7g865g{$pYOF^>PZr2m6Ay}v44m=Bz( z5C=^p(E*vjIXc^xyj_L}EPBovGI3t+KfE2QO7N~*u}f#HBbm^?w}lK$jpN*$d3Gz5 zCNjz*t*x;AIA_6+P=c<#C%Z~8(XIHj8Hn3#6yY5@WVkeqEuNn11H|JiJ_&p@QR_|L ze;l=dIcNSwWX&5dLNw!XW#aqW=A4u9`9T6r(%dr0@bi6JbX#7xnl`IszwAvkIe0fU z2S#46o7-Ih9L+Oi%~WWWsrR_&i!|>EapEkM;*I(*^5+u`xMuJxZY9>V-Q}?}pFhNW z-K6FoE(QlRS8U?zIe_G>0&tT&xr^Yk3tX9WEggS({#(u7uruHRa4R61W_05(4!hByEn*oWVUv(m4(K+A`PYrmbF#viaru}gt z^pN!@y$YfC?$?gJ8BYKJbf&M?yyh%$<%SAWXu~xqxX5dxf=CL zqIy3n_{LVl8jL@TjJ~?_sQD*78(a2c`L-`1vLCSS@nQdi81H+~PVc}=DhR$3L(|`S zi^E@P@MX$>@%S@a$~-c&cOpnxM(ci=B=_+8MW?s#KEOZ(d~aahkvrDlKOKkEF@^Ph zqy`2N95x3td;Y|RD!=;Ctg2^PvXPkAIS;^7n7d#l4@PJi zPSJ}?HOt*8Bj}DUqkwxr{}k%c#?O41CJs^^Dg3ez-qVFEgA?f zT~XcpV}d!Z}1)k}i0(_pJ-rZs2XWEc3+EwQ4C8ty1hYhf{$gqD3g zvs5i^zXPNKDS%TA!soXMCEx7A;VWG_dnAu7P@2Nnmba!c=d31vvnXp_tn|^C_ zbN0EuVj10uv}V9uC&ozc%&k4)lf1N4*{^=bK$@0Y_Wl+vN6i8L;e#!2J;f`PRuYyK zK*{*w8pEcD%=b&D1!6(`#7$p*+@)!MVX;;8U}vbQ+hM=f&?j1C@RepB8VQ1AhgPk9 z=W~TCwQN_JZ63ceW7!*a5z0=@zVZbSoHw$qpED5P26^p z*G=aQwqqnyL)Z|~DTyC%^J>k`%$a>Fiw$sUSKIBUN(7e0Y>u1KpRhFd|M{3x6@*=7 z`XnY+V@leQ*3?z?Du{+gj_i_wW;{%zM$^U__j-Xj^56z)5)pxZ#ytFe_Ep4w@*tP4<9wb}+${FP*);F>Iyt14E5&Q$u5H@6TNA zD>#04v6=3e*u*k?!Se#1blTEfay48-wFh*mpPn!E`LrTq_^-963!0MfgaQ;762YK@ zH3RcNOL2Aog{r|j5Qr|hrd=#HPTEiB++QqeAhk1bV2a9>eb=%%)2e~ifNI8aDO<4= z{uUr7sj$(V4IGjS+&q4_NhxFRJ6}iSP{WLfAuu#Td^A)peBNpqbY&~mX;OiDRKI@< z>{JE!CIJyZ0yFq@yQjzL8#`Bd?5_{y-|UfeZx_k3R;xsZ|FWuk8S@#&$l5FWOD;hv z^V93ZtcuA|m!`&pIu;Etu;$BOQ{y`~>#kgfpLHH$T=yV`$5as#D=k#C@{xVIi<{`~ zNuWjO&rLE*79Fw6uKdiKqG~iYEYq8u*}s3b&1fuiuykN?&(}g4FP#>yGG=yDuMTJo zY#W2>JQu3fb@2J)4ajaItqJwmDgGXlACJ-LkKY!N(ibQmp6L$bXm!w<0NxQEh9_8Njvr@WAe zZj_ZDPzIsqe~Ml&PXdfLRrPaQo@?f)%$D3Jz7+VVwYs6klBn-z(ohP|YJ1ylK`cpS zHX|r-blnDS1;~DZ$kuz7v8YZfLGaOBf zKuw<%8+$&nHzSr*r)Z}23om7R-nbkbBFp}PogY*Z2 z&gFry5=sDZPd<%fHB2#wrQsXxJas)9GtWoe?B-uIz)eL(bOQ(}sIM@nf>%7h&ZHVv?}+2qI;YvKs^)vQK3V&Ia@dmd&;-!;(`5s5jKW z=6$BUPFse+!4t6~4j6$Wer;^-s=}k-K8)TX#Wv17AXgyW!*XItn34Tq+v?esmV2(M zEy8D4dW52+&8UKvovY|y(sZfXpzP$R{eVlkdtVdGa~U*ZDWl*6_uRu<9sgPqyoxj4g@ zKt3w{Gx!E3E?~8)G@v;3^u^eEv4*LWL?5Sa7v9-2;2UzNVwU;k`W1CNDjy|b_U|Ak zS>?V(iMr%v8nWE|sXs0{JrB}Mm!`%wga>_>G9RBpn*w1E5p^nRfW(qnEjSB9S=Ukn zpjDLZ42nRH)x}y^I{o7J#B)__MFl;v)}*A5h*NU|`NZQZBUG$Jzt|E|8V+=C4}ObS{#kF9UT}xwT*(1^;X-$U{W8V+J#V-4 z?vD|mOEAmeY_s`+aoFw>9W6>UemgMqIlB%&OB_AE*q@T|aTiv-da&6P^uFlp42$6G zeU-2JY^7a=I%H&QxKEJ6{o*Uz=Bk zYn(Vb&cT_2tg|ocWEa8`(2YN=VS*hWQKEfL8}g^@hTMWI*7wh~X%V)?Cq8T+jig8Q ztVuFw2)eRKZhzKDe~V+Gx(elQ!94+!SIG3b+(Hh(R%chY#d}{3j-ioTJ)9KO0%dtu ztC5(P;f-f`vUMZLSVi?b9eg^Br@eYQa+ptgH5_q}Q0p}ru$izl!z^4y$NgP-Abq{a zR_&8utciXkmwaaj_DDjeGA;m<=u(Uh*k3d+b#7cAv^L{7hVz<0&8iuC^9E;*g=BdF zc}=48-M90KGCRFboIggka%uWc)qsF;`wbv1&{?X;*msh8bx!WdmUiDOw`T+Eu=Lt0 zHK=!yX*-8p^w^YZq00c2qpN#~M$W(tWnFx1q@|@5Z(u(DiWuL631HrCVaXf&GwTPI z?&00nfk59hWVScWl0^D=qw8Jd!f{GK$%y4)ehhmiG)luwhY%) z`NsDGgW+(oKiaB+;PQqPmhF{p`YTOZ-4AtFHB?t_HF*VRB0KA)c+p&UA+{d^flV%K zY9cf+1P`}M;xgNIw}8^d$TjMltz-|hul;3kTnGbLf2&kD)jqkiEl(mDN4<;hD>+d$BjF_ z=kY#k>SB6^*p3xRfSqCzeQy3SsqHGmxx?A|^8k!^2E3L`-F_m^Ez2<4gUuSqYEgEN zfW6=Lswh{*Mce-w8K)nd>2IvI^di>(fsr)smZFe@XyK}g>DFD%LUH*+IXg7xpbsyM zV_=#VXJYBz-pWTVj5QRu4k@yG>K1wrG>OY&UObFZ#0*-L?NQ_C^ibUkN94Ad8_SV6 zwGvc6(Gx;J4?0E{4<3+ZC>2F4P!(?3uc_S68LT4+O~RhZ=Jm3gVl(3$OnMC53$;L_ zI&`E#FCaTX1^*|{L}XX-sAVlxSJxsi(Gd+Lz&#zV>EM=}-iu<~s9IL^tdSb#pcDvc ztj(5%nv!o3C~4^xb>c&}2C+l`o<&Wb4c*=8_s3yHEwP7OukR3HL0@YS`>1ts{FbG)1dtELL?1*z@GK(NY`3+21+eJu7gpk5_zqwWFo9PtW065ay9m zl?mj0R<3UR3Xn#W@FwXC=)ER%xcko+BGkX1YIIzs^{WQ+H4bEa%heb~!ECfDY9MvT z4bM`*Z{V-wqCP%Yi}H&u#h4!uBCXxg2w`HNEh2EfG|{C&s_Yo#sWXwSSbQY+$Z5&9 z*9@w-!im#T+T&*4q|baB2oY~P)4_17O6BqILsuCFic#3MyH*U`A?J(xsM6eN2BJ&C z^hU7*=Uu^~d*io75(T_%Ca-oNmJRaLZQW%((ckWLL`w;kxte!wq~g1N8RW5}TE)oN z*xC~iVj!ViI!fPeNxRZ3>b@46Io|nEe_GoGtls$Gs1FjE`b%Sz3}%6J-kT%8W!80E zHyXfYc`o>905}wwhZ^*-bQij^j$&)9gjix}XueAo64xb2ln&MQcs^32PAK0#uG)?r z&e?lfrP>`_)o0;FxKR<1-V~2Dy)rqIn8oycEF#bOHR82sV$&#GjO9A~&zQ4m`=a@1 zp1Dsqv8U|PO@r7yyUmjqO9@J+-X;vyiYU*oiBDSIwg_D3hO@?;GXK)x8}Q@rTBuENNp{zsXjbUSU)Nr`6fc8NwSps;De#Rh%{3s+B+aUx{=73N;Ba z(+JmTEc@Rg8eeX=Q(cwk=>rF1gM(S~vJRGMl4Y`tureMT@w91RjBB5u77HKT4Mv^E z?smqW<#DRoAx+l&(Cs0p{0VtHl1umYBfsIOGuHGx?`?&{m<1Md4sIZE_FdA4^$&rw z*v=m7IRW5_Wwib2V}^8c4P{i(p2g- z)PK!kKElJGd;S)TEn7|`AHCOD;7+c_Xr)X?8o8rm62^vf$wl;~#`xMD8jd#GWwPyi zQnA_HL&71M!dZqn{ufQ@X@ItPl0#piJz~Dr<;_+SsTj@_Gl?V%UUc5-S0U?5pESxX zj0vY}mVfAk9$5B?ZbrB-Jk`M!^AQt+yHe6ChVZV#pT#l08J=z>g;xg_12mnnJCj|j zTF%)?6il=6vN^QHs&2sHL5u!uS?iK+hE(yOvq`r*_4vwq8U*`RhkG29?*E|ur3B(q zsB{ZA*C9I#3v718k&4+`cyW-P%S828Iwz(~s}{mw605Z0p$J(DB zgW8wdr9(X#I|KQN`ZBegVZM4=t++IZuZA z3B<-Hxl!cifz3u4T|vB-kGOAdwIJHm>fAy$d+qI~dMgPtI@I_u`MMC^`FzUI3IKjf zwRyE{?yJ7TF|l+U3YwzYmFQHZ=F(#ey3$MCR`|Uw^TwtN`6zWNZ%7d2Vz`CdHS3~} zN|5F76(e?MsyJrWNL{0mLk+KAOHyfh;7BJV&&7{O|G2?z$6y_1o`%t9S*jkt-Q>Wo zyC1vM!t(Lz!!HgO@jG~USI74kgWAg{>Q%e%K=a2q)!Og8NiyE#el9wC);x=3hq%Hp zgdC^al1FdcA-EMIiQ(IN+kRTJt$Wj`Wo)i$#Nh2{0UCXLU^|byNO$s@kU%x*?zMFRZJZz(pCWym8OQ`5jrQs|6Te;AHe-_)T7Qp8ogoSu9>K&VWKKK1@%COXR@whu>?6$_2VazMOUB$0 z{t#K$DGqF?$kAX+yuLXkt4cG64lds$wbTa$+6FrvC{kR;81lt=wbBr}UKw?~TTY!C zfr^geM?c-UpS0bJeo%ZAS&JTwK9Aqizxer%Q~ua_@7H*$J+8euu+vdP%Z-cwSUOd9 z8g1UPc)atc5+04gPrIh*E~{cY+j)wr5IMxkR+6{z?92(O@cBleIKx>c4D6!}kohGW zr2O2WD*T@et!5p&{n6R9w&JG}8ADGMbncLnnoL5Z@RbWe_#4-~RZU{n7pmXR-7Kgj z>cL1DQZ%LURz{>%R#{O6YQi!9>-X{ba&Ue{?+<^j&Fd$1do`-Z#GOvh!bm0|H zWg{G10gVe8alVY0ZSlhjhYcdXw$3v)$M%%miw;m0xlZ^cNQ_E)B>Tl@$))Ow&>+oX zX;WbPorL@vJu7g!fU--r!3v|pP`L~-r0yg0?ht3BkO zeZ(MgMh*UliusQA>juCv-0DeRUF0VW0cmNuzsF0HT&c~m3d#NPl|Tb{TuGkTL9F^+ zOiU#~G}G?V8G+ME8Kf|ABFZDtbR4J4duuN|QEJtiC*)r5h;3c`v)39_Thea1>ga=s zsm>o&=#(LS;sI1J&A3?4FX-R{Zm?c0Zw@&Z7I0 zo!@`lclFNdEZetIsEE z)FovIQ8(Ld^%}g5owHHftwYhHMIrvtWGTLXDJ^D_n%|qG`P_rdJeFFEfLxbXy4l;lE>A->37AWtia`1_xLrc1awZ{OY=b9daX#_W{W#=b5$A%s?g6ejHh zwmmfQkP@K?v5d)2JYsDfxO_7|(dBW95Cu@JB}4ll$;bw}S*4qiVN8m@v_f{iWc?c3 zxa7Q!TD?qBHZvCs?L(?>d)42N@eVU?@0D!AFSfqP`~W>F>#(h27|_dBgeIDIFMfc| z2BuLPggmKF>NH>G1zPQ1wX}=a{yj-2jMZgqw|%;42fI|#c3Fm7WGdp+#6Z$#Xo*8@ zu71G&usUwPR9eP6!Zai4aaz@w?;B}YD02^c*wE~(KI1wnTEo(3ei+G-P>^Wcgty!p z_A2b(BNeD!iIr5Ic%tTQ1vn(FlcwoMUNf~1Y;jL2AllP*D;-izf+lXq`-%8R|A>tE zpIQL^oj(Tbd9B-v?=0-z!a^`fSJJ=3_N~p3&bOxwU%k~#VFB9>E;ZDdgG*a}3?OH$ z|92kG+1o*)=^qF;d5%^)y8ig+$9MDtZ=pICRt;8LIg(?(jJZuKVR|3TT`BS4ar<7r zBvAlgbUqCn@)jxtyL)RrH3I@PsAOuwBQV?NO}T}#03kpPE!F4BYS75JGNz_%4Z-Hz zGo<6BkBriWSdO>CrT|AY0&;uIar}x5Jd5s};eqiRx< zhP>Eefmq2K8Uen`-geq3;0N^!VPbglm}OzQ3!#-7Fdz|e2(jvma%uh@x&ifN$blxk z9gr)Ai1cmUsjbq4ZqDnB`YskbGTug4zkbG5N;Lk+TO@7D#ZSX?8n7xIqLwMOErUnfiY#!c}zW^q2e zUD2=&J?) zWkQnific#xx2MBijfr}`q@MU5^c4cn^U>swOIc=Vx4Uvwb!bCK#jn;0Ili%?f4F9w zENP3)7OD4$+$?l~(a5Ex-kfe=ewW@ZyqTask%YS3aszt$o~FdNM@QPp-~>Cr(LxPS zbYR2q*d}Uv!_w@i?P5Q7DaZh-!*$sBa&$noIz+i^1VPB^ejb>x?I;){s}_JzjN`5m zBL^>uKSS)1MutbScJ4c|&_!&c_~*_@hOc83A@Zc+vsGghjjwmB_FqxJ zZp#~S+74PM%pW8bb2H)M+uU4(If^eKa$iG2xLKKkbM0vR(0H|~ak>i)fG#gRdJYZ$ zu4U&Sv0P(STq`eEWwtJJdfki^&a8;9WAq4v2ki7`lrX3K462jWyvA1Sjb2hams21* zwv{VG7nQKIUdJau3h;+*%U zzt*S{$X{o~^tkw|N5lU;L02(f;s2VTELw}@@Ab@bk#^(f+L~35$mzY75VviqOG|aM z(VjE|^$x^s@20)?19 z@>1;9i%WgE;Goc)&5FhCc`}9u5_v>pT)n2XTIW zMEH(Y0;}A$9n-Lbhtv>)(uZ=z{0w1;PyML~D}M5ESc4D|Hj?X-G&A*RSL^fKFkInG zBo3BESe%3S|EgNXg~rdRoFZ|zfDJoZu_3kxVbPI35Ht$#7>;BD5CQC42* z(7f(A;1=@xeD&$*`d)hfjSPqkeqAq z<5Td0sLn(ePPWx#erXU@y!!m;J$`5+3Ht5>;x3UYYzRI~2ja1ZOcTLtxMaDI0%IT} zqmP;y0@AxO4uh;^Sq)mtPk=Jtn-$oUPb7VckQ%R!rbdvP?La6udgDOH_t)211Or5J zji~kRSG10?(@n z1<5O)&{lzpEUrJZeo9Bv0^1le4+@Al^(33@T6i;f+fCI&-_KEvPYBu82AG@|G2%0? z)<9m(4XL#^r_*BE{C_p#=HWE?o)z|zin;B^ptfQ!cq9i_ss#6|bJTA;!uZ8#O?G%U zvC#O#jsld2`p{oe814hw;*Uj`_wN^ECjq_m&lCTQ6?M5Z|KGeym^qy`+yJz0en>Y4 z9M^oRd$-cB{Bq#G`+(3O1=+CL>H&^5ClwDTSehNX2GmFPAZP3zrnnkGtv%4^4$^h0 z39z|9U*QC-B2KRzWrbVn?JbBUE4XaLZMwLyIa_CbRSKBzvX=H>wU&u2Hn0FJ7`${I! zbtcFb>kGD-Lf8qYoZHJAW9~iF%;uZau&#ST)sokxZw3a3d4=3hgzU*QpWE{Qu zgp^k%sf2)OyiQ5k%FFR3FGr@idRc#~=2Mt zU`f`yTCu*mOCaI9m}uNQGZQzVPv5D{71MH?moVJv66sddDlB+XzZqrq_P{Lz0seR3 zT$wBk=wvTyev1pD4auXpS<(Zy0LYXgKZRO2jR~g$ZH*)`&RC{w%*Do7qGoe0L8JpUJ0W z_8!V*9`tn-AEoIvk&GF?9ZE1v+~EPMkJGjm%zI6SSVPe4OieldfG%ox7c0|FNQ-2a zBl}St8G!(hX3PHo@-Q z;(X&iglKA^kTR|>LM!~%DcW2#uUt>Gm9e!|-ony!Uvyt|T-;}59Rl5U35tQ$b{03+ z;DToL@q>Z@mn**yqJhj5HvU6Z%+~8Cd_^zDk z7rek)5;MPyy}f5#uzWDe7i z^}kWoKnSV~6I4B$ac0+Zl7vjN##o8R*xMt)YMB`v8rauTICxIy3Qkgm$|bOMxT>nS3xX|-#Z^z32fIqR_)s=Bw_c8p9g7;>^+t*~ zH=Z$U@SO(V4w=C;1Kov~GheUwn5$w$?|Hb=>A;_l`?Dfr^iB3l>X(%Pd!T{vUv23> zqjIfRsmiaqTPPvdHX-@L$(CY^|M{k`-mfMwA9NExYkv^49_zk%6GhSYD~#)*nE5Yb zB|=BlO4}DHTlCbg-`n_k-$4>FY4V5FWdk6jk-bk$Fi<~@`g8~Go9a1e#KTwF#^D!V zw_MR9eFU~Elwkdh^%yAF)_w8us44iTBEvr*kFxZu@%_-GE3hNQ664~x?(NhWSe9ZOW> z3(Y@G{zw&uf#iBM`^P6mS!XP-UJQN%>@e%Xc=khx&6S^hNw{0=o&#Dp5wlZZy+`?I6V|0)%{Z0}%dZ+{^=pzcP8xs)@g*T*j4n>U0&Zzhf)lVc$gB+hP zwU5XOvtEOj0U%YmUi)3szkAo8qnhUqimp@RFgCQSXCJK(p*x}Z4JZY#u?u@6o&bTC zbGu%)RYxKG_yLH>CLlqD8v(5F=<64=pwN&M25-3yRJ)ht+miP0jn9as?Rt35dF{_v zyBzF+_SXbaymqD2m$dqv1IKYI^`=49>s9_ikm>!g#hxjzE3rL$KAocDZXHnegtAe0 zy?QqV)Bz|)QI#S^TvI=Zj`pe#tYW9oN4*5lo_`5yeMR=Z9W4^KJyW@Y+s)&XoT8_u zCo8$u4^!PKz`z0gRdrveR2a2RQ97tJQ>vX0jFn{L9ucOo8u{SX@z_d`VxDLXmK|pf z-AC%-R)ear@SVHEnM_R)t^|Blt~rnmSbyDe?-Xv!v6Up8w>IbnM1|M{;g-1_OyOEPt5HsXRd|mNSHeHXp8W~ z>-{P*qaC_M0pYBx>>gj_*iAju`S15fex1QSoO-Xl@-{Ydjasr}9@XP#Mm2BB?3{nm zNTs;dQLM3x*s+3eE3oNYDJ2<8yEUUnY^t|CcX2CT`^ZJd{R27^kgGiJS?Xt6dn&AF zev2(8@nii`wK1HsN47Ny^UCO<9;!NKhp6(7Gh3(*;|cAinXo-jqo$&AGH|zQue0<9 zWi>V^H65IIk`}{i8lmm1i_4&}ioi^jN`;g`)u=jQZR^DxwML6)gd5OIjPNQv7zJT6 z9g|%UXFpkI+Egn^RVjgjM$9uIEqAzuI3dh4Tz5I90yYE&7d^(QYD3{Xf9O%K5gY%F zueL+_<^k*FCf8D6b`I2XEc>3g+n8~{1u+fwu*g^+^rvrBj!<-Aj(rR+5ceD{6KkCgSlvNV=>!w0hb7PY zHI*!8$}>QvKYUML#l}_WVhfa`B{H$jtvg);Y;IrjaVGJe0In6ZP|yuplY1SbPxM9; ztl$3$HS>5mNv#awYjm*Kbbv9N;I}&EfuEH}xib`d^+;{1eySZ^=W{Chz%tI(!)_y< zi18%+0Q0o{8AUk2#*h^#8kZ_il-60@R5FlG8F-s~q3WxM@Haa}k&chpzP$SRnnW#7 z^?qYCV52Gpx4$u(wA8Xqq%x!dqpn?A#EX<6fFKd4B!L3Q>|@nkj?f02)2W7m3zONa z;yys!u7N0xb4*pbXrIiGQ7k^J15=5YpoED|?brZed=<6w3e(_^!A#K3vw5*F8+Xag za>Exc6Jj_O8%TI_0X1WNghOYd5E6QW!2{?)2Vgl~K&FW}$gNFu8Ab1d5=BJ;K$cZp zTs`7I{F5(4E?Vv>D6s+1EZ)+r#0MI3xz^=TxtafS1G(2VpE}a?hSbH0@d%`-X*C6i zX(}N`mW?$&5s}@7V%~>LOAcGsQq^wjkwsL==KCtS`r}5q#v z&fRKig*|YUW7`Fls%C?fjjSkpK+ILz2sFPXN$a4F0p&?5uuSh;;r_w}$KRidlqd&t zCTYs`3C!4sto}{cqNN7mdaS-!8~KOZg_vAhok`>oIcf=({iCr|q?>45K5ziunY8!_ zz}v-jPB##*T^&KqrWqV{p!y3lT4D5kb!32XJ*bA&G3)NHp5f7_wB9pCN4j-zD8ZO_ zh?`Pl<3lZzwo9%51XJ{1ZmGK08MzqU%IOG~GQf*X%qSs^Oi0F-;a41LpXxxd$%5-) zEj{A1sfax;W+G~^B3;Dd_&Dpw&^ptgKP`5)EJfSgVUg#yXZE=m;u$Z~Wb*=i>)@w~ z%0I)}F0095onumTck#9UJlND2%6t4)hyFRb2=i~#tMeK`zI_XNn!Gbq%0K;2^R& zR4Zd*>++c2+5_Gl!P+1mv>RuALl^ zbcs_FE!2nzq)*m4$b}X+9qd@Uoj50xQb|lSPLz&G9tuA=mBpELZ`-n5T4G1kx5sZH zE=H2{sc%*`K;c7N&5DdRMmp{Sb4qW&*>{$J^Tj}F8As}9s?MVpuoBqIBD7QMdc-A$ z(~p--x@Gfr`yHM=IU9}amLMOTywQBD)wZ8n_sgC$IwOC3@JGDye!R9~hJSCHntMA` zLuYNV0?_dJY{PcxQj0^qr!DHWWXLR=eNFeB)~FRx#Q8O=n}8=CB8zW-k6Y})@b%_@ zqZ*<3$BFhpCny-_=URfUp4^k!(Ts8o9fKF5^ik{q#}(RTuHhFoc_n}5Gd(sdnXG~K z&_|&`+(OZ_M?eg)SS+8^%2m#1F%kU*4cpal#)$%(1f9Iu(cE@gHOPevIv!@X^?aw9 zGKSr}2(;E7tz%b-HWS6X8WP2(>1CFB88xyPZAi>?Zz?sNAP*F@J1wvo{oHM_VCE}( z)3SUZ<0Ho;w+sydu>=hnAg9#!SR1+OSs#dTP8OvAq?{@?!cM#?zEWvCw6;u}PEMW8 znU{{`NowsG9cwd}BCvphC+o9a8g4=U0vwI^w7SO-RZDr(71VlzP?-ncjqCfsZuvyY z)`Wy-H3&bvgM5?f1#};z58a^`k+0l3fP-z534AZ==Ltksy+_SH%AV~KQ21F7O4w=R z)}=&FKj>|6;`<|d6dyw(0{cWVAYh4aun zZ`0Tto6)7#sG8weSiMBYb=zctsw73y4xwO+i)O{xH*vEnwj9jnO z(1G8=USRW%b$8mN^g|`$tO`aH`~!i+ppROurVqidK!{*e#AqpIk5Tca?`IXWm)t-F z`sQ9E+#n&kYKv(|y%!Bv5R4qWXJKT@Q(2!^DB52rS2|c)O^^YnpjQW@E&N03UjYsm zdPXC>W}j2@v|u}_((9{&U@|KLr`#LWP-d+m67YqRVhmL?NU!nn+wL$=Cx!7?_ZIZW zrLX10?;77@oNH&lmrp&Q5OGKIMhBv1yi?2vTGohX~-Py1EiD zzO=hFX`KAE0!jW?pKoGvb9V%u?>-m&x=wbNjLj3VK8NWkF)Uam3nEejfErb!w&z{E zqPU7}nx}sIW9j(BJFb4ll35}TAraH?T$Wk17wtYM(5FkIQ5&~des3&>8T+nHB2zfh#7X#V01O3nV(8JlO}Y$Xdr)F)vWp+?)Q~|+fmof$09vn4w;lNbv5wN zoekJz0|<7Wnk_*8yh|^Ae1-*@djMo*36b4i)=>wH+#(ZyXhKJqcJN+%oHF1D8MH!1 zPPS1d`a07+Q_Qg|yU3voBPbNF5xhXYbUH^9QBk=YJO!$0s-X@&)2^&ZYe?~+)*r3) zUH^LSxE>LsA9nu%M`5KT=@HX>jKuQ2TBh|iP#>!t5*oN!KbatXJUdq%)~%<9i>6S3 z2H}^g{mP+p0jFAv3DE(pE#-{-$yYtC7)F=8_6yr>ID)mPTy?<@0pexZ`*RR27=+t(F=ttxxpwG6?xA1E zBcNjK{nG}^bYAjk1p&tr4U;Ek;Gk^nc5PB^K`&7Fz0uxb2AW{?_|{ z$otE7}h0A5kx>rK}rRYR!T)=Axa}9Al=fSbi+bLMFkWP3F+?c6pQXoMLMOM zXD*EG-uM6g_WqtP&;4QF!eU)<#>_Ft95ZKme1kho>ekwWjKR-b0=iCtzoyjKm92{w zTgX;A^>AE|cI$4P)W~Xr$#M-e2#XrMloRZLbU#9BcSe~Xuyl_D#J*$+!e#Leqj!c8 zu6DJ;RS8OGUno)43a11mbeJnx+omUP4m?*I(L2|wl$7O`mVWB$MWgzJz7?v1(JNRv zGGOjH1QPloxO`kGdVTSgybp)VSNzEaOE>FcU`M7$i$P`3rt^eaiB_6bwYL5mP2UseKz8{I3N{OA5x7?6cXGdGpfa$nkvF9zP5dE;e z5clP6gP5axh_H>hPJ9nAa_eE~lT^rxvbW_J{M>Nl<&sFw(=INur^FmFGVg+~CHE!h z_We*HilBNG&2wA*2*?j)F8Hord(jth<|EGy8}EK|?>GDxC3>g^7rb68hso#~twT$* zy#8={xohv@`T$YTTiPC#Sq*+ecyEj+FZ zN*s&NpD+5lEypluQ=~cT;`Xht@UlOOsRspEQ=3UjT4(-cutav%_g*{(UPhwKfwpN( z8>ZqXQNp{DXXcI*9XN$>`J2Pf`jVpe!q;Cs7j=pMkSEk1vI6f@d>-vh&T|oEc+8fT z?d7)|UW+oH1G3v5#5+SXP}b?cxQX;eMnb3n@>03_1`m>6BqW%gwXCC*!ZH9=Juv>?tG{@g--oib<7Z2WmspycKrsO=Qhu`&LqB z*JZQ=(B!8^-?MDxqVQ?&>Q);EDhy9E0 z+x3aqi=KER0cXCD?4Up1QnQIZRo%fGk9*=)+cHuyc#N>`a)$qSP6L$LHcBG(YcjyjzRmKYvImv zgBArBzif&K3jDZ2!S?tpCn?(_C+LnUg(HPTFHWW%6L+ql7+NNmMrhUw-yGuM9;X(( z`|*ozm%6pIaTZD1!|2D2QX*W_D@DvAL?z$6StN>L3}ibJ)R4es;*kMI&H(xs!zSesFs@I zc*iZtrET0&v!c;A4gUOY1JAd==?b6}-o?XfacfOC(C(w|4QuWc@#W_VJHwn^lr4B` z&|$JSHTxO4)Q_{$ON2aJJz^P6ia+#v?g+kZ_?}Zl^6*nJkYr%sAXS61AfSpXi%b1a zI`;sU9n!YNl{x=y|AWyEfDc5BEibSj^{jvAfFtn_y5iVk4Hfqj%52z^7tVLPP@;XZ}r-oMk~?sgQC z2yU}7c}Uj$U(cBfe#Jym@9@8V32&k81{6j^#!eFNzh4R%+k7Cg&5i3JefjT8)pG!q zyOAf@;q~v|6VWv*L6tAt%TxbzsSO4Qs4G?zdjB4`{mb?OBbyg@@!@|j#aRKvJ=V2* zhw4A0?SWB)+vjuN`=3jVA%e;6-Ut)n|2!~}5r}`=-+Vgu-%C|)!>Jx2T3ZU{|C&(b z>*g>My{&JB{(GsnS}-b0rtZlE{~gK5G{_vNQF8syrIwJ2)c^lY%JYJEKc|rt%hh=K zAr3YieNzS~3229+$zSCua0t z`cD^I03NsDf&ah$jmj|mekd%MM9L0}q089#HI7pdGx4F2YC1>SA!spf?T=0S)ly3j zS>Q?~Y6bv3_^@i0WjxeQ6H$}UP=B>hQN#6UJrx#dl|Jga4K?A`gYDwCel8Pi719o& zQM2*amcbgMl~j%6mXW6VK#=RUv77#0XV5|8xlDn_#-GjgkOX1D-XQ6UpHu5Z56c|? zRv-LWc?gB5VfOoY0BQ()1KnqHftyJH4$jE4=`n@oTO>V^3u4GPq*dAuP%L85%hX!c zvTUW5Bx3!;V#*%4o3}`1V1nSzq>M2z#ZvHMwocvEG`u?eKhEUaLiuh&A=<2lAxH`- z-D*Rs%}?c>z6`FZ6)2hDPDm{71E~U6$b<%7x-|&*y4MTo!nVaa4y}m!qX$s~_#d`b zA&3_XHOsDvU9&6Y7}+^BzvCdIWmUs)p^*)W&<#{GwS|gj^ZRNy=m$$cbf)fD@#)f! zZE+c=);+`d+rw#;Yl1#|l!^hZv0|f)7jn+j_{nPzS!3&fR< zH>I%>e|+rK4QB3SZa;~*Mt4`*H1l`XtJh~;8+*oQ$~wm#%j}$I^TP8wmr5gCZ#(Zb zaur65H-z%_)dzD8=+(Vpo4dNXbK}N=gE&MO_zy154;tkEy~T%t0?X5>_6laSVbdxv zb?%H?8BMVqDp$(Vy=9+}rT4kGCbTP6MGvyE8G&qSyl>VZ4WB1`{K|*J$4+WbcV*Zg zq;`#>;xP#zpt;sf$!#>89wi>RwKhLre!xUDP9jiURY>sq@Fa9fe0^Uf5Y1Dv8Jg z>)<&hWaL7}|nAp?+zumdeZ(Sb%#a@~Wr5&17^1bWdg$vHGNG zkZ{{pYig=1V;;pz!YGvqwBZ+0qYaQUn|*jnTr`)g$=93kEG$r2tCy0`!VH>1m*Rg5 zv#PxSifi8@y?6ZfcGzG@L$TYvgm|J8jlF6&V^sZwu;9nY=RO4)fl;UV3CY-U{ZEop zr|yG@q=>E=g#7j;V z>5+%i#s#q$% zUTwgI`0&ea@|ZDb>~p{Ra)7e%RX8{V$NsMWd*=0+ud18TiG!%x9HW*qdp2D$VnPY{ zWrH{&VoFI)D^s22uFeH%qw!0OE3EP|SSXCD)2L7@uuzJZ3ckyWcIC!NwVxl%iqb8a z$$pM6HQB3L^z_jI)W@@tQ88(11^HL3#|k4~x_1yz@xJven(9dy6hJeev83BA^Mos` zcYJAgQj?W+gV;56g=?ldQVc<)l3Nns74v$3p%Y=~dQry+p4)9HIK!wN#=@|CY&hDU zZ6Miaobb*{BTFKXc`jq6$j*s_)I)ApdvW}LNngJC{Q~^MlUMP+uZz1+5;78;4du5_ zF9SqtFUlg)W|NZ`b4`_UF&nDE9UM?Wm8O^J{}dvYt^XsoVF7Su#9gyzPG!9Ny7HE6Y&Y0U?4C)OB5jXWCu5v zF><52{h_GfXE!>rvLhX8OqP+C2LpM0l?P*o$U-h_3eHyrtl5h#oa}>rf@~HdHthP) zjc5amIWgt{_RLYE?ljY75Ch3HSJWtUm zv9qwvyrK1$1a7uJCRhMOIAk8x^st2KP91FMh~c&zEG_IIvysc7Z=fZe=U9Ir!>pa) zG|Ge&XS8*snSvoSN0w@bNy8*GU)QB6b4a|E4OPeZ^NX1zAUUfXMr(kP1esPyQF~xJ)26g?TJg=-djSJs0dMefOi^|@=`&6(hx7;P! zMS?kWjIlNIqO>NJ6pM1YP1)1jtHDEC7aYROVX2mScnUfhIlT39X3p$SRmpj;<;V4B z1k7=p&l*u-PK&^Zye3|Unj>TAU6Dp`lCu(csyn9biLV;oUaWR4QdqcI74%I)^@5mi z8L4(_7{)1W*v}I)l^4V9x@kiZ=1ozUbSLSxnsFl7{CN6%s<$ViOBE_<@JU=AZ^`ZV2k33;46bTJ#IDh~0ihVtH z@TzLhG_=aMV+ktT+ugDV%(%JM>U~4iL=_u;KEkEdKta}cqB$y^@BO2rB>n2bF(NOm zgd*XzqsHD`WB&=O8L(n!aE-$02pakeADpdz?>+n_>DPn#g<*ofXTM^RMBlSuvwzOY zCiqc=j)^m;DCl0}@=#}$Fxs-f=*PG(MyIhOkgQpp!8kmeLGsPrmy|rM#qZxRCTJGh zBvBn6Yx)smWs>BuLexs+KkS!zXDZS((cY8sRVPl%K?9N{tH?gJRV2&z za#;M9?d_E{5u9g9R`fXQWv@=F%DKDwm*5u>F`dG;Itfmf=ahc$cs>7`p{95U){79z` z7ku9l-hTLhKNMpEb45>Z)XEsUG?#r%&*Q`1|W|-K)>+$uaDE!=`2y;kudyb@$b^ zLE{zJlP9SDdd!;`56c6@M*7K551=L~PV(EOvfa(B?EDL+@xEG0N(i6wymaXBvCNg} zt`d*Nv4&9d_Cz^ehXrlZ=145d@TZqlY3_Tw>`!B<1DO@Gfj=3bHtk3bT$$}FsMPjj z^i!gdrb8fb(bBRnN}{+4kjz+ZoGad89*h2Ni?S7^6s5EfyH{WYPoj}S5Ohl^gZ?Z^ zZyAp$osWh!fv{e_p;(ksGl|cN6pDwPmwKx$bA2NgKmQtNRoF4swHFcqGYQcZ(Mxt;NV^-vT(7p>}037@(i)-p*Q9sUzQ5eXt&!oDqY+ ze2kL&O9-DOO`R1`^W6Y>XG2WWAFR>vS%j@jv{aL`seZ4etow5K@2P&I34!lK=v%xq z9vJbXi9roy%IZ2G-SnF-Qiu^D;Q#uFq<=P)+xVNOAoPaIq4Kc>3a#)O-luhO0dD|? zmT+1{R7aA+=h<}4vf@0;p+WPM&oZU4I{IbKYx&8FDZLO&7=!WfHrRsmpfv6$Nd~cZ zrfcx0$VPcm@peWDADM1X@`q%Uy~UTiGdR3|?-;sv04?~oOY&F16NqCk8Pwhd3~xQq z8gVGmyf43K)zl)KSuy#o*VBz8nTYGV@y?x`*+tglyfyIEjij8^_0DX%I)rCkeuM5&4+%+p#@o4mZ1C9OrzrO(6H*8iS4JMTohnqt- zx+Vya&6W#;{4mqo_86nZ?=|e4G2vJ1f;jA-bUbI0lP9EGp#ek_(J!0pwzD}}o2)4! z;rLLhN{X^on~<{#&~7w)aCoy z{gg%qnTSu|Ce9lpb^@0Jz|O{#)}t7zg9rRF9Vl8BxK8{%lQ)H7Cb@fqRbeK*@Dr7N zJTa=BJ0O*=QB)<^k_g~F|7vV!vL`3Oc7AZci-2kjCC+2oX_GJmqUAY97=H+sXDy4^ zW5{Z_pDo|!pA9e-%>;=34;!+go!nteFp;{CE zGS{XTzZTp6kA~3sAWIOift&dK$g1fYWj& zpzL-6;uOV{HyY5A&q8GrBE?>4(lOkqsEK|I$F%A4<+9G9d;WFTD3UK_`qC506dQOA z{mjj07Ec^_B&2=Mn#1^Z=pBXU(}BvG;wx_+H)f1}2V8GLc{n~?Q1zLV zaa+7}`V-lm%@wwJ6TPHF+)*?Igi#$^$2fh%lS2Pq!g!o2Hr##hRL}m~WdI<;LF3}w%4|ux^!L*Ii5C@+_4)tu zmoQKV+z@IQ;kKDH`u+Pk#hB6!K8yZ|MlB#E&b=ZxJ3 zM}x`2heN2j5Wm`2>S!(B)fno&GSxA+4T-cXNlQ6A3U{W0(`JQ>{#nKnx8W^oRIJY< z-W@A{A`#3Sk8$e2Tan06^!8;xTE@q!;#zojL34& zWhDkr&p>eNB7{CU=}ZaI+3bT}ZU$TP6&2un*_C&Z@Nuy*T%eD8=w-Si^t2qSC2@IWvO8R~yXsBGG6M;p zT*QM{dl8s}=M}$8xOaL;2gs%z@Z~K-v*GT0JN8?EM>H}{ceKgW^c!w2Pg<<^2_2!< zN4C65&b{EN?tF7KKEYNBEtjvSyVBGQ@iuc!I%*M8Wa$02b61Ae(8@7y>IGgKca3>? zv;x@72-n7EI#^m1ctL^j%`DJgJb_WL((|DKRnLo4T>U#>=k?iCbMMcR*VX^+!A`M) zseVZHB%0m>vx-{^5@cTE)^pQf=LX7qO7Dz@_8@ zj0p8o1k$EvbfP8ZiS0x)z06#NCiC0qCHB&V8~mWj-Mu<&E*;9-0BvrMLp)!ni}lx^ zUxHAgxUvbv-zIyYp_9g*LSt3)&5Qw>+{Xq(h=erG9Iw6RvVI@&h|Y;bwHeM4kUK3x z2_d|W!1h9QW4X)5;wp5(EWE_>C7xG7Tg>>!w~uq%76ZjSV7Z1-u2tJWy5vJF{&{qi z_Ai8aNd?v=W5IC)K-8%uvP6mChLKi~MZ-Q$?Z<$#I(+=J5ximqatLSo?SFs=9Dd50 zqLAP-O@>sWlz2QtF^9B?V9!Y(PQ!cvmFoBO6zwNYM+N{30w%0g7fBjxvR&+?C`JPC zS*O#8w}IwphHAM+uLK}j%xg0tOK}>SXbFFNXCiv~!6R6b^=Qe(!=8*X;bwT@Ouq8T zYjTk@AZxZ8l^ecQ<_)zN1Qa+(Daj0E&8)F>Dg*#@!reBF`JpOAs?6gXLt1|PS6$ed zIUu!b)tCookb_v_=N#n|y;4;%+LS=dB`LQ)B|fV@&y+V0%%*{f4Ejqu)idB&es4(( zBR-h5KVx(mSh87Y;8Oq~rZ}7%kFp=A&vY6*sde$XJQhq;I2Ru3LeiS}*6Qp){?OJq z&~x@ek^Z-@){xG0Lx#WKmhkfMfi0knM-~t7iSxKGsJm~y5~X<{z6Pd8vra)mC{u000;C!;$Iu6ES~{k z8a%%XH7ihGlT}S9*bkqNu72DSE%R!BJ^=@YSqaa!w@Fm@AWd(J21O{@S0SVvyQ;W? zh|~rac;BVyxBybO7I@~i{o}K`qa*h3v4F6r_JdNC6u8Db)DJIdSsZ0Ha8!rL9g1p8<@im2v zc+o_UOTv}{pKb>cUM75SD0W;9*<^i6AA_{vm(Q8;<86 zu5uZ>QLlu_4K!X`k}7ZpvM`;`e{5NfdB(LJA24IJSaor@!?&Yt*KRl&wVI#@_`LEAefIoixd6G614_+x*FFT{;VA+eq@+YLwX(JAQm>0ACR!~wSq5oLqiO`K3=TRL?0lT?_RPSD;)K%+d2=5MLDSfG^KwIa5_{o4<#ek zYi$BNS-C^P7aV*T$CVcQ7#vs&hnj-cE@%AX`-r|MK{(mdYZU-s`jR}R%$ZwDn$wtp znLt>cFa!*E;b<)xPOH@n6oW`|rchlc>yM&0kWl5-w4TTDXY@M0dbl;~&eD;L1as-I z)FMmL`XA?QkB_zNaL@Y)#Zf zoiI+WQ$R&~vDUeUt(eie4oMN?o@|4;9>U&2ia|lxKb>lpMM1lqXQgkd$4i~!+mjv~ zWGPPfGD0gPZ#*0~Q4_GsWi4T}TYX6HMw2ER?`Y3mV|72AX3t?krHWR7`);qlT|FEFQ zVlX3*6qV&IpfLcF=LJb~KOf#3Cc2mneeRwG=%X+-R{vwX&bT`>3jZcpOMZ*n(4zdl zm=K+NV9I1$LJ5uVRXf=2C8iU_l!xQOZ&9M+;@Ws9n!pF^9ZwqbeRr}O+K%I^`%(_b zEX_bX`2I2eENCpO1)OuGoG*!!s0(gR#8`B2Rm@c_d5>J5GKi(u26tU%{E@dID>gEe zwSz=bDD9>=OMZ>x!f;i5_$_c75f{2wfY|RMFh<i4Y*E{(v01Hh_%=PWW=b z>3?CyLI71{(2~bp{Msqb=-H2y^x|5yd9F-#HEE#(RG*jwOV&Gz5~jS}k!Pwr-I*3@ zd}afq_L_1ZUi>lYtL?D6uZME# zc&a^?+~VnKf=r8>y+d(@I@f5?2m~_c{^r|Uxtr?qTY*EL^SlJXKA$vaD!rQOZT0Gk?W3=k zG=;6VSv88=8y<`_1;<>YRE?*(L3uk)N2lsLA&Y`6V{m}UZfVyYm0V!y?16@Jtn=q{ z*U_R_xlXB+FnCQhFwLYRIX$(HvYebXrT50l9m^E>1)4CXwmJUkiEX{m9q znPtDqsGepmLWw$#6E6Cmr#irF=C*fjPsO0+evVB-^q}a{M&fy2n#bZM_-Va=2gP(V zNMyN69`E+s7ysjlDiNDfN`|a_U7}{@J0V-=HG~tqqn@gx{wyD4A#uWA9sTuJ`ptTu zeQf=&Q&f)@;{5UN$Tj_-(Y=td&S>=i-Uu@W#9VDDZ`95I<1PXKV+6XBE3tdSy#roUmO3uYz;Wk!VPRph7H|3XfO;C(fRh--rnaTN zEQ1d(gkq%6w865cYyY|yMhSVaBsgu|XZNfBkUTsGz%j^O6|8l{f^Al*lS#@l~?Q;b#i`VV@apLC(0BkJ8%+e-pse-D~pF~m^q z4VMuX^2_-J=$75LI9fLgLWi8k_!PZJGImE^&e)Z6>mxvX?3$6NE)Eo7L#)ORdPNtUoPDi{_^(zDsLK<8x%lX3FHtwYgao50a7Phv;E72i)(u*&k3R*aI(X z9wX|Tk3@7xnmD~LO^t60QqSr{=@4Fw$uJH81<5}iX02|cjbZq6iJ}d|RS(UFE3rub&^HddQlE&7W@UN#RH~#+)Bf6Z& zp*Znq_=!HH1CjzZr0d9euW1m5rY9({J}ii$^PfVh#7Z=;B|B%+R84;$pn*65DMyUg z4sg=0*@alls!M4h8P=ks%vYYjCiWGHcJ{;ADBG_kC zmZdENivY=e22|_vka!je{3vi*4-)vA43(GFjjZjqG&7^I(7WdxoafzU_JrE*ewP6B zL7D+kq|A)du1NC6&XQ*nbPa!sADcK1T%81x_??c6y=2mnep7rSP+7dp1wv+R;H_!{ zU8mZUkRAXWEkAYoVh7NwU0h<-&Hvhr+9JMm zgJ{HbT*Ui8#T%z1hvx%&YKkoWmC3gA`Dp|MRUyCAy6(4c`#5*13rj%n_!rK{cXOa- zxR6GK(rPr}&eGQSMiZ>0(CL@48}{7ea)Yss4%c_r^|oLa_GhM+e)e}9fH2l&Y4zN+ zT=NQt8{{I*D}^-m>Uh9Q@|hH@heDOFaOrzOZ(Jj4_tjIgT~no&^s z%1&-m?byH%0o+A4rLkMT0G=~I8T*oU({U*GX>`84dbNewDC-#7>;rx`ymF<% zmA|6N11NDIdL>ZLSdpR6MnQ%1hO?)5in6f}6Izz7&^Z%y0MCFX|P%Tjwf9eMxh#d(_wv10XG$`Kbz60U` z^@R9uucmMQ7I5r4PE0NM(|4Cnokwi+Gj#4|AfVa^$)#N*hBP>rxpo$KhhXQ`hrDjr z34EdOw4h(ooynh(Uujq$5|mk^qdVKh>7Gng04RowdX;?R^3D5-70V#g;6J?pq*59< z9A208h3?kXqDyA3UynH5iO`p4rm0c)z4k9ZaMy!I=$ZLtPk5?W*2pV%q^xon5#dNk zgX(0IZzih|&mdBOE>nQ3Y8R#UjsZg1CSl* zTw!e`%7$!5wT5zCw0xgvSLVQb<4$fvIrYk*dp> zj*3-GMUqw}(Dg*US!9cp38|Gs=pb&iL_qGN;(w^(++*9}&U40iA8dg5V7r)M&`0aG zt`npsoRfQJGgjPxQF4JMMtCE}(|2c8AE30s7w!pyCQ;xWFY z=I{LEOKS;fd1g+cfMn5a{%nfBanEbwF9LGHmkj;RhvMw2+xMp4;w1~oKI4Z$2<45s znDbvq?CpPZ${Gv6DL-=_7;Xd~Fn~57T~CCpH&u2VLmkg{T?F0Lw_1H6BZiZ$aq&9Z zyZzUCGolQp=wr#Mwsn+xGrkkX$p3>f5zn!o?cIl&a+(yMVwl!gX02r)19pQfDsg`N z#sIhH`LCy|L102%K-bC7K`p(Gyr-XY*-QQ?ulY_-k_u+^M-jJ87m1Qfsjg-1k3bmzeFCkQaq zM{|Gn9V-?eZ%)#$y8)5LAKL&^{R$Av%}e_+i9AuWesjoam4ozugLuG>Ns#ZolxM(1 z@z>3NEqB>^_{{rMK;8cV0)rv@%A}y?Wd#gn*3keqwNADcO*4w`H*f=@F<@{f1*9c1 z4pi{9PD9A4L}yaLx4yyFvqoVrnSp6{b8Abf>2N)GU3~MzwX+@=BQ#)k1Hrv*Yolq= zkI@W~YuSTsvCDW1&S%TY)ywqJ(W}l6BZgkPx<4pb#=AK?&Q}(qJ^(P~yX3-)1E?sn zM6HT)d>jaOF67$A>b4LyJKv?nT?uE5Q}!65np_#3VmMFqafoVr`%VAWbN)DGI&Z;> zNwU$Y@&i~pEl)!nBHT0#u)_n*nZaC!Uim(ESSe9XgelJ*K!gx2_fI6{pg z1w-m?xv->`YDv$o%y=^%$UFYhV~=L8 zJ4?@KDjzYV>{;^RBMLKU}IHXrJZtO~tJ^j{X8J~ZvyB2Ed%-AGc_qQm>;QGUu zM?SsOYn^bpI`f76;0jKw^IdDLiND8Hgf4l1T!Ud;#WOH>++kd6VO&u{(0v)SJgQg# zo;@I~G(xi7un_XHS8ivwlYfD}V&gRMA{)UuO>%uzVLQ zDd8I#oAd=z;61F79F03$YXy)2Eglr=Q4f-0+X}m4Ek!8*13ytTbg;yUkOYU>ZRn=* z_cf6-r&bLgV%m`@3MH7AuYD^qIC(X@1GY5IEMWt#-PJTdovK_0)HLWnWk34(TS7{`e^!lvu6U` z7Iv6`vhYd1_$`WNXN7sWdBct}L&psAf!s`GxR1{0<&0P~C`q)!B8IjZXKJLdQd=7g zL;;V7-pDgu60Vl7Ji`J3YK6zx;Sfl&I=a)_kcgTN242+rlo@iDI8-SfVz2vz=Kgpb z!>-qC=f%tTc;sRSQ!W{3ro0a&Uy*XNBvtfVYGbRMfBE74_BLbwSFwAzFkYu-F&Bt% z{SOFuzHbj{kGt!HgWfyLt^dU@3}}=R8VYbe_Kh9~4_RG@+>I{ER_ zWd+7Q`Qk}OCj=7K!UrEZan#DqOP7g?2Q$E3>P>u^g1m;Pk_xU1RwRW!5OKoE{yH8+ zBlLB5@nf>iR|_q7sqW*`md3SXR{mAr37;&@1q|NU=bm5&zc&~^ zty^d=03*6kOct^Od45E4xrTOy0LYCzoCJ45IIk3jX54DMFV!OO0BdH6Sx+|m!r2le z3WSK)66_KEQ*zGQ@6s^lX|~IcpK3c#d2J7h4X_TL+TQ-8AZ{M9cjTCxkH39ge1sNC zsHVV_ckCQdz{|8ZF2P4}7%Xo!)>DX?LL{CNQN&=@m!IZX9Ws*;;b!%?AI5#w-K$DOz##+5v*>C>f_03dInxP= zjQw1}J7i}xFkJ-C;dc?Wdj!*9JaXQ>PjGC09J%ZQdaNnJ62x=yh5fXgSe2~oEOhUxxOX+&#)g_KAL`IBbhK6u(M>|pwOIy3{wkr@3X_8H0fklkc= z=!5l^QE=HmKTqa!0P+;LIA~HjBtM{6Ldwku9?b42)7upB=#^$(P$g$e4rj>B^(b0M zJV$`1Y%OZHIfGrJ_&dw9iuq^f!@tPy5md`rA6dCPG ziso}IRk2YFpi9P`(?uBPyL5xJj^AeMhM8r#an5jn*Il$ccmQ$zFx@Yy41p=+a@U=3 zP;S0BDyNA!cRDQC-9kDmSz%#G5MMIA=S2mxL`J=vB2JA^6Es%adP~tv@%0q89$5+- z<~NV-4?aork?V1gTe{3E>TwqidaoMDUH@ZdG?4XDEiOLD85L2|BW zbqR>7OESY?|MP(TD5lqOn_T8fb-fA<72ICDLFo{Y=@9 zPgHZZPmWVt%_`p)tFDGT)SG=t2Ava%l01Is=;pVRQch`c-!n?)iubh?bk#U`nY6Go&{Q7=XEP0P6ajmb%`9FHUxB4L%1Rf?wy}XoWz(j}& zreA;!*@tMc3OMnOBit`SvwYeoftXwm?!Xao1H~3K>ke^3fbfEWASuIjOU+FR%+&zm_`#Leig9kq71#&f@J8)TM8oaDa<>h><0FY>snD3S+%% zY_)b%{PQOt>d*G>_Kq2X6*w0R(is0zYPrN#al_xoQQTJxL zc=QtaO@Xcb%EaEv4GbR@?s$v3PzqfzD(aO>9;keEIHy4;xA(Z8JO z6fq|9E-=_+EaxfyNiX2yxd185Yfu|7oQ^8U`*005%q3Aw*)kZtKpSj8P`InO^($hm zJhAr1y1dSK{GV~+#(}xMhnNHN!O|o6f{hE3s!$>{;wfu=9xnqu;uXNQmF-K3Mm z0eONY1P#jNt(uVHwip5?W{JA(3P60<@|_u2{q+EX1NME^FT{(L#cK!FVi`AHQ288h zpZ$<)8Yum%gv|h8dL3nov>ppK9Ai=3+ua!`clf+$a1Y?Ne!v0Y>PyL2n2!A(8X~uY zE8B9+%IWIsERqC2(?DN1!K|F>q$7pR@DpF-mH@KOEZ|((ADA!t@Og0m)c}G&pCg(_ zB3@?2eTB&v%7bILwIT7*kuVq-6V%dplg#X+Gh=R0sNGK{v009EcOsT-oCIUj3Px2z zHEioAe|{^U9!ji!$&P=%B$Fn**2n3Zj-Nk@ z-GXF0v+DO-f3Cok4OcirV)g6c@YtYwtJx+DP}{Fbmputrcq}(^`sXdYMd1oRE?bfP z^PJ~kfiWDt#ryL&tmu*VQyo1Q@E;?MVmHI|4A z9{rN#Q!dvp16dT&EfCgJ<3* ze-ZfxmKDq++93An{vgp4tpj1o2E{YXcM`Qn>z0OOa612(Np^G26`ST{;tQg-v zLygtPfnu3srk$z50&u=TjsAHsN008pt3Ra*O5R^N`lT>#FD%dC{h5qMl9hxwfoyf6 zE%)pza6PD1a0 zZil4Q8|`A+kv~?&0XLzK#06kMV{LYjf(+Qm5v^C;?!8Odg0c;|t1yGnZJq6;$Q08p z;oQw<_?>28V2;GpgS=V?k%WR?rzEh9+fo$5vVFkf@cy$v4m`k~Cm=NLFU*2kbYrBj=+u(ey8N}X zM7Lp#znUqVJVxHkj~c-N$vfkN7qTzL9fHS0%i1!D06&J+c&b{8R5##@neF1Q2 z37sG6%GAITl7fJYZ&V79~a zqhpcT&cZjk8zhfV;ZQD+wK%#r!<4Lfl|sYjmF--A4|NwOryxR$Z*x4DA_JAG#-1OK zW2Q9v9YT=7Xet4g>QEsQcP=#%K@ZX*O{XEf#xvWgXzdrGW`J+$R*7B{q+cSD=8RcgyCR6hyIGjfmSaapZ}(z2`0RhF z;ps|{IEC=Y6?93;{aFv8z8J^Va`$Fw+T_j_BF9^$ebkX4YtmUQ&z zDnJB{TfHw%Aez39jFB~AFwb^vvYC)+>Q0o4nSti!3A0dFhG>uZ)FG{NYh!K=#969P z@HDUss2Zv18(y3)h2D`L&_O5IglJDoyqSNGxKc4hynB;NL44ml9U}wt6-6MNh1jas zCz_LWLJmy;`*MA1rGq&YyAmg?klPV~FJESlH zDuG&_0@-wbMDX(pQ4_?5M%3;rAOjFLC2w)S0!RSdfH#%Jw$n4f&sh1ES})E(sKH*j z1e0F$4YZVnn!-*$dJH3qtd~zqAps_fFA}@J@rXqB6L5ka?mZnuQhgsRg~QpO08kcM z$9qaNh4Ul#ier|A+REvUlpsW#4w=BRLW3)izZuvTx|+BxLF>BwieGe`Q6j`aHR;A< zyl2^&w1BS>vETcbG9CMVNkRkccoN@1Gy-cRL?a=2l@-n(GUEtavIs9)6N98zlE(ey z0r(H(RD3A$@kr&f_4+zsXa*F_CWEJzAB{~bb_;8Lb<{T%=x+>@svf<&gH{KDKT%rZ-;~$#W_k| zv$p~}%WX(HQw7pE;2;ew(&?meQp6K7;DBiPjI;3M*m{zMLqZ01i+nZprqmMq83ZCb zu(mpuI#W;JSxD(3i#*z$#~~qFfCvY94I59dEy#e3B>0{C?gmmW;2z{$4i+^{=;2{r z2>g&Ly-@AmCfOn@1B5(BPA|*`W3pM|h*V1HqTJ}7Dpu>B1EYn!tE?_6^{bk0dz@8F zu-bfY^qsjzX<{Z#Y;Z$aY+#aD&C&V@LBXOIYSzZM-Jk~1$Os4RY1LF`nyD&dd1UNY zy(#Hj2=~=0+{z#W)rWpcdW<%>P;h|pQNUjP9< zmx9j`>IS|4c^~q*+H*P0Ki{uIZ)2K)gwHNzB!y*%JkJct)Be*!f{^dJy~?BTO7e8@ zRc98`iwB)v4Y8~L#Ch0dJSL*pE1x8+bT^hepQh|@DLOv^fIPoF!OykDghPs}JGm~) z%@DYc$iLix`Bv#RBK;%#{c;a})6?%1K>hOx5qXR=zP#H%Pl!CT%0oHz{fCo868pf0NFJ_8Z+>g+0I=NCHwDrQw=s_ZeK+2&$osvFtCHU`cP9u z|MLqM;rCv^TvGGf6%YQ!!j4Y8cUjZ?hh{VVvoU!r5Z8f?aS`}&lkr=N|~Rk z-5&}fG(4D`k-^j-{HN~+-#=z@%zOWeL~x*&&?{3b<`bZ=#?N+ktLO&MZ7 z*Ixt*GOjf!oie7J^Xp~cO1qX67~YTj0f!vu#-@CbO?MyvInLP%kbvvyu?V0;u=tn%wx1ZvAc97_6FDc%Q$D-DHh>YH1#&+mL_b)FpDXm@%2Pu6q~=1L+?xmJOBXqrYwImVz|?jD`$}0mVHUig#Z?i~uaV_UPu4tc<~q1~-JWnQx$sJC#|xLdyF6x8 zmIrq}P65r8S>u1c56Z&s!IyH8Vl|}qJL1}qsuxD6^rGgs%bgo2DM>*J)Zla|s{#;h z;HIeL#3Sw3H?i4dETngN)Ux%(&B7yxi{`XX>?M9~0X&Ul3+{Kd6Xk;WM*6H8LNR^1 zabEVFR!}Z@l$PWGTIvK>&u^awg17cY!95pcpF2}R zgU$HImIoz+90X~#-E>3?I{XAh=fn-A5#{UO@F}=|a2eCk%_=N9e`3n0klA|ZNP1tK z7ortQI|%(jwz9*cuAt?ms5B4*#d@odHZB|v@VwF|0lqy4`dZf@a+$hf_(G@?+7)`6 z&HVuKun(!lnWC)wZq9T+1{0tpQUdh>v%tjJJ~4=$ZVcx7j7aoDc}(B%BO+OlHl9T) z2f%5|%=t}4X$pa~uiSZ}95~$oC`xCWPb&3EuVkJVj0*(`%N9LS{D%KQB*-%aStdO! zfKHtId|=IB9MH=fy<1^v6~jZ{1(Y5a#J97$@e!zpc{sgi!o5@O;s`unge&l>t4L@9 z=*MuAKA7A1HPV|w+!zX<%vYhIcCp8^+eq@?InC8gCR(38c974FXaba!1xP#J9AJn( z!MWqqU0k9DiXI}LUyE1K$tK)>r~Zz#P_9|RK+=4pVT|aSrb+)f$u}*0X=?|B7``y*_V@MYVmghK42KqD{T{LjF zz=XYIZK0hgIE=l%Y?9j46YGZ$L5R?yt*U2G@5-V~#Pn998gM~$HQ1g00nL=x#x#)* zleH`vHcN}!(0L{E)^qZYbtn4PKv_~;0_?<&MvB>{h9ITiIq+s2C0?{P}J$7v&zdNtEXEguS>9AupOMy{& z`nND{f!@+~8bB0c86GI1XJdp?^Y4nTB5&h?vs|PS5wUXRYBPygM8?XMP*SUdPB;uk z+R_fGq#TuBBjh10g#sPN)F9sqL5-#y!zR0+x#lS0_k=P?-IDWM(fOn^Mz`r1IdXOX+@=|8;_0f<)DcL$hFZE#J{31 zC=?B;P>PXS7IoxU_mAnQ;=Y)xY}|14!~IOz0=KzZIJsq(*f}n+A^>ST+gs+Q25>uMIw(7-iw=%YroOGqEG(Z8Y#FA6IGa~(nWbK@6ZC$m2lnRAvnXzEEQ{J&;=KxsaNkvV{DAgFS<`2h zvRUH6@d@l87m2VW*yua#qgZ{R)tmj>y*w8BGxqKYat2ggHr2IrqYR?#H%_QeFC*WCr3Ro$N} z-?@e5*7{61IKne;y+k@+G|W+=d0753pwy4X$}d$cp|JW*<8wB1n)9K3dG~c{T-@Y> zku~kEI(Mz)rjT~kB;AkR zSw)`mWrsJ2#pojU>KV;l&wY76$BkFt?=(wpTzVu34S;j)tUxOKrG~FIjLzn zq1qHKfRuJ;b!m0c9{UMezzChdTB%J7(J`GPkl=xc1F< z80aOx1Bd12LmSZ!P``!`5x`y>xDZj$0wITNk zVEm*>B6Zz5f=-!c$eumpFj`nyLHS+FNm#xC$SmFK=Em2Cq8LFjZPZsIU`w zq3X@pb`dnfm}!;d=t!Nn_?QNrR)W1$K{Ua{D*Kj248R|2Kj zX;6oLtB@q&SH20-C*Tz_4@X_zhmtKP0J9&QErWK?J&^jGM{&Tg_!@f}tSGg}Cm%XreB@vZ@igh$LcQeTI zD)553tt`*c(}|I-kTH@HcutBVKOcv<@F*4V?Pkfh2JY=5g%Ff;+dyi7tGkYg)d%MYg&Nub> zet*yNdY;$wPp@w4x~}*29_M+S$8nr)dyXdrZ3|aP%1VHhTws5eV?W<>{U1d01Ni&0 z1+!E^ZPyF9TKjx$YY6epcg|vTx2(YgAece390VqfAFZl|)X16iRUQDwFR@-Mp8XZgkMHLpZepq?h&X)p|5~=Y^e-m8&bl0H^ z5w$Tn`G=Vh5fx$?KuM)r9HtJ(Y70tJ5urvlqjg2*12;-06T6p012B?M!Oao|uFhw|8Xd+!*>2=|q(JuJyi6#p$oSL2^%oRA9iSy7Uj504 zPq@X@((M>JE?^E$5?e)YT>$GPMkfLB8L#sf=jC$@L+F$dE?K7LqAmRN=sNLG$*)RFORD0Fpf12kZ|7}dZDt9 zs-Ulcp{K~*4lQy-RuHB$g&p=wXH5IQHO3lK_E)aL6ghvxzS^Qg zY5E_oUz`r?v>RSUaBQMSI}D>pANTX`K)}qR0;C`wFHmNGAbit=*Y&S?(Cfg0s(%sB zD%Jh~+sS{$uSSOQ&lQEEF2ZjFzTjBfy8&T3Hxx@(e)^9~1>F_6%eF6d;`@UpiFD_d z@}qywCG=1s%bWJ&R1?{~Bay5VmUIzZ+-@o!35%@b- z=`0Wby^#ZMap-B}#Bcri0raSgVV2_m`GK!+CuDt(3HFA|Ef*~!zN7*n_17EG34SKFiPP$6&d~#@Lr=KDU)!TSFCp=U0EfSrrS1~ z7QhBomK8z)O+QmuHFJWbv|4n2(PUNax!(m+do;Hr3o0oylY|_7z7765G{0E97Ca&l z3_``%_s^4(I4e^%pdTn@J{a65irQifjS^74)8C02+=3B`z2o8{Cwwv-jKMo0$l zl|mh2vgEKjgcWHgf5X{h3e|ypI8Wf@>&Pw_R5{?5M6|H$yqa6Ow%#4YrW3im-24U; zvZ*`cB&hv!mUgbJywNeO4Nu9M;I|F=Tt35}%M)qc1w3xI4-D7xkPi!L$z>ofxURpP zpAP8uslX`LhCnc2a=`@*Hx^OA<9bzsQT7AS)?`D;9jR2<4r=_AsCb_BK-x%3LB;S(^QP7N(P1909k*j@3ocBFh~W5vhn<4Cr0E zohMRByVL3<-#KpJWZpUc$jN(^mG7(LgjJrPj{kYIyWo4tQXD6YcHQw0oe*eRbvTfG zNV*5qqkX6FDZd0INJEfv16ESBi4rsHqIVginj1H^(wbX0_-EsD%3cZToH*iltLB7T zlrIl^hZtQP{)JM9ytzL(3bQ(PH`O=)S_jZ(NKutoFII^j_67|+C-2U2Bd3g2C6oE+ zz1$1FY5$M10D&mu<3ROGT_+4%qD0Q6(dlA93G24vbSWI;JVVs-_5`W;lqa~mh%K3nVh4X z57G$yK>jbjDlCa{FnRh3&iwStRi8zBzo-BwK@<^FR^((cuzSh?HlXn_2WQ`xDms5T z{oprwQMfdjq`n9+yGR+w<&3+YPLnO3R-_}^V=er*oQjbN5Byd3z^=L4C2x9+&X=`w zq$*#A`(f`Ry3)dM=~80C+DHGjU8N9FofrFVankDW_a`^3K)53uMfWdSwv@-j$X1<= zzPXxzmR^F+7{5C83!I~t?Nf4~lQ`g}aM(`lsFqV*;ICti(Z+X}p96#VIN`HFn~xP@ z%2%1m5M3))7qD9nt*H0v<ok5^lYD$v=8 zQRD{N4aEXY0N%SnFa;#KdZ_68dYf1QPYte022?i{(UyMXtHl`SDrhlnRq$boVu>s~ z=qGG79HnlIH245_f>P#QSgqTA>b*in5Ay8c>+@0q^d*-d-+9iaHOjDqpDOPPt(UP9 zxa|Cy8ynXLxLftes6q2i~h>TkeO}2a*>1j{ZM6{lnLHx6Pr)@ zO_KfO*R4$;Dl0idH>J(_greIkkv|;S28U<(hZ@JhVr$Pb+Xpx_L>e9fYO!XrHi~5&;M6(KU-aQdR+I0x4y**$~%@+ z`NA|uVPLq4P{A!9)$74yCrD`#(dKL;n>s#BvnPOcW=F>&pfH3FaBt$}p+%G1wPGkC zDi;qzaeuC<1OhYV8*Cj7$jbs8Dn27*l@DO8bO|3ClmZ*QJjMqW3~3O4BB}J|`uN8a z+P%A{TO9=k9{|VZ+88ukeqah}kSCcxGIc7tp`OOzuVOxs2}Xp#qPQ&y6%t>3cf`co zJ3(PDKE!HoHMa+%gPG2STkqI2!*Yr91Jo8F*fuli$&RmL4Z{BQ0zm$o8IL}w-MfKQ zQD8m$>a5+cTgIa%2!;_nln~-z_oGphT`er_7Xb$r zco(!iL+<# zhnXN!SRrJ7sK5$k#a1!|&xk4N$&I8+==qhmr{62dXHUE}65f%jUd^D&YWlLHUgG;y za>7r&spBdNH=vG)NMfw@V2jc}+JzarZH*F=PYBkoLcWaxugJy>gz#(87OD2;*?x!- zhj>hTQ9Yr4w5!8y!|ZzkaBR|~x~V;ED@RAetRuLN416t};>v9I`4AO*Jabr`nVg&dn7VFA$*N_zSeIH`;B$S1 z@WXD*Th=#@lZb{xQma(PwZ53&_IvTy08XR8+-wow65(CdPs@H*y{aw?pNst!s$6{= zaYr`6H)X;Gc|?_!!~OT0G)DX?@akpKZX@NuC~zh!fc05_zra*2N>Gaezl}ur8-DaK z7FcN!n|nOo6am?>gJ(9jcNcs@el(uQg@qgD;X}aMX{OVl_yMnn( z&cjB_(&z`ggV|kkc?Gg|-USUdGt?Hra^5_*WoG!z>`rVJ|I$`C*R?ZDoH_1M{|n=F zzog9G(6V%_a=>~1tz8p9pp<+GUWkcOJ$-h>dkE?lMcx!EU#6Kxnu?%PlVxv##eBZ1 zjs<;UcfK}{aXaHUWV}9(OqmkW-Y4I^9$RC0*0_jOufz2c8;{xPtXf70*)0s;p<#O+ zI328Qx7<Ee&rOvN^44f`Lk zSDoy`fr@SP3=Iz2hy;7^+2aWmFsog*iX_36g3*Hy!gy>Ya`{WNr+;F^mPWae0rRwC zyuE$9l{B|MgPi63S%Xf?ndFt0b-0s$f!vv5HGG!ZxO^2yMkB#jz@F+u{FL7f6QF`Izgew!!snD8w+ACH6U zpO0c*LIv1JJM-vCex`;GO!F4%=D!2oDFE1E$LLxMV1hZxo2GH2?iXQqzMU~;czy(q z-=cH3wSCSh>q?}iCm*>X;f8<3AS3Rl+GkvzFht)D#^3O$1vg(F0^ z-Wi#CHnj=B^B6Gb7_GJU#}^^ewkuQU-_9jdV_zR<9FEs};@JkaT4Np!_GNKelpV@q z6DtdAm}b=|b1-C0pbL{#od?BaD895qGh_Uc;Nm|b+Q+hn%`DAQ<+#oJgAu{nWwfp$ z{Ui|w^PTv7l}Z^Yu~C(iWccDhC{j@oY1${EMj1l1gESTpH(zzz+1}Lb$zFGPYJA!} z9yXk!1JX~j7};a!D+4(Tyd7qe4*8Y)`cQE(Cr4getc$UCB1?Za_7ub2Xch<36jJ3D zNh!QD7KA~4LC>)jwP7k50MGQ{vRgUEFGzSd1d6*I?-22rijTFoZQ`5f?=rddnM`OQ za5@fOI~*3?Tv8qOkZi74-&9_Eg4~&|=Te&t^5IE|cb5J!+2Oi>=uKht0qs)VAhmH+ zRVgXPo|fiLJ{hDWQuc(rq3vdyCrxNGleUs((%I+7EDFPkqmT?6OY+t_(I_%XrH^57 zyZezSzW72vd={$ESUPtGl2u0< zqhLP|Lkeyb=Ezw`OgwYBZwizmf)?qim!p>|x_owT{6;n!%}2wrTL6wLR&4EIL`je>&h`} zujhVN+*Y#p`CS&siTCu$CEQl)Q+OihlsgNz1qRIve;!=M#&M+6-4czu4v$0e>U`Iq z2U>uaNn}wSf?g(`#eGAb_QbfeV{H$*FL0v>g`!cz1Y@HJ$X<0budJua3SrCS-NQ{mjFMJ6!MwbkY9LsRj}XLZvp3Q2q0bHqQB zr$0=neC6jZO2rz_8?-h2j!)spu4z3(hvx5PeERHmT)VOiUP=IpdWw%aoR~TPT9e0A6gfYSJh#nDEdV&q zS?H!9G8LWgBFZ4jJsIhM2YgT1YLYwvWCzpO-qxE{&cmXq=8d6TsZl?O_*Wt5r?lfg zCA{ExXua%NyUfipaUyBwd5Ya|U2QqzG^u2YU=$-G#UAzwhBl1-MlesvEQB0I(AuV7 z`oukxa=OOPR$v7dLc37M$A38jiZFHI(lofK#pg1Ju-}$AvqhiZnQVycw)IahxJ~LhVfR_I(c96}#JPosJCJgUEJ%R;`(?aXprNYbj%_eahKn$peAuCmJn>s4R2hJ9G~+_|Qc zyILUJl*RJ|pI*xAcvjLF$p=`J1tHRa?I9dNn2?LJlI`?BNr7hImE?w(HRMMK*0Gp- z4-scpTW4|bBqpLh>+xYA;0slr&gfPAs5oRilxuR>6@X_bVEzY3#@l&zxz^Zrqp=__9^~=dLu*9m*m1 zRam$wC30}LKD_VvZ;9oTMo9`_AabwyW92zz*IFIF9XL84`|FzEo4S0#%CMES=XYo_ z*$KGx(b6PD3OXx@F8{MG<*#nSb(w_3@#o8v{pLb;qvKMmJv!o7dbYm3hmw~_Lf~L~ z*i@Xqy!7>(4HX|e$ONGc!cKiIe`;bkSWQf2q;nrzJ^$8L40m5ysX1S_?`wU1`D5Y< z+gF2!{!OIL0zeY~3(-7Icaxzb=V%oz~SA#Hzx`w-%R~PW?LM+8&0@>(YK}yTQG75BeXhf(7d+(L~M2UUB;r?jRMr zuJFu_jp?!=5A`-VK6>;a^3qQ@f zL!CQ0UzovZx^A9$IE`Xr*)ilEFjq3vHOt@4yng5ET>gC7S3F@vWse-cmx1EprQ^1` z9GdB&!Kj-s73QIXSWM~sV4wWp*CJkRUO#8#;xQL9NSfZwJNTii+e_Pd-JoEI`0Llz zv4I8bSI>xPoT2C0RS&5#J3R01s}_Rc5Fsmh7i0(Y+0JBDW3|9+!@LAoMOn*Wd-02U z{%wBWP=(8Gx3fzXG2@tBnO6T33yLgiHrHRY%nLo5mV*(ok1}7q>sw_|ZCYedZt8&+ z9hRlKK7HeukCrsg7Csktp{9UV)>_{OZ#M8EpDey1A@$8NwjJISM|Pu+-u9n8so8YN z`A751^z`Nmm%gT2m7iFe`3)__Dfo17^N69^#AG zBVd(0iwr9*?{)5;E}DpW3rTVVu|>`O8K>Ay2fQ5WyLNcr!hvctZm0A6!PpJRVyvJ- zrXJk)SpGEuGQXy_5M3wjc%4E=Du?I9Dzln%m-A(INvE;q{O;3o83#f=S@8IMkg5d! zs5PhXNXA2MJWa9JalIG9S-S)rjmAxm8SL*?QW}?uKpjIlhqT&&ywakeJM%vheOwq< z(*{MR=;H=Or@mRi8vO1}pYs!QOm=xIN!~}N~dtS%!h9-&*PcV@4%_KOSs>Q>Vl3F(XU2WdopN5rA0;yZd( z)<9rdiKChZ5qt03#IZ4`-zu+Zw>7q>P6HwQ2}_RAh&fJH$tU6c1&QJ{uZkG0UgGa0 z9BRS>?sskoJ_vi|z6YG$6poT5m^wAaym7o=sEfuso8alO=dKO}TwT6#rgEXv<49+L zN0!N)6&NL5BB*gsn#FgLxER~E_$rECnbj#;If7Fpx2xnU$yyDTTRNdBBtXI}F$$6W zL zW>237CQxg3+DWjTEbrNe%Y@7_K3A1>g1D)_0Z=}3A_+NvLzV0oQAUwHDrSoY%7`4= zeNJcPO#bGRz{f*uN7q|%M09Dm{xbvP29ZbCQ_C2y>>cb2@Cox*$$|ov;k$4U zM4C|%A^a4I`|#=9e;f7NiA1HPHISb?=@`uz@TdH!fM9WhKO!wujfrAMh|fDd^?fB; zC1hQeWc>f$BH^eO9JEcIMd%VRuz~RVDFLq{{@x={+K@m?PHt(xtVH(|?gC`#|MA}y z4uv<>_Tm@kJ`R|D5#EK2uRh>!fBuF-o-4kHxOgchUE|G#eXOv?>%cDY`1e4q{mK;z zk4K=92jfwqus=V5e(dz`_iw`YMccbo zD9}S!`okywp3CG!=o2~{2ovBB>gKcL?LEeV&jB~O-CPR#f8}}40GzK`las{k{rEFL z?rO$KBp3g^KOq>mC1n#uwY|&Jp;5lSw}tKfb>YfbbirPB@;&V#9Vfb(AMO8jG$DoA zmjXQc{hKL(PmlLn4gI;lC1eni*{`+7RzKiIgx|qps`saq7)WT^{pmB{u(4l`BdTLx zf_#LAo({N4Hv&gcU61b!WJk~7ieI|0?Wgu0UlTr+cF+VM_dg0zCP zG6x)UK^ahlv0!D+RNw_!h;eV8$)nUoFe&V<2789?YLK6WA;U46R4b6WySh63V+KkcIzIDuSZviL+*fA zxaId2%wv4iKI1lOS<(NAy`b@I6{1kk!CugR=+BGNj&k_>SC*#i8k{K;ZArt z23~&e_8(pni3@uF=w*iq0K_!{n8C~i$US4+%c$lM0Hr7u@8^gc4xeD+MC)Pn#$({4 z@FxPKRdHCE&|^^ra;&bZcQ1VP|8W7*E0w;V3`ruPki-x7yr|N>w+FilJ3}ru))kg| z&ec@{v+`3Pc!CeX6zd_>g0O&DbAz0d!@$6R=1SJun>^r0Wo)AyK)-U}94+_cNtT3+Y!+#aY~{Ax7)^7-`<(p=S#uK}nZY3(j1|gH-pP6B zbz1OQ;k{KYH(GiO$I4GFAp$Nxi~DJQ1(p)b2kvTU7;D>)QN#*|&WR7$f{98{5fl^T zV72#0^PeXr1TH5g=5+F&8K!G9JjQw{9upwSact}l8%|v_?Pkxhnbr7F{M5t>0;Mnj z^~3!9$e;s=*UB}lEP)bvH#8PP`n?cAr(zVd6+vOD5dxjrZCBrNr%N zcYh2z79V=bHWk@kmh7mqmNIvj3||Btr7zUtj_$o(XXIwIMj5vS*}R{tpaLyO60up; zG}kn+Tfp{VhsGlh3GYb1p$?b9Bgy(9knE_umNch2kVEj?GFg6b9T{n`8D;FBQNs7( z4SSk(!zVoQmVOoxob!lz9Lb{n1U+CVDm9ei=?EtVRY)b#oeY#*B-mmd#uo`?jQC3f zdU<@<<>!||D^;jfx>kZC`~?BGTklROx{^JXB2YTnQa0sq&`6tw4Bwd&b`CX?Wh)jX zo4v(Z0qc8dU(_QhEu8usNdX=<3#{d`yr6$dcIe#jaW~W^&r#h!Y+9S)+^{;_sk0ex zofk}SI?{{qNyJ!g3O0(vnB2<+YKKZdn|h?`GZ{x#RV;q-qo?yzG@WI#qLrDjvbXk4 znPJ^Y!YB8A>C*_&#Pe4UDu$kRBw6Rq4NKq+loY>u%2smhhjZE?gQEqmCAO!jDCag^ z_ccAh6cJX?+eU=pQ4w7<_n~r|0KNoiM}rPou)aTom(GyjcxNqp&Iq0s0*BG+Jz6*{ zND)*h&ZUeO1dWS`E-V=v&9%7!mYE0t;5(B&Ui(y%xUFJS_vZh$iStt4$=3IrJ*=Xn zV~8sj8R;AUkI)}9v1!qMEYe= zuf9}pXNIj&5Yex1n`P|l$%LQ5JulwtMozZ`x1ykp8WiNpJ+};Fe(r9K3!!rg3spEy zzL`(Y*Fo7J>q8^Z0EKPJUp!%Af$0>K_2*Smq6a zQ*$E}J{(JCLFmNew6Ss(nPi*>$5Z z)2bkEChr?lj5$yr3`Km%qaT$rL!fGWJ^2lidtgC%ULf!UIKdB`*Gh&?<4P4P>2G+^ z3JtE78-X)yyjvg?r|W6JXtZMDN6zXq0167MFicfIlN|zWW`*b=(j8SN75Gje@pujx zXHt;}fIYDG%U=(8`ax5j0H{UhO_57D6=P5cDGua*xtlVH1f>kjK9SW4`hArVw&0rH zw*~SXvl1!$6lV=RGY}Ndu|s%7_YdAlLT2WXD~1Q(g`@WiYwH&N7F+mn+DkAGS_sHz zCZJ(O5qKjuK%>z{5DmX`zX{6OsYyT#LheJ<-qr4vz5y_Z zW*?%8*z%jB1W@Rzv7T)EX6QEw_)C}zwD!%HCkieFo&=!K-dQ-mx7XgVwg9Wk4B?@g zJIJM$(SoEysSNcFGj31P6Z}2=&or5ncFefT_O#AyT<6j$@WVX~9y�%4VR$vv693 zSUDKr;`#iO!R`*BJW^#<5Y+n?!HjMosr{2?%EC|>FxRZZP@KZUt)L2knG>MsChG0R z(89%OAWrH8!e>GOwH-XjIijjjrpehLC2|q>OmSGEDlvfbm=CzO)2OMbJv1+#Opv4m z!=nVJ46BhwgJoC@ecNYmShHvFzxU7iebbEs;}8(@j|d#fS7AzXBb!`%z^3J(nU^e# ztyg*L5bRTFwMdoStQ<8CQ=_06=i|^*xP@<8)X3rPb>= zZAH>wumNNV^$wNTTJRn7^5A4TMHbc`c1udY(=wvnUrGwXJKXCwEk0?)O!-W=d47r( zTK=pSRP+qq1R`)g9$SHCnGxsDe0(NQ$9VT?%`HOzt{j1g_A@3BhK{WkoveE>Ts=hVk$_~dwk1N6+DhZP*&0JOT8ah*QmIT{kldm`fM))2_u9z#T-_n z;&z6F_Kh6XU|gIwS7-@?qEgf(a5ZwuJaS)@xU-?QTaZh=*=`-JXPvxGQqlH|AN&lg z!>m>jsid#;Uh@Uqrzf?zjVGD0w9xoP&+(8^%-DogzZIsMEX0^&4*Cc)-YBtb?szUO z5(l;tmFB3cKqGWEQJ6cmc76bspPF!)eE%VG5`mP0F|%`jTr}Iu$*NFJgYz z7NeIU-P9OyLAih|7+-3VVZU9_bA2kogh92?OsT4sn22j;|ELQ#9ufymWIthHMNS%3 z&>5Y-w{;qBtCsNzD2oZSg{y*Ap)g)@;{*0v@?|4TR(G6dP+o2cgG4#NybX}pQ#ne; zXX2tIKhrkm#YT!al!K-pDy3W1oQ5rxtb95~2k&-IVkO&{wL>r81#{{EPS0fsV`L%6 zn3=ZQEsPYNqYhz?iqjT>Id4#^e7>*!hlJaOdzwP6QUcb>pdXD`udkf!l|9>BUmC4^H{MIp8mz81pR)ZMS$QD<8AGqEDFtS9?<14`-A{ zvB{o}EuCIdW7qV)ys#8YvQ7)zy2IjcGPqCVbaAq$o7zE?$+j#XbA#-CTtGr=@^2rYCgC+zv@`-!tKbd6>ls_=#Uvn+Np9fJNo=(}Y>>sm<`wtl%lN|)Yn5C!3N#)zJ!=Rb0+LftU|Mc(pa(Tm za-t7|nf%I?J3zc>0REQI3PUoW8Mi*a2Hi@Z6~&5TrMOHKOY#R9>*muXPo{;YC{}=3 z(i=a4>xKx_3YwM`4;Tq#=u`||)-EmVFFM0*`l)KT{T~SdlaGYFz{>1VxOi-D z*A2R={z*cn)+ZjerP@UM!{_u=alhEX68w9X(Z(6t+x#(!PYL)7v<%yiAbUXW5Bb^6^Wi$=86Mi`IAni6M6*@okX(vLt}(c zOqG5OfIlfSUZYVgYduwQyi_QJ;e352;v#f%&w!!9I!t=(`<3X7b23wR zbHGu>v7CWNO<@#>aXI#2veF2XfM(u0r=GQ>Tl-3Ha&6EiZu;a}p)B{*nd|9a;$6Ft zN~@VZkV%`g=H56IwRzwK$rA|Y+>)^(LiGMEh%IbHCsCZ%H2*88%yxWIu`GtuMPYuh zOb$A^%xWD7eIzA9m(H2MQ3`%4Hr`Pa%P9^Sk&+OpCae3t3xxeD?C|dXAp{T-W+mRY z4c3$-DdTjp{%heS{M|G2zFUjblWcBnLvJ)kGVd#E5R4E7n4e|nkgoDqk_*+n!xUlF zfu4CxvFFWQ&$>7s{;7L$QEe#y_=nch8&_}BxEOuI$lb8rT>6Z<4fUuEe0w49n*L0r|PpzFYZuY%uHH4 zLtK!M{%GMYhduWYb!`j#5GMsv9*7b$Dnx&@{sFaYRzmMlc%5y0s%6F8cmYc zNpbNZOk_UQ8hQySBO%4GI;Jt7FM@ zzJqU#YMir|5BD}8<4rOm{Y$x_?2a~n-pyW?0{@IE2@|V;O>tK#)JAWK(BWc|f#>S9 z1MRYUCjjW}|!-2=&{hYzM^O+mfP6jQ;fzukzl!CXWg}?F)UwQ)GF7SV{>w#n zx1cTS^wAU+BUWcbzGl?W4UYSEnOZYLqs%B}OA9G#SPQ8*9pg2jrhORr@Awcgv_Q00 zlLw7DCu^+MZl1U%1j4!eH945OGg#}UC+kHLavRhZWRKFNQ{a#mI^Bo~Fts#LI$$(b z7f2zmY;hh(M|H#6@THR}Bx~&i!Iy7vv3dW`i|}7Sz%<|0Wn|TH$pH4T&{zq7G2kN& za^E>oQX_U_{Ht0aQ0ub7W>7N%-t2qJ6h_Iv^_AdoFsn3%6FUJ;s{BW-VCbcXx?%H; zzsC}{9Yf=K5|o};k5Pm+{5!002`@8d9K) zXOrQ-I9e1wnI1AAFQwwOj5|w&`n({gtq{bJ?i6`i1l#R}6EY>13xIV`Ld8M4*(>Ki z6AS(CHX$cyANc_&vYBYFbo-|f?+~@_c(N2g%R%@pSuBBA8H5sZQ+=Wj8x9M)w%s0&{^N8{J$qNWKns0xmg6IVV z!o~7Dbh!AZSnSG!L1gQ-IT)Z5`prh87Ju5vK{@`{)s}PWIbH~&L09y|%R7XuK zXSPIolYc?Q$rlXqqgH0GpMVukm@5oksi+|LXqaJ?Kz&0CM|F33hR z65`XGgvGq0E0gq73Hsl?y_YXgTmzAEJj$d75OTN~+{D{z|8TR2;iFAi5N9h6i2e$5 zWK$Sxp#S%A+z}KsVm^v$C#6gaB}3ayX&QNT@-cSMMcgjOHvm;W zB}In;tjnW40Q4Mb_ZBI;CxlSpFBOKdjU)-^2Rm?Vs{}2VgM|WMM$4$4SLB|1KbS$A zhSShjTxL96%LH-mZEAm`JjEIq?uMS5V4X5>&;$9T zfC%%j1wz=%PWTK`bZ*iWpnyNhD~8-G%c2-Gbb%euYWJ(0dy&H70AL^72RP#q7^PeB z)n=E7LbXyB1%{x(mIKy$|3kh>%ark(@=c#lfFEF_%y9)Mk; zh@rGH-PLS10%1rSc}-)nixH*Ak8D93E5*?(6IpfcmX=>I*{Sw-O7tC9LAj2_s?S{sj z;E%6E?xPr-^MjwAxws>OG7QbwunUvNN5l4vZWS2aV4ek|eoTg*9V%^^6+6b&lO~Z7 z)_&O9mSB6HtOqyjzw$kTy3s>GJRZVXS~I2p7L|hsT7KQj$u>g~Tx(dVwI4&anyqd6 zJJCHD71hlhGGkn&m__`pn`-l{;FlXw&n*{Arz=Ym;HeB<*Y+2-LcA-(#Tn>!y?xLa z8fI~9Fr>pih=r3rd*~B=oi1#$4*^xM`BHV`=~Q>+>tc>I@c#d)l!!`A0z1I1$#si0 z$te0R@17*=HArY=*oj){{zEfY40~O^fW!$FW5eJ!KnG}h5_q4-_h+Vm z&-B6g;g(;hX^Q$}3y($CBtU3+>m;;-{Mja8c4eap;n{3H>H!96L55yU)P2JQbhfmx_M=69(AP4|$ymg*}Ic779h3J8%UGQ0g0NIoYMw85w&8XxK>I8W65sbdDELPS-teW%6Kq&>Wr5Q}T%fB)Gel zl4A5UpG}%Q_!2R%I4hEhJ?QEbeq@S}{gd@kERDteRMHo^#ot00BzPO|d`z$(Vh%ufRuuen~@Q4z-rYBE_lOrENLX$p_*8rn-C^^M<1r zBmW$j!h!g^Yf^D_V!S9UAS`!Pi%}qz#3O>VtnBwXAAYxQfdz0&qX}wiiDwhyH8i(AtQ4ZpB85-&lkAjSGQS0(+$q zq$qQ8hQv7pH5A%uPgQR=|6zQ_ToWu}XSp%j*5&TP0L$juLr*f(1Cr||I4|U2MUyyP z3dixPJ;&}dNK3eNt_Y>lc?k|D&xH;vR!vwBKr=ek8sydi2*Ly8vhY7%x&hmEK{Jn&JZT8`8ez_G3HoLDs9>-2+&g; z*?@XU19C2%&vc%T^;ct6vXI1h;ds8_EZeQNue60fBwG3^L&Tq~j)kda`doPGY;wi1 z4b9YH94t)gdCvSlr=2hvR7L!RN5GDm@Rch*)kePNB(f_NONT?1ZBwJlx7~jnni1z_ z=7ox{EU;jnv=d8+#SS{H-7A?%7A%}NV~R^sWD7w}jw4eK_zRAHWxyzrzi1<6#aNQR zn`()Ee($InlqQOzl{Ls#p*dTYkonPbB+G9GfJ7U5Oj0a~T#FYhx6TjsvYd)Nrb48} z)c8?oNRVOK$DIdZ`rEFF5`nkN-mw->#MBtmWTdeQ$#tejKi!~23B&61*s16^i*<@q zZ42xqJcP117+3{1ZU594A`lLmg(Bd}9X%rs9(-tU|?B zKri{pWqd_rQ%mWZPKTaHC$K)sIgveZ83qlt$Cm=*ioc%;jixyu9co+Cs1w&kGy*@(c58VXst596%yGvr-0F_< za}X>Vyq$HZ)_eGN8d2u)nh00?I_ZtgxIr&g_H^B~k2@=gk_+lpm0C*2#BjrkN~HH( zs`3^TKjjox`5RP0rsFcouJ@#@f)ITJB{I+PZrKP zSbwH>s*jXbdMqWBH3DS?|(yj?f(VBqS;fDo7t57gJhW~5gbg(k|CG5+<=`7ebvKJzgCc|hId zl08BtB&GcMOmJy+enFzDbTtttoJ>{6Z_$QLS&{U$=yz;<*Y;OVRrizGu+b5CswA~h zIgULiy~W}0t1Zia*Z~n>nB7i))Ks5!Mn!&ZJL+V@d$KwrYpfG8h%I3 z+a@2C8A#E=(uRB2zGXC36e~42@a)0d)QL;-oF8077Mu{R zo_0Ge5YRAIji^E>ve!pSP#fx6uv~q{8g=Y^V;E0CRa{Ewt8qAavQJQ@s=sqyna%fO z@aY^)FsI10yO?pwm+#Wx)t>$YgK-K1r32C@xJ=5uCg?|U?9=O(l!gsa-z1iIMxv9z z*U!zaIx+Vn`vP~9Bw(&WxHN}W(G^BeE1BYf$H0B z)WV_s+IWvhj;pduotT)G5OB#YzGFJf)1hm%BVKrZ|IT+Pbh8b2kv<;82*iaHUpUup z#>5%?z`2Rz9d2}tjyv#5CJ>jV=yM@GG?v`Sn~&$DkH&!PBx0RGqG{7lr?A4Fr4{gb zY|`UtQIUK9YPUrC5l#TE3#-QO`;39v>F8}?5pxJ{qp}&EcMg?+`CadDfdj3vlHti2 zn|LMRL@*h$dCI=s*uG^L8Yatfp4lh~He0cQpMSZW!ZZkw#tV_5b9EIWkiY(uKvTh~ zZyVtIOXkse8$m%gI!kXLGYLeu`%VpT@oJI7aABb3{c9~8HPC@uz%qC0PQJf~j`*cA z?Oen6193mkEc%sqbSbq+XN_0$3HqBs-}X3*-Mlu9(iP|gh1h$&m8u!_C+0UyA>?1e z*VwiI2l3TQHu5x$i3wvv^F`w{j(*OM2I}-QaJR?5W$YxL%hj--0nGO*!dV3}vxgnm z+ZO)PG_#?t0^^xCqktZ}kzNNG--W7DPTnb~V=887LoSdlLt_$)t$HkWdc~^%DHjR` zAeq)G>M&iAD=J^KsWyZQ`}?Pl)9Zbm@@Izd!b_PSZePL9UEAUC!u6BC?$$p9FMyoeAfkCNj<0l}=1OYXPhIH^HEn}7o<(M5 zsA9iU+flp+jl|o)LGV#Q&q>}5i7wxvws;w{PZBKgwW?zZnJM4ip6m4i%CgDir@t9E9{5 z3vIpG5#{49G(C-b<>Re(H&-;o;ert>9VR&-w>_sWHqe!ma|X1mxz$B_XIG=Bw=Nt? zr?&n^p&ks8-qe8a@#I+oe7>HAbjviJeErdQe+$|Y{{u=3uB1YR+{u0oIcL=g8 z)Sm3rH3ghqIy4!^-*_@k)tQsY#xzP?AcE7#q`W@mSdeAgtIx=4 zcGXZiS|ggD(|?mUKv$($36XkKi-%^>TWN$k*0I>b6Fe4!Gu(X=xZrkbT?lSj3htG? zq=BxKeLs}OB`#q)xwKS18*iIEm2zW{V{{n{RHsU3KgGsGCuG*k$@dYN=8v3J(hJ-1 z@F!h(E}UKA-;?szBVhXNscn@YP=Oj|l&xy~cs5Ox>gnP6Ju-7pjA$n%F?Xj+Km#vE;;`e4!2lt+w;o#h zlI%RuPWoxA{zG3>Hhw>(UAgsN)jeWeecj8awi9XIAQr&cloVZsax&VTCHlsv92Al` zTaJk7^cejP7Q?1+-qmrC^W_MvxBE+= zO8VO)PrFQT>^5RU11C@PW}KU%I-jYBV`Z|yxCB-M)u{o%(3wvQ6T<#{K|mI+$%Ja? z_#W}}zu`sulL`wGR*RF0#9W4h!i~%EtwJaaKAVsZnrVCE*ClNz2Y+=R^1$`gJojR% z(BzL3_Q$xwe_-5Li$=+oA&l&c9xC=&HJS)R+zoEXK4;> zm{^Dm-)Yr1_QcZ&q7y~H8P#^xsX9vCr9(?V$y~*G`k8fGOBNh{o#t)K$>v;2W-X&j z8+V~CcLduEq$HfM?jr8xp1*MnacrGY!mVSGu}`x7`+%dW!ey?g2X|(&y~_xF<44ps zyJQ+5D=%BgezRlz3$6o$$i?`WLh19|8@h;3hA1GQk4v?Cg;IbeMtGF}mJ#g7utWG% zLGf|5q3~ipyJMcb{e%c#vl^#mPn2EC8R>#(#JCydD6)!*Th0eTW_S&u?foX%fGzzu zWBcE+A*~5>CGIr5Z-}c)X5NM|eF(KA1kxl;Dg08qq3lBku=j5Ssjda=hV{68j8Wnr$BoYL0%@vA#mZ;vxyp52E`-k`TfJ5r6T}6Uq#Jt$%Ii z|Ah5#ftVhrEeNR9?cilUdYJ~$zM6w&V91N=uXEjyt2{=`&f(OQ2Oz?TNW&6x)-XQ8IU@%AKJ{`eFQ9mF3?T$}-9kQ6osYi{#vKJd~X0GE60 zRPf9`+3@cyWFth>YFY4xZ0d@wylrHZjF21>e52pNU~&lsUy3&=!Y$~~3kAsaQnBoqT~=@O z7oN1}VygXtD?EZ`5m?#pXJE4Qc0TTT`Zw9~ur%@y5}@^0QM0 zLBOEtN$G#P9}q|jTe&gX$3KE~v1h&y>*{~5!xP&R|zn6NUT8VK#MMc@k=jyOmce6ucF?9`h%YP3O zJfzwf{vL0+I06EY$QxuC|rZ$)O8 zvd0uhcRs2N$(LYW5$aR`pFQEP(dX;^S7*K;B)2&1XS$#0*C2a;57%(-znd|gLLQvV zDPGs1l_A1NpCPi&D~>NWMh^T-h9V92oqGS;TiGq^jU}lS;do)Hx>p!#>L!O1;+)ZWBHT&T zAh!aVLPR1oJeXil`u;c)p}5M&$*Wa(8zB+O8{ia1EeF&t^fN;yWlaWpC82Z{D|%nD z2r0s^N5G=|TZ9lzoM%ZP6S?De;QIp)&WWuK!@mP+s^#HzS@$Rspq)HWAQZ8tLB~*7 z8i-Ig!XcIj=#Uo;5+M>pUcoxVD!&fH0@$Pe`!{2EuC|{LfM%x9A!RqJ()%Gg5mWLw zQ7$A>@x7WvI5hPp`}#~CzVkLA4>Zjd=*CNPISUex9(vDN7uX773F;*{iPpu?08V&N~otc%vNgW?Ue{l)b+zCGITF~RiY0- z1!sNsuMjw(p8tlvp*Kh%oJiV@1l-c10;~;tOg`EbiJdtgND>Duw(tnab#LsNlM-#N!V3IsZ zco*pK8$G-Is{`&0p|A&Of(E}tt&Z_o6lionK1XtOR%zr>`l=^e-N>z}PJ-6BTU_B= z?Gq4QQehbwzP~#XU$;6g`^%OqSyofgAOXW_g8)lfpB*6`FQh%e(^8;@lz$gJ{{ee< z05ziKCB3ZyeZfZtE3R0s*I_=OjUi(;_9$tdKcP_R5BW)hYx4a8$+us`X7UGWDbPkm zm-l3TJtQ1|VAn`ufEbTS-ZJzMRjCDLoMbBe3bkNwJ3Oe2r>2@Pdt(r)QZsE1AsM1^hsEi(`R%R<^B2xTZ1rMPKZ5c%+gCH{x2 z9DYqeya=9EaEHuV>AxVoqni45TZzR^&ma=zFa8+jmo0NtemwhXN4(A8T~kfnNfo@>Vf5Tb0|eVd>rB=d%x7?y}Q=MzXYOyriY!zkb&imM|IE}_^R zx5(oIN;XDPWCQz3L^SvtzF|hT$4lcmzd2!M$U2S>%o;zi7n3I$);I>O?lM26Hq!ET zM?0F6iY#$WPxpcyZBoZgT^Fr)qA`pvmE%thvDMWd+Y4es^#Myy-Y$r@VZ3pas^q-_ zRbs!lmlR4r^Y#Y63wc%nCcV6y43l2%4cBH+Bmg_DT(yZjR;DOR-aCaiG=LRhkF||0 zut7+t)EENTix69;kB)!^y&^?V zEh*+s0p{+Z?ms!CR$x&Dje21Va8X6^tyXw*Kq{Eydo3w?UQ=Pe#6avK+66o<_-aVg zPjeN1Ns`msCBp61(frJ%OI%F2AG(CO?3*`H>dW?-*ojX_q)Ua8DoOqL0MgdqlmEyR zwx9z<$T9vt3akS$3CwRcK(4rFZqMKrcY|x4jtpJ}O272VZbeZMLCKHTYvsVUZM!Mg z_Tz{8l*2&$#qKwIxcSFJ!Jem8^% z5xLia`OzHG-231_k8CVSbNiA0v_5Apm!C)SkrdK!b5&F;$mbcv&{tpcI|W_B)0mi)z@gE&C7(_(UO;h)QCIo|J61K%Y`Tvj zck~fRfjR}|n=lAVjRZ8$Kmm>ZR^oo_QD$WcB0UxnzcrO8p@7qA+@xi#x(!Lt@l?R= za63aW8iwMLx6g5~{GQ*4qiw(aFr)sYWZC5TrWXtUosDP1nH zLMlTdy9;@14+w|#Kn~$^=ZvLp4dhToFvuhXul@q;FbGrkRz0vRPd9hp^+aJ=BaoThYdY#WArssB(+px?XXP4#uZ|;zC0U*x0fDQT{cnUn z!W2&O)stCY|2XjRSj*R=i}lM0PfJk;3pG?WW54_r4Uw5jx63Qw;)BjPz>K4^WPB{M4B;4bj z_L4sGQv&Hgh{!T7oxHrnvBM|#UDpDMqEY2~}4nBwh zvIvs%8@BGg6=uRRF?hSUWk(R=yPt?ILm`hqtuq>MNXHqdxrIjAkidc#QofQ()Y}Q2 zjY)i*C_gPIG&8>xg1Eavhne%iuyqKIBg&@+DWn_5U)OxK%S3=+g~KR|pA*Tog?)I_ z+*6?1e#O>%({~3WmeYBpnDESt^A1ROLrP~ayUFALHQqL&X9Z$YhyqNIPEGzKCsh0r6x z=L745)8heTCV|KAzz29WvXw=zfgs2mEQv&C zB7xvw6cTZgq&)!4_H(7*#nxnXhDYx zRLEkB_g#al{(z$|;G_<8 z$R7m!bwft$1CI<&ms;#rT+wHdvhAR8cpNbCsj1aF!*TV7`6{mwkjM zee{T-*Xwg5%9=vjh~R&dY`y5t9l~mx=|h@6!UYga?v`_`tJMJgBK+kr`~H<CK~}gXz-EBY%EJvvt~Lh4 z9!sE^T2Gm-p0W*9?`;eF#C-l2PEw*8R8=psJ1v=WD3Y#bAl|V2%*`y;E^743Fr+Vs z3s=&${x{7SOn#by#pKDZMz3G=j9Rnz;MHz`>~MR(C;x@O0u3109;rm0A{f=bVc~{I zJ!=bjq`xvH-UI@=Xq^${#ak-~lIr+{SeBNp<+rYhmnvk*t8S^2`b7H6Ni>|MTTeNV z2L(<9cQ^oQ3XZj-B?^n-2{fo&=-b*L(Lk;lVk|HI!OErMENya3f_pWI_vl7h&V^cc z^;0Ky*aL7Z|NYLP%o#+S@ciwYT|{Q2S?oP{mb7Z|0gpjCHL$>7vf40+AD~AO@787( z7R)RCx555k6%o$tU%SBICnDjyXAIEtWRkN6FBB9j9w-bft&2#F;(h!BfR zc-pjfBv}RsC$eMy*6Zl%e5@;4moy4sAP0M5*A0!e`6Z%TKv-h@KGKZ-&1Qxq zG86>DrO<(sWRD}fM=La=B84AGE2%+dP`%5BA+k9Vrv*3YWLjp&wc>H62h||{8!m81 z8g8#dBNOoG-nEPF0pgz^mEsMHa2l7Ye9MFUhyS*oFpy9Lvh$J_#W~^#%fa0;q&=N4 z_R+Qz%fFuTod-}kmEk*ZLhPD5mj=Ool>F3`Xc(ise|Gg7lK6vAbvNiBhU3$>!ozT8 z>Gz_L8$gm~EvnNbHhuw;$|n~J$BMpE8PI2vvZKRz{y{jT1%oL$U%0kBY1}DhVtJ7W zP3T21k&cfalHSYr7aJ9WR%7L8CQ`h82oS7ruI9EWhNe>Ay>fj}sd+wp|JecV6e|d(uh*XK2g8v7>}nj&iNh#2TflMG`xq^vxkJ%PNuzvdgfgix%K6$?Q91%QBu6rmLS z@&PbAlp|gG7bGqhMg*4~z&TzA$u-ZzHArWm@H-XCco_kFLtU&A{wh>g4ZId-5`lL# z5sY%O7bHI4J@OX8cj@>KA%2th#UH$uJtBiAA#ZUUqSHiCBIloGO)NkFbIyu zgoXbW0${8q#2iLU3rY}HH%#-CNW!amo7X@$WbLosQwq5>3Oe*5D~97wDCJ@a;^;sS zo35KEL7NB)Mu$)b>!Vf&x-J1R$80BLMah{5m^t0WtbDD)ja!+5CU!dFdx0rM($iM5 zhNSfXZ%c?jweCL}tl+ZuEdyeKI-{jY#v`^Lisd~a*N_lIsvc?(pStS;m=>$XeK|)4 z?;QWdll<7*TM4oQL~l(7Hjj_PB&Y!KHpCtjNqzea?9;|5WAG73omTD0r}_f8Wu^d0 z9S$(#a6q+(BUlS$cIB{FzMzUo7;GT)*R7h8QQ!C!Wcane^#URp^N)XYpYrR;{F*ZK z7JQrE{A(5_obd-R1zgmk zro5gLj_7H+Zq{4@0+mGYui8D;$yRmHQS-e{Py=D3%)5}nh!#A!D8QuE9un6L`iPIA1xmmy&CM&sZREZsuGqclTA!sZ)Lv`APYAFgF z-m?Bw^24I3X_);8NpT7CSZ>WlktNTluQm;Rb*3cJ2Wc4Cf~&cu)Z^!{6t_nvi^xbx z9KI33W!GPZ#JNlYCfBWnDPJMbv|bjvBQYcVFfG+LF5{rOR!Gu(1=7mB=LL=puc|s- z3j87`u$8lo30O(vahS334Og&~j|5Xd`2_ut2AjI^5p65t-Kb9VcCF%^h>QOu%~Uhc zjHp2t=EiMr@ypmc{hUc&=Zo2@2=J+cjM>Q2V4W>3Rd;m1{L4_-2~IRWPtMJ4H9r>K zJn`wZb!>I%`vrg3ZC}hFszYdE642!l;0LcqNw>!WsQjsX%$*~Td#B9RrXi0s?j(-NIwo9CZSC338raEmlHE5sywlinJ8i8Wj|F#|;LmQ$Iiuc6X2)M3_-j z>N?1Y%dHdYdT*>*j1R!5+Z<;V*72`>Eg=L5DB9RN!UX~MegY+X$Y~n+q11Gg$he=W zGo&tFB;@fm!ohLKxm{7;6W`8rI{k9w)7j)t#!Rz`k1VM=E7*mnvvTJBO&WlD<6oRu zJK(el&KLWoiWM>aAn#rPA%0ki;|EO7wyHTe_`zOc3%=r}tbG=w}`=`isd(y$#ZWQGaUu z5>t@Y9rx8?8aSBVm-*d%TNv+Z;WMoKfW5U|TVp&;`VxeN8Cot+qU#+Bq-{2)mEsSkHBhx;k!NXTmzF1#T<+&jG+bO&=htTN5~Yr8UX z60{3*P2CSGl$ZfeO@uI4U_3$o+b@yuenvN&tK$jpJ-!qlACA37JNr(v z@d%Wje%}reiRUg+(ch=R6@l8jViyH-gDpnJljqO?@?F^TWkb@@ z_e#%#U(*_>D!r;wvZ-!|J(qabJ_@>UGvJBI>E=~i(*bc zX?MDExWZ>70ck)l;Evb=U*nMXb1cWj62YW+q=w$hU-MOM9 z#`PO;R@2$d6}FWfhtS#UC00kZsxx*Zev-7QzOEMm&>N$I64^cj!q6r8(0Y+?*{=Z! zY14Is(a;=mb#+*eFvNe5Jji99knk}Ww8MjFgSqLjh51+`(k^7_4h$U`1%jSgdv2-X z;DX$qjNFh0`viMF%5MN~u)-_7e!X)?h%Zak21(fJXmsz3lDE3wmIT6SEd5;(NRnk` zNcA{*r(I*+qX+A%8pfg?@XuFO3E{1Li54SRPuP|{@T#~@p^N5&^pU3Nc9Wx+F z;N565`7Qu=e|8)gP?t#Pc9*95Ax&NI6DY(;Dh{1Q`9c00il6so>NElcqj{HBT6!LY z+WJKuqoi0yeh;HabraG>=MTS!yk;+?8aWE7!;?Cm*~K;4_l^jnO2CUU zIHJE0a%7-zZ6RvPfTQtWA@=%ilujERb;S&l}9d+BaP!0`;7K@ zAXtMt6r$Rc28@;$LbuBA=XM3KR~^d1a7`YWfFl~ul(^&`^#HWxTTp_mZ=FS%$QW|e zyg!2~Rxoc_Xt!0W#a()2{;|jvlmz|=oLXIx$*<^zxgOc z+&1_`c<^y=LVMH_v?kT@@_|g2^^p)~{@$37Gh`XeWdO=;t5R_a$=qmC=roD95R`62ph=iVgBw^D-^kORw;PR+IyMDBfZ zQvA)@2Vf=GiHTSSQj)ogFB@@&e66K=FP!zB3H-|n@E7eg`n=2xK5&0$Ob`1Q z6oxX&U_O>>foG50f&dvhUvhr71IWy*E)M{Ki%&a5I=lh3Ih3}Mw63CNkG=tm4U$U_mVuN zqg&k1Yf6$nGs0=j$#8KYxCXcnQQAH(j+{FToLpQ_A{CXC&O3bvz|Kit_zqw76{(k`&ioB> zL2dmTIrm>sB)-tK-^M6=WGbyHGhM^wTt{B}FujLG(`CvizyQ57s+a)31clXaA3ujs{zVK(BWMiLzJi;u8iQH$XU`NgP z+vV13z{MjYp~|sSE5j*=X}c1{-$C}dq2a!++zVVBLO=|2ID)!*(H-I-PoIOppG1f~ zxj)+Z5s=s|5S|ghrEWlO6V)HSHx=R@Sb+Jf4pm3|_IokqU0nYF!h-6m$pAbJsk0#mL!?|l6V)7q;=0&JH# z!L+!yo6?i$@kz@zAMtg3-bUThExD2TviGH~{EPw~!WXg>jx$cHZ*G0oDciZ`9s2$U``Z+gWHSB{sNNAt)%`pe%* z5|W8H@0+){1Qc>v_54?elmVWjrluF-y0)`W?ig-P_$LLfn_K|R2k<=S^7G8s*Rur~R;;shNgIDmI*m8iQ=a_{C1(^nHP6Bd2 z8p@+`jldPKr#w}i1AB7s1cXDd8>sH-f@34zpSp!8BO{aOxwvq#pvUnD^3O+$ISGM( zzTmal>wbd4=HH&@vLp-;SqZ$pMJhEL%WJnC%)m00IRnAt zO|WwFllUK)g~@C%*AzoCT1Q%s68~1W%BS_BwHs;$lbAOBaexrZHlaWVgbGQApod(W zo=$!D?-hw|b7Qh4=Zf>xPkYSH!W=UiG$c$&;b;3bd9|Ibp#U!8ag6ce?8T6N!4QndDjqHqHdY%hQ zm0{Xyv{nz~zD>g=4M2*q+cvZ((c=l79EW(=p6wJf@y$OKO0t3;elQzZWPcfp+pcPl zGL&yjHU%^HOZvtF{Z*#-Wd%!yt!wL%W&;rzPQpO>7Cfh&r&<-zOxO5Rk&-xlTytFb zi$QPh(B& z*+l1I9h7KbAtS}(mz51$7f%Y9Rnm&`R`9_k>f(=SAhp1O-cw$P;}JgvlpqsxAGbTvm>7 z;CiH*P99R7j&E|!w8j#1+P5q;tB;G#ENPBQDd0>|CWd^OHvozP>Q#u@i9j6n2l~R- z38<+kU05w-D-jsNw2tw7$fw%)WbXlrl5r~_p7vl(or~L#%|VA=WJEt8Sc!CD!nQGI zff)!4fgEYqC^H>WXjb;dcy2`3H3ejxO{M(J*@qYI zFhe71_7P$oF#-q0O1%$|z9V?ABQ9?$kA4;Zo$>a6c@?O|k1AXptxw&s9S`qPV0x&& zfqS55$GK%8EN~C%`zvByx$rctN-7uN4P$T0ZNQU7VN;Swhu$Z?qzyYQBMbVhcL0Nka zxco;^f;0f|f2q)l{Z~``eH>C9-9)ofrWedGI=Y4S1DqmO`~DcgGUH7+?ACYl?{qQi4@{Jdd+FjuRpk(QjhriiZ*!+y{@aEgkYNtAvn94VEM9 zIrkQYfL(7ISQUX> zf2t(L?7l!;sbzhHsGUEibSA?#L8rKnVPM{OYO#ChxNyl-^f>Q{DZOonDjC_@hMqoL z{Pv9Uj7{M{$mCD%4_T_N&4JAuc3TAibn;f~m;kHm@T02$P)!6ud*VKBoAZYK5`-Npxq;U|*hSP~dG5usK`}6T*#?Om zJ*suxZAOA-Xvxy(r>>Gmr%RKKN8gVUWuy5T9be|FR;^yJvUyBWI;T=~N>bkg8+UCx zxUBmuZXwcq{cPh$7@OB~K~ zD)J+5{(F;u?DeqM`j)RrdzbYyge6<56Jwh^L#CJW9Tr|}(#_sL%$Rb#-z|vm=w`&| zRb6iYxVFXfoYA+GK`&-^4^+_H-)-}3FN)ayQnsF2S&rFGT7*%)w9%&jJ8WW0+mAtJ zRL!^Hfo|3DFtY6f=e2+CG@QD|`U^k6a)N8)290gg@#u2Q>`CVWNQJ>^KU?>>w@Wwp z-S-X{%j8*#1aU1-obsBlH9~JP8!Hx?of6s4bGSl!=DvMEBJ+dOrOw|Tg1e$5s%**H z)=jSn;0=+&4%5FN&jbIQ+o+{c@ERjzMHQHjL(SN9Y!fRIHIp=oOcrz@;h1N zOlu>8yKmo6cl3PaU0H?bq9J489p>NY+6;HTXnX!-*>JlS@12Q^QVXR&D)&cs%VC4Q227TS}8{iWVK1^rsK|7FrfW-0y439EdU6{u#~ayS9x}Yyj|cmc>yd+ z&0v}C@9>?)cWw-PB}T_9i?mxd9-ko6Pq9@5RCZ6LKM}bZQsFj0$?q&FQvNM5(uVtE zR^52C>_t_yZ^!0i263sQDSW#fU6$tRU4_-ZlD6}*5zJ=BgSz*UJRxn_^# zHC4~azyWtrftU8!&8b0d$w9Xt&!K7UJn?w6yfZpeMK7f9m=y11uD!M!fTG>-|ti=+E_DDI?V4cj=V-t*H0ST ze#ZAs!1-;D2k~lG7|t6;bxUzCj_ADL4jr^+lNirEevWxOGFcUyczP?Q;=75#*g&AY zYW7vY?!*J*`m^*(fK?f8@an5@`}p%i2wp9G?{c79Wbo7p?W{;vlqpDfP|w%?mXe-z zp0?;na$Jpy(L!_R%~})4iwpkRi)+CQU9~^B4Z!2T1`J zyQ$-CQ$O~yQp9~~hB=6s6hD||Z87GeX?-_F0bz+3-l%#$n9RDTtd>BUcj&p+%uN zE=#wA)~mek&-PtM^rWwznh$ELpNfwcXRS$AaentSMtg%#s>i-7GoM8phK`JsvRS{S z+ZbAM*yM~VHvDn%@Do}t*Mw>AfqBcCz4GHO4Q@4EEnZ_+9x6SNrV&#$HP~|09pnTLPaZF=8ncG%eHsJ6WFC$Uv4pMM8NX0to?7Q|U0S?OPCxo`2 z95z|{briV(%JKivzI5CsHI;6sGjXRq9oN!}EUt=ff%~Z*lkr7bBaho2wYb+Z-{3*|ZI=O9IkdAb~1r_LF@e+s=j6$F&kLE+{?3bz_HDEO{Xav89$w-POeb$(HL zsCZqljyrZ14!m+qQ!R&va$hm1f9hWSr%QxWJxn;yw6*l9dM&ghXS{W?Q`~;2+Nqa2 z6rmEM?h1d@I4%=A5AJj2Hu2-ylG`~)9DL7N2=MZT$j_fTI}jx&(B!bQ+_e4suCQ@T zV5*~|NsCR%`TB$=4dsg^79*7sKGp(Hu1tISx;VJrFG^7ruH1g;F8!6`GK5RH-i>IB z)zwVP;wfhI#W=8~W#@>+WE>!ug>>t`ixugGu>Co4{SY=tMdEK0MJ#KlM6L`T zt+83UTjS)<-9EHepKo5y_5Lj>2A8n2pwM0363N$TFd(x0YpPbfr&DX@ZGjN$Y?#Q8 zR&%2m?-vJD$1Z8x)HcZaY81Lnbj{QDo2zL%y=_Mf}U_tRApGzD7F6q=6NEZJBu zALI-C-hNFKSpx_kH{TVZY5gbb`<;c!QJJ1#NnG)=xcN~=|eQsR1SZC{b<-8u&bTY}h z(0wM&$U5Dh!^Nks58#TnR{rjYWRgvl!Fbn~DmS+CErH;X&*HVMu=5mB$+9S`XbTJ2&-nfAQxMy^@AL zyEZ~2z(YR1_k>%*j>5UqaVdpzx_00w5YAtoxOKwHY6>)>Hynr5+aCi%M2j{OFXARf z8eD{XJG+Inu?FrgJfhw1O6XUG?R9?vq0qU6!%3@BIC?G)+BQoM*lPrIV>e3H;PhLJL9>H9JY{BQB z0NO;z=OJVQN=cGE4f&iDc)?Y4B4ahGic$R?;9^c#t>gMw`usMF{QWH5eYoK9UM-X& zPh5XXRZ9MVm}~G?(AO*O3A{yjU(4xNDK&modmP_Cs-}|-9iX05O%igs7i|LM<+izd z@8yATGNN1WeOo=l{=$wL>u9Xcyzx!m)gr}I?B{G;C`X?{-DRDSCLYcMDqhofn@boDM&piJ*0Mm=?9lZEH}kzn2mI4^1Nt#WGEX9pN%n7}AdbeI0nCCk6TV2B z`12DrJt(s?Mt3XQYAvg#yId+Trnv$&&%ZuN&H!_vwQ9!A75nG!;&dsoKDrXczmJWT zB3n;`nJGW%^y@z>COyl5=4P-M{zpj`JYae3(o8$|yKwTi#M98+)6`l2QBofC4=x66 zhyLy!?8#}++_YOd|4~vW^sjBU)dHxx{(cs#4b5$P&GWB6dlRgoe->^Qf_MKlBKSsb zFqe&d0$j$wO8S2@9kGlU%|vAd8TIMMoELsi^3Fhl{;kTHWa-#Y%m?+7R;?Wrm^`{L zA!}XT98-nS&IHM|HUt{sGo?PeYVB)Of>ATH;QnVFv2&BvlZqV|@aB-bBhNB&)rbvC z%@Enf32J}auhqne0Cx+LdJR#;YR~g8(X4h6F3Y#SHtb{)0vP@qJq5w9sESf?Yi6>Nmir|!G7(^zf*CKC7CygM zmLVDHQKPryps1;4K+ryfJZ&$pwi`Gls0ekKW?X?}mpnZ&n#U*7^E@waWOkyE@TrV` zPl2Nub|U_mlvtc@X@P_(XJ@yK%*`gBSRDW~Xdi~{dUmxxq}z^-C_ZfQjt9+*0wnw( z>TWLn*)x6$*{Vduzg?$0mF%D=cyCqIj7_k8qDOAtsBkdT_fo`GN*PCg=hO;^flZDp zgBbKE*l(lYRvDFh`O~Ybkj-c(`{^pW43*W9&*P zV$)zHZ^9_n#3mL3Kk}Q7a8}whM2ofelv_9@3f^9+KHCx7H)d=nv8z2;TQ&CChoc+= zo!?nou*FPjk3gOgOS@u6;iqR(%?BkV%h-2~l9qZx73Ue$!9u z&9lh&XU(Fuw{6!0K@Yb!H|K;1rVRbQLg}?F3Xgp}r>cLXlD2`Z!r}auO=4f-1KHck z8^@SV$Ha6leb!MVZCuIGW$~M_+OGhfE)Q-`RmjHT0L*;qMF9J+;r{^ybf?dX)2?oW zTb`9c)NoB%1H{&}fCUhxB4~!IbPI|yiM~r{j2Al~krD9bv0F_FY(E>+@ukIcaK_v3 zDxbAHvf5!7$wfBfd$CI*Iu?p+4PN=jup%r6p5tO`lDE3v7m$C#t3Ehltu?DpifCzY zBfbswYpZ>ArkB70H1hCDLgIfs$po)WQQy-)n?!mi)(aji4C$wE{n1$R6C|FQKfWpd zM}t}6!T+bbLX>1YbC$j$cDMQ3@>@r=V|4yjyX*{><@uQ5;Si7h(Y4+hH&TunfcWW_y3fao{Z%yG?XRQH#_GR(uoJuO z!vXu7`N{b`QR|2TINo}fTd;o~x@ll~;P)wO%!~0iu%#aG=v&ZBeFTTpR@GGY?a3)+ zZB3c5bGGHWMEpK`jr%AgG7D{zgKCfK{ zQO{Gi89>rJIs?Wkjw_QZt^0I>Wxb^j$A%2@CulMr)ITF>9;)fCQ0K>=v9ng(X{?*xNhDzH90Jwd0G}E8c|jyIj&(CeMinm^ z+tgw4RNFtcGfHvm-b2}#@i%nx5ARnUwhekw?b|J7ZVpfakaI`riD#Bbe_v zQ|j*0y`2{jHvF#Z80?&~-S;f|R}F&Y;fi6yS8&;BjZdXHU^mDZ2X;3G-n@D9-dnGp z_tQ^g83E3s1SFPc4M*c_)IQ)k71rc~o2JAIa+MHGfP4ALqvAvL?N!u+GmJy+W&;)F zjeCy}-ekp5h;~FM8n45eV19MDga?L(yM)JtUUCuGBE$wd`pSm1o-6xsf#_qx+`t8O z!_4~HU&ZOw|NP#WS1Vi-tpELEb>!BD@PGeU+kF4`k3V+5|E-->rRRT3=HD9H|77ET pvax!-{-138|4FGRE2sL!il5zmwZYZx;X3$FLFSZn=1F6p{{z&DnM(iw literal 0 HcmV?d00001 diff --git a/paddle/scripts/docker/doc/paddle-development-environment.graffle b/paddle/scripts/docker/doc/paddle-development-environment.graffle new file mode 100644 index 0000000000000000000000000000000000000000..5b164c4832809de94ead7309af49c579135d7f48 GIT binary patch literal 2691 zcmV-}3Vih+iwFP!000030PS2`SKGQ0elEYlo2PSuM84bIl)b{GKsOLJ32@q#wa!v( zCDF#QTv_4LF8=$G>?FPhLK+(AF%Pj5jij+O^XZ#On5{qVhMshTXo!jbVO_3)bqVFi?X_-I#J->lagLExc!eXqSIogE*w z+7ch5Uf(}mm(~Z21@G(i+uPe36jC*Z4257=Kcgf-l-(cm!8Un^n#hHua@yhrrNchTKeU3|(LvCmK+(QU9*pWIHm@VT)A86Wa89(mdc zQVPY*yRDE>{>a;0!x|a-xKGI_s5SZB0foJuhm5WISR^Ga8+J`mx%!G}7>1G(*{V;5 zj#U^j(s2JCg={)Cqa#$GwC_OYT1@G?e7azOovGOP*~el+BAm2M)%gA$Y)If?L;7ym z>V~AN%ELoOT_L~g!LZo1MLe9$nd@@2m&i}nR7o#yEY@Q89!ex98pn-%NOGODrkdmS zJqm9VwGzcnn`w3UdQKN&a+lo1D;T0Lgbe4fhh6C3z_3YiA19VDS;}Z7(;J!eXyI?D zAU|F4LLcIdhJM4teZQcST+}I&ZUxZcLrmo`AQV*s#(LR`C6l^JfgV)cp%n_ps-sr%sxumY>*RULqFhrklsggPNh4K({PS(+xw_ zbfMeyX(m1U3-_c8^h~W`_C5l_0d&)wT3m@d6wboJlCg6jyPUD}CXqAZoDLbJOB)Ty z+DzG4i6eF>>&YGHhd6q1g1)TiO2;7%fuczVTKpy)b@_Ws25a9tJ#PQ3>vejd)AKRE zIs-^U#DH{&yc;BZ>=cdA?+;wo5$48)fA3J_D!Sh3Q-pkR)AdG(Un8l1bM%4M7u;@%r zpO%mKP~Id~1;LiC*+8}pTT>NF1=jo!JWo%$K+pec2!5NO8X+mW(+$91Wm$NO5BB~vUXj&w~bq8^8aWUjb#2DkTi>VLUh(fRP)tL>@suBZq7!DqX-YX7f zMHI^J-%P-r0h0oOW`pFD%uYy@?D=$$a;CKy89YSLeJ)Dck3&`|MCXxp46OX4^ukTq zeagfYjfY{}s2_6AJ89=&QPQ47>GJ{UY5_#-LW9zb0@Lxw4e2SyCmiZS=RVV=2fZ-L zJtW8hvG<&zAA=}9)N^gcnq?~suoqp{I8jkeeo&~E0W=k;ifYJ$!kFNVQyAG=7KJgN zOJPh~1G26I)6`^3$)H6=VJZq!QJB|3VLakMPdchMDs_r??GoxE zdJbJY%1#l%ft5XEfBZOGU=wBW)q<*OHN`e8Q?m?J(5fiCCIjQGlT&?JYBxQRYguKa$1qoS4U1=;#?yt(UC80-c$s2X#`bWgrI6Km1>LH3k3ETBBd{54M?AGp1K36_y@ir$4e*GIhEflc8F4`C1;&D z(KJ~rIni?6^UywCEmKV|*W5{44S%d75c#Syz4{U} zy*x(ZRLqK4nFhF}=-La%NqQA0Rp;%?{5~!uBkCX?{pRA|6F-~FZxnaWcOrqL$bjO zJEZ!G{z2O0QL6%yKV|Mvd7hqhfu1V}m8;|RXCAKym<>iGBk((w8 zPR?K-g@u|H#gZ0BLoT?>A&ukb9Hm44L?~#JY;BHd3=_vMqFPS)WOWSj;7=tPV#!lD zL?sCYF^wb_#m{JIEL@s%ZF$&1;e2XbVs!iDu(J$~YfDCKw#hYoJWN$snU!=Atq!IR zIl=y3ss_m^brw_ny`OA(k`zsaKp3 z7x%_>p`Na#quKAlj9NPJKxp1#I;Vu8`5&Z7F{ehErF{fC;-W@#4=xaChwsjHIESoo z;@R$U+S(zEk>LrXsj^DxWYlEno<|_dI`jBmle3PS098E0A))x2NRV;8v7a-y;zT06 zb4w@MX|QkZ;V19&l?>2-cDye~hoqtXwe#0eYk$xfG%tWIcdtHqPJfrbcbgaTmqGXN z(rfVc-F>~?Rz7|Ga&-zCM_2oe)>kg`@y@&5J$PLGW7K0mvOEGy9vlCR(^ZB-76RuCrtoVF%|(6fQv}#`zXD;#_c;m)waq30`9OEPzHE(e#mU;u#13 zIf8D}_wJwe?sIZGZ>cgaI#->t2z=X^6@e%&roVdKGmR zCouFK^vu2&KK?pFNkI3=(_e&W-)ERUw(OVXOs_cD74~_^GaFQQN0joFh*q*#iVqY^ z@$HYpga5`$oFY=m;Wz&JhNd}ODqz;5h*{C&nAzY#vtgIj6BLv@^9|9oqG8g0KJt=d zu^_;=s0*KJzQS&r#3aRhN}is(UCV>rLnQn=M?_RL$6LmShR(5qNdQJ+~-FC8L*plm?;#Xe8ID$maIi{{Ws>Rm}TS004poIK}_~ literal 0 HcmV?d00001 diff --git a/paddle/scripts/docker/doc/paddle-development-environment.png b/paddle/scripts/docker/doc/paddle-development-environment.png new file mode 100644 index 0000000000000000000000000000000000000000..707ed45a335a981c23b3533984045f53848b55e2 GIT binary patch literal 67311 zcmeFZXH=9~(>97ND1rzEKtO_`1j#wmAS$4ea|QvCCQ6QtfPkoE1j&u$9Ge_elpG{9 zNN5lVO=^%F8@}Dn^UgElyyxe4)>&uGtTotu_Z{|KyLMIWx~{5FRb_cH5?T@h0s=Bc zg}dqm1cU(u1gBky&wwNC`+@V|pHt52@^=V|d#^2mFXtQ;^qdI@F1g_UKQ;M;(F2@- zvewjf(N%gRZth^mZD!%{*pl1B&JmnVKp^QM4nEphx|lI}*xA}Ui+f1j_;rRj_>4cy zbA##EDK0irH*}R$ndBUtESZG31-W@|NRu!zF-baEJP}vFd;dR|gKttdp1Qa=iu3Te zySsC{3vfF)S@H0RiHY&>^7HWXbAdCsoIUMb%sja4otgi*$^YEvuBEfNleMFZwSzqq z{=R099b8?cZrs2>=+FQD7^jQ%lm9-+-uXYT1zwN`{|gTvH!shh_Xd|r;*W~UIoLTm zSvos|`%4Q+{yOt-$Nu}AKh9Tu>fqu4q~K(2u4wOK=>#rzF~g5en*Tpn|39Dc`@K}0 ztS!M?|GJv*KUe?t*niHKTq)AF=_|XIeG6afuWi>rc zt)NMdZaL%H*4N58EV?Y`qU#OEkPkX6U#re4o8?7vMg7!ye~-Hsd8>Ao>Ql7$t*ELy z3U^<MZ}GSFre3hGJSqiMHKad7Sq%D zcYNlpJjfw+6tx8@1>5zX-^?Rw<4xe6aYc{sgrZ8>ejj-d;eF~v8^R~*6bl<&JdFNA z6%PpWnRz`oEOx)jk!$6Ah`n-(kV1yw&p(I(h_hDR))j_7kNoAgQUZU(C(Z7c|6C)Q z2G3ke?F0rQV`#M^ySTOZv?)|Ac=?xwY*Q%y#6+ZUl$_&eb`^71&HKd zG5%d8|5cE`uG@bVU+4IQES_q%TQNUSwOshvWL{4nbZZH1b*hvKm0j?bs)|`Di{|&!z}DQCz>eG;HjR z?Ov3>_UuLd&X^5ac-(g8g4EX6ZqM@j$4ffRPH87c7+>D|E$9bl89Wg6+rtLvHt8L{ zG{5SNTa5p3gbU>U`qH#J;G#C)#fK(Ntf?N4uMv7}CQYqO9!~Do35^&Iay9M!fUg}L zj5NC)u2e)wEM~^&Pa2FnG&df8KMP$bZazMUi?{r{`TXT%O>YvybeI>`{7&k}ZAuy^ zz*DTWaV-Eu*qW-iX4_~R~kB`2%oXTgJq_o$6#udpTZTRv!!bnqS@PadxRM8YBX zW+cS#XoeB4??GtWe*Vqhz1A-ktm9OzrBU7Hw3t;0Ov6?Rv>hiw#@c)VTWheR5- z%SW8#nXNQrVo{Pee}B-@P1OSVmR_k{!`&aRdFpEC6AK)<7`Crfb+9DWfSLP%TU7Cb zi3%%2aW3wM*y{S-<`bMUGVn=H;#|`XvSi=$3W3_eaCOrhckS#YsLw_RYdsOQ;a76j zlcOUH)ScScBfTDz>h1GWE+V}Vhjnm2Tx)JNfbY-6RMas?NDfHteGfc-R>f64QEsNc zhmb@OZyzj|tVwU@sRZ?tt&zM)cIsqb?Uh*D8W)BQu~lz$a@2=jX01cJPf1lIlW{!V zn101k>?neEiK*Y6^hP`PNnyM-4ZIe!(DKq77pF?yj8&{hjNDU__{@XBqB)$?U{%^_)f(t{hCLJ#ByQ91ikyjw2iOzF-qH@ z2V2(008gZ9S}E^JZm{G!jg9CLnVNv^#3LPoE!v5H7wv6M)dHEX;KhEpqt4LT^4gC4 z?C|wkDQjbawzfn_?YtvDF034Q0G|L7bvO}A{dndW==MQ;YhJ60I6FZmK~!{|s}7b1 zDd941S6yr5Hu}Z)aD2+Gptug!@D8F*QYwkt8eFg0l3^PGB6ycN3++n~X!BilPTM?n zIZV~`KGOV3`@k}F^Ip&)xbtctZvsa_Ve8d(ob`IW$yf1i2wa#s!9ageEN>o z4h|Fs2T$fT`}A@}1@$9A+PBUxj9y|GQwEf#b!I?F`N~ywI=TGy_h<4Pn)aXU#-ts6 zt~SV&34-zD1wYJ*3E;~YszAGq*FNNF96AdxZrss#-|Cl_m^=J@Sw$38+CQULbd#%O zO!z^5&?aupZ^|JifJc97!nqG?!$|+^^F^`(r8EB0N7LkorctmfdBI;lo@HD)t+hMl z*F5gM*67TL5ubi>+2;f1oq71Wr7G`3w|t{4(>{6Vftm>ec=PSsrJ3x`LjOyk~3A)XCK=A5wovMngS6 zQXe4i_4!eRr8N!F>!Fy@CrNhmRvKY(Q40BXE?;HOilJ!L$!O>57qgQN^o3w_iV;5F<)HbyIA@?dR#N5c1R0Z~u~&YoJKk4By27_I828P%oZGL5 zh$gVJzs!*7Nw7*??Aoht#`T=-KKeYwNYpdCmrm~6ezu}g3HH-FC&g{UIo~ZZm@;y} z>uqK81<~)%_na3LdsSx1f8T|jW>Za)`I-?SO}p0gh1x`tLHWYbyh!u27#1bif!EHc zRm|&H2JIdiq+YBt$x}A=59#t>#1po@y&V(me85py8ZZtkC_1`H6!1!T>)M!XG`4Eo zJ_!k%GL9OXQTw2X43E{k{J@j5>f7TcdPE^nTLdFxykdmAIB~I zE^!x9Ez-0whR)dd-BBu16Gy071|IUDqB(VZwfPWSOSs8@}A$IyHfL~MyqsVit3gBl?5yp=<) zueQO$?Jf$QHG!7$KQgzYSMRpML|n?YWp9!qAyC z+jg>3-MsDpt|7s`8RrI76pEV!K5M>bm{(e|Zo7U8S8V395jeIpY8ps3;k8_Zv2R$v z4`nBGP2|d3$@If}&Lc_e4m+TLJGKTX&R}P8Ric+BTamv5#G4|9vGoN9AIfkMy^Q z5>^{`CucJ~bfsS8u|$q7sRg1Qq=)dM>JET2rtM3*_I1WbU)%aMR~6e3gQ}Bf@WtQ5 zk?(#GBnhE)jQfa2R%2p1gmHDX`2?P`1G}RhmTq8Q`{iudP?k^m*%ID9 z$2C&U(&uTG!-aZHqBRYD2ow;+(V|ER@l|DWl~FGR3G zT^A-bg2xA2i>og#EC2Klr4e`{+BnqM5Wu;le+*tV{Pay&%lsRGUPll(RnQey8m-8hGOMj;Non!~we>691+Tcm z-SG3%gd~en;_PW56514xAE8YgyWl;0OXNuNT6<5Oz4kp0gxtZdY)`K+C~A~)CS37R zSG8Lj%4?EcW!K2A`9fvToktZ1c^DzN@iNUQ4Y_KeuBzC@zK~&e0IcL-odg; zH?`MVD&@$2)fw37!wvS*j2(A|s_*duVl!@yGs_f_>n)OP-!78dwy94tuQHfjcI+s@|Y6|g#Ed| z$*^ZbwXjhfPCxkuoc+lT%@>eLNv94vcKt}as<-!AXky8*^*YO!!p1)1R zR|EW3W!S*U>3_)rdNIo|<%+D%zWV!(@e1r>1d@j%9UT zTL0A*48n+&Rqprb672j%AcO&4Gs?yP*ttEI838Fr*OcV*|1b;${*%NM#7kikwKx8` zs*I?aD27~}`0xJaGr`w?U+izw_V0}SO&|YN#lIz?{#|2#k4OGB9{<;k$9-3+wi8QH z=k9lWGx7ScJO)dX2-`I8Rk7xDQm#r>DdCy`gZ)2oG)mt)M6Xq%e)A5uS-ygVh1qxJ1 z+)&``5?7uIU;f9fMFU{`=eW^;du_PGp2+dpN(qH44Ua7g{T*7(xsj@N`S`S>#HP94g)v4q ziFQR_qu_FB8*O?m#c2!y>Oz25my9>FzugS=ELZAnfJkBPR3{j zC8^W#r*Fj(X=Uq}F^u(&m-1zW?Tg=6pO)?l`6E6~Hrt|42zQb|q9)vH8>KTPX!hyF z(~a4VjH*d58><$35P4kv?`WBxKR?&7HSo|Lq?1#Tq^U)74C|WE8aac_IlYmCLmJS-ZiV>0z279A=X+{suT;D9ew$aX^Vq7 z_51UKY<~+?u28-cunGt83ff1tJ+8LLm~Qm7sjAwCSH1G5^C{8KuF&}>lIF5}71zj7 zc8xc3qme}ctZ~Ov_=N&imNKa8(mVAXdO?%z&l;YzW>#9ZK}>5jj?S} z(Nto>4c_>xbYwjJ0kp(7bber4*zU|*R_wSC*p`nQ4MqiABwiwB=2&F*4=R+c4*?Gy zB`SPYU+ZzLFkb{WBJ5iy8;eEw z6~uj2=qhC=NDAvgVph`nZww1l$RarDTmK>gRdpF?)smON2)Zg?@s>{858vA$WK(fJ z|K$#W|8+=`b@#A=gOmk?`LA&oLI4uFZ0CnVCm_q?;FrK8t9S>NgZC8hpR;{zLH123 zDa87}xBExly|i<0 zRx37CuK__ZshAeOmkPos+nx<$vrPH`+K!go?2f7Ky*>l7#gur^wRHv+_N3ouCP-Y8 z*Q)ng4#Tzb=+10xlJ{zD1;B#UY@JBMG+8M7 zexP`hYcfX<^yFw?U}rHl-q@=H2T)qqA~^KW#hHx?;>-Fo!tbah#brRzK1^uX(ZCL= zHT5=?OPw_K$;1d_Hq#@fq! z<5S1O0~^eIn*d-MRLw56^U|S^5gEiMjtzXbQE2L>pG3PL6C8e*Z_F6uyNjldKLG%v zgSS**&36a*CSCP%aeW;WV9G&(GER8t{jmm-R(@P%k2oBFw`=*7DH}Ib&-sfqE-I~p z>33y3*=s|`s7ULD@@pvIHP3(b9GJpXAu%vaa>CVyO=b68v^#aDo{&EPprh|>?uZ!5MVV_r;4p*fk@c>~?_Qpm34Tyu^;dqJ`)*V3T zyIm!3gkmo(d}-S2)M@&GN6%{mFNo)Aj_O^-FB{XJcxyE&ZTT8Z>@8m3gRP-=D-n3C zbPAQN7zeO5T<(dbPV;K;$YmAD>0($5J2$ z`+)$$HDV@SN62MX<*jj$-lX2&Q3xLYu-Plo*LNw7&i>e5Mdwf)AOyNCHtS!g-SJF@ z36><=AKdjT{M7(jJ9up5jYS4k)sBzLz)vo#)+8*6~+FS9415(>V8ulOos}!%=LyO8+8v_>g z$h#j3UO4&iaO%X^WyeEWXy*{1#h=q8_)5XdAE1sl(@r?I6q0GL^w_-gs@cdjvy+M* zc2FGGWDzoG(ilI&W4&m<6P!H~rR}`t(I$~drE@5cJU50`5%1l)s$Z?1v$_1jv%**Z z^Sy`>!H0d&6Q8cboI;IJJ6M&J=U``Yh32GNgxOoGn>(`KdX0618gCvVZu|8~lovX~gf(Z2T&a#6u5cq^z;!UEJ`b5O2v9Fm#u?X+wv` zZ6C6vC#tYyon-?cLEpgYx^`nQKj?2OJB0sU|)$Z z!gOqV#AHo)%Evi}R(&kHx@o&;wT1Dx6!2)8;`K6t18?PFWJuR2F@kAoA&UouKHYzu z)F-~oX;KUbDeM(K;#AFeX2(vdi!E1q`we)#;0b;mc3Va>Q;LoOND22W8<*+Z06^%v z68JWNw3veTTgoJ@2Qst$8#R>0z9eCRFwFY*8s8rM2ueI(X>PppPWAMJcgeaD!S_B!X177g5|ugPJha_~ zE~~}@PPfZzn}Y{~9T2l9p*{L|`Ek?6PhZlIMNSv8hmRk&+_tY<%zm1e756G<`P)zv z$z}<1SU(D78X8|_jQ(N0<6$N#ErnQ18-6yVCNJ~|CIscO{Ycf!O<%G&RO~rwerHO> zE-hSMY{9#*ue_Su1y?G)+^`V>J-SZP_NJGsW6ggaX{zc~@!l;t+OoV&YGyNNj9!-ZP7qNgvkedDq=|TVG{(v|1bg;%aKr1=wk^MVZ6CcSH&cV>e!-4{mp!m2 z(!0@akXzSPg+NwCYur4xIzh(3p9vROVe$-juN2u4y?+AMz`~PW*h4`wLJ+w2lspu9 z%$nxoWYNqoU($4tZ&HGK8G7YzY1m8Yy^rL=vzAiv_ej*ACQ4nH4ZOXxr)?3TeGdwL zc@LKEluAYM_Y9MhgAuYWs&j;TWUm%pGKc?MRxMowafD}!_||nA!U3|j)8vxhF`n}Y zR__J+Bn2)!VSkB8inb^}lQ%=BQebykuVt)hg#@M>*ar7T+IeNAY(iwop+}Ba0^I7p zc>oU>qbE#%;qcM^VsHmcVGoy05Yo8?pm6Rnvk-b8PeB!NkBjz*LaIlZA70L@sV+P; zIZ(Hd;@ofOvURTbWWeQLW2kLz-d-r~rpZ9hqs_rqPD=w%Z3gINzjaOu)!9bD2;&Z! zNQm9_r>Dg|-_xa6G21QWPD0^)3b`=* zhXV6)nET#`oxGe>TD>s7w49ylXsK-4jyok39>eR5t$Jat7X3~kUC9+=QX?<0^r610 zJ9|%ADTX?m_JFTm+s)X?=~dC!?tpRJ$1}o?1nJ{-j6E$I`d%fs309zl!{YnIF`qds z@P1GMWd6!e4sH_O#z(F1CBh$DkyO$Xmro`f-7DQjMDOkH93ZsncL%Y2{Lpf}FeW0> zLYIEPMmRVixg=+m=3A9koV0FHF8}Dr10`85926aMR0?Z-p-;54pw+TPK|IidkXl9k zEDwG^Gj{8Cywvcl!!=rMQ%0A~s}MDktCm?3t2NB2ZFK?;@9v#~6iqSQ3)M8gmEyu| z8go#XkQ{WToK@x zzN240JOcC>Rq}jZEDlZd1cq}qby*uHolF4|=h5G;fS>0R_Xy0cQFRL95>6J!)62866lEvG~q6 zb7q^fpGG1TidoMah%_Fzc2zMoMGf)P9TbiLniQ6IIntddsu1^fAyV#w^*btvtQmdJ zBr;_<{LxlBbA(`_IUpNu?{n-S?0O##2fj^HX?_qA%ju{6;bwOZQy}a?o)2q^i~3cp zP1Xzk<#|03sr305LJRbZXX;U4b|h9xTb!tMxH42(mW5bTc_w7?4C@qCa>~b(jZHS3 z_vTDQ;uf#K`1I-p5=3xj$Fnn`*C}p}e=i)n=D7&S8e5w|v^Nt~rlNj&pz$Ou3pJas zV>}t=f|>mLmP7%aiSbe_(O_{2ln>C_6i(A?9v5b|RezFJ<>JiZSMK1~GvCqEyFXm4 zoNOhI7~|ZFtSuk8&$4^308_}1Nu0&I*ryep42;Y`8{B4&q&2xJ8#2ejNvIb0NFq*M zaA$Z}%pp0PBq_q0Bk%_7r7{5!Jxb&Ld~o5^q_8jIop0W8v@v6oFt1l;}oQyzx@k`_?3+(E}KpmLge~erY?o6;5rYM*H%Yq}&i1p$Bwv zsH1f6gF;IzACHyYc`>2LL_-#_67&aA@6WBHWCe_N{z6lxZceG%A=qq5_lw@$J$V-` z3hu=iWOEK$?04Ba*Ezz7^%TQ!*&0uY^*!~m&$njxDyl~96KDEpO9_%rY>{d{8GSo1 z&04m8%95Su$iwj0FF5bi#5=4QPDOR`G*2{hic&ps->HeKXgfK2X6~Qw;gfLmc|A65=zm=_- z1&DjTMPQR{?p%~;B>&{q;+-(VT$mBFMpd(2puxQBIc3m;2=3VS)#Q1%ICFhjGLAFj zm^|P8(e``AKM0xsfe(i6$TS}6&Sdm5MEz{!Tf8r~2*S6`wH=PC=Ic9*!TTA;`FM6O zqlenp7TBlHXy=e^7M=9P2TNug%kxk%hYB#bQ_-sVF!Bx z2W}S>MpxG`HUr*n+aQV~8Y1hZGoVx|E%H#`n8hq%t09I}#Z5c#%UQ}HN~D+SV3*)Vtlb3 zyvyGvuPe_wCHK8&Ovl-aEWX@JyIVZs*2v}KT??+HX$?!Acyw(P1Ec0<%Vwq<;QrADFr{=1aApBIqgE)^Mc7Op7!h`v43P-)f}DYqGD z@EBo-bqWJ~>TnWcgVU9ucoufihNJDQNvuxAm`q~v>Y!Y?eEX592cx}S&$T%6t{1LB z?nM5Dxa$tLSZzKJjl5T{?!w+gmtxSdA=^QTLOZ$2S1Fr0uEroueu?tglM=EQuKM-# zWHFU#g+??`Zc*Xx$UW1DXSbwr4!nhZ3_;HLXffnbA8k7A&Fd7IOc_$;n8F-ME9}`? ze4Gxyh~n2!FBMGaoR2q*50#Gu38b?rLZs&drC- zBNtH^4b!4Jw-s3X`?KAGRF`&gBl=yaNz1j45J zKC{hxOzASqBS#X#y!p`K9E5)-!X!nm=yiiDYfi|R?6lGHAlYN$YZgV!oQ!dbedba{ zVm9Z;BdUtyQjqd9g&Zy?3#kz6A{O*rJqLYhiume62>gK=^d_duD^`9`4zG_LM}oH4 zsy(&*u$V_AxJHa8xv)3 z1qL8PXLlyO9jZ@`539>-(nQuI;|!ykx*kj)%|L&i1$I^jQ3cs2qZo>NK_*Y&FH8A# z#_R#i;GW1H2{C)n`mtOL6m}YLvy5t%`%7IY6kb0;6X#Y%Mh7Uvs#$rkV5`v7n7rG7 zk@Bp#LdzIH3}a*+x^^zY`e-A**|zS&GGtsurzm>P*n9RiXTdo$ui*!UukEbDo*gQ7 zSVnK^o9C{}OF`0;F5zg4eJS9S5ms!X6fjrIp5nI5LOvsChnH?w;w@*x@*~S?v6PRG zUvtgNXPuU%rLlim0yfE8kIB;fb}G_@e1=0P9KV_y4eRbjE}DG?JI1wMu*+v$IXCmk zJD0|a?DC4k>e*e7juyxKstM=u)rxtUT6pHZZh?u0uE2Nky$VEF`+2b%DER)w%^T!pmg0ld^#c&j?{V>(6($ZGsmP`ZY@8VOz~RE zgA3zmQ&C@c9GAatT~-#ozFT%SA^5vd6o1gseOMCmm|~?PA7$QHwSOa*su#a*Rg;6= zmxDS-p4eQqW~`h98@8^9;imQnB=_^PBOng>LUGDt_RBO~BR4hVmZr;iQP~f3lxO1& ztyp{Hf4ozOL%y9>mOw_kzY|G*{RuNY34-d*WSd|Pot*4yXP?|K;jF^7u-4`J=gmHQ+UV<%X@>V(Mhu<9PE4}a?1#k%Hd&#pBQya%9gF9g|3ErEn4asd?CF2~ z;sXg(C#x+8ixr-LGN>?& zq8mqZ8-w@VVO1x|9tdRV#EW~w&YF@+r_&ETT)FJSYMwON*JG>N8R}nQkit)n zd^#xc?AxXzs-st&GSSIb^bDk;h&qp4Yy?=#+rHCgnygjF&(sz)&LZWX^MG(5e!kLN+(2Bx2 zF}PRTV#U7<3YsE6`lPx!kMK(Gr*{jpj>Iq!`8_}RpI!iQcB1zS-ITqYEXbq|7sphz znOV!fD@jJ6YF-s)Yd16un@5~{87xv;@kk_+U@mB+P|+CqG5p*^5)Hfec`*9)lCl0W z(m3=+0gfu~@LR)>J$%?Ermw}T!$#MYX0yEQ?n2!JB!SHG0lc(L!MCMRBHm)0~2&QZnRJMA{OmZN9a5^SDsmYqy)*{g?5wRvUhP=AP6S${V1 ztnEv$xF1X3^g)`LKUSeFyh(!UJl%WoiKcIWNqtm4JZPRd!*=80lQYXf3qu(Myf1n?6tco}i%A(0V7k(&z=jIUNjU!|~rjgpDv4k&u%=OgQ2 zcW{26@f|hsKzT=!qs?p9G24osqplght<3wx={rXsyzE zP*7Y18DnD1q>xZ)13c~34yG4X1@t*LEWy^LT%K=c=BAH)jI7~X4xYDSR!Hw+_VUIQ zVnU>I?^jQykE!3E+GXvpWo>6JwCRNq_TxFrAr%nW7VK+Kallh9|M|x@Pub^7aNN$G zt)#LizPfOoo5PEtA~6?EPqG?~Prtthx#yKPDt8e054Y=P*xa&ZS3gc_&%&LNSry;V5dPw(A2Z==NpI*g?;7Go*dzjjpd|v)q=fa$oNUF%_p8)KMZ)K;@(xez(g^^MY8rNSu;g zqe3PUMxoC<6%|00kIphjJY`~wtxa)`i?hQRYFKznTqK5JlGE*?x9gUR>HKzQ^S;+2 z(<$}5E$}IVr5vzRd$mG}x^cK-1*IF!Xe0AnkUxV;eVCR8QdBEty3NvWBk&a$SHPCm z!fOANa>f~t@o$wQjHF2fXCSAU5HepqNI!3V$<95?hpb2JIBEmT=Q_H)pkKxHRO*OK zSsi~fAg?E*pB`2j4WLiCn=_+IkzSWH#;IX@`eCNjN-)-T1~X?ElfGTOh3iLS7b9%9 zl|N)^GA2%o2A{m9n;BkD--#Pyh{btL_U1dsFU|@t@0H194#KPHoUKkJP57XGRuGvd zdj;AIn;Q(13^ByfmTHNtFC~rznCn%z+8X9;+I(rCZo6CWnBoSjzE;GkGoP$AZ#ejX zCaGtiiMy?(A`oBBfsM>)>>YMJO0y?}}Fg_p7v3xMCo^jzDMuH1epze)KybHB>2L1QKHO1oY^I8G`UQDz$$)Fgzjj>0 z2WAAPq||GG`0GH$LQRrG6AcAETFZbeIUUD0A7xckwI^>Ee(QIw;K!fr4+EfO{?2j6 z)-PO>l1db;%am=B`3vF;2oeXPdmy@lUoH)&IP$z__d&1E52r2J83>ev>8rVazf^D9 z${#uHr+Odi^xTF|$C%(N`y4#?;d(N%Ot{A+z6W%Q`RpugLNVM374An?!Q!`?KCB!*W<`g{1{uAd>N8AsFXp2dUSL_Nz)i3QRbD??7Sd4uf_8s@Te}RU3Rh`R1$FoC7e#6W@}f* zo`(+QeX7^Baq6rqrz|ved5?Q&U10?$ygTEbzI!_sK@G|9iemS4Q!<^m5mQ>w54U*n zq5WL%8g^wJZBG%Chog z9E&wc{FbjIKy^6!4k?$Qx?aU(;yK6Sg3$|W6+PG-XZooiX$eu7b~5~c`@B5V>=U6# zS9DKXg{$U!K;2Qyt308M21;_59M*ZME57hVC2COaBq8$u{fMeH>lMExFceEO$sO91-V51v6g z%4~Y?n3bO#!pw~@S8K}*SCUx8mF1hqy^tvY57DPIKei`v*)(i(PfzlWYtW}$n0HTq zTYi!F$C>l~OTdQZuQp}O$`OfQA|CkiU3|H4#-Wr|CH2~~Ky>G=Q=4uYGHu!6YKOP? z%;U_FNe8ID*Um-L%rm5Q_D|6{-FMAK7~IbU4%JMFxa@|jI~2K7a$W9 zJ|8o#1sn-T{nw z6KUgQ%dCl}X=XF&y{?a-rIV^Jpg?K)F_sgRkj55Ybomj=+A8F)kJEawe{ms8CYR=J zl5FDyDXJsM*~oQ=@|N9|g{|7~wL2L+Y5B}w7uW7c7$}D!MjY;(dG>PE@yxT5rY5EI zXFU2|e&`qJOC)R*LvAHzOSde7D7775bC$ov7tH3J(?ri8;%=n@I#-hDv-jQr7n=wq zmsW6ijFcpKWhaQKd#v1zN`s_Gj;tFk^!Yx@dv(rbqh6J>E#0}_=9_AaENx4MSzx8e zC>0?^JEbc1I$#qtNB}(G9Axe`U%CmArjcP=akv?M*G<9=LeY`A4&Ek`KW>-%i6Q5W zCnV*Fdv@;Qo)@VqGa$`Dg#ym}a*kug7q{?@I6yjnqWKs=#5HOeqT49!cbXSiBn)j2 z$aL;1()*cyJyN?9fu(x3a%YJ|M_eglw6-51F5?9(HSTLMA?vT~L*02LgjUva>%dPB zM!0<<{r&}zw)%HGQwFeH+yHoRS{4x<$2lo+iip?s{Ic#apVA#Bt`D``T%RzvNnC#5 zHS3jkDt``<__~ooNlYIxA#k;_0iXfJRb77&kL=lv1Gwg_8{KHB7x4>o(QAX--(Xec zxiX^MpPs3^h|S9(rU*arT?=2-?xKz(UAA`hoR>$K{Ba1*PCb{~1Qx;4np_p|pTQSB z_NGI8m6)`i0rb@J6n)Dn51a4%_G6%5bd?xYYmacI>Q)z%$6WrCowyqcvn>!x-w%KNWOfUH0W3QU3N+ z!-6&+BXGC80T5u7UwhBcO)Z$znJ$l1cJp_Ao;yE>s^>PPHn8Q}ZQ5<@!`IHQ#%d^s zo;JW$a)E?e0nZ8GSD{tDyyqm=)R&pU&F&-KPJ?bJRZioRDwrtv?T2%2v&2-QYkrm$ zYpsC3qg{8`vrTStZ-Ay80zxpR>oQDi{vVmZ5LBuC0TAY(e5z=(4?2aLI>DvULE#OD zAIagW@jX@&e<_B@Eh2~rM!AhtO#iwp0(2l)1qkCC5uJ$FQpbBUzv!W>^=oeU5>szH zX$If41E18ackr2^TWD?4c*EfM;nXTVVh2>RBs8Fx#n<-Zt6S#x;0$8V ze%!@ZvG19>L*rx0w~d)_&E~#bEW{zRRnt~@+AV|uuyLZUys*{e_Bm4x7G{g3UZLZ>gsd3?H z_<3j0wNhNovk7o(xVWss%DY0ge}>Dv1u?dkwS$^As9JsXJm8wi;n>K; zeLA~YDnm!v>h>G2_u6i0+RTF-qj@^N;j=H4_JD)-v$-$=^z``E{w3ct`$hhZ4c6y&8_BOcP1@m~jOG*VR7N_#AcX!|0JcG1`ccrwb8h`6|R zp7x}148%Q+j0C z?j3my;BZZRO-n-|N?>BX)1)RpBc2C&emcbO??-TNOMGs<*;wn4bT#(|-8gFS{U*|u zwIm*3vjKgCON7D|P@U=;cgDK7Kt|^HS~~xNVHp4f>+&>ms$2unsvm|zv^SH)oU^Cw zQy5(+BD8wU7RPXKj|-P)BwqurUg~|o?E^;AylBTK-SY-~{no7Z3 zz%>}@H|cqE*7G9~^<65a(2*F)DVr31>6BT-36WX7mUfE6Hbbkm_uvWRMrX9K-+{Bl zJ{rv8GrgwGl(o-uS~M{j`?n(jPoq#Pu=c8z4Kfa!O}T`b+P9VlIFkgV81$e&;)bvL zr{j^`PY~L5t1y)llODWJa;O@80{1;AhT}8gBPqvz`6rfC9<=I@e4AaAE_FWjk7x62 zSpHT6tW5~#NbfGZ{ee2OOM_l1EBEWpOq;CPsy8$OEOmgrNE^p#qw2OF{K;iqj-$<) z^>vny-IODbHStZtd={|fhIzl)5Wc+dfN3F+@d(k(+fNjQBH0jOILJ8w36fjl*vi2l zudO+9(|CN-9h!yr*Xv~a$yAB$zmXJ(K+xy?{HYQlzURQkOUJ&%tSz&%xHX{S}{JY zr64Bh8ZViTLOp(2h|XL9?I?yheWScwWxr9=sa#Jai4_wL2B~D#YxcC&W3U2+DU%cd z7w$b^aT&RQbvpLWGgal!rGUt4FY^ru`nprzEOS#mavDQW#P%y8n%uQd_oYc7k4=t# zP)oPJpL5B`rL*O-A(QbYO))R3$}%aLPvA4ZAE<^`l{|uK2y;#v+{*&VfD=ahZ1zO; z%Il{Y+P@CAgXmVC!itOrL)Z|{c#I`M)D^9aE`~1h9}3rp3pWJDy2_lxP)Lm ziZ}eqO>IvX->65H=POgoyI8vhdZx9*G0Jfd_s8HMSNeAiySgqD3EwVJu1hq7(vqAcroj_mv|83P4*P%$FkTVt#GloXSiZIa#5Uoc_PDe%)%o6g8oVkU~*! zhr)x2_$ENCv1pRm#(m&3ehEg`0vkWC*A2JZodHcuMy?NvOWoeLb428@GvHGa6@ecJvYdvfH zb~X*SVuia|A40fPeum$*W!DjBV7xZL)whcNX5tTN^-zlD^ z7dbh~+^7tPWsWfT_i$>A&I#y@g)k$V3(u^Smdx^HpG0Xdw>gB3<&m7o#UxoF%aV^p z^1`5`wl`UkLWRNy8(yC(=-j8~Rmd#e_%*Mu0fSGr6s)457d^-M6pAxeHacAS-pFVH znx2+xpPD@g>uY3cJgTh*lZqt>6$b=@NblINwk--74ur`mmjKB>(oqV6&Gi-xp*ke%N;;8S; zWlA<2>QWScJ9^g4h|nKj$#SH^L`YoNNIO+A?8sf0Ua_N?>~-Z~_H>g$K9`tbXA*0{ z@CV1`fvxFY8_#7`9@Qz_s=IGQXx;x|>$~Hr{KNNAM@7;wij1axl#Gy785!BzLD`!) z$c#!wRv959TlVJI4P@`VN@W~-W&7Pv>ht-0zrXK4y-MXgp6C7C7dec+5ShZ z@12S8k-YG#YLZV|vWMGZ?_gOTbo!lu17w3 zpIZWBeF7FHMkT70k-EGhP=oPxf2|#**`-%X&TAys{QAg0$1!x93cK7_)IuA{>a&v; z@oGz@unPR#V)-qeHSWbdm@VuTusA&2d$wcT$|cR`Es^-x!&|%nf1{F5f@gsejkCSixG~@l?@?nvWx`zW-RnzA0!!tL<*|9R?kzS?5+sJ04 zWqRQbwN#HrUaVZ9TZnOM%&bFARC(_5sgq!m7nk!=mQtZ3+wBW4?@7N;kKa!vR0l8J z)r(7`^76Q<5=CeJ@h0{A_oX22LYJ}YO7^=SSL>R)aQWWDkrSu6y%b(cirE&P>$po3 zG9OBBrg&V+YaW?(DYi)_=$A^$+h7sR+SA0o|~_7Zoi@J4is*LQ3U$X;z}HIW=m+qt>Z z79S}Py*~2H?e)C;jm0PVcbVM?q-SlX>>3PdB_h?m;*HZ@di7{KZWtRG&=9GfsJA(o zz^X0E5BQ7Mu_I3id_2f3!dCpW)JUl1-bF`dJmkJx=!U4LsEL^FRkp3?zPCyL+OnYW zX5eT;S;^7HFn4vq+rZ1G>G`kHQ>b4e_m?ognXgx(x-7}#8}h30NfbF=qF=soGr{qN zil8)~FI3q-f;16>-c81!%c&d(@J63n6QXhz3|QG!96>FKAZgj8PvCxOTR9%U?(O{A zve!=KSl*-|Heu$T%sz%fd|o%Sl%_IegfEB1^h{6O{hmU*U7lEHzw##NmAS5QQeLA; zroBS#(>=S*b02X+t;6hiy3V~o*H+h&*MjoxM_gWVc8RuVU7q3GIh{b$AYDA{7`4Zc z{mW2E)E+vv6`rRmjcP192mHH!=~@v?(MXE1px$i6`}I(2cH> zT~0!5QxY0HXRy|!70gD>>ObrROxXI8C|p=%Yyzzd071|d5pWNXZxtMxxcei;-qP8S zFFM6s(mtY_$%gaM4Z#wWj*a5Kp0dc zD^UymCGy%mgPWO*Se|srD_xDx24b0CudwX`Ak(+dQ8by$ia9+^EsaT?_Ez-QVd89B z@r$NcS62EM4Bm9zxLZo9&R7H5pNfO0zfwyzCk%+cOYR2HNcem1N!7h$)S2doSo*D% z>M!hM`Ke%3*lZIY;7{ZR3FMN0z&Qi4-irzloV&jF6$W36L;|PmT7qKS*EEs&(YlID zK5;H@gqi0B7<>qV%(1MGUdg_1PnFdDu;;(B^1b}ay>2emD>2>;e1e@Wd$;Ebc&YKnz-#hCt08+=#S7O|Xc0DRz3p(Et^!c#JXmH2R{UbdiL%)U=F3JcLyz1N@mhL zS$+CzALxUOwvzB@)7_(De!_$OoYBp=l4@B}-@a8W{!Sw$riZ@Qcjl%@-u;bUz+=y9 z+Ry~=e47x_K7M`x5YBH?YfYZFyfW~e_2>uRrVwXxy^(>97j>GPJi(YCK6YwhjX}QP zrNbNY0CvfmZw%8Y1@xZk3IAnX6Ibu?xutm=(P8PJEn1oLYR@95h)5nW-S`baAfoEg zF3&PP(hEM%I5*=gtf|!;TTgUv@p^>jcph2ffd7kFYs?()OLB%|!s}zs4@u3Rwkwux z=|hHJ#tVYbY8<;dE(%?QNLVbF|69=rgOA=tyX(b{)elHb`ND#S^*1*%*7OW{PA4Fc zFMY~#BlPYgIom5!ojcn-7nDJrc_|TD_1{&p@>ne0Kod% z#-E{|&EsnE1zKkpw`olIM7GYI)13e(#k1tvQCEXesPp(AtZ+3|g>0QcP%qM|_56Ic z-)}Efacz?qYLzXjO}!}?6Q4=j5jN|%xF;>3t$lrwi{>|xNVdx>L;?h zBGN_l%;Rm=Ik5F!@hM*s8}(Q42yOJ}^PM@AQ}1EvyW)xIbZCB6No6|Wn!TMtb?a z?wNXeP2UV-xf`)zy`TNPY1S;FjT_~9lzsH>^65MzOc;84>eLe28e3yxC3d^P1S~d# zg(N67n`1t`>VMp$9bT1F$cy?mt8^x_R^DO2EX9a(Wk3T};$Rn5&K&llCyC1t6|{u* zxh0`?;oLEtnf7dN+m*CT_v2{UBL_dfJgOoa*VKFyq=(mMuC~_GSOy;W2oPRU^&A4o z-L`Ba5*rXjWq@gJ@cNC^*IH&=4@e7anw=6QUJa11?>Hu7}lNqZiPWS(t}W{+7abic(|f1LQL zKg}oq`%(A37CuKG54=YJL^JboZrwnp!pN@b93{cV_r8ViJ1yv%U78x$Ub~#!7-!g< z>{%94`)NX5gRTL)MC`_}K1JKzP^sYpRVcVKm#Q4M8?=&B_ zqHqr_Q!$){PZO-atzJ(>QMy#_46{FSd8FhjHeI7YMx1UjXV_sN;c_%3+TQSi&^-{> zvD?mP6ou*OOqi!ve0MeO+7lD!_3qqq9x)UOKc?4Qvja?iSk(2h z4}#FFtfI3lXmk?&U5pCReHfuAQrpsp6G_=Ch*?-{Pobp;`q3EM>F!MK=UZ{wj^7`P z#yqkfnmN&4{^fv(uGPoM8*iM@Dq}{_#m~OE#k8cXf72_?5xVig=3?{4G|b!MPiUv| zw41p{9q+Miq|UOS(494n=kb#(zjO(|8;gBEUuPjm{Kbl+% z|3i-g(Ht)B2vEJ37Bv3XE+j;dmHG(YpOVv|EBuK9uXQ`eC{f2Dfk^cW~)@cnIr*+%^Uk^`i6`P;O0^j`o}X zBk1xXq*O5IhP*E}sIBg}QFP6?Mu%{fMD3SKPu$GXUh7D7wG7Xm=h?l2R4LXIPa95} zoqP2vlj7Wp@>*s_zprPQM}v~b?XZH>*!2@91~N1~q@;>&n&nd7Nxdjjz&yNHUbQ>A z5i98Qb0nhl+OL)+Scp~}!M`jt;nup`WYx0sw)NLmvy*3lNb$i*nlSB&*9l_C-T1F^ zjPT%nkX`uR`AGYJ+)!(n3TVh&{?h)BnC#zwbQL=4&xv2j{`a>7CbjtBX?B6)7A8qV zrI#o>210`ZV0|=yG=yeKhqAH^qG|stM)O)a#dslgd#3+4iP%mA=KBGv*}B6B1M@=H0A=IrnZU;p?>0#Qmt&I^{()JCo%MRPXU4N9gxCf?o_H;JmO4-s7f*|H{nf zMRI3U^sY@2R(K-OxT#2rJV7eJFDRcI&&m@LS3WSI@)Te zq`gYEKcQ#?;PXi0$9Ii3VEE+Kz|efvUQCyZ)}6W>kDblLUK8_*h!bl#A0T2oOz!}33vQ^Q;<(~_ih{ghM zDl$!z`~V7jpYXJFjT|>{PjdWnDO{t%Wi`XR`@LGRHRe|d@JHHJ9y^w^1BErya~n{= z%op)FFN`g?UNdQNXhfjOVoR{Dc5Glt(b(Bs#klgQwYN($SPkge{eS`HJTwM(U60@F zZGsYRM^tB{jgdL%+-SZEFXp2k!)&`Hxw?MeY!$G*Ux{% z0+-(Gtx$bYr|SM^Ule#MVZ7*fT%OUpn2MRce77Lr62CLG14hn_iTR2N{A z4V6l|T64uQIZnMC@W_5rOil8Z(hX!IUstzZD^#bA#rvU2dear&-4tU!PT|3k5Az*O z>f|D@45Yrh-R>EY)z?k)QSgDH$Of4fy4Z9iOcHb#1OEL4HqP#_8NhKYNPk@I?+=P$I4u5W*fX32Wrj{ESNV{LB2}Bn?jpH*%H3!I z)H{Obp6S&l2hFND!G7Jj)HBa*W68TAl&uU4Uhv1Nu5X{7{`q4Bv?OlM5uV?!JqtNO zL1Mb``6>M^FSg=cShApM`q1u-n7IY>VU>$zcW4o+ad<6+jFNqMqSdMHTp4(i@U5$@ zKTLIfqwNH*2{~uysn`>VSn?YS)!vt6!YEusVs%4$qifu_T6zqA%FfTHXRu&hjhM zyw)XOiIJXS76z1ZU^_`|_Xg~)kuw-cWhRvv|9(46f#BmDIA@V!-#v)9-r)0sy&QHoHfVEU6unkh%iGZ>cI(-(LIvvLRP3$cvLRU1dP&J4Lm;a@elbXe zdoGBfN-FyqIfHc+uN9a7uY{{X@u9H#X!xcWXZXhV{MhsOFLx)Nd;bm4NT_~6dd1!C zn{kF00TDlIj(*K3pAbdrbE>hnd(ZM7vU7*YpChb_q4LS|AgQ2s%N?{XLr#VP&BFDP zfb3HZ-iz{nH#WzV;;&h|1C`n*#C=M;+}ZB?pj|~kx9+H#)$Q4C{Xy1g zrIh$Adj_`!FIW26Sa??5AUlyEHImf)E=MjKE2i6&okocL$4xA^W z;g*`5$0{ zJ;_4D_9gHZ#k>nCHQGUyr#8?-nw3ONoXT4?_a=Wk^x zl1yGy+ICo~^i#FNzfTYI)nn37KcJF=W3ILb?cl(t7pI1kv6h)yg?Y99v}H{+WQ-6# zZTOh6=eO<_WKKUQwi}lyCN+}yeCqal&3xnF%ZOEwcv$X#vN0qla8#$LqU1Rt%J>Y` zMaj<1n2W!x+Hw5`zBii>6%az1z+kkInxR$LfMm$!d}s0Tr3g`vXj80GX@=+CF8XjH zL05h7h2eDxnv~QGKO#x6914%R2P=rs`IYWlm%E>6jGw(!cvbB`6$2wV6ahcJ-kAu3 zkHtO8Gwc$Yft(tkd#*sGGuh&36j+6IqBQObZx~XKEwGu=tijA+O%?k}>dOoMjS2B_ zQkoQU2H73?Ch~$HSxA^u>Z}hJS1d-$@yyqe_!hPzf~cuUo+3p}RMY*!r#}vVe_ai4 zM3DTm#@w)A(Qq`T!Bl>R7%58_b-m&{D?XK7UWF3H>+GrPfHMuBK;%Ed+Y294B~$u~ zDGqj&2uRXlL()6r#Ysu!21_X7~<~KDu@@kO{t?i#yULvqR2DsKh|?W zcqm05>>RZRvFh>SYG~{zT15Fl>NyMZrJMzG`0OfcWdD!0BgjYIFnp94j^XAh_ccC% zA1ck?f%)CJ!rk`kLI;SC9`0>_bAyJ7N%eEF0XQ8dTU%77VuT40Cx{Zp5Ks9)Py@ci z;(IvG+A0RIM*n~X7rAS*gJq&x&yLXzRd;)Ml58@s_oT=(6i<#lCP(LIu#6JZ|3^Sq ze+UllhHV(X#KGgwWkfZFVbxn9qTS&(P#r9?lDqaCb~QZmC05hTH}@XHvg=|_gXc{_1FU5o;`()7ZqJlq`urN~zIx*=6v-l& z#igzW+?xB3Fd?9q&CCg-{+$C6WvoKZ;;)T~NjQr)pQabM8a7ptlGXIx=Z7hw`K!A#D+0~PFWQI5f(YlYK5C|!v?VMr^*Qu{?9AI z=6=9^`0A}nlc*&Yr(X)F&-Lqteg6D}yBYk{vx|g&fdA7FH86&Xa?3(vC6^#`jn`SB{N!!ipdWE$L zZ#Bu?$-esJWb`ZHtmumhcf{!ZsS_Xh5q}OiTEN_Cw!6Kd)?oZ7PX{SsnB9(MSUQnT4Zc{tO8f^ZJ9y+ z`?bHnV|Ie0{KJF9^uI~!&$n0Lg*URHHlpI+9^;?i{Xl~xXh@!r?0@!UzdyV|0`f@z z^?d4ol8YDjk!>i;)&R8|fFbs%WT70*{l9!53IBHPtxN#Y@BIE=^)g;yHToI+t1d@@ zsh2Lj$RV(`gG&-U50`VYJs$nvL$H5AL7(BT^47n%JN)yJf4`k5;|Z^Z7b?mB{a(LHPoj~zMGf(z9Ltorke){x)2#)(Yl4Pu4`5v;bu*n?ajH#f@J1ckSJd)A+NV78w`+|D%jAV-a&xc;1 za)Z&dHOu3Zx=;oYBB{r>i@+HsIR1$i8PBjRBUu z8QJ)RIN{F1;VFK7SKtdo5<_TWqUB<4^aC1y)fI#)GsvpJL1)-mhdDkunGjfn5P;J? znfN?!pQC5)=NmWYWmoMEKqd)X*xi2J9K|;X$$J)}V@dFW)kUA6ngp;Rp< z)xm%_6Ix~x_7dhRCPU45W525lvLJeSBE-TZWc)8M5zZ4Yd&($NL5V3& z)St^1m}{!uZt2$0iiAhf&}`0vwB@SNxApi(P2g!~4b{s7d|C@gA@L(>1QS5wJn%Yg zM8(3KILpF3?w9;HTd%5eb*8@)JW0G zw3xgnT_~g^`I>cKaJJxeBf6s~7BqBYKb`l-<3wAI1f2-6PYF5j_gSDQe>O*%KwF&& z9=1l05=F;)^*pf#M06nIE&|Y-YTz4m*hYcEo2KuKT;379oId2oZ~*aDI)Zi_ZYrQd zJ#ag7MgNiKG(F`I)SS5$Q18!SuS_PQroUm~+4Mv9pU88$D>%!o-nzF8;@mR!c(ClO z_t;*Pgmxm;?g@8tP(XL_we8FDe_r_$5QO+2oaaCK_w{9rCTa}lbc)JW&(<9(YcCOP zSbL%N9{ylxcg0@PhBd(y<{)_uYL9L~AN8>&YAmi5gekD|)SCUg(RM1n54;BFh*Q?E zA@>sF(bk(URJ)S=MHSfT!hR1M8kUK!*!;8Zn&^LCa{j#rpq#&XpCS0U^M?ML>5C1>`Q zNK`UM-KpV46>_V(e{TgoV}>Yk;Lv_{_}`I-@nT$B8FUb0f-7yeGGPd9j(j>vCfxXm zKM^sMtoiueQ#OxZX)={q`R}}XFDhxTkJCrG?AHzgh#(he($1N?{lxCXds-$;KyEmo6dCM7eY+l%f z>4O3~$G#e$V{}^9q9mU{EqcNFy)8<3lRq4?k1EQ%yONxU^1d2v2Z~4*J zf0SjSeqkW=PB`tX_D12}Y@qLNzHE*W8brAx;6!q)UXf))1wav5jfo!ee*nGjc>wn5 z2$!9u{hb4gT6on_t+rw0;aiOFb6L^T8Hg>)9;Z_vDF8SO?{ey7DX}ryYRe&u8z03^(WRoOFQr@uyHHB4wblW@YMfQhe(?)We%lnC_WJ zSH+zPbwc58j>dRP)@Tv`&IE!e*@WgC>~b@oq*WLZ$AdsMdRTvU({HfAY#g3u?)|5v z0vlI7RPJCVLe3@T`Z)v`SSf3&ZXTgBsvU+wudJvsXmsXlQ5^*B@H*6N-q>Gi#uyHB z9zQzzzia#sn@cxEhtop%6BwW2j-$#vSd`;S(6>E?D6wKR*KPG1%5@=)oh<&5f(K+P z0YSpPZZ>RRpD$mw{POA&I$;XQN(_gkjiVQQZpl+*D1?Q0->odL9v;A&b%=h7B$VlV zZ7~z&>`Q#^uH>Qt;l}c*p=aI?!2z>M$0vski8^(5&|G3kuoA;o5E2KiL?*c_}+g zL4~)X>wce#pw~nsE2eKLc1+!9M|s#|at*nU`TFA4OxYtyJe6#Ru8UQxCYv{CEWdNo>ZK(ZR{ z6CDVv4L)-BR#xtlhBDKaG3Rdil$>emq9-%YylM-Z{uq#k7Qwf{$Y_+(+!)?VzPMcd zyxe+8gE3Q@htZPJ@|4Ywl9n?xafa$W;|GiqE?ND7RlGjHT_$l|AlXkP8I)}rTqLJW ziP3VonPxd60C7??cgWsBtS+HCb^YPu7Re)PKw$PW4RI)XOPCPxiTlb5c#ZxvCL`tL ziokAf_J>_cSWZ|@Tuvg0Z5h=xi8g{_I8rd1hK>0(F~P5vzt2H*?J$I6*A_~S{Z5k+ z?d40!hO2qQ!4X;Y1YHIQifUS;pnbfNe1^WIciT}+ccrJ zwuFp%wD@8K_5+(q`zdU{@*<|H%Uxbm?;os+QI*K>V3PE!**_=Ci-_cWDrqpWxLWvm zEJw%6k`abGzaQ%C36Jx2-xXc6aMP??Fgh?BnM9Zq$`jHgVC1foHWYjIe3{3IgVIS? z2LBdEP%M3#*neiK2^q+U=roVrSP@M$?#g3;=9?{M-eBlnzdy!dZa=cC6-#+QpNyZDz zEWCdqCa>ElYuLohXSBWaqj46Nc-lXvhgZF2E^r%+s=GWkr1x&S0>MfxS%{FwwsSh5Wj}dC{s1yT;z&O=Y1po^fAx8qq6TBKZ?Ct*4pd!NKnASZ zhgta^2R4!LkRP z9}lHG=S(yElA+we`%vVn*T(WhncZ9NoMNJfv2IH*GIc8kpd#cPggHf|@ke+x1exc+ z;JQG=>x#H~#yjFWin=Z>z;=sV7_9F=xH*gdyre;iJDFM#s1X!It8^ zI}JwBKk(X&SaN6;uLt7<(w3o&xKEcDCn+mTLU)n}nY+L3-7PEFEMe!m2P2$Tz!;#PC!zoo$X~_x zmi^Kywzh&&-95L4k&H1@U|=2*D))4XkT9ExfNWpUdCUs+7}@x#Ogz9lXn88D@znn9 zTGD)^`EhAu#vbIHf4~B7BAPDOnZ8^{@Db<-EzAtCT-cg0Am0X1rciX4@4@`dkSopn z9kF*QF)pm-;Yh+nc?CdMfDD9Mf_pyFPnFI@!@22)%c9Go-MBq24G=RH-DVGMu{JQ2 zA64goLeS39_rhSiRjo(iwB(IcxDG;_H8DVwRsZ(_bpQj1@g%tJwtv9@P@W((>eqpN z^gZuY*}_Ye%N(tcq3_|#@C&)}f(i7`4#T;31*R^lOV*s$3sF?5@^HuZ=cjaMYWH$v zsiucvfvFKvuaz%|w>I3_`c({jSNb5~nfcT3kf=k4@=M1yO)`+a=iRMCgWz+gQ=AJe z?BE%bxAy>iNWybb>3T8k->&buq9a(cEzqZtEG@WAG+*Vj?8~`HAES5d%zlvNI+Y6T zBho)TU7JX86!{)>i-aRvr!>#Zk_mfa*{J#v;cl9%N^RS>&wjIrsd+X}i@agG#%+Y& zfg=44mlt=5bIzoDXdCts9!KRSkNK!#-1O#aUV!)f;@SNVaigd&6IBq7F$~!MRQRe? zsOV>3N`qsSI>2(CFH;<2)(BCAZ7#CZZxoYTAZ<;Ks8K0q#2;leBInSn9Kwn#7Z>O& zh6PI-2C-;JVg~+l!}=18&*cgKK~+Q}jxgs>0Kn^X?MXBQ4(sD)WEV3tVrm{AlC+Wp z>|qA{nRv)2^D`@&MQ)_(%RBhfQQnAs%780GLD z2#-0Cj&uLaO7mK3d>7$?k=?&#Kv`lK(}0z-4uxmcFEj_daS^b#)B`V#FLp9^03BlW z?nn7P15Kzk^K3$f5qd%aQ^Afz|n`w$1raH#%$NYJfixH}PP`TA>|IhH^7_r6G zyoCfcjdL`UWvo!Ml@g1K_1x{}0a%nDZIG&4;i5>}nei=ceyN7j-g zC&hve77l$wW;%|91X)>;Wue(cmbbs~*6YS8<{SRE=+C;v++~P8WU@AY%W!l3t#CJj zq1HZ>DgaCUrp#AOxEFJNn}-v%8I?s^H#%mM#U2w+34gw0Q&kY_Pp#{GZpxhr7oYqZ zDQ(ke-*93~dP@OWlS!WK(7xzGw4cQAL3mnTsJ_iVvC)_m z5R?9iXFdPnCicT70TBx*=0donc@dkhKe6=h?+lBeA!`yADfr(j-H|sHQD%fF=AHZd zZ2#7V5b5#(ick8wjX-VCXbuI&ggbpHSPLmWw6$OK6L`&>Tn9#e2AZ-0=j`Hw_+m>l_r5qQujYFzLoHWn z?3E0}{TaFNhfE7F@!u!MH*5&n4|Sb7dd)q^fp&QLXaSzA)%-%G%Cja#F$@F9yJd)2 zsBj6Ng0&_EVI0~6N0{C!)Vdlw{d)^a*%6sQA0Tq~AlJF087}YF;tJAU4M{q&-3f;+ zLF?&=3{^V;R;iR)3&{Tr^vgwnR#IuTkpAb!NMBHZa8Ju?sbR)jgWJDaMkw&fm>#A? zJLWBp*QMNRwEW}Q1wt1Quipg@fJ=xk4T9Ph>DLhb!a?)hQ~uCC9U7mm+U8i1(Sj@- zddjCC92aJ}5)n~FYNhp?v^i=$w|g}65`&1H0crP?2aQ?eUzS;s*(emqur^yCgWD@>?+)i^k z6D0sDE4QTp1X~1i4C|OTqfUsx#8_e7zwOkl-ryEc)QD05iFQcl%)I%OmFo}stsX!NGO~dl%@rsC62zf&qB*uW-%?Gm)jaXs#8MsM> zP@0y_yZH|%=!=b3!iY?n(M5#^NZ+5r;s_^wb4XM9ETQA!a#20122w6dE z=`s~2AIZ&PUM)4xq@{$)7E;TRW$clLwd^b}>->QIH#tZpRtOIVq*1m`9s@;Ic(kFz zS1QfltqgRLv+$E)shK}MBQJIud0036r4y5m(EEtgqo-F~rXXJMF-RA%)N+uaO3QNE zKevb-qBBJLJ`2f$)j#+N0nrbpI7S7)ZDtVEu=i|3YWuGZpd+n5JwE}tx2&`lVGn`t zwut^8p!&DUElP#`~fkXvC6`2tjfvZKq8~x zP=hOlv!Yr%V&vk~oPuY?;J-+y0fkhC7?t!?y1Ob!q3AeJPEh4#hoRxPN4sC`eMb!7aGStAc>8(8P> zbJde~eZ57&iz<}VWAa!Nidtu^!y+`9_HBR)OSx7ugQv+I!c!G$G=N+pv z^tG+fLS4JSR57@j#Wum9qsi3#_1sIAXbmwIOq>}VMq@Hdt6&L!nlis5sDY688LJM9 z%kU~kppRqUB?xN&?TG8y2{9H9^CQEBOyv--bw_D zdakr}x813Vh!)D{w`DI9n+&a-AjIS&3w^rDc3=N~WE%+K(=32dW;vV~KeyAMq(lph z!Wq57`hO11-9$e&PA{<)u z>q+pqI->UDx4Kc33c?%f&qS6abwbAB?sg38>Mym}1)%OaF>TB|S}f&@A#hTghp~#) zx5b@|9H=EkgkLkq%rbBN1i9-2)ljYs{WduG#t>HRBlykJiMU-jW8(dYk}&Dag&K8< zq(ms^E@M@IJ!?jsbWLetA6$;sMaIsHSheIpl|a;Gmc) zi8=ayC$GCQ6qn&AUMS}DfkGwzZ-4dk@NGPA(xMxMRQs=DPLL$n!ee-wI(a#Y%Mj6z z{n9$~iIv7eDhZO)+|~6!u3UGYV7(Yq_o^7rE`J`Q3uf$R{U0Vi3f)soE)2sDh!4*j zP%<#@#q#)b>|Om!!TXJ-&KImS&Pu zO|ea!kT#SpyF5=}LCVP+$u>uaSCy+JSlVb#G3Fs)KQ>lbD~}U*>T(Jsj$uqHvz*pr z&dvC%1R?&3WimMBAAUsg$abi*I7M73`ZAV29&>sVyHAureUnGIaE3eBRW-~Pr_Ms~ zF5?u2ZHh*eFEXSw15XiJzlx1MVO6SqdQ)<62TG=G87IsushtJiy;`JfHuVaRN~FYRp2j&i|iV%jhB5kW2lLjwZM?G&`NpGlqqd>26EN7 z`zYDciPR>DaMt`<8RGo4dVX|*=XWV$jmWa?bVG|WPI6R#RqqENai}0lxUr5?zBeIR zNmWt6F{36Ndr0yzYpt?_$zAa}jUm*dwG`2yT70{+sKA`QVvj<`H_kZj!AQ<&TynfS zE1T3fi8!0^^Q0_i5=&GHpTAnH(VxN-^a(hhHfNri_=walvO8zEX_OWFxk-pvE@%wb zJ-)=1lMtoD3IVH6uqK|?y2g^RA{;{8+KLWejRQv#%=vWjYe;T&mIzkLXfITZ*`xGe zOj(Z8XfYM9eI+NP$s-;!o)%S^_0-{Uhl!WP*kpgc$zZkD0ZgrPfoKp{no7L?>S>+1 zYmaq|cEymAzW$+?FTdxLJM?k#KRBlR+C6J7mqtBN)5bZZrO$JEoYdl%Hd^&pUv#fb zP+?i}`2O?_>uqA6`&M4DC6)k#N;l#o^wUxOUw@$v*{}o@l=0 z#*wRT7Tj}zmD$+Gcf>Jus^_z{yD+k+R8rTiB^u(}$ry&6mo!yQ+0>%10NLAlweGlYq1+!vxfpsR<_UQxi7rpfnQI;kADFv` z&Wq5wW&Bbu)Ui@>e3j?zJwAJTu+p0c_c2-Uc4eT8p^%k4wW9RwT7+0Ib{nrxrEsVJ zmPhnAdhENT_^EF@8e*TjCxrgAMv&v1li!=QuO`G>9P>Yr_=Sn0K(XDQ4TT0dh7}?y z)=rL+pZ?(R2OyB`teyu10)0F|Q-!A`0H8JAXSNrs zyHwEw9_Pb1M1%4mulg0rC0)hejpljOy^(r$XZi#y@jVlyd?2y_{ht|X^IIx78W^MS{dUi zZ6*k2+m|%15>*T{kz+rH?XebqKNI;vcaj)Leh0`k(kl`QFYOV36`T`yPHwD(u!bu2 zHK^ms(bAKp^QU6#r9FPK51;Vk5iL}DjlG)SN3?o`Gs~3s3MJV~Iy{j?XSsP4D^S*) zx!L9E8XrVRL|HgA#hp^)DY2N&qeki&Y$z=)qZZv?X{G1}3A1TOs$xQ_7~?1x%%S!r z=m&GXa#tLaYK%u=DB2dp81<@3>TcsDt!1nfZDj~wo6f$-h@HWA%t?Bfk3-2m=jJUI z=RA>Bzc&!6AN-%Ic9?hGeG9_(m*Y(Q>ylmONLvn3io{T6KI(*`&Oe+S?H z;5>}ks7V^NrYUtj^Hn{)65Dr()CEl|kyar6MO88Ab`4eRRpqAS4>lVS7r76=Ywc5f zRM#3rnRWWdi_Otd@Lxm7-Y70?7lRH<&$Rg}RD(~;(xz79#2&|6zMG5(#_kOfrh!Ds za~(j@GQzjrMl#CP?d#O6O}~`wkLlo2;>`g^m`Qj1vHHf4^l3(n6|}v34udDdLjM{ zM@T@rWhiJlH0MVuWB`<`{(r{%76L;(X}mgk&4lwSdh&JX3q+w_qz3MyUf*7Cxk_ z=Qz^^JtEBvfLh9;jr3NgQMnp%a(Xn@DR#>6ZJt9kO7508;m6y4-o>n6GNL9cQv*eD zsP!8u+Sgr+^#)pN5wUZ{A^FB@YH=kmWPB|gK&_diSzN-qNKf?RoPutw$t+*jHRF4e zZ}uvK(1DLlV$Dv)S!f*9!pbWp`zMPE;WP4eIEHsfMhKVlwWj3ALNw^bYiM>7-y_2y zK>1axSfS@;nFU!__&kPI)BYv9{=$PVK!fRfFIo09$j6y!aDaDGzApvj=Xc}7cNul1 z%jjQ%@L&AU5Ycj+Hjy+SV!8GcYDU|~H-W$UV(pWOP*P;{nRd>V>)%L=*B%ibAd!1U zWAK{&KPvjaSYm@JAQneGj1D4;P->!#LPU`i zx5iH+%(|Vw-f2xegA*-Le67uoA{)@FQ~U!}#qZA@MrcEXt&zJ{r}P^Z0n&kBl=ZU| z>y`;X1dbqB8Kku-|C(&t`}9fefsPC{IAQYdu0Hndr?B}KKfTBjrzrs&)jSD^1=KrlnqPZ zMjsUs<0k)udIA2rcMu6of7wPa6?lZwh0KHh%S{x3Ioazd- z;tdeeX@h7po_=F{V_6RW-Y*O{0?ZO3`fNPkj|K?yN_Y2@vkfKfygX=yj8~>Rri1O* zZy?+x!a*Y1qsS{4d+gR%MYb1iA=r{@>4P8AuK3JiOCV~ov)vjJG=Wn24d#L{#^o=V zeCrthCq#_c^P+AX%8+78Hnlzk)&Xybry!fK;e}1*h`w~VvVc574JNpnZP}E6CmsT# zMUcw)O1}8}62p;2Tnzfb{Vzj4R@f1)djNhO%#{gXBOegHfH4M~rkYTwH~)b{#~i$< zIF#G&17bNhi-<$*P6K@UU3U+JR*rD7k;He@+L!LUzAR35dTQr3-Xgl5%YD;-9w zQ@gkYr41FUG{mb+`-zs|?vL4kjnP_9e0WMa{4!5~9+kzdu!=x_K+`#SMIy*oW{OxW zV!&4-glJf98XRc$VMHjZkD~xIRIE zH3Fe*A?+bauZv_qGxX~qd`%Q-7n-Yv-+>$a>q)+io59J<`Cd|CR<;Sqk&b615t9~# zdAGhF1lV&wXiPC{cY!YRPJa?>asGtSB=Z6mHpCAg>!!G(KTzJ3r!efe>Ikrn(c)y^ z`08P6rlnHj!tSvy5($=ZsWre3!v~AJ)e9|p19YDD?!N&PCsQ>hDLMZ=_Q#pU#-5zI zYfBga6laMn4eD}M*_=Re4}j40x?2eI46#5P5Z2+YS(jVG2?CBYlhf-n4A@L1aFrSl z$OnZ(KVV9;MP(4L`oSOd_(+WBD}W+&7wi!AQ&4;p9m-1uQY}+>QX;Q;PnL%Gs;auU zrf?*`CBV{bDwzDYh+G9Nm6+u~V)+@bXnN}eXIBw2F@W&w=^DB5jc=yg-aT$Ss>BEV z+3%m^5@;<{ULF6#oFfsG;^gxlE9v2Qin9@dPSz!J5bZv+uHMGvP;u&7LPejat*f9g zM$gj|Bn^{mLk0`6Kruyez3lqY5PloKwYohH)2VZL!Q8t6wKsg(X`W3$EX{$($TO8T zNDh!OAC;2r;G)aH-WP6JaZ$J@D8JTft$brS+pDenB;0dWg>~(V=9K@Pwp-XE5`O%1 zFMNAPwqFE^Vb>qUUws0U-&8H-CL%r+3$I@d=hU0yJzhFwqc=!K><#Clc$^_LA|XD5 zZU=B4DIA=q+!3^6DS695w$JzzI!sFs?J!kt`Wz4FbocqidrPkWKHc0Y^|aCj&1C4u3PrT7{-pV*JQPe8eXe?h=QB_#Ko?{OV);^g|dK z9zJh0d)DsLiC~*&dg0uz8YTBbnA(YW7hP`mpY{6_$Rk8YP~JB?z15^5bbs1R&fl=e zH;x!p(fXU#`p?JB(EE>`VvnqlHt1r0EHVC*IZ6Wf+d725J(#X#s1M;e!E78Pf!e8A z#6HK24@@=R##0aIPm05Pp*gV|Fy3NQpeyFu4_UbhVPP<`TYXq&H1y76 zIyPG_P#7+(ELBz(r87$BT+|XN$%6ghApJxNVp(*5p4sZ(LnxF3doCyfsx zpb;iXfvPBq#R57DYc+I*+Lha4j7Ed0O%d1XjE+uKAk(N;w+4kkX^Jrp#s%$w_6R%J zU9envQEN1t+U*7mi(A65%iQ(f`p(?H`c7*FJhC)eqoUs4NJtfQnsgt5^s_7jqi{Ny zZWW*2K)rBgHrNNQABo~1)FsJ;&F*7BaKZenwxH*TNY?8O^fk^k2N?_I;g3+XX@U_+ zhz^XXA!Dz~aks^lQ!S>~A&T`PGp3Qm9-!yqLfuCgc)ri&$yQY`LaO5qt)j{l?4)4k zO0|z)9VR}oXR1F9-1z{AnJr;*JsEAv3VFB?*<3$RXhEGyx0B|52cZaI5KSB3lj%ox z@Y3`N_dxc2Iau5Z(NjUra1U3@bNL5PUtrE}Ih4r~UG1Ix?pw;yyN1fmkvl@NYFzls zc`(PMKt<#h(|D6#@R#C#VW;rTiyY!I%7|h;RV_0TpO?ha*uB$8ew?oK1WH3KL*w2w z@z1)SigZ{GWuGA9gUiDi+qg>FP;aiq5TvtFw%Py~eVDO>hEX6-D0o$7S>Q}y$4qv4 z?<|aQD9P;)n_lkLP`kX|Tb``A@do%Nxzc`18m-QX`-T3VuzM|o<`_>}h|2KMA`v2uG zx`h6V3AxrC{aDtKB89)z$}5C@eoJfdPhs_tDXu1t=XxH|&3~AHyTAbVKO8arjmQ0g zRQ??4pd6@vo_x_D{HOT&TRI-E|IdHB$fy9MF`tzkQMv-9*GtMailI$e<>SvO#`bVh zCv;8nMa_1z$DPhawcfA_q+o8Izxln$+5Zvsm~@BC0t63!wj4x#@Lo0j!P@%uob8is zCYrHkBt*eP+j`V2AMln`%*}h)ZM5NXM9OS1QPKwH`4Cn1i3NaB%MSvi6qHW{dM78Z zIH+TP&Rl( z$AcgkKtPlr2r5xf2?|IW$vG!MP$UaV7KsPOL`E`5keqW65Rj4N0VRWg1eF{mXZZGm zqRu!~x9+`Pef7Jy=KbR>=NwKy{dDi{-o5u)YcnbI3_Pr#o}<8F7ox5I-ic;VOz7ge zQ{|4!^x%AlCY6G7+3lh2y=(a%iukgAOJ%<e3@v*VJqmv)Lnu)yIK!wZ zn%i~T33SeGy+6f(7t)JbpPJxsg78m-KpQEV4i|c+^k-K$qrd0Qm!jg zb&)kFSl0PKRr&xK_uIP8OTiskI(%M!kCKv=-F7BhQyYrf&iiALr%9}I-`W@_l^CA= zBX9anXGZw>L&44Ko9Dw>Zym@M7gIP8LlpsN-Bglhn)l#f7Swo=mrcdCn}?r2?eBtP zP%xgK-gH4YcG1yj;S{__WYZ~g`C-IrzB*tHF)G_^xAXU#gA~{hI*52E-~CxH%Ivg1 z4+@J->|!>OchEZG4Ws=RR!o^uL2dUH{tB#3YNb<9t0Xirxk@Rz595vC zTVO z0$wi$h*!F)0D;l7^6rS0F3hdf4wX>@amdQ9nK=)*( z&#ti`spZu7qeCKh-@pdbfpV%QJsRngyYeyGb0yad?TZXIM09snIZUE9#b|NLf>!{jCRd3 znnwyAN}Gv_7;!?bZsJW$dnZ8|%4v3VrewH&{u*p_F3*j85j+jBo<8L2%X+|p`G5JR zk+0)Kz3u=B%VOR79c|pGme}_o9Ofg*prhc*dnWN9R|E|^``o%3>75iVqPs6E? zarXFLpF0nVZ20;Wu}&WGK0xm;j{IQkwsi{fgEU+OW*fmJqVtzii}m?7#~)`LHBQBf z^{BShnYHtP3#x1q6h~NGK~j1^{wccCeW~kw<6ViqHw7}Yir>9TT6^?re-k2nGlazH zHwAX6_>pmB#xdBgX09nGH~UTugXfMAkgoLI_~;n!C{l0;wtv{bV@9Lmk%)`b`LA%H z>V|Z%zJ>T~re^-Rk&Rb-iHmOf*DEf|&snrQg(TrUq>g=dM&=xofybv}7g-s?@v7psKPI~zdBki<{4l-qwH$y|?LbSjL?y>6k0v!8`@9M5xu^DTon>C37 zBs{t1d!&TTZrg)Zg$h)3`2wQ@|3^1mo8Y=`pBej~aOR;#d!dt8@szYtM>AS55&*{{HU| z`ZY$(0q(OwtwSu?_eJq-6@6PuJqRc-DvbKOneKl0{r|#|k~FTMy|NeQWA*w$FYckyO6b$Ka&-1r~?@SmB@qYWpO~?>oIC(D!vHa53_a|ctkXRMp zMZSOO$t7f68$QR=+Oy7InoiJx?|Png|3v)$ct8J>V7ZsRTx}-!mjRoG?^@*=kNitd z8i`<)oE*id{mX!vfT0H2-Lm^-eD|Jr`w=Y0#w$k)P`|sR5@qn+e^dAOQTu;Bb-`^F zc)L?@+y6D(csNqHN;-ldV)d4qsCJw$8?6?o{igz4c0Fs!7InaMYYv`V291YlTK=vN^IJdJoepiid>plED4Kj+_u z_>#PvcWfgcLjhEJ%@fgskV9vJB*{uVt&BtH&RWclrFqGxN0ityrYV|VC+reZQ}e*8S zz5)nfd7O8zm!KwBF9P5+>wi@;4b**Tm;}0s%_)BX24M{M&x+5#vE=d#!f;> zk^&^i10V)d(@7|DZ;pWUjWOW}s?B^dP+q_?vXz_9W=s#k`9pC{Hme2E8z8IuE)!w7 zm8>A3V(0Zm%Q}a4Mez~?!1G$!mfFwWxb0jy2vA6STbny9>RF3k(`o5hRt*rBH6P`= zT~9RGQ<%>87!eyn=q+%UHHg?c-(k?2AZanc5Pd%mFuL3$BerZEGb~vz=?tm+ch_jW zEKZ*`u!o{HnBMs$&38&JU$Kb-T4VI=XDD3NlXLTpOc!A;7ScRMrOOWxb`OMom!SZ0 zH?SaYsX`9lf#vn$oP*;8&pjAzxj_UPjc{iBAt-*YFtex!3s}sfZ4c`K`t{lU_@~SC z(PMtDxT>T?UlSq3#1=abaN=+m1o%e&1Xx4-PGl>q@S43qV>kb%I+m2&RnrP4VS-!+ z;Z^=ZP_^Ke<99L5yam~0J*2!|x)n(JeE^WJV#+%FeAd@!ZGPCIWI4Gp9AOY|rDXc? zFNYz(=TA!rXBlW_)^!QoOep~pYjDdkpZQo~k>*Gpin0I}2Xnzw7}s|l-N61KIDfPP z0<0eC@di~gF`)eTQD-w#Blj=1iq%_@0?wt^Wk(f9#j9Q+WM~^IfB{KUKh z(++IZHS+~v%oKm|twkytC7_@)e*8VZr;}o~lEVTTrMcAA|-i1wL^3z>XhbPh+FJt*8o{ex)$PC4ybVr1ciOQ*g|o@=rn5W z9H4G~h%G7@0$`PlS>+!2DJ=6EV2P&@0Hf>Hq^<=e;(j#(HB5-06z25}E*0tOOVD|c z`6hz%{^~*_T{l1IL6~2`Phk2q0035X^^=?rvaN=(Nwkmqy^=wQ=qeSO0yH1^q6Z%k zY%hW|#bS!vpUd(3s%}FS8#DQUIxZDzK&YXmj^vsuy27-$i8dKXf)8>RY%H?I+w?JC3HS8;SZGw4Ljitu0p6EBZ1QwtUM?#+V`|QxFHl0B)vSN&%czi*>EidDUElSOXKIU+jm+RsWe#LoT(1q$T7pT!VNLVvzJ8 z?hP9Tb40HbMt7ZA^b{Fdais{h=8~yUkS!C`i&Wbrwebw#Mayo^i z)kH2n7Pzz}e%q-g%LuO)+;#&dI>aD-gax>(2mO+u^rQM=n69 z!UYHSu*zax$#$5Z8eq~XEQA{N;|EBdrgwjlSR z$_}K|m-fKY8AU&cYLV_5-V@Fl@%_l z|6I0Bc6nQIi1)kB9PI#_w0;fplsZY7wW|w=;=(6d|4RWK1~f~XYxP^JgSeH z$;Tt!@{Zkyn|StJQcH_BGMoWD)q9wmx`cm6%f1px#1~~6Ncs3Sex9@KsF4w!Zhy&B zdc{-Csj~%UV;psP;?k&>ukwD8&?IA_X&W*F-5O@wiFV7oO!^G{adXc$jn4G&LK@g) z9Ecn?eN9215`*jxt4uclf_A!efJp+OoK=Q>hY8*yNHGg4<;AV^&{y-lf9=cmc&%5S zGYL~fp~?oKFL7A?dTPpKK8CLDvWmJRnT{Y5mzk+m&)8ba+NS7L2p29@Oqw2#fbB#8 zu$-Gp*cNCMhXSCgkESfkHjQM|&l_9;^qdSC}&wjUpQ#~h4La3(MAL-@o? zuIm26-HZfksg$jN2ETrrtIt5W1vl=Jw-hr7USU{pd7Pe7P4%$L4_zedeNrf>Lz|Sx z=mF=CS5Y5$Gy}Nqi_v*GD(X0#hfyyU~b&z!_N%{m@;=B2Zav5AhLY?)z^RFUxx zFo$f&EkPm4PJhGr++m&7wZ>DG=7J3&hQqBvb91ByP3Zl?6qi#uMKS*MbJkM%<@L6K zCDs%rK7*5h;-NJx3YjpudQKNnic==%^qE|Q0WYm$U5cimd17(%hUf0IV`7C&3~4&- zY)B*mr5NMIMKxX0O6Th_?~6C-);FLi zi%R@TWAJGkGMPySS=_s2T-4Ohdo_|WTdb6JDqg+>sZ2#}=l*N7>WF0AMELMam;fdm z9Y&@naWgs*H{)I&Gk1y}To>SA?Bou@9rDWaD9Ac+y_5RYv)YU~rv3?cQl7%nJhN!0 zc}a4&=0-y-y7ggEXlN=PyWJAzOyD_Y=zc(QIp2?YlJmZ` zWM4&aNqm+bg^;HH0n>BRI!28C01VlzgX#nO+djBa2#S*kq%gjYr}fWb31W(Gq_N%V zJWS@(V!bvTT9$F<9jTTYhIZgQtChd8tY&uHBtwk(Z*5!*rwcX0nh8o2vsH9tzPBfA%djLE`X1@RClGlrkd}K7^kwGq{YA-5B!V%$=6j!`hL0r&R0q6A(P+rzRs)el~51gygzSP4l zhW)Kpf^AxwMJxf%OnS+4RHXGvD;G*1o{baNLH36o^df(@klZD(hV<3SovrL^Wncdy zB(xn~en!D}7r|&sbA2EyhZ8j^T+$$#4~es)%0=Dl!-4UFCGUSrE6>v7luq6~R}16eXwXjd1xvp=sthbH|O;@$Y9p9j;SSI*Ox;O*ddEs4xDr5lx46WmdF!sq4n)6#DKfsplhN1pcS5Idv2WnpQ&e(yB`m8|%aB;`G<; zOAp*9gAA>F>$&GBFn5m8GVn*fKW+{ghf5?c;i99x^I%C>-Lt@*7(1ar`-lCD_C#?^ zg!n0kGp3O?xe42wSc1}#2(vq12dgYD7}{3O=Rae5!eKL}kPeRReafD!+NBnOV6KEY z0~>LL@ji;cdOZHc2hFj>(NIZdZp4DAsi1=EKB!QjKz6r)0)BcH_7W3wO4C5*TXWD4M2Hg_|(sy z3>w`iEdzJ4xU|Wmrv)7B`8!88cJX!fdN{6aZ|u4JI@$*}N(B^>{3>gkU-Ha-WcB=5 zbl_oU#!akD=Nyz{*%b*C58&?jXQ16XP()ooUbZ__pt8Z_SW3E7FSZ zr(lT`Cs&DBF9a4Bti=HFQ<}GJAw^tZVGcRfp5$8kR?+C%MnyMCa+4~X89K_Rd4nLc zXM=OLJdy2I>FasXSrZ|#r7HD%qMFhWCbn#{eUX|$+jo}LFD@ekgyR29exw@8;}a7r zTuz1tsLnM9PL>zW6C4S&A)d9@n`2vk7*zXkx8Kvzy0=~#Uv%;EA3AYo$DCztF+-ax zcQIj&XMhyR4}3<8%cXKHwsn9Nn1a0fN}1e-eui+|maj9B%;l*d;$VrEB_GcLF}g`5 za*Q@H5g0K0n628XZkz(=2Qu zWTNB7+$*NjMcUj-U27~#?w!a)khjLBTw^Yt|D2IUf+M#652ii}`{aZLqNdlKr8yn) znn5x&4JoXLzUT1gP>JErD%U$@kC7RW-R&u08c9G`NuJ<*Qc!U{nCF$@I|_UL*wo>z zSgeJ|3s&CEjt>6m1Ap?9iUusso{@wPJ0EF{8zTiK>o%@|dpmctaL)9}Ao`y}tFWey&257cE;2=tAUIa?^J*?87AetT4KXFMWyib1yM#nhyR zsLl-O=6rsPY2H&kL(7aM4PT)w#5L{UXt4Z&gg}nq&VxaQ%Zj)PoRSYcOFhyZAKVL3 zHrQgMjWeLom&gxGGd@N@=5uio0vjcx+cdbooT=3ac9r~L{^P$0l@>)dG=RuR$&i~+ zmBp3y0>#UWbx|5LhMdyF+CxhAE8siU+g_yMv&%?NZ@eYDDlAMJgUdX+Q1w$+Wj9hO zlyvY6-$sh>H4bIdt1-Z<_y9K;i$Df>I}VEpep%cdxJ6E;A_aewx{FnINfjp_)s8AQ zi+`Cl*8bz2M%taz2K!7(O4XQNww62EUw$fLsoY+6m|@kgCq3>^%zaPy8(8<&_#R** z$zMLA4`QBu872{WbSLnY?JpdDq;p(n`iqI6EfFI-ghsK_(OW+NLc1Y5MKq_Fd>%Ll ztbZXIpa^dx0@oH=Vbd4W1hObopjh84CvJbcYzrl9n4*w`D2X;BpLo8NNt`!eJi`r7 zOapa$)!Bd`G0*t5slg)SllYS=UjWf+lMC1Neo*hdKt8D0JOOyA9O&a?4rf&$ z2Xeb4XixHx2R!E$PAB{MdR+hNihdU>M2m+${^!gpXGg(EB9n9|+xd4B8D2sq1R|8L z^0g_e-|vF+;s_GfAJDgSi+lJ$8a}fD-Rgat;81Y;uoTY8cde;!=}FN9a?07$E})=g z5q|ETgh7jUBH)+qVVGr`17up><gV+T5LX5{5KA3yyyvi_}Yl}AVbxt;+7_<*6d zPa)&E+@Jt5jfA5NN_FkN$?kps*&GOEe3McCThJMaRc9QBn^H%zU+oX^l%HLnLqPHa zo)JGk+54A}8hnMFXcPbP4UYNi!2Mc7LU6|9=jr&?d~}=i<0?zix@&;^qEGBs^~&$}Im&l-vaz z>bdKR|B^nbH-)b>o!5=8{@x@Jl6Y9nO!=4jyj=)iWe7NPp87SwziZG$Pjn%azp2*# zFA1JJB+9QABS5hykNV3iRs+b2(S^nIUkpwY$;bb5BECQPr|AAartkl}wfuj2+O~_! z`EWv@vi!b&$$dO%VfXsEfG2~(_$V#6PmFA23c|xhL-TA$a71~eMdrUE$`M1Lm9NE{u zGdX^`xfBdxQqAfZt&rP3926R#==(ov3*3a7!vvg}<^TRUK9RAG-ows(tmNN5?>qAq z^k0(z=2W2`2&DnWt@?nR*lX5%!*Mzo^o9#>OlHyw>Dl1pqkViAVvqlP5I)ffTU2^R zo)qt=KS^5Q#@KGu2l^jwNlTRG1&Te8wTSF=5O#Y#c8mz0LTB`vIsCu_vKy)uaI3{eCCh}jx}07<^X=utqFy7YSgR+c6-PuBU-#6WmCtP^15oDTCt1yJZF0fZ|O50ad-A@L~kMd4s#74IEW!R*82GC@5L zcN-vRCxGHfC?>|Z=B4EqEc@!}Gr!Se-v>`GgD|6HB*LfSeO}Kohr|lp&wh`=A(sR_ z18Zi_xG8b5m^rA(E&+OLR$6A6h3ow-S}5w{MEDaIQa8Z3gx+R&3Hgze>^yA8P4!OjifqZWzF$sauA=T`RUuwS~;L86q@ z*Ff871I|ehLc1&gdA($S`SbQc(0Uy1r_#`{LDvC{>4Q=(uWt{(FG(wwxk!%oWX(N4 zFVS#Z!z3{n>!QC%6Hjw;>+P7j*v_T(@Rorf{6ooLQTHa8Bllsn4pxWG1C1d9RL}x$ z-KquzGz<0pK@1yCdWUA`iEm~Y%Z?e_$apo6bv9@2swqlj@SVAwfA}!wgd`c#?jzkr zj`&KZ6YaF~w@;wa?vHR(1JL;G%Cy^9Te$2c0gW|x^O^|R{Wn!o$=VP+R`-;5^cRLr$h5`X_aK2=v5V=zQirAM0CLV{m zB@olLbhI%ULs@Em0Gn+CS2Ow)j1+N6q{iap{GXjGL!GPVc@~vBcl`2 zD>ztv{_~d^Jwy(Mw%M4EXX4to)yjGqgTGi^9}^uYR=1$RfZtxyM&xSa?P9bj7*E4Y zJO+ziAz3sSH2X8rz{+@@Bp-95pROPMq>#6e;(Hqfr|*5Z`FN=(J=?bYlIBH5%$PkW zQ;rx2a<0c8!^jcfqivOxHDM~`kc2c+Xu$;XuJ-mhJyAM`>AtPR11vdj&qXGHI8VhC zd5yu%xvQO$QDpeNW3FQV(hQ_}4I)9gaki&W)?;snn#QlD&iwo=N8@8(8g*6Ya_V}F zs%t@E`4C;s9ayslly8OIgvbdSqxgz9z0|pgt;m&)dA~2xSkfl}%#h5qDtT$+vf5AD zvf~{f+(HwKZ=>!9H$^zhB29m9}`XZ9}_YA2B-KnAPQviNq@J3d} z`hnW?5kWceD#>>AKbmqAVnNmuTgX^z=_r|c*Yd<+c=Uz zTWUc=r~f&Q?Lsh&2N)Ddej7)a5_IAJ%_-4)sCMbc_@2l_i7F;Q<%^Jx>juH9t#CYv znW^GsbitVtw!Ja1lSHPbaXPc0Rcil8VSo@mR3j!dnfqL00yWQ0`3 zJ>5oxLLXotBAKi;uCi-$$0byfk>meBy1@@Ed-Rz%uR)cgAVkZ%NbK5?4#Vc)>D+5F zL0f>AU(smWEYMkjb_L(z+K#D~EuHk712Rur7-@Nw;h%S9#Mr(qS`6(hSVE~7viFU@ zq1nEt1^M~5(~qg1xy*U#7A)zfn!V->5|Tv&cQtqMsk{>%BK>9poF{p+j9jq?Zz%iy zA!aa08bjD1g@~kiN;-VW{}P<-WZ=mcXbv0gIE(I@RHa}eea=OIn9PS=46vOlhL~#- zNNV$xPwLD{0aN@<-jUw>2rP)FBHfTV7&c(Tg9BqJM_rVay1=sipT5c z@3?zecTQI<*+5CY>X>OfZEkqU&-pQ1cjk&Kw=$dDzl3@4Tg2dMO1izyfnqdGt`?hu zJJ7Kfo99&bn1BFTUb_rV9qC$~P1__@2IIVhPtw+B(Np{yQ-CJ&c{S(o%!5}p8<>Y$ zLjeM^8zYM3qDPiFhZ}I7COI!mS9pN{AosLA5IjRRhTLr1)-g94V+1SfF__w1E1^UU zqbOTXz!_ccM5MArPvLf?huN9;&xaHT8N@*KfpTny+N+(dS@wmYSjECLr{`B4tV#s% zK6_C;H*3FmC;R1R&r9-UGeAAno@?=QDNr1N#p`)5y8`d?N7q@$0kbrD7WSS@`Jlcm z%c+g1S*PTB;X1J`buE)bx{x&fS(lYEuPD2oR3~R3yw3eEWQCAsy3h< zlIuyjLLKJu6K=)(a5ERz%{eC7p@d`B8jY}d^AdiSa)Q_ovv@UziUil)xcQI_JMC~= zq3K$1)oj|fN8z{#?HGv5vrh(#myptIe5mvEVk=Dv1D>g7on2r0rG@%*I=@;|^cdni zeFoAZiyt{o9?i~bo$T?32Vk_1=C{j$2ZLR&b+iZfk&P}T|K6v+=ydy4s~wHOF4>CJ z2X0E^xzo|E&xcmNY9zslzu#2rcmd@uxcTX*7hr=bKOV1&UEQH2roHe0S1gU4nTWHw z=~yqmw*>|$%eX75LiO}sxtpi%3%IDL>ORW6y1XfEyzSaNrn0i~YC>UDn#J>|)8PJL zU8{w&xMFCZ{)&JIVn*C`LUj48^DN8GKplQKvMkFBZjm-S_Af+VYJy}`A1Ci;dDnGi zMdx!)3y}j;g?BTqfQWK~GyO|{_8pTUq$Wti(#4*f>|L+F_sio4P3>}U!LP3~>Uk`% z_##@AD;l9Y4KD*KCo`8V`cO?02`2V}SE+hcOqvxS5Mav{MVH5Y9N1niR<)G%Tbs{m zr9-h~CfQAr3?Hqz50chQlM(E~chCh7I@bV*$JjsTYaitBd6S(Q9~;Ua(mQ|Fhu_}p zDo7D*P|~-DoC@!(AA-8b@qfflEU?XDA@MBVMZsW=?pPffNz#c5-I(kX$)w*sILmPX z&QfKS752fZ_M}k))uD-CCcML~p%sgXRsvCBdORbGsY<6_D9I7_Um+bk8KluHkQZw){!_TChaCV$|n6AIYTsOba zG$iKUbYqzAG-9kpZds+{Ns(AB=lQ@G*3km3QqQ%c^uwF-(%F7p-yHiy06sB*t80DgZMPQRBjHl`%F}?O|zG;y}K)y?L2L!e5$W4=&YYF zvu<5=tL5WY+gH;D$C_N0^XDC#S9D0?$-#d}DFpv6Q$D+vm^OzMv)XbGq@9zmx{6>i zNd^YRQTs3#>HJ3*n9L&BE2SI+f^f{XO1sLl4xv>yL`Ej+Y@?fuUOR8}hB2_Qi5pbc z`{G{`b?C?}(^rv7R*J4XH(c_r6_$v4fAqGQ?uM4<^BYyA{*`TT{$@^PxcsUzh&))4X42V@;tj@mcKtGp#H{$(~!tWbU=NL|F=Kr^*(U8LmN z#VpjLjt#5W2Wp`wmm3$LT3HDDmw7~2KW{Y2C8o{*RBE>7Kft=h^dR?-nIKal9Msl9 zl~9)&-R96G_0DZCbO(uT&oWFz7^{ zOp4>-gqLOIu-GE59DU{`RTK$k1VE`*RToxOa7ANN+L+jsy&C?y^)zTUNcFwLqR(BY zc+PtJ5vjS8VGKJms(ihjdYBQP)IpHxJLn%hp8xbIxuf9Zp5=}C|m0aiRZ=KZN4uxP#srD@q@g`Dw{r9|+8Kl;U zqMmxHWNs7P@9plS1&DN{ghtn8MvwB9sLJCleaLbo+e{fDn3#4qWV?O4gT7*#y2g&rBX@1`ZEN0 zLKS8GnLEToa_zAqaWVMQ6sgzkjwHTcKHF@La_$olz!UJC;2u>@l3|)3=623=T5pSC zyPF@y9?vxIbiwDULv)mt^EN5#Je8o&5`T=-gmMh?LbZ9r0u{?}Qj_zH^>C?ExNK8~ zY!hsmO=8{rg7F6&H&&f^R07yJSQ@Z6zDfEj>~Vdn(b8-s3EKQBk!jJEW(yTcM+CTf zLKz0{VxNg?1^3QJrA2hAv%~HjS{*!G>(ChgBJFd;DOczX_eAi1M%mtVN91HT)h@SR zpc34wu!j`in3!Woh4Us9P*ApL2;INjU2uOu)nVSi+j%7S**;9m=?hBEEmthIkKEZ} zJ0Cw`yTanlZ2&ufnkBS&XF{e6lv#3FC) zSD5FA1r4I{Cl+OL`TZIhi_0fevzaZ*mpY>(y`0LH=>~LiK5kbXb;-#+kGiyWtysHa zqi>eGvo7$eu%#v7v$>1JD-gJMKr+-&U-?oVkXsiBQt;`lci$Yo%)0&cvEbqdB~oJV zD=>VNbq)6C&F;{6z9X;r`cN!fUE5MLfZfk5Xd^IYGod~-m5kW`>GEN2qV_CD6+Og} z@e98zsk=FHxzePaiHa^PR9rH+cY(TtqBmseHX~eXETd;rZ&G32M9&6(^0V@{n}(gTk}!2a#im zQZQ@*U02f!UI(4-a+^jd`x~ZPi#LPkokT>rCeC7gTO{VL3GlWn`kmmn0okGB4%;Dl z-Ws|-BPI+)r14>$pah(>UOk7nq;x0^#m5?u0AEe>^r^Em|BKn3U$y(V;SG0p-K(7S5mdO;glW z%n7``%){7Ag45tc{*ta~q5b~s6zHw^DBcA(F>Nbd$Ot`B1R^)LK|^P^fMu7T~mceqdn9c zLhfKuWObbn6PsehH&J?Wf;KV=R+7}@ zC%Wd2kx9Uz@rn3}P_yA0Ugy`Ke)-h3?6u*LVdsx7Y5H6^sB1*^3voe#leOp;2OCib z%9Se8%l0!LNu_1}h+!9$;(5t2IAe*Ii$~*hVYuZ4Q&Vzanq)|N)U8`zUedRpi@c+h zNztk=b1G^-HIr{7`*mrDMQLoPccF@58MlK)LMx|ls9;=!WZre&LE1})JIk5Z!LE%o zi?7n9JTGvJExW6_Q851j?zUXMyfxJy?Lq9ChsQxiBx-+nrb=M|g9TH}{v@VzWPVSp z7mQPDx#hwB`%v^9ynHLf0`4%>@C>79X~ zl-M8T+jUeQV@M`uUp)LRcG3gC?pSU&J@H$tNfSa!-EUZa4T|h;TjsbBG;=iN^4@#O zw=gi+9TK0J`b73yOGRLhh5UsSVtaqN7rruv9EIufSI>X8q;MUUc~7*~;cx!>FP-Kj z#7A>E-iJ`--=C6_2@ajPeCDI?Va}gFiGc1#@W1*VB>ou@YFC5zD@s;-^YbI*^`PB% z(A{^ZZv57gDGS8@dX>{}{%ri)Z!w0@UBjc~t$meg*9IzzRT2+mRq26e~1k&joQ6+?;caE##n-mMEa|KJ7SdK<7@ zqCl}UqNh;)N8leV79+IC7r*0BO)Q=;<|l>bWx9|R_>c+7N4;UZV3hw{>{eDZc$NuJ zK

*9Gv5DNs)pSBlZHfa?6kY_Oa%`4WARB_=^xZNCxZ`Z6*Yw8lv<;{boG+`8+0> zb;W~kFo*C2G3QkE5h1tN8styd2RPneq3?rQ62oJ$x&5b>81@zj4BABs{`IJq9jd3a z&11oe`EuX{+e#_%z|~G`5O`6t@C-c%hYykrQ7V%~rc`^&;&nZ~h#&_Wp%;>eIg5I8 zI_Ix@q-iq9l~8dI4nRAkb}=s$g>M7Ft1!sP%v{RljW#ylTPLhWn3q(nFqlNf5KEM9 znT2Fqs2j2Ho3rWKD6}a9k@fz*`(J!~0)#E6mq8Hjqilwqk=Bm^Z4jrOIdUAysj1Op z;wnw#u)|*lGR2AUCptN@`_T9WXNh#NP-(}u6vD*~2RDIY7rPtHgPIB~eGn!x0ScaZ zuo<9RzOS(vq9@z87AfO*PHfeW$=6ml0~boMyNMJ%;k!~LQPisQMoeiS_aM;;@7GWI zxH2lGYWu1ot@?caST5A~3hf5nuTaR!YOu{|@4X|?IZ+qa8P>gXs~03%?s`lAB?n9t zMXrklWsicBlZ$p%gOzsIyD7tk36yZ~S188R23306GBopB0ngMH?HLiTdpeeQ{WcQ_oqr&hDArUaI1dBzz;`WOweLUwBi)j+bkl)gJWg6OgI8a0b3P zJ=Y!n+ZT4w<^SO?BUDVt%H$c>d$=IxNB3Xn5WZ@ky>V#@&&rJ$8C+sJyt6{FhcRmi zaZo~m@}Aw#bj*bln9%jU1-yK}S3gI!c^c&hVO>E)TCA7)Z%rQ*apha@i-C`E;F*011;Y>BBG>$7!oawtx27!}V#Zvhxk%w~`6StK2E|1dMmCM) z;$Y4_@5!}l8Kl+W&s*8bqQxd=+S;H^i?RQ18IXpDZRr8sc`19=0hmg>MFW_$+ z#Ne(;j~9S{c5f4^#P>g)|K-?o6FUB6+jcQOzA{Re0M)UobMZy3c^fARJ;C&~>|@5| z5n|y;LD>IH0Mngq#8UXv33+XG?8nL85EKUc}vAK&)Qa!$O_%y7M~SFBTicxIXANxvzburXuB z^x#*ADmzKOd%~XiGQ^?Q`y|th6@++yEV2tZLW_~{J5*a^AU$;Pm;{lrUQy2-F1i%% zWTk8PHkq$29Z=&O3x$nlL+p8Uenb?-5UQyQ)zY%8AuBsB`sSy>EFxv06+aTQLNOW+ zv3+Eg_BkK@T-FnLFl;m7T?-Guo9fuuG^NH^5NohpUJL>dKXt%PY3{PL58Ez;1^Ah- zdc<{#UlA7s?k$N}ODMhiA_^|1+EaY~!C^(f=^p@DMD>-?tC#At@rP@}If>XRiOl l&GkPM{r{(l(zAT6WZE$$Gj61u+6VuW6qOUn`a{qCe*rP0S1 Date: Fri, 17 Mar 2017 13:30:53 +0800 Subject: [PATCH 43/62] Remove book submodule in PaddlePaddle --- .gitmodules | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 832698b8bf..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "book"] - path = book - url = https://github.com/PaddlePaddle/book.git - branch = develop \ No newline at end of file From 1ab419a1bdd5dd5013d923b156ac44c8db0d8955 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 17 Mar 2017 13:31:02 +0800 Subject: [PATCH 44/62] new design: #1625 --- .../docker/Dockerfile.gpu => Dockerfile | 18 ++--- paddle/scripts/docker/Dockerfile | 71 ------------------- paddle/scripts/docker/build.sh | 69 +++++++++++++++++- paddle/scripts/docker/buildall.sh | 31 -------- paddle/scripts/docker/paddle-core/Dockerfile | 32 --------- .../scripts/docker/paddle-core/Dockerfile.gpu | 32 --------- .../docker/paddle-core/Dockerfile.gpunoavx | 33 --------- .../docker/paddle-core/Dockerfile.noavx | 32 --------- paddle/scripts/docker/paddle-dev/Dockerfile | 53 -------------- 9 files changed, 73 insertions(+), 298 deletions(-) rename paddle/scripts/docker/Dockerfile.gpu => Dockerfile (88%) delete mode 100644 paddle/scripts/docker/Dockerfile delete mode 100644 paddle/scripts/docker/buildall.sh delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpu delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx delete mode 100644 paddle/scripts/docker/paddle-core/Dockerfile.noavx delete mode 100644 paddle/scripts/docker/paddle-dev/Dockerfile diff --git a/paddle/scripts/docker/Dockerfile.gpu b/Dockerfile similarity index 88% rename from paddle/scripts/docker/Dockerfile.gpu rename to Dockerfile index a687d490a3..b835cc52cb 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/Dockerfile @@ -1,3 +1,5 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Authors @@ -14,14 +16,12 @@ ARG WITH_STYLE_CHECK ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=ON +ENV WITH_GPU=OFF ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -ENV DOCKER_BUILD=TRUE ENV HOME /root - # Add bash enhancements COPY ./paddle/scripts/docker/root/ /root/ @@ -50,9 +50,7 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd .. && rm -rf cmake-3.4.1 -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh +RUN apt-get install -y swig VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] @@ -63,9 +61,5 @@ RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config EXPOSE 22 -# Jupyter Notebook: Paddle book -EXPOSE 8888 - -COPY ./paddle/scripts/docker/entrypoint /opt/bin/ - -CMD ["/opt/bin/entrypoint"] +# development image default do build work +CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"] diff --git a/paddle/scripts/docker/Dockerfile b/paddle/scripts/docker/Dockerfile deleted file mode 100644 index 48af9e5b5f..0000000000 --- a/paddle/scripts/docker/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Authors - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG BUILD_WOBOQ -ARG BUILD_AND_INSTALL -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} -ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -ENV DOCKER_BUILD=TRUE - -ENV HOME /root - -# Add bash enhancements -COPY ./paddle/scripts/docker/root/ /root/ - -RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 && \ - apt-get clean -y - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' && \ - pip install -U wheel pillow BeautifulSoup && \ - pip install -U docopt PyYAML sphinx && \ - pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install -U pre-commit 'requests==2.9.2' jupyter - -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -COPY . /paddle/ -RUN cd /paddle/ && git submodule update --init --recursive -RUN /paddle/paddle/scripts/docker/build.sh - -VOLUME ["/usr/share/nginx/html/data", "/usr/share/nginx/html/paddle"] - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -EXPOSE 22 - -# Jupyter Notebook: Paddle book -EXPOSE 8888 - -COPY ./paddle/scripts/docker/entrypoint /opt/bin/ - -CMD ["/opt/bin/entrypoint"] diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 668b6e6b84..9f91d8b571 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -7,7 +7,26 @@ function abort(){ trap 'abort' 0 set -e - +mkdir -p /paddle/dist/cpu +mkdir -p /paddle/dist/gpu +mkdir -p /paddle/dist/cpu-noavx +mkdir -p /paddle/dist/gpu-noavx +# Set BASE_IMAGE and DEB_PATH according to env variables +if [ ${WITH_GPU} == "ON" ]; then + BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" + if [ ${WITH_AVX} == "ON" ]; then + DEB_PATH="dist/gpu/" + else + DEB_PATH="dist/gpu-noavx/" + fi +else + BASE_IMAGE="python:2.7.13-slim" + if [ ${WITH_AVX} == "ON" ]; then + DEB_PATH="dist/cpu/" + else + DEB_PATH="dist/cpu-noavx/" + fi +fi # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied # source tree to /paddle, and this scripts should build it into # /paddle/build. @@ -29,9 +48,13 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` make install + # generate deb package for current build + # FIXME(typhoonzero): should we remove paddle/scripts/deb ? + cpack -D CPACK_GENERATOR='DEB' .. + mv /paddle/build/*.deb /paddle/${DEB_PATH} if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then - apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev + apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev # Install woboq_codebrowser. git clone https://github.com/woboq/woboq_codebrowser /woboq cd /woboq @@ -65,4 +88,46 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then fi fi +# generate production docker image Dockerfile +if [ ${USE_MIRROR} ]; then + MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\" +else + MIRROR_UPDATE="\\" +fi + +cat > /paddle/build/Dockerfile < + +# ENV variables +ARG WITH_AVX +ARG WITH_DOC +ARG WITH_STYLE_CHECK + +ENV WITH_GPU=${WITH_GPU} +ENV WITH_AVX=\${WITH_AVX:-ON} +ENV WITH_DOC=\${WITH_DOC:-OFF} +ENV WITH_STYLE_CHECK=\${WITH_STYLE_CHECK:-OFF} + +ENV HOME /root +ENV LANG en_US.UTF-8 + +# Use Fix locales to en_US.UTF-8 + +RUN ${MIRROR_UPDATE} + apt-get update && \ + apt-get install -y libgfortran3 && \ + apt-get clean -y && \ + pip install --upgrade pip && \ + pip install -U 'protobuf==3.1.0' +RUN pip install numpy +# Use different deb file when building different type of images +ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/ +RUN dpkg --force-all -i /usr/local/opt/paddle/deb/*.deb && rm -f /usr/local/opt/paddle/deb/*.deb + +ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" +# default command shows the paddle version and exit +CMD ["paddle", "version"] +EOF + trap : 0 diff --git a/paddle/scripts/docker/buildall.sh b/paddle/scripts/docker/buildall.sh deleted file mode 100644 index 63e5301ad9..0000000000 --- a/paddle/scripts/docker/buildall.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -BUILD_DIR=$PWD/build -DEB_DIST_DIR=$PWD/dist -VERSION=latest - -function build_in_docker() { - if [ ! -d $BUILD_DIR ]; then - mkdir -p $BUILD_DIR - fi - if [ ! -d $DEB_DIST_DIR ]; then - mkdir -p $DEB_DIST_DIR - fi - docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile - # FIXME: need to wait a signal not sleeping - BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600) - # NOTICE: build deb files for real paddle image - docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh" - - docker stop $BUILDER && docker rm $BUILDER -} - -function build_paddle_core() { - docker build . -t paddle:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile - docker build . -t paddle:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu - docker build . -t paddle:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx - docker build . -t paddle:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx -} - -build_in_docker -build_paddle_core diff --git a/paddle/scripts/docker/paddle-core/Dockerfile b/paddle/scripts/docker/paddle-core/Dockerfile deleted file mode 100644 index eade296a40..0000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM python:2.7.13-slim -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpu b/paddle/scripts/docker/paddle-core/Dockerfile.gpu deleted file mode 100644 index 89386b0379..0000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile.gpu +++ /dev/null @@ -1,32 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y python python-pip libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx b/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx deleted file mode 100644 index beaeb5c51e..0000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx +++ /dev/null @@ -1,33 +0,0 @@ -FROM python:2.7.13-slim -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb - - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-core/Dockerfile.noavx b/paddle/scripts/docker/paddle-core/Dockerfile.noavx deleted file mode 100644 index 853dd7703a..0000000000 --- a/paddle/scripts/docker/paddle-core/Dockerfile.noavx +++ /dev/null @@ -1,32 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04 -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root -ENV LANG en_US.UTF-8 - -# Use Fix locales to en_US.UTF-8 - -RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y python python-pip libgfortran3 && \ - apt-get clean -y && \ - pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' -RUN pip install numpy -# Use different deb file when building different type of images -ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/ -RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb - -ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" -# default command shows the paddle version and exit -CMD ["paddle", "version"] diff --git a/paddle/scripts/docker/paddle-dev/Dockerfile b/paddle/scripts/docker/paddle-dev/Dockerfile deleted file mode 100644 index 11465a4ed8..0000000000 --- a/paddle/scripts/docker/paddle-dev/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Authors - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' - -# ENV variables -ARG BUILD_WOBOQ -ARG BUILD_AND_INSTALL -ARG WITH_AVX -ARG WITH_DOC -ARG WITH_STYLE_CHECK - -ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} -ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=OFF -ENV WITH_AVX=${WITH_AVX:-ON} -ENV WITH_DOC=${WITH_DOC:-OFF} -ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} - -ENV HOME /root - -RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 && \ - apt-get clean -y - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' && \ - pip install -U wheel pillow BeautifulSoup && \ - pip install -U docopt PyYAML sphinx && \ - pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install -U pre-commit 'requests==2.9.2' jupyter - -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -RUN apt-get install -y swig -# FIXME: wait a long time is OK -CMD ["sleep", "3600"] From 8c8b2efdeb95491d29d181b0f70e2738d17c2762 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 17 Mar 2017 13:46:00 +0800 Subject: [PATCH 45/62] Add comments --- paddle/py_paddle/dataprovider_converter.py | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index f1ed57f13f..6d6a406cf6 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -22,6 +22,19 @@ __all__ = ['DataProviderConverter'] class IScanner(object): + """ + The scanner will scan Python object two passes, then convert it to Paddle's + argument. + + In the first pass, `pre_scan` will be invoked by every data instance, and + then invoke `finish_pre_scan` to arguments. And the second pass do the same + thing except the functions changed to `scan`, `finish_scan`. + + During the first pass, a scanner may count the shape of input matrix and + allocate memory for this argument. Then fill the data into this argument + in second pass. + """ + def __init__(self, input_type, pos): self.input_type = input_type if not isinstance(self.input_type, dp2.InputType): @@ -38,15 +51,39 @@ class IScanner(object): ) and swig_paddle.getTrainerCount() == 1 def pre_scan(self, dat): + """ + First pass scan method. During this method, the scanner could count the + data number, and get the total memory size this batch would use. + + :param dat: The python object. + """ pass def finish_pre_scan(self, argument): + """ + Finish first scan pass. Allocate the memory. + + :param argument: Output arguments object. + :type argument: swig_paddle.Arguments + :return: + """ pass def scan(self, dat): + """ + Second pass scan method. Copy the data to arguments. + + :param dat: The python object. + """ pass def finish_scan(self, argument): + """ + Finish second pass. Finalize the resources, etc. + + :param argument: Output arguments object. + :type argument: swig_paddle.Arguments + """ pass From 01d9fa41af551ac1409f1f762bb0309c4ea7897e Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 17 Mar 2017 15:15:07 +0800 Subject: [PATCH 46/62] Remove book submodule dir --- book | 1 - 1 file changed, 1 deletion(-) delete mode 160000 book diff --git a/book b/book deleted file mode 160000 index 6e3875eb62..0000000000 --- a/book +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6e3875eb62533de1f2c1088a477719eb57b9732c From 653c981df920cff7b1e7b194ddf5121f23614f15 Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 17 Mar 2017 16:09:26 +0800 Subject: [PATCH 47/62] tags for build docker image --- Dockerfile | 2 +- paddle/scripts/docker/build.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b835cc52cb..8f3137df08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ MAINTAINER PaddlePaddle Authors ARG DEBIAN_FRONTEND=noninteractive ARG UBUNTU_MIRROR -RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' +RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi' # ENV variables ARG BUILD_WOBOQ diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 9f91d8b571..b617865016 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -16,15 +16,19 @@ if [ ${WITH_GPU} == "ON" ]; then BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" if [ ${WITH_AVX} == "ON" ]; then DEB_PATH="dist/gpu/" + DOCKER_SUFFIX="gpu" else DEB_PATH="dist/gpu-noavx/" + DOCKER_SUFFIX="gpu-noavx" fi else BASE_IMAGE="python:2.7.13-slim" if [ ${WITH_AVX} == "ON" ]; then DEB_PATH="dist/cpu/" + DOCKER_SUFFIX="" else DEB_PATH="dist/cpu-noavx/" + DOCKER_SUFFIX="noavx" fi fi # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied @@ -95,7 +99,7 @@ else MIRROR_UPDATE="\\" fi -cat > /paddle/build/Dockerfile < /paddle/build/Dockerfile.${DOCKER_SUFFIX} < From e6dca9e58dc5c7d2d54d78f7913fdcbabd472369 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 17 Mar 2017 17:02:32 +0800 Subject: [PATCH 48/62] Follow comments --- .../multi_language_interface/why_plain_c.md | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/doc/design/multi_language_interface/why_plain_c.md b/doc/design/multi_language_interface/why_plain_c.md index a780d99bc4..a3f41ca7b9 100644 --- a/doc/design/multi_language_interface/why_plain_c.md +++ b/doc/design/multi_language_interface/why_plain_c.md @@ -4,9 +4,9 @@ Paddle需要一个多语言接口，这个接口需要做到: * 有标准的，良好的文档 - * 例如Python可以使用[Sphinx](http://www.sphinx-doc.org/en/stable/)生成API文档，golang可以使用[GoDoc](https://godoc.org/golang.org/x/tools/cmd/godoc)生成文档。这都需要这个接口按照约定俗成的规则来注释完备。 + * 例如Python可以使用[Sphinx](http://www.sphinx-doc.org/en/stable/)生成API文档，golang可以使用[GoDoc](https://godoc.org/golang.org/x/tools/cmd/godoc)生成文档。这都需要这个接口按照约定俗成的规则来注释完备。 * 不同语言的接口适应不同语言的特性 - * 例如Java与Python的错误处理是直接扔出来Exception，而对于golang错误处理应该使用返回值。 + * 例如Java与Python的错误处理是直接扔出来Exception，而对于golang错误处理应该使用返回值。 ## 基本要求 @@ -23,9 +23,9 @@ Paddle的多语言接口实现包括一下几个方面: ### 使用动态库来分发Paddle * Paddle的链接方式比较复杂 - * Paddle链接静态库使用了GCC的--whole-archieve参数，它要求使用Paddle静态库的二进制，在链接参数中指定`--whole-archieve paddle_xxx_lib --no-whole-archive`。且这个链接参数是GCC独有的。对于clang或者msvc，参数会不同。这增加了用户使用Paddle静态库的难度。 -* 编译型语言，例如C/C++使用静态库和动态库难度差不多。但是含有解释器的语言，例如[Python](http://stackoverflow.com/questions/19560594/how-to-import-static-library-in-python)或者[Java](http://stackoverflow.com/questions/24493337/linking-static-library-with-jni)，调用动态库远比调用静态库方便。 - * 解释性语言实际运行的二进制是解释器本身，如果调用静态库只能将静态库与解释器链接。例如对于Java来说，便是将静态库加入JVM中。这对于通常的Java的开发者来说，是不常见的做法。 + * 如果用户要把Paddle的静态库（libpaddle.a）链接到自己的程序里，得使用 `--whole-archive` (for GCC) 或者 `--force_load` (for Clang) 参数，来确保把 libpaddle.a 里所有的符号都写入自己的程序的二进制文件里。这是因为 Paddle 的源码里使用了[object factory design pattern](http://stackoverflow.com/a/1310326/724872)。 +* 编译型语言，例如C/C++使用静态库和动态库难度差不多。但是解释性语言，例如[Python](http://stackoverflow.com/questions/19560594/how-to-import-static-library-in-python)或者[Java](http://stackoverflow.com/questions/24493337/linking-static-library-with-jni)，只能调用Paddle的动态库，否则得把Paddle静态库链接到解释器里。 + * 解释性语言实际运行的二进制是解释器本身，如果调用静态库只能将静态库与解释器链接。例如对于Java来说，便是将静态库加入JVM中。这对于通常的Java的开发者来说，是不常见的做法。 ### 动态库中不嵌入任何其他语言的解释器 @@ -51,36 +51,54 @@ Paddle的多语言接口实现包括一下几个方面: * Paddle内部的类为C++书写，直接导出到C的接口比较困难。 * 在C-API中使用`void*`来表示Paddle内部类。再在每一个API中自己检查类型。 -```C +在C的头文件 `paddle_matrix.h` 中: -// in Paddle.h +```C typedef void* paddle_matrix; +typedef int paddle_error; + +extern "C" +paddle_error paddle_matrix_shape(paddle_matrix matrix, + uint64_t* width, + uint64_t* height); +``` +而在CPP里面实现这个C的接口，文件 `paddle_matrix.cpp` + +```cpp +#include "paddle/math/matrix.hpp" +extern "C" +paddle_error paddle_matrix_shape(paddle_matrix matrix, + uint64_t *width, + uint64_t *height) { + auto m = (paddle::math::matrix*)(matrix); + *width = m->width(); + *height = m->height(); +} +``` -extern "C" paddle_error getShape(paddle_matrix mat, uint64_t* height, uint64_t* width); +其中`paddle/math/matrix.hpp`文件内容为: +```cpp +namespace paddle { +namespace math { -// in matrix.cpp -struct PaddleMatrix { - int type; - paddle::MatrixPtr mat; +class Matrix { + //... }; -paddle_error get_shape(paddle_matrix m, uint64_t* height, uint64_t* width) { - PaddleMatrix* realMat = (PaddleMatrix*)(m); - ... -} - +} // namespace math +} // namespace paddle ``` ### 不使用SWIG这种代码生成器，而是手写多语言绑定 * [SWIG](http://www.swig.org/)是一个多语言接口的代码生成器。他的目标是使用C/C++写代码，SWIG直接读取C/C++的头文件，生成各种语言的绑定代码。 - * 对于多语言接口，SWIG需要写一个interface文件。这个文件具有独特的语法，学习成本高。且增加一个第三方语言，就需要对这个第三方语言增加一些定义。有的时候，interface文件的写法非常[tricky](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/api/Paddle.swig#L36)。社区贡献代码学习成本高。 - * SWIG暴露的接口保留了C++的接口样式，很难保证多语言代码风格的一致性。(函数命名，错误处理) - * 因为SWIG在第三方语言中暴露的函数名，类名和C++中完全一致。C++的命名风格并不能适应其他第三方语言。如果使用SWIG我们需要将在interface文件里，将大量的`SomeCppClass`重命名成`some_python_class`，或者`SomeGoTypes`。 - * 对于不同语言，错误处理的方式也不尽相同。例如对于Java或者Python，最常见的错误处理方式是Exception，而对于Golang，错误处理方式是返回值。而SWIG只能简单的暴露C++接口，无法做到对于各种语言错误处理方式的适配。 - * 对于大多数语言，直接使用C语言的.h并不困难。例如Python的[cffi](https://cffi.readthedocs.io/en/latest/overview.html#simple-example-abi-level-in-line)或者[Cython](http://cython.org/), golang的[cgo](https://golang.org/cmd/cgo/)。 - * SWIG支持的语言或者解释器有局限。例如对于Python，使用SWIG只支持CPython解释器，而不支持PyPy解释器。 + * 对于多语言接口，SWIG需要写一个interface文件。这个文件具有独特的语法，学习成本高。且增加一个第三方语言，就需要对这个第三方语言增加一些定义。有的时候，interface文件的写法非常[tricky](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/api/Paddle.swig#L36)。社区贡献代码学习成本高。 + * SWIG暴露的接口保留了C++的接口样式，很难保证多语言代码风格的一致性。(函数命名，错误处理) + * 因为SWIG在第三方语言中暴露的函数名，类名和C++中完全一致。C++的命名风格并不能适应其他第三方语言。如果使用SWIG我们需要将在interface文件里，将大量的`SomeCppClass`重命名成`some_python_class`，或者`SomeGoTypes`。 + * 对于不同语言，错误处理的方式也不尽相同。例如对于Java或者Python，最常见的错误处理方式是Exception，而对于Golang，错误处理方式是返回值。而SWIG只能简单的暴露C++接口，无法做到对于各种语言错误处理方式的适配。 + * 对于大多数语言，直接使用C语言的.h并不困难。例如Python的[cffi](https://cffi.readthedocs.io/en/latest/overview.html#simple-example-abi-level-in-line)或者[Cython](http://cython.org/), golang的[cgo](https://golang.org/cmd/cgo/)。 + * SWIG支持的语言或者解释器有局限。例如对于Python，使用SWIG只支持CPython解释器，而不支持PyPy解释器。 ## 原因列表 From 7dbc77ba4d7be39ca666ea0c1139917c0fa1fe1e Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 17 Mar 2017 17:11:10 +0800 Subject: [PATCH 49/62] rename regression_cost to mse_cost --- demo/introduction/api_train_v2.py | 2 +- demo/introduction/trainer_config.py | 2 +- demo/recommendation/api_train_v2.py | 2 +- demo/recommendation/trainer_config.py | 5 +---- doc/api/v1/trainer_config_helpers/layers.rst | 18 ++++++++++++------ doc/getstarted/basic_usage/index_cn.rst | 4 ++-- doc/getstarted/basic_usage/index_en.rst | 2 +- doc/howto/usage/k8s/k8s_distributed_cn.md | 2 +- python/paddle/trainer_config_helpers/layers.py | 11 +++++++---- .../test_cost_layers_with_weight.protostr | 8 ++++---- .../configs/test_cost_layers_with_weight.py | 2 +- python/paddle/v2/tests/test_layer.py | 5 ++--- 12 files changed, 34 insertions(+), 29 deletions(-) diff --git a/demo/introduction/api_train_v2.py b/demo/introduction/api_train_v2.py index 84125c3b4b..1ba971b368 100644 --- a/demo/introduction/api_train_v2.py +++ b/demo/introduction/api_train_v2.py @@ -14,7 +14,7 @@ def main(): act=paddle.activation.Linear(), bias_attr=paddle.attr.Param(name='b')) y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) - cost = paddle.layer.regression_cost(input=y_predict, label=y) + cost = paddle.layer.mse_cost(input=y_predict, label=y) # create parameters parameters = paddle.parameters.create(cost) diff --git a/demo/introduction/trainer_config.py b/demo/introduction/trainer_config.py index ecafe955f9..651dfaa4b7 100644 --- a/demo/introduction/trainer_config.py +++ b/demo/introduction/trainer_config.py @@ -34,5 +34,5 @@ y_predict = fc_layer( size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) -cost = regression_cost(input=y_predict, label=y) +cost = mse_cost(input=y_predict, label=y) outputs(cost) diff --git a/demo/recommendation/api_train_v2.py b/demo/recommendation/api_train_v2.py index 9b254933a1..f6a061799e 100644 --- a/demo/recommendation/api_train_v2.py +++ b/demo/recommendation/api_train_v2.py @@ -61,7 +61,7 @@ def main(): inference = paddle.layer.cos_sim( a=usr_combined_features, b=mov_combined_features, size=1, scale=5) - cost = paddle.layer.regression_cost( + cost = paddle.layer.mse_cost( input=inference, label=paddle.layer.data( name='score', type=paddle.data_type.dense_vector(1))) diff --git a/demo/recommendation/trainer_config.py b/demo/recommendation/trainer_config.py index aabcd33525..25f529d7d7 100755 --- a/demo/recommendation/trainer_config.py +++ b/demo/recommendation/trainer_config.py @@ -86,10 +86,7 @@ movie_feature = construct_feature("movie") user_feature = construct_feature("user") similarity = cos_sim(a=movie_feature, b=user_feature) if not is_predict: - outputs( - regression_cost( - input=similarity, label=data_layer( - 'rating', size=1))) + outputs(mse_cost(input=similarity, label=data_layer('rating', size=1))) define_py_data_sources2( 'data/train.list', diff --git a/doc/api/v1/trainer_config_helpers/layers.rst b/doc/api/v1/trainer_config_helpers/layers.rst index bbea823de4..24389c2d85 100644 --- a/doc/api/v1/trainer_config_helpers/layers.rst +++ b/doc/api/v1/trainer_config_helpers/layers.rst @@ -432,6 +432,12 @@ multi_binary_label_cross_entropy :members: multi_binary_label_cross_entropy :noindex: +mse_cost +--------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: mse_cost + :noindex: + huber_cost ---------- .. automodule:: paddle.trainer_config_helpers.layers @@ -450,6 +456,12 @@ rank_cost :members: rank_cost :noindex: +sum_cost +--------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: sum_cost + :noindex: + crf_layer ----------------- .. automodule:: paddle.trainer_config_helpers.layers @@ -486,12 +498,6 @@ hsigmoid :members: hsigmoid :noindex: -sum_cost ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: sum_cost - :noindex: - Check Layer ============ diff --git a/doc/getstarted/basic_usage/index_cn.rst b/doc/getstarted/basic_usage/index_cn.rst index d01cdaaeb7..428f58830e 100644 --- a/doc/getstarted/basic_usage/index_cn.rst +++ b/doc/getstarted/basic_usage/index_cn.rst @@ -55,7 +55,7 @@ PaddlePaddle是源于百度的一个深度学习平台。这份简短的介绍 # 线性计算网络层: ȳ = wx + b ȳ = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) # 计算误差函数，即 ȳ 和真实 y 之间的距离 - cost = regression_cost(input= ȳ, label=y) + cost = mse_cost(input= ȳ, label=y) outputs(cost) @@ -69,7 +69,7 @@ PaddlePaddle是源于百度的一个深度学习平台。这份简短的介绍 - **数据层**：数据层 `data_layer` 是神经网络的入口，它读入数据并将它们传输到接下来的网络层。这里数据层有两个，分别对应于变量 `x` 和 `y`。 - **全连接层**：全连接层 `fc_layer` 是基础的计算单元，这里利用它建模变量之间的线性关系。计算单元是神经网络的核心，PaddlePaddle支持大量的计算单元和任意深度的网络连接，从而可以拟合任意的函数来学习复杂的数据关系。 - - **回归误差代价层**：回归误差代价层 `regression_cost` 是众多误差代价函数层的一种，它们在训练过程作为网络的出口，用来计算模型的误差，是模型参数优化的目标函数。 + - **回归误差代价层**：回归误差代价层 `mse_cost` 是众多误差代价函数层的一种，它们在训练过程作为网络的出口，用来计算模型的误差，是模型参数优化的目标函数。 定义了网络结构并保存为 `trainer_config.py` 之后，运行以下训练命令： diff --git a/doc/getstarted/basic_usage/index_en.rst b/doc/getstarted/basic_usage/index_en.rst index c10b897d42..6775da20c2 100644 --- a/doc/getstarted/basic_usage/index_en.rst +++ b/doc/getstarted/basic_usage/index_en.rst @@ -49,7 +49,7 @@ To recover this relationship between ``X`` and ``Y``, we use a neural network wi x = data_layer(name='x', size=1) y = data_layer(name='y', size=1) y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) - cost = regression_cost(input=y_predict, label=y) + cost = mse_cost(input=y_predict, label=y) outputs(cost) Some of the most fundamental usages of PaddlePaddle are demonstrated: diff --git a/doc/howto/usage/k8s/k8s_distributed_cn.md b/doc/howto/usage/k8s/k8s_distributed_cn.md index 2a7a6c8c17..3121b3f59d 100644 --- a/doc/howto/usage/k8s/k8s_distributed_cn.md +++ b/doc/howto/usage/k8s/k8s_distributed_cn.md @@ -213,7 +213,7 @@ I1116 09:10:17.123440 50 Util.cpp:130] Calling runInitFunctions I1116 09:10:17.123764 50 Util.cpp:143] Call runInitFunctions done. [WARNING 2016-11-16 09:10:17,227 default_decorators.py:40] please use keyword arguments in paddle config. [INFO 2016-11-16 09:10:17,239 networks.py:1282] The input order is [movie_id, title, genres, user_id, gender, age, occupation, rating] -[INFO 2016-11-16 09:10:17,239 networks.py:1289] The output order is [__regression_cost_0__] +[INFO 2016-11-16 09:10:17,239 networks.py:1289] The output order is [__mse_cost_0__] I1116 09:10:17.392917 50 Trainer.cpp:170] trainer mode: Normal I1116 09:10:17.613910 50 PyDataProvider2.cpp:257] loading dataprovider dataprovider::process I1116 09:10:17.680917 50 PyDataProvider2.cpp:257] loading dataprovider dataprovider::process diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index b94f8f9a78..1eb4cf7968 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -52,7 +52,7 @@ __all__ = [ "cos_sim", "hsigmoid", "conv_projection", - "regression_cost", + "mse_cost", 'classification_cost', "LayerOutput", 'img_conv_layer', @@ -3572,11 +3572,14 @@ def __cost_input__(input, label, weight=None): @wrap_name_default() @layer_support() -def regression_cost(input, label, weight=None, name=None, layer_attr=None): +def mse_cost(input, label, weight=None, name=None, layer_attr=None): """ - Regression Layer. + mean squared error cost: + + .. math:: + + $\frac{1}{N}\sum_{i=1}^N(t _i- y_i)^2$ - TODO(yuyang18): Complete this method. :param name: layer name. :type name: basestring diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr index 811b38ae4a..3244181a63 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr @@ -45,7 +45,7 @@ layers { coeff: 1.0 } layers { - name: "__regression_cost_0__" + name: "__mse_cost_0__" type: "square_error" size: 1 active_type: "" @@ -84,7 +84,7 @@ input_layer_names: "input" input_layer_names: "label" input_layer_names: "weight" output_layer_names: "__cost_0__" -output_layer_names: "__regression_cost_0__" +output_layer_names: "__mse_cost_0__" evaluators { name: "classification_error_evaluator" type: "classification_error" @@ -99,12 +99,12 @@ sub_models { layer_names: "weight" layer_names: "__fc_layer_0__" layer_names: "__cost_0__" - layer_names: "__regression_cost_0__" + layer_names: "__mse_cost_0__" input_layer_names: "input" input_layer_names: "label" input_layer_names: "weight" output_layer_names: "__cost_0__" - output_layer_names: "__regression_cost_0__" + output_layer_names: "__mse_cost_0__" evaluator_names: "classification_error_evaluator" is_recurrent_layer_group: false } diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py index d30f70a55c..1c0aa7f9b9 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py @@ -10,5 +10,5 @@ fc = fc_layer(input=data, size=10, act=SoftmaxActivation()) outputs( classification_cost( input=fc, label=lbl, weight=wt), - regression_cost( + mse_cost( input=fc, label=lbl, weight=wt)) diff --git a/python/paddle/v2/tests/test_layer.py b/python/paddle/v2/tests/test_layer.py index 0055679a91..5ccd3d6913 100644 --- a/python/paddle/v2/tests/test_layer.py +++ b/python/paddle/v2/tests/test_layer.py @@ -126,9 +126,8 @@ class CostLayerTest(unittest.TestCase): cost3 = layer.cross_entropy_cost(input=inference, label=label) cost4 = layer.cross_entropy_with_selfnorm_cost( input=inference, label=label) - cost5 = layer.regression_cost(input=inference, label=label) - cost6 = layer.regression_cost( - input=inference, label=label, weight=weight) + cost5 = layer.mse_cost(input=inference, label=label) + cost6 = layer.mse_cost(input=inference, label=label, weight=weight) cost7 = layer.multi_binary_label_cross_entropy_cost( input=inference, label=label) cost8 = layer.rank_cost(left=score, right=score, label=score) From 36ed2ff1ea2a92be2ade3fb990bd7317d6d48d75 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 17 Mar 2017 17:35:33 +0800 Subject: [PATCH 50/62] keep forward compatibility --- python/paddle/trainer_config_helpers/layers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 1eb4cf7968..7cd3ce9131 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -53,6 +53,7 @@ __all__ = [ "hsigmoid", "conv_projection", "mse_cost", + "regression_cost", 'classification_cost', "LayerOutput", 'img_conv_layer', @@ -3605,6 +3606,9 @@ def mse_cost(input, label, weight=None, name=None, layer_attr=None): return LayerOutput(name, LayerType.COST, parents=parents, size=1) +regression_cost = mse_cost + + @wrap_name_default("cost") @layer_support() def classification_cost(input, From 0b4d4560460718337639d9ad122f4277c192c4bd Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 17 Mar 2017 20:55:57 +0800 Subject: [PATCH 51/62] add python requests, add DELETE_BUILD_CACHE arg --- paddle/scripts/docker/build.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index b617865016..1779a0753b 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -25,7 +25,7 @@ else BASE_IMAGE="python:2.7.13-slim" if [ ${WITH_AVX} == "ON" ]; then DEB_PATH="dist/cpu/" - DOCKER_SUFFIX="" + DOCKER_SUFFIX="cpu" else DEB_PATH="dist/cpu-noavx/" DOCKER_SUFFIX="noavx" @@ -40,8 +40,11 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then fi mkdir -p /paddle/build # -p means no error if exists + cd /paddle/build # clean local cmake and third_party cache - cd /paddle/build && rm -rf * && rm -rf ../third_party + if [ ${DELETE_BUILD_CACHE} == 'ON' ]; then + rm -rf * && rm -rf ../third_party + fi cmake .. \ -DWITH_DOC=${WITH_DOC:-OFF} \ -DWITH_GPU=${WITH_GPU:-OFF} \ @@ -54,7 +57,9 @@ if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then make install # generate deb package for current build # FIXME(typhoonzero): should we remove paddle/scripts/deb ? - cpack -D CPACK_GENERATOR='DEB' .. + # FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must + # install them in docker + cpack -D CPACK_GENERATOR='DEB' -D CPACK_DEBIAN_PACKAGE_DEPENDS="" .. mv /paddle/build/*.deb /paddle/${DEB_PATH} if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then @@ -123,7 +128,7 @@ RUN ${MIRROR_UPDATE} apt-get install -y libgfortran3 && \ apt-get clean -y && \ pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' + pip install -U 'protobuf==3.1.0' requests RUN pip install numpy # Use different deb file when building different type of images ADD \$PWD/${DEB_PATH}*.deb /usr/local/opt/paddle/deb/ From e8d33bee479a1bd574875f8b5ca1463955d6323d Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Fri, 17 Mar 2017 21:25:19 +0800 Subject: [PATCH 52/62] gpu image dependencies --- paddle/scripts/docker/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 1779a0753b..c44874eede 100755 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -14,6 +14,8 @@ mkdir -p /paddle/dist/gpu-noavx # Set BASE_IMAGE and DEB_PATH according to env variables if [ ${WITH_GPU} == "ON" ]; then BASE_IMAGE="nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04" + # additional packages to install when building gpu images + GPU_DOCKER_PKG="python-pip" if [ ${WITH_AVX} == "ON" ]; then DEB_PATH="dist/gpu/" DOCKER_SUFFIX="gpu" @@ -125,7 +127,7 @@ ENV LANG en_US.UTF-8 RUN ${MIRROR_UPDATE} apt-get update && \ - apt-get install -y libgfortran3 && \ + apt-get install -y libgfortran3 ${GPU_DOCKER_PKG} && \ apt-get clean -y && \ pip install --upgrade pip && \ pip install -U 'protobuf==3.1.0' requests From 5eeb18705a2c468b1efd60f05487a4c7f69c7fe4 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Sat, 18 Mar 2017 11:39:37 +0800 Subject: [PATCH 53/62] delete deb path --- paddle/scripts/deb/build_scripts/.gitignore | 1 - paddle/scripts/deb/build_scripts/Dockerfile | 5 --- paddle/scripts/deb/build_scripts/build.sh | 43 ------------------- paddle/scripts/deb/build_scripts/build_deb.sh | 8 ---- paddle/scripts/deb/postinst | 7 --- 5 files changed, 64 deletions(-) delete mode 100644 paddle/scripts/deb/build_scripts/.gitignore delete mode 100644 paddle/scripts/deb/build_scripts/Dockerfile delete mode 100755 paddle/scripts/deb/build_scripts/build.sh delete mode 100755 paddle/scripts/deb/build_scripts/build_deb.sh delete mode 100644 paddle/scripts/deb/postinst diff --git a/paddle/scripts/deb/build_scripts/.gitignore b/paddle/scripts/deb/build_scripts/.gitignore deleted file mode 100644 index 1521c8b765..0000000000 --- a/paddle/scripts/deb/build_scripts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist diff --git a/paddle/scripts/deb/build_scripts/Dockerfile b/paddle/scripts/deb/build_scripts/Dockerfile deleted file mode 100644 index db365a65b7..0000000000 --- a/paddle/scripts/deb/build_scripts/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM paddledev/paddle:gpu-latest -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -CMD cd /root/ && bash build.sh - diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh deleted file mode 100755 index 51cf7f1b9b..0000000000 --- a/paddle/scripts/deb/build_scripts/build.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -e -ARCH=$(uname -i) -apt-get update -apt-get install -y dh-make -cd ~ -mkdir -p ~/dist/gpu -mkdir -p ~/dist/cpu -mkdir -p ~/dist/cpu-noavx -mkdir -p ~/dist/gpu-noavx -cd paddle - -# clean build dir and third_party dir cache -rm -rf build third_party -mkdir -p build -cd build -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -make -j `nproc` -# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything? -cpack -D CPACK_GENERATOR='DEB' .. -mv *.deb ~/dist/cpu - -rm -rf * -ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -make -j `nproc` -cpack -D CPACK_GENERATOR='DEB' .. -mv *.deb ~/dist/gpu - - -rm -rf * -rm -f /usr/lib/libcudnn.so -cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -make -j `nproc` -cpack -D CPACK_GENERATOR='DEB' .. -mv *.deb ~/dist/cpu-noavx - -rm -rf * -ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -make -j `nproc` -cpack -D CPACK_GENERATOR='DEB' .. -mv *.deb ~/dist/gpu-noavx diff --git a/paddle/scripts/deb/build_scripts/build_deb.sh b/paddle/scripts/deb/build_scripts/build_deb.sh deleted file mode 100755 index c38c6299f8..0000000000 --- a/paddle/scripts/deb/build_scripts/build_deb.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e -docker build -t build_paddle_deb . -rm -rf dist -mkdir -p dist -docker run -v$PWD/dist:/root/dist -v $PWD/../../../..:/root/paddle --name tmp_build_deb_container build_paddle_deb -docker rm tmp_build_deb_container -docker rmi build_paddle_deb diff --git a/paddle/scripts/deb/postinst b/paddle/scripts/deb/postinst deleted file mode 100644 index 1d2dd3171a..0000000000 --- a/paddle/scripts/deb/postinst +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -echo "Post install paddle debian package." -echo "Install some python package used for paddle. You can run " -echo " pip install /usr/opt/paddle/share/wheels/*.whl to install them." -pip install /usr/opt/paddle/share/wheels/*.whl - From 48a1e57f1cde87a3d8384a91acd6629111c3a99a Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Sat, 18 Mar 2017 11:51:07 +0800 Subject: [PATCH 54/62] add gpu arg in develop Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8f3137df08..536adb0716 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,13 +10,14 @@ RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ub # ENV variables ARG BUILD_WOBOQ ARG BUILD_AND_INSTALL +ARG WITH_GPU ARG WITH_AVX ARG WITH_DOC ARG WITH_STYLE_CHECK ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF} ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF} -ENV WITH_GPU=OFF +ENV WITH_GPU=${WITH_AVX:-OFF} ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} From 1ec66a334bb7a161b2f2cd75aeee0597eecdcf13 Mon Sep 17 00:00:00 2001 From: gaoyuan Date: Sat, 18 Mar 2017 15:49:08 +0800 Subject: [PATCH 55/62] change resizeOrCreate to create --- paddle/gserver/layers/CostLayer.cpp | 34 ++++++++++++----------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index e2a4153bad..4ae5b82870 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -206,17 +206,14 @@ bool SmoothL1CostLayer::init(const LayerMap& layerMap, void SmoothL1CostLayer::forwardImp(Matrix& output, Argument& label, Matrix& target) { - MatrixPtr targetCpu, labelCpu, outputCpu; + MatrixPtr targetCpu, outputCpu, labelCpu; if (useGpu_) { - Matrix::resizeOrCreate( - targetCpu, target.getHeight(), target.getWidth(), false, false); - Matrix::resizeOrCreate( - outputCpu, output.getHeight(), output.getWidth(), false, false); - Matrix::resizeOrCreate(labelCpu, - label.value->getHeight(), - label.value->getWidth(), - false, - false); + targetCpu = + Matrix::create(target.getHeight(), target.getWidth(), false, false); + outputCpu = + Matrix::create(output.getHeight(), output.getWidth(), false, false); + labelCpu = Matrix::create( + label.value->getHeight(), label.value->getWidth(), false, false); targetCpu->copyFrom(target); outputCpu->copyFrom(output); labelCpu->copyFrom(*label.value); @@ -230,17 +227,14 @@ void SmoothL1CostLayer::forwardImp(Matrix& output, void SmoothL1CostLayer::backwardImp(Matrix& output, Argument& label, Matrix& outputG) { - MatrixPtr outputGCpu, labelCpu, outputCpu; + MatrixPtr outputGCpu, outputCpu, labelCpu; if (useGpu_) { - Matrix::resizeOrCreate( - outputGCpu, outputG.getHeight(), outputG.getWidth(), false, false); - Matrix::resizeOrCreate( - outputCpu, output.getHeight(), output.getWidth(), false, false); - Matrix::resizeOrCreate(labelCpu, - label.value->getHeight(), - label.value->getWidth(), - false, - false); + outputGCpu = + Matrix::create(outputG.getHeight(), outputG.getWidth(), false, false); + outputCpu = + Matrix::create(output.getHeight(), output.getWidth(), false, false); + labelCpu = Matrix::create( + label.value->getHeight(), label.value->getWidth(), false, false); outputGCpu->copyFrom(outputG); outputCpu->copyFrom(output); labelCpu->copyFrom(*label.value); From ac5ff8785cea9fac529c67223c206bdd3f0cd50b Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 19 Mar 2017 12:17:54 -0700 Subject: [PATCH 56/62] Use Grammarly with the design doc --- paddle/scripts/docker/README.md | 42 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index fc38fd7fa6..8c35411fc3 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -2,32 +2,32 @@ ## Goals -We want the building procedure generates Docker images, so we can run PaddlePaddle applications on Kubernetes clusters. +We want the building procedure generates Docker images so that we can run PaddlePaddle applications on Kubernetes clusters. -We want it generates .deb packages, so that enterprises without Docker support can run PaddlePaddle applications as well. +We want to build .deb packages so that enterprise users can run PaddlePaddle applications without Docker. -We want to minimize the size of generated Docker images and .deb packages so to ease the deployment cost. +We want to minimize the size of generated Docker images and .deb packages so to reduce the download time. We want to encapsulate building tools and dependencies in a *development* Docker image so to ease the tools installation for developers. -We want developers can use whatever editing tools (emacs, vim, Eclipse, Jupyter Notebook), so the development Docker image contains only building tools, not editing tools, and developers are supposed to git clone source code into their development computers, instead of the container running the development Docker image. +Developers use various editors (emacs, vim, Eclipse, Jupyter Notebook), so the development Docker image contains only building tools, not editing tools, and developers are supposed to git clone source code into their development computers and map the code into the development container. -We want the procedure and tools work also with testing, continuous integration, and releasing. +We want the procedure and tools also work with testing, continuous integration, and releasing. ## Docker Images -We want two Docker images for each version of PaddlePaddle: +So we need two Docker images for each version of PaddlePaddle: 1. `paddle:-dev` - This a development image contains only the development tools. This standardizes the building tools and procedure. Users include: + This a development image contains only the development tools and standardizes the building procedure. Users include: - developers -- no longer need to install development tools on the host, and can build their current work on the host (development computer). - release engineers -- use this to build the official release from certain branch/tag on Github.com. - document writers / Website developers -- Our documents are in the source repo in the form of .md/.rst files and comments in source code. We need tools to extract the information, typeset, and generate Web pages. - Of course developers can install building tools on their development computers. But different version of PaddlePaddle might require different set/version of building tools. Also, it makes collaborative debugging eaiser if all developers use a unified development environment. + Of course, developers can install building tools on their development computers. But different versions of PaddlePaddle might require different set or version of building tools. Also, it makes collaborative debugging easier if all developers use a unified development environment. The development image should include the following tools: @@ -38,7 +38,7 @@ We want two Docker images for each version of PaddlePaddle: - woboq - sshd - where `sshd` makes it easy for developers to have multiple terminals connecting into the container. `docker exec` works too, but if the container is running on a remote machine, it would be easier to ssh directly into the container than ssh to the box and run `docker exec`. + Many developers work on a remote computer with GPU; they could ssh into the computer and `docker exec` into the development container. However, running `sshd` in the container allows developers to ssh into the container directly. 1. `paddle:` @@ -49,9 +49,9 @@ We want two Docker images for each version of PaddlePaddle: - no-GPU/AVX `paddle:` - no-GPU/no-AVX `paddle:-noavx` - We'd like to give users the choice between GPU and no-GPU, because the GPU version image is much larger than then the no-GPU version. + We allow users to choose between GPU and no-GPU because the GPU version image is much larger than then the no-GPU version. - We'd like to give users the choice between AVX and no-AVX, because some cloud providers don't provide AVX-enabled VMs. + We allow users the choice between AVX and no-AVX, because some cloud providers don't provide AVX-enabled VMs. ## Development Environment @@ -60,20 +60,20 @@ Here we describe how to use above two images. We start from considering our dai Developers work on a computer, which is usually a laptop or desktop: -![](doc/paddle-development-environment.png) + or, they might rely on a more sophisticated box (like with GPUs): -![](doc/paddle-development-environment-gpu.png) + -A basic principle is that source code lies on the development computer (host), so that editing tools like Eclipse can parse the source code and support auto-completion. +A principle here is that source code lies on the development computer (host) so that editors like Eclipse can parse the source code to support auto-completion. ## Usages ### Build the Development Docker Image -The following commands check out the source code on the development computer (host) and build the development image `paddle:dev`: +The following commands check out the source code to the host and build the development image `paddle:dev`: ```bash git clone https://github.com/PaddlePaddle/Paddle paddle @@ -81,7 +81,7 @@ cd paddle docker build -t paddle:dev . ``` -The `docker build` command assumes that `Dockerfile` is in the root source tree. This is reasonable because this Dockerfile is this only on in our repo in this design. +The `docker build` command assumes that `Dockerfile` is in the root source tree. Note that in this design, this `Dockerfile` is this only one in our repo. ### Build PaddlePaddle from Source Code @@ -92,7 +92,7 @@ Given the development image `paddle:dev`, the following command builds PaddlePad docker run -v $PWD:/paddle -e "GPU=OFF" -e "AVX=ON" -e "TEST=ON" paddle:dev ``` -This command mounts the source directory on the host into `/paddle` in the container, so the default entrypoint of `paddle:dev`, `build.sh`, would build the source code with possible local changes. When it writes to `/paddle/build` in the container, it actually writes to `$PWD/build` on the host. +This command mounts the source directory on the host into `/paddle` in the container, so the default entry point of `paddle:dev`, `build.sh`, could build the source code with possible local changes. When it writes to `/paddle/build` in the container, it writes to `$PWD/build` on the host indeed. `build.sh` builds the following: @@ -109,11 +109,11 @@ The following command builds the production image: docker build -t paddle -f build/Dockerfile . ``` -This production image is minimal -- it includes binary `paddle`, the share library `libpaddle.so`, and Python runtime. +This production image is minimal -- it includes binary `paddle`, the shared library `libpaddle.so`, and Python runtime. ### Run PaddlePaddle Applications -Again the development happens on the host. Suppoose that we have a simple application program in `a.py`, we can test and run it using the production image: +Again the development happens on the host. Suppose that we have a simple application program in `a.py`, we can test and run it using the production image: ```bash docker run -it -v $PWD:/work paddle /work/a.py @@ -121,7 +121,7 @@ docker run -it -v $PWD:/work paddle /work/a.py But this works only if all dependencies of `a.py` are in the production image. If this is not the case, we need to build a new Docker image from the production image and with more dependencies installs. -### Build and Run PaddlePaddle Appications +### Build and Run PaddlePaddle Applications We need a Dockerfile in https://github.com/paddlepaddle/book that builds Docker image `paddlepaddle/book:`, basing on the PaddlePaddle production image: @@ -143,7 +143,7 @@ docker build -t book . ### Build and Run Distributed Applications -In our [API design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/api.md#distributed-training), we proposed an API that starts a distributed training job on a cluster. This API need to build a PaddlePaddle application into a Docekr image as above, and calls kubectl to run it on the cluster. This API might need to generate a Dockerfile look like above and call `docker build`. +In our [API design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/api.md#distributed-training), we proposed an API that starts a distributed training job on a cluster. This API need to build a PaddlePaddle application into a Docker image as above and calls kubectl to run it on the cluster. This API might need to generate a Dockerfile look like above and call `docker build`. Of course, we can manually build an application image and launch the job using the kubectl tool: From a710682a9f1b40e697b8e607d562d3c8b7c5bfb4 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Sun, 19 Mar 2017 21:50:22 -0700 Subject: [PATCH 57/62] update url for wmt14 dataset --- python/paddle/v2/dataset/wmt14.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/dataset/wmt14.py b/python/paddle/v2/dataset/wmt14.py index f5a16d5147..c686870a49 100644 --- a/python/paddle/v2/dataset/wmt14.py +++ b/python/paddle/v2/dataset/wmt14.py @@ -23,7 +23,7 @@ __all__ = ['train', 'test', 'build_dict'] URL_DEV_TEST = 'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz' MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' # this is a small set of data for test. The original data is too large and will be add later. -URL_TRAIN = 'http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz' +URL_TRAIN = 'http://paddlepaddle.cdn.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz' MD5_TRAIN = 'a755315dd01c2c35bde29a744ede23a6' START = "" From e6a2daa3eb8a443c778d8d25d38d84ae1c87bc36 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Mon, 20 Mar 2017 13:14:37 +0800 Subject: [PATCH 58/62] update sphinx cmake --- cmake/FindSphinx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake index 1c29cb22a3..f74cd4ff8c 100644 --- a/cmake/FindSphinx.cmake +++ b/cmake/FindSphinx.cmake @@ -72,7 +72,7 @@ function( Sphinx_add_target target_name builder conf cache source destination ) ${source} ${destination} COMMENT "Generating sphinx documentation: ${builder}" - COMMAND cd ${destination} && ln -s ./index_*.html index.html + COMMAND cd ${destination} && ln -sf ./index_*.html index.html ) set_property( From ae57b4e774c4af14a2d996a59d429b883ee6b544 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 20 Mar 2017 14:41:17 +0800 Subject: [PATCH 59/62] Change unittest's weight dimension to 1 --- python/paddle/v2/tests/test_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/tests/test_layer.py b/python/paddle/v2/tests/test_layer.py index 5ccd3d6913..dad4a5d949 100644 --- a/python/paddle/v2/tests/test_layer.py +++ b/python/paddle/v2/tests/test_layer.py @@ -22,7 +22,7 @@ import paddle.v2.networks as networks pixel = layer.data(name='pixel', type=data_type.dense_vector(128)) label = layer.data(name='label', type=data_type.integer_value(10)) -weight = layer.data(name='weight', type=data_type.dense_vector(10)) +weight = layer.data(name='weight', type=data_type.dense_vector(1)) score = layer.data(name='score', type=data_type.dense_vector(1)) hidden = layer.fc(input=pixel, From b3313f2b6d8b5b876de0774a74e3be6e40c41ea5 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Mon, 20 Mar 2017 15:48:23 +0800 Subject: [PATCH 60/62] Update CostLayer.h --- paddle/gserver/layers/CostLayer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/layers/CostLayer.h b/paddle/gserver/layers/CostLayer.h index c2b7803710..569a6840f0 100644 --- a/paddle/gserver/layers/CostLayer.h +++ b/paddle/gserver/layers/CostLayer.h @@ -164,8 +164,8 @@ public: * tasks. * \f[ * L = - * output - label / -1 < (output - label) < 1 / - * sign(output - label) / otherwise / + * (output - label)^2 * 0.5 / -1 < (output - label) < 1 / + * (output - label) - 0.5 / otherwise / * \f] */ class SmoothL1CostLayer : public CostLayer { From 979334e7a7d71e62b7c697954843b96a8682dfe5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 20 Mar 2017 15:57:14 +0800 Subject: [PATCH 61/62] Fix other unittest --- python/paddle/v2/tests/test_layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/tests/test_layer.py b/python/paddle/v2/tests/test_layer.py index dad4a5d949..89cc928dd7 100644 --- a/python/paddle/v2/tests/test_layer.py +++ b/python/paddle/v2/tests/test_layer.py @@ -23,6 +23,8 @@ import paddle.v2.networks as networks pixel = layer.data(name='pixel', type=data_type.dense_vector(128)) label = layer.data(name='label', type=data_type.integer_value(10)) weight = layer.data(name='weight', type=data_type.dense_vector(1)) +combine_weight = layer.data( + name='weight_combine', type=data_type.dense_vector(10)) score = layer.data(name='score', type=data_type.dense_vector(1)) hidden = layer.fc(input=pixel, @@ -81,7 +83,8 @@ class AggregateLayerTest(unittest.TestCase): class MathLayerTest(unittest.TestCase): def test_math_layer(self): addto = layer.addto(input=[pixel, pixel]) - linear_comb = layer.linear_comb(weights=weight, vectors=hidden, size=10) + linear_comb = layer.linear_comb( + weights=combine_weight, vectors=hidden, size=10) interpolation = layer.interpolation( input=[hidden, hidden], weight=score) bilinear = layer.bilinear_interp(input=conv, out_size_x=4, out_size_y=4) From f6f9fd7a663dd977899538d39be3cf4a619fab06 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Mon, 20 Mar 2017 16:21:58 +0800 Subject: [PATCH 62/62] remove unnecessary param in degradeSequence --- paddle/gserver/layers/SequencePoolLayer.cpp | 9 ++++----- paddle/parameter/Argument.cpp | 2 +- paddle/parameter/Argument.h | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/paddle/gserver/layers/SequencePoolLayer.cpp b/paddle/gserver/layers/SequencePoolLayer.cpp index 35260ca912..5807c42496 100644 --- a/paddle/gserver/layers/SequencePoolLayer.cpp +++ b/paddle/gserver/layers/SequencePoolLayer.cpp @@ -56,17 +56,16 @@ void SequencePoolLayer::forward(PassType passType) { CHECK_EQ(newBatchSize_, starts->getSize() - 1); resetOutput(newBatchSize_, dim); - if (type_) { - CHECK(input.subSequenceStartPositions) - << "when trans_type = seq, input must hasSubseq"; - } + /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, * thus, in this case, output_ has no sequenceStartPositions. * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this * case, we should compute the new sequenceStartPositions. */ if (type_) { - output_.degradeSequence(input, useGpu_); + CHECK(input.subSequenceStartPositions) + << "when trans_type = seq, input must hasSubseq"; + output_.degradeSequence(input); } } diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 7a343cca33..2f025f7290 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -583,7 +583,7 @@ void Argument::checkSubset() const { } } -void Argument::degradeSequence(const Argument& input, bool useGpu) { +void Argument::degradeSequence(const Argument& input) { CHECK_EQ(input.hasSubseq(), 1UL); size_t numSequences = input.getNumSequences(); size_t numSubSequences = input.getNumSubSequences(); diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 9ef44be0cb..129b7c4f8b 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -296,7 +296,7 @@ struct Argument { /* sequence has sub-sequence degrades to a sequence. */ - void degradeSequence(const Argument& input, bool useGpu); + void degradeSequence(const Argument& input); /** * @brief getValueString will return the argument's output in string. There