From 2669aea67fffa9a0db1616f8f15a9c6b05be4352 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 13:07:44 +0800 Subject: [PATCH 01/57] sgd_op support optimize SelectedRows --- paddle/fluid/framework/selected_rows.cc | 12 ++- paddle/fluid/framework/selected_rows.h | 13 ++- paddle/fluid/operators/sgd_op.cc | 24 +++-- paddle/fluid/operators/sgd_op.h | 123 +++++++++++++++--------- 4 files changed, 121 insertions(+), 51 deletions(-) diff --git a/paddle/fluid/framework/selected_rows.cc b/paddle/fluid/framework/selected_rows.cc index 504344e937..162b0355b5 100644 --- a/paddle/fluid/framework/selected_rows.cc +++ b/paddle/fluid/framework/selected_rows.cc @@ -1,8 +1,11 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -13,6 +16,13 @@ limitations under the License. */ namespace paddle { namespace framework { + +size_t GetIndex(const std::vector& rows, int64_t value) { + auto it = std::find(rows.begin(), rows.end(), value); + PADDLE_ENFORCE(it != rows.end(), "id should be in rows"); + return static_cast(std::distance(rows.begin(), it)); +} + void SerializeToStream(std::ostream& os, const SelectedRows& selected_rows, const platform::DeviceContext& dev_ctx) { { // the 1st field, uint32_t version diff --git a/paddle/fluid/framework/selected_rows.h b/paddle/fluid/framework/selected_rows.h index c9c2c1bb72..e461b7bdc1 100644 --- a/paddle/fluid/framework/selected_rows.h +++ b/paddle/fluid/framework/selected_rows.h @@ -1,8 +1,11 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -10,6 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once + +#include + #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/framework/tensor.h" @@ -59,6 +65,11 @@ class SelectedRows { int64_t height_; }; +/** + * Find the index of value in rows. + */ +size_t GetIndex(const std::vector& rows, int64_t value); + /* * Serialize/Desiralize SelectedRows to std::ostream * You can pass ofstream or ostringstream to serilize to file diff --git a/paddle/fluid/operators/sgd_op.cc b/paddle/fluid/operators/sgd_op.cc index d0aa2f9cba..9cdc5b3f1e 100644 --- a/paddle/fluid/operators/sgd_op.cc +++ b/paddle/fluid/operators/sgd_op.cc @@ -43,9 +43,19 @@ class SGDOp : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - return framework::OpKernelType( - framework::ToDataType(ctx.Input("Param")->type()), - ctx.GetPlace()); + auto* table_var = ctx.InputVar("Param"); + if (table_var->IsType()) { + return framework::OpKernelType( + framework::ToDataType(table_var->Get().type()), + ctx.device_context()); + } else if (table_var->IsType()) { + return framework::OpKernelType( + framework::ToDataType( + table_var->Get().value().type()), + ctx.device_context()); + } else { + PADDLE_THROW("Param should be LoDTensor or SelectedRows"); + } } }; @@ -53,10 +63,12 @@ class SGDOpMaker : public framework::OpProtoAndCheckerMaker { public: SGDOpMaker(OpProto* proto, OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Param", "(Tensor) Input parameter"); + AddInput("Param", "(Tensor or SelectedRows) Input parameter"); AddInput("LearningRate", "(Tensor) Learning rate of SGD"); - AddInput("Grad", "(Tensor) Input gradient"); - AddOutput("ParamOut", "(Tensor) Output parameter"); + AddInput("Grad", "(Tensor or SelectedRows) Input gradient"); + AddOutput("ParamOut", + "(Tensor or SelectedRows, same with Param) " + "Output parameter, should share the same memory with Param"); AddComment(R"DOC( SGD operator diff --git a/paddle/fluid/operators/sgd_op.h b/paddle/fluid/operators/sgd_op.h index 0ad8010794..237cd2f812 100644 --- a/paddle/fluid/operators/sgd_op.h +++ b/paddle/fluid/operators/sgd_op.h @@ -23,60 +23,97 @@ namespace operators { template class SGDOpKernel : public framework::OpKernel { public: - void Compute(const framework::ExecutionContext& ctx) const override { - auto* param = ctx.Input("Param"); - auto* param_out = ctx.Output("ParamOut"); - auto* learning_rate = ctx.Input("LearningRate"); - - auto* grad_var = ctx.InputVar("Grad"); - // Actually, all tensors are LoDTensor except SelectedRows. - if (grad_var->IsType()) { - param_out->mutable_data(ctx.GetPlace()); - auto* grad = ctx.Input("Grad"); - - auto p = framework::EigenVector::Flatten(*param); - auto g = framework::EigenVector::Flatten(*grad); - auto o = framework::EigenVector::Flatten(*param_out); - auto* lr = learning_rate->data(); - - o = p - lr[0] * g; - } else if (grad_var->IsType()) { - // TODO(qijun): In Sparse SGD operator, in-place update is enforced. - // This manual optimization brings difficulty to track data dependency. - // It's better to find a more elegant solution. - PADDLE_ENFORCE_EQ(param, param_out); - auto* grad = ctx.Input("Grad"); + void Compute(const framework::ExecutionContext &ctx) const override { + const auto *learning_rate = ctx.Input("LearningRate"); + + const auto *param_var = ctx.InputVar("Param"); + const auto *grad_var = ctx.InputVar("Grad"); + + if (param_var->IsType()) { + const auto *param = ctx.Input("Param"); + auto *param_out = ctx.Output("ParamOut"); + + // Actually, all tensors are LoDTensor except SelectedRows. + if (grad_var->IsType()) { + param_out->mutable_data(ctx.GetPlace()); + const auto *grad = ctx.Input("Grad"); + + auto p = framework::EigenVector::Flatten(*param); + auto g = framework::EigenVector::Flatten(*grad); + auto o = framework::EigenVector::Flatten(*param_out); + auto *lr = learning_rate->data(); + + o = p - lr[0] * g; + } else if (grad_var->IsType()) { + // TODO(qijun): In Sparse SGD operator, in-place update is enforced. + // This manual optimization brings difficulty to track data dependency. + // It's better to find a more elegant solution. + PADDLE_ENFORCE_EQ(param, param_out); + const auto *grad = ctx.Input("Grad"); + + // for distributed training, a sparse var may be empty, + // just skip updating. + if (grad->rows().size() == 0) { + return; + } + + auto grad_height = grad->height(); + auto out_dims = param_out->dims(); + PADDLE_ENFORCE_EQ(grad_height, out_dims[0]); + + auto &grad_value = grad->value(); + auto &grad_rows = grad->rows(); + + size_t grad_row_numel = grad_value.numel() / grad_rows.size(); + PADDLE_ENFORCE_EQ(grad_row_numel, param_out->numel() / grad_height); + + auto *grad_data = grad_value.data(); + auto *out_data = param_out->data(); + auto *lr = learning_rate->data(); + for (size_t i = 0; i < grad_rows.size(); i++) { + PADDLE_ENFORCE(grad_rows[i] < grad_height, + "Input rows index should less than height"); + for (int64_t j = 0; j < grad_row_numel; j++) { + out_data[grad_rows[i] * grad_row_numel + j] -= + lr[0] * grad_data[i * grad_row_numel + j]; + } + } + } else { + PADDLE_THROW("Unsupported Variable Type of Grad"); + } + } else if (param_var->IsType()) { + PADDLE_ENFORCE(grad_var->IsType(), + "when param " + "is SelectedRows, gradient should also be SelectedRows"); + const auto ¶m = param_var->Get(); + auto *param_out = ctx.Output("ParamOut"); + const auto &grad = grad_var->Get(); // for distributed training, a sparse var may be empty, // just skip updating. - if (grad->rows().size() == 0) { + if (grad.rows().size() == 0) { return; } - auto in_height = grad->height(); - auto out_dims = param_out->dims(); - PADDLE_ENFORCE_EQ(in_height, out_dims[0]); - - auto& in_value = grad->value(); - auto& in_rows = grad->rows(); + size_t param_row_width = param.value().numel() / param.rows().size(); + size_t grad_row_width = grad.value().numel() / grad.rows().size(); + PADDLE_ENFORCE_EQ(param_row_width, grad_row_width, + "param_row should have the same size with grad_row"); - int64_t in_row_numel = in_value.numel() / in_rows.size(); - PADDLE_ENFORCE_EQ(in_row_numel, param_out->numel() / in_height); - - auto* in_data = in_value.data(); - auto* out_data = param_out->data(); - auto* lr = learning_rate->data(); - for (size_t i = 0; i < in_rows.size(); i++) { - PADDLE_ENFORCE(in_rows[i] < in_height, + const auto *lr = learning_rate->data(); + const auto *grad_data = grad.value().data(); + auto *out_data = param_out->mutable_value()->data(); + for (size_t i = 0; i < grad.rows().size(); i++) { + PADDLE_ENFORCE(grad.rows()[i] < grad.height(), "Input rows index should less than height"); - for (int64_t j = 0; j < in_row_numel; j++) { - out_data[in_rows[i] * in_row_numel + j] -= - lr[0] * in_data[i * in_row_numel + j]; + size_t id_index = framework::GetIndex(param.rows(), grad.rows()[i]); + for (int64_t j = 0; j < grad_row_width; j++) { + out_data[id_index * grad_row_width + j] -= + lr[0] * grad_data[i * grad_row_width + j]; } } - } else { - PADDLE_THROW("Unsupported Variable Type of Grad"); + PADDLE_THROW("Unsupported Variable Type of Parameter"); } } }; From b154f311b920200a8ff714bb5d4a34fa2e689a27 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 14:05:29 +0800 Subject: [PATCH 02/57] init TestSGDOpOptimizeSelectedRows --- .../fluid/tests/unittests/test_sgd_op.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index c498b23db1..b3fd636114 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -97,5 +97,69 @@ class TestSparseSGDOp(unittest.TestCase): self.check_with_place(place) +class TestSGDOpOptimizeSelectedRows(unittest.TestCase): + def check_with_place(self, place): + scope = core.Scope() + + # create and initialize Grad Variable + height = 10 + rows = [0, 4, 7] + row_numel = 12 + + grad_selected_rows = scope.var('Grad').get_selected_rows() + grad_selected_rows.set_height(height) + grad_selected_rows.set_rows(rows) + np_array = np.ones((len(rows), row_numel)).astype("float32") + np_array[0, 0] = 2.0 + np_array[2, 8] = 4.0 + + grad_tensor = grad_selected_rows.get_tensor() + grad_tensor.set(np_array, place) + + # create and initialize Param Variable + param = scope.var('Param').get_tensor() + param_array = np.full((height, row_numel), 5.0).astype("float32") + param.set(param_array, place) + + # create and initialize LeraningRate Variable + lr = scope.var('LearningRate').get_tensor() + lr_array = np.full((1), 2.0).astype("float32") + lr.set(lr_array, place) + + # create and run sgd operator + sgd_op = Operator( + "sgd", + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate') + sgd_op.run(scope, place) + + # get and compare result + result_array = np.array(param) + + # rows[0] = 0, 5.0 - 2.0 * 2.0 + self.assertAlmostEqual(1.0, result_array[rows[0], 0]) + # rows[0] = 0, 5.0 - 2.0 * 1.0 + self.assertAlmostEqual(3.0, result_array[rows[0], 2]) + # 5.0 - 2.0 * 0.0 + self.assertAlmostEqual(5.0, result_array[1, 0]) + # rows[1] = 4, 5.0 - 2.0 * 1.0 + self.assertAlmostEqual(3.0, result_array[rows[1], 10]) + # 5.0 - 2.0 * 0.0 + self.assertAlmostEqual(5.0, result_array[5, 8]) + # rows[2] = 7, 5.0 - 2.0 * 1.0 + self.assertAlmostEqual(3.0, result_array[rows[2], 1]) + # rows[2] = 7, 5.0 - 2.0 * 4.0 + self.assertAlmostEqual(-3.0, result_array[rows[2], 8]) + + def test_sparse_sgd(self): + places = [core.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(core.CUDAPlace(0)) + for place in places: + self.check_with_place(place) + + if __name__ == "__main__": unittest.main() From 44d5f42a7e2400074790171f77920473aabb7eba Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Apr 2018 14:13:04 +0800 Subject: [PATCH 03/57] update reader --- .../reader/create_batch_reader_op.cc | 7 ++-- .../reader/create_double_buffer_reader_op.cc | 8 +++-- .../reader/create_multi_pass_reader_op.cc | 9 +++-- .../reader/create_shuffle_reader_op.cc | 8 +++-- python/paddle/fluid/framework.py | 15 ++++++++ python/paddle/fluid/layers/io.py | 35 ++++++++++++++----- 6 files changed, 64 insertions(+), 18 deletions(-) diff --git a/paddle/fluid/operators/reader/create_batch_reader_op.cc b/paddle/fluid/operators/reader/create_batch_reader_op.cc index 277f2856c0..04c5872bef 100644 --- a/paddle/fluid/operators/reader/create_batch_reader_op.cc +++ b/paddle/fluid/operators/reader/create_batch_reader_op.cc @@ -39,10 +39,13 @@ class CreateBatchReaderOp : public framework::OperatorBase { private: void RunImpl(const framework::Scope& scope, const platform::Place& dev_place) const override { - const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader")) - ->Get(); auto* out = scope.FindVar(Output("Out")) ->template GetMutable(); + if (out->Get() != nullptr) { + return; + } + const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader")) + ->Get(); out->Reset( new BatchReader(underlying_reader.Get(), Attr("batch_size"))); } diff --git a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc index f9a8058f2a..82bb668e9f 100644 --- a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc +++ b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/operators/reader/reader_op_registry.h" @@ -98,10 +97,13 @@ class CreateDoubleBufferReaderOp : public framework::OperatorBase { private: void RunImpl(const framework::Scope& scope, const platform::Place& dev_place) const override { - const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader")) - ->Get(); auto* out = scope.FindVar(Output("Out")) ->template GetMutable(); + if (out->Get() != nullptr) { + return; + } + const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader")) + ->Get(); auto place_str = Attr("place"); platform::Place place; diff --git a/paddle/fluid/operators/reader/create_multi_pass_reader_op.cc b/paddle/fluid/operators/reader/create_multi_pass_reader_op.cc index 47d9989bc8..b72ccc77a3 100644 --- a/paddle/fluid/operators/reader/create_multi_pass_reader_op.cc +++ b/paddle/fluid/operators/reader/create_multi_pass_reader_op.cc @@ -62,12 +62,15 @@ class CreateMultiPassReaderOp : public framework::OperatorBase { private: void RunImpl(const framework::Scope& scope, const platform::Place& dev_place) const override { + auto* out = detail::Ref(scope.FindVar(Output("Out"))) + .GetMutable(); + if (out->Get() != nullptr) { + return; + } const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader")) ->Get(); - auto& out = detail::Ref(scope.FindVar(Output("Out"))); int pass_num = Attr("pass_num"); - out.GetMutable()->Reset( - new MultiPassReader(underlying_reader.Get(), pass_num)); + out->Reset(new MultiPassReader(underlying_reader.Get(), pass_num)); } }; diff --git a/paddle/fluid/operators/reader/create_shuffle_reader_op.cc b/paddle/fluid/operators/reader/create_shuffle_reader_op.cc index 3a1f3805a0..b164ce232d 100644 --- a/paddle/fluid/operators/reader/create_shuffle_reader_op.cc +++ b/paddle/fluid/operators/reader/create_shuffle_reader_op.cc @@ -80,10 +80,14 @@ class CreateShuffleReaderOp : public framework::OperatorBase { private: void RunImpl(const framework::Scope& scope, const platform::Place& dev_place) const override { + auto* out = detail::Ref(scope.FindVar(Output("Out"))) + .GetMutable(); + if (out->Get() != nullptr) { + return; + } const auto& underlying_reader = scope.FindVar(Input("UnderlyingReader")) ->Get(); - auto& var = detail::Ref(scope.FindVar(Output("Out"))); - var.GetMutable()->Reset( + out->Reset( new ShuffleReader(underlying_reader.Get(), static_cast(Attr("buffer_size")))); } diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 3e78788f47..2f943457f5 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -640,6 +640,21 @@ class Operator(object): """ return self.desc.block_attr(name) + @property + def attrs(self): + """ + Get the attribute dict + Returns(dict): The Operator's attribute dict + """ + attr_names = self.attr_names + attr_map = {} + for n in attr_names: + if n == 'sub_block': + attr_map[n] = self.block_attr(n) + else: + attr_map[n] = self.attr(n) + return attr_map + class Block(object): def __init__(self, program, idx): diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index bd7e9c30fe..f6bd3c7d0a 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -255,7 +255,22 @@ def _copy_reader_var_(block, var): new_var.desc.set_shapes(var.desc.shapes()) new_var.desc.set_dtypes(var.desc.dtypes()) new_var.persistable = True - return monkey_patch_reader_methods(new_var) + return new_var + + +def _copy_reader_create_op_(block, op): + def _find_vars_(block, name_list): + res = {} + for n in name_list: + var = block.var(n) + res[n] = var + return res + + input_map = _find_vars_(block, op.input_names) + output_map = _find_vars_(block, op.output_names) + new_op = block.append_op( + type=op.type, inputs=input_map, outputs=output_map, attrs=op.attrs) + return new_op def open_recordio_file(filename, shapes, lod_levels, dtypes): @@ -283,8 +298,9 @@ def open_recordio_file(filename, shapes, lod_levels, dtypes): startup_var.desc.set_dtypes(dtypes) startup_var.persistable = True - return _copy_reader_var_(default_main_program().current_block(), - startup_var) + main_prog_var = _copy_reader_var_(default_main_program().current_block(), + startup_var) + return monkey_patch_reader_methods(main_prog_var) def open_files(filenames, thread_num, shapes, lod_levels, dtypes): @@ -313,22 +329,25 @@ def open_files(filenames, thread_num, shapes, lod_levels, dtypes): startup_var.desc.set_dtypes(dtypes) startup_var.persistable = True - return _copy_reader_var_(default_main_program().current_block(), - startup_var) + main_prog_var = _copy_reader_var_(default_main_program().current_block(), + startup_var) + return monkey_patch_reader_methods(main_prog_var) def __create_decorated_reader__(op_type, reader, attrs): var_name = unique_name(op_type) startup_blk = default_startup_program().current_block() startup_var = startup_blk.create_var(name=var_name) - startup_blk.append_op( + startop_op = startup_blk.append_op( type=op_type, inputs={'UnderlyingReader': reader}, outputs={'Out': [startup_var]}, attrs=attrs) startup_var.persistable = True - return _copy_reader_var_(default_main_program().current_block(), - startup_var) + main_prog_block = default_main_program().current_block() + main_prog_var = _copy_reader_var_(main_prog_block, startup_var) + _copy_reader_create_op_(main_prog_block, startop_op) + return monkey_patch_reader_methods(main_prog_var) def create_shuffle_reader(reader, buffer_size): From 649ae2700e94233fe58a2fcdc18a8ee59f40f335 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Apr 2018 14:48:14 +0800 Subject: [PATCH 04/57] fix bugs --- python/paddle/fluid/layers/io.py | 30 ++++++++++++------- .../tests/unittests/test_recordio_reader.py | 4 +-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index f6bd3c7d0a..fb5bb6bcbc 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -259,17 +259,27 @@ def _copy_reader_var_(block, var): def _copy_reader_create_op_(block, op): - def _find_vars_(block, name_list): - res = {} - for n in name_list: - var = block.var(n) - res[n] = var - return res - - input_map = _find_vars_(block, op.input_names) - output_map = _find_vars_(block, op.output_names) + input_param_names = op.input_names + new_input_map = {} + for param_name in input_param_names: + new_input_map[param_name] = [] + arg_names = op.input(param_name) + for arg_name in arg_names: + new_input_map[param_name].append(block.var(arg_name)) + + output_param_names = op.output_names + new_output_map = {} + for param_name in output_param_names: + new_output_map[param_name] = [] + arg_names = op.output(param_name) + for arg_name in arg_names: + new_output_map[param_name].append(block.var(arg_name)) + new_op = block.append_op( - type=op.type, inputs=input_map, outputs=output_map, attrs=op.attrs) + type=op.type, + inputs=new_input_map, + outputs=new_output_map, + attrs=op.attrs) return new_op diff --git a/python/paddle/fluid/tests/unittests/test_recordio_reader.py b/python/paddle/fluid/tests/unittests/test_recordio_reader.py index 640264d82f..24a0074d9b 100644 --- a/python/paddle/fluid/tests/unittests/test_recordio_reader.py +++ b/python/paddle/fluid/tests/unittests/test_recordio_reader.py @@ -15,8 +15,8 @@ import unittest import paddle.fluid as fluid -import paddle -import paddle.dataset.mnist as mnist +import paddle.v2 as paddle +import paddle.v2.dataset.mnist as mnist class TestRecordIO(unittest.TestCase): From abb7deee39f023f16d2afdad9e369105e7be0744 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 15:03:34 +0800 Subject: [PATCH 05/57] optimize test_lookup_table_op.py --- .../paddle/fluid/tests/unittests/test_lookup_table_op.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py index 3f739afd25..f8d5785fbf 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py @@ -115,18 +115,18 @@ class TestLookupTableWIsSelectedRows(OpTest): w_array = np.ones((len(rows), row_numel)).astype("float32") for i in range(len(rows)): w_array[i] *= i - ids_tensor = w_selected_rows.get_tensor() - ids_tensor.set(w_array, place) + w_tensor = w_selected_rows.get_tensor() + w_tensor.set(w_array, place) # create Out Variable - Out_tensor = scope.var('Out').get_tensor() + out_tensor = scope.var('Out').get_tensor() # create and run lookup_table operator lookup_table = Operator("lookup_table", W='W', Ids='Ids', Out='Out') lookup_table.run(scope, place) # get result from Out - result_array = np.array(Out_tensor) + result_array = np.array(out_tensor) # all(): return True if all elements of the iterable are true (or if the iterable is empty) for idx, row in enumerate(ids_array): assert (row[0] == result_array[idx]).all() From de2d82d6a9838b4ef192aaa2b1d57115d9250fa0 Mon Sep 17 00:00:00 2001 From: JiayiFeng Date: Tue, 3 Apr 2018 07:28:23 +0000 Subject: [PATCH 06/57] fix a bug --- python/paddle/fluid/framework.py | 3 +-- python/paddle/fluid/layers/io.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 2f943457f5..772ee6dab6 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -640,8 +640,7 @@ class Operator(object): """ return self.desc.block_attr(name) - @property - def attrs(self): + def all_attrs(self): """ Get the attribute dict Returns(dict): The Operator's attribute dict diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index fb5bb6bcbc..969398bda4 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -279,7 +279,7 @@ def _copy_reader_create_op_(block, op): type=op.type, inputs=new_input_map, outputs=new_output_map, - attrs=op.attrs) + attrs=op.all_attrs()) return new_op From a994327fb158ee7692238fef6e29c7f1ed6dc1ca Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 15:39:46 +0800 Subject: [PATCH 07/57] add TestSGDOpOptimizeSelectedRows --- .../fluid/tests/unittests/test_sgd_op.py | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index b3fd636114..191f21725c 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -101,31 +101,50 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): def check_with_place(self, place): scope = core.Scope() + row_width = 12 # create and initialize Grad Variable - height = 10 - rows = [0, 4, 7] - row_numel = 12 + grad_height = 10 + grad_rows = [0, 4, 7] grad_selected_rows = scope.var('Grad').get_selected_rows() - grad_selected_rows.set_height(height) - grad_selected_rows.set_rows(rows) - np_array = np.ones((len(rows), row_numel)).astype("float32") - np_array[0, 0] = 2.0 - np_array[2, 8] = 4.0 + grad_selected_rows.set_height(grad_height) + grad_selected_rows.set_rows(grad_rows) + grad_array = np.ones((len(grad_rows), row_width)).astype("float32") + grad_array[0, 0] = 2.0 + grad_array[2, 8] = 4.0 grad_tensor = grad_selected_rows.get_tensor() - grad_tensor.set(np_array, place) + grad_tensor.set(grad_array, place) # create and initialize Param Variable - param = scope.var('Param').get_tensor() - param_array = np.full((height, row_numel), 5.0).astype("float32") - param.set(param_array, place) + # create and initialize W Variable + param_rows = [0, 1, 2, 3, 4, 5, 6, 7] + + # init Param + w_selected_rows = scope.var('Param').get_selected_rows() + w_selected_rows.set_height(len(param_rows)) + w_selected_rows.set_rows(param_rows) + w_array = np.ones((len(param_rows), row_width)).astype("float32") + for i in range(len(param_rows)): + w_array[i] *= i + w_tensor = w_selected_rows.get_tensor() + w_tensor.set(w_array, place) + + w_before_optimize = np.array(w_tensor) + print(w_before_optimize) # create and initialize LeraningRate Variable + lr_value = 0.1 lr = scope.var('LearningRate').get_tensor() - lr_array = np.full((1), 2.0).astype("float32") + lr_array = np.full((1), lr_value).astype("float32") lr.set(lr_array, place) + # optimize with Python + w_after_optimize = np.copy(w_before_optimize) + for index, id in enumerate(grad_rows): + w_after_optimize[id] = w_before_optimize[ + id] - lr_value * grad_array[index] + # create and run sgd operator sgd_op = Operator( "sgd", @@ -136,22 +155,8 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): sgd_op.run(scope, place) # get and compare result - result_array = np.array(param) - - # rows[0] = 0, 5.0 - 2.0 * 2.0 - self.assertAlmostEqual(1.0, result_array[rows[0], 0]) - # rows[0] = 0, 5.0 - 2.0 * 1.0 - self.assertAlmostEqual(3.0, result_array[rows[0], 2]) - # 5.0 - 2.0 * 0.0 - self.assertAlmostEqual(5.0, result_array[1, 0]) - # rows[1] = 4, 5.0 - 2.0 * 1.0 - self.assertAlmostEqual(3.0, result_array[rows[1], 10]) - # 5.0 - 2.0 * 0.0 - self.assertAlmostEqual(5.0, result_array[5, 8]) - # rows[2] = 7, 5.0 - 2.0 * 1.0 - self.assertAlmostEqual(3.0, result_array[rows[2], 1]) - # rows[2] = 7, 5.0 - 2.0 * 4.0 - self.assertAlmostEqual(-3.0, result_array[rows[2], 8]) + result_array = np.array(w_tensor) + assert (result_array == w_after_optimize).all() def test_sparse_sgd(self): places = [core.CPUPlace()] From af1d3f5bc085923a3000cb25185a24169694ee67 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 15:57:30 +0800 Subject: [PATCH 08/57] remove print --- python/paddle/fluid/tests/unittests/test_sgd_op.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index 191f21725c..4761b43548 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -131,7 +131,6 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): w_tensor.set(w_array, place) w_before_optimize = np.array(w_tensor) - print(w_before_optimize) # create and initialize LeraningRate Variable lr_value = 0.1 From 31e8d807d9dfa8682bdc8e5a0fc80fa34b577171 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 18:52:12 +0800 Subject: [PATCH 09/57] optimize code --- paddle/fluid/framework/selected_rows.cc | 6 ------ paddle/fluid/framework/selected_rows.h | 14 +++++++++----- paddle/fluid/operators/lookup_table_op.h | 18 ++++++++---------- paddle/fluid/operators/sgd_op.h | 2 +- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/paddle/fluid/framework/selected_rows.cc b/paddle/fluid/framework/selected_rows.cc index 162b0355b5..d9d6b7dd67 100644 --- a/paddle/fluid/framework/selected_rows.cc +++ b/paddle/fluid/framework/selected_rows.cc @@ -17,12 +17,6 @@ limitations under the License. */ namespace paddle { namespace framework { -size_t GetIndex(const std::vector& rows, int64_t value) { - auto it = std::find(rows.begin(), rows.end(), value); - PADDLE_ENFORCE(it != rows.end(), "id should be in rows"); - return static_cast(std::distance(rows.begin(), it)); -} - void SerializeToStream(std::ostream& os, const SelectedRows& selected_rows, const platform::DeviceContext& dev_ctx) { { // the 1st field, uint32_t version diff --git a/paddle/fluid/framework/selected_rows.h b/paddle/fluid/framework/selected_rows.h index 2979d40870..8e2d9470d3 100644 --- a/paddle/fluid/framework/selected_rows.h +++ b/paddle/fluid/framework/selected_rows.h @@ -50,6 +50,15 @@ class SelectedRows { void set_rows(const Vector& rows) { rows_ = rows; } + /** + * get the index of id in rows + */ + int64_t index(int64_t id) const { + auto it = std::find(rows_.begin(), rows_.end(), id); + PADDLE_ENFORCE(it != rows_.end(), "id should be in rows"); + return static_cast(std::distance(rows_.begin(), it)); + } + DDim GetCompleteDims() const { std::vector dims = vectorize(value_->dims()); dims[0] = height_; @@ -65,11 +74,6 @@ class SelectedRows { int64_t height_; }; -/** - * Find the index of value in rows. - */ -size_t GetIndex(const std::vector& rows, int64_t value); - /* * Serialize/Desiralize SelectedRows to std::ostream * You can pass ofstream or ostringstream to serilize to file diff --git a/paddle/fluid/operators/lookup_table_op.h b/paddle/fluid/operators/lookup_table_op.h index fff5edda62..cb088c267b 100644 --- a/paddle/fluid/operators/lookup_table_op.h +++ b/paddle/fluid/operators/lookup_table_op.h @@ -30,13 +30,7 @@ using LoDTensor = framework::LoDTensor; using SelectedRows = framework::SelectedRows; using DDim = framework::DDim; -static constexpr int64_t kNoPadding = -1; - -inline size_t getIndex(const std::vector &rows, int64_t value) { - auto it = std::find(rows.begin(), rows.end(), value); - PADDLE_ENFORCE(it != rows.end(), "id should be in rows"); - return static_cast(std::distance(rows.begin(), it)); -} +constexpr int64_t kNoPadding = -1; template class LookupTableKernel : public framework::OpKernel { @@ -55,7 +49,9 @@ class LookupTableKernel : public framework::OpKernel { auto *table_t = context.Input("W"); table_dim = table_t->value().dims(); } else { - PADDLE_THROW("table only support LoDTensor and SelectedRows"); + PADDLE_THROW( + "The parameter W of a LookupTable " + "must be either LoDTensor or SelectedRows"); } int64_t *ids; @@ -107,7 +103,7 @@ class LookupTableKernel : public framework::OpKernel { memset(output + i * row_width, 0, row_width * sizeof(T)); } else { PADDLE_ENFORCE_GE(ids[i], 0); - auto id_index = getIndex(table_t.rows(), ids[i]); + auto id_index = table_t.index(ids[i]); memcpy(output + i * row_width, table + id_index * row_width, row_width * sizeof(T)); } @@ -128,7 +124,9 @@ class LookupTableGradKernel : public framework::OpKernel { auto *table_t = context.Input("W"); table_dim = table_t->value().dims(); } else { - PADDLE_THROW("table only support LoDTensor and SelectedRows"); + PADDLE_THROW( + "The parameter W of a LookupTable " + "must be either LoDTensor or SelectedRows"); } bool is_sparse = context.Attr("is_sparse"); diff --git a/paddle/fluid/operators/sgd_op.h b/paddle/fluid/operators/sgd_op.h index 237cd2f812..8d2bdf7590 100644 --- a/paddle/fluid/operators/sgd_op.h +++ b/paddle/fluid/operators/sgd_op.h @@ -106,7 +106,7 @@ class SGDOpKernel : public framework::OpKernel { for (size_t i = 0; i < grad.rows().size(); i++) { PADDLE_ENFORCE(grad.rows()[i] < grad.height(), "Input rows index should less than height"); - size_t id_index = framework::GetIndex(param.rows(), grad.rows()[i]); + int64_t id_index = param.index(grad.rows()[i]); for (int64_t j = 0; j < grad_row_width; j++) { out_data[id_index * grad_row_width + j] -= lr[0] * grad_data[i * grad_row_width + j]; From cbfec1f7d614835b806e0f37adb1cfcd8b15b444 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Apr 2018 19:56:19 +0800 Subject: [PATCH 10/57] diable test of sparse_parameter_sgd on GPU --- python/paddle/fluid/tests/unittests/test_sgd_op.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index 4761b43548..3126293f9d 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -157,10 +157,9 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): result_array = np.array(w_tensor) assert (result_array == w_after_optimize).all() - def test_sparse_sgd(self): + def test_sparse_parameter_sgd(self): places = [core.CPUPlace()] - if core.is_compiled_with_cuda(): - places.append(core.CUDAPlace(0)) + # do not support GPU kernel currently for place in places: self.check_with_place(place) From 09b53c086de7011d445758f4dafa875d2d14ed4c Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 3 Apr 2018 20:51:16 +0800 Subject: [PATCH 11/57] add remove_var from c++ end --- paddle/fluid/framework/block_desc.h | 3 ++ paddle/fluid/pybind/protobuf.cc | 18 +++++--- .../tests/unittests/test_protobuf_descs.py | 42 +++++++++---------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/framework/block_desc.h b/paddle/fluid/framework/block_desc.h index 468423e0e8..873969b2a8 100644 --- a/paddle/fluid/framework/block_desc.h +++ b/paddle/fluid/framework/block_desc.h @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include +#include #include #include @@ -96,6 +97,8 @@ class BlockDesc { */ void RemoveOp(size_t s, size_t e); + void RemoveVar(const std::string &name) { vars_.erase(name); } + std::vector AllOps() const; size_t OpSize() const { return ops_.size(); } diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index 45a64f4384..985984983a 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -15,6 +15,8 @@ limitations under the License. */ #include "paddle/fluid/pybind/protobuf.h" #include #include +#include +#include #include "paddle/fluid/framework/backward.h" #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/op_desc.h" @@ -98,7 +100,7 @@ namespace pybind { using namespace paddle::framework; // NOLINT template -static py::bytes SerializeMessage(T &self) { +static py::bytes SerializeMessage(T &self) { // NOLINT // Check IsInitialized in Python std::string retv; PADDLE_ENFORCE(self.Proto()->SerializePartialToString(&retv), @@ -107,7 +109,7 @@ static py::bytes SerializeMessage(T &self) { } // Bind Methods -void BindProgramDesc(py::module &m) { +void BindProgramDesc(py::module &m) { // NOLINT py::class_(m, "ProgramDesc", "") .def(py::init<>()) .def("__init__", @@ -151,7 +153,7 @@ void BindProgramDesc(py::module &m) { }); } -void BindBlockDesc(py::module &m) { +void BindBlockDesc(py::module &m) { // NOLINT py::class_(m, "BlockDesc", "") .def_property_readonly("id", &BlockDesc::ID) .def_property_readonly("parent", &BlockDesc::Parent) @@ -200,13 +202,19 @@ void BindBlockDesc(py::module &m) { return self.FindVarRecursive(name); }, py::return_value_policy::reference) + .def("remove_var", + [](BlockDesc &self, py::bytes byte_name) { + std::string name = byte_name; + return self.RemoveVar(name); + }, + py::return_value_policy::reference) .def("all_vars", &BlockDesc::AllVars, py::return_value_policy::reference) .def("op_size", &BlockDesc::OpSize) .def("op", &BlockDesc::Op, py::return_value_policy::reference) .def("serialize_to_string", SerializeMessage); } -void BindVarDsec(py::module &m) { +void BindVarDsec(py::module &m) { // NOLINT py::class_ var_desc(m, "VarDesc", ""); var_desc .def("name", @@ -257,7 +265,7 @@ void BindVarDsec(py::module &m) { .value("RAW", proto::VarType::RAW); } -void BindOpDesc(py::module &m) { +void BindOpDesc(py::module &m) { // NOLINT py::enum_(m, "AttrType", "") .value("INT", proto::AttrType::INT) .value("INTS", proto::AttrType::INTS) diff --git a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py index e4cf4a8bce..f98a8bbc68 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py @@ -19,9 +19,9 @@ from paddle.fluid.framework import Program class TestOpDesc(unittest.TestCase): def test_op_desc(self): - prog = core.ProgramDesc() - self.assertIsNotNone(prog) - block = prog.block(0) + program_desc = core.ProgramDesc() + self.assertIsNotNone(program_desc) + block = program_desc.block(0) self.assertIsNotNone(block) op = block.append_op() self.assertIsNotNone(op) @@ -67,7 +67,7 @@ class TestOpDesc(unittest.TestCase): self.assertEqual(8, len(op.attr_names())) - op.set_block_attr("block_attr", prog.block(0)) + op.set_block_attr("block_attr", program_desc.block(0)) self.assertEqual(0, op.block_attr("block_attr")) mul_op = block.append_op() @@ -88,20 +88,20 @@ class TestProgramDesc(unittest.TestCase): del program_desc def test_append_block(self): - prog_desc = core.ProgramDesc() - self.assertIsNotNone(prog_desc) - block_root = prog_desc.block(0) + program_desc = core.ProgramDesc() + self.assertIsNotNone(program_desc) + block_root = program_desc.block(0) self.assertIsNotNone(block_root) self.assertEqual(block_root.id, 0) - block1 = prog_desc.append_block(block_root) - block2 = prog_desc.append_block(block1) + block1 = program_desc.append_block(block_root) + block2 = program_desc.append_block(block1) self.assertIsNotNone(block1) self.assertEqual(block1.id, block2.parent) self.assertEqual(block_root.id, block1.parent) - block3 = prog_desc.append_block(block_root) + block3 = program_desc.append_block(block_root) self.assertEqual(block3.parent, block_root.id) - self.assertEqual(prog_desc.block(1).id, 1) - self.assertEqual(4, prog_desc.num_blocks()) + self.assertEqual(program_desc.block(1).id, 1) + self.assertEqual(4, program_desc.num_blocks()) class TestVarDesc(unittest.TestCase): @@ -162,9 +162,9 @@ class TestVarDesc(unittest.TestCase): class TestBlockDesc(unittest.TestCase): def test_add_var(self): - prog = core.ProgramDesc() - self.assertIsNotNone(prog) - block = prog.block(0) + program_desc = core.ProgramDesc() + self.assertIsNotNone(program_desc) + block = program_desc.block(0) self.assertIsNotNone(block) var1 = block.var("var1") var2 = block.var("var2") @@ -175,9 +175,9 @@ class TestBlockDesc(unittest.TestCase): self.assertEqual(var2_re, var2) def test_add_op(self): - prog = core.ProgramDesc() - self.assertIsNotNone(prog) - block = prog.block(0) + program_desc = core.ProgramDesc() + self.assertIsNotNone(program_desc) + block = program_desc.block(0) self.assertIsNotNone(block) op1 = block.append_op() op2 = block.append_op() @@ -189,9 +189,9 @@ class TestBlockDesc(unittest.TestCase): def test_remove_op(self): program = Program() - prog = program.desc - self.assertIsNotNone(prog) - block = prog.block(0) + program_desc = program.desc + self.assertIsNotNone(program_desc) + block = program_desc.block(0) self.assertIsNotNone(block) op0 = block.append_op() From 187ba08789619a5a92b67d457dbed165c1b086af Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Tue, 3 Apr 2018 17:50:55 -0700 Subject: [PATCH 12/57] enable tensor core for conv cudnn --- paddle/fluid/operators/conv_cudnn_op.cu.cc | 22 ++++++++++++++++++++++ paddle/fluid/platform/cudnn_helper.h | 4 +++- paddle/fluid/platform/dynload/cudnn.h | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/conv_cudnn_op.cu.cc b/paddle/fluid/operators/conv_cudnn_op.cu.cc index a32aba4c1f..c70e3cc3c9 100644 --- a/paddle/fluid/operators/conv_cudnn_op.cu.cc +++ b/paddle/fluid/operators/conv_cudnn_op.cu.cc @@ -128,10 +128,32 @@ class CUDNNConvOpKernel : public framework::OpKernel { handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc, cudnn_output_desc, CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, workspace_size_limit, &algo)); + +#if CUDA_VERSION >= 9000 && CUDNN_VERSION_MIN(7, 0, 1) + // Tensor core is supported since the volta GPU and + // is only enabled when input and filter data are float16 + if (dev_ctx.GetComputeCapability() >= 70 && + std::type_index(typeid(T)) == + std::type_index(typeid(platform::float16))) { + PADDLE_ENFORCE(platform::dynload::cudnnSetConvolutionMathType( + cudnn_conv_desc, CUDNN_TENSOR_OP_MATH)); + // Currently tensor core is only enabled using this algo + algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM; + } else { + PADDLE_ENFORCE(platform::dynload::cudnnSetConvolutionMathType( + cudnn_conv_desc, CUDNN_DEFAULT_MATH)); + } +#endif + // get workspace size able to allocate PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardWorkspaceSize( handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc, cudnn_output_desc, algo, &workspace_size_in_bytes)); + // It is possible for float16 on Volta GPU to allocate more memory than + // the limit because the algo is overrided to use tensor core. + PADDLE_ENFORCE_LE(workspace_size_in_bytes, workspace_size_limit, + "workspace_size to be allocated exceeds the limit"); + // Allocate on GPU memory platform::CUDAPlace gpu = boost::get(ctx.GetPlace()); cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes); diff --git a/paddle/fluid/platform/cudnn_helper.h b/paddle/fluid/platform/cudnn_helper.h index 7c604e14eb..c0d399d078 100644 --- a/paddle/fluid/platform/cudnn_helper.h +++ b/paddle/fluid/platform/cudnn_helper.h @@ -257,9 +257,11 @@ class ScopedConvolutionDescriptor { } #endif + cudnnDataType_t compute_type = + (type == CUDNN_DATA_DOUBLE) ? CUDNN_DATA_DOUBLE : CUDNN_DATA_FLOAT; PADDLE_ENFORCE(dynload::cudnnSetConvolutionNdDescriptor( desc_, pads.size(), pads.data(), strides.data(), dilations.data(), - CUDNN_CROSS_CORRELATION, type)); + CUDNN_CROSS_CORRELATION, compute_type)); return desc_; } diff --git a/paddle/fluid/platform/dynload/cudnn.h b/paddle/fluid/platform/dynload/cudnn.h index 81acc445bd..3f91bd2fe4 100644 --- a/paddle/fluid/platform/dynload/cudnn.h +++ b/paddle/fluid/platform/dynload/cudnn.h @@ -16,7 +16,6 @@ limitations under the License. */ #include #include -#include #include "paddle/fluid/platform/dynload/dynamic_loader.h" namespace paddle { @@ -140,7 +139,8 @@ CUDNN_DNN_ROUTINE_EACH_R5(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #if CUDNN_VERSION >= 7001 #define CUDNN_DNN_ROUTINE_EACH_R7(__macro) \ - __macro(cudnnSetConvolutionGroupCount); + __macro(cudnnSetConvolutionGroupCount); \ + __macro(cudnnSetConvolutionMathType); CUDNN_DNN_ROUTINE_EACH_R7(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #endif From 9ba36604d845fedc4f82fad637a5f056648301ee Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Tue, 3 Apr 2018 17:58:50 -0700 Subject: [PATCH 13/57] fix cpplint error --- paddle/fluid/platform/dynload/cudnn.h | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/platform/dynload/cudnn.h b/paddle/fluid/platform/dynload/cudnn.h index 3f91bd2fe4..49a54d8478 100644 --- a/paddle/fluid/platform/dynload/cudnn.h +++ b/paddle/fluid/platform/dynload/cudnn.h @@ -16,6 +16,7 @@ limitations under the License. */ #include #include +#include // NOLINT #include "paddle/fluid/platform/dynload/dynamic_loader.h" namespace paddle { From 4bbfa9eccb726071f55c14263fd62812cc3f746b Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Tue, 3 Apr 2018 23:14:13 -0700 Subject: [PATCH 14/57] Add feed to ParallelExecutor --- doc/fluid/api/gen_doc.sh | 0 paddle/fluid/framework/lod_tensor.h | 1 + paddle/fluid/framework/parallel_executor.cc | 21 +++++- paddle/fluid/framework/parallel_executor.h | 6 +- python/paddle/fluid/parallel_executor.py | 38 +++++++--- .../tests/unittests/test_parallel_executor.py | 72 ++++++++++++------- 6 files changed, 99 insertions(+), 39 deletions(-) mode change 100755 => 100644 doc/fluid/api/gen_doc.sh diff --git a/doc/fluid/api/gen_doc.sh b/doc/fluid/api/gen_doc.sh old mode 100755 new mode 100644 diff --git a/paddle/fluid/framework/lod_tensor.h b/paddle/fluid/framework/lod_tensor.h index dee505fee0..4f130d2659 100644 --- a/paddle/fluid/framework/lod_tensor.h +++ b/paddle/fluid/framework/lod_tensor.h @@ -142,6 +142,7 @@ class LoDTensor : public Tensor { return (lod_)[level].size() - 1; } + // Split LoDTensor and copy to each place specified in places. std::vector SplitLoDTensor( const std::vector places) const; diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 1788514324..7be93fa600 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -150,13 +150,30 @@ void ParallelExecutor::BCastParamsToGPUs( #endif } -void ParallelExecutor::Run(const std::vector &fetch_tensors, - const std::string &fetched_var_name) { +void ParallelExecutor::Run( + const std::vector &fetch_tensors, + const std::string &fetched_var_name, + const std::unordered_map &feed_tensors) { platform::RecordBlock b(0); + SplitTensorToPlaces(feed_tensors); auto fetch_data = member_->executor_->Run(fetch_tensors); *member_->global_scope_->Var(fetched_var_name)->GetMutable() = fetch_data; } +void ParallelExecutor::SplitTensorToPlaces( + const std::unordered_map &feed_tensors) { + for (auto it : feed_tensors) { + auto lod_tensors = it.second.SplitLoDTensor(member_->places_); + for (size_t j = 0; j < member_->places_.size(); ++j) { + // TODO(panxy0718): Do I need to delete this var? + member_->local_scopes_[j] + ->Var(it.first) + ->GetMutable() + ->ShareDataWith(lod_tensors[j]); + } + } +} + } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/parallel_executor.h b/paddle/fluid/framework/parallel_executor.h index 964b476234..c7c58b2b80 100644 --- a/paddle/fluid/framework/parallel_executor.h +++ b/paddle/fluid/framework/parallel_executor.h @@ -42,9 +42,13 @@ class ParallelExecutor { bool allow_op_delay); void Run(const std::vector& fetch_tensors, - const std::string& fetched_var_name = "fetched_var"); + const std::string& fetched_var_name, + const std::unordered_map& feed_tensors); private: + void SplitTensorToPlaces( + const std::unordered_map& feed_tensors); + ParallelExecutorPrivate* member_; void BCastParamsToGPUs(const ProgramDesc& startup_program) const; diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index a2c830b3c9..4153049c05 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -26,25 +26,29 @@ class ParallelExecutor(object): use_cuda, num_threads=None, allow_op_delay=False): - places = [] + self._places = [] + self._act_places = [] if use_cuda: for i in xrange(core.get_cuda_device_count()): p = core.Place() - p.set_place(core.CUDAPlace(i)) - places.append(p) + self._act_places.append(core.CUDAPlace(i)) + p.set_place(self._act_places[-1]) + self._places.append(p) else: for i in xrange(multiprocessing.cpu_count()): p = core.Place() - p.set_place(core.CPUPlace()) - places.append(p) + self._act_places.append(core.CPUPlace(i)) + p.set_place(self._act_places[-1]) + self._places.append(p) + assert self._places, "no place for execution" if num_threads is None: if use_cuda: # Experiments on se-resnext shows that too many threads hurt # performance. Worth tunning for other models in the future. - num_threads = len(places) + num_threads = len(self._places) else: - min(len(places) * 2, multiprocessing.cpu_count()) + min(len(self._places) * 2, multiprocessing.cpu_count()) startup = framework.default_startup_program() main = framework.default_main_program() @@ -53,7 +57,7 @@ class ParallelExecutor(object): self.executor = core.ParallelExecutor( num_threads, True if use_cuda else False, # use_event - places, + self._places, set([ p.name for p in main.global_block().iter_parameters() if not p.stop_gradient @@ -65,8 +69,22 @@ class ParallelExecutor(object): allow_op_delay) self.scope = scope - def run(self, fetch_list): + def run(self, fetch_list, feed_dict={}): + """ + :param fetch_list: A list of variable names that will be fetched. + :param feed_dict: A dict mapping for feed variable name to LoDTensor + or numpy array. + :return: fetched value list. + """ + feed_tensor_dict = {} + for i, feed_name in enumerate(feed_dict): + feed_tensor = feed_dict[feed_name] + if not isinstance(feed_tensor, core.LoDTensor): + feed_tensor = core.LoDTensor() + feed_tensor.set(feed_dict[feed_name], self._act_places[0]) + feed_tensor_dict[feed_name] = feed_tensor + fetch_var_name = '@FETCHED_VAR_NAME@' - self.executor.run(fetch_list, fetch_var_name) + self.executor.run(fetch_list, fetch_var_name, feed_tensor_dict) arr = self.scope.find_var(fetch_var_name).get_lod_tensor_array() return [arr[i] for i in range(len(arr))] diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor.py b/python/paddle/fluid/tests/unittests/test_parallel_executor.py index a79e4b3e18..0cbef82e33 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor.py @@ -21,13 +21,17 @@ import paddle.dataset.mnist as mnist import paddle.dataset.wmt16 as wmt16 -def simple_fc_net(): - reader = fluid.layers.open_recordio_file( - filename='./mnist.recordio', - shapes=[[-1, 784], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64']) - img, label = fluid.layers.read_file(reader) +def simple_fc_net(use_feed): + if use_feed: + img = fluid.layers.data(name='image', shape=[784], dtype='float32') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + else: + reader = fluid.layers.open_recordio_file( + filename='./mnist.recordio', + shapes=[[-1, 784], [-1, 1]], + lod_levels=[0, 0], + dtypes=['float32', 'int64']) + img, label = fluid.layers.read_file(reader) hidden = img for _ in xrange(4): hidden = fluid.layers.fc( @@ -42,13 +46,18 @@ def simple_fc_net(): return loss -def fc_with_batchnorm(): - reader = fluid.layers.open_recordio_file( - filename='./mnist.recordio', - shapes=[[-1, 784], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64']) - img, label = fluid.layers.read_file(reader) +def fc_with_batchnorm(use_feed): + if use_feed: + img = fluid.layers.data(name='image', shape=[784], dtype='float32') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + else: + reader = fluid.layers.open_recordio_file( + filename='./mnist.recordio', + shapes=[[-1, 784], [-1, 1]], + lod_levels=[0, 0], + dtypes=['float32', 'int64']) + img, label = fluid.layers.read_file(reader) + hidden = img for _ in xrange(1): hidden = fluid.layers.fc( @@ -135,7 +144,9 @@ def bottleneck_block(input, num_filters, stride, cardinality, reduction_ratio): return fluid.layers.elementwise_add(x=short, y=scale, act='relu') -def SE_ResNeXt152Small(batch_size=2): +def SE_ResNeXt152Small(batch_size=2, use_feed=False): + assert not use_feed, "SE_ResNeXt doesn't support feed yet" + img = fluid.layers.fill_constant( shape=[batch_size, 3, 224, 224], dtype='float32', value=0.0) label = fluid.layers.fill_constant( @@ -185,30 +196,28 @@ class TestParallelExecutorBase(unittest.TestCase): memory_opt=True, iter=10, batch_size=None, - allow_op_delay=False): + allow_op_delay=False, + feed_dict={}): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - loss = method() + loss = method(use_feed=len(feed_dict) > 0) adam = fluid.optimizer.Adam() adam.minimize(loss) if memory_opt: fluid.memory_optimize(main) - exe = fluid.ParallelExecutor( - loss_name=loss.name, - use_cuda=True, - allow_op_delay=allow_op_delay) + exe = fluid.ParallelExecutor(loss_name=loss.name, use_cuda=True) if batch_size is not None: batch_size *= fluid.core.get_cuda_device_count() begin = time.time() - first_loss, = exe.run([loss.name]) + first_loss, = exe.run([loss.name], feed_dict=feed_dict) first_loss = numpy.array(first_loss) for i in xrange(iter): - exe.run([]) + exe.run([], feed_dict=feed_dict) - last_loss, = exe.run([loss.name]) + last_loss, = exe.run([loss.name], feed_dict=feed_dict) end = time.time() if batch_size is not None: @@ -242,9 +251,19 @@ class TestMNIST(TestParallelExecutorBase): self.check_network_convergence(simple_fc_net) self.check_network_convergence(simple_fc_net, allow_op_delay=True) + img = numpy.zeros(shape=[32, 784], dtype='float32') + label = numpy.ones(shape=[32, 1], dtype='int64') + self.check_network_convergence( + simple_fc_net, feed_dict={"image": img, + "label": label}) + def test_batchnorm_fc(self): self.check_network_convergence(fc_with_batchnorm) - self.check_network_convergence(fc_with_batchnorm, allow_op_delay=True) + img = numpy.zeros(shape=[32, 784], dtype='float32') + label = numpy.ones(shape=[32, 1], dtype='int64') + self.check_network_convergence( + fc_with_batchnorm, feed_dict={"image": img, + "label": label}) class TestResnet(TestParallelExecutorBase): @@ -400,7 +419,8 @@ def prepare_batch_input(insts, src_pad_idx, trg_pad_idx, n_head): import transformer_model -def transformer(): +def transformer(use_feed): + assert not use_feed, "transfomer doesn't support feed yet" return transformer_model.transformer( ModelHyperParams.src_vocab_size + 1, ModelHyperParams.trg_vocab_size + 1, ModelHyperParams.max_length + 1, From 3cd9e450ac1c9562823f11feee4212af9bde7682 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 4 Apr 2018 14:23:56 +0800 Subject: [PATCH 15/57] fix lint --- paddle/fluid/operators/reader/create_double_buffer_reader_op.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc index 82bb668e9f..f484dbba53 100644 --- a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc +++ b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include //NOLINT #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/operators/reader/reader_op_registry.h" From a122dcccd5538ae0198d727047b150530f0d8133 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 4 Apr 2018 14:26:01 +0800 Subject: [PATCH 16/57] fix lint --- paddle/fluid/operators/reader/create_double_buffer_reader_op.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc index f484dbba53..4c423166a9 100644 --- a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc +++ b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc @@ -13,6 +13,7 @@ // limitations under the License. #include //NOLINT + #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/operators/reader/reader_op_registry.h" From 84413478089a90a599391e26651ff1b614fc1180 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 4 Apr 2018 14:27:10 +0800 Subject: [PATCH 17/57] fix lint --- paddle/fluid/operators/reader/create_double_buffer_reader_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc index 4c423166a9..ed868786ab 100644 --- a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc +++ b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include //NOLINT +#include // NOLINT #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/operators/reader/reader_op_registry.h" From 39277e9282294dc18b4c2b93aa000a15b58bea5f Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 4 Apr 2018 14:55:28 +0800 Subject: [PATCH 18/57] fix transpiler condition op in optimize --- python/paddle/fluid/distribute_transpiler.py | 32 ++++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/python/paddle/fluid/distribute_transpiler.py b/python/paddle/fluid/distribute_transpiler.py index 9311fc9904..6d76c1a8d1 100644 --- a/python/paddle/fluid/distribute_transpiler.py +++ b/python/paddle/fluid/distribute_transpiler.py @@ -408,11 +408,16 @@ class DistributeTranspiler: pserver_vars = pserver_program.global_block().vars created_var_map = dict() for _, var in pserver_vars.iteritems(): - tmpvar = s_prog.global_block().create_var( - name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=var.shape) + if var.type == core.VarDesc.VarType.STEP_SCOPES: + tmpvar = s_prog.global_block().create_var( + name=var.name, persistable=var.persistable, type=var.type) + else: + tmpvar = s_prog.global_block().create_var( + name=var.name, + persistable=var.persistable, + type=var.type, + dtype=var.dtype, + shape=var.shape) created_var_map[var.name] = tmpvar # 2. rename op outputs @@ -708,11 +713,18 @@ class DistributeTranspiler: varlist = [varlist] for var in varlist: - program.global_block().create_var( - name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=var.shape) + print("##### deal var: ", var) + if var.type == core.VarDesc.VarType.STEP_SCOPES: + program.global_block().create_var( + name=var.name, + persistable=var.persistable, + type=var.type) + else: + program.global_block().create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=var.shape) optimize_block.append_op( type=opt_op.type, From e0b396e7ba80738fe8c87edb80e5743b8d692cb7 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 4 Apr 2018 15:14:48 +0800 Subject: [PATCH 19/57] update by comment --- python/paddle/fluid/distribute_transpiler.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/python/paddle/fluid/distribute_transpiler.py b/python/paddle/fluid/distribute_transpiler.py index 6d76c1a8d1..134dbe573a 100644 --- a/python/paddle/fluid/distribute_transpiler.py +++ b/python/paddle/fluid/distribute_transpiler.py @@ -412,12 +412,7 @@ class DistributeTranspiler: tmpvar = s_prog.global_block().create_var( name=var.name, persistable=var.persistable, type=var.type) else: - tmpvar = s_prog.global_block().create_var( - name=var.name, - persistable=var.persistable, - type=var.type, - dtype=var.dtype, - shape=var.shape) + tmpvar = s_prog.global_block().clone_variable(var) created_var_map[var.name] = tmpvar # 2. rename op outputs @@ -713,18 +708,13 @@ class DistributeTranspiler: varlist = [varlist] for var in varlist: - print("##### deal var: ", var) if var.type == core.VarDesc.VarType.STEP_SCOPES: program.global_block().create_var( name=var.name, persistable=var.persistable, type=var.type) else: - program.global_block().create_var( - name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=var.shape) + program.global_block().clone_variable(var) optimize_block.append_op( type=opt_op.type, From a16a872783d52d9ba7d32d53848e95cc4ccaefd6 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 4 Apr 2018 15:56:21 +0800 Subject: [PATCH 20/57] update --- python/paddle/fluid/distribute_transpiler.py | 14 ++----------- python/paddle/fluid/framework.py | 21 +++++++++++++------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/python/paddle/fluid/distribute_transpiler.py b/python/paddle/fluid/distribute_transpiler.py index 134dbe573a..31bedb592f 100644 --- a/python/paddle/fluid/distribute_transpiler.py +++ b/python/paddle/fluid/distribute_transpiler.py @@ -408,11 +408,7 @@ class DistributeTranspiler: pserver_vars = pserver_program.global_block().vars created_var_map = dict() for _, var in pserver_vars.iteritems(): - if var.type == core.VarDesc.VarType.STEP_SCOPES: - tmpvar = s_prog.global_block().create_var( - name=var.name, persistable=var.persistable, type=var.type) - else: - tmpvar = s_prog.global_block().clone_variable(var) + tmpvar = s_prog.global_block().clone_variable(var) created_var_map[var.name] = tmpvar # 2. rename op outputs @@ -708,13 +704,7 @@ class DistributeTranspiler: varlist = [varlist] for var in varlist: - if var.type == core.VarDesc.VarType.STEP_SCOPES: - program.global_block().create_var( - name=var.name, - persistable=var.persistable, - type=var.type) - else: - program.global_block().clone_variable(var) + program.global_block().clone_variable(var) optimize_block.append_op( type=opt_op.type, diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index e15456bfc0..39d4017861 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -946,13 +946,20 @@ class Block(object): The new variable cloned from 'var' in current block. """ assert isinstance(var, Variable) - return self.create_var( - name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + ret_var = None + # make STEP_SCOPES var can be safely cloned. + if var.type == core.VarDesc.VarType.STEP_SCOPES: + ret_var = self.create_var( + name=var.name, persistable=var.persistable, type=var.type) + else: + ret_var = self.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True) + return ret_var class Program(object): From 6af178356b064c1de104810ec51f8a49410e4869 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 4 Apr 2018 15:54:17 +0800 Subject: [PATCH 21/57] expose CUDAPinnedPlace to Python --- paddle/fluid/pybind/pybind.cc | 45 +++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index b0a3f06a88..e7fa450832 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -11,11 +11,16 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include +#include +#include // NOLINT // for call_once +#include +#include +#include +#include #include "paddle/fluid/pybind/protobuf.h" -#include // for call_once -#include #include "paddle/fluid/framework/backward.h" #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/framework/executor.h" @@ -32,7 +37,6 @@ limitations under the License. */ #include "paddle/fluid/operators/cond_op.h" #include "paddle/fluid/operators/net_op.h" #include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/platform/gpu_info.h" #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/profiler.h" #include "paddle/fluid/pybind/const_value.h" @@ -100,6 +104,14 @@ PYBIND11_PLUGIN(core) { [](Tensor &self, paddle::platform::CUDAPlace &place) { self.mutable_data(place); }) + .def("alloc_int", + [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) { + self.mutable_data(place); + }) + .def("alloc_float", + [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) { + self.mutable_data(place); + }) .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) @@ -317,7 +329,17 @@ All parameter, weight, gradient are variables in Paddle. #else return new paddle::platform::CUDADeviceContext(place); #endif - }); + }) + .def_static("create", + [](paddle::platform::CUDAPinnedPlace& place) + -> paddle::platform::DeviceContext* { +#ifndef PADDLE_WITH_CUDA + PADDLE_THROW( + "CUDAPinnedPlace is not supported in CPU device."); +#else + return new paddle::platform::CUDAPinnedDeviceContext(place); +#endif + });; // clang-format on #ifdef PADDLE_WITH_CUDA py::class_(m, "Communicator").def(py::init<>()); @@ -330,6 +352,10 @@ All parameter, weight, gradient are variables in Paddle. .def(py::init<>()) .def("__str__", string::to_string); + py::class_(m, "CUDAPinnedPlace") + .def(py::init<>()) + .def("__str__", string::to_string); + py::class_(m, "Place") .def(py::init<>()) .def("set_place", @@ -339,7 +365,11 @@ All parameter, weight, gradient are variables in Paddle. .def("set_place", [](platform::Place &self, const platform::CUDAPlace &gpu_place) { self = gpu_place; - }); + }) + .def("set_place", [](platform::Place &self, + const platform::CUDAPinnedPlace &gpu_place) { + self = gpu_place; + }); py::class_(m, "Operator") .def_static("create", @@ -363,6 +393,11 @@ All parameter, weight, gradient are variables in Paddle. .def("run", [](OperatorBase &self, const Scope &scope, const platform::CUDAPlace &place) { self.Run(scope, place); }) + .def("run", + [](OperatorBase &self, const Scope &scope, + const platform::CUDAPinnedPlace &place) { + self.Run(scope, place); + }) .def("type", [](const OperatorBase &op) -> std::string { return op.Type(); }) .def("outputs", From a84a580e65c7008f7e4aa03b5e93057ac65e988a Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 4 Apr 2018 16:22:28 +0800 Subject: [PATCH 22/57] Add CUDA kernel for prior_box_op. (#9553) --- paddle/fluid/operators/prior_box_op.cc | 7 +- paddle/fluid/operators/prior_box_op.cu | 167 ++++++++++++++++++ paddle/fluid/operators/prior_box_op.h | 45 ++--- .../tests/unittests/test_prior_box_op.py | 56 +++--- 4 files changed, 207 insertions(+), 68 deletions(-) create mode 100644 paddle/fluid/operators/prior_box_op.cu diff --git a/paddle/fluid/operators/prior_box_op.cc b/paddle/fluid/operators/prior_box_op.cc index c22a55bce2..82e54139c8 100644 --- a/paddle/fluid/operators/prior_box_op.cc +++ b/paddle/fluid/operators/prior_box_op.cc @@ -73,7 +73,7 @@ class PriorBoxOp : public framework::OperatorWithKernel { const framework::ExecutionContext& ctx) const override { return framework::OpKernelType( framework::ToDataType(ctx.Input("Input")->type()), - platform::CPUPlace()); + ctx.device_context()); } }; @@ -171,6 +171,5 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(prior_box, ops::PriorBoxOp, ops::PriorBoxOpMaker, paddle::framework::EmptyGradOpMaker); -REGISTER_OP_CPU_KERNEL( - prior_box, ops::PriorBoxOpKernel, - ops::PriorBoxOpKernel); +REGISTER_OP_CPU_KERNEL(prior_box, ops::PriorBoxOpKernel, + ops::PriorBoxOpKernel); diff --git a/paddle/fluid/operators/prior_box_op.cu b/paddle/fluid/operators/prior_box_op.cu new file mode 100644 index 0000000000..76bf2b3b7d --- /dev/null +++ b/paddle/fluid/operators/prior_box_op.cu @@ -0,0 +1,167 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/prior_box_op.h" + +namespace paddle { +namespace operators { + +template +__device__ inline T clip(T in) { + return min(max(in, 0.), 1.); +} + +template +__global__ void GenPriorBox(T* out, const T* aspect_ratios, const int height, + const int width, const int im_height, + const int im_width, const int as_num, + const T offset, const T step_width, + const T step_height, const T* min_sizes, + const T* max_sizes, const int min_num, + bool is_clip) { + int num_priors = max_sizes ? as_num * min_num + min_num : as_num * min_num; + int box_num = height * width * num_priors; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < box_num; + i += blockDim.x * gridDim.x) { + int h = i / (num_priors * width); + int w = (i / num_priors) % width; + int p = i % num_priors; + int m = max_sizes ? p / (as_num + 1) : p / as_num; + T cx = (w + offset) * step_width; + T cy = (h + offset) * step_height; + T bw, bh; + T min_size = min_sizes[m]; + if (max_sizes) { + int s = p % (as_num + 1); + if (s < as_num) { + T ar = aspect_ratios[s]; + bw = min_size * sqrt(ar) / 2.; + bh = min_size / sqrt(ar) / 2.; + } else { + T max_size = max_sizes[m]; + bw = sqrt(min_size * max_size) / 2.; + bh = bw; + } + } else { + int s = p % as_num; + T ar = aspect_ratios[s]; + bw = min_size * sqrt(ar) / 2.; + bh = min_size / sqrt(ar) / 2.; + } + T xmin = (cx - bw) / im_width; + T ymin = (cy - bh) / im_height; + T xmax = (cx + bw) / im_width; + T ymax = (cy + bh) / im_height; + out[i * 4] = is_clip ? clip(xmin) : xmin; + out[i * 4 + 1] = is_clip ? clip(ymin) : ymin; + out[i * 4 + 2] = is_clip ? clip(xmax) : xmax; + out[i * 4 + 3] = is_clip ? clip(ymax) : ymax; + } +} + +template +__global__ void SetVariance(T* out, const T* var, const int vnum, + const int num) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < num; + i += blockDim.x * gridDim.x) { + out[i] = var[i % vnum]; + } +} + +template +class PriorBoxOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* input = ctx.Input("Input"); + auto* image = ctx.Input("Image"); + auto* boxes = ctx.Output("Boxes"); + auto* vars = ctx.Output("Variances"); + + auto min_sizes = ctx.Attr>("min_sizes"); + auto max_sizes = ctx.Attr>("max_sizes"); + auto input_aspect_ratio = ctx.Attr>("aspect_ratios"); + auto variances = ctx.Attr>("variances"); + auto flip = ctx.Attr("flip"); + auto clip = ctx.Attr("clip"); + + std::vector aspect_ratios; + ExpandAspectRatios(input_aspect_ratio, flip, aspect_ratios); + + T step_w = static_cast(ctx.Attr("step_w")); + T step_h = static_cast(ctx.Attr("step_h")); + T offset = static_cast(ctx.Attr("offset")); + + auto im_width = image->dims()[3]; + auto im_height = image->dims()[2]; + + auto width = input->dims()[3]; + auto height = input->dims()[2]; + + T step_width, step_height; + if (step_w == 0 || step_h == 0) { + step_width = static_cast(im_width) / width; + step_height = static_cast(im_height) / height; + } else { + step_width = step_w; + step_height = step_h; + } + + int num_priors = aspect_ratios.size() * min_sizes.size(); + if (max_sizes.size() > 0) { + num_priors += max_sizes.size(); + } + int min_num = static_cast(min_sizes.size()); + int box_num = width * height * num_priors; + + int block = 512; + int grid = (box_num + block - 1) / block; + + auto stream = + ctx.template device_context().stream(); + + boxes->mutable_data(ctx.GetPlace()); + vars->mutable_data(ctx.GetPlace()); + + framework::Tensor r; + framework::TensorFromVector(aspect_ratios, ctx.device_context(), &r); + + framework::Tensor min; + framework::TensorFromVector(min_sizes, ctx.device_context(), &min); + + T* max_data = nullptr; + framework::Tensor max; + if (max_sizes.size() > 0) { + framework::TensorFromVector(max_sizes, ctx.device_context(), &max); + max_data = max.data(); + } + + GenPriorBox<<>>( + boxes->data(), r.data(), height, width, im_height, im_width, + aspect_ratios.size(), offset, step_width, step_height, min.data(), + max_data, min_num, clip); + + framework::Tensor v; + framework::TensorFromVector(variances, ctx.device_context(), &v); + grid = (box_num * 4 + block - 1) / block; + SetVariance<<>>(vars->data(), v.data(), + variances.size(), box_num * 4); + } +}; // namespace operators + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(prior_box, ops::PriorBoxOpCUDAKernel, + ops::PriorBoxOpCUDAKernel); diff --git a/paddle/fluid/operators/prior_box_op.h b/paddle/fluid/operators/prior_box_op.h index 18bb2deb6b..1e4a12aac1 100644 --- a/paddle/fluid/operators/prior_box_op.h +++ b/paddle/fluid/operators/prior_box_op.h @@ -51,7 +51,7 @@ struct ClipFunctor { } }; -template +template class PriorBoxOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -106,49 +106,24 @@ class PriorBoxOpKernel : public framework::OpKernel { int idx = 0; for (size_t s = 0; s < min_sizes.size(); ++s) { auto min_size = min_sizes[s]; - // first prior: aspect_ratio = 1, size = min_size - box_width = box_height = min_size / 2.; - // xmin - e_boxes(h, w, idx, 0) = (center_x - box_width) / img_width; - // ymin - e_boxes(h, w, idx, 1) = (center_y - box_height) / img_height; - // xmax - e_boxes(h, w, idx, 2) = (center_x + box_width) / img_width; - // ymax - e_boxes(h, w, idx, 3) = (center_y + box_height) / img_height; - - idx++; - if (max_sizes.size() > 0) { - auto max_size = max_sizes[s]; - // second prior: aspect_ratio = 1, - // size = sqrt(min_size * max_size) - box_width = box_height = sqrt(min_size * max_size) / 2.; - // xmin + // priors with different aspect ratios + for (size_t r = 0; r < aspect_ratios.size(); ++r) { + float ar = aspect_ratios[r]; + box_width = min_size * sqrt(ar) / 2.; + box_height = min_size / sqrt(ar) / 2.; e_boxes(h, w, idx, 0) = (center_x - box_width) / img_width; - // ymin e_boxes(h, w, idx, 1) = (center_y - box_height) / img_height; - // xmax e_boxes(h, w, idx, 2) = (center_x + box_width) / img_width; - // ymax e_boxes(h, w, idx, 3) = (center_y + box_height) / img_height; idx++; } - - // rest of priors - for (size_t r = 0; r < aspect_ratios.size(); ++r) { - float ar = aspect_ratios[r]; - if (fabs(ar - 1.) < 1e-6) { - continue; - } - box_width = min_size * sqrt(ar) / 2.; - box_height = min_size / sqrt(ar) / 2.; - // xmin + if (max_sizes.size() > 0) { + auto max_size = max_sizes[s]; + // square prior with size sqrt(minSize * maxSize) + box_width = box_height = sqrt(min_size * max_size) / 2.; e_boxes(h, w, idx, 0) = (center_x - box_width) / img_width; - // ymin e_boxes(h, w, idx, 1) = (center_y - box_height) / img_height; - // xmax e_boxes(h, w, idx, 2) = (center_x + box_width) / img_width; - // ymax e_boxes(h, w, idx, 3) = (center_y + box_height) / img_height; idx++; } diff --git a/python/paddle/fluid/tests/unittests/test_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_prior_box_op.py index c21138c13e..bcbc02a2ba 100644 --- a/python/paddle/fluid/tests/unittests/test_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_prior_box_op.py @@ -28,7 +28,6 @@ class TestPriorBoxOp(OpTest): self.attrs = { 'min_sizes': self.min_sizes, - 'max_sizes': self.max_sizes, 'aspect_ratios': self.aspect_ratios, 'variances': self.variances, 'flip': self.flip, @@ -37,25 +36,28 @@ class TestPriorBoxOp(OpTest): 'step_h': self.step_h, 'offset': self.offset } + if len(self.max_sizes) > 0: + self.attrs['max_sizes'] = self.max_sizes self.outputs = {'Boxes': self.out_boxes, 'Variances': self.out_var} def test_check_output(self): self.check_output() - def test_check_grad(self): - return - def setUp(self): self.op_type = "prior_box" self.set_data() + def set_max_sizes(self): + max_sizes = [5, 10] + self.max_sizes = np.array(max_sizes).astype('float32').tolist() + def init_test_params(self): - self.layer_w = 4 - self.layer_h = 4 + self.layer_w = 32 + self.layer_h = 32 - self.image_w = 20 - self.image_h = 20 + self.image_w = 40 + self.image_h = 40 self.step_w = float(self.image_w) / float(self.layer_w) self.step_h = float(self.image_h) / float(self.layer_h) @@ -66,8 +68,7 @@ class TestPriorBoxOp(OpTest): self.min_sizes = [2, 4] self.min_sizes = np.array(self.min_sizes).astype('float32').tolist() - self.max_sizes = [5, 10] - self.max_sizes = np.array(self.max_sizes).astype('float32').tolist() + self.set_max_sizes() self.aspect_ratios = [2.0, 3.0] self.flip = True self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] @@ -79,7 +80,7 @@ class TestPriorBoxOp(OpTest): self.clip = True self.num_priors = len(self.real_aspect_ratios) * len(self.min_sizes) - if len(self.max_sizes) > 1: + if len(self.max_sizes) > 0: self.num_priors += len(self.max_sizes) self.offset = 0.5 @@ -105,35 +106,27 @@ class TestPriorBoxOp(OpTest): idx = 0 for s in range(len(self.min_sizes)): min_size = self.min_sizes[s] - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [ - (c_x - c_w) / self.image_w, (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, (c_y + c_h) / self.image_h - ] - idx += 1 - - if len(self.max_sizes) > 0: - max_size = self.max_sizes[s] - # second prior: aspect_ratio = 1, - c_w = c_h = math.sqrt(min_size * max_size) / 2 + # rest of priors + for r in range(len(self.real_aspect_ratios)): + ar = self.real_aspect_ratios[r] + c_w = min_size * math.sqrt(ar) / 2 + c_h = (min_size / math.sqrt(ar)) / 2 out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, (c_y - c_h) / self.image_h, (c_x + c_w) / self.image_w, (c_y + c_h) / self.image_h] idx += 1 - # rest of priors - for r in range(len(self.real_aspect_ratios)): - ar = self.real_aspect_ratios[r] - if math.fabs(ar - 1.) < 1e-6: - continue - c_w = min_size * math.sqrt(ar) / 2 - c_h = (min_size / math.sqrt(ar)) / 2 + if len(self.max_sizes) > 0: + max_size = self.max_sizes[s] + # second prior: aspect_ratio = 1, + c_w = c_h = math.sqrt(min_size * max_size) / 2 out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, (c_y - c_h) / self.image_h, (c_x + c_w) / self.image_w, (c_y + c_h) / self.image_h] idx += 1 + # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) @@ -144,5 +137,10 @@ class TestPriorBoxOp(OpTest): self.out_var = out_var.astype('float32') +class TestPriorBoxOpWithMaxSize(TestPriorBoxOp): + def set_max_sizes(self): + self.max_sizes = [] + + if __name__ == '__main__': unittest.main() From 72913dc2a60b0e5c9e46e475d74e708fe0b7e80a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Apr 2018 17:21:49 +0800 Subject: [PATCH 23/57] change mklml download url to bce --- cmake/external/mklml.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index df3f0c7f0c..796bcf28a1 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -28,7 +28,7 @@ INCLUDE(ExternalProject) SET(MKLML_PROJECT "extern_mklml") SET(MKLML_VER "mklml_lnx_2018.0.1.20171007") -SET(MKLML_URL "https://github.com/01org/mkl-dnn/releases/download/v0.11/${MKLML_VER}.tgz") +SET(MKLML_URL "http://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.tgz") SET(MKLML_SOURCE_DIR "${THIRD_PARTY_PATH}/mklml") SET(MKLML_DOWNLOAD_DIR "${MKLML_SOURCE_DIR}/src/${MKLML_PROJECT}") SET(MKLML_DST_DIR "mklml") From 92e92ceba1e99e07ac485c9538f53125dd9a77a4 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Wed, 4 Apr 2018 04:24:26 -0700 Subject: [PATCH 24/57] Add type check --- paddle/scripts/docker/build.sh | 0 paddle/scripts/travis/build_doc.sh | 0 python/paddle/fluid/parallel_executor.py | 3 +++ tools/codestyle/cpplint_pre_commit.hook | 0 4 files changed, 3 insertions(+) mode change 100755 => 100644 paddle/scripts/docker/build.sh mode change 100755 => 100644 paddle/scripts/travis/build_doc.sh mode change 100755 => 100644 tools/codestyle/cpplint_pre_commit.hook diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh old mode 100755 new mode 100644 diff --git a/paddle/scripts/travis/build_doc.sh b/paddle/scripts/travis/build_doc.sh old mode 100755 new mode 100644 diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index 4153049c05..1b3ba414ec 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -76,6 +76,9 @@ class ParallelExecutor(object): or numpy array. :return: fetched value list. """ + if not isinstance(feed_dict, dict): + raise TypeError("feed_dict should be a dict") + feed_tensor_dict = {} for i, feed_name in enumerate(feed_dict): feed_tensor = feed_dict[feed_name] diff --git a/tools/codestyle/cpplint_pre_commit.hook b/tools/codestyle/cpplint_pre_commit.hook old mode 100755 new mode 100644 From bdea5bee7d263212b1c2e367b8d6b61daaecfa18 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Wed, 4 Apr 2018 04:32:36 -0700 Subject: [PATCH 25/57] polish --- doc/fluid/api/gen_doc.sh | 0 paddle/scripts/docker/build.sh | 0 paddle/scripts/travis/build_doc.sh | 0 tools/codestyle/cpplint_pre_commit.hook | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 doc/fluid/api/gen_doc.sh mode change 100644 => 100755 paddle/scripts/docker/build.sh mode change 100644 => 100755 paddle/scripts/travis/build_doc.sh mode change 100644 => 100755 tools/codestyle/cpplint_pre_commit.hook diff --git a/doc/fluid/api/gen_doc.sh b/doc/fluid/api/gen_doc.sh old mode 100644 new mode 100755 diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh old mode 100644 new mode 100755 diff --git a/paddle/scripts/travis/build_doc.sh b/paddle/scripts/travis/build_doc.sh old mode 100644 new mode 100755 diff --git a/tools/codestyle/cpplint_pre_commit.hook b/tools/codestyle/cpplint_pre_commit.hook old mode 100644 new mode 100755 From 8e4e155c5264bc38828546a86c41790a0a17350d Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 4 Apr 2018 16:38:18 +0800 Subject: [PATCH 26/57] add PyCUDAPinnedTensorSetFromArray --- paddle/fluid/framework/tensor_impl.h | 16 +++++++++---- paddle/fluid/pybind/pybind.cc | 10 ++++++-- paddle/fluid/pybind/tensor_py.h | 34 ++++++++++++++++++++++++++++ python/paddle/fluid/__init__.py | 3 ++- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/framework/tensor_impl.h b/paddle/fluid/framework/tensor_impl.h index 7a48390440..07d0906ea7 100644 --- a/paddle/fluid/framework/tensor_impl.h +++ b/paddle/fluid/framework/tensor_impl.h @@ -128,13 +128,21 @@ inline void* Tensor::mutable_data(platform::Place place, std::type_index type) { if (platform::is_cpu_place(place)) { holder_.reset(new PlaceholderImpl( boost::get(place), size, type)); - } else if (platform::is_gpu_place(place)) { + } else if (platform::is_gpu_place(place) || + platform::is_cuda_pinned_place(place)) { #ifndef PADDLE_WITH_CUDA - PADDLE_THROW("'CUDAPlace' is not supported in CPU only device."); + PADDLE_THROW( + "'CUDAPlace' or 'CUDAPinnedPlace' is not supported in CPU only " + "device."); } #else - holder_.reset(new PlaceholderImpl( - boost::get(place), size, type)); + if (platform::is_gpu_place(place)) { + holder_.reset(new PlaceholderImpl( + boost::get(place), size, type)); + } else if (platform::is_cuda_pinned_place(place)) { + holder_.reset(new PlaceholderImpl( + boost::get(place), size, type)); + } } #endif offset_ = 0; diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index e7fa450832..046721970a 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -125,6 +125,12 @@ PYBIND11_PLUGIN(core) { .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) + .def("set", PyCUDAPinnedTensorSetFromArray) + .def("set", PyCUDAPinnedTensorSetFromArray) + .def("set", PyCUDAPinnedTensorSetFromArray) + .def("set", PyCUDAPinnedTensorSetFromArray) + .def("set", PyCUDAPinnedTensorSetFromArray) + .def("set", PyCUDAPinnedTensorSetFromArray) #endif .def("shape", [](Tensor &self) { return vectorize(self.dims()); }) .def("set_float_element", TensorSetElement) @@ -367,8 +373,8 @@ All parameter, weight, gradient are variables in Paddle. self = gpu_place; }) .def("set_place", [](platform::Place &self, - const platform::CUDAPinnedPlace &gpu_place) { - self = gpu_place; + const platform::CUDAPinnedPlace &cuda_pinned_place) { + self = cuda_pinned_place; }); py::class_(m, "Operator") diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h index 6f8c597f8e..f52ffc9ef3 100644 --- a/paddle/fluid/pybind/tensor_py.h +++ b/paddle/fluid/pybind/tensor_py.h @@ -14,6 +14,8 @@ limitations under the License. */ #pragma once #include +#include +#include #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/memory/memcpy.h" #include "paddle/fluid/platform/device_context.h" @@ -208,6 +210,38 @@ void PyCUDATensorSetFromArray( sizeof(uint16_t) * array.size(), cudaMemcpyHostToDevice, dev_ctx->stream()); } + +template +void PyCUDAPinnedTensorSetFromArray( + framework::Tensor &self, + py::array_t array, + const paddle::platform::CUDAPinnedPlace &place) { + std::vector dims; + dims.reserve(array.ndim()); + for (size_t i = 0; i < array.ndim(); ++i) { + dims.push_back(static_cast(array.shape()[i])); + } + + self.Resize(framework::make_ddim(dims)); + auto *dst = self.mutable_data(place); + std::memcpy(dst, array.data(), sizeof(T) * array.size()); +} + +template <> +void PyCUDAPinnedTensorSetFromArray( + framework::Tensor &self, + py::array_t array, + const paddle::platform::CUDAPinnedPlace &place) { + std::vector dims; + dims.reserve(array.ndim()); + for (size_t i = 0; i < array.ndim(); ++i) { + dims.push_back(static_cast(array.shape()[i])); + } + + self.Resize(framework::make_ddim(dims)); + auto *dst = self.mutable_data(place); + std::memcpy(dst, array.data(), sizeof(uint16_t) * array.size()); +} #endif } // namespace pybind diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 5ea4d977f4..f01d638efd 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -31,7 +31,7 @@ import regularizer import average from param_attr import ParamAttr, WeightNormParamAttr from data_feeder import DataFeeder -from core import LoDTensor, CPUPlace, CUDAPlace +from core import LoDTensor, CPUPlace, CUDAPlace, CUDAPinnedPlace from distribute_transpiler import DistributeTranspiler from distribute_transpiler_simple import SimpleDistributeTranspiler from concurrency import (Go, make_channel, channel_send, channel_recv, @@ -57,6 +57,7 @@ __all__ = framework.__all__ + executor.__all__ + concurrency.__all__ + [ 'LoDTensor', 'CPUPlace', 'CUDAPlace', + 'CUDAPinnedPlace', 'Tensor', 'ParamAttr', 'WeightNormParamAttr', From f8dd03dced4b725dbd161d1b9ebbe89b97ce6173 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 4 Apr 2018 14:04:17 +0200 Subject: [PATCH 27/57] Prepare code for CentOS (#9651) --- paddle/fluid/operators/fc_mkldnn_op.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/fc_mkldnn_op.cc b/paddle/fluid/operators/fc_mkldnn_op.cc index 9c704a2949..847b7b0c12 100644 --- a/paddle/fluid/operators/fc_mkldnn_op.cc +++ b/paddle/fluid/operators/fc_mkldnn_op.cc @@ -27,8 +27,8 @@ template class MKLDNNMD { public: explicit MKLDNNMD(const T* in, const T* w, bool bias) - : in{paddle::framework::vectorize2int(in->dims())}, - w{paddle::framework::vectorize2int(w->dims())} { + : in(paddle::framework::vectorize2int(in->dims())), + w(paddle::framework::vectorize2int(w->dims())) { with_bias_ = bias; } @@ -78,7 +78,7 @@ class MKLDNNMD { class MKLDNNMemory { public: MKLDNNMemory(MKLDNNMD* t, const mkldnn::engine& e) - : md_{t}, engine_{e} {} + : md_(t), engine_(e) {} virtual ~MKLDNNMemory() = default; template From e66bd4cb732003d083f981bc5b2c7fe238590aa0 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Apr 2018 23:31:13 +0800 Subject: [PATCH 28/57] add GetDataTypeOfVar --- paddle/fluid/framework/operator.cc | 11 +++++++++++ paddle/fluid/framework/operator.h | 2 ++ paddle/fluid/operators/lookup_table_op.cc | 22 ++++------------------ paddle/fluid/operators/sgd_op.cc | 15 ++------------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index f6a43804ef..a3b4a8c082 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -35,6 +35,17 @@ std::vector> kKernelPriority = { std::make_tuple(platform::CPUPlace(), LibraryType::kPlain), }; +proto::VarType::Type GetDataTypeOfVar(const Variable* var) { + if (var->IsType()) { + return framework::ToDataType(var->Get().type()); + } else if (var->IsType()) { + return framework::ToDataType( + var->Get().value().type()); + } else { + PADDLE_THROW("Var should be LoDTensor or SelectedRows"); + } +} + static DDim GetDims(const Scope& scope, const std::string& name) { Variable* var = scope.FindVar(name); if (var == nullptr) { diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index 41214b41cb..b7a7c69b4c 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -61,6 +61,8 @@ inline std::string GradVarName(const std::string& var_name) { return var_name + kGradVarSuffix; } +proto::VarType::Type GetDataTypeOfVar(const Variable* var); + class OperatorBase; class ExecutionContext; diff --git a/paddle/fluid/operators/lookup_table_op.cc b/paddle/fluid/operators/lookup_table_op.cc index deabcdc99f..bf33be3106 100644 --- a/paddle/fluid/operators/lookup_table_op.cc +++ b/paddle/fluid/operators/lookup_table_op.cc @@ -18,22 +18,6 @@ limitations under the License. */ namespace paddle { namespace operators { -static inline framework::OpKernelType ExpectedKernelType( - const framework::ExecutionContext& ctx) { - auto* table_var = ctx.InputVar("W"); - if (table_var->IsType()) { - return framework::OpKernelType( - framework::ToDataType(table_var->Get().type()), - ctx.device_context()); - } else if (table_var->IsType()) { - return framework::OpKernelType( - framework::ToDataType(table_var->Get().value().type()), - ctx.device_context()); - } else { - PADDLE_THROW("W should be LoDTensor or SelectedRows"); - } -} - class LookupTableOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -67,7 +51,8 @@ class LookupTableOp : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - return ExpectedKernelType(ctx); + auto data_type = framework::GetDataTypeOfVar(ctx.InputVar("W")); + return framework::OpKernelType(data_type, ctx.device_context()); } }; @@ -138,7 +123,8 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - return ExpectedKernelType(ctx); + auto data_type = framework::GetDataTypeOfVar(ctx.InputVar("W")); + return framework::OpKernelType(data_type, ctx.device_context()); } }; diff --git a/paddle/fluid/operators/sgd_op.cc b/paddle/fluid/operators/sgd_op.cc index 9cdc5b3f1e..074fa9e00f 100644 --- a/paddle/fluid/operators/sgd_op.cc +++ b/paddle/fluid/operators/sgd_op.cc @@ -43,19 +43,8 @@ class SGDOp : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - auto* table_var = ctx.InputVar("Param"); - if (table_var->IsType()) { - return framework::OpKernelType( - framework::ToDataType(table_var->Get().type()), - ctx.device_context()); - } else if (table_var->IsType()) { - return framework::OpKernelType( - framework::ToDataType( - table_var->Get().value().type()), - ctx.device_context()); - } else { - PADDLE_THROW("Param should be LoDTensor or SelectedRows"); - } + auto data_type = framework::GetDataTypeOfVar(ctx.InputVar("Param")); + return framework::OpKernelType(data_type, ctx.device_context()); } }; From 7bf82f82b1f1e74f4755eb327e1528a2544437bc Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 4 Apr 2018 16:32:11 -0700 Subject: [PATCH 29/57] Fix CPPlint errors in channel.h, channel_impl.h and channel_test.cc (#9628) * Fix cpplint issues * Fix cpplint issues in channel.h and channel_impl.h * Fix typo --- paddle/fluid/framework/channel.h | 7 +- paddle/fluid/framework/channel_impl.h | 21 ++-- paddle/fluid/framework/channel_test.cc | 160 ++++++++++++------------- 3 files changed, 95 insertions(+), 93 deletions(-) diff --git a/paddle/fluid/framework/channel.h b/paddle/fluid/framework/channel.h index 019bea600f..722bf8e8ec 100644 --- a/paddle/fluid/framework/channel.h +++ b/paddle/fluid/framework/channel.h @@ -14,8 +14,8 @@ limitations under the License. */ #pragma once -#include // for size_t -#include +#include // for size_t +#include // NOLINT #include #include "paddle/fluid/platform/enforce.h" @@ -216,7 +216,8 @@ class ChannelHolder { template struct PlaceholderImpl : public Placeholder { - PlaceholderImpl(size_t buffer_size) : type_(std::type_index(typeid(T))) { + explicit PlaceholderImpl(size_t buffer_size) + : type_(std::type_index(typeid(T))) { channel_.reset(MakeChannel(buffer_size)); } diff --git a/paddle/fluid/framework/channel_impl.h b/paddle/fluid/framework/channel_impl.h index e056779ea0..26d454534e 100644 --- a/paddle/fluid/framework/channel_impl.h +++ b/paddle/fluid/framework/channel_impl.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once #include // for size_t #include -#include +#include // NOLINT #include #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/platform/enforce.h" @@ -38,7 +38,7 @@ class ChannelImpl : public paddle::framework::Channel { virtual void Unlock(); virtual bool IsClosed(); virtual void Close(); - ChannelImpl(size_t); + explicit ChannelImpl(size_t); virtual ~ChannelImpl(); virtual void AddToSendQ(const void *referrer, T *data, @@ -60,7 +60,7 @@ class ChannelImpl : public paddle::framework::Channel { const void *referrer; // TODO(thuan): figure out better way to do this std::function callback; - QueueMessage(T *item) + explicit QueueMessage(T *item) : data(item), cond(std::make_shared()) {} QueueMessage(T *item, std::shared_ptr cond) @@ -88,15 +88,15 @@ class ChannelImpl : public paddle::framework::Channel { } std::shared_ptr get_first_message( - std::deque> &queue, ChannelAction action) { - while (!queue.empty()) { + std::deque> *queue, ChannelAction action) { + while (!queue->empty()) { // Check whether this message was added by Select // If this was added by Select then execute the callback // to check if you can execute this message. The callback // can return false if some other case was executed in Select. // In that case just discard this QueueMessage and process next. - std::shared_ptr m = queue.front(); - queue.pop_front(); + std::shared_ptr m = queue->front(); + queue->pop_front(); if (m->callback == nullptr || m->callback(action)) return m; } return nullptr; @@ -147,7 +147,7 @@ void ChannelImpl::Send(T *item) { // to send to the receiver, bypassing the channel buffer if any if (!recvq.empty()) { std::shared_ptr m = - get_first_message(recvq, ChannelAction::SEND); + get_first_message(&recvq, ChannelAction::SEND); if (m != nullptr) { *(m->data) = std::move(*item); @@ -198,7 +198,7 @@ bool ChannelImpl::Receive(T *item) { // buffer and move front of send queue to the buffer if (!sendq.empty()) { std::shared_ptr m = - get_first_message(sendq, ChannelAction::RECEIVE); + get_first_message(&sendq, ChannelAction::RECEIVE); if (buf_.size() > 0) { // Case 1 : Channel is Buffered // Do Data transfer from front of buffer @@ -219,8 +219,9 @@ bool ChannelImpl::Receive(T *item) { if (m != nullptr) { *item = std::move(*(m->data)); m->Notify(); - } else + } else { return recv_return(Receive(item)); + } } return recv_return(true); } diff --git a/paddle/fluid/framework/channel_test.cc b/paddle/fluid/framework/channel_test.cc index 1184bfdae1..542d791f6b 100644 --- a/paddle/fluid/framework/channel_test.cc +++ b/paddle/fluid/framework/channel_test.cc @@ -14,8 +14,8 @@ limitations under the License. */ #include "paddle/fluid/framework/channel.h" -#include -#include +#include // NOLINT +#include // NOLINT #include "gtest/gtest.h" using paddle::framework::Channel; @@ -166,9 +166,9 @@ TEST(Channel, ConcurrentSendNonConcurrentReceiveWithSufficientBufferSize) { std::thread t([&]() { // Try to write more than buffer size. for (size_t i = 0; i < 2 * buffer_size; ++i) { - if (i < buffer_size) + if (i < buffer_size) { ch->Send(&i); // should block after 10 iterations - else { + } else { bool is_exception = false; try { ch->Send(&i); @@ -212,12 +212,12 @@ TEST(Channel, RecevingOrderEqualToSendingOrderWithBufferedChannel3) { } void ChannelCloseUnblocksReceiversTest(Channel *ch) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; // Launches threads that try to read and are blocked because of no writers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { @@ -230,7 +230,7 @@ void ChannelCloseUnblocksReceiversTest(Channel *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait 0.2 sec // Verify that all the threads are blocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } @@ -241,21 +241,21 @@ void ChannelCloseUnblocksReceiversTest(Channel *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait 0.2 sec // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } void ChannelCloseUnblocksSendersTest(Channel *ch, bool isBuffered) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; - bool send_success[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; + bool send_success[kNumThreads]; // Launches threads that try to write and are blocked because of no readers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; send_success[i] = false; t[i] = std::thread( @@ -277,13 +277,13 @@ void ChannelCloseUnblocksSendersTest(Channel *ch, bool isBuffered) { if (isBuffered) { // If ch is Buffered, atleast 4 threads must be blocked. int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (!thread_ended[i]) ct++; } EXPECT_GE(ct, 4); } else { // If ch is UnBuffered, all the threads should be blocked. - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } } @@ -294,21 +294,21 @@ void ChannelCloseUnblocksSendersTest(Channel *ch, bool isBuffered) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } if (isBuffered) { // Verify that only 1 send was successful int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (send_success[i]) ct++; } // Only 1 send must be successful EXPECT_EQ(ct, 1); } - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } // This tests that closing a buffered channel also unblocks @@ -409,13 +409,13 @@ TEST(Channel, UnbufferedMoreReceiveLessSendTest) { // This tests that destroying a channel unblocks // any senders waiting for channel to have write space void ChannelDestroyUnblockSenders(Channel *ch, bool isBuffered) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; - bool send_success[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; + bool send_success[kNumThreads]; // Launches threads that try to write and are blocked because of no readers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; send_success[i] = false; t[i] = std::thread( @@ -438,14 +438,14 @@ void ChannelDestroyUnblockSenders(Channel *ch, bool isBuffered) { if (isBuffered) { // If channel is buffered, verify that atleast 4 threads are blocked int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (thread_ended[i] == false) ct++; } // Atleast 4 threads must be blocked EXPECT_GE(ct, 4); } else { // Verify that all the threads are blocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } } @@ -454,13 +454,13 @@ void ChannelDestroyUnblockSenders(Channel *ch, bool isBuffered) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } // Count number of successful sends int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (send_success[i]) ct++; } @@ -473,18 +473,18 @@ void ChannelDestroyUnblockSenders(Channel *ch, bool isBuffered) { } // Join all threads - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } // This tests that destroying a channel also unblocks // any receivers waiting on the channel void ChannelDestroyUnblockReceivers(Channel *ch) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; // Launches threads that try to read and are blocked because of no writers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { @@ -498,18 +498,18 @@ void ChannelDestroyUnblockReceivers(Channel *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait // Verify that all threads are blocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } // delete the channel delete ch; std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } TEST(Channel, BufferedChannelDestroyUnblocksReceiversTest) { @@ -679,12 +679,12 @@ TEST(ChannelHolder, TypeMismatchReceiveTest) { } void ChannelHolderCloseUnblocksReceiversTest(ChannelHolder *ch) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; // Launches threads that try to read and are blocked because of no writers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { @@ -697,7 +697,7 @@ void ChannelHolderCloseUnblocksReceiversTest(ChannelHolder *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait 0.2 sec // Verify that all the threads are blocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } @@ -708,21 +708,21 @@ void ChannelHolderCloseUnblocksReceiversTest(ChannelHolder *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait 0.2 sec // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } void ChannelHolderCloseUnblocksSendersTest(ChannelHolder *ch, bool isBuffered) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; - bool send_success[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; + bool send_success[kNumThreads]; // Launches threads that try to write and are blocked because of no readers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; send_success[i] = false; t[i] = std::thread( @@ -744,13 +744,13 @@ void ChannelHolderCloseUnblocksSendersTest(ChannelHolder *ch, bool isBuffered) { if (isBuffered) { // If ch is Buffered, atleast 4 threads must be blocked. int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (!thread_ended[i]) ct++; } EXPECT_GE(ct, 4); } else { // If ch is UnBuffered, all the threads should be blocked. - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } } @@ -761,21 +761,21 @@ void ChannelHolderCloseUnblocksSendersTest(ChannelHolder *ch, bool isBuffered) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } if (isBuffered) { // Verify that only 1 send was successful int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (send_success[i]) ct++; } // Only 1 send must be successful EXPECT_EQ(ct, 1); } - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } // This tests that closing a channelholder unblocks @@ -813,13 +813,13 @@ TEST(Channel, ChannelHolderCloseUnblocksSendersTest) { // This tests that destroying a channelholder unblocks // any senders waiting for channel void ChannelHolderDestroyUnblockSenders(ChannelHolder *ch, bool isBuffered) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; - bool send_success[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; + bool send_success[kNumThreads]; // Launches threads that try to write and are blocked because of no readers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; send_success[i] = false; t[i] = std::thread( @@ -841,14 +841,14 @@ void ChannelHolderDestroyUnblockSenders(ChannelHolder *ch, bool isBuffered) { if (isBuffered) { // If channel is buffered, verify that atleast 4 threads are blocked int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (thread_ended[i] == false) ct++; } // Atleast 4 threads must be blocked EXPECT_GE(ct, 4); } else { // Verify that all the threads are blocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } } @@ -857,13 +857,13 @@ void ChannelHolderDestroyUnblockSenders(ChannelHolder *ch, bool isBuffered) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } // Count number of successfuld sends int ct = 0; - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { if (send_success[i]) ct++; } @@ -876,18 +876,18 @@ void ChannelHolderDestroyUnblockSenders(ChannelHolder *ch, bool isBuffered) { } // Join all threads - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } // This tests that destroying a channelholder also unblocks // any receivers waiting on the channel void ChannelHolderDestroyUnblockReceivers(ChannelHolder *ch) { - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; + const size_t kNumThreads = 5; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; // Launches threads that try to read and are blocked because of no writers - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { @@ -901,18 +901,18 @@ void ChannelHolderDestroyUnblockReceivers(ChannelHolder *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads are blocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], false); } // delete the channel delete ch; std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } TEST(ChannelHolder, ChannelHolderDestroyUnblocksReceiversTest) { @@ -945,12 +945,12 @@ TEST(ChannelHolder, ChannelHolderDestroyUnblocksSendersTest) { // This tests that closing a channelholder many times. void ChannelHolderManyTimesClose(ChannelHolder *ch) { - const int num_threads = 15; - std::thread t[num_threads]; - bool thread_ended[num_threads]; + const int kNumThreads = 15; + std::thread t[kNumThreads]; + bool thread_ended[kNumThreads]; // Launches threads that try to send data to channel. - for (size_t i = 0; i < num_threads / 3; i++) { + for (size_t i = 0; i < kNumThreads / 3; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *ended) { @@ -962,7 +962,7 @@ void ChannelHolderManyTimesClose(ChannelHolder *ch) { } // Launches threads that try to receive data to channel. - for (size_t i = num_threads / 3; i < 2 * num_threads / 3; i++) { + for (size_t i = kNumThreads / 3; i < 2 * kNumThreads / 3; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { @@ -976,7 +976,7 @@ void ChannelHolderManyTimesClose(ChannelHolder *ch) { } // Launches threads that try to close the channel. - for (size_t i = 2 * num_threads / 3; i < num_threads; i++) { + for (size_t i = 2 * kNumThreads / 3; i < kNumThreads; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { @@ -991,13 +991,13 @@ void ChannelHolderManyTimesClose(ChannelHolder *ch) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait // Verify that all threads are unblocked - for (size_t i = 0; i < num_threads; i++) { + for (size_t i = 0; i < kNumThreads; i++) { EXPECT_EQ(thread_ended[i], true); } EXPECT_TRUE(ch->IsClosed()); // delete the channel delete ch; - for (size_t i = 0; i < num_threads; i++) t[i].join(); + for (size_t i = 0; i < kNumThreads; i++) t[i].join(); } TEST(ChannelHolder, ChannelHolderManyTimesCloseTest) { From 30061d48313d9c96af49dfe65939c47ca3bb3470 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Thu, 5 Apr 2018 09:00:59 -0700 Subject: [PATCH 30/57] Fix cpplint errors in paddle/fluid/string (#9667) * Fix cpplint errors in paddle/fluid/string * Fix unit test error * Correct --- paddle/fluid/string/piece.cc | 2 +- paddle/fluid/string/printf.h | 2 ++ paddle/fluid/string/printf_test.cc | 5 +++-- paddle/fluid/string/to_string_test.cc | 7 +++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/string/piece.cc b/paddle/fluid/string/piece.cc index 454f5d8d38..8e8cfb0e91 100644 --- a/paddle/fluid/string/piece.cc +++ b/paddle/fluid/string/piece.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "piece.h" +#include "paddle/fluid/string/piece.h" #include diff --git a/paddle/fluid/string/printf.h b/paddle/fluid/string/printf.h index 693cf9d6df..062095a1c3 100644 --- a/paddle/fluid/string/printf.h +++ b/paddle/fluid/string/printf.h @@ -71,6 +71,8 @@ #include #include +#include + #include "tinyformat/tinyformat.h" // https://github.com/c42f/tinyformat namespace paddle { diff --git a/paddle/fluid/string/printf_test.cc b/paddle/fluid/string/printf_test.cc index b6a60c8d6b..678029f935 100644 --- a/paddle/fluid/string/printf_test.cc +++ b/paddle/fluid/string/printf_test.cc @@ -11,7 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "printf.h" + +#include "paddle/fluid/string/printf.h" #include @@ -21,7 +22,7 @@ TEST(StringPrintf, StringPrintf) { std::string weekday = "Wednesday"; const char* month = "July"; size_t day = 27; - long hour = 14; + int hour = 14; int min = 44; EXPECT_EQ(std::string("Wednesday, July 27, 14:44"), paddle::string::Sprintf("%s, %s %d, %.2d:%.2d", weekday, month, day, diff --git a/paddle/fluid/string/to_string_test.cc b/paddle/fluid/string/to_string_test.cc index 8fc293af0e..1d9c0e5e0c 100644 --- a/paddle/fluid/string/to_string_test.cc +++ b/paddle/fluid/string/to_string_test.cc @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "to_string.h" +#include "paddle/fluid/string/to_string.h" #include constexpr char kOutputString[] = "User Defined Output"; @@ -26,14 +26,13 @@ std::ostream& operator<<(std::ostream& s, const UserDefinedClass& ins) { } TEST(to_string, normal) { - using namespace paddle::string; + using paddle::string::to_string; ASSERT_EQ("10", to_string(10)); ASSERT_EQ("abc", to_string("abc")); ASSERT_EQ("1.2", to_string(1.2)); } TEST(to_string, user_defined) { - using namespace paddle::string; UserDefinedClass instance; - ASSERT_EQ(kOutputString, to_string(instance)); + ASSERT_EQ(kOutputString, paddle::string::to_string(instance)); } From 09b4a1a361003840e3690e61003a37db4f65564d Mon Sep 17 00:00:00 2001 From: Lei Wang Date: Thu, 5 Apr 2018 13:24:10 -0700 Subject: [PATCH 31/57] Build: generate all the build related files into one directory. (#9512) --- .gitignore | 9 --------- cmake/generic.cmake | 6 +++--- doc/fluid/CMakeLists.txt | 4 ++-- doc/fluid/api/CMakeLists.txt | 2 +- doc/templates/conf.py.cn.in | 2 +- doc/templates/conf.py.en.in | 2 +- doc/v2/CMakeLists.txt | 4 ++-- doc/v2/api/CMakeLists.txt | 2 +- paddle/api/CMakeLists.txt | 11 ++++++----- paddle/api/test/CMakeLists.txt | 5 +++++ paddle/fluid/framework/CMakeLists.txt | 4 ++-- .../fluid/inference/tests/book/CMakeLists.txt | 2 +- paddle/fluid/operators/CMakeLists.txt | 4 ++-- paddle/fluid/platform/CMakeLists.txt | 4 ++-- paddle/gserver/tests/CMakeLists.txt | 19 ++++++++++++------- paddle/trainer/tests/CMakeLists.txt | 13 +++++++++---- paddle/utils/CMakeLists.txt | 4 ++-- proto/CMakeLists.txt | 5 +++-- python/CMakeLists.txt | 8 +++++--- .../fluid/tests/unittests/CMakeLists.txt | 4 ++-- .../tests/CMakeLists.txt | 8 ++++---- .../tests/configs/generate_protostr.sh | 1 - python/setup.py.in | 4 ++-- 23 files changed, 68 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 2badc3bdaa..9e3a0b499f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,12 +25,3 @@ third_party/ # clion workspace. cmake-build-* - -# generated while compiling -paddle/pybind/pybind.h -CMakeFiles -cmake_install.cmake -paddle/.timestamp -python/paddlepaddle.egg-info/ -paddle/fluid/pybind/pybind.h -python/paddle/version.py diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 3fe750f47e..e8bc285bdc 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -251,7 +251,7 @@ function(cc_test TARGET_NAME) add_dependencies(${TARGET_NAME} ${cc_test_DEPS} paddle_gtest_main paddle_memory gtest gflags glog) add_test(NAME ${TARGET_NAME} COMMAND ${TARGET_NAME} ${cc_test_ARGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() endfunction(cc_test) @@ -561,9 +561,9 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS ARGS ENVS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${PADDLE_PYTHON_BUILD_DIR}/lib-python ${py_test_ENVS} + COMMAND env PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_ENVS} ${PYTHON_EXECUTABLE} -u ${py_test_SRCS} ${py_test_ARGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() endfunction() diff --git a/doc/fluid/CMakeLists.txt b/doc/fluid/CMakeLists.txt index 9fe79323ef..8086507bb4 100644 --- a/doc/fluid/CMakeLists.txt +++ b/doc/fluid/CMakeLists.txt @@ -27,7 +27,7 @@ sphinx_add_target(paddle_fluid_docs ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_EN}) -add_dependencies(paddle_fluid_docs gen_proto_py) +add_dependencies(paddle_fluid_docs gen_proto_py paddle_python) # configured documentation tools and intermediate build results set(BINARY_BUILD_DIR_CN "${CMAKE_CURRENT_BINARY_DIR}/cn/_build") @@ -50,6 +50,6 @@ sphinx_add_target(paddle_fluid_docs_cn ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_CN}) -add_dependencies(paddle_fluid_docs_cn gen_proto_py) +add_dependencies(paddle_fluid_docs_cn gen_proto_py paddle_python) add_subdirectory(api) diff --git a/doc/fluid/api/CMakeLists.txt b/doc/fluid/api/CMakeLists.txt index ca40dfb964..48b396f078 100644 --- a/doc/fluid/api/CMakeLists.txt +++ b/doc/fluid/api/CMakeLists.txt @@ -19,4 +19,4 @@ sphinx_add_target(paddle_fluid_apis ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_EN}) -add_dependencies(paddle_fluid_apis gen_proto_py framework_py_proto copy_paddle_pybind) +add_dependencies(paddle_fluid_apis gen_proto_py framework_py_proto copy_paddle_pybind paddle_python) diff --git a/doc/templates/conf.py.cn.in b/doc/templates/conf.py.cn.in index 260b6c9fd1..76b82fd97f 100644 --- a/doc/templates/conf.py.cn.in +++ b/doc/templates/conf.py.cn.in @@ -13,7 +13,7 @@ # serve to show the default. import sys import os, subprocess -sys.path.insert(0, os.path.abspath('@PADDLE_SOURCE_DIR@/python')) +sys.path.insert(0, os.path.abspath('@PADDLE_BINARY_DIR@/python')) import shlex from recommonmark import parser, transform import paddle diff --git a/doc/templates/conf.py.en.in b/doc/templates/conf.py.en.in index e5757b86b4..5aa5c1381f 100644 --- a/doc/templates/conf.py.en.in +++ b/doc/templates/conf.py.en.in @@ -13,7 +13,7 @@ # serve to show the default. import sys import os, subprocess -sys.path.insert(0, os.path.abspath('@PADDLE_SOURCE_DIR@/python')) +sys.path.insert(0, os.path.abspath('@PADDLE_BINARY_DIR@/python')) import shlex from recommonmark import parser, transform import paddle diff --git a/doc/v2/CMakeLists.txt b/doc/v2/CMakeLists.txt index 82de7a3a3e..be957d37b1 100644 --- a/doc/v2/CMakeLists.txt +++ b/doc/v2/CMakeLists.txt @@ -27,7 +27,7 @@ sphinx_add_target(paddle_v2_docs ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_EN}) -add_dependencies(paddle_v2_docs gen_proto_py) +add_dependencies(paddle_v2_docs gen_proto_py paddle_python) # configured documentation tools and intermediate build results set(BINARY_BUILD_DIR_CN "${CMAKE_CURRENT_BINARY_DIR}/cn/_build") @@ -50,6 +50,6 @@ sphinx_add_target(paddle_v2_docs_cn ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_CN}) -add_dependencies(paddle_v2_docs_cn gen_proto_py) +add_dependencies(paddle_v2_docs_cn gen_proto_py paddle_python) add_subdirectory(api) diff --git a/doc/v2/api/CMakeLists.txt b/doc/v2/api/CMakeLists.txt index da1eafc02e..2670a21a22 100644 --- a/doc/v2/api/CMakeLists.txt +++ b/doc/v2/api/CMakeLists.txt @@ -19,4 +19,4 @@ sphinx_add_target(paddle_v2_apis ${CMAKE_CURRENT_SOURCE_DIR} ${SPHINX_HTML_DIR_EN}) -add_dependencies(paddle_v2_apis gen_proto_py framework_py_proto copy_paddle_pybind) +add_dependencies(paddle_v2_apis gen_proto_py framework_py_proto copy_paddle_pybind paddle_python) diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index cf84568ecd..06e1f5d5f0 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -89,16 +89,17 @@ SWIG_LINK_LIBRARIES(swig_paddle ${START_END} ) -add_custom_command(OUTPUT ${PADDLE_SOURCE_DIR}/paddle/py_paddle/_swig_paddle.so - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/swig_paddle.py ${PADDLE_SOURCE_DIR}/paddle/py_paddle - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/_swig_paddle.so ${PADDLE_SOURCE_DIR}/paddle/py_paddle - COMMAND ${CMAKE_COMMAND} -E touch .timestamp +add_custom_command(OUTPUT ${PADDLE_BINARY_DIR}/python/py_paddle/_swig_paddle.so + COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_BINARY_DIR}/python/py_paddle + COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/swig_paddle.py ${PADDLE_BINARY_DIR}/python/py_paddle + COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/_swig_paddle.so ${PADDLE_BINARY_DIR}/python/py_paddle + COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_BINARY_DIR}/.timestamp WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle DEPENDS _swig_paddle ) # TODO(yuyang18) : make wheel name calculated by cmake -add_custom_target(python_api_wheel ALL DEPENDS ${PADDLE_SOURCE_DIR}/paddle/py_paddle/_swig_paddle.so) +add_custom_target(python_api_wheel ALL DEPENDS ${PADDLE_BINARY_DIR}/python/py_paddle/_swig_paddle.so) if(WITH_TESTING) IF(NOT PY_PIP_FOUND) diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt index 761aeb5b17..13cb79129c 100644 --- a/paddle/api/test/CMakeLists.txt +++ b/paddle/api/test/CMakeLists.txt @@ -1,3 +1,8 @@ +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/testTrain.py + COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/*.py ${CMAKE_CURRENT_BINARY_DIR} +) +add_custom_target(copy_api_test ALL DEPENDS testTrain.py) + py_test(testTrain SRCS testTrain.py) py_test(testMatrix SRCS testMatrix.py) py_test(testVector SRCS testVector.py) diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index c425c71160..a473ed7400 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -74,8 +74,8 @@ py_proto_compile(framework_py_proto SRCS framework.proto) add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py) add_dependencies(framework_py_proto framework_py_proto_init) add_custom_command(TARGET framework_py_proto POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SOURCE_DIR}/python/paddle/fluid/proto - COMMAND cp *.py ${PADDLE_SOURCE_DIR}/python/paddle/fluid/proto/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_BINARY_DIR}/python/paddle/fluid/proto + COMMAND cp *.py ${PADDLE_BINARY_DIR}/python/paddle/fluid/proto/ COMMENT "Copy generated python proto into directory paddle/fluid/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/paddle/fluid/inference/tests/book/CMakeLists.txt b/paddle/fluid/inference/tests/book/CMakeLists.txt index e7ffb00ec8..6ed77adb9d 100644 --- a/paddle/fluid/inference/tests/book/CMakeLists.txt +++ b/paddle/fluid/inference/tests/book/CMakeLists.txt @@ -4,7 +4,7 @@ function(inference_test TARGET_NAME) set(multiValueArgs ARGS) cmake_parse_arguments(inference_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(PYTHON_TESTS_DIR ${PADDLE_SOURCE_DIR}/python/paddle/fluid/tests) + set(PYTHON_TESTS_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/tests) set(arg_list "") if(inference_test_ARGS) foreach(arg ${inference_test_ARGS}) diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt index 952ac8b1dc..84eabab563 100644 --- a/paddle/fluid/operators/CMakeLists.txt +++ b/paddle/fluid/operators/CMakeLists.txt @@ -3,8 +3,8 @@ string(REPLACE "_mkldnn" "" GENERAL_OPS "${GENERAL_OPS}") string(REPLACE ".cc" "" GENERAL_OPS "${GENERAL_OPS}") list(REMOVE_DUPLICATES GENERAL_OPS) set(DEPS_OPS "") -set(pybind_file ${PADDLE_SOURCE_DIR}/paddle/fluid/pybind/pybind.h) -file(WRITE ${pybind_file} "// Generated by the paddle/operator/CMakeLists.txt. DO NOT EDIT!\n\n") +set(pybind_file ${PADDLE_BINARY_DIR}/paddle/fluid/pybind/pybind.h) +file(WRITE ${pybind_file} "// Generated by the paddle/fluid/operator/CMakeLists.txt. DO NOT EDIT!\n\n") function(op_library TARGET) # op_library is a function to create op library. The interface is same as # cc_library. But it handle split GPU/CPU code and link some common library diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt index 686c088914..6780b8cc6d 100644 --- a/paddle/fluid/platform/CMakeLists.txt +++ b/paddle/fluid/platform/CMakeLists.txt @@ -6,8 +6,8 @@ add_custom_target(profiler_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch _ add_dependencies(profiler_py_proto profiler_py_proto_init) add_custom_command(TARGET profiler_py_proto POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SOURCE_DIR}/python/paddle/fluid/proto/profiler - COMMAND cp *.py ${PADDLE_SOURCE_DIR}/python/paddle/fluid/proto/profiler + COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_BINARY_DIR}/python/paddle/fluid/proto/profiler + COMMAND cp *.py ${PADDLE_BINARY_DIR}/python/paddle/fluid/proto/profiler COMMENT "Copy generated python proto into directory paddle/fluid/proto/profiler." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 9839375c22..9d7cad7584 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -14,6 +14,11 @@ function(gserver_test TARGET) COMMAND ${TARGET}) endfunction() +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/concat_dotmul_a.conf + COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/* ${CMAKE_CURRENT_BINARY_DIR} +) +add_custom_target(copy_gserver_conf ALL DEPENDS concat_dotmul_a.conf) + gserver_test(test_LayerGrad) gserver_test(test_CRFLayerGrad) gserver_test(test_CrossEntropyOverBeamGrad) @@ -31,12 +36,12 @@ gserver_test(test_Upsample) set(PYTHON_PATH ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d - ${PADDLE_SOURCE_DIR}/python/:${PADDLE_SOURCE_DIR}/paddle/gserver/tests) + ${PADDLE_BINARY_DIR}/python/:${PADDLE_BINARY_DIR}/paddle/gserver/tests) function(gserver_test_with_python TARGET) add_unittest_without_exec(${TARGET} ${TARGET}.cpp) add_test(NAME ${TARGET} COMMAND ${PYTHON_PATH} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET} - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle/) endfunction() gserver_test_with_python(test_PyDataProvider2) @@ -57,7 +62,7 @@ if(WITH_MKLDNN) LayerGradUtil.cpp) add_test(NAME test_MKLDNN COMMAND ${PYTHON_PATH} ${CMAKE_CURRENT_BINARY_DIR}/test_MKLDNN - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle) endif() ############### test_WarpCTCLayer ####################### @@ -66,7 +71,7 @@ if(NOT WITH_DOUBLE AND NOT MOBILE_INFERENCE) test_WarpCTCLayer.cpp) add_test(NAME test_WarpCTCLayer COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_WarpCTCLayer --warpctc_dir=${WARPCTC_LIB_DIR} - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle) endif() if(NOT MOBILE_INFERENCE) @@ -84,15 +89,15 @@ if(NOT MOBILE_INFERENCE) endif() add_test(NAME test_NetworkCompare COMMAND ${PYTHON_PATH} ${CMAKE_CURRENT_BINARY_DIR}/test_NetworkCompare --use_gpu=${use_gpu} - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle) ############ test_CompareSparse ################ add_unittest_without_exec(test_CompareSparse test_CompareSparse.cpp) if(NOT ON_TRAVIS) add_test(NAME test_CompareSparse - COMMAND ${PYTHON_PATH} ./.set_port.sh -p port -n 6 + COMMAND ${PYTHON_PATH} ${PADDLE_SOURCE_DIR}/paddle/.set_port.sh -p port -n 6 ${CMAKE_CURRENT_BINARY_DIR}/test_CompareSparse - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle/) endif() endif() diff --git a/paddle/trainer/tests/CMakeLists.txt b/paddle/trainer/tests/CMakeLists.txt index bd518d8598..12c9ea8cef 100644 --- a/paddle/trainer/tests/CMakeLists.txt +++ b/paddle/trainer/tests/CMakeLists.txt @@ -1,11 +1,16 @@ +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sample_trainer_config.conf + COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/* ${CMAKE_CURRENT_BINARY_DIR} +) +add_custom_target(copy_trainer_conf ALL DEPENDS sample_trainer_config.conf) + set(PYTHON_PATH ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d - ${PADDLE_SOURCE_DIR}/python/:${PADDLE_SOURCE_DIR}/paddle/trainer/tests) + ${PADDLE_BINARY_DIR}/python/:${PADDLE_BINARY_DIR}/paddle/trainer/tests) function(trainer_test TARGET) add_unittest_without_exec(${TARGET} ${TARGET}.cpp) add_test(NAME ${TARGET} COMMAND ${PYTHON_PATH} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET} - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle/) endfunction() trainer_test(test_Compare) @@ -22,11 +27,11 @@ if(WITH_PYTHON) add_test(NAME test_TrainerOnePass COMMAND ${PYTHON_PATH} ${PADDLE_SOURCE_DIR}/paddle/.set_port.sh -p port ${CMAKE_CURRENT_BINARY_DIR}/test_TrainerOnePass - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle/) endif() #################### test_config_parser ######################### add_test(NAME test_config_parser COMMAND ${PYTHON_PATH} ${PYTHON_EXECUTABLE} ${PADDLE_SOURCE_DIR}/paddle/trainer/tests/config_parser_test.py - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/) + WORKING_DIRECTORY ${PADDLE_BINARY_DIR}/paddle/) diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index 7a4977935e..6292e7fa52 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -2,8 +2,8 @@ file(GLOB UTIL_HEADERS . *.h) file(GLOB UTIL_SOURCES . *.cpp) create_resources(${CMAKE_CURRENT_SOURCE_DIR}/enable_virtualenv.py - ${CMAKE_CURRENT_SOURCE_DIR}/enable_virtualenv.c) -set(UTIL_RES ${CMAKE_CURRENT_SOURCE_DIR}/enable_virtualenv.c) + ${CMAKE_CURRENT_BINARY_DIR}/enable_virtualenv.c) +set(UTIL_RES ${CMAKE_CURRENT_BINARY_DIR}/enable_virtualenv.c) if(APPLE) file(GLOB UTIL_ARCH_SOURCES . arch/osx/*.cpp) diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 556bcd1d7e..a075eeb83b 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -15,13 +15,14 @@ foreach(filename ${proto_filenames}) get_filename_component(ABS_FIL ${filename} ABSOLUTE) get_filename_component(FIL_WE ${filename} NAME_WE) set(CUR_PROTO_GEN_PY - ${PADDLE_SOURCE_DIR}/paddle/python/paddle/proto/${FIL_WE}_pb2.py) + ${PADDLE_BINARY_DIR}/paddle/python/paddle/proto/${FIL_WE}_pb2.py) set(PROTO_GEN_PY ${CUR_PROTO_GEN_PY} ${PROTO_GEN_PY}) add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY} + COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_BINARY_DIR}/python/paddle/proto COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS "--python_out=${PADDLE_SOURCE_DIR}/python/paddle/proto" + ARGS "--python_out=${PADDLE_BINARY_DIR}/python/paddle/proto" "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} DEPENDS ${ABS_FIL} protoc) endforeach() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index d074b0136d..7cbd7f22bf 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -47,14 +47,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py) -add_custom_command(OUTPUT ${PADDLE_SOURCE_DIR}/python/paddle/fluid/core.so - COMMAND cmake -E copy $ ${PADDLE_SOURCE_DIR}/python/paddle/fluid/core.so +add_custom_command(OUTPUT ${PADDLE_BINARY_DIR}/python/paddle/fluid/core.so + COMMAND cmake -E copy $ ${PADDLE_BINARY_DIR}/python/paddle/fluid/core.so DEPENDS paddle_pybind) -add_custom_target(copy_paddle_pybind ALL DEPENDS ${PADDLE_SOURCE_DIR}/python/paddle/fluid/core.so) +add_custom_target(copy_paddle_pybind ALL DEPENDS ${PADDLE_BINARY_DIR}/python/paddle/fluid/core.so) add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp COMMAND touch stub.cc + COMMAND ${CMAKE_COMMAND} -E copy_directory ${PADDLE_SOURCE_DIR}/python/paddle ${PADDLE_BINARY_DIR}/python/paddle + COMMAND cp -r ${PADDLE_SOURCE_DIR}/paddle/py_paddle ${PADDLE_BINARY_DIR}/python/ COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp COMMAND ${CMAKE_COMMAND} -E remove_directory ${PADDLE_PYTHON_BUILD_DIR}/lib-python diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 1b2d29a47f..f10ef9b634 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -22,9 +22,9 @@ function(py_test_modules TARGET_NAME) set(multiValueArgs MODULES DEPS ARGS ENVS) cmake_parse_arguments(py_test_modules "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${PADDLE_PYTHON_BUILD_DIR}/lib-python ${py_test_modules_ENVS} + COMMAND env PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_modules_ENVS} ${PYTHON_EXECUTABLE} -u -m unittest --verbose ${py_test_modules_MODULES} ${py_test_modules_ARGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() endfunction() diff --git a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt index 580aef935b..30e0b9906c 100644 --- a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt +++ b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt @@ -1,17 +1,17 @@ #################### test_config_parser ######################### add_test(NAME layers_test - COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python/ + COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_BINARY_DIR}/python/ ${PYTHON_EXECUTABLE} ${PADDLE_SOURCE_DIR}/python/paddle/trainer_config_helpers/tests/layers_test.py WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/python/paddle) add_test(NAME test_reset_hook - COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python/ + COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_BINARY_DIR}/python/ ${PYTHON_EXECUTABLE} ${PADDLE_SOURCE_DIR}/python/paddle/trainer_config_helpers/tests/test_reset_hook.py WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/python/paddle) add_paddle_exe(protobuf_equal ProtobufEqualMain.cpp) add_test(NAME test_layerHelpers - COMMAND - ${PADDLE_SOURCE_DIR}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh ${PYTHON_EXECUTABLE} + COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_BINARY_DIR}/python/ + ${PADDLE_BINARY_DIR}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/protobuf_equal ) diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 8a31887963..44a75a60cc 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -2,7 +2,6 @@ set -e cd `dirname $0` -export PYTHONPATH=$PWD/../../../../ protostr=$PWD/protostr . file_list.sh diff --git a/python/setup.py.in b/python/setup.py.in index 08a448934d..2707d34a2a 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -58,7 +58,7 @@ def mkl(): 'istaged': ISTAGED, 'with_mkl': '@WITH_MKL@'}) -write_version_py(filename='@PADDLE_SOURCE_DIR@/python/paddle/version.py') +write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py') packages=['paddle', @@ -109,7 +109,7 @@ package_dir={ 'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework', } if '${WITH_FLUID_ONLY}'== 'OFF': - package_dir['py_paddle']='${PADDLE_SOURCE_DIR}/paddle/py_paddle' + package_dir['py_paddle']='${PADDLE_BINARY_DIR}/python/py_paddle' paddle_rt_lib_dir = 'lib' From 0935f2cf05954b97caae5b0216fda5f503b5e481 Mon Sep 17 00:00:00 2001 From: Xin Pan Date: Thu, 5 Apr 2018 19:59:10 -0700 Subject: [PATCH 32/57] shrink test_parallel_executor size. --- .../fluid/tests/unittests/test_parallel_executor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor.py b/python/paddle/fluid/tests/unittests/test_parallel_executor.py index 0cbef82e33..0f90e0e4df 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor.py @@ -144,7 +144,7 @@ def bottleneck_block(input, num_filters, stride, cardinality, reduction_ratio): return fluid.layers.elementwise_add(x=short, y=scale, act='relu') -def SE_ResNeXt152Small(batch_size=2, use_feed=False): +def SE_ResNeXt50Small(batch_size=2, use_feed=False): assert not use_feed, "SE_ResNeXt doesn't support feed yet" img = fluid.layers.fill_constant( @@ -161,9 +161,9 @@ def SE_ResNeXt152Small(batch_size=2, use_feed=False): conv = fluid.layers.pool2d( input=conv, pool_size=3, pool_stride=2, pool_padding=1, pool_type='max') - cardinality = 64 + cardinality = 32 reduction_ratio = 16 - depth = [3, 8, 36, 3] + depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] for block in range(len(depth)): @@ -290,7 +290,7 @@ class TestResnet(TestParallelExecutorBase): batch_size = 2 self.check_network_convergence( functools.partial( - SE_ResNeXt152Small, batch_size=batch_size), + SE_ResNeXt50Small, batch_size=batch_size), iter=20, batch_size=batch_size) From 4ff237f93c85521fbd69ac618735de3acdd822e2 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 6 Apr 2018 10:22:22 +0800 Subject: [PATCH 33/57] follow comments --- paddle/fluid/framework/tensor_impl.h | 5 ++--- paddle/fluid/pybind/tensor_py.h | 14 ++++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/framework/tensor_impl.h b/paddle/fluid/framework/tensor_impl.h index 07d0906ea7..f49d1a47a3 100644 --- a/paddle/fluid/framework/tensor_impl.h +++ b/paddle/fluid/framework/tensor_impl.h @@ -132,8 +132,7 @@ inline void* Tensor::mutable_data(platform::Place place, std::type_index type) { platform::is_cuda_pinned_place(place)) { #ifndef PADDLE_WITH_CUDA PADDLE_THROW( - "'CUDAPlace' or 'CUDAPinnedPlace' is not supported in CPU only " - "device."); + "CUDAPlace or CUDAPinnedPlace is not supported in CPU-only mode."); } #else if (platform::is_gpu_place(place)) { @@ -153,7 +152,7 @@ inline void* Tensor::mutable_data(platform::Place place, std::type_index type) { inline void* Tensor::mutable_data(platform::Place place) { PADDLE_ENFORCE(this->holder_ != nullptr, - "Cannot invoke mutable data if current hold nothing"); + "Cannot invoke mutable data if current hold nothing."); return mutable_data(place, holder_->type()); } diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h index f52ffc9ef3..868966433e 100644 --- a/paddle/fluid/pybind/tensor_py.h +++ b/paddle/fluid/pybind/tensor_py.h @@ -143,7 +143,7 @@ void PyCPUTensorSetFromArray( std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { - dims.push_back((int)array.shape()[i]); + dims.push_back(static_cast(array.shape()[i])); } self.Resize(framework::make_ddim(dims)); @@ -152,6 +152,8 @@ void PyCPUTensorSetFromArray( } template <> +// This following specialization maps uint16_t in the parameter type to +// platform::float16. void PyCPUTensorSetFromArray( framework::Tensor &self, py::array_t array, @@ -159,7 +161,7 @@ void PyCPUTensorSetFromArray( std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { - dims.push_back((int)array.shape()[i]); + dims.push_back(static_cast(array.shape()[i])); } self.Resize(framework::make_ddim(dims)); @@ -176,7 +178,7 @@ void PyCUDATensorSetFromArray( std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { - dims.push_back((int)array.shape()[i]); + dims.push_back(static_cast(array.shape()[i])); } self.Resize(framework::make_ddim(dims)); @@ -190,6 +192,8 @@ void PyCUDATensorSetFromArray( } template <> +// This following specialization maps uint16_t in the parameter type to +// platform::float16. void PyCUDATensorSetFromArray( framework::Tensor &self, py::array_t array, @@ -197,7 +201,7 @@ void PyCUDATensorSetFromArray( std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { - dims.push_back((int)array.shape()[i]); + dims.push_back(static_cast(array.shape()[i])); } self.Resize(framework::make_ddim(dims)); @@ -228,6 +232,8 @@ void PyCUDAPinnedTensorSetFromArray( } template <> +// This following specialization maps uint16_t in the parameter type to +// platform::float16. void PyCUDAPinnedTensorSetFromArray( framework::Tensor &self, py::array_t array, From b5b7ea12fa0ba6da57fed0e739ec2cf094ad6c3b Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Fri, 6 Apr 2018 00:39:36 -0700 Subject: [PATCH 34/57] Fix CPPLint issues in tuple.h (#9670) --- paddle/fluid/framework/tuple.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/framework/tuple.h b/paddle/fluid/framework/tuple.h index 78996908b1..f6c6a1fec1 100644 --- a/paddle/fluid/framework/tuple.h +++ b/paddle/fluid/framework/tuple.h @@ -35,24 +35,25 @@ class Tuple { public: using ElementVars = std::vector; - Tuple(std::vector& var, std::vector& var_desc) + Tuple(const std::vector& var, + const std::vector& var_desc) : var_(var), var_desc_(var_desc) {} - Tuple(std::vector& var) : var_(var) {} + explicit Tuple(std::vector& var) : var_(var) {} - ElementVar get(int idx) const { return var_[idx]; }; + ElementVar get(int idx) const { return var_[idx]; } - ElementVar& get(int idx) { return var_[idx]; }; + ElementVar& get(int idx) { return var_[idx]; } - bool isSameType(Tuple& t) const; + bool isSameType(const Tuple& t) const; - size_t getSize() const { return var_.size(); }; + size_t getSize() const { return var_.size(); } private: ElementVars var_; std::vector var_desc_; }; -bool Tuple::isSameType(Tuple& t) const { +bool Tuple::isSameType(const Tuple& t) const { size_t tuple_size = getSize(); if (tuple_size != t.getSize()) { return false; From f53beedd13747e9925a523d1a2ffff36e3bb7893 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Fri, 6 Apr 2018 01:30:15 -0700 Subject: [PATCH 35/57] Fix issue 9673 (#9674) --- paddle/gserver/tests/test_Upsample.cpp | 85 +++++++++++++------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/paddle/gserver/tests/test_Upsample.cpp b/paddle/gserver/tests/test_Upsample.cpp index 9d6fa1d130..39b902fcc7 100644 --- a/paddle/gserver/tests/test_Upsample.cpp +++ b/paddle/gserver/tests/test_Upsample.cpp @@ -20,10 +20,8 @@ limitations under the License. */ #include "paddle/math/MathUtils.h" #include "paddle/testing/TestUtil.h" -using namespace paddle; - -void setPoolConfig(TestConfig* config, - PoolConfig* pool, +void setPoolConfig(paddle::TestConfig* config, + paddle::PoolConfig* pool, const string& poolType) { (*config).biasSize = 0; (*config).layerConfig.set_type("pool"); @@ -42,21 +40,23 @@ void setPoolConfig(TestConfig* config, pool->set_stride(sw); pool->set_stride_y(sh); - int ow = outputSize(pool->img_size(), kw, pw, sw, /* caffeMode */ false); - int oh = outputSize(pool->img_size_y(), kh, ph, sh, /* caffeMode */ false); + int ow = + paddle::outputSize(pool->img_size(), kw, pw, sw, /* caffeMode */ false); + int oh = + paddle::outputSize(pool->img_size_y(), kh, ph, sh, /* caffeMode */ false); pool->set_output_x(ow); pool->set_output_y(oh); } -LayerPtr doOneUpsampleTest(MatrixPtr& inputMat, - const string& poolType, - bool use_gpu, - real* tempGradData) { +paddle::LayerPtr doOneUpsampleTest(const paddle::MatrixPtr& inputMat, + const string& poolType, + bool use_gpu, + real* tempGradData) { /* prepare maxPoolWithMaskLayer */ - TestConfig config; - config.inputDefs.push_back({INPUT_DATA, "layer_0", 128, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - PoolConfig* pool = input->mutable_pool_conf(); + paddle::TestConfig config; + config.inputDefs.push_back({paddle::INPUT_DATA, "layer_0", 128, 0}); + paddle::LayerInputConfig* input = config.layerConfig.add_inputs(); + paddle::PoolConfig* pool = input->mutable_pool_conf(); pool->set_img_size(8); pool->set_img_size_y(8); @@ -66,9 +66,9 @@ LayerPtr doOneUpsampleTest(MatrixPtr& inputMat, config.layerConfig.set_name("MaxPoolWithMask"); - std::vector dataLayers; - LayerMap layerMap; - vector datas; + std::vector dataLayers; + paddle::LayerMap layerMap; + vector datas; initDataLayer(config, &dataLayers, @@ -82,20 +82,20 @@ LayerPtr doOneUpsampleTest(MatrixPtr& inputMat, dataLayers[0]->getOutputValue()->copyFrom(*inputMat); FLAGS_use_gpu = use_gpu; - std::vector parameters; - LayerPtr maxPoolingWithMaskOutputLayer; + std::vector parameters; + paddle::LayerPtr maxPoolingWithMaskOutputLayer; initTestLayer(config, &layerMap, ¶meters, &maxPoolingWithMaskOutputLayer); - maxPoolingWithMaskOutputLayer->forward(PASS_GC); + maxPoolingWithMaskOutputLayer->forward(paddle::PASS_GC); /* prepare the upsample layer */ - LayerConfig upsampleLayerConfig; + paddle::LayerConfig upsampleLayerConfig; upsampleLayerConfig.set_type("upsample"); - LayerInputConfig* input1 = upsampleLayerConfig.add_inputs(); + paddle::LayerInputConfig* input1 = upsampleLayerConfig.add_inputs(); upsampleLayerConfig.add_inputs(); - UpsampleConfig* upsampleConfig = input1->mutable_upsample_conf(); + paddle::UpsampleConfig* upsampleConfig = input1->mutable_upsample_conf(); upsampleConfig->set_scale(2); - ImageConfig* imageConfig = upsampleConfig->mutable_image_conf(); + paddle::ImageConfig* imageConfig = upsampleConfig->mutable_image_conf(); imageConfig->set_channels(2); imageConfig->set_img_size(4); imageConfig->set_img_size_y(4); @@ -103,17 +103,18 @@ LayerPtr doOneUpsampleTest(MatrixPtr& inputMat, upsampleLayerConfig.set_name("upsample"); for (size_t i = 0; i < 2; i++) { - LayerInputConfig& inputTemp = *(upsampleLayerConfig.mutable_inputs(i)); + paddle::LayerInputConfig& inputTemp = + *(upsampleLayerConfig.mutable_inputs(i)); inputTemp.set_input_layer_name("MaxPoolWithMask"); } - LayerPtr upsampleLayer; - ParameterMap parameterMap; - upsampleLayer = Layer::create(upsampleLayerConfig); + paddle::LayerPtr upsampleLayer; + paddle::ParameterMap parameterMap; + upsampleLayer = paddle::Layer::create(upsampleLayerConfig); layerMap[upsampleLayerConfig.name()] = upsampleLayer; upsampleLayer->init(layerMap, parameterMap); upsampleLayer->setNeedGradient(true); - upsampleLayer->forward(PASS_GC); + upsampleLayer->forward(paddle::PASS_GC); upsampleLayer->getOutputGrad()->copyFrom(tempGradData, 128); upsampleLayer->backward(); @@ -122,31 +123,31 @@ LayerPtr doOneUpsampleTest(MatrixPtr& inputMat, TEST(Layer, maxPoolingWithMaskOutputLayerFwd) { bool useGpu = false; - MatrixPtr inputMat; - MatrixPtr inputGPUMat; - MatrixPtr tempGradMat; + paddle::MatrixPtr inputMat; + paddle::MatrixPtr inputGPUMat; + paddle::MatrixPtr tempGradMat; - inputMat = Matrix::create(1, 128, false, useGpu); + inputMat = paddle::Matrix::create(1, 128, false, useGpu); inputMat->randomizeUniform(); - tempGradMat = Matrix::create(1, 128, false, useGpu); + tempGradMat = paddle::Matrix::create(1, 128, false, useGpu); tempGradMat->randomizeUniform(); - real* data = inputMat->getData(); real* tempGradData = tempGradMat->getData(); - LayerPtr upsampleLayerCPU = + paddle::LayerPtr upsampleLayerCPU = doOneUpsampleTest(inputMat, "max-pool-with-mask", useGpu, tempGradData); #ifdef PADDLE_WITH_CUDA useGpu = true; - inputGPUMat = Matrix::create(1, 128, false, useGpu); + real* data = inputMat->getData(); + inputGPUMat = paddle::Matrix::create(1, 128, false, useGpu); inputGPUMat->copyFrom(data, 128); - LayerPtr upsampleLayerGPU = doOneUpsampleTest( + paddle::LayerPtr upsampleLayerGPU = doOneUpsampleTest( inputGPUMat, "max-pool-with-mask", useGpu, tempGradData); - checkMatrixEqual(upsampleLayerCPU->getOutput("").value, - upsampleLayerGPU->getOutput("").value); + paddle::checkMatrixEqual(upsampleLayerCPU->getOutput("").value, + upsampleLayerGPU->getOutput("").value); - checkMatrixEqual(upsampleLayerCPU->getPrev(0)->getOutputGrad(), - upsampleLayerGPU->getPrev(0)->getOutputGrad()); + paddle::checkMatrixEqual(upsampleLayerCPU->getPrev(0)->getOutputGrad(), + upsampleLayerGPU->getPrev(0)->getOutputGrad()); #endif } From 517f619501f6b0a177a184f949e1273fa7693e68 Mon Sep 17 00:00:00 2001 From: lgone2000 Date: Sat, 7 Apr 2018 01:04:40 +0800 Subject: [PATCH 36/57] fix pybind.cc compile error (#9681) --- paddle/fluid/pybind/pybind.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 046721970a..9bc3ff5128 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -11,6 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include #include #include #include // NOLINT // for call_once From d00bd9eb7256b2022b11491374e94adfc82e720c Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Fri, 6 Apr 2018 10:19:31 -0700 Subject: [PATCH 37/57] Update the cuda API and enable tensor core for GEMM (#9622) * change from hgemm to gemmEx * fix cpplint --- paddle/fluid/operators/math/math_function.cu | 33 ++++++++++++++------ paddle/fluid/platform/dynload/cublas.cc | 4 +++ paddle/fluid/platform/dynload/cublas.h | 14 +++++++-- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/operators/math/math_function.cu b/paddle/fluid/operators/math/math_function.cu index 1e909db528..82e1294314 100644 --- a/paddle/fluid/operators/math/math_function.cu +++ b/paddle/fluid/operators/math/math_function.cu @@ -39,18 +39,33 @@ void gemm( cublasOperation_t cuTransB = (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; - const half h_alpha = static_cast(alpha); - const half h_beta = static_cast(beta); - const half* h_A = reinterpret_cast(A); - const half* h_B = reinterpret_cast(B); - half* h_C = reinterpret_cast(C); + float h_alpha = static_cast(alpha); + float h_beta = static_cast(beta); // TODO(kexinzhao): add processing code for compute capability < 53 case PADDLE_ENFORCE_GE(context.GetComputeCapability(), 53, - "cublas Hgemm requires GPU compute capability >= 53"); - PADDLE_ENFORCE(platform::dynload::cublasHgemm( - context.cublas_handle(), cuTransB, cuTransA, N, M, K, &h_alpha, h_B, ldb, - h_A, lda, &h_beta, h_C, N)); + "cublas fp16 gemm requires GPU compute capability >= 53"); + + cublasGemmAlgo_t algo = CUBLAS_GEMM_DFALT; +#if CUDA_VERSION >= 9000 + if (context.GetComputeCapability() >= 70) { + PADDLE_ENFORCE(platform::dynload::cublasSetMathMode(context.cublas_handle(), + CUBLAS_TENSOR_OP_MATH)); + algo = CUBLAS_GEMM_DFALT_TENSOR_OP; + } else { + PADDLE_ENFORCE(platform::dynload::cublasSetMathMode(context.cublas_handle(), + CUBLAS_DEFAULT_MATH)); + } +#endif + + // cublasHgemm does true FP16 computation which is slow for non-Volta + // GPUs. So use cublasGemmEx instead which does pesudo FP16 computation: + // input/output in fp16, computation in fp32, which can also be accelerated + // using tensor cores in volta GPUs. + PADDLE_ENFORCE(platform::dynload::cublasGemmEx( + context.cublas_handle(), cuTransB, cuTransA, N, M, K, &h_alpha, B, + CUDA_R_16F, ldb, A, CUDA_R_16F, lda, &h_beta, C, CUDA_R_16F, N, + CUDA_R_32F, algo)); } template <> diff --git a/paddle/fluid/platform/dynload/cublas.cc b/paddle/fluid/platform/dynload/cublas.cc index e90e3105f0..eb541579a1 100644 --- a/paddle/fluid/platform/dynload/cublas.cc +++ b/paddle/fluid/platform/dynload/cublas.cc @@ -24,6 +24,10 @@ void *cublas_dso_handle = nullptr; CUBLAS_BLAS_ROUTINE_EACH(DEFINE_WRAP); +#ifdef CUBLAS_BLAS_ROUTINE_EACH_R2 +CUBLAS_BLAS_ROUTINE_EACH_R2(DEFINE_WRAP); +#endif + } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/fluid/platform/dynload/cublas.h b/paddle/fluid/platform/dynload/cublas.h index fa9041134d..3b8d192b6c 100644 --- a/paddle/fluid/platform/dynload/cublas.h +++ b/paddle/fluid/platform/dynload/cublas.h @@ -15,8 +15,9 @@ limitations under the License. */ #pragma once #include +#include #include -#include +#include // NOLINT #include "paddle/fluid/platform/dynload/dynamic_loader.h" namespace paddle { @@ -70,6 +71,7 @@ extern void *cublas_dso_handle; __macro(cublasDgemm_v2); \ __macro(cublasHgemm); \ __macro(cublasSgemmEx); \ + __macro(cublasGemmEx); \ __macro(cublasSgeam_v2); \ __macro(cublasDgeam_v2); \ __macro(cublasCreate_v2); \ @@ -89,9 +91,15 @@ extern void *cublas_dso_handle; __macro(cublasSgetrfBatched); \ __macro(cublasSgetriBatched); \ __macro(cublasDgetrfBatched); \ - __macro(cublasDgetriBatched) + __macro(cublasDgetriBatched); -CUBLAS_BLAS_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP); +CUBLAS_BLAS_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP) + +// APIs available after CUDA 9.0 +#if CUDA_VERSION >= 9000 +#define CUBLAS_BLAS_ROUTINE_EACH_R2(__macro) __macro(cublasSetMathMode); +CUBLAS_BLAS_ROUTINE_EACH_R2(DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP) +#endif #undef DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP } // namespace dynload From 9f3ac225ada3f9222b05085340c3b4b10965b911 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Apr 2018 11:18:10 -0700 Subject: [PATCH 38/57] Unify Fluid code to Google C++ style --- paddle/fluid/{framework => }/.clang-format | 0 paddle/fluid/inference/io.cc | 9 ++---- paddle/fluid/inference/io.h | 3 +- .../tests/book/test_inference_fit_a_line.cc | 4 +-- .../test_inference_image_classification.cc | 14 ++++---- .../test_inference_label_semantic_roles.cc | 32 +++++-------------- .../book/test_inference_recognize_digits.cc | 14 ++++---- .../test_inference_rnn_encoder_decoder.cc | 8 ++--- .../test_inference_understand_sentiment.cc | 4 +-- paddle/fluid/inference/tests/test_helper.h | 25 +++++---------- paddle/fluid/memory/.clang-format | 5 --- paddle/fluid/operators/.clang-format | 5 --- paddle/fluid/platform/.clang-format | 5 --- paddle/fluid/pybind/.clang-format | 5 --- paddle/fluid/recordio/chunk.cc | 8 ++--- paddle/fluid/recordio/chunk.h | 4 +-- paddle/fluid/recordio/header.h | 4 +-- paddle/fluid/recordio/scanner.h | 4 +-- paddle/fluid/recordio/writer.h | 7 ++-- paddle/fluid/string/.clang-format | 1 - 20 files changed, 52 insertions(+), 109 deletions(-) rename paddle/fluid/{framework => }/.clang-format (100%) delete mode 100644 paddle/fluid/memory/.clang-format delete mode 100644 paddle/fluid/operators/.clang-format delete mode 100644 paddle/fluid/platform/.clang-format delete mode 100644 paddle/fluid/pybind/.clang-format delete mode 120000 paddle/fluid/string/.clang-format diff --git a/paddle/fluid/framework/.clang-format b/paddle/fluid/.clang-format similarity index 100% rename from paddle/fluid/framework/.clang-format rename to paddle/fluid/.clang-format diff --git a/paddle/fluid/inference/io.cc b/paddle/fluid/inference/io.cc index 52e9c0baa6..a5b62ef322 100644 --- a/paddle/fluid/inference/io.cc +++ b/paddle/fluid/inference/io.cc @@ -41,8 +41,7 @@ bool IsPersistable(const framework::VarDesc* var) { return false; } -void LoadPersistables(framework::Executor& executor, - framework::Scope& scope, +void LoadPersistables(framework::Executor& executor, framework::Scope& scope, const framework::ProgramDesc& main_program, const std::string& dirname, const std::string& param_filename) { @@ -108,10 +107,8 @@ std::unique_ptr Load(framework::Executor& executor, } std::unique_ptr Load( - framework::Executor& executor, - framework::Scope& scope, - const std::string& prog_filename, - const std::string& param_filename) { + framework::Executor& executor, framework::Scope& scope, + const std::string& prog_filename, const std::string& param_filename) { std::string model_filename = prog_filename; std::string program_desc_str; ReadBinaryFile(model_filename, program_desc_str); diff --git a/paddle/fluid/inference/io.h b/paddle/fluid/inference/io.h index 6817a6fca0..d07d315b93 100644 --- a/paddle/fluid/inference/io.h +++ b/paddle/fluid/inference/io.h @@ -24,8 +24,7 @@ limitations under the License. */ namespace paddle { namespace inference { -void LoadPersistables(framework::Executor& executor, - framework::Scope& scope, +void LoadPersistables(framework::Executor& executor, framework::Scope& scope, const framework::ProgramDesc& main_program, const std::string& dirname, const std::string& param_filename); diff --git a/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc b/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc index 9ab808efec..8d8365a839 100644 --- a/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc +++ b/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc @@ -30,8 +30,8 @@ TEST(inference, fit_a_line) { // The second dim of the input tensor should be 13 // The input data should be >= 0 int64_t batch_size = 10; - SetupTensor( - input, {batch_size, 13}, static_cast(0), static_cast(10)); + SetupTensor(input, {batch_size, 13}, static_cast(0), + static_cast(10)); std::vector cpu_feeds; cpu_feeds.push_back(&input); diff --git a/paddle/fluid/inference/tests/book/test_inference_image_classification.cc b/paddle/fluid/inference/tests/book/test_inference_image_classification.cc index e9a27171f1..954ca7a3e3 100644 --- a/paddle/fluid/inference/tests/book/test_inference_image_classification.cc +++ b/paddle/fluid/inference/tests/book/test_inference_image_classification.cc @@ -35,10 +35,8 @@ TEST(inference, image_classification) { paddle::framework::LoDTensor input; // Use normilized image pixels as input data, // which should be in the range [0.0, 1.0]. - SetupTensor(input, - {FLAGS_batch_size, 3, 32, 32}, - static_cast(0), - static_cast(1)); + SetupTensor(input, {FLAGS_batch_size, 3, 32, 32}, + static_cast(0), static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&input); @@ -48,8 +46,8 @@ TEST(inference, image_classification) { // Run inference on CPU LOG(INFO) << "--- CPU Runs: ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs1, FLAGS_repeat); + TestInference(dirname, cpu_feeds, cpu_fetchs1, + FLAGS_repeat); LOG(INFO) << output1.dims(); #ifdef PADDLE_WITH_CUDA @@ -59,8 +57,8 @@ TEST(inference, image_classification) { // Run inference on CUDA GPU LOG(INFO) << "--- GPU Runs: ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs2, FLAGS_repeat); + TestInference(dirname, cpu_feeds, cpu_fetchs2, + FLAGS_repeat); LOG(INFO) << output2.dims(); CheckError(output1, output2); diff --git a/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc b/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc index 1849240166..31100494ff 100644 --- a/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc +++ b/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc @@ -36,37 +36,21 @@ TEST(inference, label_semantic_roles) { int64_t predicate_dict_len = 3162; int64_t mark_dict_len = 2; - SetupLoDTensor(word, - lod, - static_cast(0), + SetupLoDTensor(word, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(predicate, - lod, - static_cast(0), + SetupLoDTensor(predicate, lod, static_cast(0), static_cast(predicate_dict_len - 1)); - SetupLoDTensor(ctx_n2, - lod, - static_cast(0), + SetupLoDTensor(ctx_n2, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_n1, - lod, - static_cast(0), + SetupLoDTensor(ctx_n1, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_0, - lod, - static_cast(0), + SetupLoDTensor(ctx_0, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_p1, - lod, - static_cast(0), + SetupLoDTensor(ctx_p1, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_p2, - lod, - static_cast(0), + SetupLoDTensor(ctx_p2, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(mark, - lod, - static_cast(0), + SetupLoDTensor(mark, lod, static_cast(0), static_cast(mark_dict_len - 1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc b/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc index 1fb0f9e777..82275d3ee1 100644 --- a/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc +++ b/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc @@ -35,10 +35,8 @@ TEST(inference, recognize_digits) { paddle::framework::LoDTensor input; // Use normilized image pixels as input data, // which should be in the range [-1.0, 1.0]. - SetupTensor(input, - {FLAGS_batch_size, 1, 28, 28}, - static_cast(-1), - static_cast(1)); + SetupTensor(input, {FLAGS_batch_size, 1, 28, 28}, + static_cast(-1), static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&input); @@ -49,8 +47,8 @@ TEST(inference, recognize_digits) { // Run inference on CPU LOG(INFO) << "--- CPU Runs: is_combined=" << is_combined << " ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs1, FLAGS_repeat, is_combined); + TestInference(dirname, cpu_feeds, cpu_fetchs1, + FLAGS_repeat, is_combined); LOG(INFO) << output1.dims(); #ifdef PADDLE_WITH_CUDA @@ -60,8 +58,8 @@ TEST(inference, recognize_digits) { // Run inference on CUDA GPU LOG(INFO) << "--- GPU Runs: is_combined=" << is_combined << " ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs2, FLAGS_repeat, is_combined); + TestInference(dirname, cpu_feeds, cpu_fetchs2, + FLAGS_repeat, is_combined); LOG(INFO) << output2.dims(); CheckError(output1, output2); diff --git a/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc b/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc index a0523905bd..ea2d5ee092 100644 --- a/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc +++ b/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc @@ -32,10 +32,10 @@ TEST(inference, rnn_encoder_decoder) { paddle::framework::LoDTensor word_data, trg_word; paddle::framework::LoD lod{{0, 4, 10}}; - SetupLoDTensor( - word_data, lod, static_cast(0), static_cast(1)); - SetupLoDTensor( - trg_word, lod, static_cast(0), static_cast(1)); + SetupLoDTensor(word_data, lod, static_cast(0), + static_cast(1)); + SetupLoDTensor(trg_word, lod, static_cast(0), + static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&word_data); diff --git a/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc b/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc index 824b3274eb..21ffd8d368 100644 --- a/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc +++ b/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc @@ -33,9 +33,7 @@ TEST(inference, understand_sentiment) { paddle::framework::LoD lod{{0, 4, 10}}; int64_t word_dict_len = 5147; - SetupLoDTensor(words, - lod, - static_cast(0), + SetupLoDTensor(words, lod, static_cast(0), static_cast(word_dict_len - 1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/test_helper.h b/paddle/fluid/inference/tests/test_helper.h index dce541c097..d8ffedf672 100644 --- a/paddle/fluid/inference/tests/test_helper.h +++ b/paddle/fluid/inference/tests/test_helper.h @@ -19,9 +19,7 @@ limitations under the License. */ template void SetupTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, - T lower, - T upper) { + paddle::framework::DDim dims, T lower, T upper) { srand(time(0)); T* input_ptr = input.mutable_data(dims, paddle::platform::CPUPlace()); for (int i = 0; i < input.numel(); ++i) { @@ -33,8 +31,7 @@ void SetupTensor(paddle::framework::LoDTensor& input, template void SetupTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, - std::vector& data) { + paddle::framework::DDim dims, std::vector& data) { CHECK_EQ(paddle::framework::product(dims), static_cast(data.size())); T* input_ptr = input.mutable_data(dims, paddle::platform::CPUPlace()); memcpy(input_ptr, data.data(), input.numel() * sizeof(T)); @@ -42,9 +39,7 @@ void SetupTensor(paddle::framework::LoDTensor& input, template void SetupLoDTensor(paddle::framework::LoDTensor& input, - paddle::framework::LoD& lod, - T lower, - T upper) { + paddle::framework::LoD& lod, T lower, T upper) { input.set_lod(lod); int dim = lod[0][lod[0].size() - 1]; SetupTensor(input, {dim, 1}, lower, upper); @@ -52,8 +47,7 @@ void SetupLoDTensor(paddle::framework::LoDTensor& input, template void SetupLoDTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, - paddle::framework::LoD lod, + paddle::framework::DDim dims, paddle::framework::LoD lod, std::vector& data) { const size_t level = lod.size() - 1; CHECK_EQ(dims[0], static_cast((lod[level]).back())); @@ -92,8 +86,7 @@ template void TestInference(const std::string& dirname, const std::vector& cpu_feeds, std::vector& cpu_fetchs, - const int repeat = 1, - const bool is_combined = false) { + const int repeat = 1, const bool is_combined = false) { // 1. Define place, executor, scope auto place = Place(); auto executor = paddle::framework::Executor(place); @@ -132,11 +125,9 @@ void TestInference(const std::string& dirname, // `fluid.io.save_inference_model`. std::string prog_filename = "__model_combined__"; std::string param_filename = "__params_combined__"; - inference_program = - paddle::inference::Load(executor, - *scope, - dirname + "/" + prog_filename, - dirname + "/" + param_filename); + inference_program = paddle::inference::Load( + executor, *scope, dirname + "/" + prog_filename, + dirname + "/" + param_filename); } else { // Parameters are saved in separate files sited in the specified // `dirname`. diff --git a/paddle/fluid/memory/.clang-format b/paddle/fluid/memory/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/memory/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/operators/.clang-format b/paddle/fluid/operators/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/operators/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/platform/.clang-format b/paddle/fluid/platform/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/platform/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/pybind/.clang-format b/paddle/fluid/pybind/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/pybind/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/recordio/chunk.cc b/paddle/fluid/recordio/chunk.cc index 187a6a4ea7..e828cbabef 100644 --- a/paddle/fluid/recordio/chunk.cc +++ b/paddle/fluid/recordio/chunk.cc @@ -58,8 +58,8 @@ static void ReadStreamByBuf(std::istream& in, size_t limit, Callback callback) { * Copy stream in to another stream */ static void PipeStream(std::istream& in, std::ostream& os) { - ReadStreamByBuf( - in, 0, [&os](const char* buf, size_t len) { os.write(buf, len); }); + ReadStreamByBuf(in, 0, + [&os](const char* buf, size_t len) { os.write(buf, len); }); } /** @@ -68,8 +68,8 @@ static void PipeStream(std::istream& in, std::ostream& os) { static uint32_t Crc32Stream(std::istream& in, size_t limit = 0) { uint32_t crc = static_cast(crc32(0, nullptr, 0)); ReadStreamByBuf(in, limit, [&crc](const char* buf, size_t len) { - crc = static_cast(crc32( - crc, reinterpret_cast(buf), static_cast(len))); + crc = static_cast(crc32(crc, reinterpret_cast(buf), + static_cast(len))); }); return crc; } diff --git a/paddle/fluid/recordio/chunk.h b/paddle/fluid/recordio/chunk.h index bf20ebd455..71a1556a33 100644 --- a/paddle/fluid/recordio/chunk.h +++ b/paddle/fluid/recordio/chunk.h @@ -24,7 +24,7 @@ namespace recordio { // A Chunk contains the Header and optionally compressed records. class Chunk { -public: + public: Chunk() : num_bytes_(0) {} void Add(const std::string& buf) { num_bytes_ += buf.size(); @@ -46,7 +46,7 @@ public: bool Empty() const { return records_.empty(); } -private: + private: std::vector records_; // sum of record lengths in bytes. size_t num_bytes_; diff --git a/paddle/fluid/recordio/header.h b/paddle/fluid/recordio/header.h index 9200ac090d..245425990b 100644 --- a/paddle/fluid/recordio/header.h +++ b/paddle/fluid/recordio/header.h @@ -37,7 +37,7 @@ enum class Compressor : uint32_t { // Header is the metadata of Chunk class Header { -public: + public: Header(); Header(uint32_t num, uint32_t sum, Compressor ct, uint32_t cs); @@ -51,7 +51,7 @@ public: Compressor CompressType() const { return compressor_; } uint32_t CompressSize() const { return compress_size_; } -private: + private: uint32_t num_records_; uint32_t checksum_; Compressor compressor_; diff --git a/paddle/fluid/recordio/scanner.h b/paddle/fluid/recordio/scanner.h index f3f17b69f1..8812e2c952 100644 --- a/paddle/fluid/recordio/scanner.h +++ b/paddle/fluid/recordio/scanner.h @@ -21,7 +21,7 @@ namespace paddle { namespace recordio { class Scanner { -public: + public: explicit Scanner(std::unique_ptr&& stream); explicit Scanner(const std::string& filename); @@ -32,7 +32,7 @@ public: bool HasNext() const; -private: + private: std::unique_ptr stream_; Chunk cur_chunk_; size_t offset_; diff --git a/paddle/fluid/recordio/writer.h b/paddle/fluid/recordio/writer.h index 0c478d5075..87349644a7 100644 --- a/paddle/fluid/recordio/writer.h +++ b/paddle/fluid/recordio/writer.h @@ -18,9 +18,8 @@ namespace paddle { namespace recordio { class Writer { -public: - Writer(std::ostream* sout, - Compressor compressor, + public: + Writer(std::ostream* sout, Compressor compressor, size_t max_num_records_in_chunk = 1000) : stream_(*sout), max_num_records_in_chunk_(max_num_records_in_chunk), @@ -32,7 +31,7 @@ public: ~Writer(); -private: + private: std::ostream& stream_; size_t max_num_records_in_chunk_; Chunk cur_chunk_; diff --git a/paddle/fluid/string/.clang-format b/paddle/fluid/string/.clang-format deleted file mode 120000 index 7d28cb3924..0000000000 --- a/paddle/fluid/string/.clang-format +++ /dev/null @@ -1 +0,0 @@ -../framework/.clang-format \ No newline at end of file From 797a7184ace6389930f8096ffce213e63207bd8a Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Apr 2018 13:05:19 -0700 Subject: [PATCH 39/57] Unify Fluid code to Google C++ style (#9685) --- paddle/fluid/{framework => }/.clang-format | 0 paddle/fluid/inference/io.cc | 9 ++---- paddle/fluid/inference/io.h | 3 +- .../tests/book/test_inference_fit_a_line.cc | 4 +-- .../test_inference_image_classification.cc | 14 ++++---- .../test_inference_label_semantic_roles.cc | 32 +++++-------------- .../book/test_inference_recognize_digits.cc | 14 ++++---- .../test_inference_rnn_encoder_decoder.cc | 8 ++--- .../test_inference_understand_sentiment.cc | 4 +-- paddle/fluid/inference/tests/test_helper.h | 25 +++++---------- paddle/fluid/memory/.clang-format | 5 --- paddle/fluid/operators/.clang-format | 5 --- paddle/fluid/platform/.clang-format | 5 --- paddle/fluid/pybind/.clang-format | 5 --- paddle/fluid/recordio/chunk.cc | 8 ++--- paddle/fluid/recordio/chunk.h | 4 +-- paddle/fluid/recordio/header.h | 4 +-- paddle/fluid/recordio/scanner.h | 4 +-- paddle/fluid/recordio/writer.h | 7 ++-- paddle/fluid/string/.clang-format | 1 - 20 files changed, 52 insertions(+), 109 deletions(-) rename paddle/fluid/{framework => }/.clang-format (100%) delete mode 100644 paddle/fluid/memory/.clang-format delete mode 100644 paddle/fluid/operators/.clang-format delete mode 100644 paddle/fluid/platform/.clang-format delete mode 100644 paddle/fluid/pybind/.clang-format delete mode 120000 paddle/fluid/string/.clang-format diff --git a/paddle/fluid/framework/.clang-format b/paddle/fluid/.clang-format similarity index 100% rename from paddle/fluid/framework/.clang-format rename to paddle/fluid/.clang-format diff --git a/paddle/fluid/inference/io.cc b/paddle/fluid/inference/io.cc index 52e9c0baa6..a5b62ef322 100644 --- a/paddle/fluid/inference/io.cc +++ b/paddle/fluid/inference/io.cc @@ -41,8 +41,7 @@ bool IsPersistable(const framework::VarDesc* var) { return false; } -void LoadPersistables(framework::Executor& executor, - framework::Scope& scope, +void LoadPersistables(framework::Executor& executor, framework::Scope& scope, const framework::ProgramDesc& main_program, const std::string& dirname, const std::string& param_filename) { @@ -108,10 +107,8 @@ std::unique_ptr Load(framework::Executor& executor, } std::unique_ptr Load( - framework::Executor& executor, - framework::Scope& scope, - const std::string& prog_filename, - const std::string& param_filename) { + framework::Executor& executor, framework::Scope& scope, + const std::string& prog_filename, const std::string& param_filename) { std::string model_filename = prog_filename; std::string program_desc_str; ReadBinaryFile(model_filename, program_desc_str); diff --git a/paddle/fluid/inference/io.h b/paddle/fluid/inference/io.h index 6817a6fca0..d07d315b93 100644 --- a/paddle/fluid/inference/io.h +++ b/paddle/fluid/inference/io.h @@ -24,8 +24,7 @@ limitations under the License. */ namespace paddle { namespace inference { -void LoadPersistables(framework::Executor& executor, - framework::Scope& scope, +void LoadPersistables(framework::Executor& executor, framework::Scope& scope, const framework::ProgramDesc& main_program, const std::string& dirname, const std::string& param_filename); diff --git a/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc b/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc index 9ab808efec..8d8365a839 100644 --- a/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc +++ b/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc @@ -30,8 +30,8 @@ TEST(inference, fit_a_line) { // The second dim of the input tensor should be 13 // The input data should be >= 0 int64_t batch_size = 10; - SetupTensor( - input, {batch_size, 13}, static_cast(0), static_cast(10)); + SetupTensor(input, {batch_size, 13}, static_cast(0), + static_cast(10)); std::vector cpu_feeds; cpu_feeds.push_back(&input); diff --git a/paddle/fluid/inference/tests/book/test_inference_image_classification.cc b/paddle/fluid/inference/tests/book/test_inference_image_classification.cc index e9a27171f1..954ca7a3e3 100644 --- a/paddle/fluid/inference/tests/book/test_inference_image_classification.cc +++ b/paddle/fluid/inference/tests/book/test_inference_image_classification.cc @@ -35,10 +35,8 @@ TEST(inference, image_classification) { paddle::framework::LoDTensor input; // Use normilized image pixels as input data, // which should be in the range [0.0, 1.0]. - SetupTensor(input, - {FLAGS_batch_size, 3, 32, 32}, - static_cast(0), - static_cast(1)); + SetupTensor(input, {FLAGS_batch_size, 3, 32, 32}, + static_cast(0), static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&input); @@ -48,8 +46,8 @@ TEST(inference, image_classification) { // Run inference on CPU LOG(INFO) << "--- CPU Runs: ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs1, FLAGS_repeat); + TestInference(dirname, cpu_feeds, cpu_fetchs1, + FLAGS_repeat); LOG(INFO) << output1.dims(); #ifdef PADDLE_WITH_CUDA @@ -59,8 +57,8 @@ TEST(inference, image_classification) { // Run inference on CUDA GPU LOG(INFO) << "--- GPU Runs: ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs2, FLAGS_repeat); + TestInference(dirname, cpu_feeds, cpu_fetchs2, + FLAGS_repeat); LOG(INFO) << output2.dims(); CheckError(output1, output2); diff --git a/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc b/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc index 1849240166..31100494ff 100644 --- a/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc +++ b/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc @@ -36,37 +36,21 @@ TEST(inference, label_semantic_roles) { int64_t predicate_dict_len = 3162; int64_t mark_dict_len = 2; - SetupLoDTensor(word, - lod, - static_cast(0), + SetupLoDTensor(word, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(predicate, - lod, - static_cast(0), + SetupLoDTensor(predicate, lod, static_cast(0), static_cast(predicate_dict_len - 1)); - SetupLoDTensor(ctx_n2, - lod, - static_cast(0), + SetupLoDTensor(ctx_n2, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_n1, - lod, - static_cast(0), + SetupLoDTensor(ctx_n1, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_0, - lod, - static_cast(0), + SetupLoDTensor(ctx_0, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_p1, - lod, - static_cast(0), + SetupLoDTensor(ctx_p1, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_p2, - lod, - static_cast(0), + SetupLoDTensor(ctx_p2, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(mark, - lod, - static_cast(0), + SetupLoDTensor(mark, lod, static_cast(0), static_cast(mark_dict_len - 1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc b/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc index 1fb0f9e777..82275d3ee1 100644 --- a/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc +++ b/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc @@ -35,10 +35,8 @@ TEST(inference, recognize_digits) { paddle::framework::LoDTensor input; // Use normilized image pixels as input data, // which should be in the range [-1.0, 1.0]. - SetupTensor(input, - {FLAGS_batch_size, 1, 28, 28}, - static_cast(-1), - static_cast(1)); + SetupTensor(input, {FLAGS_batch_size, 1, 28, 28}, + static_cast(-1), static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&input); @@ -49,8 +47,8 @@ TEST(inference, recognize_digits) { // Run inference on CPU LOG(INFO) << "--- CPU Runs: is_combined=" << is_combined << " ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs1, FLAGS_repeat, is_combined); + TestInference(dirname, cpu_feeds, cpu_fetchs1, + FLAGS_repeat, is_combined); LOG(INFO) << output1.dims(); #ifdef PADDLE_WITH_CUDA @@ -60,8 +58,8 @@ TEST(inference, recognize_digits) { // Run inference on CUDA GPU LOG(INFO) << "--- GPU Runs: is_combined=" << is_combined << " ---"; - TestInference( - dirname, cpu_feeds, cpu_fetchs2, FLAGS_repeat, is_combined); + TestInference(dirname, cpu_feeds, cpu_fetchs2, + FLAGS_repeat, is_combined); LOG(INFO) << output2.dims(); CheckError(output1, output2); diff --git a/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc b/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc index a0523905bd..ea2d5ee092 100644 --- a/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc +++ b/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc @@ -32,10 +32,10 @@ TEST(inference, rnn_encoder_decoder) { paddle::framework::LoDTensor word_data, trg_word; paddle::framework::LoD lod{{0, 4, 10}}; - SetupLoDTensor( - word_data, lod, static_cast(0), static_cast(1)); - SetupLoDTensor( - trg_word, lod, static_cast(0), static_cast(1)); + SetupLoDTensor(word_data, lod, static_cast(0), + static_cast(1)); + SetupLoDTensor(trg_word, lod, static_cast(0), + static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&word_data); diff --git a/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc b/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc index 824b3274eb..21ffd8d368 100644 --- a/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc +++ b/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc @@ -33,9 +33,7 @@ TEST(inference, understand_sentiment) { paddle::framework::LoD lod{{0, 4, 10}}; int64_t word_dict_len = 5147; - SetupLoDTensor(words, - lod, - static_cast(0), + SetupLoDTensor(words, lod, static_cast(0), static_cast(word_dict_len - 1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/test_helper.h b/paddle/fluid/inference/tests/test_helper.h index dce541c097..d8ffedf672 100644 --- a/paddle/fluid/inference/tests/test_helper.h +++ b/paddle/fluid/inference/tests/test_helper.h @@ -19,9 +19,7 @@ limitations under the License. */ template void SetupTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, - T lower, - T upper) { + paddle::framework::DDim dims, T lower, T upper) { srand(time(0)); T* input_ptr = input.mutable_data(dims, paddle::platform::CPUPlace()); for (int i = 0; i < input.numel(); ++i) { @@ -33,8 +31,7 @@ void SetupTensor(paddle::framework::LoDTensor& input, template void SetupTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, - std::vector& data) { + paddle::framework::DDim dims, std::vector& data) { CHECK_EQ(paddle::framework::product(dims), static_cast(data.size())); T* input_ptr = input.mutable_data(dims, paddle::platform::CPUPlace()); memcpy(input_ptr, data.data(), input.numel() * sizeof(T)); @@ -42,9 +39,7 @@ void SetupTensor(paddle::framework::LoDTensor& input, template void SetupLoDTensor(paddle::framework::LoDTensor& input, - paddle::framework::LoD& lod, - T lower, - T upper) { + paddle::framework::LoD& lod, T lower, T upper) { input.set_lod(lod); int dim = lod[0][lod[0].size() - 1]; SetupTensor(input, {dim, 1}, lower, upper); @@ -52,8 +47,7 @@ void SetupLoDTensor(paddle::framework::LoDTensor& input, template void SetupLoDTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, - paddle::framework::LoD lod, + paddle::framework::DDim dims, paddle::framework::LoD lod, std::vector& data) { const size_t level = lod.size() - 1; CHECK_EQ(dims[0], static_cast((lod[level]).back())); @@ -92,8 +86,7 @@ template void TestInference(const std::string& dirname, const std::vector& cpu_feeds, std::vector& cpu_fetchs, - const int repeat = 1, - const bool is_combined = false) { + const int repeat = 1, const bool is_combined = false) { // 1. Define place, executor, scope auto place = Place(); auto executor = paddle::framework::Executor(place); @@ -132,11 +125,9 @@ void TestInference(const std::string& dirname, // `fluid.io.save_inference_model`. std::string prog_filename = "__model_combined__"; std::string param_filename = "__params_combined__"; - inference_program = - paddle::inference::Load(executor, - *scope, - dirname + "/" + prog_filename, - dirname + "/" + param_filename); + inference_program = paddle::inference::Load( + executor, *scope, dirname + "/" + prog_filename, + dirname + "/" + param_filename); } else { // Parameters are saved in separate files sited in the specified // `dirname`. diff --git a/paddle/fluid/memory/.clang-format b/paddle/fluid/memory/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/memory/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/operators/.clang-format b/paddle/fluid/operators/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/operators/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/platform/.clang-format b/paddle/fluid/platform/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/platform/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/pybind/.clang-format b/paddle/fluid/pybind/.clang-format deleted file mode 100644 index 29282dc87e..0000000000 --- a/paddle/fluid/pybind/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/fluid/recordio/chunk.cc b/paddle/fluid/recordio/chunk.cc index 187a6a4ea7..e828cbabef 100644 --- a/paddle/fluid/recordio/chunk.cc +++ b/paddle/fluid/recordio/chunk.cc @@ -58,8 +58,8 @@ static void ReadStreamByBuf(std::istream& in, size_t limit, Callback callback) { * Copy stream in to another stream */ static void PipeStream(std::istream& in, std::ostream& os) { - ReadStreamByBuf( - in, 0, [&os](const char* buf, size_t len) { os.write(buf, len); }); + ReadStreamByBuf(in, 0, + [&os](const char* buf, size_t len) { os.write(buf, len); }); } /** @@ -68,8 +68,8 @@ static void PipeStream(std::istream& in, std::ostream& os) { static uint32_t Crc32Stream(std::istream& in, size_t limit = 0) { uint32_t crc = static_cast(crc32(0, nullptr, 0)); ReadStreamByBuf(in, limit, [&crc](const char* buf, size_t len) { - crc = static_cast(crc32( - crc, reinterpret_cast(buf), static_cast(len))); + crc = static_cast(crc32(crc, reinterpret_cast(buf), + static_cast(len))); }); return crc; } diff --git a/paddle/fluid/recordio/chunk.h b/paddle/fluid/recordio/chunk.h index bf20ebd455..71a1556a33 100644 --- a/paddle/fluid/recordio/chunk.h +++ b/paddle/fluid/recordio/chunk.h @@ -24,7 +24,7 @@ namespace recordio { // A Chunk contains the Header and optionally compressed records. class Chunk { -public: + public: Chunk() : num_bytes_(0) {} void Add(const std::string& buf) { num_bytes_ += buf.size(); @@ -46,7 +46,7 @@ public: bool Empty() const { return records_.empty(); } -private: + private: std::vector records_; // sum of record lengths in bytes. size_t num_bytes_; diff --git a/paddle/fluid/recordio/header.h b/paddle/fluid/recordio/header.h index 9200ac090d..245425990b 100644 --- a/paddle/fluid/recordio/header.h +++ b/paddle/fluid/recordio/header.h @@ -37,7 +37,7 @@ enum class Compressor : uint32_t { // Header is the metadata of Chunk class Header { -public: + public: Header(); Header(uint32_t num, uint32_t sum, Compressor ct, uint32_t cs); @@ -51,7 +51,7 @@ public: Compressor CompressType() const { return compressor_; } uint32_t CompressSize() const { return compress_size_; } -private: + private: uint32_t num_records_; uint32_t checksum_; Compressor compressor_; diff --git a/paddle/fluid/recordio/scanner.h b/paddle/fluid/recordio/scanner.h index f3f17b69f1..8812e2c952 100644 --- a/paddle/fluid/recordio/scanner.h +++ b/paddle/fluid/recordio/scanner.h @@ -21,7 +21,7 @@ namespace paddle { namespace recordio { class Scanner { -public: + public: explicit Scanner(std::unique_ptr&& stream); explicit Scanner(const std::string& filename); @@ -32,7 +32,7 @@ public: bool HasNext() const; -private: + private: std::unique_ptr stream_; Chunk cur_chunk_; size_t offset_; diff --git a/paddle/fluid/recordio/writer.h b/paddle/fluid/recordio/writer.h index 0c478d5075..87349644a7 100644 --- a/paddle/fluid/recordio/writer.h +++ b/paddle/fluid/recordio/writer.h @@ -18,9 +18,8 @@ namespace paddle { namespace recordio { class Writer { -public: - Writer(std::ostream* sout, - Compressor compressor, + public: + Writer(std::ostream* sout, Compressor compressor, size_t max_num_records_in_chunk = 1000) : stream_(*sout), max_num_records_in_chunk_(max_num_records_in_chunk), @@ -32,7 +31,7 @@ public: ~Writer(); -private: + private: std::ostream& stream_; size_t max_num_records_in_chunk_; Chunk cur_chunk_; diff --git a/paddle/fluid/string/.clang-format b/paddle/fluid/string/.clang-format deleted file mode 120000 index 7d28cb3924..0000000000 --- a/paddle/fluid/string/.clang-format +++ /dev/null @@ -1 +0,0 @@ -../framework/.clang-format \ No newline at end of file From c839ec6c470d434578ddecae02caae974ff97228 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Apr 2018 13:19:49 -0700 Subject: [PATCH 40/57] Update rcordio --- cmake/external/snappystream.cmake | 4 +++- cmake/external/zlib.cmake | 3 ++- paddle/fluid/recordio/chunk.cc | 6 ++++-- paddle/fluid/recordio/chunk_test.cc | 12 +++++------- paddle/fluid/recordio/header_test.cc | 6 ++---- paddle/fluid/recordio/scanner.cc | 4 ++++ paddle/fluid/recordio/scanner.h | 3 +++ paddle/fluid/recordio/writer.cc | 5 +++++ paddle/fluid/recordio/writer.h | 4 +++- paddle/fluid/recordio/writer_scanner_test.cc | 7 ++++--- 10 files changed, 35 insertions(+), 19 deletions(-) diff --git a/cmake/external/snappystream.cmake b/cmake/external/snappystream.cmake index 5377a0b046..8f7a3bf8ee 100644 --- a/cmake/external/snappystream.cmake +++ b/cmake/external/snappystream.cmake @@ -54,5 +54,7 @@ add_library(snappystream STATIC IMPORTED GLOBAL) set_property(TARGET snappystream PROPERTY IMPORTED_LOCATION "${SNAPPYSTREAM_INSTALL_DIR}/lib/libsnappystream.a") -include_directories(${SNAPPYSTREAM_INCLUDE_DIR}) +include_directories(${SNAPPYSTREAM_INCLUDE_DIR}) # For snappysteam to include its own headers. +include_directories(${THIRD_PARTY_PATH}/install) # For Paddle to include snappy stream headers. + add_dependencies(snappystream extern_snappystream) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index 20b8506e67..c3d7323545 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -25,7 +25,8 @@ ELSE(WIN32) SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) ENDIF(WIN32) -INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) # For zlib code to include its own headers. +INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include zlib.h. ExternalProject_Add( extern_zlib diff --git a/paddle/fluid/recordio/chunk.cc b/paddle/fluid/recordio/chunk.cc index e828cbabef..e7ebbba452 100644 --- a/paddle/fluid/recordio/chunk.cc +++ b/paddle/fluid/recordio/chunk.cc @@ -14,11 +14,13 @@ #include "paddle/fluid/recordio/chunk.h" +#include #include #include + #include "paddle/fluid/platform/enforce.h" -#include "snappystream.hpp" -#include "zlib.h" +#include "snappy_stream/include/snappystream.hpp" +#include "zlib/include/zlib.h" namespace paddle { namespace recordio { diff --git a/paddle/fluid/recordio/chunk_test.cc b/paddle/fluid/recordio/chunk_test.cc index 1f0e36a14d..98ca99b9a0 100644 --- a/paddle/fluid/recordio/chunk_test.cc +++ b/paddle/fluid/recordio/chunk_test.cc @@ -18,29 +18,27 @@ #include "gtest/gtest.h" -using namespace paddle::recordio; - TEST(Chunk, SaveLoad) { - Chunk ch; + paddle::recordio::Chunk ch; ch.Add(std::string("12345", 6)); ch.Add(std::string("123", 4)); std::stringstream ss; - ch.Write(ss, Compressor::kNoCompress); + ch.Write(ss, paddle::recordio::Compressor::kNoCompress); ss.seekg(0); ch.Parse(ss); ASSERT_EQ(ch.NumBytes(), 10U); } TEST(Chunk, Compressor) { - Chunk ch; + paddle::recordio::Chunk ch; ch.Add(std::string("12345", 6)); ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4)); std::stringstream ss; - ch.Write(ss, Compressor::kSnappy); + ch.Write(ss, paddle::recordio::Compressor::kSnappy); std::stringstream ss2; - ch.Write(ss2, Compressor::kNoCompress); + ch.Write(ss2, paddle::recordio::Compressor::kNoCompress); ASSERT_LE(ss.tellp(), ss2.tellp()); // Compress should contain less data; ch.Clear(); diff --git a/paddle/fluid/recordio/header_test.cc b/paddle/fluid/recordio/header_test.cc index a7d627c3eb..00f1887dc5 100644 --- a/paddle/fluid/recordio/header_test.cc +++ b/paddle/fluid/recordio/header_test.cc @@ -18,14 +18,12 @@ #include "gtest/gtest.h" -using namespace paddle::recordio; - TEST(Recordio, ChunkHead) { - Header hdr(0, 1, Compressor::kGzip, 3); + paddle::recordio::Header hdr(0, 1, paddle::recordio::Compressor::kGzip, 3); std::stringstream ss; hdr.Write(ss); ss.seekg(0, std::ios::beg); - Header hdr2; + paddle::recordio::Header hdr2; hdr2.Parse(ss); EXPECT_TRUE(hdr == hdr2); } diff --git a/paddle/fluid/recordio/scanner.cc b/paddle/fluid/recordio/scanner.cc index c22281dc97..88b4d4001b 100644 --- a/paddle/fluid/recordio/scanner.cc +++ b/paddle/fluid/recordio/scanner.cc @@ -13,10 +13,14 @@ // limitations under the License. #include "paddle/fluid/recordio/scanner.h" + +#include + #include "paddle/fluid/platform/enforce.h" namespace paddle { namespace recordio { + Scanner::Scanner(std::unique_ptr &&stream) : stream_(std::move(stream)) { Reset(); diff --git a/paddle/fluid/recordio/scanner.h b/paddle/fluid/recordio/scanner.h index 8812e2c952..34f1b0c78d 100644 --- a/paddle/fluid/recordio/scanner.h +++ b/paddle/fluid/recordio/scanner.h @@ -16,7 +16,10 @@ #include #include +#include + #include "paddle/fluid/recordio/chunk.h" + namespace paddle { namespace recordio { diff --git a/paddle/fluid/recordio/writer.cc b/paddle/fluid/recordio/writer.cc index 196d66edff..8046f4ff78 100644 --- a/paddle/fluid/recordio/writer.cc +++ b/paddle/fluid/recordio/writer.cc @@ -12,9 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "paddle/fluid/recordio/writer.h" + +#include + #include "paddle/fluid/platform/enforce.h" + namespace paddle { namespace recordio { + void Writer::Write(const std::string& record) { cur_chunk_.Add(record); if (cur_chunk_.NumRecords() >= max_num_records_in_chunk_) { diff --git a/paddle/fluid/recordio/writer.h b/paddle/fluid/recordio/writer.h index 87349644a7..ac7e50ee90 100644 --- a/paddle/fluid/recordio/writer.h +++ b/paddle/fluid/recordio/writer.h @@ -11,8 +11,10 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - #pragma once + +#include + #include "paddle/fluid/recordio/chunk.h" namespace paddle { namespace recordio { diff --git a/paddle/fluid/recordio/writer_scanner_test.cc b/paddle/fluid/recordio/writer_scanner_test.cc index 7e764f0d94..6583df21a2 100644 --- a/paddle/fluid/recordio/writer_scanner_test.cc +++ b/paddle/fluid/recordio/writer_scanner_test.cc @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "gtest/gtest.h" - #include +#include + +#include "gtest/gtest.h" #include "paddle/fluid/recordio/scanner.h" #include "paddle/fluid/recordio/writer.h" @@ -66,4 +67,4 @@ TEST(WriterScanner, TinyChunk) { ASSERT_EQ(scanner.Next(), "DEFG"); ASSERT_FALSE(scanner.HasNext()); } -} \ No newline at end of file +} From b2a1c9e8b7f0fab5f81282783acadc2d35f3e207 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Fri, 6 Apr 2018 13:25:29 -0700 Subject: [PATCH 41/57] Add float16 support to non-cudnn softmax op on GPU (#9686) * initial commit * fix error * fix typo and order --- paddle/fluid/operators/math/softmax.cu | 3 + paddle/fluid/operators/math/softmax_impl.h | 2 +- paddle/fluid/operators/softmax_op.cc | 9 +- paddle/fluid/operators/softmax_op.cu.cc | 11 +- paddle/fluid/platform/float16.h | 227 +++++++++++++----- .../fluid/tests/unittests/test_softmax_op.py | 11 + 6 files changed, 189 insertions(+), 74 deletions(-) diff --git a/paddle/fluid/operators/math/softmax.cu b/paddle/fluid/operators/math/softmax.cu index 5518ebed3f..a579182ec1 100644 --- a/paddle/fluid/operators/math/softmax.cu +++ b/paddle/fluid/operators/math/softmax.cu @@ -14,6 +14,8 @@ limitations under the License. */ #define EIGEN_USE_GPU +#include + #include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/softmax.h" #include "paddle/fluid/operators/math/softmax_impl.h" @@ -95,6 +97,7 @@ template class SoftmaxCUDNNFunctor; template class SoftmaxGradCUDNNFunctor; template class SoftmaxGradCUDNNFunctor; +template class SoftmaxFunctor; template class SoftmaxFunctor; template class SoftmaxFunctor; template class SoftmaxGradFunctor; diff --git a/paddle/fluid/operators/math/softmax_impl.h b/paddle/fluid/operators/math/softmax_impl.h index 3e123f7bf5..dd9971ba09 100644 --- a/paddle/fluid/operators/math/softmax_impl.h +++ b/paddle/fluid/operators/math/softmax_impl.h @@ -27,7 +27,7 @@ using EigenMatrix = framework::EigenMatrix; template struct ValueClip { HOSTDEVICE T operator()(const T& x) const { - const T kThreshold = -64.; + const T kThreshold = static_cast(-64.); return x < kThreshold ? kThreshold : x; } }; diff --git a/paddle/fluid/operators/softmax_op.cc b/paddle/fluid/operators/softmax_op.cc index e2c0f915d9..6bdefc0f23 100644 --- a/paddle/fluid/operators/softmax_op.cc +++ b/paddle/fluid/operators/softmax_op.cc @@ -13,6 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/softmax_op.h" + +#include + #ifdef PADDLE_WITH_CUDA #include "paddle/fluid/platform/cudnn_helper.h" #endif @@ -20,6 +23,7 @@ limitations under the License. */ #ifdef PADDLE_WITH_MKLDNN #include "paddle/fluid/platform/mkldnn_helper.h" #endif + namespace paddle { namespace operators { @@ -60,8 +64,8 @@ class SoftmaxOp : public framework::OperatorWithKernel { auto input_data_type = framework::ToDataType(ctx.Input("X")->type()); if (input_data_type == framework::proto::VarType::FP16) { - PADDLE_ENFORCE_EQ(library_, framework::LibraryType::kCUDNN, - "float16 can only be used when CUDNN is used"); + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "float16 can only be used on GPU place"); } std::string data_format = ctx.Attr("data_format"); @@ -70,6 +74,7 @@ class SoftmaxOp : public framework::OperatorWithKernel { library_); } }; + class SoftmaxOpMaker : public framework::OpProtoAndCheckerMaker { public: SoftmaxOpMaker(OpProto* proto, OpAttrChecker* op_checker) diff --git a/paddle/fluid/operators/softmax_op.cu.cc b/paddle/fluid/operators/softmax_op.cu.cc index dbd13fd38a..0c1f7cef7a 100644 --- a/paddle/fluid/operators/softmax_op.cu.cc +++ b/paddle/fluid/operators/softmax_op.cu.cc @@ -13,11 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/softmax_op.h" +#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; - -REGISTER_OP_CUDA_KERNEL( - softmax, ops::SoftmaxKernel); +namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( - softmax_grad, - ops::SoftmaxGradKernel); + softmax, ops::SoftmaxKernel, + ops::SoftmaxKernel); +REGISTER_OP_CUDA_KERNEL(softmax_grad, + ops::SoftmaxGradKernel); diff --git a/paddle/fluid/platform/float16.h b/paddle/fluid/platform/float16.h index 2cf311c7e5..e77f768bf9 100644 --- a/paddle/fluid/platform/float16.h +++ b/paddle/fluid/platform/float16.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include #ifdef PADDLE_WITH_CUDA #include @@ -293,39 +294,39 @@ struct PADDLE_ALIGN(2) float16 { HOSTDEVICE inline explicit operator bool() const { return (x & 0x7fff) != 0; } HOSTDEVICE inline explicit operator int8_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator uint8_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator int16_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator uint16_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator int32_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator uint32_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator int64_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator uint64_t() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } HOSTDEVICE inline explicit operator double() const { - return static_cast(float(*this)); + return static_cast(static_cast(*this)); } private: @@ -370,7 +371,7 @@ DEVICE inline half operator+(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hadd(a, b); #else - float res = float(float16(a)) + float(float16(b)); + float res = static_cast(float16(a)) + static_cast(float16(b)); return half(float16(res)); #endif } @@ -379,7 +380,7 @@ DEVICE inline half operator-(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hsub(a, b); #else - float res = float(float16(a)) - float(float16(b)); + float res = static_cast(float16(a)) - static_cast(float16(b)); return half(float16(res)); #endif } @@ -388,7 +389,7 @@ DEVICE inline half operator*(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hmul(a, b); #else - float res = float(float16(a)) * float(float16(b)); + float res = static_cast(float16(a)) * static_cast(float16(b)); return half(float16(res)); #endif } @@ -399,7 +400,7 @@ DEVICE inline half operator/(const half& a, const half& b) { float denom = __half2float(b); return __float2half(num / denom); #else - float res = float(float16(a)) / float(float16(b)); + float res = static_cast(float16(a)) / static_cast(float16(b)); return half(float16(res)); #endif } @@ -408,27 +409,27 @@ DEVICE inline half operator-(const half& a) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hneg(a); #else - float res = -float(float16(a)); + float res = -static_cast(float16(a)); return half(float16(res)); #endif } -DEVICE inline half& operator+=(half& a, const half& b) { +DEVICE inline half& operator+=(half& a, const half& b) { // NOLINT a = a + b; return a; } -DEVICE inline half& operator-=(half& a, const half& b) { +DEVICE inline half& operator-=(half& a, const half& b) { // NOLINT a = a - b; return a; } -DEVICE inline half& operator*=(half& a, const half& b) { +DEVICE inline half& operator*=(half& a, const half& b) { // NOLINT a = a * b; return a; } -DEVICE inline half& operator/=(half& a, const half& b) { +DEVICE inline half& operator/=(half& a, const half& b) { // NOLINT a = a / b; return a; } @@ -437,7 +438,7 @@ DEVICE inline bool operator==(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __heq(a, b); #else - return float(float16(a)) == float(float16(b)); + return static_cast(float16(a)) == static_cast(float16(b)); #endif } @@ -445,7 +446,7 @@ DEVICE inline bool operator!=(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hne(a, b); #else - return float(float16(a)) != float(float16(b)); + return static_cast(float16(a)) != static_cast(float16(b)); #endif } @@ -453,7 +454,7 @@ DEVICE inline bool operator<(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hlt(a, b); #else - return float(float16(a)) < float(float16(b)); + return static_cast(float16(a)) < static_cast(float16(b)); #endif } @@ -461,7 +462,7 @@ DEVICE inline bool operator<=(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hle(a, b); #else - return float(float16(a)) <= float(float16(b)); + return static_cast(float16(a)) <= static_cast(float16(b)); #endif } @@ -469,7 +470,7 @@ DEVICE inline bool operator>(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hgt(a, b); #else - return float(float16(a)) > float(float16(b)); + return static_cast(float16(a)) > static_cast(float16(b)); #endif } @@ -477,7 +478,7 @@ DEVICE inline bool operator>=(const half& a, const half& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hge(a, b); #else - return float(float16(a)) >= float(float16(b)); + return static_cast(float16(a)) >= static_cast(float16(b)); #endif } @@ -489,7 +490,7 @@ HOSTDEVICE inline float16 operator+(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return float16(__hadd(half(a), half(b))); #else - return float16(float(a) + float(b)); + return float16(static_cast(a) + static_cast(b)); #endif } @@ -497,7 +498,7 @@ HOSTDEVICE inline float16 operator-(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return float16(__hsub(half(a), half(b))); #else - return float16(float(a) - float(b)); + return float16(static_cast(a) - static_cast(b)); #endif } @@ -505,7 +506,7 @@ HOSTDEVICE inline float16 operator*(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return float16(__hmul(half(a), half(b))); #else - return float16(float(a) * float(b)); + return float16(static_cast(a) * static_cast(b)); #endif } @@ -516,7 +517,7 @@ HOSTDEVICE inline float16 operator/(const float16& a, const float16& b) { float denom = __half2float(half(b)); return float16(num / denom); #else - return float16(float(a) / float(b)); + return float16(static_cast(a) / static_cast(b)); #endif } @@ -530,22 +531,22 @@ HOSTDEVICE inline float16 operator-(const float16& a) { #endif } -HOSTDEVICE inline float16& operator+=(float16& a, const float16& b) { +HOSTDEVICE inline float16& operator+=(float16& a, const float16& b) { // NOLINT a = a + b; return a; } -HOSTDEVICE inline float16& operator-=(float16& a, const float16& b) { +HOSTDEVICE inline float16& operator-=(float16& a, const float16& b) { // NOLINT a = a - b; return a; } -HOSTDEVICE inline float16& operator*=(float16& a, const float16& b) { +HOSTDEVICE inline float16& operator*=(float16& a, const float16& b) { // NOLINT a = a * b; return a; } -HOSTDEVICE inline float16& operator/=(float16& a, const float16& b) { +HOSTDEVICE inline float16& operator/=(float16& a, const float16& b) { // NOLINT a = a / b; return a; } @@ -554,7 +555,7 @@ HOSTDEVICE inline bool operator==(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __heq(half(a), half(b)); #else - return float(a) == float(b); + return static_cast(a) == static_cast(b); #endif } @@ -562,7 +563,7 @@ HOSTDEVICE inline bool operator!=(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hne(half(a), half(b)); #else - return float(a) != float(b); + return static_cast(a) != static_cast(b); #endif } @@ -570,7 +571,7 @@ HOSTDEVICE inline bool operator<(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hlt(half(a), half(b)); #else - return float(a) < float(b); + return static_cast(a) < static_cast(b); #endif } @@ -578,7 +579,7 @@ HOSTDEVICE inline bool operator<=(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hle(half(a), half(b)); #else - return float(a) <= float(b); + return static_cast(a) <= static_cast(b); #endif } @@ -586,7 +587,7 @@ HOSTDEVICE inline bool operator>(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hgt(half(a), half(b)); #else - return float(a) > float(b); + return static_cast(a) > static_cast(b); #endif } @@ -594,7 +595,7 @@ HOSTDEVICE inline bool operator>=(const float16& a, const float16& b) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hge(half(a), half(b)); #else - return float(a) >= float(b); + return static_cast(a) >= static_cast(b); #endif } @@ -679,22 +680,22 @@ inline float16 operator-(const float16& a) { return res; } -inline float16& operator+=(float16& a, const float16& b) { +inline float16& operator+=(float16& a, const float16& b) { // NOLINT a = a + b; return a; } -inline float16& operator-=(float16& a, const float16& b) { +inline float16& operator-=(float16& a, const float16& b) { // NOLINT a = a - b; return a; } -inline float16& operator*=(float16& a, const float16& b) { +inline float16& operator*=(float16& a, const float16& b) { // NOLINT a = a * b; return a; } -inline float16& operator/=(float16& a, const float16& b) { +inline float16& operator/=(float16& a, const float16& b) { // NOLINT a = a / b; return a; } @@ -784,19 +785,19 @@ inline bool operator>=(const float16& a, const float16& b) { // Arithmetic operators for float16, software emulated on other CPU #else inline float16 operator+(const float16& a, const float16& b) { - return float16(float(a) + float(b)); + return float16(static_cast(a) + static_cast(b)); } inline float16 operator-(const float16& a, const float16& b) { - return float16(float(a) - float(b)); + return float16(static_cast(a) - static_cast(b)); } inline float16 operator*(const float16& a, const float16& b) { - return float16(float(a) * float(b)); + return float16(static_cast(a) * static_cast(b)); } inline float16 operator/(const float16& a, const float16& b) { - return float16(float(a) / float(b)); + return float16(static_cast(a) / static_cast(b)); } inline float16 operator-(const float16& a) { @@ -805,51 +806,57 @@ inline float16 operator-(const float16& a) { return res; } -inline float16& operator+=(float16& a, const float16& b) { - a = float16(float(a) + float(b)); +inline float16& operator+=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) + static_cast(b)); return a; } -inline float16& operator-=(float16& a, const float16& b) { - a = float16(float(a) - float(b)); +inline float16& operator-=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) - static_cast(b)); return a; } -inline float16& operator*=(float16& a, const float16& b) { - a = float16(float(a) * float(b)); +inline float16& operator*=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) * static_cast(b)); return a; } -inline float16& operator/=(float16& a, const float16& b) { - a = float16(float(a) / float(b)); +inline float16& operator/=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) / static_cast(b)); return a; } inline bool operator==(const float16& a, const float16& b) { - return float(a) == float(b); + return static_cast(a) == static_cast(b); } inline bool operator!=(const float16& a, const float16& b) { - return float(a) != float(b); + return static_cast(a) != static_cast(b); } inline bool operator<(const float16& a, const float16& b) { - return float(a) < float(b); + return static_cast(a) < static_cast(b); } inline bool operator<=(const float16& a, const float16& b) { - return float(a) <= float(b); + return static_cast(a) <= static_cast(b); } inline bool operator>(const float16& a, const float16& b) { - return float(a) > float(b); + return static_cast(a) > static_cast(b); } inline bool operator>=(const float16& a, const float16& b) { - return float(a) >= float(b); + return static_cast(a) >= static_cast(b); } #endif +HOSTDEVICE inline float16 raw_uint16_to_float16(uint16_t a) { + float16 res; + res.x = a; + return res; +} + HOSTDEVICE inline bool(isnan)(const float16& a) { #if defined(PADDLE_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 return __hisnan(half(a)); @@ -886,28 +893,116 @@ struct is_pod { is_standard_layout::value; }; +template <> +struct numeric_limits { + static const bool is_specialized = true; + static const bool is_signed = true; + static const bool is_integer = false; + static const bool is_exact = false; + static const bool has_infinity = true; + static const bool has_quiet_NaN = true; + static const bool has_signaling_NaN = true; + static const float_denorm_style has_denorm = denorm_present; + static const bool has_denorm_loss = false; + static const std::float_round_style round_style = std::round_to_nearest; + static const bool is_iec559 = false; + static const bool is_bounded = false; + static const bool is_modulo = false; + static const int digits = 11; + static const int digits10 = 3; + static const int max_digits10 = 5; + static const int radix = 2; + static const int min_exponent = -13; + static const int min_exponent10 = -4; + static const int max_exponent = 16; + static const int max_exponent10 = 4; + static const bool traps = true; + static const bool tinyness_before = false; + + static paddle::platform::float16(min)() { + return paddle::platform::raw_uint16_to_float16(0x400); + } + static paddle::platform::float16 lowest() { + return paddle::platform::raw_uint16_to_float16(0xfbff); + } + static paddle::platform::float16(max)() { + return paddle::platform::raw_uint16_to_float16(0x7bff); + } + static paddle::platform::float16 epsilon() { + return paddle::platform::raw_uint16_to_float16(0x0800); + } + static paddle::platform::float16 round_error() { + return paddle::platform::float16(0.5); + } + static paddle::platform::float16 infinity() { + return paddle::platform::raw_uint16_to_float16(0x7c00); + } + static paddle::platform::float16 quiet_NaN() { + return paddle::platform::raw_uint16_to_float16(0x7e00); + } + static paddle::platform::float16 signaling_NaN() { + return paddle::platform::raw_uint16_to_float16(0x7e00); + } + static paddle::platform::float16 denorm_min() { + return paddle::platform::raw_uint16_to_float16(0x1); + } +}; + } // namespace std namespace Eigen { + +using float16 = paddle::platform::float16; + +template <> +struct NumTraits : GenericNumTraits { + enum { + IsSigned = true, + IsInteger = false, + IsComplex = false, + RequireInitialization = false + }; + + HOSTDEVICE static inline float16 epsilon() { + return paddle::platform::raw_uint16_to_float16(0x0800); + } + HOSTDEVICE static inline float16 dummy_precision() { return float16(1e-2f); } + HOSTDEVICE static inline float16 highest() { + return paddle::platform::raw_uint16_to_float16(0x7bff); + } + HOSTDEVICE static inline float16 lowest() { + return paddle::platform::raw_uint16_to_float16(0xfbff); + } + HOSTDEVICE static inline float16 infinity() { + return paddle::platform::raw_uint16_to_float16(0x7c00); + } + HOSTDEVICE static inline float16 quiet_NaN() { + return paddle::platform::raw_uint16_to_float16(0x7c01); + } +}; + namespace numext { template <> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isnan)( - const paddle::platform::float16& a) { +HOSTDEVICE inline bool(isnan)(const float16& a) { return (paddle::platform::isnan)(a); } template <> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isinf)( - const paddle::platform::float16& a) { +HOSTDEVICE inline bool(isinf)(const float16& a) { return (paddle::platform::isinf)(a); } template <> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isfinite)( - const paddle::platform::float16& a) { +HOSTDEVICE inline bool(isfinite)(const float16& a) { return (paddle::platform::isfinite)(a); } +template <> +HOSTDEVICE inline float16 exp(const float16& a) { + return float16(::expf(static_cast(a))); +} + } // namespace numext + } // namespace Eigen diff --git a/python/paddle/fluid/tests/unittests/test_softmax_op.py b/python/paddle/fluid/tests/unittests/test_softmax_op.py index 33d60c7e31..279f3073f7 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_op.py @@ -68,6 +68,17 @@ class TestSoftmaxCUDNNOp(TestSoftmaxOp): self.use_cudnn = True +class TestSoftmaxFP16Op(TestSoftmaxOp): + def init_kernel_type(self): + self.dtype = np.float16 + + def test_check_output(self): + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + if core.is_float16_supported(place): + self.check_output_with_place(place, atol=1e-3) + + class TestSoftmaxFP16CUDNNOp(TestSoftmaxOp): def init_kernel_type(self): self.use_cudnn = True From 560d960b2709c38922e913e7bd00e5362d75e1f2 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Fri, 6 Apr 2018 15:42:52 -0700 Subject: [PATCH 42/57] Fix a minor bug for distributed_spliter.round_robin Also fixed typo and comments. --- python/paddle/fluid/distribute_transpiler.py | 12 ++++++------ ...ributed_spliter.py => distributed_splitter.py} | 15 +++++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) rename python/paddle/fluid/{distributed_spliter.py => distributed_splitter.py} (78%) diff --git a/python/paddle/fluid/distribute_transpiler.py b/python/paddle/fluid/distribute_transpiler.py index 31bedb592f..7a2a81be9f 100644 --- a/python/paddle/fluid/distribute_transpiler.py +++ b/python/paddle/fluid/distribute_transpiler.py @@ -17,7 +17,7 @@ import framework from framework import Program, default_main_program, default_startup_program, Parameter, Variable import optimizer from layer_helper import LayerHelper -from distributed_spliter import * +import distributed_splitter as splitter import math from . import core import debuger @@ -36,7 +36,7 @@ class VarBlock: class UnionFind(object): """ Union-find data struct. - + Union-find is a data struct that keeps track of a set of elements partitioned into a number of disjoint (non-overlapping) subsets. @@ -138,7 +138,7 @@ class DistributeTranspiler: program=None, pservers="127.0.0.1:6174", trainers=1, - split_method=round_robin): + split_method=splitter.round_robin): """ Transpile the program to distributed data-parallelism programs. The main_program will be transformed to use a remote parameter server @@ -303,7 +303,7 @@ class DistributeTranspiler: # If two ops are connected, we could add these two ops # into one set. ufind = self._create_ufind(self.optimize_ops) - # step 4.2 + # step 4.2 # Iterate through the ops and append optimize op which # located on current pserver opt_op_on_pserver = [] @@ -312,7 +312,7 @@ class DistributeTranspiler: opt_op_on_pserver.append(op) # step 4.3 # Iterate through the ops, and if an op and the optimize ops - # which located on current pserver are in one set, then + # which located on current pserver are in one set, then # append it into the sub program. # We try to put optimization program run parallelly, assume @@ -752,7 +752,7 @@ class DistributeTranspiler: def _is_opt_op(self, op): # NOTE: It's a HACK implement. - # optimize op: SGDOptimize, MomentumOptimizer, AdamOptimizer and etc... + # optimize op: SGDOptimize, MomentumOptimizer, AdamOptimizer and etc... if "Param" in op.input_names and \ "LearningRate" in op.input_names: return True diff --git a/python/paddle/fluid/distributed_spliter.py b/python/paddle/fluid/distributed_splitter.py similarity index 78% rename from python/paddle/fluid/distributed_spliter.py rename to python/paddle/fluid/distributed_splitter.py index d288b27ba0..060c1df8ad 100644 --- a/python/paddle/fluid/distributed_spliter.py +++ b/python/paddle/fluid/distributed_splitter.py @@ -17,8 +17,10 @@ def hash_name(varlist, pserver_endpoints): """ hash variable names to several endpoints. - :param varlist: a list of Variables - :return: a map of pserver endpoint -> varname + Args: + varlist(list): a list of Variables + + Returns(dict): a map of pserver endpoint -> varname """ def _hash_block(block_str, total): @@ -34,9 +36,14 @@ def hash_name(varlist, pserver_endpoints): def round_robin(varlist, pserver_endpoints): """ - distribute variables to several endpoints. + Distribute variables to several endpoints. + Args: + varlist(list): a list of variables + pserver_endpoints(list): a list of pserver endpoints + + Returns(list[int]): the endpoint for each variable """ - assert (len(varlist) > len(pserver_endpoints)) + assert (len(varlist) >= len(pserver_endpoints)) eplist = [] pserver_idx = 0 From 6ba262572c0e4a1b8c39830ed336693b301f39fd Mon Sep 17 00:00:00 2001 From: lgone2000 Date: Sat, 7 Apr 2018 12:09:02 +0800 Subject: [PATCH 43/57] fix test_conv2d_op when compile without cuda (#9698) --- python/paddle/fluid/tests/unittests/test_conv2d_op.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_op.py index 4b6e3fb69a..65606a0b43 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op.py @@ -97,8 +97,11 @@ class TestConv2dOp(OpTest): } self.outputs = {'Output': output} + def testcudnn(self): + return core.is_compiled_with_cuda() and self.use_cudnn + def test_check_output(self): - if self.use_cudnn: + if self.testcudnn(): place = core.CUDAPlace(0) self.check_output_with_place(place, atol=1e-5) else: @@ -107,7 +110,7 @@ class TestConv2dOp(OpTest): def test_check_grad(self): if self.dtype == np.float16: return - if self.use_cudnn: + if self.testcudnn(): place = core.CUDAPlace(0) self.check_grad_with_place( place, @@ -121,7 +124,7 @@ class TestConv2dOp(OpTest): def test_check_grad_no_filter(self): if self.dtype == np.float16: return - if self.use_cudnn: + if self.testcudnn(): place = core.CUDAPlace(0) self.check_grad_with_place( place, ['Input'], @@ -138,7 +141,7 @@ class TestConv2dOp(OpTest): def test_check_grad_no_input(self): if self.dtype == np.float16: return - if self.use_cudnn: + if self.testcudnn(): place = core.CUDAPlace(0) self.check_grad_with_place( place, ['Filter'], From 1543c4cf6ab2df8ec4e8f5b526674294ef3ec56d Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Apr 2018 21:15:58 -0700 Subject: [PATCH 44/57] Fix cpplint errors of paddle/fluid/pybind and add some tests (#9694) * cpplint test and add tesnor_py_test.cc * Update * Update --- paddle/fluid/pybind/CMakeLists.txt | 2 + paddle/fluid/pybind/const_value.cc | 12 +- paddle/fluid/pybind/const_value.h | 9 +- paddle/fluid/pybind/exception.cc | 7 +- paddle/fluid/pybind/exception.h | 7 +- paddle/fluid/pybind/protobuf.cc | 278 +++++++++++++------------- paddle/fluid/pybind/protobuf.h | 14 +- paddle/fluid/pybind/pybind.cc | 14 +- paddle/fluid/pybind/recordio.cc | 12 +- paddle/fluid/pybind/recordio.h | 3 +- paddle/fluid/pybind/tensor_py.h | 107 +++++----- paddle/fluid/pybind/tensor_py_test.cc | 44 ++++ 12 files changed, 291 insertions(+), 218 deletions(-) create mode 100644 paddle/fluid/pybind/tensor_py_test.cc diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index ada69ea4a4..787925d9f8 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -15,4 +15,6 @@ if(WITH_PYTHON) target_link_libraries(paddle_pybind rt) endif(NOT APPLE AND NOT ANDROID) endif(WITH_AMD_GPU) + + cc_test(tensor_py_test SRCS tensor_py_test.cc DEPS python) endif(WITH_PYTHON) diff --git a/paddle/fluid/pybind/const_value.cc b/paddle/fluid/pybind/const_value.cc index 6657b25ed2..3f28e61649 100644 --- a/paddle/fluid/pybind/const_value.cc +++ b/paddle/fluid/pybind/const_value.cc @@ -12,17 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "const_value.h" +#include "paddle/fluid/pybind/const_value.h" #include "paddle/fluid/framework/operator.h" namespace paddle { namespace pybind { -void BindConstValue(pybind11::module& m) { - m.def("kEmptyVarName", [] { return framework::kEmptyVarName; }); - m.def("kTempVarName", [] { return framework::kTempVarName; }); - m.def("kGradVarSuffix", [] { return framework::kGradVarSuffix; }); - m.def("kZeroVarSuffix", [] { return framework::kZeroVarSuffix; }); +void BindConstValue(pybind11::module* m) { + m->def("kEmptyVarName", [] { return framework::kEmptyVarName; }); + m->def("kTempVarName", [] { return framework::kTempVarName; }); + m->def("kGradVarSuffix", [] { return framework::kGradVarSuffix; }); + m->def("kZeroVarSuffix", [] { return framework::kZeroVarSuffix; }); } } // namespace pybind diff --git a/paddle/fluid/pybind/const_value.h b/paddle/fluid/pybind/const_value.h index 79e71e039d..2fab3160d1 100644 --- a/paddle/fluid/pybind/const_value.h +++ b/paddle/fluid/pybind/const_value.h @@ -11,16 +11,17 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once + #include + #include "paddle/fluid/platform/enforce.h" #include "pybind11/pybind11.h" -namespace py = pybind11; - namespace paddle { namespace pybind { -extern void BindConstValue(pybind11::module& m); + +void BindConstValue(pybind11::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/exception.cc b/paddle/fluid/pybind/exception.cc index 4bd3ecf728..08a2f185e1 100644 --- a/paddle/fluid/pybind/exception.cc +++ b/paddle/fluid/pybind/exception.cc @@ -17,8 +17,8 @@ limitations under the License. */ namespace paddle { namespace pybind { -void BindException(pybind11::module& m) { - static pybind11::exception exc(m, "EnforceNotMet"); +void BindException(pybind11::module* m) { + static pybind11::exception exc(*m, "EnforceNotMet"); pybind11::register_exception_translator([](std::exception_ptr p) { try { if (p) std::rethrow_exception(p); @@ -27,7 +27,8 @@ void BindException(pybind11::module& m) { } }); - m.def("__unittest_throw_exception__", [] { PADDLE_THROW("test exception"); }); + m->def("__unittest_throw_exception__", + [] { PADDLE_THROW("test exception"); }); } } // namespace pybind diff --git a/paddle/fluid/pybind/exception.h b/paddle/fluid/pybind/exception.h index bc6b0c0679..5e05426736 100644 --- a/paddle/fluid/pybind/exception.h +++ b/paddle/fluid/pybind/exception.h @@ -11,14 +11,17 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once + #include + #include "paddle/fluid/platform/enforce.h" #include "pybind11/pybind11.h" + namespace paddle { namespace pybind { -extern void BindException(pybind11::module& m); +void BindException(pybind11::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index 985984983a..2fe8290363 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -11,12 +11,13 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/fluid/pybind/protobuf.h" + #include #include #include #include + #include "paddle/fluid/framework/backward.h" #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/op_desc.h" @@ -97,10 +98,11 @@ struct type_caster> namespace paddle { namespace pybind { -using namespace paddle::framework; // NOLINT +namespace pd = paddle::framework; template -static py::bytes SerializeMessage(T &self) { // NOLINT +static pybind11::bytes SerializeMessage( + T &self) { // NOLINT due to pybind11 convention. // Check IsInitialized in Python std::string retv; PADDLE_ENFORCE(self.Proto()->SerializePartialToString(&retv), @@ -109,24 +111,24 @@ static py::bytes SerializeMessage(T &self) { // NOLINT } // Bind Methods -void BindProgramDesc(py::module &m) { // NOLINT - py::class_(m, "ProgramDesc", "") - .def(py::init<>()) +void BindProgramDesc(pybind11::module *m) { + pybind11::class_(*m, "ProgramDesc", "") + .def(pybind11::init<>()) .def("__init__", - [](ProgramDesc &self, const ProgramDesc &other) { - new (&self) ProgramDesc(other); + [](pd::ProgramDesc &self, const pd::ProgramDesc &other) { + new (&self) pd::ProgramDesc(other); }) .def("__init__", - [](ProgramDesc &self, const py::bytes &binary_str) { + [](pd::ProgramDesc &self, const pybind11::bytes &binary_str) { std::string str(binary_str); - new (&self) ProgramDesc(str); + new (&self) pd::ProgramDesc(str); }) - .def("append_block", &ProgramDesc::AppendBlock, - py::return_value_policy::reference) + .def("append_block", &pd::ProgramDesc::AppendBlock, + pybind11::return_value_policy::reference) .def("append_backward", - [](ProgramDesc &program_desc, const VarDesc &target, + [](pd::ProgramDesc &program_desc, const pd::VarDesc &target, const std::unordered_set &no_grad_vars) { - ParamGradInfoMap param_grad_map = + pd::ParamGradInfoMap param_grad_map = AppendBackward(program_desc, target, no_grad_vars); std::unordered_map< std::string, std::tuple) + .def("block", &pd::ProgramDesc::MutableBlock, + pybind11::return_value_policy::reference) + .def("num_blocks", &pd::ProgramDesc::Size) + .def("serialize_to_string", SerializeMessage) .def("parse_from_string", - [](ProgramDesc &program_desc, const std::string &data) { - proto::ProgramDesc *desc = program_desc.Proto(); + [](pd::ProgramDesc &program_desc, const std::string &data) { + pd::proto::ProgramDesc *desc = program_desc.Proto(); PADDLE_ENFORCE(desc->ParseFromString(data), "Fail to parse ProgramDesc from string. This could " "be a bug of Paddle."); }); } -void BindBlockDesc(py::module &m) { // NOLINT - py::class_(m, "BlockDesc", "") - .def_property_readonly("id", &BlockDesc::ID) - .def_property_readonly("parent", &BlockDesc::Parent) - .def("get_forward_block_idx", &BlockDesc::ForwardBlockID) - .def("set_forward_block_idx", &BlockDesc::SetForwardBlockID) - .def("append_op", &BlockDesc::AppendOp, - py::return_value_policy::reference) - .def("prepend_op", &BlockDesc::PrependOp, - py::return_value_policy::reference) - .def("insert_op", &BlockDesc::InsertOp, - py::return_value_policy::reference) - .def("remove_op", &BlockDesc::RemoveOp) +void BindBlockDesc(pybind11::module *m) { + pybind11::class_(*m, "BlockDesc", "") + .def_property_readonly("id", &pd::BlockDesc::ID) + .def_property_readonly("parent", &pd::BlockDesc::Parent) + .def("get_forward_block_idx", &pd::BlockDesc::ForwardBlockID) + .def("set_forward_block_idx", &pd::BlockDesc::SetForwardBlockID) + .def("append_op", &pd::BlockDesc::AppendOp, + pybind11::return_value_policy::reference) + .def("prepend_op", &pd::BlockDesc::PrependOp, + pybind11::return_value_policy::reference) + .def("insert_op", &pd::BlockDesc::InsertOp, + pybind11::return_value_policy::reference) + .def("remove_op", &pd::BlockDesc::RemoveOp) .def("var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.Var(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("has_var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.HasVar(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("rename_var", - [](BlockDesc &self, const py::bytes &byte_name, - const py::bytes &byte_name_new) { + [](pd::BlockDesc &self, const pybind11::bytes &byte_name, + const pybind11::bytes &byte_name_new) { std::string name = byte_name; std::string new_name = byte_name_new; self.RenameVar(name, new_name); }) .def("has_var_recursive", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.HasVarRecursive(name); }) .def("find_var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.FindVar(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("find_var_recursive", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.FindVarRecursive(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("remove_var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.RemoveVar(name); }, - py::return_value_policy::reference) - .def("all_vars", &BlockDesc::AllVars, py::return_value_policy::reference) - .def("op_size", &BlockDesc::OpSize) - .def("op", &BlockDesc::Op, py::return_value_policy::reference) - .def("serialize_to_string", SerializeMessage); + pybind11::return_value_policy::reference) + .def("all_vars", &pd::BlockDesc::AllVars, + pybind11::return_value_policy::reference) + .def("op_size", &pd::BlockDesc::OpSize) + .def("op", &pd::BlockDesc::Op, pybind11::return_value_policy::reference) + .def("serialize_to_string", SerializeMessage); } -void BindVarDsec(py::module &m) { // NOLINT - py::class_ var_desc(m, "VarDesc", ""); +void BindVarDsec(pybind11::module *m) { + pybind11::class_ var_desc(*m, "VarDesc", ""); var_desc .def("name", - [](VarDesc &self) { - py::bytes name = self.Name(); + [](pd::VarDesc &self) { + pybind11::bytes name = self.Name(); return name; }, - py::return_value_policy::reference) - .def("set_name", &VarDesc::SetName) - .def("set_shape", &VarDesc::SetShape) - .def("set_shapes", &VarDesc::SetShapes) - .def("set_dtype", &VarDesc::SetDataType) - .def("set_dtypes", &VarDesc::SetDataTypes) - .def("set_capacity", &VarDesc::SetCapacity) - .def("shape", &VarDesc::GetShape, py::return_value_policy::reference) - .def("shapes", &VarDesc::GetShapes, py::return_value_policy::reference) - .def("dtype", &VarDesc::GetDataType, py::return_value_policy::reference) - .def("dtypes", &VarDesc::GetDataTypes, py::return_value_policy::reference) - .def("lod_level", &VarDesc::GetLoDLevel) - .def("lod_levels", &VarDesc::GetLoDLevels, - py::return_value_policy::reference) - .def("set_lod_level", &VarDesc::SetLoDLevel) - .def("set_lod_levels", &VarDesc::SetLoDLevels) - .def("type", &VarDesc::GetType) - .def("set_type", &VarDesc::SetType) - .def("serialize_to_string", SerializeMessage) - .def("persistable", &VarDesc::Persistable) - .def("set_persistable", &VarDesc::SetPersistable); + pybind11::return_value_policy::reference) + .def("set_name", &pd::VarDesc::SetName) + .def("set_shape", &pd::VarDesc::SetShape) + .def("set_shapes", &pd::VarDesc::SetShapes) + .def("set_dtype", &pd::VarDesc::SetDataType) + .def("set_dtypes", &pd::VarDesc::SetDataTypes) + .def("set_capacity", &pd::VarDesc::SetCapacity) + .def("shape", &pd::VarDesc::GetShape, + pybind11::return_value_policy::reference) + .def("shapes", &pd::VarDesc::GetShapes, + pybind11::return_value_policy::reference) + .def("dtype", &pd::VarDesc::GetDataType, + pybind11::return_value_policy::reference) + .def("dtypes", &pd::VarDesc::GetDataTypes, + pybind11::return_value_policy::reference) + .def("lod_level", &pd::VarDesc::GetLoDLevel) + .def("lod_levels", &pd::VarDesc::GetLoDLevels, + pybind11::return_value_policy::reference) + .def("set_lod_level", &pd::VarDesc::SetLoDLevel) + .def("set_lod_levels", &pd::VarDesc::SetLoDLevels) + .def("type", &pd::VarDesc::GetType) + .def("set_type", &pd::VarDesc::SetType) + .def("serialize_to_string", SerializeMessage) + .def("persistable", &pd::VarDesc::Persistable) + .def("set_persistable", &pd::VarDesc::SetPersistable); - py::enum_(var_desc, "VarType", "") - .value("BOOL", proto::VarType::BOOL) - .value("INT16", proto::VarType::INT16) - .value("INT32", proto::VarType::INT32) - .value("INT64", proto::VarType::INT64) - .value("FP16", proto::VarType::FP16) - .value("FP32", proto::VarType::FP32) - .value("FP64", proto::VarType::FP64) - .value("LOD_TENSOR", proto::VarType::LOD_TENSOR) - .value("SELECTED_ROWS", proto::VarType::SELECTED_ROWS) - .value("FEED_MINIBATCH", proto::VarType::FEED_MINIBATCH) - .value("FETCH_LIST", proto::VarType::FETCH_LIST) - .value("STEP_SCOPES", proto::VarType::STEP_SCOPES) - .value("LOD_RANK_TABLE", proto::VarType::LOD_RANK_TABLE) - .value("LOD_TENSOR_ARRAY", proto::VarType::LOD_TENSOR_ARRAY) - .value("CHANNEL", proto::VarType::CHANNEL) - .value("PLACE_LIST", proto::VarType::PLACE_LIST) - .value("READER", proto::VarType::READER) - .value("RAW", proto::VarType::RAW); + pybind11::enum_(var_desc, "VarType", "") + .value("BOOL", pd::proto::VarType::BOOL) + .value("INT16", pd::proto::VarType::INT16) + .value("INT32", pd::proto::VarType::INT32) + .value("INT64", pd::proto::VarType::INT64) + .value("FP16", pd::proto::VarType::FP16) + .value("FP32", pd::proto::VarType::FP32) + .value("FP64", pd::proto::VarType::FP64) + .value("LOD_TENSOR", pd::proto::VarType::LOD_TENSOR) + .value("SELECTED_ROWS", pd::proto::VarType::SELECTED_ROWS) + .value("FEED_MINIBATCH", pd::proto::VarType::FEED_MINIBATCH) + .value("FETCH_LIST", pd::proto::VarType::FETCH_LIST) + .value("STEP_SCOPES", pd::proto::VarType::STEP_SCOPES) + .value("LOD_RANK_TABLE", pd::proto::VarType::LOD_RANK_TABLE) + .value("LOD_TENSOR_ARRAY", pd::proto::VarType::LOD_TENSOR_ARRAY) + .value("CHANNEL", pd::proto::VarType::CHANNEL) + .value("PLACE_LIST", pd::proto::VarType::PLACE_LIST) + .value("READER", pd::proto::VarType::READER) + .value("RAW", pd::proto::VarType::RAW); } -void BindOpDesc(py::module &m) { // NOLINT - py::enum_(m, "AttrType", "") - .value("INT", proto::AttrType::INT) - .value("INTS", proto::AttrType::INTS) - .value("FLOAT", proto::AttrType::FLOAT) - .value("FLOATS", proto::AttrType::FLOATS) - .value("STRING", proto::AttrType::STRING) - .value("STRINGS", proto::AttrType::STRINGS) - .value("BOOL", proto::AttrType::BOOLEAN) - .value("BOOLS", proto::AttrType::BOOLEANS) - .value("BLOCK", proto::AttrType::BLOCK); +void BindOpDesc(pybind11::module *m) { + pybind11::enum_(*m, "AttrType", "") + .value("INT", pd::proto::AttrType::INT) + .value("INTS", pd::proto::AttrType::INTS) + .value("FLOAT", pd::proto::AttrType::FLOAT) + .value("FLOATS", pd::proto::AttrType::FLOATS) + .value("STRING", pd::proto::AttrType::STRING) + .value("STRINGS", pd::proto::AttrType::STRINGS) + .value("BOOL", pd::proto::AttrType::BOOLEAN) + .value("BOOLS", pd::proto::AttrType::BOOLEANS) + .value("BLOCK", pd::proto::AttrType::BLOCK); - py::class_ op_desc(m, "OpDesc", ""); + pybind11::class_ op_desc(*m, "OpDesc", ""); op_desc - .def("__init__", [](OpDesc &self) { new (&self) OpDesc(); }, - py::return_value_policy::reference) - .def("copy_from", &OpDesc::CopyFrom) - .def("type", &OpDesc::Type) - .def("set_type", &OpDesc::SetType) - .def("input", &OpDesc::Input) - .def("input_names", &OpDesc::InputNames) - .def("output", &OpDesc::Output) - .def("output_names", &OpDesc::OutputNames) - .def("set_input", &OpDesc::SetInput) - .def("set_output", &OpDesc::SetOutput) - .def("input_arg_names", &OpDesc::InputArgumentNames) - .def("output_arg_names", &OpDesc::OutputArgumentNames) - .def("rename_input", &OpDesc::RenameInput) - .def("rename_output", &OpDesc::RenameOutput) - .def("has_attr", &OpDesc::HasAttr) - .def("attr_type", &OpDesc::GetAttrType) - .def("attr_names", &OpDesc::AttrNames) - .def("set_attr", &OpDesc::SetAttr) - .def("attr", &OpDesc::GetAttr) - .def("set_block_attr", &OpDesc::SetBlockAttr) + .def("__init__", [](pd::OpDesc &self) { new (&self) pd::OpDesc(); }, + pybind11::return_value_policy::reference) + .def("copy_from", &pd::OpDesc::CopyFrom) + .def("type", &pd::OpDesc::Type) + .def("set_type", &pd::OpDesc::SetType) + .def("input", &pd::OpDesc::Input) + .def("input_names", &pd::OpDesc::InputNames) + .def("output", &pd::OpDesc::Output) + .def("output_names", &pd::OpDesc::OutputNames) + .def("set_input", &pd::OpDesc::SetInput) + .def("set_output", &pd::OpDesc::SetOutput) + .def("input_arg_names", &pd::OpDesc::InputArgumentNames) + .def("output_arg_names", &pd::OpDesc::OutputArgumentNames) + .def("rename_input", &pd::OpDesc::RenameInput) + .def("rename_output", &pd::OpDesc::RenameOutput) + .def("has_attr", &pd::OpDesc::HasAttr) + .def("attr_type", &pd::OpDesc::GetAttrType) + .def("attr_names", &pd::OpDesc::AttrNames) + .def("set_attr", &pd::OpDesc::SetAttr) + .def("attr", &pd::OpDesc::GetAttr) + .def("set_block_attr", &pd::OpDesc::SetBlockAttr) .def("set_serialized_attr", - [](OpDesc &self, const std::string &name, - const py::bytes &seriralized) { + [](pd::OpDesc &self, const std::string &name, + const pybind11::bytes &seriralized) { std::string ser(seriralized); self.SetAttr(name, ser); }) - .def("block_attr", &OpDesc::GetBlockAttr) - .def("check_attrs", &OpDesc::CheckAttrs) - .def("infer_shape", &OpDesc::InferShape) - .def("infer_var_type", &OpDesc::InferVarType) - .def("serialize_to_string", SerializeMessage) - .def("block", &OpDesc::Block, py::return_value_policy::reference); + .def("block_attr", &pd::OpDesc::GetBlockAttr) + .def("check_attrs", &pd::OpDesc::CheckAttrs) + .def("infer_shape", &pd::OpDesc::InferShape) + .def("infer_var_type", &pd::OpDesc::InferVarType) + .def("serialize_to_string", SerializeMessage) + .def("block", &pd::OpDesc::Block, + pybind11::return_value_policy::reference); } } // namespace pybind diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index d0dc8936b3..e7370672a8 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -11,25 +11,25 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include + #include #include + #include "paddle/fluid/platform/variant.h" #include "pybind11/numpy.h" #include "pybind11/pybind11.h" #include "pybind11/stl.h" -namespace py = pybind11; - namespace paddle { namespace pybind { -void BindProgramDesc(py::module& m); -void BindBlockDesc(py::module& m); -void BindVarDsec(py::module& m); -void BindOpDesc(py::module& m); +void BindProgramDesc(pybind11::module* m); +void BindBlockDesc(pybind11::module* m); +void BindVarDsec(pybind11::module* m); +void BindOpDesc(pybind11::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 9bc3ff5128..748ad75a99 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -74,7 +74,7 @@ PYBIND11_PLUGIN(core) { // not cause namespace pollution. using namespace paddle::framework; // NOLINT - BindException(m); + BindException(&m); py::class_(m, "Tensor", py::buffer_protocol()) .def_buffer( @@ -478,11 +478,11 @@ All parameter, weight, gradient are variables in Paddle. m.def("set_feed_variable", framework::SetFeedVariable); m.def("get_fetch_variable", framework::GetFetchVariable); - BindProgramDesc(m); - BindBlockDesc(m); - BindVarDsec(m); - BindOpDesc(m); - BindConstValue(m); + BindProgramDesc(&m); + BindBlockDesc(&m); + BindVarDsec(&m); + BindOpDesc(&m); + BindConstValue(&m); py::class_(m, "LodRankTable") .def("items", [](framework::LoDRankTable &table) { @@ -553,7 +553,7 @@ All parameter, weight, gradient are variables in Paddle. }) .def("run", &ParallelExecutor::Run); - BindRecordIOWriter(m); + BindRecordIOWriter(&m); return m.ptr(); } } // namespace pybind diff --git a/paddle/fluid/pybind/recordio.cc b/paddle/fluid/pybind/recordio.cc index 16f8bfb1a2..0644d91425 100644 --- a/paddle/fluid/pybind/recordio.cc +++ b/paddle/fluid/pybind/recordio.cc @@ -13,13 +13,19 @@ // limitations under the License. #include "paddle/fluid/pybind/recordio.h" + #include +#include +#include + #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/recordio/writer.h" namespace paddle { namespace pybind { +namespace { + class RecordIOWriter { public: RecordIOWriter(const std::string& filename, recordio::Compressor compressor, @@ -49,8 +55,10 @@ class RecordIOWriter { recordio::Writer writer_; }; -void BindRecordIOWriter(py::module& m) { - py::class_ writer(m, "RecordIOWriter", ""); +} // namespace + +void BindRecordIOWriter(py::module* m) { + py::class_ writer(*m, "RecordIOWriter", ""); py::enum_(writer, "Compressor", "") .value("Snappy", recordio::Compressor::kSnappy) .value("NoCompress", recordio::Compressor::kNoCompress); diff --git a/paddle/fluid/pybind/recordio.h b/paddle/fluid/pybind/recordio.h index 60e6a9e859..2555f9b719 100644 --- a/paddle/fluid/pybind/recordio.h +++ b/paddle/fluid/pybind/recordio.h @@ -21,6 +21,7 @@ namespace py = pybind11; namespace paddle { namespace pybind { -extern void BindRecordIOWriter(py::module& m); +void BindRecordIOWriter(py::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h index 868966433e..fbe953b2d8 100644 --- a/paddle/fluid/pybind/tensor_py.h +++ b/paddle/fluid/pybind/tensor_py.h @@ -23,12 +23,8 @@ limitations under the License. */ #include "pybind11/numpy.h" #include "pybind11/pybind11.h" -namespace py = pybind11; - namespace paddle { - namespace pybind { - namespace details { template @@ -36,16 +32,16 @@ struct CastToPyBufferImpl; template struct CastToPyBufferImpl { - py::buffer_info operator()(framework::Tensor &tensor) { + pybind11::buffer_info operator()(const framework::Tensor &tensor) { PADDLE_THROW("This type of tensor cannot be expose to Python"); - return py::buffer_info(); + return pybind11::buffer_info(); } }; template struct CastToPyBufferImpl { using CUR_TYPE = typename std::tuple_element>::type; - py::buffer_info operator()(framework::Tensor &tensor) { + pybind11::buffer_info operator()(const framework::Tensor &tensor) { if (std::type_index(typeid(CUR_TYPE)) == tensor.type()) { auto dim_vec = framework::vectorize(tensor.dims()); std::vector dims_outside; @@ -84,15 +80,15 @@ struct CastToPyBufferImpl { if (std::type_index(typeid(CUR_TYPE)) == std::type_index(typeid(platform::float16))) { - return py::buffer_info(dst_tensor.data(), sizeof(CUR_TYPE), - "e", /* np.dtype('e') == np.float16 */ - (size_t)framework::arity(dst_tensor.dims()), - dims_outside, strides); + return pybind11::buffer_info( + dst_tensor.data(), sizeof(CUR_TYPE), + "e", /* np.dtype('e') == np.float16 */ + (size_t)framework::arity(dst_tensor.dims()), dims_outside, strides); } else { - return py::buffer_info(dst_tensor.data(), sizeof(CUR_TYPE), - py::format_descriptor::format(), - (size_t)framework::arity(dst_tensor.dims()), - dims_outside, strides); + return pybind11::buffer_info( + dst_tensor.data(), sizeof(CUR_TYPE), + pybind11::format_descriptor::format(), + (size_t)framework::arity(dst_tensor.dims()), dims_outside, strides); } } else { constexpr bool less = I + 1 < std::tuple_size>::value; @@ -103,7 +99,7 @@ struct CastToPyBufferImpl { } // namespace details -inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { +inline pybind11::buffer_info CastToPyBuffer(const framework::Tensor &tensor) { auto buffer_info = details::CastToPyBufferImpl()(tensor); @@ -111,7 +107,7 @@ inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { } template -T TensorGetElement(framework::Tensor &self, size_t offset) { +T TensorGetElement(const framework::Tensor &self, size_t offset) { if (platform::is_cpu_place(self.place())) { return self.data()[offset]; } else { @@ -123,31 +119,32 @@ T TensorGetElement(framework::Tensor &self, size_t offset) { // TODO(dzhwinter) : fix the redundent Tensor allocate and free template -void TensorSetElement(framework::Tensor &self, size_t offset, T elem) { - if (platform::is_gpu_place(self.place())) { +void TensorSetElement(framework::Tensor *self, size_t offset, T elem) { + if (platform::is_gpu_place(self->place())) { std::shared_ptr dst(new framework::Tensor); - framework::TensorCopy(self, platform::CPUPlace(), dst.get()); + framework::TensorCopy(*self, platform::CPUPlace(), dst.get()); dst->data()[offset] = elem; - framework::TensorCopy(*dst.get(), self.place(), &self); + framework::TensorCopy(*dst.get(), self->place(), self); - } else if (platform::is_cpu_place(self.place())) { - self.data()[offset] = elem; + } else if (platform::is_cpu_place(self->place())) { + self->data()[offset] = elem; } } template void PyCPUTensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CPUPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CPUPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(T) * array.size()); } @@ -155,34 +152,37 @@ template <> // This following specialization maps uint16_t in the parameter type to // platform::float16. void PyCPUTensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CPUPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CPUPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(uint16_t) * array.size()); } #ifdef PADDLE_WITH_CUDA template void PyCUDATensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CUDAPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CUDAPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); auto dev_ctx = @@ -195,17 +195,19 @@ template <> // This following specialization maps uint16_t in the parameter type to // platform::float16. void PyCUDATensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CUDAPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CUDAPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); auto dev_ctx = @@ -217,8 +219,9 @@ void PyCUDATensorSetFromArray( template void PyCUDAPinnedTensorSetFromArray( - framework::Tensor &self, - py::array_t array, + framework::Tensor *self, + pybind11::array_t + array, const paddle::platform::CUDAPinnedPlace &place) { std::vector dims; dims.reserve(array.ndim()); @@ -226,8 +229,8 @@ void PyCUDAPinnedTensorSetFromArray( dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(T) * array.size()); } @@ -235,8 +238,10 @@ template <> // This following specialization maps uint16_t in the parameter type to // platform::float16. void PyCUDAPinnedTensorSetFromArray( - framework::Tensor &self, - py::array_t array, + framework::Tensor *self, + pybind11::array_t + array, const paddle::platform::CUDAPinnedPlace &place) { std::vector dims; dims.reserve(array.ndim()); @@ -244,8 +249,8 @@ void PyCUDAPinnedTensorSetFromArray( dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(uint16_t) * array.size()); } #endif diff --git a/paddle/fluid/pybind/tensor_py_test.cc b/paddle/fluid/pybind/tensor_py_test.cc new file mode 100644 index 0000000000..1a0ae1d658 --- /dev/null +++ b/paddle/fluid/pybind/tensor_py_test.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pybind/tensor_py.h" + +#include + +#include "gtest/gtest.h" +#include "paddle/fluid/framework/tensor.h" + +TEST(TensorPy, CastToPyBufferImpl) { + typedef int ElemType; + + paddle::framework::Tensor t; + auto d = paddle::framework::make_ddim({1, 2, 3}); + int* p = t.mutable_data(d, paddle::platform::CPUPlace()); + for (int i = 0; i < paddle::framework::product(d); ++i) { + p[i] = i; + } + + pybind11::buffer_info bi = paddle::pybind::CastToPyBuffer(t); + EXPECT_EQ(bi.itemsize, static_cast(sizeof(ElemType))); + EXPECT_EQ(bi.size, static_cast(paddle::framework::product(d))); + EXPECT_EQ(bi.ndim, static_cast(3)); // 3-dimensional as d. + EXPECT_EQ(bi.shape.size(), 3U); // as Dim d. + EXPECT_EQ(bi.shape[0], static_cast(1)); + EXPECT_EQ(bi.shape[1], static_cast(2)); + EXPECT_EQ(bi.shape[2], static_cast(3)); + EXPECT_EQ(bi.strides.size(), 3U); // 3-dimensional as d. + EXPECT_EQ(bi.strides[2], static_cast(sizeof(ElemType))); + EXPECT_EQ(bi.strides[1], static_cast(sizeof(ElemType) * 3)); + EXPECT_EQ(bi.strides[0], static_cast(sizeof(ElemType) * 2 * 3)); +} From 5bb7d59e3aefc77ca8c8457d35082a66beafce75 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 00:14:10 -0700 Subject: [PATCH 45/57] Fix cpplint errors with paddle/fluid/inference (#9702) * Correct inference * Update * Update --- .../tests/book/test_inference_fit_a_line.cc | 4 +- .../test_inference_image_classification.cc | 4 +- .../test_inference_label_semantic_roles.cc | 18 +++---- .../book/test_inference_recognize_digits.cc | 4 +- .../book/test_inference_recommender_system.cc | 16 +++--- .../test_inference_rnn_encoder_decoder.cc | 6 +-- .../test_inference_understand_sentiment.cc | 4 +- .../tests/book/test_inference_word2vec.cc | 10 ++-- paddle/fluid/inference/tests/test_helper.h | 50 +++++++++++-------- 9 files changed, 61 insertions(+), 55 deletions(-) diff --git a/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc b/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc index 8d8365a839..3e77dc166c 100644 --- a/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc +++ b/paddle/fluid/inference/tests/book/test_inference_fit_a_line.cc @@ -9,8 +9,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -30,7 +30,7 @@ TEST(inference, fit_a_line) { // The second dim of the input tensor should be 13 // The input data should be >= 0 int64_t batch_size = 10; - SetupTensor(input, {batch_size, 13}, static_cast(0), + SetupTensor(&input, {batch_size, 13}, static_cast(0), static_cast(10)); std::vector cpu_feeds; cpu_feeds.push_back(&input); diff --git a/paddle/fluid/inference/tests/book/test_inference_image_classification.cc b/paddle/fluid/inference/tests/book/test_inference_image_classification.cc index 954ca7a3e3..a6b6c3f828 100644 --- a/paddle/fluid/inference/tests/book/test_inference_image_classification.cc +++ b/paddle/fluid/inference/tests/book/test_inference_image_classification.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -35,7 +35,7 @@ TEST(inference, image_classification) { paddle::framework::LoDTensor input; // Use normilized image pixels as input data, // which should be in the range [0.0, 1.0]. - SetupTensor(input, {FLAGS_batch_size, 3, 32, 32}, + SetupTensor(&input, {FLAGS_batch_size, 3, 32, 32}, static_cast(0), static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&input); diff --git a/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc b/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc index 31100494ff..84bb855fea 100644 --- a/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc +++ b/paddle/fluid/inference/tests/book/test_inference_label_semantic_roles.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -36,21 +36,21 @@ TEST(inference, label_semantic_roles) { int64_t predicate_dict_len = 3162; int64_t mark_dict_len = 2; - SetupLoDTensor(word, lod, static_cast(0), + SetupLoDTensor(&word, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(predicate, lod, static_cast(0), + SetupLoDTensor(&predicate, lod, static_cast(0), static_cast(predicate_dict_len - 1)); - SetupLoDTensor(ctx_n2, lod, static_cast(0), + SetupLoDTensor(&ctx_n2, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_n1, lod, static_cast(0), + SetupLoDTensor(&ctx_n1, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_0, lod, static_cast(0), + SetupLoDTensor(&ctx_0, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_p1, lod, static_cast(0), + SetupLoDTensor(&ctx_p1, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(ctx_p2, lod, static_cast(0), + SetupLoDTensor(&ctx_p2, lod, static_cast(0), static_cast(word_dict_len - 1)); - SetupLoDTensor(mark, lod, static_cast(0), + SetupLoDTensor(&mark, lod, static_cast(0), static_cast(mark_dict_len - 1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc b/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc index 82275d3ee1..f12828a268 100644 --- a/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc +++ b/paddle/fluid/inference/tests/book/test_inference_recognize_digits.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -35,7 +35,7 @@ TEST(inference, recognize_digits) { paddle::framework::LoDTensor input; // Use normilized image pixels as input data, // which should be in the range [-1.0, 1.0]. - SetupTensor(input, {FLAGS_batch_size, 1, 28, 28}, + SetupTensor(&input, {FLAGS_batch_size, 1, 28, 28}, static_cast(-1), static_cast(1)); std::vector cpu_feeds; cpu_feeds.push_back(&input); diff --git a/paddle/fluid/inference/tests/book/test_inference_recommender_system.cc b/paddle/fluid/inference/tests/book/test_inference_recommender_system.cc index b42a33c9a9..70aa6b194d 100644 --- a/paddle/fluid/inference/tests/book/test_inference_recommender_system.cc +++ b/paddle/fluid/inference/tests/book/test_inference_recommender_system.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -36,25 +36,25 @@ TEST(inference, recommender_system) { // Use the first data from paddle.dataset.movielens.test() as input std::vector user_id_data = {1}; - SetupTensor(user_id, {batch_size, 1}, user_id_data); + SetupTensor(&user_id, {batch_size, 1}, user_id_data); std::vector gender_id_data = {1}; - SetupTensor(gender_id, {batch_size, 1}, gender_id_data); + SetupTensor(&gender_id, {batch_size, 1}, gender_id_data); std::vector age_id_data = {0}; - SetupTensor(age_id, {batch_size, 1}, age_id_data); + SetupTensor(&age_id, {batch_size, 1}, age_id_data); std::vector job_id_data = {10}; - SetupTensor(job_id, {batch_size, 1}, job_id_data); + SetupTensor(&job_id, {batch_size, 1}, job_id_data); std::vector movie_id_data = {783}; - SetupTensor(movie_id, {batch_size, 1}, movie_id_data); + SetupTensor(&movie_id, {batch_size, 1}, movie_id_data); std::vector category_id_data = {10, 8, 9}; - SetupLoDTensor(category_id, {3, 1}, {{0, 3}}, category_id_data); + SetupLoDTensor(&category_id, {3, 1}, {{0, 3}}, category_id_data); std::vector movie_title_data = {1069, 4140, 2923, 710, 988}; - SetupLoDTensor(movie_title, {5, 1}, {{0, 5}}, movie_title_data); + SetupLoDTensor(&movie_title, {5, 1}, {{0, 5}}, movie_title_data); std::vector cpu_feeds; cpu_feeds.push_back(&user_id); diff --git a/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc b/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc index ea2d5ee092..e15c3f59ac 100644 --- a/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc +++ b/paddle/fluid/inference/tests/book/test_inference_rnn_encoder_decoder.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -32,9 +32,9 @@ TEST(inference, rnn_encoder_decoder) { paddle::framework::LoDTensor word_data, trg_word; paddle::framework::LoD lod{{0, 4, 10}}; - SetupLoDTensor(word_data, lod, static_cast(0), + SetupLoDTensor(&word_data, lod, static_cast(0), static_cast(1)); - SetupLoDTensor(trg_word, lod, static_cast(0), + SetupLoDTensor(&trg_word, lod, static_cast(0), static_cast(1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc b/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc index 21ffd8d368..0dbb6a3040 100644 --- a/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc +++ b/paddle/fluid/inference/tests/book/test_inference_understand_sentiment.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -33,7 +33,7 @@ TEST(inference, understand_sentiment) { paddle::framework::LoD lod{{0, 4, 10}}; int64_t word_dict_len = 5147; - SetupLoDTensor(words, lod, static_cast(0), + SetupLoDTensor(&words, lod, static_cast(0), static_cast(word_dict_len - 1)); std::vector cpu_feeds; diff --git a/paddle/fluid/inference/tests/book/test_inference_word2vec.cc b/paddle/fluid/inference/tests/book/test_inference_word2vec.cc index 1481760c52..c9328eb21b 100644 --- a/paddle/fluid/inference/tests/book/test_inference_word2vec.cc +++ b/paddle/fluid/inference/tests/book/test_inference_word2vec.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "gflags/gflags.h" +#include "gtest/gtest.h" #include "paddle/fluid/inference/tests/test_helper.h" DEFINE_string(dirname, "", "Directory of the inference model."); @@ -33,10 +33,10 @@ TEST(inference, word2vec) { paddle::framework::LoD lod{{0, 1}}; int64_t dict_size = 2073; // The size of dictionary - SetupLoDTensor(first_word, lod, static_cast(0), dict_size - 1); - SetupLoDTensor(second_word, lod, static_cast(0), dict_size - 1); - SetupLoDTensor(third_word, lod, static_cast(0), dict_size - 1); - SetupLoDTensor(fourth_word, lod, static_cast(0), dict_size - 1); + SetupLoDTensor(&first_word, lod, static_cast(0), dict_size - 1); + SetupLoDTensor(&second_word, lod, static_cast(0), dict_size - 1); + SetupLoDTensor(&third_word, lod, static_cast(0), dict_size - 1); + SetupLoDTensor(&fourth_word, lod, static_cast(0), dict_size - 1); std::vector cpu_feeds; cpu_feeds.push_back(&first_word); diff --git a/paddle/fluid/inference/tests/test_helper.h b/paddle/fluid/inference/tests/test_helper.h index d8ffedf672..5118e66f1e 100644 --- a/paddle/fluid/inference/tests/test_helper.h +++ b/paddle/fluid/inference/tests/test_helper.h @@ -11,53 +11,59 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#pragma once + +#include +#include +#include +#include -#include #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/inference/io.h" #include "paddle/fluid/platform/profiler.h" template -void SetupTensor(paddle::framework::LoDTensor& input, +void SetupTensor(paddle::framework::LoDTensor* input, paddle::framework::DDim dims, T lower, T upper) { - srand(time(0)); - T* input_ptr = input.mutable_data(dims, paddle::platform::CPUPlace()); - for (int i = 0; i < input.numel(); ++i) { - input_ptr[i] = - (static_cast(rand()) / static_cast(RAND_MAX)) * (upper - lower) + - lower; + std::mt19937 rng(100); // An arbitrarily chosen but fixed seed. + std::uniform_real_distribution uniform_dist(0, 1); + + T* input_ptr = input->mutable_data(dims, paddle::platform::CPUPlace()); + for (int i = 0; i < input->numel(); ++i) { + input_ptr[i] = static_cast(uniform_dist(rng) * (upper - lower) + lower); } } template -void SetupTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, std::vector& data) { +void SetupTensor(paddle::framework::LoDTensor* input, + paddle::framework::DDim dims, const std::vector& data) { CHECK_EQ(paddle::framework::product(dims), static_cast(data.size())); - T* input_ptr = input.mutable_data(dims, paddle::platform::CPUPlace()); - memcpy(input_ptr, data.data(), input.numel() * sizeof(T)); + T* input_ptr = input->mutable_data(dims, paddle::platform::CPUPlace()); + memcpy(input_ptr, data.data(), input->numel() * sizeof(T)); } template -void SetupLoDTensor(paddle::framework::LoDTensor& input, - paddle::framework::LoD& lod, T lower, T upper) { - input.set_lod(lod); +void SetupLoDTensor(paddle::framework::LoDTensor* input, + const paddle::framework::LoD& lod, T lower, T upper) { + input->set_lod(lod); int dim = lod[0][lod[0].size() - 1]; SetupTensor(input, {dim, 1}, lower, upper); } template -void SetupLoDTensor(paddle::framework::LoDTensor& input, - paddle::framework::DDim dims, paddle::framework::LoD lod, - std::vector& data) { +void SetupLoDTensor(paddle::framework::LoDTensor* input, + paddle::framework::DDim dims, + const paddle::framework::LoD lod, + const std::vector& data) { const size_t level = lod.size() - 1; CHECK_EQ(dims[0], static_cast((lod[level]).back())); - input.set_lod(lod); + input->set_lod(lod); SetupTensor(input, dims, data); } template -void CheckError(paddle::framework::LoDTensor& output1, - paddle::framework::LoDTensor& output2) { +void CheckError(const paddle::framework::LoDTensor& output1, + const paddle::framework::LoDTensor& output2) { // Check lod information EXPECT_EQ(output1.lod(), output2.lod()); @@ -85,7 +91,7 @@ void CheckError(paddle::framework::LoDTensor& output1, template void TestInference(const std::string& dirname, const std::vector& cpu_feeds, - std::vector& cpu_fetchs, + const std::vector& cpu_fetchs, const int repeat = 1, const bool is_combined = false) { // 1. Define place, executor, scope auto place = Place(); From ef4ee22668d0f135c0d3e7b3cc17fdc5262181a3 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 08:32:01 -0700 Subject: [PATCH 46/57] Fix cpplint errors with paddle/fluid/platform/cpu_info* (#9708) --- paddle/fluid/platform/cpu_info_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/platform/cpu_info_test.cc b/paddle/fluid/platform/cpu_info_test.cc index 78332f90cd..aac882e846 100644 --- a/paddle/fluid/platform/cpu_info_test.cc +++ b/paddle/fluid/platform/cpu_info_test.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "paddle/fluid/platform/cpu_info.h" -#include "paddle/fluid/string/printf.h" #include #include @@ -20,6 +19,7 @@ #include "gflags/gflags.h" #include "glog/logging.h" #include "gtest/gtest.h" +#include "paddle/fluid/string/printf.h" DECLARE_double(fraction_of_cpu_memory_to_use); From 809962625f1f6980ba7484acda437b16db6d32f6 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 08:32:52 -0700 Subject: [PATCH 47/57] Fix cpplint errors of enforce.* (#9706) --- paddle/fluid/platform/enforce.h | 30 +++++++++++++-------------- paddle/fluid/platform/enforce_test.cc | 4 ---- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index d303fd6d63..7b8c29e1e6 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -16,35 +16,35 @@ limitations under the License. */ #include // for dladdr #include // for backtrace + +#ifdef __GNUC__ +#include // for __cxa_demangle +#endif // __GNUC__ + +#ifdef PADDLE_WITH_CUDA +#include +#include +#include +#include +#include +#endif // PADDLE_WITH_CUDA + #include #include #include #include #include +#include "glog/logging.h" #include "paddle/fluid/platform/macros.h" #include "paddle/fluid/string/printf.h" #include "paddle/fluid/string/to_string.h" -#ifdef __GNUC__ -#include // for __cxa_demangle -#endif - -#include - #ifdef PADDLE_WITH_CUDA - #include "paddle/fluid/platform/dynload/cublas.h" #include "paddle/fluid/platform/dynload/cudnn.h" #include "paddle/fluid/platform/dynload/curand.h" #include "paddle/fluid/platform/dynload/nccl.h" - -#include -#include -#include -#include -#include - #endif namespace paddle { @@ -185,7 +185,7 @@ inline typename std::enable_if::type throw_on_error( } } -#endif // PADDLE_ONLY_CPU +#endif // PADDLE_WITH_CUDA template inline void throw_on_error(T e) { diff --git a/paddle/fluid/platform/enforce_test.cc b/paddle/fluid/platform/enforce_test.cc index bb9a3543ff..57d751cc00 100644 --- a/paddle/fluid/platform/enforce_test.cc +++ b/paddle/fluid/platform/enforce_test.cc @@ -96,7 +96,6 @@ TEST(ENFORCE_GT, FAIL) { bool caught_exception = false; try { PADDLE_ENFORCE_GT(1, 2UL); - } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; EXPECT_TRUE( @@ -115,7 +114,6 @@ TEST(ENFORCE_GE, FAIL) { bool caught_exception = false; try { PADDLE_ENFORCE_GE(1, 2UL); - } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; EXPECT_TRUE( @@ -135,7 +133,6 @@ TEST(ENFORCE_LE, FAIL) { bool caught_exception = false; try { PADDLE_ENFORCE_GT(1, 2UL); - } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; EXPECT_TRUE( @@ -171,7 +168,6 @@ TEST(ENFORCE_NOT_NULL, FAIL) { try { int* a = nullptr; PADDLE_ENFORCE_NOT_NULL(a); - } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; EXPECT_TRUE(HasPrefix(StringPiece(error.what()), "a should not be null")); From 55ffceaadbd3ea27cfb832d360bb2fed57cd5032 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 08:35:07 -0700 Subject: [PATCH 48/57] Fix cpplint errors paddle/fluid/platform/place.* (#9711) --- paddle/fluid/platform/place.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/platform/place.h b/paddle/fluid/platform/place.h index d0bdcb0da5..ad54a87899 100644 --- a/paddle/fluid/platform/place.h +++ b/paddle/fluid/platform/place.h @@ -11,10 +11,11 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include +#include + #include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/variant.h" From 0c43a376e22f002b067ad6a0e1cfeb0e3216b493 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 11:56:21 -0700 Subject: [PATCH 49/57] Fix cpplint errors with paddle/fluid/platform/gpu_info.* (#9710) * Fix cpplint errors with paddle/fluid/platform/gpu_info.* * Update --- paddle/fluid/memory/memory.cc | 2 +- paddle/fluid/platform/gpu_info.cc | 11 ++++++----- paddle/fluid/platform/gpu_info.h | 5 ++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/memory/memory.cc b/paddle/fluid/memory/memory.cc index 09f82166be..2c13dbc6d5 100644 --- a/paddle/fluid/memory/memory.cc +++ b/paddle/fluid/memory/memory.cc @@ -95,7 +95,7 @@ void* Alloc(platform::CUDAPlace place, size_t size) { int cur_dev = platform::GetCurrentDeviceId(); platform::SetDeviceId(place.device); size_t avail, total; - platform::GpuMemoryUsage(avail, total); + platform::GpuMemoryUsage(&avail, &total); LOG(WARNING) << "Cannot allocate " << size << " bytes in GPU " << place.device << ", available " << avail << " bytes"; LOG(WARNING) << "total " << total; diff --git a/paddle/fluid/platform/gpu_info.cc b/paddle/fluid/platform/gpu_info.cc index dd70ff9ff5..aaebeb1353 100644 --- a/paddle/fluid/platform/gpu_info.cc +++ b/paddle/fluid/platform/gpu_info.cc @@ -14,8 +14,9 @@ limitations under the License. */ #include "paddle/fluid/platform/gpu_info.h" -#include "gflags/gflags.h" +#include +#include "gflags/gflags.h" #include "paddle/fluid/platform/enforce.h" DEFINE_double(fraction_of_gpu_memory_to_use, 0.92, @@ -77,8 +78,8 @@ void SetDeviceId(int id) { "cudaSetDevice failed in paddle::platform::SetDeviceId"); } -void GpuMemoryUsage(size_t &available, size_t &total) { - PADDLE_ENFORCE(cudaMemGetInfo(&available, &total), +void GpuMemoryUsage(size_t *available, size_t *total) { + PADDLE_ENFORCE(cudaMemGetInfo(available, total), "cudaMemGetInfo failed in paddle::platform::GetMemoryUsage"); } @@ -86,7 +87,7 @@ size_t GpuMaxAllocSize() { size_t total = 0; size_t available = 0; - GpuMemoryUsage(available, total); + GpuMemoryUsage(&available, &total); // Reserve the rest for page tables, etc. return static_cast(total * FLAGS_fraction_of_gpu_memory_to_use); @@ -101,7 +102,7 @@ size_t GpuMaxChunkSize() { size_t total = 0; size_t available = 0; - GpuMemoryUsage(available, total); + GpuMemoryUsage(&available, &total); VLOG(10) << "GPU Usage " << available / 1024 / 1024 << "M/" << total / 1024 / 1024 << "M"; size_t reserving = static_cast(0.05 * total); diff --git a/paddle/fluid/platform/gpu_info.h b/paddle/fluid/platform/gpu_info.h index fa469fa77f..57962e6795 100644 --- a/paddle/fluid/platform/gpu_info.h +++ b/paddle/fluid/platform/gpu_info.h @@ -24,8 +24,7 @@ namespace paddle { namespace platform { //! Environment variable: fraction of GPU memory to use on each device. -const std::string kEnvFractionGpuMemoryToUse = - "PADDLE_FRACTION_GPU_MEMORY_TO_USE"; +const char kEnvFractionGpuMemoryToUse[] = "PADDLE_FRACTION_GPU_MEMORY_TO_USE"; //! Get the total number of GPU devices in system. int GetCUDADeviceCount(); @@ -46,7 +45,7 @@ int GetCurrentDeviceId(); void SetDeviceId(int device_id); //! Get the memory usage of current GPU device. -void GpuMemoryUsage(size_t &available, size_t &total); +void GpuMemoryUsage(size_t *available, size_t *total); //! Get the maximum allocation size of current GPU device. size_t GpuMaxAllocSize(); From 544254fe4fb6eb2fbc1b69374b8ad2bbc709fa97 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 11:58:37 -0700 Subject: [PATCH 50/57] Correct fluid/memory (#9716) --- paddle/fluid/memory/memory_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/memory/memory_test.cc b/paddle/fluid/memory/memory_test.cc index 03829702a0..9fbbe62559 100644 --- a/paddle/fluid/memory/memory_test.cc +++ b/paddle/fluid/memory/memory_test.cc @@ -13,16 +13,16 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/memory/memory.h" + +#include + +#include "gtest/gtest.h" #include "paddle/fluid/memory/detail/memory_block.h" #include "paddle/fluid/memory/detail/meta_data.h" - #include "paddle/fluid/platform/cpu_info.h" #include "paddle/fluid/platform/gpu_info.h" #include "paddle/fluid/platform/place.h" -#include -#include - inline bool is_aligned(void const *p) { return 0 == (reinterpret_cast(p) & 0x3); } From e185502ebe35c3d72ce1448fe55010374fab7807 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 13:14:21 -0700 Subject: [PATCH 51/57] Fix cpplint errors with paddle/fluid/platform/dynload (#9715) * Update source files. * Update headers * Update * Update * Update * Update * Fix a CMake dependency --- cmake/external/warpctc.cmake | 3 +- paddle/fluid/framework/details/CMakeLists.txt | 2 +- paddle/fluid/platform/dynload/cublas.h | 42 ++++----- paddle/fluid/platform/dynload/cudnn.cc | 3 +- paddle/fluid/platform/dynload/cudnn.h | 26 +++--- paddle/fluid/platform/dynload/cupti.h | 29 +++--- paddle/fluid/platform/dynload/curand.h | 29 +++--- .../fluid/platform/dynload/dynamic_loader.cc | 89 ++++++++++--------- .../fluid/platform/dynload/dynamic_loader.h | 56 ++---------- paddle/fluid/platform/dynload/nccl.cc | 5 -- paddle/fluid/platform/dynload/nccl.h | 28 +++--- paddle/fluid/platform/dynload/warpctc.h | 29 +++--- 12 files changed, 152 insertions(+), 189 deletions(-) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 9a9a20f897..a631ad14b1 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -62,7 +62,8 @@ ExternalProject_Add( ) MESSAGE(STATUS "warp-ctc library: ${WARPCTC_LIBRARIES}") -INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) # For warpctc code to include its headers. +INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include warpctc headers. ADD_LIBRARY(warpctc SHARED IMPORTED GLOBAL) SET_PROPERTY(TARGET warpctc PROPERTY IMPORTED_LOCATION ${WARPCTC_LIBRARIES}) diff --git a/paddle/fluid/framework/details/CMakeLists.txt b/paddle/fluid/framework/details/CMakeLists.txt index bf1a705ef5..89b5c6847f 100644 --- a/paddle/fluid/framework/details/CMakeLists.txt +++ b/paddle/fluid/framework/details/CMakeLists.txt @@ -16,6 +16,6 @@ else() endif() cc_library(multi_devices_graph_builder SRCS multi_devices_graph_builder.cc DEPS ssa_graph_builder computation_op_handle scale_loss_grad_op_handle ${multi_devices_graph_builder_deps}) -cc_library(ssa_graph_executor SRCS ssa_graph_executor.cc DEPS ssa_graph) +cc_library(ssa_graph_executor SRCS ssa_graph_executor.cc DEPS ssa_graph framework_proto) cc_library(threaded_ssa_graph_executor SRCS threaded_ssa_graph_executor.cc DEPS fetch_op_handle ssa_graph_executor scope simple_threadpool device_context) diff --git a/paddle/fluid/platform/dynload/cublas.h b/paddle/fluid/platform/dynload/cublas.h index 3b8d192b6c..a41018d350 100644 --- a/paddle/fluid/platform/dynload/cublas.h +++ b/paddle/fluid/platform/dynload/cublas.h @@ -1,16 +1,16 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ #pragma once @@ -35,18 +35,18 @@ extern void *cublas_dso_handle; * note: default dynamic linked libs */ #ifdef PADDLE_USE_DSO -#define DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - inline cublasStatus_t operator()(Args... args) { \ - typedef cublasStatus_t (*cublasFunc)(Args...); \ - std::call_once(cublas_dso_flag, \ - paddle::platform::dynload::GetCublasDsoHandle, \ - &cublas_dso_handle); \ - void *p_##__name = dlsym(cublas_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + inline cublasStatus_t operator()(Args... args) { \ + typedef cublasStatus_t (*cublasFunc)(Args...); \ + std::call_once(cublas_dso_flag, []() { \ + cublas_dso_handle = paddle::platform::dynload::GetCublasDsoHandle(); \ + }); \ + void *p_##__name = dlsym(cublas_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ diff --git a/paddle/fluid/platform/dynload/cudnn.cc b/paddle/fluid/platform/dynload/cudnn.cc index c65b060ab4..f3cd3b2bbe 100644 --- a/paddle/fluid/platform/dynload/cudnn.cc +++ b/paddle/fluid/platform/dynload/cudnn.cc @@ -44,7 +44,8 @@ CUDNN_DNN_ROUTINE_EACH_R7(DEFINE_WRAP); #ifdef PADDLE_USE_DSO bool HasCUDNN() { - std::call_once(cudnn_dso_flag, GetCUDNNDsoHandle, &cudnn_dso_handle); + std::call_once(cudnn_dso_flag, + []() { cudnn_dso_handle = GetCUDNNDsoHandle(); }); return cudnn_dso_handle != nullptr; } diff --git a/paddle/fluid/platform/dynload/cudnn.h b/paddle/fluid/platform/dynload/cudnn.h index 49a54d8478..24475b62ca 100644 --- a/paddle/fluid/platform/dynload/cudnn.h +++ b/paddle/fluid/platform/dynload/cudnn.h @@ -30,19 +30,19 @@ extern bool HasCUDNN(); #ifdef PADDLE_USE_DSO extern void EnforceCUDNNLoaded(const char* fn_name); -#define DECLARE_DYNAMIC_LOAD_CUDNN_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> decltype(__name(args...)) { \ - using cudnn_func = decltype(__name(args...)) (*)(Args...); \ - std::call_once(cudnn_dso_flag, \ - paddle::platform::dynload::GetCUDNNDsoHandle, \ - &cudnn_dso_handle); \ - EnforceCUDNNLoaded(#__name); \ - void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using cudnn_func = decltype(__name(args...)) (*)(Args...); \ + std::call_once(cudnn_dso_flag, []() { \ + cudnn_dso_handle = paddle::platform::dynload::GetCUDNNDsoHandle(); \ + }); \ + EnforceCUDNNLoaded(#__name); \ + void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern struct DynLoad__##__name __name #else diff --git a/paddle/fluid/platform/dynload/cupti.h b/paddle/fluid/platform/dynload/cupti.h index c1bf88f8cb..d0d676b9d8 100644 --- a/paddle/fluid/platform/dynload/cupti.h +++ b/paddle/fluid/platform/dynload/cupti.h @@ -11,14 +11,15 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #ifdef PADDLE_WITH_CUPTI + #include #include #include -#include +#include // NOLINT + #include "paddle/fluid/platform/dynload/dynamic_loader.h" namespace paddle { @@ -36,18 +37,18 @@ extern void *cupti_dso_handle; * note: default dynamic linked libs */ #ifdef PADDLE_USE_DSO -#define DECLARE_DYNAMIC_LOAD_CUPTI_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - inline CUptiResult CUPTIAPI operator()(Args... args) { \ - typedef CUptiResult CUPTIAPI (*cuptiFunc)(Args...); \ - std::call_once(cupti_dso_flag, \ - paddle::platform::dynload::GetCUPTIDsoHandle, \ - &cupti_dso_handle); \ - void *p_##__name = dlsym(cupti_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_CUPTI_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + inline CUptiResult CUPTIAPI operator()(Args... args) { \ + typedef CUptiResult CUPTIAPI (*cuptiFunc)(Args...); \ + std::call_once(cupti_dso_flag, []() { \ + cupti_dso_handle = paddle::platform::dynload::GetCUPTIDsoHandle(); \ + }); \ + void *p_##__name = dlsym(cupti_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_CUPTI_WRAP(__name) \ diff --git a/paddle/fluid/platform/dynload/curand.h b/paddle/fluid/platform/dynload/curand.h index 1b3ff962d6..4697fb6cd9 100644 --- a/paddle/fluid/platform/dynload/curand.h +++ b/paddle/fluid/platform/dynload/curand.h @@ -11,12 +11,13 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include #include -#include + +#include // NOLINT + #include "paddle/fluid/platform/dynload/dynamic_loader.h" namespace paddle { @@ -25,18 +26,18 @@ namespace dynload { extern std::once_flag curand_dso_flag; extern void *curand_dso_handle; #ifdef PADDLE_USE_DSO -#define DECLARE_DYNAMIC_LOAD_CURAND_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - curandStatus_t operator()(Args... args) { \ - typedef curandStatus_t (*curandFunc)(Args...); \ - std::call_once(curand_dso_flag, \ - paddle::platform::dynload::GetCurandDsoHandle, \ - &curand_dso_handle); \ - void *p_##__name = dlsym(curand_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_CURAND_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + curandStatus_t operator()(Args... args) { \ + typedef curandStatus_t (*curandFunc)(Args...); \ + std::call_once(curand_dso_flag, []() { \ + curand_dso_handle = paddle::platform::dynload::GetCurandDsoHandle(); \ + }); \ + void *p_##__name = dlsym(curand_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_CURAND_WRAP(__name) \ diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index e590e81bab..3c1ccc7445 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -11,12 +11,14 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/fluid/platform/dynload/dynamic_loader.h" + #include + #include -#include +#include // NOLINT #include + #include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/fluid/platform/dynload/cupti_lib_path.h" @@ -65,22 +67,21 @@ static inline std::string join(const std::string& part1, return ret; } -static inline void GetDsoHandleFromDefaultPath(std::string& dso_path, - void** dso_handle, - int dynload_flags) { +static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path, + int dynload_flags) { VLOG(3) << "Try to find library: " << dso_path << " from default system path."; // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH - *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + void* dso_handle = dlopen(dso_path.c_str(), dynload_flags); // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to // bring System Integrity Projection (SIP), if dso_handle // is null, search from default package path in Mac OS. #if defined(__APPLE__) || defined(__OSX__) - if (nullptr == *dso_handle) { - dso_path = join("/usr/local/cuda/lib/", dso_path); - *dso_handle = dlopen(dso_path.c_str(), dynload_flags); - if (nullptr == *dso_handle) { + if (nullptr == dso_handle) { + dso_handle = + dlopen(join("/usr/local/cuda/lib/", dso_path).c_str(), dynload_flags); + if (nullptr == dso_handle) { if (dso_path == "libcudnn.dylib") { LOG(WARNING) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n " "For instance, sudo tar -xzf " @@ -91,28 +92,29 @@ static inline void GetDsoHandleFromDefaultPath(std::string& dso_path, } } #endif + + return dso_handle; } -static inline void GetDsoHandleFromSearchPath(const std::string& search_root, - const std::string& dso_name, - void** dso_handle, - bool throw_on_error = true) { +static inline void* GetDsoHandleFromSearchPath(const std::string& search_root, + const std::string& dso_name, + bool throw_on_error = true) { int dynload_flags = RTLD_LAZY | RTLD_LOCAL; - *dso_handle = nullptr; + void* dso_handle = nullptr; std::string dlPath = dso_name; if (search_root.empty()) { - GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); + dso_handle = GetDsoHandleFromDefaultPath(dlPath, dynload_flags); } else { // search xxx.so from custom path dlPath = join(search_root, dso_name); - *dso_handle = dlopen(dlPath.c_str(), dynload_flags); + dso_handle = dlopen(dlPath.c_str(), dynload_flags); // if not found, search from default path - if (nullptr == *dso_handle) { + if (nullptr == dso_handle) { LOG(WARNING) << "Failed to find dynamic library: " << dlPath << " (" << dlerror() << ")"; dlPath = dso_name; - GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); + dso_handle = GetDsoHandleFromDefaultPath(dlPath, dynload_flags); } } auto error_msg = @@ -124,70 +126,71 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, "using the DYLD_LIBRARY_PATH is impossible unless System " "Integrity Protection (SIP) is disabled."; if (throw_on_error) { - PADDLE_ENFORCE(nullptr != *dso_handle, error_msg, dlPath, dlerror()); - } else if (nullptr == *dso_handle) { + PADDLE_ENFORCE(nullptr != dso_handle, error_msg, dlPath, dlerror()); + } else if (nullptr == dso_handle) { LOG(WARNING) << string::Sprintf(error_msg, dlPath, dlerror()); } + + return dso_handle; } -void GetCublasDsoHandle(void** dso_handle) { +void* GetCublasDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib"); #else - GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so"); #endif } -void GetCUDNNDsoHandle(void** dso_handle) { +void* GetCUDNNDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle, - false); + return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", false); #else - GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle, false); + return GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", false); #endif } -void GetCUPTIDsoHandle(void** dso_handle) { +void* GetCUPTIDsoHandle() { std::string cupti_path = cupti_lib_path; if (!FLAGS_cupti_dir.empty()) { cupti_path = FLAGS_cupti_dir; } #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(cupti_path, "libcupti.dylib", dso_handle, false); + return GetDsoHandleFromSearchPath(cupti_path, "libcupti.dylib", false); #else - GetDsoHandleFromSearchPath(cupti_path, "libcupti.so", dso_handle, false); + return GetDsoHandleFromSearchPath(cupti_path, "libcupti.so", false); #endif } -void GetCurandDsoHandle(void** dso_handle) { +void* GetCurandDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib"); #else - GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so"); #endif } -void GetWarpCTCDsoHandle(void** dso_handle) { +void* GetWarpCTCDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(FLAGS_warpctc_dir, "libwarpctc.dylib", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_warpctc_dir, "libwarpctc.dylib"); #else - GetDsoHandleFromSearchPath(FLAGS_warpctc_dir, "libwarpctc.so", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_warpctc_dir, "libwarpctc.so"); #endif } -void GetLapackDsoHandle(void** dso_handle) { +void* GetLapackDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapacke.dylib", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapacke.dylib"); #else - GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapacke.so", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapacke.so"); #endif } -void GetNCCLDsoHandle(void** dso_handle) { +void* GetNCCLDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.dylib", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.dylib"); #else - GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.so", dso_handle); + return GetDsoHandleFromSearchPath(FLAGS_nccl_dir, "libnccl.so"); #endif } diff --git a/paddle/fluid/platform/dynload/dynamic_loader.h b/paddle/fluid/platform/dynload/dynamic_loader.h index b5b9c4af91..4c85093a43 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.h +++ b/paddle/fluid/platform/dynload/dynamic_loader.h @@ -18,55 +18,13 @@ namespace paddle { namespace platform { namespace dynload { -/** - * @brief load the DSO of CUBLAS - * - * @param **dso_handle dso handler - * - */ -void GetCublasDsoHandle(void** dso_handle); - -/** - * @brief load the DSO of CUDNN - * - * @param **dso_handle dso handler - * - */ -void GetCUDNNDsoHandle(void** dso_handle); - -void GetCUPTIDsoHandle(void** dso_handle); - -/** - * @brief load the DSO of CURAND - * - * @param **dso_handle dso handler - * - */ -void GetCurandDsoHandle(void** dso_handle); - -/** - * @brief load the DSO of warp-ctc - * - * @param **dso_handle dso handler - * - */ -void GetWarpCTCDsoHandle(void** dso_handle); - -/** - * @brief load the DSO of lapack - * - * @param **dso_handle dso handler - * - */ -void GetLapackDsoHandle(void** dso_handle); - -/** - * @brief load the DSO of NVIDIA nccl - * - * @param **dso_handle dso handler - * - */ -void GetNCCLDsoHandle(void** dso_handle); +void* GetCublasDsoHandle(); +void* GetCUDNNDsoHandle(); +void* GetCUPTIDsoHandle(); +void* GetCurandDsoHandle(); +void* GetWarpCTCDsoHandle(); +void* GetLapackDsoHandle(); +void* GetNCCLDsoHandle(); } // namespace dynload } // namespace platform diff --git a/paddle/fluid/platform/dynload/nccl.cc b/paddle/fluid/platform/dynload/nccl.cc index 3edc70c46d..2c40c48ee0 100644 --- a/paddle/fluid/platform/dynload/nccl.cc +++ b/paddle/fluid/platform/dynload/nccl.cc @@ -25,11 +25,6 @@ void *nccl_dso_handle; NCCL_RAND_ROUTINE_EACH(DEFINE_WRAP); -void LoadNCCLDSO() { - platform::call_once(nccl_dso_flag, - [] { GetNCCLDsoHandle(&nccl_dso_handle); }); -} - } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/fluid/platform/dynload/nccl.h b/paddle/fluid/platform/dynload/nccl.h index dc78bcb44d..d21e29df3c 100644 --- a/paddle/fluid/platform/dynload/nccl.h +++ b/paddle/fluid/platform/dynload/nccl.h @@ -11,12 +11,13 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include #include -#include + +#include // NOLINT + #include "paddle/fluid/platform/call_once.h" #include "paddle/fluid/platform/dynload/dynamic_loader.h" @@ -28,18 +29,19 @@ extern std::once_flag nccl_dso_flag; extern void* nccl_dso_handle; #ifdef PADDLE_USE_DSO -extern void LoadNCCLDSO(); -#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> decltype(__name(args...)) { \ - using nccl_func = decltype(__name(args...)) (*)(Args...); \ - paddle::platform::dynload::LoadNCCLDSO(); \ - void* p_##__name = dlsym(nccl_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using nccl_func = decltype(__name(args...)) (*)(Args...); \ + std::call_once(nccl_dso_flag, []() { \ + nccl_dso_handle = paddle::platform::dynload::GetNCCLDsoHandle(); \ + }); \ + void* p_##__name = dlsym(nccl_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ diff --git a/paddle/fluid/platform/dynload/warpctc.h b/paddle/fluid/platform/dynload/warpctc.h index f5ded0eb6b..7fa4683704 100644 --- a/paddle/fluid/platform/dynload/warpctc.h +++ b/paddle/fluid/platform/dynload/warpctc.h @@ -15,9 +15,10 @@ limitations under the License. */ #pragma once #include -#include -#include "ctc.h" +#include // NOLINT + #include "paddle/fluid/platform/dynload/dynamic_loader.h" +#include "warpctc/include/ctc.h" namespace paddle { namespace platform { @@ -31,18 +32,18 @@ extern void* warpctc_dso_handle; * (for each function) to dynamic load warpctc routine * via operator overloading. */ -#define DYNAMIC_LOAD_WARPCTC_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> decltype(__name(args...)) { \ - using warpctcFunc = decltype(__name(args...)) (*)(Args...); \ - std::call_once(warpctc_dso_flag, \ - paddle::platform::dynload::GetWarpCTCDsoHandle, \ - &warpctc_dso_handle); \ - void* p_##_name = dlsym(warpctc_dso_handle, #__name); \ - return reinterpret_cast(p_##_name)(args...); \ - } \ - }; \ +#define DYNAMIC_LOAD_WARPCTC_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using warpctcFunc = decltype(__name(args...)) (*)(Args...); \ + std::call_once(warpctc_dso_flag, []() { \ + warpctc_dso_handle = paddle::platform::dynload::GetWarpCTCDsoHandle(); \ + }); \ + void* p_##_name = dlsym(warpctc_dso_handle, #__name); \ + return reinterpret_cast(p_##_name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #define DECLARE_DYNAMIC_LOAD_WARPCTC_WRAP(__name) \ From 535646cf256cef422f34ebe19a7a1e311e4225d6 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 7 Apr 2018 17:37:42 -0700 Subject: [PATCH 52/57] Update (#9717) --- paddle/fluid/platform/gpu_info.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/paddle/fluid/platform/gpu_info.h b/paddle/fluid/platform/gpu_info.h index 57962e6795..36345e1740 100644 --- a/paddle/fluid/platform/gpu_info.h +++ b/paddle/fluid/platform/gpu_info.h @@ -23,9 +23,6 @@ limitations under the License. */ namespace paddle { namespace platform { -//! Environment variable: fraction of GPU memory to use on each device. -const char kEnvFractionGpuMemoryToUse[] = "PADDLE_FRACTION_GPU_MEMORY_TO_USE"; - //! Get the total number of GPU devices in system. int GetCUDADeviceCount(); From 6651301b2926790081e4edb2ad4ef7155fd08581 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Sun, 8 Apr 2018 10:41:53 +0800 Subject: [PATCH 53/57] Modify comments of `sync_with_cpp` --- python/paddle/fluid/framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 39d4017861..401e26f474 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -838,7 +838,7 @@ class Block(object): def sync_with_cpp(self): """ - Sync with the desc on the c++ end. + Sync from the desc on the c++ end. This method is used to synchronize the c++ desc instance generated by backward. """ From 50e036a4ed37b728d91342ceb8e05f804b185264 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Sun, 8 Apr 2018 10:43:14 +0800 Subject: [PATCH 54/57] fix compiler error on `tensor_py.h` --- paddle/fluid/pybind/tensor_py.h | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h index fbe953b2d8..4a9dbd324c 100644 --- a/paddle/fluid/pybind/tensor_py.h +++ b/paddle/fluid/pybind/tensor_py.h @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#include #include #include #include From de6802d2bd53a471de6a4ba644a27a96d718aa6c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 8 Apr 2018 11:07:54 +0800 Subject: [PATCH 55/57] fix missing core.so on mac --- python/setup.py.in | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py.in b/python/setup.py.in index 2707d34a2a..5e7096e225 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -107,6 +107,7 @@ package_dir={ # So that package points to other directory. 'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform', 'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework', + 'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid', } if '${WITH_FLUID_ONLY}'== 'OFF': package_dir['py_paddle']='${PADDLE_BINARY_DIR}/python/py_paddle' From 756a7d9d506780c8d395d12be1b798a073049f55 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Sun, 8 Apr 2018 11:10:57 +0800 Subject: [PATCH 56/57] rm filemanager --- doc/design/file_manager/README.md | 87 ------------ doc/design/file_manager/pfs/pfsclient.md | 129 ------------------ .../file_manager/src/filemanager.graffle | Bin 3438 -> 0 bytes doc/design/file_manager/src/filemanager.png | Bin 145125 -> 0 bytes 4 files changed, 216 deletions(-) delete mode 100644 doc/design/file_manager/README.md delete mode 100644 doc/design/file_manager/pfs/pfsclient.md delete mode 100644 doc/design/file_manager/src/filemanager.graffle delete mode 100644 doc/design/file_manager/src/filemanager.png diff --git a/doc/design/file_manager/README.md b/doc/design/file_manager/README.md deleted file mode 100644 index 3df10d801e..0000000000 --- a/doc/design/file_manager/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# FileManager设计文档 -## 目标 -在本文档中，我们设计说明了名为FileManager系统，方便用户上传自己的训练数据以进行分布式训练 - -主要功能包括： - -- 提供常用的命令行管理命令管理文件和目录 -- 支持大文件的断点上传、下载 - -## 名词解释 -- PFS：是`Paddlepaddle cloud File System`的缩写，是对用户文件存储空间的抽象，与之相对的是local filesystem。目前我们用CephFS来搭建。 -- [CephFS](http://docs.ceph.com/docs/master/cephfs/)：一个POSIX兼容的文件系统。 -- Chunk：逻辑划上文件分块的单位。 - -## 模块 -### 架构图 - - -### PFSClient -- 功能： 详细设计[link](./pfs/pfsclient.md) - - 提供用户管理文件的命令 - - 需要可以跨平台执行 - -- 双向验证 - PFSClient需要和Ingress之间做双向验证[tls](#tls)，所以用户需要首先在`cloud.paddlepaddle.org`上注册一下，申请用户空间，并且把系统生成的CA(certificate authority)、Key、CRT(CA signed certificate)下载到本地，然后才能使用PFSClient。 - -### [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) -- 功能： - 提供七层协议的反向代理、基于粘性会话的负载均衡功能。 - -- 透传用户身份的办法 - Ingress需要把PFSClient的身份信息传给PFSServer，配置的方法参考[link](http://www.integralist.co.uk/posts/clientcertauth.html#3) - -### PFSServer -PFSServer提供RESTful API接口，接收处理PFSClient端的文件管理请求，并且把结果返回PFSClient端。 - -RESTful API - -- /api/v1/files - - `GET /api/v1/files`: Get metadata of files or directories. - - `POST /api/v1/files`: Create files or directories. - - `PATCH /api/v1/files`: Update files or directories. - - `DELETE /api/v1/files`: Delete files or directories. - -- /api/v1/file/chunks - - `GET /api/v1/storage/file/chunks`: Get chunks's metadata of a file. - -- /api/v1/storage/files - - `GET /api/v1/storage/files`: Download files or directories. - - `POST /api/v1/storage/files`: Upload files or directories. - -- /api/v1/storage/file/chunks - - `GET /api/v1/storage/file/chunks`: Download chunks's data. - - `POST /api/v1/storage/file/chunks`: Upload chunks's data. - -## 文件传输优化 - -### 分块文件传输 -用户文件可能是比较大的，上传到Cloud或者下载到本地的时间可能比较长，而且在传输的过程中也可能出现网络不稳定的情况。为了应对以上的问题，我们提出了Chunk的概念，一个Chunk由所在的文件偏移、数据、数据长度及校验值组成。文件的上传和下载都是通过对Chunk的操作来实现的。由于Chunk比较小（默认256K），完成一个传输动作完成的时间也比较短，不容易出错。PFSClient需要在传输完毕最后一个Chunk的时候检查destination文件的MD5值是否和source文件一致。 - -一个典型的Chunk如下所示： - -``` -type Chunk struct { - fileOffset int64 - checksum uint32 - len uint32 - data []byte -} -``` - -### 生成sparse文件 -当destination文件不存在或者大小和source文件不一致时，可以用[Fallocate](https://Go.org/pkg/syscall/#Fallocate)生成sparse文件，然后就可以并发写入多个Chunk。 - -### 覆盖不一致的部分 -文件传输的的关键在于需要PFSClient端对比source和destination的文件Chunks的checksum是否保持一致，不一致的由PFSClient下载或者传输Chunk完成。这样已经传输成功的部分就不用重新传输了。 - -## 用户使用流程 -参考[link](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/cluster_train/data_dispatch.md) - -## 框架生成 -用[swagger](https://github.com/swagger-api/swagger-codegen)生成PFSClient和PFSServer的框架部分，以便我们可以把更多的精力放到逻辑本身上。 - -## 参考文档 -- [TLS complete guide](https://github.com/k8sp/tls/blob/master/tls.md) -- [aws.s3](http://docs.aws.amazon.com/cli/latest/reference/s3/) -- [linux man document](https://linux.die.net/man/) diff --git a/doc/design/file_manager/pfs/pfsclient.md b/doc/design/file_manager/pfs/pfsclient.md deleted file mode 100644 index 56bc70c54b..0000000000 --- a/doc/design/file_manager/pfs/pfsclient.md +++ /dev/null @@ -1,129 +0,0 @@ -# PFSClient - -## Description -The `pfs` command is a Command Line Interface to manage your files on PaddlePaddle Cloud - -## Synopsis -``` -paddle [options] pfs [parameters] -``` - -## Options -``` ---profile (string) - Use a specific profile from your credential file. - ---help (string) - Display more information about command - ---version - Output version information and exit - ---debug - Show detailed debugging log - ---only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. -``` - -## Path Arguments -When using a command, we need to specify path arguments. There are two path argument type: `localpath` and `pfspath`. - -A `pfspath` begin with `/pfs`, eg: `/pfs/$DATACENTER/home/$USER/folder`. - -[Here](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/cluster_train/data_dispatch.md#上传训练文件) is how to config datacenters. - -## order of Path Arguments -Commonly, if there are two path arguments, the first is the source, and the second is the destination. - -## Subcommonds -- rm - remove files or directories - -``` -Synopsis: - rm [-r] [-v] ... - -Options: - -r - Remove directories and their contents recursively - -v - Cause rm to be verbose, showing files after they are removed. - -Examples: - paddle pfs rm /pfs/$DATACENTER/home/$USER/file - paddle pfs rm -r /pfs/$DATACENTER/home/$USER/folder -``` -- mv - move (rename) files - -``` -Synopsis: - mv [-f | -n] [-v] - mv [-f | -n] [-v] ... - mv [-f | -n] [-v] - mv [-f | -n] [-v] ... - mv [-f | -n] [-v] - mv [-f | -n] [-v] ... - -Options: - -f - Do not prompt for confirmation before overwriting the destination path. (The -f option overrides previous -n options.) - -n - Do not overwrite an existing file. (The -n option overrides previous -f options.) - -v - Cause mv to be verbose, showing files after they are moved. - -Examples: - paddle pfs mv ./text1.txt /pfs/$DATACENTER/home/$USER/text1.txt -``` -- cp - copy files or directories - -``` -Synopsis: - cp [-r] [-f | -n] [-v] [--preserve--links] - cp [-r] [-f | -n] [-v] [--preserve--links] ... - cp [-r] [-f | -n] [-v] [--preserve--links] - cp [-r] [-f | -n] [-v] [--preserve--links] ... - cp [-r] [-f | -n] [-v] [--preserve--links] - cp [-r] [-f | -n] [-v] [--preserve--links] ... - -Options: - -r - Copy directories recursively - -f - Do not prompt for confirmation before overwriting the destination path. (The -f option overrides previous -n options.) - -n - Do not overwrite an existing file. (The -n option overrides previous -f options.) - -v - Cause cp to be verbose, showing files after they are copied. - --preserve--links - Reserve links when copy links - -Examples: - paddle pfs cp ./file /pfs/$DATACENTER/home/$USER/file - paddle pfs cp /pfs/$DATACENTER/home/$USER/file ./file -``` -- ls- list files - -``` -Synopsis: - ls [-r] ... - -Options: - -R - List directory(ies) recursively - -Examples: - paddle pfs ls /pfs/$DATACENTER/home/$USER/file - paddle pfs ls /pfs/$DATACENTER/home/$USER/folder -``` - -- mkdir - mkdir directory(ies) -Create intermediate directory(ies) as required. - -``` -Synopsis: - mkdir ... - -Examples: - paddle pfs mkdir /pfs/$DATACENTER/home/$USER/folder -``` diff --git a/doc/design/file_manager/src/filemanager.graffle b/doc/design/file_manager/src/filemanager.graffle deleted file mode 100644 index 7861a33072bc1908f69d12b37c20491dd8663103..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3438 zcmV-!4UzI6iwFP!000030PS7fQrk)rzCL*hEjN2{z@8ud=8&9{4HyXjHuykMwKcW| zTS1l_Nd}x$ZuV;T8MgLj-(X*0-{#cTzQy)P*w&xH5J*CfsKSyp-94>Q_t!l?8n6C( zGjOCU?gzH#zFEQ*v?Ou2>shwjd$V-h+E~$-etr9D^-pX2_15XZx-@j`Ae0V{Ydc$Y zX=$ZeHHSloSF3BSHR)hytJ#u3Mzy-Ww3Rb}7*r2@Z^->{ zyaU2kfT3cA))G_`OHZi-S1r36zJ2x6>Lnk)HM^mG#dp}4`&+KXZ&s_~Diqpo$a~y> zi&m?X(+LaIZFgA+B2Qw;Qzqv7Oq~3*8iYRF@>|fb;tgE8=X;}JWgpHOKD)SZxVBoo zB}q_YO|1~no@mrCAa*x@tEiQaJ`i1aRJrHSIWe9dRqGreY-?d>PkfhL|d$K-5w61QW(&pdCifQ8hV zd&Po%-|pEn+n6*cv?ofoUV@`(f8;s(xY5NE$T4}&XyBTT-E(tF;k;52b#utNAYw}I zbLPCI>S{GQnUKO4Vcu&j%;`fA)>e?Rf~bUy*Mz*LhV(OnznpeLi05bdd!F07?qrGA z>|p4yakI;uT#ewh(lC_XNYTknmUY8+cxyb&u`zIxENx_mYhHIW;BJ_8-f7d(T-)8+ zZy@lcz}eFVnen8AD`BL=(^s%AoXOCaC5qCHROCtbyV9f`!WO>?^L(f4%>ru2yUvpy z@gOq6EOo+9XYHh=48PD-bIy2SJ5JIUXT`fGWN zA?s)ay*eK<--z1-ug>LzTff;TkxsMrESOszQu72FiMc2tq_PU8P6| za8~56;Kl$~;=wr2;>)UD;LAIM)17B60eBU(%SxoH-~Ud}}@iKs#@_7vB7*Dkut`>J+{+ z#uNELBDtLUA0`QR%No`uXV!!L-?-Zwi_wP`pwEE^my#_v&9@R(tmtw@Q!lBA?L_V)v2YinNYS~9UtN;7Q74hgy-%lwpl@Gzz;gjNIMYp`( zP*`VLwa_2&YKAHU$cYRZ;Y<{`4Jf?Fy5khf1#^P5;1;u;xvHbZQ4z#E)Ob}?6ui=& zIH!wMV!=~k0eLhQP(^qyRcK|t_fr)pbfxtE()+XKO0?3U?{R4LF%FF$AKb~Eq59#$ zUzcNuz_da!!3gO(WR_~_>@U>WOUJw&^+%6zOhtRFW9o()dmS}oRgo#ybwgL_Jg>uu zXceUC(3PW_3?3?iv|IrJgb)RsPy;!0nK1^BC5Dj=>ZO;ycrU$Uq}%}L$GRE~l;dSd zdBTD6VrNhvKA8PH7WJ^$le;nuq}iYtGhT z?cFsfH85zGoQ(V4VVxac)@n+`unP?9ti2c4z}m*>^tWxr+-|O$?aQ@Gui18b7O1j! z)oB>x{xD+N1g5pP26A)N?bVO_=Ei%_y?!=0HO*7j>~FL7`nu9OZfuNC+t&E>x0)9* z9MdDE(`Yw9GTfw)jIq_&use;DL#VlbMz{OjMj!6M3F4bRtOH?f{}|W$-NA+n|JQ6| z5ApVmBh;*|@2wqf?AFb#V>4n9Iy%GC2I$!60D9f(JpH~oY@Tgz9d5Kg?vjY%6r+X- zf1Bdtp!##utT|@m+O&Fb+&nbdVeP%yST|enJz_97k4>SL**-lST-Q71SzyAk(f3Z@ zOpImvl)v-20a1YB%P6{tzNhg%B6R@Pg86d}Wq$D``NgA0^%hIhXY>Yi1mK^F&HzPP zMbk7YzLhye+0B1IC-yOU3eq5(C2BM^s1dPF1K50%rUX~8p(+|ClPfm3w_sXCQ470F686OIs+&=rM2B&UzCSX$Y(WL+2SOGAU=h%USPlh9O7m&DO=pk<5dwZ?gJEM|4?|F`4xC zwdd~b=VqqLnD`^b#Dk4yllxbX=-1t4qY&+gre}I2U(I6G(qc=CEk`0AH3dn(V0{s- z=$J&y3dsCr1<_2Upa3Jlib~1}$Z~39ULcVTB$6Ha^+E?OL&pysI<{QUF9^!O@z~Wp zceoDG9~(Fb)gqv$YDGl~l@(RT096{E8$#x;UMr))A7D`W3CEE@D~#5#F{U z>Qjw7D-RrZR-qv=G_|1uKq#7~m*dXmxbrtd7N7U*AmqNpTubWlP^urb&i~9X{F8Tl zJUOkKWpaBD4#-90S13dUGm6Jb zuvlIpljqK5@qKbwU zG1)I`s-YN>HLKbSUpqmbZ0!VjU=E9kMxdXdVogDYE@Pw2Vapu$8}^PNfpvNKj0-1T za57sA9glv+mm+{ogUri1Dbg#7s>;yGh4fS*az&wd{$!jWk09UplXj)NuE-{c(ePcfK?Mr~&psUz1)n%Uz=4 z6PsTbmCo}WxKVEu&i5YEwFik!QRijyqO@!KU2oTR*Ajb%PCQM|xLQQp8UaTV1+u?36?B^;VDE=dcjtC`U6f9L?#Qy*V!)tG7 zFGQbqAW-pgo=4;zXqWY%=k*YJG277Wa~@`ld1~pqo@HO$?(Tfo_P1D;5PeP2YoZlz ze;W(p3+{ir>s@iNC+#M4Er&0BLwjDx7r*^|UnC}}2U5K2@`M`9YhmF6n`-b(FM|bW zh11S=vb|=_3qx?LA~}wb#w& z2cX%wajxr)yVc*}7vYyQk3gWiuyX zCT5u`5o)O#BD#)rby?Cf!vh7C3hO)KVpn)x;1Vz4@gIpk$a6mbmN$G3@oBJ+9>l)O z1$4|wJdQZC_FZRuKc*ac*Ng6i4x|fhgB}HnmL2Bd@4tWl+yDOguT(w~a}1z=7^xw^ zJkIapUl>>*y^EMdw!qvje<1I%YkwH=Sx@ecyFU*2x*OX5UCn+{oN4BT-G+w~&$iz2 zak@Eix;5Lj3w^lxs9zuXJ~Ud?==r1@<$o1m!%yK;@&^l45-%Q-;atZ`!LJoG`)OU0 zndp?@Xyouj$7hy3#)F|2@38xd+qRWVw`NP`l!Bi*h*|#9Vx5H=;I{FPcrOKCvvC8N zh+PgU@!Pu5+9~lf0{O;B{;a{A&Y&PUVa_NqRG37Bg2CDXANi?3deNmM9_6rOc8q@A QNOyOGDBZQ_?nQU=U(0=d=j`J? zTlV#Sem}^y7S8$1ImfuiJ>nUYAbDAFWJG*K2nYydNeK}}2ncvG2#6=g@UWoYWGe2m zLqH&wnF|ZcO9~5<%G=wTm|Gb`Ku83|#KI{nY2bGqcrZ2(V+ciHxE~N@0py{#^~for zz{Q7$(#Va@Y&!_w_wr%uId+N){K1`+VS@1}5bt4!)sq0va)iUK+yp7IJ`CZ`6-`c$;)8VqN%6>S+tY6#{SPs?NXRoWq^mYzN6NdM6yJm z0B-zfk#spGZkA#8LRsBCDiiPB$TqhPJJZb}`;2ZXtJGKK`y(g)#vYdgl`miQJEcy4 z`{?1eT{P_WtOVuR&52U%W$@jqx1Sf{=L-$X}m^4EGlKJ$d(<_8i8K6idV}#`g)v zYZ|oo)p}2+ppFG0rx4EHv%JGqesA#(BNYw>jaEu1Z0N0U-4wL7yw0JzgeWNE$B(f9#Wa*W3>z{Z8h^a^)h7NeIhuV}0^+gsJK&UMqweL=O}X z)?cG;D8T(Gimrv4}GqRWx*H0p%rUTZ%o!3OvfbjmCpxxzF-E})ZqGF=@za=W~{ z#Judg)V+nj$Mlnyr;NvJf**nAe;z0=lt-yec8Z~kmh(wbgg%3h`Aaut3Dq<7aCEIt zE}zK1Q%AB%2ChjUQc_0Le0d{zmSvgMm33jnqLE@HuKQ*qqOvD@O=*o_?fJ3kXC%?I z6^EXfG7JBbtIelRaBJ{XNre zabR-Z=L%JPWBB;uBh_bIQLi7_pDnRbsdbe+n?JR5>!X`0+-Hmh6VtT6xqj3BM&nJ) zo0K=0y)?ZGy-@3Sy=pOU3z?L)l_QnU^RyN5l(O@=^N*D;@^n?bdZLXRSd`+k`<*xH z5+f7C6D9WYG~;HX8^YY<-NW4duQ8D$1B%6(I*CI$LhWQ!1|SBI6P*%Kt3PShXa;HC zRB>1FTL8_(&1}bPMu{gP0Q%cTJLB8Dli54mz><;T;pW`HvC0XlNuE4FF)C1Nn`$Qx zxXA2_T!icj3y18ER36|Q;MsxL0ZINjqEI0~A!XzXbH~&oS65xnWoS^2WK|DC6iU=V zj%p5dPM0)ELQniv;(B~_d^0P6b2C0JX|eLFrmKckjaiMWrMVuSsayAP3-$m_n?;gJ z(gdd^fn~Z=^I3doj!eh^WD=xSgY_5d^gU#rqLie0lg6jcX?w+cV|#M558HZcUv}kz zObzO$c5^F+!GS~*MAKp7L~tqbvdpsL$*?IC$zoT_jEz+`c$X@AvH9C1iI_PJc41+ zU=U?csoQ=N%fc*0T2_Z7Nz{|5a4NZ6ku0_>3`Il5p%dfFp4;;YnnFPFJ~}S^F|@zs zE?`%OmEjaqHT$(W?P%iIwpryE^(gioDSq#}-j_p{RUDps2~MNHOCUD@Si0D9FPfva zF{BYsC#yOOTaB#6fn@DC@9KzIMeVFKYZ5w8pCn$4N= zeQ*7AeTRVSgM1uS1XVx5)JECfVR$@=n}D-*?Np1;&4s7UeKFM7`^fjDACtdn2_)Pp zUyq%JT@vYtPKpLPuQ(sOoFCXPsOZq?Bx~?C@#wj%5^tp5@{ydE%y?9OuUI?eALSou zX;*w0-MCAM&{Pjv~!+$=gxEKJB{K>xWSRZbxxsBAVF6WMiOjNW}?U7 zO{%S>o!P64!lTkI#yp*utuH;ww=lL9_p`VA65^;dVTwyjDYWryv?k8CEY>DG_M~PT zORjVp?dE~k6DWYNK2Abg?K*RxlUw>(Et%S4TTW+9r_`H)omvN-Nu5v^t9>??S*O}l zy8Zfb;GD;LN!L8^LjA(#f_@vgjS6fPDD%`iAl3wDo|qIVdo?gr-0EyY(Shb#4XgRRr}n2 zEoNzA9N5$<`LNeUFwbfF;CNN(z-#ZhR(!a3+W74P@gUr%>~@#~o$bhx#mRoh@Sf_U z+(eqSfQx6)<*|Xqp4u&7)Yg4bv*pIihc|_rk6!QTbY(Fp>LMgTZR4qu`zi)02!!>e zUNT8Q*%h|A1XZ^v;l;ZYc0QZ^wS2rK>PVLmVE7ruE8 z2G4z{c|kcgb}+JlRtncv>$km~pR>Jft72A;3Tc*-3>+|}_;~&xUeVvQcgv&JW9;uU zvIyx!1S7P5<$wYTVG-U*XgEMXJfj5vgOpUHID~)@gpd?@t?UZ9JCEw9yy^9Dh59NI zeJYat6BD`}G1c0exm>8gqRy>x6|E&e;Qn-q>M8iLMU&5mDQkVpdc0Wq6S)#$HO!$q2A?wdohNM z)qRoBZ9i?3-$!@I0cP2I(`~~oRK^OP)E5Hk(I39ZP$Xr~Q7Z%?AfZ9Q@MAx2p`gi_ z(Eh6 zp=V$A!2hG+@FIFh|4hEtAc;+hNs2wH39zyRNudlM)xppGT49o+wR5~-5&B2Nq0n)k z{G$!%B|-SX*C<4yKaL#Ge$hU);XaNT@P22Z;YG?qBSZcXazKjc{0Gnl4Vii&Nc!G7 z+UUbS8V-Uk>_6JT{|{ZT`hEYuKo>)3X*rN2_8<`XF*}OX(n|$QMK&Ug6GK8PzFwNt zvYog&Xf3^1@Q{0o!yx_f#R%*p)UlBT!AjQOW0N}qPm$Wzcnljx*xXo94%AR{R^;tKyg>ICgUO!YJ9hD*rbZmradSyhvdMY#!1B!j9 z2V7L){#MbX)ga54D8A2Pho>k8&6k{YQYQTE3<$_z(70zPC}@IuHK57y^lrVPzh}8N zU66Qp&nP%S>I82x>cc9>BhbMBiwzoxR^Z>JjSUid$vN+y*l$;nUHN|Hv9 zR4CG!aN_qf1WCb*FrlHK3+g$;H0d7fs``#T0y9CqHz0>09Tc8~F4)H4*A@crye`iF z2nI-*ctDf-!+5gEdE=1jQ>sU#CczETO=L-59urhP2--px=d{JY_l)r#V2L&J=l+1FsPY8>Wwx#rUOjRY zv`>tnXM2*AkomSDAwZjm#HX*kg#E3=KpM!A1cAW(J*5^1od43ziw!Z0N7ljsg(DwC zfap8^ZG>QahB@;N{2px*ufX^$kupbv>H*8Z{+yut_cH`ZwLy!#Lk|PtQx$A!*v^QL zS(-Cw5i2T6VbEbhz?P=4O8nSKB- z(wTcy{q;D}@5Su*3Z$BE$s|w^q+f8MK=8-N7qp;BM(cH`kkD4x@SyO3 z{C#neQNZtMy)EP^$Ou-~us%URjew-~Dd*c`z=;HlU=7G>5hC~o5#$ph=ig-hJ`maX z3KFhE*_b&CR1nC3gvibjoqi8!LA@0)am&?WRCO94sWiZjh0VW+GXFu5#tRiXK1p~3fm`hpdc9r?%{ zQZyxy$l}?w1tI(ruzsrMn{sJWw>ng!Ox1qDk0*fD+mGkjDo_O^ybV zkukoA@FR2LGr?p#5|%p+Pm%bCd%`F-%|m@eo0#Be$Urws48qSG1N1Rbujkn>`~T%D zNbaC<0qxW%kkA<*Gew*KSosLjD8Ly{&!pivjvp4-*@*JQn*2U`><1I=hIZVT9yA6R zrQdgPEI$6;Y}!OX(Oi5U?ly>NHNiIbOqJykrq$~NPqDK{1lgu5lx`{Kf^o_3_eJU} z_#SlTFH1xsXK3-XU>4kX<6%zqh?Mie!A!w9-JTFAAb|1%8>kVHxYHw0g9qv6InoEP zA0PsW!=x)k@sSy2S|D&XiuQoW_5~#foLN(K2*0TBm#=;wz>6$5d_a88#0i#0(w%JK z?^jU-GwnD~N%MoCBABV7=f$#rKLZ33&?4_YV8W9!5rAPDz-|1~RR2D`*J}kq<~%8q z9133gH8|v6wy}9cn-G4}U}V;K_#T7eI!2J)K%zY2H-E9TpYe9|wj%!f-$IQ^94x3z zBDqR*2{w>oFp9+=wX-O&o!Q&Wfk4aw0+1f-%jrju#v}!n6bA1PGGAX6kXj4j#2+)W zLa>?5IE)kcVS%;!gFAZT?@l4;mhVWyChGHcPi3sU?^G+|=cw4Tw4DT|aK@B8pO3s@#>O@p z5@RLF!B~?ptB{An}OvX=(+a; z(Rzk+lw14^Mu+C*o09tk1M592pQfgg^(t69_rfqY-RR{ER?8{rR-VmuvF>G44-0 zVj=?-(#Zd^o4=pb77a6+ll%k^{0t8Gro(SfIo5=p{d&kht?=_vOtWAsa7LBq|C2lv zU;><)PYPycc*X%YJlnBaj#}qLy#vikeJ2hv=x3=2sf6cwCp%ge2bV5)`!1YLV=oOy zh-j7H0ch4ux)}6_S4ZT(=AFXy_J6wbp%pS8i%G!ak$dax?=|lZ#IbHTmo2q(Od46B z?4?!DzD@2IG*`MNBqkhh8zOrDu~Uc-z#lyM>VR%-WjjG8gdJyVuF>sd=Z@mNaKB z&NrDK&gbx`7LCpqo?r(vj@-242Hi7{n1moKX1dH;*P8cKcou7>;8;abG4x(Cx;M-^ z)$h?a8)6x4ZP$ppm`x-$S`(c3CAyZl>=f5H-!9vwX>G}0y!oD%&jh%&LM8B4V_V^5c~K!y@#cbCtn2$%abllX z!v|UT@ie)=n1X~5BqPC}bOJv`qj$RqReE4rp(U=AQajEZDNf1=G%c^4SwBblh{N=5 zM2lOqX6;mW7gdhQop*fII*>Cq{UhD+#{`Zc)(VfcjMnnb zr$pyGz+UmR3SlGvAfm03TI)q`k4`(=V4~}~7=5Qlc_&^|>uJ{PtX=w~7U3ykHMQOf zUFn$^CA^8iTJxwp(5bVhBQ37={z-e|H0Cks*8>D-grY@LV0Dz?^Bm6~4ZoURVHo9(!m z$ZAbEK%v#b+wqB|=GA$Gaj&LrNf5#PVg{Auhmfle-z)#@A_RrtkqWO8QZ!r2rO)N>cD9ZHHH+0=IwJ3URRa5%534%7{ol-F{BUpN^0Mt6w1$Vz3Fq_1y>`w`E02sK;0!&jnd z01R0(a+$UPusqVRcCnfBa_k{vzzf^y^;|4hX+-P>lP$`Z{XzKRiQG zE_k!uI(qGu9_QGSgB7Qmw;wHKN}yek&$PkAWW7&5TYXT^F_ysW95{P+nc;-!K`{3biN#Rhlsyf(c(2^ShbdyUjyJCd+sY2w_hWmc5w@ z#_Ufk?TIT|${80_)(?hY0g@^NV>nagSDgcIOtUEqM?d4I91Hlc(^Mnj*Cx|m2IvGIWKNQ8l~9S;#hw>xB&Mlr2(P@P1vN4Wr>zHEGGu@t7urv(NmInks=z)dS_IR7OQ&^w1QF(re2^QYjA)Cne}b=|{D0CR>;so`ed z#Z^vsH>lg-Yi3W??R!R@`U?v^SQjZ`eWgxa6$^^P3ZmV$wMMcJ4O>2T* z!=3Sz_TVnx2f=YSRp(G|dzq(H?=KmbDP3MoxWH8$a7$Yct<`=Mo=_-s*OL65p` z@O(BQZkl@TYKIzVk2G={;LhMqOD7eKBoZ7{!8)^*K;8$ z7U*k%-B-aENcesD(Qt8&hx4`}NhODrJONEOa+;w;Dl$3coMH2jE>j2aZiU)vWHy9b z4QL~&p96-Yi0^M>(GEW$2^){AmR-eSc7%s@MXo7tktZ`xZT=l{>6d znqX{Oz%mhI9$s%ahJ-}rMOfj&{$7Yj+K=ve&V8^3@n#vc1Pw zxiHvG;4xUwOxS|Uj46J2*Uv&LKJf9tayId&z{8x`KMXjc_h12~ZxT#pDkHE8(>L=z z+jjssB$bvk<+M@-T#UWGhY*(c9;VpdbeYn(O7+U=Otgz5lG*fh(~pkmItWMxqNJ(L zQOzqncX`0`@^(nKuN2=|xXRq5?~59*moszlHp7b+Y`;oE-N{(Lc(a6<_6uW!!okm= zNF~dlbo9l))#68idAQ~2wFOfe5_kM|4h`=Enwj$8-Kkudi#edgqPUU#*4Tv9 zcx&urQjQKdqsDd{gP768gF2^B1(Zy{dCzrtD?m`#He2h)ty|edSIl3fRIWuAhf3yUyA;;6=R{gYI?A!|ZLuB4)V#0hg>(bxpkD-e_#K z;aaj+={@JFmwD!PN;$QE&;h<5;YFx|(cm2_K|5}z63Zs(FHH(8y8LG~wMz(S0NUtUf4nj>*00UdKbM}P` zZeyoHO-H%c+3j(LZb8;_+a9AWyi|Vg*yR;#60IK9} z?`Ev^9?duPvFWmOnP3^+T&MdL)!qo zT?BiBG3M5aCSq^PPCAI(=?{HzV_rT8l`{uQCexqTSTnD1Y3KCFuVigt{6*jt<)Cyx za?VZpKNVq8QE1qO3FEPw9k@yQ+GhO&XKj@AZBN0%l@@wJ$wYpE+hV(y!TI~hZVL#; z%eQs0;Tmbu#cpk%MDjSB&yd!cmibJV54wExclwwOV~Ck1aq>vKgl8`|N}bBSUu)Jj z7HbGJA7VDUpHmX$lRV6-{EU;2NEi3mLZ4rtzv4`6K_EHk9I@P180~i!Fx&D<5X&G| zucdoKU!2Pl%X3@#QQC~cy=dsNoKIHjt6%kp?#!ZYrI8_|Th$!h(b$_faXC}|B;rX{ z;f>&Qug%ySa(_11g*fUKAsAYNArzKKrJVhFD_-SLzp_C2``tfBUO&S#K@0GkM$J1( z%l{POA{E+PQO&&9eGIz?MbX3rZH9EvBkiiMOgWk2@EUMu`mPUC()~MObxw9w&(%3U z?&BH`MMnTf>vmYh0x%mr$vbCNr5OBon41}^n~=_{whspYe(Z{5*d-ZK@4k zcKMiS>u71-O65GN{*bO#9J94>ayCpxtu>)(M=xF%Q7pxszrq=v`7PX2o49RBK;2AK zjk*g@hz6pVfBGQ%+p&d=-R`E$GZpp5&XY#V<-MFEd<{g?|D5ArpFlbWy{w7|%fkB) zxk=LLH5s1DT$4;~OI?v@UV6zQi5q0Q+0DuR$aH^y^&C`TI*$PO}3^<(Zw+CTjmB`^L=TKX=;KwAC z+gS+U>g((E*P~S@dg`w^=GksL7m%7gmE`4{2w?9!xP{vz?W#iXYa-C}&yW{VV?KeD5B=S%!mqV=0rF)zf`$*#<_nMm)(RH474aj;nWh4FBetwa*zChPGHncIo_$*0WQ6THpJI<7VkD z{8?};Q_MMh1JYWu=Pg02%s5=J%Ur6sZ=}#rbIkBGxOY9#5-7B`?9>jYu;I(l?bo)j z9-5IPCO1O$S`*W@h`&5 z@@<1t{EEy5Dh>P}!u-~rb*twHEpi@+*BbiV7@onPWO$u7=iC|JRN1B-Iq0R`p8=kX zC~6GSYu0>s+#Nj;*kJ?l_8A*+*l(mZr_;FWx>4-yjH#tc`H>iV*#PH;NqF_g=59`^ z%E??^D*Udzp)<8JYo51y4XKlbfC2W;bFKF?;r@*3UTU@vOA=Y%t2wS-NrhcaN5f)h*(73NP*!kHVULHt(8!z}5cc zWc%#5*~)_tGr~BT`_;*pInr_sv5cYE=vdh(=2DJv+b(gIi{TSQ)zJ}AUN~>ZBe($d ze{HhK2fff0+Bv19|4%X_q3I_*Uz^RQeADUWey+U>aI;2UjN^%)$ros$<#ENbq%u4+ zQHf_Dp1%K}+3zr#1jQ_??I#EXO32+C1(@tTHIRD8O!0yK8biKt<1GW5yx`vq?b0?kHx(4S_P$c<>oGRGAh7}1lh9S zzog>%bRhZDTrgM*|2vtK!1w$1P$QM&2CH0e9vqg6`e1WjHlxd3G{F=0aRwCIS$2g9 ze;GYKouWQ@+TmW@H&g7er_U%HSPJ||RcqdtKJ1dUo;Q*!4SbO`go<5iIID6cbQ#?B znw?~^_zW&2?ywyPkmGUN*mAwglEV8!d}P(4p7S0%Y zu)Yi{Hb;f!_+5IJTp>@9gxkzn|6mMDtF7~-_iLDug`tMt)BgHH@5qwc=B$)p4$UXb z6&p!HxS(1HC~1G0AiK8X04HOZ=N(8q<)fW~K1iG*pqtJzb`7IqoEFj?(ouBoAALiW z7G+Ek`|9-4fgb%e4M0l{m_ETHO^i7^FnyUP?M~ys`*n37!mw-1O&2?j`l<1L-Xzsi zp+2rEqKAx%XCFHoPVj)05f{UTvGzL`MPvJ!n`H0BJolz+JcG?Sp!2T_(^x`BStcWMo*7l(N1c;^S)Z`&vO1JF<-E&oKsKXdo2_%`0s@@(}aGn7A3P$dnM?E+IK7MiG+MaqLd#CtggASkjdSgsMZIgpzUOF&?1Fof=cPb<0>11fhTid2j z2Lj|ux4se6PJBugA~({^u=iwd`l_#5ut!UMa-ssmMEiN)?d8impE6bH(R3R>JnmdhIo=s-QaWb)8?)H0Vo#)oBkj}v-y)_MXQ8BaBPZcPQ z64Q-+5xzha5jS#PEH*2Af+|wedX~4`*mdeWryMoAMcm@a7Ogi<$1Am`X8K{}wuU#>{e8Pw%Y>PP zrfwhKARZe*>`1#l-~b9P=4@bAjQg*&(7d=SM^QVY?C#5At$5K?Qw&s;ZB-Wb{4FfG z^2tlgyzn^&J6&NjV^?3Q{0O@VvCpbe8{Cp;K7%Z-A1-Vbh2*22W2^`dIcF2dq(rYo z#VTyFs8q#WzuldaBzkHqcSViN+dP29v@&v3tU*KIm1n_#fA$h-q9X*mT1N~^v78aj zynLMNnwKlYur_VPrJZQq<^3zb)=kvi9X=oDAFpu#Q=I{o`l}Ewq0p!P&zFF`N*h1q zD&s+m-lC)t*P3+!_1PjpH~E68wNhMlh+pu+=txB<;C>&%OH5+(`F{S`&8I(Iizg0~w>#tk-t(!c+Y{7eJyl zk?yL>)|B-y!@)=R_ui$<2_0PfF#uO|f{2p3YoYG;lJgt;Dr&|`r4N;@$%P9AE8Whb zBy#JzE-a`1BPFyYpFg#HUU=a!+tHa|Oep39TiQ_2sUv$OSt~bJk#N(c*1BRg6k{h2 zm`=2dTYV|89g4fL;BA}2I<88N)~*aGwpfX_1TSiszA*ix(?y9iz0N~+9IRdWxl@({K;g}$#$7%9b* zV%z0oCao=0&HgbGbvdatrQ)QhcEh+m!amXp*}QC->vUwK(jSA9yk6Zf_UO z)Gx&iJ!pcpl`qxIt)*tHkx2G#`eHlDI(^xqwWL7^b9V74{o@8j#7&r!Ccb}LTwT@v zO=k$Y%|tX0`N3J5U0zdXc1e0Bix-|3n{`eN(VmL1bsqn60=NR#K40h{Hfz!Gxx0an zFM95es4zg`hVCug&{^TOQMCc;n1M=)m160Id9h@GH`9#PVYkvwv%9JaGYQgSJXhhp zUtvx*s%rOK3S>>h&=&HL&)U~(o%xU40~Rt*JGasv3tFYxrYS8# zH@8O>ysk@zb?(x2&i@S3pYDS62ef+PFE8TyTYPnt0#QA#$0tt0t`|v0hljt6;w+Se z(+w#(cW&WBaVsF%fPt*BR;CTTN9v#Qp2MU131AIfr zwViK8*_tL*%&~7*F8tYc>q|;XTqX`|l^ZHTnC-e;yF@4ycUk?tI;Vi_odL{To?6#J z^g6MoAy-@EMie6y_dML4UPJk|Qk=aSo?R^KMqNAElF|D0KX7q~!vg|Cqs*U%Y(AF{ z6BiF%Cq55hxR7_Mk%E(2fl=0AIU|`myJ2TB9sDyf{B+g6DNq~_zjp$n~=~O zHG-NZHOC=o+jA_H4(m4wx3THTO`hx0{7=SG%zJy-b-&4GbLY%;HYdgA&+)FB&T+wl;ep=xWb%$#kNm5sUAosj&1Q^Gq^c*LODQdYdoalA5W-(+s@yfEBvxa85}H;Q zWL3w7JBe?P#_Am2=`y_O#bj> z;65bkseHCyU)B31py9eUYQ%Jfg+6n7?bB2}r9Mj8TY=S)!mVwh8`$r^-6w z-TRlmTE8tQl26`=pw5ghc~Bi*d)zMj$%6y}1J(e{>csK?YC+ua72GHw!-kvPo%JVx@zXN!$D=kd{O1keNgI`xENCcR$6_+b+W$2(EHpbtZC1isZ6XzPn=2!QA~z+kjvJI4`&0d*h+_ zmK5~*Gn96Nps-YSk;AiDRBoS6Ve)u_=I(xdoA#x#J+W$^gPg@4sEVe0fv58V1uyw*2qHqEGN8i1Q{I*NWnWEiKa0Lb!V^@!ndR#f~8 z3lGR-u^9wPjQF0R3DJW?X>s{C@1s8OPm9 z+(XIYLS4F_pRRd(pwp0-Pm!EwhD1GG7n82y=&v=4FXNz!B!$z3f*ZJPnfK-!8%_+( z^w7|onAqWebw()4k&;u%!Mw@`lo(l}-+i&-ScbWiq@66Za59u$j z;4nk?T^nXkcjUCjuS%lhT!P(0r`6*&9$F^nnl)b~FnjNb6yKC_YbBmqXq7&rk97BO z2}a~Ux5M5bcxxPzsGBIE)dAe&NqsjjbEf`fU%{+8#Bzt34lb0}pRa;c36D9Eq?jT3 zFF60NV+oSt!MIY!{%qCz^nA&VL}{|Yi(D|ckyb})bZ;A(HV;A4@s}a9ffsR zA57QN2ky$MTxOQ~M*FB|%87$K7ULe~r4=%{Y*YEG*f+|~-;Mb#r(IOX^K*_*dY@_$ zOe$U&fC&7|eX?FNX4wgkp3}P5x>{K(s<<(GA(n1_rlP+=R`~3Pu2=CenMu9>lSg!+ z_?qtaAtn#A=Ur28td;T| zZf!kEOpnu6e3?{Bz=j5sqq);WL! zZU%YUEKRW4BaX9@vRI-`%A46v_>$W~Wd8HT-%tLuNm+rUBg#rm5NV zG|vYOYk_xGr(xvwTxxlnbGmrJbn2Xec3U=VFweP^749h4=e>nfN|VPLFTDEmvf1>S zXvwR>a8j zYi{xjJ!B>ROjR^9%}}B8>6Xm-3h260BVIfM6gIvIb` z`El8q?`o-cxB~7+C(BbYdBMvJ$PVkSn3jhAw5ZC-V%A2$5cj<9QtleRZt>i)J z@Y;-p`25|En+MjSP3IiTKmq4W<^6O_P2oDVPl0?OAnBnX^e1r+Eo$Bu@C9aW%Pd3Pd)W|Uwr|U?V4q)FB*m56AW$Rqz>!r z+3IbH>8Uj^W6f|LxFwzq=%Kw=b~?XN0QtvSqi0rQF0wIj2PK{xqN3E!6!7f|*2@hf`ZfaOKC{i`OuFh}w=RBLrT4Pk^Q}^J>wi zeAt_2_SPvS4PE&!RNsr{G_(-^t$?G1iYAFN6ojxk7Q66B3H>qTDjT#3I$!% z|1IUeEV&U0Q5ZLry^vGvAobO!Xv4&!2a!C~qsZ~7wAEI#`E^$fEl~aTbXjdPSNw{- z${>iP1GAGS_2=V{c6!(!d%Q3B_K%-zP7u6wnqb-j*J`5TIlaI^P?Jx8+xu><7v3g` zJN!qV-@*MU3IAW_Kuw9ZUf4}N!H*B>FB_xzQhZaxo!u$OBve1Bdl`MYFkJVhIbW-K z|Nho4yIMej1u*Z8Z2w4WbsGan+PjeyB+wgqg4eh59VzTQDvHXu+!0G2J=z!0x!Hd% z-~M8h;I9tqAYZoeZgb2LdKXQ-&Q+9&0MLSIflB;i3eRfdzS>enQ8A<1(xvbRCiy&4 zvg}5~5Cp2||9H@!Ps~IO+SQ96oigyGXlUJwr_|VRD2^hFUg@l^Sk}2`R~_6d&V}XW z3O4JO<|b9jtPVB}K>ijO8k3yj=%0OuOsr5TDJk4u*QfnhV$^?r^q-om1mA;OFTWBA zSEGS?5CI8l<2c2N(*&m?G32i6Urx}8Z1+)!lUy2-!h01|NqAMf5660ui3h$7vW?$~ zmEke4zYU{#Cf)wCb|Ps1)X~Xl8&2TTujT%%6qp`Jtmy1q6U+FvWN)^X%B2I;hy1^{ z!6fQeMZGYz2Z}PkL2YlJ2p3z<<;q#zB@nlGq{G6hnz>BqVuuWx$SdmRj`lj?BQU=! zw7O$S$z7UgRly(uFU4l`phDdo0scS!roIKNe%2xU9c;Ew^R5 zj@pMd_E})QxH;*mNp3{+XK?7xWbF5F8U8Q~ENv|T=z7Qu3=G^4y}jE}L%;Ig5+)h- zx|q)m2nhIOnfxbC_%EK=_zViX9kO%HO?Tj>^;#a55vo8PN_a4+g`Mgy`io|_zOzo! zHUNnnlKWj*Y%Jq&LZfE|CApGM92;n#r=&{EQzb5p zs%imK)9K&lN9%FX0y=bhQ9skXVc%Gknn#M%YJNd zFXtP%vB79`^|H&3WBLzEKFulEjT&LqB$Mx_;yOy^(m!tN=iUasnz`vfF;?MA&qbXG zkA9W7dtP>G{))iSF;DyfSerH)>m{CHY?ADdx3gAB~>8TMmbYQu4Tx895G z)~qYFZqt0GI`47N-2U~+ViBCnyi*({GTi^h7U+oNP$wZiM%DMUv@D=8ZOSmPH{;G% zGENt`4ybc0eRMh`{dal$x(T@H@At}oWb^)~t`?XqhDXrjxab*YT-75R@j>mtY z1uY&KBw!Qa6qf(=xa6oTwcZ0;PfyYw<}e%&uYsk&&QO9_`?1kcWqJ9fx0>yoeSVBv zZJZ~s-QB0g1!5aJ`Ikher^K?rn0!Irb^z$fv-?YrJ}cvwtwotsH}p4uFd*o@n!klD z@Nn}>b(7#LJf+@{Buw?Md-ZqM(r1+PlZ`XCj{YsGkx6aUGIs@c$8oL7dgd^T$A~R= zpo^YEqwf~`3RMbKHQu947Hj3gJnd|akD%5*lRZ*E>f*;vcGPB?FA@=(DQ1bQWDtHF=9E^~H)lv`i(@+5+B7*XUn47(UxE>}WgL|^|6UIjD2nW!0`kk?O%Mylzx z@1e6|dTyO>#~*tMmfjUJ!nfBJ509%>$m!Zx@TWjk@^qq*@DnB{bpq${m0AHaBswwLq6X(i>2y75i*!^#X^6s#oMm>OM#F z_1H6~7TWdo`HTf!bMyNH%YpgAn5b6^0>_||g0~F4Z4;#zms~BLF@C}pDu0b^{WQ57 zG%z7GzyFgvQr{}xVJ+GG%NP6X)_GXC7h+VNj40c7SSh`nW44RtxJXtfQZsh zJ1D5VtV|h|r1P2gg8U6R#2wbdTqw0qU*OFr6v>q}i}6Qo#~r$V=OsSRyX!T++uhvT zX?6>aU2s-v-8wY8;_IwkI&gT&31v)?Yj(y88>Y#~QCNMK^KAZ{$LW42=XhVIRDg!& zlR7`Ssu%BkvHO{iu5#{Xz{XHosg{ja0s95Exs0xJk(L#gz{HVh!TaTD=OYzEIv3ck zW0_3k)`!-jk)7M*Zb3YP&iMRO>OKCO-Abi~G|+X{|7NUze>Kr)>-(c94C*31^{@XjaovXWp(*AIvjuL(`yZ`C|BBVxv~ezdk5F&^^#?mBKKQ!HTpy6!R;hB+W={6;_O%_Joy^qh)%S+g zW#k|yI}*{19mCvA>>wnUd)I(kQTrwjEbj-`hk#%w+7pP{hzP?<6AqQ9=Gh^r2odsV7FrO#2o{18%WFI9Y zJH2I>hGkn-qz(Q=hosrhr? zI=<5u36WtVlpUKGcKF`At#>C9W?fmGQM582xgWmX{ZvR0+3-QJLqIGJ z(?ZO3ZgVNW2fGPOQo-jt^ALgB&>@3TpvBtvyOyMz(&P$hrMFHsRMDHmDy8TJXuVwu z%TD40Cf&2jn2Dc#vZQk_4AeFIMv-E-uR|%JrS}{r**@;*O#{y+V*++o^SCVQx$T+O z`0%8acASy`@0hY{esrlL>1W&-6bRkUFT9Xy-Q{+p6uW+IEBrCPlxSjMcm;9t`s(_J z!t&>e5B`AJ5j~w9a?haflFCUSc5VhBchQT*U^`HfaX@eL68OsUh_2-Gz8){SZrqQ` z-Zi|f_eY`Sf)@OLOGIekOPAlYKXa!SgJdCyfpFOptvfNCUK+C|URs^2u~=k7xZ5pj($3+vsqA)Wwa&^&Qzj&Q(YH8Jz7%7S;&nw`9F!=N$gz7D76Rcb zY10WfxDl=`Y0M>c)SK>`%)J0W57jX~XqA<;N8t|-2tElZ6n>IU+s>Ed`Ti<&KV+J5 zne+Rrb8V0LDo??WE%#=0QEb)=4GvrPDY1aKHH!Z{`ZLK7(h%xBB9m;pCt+J*bf&2>a= zcHIVk)ksXP&({g9XHO9sMwhrVIj=o?0@m+Gg4<8SK9^E^bKqT%19skTQ`JP}s{n-^ zR@7{+F^Tv9CG9U0jGlQ*-ZeMxuzSPG`ta{aoxQBpkeyyN?6F_d3-J!J7w=bUalVdy zyS2BiAZMFxZB1I4l*}lhRL4ih^t~NDN{qm{ z-O0qKyQV;X9DF%+wJwiuokyH=C73twQL2Z=2ME#B2EfpF_bt_Jr=sEn%KhqPx5&E5kzK}=r< zEn5yQ!>z)tZ!+6#`iDYPfNb@W=%c6Q)^AzXfs?5+wAFgLLFXj=B`5TII|OVdGKaS2 zz5Ro#HAhkCC)o|ug)bY=FGWmvRa-?C&<~^&Sy)(NQlc`GcmE$_UmX`^w>A2TAYc#z z(j_VBfOLa|(v75mbaxLD(y7wj4blyQFd!}6HFOLPF*MwV_dVy_?|w)4-M_>in`f`J z_Nu*VKlT=t*Q=eL(Hxr|&U^;VtFM38zaE~p#;A$4(>m!EJ6p?-5Qp{*s~$@>!`#mM zhb)mg^@amxL5m{~4?k@wD2R>9bn%-(Pl&HdWaRzT){V?vUm^|`K@d$fuxU*+ByRrY zn*aOTN%Eb{2XqxPmnz!*PZzXL@I$84_fYqnG0srlkavj(@!Buqp6VI1$q9i(a`B-0KU^lwStHOv(=n@dqfAA z;{goR5dLSNhSXypVmrMa#AZ1AmmuVq93`vk0=ZeiJGnp4_FbPJzpW>B-X0c?Ow_Js zaN&J!ZS6=plDbP=&CjTF9+?{2;3(E1i%pk1NC$3xt@5D{$BVhEn$s(o6ZVY?O}}p$ z5|{4n`O}zA^$@;Zk8n3F`216)Z!9y_u0uqI)&ll_3$6a+KmFr{1M@7Pr|k=~${C%1 zD{^#}n=ls9$kSI3e6f3+5bb%{#{m%4y`%=!ZP>A_^j@@^!KYK`DcQyQz<~!Lr0Qmh zPmNYge^51E3VA~}mDjaw3QuFM3%pOPbBrWDQh9K0DhNH`4%KW~FmFhXVw%)cAI_yGy}lTD4N z5uyjA^a$l&za-F~_*u!I!w`NR+^v)6d8I~Blyv)0otGl_u70}tzNh}XRbT4yy#txE zfkdEMkRkbX=Wz82#zHYN)%HEl>($!R1fRy;d3@9(=MXBZCM0-ekIeb&LwZ( zzO8cJwXQx{pfuTNk&i{TG3EcgvvImu@GV9Ea%5yoe;wk&J?X5eH4$sNJ^nOiN3gVo zpmucye{*rOh)oE6ZcpywQ>}m?h{3FP^Io%)KJ{N6aXMpDHeDE^FH139E=BPzUvRn8 zCKSPr2I&`Dw*=F#%e-E?w|}2n;`G}4u0lm+N#cD|y&F7q5T4?b2)VHIux?!=G_3fr z%fBl){gj;)|CX!Z%{3|?2xQ^lQ2}8(nb40nw@kO2zT7k!WqAtkkvtU=%W-V4<4-57 zx>{*M#Z=a{M8w45rT7roP9ro;VNd%G~d zFr!uznWiE7xFW^yu2i*yBW|v2<`IDwH$+!`LTev+T_;PLFn9#cvrvWH{yEuIZ((n| zwkJ83hQzW=er;}=hzP9^jOYphJ-!gHP2(vv5_Uga7NzVCq6i!~+4vRV_9B z4}2@0ue*{O?+3;X!q&3vzkYhALN9b;w)#NnL-8+fp=w53>UxBk+nJnzON*?RpC1v~ zVg8H;%hGRHwWhkU`wK}ZbS zzV6)WFHpP(CMvXTwE@%Q`aT0RsYFg&>5Tn4kzmaJcdXUQ>mq-kwx+1DQMsg_mGB-i zjb;rvvLFYKVIkIrH+H}Fa>2Y1Brl>+MoO%1@yi$GtzJ})5#z-6_YE1eh<$U_7@{%{ zI)0F7M2&zV9#d_({g+{B=j{{e^|A4(f`I|&YJwxLHF%0ullPFY(XvI1~%C| zM9H2<1sRj>+mux?Rt~G29=j^H-b>q55@?ZmM&9;Yj z{aRI;v3OXbHq%xm;jEQAnb$gWvX#$A_h#}BOXRn5Vx4vN8m*DDt2XNCT+F8=4Ed)t zFlQioa=}~XVoz+z8AmD(OL))P-Kt%jiz=H{Sp{4)Xf}JR=y2n*?A7b{sSI==ToB7W zWVTxkkrcX5khim3u-Y#)AfNNEnMY5v+7mt%Zkr9a(-8ZOtTbZmMsF*GZSnLtEv+)F zifhY_1T>pC9;IfTE$0e^9Ibko#7NU&2vvKcFIVlif}HOz%QCe8P+YGm}!HH*OoAYvC(JSjUEiLB2?G~5V!Ug zxO!cq9DbF;_AxRfM%p7ecax*tTt(MK#QQ*M1&1=+A?(v?*MC=%{fFBGfxP{D62Maz z1qKmDVa-Je$Jm+z-G9(RdCh?bq@-JlVzO?@hZr}tSl2mus$X&V9**31S;7Lcj+CUJ5n6C7kyBom|S9^3_J^lWS6ClLS8R zS#@JKxe`|Nt47l@-o!9M%8(yyn)*83ZF1xbul|DW#sxi;3S?v8cd8@bu z6_-f3*Qu7Ls@w#sgHU0k`q~RBwTrfNNw51qvUau7cXq-~tVk_?jiYS8+9!3$V>wJy zj5bT~KjBI$!`OCTSH0X{DBW((h~7J1c*SC}zOs27yZ(ZXUF}o9n~beDn;;I8LB&th zb6%&@e&tf6aCwr<@dBG z8j1R})h_@Sz~CWrtZ`7n$46)~UxYPbnySbL&YK=WrjN+PcOKVnPwAlg=8>wVDG5?~ ziC{X@vV1>Dsv~sy-^l$3YwDXZOTi10qgXyXl>dz9S~ad~ApKoOwxHKr<$C{F^<@2NhV6*Wz#XDE@&{TJOd?>8V=1+Bu$BPrPbhKWfUO zRosw^k#gdhvC4#sCZqci&gI4jyIpg>zSq)|5Y#N1f)(!vY|z&xCtpQtvSM*OvVOSG zqvdopmbNvVl(I$Ys=n9C3SIYNZgL10Jf+h*%p;zl{~{!SQ*!*dRWPh|Wlv1=KQ#Z} zhk=E=sQNa_7YJF$%ibhpl+!Buk*3@%!f5cr$PeBiu;VH3K^7j1C_la}{`I{In56OJ zpPWIMI^aCO-$nz&jBfX@fNWg{Fj|JP{5)`)^7nP63zw~$>T&b$Ya+B0HiDP=KAJD{ z?@?v>5H*3O|9ZoyKsxh&g!q@_=e+NR%038MZ5dIUH$>+4#NU(X$kPX(fS)JQejW#* zSXw(3n~pvWddxsA`c)(UE$%0J`mBRwFXMrkm+XCy5A1bY&JTDJE-Mhz7G4vnZSsh{ zVjVlA@bGZF2@_IU;g5gaLqbNwqWJG06y%V6(ND&^MIz~B3P z{Yc?E1N@tb)s2kZ7o>puBLn---~MN}1}2K2AtPS8rxZgez+Q6&1uVC~^7nc#I$*gA zq7)d`nEH`Ilq^8?uGKWa#}(TB@-~rs-Uwr5Bugmk{SnQ7Kw)?Xtobadw}6ft4SXB> zNkHzOX!6fuG%Rk6dm{eO?+y(2kmhcn@`^Qo{_o%Yfs{jW0D4{pd(+5hBs4>lldhwJ ze0KnYhDA>be1Djbk(owe94UbiyQkp`ujK7VB@9fFLB?iyALSG+`~vNRFPiad`cJ=x zCGMh+)uTg;^C&9dd^QAt&E_jvLK#Wt@wLGFru-M(hUJ_1@?EI{sELgn^Z|N~A~G$O zZ)K7%^b9%{4KLVTY+Wp8s~9uAuMI0mcs&o8MAJBA9?dbm1DyU4@mY;p=M zR!vQuX(ose8H@&p<HMCLn;`az&+#{ib?c}pXH=`<>cgRuwCaQ_Aie9 z3y&iy61U@^I)nCnmT}d2Xu8XvId6!=r}XlALZ>md5;%dOT;Q;iUSX^}dbJZ?y>PKz z=iF9!(fn!Ksk+>F>hoa>Vd!31*}%Eu){={Ubl-TJi8(anrvrNu>D4A`1gKjL{&M|+P)WRkh zG{G4zHiFJ>aKISX6@-zZQ~wEJ{f^jM;9y!~Y`{xJtC2RANs3T)Fo;jxi$hQ#<4fT= zdALx+VMw=5CNIp}%tD4pvmWLy_h+VcIH=Udf&iWd3a0NRb`vDs$H3Wp_eg)aZV)`lH|la)|y< zc3zYBU2E=>&J!2ySfWSmli-aAf(iIf+>ng*=J|jw^?`^b=nN($?yEwuthUQ$cb)0g zv?&yWl|FT9vRYqO7madCi@fOCPHXL3+MiEGc(D$pFUWS*pMgR+oe$vW2C?$6?a5Y} zQX%ND#+QZ8(b_h3Pa`$Yi&kxdAE@d%c`-KJRYIw5%fC+M7rc04M~z$dbnqusy_o{| zYk5jZehmnDaP&em#0zZBTT|U@MMGET_P{Pp)z_!4hkD~3zDCtkSx%w39qpm1p*iYw z3sOQaa!bJUf`{%1+qRxYc{dl!Rzv+hZMC?0A)BLslP@4YXi05Ei>}h*SN(PTWIYCA zE$cK=sL5Kt1_(Yrc?xCi$amh5&an}g8enEDiH;J1 zj7E7mNm)09ReOG%bbd7L#|)Sxl#MXOmxU#GnmH{pNmffS-N5h9Ca8fH;CLMehiCVY z1gHTOJ1NFdRHK4|n6G`c#wq(e3f5jcc2r&wgC*K~6<+7*%zrW&G7CFHDL;dI)$hhy z*f{T~qTLUS2<^OfHJP6p>_}B7+OL_>71%e1O(scVta82umr!!fJIgL{otczG#Q9_K zzO3+oYQ+uoGjcI*VHR7`_(!Qzza!;-ZmoU#zBvEquHNtZJ#ju4*TVN={3p#hQ#j^o z{LhYD26})V?|LBSDf(yOMUK!D##%agGA8^{m<#QL@#~F%LKQ_dq;rFQUz&Qd{sz9- zg~|D1l={q{(~IbphCkZ3b_V#oo0uEH{EU|pUeB#l;tMiZ!0`6AJ_(d7eiBSiFL!Ma z*KtqCmYYkLPYhQL&-v>jsVAt0UC`rB9Y%GC&hb3{@w}6Iar3dyWQ~O$F=iL>@?x8x zY}yY}56^kqy84T@A0?a_tj@57I@)KeIwu3Wy@r;m^AXPKs9ifYHbFb2y591Z7OcUx zwhqPJGON*r$B|{pScXFI!1@Dq=1|GwS`QIa^gkO_;Xpv#c{{SPfWu>sLVq5QospL# zU3BF)Gv4XjWVW(*(N0cs#2a9Ev{7+=Hf0%o*54We3x2H9N$6KGruMsMsqQ$o0OTgZ z;{VcaDZtfp^Kj#L<>A+PSa@w$_LZMOUykx}1CPUUL*ffXhdC!ZtNxXI!Q-$E?_V5t z=a4y+fkxO_CBZ9Gg^~4aaYfgvbkh+?8#CxKdu`uOE+j^)VO`F2^=-;zwUD_wb9an9 ziq(~?WA6lL1*5isbEl&21uO1Bk+UeEUsuVW^C{uPbl+*>RKPFd+%Y|3 zbAI{<*8uPudr1-<#n*VeCZ_z<8q;5ilKe{@BPn7Pw z)}TIm8yGXu)y-%uQ5vhZvt?nphDT+H(5WY6mhf6uuD99m`kX&Ee5HQsGBFqJfkw{0 z{P}((iN5o7qSjtS^At1VA9WU>-Yo!s=pSL7yYT`0(~T5$#CE2O@5hfAauH)4;phHRcR?YF7qm*(NGQ1d>^~0v?!uXX#N7`lR+NL zI@ojM1S2J$as4EwWb8eAoH8qRven6MsEz&jomgSypmF0%Dna|U#T_*hwX-M1Zx5?`Y7RIy{7L} z95F^R0xYvafS>!U4lizF+JGQWc%)h+Ka6OmE- zRsF%4?e5T{Mk?af{8Q5~OFl{4*^-N&bJu#M5pHS0@Y94qAE?1vyY{l-fD4%!-Dq?X zgy@!wgsF785?M;snD;q47X4Ge)qGAE^xOjSu?f*M^&?WkqV>Auq{pHa4{kg!o}`ZY zm;kHnYQ09IqIh%Wd~9q?QBRM2?2!d@&xXX~5Z$ODZ>u`3Yxlt9wJtclATM;h1De*z zPrD*CD;UrgyEhl>K`M5%tZ}efG!qZcWGKCwUg&C~S(o6wiavu}Mb`F9xB9;SU6-gq z;GoQEEEPL_GH}^w(0(|}b;&N9Uv?>H%we#+f6}o*=DD2KI9xulU&&hkQz*hq*F>6R z{%H~Zs;EG3S?fi90wc?58J9Ry!idU21zx!uJO-43W*oj@sUAc3=aXN?0!}`{Zs;Kr z9xHJ2@LJ)8`=ccxO(xa5HFW5dqqFbGk3AdgMbXf^6Hz84lyS3Y1`F#PyMLBn4h?S% zv0nw~*mcd$;-ZAcfjI-Ji!0uHsPyRyFvfDwQM5EJZK_~VCiv*Tp8e)7!IUsHTBxdb zs_*G^hHqfnR^Mo7HDA3p_o}25?%QruUw+(H?FOxRX$~PxAt)g7P-QAj#R(l&ew8gs zdOhr(pT*?k7;=lD)%jjT*pQ;~3AoEd3L6?i;mI&oMhPmM8(`t0LMlCHXSLW3I86o4>s;@`{2~aU>;5@TfeiEz3y!nlL!QD-t z^d0+)yIblqz#_#ncHCuaL4H~=i&PNhln&0sHMlg@*Vn5Vl%GfRO$dB=l*>c)nXyj; zuBQwSACJ|y+r4-;WdiE9QA_S%0K?!rW zQknP36)VU_*IFfZ{_=}9@YECk7VN~?xzT+onLu5pjF|iLFTCMod%(` zmH8el^5&I8?mBPpotfIQ#Y*q%HtuG3ZCHDWKd(J%3HB|v1%ZN$J$-EV^)DaYr+?qd zhol#2>=N4S+G6ShXEFJcuSw4a9MdVCt)>WwxpD=yoU2d2xeR?j?pX9u1-brSR1*@D z42xj2xv;nNS}l){b~z<4_TaTCnpLNh*c}Ljju<*9pUc;sl4o+VqG4GWL%OW%S*Jry zNIVfH(HfQmd(xWv}+d;@ZhT#bM)v;jZ-s+jlkEpkkGj z%Zyn|3@74ceALGA!T>-nq4#BvPf9P}#XA}$dB;C8pypc(BPa6&0LY8 z`t>@bTG9exS0ixs*CTQ5MbPs5T88a#Pe$->{}Z6)PeZwq$?I|&m9w)=r7?&jzWPH>Z%jZFOLCLjOS-pNRvf%O3?c-yf2=utoH3Ewi`qy1d5 zjSpPgam5kLkLd6wn4YHW_$~k091n3Jxm|OC7hZi=1^lxpz!|~4_KK2O6?EJf(zV)% zXB7J4jwfoBoyI!B8d9w#h|d+->qXp375;`y!i2n2?!q#8;mKg;;Kd?My)?Ni`{nMn zbv~!MNqK4Wl$y$W(>~oscIQYTuGm-d!U}xH!|~ViyXA)?Sz4R2*G*m=@Pr~-Qax^J zJo(c@-*@F34XT^&am6QgOfCEJ%Po7aK8b%$scgCUxdDvJDhstL!6=U4DVU-*PcpI6 zI2XkIXBaP@*U7wCtG1My4^ZFQ zKY)l@Stn4d3e6L`G_#zpdT}|`?BQ~VC}KYgk0&@#)=Np7x~N9fv$x`n)Pn1wmCs3L zpX!;X)>B}o+WqK|6I`+K(tdzJo&*;7k+@>qvE;S)(^jk zf=4^Q+0}NQizl!jPd%~kwK+|5)J*xyd{2HHEVMyf z7M^0%Yi{^!jLW5WPu63?G3?JqqeptNV}DDo%bm)92t=3jRzCgfo|TAyOXI@4D>U47 zC0f$pbbl3O2niz)aCdx6#?MVFpGxxR8BsAb)(?10zaNu&}kaUjNGXl39mghhH0GJw^H&;GcbQk zS;maTUhG5E$k4GtE2&Q?_Pv=ue?Sq;%o0p^nkQeh9CBjEio@pWl_PZdnrN=p8b)^> z962e*CE6|34oEk5WvnW32(B>|ThD9ZPsp{faN$6jctBV2wKfYU7$oOEHKLpHGTq+2 zqJdAxO<5kV&xhBFAN9%XP-Pl-UJ&qo+x!~P>{R}WxFaauTWp?*YMXQoE(fzVj>f=LrD4-&V<;0TJXeNx zHlcx2IL6=?N7Lbb71X72C^Cqe*fP^eV#e!NLJDlGf?nbl><_Yy@$2-#x$VAbjDimD z4jCsM4kZ>&)jCb$Y`QC@`u}3P3aeX1gqOE+L}qWI9z&XI`y-ar`}&=-&&E%j=b5mN zELukn-+sw{eYAYi{HsgG;KMc%x4Y(SB60EwLMN%BtN72C(7${l&`7qg<` zGsx~9Ti4bWuGnga0J*vlyEBz#YErtD-<-C`rP|R>rCY(5%gdpx5eug&3&GMX_0*Sy ze9o5C=b?t1`0C)@HjT@RsBZ9SqxBnYJ8N^%vtV`uIL8;mzV1qc?#6`>eCNHOCkuL5 z7Kz3&gPh)w{jLbGfH1;TUF|_G5=s35nU}r%EO&!R-W55Yg={e}d#X=6Ch;l7QP63_ z7A1VUOE)Bm?r}dyw$ykbWS+nS=KUf-UUz6V^DLj|SbjMp4i?j4HI!-EATmgOnt=eQcqi>|rVh z{H-1?fDk)rIh1WD#sN*vYhfkV$qLk8<6P!8holA~tBN zpsloA8nq;kyxhjOFJm&B#B*86u{G{%<#b_kQR1@@gKnWz8nihT4I6DeD+#trYvt#g z5#^)feSJ$jWEjcnLhsb^CSKck6AgDiE?jk}rv6^;us-0pL4i(kqkL;Dr=I?_z~7I< zk>>|N!U&bIc%?+q?xS)JuBYnN<&UFwg}aahRYG*hm7lH1Iw;iAKFblbVjZ*F4%q9$ zFD03Cj%Fb(2-$j5aC&HwG+j}MfS;s|NOqQ!ySQu`$kII_dB#=h$v<#bQUEeiR30{f z<(PDQ>@VxsvGa?}T2AG0n#^YHK=56&L#WGCq*$x%ItdT5uD%aZZ^!qvi@$Z2zA2oEokteQ+}<#{D|6!WP(ofc2YOxKusV^Qox5JAchgK&TI zlF}B+rOk=b8fVhN-JM@zkL@~c^>SE|cm1~ai(EoNAZ9!`VT9NFIx=;?U^Jm9QH-K! z*#q-RHLSU!I)UEas#@X}y0RhRU780x_5}j6ht0yYLBLZe5#gO>;G?3Gn=TsQaksEY z?+fUS7s!2maR6=DE)C6*j)VNb6iUXRY`VVOixj$o3Psp>LWvdC)$z-k5t{s$ha+$x z%*u6jF?h|-(tJ4l6piwOOyM$6*@aWK<4GUOnjAef1sUAZsB7X(<@;6*rS!jNr7{MA z9f0=|XQMf4A+1%eN5Zi8+ZHKJ+72}Lp*^hE+%UalFV@$6gxBR7Ge0eA=QA;Oc0XiE zdb^wPSMhrn$BG|ip6|1aveNeH60(lj1i96-tdiPjmZX+*lexo$nVRj|L3-scFZ`hU z18D^8xsGd;o(pGc;Vrx{p8dD{j>r4)zak`u+qM!vDK?(gPr6I$qfzqIK->7C@3=Z8 z*(w#NQTZAkUrEirVmt;1IAN~Jv;X?__UoS&w$EEUZpx#fvhrcxJs*>ta95+l+R?pj z@2hq1h@#R4hMTrL3P3@Vt3uM9zQwUN{T8LJB>7;>D`m_#30Mr;e&f>hx|C9`JxhD~BOry8O7cmlK^k=F1hQm1>(T3<|L(IJ zH$M=OEMYn=I5^JJErG9;Hvi&J{B`{`13t4S#47Q4_r%otq9?m9(+v_7uSzVbrt8)B zyZx7;B1uc&@m{FFHQaLtungS8VfH;90i9FpVPX1yeAf1k*E^KASSSHTz^SfmF{c}{ zK*}L|tx;>O!|VUjYPw2F6g{S4XQ5Sy(`G&yTrK*PTdEy(Q0V$Xkz~(n&!e(TT}l*{ z{OWh0@?MnCcD}vy*UIl#8gBhj@m~_UydiUT8xfmEWZ$_Vu4J=MCms`}c~!UVk6JKV z!}cv0Psqe(^l$U|bA?+uBe?;1eROf@k0`{|^T7v63mt=CM5XUg+dv9mWTE*`x(En- zabo#<@v}5ba5R-9nd(5qTO0zC%Igflnq(EyA_ED_6Y^HQDIhEUd+#hRW(rHRV>y0) z6-(5f6<$)cKz~W|N0Ru!aIju)2ugEcQ`nLqt&+WVJ?Q5$+#WC491>Z>do)vP6X`PV zmM6Ts@smhcipLvNAF#Q{A@;0NiV=W|a0^ zZNNW09LDy*$Bl+NljT!GBKskq9y@FpRxbG)Ty`b(eb?@+yq zObW-}9XRqG8DfDC<=(WTkY6mKQn{6g?!OU5#n*9E1e}%h^km>R*YMny+Z=zeKZP5b zTW4TuJ#t%^>BctDWw&@~pXmv~vfuG|9W;E=QEv)E=~-%-uXixHK-gS6Q?sMsZtw3@ z4oa+zJxt=X3IRIJy4w6uA0;M*b>=H(JrqTSLc&9p<#=-b*-RBr}nt+V0$zqb~W_(ut@MjSo zTiH8qXc)WZvr5^!K4k!e0T`S&oRWoCAChp}Qv7W#fxpi1TZwtZ9QQ%ZkJ>g0os?SW zi@K!7ZP@*NRWj8AxXX%I5=c_t_Yn)k=mhMoA{Ot->7X|mz-cJWe5+rTm&M4>6r5xI zs2y#A9$AUC7<`mZR=2oa|6IWSor%SwP%uZ5j35cNnyE1|;gK@tcik$;%go5gNN2qN z;BnB_))v6ODEQh~bg8ZeB0R)@zSI6oA{})BU=NH30Aivc&%3RsOtyG=h-|zEq!7NA zWb>@5RIe2bFg_rUZ_qNludh&6E$;?X;HFRkn*R)l)oZ+8A4r)RYLoXq8<}i!b7;Cc z+eT5B)y?|tgXu{jFd{J4Hf!^enwADYP_DQ`_6+BuTIP~v)egtB6efPP4 zUVM2CC%?x2X@U1y{wc4NAW#D-R470t8Nq}`8b&T8_}~eVqYBU6bp-!P?K{&T(NO;W zL4Nh}YM=F*&3R|?7o)y^oUVJp`&>V2&)!x z_13d7#R+b}O`uAc1AMm+qNo7ILWRkscb+6}8W6dIhX+5y-ca?aT>$g79F^E~;9a2EDP&Q&Sk9&zxbigaAiQSx_^os;YvlXX`8%DbUGo{1#Sv z!@DsX#s*@wDdodTNOLT|E7LBSjQbN~FdkJE^P^)pI=TUOE(3sv0M229wvEUOk-q7s z^d{uCeXk8XQ-_k_hv~U^-)=dWZVkWkmJIzr$O@waN!zc#7tMHmN*_|QlV6PKU8*wa zG`zauRkflw?Z_p}Z|W$}2|wtTM13tPD~qHKi)uaGTM#NNDtZK{N?lj|!|nA!xt1?Z zf7ZnRgaub=ZAHx!58K^_drJaK4%hv0Z{kIIUcs?wt~H89YVehT?bWU5@E><_R7ZF4 z+`Iq4@!ir*%qnhhw=a0F=|Jx0>rvOpH{j?ah40K zWZ`qS?pA^uGYVAfI2jxK1leg5)kdBzRzz=0+^47~Bql}S(~ z>id8Z6!dRAVidAq(~AT6Dv@wl-GMKKZPiDL_FuFJ zAa)RVQ}yWO>J)d^df`P^z*UM+A38-wa|{(WmhiVOkfFH?546Js;a0)Mibu7KwyxLn zu50*>%TxHrU0OEHczc3JecbC!zd**LL#7TxL>OIdVv>p5yX2~>_md_tA*ELG9m<0( z1UAj*_u;7O5#fE&GE^9_8PdU>NxbXwYp#NA@h$f}2oIHHms?gKsVb+JTwC>Xxxzy8 zN}WJVvM|dcxxx@1E64Pu-)*MmVDJn(bJ%1n*J`e8B##^2$cwdlT<1*qJkq2^6)gdM=48}XY*6B-qwUrM=-B#~R zhdU%8)LA3Nv}_{q$u$Vs=>~%>tqLy7R#31JR*hUOoVB#EMu3T};eA$mrN-0*-_P&i z8~wK41v330>pWQ8=mP24l&me#KW?^D(yb5f|=wTP(y@!*i~kt*YL9W>BrPb-I|#BsNs(^%qAVokLq|`#?+i4)Z^e%5 z+7GU8P#e2q!dU05wD16auOqLv1)IV>e}s5tMKXV?!Re0$0rMAsPn2dGU_?Q7I`o)TaJ;1ME1=;i1Er0vO^A+_?6; zcsWE3K;1Nu)&SDJYIhv)MA73oBwk%vXJ~l3UcF;|0>&_@{;k~HSew^Kv90uj;d0J; zo(boq*B-N?mrm%^J=%egRhMx{z;q*Y{1f}b!4fSjs=TtkN7#=eg1J^=-W_-^CuDs_ zds5zPtrP0b*};nO4EDAr^|bcffzLt^FYd9d6`3bLN0OL6IqiT30VAx557u%%pwZ&0 z21`Di{(LfGBRFrcFn@A33M$VRv~f8K(`Y$^O@3Q~Y0PJ)9Z8!Tz;tHzAJ%FJ1XzR) zbMgU-Xb7E2{7wa0{vEyGxSZ>m-MM?VnfTH7gWs`rSJ`m4%6rs0srZr2hvD%rtA6(r zHge5Bqbf56ABk!jOIQjzN`_g0N!!}27!4JUlC%0gPdsBf+u7?xp$h2yp560fMrop$ zrEYC`dKbQ0B#EEw;kpL}xt!#bGuZ64Kk-<$P{)woGn;JYJd14N^l%iv^obYXyhg!j zgvR1XKt5Xr=fA))rr%lN%R{0My38QZ(N-|3b%e}rrxi*X5<>1u(P4z~%Mp}!ET zaR`T(QxmJj6CAf53d_80$12GAQ7YSczT+Lb)dKTv2Z9m=CMiy8$!=>2aKHvSW<1Ed zODHH59wPyj#{r7;regb8o|}r4{@L9@y*3f+oJ@U;&9tzSb!TI7shDJ!V?^D$1s9}F zu^))4w2)q(l=xz`H#NGD;fYGIu-Q)fYn=8NDt@ti#VXy>!inH0J0^xl*Kycn20U>Z z{1B4k&ABs2w0VlP-$RWGJFV5j{Tg~H4>@KR@y(AAx|j^gg zxG<1op}97l6{S%8krm`#BamWDnH0->NQrmVrwveFf;WRmklZNb}XJ+}W>g z*=Y=C`xaIYu+{ofwSPbKb?kh?+6kMUr7^c|pw&z;p-`u&=0to+V~oes-|G#m=wdxN zAvc-Gf~Be`P*hG~9bx9Ii%IR~M=Yf2;*h6IdKSZcThui|)elwI zW$EjzFB9t(y&!Kun`fux`=uU+y%BTe*S2EjZ~Y1hI1OHhNqGf4v6iFEHup3dgjqH# zdMWOBFT*G6>D887bXD$h5{OCyb5KCaCe(06QZSLXDtFWNQhxb#^!|aqDzo%pkTeUyti4G`#>Dbw zQ?s%l8=Fm0_}CT7o>UO`XGSjDQKRT4CWANb>N*$OHO(M1rKeVtI7#C-Bq)&&-O{I8 z4r5fVq#LDq5j5?3h=0Dkk1nRX-3ZmgH(D19N(?qsI!}xnWkT`3o(#!vc9b)gKQoz2 zG>Z0Q>+K*sSWbMQF5a*D#o%@7kKV=ucg^G$kI(UaMq`lgHO8PQ1zMn;d~m zTpjehZXJR8Zp}uZl6iC1VpC(=8i z`pIAB&ic#Q+^yB?zXQRfry$>7CRJ+r>1RO!anMbfdq89? z6R@DE&=%6hG!NPup?Bn%0`}g;S~avD;WDA?a%6;loUKo}B8W*O=ldSLQyB~y)4O_| zd^WBHSIf2K6IuvShVFdPd@&Sj!6?4F^`BCvTb&dNg<`-_7w2HJFc@)Ih_Ex9!9?X( zuBDvV+Jf6TTXdp62~tC?-6|$XOPouu;i37qv}9s{sI#j-)70WSbSE*j@;+$sF=UB! zD>%xkij@Hn|`Zu~%L@^N9Dw$$T`iA6Vjeoi-cjltuh|p~>h!A3v787tHOY zeev^f({2Y}ugY$>4_2ei-gUWC0l52do5{dq#5{wB{vo@j_dSuMpFWg!&h!2csvY)%bh_%6WBIOF><3Sa3|HPUf5yC&=p<*Bqj{Vd z*FZke*=jYA3}R;KX|dL~^*l`MvmjA2B6FkOQZpLRB!~+ih(sp|tFF~1pU_M5H16tY zQfK)v5TsK4zGIRRYJu5=%g8pCi!nZN$%M!Qr#eojYyWDn=@Qzr?zXC}sPXRIwd>Ee zSU$5T4L)V$k|@fVakR=2ZRDi^{FE0gpYHTiN&rekKasus{L^kC;)hj@`Em0?i1< z-Be#gw{HrFAXL=T+Rj#>ePDd^{~D@h%BH%mEP9+p4wO-G<17H`eS3Y;BaHT8Bn+ zNbe&CgQmP+GT1a47E$GbIWHML$!u~sK)QBfFiDQ0zza3iAg`6LO;cFUtuK3p%sEaJ zR4*GrjLQ1+gJ!Klxr=z%fb9rs`uV7WMu=l|dB@$@XwoJn?p?TslhuWpd93faFYt%>G_o$-(aLP2672IA7Uh;U>nK)aCa7=J>*$c}VaS-wMn^A+LZ}G^k%WGq!lEu%Nn^L^^wg;bC4wGKz zjOoAH>xi#C>ET|l3d$p^V^qRHDFj(YgSZh;wU*z@_qp)kvNaILQ4 z{ew@J<@<@0AUd1RpHw=_54wj5#_Tgc`MpRIVYVtX9B(oMZF!l9H=T-F7HQdk?G5=S z-JRr7&DA1k=DM@yu5japc<@EpIN-~pQJ`+n~D0(89xu|tDhEtT)QhKTno-5QCeeN06 zmQd~PAc98OETpO~8JB@$9U*Wm`8X|^Tp~Zz$V&U0`+N%geG;YrO5vq}XulISiip3N zog(lDFSxi_6421#0kRD|7|m5YYD_XH_op=ql`Gk1PlpC38xBXN2$81giTqd-m43Si1cwTd74V=& zXbzlrOFlw))vSr3%$yRd8rpz1rWo9pODP9a_PFmg`40K zTa2f4X(T33$jQWt9w1~wi7_Da0<4|AsPj@23$52b?amSkwx7Tlm^vosP-4J5ULcw-?3GCi%;m@&)gF_Bs>0Rf9CI?f9OMdw zy1jt}0CZGYJ{(Bf(cENs51(Wf-6?^#vw&UflFdg#>b&vr{ki5{sYpe+dGH$Eh@u0j zojWjns|=Jszjc_cN)(<#|LPuL6%M0-4KUjZV}Zt&8*M)z zZFUz$I|1%eD5lENL1wsVerbQ8?Q(||?70b0SO%(fp?!@7wlNEj8^7zMO{Ib5%!EMa z!G7cLAr4F4wS^S5sR{DjJ~4m*xh4UuVKQ@xSR6^{Xoc@OJQdmm25vX&yUBM&=hHQS zzX&m+uJZfK2kW&h&Rxev?*)*1m|RSNLV**Yq!Ig9-rKtU0Og_32CTv3KWUIlpS;<_ z>UCA%U4&l;#E{o!?J+X+4DG8gz|$|n zK65_Jf9V5(Kpt+F!|swXdU62OeL`sgN`?KKl+fE2D&AXfUpIvJ1;Al&1!lqtm!tpF zt6mfRi|nq`1?E=0-l&WD4AiN8yHPIpi&F0*4=Wnl^FyQxX&~kg0$_{ac_DrmdExL^S8{fA%??M7yWhD$vIZuwcNCXf_LLVS^W~3&)Z}N8$;AaxS@Vjy(QP1+b39*4W z;3g+POP45s*K-nF**gayyzvQ1#?JU65rRW8Bn^Z7HbzC}2l^dsEC()g`}umq1Vh z@WSpRA&1DNmzS4sOlOPS1v#4IjTtkPdy7pm^yu$u)g>jCyVS}@VN8JS4zY@X$Ducx z{wc#8nBlprK0&w~;BeetDO1#YNR&Z~w09{wg);#B9H9Vr;dD!Yl0C-sPX4zG&;KKw zz#cO}rkHmTegLp!j`MwlfMov1?BBj#m=zG7!{;^90Bu?TK|U9ZxKp#B0Ia1%Kx0L% zp#S&8l5xYiyYj^ff&i+k?6K~m=_`LvEb)|dyhHFm3ZnqftNdQv=Jhea`DbK_B~_Nd zU8l>-0tl!bBao5Z&8ktNgZy_3zbUcL zLm-ls|Lxv{&h6U5r{ad`d-ca&AmzLO7&;M}3p`eHyHVN))Z9KtF(80>04QEPd&_sN zZV(5!Sc^ba^rNuL!&jV~&L&P)8id>YU965@T>w5+eLHug-me1xcP<||l$ zzfJE_o<&{dI>FuDUY=F1%=gAm-+EtOOixwkQarSj(H_-tf;U>xxp_)z-y zuSK=o-CFJmn{8biOwH#41_wghnAlm$o1T7^y9EM5{_+0@)au>?pLj%D|b}sL* z@IUE8?a#9`n_F|Ye)<^ji^G8YSR()mQ3{X|(?ButUq~H_BdoGcuj4aXNJcJ#oZI`P zw8zRnBVEDt81y*|diCz@@H5bTFf1Mqs>u)g)o6NNX6Q|x%69_D7rx&wG;}WhPv9oV7)d2034t5>GYyJ54S3{n<&s#E^g$3xoznz+&EAhWnEA%gY`}2byssOC` zcPR-E6jhkFgoxcx_$xd`1n`|njO<7p21=N_ceoM$>x&3Gp!d*7ZD;9}`e5cQv^rwy zX9NS7Tx8HA<|rf+cIU<=cxTVJTG2x?>JWaz7Q{_dZ3693ezs7MPQ>vdz4?O{NzGvZ zbHva7UNQp@<~rJqp1|-iKY1==2I10CDj-iKS8BD{u)28@uwSV}UDrbYPY(ViS8%cQ zY>k24AQm9s5MUuukVJnO(jm(QA|GV*86$v65C)cJw^SAXr-K8;y8)JlXL6gMgDfxr zU(b@+{`tlOHv@a%tkLnjejuN54RZ*48)Myn848yNR;b5O-zf~jhz67nVQ8HT91x^=L_^2Z>x}+x)D&~5&%(ji^!e(xoo4T z<}`)8o3yG2uBgN?zuiu?Ead0H@OSS~=+DdTFI2-{z9|{t8vgh$CHIT5;4nA?J7_>j zwL_x>{mnu7Zt*Md6ToSpR%65PxD zk;13G5Ib1(VXev(Xj%2Yg)AE1)YLQ>9vQ36qNqOpHmPxLs$u!u28a;y+Aqso#-^kf zR9;VxH!SzuT%7?R?vpLGea^{wdp%|)VM4d6wQ7+S@@^dK`*9g z!LE6L=>Tv!XuNNa(sO-aJzW5RIzY1EA8lH8le(3M?+%B*;6yJg7qVSwdSb%al|GrV z;y(N-zLar!AZ>8`@<(eBy0KMPdNq}Xjg>%BH5IDn_Eb63jT!bb0A%k8B>2X+mI5Q` z0tjbHnLEL=83)5AU$~vrUG*ebRO{m276tVA@1+KmfJuRaXbGJgkIwXpsqW*-qN7Ape99&Et;6=>2Hqay>+u5_wHU4 zngVP!aMjH=0xqv=bz+)QzK|l+-_+IdgKG4jEG;yKP)0Ww1}{B7^SL`a9(XO}Rxf7p zboGbpq16pQuVuTD9gkH!MDqh^6mohozLExSJtNEO0DidH*6a9?<)c2%fsZ2HmoO}> zrOd0F%M-*GbWVCOoc%zM$n}q7#UeKbaOi)0iFh6zsv;H;!Xua&AC}Vg`N`cBxERMw zpjysUaZtD}KtFx{9QH-4l7hQe6px{5{X^KIA3Cc{5C*{TIBUGSsek+#vgWQ@la0T& z%xT=0n8p9L4v8j-g-Rr4|9C_(x04AqbaZ??N2EJM7D7u>6oOCR@liH*@vxUIYWajP zewYw&y~WH%%z*38ED*jMoc*x*zo~CPM4ByHu&QA#&0{hBb|ZbU8JRL1=%Htso1dR= z&=P=Zw~?LD5hQ$^34=$&0KJSCW;>Z2QU&~1p(svw&6jL&v}qtKlg;AsdFHn(W5Qn^ymlM9SxhAL6Kf` zdo>85DhpA!>rumZxCc={qrVLw?a&O9hX54BQ62Tc}M<>&%>PH*A!+C=bO?6 z-Rv(NCXEu>-w^2%cW)suXEW`?7{LI9d68b^LUzXI?q);ra4P7vY#Kd1CvD$*Zl)Z? zbZej8`R!RFWv%m$oXaV&n^AlY zt8D<~`{J01sz)tD(B0X<%~2ev&cQ$rmCr9px%|lXlT+ynnFhfC;lEK3?rzUI=db(c zB_7kXtO=i0|0ez40R`i(2cZ6O#HIsR>}g4sUKhk(fl02GDP(rvET_qw(w)iJ7tx-D zXTyNBp)QtX;gELJ*SZtW?Z_uy{X|s*0nkREYvfW7Q&D*1d01oc^7Q`Gc;=9`bRXf{ ziMvZZyKcG!)#C6%dQi7e^bC5p(Ayc(ZlRbNrf#9w8G)55m=hAC$jG+J!niUt;tVo+ z_j7m>H)%fIAoRcjvEfK6YCn4$GJXZ0Ib9p?r&*7{XPi<^%#$?=bj+12NJL{6Wxl8%AVr+k_d8Y!0$SCKI{FN z7EnWoLGI~3tn|6694boho~boZ4_Tu%0&{hATV_FTQdta6c6Vfevrh_3N&X4)sMR2?Fi;jNjoG{FNaMZ%mI-u%m-(X}I4 zih=%jfbHdV7NtY^h!EpJS8c-F4)8;j>I}1t5FR}PrnDHS9rUw7Xl7JYJSnVRegD1I zg$)<9gGT#5L<%j@JI^3UxATAh2oMDoJ_u2iIMh4?s!FuY-|ozJd@gFx%*3mijfdDP z(w<4#M**v3KcF58TVdBusa_P-6qu0>-KxU+$$bfmMpU)eaJ85`e{FocHNhsdatr7w zau}_b0$>y7+mg&G?n0M!Wp+cBoiP3*pN5ROfS0&PPRc@~WcNGP>XG%jQ{rEK&EFlE zN7@SjFEae%XswgY=c4y~k*JXv#(T5z+R{1zSlPVagwAG;8>9h93M_4emhk9qm)8=-Etb^gb`rPJkAbG)FQ=Wh2kRx?|~c zMG|zJHMCuSyg7j7zxv+w)OyJ7FLcwV%LifaHs0OJ2*b$XH6z_Ob{8uNoxocSE{``h z*u@&8w9XX0Nw&Ob26IIbc%Kf&Y8;kREb@88by`%5KD*i6PKUWLn@d)X^KmE*t$K#inuqY&oWpLl zPqi)Ej7Z(L-6W4&FX6Q#H}6P*?l^G{s(z^c`{F*wqo+k z_}chfO(zO_@yB_Np;zPLfT%N;&kvT)_gk?K0PC2MnF$B>RB8%yoU~X~&IZDm z(dsphjEhT4V-BaJaso`yN2hDmdm>QwyV>_HIrk75^qHFO)W5siMu7-2fPX^Z(3E`t z$Hzqe4{BdD5lft%oRXwu0CG7cK%pEQ9Ss0Tbkk*zUK)%B0OmsfbbB(9&mqU`zWBAC zLq9i`%_x)E`Gyu8$A>qGVm|mLB$*9VJ5KOqZKfGQCH9~mkhZAtw1$Zm0PR6guZmG@Fst_m@-7k~^JUOzh# zC}`_T0VV)xLJ!6=Kd6m-jOb}%LXSluXj&SIOR%}MrSQ2z>_h7n;8eea$BzTQX={^E zDfUOvUs$a6MPa}hXJKJ6FNWf|?q95>F9E`8_dbdEhd$8pcGDf;1ulN=Wp!|8!A2^& zmurkNS7F%Ah+_|Ob8~Y&XeUzE(TRA+q!y@sqfYgYb!?JBWoW*oaT1ve0?{xwWhO{J zPd@tqtU3ENsArF*;rhb+^4N@vhbMVlQTRgQM$m3Cp!b`d@<>;gjC|5#L^WNvY_k^? zJ0kRoY2ko6!Vrn1+}%H!(95c?CkI4h-ga}8F*m)ZIxUSNfsR0w5-2;=>NKGf^zkF@ ziHfD3UHg635zNyzTR{*_>T22Pd1 zcMl~cB}KzCz*4448#*LAYow+&P5miD1e)N)1 zT#Ps?wX3y(^q|~E@z5qr+%hUDUO@6|9$13?u-i=W0d}=RXO{yNYZD{@$438U6sVcD zN#vn_Ub*_qf2{l8q#9z28XS25yE~QfGhGnrd1fNf1v4T`*!rESScWNs8ZI}P99>+J zUz%JXT$LOwbtX#%;=mypyzs-^7U4^6*)K19d?LfHDBBJR_~9X($O&wo3>+O!^N38X zsPRBAOZXB2gn8r4ZhlvCh2%WwHSKGH9(g9{E2s$)8Wb)=J+I zMf)*-ez)dxUr;j`cuL=tF8=MmJ>?79MI;VhmmH(%4CDgZ0|~$dty%r+AVAN+1eXIh4}t-CDfG2aB@6};?5E#@$GtWg5K#GFy8h33 z!{*nd21+=8M6)Um`>X}5gDX?l@xL$iek1G+^GM8afQiA?2ZJaP|Ig3<&-Wt09`b9% z)PP3|2nRlj z2nHXVkwn1TTAICoSebuJ721T?)zjmr?(c?*zS>f{4VW5JR0b#t87@`X)VuP`f0K)S zZT*QtoishN3^jX+fO}TNgkhj8W#Hgum5gba{M)dhO8EO*!6->dNlN?_;+XL#!26Kc zGOh&|f>Hb{4blGXc{ntLP}pgQ2~#_C1u*#_2=H#WK@?`tzYBC7!NMqpF6c2IU^4_@ zXX0B7pYp%0vr>~D2t#5&L2&OW92zeQJ5Dn+0Qq0VmI22RcnYQFp&^tJ=Moe{`^KA2 zL;T;T7w`!tIAyOiSkMr%d>>PbvB#$x{l6U6KG3TGmoBD> z!JM4+=^Lr10fr~u__9t^JtpM}i_NQr`pz5{b%j9z4DG*Gepx+-cA3uJqZ;^&ABjyE@irExBNpBvX+VS!j zSZEUPX-m7i#D9?dz_#ui3TG|Wh?7~Oz}`z{O-JPCv5QC^TDp`0AaPeNFSXvODb0DK zjuc>9Gl&4^dyt2U>6eyn|8rAPtIeWGwYzEvh3%DOq!jy&UMk!Rqt)2<=}qOYp2B`q z6q#$TEQGAuhIudeAJm*uu}(F_!0%13a_*m;s;P20zj6va?WpJ7@8OiuQa13(Gn;3a zSs7DOK^(bXy}~Yh#t>t8GxcHzsGPAb0QanaLEz~spsHONn;-~ma7i$S|EGGFJrohE zzKSr`iPaBByDILBx^GwZKxX-TJ0jGP*E1!Divwk18`FH`VY+B`nRY?@W=CXs9;-i2 z%BpTDMNJ6bcOR+bs1YVN_FzO}W4yMA&JQZ!Q3T>(9CMbQ%fH!^zdpVZ|B&}bEr8!e z6vBX7R$l&NFjM?c>jE8@goH%tg}|`^-0JF!$#~g>upUgeDkojM!r-^P)AMhZ`jJbr zHf7_+OatZF3!2M<7Kci?=^3P?iobk3^Vq1Clxci~Cq(gj%z-fRN%K_u%yrLQ``7*l zPx+#*53OAFWzllv-y<~7IbCwrovos)m*`FlBQaO4xjAk#K?;!1R_L1P^}Xc$inf_K z0EPDfrd>_@)meUZoFMyXK%kwdYOAmoXz>(A$Rg(K{N(G`ueR;?)y~VG&jwZSBTy08 zSV(Lf5`1?K?4W+a3deZlu)p70q%S*#=9Z7t`@9?@TZ6usLqL@8vAOV`hFYNCGL`(Z zq4{tHrm|+ljqhFtM1=I;SMQUO1_gXnok=~r=grTDnS6h;M(o;tUgvu>ZV}lJ6BJZi z^FSPP`Im1%&8!*W!8qF|`WL@EikS>Jq!!*T^|`soNd9ox!slS1_JiM^=b8Wm6H{v^ zrO$Sdd3tlv5t`MmmSr?N8krxCC9A0CrhT17K?R?B#Sf>H628a$&aB67j|u&nZC`GkuLQUHXhd4bT?gON|D24w+==!9TqOV?VOTI_o0$!0Nr`HZ zASHr(t;Zn!w?JuX{+1LM8xvCD8wd2r^xd}hF#Bi9YWkObw0&~nrKIi6k`?b;Z)Qzw zw?_sB$Gsnr6G1PMooVAnerFbDO?H)+<56_;3tf21y>=g~vB<#aNQE~f@bn|v z9Bd}{m2#Ht2+4JBd}AR2-&QM?F2CbhRybx@d}X)VwC`2E36p+nkX|tz+?QDWiBsnt zw&4{&*X?#^*XPHSB?HKVS@wIS3|%sgf<_J!X;@j6_lx4bG>1MWdLzdswX`Sq7EeH4 zrzK|{%_6fygJ;%KlE%vk@mS}|H<4BUQk`caa!w_4(B~H0J}7;;=;7bO$PeVD*Wh?xr_#kh5-vNg@%U8kMPja(`Nxy zDrDU9Lbm6+*!pX&21QY***Lrc%wBL-Sf@3M(9#uW}@{q%_NaC%oi9_bAe$X}0*R-EwLSTcT@Kc~@T;5lL3ceRVHYVQd(7zjK z3#ZgvfP5b%jW&*Tz>Mz;ZXPZl`0+)&e|@_G!S%a#Yw8g70`n)6veDF&kAR0I5A&PH z!OU3y*uK9!;($0`6k1-NTS0ySfde3eHd}VnPNbK3+MfG0^r)xeT+m@P-0ZI94Z83_ zB99G)_hD>0n4TUP5fM@RTj^Ds=$0vpJ!>o*sb)}3FX*^Zf64FU{m6wWuLg9~)yNgv z-lb8EW}$b%xLWw#vbqqhdSLm3S*xc4|5ZOVSDA>vYCjj&B;c%1-I3+6sAuWcxlQ7q zzZ&S3QMS%aE3y7aM7m!aFTd@_Gx%UBdS{~u-{^N%u^J{?2_a}|GA**`bbo~&SF?K>ba#n; zSJXqBqgJ&X^K@{+gogl%TpcRxTf1A+OW<3q-weWjj&U1f_rqUtiB+=qn_hnE1%%(Z z(&n8%^0rC};kxKIPB%LKaQqT}wWtSMWSHjehN*Bs?1(x#Eo|5${?)~7l1#W0+Q*y~DO>bn zC;!?Tb$7IYkG|)Uq3FMmAtf*YqPvF&G8q(tN>+?byS~2e%QK%D1_5!Di&)phD-VnS|3l z`OC78w3(f_8E`GAl(c7khu`Jt2bH6L28;ALV;&;CK48R=gN0QxRq4lnr6})ahv%RfRs(#ZuJq^&^9};|o5O zmxtY-XeFkMKTn2149uU5hd3=2+2))OzfZ>c(MWn5P-*^IX^r-ACNmqRbj^ce2dDgp zl-6j_f&ylI2`ekBK|kP%W11X^mF90JqOEPfS|XdpPh+$tIx8_RxX0ro=_ z#AL>SA{a=(+r--~bzt&E>o2y2HJUm1n3R|lKf~MC* z;tCb{ikIc-pBCYtOP?vm-}!mvY?&2jQRwdZ4h#pEuaWoKIi zH^8a|e}q7|8ir{xyubK?qCoy|jV8chD@_&g@s@!iP_Hk{R6FQa2X9~sCiOr_pb*bT6Jc-RT3To8lXqVXNzs;k zh6GI8G?I&4bSv%ym4%6V&Td+DGJ?(cmwR|h948IuYo&|%jiPNP&K?~2cRMJXFcuWn z!6|Gl3oaYmkDu+CrPjDh#`)e1M+js4AzwC{mdovOA0(a!_(y+miQdh$R`=#Z}HfTZxWm^yCk=15j>H?AN5if?%3J`FW-bNnUp9pjT zEM?t&C2x)&!65aaWEK9p@XdcI&-iy++R=N3-`%F&K|<|oQBfZ z&j>Pn6JQZ9yy)(qUrSsWz)}Rn=3pqw!*f7$3)RIQ^-@z)2a?Hv=emfp9@rj4L8PRR zZZ9Vq7l)s+ohZU>@ABJRh=9z&pyG!+B(G1bBtFpBXUrlOQx!?Y8WOJ3GN5MJU8rv5 zwGf>yG%HM6^Y;lH7>H6hd(-7^q(!D%x*;7cy{jv@uyi{Cqs4F*Ad^*0DP7eKi?gMt za7}(@q$c}%HU?d{umW*o6zxe^h<**qQa;Z5{8RlLJhFjTjRZ}716%esolz{p7Hr90| zE+AMXJ{OX6UMGn~olo)eg##*AyS?l*CTP!gwK2~OxyxQ{<#U~6dny-P?_itr57XF6 z>oP9?Anc=?E0FGo*HPav;PRg{hCj?76B$!Dqet zE2v2pWTP z(Jk~*C4Eceqq=CaOB>6H$oA_#{_*-#Pp5pl2b-I*8otqD!B3TL7z;p!}(V|S&>A+cU(Ot zdh_bEY%-E5u{K--T$D|eXWPx`SWiAQyOK#|t-lx)O%NtGb68y*-(zhduk1EMb{A5t zPP*nJb2I-Q;~qC4{+yAnMRz9`9r>Z4T}Cd^7ky3BZvV^UQPa5&9{R!N z{Z&PBX(g1WZA&LB-|O`2>GWt0hjXYD6T%knAAwCmR$W{SN_7J7zYvj1DW`o_u~}@e zt6Cs4AiXh6ATK2_H>>5p0Ch*vMNwv`-QGteNd~`hzj)hSW?1vqQ90UFY<$ z4V8x(XETgt*B2=H<^?oT42Zq+r~F@2!^tvJ)=c)%crVI@T$x&9fcRxIOUCtZFAz|u zCD(BOFI7o^f(cDqDp#*kIx=d=9UERT^cZUd(~q5EioEsga?{+^8T$TajB&_Z)_odl zfy7N_1+R0jY{ep+eLQ5$H@|=JivW2KCbxP*YC(=m1qYcMe=T+t5a9R6ud@B&$M5G` z3xVT^f!Np-xqm73y=Sr^S!2_1VQl;6LCe^pz#xz?{k)Z(pqp=P5fx^|ab1u1dV#qZ zLWYiR$TNWR9x5xQ3TAcU<_|wOhgy?ZE@Ff`9d;o9WW6L@j(&EtA$wWyV& z718Y@s9)>4<3U{a=hAG`WZcgxso_Za)#?d_oHvmMQ(r!dg@k=rQR6X)okhW1uhn}4M$@AH$BIMLh?4pvb-+-%Y_nD8Z3frTphZA;gF*O62pMe~k z@gQ>8#Yv-*hK9z&ejx1m`B5bh^;Bj8ZY#>u#|6CqZHN&*K+dZ&-^qDj*=oGMMl!h2 z-BwR{CBWaJbfo&!z$><@(hZr~ghgclqwZCkdp*`(!i3>k&BZe|%1FV~xnN{Q95K*` zqq&Bqc@62tCt&^Jwe}O5gTt8h(MoJ2TOtNH^445a@N5qE7qZi5Lr#dI+eQgWKdktDwtZW{9iGvbNSMyzrsP8S-3_@3@x@6UQsNPdsj?`fsR0^L|pdO(Qj+PSJ+&VKoA&SRbz*>%f zX{hs~UWTI2_QmF`RHqcfar&MAHKk3rUVr;DX`k++1_Bg$D2rCbLS_&}3jm1R|EzcO zh#kHA`9Bj!evl@j`C2E+xlm%AOwH-z8S0OE_d_U7g1-k#ync;+JEb1>h<^TK!bGLv ztmjr!TiS}51H4o28gq0*A@O9$VK7R4KSOW|qFb@i7B#k2oz*P7OcEa=2L}hr-d2lA z$Ju_%2$Pc0Sd}*Nu84`UJiB09QSP`F?D-?kxMB8)mrX(a1^w~Uusb&r8-Vk{m<#_O zNe3D_+ku+L1hlo62A(vJzL80d4cM3D$hlQwso&Xh+b56neXF!&EV~lanJ8GnPPX*J z8Gh%_DV?^7SuF}j5Bgx((crH) zIpKWF)LE9EmO?afd~#vCZAI^d6U17*j3*%>!HbDE=zJ`I4%$DiBX)vZ><&tDxF!Pj z)fXSi-qu0?ew={bB*>FNY*#5kTF5pfpr@3ql{-PQvqQWfvmE6NVPw-=tY9#o+hU0U z$OuT|QvlZJnl|H_<3}e{^IFwmF8&|J8s{8DblIjgPzb_Z=CtcpuRpOm_U0$qdWVY= zzPF^xgJa{iHP8?bzhdT7@773qWx-`gwJxW;gcFdVKoS`DZlGzf4EwS_9*kP8?Nh=y zVY{5i0$=H`0I6Fk+BvVBKc&TK5{_`z`$&{^ALESL4*`kO$+uJ=3LOsP(*85^;|nvT z+mcl<%!L9FQ1Z@Ni}ETI-F*Fr1)tc*g1lnjP++G+NtwMFyWRYkj+r8+_0}@_>6irZzWy6Uy zEuXC&vOgF1^t|R`jjYFVVSVO|KfDNDu!T4-UQy#fNQw0uh49#V0)m2)rErNqW%+zs zYgdI|r1;y^MG&-aA~&MGhJP0-{rVLKPr$9Ew2^?m#p~+p11`inGPBm-cRBHV)-kny zBmtR<|H?w?$Ds%4JQ1bQmzd@|qhT~?G7?CUh!fg~ z^ShDpB(NAz&Z~516tOOrLSOIZ@}@kI*8SlJqo2ePi8IqnkUG-7$c(p9o7S8vCqA5* zekI#^vN12s+M`dx3&*x|tAr(yxM!QqYrER2sg7*iCHbv?yaeA+_CsaO zZa7eGPN9641byXA%eKzf18+Q}e{B>g=E42{MOmJ<7LNI4UoN5CWog+nCszE95n2Ad7%Xp)IM zO1Ljx18@;UAN>w)5uNe{g0TOk0s_jK;uCaR`=)aj7A2?6Bj33wmQ%F(5X76#mGS7s zHr~9Vt-eAURu8NboW4%k?dIfg$(Sj_Oxpt%5?p#}wzPxqsHYB}-d@D`0; z6lG9tVT~`ux%%9mhj4PM;zRBYF~-tits50JA-br94CWWEM~wpv0$AzOa}0JUVmL&Sv@BwIfeR<8l>8re z1kWq@z2KEFyh8D&IOJFQ=Oays0~Q9c1pm^`QvSZRuFFG8KH~tC?^m;6LpWq|UdkH_ z;lse-;HMI&Jx)0%YQNlHCz7$pzl#AGc0xgIzW~QcaLGqqI0v`F?O=ZO>!ZHq(xxcSH(d?o?s@ z7T!(XqqP^+3!w|?8O1WU@0AjaFYK3;)r~M~ER;w~6Q$5-<`)tdIc+RaP>9pbl=iFP z5nTDgto|voDq}A6Khpc#)@6wS7CY-hl7=iALJ;gqoRz9B_vfDeXFZ5hLGEtm{^1)- zgVS3Dw?wvZ_|b`u@YZQ}r{zm5Pj@AqzkZH%GC8yJVP@vXl4UeVjaot6DlV@f z`XY_pvcFlXu40^a*_7fBXZBf7bfCAFV{kSo&g4vkj6<_m^Tnr1RIcJQyN3Hq*X+kn zt?fx?-qrz!?=TU0Rcl&dV9;aKSTpd2hC_i1N&Mle!JH-0TZ(Z3hp@jCELZ|j@G1-} z!8BkFoH-~XSoXO-VR$wSo&49f!ajx@KadP-<<=wOlfy;is#Ur*m z07ml%+yh*NICC9OF>~K=15lj{_)1a|C=D^CZGV2@k6avdkcilP(6`7)bKa9|QFkI|$w~xZUprY}Tk&;CCK(;WySXD7 zlHjR4TR1c7(Z=1VT8orW*jefS!C=$Ji=nMKZSZ&Ce-m@`e=_$9fHrCdoHFps->}31 zzh~?{DgH|uVCu`3j$Mdb+#w%HVBTzOyLNt?Z({ctG`~g?(VMGj^WwaP;#kWi)zWFP z-7j1HQru0ahVW`yTW4pp>R~Sm`AdSWQTU?9Ss+7?q{=95e}~JXk&~r_6%sEnxSe1Qsb|xSJ06I&w;5VNV1yGjze1FKrAKMO`Q++V>J6~m& z&9zoS44s>6sU!vAtEOzsMn>JDYPl#aIUg;(XyIoA>a~7o(+(G5Bl&=1hpc3U(Y_1QYtrFG=2J1uyljYsgY~%2`?7-$G~B}Y>7*pk&});fuV?sb>y<}r1W z%SL8`#?g`HLZ4}b8>~Dw#O%qB3tcnVP&nuzNmlfUpU45v{4HFEf(rL+X6FUNKM9~fw7a2B|N~%j2ktP zV0Z*FB0&0EWG$K<&eYq5{>j95!om-mowkX5%_M%)hNxY6qge%m`VQ|3d9k|-4alxF zb(0P3|E>=Z5Xu@Yf*?-n0Nxm!2*46?`o46+aDV?tBQz-?7S`_adjh@mcs<8T(efy(-wKmE`PKA| zwZntK`&kF65w>KGO=oT${dz%$f#`{uL{xb3lHp3=qHpmZtfUH_aVrx2*uK_}+#nPI z>sd3EY&N)Am+sfp30vq@wlB;6Wip^(9DNxl94@>t8R*bM0HJRB`SBJIWB9+jk)^%> zD}JhZ1+Y5toSd9sz0HapH=b*|t-OGhrrdOE$Bn@^h*xeD&kGzg6>e?D^`}Yw!Kz@~BUAYAX7*T?U!Hl1W2!a>BkdGiY zKo~sl?w|KCQvquReM_`3aQ2o5J6j-d^9Ud9WN{QD^O!BdDH9Tn9?*h42DSkarUNVF z+XA+u_HJ zqXC~=PT}(fcrrfw*Ih<605$#qV2v4$7ifro^GE%!=*ao=rXnaxvtaHUHd3e?Fa=|n z9gu}-S{zRh*(T5VR5+b85Hn|8yhKYQ*#V{RUSN~@^=r*Zg^&f~SU5PA?Hw--G8uJg zDT8PnP1nuE@?&KJ-j-mrRwWM2Q4};bBCu*#`LZ>fgXE$`wX~AWw6=Y&4?2aJ#(|Xc zfz92iP1R1tv}Zg$=if{f?EMSqFj~y{2T-3s;0BY!t@Qdho4VyohjHU1~G>Cu5aSOi9Z61spihh(xyT z$RAF^*+HcvF7DsTaRm>AV_kvuD>3eWAR)0&A6<`q_Q#&0$iv|}4h|3d-QCRJy_f*{ zLT1+1k`4%+Y3mcdhk2uQga9mHBA^e@6#M1(q$T#t8IAx#%pV$OKLq0V_{p|q3(`Tn z1MywTX-{0d?g7WaJ_-XI(v&{@0iv6UNp3G`B&EpQ!|xDO-l`vgq)6lp;Ncq0mgRIl zpm*6_Kuiptw)e$KA?uRepL!Pu2W0S(o^Bl|-G^&%f8^yP8&I(Vo5F1ww?Mak>XrbQ z0Tc&kfrp-IFt`AOC?*LZ!KG5&Y<2%raRWrgbxW%bWCh9}N0DKPkLl2IpE-fe<*2O_ zudHZ0CKYrRKpC}PNcQS|P`;Qz0|H9c*YD-f>IN+XFc+eo$@ILrc*9Q{S01%PBlK#_ zf_Ou?@CoImC+t{cg3J>qAG9YJ%6i_V#j%qfJ#X|6P@mZFSmE(+vueYmAi*5@aw%X} zb#M=>QoF6~(~;>-a>`oeh^>P;o?tA`8qdPrsq=^G4R}w-V+ZXNUPR?JMNw-b+K02g zHd}Z<1|@Fl*PmqGd;4Q{{ke@T*^a%C`-)dkO;;z9$z)W;s#M+lKGkbcTKT$Q6>>wn z6XuKRq}Xs?$~xj*(>b=r%rV2lh@hP>ru4h_M%IW7%@{Ch4`lYt%) zGVWdggy$uAGOCi}i`@Up`cIPs^~E{AJ{qv{!;-is7IaQysPyxN#ez;vakV_iM^eQu3ED-Hj-)7!aYfpjlTBRxHBmeh5-)HjtY`>b9G zs!4^!>)BE~D{=hc&m{}LS?M_RVm_}O$v5AF&l~>EO3oSgl)rcWYv{WZ6*emi-VLr? zQSxnz1ZOqEYL-$#`ZuZ#^~%E)_G*Uh_sj@5WmY1{@)t+*3&2So1e2C`@PDra;#Ey^9x;6rl=}PHPLKFK9gV$bnwi3 z(xeJ|X}SZZJ{#yVDeAsm()khArT4kL{a0G##_5XE4m+UI%rH=n@1tZGp)9u0akQ$k zGS$vpy%s-ErDC68RI<>_%*>pmz2Z}+64A;evAKIr76gp8UyarF@>LRGIENnUb3MpT zzu7h!j-h+{v8syq#D!?Zq>MftAQcY-z?YB1jAoNIx}orU3O>8F0&&gKrofTfvXSD^NQPOL zn?3z!mBv<+o3Wcr4nh^*=8YY@L}MH%%mtD;kDk=WQez$rRy(YH^hd^)E2=vTJq9WT z$n++#C9L)*(}(zi#)fz2!vw!Vxr}-U%$ofWhyAgI8Qvu%Jf#NfUeTT7ge{f6KAZPx zbOb7<$P_i+)=aoP!qPQq!(hOHfIJJ~KXIrN0JpADw5}0WnZsjMR-d)!-Y;vfJb{WW zme+9>R#gobTm3 zz~zD1q6M`0xcI50Wm7x@4i&58Rixa-lT^2j=k2HN8xkHtqhn)q<>~Bl(@bUL#rB=m z*Q=EZlwMIotk^P=?~pdW%PLT;OjQo72o5t@N~7;rkj>@r9GD7JtW=&`Ju*Ap5_ol+ z`qC|}&BX3RH&#tCYTrCAKWvA;L7A4k)D2zNgHi!3=3Y`(dyshjvVPKQ!Be{|FSfO_ zV)`lb;dBcm6<-`zJ200P2(){?)?yvqkTr8x;7#;!HeA|B6ls$Hs$>?Xp2eR z^WJ^8NHHZ-6%o<8v?R;}{=-ri{0eR>o;{?NoiqD|sdU}b$5`(kEC6w1qK`oL+^0Yx zlR#QU1+dV56Z|ksM(-RuDe zYAHG#AOqh>OFk=KRd^Jt+DsSaJ^Wh#jDBZ4JXA?9e~0;50!C-b@j@@36Zxz)=d$O9 z?pD9$5v3<^C~XSYA*pLqM<2w$JLyF2RNO2+(;NDvIpvZOW}>2IIp`Rz#U@muHGROw zpY|>fyI=yNj!rQ)V?$y>ZM0TqnCiuvra+BT`|3C6>9DJhR-4?_F+)a;MdLjhV6j?N z>yf-Pw(9)%U!cz(e;P^yP8?YOkxV!)P5<+sd9jh42yZMLBF{o4PS%ij~TVO_P$*OJowkWL9n=}zfXy1ToP?rso} z?vm~lq`SMMyStlr<2g^9|NFrYUe^}c_nv!V&6+ism?_{|%Wfw;dxLbXc(>3wn{dxg zma)v26vbp22eLD(CA2_*ssJZ$lYgYmd(^nTWFjRM-ak5WC{Pv<@TDxz5&g!Qx&gGb zQ9(XU>vyQ5poBKmpF&o6e@Yv00lxI|Nd=@kQA57ipn!jY86a@X5`;$cXe{HUX)EFU z#R#~D{HZ1^+kJ~~F4Yfj(rF6ir84bL4*I0Rm2PTr$Hh5rhp&mRWct6>?ke3hQ3usu z$qcT!<~*E!>|;_HRc6J}m|?L!}?$W-g!EgaX%iCY^{xMmFrhg|z= z&{MKSD_w`YF|X?i+~yH;)}RioT%nG|LMx8V>q6^ZpgF{))EKEzgwf+>J|0}JjTVvE z>`w1wK7X8u%Qo`6*a172k7+(*lX*eM{#>zB5ffd1bZF^neg!@6`fyZldwr-H+=}w5 zY_bUKYj(aI7d<=Be!&5D;R=h2z6A>SSDlTcg?fE}g`m=4M{3o$N7mN~i5$x_`Y&9= znG%G75eSHPC3|XLFc-t3lIiqD;$yK{r3%LU_?(a&$pcKB?4BfeooosK*RHFY_$_Hl znL;`da?DxGzlP|&-GBn>=gpex-^uSPvd=E%tU&&v%JGzjTm?E!TJr$PuramL(LSRC z$kF~v$FQoQx^}Bo%Ws~I&MEU#;u>*GdyM@0#M?~AQC!FMPN@r6>%lckS4m+_D^Gw` z=_;iiu|+AxnOdq$xOZGT)0hoy{k~E`>1)K?l=3HUp$7uHt7MbKnl79WP!7TSArWSy zyu#A(n1Fiy17L-(`POc%;%*F=k0gFOml7vpB-T)xrsl&=y>@;2z>+r?ss?!9Y`5+> zZ}P8YVVY;p>r|FfBXbSs4`VCgy*pB!u2)%*5{SJO4u+xI5=Tbh#|c3`GVw*^UO`L) zk@vqt@y$!HF-iwwK5ts|cO##Cv3KmSr*_@uR#ATkqg-}q-9DK7_?e9-*m)~;a#RRS zq4*{AbmEvKzPQQK*m{zSrex9&7yt%}*$=?#UtB;Hiu`u; zmbbzG_YnitKPn*e!t~_1y@#*ux<~pYAXK__G@Yq43qPWLil$@IaEBrW+>mMIGHeBCG3uJRv`GUR2J7G3= zRz-47@oZz=>uL9x8QGnfR)uhy2qIncvHN)ZKr6!3m&i7Jw3I*8)PHp6BOSOFn5vjO z7*3(V4iWtlnN@pAeZ;h6f@B6|k+pn{=hy8~aiqTf|8S()a zEAc_LkV5kD59M^2?8sE?PVV3E{OREh_r>7tKms0ItOmc3kU&nKCw2rqK13YIE?D4Z zk9+LW#1?i>Yy_=1L}xr>4B0D32UGs+$3f2C4u1YR_V}@*#NXudrtz6=il$!iOLi@S zgS7|DMJuuO!taI487jmY0OJU`b-r1{fY@rk_`5du2JiexC{h7{Q{~)^{f!O>QE6#M z=V}Z^ikh1EfY=dIaORY1wb2@|>bnDw5?NzJln$2yJNpuBEXYl=u8J}m?(E3UsHXD? z86e6IIqDWt&{ILhqM(QvZm9wV#ecyrz#UhrG;BapU0Aw_9hk9_NVmRW%99fMxu~mo zx~X)?yO;+}N-=#W!IT4D6i+1L`H_c=tY-^{fCK%Y)ME?_v+UB*O&O_e;Zh<(!^`X~ zR+@X#F9Kt8f(`PY?^)X2f4qa#k<4ET4Nl2i!k=)okN}gKQC1UerQebZ8a}wF&^bo< z5{@9+GXZX~RQbMi>3EPS7krL%LQOp_#uKh-g)RaVCr_5nsP$Fl1I%_AUR zMZZNExY)X1RD9Y;@@;js-S60{{c8>%A2-ChDrnT-nYHUHc{)0E?679U_{T&1&$^RMp<*~+H^4p+yT^<=_B^*QX2vsn0QUq(?b7^<-l zW>h82qjUgwBkk5E-yzmJix2NUcPZJTZ>z!paAW4;(2I?$pqMpo@s$9p9r8yTW=~JB zWY>9F>jI(Z<@vKRv>zif0Ud_$^fY96=Jm&4IUW4$O09W==uWdN`k9-YoikZs1n(hc zP)Nc_r`b!m*Owxkb*mkM18Q$p4i}0B8yw!hv~f1~%h<8u3H%BiTX4K)eUd3^WB%*6 z2h7KZ=yWeXWBpY^@abmd8-s>gViGY>(2A82)5;5Y$l_`NMJ#|+baX^=(`BXpx>Ik{#N`td$d6#^T6f;~&^>U+ z{0jcea6@(gx{%0jPw2ds*0!GfxK;5oh4ie0-{1aBweWO`B`qRUA<0zYhGLtTp{b*m zv3D$VkpF(I?bpdFZ#tb%`}|7eZ~*-HR12mXt3iSF0ulDDf6w~P0T?R6K$}5zj=j+ZjGtEW-EiI1ZWB<(&HV+f}c`NbN zr@Mt~VkfxOv)1g}2C*9Q%eu5?$&=({=j|lp)6MVF>%eeda9CL9 zq%`lVG*q(0FT+8Mp>!Tx9v&XEJ>j>nYDAGA%bVnQw)c7fn)2yI#tR-Ut3_KRK9`K5 zVgMj2nF+FqQo19`*t_y+n4sYK*kWDGMdI^ zOy$4J^^U}Rb7zNyoV*t;oWAPE#!?`a&P)ET%mua_C&Nz0#2I@oLL#6A149Y6wF3s| zT@)G_8vgYfFyeoBcLQnDdYX}l=*Bjy5fZtj6qQqTNT1Vx2ee53hM5hoEHA$$Vw972 zr&xF4NPRgwuykE0GARn3`Hs(b-j+6D{A@Ro9Ks55AlqsgAlysLVAx`++2RT@QiI30 zB|Rad7xJ<2qiS&4%?%NGdqWu;)GV~a<+Z8Sj#IR$*8EVh`<;>zbR$cSGk_cND6!hK<=JcE}Q#D`E9oG9B37 zkBGc{se_lxsW7u2W;T3&F-CEo=KK)f@SJ?TQq2z&XVm|ABJWlxKp#w2M#g)4+ekiR z`J3k@$z;?mD!G(G7^~$xGxu2+s!HoJr*I<7hgZyZ$~>@&XxUS{QZK;N(%ifgz-iAO zR$d4I?MtL(=MBV3s`H36qolvJ03str%*?1JCMEzWNVe65m1kB!-*eJCHweo8 z#j>3cPEK8&E4bK&5=id%U#BvzFpQr5{*s@3v<{sgwJbP@6CRy0j|-pi;p9^7PY0|; z^J!PS^<{Yyv*?(VtM?$glwmf=Rmq##N?fpMise_?dwfLlZs=$7wzEwqKBV$+p-!Jz z2}{(}Z|!z#(H*Si&Lj(j4V&yTZr;owLQkFnOXhC()~9V!${s-J9}DF#`V)<7<&Y!W zM65567TrcM_*y(>(Qh`K)Rj4DoHX>PSP$6)(@(K0Y8dB721-*`>*56q_?cTgr`Jlv`VCg`TbK;BcF{a&*6rzDN2Jrd|T9QRhU8pnl4>u644jfVmj>QeE^x1}0Q|T~TSWY9QTYLUx}$`fjO>H+n-t9-LQ64c(+P z8#;9ex#c}Mx~)8;r{sbi-Y5nT)$8=8nFoy2S?Ro&DWNpq~Lx4|th z;RBkN&`WjM50*l{!@|jF%YwHr%88<8Aj%2jftTQ|?mvE@5cK%|uR{WYS0-q#&Jq`B zri8slRnDWZurQ(-pi|aJ-RkV8`6Y``gx(oIvjQEM!N>X2;^GX|gnfVsZ`;#V{uISm zt-G(FVn-DGWo2!TTAaj8NXidnWXa2om|zGR+?XIRsE)=TKZFrkWx$a?y*z$>?Ltd2 zq56*l=Oh zoh|G{$kIPuFMaJKfBo=(L}qRyMMjj#TVdnk9Z`=}Y`CnNX`pPaw2b#oEaGVvz9ebI zvm*oZ0oL|HzgKv}>9(p9dR7P%#(A}8C|S1iPhOiz{}E^TZ-(m*@I$W1I6i*eX{7}~yH_-KcdJ^6dWZCT4p&H5my+Ae`)!)P9Q-! zXIu6IvO~GX)+?r3jxjJSBD9`1myS$b%&4tUWkW?an;`_`xfl}bo_Nk@M7fmw@aJob zmgdo$+twdQw<_F4v;zlAs2KJ&AZvS?D(RVNj;i{P4)ig?-2h^L%~l`$u~^9n|zRit;7bH8evPV zYOrr3pII%Q&NNJ((RJ^@2QPLkx!h7b`)Z8^{$+&G??Bcim>3uW;mm~b;o%4hN=nI` zM?92%i8yPoTCpHpuG>~H?*5f&mUT}+kcF{yfF)l#gH@88C@=OFNIkwRtY376pzh!I z@;=KpXgr07hj+zN%G+$kDJLr#E&oDcLl@wY?mBnlSHOuK6UTnC~MdT4BB&}aDtDt@W7?bN7zWZM$YQ3b+Gug z7Sq08D1L_GkZK9WwM&ne`;V|s9_sr)%09^r>Xu>Vu{qFPW(9@5rrraKc32s13Ir{7 ztD;*&YiwuL8ZMa<7i=Wg*n`@MeJ{$!d~t>dqwAJe6sn(A_D+*UwY35Q;;nt{w-8O# z5{kB+ZtHnPlD8}|CmTK-a9m}-ncqD+Qv8gHfx=LtEmg<{caF#~go^pEGi?I}$K{Jt zOqT|6L}ns=&I=@IZNh2zJYjQ{D|YFW)}0@;++BChQZW8VBNOZPi(A#lrJo|EoKBWg zG@|cwN@!Yl+-<){J&COx*2wb~g=6a}^Ng~ptDj_lsBEl0{{HfT-EAfTM7@+vCqaZQ`p)AoasHPSm_t#67o2 z|FypKjl^`7SGyES2u}$Y$A+Fv@~DX}^#NkYVkHQH{0sp_T-DASk{aw?yoLRbpC47_ zO(l;%Erbv9IT+GzV8tXOF?O4gkl={t;r&f^yNwTaSfVrnZHm4~IZ$4$O6Iqg``BKW73iHH-_ zPRS+M7iO8?!u{a3?sN)T8&ys3>^G#Rqi0sW>%)dwl8 z4|f4BS5Efb16%nmaud&_#CFH#o?3`1yVj6SJn&sX$PwlSX zJhWUxj~*l+99k0}c@X}7*4&wsns3EDdhxVtnxBp)Fm*C*G(OTd?Hz!AP-5L#8kbq) zey;Y02@qipUhPY>^HV&_6Q*Ud+$Co3xI%!!^S!G#cStngWYX6W56p@b5 z!Bbd@(V^2N-P^B98|zyGHfNQ5u2v`$<)N`FlVh~e!QqGP+PLPjhGR{%AGGyUT0yP1 zQd~2`06jc_6n@9W?4sW4CKcF0RKaR{@$m(7kac01sns2qKqbbP(#~J)%ZIVQ)wY%y z;|nOcATSm$xH+%U+kWeC25C{ZEpi`S*%52fQf;xxTcwdBmlHD{pLCpU`^^!J9;xPe zNDfp$PZETRj?UoalaQ#G>o4v9vXNWjSmv6-7M-RJbP`Pe(Rr`+1pzv}M&CIMb+_TG zSs~-GxOX^C+;;4C(s#I#w+;3-X4MB|GIa)v`;5{jb9YP%2RsiFIUBtPS${fujg)zTN2w<))_kGFez2Yw z<=m1l{Y`%C+gtn$R~Xxx9pm;)BC3zxmwfiF9JI?1FBN3wND@V*gK{4*SEr|c2Uog! zrt*xA)YX?U#k~U~=7S7=gF*-{jRXSo12@f0Qv$_YW?bTw*}}mMMNBV87>y| zzUF}LDU61pnx#LD!N*g)Zz*r+bxOnCa4NR?`7Z4VJqHfX7}JH2;nwpPi+xqdJO{qn zjv_AV`z@;bC1Ub!^}1ju2%74^4yAEnt~dQ|Gn;r=F5jp_YPLSs7<9SVh6M*OoUPPM zf;tT8+*E+r&h;Xv6GMoZiYlBS4DM3TB&AZRe_--R_9CrTGp)44&#tnZ z5p)U9u`bfX;T3#&K%qWI(Htw1nQVk0i)Pd)P}bI*rB$d@mz|!T78s)ZA!i!SKW#}; z^xqSP5YVA`T)_s}Ocko5W**E}PEP(__Mgme5t5&D0uA{l!Amc$ev2C80=+w28P09Y z`gV;mMeqV1ftdtJr?nk|>u22=D^7Ikc&ZunYMRRoYC~k|v;L^P=Hp`>o}JE>+6)K@ zFSjyZV3B>w{q!Ab=w_!6gyf%0-S1Aw+wu#GC;mDnT2y`D1p& zVU-4OF3a9Jj!OZR2W7mIK8m)%!)z*Gxu#>X>8o=NCwipn`{rlDnmNyI6ss^@mi5vi zVwqPfckxSHZs}>3)`ci>KjYs#X@8}Wj@c%F+--FHcEP6x0wbosg)Nvid!@DUo4UfefYRVh$j4NpT@hfmuL*vhb=xMA~|Lz(L*9pnPtnt6Gzf0 zlwyD11Q3|B^IPI@Y=|I^&W?rmy3q_${WeO-#tU4W?#L^ALPFKUMx9}7IWUf%7Ggx!dl|j8#zZr-!a0#IYkEve%5s%T$MhuIv93T z_4H}L)$s&0L2}tz`GxdKTk{80bu>NMQZ#ZD$wgHm#hmf_beqb!MBuOH#f0LhNRyj& zmG8^7;R+@nXJON%$SsbLnAeJpkQ9%)M{bro-A<0WmWu=n%wxWFm1K-K7N2veC3vBUu^Dj37yxQsv zf9QiNA(n;x>0%TV6e27lNN8w1pp|$F08k<# zFg&YbiXp&5*gjxa!DCE{K~|REaC`fp^MyCqu6Zc?sJPOn-6ol5oTx1d=wg$8$kZx< zd6u>bHGEEYeEgwtjL09d9?qWQF3LFhE`d=QCOdr4#L7?!%4bPCrdHi6;4~L53!W5y zK?e9kcvAnvkq*vtaNS74I%G2_ON>@vYfzeKqC`5a&Nm>XZ(>%_l7Auc;jDdtJW9bt zFp3eA_*{~0!>zrndL-W6H zj->&KQ|S=*Tu(8})nM1cP_n*?^Van9zl@KVJ{86tU?cis`UCU-M9c0H5^w;V6Xfmf zz2@)JBLNxQ_6D@;Ipg#u7ROOD85{Q2rSZAuHXI-R*o)6zCkizMwtMd?7`c`gh2UL6 z2);$N*imm*{Bmmr8+~hAb7Yrd6zqA#7g5pPcOD@@L7m6cT=vT0a=E5Ju^v^p&;a*c z;Vv)0$W6@wDSJvilfwfBHfwo8Ryrw3R=3(x33E+tyY74@WwdnUFXq95|eg<)tSlXnk?lf%gw3$VsnPke92N4D}F zgCK!)tLoC1d^@9tMf71OUhg5DXHH8s)ZmB8cQ3Q7 zY&Z;|e07696oQ+t6tWz;>SfEp#z}Ouejmw18&Hs;irt0xLlw_xPfDTJ1?T4B=|`Vh z<@U&B`9DU`0Xd&VN$)(ek@H0>bdCUL16h}ynfZUN5gSRSQZWh!1{oHSvzyxn0PXbx zOM|3DV5}U|#qf#gr(l12?mfwn&-k51;B8Gx9y#L8Cx$_x_mHBOJPXa4$KO+^Z?#fG zDwofr3?nUPyCcA&RMis`WI2nghrkIJi3>`KV=$B669>ozup8pV-VC=*{mLioTB$Sj zIX=`?)%ZfV+p~-H)sbIu*u$ z|0iyT1GZ+r{_5Vfd6r43rn%$0`%Y&zk`)8Sj9H0==MNo+Y3P#3v$-3hF6FCxC#Fv z=~kpfl3|U_XSKzTmMgTNa+cM$D|@UHlG(X3CBg(p5eA21t?N&S$l$sMFQy*|gDxNo z<>*<5qstJFC;|VTfnhi51AU5b8=+}^Bb@d=BML>m5Phd!-N)J=c>jKrf4`QiI6xX_ zW@cVpd3k07x*S2Tz;!}Q0rubjfPU^@ZdAi*$-#obyhz6%1Om~|1q$2)$uE5u3_<}> z2f&#RNLdHz&lj)d#vQlAC!Mj9HwkC;3&*K|Sm!$ieVsL4`Zegok^GL@K+7-@Qf+W8 zrFYuQ(-|U1<0Q9-tG#o-iovIqnn=;|qK;m{g+H- z$d;F9H==MNHu}{C0LvwH4}Dm?T?IONBELU<`g9Ahbj%j>|B1Hr%&;0Q?${h09L@k$ z&L8X_e=7L3jhPDg!O-|VU@SsdLku){9Fs+d;9Zh?|NCjp^j*y>DXImo9srz|uhiVL9WVL^U?>r+OW}V62X?wf6!`-BQC$;0=pIsAR_ewe?M#U+Z(c)&9nR38HI}joD zbkEG$H5g$@1M8-zU-;megieD@LEw_{&{xzJg25#V$5m;VEzCGXr;Xw_i_4m*54}nF zK2k~~4VmwXYzTTflx(0?4Kdnu?RQJhefd+g0WR3%<0HM%5LP$=56<)RbB{-5s+a3r zwFq`i&WV1v!xg*DPoyDIv3h(5d;7MFK)yxC2S|U&d8X(+p{M9knVVWcz0%7CAEa_n zmq<{@Jz_IzL~X`eCo*q`age!9QPwm?=cqMz-Xa(L*KKZZH)7w?niHX=tWlZK7J;R2 za}+aXffq0>To}4a&8;^0wFQ=s$yqFyUHfiOD-pZ&qQ{C#ziKF^rmX_@$7k2rt{K(H zclhL6#wa&P!;liaFhi7`cDhR(kjrSwajgtita-x`e`5PQ*Ku!(HCDhw>fVTu%qb}{ zDVikwuLB18Jd?L6GqeCQr$fbgiuK7zl$ZbYu}ITDB>uMQwpHS`9S%TB`1G|ct6Z)n ztfND?>-DnA8I((df)xo_GOCU!`cBYg$TGo3gSDloQ=7vixD6Ldw7Iio*OuL;qRyq_Yrs;a`bYm(D2zlzC3`k6;I{!>pCwK5|OzR6BzvIv*WYK++{>67b(jr16q62S6 zDhXP0&kW<9RE8`U@z+G^lq8v?s+wQXkeDj$U>#ojKv0eh#j=EfC=?x~u9UNQbpN|5 zd`<=P?nwKd3i|D7*XEUTY=UG0c-VUI2U-H*RJ=mOwdU43JK4t$yDn8Mi^TL_#75qA z9Yhe!xNvd|e>;4>|J-~JiPd;@tM-W>pU(hqaZ{9m{!wWaprsrus+;}e9DcI-0Pr0a zP<$1XmUh4HqyolA+hK?Z3EP^{$&Y|c6YClhNa5Kvz+0RWzl@pJ9r1a%`ui=zbB@nb z`E{MA1umJwdx2pgqIj3=(dlXR;`npgDefT=(J9iPawE$N;+ayI>qTF6OK$w=2xsH% zidl#JSlrajNi&z29*VC#*J4GSuJuB!=bJXoAIoEtu}EAMm-b3C?l~r}Pv-V|ZmQtO zDK%-*u`19hUTh***^P%cFAtIb{SIa6su3nvHccE5R8K8ClfDg+otIFK~v>lo|Lb2&&*^ z=|R|4nb0?Fk6G@_*ab=lL9%Jz{Nvscpd+xC)Yahkx$Rb6xT) zFj~VF_B4zLe38Hl3dB#Us5@4VSqeV?F&QC+;p{ShO^SpgA|^-ub@?Z6?VBM(wS|>ip{DxJmYOj@r14WgQ_vs%ibBfrT1#Gc|*b5azh0z>A%y<+!}M0CxH}Z z98ls3xpm9aX=4&wk9c6xb#6$>GS)CGNrY#%adk ztg#)QoOOq|R>1!F$P2LbUI@Opo(u27S=i*za` z;!uBo?f=G~pAbHonS_sk3g~gPEI&Zf22Hf%Uy|ehDbD^pJc}~dcVz-2RT+!WaKM zJJM~DKow^alR_?U-yl6%;uLrP>^GyNfuZkG?-9Z^gDp05x(G=X86enyqEe3Mp|&>O zL-ctuEAD@m^?&HRcO9<43HpXX#K@_#@AT&DB0QMWQ(ygZURfD!J_5E70BCYScxA4q zFok^UUs+kfGV55BX?OPmW>T7urC3Bm62&Ibck?>3QBX62hvGzK=+)J(w6oRxdKU(C zdCY>}=M9JR=gKGOp|b7_bchxV*w~744&~?ba5te~;e^DpaAmq$d^Dnp2`|3G1kW8{~E4jC-@G71$BJptx;&te0mkakB{$uu|!tQOyLjAfAa`-zw;O_ zK!aER>K{MR9Yakdtef+GL((5n^D655<^iW4CSI@CUwt^hkP zA-{=#CzG`Ev46)8z!N&e^l00oZ>K=IH^)nUd+x|qnjV0r;4^W7K0qK4oY>UoT*#Cm za??U1i^TQ2wjRkj-n3OH_<=s5cU42P5T&!t&_?nB3==sXkUp|lmmkz<60MADha>%t z`>i61;pe?157G&Zzgj8-YJdRP@W|GZvkbk`@uv-x&^U`!(jHs0Xig7sx!<9ui@Byc zMVZ&%@hKSOb8FT^T9TR$bY$Nfs2P8G{|!M`JA}vBiR4pPA+~YSrceL5vjW?)a}MBk z!Mxf#nwf$FVVj>ewFrQf<1a8NB=i0lja0u%e?l*$pTqz zK)Li*x}eiaR0z45uc3>y02;J=w#}o{xliSUPBBNkp@0#CK-KZ zAdFvfD$5prNpZyl^7QFY@+K&|w~ zs{*)^g{7sXqR@yVm($K=Q>pI7}E9Lc(; zzPYynZ~d$t8h3!O)x)UH)4%U~we}IFLh-loRwGU3r05uck66&LaP+}5>-N5O|*PiT3jWA;)1+`NBMd!4X0fMX2%Y3&pML)IQn zS}_-C$Vo}p04A)T6uI_1p?D&@E%Ej7(!pi+8*$>Tzdj(y$LGW8S?_xi9v=Kf`e?dm zHvmFBTMwfnCTwd@_ytkOeI@yN-75rj{L}R93xXisdzL%M(sP;R2Cg}G%#w&0_Sokg zF_Nn^DqO~`-C0%4t;8DshNq8 zOO0=GC?pN#tqs+{t~0ER^qCyWPuvdDCa}=tZN7h#L^;LABre7cu*GnAihgdv`0=}6 zrfM4Q+(6L51%2Qq%KO_lWM4l&0P6|_ghpR?j*ETyf<~|yPoo|PEL%qcBqbQNyq+8^ zR$7{aCejLz!w7ke=WjA;QhF1#6bofNYPu_X2vF(i5Uv}Zyg`GeH|;~9n8 zmSCr5XW^|9ydF2+Zw#}obOoc3R2cNXO9h5b=U#@bJ@p-67QJ6yFC*1Zwqm=&KaGb= zE+1g%5&q=`7@nuZBJvLk^1~%+YBqpZaSqEOj%(`0q^+lvJ0K;aM_Y@~${P`KC<;d|1*82kO2n%7lV&UcXVChkP zb=vRh-NpmEcLon*H~!P^95pWbcU zlVql2P`~nq{FI&!&z6x6Pj&Krk0i@N1$`BlHR`gMN;QP0dFt-3g}lR>HIt9q-_t4ah=4 zKelL-{ax!YNr8LAQu)c_ueK(Z0i=DcdY0>@!)-)!_)I^?SS($vix-Z8UuCw{(Tt&U z%`QdHI^%lSjgGKJMV#6a8Vi0~_HXHIUFzZ;vhcurmud%*`ILQ|c;0i@+e%Qt5|7tH zjKG#)8lRvbgqHIfl!Af+%!%&LptkNX%s&V%z?Kg0932@E5f=|xy^^NByt?WJs&q`5 zaFb}5e}1thNPxTRpP%80lR&HaPJpIVqv0I@(fB&$wsTQUq`p>`IuhPS)L&c>s(Fxa z-v+^?6iUA_X0U0<>M&aQ=3D*5KDV5Lt>j-d+`g|60rNAN@95VS^z5C69gb&FlrMSW z_10hxEPK*pmYS@0i5Ua4I7>{nuT1WMh}1UmqE|V%&l5^n+W-aHw`Tu==pb8%O);bJ zwScgPn5di?WrYtrdcRA!!b{8Qw7O#}@kHJhAG8!UUzZ0);cAGY6J(&9DX=;l5y!~F z=myWbKklipPLNonx#?FE7|u(-*8yYc$`zP%o*lp>oh1KHd;$4@kpQd4Y8e(&kkTOX z#8&JVv$!Bi*<0vFZU#X7PRH+M%!Q?8tU64BGuX_PA=p|@;iVY~29`Z>x`C64$&t}j~k{h2hFuN@yBcn52;qB(C-r~-2n+qwerKL4y!u<2$_*W2( zAP)Oq|9=+W>Sq6UU*+6=GY{?^VBxJkfDJ2UybbJdHbwYP1T+YyZyM!|NQVLWEWg9r z`|UA!O{UOX3ti2aTe!YCK#pQ`F;d)9Y+Z2rVs)nHDTF6Jw9r@kW|4~*shu&2=**>{ z@DWZHC@zT9Bw=IbdcMa<-qR@!iwsHU6ERo3kd^U(lSIeRCXqWxwe}#O$U&u%jW{Dv zQlbbfUm}OGto7_o9+iKdH{^|^@G`|*jusnMqnz!q-tU((s#L9lNB~o>t8L1dC`kKhaw(7o0;8_?3!c1^kWSJRAa3t%;RNhTH^if6M9UF@l zPX~06DJYQTTw)f=3^!mBJwLr0Y-5kDL&9zj z*Z=I(f#6Xb!#IW3MKXV=(Jy@dm~>x0mYV`oqh5Arprt(o*#pu?T!i`}-GJlfaUdm} z95-D^1=-{CxYn#@C%K6_?`8$znF+@IeWGADB&Xw|?_sq-&sXJMu%nQ#Yu5m`CkzP* z`7@&`7X>h9D?uF~ueQqhY>l2p8MYjgsed@1C*m1UC<|UKrlX5D*Tbo7n4mj7KpG5_>xCGZ8rJh| zc``^7M71M;ZrZRs%(l*tM!^_G$1Xg!I@~%%)QF4OJTHLm3x_|>r$lba1-C9}02TW` zJ&9dua9!jwJ|BrNcSN^S`49S|p)VpY$#>TPvxd$M@|D^d8y^q%5zm1$O~wI`#khSs z@%r{UTdgHu-0OU}3+(h8*=L{vBHn)+9g8V%Zp^v_X%v-|;6h(f?4~9>K(^($l-*SH zHLtNj@c~P-T<7iD?Trl^`49qNt8JyQCNVLUtg8?u!S$HrN+C3FKcjHK*-Y`a71(}9 znIFcLLO$UW>LG#O zH<0cmQS}=X=bTB(ACG%NCn`wVB9NSKp;{WFaY;_?U!HE@pCk^ z0*r0m@8)VuQGfEU0W-6!M@JFH=`PS8l}fx55@x;kh133IkoVu)yq`G<02KfJ;)ATV zkj_ug?*9JkJ~|X!+%WQV=Z?~#QtJ7j8>ZibxvZaW?ljIuX5b47mm^cB++o7jey^l* zd_T>Zv{W&fv};4wl_vqsMCkg+>oh%~;GmFLP=17y_1Yw{pc>(CQ7b1+NGlTSwnRK3 zqIKqb5oTu@YOz{Wtw+XY{1}Nn*&{GJ(UDD=sD(^Gu#8n496L3>t)py7CU=->5I}`1 zZp+Oq_-kq;@z;P}Uxd(DY5hjo-XuK>wZGCl_HUWMzi`ztIWL4Dr1yePTjnsZ*b(>l zgIKtXP?Omwy1E5;UUlOv(`&btUfIVh!>2fGNVEIW0y|t_PQuTSvbya^xNr7xzO9|S zpU%{BLg#i*nBBDvPQ-`^j%s1LN6Qf5RNt5Vx=SIIYHhUEHOJJ|SDE$l)k;+f1llL zzPd0DcGxt%F+T_T(t7jBKdG0ox=?`}$K3pMuYI(=jKCjKqw~&)ZV}^IU#PZFWK7#F16pa%!d*vtw-p1Fx9FU*BOCM1QY7^%A66?m?+1z+c4uo@CdDrF7LFKM*!oc%zHE&yg9k62< z4Upvx`<&N*uQN!Vt*x?lZ|`5`bGxs`OjX(!6JCZ12^U@7cBcNK7AF4H4k3tH<+{h1 zJz12tJE1#`kn_$Pp46W{a@YBwpH>m zJA`rrGLp|m8m6BLHi;3nw4!xpnPfLA)0`RTkVzMtTS{|>dRk`TK$vF6ML(-y)h{ym zW($LMib|Z zKryM+b;uOwEEcD)e@L}WX~}~j^7b=+$_^*)ZCFMu zxzu+?v?14S;+dtTk~XDH027s&g#}jbb@5vJ8z012xnMN$E_ zz;;~zF_R1efYvT8qGxPginE}|u zmrH6qh1Sa3+-v*wcS}T&ro?m;7)tA6R(QmgnvlxfwRY!9PZ>8r_MFI| zzq+$CdoYS?^)u@QJ)gkx-KvNdFjUY3q#2M2-mD)d{y5k+LXt7NKhT-Qmt6@z*`0#S z<(P=%Z=oorTSWPuI}%@H#Iq7|KSswN?LbNMIi5%;fI2u5wPs0rcjwXs-*Os8c6 z*fI<3InhesTJao8l`RzV*3&_&sPKT=4X0AMRs@h5!%H^o_$d}E=>pjp4~nqE-!uRp zB7s$44(weM;Umx=e%BFAr&UG5kcxIyq3m(^4#kp6xmB_V7ro5g73_#0E;8Oaxb~6$9oEfLn z-CNtZkH%lFYa4C03o4DY4U?~}F)1r)V=1WA!!wT$-LZFlvz(at5*p3dHC2uvybp1j(@+*93TYVL<+ z?pSii7mF0ly0_Awup$UEq*9-HX*Y)@3@7gcyE+VbUY0vC?Cg74ld*{?02qc(ZwT$y z)js>~5Ob{Q4zDV_>>V6RIu|~EH!N#L5>NN1BM-(CG&e0Z-`#%xL$ox*S+jKe(kjQh zYOOATF!K_E;-PTRnKkUz}Dp;^Edm$;{21j3Wm0A4JK5&-`N2ZL2w(%jD~On0|Ovb!E~db zsgS((SMxFlKUbz}#t=aKW0qxv^a_Y5O}(PtT&r?l#ikW+<$9IAs?vV?3MCMp@=iB4 zhlaW{YtdPVYc4^wmqlN_rphUCV)@NBH^a4&>J9#Njal!S=K7$6DsF<2!`xLUGgwxi zM|^;61-=pgvcqn7bHT`LD-12=I3X|5WGRnSO4f zq$C8SLrO|Qx;rGKySqWUkuK>JB$bd91O#aVY3Y9Vpy!;=_j-T-a2*d1GyBD*pd(6bz={0nMBcC^Sp*QHKOHs)ccJle9?(aT7x&uNl_P2Ww zz-S)6wPm{iQiKv_X+&!vvk>ezWe~)eIs0o5P0ZH0bVQam>b)AMq~H&r^gTuVF`9iR z?%7bYYKbDr!GgrlveM(yxnIsqO!@p(Vu38Cfo?K%=n)Nf9@hTmP@{MEyrS8P4;g-Z z_~mg>H&trN$n$pM%u3gjV=S?S9zo{?UczViYRJ3j^s{Y#t%H$A@5?y{e)u>w=XPaC zf{8)~eT9``HdEPYEB?eNR<>B{4Jz4Ntl9+ZuX4~^No>76A8PMrGi)~BQzFt>DP=Kz zRlz5dp*i@_BuqEmt*n8SF^!m=pO#3Ng(fxxyP~^6hq8+-$So{VeRr)@a&r;mu;YGu zDoRX5YFKy+!Pd|4?s61{?!CLAkw@)&;suxdm8hhNs%DYJD~3+|1!nqO-n7Q?gPLX6 zUW5!E9}1$UtQ(#DZk4Ga-V|e)7OWVUUQXgcO$?~B4|b?xHZk+z$VJ(mSvLuhPj5T< zEb4g=>1DIc9FMtY^=p!nj8L!m$HJfQ(u}_K`{I@*YGJ|n$bK;zRKZak9UX~FTyOPZ z;(Q;=-EE;Nm;E=Dfx!f^?>Hg?EO|@IAO{@yV-Sv)L3Cy7DoBE%Od=8q^A=(nMj?>U z{4F>Efwz3?aD?xv>K26?=w5LJqmRE~V_ZZ`95K6XJQlhUY9M3`tcv-?u(4<7SgZEo zg&++FJ9dH;7nW&<6GHj~YGO{EuGhAJrlG@6st2xCP4%_+`rp&Lbk+59+7o{TEVkXo zO_d`xG*yq$FMdtN-~Doxz9D6sOa2}I#M_C%2Y~;;he7qomqvezF{0wInI*IbFo0nQKWw?Rf3*U`OxwZQF65Hl* zn9GoIcax!JyIOX}!BnYhxQYH6=&!X zAjBk}s?Z$k8^N5{UO@zI;8gUfRp%59izQAXTI9_dGnR_q=a+3#O|uv``Op>^^ocB02MLs)hc>+*LBKQmm$Z_r#7X9`M`ObaBe2sVUS33Hj{Cq1}w> zMm3g|^Do!$tly2{BbfC)_5&(B`lTiVo$#HaA@87C)C1lhLTC%_pJ3T2a!4{BuGfN@%ghvzv_rgyPyU?3GH=B(FjGs9-n_6%O3)*J1A`qH#C z#N2O*lrqFyeMYx2++WKC3bNhkkW)dUPD0B8>UxH`qOps*^Gs z!N8#(stM{w-glx3N`A;z)FYohCgV9~kPX5k9P(TAKeYU$*_s`r?oOlInW$jBI+S(X z!CQ3tDH0-RNMBu7b^W&YdwzJUJduI(n;?+U~1W^Q8Nins%5gror3Z&R0T_!91 zq}Xq&sz#W`FuWl-Vd>)=&OP64TOtJ!d zw(S)^zA4t5L0wuFm90)*c%fQSr*idFTI6iM*v~MmGcB1pzGI!@>{*oez(T)_2_l-V zjdYo){;_R)jsawih&HvV%%YoGb8`8$A2$s{tcXr)co5ahG3C-gk)_SkCz;jc*%X~M zPX#y#@LtS5{HgH1Aa@RHep$fdUdt}V#Db*VUGLxa^K@oO2YdB6UTk_wyU&iEoD+;+VSj+2KxtQ9acPYcYipQ_4gg!;~TVtxw9I`K& zPoX}koolh0BD_4Xx0guH_e);1kaOtHcA3qh@9acQ zOl9>inK%t>##N@Yy1eg5s=80vE6AN~^h~EjutXKo(jmz^Ryu)kGhZu;w4y&QC z(cp|_pfm3mw63^S)dIplR6g?9z27ymyHQuumGlu zTMCq>dAs`O$wL6W{JaAZEX2H_h@`-jG`VvH)lbL*17uTHvQW7~g64Yv=~7jUt(Z3!Q)(nxc9@V!3r z#e&l`Pm|BgmPx7UdA`6&NW-K-E5XCBty{dGKB#H4pJ=|T-%>1eYNeK8QO_UW)1^K6 zwTy=K)n=z#cjg=|Q&VGRGCYcyZEc89$RdD0Y+~p|V7GkI!Y;?&8?o0qGYMRTdtBa>dzrx@Taok6RSJvjyNx`q_xCJz{ znmUl3z;Uv?7;S|5ZJ%ySRXCHd`6)gkL^Q< zuou-+PUAY2`MKh;MT_q^#ws)}`e`j|dls`<()p&F6i08`AM71Vk5SVI&#oK`mGL=~ zJDvmws--R&oURKm`5wNeltuAOTYDII7Tn2&jQRG3msfj@UW-G1*gcu)vo!Y>RB0jZ zJ<8!&24Sh|p!09321L5p78VvJ=Z71yk4iw;1T^k+F=~!w@zc6L;^5_jb59!%a}nHh z9-uyS3JRjUIsMRk{B1^t=bEtXcxSAJ&eX0gkMIkYMa}H-r8g4nQ zNK^Z#52b0ZNl-41$e0}C=Z5cjIYtIOljBVvWgv4uc~SXb=-2eYn~1~bNkR_codlj&QP+nh zBB;nH+Ev#y>7Z%BJ+8po zaGi1y-8y4HlwuLCGJeJ4kTZgi(Gu#cC*sm>rGHUQMYLJO-cKYz2$u^M=TAx7ulPJ;(uMKn3HiyeK?%PGhiGY;QRo88SJ0|HFf!85?z@kE z>=m+%`N1aez5HL4EJ_EjEa-1s=7A}12o?+ooNkCDko)H2d za~|gZ6F1+_bJK&p6L-Tl+1PW8w2r9Lh^g{LWO7rd%-jT$jdZ2jj!^=NxC!2~k4@s= zcaFExoh&cgLI(FZ2lgcm_gki;$$)`eLiYlnuNH zTrB4JW=%&LZA8b{VUnTuDj}EnFRJN|V6t7$9zC3zCrHu=seZfIL^SU~TYGuGZK&Vp zs)TDMrHsyT?9RSTTBl8)&$%-%#JD8_44OaE<`JG4jKLw}v4nFML6xvNw)X%<8=T5A zL>W_1>UX?M_V#$=#@8bEa}M5>P#q)2b_?rwrTrSj86jRVGnRhxunv49>e%zB4M`J_ z)X#Yw^xghV?fkUm>5V1ljtYl8-`hA>a%b`8oWKX}_$iN^CqYE|z~rVcToOe>+n3Xf|StgI?~)7w0~r+-&1d-`(QP;v`H zn-yVVd3gm+u!pvA5YfnO+y~oBZZbZFw$>uD|MmSGRB$X`?=&NcE|5vfbeNZ%sk3YHyU8k*c{$VvxS9>=quyJtkR-tp+5BF zx>a@tGVHO6WS?GGB6_Lljdp8Wu!hB3n+L~#?6E6ixF*2Gw>LjAzG_`h>RMcU{Avta zJhVG!CWCJ#h2NFDwP}n(Cdupc<=#D;(z8Moa*W*ai{ov4K?<_R&N6qod5?z%2XS@T zoL+PMmnZAmqwf-u`^|wtS?hrD4`#*@oI*u9ytNvlP%?()@9*l!DNxiw=Eo3*gM*2S z93mYo_I`d|Vtp*rurg7ji<{W~p$boWo+ZIzOBB3#ax zYQ7JAm^&uQZZ$&WJ;s%|AD+7~>u-PDsoK2o5`7)eaDt{50$S;|zN;A0$zKM^>jaym zX)DUXF0^-ApDX;hSEv`{Kl;8yW;n64vAn>?-Qr2s2lMA ze0TknLlERrPY0EJA^XCtZ@5Yi>Tn)fUfpetN;KqU@phpufUrr-KaYh)@pC3I)?;b^`q1$Rt9(l>?-e@ zYUZLB)X-}LN!Z}YGd$k}G1Z<*?@zy;A9IJ4jT6+VUz6M5Fkc)k+j>3FYgL#4WO;#r z4Hx}A^1^07)%xzPy|Haq2U$2W3ZO4;nvNdQ6YnC91X;}u!`3Db*j;b$;+|BhvvJy+X_IGZFz=>q z+5u5>*Ig3hehTmB7Dh^X8F#Ma$aZ2}Sy3*&G_6dDZ4|55*mu03g))@Zuh3O~YFdg~ zllF0(>5wzwtH|O#q2_u#v952$w%M197SG&fd4G0H(7ZoQW24?DF1|fsq;OMVj9c?$ z2e+?K`pedv`kgVUwMUuBp@WEa?(w;~alfpu8vQ;g@UDQOdMR^r2C0~PJo5L)IsT0| zBkDj;S21^x0=y)acZJ>4iW_Za0&l**zO4(tdmjQwrSRJp!T%I3TXud|rxBqzOoIr& zC5R=kaD3f$+-|1msWdo_G)(rV4TW6_doTH%1dviVjG=%WRUNa<03q{ic=-bid;xD~ zx%Cvb_lE-SSG2E;Du)JA8JM00H14wWet*S*j{(|!&5VNchS+*&<&YrN*&ht1`tMK3 zB-t~>md%b3ubP;C)3`f-C&Yc#jIjKnxD`@KbQYR{B(oVq1L@`PLVJ3MqVnSyCRl>7R`n+w?DE4=P8Lo*j|M! zI?HgF7Akhf9+*pEGL|FMGjL|wQ)f0D9E0&bbgo~t_n#sF!;mhJtcLE&kn-9Q#44t9 zA^|Q5kje>7{-Sn!3w|0v+TC_{mar}gcpt6s8b-vFMA=_uBbP5-Q!pO!zJ+0gY@WC8 zcH-}#+CCZDd+fojeXArW@o{$FtlprVF*@Nc9RbVk+HrlpsYlGF>!6AMEVLIBxwN2( zMeB5l=4S=VrgiehL^*3xEJ@DO$pR47Pbz{>sT-0~UHmoh}xIPPY2+d1A*Z$UBjkWg7r+Cd+ zL8DsyRkuj8-UZvQxW>a-m{=Wl*#4|3s0S6Q-_(w&NCYQ888z|uc)X< zudJ+0&uz)ajX1OA7o1IKL1NITIAri->au-i`g5a@f@@6OuF;(DgBi{XSF%cZuCq}( zdwwIjIc-&Dfs}>DrB#N!H=g3EbQLALGlv2T)6?GL4T>)}e4}<5Qa{%$9SRJ*5pvR! zLTVpOr#f?u@^IqxQvC?CdFQ40#@Ws$8;(5fz`Alc- zhB0chU`jnr9lA32la;G<6k$w zV{Mapl*_(GpDGWlcjklIE~am>^Iq`;t>aBQqH^AJfmaBdQ3NDt9t?_gtH13u;$e21{#9@6j$)(~Vx?MDNpphP`#V+c=;&1Dk zbG2lAK6=u3sFuEqXoed}B==y>$G#);Obj>aUM=E5ZS5~uhPE@9>2B!2P{sg^c%#ppjaWqk0_6+Q*NKR21x>cpEt3ht$|enMFX zf$TqZDUw%p#f(1b8;bFIrn5BQT;3ayOtSX@Fvdch@!Hz@C} zw(T9ie_q5x!mL<(QAGPOXG9Q0kuop(3B-uwN;aPr&b!JLvm z+#})soAB0ONSxLF50RTWm)F#?;LkCvK?;ME{M@*NOGw-FH zWIlS)BY`ax3^LK(F+95l)1Y^`mRp>obea4v9h0hF2)oOwoxpWb`jfsY3LUIPY$64t2Xpg>9vk16l%S1 zTaj$?{fB#~cSU?51fSIJPYv9oPhH&#dthkniGMffsBSl6FFTDeBo9FZS|h9Xq*5N4 z3+qe0w0Ryae%08y@yd7PSA)^Gd4nIaN@eBsao#IMimP-|y9Pavn5=`OD&keVe44;I zdQfrj@M-rsCg-MxGB?QuDKjq9JH6J#pzH&pBrR*l&H@H#_ZU0E8sq)Eq5Iq_BVX&n zYE7TRLM`u_8^;{EzhC{r!EfDW!uEle1_={oSe}`%cvS2(cwfFmHe9_(oHc47pZQBz z;3-bh`Bm#0E?F3PRmrmyh8$a)4n~s;RLq6 zJb&xV=Ap<_INeR!&T!?iBJY0|L^9f2QVaA7psP_d1mi7wBsraU*fp=b+%fv_6jQN> z`yuLS52t&G14o3TZRj44t=1SRCTNWM>Y_SUFIt-rN%+Qgc+h&#s*`*V)|RueD^0a# zDlYhTAi?Lb?0>X9j@wix(S-N*T(vvA$L(c9gjn$NY@{dg*sDaW5@(;Y`FfrwH8<2- z31J&RxoC;(28<5}x;)*QI=B&ra1k%yMf}EDr5f@bNWVRtYKx=|xrYSU#8sJjrE%gS zb8XE|kMB9#j!K3n>ln`Vyv$nEU^s}TYQh&uzCT+>IU+q{cnlS(#ktF5R;0@@$=kfL zOw`IqxKSFfYG$%Dhn+w|$)Ary*~ zm%MON;aRyMZre9jU9p6YMEaKUnVRh?{r5mX$Wd2kbLMW+H`2(%-^|>()8h3-7?*zD zpv!QGx!L61G0T^>N3;(=5)*HZMUZ0;jsaS6GzuFU8k#+jwDAwg!ovU^1HYfX1^u@z z8^xKS>n;Dsz`)3aJk|En4grP<3Nh${2S}zqf2Lt#DzgYIee}m)ISlV|5k!!T)5ZuG zYWU>|%aXhoh!l|TpyyPfMilqd-D7Ieu9f+d@i<8P-e$n>NEy~#F7X5Ms7Q>UA_Vk~ zaqVQgtbr-=2VuE1y|6*R;Bs9@6UW?cq$j4CtKCh3iVX-I23vd7jUjWeal)t%M|~9&bgFK>{I`~ zj(vgr==)24Z1E#QGVzb!r?XzzcYFw7B?kl8x>R0*jz1R#KF7C2n5;|P!^2S>rlyMc z@57Ac++p6VKfYFB6q6q{et0%>+UW4{(*ZJc(N%yU1nSDQZ~KIR-P&Gkdx9N=yW&cI z#K5q!!^aIjVCLG&>@>Axha~g4goc=(W>nq$pt6IB|IAj!p$YYy6TG+Q0IKy3-$#@9d|^kuwH7UbUCQoC(<4gHH4JG; zZneb+fIBnGO=yd>|0oX*Q&`P(>+iF(3i7W4D3m->i#=Eo6+Iubo**eDQLtjUsg9#P zv!>Fyn*Yp!Y2Tq89W5SI8>+Qw?Sgum(m(X{B>RbZ)_UZ^+_Aad1RJgcD&ijksj8XD zx43vjgH8JA94~%u$JPA2+ai}$=5XP)c-AL!A8q#9|s1yhS}d?Q07v&m)sljx3q;n8{kVN z8Vb8YNN=M8#k3EBZCt*~SpMYCg}0ORANjTESs07O<5j72o*C83>sjEDPy>wE=hrlt z$sQ*r+&XjJp9S19tBttrk~^ICv0^39Ge%^_4C2n~OB`0wS&wggC|#3F>D~5`%e3kl zy51C%FeOaN*;&lHRdA?-FNIrl5=89#*8;@%nFxl;>xZ7TH;@@UTHKo2WtDi|IIA0h zvg;_lhW1S4A{P@A6CPd&CXDvxp~dFvXC%t;+ToYavb?$}@`-SM_3zcVI|OwfV6b!F z1Kl}>3oVZBfb94oXt~K3G`&X(m;WwS)Y;S4Uny;gDqsm1j&M}ar^JC72mc5T0@Ty! zZh}W_jvZFhjEo5+Brm)QvxyzmDs9_S9Qx)zoxeWs`^i`Kp*G>v!Y5p~@$S8<$V0d$)+O(-pxRlil&5TMt zI&$0>q7N+Sm0=!xBx{W7Rr+xvGdgb=pIh_S;U27gH^KEntgpO8h+?ho^?E>TG(po@ z^!jbNekJpTsofmyYHjf^njzD*PtR}euqrmEx=3gN3k*-zksvO~R4!>yHI}h;XO*Jv zwc0VS&ieTJE~vSm$MEuqi{7(F!!!K)RsrvuVry|R1Dr?-r1hm89Urz32P2^qE$7}` zGB(-HiA%wDoa(w0E#m@iX@(`E75(ca6`R*oR#| z7Z`e`ROY#9?be_u=i$s-*DlYwZ3l;0h86P>W>{lR-NTd<=Kp|!>+N_DgQ;9GZCfGw zzNy!exL?Z~d9~UrS*0wOGIQW%QwLq_%Rk>T?gZ&}^x$iMgHhMcg<&4U1#SweXh)C^j=}6 zrk)Ck+Bi(yi5$!Jqjb4;qI$gNyX7{4Q|TnFeCqGw{SZOiR?5v!Cei1StQoPkPqbqR zGQ~(raDD-5&R0sge>Xw^LhZOw`pX^qL0G_FNkky>{Dyn&ck%;-`D#M~8uNuX{{5R6 z*e5@Xj!jzxZ9o&_7Kkze>Lgb{c`OA2X6e&^?tr8r2M(X>*4@OUr0y6ZAwj{tk6dF^ z?CjWJxW$#Ba<233reS!%^})K*L(F&>hA@O6!)Py?Yw|W+#Zigs9&B`UO{3QL{q92o zh9~(PvWWC2x*RDAXQaqgr#*JaRp0nlH)nTN{&`{M|WKZ1__FIUo}hoHJg~2L5%(6fmmj6K$iU zG$yAZq|BF2)bOBnVkXNq=w&p+h2G{pV zaJ+i;%5bXmnK9w{-%HO6h!qtMso-Kw1;N3@c???c-C$U7?FWD5Mclo^4lQdwU%K_@ zEuGDaO#L&@<+~MxRn!l%##(T+zTS_nSK+0zL;7;Bp_D}i|B^gyK>2=xPO(_;#7jZ% zFy}(`A?xzSu_>o2yEx#zE>MdrUk!1@TOH6cYGFAw_sYq z+Aj|8oz1qCaxAdY1@ku>7{4|1JV#<{|MTy*uM?#=nX72T_*#qC|2X2?)Y4k~cj;H|lb85xh)sH7)c5 zN#V2G-Z5li#C-}O|F}dutV;aH*;s+~l-QJ~7oMbt)A`<=l$;oL=xFewWVTr(6Qe60 zJ;~m0!#LY#QfD!}r}3iULi5{&_2de~?8`50DmsM@6xV<75+Nc6aFp9A`PXl65AJ+0 zan?JbwfzNZi*MiKzkcUsssIr8Ku%9j2Td+Xg z`WRk>KE82%GuMuNdVgqpW$j0?vUqZADGVm3-j=FgNXLy^JAo@k04+j=pz3A7U^G-z z{eeg=!BDZs_ch8@Bwy)EdhQ0y94xr~U`I=<%gj!{63QLRps(k0%Gu?zGq)Ja`hd?7 z0i!_mrCD}BY}Vz&#=?3e3v+aIERaRzhT@1KAbO#-eA|uXb$&y6@djf<=n2-Z_d6A| zuP>(F9i-k|T+`lo)&w-pPmn4O+2m|sRW5-M@DQ?@>qLupFe-lp$Ui^2wvpg;(j%F? zsXiCSvKib~rl)%=lbt>n^Y7A@Xuy+fcn-ZTh1<`F zz;AJ0em8j%rxJf-V^j1yN&Q>MyV0OL^f$XnMpf7BsA7fnQ!LJ7HwJN(|9t7cU%!mo z;(1gq`Q*tU=*b+3u`wHZl>p{^$N@Yc#J<7^U#0&D(q%?C4leG>%;aVcyWk69so*eC zQB?QqcZE}KqL>2NPY@8HWUTRQep+Sc(%32K`j__=B8yU8UW=rDb{QHBn&Xr$)lKZk zOdTl2iZNS_yP26TR@E7%-Ou1%x%s?8^6y^!U6vj3Y^P5@0Hl-v75@DgKLDk9;rPH@-Ah03tQ5-`|EU%BcjePuD)%xCeqKp^^Ti)v8FQ zG~#(=d58FjjA-t7xYfF?H0N3Wd@pYn%uDBZ?>&2=bj5ql10{8AyXLqYRSmhN6v7Z= zSo5d--Ce%wtIe@RTyVi|ZyESO<$#1mDa=x(-Ixfb5R*WCgaN6REQmoZ7mPbWabT8+ zyt*YcxYT*rMvvqCub?W$@>b>4@!^K=YN=_!-!G3r?D%l6o&4UOT(*%d9=*P7u?pK- z@KGMUOwe@On3PSJ;xCff7*Z*c(`rq{fbKeac|0YX(ML9O`tS9@e~Ai0JApTCHB+6P z#pftqXF5m&9PSik%zJ=2Qy;zsEgD^zGi5T84!irZV>k|lqUCC3l1VrXhbav7<8YaQ zZZR!~;v0Dl;U$ZJUA`(C>-;}&Iu0Kv&zLRXmcpc3oK_tc7MA&gA50D7MmvuJpMR+U z|NOw~&kuY9z&}r)bz$09`mUSI{GWA#-~1IY|7|3AKWd!td8w?RO!ni(Xb?>XtV6+c z(0OcfMEC*k6H(z(FRH!8L4+ns_)7+_E1UmcDG|Md!sQcGMG#z77RPPP)ejFO&?*qN zpS}ptp#JBI+%P(9f8p7fZ+@dzt*dpsHAhSTH06~-<|ElKEb{B!CxR02^NbG}&eV$2>tO_JAxVR*KFYT;?26Ao%Mb!~b%n2MHA&{a`V44iEkw{u)R!WCzTQr1cR9 z9~u>q137R+yNDp3sm8g&KBjYZW(*YBZ(2b!#P2UQZ}zoDh-f8o{=7b)(lRKJPGL~U z7zo8ArQ_q%1SNqCU}RV7__*>9m(cLroxgt8^(F$Q`}5+%{t6NdnkD+B!Ab$I=j`Ce zlO#OL+fO2#b0hw%%HTP_4gNmUIW%LIW@IcEV?b@&Od>Jl>o5OSHZ)B zdzk;+T$$)D&<4K_Sj6MO0%daqJxqbutarK5&q;a)y8nm;uoUC?tmJ!;Mc*OBA!^34<|6IA@uk2a5fbk{>WK{+E*4&33GS zAC}>#`AEiKi{l!dUYqmi1~lPTkm=;W=}Oktf?Yk;UIyc&I~S4WrT_l@A5nwoFMpu) zd*yp|J_wM58B4>jfYFJC4llO}YuF#150}PTc-ArrwCxXoZWMZ_?;n2yOC|GKDh98e z|MTrJJES2*z_3U;dY*J`LvkWvcDsyMx4-yTs4tmnd@fwM7!Sa8dV=5G+tX7Bl>hJV znf7vEK;9+Ey$h+oD+@#a+0YK&BOwBxF(YTHQcJa1F`F3-!+X=~C3SjxKB&0h3R7#p zB|?OHvn|U?=FtR>mCYgk-Nps6Ww33DVA7rd%Qv{7_}3%3?kq6fLB@!Ok1eoUXrV4s zDFSm6=H=e#chGqqZ;kGdP0;?i+5SsNz@dBqE^;z(deRS~sU{5O2kd zE>l#wZaab;E(`lOds*>`;W*eVxB3A7dtuyx6xK|Dx+ECn)2M(n{2mmZy$==_zW`7C zJv!hbW>%9i2Rs+l)YQe0Kwnu7&qK>ZoJ23tS;{}}z=q%Ad)1b{QzjmQ#()a1g-yaF zeIx@slzyj?88f|N0r+bvA^n;v1Jpb(FJ>qWj?Z4uxQq!ONa;tkM>cc*ElXeWQ}Me8I!%>R6&hn7&CUT)&W#6Q z4ty_~h-csVy~DW2EmJI?ZoD`zE+J6_wDDxKy+P~E^~DQHky~{FU&s}b@7XtERU|lP zODbrq<6`aY3b;f6B$D75I|l}rx=b|pYE}L-FK0YlTTcgEB1iypJd-LzQZSrMX`&E% z5B}c%LP!e^G`31w1;0!Df3$o)hzHnb5LA+X1KleL=)`QAMuhj3Md9RHR4Hg*FsKrHJQnTP5mtc&YCFfI$TLED+d6U@yv3R z^HwG>R-O}7O_iyYB}#Zn%)-Cl2Pz0x<}&MPmi(wsqGMwf*NHTNa4#^#^E;VOMBOkd zX5lEv<=ra5S=BB~#-8I>^vr%}3p*`9#_QmkfsURM~HLoX$U$xjLdd!#4CH#)M9?03eYC5`XnSfgry8lS_J)M?4Qyw@FB*u#ab%PII-Tia4;kk~a!)=}lB!o~SqE{lfXo=U%i-(K%LkA@1q4&f0CUeN ziwd+&0;^7wX8xvNDR9bzW#z`#gR}UZw1;mYh`WyHgCRJhqf{Kli{iwIlR99W)1bki zLDJL9%@z)u;3^2B2_JDY*=VDYA zS1^s=u<>XZv;ZdJHE6@rW$O^8HgHO=IDAguqoeQf3<0YmWFtPw#DRiwmkM;dG)si{ z-Cj)agAGr!*)E&=#TPZWvI1z}l~@q+AecR2Bz(|Pn(2%LmRS=cs#2{wSA4Z(VeA`5b%b=)mX+4R9-r$jr zwl5xrrx9>uzz6C!qe-}=q@wRq!-rRGP6PFej74@IR5Zk%xVlsSQBN=`97;rmI(XwE zJu|ahwZS{K{z$?k0He{8Q&(SqumPJM(1-56`B;6GR^~i|j)v}QJSz||Vx&5YF}p+@ z-mo@;D^5TKdoyGrj;(4IWgOwnxY<_Jr@MyN)3s_L{8J;yMH24tYEmpg+w9;;D;i0-I0z zk8f{=9pPTTOS#U;@`YbQ__CMDG5AAN93Y@!mY`XbR$B5uOyJNPQ6pp)F_a7G6s|tn z$p>wJnT5dsHw<52T~zIue@uG@a%~pH4&wruAlhKVx(1{j+61o#G3^OP1w?68WF-no zsc`8@exQM6OXe8|S4!X;h4IrKy~uFhnte8!E5d%VXmY{onE=57N?*XRSgsl-$9*GVs{Wql&|pQ zF^H9ep)#9{Jl0b%77R}~CWS9}kih{YAw9sheO=(}R}Ji6sU;N6D?T&92+p zt&VG<8}iD^@oJ^Y-E{O*Zww58|B>3DSO+9?R(~1z!<=SA1YF;Gz)BU~7Re=ZqQbyH zg{HV%guej0s}=INN*A!QjKmm|aGJ&g@G^|f{&sl*3zr}M7XAiEGLm}_LFas&=8j1I0N7tdagS>5jYF+t5Rantb>HFF8t z?0(DVGffj?V|kUp#cb#EL#rqab~J18K*g0jb$oGg@k%p602hD2`uh6^0nD>yrm%D& zcmb?(!oYxx)l_LhI{p-Y2erx{9f{+R(e*xtvn)|q_3l1-oqk&WE?T$O3*^61t) zBS2)~xQofN65IlP;ub*eBF_WG!+onbj11rMH(*8RI&WO#V~`5{zsA|H&Que^72)OC z!3cknMIJaWIxu;ef&S;hzLAkp&XY??I57B0z*289_J|PM_xj8N{~K+bAPv~NWQ$y% zl&G6W;OGZYf zOyLZm=_cEELxR)=R}S?5%~1PaL`+q6Oe0nUPq;%-l|nve>&yHVol2ONiD z|CGC*>4Jfi67@g&yO7;>n@L8^mB8V*V~5iZ#e@Fh=TFC>NLK}_nZEg2Wh(T*bQ9uJ zc<+fFcwGkpAb@)c!t*A~6mVj2281iWIuJpaEO>%mtg$~9U!(D7aS%m#60jfw#bPWc z*=x5$qgWx+yu2KV3`!11-?nVZC5q(uD}hMpwiFxLy|lP1qmNj?6%v81k7m!)QE$yR zD{(Ug{Nvm+`We|MJu9dS^4E#oT=KmbQIi`l3qeyc*1fjDc1{HzrT`Z*b>g*jrr(kWX3DLMpX$K9b`V3#qo*f$`1tXc zC6mExT+uEDh^Lrt5)Ry1&Uhty>mb#X_>(FIZ!MC-paFfTm729ZbX1kvjYGCnI!(5ducfg)z#z%fn+J}C zl_7tXbOcTd0Hv~lQ+;=`JFATQBjI?rx}BjXQ621Jz&a6#vN6A7prJeQuN48{;+^Q? zTMyt0gIdc6IK^tBL^esvdZzkCA|AKLC2)#sQ})3@ z@37UtN5X;V0E=GR)LRY!<0h+gcsFKiUk8VTeDyI)`8U?2PXHKm>*2Tu2$p$qU}SVO zw=Aldn3(ybju?@fR(C3P!-dC<{Cv#sA+P!q(Tc*%hsepgupz zjx$R0Sn22_K}^Ot5%1T`8}`R6c1d*Jn7r~q z@&)2_E2XegnPs35e}l@30Ui4edTqig!|p&%(0|PItROhvaKU27sklSvk$tmqohmt|Rw8 zfLK)eA8E-K52*yw5+F)A+v4~dI0oq(ic_LLbOBXVy{tIEFA{lm4B?wEAR~4~K3b;sR(4w{CS3@ZRlO6ra$D8%yUN0X519!4^7o9Cup8Dz|T1x$(V{eA0+Jvsy^0H4NqIGyLe9(ZLf zK%v!mUVyAz>GP_LLizMb?f3Z51-d;j?Opbr@0BSKJae=pkOg!qF^Z!Qq64LncE6Z? z{)wrC!7P8*Fi?4)Z8n@f|c``5b)xj zru-i~)AFy@iQPQi+?fg;cPB@Ke+%Vq&#l)jo^M?rv$=syqEAJ`dwHpyge z#MsTgi>uSMxaTT8z$jE=hJ=PjK9?EaAccz!JowTW%?RTYgR`%3IxLH(3;VPISRorp z!1S^Is(H~fYHbenj!g6)!D&%j9!h>N+3sdf!vC_cY!XBW`O`rK^!iBVOp#Er&h1l* z1ba3#5aRPJ_W>L(NGGBDdV;h(UQaB!!2e+G{}YGHwb%xBau{f4dE=gP^@@+F&292N z!>^c3fWxKB^23wjq3@SxqH$|*#|iZ4tpng%Aj<>XM{eRvO7~?%flzt*KP#ldp@IOt znE+yi;nx_bs0qS8=iv;p$S5c%;Rs*h=Z)hK31V7`BuuN4RH9HcB9{9Z!Ff(^tp7t2 z-bOZmu&aYE!op|i|7stqJ&^z~!(#|gA$$YX8U$70dDEtF zc2wX`gu?z7j+|UJ@F30WpgtyKciQZsf z2#}W_yn>Fa!Hp8xb@^bc0MxObF8{vcWJ9flBZC}3D;+xk76~`FFG0;PDjHfMxHbAT zw6yeo*O!Ahi0!vF8vbL4#Q_Z2wwfpixmEL}z*urxPu&OM-1Y-GIC22O-8BZGqdE0J zm<3X^z^q)VOs$F;AQ`NqcS;Q2)Q}gkv!-!A94C$!%2(ZyfA`%BaIYd1=Q-SVbPr~m z?QRu6rWzPPgjEZgtPEDW<_3p?2@nl`s_$6_b1smHY>7$S5M}C45Ln_{#-=hnuLH z=|k<`C4xr4rVE6|2y{CBkBsjDr{F_KX(B_HV>b7@!paV-PYe&3R)G9AJW2*op-jJ{ z4X7Z4cb%+|7TfPW|7`S~&Fzja5D9asNraBAwiA_*T90)tZr+Mj^45F?vs z3lb#R5)6-SA7Mv#FJA&Nu&!Yp#msAdw{yMALt($!5@LyDgRU{0Pg{=M}U1(7BKjPD^%ItJ4qpqaZM^ zV*ntpk*DD9n&IILy<&1|S3f2HM|k45AYi~qGH`U@=v#IGD1pkd@OTlwYty)pKZ&l> zzqy;e-G_Z3{j|&u0YMBdS(tVrn5E>`y#gFMfBzULJS0i0C|>&~8?lOIq}$>k2C9rK z6Y&o15VC1!KE3=lWcGbgph)Y1cgDlhf z@*6}qZ0Da)b2iku?F&3z0fO`NAYI`J;4s|xiI2w-2$5yEy}*OFb_clo<57~4cge}K z_>6NP#VKN%i01|A!`*TaHV7dyajs&Rr&6Ho`4XTo4MjHmMA7?9|Fs z`so;eIH9}6X>lVS!DD-zMmBum4AN=75Zt zl(ZU*PC=9%I+;aZ14L+o85NLP{suFxW?t1E9g5=P{`y=6;y{N{mI`Wltr~EX!f7>0 zM9SHYbcf+T^J|f+QBIQ5-t2`~u9$fXrh5FA!4`+*wGwY?*hj%}` zf1hzu3P(g>0+E%AOo{C{`)ijhz$V7}xaiIZIh;m52?Xv2K=8X?mHIW<<;gb&A`#)? z4$_PpNPn+pi`+5@lg0yD*z^MsUaQYA9Re}Smhw1fWVjZf+6VGYw;0OGW z@G(e=rssvn>A#l2UlameU0q!VHBDTysM-(3N)RrbJ~=5V$c;+_>_vC~{?D-}j~xWW z>|%IpK8Qwic%5W~H`Zk^P&rU9A>NS*|6>NBd2nee1awWDF2B@1c`0N}1rLD?4N4xe z-fcdnB?%Zox5Skb@VqDulH9BN)EU9>Cin^T-hUHBKzqHVhlftMLV$PAqE#o!rrTop z;xPdIcoY;s$yk4`WCH7|OWE)9g=Coi23+l#!9no!3Id4gv%o_nrO#;hb!%!3)+K}g zYVH7*8ZfiX3K&OAh?%Bl8V2bjd3~RwX;Fs!-P_f>;p}-=n^}hnAne<~H!dFnGLYP| zFhEB0Z;<{^$YmKl;DJW<0MJ{NebBMgxwbdU%Z6ts?{0wCa){LkJfkxjXj3{Y0JLlk zfY(>(=>|n$woQA$nSN?&S}1t2Rj<}46UK$0zP4ag=7 zemwlUC3cLWG(FpDx7sKkiYSCSLl+Dx6#w<`|C$}F0Km8Nq~P^ygy5&$xdp z&$rvTzyenyHfSh9O4Bdfa-QdjBEl0Ivht;xwFQMT2|e}Uxv~V~bU0P+dDD{(R%Vd^ zwLXQ@3$I}H(5?P|Y<+c9)!p;;1A-DtidZ0xf(S@Tr!=T^qtdN(UlavV6iI1N1nEX; zYzgU*Mv?B6=AA?MJiqUH|8cpN*Ly#4&di>@_sq!`*uU9TH6+iplsZ|#i?xXsyW%{h zB?quwS_)WF)`J+l2ac+hu|X{2pbRU@9{@4pH_#ax zu8%Hx~Dut24NyyE=jzc1wiD4!cXBU`o*yrwc zKjBOf@c3~ks^HTja!u+YbS2oJl;weK+jSThw=X}|rQ|C?#+2h~EXaJj%1iVsC zAkEFI2C#8S2Vfu08Dg@3iyFLMjX@Yp05u2XpJ5@(g4Ac1=J%cji551}9gw9$@pHJK zi!CZK)zZ~2^zw$S(3Hf}X zk4AZab%}`qxxj*0$~4E!DjtM0S?sUR&QWHCkdTw#PG^Ei#4QuvP}3v>iechRf)#uhv{s$E zsa2^uia(Q`Dk5vY$wT#?v#7Hm8`9ELmnxFbz6L)6Oud&!-o2^@iv2Y>7MM=`M79)s za;9&f$T)oMEZ8e7e3S8l1A>nWcc4PUmOhcO9*P=RZhwoKy>G%+r@3^gK~Wc2Q~~B> zWt%|@ck!35^S?BT{9u2^lspoy+gft~QWTX1sGc`JR~UHTpm4T5Bl34`cwItL-T^Ho zM)MvU8k0%lI#3&C~l&IEoVP{UmwR_ak|AuV*eit6~4nlomH_CD4t z4YrxG!CX{g%RUi<$E1W*4zgEY1rEOFA8cPb`{7y&?MlxQImM|w9OR3MEy@>X( z96sn!Hww+KTSc+Q$BOzGxG#;-BHtM7+Q6AAIGPuEC{V^@cdp_6o52zY%+*to*~B8{ zKv0E`f(#?=S!&4pbQcw2WFUq;!x2UQi5(xMUT%DG_Vfc}eNaYjYa2oM`xW)!=JG^4 z6K!=t2k@PCjErg-8cNU@F{J%FJVDZ*C>#<`tKzYF=mtY52ROF@)*e&(gS!;FOK5&C zRYP2>%sDH~Gm(OX`+|6C%*(yS9IS_6Y9#lD_&>fF(l2eZU#nw>=O3T5ekTKcM z^j<-@AuR&9e^v-@F-2Hx7cxhCbuNgXTU!_|h+tfKWa|S3+(MhGiX1q^$jeUc*@{pq z-Mo}I$_?x+S=MTDP3$?ZrRP^%7Cek_ASJ>Av4^XGaf7WlLBcPBLzQ34;R%dp8PNQ9 z%VETR;oA0>U)VDgfMgo%G}%!DWpL&(k%e#cu(|;l%A1?NK@`_MloCtCJ1gp~XW%~0 z7zJr}d;w;^GJ{f7X$&ak8s6BxPZS5#qT4KsK$Z4f%wP3i%n`jL-}_{zu3|$Tw=IQI z64}Wx1Zp`3eGIe+7axcxyB80C^piI|p3-3y7xib?DU?-?6ELw60_)R_0#hZ`3$#K) zdPvmW)sgpujRux5pJiWZ5z+=2sGpqSQ$}a8ymgNB-hVCD*ib6kOUgYmK6Am@PYi>H zWCMua1D=aPa`Je=wyjDBaSa}Bh(7Fit z@Jdp%4#cg!?-8SM!y!7$Z^Xzrre=YdGUr%Eg%eN#Mu&lPIa4x94ZwH*qyJ!~;)7l! zj?$K(^HlKh3pc)Dunr|d*~MTWU_w^AD`-qGG25d+fqBG+3g9ca?W%3!|LLKbqOffrHNKgTjHNel9oyFm2f9qpFumkeDG&RT}ZhMx7-3!ePs*V@-NQRw{U{ABf8v~DN9#obyf=*og z=(*Vbi$ECE_O#OaJq#fZVn9Mj(7_U_3+=f2FA4wsaXhFtLA4%<)el8dEF2VVy37ui z=NmY*>d;!F!fPSLJpv-ibwk=05Q4WZXIl%&^KkAEuG2mKcGO-`};mGf)c&{rl1}9X9b) zT|uXbu%ayhbW!PM?wrz%%TUDrpB-p}B}{42o*s@!hL%(ncEsrc-|OnZci=Wkz#HS1 z=w3Sf&wqhRp=SmY?bri_1MdAG&a~hxz5EPXw<#Hk^VE-&1t7VC4T?O zHUz{U6qIG15nhqab`d{+%hA!1?Cqxs;6F}ac^S^>y=mL`i|v=-cnr#7?B$Ih^#O}; zn-S}GU8D*WrY19oj}j3n_dX;q>F_;t1l&1HjOL6FA5*gH=+;K_WXMT%G#&z~F$}Ck z`#!DR+x?_z1V&OwK;&KEc8?;HXBA%_YrgOIbrpL-VE4WE08TNANunVE2=!R3uhi-L zXE~*7Esul$%QcY5zQcHT4S>4P{CK|m?q~xWs+TAYJ+DhhNRU;Ev*#qaumE@18@pje zw*WP8mazFfDabV6N3oQYFWpEE_)nm(p2l$`NC%@&^kM?!XL(q!-^0|vnpM?Tx3ykG`?`Rb6lepkNX>=zD}ZH-rpPhG2&1xC}1F{ihDI#JJkyH8Qnsasc09c#0=`1UFLvH&&tp+_g8yoUFS8EYGv`$jGO9!FrA1pzF-8e_%2@PLN<&Z37 zLAL>Y*x)%-(A}KB1;02#R&Mqq$eKV&S?1~4JcAg3m}!v5%|SawLe0Oi5Kjo57yKi3 z7L=E(s6GY0C>#hjTxuNQU6#l`4uV4VnHvJY-E5yg%s23oKHH1u4!kr6>#0{;EaQkd zt{?D<(Ud0Nqt%gGNG0g(_*ox=ycm#_qmyxz6r1FUb=Tiy;FEHt}Ih14zbq@peUY zS|nTlZxZ&QTm?GuwcS7^82rf!uCbR^!Cak|5(hJ=GrtdL+Sk~DC}o!Emp$Yg)aL_Zcpj54%YBXx$%`tXB(@iaO7 z_49h~a{rAMM(kL^JEwRo!^~TJHKfj8OT<9mXq}p1zGkx2fos4Aygfg2=r+#)bY*D* zS~Jvd4hE$L{$BuuC*=#v9^n{J#zi>sQIkXjqJ$sMxx0w6-NY8%S_*}S1q^73^K^6( z1Pj+{01U7{0WfIqnO{-F*e2(9ol8WzaKt5j%X>`@v~dx>Rqfkn$XA~1?m96Ez+2&axO0 zZ3t(L?EC1V=82p);79HjD(qbZpHZeIK15$iCFLgnVvxRd+R1=UWKT*;dOE@Se$Q_* z$nDEz)*uU}d}Cm2nU+8yt^;cBLEtszGjhN77MPe{Al_MAkS1Au;b}b9kX(fV>H7n5 zU0fNC{~#+@?LpxAk(BG;tDj1U2O=_bVaTCE3d~Y<9=9EK80@EhZ~~BLV)~+{)KtbH ztVDhixNr+Ha&XKjJs==Hx(|8gLjG4&i+Y#@VyZ#6K+OLB0=o``vy#tU&?3;QJ{V;d z&nbro#<{<0{$BnVyy^=?Ox=NgyF8(ePec@f^jqZOgs|asJCdq16`oTaFp;36!-Fyg zeN*FHVTf)`2xFEdK6oj~(#y1kka-YZ7L7O<@7hvT&Yj}C(U(qI27etXaFo!lUJTqF0z+uva9Bs+fGn9P0y zoUynB$L&=$^*bpm1XbDp8NUq;KX>PUNoPERzk~v98v5eetEgOFsv*G9xNar z7-&r0fA{AOc@w+(m#0*$!tP5PHBmZSMzHQuPGeu)Ur96??ZV8=bgMjB*F=Lo@~`kG zB3MB&SI*4878L$`Hz@C>1+i*+l=D#ZaRXAJf3pF0VzPh&Pw)!LO~8HR;mu&4|_|!x(r)DTD>lU4H;=nKLRoL z9FU79;t2Yv@_6>Q>RblKkd@B*s5D%qXHrxE17Pw6JA^>)Ec!ChzdhLg@;p!N0V!&oBW`uu@ zls5NEO#fednJsBk`VpYi;*ow-c=}#3s4Ou!e!xkVzK&TMx`h2Vh%7mLhadtAM|mjM zd&^z*ig>C+Q%dXIwlvkR42)xrL=Z__x-LK7!+rMI7^aDem2RsAaG*vhj}26T5PJ<` z-Au3Yp1&BbuvZ-ui03kV6Y>$mTQT4_6?y!iw0oo>bCB@RZ4Bc00(P#2nvfj|v|=D0 zdj0f}#}jicNdlMI(f4Gi_#mjVFV)F_f~mU>eQ@2~8B82)>FRZGVPV9coVT0jc8m1t zpV)w6BTze7e}xAlu$m8v&t1wXUkLo@A-&f;g}16veT;$>jIz_5A38|IXne7OI)sh- z;)+Ac|IjyrgsAb8NjdgB1ma@(IjdjdJKI~YVI4UW#r@d4h!}3TpCEO%o3P|Gq}<;_ zJIxJnn9QDRAB}%}t>f`-mj+S`aN(SLCQ$mAt}jAV9hr?XSo8oVh7I+VpL^8Bpbu7~ znHaMs2{Y;Yov(1%Bc?h5hg(obG#Cj({};4jsBp6lDlBX9iO=$10}rOPL|Yo=(k%~I zYSSk#b9U(&4O9X*j^;BWXDge5(gkw2~TgC_Bl3)PpANlq)I(_&1Bk!~}@OSa(<6FR&WRmQz`wh4z-h zyuZ9ePT~bn))%^*dno4>_O}k&x|}^Ygx7kdlE!J_Bp;}P_Q7uUi>@tmX-gG4FSdF;fT`OKt(hhQ9F%NK#aZHpw z6x{F5l}oRS;maxM0O_KnEc>Ix{*S*?7ZaTwFY$16T*@LmyTo3lfqDi6_~?)q`kF6P ztpQMze%3#zNq?wf1jnb4AIuSJCRw30N%w7{$&wR6-WyRCYS+r+mmt!}iMc}ONN=~z zx(PW}T(Njg9H8Qcf~>EvQimb!$;&s5i-EglTKkg_3qw*PWau>bW%e^r7kFPNWGKMh+)ChD;9mSHt8#{6w7AN#a$jRGF zHQ}J0kCTWQ6}t3KnR4VIF$}vYwox#7lXP6;DscKd_(uS~XK1U=ip_%kRA{lDYh=a` z$NFg`U9d4q{@*~31LHoxm`yN#mwQkHVmzLjFSFLJp-ns zEzv;Kg#;KI5e2CK@$#KtQncEwJ3|sUdtrQ46@wqOx!c=2Xou}-zj+3~3f+!1TQ#n= zE-~?Y#%t9M)FIXFt|bb%OIy9HMF3-27ku_NF33F zR_*DGs6K;QO(2I$Y7Yd0U!MY+BsH&eorFlFE>`!}g2xXv9N(~&>!Y6BP2#Ips}~=z zfpKHrBx9Qt+#}D*`c~3st)OJ*!aoAa8_!Qz#6YX_s`}#|Inu*s{7bx=jNg6fO?Uv1 zc7^WuAdpM$6-BTQ8Nqb~7XPChSlj)$KQKpRU9MzraW%m(+r1vA*?yap!>0ut-$2(( zv*`N+i#kbe+&>hg)_e>&equ(1t|I)0i?J*IZCai4uLi&ZN;o)XzR$o-vsJh+hX8Zw zz+57F*w2ee3)!Jxw)Tw=0p7_6_#FBo@$1UUbkFc&qbMKoRxi+K7cz7t0O~}UGQ983 zgzbxmla1L9I>ljxLEVmEtVjmYIZh98xyd16`_ZT z!@W!Q*bkZu;y-b+(EV`m3}Cqf=Bq~z4j*;gN8f1(q0X*xW!Rm`;JjaGQD8^L$=)XE zbqoeP^^J-@Au>T04h1z_nF->InOe7{C)Oaj`yBVs^H(hHqWaWrkaw zWT{*(H608xF+l1bJEl}g^Miap2lGoHO7bYVI336*kkL6EYL(!WnS%fd!vHQVb^YrP zbNJQ0ZZfs&Z`RP#h5h+g@CAW{yiBSd?~{U9D9O@lXR_am7bm;QTylyGWMu>P`f|+g zN+7$kiO?PYe|%M3*?QiNR??e?cR(V@o9bYq=9;&Z>asMabLF##C$| z{qVRff}{FuDB?&8bKgo|`n;4!49|7Rj7ZSx!i@+Fs5K2MOUi0~MXlO8Dv%+C+ zkaV328uDjlhr)3|;^yHyDh-C`X&&&QLw;h0Z~s%f(uKGGP&f6fkB#y%&~19@tsMh7 z_Ie_qb8!Fu`t&H}c=kETn|lrUE~Ma=pVKe>`tl!&!FSF()mFpz!Geb5k8x{gqfLev zjH-ELo7=z|qlNZi)EYSE$mrgw1QBU0QYAh@z{xTV{L4H+_xq12IK;zqjpZ|y+H8as zwPPLjL$e+Xa=jOmeTbKOEx@A*fnw;B?bCgFPmCrWW*=1HKQc4)TDZ(MKPtXdLxj4I z`IV~|KiORRU>$Hd9Wd98mK3Hv~?{o4NL2arzyzdu%<-0Nqs_eXtsiXT#xza~yUVvY}n zN>UCMes8n5JDOmCsuU5*IV)+Lgg;M#$i)od7-Ryw$Jfs))d<)n;sx`c zI8RTdwe|a*^MEUm$6X9Gz}_@EDu)swPJA4uQpjDkH+%-X3fN9S;8F@lki3>oNBDg! zw{H?O@ucV=dd9de;yai>cwyX6-1ozN_T|VAOL5hT9-34u#+C?SVB%wF&yhb&Ol=*) zFAQe`yP?qzyH`p+x7p{k5LA|&zA-gt1g&^LIx|=0F})r@O;CHmpC;JR7bqTiaG3vw zNpjpSj+2Fi?th%YVXi3mGkMmRjfUg}HWbj2;C+n#3cR=pkQy2f{1Uv^dQ-~VZaCz2_+@-{lmvXz&aRE*(ka`_G-8L02s`V?h^0zf8FQ^ z7Q~#J#{;mia4z~nq>Hh{LrRso)4yHS!OyLByQ*QJg6MKZ!7OkOvH;|Uck?BlKte^l zDikuRAURCL#pS}pm3LXyfFGe`K;1k5PBJ?Iq-)5%fl&^qvAxh8`@0YJX2oIxP_J?S ze06HEahdKmNvFxwFVle8JoU#lqQipeqyv-fVxjMI_eqJ}18bl+dDx&q0X96jK}jsW z^1St6f2F+=5`C!m>LBR<;Q<tWAi5@a^!v9F&NBk3Y6~o72rY?e zLNFDV9!xgDF#M!hY7u0P$I9c0i-929XrWOZ&=499F3|vu>w92()o}15v1r&=Y7a&5 z5JB#bgvb32`Vr+M=60G$u=ZZ((-TIPhh>P*y{8`(0j9i{FJ)=pYz-1)IHuoM7;3Jy zu~*L&*NiG|=fIDE3UIClD2W{BI%lwvp2I0T{YXO;I0y(4QI`18$*2{kIS_w+3^mhX z-ybcCPE2%ozQ-Sj$e(NC=r*HriGlF zU zN$o8vcv(Lr#YD;7D!B?|r0}180LN#1{VEKZV-ULA0ohWMz_niucI|lWcJqaUD@9Gg zNH^*C-mr<{zw;8VF2+s zQZg#=oQcCVoiU76vw?w%Q8#f4y&e0sYM|_uq)#geHKnwh7nX)8*X+e3xReb%R}soC z=9hjl3pt`9OKa0TB9-=vtC+mQ*tV2t2B7iOkG#t;VFvlGd&(4cP=WWA0462;NTQ!B z!?V{L!2R0zcuBbeE+I;A8Ki}aK&*}^)FTVGFE;MxS*kLCD0cW`JCpAI*&@jP4|%?b zH_K%brm?g#Fd?A~Y7tYmA-NAYs~(XY>;}_EAy2Lbg)({az}O$u``COT3yh#>gvXuD zzKHor;hcR^7^oizn0~)!E^dHJXM=F=eKVY-?YF-)fZWDMS~WEO+$2+9o)H$-w1(c! zrP%>PP}aiqZswkY*U01xzD9)%jjVdfRDIN%2#xfDzTWHO%u&J~#cpFB;=nKi`NJq5 zHmFc&e45@F2+fA$$SwvFXf(@>^=+MGS<)yTtBB*Q8ud|Bbe8*X)!q+K(OFJKoJV4V zc#acoHENQsAUhlgBvTi}e!cil^0v$N0vZEtXQ@Vf@OsQ{Mt0r>idvc3ZUdohW9 zFj7D>ll_2bfDS|uZaxg_IZwj`kRuZq#A#0xEe1vC09#6n}#0US=|c-kLam^A=lzSD!} zWY>vA#(gCT99|*;50y||*g&!hfug=G71C-bEcQp+Wgwq5;Kd*K7ORojm){$2{2%NY zGEl91%iUV_sb29Nyjs|{$AP}M{q<=$kYzwcHm!UjS3&;b*VbM6E_%;^ri9!A3~6zS z$u*RPfb>Li{%*K|3<=>z12tuW%p?!=5H@*`0rU-s;iGXzh*?4M>i8ejzD>@g7z}5n zGbf%MySSg&RyAq9Rh5|`w{3M6Txjpa`>H+i12zaBpxtl``Oh_@8TL24WBLkgrL##Hi zGQ0kYse!72$E3L^WK@e_gM&K+%C%?)oVavTc=sb-&lzwwo{&D@%PD9I!J7{M&@&XI zu|n>jo?-~0D7XKa*ur=aalq6s@OA@ys3@L1hlJmEvHivb(kEOl?hFraqEP|?Z^fW1 zR(f``{VV8C#(;86368{lxrwTOFx*j>iYpn-sVSV-%#QW*vyZyi--Ko-Su-b&Y9dlk zJ*BY495_=&G6`VWY$#c=_7_&QDB&2lqq3cvl>Is$nJD%pa*z>C4#~S5JVRemd^D`& z%d>OrQ16GrAgCC)QFmHi-VQbKru=avdRFUVL#%=Id}_QiCqYh@-*Yt=Om@<+85rCn zbCt(AV30_&{6z?%;fT`ov`AH_*tMO6?n*I`;=*7O&WmIgrIoe|ma-D|E$7L084Or* z^?l_6hQl zFJof3p<{H$ZsV2biHlm-fV5(SC{BD4@9&s_;^V7mw9ULt0-OKCL=rqjthi1G7#4?S zK8!=vX9Q;}qlbir9shVfR$jm;7mu?aaPoXg;2*-FU%$}wCDU5mxKw>zXxC@BeE9|# zX472l{3+4)tUS^5=X_kH_&E@ODB14JSU}|kTR-K_o9nM0rlkyOk9acI2krB-4amK6Aj|GL~Q#83Ma8Xraj1S1rotO#$8QGH$CUIryq0XBowz^G&YGOV0v zvN1^O;UJ*a0x@ov_A>bFoqS@L;U?q@EJf5}8loxz8!O#St(1WZjdhM=jB`K!{4jt@ zwuhJ$iw?}8@lrS}bT}RFT6OoY)xH=LU-mtU>uYaRYRzHT3*J}wU?K#v7C>&h(uoEp zV90Ty(58cJ8Vn6B-|}vMRdk|c^N;7T(r6B3N33vEDJ>8>H*ySw;Q+vUDEct#gEc5i zrMT`YDD2zV&?EXBg`ng{fap<+nJ{1d#!`jY1fkO9R(WH zk>D(BTbzg0AnWc%7)p$hUdw6>R*HZE%uqYqMkukTR{(&-L&jp{2P{ap=~(ML$H;BB z&U%_S-aB|#?O{Qu`_+J{97zr>8Jg_Zc5lj>1yG36k%ze<4Apn2L@Rgs*7qn=7Dxc+ zp;&Xb;t)UF@q7>7F6z>|WDnErRPjN&ySh{wsjNK&!2ucK9Iju_38G*(&=c~xFO*Z# z*9hK0IE6AJOL~2K1G=MSVbW9Pz~@n@zrD*DP9|`;9mB>av+uc+wc}H{*dRC}5TzmI z3dFYw(jPC-igT2o+~1fabFVPax7+J{Z|v(G<4(MHR}*1~0~5Zug3^J(6XsXMe*H_{ zxct$D_Hk$(?d*M!n$uS1Q3m7gs018?0h0{ux7cG{%7yA_9vbQlEA>!A-6L}e-%(p; zwu)?rs+}Tukn&I~Exf&OiaU*#g{9WhyJK|TVzc7$GqRjJ(U_R0li;nDg*=QV@0p(D z?s9-Js$?flj1=CaHP~M;c=o=lUK05B45eTJtYIEZA%0qu`X&A#&0!7TBOeMfHKqYX zfEM@FG-QP`uuMn7eM2pfeXL_r#2 z;WD+`B!;#rYFLE+U|y5T(eE*45$2EPIt@IOz))iu0*8#=-QHEmf~9;Rcm;|e3$g(2 zNBDcaev=Nac&g(`L1*jol|%~C!svTU&zHwrWqt{&OP2FCfWB32hAp740K~6!sl#a>`~cJM zF2Pj*e1(!bn)?i@Z=i)3=AV6oqYPMgt)#q&LI87-(QSSt(c=S;L{6zNO+BnH|a)`@sW_ID8(n;4zM%W*AIm^MNSRw;J1uD|mYK_`y8 zzX?=32V+Im>MugMaF*~WZ*g*xheT8?0)|(1X^7FEt#it@ zP-nO;1yEs8LR-$@2l3a?^@vIz8-(%Ei@XK-@4Zz84m;ObU!N~ z*Mk_%ZsiW{o-lLlt(F3?Co$3fgifhP{Y0!Fytd#HQ3)8QD2>CA3eXI@6yGQA_*e~- z|3#8uOd#=bW}%~#5Hb{4=OS8xythIMI``OTvcDg~VE*7Z&isUtPP=IY^;1IMm<1S~ zQG+~K{iZ9Z^7g=e4*sVvU))V}5p9kr9l#O8;nR7fQ0u;^h6H;F5_LXmuTYDN7IMpn z0aBtj&-`rPziSNUBXSW1|7f{O#Kois{|N&YHvH;euMa)|vtQ9HE#N-`kh2UZ%v-hF zsL7}h#~%7xe&ylrH7X3U`>q-YDnL`Mac!6!;+y_*3pgR1f*St;fO8hgB(6=<5Vq5Z_P^t+T@^e0Kd{fZ4E*_{Q%q z5o9_U?3D8J2?+WOX%hp>31={|)nxyfyvwMDfm9|iU*ocQ6U`NOT_dc-a=L zxgEnQE)+wOExd(S=QPMVooI_C7|nHiw*xwHdJGkGEZ8z}84F?5k}9F*ZwZ!HWHhzH zW|z%I>KQ_-7H8|r8-_T)M>$lEZftob3~&=Uh>UI^kW0rp2V}4XcOk7IWdM(8004!u{Zh zGVJmyu@{ZLLtYah5j^-uj+0o6kEfxpFC#zy2DAmQXoM#uwsM&bu0o|_G#QUunqvRbA%=Jz*Q)Za z*j`K6(MC2z2o0&J@$T|~HfD#~j~}c_oEw_Z_ti$_;r>)ifAsa5y0f=E2i`@@x zzR9E<~P4M{T21`ccTRW?^ov45q zMiz@JtoN=+Qk*$>XCz>4{PIPMhoZiCHrNA9b>u^M6N+I`l)JWkx!$AQcOXQ2hURI6FCOCQe=X8@2UmpY|-V4mk z%vS_tfg;)sP5=0wB>1#^=2B$cXMt4Cv)0iP)%dWm_-}><09xDq&|#Lj#e|(+!J+p+ zcuZy80%pDxh7DJ!^d37W;+fcfci)oFNik+Fm`rJC(Vlzrqr1%?5Pw>7Raiy~A}0fT ztsiV$&KGrh$m>4SYd_p~?ah=ei)eqo?#>RochW$4xlCC!aYji=*E=D@HR&N9n_4xm zpR;xyN_?j~Q~WzlZ+mf19341qc8d{}PT>&|6HrddtEhCsMAu{}92d5nG7n>%58~}O z&clD$*u*Y5;>@3rixx@dvh^*PNoUJ5({ zN%Tzbj7e9wW_@yCGjGA^P)p7hpuMd}`m9mTW~wA?{$>sZqacjhMK3%@?E3g!3jK(r zR&nZ6wcq%&s-`Ti*;JIAl`zUMyrTNdb6dYpP*~8O!#HO{K$2jsKVF13=*;j&bU}r5 z9n*KhOIn4px-(np1li?M+S=NuooB-Ot+(Ui^~%JGDxU?8;4ZV;HN7(APka4t^#kRHx$xquVPYQ#v# zL4VKTYw$OCEWT1?CwXsK)5Jc$>Zc#de=R^5=P>h}zv$-s7z`R2WhV`@+*|mn)|6%= zdq;wZ_cQ)FYp&6IY^4_*;wC00r$xOzeb}GLt?ABWwfri#FW3}%6Fsz{z9E(t(SDeW zOryWz`sCkg*l!%IXxh6G zZ9qSf0-&PB4p0Law;QB6DP6&npnl`VORMaXtEOZUrlu*>F!c+LfYa^&gcCw?ON4=Y zcOG95W^Et@Ai?B99eXEWwL-T=Q!n6T@Y}1nM(>SE;2papEU2Quw|T=V(Aq58utku% zXCt_;f8ywc8IQ^41_9mE65g*TN}=-1QB-|TMV#U7<~^85+Swz1NBUKRn8OsDvz(WA z-ObIdpTQ4i&R%O2T6_YH4H9G$*8aX72cbh6=LIRP^xe!W5Nb=Q(ty6mGSm zfWdQnG2xu!nY=U_T^Db&APwHEB_0>pi#^%*>IX{*4xg!uN+h^?o_@^B#jeG1QFAsV zJ#tmKW3v6N5KDdC66MpYaoe?Cg7L5XmvrjR2skfz#kJc9ops2|u8+yY@ABoB$hj_4 zCY0LfMoe7yO|5h`#-MRt(%oq%!Eq|C;Izm0pwjga(zr;Xx@#%wZxl=_4;2e=J^55? zZ%F+|&_K7Mg@fLr^|Z?~qG=y>9|2EAGyfevuQYykr-<8>SQRTNUg)YOXO|$)k zj{^hC6*%>ryG*t+0$L8Ty4VvOy-Sgq9;Td@mw$ct+qigY{$XiG)3mb(F&{U#qf7K{ z2)XyLJI<+ZP_kY4CKh=R9p2+^PgZR!3~aBwU2aX-Rzn`9x!va*%h1$U^-P%h8G_`qR^ayO_wxgP%LG@#nxcdY+qA?++n?_O>+x4Y?>R{d zcqgT*tiQB85pMpKbG+?ln!oOj7g2tWQ*lvwV(o@$T507*`l3Ro`4Ihpnz-{==eue5 z(u)iagmd6Boz z?S?&Hal#+*wfUmr9nH>a-+io`4WGt^JtXd&d;W^Y_CtI03&GN7O3q_rNrx=mhH{lF z?SvY6J(I5*y1y2Q6cvveNu70mCe(kVb4umjS~?MV=vG>Bo!w>#!Ci^U+*$M1q(eEA z$A;wc|1>%myUnk%^0#^Py;Ctm+7lC+z5A z*9Pbo)6(+uFMF-k8FnB5QF>8iRN!3ypSN*57h4RKh+56Q#ME8nb}4QyzoBq$Yu4R! z{7HN$VGGkq1sUts8$QNxUR7I{_D<5M#$;d!fyZtAm6%BlOUu3!?dN*4y2z&zuo`Q= z$;{i)H%zy7r?nQgbUICb8Xbu|^9WD9@~ygMw_&!Rb%C(faJi^(PEBRL^xCn|d&L$6%=7@T0I8%Y1gzx?0! ztAJ#&WYKAv%e)g3|M(t14#5%kNZciaCGL>nxwB*#KUTIYlM^l5QD%EXu*k1-y<}xm z<~6}p{~f%(g%5}O_^ck4=Fl2i74JwHdX9QT>u8KlFE3;rDYBixNe~-|x700|OlBiZ zQewA$)0WBDe)D(`?_1k8zR?qof>|^36^#?VK2gj)k`DS47cLCEnbqt~N}R3mdVSc# zEAk83Z_E{k9y{;FGx~5Obtzz;a?!6(PQ8L-N7NC(bUEw?1wndZq8RY36`V!(DbyZQ zoU<)QO)r}m8jL8tt_&m8d^kxzY1SXh@wIL9qGjdOM)}wK1OC-Je3?7X7``P;hT(tQ zv=cYvU936=Ll9o{5Vvc9d_PFZ?)t|{v|x01`WAywx!!6 zr50Z<6Z)3TJJp${&-gQ@YnMpz4QRb#y?fPVqvU)tT~CjB+XbqDl41oi2@AI%dE0mL z?xRlaqD6{M+iBR*>UTe1=1J&J6w73J# z@LJ2lxQWmD6un=$Qmp2XLs%b2^2l{7s3XE6nrHIfPNEw3bb37bRcjNE9$_d^T~7Ex zrCqZzumZ}}oAmVB7^&`_qS$|}umMlRR_(6StBMw(oqULFY?6}gPdIMx-LyF@les&A)b+M3>}m9QSLu{9We?>uOs5N11_wqPDlG z-7Q)EXL@uCOYK8gum=iyCi0zwct@4mBNp7Jz3rWwK7Gv+)UfKU4^C~$tg{xK%&1i} zTrMSw_qaxGz2kmzYPpz+?CziY#3D_=e2;rMQ6{<-n88TweodFrDzfxY03#{WGcqc2z|G)JP6l%Tb3hvEk(ai0`Gk9_LaigaFQzu= z!p`(Ly)PN5b+xvfBB`0*+M~FH)ofxn6`O6M<07Zoxo^(wv_!P%?1bR=WHaYX#QCi$k2PKXZPi%t%jc0 zR@C|Vo}sNz%pyM%lIR?Z@%xW7ijZ0nbBTBLr+A4jot?d(^>X;;_D+9hk7rcOmE|kC zw+KP1mKm7@$k+_?+_v7}Y^i%-;L3MPX50@k~$Q>$8r z!=qg$Jxd6f{i4`t^PQ->SG1bU8mDJ%uid1H{6J|QO)!$vn!V$p)f(Y6!J`Qy#+UEafJNOzA@-_P4E@dmjBVG*BIh7HrB9JY%~x>BtD!@9&L3-06k zWEWqx$Q7+x2+w(wm8AroJWqA78;oxMcK=I_bm2Zcxnn=I8F~AxyQTO1ZQDCXX^ARQ z&+%(<(UQPNhiy%w{{$J z;bWl;1<)FEs`<1?u}E0&*Mr1`QxmYjHgN7YBUgS#hklCj&{z95iU z>shc9@11X2NB#r`JJ~B(u9)|hE#wuugX4O6Md+n3XdB^$GwC>F5|uJ+ zmB6`pt_3bj$GbA4k(>7`MBXKRavKdBnEXUaPX<+}I@Y4c*+uDFvesly)Na2?sf67f ztE+4^_3k{dt=e*)y>Sf8JBv)gQQ^l%Hr~yM$Sku~hn%h`vq_u!(|<(ItaAPo<9SkA z61k0e$#vO6y`c8xhF;kOOHdU42Q%5DmCDa}PbREItcomBWOY%~#IRvv6S{ zKmU9|44y)#bzAu(`m*|b61}BN?z=BPupNCKFDza#`>u0iov-l@mFFvI55w&W@5LTU zHC;0k-EK?nh+omXD?HNzB}xy(qtbjY-_Kc0)*(JJQFgbDbGpAo^{Pr~-$P>25fupz z!t%M=o=Pdd;9rcBJQ+O5Z=OKY3mvsHbNCv=~)K*Fdii)ebY>x9n(X&%N( z2Asufj=%J)TuncLS;n{EfXv2Yc#a6Vl`pX+t!aSWl&P-#x=JI5=*bq}*JzG4<`9l$ zdL`Kk;r!B+$`SXbLP?=^}}F44GltL~OmC-+6q<&JgC z#8PaN)%K~3Cl9J3!f+T*JN_iR5=^ns@2-1!vO71yg+j3MJx6R1z{dt)Z`GkQWZLU`0_WL9BM21S=o$A8u$^+#iCr{8GIb=Xwx(#r8;?8Y%D2fwJb4Gxr7Q54A_9UF5@wY|pjF8CQswHe{`AMSYubht zGLPwBa2p+V8|k$e^6^l>9qW#BekL;{a7>1Yq`paZ-iOOU|8sYDq z0F6I~Dwe)^&W3215|U=Mu9~+mx2kt~`w8An?^7h5HWefvd8f`jDv-q?@{dj(M!0fX z{dRqdM`Q7{)#V$AFZToR1R_n7;NyC<}8HPo6Gbr z?UXt;+XbBfitN7>tn3Y6U(ZwS>yllj9qpgyZ@a&*X&Vg;vctq=9>>gm{wUn;{BpUd zTKaoCOUZj)ddWe>AHuVL7QQIrt4!(StXnADtL14|uk+N7iF(0d))A?`Fno%jRt8X^ z#vydt%PtFNfD8Qn&?S?4zx&{9^B^ZlQOpyr=znc&O>~sof~KXpNW`V;OSqboMvURB z(2a-{eg#asxAS{Xo8I;OFKdximM!N7Y`Mrtud@DfGO$p*Y{X-|c`>W^O^DnG+3}wz z%yKDsPuQQl^7HZr*3Nd(gQ2aBzFFgm5ys)GOqQuC1wL*MiKer6l6FF*4PB1JoJx33 zPi@zHRG;U?)M1~(seh3eL%x_EIcaQpiZmTDOCIf@G2&>D&lFHva-!Hb@>^yNPD_FI z@85`cPo5r0E53Q(gXw!&{k>Dh&OdL(iQEnHUiu-xmbM^zi1K(<_>0_nex1dPw}p;Z zvv)R)a{g$3BHAy*m(GK#!U2cdgolN>h7xbXiPZ)c7I(IEa=yM8a$5MLyW;m9qVMgr zfMzexiS1t*Wh!FSOmcF9N_4{m;L)_Lty?^_D z8-iP{qFbT1CBrB5U4K6K8F;6cgEXUD9B|zQ1@5$HJvfaEsG}#?eO{JIU%XpjTm{JCJoJC% za9=gbZzo^ZJoH_5`tx+4GW#{|gP-aG{p}(d_NzDgxqil560Tp~e2Lu$LpTQ`JCz#M zW7)$3VW!|n$~3n#s`AbmRVXd%2fTETHEJgz3P`ORWVzC8W6 zz=oA{1+_ozs-^Tf8`=fkx)bUeXIB4N3-^EWnxa%mD)+%TXFjFQ?-dH^6MFF(7InAX zSDQ$E#%uG>l&$GSWv_moBo3nbcUc@EIO=3n?*~n!au;_gWeGCk%lb$Z2$7?iZY@n` zXi8c)scLGj?z!Y)<8fN8Z`l0Zp+3>}2u>DOd-2vG_sXJ;yBbNw6*1gqU(T|{a>=mr zPO16l)y@|DVQ>5JV8kkqcgnmdu-iL@l`lt&c*a$8(q_xb)?M1>er12h(XzkUfC0CK z&~etCi{gEEwAx+ihZ>?~x;9@fAM3NR$to*yZ_vsrWHe0oe6)~Njyc~J<@M2%?Jcfq zgii(U^vrrxeQT-e@Y^t4`H`GvC2|jibiCuUw(JFoTEiWWMg-K`b*R|ERpgM#B`!L2 z%I+G?e`po^uz$Gd_$}Yzoq!Q?K&q^LeL;=3Mh#Q&i_%N>8}q71LM z{~G^)?R|Arlw0?(Am|8)fHcMsqLL~gf^;i_($XN(C9U)z;?P{AQ%br7L{Lf)M7m2# zBuBcG{LTZ4dM`5T_vg3PdtK|UXNEcFIcN9YXYYNAzrCZa4tg7X6vguT#`Y!J!pP*o z`iPgyMaHJyxUryXB!j~sL${b8E&)r5pu|?LEvUbFn)I%>=iYoc#hHNnF zYq6q)#E?NVIh*LrlVXk>1$LCNjYr;$8fE=ss8T?8sm@JV$N5qC!n@IHRU88qE1{?# z@l-$wpU;7_%C}z(K^wke(EBD%l@xzRa?s@nWEtm@QY@ajYmDk-2}|_O7av{31wC?O z65c2%aA%&)qh4wsKGk1pR5bsjnDu*{86s=DU71(u(NoLEF9~^ii&BMjZ4ku_b!H3n zr@pA!Cs{PFzPFv`=~teMKf_5x?8d@*^VHyaK55UH6HF2R%rdieqYe}9DQ7?8Yh+JS zXchHeg=~Fh_Wf3E!koISXSVh@Gdq2SHFcEVV6c|8D@Kz)CF>c!NNMBO$<-;XO$v3H z&#LhiK9x~%o_oW2_%!=4E<@oL6I0SA6RzG_$LpjXO3ja&Gkf!1A+96e{G?kgr{?#b$4TztQy6dclKg z0w?<^%$~B+veC8q0m&l{B^@k3Tke&IV^WYgiLKmbqorx~MRtaVq`#+HnfwrA9~|1t z$Nt1S&)}b{X6BufPAAgfpGvkav~y{%FhtvQW77wzF?uQ2m%KZ=qpu)9AtG*qe$fD)v%kb=26;vRYu}=3Rq3uR4;}{jrap7k`;ocFnn)nQ3#yp^>q=BjO^oOAXFtLSc90BKT*_uB(D`Gvqz zcWH_BKO@t(u5Z&xUj9#|xY1JgDRtY)Yok8+h4g~o{_%SlE#@QcAJy)8g8!?DbvJhL zeUq%0Y$vUS`mnq?NnElkw{0!=az7>T5LlaZk>uAzz~NUe+0a?PT+U=sAh+S+*dDqX zrzQrv2?dQe^)ze?>c>akuFfx>N6Aqj;^Q1BLcjEWlg!(mXbZ7~f4oF_st3C0Zdb ze4c(WxD0m8qcMtE6BuaVo?h>?b6Sb0B8FRJ6k1FM)Qrf8Kb2u&7EWNxO#aH#t5t8W zCDnT8dpfNO2Q85%C)M<67bS7&sAjIgwKrt<)#G!EdS0r`T>k85d)j3+Ogf6Y(1KWCIPsOnerRva?aL4O3Cdk$>QWtnVm&K^6&+mIfv7tYH52(Xh$xT z`@jynkAQ3zq@U~ZiJFOa>CD}2h-f??`DUiTJD0e1Pthq=l_On`igUXyD4c@lv1{RI zC%p@i+jr;oC;hiFhnTwuw=ehBHMWS1a6D&bR`21eZZA9hR_$?#vw)!0UWzc3DDa!X zng+6hJh`t_JrVH2x$6lVZ;HJwgHgPbss=h=wVBT(~RyJs0%j1?@uPxxzU_>c%9P8u&F3^a^+1o^)Mxk*ryQ-PN__cfGLXk5l4 z3gG;bjAxt_*H4`02+=oCdXp^0d_A_~bLvR`%UaDlNmONNo=z&9)DMpD`FMY<9v-lP zX;!E@R#!^}6fTskilBo|F^>I6ovmdFP)_mfW8^SXd{UO&Q8$)qAHHqc6&*;KSwx$S zvhe$DwZFkd;Ny-TIp9h`c#7ZM8Z?51`T`bJKO2C4pGp&;`u3*yiiv`u^DHmR;J0{b zbJet{0dHB%-4p*`#?X(a2Z;h|`XUdC)Lxo#bIbaJD@EX+F*^s{!Fjd#TQi1TF9#lh z$%SoI@Mw@LldC;rOi5*FEz+gMLBDZgY4VwDwZ8Q-hi0=cCn>|zQSx{$y!Kz&guksE zq=Lc;Kw6uJhX)oM%H5No10D#GAv+QqTHEt&Ov8`icZQruR=4kE{YbI-&!Pi`T$3mQ z?~zYpfz*6Jnzg?dH19hI&H;6d^R;*p3cDs&9zj@f8RI0B^8Uxl{{0aPR3#HsRR9YX zS?C&!1TUIDbFBB*X~n*8($WGc3wZUuP=fnS@F;RD#ORk{v3tV(uYt&a1XO%_<0zp0 z-v&dLJUxZ}y_P>$dV^uevgm$4o0SpS2oY^F;aOnaPD9X$SQ)R1)a9%_7k`VQpp3tMug<|pZ@1Ua|s{3 z>A#{WJAx?A_kgg*(%jwMz54E$N+1Oxk+l{K7aZk^N+D$FLtC6BG^LR>*iY2H?$)pz|;hWGE>5b5J*@KyRZm(>R!+D{dbHAgN9gHQi>cx#9(Dm z5ZW_Ta{X=opD2+~gXFiJ2@($xM4UrJ2(o|9J&lz{L(IRW`a+s3Tyf0D`P_8 zrkU_QB+0fU>UjlZeOqDZi6FVSj-wHJ1cU$wH18zNfgu+$jK$m~I>dR=P!ya%6%4wE zk}`(Do4t3{TZw=L8aaZ%f_5PYiVmRzDylKU+usLe-y7 zG*FdRqG&4UU`7K8O+xKNaIW{~Z~pc19<-YJT0>BKZ%>q?Co++WH~z1m{O99ul3P}G zCzISg=#(=z8nK53sl^I-D>jI70YP~mYO{Z!v})GGDC}ci9R0%FP+M^lWX99QLfAuV zX+59C^!4?1VvTx!F8u#YDOEONSfO?2P%m^|y0(IfYVs)8WzJ#Hn*4D(GW}p34$LhU zOvsioFhEKs8-hMBL{Rd3)oCi4y>$3pSW2A1Kn)Jow$a-E0Rc}sfHO~a$|~@!GvK>9 zIZ#&eCw&Q2VbxRI>>-d41F(tq>5uB~=2$Rs4)1K5W% zU~csL;V=g8v9QcZEsxfe7TKQ_;XBf<8)QvZctDOym1cQy+@#`|DdfYUp$Z7gR(Xs`OPIl5@JYR{joIsUEU z5f7pIs1r85K;Y*90&!0oM#E7!urRh(6(|aN3wwMKxXgL=rBo*QV?p+sob7@!kR{*NNy=a(U=^5PTwUM1OYIv(}85Q<{17JsreUve|(gNs+x`^i1@$~10b^#$0j^$ z4}JTKivW3uzd9bYQeZ`~?wXy8HGyaL-^S0kCcXe1mC7}R0HFQUqx_(~0NoGY z3BN_XP2%EucuySoeW3pa_4n$aGY5;aC3TRxn>+jDWIi1m4c$aJ0V!rG0}wrFemt9d zUjDwXmLNsEZba^Q8`if-??86by3b;YLcz#&;E-VoWPo#1$EJ2k7f2TvZGFEF~5-nsz1VVn$ z7jkiX0LlY^aOn~TBne}B%8iiQ3P7*{heHDa+~#kRP+KoLaHIu!c|`ZF!46iZ@ao_A z?edu`5x^i6fr9=m;>d3YEEy6O6Utf!P03tPTe#os^IzEA=jk7)rFUz5JJ7X+gj$9k zyMDkvKm4`#%g}1ZzCd~GyvOdc(JaY(SpE9z{!am~A%>gjCZNE)cyZTVxQWjnjfR*p zfyol6MKFnpSqnOb2wm}|-@7P%f9y?UgItM_@0JT!s~%2$b9~EPe!S*kZ5s=wC*vHr zEoc(7jRtuk2}__ZiNoUXIdslcJUjqzwFT%!oHBBGkQRTKOu~admfum@bWopkl|j6k{U z!b2VzGv6sRtR=yOxc~4$C?GeKotFVPV27eE8w>|x6||+g1;8D}WhUY)(28@HLgPlC zAU8S*9|$=_Uk=||C3q0AYm`b;aedyax7&Z8&*dPMcM@pZI4I1MG7Kl*ADw#)y2c#P zIm`GOqH$zA;9!}mq=OB5ggY4&9-9e2=a{37dFtLR7iYfuqLj0?Z~6zW0wX_ufb=-f zy=+*Zt~Oz5$=a9vve5MObR+Ed`#kqweK{-e%&{1~Y5j@_oME5_3w{}^F@_fvQ0O-t#|BS`JvZ05MkJFdsJ?jmR+BV+KM2ts%_Xxu-TmCF3^_UIWnk0i;bV z;HYd?u4&!}@d^z~?UEwg64OE*6#w);rw5)5J%*Lx5i8Fl;Qz)R?)Rt8qLEGl@F-Us zm-@GwW9`046g!=71aM&2zWW>F;GIzat2eMtR{NsX`K3+Z= zy$1vqus5`|%+#J7!j$>p?R$NYM>9+y2yi8mh^d3dwY(fq1o`XJ=;R%NIgocp1=fM^ z7l7i6aWroaj=M^b?5IoUaJhA!s<5>U$G2{FfHhxG0sor>a~fD4m^eOwCqWNGkY26N z?)YsE%Lnvf_zfWY0SO)jIq?}xe%Ho?+^vQ|)nk`6(|E( zoggE70)mO8Kp9sFgEc(wvuXWqVvp500|I zhxQs`2@v6ev$J|+K-J@0T?fA?5DLU5o>t-ZaJ2%I_+X}c@9KyBQE>>oxz{M)G#z8NN0U8 z*<&1|tK3`!ekjPP#D~RzDiP2&7Vyr@e3IQk7y!XT=1c5FZ^UPTCF2Q5_nJjJ60~bN z2rdX|0R3-LAnNW;0<2p1-#S11WgI*M$SrNdYrxE(hhjTOrS<$@{gw9|oj-wf7;vKY zkgXc94P}USqp3H`<j>VxxLnPmtCCdKfnAJ5(xysKpNx?1rUa%z+faCPEC&Sy7dH|L;~Rt$cxu& zX9oqqP9y;9mX?-l_Q!zL^iy)~{{$DrY9*q>e+T_+1DIFu#`#vcwl4z<^1yKt*n6sy z3K*V+ebY9^1_lq(&RPIrM2rPm&AfmL>z_P%B9jsgOrV1&Q-cmQ_`bZ&eed1<@iS5f zHb2D;sasY|s0`HuUw1i7BMhxW{qER~yIon)dpuD?tPrEGZDg|IQYZ7!-zVYsSg zfYHD3Yv)$NS@QK*`GMn4KCTQ37W7`zkE-kBXj=^G8GL2WJlnDr&5C;6f@{uS-)-sJ zO>bS@i5h){8NGQeU7Ih!YTV=LQjKDmI#xW>B>-E|BHx^j6BJjf99aCQ-F^)tJRiDxDS(r3jLJ}C+(nroSDWE$sYePwkn zy3=kMOV_HoQ*Wu}#blUweoex5!b=h^a~PZF3rQl!%q2t`)3^bHpnyl%q*#&>?OUud z&lRRmRa_4m81&Os%bIQPLJbU8+N^v%Z}VvI3dLIA&9b^KRtLpQF&ZH=&8#JMW2X** zu9OMM!$E=|!2ycpEJToQCXeBuA#wtwS9vNMx2gQ6SrPYq6RB9<>kQP#vsxKCW=q5+ zam7|u8d?T<)9v%*olDzq2?=Fu5hFbzgY{mkFy*fz+Wmo4ulb)Wo2 zeZ6z?6)d&uz19=uDEpDQ%Ox%=8Q`X+RPP$i)Y@nC7xHdb1RKaqUhL@Pc!W}W^{8-$ z;lhlX5KZ&V4;qnMt7~bEI-Al$6V3CTiY1%liY0w3rHzyYcxSm(5Y$n%rk3iuTC|$gyZ}Etqm@Ek4w632BxEv8}GfH8j z8);*cXOmyo_TBq_RDRz>YlH3dwP*7V7PjA9+Cse+*J3ZVNm+S1r;rVZoQOpi1p~Z} zfn%LNK}GRQ9#!&I#6j9rPB3h}epuC@4VQ|6!}x|x7|nN;Je!14uEOV`iEiFDb@5zV zB3KI9N>;iS{`1zS`crF4+CYg@sKH1lg$k#6xP1&+oG>%%#+}s$l~%52LSZQivRPzj zvzwPrInX1-`59_k0~^(jneseTIXAm`OIMD}Ju}_0?hNm>GWadV$DZ#MlafWL7-rf6 zjrcvL6E7=^82UOd^b0wee`4==)Wp<GB2nNJC{y+ ztK*&2`eCx3l_ON%$_h0trz4(mTr>fVRbGJdP5ST82Rt@x5DuZXVj7Rd)UW zVkRFb<9rZH7GdqP#|Sp2Pj#qt(J6E%pZ&n9&`_P(Tk4c<^igv2TtoC&8WFQY@jBH) zG&ZlSnBUg8SGi5c?Na!NL0G0j6U(vsLw7W;%Ehj*}^+!t!1f}vfhj*A1&X;(N46gobxo|7hKX5iGjpR&4A^0GZvYM5C=>6PNT z@JydL9Vsp07b+cnkNWC6@f@f^ato8LYK}URQ#auDIunntvc)joWN`Bf4(T#Tmuc$s ztbOV{-OaVtBywr)N2cFlzun1tCF5`B=m!Y!_n*4NCBsChN}`q z+&b6Eiyg%D#nh7eZhDhkNq07=;49%@`dXgbBEIcV0h*{U zs279DGhal>@%Js>o*Hl~ms;dgCsz6J+G4Yt2Sxv8KG9(`JHJ9g2-e^F5_f6H+-|Y} zsPJUaPcJu4lx-%ATqS>TlWRnL#YRPjCiu~L7K6@b%MB4F`S3)?rx91Tn_Za791#mX z*UN_WZ(s6`>1o=&fDGH^H~BUEM>;*rn}qw$ zJTql4*z7;~KU$IDqCo2j|zfA^W=(A%j!+!#m2*h|-J&KCF8wl_#JZl0eC^_LsgjwPC{hT=8$wBxdduGEEm2Isu~IGmIGd|Ea2UE#=TP0D^MWCIw_?yd>W1p;)qqc?B;+&e*0B#VdFM`CpM!jYDSOUH#pa{Q5;rKC)EIFXPSL!C1mGxyZ@5k6JlQHGiKS_{j1 zFOjB4u(vzTci4Atiz)eWzv#bS>GuNEvXbVXau3(Cm|e;W$3>Af;l-8{8n76y0;jz< zE(!Fv`Ikra=f^~3`L`xiXHj18nS_}b11IKYS*wDZW-uO^v)2fn&F_Abi`H*llvj+& zE>3H7ch!xwLuI+$8}yyNT5{*KLH7O4Q=?0YnjgL=Fuz%D%^IArE3+5uu!)wuSUZGgWI^fM&77;JIlpmH`ynS(_E-{ zr}lQwzoO{_F*;x=AZ?BX@m%Jf?fsoU`KS1QKK=UqV&il|3#(EHE=Kzj|Lm!UL z75A5o_c~A=YRVayI-ei=bw;avZaV#;E0Fc}RAyTFqubt{EK!NU>NC|xy{bUJnA|08 zQka8&E1#hYSvTiPp*3Z>Ar+ljc?Y{%@-xxiqpbJuh{7kw$Si507g+*Q0$;i|) z9nhsVYn9OHe;PyPG4XaNBiB_#Q}L-p(;MaB@3K!;4L9vg&5RT(zN&F_@f#K}1!pwd za5=pl?Q2OZ=%Ts-dzHMMIFDoW(}`ww45 z;UQKpeJd`~3MPbp=p8hg;ihtXLmv9I6t)|$baHOt0%fbY+ z!ami+xVX5&_+t*v2Jghgw9rsFhvnZ`=gCNOU=G6PdcS9c+Ox~ zM`4#{I!8L^Pd-h1Igl=IWnMpDhB6(WOK0uz3QDI3moF$^FsMG-r|?g&uLu*Fe^bE| zx$L#U(*-3b`c363M>^t;I?b{zr{*H$Bx5#2Sx3N?7~bOsjY1RW8H|fn#SfIjz(w;UiJe@Or#{QLTog>?sZv1uIHXAu*-Ju6Io>MD* zWX7&6k@$oTCjmy7T*Xcu3j)a~ z(up;a$oAgRtpf4uoWXU4#cqX;x;;{xcdM!8IQz^4BRSaZt~vm1Vl#vH#RXNjG~w6D zqIAamksxl(CoSfO&5s7|>OFr-a-z7sW%lr4-*cvJW`i$3bgC6BsIMBY%JjjlALg%5 zuG_o^9qFj8HDW;7Uyqoa?)NU^wwWxxn9gX1vLvQvy)DXq_S~~Qa0KHsXIADIUTfe} z&RZw^=mh_FOBR_eF!RYF|3!)lqtoqPPNh)8sCfoujVy|jzPg78Oj}-g8DaAxHS;y> znm)`Z4RLifUMAH}xuf2!5WHrovE6#OTHT>UM)-M1Fk8*!lh#p&QoXmWeS>XvtUT$( z^qL`$OPFb8w7LT{4eMgW*~|FlKvyeRkN+*Eo``GPs%hT^O|@P zVqo=*)x0uFEI!Ztd$pOB0jF=G$GWCk!W5O_8jY!9PrikikWlJJGPQ(HGO~5%g{j3X z2J8xQ4#{hr5un}0##NADa^vewG+}KJ9QMN0kH2d-nAX@*;(OGl+U5y^$!anxJ>5;8 zcuwh+TEnt8NAGqis<9~RY2p%3umWjnW2eOA8rR3kERzqH9h|}+Ef(0c5H+M9CpJoB zBRYp!bZqOSXm!KdP?S6O=1`~|=%%1FS%1;xa*yrdqoxikFJD!9-D1+i5%7bm^$MC! zCkHPhNFyRm4u9tTlp$)|IZ?^Xs6ZCsKJ`k#Ne+>E+gfMYF=_i>+>~w_lc=D7>zbaOLax z6%M<~8>X7(n(pP!8Vn4~T`ObHdUIzJJ43Nz>nL@_LQ`wsBTmXyQ^;Yv_g}G2;$lM- z_~zKw$NqzSucWDxHPdga($L3gB*XbwYbDjW=}qrkLr-TSYykI79aY#RhhXWF<@in} zoTZj7uWj*`_?smU=Mp;`nikKp*e7#mo{Kw?h(a#wMJB`938WQQ13D8 z4^|uMr@1VI#vSd7SU(ht=vd_i$K0`?tgy~Uefx)&+hR&MJP!X2T$0SiK0{2u_k6p# zk}+ruI3H(SAsRY?lW=w=j}Rs6t}|({`oXHWl1f9Y+8}9tj@N{hsU|%$AAN=-m>k3bso3Z+M}T}O z@YL2ktyQON+neQ*CHAMmtd_6GhGqGICgH-wz9^s7_~O6IbvNWWlI|5eC{8aMGvnjDdetnP(Kk zm1j8D7nopAXTDyaU2NnWJhSo1ba23NL2#%yb7i9O_VgRc@xC6`#{A6^{ptD2!Crs< z!R@&y+mh)Segm@GYjf`YF_Sg(m@V*kGG02*%KGl{l+Yd{fD5=D!ob9W?famWU*6}^ zmpvF|zPhz7@80w%dN7vHoU?R$$|`1x>m=4I`ctSoe;fRB-T=aGzOI?A`M11lqie~i z?FVmoonEoJ&l#o`@Km0s$#~M@z3e`Qs3JAYfF)X2Qoxv$T7 zx(-*f5Oj5XRY&HC^UknQj9{3(po8Y1EkU!s$9Cipp2slu|NG8&Umo?s75>i9C|&f| zTJ|5J2*ayv#@{kos8nJzSy+m?{#TIh&J;v6Q9P3q`~F^TuG4oBci+>|C(ol>_2S8H z=@{9;)#(cj|L~kBK!ZBhLx^RVNLhq7FY`kZbSSzbJT$?-xNhV@x!8`4X6Xd*OEOx< zpXc$>E*J~OgpEZQLhO7XjqL#B1rChn7Kt?v+IQy=cWZ2YIf`yWAlJBg)d^M1VvjH9 zZRfq}pUQe=<(oD+{)zsc9*T7$KU#Fq#?qrBI8Jja>6)7|lEU|0DjE_!glHh^?%g1@ zMmsO8UQ~cUo8{SqGk;;aJz9J2>IxS^P#Yokr@3D&`+u0RoXHKQ-c_JYF8zh)TjAsJ zjJ1)6&gazyvd*IPJ4z}yYc{*Kw6}9yPO)lj=ezXsZ*JvIQRaXL|3Ti~;n|h+tkbh) zCEK>!55T``8s+Qevt`q<11{OcN500cG>zDufIhD@DXcb5mAS-NYlgKAia&-q4YD%h zvv^!QQL|{Z=|h&|oDn;(bo)A$UDX{#nkAqDT5*Cexd9$T1 Date: Sun, 8 Apr 2018 11:32:54 +0800 Subject: [PATCH 57/57] update release doc --- doc/fluid/dev/index_cn.rst | 2 +- doc/fluid/dev/index_en.rst | 2 +- ...ing_process.md => releasing_process_cn.md} | 54 +++-- doc/fluid/dev/releasing_process_en.md | 210 ++++++++++++++++++ 4 files changed, 242 insertions(+), 26 deletions(-) rename doc/fluid/dev/{releasing_process.md => releasing_process_cn.md} (74%) create mode 100644 doc/fluid/dev/releasing_process_en.md diff --git a/doc/fluid/dev/index_cn.rst b/doc/fluid/dev/index_cn.rst index f627437f35..b123b756e2 100644 --- a/doc/fluid/dev/index_cn.rst +++ b/doc/fluid/dev/index_cn.rst @@ -9,5 +9,5 @@ use_eigen_cn.md name_convention.md support_new_device.md - releasing_process.md + releasing_process_cn.md op_markdown_format.md diff --git a/doc/fluid/dev/index_en.rst b/doc/fluid/dev/index_en.rst index 0b65fed67a..98988fc22d 100644 --- a/doc/fluid/dev/index_en.rst +++ b/doc/fluid/dev/index_en.rst @@ -9,5 +9,5 @@ Development use_eigen_en.md name_convention.md support_new_device.md - releasing_process.md + releasing_process_en.md op_markdown_format.md diff --git a/doc/fluid/dev/releasing_process.md b/doc/fluid/dev/releasing_process_cn.md similarity index 74% rename from doc/fluid/dev/releasing_process.md rename to doc/fluid/dev/releasing_process_cn.md index c5943ccd81..4c6728fba7 100644 --- a/doc/fluid/dev/releasing_process.md +++ b/doc/fluid/dev/releasing_process_cn.md @@ -10,19 +10,10 @@ PaddlePaddle每次发新的版本，遵循以下流程: * 使用Regression Test List作为检查列表，测试本次release的正确性。 * 如果失败，记录下所有失败的例子，在这个`release/版本号`分支中，修复所有bug后，Patch号加一，到第二步 * 修改`python/setup.py.in`中的版本信息,并将`istaged`字段设为`True`。 - * 编译这个版本的python wheel包，并发布到pypi。 - * 由于pypi.python.org目前遵循[严格的命名规范PEP 513](https://www.python.org/dev/peps/pep-0513)，在使用twine上传之前，需要重命名wheel包中platform相关的后缀，比如将`linux_x86_64`修改成`manylinux1_x86_64`。 - * pypi上的package名称为paddlepaddle和paddlepaddle_gpu，如果要上传GPU版本的包，需要修改build/python/setup.py中，name: "paddlepaddle_gpu"并重新打包wheel包：`python setup.py bdist_wheel`。 - * 上传方法： - ``` - cd build/python - pip install twine - twine upload dist/[package to upload] - ``` - * 编译这个版本的Docker发行镜像，发布到dockerhub。如果失败，修复Docker编译镜像问题，Patch号加一，返回第二步 -1. 第三步完成后，将`release/版本号`分支合入master分支，并删除`release/版本号`分支。将master分支的合入commit打上tag，tag为`版本号`。同时再将`master`分支合入`develop`分支。最后删除`release/版本号`分支。 -1. 协同完成Release Note的书写 - + * 将这个版本的python wheel包发布到pypi。 + * 更新Docker镜像（参考后面的操作细节）。 +1. 第三步完成后，将`release/版本号`分支合入master分支，将master分支的合入commit打上tag，tag为`版本号`。同时再将`master`分支合入`develop`分支。 +1. 协同完成Release Note的书写。 需要注意的是: @@ -31,13 +22,18 @@ PaddlePaddle每次发新的版本，遵循以下流程: ## 发布wheel包到pypi -使用[PaddlePaddle CI](https://paddleci.ngrok.io/project.html?projectId=Manylinux1&tab=projectOverview) +1. 使用[PaddlePaddle CI](https://paddleci.ngrok.io/project.html?projectId=Manylinux1&tab=projectOverview) 完成自动化二进制编译，参考下图，选择需要发布的版本（通常包含一个CPU版本和一个GPU版本），点击"run"右侧的"..."按钮，可以 -弹出下面的选择框，在第二个tab (Changes)里选择需要发布的分支，这里选择0.11.0，然后点击"Run Build"按钮。等待编译完成后 -可以在此页面的"Artifacts"下拉框中找到生成的3个二进制文件，分别对应CAPI，`cp27m`和`cp27mu`的版本。然后按照上述的方法 -使用`twine`工具上传即可。 - - +弹出下面的选择框，在第二个tab (Changes)里选择需要发布的分支，这里选择0.11.0，然后点击"Run Build"按钮。 + +1. 等待编译完成后可以在此页面的"Artifacts"下拉框中找到生成的3个二进制文件，分别对应CAPI，`cp27m`和`cp27mu`的版本。 +1. 由于pypi.python.org目前遵循[严格的命名规范PEP 513](https://www.python.org/dev/peps/pep-0513)，在使用twine上传之前，需要重命名wheel包中platform相关的后缀，比如将`linux_x86_64`修改成`manylinux1_x86_64`。 +1. 上传： +``` +cd build/python +pip install twine +twine upload dist/[package to upload] +``` * 注：CI环境使用 https://github.com/PaddlePaddle/buildtools 这里的DockerImage作为编译环境以支持更多的Linux 发型版，如果需要手动编译，也可以使用这些镜像。这些镜像也可以从 https://hub.docker.com/r/paddlepaddle/paddle_manylinux_devel/tags/ 下载得到。 @@ -48,10 +44,20 @@ PaddlePaddle每次发新的版本，遵循以下流程: 上述PaddlePaddle CI编译wheel完成后会自动将Docker镜像push到DockerHub，所以，发布Docker镜像只需要对自动push的镜像打上 版本号对应的tag即可： -1. 进入 https://hub.docker.com/r/paddlepaddle/paddle/tags/ 查看latest tag的更新时间是否在上述编译wheel包完成后是否最新。 -1. 执行 `docker pull paddlepaddle/paddle:[latest tag]`，latest tag可以是latest或latest-gpu等。 -1. 执行 `docker tag paddlepaddle/paddle:[latest tag] paddlepaddle/paddle:[version]` -1. 执行 `docker push paddlepaddle/paddle:[version]` +``` +docker pull [镜像]:latest +docker tag [镜像]:latest [镜像]:[version] +docker push [镜像]:[version] +``` + +需要更新的镜像tag包括： + +* `[version]`: CPU版本 +* `[version]-openblas`: openblas版本 +* `[version]-gpu`: GPU版本（CUDA 8.0 cudnn 5） +* `[version]-gpu-[cudaver]-[cudnnver]`: 不同cuda, cudnn版本的镜像 + +之后可进入 https://hub.docker.com/r/paddlepaddle/paddle/tags/ 查看是否发布成功。 ## PaddlePaddle 分支规范 @@ -76,7 +82,7 @@ PaddlePaddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git- ### PaddlePaddle Book中所有章节 -PaddlePaddle每次发版本首先要保证PaddlePaddle Book中所有章节功能的正确性。功能的正确性包括验证PaddlePaddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 +PaddlePaddle每次发版本首先要保证PaddlePaddle Book中所有章节功能的正确性。功能的正确性包括验证PaddlePaddle目前的`paddle_trainer`训练和纯使用`Python`训练（V2和Fluid）模型正确性。 diff --git a/doc/fluid/dev/releasing_process_en.md b/doc/fluid/dev/releasing_process_en.md new file mode 100644 index 0000000000..f989b964d6 --- /dev/null +++ b/doc/fluid/dev/releasing_process_en.md @@ -0,0 +1,210 @@ +# PaddlePaddle Releasing Process + +PaddlePaddle manages its branches using "git-flow branching model", and [Semantic Versioning](http://semver.org/) as it's version number semantics. + +Each time we release a new PaddlePaddle version, we should follow the below steps: + +1. Fork a new branch from `develop` named `release/[version]`, e.g. `release/0.10.0`. +1. Push a new tag on the release branch, the tag name should be like `[version]rc.patch`. The + first tag should be `0.10.0rc1`, and the second should be `0.10.0.rc2` and so on. +1. After that, we should do: + * Run all regression test on the Regression Test List (see PaddlePaddle TeamCity CI), to confirm + that this release has no major bugs. + * If regression test fails, we must fix those bugs and create a new `release/[version]` + branch from previous release branch. + * Modify `python/setup.py.in`, change the version number and change `ISTAGED` to `True`. + * Publish PaddlePaddle release wheel packages to pypi (see below instructions for detail). + * Update the Docker images (see below instructions for detail). +1. After above step, merge `release/[version]` branch to master and push a tag on the master commit, + then merge `master` to `develop`. +1. Update the Release Note. + +***NOTE:*** + +* Do ***NOT*** merge commits from develop branch to release branches to keep the release branch contain + features only for current release, so that we can test on that version. +* If we want to fix bugs on release branches, we must merge the fix to master, develop and release branch. + +## Publish Wheel Packages to pypi + +1. Use our [CI tool](https://paddleci.ngrok.io/project.html?projectId=Manylinux1&tab=projectOverview) + to build all wheel packages needed to publish. As shown in the following picture, choose a build + version, click "..." button on the right side of "Run" button, and switch to the second tab in the +pop-up box, choose the current release branch and click "Run Build" button. You may repeat this + step to start different versions of builds. + +1. After the build succeeds, download the outputs under "Artifacts" including capi, `cp27m` and `cp27mu`. +1. Since pypi.python.org follows [PEP 513](https://www.python.org/dev/peps/pep-0513), before we + upload the package using `twine`, we need to rename the package from `linux_x86_64` to + `manylinux1_x86_64`. +1. Start the upload: + ``` + cd build/python + pip install twine + twine upload dist/[package to upload] + ``` + +* NOTE: We use a special Docker image to build our releases to support more Linux distributions, you can + download it from https://hub.docker.com/r/paddlepaddle/paddle_manylinux_devel/tags/, or build it using + scripts under `tools/manylinux1`. +* pypi does not allow overwrite the already uploaded version of wheel package, even if you delete the + old version. you must change the version number before upload a new one. + +## Publish Docker Images + +Our CI tool will push latest images to DockerHub, so we only need to push a version tag like: + +``` +docker pull [image]:latest +docker tag [image]:latest [image]:[version] +docker push [image]:[version] +``` + +Tags that need to be updated are: +* `[version]`: CPU only version image +* `[version]-openblas`: openblas version image +* `[version]-gpu`: GPU version（using CUDA 8.0 cudnn 5） +* `[version]-gpu-[cudaver]-[cudnnver]`: tag for different cuda, cudnn versions + +You can then checkout the latest pushed tags at https://hub.docker.com/r/paddlepaddle/paddle/tags/. + +## Branching Model + +We use [git-flow](http://nvie.com/posts/a-successful-git-branching-model/) as our branching model, +with some modifications: + +* `master` branch is the stable branch. Each version on the master branch is tested and guaranteed. +* `develop` branch is for development. Each commit on develop branch has passed CI unit test, but no + regression tests are run. +* `release/[version]` branch is used to publish each release. Latest release version branches have + bugfix only for that version, but no feature updates. +* Developer forks are not required to follow + [git-flow](http://nvie.com/posts/a-successful-git-branching-model/) + branching model, all forks is like a feature branch. + * Advise: developer fork's develop branch is used to sync up with main repo's develop branch. + * Advise: developer use it's fork's develop branch to for new branch to start developing. + * Use that branch on developer's fork to create pull requests and start reviews. + * developer can push new commits to that branch when the pull request is open. +* Bug fixes are also started from developers forked repo. And, bug fixes branch can merge to + `master`, `develop` and `releases`. + +## PaddlePaddle Regression Test List + +### All Chapters of PaddlePaddle Book + +We need to guarantee that all the chapters of PaddlePaddle Book can run correctly. Including +V1 (`paddle_trainer` training) and V2 training and Fluid training. + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Linear RegressionRecognize DigitsImage ClassificationWord2VecPersonalized RecommendationSentiment AnalysisSemantic Role LabelingMachine Translation
API.V2 + Docker + GPU
API.V2 + Docker + CPU
`paddle_trainer` + Docker + GPU
`paddle_trainer` + Docker + CPU
API.V2 + Ubuntu + GPU
API.V2 + Ubuntu + CPU
`paddle_trainer` + Ubuntu + GPU
`paddle_trainer` + Ubuntu + CPU