From dc8390d8c3038173d70d4c7cc9f4e76bb1ddc587 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Thu, 1 Feb 2018 13:46:35 -0800 Subject: [PATCH 01/26] initial commit --- .../tests/book/test_rnn_encoder_decoder.py | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py index fdc6086176..593d0013c9 100644 --- a/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py @@ -145,7 +145,7 @@ def seq_to_seq_net(): cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = fluid.layers.mean(x=cost) - return avg_cost + return avg_cost, prediction def to_lodtensor(data, place): @@ -163,8 +163,8 @@ def to_lodtensor(data, place): return res -def main(): - avg_cost = seq_to_seq_net() +def train(save_dirname=None): + [avg_cost, prediction] = seq_to_seq_net() optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) @@ -196,9 +196,52 @@ def main(): print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + " avg_cost=" + str(avg_cost_val)) if batch_id > 3: + if save_dirname is not None: + fluid.io.save_inference_model(save_dirname, [ + 'source_sequence', 'target_sequence', 'label_sequence' + ], [prediction], exe) exit(0) batch_id += 1 +def inference(save_dirname=None): + if save_dirname is None: + return + + place = fluid.CPUPlace() + exe = fluid.Executor(place) + + # Use fluid.io.load_inference_model to obtain the inference program desc, + # the feed_target_names (the names of variables that will be feeded + # data using feed operators), and the fetch_targets (variables that + # we want to obtain data from using fetch operators). + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + + data = [[0, 1, 0, 1], [0, 1, 1, 0, 0, 1]] + word_data = to_lodtensor(data, place) + trg_word = to_lodtensor(data, place) + trg_word_next = to_lodtensor(data, place) + + # Construct feed as a dictionary of {feed_target_name: feed_target_data} + # and results will contain a list of data corresponding to fetch_targets. + print(feed_target_names) + assert feed_target_names[0] == 'source_sequence' + assert feed_target_names[1] == 'target_sequence' + assert feed_target_names[2] == 'label_sequence' + results = exe.run(inference_program, + feed={ + feed_target_names[0]: word_data, + feed_target_names[1]: trg_word, + feed_target_names[2]: trg_word_next + }, + fetch_list=fetch_targets) + + print("Inference Shape: ", results[0].shape) + print("infer results: ", results[0]) + + if __name__ == '__main__': - main() + save_dirname = "rnn_encoder_decoder.inference.model" + train(save_dirname) + infer(save_dirname) From 71a70f209ac63b6f351bd3c399ddff804da090f7 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 31 Jan 2018 17:07:51 +0800 Subject: [PATCH 02/26] refine gradient --- paddle/operators/layer_norm_op.cc | 42 ++++++++++++------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/paddle/operators/layer_norm_op.cc b/paddle/operators/layer_norm_op.cc index 1c6d2ae4d0..8fcac00e08 100644 --- a/paddle/operators/layer_norm_op.cc +++ b/paddle/operators/layer_norm_op.cc @@ -291,32 +291,28 @@ class LayerNormGradKernel auto d_x_map = EigenMatrixMapRowMajor(d_x->data(), left, right); auto triple_product_func = [](T ele) { return ele * ele * ele; }; auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; + + auto inv_std_map = var_map.unaryExpr(inv_std_func).eval(); // TODO(zcd): these code can be refined if (d_scale) { auto scale_map = ConstEigenMatrixMapRowMajor(scale->data(), 1, right); // dy_dx - auto dx_end = var_map.unaryExpr(inv_std_func) - .replicate(1, right) - .cwiseProduct(d_y_map) - .cwiseProduct(scale_map.replicate(left, 1)); + auto dx_end = + inv_std_map.replicate(1, right).cwiseProduct(d_y_map).cwiseProduct( + scale_map.replicate(left, 1)); + // dy_dmean_dx - auto dx_mean = (T(-1.0) / right) * - var_map.unaryExpr(inv_std_func) - .replicate(1, right) - .cwiseProduct(d_y_map) - .cwiseProduct(scale_map.replicate(left, 1)) - .rowwise() - .sum() - .replicate(1, right); + auto dx_mean = + (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); + // dy_var_dx auto dvar_end_part = (x_map - mean_map.replicate(1, right)) .cwiseProduct(scale_map.replicate(left, 1)) .cwiseProduct(d_y_map) .rowwise() .sum(); - auto dvar_end = var_map.unaryExpr(inv_std_func) - .unaryExpr(triple_product_func) + auto dvar_end = inv_std_map.unaryExpr(triple_product_func) .cwiseProduct(dvar_end_part) .replicate(1, right); auto dx_var = @@ -326,24 +322,18 @@ class LayerNormGradKernel d_x_map = dx_end + dx_mean + dx_var; } else { // dy_dx - auto dx_end = var_map.unaryExpr(inv_std_func) - .replicate(1, right) - .cwiseProduct(d_y_map); + auto dx_end = inv_std_map.replicate(1, right).cwiseProduct(d_y_map); + // dy_dmean_dx - auto dx_mean = (T(-1.0) / right) * - var_map.unaryExpr(inv_std_func) - .replicate(1, right) - .cwiseProduct(d_y_map) - .rowwise() - .sum() - .replicate(1, right); + auto dx_mean = + (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); + // dy_var_dx auto dvar_end_part = (x_map - mean_map.replicate(1, right)) .cwiseProduct(d_y_map) .rowwise() .sum(); - auto dvar_end = var_map.unaryExpr(inv_std_func) - .unaryExpr(triple_product_func) + auto dvar_end = inv_std_map.unaryExpr(triple_product_func) .cwiseProduct(dvar_end_part) .replicate(1, right); auto dx_var = From 76e188e5c6b39fc7cccab9d5b64bd19163fafb77 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 2 Feb 2018 15:09:23 +0800 Subject: [PATCH 03/26] Add layer norm [GPU] --- paddle/operators/compare_op.h | 2 +- paddle/operators/elementwise_add_op.h | 3 +- paddle/operators/elementwise_div_op.h | 3 +- paddle/operators/elementwise_max_op.h | 3 +- paddle/operators/elementwise_min_op.h | 3 +- paddle/operators/elementwise_mul_op.h | 3 +- paddle/operators/elementwise_op_function.h | 4 +- paddle/operators/elementwise_pow_op.h | 3 +- paddle/operators/elementwise_sub_op.h | 3 +- paddle/operators/layer_norm_op.cc | 9 +- paddle/operators/layer_norm_op.cu | 245 +++++++++++++++++++++ paddle/operators/math/math_function.cc | 6 + paddle/operators/math/math_function.cu | 25 +++ paddle/operators/math/math_function.h | 12 + paddle/operators/math/math_function_impl.h | 82 +++++++ 15 files changed, 393 insertions(+), 13 deletions(-) create mode 100644 paddle/operators/layer_norm_op.cu diff --git a/paddle/operators/compare_op.h b/paddle/operators/compare_op.h index b275fd75b3..79b8c6f59c 100644 --- a/paddle/operators/compare_op.h +++ b/paddle/operators/compare_op.h @@ -62,7 +62,7 @@ class CompareOpKernel z->mutable_data(context.GetPlace()); int axis = context.Attr("axis"); ElementwiseComputeEx(context, x, y, axis, - z); + Functor(), z); } }; diff --git a/paddle/operators/elementwise_add_op.h b/paddle/operators/elementwise_add_op.h index c32288d698..c24f97a850 100644 --- a/paddle/operators/elementwise_add_op.h +++ b/paddle/operators/elementwise_add_op.h @@ -35,7 +35,8 @@ class ElementwiseAddKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + AddFunctor(), z); } }; diff --git a/paddle/operators/elementwise_div_op.h b/paddle/operators/elementwise_div_op.h index 07ebade31f..dc863cc598 100644 --- a/paddle/operators/elementwise_div_op.h +++ b/paddle/operators/elementwise_div_op.h @@ -35,7 +35,8 @@ class ElementwiseDivKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + DivFunctor(), z); } }; diff --git a/paddle/operators/elementwise_max_op.h b/paddle/operators/elementwise_max_op.h index 717e45ab31..67efe4e151 100644 --- a/paddle/operators/elementwise_max_op.h +++ b/paddle/operators/elementwise_max_op.h @@ -35,7 +35,8 @@ class ElementwiseMaxKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + MaxFunctor(), z); } }; diff --git a/paddle/operators/elementwise_min_op.h b/paddle/operators/elementwise_min_op.h index 0de9a91c52..cf11759404 100644 --- a/paddle/operators/elementwise_min_op.h +++ b/paddle/operators/elementwise_min_op.h @@ -35,7 +35,8 @@ class ElementwiseMinKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + MinFunctor(), z); } }; diff --git a/paddle/operators/elementwise_mul_op.h b/paddle/operators/elementwise_mul_op.h index ae7a71e024..773125f5ca 100644 --- a/paddle/operators/elementwise_mul_op.h +++ b/paddle/operators/elementwise_mul_op.h @@ -34,7 +34,8 @@ class ElementwiseMulKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + MulFunctor(), z); } }; diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h index 213fe1f5a8..74abf7c4a5 100644 --- a/paddle/operators/elementwise_op_function.h +++ b/paddle/operators/elementwise_op_function.h @@ -365,10 +365,10 @@ template void ElementwiseComputeEx(const framework::ExecutionContext& ctx, const framework::Tensor* x, - const framework::Tensor* y, int axis, + const framework::Tensor* y, int axis, Functor func, framework::Tensor* z) { TransformFunctor functor( - x, y, z, ctx.template device_context(), Functor()); + x, y, z, ctx.template device_context(), func); auto x_dims = x->dims(); auto y_dims = y->dims(); diff --git a/paddle/operators/elementwise_pow_op.h b/paddle/operators/elementwise_pow_op.h index 874fd3f09f..0c5dd031ec 100644 --- a/paddle/operators/elementwise_pow_op.h +++ b/paddle/operators/elementwise_pow_op.h @@ -36,7 +36,8 @@ class ElementwisePowKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + PowFunctor(), z); } }; diff --git a/paddle/operators/elementwise_sub_op.h b/paddle/operators/elementwise_sub_op.h index c2749a8e6b..6a88c5f6b4 100644 --- a/paddle/operators/elementwise_sub_op.h +++ b/paddle/operators/elementwise_sub_op.h @@ -34,7 +34,8 @@ class ElementwiseSubKernel : public framework::OpKernel { auto* z = ctx.Output("Out"); z->mutable_data(ctx.GetPlace()); int axis = ctx.Attr("axis"); - ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, z); + ElementwiseComputeEx, DeviceContext, T>(ctx, x, y, axis, + SubFunctor(), z); } }; diff --git a/paddle/operators/layer_norm_op.cc b/paddle/operators/layer_norm_op.cc index 8fcac00e08..6dd18277c9 100644 --- a/paddle/operators/layer_norm_op.cc +++ b/paddle/operators/layer_norm_op.cc @@ -13,6 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/layer_norm_op.h" +#include "paddle/operators/elementwise_op_function.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { @@ -353,8 +355,9 @@ namespace ops = paddle::operators; REGISTER_OP(layer_norm, ops::LayerNormOp, ops::LayerNormOpMaker, layer_norm_grad, ops::LayerNormGradOp); REGISTER_OP_CPU_KERNEL( - layer_norm, - ops::LayerNormKernel); + layer_norm, ops::LayerNormKernel, + ops::LayerNormKernel); REGISTER_OP_CPU_KERNEL( layer_norm_grad, - ops::LayerNormGradKernel); + ops::LayerNormGradKernel, + ops::LayerNormGradKernel); diff --git a/paddle/operators/layer_norm_op.cu b/paddle/operators/layer_norm_op.cu new file mode 100644 index 0000000000..a84f5a41ea --- /dev/null +++ b/paddle/operators/layer_norm_op.cu @@ -0,0 +1,245 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/elementwise_op_function.h" +#include "paddle/operators/layer_norm_op.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; +using DataLayout = framework::DataLayout; + +namespace { +template +struct SubAndSquareFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return (a - b) * (a - b); } +}; + +template +struct DivAndSqrtFunctor { + explicit DivAndSqrtFunctor(T epsilon) { epsilon_ = epsilon; } + inline HOSTDEVICE T operator()(T a, T b) const { + return a / (sqrt(b) + epsilon_); + } + + private: + T epsilon_; +}; + +template +struct MulFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a * b; } +}; + +template +struct AddFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a + b; } +}; + +template +struct SubFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a - b; } +}; + +template +struct MulInvVarFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { + return a * std::sqrt(1.0 / b); + } +}; +} // namespace + +template +class LayerNormCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + const float epsilon = ctx.Attr("epsilon"); + auto *scale = ctx.Input("Scale"); + auto *bias = ctx.Input("Bias"); + auto x = *ctx.Input("X"); + + auto *y = ctx.Output("Y"); + auto *mean = ctx.Output("Mean"); + auto *var = ctx.Output("Variance"); + const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); + + const auto &x_dims = x.dims(); + + y->mutable_data(ctx.GetPlace()); + mean->mutable_data(ctx.GetPlace()); + var->mutable_data(ctx.GetPlace()); + + auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); + int left = static_cast(matrix_dim[0]); + int right = static_cast(matrix_dim[1]); + + framework::DDim matrix_shape({left, right}); + + x.Resize(matrix_shape); + y->Resize(matrix_shape); + + auto &dev_ctx = ctx.template device_context(); + math::RowwiseMean row_mean; + + // functor-> get mean + row_mean(dev_ctx, x, mean); + + // functor-> get variance + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, mean, /*axis*/ 0, SubAndSquareFunctor(), y); + row_mean(dev_ctx, *y, var); + + // functor-> get norm_out + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, mean, /*axis*/ 0, SubFunctor(), y); + ElementwiseComputeEx, DeviceContext, T>( + ctx, y, var, /*axis*/ 0, DivAndSqrtFunctor(static_cast(epsilon)), + y); + + framework::DDim scale_shape({right}); + if (scale) { + Tensor scale_matrix = *scale; + scale_matrix.Resize(scale_shape); + ElementwiseComputeEx, DeviceContext, T>( + ctx, y, &scale_matrix, /*axis*/ 1, MulFunctor(), y); + } + if (bias) { + Tensor bias_matrix = *bias; + bias_matrix.Resize(scale_shape); + ElementwiseComputeEx, DeviceContext, T>( + ctx, y, &bias_matrix, /*axis*/ 1, AddFunctor(), y); + } + y->Resize(x_dims); + } +}; + +template +class LayerNormCUDAGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + const float epsilon = ctx.Attr("epsilon"); + auto x = *ctx.Input("X"); + auto mean = *ctx.Input("Mean"); + auto var = *ctx.Input("Variance"); + auto scale = *ctx.Input("Scale"); + auto d_y = *ctx.Input(framework::GradVarName("Y")); + const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); + + // init output + auto *d_x = ctx.Output(framework::GradVarName("X")); + auto *d_scale = ctx.Output(framework::GradVarName("Scale")); + auto *d_bias = ctx.Output(framework::GradVarName("Bias")); + + const auto &x_dims = x.dims(); + auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); + int left = static_cast(matrix_dim[0]); + int right = static_cast(matrix_dim[1]); + framework::DDim matrix_shape({left, right}); + + d_y.Resize(matrix_shape); + auto &dev_ctx = ctx.template device_context(); + math::ColwiseSum colwise_sum; + + Tensor temp; + Tensor temp_norm; + if (d_scale || d_x) { + x.Resize(matrix_shape); + temp.mutable_data(matrix_shape, ctx.GetPlace()); + temp_norm.mutable_data(matrix_shape, ctx.GetPlace()); + + // get x_norm + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, &mean, /*axis*/ 0, SubFunctor(), &temp_norm); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, &var, /*axis*/ 0, + DivAndSqrtFunctor(static_cast(epsilon)), &temp_norm); + } + + if (d_bias) { + d_bias->mutable_data(ctx.GetPlace()); + colwise_sum(dev_ctx, d_y, d_bias); + } + if (d_scale) { + d_scale->mutable_data(ctx.GetPlace()); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, &d_y, /*axis*/ 0, MulFunctor(), &temp); + colwise_sum(dev_ctx, temp, d_scale); + } + + if (d_x) { + framework::DDim vec_shape({left}); + d_x->mutable_data(ctx.GetPlace()); + Tensor temp_vec; + temp_vec.mutable_data(vec_shape, ctx.GetPlace()); + + auto &dev_ctx = ctx.template device_context(); + math::RowwiseMean row_mean; + + if (d_scale) { + // dy_dx + ElementwiseComputeEx, DeviceContext, T>( + ctx, &d_y, &scale, /*axis*/ 1, MulFunctor(), &temp); + framework::Copy(temp, ctx.GetPlace(), ctx.device_context(), d_x); + + // dy_dmean_dx + row_mean(dev_ctx, temp, &temp_vec); + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &temp_vec, /*axis*/ 0, SubFunctor(), d_x); + + // dy_var_dx + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); + + } else { + // dy_dx + framework::Copy(d_y, ctx.GetPlace(), ctx.device_context(), d_x); + + // dy_dmean_dx + row_mean(dev_ctx, d_y, &temp_vec); + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &temp_vec, /*axis*/ 0, SubFunctor(), d_x); + + // dy_var_dx + ElementwiseComputeEx, DeviceContext, T>( + ctx, &d_y, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); + } + // dy_var_dx + row_mean(dev_ctx, temp, &temp_vec); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, &temp_vec, /*axis*/ 0, MulFunctor(), &temp_norm); + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &temp_norm, /*axis*/ 0, SubFunctor(), d_x); + + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &var, /*axis*/ 0, + DivAndSqrtFunctor(static_cast(epsilon)), d_x); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL( + layer_norm, + ops::LayerNormCUDAKernel, + ops::LayerNormCUDAKernel); +REGISTER_OP_CUDA_KERNEL( + layer_norm_grad, + ops::LayerNormCUDAGradKernel, + ops::LayerNormCUDAGradKernel); diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index dcf4b85e1a..ce0a5f6cff 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -331,6 +331,12 @@ template struct RowwiseAdd; template struct ColwiseSum; template struct ColwiseSum; +template struct RowwiseSum; +template struct RowwiseSum; + +template struct RowwiseMean; +template struct RowwiseMean; + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index d47a7f818d..c0a107470a 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -325,6 +325,31 @@ void ColwiseSum::operator()( vector->data()); } +template struct RowwiseSum; +// template struct RowwiseSum; +// TODO(zcd): Following ColwiseSum format, need to confirm. +// The RowwiseSum failed in debug mode, +// and only failed for this case. So reimplemented it. +template <> +void RowwiseSum::operator()( + const platform::CUDADeviceContext& context, const framework::Tensor& input, + framework::Tensor* vector) { + auto in_dims = input.dims(); + auto size = input.numel() / in_dims[0]; + PADDLE_ENFORCE_EQ(vector->numel(), in_dims[0]); + framework::Tensor one; + one.mutable_data({size}, context.GetPlace()); + SetConstant set; + set(context, &one, static_cast(1.0)); + gemv( + context, true, static_cast(in_dims[1]), static_cast(in_dims[0]), + 1.0, one.data(), input.data(), 0.0, + vector->data()); +} + +template struct RowwiseMean; +template struct RowwiseMean; + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index 8cc03c2ba0..cb14d1e574 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -128,6 +128,18 @@ struct ColwiseSum { framework::Tensor* vec); }; +template +struct RowwiseSum { + void operator()(const DeviceContext& context, const framework::Tensor& input, + framework::Tensor* vec); +}; + +template +struct RowwiseMean { + void operator()(const DeviceContext& context, const framework::Tensor& input, + framework::Tensor* vec); +}; + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function_impl.h b/paddle/operators/math/math_function_impl.h index de591626df..af4127788a 100644 --- a/paddle/operators/math/math_function_impl.h +++ b/paddle/operators/math/math_function_impl.h @@ -87,6 +87,88 @@ class ColwiseSum { } }; +template +void RowwiseMean::operator()(const DeviceContext& context, + const framework::Tensor& input, + framework::Tensor* out) { + auto in_dims = input.dims(); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U); + PADDLE_ENFORCE_EQ(out->numel(), in_dims[0]); + + auto in = framework::EigenMatrix::From(input); + auto vec = framework::EigenVector::Flatten(*out); + + vec.device(*context.eigen_device()) = in.mean(Eigen::array({{1}})); +} +// TODO(zcd): Following ColwiseSum format, need to confirm. +// Specialize for CPU, since Eigen implement a general reduce. However, +// rowwise-sum can be easily implemented. General reduce has a huge overhead in +// CPU +template +class RowwiseMean { + public: + void operator()(const platform::CPUDeviceContext& context, + const framework::Tensor& input, framework::Tensor* out) { + auto& in_dims = input.dims(); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U); + auto height = in_dims[0]; + auto size = in_dims[1]; + PADDLE_ENFORCE_EQ(out->numel(), height); + auto inv_size = 1.0 / size; + T* out_buf = out->mutable_data(out->place()); + const T* in_buf = input.data(); + + for (size_t i = 0; i < static_cast(height); ++i) { + T sum = 0; + for (size_t j = 0; j < static_cast(size); ++j) { + sum += in_buf[i * size + j]; + } + out_buf[i] = sum * inv_size; + } + } +}; + +template +void RowwiseSum::operator()(const DeviceContext& context, + const framework::Tensor& input, + framework::Tensor* out) { + auto in_dims = input.dims(); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U); + PADDLE_ENFORCE_EQ(out->numel(), in_dims[0]); + + auto in = framework::EigenMatrix::From(input); + auto vec = framework::EigenVector::Flatten(*out); + + vec.device(*context.eigen_device()) = in.sum(Eigen::array({{1}})); +} +// TODO(zcd): Following ColwiseSum format, need to confirm. +// Specialize for CPU, since Eigen implement a general reduce. However, +// rowwise-sum can be easily implemented. General reduce has a huge overhead in +// CPU +template +class RowwiseSum { + public: + void operator()(const platform::CPUDeviceContext& context, + const framework::Tensor& input, framework::Tensor* out) { + auto& in_dims = input.dims(); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U); + auto height = in_dims[0]; + auto size = in_dims[1]; + PADDLE_ENFORCE_EQ(out->numel(), size); + + T* out_buf = out->mutable_data(out->place()); + const T* in_buf = input.data(); + + for (size_t i = 0; i < static_cast(height); ++i) { + T sum = 0; + for (size_t j = 0; j < static_cast(size); ++j) { + sum += in_buf[i * size + j]; + } + out_buf[i] = sum; + } + } +}; + } // namespace math } // namespace operators } // namespace paddle From e03337350356626f36cdff371a3e82292e25a1c8 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Sat, 3 Feb 2018 14:34:29 +0800 Subject: [PATCH 04/26] unifid GPU and CPU implementation --- paddle/operators/layer_norm_op.cc | 187 ------------------------ paddle/operators/layer_norm_op.cu | 228 +----------------------------- paddle/operators/layer_norm_op.h | 204 +++++++++++++++++++++++++- 3 files changed, 206 insertions(+), 413 deletions(-) diff --git a/paddle/operators/layer_norm_op.cc b/paddle/operators/layer_norm_op.cc index 6dd18277c9..edc26dfb96 100644 --- a/paddle/operators/layer_norm_op.cc +++ b/paddle/operators/layer_norm_op.cc @@ -13,8 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/layer_norm_op.h" -#include "paddle/operators/elementwise_op_function.h" -#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { @@ -23,13 +21,6 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; using DataLayout = framework::DataLayout; -template -using EigenMatrixMapRowMajor = Eigen::Map< - Eigen::Matrix>; -template -using ConstEigenMatrixMapRowMajor = Eigen::Map< - const Eigen::Matrix>; - class LayerNormOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -118,75 +109,6 @@ https://arxiv.org/abs/1607.06450 } }; -template -class LayerNormKernel - : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - const float epsilon = ctx.Attr("epsilon"); - const auto *scale = ctx.Input("Scale"); - const auto *bias = ctx.Input("Bias"); - const auto *x = ctx.Input("X"); - const auto &x_dims = x->dims(); - const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - - auto *output = ctx.Output("Y"); - auto *mean = ctx.Output("Mean"); - auto *var = ctx.Output("Variance"); - output->mutable_data(ctx.GetPlace()); - mean->mutable_data(ctx.GetPlace()); - var->mutable_data(ctx.GetPlace()); - - auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int left = static_cast(matrix_dim[0]); - int right = static_cast(matrix_dim[1]); - - auto input_map = ConstEigenMatrixMapRowMajor(x->data(), left, right); - - auto mean_map = EigenMatrixMapRowMajor(mean->data(), left, 1); - auto var_map = EigenMatrixMapRowMajor(var->data(), left, 1); - auto output_map = EigenMatrixMapRowMajor(output->data(), left, right); - - auto squre = [](T ele) { return ele * ele; }; - auto add_epslion = [epsilon](T ele) { return ele + epsilon; }; - - mean_map = input_map.rowwise().mean(); - var_map = (input_map - mean_map.replicate(1, right)) - .unaryExpr(squre) - .rowwise() - .mean() - .unaryExpr(add_epslion); - - auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; - // TODO(zcd): Some thinking about output_map, is it appropriate that - // `output_map` and `input_map` point to the same memory. - auto inv_std = var_map.unaryExpr(inv_std_func); - if (scale && bias) { - auto scale_map = - ConstEigenMatrixMapRowMajor(scale->data(), 1, right); - auto bias_map = ConstEigenMatrixMapRowMajor(bias->data(), 1, right); - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)) - .cwiseProduct(scale_map.replicate(left, 1)) + - bias_map.replicate(left, 1); - } else if (scale) { - auto scale_map = - ConstEigenMatrixMapRowMajor(scale->data(), 1, right); - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)) - .cwiseProduct(scale_map.replicate(left, 1)); - } else if (bias) { - auto bias_map = ConstEigenMatrixMapRowMajor(bias->data(), 1, right); - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)) + - bias_map.replicate(left, 1); - } else { - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)); - } - } -}; - class LayerNormGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -239,115 +161,6 @@ class LayerNormGradOp : public framework::OperatorWithKernel { } }; -template -class LayerNormGradKernel - : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - const auto *x = ctx.Input("X"); - const auto *mean = ctx.Input("Mean"); - const auto *var = ctx.Input("Variance"); - const auto *scale = ctx.Input("Scale"); - const auto *d_y = ctx.Input(framework::GradVarName("Y")); - - const auto &x_dims = x->dims(); - - const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int left = static_cast(matrix_dim[0]); - int right = static_cast(matrix_dim[1]); - - // init output - auto *d_x = ctx.Output(framework::GradVarName("X")); - auto *d_scale = ctx.Output(framework::GradVarName("Scale")); - auto *d_bias = ctx.Output(framework::GradVarName("Bias")); - - auto x_map = ConstEigenMatrixMapRowMajor(x->data(), left, right); - auto d_y_map = ConstEigenMatrixMapRowMajor(d_y->data(), left, right); - auto mean_map = ConstEigenMatrixMapRowMajor(mean->data(), left, 1); - auto var_map = ConstEigenMatrixMapRowMajor(var->data(), left, 1); - - if (d_bias) { - d_bias->mutable_data(ctx.GetPlace()); - auto d_bias_map = EigenMatrixMapRowMajor(d_bias->data(), 1, right); - d_bias_map = d_y_map.colwise().sum(); - } - if (d_scale) { - d_scale->mutable_data(ctx.GetPlace()); - auto d_scale_map = - EigenMatrixMapRowMajor(d_scale->data(), 1, right); - auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; - // There are two equation to compute d_scale. One uses "Y" and the other - // does not use "Y" - d_scale_map = - ((x_map - mean_map.replicate(1, right)) - .cwiseProduct( - var_map.unaryExpr(inv_std_func).replicate(1, right)) - .cwiseProduct(d_y_map)) - .colwise() - .sum(); - } - - if (d_x) { - d_x->mutable_data(ctx.GetPlace()); - auto d_x_map = EigenMatrixMapRowMajor(d_x->data(), left, right); - auto triple_product_func = [](T ele) { return ele * ele * ele; }; - auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; - - auto inv_std_map = var_map.unaryExpr(inv_std_func).eval(); - // TODO(zcd): these code can be refined - if (d_scale) { - auto scale_map = - ConstEigenMatrixMapRowMajor(scale->data(), 1, right); - // dy_dx - auto dx_end = - inv_std_map.replicate(1, right).cwiseProduct(d_y_map).cwiseProduct( - scale_map.replicate(left, 1)); - - // dy_dmean_dx - auto dx_mean = - (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); - - // dy_var_dx - auto dvar_end_part = (x_map - mean_map.replicate(1, right)) - .cwiseProduct(scale_map.replicate(left, 1)) - .cwiseProduct(d_y_map) - .rowwise() - .sum(); - auto dvar_end = inv_std_map.unaryExpr(triple_product_func) - .cwiseProduct(dvar_end_part) - .replicate(1, right); - auto dx_var = - (T(-1.0) / right) * - (x_map - mean_map.replicate(1, right)).cwiseProduct(dvar_end); - - d_x_map = dx_end + dx_mean + dx_var; - } else { - // dy_dx - auto dx_end = inv_std_map.replicate(1, right).cwiseProduct(d_y_map); - - // dy_dmean_dx - auto dx_mean = - (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); - - // dy_var_dx - auto dvar_end_part = (x_map - mean_map.replicate(1, right)) - .cwiseProduct(d_y_map) - .rowwise() - .sum(); - auto dvar_end = inv_std_map.unaryExpr(triple_product_func) - .cwiseProduct(dvar_end_part) - .replicate(1, right); - auto dx_var = - (T(-1.0) / right) * - (x_map - mean_map.replicate(1, right)).cwiseProduct(dvar_end); - - d_x_map = dx_end + dx_mean + dx_var; - } - } - } -}; - } // namespace operators } // namespace paddle diff --git a/paddle/operators/layer_norm_op.cu b/paddle/operators/layer_norm_op.cu index a84f5a41ea..77d13b216f 100644 --- a/paddle/operators/layer_norm_op.cu +++ b/paddle/operators/layer_norm_op.cu @@ -12,234 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/elementwise_op_function.h" #include "paddle/operators/layer_norm_op.h" -#include "paddle/operators/math/math_function.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; -using LoDTensor = framework::LoDTensor; -using DataLayout = framework::DataLayout; - -namespace { -template -struct SubAndSquareFunctor { - inline HOSTDEVICE T operator()(T a, T b) const { return (a - b) * (a - b); } -}; - -template -struct DivAndSqrtFunctor { - explicit DivAndSqrtFunctor(T epsilon) { epsilon_ = epsilon; } - inline HOSTDEVICE T operator()(T a, T b) const { - return a / (sqrt(b) + epsilon_); - } - - private: - T epsilon_; -}; - -template -struct MulFunctor { - inline HOSTDEVICE T operator()(T a, T b) const { return a * b; } -}; - -template -struct AddFunctor { - inline HOSTDEVICE T operator()(T a, T b) const { return a + b; } -}; - -template -struct SubFunctor { - inline HOSTDEVICE T operator()(T a, T b) const { return a - b; } -}; - -template -struct MulInvVarFunctor { - inline HOSTDEVICE T operator()(T a, T b) const { - return a * std::sqrt(1.0 / b); - } -}; -} // namespace - -template -class LayerNormCUDAKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - const float epsilon = ctx.Attr("epsilon"); - auto *scale = ctx.Input("Scale"); - auto *bias = ctx.Input("Bias"); - auto x = *ctx.Input("X"); - - auto *y = ctx.Output("Y"); - auto *mean = ctx.Output("Mean"); - auto *var = ctx.Output("Variance"); - const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - - const auto &x_dims = x.dims(); - - y->mutable_data(ctx.GetPlace()); - mean->mutable_data(ctx.GetPlace()); - var->mutable_data(ctx.GetPlace()); - - auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int left = static_cast(matrix_dim[0]); - int right = static_cast(matrix_dim[1]); - - framework::DDim matrix_shape({left, right}); - - x.Resize(matrix_shape); - y->Resize(matrix_shape); - - auto &dev_ctx = ctx.template device_context(); - math::RowwiseMean row_mean; - - // functor-> get mean - row_mean(dev_ctx, x, mean); - - // functor-> get variance - ElementwiseComputeEx, DeviceContext, T>( - ctx, &x, mean, /*axis*/ 0, SubAndSquareFunctor(), y); - row_mean(dev_ctx, *y, var); - - // functor-> get norm_out - ElementwiseComputeEx, DeviceContext, T>( - ctx, &x, mean, /*axis*/ 0, SubFunctor(), y); - ElementwiseComputeEx, DeviceContext, T>( - ctx, y, var, /*axis*/ 0, DivAndSqrtFunctor(static_cast(epsilon)), - y); - - framework::DDim scale_shape({right}); - if (scale) { - Tensor scale_matrix = *scale; - scale_matrix.Resize(scale_shape); - ElementwiseComputeEx, DeviceContext, T>( - ctx, y, &scale_matrix, /*axis*/ 1, MulFunctor(), y); - } - if (bias) { - Tensor bias_matrix = *bias; - bias_matrix.Resize(scale_shape); - ElementwiseComputeEx, DeviceContext, T>( - ctx, y, &bias_matrix, /*axis*/ 1, AddFunctor(), y); - } - y->Resize(x_dims); - } -}; - -template -class LayerNormCUDAGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - const float epsilon = ctx.Attr("epsilon"); - auto x = *ctx.Input("X"); - auto mean = *ctx.Input("Mean"); - auto var = *ctx.Input("Variance"); - auto scale = *ctx.Input("Scale"); - auto d_y = *ctx.Input(framework::GradVarName("Y")); - const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - - // init output - auto *d_x = ctx.Output(framework::GradVarName("X")); - auto *d_scale = ctx.Output(framework::GradVarName("Scale")); - auto *d_bias = ctx.Output(framework::GradVarName("Bias")); - - const auto &x_dims = x.dims(); - auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int left = static_cast(matrix_dim[0]); - int right = static_cast(matrix_dim[1]); - framework::DDim matrix_shape({left, right}); - - d_y.Resize(matrix_shape); - auto &dev_ctx = ctx.template device_context(); - math::ColwiseSum colwise_sum; - - Tensor temp; - Tensor temp_norm; - if (d_scale || d_x) { - x.Resize(matrix_shape); - temp.mutable_data(matrix_shape, ctx.GetPlace()); - temp_norm.mutable_data(matrix_shape, ctx.GetPlace()); - - // get x_norm - ElementwiseComputeEx, DeviceContext, T>( - ctx, &x, &mean, /*axis*/ 0, SubFunctor(), &temp_norm); - ElementwiseComputeEx, DeviceContext, T>( - ctx, &temp_norm, &var, /*axis*/ 0, - DivAndSqrtFunctor(static_cast(epsilon)), &temp_norm); - } - - if (d_bias) { - d_bias->mutable_data(ctx.GetPlace()); - colwise_sum(dev_ctx, d_y, d_bias); - } - if (d_scale) { - d_scale->mutable_data(ctx.GetPlace()); - ElementwiseComputeEx, DeviceContext, T>( - ctx, &temp_norm, &d_y, /*axis*/ 0, MulFunctor(), &temp); - colwise_sum(dev_ctx, temp, d_scale); - } - - if (d_x) { - framework::DDim vec_shape({left}); - d_x->mutable_data(ctx.GetPlace()); - Tensor temp_vec; - temp_vec.mutable_data(vec_shape, ctx.GetPlace()); - - auto &dev_ctx = ctx.template device_context(); - math::RowwiseMean row_mean; - - if (d_scale) { - // dy_dx - ElementwiseComputeEx, DeviceContext, T>( - ctx, &d_y, &scale, /*axis*/ 1, MulFunctor(), &temp); - framework::Copy(temp, ctx.GetPlace(), ctx.device_context(), d_x); - - // dy_dmean_dx - row_mean(dev_ctx, temp, &temp_vec); - ElementwiseComputeEx, DeviceContext, T>( - ctx, d_x, &temp_vec, /*axis*/ 0, SubFunctor(), d_x); - - // dy_var_dx - ElementwiseComputeEx, DeviceContext, T>( - ctx, &temp, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); - - } else { - // dy_dx - framework::Copy(d_y, ctx.GetPlace(), ctx.device_context(), d_x); - - // dy_dmean_dx - row_mean(dev_ctx, d_y, &temp_vec); - ElementwiseComputeEx, DeviceContext, T>( - ctx, d_x, &temp_vec, /*axis*/ 0, SubFunctor(), d_x); - - // dy_var_dx - ElementwiseComputeEx, DeviceContext, T>( - ctx, &d_y, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); - } - // dy_var_dx - row_mean(dev_ctx, temp, &temp_vec); - ElementwiseComputeEx, DeviceContext, T>( - ctx, &temp_norm, &temp_vec, /*axis*/ 0, MulFunctor(), &temp_norm); - ElementwiseComputeEx, DeviceContext, T>( - ctx, d_x, &temp_norm, /*axis*/ 0, SubFunctor(), d_x); - - ElementwiseComputeEx, DeviceContext, T>( - ctx, d_x, &var, /*axis*/ 0, - DivAndSqrtFunctor(static_cast(epsilon)), d_x); - } - } -}; - -} // namespace operators -} // namespace paddle namespace ops = paddle::operators; REGISTER_OP_CUDA_KERNEL( layer_norm, - ops::LayerNormCUDAKernel, - ops::LayerNormCUDAKernel); + ops::LayerNormKernel, + ops::LayerNormKernel); REGISTER_OP_CUDA_KERNEL( layer_norm_grad, - ops::LayerNormCUDAGradKernel, - ops::LayerNormCUDAGradKernel); + ops::LayerNormGradKernel, + ops::LayerNormGradKernel); diff --git a/paddle/operators/layer_norm_op.h b/paddle/operators/layer_norm_op.h index bca35b91e6..309f1b87a2 100644 --- a/paddle/operators/layer_norm_op.h +++ b/paddle/operators/layer_norm_op.h @@ -16,19 +16,219 @@ limitations under the License. */ #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/elementwise_op_function.h" +#include "paddle/operators/math/math_function.h" + namespace paddle { namespace operators { +template +struct SubAndSquareFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return (a - b) * (a - b); } +}; + +template +struct DivAndSqrtFunctor { + explicit DivAndSqrtFunctor(T epsilon) { epsilon_ = epsilon; } + inline HOSTDEVICE T operator()(T a, T b) const { + return a / (sqrt(b) + epsilon_); + } + + private: + T epsilon_; +}; + +template +struct MulFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a * b; } +}; + +template +struct AddFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a + b; } +}; + +template +struct SubFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { return a - b; } +}; + +template +struct MulInvVarFunctor { + inline HOSTDEVICE T operator()(T a, T b) const { + return a * std::sqrt(1.0 / b); + } +}; + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; +using DataLayout = framework::DataLayout; + template class LayerNormKernel : public framework::OpKernel { public: - void Compute(const framework::ExecutionContext& ctx) const override; + void Compute(const framework::ExecutionContext &ctx) const override { + const float epsilon = ctx.Attr("epsilon"); + auto *scale = ctx.Input("Scale"); + auto *bias = ctx.Input("Bias"); + auto x = *ctx.Input("X"); + + auto *y = ctx.Output("Y"); + auto *mean = ctx.Output("Mean"); + auto *var = ctx.Output("Variance"); + const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); + + const auto &x_dims = x.dims(); + + y->mutable_data(ctx.GetPlace()); + mean->mutable_data(ctx.GetPlace()); + var->mutable_data(ctx.GetPlace()); + + auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); + int left = static_cast(matrix_dim[0]); + int right = static_cast(matrix_dim[1]); + + framework::DDim matrix_shape({left, right}); + + x.Resize(matrix_shape); + y->Resize(matrix_shape); + + auto &dev_ctx = ctx.template device_context(); + math::RowwiseMean row_mean; + + // functor-> get mean + row_mean(dev_ctx, x, mean); + + // functor-> get variance + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, mean, /*axis*/ 0, SubAndSquareFunctor(), y); + row_mean(dev_ctx, *y, var); + + // functor-> get norm_out + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, mean, /*axis*/ 0, SubFunctor(), y); + ElementwiseComputeEx, DeviceContext, T>( + ctx, y, var, /*axis*/ 0, DivAndSqrtFunctor(static_cast(epsilon)), + y); + + framework::DDim scale_shape({right}); + if (scale) { + Tensor scale_matrix = *scale; + scale_matrix.Resize(scale_shape); + ElementwiseComputeEx, DeviceContext, T>( + ctx, y, &scale_matrix, /*axis*/ 1, MulFunctor(), y); + } + if (bias) { + Tensor bias_matrix = *bias; + bias_matrix.Resize(scale_shape); + ElementwiseComputeEx, DeviceContext, T>( + ctx, y, &bias_matrix, /*axis*/ 1, AddFunctor(), y); + } + y->Resize(x_dims); + } }; template class LayerNormGradKernel : public framework::OpKernel { public: - void Compute(const framework::ExecutionContext& ctx) const override; + void Compute(const framework::ExecutionContext &ctx) const override { + const float epsilon = ctx.Attr("epsilon"); + auto x = *ctx.Input("X"); + auto mean = *ctx.Input("Mean"); + auto var = *ctx.Input("Variance"); + auto scale = *ctx.Input("Scale"); + auto d_y = *ctx.Input(framework::GradVarName("Y")); + const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); + + // init output + auto *d_x = ctx.Output(framework::GradVarName("X")); + auto *d_scale = ctx.Output(framework::GradVarName("Scale")); + auto *d_bias = ctx.Output(framework::GradVarName("Bias")); + + const auto &x_dims = x.dims(); + auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); + int left = static_cast(matrix_dim[0]); + int right = static_cast(matrix_dim[1]); + framework::DDim matrix_shape({left, right}); + + d_y.Resize(matrix_shape); + auto &dev_ctx = ctx.template device_context(); + math::ColwiseSum colwise_sum; + + Tensor temp; + Tensor temp_norm; + if (d_scale || d_x) { + x.Resize(matrix_shape); + temp.mutable_data(matrix_shape, ctx.GetPlace()); + temp_norm.mutable_data(matrix_shape, ctx.GetPlace()); + + // get x_norm + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, &mean, /*axis*/ 0, SubFunctor(), &temp_norm); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, &var, /*axis*/ 0, + DivAndSqrtFunctor(static_cast(epsilon)), &temp_norm); + } + + if (d_bias) { + d_bias->mutable_data(ctx.GetPlace()); + colwise_sum(dev_ctx, d_y, d_bias); + } + if (d_scale) { + d_scale->mutable_data(ctx.GetPlace()); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, &d_y, /*axis*/ 0, MulFunctor(), &temp); + colwise_sum(dev_ctx, temp, d_scale); + } + + if (d_x) { + framework::DDim vec_shape({left}); + d_x->mutable_data(ctx.GetPlace()); + Tensor temp_vec; + temp_vec.mutable_data(vec_shape, ctx.GetPlace()); + + math::RowwiseMean row_mean; + + if (d_scale) { + // dy_dx + ElementwiseComputeEx, DeviceContext, T>( + ctx, &d_y, &scale, /*axis*/ 1, MulFunctor(), &temp); + framework::Copy(temp, ctx.GetPlace(), ctx.device_context(), d_x); + + // dy_dmean_dx + row_mean(dev_ctx, temp, &temp_vec); + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &temp_vec, /*axis*/ 0, SubFunctor(), d_x); + + // dy_var_dx + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); + + } else { + // dy_dx + framework::Copy(d_y, ctx.GetPlace(), ctx.device_context(), d_x); + + // dy_dmean_dx + row_mean(dev_ctx, d_y, &temp_vec); + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &temp_vec, /*axis*/ 0, SubFunctor(), d_x); + + // dy_var_dx + ElementwiseComputeEx, DeviceContext, T>( + ctx, &d_y, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); + } + // dy_var_dx + row_mean(dev_ctx, temp, &temp_vec); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, &temp_vec, /*axis*/ 0, MulFunctor(), &temp_norm); + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &temp_norm, /*axis*/ 0, SubFunctor(), d_x); + + ElementwiseComputeEx, DeviceContext, T>( + ctx, d_x, &var, /*axis*/ 0, + DivAndSqrtFunctor(static_cast(epsilon)), d_x); + } + } }; } // namespace operators From c3d27b15b7575a48e02a24eae6dff6b58d23cf70 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Sun, 4 Feb 2018 01:43:01 -0800 Subject: [PATCH 05/26] modify prune.cc for multiple blocks --- paddle/framework/prune.cc | 71 +++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/paddle/framework/prune.cc b/paddle/framework/prune.cc index bff8e0bcea..6a3882f199 100644 --- a/paddle/framework/prune.cc +++ b/paddle/framework/prune.cc @@ -49,11 +49,28 @@ bool IsTarget(const proto::OpDesc& op_desc) { return false; } -void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, - int block_id) { - // TODO(tonyyang-svail): - // - will change to use multiple blocks for RNN op and Cond Op +int GetSubBlockIndex(const proto::OpDesc& op_desc) { + for (auto& attr : op_desc.attrs()) { + if (attr.type() == proto::AttrType::BLOCK) { + PADDLE_ENFORCE(attr.has_block_idx()); + return attr.block_idx(); + } + } + return -1; +} +bool HasSubBlock(const proto::OpDesc& op_desc) { + return GetSubBlockIndex(op_desc) > 0; +} + +// block_id is the idx of the current block in the input desc +// parent_block_id is the idx of the parent of the current block +// in the output desc, -1 means the current block is global block +// dependent_vars is passed recursively from the parent block to +// the child block to help pruning +void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, + int block_id, int parent_block_id, + std::set& dependent_vars) { auto& block = input.blocks(block_id); auto& ops = block.ops(); @@ -72,11 +89,9 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, expect_fetch = (op_desc.type() == kFetchOpType); } - std::set dependent_vars; std::vector should_run; for (auto op_iter = ops.rbegin(); op_iter != ops.rend(); ++op_iter) { auto& op_desc = *op_iter; - if (IsTarget(op_desc) || HasDependentVar(op_desc, dependent_vars)) { // insert its input to the dependency graph for (auto& var : op_desc.inputs()) { @@ -84,7 +99,6 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, dependent_vars.insert(argu); } } - should_run.push_back(true); } else { should_run.push_back(false); @@ -95,19 +109,48 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, // we reverse the should_run vector std::reverse(should_run.begin(), should_run.end()); - *output = input; - auto* op_field = output->mutable_blocks(block_id)->mutable_ops(); + //*output = input; + // copy the current block from input to output + auto* block_field = output->mutable_blocks(); + *block_field->Add() = input.blocks(block_id); + + int output_block_id = output->blocks_size() - 1; + auto* output_block = output->mutable_blocks(output_block_id); + output_block->set_idx = output_block_id; + output_block->set_parent_idx = parent_block_id; + + auto* op_field = output_block->mutable_ops(); op_field->Clear(); for (size_t i = 0; i < should_run.size(); ++i) { if (should_run[i]) { - *op_field->Add() = input.blocks(block_id).ops(i); + auto* op = op_field->Add(); + *op = input.blocks(block_id).ops(i); + if (HasSubBlock(*op)) { + // create sub_block_dependent_vars here to help prune the sub block + std::set sub_block_dependent_vars; + for (auto& var : op.inputs()) { + for (auto& argu : var.arguments()) { + sub_block_dependent_vars.insert(argu); + } + } + for (auto& var : op.outputs()) { + for (auto& argu : var.arguments()) { + sub_block_dependent_vars.insert(argu); + } + } + + // GetSubBlockIndex(*op) is the idx of the sub_block in the input desc + // output_block_id is the idx of the current block in the output desc + prune_impl(input, output, GetSubBlockIndex(*op), output_block_id, + sub_block_dependent_vars); + } } } // remove the VarDescs in BlockDesc that are not referenced in // the pruned OpDescs std::unordered_map var_map; - auto* var_field = output->mutable_blocks(block_id)->mutable_vars(); + auto* var_field = output->mutable_blocks(output_block_id)->mutable_vars(); for (const auto& var : *var_field) { var_map[var.name()] = var; } @@ -118,14 +161,14 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, auto& input_field = op.inputs(); for (auto& input_var : input_field) { for (auto& arg : input_var.arguments()) { - *var_field->Add() = var_map[arg]; + *var_field->Add() = var_map.at(arg); } } // add VarDescs of all output arguments for each OpDesc auto& output_field = op.outputs(); for (auto& output_var : output_field) { for (auto& arg : output_var.arguments()) { - *var_field->Add() = var_map[arg]; + *var_field->Add() = var_map.at(arg); } } } @@ -133,7 +176,7 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, // TODO(fengjiayi): Prune() could be inplaced to avoid unnecessary copies void Prune(const proto::ProgramDesc& input, proto::ProgramDesc* output) { - prune_impl(input, output, 0); + prune_impl(input, output, 0, -1, {}); } void inference_optimize_impl(const proto::ProgramDesc& input, From 5092f5291c17e46b4c5e176c00b46a69f5e0d466 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Sat, 3 Feb 2018 15:05:55 +0800 Subject: [PATCH 06/26] Separate GPU and CPU implementation --- paddle/operators/layer_norm_op.cc | 186 +++++++++++++++++- paddle/operators/layer_norm_op.h | 29 ++- .../v2/fluid/tests/test_layer_norm_op.py | 11 +- 3 files changed, 202 insertions(+), 24 deletions(-) diff --git a/paddle/operators/layer_norm_op.cc b/paddle/operators/layer_norm_op.cc index edc26dfb96..910b8ec0a4 100644 --- a/paddle/operators/layer_norm_op.cc +++ b/paddle/operators/layer_norm_op.cc @@ -21,6 +21,13 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; using DataLayout = framework::DataLayout; +template +using EigenMatrixMapRowMajor = Eigen::Map< + Eigen::Matrix>; +template +using ConstEigenMatrixMapRowMajor = Eigen::Map< + const Eigen::Matrix>; + class LayerNormOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -101,7 +108,6 @@ class LayerNormOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Layer Normalization. - Layer Norm has been implemented as discussed in the paper: https://arxiv.org/abs/1607.06450 ... @@ -109,6 +115,75 @@ https://arxiv.org/abs/1607.06450 } }; +template +class LayerNormKernel + : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + const float epsilon = ctx.Attr("epsilon"); + const auto *scale = ctx.Input("Scale"); + const auto *bias = ctx.Input("Bias"); + const auto *x = ctx.Input("X"); + const auto &x_dims = x->dims(); + const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); + + auto *output = ctx.Output("Y"); + auto *mean = ctx.Output("Mean"); + auto *var = ctx.Output("Variance"); + output->mutable_data(ctx.GetPlace()); + mean->mutable_data(ctx.GetPlace()); + var->mutable_data(ctx.GetPlace()); + + auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); + int left = static_cast(matrix_dim[0]); + int right = static_cast(matrix_dim[1]); + + auto input_map = ConstEigenMatrixMapRowMajor(x->data(), left, right); + + auto mean_map = EigenMatrixMapRowMajor(mean->data(), left, 1); + auto var_map = EigenMatrixMapRowMajor(var->data(), left, 1); + auto output_map = EigenMatrixMapRowMajor(output->data(), left, right); + + auto squre = [](T ele) { return ele * ele; }; + auto add_epslion = [epsilon](T ele) { return ele + epsilon; }; + + mean_map = input_map.rowwise().mean(); + var_map = (input_map - mean_map.replicate(1, right)) + .unaryExpr(squre) + .rowwise() + .mean() + .unaryExpr(add_epslion); + + auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; + // TODO(zcd): Some thinking about output_map, is it appropriate that + // `output_map` and `input_map` point to the same memory. + auto inv_std = var_map.unaryExpr(inv_std_func); + if (scale && bias) { + auto scale_map = + ConstEigenMatrixMapRowMajor(scale->data(), 1, right); + auto bias_map = ConstEigenMatrixMapRowMajor(bias->data(), 1, right); + output_map = (input_map - mean_map.replicate(1, right)) + .cwiseProduct(inv_std.replicate(1, right)) + .cwiseProduct(scale_map.replicate(left, 1)) + + bias_map.replicate(left, 1); + } else if (scale) { + auto scale_map = + ConstEigenMatrixMapRowMajor(scale->data(), 1, right); + output_map = (input_map - mean_map.replicate(1, right)) + .cwiseProduct(inv_std.replicate(1, right)) + .cwiseProduct(scale_map.replicate(left, 1)); + } else if (bias) { + auto bias_map = ConstEigenMatrixMapRowMajor(bias->data(), 1, right); + output_map = (input_map - mean_map.replicate(1, right)) + .cwiseProduct(inv_std.replicate(1, right)) + + bias_map.replicate(left, 1); + } else { + output_map = (input_map - mean_map.replicate(1, right)) + .cwiseProduct(inv_std.replicate(1, right)); + } + } +}; + class LayerNormGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -161,6 +236,115 @@ class LayerNormGradOp : public framework::OperatorWithKernel { } }; +template +class LayerNormGradKernel + : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + const auto *x = ctx.Input("X"); + const auto *mean = ctx.Input("Mean"); + const auto *var = ctx.Input("Variance"); + const auto *scale = ctx.Input("Scale"); + const auto *d_y = ctx.Input(framework::GradVarName("Y")); + + const auto &x_dims = x->dims(); + + const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); + auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); + int left = static_cast(matrix_dim[0]); + int right = static_cast(matrix_dim[1]); + + // init output + auto *d_x = ctx.Output(framework::GradVarName("X")); + auto *d_scale = ctx.Output(framework::GradVarName("Scale")); + auto *d_bias = ctx.Output(framework::GradVarName("Bias")); + + auto x_map = ConstEigenMatrixMapRowMajor(x->data(), left, right); + auto d_y_map = ConstEigenMatrixMapRowMajor(d_y->data(), left, right); + auto mean_map = ConstEigenMatrixMapRowMajor(mean->data(), left, 1); + auto var_map = ConstEigenMatrixMapRowMajor(var->data(), left, 1); + + if (d_bias) { + d_bias->mutable_data(ctx.GetPlace()); + auto d_bias_map = EigenMatrixMapRowMajor(d_bias->data(), 1, right); + d_bias_map = d_y_map.colwise().sum(); + } + if (d_scale) { + d_scale->mutable_data(ctx.GetPlace()); + auto d_scale_map = + EigenMatrixMapRowMajor(d_scale->data(), 1, right); + auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; + // There are two equation to compute d_scale. One uses "Y" and the other + // does not use "Y" + d_scale_map = + ((x_map - mean_map.replicate(1, right)) + .cwiseProduct( + var_map.unaryExpr(inv_std_func).replicate(1, right)) + .cwiseProduct(d_y_map)) + .colwise() + .sum(); + } + + if (d_x) { + d_x->mutable_data(ctx.GetPlace()); + auto d_x_map = EigenMatrixMapRowMajor(d_x->data(), left, right); + auto triple_product_func = [](T ele) { return ele * ele * ele; }; + auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; + + auto inv_std_map = var_map.unaryExpr(inv_std_func).eval(); + // TODO(zcd): these code can be refined + if (d_scale) { + auto scale_map = + ConstEigenMatrixMapRowMajor(scale->data(), 1, right); + // dy_dx + auto dx_end = + inv_std_map.replicate(1, right).cwiseProduct(d_y_map).cwiseProduct( + scale_map.replicate(left, 1)); + + // dy_dmean_dx + auto dx_mean = + (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); + + // dy_var_dx + auto dvar_end_part = (x_map - mean_map.replicate(1, right)) + .cwiseProduct(scale_map.replicate(left, 1)) + .cwiseProduct(d_y_map) + .rowwise() + .sum(); + auto dvar_end = inv_std_map.unaryExpr(triple_product_func) + .cwiseProduct(dvar_end_part) + .replicate(1, right); + auto dx_var = + (T(-1.0) / right) * + (x_map - mean_map.replicate(1, right)).cwiseProduct(dvar_end); + + d_x_map = dx_end + dx_mean + dx_var; + } else { + // dy_dx + auto dx_end = inv_std_map.replicate(1, right).cwiseProduct(d_y_map); + + // dy_dmean_dx + auto dx_mean = + (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); + + // dy_var_dx + auto dvar_end_part = (x_map - mean_map.replicate(1, right)) + .cwiseProduct(d_y_map) + .rowwise() + .sum(); + auto dvar_end = inv_std_map.unaryExpr(triple_product_func) + .cwiseProduct(dvar_end_part) + .replicate(1, right); + auto dx_var = + (T(-1.0) / right) * + (x_map - mean_map.replicate(1, right)).cwiseProduct(dvar_end); + + d_x_map = dx_end + dx_mean + dx_var; + } + } + } +}; + } // namespace operators } // namespace paddle diff --git a/paddle/operators/layer_norm_op.h b/paddle/operators/layer_norm_op.h index 309f1b87a2..2de58186fb 100644 --- a/paddle/operators/layer_norm_op.h +++ b/paddle/operators/layer_norm_op.h @@ -78,7 +78,7 @@ class LayerNormKernel : public framework::OpKernel { auto *var = ctx.Output("Variance"); const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - const auto &x_dims = x.dims(); + const auto x_dims = x.dims(); y->mutable_data(ctx.GetPlace()); mean->mutable_data(ctx.GetPlace()); @@ -87,11 +87,12 @@ class LayerNormKernel : public framework::OpKernel { auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); int left = static_cast(matrix_dim[0]); int right = static_cast(matrix_dim[1]); - framework::DDim matrix_shape({left, right}); x.Resize(matrix_shape); - y->Resize(matrix_shape); + Tensor out; + out.ShareDataWith(*y); + out.Resize(matrix_shape); auto &dev_ctx = ctx.template device_context(); math::RowwiseMean row_mean; @@ -101,30 +102,24 @@ class LayerNormKernel : public framework::OpKernel { // functor-> get variance ElementwiseComputeEx, DeviceContext, T>( - ctx, &x, mean, /*axis*/ 0, SubAndSquareFunctor(), y); - row_mean(dev_ctx, *y, var); + ctx, &x, mean, /*axis*/ 0, SubAndSquareFunctor(), &out); + row_mean(dev_ctx, out, var); // functor-> get norm_out ElementwiseComputeEx, DeviceContext, T>( - ctx, &x, mean, /*axis*/ 0, SubFunctor(), y); + ctx, &x, mean, /*axis*/ 0, SubFunctor(), &out); ElementwiseComputeEx, DeviceContext, T>( - ctx, y, var, /*axis*/ 0, DivAndSqrtFunctor(static_cast(epsilon)), - y); + ctx, &out, var, /*axis*/ 0, + DivAndSqrtFunctor(static_cast(epsilon)), &out); - framework::DDim scale_shape({right}); if (scale) { - Tensor scale_matrix = *scale; - scale_matrix.Resize(scale_shape); ElementwiseComputeEx, DeviceContext, T>( - ctx, y, &scale_matrix, /*axis*/ 1, MulFunctor(), y); + ctx, &out, scale, /*axis*/ 1, MulFunctor(), &out); } if (bias) { - Tensor bias_matrix = *bias; - bias_matrix.Resize(scale_shape); ElementwiseComputeEx, DeviceContext, T>( - ctx, y, &bias_matrix, /*axis*/ 1, AddFunctor(), y); + ctx, &out, bias, /*axis*/ 1, AddFunctor(), &out); } - y->Resize(x_dims); } }; @@ -184,6 +179,7 @@ class LayerNormGradKernel : public framework::OpKernel { if (d_x) { framework::DDim vec_shape({left}); d_x->mutable_data(ctx.GetPlace()); + auto dx_dim = d_x->dims(); Tensor temp_vec; temp_vec.mutable_data(vec_shape, ctx.GetPlace()); @@ -227,6 +223,7 @@ class LayerNormGradKernel : public framework::OpKernel { ElementwiseComputeEx, DeviceContext, T>( ctx, d_x, &var, /*axis*/ 0, DivAndSqrtFunctor(static_cast(epsilon)), d_x); + d_x->Resize(dx_dim); } } }; diff --git a/python/paddle/v2/fluid/tests/test_layer_norm_op.py b/python/paddle/v2/fluid/tests/test_layer_norm_op.py index 68cf8673cd..f456b1194c 100644 --- a/python/paddle/v2/fluid/tests/test_layer_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_layer_norm_op.py @@ -62,9 +62,9 @@ def _reference_layer_norm_grad(x, grad_y, scale, mean, var, begin_norm_axis=1): grad_x = dx_end + d_mean + d_std - grad_y.shape = x_shape - x.shape = x_shape + grad_x.shape, x.shape, grad_y.shape = x_shape, x_shape, x_shape scale.shape = scale_shape + var.shape, mean.shape = [N, ], [N, ] return grad_x, d_scale, d_bias @@ -112,10 +112,7 @@ def set_output_grad(scope, outputs, place, feed_dict=None): class TestLayerNormdOp(OpTest): def __assert_close(self, tensor, np_array, msg, atol=1e-4): - self.assertTrue( - np.allclose( - np.array(tensor).reshape(np_array.shape), np_array, atol=atol), - msg) + self.assertTrue(np.allclose(np.array(tensor), np_array, atol=atol), msg) def __assert_grad_close(self, tensor, @@ -123,7 +120,7 @@ class TestLayerNormdOp(OpTest): name, place, max_relative_error=0.02): - a = np.array(tensor).reshape(np_array.shape) + a = np.array(tensor) b = np_array abs_a = np.abs(a) abs_a[abs_a < 1e-5] = 1 From df0e74dba0fcbb894eeefa727d7a8a4d50025ccb Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 5 Feb 2018 11:28:22 +0800 Subject: [PATCH 07/26] unifid GPU and CPU implementation --- paddle/operators/layer_norm_op.cc | 185 ------------------ paddle/operators/layer_norm_op.h | 2 +- .../v2/fluid/tests/test_layer_norm_op.py | 4 +- 3 files changed, 4 insertions(+), 187 deletions(-) diff --git a/paddle/operators/layer_norm_op.cc b/paddle/operators/layer_norm_op.cc index 910b8ec0a4..76d5d571c3 100644 --- a/paddle/operators/layer_norm_op.cc +++ b/paddle/operators/layer_norm_op.cc @@ -21,13 +21,6 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; using DataLayout = framework::DataLayout; -template -using EigenMatrixMapRowMajor = Eigen::Map< - Eigen::Matrix>; -template -using ConstEigenMatrixMapRowMajor = Eigen::Map< - const Eigen::Matrix>; - class LayerNormOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -115,75 +108,6 @@ https://arxiv.org/abs/1607.06450 } }; -template -class LayerNormKernel - : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - const float epsilon = ctx.Attr("epsilon"); - const auto *scale = ctx.Input("Scale"); - const auto *bias = ctx.Input("Bias"); - const auto *x = ctx.Input("X"); - const auto &x_dims = x->dims(); - const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - - auto *output = ctx.Output("Y"); - auto *mean = ctx.Output("Mean"); - auto *var = ctx.Output("Variance"); - output->mutable_data(ctx.GetPlace()); - mean->mutable_data(ctx.GetPlace()); - var->mutable_data(ctx.GetPlace()); - - auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int left = static_cast(matrix_dim[0]); - int right = static_cast(matrix_dim[1]); - - auto input_map = ConstEigenMatrixMapRowMajor(x->data(), left, right); - - auto mean_map = EigenMatrixMapRowMajor(mean->data(), left, 1); - auto var_map = EigenMatrixMapRowMajor(var->data(), left, 1); - auto output_map = EigenMatrixMapRowMajor(output->data(), left, right); - - auto squre = [](T ele) { return ele * ele; }; - auto add_epslion = [epsilon](T ele) { return ele + epsilon; }; - - mean_map = input_map.rowwise().mean(); - var_map = (input_map - mean_map.replicate(1, right)) - .unaryExpr(squre) - .rowwise() - .mean() - .unaryExpr(add_epslion); - - auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; - // TODO(zcd): Some thinking about output_map, is it appropriate that - // `output_map` and `input_map` point to the same memory. - auto inv_std = var_map.unaryExpr(inv_std_func); - if (scale && bias) { - auto scale_map = - ConstEigenMatrixMapRowMajor(scale->data(), 1, right); - auto bias_map = ConstEigenMatrixMapRowMajor(bias->data(), 1, right); - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)) - .cwiseProduct(scale_map.replicate(left, 1)) + - bias_map.replicate(left, 1); - } else if (scale) { - auto scale_map = - ConstEigenMatrixMapRowMajor(scale->data(), 1, right); - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)) - .cwiseProduct(scale_map.replicate(left, 1)); - } else if (bias) { - auto bias_map = ConstEigenMatrixMapRowMajor(bias->data(), 1, right); - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)) + - bias_map.replicate(left, 1); - } else { - output_map = (input_map - mean_map.replicate(1, right)) - .cwiseProduct(inv_std.replicate(1, right)); - } - } -}; - class LayerNormGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -236,115 +160,6 @@ class LayerNormGradOp : public framework::OperatorWithKernel { } }; -template -class LayerNormGradKernel - : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext &ctx) const override { - const auto *x = ctx.Input("X"); - const auto *mean = ctx.Input("Mean"); - const auto *var = ctx.Input("Variance"); - const auto *scale = ctx.Input("Scale"); - const auto *d_y = ctx.Input(framework::GradVarName("Y")); - - const auto &x_dims = x->dims(); - - const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); - auto matrix_dim = framework::flatten_to_2d(x_dims, begin_norm_axis); - int left = static_cast(matrix_dim[0]); - int right = static_cast(matrix_dim[1]); - - // init output - auto *d_x = ctx.Output(framework::GradVarName("X")); - auto *d_scale = ctx.Output(framework::GradVarName("Scale")); - auto *d_bias = ctx.Output(framework::GradVarName("Bias")); - - auto x_map = ConstEigenMatrixMapRowMajor(x->data(), left, right); - auto d_y_map = ConstEigenMatrixMapRowMajor(d_y->data(), left, right); - auto mean_map = ConstEigenMatrixMapRowMajor(mean->data(), left, 1); - auto var_map = ConstEigenMatrixMapRowMajor(var->data(), left, 1); - - if (d_bias) { - d_bias->mutable_data(ctx.GetPlace()); - auto d_bias_map = EigenMatrixMapRowMajor(d_bias->data(), 1, right); - d_bias_map = d_y_map.colwise().sum(); - } - if (d_scale) { - d_scale->mutable_data(ctx.GetPlace()); - auto d_scale_map = - EigenMatrixMapRowMajor(d_scale->data(), 1, right); - auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; - // There are two equation to compute d_scale. One uses "Y" and the other - // does not use "Y" - d_scale_map = - ((x_map - mean_map.replicate(1, right)) - .cwiseProduct( - var_map.unaryExpr(inv_std_func).replicate(1, right)) - .cwiseProduct(d_y_map)) - .colwise() - .sum(); - } - - if (d_x) { - d_x->mutable_data(ctx.GetPlace()); - auto d_x_map = EigenMatrixMapRowMajor(d_x->data(), left, right); - auto triple_product_func = [](T ele) { return ele * ele * ele; }; - auto inv_std_func = [](T ele) { return std::sqrt(1 / ele); }; - - auto inv_std_map = var_map.unaryExpr(inv_std_func).eval(); - // TODO(zcd): these code can be refined - if (d_scale) { - auto scale_map = - ConstEigenMatrixMapRowMajor(scale->data(), 1, right); - // dy_dx - auto dx_end = - inv_std_map.replicate(1, right).cwiseProduct(d_y_map).cwiseProduct( - scale_map.replicate(left, 1)); - - // dy_dmean_dx - auto dx_mean = - (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); - - // dy_var_dx - auto dvar_end_part = (x_map - mean_map.replicate(1, right)) - .cwiseProduct(scale_map.replicate(left, 1)) - .cwiseProduct(d_y_map) - .rowwise() - .sum(); - auto dvar_end = inv_std_map.unaryExpr(triple_product_func) - .cwiseProduct(dvar_end_part) - .replicate(1, right); - auto dx_var = - (T(-1.0) / right) * - (x_map - mean_map.replicate(1, right)).cwiseProduct(dvar_end); - - d_x_map = dx_end + dx_mean + dx_var; - } else { - // dy_dx - auto dx_end = inv_std_map.replicate(1, right).cwiseProduct(d_y_map); - - // dy_dmean_dx - auto dx_mean = - (T(-1.0) / right) * dx_end.rowwise().sum().replicate(1, right); - - // dy_var_dx - auto dvar_end_part = (x_map - mean_map.replicate(1, right)) - .cwiseProduct(d_y_map) - .rowwise() - .sum(); - auto dvar_end = inv_std_map.unaryExpr(triple_product_func) - .cwiseProduct(dvar_end_part) - .replicate(1, right); - auto dx_var = - (T(-1.0) / right) * - (x_map - mean_map.replicate(1, right)).cwiseProduct(dvar_end); - - d_x_map = dx_end + dx_mean + dx_var; - } - } - } -}; - } // namespace operators } // namespace paddle diff --git a/paddle/operators/layer_norm_op.h b/paddle/operators/layer_norm_op.h index 2de58186fb..608447b1ff 100644 --- a/paddle/operators/layer_norm_op.h +++ b/paddle/operators/layer_norm_op.h @@ -31,7 +31,7 @@ template struct DivAndSqrtFunctor { explicit DivAndSqrtFunctor(T epsilon) { epsilon_ = epsilon; } inline HOSTDEVICE T operator()(T a, T b) const { - return a / (sqrt(b) + epsilon_); + return a / (sqrt(b + epsilon_)); } private: diff --git a/python/paddle/v2/fluid/tests/test_layer_norm_op.py b/python/paddle/v2/fluid/tests/test_layer_norm_op.py index f456b1194c..4460ffaf9c 100644 --- a/python/paddle/v2/fluid/tests/test_layer_norm_op.py +++ b/python/paddle/v2/fluid/tests/test_layer_norm_op.py @@ -20,6 +20,8 @@ import paddle.v2.fluid.core as core from paddle.v2.fluid.op import Operator from paddle.v2.fluid.framework import grad_var_name +np.random.random(123) + def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): x_shape = x.shape @@ -148,7 +150,7 @@ class TestLayerNormdOp(OpTest): x_shape = shape D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) scale_shape = [D] - np.random.random(123) + x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) bias_val = np.random.random_sample(scale_shape).astype(np.float32) From 677312973516832a495a0f83fbcc8b5c55e977f6 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 5 Feb 2018 13:40:11 +0800 Subject: [PATCH 08/26] code refine --- paddle/operators/layer_norm_op.h | 42 ++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/paddle/operators/layer_norm_op.h b/paddle/operators/layer_norm_op.h index 608447b1ff..3c436b8926 100644 --- a/paddle/operators/layer_norm_op.h +++ b/paddle/operators/layer_norm_op.h @@ -97,15 +97,15 @@ class LayerNormKernel : public framework::OpKernel { auto &dev_ctx = ctx.template device_context(); math::RowwiseMean row_mean; - // functor-> get mean + // get mean row_mean(dev_ctx, x, mean); - // functor-> get variance + // get variance ElementwiseComputeEx, DeviceContext, T>( ctx, &x, mean, /*axis*/ 0, SubAndSquareFunctor(), &out); row_mean(dev_ctx, out, var); - // functor-> get norm_out + // get x_norm ElementwiseComputeEx, DeviceContext, T>( ctx, &x, mean, /*axis*/ 0, SubFunctor(), &out); ElementwiseComputeEx, DeviceContext, T>( @@ -129,9 +129,11 @@ class LayerNormGradKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext &ctx) const override { const float epsilon = ctx.Attr("epsilon"); auto x = *ctx.Input("X"); - auto mean = *ctx.Input("Mean"); - auto var = *ctx.Input("Variance"); - auto scale = *ctx.Input("Scale"); + auto *y = ctx.Input("Y"); + auto *mean = ctx.Input("Mean"); + auto *var = ctx.Input("Variance"); + auto *scale = ctx.Input("Scale"); + auto *bias = ctx.Input("Bias"); auto d_y = *ctx.Input(framework::GradVarName("Y")); const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); @@ -155,14 +157,19 @@ class LayerNormGradKernel : public framework::OpKernel { if (d_scale || d_x) { x.Resize(matrix_shape); temp.mutable_data(matrix_shape, ctx.GetPlace()); - temp_norm.mutable_data(matrix_shape, ctx.GetPlace()); - // get x_norm - ElementwiseComputeEx, DeviceContext, T>( - ctx, &x, &mean, /*axis*/ 0, SubFunctor(), &temp_norm); - ElementwiseComputeEx, DeviceContext, T>( - ctx, &temp_norm, &var, /*axis*/ 0, - DivAndSqrtFunctor(static_cast(epsilon)), &temp_norm); + if (!(bias && scale)) { + temp_norm.ShareDataWith(*y); + temp_norm.Resize(matrix_shape); + } else { + temp_norm.mutable_data(matrix_shape, ctx.GetPlace()); + // get x_norm + ElementwiseComputeEx, DeviceContext, T>( + ctx, &x, mean, /*axis*/ 0, SubFunctor(), &temp_norm); + ElementwiseComputeEx, DeviceContext, T>( + ctx, &temp_norm, var, /*axis*/ 0, + DivAndSqrtFunctor(static_cast(epsilon)), &temp_norm); + } } if (d_bias) { @@ -188,7 +195,7 @@ class LayerNormGradKernel : public framework::OpKernel { if (d_scale) { // dy_dx ElementwiseComputeEx, DeviceContext, T>( - ctx, &d_y, &scale, /*axis*/ 1, MulFunctor(), &temp); + ctx, &d_y, scale, /*axis*/ 1, MulFunctor(), &temp); framework::Copy(temp, ctx.GetPlace(), ctx.device_context(), d_x); // dy_dmean_dx @@ -199,7 +206,6 @@ class LayerNormGradKernel : public framework::OpKernel { // dy_var_dx ElementwiseComputeEx, DeviceContext, T>( ctx, &temp, &temp_norm, /*axis*/ 0, MulFunctor(), &temp); - } else { // dy_dx framework::Copy(d_y, ctx.GetPlace(), ctx.device_context(), d_x); @@ -216,12 +222,12 @@ class LayerNormGradKernel : public framework::OpKernel { // dy_var_dx row_mean(dev_ctx, temp, &temp_vec); ElementwiseComputeEx, DeviceContext, T>( - ctx, &temp_norm, &temp_vec, /*axis*/ 0, MulFunctor(), &temp_norm); + ctx, &temp_norm, &temp_vec, /*axis*/ 0, MulFunctor(), &temp); ElementwiseComputeEx, DeviceContext, T>( - ctx, d_x, &temp_norm, /*axis*/ 0, SubFunctor(), d_x); + ctx, d_x, &temp, /*axis*/ 0, SubFunctor(), d_x); ElementwiseComputeEx, DeviceContext, T>( - ctx, d_x, &var, /*axis*/ 0, + ctx, d_x, var, /*axis*/ 0, DivAndSqrtFunctor(static_cast(epsilon)), d_x); d_x->Resize(dx_dim); } From 6f0e630c5ce67bef5e87e26441c60870d1ab207e Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 5 Feb 2018 13:25:20 -0800 Subject: [PATCH 09/26] fix prune and program desc constructor --- paddle/framework/block_desc.cc | 2 ++ paddle/framework/op_desc.cc | 17 +++++++-- paddle/framework/program_desc.cc | 18 ++++++++++ paddle/framework/prune.cc | 36 ++++++++++++------- python/paddle/v2/fluid/io.py | 6 ++++ .../tests/book/test_rnn_encoder_decoder.py | 31 ++++++++++------ 6 files changed, 85 insertions(+), 25 deletions(-) diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index dd2ed87252..ca3d03e554 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -155,6 +155,8 @@ BlockDesc::BlockDesc(ProgramDesc *prog, proto::BlockDesc *desc) for (const proto::OpDesc &op_desc : desc_->ops()) { ops_.emplace_back(new OpDesc(op_desc, prog, this)); } + std::cout << "Constructed block idx " << desc->idx() << " from protobuf str" + << std::endl; } BlockDesc::BlockDesc(const BlockDesc &other, proto::BlockDesc *desc, diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index f8df2cf97a..5ebd2b3ad5 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -124,11 +124,24 @@ OpDesc::OpDesc(const proto::OpDesc &desc, ProgramDesc *prog, BlockDesc *block) // restore attrs_ for (const proto::OpDesc::Attr &attr : desc_.attrs()) { std::string attr_name = attr.name(); + // we use a trick to handle attr.type() is BLOCK here, because at this + // moment the sub_block hasn't beed added to ProgramDesc's vector + // so we cast the block_idx to a dummy BlockDesc pointer if (attr.type() != proto::AttrType::BLOCK) { attrs_[attr_name] = GetAttrValue(attr); } else { - auto bid = attr.block_idx(); - attrs_[attr_name] = prog->MutableBlock(bid); + size_t blk_idx = attr.block_idx(); + if (blk_idx < prog->Size()) { + attrs_[attr_name] = prog->MutableBlock(blk_idx); + } else { + std::cout << "Setting blockdesc attribute for id " << blk_idx + << std::endl; + attrs_[attr_name] = reinterpret_cast(blk_idx); + std::cout << "Testing reinterpret_cast result is " + << reinterpret_cast( + boost::get(attrs_[attr_name])) + << std::endl; + } } } this->block_ = block; diff --git a/paddle/framework/program_desc.cc b/paddle/framework/program_desc.cc index 15ea4035c6..9124607623 100644 --- a/paddle/framework/program_desc.cc +++ b/paddle/framework/program_desc.cc @@ -52,9 +52,27 @@ ProgramDesc::ProgramDesc(const ProgramDesc &o) { ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { desc_ = desc; + std::cout << std::endl << "starting in ProgDesc constructor" << std::endl; for (auto &block_desc : *desc_.mutable_blocks()) { blocks_.emplace_back(new BlockDesc(this, &block_desc)); + std::cout << "Done constructing block idx " << block_desc.idx() + << " parent idx " << block_desc.parent_idx() << std::endl; } + for (auto &block : blocks_) { + for (auto *op : block->AllOps()) { + for (auto &name : op->AttrNames()) { + if (op->GetAttrType(name) == proto::AttrType::BLOCK) { + auto attr = op->GetAttr(name); + size_t blk_idx = + reinterpret_cast(boost::get(attr)); + op->SetBlockAttr(name, *this->MutableBlock(blk_idx)); + std::cout << "Update attr name " << name << " for block idx " + << blk_idx << std::endl; + } + } + } + } + std::cout << "Done ProgDesc construction" << std::endl << std::endl; } ProgramDesc::ProgramDesc(const std::string &binary_str) { diff --git a/paddle/framework/prune.cc b/paddle/framework/prune.cc index 6a3882f199..3c3ec87585 100644 --- a/paddle/framework/prune.cc +++ b/paddle/framework/prune.cc @@ -109,15 +109,14 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, // we reverse the should_run vector std::reverse(should_run.begin(), should_run.end()); - //*output = input; // copy the current block from input to output auto* block_field = output->mutable_blocks(); *block_field->Add() = input.blocks(block_id); int output_block_id = output->blocks_size() - 1; auto* output_block = output->mutable_blocks(output_block_id); - output_block->set_idx = output_block_id; - output_block->set_parent_idx = parent_block_id; + output_block->set_idx(output_block_id); + output_block->set_parent_idx(parent_block_id); auto* op_field = output_block->mutable_ops(); op_field->Clear(); @@ -128,17 +127,18 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, if (HasSubBlock(*op)) { // create sub_block_dependent_vars here to help prune the sub block std::set sub_block_dependent_vars; - for (auto& var : op.inputs()) { + for (auto& var : op->inputs()) { for (auto& argu : var.arguments()) { sub_block_dependent_vars.insert(argu); } } - for (auto& var : op.outputs()) { + for (auto& var : op->outputs()) { for (auto& argu : var.arguments()) { sub_block_dependent_vars.insert(argu); } } - + std::cout << "pruning the next block, the current output_block_id is " + << output_block_id << std::endl; // GetSubBlockIndex(*op) is the idx of the sub_block in the input desc // output_block_id is the idx of the current block in the output desc prune_impl(input, output, GetSubBlockIndex(*op), output_block_id, @@ -147,6 +147,8 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, } } + std::cout << "Starting to remove unreferenced variables" + << " for block idx " << output_block_id << std::endl; // remove the VarDescs in BlockDesc that are not referenced in // the pruned OpDescs std::unordered_map var_map; @@ -155,28 +157,38 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, var_map[var.name()] = var; } - var_field->Clear(); + std::set var_names; for (const auto& op : *op_field) { - // add VarDescs of all input arguments for each OpDesc auto& input_field = op.inputs(); for (auto& input_var : input_field) { for (auto& arg : input_var.arguments()) { - *var_field->Add() = var_map.at(arg); + if (var_map.count(arg) != 0) { + var_names.insert(arg); + } } } - // add VarDescs of all output arguments for each OpDesc auto& output_field = op.outputs(); for (auto& output_var : output_field) { for (auto& arg : output_var.arguments()) { - *var_field->Add() = var_map.at(arg); + if (var_map.count(arg) != 0) { + var_names.insert(arg); + } } } } + + var_field->Clear(); + for (const auto& name : var_names) { + *var_field->Add() = var_map[name]; + } } // TODO(fengjiayi): Prune() could be inplaced to avoid unnecessary copies void Prune(const proto::ProgramDesc& input, proto::ProgramDesc* output) { - prune_impl(input, output, 0, -1, {}); + std::set dependent_vars; + std::cout << std::endl << "Start C++ framework::prune" << std::endl; + prune_impl(input, output, 0, -1, dependent_vars); + std::cout << "Finished C++ framework::prune" << std::endl << std::endl; } void inference_optimize_impl(const proto::ProgramDesc& input, diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 613dc20b6e..e410549f8a 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -342,6 +342,12 @@ def save_inference_model(dirname, prepend_feed_ops(inference_program, feeded_var_names) append_fetch_ops(inference_program, fetch_var_names) + # save for checking + curstr = inference_program.to_string(True) + f = open("save_inf_prog_after_feed_fetch.txt", 'w') + f.write(curstr) + f.close() + model_file_name = dirname + "/__model__" with open(model_file_name, "wb") as f: f.write(inference_program.desc.serialize_to_string()) diff --git a/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py index 593d0013c9..15f00f95d4 100644 --- a/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py @@ -197,14 +197,15 @@ def train(save_dirname=None): " avg_cost=" + str(avg_cost_val)) if batch_id > 3: if save_dirname is not None: - fluid.io.save_inference_model(save_dirname, [ - 'source_sequence', 'target_sequence', 'label_sequence' - ], [prediction], exe) + fluid.io.save_inference_model( + save_dirname, ['source_sequence', + 'target_sequence'], [prediction], exe) + return exit(0) batch_id += 1 -def inference(save_dirname=None): +def infer(save_dirname=None): if save_dirname is None: return @@ -221,24 +222,32 @@ def inference(save_dirname=None): data = [[0, 1, 0, 1], [0, 1, 1, 0, 0, 1]] word_data = to_lodtensor(data, place) trg_word = to_lodtensor(data, place) - trg_word_next = to_lodtensor(data, place) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. + print("Print feed fetch target names as follows") print(feed_target_names) assert feed_target_names[0] == 'source_sequence' assert feed_target_names[1] == 'target_sequence' - assert feed_target_names[2] == 'label_sequence' + print([var.name for var in fetch_targets]) + + # save for checking + curstr = inference_program.to_string(True) + f = open("loaded_infer_prog.txt", 'w') + f.write(curstr) + f.close() + results = exe.run(inference_program, feed={ feed_target_names[0]: word_data, feed_target_names[1]: trg_word, - feed_target_names[2]: trg_word_next }, - fetch_list=fetch_targets) - - print("Inference Shape: ", results[0].shape) - print("infer results: ", results[0]) + fetch_list=fetch_targets, + return_numpy=False) + print(results[0].lod()) + np_data = np.array(results[0]) + print("Inference shape: ", np_data.shape) + print("Inference results: ", np_data) if __name__ == '__main__': From dc68e7c44b198acbdf588e97d219822602dc90db Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 5 Feb 2018 15:36:54 -0800 Subject: [PATCH 10/26] fix constructor bug --- paddle/framework/op_desc.cc | 9 +++------ paddle/framework/program_desc.cc | 26 ++++++++++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 5ebd2b3ad5..7859c391fa 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -133,13 +133,10 @@ OpDesc::OpDesc(const proto::OpDesc &desc, ProgramDesc *prog, BlockDesc *block) size_t blk_idx = attr.block_idx(); if (blk_idx < prog->Size()) { attrs_[attr_name] = prog->MutableBlock(blk_idx); - } else { - std::cout << "Setting blockdesc attribute for id " << blk_idx + std::cout << "In OpDesc: set up attr block idx " << blk_idx << std::endl; - attrs_[attr_name] = reinterpret_cast(blk_idx); - std::cout << "Testing reinterpret_cast result is " - << reinterpret_cast( - boost::get(attrs_[attr_name])) + } else { + std::cout << "In OpDesc: We don't have this block idx " << blk_idx << std::endl; } } diff --git a/paddle/framework/program_desc.cc b/paddle/framework/program_desc.cc index 9124607623..ba461b0933 100644 --- a/paddle/framework/program_desc.cc +++ b/paddle/framework/program_desc.cc @@ -48,6 +48,18 @@ ProgramDesc::ProgramDesc(const ProgramDesc &o) { auto *block = desc_.mutable_blocks(i); blocks_.emplace_back(new BlockDesc(*o.blocks_[i], block, this)); } + for (auto &block : blocks_) { + for (auto *op : block->AllOps()) { + for (const auto &attr : op->Proto()->attrs()) { + if (attr.type() == proto::AttrType::BLOCK) { + size_t blk_idx = attr.block_idx(); + op->SetBlockAttr(attr.name(), *this->MutableBlock(blk_idx)); + std::cout << "In ProgramDesc 1: set block attr idx " << blk_idx + << std::endl; + } + } + } + } } ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { @@ -60,14 +72,12 @@ ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { } for (auto &block : blocks_) { for (auto *op : block->AllOps()) { - for (auto &name : op->AttrNames()) { - if (op->GetAttrType(name) == proto::AttrType::BLOCK) { - auto attr = op->GetAttr(name); - size_t blk_idx = - reinterpret_cast(boost::get(attr)); - op->SetBlockAttr(name, *this->MutableBlock(blk_idx)); - std::cout << "Update attr name " << name << " for block idx " - << blk_idx << std::endl; + for (const auto &attr : op->Proto()->attrs()) { + if (attr.type() == proto::AttrType::BLOCK) { + size_t blk_idx = attr.block_idx(); + op->SetBlockAttr(attr.name(), *this->MutableBlock(blk_idx)); + std::cout << "In ProgramDesc 2: set block attr idx " << blk_idx + << std::endl; } } } From d5686f5831adea6bc9b0ceb94b81cd3f79270800 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 5 Feb 2018 18:00:07 -0800 Subject: [PATCH 11/26] clean code --- paddle/framework/block_desc.cc | 5 +---- paddle/framework/op_desc.cc | 15 ++------------- paddle/framework/program_desc.cc | 9 --------- paddle/framework/prune.cc | 6 ------ python/paddle/v2/fluid/io.py | 6 ------ .../fluid/tests/book/test_rnn_encoder_decoder.py | 10 ---------- 6 files changed, 3 insertions(+), 48 deletions(-) diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index ca3d03e554..3e344ea379 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -155,8 +155,6 @@ BlockDesc::BlockDesc(ProgramDesc *prog, proto::BlockDesc *desc) for (const proto::OpDesc &op_desc : desc_->ops()) { ops_.emplace_back(new OpDesc(op_desc, prog, this)); } - std::cout << "Constructed block idx " << desc->idx() << " from protobuf str" - << std::endl; } BlockDesc::BlockDesc(const BlockDesc &other, proto::BlockDesc *desc, @@ -164,9 +162,8 @@ BlockDesc::BlockDesc(const BlockDesc &other, proto::BlockDesc *desc, : prog_(prog), desc_(desc) { need_update_ = true; for (auto &op : other.ops_) { - ops_.emplace_back(new OpDesc(*op, this)); + ops_.emplace_back(new OpDesc(*op->Proto(), prog, this)); } - for (auto &it : other.vars_) { auto *var = new VarDesc(*it.second); vars_[it.first].reset(var); diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 7859c391fa..46c50d9250 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -124,21 +124,10 @@ OpDesc::OpDesc(const proto::OpDesc &desc, ProgramDesc *prog, BlockDesc *block) // restore attrs_ for (const proto::OpDesc::Attr &attr : desc_.attrs()) { std::string attr_name = attr.name(); - // we use a trick to handle attr.type() is BLOCK here, because at this - // moment the sub_block hasn't beed added to ProgramDesc's vector - // so we cast the block_idx to a dummy BlockDesc pointer + // The sub_block referred to by the BLOCK attr hasn't be added + // to ProgramDesc class yet, we skip setting BLOCK attr here. if (attr.type() != proto::AttrType::BLOCK) { attrs_[attr_name] = GetAttrValue(attr); - } else { - size_t blk_idx = attr.block_idx(); - if (blk_idx < prog->Size()) { - attrs_[attr_name] = prog->MutableBlock(blk_idx); - std::cout << "In OpDesc: set up attr block idx " << blk_idx - << std::endl; - } else { - std::cout << "In OpDesc: We don't have this block idx " << blk_idx - << std::endl; - } } } this->block_ = block; diff --git a/paddle/framework/program_desc.cc b/paddle/framework/program_desc.cc index ba461b0933..0e937dda4e 100644 --- a/paddle/framework/program_desc.cc +++ b/paddle/framework/program_desc.cc @@ -43,7 +43,6 @@ ProgramDesc::ProgramDesc() { ProgramDesc::ProgramDesc(const ProgramDesc &o) { desc_ = o.desc_; - for (int i = 0; i < desc_.blocks_size(); ++i) { auto *block = desc_.mutable_blocks(i); blocks_.emplace_back(new BlockDesc(*o.blocks_[i], block, this)); @@ -54,8 +53,6 @@ ProgramDesc::ProgramDesc(const ProgramDesc &o) { if (attr.type() == proto::AttrType::BLOCK) { size_t blk_idx = attr.block_idx(); op->SetBlockAttr(attr.name(), *this->MutableBlock(blk_idx)); - std::cout << "In ProgramDesc 1: set block attr idx " << blk_idx - << std::endl; } } } @@ -64,11 +61,8 @@ ProgramDesc::ProgramDesc(const ProgramDesc &o) { ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { desc_ = desc; - std::cout << std::endl << "starting in ProgDesc constructor" << std::endl; for (auto &block_desc : *desc_.mutable_blocks()) { blocks_.emplace_back(new BlockDesc(this, &block_desc)); - std::cout << "Done constructing block idx " << block_desc.idx() - << " parent idx " << block_desc.parent_idx() << std::endl; } for (auto &block : blocks_) { for (auto *op : block->AllOps()) { @@ -76,13 +70,10 @@ ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { if (attr.type() == proto::AttrType::BLOCK) { size_t blk_idx = attr.block_idx(); op->SetBlockAttr(attr.name(), *this->MutableBlock(blk_idx)); - std::cout << "In ProgramDesc 2: set block attr idx " << blk_idx - << std::endl; } } } } - std::cout << "Done ProgDesc construction" << std::endl << std::endl; } ProgramDesc::ProgramDesc(const std::string &binary_str) { diff --git a/paddle/framework/prune.cc b/paddle/framework/prune.cc index 3c3ec87585..00fe551e55 100644 --- a/paddle/framework/prune.cc +++ b/paddle/framework/prune.cc @@ -137,8 +137,6 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, sub_block_dependent_vars.insert(argu); } } - std::cout << "pruning the next block, the current output_block_id is " - << output_block_id << std::endl; // GetSubBlockIndex(*op) is the idx of the sub_block in the input desc // output_block_id is the idx of the current block in the output desc prune_impl(input, output, GetSubBlockIndex(*op), output_block_id, @@ -147,8 +145,6 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, } } - std::cout << "Starting to remove unreferenced variables" - << " for block idx " << output_block_id << std::endl; // remove the VarDescs in BlockDesc that are not referenced in // the pruned OpDescs std::unordered_map var_map; @@ -186,9 +182,7 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, // TODO(fengjiayi): Prune() could be inplaced to avoid unnecessary copies void Prune(const proto::ProgramDesc& input, proto::ProgramDesc* output) { std::set dependent_vars; - std::cout << std::endl << "Start C++ framework::prune" << std::endl; prune_impl(input, output, 0, -1, dependent_vars); - std::cout << "Finished C++ framework::prune" << std::endl << std::endl; } void inference_optimize_impl(const proto::ProgramDesc& input, diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index e410549f8a..613dc20b6e 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -342,12 +342,6 @@ def save_inference_model(dirname, prepend_feed_ops(inference_program, feeded_var_names) append_fetch_ops(inference_program, fetch_var_names) - # save for checking - curstr = inference_program.to_string(True) - f = open("save_inf_prog_after_feed_fetch.txt", 'w') - f.write(curstr) - f.close() - model_file_name = dirname + "/__model__" with open(model_file_name, "wb") as f: f.write(inference_program.desc.serialize_to_string()) diff --git a/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py index 15f00f95d4..2211637b5b 100644 --- a/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/v2/fluid/tests/book/test_rnn_encoder_decoder.py @@ -225,18 +225,8 @@ def infer(save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - print("Print feed fetch target names as follows") - print(feed_target_names) assert feed_target_names[0] == 'source_sequence' assert feed_target_names[1] == 'target_sequence' - print([var.name for var in fetch_targets]) - - # save for checking - curstr = inference_program.to_string(True) - f = open("loaded_infer_prog.txt", 'w') - f.write(curstr) - f.close() - results = exe.run(inference_program, feed={ feed_target_names[0]: word_data, From 450c39a41301800f9b71e499d539ff7bbbd7414f Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 5 Feb 2018 20:39:38 -0800 Subject: [PATCH 12/26] fix bug --- paddle/framework/prune.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/framework/prune.cc b/paddle/framework/prune.cc index 00fe551e55..ddd6b993d4 100644 --- a/paddle/framework/prune.cc +++ b/paddle/framework/prune.cc @@ -182,6 +182,7 @@ void prune_impl(const proto::ProgramDesc& input, proto::ProgramDesc* output, // TODO(fengjiayi): Prune() could be inplaced to avoid unnecessary copies void Prune(const proto::ProgramDesc& input, proto::ProgramDesc* output) { std::set dependent_vars; + output->clear_blocks(); prune_impl(input, output, 0, -1, dependent_vars); } From d2a31fb8aa653485c9364f896ba0f3c9c9253737 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 7 Feb 2018 15:20:50 +0800 Subject: [PATCH 13/26] refine unit test --- paddle/framework/channel_test.cc | 147 +++++++------------- paddle/framework/details/buffered_channel.h | 7 + 2 files changed, 56 insertions(+), 98 deletions(-) diff --git a/paddle/framework/channel_test.cc b/paddle/framework/channel_test.cc index df9e15e22b..3b8150b427 100644 --- a/paddle/framework/channel_test.cc +++ b/paddle/framework/channel_test.cc @@ -115,7 +115,7 @@ TEST(Channel, ConcurrentSendNonConcurrentReceiveWithSufficientBufferSize) { sum += i; } }); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait 0.5 sec + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait 0.1 sec EXPECT_EQ(sum, 45U); CloseChannel(ch); @@ -144,38 +144,34 @@ TEST(Channel, SimpleUnbufferedChannelTest) { delete ch; } -// This tests that closing a buffered channel also unblocks -// any receivers waiting on the channel -TEST(Channel, BufferedChannelCloseUnblocksReceiversTest) { - auto ch = MakeChannel(1); +void ChannelCloseUnblocksReceiversTest(Channel *ch) { size_t num_threads = 5; std::thread t[num_threads]; bool thread_ended[num_threads]; - // Launches threads that try to read and are blocked because of no writers + // Launches threads that try to read and are blocked becausew of no writers for (size_t i = 0; i < num_threads; i++) { thread_ended[i] = false; t[i] = std::thread( [&](bool *p) { int data; - // All reads should return false EXPECT_EQ(ch->Receive(&data), false); *p = true; }, &thread_ended[i]); } - std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait 0.1 sec - // Verify that all threads are blocked + // Verify that all the threads are blocked for (size_t i = 0; i < num_threads; i++) { EXPECT_EQ(thread_ended[i], false); } - // Explicitly close the channel + // Explicitly close the thread // This should unblock all receivers CloseChannel(ch); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait 0.1 sec // Verify that all threads got unblocked for (size_t i = 0; i < num_threads; i++) { @@ -183,13 +179,12 @@ TEST(Channel, BufferedChannelCloseUnblocksReceiversTest) { } for (size_t i = 0; i < num_threads; i++) t[i].join(); - delete ch; } -// This tests that closing a buffered channel also unblocks -// any senders waiting for channel to have write space -TEST(Channel, BufferedChannelCloseUnblocksSendersTest) { - auto ch = MakeChannel(1); +void ChannelCloseUnblocksSendersTest(Channel *ch) { + using paddle::framework::details::Buffered; + using paddle::framework::details::UnBuffered; + size_t num_threads = 5; std::thread t[num_threads]; bool thread_ended[num_threads]; @@ -209,34 +204,56 @@ TEST(Channel, BufferedChannelCloseUnblocksSendersTest) { } std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait - // Verify that atleast 4 threads are blocked - int ct = 0; - for (size_t i = 0; i < num_threads; i++) { - if (thread_ended[i] == false) ct++; + if (dynamic_cast *>(ch)) { + // If ch is Buffered, atleast 4 threads must be blocked. + int ct = 0; + for (size_t i = 0; i < num_threads; i++) { + if (!thread_ended[i]) ct++; + } + EXPECT_GE(ct, 4); + } else { + // If ch is UnBuffered, all the threads should be blocked. + for (size_t i = 0; i < num_threads; i++) { + EXPECT_EQ(thread_ended[i], false); + } } - // Atleast 4 threads must be blocked - EXPECT_GE(ct, 4); - // Explicitly close the thread // This should unblock all senders CloseChannel(ch); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait // Verify that all threads got unblocked for (size_t i = 0; i < num_threads; i++) { EXPECT_EQ(thread_ended[i], true); } - // Verify that only 1 send was successful - ct = 0; - for (size_t i = 0; i < num_threads; i++) { - if (send_success[i]) ct++; + if (dynamic_cast *>(ch)) { + // Verify that only 1 send was successful + int ct = 0; + for (size_t i = 0; i < num_threads; i++) { + if (send_success[i]) ct++; + } + // Only 1 send must be successful + EXPECT_EQ(ct, 1); } - // Only 1 send must be successful - EXPECT_EQ(ct, 1); for (size_t i = 0; i < num_threads; i++) t[i].join(); +} + +// This tests that closing a buffered channel also unblocks +// any receivers waiting on the channel +TEST(Channel, BufferedChannelCloseUnblocksReceiversTest) { + auto ch = MakeChannel(1); + ChannelCloseUnblocksReceiversTest(ch); + delete ch; +} + +// This tests that closing a buffered channel also unblocks +// any senders waiting for channel to have write space +TEST(Channel, BufferedChannelCloseUnblocksSendersTest) { + auto ch = MakeChannel(1); + ChannelCloseUnblocksSendersTest(ch); delete ch; } @@ -244,40 +261,7 @@ TEST(Channel, BufferedChannelCloseUnblocksSendersTest) { // unblocks any receivers waiting for senders TEST(Channel, UnbufferedChannelCloseUnblocksReceiversTest) { auto ch = MakeChannel(0); - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; - - // Launches threads that try to read and are blocked becausew of no writers - for (size_t i = 0; i < num_threads; i++) { - thread_ended[i] = false; - t[i] = std::thread( - [&](bool *p) { - int data; - EXPECT_EQ(ch->Receive(&data), false); - *p = true; - }, - &thread_ended[i]); - } - std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait 0.5 sec - - // Verify that all the threads are blocked - for (size_t i = 0; i < num_threads; i++) { - EXPECT_EQ(thread_ended[i], false); - } - - // Explicitly close the thread - // This should unblock all receivers - CloseChannel(ch); - - std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait 0.5 sec - - // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { - EXPECT_EQ(thread_ended[i], true); - } - - for (size_t i = 0; i < num_threads; i++) t[i].join(); + ChannelCloseUnblocksReceiversTest(ch); delete ch; } @@ -285,40 +269,7 @@ TEST(Channel, UnbufferedChannelCloseUnblocksReceiversTest) { // unblocks any senders waiting for senders TEST(Channel, UnbufferedChannelCloseUnblocksSendersTest) { auto ch = MakeChannel(0); - size_t num_threads = 5; - std::thread t[num_threads]; - bool thread_ended[num_threads]; - - // Launches threads that try to read and are blocked becausew of no writers - for (size_t i = 0; i < num_threads; i++) { - thread_ended[i] = false; - t[i] = std::thread( - [&](bool *p) { - int data = 10; - EXPECT_EQ(ch->Send(&data), false); - *p = true; - }, - &thread_ended[i]); - } - std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait 0.5 sec - - // Verify that all the threads are blocked - for (size_t i = 0; i < num_threads; i++) { - EXPECT_EQ(thread_ended[i], false); - } - - // Explicitly close the thread - // This should unblock all receivers - CloseChannel(ch); - - std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait 0.5 sec - - // Verify that all threads got unblocked - for (size_t i = 0; i < num_threads; i++) { - EXPECT_EQ(thread_ended[i], true); - } - - for (size_t i = 0; i < num_threads; i++) t[i].join(); + ChannelCloseUnblocksReceiversTest(ch); delete ch; } diff --git a/paddle/framework/details/buffered_channel.h b/paddle/framework/details/buffered_channel.h index 00b63da4da..44bf84eb30 100644 --- a/paddle/framework/details/buffered_channel.h +++ b/paddle/framework/details/buffered_channel.h @@ -25,6 +25,13 @@ namespace paddle { namespace framework { namespace details { +// Four of the properties of Buffered Channel: +// - A send to a full channel blocks temporarily until a receive from the +// channel or the channel is closed +// - A receive from an empty channel blocks temporarily until a send to the +// channel or the channel is closed +// - A send to a closed channel returns false immediately +// - A receive from a closed channel returns false immediately template class Buffered : public paddle::framework::Channel { friend Channel* paddle::framework::MakeChannel(size_t); From dff5a8e6da5038e549c7ab66e1dbb94f619646d9 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 7 Feb 2018 15:48:50 +0800 Subject: [PATCH 14/26] add the properties of buffered channel and unbuffered channel --- paddle/framework/details/buffered_channel.h | 9 +++++---- paddle/framework/details/unbuffered_channel.h | 7 +++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/paddle/framework/details/buffered_channel.h b/paddle/framework/details/buffered_channel.h index 44bf84eb30..4275f919ba 100644 --- a/paddle/framework/details/buffered_channel.h +++ b/paddle/framework/details/buffered_channel.h @@ -27,11 +27,12 @@ namespace details { // Four of the properties of Buffered Channel: // - A send to a full channel blocks temporarily until a receive from the -// channel or the channel is closed +// channel or the channel is closed. // - A receive from an empty channel blocks temporarily until a send to the -// channel or the channel is closed -// - A send to a closed channel returns false immediately -// - A receive from a closed channel returns false immediately +// channel or the channel is closed. +// - A send to a closed channel returns false immediately. +// - A receive from a closed channel returns false immediately. + template class Buffered : public paddle::framework::Channel { friend Channel* paddle::framework::MakeChannel(size_t); diff --git a/paddle/framework/details/unbuffered_channel.h b/paddle/framework/details/unbuffered_channel.h index 815cebad2d..bc4233af73 100644 --- a/paddle/framework/details/unbuffered_channel.h +++ b/paddle/framework/details/unbuffered_channel.h @@ -23,6 +23,13 @@ namespace paddle { namespace framework { namespace details { +// Four of the properties of UnBuffered Channel: +// - A send to a channel blocks temporarily until a receive from the +// channel or the channel is closed. +// - A receive from a channel blocks temporarily until a send to the +// channel or the channel is closed. +// - A send to a closed channel returns false immediately. +// - A receive from a closed channel returns false immediately. template class UnBuffered : public paddle::framework::Channel { friend Channel* paddle::framework::MakeChannel(size_t); From 56ebb76c000fd56ba46baf64781a8c2df18955cd Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 7 Feb 2018 15:48:50 +0800 Subject: [PATCH 15/26] add the properties of buffered channel and unbuffered channel --- paddle/framework/channel_test.cc | 4 ++-- paddle/framework/details/buffered_channel.h | 9 +++++---- paddle/framework/details/unbuffered_channel.h | 7 +++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/paddle/framework/channel_test.cc b/paddle/framework/channel_test.cc index 3b8150b427..95360d7b77 100644 --- a/paddle/framework/channel_test.cc +++ b/paddle/framework/channel_test.cc @@ -149,7 +149,7 @@ void ChannelCloseUnblocksReceiversTest(Channel *ch) { std::thread t[num_threads]; bool thread_ended[num_threads]; - // Launches threads that try to read and are blocked becausew of no writers + // Launches threads that try to read and are blocked because of no writers for (size_t i = 0; i < num_threads; i++) { thread_ended[i] = false; t[i] = std::thread( @@ -167,7 +167,7 @@ void ChannelCloseUnblocksReceiversTest(Channel *ch) { EXPECT_EQ(thread_ended[i], false); } - // Explicitly close the thread + // Explicitly close the channel // This should unblock all receivers CloseChannel(ch); diff --git a/paddle/framework/details/buffered_channel.h b/paddle/framework/details/buffered_channel.h index 44bf84eb30..4275f919ba 100644 --- a/paddle/framework/details/buffered_channel.h +++ b/paddle/framework/details/buffered_channel.h @@ -27,11 +27,12 @@ namespace details { // Four of the properties of Buffered Channel: // - A send to a full channel blocks temporarily until a receive from the -// channel or the channel is closed +// channel or the channel is closed. // - A receive from an empty channel blocks temporarily until a send to the -// channel or the channel is closed -// - A send to a closed channel returns false immediately -// - A receive from a closed channel returns false immediately +// channel or the channel is closed. +// - A send to a closed channel returns false immediately. +// - A receive from a closed channel returns false immediately. + template class Buffered : public paddle::framework::Channel { friend Channel* paddle::framework::MakeChannel(size_t); diff --git a/paddle/framework/details/unbuffered_channel.h b/paddle/framework/details/unbuffered_channel.h index 815cebad2d..bc4233af73 100644 --- a/paddle/framework/details/unbuffered_channel.h +++ b/paddle/framework/details/unbuffered_channel.h @@ -23,6 +23,13 @@ namespace paddle { namespace framework { namespace details { +// Four of the properties of UnBuffered Channel: +// - A send to a channel blocks temporarily until a receive from the +// channel or the channel is closed. +// - A receive from a channel blocks temporarily until a send to the +// channel or the channel is closed. +// - A send to a closed channel returns false immediately. +// - A receive from a closed channel returns false immediately. template class UnBuffered : public paddle::framework::Channel { friend Channel* paddle::framework::MakeChannel(size_t); From c74445017d35ad344c5fc2a19a35c47a72358f3c Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Wed, 7 Feb 2018 19:18:48 +0800 Subject: [PATCH 16/26] refine distribute transpiler --- .../paddle/v2/fluid/distribute_transpiler.py | 124 +++++++++++++----- 1 file changed, 89 insertions(+), 35 deletions(-) diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 121b407cae..4eb103cc6b 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -300,6 +300,9 @@ class DistributeTranspiler: pass return orig_shape + def _op_input_var(self, op, varname): + pass + def _is_op_on_pserver(self, endpoint, all_ops, idx): """ Recursively check if the op need to run on current server. @@ -309,29 +312,35 @@ class DistributeTranspiler: p.name for p in self.param_grad_ep_mapping[endpoint]["params"] ] op = all_ops[idx] - if op.inputs.has_key("Param"): - if op.inputs["Param"].name in param_names: + input_names = set(op.input_names) + # TODO(typhoonzero): using Param and Grad input name to identify + # that the operator is an optimization operator, need a better way. + if "Param" in input_names: + if op.input("Param")[0] in param_names: return True else: for n in param_names: - if same_or_split_var(n, op.inputs[ - "Param"].name) and n != op.inputs["Param"].name: + if same_or_split_var(n, op.input("Param")[0]) \ + and n != op.input("Param")[0]: return True return False else: j = idx - 1 while j >= 0: prev_op = all_ops[j] - prev_output_names = [o.name for o in prev_op.outputs.values()] - prev_input_names = [o.name for o in prev_op.inputs.values()] + # prev_output_names = [o.name for o in prev_op.outputs.values()] + # prev_input_names = [o.name for o in prev_op.inputs.values()] + # NOTE(typhoonzero): consider list input/output + prev_output_names = prev_op.desc.output_arg_names() + prev_input_names = prev_op.desc.input_arg_names() found1 = False found2 = False - for _, v in op.inputs.iteritems(): - if v.name in prev_output_names: + for varname in op.desc.input_arg_names(): + if varname in prev_output_names: found1 = self._is_op_on_pserver(endpoint, all_ops, j) # later ops may produce output for prev op's next batch use. - for _, v in op.outputs.iteritems(): - if v.name in prev_input_names: + for varname in op.desc.output_arg_names(): + if varname in prev_input_names: found2 = self._is_op_on_pserver(endpoint, all_ops, j) if found1 or found2: return True @@ -342,11 +351,11 @@ class DistributeTranspiler: new_inputs = dict() # update param/grad shape first, then other inputs like # moment can use the updated shape - for key, var in opt_op.inputs.iteritems(): + for key in opt_op.input_names: if key == "Grad": grad_block = None for g in self.param_grad_ep_mapping[endpoint]["grads"]: - if same_or_split_var(g.name, var.name): + if same_or_split_var(g.name, opt_op.input(key)[0]): grad_block = g break if not grad_block: @@ -376,7 +385,7 @@ class DistributeTranspiler: # param is already created on global program param_block = None for p in self.param_grad_ep_mapping[endpoint]["params"]: - if same_or_split_var(p.name, var.name): + if same_or_split_var(p.name, opt_op.input(key)): param_block = p break if not param_block: @@ -389,11 +398,12 @@ class DistributeTranspiler: new_inputs[key] = tmpvar - for key, var in opt_op.inputs.iteritems(): + for key in opt_op.input_names: if key in ["Param", "Grad"]: continue # update accumulator variable shape param_shape = new_inputs["Param"].shape + var = program.global_block().vars[opt_op.input(key)] new_shape = self._get_optimizer_input_shape(opt_op.type, key, var.shape, param_shape) tmpvar = program.global_block().create_var( @@ -412,30 +422,46 @@ class DistributeTranspiler: shape=new_shape) # change output's ParamOut variable - opt_op.outputs["ParamOut"] = new_inputs["Param"] + outputs = self._get_output_map_from_op(program.global_block(), opt_op) + outputs["ParamOut"] = new_inputs["Param"] program.global_block().append_op( type=opt_op.type, inputs=new_inputs, - outputs=opt_op.outputs, + outputs=outputs, attrs=opt_op.attrs) def _append_pserver_non_opt_ops(self, program, pserver_program, opt_op): # Append the ops for parameters that do not need to be optimized/updated - for _, var in opt_op.inputs.iteritems(): - program.global_block().create_var( - name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=var.shape) - pserver_program.global_block().create_var( - name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=var.shape) + inputs = self._get_input_map_from_op(self.program.global_block().vars, + opt_op) + for var in inputs.itervalues(): + if type(var) == list: + varlist = var + else: + varlist = [var] + for var in varlist: + program.global_block().create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=var.shape) + try: + pserver_program.global_block().create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=var.shape) + except ValueError: + # create var if not created yet. + pass + + outputs = self._get_output_map_from_op(self.program.global_block().vars, + opt_op) + program.global_block().append_op( type=opt_op.type, - inputs=opt_op.inputs, - outputs=opt_op.outputs, + inputs=inputs, + outputs=outputs, attrs=opt_op.attrs) def get_pserver_program(self, endpoint): @@ -472,7 +498,7 @@ class DistributeTranspiler: self.optimize_ops, idx) if not is_op_on_pserver: continue - if opt_op.inputs.has_key("Grad"): + if "Grad" in opt_op.desc.input_arg_names(): self._append_pserver_ops(optimize_sub_program, pserver_program, opt_op, endpoint) else: @@ -499,6 +525,30 @@ class DistributeTranspiler: pserver_program.sync_with_cpp() return pserver_program + def _get_input_map_from_op(self, varmap, op): + iomap = dict() + for key in op.input_names: + vars = [] + for varname in op.input(key): + vars.append(varmap[varname]) + if len(vars) == 1: + iomap[key] = vars[0] + else: + iomap[key] = vars + return iomap + + def _get_output_map_from_op(self, varmap, op): + iomap = dict() + for key in op.output_names: + vars = [] + for varname in op.output(key): + vars.append(varmap[varname]) + if len(vars) == 1: + iomap[key] = vars[0] + else: + iomap[key] = vars + return iomap + def get_startup_program(self, endpoint, pserver_program): """ Get startup program for current parameter server. @@ -529,17 +579,21 @@ class DistributeTranspiler: # 2. rename op outputs for op in orig_s_prog.global_block().ops: + new_inputs = dict() new_outputs = dict() # do not append startup op if var is not on this pserver op_on_pserver = False - for key, var in op.outputs.iteritems(): - newname, _ = _get_splited_name_and_shape(var.name) + for key in op.output_names: + newname, _ = _get_splited_name_and_shape(op.output(key)[0]) if newname: op_on_pserver = True new_outputs[key] = created_var_map[newname] - elif var.name in pserver_vars: + elif op.output(key)[0] in pserver_vars: op_on_pserver = True - new_outputs[key] = pserver_vars[var.name] + new_outputs[key] = pserver_vars[op.output(key)[0]] + + # most startup program ops have no inputs + new_inputs = self._get_input_map_from_op(pserver_vars, op) if op_on_pserver: if op.type in [ @@ -548,7 +602,7 @@ class DistributeTranspiler: op.attrs["shape"] = new_outputs["Out"].shape s_prog.global_block().append_op( type=op.type, - inputs=op.inputs, + inputs=new_inputs, outputs=new_outputs, attrs=op.attrs) return s_prog From 7a6000a0b879719ea25e4c882ae6be79845ee57f Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 8 Feb 2018 12:08:13 +0800 Subject: [PATCH 17/26] follow comments --- python/paddle/v2/fluid/distribute_transpiler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python/paddle/v2/fluid/distribute_transpiler.py b/python/paddle/v2/fluid/distribute_transpiler.py index 4eb103cc6b..c5f1d51bd7 100644 --- a/python/paddle/v2/fluid/distribute_transpiler.py +++ b/python/paddle/v2/fluid/distribute_transpiler.py @@ -385,7 +385,7 @@ class DistributeTranspiler: # param is already created on global program param_block = None for p in self.param_grad_ep_mapping[endpoint]["params"]: - if same_or_split_var(p.name, opt_op.input(key)): + if same_or_split_var(p.name, opt_op.input(key)[0]): param_block = p break if not param_block: @@ -403,7 +403,7 @@ class DistributeTranspiler: continue # update accumulator variable shape param_shape = new_inputs["Param"].shape - var = program.global_block().vars[opt_op.input(key)] + var = program.global_block().vars[opt_op.input(key)[0]] new_shape = self._get_optimizer_input_shape(opt_op.type, key, var.shape, param_shape) tmpvar = program.global_block().create_var( @@ -440,20 +440,18 @@ class DistributeTranspiler: else: varlist = [var] for var in varlist: + # TODO(typhoonzero): will remove below line later. program.global_block().create_var( name=var.name, persistable=var.persistable, dtype=var.dtype, shape=var.shape) - try: + if not pserver_program.global_block().vars.has_key(var.name): pserver_program.global_block().create_var( name=var.name, persistable=var.persistable, dtype=var.dtype, shape=var.shape) - except ValueError: - # create var if not created yet. - pass outputs = self._get_output_map_from_op(self.program.global_block().vars, opt_op) From a1fc570197dd9cacaf5dc6941a47bab5fc2d4fba Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 7 Feb 2018 23:45:16 -0800 Subject: [PATCH 18/26] Fix for program crash when destructor is called before channel close with blocked readers/writers (#8197) * Fix destructor crash and add unit tests * Fix typo in unit test * Reword comments * Make close channel a generic test * Refactoring unit tests * Fix method name --- paddle/framework/channel_test.cc | 181 +++++++++++++++++- paddle/framework/details/buffered_channel.h | 22 ++- paddle/framework/details/unbuffered_channel.h | 19 +- 3 files changed, 213 insertions(+), 9 deletions(-) diff --git a/paddle/framework/channel_test.cc b/paddle/framework/channel_test.cc index df9e15e22b..a307abb4ed 100644 --- a/paddle/framework/channel_test.cc +++ b/paddle/framework/channel_test.cc @@ -22,6 +22,8 @@ limitations under the License. */ using paddle::framework::Channel; using paddle::framework::MakeChannel; using paddle::framework::CloseChannel; +using paddle::framework::details::Buffered; +using paddle::framework::details::UnBuffered; TEST(Channel, MakeAndClose) { using paddle::framework::details::Buffered; @@ -60,13 +62,54 @@ TEST(Channel, SufficientBufferSizeDoesntBlock) { delete ch; } -TEST(Channel, SendOnClosedChannelPanics) { - const size_t buffer_size = 10; - auto ch = MakeChannel(buffer_size); - size_t i = 5; - EXPECT_EQ(ch->Send(&i), true); // should not block or panic +// This tests that a channel must return false +// on send and receive performed after closing the channel. +// Receive will only return false after close when queue is empty. +// By creating separate threads for sending and receiving, we make this +// function able to test both buffered and unbuffered channels. +void SendReceiveWithACloseChannelShouldPanic(Channel *ch) { + const size_t data = 5; + std::thread send_thread{[&]() { + size_t i = data; + EXPECT_EQ(ch->Send(&i), true); // should not block + }}; + + std::thread recv_thread{[&]() { + size_t i; + EXPECT_EQ(ch->Receive(&i), true); // should not block + EXPECT_EQ(i, data); + }}; + + send_thread.join(); + recv_thread.join(); + + // After closing send should return false. Receive should + // also return false as there is no data in queue. CloseChannel(ch); - EXPECT_EQ(ch->Send(&i), false); // should panic + send_thread = std::thread{[&]() { + size_t i = data; + EXPECT_EQ(ch->Send(&i), false); // should return false + }}; + recv_thread = std::thread{[&]() { + size_t i; + // should return false because channel is closed and queue is empty + EXPECT_EQ(ch->Receive(&i), false); + }}; + + send_thread.join(); + recv_thread.join(); +} + +TEST(Channel, SendReceiveClosedBufferedChannelPanics) { + size_t buffer_size = 10; + auto ch = MakeChannel(buffer_size); + SendReceiveWithACloseChannelShouldPanic(ch); + delete ch; +} + +TEST(Channel, SendReceiveClosedUnBufferedChannelPanics) { + auto ch = MakeChannel(0); + SendReceiveWithACloseChannelShouldPanic(ch); delete ch; } @@ -381,3 +424,129 @@ TEST(Channel, UnbufferedMoreReceiveLessSendTest) { EXPECT_EQ(sum_receive, 28U); delete ch; } + +// This tests that destroying a channel unblocks +// any senders waiting for channel to have write space +void ChannelDestroyUnblockSenders(Channel *ch) { + size_t num_threads = 5; + std::thread t[num_threads]; + bool thread_ended[num_threads]; + bool send_success[num_threads]; + + // Launches threads that try to write and are blocked because of no readers + for (size_t i = 0; i < num_threads; i++) { + thread_ended[i] = false; + send_success[i] = false; + t[i] = std::thread( + [&](bool *ended, bool *success) { + int data = 10; + *success = ch->Send(&data); + *ended = true; + }, + &thread_ended[i], &send_success[i]); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait 0.5 sec + bool is_buffered_channel = false; + if (dynamic_cast *>(ch)) is_buffered_channel = true; + + if (is_buffered_channel) { + // If channel is buffered, verify that atleast 4 threads are blocked + int ct = 0; + for (size_t i = 0; i < num_threads; i++) { + if (thread_ended[i] == false) ct++; + } + // Atleast 4 threads must be blocked + EXPECT_GE(ct, 4); + } else { + // Verify that all the threads are blocked + for (size_t i = 0; i < num_threads; i++) { + EXPECT_EQ(thread_ended[i], false); + } + } + // Explicitly destroy the channel + delete ch; + std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait + + // Verify that all threads got unblocked + for (size_t i = 0; i < num_threads; i++) { + EXPECT_EQ(thread_ended[i], true); + } + + // Count number of successfuld sends + int ct = 0; + for (size_t i = 0; i < num_threads; i++) { + if (send_success[i]) ct++; + } + + if (is_buffered_channel) { + // Only 1 send must be successful + EXPECT_EQ(ct, 1); + } else { + // In unbuffered channel, no send should be successful + EXPECT_EQ(ct, 0); + } + + // Join all threads + for (size_t i = 0; i < num_threads; i++) t[i].join(); +} + +// This tests that destroying a channel also unblocks +// any receivers waiting on the channel +void ChannelDestroyUnblockReceivers(Channel *ch) { + size_t num_threads = 5; + std::thread t[num_threads]; + bool thread_ended[num_threads]; + + // Launches threads that try to read and are blocked because of no writers + for (size_t i = 0; i < num_threads; i++) { + thread_ended[i] = false; + t[i] = std::thread( + [&](bool *p) { + int data; + // All reads should return false + EXPECT_EQ(ch->Receive(&data), false); + *p = true; + }, + &thread_ended[i]); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait + + // Verify that all threads are blocked + for (size_t i = 0; i < num_threads; i++) { + EXPECT_EQ(thread_ended[i], false); + } + // delete the channel + delete ch; + std::this_thread::sleep_for(std::chrono::milliseconds(200)); // wait + // Verify that all threads got unblocked + for (size_t i = 0; i < num_threads; i++) { + EXPECT_EQ(thread_ended[i], true); + } + + for (size_t i = 0; i < num_threads; i++) t[i].join(); +} + +TEST(Channel, BufferedChannelDestroyUnblocksReceiversTest) { + size_t buffer_size = 1; + auto ch = MakeChannel(buffer_size); + ChannelDestroyUnblockReceivers(ch); +} + +TEST(Channel, BufferedChannelDestroyUnblocksSendersTest) { + size_t buffer_size = 1; + auto ch = MakeChannel(buffer_size); + ChannelDestroyUnblockSenders(ch); +} + +// This tests that destroying an unbuffered channel also unblocks +// unblocks any receivers waiting for senders +TEST(Channel, UnbufferedChannelDestroyUnblocksReceiversTest) { + auto ch = MakeChannel(0); + ChannelDestroyUnblockReceivers(ch); +} + +TEST(Channel, UnbufferedChannelDestroyUnblocksSendersTest) { + auto ch = MakeChannel(0); + ChannelDestroyUnblockSenders(ch); +} diff --git a/paddle/framework/details/buffered_channel.h b/paddle/framework/details/buffered_channel.h index 00b63da4da..77eebc9924 100644 --- a/paddle/framework/details/buffered_channel.h +++ b/paddle/framework/details/buffered_channel.h @@ -42,8 +42,11 @@ class Buffered : public paddle::framework::Channel { std::mutex mu_; std::condition_variable empty_cond_var_; std::condition_variable full_cond_var_; + std::condition_variable destructor_cond_var_; std::deque channel_; std::atomic closed_{false}; + std::atomic send_ctr{0}; + std::atomic recv_ctr{0}; Buffered(size_t cap) : cap_(cap), closed_(false) { PADDLE_ENFORCE_GT(cap, 0); @@ -58,6 +61,7 @@ bool Buffered::Send(T* item) { if (closed_) { return ret; } + send_ctr++; std::unique_lock lock(mu_); full_cond_var_.wait(lock, [this]() { return channel_.size() < cap_ || closed_; }); @@ -67,20 +71,30 @@ bool Buffered::Send(T* item) { empty_cond_var_.notify_one(); ret = true; } + send_ctr--; + destructor_cond_var_.notify_one(); return ret; } template bool Buffered::Receive(T* item) { + bool ret = false; + // Once the channel has been closed and all data has been consumed, + // just return false. Don't even try acquiring the mutex. + if (closed_ && channel_.empty()) { + return false; + } + recv_ctr++; std::unique_lock lock(mu_); empty_cond_var_.wait(lock, [this]() { return !channel_.empty() || closed_; }); - bool ret = false; if (!channel_.empty()) { *item = std::move(channel_.front()); channel_.pop_front(); full_cond_var_.notify_one(); ret = true; } + recv_ctr--; + destructor_cond_var_.notify_one(); return ret; } @@ -100,6 +114,12 @@ Buffered::~Buffered() { closed_ = true; channel_.clear(); NotifyAllParticipants(&lock); + + // The destructor must wait for all readers and writers to complete their task + // The channel has been closed, so we will not accept new readers and writers + lock.lock(); + destructor_cond_var_.wait( + lock, [this]() { return send_ctr == 0 && recv_ctr == 0; }); } template diff --git a/paddle/framework/details/unbuffered_channel.h b/paddle/framework/details/unbuffered_channel.h index 815cebad2d..92a16b4d22 100644 --- a/paddle/framework/details/unbuffered_channel.h +++ b/paddle/framework/details/unbuffered_channel.h @@ -45,9 +45,11 @@ class UnBuffered : public paddle::framework::Channel { // A transaction occurs only when both are true std::atomic reader_found_{false}, writer_found_{false}; std::condition_variable cv_channel_; - std::condition_variable_any cv_reader_, cv_writer_; + std::condition_variable_any cv_reader_, cv_writer_, cv_destructor_; T* item{nullptr}; std::atomic closed_{false}; + std::atomic send_ctr{0}; + std::atomic recv_ctr{0}; UnBuffered() : closed_(false) {} @@ -62,6 +64,7 @@ bool UnBuffered::Send(T* data) { if (closed_) { return ret; } + send_ctr++; // Prevent other writers from entering std::unique_lock writer_lock(mu_write_); writer_found_ = true; @@ -81,6 +84,8 @@ bool UnBuffered::Send(T* data) { ret = true; } writer_found_ = false; + send_ctr--; + cv_destructor_.notify_one(); return ret; } @@ -88,6 +93,12 @@ bool UnBuffered::Send(T* data) { // data that was sent by a writer is read from a reader. template bool UnBuffered::Receive(T* data) { + bool ret = false; + // If channel is closed, we don't even want any reader to enter. + // Unlike a buffered channel, an unbuffered channel does not allow + // readers to read after closing because there is no buffer to be consumed. + if (closed_) return ret; + recv_ctr++; // Prevent other readers from entering std::unique_lock read_lock{mu_read_}; reader_found_ = true; @@ -96,7 +107,6 @@ bool UnBuffered::Receive(T* data) { cv_reader_.wait(cv_lock, [this]() { return writer_found_ == true || closed_; }); cv_writer_.notify_one(); - bool ret = false; if (!closed_) { std::unique_lock lock_ch{mu_ch_}; // Reader should wait for the writer to first write its data @@ -110,6 +120,8 @@ bool UnBuffered::Receive(T* data) { cv_channel_.notify_one(); } reader_found_ = false; + recv_ctr--; + cv_destructor_.notify_one(); return ret; } @@ -135,6 +147,9 @@ UnBuffered::~UnBuffered() { item = nullptr; closed_ = true; NotifyAllParticipants(&lock); + lock.lock(); + cv_destructor_.wait(lock, + [this]() { return send_ctr == 0 && recv_ctr == 0; }); } // This function notifies all the readers, writers and From 87e3bdac4eafb3709f146cff358bca9568c24aed Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 8 Feb 2018 15:49:14 +0800 Subject: [PATCH 19/26] refine structure of cluster and quick start --- doc/getstarted/quickstart_cn.rst | 6 ++++++ doc/getstarted/quickstart_en.rst | 6 ++++++ doc/howto/cluster/index_cn.rst | 14 +++++++++++++- doc/howto/cluster/index_en.rst | 14 +++++++++++++- doc/howto/cluster/introduction_cn.md | 13 ------------- doc/howto/cluster/introduction_en.md | 13 ------------- doc/howto/cluster/src/ps_cn.png | Bin 0 -> 33865 bytes doc/howto/cluster/src/ps_en.png | Bin 0 -> 145107 bytes 8 files changed, 38 insertions(+), 28 deletions(-) delete mode 100644 doc/howto/cluster/introduction_cn.md delete mode 100644 doc/howto/cluster/introduction_en.md create mode 100644 doc/howto/cluster/src/ps_cn.png create mode 100644 doc/howto/cluster/src/ps_en.png diff --git a/doc/getstarted/quickstart_cn.rst b/doc/getstarted/quickstart_cn.rst index 51dd00f1e8..d511cead26 100644 --- a/doc/getstarted/quickstart_cn.rst +++ b/doc/getstarted/quickstart_cn.rst @@ -1,6 +1,9 @@ 快速开始 ======== +快速安装 +-------- + PaddlePaddle支持使用pip快速安装，目前支持CentOS 6以上, Ubuntu 14.04以及MacOS 10.12，并安装有Python2.7。 执行下面的命令完成快速安装，版本为cpu_avx_openblas： @@ -16,6 +19,9 @@ PaddlePaddle支持使用pip快速安装，目前支持CentOS 6以上, Ubuntu 14. 更详细的安装和编译方法参考：:ref:`install_steps` 。 +快速使用 +-------- + 创建一个 housing.py 并粘贴此Python代码： .. code-block:: python diff --git a/doc/getstarted/quickstart_en.rst b/doc/getstarted/quickstart_en.rst index d1bcf82ea0..70f7fe0646 100644 --- a/doc/getstarted/quickstart_en.rst +++ b/doc/getstarted/quickstart_en.rst @@ -1,6 +1,9 @@ Quick Start ============ +Quick Install +------------- + You can use pip to install PaddlePaddle with a single command, supports CentOS 6 above, Ubuntu 14.04 above or MacOS 10.12, with Python 2.7 installed. Simply run the following command to install, the version is cpu_avx_openblas: @@ -17,6 +20,9 @@ If you need to install GPU version (cuda7.5_cudnn5_avx_openblas), run: For more details about installation and build: :ref:`install_steps` . +Quick Use +--------- + Create a new file called housing.py, and paste this Python code: diff --git a/doc/howto/cluster/index_cn.rst b/doc/howto/cluster/index_cn.rst index c68b2655b6..a60521b4a9 100644 --- a/doc/howto/cluster/index_cn.rst +++ b/doc/howto/cluster/index_cn.rst @@ -1,10 +1,22 @@ 分布式训练 ========== +本节将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： + +.. image:: src/ps_cn.png + :width: 500 + +- 数据分片（Data shard): 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用。 +- 计算节点（Trainer）: 每个trainer启动后读取切分好的一部分数据，开始神经网络的“前馈”和“后馈”计算，并和参数服务器通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients），然后下载优化更新后的神经网络参数（parameters）。 +- 参数服务器（Parameter server）:每个参数服务器只保存整个神经网络所有参数的一部分。参数服务器接收从计算节点上传的梯度，并完成参数优化更新，再将更新后的参数下发到每个计算节点。 + +这样，通过计算节点和参数服务器的分布式协作，可以完成神经网络的SGD方法的训练。PaddlePaddle可以同时支持同步随机梯度下降（SGD）和异步随机梯度下降。 + +在使用同步SGD训练神经网络时，PaddlePaddle使用同步屏障（barrier），使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大地提高了计算的并行性：参数服务器之间不相互依赖，并行地接收梯度和更新参数，参数服务器也不会等待计算节点全部都提交梯度之后才开始下一步，计算节点之间也不会相互依赖，并行地执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台参数服务器上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 + .. toctree:: :maxdepth: 1 - introduction_cn.md preparations_cn.md cmd_argument_cn.md multi_cluster/index_cn.rst diff --git a/doc/howto/cluster/index_en.rst b/doc/howto/cluster/index_en.rst index af957e06cd..2640a09dcc 100644 --- a/doc/howto/cluster/index_en.rst +++ b/doc/howto/cluster/index_en.rst @@ -1,10 +1,22 @@ Distributed Training ==================== +In this section, we'll explain how to run distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the main architecture of a distributed trainning job: + +.. image:: src/ps_en.png + :width: 500 + +- Data shard: training data will be split into multiple partitions, trainers use the partitions of the whole dataset to do the training job. +- Trainer: each trainer reads the data shard, and train the neural network. Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. +- Parameter server: every parameter server stores part of the whole neural network model data. They will do optimization calculations when gradients are uploaded from trainers, and then send updated parameters to trainers. + +PaddlePaddle can support both synchronize stochastic gradient descent (SGD) and asynchronous SGD. + +When training with synchronize SGD, PaddlePaddle uses an internal "synchronize barrier" which makes gradients update and parameter download in strict order. On the other hand, asynchronous SGD won't wait for all trainers to finish upload at a single step, this will increase the parallelism of distributed training: parameter servers do not depend on each other, they'll do parameter optimization concurrently. Parameter servers will not wait for trainers, so trainers will also do their work concurrently. But asynchronous SGD will introduce more randomness and noises in the gradient. + .. toctree:: :maxdepth: 1 - introduction_en.md preparations_en.md cmd_argument_en.md multi_cluster/index_en.rst diff --git a/doc/howto/cluster/introduction_cn.md b/doc/howto/cluster/introduction_cn.md deleted file mode 100644 index 562008a898..0000000000 --- a/doc/howto/cluster/introduction_cn.md +++ /dev/null @@ -1,13 +0,0 @@ -## 概述 - -本节将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： - - - -- 数据分片（Data shard): 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用。 -- 计算节点（Trainer）: 每个trainer启动后读取切分好的一部分数据，开始神经网络的“前馈”和“后馈”计算，并和参数服务器通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients），然后下载优化更新后的神经网络参数（parameters）。 -- 参数服务器（Parameter server）:每个参数服务器只保存整个神经网络所有参数的一部分。参数服务器接收从计算节点上传的梯度，并完成参数优化更新，再将更新后的参数下发到每个计算节点。 - -这样，通过计算节点和参数服务器的分布式协作，可以完成神经网络的SGD方法的训练。PaddlePaddle可以同时支持同步随机梯度下降（SGD）和异步随机梯度下降。 - -在使用同步SGD训练神经网络时，PaddlePaddle使用同步屏障（barrier），使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大地提高了计算的并行性：参数服务器之间不相互依赖，并行地接收梯度和更新参数，参数服务器也不会等待计算节点全部都提交梯度之后才开始下一步，计算节点之间也不会相互依赖，并行地执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台参数服务器上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 diff --git a/doc/howto/cluster/introduction_en.md b/doc/howto/cluster/introduction_en.md deleted file mode 100644 index eb70d7cf35..0000000000 --- a/doc/howto/cluster/introduction_en.md +++ /dev/null @@ -1,13 +0,0 @@ -## Introduction - -In this section, we'll explain how to run distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the main architecture of a distributed trainning job: - - - -- Data shard: training data will be split into multiple partitions, trainers use the partitions of the whole dataset to do the training job. -- Trainer: each trainer reads the data shard, and train the neural network. Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. -- Parameter server: every parameter server stores part of the whole neural network model data. They will do optimization calculations when gradients are uploaded from trainers, and then send updated parameters to trainers. - -PaddlePaddle can support both synchronize stochastic gradient descent (SGD) and asynchronous SGD. - -When training with synchronize SGD, PaddlePaddle uses an internal "synchronize barrier" which makes gradients update and parameter download in strict order. On the other hand, asynchronous SGD won't wait for all trainers to finish upload at a single step, this will increase the parallelism of distributed training: parameter servers do not depend on each other, they'll do parameter optimization concurrently. Parameter servers will not wait for trainers, so trainers will also do their work concurrently. But asynchronous SGD will introduce more randomness and noises in the gradient. diff --git a/doc/howto/cluster/src/ps_cn.png b/doc/howto/cluster/src/ps_cn.png new file mode 100644 index 0000000000000000000000000000000000000000..f9525739cc8bc6506adde642aafa0a85ae3ebebc GIT binary patch literal 33865 zcmZ^L1ymhfwpMaF^-(egAuJX4Xu4 zu|nT;-#&e0pW0Qm6Q!ywgN%re2mt|sEGH|e4gmp$0s#T(4-W-=Vu1<64*Uh_rY<86 zQ9Dj}2%I1|%YJZ!fIz|o|3E@y<=_Gr%-Lw_y6Y+_@|!t1Fq@b=nOZP=J2(SZLqG_6 z^8>#+Sh$;zdOO%Vy77Apk^gfAKkz&FV-|AKe=c#i6C&4DQYDpeamPliMdp zcN1?WM>mRpzvREZN7BO0%+7|2*g4>s76t+?{}4 zaJ4a$b9A?G1#WgX0k=+={XcjA|F8JJze~l{#sYZjf9_`c&)xs~XaBigkOlnW|651= z+spra3UsqDq9DtE51BAx{Wkj=1cWGroTQkhH{{8hM}e=@67RW=4P*&H>KV%6nWO3@ zj2J#;+L>c`#x`f z-(I@kvc zJmo)!J0=u@mOJKS#VIHVeB=d+ien40DOIE(K!K07P~dzm)ZbP>j1)KuXh;)PrBoC( zj?9Ba!3Pf+aL!YhzpRWV3mgXs@RXWUm@}#TnWTxpF{uPTS2|djxfOYPn22HSI5@8B zt}Sz>y}I8{=ze>BYQJ3fyIZ@O5Xt`?o3QSCrKG6f`*hm6H1s&*Sd`~JE%&$U?&C|o z@5Anw24~NVB{JXp&4_%z+YgNC7~lpHg#!&Fg1t}7Uh(eCBR0~sq02N?(=jNBS`V?Yl3jrSVb=E75LVJWP^q;A3xg^R7k!z zH{w$KXnhd`N8I+XmwkNaeX-E}ihYRLiQoEjh$J9qEMPZH$22q58Uq+2PJ|wA|FeR1 z+o~Ez;6dNU`F~%GWlz37{SEld)cS6Ahq?1)tN7M4RZTk6Bpgo~Xw&-ZZL2=elIDev z3|DzRXARiEYoAu;S3S3F$3OMguXo-3dHPA<)Po=J@Y|?m)%CaE?b3e1+q3`6vC+EA z5K&oiuViugz~U6VzpP;_bM$NQ?C z^L|to^vkkb&*dZ;;`TC@PHiHeZ1&B|{I}XoXdHr!M*em`B$7z<_s5fC%bHol3v3_ge5SVZ z?6Ldpse8fD%kZ`e|FCu4-=`M_XEqUq#A8;?ZImrSnBi^FEP<(I(ez&LVcEdCFf5*T zD-!>vSm#E9l<&5jM4O1ohJ*L9#)rn6m!AxR@4yaz1DKHczLz6zM0%eOOTaUyeLwH% zY5ffcrXW0q<;Q(Efn$bUZM&d2HsYw@)o6QN-?no<@?Ag4EASW3we>=-%TK%*#zy$) z>1}0!(;Y2Z&$IUZ%JV>I6!?VpZX5&WUSRJ37<{j9;5tHeF=yzt7ABxJqR27EIj5;H z_?+P^`?3Kil3|`Kx6=ccqRd~BbABr$@~Ac(ar=|dP0?vvhmH*mqj>I z!1FeV01Mhv!_Z>tZn{3Ru(ndcvD?Ef&9r}UlJu`#pQe!n!OP(fR=IPT$^}@6#!-nL zk;1%|&lf{=&z0;nh-j{TiK}M25p-~%4ez#~MdyygQX~kM$!rHJ~ z(=h6e6k25Or^4~;Ryp{ciD2p~g?t`V$b%qapRFhwe_&LSHqFH(Iqhni-}j<$2IlFR zMo~>F2~J8y%i;H%v;t8{aK-*x94gyW3yHm0aj2_--_4wsDJmxOAsGtvw_pK0Sz>Hy z^ol273C&b&8aTsh?Y>KWo2&DOp%OK-o;Hi;`$?gOT>&-yw-1Shvpk3tn#WimvIWZ< zc)gQ=h%al(#Bs$`5J_*t6bZZwJs!4}JQ_zYt@?+-?SjdO$-mzTr1`@qaV$ zFo(a?{dlzfMOQX!UH&0|!6?A53hFjFlFE(-Dl7>37k1U?2)1m3hd60rG(Y5AyXm+) zy~Vz(HHOY>naE2NDX$}Ub%+LTnvymU{8s40r)63cRNv7ig!&#;lxZ8dNWn?+o+YI; z4ICBwCpt3=TtiO~!`)8Fkc7D{$q{i#h=)}M=i7g0Y{rdQn%glvngt1(tZa+0az2Wh z=WP3L>5xHEO|}YT3zDlLyZw0B>;Ho0D`0AjzdwAlV1%l%C#;z80Uqsz?|{vBLg
D&IYp#aYMgWCxQa%tf*Hgc5N|iG57Q z(3s~$pN~)##^nXBMmcKKyOQ~CW;Lv`QEZM^fAPINU$On*q}oZ}j^#kH)SHQAJkX;T zLEwXAGv|OwvH$l*qtKcQKaOEDWNTdyohQc0LIx9Ur6_wB(a*0#lA^We3YW zm3xTp43?mZRsXTDAJRI#CS_9@Xos5 z{GW}`ki*DQl*rDB#!z-i)0cd&#`+n4(IAaLI%tu~k2d=%!1TeV21(@0L(tHR5YB8$ z1R%(URAB2tB4J#Cm#YH0(eFyAuN**!4e5{tRR@h$t5->(7{dI)`5ODH{iN}?tcKg} zw}fFNNrY6(3Q(0GVOp1jeHdrZ1fyWp6k>7yZ+~4b9|UJZ4oryemP)58DrFgw{G@9tA4B;aB^MUW-zKk&PQ2B^d*(mCgGA)>^-#HBX z_vuQ-V=$NFwFU?HkE=1!Fpj?h3lS{=uK_~!rBry2;;jaA=ySQzk3Sfxn`YI_@Dpve zisbW{%uR{D2P|6OiIaP6hah>PX}#QKc!gUnG}t4Gcn=hy&M0`q4%JyH+(mu(5kn5L<-c ztk*^D7Ttm#SE?Bb03xdmSOo5g$Ft$Ps2-90gu+%S-@>vKi#+(o(+7$FS!E9i4n&}B z3c?a{{lfrPRTC!LS{IzU0l+EbA@_UMSC9Ec9a=BG+fSok>ZKei>#JN)BPG?t0%^XN z2vF7ZA@VMry77oiT8!flqQxpRNEYRrMYhGxy}yu3_35$lFJw}8_2fMVL2BBs(Lz8D ziCSWCz{!K&nG{1~A1pjqGba7|MHlrVPL27}U@dB=VIaVkDZWc;oY{-nqwH+{akzlB zAYNf+0AeCp;H18Pv`((61D4%tcn*&^cl=BtnFYo@44QO@j4ryouNjr9hMyLByr)i7 zEuBp%1VoPTDd^DLf`C5ssQOcRuf$xs|sZ4iBxN)tU$tur zk`pxljpT$RduXcQ_<({yzf`6(SDIh9J`rn4ys}b|-M54+ACwyu_XRqrwGC8EJDo5h zKY)pbIvDO2Bu&ZAZ@xMXqw{9U=0wL~ZzCRSFt(|d-xY>F6+%Yi%%sRCr!$NRU$^9k z*G%y>S+H8q3W3AW8`k!oIFA*nO~?~6UfDO?4UHq>b+|~qUdk85hVrarU_%y^-EE1u z$NEiI%bh2^fxgli#G#B-(BeojdO@pQdJ}FCNJiLet|S(Z|4Wx|9^))!iwLFuoVHaJ zmKU0Pby;ft{zJ6lUo45&-`6!#xRbF>Qbq5k-p^XUL}V56b9*~R>nfl0id)WgU*oZn z6cmfm?G;O+0I9?d#PqJrJ4o%^ALG&<)qer>JQ{S8&h`@*x>LjfzmqE0u+TRNC z2=y$=iH+93Bgj3@0Od6>5Gm{cM?P7wJjLdH3xz)c2CPNBr@-aB&LX;15HWc~T<#y3 zA3?k})`Xvg#oq9x=FAv$+?I%^Hsbwx=Qx9~Cwr?|Betg#ASNs8{dX|U$V;X+%8^-R z)OmLs!s9(`bRnOaI3V2PQ$7~0D#=r2wOXe33d15&Od~>G{=$^4)F2tf7sRsKx%|y? zSA$7X6rDLC{AAyQNd}kPqblw+Ut`4h9@E@ZT2jzrqb)9xg!WzTP?TQPHEke-&63hZ zeQNRFAV~7JWWJag@$nEPwJ_J>aYo|_R$tvQ8ZvZbXXPZTwD%AT{LA{VM2T@JrFqA4iFbf|?M z^Fw9(rGwF40qpQ&Mc4(VWLT&-){{1p@H=}YLI{lENy{Xw&oRZMAIPyKPc#Hf#3C=1 zf@bi7RIWo|LMt;z`1#QxC50q)#Wqr43VsREV|phMRtAnQu|J^e6k#`#n4({Q7%OGh zRZLV#Hqq<&Odx;BC{7a;OB(fyE`L(u8gY`CKPnH_!=hLJ?We+^IHA?=MWS#B$a3U7 zihysfjef(WViGE(!x+qYjD)sjn3d(o`Qu&91yLEQpB5buKgxU;dKX|^i(r-W7hc*2 zA!H`=%%Y>7>j7UTb`wNtb~X*D&mqb=P~Oy!4!nw`{o_d>ERz}WXfYX6tN~In z(3cv@>^-eMGwpZDek*l0P~?%leAdCNsm1&#s+JQvi^10f8yaR&?WD~mjyPd!bdPx3rxrH80)MX;8-pFT7wNEz;q9)8 z^h%Ijf*~l6A+alpMGW*8V)~gm|1Yi+#p`d6r-avSPmrEEP;Y#PnDhne;-?eiNqywu9_FN#s`2!vBjU^U#=$*(I%DKENE!<9?B;U3(09fk z#3`T4HPYImFoIpM&LHPvOL}Vdq#-+pA$f+FCB+Rw5Th)7k0IjzK%4ylT|!Bz%SFi` zLgUC5S)BH+ALO01E;T(CA}d-AVE!pepN)+hPfw-(X-+aI4gDg?U4?M-8}(8ppR^Kc zX;U3Nui5qc5@G`pH_nZKAwM;-s}us2W)h5=B8?`xm|8(-ZlGJ4*#clR=Xli6BkLzVoQA&{|33$K!#!ua6Wzsg9_!R1q?vr%^vd+nnpn zu+L3AN0b9^mhhNTYsdBH50Q@6*H{(g>Q5;sR|t_$o%h z2hk5H`F_Dz3x-hcQ#m#l9(Q?)OYD7cJjPfO_*CWjPHjSPq(Ol{mRDTObkn9=t|>Aw z$@qtQ6gwsKfs9N$rX=-myhhTMML8wJ*n$x8jdu|-t>Xp5RP}b^mi&&mIN=H{h-i`p z>am{K zk@4kPofzXSB=U$$NsBp_(u|%K_Xj6AA;X15FyxSZbNx;R=Kv|A=QJOS8WaQn#oDXj zl4?Swyhs-b;)6`^Koa|$=|*)$=}7#ff%j?9qpd=s$f9?P3-3!>s2SmwD^w;)x6~FH zb_3mJ2T214vN>;Ll=}t4c&e!Wk|6gdW%xjYn?kceNHJF_Uz3ojq-jFS!C!48X}D0r z6nZ>Ah0QTlQEF^(b8BH=xMNskKF-F}{~(!0zOsLzk~3ooKI>&SMmN3<35Ill4AaEM z4UVPwV@m4Z%is3J3^oZy6^S~K`dul40a7Z2g9L%INCovQ3f=~m653qqH)4;XO3`)D zEsP%;DeBTO!W2~p9DTHRupW}g=4a9rzL^@<5FMqIj9z}*aXslB>M7h^QR56{&szS# z#w2JXQ}^?&=?>7vj_{~(xU@w4;Jaf=!K5%1=9jL(e7E*PHl!fYLAc+?_WlCXte_E8 z{_!ciWrsFw4t*75$tG3tPn6uq{ogM8jei^Yzr5 zVr;R$Ys@o3kB;a)KziVoWk$4tJ)Et|@RVl4e=&t81`7Q^HW7;tuU=<-ae_IF@C((z zzPguPf(H~E77{e-jocF=?a7*6tQ)?+C<7A{o{_J#fop9M{6K7W~64qzsL%7 zLtjl<33Ru7?1DZ&`AdC-^XrC(5TlZI^|>^6eV$HEpu1*6DeaB{lw4b6WW^oF@Gs|X z0p`>!3KX%a{-Nxz89*472Md`>+w%W#?$&s~xiucGhSa|edO-{drGNCWoN2=Vfe$@c z;9L*(m!{gklzlgtWbw~k;;j9b@=q57&eMa~wNn43?CZdf`+q+k3lD0G-@FrtVqgTP zp7N=3q{;Y$$2@6ZF7NxLzF-Rr68&s$^M)V#`c|K870Qcr_W4hro34(HI^$(c#J0s4 zpVeXwC=u(dOP4A^C>WXzmtUADZj;d3mvR!{59vd6(&YP2f0d09ztv7hpHT%FJRmVO zCF*(`xH623hDil?Y5P_*gJb({x@sA(7a6ao1*Yr3@%wp!r(dmoT$XO?9sb~a zN~9W5ll{~;CCSqtnnAtTF6kt6O3GIDUJTo`G`#8jyW-bj?mFxMz%iLA%X2~m@$F@pwXS)76CeE=iti6d8VLKu(`2jH9sT}&@x<{83>P%%?aB?!&m7$Ojs1T z9HuA)Z9dGZ{AgP9T#@1w0SH-{apQ`T^yvd2?Nd3rIr%m*4v=#iDniYz@bEb@Eh6C z#7#n`-AAX=QNJ~pxFdt-Jv1+jUz(?XMh39#3<@OWcoD+Z{hv+`%kn?=wRa!ac3O&% zSmn5EQOt!=c;tQB12hhwdYIHdlMn{tNoccE4aK1p{&iiq(R|JrCtC+!YEr^LpUdIzX zrz3D|5$Ks3OEwDQ6gAz-RyaD-#QDhaGnamvH zHPPC{fVQUr=8w8^z4@S4eMW}&R#XFuJ4UvBBSLNPUJf0TU5v={Ch9DKHxyh)_sNnm z`)&g~$U=d!L9%@HFkmm!N*FK}JB7rFR zq?0Ou_ZDL1Um8uVMFu+Aw!bXWX=Z$4On&)+FVLKXs1BHE0+obh6C0<1Oy@En@FnVg zYOUJr5IZ%39zMP=BK8aSn!84D!TE4jyk>;O8fA*&)($c~DBSjHT%Zn$S;bgH5JP{H zQuRU?Q646C`1xug!KMJcklI9E<`*c*8l@`eVG{FJlTv{m59lX6c{zJPmF6U8)<1Bt z7J9Wfp;YP;3mJpuVdp^=x}Gfm8jl|{2ztH%)JzS_ft-vsKm-gT2co7Bcq<~N#V%J| zenBBqKHP zdOt*u+NNoFUSj%-#Z7ztX}u}a0ZdofrXqI+ecZ~PA<(Gz(}kgd)ZLlgK%ce_$w}oZ62u|FEa_ znV_i*_AP}9x9#H81`hb*-&cCWxSD>;o^u_d-YC2OBtMG)4EYgAQ!TnAH$NL8LID`` z40Do9)`z5((9v%b!Ye@xr}xzGrhM{vJqBE|`N2e32-vTr`UtNftTbNI`zP8B@U_Bd zq9#bs9+~k!vg9yyNUj%`mRD!y%P-GCLfpuRDNCvw8zHFF%9Le@)b&esvQ^Fq!q+_6 zDpbTG2@in2O&5nWCO;|{MGbZbq(Gcp@7wHq;cvan5p!2A07h@&$GQXi6<#G(ms06P3es4U+(QiJ)BG1L6>h+U(xJFHK)Sm2gv)mK=?gJ zE*>nOP9Ptmq zG(B#j(Dp?e9Is}A7@uq+8(6r92k~#}Tlyl+1>PYgN3|z?8B;v`ukN`7ch4uN&&8i! zzZgt=exJtWP(f^E5MS>?w1uuUM<|fJii|xX)aJ}m|3mjzALrP*=38h$E&cc4Cof7p zOeIb=?=a|d3YMas$jrouCkexP%@UKYr|;(e8&W8J1kby#|6Dpe~byV}ckl4imXZmZg5jTXq) zQ%6Qzd_al$l{=A_{9w6@D4#BeWmfi+q3DJAX8S952B8vDU`Sd0}t zqG$)_!Fc5ahCq5|dQNA3=Qwdodiu=Bxtw%%KJIk4C5Kc>6hoG^K3ne@j8o+U4IjgIG`4|31!vG5+nv0}@=U;dX19r5)qwCQ1sXHfoB-O#@Q~>=YvAU{7i#R{Vy=^E2d|^*7^vV%x|kw z&-g*f4JR8Kla4+^_NdV|Ps25wGDys0mw&c@YPa-z3O^?4!}C}UT1hK+R2*Z}bq0}} zUh<|g8HBn=IFPJ#c}KSx8w_mP^lP26I_s3Sd%M=?Wkj<+o$176ZZ0EF>@+L9H)%eQ z0QyEu+Hfh(UB459nr^=vjg)+YR5nyW=kzr@TMmIKa1L$e=Qxe?hBj>uI%@<+Flv;y zw7IbS$d51x`XHL}df0dDp}f1;wg7%ZHIo)PXB1ckd=7ezPA0S2&;YpS*{{*)+Jhns zZA32#3@P|!3GCyca+z)eDZ?FO8@mY0JzlUG#uQlICk;b_^UNPCao@Y003+k}F?yPT z*e4R{I$R=NUK0rFYI_x|FT>%epDqS*+Np3z69jf9#d?*(Mll&y%ubWT@Kle6yU7w~ zmu=`cQG!;-aZ;&gwLVF6{*iuc9lC(uFyi1~C)ZqSu_>^js!omT@pds^x6L;ovQL+@cPR}CFEZp87?BQ zui_#$sW1JTqpE{*R&`|`eD{BH*dA3Nj1Q_7r2YEW$Ce7VNTs?i&lLRoG~nr>^x;r7 zn%w_p(5Yy$t<4JHr~83|HUx?jU{k}@CBctu`v7F!&Kl|)D9yj}wSpotI<(=zpGVq6 z=E@3)1d~&NTOSP_R9r>D4Q}x#W#2DxxtUA;4cG+Jcy(XGl7&ik| zv~1qflUHcvUB;IsNLM*+(S3?iN3#EB;A&)vZP7*JiVTlb!56U{<49YEF#c69-nM~lADNDC70j1rF_H%$rP^Miq0 zReijn0@ZfEe!e<(a*-|*i65-J+h1|&o=ZWO#xS<#%L8B}%?`9{vw%uK)fiY0Q<;E`qY~Po zgc>#s7{HL=avCH;0OEa(Wsm-P-U~;AVmQCoH4Xxd zU$uTU#aYMTpzQ%wX%+nf_)Zo-AJ+t&hmi0_O9L5Vg2-zduVL%e&96`HK*s9Tr2{}} zxdRo!HnqEq*jtZxJb=S$W~Z8xI?sEHrTe+jFK%ArA2%N0?GUNPV||#(f7P7OIc0VO z-oUK=+7`grPG0&IuVUb!b5U>*F<|uexB%pJn?QE?`gmLyX9c9i1i(HP&B6LIg_mmp zbY=dhz05PulMJ)?kwQeokp29CAAViT<-t>uVV2H0;FAxB*OZWgiWldtm?;#PziA(4iBIo;t}^+aJ-(+{U1FR(jlO%rEYB)Y^ds%k;ZAlN~XF_+Y zSHMJi0c;1*+b!P}CmRg;up1p8}I*yahF6q6o1u3~Vfw z+;NCd7=zV)n#>|UCE5jN6}V=xKmc=NWv_jhn*1pIc<|K?U{GdMMB=>HfsKqTN`a-z zJ({kR^a@W;%7NAo;93sk(wl>OmM4`dv@D~D*S96h@;)DVl3q&l>-0In&RG^i34tIx z?Eu@9<){Rr(_239jbTU6fN{_%21bBZ5DgRO@<9AX)oNVi%@;8b2+yq#@tN`9$+I2D z^HI#+q$)m=XaI0@#r75OlBIUuF56YTKlY_IG2~6Gu@X7>uDa)&3n%NPuV^4;lmw?D z{IF96EI44xVZ|2<8xwhZ4#OvYBr+2_;pl(ILFjiq1)BUok{xH z2rdEsxH}+Qe!eafsZkT|vSk5v#8jvs*M#*1BQXy@)6G5r$#F}|<({3Jqqua7!Q!m9yqFNL^sYNwaEzIGL%cFbo@fZ^~u z*Pg#w3jx?4iZH0VvVfVrFv>ZjxPvfA0%5y`91fVnx;6}jE(S1ft^kUzvOAwQxuP@+ zkwC57%&XuPELd|sKzA`IznZ5g{q(QqeW5c^0p>`G9L>sRM}%4UN4yCXfUFxL_QC3I zO`R5by-3KUN3PWFjZ92wONFq)2{8D4P`L(PspjqXT_u-G4y|j^>l4CfCevC?IzJW~ zmM4Iz3#2K|0L^%;8>7T6ne7VX-&;Xh)n#k>n5ikr-S=BDU~*C)9-3#El*CLFnd@ee_EesV}d|> zTeR}Vwr6W^*6^;?wNod7t;36A?CKXshg^rsxF8ltcrGraalluNvj>k15M>VWtkd1# zeh?a%K70myb~QY;D;r@ zb=nYyxgx-sN!)?{LO-7kkdOJq8ASUPz~UGtix9;gu&y=1srnLfvj@PVnFqkSEV$q4 zT>rP@ArE~3Fv|9q^Y|ov%O;}#uo6{u@PGi&BQHFFyEm^NbK`6oCqz>(oj41+75?E3 z7htA%l{T#>y3T-q9G&n4wk9BA)JB|#Vw;9>Fok6MST|p5=wMbJoG!U<#&;yV6%iA` z;-#vJP=4>4egh^~>boOgj!LIt=58~6F4=F$A4vP|)iy`s_L-^|v}F4??a%E85%B#8 zYe4rbxsbd1xZ?t2*ExQNSRsorfyyAFHkyOBmmqYD>!b`ARil5Kc0mO|<$R6jm&f}0 z@zA*?UoxPyAd}&GX<7X<;CU}MmEEA%Hj!oo_MY`R|C=gP54*^&Lxf%K512k<07vuy zcn@eg5QOpCEe3VZm&b(ALzc0bxQ`TH|hB9}QynXOB;m^!stwf@&2ZtQQTnb}}!uaSjDzLU* zSqT|S-$ldD!Psl_Jl~)k&7?wWb~rW^*u)wEwo<5C`n>9c9WdU2x~+L-jv8(pz)-+d zc*!qLt_c#Ya#5+Q!$dRY+8H;b(L2&4U~UADMy{O77$bw?iwZi74v7RYhl2i##M7nLiQ}VndZ)N zv}+CLJ2q!xd_1I?($2MST5|v!oyomAl727Pbgtl+`{!v#_0?km-rkBn8s$J_X<0)qd87a|9rEBJ9*k=4LA)@g z&tE$|?$l$a!GtpqTg#tf-s4ctU=ByZNr;zGUxen7`H&&d;2U8X@|>*#WeC^BPUoQW z(oS!tk1rmYU)rMi12v1u`vfO#@}2t-q>eQ09$kS}U7Fkh*uJ&2qr(opzE=ynluJBm zMrHQJ&=4k5xu$tu&)kl@-g6MP*8%JZ&k0J;4)M5Y!$D+w-j-kLDyns}?}DhAzh-m6 zt_d=;vL(w0`%_QN~Bcktnqnb+3g3ro= z7m{~yE*yY+?$DQbmhHWRA-xcC{=C-@9cR6fr%yn+K)u%qDi3jU*=1?Aqw8k7XXZ~p z?!`?op$3TW`|-&et*3|6*7Y+WqS9`O<%Vg#d> zY6#R(aR2c(TMiDsU;k)Rl12=W96;>Mrb1m)w6l88L6!*HUpB9;-?Qu;uEJhEKB~~6 zGlj>?^Cw!AIW@)nc$Ar-uB7Y-*n4jf?B?|uC|0p)+@8$?(i-$a%mu}aG-^A$hkd~S zR?9A+bl{3>pZ+xOv^bDWAN9&O4OXUcy*)-UX9UpcA~zB6^g z)KSCAF>EHWKOe+AqxmbO74abxtYjuLaiZP#T0k);=td&y?GNk(w>Dlv?Xfz!Ifd7w z!`W;g-<<;_Uav@R67s^N8wofS9Prxf+W-7VpNP2jT}R!YtmH*~e>uKJ>Igibg2K#F z4nuOnk=Xf?t=JAS2dceMG{FympRUgf`t|rO1LzC7X2{TGmAm~05DA61`2-AH;ChtW zvUfKtf0$Tki6JQ>1|}W;#uwcSso0)wjiZye*+V|fZO;@qTg^oeP>Z>Sqv+uZ#UGHJ z;oL!G@>xHSDoZbJ0aD&JF~#1UT`(VJX)jaCl2txYiW;9{ z;m46&`5(sK2A1TjJI!PAi={bl;svC%3?3AsfiN+hzfc9wMWJLvAtsQ!pVs~g`U8nU z)9Ob5lfAEg9;kuRViZ7Bc^)wekSJf8Ry$2u#I*>^ENdNZw-N=G_6M|{aqQlc4bB~` zx=oDLNTKsM;zNs`{HkZG1M7Yq-hxx!Yg$aM>V^0 zb)^W=S?X_?IXIJQkDlFpu1jvP53)=g%a2*oMdUw)@~B!d`~)PA8RA8`K*f{r%ITSV zyI-vU01b7r5Oh-UMbJ-gOo!O&4FI21f62x%);_9hpg zGUbnQ2ZgwzWuJqF+fuy%a?9Db>3Uk>TPZdH+h$aov3k!bd~`YuhR&30BDd})fH?So z^wF8%i>H4e=@2oYI~cdlE)QF6@(0RhjKXfNc;y=cScjRowq@CNL*w#^$Hj4kW=$6> zEp%caBNKxOYzxtY`uSe75lFrf_0!KKA*h77Im&=)kz=rN0MVyq$&w;)>!=JS@GOQt zCn~r*I(7g$+3vF`-pycr77++lZ?&V?SZ1uR;pi@Rwy>-PHW-437|5T4tgWH3X;p-0 zygPFT5JZxQxbiD@n3i(b%cN{gao>}QhC!D#HZXIDZY{RQVhw<`v?|iX+Kpz!WTm-p zfQdI3sH{}dn!+J*8xN`@57=*@;2}+K98iTel=zK(_E&}yrZ0Hrl2`FN&Q1rQL$kKT zXO<|5VxvjhM68zv(LicyeXRPLnPYm%Gfvs`wdQ>_TkXZXf!i3Ms13o`6Klx;6p#U& zoSri_?SX+m0tw%DQ(XBotRZFHJD`LLNL*z`s{umxCc|i460vP9{gV?WqWs)E0;IT= z^hNCMG2{A|L2GM6=Izs3Zne=6`(&m9dhelaspcbjXO^L{Q$U0+Jz7aNkB;O5ijd}S zv$QZ&o18NMlqzn_tv>g#pFiQ{PGNLA$P+KfGYq9>FcG-g*9qv~M|4Yd7H#G@9N&K) z0Q(o2wx}RxXOP-g-GJ^bkW|m`_d|ctI6M|RgS7JWBEIf(^U(HFn<=M;URU9MKG;EoX zAZ3G7!tIsirU6On{XU^=H;_8n%1U@H1w*R^L|khHU{O~U=NlvFO*mtDeocvk&#3U>$(rBB%T@h2j#^MC z*XA7=yzR8cHIQ2iz%8p0f4Pa5zs2dkhxw*cREB4nQP|KsrPWp?+cUkmFZ4JEWawjr z>FYGg`Qe-(;-+#QX*>(8lSp1QfMqB{>Wp*)QXwPXi$Niw=q-0)fF907de_Jb4VWB^ zyt4j!SC#AG$ncKswnvg_^m~gBMQyBg6nsCuC+%3;Fo__t+>jW={Col%4o_n4_wJM! zC|`7pFZ|}eNyZLm=gNuW`e^r#mTYr zACo(y^%c825?V_0GYt7|)m~V%1p*RX$4^f?0DcZRUm*`4WP@IyDW%O3Ef2* z%aDuV{RQ}$RkvIg(}>P;8ajs=_jW=2wB9(lm0Pm}q4d=dn%Vcu?VaCFCn<;NM=6^oZ*gb*s~rRaE=L-;(FjLpBbmHbX{ZU z;mZB4%B6*>aafvj{G&QRW9`rkQyo`6#&rW!YBo@vnc=}8=qRr%oKD?ACP~ia0%ie2 zqkW31K#mnN!y|;yd6nE9vZeN=bxU8Mewt{XFe3`6W)7OO6ev|0)pTK8R&$#=D$eHx z95wnY4C!i?5fi=tAhYu85Aj&_Wn`8*BZ|jHZq=7D189stGb=q zW%B;c-T9F~H)bC-TuyY5eis8+Nc;g9QF`yN&gs!*xe1jC?cPP;HyLD!9V_%x!lScO z@lKMBLh#U2W_eoJaw|~+JhEAc=rkUjh$pyR8E-N)Z zpt87`q5zYLXT7O3_5;3}$pYL;DGFov^W{O*^dCJZD$OT|wVPjPZ5L=1j{u2zn1+H{ zk?(%PJkNIQis>Uu!flXY2xj}Hh9n!R+!%!k!c}rf#9KqvR=XEI{Me6H%OC<^k(M^?itrR~~IsTp?R zLxMmta_8dW7kbWGCHM<|h@Y>H&hG<+K1ig!%Y<8C#@b4+ z@NhH5KvjyMo`dPuf61rn(O(Lrb|b>dwOJP zY~qyy05TGdGLc^H+m*;!MWoePJaH%cyakb6D>Sm2ol6cCwLIMN(LVKKwhjI8L9iy# zh<4&D=Q?uUEAPIpL-Sl4uI-N&OIzo>4%$gYWSHl47Ab$Dj%l{p)=qfe?HO;c66jOw zR2fX$XU|!KJpPp0B1|_W$!*DFNrh9VkT)tzKF92DV@?5fA+3Q`Xm@cVJsVO!#UQ}k z`fY5!@As($QR_xC;9~(1%+$_6N}PU*yG&&ZE#X2e)@de71Y)ff9=M|4@i)L!wYId* zzQPX7xeR`(y#XK%MYVN6NJuN|AeP1f#??b_%&j;AZ-Qt750+_)(uo&2{V` zpIvLm)Uu~W{a4G)4+JU2tz!~_7CbZed&h^4$oA?ZpUUC*LJu5O==F!z3xXN0eAptC zV|43Cg}Q-?QjiRWW(koOgn*#*0CKM$$-Cly=a?}nNKKHGKE|`iwy)0`EV_*iR?fU1 zP-*M{so|DZG%!*iTa;B*`ecN8-?nuQKpXdNhPOjVE&`RN*yJbD%K~0K?aKh^^jB15 zC_^*X?A`Fsn?4$3@7Rk*kxxG!IHd{u1HL7|l7g&!L6>rc%l!3N6WK}P_fFbf^50WE!c@Xjb z{b#hkk3;v7hJYOlak0PBa2c2&$^au=Sr?su__zYpcMd?BCUwfQ?*;dv2HZ>dvs*{S ztj+~Jrk@nR29It$yurB?N4yt<&^p`e<(Lqh`Cp}dbySqy8!gSyjYxNcgfvJa-9w{* zfV4 zX7%skRE&nUR~ql}0VpmYLcR~MEiO4afBu?>^Y1C0pw#24Cu8Z$JqFAEh4OI-UE^y1QomNwA2FbcGREsT}u3wYV8fM9BKpU&RU|57D$pZ`9-j zXZ|_hl-M&&~Q4?V{IEuEeppgEd#K&IqWNgobirQb&G@2fRss>Hdx-g1<8+b*_t@&oqi`ok<0N~7_fV|f5i8f51L zmv}{;=i-y=G^+HuVfVbM2(pOw(hi8sIA4D;HpD`Fipdb@;g-l#OPOmoHYH7c$jY9* zbjw_~ITqt8ggh7#9xE%ao{2(6^E&ay1=KGKD5f%VkWKO0mzAlyZ+|N3oO5Cdmq>|i z|2VZTp1OLk3>z@1=?Ge zDJ8cnDQDxB^qM>X^v|X2N#N5P0GA-^TO8QO1rg3$k5SW&c7_xR)#krLyKR(C<1_y9 zo+hD^vOY4XwH|l#Td*p9HElM>B;yMZ5vvDR?Pl|TpqyMT;nKr6=R0U8SR(ZXeH`uzKF{I}lxCC=jcq(5JY$cA~wNE;fxhj=2$p3B$W zlM1WjUpaX&m_-gk7jVu2H7pcbl`sgoEyGJHAeNQ2Fw>}T8QwB|D|oa#x>A2vnJ7zu zH~5_|MbDT#hpa78+on0z% zlUA5bsVN{#`ab89E4K>OM#9xk`z~n>Vx^{zv-pu{WKV8T2U2U*P9npYTBInD!Wt%S z1yF(hodUf!cTm|OGK)UW#Th^y_bE&K94-U?ECl#b3YuCL@7`BV0CN54+Mvr3YM_iWQuXvFv8RdSRR?CD7fX}Z z{)#V|D_joe7gCfWi}IpKrip&QZn!WBX*R_?^+b=RY|1B~*RM1`2PN&}93$l4?>UU< z8Tx7%9$V^>pg|iQ0}IPx?1y_S|Ag)31kCwD#wNG5_5X0^mW1*p zUz&uPqwl6U1D3A?y9J5mz^MY&6(^EGvusTpAe_dCsA95rSOA~|H6BLo0ro{WhZs*f z<3y}t!fM*Tm1j7#hu>K2(11Pr6JR)AFp2*jo%XqE0E+GRpr*JEM$H4LvJSOo!&k-- zaeGYzS-i;i)>oj&m{OJua?a0y3oHJuv_LTH2mJ-O$EAQq`mkouNhfRSh6<;)H8>3x zrDicgUvQyV4vqmzdhiGopgep|BleNx)d-&0>~vRhAhvMW_&p$9q&K+dIdt`h!4=7T zfA=4$|8y9T`jNM80iZ6)|A!3@abl3@9+V?KjjB6lOEOm+2P_Rx zID;t`V3d&8S*|GrqF!#V=wvd|f$2>p%8tsjy9+H1*$>8U`n%StSHMH5*Dcd&j0O5( z6I5IneS?W~Aa7*twd8YK1*C+hDypxY%vKIdB zTE~DLIuw^-hrW`?zj{z6$?^M-_M__6MX<+w9vp_bH(h)J&K*Ggu@T4tb0^A$Z#4(m z3(;#qVhl9VK~TaV;@zQNu>h!Hyh7R;cq^P|7#zA%6A?7YJQZ)DO6A`?sxl@6&AsD> zHP%|C-puR!D!{Bp$L#SPok~txNu+ zuY16#@OmKSBZkEWZeqyFxK5t{xyIOS)dK8u)5H+#@)!VKO@&8!^)dua0M1IV48~_E zcute{mLvjz<5N}#R4srN8YT>Q(zKxYm(89(*XM4*7QGKz9SvWSU85La1s=JlW#<&n zzlML*2%h-{W1)Xu2F4ljWfhUwG1maT2-qP|-35y|mj7~IgcaYkyEPy=ZGs@NU(xKj zQe9uM8>rNPgiH+v+m5^hsrMI9ibMOrR7*t}Uge0dY+XkPONQjkVX>%#5}lMve^?3x zgBERau}$viiB5v)5>8xR3Oiap$0%2*@J(RK6EvTbP*W79cC!HsUq(HjldWXk7l$)U z6q+kUopw8*yx|qiw>0tvGwBn+e(UHEyP$2lq)sPO&}Zd#q;|TuD{EuJ_Px^Sj2Wb* zAx7QR#in0OG#_3e&VVTDo*&q`bc5)^G=5v~*ZLdre8V2#z!HGOQ+Q{g_{cM$&BdDW zYAl&lOYJe^a7A0_9f@9tA>CJi*1V>8P0E^Xp@qem7J=Jg zi%arJ&Z$?y{>yMmC%Ra0D#6$@fI7a`Qn=)>=>Ytlp2kx>Y;A!WZp?R3&7en21F;r| zXaaL4APLb_O@Q)JjA~P<(F%H+B{{tCtPFuYU=1BC1M-&@plo&nf>Yt+*X7UcpB7g- z>3s`Tvw7h7$~CZd{jRa&7~bG(zNsI8PoKzY@1!)Tkol}YdLBgYj>PK{gnEg$QR>AT ztbW!D&FvV4Ia?9~==A-0`vlK;l)xu%tfIogWA= zmQ_K|fXAiPG&!5VAi`Lo zFC=4#H<|8hhRS{!i!hd1A0DC9p%|bS?RC?p1gWxOy+Et%&qCX$0>!>ww@$z@aL0&| zW<45(u6GZE)msIw5ry>-4aH91Ghj=B=fDmXjA5&^@d@HM#-CRh7f7~|5^69M1#5<& zEuGAreUyvLPx*>~q#vj1F;DzT(d*Xp-U&+$NcCJ_t=lhOi|0$b@>TYvLQ2 znlH#aWu~;DJcW>Ll* zx1AaVc_+^kSH#p$vgP$d>;mbAS*5jD>uay^vhFiNOFh~4U|;*}Z04_A{&fQgrCe_{ z0#2?RaFsHF{hK7a=_;<0;|+hQUH~Cmi)FnT8f9678Cs$JfLVCRvYZ+YZ)G$H!Zkfqi*RLjDcxjfxQeV)Y@g`Vw*)JZYSf zdbjENq2ohl5F)Lv5vO#p;SyrsLbT#9u6{$GpLohU| zxU@aMv;$$>_r-*7Pe_r{rHH_SPO)4Y?~l73hR9Awn6GLgvSOAy>vT0L#N48h2REEw zSrYg#JEn#-)v+Cb!Fn!|0t;KwvKb^BL0pnStN@}PHJk<(ZEhGc$NlD(7?)Rs^F4wt zJk+uOyyxNf*+!h9>1ojEns;r`EhvJ$FNo$J8dx z$wR*%@bLQD7)Zd$>DdhU)iD{{;ox&AM@-pSgsniw=){?A^2nm~Zj@6b4M+Tz>y6*) zL_i_f)A%Jj{tL@!S-?gTYs$Zm7-1DgYYIwpdFIW~RtOOlnHSARNKYQZ)F~=jLCC2a z@nhGgvBn5rXEpbXJcVL{8T0|fhpZq5qCH?S*m|75*8o3`?+lA@REEjze{*%)d*p-2 zPiR9b*pgq7G;my;ehBawDGt%z0EVD>!h{_l2l*F{SQwZbpm96a|CW{iV4hlq?8@M? zKcYsU!H$>i+!OmSY&VY>&XCE!YKE-vYZw;JNyS!yXqNo9Bno*TjB+W87TL@Zgh%D8 z9y|!Rb;OJl;kmtleS#9dEyVgd-r*L+S=)c&=K{8dE6W2vNX*W_EXuSg5W8#?d_XVb z>p^ui{|1dZQ*U^M{_kGz$Z2$qUOAceNSnr!A_7jx?|=XiYOZB_0Fyd_Lezj6ewqnj zA`EF#vj#JjxA$vbf)K}PqRf$9;t>K=f`+>!F&vDi_tYH!b)D+Ju8S^+0KPTSqJDs@ zcM1TbKzF*o;yHHxG$}OOC|TSY?YNHG&dmFO2zf0l)Zq&fDX(6_vrD}kBjoHkn>McKJob4R)~pv}HlVzvAWK*o<>-ORzJ2;6^xyJI{oaG!+r zG#FPj^98uyzJmzg?X$3)Eb$J0sH7;=)gDhGTsAQ>MLA!Ws74ed5S#}VO&fWtD!ppZ zqc`@JpjQoYht~c@aDr7F3b3yPgQs10Cz~G&Da8!XW?>awH$WA(G&RMd0M!~owBy{= z-Y+^t) z_6KmJqrq(OuNO;~r{Q{~fc7Wq<=r>k1xm21-(r{io!?FWo8JoNGEX#=XudCIJp6hs zIZ|`?d3ZFs9F)d=K(gUjFvcPDh@INz(HR%2C`!mQOADSS;1H%hCkgN;s3-x~9-!-3 z0Tl1ADHuZ`Jhf8QvW+M=1|RF&ZcmvdID?-r>LOZ@>bm7xry|0a+xQ^PU|hDAD7@V_ zV3;-mv^Nsv)*P)`!-MO%-h;KHYh|s@0O6Ylh~R(B{neHX#;mX> z-f-|aS`@vBg1XTFIcT5wz&cnWakX|~Uc9EF4LLyF=K zEb?PD_}eH2SlyVUW<4hgk2>Js+4WLQF@m@S2v?QY+i6B^QQ{OA{oZ}^{XhMh{-yomgcCLav$B;G(L;XU&GB+?Ks73Y zeR*?sv50A1;~_&*T>BGlx>KOG z!iFvIgc^ghmAYgX7@snGp}ceSPTU!-a+3jy+)Pn9+&enkM4S+&`9fs1KIG~HK|o8tx?HM869rw#P1J@#8%s3+c! zLakqtxHCNcHepRQV}{xqmR0*v%@J+C1(bFWT=w_uYx_{7Xv*wC_W0IWMCT}``zKJ1xKCI=o9#}c zY&?I@HE|{RWR}3x{#|@FvtYTrxy zp7;Tc&e+_bxh-XHI4a%#c9bE68>#{CBspnY(@<|SX{AW2Zg&Sb!)n7GfNB`J-`Bv$ zBeP1FT4O;ie(8IZyM0z!#W zH!Nfi!+$p>DKbj{mvInYUcnZR2hVQIMAsh08kHj@(jrq0VgL~5Y0GqunN+kTAbVG? z{k|u&WTM#;2;-k6z+%skk%Ke98wtbKmw+16lFtBDLL!I?+`MlyIgCH)E7at{&6HQEAWefKg-g-T7eCwhu+bH!ak11&7*xNVV zntO%-Dzc?EV22_DXbP;4>jtnzGJ$3X=B+C+CHCAw=fqnwWR*|*E#N8e#=QinNdNBk zSGNu9Id_kGIBC|q{L4u$5uEeCH)eH$|BdO^z?&!EwiMxOE70M| zRty8!B<8ipO8=9N?9<2JUsyP!r&iDY_93LL)!SWx9ih4Y z*Q?V$U!E<2#f*#?u7%3M3d$L>6v43ebOJPuujyz2EK|-#vit1CAgx%#g>dHCd^ikj z0qb=lZ}K2MoSwAbEWJDGohZO~C6mx$syISy1Q5auJ+>710VQ;}EW-ZhRznZb4g~6< z#oUCS|Fm-WNIbCo1TiW}`LBJAJZ>q>pr%}sVgG-hFsJ6qHOSD{E7sd$1ewQUijCeG06^?hIsdAX*=}~Nzp>rbKE6tz4G~g#L4?lY3;ozvp1=`fN2LAd zB4Drm1n`X;h;9fC8-Lo5?AVX;k$>_8ndQMj!*T^12$z9v0sb4;cv<4JgWY%0C+|T3 zdqzp8&ig|dUE~aaJusF}gvk2LSgtQ0y;IJ!i_GD6$xt0%|LPfn_Fn+seO=OR6UqU{ zeZ-~^@#hdtl4KTl$%eexs(2(4_g#%mL7Or+%2NS=fPVR@^i%H@;*~Jc?iyFzGfGQ` zbm3@L@_n|-S_FI_07yvZ#$Necd}Q)XF98oC6@jBNa{~Z$-vBrerYXOr|KRwN5Vat| z?jA^`_PTA~fKBkKNAR!(K!hON)CVcyXL5_1Wh-eU4I1>t1WmPWBoFLL5kLTv49>AeH(sfBbjCH!KJ4bQs zqo3TZ0phdhuJSH^pFi_C*zNr8zpLB%Qw$y2@~Wc+CxAN8N#0gkFgn|iEK82a$Vie1 z2uDIpM&1&H$Ra}JwY##Y&;kAEkzS~EaoQJ9MGYde6Q39q{2W!l>hpL9lSFEw5^VZk zn%5lyLy3)DG)4v{ebz<10(=48-(fBX0LiOE5tcCC=pGF4KnREYyzSV!bD$1SS-T*P z{QQVj@*DbaBY@|QOq=SiR_@f z-+c$1GA1tpg?CJ#X_TlyMEMAKb`Lu;DaozyvyX*H;! z*w{pHZ~(1^N!aXv9!%T0%%kL*KOcl1gW=#Poyw?(x{j&{ z08a^k(OHSUKXd&y$wAmB)lK9Co-_xnrld&cKL?fp{KN1#5>P;N+Kh~aKT;3RvotFX z%4P4IVAPHe9*vu`w#P&>s$$&$a?K+xWo}uA0-YGty6F$*VV(9~u^=x9T8Ka+lx4#< z+@>c-7xXNp^;Yc_18POy*AYo37EMkjESb-}i9hgDO1nGOzKw ziYkrJs=#5Yy0`(?I#EK1y5{mj9Gy(5~+;|6|&G7-uPg zECl^hLXOLPzLraM2Tpp3IXXF2JRmQ3Z03|>XYcsBys0nk z?Hz(lemcAhVK-3DWlbf=IoW6@tFElrk=_4U?)g|S%-y#oMwVg6(Eb&7xZ9lfl>yaE zDtp0RP|q0=hp;wYy#B;Aql;gHPKSXhK_>GgyQA3}i}DBD!rPhVAwQyC0zpz*Dr$rL z)gGXLd2cb@JEMc$KujyC$(R)6?W-dER;yBok&FC#s$2vMxfh!!fs}WG3-P_)Ef;;N zD6aCXeA%GcRF_5eb2UqA8i=~oCtAs{Dk<)!!&Fk*l7t&kQ5ebK=0}hC4y+0p8fv+y z)ugw|Grv~57B3P*x)N|ZPkTtuJu9T`fAp)Nd(=DY5NUXBB5gOK<*x8G(11%K!sq`e z|AvnjE7ay#xuHNYk3U7R&If3Bj2uNh+#BzjI~xm7Da$%ACDx8M;TA&A3f5yc3$v-5l^Fq2kpw0n`e+(}NypE-{Zm{#zg)+D!x)mzl|tU(cz z!fzx^+TJ8_sIR$QETsWJwcMk>axu@IIIOo=ikL z*e|L6yF8oRF4PPf3-?>{#<(C`662abQ$1dyTEkH2rI5`#AkXJ=LzA-*6q>LgCS`=A zzZ2rj&`;B0XKr_@!M)k7{lYU)pq2t;OV{YW*2PX2K`lHQl+Yq&Q%lpPuoW|Ve9Ihe z0qqZy6Ge-Dzad9RXD*=3Ht>S4`g-r#kI%yiHCUNQ*dnE4VQOif3D)nN+HI@WRIdu| zn<2P%CFM6*-15R2zRtq zRGO;92z&?OwI=EGEH)sxF{ObT%TBO;D`Bd9y}_#??PkQXDT?Iz-rgJVzU=}IMD(2m zaH9X%9x#?WR;@+TBeeSSzgRR?|AV(OD?e}F9z`Jd-X;XZciCtZH}WsDz^N<}3JV#F z!P-?Sj4H*^RX9;+6^catp$_TgDyiDuI&z z=L~To`g!ZBWk#?e9yzajrDx^xK}H*8ScN0z?>N7TZwYcU+Fif2R#PsOjepd?!1{`u z{IV$1lX_;}Ht+g zKctRCeqPk?SZ*QA>*E3{A9Sd_7D>hSu|rfzJw%&X|7Q707Z zn~m({(g=rU20Wzshv5EPy;ltCX;La~!9&=))Dt#|6$)eoHG00{>f@u0PE#@p2;}9g ze}}82vqMAPl)RVGv?!?HpjO5Gj=ouMd~`c_?MR(~uoKYadikU>&Zr5XMy+iAZ4+b9 zlTiPl5fRyOH%6{futpl`1Jmdv8+$#HDX}2z0d!&l(ZO6MVg4#(38FX}{tZqG$U(6b zxee}TOPsUoh+*uW>yc-7ppw-@Q(#QsiKX?$4)`)pZtbv|c z#Z`@)_Jx$}(MgNR4m=#c(B*KmU0QUHQ7yr4rmfHoBHMl5!(~D!GBi z&(-4B*(@uH_cEAt_p?N}U!oJ~2YO@)IK2}iqUqP@PHXHO5s#~ool8fOH1)yNz?*0G{CVD!lHOwq)U8%sYD zN*_E<07VNp4yDN|Vw$-u#ciHi3V#W8em#Y}vGQ)8>;{9X7o7{&3{vqY^5t!SeKH?4 zs+gp3dHEP|d^Gwiqg^Pbh5I7;K{YTRwisdgx;8S!-XP&~(kdvWKy6lEv-Y9d%=lCH z$AzdQ>HW7qv}lgZ!D~!53-8JMao-^~W2wlE{N07#Q*ODZj&%}>eiA~i(%_wELt8i^ggmB*FP$uN`Z@ylq~TBA3YT+K8h74YjFKG~skYKSVV zop9Pm^^BsI#ny$UgrXR+$imCKkZeJVdHOmlGB7uEez&7E*rySm8-UIq_+Mc?R=aJ2oMj zwfXXqOLzqhCb1pqV6>QcF(Bj_tCGH((JKX`j>OFF+-0_W`H z?%5(T$hZ!ls6bESN&9)k-$kVwqwGg3&YUtSTs1^DS-HFKhOoZcvvdKne96>mx zW62-uI?DXiO+s*;ZRq=#N;Rl&Y~+I*sEC<1CxjJj3qFcAo7+1UMvs26iqXJXT$Z>G zwQ(s((_NC2BASG&73jAA0vU;^r5=`SR$|$df;??dom;cok^&wtBN3k&xr@nmR5>_Y9sfZBf1#@38j!cWEH-I&>lGpK7i20y2^& zB=52WpJ2EIgreg&J=k2)6$%hY?(ULHV)DhzSMU`~+3vJCf%jK4^hMQflnCL42ci>R zrLy$B6f0%;PP7NsCo zcS?|OBK4gA0%!Q7@M^mk{j~6tpk*238pn!_zfeFbH{tj=T@i+YrMoCil>&i6G?Olk zI3{b};HW^q*}SF(M8m1xws(fGL%m*&u>1^92x4K!EAM5sq{~$}fB4jK#CO@Lf-6ri zn*p~ju!wS~MG_+s+D{R&`bnUG%!-;piU^SW)2xbVsEc~~4beaF;~HuDSi=a6bE@yf z5Ffq0TeGd1`~)@Eeh(aLmd@?fKCozXM!@U@m6OYY`8_??G{3H+`b%i)(dYQy3nc^? zu*DOe1iDQxvYV93AKK*vV2+{p>B)F#Ph*wbV}JPG%s(TDTrVSe*Q>~&gzVp_s>rt~ zO>qp)vxB+E{uWul@oMxAM^YCPmM;5CTtp7)WiE&c=wdb*URx*vMzu&Q+V-$fBrXlu zj6T_$Pn-W$)l*>X#uPtpXu?0w;IbE6nZ^=AK zuW8?V-V8`=h<7hMDyG1OV}RcNJ+}DmDP8naaG016Hy))1ZUNSf$FXc~a_zKrR+8f! z8Hv-t(H{6D{vro!wV?qSDe!A5HM zne(66XUC6JoqZd6@Sc;;);SENR&klq;NACQy+F>SS;;Fe{HzvJtK<7{vqh6}ieY_y z0JGeBdSPyQcko<1fDdpnF~ealM6eUWUa!2_Fzo>jmA(OJjH#do1rti65Ig)aSBDh3 z@-T;K0^-I|VeojaZRsw^kv##!GTZka=BIfGn91h{U+uDh*~8~LG90sTD(o8tlRwlU z!~rIyP)Q=(k6f9TLoe@tegM?TpCEE>W;W#!hhUo}BJTImCSu*F0QU<(zq|y&rvTvg zkZh_OvO9ZJ&{ie#Jn<7A&R$Z;^0R5x`>Kxo=MJDko#5;}pfjol)yORhg*jNQ83L_2c~yAa83sEY6_x zI_6Ff2P_jYX4c|VGxU#AcN{gtPk0aV+nEeuRuMTsxd+_765GuH?{~mb1%OK76=6!^ zRf&zNyJn^FpvTXT3Y|;)e_MS|%bYV51%>M=74fvFV5{cC#+@ioV-idk2h2|QY>mr9jYjm^S7)_+&#O8V*C z&`*Tkb6gY=XK4|ng|>XO>xw?kwkL01qc!Lve=zO)UE4dc!z7P)p!)9p{&(a-++&N! zT z{yWT;5B8dGrX=Jalgbkrw%Cm+7y0~iA#}k4uU)V%%69%Ob}wKahh02cWjX&AJ7d`5 z2Rx*01ph5|y430}+B@Z0>rj2G_NI*Y6U81_qtD!^PlQ_x{VA-N`83E7tg{wE)I7-m zE~hZSmY1u0`n~g9^{poGaEJ!wBAC$j5klD49WW1bzQzK;NUamKrP%q^7jdOpYM-LL z^OZ5~m(_mHIR$iHcv%+`YehKb4__iefI9h_VG*aK5W8stRA9=rSvVRGESQZW} zalKldUj2Ie%ciE`YVD?n^F=4KuCY>+OT^J=*7}Qxennh}SQ*mQM?O-|IgbyE3l%0L z*yMtKyDVKatV@?i%ZX{|4a!5h;;baZMI7jn#2jlK@80+v?hJ%}oyrOw@`;6yk0`ah4ykzAu)z|m@03Gm zZa9|8n3Ug+M0|U#$##?^&cZ|`xz1v=^|{aOHMioacB+pG)qcrd!|UmKG4SPM(Di1k zd$B?{L*Mh~-e|RPhlJny<^K4GpX6Mutf|?;PJ3ZezAnk!?pMcYNzST~L7)0(rKoQ) zuPl50;DusZ2cHVx-kvcs)I+9f3dHY|$A^7=ru38%vcMX;?QZkdaRj$(>NwD zNz!zmDI_x5Z*^y}|2%e3P`&h^aTpnG^*Aq{j@C|nP|X(o4H`wpZ->kJXn0DR&T4ds z|L0g6`)zhk$c8%rUIuY(Qn#g(|Iy27MF#FqL4}%l7wVLj3ekrvKKga8S=5|binE22Y z_j)qj6`a0}@rZLzKfJOrvCCm9Q%JT6msJKdS?ZK*!R0`q#zluhCAf{QB3Td}|82(f zS@aLXPfs5FD&8K<kqV+r-43hBdXke$eg9L|R$B zxs2Sye2JP!mY7#_8gu{e^RJsZbP3-?-A}j2KRmIlz@~bZsW?}|!h#kPv9I}?A^Uke zBHt67HZMp}whPXBx69X=#21H?!s+Nwz!yFaY<(=JrNBbYfycts%OJwxu+iG5?I?ne z)ETOh!LMM$5xNPQD%*V0tImOSXNPFpr=pO2{?avbP5O1Q?R2)7QMAkDWQG9#r+vP} z37?TYgP@sn^xN2Aj7MJ>Xg>IXafe0?*eX(L!aoMn(<8zOnAW6+g>b!z{c$V#;gdhP zed_c;xA!9vb$4XtCSdq(FP7W#M`La~p8vzF1!c@l=yX5Z1ubpKWo+fRb2Vi$ z;)}G1hbr-vNIhcHsgx&X>uEP1O}Kr!HwPvt3)VL|0oXYIwdjJ8J&8rD(S4y?Wnf_= z@7>?1>ACK6_a}{%^*Bh%tLiaaqIOw-C=Qj~d_(mp<2N({##Y3VJ|x0{#QaA>HeV{e z-`Z`q^F1-Q7UcF&NDxa1nc!<8rwkXnd3VQr$d8c6v?8(4|sl$VGGWY#r8;GUD=2?XKf`Z^X0SFX&9}TA9%} zY<1t=-+9qEbjz9&Y5E!1eg4$t6^s$Up;$@VcLBQ1dzLT-yW<4>_G@7r=%r4$Rx9TEO*;qEZQ=}HrW3g|(zUTNvad^T z+&l+~EYVFCVw9n7W~~_|=_nQHaHQQqCl|$>E*}8Zb2Et|7pgB3&04$lcPg)ywm-Tp zS10ezieaXx$&VJyl2_aBo? zS&w7S6XdU6eVl=VgTGXek<{+5c)!@*F>chr$ zPfe6M)TwT;aMOh~;d!n#TI*$SvES8&``O+4++1;x;+tzRJH@Lvom`98a$V5>xR^;K z2j?n+wlS-yDXvwWY~sEe!SuhzIYZ%bT&TL{H4m2<@~=R|OB5{@vVV#FoWSCOo}6XX zkURZgs^(>n=h1YII32l6bj(X6j%uR~$VA4q+oso-nZ%jIY%ARZg;7tgPHa-iJFB>(2@GPi{Z%`K5V@+CE>G4(q;SYpRsOqpx}H{J;8({ z#sYt0qb$4rhkl5|V@+hF)}xZ{*SF_RqJ0$ z@B8t7;{Egc>wEnv9`>F+duGjAYp%K0S|?0ZSq2-E1oOs?8`yHPlIk~Z+>=naOmp2r5PU{lW~ZULZUS)-rqNYWrIK)Q zx1!=<<7VTa5y7OQq7rhqv=&sCl>X~+@S8A=Ed=5!$jK!BZt zlbw^36|`XW@NtHid9yluJow$o-~C8hd04pHxkBt*oT(7~nwh&ig$UEoAYSyZKflKb zv9td3CTEYorUfR*j=01Ah>e5&Uwwl|g%DQ-C0v|b-K{)4K>s2jH5BQgFAkkaLDuxr2uxW{A;=aQ^l5|GmY3_NC%(X9Z?`{q&>1p8n^xzuF72BPRY2 zLHw5T^;IBd5lkWWe=V5^X2l}s?2Q{@H{>LrXnG@U&HA-z4qtZdn|xz6dM_KigRa() zA}=Y!bmz-E0y(yZ993r~=0<)wbOz?7so;}?*{@|n)<6x5*NV)l6cMz?E+&A~U7fAYU>BPRDhN00jd zCimZR`Tz0C$%_HQ5uo2Y=wa_q5#Z_)l=8aP!z`A8YP;dFQUjTW;A-r@MfU0~Q1LZG zEd3xcdsKq3rrnlb>KgZIej6W7I=Fe7!WjIwIiBAg*7U*uFUv&Sj-p1A?zBQr8vb*a zmbq;TD=b`%0`W2`$>&cA#a$<2<$%2nm@ieqtPT>>eD-oKIn?S6`7$xfjq z5(qPezCr(Zb@>bH9$RmBAwS07 zz9Q-cuu1p5tRXs7A>{DSvros3(iC}dukBY>`N#3J3&Z_*GIE)JxgI>zBxaZYn>T6e z{&%+kZN~yrSimks0yF5wM;g=Wj=@Ten5DfRAE6V$`{3_6_#h#d5)X1C4qD;K~noalAeGd$s>@8T3SqM|D4*+E)3+|GfQ)e-7pEH~d=zOWEh9hW~GJ|F*RM zZ{_}5RsKI#OI*U!jpB0sTF0}`_$rds{@hr9d%)i=zz}HZPirzLLU*GApfwKTwJ!4k zm*-~=Vo9ld&4TCPP2_Xq-X&b zFde11H~Kw+zqaepYnip=AA9Y5+$?LNS4y&cIAfFU?w3ikHr zM|8`Y8$B$Ru;uK1#Cxn!#bXSBL$gUfWYoH%1D1Wy0E3G+3W?%p5bOwx5LLue2#O&!ZN*5&2rM zo3wn?EL6Wm=K8$B60c0FIBsqgdh4dOb(6GdzPXjPkxhBiP9xv^GmY#gC$nBA7hL1T zko6%1u-yUMssArerTH-vio>aTw`^zkkzyT}^V9vxMimjT#KzC9h&Wxo6N7yS>PP_m z>-BCCMUK1+^y0^-7lhR69x#g`Zhf^kV4GvgF#Tnk#mi;${o~1;U$J)SRFl^(5a~p- z8@=b&MCOh8y@if@EV@4^vYLkN+p79Sel5kQ%z5++ha1-WS>zICk#Jge_prB?Am1W~ zL@^zMr8f?|fTjB%CkorckJfivuUfKbD~ptzid!y@r_DM-gn>GcJkFblv+MMl_a86T zVGl-kDt?bs?pvCXDI^m?lAFG{V?3IXX;Z9IUYn?{{=UnRnU=YLS*vKGd7l=7d+QEa zLB{jOtvHhg_gr(2Z)%ErNe*QV>v_rID=7iz`(5!yo{7KaIJDLe*0&e59freS8vCy2 zdJd<{%5}`xYry7QH*d>5j5~@9JeRkQF+m~$SUQx^Lt#zCPYSijD2UAaw!(FgR^XaU;$_%Aw1ghKzg}XGkFGJq{RNWje z*ICmhD3bZ)Y%oV3UQx9WNaToxg5O2v@uqBRC|BVrJ9WkLzL8d@3l!0_r5L9>7yzLJ z0b7z>U>>-Muw^*F6($+T|I`);xZ2dwlaTJ#hw%L9e(K)HE);5lcq3mzsN<5Vt37K2H0p?z#S zemI;-Xdk?JswzPnqsXK+fW*l5P#D7}?dh1_1DUs)^EC?8$qwbs+V7B69(ifFH3R4I zhC*2IlG>fU<@9F+-&}!rc;V`+2XE^SzC9#VAIuQ{atdj@Md2+b;Z2~W)~2Z=k3u-h zm2_0^w&;7jb=E~0ca5qilXalWbgj~E@6Ruu2%glkLaXyf|S7_!%CrB!Ds z^d<9oagp-a=j>HpCOo^kIB{-~dsyc6U10U}@O5n!bs*8X5^RaFz9gQlVCUhQ%UfqV zZ%kW_D{+^WGwX{?jc3EH@hKyiwuxx7;*95~Gb6P{ZBs||BlDIKg6tKR^jK@d@__?c zV?_j*YubsIlh$82v2dUHapIIn* z9_0l5Q{g9#v;&e=_mxB&=7gtvpKkei`tXjI=w<5}S#QM#ut(f~^z#kZ_X;a+6IiqY z+u#l2m~13|<`R?j%d?{0kB4uO?MakY?hYWVdZIqcjE2Kgt$GE! zg~Y=zo88Mj)6Xp(G2*D|Kd5%pH;mR7KWOB8k)wl?W6|}N1l8uw;5d~LGWiF`NzQ)i zprQL>XoCdndyui#!ivc5MRgqq%_s-H42l)+d1{gpgmWqF3vb(S=Vk;7yshu61+1#lP@%B=~v4 zwcnXsiLNq2#;$BW%`Zohk(cQvJKLrBA5Q|T4^gl$DZ3o9kIG1v6I4P@G_8*YWm6`j1MnZW$^DwD5HbWo6Gd%G6PH$Rr}!NVd)=Jc@F|c(W|tGUvhK16yN-m)IzPKIA8%9?B%)2%xMdFSc5&GvxEs$v?r z&*jrMJvv|W@;tzC6RnpOu2AVZ{DRU<4?z-@XvPd zKDU{xZ91AhSnPlHGmOExVVS7zBhd%VrKvuq=<3UEA^2MMP9ZE`Y|(f&MKkyHl0=`8 z^^;bHzIoy+PAq3ip0x*IU|ql7ukQ>h)k|mpT3VSAs0z2q^*wiBVbrKAkJh!IZqW7_ z)8#0|G_?3M#@_2ardzQbE4xjCRcq}(MFbwi?9#W+v>VRzti&?NmwU-xh2xObTuS>r zWb#11zK4ZiWqV&t{q| zVW9PGrA1d~(hQEbtb1bOd7FG$Fj;IJ*d`8+hlcl42pOa2S*10gco6~W&&5QC&R;YL zLe?@KX;j{PV|yb=y2ZOw+6XczRT(gAm(wf9kZLK@T!@r}TR2&mwpsM!ZOd-sm!lqS zBo>uN7i2?Xw6o=;f{=o^?1Qb*6p}BPuvZu{<@ii9g$v%{n09UPzdHD9O4S!(qk5%7 z?2n{ugi41d82!U4AMtpVza7dZ6-r zuINO%WbEdOgZ76@>S-f|IF>M;J*6jtMvsF_30-QIXC+a2B#V$tuHfUXGoJReC3Drz zZy8Dla>9HczRGxh3cW?K#+1>A?m0H!{nGu&MIJ?AR)Qc~#~Yy)5crJECLigYvg{!8omThcT?Q*6tVI!LXR z$?JQ?hg>3cXZM%78hV4?4zZ5v&fiu_RU6nl`rcZ&R~d)2K2jJnn>#8Bt5cX~y_gTx zl1^WTYj3q`Py~PPg!h;>5v2|A$wh3A3y18PTbI8&S{6-<4rYq^dbgy#I>zEq^|@V< zkN4m@C-OSpfR4?q!>EEA*~JZ46^~D7_txSSyImpv$D5tmmzxy1QJ;hF85dyAD#mC# zIln3eX!}w&MSkNAkHZ*5fw^s|T&a$jr(ThL_LKJKZ>$+>(Kp{Vc-8VW9ln#HP$Nd` z5fYYnD%LzJJ9`yspU9z7%24zZFJQ`8Z$3*^ zRXJ0}$o_g!;j8VA2XmE$A`=^1Q)>Zd?n&Waq@U~RlU7ohaiaAtUQc#Z(yxZK#w8N@ z3U?Bj3HW>(lBCegX<4Yn)r0y@aa?0xCcD$xh|lYOB_icdue7Lj{P(MNagqm6lDL~Y zQccXg-hsmys%KBA@{kN5ZYrJ!QWUrgxjhVw;b!lDD^lfG^X9!$6GmT?8=nLPQvrXf zfqG;ngiqM6&trdykz`r==S=e@g;j~x%09ZYg^-=fp0eLIijy$icAhyIuM-|zWczDn za+1xGh_3dT{!0H2wxYLqXp(X{UV$kYiEHB~?tT{%M z+-4~pd`OazXGKK7IkrtUyE<^21v~x{90961Upyx2BrD4-X}q#bE63B#yvgpMTdB9L ziFU+6)OTwXyY5cYWRCUp9)s{1($~B!qV=1t^q~rOm(yNmyD&$$MTxpoFNKF;I z@p>1Y{wpoXMO~gfnT|jR2DhQ?F3~LWvkk=t64{&VIQ9a zLf5_WszVR@3McCG{$sqF5(@vO}FO(bE zhYmy(%Gvl*r2yOCh!HoFc#ba%(;6~5AJyLM{KcK4tU%M+n@P@3qV)_9en(wvkj1IP}3e(DTM**rI zJNn#)^*z3!GQu8ElRn-&4u)`}9las=qR0JXobtvfC7lxEaOgYHjZ&%G1R4XDD*X(v zk){3NM;Kd$+FS!3jz^(`J5G}ya+RiE{^Wt%Rwc`OJ0B+)k?fX8@xo9hL^neD718~ypC-s<~OtUSybk*Xe7$c&|~wboJI_J=|_-Bs~QsO|>9NjLcWuahH7J@}EX zPiXVFdZb59hJ5)>!_Ymii-3U`U11(WaD>IYVHak)$t<;13rYVg+~1RNaBgRsT0AvPES*onfX4P zpdR0mhfQo)Yldy2+B315;iV0hRP+c}#tG zTzA0FV6ZV~et{|7wmdFpWBZ+IzWRq=ac-UuRuREjjYrmIw!%KWRU?JJF4fxf`>h9Y zbdt}`R?{5_N+WuvpW&e|y#PUjXZ7EXaG{QP11{8j_w(}{c*8~S=aar^q$|^b`YUkX zUnPF8X?o!dm93E<7RSfzEPN2&h4xkg@~fcQJR^+^92yKM9;bF`w6zxM79kg2p0_0` zU7AA$?3@z5>n5B?o4&BH0z^}o??7gg&OX1Ar3RxsU&`WbPZ>jrMHB8h$*NvQJ{~r? zMZI(AQ#4N=EX6O}Xo~&)2<-9s@g{F92ZlqQ3~?acCr=GN?-Eov2qW~ME=;s zv;G@=SD&Q~hPcqV1IKjtSiJRrbcK)Uy518a%m^{+(%<8zJDlX%Qgb8r%XQ4+;ENfW zs>lMk^zQ2lNoVmMmbFS+C|@E?u^D*ALQdb}Nz6Bt`MY^liFy{)V}6`5@y1u~JmIaz z&AvNY$N4;Nb7&*+P#oc_g(a#e1w@=;-G1mEs{50+)R*S)#k2KjTL$4z^~kMb4EllY z%UlCRv@J6Pr!R&f+6!tb<%HDXHPP|;meAy2V!f!b5W!lFcn6=IgkkS~S1gS#bH$nWr#pzSPG z83JL>`MsQ2zQ~cug#&3uXq?LqEVRNUj|EG^+extC7RB)*c{M{pD|dbKh?01S&N~{P z#$w^Wd2OQw%^}SrwI(K=+&-hpEKE(3H_THcJ~4sH70c;ce@ZV;VRTrF>ApzebYS^R zGL~AzL-gV2P^jpa9%EbV@H#uXYO?%Q;l&-g2r*|Y5E9+qRM^Go7C{Q+V0*S7uPZ$B z!Rqx`$)Ju{>Fi?MBa8Z(6ve#k$s$R#qtRiHS1sy(?$L8By^)F4ye#V!bZ!~qok=`- zR?+7UZGeLqvlwzs257g|ev~=qV9mkS4HA5Vz@e&nV2|iT=IvLVb2W^J?4$XGx?9q% zRkek*G;BDl-5P=HKLNfFEGDiudZk?a!7BpIhyLO$b|>uE-k>wcE!f1G`X|X`O{X|L zd-E>_I@;1;s@Z1Wq=+Ut66i-n|{)QEI9ps0dr_SyLAiAQ^iA0`px~!(_rcJ!sjhs6oKdajSI&E5_cOE@~ym+ z?70mH(eDT5U4_#=qT?M~$Haf5;Lrh}j{ljyj#T6m&H8QNIO;qEW{2%QB+o7QDwZN^ z0dB`8IK=#|@0(q~IND5?+wjzXss|^k4c1IitjpjB+tl)wQZ}>(zsaeQ`KaPEzLp&vOpk-i$BRi*E)@Owz{{qQji3tg1BLszg8JOH)HU z;6^kQ`&}CLJ4v%^JH=5AV%H41uh1R@JFpZyx@_%mtL2!*eqBY(#AF?($Ef!qri%)dw`mgnmH^=*}jh~Kc0U|womKG{TfeGxZ zH}>m;Xk5j?P`?7QuS(_mUi9Lu#uD~U!^EK(TYeY5~Vg4`0uG|yKKKW-eD!n;1J+- zryao$4>0Xf4m)A2FVgxjRC_*-Rd&TY$Y)dlxlxvJ`b$zJqY9}+X%5%8`kCH3@-MUM ziL>Z@^9Wx;(-Dy2aI;-v%#&%C)`rl0KZGtXV-kpLx6<^)u|9lOlNXYcf3~&-mQ<35bo`)YQ$}_kWzNr?HZRXQu*!s`i0AF@_Jf- z8ia#w7F&U!Xm7cX6Kgd+Frb9^vvp#yTDJ6Ae=H$l$?NuyV|pwS4N?>`gL$6obcfh%4KnR3un_&#t{ZS{i#fsEnfO+ zkDLCjdy>F6HH)usJ57yfbbwgLqT4aStH3G_SWb=Ez87^V9S^wy{xjdBwS}|bD!XC& z3vW-AR4I})STgK^)A~>@37J_Qryc$4>GC!R(No@v3(J*|=j2CbO^a8WADZ63TCu-5 z=oq^h2N9e-oQ%0Vm-p(3CVJQlm1D1pA|xrPj2iHh;G5!U_XmueV3Ax5y^6@0rG9G! z!w4_t1J##c&kV;&*Ld(;?D=zsj10V_;0^ZtyD$*?8C-E_c@bDiu?@TZT0>c1^fG&r zx%hq2%Me?ixEF&esu2_Nau=7BT$jm9mCEnOLcR#vtsL?=9gZ2X_ZPNaUHb7cyyjNs zUwHZ8Eq}YyNBTMDk7d;2*d*NiJN1h*M$mPZ4M)>oUwg5k2=d{28d{k1q?Z%6nB%&t zk1x_q+;~NLe%Rv1gegbr8xy|`*9Dw^zu)6f>*oBU9u?2otW&nulw z@3t)dnb@;#=YtpOZC_!eEjC=W*5ZjxAv96$ba$VYdu<&=1dc6}l&;VY?($*?sV0KG zW#iZr=5Gund>LwH*_U5JXt;AFs5{f=A<&o>`>SW zVl%W89=k{OQzT%{qZ^T1b|#q1)EFrOFUZ&Zk*A;a)bgkxqK$sE%xK!s5D7u!u#<(=D(_gXi$?RH=%R?I^?h22vW#0!X*OkI13D z7zuvn3#%GyGWf$YoZ&K>#oB#+kL<2uYU9{VcG&aC`acUVd;erg%RZ7A=c*bN9s1<) zYPQ$1SiIe`Ryd6_P065_zPcrCg>h1ej@NPGhjZgP${l_&e7ae(T-a^sk$YKTexsa&5qJikjhCCuw-UJ% z*}VsmXli|xTRgi>ZEFp}w66snrVUNouTHwIyz<)Q8KWK{T|L(Zob8}tA5~JR{8_NV z$um!Y=zuJSW-Hz+&>NWvfe`h(n+_~5KjK4dDiz|{T$*8V-)7u_9z$uvnZ6G^-0Vgz5~3z5sqlh%3H4s7VM@bJE- z^f2W)UCpz8#2)0 z{~oLU(t5nR(1Ji+8_&35c9&tJfc&PK9;<#e&WBc?^k)ZQY4;1{F!hsL0f`QjBY#!5 zO0ULVtN*j*nO|0Q-V)g4>KrG3meVR>KkNl`d3O}q7slWWWZ&wa(R2#tu_IeXq;FpH zglKkIzYB3rZb}_Z;=G(3a7GFgd@Z=?^33$@<*4_8O@G?M^gJdODL(+mQNU?bbaS*qTthw0<~<%yby@pw;X!OHKugl zVcMfU)6cJP;bFz>w@9*J02fXo`_g;{p=}2~r!2pdolCLUSqFSO``DGQaD$qF3pjvz zQyPAF@aT)Zj=O%Gg!LG=?Gnx5Q%mWVMXK@Ip2I9CDq5b0pKeh)PE=&ha$^LuKRqK* z7KDvx5Grqsm#5P+rGQf*=1^fDTrXF4#v)Rf$l-HLOP$@BfJ$l-)Ekj;!2A%ia6Tei zCUR^`ij@Gd`o8i>eed~7qIM(Nq(dx|KDjj>VaomLno2<7K_48;=w1OaTNggzFvr0G zE;4Sac3xwY9g zuH$V;?L~bad{P3Hm)28kex3eRok6i~_;#uNJ8a~p$|S%rfHx+IIhyVmeaKPz%auvb zn))?2O-rYECtLHcxq*|;hh_ER^BYe!whQyq8u!|zebz4ObQkbg8G&6QI{B$vr&~YdMIYzHU_fhQND_%OsM=Ij) zQn|kK1Dlz1Jp?Oq(S~x@cGd)~s2ZoH1(UIa%k_s?1n#hJWo-_!7TA1wlHosea8#)o zzCgf<`2?iR-6}KBlg-K_vEvLfCYjKxb2c(aPL_2lK2lJeHyvUAk|s2t*(dijc^}|O z%{H~{(5)&Z9wQwj_6P~X3RCDbZ_HTq6(EG}$2l+URCZC&YiOWA?2fk2K*|e@GAlj4 zX#2f}vYKTnAkET{lk^J&a*UchpQ04zy*VfRbad-c&D-55YKithE#|n@))vgR^<~}+ z=?Ie5G{3TBi{V0h-J&_tS~EBLEp|a8@-J*WgA&=20K+-l8JGNWdGSjniTkVLM!<*m zRB9fZchalhQ@7D2VS+gtm7Z2=HYiOQ={q3dLUTax>iDALXD>C}AU|*g({RH-OEmv7 zaTdxGlUV(nEz~B_Gf=BBcd20-x4t(hFz%593#xfz;<~QI6R)rt4ciE`(*A-#J-GId zl_t9jyP;G8f4jl?XR z0v{MG%?7D8?c0ucz>0t6*3FLV)9Hswda1-_m-mru5jlKJZZWs>4Hr4+J)kc&sAB_Z zE2W{R|G;}0YiI<|1}}yJFE&lW(-`wIIKo?cnBj|hv5FF-)n%UtWhjkIUx(fyFFH-d zUPwG3X~~I~qYOA*fcuv22Ejv0cah*c%Y_;a9S+aN0L`CoMh`f{{VkZI{f~(hqjDyV zywgzLiFPK(QIHnEP1HLEf?;jM&6W-pkGy7|#zzdIQe+)(=I*}r?fuBa0Z8C}7ukL& zSPN2+4}qM7nXFxAI9a75YJ^&#<}&2jYTy@cr_po8#9O4WHNV_H$RTWIKa;g|;YTPp z@?QGF^@EyhVB&jjMYC_3&3xyqx4#aG;xy@8xn4GGYWa^Vd?GQz(2ShV?N<@WhAU%Dkn)!?R@iP8wDGb>GD7r{2obtPu~bRa`XG_);RcsA3HRu z?ntvy1Km4O2I!4mc&nUP&|>0K9fdc5w9fNSoU>~yxrCFucT?9^@rFBj^par7kUxFj z7ZwYk*pXgIs~vSh<(MvdgXn7OW;Pj!OsX`tu{nq|wGi4Sd~DJI0ZPVqNAZY}G4e7{+GnXMO91j2Cy@)Ah4EhQ}+~KMEV>A`b)14>EO? zsbyz#4@K{dj^OljV(H~%vePCNVV~Wm?Rp3)?gjJKi%jWSHz*ZZWpiSe8uKyaekZFU4Nb}ABM&9&6$(P6@VKR<^xqUF1+*4U2@MR{Wn4n=xu z6+F=;e5kG2(jvLx>Mx+J&%Qm>C+MF!?kX^wIIai5S9ym#eKU|IxKy1(+(;v)R{>ii z#i-UW*u>39Nw3^~m>tWoB@zmP|CSKtyjN(NSy?PKD zsiqM<@?4L^s+zoFx3qtL)s*`( zHF6L8?nu9K^9bG-mAfLRrZu&|r|i#&fDAD8)44qe$1{ybOfXFpAy|i2D>KZ*2rHs{ zK;hPg`FdBCw8O@@@_CM4W+tC;*IRPfuHg4d+RFRj><*0>cV+HV8f%3ZSv4Q2s9$l^ z;KZhhRVjaHARvCOkP*Xle>*sh}jv3`A7r zs-D|fE%CV4QG`<#Klu z-$s0w4{Z_9g5jp}LxH9s7;gDGM9<<=Nb^wyBhWnc%e3CT-0>6c*)`aC)q%-H8`-R` z;PHe7ir2d^u{&4+&si|?9_3P>y$2M{cVq%kAgU{-ELF)b$JY@4TFik_GpDZOOVh7zpE zRB177nU)k0un(Mw&Yoy*iA9g1q z)XLV7HZm>J51rze2D$vNrI*lZKgASE;cby%H9Fh}p>=_=i!WukHlT5W1h+CTV!m2y zCJv3co)KtT#ZmIq0Ih-6EngrlklUmDC+;Ft?e*Xz2_wq`X2RS;;@?y-*=S*=A7dx(j&WJ zS$g9N}3fOVhv9=a4gBB3|}Ig!jZYT<6pUVr%rTNO^<=xsn9-ah9Vd{qkpY7}?R zJu&!*hXtibF!)j_*EKP?+n%xaGYM9rj|`n}oMM0}-4yS))rK1n77xy`#ZG?LEg&+N zf8q@^ZppFqQ54&k78UB0G&=`$YCj&812l)u10}l%+u<0GBYB^T;E&JN9dy6>k?3Z| zy&Zk%s)LvKZgk*B`AC^3b4s@-Zx~h=H9FOgdlP->Qg{9>S^Zx{1UV8wYXmh5@&AbS z&nGCNj0%a-@?Z56|4e_5x&v;mtL7@B9vF%B%O2o3&bQ3QbRb0Yuig=#d>8>WcNgF^ zX{v2xkehju(viN%)b?NP{z2glBX5qC9;Uc&Pt~N6{Z-HLrzopi0=)QlN`>7$Z-8=z zRued_!Wlx95PFV;nnmBniKW&=Q+?}tCjVf$RV|T0Vl^2^TgGDl_`~i~megxFZ1V&h zntTr?C9ey#s05KOr3lYKD36u(uaYe&3nAmX-0i$cbSfMkD9Cyf{k2tfxYG9vWZEK6 zANL1d9)4v=F98{u0QEFM*70;ht}PA;HCz>>0{`Z)u2axOk>9-ONq10E!Idp<15Tr~ zp@1Gw2V`4wT7AcXxq>`a&v}2ZR78J>oMDgzGc-+LFY|kVOh|Ehvf&Xg zuICDJRUd&89l8}MJCMUl$MNiRFFo+G;iznU&Tn^{9USXwL87J0RP|TXm26p zK5!B?^gtUhonW8>4rwc=P0*d+3KoS6-b`$^LigSXLAC zn`Qp~_X=Mx^PeA7a!@=uiY~K%PC_*Y`LXBLeee&YGrz7TOd&%;{h*%V3)|S9uA6t_ zySZk$yqrBa$p%bsx}0kJPsiOFNK!wY>lZ?VwuNS;Z$LHC4ZRnF3kyO7uT?4QN0^Tq z$v-^6D|8NeHkx}u)>Kis$miH!hp=R-Jj8%w6mZ2|M@B}^(S5Eb%5*bV5cNO~d^&^F zJQx`VCp8|YDQEGA$xX3#`%*h=JLXvfa6LtknE1lUY#=MfG<*rsDDzvmr!qbfj% z$}!JC>|On*7q{|%v5bJ#YoHzg+3A3=aAR7RI>zKs;_)jdXO+1Dg}?|RXQ!zA_mgZa&!j?6_XCM z6W@2^K=z>rpM;uzK<@o51hD9KMRM(K_y6a*h9TBf^Uc9OSLa{ukCq9@UylEd#Glpt z^NC6gX#3xVbbl`Y{sg53q%j@FzpwhAewjhPYLCACs|4+zZ;#3W^P*kgzy0?F{^%Jf zNRUMI%Pj5lKb1ck5f7v#3;o~zzCbKY@@~ezt9Mj7!lR}L`yB_NVQFvsmKrxV8aB9t zV|}T1Y0{ZXUlPyjd+cfH{72VHMWqbp>?W@EZ(}>q5ksoaPxG~Je~Hl(&wf{}0L0My z(}Y3Ch94}yMwxq#gTg+(fOGGv$!5QkY3HWC9eI56sIsVMz_~)`M#kT=g$GI6c1_CpZ9X~O*&VT`d^!_QpQ?0l90*n~2 z5b8Kn?*;~**9+K?4y=6p-4JZjEx9`9*`y+S4Fi>T-8r>bBM2Yx8N`6^DTF*j2oun` zK*2o5QZNv5r^1DTEN6v|8F2bd?0#}pL?Ze>X7~ zdE!qn+x)3eXh;bg6y|LE~LqF$k$2;eUD|kUU!FF zu-@V);HBQEh+Z54O7bmXm>JZj}Y&bQEJN!1Llhvz*|lL3T!jG6w4yaa*3D(=>{OC@o#>TQ>ZhIEu&q>Ocfe9Q zPgNJ@6(;G|xX1t%XTIuOX_`@~=T1Tn>K4`}_)_EH1x?6s$r zjRoFJolZ{pdI8q)Re$uBK3f~eq}I@wjdpH)^5P~q^i>OhwKDkVT?s1zNvsY1@&XK6njaKK9IBPbG95mYOPgVwFM#)zIRF{ z?&oL!{apZ4&*pqqnQ9YkYka?NAxgSd4~)t|(HyAZvTikEe`mk4i*$=D$~N_SqjE!4I(-RDl=TB65QaXS%V_jjqJ7IcD-2Yu_=7Z95*r)iwEj<$4!;0Lji5?6v`uNhkL0Pj z>nCeba|u78xr759*9T5O_Z`1Rq~H)>!`v1-33az)R746_(;~?PIJLnO8OFT{k0kc% z5pm--K_-ODUP=H6-+e1+h#VoCEMS9XV=9vKN!QhiuBb$8hNqyaY2tQinQE5g;FY=h zt8L1w(=MREh&nUCr4FF;*SL-rfh-DMfaT^uQ5K*$@39-t%r2{xgBZm?#uMisidai_ zWt+fb0Tq#{`}nQ;ccmi<Pu(U!#>0u(T|}N*tSA>QRWjgocL0<~vy(9m&G7Gm(~l zb}xaRNBHFOa|V&}4#{8UhLCd45K~3H`4`NXY}<^LQRc2OiEDH80msFabGp122!$9#n9>*z zfEp~s2MF7u!5!dInLX47@wQo(=xXT#*!J{zn!Z}8flKq#u@XH@zeL#-Pjbk>6GWI6 zT;wrryW!QF6!LMs661KH?PbJgrWRhXACZP>n3b-04ocl)@|dVFYuY}(Y9-qR#bLs8 zpG>AfC1NY^pfTnA8P=_Lh&d%+_f3L+V?phq$Le?cz*uH2M{N5Br5J=z51#zEy2@FR za9MwV*s#Rjp-Xy>MaJ#g4dr|VTa++M5^OP@-NO3fXnnXV)nrj%zks)uCnifJxfn&8@bW8eLY*%$T zb@W^(xsYQ9ki|A2d;+$!HH{(bDK&RqKuiNSHjS*P(RZEQ1w<{2&aoM)C;HC&!4Nvo z{tPN4rL1bHqtoCIRy|XFZ7;ROJSb0bbUXXD1~NN7vB1pA zlZzvh+8wb6y=T(sHCA%l9xI&&CAk#36PCOVV~WL#!2|JdqGTjOqBqS z2<^3@Y^zPzI(dlhXywlmCJgj(B)7UPN+;7J9M+$NM&Mu>&7ma;uxzcTJB^!cUL{cL zi?z|lrw$QM1_-DhyDL=%ur(j4N2FnyJ+-;;?j*5&mSUWAnmw<{H|L!l2YUotTh%_< znSH2P@S|`{bhArztgRPr2R0uH1)HvWwqud2BWX#$&zF1XAn(aL%7SwJP8G@27{XrH#cF6R=LZVO z#|)qp7*^MM(U-p4Z#R$ueLSbslh$Yj`6YkXfwO#}BebdM+-}moJ54m54B_WgQ127R zGu%sZMtBq{rXO<`)?<-=bP2FU`zUFk@ZpAH*=g~t2G&WLYA(Q8Kqy6`&Q|P);`o zQIyQi6FW%S<*!Mb^!0_iwKoxnOKc92el@M2(uMFfy%4~}- zGh1JL#*li17+&1-Rkf+`;NY&JYMssS&LX8V5}SAkhqA+S<~W;&qFWUmj*V?EK_cMK zoW>qP2h&uFUL#Yd+UGzr!~jCvI8-jZ}0hi_&0O(@vr7yKBGnFjobVXa4>&SvR}0(3C=^D%rq*F>b2`8Re9 z!$d@0cc>QvH)f+NhrsUhwx+o4S#aYFCkO*T_&SEu?kDCmpXjnuE;xteE5tG{6Y192 z$2JunArK=s^&^q}&-l=ETVK^^5k*io%x^bD+h2LT%bYy`U$}AJFm^si+Tx4i5gc^A6ZC`ml@5bNDe0@B(71{K7(zHI_YCze|QXl*lbEv zHD;5auvSA8B5eYUVjHDN=~3U%)1HTT;{YrG3Zx+sBtg_u2&*-DIteq0^F@2Ib(W1jr(h{_!!z>rLw+!emF6^QsU78U&7$VJ1~ibU|bb@f9kZ zOs)XAO%L0VKL>do9nSPgx&TC-?l0mCe`76ov-BK}D4*=r#o%+#-yg?}_`FmsQ+lHNe4U4{>0-d)+~Ohk=4$xEviqEtucV!1Ex>%?yH0hpaRcGv`(7#0h*4jn z8-hGUl0&OszrH;zKi@RD`jCu4ca)(;nl8U4W-#Gz+wsxokbtbZsU08efi-9|@{V~* zis>T4-x*Riwsyo_c0kyoPk0eT^y-F{ANQrvaG1%QqwUZk_{nZ30NQ`2=>u8;1Bq+1 zix;@F%49iOp?8HZ?KyxhQ}0Y5B@;hfcq`9Xq$diFi(~Jz*jrA;RUGR_f9; zn3p+E^6x|2DcZL<)?)7z1LH?L!evu^VGN}_gGV?oyyZjQ7wVL!%{JK%W+4(7`l3FQ zx_r!BO`Jk@rsEx^54b$D5k4Dn4xuHet-|-#>P>Af&TnZ{K@nYi!zI4)sA-0XNH^hn zD}Rg7tx7C;#5eXpUtxlEU1|qEP(zInB3nUoZoTmIx|2t-H~=<90L!}O@vvVt!8eK* zDZ7b^t^Iwq3)R2uXz8l=}(05GfeTDe@6GKzj?Rl zg{Q_{E8q0FQOY!MU^2vKw)9 z36~dB@Ug$0^mtA8MLWolLyeGsn=co`pdzwSkbSfi>Wvv1&!qQZsSS$al!$PE0OC_a ztH>XF+TN&1%Qd|_nB=wmEw&1b6T99SSgm)03xKC26V7*A0gfP8sY(tvYU~;6OB0^n zIbb;YG!96@*oQh(Mhp41U#|5}-~k{vY}n|?5o9F|D`J5007CpdK-C}hi*JX$D?xGG zBrgldkOg08fX|?On|L?x%XrzxgKvODos=C0h1oU^vTTE3?Eu9)Z9(a?^bR0(fj!&; zz)c|PmE-Vhf z8at)1y|FykvB-ep65Y!76y>Ub>Vqk8+$MKFUb4P01YavbCr_}a)>D#>9QldX#8q5BoX+S124Xu0YA{>m6i_z)HU~QOWroCpf&G znfxdmMU&dER_HGrc;51wtx@)2OJQVqIaAIlAh(4;Cy3PsC3gd}HiijCfZ+Sk(`hH~ z5<{;f=(RJGr$nknCx*>DqVSI2AAev^m^=$EW#hy{PzeVA&5HrOiGdDrgkOplLu$ow zFDyO+-%|p7ye(QUK&!r$q6}bf0#*1m;82TJtJLbs*0ja>cr2ssfEu(&S72-S{Qgl@ z0L8PtPrA^m_FlLTsIeDsf41KXe-uq`PEM z0!nv-ihu}635bXw9SR6acS(v8(xTD|(jcL9h#;UyNWAk}-ur&;{XBbrKffREc*j`7 z0l4B^XUsT`W6m=#{_1wr-y7vIbstyAI?c*!mvCn&z*%3eGl`*?jF8D@r z_<6SLK7qrR=BA5F$%D?5wYu7t1a~$bDBA2}3D1<7@QLpEQuhIvVqcI|ZrSjw+48wb z*fQl>=J7${A=d3EP+|If9V+rVZBVTvq}((qEZzj%^f|Ahq(z9>H49A5jbiY;3m3+> z*pZXZKnV?@Fa$l!0|x6^U$`<7Q=)*xYvBbg5JQwLx$qjymq$>4Hqdk#-W4vRwa<_! zwz2Tp^ZDg?Y7!bUl=75pQbE^{>Rv`|cCG5u0#*m`y5STzyb+7si^Os$F+qyLYY2V= z3ABpDXQEQoN*DXyv?8h3Ukw67{^X;1NIv?n0IB{@KI-`;y$V6)2q^lT1|gwdQCFsm zC?L0eL2WdcO{5}r>$61()A9|{8cw`6(X7&qkKEjr^;H>YUs#6=5PYT-Idb`oQ}Pfz zOSQ7B_Y}(g;N%||0l?Z#0MpFOvY`->DE<@nw_h!YM6vQJ{wbO)a%b_uOq~tgC|!rV zx{Dv{t`TagtB4HC-Xqpg>sJ;RlRbbw5@PYTn#m?gLx)SO2%qy0b^=b_6Bzuag*aq5 z=rIJDV&!)IRz8a4oi|6$mt_}_J+^{YKe|}!pnw|McQfPA{D_7!XF1u2#?N@?S8%U+ zHOYnrJ2?`hTA9&n1r338y6l95SWskx72VAOylKTp)eQw2$yW!(vwz*pSb^+%SfPi} z@Pn?Ueh3uh8uYl4oH9_NsvPY59&+-`R~|~=qVJIlv`islYbyylJpg?g`RVhnuPA8W zULA2BDNW{!eHJYji^QO0_rRI;zrI)nI5=6S0yr~O>yhg7RhE=|jhXc`NUt78b-J>k z+{Y=~ls7q2Hhd?W>_)V~q(UlmLqcA#%v9ddIV~m?ydISogS@|=jkCBuB!f=PKm&YY zkhlU_4F2PczNN+AgJ6!n@ToH0tCGH+IK6p>4a1CO38Es^d1ecYubksKcoFHi5?{5k zy?e87SBALx#}Zcp(rSj=ztA{Pb;T3(sAM%FGvyj6++{Mee10EVq>%ad~hd3zjat^tFyyEY1gx3Q49&XK? z683*y2m4h9Nlli_1}Og;_5X|?2@QbTiu(UP^#8xpi;g3VW{}!yIc|i04YBK979eUn zmYN6zi1yOX$}=X|4%`%h6e{jUT9N<0qGb01L6KK(TG?LZ0b%uu%Lb^|*H*-DeNI~+ z)Rl)_qr(n}E-!`G|9&NAOU1^(Uf9|!9<)dHLLloI$T{MyQFQ;^jTd;IA*Zswe>nV2eywWIYhvxVUC0l5 zXMq*@2A-=C0yfQ5aYKL}?RrvxxUs6V>v5trUj-rqEZPc)`IiBWFveu$5V!)=-uLLN z%u`6BS#C?J~hK5audZVgf1T>Hb{@T2n#Fd##DfMs~CA><9BT-|BE#63R>BZ7#v-tg1g zkFq~LfIQt}#MJ&Fqnug4Xev{Nj*j+OgYZ6mjXfx7`UJh6piR>!_=Z5RcjzFbPI zdQZWmz*~^q??Orrz>~v<9)+eH#>ZBOy$n1chcM*G;->GvZD(~Nhpb7^iXK9Pt%f+s zSLIOebt#UM_<{c7!QPJ;sYM#%tdpmpAaM33oQwgV`j)IG=4+5un>@FYQ<=PKnE%%C zv0||F9F$hAG1Bwi1KomSX{J1d{rh9k69f8Ns$cksSasZzB?5u$A%+K0L5HHW7z8KO zd}iF|&O{02`D0>VDFhfut4r|2h27Ul5~$X*ufCVSWQlWF4`;-y@yrbl*mq>FTA-sH z^n_alE@g}jKIApPA9o?t6IUhd`Pm5B9#znwq*za1RRa`M8;HF&RNzl73Jx72rV*Te zB(v@+;%vsm4M4lRXs8QRAc3@96jiPqK>_)J`;Y2RP%}Ynzxw6%{mt37DNFG~!00W! zK3%8e)Tof+%vLW-xmDukTs#jMgJ3fB?FuR=1+~S9zN9OnWH*2a?9ki$xwI8fb@`VQ z>*JI|z7DC~!zDc{J;bMhf#{eZ0NC>=TR~1Y8DU^J0COD)OnbtsTokv2=@1Af&#dL* ztf+~`prE$&3OffBa}{i)QL7FHLpjf@6pvnD8lq~G)hv>QuZwc%2TGMu7B0+F#SVOR z0=j4kFm`+TXFZU%>U!7!kPdcEPk|Q_nv0*em;2{HIc^*>ZouujK3PW!UjTq*O`*0r z(VQ~C2VPCON8QcREGcqfcY!lJYG4|&RArFHlY0>Y^zG)~>vvbnUM1wt_|vu7k;XjH znOUD|m}nweE1Cl$_bb4&DuOJ;-m zU}KmgQ7Z7o&+7XJLlVT@{LX+%Q2;umCud zlkpDBN#gVpE(dUSd6n}e6Yo?6z12esJKKBe+<5T{!S;ykxWLdo$k)3M?OFfE;^uK7r9sNrYBV5T8+_}M5}dr98zDkxG0pFqE3gB~R+R#|g`QF- z?SfB32~1Nnw+8Ye_VVYS_9rm)^tpK?i0k`9F(c)XzH+7ic=gS~Y~qY$Wjea7Y|50B zylng$I|#}KJ)E@E6f6A!M&DgXw64ECv}soDSogZ2YV^Z?KNZU>_8W+C4>TZjII#@2 zq~MLq5s{cp&ktqPLr?v&%OawOAy0BQUDlW00=IwA_VY&mlFFDLU#qy!hK;6XwUl_a znl;2|V(T9k9e#R)<@{CvEhzlcfY)&BPT(mstA2TRD)=h{lqNw7!>JD@Hy(QHCw=^8 zSh@xz>dyUy)Fz@X%?#O37pN37zt_i{E-}NnSdJvZx@TPN4`qSO@MTV4 zpXW6~3QEA6Or}@PC2#+Lu-*?cV4^^tOV^JkvX$^(y-L}>W`C+~&#pU$UU*P|`!Jv% zH5^IkXcX65Za-7^y68hf=EiO31DN;*HNK&r9K=z5K14lz{oFWwrp|QneWgqdM~@Wm zM)usm3h{5^epGbq7!aD_wRCaeXUWBw36BQM;YDX{p!P49Oeg%_8PPAi_^eK9pUO2X zwXPxI0`Lx}pT11akm2th4U6N997omj=5Bf;JjMx!dlC(oGrcn$#mt67E z(}>FnPOxJI`uOWFlX;Cw`=Sjhi)QZn=9CklXjk7LPnD#Ii$Q?5hykn7gV4Iy~JS(p3#H(64j9?)|*Lu|D;v zN``vkg5VqK!LvM5Z7)(Iae!`6+*TD+I5rQMNb4~Znikw~Jww2YqmJL0NzbT!8Owr8^mH)ChX0OW4m7$gvc>^nbB&tHJ5h|(0H8J=pB|_=D zXRjb60GUdlMlZ&mCWBdtymfv&7-LVffQz0n9A~+4uD}ovmH5=-rtABLJRIwTj!Q)`hW5z75_>`jVy43C}=6=-Yu$hLBH- zWO7lHmdX6tHWr}oznU)5%N5iiJvmPqeRAiD001?JiYTqI)RX9s6NcGz%e7lSpMlDs ze4D`|Fa6Y19uK9535=Lo0*6bC8x~;eE(89)oo4hZpzMt~H~6<57+U%8z9L|fon5Nuv3Q`*oXy?y0Oi3YT^ zC4ssFIo2+!T7Yl2V}F{2bYTP`6_8I>Q?X-WX?LR*5nN^t@N{jZnQ8Yr`Vb;0OAT+5 z+}_2ntmR@|TtkZ3&HAA3seBfsVJoWSC7x{Vnp3m-P?3JeqQgP$AI6xB0NSNn5Rf$d zc~|9@-a(;5^{xaaLwH|?JNNsD5yx{CG*aJWVE3qXX&)lA#To)zFk}f$c*56*{jdOdz+-S`Y}~QK~k4`xUCv%U)y2g z$bYla1P7UD>4Jl2j&_F|f7EVjVdy5`ZKvueYCbv{FDEEKa?&adR&){%E);g*7>7!~ z84$oaKSBadYu^VTTKl-92jP=??VKnO^Kk}6QecCV9Ylo)SS()6-3r3hTi$J# zoIbt04uG!LcWBh+Fo-N@pe#AKtA7!ld;HVABSQNAvs8r1Te)OX;!pPOjr%);7{h~TCCn85Tz0%8;MPKKL+-z~-aC)ZYgssfGxnn$|z?)N`u z^T_g_7e!Fb+>14g_YokB5_2F+sr$$(KULb_)D{0-lx@!w{dqfb_<0bC^m4L&2B;Bk zl%kOGNk;!-Ze%GmlEJ90z9ISbi6jFDLg>;Z*-6hq%?!Vfix9S6xU*$@T+jcnL5;oF!2N4cA=2s@+AVNb^g`FJD}WHcSU0;v0$E0S6Z8|w@H;M8 zH|BZ!tfS7cN$Rh>)sGCU9i}oI93Jz=E$n;0N0X|7BV0o=Fjl*6ZP0pb-&7Si{M^?k z-c)tz(OF>|l4!97)w?~Vnc((6?F_W z`|t$5PuCcU5bSi*gN{4fF!<*<$mgE&TNYbip!>2^&e~dST19i#>o9jvg_Y}|4jVe;xABfL!NciG>_P{Jp^yOUoIy12iin}D{|gSf1g`O zd|bzsMbEhDeQ8i?&M&K`=*NuZ^WD`^KBgwm+-0y?)=j?Ec0rC(czCaW4&(+y2XCtT zh6Q;~O?3pd1Rg#{=Ih#Vu)-) z%+)UvWP-^+8usN*^KYzk*^O1 z?QY_Z(Ba=&bT|d3V7$7z-56lJ0cb4(#CF*vZO2o7Cy_?p;4c6T{S~8LWNJaoU@-v+nHO~M2*dk=V{J-Xla+4=r z{y(Km{ZuqUS6f>nD2Ho+!MZn)rw-VkvFL<)2@){;hZTXkmNeJ5ga8^ft<1o-gP!4) z7VFN@%?|;a2CPa@hJ!hb1~0zr2y8Q_cE$lD(4hQW1fuUbFq1P4R^U_haQTwyjy*mg zbu|usSzz9I=b$Tf)ai)J);LFlWJgFj%zr9L2>f;$#3W@8lnV~QhQ9y&h^k=0{FO^J z0ll~rw}#>R3s45tK|l*<%#y*-wyVLewFkuC=C@(H zdirU3HYsGgDiU_y(y`MB=N)7TO!}>^z5!68iiS6H&GX#_%9`YQJ zUhNo0{O$*ID7062dwjE%k5(HZ!cA^n%vE8`Pg{V<6Vn35*X;#S?OqC-A1F8v!CNCe z2!5Q@cy3X*1rsdM1|;-LL}#ERJcxBy8VP;S1Ay8;E|^0jCngXJ+wPEn1N4Eila=-# z5vQhEK)ui7C-}%zp7F|`!ZiwtWr%0!BX~_;+QRH2p#qd3I68g>w<3Q3=pb7utlX@+ z+h_vb0jZmD=xbS|N}_=pHShw4rGQ5!TKN{4Ul1X7{iufEhH6?+kJ44=M)l`BtDLvt zTFEOuTcc=qm=~Ggi;U}l+ys(7P^;ViHixn@z@vFw3w%tWrU5jHuH8I^KbQNjA+RO{ zzimHl!oWL1hTXLo%mv80l*NT$aBrXcrJZjT?4Ta#Ii~da z;4JcLfQ3#6R5mp4Ez0n9_VCS54TbD37-T$;>8|_7+$Vy1fOk1^z=X#3{dx);C!Q1N33lKg!jfY za*r_17tNfDGFvu#tt2{V48@YPLcZEM>gr~&h#VqujdZyBiv?UMbqrI7E5O;$Jy+Fk$Pm2p7Fyao{qD;XYXFk5o-aeIdGT6~g9SauFTI~)>oH8xHRF17 zNxr%6MoO19t>qfV95~%vdxGoD19R60c-{o->+56|57>8_ioP2yrly}=ODoWO%)G(q*+10LZ0ua61DF1( z693h6#VeDz1GG!un{xdqD$Zl?*7fn*fYdjeeED0!4@AasDR zoRBsjFVPyreSM{rSDkTQeq#?$gh^H~t=jvs6lSE2l?b7Dka+IyZpfFu>{#v03vb*w z`B`!)N#xubVOzo@UGARyAE?E#W+$PcGzzStqeRPU zqHAb(olE_B=RpG@ix}c1Y#g!r1vbVd*#z>gJtqW%F^m>Kj{4E`wKDehx$#=HZ zQ~A~#CSzzb6ElycH<|E#Uf0K4UpIK4^R&&Wn$9zw^0eG^hf>pb#+ABM$+Rn z1v7RT0rNcEn)9M&0J8L;Mm2UR){Ce5oi{+b4td&{Ekg~V-BFhWfu7>LB^=GG9YEMJ5z;A8$L;5;6h=~} zla0F677@SEoK3O_RZzD1_vhu`E(CX{Y7Jbz`*S()7Oxr_Ph&nO2bVr)K1Vs?fk=2z z(Dz$I&vXYK;yovGmsPsx&N5U2t|`2eGNlDC6pP=_i0uUNx}qD!k)&-ScqwDMpGrc0 zBy-U;WaF=&{r&}~b;3%8MT$Mx4iYVuNUB*s(5gDgxq(VNER1|z%I9<&mLF*j$=MH| zs%}vBT0pc8h)a!d(*;IRhgek)^3Kr5Uha{X?HPzR_{?iGqE)MFU1z@Cx^`F=VDJPV z*}qwYpq6!f5T}S7-j)sAgSagm?yvTTweedxN}gT&Z|ZB?-adT#j>#=?c}@nD-xpBx zo)pZ1Sd{4fLJJ|QMxSt=ph|V$Syw{FS9*mRxPq#YYSgq^>Iw?!)JfgByd;$w$z-eRdaLQ6fDYQ_1;rI3` zerAeaM3~an*bH`>(XJ!N!=Jn>>0EA=NNL2)OqYF%PmuMr!}Z-lZ@PJNl}yVyfCOT> zt|z3)Uh@w4E=Im};v;kMxMs32o{$?9#JgVTelC*Nnu9G-&oI_SF0g4SYxjYsD#_JH z42vr+OB!^Hb!4+-mUi#EZ}Iw=IZs9Dg{$IQaZktjk#J&zaY)2nj`oNM$H8owd0fSU z17_@@yKs&WTx_7Ip3_LHTDz=G=K-Q z?ItaN)Np00VF!TnIANSAm&w}AUu)*dv`;{HqxS zXS*U~nUQfz5RP4#cKghNjdFfn2SLE1A7O1+?Cy%~sp7|7B|bk{W7@DUVC}3;aA{Ku zSV50%*#P`Y84z36$$I%P`(9!mTkDKtb0DX8k0ywVUxL0#rnRnD_6BC&KynhMh7L9`2o3j?2`z3hxt1NXhv4KSl{i?or_ToCm|h#vVJ1?_Uly^d~FgJJGXlEZWlY zIPwNi49xH-S?F4*Go{J$v}@9oy6ogjnYr(Y%T6Z+vYpux&s=%^{i2_xEF}ew4r|{C z$5+3n?@B)NOzTi-KgJ@A4G@4W@FRqA3G)jr!LGmpzGp(uz8lv9{yshTYDd&gPGb|O zGSB=(;sW#;l$6I}!YXV1In_rDK6zATdE(s%9lOU&wl3WfETmhDJ@i|7Q?v$LvGzL8 ziRHIANFH-)Mt;-XoD;^Ox=reo8T_f6aE(Uy?gn3|*J*`MTNDqrEb38hHp(QSPoMJU z-?$x+DLKNXkGEuj_ci|2@*Om?(gy-hQxlT4}08`3oDG|rC2^)xPTHB4a1dZ zE)lMWjfZ~ERw$F<)49L5aAsyldG*A%mmX1y4PT>B*~G>@HGh)D^JEO=qhPiv@vaF6H_i#ckh!Auqua7 zZ`pMXP6%@G$eRqUmS#fOA;XpX){vIEP<@2-T-?M7nc8@-JQVI%Bz}=Zbq=-~pQd}< zeTzRSK)P_ZwVm`pY-O-eKqm_;=N;LPsE-m7#2WFO4^!FX=t3tAW%q#fKw^|lfv;&f zJh7@2^EO;x0+uIS*T%)9D9Y9EeA{ZNxmf)Ejn%0DrZVH_a1)FDYJ*0SAWQKe$b>$h z4cwYQ%4-!Th`bBN9&K7;o8OSd7?v`EKH9?5>fJ2JY0afTcayrD4*mt-b;ikO_gmj6 zr=I_$=x{@6CV)|w_E=X{n?3!fZ;ZtJNMI^mxI98CEh{-{^rZx%PUd)TXs%fEhD1%q zooh_N8fB3V?yVNLkDV&+{jzR=j-(9lhAHl=GyZ+U_>ZKNuVwquXHZYSfA5kv^`2Wd zXDoelfn&B}+DY--1jtVO;;zX^R5`lQ1iiiUk_lzyJ0D$H_Qt*;jk9CfCQ6>`TQgZ) zQKaqX*n7ig%Y7hjdKX3-b=j)QWy2CX%xse#3AuK)T2yp`zMs@VA-}~2%;Fv-Y~hro z%eIp~j982RfkeMMkaOSZCNjRo_dwwP%af zlyUFOVGF(2EKDl%^L|8*H3m5}yxExV>@d+HWGm-u=x(=k`*jFSL5_4FlG3yAOY` zrMUq1fo7*u9O#o7))LBd1unn?RB1FW-s%&H8?8_HE%y(9@6=osvF!{v%Ido5h zy}@Qi_HE+TN3W#!*SI!ayfkUnzr6mgm%b#cq2BnYy(C+AHE#OoP2y+Jt+DRa%6 z@BE2M+d+6Isl%^;kV#p#;XTjQ03)K>I)<<~WV+8F77ImJh?)CO1iNet_|a4!_uVM2 zV=X-CfwA{M+0>T8p*AqPMJRExtW9nvs1ijzId zshiZ~H_vKEoru<(Ql_%cULZe-WM6_Mu`A66%YrODo>!I19U=A+yJaNP-K=czOoiBm3O z>}R9m2?VPezHa2#5J_4*;8KSu7R?I-9Q-pgu>p4o!V@x~5ylMZK0rj z?_m1Y_17t9g;xu`zkl|b_XosAR`5GSA?^=iJ=ni~_bHBuPhbkCFqPLUXoi1?zmPWe z?kn`LO`ydC88r=OM5VkZ+sE$i(s9Un7Dy0)ZG3;V3+`S$0i#CMD{X1SdkKT=EP(aVeNAR*0?>V78G zI6tK>`cU&RdqkQHgn@S$#<8gYyHIp6mXjlmg2wI4_SyEYMuptS9%yC&!5wS#+W%hd zue|v_k}5;yDnp+P1OcdG1w_DzohJe*auyE1aIFgvP86_ADRA z&bCZ!gZj*e>Zdc;n{|%nGZ}JsIkiA!)&!;dUO2M8b&#ubX=jAgt3q!&l znB`Y|g*s3wDP=r?(>i&*GghHn*+e3xzB{o7AL1Y$aIlOUV;R9BBzSVpAAQ^=;Rr)< zb~kkS*9ulW^(AjBTGnTC7O(!903>f2o10kA-~wCb5YOd41;)GH8$dTjMSxKA{bP=( z%Pc9p)9xxHm_V0_aN|Wy43-r5ya3#2O;`W|dgX3yXpohbN|ltrw49|lLhAAZ;54=X z=GBgxU$IJ;#_bLzE#9uiLm>#=SuTj{o+)8HQJU#`d_~_!>h@Oy+?4DYJIHHi+aE@4 z%!850-TMfVW8HB*#|(bUdbpsqGckoJD>VH)eVBVm9~8TB0w$`14k8FUs7?z1&4$#vtXMM-zJMe(+4xHHmk#RfaA3 zH4Jbd0j;5g^xr!aIh{z5$E@eyM}aL4<^bA0$h_hbh5Yu1605(R+d!dqOfMG4rG(g`bcd(X(64nV z^7@A)H-M&(N@&A|H(O4mt6yI-61~{6gXhn1F0=p_&i!-qSL?v&wFUqvGKW1#VKTp?ed&R`ccuO+#Ab%6Rz@Qg7}B(5{CRT`3{ zLiir;6};2eV{_5@bZp87275bD1x)+s+T|{cK{$nS1cQ(c{#o??j^Mv6LKa8??jY6R z3J-8y$}{-P#g)xCG@z`1^E0v1(g>nJn*07fjtw8R(|0VYogw?VY{8a#cAD&dqKLf~ z*R|GsC_Uvjt5ST@- zxvuYzruMdY+OyyP>mi&T=R10I;3k@Jp?i+c>Ih%$ybf(o&2UVa(Z(FJUMX}2vA(x$ z=s@06Gj~rubd=PJ=RX*EVR<|z3B}ggXnyEdOwQ-rC1NzS+2y9R^*ywQ$Fa$4bk{_vv6So!mT!4A(&= zpjYyb>%tBjwbUoD+NOR^Za1ZV|+>RX1W<<>b#KO^W|M+8! z-^vs=>aa1D#+}-t#9=((2Y2p_mS1LURyK;oaQ07zi4X4KK;eln;uTwLaFx4BiADh9 z?K?ZXxxr+3V^W*kUA5qy5bG+wgkbG!cS&@H@(3w~`AfT=NYKoFf1VfHM*@E#T4j*) z6|EAQZS34PZT74CaRs}%yEGFEXUQG*Ss*)eWabHXmBgDY+@z+@cHN|e;i`+}3p4IX zqd4w|KjyZ7wrIC1)wT$9pGF44yn$;X!oTsMjun~lXKG-bAT}6PC#0XWWIk~P%Aq&Z z@meUk1S3kB<_FrLc7a_oZ>;4QPht{qubGHQ5U*G32Q)?%R-DKKYC5Jd!Mi@DOj(Vt#uwd1B|8aP{w zq51P~3REc;FRJmx{@Z)}&u9X084&9vbS8rXiS_>DqlXaEAt^l~_W%6f@R|+;iTMy> zkId_Te4ss0p6xdtC7}N6Gg^dTlQ4M9nEuBhAb;#j1E3;>Eg$_~pW&*(oHjQo`!^{5 zXC7?Gke+6M{}(DG|Oh4#|9-l3?Fg%3S{OwEk-&-V@6Q-(F2QS>^7Id)YORjV3Lz#6{y8N4ha%~ zX-T7i<>+Az3p;&oOMLGmrTv7^_tlO+e>yXiaB!x}_e9cIgQ^Tx~ZXs=$ z494(~3>h9|bb(NPQwmXJw&emQEyt{w7@?r1Vb8s6BY2K4g-vU7#?Lx&LYho0d8O#r zVO9TFkORaF`UhX`hi9IER!U4kE)2t**r%a6foh0l776kpe;!%FFf#_9&wjlKs1GUd zqRa2nOAwUhpN{}MBsD;#YEb^SWgS9D8M@zSzW=-}0&vg6Zk9ahB&_-0HkVEiOo!t) zoa+CwIVmczkTtC65{>(Bdyg3QUeOEEp$Grv2`wntzLhPUgUbJ0-hciqh5!QX7j#bk z^N;`h`6S>tz5l-_f=r88>=_<7879I5sGxvwaC;6#rHPqv8xGX2S%1HUf6)p-=H+-K zPkceJ{5$0Od7~lm78u=0L05iNlr)*-_Qx2KlOF1SeRf|8*6X1v&P#XxUbsNc7SUle z<&y8;fB9cOJmr8Dx&A666#10@^uSGl2@mA#-Ttpjb{UrJy^9<%*MG4G1ftmDt^}fP z<*KM7PMsM?ti{q79T<&ypJD1hTSrTi?mAhSkk%% zw`;9=uZ`c{{%Xi;>Lh6XF6876C{=+3Erl2hVxePjnZ$MhZA15dED~P;i0!Odwqgjx zy9)ra!p&*a*KE{0W~7KoNJ76)#EzRQtw8&P3@MLxx|)GY$U3-2LH+gQ+g1f+FgAE7 zkHrr+`g7WVRC@5>L8;AiLMP~;cpbQ-U+Yt;%nB&Vq-&QbGcMdqVNnVVBV~G94=54{ z((1vb0%AM&l;`CX0&E>-G956^41OHRh9IyOn(cAta19eYY=QbPxI_b~VuC3N6%Spy z%~b*X6#ytuzfR`jGW8@Iv!}99$O-Bz0(`gC=VW58K`l^k z0)%lAuQC7Rl5!Mv&qG><9GpNXe;z3@{^&IW2IoA$@6oqs3$#B{xX)+fWZL0mMVY$LNBNKu;dCs&qTS`Y8`IdLT}eW@>-##`T(}#FC(749H_=P|2FZr2 z@Z&_xequ!|MCZjp4a|aTU`crbsQTpg>_CC$st?1*bL!}Q7pN4Sgrx-Nhq*>qU9+rw z2remtPN@e#sl_svTfe?AlgX68#cN)3zR?{>-P@p7QF=5_8$$s5LnZWew#jLREEg@+ zNmfQJ!Q@^|3twUoZ!>;9*C`zh^!UJK%JNkb?3w6yozQ>vY9+L3_||*=5eB2AM0|K_ z>g7SqVDSRMwPbspmOjz#2K~n3I%IV+1Hz*zeUAVIfIGvWVRH1oqq)vIXyTjQ;s+o= z`q&X%g2Qoq5KK&8gAPP(bkE>#FQ#eCaG0ZNCsY%j5!?@=*TU8KMB@6R7O2p+s59Ok zf?;?WSmRbLYN$=NPWKeyd`j2}rE@FiO(Te0SnJHGP`?%N36Z?FS*EhVU$y^-gC zZ1SSNrr9_qIO(q7mNt;q*xx5$BV12Q80B30n8}pvsk|nJzOSupsBeH&Ks;Gf3@7m4 z+v1?h8_Sq~p@+NJn;x~{T%-p|nYf~lk;xPUq%f58YSSVo$2H>rSR$R}dXpDdy)q0qW5Kx>3*fj zf51t=j>{@paBruzEt-~cN_-jm$(f)18sJ4tKv(^v^@{D*eAh~Zhnhxud`+BNA5pKq z^3etV1Cz7FMTCek#!Uj#i+!SZ0a-@$eH2)^a>J6R@3ss*+gD!8WjK%x+zi0wubMM? z?cc<8RZ<*{3(dUVL*3~}L)&za5~n-V^-n)hqF5MvaZQ)9M@E-3^$8d&FfI;PsT%V^ znL4CQ72==;RbrsaRhb)#ef+73g`IM^msxa7i-W2geM~IFl|Dt&j4{FgzO6D^K)ny$ zOQYyV`y7*@d*Zl1i8ZeVbDjuEH=s~2pl}wSl+nP%ZM8U5(Rg`iCF*eO3n*U!8javL zj(7olY;)qNn;@4!w5J-3+$vaWbYx6)W@pjTa|z?;!@4Cs>sdG^SC=SBjAt+|PLK?2 z*r_U0gD1$w@nhZiI?PLz37Qr?2fb-Fn*wbI*CAOZSf}NPmzSPBytg^4A~sy}(nu{B zk82No8+JT-^@aQS9Y+58eUVGmj%2?G-)R6|{c}Sb>hukw~8jF*_R)Fc}A$Nzj-Xvpcw7 z#zUtnF#S@-A-$4IepHPhe86{p4%#13i@~YjeOA|Is2ET@TAm41!g;VeW>pT<_A~cu6c@|3D+pm&+ zhl32G0`V)Q74NGc-f9`Z6?d650dy8hFdFivr|y+re!(y;kIK$3>OMUaR%TK6GX5vO z)MNZW_E=VxNNk)FKCfo~n8j~X79JlXq0Q!FL%B8MsNi*QaY?DsV~$* zHNWSigavWaUfrx143sMNLJ#?^Ow!_~lXv$l7CWw=t%jg{gA!Q>+^1JD>tK)K_MHMx z4AtW~J2|PveY8uQ+_yvP=AwgY0A;s-asRfLrUL~OO#-lTXUGGqw*eCJV!Lp?{cixx zL7|gNG48JGK67bz&n5#k7HT&2{hfvrGGgFxaB>zeC-zD{Yl6GQpctd5NFMooX6%1U z$PNja`Cfp!9H^chwbRmbvTyU>+DN3k>71^62({gM3ie-_31P|$At2hy|A8Y)Yx2^T za6D9+$?j7Lpq7y6(vJ_-eo>~mK6g2R`-9@E;LJD@Mw+L0$mo)K4@!xF!h5N@Me0B2BV zmT~S)g`{>wk>KaUPRVS(?i+Q3DLQzA=W~?6{!e|BodUM%>3LJ2A{@vi*OT^OKY*k$ zOZDNY@b{?_0@A7o{(fg`K3lydC=6PywhdZm_T_DYMVY)cO6JZdJ*OzFyAl1nS5n;A z(DnB-3JBQcAxBsuEl$8tRQqJ;rYBIXvjwQA)p>11nC6)|TO=_9ipWmuUvG^|8uB@f zvY-d!vd?NOx_0!%&8sc29r|;z{cGv}nP!zm-Suro86uB*1)34~T! z3G*k3+dx-~w0s*k%peuSju{slTcH+n9J1|t{3(6@G9F>=_voN4r}E-y5U;cZnmpmAG0vbnFfp` zz!b0t`;qi~OIYF&)_|%WE^(7+XM2Erz_$4@gzedW;_Z@qOIRfER z5SFPq0e^#+&!hs&*sTD1G^^=oW?pEjjkF_y3mM+tYGQCo4}!XTyWX@r^PO?5%>3^2 zompYE(}+$%0R+3zmg*O+FO!*W!6L3~WHe<-WnJ-N z0~*bLhjQTxtxnVr|qkEOnX3582o7rL^tvfwNm7aORjDLfG!QdOEvW3br|f&=FHZH%0y zMZ#s0tUc%6SHSUnch=VgWzX4tNP^uEXVMo(P2Q^Jlwj|ZC4=obu?(BIxe>tJg|@^q zC)-z=gIvx{hlr3o?zcGo&~(z`!8;_*XfZ)f^N7E$Ibz*iFEYE$w!$RKL&Z=C(do2@ z9-a?-B~=Yhx%TFU7T5erlkq0mEABL_mg~CKM9@MEoq=k432M`HuzR$aY%6o{@CVFh z-nMW!S%Xj}MdH@c`jkvV+~VE3EALL!vUNP9=h6t$b_P-$gjj0n($$pLHm^W9b596s z&eF^DHfvezeg^CjR9!Yw*jZx7-gq@>M{FgE+p>|0@FiaSPAm5<- zS_NL$lV_A%?=wHqj@3Aud(I%dsige1_EVyFHp)Gs(rv}8FWmigDlQ0-VX{hxgt~sp zTE7C>9K(s$Po_c;_&{fEkP}$VR@1j6Mv|c`r$OQd7>%@TyPE?YxN|z%;oiGuB3fTR zE*4YZAz`P+MaZDzow<9E;63n|4E<{)5C>n!glNW{KU?s8F9+*&e4nq;N`ZX1L~o4m z9v$rUrpvI!Xqyh6e>Mho5facxa!5@KaPjG6Xf58i;nOx0*ZLZ;IOdH?O=fH(6$HTA z6O+>DELtze%wB~cA>{qVeOAlykT7D0`$8b7O=OcoUl1b@Ad5yM#{Dc1Afg`K9A*CD*~~hAavqtMLs971 zU76!~2sUh~Cl%ki>?O6SsZgHP+f`Ax!vv(-_45G^My)M5qTEda)NC`{>Z`piU|5{su~blqlKT7{cjHCqRycoRi$vmY zrU3qYw*YHGpZVx__H^CX-v%Cm}x)A&&%6EJL}beVWqwP{|c7TMNiL$vGpYdFJ=QhP@~AGBr{kl2mHVNnK=9 zr+nkN%pWPLAq3S+Ik!`=KBLiuAtSPPG(c(cwi-UIhZo@DVd& z(4QiLEal5c^BVPyR?u)08s)((=kX|a03Bd0{G$@lrNsZ11T7rXFfWl7QhA~%E&VU{=dGN8gz?1Q!$ueLqQvN zF2`R{JV0Qm7*M5EAcY)iqvo9=jC`$c0@O{Xh( zYh5|HrKcHR?@&5Lm{fxlxO6{bxvwuO>T|Fw^dO>7ma6(APC*doNtR9O)3X4MoXg9e zoPbI*^8F>P`X){yo$z&Bs3(RhHnC@cpWi^7L$Ti?6?JiDS1*cQqI^gYB$-=DkX5=u zKc#bq=MLOZi6uIWOjje#6)?aSS_yRpBJo?qZMsw#94vqpR|kC#fZ1BEunV=B3LHE# zZP1MeDSQJ^VosJ-$3&;=QH-ut&M|Xj`bp=Lr_TjaKzOxI<5pPc1J3hLHyK%I?`h37^mvZSt@;e$@!d4>uXXHriGgh&DR%(A^&M@3 zPA3SsNLb;f1|cWjysZm*5}3m(D6FNFL~zb2{+_y2%eT zvsRWYwdaZ+hzbsQvd-h#%OOaFW+H+fNjdN5)0_%_W)~h$zy<7)A_fU6uCy|1UGkaj;~SlkIJ5sWE1f?B zowag~ZSGVhAZ_yD(Mm(&52gVsiOzWnr~gcbWKt|pe?B&D0jI8uJ{FJF{VUh|r%Igr z64qIf&B8T+L$a9MPlMobMOimkEow-d>za?cDay5F<|!wTVI*S8SI?|_xDwv(yER$+ zW_E{!C;2=3`d#wt;-D?LLMK~OyX`z!@bJOY&vhP!fZ#yv8(;sX3)M4o^{Ks&4LK@2 zwmVs*u(K>wf?A=OT4<8Uj@98zdK~%h#lA5}nsS9i`Lj`rzy&cp`YS`Fg%DJuCR-FE zGJujS-&mHWllZP(*&B87bjS18eNhftVPtAa5$0#}ypum$(Qy;Dr1yidgs3S(1gPm` z$U5|RY#_YYho(oF^7r{gmAA$rV9zcT4a9%S8lZI+B7?9lNY9j6@GgIHal0m9QOi8b z&~HQJ=J8Uls_jLeAFR3u!t~VbB#r<#XcC9jc#HcE|M_DGs+S`wXH#3+?F`~5pHF9s zSxDWdm+@H^!Zx@r?LMyt9k|*w*aKO=5vZ3kp4@sD43i8OL=-?qF+gctW+vw6)la#E zKh{nLQPZWF2&FvndPh2?8;(WffsoWj&F=#qnEfIDq8?8ulB6eaf*Bi_P{LpsEhkYR z1#8c&OiGf!e*`rBU6w%o56W0YRt}d4qNb0~IkCwaxqSSDd6yH{!Ga4&No zPWE-+>=nboi=pD7;LKH)PCL6?d(QEa@A=+WgwUl?=^QH+naV!PH|RzNv4aL|{(F=k zKDBc~Nlh>!rwRq^FXwmci-+02tjOW#P;ToeA1hh7=kEj;Mxqo434(pzR8Bc2Kztl< z3c!da>HYkAhL6}r%zb(4lvg-e=K#dD!Hd6 zVb*IMC`}FytXKV>k|d%D!Y%-`(%rs(!xO_Gh}MMDHNPOVh7l3&))HfYPA7lU=KLCV0TxN3;o1jMq z3iBbDZF=0HUclN{sLiA+O}3r{I8;)%INcm!AQY@e&Oo}hZOhn!_>ce=B+l=2&Kb40jrY7?&+C5f`+n|cvM;s&EB-H$x}`Jv0m}?8 zPwfTO{+UzVqBa4=AhTpKAdhu9_rOx~J5?0VVwZteBW{Rzmf9ESbZ$U}*lZ>%JD;DrQ6|Ke)lBBsPl%Oo@os?d%v9#vJrr4u3t%j8F>Ua1F6AP|gBEFY$Vl<)ug=*xC z#(bK_6aNluk-Y~zC|P)g#}M_sSblLGvCjnSM^5r{r2ft2;(Dq0V)E|Qu5VBN{W1H2 zl7U@Dn4o8+YX5oUDY{a5(g=DEo{hsBKi9P1;4;6V_CbF5=E+DuR7`T#js5u4*Pl99 zXmH+|3q=<$IgoPp+tC+<#fkCW531$PAW|EUK|T4;?2pYnt0Er2mxk*> zqlPE^87w0oxLhU}279q#<;E8aOn*^Sgq1Ai_1%U$_$d1I1b8z?H!T*Q

>qd&Qx~cKaDi%@2sG0elMko_oO!sN4z=YP36p&C&MLtp2hPVGUQ<9gKN-C z+|9aL)f2OAR52CaNsw5eHJ`

r*zQskxWNV;c#^NX5Ru!a@>#4odlXRLh<y)6e9J1CIs2;Q3~hQ|E=-oo&qw+?*u>{j{@RdMx7RH@V-Q@QGwX%a z1xkCGV0vR4KqgS1_?v&z?+ty9*#6Nt@M?BJK?xEgVWLxUkQ8Sg$Il2)8yv_NPbDhu z`>uvrKWXkSL4fkquH<-&1czQ@-m=1ALOw-t_bK{gdDB8xT*;$drqH zBi$uqV?O>>j$$^4Y+lV`k=~|j3G7ZI2CD9bU>Kart6b5}qMQ!C5{R7UsX)m^s=K05 z@g}~eoAPz?JD|RV+Y~$R{#L8ykg>}~K#ieR01%0V; zyYx37vm4d-gwVs$DF>@%$uti_xaNpa%PqVy&ImE8LN8Ib;cu>3?D5=1;;gMYn?>!? z&Oq@qzs>3ouroL_htG7HRbc%#7FUi!HLv$HxFDw?ZMH zeq~I;>zCYJ#fWB;c0NPzxQYkFpG$YqR87dFuvmVl!>e^(h~f?Wzpuf;QXUM)N|uRK zRd~xEUD~pC-*T5Pzggu_6Y+zkg%n;7Zi=o@@+)`TM`s;2rx-&g8+@qf^Ox-#Hx)$i zM)(VdKqkTmeN>v0?$meGVVlXp1CAt0lUmN#22nk z9RMJLYTDipA)!7$OMPq5dR4I}YAK@!YwOH{zH~nWuM0ri_j&2T)T($Hfo&rlG@n@L z)0YxX9#j9T#(85uMoE^$m9{&4?JfhmCmKwSX@b{$k-=Sat`~=UubYFvMi`e|RLI1P zV=9|PN)oaQ4n@jsl~pG-%iG~R)iko7s_?#CJS+LIWm2I2tD2laN*UlSq9OKcd%p&k z%sv1PbvP-xNTeZXR=dvn{+d33gW>SL<*QGhG2jBJAc@C*jQ_>yA1}R@1w~d1aWM`~ z4U%pJ_7UMo~cmR^Y*|jV;v! zVP|rlUQJqDVs@*F9WIfpxJ34^gwKGexQ0;V#HjQJR_bqbmy+5`UqX%!HoTtsT4Msb zdEAG`UXQWj#Wz0g&IZF%_wz{ue}3HNvMfxSV@7xH-m9HOymGOcc;!AVXU(2@aRyjG)G#}r;MA{Zg~*0Ajs zI%k2fUCB#@O~8|&rZe{8uM%CSJh%}3BzK>CB7I51X9~9{W3P?0Pj{QessM8LYWvZk z(4DAS_p6*%1xT0s|vcHFWNLGcHTfrve-H=nGfTI~ESy{!dgmEA7sQ0dA+Y^}L}I%u)x*2>Rw?=5?D zYWZAEcX=V3F@bm2nR|1SL=UshJw1lz^VZ;tr0J=%B|jSsFquB1#6-+ zo=`EUEyWju{*$r0Ggn~;+}Je&2h^Mx+g$Bf@=N`2E$hzN^t{;&*D5xZ#H=;Ae((?v z_FPRMtf+UUpVmaX9_VH)aPXHxJHhD7rmNrN?ak{;<*W@XAF6YL@|)@=6F_P z(0qqcv5k{*EE$@ZK6ZXIS%QM-<^P;TqbY~i!p4dWgT$$^0wd9w4PytMBAYSqq@AK} z%ccg(Fc?NLf6ZT3@oHHqcWg<01C`S1SI>cI-l%au!bO?Ze`@(Mz3Z+ISYdAhQ9}L_ z{b(!A+mc{;B3Ptg#*|lDIqWar-#_x0l__CO{^!9OR6FYb7!c*}Khu8;7IFbI&|$@Z z2W;;l@M|8wP6uI47Q9vP*#eLV=ArVhI@O~;iGj;yh*5$*m%z;Nc4dX2Q~dng48I#o zo=bRIC2%=g7-gWPEV4iO3K;`&ecTz+jPn|R{=U5S>=^{qMM1NWLs4Qj0={f~oxAQN zrL*_|$wH&@_1uxFp>pFh6^$RPC@>bMH}9+INbOP>@c}tcr+>#J!H@15ph|rD9{BI` z&p}f{K!W0u?fnjCOE;R6$40R<#i3&UU=Tj2Nh!_8Ux6GwIl9AnEM@JScUtytGiRD0 z^nSAQwh*UFnNC4}#H_0WTYnKZCf9%OdjCuoIQ!MFnn2b)ajst8=ht}0F3tSOfrYSd zqy4?^BC+Nk#Q__TuoQ|IHef>JUq4$w;^4fsxf<=-9agNbD|9{>z%3dD)(Ma3*0@+`0QOt<#n>)g@YNa2U zi1SU|kw~NVDN+(xm>V(L7uRSIC;Bvv@*?c$KtlmCeol<42ZOyF$ElNi#jFqoG2VknpMTWJ;%I%<}nb zQn1?el@;an-6MHHMP*Ge9$~+itooJTFEg*zp$mN8y349eohmG)=g-MK@uuhSXwktk z9^RW&O_6bf#595|k|vQeh6oRce@1Z~AE`$jU)KL)03K53Vk?D8v#qOrLnO4*7Kx5^ zYC~C^sI<)Nb}%!SWP9H4W=8*+jx)V&m*OMlL%vdAlO+&Sllf_r9PYubFT2G|qa+1G z`z#VZ*Bex{o_(+T59D;jBjs3#cPw1SUmv8OIJBE{+sDrM=J)Xzhq?@8fek|f@Wupe zeZS5eS0*1d7CBpPw2BB!21fSRgpzVac&*V^e|E?UlYvR zmDWStoe(+CP949{r9#Jo9*PI;@RP1K{~d2UpP_}o4oa{FYgFzktb)NVZ-j#q9cMmx zMia3LXV*V6Xd-v&KzT9nuUBS&``xUS#$U!8Z~d1m!VC-QxQ>q1U}90U-8lvkvjlx1 zEe`*S>51aTJ!sz-Q6BT!7VU6z!TA!hFUNI`Qp&w2spB(b6Oyu_%=m%bcbkq7_O^j9 z11$;jAi;GtUV9gKbV6yM7M`B%UtZ^gaC3=G_mEywuSS$vBqYxlfr20rsk~?tN;Y42 z(St{k?@#7B-UdyCj*YHy^y*bT&jBnS& zBASYfnhn@!iJLy|dk5DG*sMV98L&>`$zk=)cMwm9m;$SZT0UdKFH>|P7%CPq6R&SI z#8`HZ`Y`G>Ih$%QPEKA6!~CA#+kV|joM{es6RKXtg*`V>8vRp98{LQrs;JH!Q)kP7 zYa0V=t-u3?g2u?$(zlWsIW(UyapP}{WsMbce5{SNagm_WaCl-KH{viPFK-gw4dvPP zqvjZq$XRti>-C|H&u(&WigCWo-RM^#mDmPr-BNcuop1rN<3+H6$LqAe?X%D49D}L) z!M$f5qxi=9)i}w{TG1lUVPQ!%KLG8#wpR1Ji4vdbfNP;v0kw`z-)Hc(^R|N-Nvgt2 zTmHL>0qquZN@%CAt!9Np2+8kzSSHB0%Gwhd|2kuTypCQ^h`Y^Rs$JF{8 z0gupL9Z&Y_Z%d?reccan{c&1WYV^pL!)vbUSKFzDC}5!QTu<=*rTF4j(K$-tSjEL& z=95&Z*(3dIjg)@hucdz7xPEOu=*Svedt0RHAo4a71^hYA&2iV_%9ENRAzlS^uN|n( z;`v^GYXRBeatgDkW)8+}bsoH$B)3D;eGWhWeXFqR>4s)%oE3Z>|Jcfw)4kGxHaGqg z6!>tVuHbODEF|8ZXlqiAw_TTCsd-HE){pPPObz#O9ulk(ioep5?N#UFZ{FX_1JL^N z>K0msH>e*r2IP^UL86*-$+H_*bXhgfADiHXhrTL#*2kYONpm6-GpP%w5+?2@7rrwj zj}_^*11GrYDxCZqn7vuwgJo$PR$ce%g&>7R?jo-Dv)}Wc;`Ke4O-fX1_+-|wvtvo! zji{=HS>miS&E@6FTC{&XINB1A9yYv70wIQ*44=hbbOj%uZ!_R9|97c-&xBKtfpjzC zyJM5u6VaBb)r$U6x|_1+L{|Q79Bu(B_(2gR%a%@VZ{jZTnt*fF{^TR^Aof0g-&}}n z1Qz-Kk)z{nNHk0MvpQU-1lK>P+rL1?Je`uL!I^%~o)h`+mtHoxQIj+-IS@^CNOlpq zCl>Y)hGFQmZ3T6I!u}pOXtLVb^e`dA_zP@%6hU3Q;i>7rGq)v8l{w$b91ZMmw4!20 zDi^3Y^0fI9!z6Tks5TQ_%j-k9_s_P>&~xv=iMc<&Qn3b~%uh8Md^Mtas3(9E6ph1d zPl%OYfmqGRkN*{f@CZ;nSrK+-EYffmu3Y;_9fr?rdcTQ-yo^eus%=Kd3fuVB3hp>c z3*Hw2-J$S~`_zusjCAp?^S7o-R6RUn$L+GF0P5W~84CNPvQV>9HB5B8#X4)yQjus) z9#8?l2N2{KCeRPC@Yrli1sfUxR$jg6)H+OIYjU=ynTTd=kY~S<7}M?M??|^nS@60f zzg|_T9+p#l?xKZdTtCP6@avu6mm^9c1@nk{cvtRytoa&Q_(qqDMww~h?YM5R<*KG%8_n7R`#e{Q>z>@EZT%+ke^Sd1Ob=MJ4{OmducyGUOS>cj#8 zyMo-O;MSYQ@5pAPx;{WcAl9Wgy>uQ`$iTh~w?iYWw@D2LH;+x{NU5|(v@pfh97Bvr zrpwm;uZvl9qWu*C_;|XwFB8&o_RVw=-sDz@l-Q7_%vik*A<}iVQ39%bo2bh3y)V#1 z<@h&IBaNZ)F91oFx@rnFi>(-FTJAMVeoMsD@=k$wHzS@me+AU-yJ=sO(RGztU{hY& zEW6zG9OhPlzXa@$BX<-6wbZ$?*Y7+?IS12P6_o@(w4VI8)01!*-UWI3X5qXZlozZQ zrX@~K)#7_)t4-5XDE8iy687X^2J#TOj$GTm)A(}|wA1dlsMo{n-*J^G=1b{7LeaV* z9v6+A%JGXT!t+IEj}9di3NMKm)5xuD2I4^F~BSYvUzskp7IT8V{u2D zzrB8N28X}hVFeh+fT6oBukV@I(UiGC5p$HT#M|i9X%XW<%6tx#_TPwO%I*0gPn;w| zdsi0LM@xby)EO4xf;g;t*2$Y@j=Uvg2zeldOuOv&NC~S~PMq2C#r)IK(6UGW^fJ8)$O1|`lguj@q+HRy4Km|SvCoX$ z=B*t4nEJ}hUct%2bUXajjziC~zHZh3mx~c9BWXUDf1zvZuT(ap=B!cUY>aIep9>St zlWT3n1GZrRyx776U9l+aJYexAN4e~yrtMcfcL?f}k7=GJkldZrLxUq_<{y{>ZxmqR zMfVkEg-hcIz#F&(>9srDD&L|7$?Ucbb;b6bx0&;9T$R0_jehP?jhD?>PuZ=m0h>rS zauIPa9J_d=3-`j9%MgN}P`;%WfuFmkbo`7ZVmLlfdHnrA?lF6d>$XNWCSAfN3RtgT z&J;LHWv3n|+J3Na54SQ$oo!m(DXkE+W!Js-LV}6sH=zI%d%X#1)uPh`ipJq zLyykCdEIF{GLzO_Ru_@l`QdARVm$Om)@sa;`6g_ukwdMyL2KObY%7kn5u?pvtf*V| zvBrx{t4g;0`nDmq(e{v7O@KP$j9q)h^3Aa!NJ4vQ_dJVwbq=Qq z6CBrMG~HEJh*%p(KX{TmeAbGJiAs8$B)y$B9Gy~GQD@~>0rYl+2VPHH-8oau{oDVa z1(51+=wJiH+<-UzNwohWHMS#rzBKsdCFHws9?;57FAJHmYr#yO{JY!p!C(042!~U| z+V@6FeOIQuJ-_b0My$jBkX0QS#-Z($!knX;JvZ!O!IKK|!$q&zg>5Rr+9$Mh-h>Mn zu49vOTGPvmsT%!k@{f=)dcJfsqHc;vgKxKm;osP8QM1iVE1jtyAPocqxP^O@e;fon{Q~YyrmvLrKKr5Y#8qUc zI-lHg^H4@!iqbq@q<`eXLPNB~)kX=p_>nNtQ25i-FM#~num=9^j}U=xzH{oF>4I}n zjkmIX%;eOOtw{=F5Y?D|?Lj$OT#1adOLJ)P?j(1Wmwcg$3?0JodWqCiuUU^M6iUYp z^IQxMVEu{A>QQ=ngjL}g8z$W{G2T|9@^~a%r^b{gkqV1qYgSw`89Y{TduZJeD(rk- zxV12Tzv$Vt+Maz*ti|N9Q+l$(ZQ|*!z8@*xu$2CNJ&ay z#o254pddJ@6b4+`ZZ^GRBwL`l*eM-`2&RIx)No%PF7i9!vKx#S@wI4*t=2o4?_r+3 zQQ|9%*KmK&kFIYwxNnbYbe=l2Rcu(Um$u7#q+0AjZvVZ!r}-=@dF;qR4L8|gYfG#p zfat~+8m2v>^p1-zY^{7~ZVy&rZt;E$U&Srir&`KjeTzj(OmCNUHk;;dD?5MPy^gnP z!LbKOB|eo8#&M+@RlvN7sv5UV<9Y#$MJ)yE8vjzvXve?5e~`~c2@f!s(JCy7ci7{H zImCYs#X_B9s?^VuI)`;Rjvl4QNehGLlCjW%O*-Je9b+8LV&x?F%e@<1bp5 z`MODdWe3gk$*}P+^)!H&(EJ>-TPZX~P0i`hIJVXR5&{V1tH;@4fC+f&OVg9q&RQbh zC*hSy)6uHuHSBi&Ah4fmO)@?-QktDFY@Dh|yvQtlT7ei$*h@~ z;{V4xIG19dr>Ote-u6oeUeYG%JGa!z8C>X(-6|`hMu&QO^x)Usf1%ghSI8YnO^f~v zJNz(xUcfUBrbA}zf@;urO!mQ1BZcC+CKrO0+`Q~c{+dZ`NF~MdH%K?b^)rC|6{QBU2 z7T)E~!wWx&BDZ0^-@J7P;MmB-R%5$R<)?Kw<7LF3y-!=GJP+9hKH@JL*X=u427_|V~A`yMp_X|Hd=juCTAG11-uc}@a59j2C(Z&OKgOtb_-T6I&u}AkvXz#V$9_`DR z$l*kd8o~XYr(`v)aXaeVMH-mhzL{5*`*Pz%;^!6ee7A6K-qHDJps6VEILZy7+sZL5N^nxeJ=b)L(;FUF(&Zs zXXvyGhvMwLGE$_Jkwz{-6>UuTvgXNUsVml?%XZ>-@pM;}OKO6D`ujUUE8D@LMry6{ zo!YwnG+vC}?Ul&U-@mYTD-N^`G1*IhW~&aMob5IQprNH@5XB0J5jz?hwXf!iFU>Yr z9_HrV?b|QJ(B<<*eK8jV*Gk1QQV;Lf{ic{vc#5zV>S^63@mC*qe-|!Zk8V<`GrZV6!@$IwI1 zB*qV=A7!4hO3J-!`fj$xIF9K0*N0$W07QeJ?2p--o;17y!U8Dn+Ll>DNc;;>0&xpv z`mF!4bacV!S6?ZueL1tp0c3^G9<2aR7sjp6d} zRT&tSUP3?)$Bs6%<_thWdwk($2Sckp$>mR8t0+G+FW;gy`0}+$&v~;6cIBmT&`3a+ zhW5wZ4s8W>k01<0cGtcCDPN(>g-@m8TqFq1R4F1n>}K77bC9oh*d$;KNCdvRAwied zRk{^2`;q8KRW@-?6nW7cw;rc#1qus5$f3dD<+HqmM7Z2&e`5cgc^)GlO5qX^C74H!N}u#!P43nh zyu8hDdQd<;=6-J7E?`5ifH%Cc-2+>uj!3R=8@(-XoZHspYpO=jo2If1G7;z}tnNKy zag8A6w*Lz{A<=%j41o}DnDdQ@jD*kZ%wz^7+m%#iZo6GTu-fCu#&B#iQ094&eA#G$a+Z6wLOkW`8{tMK2fr#UI0@RQPC-oeQg3j&X4!mCwX>GSpi$ylw44AMIs{Yj$8aDlN-in!{n!$fNNadG znxWfrmmkud@#{T<1>?ZPU`{HkC7P)4~P|u*H&)&F$9vV){ zx#BwspD>;hu9Uo~j@3P-GU{FV9S$C)7d<6Q4j{-J>Ag1`?}CnF3d@LX5vY}>`$9HW zl$X~lnufgjHdBxqXD~9C9B=DG=qDw`bvvzHEAU_fiI7OatU_yoWMZNi66y5Oi{LUF zyC^-f$RnRKOaqhR{Ft=1G+@@pf&zhyMEIobY4N8qTi-u)v!5-yQ*a&6ijrTuXxHCF z(?H2V-^S~wFpx}U+8tT3ht_CioX6CSBA;1r2yU7v$K%i3qeFKwIF}cK${Vpr-5Df&?+HEBT2C#+nGEM6d%17uQ%KWM8gd0`0>|aT(x9?D&o79|4Y%Jua=Y;s zbAz?SOE?j9ee2W=D(24bL7=_qOUHeOfwK4f=wW|K07`XTqj2Y}G8dLl6x!&nfP9i0 z5sG*O0RaEA#f2D=Q30;~In@318O+q^QH0iz9;owOq?I3tz&SRO|CxkdJx5$K_nEh( zLI$f9;5J9vcmzT|L#NVXHpo%`Wj?FwUh$I&CeASGNxIloom8hv`hKONdn*a`9ygf5 z6n6fcuz0v~UL=cbNoVc93(FXbR90;fKSA=yOQd(z;$2ZIUjlM}-Q7Y4E-yWIlKQb! zO<2UjraA(C-wE0NMNmpFBO4yJsvgXN$~_-MMNl=NCgr1BQ|totRCUfRGe0_h^7!;o z=ONq^`DywZxu}njFR+bw$|rYvbkcA9@!6xj=RloKb=0=5?(cOYQ^Vb1vzep%?!Lg} zsf}U7Mkt1HgcRHp>?w|4&!hi9FXCw-Sfp&n-s)|Z657b6u$dvxx|oH`Ipz?A2=)nE zWF{ejv8s%Ru^XY=K?XZRczi8-f%rcgIq^>U8CBX5cleKAE% z`CtP4BVe|q`+}?9R`^&|9pbXHnFKN>^WX`$bKv5Dl};ky zftq@|CoG8n67L#x=I_UPp~*5?#Sa%My}^!kjkv0Y51u*Po@vy(^ao;%qhu8 zUzG4!ZF_y*6*L3&*HOAwII5yhT+qQt1!TFAZO2YR|-Ww_v-meGYo?* z!e75D=R2Urw~N6liG+q!PnI7O%Kf6b?j~O2iQS}sy;Hko-^>t8j${~wZ0rx2zrYs{ zoghvA+C1VcVcea*75M)H3cdYVF>4R4ISar6R77JYsG#~HYA=GZq&q3^WTeT2$Kj@q z$$`$zTAJzY6B*}ex8LmyoGmg2!!m6+LjLyWZ#zd^&rTCUFmY2QnGL@qUfU1?8!uM= zFC!wDAq^r(Yg_?RV)4m$$MoD)`E0~kXZ9kZj{OSn;q5~UJd&QnI6qkJCO5ULK>cTE zJ77hxskfS1-6pb`kP#PzYTK`exSh{_f;E)xNn2awxIv`eq(m=$YWY+Uw+6%y=!^dv zJ2NsyDd|VvfpNaN{cBz1J1=7VT9=4exOv&*8IF>vDz@u-SDDaaCOVj}9Om^&*phFG z^G@Zq6dwW4Ffxrj4s!D*!$8tA+iOyCz(o!krC50A6T^2!H7G_?t(u z@*yU2V{vjIvv&@2J}yehW;m1P1o`F3FRU!DtEf$luxt|R^lhwSWkf`%CX`WJS*$g6 z2Ei?lDX)7ecr5n`cx<}d$p*(ORy7)lqNy$!uaE%Zd8 ze36>ABd{(KN`yKXt<06r(3)TY&I-*nD<0NRdgEfz2#u-g2{pkd*_J%})pD1l6F8Bo zE$OKEHsYaQC98L<4GD7Vd{3yHlw|ajE!i`YcQs>V1wIR-Hy?mYw!MmI#P={*!F4BF z?!!B>XKDG9B29ol<>?^+t#=ccGMdEx-xO*&vOxq-MJK+#Efet1MZvc=(~CM(A$U_& zMSHnLQrj|8vVj=qX#e6DOxK9V*fkYH+b>QGt42$E?B%QjB=`&qmHusz;`gI;g_(Tf z#M7W0aumnYm{N=W{Uhe*MTJ1)hR?ZWhuY4^sdEZFeBN*cB5Il=`c#7Aer}s2I2UF2 zHJWhV$Z>a|jkdG@*_sk>OAZ!`fUt_OK)ROZn$gS3X#X>=PCEN@G+|Y^`>**(^#$Q-rX8&(+!DlD9kO1S z*VkmPq}k@kN{X)OU_g$adLJm^PN1lbM1Se#DGxBiB7dXM>5tB>^@x#@tfdQwS|y)x z;)PKV7{OdT4`APS?)!q|bUy}Y#cBB3Q^~Lg!N|7LDXuSlPhcvs^A~<9{a@UnKPy4_ z!~R~k>My2z!~dF(jHbT#RQOs$qy5;=`B2*omp?oiKjZWTX0nsE;Zl4fqW)io3qQs4 z%JRln?fQ4htyAQ)hqM{>iUd`JQ^3s=Vt3oX>I#oOY~ znEAG;F|U9#r0W>dVLxbRy&sNPVzJ~MF6XAD@L*3F#h7NjapW~pCN4M}wmq2MKHG7< z(W%3x2$$oTqnym^3HI{Fx!F(U-J%1NV%)D8mAs}(A&$xSoJ!bQ;PmlV#@WBwmq)=3-sE|wQru6EUM zD9?Mv@;V3HhTB2r74O`$P8sL;8)c5DqF&^+(KF4E9V1#v>U%i_v?{klQf1W8Ho)cI z;R2B%%+Chc@pZ4Q0N)@go~nW&i0qw#g<(#-B=V2s$B+|s&NKZ)2v!ew!8(iXw&jnK zm{;>!w!F!W`j+!MAXe&IkcK)j$)KFDV$OAA?*|!+9S=M4!YS>PAEM-a!$%k58K(vn+8hNnP$VDDSqLceT@z>B-w>Me zdrQxRMjw%8xQc;sK;!Q@frJ2NOOo8xt17zZnkr{xYSFs3)lNJTC7-TYGQZOGF07p1 zRgS}Z?xAO#p1iIEyAyNy&@uPJvI!#-5kFLlwm2#CDAE7&c2*->M(RDbwk6?8-YM`7 zi(Y2#(X@%h8NDsor91nvKVs?x7{oqeWImKjmmROX9)q@0!e_4zF+#var}V$<7?vEl zM%)mQ(wgr>*Q=H+b>ziT5SS()rAIxN@BeO9w0!eXDJ{<-EIY=OXuK4$X35BR`-toJ z)5fjVHA;E?Uc-W%^8nipg9A|QOfSl}>f!v(4HGLs*Zn$GySJR;xk?Qjg{R`(cnGxR zhaAs4@#KkOcir-y03K4SIIG$NrL4mQ#r@_HtJW6>UgE$-&!Ai>_*D5XE(!MkGCAt- zGUX`mAq2k8DNw~g#&SQCpk+?7m5&n<^FxBXUjeUg1|b+~rB>zjqHLFcogS{qVuG!b;R3KmrTgSBf`=LkLT< z$(hTsZzfr6s}G~k#Q!BEH^e(EXA+0#15LrsFWRM*Fr4* z`u4tq@y8kwT8#_qLLU0SKR7D`UJ@eaO~XTO92b#Mu|4o1JpwKM>dv7@hu_NEkmsmy zp8BE)7bLm&$x3hf3*O;J7CQ-w=N_0WH6u~hTT}V zwiO+J(NtYq%q+Ddzt$ia?@2j+(Wc*B#usli=7OXDKU0dVh2$oO?MUM3cZO{8C8_Y4 zqE3|sfgf;GEA8`a_Iuq^EW3=k3EqZuK18RDKREFS|6%uRak@7asJYsZ#&H{6Jla)R zPyIM|$k}GKu7~gnFeiwfFxJJ^3_QiJv5T(n1C;E@Q)+cBHfi_25L4e>4J7HA1K?fv zk@^+0q@H_t_;b8gEgv(5mDBAc{(15rhKz2cjo+1G^zLQB z%pewuWXIVR3b*gqsAm7z_~mAMvAf?Nge#tnb~f9!c*kDOQEGQdUA9kLf3tT0F(8{0 z^X}wqb>fFAAVpE&oJ-zF8PDk%Q99 ziwJZvWP6L1*IfuK%*E|RT1Q74n4UXE^37^L>b6C)w96+A8N6!=N(xWJlT5 zBi{H>HSQ!cc{|w;m_zN}6jdka_S2EIgpAF5clPVfeaV(ST=ONP(QhINGlx|_B=;yR zp8tV5;f4sq$R!F+eX|8oha#U`j1YU^J z*08P85vVBCUFP0#ze23*6Nq9}Zc<}JU$U;mT2;O{V)R^O{(D@)0%DLtSAfZp*c!cj z_UxSl4S~kLo9reIH#d8kFg#OONVC9CNeLKAz-)}YS!$&OK9oC?wW`$HuVg!{3qNd0 z9*a$czwH>mqWN4rQ8x83z9}`XsMT-!&!K2Sb|#3A6tci#gvyiuTEaNKZ%G0O4`pgO zho(bC8oG4m+&=PMg{GD^H*iSL3^LHysr}wQu}eNMuVpU{PL`NBo5jU62G27FBf^MdwQ45*cp_q!rR#?MgTiZ+}^ zjNAvfX@Y;9n?;aTlq^!~-#J-o*KtUociap7iXHc3UjQTSuXQ+b?3e~qg%dT^EyI7r ze4*9N<0=dbK=y_-V&|KfSodD_Tj*rv%p%nm5xGKb&nxFof|m&Gllo|++cMcDCnnqu z5so`SnF!Q^d=()oD423=aU%Z(f}`Tt5kECmFP+dXjk!eJ7C@Y*GDM_%I%`)YX)U6q z=JGIml&^mpMnf1jOr6ERg2hitFDK%ehq$DyQlhAUIaE=S50)KXAo%bivEZs-)&3WZ z_!ikb;=LQDDKD$d{;GrFQQ~6PlvVZr66Mt&B)%*z0Y?@GK%lnknDfZ^W~9v`yHv zw*zLjkB`nAX$YJ7heV{E-U{c?K6~|aSOT$J3^{|NFGRQ8D2Z6yzSkVRBIp<$xHTfYSdW)RTO5&G0S2wy`!ao)nm78>K( zCsPP>+0NhbOzD9UdUaiSzC)3-eX5|0zx$HT6eC{Clxa( zz^hD=aXVvr)_l`VgBB#!%~a$95<0}SQ2y5!z?{&oV9<_;M9Q=H~Ge*o`#XF3^78Xgxe(Bq~-2EJm!TdHlBzGQkoB z|D}O)QSPldx3z&kQ0|Q_aj&GGj5esjILMOJ>O!1qF2)vymd$Z`be_AMbA&jTbzDwab~76JtLi2zxg3*)jVVQc^^C8_wU+yf@bINZ3Rv zND;N9A5U}%?`U6zXYrm|Us4wrwa^6L_hBlA8qNdbvq#{KKzLHS`#oi+YR}<60_ZEd=_TGVmMsO3Mf0XPMrUSUZ%*ERSa&`A~$zvi9#aV6paMmASJ7VC-VAA)X@Qg1}nJFu3UUEk2*sT7o z^}$PfU^<}l&dB{9O10GbsjnAft`DWT-3QW~z5#mFbBJhy;!8jN8%7A;JiZnc#>#(0 zSkR*D32Qn};ED2~=&xWIZ6;*i9pXf`2{HAkKz|K#kiJ9L=B<;m79~H#7&>N6fzBE% zN_+t43JOpuruFB2xn&{}|60t~`}RPmCH}OUDMU}ZMe__5+P!jE`A1@4`N;@LTQB<5 z%kvcmygJgKf>_9YK2x)ba0#w<^Szk(7>zI8pDg4ptmlyn9xKF(3~njuRH0UO)5TcXlOPt3k4vt(LSLT1lz6s1enT*sT*axt){hMIIE$6Qi`af!Hf@5+7d&r zatzbmdrKDKN=nT_nPS_OeZjA5*h83yWE3?4n|;N4QOUDf{InVeQrlwa?P>a~z>$f8 z{Y>2U744YutzMnnO;LTDtS^&o>#8e3<< zPcAXrmG8^Hw*`OAgt@1++D#U@PK;wC-+mDmfIXXY*z;`fYC~0y6?r9_+A!YoMMQ6XuwjztYVnQK5O$RY?7Y{=)L_(n2=UF&}U^FT(lc35K}GcRD9Go=(FkS2>Nq$Jd^4w0i@652xLQ{zI(v%1+te?J%M7LHpQsk>zaLY zv#=7Sz~?U--Fh*tql;!U;Y#Ek5LWVMS7SH38qkL2jRP2j@e%q5T-+HH{ZkuzeJqwI z5$KG^q=dUYK)F22IBO$yIC3w`k=iRgm+_&r5LeC=W#^x-%le}wHY@v^VIs2PKqe!f zV?vn_TngWl#W!@lbaor_G^~u#x3zKBNvl)y@?6<>O+@%8w<=6kjnDuUcroEe2etm$ z<}%Dc?f)cv^X#aDP)564=a zvsTVnX{jGjxlYZNEkNkFWcBf)NfbXPcBfJjQ3~;q76O+6iumg^=#7%B4r5d|V9{Iv z=cYwS2^Ji7NC0({VcO#f$RU_uLkU{thcU@H<^K;a~WJSu&1BQ9| z(PNu|(!iO$uaJOGi)ejwV(Ly&U*R;>my%3AXKnr(U&sC0_W7qd=uHpOF-bV+?jc5e z(lmEv4&wAbz`QxJS1|blmFiq-Zmfaw+YlQd+}=`}3;@M>IayyoV*<)}p(i2<44hxT z%zqj=^SWKO8)8~Y2nPr0_>iYsh&Q?3`x;w6b!}Ch+uVY%eVl2)VeefzT1;XtfX4fC z96xKAZO1R(cg_5xTl5(9a(w1m7aA1kc~)J5BlNh&Zy62f%xtminoJkm-Jp^ zq#IvLt~%PGRR_wl_Hk@zu5fDr6L~5gdo}&-<9km5pF@rlwGen~o#S7nDiYCm_+Ye0=1KDn>5km$GsBJWj*+vo%2bCMIlvm%Fp);ZPdW zD<^=F#W~gsDYVD=q<8a%7AlF#IleIH0Lc0Wk59Ug33AyF>aY}iJHx3$k@J&c}Mt2}5!-l~DTGJ}JI z6F>AzSB4~AJ8G#H6CqR)*pNMo{Uu*VH+64xB!jw;usVThw9n{6Fh+q--_lwT!rA!` z2$UO;?=BLDB?-?X-&V<&Vwbb>`Z_EjP;@`4v&Z=ffhiqvs287>T}^2J>7?M#x7Nya zyuuy8JK_~DN_$+)L$6$L{tCo%_$uhzl~p8TYU7{Ijgz#gS%HEEYl0|WK3g$n3$rlQ zdmpQth6#yRI85|enQm|njt+J2GYDGb|AMd=R$ohBgNgNgOUBjBZDO2r`JAN`IQp<3 z!9622_8PP~ALsc3Hn6Pl0e+kM+tv|2fGkqtE?a>?a3fe8aj$>L2V1hN7}Sd$P#}ku zbbj}E9tp(^WX%Ikmat9)&80NfCPao`(l6B{HH!!A==F8^ZA-xR5ZRkv+F@u7fZLO> zNuEh6T{~>|7kPWcWR;JpxgjkzpM~7Bjt|*B!oxavxV`&X5?4Rmi(6Ma7wETu^qUhB zL$^+d?@k!M@;!T5f-8BuI7dcS23A)&O{*!f?D^0fVZzZmX3k!1_uAZ_f+vs%kAjZ6 zVj7dHOy?PD?-cZ0F?-9EuMd}3)NR;~CIL{nTjf@FcwU}+VUP%!&)u5*j!x{sM0Sg^ z=+f?CZ*5rC-gVnM5#N_Prz~dcO6!{#HpG)*e@Rsr5nb5WQOxdgZ>B$nA0=!9)>P^0 z#lV6kgH>p_Dt*p%a2F_k9lo4CtAoPaQfi>fTH?s#C&NG{Rb+17ku=#k*K$Cn_p}j9 z`BFGiY&vSFiVpFMZx-;!GDPgjb>b|&**Xf=&WW6R3*Emys7zD~SpfL(WJ9NxA)d=P zh5q>q#Epz!^rLTnky=tHQBI^+DbLQYPp`{p{rC5>ClLz+zSVk>PQQnbx4H})m(IOB z4>vDuNgAb;RO(O24ZAlysFP%L%}x~fG7CR~-26%8fFW7Fsp&mT6m;6zmvHRPv!a5c zdrs^AO#@syciEuu2slt}AYk9fg<2TO=cvz!Z3})&%Vut>^6lAMY_3JqvLTt#LmoF{ ziDkYP*2H6~^EX5kvGqr@vo}&u1Kptnc}9>Q5kdAKe-<2mY)dX-ur_-CE@NL(bJ|mzm=99#;=;9G?LO z|3*j}_SHIk`K#sGmkZ2y=$|-R%wnOR>z+1hzj(Q2ZG^|rb){V$ zmPbrhXZ4yw;l;e#0E}x;Z$4E1a!^WAvqks*7Bs;JmHDsKxQ>jmho=rr(9+(!!K$vh zFfe4(^_6>ZEo1|X!nrB^6W|RV+DOa*0^^Vui#e$B&8Brb`u>*E(o(d;6kEMEPIy#v zw8SBFR}y>&)!fT0GEX}OzQCxs_xE~n+T(qs4}SILAS3^vYhtIM^H9Ai93IuIX)^PG5C04y2{^nvu#W*=nPk(DSu{TiPxBB==Hb5ky1t@!NTx#v5 z4XbB9Z}E7G>g`^tx8zq}vblLWK25Y9K8^h4DW6ixlqN^*KISy&+qYt6#q76VS8nEC z-xSsWJFbgcd*wojV?QJ)oLoy|^Bcy;`?+K?{a<4d-k44)NTr{|BJr94#P ztK%1uCl)#!oSD-QkW3a|-%A;?Rih`8yN@V;Z!s<5Un$gs{;gbwD0a&|>rnV6tjHL0 z!wx^49&}&1p9}t7=BYAzGk9>w;HF!KIR$iH)bnt@e7j}X>bRpwAr4^|711^v6{|l} z`xJc3Z>xork?Kr|^)-JAp2cG7==CNcr@QE#;9_QsH!JH?SbLp+2*;sqN?@{8L^ah> zm}*wGUOl)P_L`GZ3X~y$kWv;d2F;TuacX>;433Puj7DNQZq&@kA{`ZbfgzvEI>m01lV zP--LADrtFFXJ;2ITRGd=k!&f(TYehf=V%eyTqJ%yIKJb_%n;82qujVhUur&ajhC@d zs>P}W%!09q(Wj{Kilq;S^4Ep*`t`PdKQX)4-1HeI^ID$Jy3xPS`cALKO*ufv9IN5w zI_pbLJUKoePdf$Qsdg9D;=s0NZq6%++2MAd&=m1|yv+0H_`xJCW19Oe1>nAbg|_H571^hz12I6Mx~?+L_H;j(Y$s3cerJnWX3lHcR7- zHqa9Wu1)*0BJ+7JhQoadZ-*xwIZtbo`5VkIMM+GIJ`xfw`?$O@f>~IyD6VeJQ!fRi zNaALU0d$imY}}w}wss`nj`{!idJk}_`~QFZoa4kHaqP{p_a>CGXJr(X8B!W{5+xx! ziXxPdk&!}+N;+mzX;E4_RzoV9D%J1tc7Hzq@Avcley{&^x$f(_@4GnO@7H)fAM2U! z+I=el$DVn7;fSAmvV+r-_fK6_`&I%4Eo*UvY3yD+z={Dou79v41(pDoa9l?>5p}qj zL8PpL4d=9&NoM#yer6t;X-^rZ-7Z-sp}5A?*gXoMqT=w~u^Z9D2E&5C|2{r$uvJd4 zt77*U7{*n@-JuG8B9x&)0&#m#KALmpM2jm~SoSnp+AA&`?R7MdO<#qs2HW&)O*IFC zK8e$Bcln#lBCnzOoK91;2mNf4zQh@$Nbb5tyPO-+jM++;1`;igF!ymXss_Bmt_>oa z6;`Ko_3)JaR+7(0|2%U&eI7aq7UzbVE*@Ih+q`0`Hw{NytB@7ff`Zeg^LD3FbdawFw4w=N@CjmO0$4px zRqWo(;E1{dD}I1;B@5c$LFzwn73&R!HFUmIg@~FH+eSn8?8@CFnWJ;udHFZr6^?&5 zP|%emz2yp-y&#TSbf5OquiX86i5&WPFPj%PQQfz*(s!}r28njhh&+;zNRV6)t5&43 z=g~Sjnnt*IK|k0tgK+%+$jUN8@20YX^zv8kKD+cZQZxFme;{Rj0%c26FodPMV8R-} zSNh>vmEvu$q8I&Se0Brvev(FG{;5f{bxWk06HZ!O2z)9oWE13eF7dBj*D#E&&9_E7 zuVckI%M>n{9PO2*Y7gjhGCpTS9o$%1r~jRFQ7Msx&8(H6V*)RNc{(TK)-POe3~K~O z5N_&#t;WkZt&>Pej{#Lv_v@(%W%~QZPUPdF4I7W5JaS~_IMdU|m1A_Uha}$CnG2u2mwczaJ>9${j zAJsnr{=rgu^}|-P!E#sEjKxK$4La3bR!G(6RnSlo?}R7Va#PF>_}d*I zN_4HT^|>6mU`NGt91%b)PYqH`?BxT9)(zSbzOt(!&lc;h26+M3w+`w={8!u1;H$&g z_0!;SgdaE;>2;so<`T+}zy-*(fXE!spNrl#H8k^b^RmBo%Qyzy?i8Dxcj0|?*=RUs z45-Y*HD22$xG9Uo4d|J2%gXf)P94RzK1%E>{(4CQ6X~S_pkm)xTu3<10a9_DBNZQ{ z$~lvGYY;CriFmol2a(XaWl+0!GHY5Os%Y;>{Vuiw_G)=j6EFBPa+)^G(@?RojTqLR<-}ThY+i zGwi8&r^=L{K;V*DZHzm8TkRR<^HF@=A{!X93$zph?2Qix16}6xEm!eRc-8byM`isC zFg~lF@LlO@^qxI!fBCQMz1lU(#>~-1z|4dzl_u0G>xF|3e#3Hzck^dr7wza#E{#hv zH%p_J9b3ZJ3kwx@-#yy(TlE^15BpY&@YD2g2qtkCKSXm!Eg`V8z|G%L2;D$cG`%C- zn!hoN^r=ip+m|qv+Bqq%Oq$yQ6x_(Mc50v)$0N5vKf_wwF`4Fq z1KSr4J}(A%4PA*PLGN{|@8WiE^lQi3ACXmNbON|;JjO&5K=*yo9Jb-Dbkc=;D#o;D z#e-{}g~km%Kfmb~d(+IqgE79wN<&TyxTeA>vxax`)gR*)YCnl}3Fm8+QVy<6`H4q( z2`N93{AeS085owy5%!ChR1<-vS2>sZ##|mP@Z4;3p>_#T?J*RnZ^ZtMuK7`2{9Md0C8ZOK?D>cJDJOP*~*Kw%}i{tkW-lYz z5r;3?o~(!}Jm%c7x;~1Rf)^5~A?SLD^dVP8v8LDQRyS13ziJ-K?L2JGEa zM2i(;!r(L}h;$y0Ehx*M&f{wL-h(I$DA^NI;Gx}((Tq!g5d;>S#7vjn)4=7}E1Y&A z!m0PwhwlM>dOYE(6szkuMzV==E}R3Jot#3EuWg%ZHcuC5YDC-a4t;xukiWa_v#uP1vttXj37v#gHsp!B3|onJ!Kt6t&9=<{DK`Eo@yXAQ3s_w4;Lup^A+FWzZRK^sFi z-*gi1!J=@TZ|s7IN9Y1wtsC!UCayPgYsyL44t{^ZRapvrh?nhRG;4+V%c5dBouE|WCo8t$i76*%JMnwAf`NAfr-dC3? z9Mj`ozdi!Cu9aIg`DV;LDE7OEMo3m#-aX$(iOUwy2|hsIX)6M)Fy%ueP9J)eEa@0) z0(ySF$n@(gLw=enS@Z>N(rAdSWf%1Ahv8|_B@@YHH?8BVcq(jBeKHJ+{RzhxsHWqp z<`Gg3WiYFh`?gDjwKf4{d9`@a^@T1$!djiUNJsf6P0H%nvZO}#oJ*k&$Re9Mj;}s5 zfQN`@Yq+#zg_VOnQy8l_=)k7(9GpjIsR(SphtB2xQ+P8xN4?22-eMHm+2j{zaGX5g zSMK}6BdU{d)Of=>RTKiak~)2A{Ed+2vmq>KA3tq55jNP@caqR3ja_-BSmSebvE}Th zvX>aVlz#mPw!+S5MF3@Kf;$VbXR%Vzi;w)4my}mk~4Xc;41QEA0gVwo$@85EUNUkGz`MdjVusQH~a32TUIH81~Ei9*?;MBLeog}ei* zE@cT(IrnXsss(H(h)|~fI0T3Q!^2dP?9~;F5UGz8nJ2+#3bu8KAb9HL3t6FnB$!6e z9*fCKLyd?(t@rXtj;s~yzJtbczI5cnJGvt%g{Oz-c{-Db7pzEL;y5~Mz_av7!XVY! z!u|HS;3s+?oKJ7v5l1K-vLlw9QG)LLH|HE~OjN=AyX91u-+J)&`uO7e(^!VMXxve6 zbFjwQeS=jKhPlIK1gpssF|lK-H`FK_a5t=kH_ zT*t~vx*$vB(C)qmABXO5scF@t=12+;H_$o6+`yQf%<>k5=)im@CC<`iZ;FZx0Ra+1 z|E{+Lt=^e7!+*$u;YM|_WdH+0yDm&&p4^BQnb4X?FkLb#f45F&3aJib11wCle*d*9 z>6=_5)JYybfgrx)!UT&Z_Wb_(*!AZr5FY_KXujIpE(2)o&APdV*i6%Xi#FMIB>?AI z$m*2@aAO{Z((vuIwt<7x3vvnHvr)gKIL0*kdn`F~^wFh0hO8T!CMe1>`${Y`;HZ|n z9|LlKb~fZ~9gIA;mAmzLj+Vb-weiRNnKOAl+T~-fepAInmOxH!U1P__ee4@Z1Zd%BhmZ0kVo*h&(>RwT9fr6EEdzpF z`~3RuM{BMD4@HXE+%%iyC{g4kC&j|Y;-GR$NxsgFf31~AVC5vlj)?p{c{0`QW(jqA z7YnU_j?~u_Fl_$Lig`!G8;o{;o4O3C6re-S5rj^-Ukdj4A!lupp_+4qR>legI$DN4 z;1e++4+oo!hkCvjjmakY<>VmICDH*{zyVB1qZeTpZopmIGk}(8tl{F zvs*F2yBO}KP%5mmAL zn}UuXlQdF|BGGZ+&!uX9#}$k28dqjaJ-v9^egWxej0j~4=cS8hC|C7TvRG|qRufRO zK_8z<_y)jspqn7crGceFwFg26Tw1aRls+9dB(T%w(rEsZ=ocywf<{8MWib~b?(m4Q|&sGWT9q(3S9J+chZIk2|k9E(s zeS#zWz+p=7Hax~P&nEqO+g&#ZxvzEI?w)=i?u1SXpp}Qx57szJaU#6vhI^G0;z99p zG?H(f#JZtalEg*{n20+cV_^krx#vILZGjf#BO%uwz?<@E>cCka37&}XTY5mCJ*~@7 z@&p79Ipm(bRqkP|fya{Yl%g^zcO_eT*%BNyBm82;vnTyc5K_m{ZR}$MVwW=fIG2es@r*b@2D}8$CU@;l6kdmG^ zHBftmE_CS^DxD{m!Hbg7hrBA-5Kpx1*lkt>3+gXB-KL!&_-9H{(y z=d<=-dvx%@e=y-p4RTjixtrYX*S889a7=6t+i1q0Za_BnU~qG}ufbiT$$r|aSZ`kJy+xjmpptW z?-_=h1`?V04bdm5>{WNU&MS@D)0HN+oax6ou)4{*4DZs{aG%0hq{m2QJ`j0PKu)0c-=kiF7?7^eP zn~ZD-b;_f)>2|M;Hx)Q+Yc{ z5^uM!ZW$MJmY{$JDR{Z=tz`9`bF2CI8`E(`3MWo9V(H{6PQ#T36~9;ogL~E5HQ7^T zt))~sCDJYJlF?~t3HKk?4G{4J$dqqW+dgZXwJJkBW?VRuv2y@w`MjJAOXK|{{^Piv zo!%_*VGZZ=-s`A#X-vUU~&aM%Oa^H@UW$rRWL>JJy)&hH<5?pGtrvzNXOy2$iU z|Eo)=tc-+hw{kd+8ZVKrIj`lELzN~1W>QbtuUE?MaB}h9B!zqB`o6z^b`8hc(K4=& zIaST8c!`%R_-!xJwo+ON zn(7cMkHAfasOy=sLtQ3BC9#eIB{@PIp8@%~e;yhrOs!ABQJP4dlgA0GTAB8&X=7T8 zlf2p?9CxNU8J)hu4p`)>d4>!j!qd0o^D3NO8SC$>7V#T zCCgi`4a{Fo+uh?T62@UY^M&gZIMMs61yvVC5>Z`tn(^q#SE;I--Jfz`A;2Ek{IobbdwWS6(3DDt$eYh=AUU31M)9YX+shir1%>$cTy^Z5uEj`>@DcsAHfuM%3VBuxJ4kl25GNS~Zt>zc# zmvd#OVV*du;!24x2=DANhW7FC*%vrrKoyA;`NxKB;q1-w0R<+cF5ww#e6@#4|EBU% zh{&yj4oX~o`>sJ3RwRN%QES_nE{(d3=ALU8 zRuST?At(`{wTXII@^gnsdR-|gDI;_F>wMqdJK6bHYdW2ZxH#txD*aw?fTF`VFmrD6 zoy;z00i~7cko_?>vqWX(;o!K4J8a`mz|RbR&A`ZTl<%UL@P`+2Qe{}Qy>E<^{r-&H zZNznd^*Xp&Tl6X_(!db>lo5NV)6lffmc_YkYK1 zHV0GW%C?TysmGJ4k$E(r7$+EnX4`aTKm=H?F#Niv4&VXJBgN~2O9(i}7+v`l-``Yg z8*<4e!Z~-!F`tXVTJV{e6>RBXY=|X(j*y~6Z&eW`O2qPb>#FlOJ`IA0?q$$w&Kptz zv2azdsQfT|ec$!Mk&BOgpIzReTv9XEm9gas@2;zprpEeRS?J_@IC6&np> zpuR#};9>wa8FXZHLeI~dLKs1jMZhx5a^Y0H=67;`fn?mo7uUnbRLn^>5-ujzv<`ds z1;WX4ISHA>CyH=a5#Th2LCT-#Xx}Wy9G~20-9;)jJN*DAQ*3zAZ)>i+Hk+9hwABFW zUV}z~hzmIa=0ZHlC7imLi6ttk^XPz9FLn7Wi%nOaD-{8oM!n~EEWvyF%6U(;`j5D)C4V?qBFeJ# zbq}!!>}1udYgR2gqdU4 z2yC|_$*g`*I|-R+-h#M`Yuxswpf_4zw>7zz%N^tUQpTX~^VoN%*1wU)&}1Lx^)tdLjyG+MY#{!JD2M`{WP$HUNn4 zyxWAD$?_po9(ii+lPJvxd?VCE4JZ#xaBsz09M2OVR5Aah)IrURbkq*^lvM0db+?+F z{->*r-f^-tqo4ygAf!$Zpy^MLwR^~Ji|DI>o~*6E=ZalrwA{q5I8;+-VIOTo|7FE+0P}_EL88hUVLo2?FUNN zM=LvAal21BV%uLmD>uNt$->lDe(cuc)?sCwI zzP=?~R?SLq>Ng*Lvni^ zd!9OnJfzlGw4G|J(d!r{qW}5}mxl{^=bfAfw;o+_nWgg+zT#oCebAJo?@kq^Ef-%B zrP4Yc`MO^UVWK;0X!fsL&_|_LCZ0T;{CU!XM6RQ)NzsBm&PJg^+R=5Pd?Jd|PH&`v zTj$;(-Fg9q)|_Iyy46M_O!9K3`iH}%8I>o4Kl1#}3s#nV15I0D^U*Fnp}JHL)sS+_G>fQEGO z2b==>`WJeg_QJzE-mD<$W)XMCPb?;`LBB79gKv)kE{aA^Oi0=pEYF1^CO<|-qm~T3 z+(j8$G3%7{i&eYoweiazGd%qFiU|0kL2IWxb~(f9Is7M#AoM0?KgI>VE;yB3#|Np- z8emtLP?ykfIY$%02{CTNxbbiQsW`^WQ#PhH3J^YsR{>&aES<+OM{!zbUe`f^OiY#R zIkwu6Zr;%X9Sef3eeM~DpHfnT@5!0={4 zIrW2oVZms9&)cMq-D~Q>}!lpm;?CR;t_lOGx!$vsYGw!H8IxNfAE}@aWMc;+b z3|2*c;H+F(6p6xO`mn&qJ&;xW-wSdGFZ zvAZOj$K494Zj6NL08tkPXWP?*qPms~a#i+^Qe9%2K?M;L8*#^|TJQaQ$LNsM(nodf z>Pxy*+oEL!d0%-#XiCx;qj#hp4G}YV7x(z*+iB~F-rbq>GGG%|2{BBx z(kZbtReqbbAJPbrfB|y#(mhpN$9VldsHC8^GkkEkZar}*BLQm^ef`nf?G`%~5_TUW z)CR{!s$A>WH$@E`Uh>17#1ldD&|F=a-oo)w&{dAT`|4iNBp<1jgc=z`14Hw9hG}jm zsDJDddRqQAvH0-P!fCfb=wql~;X=imNNA8Ob}%`xij%>hz1;9J_Vo!uxg&(o;)4N! zpxK*iZ3UP1Qm4l?4u3#MXcYAR79}ShJ}DqM24!-*LUnf?YSC}*$%i>|NW-KMDzG4IS8R0kET{>d@KqPdkyCl51EW zPRjIEf14a!5rqzSCw|bfs8@<<0_!hq;nVB7gCZ#0JdQ5u&U%@*IK(sU$jP6}`EvRr z!fOO;)%IRFaYD+T@ZTc>NsLU!O@6oXtR$(IN$3F)jvG)a1rT`9nXy|7iFqq_n* zopp2L3932CWGu(Tsm!lZt(i&~obHLt#xl@na*-8OltvL$Irt#>&Sw{P_XE{^NA%FaSRa5Ni`pHMFn&V3(r73 z1GBhAHD6hqFDGjZFxy^u$oZB*k>`uhlH|pEuH{MU#@+A+2jPP;dY#qa>`P zvBdz#XbNQ0chE32!#{(|=mahc9B|nc_)Ze4IW`L)^g!49JSP7Je430LF`fuyYw%SM zDZkwNUCIfzb{lI27DpoC(f>ZYXVms+#57yk!%(rM(Nhra@SJhYEVwLKeCg+_uXoV% z0r&AgE7V+!1&SJH_SHt%BBI0-;X+9-zdc<68NtPe1J!M)>$ULtc91&|%93c$XT104 zg%%kx-Q`e~!e&3Vvq_TgsPWKP_%R43oXvi_5k5C6dHiLqYX$pUA^oF43j@Rdp*7Xk-YU>Y$}eZpkNX22F5&F8y|s5$-|mmf^5(X}i@4DJ`e-b)GU z{vatck@x=D&AyO+tYhDppm%}A? z0)1HW##ZyEmrxO@ble|Wo6@zFAhQEvy}Q;jir;8MVfN=06QT(!9TpU zm`HejuGZ>z&WrGWMs#6mCrkd)VS`ZIuwSa~>o9%Rjh_x*tgg)grOHuC#_fg)IJYC>gMHeFT32Ps>eTmnt6 zrS*)@)kp7q<|@X8a`!H6Sv8h&&&{~&WK`De!h_;^>ADvhj^&wNpwiQ@9NLj)wkDmd z{2c9CUsR3KrX9cPnB&)`PB9V*2e0P^TH@{AG|+?_8u(Eh3h}`n#*X9Vp&uV4BQ=)H zN?oV)JrjLGrVN zmqyoPdzXZbBON-Q^t=p($#zpt#M+2H+Bg1YW=T?(_UETZq=FG{k|K9l?9ySJt7B@# zYjZBCmWSfV(5DjP%eM`xQOW7FEd#l*$SOS$JeDENoDDBg!|+(^R=u`WtbpxIo zSP0}kbS^mvHnLpTuUcYhv@v*7r|&R0VS32)@i1roLr>U5nM*t(iGdyjZ4jXrExNI&6pM|7+)Xu6v7BR6zy)ki)I zC@N5MM82YOnN5}7eB*%1xC(5`=Q;e_W=9X8sXxiKTqa`bq_TGHO48Zz>5wm1X>7~J z+fmHWil zC3GNZ6HagPaSp9t3GNHgj&WXF9A!hSNl)73#5Bz_{_$>CPkxsI61Uc$!G-(N1RY!DU38&?ovgFF`J69- z(>>55@^ZRor2T-R_{_=!!H=-kZEQ_0Fd|=L2E0wr7i^BJ&%4*~(`aN<${9>^!1?_r z+xD*U->2r}q$W7>Yc*1Q{3dPA zx^KJ-0TnpRUp;0Py+7}M&$G4uw#3%er>hmr(KFtrty!Ph#KUCC&tSXnk6vM$=wR+j z(dnBVaZUQDf}NCBW$SuIWo;L=?Z>O**s69>Qro$l9a`t?KFr~e%?yP}6E(~)RBL`( z8EXY__kkFvj6=xfbBew~lPL2l$_u zF8@eLL>vFcef*8`XmO@mw&fL3-4s}-b$Jf51JNnpYTJGBi;@7v>D}Pz*M^2|^-`(9 zKV#<1pBn#7GHlK0e5b%FAyK%0)A-82AK7#4TiDlUqG{e-pz%*&Y8x7fsY8k4$%Ulo z`_Ol~`szcaMqlvgFAS;FSP6C>NzvRud;J@vq;%F$`CdfS;X$5tF2~6{uNyv2XdF*1 zv}vhrLqA%pnagYsS;nC{2T&TS zP+8|#KW&Q6o6cj0EU&!kKJ~9o;urt)$h+FT`IY0SNi7(D!=fD3b?81BwQJ~=qk?4R zgh{^*@jt45_&4d!o_JuY^^My&5gk9#<-JO)BEHL>uU|U#dD{DHLOqapzGtiB)({H@ z06F`b%*;ibD^qGepqdiiDs%fwkjx-0eY6fLzs0JeJ9>(!gd}`Bp*Pt5L&X$BH*r&) z1xOq2=HGb8QvLM-m0ryka5-=I5W{IwhfdV1G}J$cpQb@J?Sf17@p$2Fu^|4uYgc4j z{y-1&-C=WHV(LfYfR4KY{*Gx%AWp{{^oc?_aF33e@t!(cQn$OS>*K&Sp0;kckRK|% zoa+hfO}!=(<8&m+ZsetVzSRd^1V?w!_<*ccD%0oX4rgEYBc!!rxrPYe_~LG=P>LUL zu3=Y`89XQE$ffc-tgp|&1>$nfrxa^_RJ$;?X{I?8?+S#8!M=s?H8oJ_4{ll`F6NsWlgs5TI@N+8} z-_^w=?pjE5W~_QaF(4AW?~nw{#nc>9$LTv=5)VX0%wJ7%Z(43E)No+BndisXx9w;8 zKW$aL2`5e5SA(?uK>+tJR*pzkxp-g1{p9- z&8g>z9zk)uM>MS@LYg)7{jk_Z$yBCCgNiwf;pu-$n2wKERmv&Q9VvWbW9Z`BAiVxOSYcVq_WEF^F-Zwi!~`sR(I&RvO}=}7;&utv?MGVB{VyNsDNLnSpPPrB(^!m18RXP ziYq;y2qO99&Jm~rrZrx`{`g}2?%T27Uqw?>Yp-+%ytB;UO@@ zxGnZtc%g)YxjyB-;ls2^f@>}%wRxAT6c!r+G6g`30{K$#Na89^)`58VQRiZUnFYOw zW-Gv$F`~CNg8gf;qIr{{k_(zb(4ggQbqBuQnQ+@k>37vE7j-udy<4PzV`a9_>&Eg^ zNBI+GFof$pc8>KY$GMp3p)VRY2^%8r@m|5TpG=oGu&0sQHo;I62gAD7!~6RE3uw*` zu-Cax3!bS8ApCWVEkrC%9&E3=FThc>!ibBRq12FQr$7Xy6sGQ}xH-f3C9QHr$IsM) zYmGOLwdO)Am`tU=9|R~|K?*N`Dx|F`pmm@P0rdL-H_HL*X9m zV}P%S_~3qvyT%!o$M;W{M8xc91RM-|>$VDcp(go6G*c%9+2Vn{0|ruCKata~r=fAX zl9*a#(Kv}l@yE#a!-dCQG{ym8G{sK=KUtalCF_Vvj5$Rdu$9wTHw6CBj&_KygWSvX zS=nm;oXrykYxxtJHv*#TxJ%k+6myO9okYoIdB;SVKem?$nh~HyG$gMrvNX=EU>vLF zgl~6?B7?1FlNdVA`9K4Nq!0qPOI^HNy=Cmnd{bp>73=kVgPuT!eE8ty->u&emr-M7 zF1W7yyLy}`XKf);e+N%0Am@pkTI+e^D70W}Xw2Gi&@0w=R<+%DxbdcIY$MDK=T=MT zwf!#r{LJjhOD;hbrgr-7dcBJ}{N{Q0IAr3Wx2N1skI{8IsT<8|I>QMrm5BLqAxo=( zKU%!R@~NjbSA$U^F3$yRk+V<04lXZ{UoD*43egn;nB>ZNf}0m6qLMqRy5<$$G9A{< zCp!I4Z6f>NCuPZyZ)HP0&6JQ3>wZ<5*NQpA7&JFsZkZUJuUHwpoNwsjb`eR3##0bq zbgX%Y2$so*@4=d&IZtSSix3zvUq?k0=LXH82)-A6+|B~7R;d%>&p-KB{G`Gueo&Ip z*axN+cEUX)3sMK!FCCwLAKWBGZWXX};7HGAQ#gG7#GD|JA*M_@DlrB+;ZeEG!qUKi zAmFt4vY{~wPeWyqNwd9AHXYWoP2DoKUOtCmkp%Nlr;y;*xHw3vzCH0Kx8=m>#c$;lu0sW_LK6Y^zV0Ejd^gz3ey(x}NbkhF5q)$s-+Zj=!&)Np57p1(CUCdGI!gT_js zOJ4Bc*Q9Hhuyf8jGias8?Z38BR!B-u&ZV_4MeB$kon%AOc1_mRS}(vJCS9q!=~FUm z8Q)?s7gAFGB=>PsQ&p{y~qHwPmgU#lD@BjTzfV%&rjQ_I;|MQ=T3?jnJ zz6^Tw|Kq#Q|2!2><_Hc@aMegTl3bTg&6CT4@C65rZT4h$=AA|L19h&V4$N`(qrZvl zzW?zB4_Bu6m|w;_+{4U^6A21*It3*5XHy=d#-Qb`;+_;=qwp==V^jb0FM`X|=2?Dn zb;vv|Fl5LiaRZ?IpP=}$`bb<{3 z4-=g|f{`ZPO+9B4P0j(@Y;&R}e~)da-uQFd0~0E)t5S=1ur1xT4m{rxi6L8YSGRvX zrRn^H$sXhNqnP?Sd3x=XGli@SxNetOR>X5Ac>|5-jSTtk;>LYD-bRU7_3!jt^*d@t zJhFdL{mb%qmAQ`>)gL)WNU-K#j5ke1{M4kJqVvB#Gh>!2Iw%qMs`$PFS^xOMl`=;j zHc4<&kCKdn{9+>w{`mmU_*`D-_eYzn>de0$C@n-I&%QkN)J^`cW&h>GkSouBX1ssN z{^-ztuWZqEA5b4yIWYshO^m{O*JV;#6#XLEu3mdQX)8WfuXKCnpPwA;oFM!9(gsT( z!D;RBVu|$yUz@K<{|WJXb~?m~+>?KNtjqVLEX(k0v4pX~!cLDneA8DF?<^8v`65|M z$At%ir2l-w%lZ7?oONy)GAX3q4tCPto207$T8`{s{^X9sUE|BPrt?SNe_M5k?(y@E zCGuiNM`G5SK^c>PypS*SF!gfX02I991Uzc&- z#-4e7k>p6Yu(w}f-Cq~X%P4S5#YBjoeSe#ZcfW=~O^y$7vR$>JmK8=u;p3Ngu-TQ! z+Wn7j%^-s)2`c`p{s*Q7MAi5G0w%A0DAw;V;LXCo7%23P>}v8dr-ifGNz3Eg{FAcw zAD=K>9MyzLNOvs~ApZ9j#eV^Mj$NwvrFeeK9@c{#@eXo>dUHPa>Sx`iJ@3ptnYQP0 zqWnK8d&D;}pXStAX*he^1pjURds4GVR{ipa4ikS*w1}@IP2P(4dnB73KO<~DJ0F-6 zi->`nF4{qiqWnL<(uhX-fia+CtTgZYsar*6ppmOFtiG}?1%p65BdMjC#EH|b?51! z<7q^-kF82r8YfCi0sSnUFf+rPBoOm$&8V4~D0;wTQ2{fwS-907q3oP`m@ri@M(+NuW7N3$S2j^Q z|Hs!qNkLKBpphFOZ;VHeh7N{TdlU*(FdU!31%PTZgr|RGX-kL+rVb(+1v3ubNAAQ= zE>{GZ50~#22-s)Q00?wy4bkNA}pQVX5~qsT!TD=x*N`{WY~h13Lx`GZ=& z<77jlcPmG>6Zl{D`;kKc5sFZ-6A{vZ83N7bf55JGO7DWYb9#TH%|=S*NB>a2?C;X< zL|*KFe06i)(|08anCt2?8 ze;>Hgu?VKw-?{Usl_5g0<|w!tQzt|IO1!1`s|UqBRxvYUcJxXzFK}|3H1Fme+8d|- z4gc@Ot_2hSUh;IpM4deSV&$R@`s*KMiC{530$746BvL%vx7cmIgx@3{Cj;0N!52WC zbU_XvI~3Zv!I%nDY2$DfGU&#)B3ud)Esv2;iLcBaO!9(G<~rz46gwiCjW>w#X`xYo z^a_kJ_Y+175StIyB6@oi!sHrM(+$_Yc_IVST4rbrlYx1Tj3Q;&`W6ZH zYku{`d*SzTBz`a2zkaVMwkLVF4&*6+`@5c=0u#<)NgJtgk>=Wo$(JaULK^H(*7 zz>ABfH$*3%bA_Z9zCX_d#>=axt|SR_BF)%mf*=v0Q&IqSS;a)yZ=- zTosHP6ldLqw#TL7KCo6QhvjjgU%rCg$~a{K{o7SUE+KJw4gM&n>g!{RL%41dV^+hu zWH?>1zl3Pcx2)pbTa|a@5pNntAmMmbFgewa@)a%cZpAb3ZJS-q90koLa?QU@bqo zdZckJYTJV4p9I@>A~3{!r`vr@W$v41lZ$izc7uhbb+lTG0|)#z=wGW6op;~795q(! zL>(Y@(~|c_lL1m>_0c=BA$|{)bjF`wSya~EyLJleebHiecbC!xA&ZH7$08;swXZGa z_iReLoOtnxw5`pSEd2+xi5Z!FbN*5O`%imXA2n80^d29M`+Ns0B2ze-qny4C5y63J zX!=$b8@ras84T`Mbc(XdsW<46EqIPb#8uw%;51GuVBvG|a$=i;c3S*Ed5 zBYs%2dh3N^pJP~1@Kn~CjTawc$fj7sm1VXHG_%mB7Bl-v0 z)XlBpe?)JNbr_wqSx1>9P6MCs(qFkW=AWGZ{F5_nJaG2OmiZ@lXEEoXYxew;%Z-?* z)4b60X+81e42N}GvU`l(AAQG8%P)tBk!z>Kcx~E#AWINgx<_MYB%Ah-B{{~o!mC4n zq~^R#k~zrJkuGEWP;8If#ut_Q_VpZ+32PPnBidupTEO$NU)YD!@2>sez;nFK$V7C( z(zN75MgXK#`eM_r+l;SF@LB20Zz3>WJHu2A>41iD7HPE-OGOXmE++51wrSx5rQVGZ ztnIy*TThq>C~#%-))PxpJWRAs*pot2yYW$DbSskhcZoQ_XpjqLzra@E(pkv6z(3GU zK1AM($vmAtTZ=0LuzS|~-e&|%jDQY@H1-IgukWTW@6TG+)Bnl1yo=WR4hj}6+FQM+ zgqjrXdAgb%E3t?A@1IyJo*lt&lg>-$&{AZ+jEbH5${hNEJOTZ0gpJV@@v6n9m%fnp zW$9?&4xc_w$lI%SM1)t--VjeW|EZgx(kylEpix2j=*R1~j>~#~(hf4+a?HzU>1C0v-34qNs^o@PBDm(3#3gK@sKNI zJGakPyDDsa4O#|)E5_ge$~&=yH#v_5?GNVeB`@E@%3Ii=V!C%^LmXq|hz3J`2YnA; z<6G%h?3W%duDGyPV0!6r95#A~gT?qtoy`_0x8ti<^NT2#hz`WTJfctkzzrT@k| z)VS1GslI9+NPYx{zcIfr(k_U*l6&H!g5TBoXSlp_b*?hKDCRRU<%!&}r{~9s#qDQo zXRWRJsLcf(MtVV(F-p6e=eOY3Vf_KEWNHUTzMWf#lY*`JqZ9Ln5Dk9$7SaAL#exN= z60*1lF_S-xk-XEx6R8rjO5&N%6*27}yR&!|TE9U<)&=$8b7a(NEo8TdIRp{!z~lBO zFHcd&djvk6ZpHcyxpG4#I-Dlzg-r65JHFa2@{GHt_gksfyO{?gv)ciKN@EaU!QfFLFTJ{L@-q5l&GRZq|H@Yk- z%CFb_vitZu-OS_%r|pMI!7zcZD3S0ixd$kTsk0Z3laIcw^+ zaFQ82f64Vfu{Y>ZZF z8LeLPmquMY>>VH15);m$7DAEq`Ss=Pc@!M!c{csjaX(@_Y9#l_s@2vnhRzC*=PNO% zrO8)RY?!lx@@9Is?bG}9R&1SpeJZCbzryq#bSAmWxt&*UcJq^2M*C;U4oPgjfDVMa z%I`}%8=LRP14S6{`G5~qm{9)2bpZyoHf_y@1l4&O~(8)^jlj=JVW!f-q$VWpW)B<0^2HS#Bo&k zj(Iw!r+;M@OGsqtdOPCK>Y_b@V_W{AXmwxa_}$VCnUsZ3iRr%uKh7zZ?+1(IARzry zl%u_NW5(Huds;HmxR=i(;NqQ;WTtIi{AMn_uA{t|q}z!^|v;+|t()X;|R3qIUk{dWPnw z=LHxJruR~>UPHYGXITDuBETV9H+J$H3-mQX_O~W$m%(0DP;E>&iGkz?-U5p;5n)Y~ zbceUTf;ww0{7p{;Qr3V1(sBkX!8r6nnyDEIt8b`wlk19q+5pX6Qu$oQjo?UXJs!&sO>k3VGsu`(6@p|6q@?2t#%$ zN|Lj>48cu@m+qXT?(njvAmy|i)!{Txj#SPPW#j~W668QpEj}fG4x@uj0FvibkMK?> zVC)pf6JMla6eaRx*W_(8eQ|k5^_18jlK&;5%mEiXq1bTA=E)I`#foLK#Oe07t$fHr z$#3N@Mx;`qAXAh*RMPtS!3&&`lBExn^8S1eg5x5RI_7=5h|x4{PS**&ZvGx+(FZMl znsMT^#l)-XaNYHyY3H=gJ{L8my6!-x(tSG_jHsnJT!6gy8>^jd?f|zLL0mS__B=h7 z5HV)KR}k{n!?2(~QYF;y?sJ2y;F2sCK1HiRS-@qVNK79HqBl!ZA<0A#o!+0HxB6#~ z4S^|(RpTYnpTMX@a9v+K+lM(u-oA#?Q)otDend4{&i7ReGLoRFgox={c`sIB!Xe4n zZKg~YLhe92)5O-cSjrkz$uEHFXKTX6sK&8-Z+CXu?S1yg0XINBiu2)N7#$c8zcRf+|{3uaW&Hl>N ze|nHKS37fxW2;N@_-rq8qb;m>Na^0toTl*DDOYS`k37{4|VYce>xVxYnQ#qBIYqNIOPKT_7 z8}e>d7jc~v(i1Cf|m@88W%6^Ixhz`*Q>G4!99QHX2Uzpe6T8lTNfq!P;(m3KpI7h`sa7)^gptVK#7J0>nuB_xSv zQV@1%`lJWq4byx4`;*gqw>%G>ziDGST`to3&0Q|ne@hJ#nCm`>FkrD*)-lFaG5Lo! z@|4s{&CcpB8i;oRdC}6fSvz!fle+rZEas1psp1#iI_?vX#FvK?h(SxK*v)N^>SJf8 z{dtcXb0>-O;1Ypzgrk1S_eR$NhG~?n<)aX1Y7>#hp!Z=gMCLxNbSYKtTGONwDxooR>R5 zul+1E=15jRJ?-ySWSCh?(_8rb&jW9;JxeVJ;0su3{76`JHMQL?=4!rCn6Ci=U~aPI z-G5amD?9q1PkU)By8QNXz62Qqe!tJ%y?jsfL}7i}{eYJR64H{dTGf zy8s+L#r-$KM_aevGhQW-`uvrjJkWA}C-)i}dRY|%G}NaEkj#0kGpm35#s?EDzEL74rQHQX z9b{o^zP}^CUI&E|9Z_{{SVs~Nc)Q%_`9g~YEGhO+cU} z-S@X`SUndj^b+hk0BLG_ulKXh&rOI_oqsUDpY>#T&I_p5R4$yy#bVHi!X9Tk(}Y|#0;4?p#0SMAVSZF zp^ev)9(DD&SRNbmnA5z`Fv(m_Lm+|(II9eHvKKk!L+0*)li%0$^K=R^R?r9YTVUkC zcv&&khBZjRLdHvD9(aVJ*6M#C7+)5`uYO)=*ovqOb1KLgQ|%OW z|K~G{X;=o+(In;x^Sj%yCV48`an9c*5a0`KZ;&f!cPb#pB>214_@AykV5VwPh(r{1 z=TPctHe>c>-r)DM-#|zUBd$Km5}%u}xcsPg7d1rU)#M$;$WOqe1Y#E5lF7|=+?&>6 zK!;>jifgmGn?`R*{-Gf@!yCAkidv7!q3}%Meii#Zx}E$NaT)3sP2G`v7MqstVbA&o zM7RHJ#sdz!9oqfIIFjSeUJ_nz<2JN>-D?GhbCK;LL8Ecc|MlxJA;s7TiM-3#V5K&> zti%`2HKQufnhPJXKJw}bB>d%s`ai!b{0jzX*lW?GU`o`V)%^DlmJfk*^j{ZwEP2o; z@wLG5mTw@^Cub!m(b~W+C8$(E8)xLZx`l& zh(#HvDo6MI%ZYQ)m!py|v#X%axDoiJ;)iJI_X{9=c<9hHIy8~-eQVX996^hQjwOYL zK^TJe`^PpZG#k^4gp?Tn`IrBC5q`A7vW&!)jS%|t`~Q5E@DUQNy_NzL@Be<^zkjcl zu!Z8%Lt@u|ed?cgAv}j3>A@#ui}l}^^yfvm&@j3XKJJT5lmGQk|NSHFc?^mr>yS>u z|Hsu~$-q}#*W7&UxslWG? z6fHk|cko=Y|gRBNOg- zKZvW{H6G1%D&n%Ekw^B-*h(yQGsy@3Yc>A-%}vG-2UACPP-*P}u4Ori0yfsO&pJX6 z7s9VXt9=(Hw53DE2MyDPBZ&GnvjJcsF=v--=0Yug1XR0#4w^fSdxI)hgZ5(OO`MCX zCaYZ=79LU`09XQq?P{a0RxtNDL{0BHT#nD+Iy{C>F<0{4)1MuZJMgYR;^7tz%|*)l z?bNM8S|Q*(%87?E64Wy=^LgPjI$;xi{b1UwACxE)LpP4IA@QvZ#Wn>vuRLes zXg)MD!1%@1{dD_+t9;gNptk{k`L`fB6n2{)AEN!Koj}TW4@|&NLm*C*lFfW4s;Arn z7am;NsoyL)Z5VD-&wNN`SoCROoa`a`zxOk(7DiZp(o=(b^5E8gS=cSjIl@Z;WS)0y zUE)a6*df1dp0H2tLY8D=s^QsMH!R1gENd8ia{dyX$2%{rG8CQk=qpw3@Kv zAmbEYruzpngU*5e+YuFJ4X%8LKF=K($_xFG=Iv2F2d9CTdv`UrkGu20!mS@qsk?>Z0B zqL9!0W6;f!l~2Mn9F3*W zQ;5{?(Wix*`(f`YlrpPj`}WHi9At#Zc)0WVvs;Ww{UsMCjoSZ2R9XyFh>@;H5;=Ur zM(hRZ7Xz#O?4&q<^}_*D_W}l$kB=Zl*1sZb6X87gI^q?--7}qZk?i<*HD&Bc?%Mg# zhMoEz_0k>?dzkT+RkxFR)4qw+cgvpMfeAJZBl;$e*G78qMIa^GENR$@U`glgwOx%< zm{R5__8v8IsH1T;@~lrn30K+G^*@VdKNfIhT!0$#^sqk|rQIxuEzM{gcllwBDTsPj zx6M)xDmS2$U5C4R4>tjI}COFEfwk`a>g z{me)zd{LHJiB$GWrMi?_mRye;uYTL%%=#1136W?Jzog@JQpMJ6?*r-0+;K^0*3ZUh zQ8!DLB1&w7RkXJ5eb*(`6lLqO$!dKo+mT)e^LTSLFAOhv&@ib| zSIoAoA5_ewUUMyK^ygYubh_`PV@|eU#za(lD@4L&ZiQUKMj#-RKV!X<9;-~^3S&`r z?*|@CI3()#`(A2C5nIf)6%(ZJSj&E9zK>wXzQ6iFsy8f+|9G8#e(fVu4>)5dNUWQ-s>^1krIxJe(#Mpe96X0LpwVO?=Hr3A6xw8F`+N})z zB??&uOULQ+y9z^AMAh;UmM$!^m2&5*5_w7Mj0(vNX%3uYXT? zN{(^`C1kD1#*ll;k>R+MGT^$NBv?hGgDKiucSlb@uV^bE=V^QLm-JlZ*(OtH#KExfg6S2X{qv{fwy3l8U`5wV9^c0Wq<_s9$f zM{`kEKVyL>8pJ-*PSPPk3X{47?8H5S6>8@y14v3sjeb>SB5nj94P{7JR8%)zkq0)a zVVNb;)uQMtR1JGTNf{TR(%jvG6Bl4o|2IU(`_MGh!nw^6nM><{aXn(MM3E-(O~)+P zLED?9)G-xiobCfp+<;r`kGiXQ3_JHF$5l@ld@@AzbS<|%&DPyZ$ZFX&U@+wjPuHN` z;S1OL^&IqV;~KZ^S-r}fz;&L@7KDkT*|O@>dzxK0?Y?v?^(LLed3>%)gY)ET?Rrk? zbo*}cP;zLo)}uVG3#8u6vMpvaad`n4DMDf|tDY6U5P#HM%Ehsd=Z{e&j z<`=tcHMg!Eh}gSypFDf+{Icp-7j97!38`}blyVGQ41dY;d~C(UOBi|)94vVfJR7k= zqJ^(oT0`+2EQAX<=-O*{e{QNioW~bi7 z1kTy!+tVJQn6dj@g_&6tTiF*Mxa|m;eBrt5-hvLHwdC#WXl-nr6{p8W-X)_FgUnZU z+qX#L7-H^Vl2dw%i0)f1-zDOuTixaP`oz_fxptq)i@B4dx7wf&VIqVX+!v<8)1=!s zmTq5v4Ip$rhubM>*aw5;65l?1I!r+cvSn6d>B(&B)hp;`_oiIZA(X$6+{ML{`5I&g z(}JQTIMuKDu`wLk%a#?w#~*h6FvJxSI2cF+%D;iHk`>4 zT-WVb=`woP#PP-P@L7*RUl4xxn8`bN94>DYa*m!uW38L%bEd}oK+?K>ol(`|u#7`V zONl4knndgBQ>h~yPSJsdTd8`tn;_MyS)z6+lXk!GL)^e(2Q!$tj-hsYCLqx)>wc8< z*J-?3Z|W%tx11A@+Bk^s)GkF;UpD?>v zNy8Q+pBBg=z5!|F_39Ik4jO}9xxTs6gPIVQSV|{=IK)0oCG2-QK)QCC5$vl?)@BPj z=4yG#Jji}rc-R;fXw@)B**`5TSI5O}B12y3koguQPPSo*F{qyv9Y+vB3%uw@GstxF zsi0F)9=c9sdAg|->HZG$601*0#>;dne&afQfm8}M@&|ut19^To5D~FvwV!Bwh-;fSO$f3 z7fSKCV|`BhsyGs2hKf|U&0NRx%Y-(j{!>&jSS=Tq&m)suin!eZqghLW*k9wOQoeI7 zQg{+1v1S+h>t5HHd=EoNpyN-7dn0yBH zI^&))rBBT=xNhMTeH*YMu;t!_vre`zLb%jqL=aa)2Pw$lFS%hlXL#=cLOY5h8GRFc zl56-kdmrE9XQcz5#hL9dL?@32Os@VupPQ2hBi#8(C!4EjzaVJx9e7r%VceC-zK?~D z<7KAcnCm;0L+S!t=Z?2j&k-{izBb~1UDP6j#CI^d%FjgIUaRWbafXvM_Um(*mJZXG zowFu`f@LsXdVUKKww!Q@_L)j=JIbYL&S&X|vi0tP0sh>4?fxOlC;Zr$}r_gtllDHlW{p; zDV7Fbo3)#Yz3FtFDYsMk!CzYx!cc>x>GpdzWl~sN{hTYA6R}+Fg?E>!gKbobyb0}U zp>a)CnbH38+!@u3$7nDgGM)#{EU*nPduBUrRaeUT z<6C`wrZUZaTHbX!Uuy7l^HHvf=+g^^J2+96>}2ZHY11s3JT>L=Elw|pILR?bHP_rn zg{b@swX1f+1eDhx_&WAaPIticOlR?fP1E$*`XD3{9K{@52=C6*j^E-z7)!d zHEj9uB{E4aoc_}X_r86yk|c4WB}|tBZddI6ZlB9x5$t+7bx=fgsrgWe^0mT{fy+`# zuh@yw9X~0l_F-6YmQcI7*rFrhiQgv&pNV}B;;6%gTO z8a1zW(*AJJt&1Rwy>9!XZ753Z3J7D|Q3 zPjydHr|^z!?c{Ij%ukLfD{tNK95uOp{YPRsH3jik?{69l->Sb~`?R23-M-venM}Ho zBt?wF`E1R7#)%ZS6bC(;z=xA=wJp`aj$_kT+q>4SCjCK&T+7K&;rz25BAkHR1zx(i zLKTT_fhLYPJEauD>eE9XgV%))j@>*@2z;kK6oThM1&`3l+{0?h_kh?r0`|ObmTDGm z!1$%|rs)rp$~bRG6m^hHjSa;m#!05hQxr7Z5rslj^OVUmLBlWc`mg^;i~R!QNp!Km zh&_zx4nTU?)OO5re7`JYrLH+=>wHx z+d%)=(8LP4Ffr`otoQy}Xv;--7)q$b*qpjPHd!z8z5fK;@<&eeA5Hh9>1IT}#XS`a zothQMbnbMw*bI1n$TQQL2&z!cSE5*yN!>#m7HV3jE#P zVnUcYR138`Xj&YdACuWyS1?Odd0ZICp8Yxv5k05Wv=(5KLekqG9+vMz*J2ZFC~SC6 zFz+M*zcS6MPZS{P1L5z(fX*mL@8@VMJOj1zr=gG@Qxf+odTvZ|LU_=PkzeZ=Vs3Vd ze*0XfbKev<5`Q(|7h!V=7NM@W!IWA26S;6_Pm_2PQmcZlH^ttz69uqlHVTJOi=0Af zG~N@6na#O9R+`b2Eh%+&Q1f;|g)HjE(XBflAq5zQRsrW~m2*j5&ODci|IdZ+8rP{& zQ}!#%1%)N$HIK<4o{OjQNxPozU?uxns4-g=Fs5D)OsWVP(D_q0bod{}$P^(+=*jiWhc{q1qFo-v|qX zp0Ro0b`>5K>1@^9ym(2YZh!iecJ*1KFD;vymgIarMIK8~BjD^s(YDAyu{^9kRx{o(d4_~2t zc*Wa&MME1z92S&@jqM4HOFh!z0?pEoE}+YXWO`ns`-1qi(D=o&d-syi=4(#G5E;WK z0K`BTb_5+LAw0LnZ|$gw)87h4@p^sJ#P5_)HFO+xHu@|^uuPt>cK^uN(P?-9;pG{X z(GT4#f=&sNWxD=~1S0bxU{5^NjE-4X&30}Eva1}SjoSD4;++zAKBqVw_1^WIi;o@H zPZ!_F%fgWVtbY%+$gg0L4VYpiqW`*HiRkAs7AXS@$2&A}V`S)|-J4Dg&Yo}9+|}Hi zJ)h943W`?qMRKYnMe2j}s?uit=Yw9MsjYFFXqYSVEYch5iu%Elt|7JXpS4LMJ}5iu znJ8a`UboZd`myVKpJaj45pOYtCp`TXYPPk6k=*nEWpf{~!4JlZ)ARb`2_~N+%fC%5 z(b{@!&b>WLGI8_Y$j_biopR+L>}o>U1sj-pJh)6hz2_c&*|QUz55M5c)!Adg7YV(O zKNk70EXI^jksejanYsk^VnyJOebg_HdCnrG>ECoiA zYsaZH=3D5bfCXOZrnX8YQyf0{F-@89!}b0Tb-YQlY*?9-PzcE30fwH z*~#W{qCB-JJHG3Ka?(X5EM`*4Fzm>zqGyf#Q~PvWe?nByCy$bDVS{o$JWn-RNJjn+ zIEVC^7h=pu3{&>cEBgBW=zZLjorWDi*okBBi~+?&zFvZWbU)9FZ5#2)Q0=Q*U{DK< z#f)NXRwyXVbyCj5F%8{W{ZAir=VX8>1n@E1N8|bj_+x`^47_< zQ-d@&6O|(^Z?n}^-wPV=32B8`M;I-rHsaAvN> zqGHt8xocJ3;YLx@!Q^Q}f!4Ev;L6Kh~d$8jlkT=g773MGt!F*bZ8{!!*9Be& zLsR!C-|h{FXMmg3^<|Oj6#SgVTU`p~o+!4H!h7&8B5f`}Um>i47VKmajt(A4zSCq6 zzh8G;KO|l&FE~Wsf+-D|fvFaH+2+<*_Sqnln6rbyv&ls&W62mGLuN~Hs| zp->v$s(vQ|yLgb9l*CY!WCt3VXd130w@W_06dDTcO3AFm#_Sj-IcPaYAh8rFfHPZ_ z2)|AIa%0_weZ{OK+I5}>ws!yP1E)_5lT$kSx`7B)%{uMrXW1!jSe`Gq$+Jjq@P%r- zr1Gg*BM$L8F55w)jce3=pHUtwL`R_J z)57zKZc_44K5ooTJ`FmX2{;@seaU>0e0PyXCkt+4#(210izx}}xkS+PRq4~-<;Hpz z^Zo7Ve(8ruyB&5?uY&Inlmnbn{k&4fVfM<7?~i1)zA%(qUvQ6wjVCiJKcC1_v>26x z>&YO4L+DkT?UBs)RW-Rqc?Vc)g(fdW5AYrnGq7HWvOWPgVbiE+8f$p`dMh_AUudM= z!AWKFcRl^i*f~k5Lp*$ud9=W=7Y_3M*~AQ;A52z<^Jf|5WBhlZAHSD+m_V@rk2^D8 z=mpDS^H=Yy>MP!q;@X&QDN+*tg|%p=B2+sM!RX;SOwDBC`^u7$+5*+-LuAT!qtMF-}L*!;Q0Ow5qY3%sd|?DnlTt1^+}%H z6~?Yz?r^Ho z?SN*wwkY8-tEx2ulcI~&e&@zt$HNdV-C_Cq5SY%^*@Nx0R{r7^kuNlQ!5kwh_?k%; zrQMQ_z7ENy>~}D%P7-7ZhYRcj1vmH)x;9vBUVRL9d!<(w2D*xGzL#+X<#^c-lG+u8;VieG1u$M6P9$H6 zsdeT5z%LRY4NND#m7^4iCYDAqd6_`BQ zpO7jtfaz1F`K9zk9~9OIr-Z;DYCg9S5r>&5YVHEA)#>r!x_X}r8Ict70Rd)-V))v{ zf8yVneJJGDfx6H0f4}Vo3UJ%^#{~rb`N@C3iPQ_C7K;D=>l+x$!KB2yl1F>=!ep$x zIBC>&a+-fZyDh{R%aO7~f8!1N>LJI`gJ`>wpMPYa<3wVt8U)5({=K*x@o4ED`*mN6 zsQ$gyzyCB72jEdXbTJ*7e?rxHvKVcN&O=I(F&4cFNUizW%4FUVs4?*Wec=cq+7eT5 z#Yn=}^b^hD7+EbIj;!sq%1S6-5AfMaZCd_5;Q&H`(IT*L0}7Jy?WTR0sgy5MiH#a8 z0*6E}${qCc$3ti?3T`OmN6NfruSczMSQ=bvNF(Zc;*AK>R;3ZR-@OomFr<5T@41|K*x6vQ5{&{E7FzRT5aB(Em)hP%z zEC7Yqj}tloVhv?m;BFVo7ie33nEbhEH!HRfMsnruS81bii92@t)WE{cl)is$cTgo6 zi!U8in1W-bUVzVTe0bt;*(=|KCEafkUwq3Qd~|xe*KU6YT%y~_eq*ZAk-<5g4Ggm0 zJEZ#qtFhU!;Vz_p)h@VDo-IQqMwb;?%!fG=4ZeMlqkjR0DL57b>2S-j#m)&4_sNP> zq-xuS{@+Z1P6j0L`4H>@OS z#DK?twJu5IM+sMtK?$%S;T>Gdzz63EPcv%h?JiqoBIzJwOc>A`9Cv8F{y>F)s2WmN zuz@{N?hK23BzbnIQM0P8!j-)?3IJKl^_>&+way-6tCn_}u zf`^X}2X2zxlKAImHny!Ceq*}eRjFvNOc%#B=$ezMn=%VIEmv|sjD6%>ZGhDpbX5|? z#86s;r?R!35A)y8#qtY)J?*|^s+xKJDxXR4+KyAU;AAsZGks%6rVp>!aZ_C!fPSF;^myL(l$zi6k;Kx+XOFq` zB2oTWz1b(FcSm6wi5s33KavK@tl-n{8FpClA3j~vAr_Nsgsi_#S<%6XaSFs$dNv*3 zE+y=359?_7Y$htIV{b)b4~}Md4=Ul@t4!u^V$0EEmH?DEv3!M%>_yUdz{d+eqwwIZ z@*bB!95zBlMaXGM52Dy;H6x{nyRHgeI=uP>J@Ef;9PNRLb$vrCOq$&?K>H`C$cE(# z$AP9%?4@5;_+l#AlM_Ka2oFutj1?hP0+bxf(Q(M*=oL4_HUlKT;T2get*_pC(X%Wt zox5$@^orrHTTa`9z|!Fgyff|X2oQfp2k|QY9k8Ij4Ly-+`SnVdIz#Cl8=PDFhKnvX zCrr5_`hkZ~_K%lytwjr&z{Jk7Ibw$|PD$Ww1EzS16@qTk!NXL+!YP=EgF8?YC1|8| z5a%B7-!Sf(gkUCe@7OPzsV|Bxeg!Bp`3%f1cn^lL7;_4y-6uE{yj3fq5vcFi&F z?z_;{24MJ*{i6INAI@SpCctF+y%IIV2ZoL~aom}(yJW&`sL5DVdC0%=nCfO2pzTcOo{CqB?A*u0DZT(K#no*XV4+n^pRa)?T2z=Xaf}tC0~nmC5XD z;k~g6R~((Zt=|J9U~P6kdEcbdf>JYU&kF+Vh7X)<6sa#LrC%1I%TAa$qN}e7taX^VT#e*Rd}j#s(_m!^9uIzLL^67%tIyAGkMb|`WBpMQ05djH7Cb`Rp zqPoPoB?FH*elYYJs}YN((O{BqD|^G58JS`0a8di^s=si#+N3*Y{e&w4sdD8)^&l6@ zm$@tWh9|=P_F)f9YX4>s;IyYaH@fzr`No0v1Hn;4S@hSRb@(xK!zF+MGVOtrWM`D@ z`evR>Cm(QFGtrgAYnad5J+E+*Cpjx*5%M6z-~*%Xq*CBvyhBX$xW~8XMp+)P((R8# zldtg52-rvVninX?!#D=p)HAIvLUr-&Ybz$a3fTF{u$ksmd=c?2!{(ErlE{31mbIX` z>&l8;Y8X0oV=#@W-S?WquW^;&ckrfA)!{sK_64@PYG5#i&PiIs+N&p9S~5=d#LrMk zi*eTaO8xrbYBsYk2KKspJDDMc$_6ja<%V)TOHSw^vL{Q;XEayiDh)uir71KD4`N0+YE<|b zGN;;gMyYcl=xGF_JJ&-h(C#HXzYC25?_i!o6DLEHb&p3?`=_DzxHmrH1YZ~IBKjSs zm#ZUV>E>BtOm`J}?4Me2hz~OL;^&iJ!LFro}5L z+%1qaFAd)z5-+?h)-tVYO$}3Q#@`XyeFe50O`&C=ZCcH@ zp*oh49FLgtv^}0bR`+Ql7&f+8!5K`bQ{KTo>Prj^RllkC>DDKy=0He_noGGK;zZE# zoUdcN9JDt8{lntR``2PGKDvs^wX!ax8t*;wypZ6-e>@+c`tY`DPd7H>YY2-gChxHb zu{2sv*0njWbVM;3r|n6G$TxM zLy`v$abL-Y;Ygs_w@E6;`gy+_B#0#$l8@{3ZP%a=1mm+XvCRL;%P8R{o;ib*__SA^gr@4w+sE+_@Jb@6Ms#9c$*q45R!*{u|HL3Dxm> z&K`zK0%Ta6q!anBWPqku3L!kAsBoH&uH@I%k{Z*@=fZ?I zmR8=?UH$*(eOUSs~R5T{q6MLfH#n%E1q*>Vu`HEuzw6xBEEh=&U zCkcZ~-AYor5bPg$oz`0Ln0eN~Pm4-w(vxUOQ?xd0z1rNxo2#?1 z9L=1;|8|0Vv5l^t?fa|rYr)4$SD{uJ4NVd;I=@mD1SO#}HeLI#4CZ(3B29-L;e#Ic zcI@0IzC5}zo!;<f?Wnp~Vz&v1`X`*@$ z|24=`Pl?FhvPo_$T=8!U<~X3DN1fii*&jelVs8pSK7co_D zcD!QgAl%=;DcW|Vz^bnUxQn69#u8JEAZ(MR84LMBD-3L=VA419uMwEBNf&SK{-8(Z zXDBs|z~e({0Q4VsVsrklqf?S$rbWVanVwvdaOF(*yO|_BR_Zh@u~YuK@{Kr%Z3YHN zl@4`%gJ#o@CGNUSGlD#R`}c49a${y&!0G>Zx^kD}33AJ$`#x>*H51A}Y4q|V+%J&k z^E33=m+Wy>_{PI{qmtKj>sND_4iGSDq zds`PZVL??@TVMZ}kK6vVJlMx{S4$ne?q%2Z&{13WRke2o5mZYFvhN-~DXAP(CeDP) zhp)8kr#ld0kb>=Oc8=d3&WbA+Zxk=q2OtXa;Ct*>zew>?LZzA*(x)A!PO>*1sLP||a@<5k=F-dy^z=HXq@PgA57>P!){aJnwzy*9mmlr`|yZ{_`fzsKoOsNle2H`IMD z3OZYTP$J#pxsU4^?d}zhDLgtepXn)>$5_%>l%c{Tmo3eUdOl${GQWeHb)isg@DA=> zB=dE<7m=kBFFYR%#m<%(Jde^*^gaafO~uj|pdT`Ed5rqn2kL3N&&hExkJWxix~!?d ztoDrerL6$fYYo~USh%4%JNu8~$dAmyq=wLgFl!i$m8=j;i-BMi!q?ZVE%>!(*AcR2 zF)EXc3Pw_0?q}#xuRdr(6riBoIEVGuMnk9LlL^t+Jt0r`SiwWeU-;t%;Q9`#((H4T zs#lvxfpF~P9()p!AB+a&Tb1MINvcqB`PyHAwHWb8_!w}P#ob?P8;Nsm6&|-wb%m}j z{*bwrqUX8cNfqBQHWHKWH~enOdxkG+O#akU9!AYZyWKJ${~QNWMj zs^^FtXkO?ny=F@}juvtaO;N{Ih4MDAPHS`J>FK?ZTo@ZsueWQpp{#<^4ZQVbl8}QPvU-YOUmB@5}s%Tk@d3CIa#LPPnB!h=_q@35TiV+ zJabh5@PuEE5@n0&PVEgCG)nX&uyXlf36G!>ev0YZxCSwik?V{NQ2M^hQu+zX*Xy-v zMtAIT8iCBcWwqdG!Q&Vi-vVs~HKU48wK5nfBd|+bUu26T_~VYXsHIO8D*338k!$X+ za5(vQEFNCbw4L8aTi_Q*KnV?2f&cKwRY5kyu*C4rYp zuxUCV@73AUXsDJ2nuceJAc6Q+S$R}I*?pZys=HWj{=Et_CSWB zzTNo7)rR=lWDg@w`f)VM^$8NN>DPEls!kmVt1nvHbS$}jr+sBK;E^>(HS`D+z z;WL{kXdd~vbO6Nc3$#&#pD&BwJOGbg{o5?Sh`(VXODBjifx0+BOv_jG^fC8#6&8HF z_+wfsOZK=TKID|5qS&v_XM(a)(knn#sF1PRyT1q>lF_;EAfDbY?E_-O@$^^WC)w?7 ziIV%nIy<+Hj9$w!g8H_IWP2}FQPOK~4*lHuN|%X2S8`K)EOrBh#qRs-b_Z^wzjq=z z0(ujsK4eU)Ra;ylzqTs&O%H+Os;Xf&>pi5l%MZ4YoACc7TXkZw^YEY#NftnN3I$jD zVqm%eZ)?h&S!*0%XV=MEz_g9&X?%vYE?jmGIVI}m|2Y~LeI+e`nKE23Mbh?u#k*qQ z_5X+x-(h`6uke=RS3kI7*?J8d$=M@4FAi6@qZ&spM{DlcC~g5j*ve$Np>UX~QBzMp zN07Jgp^i@#Sxw7+djMKP5&WsgT7z*g(JD!6oi4@*Ldbl6POX#Qr$jpGRJ+TRs+O={3G$^&e-<+&fe_&$axA5TIIhYuE z4~!ckK1ar*DG%kuD=|eA$xk$t8t<~Prk);Nhuj{z#KWl^LsBdWg_#EO0lj)Qd^}o ziXht3xI)a}H^01A{usX3s-lMVr9|ElR>hUKkfQwvle~SeqE{#trm@4o_E)O<>Q^7U zgI?VkzRm8CR@v^%&|$8LYn7g;K$sJUq{!i%4|G{=H$l)JNhMeyeJ-+Sa=0AIV@ zbOz*zKR`1v>i8PHy$EbaIJoyghwu!XR{VO*ZX!>I)E>SCRS+Bg8~E>8K3rbU2G6gt zM??&uDv@h?JBYW^sN=nTR{it+RcpSdY+5zyF))dySg;%I#|&Mk9@}HeC-o9+>O28K-u&?okg_ zUUN777+4QaX+MA|#J9nBaCa@qp=z*H`7is991)@ z%(0I4IIfpoc&%}nq)wbhhc-qUwx`bcot#d=-zT^poZ!Y)+9Wqn;W;iI5~jG}~@$MWnV!>Gn5cp#uUdh=%NB=o-@bdt$$jTcRR zI@9wEGO+yBV#{>H?hJ;2kKUkxF@z>US(m6r!;Ey4A3n!dE=!v#y|O54>hB9Aqc~d~ z2=tMV*DP8^syydi`dbYaM<3I>9Bs^Xcr|`<_c^sNYTJ*cy?T@R7>hLzG)SL)vEs>& z0F@6s{??I5rU0r^#jLCaln95bX4(_Q23~=_OkTo`1OA9mmm^=pc!~vR_{a0^-Ux)y z23sHcV#)h!1F}k<=|H%zN;s5NI&w`0QJBek_5K^d6>=&e7ZKY|LgDr5%jdxH@D2#l zrrqD;!T5s5p6Jl1)@jX7m;gbL&{7Qq#_5P#7TCy-&{Z()s{Y)>V^!ckC%ya|lSI}t zYx=`!!f=*F8*}!?_$-EfY=M)bEKff!Ecs&Nh_=dSAJ{K>rZU6^J3FK+)_?8tH(Z6` zkZA0e^%VW3J9NsBIv+2IBL;Pf#rH)MtMjOg=yo6_+Z>P!{GnNX1Ee;*9I=7<$Dh4J zO`o+VR`PkQh`mZ1p!Ftj*l>zrVZ@4np6f31|I(-Lb#nLQOlkwt_}JXBqInzkCE5=aPE|&lCTJPW~;V)lHGQ&)(m^mrqg)64afy zXZO^mz%B!2Q489@{3k78Fa-*j3q&`>{)a4754C9Qr|$76ngp1L7)$|Etr{w}1u-(W zedsrtdv-Bp+}(|E1IM}REyO=iOd-GuQM7xA822Dh;0cXq&cyXYyPJo@d|@E}0bsHb z>>7%YjhsKA{2y>DVG68d=bH8gYC<3?1+y=QBOzPHCT+{Poj4ag8sre6&VHb|Qx{nX z1838&jfLBvDjkO)$xFMt+lBs6BACZ%BTNdXTSq%wC+4NPZ@k^V7eUjWZ zpQ@Cj#Kq9063U|S`pBokRh1yV3v%rh4q-5bs3|u*%!+4LTuI zRP)F177?V}a4>;gE`jJwnMZH6JHTd4E)Pk=o|p?_4EyWJu|+N7)=nP~YAgG1{JION zw?~ACb=xBl9m{12-ug5T0>6Wc;$Y1>Mmz@gN8n$!1^GU#ZI+DA9LT@Gi-G{gG{O+! zY8#k`Qw>NaN@zlbv#&%exPtv@Vc0gWpUZxP5Oh*vF+3Pbxtz2PGR=GFf0!Ib44?%c z^<6&eVTcy^u(%E90QBdKOm|N}>I}1ng|oI%vdPlsdhuOo5<;MHa)yC%LNH32@`i1N z2{e_!#h1wC5LnYTt5FVmu6aN*Im# zh~{Y;_M5M^gak-ZZF0n>I}V0l(nlr)Euj?ff6+vyyD`zdMJ`S0mc;nAC#rD99>A;$ z?fF$e-SV?kI-;+ZCb*(N@kRj@`iK)CXmb?pCbF95WosmGCsp|N4VYmAguI`ahK|{Z zKy!KyLne-)*~WK=31vc~E_vb|Y=hc=PTeG#@AJDfSpvKH>E`LA`Va(!+yW}JFA^}9 zRXd9?XbpP_Y58P&UF=Rc!`iU+E5S_0)HeLoI34rP2E1>LQ=Cobh$H z39>HPK5YSItNo(6-NWx!XTCy>XN>#-RXj*|d(Fu?z@qxY?9t(B2BeYBfl4n+u>N!R zt139FZ(y^2Z-K!lFu5`aK(7tNOW@M77rz5M)%rCN;q4M53A$CNs$NH7@AkU^+0HS) z$s1k768PDxA0JRFkM+%el8_;1eFP;<7!>3yyl%$bn9`+1w%1%h0luw*dh)X*;mMCe z^QoDdaq>z`X@icWe^o+EN2qeBx@=ePM)2t|Xf^s(vs^qtZ!rE1m#49eQ*`h3T`+xcxdach z91Xd)W_p}hS}26L3eGy%%2JF(z1OqbtDrkC8n$D3w~pIt;Kedl2)=bvd05xU&SvCq3~M)aNcaQg>qLb+Q#8*7v=VQGOTpHpB{VRXd>+%LJT#C2F+(AQ@e^N; z3YtxtO4j+F(dW$oR;^2X={Cqi8$l^SqK1It1P2p5ql8$rglhhaBMZ0q5uD&+8X~2L zaWKa(N+Ls?Wn_fVRx~0kde4TXNlr*5o7snrKqNl7=&|Bca)pdr^3TH^Lp-BG40#l> zRupyP*H&DV0KrJ=T^4n>St>rkBs8TGk~_Rj?NjWg8*^8pV#!*IX(*sSsl>zid_=NO8Y2iH0(=p#L@ z&}8VM|h28X8^KBGpduGK~l-VZii!MdrRPT?a`esRA{8Y&MZ zl1JBPO)*G3)Ave74t}m|Xl#j@mY`KN(G->7W5c`S3pd#(sal5tDS3k%VPooZ$RhF! z5(tH-RCdgoi`(F5AfK-4;0OAhudr#y7P@^2Y38+eR4k{GWljk5Zu&0{E-2gPWxTM? zCR9ua5iE)qpwsHRcqL9MECKV|hC2dXhWn{KVMqwpcyJlG_g;;o{S&uC7;E_;32zr6 zh?%Twge~0r(od}ZB^V~gUdp+z)9*#9dQsvfj_1t4Jmc$$J3-SRoGg%iVmA6s#VgmT)@)3&_#0|?eSZYrn@ zlBBkzSEy2IJ)b1YATa*i2cl8MWGu+p_PM>CYjcpCW=(w6#3q*25v+^R$@P06V?Wjak!q#vMI1Rg7ix0SC)6G{ieV~B+@ z{QX{bqi>`K+c^yzvATJ|gmz$GET0gNu<=OY=-IqeE*5`Z?4fn*b*chfvi6KPQFp*nrJf37BLX z8V|%w>(KVA?I5Fpr8u)(QjP>E)a@}EU_K{qsWk!0a3kYvtQ=Re;3Kw%*F zQ%Nld3I){gJ^Q_KYhCctW>T4kv;7v+=5*b6P}4MjYp0%~!Xac;q%~iO$a(**Thl#7 z7s>@YqCDPNQ!lnG#sbMO1*(hXkHo2%cD$T=aDwblI5@b-6 z+2Hd4R)GYfi(vBOJ8pHsIl}o<<~wkv`2n1Gn+RO&vh>ssB@#s5BEFOx{<22mh(&rr zQ3wsJ%g1db8YnG@nM#@`bL5rl#s>kM9v<6T#Ze^5r#D9zY7u!Oc-Cg96OL1Qg;>>N z1?Z87AcOgAy!h}UWhfJOgjrz|AICt3p?57Yy3=j(@I@~7tWx8-el`JhcM(k2A6B^yB z$F@w+|Gkj@ZJQBrt7>Wg0ZAagteCz`=58Ybo`n?*-SWFfNpK37*0q(-&NjE20mqcA zlSF$O5?wdw4CZ{p;V24W;${?n9`WbHkT0M`z|KO*HEa5v*wb9}IN%DM{0ZDYJ{-;d zU!9Y5@Mz)O?-r!-SXtrEZBohfJjvG#=eX2jhQC6JMVRv(V|gz})BPvSooTDjFfumz zqm$KV)QIm3cvQ9~-J;|fA}aS~c#-3YETQ-v+_GQe>TtnJjFFIT)|7$7^9nIfFAnh| zgJAkK_eKm-Q-(x{DAE`Qt|hzwhr6%->uO!vmam|cN{1lb4N8NgQql;Dlt@U2G$J7_ z-6eu_cc+R-3X+m4p|psgg2X!u_I~y`?&p2~fcN~cKkT#Dcda|-o_pq+nQO=>od%^a zrih9lng~(5S9rln-gK+>dMN+f95%)XuT&UMkwo128%U5LjPlbxEz^)Yz)7*3vyX{n z`R?fWp$pUEHEqcPk&$T^mS=~*WG6H}zpBnw4#yAHcZUUB;~RNTRY8fV6(H&F_MjW6#cm1|nK6a`wYnpHt>t)_ z*)lZJY7OYIr`MpO(k!wZIV7Mh-a<+;@eqFMZ73EqI0(sj^$j~}wWX_I^WnP%r+MsP zFEE3X=Ic_gSUAE>KyqI_e+4r1T#B5X9($7K7I%&D^xxO?L^sw+#}uJ#!mX)OF2+PH z69(hianWK4t9s{IT6F%s+?+P&tb8Sh(CN_Ql^EuFVrLU4sU-gTsM=I&{KTk(kBnJe zuGow1G6(3KovR8BPa)t;S=pv{N2WHL931w!o9XvOD9douF&webFlFi0Y?~X*XcxpT zT6&P9Dppk+U=*j;IKWv=m-NqYJg1`P?)I{JoA7T@`$a0RjE%k-riA8rQL`({2C6vB zasiPM*pCA}XYi}D+TTa%Z%y{GG=dM_6pk#bvTT?8mC0>B^tsVzFx9eC*6jb!kBGqK zmh(z@k3-;`-Xr($aa`h&c9)iEN!RfKrd9yn9gui zxS{91Q^z?#&`~O}xMIJCT5uQK;uP(yfB2Pp=r2eM5OCe*h6UyYG11f{IR)d9XRr}j zr`#7J`|$Efd@(^m#@BCIuIt$JQIs9R;d8^t4_EG9X5;BxbehMb_$wup3eAxz`%Ox6 z4>R7q9;iHwi+o0fmwyrYeGL4*(B_u%4a&cNA0pnpN(aAp{u1@~@A*+3spd9vQF(f( zCHsbs`xksI@qk9O(Y`|*N* zj9tPgCCBLcE&ZI;epV-Z|8F35gw6N9)Z%j~J@=AAS&`LHb?@iP_F<@OnXucT_+5>> zwkt8JyvqealATx??c^V5I{u^5r;7-TR})K7yiOUKPQhsisFs)1^@d%XAA?G0k14FN zR8rL4xXU+En~O<>t-?w^*J-jk>mg-!51^?hwG19o87t^&9}Df!P?9tTDF3uKy|M2!n0!!qxM|CO zjRzDlk8jRny9cBvJEm5L9=uXARWfk+@)e7DlE8!^E5R58T3i)VUHr9U)LPOFMZOb; z({>6GUt~QA=p?(`rH#mIrBQq)QI|8N1D(p~e50`Su3;64@;fY!E)J|ENLJT761_)d ztLdZO5eX?L^LFMaq&6H6f<_e`gmu`KwkYxx_L__|2D+o9PyH69H31z7vk!{Y{M9AQ zeQL1U4yh^i{u=?ulTSXC*~EI2b!0v%=+7C-=#&n_7 zJq76Dpfc~EPU`V{CnKO2{;>ZAsd4dws3BP9E8qh;gw+49#F#5jB&`FJ_Fg8u8hx8C z{u_|vpEyYR9#`GMJ!~jzKzuZ!?_~o}?qG`3cJSppT%4(M;Lq*N#|!|jnhjJ%2$d6`=W_h8;Y2I6Gucj{seGY z`Kidw^N*pDTMY^cG}E8{BWC}+FW()&ealBrj?8_5J?a}bl+5VNc&wHX*IQl%G((f3 zp&1~9uSkVDIQC-tL@pkC%XtWCBoOp9+jc8$0FJILTe#abD+owlT`v@o90Gs=NN`t4 zpY`rb9B#_159{&2LTrtU-%<-Maz1uWKSX%DxY34~7F@qZAyrJ08FyHQ}H?#h^m2??Z&pAQ|tx> zAW3VtAvRSPj~zLx_k*u00AkX&mD2q6ppU~fQhg_gs+(vVp(FYuN)wf>+_l_ED*zD} zUonD@nXzy5Wy~-~5KdNzpm{?!0cqS=KSvD|H6i{7ur@!|nFUA=B7xperDIzKH%en- zdSy)iddtECSe;qG-#>5kC0~AUH!LeN027}Dp`tXsjkJR76+~?CVoqIn%98;f$&{tQ z>_AM_X0a-vl=H%nU@ONs0cA-UTK9os#dt_#1$PXI$b*$Lgm#{2}VviC)GQ5yU#wqy(sgN0qxxQ3;eh%H3x!h@u zp3i6ubBQt#EGZzKGy#x|_10Y9mh;P3yl*J8w!m+2dMh)QL=^W0fIhYKdI-PF56WBs zdZ;zXO3^7N2jdi(MY0dG%}cQSb{dy}ca$rseWcHLh{??iH&);-(0a;II{2?IsmdAI znHGLbjIG8>o}}>yl14{a7*SIHTmvCO!iy&gTU$ZjKkaitd^^fS1R(NM0h^Ru`#pyo zawr#r!%dwlY`4&`FgOlO6_EdRj!G=)ffoI}Y@`sx_Y}%@(jR&%#@{fq4BTQ?*c>NH z)3P80E>qP9?A+;-hYVP-Pfv0Pjy16uy4#YQ4^i{Bgy}v%hNuSEOyHZ{(_`Fr9Q+B1 zX2UY~0dGY}Dl|x31BkVSAck?JAZ=bFJ>}$}(mY0=uEqi^(7OvgH=d&-Tlp3wSn|so zggG$T`04XoAcz^VJUw*nGRzEI>7uk!?{HDy2x!bt?!qt>z#& zA7Q*yLu4r?*!pz3CaPTR`m^Kk^!_Xd8x@g({p zHA2@PCvX>V?YBg+elL@%vG`sQH!DF<14PE37KVR9} zsKtRihd%#w=J@FjW!TPXS@H|ub|*v4BkyTbp}+5xeGtcw=F(uO!D|_ieTQ2Rijjk# zfHpyxQ)E$3AekFlw?M)N-4E5`DQG_#x5>4?iLehNX$p|cy{XSP#8-7E3jb;_8X7{n zQjO&hZ9;kc4HgA>=gFxml!uuUf4aOw4fQgGGE^yocD4hTj(}AMx}_Iu{s4>;;k#89 zm{C^Y^oORb zTR^kA3()ddzslE#9toLp!X6M3n3XzX)YW?TZf5D zKxQA3dD)Z=Atk-B=g^THG9Lu1<85CoJgx_~ZFIck2z*pSu#u6h0RRaS4LmF=Hf%}b zO#^=eOC!H=2d40oWxhWA>dHfXxQBqfxO}pg#q7*Oa1S88g!0lS6RsV@?yQ8*j!5$w zXi5MX0qZk77|F9-ZTpVNtTiTeEgX>u3u1WlHzd|NtRl)8_TG|uoTUEcuU-*!wa;D9 z=EoaoKM$L1!xs#J+?cIGt1(MG5X2iM_#bmMi3Pc_-cagS%z3rOq>|=5dilB< zsXB%eeFP~wkn*Ry}nhaPRJk9r7{U+s2StJc@8oG zxt0k)RvVs2^NdwBT<`sYjJep$1xy!MCm=SZstq@bj!X)f)5`IqCgxy$l`LU!Gq)Oe zd%KLx!(CmX;(tRPi_6aMxJk?zs zvBzf(xvKj35G)F^hK7Jy_}bUT@I_Kq3OWf-JthPI-p1-6t5Tixx6W1GWQjL8*At{a zyI6Y&o8|%ZL{0^QmPops}qi5%8c`*>7BPIx5Z6KVNkz?<%ZcKmwVCQ~?gN+oK z@?hCpBoh!gTq?tNsN^N2ouOCnAS4yCx%A9Fhc$=Hi|*?>-pO0<`3wagsmE#d-C80V zg3^jZ^-npaP05qJ`wKrb9xgsR+-9cu;RZ$!)a=%b44gU$;A~1-&neSwIhX|)4ycp6 znv`u#(-j|BOi%CEp-7PWqx-a#>`y&b3b5<60)_5fA7M`ffl?y5k|SSrdL{DFZRVXX zttG6tqQZx2I=VyFz*_lH7pI`4;rtiK=G~J5LRJ$|p z65V%ePLBpztvIX>;;OQ*-is29Hh#6-sTPFA2Hnp35}t9dUjhvg)X&x!{5pWF!q)ZL zskl!g4-%FBTCl|i!mt0iU=`L<^8AA`auWU5BPeedw>%o9g0v54Su<8PX>5~UrhqaC z%1Y5=c+!JH^J?hAUxQq*?{^HgIigcb^<3hmOTB?89C&wh*=n^o7msq>ZtOmqU}U+v z4V5@`2|}Y^;q?3{<;?#LTwa#kH#db$lJ9JF4L3P>N$lmYeI_Su#+x8y+=uQ}oMrb` z2bq5gjAi?BbRRp0aPJ zQ3Mp~sSE;BJ%r}mHc~BP5aOmN zDD=2AVEJ{@FNIYq>?-7N{*b@fM@CwXcYAIDO{XtM#T5zM{*G1UTVnQ&@L+M#E*|6h za&aHc8H5??W)!_cIm_a+ATY?qUT+CU1`Z^1X)A(=^(2xFQ&NC>|%qDq( z_X5qBiv2}6XQ)1wwxKyC#YBjXQ(P6la-f?xn0d+N{R=PpB|90!Ch?6yH-5O$u`?Lm zH=#OCs8dSCDbUHu(lAr&`*1E!s4H7h$I`l`RU^WBkyl5T7K5}6PEKtOE6XGKvbnhS z5zPV95+ndvB$_(oybhvE^{LTmc@F5rny8f0Y~3lRQRhFzJ$-v=1uOsuZgJB#EyfFk zo38Kb%z2g4l2wAp>!G7;k6dFCuS$@L+ddy+wMM>AS2}oZZ$%`)ek8P-VQ#m}$ZdtQ zCWMczNcCm;W=vI{__15T#(;OPueJ4+Lh8qf^WuY-L0op}!oFu1NOeQvRKC;(0hr1Z z;O4DXaLqs6YBvu1Zq17XsvuJ4vC;8s@ErkPdMR_Kp17{H-W|2EF?NMC4u`F19T7Xm zw=Uj0OS{kgKV$nk5+?Ry$y}E?&{rShxbl`8?WNZDCRg`(y-xy1L3XKEy5t`A2n^C7TzWnrQ z?bu0`YNOs(G<@XUV>^#!9FKGN#W#BS4Mk}rjZ3l=(Lz=v>P*h^=4VJ;ycVn4+RCTIdOE_u zs%RPvE8}?<*$*v@IB`I_?3umw@ZR23QRi@u5k;NGm`p?90bmNX8WtC=CpeettuA@3 zkKGqm9Hw8LQa#$O=hm1`u-yed`I~)ZckcBpDsE8wLXhVaN|+H(r|+r-A&Q9Ru?1n= zp>(8?Y5nl-{A&gs4T&6#hX8=soIXey(wzD_W<#Nd-2ZAZk{kn1-Ka%kgsYZ4j0aBU z$0#^ylkuU@<4so9%C=umiED39O!h){>DiF;;m#Clr8Wql;OTw2QN(-G!cEGV%#YyS zs%E89L#I%r*4;3NbS}ngL-!=3vu&B0jP>_-YisQJ$Gnv@Z^_jeXe%+kePCtXpx139 zckJ3&21*9-{szx^&>uHfc`#GK18odwgHBaATY+EW9PY4pPjX=6BiA(IbKu2DG7dcT zrxJ;o=DykeLF$l;H+NNmOGov^Yo@qTIf)4;1GREv7QdUegd%(00-9_O?&PX{NAA#l zlJT#onMB;@YD1Ub%CKdQf21ujQd|Mx|1Q7C_M~R_iMJxlWMuq<-qVuZ#!o!x%8zQS zQZ1;eyLL-{&4O^Dy5vo%!DCi;KR)Ip&7`$4aC9E2u4Beb@%D0dDbU^^eag8Zpjo*d zb-}id)D+ng3@oj!oJ6>?9QSexHy$wGC{4eA;i=yf&U(#)n=6^iZ$}F>nSaJO+8`N} zQ@`&-79p83i`uzi+3AFq+ezK{=YMwGqIY2o`Qcb?FN-We6qB!_@ehU6&1c0}5~CWE z=;Ja{h3`QTULlSF5X?7)8E>~Cc3MWRPh2DZO6P7<+;n}^vPaC@Q9`M$7^NrEl4sq$ z0T`(D1mX~-^l@_v&3rUuLyZV;zdT~jr~{_+$ngSm$jd8!MD;6kp_lB-04WrJwdIW< z#&dp@B%_LIs3fh8wEUsFLTZNWHKfYyiLUp-j`Q2cE!l{X4OmF@Gtq8wr)rkJC8FE^ z#(1n#DbM$fF~q(Z{j`F)5mk?cG40WF43k8$B3~SF@h80=r1IXsWUEX9 zQjlK#%t6E#!1|Awnw(qleV$B+h?qoA^lPVnzwjYEG#q@L-2s1pU8E7gP^; z;H+1R&Y0R2#Horpk47c3Y#-rDa>*Sb3xL!D{Z}qagHn;H0JXBuvXXIf!wFn8&5Qc( zjrkBPi!Gnu;K24dgPV@j_O&r?AS@~n?$)#1?IF*LLlc~~f6RUaX$g76-=^A4PxSY7 zWF+(y_{w2WY>^K12cK`e2a8cfyW!zMKc3C#w%;lAEd-xZ`DqIjPs~r9h9Je@-a;a| zikzFE)tGfoymrrWC|(l*NIP4m+y^$i!Dqc=1bNF~6Mg_5gIC-^Z#Y%XZS4f@XhS;u zOKkS~H$p5fK&HS8zHYg9#j*VRNY=nWGIbZ`Q;_t(gYVCd_JQrxQZnc1Xe!U8+hpS( z!w}dQA^>7ebBoE7_mM^=V)CVpD{McLGJ+BxLJdLws|jb5DC~mnSF*Qh%AH93B?GMq z26Y1Ecp}S^zu5j#8W7qt#+lGD=SM2JZ0CVBC2}K&Q;w(89k?DQqi6i`!oLjJSoEd~ zq$^F`$bBVou&nxHhzR25bB2j|;Vt`cgOt_JVS6sKx_v&^hLP@2k2w2k1Ly$MK)P)k zG`IoM1XD6MKy=Q7rxHn8^`%JEkUPjy)aK>f>nD-IpOXMHxMSH2_S6Lm0bC_L@uMeZ z>v&y4u-)VzGK}Yxp{FM-osLtXRY+C%PSzc7c}j4q_IAzKS^vgZ)yL|MUZ9?K%`X?! zy-82GpO{{T^vPgb5E{!HujT$c{2MN^34vU-t(%YK*amXUuRK_`r~+iguYX!mm0|$X zwiim1X1_mHrfr1wJZ;kguX`p70Gz*YNu>IHf;%#|*Jil4zdq~QIm8jh>_0wszCH@6 zMubUY_LJxrY+?moXePf=Iye*tG%%i1<>e|y5>*Q0kyh6{Uo=<-`R8Dx92+A6bj_OG zYX)>Z#v7daAN#FUPasN=V(|GB?Oopy8MJ$P;%km^{6P8Cc89|pbBVM1jhtj8$&nt0 z)qXBOO`nOZEiQ!Cx{7~hzGH{|sDNnbtHpdn9C3G5lyL(0l=`~3KFaJiz#ag+IZcfd zL5>37rN`^!9mCfR7ei;g=EDV%E*O^v=NpQYj*MT~`FdAZTJ95m!_MdML5^CbOh)Mt zQ~i78N%T6VNK?~b&AD_So|1mKD)~c+tD$3lV2MjnVNnI5pMbn{mt-TnW;Dt$y`k(& z^Lp3|NEVqbRKo)UVJkF-0v9*ePg7NNz52|#X7X3t9*Bl~;$RurBLpZA>~Ud;D}Z`? zU&iVluL&LQRa%;`ZB>tx#8DV0%eY3&u`Z`_Au__t!;Z>YBX7OZ|twRLNC zUqgYR%`QXqGjPRxH7G3BD@?yQkb`TIe$k8m?5#1VF%&wI8dv1typx)6PeZxsG(t57 zlqirOD{i7R1EezpfCmBh)NfLXM>uOV@34E+SC)mGSG4|8zG4U(h&I=@%|N}WcWgd& zKCoySF_}bfpz5lJ9RJAfQ?6k0Al=GbS`Pj6CU+dIrt2Lxs0j|ukdw?WL(>oxf#=?8 zE#o20m{RnHJ6^8KPjnUbENq%_IVi%rg0?Q>W9Un*l`gG=N^KR5*fUZB^~!=X+kob@ zK*A98BVQ($QdrzaGe9WIaS`aOD!zhKHjqu>c-;;(1IVOGH+T#C7K%eyzs)^j4in zRK@UX!35B(it~HqtzWlgYy1JB7xf#he)q*VcLaYT{Q@7vx1ResVNNZKJ9S2JYAsQ5 zSv@|h|AbjPPL@uE^{2J>%dm8BzIw$mi{P1>F*_`bDwleXT@lO7hf;eC-7SOMM~j_ zQ|#tITs!Tk^z&>1xwg9OVM~P7_0H~#Xk~ULV+%6rPMBSx*=fU-HQkr`vU#}#zQT3O zayhE_7lN>!p|H+4TA3a&4!3F1>w`|u{18n7thCaIBky5PoX~AUg{Pvf2|3o#53Is# z<|X*~O>d|?!>A=@OSHK9kUA0aT;ffZNS=e}64Dt2qy-}#Ze@!*E6)88RsvL#xkfv4 z{Cwp(1-2F|wqHC^>()S52Nys~^ApDOy_5h8kh?Nzf;&JmhY$EAJKw~239LD zvdV>^Qn|uAUlSICOA>7+Ut~pyu+fA08eXc#P5891O=R18=uP}Y!j6iH_oZdIXa}S5 zXJ`-!ta4?AV^O1s9E;#i7SXXVRMZ=yT;5vvoTVj2}X)| zW&s924}WwLi!xlPL)lMf8cdp^=Wn^~Za=^E?V=^OW;E);>7n8zUel1_uLf8D<^qU) z!RYqTJ*cI8fgTp1J(Rx{n5q_cm}F9?Bxeowk}d{R_7IE%@-1Gy(D9|tyT)>K1%iu% zuFqP3jSO9SV|WPVAtvvJ<-F^AwTR4UD~VTzt1UDqe)su^6nJbcbRWL33Fq@zq0dTv zM8Hy(S0adU@HYB|N`xJD4!-+<{!d2;0uSj~E^DVal6A}2`g(8rT*T*EGF@J#V=up2 z7)Sq8=7RV_LoXOraASt%C=c@T6H`6B&0*#ZN}3C+*=W)qAmI1I6}Kk+uX^nzC2UJoouVY#ezO?o|E1jftoAcVN$fbOvUdASBhYozksX@f*Mni!$jsE87oSozAxcwoe zjZ96o!a@mG0*@skJ?kz`dHK4XM_u>Ah6e`xKO4U<1`f)^no9){Vm)_@G35>l-^sOM zZ%4-~Qy(mI|18Z(&{b5q6+eJVvue?*Pbs3z4cFn*>Z3Sf@mD}QA z`dV63+-DIUvp8Gx!?-QWcp4>Ym*hA0s2%OBd{*_t5)~X$H2hct3oJ*$!L9X;C*OgZ zlEm!RBnbMBq-0^IMBBxSl|m3Gl4>+Gn-tShRq zm-8i=cV*e}eShck)!mr%2h3JiWo&OW@1xxBLkT#Ez}#5!s)jw)VCWYq#4gF6 z+16FCp3PQX8ZY~RP;7sFe(`Sf*+yZ{5dj_xLWZPS9i(t;?|foaAltg=WsngZp2&i? zPG$Bu#kjOM9@AY?1N28nP^~t}ey%_d&HbY3}UvRPEj%jeq_F3epoMj z`R5JKXQ^L22Y#j}ch_@C5ZOTe9Xb0QWi$*&w!xu{#|re`+qy(fLWQN_qRJOU&Iw38 zxjG80;RKgf7QE(ebk+nn;s}afXuDOrUH)sMD*$)RFZ(M%>SIG*4c`n1GtO?oJsHi* z@U_#ePVwOEt@`WyN<%!{J`3fTU&IC?9+EJ>)nYxHn0R?hs7cwUZEE|OFx#Q+<+Yrt z^JUTVylni>#{-?KJ_yiK-Waw^Z8r;HIz21kYq@VbP)5&6$W^K^Bk(DhK0n zvZ@ExibLqz^us0lPY|!J0t@K;t$yqh!Y&IfzPaiR`gg^gbs|glY5Qga#B6$p8p|%? zBi&naWg${QA1cl?>tB!?{Z>C5moePIWTaV=R6ipuVf>mCL6VUGP+HC4Bj4I%4aAkL{Ybo;HM6qF%x0fo zYM8Jk?Z*)_?$)9UURc%}foZ?Ty?$h& zFYTPCG3jpLKF^cuSl1+Xe%tXBmaaFv;MELq<=5U6WQe}nXD_j&IG2$f`th$Ch*RD) zupNK(;met?qcHjVug|6qLWFRDq0X9*t3Z9Nvb)~6uFO0S8A66+I(ofPY(WqHcTr)@ z98qiS+q}&+)<;^k7k~28E5|ytJHJO^UE-QDxwu)2H7$5~gu`pEkGn$8DUx{b+VkF@QmI!_B^4n;p;aK0inG0dTV>p+Hh zB}i)9Wa<7`c{ahl1`UHHaEtxb@esj(>XckFJ{bU*G2Ezg^x$_#Gal)u_3_Cj#t@ zRn`9WaHG6WQ&ze|C;2y@)~$F*!xiRxtIN{t(_Q~ku!1@Nct!v7ANmaTJ*;_v7>j=L zfuuN_4bG`+XTM}t8?>9~t%lD$Mt0yop9h)XZXa$uf)*thVc@$S!nWz%&kU$uW`!|{ zk4~o-|LZ4CH;jI)O(1n=P~g*xzuC79WFwx?33KKBJMiJLlEPN;nwxIIQvJuIk#!=V zAw;qU3?OYfZ5v8C5f%XmF2w`6BccQZJAx?Q{_M}zI=~G&L`Xa|Ve##1o6Jl8H9ut%q@M<46Qz$_55gs{ z1Rm2%(5V#2V^vxft?jU37X2Snvq%^W$>H-ZS^xG-(lS7ByI^1 zm-w$$Jb9gODL$wde&tWTe+m_{bwv8`@ppILf7kr;eYho%A>|O9xjMR^~KnyF!0YDz4tqk^$QKh1s zaR32(>>Pw^v3r!&7&%BxtdidL0aF5SZ;~gVtF-J+`in+Y;YJ2H;?Lt{M zAuSAQftp{72@WXwn&JF%E}3|4TV{?984W+Bj$|{;@7P)~DEeh1`FIDMGEaT$iW&)6pQB}k zX|&MTL*_vmz?d!ex#?LbW&BOP(OjO!=yXEh?;Qe;^EnG>8sPwJyWv$N$KxcrU)2t2 z2@TK{xqP@lGmopXi_T`YspVcrc<)P~EW)V{el%CEhwwE*Vcz-^?>_-TyK>lXxuP+&K2(mO=+epJ%vOHcYVx z^nJrnO!Iuwx0QV(F-ifd0BCA?e%E`K)a-VWp7HHtP3>EYbt7HOegD_DyRJk@HdCcD z$L9sIkqzALN`BmRf$mC$G}tUW8L+-+A;lSzx*2O3k1KJ8twhR>6)XsW1}_W$l;5RB zg_9F>mEfc8q?jVVV;UepP=g+ZEMrD7yr7bUODp4fJCHI!`^dKV)_&<(HvJ8Plb`9b z@c_*|TyCNbIR809608m@tye#gJfF&$9Fp~!ezqu0&}YInLKm9up`do`-nR4d#_g^< zIlYLrfe5A&2EZzfNO`O43l5xEC62$=p3c`T+;aMm8j$&0wXhwgpl{<%9WqhG7=?Pj zH5W@lQ@@`(8O}vQ77DK(E^BLv(vzD+Qr8HWw<4h-ndLHMWgWp?l0za=cU9 zfiw$plHPZ;)U#rQG8^%gy4kFd*Qgn_`<^+iegD&o1`A|JzO#fvP;dn;$qhVaITlKW z7P-`qwOK|8DrY=DFnpMEWp&kRE@C0(>i9KFroKpG894E#7?1@U4#_cfYj?OnVWQvY+~rlHDQX9nfde3O zyIH1;QFya#zUf4EuUzxt8Uz!I-6ph-Z6}(oWbRlwpc2YEyM3haKZch+%0Iu_;52LAN5F>-LluiK$(P0 z&{i+u2Bl}t#rV`_Pr(zTx}Eu0yjGHh3N?K#pXm(#O7jJ>r-Y? z69U-?u+mV?=ST7DFKcQpj!~A~Ii5C4y=3RD*b3bj6=LGY+iQlS)pl3!J;f^`AYQIE z`#Q1Cs)mlnt;s3P?s2|^~kawY2{yEq?-TcbgasC*`pROj@qM7H*P~_@%{C8_0AZM zEFZh-DFsr3rVV@{qz5c%D^QoiU=27nGN-r zcE%DY$k5rB>4vXR&W857KSR$^S5h3?uq>65m%imh+zjPdYCvuc>?<@;2Ny@nV8 z8eE2)TN}Jd3|~wvASqWD&tbwws04ujj3W}Q@Lnn$_YVM*BE%{pW6s1U($Rz6m%{%f z^{#3%U(Q-nJP8+6#7@_k57KxClrSuuTfpN`r=jJEn8*IoTs91WPLpnrcr(;2GXW<7c{;9f}9oihqTi6V5u(gX^dogRF}` zHX%)Lu%gh*t~68fE@EQVT8LNVVqS66SAAS0xJWd8Y!^yKqpw78IG5dG5Ffa9y#G?Z zI9&A^t(?ehxl^*Fi%q1_-m8T(QJk*z03B&10tWDzI(~<_4hUz~y$G*wJQ)op!k*L1 zQPl_iB3B2={*lLjR>Q9ya=rGT{oPpJi>wy+c2}>}#stt?_9l8SE{+Ua4bI2 z1HTGr%55*8{AhxV;3Dfcbwp~X;2V!{PfQrNKPWNnmHZhK^(u#ahe)yQ)blk;hE}+A zPVSJ#b|@#=e@wlfq45;%e75GBKUOKY z&9Ix%3@hA%qEoN45Ucg?*C2nB4pe)=XoTz72X6Bqqur~lU!$Zi=8W=P2doj<3e8xJSF z0qcq!o)fd+pOwp)fkm6-+&J9gFyiAHDEU(Hj}V_sbr1udXw1MD^PfgsZ)HvQj6WC6 zDG$ECVNDuYD2nWod6S#gpN}kDH+o-w_tPye9~s-pzSHYz@Qb$-P3C_t6*etg7SdO; znBOtM4%zs*2>$sdqMNWousoq25$TDjegbfOITY$nl-jXoJ5g_u*c~Qmi3LBFnA?X- zufgm&b;tgVgb=_-7#XYDLa(&`+x5N;5@4!c8G({&H%WVj{^xj!?#siwWgeDJ z--BiHD1ITX|K}qqoFK4>>(T{Jgibf6r=DoSe>;01p_V*vxHgz?@SeolH z-kccG|1qZ^4A}2aXY7AAz^oICK5rTQiAOA5^###~|Fvd)4<=gTg6YkSQc;Ef`Namp z;BFyq&3^qU_x5#CSi(mSe*Q~%GGXe(g~H;=B;ava@s_8a=!5?dp8vTdFT=j`YYy8S z--rKkSOy;>t5W_>e6bOp7*+lf`p>NY`in;nS=iHH5`uKunkxUfd_*ktJl5~a%SX-% z#l3TgMm;0qA^)jx;mh9#;eSq0#&tBg&QVOWopchI&~X3P?th7Lz2uF~ot+*2R{pvS z$qE<|ag&$0=$~KI<-&_~Fvy*Bv~rE|h85hklNtDD(kFlUMu2=xLm?FuolLyD=kNvF z3E4JVJK9Lq{~6I=k8+X1HFsZ3x=bsah=hoR9mIj-3C^Eehgi|ToctN$oN80-H>(P7 zm+8O=cPx87Xa8KZE2=aG1(tlJ?)kO_Nr+TUl%~i3xyS27$5QpQyWXX;sZ0zYtloP? z?tOp0%PG-^e!j6tTr7nPS-02W^O=9{ViX}r)pxY>!WO(2st`FBx&Afc&+)T_!Q#3n zTi%@B+#^*~O`1^(K+$s~B9b6@YyAB%;Cakyv*P6ewPO%^;6rl$Qb z(8@OyM!+`oEAamz3NEm1HK`5PBN5>4UEt5^KSz82c{?z#y&QY!BSUy|MvE4ah5z*) zV*`Cu>)2E!hKT2bRrj@jtzXcIP&vQBcPGMjDu7XQ*^pgO`eS0glvu|K>2v<5*hmC& zL18wD{;7~M)L?4ZqiN|k*kM358sj$q5@8B>!XxXccrO#Y!wU60^t=>5YzI~d&i`_jj_)^qSe|f4?XSYHoZ{mP>E(aQm@+9C zWy1#!{*%*F z7+$Fr!3Ft0*Kitc&R+dk_x7iV+GlnD2`=_O?|{D~9T1(B$0;8Bf7MxM;fWndde6{) z1T&?St2@C3{&|rkH*EFl;@Hn=EBJ~IVPgEh7clHtdWN7@${y%OOfBnSsC%#kyHbB{ zhotXy48X823Vu4d{QvfK|CkyYh9w~x0DMp#Lp`>{soYQ;XNglu*}jkwOAtE$WN z(Pm!v`f4B!BWM$~{^jofP`Zc#$})(A4W7ohw(TN!McI$7`+v@2NdunHpX8y^PDDWZ z$}`{W|CEC+l(DG=1%D(<<_k+h>!>Y)Bzb*TNm?2IC^jM z6?q$T-!-a#PW2~nwB6qGhc*d9tv^+U}>XrhXxnb%d?L^Y5xRUU^d~S2k;_zJ!+oct$Qe>4n}+8!XT_2I zQWI05xw3|?uI@{M4D6Y(b#)pwpLZ+Lm)`ZWO83^Z>4QU4_obn}b94_5!Pnlp#5&dNng_IfE6`-X!s#-5gq_DZYiXwAhT+^l?)mv;`c- zvi_$AkYxkufR7pPhA!Ts@ppQm?uogGvBboQc*NVPrZ`Wa@^9_|GutM+_U^d`EjVCH zX{TgPeLj;TeAIc02)lb6udpx6xwPcW`^(++cws$(41PEH*woDiJ5Jr%wgI`#SVt|q zhIVZvrwQ+}$kG|B72=uoXVtWXRBl9tQDx>S8qJk7=;yo!^!k%zP;sUB?Ew=#RRPb> zu}lQb2+!Y+DwOapejvMM>^nGAV2hjB&O^z8d+jr+8`I$|Z-xIj4@J*if%DGi>C@c9 z12(nwVwzuGF?dUe{V``Yd>mW-V0C z$|)i@AZO^(O-`*&CU&6d^mik~QR1lR7zvwJsyJGi7KFK6fVuAFPT>%W+^;{ z&n(Mmu*`)YAM=d`orz8-Pvx2YE9w?J4{0N8G?oO-cWmSy8VB^Vid?J|KGI^LWwb{4 zfr_Wq)~EweS4Xh)Gs$A4#jtTSD_DO7#-Oou7L@b!@ySO`sO(;#zAAPmOD&^JmVu7y z_513NemE{Go6~mYASKRYZQni3(=H}%w?Ua`GhYitBx#pP{IEl6wJGVm6UlvFcRWiM z3Yb`sraP;j!V}=Qv{25_M(l91@m)8<4sN8V{z%OIQbQqV_{2 zJ30Ls3}g&L+GTyWmFI&L9m5R>2z6pN*9Dy-jA=Ux7K-nkzbI(8M?z*v`de1^8D5ff z8g^A2D^D7{Y$R^sQt}vH{1O-MJ0)W}w&?QjMM}e|_5BoA$j8k#UDhHDEvSI6p*xmY zg4cB}^O>_8{@!IV^q_CmH;96%GwozJlBfD_hyB7o!&W!HJuT;oxk*U)E?YQN%=FbPa&PKi&fn4*#W`xqws<@&v#j3QM``V1d52^r&1JX#&rBnM-LB%VQ z09n5cOnBdW2t2OxTQjQ1F9BhtdWd0I?-?dP9piIP#AzkDSxgp=CMaY4!tRr6cNNKp}E)jty8FgwHW89_JV|`;QEjIZ6zk&m|Da&s7 zQ*XV#5jJM>-dmX=+e@LlWJ)3&oTh`F@jP~eVUprvACwKAkrRb)NInsYP*^BX$n*nu zT{c#oCYEo=ds!ZA37mj63gSw;{uFEmj9)ASaO~(9)M>~NVW`?3irpznB)=|q6qF$= zKj#dMjX%!odJ%uPDe3zI$A0xOY%V)5uZb^r{=|jHyk}*f9nKD=k!qEf#GM?bSz&tfCb9;TM~)zCvAsIB>Zl zxeJhJPkk@1s`&bNz)>he?=G?6G#TSGLx9eswf;(sRhB^~w3ip|GEGy4QrHjre)-RXO7HQ0)cvQqIQE%V!HWL0OwTniZ|F5Ad|R=&+TNYtk&*O2 zuOoXRs|Auh1Gd+~^jWS->=J8Qs!_47cOcEO zzMgw}$)8W=mO`UDqs5)6zNh>&?aXN#YV~cjm3{!A&Ao<3^-O2@gY^|RqmaFXij zQu8&JQ%S~g26rc*-Usw5btz50RZwZR56Tps%+MZ?KtmwFhd$4Da(*v9peFRgr)BM6 z>e;7RFskEA;p(Y6)keW9O)JYNO+ZNAs3?)R^~@NSjfrg?vV2^AeV<;V@*KyCV##Zg z)oO&Ro?Tz(MZJKnWHYo;-uam@meH4Coux5elXG??>C_C}a{|}T=e!snvE=A2A2!S8 zK)7oQ5^CtwLn@wyuTRvh0-mD7tXHJd1jWk{zyyQPrkg&-Uamd&oB5FgW6SzEsXH^2 zwr!#qZX)Py4;?iN&r-ShSmDW!##1pzn?Xvz?z_lGS6tTK54j>i)(H$-=q>mG!nlwc zadU!_(kqoLX=&Z<^wYKSF^;fPr03r4(y}29EcD9y?*|NN(z79zwJ@^U_s_TW6TzBx zL94Wr%>k>q(pt;795o(x+2K3KBl4|Uv5C7Sx5?1X^n8=S|Ea*hz{~4Yx;;YH@f}Dp z((`3)OX2s`THul>xvx$L9VsA5_eg}OhEL)n!n4glR6jlSK}%EUEt}7&teUTlt6T&V z*|_zw(mf_`F1d<9U}@JZL3aakt#Db|-JaQb|DMu|ODXIg6~~oYj0i_(;&ze$Bvd5X zoH+LZ3fG68J%LpFASl1?&R_T<>nnm@FW{H&gs8W%#m{2AZ0>f&L=jck{+_IkL58J` z^Lt}NUC>`!Rcmf?eZYg9IM*stNm_t~1GRY#yOc}}s&GL~9|)BK31 zyXJ`vfbe{p1$Wpg?R+O^g^bN3zJsy4)|zgnR1@6^MetEne1gs+B^;0>W#blmb}BmW zfwhG&TOzC4b)WW*oxEvc(dIYYGU!ZPa-5o@*C0<(%Ew%kAbt)5?|KfdnYQ}Jd%jv) zbLN?LkG`jhacs?BOn_OEpxN%oXwmVKYR^fZeRQR=Obh4kf%DnXwn8g`J0BqgftxI3 zTZ1q@rMpa8n9Zfwto`s%@i~%3ViHH0zY2n4zqGxF!hn$n*nm{*U{eC694+CSSXFo1 z)8pJpAw}auJ48wJ<)CDS*P%91&n=}~y*n}h#gCQtXm6R94N}+KkqU+6`jk^8#gd9X zM^gI|bKaW4iu*@%-nR=QsyGqcf$qJBK<}{XOG_~!es?J~@hQhk7)0`z_T#g^IO2w; zx*6GYL6@syc6c5HiY7do6E}z18K7}}GXQd{SCiLsJc#wnNHM^dBTz|I(mZg;CzU`; ze_)0F>qEy;LHYWxwqpJ!$0BNgA%4GqKF^qboifrFe;kpwXFv;4-vRqNLdyevr=A_{ ze-W!reS;M+3N2Sx-e(lC3~Hj_O0f|70JJp-g9{=So-5npw=q(<+QUo9?luBD%j;Bg ziuja>wKF&zcw9~&(HpSD2R_ejv_nP)$L6kwC3dy$Uvnos%6r*5cT}nUb#nbeG(=$z zCnap4S_wL$pFI+XFyvTG(26hXpbaW}w!V$~D5(BU+J~XW5@)e=N-SKk+o?ERjTorw zpjE~iaO_NrP5lm$CC7o&CD|F?^!nSWv$+f5U(V{i17}|isD6mNzleU=9W8rw&3Y2l zr`w8tP71`Kd^OP0Ih>Yr5*_o3k>VNTm;)fMM-gQ;i{V}eVW#qn13IR2bWLzC?O%Sj z+edQ8`bA5%)$DV2mO%>CL(V41z;XXI57*ECY45$msgC>q@sJ2*?p32S?T|i1F9RO+0-@+dz?>m^gf1X-H=ie3)(!Fp#Fh=FuLFRr0p8vO zKK^FY>t;%1gbUNWm0m7lda3l`KcBt#C;&WkFL>b(A7@@8I9-5zo&I#v>&sN8?{qnJ zrN~DV<`c5OMjdfC3ajJ({D@?A(0XI)u2k(GgVx>a*)NatrMiHzTgU=sr_YXfuTT(e zJ>P!*E}tqt70~F|huf`nGtUuETRbIm`_7UHa&;M$pO6JKI^#wbOae$pkVlCo!0O(4WT|UV2#eYdVsSH)C&+IsNd#e^J$P@YPgn)EiBR1WKNE7 z6mZUA=d;@@@ynjN3mg`=eU8@3o!=g-rCf4wDof9?pvVj-I1A{6 z9$BDdvG4D?regBZ1(vsKlesTO+bLX4SsYP(vYaEP6cUx-g8MwiylhtIN03)o#0 zA&$7e5LnMb`1Fk?ai`iq!MAdB`d0AKQ{}#=i=CF5D8|u>7v{K3wN$u;x#sP;vUR^0 zKC$8MxZ~H+qY?65+FIAGi#MB&3y%*rEp4i8A)YzY)|*n$#juMx%DbzbXZ;~2v=Me1 zZFGHRO55TaDvhLH3EDk1ADYwUVhGkUfZ%E1k8aP)U0gQda6jA!&2%@1yMPcrwl3bi zw8#TgFpWcdsCFWB$mumd>D^1nD6q*2B3_L9efIkxewo)9G!Iy~c#9vCON6&(U366BGwf2S&oQ%pb;* zXY$+5pGtgp=J5Jwijab?C5FPTqgBz{*ZNFj$9Yf*;TV-h=(0&04ptJU(Rst{ACGZQ zP*SIuLAkjYrjx5AWMqUWnd2W=is|O-+0Xgv!9LPGGb4{QA~(CE1WxfMlsvk6wcaJ# z#QRhh$8c~u-JwU)e2mT+jou^99aQ_LDHfL#`CYOEC$>=;nF&U(4N1SyoWY3}L|*#} z3%iTpWN8&fM87mTd?(LzqhV=^&VZE$S1*&YdUA)#py(^0PHxs5IYdpYWvCBUh_nc~ zC2f19o2HcT1nh|1PH36X4f+0V)jEq*lvn}x1l^p5 z&WcS53HL(XP)tk|ESP!hCox(={db;L^u-dc?xov2#JTw24l3q&F$$>jZ&Psmh7oYR zm)qE!K3Bx@TyGZiy=Gh|Y4eLAnQ1J#=pfI+O|*R)w4}wIgiq=&jUHF!7ah3KP&tLt zV5Z}?{eW6k!$jdtMZ3iVLOE&aCpvdq8-+}%g%|JZIxY3^MH*zQ`>oquosATTJC2s{ zxSh?VpdNGInj(R21H;!_asG90An^=u+o9o_LI=aii1tz{0g8mHG=!b*u!64b0Zgb| zsb97l&M4qg3XQ*&DCQXNJeFKbUcP+1R4M^r=B?W5R`cf>gRp)kyZoY0rOQY{o#oxP z8hRqV8TdrL7b=flXSf-ClxYY`c z`(G(_F3~n4FMZCVw%fN~L!f+AQ4%qEa>3~MMJ*GVxRQ8u>;&blo$;+jA!e)KgKJ?@ z6{8;kstjdJ6ndegu$q}oyOt%g;H&%RK90=6hI?#O*lXFPl(;JfYyqV5>XR5t! z7PmiYcyL6lB4iX0pEMRR4Gp9#SE+0=SW4}Vz6`M!%xQKdHOf1_pwNnrv-J*fN!e^7Q z-Z&Z31J#hz0Ud8v#n4avtzX`Wcdpor7|$p#j(`665~@D_?+q7Ce6!v`7OP z$WHT~7srY}2(TO%gN_>m$yn;g3sP6RL%2Ah5mRp1l4PU6;xL-|;|{r2R}KxdW31wc zC|`_M^J)nSJ~1b9W4&9}al^HHT!2RJ6Dp()+J0C(>L{D4q;11;(90`(R+u{tUxkvI zXrh;GmLE}5&t%)kT4|7L^tGu_3#y>>MVbtm^=UH$bg_jO=xAd;ad)@Hy=t-jZKrFy zp69$d7>teQf)<9#?d+VcE}>TG__FyJ6J$8+@m<1w!$vi+zE$J1zS1H&kKJ*}EE==p zd|d6r=4GRd#X_I2O}{XBsH04Hv*PI|zbkc&bNP2w_Xl3ioIb}ytBVZYc_+)ejCidW z_26cV)7>}pp zC8e&lM$3sDB-7p}SLUKak?2KWd<=w5EBfgUe8npY(H=$3-Oxk#xAKOV5r`#M>{H_J zebV(cHE~>l)^6fmNSfMJ>i<+;( zvqCL`jVGU|aLjTa3o)4YjZkAqv`x9Lp7Mvu-<2P;mA!0}RO`5QbtW0EOvAr*)Y8Rk zGIpLhkS4;)BD_L?3eW9Gf54Vt`of*d?^b7cp0ll?k_&#bpt`LgcvdGuo9pZ*8|7Rn z&BNuRSqB1-vc_*mT&kc^))lvKT@*uSU1lIB;{VB(kgr6|E2bVYt7k&jsJ7Kc#;=f>2Fizu9kN zCgNk|Tyo5ItVUP9;#;lHYm(txQ?5%AmP$AB&ueP0UeNUan0!i7DT#(o+-Wmr@!gv& z=9!ZHMkv*E&Y;>1-fh_Pd7OS^BhjK8Y7qzhz*9N8{4`lRp6MoaQR2UOpULRE6I73~ zuqd5{B!EE)7v9eAb2V{kM%8l>!SN}21Z<#^E~)DmshZ(^btc#7+WC8Obe`qw-MYk@ zKz}re+=X@3xKx06QSP{uR>hb~Lv>CUT{h#x;%8sSBNRH( z44)I|$yw}RA@B#6Mad-*`-#d*kW%oreJFks*XL%&70on{eK`i>2lL~KVw~L}a~(Wy z2P-L8adaR5kXTP8?JkU}iHOOfm~~HX^4op5Q8#L5TGr+d3Cgjvl}hwSckcFd)vS5k zF7E#{?{b{#_@zLaHysQES#6FbI=52q-rx+C3mv#CJuVt3b28OVa$z&JGo1cuqPUJV@FTMbq2Ky0cuhcRK2CPL|U zwwAV;wJe4So-QrAh0Zl5L7Z?xgG-Wh3TF{sRl;IrMNs~U07o+?j_$Az=nZ^%;KE9q zbphRt%A&ciT#*1LDxvfC&jN-(T!fyjh{WKmOIM(c1;w1ek4H-C;Y&1l-6{y6N?%Wf zD3RZ2X57W(alo1DE!frdRn1A8G#TkB8^$>0UnicshHh1;b|8~*k~)??!ff^JUET2z zwmN$cWRxBtdwfLDf`wTx;gov>hD)Ot8)DJr_-Vzx&-Fgb-vN0k&BtpGS++;QBPZ$x zcOGNIbl`p4a33U1H3Q~hvM*?sa$mKSk$f)r;l>{=$L+) z=e~3E$dL3;4pkAmMs5LfBxxu7tLWq{^x`K=ZV6C{Qqz1QuV>EO$4Z5 z8=&7ow0dtEs5Bs$&)~cr&yYy#8GD_W8Tauh5F38S_wb)UX*c z$%$3-xT~{QMsP6A)n5AgED~|gg$mOACr>hxxu0OKRXcZFJ$*4IIH#Vf}X6V|@7(1XjfDj}>8jU6%@$Utm@0D9=9U{;! zodz{v_N#Y6<)QU(f1T2=KRuH;gjp0_Q;JVvCrIBWiNzM)$uRTERh&8+fw_|-iLMEJ z-!@M1M$&oA#w4YEs;&uIM9sj>kodL^nr~SLBU#Fbxwo^=0y{`T>uVQpTi6C80@Bd9 z^3S+mlyLy6{)qQ<>)w(`?qbXKi}ljy9TbEw^kr6^h$#f2i1ER1m~AoBA}klHj1v2N z#b|OC!uyet(lE@a+JL@-@kiInn{+Y<;igIYvlEO$Z(BH|fp6tkxrO(tX(=2}4Y=?8 zm;@j_$hV3cnA9HDufv$7bmb~c;OzfUpIP7p=**ok78)rFUSTH}ASy!taT6ScegRag z7irH!bSi@PEc$r%1K=bi{_@G7PmHz=mVmEHqhSQqWS`>uk-inCHHZo0;j&cgBs>zb zvYBSK+~3f}`TYC^^e~t2)F&C=2XUH15Eom3{+@=DJK?(i?+)e`QJAa)Yvz7d2-CcS zUW^koeicNTzE&a~>$g5IZ)p8l>im>oO=D$p^DnI@v(X=07#@u+q=<4x@SLSIqUh^b z3H0@LV6z57;5Pex7EuN}>yH_Zr{n@PzZ0X%RDnF%({Lxj1x~A^hnfvcEJongoTt0y zniT@{mcx%^3iUu~=jmp1EZKxR<_6+DcuoEi@X@ej@^M0YeJRZE5ftXrPVVXeeu7uu zZaf?|0kg1vB`QiIG1^`3dQkVLz|-KZZNe7f6P=3Mh|QiEeeBWBV2gI-PXoF_3n-z4 zn~a{k^wkAO>IeJlek)l-K+`gDvmMof*`uDDMsxzGX8i)jH8=N0?l^2BDFXiSx-N^n zk2E;%!j3}Zg6dQM^rEEb5{!{YbYCU&!r6+h0-V|^JK>AsU4(ga24#W61dPi$;)+s+ z7$0EZ2)HJ(-NvU$GaqG#@Tlxk?l3uiUpUr~yAw8C-)v8b3KoylaR-_SSUE#}WJ~GU-vl;D|>VpVwuY4ZwL%lFuzW50N zqRDRC^ot9Z=)MEpmHDRW=OXt)Tk5A_r~p|lrmY~37+8tj1OzZp7wwtEWv#u)uhn}Pzhde# z<5cojT)-{91Ejc;nr{%QAdb~km8cIE1C#>B8Ywa!=-%fqxgS4wbi##x0eoGYEoH@3 z9-Y)n)Vt&<^KY;A4I~+QL%$QThl4RL-S2*$DCz7U^7l)8K^VZ~?x2LMXo3tC)hY1& z0Q5?$$6=R7cThEw#NivsQ>BB-n zLy*>P(lmVE$l?LO(;N%V;Kis%EzBw%ivt)qx9Qf zHTT;cgMAM#4?6o+fjq(457|B-!?5?^aNpd4t17s-+-FJ9^1to-Dwsvl`pL6a^Uyb+%7~yggiJITGi+Zv#fR;uHmy;fg?Yp491PXm$f% zm^_~ZYbZ-SA0|>XXPvp?9x{-U`Rpq#S zJ#0il*TwgyMYAKM54^y=y#4CCu|f?Cpde|^(nMMsniv{3l8Ot+MtFew8$sOFk^n7l z#GL>h@dW)fS#0Ce#)Rs7>k&ij7u{@R--C0m2@K6$J=;2O=Q;J>dq`|h3YGn zKL|Gl)^r6-D-s~R_WUC`_f_(pMNmDX+nqd;b|*k}jL$3;C;)ac#t!i{LdF{7p!;R+ zoAioS!MtlwQuou#SqR-Li(-E34?b8gvJKa(3*W~42J*1|0@#@APqPJWfim2nbXfga z29_*@>6pjFYi~?;#eaoF@tH(JlTDH%7BTKLhe=CBk*4rzd7mBN%{7AQ5na`STbZ-n zA_uc{FES$rNdbSXKp}1aoyVB3WrlDn;@(FALpVT|f{KM8UHEptn_Z`br%4ixnzC(c zp(A_~Ms?rM-*Y`(?=>`ezEtDPMT!W<1_nPMiAES()c>plj6k9d*NaLx8^wM_UO06t zM<;S>(gdE~H2*o?kKQy#ql3;2GSnf1=^ifcX?iZRmkof=yX)#$(xcsPm84YlD}CB7 zeyjEHPpdh^LK7u+!3;QZsm%VopJ?f6D+5E+^TN*0k#S6HlMRn-72XlsK+2Ax6ncI{ zlZDLd{jL1XKgW3%I?U>GPAe}pxL3R9mj=ym9Qp{g7z=SUK{8tt zUd|_elS9Y%CJx#2=!de%o|kCSB)qSCH=vC$NK#xp0Su|-CTR0Ers}LAScbfK9vn=2 zu@H`X5yJrS&Uyp|;^c~n5nf*K7E%^x-7Y*3C_$S3y*8)te3cIhPxMUMeUoz%wGI%X zlXCi)J+OFF`fB9VoEUA4!uO=%C(;Y4>YBFGc9RaW#Q{q;3SgYFlh{ZhaqoWTHQz0I z+~=SW6asnvU1GB)?o{QW)4J0wGT$br*z$YO*)7#1oZU;?fiOkdW9UTX1tEBWK0zn* z0Dw51ckQBOz7&AqXBz5y&m_vA8W34>{BXeB`B8fX{V{A8XO@f*V4|*!JzS_LOiD=vys~$*>qeRiBews;7u>1rp5TU~6NvZ;aL!bhWTiYpLc{Mt)L)t(~Lzv!IK$=97pSQ}M~Q z_XR2>wx0#DgKN6+)OWd7VQbq|Nq)~QI?D3MYEwP5z6v>{+$M>Gy63l~Q_;eOH6xQR z?pE(9buZN|zVCmw*zwtA{k*U_s=e!FIlydUXkXUIHwVr+NAMO0cIKR)Z?ZidJ>tNB zGCJCt%nDFtC<6DL3x1UMX*`d>D9`9n` zwzwto@Uv#$BxcF^;i~a8u57dkncj0k0(AZNk!l%q#cB`oIWUyF@zAEenu)>9n;7jg zms1#!d2vMd7aYApXrWtwZyKHv&nX0~gm%^PZ*hNicVh%MR6?W&s4tr&vDg6ag9^RgIUpz=EKZF89%q(ITH~`)zX0=az zWc4N~giLInUYR$ZZjJsm?=VsytfZNx-l4CNiV_*J{kZ7=`Mt|K;MvnBEX6cwrH9xe zu>mstV|!<-h5zvhZnZI0L&mhIj(h?kqh03mMqM(mG#SwxU5c|89lZn!Upv~yy1sZj zFOlKA#Y$R|3N*q=rUt%Bv}|m^inTnbP8;gqvPzR&PczQnJiERj>g-h3E71%#H3}=f z0Nijp+xOk-S8nW8?r6`E%Vem8{Ql{{b2T3UF^Y0PJ_orag;zoojNI4j7ou!9IA1h^FjTMR!&v7Kw;*ckb2Anu zqe2DfpczO+lC)a4R!ye=_=hY(e%B_WkCX`vqUbjhYub3s*TyTzDP>*-#BU0BlgzgT zT|`o=B4!*bv+bt&QEv;5hRx)S=VwpYY4`jzdOYFNQb^{mzpJsA{$On`eEYAk6bSkb z{<-NLGPL%vtrLOVHJGxE0mpd@lCB>B^}!dru-~Th+|y!6I7BQu+2A@{V4`q1BIvYXfi;Gnze!Q{JXtj>KXf%gi zT2^>mn{yF_yVFxk&CDPCM1>%DtQI0Ns%4|U#?HGLZJ5c<{Z1cw+k7asDJ)Aw$ zZfbwA8JCtGj!A8T+enuA-?!1TMfUIqRA5o7B@N5T%8z{J&v$;<7V-N%U=wG>L4h!2 zz=EltAf%N$$D$<#2~K6IaXzY8!p!M#_MXsd(4wJ_Mo>p(F_GbKVn{i)UB)L~x9)NYzx!%*{`fT%*fJ>>9TkZYdh!libyBEk2Ns&fi3O%LJd5VyF)7Zl+dBh~D zs9&><>vJUOMt{8S;4l&xLV*e-+yZ50Df=o`c@tM>zjNsLca1?m#kwX^@;!gNfQzQI zYGb$7wCwj+cQ@L}zHco5n%*epd9c!PaJc{BFp?bUHI(*AznuY<+Imf;Jyh-+e15!= z*bQ-qT#8hl;roz7PjqfA&S>aKXxf`2n46HldMa>z!?2-9t)Oa@2I|>4#sW^lI}gp) zPy>aUsITt};vC}8SbVZ)g09I6Nk{Rr38Usm-}iwd4m!-Q7AYV-GI09^!Uj>Sd8EuH z8*zG9e@{TxZW9t8f2ndC6@Mep{$USa{M-GmYw3z?pJ&0YX8v->c`F?oAkW(TM68p> zyCMZM6*=Kv<(1mu&=Q4}P}7nkl>SWp0HPc^_OI%ps8n?|&pWuHpL#YHMb}Hzf08l6 z{4!yg!UR4G0;8B-hTn`}vbqc@=y|omst{><|D^iF1iYgJlAHE*p?dd`vdJTrIOxP* z)EPH5E^*vqze46K}YJt2T&KWOXl($WosCXXq-~*JB3q*IBn?dAPZpcn(8p`6m7$!l!st2x? zRTLb-^9+(u)_8LRqiYdTNP7EX2I2aTfuNe+^k3J<2{GongL{rHDrs~{HBXlw$!s1X z3r%vWeSMB&8Z!K6*Wv;SdCLN?OCm}+cZNPs1HKD6rxyl4zWem9!Wx42@Zww6DaP`* zHorkXFJ?VH9z4T5LD?!nxZGKTY#RX!@kX;2*yxJ?*3T2S?uv0n5Mq6k%kh`UyYV2+ zE#K$r2N19hfBW7v-UE>RI{_M(5KZ);m`4x7(+iBfKLEwJt3DCw7y6fUgm-Rat)X5n z6RLU1^0;HqlItn-ClOMA_3N=YOJeY1?3!r#RfC)#PLBUj>ANSdY@!svRnv!hqxQn} z;XKrX$<3c6ew&0X+6jcJl|KsED~i1@^{Uos-PS9?lvW zbxGA`0)K{KGs=Bi;wcQ6V-q(;{SqkTsd_wVt>w_Zq(m67Yw}%}Df4r8!;?mU0C9Ah)IqoR}k;)4bbK`_PL!gl4Iv@GWdskWMOXVGo?8 z`G`1_>^8eoeZVw|S^lntRzwpb^2YjYxO5(@8rf*->j|ehjgQXTK~QIh&I+rwReX;) z^us7TA~2JsC54LDj#qP|iC%imJo8HUDk!lXI~gun^qKsGmV_%~;a0CeGHxW4N8|xXV;ZjE5I3;x54UGQtlf#1lbZ4evBwno~zbL%^NY{udIW^5i@-NX@4am zHaUq2f$k4UaYXdk%*~ug_0I=i3|c9Sry%aKmKxp~m`wPd@iH-;2p;~aNyJo941U3f$=t`aCvbD>z<;F{g1+b) zK7->eglr-~y}5+eJye2=e)(kM3xf~NJ_&QCNK5(im0g^xWjrxCtT&@cEtB{+j-_L+ z{3jD#T>!q>-MCtI>0>+Mp6s{NdHNBR>&TivPG2X}0C~2?a1PRjJE-P;)qe{(=s0B0 zho9T!bTSus!yoyowjk0$?kp1jNedwUABlW7BH)rHORcn|giHV2ccuocUd4Vy5UBu9 zN;Upl87NCfM9v2eUSTrKt(NaxeOc;a`589jPz1^sJet7S+yv@Uo8Dw=rZ?^fbu2s7OF%AV2e}kasWq;@%pTW|2vpoRg!K8Sso>v@ zy=Gs@cn8-D8~cmmz)v*DMqC-3=yIL-rXX1M)(5&5_h^nL+;Fp&_O<+2Yo2nuvJb?_ zbjdo;vx4rPLwaK0U8tHUu`5ph*d2SQ4&aTMC`h1c?)WP$u z{FYw?Sis!gLqiZH*h6~s` z_H5{&VHJwJ{MQ9>`+j*?0YnhsQ;DK)4xJMR)d`MikG(b)fc?iS{UM2cm?!jYbXvxY z@~z46>U^0>)VL)=W}rFk!jj?DF6+t)6ogq@|LyF^ln~Ta`K1?cbf4d=bhdg|qVBmJo`Es=R{~jy<VTo&eN6A+uE=U^aL z6go7mcRf=q066$Ow-KXUz-LplO(lZrS(_TMDhlKpF7UfRLW6K6PB?YF0Xj1|4FQSY zI<@8Fwbw%W>T9vo%{e3|r7d8%cH*Zm4AYQdE7AyWD>hUn0%=pkS_zq4!0>Nl*zX|L z2+_-9ZrKRwfAdg{h*X~IIV3@$zahJhI#fecOA+;JJlhcLv2(;=V)O+VTgExM=j`5I z2a*xIx8N(SMA12f5z@z(u?MH#!@XJ*DMJC)Klc~p0SM_0N`=@G2lGPJ)<;Fd=fMl? zk>tmIZN@|yk1vq-Z$r+}TZ)LJ0;8fA{DBZo!Qa7eW+O7j2NI_p9hnV{)fyt+1?S~^ z#JlUEDZ)YJZ}UU6NmpRnjn~|H;SPk+mHZq??f#qBh5Zc8kGWPlW$Fmbn^GTxfG@;q z93JDeYNu#dsn{iN!)$F9N;5c*zs3wVz>Rmt?y`{j>aVxBQ2nDronPNdiAA8Xhk+ zGKQm-RZv^5eO0{ZG$4if)MyGLO=QEP!eHF23`GGP)>rir>Pj7)u0X&^F1P^G>_uak zmYyLO-HX(0{w#UHPG3aYrV2wZJk5?)^$D6*!NKVXpZ_O1#~fgA4%ZLNqUw+r~VzzBIN@23H1cSGkOtiBrbK1+b59YjVgT)IFMG|l) zf=&fcS7O{hvZpUB_bW9#KSI1G;thVr-W|usz}gtY7S$A~FH;V-Caw?akv`3yqETTK zBJRu^(9$AbRs{me-<`liVw}3j^^#94&sl_f8PZk_ni)M_=lFHb5~G-gkpEr)Kw9qw&(538)#MA>g z6>(w;_p%du{Cq@-k~TUF)5Y6}0kw;U{+ipi%;=FCVu%8~9H?xBLY|1;*Jm3FbT;2R z15}skz5asn$sP&T3B$wtzlIK_0#DFCU2Z2ktu+&v`<1DolHhYH+CTHK(}j!IH{G@m zM0D@Ol$9h>P6@P$B#HdEwinY$L&94oZSj)5vxrQVN{QS&UvknWc5s)kQ3kLW>;8_0 zFse`HSDa0K(napXTgHSK;#A&F{a}s907N@c2{$kA&ilFYlVN@yvCkvmSh9$97sPF5 zMNWH$a-IyT#A}+MRygR4-rsOC0j8a=a3d^PWs635)H~}YM)~f?r4&wVhP~@jQNy=& zsVtXOS86ic%s=va1#WL{&R;^045sT@#D{uysonyWc1P^gr01J$Tk*2SYjv#y982{= zfyS{~Uo%+J+x%XCB^!KxXUp|R*r!7si|pr%)3>0)I&YZHF_wJ&Xni@^jE9qM+YjBt zuArb!m79g~C1+|30itOtS*N||MzTiI(M5P)C<9?kk&Ki@Pz56lPa>(*Jtd@)tWQ)% zMgh?=@b+D$(-gXVbN*aLoB`8dNDak?^|dfeGhG~LjF9erdlSl-`>Q+l6dn_NmUs+| z`GzxZx5(#xLE3s8z>t%~-U6Ki_qZ^ONfUDhN77OOiSh`d zDT|r1h`>=^C!9oG#)TF0T?}luA%EjA#vLuU;wSG6xY3(G1zH@lia~VBQP*$@#}`0TK4^9;*{UcZ@!3OoD=ht=^9;+^p@&S@(luI%8#XYR%}xf6reNlaAQ7@WViOS37sJ$(uM6Op!UO-_Cdb&M+NKz!V5 zQRQ?t9ll|MP28Qz{tPET<~?U=Km43zvO6d5_7_5roBz&441z&F?pB&y*P-Z8dmL~c zBGrEW4J#bF#tlzwD%eI^x`$!NrUO_jfn`u<-?B}oQFGb^qA8rg_Wx>9LqG;@k?@6DcfTgkmUW9 z``#Y6%J`hsmW#&4iTLwngx94J^F!6XOPD=_Xj712B2aNW#ZTNIuWH`S&#JVoYwq-- z-EyP~wUcHq&!XvXrqhf`JJtVYz5nS_NlXIvt_L+&umSIV>8QXyVxT9w@wJ&l6aqyg z@eoSko!HTxh9}Y1+{(;jcV9i5$0nlC_beYQe7y6H*xZ0zac}yipdPu_KGBzROuQuY zr?Yl_4>ZzAHR@bgJ?di_5K$Ym{kF4NLV`LhL_M9OSPB5^ALVu1{)ze=BOZ})Vw2x9 zISf-#4g?i`>2YC=i5gqq*BTyAt%9>@b}?d-60-}-d;mEgJ#Q7hSCKg%4;h-*^)x?)(s0Gli}~*veHK*cILj=~K95;2zoZ+gQY`jN+=NV{;sFUJ)K zswn;K;R*l9j)~(kjbdopT)1i70<3uc?_wWIb$|==RET6P?=8iT$j&9Zu1uO}rQ6(S z>_&d3kz8rHA{Xh-gExMwFITK_*@`&ew)wK(+%=5djQ-qU!z|#Q9CoEJq_Bk!+oxbo zB1s{2Kb#A%6sFUsj+2zF5dB3kdaccSg?;TcZ4O-p9--sxq$rS{cvo_-1nRr!(zj98 zyd0e@M%>!ul38tn8Zx@xmfWl6V`=`$ZqthkR!&jGv|%W!L%aECoVTrp5EyK}Ha^Pd zyLPoDe&`X(y-ew?2m}B55KaT3P-4E2>xm6@(Uj z;gNeufHYU*k7KPBmEwP)`fk45_H+0-Q>8dO7XOJJ5$#aPrilPupRU3kj19b~35*}xa##4Xf!IgdG0Qj#2w8WW0LIU zS}((}0sv_6h_dqLHVTpHtEbPtmWQ%^plVB^_lYc|KK4FHN73g`f1pqPPDi3QiE17I zNwWy;z9`)rEFJW3VmMGn!5-{hWw$7Oq{6WH7CJHhNi4gJ0_m{J1S|#bEeLw$e z12UL1#9hgiSOL~(rQmr&`DfCL-#DdD^aOzdS(px$%VVqnlUDW5Hogu$94P-r)M2im z?3C3(#^&4b^b$#hV06s1VdY~6CAxAgF_vy(P!yCsPF>UqRl3adkr!1De2-ZQd3_}o z>rcR=vYy;OF@QJQi=s6X*yvPWcaiXZwukFJy)P=3EJt;7Wtk!O9VWj%mB=ZpxYKc` zqPV@&1wrS7H%FLAg$PcLGvrJ+3VoF48BOA=C0l0`--Y8O7s$0hoXA=3^B8T*t@itaPziTqp@o9(P}RurfsSYlGj-?`k52D9iZEK3cA)JP6|R zoBd~G*su3DcZ)hV(pzHU4P%m*)*saO3;Z@}6lJ_(^=oiLXYuejqDAW-dSo+O7ooZ` z2E|-PH($YVGvsyXe5ON-(#8}W&0%@YkE2#?W`Bmmp(9m*rgg&-^C;KmVhiJWCoaxS zw%v$=IvoI~vn$aN- zxEvx)c@5t4-?WZSA&rL1Jom!>wH4z}+zkjxg9y(bLjU@M&^kH;p@l4Rk?Y^Jju@1o z#1eb|y6Rt=RY>b70512`g_zKPd%0B5I_kcuHvI3d_Y4`*IvP>u%>H++BQZ!F09YtT z`JWH^pC|hNZs{Jja$MpkK62#DnRl% Date: Thu, 8 Feb 2018 19:36:24 +0800 Subject: [PATCH 20/26] Update index_cn.rst --- doc/howto/rnn/index_cn.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/rnn/index_cn.rst b/doc/howto/rnn/index_cn.rst index 9ecab5594c..bcc8c2f46e 100644 --- a/doc/howto/rnn/index_cn.rst +++ b/doc/howto/rnn/index_cn.rst @@ -1,4 +1,4 @@ -RNN相关模型 +RNN模型 =========== .. toctree:: From 7c880522542093e0ae23c49e3ed24e3c4100d3e1 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Thu, 8 Feb 2018 19:51:20 +0800 Subject: [PATCH 21/26] Optimize return all opt ops --- python/paddle/v2/fluid/framework.py | 3 + python/paddle/v2/fluid/optimizer.py | 15 ++--- .../paddle/v2/fluid/tests/test_optimizer.py | 56 ++++++++++--------- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index a12427258e..a517db68c5 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -740,6 +740,9 @@ class Block(object): raise e self.desc.remove_op(start, end + 1) + def slice_ops(self, start, end): + return list(self.ops)[start:end] + def prepend_op(self, *args, **kwargs): op_desc = self.desc.prepend_op() op = Operator(self, op_desc, *args, **kwargs) diff --git a/python/paddle/v2/fluid/optimizer.py b/python/paddle/v2/fluid/optimizer.py index 7844a4e2df..f8a00e3a5f 100644 --- a/python/paddle/v2/fluid/optimizer.py +++ b/python/paddle/v2/fluid/optimizer.py @@ -190,6 +190,8 @@ class Optimizer(object): # Create any accumulators program = loss.block.program with program_guard(program, startup_program): + global_block = framework.default_main_program().global_block() + start = len(global_block.ops) self.helper = LayerHelper(self.__class__.__name__) self._create_accumulators(loss.block, [p[0] for p in parameters_and_grads]) @@ -203,19 +205,14 @@ class Optimizer(object): param_and_grad) optimize_ops.append(optimize_op) - # Returned list of ops can include more ops in addition - # to optimization ops - return_ops = optimize_ops - # Get custom finish ops for subclasses # FIXME: Need to fix this once we figure out how to handle dependencies - finish_ops = self._finish_update(loss.block) - if finish_ops is not None: - return_ops += finish_ops + self._finish_update(loss.block) if self._global_step is not None: - return_ops.append(self._increment_global_step(loss.block)) - return return_ops + self._increment_global_step(loss.block) + end = len(global_block.ops) + return global_block.slice_ops(start, end) def minimize(self, loss, diff --git a/python/paddle/v2/fluid/tests/test_optimizer.py b/python/paddle/v2/fluid/tests/test_optimizer.py index 480ee70915..dc6b84dcdc 100644 --- a/python/paddle/v2/fluid/tests/test_optimizer.py +++ b/python/paddle/v2/fluid/tests/test_optimizer.py @@ -42,9 +42,9 @@ class TestOptimizer(unittest.TestCase): type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out}) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) opts, _ = sgd_optimizer.minimize(mean_out, init_program) - self.assertEqual(len(opts), 1) - sgd_op = opts[0] - self.assertEqual(sgd_op.type, "sgd") + self.assertEqual(len(opts), 3) + self.assertEqual([op.type for op in opts], + ["fill_constant", "elementwise_mul", "sgd"]) def test_sgd_optimizer_with_global_step(self): init_program = framework.Program() @@ -72,11 +72,10 @@ class TestOptimizer(unittest.TestCase): sgd_optimizer = optimizer.SGDOptimizer( learning_rate=learning_rate, global_step=global_step) opts, _ = sgd_optimizer.minimize(mean_out, init_program) - self.assertEqual(len(opts), 2) - sgd_op = opts[0] - self.assertEqual(sgd_op.type, "sgd") - increment_op = opts[1] - self.assertEqual(increment_op.type, "increment") + self.assertEqual(len(opts), 4) + self.assertEqual( + [op.type for op in opts], + ["fill_constant", "elementwise_mul", "sgd", "increment"]) # Check init_program init_ops = init_program.global_block().ops @@ -121,9 +120,10 @@ class TestMomentumOptimizer(unittest.TestCase): self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) opts = momentum_optimizer.create_optimization_pass( params_grads, mul_out, init_program) - self.assertEqual(len(opts), 1) - sgd_op = opts[0] - self.assertEqual(sgd_op.type, "momentum") + self.assertEqual(len(opts), 3) + sgd_op = opts[-1] + self.assertEqual([op.type for op in opts], + ["fill_constant", "elementwise_mul", "momentum"]) self.assertFalse(sgd_op.attr('use_nesterov')) # Check accumulators @@ -170,9 +170,10 @@ class TestMomentumOptimizer(unittest.TestCase): self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) opts = momentum_optimizer.create_optimization_pass( params_grads, mul_out, init_program) - self.assertEqual(len(opts), 1) - sgd_op = opts[0] - self.assertEqual(sgd_op.type, "momentum") + self.assertEqual(len(opts), 3) + sgd_op = opts[-1] + self.assertEqual([op.type for op in opts], + ["fill_constant", "elementwise_mul", "momentum"]) self.assertTrue(sgd_op.attr('use_nesterov')) # Check accumulators @@ -228,9 +229,9 @@ class TestAdagradOptimizer(unittest.TestCase): self.assertEqual(len(adagrad_optimizer.get_accumulators()), 0) opts = adagrad_optimizer.create_optimization_pass(params_grads, mul_out, init_program) - self.assertEqual(len(opts), 1) - adagrad_op = opts[0] - self.assertEqual(adagrad_op.type, "adagrad") + self.assertEqual(len(opts), 3) + self.assertEqual([op.type for op in opts], + ["fill_constant", "elementwise_mul", "adagrad"]) # Check accumulators accumulators = adagrad_optimizer.get_accumulators() @@ -288,9 +289,10 @@ class TestAdamOptimizer(unittest.TestCase): self.assertEqual(len(adam_optimizer.get_accumulators()), 0) opts = adam_optimizer.create_optimization_pass(params_grads, mul_out, init_program) - self.assertEqual(len(opts), 3) - adam_op = opts[0] - self.assertEqual(adam_op.type, "adam") + self.assertEqual(len(opts), 5) + self.assertEqual( + [op.type for op in opts], + ["fill_constant", "elementwise_mul", "adam", "scale", "scale"]) # Check accumulators accumulators = adam_optimizer.get_accumulators() @@ -350,9 +352,10 @@ class TestAdamaxOptimizer(unittest.TestCase): self.assertEqual(len(adamax_optimizer.get_accumulators()), 0) opts = adamax_optimizer.create_optimization_pass(params_grads, mul_out, init_program) - self.assertEqual(len(opts), 2) - adam_op = opts[0] - self.assertEqual(adam_op.type, "adamax") + self.assertEqual(len(opts), 4) + self.assertEqual( + [op.type for op in opts], + ["fill_constant", "elementwise_mul", "adamax", "scale"]) # Check accumulators accumulators = adamax_optimizer.get_accumulators() @@ -409,9 +412,10 @@ class TestDecayedAdagradOptimizer(unittest.TestCase): self.assertEqual(len(decayed_adagrad_optimizer.get_accumulators()), 0) opts = decayed_adagrad_optimizer.create_optimization_pass( params_grads, mul_out, init_program) - self.assertEqual(len(opts), 1) - decayed_adagrad_op = opts[0] - self.assertEqual(decayed_adagrad_op.type, "decayed_adagrad") + self.assertEqual(len(opts), 3) + self.assertEqual( + [op.type for op in opts], + ["fill_constant", "elementwise_mul", "decayed_adagrad"]) # Check accumulators accumulators = decayed_adagrad_optimizer.get_accumulators() From 11acbe687e80bbe1be2632786fd9fa29ecb9932b Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 8 Feb 2018 10:30:10 -0800 Subject: [PATCH 22/26] Updating the copyright year for the recent files. (#8018) * Adding more details to cluster_train * Update copyright for notest_dist_image_classification * Fixed copyright * Updating the copyright year and content --- paddle/framework/channel.h | 2 +- paddle/framework/channel_test.cc | 2 +- paddle/framework/details/buffered_channel.h | 2 +- paddle/framework/details/cow_ptr.h | 2 +- paddle/framework/details/cow_ptr_test.cc | 2 +- paddle/framework/details/op_registry.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/framework/channel.h b/paddle/framework/channel.h index b679387b11..146f0e9e71 100644 --- a/paddle/framework/channel.h +++ b/paddle/framework/channel.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/framework/channel_test.cc b/paddle/framework/channel_test.cc index a307abb4ed..8afb988914 100644 --- a/paddle/framework/channel_test.cc +++ b/paddle/framework/channel_test.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/framework/details/buffered_channel.h b/paddle/framework/details/buffered_channel.h index 77eebc9924..c6e4bec0f3 100644 --- a/paddle/framework/details/buffered_channel.h +++ b/paddle/framework/details/buffered_channel.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/framework/details/cow_ptr.h b/paddle/framework/details/cow_ptr.h index 7e308ffb5a..69bcea6252 100644 --- a/paddle/framework/details/cow_ptr.h +++ b/paddle/framework/details/cow_ptr.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/framework/details/cow_ptr_test.cc b/paddle/framework/details/cow_ptr_test.cc index 936954a233..1f4a12bca0 100644 --- a/paddle/framework/details/cow_ptr_test.cc +++ b/paddle/framework/details/cow_ptr_test.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index 6d50e820b2..31a40bcbcb 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From f605d00f66f27066d68033e2dbbaef806954e24b Mon Sep 17 00:00:00 2001 From: kavyasrinet Date: Thu, 8 Feb 2018 10:30:29 -0800 Subject: [PATCH 23/26] Fixing unbuffered test to a generic test (#8162) * Fixing unbufered test to a generic test * Update channel_test.cc * splitting over functions * add type * fix * --- paddle/framework/channel_test.cc | 47 ++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/paddle/framework/channel_test.cc b/paddle/framework/channel_test.cc index 8afb988914..35567649b8 100644 --- a/paddle/framework/channel_test.cc +++ b/paddle/framework/channel_test.cc @@ -25,6 +25,26 @@ using paddle::framework::CloseChannel; using paddle::framework::details::Buffered; using paddle::framework::details::UnBuffered; +void RecevingOrderEqualToSendingOrder(Channel *ch) { + unsigned sum_send = 0; + std::thread t([&]() { + for (int i = 0; i < 5; i++) { + EXPECT_EQ(ch->Send(&i), true); + sum_send += i; + } + }); + for (int i = 0; i < 5; i++) { + int recv; + EXPECT_EQ(ch->Receive(&recv), true); + EXPECT_EQ(recv, i); + } + + CloseChannel(ch); + t.join(); + EXPECT_EQ(sum_send, 10U); + delete ch; +} + TEST(Channel, MakeAndClose) { using paddle::framework::details::Buffered; using paddle::framework::details::UnBuffered; @@ -137,9 +157,7 @@ TEST(Channel, ReceiveFromBufferedChannelReturnResidualValuesTest) { for (size_t i = 0; i < buffer_size; ++i) { EXPECT_EQ(ch->Receive(&out), - false); // after receiving residual values, return zeros. - // Note: we cannot check EXPECT_EQ(out, 0), because C++ doesn't - // define zero values like Go does. + false); // receiving on closed channel should return false } delete ch; } @@ -166,25 +184,14 @@ TEST(Channel, ConcurrentSendNonConcurrentReceiveWithSufficientBufferSize) { delete ch; } -TEST(Channel, SimpleUnbufferedChannelTest) { +TEST(Channel, RecevingOrderEqualToSendingOrderWithUnBufferedChannel) { auto ch = MakeChannel(0); - unsigned sum_send = 0; - std::thread t([&]() { - for (int i = 0; i < 5; i++) { - EXPECT_EQ(ch->Send(&i), true); - sum_send += i; - } - }); - for (int i = 0; i < 5; i++) { - int recv; - EXPECT_EQ(ch->Receive(&recv), true); - EXPECT_EQ(recv, i); - } + RecevingOrderEqualToSendingOrder(ch); +} - CloseChannel(ch); - t.join(); - EXPECT_EQ(sum_send, 10U); - delete ch; +TEST(Channel, RecevingOrderEqualToSendingOrderWithBufferedChannel) { + auto ch = MakeChannel(10); + RecevingOrderEqualToSendingOrder(ch); } // This tests that closing a buffered channel also unblocks From 36da52950d53fdcd6903913a627b5e622648f5f4 Mon Sep 17 00:00:00 2001 From: Siddharth Goyal Date: Thu, 8 Feb 2018 13:46:33 -0800 Subject: [PATCH 24/26] Better version of PR #7985 (Modify load() for inference) (#8024) * Refine load * Address review comments: round 1 * Make API consistent with python-save/load * Add another unit test * Remove commented function * Fix GPU bug * Address review comments * Modify wrt PR 8147 * Fix filenames for combined case * Fix typo * Address review comments: round 2 * Unify TestInference by keeping default param in template * Address review comment * Fix spacing --- paddle/inference/io.cc | 73 +++++++++++++++---- paddle/inference/io.h | 8 +- paddle/inference/tests/book/test_helper.h | 17 ++++- .../book/test_inference_recognize_digits.cc | 42 +++++++++++ python/paddle/v2/fluid/io.py | 12 ++- .../fluid/tests/book/test_recognize_digits.py | 44 +++++++---- 6 files changed, 159 insertions(+), 37 deletions(-) diff --git a/paddle/inference/io.cc b/paddle/inference/io.cc index 1ed14b69c8..784e87970f 100644 --- a/paddle/inference/io.cc +++ b/paddle/inference/io.cc @@ -21,6 +21,17 @@ limitations under the License. */ namespace paddle { namespace inference { +void ReadBinaryFile(const std::string& filename, std::string& contents) { + VLOG(3) << "loading model from " << filename; + std::ifstream inputfs(filename, std::ios::in | std::ios::binary); + inputfs.seekg(0, std::ios::end); + contents.clear(); + contents.resize(inputfs.tellg()); + inputfs.seekg(0, std::ios::beg); + inputfs.read(&contents[0], contents.size()); + inputfs.close(); +} + bool IsParameter(const framework::VarDesc* var, const framework::ProgramDesc& main_program) { if (var->Persistable()) { @@ -44,12 +55,15 @@ bool IsParameter(const framework::VarDesc* var, void LoadPersistables(framework::Executor& executor, framework::Scope& scope, + const framework::ProgramDesc& main_program, const std::string& dirname, - const framework::ProgramDesc& main_program) { + const std::string& param_filename) { const framework::BlockDesc& global_block = main_program.Block(0); framework::ProgramDesc* load_program = new framework::ProgramDesc(); framework::BlockDesc* load_block = load_program->MutableBlock(0); + std::vector paramlist; + for (auto* var : global_block.AllVars()) { if (IsParameter(var, main_program)) { VLOG(3) << "parameter's name: " << var->Name(); @@ -61,15 +75,33 @@ void LoadPersistables(framework::Executor& executor, new_var->SetLoDLevel(var->GetLoDLevel()); new_var->SetPersistable(true); - // append_op - framework::OpDesc* op = load_block->AppendOp(); - op->SetType("load"); - op->SetOutput("Out", {new_var->Name()}); - op->SetAttr("file_path", {dirname + "/" + new_var->Name()}); - op->CheckAttrs(); + if (!param_filename.empty()) { + paramlist.push_back(new_var->Name()); + } else { + // append_op + framework::OpDesc* op = load_block->AppendOp(); + op->SetType("load"); + op->SetOutput("Out", {new_var->Name()}); + op->SetAttr("file_path", {dirname + "/" + new_var->Name()}); + op->CheckAttrs(); + } } } + + if (!param_filename.empty()) { + // sort paramlist to have consistent ordering + std::sort(paramlist.begin(), paramlist.end()); + // append just the load_combine op + framework::OpDesc* op = load_block->AppendOp(); + op->SetType("load_combine"); + op->SetOutput("Out", paramlist); + op->SetAttr("file_path", {param_filename}); + op->CheckAttrs(); + } + executor.Run(*load_program, &scope, 0, true, true); + + VLOG(3) << "Ran loading successfully"; delete load_program; } @@ -77,20 +109,29 @@ std::unique_ptr Load(framework::Executor& executor, framework::Scope& scope, const std::string& dirname) { std::string model_filename = dirname + "/__model__"; - LOG(INFO) << "loading model from " << model_filename; - std::ifstream inputfs(model_filename, std::ios::in | std::ios::binary); std::string program_desc_str; - inputfs.seekg(0, std::ios::end); - program_desc_str.resize(inputfs.tellg()); - inputfs.seekg(0, std::ios::beg); - LOG(INFO) << "program_desc_str's size: " << program_desc_str.size(); - inputfs.read(&program_desc_str[0], program_desc_str.size()); - inputfs.close(); + ReadBinaryFile(model_filename, program_desc_str); + + std::unique_ptr main_program( + new framework::ProgramDesc(program_desc_str)); + + LoadPersistables(executor, scope, *main_program, dirname, ""); + return main_program; +} + +std::unique_ptr Load( + framework::Executor& executor, + framework::Scope& scope, + const std::string& prog_filename, + const std::string& param_filename) { + std::string model_filename = prog_filename; + std::string program_desc_str; + ReadBinaryFile(model_filename, program_desc_str); std::unique_ptr main_program( new framework::ProgramDesc(program_desc_str)); - LoadPersistables(executor, scope, dirname, *main_program); + LoadPersistables(executor, scope, *main_program, "", param_filename); return main_program; } diff --git a/paddle/inference/io.h b/paddle/inference/io.h index 962b6c4e20..a7d7c49969 100644 --- a/paddle/inference/io.h +++ b/paddle/inference/io.h @@ -26,12 +26,18 @@ namespace inference { void LoadPersistables(framework::Executor& executor, framework::Scope& scope, + const framework::ProgramDesc& main_program, const std::string& dirname, - const framework::ProgramDesc& main_program); + const std::string& param_filename); std::unique_ptr Load(framework::Executor& executor, framework::Scope& scope, const std::string& dirname); +std::unique_ptr Load(framework::Executor& executor, + framework::Scope& scope, + const std::string& prog_filename, + const std::string& param_filename); + } // namespace inference } // namespace paddle diff --git a/paddle/inference/tests/book/test_helper.h b/paddle/inference/tests/book/test_helper.h index 32db643fca..3e66ced94f 100644 --- a/paddle/inference/tests/book/test_helper.h +++ b/paddle/inference/tests/book/test_helper.h @@ -67,17 +67,28 @@ void CheckError(paddle::framework::LoDTensor& output1, EXPECT_EQ(count, 0) << "There are " << count << " different elements."; } -template +template void TestInference(const std::string& dirname, const std::vector& cpu_feeds, std::vector& cpu_fetchs) { - // 1. Define place, executor and scope + // 1. Define place, executor, scope and inference_program auto place = Place(); auto executor = paddle::framework::Executor(place); auto* scope = new paddle::framework::Scope(); + std::unique_ptr inference_program; // 2. Initialize the inference_program and load all parameters from file - auto inference_program = paddle::inference::Load(executor, *scope, dirname); + if (IsCombined) { + // Hard-coding the names for combined params case + std::string prog_filename = "__model_combined__"; + std::string param_filename = "__params_combined__"; + inference_program = paddle::inference::Load(executor, + *scope, + dirname + "/" + prog_filename, + dirname + "/" + param_filename); + } else { + inference_program = paddle::inference::Load(executor, *scope, dirname); + } // 3. Get the feed_target_names and fetch_target_names const std::vector& feed_target_names = diff --git a/paddle/inference/tests/book/test_inference_recognize_digits.cc b/paddle/inference/tests/book/test_inference_recognize_digits.cc index 48f887e6bc..3a48db7fe0 100644 --- a/paddle/inference/tests/book/test_inference_recognize_digits.cc +++ b/paddle/inference/tests/book/test_inference_recognize_digits.cc @@ -59,3 +59,45 @@ TEST(inference, recognize_digits) { CheckError(output1, output2); #endif } + +TEST(inference, recognize_digits_combine) { + if (FLAGS_dirname.empty()) { + LOG(FATAL) << "Usage: ./example --dirname=path/to/your/model"; + } + + LOG(INFO) << "FLAGS_dirname: " << FLAGS_dirname << std::endl; + std::string dirname = FLAGS_dirname; + + // 0. Call `paddle::framework::InitDevices()` initialize all the devices + // In unittests, this is done in paddle/testing/paddle_gtest_main.cc + + paddle::framework::LoDTensor input; + // Use normilized image pixels as input data, + // which should be in the range [-1.0, 1.0]. + SetupTensor( + input, {1, 28, 28}, static_cast(-1), static_cast(1)); + std::vector cpu_feeds; + cpu_feeds.push_back(&input); + + paddle::framework::LoDTensor output1; + std::vector cpu_fetchs1; + cpu_fetchs1.push_back(&output1); + + // Run inference on CPU + TestInference( + dirname, cpu_feeds, cpu_fetchs1); + LOG(INFO) << output1.dims(); + +#ifdef PADDLE_WITH_CUDA + paddle::framework::LoDTensor output2; + std::vector cpu_fetchs2; + cpu_fetchs2.push_back(&output2); + + // Run inference on CUDA GPU + TestInference( + dirname, cpu_feeds, cpu_fetchs2); + LOG(INFO) << output2.dims(); + + CheckError(output1, output2); +#endif +} diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 613dc20b6e..0f43e46082 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -342,7 +342,11 @@ def save_inference_model(dirname, prepend_feed_ops(inference_program, feeded_var_names) append_fetch_ops(inference_program, fetch_var_names) - model_file_name = dirname + "/__model__" + if save_file_name == None: + model_file_name = dirname + "/__model__" + else: + model_file_name = dirname + "/__model_combined__" + with open(model_file_name, "wb") as f: f.write(inference_program.desc.serialize_to_string()) @@ -384,7 +388,11 @@ def load_inference_model(dirname, executor, load_file_name=None): if not os.path.isdir(dirname): raise ValueError("There is no directory named '%s'", dirname) - model_file_name = dirname + "/__model__" + if load_file_name == None: + model_file_name = dirname + "/__model__" + else: + model_file_name = dirname + "/__model_combined__" + with open(model_file_name, "rb") as f: program_desc_str = f.read() diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits.py index d8f0ad89cd..6f9d85faff 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits.py @@ -78,7 +78,7 @@ def conv_net(img, label): return loss_net(conv_pool_2, label) -def train(nn_type, use_cuda, parallel, save_dirname): +def train(nn_type, use_cuda, parallel, save_dirname, save_param_filename): if use_cuda and not fluid.core.is_compiled_with_cuda(): return img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32') @@ -143,8 +143,10 @@ def train(nn_type, use_cuda, parallel, save_dirname): avg_loss_val = numpy.array(avg_loss_set).mean() if float(acc_val) > 0.85: # test acc > 85% if save_dirname is not None: - fluid.io.save_inference_model(save_dirname, ["img"], - [prediction], exe) + fluid.io.save_inference_model( + save_dirname, ["img"], [prediction], + exe, + save_file_name=save_param_filename) return else: print( @@ -156,7 +158,7 @@ def train(nn_type, use_cuda, parallel, save_dirname): raise AssertionError("Loss of recognize digits is too large") -def infer(use_cuda, save_dirname=None): +def infer(use_cuda, save_dirname=None, param_filename=None): if save_dirname is None: return @@ -167,8 +169,8 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be feeded # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [inference_program, feed_target_names, fetch_targets + ] = fluid.io.load_inference_model(save_dirname, exe, param_filename) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [-1.0, 1.0]. @@ -183,36 +185,45 @@ def infer(use_cuda, save_dirname=None): print("infer results: ", results[0]) -def main(use_cuda, parallel, nn_type): +def main(use_cuda, parallel, nn_type, combine): if not use_cuda and not parallel: save_dirname = "recognize_digits_" + nn_type + ".inference.model" + save_filename = None + if combine == True: + save_filename = "__params_combined__" else: save_dirname = None + save_filename = None train( nn_type=nn_type, use_cuda=use_cuda, parallel=parallel, - save_dirname=save_dirname) - infer(use_cuda=use_cuda, save_dirname=save_dirname) + save_dirname=save_dirname, + save_param_filename=save_filename) + infer( + use_cuda=use_cuda, + save_dirname=save_dirname, + param_filename=save_filename) class TestRecognizeDigits(unittest.TestCase): pass -def inject_test_method(use_cuda, parallel, nn_type): +def inject_test_method(use_cuda, parallel, nn_type, combine): def __impl__(self): prog = fluid.Program() startup_prog = fluid.Program() scope = fluid.core.Scope() with fluid.scope_guard(scope): with fluid.program_guard(prog, startup_prog): - main(use_cuda, parallel, nn_type) + main(use_cuda, parallel, nn_type, combine) - fn = 'test_{0}_{1}_{2}'.format(nn_type, 'cuda' - if use_cuda else 'cpu', 'parallel' - if parallel else 'normal') + fn = 'test_{0}_{1}_{2}_{3}'.format(nn_type, 'cuda' + if use_cuda else 'cpu', 'parallel' + if parallel else 'normal', 'combine' + if combine else 'separate') setattr(TestRecognizeDigits, fn, __impl__) @@ -221,7 +232,10 @@ def inject_all_tests(): for use_cuda in (False, True): for parallel in (False, True): for nn_type in ('mlp', 'conv'): - inject_test_method(use_cuda, parallel, nn_type) + inject_test_method(use_cuda, parallel, nn_type, True) + + # One unit-test for saving parameters as separate files + inject_test_method(False, False, 'mlp', False) inject_all_tests() From 0a7ae369f6d6a6be7b87d13834b8ff0a7e3614d1 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 9 Feb 2018 10:27:47 +0800 Subject: [PATCH 25/26] fix CI hung --- .../tests/book_memory_optimization/test_memopt_fit_a_line.py | 4 ++++ .../test_memopt_image_classification_train.py | 5 +++++ .../test_memopt_machine_translation.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index 7ad5e2c594..045db8390c 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -15,6 +15,8 @@ import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid +import math +import sys # need to fix random seed and training data to compare the loss # value accurately calculated by the default and the memory optimization @@ -63,4 +65,6 @@ for pass_id in range(PASS_NUM): if avg_loss_value[0] < 10.0: exit(0) # if avg cost less than 10.0, we think our code is good. + if math.isnan(float(avg_loss_value)): + sys.exit("got NaN loss, training failed.") exit(1) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py index 26673afd83..9fbb36d363 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -18,6 +18,8 @@ import sys import paddle.v2 as paddle import paddle.v2.fluid as fluid +import math +import sys # need to fix random seed and training data to compare the loss # value accurately calculated by the default and the memory optimization @@ -152,7 +154,10 @@ for pass_id in range(PASS_NUM): print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str( pass_acc)) # this model is slow, so if we can train two mini batch, we think it works properly. + if i > 2: exit(0) + if math.isnan(float(loss)): + sys.exit("got NaN loss, training failed.") i += 1 exit(1) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py index ffd53e7a78..48abaa8d87 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py @@ -19,6 +19,8 @@ import paddle.v2.fluid.core as core import paddle.v2.fluid.framework as framework import paddle.v2.fluid.layers as layers from paddle.v2.fluid.executor import Executor +import math +import sys dict_size = 30000 source_dict_dim = target_dict_dim = dict_size @@ -137,6 +139,8 @@ def main(): " avg_cost=" + str(avg_cost_val)) if batch_id > 2: exit(0) + if math.isnan(float(avg_cost_val)): + sys.exit("got NaN loss, training failed.") batch_id += 1 From b5ffe5bce205ea9ce48a329edd6f19164c8608f2 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Fri, 9 Feb 2018 11:08:02 +0800 Subject: [PATCH 26/26] optimize data flow analysis (#8271) --- python/paddle/v2/fluid/memory_optimization_transpiler.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/fluid/memory_optimization_transpiler.py b/python/paddle/v2/fluid/memory_optimization_transpiler.py index 8bb8cf7b1a..53e0991ee8 100644 --- a/python/paddle/v2/fluid/memory_optimization_transpiler.py +++ b/python/paddle/v2/fluid/memory_optimization_transpiler.py @@ -92,14 +92,13 @@ class ControlFlowGraph(object): live_in = defaultdict(set) live_out = defaultdict(set) while True: - for i in range(self.op_size): + for i in range(self.op_size, 0, -1): live_in[i] = set(self._live_in[i]) live_out[i] = set(self._live_out[i]) - self._live_in[i] = self._uses[i] | ( - self._live_out[i] - self._defs[i]) for s in self._successors[i]: self._live_out[i] |= self._live_in[s] - + self._live_in[i] = self._uses[i] | ( + self._live_out[i] - self._defs[i]) if self._reach_fixed_point(live_in, live_out): break