From 6326c40d2709da9839edbf3fb2a280ca92804a23 Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Thu, 28 Sep 2017 09:49:00 +0800
Subject: [PATCH 01/82] Add max pool with index
---
paddle/operators/CMakeLists.txt | 6 +
paddle/operators/math/CMakeLists.txt | 4 +-
paddle/operators/math/pooling.cc | 255 ++++++++++++
paddle/operators/math/pooling.cu | 387 ++++++++++++++++++
paddle/operators/math/pooling.h | 68 +++
paddle/operators/pool_with_index_op.cc | 198 +++++++++
paddle/operators/pool_with_index_op.cu | 31 ++
paddle/operators/pool_with_index_op.h | 99 +++++
.../v2/framework/tests/test_pool_max_op.py | 125 ++++++
9 files changed, 1171 insertions(+), 2 deletions(-)
create mode 100644 paddle/operators/math/pooling.cc
create mode 100644 paddle/operators/math/pooling.cu
create mode 100644 paddle/operators/math/pooling.h
create mode 100644 paddle/operators/pool_with_index_op.cc
create mode 100644 paddle/operators/pool_with_index_op.cu
create mode 100644 paddle/operators/pool_with_index_op.h
create mode 100644 python/paddle/v2/framework/tests/test_pool_max_op.py
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index e56895c63a..0feb969c62 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -62,6 +62,12 @@ function(op_library TARGET)
file(APPEND ${pybind_file} "USE_OP(sigmoid);\n")
endif()
+ if ("${TARGET}" STREQUAL "pool_with_index_op")
+ set(pybind_flag 1)
+ # It's enough to just adding one operator to pybind
+ file(APPEND ${pybind_file} "USE_OP(maxPool2dWithIndex);\n")
+ endif()
+
# pybind USE_NO_KERNEL_OP
file(READ ${TARGET}.cc TARGET_CONTENT)
string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}")
diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt
index 91ae3d49f1..811deb4c2c 100644
--- a/paddle/operators/math/CMakeLists.txt
+++ b/paddle/operators/math/CMakeLists.txt
@@ -1,12 +1,12 @@
if(WITH_GPU)
nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc
- im2col.cu DEPS cblas device_context operator)
+ im2col.cu pooling.cc pooling.cu DEPS cblas device_context operator)
nv_library(softmax_function SRCS softmax.cc softmax.cu
DEPS operator)
nv_library(cross_entropy_function SRCS cross_entropy.cc cross_entropy.cu
DEPS operator)
else()
- cc_library(math_function SRCS math_function.cc im2col.cc
+ cc_library(math_function SRCS math_function.cc im2col.cc pooling.cc
DEPS cblas device_context operator)
cc_library(softmax_function SRCS softmax.cc DEPS operator)
cc_library(cross_entropy_function SRCS cross_entropy.cc DEPS operator)
diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc
new file mode 100644
index 0000000000..0e4d9007a6
--- /dev/null
+++ b/paddle/operators/math/pooling.cc
@@ -0,0 +1,255 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/math/pooling.h"
+
+namespace paddle {
+namespace operators {
+namespace math {
+
+template <typename T>
+class MaxPool2dWithIndexFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input.dims()[0];
+
+ const int input_height = input.dims()[2];
+ const int input_width = input.dims()[3];
+ const int output_channels = output.dims()[1];
+ const int output_height = output.dims()[2];
+ const int output_width = output.dims()[3];
+ const int ksize_height = ksize[0];
+ const int ksize_width = ksize[1];
+ const int stride_height = strides[0];
+ const int stride_width = strides[1];
+ const int padding_height = paddings[0];
+ const int padding_width = paddings[1];
+
+ const int input_stride = input_height * input_width;
+ const int output_stride = output_height * output_width;
+
+ const T* input_data = input.data<T>();
+ T* output_data = output.mutable_data<T>(context.GetPlace());
+
+ T* mask_data = mask.mutable_data<T>(context.GetPlace());
+
+ for (int i = 0; i < batch_size; i++) {
+ for (int c = 0; c < output_channels; ++c) {
+ for (int ph = 0; ph < output_height; ++ph) {
+ int hstart = ph * stride_height - padding_height;
+ int hend = std::min(hstart + ksize_height, input_height);
+ hstart = std::max(hstart, 0);
+ for (int pw = 0; pw < output_width; ++pw) {
+ int wstart = pw * stride_width - padding_width;
+ int wend = std::min(wstart + ksize_width, input_width);
+ wstart = std::max(wstart, 0);
+
+ T ele = static_cast<T>(-FLT_MAX);
+ int index = -1;
+ for (int h = hstart; h < hend; ++h) {
+ for (int w = wstart; w < wend; ++w) {
+ if (ele < input_data[h * input_width + w]) {
+ ele = input_data[h * input_width + w];
+ index = h * input_width + w;
+ }
+ }
+ }
+ output_data[ph * output_width + pw] = ele;
+ mask_data[ph * output_width + pw] = index;
+ }
+ }
+ // offset
+ input_data += input_stride;
+ output_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template <typename T>
+class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input_grad.dims()[0];
+ const int input_height = input_grad.dims()[2];
+ const int input_width = input_grad.dims()[3];
+ const int output_channels = output_grad.dims()[1];
+ const int output_height = output_grad.dims()[2];
+ const int output_width = output_grad.dims()[3];
+ const int input_stride = input_height * input_width;
+ const int output_stride = output_height * output_width;
+
+ const T* mask_data = mask.data<T>();
+ const T* output_grad_data = output_grad.data<T>();
+ T* input_grad_data = input_grad.mutable_data<T>(context.GetPlace());
+
+ for (size_t n = 0; n < batch_size; ++n) {
+ for (size_t c = 0; c < output_channels; ++c) {
+ for (size_t ph = 0; ph < output_height; ++ph) {
+ for (size_t pw = 0; pw < output_width; ++pw) {
+ const size_t output_idx = ph * output_width + pw;
+ const size_t input_idx = static_cast<size_t>(mask_data[output_idx]);
+
+ input_grad_data[input_idx] += output_grad_data[output_idx];
+ }
+ }
+ }
+ // offset
+ input_grad_data += input_stride;
+ output_grad_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+};
+
+template class MaxPool2dWithIndexFunctor<platform::CPUPlace, float>;
+template class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, float>;
+template class MaxPool2dWithIndexFunctor<platform::CPUPlace, double>;
+template class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, double>;
+
+template <typename T>
+class MaxPool3dWithIndexFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input.dims()[0];
+ const int input_depth = input.dims()[2];
+ const int input_height = input.dims()[3];
+ const int input_width = input.dims()[4];
+ const int output_channels = output.dims()[1];
+ const int output_depth = output.dims()[2];
+ const int output_height = output.dims()[3];
+ const int output_width = output.dims()[4];
+ const int ksize_depth = ksize[0];
+ const int ksize_height = ksize[1];
+ const int ksize_width = ksize[2];
+ const int stride_depth = strides[0];
+ const int stride_height = strides[1];
+ const int stride_width = strides[2];
+ const int padding_depth = paddings[0];
+ const int padding_height = paddings[1];
+ const int padding_width = paddings[2];
+ const int input_stride = input_depth * input_height * input_width;
+ const int output_stride = output_depth * output_height * output_width;
+ const T* input_data = input.data<T>();
+ T* output_data = output.mutable_data<T>(context.GetPlace());
+ T* mask_data = mask.mutable_data<T>(context.GetPlace());
+
+ for (int i = 0; i < batch_size; i++) {
+ for (int c = 0; c < output_channels; ++c) {
+ for (int pd = 0; pd < output_depth; ++pd) {
+ int dstart = pd * stride_depth - padding_depth;
+ int dend = std::min(dstart + ksize_depth, input_depth);
+ dstart = std::max(dstart, 0);
+ for (int ph = 0; ph < output_height; ++ph) {
+ int hstart = ph * stride_height - padding_height;
+ int hend = std::min(hstart + ksize_height, input_height);
+ hstart = std::max(hstart, 0);
+ for (int pw = 0; pw < output_width; ++pw) {
+ int wstart = pw * stride_width - padding_width;
+ int wend = std::min(wstart + ksize_width, input_width);
+ wstart = std::max(wstart, 0);
+ int output_idx = (pd * output_height + ph) * output_width + pw;
+ T ele = static_cast<T>(-FLT_MAX);
+ int index = -1;
+ for (int d = dstart; d < dend; ++d) {
+ for (int h = hstart; h < hend; ++h) {
+ for (int w = wstart; w < wend; ++w) {
+ if (ele <
+ input_data[(d * input_height + h) * input_width + w]) {
+ index = (d * input_height + h) * input_width + w;
+ ele =
+ input_data[(d * input_height + h) * input_width + w];
+ }
+ }
+ }
+ }
+ output_data[output_idx] = ele;
+ mask_data[output_idx] = index;
+ }
+ }
+ }
+ // offset
+ input_data += input_stride;
+ output_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template <typename T>
+class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input_grad.dims()[0];
+ const int input_depth = input_grad.dims()[2];
+ const int input_height = input_grad.dims()[3];
+ const int input_width = input_grad.dims()[4];
+ const int output_channels = output_grad.dims()[1];
+ const int output_depth = output_grad.dims()[2];
+ const int output_height = output_grad.dims()[3];
+ const int output_width = output_grad.dims()[4];
+ const int input_stride = input_depth * input_height * input_width;
+ const int output_stride = output_depth * output_height * output_width;
+
+ const T* mask_data = mask.data<T>();
+ const T* output_grad_data = output_grad.data<T>();
+ T* input_grad_data = input_grad.mutable_data<T>(context.GetPlace());
+
+ for (size_t n = 0; n < batch_size; ++n) {
+ for (size_t c = 0; c < output_channels; ++c) {
+ for (size_t pd = 0; pd < output_depth; ++pd) {
+ for (size_t ph = 0; ph < output_height; ++ph) {
+ for (size_t pw = 0; pw < output_width; ++pw) {
+ const size_t output_idx =
+ (pd * output_height + ph) * output_width + pw;
+ const size_t input_idx =
+ static_cast<size_t>(mask_data[output_idx]);
+
+ input_grad_data[input_idx] += output_grad_data[output_idx];
+ }
+ }
+ }
+ // offset
+ input_grad_data += input_stride;
+ output_grad_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template class MaxPool3dWithIndexFunctor<platform::CPUPlace, float>;
+template class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, float>;
+template class MaxPool3dWithIndexFunctor<platform::CPUPlace, double>;
+template class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, double>;
+
+} // namespace math
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu
new file mode 100644
index 0000000000..f32e6a26d0
--- /dev/null
+++ b/paddle/operators/math/pooling.cu
@@ -0,0 +1,387 @@
+/* Copyright (c) 2016 paddlepaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/math/pooling.h"
+#include "paddle/platform/cuda_helper.h"
+
+namespace paddle {
+namespace operators {
+namespace math {
+
+template <typename T>
+__global__ void KernelMaxPool2dWithIdxForward(
+ const int nthreads, const T* input_data, T* output_data, T* mask_data,
+ const int channels, const int input_height, const int input_width,
+ const int output_height, const int output_width, const int ksize_height,
+ const int ksize_width, const int stride_height, const int stride_width,
+ const int padding_height, const int padding_width) {
+ int index = blockIdx.x * blockDim.x + threadIdx.x;
+ if (index < nthreads) {
+ int pw = index % output_width;
+ int ph = (index / output_width) % output_height;
+ int c = (index / output_width / output_height) % channels;
+ int batch_idx = index / output_width / output_height / channels;
+
+ int hstart = ph * stride_height - padding_height;
+ int hend = min(hstart + ksize_height, input_height);
+ hstart = max(hstart, 0);
+
+ int wstart = pw * stride_width - padding_width;
+ int wend = min(wstart + ksize_width, input_width);
+ wstart = max(wstart, 0);
+
+ input_data += (batch_idx * channels + c) * input_height * input_width;
+ T ele = -FLT_MAX;
+ int index = -1;
+ for (int h = hstart; h < hend; ++h) {
+ for (int w = wstart; w < wend; ++w) {
+ if (ele < input_data[h * input_width + w]) {
+ index = h * input_width + w;
+ ele = input_data[h * input_width + w];
+ }
+ }
+ }
+ output_data[index] = ele;
+ mask_data[index] = index;
+ }
+}
+
+template <typename T>
+__global__ void KernelMaxPool2DWithIdxBackward(
+ const int nthreads, T* input_grad, const T* output_grad, const T* mask_data,
+ const int channels, const int input_height, const int input_width,
+ const int output_height, const int output_width, const int ksize_height,
+ const int ksize_width, const int stride_height, const int stride_width,
+ const int padding_height, const int padding_width) {
+ int index = blockIdx.x * blockDim.x + threadIdx.x;
+ if (index < nthreads) {
+ int offsetW = index % input_width + padding_width;
+ int offsetH = (index / input_width) % input_height + padding_height;
+ int offsetC = (index / input_width / input_height) % channels;
+ int batch_idx = index / input_width / input_height / channels;
+
+ int phstart = (offsetH < ksize_height)
+ ? 0
+ : (offsetH - ksize_height) / stride_height + 1;
+ int pwstart = (offsetW < ksize_width)
+ ? 0
+ : (offsetW - ksize_width) / stride_width + 1;
+ int phend = min(offsetH / stride_height + 1, output_height);
+ int pwend = min(offsetW / stride_width + 1, output_width);
+ T gradient = 0;
+ int output_idx =
+ (batch_idx * channels + offsetC) * output_height * output_width;
+ mask_data += output_idx;
+ output_grad += output_idx;
+ for (int ph = phstart; ph < phend; ++ph) {
+ for (int pw = pwstart; pw < pwend; ++pw) {
+ if ((offsetH * input_width + offsetW) ==
+ mask_data[ph * output_width + pw])
+ gradient += output_grad[ph * output_width + pw];
+ }
+ }
+ input_grad[index] = gradient;
+ }
+}
+
+template <typename T>
+class MaxPool2dWithIndexFunctor<platform::GPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input.dims()[0];
+ const int input_channels = input.dims()[1];
+ const int input_height = input.dims()[2];
+ const int input_width = input.dims()[3];
+ const int output_channels = output.dims()[1];
+ const int output_height = output.dims()[2];
+ const int output_width = output.dims()[3];
+ const int ksize_height = ksize[0];
+ const int ksize_width = ksize[1];
+ const int stride_height = strides[0];
+ const int stride_width = strides[1];
+ const int padding_height = paddings[0];
+ const int padding_width = paddings[1];
+
+ const T* input_data = input.data<T>();
+ T* output_data = output.mutable_data<T>(context.GetPlace());
+ T* mask_data = mask.mutable_data<T>(context.GetPlace());
+
+ int nthreads = batch_size * output_channels * output_height * output_width;
+ int blocks = (nthreads + 1024 - 1) / 1024;
+ dim3 threads(1024, 1);
+ dim3 grid(blocks, 1);
+
+ KernelMaxPool2dWithIdxForward<
+ T><<<grid, threads, 0,
+ reinterpret_cast<const platform::CUDADeviceContext&>(context)
+ .stream()>>>(nthreads, input_data, output_data, mask_data,
+ input_channels, input_height, input_width,
+ output_height, output_width, ksize_height,
+ ksize_width, stride_height, stride_width,
+ padding_height, padding_width);
+ }
+};
+
+template <typename T>
+class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input_grad.dims()[0];
+ const int input_channels = input_grad.dims()[1];
+ const int input_height = input_grad.dims()[2];
+ const int input_width = input_grad.dims()[3];
+ const int output_channels = output_grad.dims()[1];
+ const int output_height = output_grad.dims()[2];
+ const int output_width = output_grad.dims()[3];
+ const int ksize_height = ksize[0];
+ const int ksize_width = ksize[1];
+ const int stride_height = strides[0];
+ const int stride_width = strides[1];
+ const int padding_height = paddings[0];
+ const int padding_width = paddings[1];
+
+ const T* mask_data = mask.data<T>();
+ const T* output_grad_data = output_grad.data<T>();
+ T* input_grad_data = input_grad.mutable_data<T>(context.GetPlace());
+
+ int nthreads = batch_size * input_channels * input_height * input_width;
+ int blocks = (nthreads + 1024 - 1) / 1024;
+ dim3 threads(1024, 1);
+ dim3 grid(blocks, 1);
+
+ KernelMaxPool2DWithIdxBackward<
+ T><<<grid, threads, 0,
+ reinterpret_cast<const platform::CUDADeviceContext&>(context)
+ .stream()>>>(nthreads, input_grad_data, output_grad_data,
+ mask_data, input_channels, input_height,
+ input_width, output_height, output_width,
+ ksize_height, ksize_width, stride_height,
+ stride_width, padding_height, padding_width);
+ }
+};
+
+template class MaxPool2dWithIndexFunctor<platform::GPUPlace, float>;
+template class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, float>;
+template class MaxPool2dWithIndexFunctor<platform::GPUPlace, double>;
+template class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, double>;
+
+template <typename T>
+__global__ void KernelMaxPool3DWithIdxForward(
+ const int nthreads, const T* input_data, T* output_data, T* mask_data,
+ const int channels, const int input_depth, const int input_height,
+ const int input_width, const int output_depth, const int output_height,
+ const int output_width, const int ksize_depth, const int ksize_height,
+ const int ksize_width, const int stride_depth, const int stride_height,
+ const int stride_width, const int padding_depth, const int padding_height,
+ const int padding_width) {
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ index += blockDim.x * gridDim.x) {
+ int pw = index % output_width;
+ int ph = (index / output_width) % output_height;
+ int pd = (index / output_width / output_height) % output_depth;
+ int c = (index / output_width / output_height / output_depth) % channels;
+ int batch_idx =
+ index / output_width / output_height / output_depth / channels;
+ int dstart = pd * stride_depth - padding_depth;
+ int hstart = ph * stride_height - padding_height;
+ int wstart = pw * stride_width - padding_width;
+ int dend = min(dstart + ksize_depth, input_depth);
+ int hend = min(hstart + ksize_height, input_height);
+ int wend = min(wstart + ksize_width, input_width);
+ dstart = max(dstart, 0);
+ hstart = max(hstart, 0);
+ wstart = max(wstart, 0);
+ T ele = -FLT_MAX;
+ int index = -1;
+ input_data +=
+ (batch_idx * channels + c) * input_depth * input_height * input_width;
+
+ for (int d = dstart; d < dend; ++d) {
+ for (int h = hstart; h < hend; ++h) {
+ for (int w = wstart; w < wend; ++w) {
+ if (ele < input_data[(d * input_height + h) * input_width + w]) {
+ index = (d * input_height + h) * input_width + w;
+ ele = input_data[(d * input_height + h) * input_width + w];
+ }
+ }
+ }
+ }
+ output_data[index] = ele;
+ mask_data[index] = index;
+ }
+}
+
+template <typename T>
+__global__ void KernelMaxPool3DWithIdxBackward(
+ const int nthreads, T* input_grad, const T* output_grad, const T* mask,
+ const int channels, const int input_depth, const int input_height,
+ const int input_width, const int output_depth, const int output_height,
+ const int output_width, const int ksize_depth, const int ksize_height,
+ const int ksize_width, const int stride_depth, const int stride_height,
+ const int stride_width, const int padding_depth, const int padding_height,
+ const int padding_width) {
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ index += blockDim.x * gridDim.x) {
+ int offsetW = index % input_width + padding_width;
+ int offsetH = (index / input_width) % input_height + padding_height;
+ int offsetD =
+ (index / input_width / input_height) % input_depth + padding_depth;
+ int offsetC = (index / input_width / input_height / input_depth) % channels;
+ int batch_idx = index / input_width / input_height / input_depth / channels;
+
+ int pdstart = (offsetD < ksize_depth)
+ ? 0
+ : (offsetD - ksize_depth) / stride_depth + 1;
+ int phstart = (offsetH < ksize_height)
+ ? 0
+ : (offsetH - ksize_height) / stride_height + 1;
+ int pwstart = (offsetW < ksize_width)
+ ? 0
+ : (offsetW - ksize_width) / stride_width + 1;
+ int pdend = min((offsetD) / stride_depth + 1, output_depth);
+ int phend = min((offsetH) / stride_height + 1, output_height);
+ int pwend = min((offsetW) / stride_width + 1, output_width);
+
+ T gradient = 0;
+ int output_idx = (batch_idx * channels + offsetC) * output_depth *
+ output_height * output_width;
+ mask += output_idx;
+ output_grad += output_idx;
+
+ for (int pd = pdstart; pd < pdend; ++pd) {
+ for (int ph = phstart; ph < phend; ++ph) {
+ for (int pw = pwstart; pw < pwend; ++pw) {
+ if (((offsetD * input_height + offsetH) * input_width + offsetW) ==
+ mask[(pd * output_height + ph) * output_width + pw])
+ gradient +=
+ output_grad[(pd * output_height + ph) * output_width + pw];
+ }
+ }
+ }
+ input_grad[index] = gradient;
+ }
+}
+
+template <typename T>
+class MaxPool3dWithIndexFunctor<platform::GPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input.dims()[0];
+ const int input_channels = input.dims()[1];
+ const int input_depth = input.dims()[2];
+ const int input_height = input.dims()[3];
+ const int input_width = input.dims()[4];
+ const int output_channels = output.dims()[1];
+ const int output_depth = output.dims()[2];
+ const int output_height = output.dims()[3];
+ const int output_width = output.dims()[4];
+ const int ksize_depth = ksize[0];
+ const int ksize_height = ksize[1];
+ const int ksize_width = ksize[2];
+ const int stride_depth = strides[0];
+ const int stride_height = strides[1];
+ const int stride_width = strides[2];
+ const int padding_depth = paddings[0];
+ const int padding_height = paddings[1];
+ const int padding_width = paddings[2];
+
+ const T* input_data = input.data<T>();
+ T* output_data = output.mutable_data<T>(context.GetPlace());
+ T* mask_data = output.mutable_data<T>(context.GetPlace());
+
+ int nthreads = batch_size * output_channels * output_depth * output_height *
+ output_width;
+ int blocks = (nthreads + 1024 - 1) / 1024;
+ dim3 threads(1024, 1);
+ dim3 grid(blocks, 1);
+
+ KernelMaxPool3DWithIdxForward<
+ T><<<grid, threads, 0,
+ reinterpret_cast<const platform::CUDADeviceContext&>(context)
+ .stream()>>>(
+ nthreads, input_data, output_data, mask_data, input_channels,
+ input_depth, input_height, input_width, output_depth, output_height,
+ output_width, ksize_depth, ksize_height, ksize_width, stride_depth,
+ stride_height, stride_width, padding_depth, padding_height,
+ padding_width);
+ }
+};
+
+template <typename T>
+class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input_grad.dims()[0];
+ const int input_channels = input_grad.dims()[1];
+ const int input_depth = input_grad.dims()[2];
+ const int input_height = input_grad.dims()[3];
+ const int input_width = input_grad.dims()[4];
+ const int output_channels = input_grad.dims()[1];
+ const int output_depth = input_grad.dims()[2];
+ const int output_height = input_grad.dims()[3];
+ const int output_width = input_grad.dims()[4];
+ const int ksize_depth = ksize[0];
+ const int ksize_height = ksize[1];
+ const int ksize_width = ksize[2];
+ const int stride_depth = strides[0];
+ const int stride_height = strides[1];
+ const int stride_width = strides[2];
+ const int padding_depth = paddings[0];
+ const int padding_height = paddings[1];
+ const int padding_width = paddings[2];
+
+ const T* output_grad_data = output_grad.data<T>();
+ const T* mask_data = mask.data<T>();
+ T* input_grad_data = input_grad.mutable_data<T>(context.GetPlace());
+
+ int nthreads =
+ batch_size * input_channels * input_depth * input_height * input_width;
+ int blocks = (nthreads + 1024 - 1) / 1024;
+ dim3 threads(1024, 1);
+ dim3 grid(blocks, 1);
+
+ KernelMaxPool3DWithIdxBackward<
+ T><<<grid, threads, 0,
+ reinterpret_cast<const platform::CUDADeviceContext&>(context)
+ .stream()>>>(
+ nthreads, input_grad_data, output_grad_data, mask_data, input_channels,
+ input_depth, input_height, input_width, output_depth, output_height,
+ output_width, ksize_depth, ksize_height, ksize_width, stride_depth,
+ stride_height, stride_width, padding_depth, padding_height,
+ padding_width);
+ }
+};
+
+template class MaxPool3dWithIndexFunctor<platform::GPUPlace, float>;
+template class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, float>;
+template class MaxPool3dWithIndexFunctor<platform::GPUPlace, double>;
+template class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, double>;
+
+} // namespace math
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h
new file mode 100644
index 0000000000..3a05cd98fe
--- /dev/null
+++ b/paddle/operators/math/pooling.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/tensor.h"
+#include "paddle/platform/device_context.h"
+#include "paddle/platform/hostdevice.h"
+
+namespace paddle {
+namespace operators {
+namespace math {
+//////////////////////
+#define FLT_MAX __FLT_MAX__
+/////////////////////
+
+template <typename Place, typename T>
+class MaxPool2dWithIndexFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+template <typename Place, typename T>
+class MaxPool2dWithIndexGradFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+template <typename Place, typename T>
+class MaxPool3dWithIndexFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+template <typename Place, typename T>
+class MaxPool3dWithIndexGradFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+} // namespace math
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc
new file mode 100644
index 0000000000..d7a07a403d
--- /dev/null
+++ b/paddle/operators/pool_with_index_op.cc
@@ -0,0 +1,198 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/pool_with_index_op.h"
+
+namespace paddle {
+namespace operators {
+
+int OutputSizeMaxPool(int input_size, int filter_size, int padding,
+ int stride) {
+ int output_size = (input_size - filter_size + 2 * padding) / stride + 1;
+ return output_size;
+}
+
+class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContextBase *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("X"),
+ "X(Input) of Pooling should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Out"),
+ "Out(Output) of Pooling should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Mask"),
+ "Out(Output) of Pooling should not be null.");
+
+ auto in_x_dims = ctx->GetInputDim("X");
+
+ std::vector<int> ksize = ctx->Attrs().Get<std::vector<int>>("ksize");
+ std::vector<int> strides = ctx->Attrs().Get<std::vector<int>>("strides");
+ std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
+
+ PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5,
+ "Pooling intput should be 4-D or 5-D");
+
+ if (ctx->Attrs().Get<bool>("globalPooling")) {
+ ksize.resize(static_cast<size_t>(in_x_dims.size()) - 2);
+ for (size_t i = 0; i < ksize.size(); ++i)
+ ksize[i] = static_cast<int>(in_x_dims[i + 2]);
+ }
+
+ PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2U,
+ "Pooling intput size and pooling size should be consistent");
+ PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3,
+ "Pooling size size should be 2 elements. or 3 elements.");
+ PADDLE_ENFORCE_EQ(ksize.size(), strides.size(),
+ "strides size and pooling size should be the same.");
+ PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(),
+ "paddings size and pooling size should be the same.");
+
+ std::vector<int64_t> output_shape({in_x_dims[0], in_x_dims[1]});
+ for (size_t i = 0; i < ksize.size(); ++i) {
+ output_shape.push_back(OutputSizeMaxPool(in_x_dims[i + 2], ksize[i],
+ paddings[i], strides[i]));
+ }
+ ctx->SetOutputDim("Out", framework::make_ddim(output_shape));
+ ctx->SetOutputDim("Mask", framework::make_ddim(output_shape));
+ }
+};
+
+class MaxPoolWithIndexOpGrad : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContextBase *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("X")),
+ "X(Input) of MaxPoolWithIndexOpGrad should not be null.");
+ PADDLE_ENFORCE(
+ ctx->HasOutput(framework::GradVarName("X")),
+ "X@GRAD(Input@GRAD) of MaxPoolWithIndexOpGrad should not be null.");
+ ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
+ }
+};
+
+class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ MaxPool2dWithIndexOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput(
+ "X",
+ "The input tensor of pooling operator. "
+ "The format of input tensor is NCHW. Where N is batch size, C is the "
+ "number of channels, H and W is the height and width of image.");
+ AddOutput("Out",
+ "The output tensor of pooling operator."
+ "The format of output tensor is also NCHW.");
+ AddOutput("Mask",
+ "The Mask tensor of pooling operator."
+ "The format of output tensor is also NCHW.");
+
+ AddAttr<std::vector<int>>(
+ "ksize", "pooling size(height, width) of pooling operator.");
+ AddAttr<bool>(
+ "globalPooling",
+ "whether to use the globalPooling."
+ "int constant equal to false or true"
+ "default false"
+ "If globalPooling = true, ksize is ignored and need not be specified.")
+ .SetDefault(false);
+ AddAttr<std::vector<int>>("strides",
+ "strides(height, width) of pooling operator."
+ "default {1,1}")
+ .SetDefault({1, 1});
+ AddAttr<std::vector<int>>("paddings",
+ "paddings(height, width) of pooling operator."
+ "default {0,0}")
+ .SetDefault({0, 0});
+
+ AddComment(R"DOC(
+The maxPooling2d with index operation calculates the output and the mask based on
+the input and ksize, strides, paddings parameters.
+)DOC");
+ }
+};
+
+class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ MaxPool3dWithIndexOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput(
+ "X",
+ "The input tensor of pooling operator. "
+ "The format of input tensor is NCDHW. Where N is batch size, C is "
+ "the number of channels, D, H and W is the depth, height and width of "
+ "image.");
+ AddOutput("Out",
+ "The output tensor of pooling operator."
+ "The format of output tensor is also NCDHW.");
+ AddOutput("Mask",
+ "The Mask tensor of pooling operator."
+ "The format of output tensor is also NCDHW.");
+
+ AddAttr<std::vector<int>>(
+ "ksize", "pooling size(depth, height, width) of pooling operator.");
+ AddAttr<bool>(
+ "globalPooling",
+ "whether to use the globalPooling."
+ "int constant equal to false or true"
+ "default false"
+ "If globalPooling = true, ksize is ignored and need not be specified.")
+ .SetDefault(false);
+ AddAttr<std::vector<int>>(
+ "strides",
+ "strides(depth, height, width) of pooling operator."
+ "default {1,1,1}")
+ .SetDefault({1, 1, 1});
+ AddAttr<std::vector<int>>(
+ "paddings",
+ "paddings(depth, height, width) of pooling operator."
+ "default {0,0,0}")
+ .SetDefault({0, 0, 0});
+ AddComment(R"DOC(
+The maxpooling3d with index operation calculates the output and the mask based on
+the input and ksize, strides, paddings parameters.
+)DOC");
+ }
+};
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+
+REGISTER_OP(maxPool2dWithIndex, ops::MaxPoolWithIndexOp,
+ ops::MaxPool2dWithIndexOpMaker, maxPool2dWithIndex_grad,
+ ops::MaxPoolWithIndexOpGrad);
+
+REGISTER_OP_CPU_KERNEL(
+ maxPool2dWithIndex,
+ ops::MaxPoolWithIndexKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+ maxPool2dWithIndex_grad,
+ ops::MaxPoolWithIndexGradKernel<paddle::platform::CPUPlace, float>)
+
+REGISTER_OP(maxPool3dWithIndex, ops::MaxPoolWithIndexOp,
+ ops::MaxPool3dWithIndexOpMaker, maxPool3dWithIndex_grad,
+ ops::MaxPoolWithIndexOpGrad);
+
+REGISTER_OP_CPU_KERNEL(
+ maxPool3dWithIndex,
+ ops::MaxPoolWithIndexKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+ maxPool3dWithIndex_grad,
+ ops::MaxPoolWithIndexGradKernel<paddle::platform::CPUPlace, float>)
diff --git a/paddle/operators/pool_with_index_op.cu b/paddle/operators/pool_with_index_op.cu
new file mode 100644
index 0000000000..8007fc7ccf
--- /dev/null
+++ b/paddle/operators/pool_with_index_op.cu
@@ -0,0 +1,31 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/pool_with_index_op.h"
+
+namespace ops = paddle::operators;
+
+REGISTER_OP_GPU_KERNEL(
+ maxPool2dWithIndex,
+ ops::MaxPoolWithIndexKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(
+ maxPool2dWithIndex_grad,
+ ops::MaxPoolWithIndexGradKernel<paddle::platform::GPUPlace, float>)
+
+REGISTER_OP_GPU_KERNEL(
+ maxPool3dWithIndex,
+ ops::MaxPoolWithIndexKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(
+ maxPool3dWithIndex_grad,
+ ops::MaxPoolWithIndexGradKernel<paddle::platform::GPUPlace, float>)
diff --git a/paddle/operators/pool_with_index_op.h b/paddle/operators/pool_with_index_op.h
new file mode 100644
index 0000000000..91abeed016
--- /dev/null
+++ b/paddle/operators/pool_with_index_op.h
@@ -0,0 +1,99 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+#include "paddle/operators/math/math_function.h"
+#include "paddle/operators/math/pooling.h"
+
+namespace paddle {
+namespace operators {
+
+using Tensor = framework::Tensor;
+
+template <typename Place, typename T>
+class MaxPoolWithIndexKernel : public framework::OpKernel {
+ public:
+ void Compute(const framework::ExecutionContext& context) const override {
+ const Tensor* in_x = context.Input<Tensor>("X");
+ Tensor* out = context.Output<Tensor>("Out");
+ Tensor* mask = context.Output<Tensor>("Mask");
+
+ bool global_pooling = context.Attr<bool>("globalPooling");
+ std::vector<int> ksize = context.Attr<std::vector<int>>("ksize");
+ std::vector<int> strides = context.Attr<std::vector<int>>("strides");
+ std::vector<int> paddings = context.Attr<std::vector<int>>("paddings");
+ if (global_pooling) {
+ for (size_t i = 0; i < ksize.size(); ++i) {
+ ksize[i] = static_cast<int>(in_x->dims()[i + 2]);
+ }
+ }
+
+ switch (ksize.size()) {
+ case 2: {
+ paddle::operators::math::MaxPool2dWithIndexFunctor<Place, T>
+ pool2d_forward;
+ pool2d_forward(context.device_context(), *in_x, *out, *mask, ksize,
+ strides, paddings);
+ } break;
+ case 3: {
+ paddle::operators::math::MaxPool3dWithIndexFunctor<Place, T>
+ pool3d_forward;
+ pool3d_forward(context.device_context(), *in_x, *out, *mask, ksize,
+ strides, paddings);
+ } break;
+ }
+ }
+};
+
+template <typename Place, typename T>
+class MaxPoolWithIndexGradKernel : public framework::OpKernel {
+ public:
+ void Compute(const framework::ExecutionContext& context) const override {
+ const Tensor* mask = context.Input<Tensor>("Maks");
+ const Tensor* out_grad =
+ context.Input<Tensor>(framework::GradVarName("Out"));
+ Tensor* in_x_grad = context.Output<Tensor>(framework::GradVarName("X"));
+
+ std::vector<int> ksize = context.Attr<std::vector<int>>("ksize");
+ std::vector<int> strides = context.Attr<std::vector<int>>("strides");
+ std::vector<int> paddings = context.Attr<std::vector<int>>("paddings");
+
+ if (in_x_grad) {
+ in_x_grad->mutable_data<T>(context.GetPlace());
+ auto temp = framework::EigenVector<T>::Flatten(*in_x_grad);
+ temp.device(context.GetEigenDevice<Place>()) =
+ temp.constant(static_cast<T>(0));
+
+ switch (ksize.size()) {
+ case 2: {
+ paddle::operators::math::MaxPool2dWithIndexGradFunctor<Place, T>
+ pool2d_backward;
+ pool2d_backward(context.device_context(), *in_x_grad, *out_grad,
+ *mask, ksize, strides, paddings);
+ } break;
+ case 3: {
+ paddle::operators::math::MaxPool3dWithIndexGradFunctor<Place, T>
+ pool3d_backward;
+ pool3d_backward(context.device_context(), *in_x_grad, *out_grad,
+ *mask, ksize, strides, paddings);
+ } break;
+ }
+ }
+ }
+};
+} // namespace operators
+} // namespace paddle
diff --git a/python/paddle/v2/framework/tests/test_pool_max_op.py b/python/paddle/v2/framework/tests/test_pool_max_op.py
new file mode 100644
index 0000000000..2945c8b7a4
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_pool_max_op.py
@@ -0,0 +1,125 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0):
+
+ N, C, D, H, W = x.shape
+ if global_pool == 1:
+ ksize = [D, H, W]
+ D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1
+ H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1
+ W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1
+ out = np.zeros((N, C, D_out, H_out, W_out))
+ mask = np.zeros((N, C, D_out, H_out, W_out))
+ for k in xrange(D_out):
+ d_start = np.max((k * strides[0] - paddings[0], 0))
+ d_end = np.min((k * strides[0] + ksize[0] - paddings[0], D))
+ for i in xrange(H_out):
+ h_start = np.max((i * strides[0] - paddings[0], 0))
+ h_end = np.min((i * strides[0] + ksize[0] - paddings[0], H))
+ for j in xrange(W_out):
+ w_start = np.max((j * strides[1] - paddings[1], 0))
+ w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W))
+ x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end]
+
+ out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4))
+ # mask[:,:, k, i, j] = np.argmax(x_masked, axis=(2, 3, 4))
+ return out
+
+
+def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0):
+
+ N, C, H, W = x.shape
+ if global_pool == 1:
+ ksize = [H, W]
+ H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1
+ W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1
+ out = np.zeros((N, C, H_out, W_out))
+ mask = np.zeros((N, C, H_out, W_out))
+ for i in xrange(H_out):
+ for j in xrange(W_out):
+ r_start = np.max((i * strides[0] - paddings[0], 0))
+ r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H))
+ c_start = np.max((j * strides[1] - paddings[1], 0))
+ c_end = np.min((j * strides[1] + ksize[1] - paddings[1], W))
+ x_masked = x[:, :, r_start:r_end, c_start:c_end]
+
+ out[:, :, i, j] = np.max(x_masked, axis=(2, 3))
+ # mask[:,:, i, j] = np.argmax(x_masked, axis=(2, 3))
+
+ return out
+
+
+class TestMaxPoolWithIndex_Op(OpTest):
+ def setUp(self):
+ self.initTestCase()
+ self.op_type = "maxPool3dWithIndex"
+ input = np.random.random(self.shape).astype("float32")
+ output = self.pool_forward_naive(input, self.ksize, self.strides,
+ self.paddings, self.global_pool)
+ # mask = np.zeros(output.shape)
+
+ self.attrs = {
+ 'strides': self.strides,
+ 'paddings': self.paddings,
+ 'ksize': self.ksize,
+ 'globalPooling': self.global_pool,
+ }
+
+ self.inputs = {'X': input}
+ self.outputs = {'Out': output}
+
+ def test_check_output(self):
+ self.check_output()
+
+ # def test_check_grad(self):
+ # self.check_grad(set(['X']), ['Out'], max_relative_error=0.07)
+
+ def initTestCase(self):
+ self.global_pool = 0
+ self.pool_forward_naive = max_pool3D_forward_naive
+ self.shape = [2, 3, 7, 7, 7]
+ self.ksize = [3, 3, 3]
+ self.strides = [1, 1, 1]
+ self.paddings = [1, 1, 1]
+
+
+""""
+class TestCase1(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = 1
+ self.op_type = "maxPool3dWithIndex"
+ self.pool_forward_naive = max_pool3D_forward_naive
+ self.shape = [2, 3, 5, 5, 5]
+ self.ksize = [3, 3, 3]
+ self.strides = [1, 1, 1]
+ self.paddings = [0, 0, 0]
+
+
+class TestCase2(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = 0
+ self.op_type = "maxPool2dWithIndex"
+ self.pool_forward_naive = max_pool2D_forward_naive
+ self.shape = [2, 3, 7, 7]
+ self.ksize = [3, 3]
+ self.strides = [1, 1]
+ self.paddings = [1, 1]
+
+
+class TestCase3(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = 1
+ self.op_type = "maxPool2dWithIndex"
+ self.pool_forward_naive = max_pool2D_forward_naive
+ self.shape = [2, 3, 5, 5]
+ self.ksize = [3, 3]
+ self.strides = [1, 1]
+ self.paddings = [0, 0]
+
+
+if __name__ == '__main__':
+ unittest.main()
+"""
From 884e31a59b72856ea1a807561f01a623c1138053 Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Fri, 29 Sep 2017 15:28:25 +0800
Subject: [PATCH 02/82] add interpolation op
---
paddle/operators/interp_op.cc | 107 ++++++++++++++++++
.../v2/framework/tests/test_interp_op.py | 28 +++++
2 files changed, 135 insertions(+)
create mode 100644 paddle/operators/interp_op.cc
create mode 100644 python/paddle/v2/framework/tests/test_interp_op.py
diff --git a/paddle/operators/interp_op.cc b/paddle/operators/interp_op.cc
new file mode 100644
index 0000000000..04bcb9ade8
--- /dev/null
+++ b/paddle/operators/interp_op.cc
@@ -0,0 +1,107 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/op_registry.h"
+#include "paddle/operators/net_op.h"
+
+namespace paddle {
+namespace operators {
+
+class InterpOp : public NetOp {
+ public:
+ InterpOp(const std::string &type, const framework::VariableNameMap &inputs,
+ const framework::VariableNameMap &outputs,
+ const framework::AttributeMap &attrs)
+ : NetOp(type, inputs, outputs, attrs) {
+ PADDLE_ENFORCE_NE(Input("X"), framework::kEmptyVarName,
+ "Input(X) of InterpOp should not be null.");
+ PADDLE_ENFORCE_NE(Input("Y"), framework::kEmptyVarName,
+ "Input(Y) of InterpOp should not be null.");
+ PADDLE_ENFORCE_NE(Input("W"), framework::kEmptyVarName,
+ "Input(W) of InterpOp should not be null.");
+ PADDLE_ENFORCE_NE(Output("MinusOut"), framework::kEmptyVarName,
+ "Output(MinusOut) of InterpOp should not be null.");
+ PADDLE_ENFORCE_NE(Output("MulOut"), framework::kEmptyVarName,
+ "Output(MulOut) of InterpOp should not be null.");
+ PADDLE_ENFORCE_NE(Output("Out"), framework::kEmptyVarName,
+ "Output(Out) of InterpOp should not be null.");
+
+ // MinusOut = X - Y
+ auto x = Input("X");
+ auto y = Input("Y");
+ auto minus_out = Output("MinusOut");
+ AppendOp(framework::OpRegistry::CreateOp("elementwise_sub",
+ {{"X", {x}}, {"Y", {y}}},
+ {{"Out", {minus_out}}}, {}));
+
+ // MulOut = MinusOut * W = (X - Y) * W
+ auto w = Input("W");
+ auto mul_out = Output("MulOut");
+ AppendOp(framework::OpRegistry::CreateOp(
+ "elementwise_mul", {{"X", {minus_out}}, {"Y", {w}}},
+ {{"Out", {mul_out}}}, {{"axis", 0}}));
+
+ // Out = MulOut + Y = (X - Y) * W + Y = X * W + Y * (1 - W)
+ AppendOp(framework::OpRegistry::CreateOp("elementwise_add",
+ {{"X", {mul_out}}, {"Y", {y}}},
+ {{"Out", {Output("Out")}}}, {}));
+
+ CompleteAddOp(false);
+ LOG(INFO) << DebugString();
+ }
+};
+
+class InterpOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ InterpOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "A 2-D Tensor, the first input of interp_op");
+ AddInput("Y", "A 2-D Tensor, the second input of interp_op");
+ AddInput("W", "A 1-D Tensor, the interpolated values");
+ AddOutput("MinusOut",
+ "A 2-D Tensor, the intermediate outputs, saving X - Y.")
+ .AsIntermediate();
+ AddOutput("MulOut",
+ "A 2-D Tensor, the intermediate outputs,"
+ "saving the mul mul of (X - Y) and W")
+ .AsIntermediate();
+ AddOutput("Out",
+ "A 2-D Tensor, the output of interp_op, same shape with X");
+ AddComment(R"DOC(
+ Linear Interpolation with two inputs, used in NEURAL TURING MACHINE.
+
+ Equation:
+ Out.row[i] = X.row[i] * W[i] + Y.row[i] * (1 - W[i])
+ = (X.row[i] - Y.row[i]) * W[i] + Y.row[i]
+
+ Example:
+ X = [[1,2],[3,4]],
+ Y = [[2,1],[4,3]],
+ W = [0.3, 0.4]
+
+ Then, Out = [[1.7,1.3],[3.6,3.4]]
+
+ where 1.7 = 1*0.3+2*(1-0.3),
+ 1.3 = 2*0.3+1*(1-0.3),
+ 3.6 = 3*0.4+4*(1-0.4),
+ 3.4 = 4*0.4+3*(1-0.4)
+)DOC");
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(interp, ops::InterpOp, ops::InterpOpMaker);
diff --git a/python/paddle/v2/framework/tests/test_interp_op.py b/python/paddle/v2/framework/tests/test_interp_op.py
new file mode 100644
index 0000000000..f82dcc7f50
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_interp_op.py
@@ -0,0 +1,28 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+class TestInterpOp(OpTest):
+ def setUp(self):
+ self.op_type = "interp"
+ x = np.random.random((2, 3)).astype("float32")
+ y = np.random.random((2, 3)).astype("float32")
+ w = np.random.random(2).astype("float32")
+
+ minus_out = x - y
+ mul_out = minus_out * w.reshape(2, 1)
+ out = mul_out + y
+
+ self.inputs = {'X': x, 'Y': y, 'W': w}
+ self.outputs = {'Out': out, 'MinusOut': minus_out, 'MulOut': mul_out}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad_normal(self):
+ self.check_grad(['X', 'Y'], 'Out')
+
+
+if __name__ == "__main__":
+ unittest.main()
From a815d6abcf49d4778d0a49c852c45264bd8a684a Mon Sep 17 00:00:00 2001
From: zhouxiao-coder <zhouxiaocoder@gmail.com>
Date: Fri, 29 Sep 2017 17:29:52 +0800
Subject: [PATCH 03/82] elu: Optimize gradient calculation;Add more comments
---
paddle/operators/activation_op.cc | 25 ++++++++++++
paddle/operators/activation_op.cu | 4 ++
paddle/operators/activation_op.h | 40 +++++++++++++++++++
.../v2/framework/tests/test_activation_op.py | 20 ++++++++++
4 files changed, 89 insertions(+)
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index 1e1d3cf7f7..e83666c9f9 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -174,6 +174,25 @@ class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+template <typename AttrType>
+class ELUOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ ELUOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X",
+ "Input of ELU operator, it shouldn't be empty. Input is flattened "
+ "and treated as a 1D array.");
+ AddOutput("Y", "Output of ELU operator, has same shape as the input.");
+ AddComment(
+ "ELU activation operator. It applies this element-wise computation on "
+ "the input: f(x) = max(0, x) + min(0, alpha * (exp(x) - 1))."
+ "Check .. _Link: https://arxiv.org/abs/1511.07289 for more details");
+ AddAttr<AttrType>("alpha",
+ "alpha value in the elu formulation, default to 1.")
+ .SetDefault(static_cast<AttrType>(1.));
+ }
+};
+
template <typename AttrType>
class PowOpMaker : public framework::OpProtoAndCheckerMaker {
public:
@@ -311,6 +330,12 @@ REGISTER_OP_CPU_KERNEL(soft_relu,
REGISTER_OP_CPU_KERNEL(
soft_relu_grad, ops::SoftReluGradKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP(elu, ops::ActivationOp, ops::ELUOpMaker<float>, elu_grad,
+ ops::ActivationOpGrad);
+REGISTER_OP_CPU_KERNEL(elu, ops::ELUKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(elu_grad,
+ ops::ELUGradKernel<paddle::platform::CPUPlace, float>);
+
REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker<float>, pow_grad,
ops::ActivationOpGrad);
REGISTER_OP_CPU_KERNEL(pow, ops::PowKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu
index 56886d8b1b..48800b11ec 100644
--- a/paddle/operators/activation_op.cu
+++ b/paddle/operators/activation_op.cu
@@ -97,6 +97,10 @@ REGISTER_OP_GPU_KERNEL(soft_relu,
REGISTER_OP_GPU_KERNEL(
soft_relu_grad, ops::SoftReluGradKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(elu, ops::ELUKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(elu_grad,
+ ops::ELUGradKernel<paddle::platform::GPUPlace, float>);
+
REGISTER_OP_GPU_KERNEL(pow, ops::PowKernel<paddle::platform::GPUPlace, float>);
REGISTER_OP_GPU_KERNEL(pow_grad,
ops::PowGradKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h
index b9f52e1af3..3428aca817 100644
--- a/paddle/operators/activation_op.h
+++ b/paddle/operators/activation_op.h
@@ -296,6 +296,46 @@ class SoftReluGradKernel : public framework::OpKernel<T> {
}
};
+template <typename Place, typename T, typename AttrType = T>
+class ELUKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& context) const override {
+ auto* X = context.Input<framework::Tensor>("X");
+ auto* Y = context.Output<framework::Tensor>("Y");
+ auto alpha = static_cast<T>(context.Attr<AttrType>("alpha"));
+ Y->mutable_data<T>(context.GetPlace());
+
+ auto x = framework::EigenVector<T>::Flatten(*X);
+ auto y = framework::EigenVector<T>::Flatten(*Y);
+ auto place = context.GetEigenDevice<Place>();
+ y.device(place) =
+ x.cwiseMax(static_cast<T>(0)) +
+ (alpha * (x.exp() - static_cast<T>(1))).cwiseMin(static_cast<T>(0));
+ }
+};
+
+template <typename Place, typename T, typename AttrType = T>
+class ELUGradKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& context) const override {
+ auto* X = context.Input<framework::Tensor>("X");
+ auto* Y = context.Input<framework::Tensor>("Y");
+ auto* dY = context.Input<framework::Tensor>(framework::GradVarName("Y"));
+ auto* dX = context.Output<framework::Tensor>(framework::GradVarName("X"));
+ auto alpha = static_cast<T>(context.Attr<AttrType>("alpha"));
+ dX->mutable_data<T>(context.GetPlace());
+
+ auto x = framework::EigenVector<T>::Flatten(*X);
+ auto y = framework::EigenVector<T>::Flatten(*Y);
+ auto dy = framework::EigenVector<T>::Flatten(*dY);
+ auto dx = framework::EigenVector<T>::Flatten(*dX);
+ auto place = context.GetEigenDevice<Place>();
+ dx.device(place) =
+ dy * (x > static_cast<T>(0)).template cast<T>() +
+ dy * (y + alpha) * (x < static_cast<T>(0)).template cast<T>();
+ }
+};
+
template <typename Place, typename T, typename AttrType = T>
class PowKernel : public framework::OpKernel<T> {
public:
diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py
index c44eb84906..9ea01d43c5 100644
--- a/python/paddle/v2/framework/tests/test_activation_op.py
+++ b/python/paddle/v2/framework/tests/test_activation_op.py
@@ -144,6 +144,26 @@ class TestSoftRelu(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.02)
+class TestELU(OpTest):
+ def setUp(self):
+ self.op_type = "elu"
+ x = np.random.uniform(-3, 3, [4, 4]).astype("float32")
+ alpha = 1.
+ # Note: unlike other Relu extensions, point 0 on standard ELU function (i.e. alpha = 1)
+ # is differentiable, so we can skip modifications like x[np.abs(x) < 0.005] = 0.02 here
+ self.inputs = {'X': x}
+ self.attrs = {'alpha': alpha}
+ self.outputs = {
+ 'Y': np.maximum(0, x) + np.minimum(0, alpha * (np.exp(x) - 1))
+ }
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=0.02)
+
+
class TestReciprocal(OpTest):
def setUp(self):
self.op_type = "reciprocal"
From 4436ba0c56d105b0c1305a739158fdc08258f7a9 Mon Sep 17 00:00:00 2001
From: zhouxiao-coder <zhouxiaocoder@gmail.com>
Date: Fri, 29 Sep 2017 17:52:18 +0800
Subject: [PATCH 04/82] elu: Optimize gradient calculation;Add more comments
---
paddle/operators/activation_op.cc | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index e83666c9f9..7d086ac5df 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -180,16 +180,18 @@ class ELUOpMaker : public framework::OpProtoAndCheckerMaker {
ELUOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
: OpProtoAndCheckerMaker(proto, op_checker) {
AddInput("X",
- "Input of ELU operator, it shouldn't be empty. Input is flattened "
- "and treated as a 1D array.");
- AddOutput("Y", "Output of ELU operator, has same shape as the input.");
- AddComment(
- "ELU activation operator. It applies this element-wise computation on "
- "the input: f(x) = max(0, x) + min(0, alpha * (exp(x) - 1))."
- "Check .. _Link: https://arxiv.org/abs/1511.07289 for more details");
- AddAttr<AttrType>("alpha",
- "alpha value in the elu formulation, default to 1.")
+ "(Tensor) The input of ELU operator, it shouldn't be empty. Input "
+ "is flattened and treated as a 1D array.");
+ AddOutput("Y",
+ "(Tensor) The output of ELU operator. It has the same shape as "
+ "the input.");
+ AddAttr<AttrType>(
+ "alpha", "(float, default 1.0) Alpha value in the elu formulation.")
.SetDefault(static_cast<AttrType>(1.));
+ AddComment(R"DOC(
+ ELU activation operator. It applies this element-wise computation on
+ the input: f(x) = max(0, x) + min(0, alpha * (exp(x) - 1)).
+ Check .. _Link: https://arxiv.org/abs/1511.07289 for more details.)DOC");
}
};
From 3c66b307f7b6173a69cd4ccc9cf9f7541de964d2 Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Fri, 29 Sep 2017 19:57:02 +0800
Subject: [PATCH 05/82] Remove the pserver, trainer, evaluators and some
useless gradientmachines when compile mobile inference library.
---
CMakeLists.txt | 8 +++
cmake/util.cmake | 57 ++++++++++++-------
paddle/CMakeLists.txt | 35 +++++++-----
paddle/capi/CMakeLists.txt | 8 +--
paddle/gserver/CMakeLists.txt | 22 +++++++
.../gradientmachines/GradientMachine.cpp | 13 ++++-
.../gradientmachines/GradientMachine.h | 7 ++-
.../gradientmachines/NeuralNetwork.cpp | 18 ++++--
.../gserver/gradientmachines/NeuralNetwork.h | 3 +
paddle/gserver/layers/Layer.cpp | 2 +
10 files changed, 128 insertions(+), 45 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4921226ec1..ec4e6e2e86 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -86,6 +86,14 @@ if(ANDROID OR IOS)
"Disable MKLDNN when cross-compiling for Android and iOS" FORCE)
set(WITH_MKLML OFF CACHE STRING
"Disable MKLML package when cross-compiling for Android and iOS" FORCE)
+
+ if(WITH_C_API)
+ # Compile PaddlePaddle mobile inference library
+ set(MOBILE_INFERENCE ON)
+ add_definitions(-DPADDLE_MOBILE_INFERENCE)
+ endif()
+ set(WITH_TESTING OFF CACHE STRING "Disable TESTING when cross-compiling
+ for Android and iOS" FORCE)
endif()
set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING
diff --git a/cmake/util.cmake b/cmake/util.cmake
index d1aee3e170..5ebfc0945f 100644
--- a/cmake/util.cmake
+++ b/cmake/util.cmake
@@ -73,25 +73,44 @@ function(link_paddle_exe TARGET_NAME)
generate_rdma_links()
endif()
- target_circle_link_libraries(${TARGET_NAME}
- ARCHIVE_START
- paddle_gserver
- paddle_function
- ARCHIVE_END
- paddle_pserver
- paddle_trainer_lib
- paddle_network
- paddle_math
- paddle_utils
- paddle_parameter
- paddle_proto
- paddle_cuda
- paddle_optimizer
- ${EXTERNAL_LIBS}
- ${CMAKE_THREAD_LIBS_INIT}
- ${CMAKE_DL_LIBS}
- ${RDMA_LD_FLAGS}
- ${RDMA_LIBS})
+ if(MOBILE_INFERENCE)
+ target_circle_link_libraries(${TARGET_NAME}
+ ARCHIVE_START
+ paddle_gserver
+ paddle_function
+ ARCHIVE_END
+ paddle_math
+ paddle_utils
+ paddle_parameter
+ paddle_proto
+ paddle_cuda
+ paddle_optimizer
+ ${EXTERNAL_LIBS}
+ ${CMAKE_THREAD_LIBS_INIT}
+ ${CMAKE_DL_LIBS}
+ ${RDMA_LD_FLAGS}
+ ${RDMA_LIBS})
+ else()
+ target_circle_link_libraries(${TARGET_NAME}
+ ARCHIVE_START
+ paddle_gserver
+ paddle_function
+ ARCHIVE_END
+ paddle_pserver
+ paddle_trainer_lib
+ paddle_network
+ paddle_math
+ paddle_utils
+ paddle_parameter
+ paddle_proto
+ paddle_cuda
+ paddle_optimizer
+ ${EXTERNAL_LIBS}
+ ${CMAKE_THREAD_LIBS_INIT}
+ ${CMAKE_DL_LIBS}
+ ${RDMA_LD_FLAGS}
+ ${RDMA_LIBS})
+ endif()
if(ANDROID)
target_link_libraries(${TARGET_NAME} log)
diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt
index b435de80a2..3eb494ae47 100644
--- a/paddle/CMakeLists.txt
+++ b/paddle/CMakeLists.txt
@@ -3,25 +3,30 @@ add_subdirectory(function)
add_subdirectory(utils)
add_subdirectory(testing)
add_subdirectory(math)
-add_subdirectory(parameter)
add_subdirectory(gserver)
-add_subdirectory(pserver)
-add_subdirectory(trainer)
add_subdirectory(scripts)
add_subdirectory(string)
+add_subdirectory(parameter)
-if(Boost_FOUND)
- add_subdirectory(memory)
- add_subdirectory(platform)
- add_subdirectory(framework)
- add_subdirectory(operators)
- add_subdirectory(pybind)
-endif()
-
-if(WITH_C_API)
+if(MOBILE_INFERENCE)
add_subdirectory(capi)
-endif()
+else()
+ add_subdirectory(pserver)
+ add_subdirectory(trainer)
+
+ if(WITH_C_API)
+ add_subdirectory(capi)
+ endif()
+
+ if(Boost_FOUND)
+ add_subdirectory(memory)
+ add_subdirectory(platform)
+ add_subdirectory(framework)
+ add_subdirectory(operators)
+ add_subdirectory(pybind)
+ endif()
-if(WITH_SWIG_PY)
- add_subdirectory(api)
+ if(WITH_SWIG_PY)
+ add_subdirectory(api)
+ endif()
endif()
diff --git a/paddle/capi/CMakeLists.txt b/paddle/capi/CMakeLists.txt
index b9bbe58951..a19a19d719 100644
--- a/paddle/capi/CMakeLists.txt
+++ b/paddle/capi/CMakeLists.txt
@@ -37,9 +37,7 @@ set(PADDLE_CAPI_INFER_LIBS
paddle_cuda
paddle_function
paddle_gserver
- paddle_proto
- paddle_pserver
- paddle_network)
+ paddle_proto)
cc_library(paddle_capi_whole DEPS paddle_capi ${PADDLE_CAPI_INFER_LIBS})
@@ -50,7 +48,9 @@ if(NOT IOS)
add_library(paddle_capi_shared SHARED ${CAPI_SOURCES})
set_target_properties(paddle_capi_shared PROPERTIES LINK_FLAGS "${LINK_FLAGS}")
target_include_directories(paddle_capi_shared PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
- link_paddle_exe(paddle_capi_shared)
+
+link_paddle_exe(paddle_capi_shared)
+
endif()
# install library & headers.
diff --git a/paddle/gserver/CMakeLists.txt b/paddle/gserver/CMakeLists.txt
index 62cff9361c..cd469875df 100644
--- a/paddle/gserver/CMakeLists.txt
+++ b/paddle/gserver/CMakeLists.txt
@@ -60,6 +60,28 @@ if(NOT WITH_PYTHON)
dataproviders/PyDataProvider.h)
endif()
+if(MOBILE_INFERENCE)
+ # Remove evaluators
+ list(REMOVE_ITEM GSERVER_SOURCES
+ layers/ValidationLayer.cpp
+ evaluators/Evaluator.cpp
+ evaluators/DetectionMAPEvaluator.cpp
+ evaluators/CTCErrorEvaluator.cpp
+ evaluators/ChunkEvaluator.cpp)
+
+ # Remove useless gradientmachines
+ list(REMOVE_ITEM GSERVER_SOURCES
+ gradientmachines/MultiNetwork.cpp
+ gradientmachines/RecurrentGradientMachine.cpp
+ gradientmachines/ParallelNeuralNetwork.cpp
+ gradientmachines/GradientMachineMode.cpp
+ gradientmachines/MultiGradientMachine.cpp)
+
+ # Remove useless layers
+ list(REMOVE_ITEM GSERVER_SOURCES
+ layers/RecurrentLayerGroup.cpp)
+endif()
+
if(WITH_GPU)
cuda_add_library(paddle_gserver ${GSERVER_SOURCES})
else()
diff --git a/paddle/gserver/gradientmachines/GradientMachine.cpp b/paddle/gserver/gradientmachines/GradientMachine.cpp
index b44e4dc202..de5faf5e1e 100644
--- a/paddle/gserver/gradientmachines/GradientMachine.cpp
+++ b/paddle/gserver/gradientmachines/GradientMachine.cpp
@@ -17,12 +17,15 @@ limitations under the License. */
#include <fstream>
#include "paddle/utils/Logging.h"
+#include "NeuralNetwork.h"
+#include "hl_gpu.h"
+
+#ifndef PADDLE_MOBILE_INFERENCE
#include "GradientMachineMode.h"
#include "MultiGradientMachine.h"
#include "MultiNetwork.h"
-#include "NeuralNetwork.h"
#include "ParallelNeuralNetwork.h"
-#include "hl_gpu.h"
+#endif
namespace paddle {
@@ -30,13 +33,16 @@ GradientMachine* GradientMachine::create(
const ModelConfig& config,
int mode,
const std::vector<ParameterType>& parameterTypes) {
+#ifndef PADDLE_MOBILE_INFERENCE
if (auto gm = IGradientMachineMode::tryCreateGradientMachine(mode, config)) {
return gm;
}
if (FLAGS_trainer_count > 1) {
return new MultiGradientMachine(config, FLAGS_use_gpu);
}
+#endif
if (FLAGS_trainer_count == 1) { // single
+#ifndef PADDLE_MOBILE_INFERENCE
NeuralNetwork* nn;
if (config.type() == "multi_nn") {
/* multi submodel calculate, thread(s) will be initialized inside */
@@ -48,6 +54,9 @@ GradientMachine* GradientMachine::create(
/* single thread calculate */
nn = NeuralNetwork::create(config);
}
+#else
+ NeuralNetwork* nn = NeuralNetwork::create(config);
+#endif
ParamInitCallback testParamInitCb = [](int paramId, Parameter* para) {
para->enableType(PARAMETER_VALUE);
};
diff --git a/paddle/gserver/gradientmachines/GradientMachine.h b/paddle/gserver/gradientmachines/GradientMachine.h
index f9c82a2bef..ebfe0573cf 100644
--- a/paddle/gserver/gradientmachines/GradientMachine.h
+++ b/paddle/gserver/gradientmachines/GradientMachine.h
@@ -20,13 +20,16 @@ limitations under the License. */
#include "ModelConfig.pb.h"
#include "TrainerConfig.pb.h"
#include "paddle/gserver/dataproviders/DataProvider.h"
-#include "paddle/gserver/evaluators/Evaluator.h"
#include "paddle/gserver/layers/Layer.h"
#include "paddle/math/Matrix.h"
#include "paddle/parameter/Parameter.h"
#include "paddle/parameter/ParameterUpdaterBase.h"
#include "paddle/utils/Thread.h"
+#ifndef PADDLE_MOBILE_INFERENCE
+#include "paddle/gserver/evaluators/Evaluator.h"
+#endif
+
namespace paddle {
/**
* @brief A gradient machine is capable of calculating some outputs given
@@ -147,6 +150,7 @@ public:
virtual void onPassEnd() = 0;
+#ifndef PADDLE_MOBILE_INFERENCE
/**
* Create an evaluator which can be used for eval()
*/
@@ -156,6 +160,7 @@ public:
* evaluate using the given evaluator
*/
virtual void eval(Evaluator* evaluator) const = 0;
+#endif
std::vector<ParameterPtr>& getParameters() { return parameters_; }
diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp
index 26cff3e677..dcf0acb5a2 100644
--- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp
+++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp
@@ -14,15 +14,17 @@ limitations under the License. */
#include "paddle/utils/Util.h"
+#include "NeuralNetwork.h"
+#include "hl_gpu.h"
+#include "paddle/gserver/layers/AgentLayer.h"
#include "paddle/utils/CustomStackTrace.h"
#include "paddle/utils/Logging.h"
+#include "paddle/utils/Stat.h"
+#ifndef PADDLE_MOBILE_INFERENCE
#include "MultiNetwork.h"
-#include "NeuralNetwork.h"
#include "RecurrentGradientMachine.h"
-#include "hl_gpu.h"
-#include "paddle/gserver/layers/AgentLayer.h"
-#include "paddle/utils/Stat.h"
+#endif
namespace paddle {
void parameterInitNN(int paramId,
@@ -54,6 +56,7 @@ void parameterInitNN(int paramId,
}
NeuralNetwork* NeuralNetwork::create(const ModelConfig& config) {
+#ifndef PADDLE_MOBILE_INFERENCE
if (config.type() == "recurrent_nn") {
return newNeuralNetwork("root");
} else if (config.type() == "multi_nn") {
@@ -61,6 +64,9 @@ NeuralNetwork* NeuralNetwork::create(const ModelConfig& config) {
} else {
return newNeuralNetwork();
}
+#else
+ return new NeuralNetwork();
+#endif
}
std::map<std::string, bool> NeuralNetwork::dllInitMap;
@@ -304,6 +310,8 @@ void NeuralNetwork::onPassEnd() {
}
}
+#ifndef PADDLE_MOBILE_INFERENCE
+
class CombinedEvaluator : public Evaluator {
public:
void addEvaluator(std::unique_ptr<Evaluator>&& evaluator) {
@@ -466,6 +474,8 @@ Evaluator* NeuralNetwork::makeEvaluator() const {
void NeuralNetwork::eval(Evaluator* evaluator) const { evaluator->eval(*this); }
+#endif
+
void NeuralNetwork::setOutputGrad(const std::vector<Argument>& args) {
CHECK_GE(outputLayers_.size(), args.size());
for (size_t i = 0; i < args.size(); ++i) {
diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.h b/paddle/gserver/gradientmachines/NeuralNetwork.h
index 12810f6425..56a1ec7846 100644
--- a/paddle/gserver/gradientmachines/NeuralNetwork.h
+++ b/paddle/gserver/gradientmachines/NeuralNetwork.h
@@ -97,9 +97,12 @@ public:
virtual void onPassEnd();
+#ifndef PADDLE_MOBILE_INFERENCE
virtual Evaluator* makeEvaluator() const;
virtual void eval(Evaluator* evaluator) const;
+#endif
+
virtual void resetState();
virtual void setOutputGrad(const std::vector<Argument>& args);
diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp
index e95f42c863..075e8166ef 100644
--- a/paddle/gserver/layers/Layer.cpp
+++ b/paddle/gserver/layers/Layer.cpp
@@ -103,10 +103,12 @@ LayerPtr Layer::create(const LayerConfig& config) {
return LayerPtr(new MultiClassCrossEntropy(config));
else if (type == "rank-cost")
return LayerPtr(new RankingCost(config));
+#ifndef PADDLE_MOBILE_INFERENCE
else if (type == "auc-validation")
return LayerPtr(new AucValidation(config));
else if (type == "pnpair-validation")
return LayerPtr(new PnpairValidation(config));
+#endif
return LayerPtr(registrar_.createByType(config.type(), config));
}
From bb07120b64528ba37de75c01ec2d1d71a2e9cb03 Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Fri, 29 Sep 2017 20:16:04 +0800
Subject: [PATCH 06/82] Remove dataproviders.
---
paddle/gserver/CMakeLists.txt | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/paddle/gserver/CMakeLists.txt b/paddle/gserver/CMakeLists.txt
index cd469875df..5f39167afc 100644
--- a/paddle/gserver/CMakeLists.txt
+++ b/paddle/gserver/CMakeLists.txt
@@ -69,6 +69,14 @@ if(MOBILE_INFERENCE)
evaluators/CTCErrorEvaluator.cpp
evaluators/ChunkEvaluator.cpp)
+ # Remove dataproviders
+ list(REMOVE_ITEM GSERVER_SOURCES
+ dataproviders/DataProvider.cpp
+ dataproviders/MultiDataProvider.cpp
+ dataproviders/ProtoDataProvider.cpp
+ dataproviders/PyDataProvider2.cpp
+ dataproviders/PyDataProvider.cpp)
+
# Remove useless gradientmachines
list(REMOVE_ITEM GSERVER_SOURCES
gradientmachines/MultiNetwork.cpp
From 33299ef972302c310cc2b117f4cb58377daa6bd1 Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Fri, 29 Sep 2017 20:39:36 +0800
Subject: [PATCH 07/82] Remove cuda.
---
cmake/util.cmake | 1 -
1 file changed, 1 deletion(-)
diff --git a/cmake/util.cmake b/cmake/util.cmake
index 5ebfc0945f..45a8d66120 100644
--- a/cmake/util.cmake
+++ b/cmake/util.cmake
@@ -83,7 +83,6 @@ function(link_paddle_exe TARGET_NAME)
paddle_utils
paddle_parameter
paddle_proto
- paddle_cuda
paddle_optimizer
${EXTERNAL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
From ea4672bea0bdef1e73f18da8802cd8a467739299 Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Fri, 29 Sep 2017 20:47:51 +0800
Subject: [PATCH 08/82] Remove optimizer.
---
CMakeLists.txt | 8 +++++---
cmake/util.cmake | 1 -
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ec4e6e2e86..70b35154aa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -168,9 +168,11 @@ endif(USE_NNPACK)
add_subdirectory(proto)
-# "add_subdirectory(go)" should be placed after the following loine,
-# because it depends on paddle/optimizer.
-add_subdirectory(paddle/optimizer)
+if(NOT MOBILE_INFERENCE)
+ # "add_subdirectory(go)" should be placed after the following loine,
+ # because it depends on paddle/optimizer.
+ add_subdirectory(paddle/optimizer)
+endif()
# "add_subdirectory(paddle)" and "add_subdirectory(python)" should be
# placed after this block, because they depends on it.
diff --git a/cmake/util.cmake b/cmake/util.cmake
index 45a8d66120..2ab1e8c8e4 100644
--- a/cmake/util.cmake
+++ b/cmake/util.cmake
@@ -83,7 +83,6 @@ function(link_paddle_exe TARGET_NAME)
paddle_utils
paddle_parameter
paddle_proto
- paddle_optimizer
${EXTERNAL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
From d94dd75e3b932fb7e792d6d1b56701ac2d76bd16 Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Fri, 29 Sep 2017 21:19:26 +0800
Subject: [PATCH 09/82] Remove string, scripts and retain cuda.
---
cmake/util.cmake | 1 +
paddle/CMakeLists.txt | 6 +++---
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/cmake/util.cmake b/cmake/util.cmake
index 2ab1e8c8e4..117ab7f49c 100644
--- a/cmake/util.cmake
+++ b/cmake/util.cmake
@@ -83,6 +83,7 @@ function(link_paddle_exe TARGET_NAME)
paddle_utils
paddle_parameter
paddle_proto
+ paddle_cuda
${EXTERNAL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt
index 3eb494ae47..7d2becbdd7 100644
--- a/paddle/CMakeLists.txt
+++ b/paddle/CMakeLists.txt
@@ -1,18 +1,18 @@
add_subdirectory(cuda)
add_subdirectory(function)
add_subdirectory(utils)
-add_subdirectory(testing)
add_subdirectory(math)
add_subdirectory(gserver)
-add_subdirectory(scripts)
-add_subdirectory(string)
add_subdirectory(parameter)
+add_subdirectory(testing)
if(MOBILE_INFERENCE)
add_subdirectory(capi)
else()
add_subdirectory(pserver)
add_subdirectory(trainer)
+ add_subdirectory(string)
+ add_subdirectory(scripts)
if(WITH_C_API)
add_subdirectory(capi)
From bee95fc8917e09f61ba46586a94d2b9003cddf13 Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Fri, 29 Sep 2017 11:45:04 +0800
Subject: [PATCH 10/82] fix code format and some bug
---
paddle/operators/math/pooling.cc | 20 +--
paddle/operators/math/pooling.cu | 147 ++++++++++--------
paddle/operators/math/pooling.h | 1 -
paddle/operators/pool_with_index_op.cc | 71 +++++----
paddle/operators/pool_with_index_op.h | 10 +-
.../v2/framework/tests/test_pool_max_op.py | 52 +++++--
6 files changed, 180 insertions(+), 121 deletions(-)
diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc
index 0e4d9007a6..da0e8ff3d2 100644
--- a/paddle/operators/math/pooling.cc
+++ b/paddle/operators/math/pooling.cc
@@ -26,7 +26,6 @@ class MaxPool2dWithIndexFunctor<platform::CPUPlace, T> {
framework::Tensor& mask, std::vector<int>& ksize,
std::vector<int>& strides, std::vector<int>& paddings) {
const int batch_size = input.dims()[0];
-
const int input_height = input.dims()[2];
const int input_width = input.dims()[3];
const int output_channels = output.dims()[1];
@@ -112,11 +111,11 @@ class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, T> {
input_grad_data[input_idx] += output_grad_data[output_idx];
}
}
+ // offset
+ input_grad_data += input_stride;
+ output_grad_data += output_stride;
+ mask_data += output_stride;
}
- // offset
- input_grad_data += input_stride;
- output_grad_data += output_stride;
- mask_data += output_stride;
}
}
};
@@ -152,6 +151,7 @@ class MaxPool3dWithIndexFunctor<platform::CPUPlace, T> {
const int padding_width = paddings[2];
const int input_stride = input_depth * input_height * input_width;
const int output_stride = output_depth * output_height * output_width;
+
const T* input_data = input.data<T>();
T* output_data = output.mutable_data<T>(context.GetPlace());
T* mask_data = mask.mutable_data<T>(context.GetPlace());
@@ -170,17 +170,17 @@ class MaxPool3dWithIndexFunctor<platform::CPUPlace, T> {
int wstart = pw * stride_width - padding_width;
int wend = std::min(wstart + ksize_width, input_width);
wstart = std::max(wstart, 0);
+
int output_idx = (pd * output_height + ph) * output_width + pw;
T ele = static_cast<T>(-FLT_MAX);
int index = -1;
for (int d = dstart; d < dend; ++d) {
for (int h = hstart; h < hend; ++h) {
for (int w = wstart; w < wend; ++w) {
- if (ele <
- input_data[(d * input_height + h) * input_width + w]) {
- index = (d * input_height + h) * input_width + w;
- ele =
- input_data[(d * input_height + h) * input_width + w];
+ int input_idx = (d * input_height + h) * input_width + w;
+ if (ele < input_data[input_idx]) {
+ index = input_idx;
+ ele = input_data[input_idx];
}
}
}
diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu
index f32e6a26d0..5321ed2163 100644
--- a/paddle/operators/math/pooling.cu
+++ b/paddle/operators/math/pooling.cu
@@ -20,14 +20,14 @@ namespace operators {
namespace math {
template <typename T>
-__global__ void KernelMaxPool2dWithIdxForward(
+__global__ void KernelMaxPool2dWithIdx(
const int nthreads, const T* input_data, T* output_data, T* mask_data,
const int channels, const int input_height, const int input_width,
const int output_height, const int output_width, const int ksize_height,
const int ksize_width, const int stride_height, const int stride_width,
const int padding_height, const int padding_width) {
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- if (index < nthreads) {
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ index += blockDim.x * gridDim.x) {
int pw = index % output_width;
int ph = (index / output_width) % output_height;
int c = (index / output_width / output_height) % channels;
@@ -43,51 +43,58 @@ __global__ void KernelMaxPool2dWithIdxForward(
input_data += (batch_idx * channels + c) * input_height * input_width;
T ele = -FLT_MAX;
- int index = -1;
+ int max_index = -1;
for (int h = hstart; h < hend; ++h) {
for (int w = wstart; w < wend; ++w) {
- if (ele < input_data[h * input_width + w]) {
- index = h * input_width + w;
- ele = input_data[h * input_width + w];
+ int input_index = h * input_width + w;
+ if (ele < input_data[input_index]) {
+ max_index = input_index;
+ ele = input_data[input_index];
}
}
}
output_data[index] = ele;
- mask_data[index] = index;
+ mask_data[index] = max_index;
}
}
template <typename T>
-__global__ void KernelMaxPool2DWithIdxBackward(
+__global__ void KernelMaxPool2DWithIdxGrad(
const int nthreads, T* input_grad, const T* output_grad, const T* mask_data,
const int channels, const int input_height, const int input_width,
const int output_height, const int output_width, const int ksize_height,
const int ksize_width, const int stride_height, const int stride_width,
const int padding_height, const int padding_width) {
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- if (index < nthreads) {
- int offsetW = index % input_width + padding_width;
- int offsetH = (index / input_width) % input_height + padding_height;
- int offsetC = (index / input_width / input_height) % channels;
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ index += blockDim.x * gridDim.x) {
+ int w_offset = index % input_width;
+ int h_offset = (index / input_width) % input_height;
+ int c_offset = (index / input_width / input_height) % channels;
int batch_idx = index / input_width / input_height / channels;
- int phstart = (offsetH < ksize_height)
- ? 0
- : (offsetH - ksize_height) / stride_height + 1;
- int pwstart = (offsetW < ksize_width)
- ? 0
- : (offsetW - ksize_width) / stride_width + 1;
- int phend = min(offsetH / stride_height + 1, output_height);
- int pwend = min(offsetW / stride_width + 1, output_width);
+ int ph_start =
+ (h_offset + padding_height < ksize_height)
+ ? 0
+ : (h_offset + padding_height - ksize_height) / stride_height + 1;
+ int pw_start =
+ (w_offset + padding_width < ksize_width)
+ ? 0
+ : (w_offset + padding_width - ksize_width) / stride_width + 1;
+ int ph_end =
+ min((h_offset + padding_height) / stride_height + 1, output_height);
+ int pw_end =
+ min((w_offset + padding_width) / stride_width + 1, output_width);
+
T gradient = 0;
+ int input_current_featuremap_idx = h_offset * input_width + w_offset;
int output_idx =
- (batch_idx * channels + offsetC) * output_height * output_width;
+ (batch_idx * channels + c_offset) * output_height * output_width;
+
mask_data += output_idx;
output_grad += output_idx;
- for (int ph = phstart; ph < phend; ++ph) {
- for (int pw = pwstart; pw < pwend; ++pw) {
- if ((offsetH * input_width + offsetW) ==
- mask_data[ph * output_width + pw])
+ for (int ph = ph_start; ph < ph_end; ++ph) {
+ for (int pw = pw_start; pw < pw_end; ++pw) {
+ if (mask_data[ph * output_width + pw] == input_current_featuremap_idx)
gradient += output_grad[ph * output_width + pw];
}
}
@@ -125,7 +132,7 @@ class MaxPool2dWithIndexFunctor<platform::GPUPlace, T> {
dim3 threads(1024, 1);
dim3 grid(blocks, 1);
- KernelMaxPool2dWithIdxForward<
+ KernelMaxPool2dWithIdx<
T><<<grid, threads, 0,
reinterpret_cast<const platform::CUDADeviceContext&>(context)
.stream()>>>(nthreads, input_data, output_data, mask_data,
@@ -167,7 +174,7 @@ class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, T> {
dim3 threads(1024, 1);
dim3 grid(blocks, 1);
- KernelMaxPool2DWithIdxBackward<
+ KernelMaxPool2DWithIdxGrad<
T><<<grid, threads, 0,
reinterpret_cast<const platform::CUDADeviceContext&>(context)
.stream()>>>(nthreads, input_grad_data, output_grad_data,
@@ -184,7 +191,7 @@ template class MaxPool2dWithIndexFunctor<platform::GPUPlace, double>;
template class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, double>;
template <typename T>
-__global__ void KernelMaxPool3DWithIdxForward(
+__global__ void KernelMaxPool3DWithIdx(
const int nthreads, const T* input_data, T* output_data, T* mask_data,
const int channels, const int input_depth, const int input_height,
const int input_width, const int output_depth, const int output_height,
@@ -200,6 +207,7 @@ __global__ void KernelMaxPool3DWithIdxForward(
int c = (index / output_width / output_height / output_depth) % channels;
int batch_idx =
index / output_width / output_height / output_depth / channels;
+
int dstart = pd * stride_depth - padding_depth;
int hstart = ph * stride_height - padding_height;
int wstart = pw * stride_width - padding_width;
@@ -209,8 +217,9 @@ __global__ void KernelMaxPool3DWithIdxForward(
dstart = max(dstart, 0);
hstart = max(hstart, 0);
wstart = max(wstart, 0);
+
T ele = -FLT_MAX;
- int index = -1;
+ int max_index = -1;
input_data +=
(batch_idx * channels + c) * input_depth * input_height * input_width;
@@ -218,19 +227,19 @@ __global__ void KernelMaxPool3DWithIdxForward(
for (int h = hstart; h < hend; ++h) {
for (int w = wstart; w < wend; ++w) {
if (ele < input_data[(d * input_height + h) * input_width + w]) {
- index = (d * input_height + h) * input_width + w;
- ele = input_data[(d * input_height + h) * input_width + w];
+ max_index = (d * input_height + h) * input_width + w;
+ ele = input_data[max_index];
}
}
}
}
output_data[index] = ele;
- mask_data[index] = index;
+ mask_data[index] = max_index;
}
}
template <typename T>
-__global__ void KernelMaxPool3DWithIdxBackward(
+__global__ void KernelMaxPool3DWithIdxGrad(
const int nthreads, T* input_grad, const T* output_grad, const T* mask,
const int channels, const int input_depth, const int input_height,
const int input_width, const int output_depth, const int output_height,
@@ -240,37 +249,45 @@ __global__ void KernelMaxPool3DWithIdxBackward(
const int padding_width) {
for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
index += blockDim.x * gridDim.x) {
- int offsetW = index % input_width + padding_width;
- int offsetH = (index / input_width) % input_height + padding_height;
- int offsetD =
- (index / input_width / input_height) % input_depth + padding_depth;
- int offsetC = (index / input_width / input_height / input_depth) % channels;
+ int w_offset = index % input_width;
+ int h_offset = (index / input_width) % input_height;
+ int d_offset = (index / input_width / input_height) % input_depth;
+ int c_offset =
+ (index / input_width / input_height / input_depth) % channels;
int batch_idx = index / input_width / input_height / input_depth / channels;
- int pdstart = (offsetD < ksize_depth)
- ? 0
- : (offsetD - ksize_depth) / stride_depth + 1;
- int phstart = (offsetH < ksize_height)
- ? 0
- : (offsetH - ksize_height) / stride_height + 1;
- int pwstart = (offsetW < ksize_width)
- ? 0
- : (offsetW - ksize_width) / stride_width + 1;
- int pdend = min((offsetD) / stride_depth + 1, output_depth);
- int phend = min((offsetH) / stride_height + 1, output_height);
- int pwend = min((offsetW) / stride_width + 1, output_width);
+ int pd_start =
+ (d_offset + padding_depth < ksize_depth)
+ ? 0
+ : (d_offset + padding_depth - ksize_depth) / stride_depth + 1;
+ int ph_start =
+ (h_offset + padding_height < ksize_height)
+ ? 0
+ : (h_offset + padding_height - ksize_height) / stride_height + 1;
+ int pw_start =
+ (w_offset + padding_width < ksize_width)
+ ? 0
+ : (w_offset + padding_width - ksize_width) / stride_width + 1;
+ int pd_end =
+ min((d_offset + padding_depth) / stride_depth + 1, output_depth);
+ int ph_end =
+ min((h_offset + padding_height) / stride_height + 1, output_height);
+ int pw_end =
+ min((w_offset + padding_width) / stride_width + 1, output_width);
T gradient = 0;
- int output_idx = (batch_idx * channels + offsetC) * output_depth *
+ int input_current_feature_map_idx =
+ (d_offset * input_height + h_offset) * input_width + w_offset;
+ int output_idx = (batch_idx * channels + c_offset) * output_depth *
output_height * output_width;
mask += output_idx;
output_grad += output_idx;
- for (int pd = pdstart; pd < pdend; ++pd) {
- for (int ph = phstart; ph < phend; ++ph) {
- for (int pw = pwstart; pw < pwend; ++pw) {
- if (((offsetD * input_height + offsetH) * input_width + offsetW) ==
- mask[(pd * output_height + ph) * output_width + pw])
+ for (int pd = pd_start; pd < pd_end; ++pd) {
+ for (int ph = ph_start; ph < ph_end; ++ph) {
+ for (int pw = pw_start; pw < pw_end; ++pw) {
+ if (mask[(pd * output_height + ph) * output_width + pw] ==
+ input_current_feature_map_idx)
gradient +=
output_grad[(pd * output_height + ph) * output_width + pw];
}
@@ -308,7 +325,7 @@ class MaxPool3dWithIndexFunctor<platform::GPUPlace, T> {
const T* input_data = input.data<T>();
T* output_data = output.mutable_data<T>(context.GetPlace());
- T* mask_data = output.mutable_data<T>(context.GetPlace());
+ T* mask_data = mask.mutable_data<T>(context.GetPlace());
int nthreads = batch_size * output_channels * output_depth * output_height *
output_width;
@@ -316,7 +333,7 @@ class MaxPool3dWithIndexFunctor<platform::GPUPlace, T> {
dim3 threads(1024, 1);
dim3 grid(blocks, 1);
- KernelMaxPool3DWithIdxForward<
+ KernelMaxPool3DWithIdx<
T><<<grid, threads, 0,
reinterpret_cast<const platform::CUDADeviceContext&>(context)
.stream()>>>(
@@ -341,10 +358,10 @@ class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, T> {
const int input_depth = input_grad.dims()[2];
const int input_height = input_grad.dims()[3];
const int input_width = input_grad.dims()[4];
- const int output_channels = input_grad.dims()[1];
- const int output_depth = input_grad.dims()[2];
- const int output_height = input_grad.dims()[3];
- const int output_width = input_grad.dims()[4];
+ const int output_channels = output_grad.dims()[1];
+ const int output_depth = output_grad.dims()[2];
+ const int output_height = output_grad.dims()[3];
+ const int output_width = output_grad.dims()[4];
const int ksize_depth = ksize[0];
const int ksize_height = ksize[1];
const int ksize_width = ksize[2];
@@ -365,7 +382,7 @@ class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, T> {
dim3 threads(1024, 1);
dim3 grid(blocks, 1);
- KernelMaxPool3DWithIdxBackward<
+ KernelMaxPool3DWithIdxGrad<
T><<<grid, threads, 0,
reinterpret_cast<const platform::CUDADeviceContext&>(context)
.stream()>>>(
diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h
index 3a05cd98fe..308a9341b6 100644
--- a/paddle/operators/math/pooling.h
+++ b/paddle/operators/math/pooling.h
@@ -23,7 +23,6 @@ namespace operators {
namespace math {
//////////////////////
#define FLT_MAX __FLT_MAX__
-/////////////////////
template <typename Place, typename T>
class MaxPool2dWithIndexFunctor {
diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc
index d7a07a403d..c51145b923 100644
--- a/paddle/operators/pool_with_index_op.cc
+++ b/paddle/operators/pool_with_index_op.cc
@@ -76,8 +76,8 @@ class MaxPoolWithIndexOpGrad : public framework::OperatorWithKernel {
protected:
void InferShape(framework::InferShapeContextBase *ctx) const override {
- PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("X")),
- "X(Input) of MaxPoolWithIndexOpGrad should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("X"),
+ "X(Input) of Pooling should not be null.");
PADDLE_ENFORCE(
ctx->HasOutput(framework::GradVarName("X")),
"X@GRAD(Input@GRAD) of MaxPoolWithIndexOpGrad should not be null.");
@@ -97,28 +97,37 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
"number of channels, H and W is the height and width of image.");
AddOutput("Out",
"The output tensor of pooling operator."
- "The format of output tensor is also NCHW.");
+ "The format of output tensor is also NCHW."
+ "Where N is batch size, C is "
+ "the number of channels, H and W is the height and "
+ "width of image.");
AddOutput("Mask",
"The Mask tensor of pooling operator."
- "The format of output tensor is also NCHW.");
+ "The format of output tensor is also NCHW."
+ "Where N is batch size, C is the number of channels, H and W "
+ "is the height and width of image."
+ "The value in it is the index in current feature map");
AddAttr<std::vector<int>>(
- "ksize", "pooling size(height, width) of pooling operator.");
+ "ksize",
+ "Pooling size(height, width) of pooling operator."
+ "If globalPooling = true, ksize is ignored and need not be "
+ "specified."); // TODO(Add checker)
AddAttr<bool>(
"globalPooling",
- "whether to use the globalPooling."
- "int constant equal to false or true"
- "default false"
+ "Whether to use the globalPooling."
+ "Bool constant equal to false or true."
+ "Default false."
"If globalPooling = true, ksize is ignored and need not be specified.")
.SetDefault(false);
AddAttr<std::vector<int>>("strides",
- "strides(height, width) of pooling operator."
- "default {1,1}")
- .SetDefault({1, 1});
+ "Strides(height, width) of pooling operator."
+ "Default {1,1}.")
+ .SetDefault({1, 1}); // TODO(Add checker)
AddAttr<std::vector<int>>("paddings",
- "paddings(height, width) of pooling operator."
- "default {0,0}")
- .SetDefault({0, 0});
+ "Paddings(height, width) of pooling operator."
+ "Default {0,0}.")
+ .SetDefault({0, 0}); // TODO(Add checker)
AddComment(R"DOC(
The maxPooling2d with index operation calculates the output and the mask based on
@@ -140,30 +149,40 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
"image.");
AddOutput("Out",
"The output tensor of pooling operator."
- "The format of output tensor is also NCDHW.");
+ "The format of output tensor is also NCDHW."
+ "Where N is batch size, C is "
+ "the number of channels, D, H and W is the depth, height and "
+ "width of image.");
AddOutput("Mask",
"The Mask tensor of pooling operator."
- "The format of output tensor is also NCDHW.");
+ "The format of output tensor is also NCDHW."
+ "Where N is batch size, C is the number of channels, D, H and W "
+ "is the depth, height and width of image."
+ "The value in it is the index in current feature map");
AddAttr<std::vector<int>>(
- "ksize", "pooling size(depth, height, width) of pooling operator.");
+ "ksize",
+ "Pooling size(depth, height, width) of pooling operator."
+ "If globalPooling = true, ksize is ignored and need not be "
+ "specified."); // TODO(Add checker)
AddAttr<bool>(
"globalPooling",
- "whether to use the globalPooling."
- "int constant equal to false or true"
- "default false"
+ "Whether to use the globalPooling."
+ "Bool constant equal to false or true."
+ "Default false."
"If globalPooling = true, ksize is ignored and need not be specified.")
.SetDefault(false);
AddAttr<std::vector<int>>(
"strides",
- "strides(depth, height, width) of pooling operator."
- "default {1,1,1}")
- .SetDefault({1, 1, 1});
+ "Strides(depth, height, width) of pooling operator."
+ "Default {1,1,1}.")
+ .SetDefault({1, 1, 1}); // TODO(Add checker)
AddAttr<std::vector<int>>(
"paddings",
- "paddings(depth, height, width) of pooling operator."
- "default {0,0,0}")
- .SetDefault({0, 0, 0});
+ "Paddings(depth, height, width) of pooling operator."
+ "Default {0,0,0}.")
+ .SetDefault({0, 0, 0}); // TODO(Add checker)
+
AddComment(R"DOC(
The maxpooling3d with index operation calculates the output and the mask based on
the input and ksize, strides, paddings parameters.
diff --git a/paddle/operators/pool_with_index_op.h b/paddle/operators/pool_with_index_op.h
index 91abeed016..5fe2f5df93 100644
--- a/paddle/operators/pool_with_index_op.h
+++ b/paddle/operators/pool_with_index_op.h
@@ -32,11 +32,10 @@ class MaxPoolWithIndexKernel : public framework::OpKernel {
Tensor* out = context.Output<Tensor>("Out");
Tensor* mask = context.Output<Tensor>("Mask");
- bool global_pooling = context.Attr<bool>("globalPooling");
std::vector<int> ksize = context.Attr<std::vector<int>>("ksize");
std::vector<int> strides = context.Attr<std::vector<int>>("strides");
std::vector<int> paddings = context.Attr<std::vector<int>>("paddings");
- if (global_pooling) {
+ if (context.Attr<bool>("globalPooling")) {
for (size_t i = 0; i < ksize.size(); ++i) {
ksize[i] = static_cast<int>(in_x->dims()[i + 2]);
}
@@ -63,7 +62,7 @@ template <typename Place, typename T>
class MaxPoolWithIndexGradKernel : public framework::OpKernel {
public:
void Compute(const framework::ExecutionContext& context) const override {
- const Tensor* mask = context.Input<Tensor>("Maks");
+ const Tensor* mask = context.Input<Tensor>("Mask");
const Tensor* out_grad =
context.Input<Tensor>(framework::GradVarName("Out"));
Tensor* in_x_grad = context.Output<Tensor>(framework::GradVarName("X"));
@@ -71,6 +70,11 @@ class MaxPoolWithIndexGradKernel : public framework::OpKernel {
std::vector<int> ksize = context.Attr<std::vector<int>>("ksize");
std::vector<int> strides = context.Attr<std::vector<int>>("strides");
std::vector<int> paddings = context.Attr<std::vector<int>>("paddings");
+ if (context.Attr<bool>("globalPooling")) {
+ for (size_t i = 0; i < ksize.size(); ++i) {
+ ksize[i] = static_cast<int>(in_x_grad->dims()[i + 2]);
+ }
+ }
if (in_x_grad) {
in_x_grad->mutable_data<T>(context.GetPlace());
diff --git a/python/paddle/v2/framework/tests/test_pool_max_op.py b/python/paddle/v2/framework/tests/test_pool_max_op.py
index 2945c8b7a4..ffc345198d 100644
--- a/python/paddle/v2/framework/tests/test_pool_max_op.py
+++ b/python/paddle/v2/framework/tests/test_pool_max_op.py
@@ -3,7 +3,11 @@ import numpy as np
from op_test import OpTest
-def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0):
+def max_pool3D_forward_naive(x,
+ ksize,
+ strides,
+ paddings=[0, 0, 0],
+ global_pool=0):
N, C, D, H, W = x.shape
if global_pool == 1:
@@ -25,8 +29,19 @@ def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0):
x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end]
out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4))
- # mask[:,:, k, i, j] = np.argmax(x_masked, axis=(2, 3, 4))
- return out
+
+ for n in xrange(N):
+ for c in xrange(C):
+ arr = x_masked[n, c, :, :, :]
+ index = np.where(arr == np.max(arr))
+ sub_deep = index[0][0]
+ sub_row = index[1][0]
+ sub_col = index[2][0]
+ index = ((d_start + sub_deep) * H +
+ (h_start + sub_row)) * W + w_start + sub_col
+ mask[n, c, k, i, j] = index
+
+ return out, mask
def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0):
@@ -47,19 +62,25 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0):
x_masked = x[:, :, r_start:r_end, c_start:c_end]
out[:, :, i, j] = np.max(x_masked, axis=(2, 3))
- # mask[:,:, i, j] = np.argmax(x_masked, axis=(2, 3))
- return out
+ for n in xrange(N):
+ for c in xrange(C):
+ arr = x_masked[n, c, :, :]
+ index = np.where(arr == np.max(arr))
+ sub_row = index[0][0]
+ sub_col = index[1][0]
+ index = (r_start + sub_row) * W + c_start + sub_col
+ mask[n, c, i, j] = index
+
+ return out, mask
class TestMaxPoolWithIndex_Op(OpTest):
def setUp(self):
self.initTestCase()
- self.op_type = "maxPool3dWithIndex"
input = np.random.random(self.shape).astype("float32")
- output = self.pool_forward_naive(input, self.ksize, self.strides,
- self.paddings, self.global_pool)
- # mask = np.zeros(output.shape)
+ output, mask = self.pool_forward_naive(input, self.ksize, self.strides,
+ self.paddings, self.global_pool)
self.attrs = {
'strides': self.strides,
@@ -69,7 +90,7 @@ class TestMaxPoolWithIndex_Op(OpTest):
}
self.inputs = {'X': input}
- self.outputs = {'Out': output}
+ self.outputs = {'Out': output, "Mask": mask}
def test_check_output(self):
self.check_output()
@@ -78,7 +99,8 @@ class TestMaxPoolWithIndex_Op(OpTest):
# self.check_grad(set(['X']), ['Out'], max_relative_error=0.07)
def initTestCase(self):
- self.global_pool = 0
+ self.global_pool = False
+ self.op_type = "maxPool3dWithIndex"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 7, 7, 7]
self.ksize = [3, 3, 3]
@@ -86,10 +108,9 @@ class TestMaxPoolWithIndex_Op(OpTest):
self.paddings = [1, 1, 1]
-""""
class TestCase1(TestMaxPoolWithIndex_Op):
def initTestCase(self):
- self.global_pool = 1
+ self.global_pool = True
self.op_type = "maxPool3dWithIndex"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 5, 5, 5]
@@ -100,7 +121,7 @@ class TestCase1(TestMaxPoolWithIndex_Op):
class TestCase2(TestMaxPoolWithIndex_Op):
def initTestCase(self):
- self.global_pool = 0
+ self.global_pool = False
self.op_type = "maxPool2dWithIndex"
self.pool_forward_naive = max_pool2D_forward_naive
self.shape = [2, 3, 7, 7]
@@ -111,7 +132,7 @@ class TestCase2(TestMaxPoolWithIndex_Op):
class TestCase3(TestMaxPoolWithIndex_Op):
def initTestCase(self):
- self.global_pool = 1
+ self.global_pool = True
self.op_type = "maxPool2dWithIndex"
self.pool_forward_naive = max_pool2D_forward_naive
self.shape = [2, 3, 5, 5]
@@ -122,4 +143,3 @@ class TestCase3(TestMaxPoolWithIndex_Op):
if __name__ == '__main__':
unittest.main()
-"""
From 6fc44800ed800e78822c6af5750e202c041d9173 Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Sat, 30 Sep 2017 15:20:44 +0800
Subject: [PATCH 11/82] fix unit test
---
.../v2/framework/tests/test_pool_max_op.py | 72 ++++++++++++++++++-
1 file changed, 69 insertions(+), 3 deletions(-)
diff --git a/python/paddle/v2/framework/tests/test_pool_max_op.py b/python/paddle/v2/framework/tests/test_pool_max_op.py
index ffc345198d..17028c3bf6 100644
--- a/python/paddle/v2/framework/tests/test_pool_max_op.py
+++ b/python/paddle/v2/framework/tests/test_pool_max_op.py
@@ -98,6 +98,28 @@ class TestMaxPoolWithIndex_Op(OpTest):
# def test_check_grad(self):
# self.check_grad(set(['X']), ['Out'], max_relative_error=0.07)
+ def initTestCase(self):
+ self.global_pool = True
+ self.op_type = "maxPool3dWithIndex"
+ self.pool_forward_naive = max_pool3D_forward_naive
+ self.shape = [2, 3, 5, 5, 5]
+ self.ksize = [3, 3, 3]
+ self.strides = [1, 1, 1]
+ self.paddings = [1, 1, 1]
+
+
+class TestCase1(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = True
+ self.op_type = "maxPool3dWithIndex"
+ self.pool_forward_naive = max_pool3D_forward_naive
+ self.shape = [2, 3, 5, 5, 5]
+ self.ksize = [3, 3, 3]
+ self.strides = [1, 1, 1]
+ self.paddings = [1, 1, 1]
+
+
+class TestCase2(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = False
self.op_type = "maxPool3dWithIndex"
@@ -108,7 +130,18 @@ class TestMaxPoolWithIndex_Op(OpTest):
self.paddings = [1, 1, 1]
-class TestCase1(TestMaxPoolWithIndex_Op):
+class TestCase3(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = False
+ self.op_type = "maxPool3dWithIndex"
+ self.pool_forward_naive = max_pool3D_forward_naive
+ self.shape = [2, 3, 7, 7, 7]
+ self.ksize = [3, 3, 3]
+ self.strides = [2, 2, 2]
+ self.paddings = [0, 0, 0]
+
+
+class TestCase4(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
self.op_type = "maxPool3dWithIndex"
@@ -116,10 +149,21 @@ class TestCase1(TestMaxPoolWithIndex_Op):
self.shape = [2, 3, 5, 5, 5]
self.ksize = [3, 3, 3]
self.strides = [1, 1, 1]
+ self.paddings = [1, 1, 1]
+
+
+class TestCase5(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = True
+ self.op_type = "maxPool3dWithIndex"
+ self.pool_forward_naive = max_pool3D_forward_naive
+ self.shape = [2, 3, 5, 5, 5]
+ self.ksize = [3, 3, 3]
+ self.strides = [2, 2, 2]
self.paddings = [0, 0, 0]
-class TestCase2(TestMaxPoolWithIndex_Op):
+class TestCase6(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = False
self.op_type = "maxPool2dWithIndex"
@@ -130,7 +174,18 @@ class TestCase2(TestMaxPoolWithIndex_Op):
self.paddings = [1, 1]
-class TestCase3(TestMaxPoolWithIndex_Op):
+class TestCase7(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = False
+ self.op_type = "maxPool2dWithIndex"
+ self.pool_forward_naive = max_pool2D_forward_naive
+ self.shape = [2, 3, 7, 7]
+ self.ksize = [3, 3]
+ self.strides = [2, 2]
+ self.paddings = [0, 0]
+
+
+class TestCase8(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
self.op_type = "maxPool2dWithIndex"
@@ -138,6 +193,17 @@ class TestCase3(TestMaxPoolWithIndex_Op):
self.shape = [2, 3, 5, 5]
self.ksize = [3, 3]
self.strides = [1, 1]
+ self.paddings = [1, 1]
+
+
+class TestCase9(TestMaxPoolWithIndex_Op):
+ def initTestCase(self):
+ self.global_pool = True
+ self.op_type = "maxPool2dWithIndex"
+ self.pool_forward_naive = max_pool2D_forward_naive
+ self.shape = [2, 3, 5, 5]
+ self.ksize = [3, 3]
+ self.strides = [2, 2]
self.paddings = [0, 0]
From 6627801a586bf93f1d872c643c121e19d5c2f1bf Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Sat, 30 Sep 2017 15:32:29 +0800
Subject: [PATCH 12/82] Follow comments.
---
CMakeLists.txt | 12 ++++++++----
paddle/capi/CMakeLists.txt | 4 +---
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 70b35154aa..7d549b864b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -87,11 +87,15 @@ if(ANDROID OR IOS)
set(WITH_MKLML OFF CACHE STRING
"Disable MKLML package when cross-compiling for Android and iOS" FORCE)
- if(WITH_C_API)
- # Compile PaddlePaddle mobile inference library
- set(MOBILE_INFERENCE ON)
- add_definitions(-DPADDLE_MOBILE_INFERENCE)
+ # Compile PaddlePaddle mobile inference library
+ if (NOT WITH_C_API)
+ set(WITH_C_API ON CACHE STRING
+ "Always compile the C_API when cross-compiling for Android and iOS" FORCE)
endif()
+ set(MOBILE_INFERENCE ON)
+ add_definitions(-DPADDLE_MOBILE_INFERENCE)
+
+ # TODO: Need Open the WITH_TESTING
set(WITH_TESTING OFF CACHE STRING "Disable TESTING when cross-compiling
for Android and iOS" FORCE)
endif()
diff --git a/paddle/capi/CMakeLists.txt b/paddle/capi/CMakeLists.txt
index a19a19d719..2c458a78c5 100644
--- a/paddle/capi/CMakeLists.txt
+++ b/paddle/capi/CMakeLists.txt
@@ -48,9 +48,7 @@ if(NOT IOS)
add_library(paddle_capi_shared SHARED ${CAPI_SOURCES})
set_target_properties(paddle_capi_shared PROPERTIES LINK_FLAGS "${LINK_FLAGS}")
target_include_directories(paddle_capi_shared PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
-
-link_paddle_exe(paddle_capi_shared)
-
+ link_paddle_exe(paddle_capi_shared)
endif()
# install library & headers.
From bb33c2b3a5f2f2aa50e9ceaf1f2120edfe76455b Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Sat, 30 Sep 2017 15:44:02 +0800
Subject: [PATCH 13/82] fix kernel func
---
paddle/operators/math/pooling.cc | 227 ++++++++++++++++++++++++++
paddle/operators/math/pooling.cu | 10 +-
paddle/operators/math/pooling.h | 37 +++++
paddle/operators/pool_with_index_op.h | 4 +-
4 files changed, 270 insertions(+), 8 deletions(-)
diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc
index 3b706529d8..5accde8b07 100644
--- a/paddle/operators/math/pooling.cc
+++ b/paddle/operators/math/pooling.cc
@@ -458,6 +458,233 @@ template class Pool3dGradFunctor<
platform::CPUPlace, paddle::operators::math::MaxPoolGrad<double>, double>;
template class Pool3dGradFunctor<
platform::CPUPlace, paddle::operators::math::AvgPoolGrad<double>, double>;
+
+template <typename T>
+class MaxPool2dWithIndexFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input.dims()[0];
+ const int input_height = input.dims()[2];
+ const int input_width = input.dims()[3];
+ const int output_channels = output.dims()[1];
+ const int output_height = output.dims()[2];
+ const int output_width = output.dims()[3];
+ const int ksize_height = ksize[0];
+ const int ksize_width = ksize[1];
+ const int stride_height = strides[0];
+ const int stride_width = strides[1];
+ const int padding_height = paddings[0];
+ const int padding_width = paddings[1];
+ const int input_stride = input_height * input_width;
+ const int output_stride = output_height * output_width;
+
+ const T* input_data = input.data<T>();
+ T* output_data = output.mutable_data<T>(context.GetPlace());
+ T* mask_data = mask.mutable_data<T>(context.GetPlace());
+
+ for (int i = 0; i < batch_size; i++) {
+ for (int c = 0; c < output_channels; ++c) {
+ for (int ph = 0; ph < output_height; ++ph) {
+ int hstart = ph * stride_height - padding_height;
+ int hend = std::min(hstart + ksize_height, input_height);
+ hstart = std::max(hstart, 0);
+ for (int pw = 0; pw < output_width; ++pw) {
+ int wstart = pw * stride_width - padding_width;
+ int wend = std::min(wstart + ksize_width, input_width);
+ wstart = std::max(wstart, 0);
+
+ T ele = static_cast<T>(-FLT_MAX);
+ int index = -1;
+ for (int h = hstart; h < hend; ++h) {
+ for (int w = wstart; w < wend; ++w) {
+ if (ele < input_data[h * input_width + w]) {
+ ele = input_data[h * input_width + w];
+ index = h * input_width + w;
+ }
+ }
+ }
+ output_data[ph * output_width + pw] = ele;
+ mask_data[ph * output_width + pw] = index;
+ }
+ }
+ // offset
+ input_data += input_stride;
+ output_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template <typename T>
+class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input_grad.dims()[0];
+ const int input_height = input_grad.dims()[2];
+ const int input_width = input_grad.dims()[3];
+ const int output_channels = output_grad.dims()[1];
+ const int output_height = output_grad.dims()[2];
+ const int output_width = output_grad.dims()[3];
+ const int input_stride = input_height * input_width;
+ const int output_stride = output_height * output_width;
+
+ const T* mask_data = mask.data<T>();
+ const T* output_grad_data = output_grad.data<T>();
+ T* input_grad_data = input_grad.mutable_data<T>(context.GetPlace());
+
+ for (int n = 0; n < batch_size; ++n) {
+ for (int c = 0; c < output_channels; ++c) {
+ for (int ph = 0; ph < output_height; ++ph) {
+ for (int pw = 0; pw < output_width; ++pw) {
+ const int output_idx = ph * output_width + pw;
+ const int input_idx = static_cast<int>(mask_data[output_idx]);
+ input_grad_data[input_idx] += output_grad_data[output_idx];
+ }
+ }
+ // offset
+ input_grad_data += input_stride;
+ output_grad_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template class MaxPool2dWithIndexFunctor<platform::CPUPlace, float>;
+template class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, float>;
+template class MaxPool2dWithIndexFunctor<platform::CPUPlace, double>;
+template class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, double>;
+
+template <typename T>
+class MaxPool3dWithIndexFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input.dims()[0];
+ const int input_depth = input.dims()[2];
+ const int input_height = input.dims()[3];
+ const int input_width = input.dims()[4];
+ const int output_channels = output.dims()[1];
+ const int output_depth = output.dims()[2];
+ const int output_height = output.dims()[3];
+ const int output_width = output.dims()[4];
+ const int ksize_depth = ksize[0];
+ const int ksize_height = ksize[1];
+ const int ksize_width = ksize[2];
+ const int stride_depth = strides[0];
+ const int stride_height = strides[1];
+ const int stride_width = strides[2];
+ const int padding_depth = paddings[0];
+ const int padding_height = paddings[1];
+ const int padding_width = paddings[2];
+ const int input_stride = input_depth * input_height * input_width;
+ const int output_stride = output_depth * output_height * output_width;
+
+ const T* input_data = input.data<T>();
+ T* output_data = output.mutable_data<T>(context.GetPlace());
+ T* mask_data = mask.mutable_data<T>(context.GetPlace());
+
+ for (int i = 0; i < batch_size; i++) {
+ for (int c = 0; c < output_channels; ++c) {
+ for (int pd = 0; pd < output_depth; ++pd) {
+ int dstart = pd * stride_depth - padding_depth;
+ int dend = std::min(dstart + ksize_depth, input_depth);
+ dstart = std::max(dstart, 0);
+ for (int ph = 0; ph < output_height; ++ph) {
+ int hstart = ph * stride_height - padding_height;
+ int hend = std::min(hstart + ksize_height, input_height);
+ hstart = std::max(hstart, 0);
+ for (int pw = 0; pw < output_width; ++pw) {
+ int wstart = pw * stride_width - padding_width;
+ int wend = std::min(wstart + ksize_width, input_width);
+ wstart = std::max(wstart, 0);
+
+ int output_idx = (pd * output_height + ph) * output_width + pw;
+ T ele = static_cast<T>(-FLT_MAX);
+ int index = -1;
+ for (int d = dstart; d < dend; ++d) {
+ for (int h = hstart; h < hend; ++h) {
+ for (int w = wstart; w < wend; ++w) {
+ int input_idx = (d * input_height + h) * input_width + w;
+ if (ele < input_data[input_idx]) {
+ index = input_idx;
+ ele = input_data[input_idx];
+ }
+ }
+ }
+ }
+ output_data[output_idx] = ele;
+ mask_data[output_idx] = index;
+ }
+ }
+ }
+ // offset
+ input_data += input_stride;
+ output_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template <typename T>
+class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings) {
+ const int batch_size = input_grad.dims()[0];
+ const int input_depth = input_grad.dims()[2];
+ const int input_height = input_grad.dims()[3];
+ const int input_width = input_grad.dims()[4];
+ const int output_channels = output_grad.dims()[1];
+ const int output_depth = output_grad.dims()[2];
+ const int output_height = output_grad.dims()[3];
+ const int output_width = output_grad.dims()[4];
+ const int input_stride = input_depth * input_height * input_width;
+ const int output_stride = output_depth * output_height * output_width;
+
+ const T* mask_data = mask.data<T>();
+ const T* output_grad_data = output_grad.data<T>();
+ T* input_grad_data = input_grad.mutable_data<T>(context.GetPlace());
+
+ for (int n = 0; n < batch_size; ++n) {
+ for (int c = 0; c < output_channels; ++c) {
+ for (int pd = 0; pd < output_depth; ++pd) {
+ for (int ph = 0; ph < output_height; ++ph) {
+ for (int pw = 0; pw < output_width; ++pw) {
+ const int output_idx =
+ (pd * output_height + ph) * output_width + pw;
+ const int input_idx = static_cast<int>(mask_data[output_idx]);
+ input_grad_data[input_idx] += output_grad_data[output_idx];
+ }
+ }
+ }
+ // offset
+ input_grad_data += input_stride;
+ output_grad_data += output_stride;
+ mask_data += output_stride;
+ }
+ }
+ }
+};
+
+template class MaxPool3dWithIndexFunctor<platform::CPUPlace, float>;
+template class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, float>;
+template class MaxPool3dWithIndexFunctor<platform::CPUPlace, double>;
+template class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, double>;
} // namespace math
} // namespace operators
} // namespace paddle
diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu
index 6aafedf912..06263737a9 100644
--- a/paddle/operators/math/pooling.cu
+++ b/paddle/operators/math/pooling.cu
@@ -637,7 +637,7 @@ __global__ void KernelMaxPool2dWithIdx(
const int output_height, const int output_width, const int ksize_height,
const int ksize_width, const int stride_height, const int stride_width,
const int padding_height, const int padding_width) {
- for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads;
index += blockDim.x * gridDim.x) {
int pw = index % output_width;
int ph = (index / output_width) % output_height;
@@ -676,7 +676,7 @@ __global__ void KernelMaxPool2DWithIdxGrad(
const int output_height, const int output_width, const int ksize_height,
const int ksize_width, const int stride_height, const int stride_width,
const int padding_height, const int padding_width) {
- for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads;
index += blockDim.x * gridDim.x) {
int w_offset = index % input_width;
int h_offset = (index / input_width) % input_height;
@@ -766,7 +766,6 @@ class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, T> {
const int input_channels = input_grad.dims()[1];
const int input_height = input_grad.dims()[2];
const int input_width = input_grad.dims()[3];
- const int output_channels = output_grad.dims()[1];
const int output_height = output_grad.dims()[2];
const int output_width = output_grad.dims()[3];
const int ksize_height = ksize[0];
@@ -810,7 +809,7 @@ __global__ void KernelMaxPool3DWithIdx(
const int ksize_width, const int stride_depth, const int stride_height,
const int stride_width, const int padding_depth, const int padding_height,
const int padding_width) {
- for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads;
index += blockDim.x * gridDim.x) {
int pw = index % output_width;
int ph = (index / output_width) % output_height;
@@ -858,7 +857,7 @@ __global__ void KernelMaxPool3DWithIdxGrad(
const int ksize_width, const int stride_depth, const int stride_height,
const int stride_width, const int padding_depth, const int padding_height,
const int padding_width) {
- for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads);
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads;
index += blockDim.x * gridDim.x) {
int w_offset = index % input_width;
int h_offset = (index / input_width) % input_height;
@@ -969,7 +968,6 @@ class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, T> {
const int input_depth = input_grad.dims()[2];
const int input_height = input_grad.dims()[3];
const int input_width = input_grad.dims()[4];
- const int output_channels = output_grad.dims()[1];
const int output_depth = output_grad.dims()[2];
const int output_height = output_grad.dims()[3];
const int output_width = output_grad.dims()[4];
diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h
index d214c68923..d819e5986e 100644
--- a/paddle/operators/math/pooling.h
+++ b/paddle/operators/math/pooling.h
@@ -117,6 +117,43 @@ class MaxPool3dGradFunctor {
std::vector<int>& strides, std::vector<int>& paddings);
};
+template <typename Place, typename T>
+class MaxPool2dWithIndexFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+template <typename Place, typename T>
+class MaxPool2dWithIndexGradFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+template <typename Place, typename T>
+class MaxPool3dWithIndexFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& input, framework::Tensor& output,
+ framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
+
+template <typename Place, typename T>
+class MaxPool3dWithIndexGradFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& input_grad,
+ const framework::Tensor& output_grad,
+ const framework::Tensor& mask, std::vector<int>& ksize,
+ std::vector<int>& strides, std::vector<int>& paddings);
+};
} // namespace math
} // namespace operators
} // namespace paddle
diff --git a/paddle/operators/pool_with_index_op.h b/paddle/operators/pool_with_index_op.h
index 5fe2f5df93..01b961ca82 100644
--- a/paddle/operators/pool_with_index_op.h
+++ b/paddle/operators/pool_with_index_op.h
@@ -25,7 +25,7 @@ namespace operators {
using Tensor = framework::Tensor;
template <typename Place, typename T>
-class MaxPoolWithIndexKernel : public framework::OpKernel {
+class MaxPoolWithIndexKernel : public framework::OpKernel<T> {
public:
void Compute(const framework::ExecutionContext& context) const override {
const Tensor* in_x = context.Input<Tensor>("X");
@@ -59,7 +59,7 @@ class MaxPoolWithIndexKernel : public framework::OpKernel {
};
template <typename Place, typename T>
-class MaxPoolWithIndexGradKernel : public framework::OpKernel {
+class MaxPoolWithIndexGradKernel : public framework::OpKernel<T> {
public:
void Compute(const framework::ExecutionContext& context) const override {
const Tensor* mask = context.Input<Tensor>("Mask");
From 71dff503ce6934fd78508879545debdbf8776c51 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 3 Oct 2017 15:28:44 -0700
Subject: [PATCH 14/82] API of GAN
---
doc/design/gan_api.md | 134 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 134 insertions(+)
create mode 100644 doc/design/gan_api.md
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
new file mode 100644
index 0000000000..65ca49410a
--- /dev/null
+++ b/doc/design/gan_api.md
@@ -0,0 +1,134 @@
+'''
+GAN implementation, just a demo.
+'''
+# pd for short, should be more concise.
+from paddle.v2 as pd
+import numpy as np
+import logging
+
+X = pd.data(pd.float_vector(784))
+
+# Conditional-GAN should be a class.
+### Class member function: the initializer.
+class DCGAN(object):
+ def __init__(self, y_dim=None):
+
+ # hyper parameters
+ self.y_dim = y_dim # conditional gan or not
+ self.batch_size = 100
+ self.z_dim = z_dim # input noise dimension
+
+ # define parameters of discriminators
+ self.D_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
+ self.D_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
+ self.D_W2 = pd.Varialble(np.random.rand(128, 1))
+ self.D_b2 = pd.Variable(np.zeros(128))
+ self.theta_D = [D_W1, D_b1, D_W2, D_b2]
+
+ # define parameters of generators
+ self.G_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
+ self.G_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
+ self.G_W2 = pd.Varialble(np.random.rand(128, 1))
+ self.G_b2 = pd.Variable(np.zeros(128))
+ self.theta_G = [D_W1, D_b1, D_W2, D_b2]
+
+ self.build_model()
+
+### Class member function: Generator Net
+def generator(self, z, y = None):
+
+ # Generator Net
+ if not self.y_dim:
+ z = pd.concat(1, [z, y])
+
+ G_h0 = pd.fc(z, self.G_w0, self.G_b0)
+ G_h0_bn = pd.batch_norm(G_h0)
+ G_h0_relu = pd.relu(G_h0_bn)
+
+ G_h1 = pd.fc(G_h0_relu, self.G_w1, self.G_b1)
+ G_h1_bn = pd.batch_norm(G_h1)
+ G_h1_relu = pd.relu(G_h1_bn)
+
+ G_h2 = pd.deconv(G_h1_relu, self.G_W2, self.G_b2))
+ G_im = pd.tanh(G_im)
+ return G_im
+
+### Class member function: Discriminator Net
+def discriminator(self, image):
+
+ # Discriminator Net
+ D_h0 = pd.conv2d(image, self.D_w0, self.D_b0)
+ D_h0_bn = pd.batchnorm(h0)
+ D_h0_relu = pd.lrelu(h0_bn)
+
+ D_h1 = pd.conv2d(D_h0_relu, self.D_w1, self.D_b1)
+ D_h1_bn = pd.batchnorm(D_h1)
+ D_h1_relu = pd.lrelu(D_h1_bn)
+
+ D_h2 = pd.fc(D_h1_relu, self.D_w2, self.D_b2)
+ return D_h2
+
+### Class member function: Build the model
+def build_model(self):
+
+ # input data
+ if self.y_dim:
+ self.y = pd.data(pd.float32, [self.batch_size, self.y_dim])
+ self.images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
+ self.faked_images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
+ self.z = pd.data(tf.float32, [None, self.z_size])
+
+ # if conditional GAN
+ if self.y_dim:
+ self.G = self.generator(self.z, self.y)
+ self.D_t = self.discriminator(self.images)
+ # generated fake images
+ self.sampled = self.sampler(self.z, self.y)
+ self.D_f = self.discriminator(self.images)
+ else: # original version of GAN
+ self.G = self.generator(self.z)
+ self.D_t = self.discriminator(self.images)
+ # generate fake images
+ self.sampled = self.sampler(self.z)
+ self.D_f = self.discriminator(self.images)
+
+ self.d_loss_real = pd.reduce_mean(pd.cross_entropy(self.D_t, np.ones(self.batch_size))
+ self.d_loss_fake = pd.reduce_mean(pd.cross_entropy(self.D_f, np.zeros(self.batch_size))
+ self.d_loss = self.d_loss_real + self.d_loss_fake
+
+ self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_f, np.ones(self.batch_szie))
+
+# Main function for the demo:
+if __name__ == "__main__":
+
+ # dcgan
+ dcgan = DCGAN()
+ dcgan.build_model()
+
+ # load mnist data
+ data_X, data_y = self.load_mnist()
+
+ # Two subgraphs required!!!
+ d_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.d_loss)
+ g_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.g_loss)
+
+ # executor
+ sess = pd.executor()
+
+ # training
+ for epoch in xrange(10000):
+ for batch_id in range(N / batch_size):
+ idx = ...
+ # sample a batch
+ batch_im, batch_label = data_X[idx:idx+batch_size], data_y[idx:idx+batch_size]
+ # sample z
+ batch_z = np.random.uniform(-1., 1., [batch_size, z_dim])
+
+ if batch_id % 2 == 0:
+ sess.run(d_optim,
+ feed_dict = {dcgan.images: batch_im,
+ dcgan.y: batch_label,
+ dcgan.z: batch_z})
+ else:
+ sess.run(g_optim,
+ feed_dict = {dcgan.z: batch_z})
From e21dcc5bdaacbd9dbab5be134b71ba8c57eda717 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 3 Oct 2017 15:59:25 -0700
Subject: [PATCH 15/82] gan api
---
doc/design/gan_api.md | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 65ca49410a..b5f37051c6 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -1,15 +1,17 @@
'''
GAN implementation, just a demo.
'''
+```python
# pd for short, should be more concise.
from paddle.v2 as pd
import numpy as np
import logging
X = pd.data(pd.float_vector(784))
-
+```
# Conditional-GAN should be a class.
### Class member function: the initializer.
+```python
class DCGAN(object):
def __init__(self, y_dim=None):
@@ -19,22 +21,26 @@ class DCGAN(object):
self.z_dim = z_dim # input noise dimension
# define parameters of discriminators
+ self.D_W0 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
+ self.D_b0 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.D_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
self.D_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.D_W2 = pd.Varialble(np.random.rand(128, 1))
self.D_b2 = pd.Variable(np.zeros(128))
- self.theta_D = [D_W1, D_b1, D_W2, D_b2]
+ self.theta_D = [self.D_W0, self.D_b0, self.D_W1, self.D_b1, self.D_W2, self.D_b2]
# define parameters of generators
+ self.G_W0 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
+ self.G_b0 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.G_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
self.G_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.G_W2 = pd.Varialble(np.random.rand(128, 1))
self.G_b2 = pd.Variable(np.zeros(128))
- self.theta_G = [D_W1, D_b1, D_W2, D_b2]
-
- self.build_model()
+ self.theta_G = [self.G_W0, self.G_b0, self.G_W1, self.G_b1, self.G_W2, self.G_b2]
+```
### Class member function: Generator Net
+```python
def generator(self, z, y = None):
# Generator Net
@@ -52,8 +58,10 @@ def generator(self, z, y = None):
G_h2 = pd.deconv(G_h1_relu, self.G_W2, self.G_b2))
G_im = pd.tanh(G_im)
return G_im
-
+```
+
### Class member function: Discriminator Net
+```python
def discriminator(self, image):
# Discriminator Net
@@ -67,8 +75,10 @@ def discriminator(self, image):
D_h2 = pd.fc(D_h1_relu, self.D_w2, self.D_b2)
return D_h2
+```
### Class member function: Build the model
+```python
def build_model(self):
# input data
@@ -97,8 +107,10 @@ def build_model(self):
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_f, np.ones(self.batch_szie))
+```
# Main function for the demo:
+```python
if __name__ == "__main__":
# dcgan
@@ -109,7 +121,7 @@ if __name__ == "__main__":
data_X, data_y = self.load_mnist()
# Two subgraphs required!!!
- d_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.d_loss)
+ d_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.d_loss, )
g_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.g_loss)
# executor
@@ -125,10 +137,11 @@ if __name__ == "__main__":
batch_z = np.random.uniform(-1., 1., [batch_size, z_dim])
if batch_id % 2 == 0:
- sess.run(d_optim,
+ sess.eval(d_optim,
feed_dict = {dcgan.images: batch_im,
dcgan.y: batch_label,
dcgan.z: batch_z})
else:
- sess.run(g_optim,
+ sess.eval(g_optim,
feed_dict = {dcgan.z: batch_z})
+```
From 2a7f59e73ef0350c92ec3174ed1aa97efe266f52 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 3 Oct 2017 16:41:35 -0700
Subject: [PATCH 16/82] more gan
---
doc/design/dcgan.png | Bin 0 -> 57995 bytes
doc/design/gan_api.md | 12 ++++++++----
2 files changed, 8 insertions(+), 4 deletions(-)
create mode 100644 doc/design/dcgan.png
diff --git a/doc/design/dcgan.png b/doc/design/dcgan.png
new file mode 100644
index 0000000000000000000000000000000000000000..15e8e290a111ff43900934341365cb4360d87d28
GIT binary patch
literal 57995
zcmdR0^+QzO)3!lEkOpax?rx-eDe01qrMppLK{}*My1QFky1SN;?(Tjs`uYA5@BM`p
z?%sROoH@_TJo6mCD}W@B5%3V6J$r^MB`K=(?Ac4^XV0D+z`p=~BW2f~2mE;HAT0G6
z9v&XHEdK}iC$6KIx}&m<iKC0Yz40?sYa1(LdIv*$V`FOvGaJXlmu&*io)JHj5*7OF
znzFZG?D^Sr4(`w9Ry`%U3z;xq<R7%WPt;0Onr0b4oM9!FR!ufd=gx>Z_2~2Fc8sV$
zB_j>K5i_sw^wy_dLW#evM6pU8>~_O(u{sR#>;JjiR1DR}c?FE(*)#QkDxJR{o_&>l
z^Yq~h9C6I^rw`>QEVxfUpWXhy{l#A7`ADO=4$qs$daj0B2`_<7m$1bB;3QllmZyo$
zq$ag-ggs9Gp3W^DZgsc1Eh{T~I6iIh!7SJLd`ddY@O0C&KSG{m)?jbg2s2xd%<|pL
z5U6>!kOvti#?IJd^mIcxFMff8cD94YVy9L2u+jb!&Ec8wFg#Ihh=uNm%iE`0cvYNN
z%LaEj33(iJwh!8~=3LF%aUl9le=Ytsuu5}4<+Uj#GVX`F<=-|C7yYIblZh@e%BQbt
zX3@e;H{W&^<E6f)l$03$^$Bk8W)R`&;Y?{rPj@(X3z5t;tt{~cSSkpA*}mq7m1g*O
zqjfkd4kF$#Jgw!Iz=6#Rs{{^SvNlZ-8Ev=-H;2!U)-M|AFAs$%UOqkJi=Ng{^3#WB
zC9nT84rA5qJm}`^-{SxBwu05lWtp$iUco;7{mR!=mgK?dQXSSrSxf9OwEu1c6-@3V
zCE;a__BfN!KmX@Idp{;EYSigei(>bMHDCPqbm~}X6&;<D>D^g|?2o^0{uz>yot>T6
zd5<Z1P(?)rc=qY(se_{<IeezgQY!-kgQeQE_J8kb-GkMYRadi-kUS?LkLmqP6FX_a
z?{a`Igv!`@MGBi_$DTPvdz#KgPJTWW8{5S2@bHC1gLXJ73tku;3ZYR|vB{g&-lsdh
z=vjcl#w@ch;oSJrW-U!}(&C;c{x$4{SlRUImEZf<P^J>qerbOCm;X&D<TK5>Ef)$t
zxs(!3Y^9QX|5xURr$?1DIypHBXB>UYWvQAi9yKFJd#k0w&iS94kYR=7)Ye`cc3*o6
zM0A6%-8GR0nx1yyR|0A(sswg0*5K+WG8&GdaSFzXns}%gbRj<S-w-9v+*_oxFzxG^
zd(AzM_tT_{ChfP)Ou8zK4ZB0liz<-1uAY|hSF7u*K&ttLD(C%jO?tkJNJ<QZj_9op
zjI2-9Y-A!iN)<X^s{Y&Ox4x^cW|z7z8diF`&gU9=U_#NEnP@R#!pAEgWp)nmEoJ^Q
z^}!!5!nv&YPZWa9`Z`c`tkeWcEGaAo?R?du*^=_!e-8AOzmcGS?F70zZLxa@re~Eg
za@ZWQGL-2N{?A!{+wm1kFlP8cGitz&E_hv%!NF3}`bNQttfo8X?#{H9!v9UR%@!YJ
ze^I&k^FxJu07%D%S2EJbQ3+w=51Z9oH3xeCf1@g0<{})cX_}Z_k6@>*IY&0oFhJX0
zn=>)^-`?g;A?9)1rg3uSu6K?-Xjk80@w#zL1$A2ogu(q=-jZiRkpwF?-IYxu4{xJG
zL9$)=KdMXy$IRIFRA(gr4QGT0l^ho%ofvpbwX+j2`<NdFv3;9^$<u|{J^w8ERn?EY
z;OEd=G}Cwy@H88~7FJA(1(pmpgXV;Ld`!S!RQ)>T|1<wxDmwWPC~&H@)y7goDL9Hz
zyV22{cER)QKd;in_08nzaEayOCeUtbTy8WT?JeL*`!aj7Ucwyr&&r>6MIwy?{YwpU
zPW~jtx(|Bw_gZi_uhXhlH>$t+Z&GMN(d{}-Aq25L#Trg!HKDnN(MoI>d6S(i|J=~5
z1l|{ZdrBl<0bjT7=78prT&|lu&w|Q&H`w^UIR`JldOwxLsYk)v?EQDeQ!+ES`_+og
z=vkRb{sxDiPEcSyO^MP(qZ{-W77`OM<RmC_ET}+$N1nU9n`8a&UFWv=hciVeYIH;M
zuE1C_w-eSr(+}!YIu_hBQX>8vvDNeNRRWLm_2ACL4^s=AYl0u;x|iv-t9ynx^?d(X
zk7LD}4j4<=1Ly@AH4Vi_xncBybit*xI-0<LH`GG0`d!hs+iQyOx+?1@9J4;Lfv4zN
zbcd}2B3}IWnqVu-#7_)@Pu>8cBa>v|U|*n#8cbk@3K;(9Gu7-|;epPFA3syQD`}}q
zp%aP*mLT-AetJdd)k+BCOz9Jrnjs=NbtbEpsg7C`h6}^q7wI|~4Ns-szSE)3cXT{O
zwUhFXl+wH1eAwr`By9AonrL4T8OTvDnLGHbs%o;i!F_$jal*$H&Rg#>p`xnI>(xh3
z{56vso+O1h#nV$dI9T48FgQG%LaLHl+8S>J^b{i4@C+NA?O{RB{T8pc?&Ty-t%=ws
z?>zPp4<G93Zs*SB9NO@Y`ffRkkg;^GU2Gz*Ji9hDOK!>*`p3p4bPAcuHQbWL0naI2
z2tepuJ_jEz74mhGMh&L1c9%l#ME-73yuoaN^2<M8QG6uBoa#M_4{nYJ?AM!fWPGmO
zb}C;BYL@Nsx>J|!%q5m|BP`u+#78VX9>Iuwj+oQ#_J65hA=;cuM(;$9$o*8nU$oVu
z3z3szJqbA79Ii`$&GK*4hKYX{e>^Q3{T`)#vKXeaHSKW<5mQXFM-t%*U&V-*7{zZ@
z_>^H0i$`L_G6hEV*CsJ3J8=G%o&U$*S~&6U=!PA+JFYV~7u;~y8G;S#wXgi^uwTWY
zwgTya3qDAB4et=BzZsfXP2LqKf3$nq7~mG<W$?(}@K{|33CH2y=xi^uZ<He(U5H6d
zf}uxe0ti77NsHIT0(*gL5!}Q^1|{V9&*|96<WvqgA3d!kD!Qu=1#GdRZ<2nuz&!@<
zU!GjOUF{BYS4{TWx~j3S1zTcE1|uAd$q~m3mZw;1TWInN(d$+Cp8vNe%@}sZWDeUn
z4qnvt-_0&LImQ$F+k>oNF1;u15{DG$+wAsZf=yiDP0RT~=fwTjQi{L^J~gfs*;U=G
zaWTM^>(0$BVluc6#ss1ZUEDwPGQ$e3xPkdJ>!NcR9Jn01^B!5j*`=`HO-@?86z(w?
zo66sPjS)Y|0>h5Z7H&FUlga^rX9`?X>xc35ylI20K9wT%P-7e@lcmmwC%^s>T39)|
z<YiYN?BTfd=69BjaF9ctkg_{kAQ{|)Uc|Ti)DC=qRAJW$yF=vdPu?8Y=d9=NQc3Sr
z#MCNFcXR3WMi=CVPCk@V&d$z$xO^O~kT*Eh`E9_g$9EOj+FIIOzO3E)%X!3z&1ip~
zq%%R0Od?n`bED}d>t8;B*I!)1+w9>{aIi=}nP+%OtUuVgiam&IUG;G6#e2<zTQiv#
zk={dFvJSsKdaTnNOE>?puYqX9;vYF_aliXZH%olnu%}<UeL>xXug0M%H0OVi=$viL
zf?L^HdG8^|V|CdM#p_^!7NqiK_wbmUnPn)<%(Jt-`j-lf5CTzUlrw7On1!>njTb#8
zliSYx-r{{)RMhD2{@#DF>=SRms;KCaQE>m@eEq?JtZZ;DZ_c^$@>=vq8BP1skAI=>
z(`jLTez`iGrP_1CciqQ99Y(?3{n~9)Kb!c`W3snF=Jz}B7VW;T--sYuNh173C^KDF
z)1l&m0{te0BT24si8Hy(SHDHTY9FeB;C6$Em&9v2loI#-A4q@QFs{*@Wq8i_JRE(w
z<<?$wMPzqsUJDeLxiU6(&ey&-Tct((U(`s7(k)Xf_$;0kwJ-b&9re{{g{e)cx~J>?
z+?!;^(F$twH?8HbHU9?IPqZfNg{qHtWi#R5TD_n1>m`(5mXZ38<mfCla(i6C&BtBz
zYQtiN540l_P;pUlM!<jkv(#wS>6-}(N=j-vS!xlV%L7PbcA2IfwRQw~Qvt*u<*_qu
zA%||3%4-nvvu8gW?i-GM9_!B5hEF-W>rbcd4_kchcZ;mrhD*z+GX3-f%6GgFNc>TI
z(aHQLxIM(>uGjCmaj6Mn+XnAFw+l<&KjJcy&(JTGG9xJ~E014!6_n|<4KWQgIBZ^d
zdXvd_t{(r8HYSsQK^T#pKXS0ZNYpzpUtfCrCqw%i>cS#A>;6P30g#=fvYDgB!{}i8
z?MW5So{a>rr_9+kEH>yJnjwf_I}a0GjP}LeUaB$TJ}wp<Re=S~66&YC@B8lN^Q}$=
z{zN4xmQ5uveOLHuSgd0GSgpA)FL+NLC5h)oSTu=;j_&Yqciko5^10x%VOQw!@v-@u
z+n#nv3GOY9NzCy{l&qK6!h$*kl_go0(!~;8OBS6%_Rq1aoVTJG<;!PZRq2=*_)!5P
z7)}ui)-O@l=au2m<`<8vrruuiqL*S!xdiQ>-+&E0V}G|j+$a}@-d%RY4=+%~U3dH3
zZFe)1KJTA|)n#UqMoFG^eXwCW;lqK9vZY2wj$B1OG`SqMI`3;H4;x*4&EEP`x=<Bi
zVq&fB;}YuwAYx9Nt-<ZoN13=#0$x>OvlSnqzh#Cde^smXYA9nIKI`4)DQ6A_)ZyfG
zne0*5Zz++l`E0~Dp2ADThIu-EI*rRY`|>KnbSUwCc=}^^n>Up*G*}}-SzUc=brovG
z_Abyc#5hbawyDWOQSm)YFyX>Cb33whp@CF!qss{ya;Ka+u9q(~Q@P$hY2h@k`~E3C
zs<wtSM&qW-C)Sj$+;UfW1aOgwUZkGvPa(@tB1s>{vTgaad8@P<Zl@a~sda?N<EM;$
z<l!Q>7y}!IFr@X+g7+i~YUPS1gF6jsi|QkUGrl&wp#Po?yqBL#SP`o7AeD6RW3g@S
zcd>RurTM>vz^s$*W=BsjvAJo&YRGq+Us&4?z?vWFF{yecS{WE*W_Tj-B*h|<pRKHN
zCi{r}aY8Vmb}z(Z_*HdubXFzSIq9RxYg|;>Ueo(!Z;u@CbHC8vfH=9t$K;qbDz{<N
z3{I&Osj_=x_2@i>v7eZ{t#&te19P0ZA0RuyW!eq)=bEMBWw5rRkoayN*Q>Lfz0<th
z$4fZ<;pMlWwXg3D5id;XU~Xr9IaREibm!qa8>@5%O`c5~hlhuIdjrmxm+#?U+1c4y
zs*N5*Vqnf?VGm;CYqm|<n%>F;Z#YW(WXv1swwTB%*Tl6q%K#Ceg$_{w=d8a}n=sz+
zmX+Z&odc}<quIWlQLuQ{VJg`6ai+t^>qOr_5QC)ucxlsz+sn<krIV53I7VJD=;%j<
z7XLF8W0X^d#~}lb^hd*Vsk~igN3PAGY?+vLz^PO}2D%$}zypZeBlBMkNl-B?3qh?^
zAMn{<0yE<@qTqHdFC0v_67-u{to5kjaigREO^%OHHA^n;u1_y&RX$2ZXJll^wIcZG
zRKT9TtXX1uxmTGaw*ZIY@v~+iib5g`e>R~F?=KGj5-~D1mP_TC>}34aJ{Y#Ic`%y)
z8SO>#UDt=OL+UE6i<DkeVb!Aby|e|CFynU^%+3P|o4O#Zzdk321XV9l_5HdZlQ!NP
zL#bx{FlfnkLHH2-dlDJCXOL9bX(>=;6H!xnw_mqdS!bRh*Ez(BTUisgzu2oj5aosT
z_T=K^mLiK9JnhT~*Liik+Av&mjrLUZeXZS|s{!NU;sT4)D20#Di|l&*s}NJA^P*CT
z7RWoLsQt6X%i10ap9<u>RG@mI`8a6Cbf8tU5^)Z;Y74(>m9k+<X|O#8tTODPIivOV
z*|Rx{tkIdZ-OxG_G`Q|7Y&atAs>03k<#56@%XmLj;a5VUA1Tro+FW)f3JzN?7f;!H
zzl#61)6Q17>T<C=K{0c^8>n4M_b^oci$S#og@wWUXmc-+pyc7<c8o|p^YyKKg6aGA
z)@1OcjW%feD*{$?YzwKFRiFBygc}XRe@l=#iW*HFUELvPOe_D1UT&I?_cL*W#%>Lk
z)Ak=gsES_b)TYyVKKCjPrkvs<g-><OSB@Tw8Y~Qq&nhZHWYWq}P4)hlay+euTIL6G
zwcv&@;@+(eqxp98ixh6TdML^l#Ly>*+)&ZdYBC$moSU1&YGsg$1{___>s$p<zNK27
zE=wZf-Me?zy&<(o0YzHfX)W^mi@3=t8!Nm0&OoZ|k1pz{bduQ8giWV@bhBF)B3iDe
zO8a;CNov;CrJsm#c%3Z~_2M(B7~)|kgOqAzy8P~EW?#O1N!EslEN>8wm2_{nG#NxI
zX?xz!Fu-N-vN8|vEwtE<W&){U2mms)FNXitb!=t7e+vr=3c|s`sg@2c+iG3~<3I@b
zT+5Le!#4gLjAqDOvff^ZQc^Tr-Bjtk6NZ0JjvJfcX|j~|Q6@h81FtG+RJqQ3SHiy_
zoD?seWAEr_)D#>?7Sn5CVUgy+{Z+Mn6iP;wv76|1D{Z~_r6~>u)Yc$bj-vqF{<wnA
zG`0QDhyM~6Y>cteVu~0rm1=zaZmm|k(sgck*EgLm+*LkhEuAd<@{tvuZv7E@ul^Ot
zN%2`(h8|jz;{bI6awfG<4Nt38uk8*85>g3Dirt1gpzizx)+l=Ul)WTP_cONbZJJ{H
zwQCIkjt&!1#%)~CoH{>iUAU<F1fh@HuD>(xj~Af>>wF0n1WFo4h`u9F>ZFBnQ)HYt
z>Z|yEt5akXz*XU$hUZ#X8Q2$kVS)ShsEz3TLwgG`6_&I06^8m0Cg8u^%#$HB%fASf
z6ozlqhU8{7nFZq=pY-eM>h2TAxqVr1PnoGGIYqor<!hN`fFseOlK}}-nO;Wc$mkJp
zzBAA$diHnF@uOyR4AkuU>caMg6N<gg-1K>C8)0sUSRnWNnhVr^{CR$1kFy7d{zDzO
zca<?EeNw^UAP|3x=`@h*QOG6`V<60y!t*_U;<U`1l}t^ek|m~w2wEU*q~@Hv23Hbd
ztK{4+?Y_O}l<w~CwSMXwgo~8S4&x9)5eRpE!{7Ce!oZA4aW^?EeT|4Eo5H0?jQX_R
z_&BR5&*}s9aZbJHkf8H!Jr*3vC91j0G4xolssnyxJ`9L{Ym{Od0TiWJQ&&ypY3$u~
zhOS>C7OOUg=&7j8P>9d~*Z*uk*Uny4C=<NDb{~y_nfk*5LwaK<wde^};P~DF1ON0g
z>C_t|$JXB7`RTgYV>=OVAS5vwoi4pPu)<C}ft~xzrS+(XNleM_1I29n=;$o%N>Qi?
zR*}y9w>>O~{^Iic#VEzwIL4B)QiG<HxK$W)vq)KM)yC5}0Dp`lcCq`u&-vxC&zRjf
z=LrX-6f39;u)bS%h2p;P`?hcYi|h#p<_#xRr^X2$o85}4|6YIAqy;}>YfLW5;#GZ&
zK)P^jxE<&Nd+4U0obEcHsKmvamfUqu^cteXT4)qEtNgN?&olJ5ahy*J?R46wj&loi
zLfT$%xtMnYF3agLv}_Q_?jCMmuc5iHK4@Rl8Xy5ExG{SPa^=(Cl^8oC0-*ngFr-nN
zUP-bZ!|lU2eH`JnBl$ejySuc&v&CwmR=%}Ol@DF#Z%L{9njA7wRNbrQ1`O6qUPD%O
z^$G3`RxV8F%mpiupiE^W;RNpIsZC#mtgRVf>ukJ8P$LkiV*4JYL`&JOC=$TXDl3(6
zjBp@(qpgmcr38Z2R}Z8bnwW-*JeMw)kcdN0%ip<V@v^G^w$@)FQPP_a+YrBH*|cRd
zXd^L)p3m3DiV=Fnv|L-pE;Rn(I>5v_z(#N4G_<?j?&x3|;ALaWmpFVKf?XE~B$?4o
z^`aCxgbU3T%_F)zRsh!}B`t2>jmv7X<;VlT1IiQ4Qs3>{)G|X!&UHN*y56rFzR}9=
z{`(>2q?<3jLKAe)0Uj{j&0|bjX87thFTV?-1Dlo@a~%-iVU3h*`gY39Nzp0SH#qGr
zTwYBr^`D;3macR7W*bB+)1l3#a67Q!A;Z?c-adu>d9t_idQB9DD?fF;pMT2|ykZfH
zYuoQRmq1>qOPEOZ@p?QjtMzGFxzyM3I<9RU6B@7VWJ%9YxbJ8Eyl+B50T??&|GAC_
zoh4_*K>Gq&w#P6!Z(Z#AW`$0_)hns4+-%%{mDh;ag_9P*+%EsVvYqK8_og!eFBtYR
zr~;2&yklLU`8<UAgY&@kUeRC<B4;r^>ZZsiVh?Ao@LN$)_W%ie?Uu~iOy7NbP55wN
zb@d+><`n51&W`p*a$p5IynTBn!F~X!H2_nOo0y&kwsJl4U;lB~PUM6{Bs3cIN09}Z
zK3tB|@!6Y2JnQ<&<FTW@5ruACc+<S+keDb@imeuJdibz)>-18*rI@iXCGqd2mK)H?
zloo5Xz;YQ`4)CcHae}4E_4Ms_^R#ROWm*wWiyD)dAT3|^^Qu(C!#Qq!RtD)KH4O*5
zT>mn|U{8Kg5%{1#(W#Kthj6aO^iXd)SoYHDzVGuQU)y5@`dscJ%9Ifkj#DF~P1aek
z>Uo#Xvu6jtm?8)bIp~Nt>9p(>41ZjS(rH{97Y7b#+vzXwJuc8qvPfUv8MgU&*sHa1
z3drOlr26{GTPOLP3>Cq(ak>acV<(^2fp8$VVq#>F-hb7WQVu!{v*s0kcZ&#=sbfH^
z{wQo(@AAarHXT1F9$Q#;YH^`Liu$5bg<GtXBF}h#w!yPAcr;6#eJL#gOJ^3_26UA*
zvO-Rbh*(7F$?js%7GBFiqB9v5q3dydiQlcv`R<l&!Mj|%<>6{0>`f|U#r6Xs%I<7;
z`vOC)x+~S+9mSL~p7U6wbJI3=M4Yk51@6sl<F7j<GFR3tF0Vy~+P3B6K!hYXthO%3
zA}=udc}U+3CUV;Jt@R!p99Vgue$0vIg#H}&aOoz`;c@f$hu3Tb$tiI)O>Ty>y<x20
z!!efHyn-X&1Ws;OLK^W9g}H=7HEwU`(;43DDou10ebA)eXe>ksnMK2a31uqGyep)k
z89tcBO^q(KbpX%xvtw>O^#CsPxeka3R`-Z(jZVAfPFQOD%}H>=JzS7b_GUdc_PFb;
z7b;^0gOXxn>gs&7J*O6RE?C#nv$qp|RP*?1R@fV#IvTLtx>yFudju%!xgPvBt^@h$
zc@*#lpbAgzN^lGbD%@}18aBxDbdIwo%hnj@Ojm+%op>^|y=gZKd(BotIy|FhX1lvt
z^P$^1u)^RFB8N9tR#Mi^7lXWQ(li#yrHzzdoq;%4j))U}+ugx}#M9_zI4$G{lLF&H
z^ab>ngY-=X6URZC{VB~Y%t*#S!WExXdR83*(h7d!04kji)hq1Rw-mEPNfl$#bmM(m
zw}OpV1tV0zyG4B#-|O0vxf!Ts^`cIp!!kFI4i?o~=*3H2>Lol*UfY8oj2k$Sc~XB*
zhB3y~!v${b?AU|S5rV&?(_-L2sQFt^e^=JpeD~1*Ip4XSk=M7#e7>n+%I_76BMh(q
z@Ny}EDcfU}*7Gl&>Pn+pq*9`)RU*faip};_zurf-I41ouO6~(}IMN3R>Io%#*?5`6
z?_$ksPOt3l2ImaoYA{T`r=ye^b+nXjL?DYVQy-mRWv1uJip{vkE?FoCAGXQ|zPdZP
ze?(B(39D)?LBYFo*EBK&4KjsJ8VJ=sH)^vfEGj+=3Evm66#b-48J)|F6vM;K>*}W0
z%VVwuj|yeV$;*>Bo9Qh{2KPx5-RO@`PTrD(ZcMHV597*itgrj95RKrU0ET+ov2n6U
z4O~-RV0`-9ZeEaQYIaG`Vf<z;_yVmTRoMG|)Ps&kb&)#OkCFYWDf)}5Yc#R(iUM<*
zQ1x0%jhIrcE<4+mK=I2Twvzr1Je@9_YD-)mX0V9<cW+D9S$%H2H&&31&)rT<2bwhv
z-TPQskPJC+!TtRV_;*=<PQ?6C1BVZpJ&lYED#!Yhi4S*Q{@~`OZSg8i;ZxL84nzIT
zG%yMsuQTjOPv*?0p?SE_uiBwy$0wQoSz4qvi5p7aXp9e}9lw-Ech7-#U%8wv;xF5$
zS#$~#J3uv6j6F2!kVw|Bao49?4HDHPFA2N7VV_lu7`0-R*$Cp3bR`sI;ux$NRUWpt
zbahJ8c(qN-NxL-F9nsObcy3>}-7W_zwuTF5DD<pG;S6q`+V1rxc34!H?R=iD0ujK<
z4+hdwJu4))*R4E)vbQ8!q<7Ye&(53fHq&>K@Qgx?4+tOLD|T~~-fPiSZGqVXG<I9l
z`5vk&vV88=+SE=8HTSCEpV&hAPD>7Ojb@J#J(QysiDd83++>SHC`Lb~rP_q>9a$EW
zKd>Ecr;=k9=I7_`W>_RphRf5u-r1<b9A<(fkZmmVfyPVDId6@_ZKha=uFt&_VZ`*6
zL+jZbWyqK~$i0r{cjA1tz17Xzo@Z14(*nSq8bV1|prn=&i;d;RgOz)V=oEZ*^hkan
zm*rorJJAN%k6(`jq?GbZaU9G){B(Qh(Z{wSDw-4?1*ltj;sNbC8t*?c?+yrE1S}KO
zF0qv-E3LO%L}dr39`A3b8GY`(+p!HQKMDjt^&MPk-uMaSD7CkC(syyx4z!3IeZ%aZ
zV4t*%AGg0F&3e&znZWJg4Afm7X7mn6M?sRZ0_E;va)~VibwE5~C`pHG3?*}xNwq@W
zwye|<&c)7;hAq>y4b2?ygQbu^_;-K^79Wn-agQUhS%92LcD>|xW=atfQ~@9KY4fT>
zO>cLmXqfdi3#^>=n<!0WZ>dRYF%=!#t`nA@5RDoYermY2x|X84?Iu(lf{b%NM$5?6
zE9%)`nm$UFIFIf&NzWFJHP`z~L_N%jcbc*6k3<Cik52%nW>DPKPkC|3=L$h>x*+1{
zSQ_P^?M|ujIsLZEUEjm(^yiNPV`t0Z`_S#HTK2rcRLU?pcOx*1nF<XJO<#XMEe*}x
z-Q7%SPRlQ*0Zy7X(Bqc=IErvw%IMzN()@rcF;No#dmiwVRt%+wt82Lqs0v7oy0b4}
zb)M!c33e>E2{zymV~r-4p(J*2-w$ehkYT(7Chr?r*VA;M`GU)}o%*YdNKH)*Tf0>~
z(o2ds>PEbi3CVA@zN7s(u@ualtb)1D-YEvSYy8(wzM1|AaB`{AsA!}=B&`5u=#`JP
zHnyNyP)r>POCbyA14X!0Fp{Gfyg5xssQ>5R;EW5vXa5MPG-y1!Z{GWD4)MC&S|-rP
zFz520KAvtc!s!t*_t?AXEtQ6F5YA!m3Y&Uft;9F&1R_hZq`{(iydHjA!Ib<`tthDn
z=2*OclUt3fsgVGrm!jh6+4i_xZXUsGA89msAQEn+J`_>|p03pCk6Gq=IA}Vo{RZO>
zUu}J_2%q7#PZxI$ks$Ur8BXSOTM?C)mj_~=GYfWn!r8-Zyy+P(x8{K*QECDPW^%N1
z<F>)<c$58l-#Kd@Edt^^&np+A3j{rKVVnzDf?3s&PP38CG+yw8c>qhp?*%9?DH>%@
zW(w_#?>`u$wD}%P)jfUb<G34}Sdko7E(cTJG;i(g5B*3@JZ(=~(|`FYE?98NDj4UX
zqNzN92a_b`8=Q`S7nS`Qw$7@z?hmiqpw^YH;+*8AJNJDLKe7@CkAdZiJfe@*t^V`^
zCD=!^;yB0lqoJb4t!OFCf&1eOu1Jr1^5gw<8>DT7cNeedFq@sr=fM+DG$<=6%`mL>
zG9i&kvGDV^s5nv*aM}JDNMyAljv?`9!oPeQAqu~*6&J;2V#)j67zgrXBS>Q1m-dcn
zUGQl#qdpD=P)&jMU2OLY(?lewv~_?Al*Iq3oh~&v*HZM_AW^rBk&&@(-6o`S1s4lz
zlrxdI*EHBbyk*gn?WEJ7s@JGSJ=K%%W4N?$CSKYrC{wFwFlyt6ekLSkcUR&!##Xod
zwM)MaY`j--8aQ{?FfrRd9Q9VuT!o9uAcOs7-w^&_I@)J=RdbxDcR*M`*H4>Ip2-7e
zh(pTdJ5isne=D!SErM;i)VYkvgaoC7@E~t&5sSi;6zQL)*jP8byCbf!xMOsmS`?>M
z{QP;MK$()E?yXvdweG9qav`FN?(Xi*2*HOhO~Dj?=>Eg|Q<WE({hq(cP=yh(=)Jvu
zUjbz~hPCy3uV_}z(%p+OR5;G2nYgHIYeq_5kBdEkrsKT!Q8lUJ25dk}OUujKBog?p
zRu&5n&poF)B|ATl`z4ggJuH`xvF=JUTs7z9d+6EspiD0OcM+6IRFXF&I_x-*x7jQ}
z&jY<%gvSX>3Rnbcetw(|qx~7Ym_^3y>7S9hayJ6$K=FL^vrV3I%DnJgwLn{x%pVoP
zOOwN1S4f9Lf{O57xF=JK_5QS~Qnx5ZnAzW&P*3pk(>V$vg)QS<Str=WtFW*TpCH}J
z^_43gLV)m-I73Vs2<VigXv}6yCpl7CdYFAiKr}(KeR7L0sG9Yz$KX|%H=PJPx2voF
z#h%yR<>1?wx{i)@ArRf@OatXH`TaR&o5kku=#(72va<dP=H}b8Ri?vf{Isa>zkmOZ
zj*iY&+Gb5{H{e)mv~K_mpqqu+yABlaET-5=-OvNq0WdQXdf!@*(5yzDui>g?#Xv9g
zC5fZSX86{GBevk9cuUJB&`{^Yg}jGnj)M`XhRzzbZ^Obqn#gsgFfQr1MI|tA=6#a*
zma%hlxkQh5wee`sLXOQ?u4cEJF=?SC)33ZMJnpDmCN?C1Pw7dTg(`fjYIH`xo@=hv
zUo3u}xW8L#HLqAOrbTBW7FeHQ(~S9k%RlnG>o_Yo$d$TGKPzCphw;$Jz?`UM{BZHp
zhb~XEAs>_-k%z;U<%l*cLv|g(O5@49P_0;=R=>v(o0cWshp0<I{BAEO8IwO2gYL=P
zl^RThPTAc(qjVv{1~%59=aUnt6Pg1k3mDk%C&Kp;u^I3My_-J~yB;WSpSvgJ+Tja&
zToi-yKCoN-h=~~=zYJzU$|@?VQl}$rn763N2ZKGc49LT!-=#m|)8@gi+rME}1DRe!
zAfx#2go79{fIeq2f!68`vh9V{_O)ze@2{Beozm1g%DM$2ECL2}ZgqVNFTEuF2X<B-
zlTUWcDeQZH%k$JrGy8)Mbz0Y5LC+rsqhjj|*F2AnO~%M~VCG96$Ftrj7c@*vk>2X#
zJZ9w4<h+hsznn4UP`d5Xd#E544;2<-Wy@rp!MU1sp$%@fz18V1;dd*KJU2X=Vds=;
za`y%&d5A9bN~q`kf>&!mB7$eaQiO>@bdkmxEntf>@i{hjH;oUqa6^2fTHYr#^=_%6
zLiai_PR{F8XNgR!`6>cm+^eZ3R3E5|Pftv6HYSGlQ^#(&=j7>R4qfcKYjb2+yTU%)
z7FWm?O>S&#r180p8L@=nFbk8TeWavZ<$Cxb6s(V9m0Gx?s(*aGE1o(<pYJEhkjIrW
zV$GkvyD(1{O)L8bIytU^7U0{D4fI$1&4PKVz!IC-dP?<;>?e1_uUK$hSsJuH)kl1p
z4i~&>T^QE%0H3#J`RS$xk&IU|F7jC%R2xj09*n`rkc4lW8;!w9RwVJe5*vD>Ta<Em
z+cWktPMYaaWD=D+!Z30SgzivWs59o^i*CENEgVP%p=MnpK%g50h$yGTQnxu0I&4o=
zxtKoxhAM1L^oZeE5fq_O>wP`d>zg?j@xXlKXM?Qu5R<o;!`n7YVlMrq7a~~E;Ug=p
zgsr2_YiI)cK15Bdq?9A(>5s}XwK&)6dQfoch9nY@i$rjx6dmNjIJTT!ewZz<6qpAO
zzn1AKb_4DE5Ua{K+%0DXwVUT8Mbn@0?F^v=C5#Z781T5=G9fwGPlwn4VB?)F1i%ew
zD}xSZ;sCNVVKghCc9}N9xI!HP5pg(`*JXc>oiGBMfPer4!>Hduepc)Yb8<-8CdQ^V
z`<A0{;y{z3M8R0%co3u!4rI-Yiwlt0H5L;Vrg}c#M14`mCZ@3T+MKP2OMBKP;zEHY
z^sko#;I@a&8;6f|l@WT%3$DgJc5OjtkKQ~7_YsSUcfCFU)AqG`UHlT;$D6xd+wE+Z
zTkCoh;-dS(_%;cB(R5ED=GH~Y9R}(sn;F71&&uKz{($>T4iRemn7{=Q3Y;$67ciQs
zDxEoFsyB^qJ==MdyNw`bE{QJ+EQ%>|P^bJToPT?yFB7k`_c#St@x0RNnj0=0kHYK9
zL{j1ral8!ki(;r$jq#4pFfLE){jq}KcBcl>AQ@T{snO!aD(`K-@S^J>x~5w6@npRF
zoc>)T$b~h2B5(W)5s)L|fO{JZor1T%bW<*+S*0exGoND7c^a<lS{W+suX5jBk@^D_
zZK#}-+MEQ^M{^#A`v7I$+7k5!`!(9KAkoZjzC`_mfq{YI;^IGl{=6p>=i=p!SyRq=
zS2S5Tov(t4X#fWYshCnzb%#KDQ?$8W2pQHomzUs!S@7sgScRg+Q%iH}gM>mmEjPY*
zkirFOP22PN`d*PT|BmcvSKL{#cfK6j`oZQV^Q7zg^W4?3Pr*FpUB<MP4^|Jq89vGm
zxBXxe2-^VxzLk!TMVNxP*!9ny94{TRF0~7?<(1pXHlNAyBVCD!0LioAA%n5BxhDOc
z=D{ohJuc!4XU@!uR?CGIjW(jzum09q6fLoYOLRH_SpwJ5!Hnc!G=5{vh-~Fa=Mdtx
zy;OzSG*zN^vZSM$g39BbCz}vGK`XqoC%pV4D=;reg2dm=Np^3F_M4xU<*HcNWasA4
z`o;ZP;Lk656eD#qo?UPKaRvwXKMY+wtfM!mfBk;H+VuKhuGL{C#13T-1XUjl$a8);
zStwlA1G%_0vUA(vUm{0{A|<ifJpAajsc)z;YeS<=@$!5RP5Ik77zB02DFUV9;voR$
zMXF`q#vMVBictN8o12^Y`FWuAk4(s?PKD9c)kQ&0&bMyMTs5n$qT+Oaa}EteiO0f_
z)R`AWcr7(bvf$3=QiD}Bn?H{0s3|8GR;rt$SWt{bD?v;y8NFtUK}h&YssIc=J3mL0
z_$U+|z^%sr0%)}1K)5~J$oz-)7I9OP7+lQ1Pf8I3Ipv?{cKk00?Nb!oowin;4i9`!
zQ_z)Dh#W|xSbjED_V$CpKtp8I4CwCEe$U;`O(MQfP!(A?Nf*#|B`#a(`%W0^@_>dr
zKK9t!x>eXu8M9Q}RKK|!@i^=<D4e%(9f1Ws{xt&%(hs+s8`;kx2?NAELbqKMEx5^M
zV-uJQJ=|?J&^_uh=~}|@HXrZ2k<nO-k#4iGZDh?51)7>kvUuvKw;GdTD&h_snM6Yl
zZ~Y?`wHS{sI1+Fsh+{McZxDIohLRiTx9`ngDwC$Qg7CZVNlD)evziRvbZgl()Yn&E
z^D|qK4FXBXC^h@S-&LSwC|T-Fy);M!<<T-Q2mlgHZ*M^SDkCXb9`?g4=|Rpva@KOF
zuaC!JBUC!)<md<<1%(r_^Ld5cuqKXSO%l=@KaiH5j#Wj%d*K>m<8?Pqa{)c#IGPr(
zBAH-Z$TGq!coO_>yPMW>svo`Yg@F}V2MxiKAa$p<crp2>E;jIUw6qOe+toEwJT;Cu
z==ECfHU^WmGs*{-)HKr~yFc)ng!+lZAl13UcNt+^Iq7kn_@U@-IbQxOIGJFib@1k|
zCTITiExNI*Md1&{=GJZ!I7R`f`{fD`nLpl#{(;7jZWiXv@{vz!(!|H_A{<bT1RAW3
zhjI$9tWmU&TLsZD#KsbION1nc)aeTw?K?ygMiga_bRV2`4~e-Uk+VjhaHOJ<Da<s(
zHtPlw8?LVWiA*U)1sr++QiPEOL4RvHkrk#$2Q5eEt#u3Vk6u$QnVWne%5ptxD8W?C
zBOu`Y_;61VL(Rj}2;_Mn`8xERUZcX25_TLUfWtJYYRJva1^&?5+R8`B$jf_cY5kxa
z*Ul8GTT)t7T+BdSSXRigRZZb1L>#3_hvgqRFsre#;=5WW=<TEuYXV(ueFHPlF!xT&
z3ZWH-hcZzzGWusI!@LhCe11+8xcwDM%zOr}tWk02gOyp5`X{@h`6&RD9ngqMWpS&_
z?4_Vw3`FHYB4F-QP@XyZA~uyCYptJ)2op@X&I`B4W4W1pmSi!N6LWlPFssmeYOF92
zUFC8e7m~s$_&74G)0Gg$4Hu_3S>J}S?f<@pJeQHNbVO47vGv<aiG#J>U0cV)i>vZ>
zNG4);xJgzAwyn^YbO8`Ab_ueZ4LtTZf%-`iv+LE!w+ffQ&`E7U9-Fzjv*FEGjrN6o
z0AJ?${bl?qQb>*=&Ib1TPW43GDw7s|W9g<KV}aJ#fY+RiI*vzYCQVo!Orch@UU@&?
zH3ZbTU#v9)iqN#QG<jCbB}<N?&xN#Vr740wt$^f)fnjK2A+>)!Hz$XGGN(ipD7ZNA
z*l1#Pv+PiIelf`o6`;SV(cD;BnPk?cia8cD@B?`mE#Ud1KJ0}czcscG>D5-z)C7F~
z!^z>+fqsqp!=06KypRZapUK`cpi`5!v<wYIM?=fc)3U(jlm2XPG4x&Vnl#GPRx-J*
zteSi&NHT@kWFYB$2xcRbGZuF}s~V?*cY?yWFm30m>Hd(O+||ZMVF-f|+nwrSbRU1x
zqJ!19-bU&VSABXzvUk~zvKFl&^T-a$>BfcF9iZ~Md5c%WVQaj+q41!0_Yhw1X5|uo
z`#2PggP}+Fi;HVme+!Q2@n*g55JhiuivTx${*W<AJjnaHs3D~7e&dum*|uv1Kar<Z
z#@6pcRt?wgC-YR3)%Y0B*>_*ABFi}6?VVdD^BLe^p%VC^CUJXQXxX&R5*pwNDKdN$
z5(?A3Uv6MVGGs}lp`~@Ox0j|VBH(NBTUoU^SY(LrPdC{Ihjd0KCc15p<(!<@`}jN(
z^1EN%-U_ZOuoiVHDQhaZ?W%e?<>cmenK8j<D`}N6gs;B$Li=#Lc<`lI6{2q&J+kYx
z`Hm2^*Qj<F6Us!9deC#jexG}nRZx2F0^~WTxBGQVD=RC+G3?Qjr9mJm!72~9-;t(p
zLB?`#piGD5;W!YvWP$q@m!()qa{8mHkMMJ%fj+nAZFg&YKjLi7eKup^7=5^j>l?~K
zW({y5@5=PX9cJL8?h@l;)+M#!nr{ccx;$Py8}f0wm{PN9BPw4XQp~)}lkosbN_&L;
z)4@gomvjL<fVvvyHtH**C9<RD+lwvZXgis#QoT}mKFSPm??Y+#7eA<nhK7!Y^RCTm
z=71_@ZDYf}VgBdOpF^p<f_BXmP$sMpu@Y4^6_x9o8&dyJal<m*mKQHy5;?4vYnGT)
z%^EPPVJ;L>(_%_yGk8Bx_+>ENqN!2YBkmE~*{;cu%bny}A1?MY!IS8uN^9xajO`A*
z@tZXxhaa=3362HvE$>Sqg$uMcYz}esIuX5Q!8OCE(Of8o=EL4b3BONG@^<g$==hS>
zKJtCmhzaQw0a*lI?tVQU&@`43@WG==)P|sX<^)kqRF9OF;e^9=Id`Ete_3aF+)8jh
zr)GIS0gwfP4_B)uJ9*Z1I1pwWBzO{rcrX(VysxCZJob0dh(||efJhA0EU86?l@%Au
zf<PA+7ZmWM+}zh0!xHfNII3D&9vcIR&CSgzDJf&Y2sP3l9qgjGn~43nJiUz!q*s^1
z!zK-wx6j|Cc4d!~`BM?1kQi|!N(FC(bh1U1P3P~@(o!d3^{x5xUJqN;pbgHy0m2HO
zV3>H8YLR|Z7X?W)t-;u>>Ha(fbrln>&zajQu%rfwgl#`wZ_#U$K!eo=ojPPIz2nid
ztI<%fbt-S?cj;2;DpA*UojTWVTR!4aWdJ!lU%&P=Z<;|J7vI@d@ji72baF<OxdvLh
z=Y6rX#>U1i9+wwqXFqz4Dk>`EJu`ZZc)ahfDh#^|i;4^l4FTqDWqtj81j&>oF*qay
z-~$rV)0c|W7;S89vPRVbViC|CVPVzY8mYRDpG#`eW2F!eP$+vD=j7Hxh^dqMb6ojK
z6+}<3-JSS_C5_;S2!RZf+q_0IOdO!ERsa#7Grg6+0!4_N&+=KgRgk`A3GdGW*8m7J
zyj(i->OQq{(oU_8H~nenn*Q|U#+>xzVm!2MBZ_E1;Nn4MkyVvGjq92|qHM=j>0$?N
z2|4U^bmFtXu_fl~Ys`Sxa!Lu}IV-(u!_3u*Iv9bgA7I-JFcJn9pY(6P0DY4>FoNyv
z@A~SBii$cq3oqquU0i^r7tcBdicJ(ga<uP2ipPZ1KRi5bQR5O3A0Hd#`7K^|c->xw
zM#awV_$A{HN%GRtQuvHyP9PA}9gs?m!C=Xm`8zv1m6erny7>xJKcsVJErEMXa40A#
zDLFZ(M@N~dsHmu^w-2&AyYqp+6c!@D!+(N*|BBi3$`%DBBGCF(T1<??+19A9ukX!`
zJ0~Znjm=?Ck0k7~oP<R8x0i4RIB}_|3kfzXCRr7=&9Y^ai+!z@BNkyisabV(MLxtr
zXr2`blxU}imd@6+@@0ygD;9EqfGF9Lw(_%@T9r=EDHS858Wn~_I6k1XHExRAY+pSF
zP$|uRKPB#)eu4r`2x!;Ec^dSF#0CH+H~^SNm^zrAVlbb%QVd<>@}FtrrOsVszh|Ax
zB>oLzxew0eb)%N;w!YA1RIc1>7#Oz;h&no2s;XL=N{R}ou)_c?wK8p9r{A>X6Sb?l
zhJWBS&fH@fW3AeC-A={Dbgk!QL<$0tZfYOcAN>wZ_g7D?P?l45?}xL$-)lB@m?~;&
zx}R++24~vh0}`ULf&#ZcZ*d?}IZ9chuBRJ=aD>s)Ij@9fSTwxuTiPT<R4{WmfBg7y
z<k}1@1+a*4x?6w#eCd}5#2Y*dUbN+r<h|?m)WI|Wu}4Se`6ocLWQrqrw<n5BvxI|#
z!};h>J0IcUck^O^R?EyZ0Hh_agviOsU%h&zfH9dPgAL0P&#EYYx|ldQ>lz!~S316v
zMg?W&2L%N^fB7azl+>3P&)V(Uli&aeYWBJ2Tt$Ti8=FB-K}FlqdYGZ~lNmGaYx{-}
zpFa!xy>=<Y-1t@PW_A3xk-amHz-+v3FT_Oopzhx<L^2dSFJ?`B?$6_OmxQ6lNmJ|n
z@w>aba7Z}6oJlU*nPkahByzIT7OD`1=)g*~AX!MokXF6zivQcNLx*lV-^U}lOQv9G
z4k2@5@mkmUyo2cu*uoW%@|H$6)nAE<za>}lj~wf+D{Y)(<?^RAn^1dQQV6uqK4p3q
zvS;oEP8#$IIFs+zg)zU>%WOc|d-mIm7To;ie)aScGWQ^BtJ%(^>Pvj)&JUcuNJ~Rm
z_&ILL0njlcT3@ZK=wbC&P1eistG3j^!1+FdimMDpw&X$hj9<Tg1;!b1B(K{y-03TS
zvSbVm4dwH?sqQs$Hz4M9InV}y-iaxCh8Xumk_ZV2vB_!CVfhlvQj7z>4@%`RX2VNO
zNda`(qN4tJd3jdJGDJk_U6-Z#U<(=>%n22h$sZWI>=tPX9oRA*2yh(|ET-{kWuY89
zP7<8OG2Gna^QuRR4Mpt6EPm|@8X5)T+Y=TwMMXtQ1#F435)p(Y`T4E`MU76o8!IQ4
z%Eel;fE3Kt)fLE%XBpP6KYsjJq4V3JZ*Xv-*^L#Z!uJZIPuf6A!+k!Vwg7LS5d$<W
zDd|%569Q!7WN|n_vnjhsB>vT!R#H;EC0cb1dac^JO2s8bt0A2)3fRh(n-xllse^y!
zLzbr3ORdFI55_98GT!POfX*!hZGE>QN6vSSvrKC(!w()wy_VKUsu~xG9hKob<XB;)
zoj74#yVGU;>-GS)G+8B^O9GW>M@I_32Ntxvr>6(bKtoI{K)1}r!2xQgO^gckR05&P
z+Vjn=KY#vN=?nq3gSEA_VRv|^-Td${BOjkOIhtLAwpQ|zLp;FjS3A)-$%QrMYL+R&
zbvSPvT^9zQH1<hX`1zSiQmGMJUX{J=woAz!q`0+0@(7)Z>UHE<0QykUGBOR?4Cz3S
z?CkD_@w7d1IZD3Ag}QZ2_w7=KH#9WNT2T}g7W%fUTLPkDWZu_%!Sr!r6nQCt7n2O|
zVnFUVagn;cc*Th!8qrbA)&mK7+O~HZ`%7()^9>H}ExL4AP4)G!-n_w5Q-Rewyr;z<
zA?vEr$H}#IkU(;{zDg%aMR$!@SZI(Vm1Lo6-JM+R9<sdUZ(MqoG~9Ooby6vdy?}u*
z?fzg0A$GRT!G4a`kD#N(5R<S$J1sSJkVzJ-Q7A%%AucAC3K)_E0~hx@DgXpDbac&b
zrxbGIfXNU7e1|uIjvR1ubodLC0wyMK#`N2_Z~Pt?7$KeTB)-IV_xI_CJGCE2EZf@R
zoGSSQ<;zsZ`3U0~G5U<d({#z@$o#cmzF2tSm$ZQlivdgrWTgFleL&oyjFmP|vaVNR
zf~<xw3V41$%V4ByO<GW57l=qoN&*g`))txNQ3bpA)gWlt)z$3nrRm^6zEGH;|F)f{
z<U=E=cDz1O0}gV^f)gh#N`Uy1pI`5^J9WN09U#06Ae<e(i<8s&>1m84xp2k^hwtWA
zVwF7W)*@Cvj(?>*;pHBx?C6=pXJeMF$;ru{p{`~@ep8~Q52m2>#Y*mM`IQAi_3vus
zB5y~xa}<STX5+R6LG%-z=Y#&637uaQ;~B25VUr#pvq9Iu`X3Pj52guC9SUld?u*r-
zgMm(1r1jmu<OMug6=sdd<ito71K$xa=rjUMxFOj21Q=BRx_gTNFE4D$@ak90d*SIa
zJ)bY|ctWUXMH<9peSLlQ_V$GJ!N9TD<KuRwJs^WY@r02}v5=GdLzjUwaSvYbNlFE0
zCnv)2I`y_0%bzexR6T%53M?N$*Z{#anmj-_L$?fwA)BYy?d|PsaxS}5VVU_U`)vS}
z2wr}91{?qeB=^7tM4a8J5}*hJ>VT0E1^-C(WhxiDb%X$nK$%gtsh^{zD%r5j5gry?
z;h<deg3ohZalcvcw2W6RV4qNdRv2)=s+Q5&c?&phSOunp=%=-l01X)DKeg&zKXdxy
zBM8`c5;gk=OjxC&c~Y$}4;Fxy9@H+Sd$pml(fM%k{y^!Zy?wYrvUt*h85aqkP`-aT
zP0&$+3QGnbpXqM)A)d*?sYWaysgP3sqFT$K;IoRVyW@Igy^sigix=w9?(Id}?cyR^
zOq8;iO<QfoiXeJQ`mq+KVNLPvcIvi|0>(wf=l^K|9C8PhBj+^ITdqerGuc@e7kCfH
ziM*8`ukMjt!Y#V;4%=?C40a#WD3AD0_Jcb42+Qj0xd5YgubS7)0p%(nZ3m9S&VL^Q
zYJ|zjSn}TDV!Eu+{OR4pF7~9{mX;Q7e2|W}wYPU#MQ!Qwl~;}uASR{`=CNDx-%qKn
zFoP#~7r45+D-&%5V6BQu?5w4ALL-n_i;_moS_*jI)dQ!#kx%RRJJ#)AzIXvN&KY$Y
zMWeF=0|P}#?QCroXkz&Yhn>l~?6hfO=e$qP1Q7sgRJ90fVUev=AW90T&oyf-Vtb8N
zPOqhN0B^|#MA&HGv-9)eN!BJFZlW~hD8Mlg<PTHxMIXOOET3KjqVJ8Ref$cUTHuf}
zP;6{+0LocEo=-tGG=EinEJ3~M6)VRT17>WhYHIQE@td2Q*jQL|)*osY6M;>dC`*BX
zkiK1e_UHP%u~4h-Gv;~{4<<w(2SVoW&%*6zXyMf2m)tbAK;V%JOY1E@I%?#FwZ+;q
zC)zXWa@Dwb?fuMnXAQc>9zNCa(@7zRCkYjTC#%@JiZWDJILwfNIB(<c1nZl9dDJDc
z=^e$PH@F1pJ<MzdlnUV(h@*skXj@^V7Y^Iwu4UI|uV)cg#E=_jlVe3%-QU0eb!vxS
zM{jkwvC%cZ)Mf5L57B=m)MsR$5upc_c<W0ng+_9AY5vMZj|$Lu`|Ki6N<|aH!!Z2Y
z+9Gq!gEN654-5ta%OEVA_Q;!IHJJ_Mx`2uyI2a+OR}@t=UBHW!l$0E;7f^8z0K3f7
z8|oukD?9a0h{7<&iUju;LNxip!oqCn3edVn?_IChW(3-y5L93FDSx0l85SfAZ%n5r
zP%1>3Tax&VI0(q#fVc@*8OQ>nB-d6~$MfWgOifMwQRT+<^@lm`eJeD>-uNjNO@i5-
zVljbS%bU4^G*Mei&1H87v#OWe#SWd9YebttY8>d*7G_Hl$E|H@3&Hb)>A)}HgHPV=
zCU_isPHiAF-{0IN9-n?R?8*NwV&-r@G`s8jre)=54+;!4;(+}X$!S=KX>jm~swH=<
zz>%OmLn@a+<^Ed}o%1L|P9A3h*wvxMi*LW$+m{bUsDKWxzDbov)+o@Q$3P%DM@jT{
z3}lDslS>I$jC2~lF^nWP@3-;^T)i^Xr9%7e>E#7<RwSa=vNr5FjA}I5@#Fvul9Q8@
zl9Cb@_En%7=<kQs*`!xhRh5<5czQOg)5S^W)VfPXTOJf}lmKTJr_@WPfKdLV(nmCJ
zbLG>GXV9rf8EUNK1)9*tfZU$oZ;GhiMO^K=J=X5`W^#IZsopLN_@56xe(bFMsApu9
zEu3L%YwJ2S{1(c@lR9L?GCM~(F*(`ovt-GfT#SV%<&X^;dPhizPe?>UuENuKzU$+B
zXdMlG&xV5+qMz`I0{WKp(Sk!*a{J)8I5nY!KuQr1^SNO&rYyF$!H%Y{ZZAt7E(BFY
zZ=F7{+s_|~pV9AN`CLCt$%y%b&-~Le)>4TI6_p!IZhUHE>j$qla=Ki!lVmAcbeC+8
z=|A|j+wM-$qoAP7rVf_urzlWW)Bhh$UmZ{9`~QC!W_HBnnPz8dOb)};<e2Vey1Qdy
zOmlSia17Ja)5G+1*K~J(ulMKo_}zaR<J|XkU)SsPtXB+ZS6W*i9=!pJL%T!|XwJVp
zB>`vYnP>Bed9_Q|42B<YON0ddl53F%Ac7Sa=>ULi$1&cE9f&P-sJ}%WDB0bvX}Z0Q
zEpIX!3cRzi&PZmr*fMk76UMB6j8Rjk7Ds2MO^VfXbbNeFW<oS>IUg<bKu!*W5fBhO
ze{Ny*o+wmIzSTS1#ywZ9q}gRh4LmPUR_W?ylv-iXh@H4EuvgT+>CE!|;sfmnazq!^
zxKFSSHq4L0!F+esZOWYj<)c{mLv~UFBhhbd0#1L5ce9^PwOM`sjT06U`c&OV4{b9|
z+}*a#9{5&R&0FI#mKOiXxbM303QMRW{uq639>D$$wDC!-`uAR+kiyjV4i1dW%v_|A
zwRG&M-_TM0Z32Hk4!->aqazM?Am<^CB#rFO9;XfOax_bE0RA~LpB$y)eH`>E8qD^I
zg0jgSDh&Z*n$FcD>qT};D=Vr^`j?pPwHXC!B{d)YnHd>HFabKhyV#mVjR|x`fo<7p
zB|yal{3GV6|FoDfX==3g*46-<vA?uL9oI`L*JUL;<m>e)CE^XV!(N<LiH>!kIfd=8
zOQH<_NoJs={qO1KM)fkCw~lL4H=!nAm!-({BKdDe`bvcJOqvmBn>aA*FXwOCE&GY1
zJ#+TbuIJ0Eu%{=mJ^v7mWF)|;u0>16uYYAM9d%I$SyK?>viu8WHHLLt)(9>({&;81
zMe4$*3!aFkW=fmE50^a96J}>qdw6?`??)boZ*;V`XPTb8)PD6<ue=<I1m-BfgBvC%
zyi%W-oaBG|797UiJw2epmXwqX%dLJGT5#);R4wq!a=OV#;cqS^!Ib_gOif4>!<73y
zAgpA{RkyC`#`fT>Zg1XYzGY{>VW5Hd$Dy){O4w)G<}bqdqh54lH*@#*4hGOT(WiL!
zEq;ejoc_Ia+W#}hIpmRfZ~~lOVJ0?L8@uw^zP+S@BKSdx4O(G_Tzoj`j0-WKwH#q^
z(0h}%N`ORuGpcAVFH3W|(M@bJ>{K7Z4?}2O6zXdw`kFX=5znieuQp!3$nt3?_`sLF
zk)(fftN8D_@ZoF#wSP(lL8Ixdsv6QAa_Z^Bl(4<ECE_FI%t!HY42i@DY=b_rjn@Em
z6>&Jo7h$FhM8Mv@?6qKn#i)l|-UMPlbBm<@s=>@%tr{_HE#a0sxij!Hhat=ehSBlE
zoQpJ63{B+l=58x#w%UR~2}3;oHa(rZWPE013<XkDTwMQyhxsX_QgC`5T@KIWgt(f5
z@8^!+q`ucI6CD{#V>Pu@rRP@VqXS#-{rU}{^b&L=?<8{1j25H4xi}b_GK`e?<l^!M
zlATj9{y21CM~^EZzFk@AA1J{Acaiy(YRZrLlW5uiw}sO4psAdWklffYrZorhECG+o
zHUrF%_mWY6fy7|#r_6^BD3F`;y^nPAHyPO$o?2&IVflQGe6zDNoY1FOPqB`lqABuv
zZz?c#nZba-?kQw=W@dCgqjy<c{aL6))CA?gVVN_Zu8T_z5*Zm8`RLK3=fR+82G!hD
z30sOR_=6yy10Unk#q|nPLuf#>5AfZv@jUKR62PaU{-ON`0x?sR<{-W6{}k<W_DY{U
zQ!XaS<he)Pi23Po&ii-o-ubo)5n}aLTG!1M0$T~>GoIk>VTaiXi;}9UqA}EnVXBuN
zKa5!>fI44EDJ%*843f<bgFx6@^`+<tH4|Tk8fgaPJJVHl?NPqN{yVPVM`VA?IjrlK
zzWnpOolQ~86~Vu`rjs-@)bD%4{L!|d?=I@>Ql*csYRc`u6u7=Po6+|>4W_>+ZuLt|
z$b0li5n32w`oh7mm^*LetF2jJ8-#`n1{D}86ygn_hGM)=)rlt0i_Pa!Rb2-e3_s23
zgSbk`RQNtaPmY8A-m9nr8VO+BqK&`p&5K1m|5|gW$<mdKr~a!0E^FYt1D>VqS09Ax
zcUB28lo1Q9cZZKZ^Ll{V#&1ZyTZxWkCr;NC5B~Nl7+tc?Zod9LVMl#lB>I5kR|zh!
zh-|CF)qUNf_jQUHrT3D5toe^545{O^4G#e5KdXBojUP(%JW}GLY64!Q^WL<0x}|Sr
zW4%NE+g0G>Y!rl9oOX1tzQE2fw-u<kYhK<!Gw837J)n<V9e5!}`q*IgcW|CAI)n_@
ztRc;!NeF3p(K5ezAV%wt&ljseJ*ts4eSSsQX?z^Xv!K!JSp%0|w*LtZ>c+;|jT07~
zpZ4uRPXvZSL~{V32Qq5-GUWEQHW+{vQd)HRJ>iNdO^7`&`4fyZ++6~XO!-a|MSP{+
zcB^j<|E<r1axX<<bbt9+!U0Fm09uPu9CTKVW|2G|1GF~!6(uGU=Z0(TR1}DrbAQ=~
zWZl<EuI>KV5B#eWs&sCH9)BS0$0e;^eGjDvWy$=K0q!E>O|5tLh4nMP<rm(LPVP4P
z^!Bng6&Rd{+X?DB8m3j~mFw49PxBu%aaxSRoAvZnRE+CpLtJU8h7LWO5pe9XzI+BO
z6K>X3<RIlHCHd^zb+N<w(q{eJ1ya%u2&B1G_cFPKOfzfuIFPy*dG!=Xu;!1B8?H(Y
z8Y)wrbkTn~H2wRgecp0)tt|iUYzgqA5_Qb6Qvcarn8|QLKrF#q=3??^-RRgmbsBW9
z^(0I(E~_L2C>QJO^v_%-iHM1D?iEGI>T($ZsDCieI^9mi+NAOG^Miw^*O-Zc0o=)i
zgoL+m--hMFpur}<wTL9e3GorsS5}S#4n1b(coUEQL>6!bP>ba$!p}M#=15u&(W|D@
zrQnw;ABe>frqvPz1C_ke+br}q*{r^YV-QH^(Z(P8v2VQ^gb-l`xLgc)$r4Z@c<?P|
z#yu;`0K8OriyWWt296CM3MY5H&I(fM7EJ`QFj!)05+2TrCw;xH^W?3CxVU%tDL|tG
zT=ZhL;4~6mtd3qTbk+R)3@v!|S@~pYl91~`J<}1b?W@$*TLMBH6o_P<O;-=shvc)~
zZgfn<cdZ|Xn_k*=<vz>7@?|tV*mYZ<fTh-S-f^dF^5J5uFw6I_?eS67H-qT6)InWc
zUEpH^A!7=NgX&=G$UGSie360rtlly{39NWg=vYaEak+b0_=q_dr|F-mdZ$7pQe0d-
zfcnx~uyyg^<O<ban2U6s*Y@goJ%HMrE7jxV-;n?-CQw~%ZEvS?Ss*0xENpD<HiRBR
z8Jwl^^h?Io)YNF{%d@i~(pIBfHXJD!C#hgf4y-e!`RqxldnPKV647H$28H_2lR|<_
z3!U-{bt5p_*2S-LBV2nrQ>xs`y?h7fFQOg<cYd?%?Cx4yT84?yx;9c|Tx0l2nE8<u
zA+a!gDoJh<|7gIYR@F+*m|uJ9e9lctO;g|1dccAVrzY{J*`_3~Ja4W)IgC1Y&8jgD
zV~ReqNd&j>seJ&rg`1nT7qx;9i(jnv`CBuYG}C#ggC-Xm94B{nvOKqoAHTN3C#X5F
zv9Xy4g5!MYxF$W;?Rbt$^$8=cLxIJyvAv_mqZS-oVq-vm(1JwX-Q9(Xp+eY_gR`=-
z<j1%W_F`#vGYTc+4fD3Ds$QO+O2ve^hlNS;X=yTJ&T#zaFxYRlzdL7A_BDBVk4Bb8
zgQd`LUh3WX`UU6A+w#DmxeTr@E+#*p!KMGLuP^iNpIiY4LU2z1x~Pqwr<WHelrc;+
z(nx?(o|(Ac{-dLl)5-Dig3kr;2FwAP9_&hnAczI89k5c`w)Ntw;|dHxgzTUo>^1J|
z?PVv8{Hmv>qjUTFMRdezaY8~QcBz<DL8l56B~GXS)zOs#UUa@W1VR<<#m^5{gg~6%
zkW+1Llh@`(%EOIDFtL8<zQMyk5`)iqe8;&{UsImFKN-?)y~yitL{|BCEdQ9@OXIdV
z_G}gy%8QQJ@Q_B#%+6BaJnnI*WX9d|?5zCWzuwOFy}X>2GWrqZhdthCHEwO59difU
zrrsc%SV0ElGx!h@xhR;1K;0pN_up5+&2nwjO<Z4C(97IVE>`jL^Yij*lq+22l`9;J
z98bC(6}GDx&~)i3J1d7|XG(q<_z@$<=<D9$r>_#0{zv=a_Twn;dSmO|QSzUN7_bhW
z*#{J*`n{iz+S7ix`P1%oFw|g@L|J85?=qA|mg(^?#p|Hijc~m?92?pJSg(#{dw^y`
zATB47T}Ble(04R2Ok}7SvRvMbYs;BWGcmS|5h6_3H8G*a^+btB92NWbkH$u_D<0zT
zFS!hggk$2Wy~g;@@zMNKUp@{L$=rBMh6;(H89LcA^#=dloxu<0@Td_`KKy1;CM>9`
zA`r!-sn#lmOM}XxnvM!f8Q>RVTf?fCcfQ69{f=H<{2bM@d%RISt4Bab8{Nan#+Hpl
z>XlELb0Og9s3^t^(xXcy!qVNRLrQjbtU)m*)3vqW(Ee$^EGjlH!($VOH)CDf6c%=K
z(CR0PGl77M(>`lm%!=p{*t1CoVRK|+nL-tAZcM~+Vl+h2G6F;Nafn|eAJcFkn>K>6
zVO@mcYGR6O9$|*mA_ly4{hy6AC8eK8D*+>mSIKln$Bzzpd80g6@(C8#wPEtS`fG4P
z?TU9z1jL=XiYMaQYWR61IiG*V15|QdUESoeeYTcN>mpTnS6EmWpb|RR!g8T5<zSUt
zdph@^Kxip!V!qX{;KK*x5%30d4yZ+{M3Zj-HAeVEZbI^aDcqf1Wjf156XZUL=P+Pp
zln<j0{|nS=#0vOfInI>&?Hy(G&<2PW>_{|?Ca^c{PoGuW^R;=WpsFph{6*fJA@jPk
zR_J*Id2rhr@=9O(WT4w$tZm8zshMT%jY8ksd-88`Z>sjJ-i)}c9+x7j>ZkE%7`M|E
z%MEwS)tuANqSMg)r<XA_Pb%(a7vD_{zu%?^x6{)zepj{sB-oHQ%8Dx$97;GL;3eo_
z+v@cHbFyImJ25RJYbD)6^7Rab_-?#ma?Xt-=?hhVnhwM8dc5FT%jed-AA~AK%8dv^
zu}=AUn1i`{vA>8i_iHgH641*rYgIF>J~<?mNChlBz?8h(fc{K_nK)8Hk;+8uL*qfq
z4MW%rAaTj;nsP?WF#)E-!MpU5EGkK5Xm}GFBAPM%y>7O~^HC>TX&cBL@v(cFD%5Jk
z0ih;}ezQ0Gskc{Txy%?&`xG^J^>1<Kb+0N82|by~W*>S(ZZvt)tH|Uowm%>&!fk)f
zqHfmZKkL1C?yWfxfG9S26_k7)|BO3&x@5)y<?O?@<xZ0%{ZLm*jz!+Xxew4l7GfEu
z1X37G*r#V?HAkVSs)}=b<Z3f7dd8Z&ZnoL|$jF&kiEy0*Aw2pS&yjfNySClz&9xO1
z<ou~iKeb$pXESjJJLYG)!YMJswkeH<eDIv+{`Z`0i)JhKnl~OAm-fA?<|<{N&>k~p
z?Usj(x=g?2FJAkUkLqNfiTd5_2M$dl>dJp|doD%3jJ{p(Dor!}IrxDv6W$Bp-uS<~
zg@V(X5jdF3K~D$(=3Z+%-vHhm2xR-3In2n<PiTC$(WIaF@89F8ici{gwvl4AQdS=l
zO%hFjII0<Zr!)fm6n<!sT8RnlTTziz-bj90_n*vnUe~8vQsl4kFmHFJALQyC-a5XJ
z3S%nz-lw6YB}H6Mf!EHt9b@8vjv}g|p;3g~F4JuUkT-8s<4U}WiRQUyvlI><Wa)p4
ziMhXKWGS%Z0OVPgm*ZX;wiz7a`dtD$0?@JSxz1X1H_qGEXh|NUvNBNzd4`voc9q_6
zH@W7x#Uq`^_b&%0-wS!3HcAlJwzO7+^z8qLHla`cF3(NrKUbMIyEt2Weg!JFZ{0p8
zStU?jkv|%qWBzW7zRO)CLLQU4Bvamuqm!q7X^-!RlyCO4R^#Jx2Jc=DHTxQ0o0(zX
zwY`)cHLWrckQ(E1?2qC3jP{dAJQn_z5X&%7M*`Y0EFtvWu}&f{$3iP{9aI$}2f&hq
z-i#kA#;2JQELZqKS_uIc8|u^eAy=pzGDh_nVox5TM86fkvlLD$?ME*9+krf4RqCfe
zM~oq?x)irWC@QEwq#4WkDl8WO#^zjM@t@vC0pQWD^{!T}ga{2~AnW*aR=L=oH-hiw
z#=n2BwQK(vuq}Zmg*Vu6Yr-FZ4z9B{258x~W1DsX)Jrjx{F>f#5E?PvUVwCh)74m$
z@@14u0c4&K5XGid1^k#~&|o{C9qmxD+}74su!)tGy(YvWA|jgiJ{jPR@&i>ETw1FX
zn27=!2z7SFMKQ6}v<FzSrXK!b2*wVl4qn)V48P0yU@dsw(P;aLfdR#JJJRjv*{5E|
z-fQC|tfq$nZ0!@X1>U#%e%xf7atKu(X7HBw->{Z9-k<Yt^*c0O@;;*P-7e_IvGFw9
zYjANZ)$^gTeJ*Z=&DXc_6m==SQ;ID&@omUZ8$js6zRK116h%~O%o*V8GBV)+3DIDN
z1&8KRhZ)UnC_f&j#X}QOB#c^(l`<}JtedwD!3)dXJoDrv)O->k9EH;tX9_cdbyqSE
zomeDJEuj~hbaEDTF2y6<nG%lO1)jgh2t=uWBbdb+co2|I8J4?pbg9tcJd~IdE+PnU
z&{!#;hc|)csyyS{%yWl@3DVBY0Raw>U@5#V-neA{*6KBmRNX3wLKF<phC87;_NU94
z`S|z%xx+9wjO%0G`C5(w!)kh<VKTfQ4(T;^oLa@|TPmzi2*i$*c)?eeUtC~ePcAve
zrL;=ssr6p(6EpF9`ep;ROHgWqeKBp|+t}8r=iLwQN8~*K8U&0eUxsp_96IDspd886
z;GwE&V{5xmXGgUi<Uf<ns#zIi2!oe+vHxts6Ac^?T>VGXIN6GieQBP7Ba|ij`fJs<
zq5Jcr&pZr_OnIEH<CMR*<8(a~R6SN#7n@gi9`3K5uv>o%&B|Hwis(3Xw9j1B?s}iC
z@2D+gCd4&&QcF3`tQ3TBw>G-YH;lV2cq}NoP3T=WANcH_Eb29%)H99@e!6|5e>qpV
z61#5{*ZUW!N@VFC@BDs`DUA@DW+$`~p-@jpN58Z$9(T@IQiy?v_Th(o=`wmAS|vVq
zDuq~tK<Z}SjRGq$XaZhDM&%3PK!z=A@?cSwu&QYRrAw5aqksTZLMBkFk$V1UX`!r-
ze(^E6%l(U^QBNby@|qu?<qol2ny2QZliFuIORm(uz7HWC4wr|^5sFFu2}wzhX?FMa
z2Jn&#EVPCy`SM~%%=!5EIH0kFPa)aO&F@w}cUD?^c)S%?EG;b+%(B~j1@L5Zu9=aM
zSO;MvP2w{z8*}r+D{mxHE`_52=qO-N94M5Fhv&r?V<!eQ$dbpx>JeCggd2nI32pEg
z5XgoKflGg&ivzVbMXo~iE}$~O-rMB>n06`Zzl#%n+s;~fp|EheSHD@(nvc(Qcl6KQ
zTs59zvE9z;Qe#T19q^v<FN8-r9P$Gs!Mg}~aM?{IFwRxCk}f~XE3L+&;e6W6Le*01
zJ4kQa&#m9kNzGun4IUFuYvX=F(%>u$fzcy7+bKPlvH7e98{d4N1O5B+<HQ&5-2Sd%
z?<D4+%>nNR2y%(}=%NeH7U*Bs6DKBts=A-|<A%TK-y<c5;2j4OTr4=%Pde8c!03Rk
zCu5f^4D?jkK0kaX<szs*{O^+B&}ni%l6t@XzpDdhgv)0&k|)ZT*QAedEuAKNn@U_U
zMo*%D)(!#z?-`(spDDJPbAfK7Zg$U^Pm;DB6q=SokA+aKypP;(50^XW`S|#wC}Yub
zR<1qh3SUv;;8xdgkoTC^M94{2k?zh}Va8~4o2^q&wOm)0S9PT7yHm#Ls2KZHhT|~q
zUoG?aHC;<SoZeboc_bIE8@t)K+YE-uvrVh5ClSNNKk7EE9sc{s+I)Mw&3_;e4L8;A
zm`GYYX|g|@9s3^SdG}{<{@{LVT3v128~t%dqt2HvUl8!U=_<*R@u(2uboJ*ds^Y3~
zNnSK5!}gx?UTG3`yyfwIJ@{dSA2zA3uc!+7EFi-3|J-b3&wcBig}-AYnv8IpaHmmu
zZsq{6F}w?OM_{99@ww4?JvqR$2!%`m*ijk*4v+FlGAPEmivCaNFqci;EDEGpWm`jD
zf4sd5L2ZDB1Nr5kLx^s7$4--iDvS<^;o8eoNL1I$K{91YWq%_v6X>I@+nrH$^S<np
zX&J<k@fGx?9H|TscBqao!AJgSJ-d`o&2YP0o+uW&Jn@hdjZqoLeri=`Gk@0|JYDBa
z3&6}yYDUs}69z5UGTZ0DBS^$1?#s`r<$a9$!{z<Y2WWn=eXp)26uoSFOa+pWvA)E3
zBB?Q@DxX3i@F((gpL8?Qdja@IBrZZ4{aiVx!<m7wZ>b0cvg^v<;{e)3H#f5%@(KaC
zqF-mM&0q~Wufb**W(2sgZorLwO?U&K><Rq$pYg!IB$|N6NnYOc!l79&8dW_bBjf#+
ziESO(*4+8Ug&7QxELGFY{f@vo8%z_m%FvqPKvc;rgXnZfA2I*C<;?Z*X+Vsjm!^W!
zyLY6}Q|=6V`Q&7m{z0tD7`!*t_}ey-hShTMob{TTI!1VCh8%$ZRQ`|39k)p6k}IsL
zsX5ym$tzX~gG-b5fFy1bRy{y#D;LM-GQ6OO`>LFeDp8oG!3p*bhOii^{FNgWf#5i}
z=h{@1jW;chgQyu7DTQ2Qn=s&SnhiCE4)V2Kd$u!Jn>P=3tJfln=AE`;n@?EjR0=i@
z(^@rVkE5~uhIKT&<Fgq$S#Az%&cszGbxiu+tE0)jzR1CKEwY+i04?4|e&cGYbH`nO
z*4>ra!7Nv^z?RMI{BObQIMu15khk#lAqO2c*mod+PaDW(pp_t*F*44?g|Gg*^)7Au
z?U0T{qKQCDy~WnpfM#>jtP=J%>Jv;N(IkL67~d;)`G=MfNF>T-&`b<zd4eVai<A%>
zp+HjFvcaSfgyIeiPOn(vg3T|V2TSRS7LH*LP=HSdN`}`$9rmXs3aW2nrQ8KFRhWpv
zyRL6;!o$LhHEk^{o}8Uet&I5lqvYR;A<P6lqMu-fr8b!NF0-Uvr^@}LlWjsPcQ1G!
zdqs%Za=Cg?)Qgt2CjHuM>vX%nxrjMfQdO1ccvMUgcp?0SsZ(Tg=dFRR$?F%D4oFS^
ze=`Cu_Sbm_Eo*?y4s0ojCXypLU2G8}Td)iqh~#xF5<Oh0LW6@iGDOtBr6nq_CO$Z$
z(wuRi@)5<}k%EF<jUuzmD70%BCYSx;&(1;9&7I58<Az45J@+T+W?)CbOo_f<kIm{#
ze!}{&PQv<VJ#Yf^XysrsRmg7xVR2b+cZ?>kTV*hkXqMTjfz4SdYD!FkTl<6{nx{@I
zZzL9W+7a;gZOcg#iS0~XVF8aY8;|#iy3+?8GgD$(B3}?Q53ZS+)TE@I%I{-aVvWhk
zvfZ=Vd#?N*AG&?w1)Zf{`c37GBrqwi|CI_Je3r@*AcCK?Vaba%Ad@!|fv_j<Vq0Y*
zq<MCFO+>V5@j+f0)@upXaZscoO)vo!>=%Bmhr0tmWkT_>>H$;jddHU^$0{_KQ)EZE
z0KqP<NYbJBg2A${y-TvzcHf52kS_~kmWow4pz**=Xs|qmKj$xn`P9+ukndT|dmkrL
z+5S<s9!@C)Xy+M3LKQ#;sge>t6yxUlnv|5Z)^2g|#QQ(0gZg>6_?@6yPb~Dk5}^cM
z$ATN4XqX{G@Cfb<29c(Oe77{mv(VF;>iiY<061fC&YPK)?dKRJ2~`@Fwt5vCXAd87
zpll7##;aIg0@~#ZZHg={N<hwgyh3$>)4)EIctym86~m0~E>}h?zqkTn7hBgG&K|#w
z`4W7@FRG{LLme^M_1ilnS?kYQ_BNl%TRjbeby7oq!7;(%-rRyBr^t7)Cj#eEMJVSO
zTx9C9rLB?Tz=CIA(Q^hH>U4;W4Yoe44aN70tiT|@G<fea4}71j=Q)rA_2mX(3OgC4
z=O)cx0>)UW?8cj=wm?90ONzrlrESg4pD_qz1pwPcG7X(bg|4Ei%Z;AKi-$*BFU&2D
ze#d$C;kOYRb+LBCKT1q?O#e^E0maI@JvenwA;zk;<;=redyYVgrY<gy?g>wk<$m*q
z!rVd`UVJh&@^vVB8P-)9p=MS+JMgw%qVZ8pCB=G_Oxy*_K6yC*V@Ng=R+cQBHdYKC
z&^h%&@Sx(Nu8KS<tm{&Pd1^81{OVF{rSdCx)=+R}*y+Q?k<bnH8~y7)AL0?p`D;0Q
zWO_4)zW6+^wLC69PA0~|R676Om+nlqT=|Qo+G?H@qbrApR+N!a;JR-Y7XslE;OhC}
zqatJfob~TF+^;?4znAxx<UFXodP7jw?nV>-G?2C*!SKz9J$ThSBZI<4lI69Vug6;m
z9>l<f55V6qmhG7nj5NEAIbI>+u|ie?Ao<bwuCQQD7UI9k9C&|DDc(knIi-JR5X@3X
zaEy7oHe&Z20-_g3k^>6F6JVYKym#ZG*YVqa?9-aR5fZs0vW6N}(|<T37{UZw<rU>*
zqB)>8?Aj1W%jYb4lhLK`YOg*j1~9lcr?}jZVuPKF=JTTk6h#4+Mv<kTT}CvRw3N{|
z&5uTGv{ORkeiiQ<)5SgM8s+1=$u2cM+GsTx&YZjBBi-SXC+8%MBo6PgticKp?yF?(
zC-1?=`7r<l)eB7?p<=Xnn3s3=!qmg#<6uexAeq=;-p9T4poX8euASZEPQyNH7o8ty
zdmE6y$rzqQK7A|qqdZBTPILaS!k<~mKeNa8s{C=wk;OY1CPsaIb@z4w?Zh=dB5GZp
z>^G!l%=#!0JaobB6MEWZ^IufwIIuk&X}oQ>JryExyNcv=+r(8i@*Zi&_-=E&4p?c_
z$o8MIxS4knji2^)FTmpXI=N$EwP#yYM@)zZM~4iPLaWtwY=*3%eiaUbf+^)k9jw0h
zWRf@6EUJemxTPSRDSID4jyHa%*FFyyPF<os(Lyd>4~zO9?uyf8F0Cyml6Jl~jryaV
zX)X3UA6wnmn;R`1%UYVd4la1Oxsk{Q>uC<7>TLAdW0{BOqRQyoAZ9^5VPhxF#MvnF
z4`MG24<OQ`W41cMaY0r_n$8BF^Ud?-{m)6P<o194g79`l;a{n6z<$Ug;TXmD!cZ+1
zH3XcXSBxRKcR<o)qmnu8R?y?CJg{%rlShMg0wD!~bTDT9S-W5?yfMZLd|mhyf)^@b
z^xm5zt`}gr3}JMj=c^2!*+7B(@qHK`Mf*#lgNHVT$MRCGgA`i=GpCK8I%7X>InbL!
zo450E;9gG@@sR#ku0~L)9QR&FkV@F~vi_co#R`r?lSa)(z(X}mdZH-8FV1-5Du51p
zAoY!msw(_K9uIBwl_cX|=#xjydab4dq<@u~f$jqox4x~d-@jX*ZIAOt+0E9n;D-W~
zS`04B!5nNTHReYsAD&+X76uYIqo;1h$~2gG27UUW2YvBjGOzjLWc-FlREyuAo;w5I
zBX@%tAjMY!%<8&x)asY_iEyHhPMYgKpU~6c!AZMH>#KI<^YwK!cLgl1N1T&`eT~Mg
z8*uTodD|0e3bAzfN#Cn>N_noO0~^aQj9WBuO-ILa%^#Tbn6VouXt3a{i5D(GDcEXh
zZI=CvX<`-D+2xa(I$>+WaK$E6@5*o2-`0gLZ;*F>-nYMcB~@A;g#3KAzS-UGhv2fJ
zBG+VVd{4@)CjVTGuKM}hjtP03lw7Dkf_%My0TM1wS;iN~o^8I5cWXJkz?N(nfrg=4
z*=jm%H1&RRR@{J|5e^R30+~7pZ-9JGvu&_2KIh>+I1gbph}6_M5?*Bgn@S@36AVUC
z*Y_FUIV0wG3M)&obgb`HUUQ3^&*9OI-n+Ixu9+mW=FVr$%IOco>Mve~k8ypM=)jS0
z200V#jw@b-|DKEB$@P2{V0%|M>D0kFB!vLsY*!36U5sI*;$aiEWJm6KK*y(5y6w2*
ztkZM<j^LnPpY!XY>EDr2;hsjn+myndydlQcBPabVxB1Stmn8E&+rnN>nwrloy$(;u
z<h`2pKz-D3vkq#LjSYTs9LyX~BsMqrt%XkTOkmvvl(|T`+hhsrQvvzW>IF;j-Bn{Q
zBm+m@xCm4f(nc*AnZH>qlZXPD-La$)f4Nb|uRH&X;qF4;*J|-w-Ma<%tB<(qtR!;{
z{ls+qR|K^E&sv8T=P6E4!;6j8-@hvvi!@AD#z%t~_Ak8q^k&)MH)4$`jcYGaJH+)w
znW)}kBiK~_K380s1`Q${e#H^D!VcwpoBmBX4S__8a->9vEA~}hE#05zNty?blYW$F
z)U~VGvf++gVCysUH=J6#db(ZPI_Yw&p(nncT=X~d@!&Lln$zqfCy4v{k|asD*DP+x
z5)Ts{h}efC{|(CC@2%GkBWJGv_SMtRf8Rlx-5uge(mqjitK=kfPLwOl(SdvxzFEi;
zIyN4>ee=ow%|<vdZvvL^`gVI6oEr49$!s7}e7@1G1lizyb?gK(iB?u2nu(1K4cj|A
zWTd3vZ1KAu%i2D>wix>c23)KHuFvDB+HyXoT{y1;g}Ug@;O33f2ASl!P$ldMo~X93
zVCT}!kl@<`|BxwVO{v#@$7ejBIPIF7;qr|KT|$Cqf#qpg-uJ01W60N|1*vdp-<#P*
z3<$vO0PS;se;*ti3;^yqKxZ*Pz_9@IzA@bYq+X>6`gr~aUD++hrwM^Q8`0|yk8r&m
z{#<By5fW+->v72F{r={;3td&!&mc~4Oo7H!+wlb?8`LX-^X!zg!mHSDvsTyfyUl}!
zfdjoV6Ze@}&osH9>zkG3n2}|!Mkh!5gt|#e)9RV@(NNGYM5xO2?00(#$J7&%y!54!
zQR*ne#j({8^6C1(F~*|2(knc&JH%4I6MU4i3o;EzKQ_#Myh`8c+t5jhkmwM!`+Kls
z`#t(#{+-7Hco%J%33ov`Sj`XR_L6W*kF%YQvdI8@p^6&z1s@5;Ct*BdT6RTRXlmSs
z3!;j^sFMooi$`-L|CWU#oi0;6BHwxC(W~P6oP3OYR2%)q8<LaO+d@|6@}|Rtbe@%?
zpScWx!R5u(Hb$obvH#ORD45IeO6~hkw2!J)KWTq>UF?IIOIO}4pgIO>O`vZ2rcE}|
zQ6o+Y4II!@cyqRt_>ng#eT3^Ws1OP2o(UUYvi4mb8?GYZFJVo~BM<5vPJ`8H>IS@-
z+Rc~u+g%W0%(h+r32Uv^nwpCB)D=!bgUz#RfHQ(}jj&ua3w;|~UtRrTkpIBFSufFK
z-uB||VsXfUeCPx&jnjy$z?>Fk%NH(_-t`g!(L@-v2{Yq;#Xx1CCwq!<ZtIRsFBPdb
z^uzi`9sS3E-j!ofsh5F1KKGfFvGNJ}JV{0{am7EcFFI3GQbkN&g4_7-V@vnqc<}@!
zV~c6u<%8m{Cq|>`J7jKqa}P7u&+9Vx6B$1+QNCoWbUjIX=AX{%6gA%+DYLlC^@F^}
zF}<qD-9GtwU@~*qo@8{80I<reeOGJlN>x^~?H_s!&?L7GQ2e<~P`JpGwrsSCcfl@=
zVIrcK`O4&as)Q(=jA8KX&VD9Z#&E_bO70Rwfe0*tb0K|X-Td`&otknXE>Js9OiW0@
za~Z-G55Qb7o)lRU)iG~p&z|Ye|1*mLkq|1R(ssIsVAE@MZjK!5i&~iuFa`~n-mK9J
zZMimr)R;*KQxIscw2qH&<hZt3h{MNhDrZOPR0x9>{(Vb7{y;lha3rhXPfB^ny*rOi
zo$NWrLUF_G+<j86&y@iYj*y6BELtAS!SFqNtqAA^4UMnIBVoBf=LcBTzli#lmivMv
zJz8me@ex??vNEG4f}_)5tm+xuqMAP<aI{Y=&tji`5>C6UL7iPUEnri6SIXi|%U<-F
z`E|Zhhi@^WB6(^?7EhgMF+;wS0)$;Upk(yyV9d;-`uM<tFdFAKo!`pH<45asrDR0K
z5PUY&kUoo;=y<FZP05()J>YJ&%1H=hJn2q0W0~`h^|Xb2Vzj>@EjeH}hglDCTHX|2
ztA&oLvgXfb3lj%fnw5OlFcxb`0}a7*hCq<HMPZ1q5|4bEdBss7LB_`&XXSYCoH~t{
z($+$b8pL0o(KqwL9%V&TXtKl)uTnvN;e00D<n89~!(tgU(#~&wfl4j`5!QCAd$(9j
zF?T7tym+}Gug`1ietu=Anes=Tn>d_;&l$*c4+VfMqJ8-=o>BeR+rV#IU{~CmZ%i_o
zE|5vm+g~?!2(`Zjk<~rXJa68B#|}7n;Hv}SxY>*U6lX>wQ8fua`3uvMPCOsr6&dBr
zsH@|lj3l+;t}e}jhknvfdv_V}83Ln-s!Y|4e0+|AiM{sgsnCPdM(E8`UF*H>je_=_
zZ*q)H*;h1TI2TJ32S8#5w4cQ)NU*AagQ=jPYj$?bDVM!9S4paXFPiET;O4--NGh3I
zDvT)|Cc66x@+;^$r_a0jo5i^p&SD)aHdF{-6s1Fg+%S}$+RbQD&ukjrd}b-*v6>XN
z@}j?Ta%XRQd#2UTZ$)K19FDnPDV=a)3qURUuT@QFY~-w|O7nVNKevWantP$IBS>IJ
zSCFWSH{3eE;buXD2D<`!br%UPW42dlVYeay%^Mfxeu!Gw0AJD91EKR8mxrs#^sQDd
zoov{(3991=b<`vBjX$P?))LWiP^Wn$e{P#_=Y}%j%c;c;icO&kySl0$uy-Pgsx&E0
zMOI|sfwCno2=bWCH}I$pW#m7*$@FmDIdk)#&vARLtJQoST6mChM8q_5aXNGMRQ+!L
zF!rE;-1kOb$gB3;?BUw3{!=o776>&!J*k=aA4gT(-0ZX39SJ;m{H=7uN%)|(Pxuho
zt~0puAq0smO4Fu|P3APifn=AK4j8b_0Sw8s3S_gKydb1&H<b6%p5h;N%6(S!8Nel_
zQB_kA2(*x=es1BsHY#`(eLX?5eWe~%)ZDRwPHo7X??<`$%-;jS>r&i0UyaE;-g2*{
zGNS(Pb5jqatGrQKD5<jFz-}7dGe!nok552>PzN(su31=F@H;XPg&U&T{+QlF|MJ#k
zJujSP`uNza?C069ZXU;1bXM3B(4Hp<c(EbJ5-j)`Ts)KUZKgN|z={E(NJ>U_b8`dm
zs&UH#1}5E8ErGN2G&HGP*trM6@>gc*y`Jb;UfW9kKXQ{E8^$V{!NNpwp(mU|UTbp6
zn>%~cS94X9rS<Zk*hHb6Q%)*~P80}CRYfat$}xxpZf|mJ`~uKS91+v&+>LYFYfRFF
zkNIZ0{#wW-R~236mMy3IRQf#(Q9R5|Z5`M={~Gty)Gn2GW1`3RCU9K;X4>D*+w9(j
zHMVl)yVhwBt2gS4<e~dRo)D`0ZI`jEHr0i9tDu_s`I-0ZoyrPD^zBMC%cE2MmWSz9
z5eVdW>-~MJ??G7v_LB@jX#{yOv2|11#>x1^M44W*=S;Olk{63+#7jc)P>H-pcXxna
z0<&WD8N<NT2a7Kqz*w2JLq`@V@s3hnUtgaZPdFOvcIIpby}i;BFTiu0+TD_U92B#D
zTH8)5FUgO__Tcz~yv$h`bKtdGF_sn%72259{`Gk@tH}(B=vPCSrHd>G?M>B0BKvX~
zLUWnJ3=f3)>K)xr8go!^%{X>!Tr+y}mrqFpQ9RV~L}ldmPt$CtA125=5|T<k+#y9c
zQg(sUb8pYa%IXy>YZm(_49n3Pz%GY$07@Uw`{142xZxl2m5a|*BP`fE6am<IE;2%`
zaD?o;#}!6ebMv|V)B&fK&BWZ?iOfw=CEAmAM4Z>ZLOr$!SIOB5t+hl&vE!kO#f*#L
zEW6fXtgj>U{0ntSgCmw@$TfEig>1eJgxgzwv#i~TAwkDIT`Cg|i_#Hc)JlWvtSp_$
zv7H3=zU9F;mW+_kg6048S(_KecT^o_GSUh^30>b(ulk-&*ZKbOgR_b`ocGJXA|&$O
zdog8UE8vm`P<sn5j^s%m{9KI!RQXYs>A*^a0^v*i&}@lFfuV!NR(*W5uw2{LyYsm@
zol>>5`}m<z8la5<9`)MV+R{?&4z@%S3LHtkO_PECx_|4BAWga%Xws$%^3*}3&20$%
zrLFCFqA#4J1V_oO<vg?BuXPNor?p-EFj0!MDt9?5kEnyt#B<QiA4|K$>wEbyQHG;G
zdRAUna{n{=gD81cr9s7nc|=ISh(XNyPtZP6PT!X0MpT@S_n`R;8-NCNZ*Nav2-s=t
zdFei;s%U8e;n|im&zyb7+ktPt9hqSfUr-IB>4ztKHl4?kSVHhpb!_wIbJ$b=yyj_0
z3cS0kR+Cgxk^Iz*_4yr-TRH(QW*8H)=^uc#FXnmP+4m6m-E}V%MkM1s(iyoEYSw=r
zxpStU!@Ge!pQvov|NNTG_^ph^^wtx8OwDGOT;F7FZ@j`2nQp~6UBVMs!PXW72!e$C
zM-hMk3ui#BYKH8FVOYAsFwwy5091d72wp)!0f2xFlL4_Ns`~c+-_Gl#aDIYUI*Rcc
z%>OQK9%EntNiJNv;bOsyCg{jrfQRQ|#;Sb%KgHEEG>6@(3fa|VOnRy5h!OJ|IwwD~
z9$_~UxfrH}kmYIC*|+aMah*?4g}}HyMn;d+;QrfJ*xX)&ozh6;hg|9>v;&op&tmWy
zO*a*MG);UH=`i%$`N%WE491t}kgp*DK9OhsWhZZq^%xb1#0AHqSo$4#$<MF6fsYIT
zjywzQpXB7^I8uNt43xNRq><!3iW`^c3dWkMgz4}vD%+QjXI}_X1+P}CSyl7*nGTI?
zVlSzbrB9e+hLL*zY{zU*Pun=y-xU}EqU?1Y8e2`>y+IQFpZl3!er%t_*<dKVHiNxm
zJ5xvcZXPdby?%qSd2zkYi#`__SvK<W(TfKMo0}Rnmg695b7+=^K)eDFo(Uhe&aQ#X
zT~!tIRRp233&I>J@h=(iL!H5l4Je>P;E2DYqqLZXs^Lq#dI}1{|49M2wzhiNs;8}s
z0$o0R%5(G#e-~^Bz(5d6<u0(rtW64TwlPOef|+|Yw1bd{2n7P<e?W{DYzRUeI@l^S
zz84nKuMz=&gPvX{5(&CrpycfMQC%Gm?*&{dNRD9#>Fjg1bx0)0YU>@dNJbMOBqm0{
zp96^b<_1V6R8@Dt={`G469kO!tgPjgm55c|Yn7R=xeU_3Rnn=TiiFbtaTVZ91b4oY
znE=fYgusF8>C+K--e&370BSgX?ytfXcA5l2d}+-@XH!myFg}{R3}+BTSQg(M@r^M!
z0j*7w&!6%F+VAkyK%Z4)tl0jiZU494@Wcq7I%aH48tD;>?qn9B#ndEv8QLr<tX?qz
zas-;g6rK$SaTQ6@w&Z@luS>S^M0#v6Sd{0x(_@p9g-9lRs!Xscj;8nbg6lb585_Qe
z>c=jlP;)pj^s<3(b@{7TdndwT98LP4sw|yF>iE|0Wr0;ge}bUij2#=h9rL0jF-0>D
z!wp)qIN44bxz1o`xyGU&>+5xs6!TZNRg;T12C<DxN-d|)bEJxcFL-P{oSdlJ`@^=G
zvV6wo-H3liD15Bpapji@OKEEp`0}tM)M~}1dy~R%3lr!ae7OIv+I+t6MM99*kb<Oj
zqnbOOIbrn@WsLl^!jg3@yB_pR^~n&BN?SvRhlhWd4x~PH_zi)CiWvcYawW5T3I`Am
z0s*O)*Qbg<zHgeS(E>$YX2Q1P^W})TFHTR5PfOYW`wdGpd7m?^@Ln3Bunq`8x8RT<
zcc2-%xVXq$5PWYVuDH9sjgUW}ZLefDLc#eaTi{Rn7j)ae7!37yj9P^SD-wdZaI+~v
zA<&t55goaw!==R)!9rU-x?BQIW<37RjT7uQvaOF8GAg{Uz>f?jM*%TEX`}=#&@9h_
zC$hMxuclTDgj>)KuzEpKDkv}745%?_OU1u4d;o_jGqDUD|HyrA=d9a^J-R25w)<V=
z+5FiVlS0R%M`+>|7>~NxHerzjXt_z?^I(SfBh3dA%T`?;?Uj~|h9=3_U58<aAkbAO
zxegZ9>s-dxzN}8N^h1JTs2SUZEwC}-oI3Y0=OeKPpAgu55l?coNqUNacNwvZ4bQds
z{5Kh_$Ds(-6NI&Z%$mZn3}wv-B+zdLz@@`-(L^@wY~;Fg@j9F(;@*^JyCaFl8;p3P
z__U*WqjSFsP(=rsD6XxBW*6eVNk|eC0a{5ye!29|ct09A54bHbi{U{Uzjof;^gkyM
zhKA!nclB=C(;Fw*W{%z;s5J_?-@CQW*xyXHHZTt%4@WW{?pE`Jb{^KLS_;FdZf47~
zu14*s2CBZj(p4eCH+gF3qL^~_yKUj0Ud?U<EAh_EN#nt7<#mo7Yq4{N`$p)Z-@|0I
z;Mq<^OYV1h*EWXNuk|X8dz15-LBawIN*s=U#ltWd@87WST2qaS8O#fg=wstZ=XJPj
zVzVp}2s1QUKkRF+WX|j6=)eJY+O!cj!zo`G1n+8<x*o3?7WMsRD=qabQ&-dx@YaoB
z?gs)?E-`i?8(x4Cfu_dqaybM{;iS@9E@Ak<ks`MIOPKnH>)wnQEsZontz=+mD2va9
z$@4-q2X6%js8glj2%!8CFD@<ywudNcOuObYH33SVw1J{<MG=uUpqCiF^@wAdD<F_g
z6|CN(Kp67&OG^8FHMD59fc6kdgKV*EbN`JS?NptY1g}#hevP9A<O7L47u6KcV4jI4
z^r@;USb#uzVpNgL5XRh(D#E~$(ar{iWH;T;Z5sfWF&Gx|MHJX5f#dYZ-Fv#y1gI*|
zM3U3@Eo73|`R1kgn*hsA{Z)YVNdT?1sasSo!@$jjXJdnkYL`?1HKSHZmjkLW80@U^
zQEoclUY|8@T;dT*bC+d*@0TgfdXtv@*HU<iP5pS^w=JxfK8-Z9mZy2#SemA*%oi(d
z_`f&4x-H-tl%y$JX_us<x)@P<IElV5Eves{UcB9p)4z8<DR~IpNR+g^&>)^O*_m1}
zPS-@3eZ17-5j@IDL}}AySB?x92J@Ijs~<WZblvi6b-Sl~!|l>|{P^Dcp62vR*I&o!
zx7@__OGD!u_I*wEa0Oy3>;zKJdS4S>0ClRyM{ztc6O)%wt3^&)iAk=ZC^X-Tig-cB
zPZR||utG{IsxT$sqtU>io&uCvLnPs1))g_i3}<I&;5Z@g0ZBcYiSp^Z{;27Y&p<zy
z&pv1CCN2F0i;H%eDcF$Gm$zzT%c{IN`2LslF|%$-oGhv8`0TEsyncVqBf^h?N7F~f
zI-*i^Z0VS|?hpuq;rz`T1XQ=);AmY>ow(z@s%qwjM<(tUQE;~NMh&IEC3u&dmKKgW
zN+>>V<I6)&zrkG4C9W9Jvr68x%&VuaUI+9tDoh3Amf_*y8-G&bzBw7dy3K1C*uEi=
zaLh`E6+&^P_?Qp1<CLLSccM&;G6?urn15RO>ni@oqMOXG&#LPzpc1G05aA!KLX=M)
zfvzg2h@z|VJU~?5i|q5Se}m6A<~T4#Y(2zj@u%zV3sMu@*czM^v|7xqXtAumJt5Qc
zUZIL2vrV?BNWtz7NSP!-eW}V><#|)l+0L(4|9!VC*oSJ@qVcTdhF;#DYDQ;rmqDSk
zbJp*6xy*dLP|(FG*J+XS!KSbN8RX02X(84A;aX|S1*O2m)##!hWk~c~Z4qsA1u6YN
zv)^cM>&<qwh<eILN!&Pr1~wqcIYP(G$ptLl(CQHo6i6d51Ax;EuN}xJKNKK(jpgNW
zJ(!AtT!JwS0cV6k1C&=rFrghlNd*z6#)~oWIK&GYHs*MSMuUk;>xE4@Zq5blNQ&_w
zsK^lXEHg->3XjJD`u|@G09&yJ-?QaR8i_HoMkM}RAZV3$)cU=8tRe9BKM#&Ic?*M@
zas?JLV7=v%66RfHija6Ajp)z=Q_dOCpJROyQ~dK2xWd6t@sXRsX7)gY);Z9-J?{7c
zwuTP{3SXMQt}!sM>>c!<?Ql6!+jeBj`S&`=00Y1A4n{~q5=~e{Mo|eb4hyKY16gBY
zqKPkQQe-7w6r2AocYcJa=$0V{M?^@73$EK(eSv+GLiA>E-oI?`!Ch&i?&a7r2wd1d
z%~BsY>k399E13a@_Mf)`s;aJ@Ht<h@!5Jj|KwJFBwp0kk@q$2n2U-YTwCPZ0>DX42
z-T~-PIH5qTLNAxj5Vq15_?WXOo$KRJ3TFuQR{-dNE8?w;jLg%gPsJ7In!VgGtO8bl
zZjBWZpt0|&f*ix2^S086TASHKb`W+`D4UYY(3kT*q<#4ZHD)fuwsQk07t;nXAivml
z(*q1EndyOjo^}LW3k!m5Y<M^!t)~U74&)$QzzxL40lK5}Leh?(4_^jB7#*aIL-`dw
z(3_U-xas%Fmq}Tc?+qBDzqfe@jZLfhBntCtw7g^7vCS$Jpn-8~n;z0(CKTf0nYG-Q
zpWo2CK^qhFp~7Di29u|o{0{EV%4PLE_`R;?IW}8t2i*KDy>|-ET0PF53>vu-s=n4d
zS&BavXzUl^Xh_ecO-y9Zo_S@b<+v)KBTaz{R!a9{oPA53+q530Lakqad$A7$7ax5l
z|L)}_Gdm#kU@09+@HybBil^nK`=*~nZMe{n4vD<nuDjd2C||fvSk$NzoK}T`OaNQA
z>jS_5fOF;%gdt3nUwr1l+}b)u&gCPGG(OO-R)f*A|CtECZS4VA0}TZtysr-Ar#E+j
z`Z%J=ram6c3(lGlUo4<S?Ku*oWp_7|{Lk7XRoDVxw!LBMSO2*b9|eO7mU&=51dldJ
zynJ$ui$xkSx%2S>j)&&2_8*NMv~17&F~9#OmI$7=p#C+{h${7kjAFbnb<DlA6j|zk
z>6$M(sz3DonhXs+@4$7t19=Bq<_2(9rw#nqjiktKB47Foop)RDM&YJ8QV@sb{;rJp
z8yy??N*dquQ?s5Ou=P{p6)j$&bmk<lsEWHL$e}T~JDq-oW1@*4g$Cs^potflu);Fr
z%qrYxW*nL5!@ZfN<_9#(9l0;2MDfM7O5S8zQb{YQ&>}R0IVh>6m4G$b&8-gDd%mVh
z<|&E*DKA^v!1D;%0BZM4T#P_y2M$p=4o0BVq^CK)zUuh>y|1_;-j(AaQ04;D2`sp`
zQovbG8WHnfFv{TxW`J-o<a17Nu?p66915^zSC4*I0&^Vb9Lrkoub&Xmr4z8oDiUUH
zm}pAS{{h(y`@jvpaRoBr0oDlsQJ}b}4h2=^*cPZFL7xFWGuV3ITNBy<zAgn%<QW$D
zoFmHy&RxufX4QOQhJdaaSW}M>Ni#*{veegSP9z!&$RW(6vgvaTA?*IGf3d+}sR{!H
z#c^XUESgi7+5^9{^=Y?xpYtwkk|!`Z`Se!Hm8Jx5<|&VrDgQdX%lU_^V_bdj*MS`0
zhU&2Ix1~h62s=)3z&Q$;1MV+K%(2%eZ~Ij$zlIDzKzIxM`WfbqJ{-ubs%h=<uny3u
zmE3Xx$CKQ?@MqJ1fFPS7@RKNMq-kF~<5wJcME;05G^B&P2bd8pYp81YHV<~d#F~_q
zV2dJD2&_5NJpS_I426JtdL8gE-CqraTEaraZTHQU!t!+P72bma*f1H1e2hb`0mcq?
zGgvx+oi|7!(Ij&twx0-!K@chi$6w|xtE^N?T(4X7ZLFzzlOhZJjNx4%Mv5undZMny
zgbfDt^0cvFY{;6UMWjTS31hHfb%TK)m9(Nsp=DnK8JST|cH8*~n0$aHqO^J|mC^O3
z1BeK?xPnA!Ai^G3Y)lWpE~B&qJ$(dW=l#I*3`lj#mSxb`pjDANKYAVJq(s?Z!Pqiy
zK|>7GVutVzMdJTFp&4>@eh<EZZA(CE`Y<Dd01<c?7#LdG+KpfcucxO*;@t{Ks}!Et
ze=%H)%HR*CjXiW10E-@ou0aLP8&$4dR|bG;5H7Lj+Bh%haco>Q9npRQO3g%l1&6yi
zMZHt&?BA8l(v=L|FAPIT#Qf#OY5C%wK&TJB{j9=~MZM-h0%OY`6jdQ8y!^-0ut%bw
z4Iw?$zZ_qcG(?X-3GG}z>m>|5;Pu3-QYT!4nGXsWUe7&=lfh|q8as-z50+1K*bz?8
zmr98~0s5JuP$|r0NWqv=#7UbZRhrr^1j0~>MlCp{pql?}K>mKYT$}3hY|%{hC+jFi
zIw-5iG!ozJoMqM$@BOvJq`6L^x<&h+ODxt5GN_fz6<&cFwF;a7H8nLrpPcd5$rL#C
zUgGPO9{_EtSb}DM66+9%sNUMTx><aP%1r<VU5&aZemrlKd<LJ(XudR<t!<(?z?dm0
z3bM~YmAugGmC^OGVVMcH@-q27p|~_4laU~q0R;kn3)Rp^Y91ay-h;9T%vt6ok2+2J
zJXrme&OczKyy-X$2sGk~^YcN{jrs9fFA#&nrMrzPfUgk)6-5LOm;!-gZ`vAg2<FJg
zl<rIIB*1D$+aH#Dc%;EZDHGITSWbcw=vji@@gwseL1v<<|HinW@GzIOBCklcdZA{m
zdtbGR0P)$CL~((*g7NdEpk5zF=UYzdZ|-?x>kNX^7{1vx>!al~1X!(Zds(hmA6k#c
z%IjIQvis;4Jc`u^UJa}o(8pY<FvZ3s7+!v<?$*<3r<ki=Z3y^M?Ek_`aJQCKUApIu
z^2z#uoj2PuCr{i5qn?uoJi4-<SHaF7#3J4HJ$5IpgsvJ33loyj1<^r0w`1+<{~Nrt
z*nnKV2Flup9wBWM=1e{oN<hapx|#RpBavW}CyfO9#<i@1JgeVvWES^|5@3pGD&`xx
ze*!`srwm4+AmI}Pd4upMbv=)ltgQ0oW2o7{(g8$K4&=Ez>l+&n+v=^P7PU@FqJ6aC
zIR>!*wuca0Dm7J?{Ju>nbm3(gPWIFluu@k6(N5!h=7$3CqJR)8upZc8peb7cgVn{Q
zjWs9sTyO5~LGdxn6^SDS9=~BS80M^5I`MQj`8$$H0m!43Ml=L~yoWZ_ymQUDAx!3r
zvE!swIbA!=Us>tA$r5Dl$kwo4yt-N$XL18+6o0tPu!{eFnS<Om)dSEs^c~!j-pn5O
z`3$i<`CJ@e%$jwyC3NdsnWj8th4seo|7dy(sH(E}fB1rkC@CNz9Yc3_w{&+S(jeU>
zptN*%r*wl#cXxMQy7S$BfA9aS8P_a_b-Cx9y`TNWC!V@RZ+v%9I|9)j_UK-be1n+F
zx%zDL`reL@cAVZ5mEXk<?he0`iV4L#zT0Qae4Hj*)xASSQ0=W1&ulNZA1kFEPSroT
z_ISD2=6!KIxH|E$e)_(jaUniK*^&l=hr#XRE+Kdc+G^d}EHDI{6S63mv8WIs<3jz)
zL}COvzGFcLk?b!Gi-0c-CoFWH7gz|=($}wF;{b<iSM$9((kr@hb-bwF#l=O|nZPYe
zW+W>+yV2-dk&+s^!QjsIY;*&O-1po>fbIPKui)ITyu5GZKyqPhb~e#y?`so~*c!b6
z4KND{jRkwK!61GBK6`@3pV8XN*QTE-lMfnTtPDgJvJi-3x>qpCz-y3+WviOyj_<j&
zkbK*tp$fIMwS@;c38{1p5c5pd_Z|{tvYMgW%=*lwW$I8RJb4L1-SzSE;`lfq-<D|M
zdUXI~y4ZCL`CE~quwaF64Pltb5Mt?=0t-w9k?@{c65}{y9H0hOVkZ6RLx8ZXdd992
zR7o5Aqci*`!6X1&Vv~;HiX^i=G!&gmA%OTXko@C1fo9AD1~PI%8n6s0IRFBMnhAiA
zT1^Mgs4;o!rRks>1_+123Q0^xz?Rvig+Gv95=~5Be}%hk_q#vn2NGPnN40gBh{z##
z|MI3EmaMmiEg3kt`Y1p8l_!Dq3l~5K&3v(Kd|0V%Dgj1*-vqFsRdge-;;Pn0PG0s$
zPRs=&Qz@Sfmpv(8F6QoIeGP{~GoL+QgHu)VYB;LZ-7z>m+wfk4$k*aq;@?7n$3qRG
z5lnL4W+<PA06IXPng#1uzATvwK8*UlERqezQpoXm-JSv_5Ort(T0t`DKeFV<U?ZRm
zQOU7RM?>n2&y|lHJYk9}bmmn!$xXK`mSC1rFgOsb)Gr!L_P!>%Xzo|AIp&AS!z&9$
zRuhEYGoya}FY-0>;1fa!#M`1ISn#PO{u^YGjv>=~{SD2%7Ax$(VQPRh8L`K$X>1<9
z*~xwF`Z$y~o3lD~klJzkcT<o>R{)nQhMfH)vM}12TWd}ZEcH&FB&*Nu8%xw}g(R_5
zdeM;I`)8Zs<4VEsQ5lnICSSfExPeT^(a{l%V6Qy*OQa=pi_jhC9z6c3(it-00JXqB
z#GOozJ3}xKzPqhhP+YUg;YIYQ(){-pw%mw04FYMDkx3Wtrqpme&nz2^@75#*92FlR
zU=RXW4yy`$Mr=Bu2>HIm;ula>XzRuWx8Ez7>>fAnV8&H}6)5q_UkE6&Q&V%N?nbd!
zl$1?=2u%_+&2T-ba_%rfK=K2sHSTZ39BPfc>X)F-$C3JZQYXZ6p?<GhG))+Z6OwRR
zdb++C(@BT3OPCxHgd>{>R~d?S7k!tiBA)m;0yArO^I&onK#cQ*sknPHyL>!yjI^{Y
zz<bd?il`X=#}Y~es_cEb%{QooTwGjivEQU##MsR2Z;iY$4m<Mo?0B3HfEX)zZ2xoE
zJ`hv}inrj(Arm?}I+B%@1!_&R<vNmhG5dRafDr^Xs?E%Jky7p#Pbp0Cm%H6el^?S<
ztzNZH!dd%BTiz23f~;s@#Q~|Rzqe<#bbwm`B#bNK<EH_N0$}ym9W_gZDq$>EC+Z*Q
z9W636IeE7kQhUvRX>*T0Vg#3z6>o-#^D#tyX@_5VfD<2wM=3%Gz`?@<>TRDte+Hpo
z^Oo}Fm3$0&)IXvE(1?L$l!>CkeAlF_ucPsBx-Ob5y93sbaTPi`IuP@eRe(A+=?Y|4
zB6E+8k0<kcazH&q@y@1ZE3dqTvK%<`g6_83Cv32?6>&b`Hre#b|I`E$z(4y<C)D2W
zn{hSi%Ey=j0U7Q5i7#wO5VntUn}*atGj?B5Lt{3JiVNa*&F*am=yk$xzK{8TymB2E
z&<7q;@dNO6Z7>j7idYmhK(0gn?1~1re&*&>B=d?=8Un^UPI`l&G24T@uO^=?JCl{v
z?1E`o*DTzX>T-)IKE8s1d_7@yqCfxZjLMWn>uE8&qT*Pm#fMFy#RW?E<k(r~$Z0b_
zaGgOA$D(z;Y)Xqds?pB^2ceo=<?W8qR<oTcI}@Eer#<3pk-a-WME#_M__Dme^c#Lv
z5>s4GRu+s0hy=WVdn_$hn8l*V0!yF-5Z9)m)uN(6|FK7&-Lp#OVWH2E$XP-2sYOBs
zQw_u@VqAP-w1razidgp3_}7fOxiF9jtIdhyKy9KOcuychlsY`~h3$$T(*mTU1o+@=
z7UwP9-r5@T$K4&x1iY2PKq7<yMQp5(${))&|7kIR;RrP8`=;AdC2MGo9JhxMaXI3Y
zsSUzUDa;SJRW`jS1pKdPDJh$DeKvcL7k<TM)MtlL0yg8%b^$2NB3EnrH5L%Tj+Mzr
z9<ehu1xT5_xP17QZz9?4;0I<GvPc5|FtM_dA>@GTjFAG&Wh*tojjXw5{xJlmq7wx_
zBc8ed<_i3{o`06GC`GJvW)08`eohbW85@fp-JPADW^XqF`4<>PhvtCRIUefj&`@}{
zF&J5MjNoM9%@Dng+yr$O2q5*mKuQmITzWA`G9&^iepj4LLF2CP1iY>*3S1}X3qcG`
z-#3xoya+*l$F}rC`T16-gNW_mX1i8~F`+!4A9qQwD%4gZp7>70hIkOyTNJ$Q;I$Rl
zZX*M+tB;<a9zar3tg4JDzPMzSX(JV*TCAD`#Q&1i81<(li&c$rt_SpL`*Nc!tj9z-
z^9Zc)GGjBEty;rI=!MX>;*r&8J^%~$JO3VF)I$X*E7CEwdB9`{%w>9dg0T|0BF*(%
z$MSQdQd9AkLp#^m>@)p^76cSy>*#@jtE%bN40O@)c=-wltV^E*^b86L3Q%wXbS_X*
z3Om1!>iwTC=J;zYgMM1FRSTeP2XsSI7Csg7w4*O5Ru#7mYc8fHC9$!CPLmreGUzC!
zRwqo4k7wr|>;C|eftI%M737<U5o@R27;X&t|EULqSM~I8bJb<+?%?3%<s~KtCmjP+
zIKlh=-;C66q`~f>cKG-9SF)an#X75-r%!K3NYZg@=}egLpI*U0=BfLUI9e-f`K0!W
zlG7z}knly(PM;<!qg+zl1skSB=5pE<5dttVF%5BM<{UHiZU0TJatCUonAtv<jW&|L
zSF$UvDE@>7H$Ie@!jN|@vsYp)2esKrliR)rnFyv$L3U&$iZEL3d%-sGY{e|90nc?#
zm|0w%*IF(UKcOAn5|S9eYg=e`=rC%zqv4^0PEx=ki^7U5`oAcZ$#?MtfnK1U^zCf3
z!Fn+vJ>dTK76O@=m>2@>HdD{Rz#OXdGf>)h89N9`c04#w8yOj0>ca_F*VLS>w3RAS
z?G5}>0|fmsAOe%EI5jf^c1D2P(+~ip;PJW)9WS>`GW^&0&8fhAkO{tXTre7TsxuD{
z?s2w(3G<F>TN!z=e{mcR;#VY%{)f3WKd|LJoW*R#KS`B;u^WW9hO_F2!f-Lt{^4P7
zl~}~S7I7^-rCeh=t*XD%B18~B48BgRbh+m_zmzgLMU8WN+bQ!i_Z?1_5iK+!a&);A
zCZoxn;U<I)C=?lzMK24`Rx<e|z#>WIr6mpMneY9kYi3aK24|=cu@D+4mDw~gh0wl@
z?oO8GM9jVc6WJ#^9iaM7%oK)lZOnv26JO$ghYRsbU<5Q%@|YeNn@f`hSqdvJt8DC#
zdr3__Fl<geo}iB1^yBas`fbXx-z<UbG9J(Q9uCYmEYQYg9;uRKDfBwho?ECfLxFS+
z;Htug#!P`m;223K$T==m^HpiU7_}Uv3E=$eR0q#`b7Mm)1{A=?RdYbEd2X(;$|eR&
zAXCugXl~i_I!{mD9_3jhf%`QaOw2%IhUW14!S9>|`o1>SmJR{ky5m`Q3G1uuh|)_Q
z7gwuU)8d>Pxbl7=&<Vr|%gf5ZiVuwMQb1l%EU)u~aeo`68_YC+Ju*_EE~h>d@aPH_
zi#L_p@_IyvfT8Bw-yPawRG9L~eD0eCa_$Hlhnu3xamvL*{ryVR@c?L+bZ(GtYG`N(
zWFnP|0j^+bZK(DmATR=%R$(d}sBZz#bcvU#ss5<m4>IpU4Ls1H$CJW>69y~J*~tkU
z^LUFD6N7P1avSg@zYzk)vI&6}bj0SwtylYxA3sF0fh_(Pw09C{-(q7iLUXjVv?A~T
zIOO2qKqg;#VYN;GNNcXF5O)3oYT@w6N|@}dte{f@zF350c4_ISm*u^HuFFPXlf?wL
zH9MWo=MLK8Dapym1U#4!MP=oHO)fn;1y6V|o>x(s1Ro48py>#@R^j1DL_|cu&>%vh
z#-tZT^nVwael(--(4Y52;?@B4>_Z}kZf!M^IoIpgDwut(8Q+N!43;LU*DziSzZurd
zH!PdKoD^w-uemyCcU&+vdA2$b<$M*n87HHrr(H7XMBT693LeDJ5O_DhCj+P@E*STB
z*kDk^&daL8=cz^aAF3+P7cb+v@X=1!d?n&?`>x~3pA>XCBP2I-5RJ+}Fy}7E;d|eA
zQg~9_Au1_p4t&LZAMeajVCi(1K-Hg{YjOrnB@hvDLef7In<!Ke)i9>_@$r#^ZrF1T
zvmw2EXIWsx8Wb217?`@Sy|y-{I`uK6(?a)DCk(7cVDqkmoQfNin5d!B!i6u-`Zm5c
zBoC`jS@<)58YVLcLeR_gyNg|r5bgUBg2e?|-pxy2Ri;yvFhMd)PfP20dun8COiHi}
z2Cfg_pYyv%aMHCsTran{t#DI)>o+E$3SDn>v`2ub?nGEYAejEbtn@G=<3y~kcS%hr
zJJzH>i!>T4dPll=zd2#Gu6t3^4+6QgkqrJpu$S|jfuL}_%+=+-O1TMQuu(!n0-Ufa
zXfe&qs9|F%tGx)w0a>N7nknE$fjp6>MVAm1^g7rGh*?EYrUGfcx+{^7LlX{M_&l|=
zf`T0m=LMjrCX5Eo)4RJn<chw{Gq6adV}Qo06`z@{xp~N@8<1}X-IqW;rzW7Vl_a}E
zx%X;<gTw)rN^NwjEPCxXt?$W4cQ&7sb?=NGd!7q=f1*S6f(ovu!fDSpqtC;7oVjQE
z9`)A$Vibhg)gef@%mxNi{BTz<5{EelfSB^~wt$@`sB(db3JHSoRFRTMHh5%;fF~6s
z0*-jJF2ox<vDA<7kStb(X-js3GQim#CjGu~M45PT?$+GQmn|i!usYW>^B<RLuDam;
zE`<)U;f0~w{2TZHVkhe_NBgXi*?O<#dPg{?mxhMu`1l@yUN+b-;Nbz21Dg1OkrDZa
zR6pYHid1=4K7G|Sa63D6%f6L4aw68cO4z*){z5{Y+3YA)g`#A8DY(da9|(lrLb4{A
z@Ypbah~Wf}PyPl0F#A2GxCI8^AsJTKKG#T3YyeGsXho4QrS`vM!u|K_VfC)oO}{Dw
zn)VblGsCRahOpf={ih9vr7QgHm-~ch4H$MUE0N+ulro(P7ozv$!};WQA5xGY#F*~{
zGubRAGgBSk8y-I%Xpu@Citr9t>UHmXf@e}zcJwz05s0je?uLnx0;duHJ3vGXyZ3+L
zikuxyIs~(uC_;V^bX=h;PkBXizk#WHiE_CzvnCw|-r`cNs)gEr0mS$OFyb+mz(qnC
zU_aG4YJCIQ0wtE;nN_2~on2o`pl$UPbM00b*)<~p!OM&ceIU`TkSUN3<jO59z=FFS
z2w@8#2BB<xe7!^Fjy+apKAf2)*r>T#&ZV)KPv#$P$oe2u8}F}%NL0?53;3F$IUst<
zQUC)p21Z6-pv?w=SgJI`jD4ehYh(LdPL8)|i?BrpC8jK|%V%)mQhSYPrcQVFp#CXk
z?ENhS4J<nzV`wd_8qdl~Zk3G({Rt7i;bCcpv|jzKIv{#yu$_$#r#ey#c`pdJ6TxY_
z$^e-GZ~?m&-Tz&hnr_ESO+e9Rette46lIi@smaO7DJj*Obj_b@u91L#iiqk|J#?{j
zKt_+(wbLFSl4Ui+V7ir5#@pa{Qivtsl{Nk)gWp3+DNj3lNTFqwNA!t_ggu2*Psoch
zi%O>O2%pE{Z@DL9#@+`t-Bd~7I&i%F(DefYJUE*H=yq<1OBJvZvgTNvnMtUq3d^BN
z?4)O4h>VB;V)fvEG&eVojEoTQy5v1A?iJTz9n**d1d{hsohD(%a<!eSi2aQZ%=aDn
zrRSFa($3oDQrLv`L0Vt_cCpWACr#@sIs{@W3oD6Bhvgrj!M6w$TKZ!tY!+IZ3UWm9
zqiON-iA1y^bu!W%ET>-dIKkl7!8L<wMTQVS-e0OwxwEwe6n4Nl&&tXQTor%-OenzZ
zZ+vaS$jJf9h+i4k(>xQIFgxwxMZXyCP1iJ-2#sCMiN=J=^o}g6vREj6ZdUm+>&m0p
z_aMTn<FYo#9Hsx{W@jrfw0U-ab?ACD$NWQVV`oR=FdV*roLt&6HW6q&Z(7SWm!XL(
zMa1U5hsz!<Rxiby)$4Y^7mn88$Rle@vD54@dC3QG9GqbG>FD9lR)NAG$VrhO9328S
zlmun=q*4Y15+w4TfIv?}!~DyaFW`s)*cYDNlw<TSSnz=)@v5;Ok?(iR7=k2v=3Qgm
zjrG$0+<rGpcFd&qNo3iHf-=5pjI8PmaMo$&ZRcshKR38FWW)8&wS?-onq1QBQu^F^
zakD(k{K*Y*><uTf8~@f6CPkjY;(k6;uEQEhV9~@NqAx**)e&20LqL!Y5Vzx%|Dj8R
z7z%XU2u^f0G+ZBV9Ri3!pbbIBt9LzFiMuf8Z+4c{r|YtZgRLa=W;kBdQ@Iac^O^YJ
zMnsAb;O6e0P1Ss}Mw0pbiaUS;SuZtUaQpgXRgi>4ICnfLC4~kv6pzz(3Ux?Xnp{dN
z+4~J6L<r>Wn}+V#*w~_?ouC2+EqL}Z;EMR&rBq%{&PMmNb8zqz9WahiTLyB6{!%P#
zY)+1jxl{+>CXkPuLAIEgNt8OwKt<Kp_XQ_!yh4wcXMgu#x)gcHcFwvU3=+e-LEN-W
z(}Zqp#Kgsk70%+|;#x={OUF3f*@L73nnE4c{gK&N4CK?``~`y4-C@{3AQ{{?r9y@*
z+BY#$qkjXHSK>;|E}l=H8o_$cFQCZeU(WVq+>%olc@I~01Es1gA8dY#s!K$D{<=I7
z^){UT7;VCJ@Ve^WE=dK+{Xe2$3X__7(6$GPxL|h|k`-7G!9e^Z@&{VIs){X#&A1i(
zcSD$?XxJQ{zp8(euR%2d$vn{3q@N~}LKNmr<>ld-umdXyXo&+_m8InYKo-&elmg27
z+s<{*uR^}0q5<ACoq;(;>Hot5Fbuj6*z%CRnfNS>)-^eaAsn4P&ipJI+%7%=4@9RH
z8yKHbMt7|YCdmn(2S*0O<R5G(cER7=0EZG#p>642+&z9y({6FvpO~un{IvoGvQQY;
zGiC%q=1|7`4=W7VUjY3RWfheV-17rVg&_ZH^SH8%*8FW;wa{Q!8F32-0q#YU%<!P9
zbZ~IM6i-e{s<oOO`B&2n$)+CqZQcTniJPN&M_z#n(Hnick6Ak_`LwV_Ws98UZC<NI
zBwMlDonxlpfg@tkxq*ZQ3|D}bJji|k_~h#H5+NW7?Zb2<=5fdE!}vJ74bjgZlX=DR
z<;#dthk=`oV4zc1=PV_K00E>Oz@th^O5#0p$XNDR5^yXjC<~{=Dvg)fWiU$CU>L9>
zAI)G(6Y+a=#T<V0YvlyT$>}LhRL<h{@f8tEcxTAMeMI&J;pOEeP|<;b+>Ca=(bP#v
zO`ZId0FEs=dHMQZzp^l__Hx=5>K8mzfKdgwfOCPMpdbL^f{75&NyX)|&k76_6&LRY
zLk)U5I$&Smvi945>vl53h=0ZD{bZe^h99w3!^=3k&G=v4Xw*1V(O41$5e<tx+7Fi4
z%u1L~isaaE6MkvDlsiKY%r#=!sw96zDd?X(a!5|{U!#<}1_uRo1CZ-K7mgd<R>_9r
z^MwU-g-k4S!Txq{73x?eObBL>NJ~LO5Its!PQ6GgA2bx5-QzOy>B0LA(mID;Jo*IR
zf?X)E2AWaj;N#<CV`Bp`6&%&TuO65$^qx_x6oCB$e4HXAM6n0*7SZ8^1rd}$KbxGN
zNaA5CudopG<#MRBb#=?4r3rZ8zel7xv3Z%T>WcEbg6nT~L<Gt{mWs9cVNSvV!~%V*
zXHi`G=;y^k>R$XVt?L@&I+Ax5CDo)J$hy$Iy9=oo4WH`w|Kv%4_|x()2a|(txCVx0
zS!?K5)(LNT?;gV{>4@O|&z+3jAqez?J(g?_C<Mo24KF(19z|0EDwY|5CD`(^bc|~0
zH0a568^b_A34ub$N0m4T+Fqde1homs^1F<g(-pFY{WzGIaM->?*V}+SCL$uTb0yYs
z3iGj!V%v)^wj0>UgX+6oGjU$*5ef9{loHp_g#3pNe{8$EH-Cn%keWcwf1C<ivLI{)
zN~&FP8GP8;EbH8a$!II2zA$om{aQ%nF90cEg|Tm6jJ_<ja<i|uMl>)65K{$%Z5iPx
zr4t-UpG!qUah)VND$q(nlOwY7QTsFS$<b@6t*ZmxHXKl)RNyoQY`wt0AQQcNco;OT
z?u`OUjK%;M06Y~s%7Kh=db*yItgPkV*NM?1*FMi5#Dik0C;o<@nmX5EN*|=^G&>)|
zLKLYnaRQYDZ;`~A!F~Aoy_v;;fpwBIT6XFy^-n>9wLnhVQThq<;G<{5Q5=<XIx?@k
zY3I?@!QqqCxh|W+<uI5~Zjs1n`7!yukboi@-2PbhE9saJg|x445C33yRN3KfMF=Uy
z4FhVrHCRI6iXs2-E@(!LoSq(t37G@kZBQRhO~plu=gI*Q)t*zMP858uXqUSjM|&#h
z*FxYAL6a|2p-3GMU})q-tIb+*S4sV!!@|Q~d))K={7#F>tVOq*WeDdNe)z-B@_Zlh
z{KqSHT)2sk#6@<y?Vi|ni`DsxcXX*%c!oBQnKR{$y0)rQw}4g|zoY))a{;U7uaY6&
z0zSqDg7Jxw*R^)I(lJ$qqg<|UH#u!mP~CUO&?9{4(Gb8kEdU9(vIWS*0XhIQt%1hh
z7de;pzppt`q1)SPuG=yT@nIwBA`TQ7oeqG>bA2JJ04Hqi{WzqzwB=kk{ojB2+zJ=0
ztg7#t6jtpyqU^ShmGm`F=^oyOuJc6DXbBT5#tIXc`xQKnbTO*S&wCnMM-x!f8!$Wn
z&QX)Re&$KIA|lMZ^LhqEwb+=LIGO(-zbZO;dj^(<W@hTQE5nGtgHm2xsdqHP6$uJ!
zre!7E`xfZMRcZc1)AAf-uH@(DUI3vE;F0-5OgS19jrz9h8=tLB!dR29Qa5EOMohng
zieF9^=5(rPTBa#{%^qx&1#OD3{9HR?YWkmX@jp{`&DHzOWSUwFa&qq4sq>WmjM=x%
zPR#d)v*V<7o^vw)|C`v*>Je=k>+Mhb%}mT>@5I96$Sz(1Q}%Klfr$VI&e)TtsOTGr
zxPWKM&t%3cl+$&~%~tmFH#aBS@iPb!bbPH|5J>N6x+}sy-I#Zc@^<u%@Kcd5P#ytS
zoi;e~z$AThb5q4ucaoj8GkK`6_+|j{Ts&9;@D{P6dUwvMu#3lVBCY86E*FvyQ&Ol+
zC(@=ogQAGycLYA{?dReYC{s&ENaROzuHr^$BP&M8th{s9<q~<{!A=w7Z7C!N?_Uc`
zcM<u#VUE{;fA`goQ!5>hAoZBTi4T>+#CxjfgR@5Dv1a{wLmX*aX;j7hWSCyw*B1qq
z+4Lb6jy0?uY^Ti2L06?^s!$$xl0O1mm;==u2$7pWhn5ZtR5&SIPJ3tNsq(@k_d8ff
z0Yx6_x|dtb`6Mt|ntU0eLTCz96GhXQ(#DK5bR<**8|O#7@S5%}Qf+R++Ilv|C&ZP-
z)T4tn$7|D%`4>c{kTj#6tqIasLA9{|TSJh6nq?3B-DYQ<-BDgYMa>YUu=Y@-PNSt0
zNxA)49i5kUL63ns$}su1ISFZ&$l*mVN*)q{qPE-dZ8FJY4Jvj_n|9xGuK-ag^TzM<
zp8@nL6B+mG=~(t9ZDJZvP}0JD>cTwW^(a&2#aI@L1#BWggq$s;L5F}Vka~)9jG1lg
z^bRg$kQjmD&sRjK2jjZedZ27}wvts8`~-#Hzqmp`x71R;vI06>;|0WsL@N<Lel$H0
z8;HYrJI-e3O5X9Nn)uPt5A-_8b}c3PRI)HW*R6m3b7P<|OZGghIo$5JH>;<RP@X~Q
zF`P%Hb<XA9d#%?Wm3fA26C4?JurW^PIhvwry<g{0W8mE5pT8i78E=1?=iWfVA!g&D
z%{)kB?-EV&N)!eHvOP@k^j7;n%(aLHNM>4iK?!EQ1bj(8S1~Guip@)0v0zrKDe_A6
z9*4mDRib>hvb6Jb%|>A{HOmh;bC%(B-Jx_nCV1P;?R1|j<@MYUYt0rcSP+3=Ey$*J
zmOpYDr}-3)cS<^&!MacE?=h2&9)&*f_NiVRwvKaPpenp8#vTz-@<>48mtq|m1ew(N
zsjkR*&aB=i{gh|%Ev~x3H3ySsf%dOo&gb5aQ${@`JFY$z9SHw&hl5OC)hl8qC&#|6
zF6~H50lq)yGI?a5pLtwm&0&Ec5Ol7JEP(jmHab{B{%BgOrAaq4HY)INxb~-Z@GZli
z^eTz%9%g*(z5TbBm*ERKuOLzp5;Q-@#y`_E@j(a7f>gCNw6bFci9p%;Sa(oT3JDk>
zpB^Tffh|&#@oUw(Qt0!A0S6u21Vvw4YFs(97Po6c3pE~OjGpT^D}z4vPyFS2b`Fwx
z)n*|n`B_@N^taYUk{Ut}$9m;vvoV>k;atf3QV=Cw2wLJus_k8v-F#X|*+at*9XwIf
zOO+voS?%uwTh26J7*I4cRJGtxl38}~@c2cJC!dGzgd0wF?~{4!Z?mYIWKz_6zyF9}
zo+VL6p+Bs#w^+4ZAuIqO$IzTh(w$ZF(Pk%Pzy+YQ%KfETW+WQYgU@CE@-^U-6n5FX
zh&ffTMY{i6lv&?3LwB3<yqq-E(m!F<{&GK4e@3$4ejcefDLb@7l>as7<zscTK?e7K
zcg$2(^+6`HOtZ%5x+@zR(XAWF^;o0ir^?EEyPx}AWA<4fQo=mSK;PME_SgR8LvtPD
zrw9R;<8^G69G0l(^X_UyRxB?YF)Zsv%u-leKT_6B&Jcw~;Gv8da5XmEA=?uFaFO8t
zDw&aD8VMSx{)7)yX#7g3sN<-#phMc-bnnJL;BdR$;$9>{AiMIyj|Y10AJDa9{jsZ&
zgrnys3zwU3cK&XBviSgH%~0{MFfd|0&H{9HD_+_>&;y9A1zrRXQcEV#<A}qd_|H)p
zA`5Kx$M0eNYu|p0T)wd}==b=V<LOtZA=H;NYK|`>r371+UYDCRT<dsN$ASq9k(#w#
z^~S|rGrG_<P&&U&>R1jQO=q$Ey~eCkfO=d6<EN}{yI4PIcht%#5$t-rV!nJ6`@GPa
zt2<tF4kNCHUGT)@`_tw&jVV$6`@9Gm+uZ1l&p^6mehRFrnY=AM3A&@fIZT1s{BQlX
zm`!=9#osv}@~FYYm)ZDRlV9T3^|F}>Bv^N{HNO9Cw5)64z=qez&!XD#32B(FN58|@
z+uqfuXB<%$`9|yUI`X`G<~Ci<6=|x9AbAIY6lItHhmlvhUA^>Jm{>u7&V9=Y@l#eS
z4dWW!qA?lis1m>w|4@+rV0<p?5XFtR8)ek?vOD?su#1w`uJhBgCWpdqF(tkScKr0M
zlC5aL^>g`V^!NE!Uf~<Y^)5Jki*<UPF^RS5bOFT7RaNazZ~n07av;X`<MUfi0Xbpi
zk#5g#S6Rc1E3}y6N_oNh7h-6b!ape(Wt7N)8uy=Hi2{e@#n}K*IzH=iy=p~rdB&vl
zxcq9jpBPOlHH(gt^-78PEhLug66xp+s&GgVw9ut*fpZlOdzr5T6HxV5OuaB`mD=(@
zO9&JG=36nPPxX1g0=}SNWpqfbpOwEW?H+o5^2qOXE1M~e$Ki{e-D&;9K%`DYTSaFO
zjaM%9L$}iO%=DBc6$Bwy?=5G?p0OT%eh<S6#?vGw%;tx5P)PRnnr~(Kb4KFFP0!Q|
z*^$`23H*b@X`2S#4L*@%8-u{ni&$<6e}~a{tzyRyVVIvQ$CJU~ZE2TMqA*!Mb$PKg
z7?j$*8V~-S_apkOOs|n+`=tKz`jY@3!Vyf$UK}zzf^dz0xQ&Ioz<S*!_aW__7wT{K
z|3#Wc!WPS&gd=3nh9Gy%EI*pQMtAP=F|sm#04`7_(P(YPr8&J}UvxY7n%`3-D7ox2
zcuW3Ed~TycfFJ>%htJ%pC`pKA<YX@X&x6B&c~=;f?_JG<N+H9+HMHQ43|40dYh;4<
z@lEBaM!25|lhgU-EQcgo*We(ca5TR!!6^}6cz58;TN_NIP9WjgU^QE@Rb$Qv`XS>t
zdAI#b7>5mFf|@>i0r@bfkgIY6!N=!i(!o6&ii1lN=5dTRQK#1nISQC>PZ?I9-^;Y+
zvpVakFNFp@2ngEutoAdVJQA&XzvPC^`siS!yRzdTAWxa1A>_%8d2UqyEeTphI$=?g
zHx>%KK=O=y^!QVx2v2(U*6E&v^mHG!W3eNLfWz=0YL~+6_|TzCXmxIKF)&dHMsa72
zTd;661%2<BbdBYJ>Fm*fbN$Lz!nLwXuQ@iE&2B5w4mbt~X>KOk=(Djj1_j^O6C(hN
z<+qSI$J5HlSAv0INj6yJbS<vWWjcwrFZ^3C$XR`%0p_x-w5*~8NUtG=?e7seQKs$B
z&gBdk2IbvreeN|j6MY`OT;$OGbY3nOn;iYTxSc60#QV%;S4)1laNo3wHC26ChL6mx
z<_7{>^WUjgv_A4p2C}5TX-xa(_LV#yeGlDOoOBCpRyLrP;iTW28Zw1&_0(p&OwhMp
zApaMHe{;cObUb)(4e<miBSU^C(-9^Hnowmq?bz`>iKnNno#UGJF@*|C-gu%Wd)H-3
z{FY!k{7pbvIopp(6}a}xe*cCBCe+Zll!&u;r2MBPRp_4$gvO)$bw>TC_92?@WgWk|
zHs1XX0p|#)g#iW=pMPW74kGzRUbR+<2?-2TkB-59X3X#{7w%c~GrabO$0bO*zDhb+
zBh2Pt7~vR+^nyX5I*vZuoOpMmaPJ-z_<{D5%FV6%c-qw)I;1%bU;Z4i={#-17B9ki
zqXWjtdrKOkvX87-sC#dx*sDIZ)3Xu&+N&f%Qggq2EcU$7mP?&b;q8{h(Y*htIZ-(|
z+U<Y6f(3tCb94N!TJx{{0DwK%2zc_(#0as&lO~g&oZ$|;Kf{NSd7K4zZ`U-AF209L
zZ}`3@TQ7B!k1s)w6p;JLUcadn8kkcAx@7-Kw~W)h>5$S_$Sm4#l<7ws9qI3c8AK3x
zoshn=&Ax&tew^-X`N4T2DE?Ndmmys;>SLYV%X96&A$vB?%9#B1%d2&5cyHf5`-x(c
ze$EX${KuLcEJR#EtG-`^NC7``aQ=KY!3lsE?k!J`=%Qw40MFC7eB1`dz`~pLx3}Ma
z4{}c3N1_)EPS~z6UO}lCJd#i#O!ebYpQPgCj{9vIk4xmkmVY_-rZWpx&SImTIB}st
zR!S*UX0`}<e`*Roq;+GoxZP={eqVKR=J)A+<X+4?UhEjITYg|)@WG3_+w?@uhuyKg
zb6Dejv8qU4z7rBxd@Shpg8gg)$O5}#$w`>v-5HUN(3Mz9>!gym?k8XJ9uj1A2okrq
z_sJOtpYICUX<Y$wAD!sI=J-*s;pvI5ulfyQLRl0y0q@tY?@#0778(rPuY=Yibe~^{
z(BQUlwOu39rrm3E<dx%NF<Wf^+<dMeBxEG&8Q*NjWR)KHPdTo<Qv$><qqAG7*NU^#
z)T$G`v!iNe_7F&(_TK(@g;8*Tgp0%G-`vC!QjMqM9n*J{46AQWlw$is?Zm?I`>o9F
zN`U1;^hYx->9^hN>FuUhm)Y4}sZ$B84HpFHEBa9{37I=XpFHJvb>xYr_n*AJO^G*g
z$3Z>&zm-t+duwao`>41QqxHQ3rUM`&CYL4@#6Fp*Ynk~-IKe_iTZ56=hgJSm)6oGc
z(j*>YFT2NxH5dKfO*?tLs@<^T`nVh#n%bw8In&ijo(OzqQXUG5%>%REEP1ZLyZf4#
zH&S(vnW8$FpZNm=t(D;a%(8=a;$qT+r-L|w8~0PLznchCMzn%VXbBafH(U|zc^DOn
zVdy2Cb-G_4u8~hY1d=bCBRqHHp#p<h+*)tpb0pUjepN~r;gm(}snB{-_sMi_h#s+*
zixi@m*IYvGIXr7Xm;O{2wc9^}a$jlp1RL^OI1F~R9x;y3r*{X(dP*agr!9o(69P!n
z2|(KdWXj@7lo%u2&c_2G-HF<0!E-*-1;cP*5u~a%YVPrAwKKDnw-kcV@7w#h*$@~c
zjz|(&ESOq`mp3S;x^3qbrnmt)S@rfe0sGUF1@#SMg>e%@HRIi0qs%D5n)4<RwU0{Y
zHt$b902h^l3RmpeutlE+68M30+vP?LlFMAhZ0}vg!x~@PaY+e#zpaUOXj=5BB=WiM
z`1hE+^o7UXY2i;AQWq?f7ZAwz_ViN!)9EUh=!Ts(D;5+3&~B6bWa<4X7uP_ZkHxg7
zDST<9zN@)u@|cAe)sb4i=5QnFu2j;ss<Ll%uF2-pQvM<f`^w6)pz+2xG!0rLafYX>
z$%&-LtB#ig!@uKHhsS9zp1bX$#R{O$z0(#V8IC(A-uD**`RKmA-{Er1<JA01+34^+
zpT&TffzC`+Y@ziJxeAg{k-sBPr-uc$gdSJTlG6#FPZpZ@yYBKna$6We-!mJ@j6x(S
zg143g_-)oJ`d7X5oSaYmZ&PV>3lG<dEH*n|%Yuo2oQuWuz?F1V=oKA5g*;&vg|JCo
zVNu@`1<l{DeS8u<RePU;d^7lVsE99ADs5Mr50%eFV~QuMdb5hEL~W4VL5`d=^rX{}
z(dyKqojq>1%jR};o4xF9F-Q(aMzr*Oxz$1PvvB;6wv#C>R@n5#Y{-|mI?fFgn<#;M
zrjE4!S!5{d`NenPdp>wbb+@SJz!OM+w5u(4tIX^2N_|%?CVO~O-oZdDo3;*$MO#cF
z`Dn_paCA@!@z`@TC^^Duz6lA)Dq>#U+!RtBTy`U(kmwl}pJTlec3{9}^mejYNfR$t
z$V_(qc-Ac>h-rd@g9}(wGS1Gc3fMGlQCF_{l&O0uNpGI-Y01t)O@t`Z?p_91;LPsB
zpMq^Xj?Rm7`<>qjW%rD<J93~@b6`P!y50~HHatg;TM2#|?de|nR-xqoo$d%&-6QlI
zahpe6Hq=Qz&l!F>S>*g`Eu0P$P~AMFJMw-0Lx(q65Yv#?{nfie#S=?aH6LkSjnufg
zbm~#_zUZ}NK~cHu5sUp6=0App2wCMj?}!RA+@oU?i<lfy#<0KUnI#(_qbfvedhtHc
zvPK^D)w7G<cb)KK%noXG02)GsPL$;b@~kNhW1q;Rnq@nwDQs2=tmTeI%bui`d60DS
z;=D@f6jVrUUOzacR`E<>RycYx>HF@Bf<MpUc653%i#|ByNuR{o)xD9wXA~Ou%*O4t
zY15O&ty13Ix9^SDgWOiVd)eQ9ToAety7*&vmadG&;p*du_wI~FFXzb^jTDLI!Mqma
zQKaywBU^XgyjJiE_fC@&+c#mQ%XkrSs--Nkm8<32>ct!PP44JiRB~Z>W3GIi`8VJC
zCr?u-eV?y4^YeyYEhZ$tSEJpPU2sTEVI~_8>1*`dJ{_tO#bi#*Y1V{+#5K+CKItyx
z+BIe#%ojLYQEttqaW{@D`z`Z7FaDCY@(Pi?_wqZesU4*4NVTt2y_ntjiSYZYa)m*g
zMzZN9*OxUM#f0YPhVGk;1cH_JuIT*7ha0;NP50`X#?e9`G2{j9dMwJ0QWtx{e7}JH
z;)wuvPAH_rMobKW*x#eanb9M!j(z@H%e{YND@GJ1=utO2r4MF`5jpp_J4D#frUsu?
z6!RxvHALX1V5YTGSBl~<{PJRrJAm1E^>K5?O16{0`Uz&Wf>5Z#Mf={1?C<GE-r2Af
zyDx&4^F+uxZ)Kb1?hAA<607&vEi5XfU(7>DD)YD5aK(SteGq?$!mrri`nHe^>z75P
ztgppqtFx43_({m3!Zz8NN~;<=@I0<B0rAr(sSvjH$9v&ha32<fN@JR#YLqX3p0?{r
zaCUr+^zGx^ED~mvUYqqj_N(;%xDyR1YAR97L25e^CNA0$lgIhrQdh*?t83<|^^!$y
z*AuJ>dyY>^`kl2I9Zt__<Igy-9(OqU$oEHMP2nrY1qiTcX1E&@j+-eKy7<M1`;HH3
zvv+xitPk7;1gp;1*&YwW<(sV$?;)*n$t1H3oP?nJj!+;?y+NrjxIt_TBUG;aU{dk>
z=3pek<=z_N7xkLbdTL_s(1fzN6ys#&0<X>JdN)@Q0*`tyOU)!9`QmO)o6)y(^pe|9
z`X1JgA(?k$m$xYh{S!=Op6a;ST%H-NuXXPqDYeW2$p#H5zxGq76iGl1S6_wU$x35W
zZW9XRqe<=8q4UlLLZ5MIO4qBNr}P~4mn!lAA(;6}%2Jxgmi!J`B3#Q$TT*;b1kwr(
zvV`W0=*rk5K$jMl@5mJgE#JwMB1Oo(YPm)_S`t@vcxj2~rMNurGW)QIrLgJ|wuD}@
zIxEv5ycDmVU`DR*jC6EqEBZ`A1aqPj`&05I>29wU1r{P*^2w1fZ+Avs^A|<m2*i&E
z;#ZaIO-Y0Zrio@670V_Eu1FV5IXdXU$IDAsuI9RleYv%TwEaG(Yf5TTzUj3(oEb!)
zjSCXeWLq_iilp2$!-kGN{1$o-@tezDJVVrDMtWVHR=^YTYuI<ux2Se&J~jk(id<Sc
z4nJH=kHWJ`r%-Yh!Vn38;8<5>Ge20^AVao><2>WW>S}Tsp7d(25Ua;*1`1wRLD6@F
zb`Mq5G)7nI?2{F5*KF@h0Q}No4mR@Acw&<}{gDp}y|y)|(Ue^7aP&(qR*L!=TU!;+
zLnsWxF^cW$95IX=YQ++d4*6W!zxrlYH38GioJI{ZpHvgJx<EDC{h6`u?cSEDn8G{b
zV}1YTfex1QPjrLFW|%p`9$o2qK)wz_CCaK|F>;n?e((%kyZ;aF940X5i=iPgWNoM$
zu^PY=jz@YGhGSDrfYeXD<!qMUb)2@ovU(>Vaz`9(5g~Fe_n?Xi`d-^2(GMD<;P=V4
zZa~a%x6D=^(lztUKbzQqqD)DmSDxPo2?_bK+T56G4E$Cd!#;CT2t+2gtcY6D`RMj#
z#LS4##8%IeK&!Gu-9}HYT-U*LbHkDM#aV*c%Dkc^VOmuk_BYmZYU&Te%1=;oHFn;F
z(PeukVNx{<s&~-?ITIg&7;@TLRpuhmc%2@#kP<}olMJNy+f7b5?Euz!YFdDV+?7*i
z$JEpZjhKB66tTSY`vx5({$Sl#jXj97Hj{c6hwaXYzAs7^#@l0_x8EuYk+5qVAds-p
zzxr`ILbYbb)N(!Bfva@-?t+Ce<kIgH>ptZBoz7&;DQ1<HPY&%OXv&KJP^F;!WIAwQ
zL%Ty@`1Ii0P&<|=gE>}=0(&=>ZH5~_?0F5P9U3a4rnP|*)=|X=y@t5e&22j!wZg-H
zWXKQ@SeX@9N%keHv+ijYJ~!Y0<TKxYj#uk2&v|K75A#HcyuXz7jXHIhP{3N#X~;jk
zYY=qbPgdlwJ43)U(&C2*#eZ`|B~fdOXYGRTzU-OT_@=B)#Bqs??KZy^#??(g`L8ri
z#qZ+rOGK@V%<0=aeQq9()M1K7gL1*k;W59&QL$};O-NQziJB4w1m{rYk~oBkT4^Hj
z?Tlr%?GUKS41p$BMMH%jF|F-`J6ly!1(XQ_rvg!}dw1R>8YH2lSE$axtQ2gxhZymr
zT8xjpXq2;;W7s$w`GqjYr2<j?B%>vdh5atmkupbvE=e(B-!^5XwAhwvkjRJE^K-#3
z$)Wvi!WjxK#Yb+(y~{lsC`_G9a`5H#RP6u50(d+pDpGCJurrV&P<#?Y3l@a7Gm#9{
z-x-MzN29*?c&|c-CFprwR`)P5;pX%X*t*zN4Q3;)UuL?zS#6BD=OQu-k~diBA6`(T
zpiZiL<kV{G3=elUaDM=(1~N&cRU(3PpY0qKd#Yezt!s3EEkM0KYbW$w;4h>6bN9_u
zEA<ZF(AvOUXL}XfcDMiU&^&0_sM}x2Cp)5I6CSP>v%bQhWyLt!xO%7wzQv$|rIJxj
zOIG%ScbmuaaR0$PE%>&fw<TnqDyL*1jnlJAdajK76%sgugDtnt#s6FFS7f%Kh&svb
zHGvf}Ta_QEEMd-N8@~4^kyoF|qPy0w8fxT!1(U_}q_1ywb!a7<wSgjl<-5|Rr(eqQ
zfe8juvasQ+e38Dqj5~0SCw+7DC^Gbov@+|MaHXA_dUvzJ;FX+Jb@N8fQrbX8Y_R8I
z3Pgyy<dZEqrAl1>PB#py@?_~j93LRp)6nwPq_KQun_%7H)<P&jS2-4Usp&=&Qc?xw
zr<M#qk-5sA6r#Tueiqfi*1v0Nf#B{|I2pfkjZpupB(M5)0a<QNitQOX9hTMB_xBIi
zCkic2<NxJdcozS-BbfxIs*vATHHFJ9H>7i!G6CkPP1s~^YkOxE0o9cq_xB^4j~j)_
zrMq4ZO)~hn+%Y<K8Q~^7JVtIOcr6L^PZuX?0e`2g3I&Lke76z>)r%mLWf$1%gm2V`
z=Ugt8B^ICtSVM~reDg~^2;#V2@VVp{3&Wlk-D+29dhfgD`_YvImWe#e%UZV%`k{_K
zyhFxc!w8qCixW81?HRSRVFKZk-~YEd{Jh@NHni2Ha*<$D5xa3&{kbM<vtT4gIZ96F
z_Tj1Qu`s#8C*?i6;ohxP)ZjrV>#jT@yTf($AvNm7*BVBGC~2t*gv(Rw9bJ5Hes7cK
z_)SXIub;lM+P-kTlpa~O7f?`gx2XQJFMPE48(>f`I6&`V&w89#rxW27z{{0#u=o&u
zOyssTE8aoS7_AZQdwBHc=c{}1k=3lnsFz#?oEUQ8&cR{3#09)ofpUFfT%01~^E$b-
zfZOR>FOhE=zvuOOXUKo(Z`(fQE0-H|JOkqA0@?Q0yArihU@xgrt{oo}BQpKa7Z)FI
zv)o))U!Q%!B(9`Usn73vtQ!`N4*D@v-~|rtjrUlSUTwS%B{x`I!^j1K-Om*a)aLTk
zrq#1VPfBB-*>E8|P*@a#h^i8~qb+MAYiRpZf@NoRhvaR8r^N>Le-c9m4Mr<#bD5Ny
z$;tQEkww+}-Wy{?_kjE8xE`5X^RV!OHoe$iw>ffH*KxeSIk$+97X1EapxXW$(MM1G
zw(G`~CbY%&N#Fa{#h(6U<Oq86iXSfryXk^*hlxJth1r|fYV1!?L#=NFOEm#9^B8;U
zb0(Ltqkwvm7SVUFWG831X!CXH`9ln?uhTEq1_#7Q@4rjGIGaOKtJ*~i({+4I1b9}4
zd*_!XKJOhl`^Wld@tP>I7Lp>RC(`Q-xdFTQiMbY6@(rve!Fvr#C9}qm!FgEFu`^%W
zWf(ZjCY(Wm2@Dk{@wL^pE$)OM{h5mAO^~>=OJna^8sc-hr4wHJw%Oui$nN+b0t8c>
zfX{7gbybjzOq34GpxKF<oSdAMHFl-Wv_A$Q+*Q@pL%TPC3jOOhZ&sU~0aUS0n*gAn
zadDI5<Ha*2^Od11Z7qPNPx`Y=zb&Qg2Ez-8zxqDkg>)OUH`>a`3;;k9@G+Q=*$fgQ
zriTZOdrp@!=2WRB`$!LH?o*-UYP<K-vTm$Grs}|U`Ep_c<c?SXPI6WvLhp%ceS+GK
zcI7bYc}hvaerF4w?)hQVPSZx}tMBi;{QL+!&f`(Rm&@~WSV+<I0g&{jrl#Hp6MZ0&
z2q@-)fO6OVe9?@_3*g07CX&|HbYb29KyL2tO+e#RRu(CBxT3iD?Ynn8Kq)Zr&*{}w
zo9|0|RTVP?ustqMR(;q1205+ZM~Gz4{W9tV1R+3K=!qhFfi|qx56qFENg-ybmx9ku
zHlVyfIvgJ|!kxuX3iyTk&Y~*v30oPZTNam7{Q>V*fXn${Dt@blz~kDezDj&(DhwOx
z2p`Et|J``I<A$aJqk^yROMy0fXNbe3I*k=Id}zD$4_SmQJqI8b;OFOuLXgp-y~!m2
z9wT7u0544db{fqTG*CEM-9YOMK?MpR0F;}%3jEnifx<ob?<<8l06_o!`*$p*!i0ht
z_>S*?h#em@;-acxnwgkj;NdMA^1{L1-TrpqBPIPC*V*jJ@En*#Wntt2q~-c2$?h(t
zgJc&Lyxlu$-Q^03Zyo0Y3^;pI=jQef|B7=+(VlMgW9aDU<YnBE38TUK12kBnLgvE!
z{PZethXrg1pn<*D0C0!^Qunk_XEix7QQJ$5z+y3xx3~8t8BliSYRnY?$)-uy_9?7d
zv+VTL9-wXDupWYh1Ym&gLU=fv5ylNe8YO$mEaz>UkWwx1l}zVxTj9sJaTu)TsO*<g
zB>Q=Xl2q*sHC929{dC7joE{v0GTl$`T($Cn^ruw5a@$wz$jt~>fQn((zZD`yK*z*n
zv!43}R05>)U!LPY)9gQhB_kutEh++QR8p_SC1?Q=_O2`2d@y`|2n=-<6WAf*<ACMk
zda#y^?yF7m^u<gob=DN=EX_dOPPc_`ZoKwqDfU*p;U<<{3U@tc?(mD9ASS>V+b%Vd
zRAL*Y#KyAPuL%W-tREhFf_Hg8eh=&iK1oYg0n1lgfPn-MEHx_3#d@1bTY$JFlZ;&I
z@O=T#Y`Iv45`3`)Ttv`P8)U)>cQEPwSoOV@7MtDD*~+!h#@|R&v%yxF@R-_MP+JTB
zWD%{;Y7abUK;{8|0<0xf=)mys(|QOI5)u*@{)mQwcO5VS2S}TlsVN8qux>i;cJLGw
zM&X4G!D|wHJncN`c&X5vgoA_=^527L3^OzHD|jR!rCYggLLdVJWFOF;HWOo-#HbX~
znt|Z=cK-bvz_)ag&U>LWz~V;fC!`;kk+H(q@dzNHQ{_6GsHjmSqQPri?WX<cAj;t5
z1Cg{klio-Mz2?T27D+KNzz+m2l;KZbzoG>Y1F&<NfOj)sYuQ?_wzmO;G=G18fVv8c
zdJ7`I9vHUoDdmPIxk+q%aK`Y*$fT>KLv(U|pfVp6Kw?Bf$eS)MFZVv*`XNh!@h%7p
z3+t+_3ouN;eVVZ(|NQy0xR^R}^B?Gmm4zid0x-s_t^tPzNkc=U+<0(w6bz-NrfQ4B
zV#q;M>t44{;f9Cx_loN?t@eJnNi1SRdG^N%gb^@e4DSJf=C;NhLq&P1)1%&4ZpigG
zX#BG!tuV0-9$^$P)&wL$;2eI&0B{iL>3H}%hle`8Vadr_7ue!TQ!`jtlwgI6&sksh
zy(hbt>^0R_sJ%h$!j)6Edxrbt(CVp)lU3ApJ4v|gS;xLw-2h*qCqj${Kmk}ltpQ|(
zoTeM76_kI)hv3(^00i;|;@Z$+z7cmGm3NE&Z@L8s9+nb#-{03ek#TMV4}W~58q|mH
zJp*8MxW*QWOo9A+s!+y1b^m&`fPTnuX+8hEx91ykqV065EOn^eBFQ-3tZs=D49rBr
za*C`96<qe_L;l?VYTM4H2n6z3V15SFHzgwQrGJ`gOEi;nHq^CTpwnU2R-~ea=6K&<
zl1Q`c4^c-Ewj~o)(*^#Uqku_3Jxc0uWO#U9Iocv77qBILoPoXyAYNZz#}r=<kGj}f
zIqZx#`-ipxcW@5lx7jE~4lviXdiOUc3QALg|2~QN`QP0E(u{a9kZo2jRxbXC!wCeQ
zm5bwK@_|v(aZZ#*$5CyCa`9duJL8-Au{WGt^x{gF7J$WG8~RP?*%uxOXYFt=+G{6H
zCi@Gty%qWYeqppsK9KYQ2v}NSG$3aM1dPT)%R#gmPUD^gSURBCX9tdgm$r%upn0ua
z9E^lJ0d%hJ>E(a~E`W~x8(bRx3AnF50x<-Du>c@>f!?yt1Gu(;We6wibT|VAM+ZH-
zI>MGof$z(+k9?%7e5NXmK&z~55tu^v6-MeZWU`veD=SYFs}xRIUI86-V`Jk4IxUeH
zQvhu>&-9i+>jDM9wB;30uV&C~Je}4sppKVmh5{1<fXAO$`=(ML+vK#btfm(4G6=qh
zsE7!_)z<0&a&{{?yTM?!c=`Y^Ykpu**8fo?LD&){=i_V5tx@^9M7<sIIU434#O7|Q
z3H&V3G6ElIY-Sc@l$4*p4HUnmL6Q0O;Tj4EJD(Eh0D+Mp*a%Sh0Hk@e4-AYiD3&=$
zNsYmR0ifh7g*_;=T#>3mw@IX<yr`%VIBG<bhysEqVea^s1O`1z3yU8{b4ko5z=9zQ
ztf-Z@2mzPS#o5*AyTF>*pnvPd>tLzLk&>8LNGBOn7|qJ+5TM65xd1^q!sqE8Fa!AK
zrlzJwM*-LWhtJ{hu>jB%08vL-yAq!T6r_Sak08<9{>lwm1CVv#2@Qy}SBxLAus95V
zL++i`)e}-vQ-z4#Dmy`QnE>E=KzNtLE{waKl$n1+Hu7ohuMRums$<uiDHUk395
zy1J(Y#L?P!NS(ic_7g(TU+2S_X~8ms4j+(l0Gh64-95130r4^`Y?2e$9q%O?_-M7d
zF((dsC*}qS3D|t$)e^+sjdL`|B>Mh;yTa=Li2PxLBWcbJ97I7P<8F!C<?;#&v*5h4
z1w3|;z^pbpY=YNbhX<&)yC)~U8(lvDGJ1My3JwmAt(`$!3H)YY6asQ<7a<{`r+7KA
zd0qlH-6}KqpVHEmN7I%1@+$4d7XAV7gan*<T#k~Glig2N`0v5S0OnHof<CRX6nIRA
zzptT-fX-I;2IOK7;OS*&XD`qr1T;1^{WGBgIvN1=eJOz{E*XyF0P+KHQaVEy)LNKI
zg2?*;$4Ej#!slk$75uf09;Fjl=0O!d_79KZJdUjwGwB06V9^k^RC)&nXh{EE#+|{)
zh4SeOjSePaVqH1nVQ{d*nF8Jb8Ql|(TLT^}I26k@tBk=f0)ZQd<z0>X<$O_91^Rqm
zAi%qTBZNSUZU*38TL6J#EffS=L0<tHniB9ffEo~h#e+|uIN%0B6Uo_3!AIxTEOBh?
z*gM3U*SVz~!_Zk>#7(jHgXBcN2Ochx#3&)ZS2}^pXm^8y7QFW(3D}P)!A3xUt>M?N
z?~#!LUbnxRn(n}%y59tt{8tyFg1x=HMBHD=fmE<)2nwx61;{`DGKG^%gTn+Q#`omb
z9RHkrCkx~zgAZx|M5;5`%-mc@ON+fRM<Tb$`7oidaQB}*0+FD+os)};Em{k_yOmT7
z9Rq{U<=*=MViy+|@I)8%KK_4=ooPIj?c2x4T{lXiJIhTZWveV9#M~%Zk|Hf+mwkzn
zeFl?|J8MW}Eo2SZvrdwI-!mB5cVpkinEx^SKhNiR@w|Avql>QVI?wYse#bfI?|TGK
z!DVnJ8BE}Hb;{)@4B(SRP`4*JWvm{^96x>>ocZ9;1Lj6$Ma5jZIbfuG&(VUhb2qKE
zvC(~ZJp*j_;3IwgL`bYd<K%svqDdlK{pCfBeqQF(=+0*~ujLiwSZXr@@iwz*l8yy`
zr`%Mh@A-%D_vSB?>=TgVec3b-t@goH*t17tRBl3So-kG5iy-Y22_W`T*9<k8;*qZQ
zZf@5QA3=74r3zeBmm-hXI|xT1LFs{}<ERhiL`s53pZ<)bqOx+=^gxCC8rYww+Tx+H
zPR#rF7RJWcpMidbH%VG;76DB$BLt8!n1|4nH)-S~o0wVO7abwl1r49B7CIm07G1{p
zj_w<ZnUQ|(f1ZEfD17?J%d$c{e3IMm`}?et>&rM*d~iS2qf3wU^@m!6?reQcAf#;H
zpR1*#b~*W4vUo!ZFYWLpEh-HfXX(W;9%k2xaqpyb2m?mno<D)G9VKE^<+d^gl1TzA
z1~5*bxB(F!&Ql5|*q=Xtid(j|#7R4@jMY+9_aHO|-k9%~cU%6re+*pp6_u6n%&GA|
zzB$jcygCO%fHMUp0+D#HSp>_&$7c@@H`x>k3|1k@35|fHxSrd7UJhOR=?=-t8Zi#y
z8g(WnFOxF!b3E3F8?|cFv-z?52w+%4w%>Giy7fkWSaQ+qFc~-0-!wO$^gcoZ5L1vI
z0MmhkLx4Sprr&iIaB95(ssyY}?<D{<GAT)OsKZ%VO1g4ivnadb1LEr5y^gWkKxPqb
zH4Zp^0c(FkWNAz#gkyHFBsU}^WE1h}9y~>#dc2G`YtdcRd@se*6Y-kb9};7uyhwQ_
z4|`G!mxiRrmE$mi+>yP_7VRhF^O7Qh({~okIqKnMEsk4Zq~vnw`+oW!3IQ$#US3|>
zS3Md(00{$I0#_v@S{oXkJ3CLV?QDcH(9;9H=yRX|<55v~^WZOmL9w;0WjuO}fk8xA
zxUB)$(7*tPpXNH?!S;VCvMo^_Gtm$h$RgC$-;W9P+4HTYEe;8robFC_J$8~!+QBf|
z&Om>t=yIo;h88+#-bUxXzWxR6eyylSw^Q!^X)$^YI$j%b(GVxP(;hYz^B%^3#)f+E
zCJ6*ob?K!**2{Ag@u|N47IQNT3=UP~eIjMNJ+|+mPee7-Gcb%`F<9kfX{a{+{r&Uu
z9LCp7|DIZ0l!YPy?sTLQ&HCCF%it6or+A`BKbsnHPN2zH^xY~>d}X?)wnUkMy#q`B
z9*qf5e;Qq6R^wpaEgt#zYd?Q~H1Zd3qFisP#sA!t&tpM8)%=tc#d5!-^zH3!D6{LU
zt4AS90N7-IWs5MP!U(t>s}f&dUx0Rv4?e?YV_&f`Gdn}^0VNp(9R^OBLhs^ktE+JI
z{si@SLAzY6oF@vH4`;Mf{VPtlaPC#yeK=UcDVUwv^37q_<b{E8)Z0lSL9XT2{xnQ8
zsa%bP9{cT*a&3%wkd@Z&j{;CUe?M6)*=RY2YkSIeC_;DKu3Wte8FL7Nynz9y61Z8I
znV1Cizg-cbkl%uvCv%Ljh6k{XaI+^*MqyS$W-~B~5dF{s9&7j_Ky&1Bpn&3WI26<k
zV3lla%&1nPJAPc@=FRTzZYl%?z8%(+yN+=8zoMws?4@_HvJgL5TS<N=Q4?jyIk(Xz
zChB7>x=hZrqh>feRNW`Db4Txukk9^sOx{V4hUYdjGlMmZbe;or6pSkY)DFBvu9~kP
zgkS^`k*;k(kXL`dFoGq+VgEC6H^9ONi<hF4M6kd-p`<MMX7`Vfx}QJIzQ3e`lm#LX
z$`=qrFo(Ts^YN$a)D1S{@7kE~?+<?}l#&^QzeMiGlT?(P@$MX$rJ<eCqi;vfozhfP
zP-VuHr_B}L_EDmuQQ}ow)|Rxfo_oU=6D?sI8X;(bjN_?~;loL$=-Qgr{Tr*COnW|%
z-SNt+zwJ_#XZ|qZ+3R5Q`|xCGUsOg(p*!rTMM;H`zkD=0k@!zLiQt$(93n6iH$7i$
z|Ms0=D$*jhrfuE5mv{oP9GaS(JPrsdVB^We3rkD8gVI=fAQ_tt2JMN)2=HCO)8ys3
z2B;Z8TW&}}Ad;U)qMfgC5GvL}40B--R%Lnj+5r68sQ9T)=x0yBxq;S%jlJaLWP0iv
zZEUI^xtY>$SUf7Tp1-}*$~nPkBfFpfLIha$sP1aM1E2e<TJN}JYv0U2!R9?d0My!+
zEWMO)Cjg!PBWu#E(F_38A0y1Ag(RF!^cPB#{VSQ98{mfsg>0>~&#CGj0>zm#|7Me1
z$QmzUe*tmMm(iw3hL_h|VIe&;DY<x%#CdqVD{z7V#dIuB^ih4t=9&%L2XG$!&b`VG
zg5QxZC?MmK94--kc<mN6j0c4`u2TDunR!!BkaD{Sr?gJ1`w#Bd)z>{_X2lVC--k8A
zZJ?g>zpoAYA}G_vF!Ga!a^3o!<W9;iRUn&=j@z#%1sk)E&BWk%9#$3Z_{@oMlce^u
z(lcKNiT3Bj*r3Ah4>Na%hM(igA$t=w47<XIe|P_6<n73?Fi}a=Pkye=-v8b%A%e%c
zGfC;#g(NxYA^yA|$`k+20Na)CsHP?HH#Z;Uq1FvS453hE>S^fT-ZU&ChWdJX^D;RC
zO>OZ=3X1ojm<b2>(_|BK|HXm<4i*+IEv=~~GuV&-UGvY2iCX~fsIPp)4`Llmr{rFT
zY%N<TP2f;d;_$#op;c#+@64&9NajVBz@nnh^NE$oTahi4QTZ4JDy0zgI%s!j0|lB^
zNl~!{vJp@(XPv56n=a>K_o15|9wJI}b2A_V;Y^^{&Z>F8A2{u#*n_3)hh}htL2Tl%
z+#!7ps|XubfbPF)hby!n$1S=<H1-skW5Ri?t6%*|CpM+#EMb&z4ZlICzEK9d4(mC3
z8zo3J1meo|$Aa%6lRHE803D4YcvNZgth@Oz81n8e3c4Oab^`@{f>Y)p_@THGA+Q4-
z6gqockdo4-J;lW(()_~EFrm}R$u+&xiUmq9SnR-Y&+N6~_8oYte*5-udiq64CkKZW
z2*RNLh$}9HPxmt@_+)|Gp=qdrX{lInWB>gTdKt&5PXi)P-}?^j==gX6$U-R8Ai@Z9
zb90M{b$Q-?KMBq`Fn|Wup7&ke+Jct=cvOLl7ad#$RJ&Fuo54Ij{Ogy1FH{A}*47!w
z<x+zx4|iyk00<Fa29q4qWC8a6LZhs8Q%+{jWts+op#B#c39E&tbk7c{@Ti8H>(Dxo
zcAbI#V8004ZM>?`QzR$x*s)_-TG?Xk!N3e!l7ylh452{sg_h;%=>jq`DA@coK!VkV
zFM{&2A%_sM&aC}KnX)s%-@ji^>z>u!dXWXq6moN6->w1-t0kLIFa#uh6OR*Y1`jJ0
z6%~+8XPwGovvrGXVAl^i#}KNB_P`qJZ{ha^P%lKv!)@5>U^;hq-|?ek-JROsJ##RN
zl@%2gVIG2u+R`#@{uUI3pvD1823==*e*J<jZZmiX^SsBfy(9rZLco}~GQj_+oV)^Y
z)*#Us)JyN^^XOKKtX>)S)d_o_+u_gvq!{E0g)0juxxumskzC_h!N}KoE?70Mm6LS}
zELtNp<OTpy6&-C=FaTVt4HO*givgroqnR%JYdEhO-<{|V9IgyVuu7V+(Nu0%5)c~N
z-q@&9Vs8Lt=KQ}}Suj95kTes~OVUP0BlbS9_XXRIpA~3hTQ`51hPHkXulF1!yHdhJ
zhZ+<%!=S<7P`e>&mKUguV9Jk=mvvh<=2iq(FceF$;}fiGYGpM81u{MLO=IJajOVXi
zb8Cqf%{gnP1IU1!^mM3MfP(|TJvurFy!{n50;(W+^x-vejhK+bP#(djAzG2%kSMVe
zuflf(fmoKK4Cc$NgE&?ZdJs!IDffgR90a@}|4rLRsY6j3{e|mp#idi@6+-UQ^MbMo
zB=&==xP7g%%HNiVTCUbySobN^UHr&>p(=JGVp@KGZ3E3NjY}|kUs{qcD_KEQ=`*1A
zuYcSY%Vm)FsnG?K{6L%qKA7*(K8tc1&8J&@5-RMzKoe27oo?@mDqDBO4wG0nZW1{5
zeycFhl*hzxw-z*>Imxx1ZZ_5M9W8%)GSx%JAm*wmzA$l-Sljd=;V}49SA^8Xy<<bm
zGYlJi<+onHr^9=Uo5|90cqh3zN|v-dUaE(&VN{?+o9gO+3Ez(ySl>REQQ7gdYUt@A
z?(XxJo}iSY83nQ!684KHd2_&#MrBUcldo}0m<sXzdjCDc?dp2%E-CD8L-ehkrMak-
z@Cz#PtA-9AbP2mmW%pg5wvV#UW`A~c&gYRqw#5rx%(8QlsOr5BEkfj0R;0wj|Kib+
zLaR=_mP+{Lnu4n2ZmW@g3rXbrb?pu7M-R}qRtpK~1W%5O857l1-Z^CETKB?qMfc~p
zAWf=pMukmG(|Vna5-qpPV5||^Vx?513Ok$O6v?P?>&AstuV>LO7R)YEmd^JFc2ew%
zJ_-snMEp+nFA)!qxvpNMt#N+cR-&csm`I?!r{Ss9;ZkpQ9a?VRls+pz#vj&bj3vH!
zi@f?HmwaVy^m#uod*#UT_VO#bsQ9zkd#aQPb7c4k3KKbOVn_J4re`f*=s39f<2d&_
zJ)wSGj#FfVwE;IH6<bf+TX){tmJAp)`I(j{kaw1L+U<$`6Al>9rHi&cf1HoGJjkJ7
zB^!IB=nCb@S&^$qs-&vI%)Z&(>{WM@wPFfWE-5)TS9Hkk^EDi@$fjuKug0n`!sy)(
zto}T*8$X&#eaimXe#fIKWXmBL+dcl@cgVU{$)(Q5Z05Dr8W^0Tn|q1;7CydtO&`cH
zmxe8G<89nE_KfWv%hBoV)UwqtN$tbnE{n*qEoIH(K8-m&epa+gz2mm87*1IS)H2T3
zhv;IpkYp$0+S3}CZB27huWdi}o6F=GM#VA&f~rYX@y5fhG~$RDX;4I-Hk6Y!iCH22
zY|vS+#dd9^OuPbZt=HOORN}w`-pcL7*l`7?;&OdD)A-_$tc{H@hf@ibVw)7TaEtsd
z&n3-b8GEKgSRE}dAgGwjVkguJx_Wmgac{S@R};m}WR3dqaK5dk*RHtniB0oolkO0C
zoU<lUGS_R~e&aEgX57(;=pK~ok(He~wii50$B(Z+k<JDOj(&I9?<6JdEkwRfRZm-H
zc=NZc#nbf;<1L**GE-<keB@46s9{?|&w$5dsEP(Ye<UK=$?KE+9rfc~0pi$^YqHZ@
zgoK@|5`9~jebx`f2DiU;W1(GQ=>ANN)P<$t>wS?V3s3dkmV#5w8l)w62BR{j1%66!
ziAMFd7jJtOKbx=WcCl>F(hV59X0Tj`iSH-X*KwiGHXJ>es*XOUe_@K8H*$uY>$76f
zPdnBJ@+=))bHzwv(sIjHsHp*<l)v1{Q>Q4M{~ev&xDI}wcpwJ<e*x|P`jsm3pUi&v
V29DepPayCiRF%{fa}<nU{RfJlei#4%
literal 0
HcmV?d00001
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index b5f37051c6..eb0bc1c003 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -6,9 +6,13 @@ GAN implementation, just a demo.
from paddle.v2 as pd
import numpy as np
import logging
-
-X = pd.data(pd.float_vector(784))
```
+
+<p align="center">
+<img src="./dcgan.png" width = "90%" align="center"/><br/>
+The original GAN paper.
+</p>
+
# Conditional-GAN should be a class.
### Class member function: the initializer.
```python
@@ -21,7 +25,7 @@ class DCGAN(object):
self.z_dim = z_dim # input noise dimension
# define parameters of discriminators
- self.D_W0 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
+ self.D_W0 = pd.Variable(shape=[3,3, 1, 128], data=pd.gaussian_normal_randomizer())
self.D_b0 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.D_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
self.D_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
@@ -51,7 +55,7 @@ def generator(self, z, y = None):
G_h0_bn = pd.batch_norm(G_h0)
G_h0_relu = pd.relu(G_h0_bn)
- G_h1 = pd.fc(G_h0_relu, self.G_w1, self.G_b1)
+ G_h1 = pd.deconv(G_h0_relu, self.G_w1, self.G_b1)
G_h1_bn = pd.batch_norm(G_h1)
G_h1_relu = pd.relu(G_h1_bn)
From 4147c7f22836fe7ae7b0c6e616adaba0bbfe3b3a Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Wed, 4 Oct 2017 15:52:23 -0700
Subject: [PATCH 17/82] gan design modified
---
doc/design/gan_api.md | 82 ++++++++++++++++++++++++++++++++-----------
1 file changed, 62 insertions(+), 20 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index eb0bc1c003..b107f2fc00 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -1,20 +1,45 @@
-'''
-GAN implementation, just a demo.
-'''
-```python
-# pd for short, should be more concise.
-from paddle.v2 as pd
-import numpy as np
-import logging
-```
+# Design for GAN
+
+GAN (General Adversarial Net) is an important model for unsupervised learning and widely used in many areas.
+
+It contains several important machine learning concepts, including building and running subgraphs, dependency tracing, different optimizers in one executor and so forth.
+
+In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN as an example due to its good performance on image generation.
+
+## The Conditional-GAN might be a class.
+This design we adopt the popular open source design in https://github.com/carpedm20/DCGAN-tensorflow and https://github.com/rajathkmp/DCGAN. It contains following data structure:
+
+### DCGAN(object):
+which contains everything required to build a GAN model. It provides following member functions methods as API:
+
+### __init__(...):
+Initialize hyper-parameters (like conv dimension and so forth), and declare model parameters of discriminator and generator as well.
+
+### generator(z, y=None):
+Generate a fake image from input noise z. If the label y is provided, the conditional GAN model will be chosen.
+Returns a generated image.
+
+### discriminator(image):
+Given an image, decide if it is from a real source or a fake one.
+Returns a 0/1 binary label.
+
+### build_model(self):
+build the whole GAN model, define training loss for both generator and discrimator.
<p align="center">
<img src="./dcgan.png" width = "90%" align="center"/><br/>
-The original GAN paper.
+Borrow this photo from the original DC-GAN paper.
</p>
-# Conditional-GAN should be a class.
-### Class member function: the initializer.
+## Discussion on Engine Functions required to build GAN
+- Trace the ternsor and variable dependency in the engine executor. (Very critical, otherwise GAN can'be be trained correctly)
+- Different optimizers responsible for optimizing different loss.
+
+To be more detailed, we introduce our design of DCGAN as following:
+
+### Class member Function: Initializer
+- Set up hyper-parameters, including condtional dimension, noise dimension, batch size and so forth.
+- Declare and define all the model variables. All the discriminator parameters are included in the list self.theta_D and all the generator parameters are included in the list self.theta_G.
```python
class DCGAN(object):
def __init__(self, y_dim=None):
@@ -43,11 +68,16 @@ class DCGAN(object):
self.theta_G = [self.G_W0, self.G_b0, self.G_W1, self.G_b1, self.G_W2, self.G_b2]
```
-### Class member function: Generator Net
+### Class member Function: Generator
+- Given a noisy input z, returns a fake image.
+- Concatenation, batch-norm, FC operations required;
+- Deconv layer required, which is missing now...
```python
def generator(self, z, y = None):
-
- # Generator Net
+ # input z: the random noise
+ # input y: input data label (optional)
+ # output G_im: generated fake images
+
if not self.y_dim:
z = pd.concat(1, [z, y])
@@ -64,11 +94,14 @@ def generator(self, z, y = None):
return G_im
```
-### Class member function: Discriminator Net
+### Class member function: Discriminator
+- Given a noisy input z, returns a fake image.
+- Concatenation, Convolution, batch-norm, FC, Leaky-ReLU operations required;
```python
def discriminator(self, image):
+ # input image: either generated images or real ones
+ # output D_h2: binary logit of the label
- # Discriminator Net
D_h0 = pd.conv2d(image, self.D_w0, self.D_b0)
D_h0_bn = pd.batchnorm(h0)
D_h0_relu = pd.lrelu(h0_bn)
@@ -82,6 +115,9 @@ def discriminator(self, image):
```
### Class member function: Build the model
+- Define data readers as placeholders to hold the data;
+- Build generator and discriminators;
+- Define two training losses for discriminator and generator, respectively.
```python
def build_model(self):
@@ -92,8 +128,8 @@ def build_model(self):
self.faked_images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
self.z = pd.data(tf.float32, [None, self.z_size])
- # if conditional GAN
- if self.y_dim:
+ # step 1: generate images by generator, classify real/fake images with discriminator
+ if self.y_dim: # if conditional GAN, includes label
self.G = self.generator(self.z, self.y)
self.D_t = self.discriminator(self.images)
# generated fake images
@@ -106,6 +142,7 @@ def build_model(self):
self.sampled = self.sampler(self.z)
self.D_f = self.discriminator(self.images)
+ # step 2: define the two losses
self.d_loss_real = pd.reduce_mean(pd.cross_entropy(self.D_t, np.ones(self.batch_size))
self.d_loss_fake = pd.reduce_mean(pd.cross_entropy(self.D_f, np.zeros(self.batch_size))
self.d_loss = self.d_loss_real + self.d_loss_fake
@@ -113,8 +150,13 @@ def build_model(self):
self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_f, np.ones(self.batch_szie))
```
-# Main function for the demo:
+## Main function for the demo:
```python
+# pd for short, should be more concise.
+from paddle.v2 as pd
+import numpy as np
+import logging
+
if __name__ == "__main__":
# dcgan
From 79c8bb9e7acbe2bc91625e4a2e396994c4fef168 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Wed, 4 Oct 2017 16:02:07 -0700
Subject: [PATCH 18/82] gan design new version
---
doc/design/gan_api.md | 23 +++++++++++++----------
1 file changed, 13 insertions(+), 10 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index b107f2fc00..8521bc8bf2 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -6,6 +6,11 @@ It contains several important machine learning concepts, including building and
In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN as an example due to its good performance on image generation.
+<p align="center">
+<img src="./dcgan.png" width = "90%" align="center"/><br/>
+Borrow this photo from the original DC-GAN paper.
+</p>
+
## The Conditional-GAN might be a class.
This design we adopt the popular open source design in https://github.com/carpedm20/DCGAN-tensorflow and https://github.com/rajathkmp/DCGAN. It contains following data structure:
@@ -26,11 +31,6 @@ Returns a 0/1 binary label.
### build_model(self):
build the whole GAN model, define training loss for both generator and discrimator.
-<p align="center">
-<img src="./dcgan.png" width = "90%" align="center"/><br/>
-Borrow this photo from the original DC-GAN paper.
-</p>
-
## Discussion on Engine Functions required to build GAN
- Trace the ternsor and variable dependency in the engine executor. (Very critical, otherwise GAN can'be be trained correctly)
- Different optimizers responsible for optimizing different loss.
@@ -151,6 +151,10 @@ def build_model(self):
```
## Main function for the demo:
+Generally, the user of GAN just need to the following things:
+- Define an object as DCGAN class;
+- Build the DCGAN model;
+- Specify two optimizers for two different losses with respect to different parameters.
```python
# pd for short, should be more concise.
from paddle.v2 as pd
@@ -158,7 +162,6 @@ import numpy as np
import logging
if __name__ == "__main__":
-
# dcgan
dcgan = DCGAN()
dcgan.build_model()
@@ -167,8 +170,8 @@ if __name__ == "__main__":
data_X, data_y = self.load_mnist()
# Two subgraphs required!!!
- d_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.d_loss, )
- g_optim = pd.train.Adam(lr = .001, beta= .1).minimize(self.g_loss)
+ d_optim = pd.train.Adam(lr = .001, beta= .1).minimize(dcgan.d_loss, dcgan.theta_D)
+ g_optim = pd.train.Adam(lr = .001, beta= .1).minimize(dcgan.g_loss, dcgan.theta_G)
# executor
sess = pd.executor()
@@ -183,11 +186,11 @@ if __name__ == "__main__":
batch_z = np.random.uniform(-1., 1., [batch_size, z_dim])
if batch_id % 2 == 0:
- sess.eval(d_optim,
+ sess.run(d_optim,
feed_dict = {dcgan.images: batch_im,
dcgan.y: batch_label,
dcgan.z: batch_z})
else:
- sess.eval(g_optim,
+ sess.run(g_optim,
feed_dict = {dcgan.z: batch_z})
```
From 3db38fce8ba88bd68a51a5e0232d60eeac7c50d1 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Wed, 4 Oct 2017 16:14:08 -0700
Subject: [PATCH 19/82] gan design new version
---
doc/design/gan_api.md | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 8521bc8bf2..e0e3440d49 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -14,21 +14,18 @@ Borrow this photo from the original DC-GAN paper.
## The Conditional-GAN might be a class.
This design we adopt the popular open source design in https://github.com/carpedm20/DCGAN-tensorflow and https://github.com/rajathkmp/DCGAN. It contains following data structure:
-### DCGAN(object):
-which contains everything required to build a GAN model. It provides following member functions methods as API:
+- DCGAN(object): which contains everything required to build a GAN model. It provides following member functions methods as API:
-### __init__(...):
-Initialize hyper-parameters (like conv dimension and so forth), and declare model parameters of discriminator and generator as well.
+- __init__(...): Initialize hyper-parameters (like conv dimension and so forth), and declare model parameters of discriminator and generator as well.
-### generator(z, y=None):
-Generate a fake image from input noise z. If the label y is provided, the conditional GAN model will be chosen.
+- generator(z, y=None): Generate a fake image from input noise z. If the label y is provided, the conditional GAN model will be chosen.
Returns a generated image.
-### discriminator(image):
+- discriminator(image):
Given an image, decide if it is from a real source or a fake one.
Returns a 0/1 binary label.
-### build_model(self):
+- build_model(self):
build the whole GAN model, define training loss for both generator and discrimator.
## Discussion on Engine Functions required to build GAN
From 583c94e4e641297820910d15fbc604cd9281834b Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Thu, 5 Oct 2017 23:35:48 -0700
Subject: [PATCH 20/82] new gan
---
doc/design/gan_api.md | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index e0e3440d49..b7c0fab201 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -6,6 +6,17 @@ It contains several important machine learning concepts, including building and
In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN as an example due to its good performance on image generation.
+| important building blocks | People in Charge |
+|---------------------------|-------------------|
+| convolution 2d (done) | Chengduo |
+| deconv 2d (missing) | Zhuoyuan |
+| batch norm (missing) | Zhuoyuan, Jiayi |
+| Dependency Engine (done) | Jiayi |
+| Executor (done) | Tony |
+| Multi optimizer | ? |
+| Optimizer with any para | ? |
+
+
<p align="center">
<img src="./dcgan.png" width = "90%" align="center"/><br/>
Borrow this photo from the original DC-GAN paper.
From 672f70ccba17a28af6842faede7c6349a399527b Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Thu, 5 Oct 2017 23:43:12 -0700
Subject: [PATCH 21/82] gan api
---
doc/design/gan_api.md | 28 +++++++++++++++++++---------
1 file changed, 19 insertions(+), 9 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index b7c0fab201..14e34a9839 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -6,15 +6,25 @@ It contains several important machine learning concepts, including building and
In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN as an example due to its good performance on image generation.
-| important building blocks | People in Charge |
-|---------------------------|-------------------|
-| convolution 2d (done) | Chengduo |
-| deconv 2d (missing) | Zhuoyuan |
-| batch norm (missing) | Zhuoyuan, Jiayi |
-| Dependency Engine (done) | Jiayi |
-| Executor (done) | Tony |
-| Multi optimizer | ? |
-| Optimizer with any para | ? |
+| important building blocks | People in Charge | Required |
+|---------------------------|-------------------|----------|
+| convolution 2d (done) | Chengduo | Y |
+| cudnn conv 2d (missing) | Chengduo | N |
+| deconv 2d (missing) | Zhuoyuan | Y |
+| cudnn deconv 2d (missing) | Zhuoyuan | N |
+| batch norm (missing) | Zhuoyuan, Jiayi | Y |
+| cudnn batch norm (missing)| Zhuoyuan, Jiayi | N |
+| max-pooling (done) | ? | Y |
+| fc (done) | ? | Y |
+| softmax loss (done) | ? | Y |
+| reshape op (done) | ? | Y |
+| Dependency Engine (done) | Jiayi | Y * |
+| Python API (done) | Jiayi | Y * |
+| Executor (done) | Tony | Y * |
+| Multi optimizer | ? | Y * |
+| Optimizer with any para | ? | Y * |
+| Concat op | ? | N (Cond) |
+| Repmat op | ? | N (Cond) |
<p align="center">
From b8a5b7f9bbe9c182ac70685005ef1662fd6d118b Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Thu, 5 Oct 2017 23:43:58 -0700
Subject: [PATCH 22/82] gan api
---
doc/design/gan_api.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 14e34a9839..d35309079f 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -10,8 +10,8 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
|---------------------------|-------------------|----------|
| convolution 2d (done) | Chengduo | Y |
| cudnn conv 2d (missing) | Chengduo | N |
-| deconv 2d (missing) | Zhuoyuan | Y |
-| cudnn deconv 2d (missing) | Zhuoyuan | N |
+| deconv 2d (missing) | Zhuoyuan, Zhihong | Y |
+| cudnn deconv 2d (missing) | Zhuoyuan, Zhihong | N |
| batch norm (missing) | Zhuoyuan, Jiayi | Y |
| cudnn batch norm (missing)| Zhuoyuan, Jiayi | N |
| max-pooling (done) | ? | Y |
From 564b8c6cede75f844ba238a4573a6514d899a90d Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Fri, 6 Oct 2017 16:07:57 -0700
Subject: [PATCH 23/82] gan api
---
doc/design/gan_api.md | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index d35309079f..9864e8b7de 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -19,12 +19,12 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
| softmax loss (done) | ? | Y |
| reshape op (done) | ? | Y |
| Dependency Engine (done) | Jiayi | Y * |
-| Python API (done) | Jiayi | Y * |
+| Python API (done) | Longfei, Jiayi | Y * |
| Executor (done) | Tony | Y * |
-| Multi optimizer | ? | Y * |
+| Multi optimizer (woking) | Longfei | Y * |
| Optimizer with any para | ? | Y * |
-| Concat op | ? | N (Cond) |
-| Repmat op | ? | N (Cond) |
+| Concat op (done) | ? | N (Cond) |
+| Repmat op (done) | ? | N (Cond) |
<p align="center">
@@ -91,7 +91,8 @@ class DCGAN(object):
- Concatenation, batch-norm, FC operations required;
- Deconv layer required, which is missing now...
```python
-def generator(self, z, y = None):
+class DCGAN(object):
+ def generator(self, z, y = None):
# input z: the random noise
# input y: input data label (optional)
# output G_im: generated fake images
@@ -116,7 +117,8 @@ def generator(self, z, y = None):
- Given a noisy input z, returns a fake image.
- Concatenation, Convolution, batch-norm, FC, Leaky-ReLU operations required;
```python
-def discriminator(self, image):
+class DCGAN(object):
+ def discriminator(self, image):
# input image: either generated images or real ones
# output D_h2: binary logit of the label
@@ -137,8 +139,8 @@ def discriminator(self, image):
- Build generator and discriminators;
- Define two training losses for discriminator and generator, respectively.
```python
-def build_model(self):
-
+class DCGAN(object):
+ def build_model(self):
# input data
if self.y_dim:
self.y = pd.data(pd.float32, [self.batch_size, self.y_dim])
From 806796cea3e8bad3706f82bb47073a2313b09f3e Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Fri, 6 Oct 2017 16:10:30 -0700
Subject: [PATCH 24/82] gan api
---
doc/design/gan_api.md | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 9864e8b7de..d0f8b47ca3 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -98,18 +98,18 @@ class DCGAN(object):
# output G_im: generated fake images
if not self.y_dim:
- z = pd.concat(1, [z, y])
+ z = pd.layer.concat(1, [z, y])
- G_h0 = pd.fc(z, self.G_w0, self.G_b0)
- G_h0_bn = pd.batch_norm(G_h0)
- G_h0_relu = pd.relu(G_h0_bn)
+ G_h0 = pd.layer.fc(z, self.G_w0, self.G_b0)
+ G_h0_bn = pd.layer.batch_norm(G_h0)
+ G_h0_relu = pd.layer.relu(G_h0_bn)
- G_h1 = pd.deconv(G_h0_relu, self.G_w1, self.G_b1)
- G_h1_bn = pd.batch_norm(G_h1)
- G_h1_relu = pd.relu(G_h1_bn)
+ G_h1 = pd.layer.deconv(G_h0_relu, self.G_w1, self.G_b1)
+ G_h1_bn = pd.layer.batch_norm(G_h1)
+ G_h1_relu = pd.layer.relu(G_h1_bn)
- G_h2 = pd.deconv(G_h1_relu, self.G_W2, self.G_b2))
- G_im = pd.tanh(G_im)
+ G_h2 = pd.layer.deconv(G_h1_relu, self.G_W2, self.G_b2))
+ G_im = pd.layer.tanh(G_im)
return G_im
```
@@ -122,15 +122,15 @@ class DCGAN(object):
# input image: either generated images or real ones
# output D_h2: binary logit of the label
- D_h0 = pd.conv2d(image, self.D_w0, self.D_b0)
- D_h0_bn = pd.batchnorm(h0)
- D_h0_relu = pd.lrelu(h0_bn)
+ D_h0 = pd.layer.conv2d(image, w=self.D_w0, b=self.D_b0)
+ D_h0_bn = pd.layer.batchnorm(h0)
+ D_h0_relu = pd.layer.lrelu(h0_bn)
- D_h1 = pd.conv2d(D_h0_relu, self.D_w1, self.D_b1)
- D_h1_bn = pd.batchnorm(D_h1)
- D_h1_relu = pd.lrelu(D_h1_bn)
+ D_h1 = pd.layer.conv2d(D_h0_relu, w=self.D_w1, b=self.D_b1)
+ D_h1_bn = pd.layer.batchnorm(D_h1)
+ D_h1_relu = pd.layer.lrelu(D_h1_bn)
- D_h2 = pd.fc(D_h1_relu, self.D_w2, self.D_b2)
+ D_h2 = pd.layer.fc(D_h1_relu, w=self.D_w2, b=self.D_b2)
return D_h2
```
From a0767228bd70ff8809a71dc4f9273d9dfac2aa46 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Sat, 7 Oct 2017 00:39:08 -0700
Subject: [PATCH 25/82] merge InferShapeContext and ExecutionContext
---
paddle/framework/operator.cc | 8 +++---
paddle/framework/operator.h | 49 +++++++++++++++---------------------
2 files changed, 24 insertions(+), 33 deletions(-)
diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc
index 2ca838f838..2fca816f35 100644
--- a/paddle/framework/operator.cc
+++ b/paddle/framework/operator.cc
@@ -205,13 +205,13 @@ void OperatorBase::GenerateTemporaryNames() {
}
template <>
-const Tensor* InferShapeContext::Input<Tensor>(const std::string& name) const {
+const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
auto* var = InputVar(name);
return var == nullptr ? nullptr : GetTensorFromVar(var);
}
template <>
-const std::vector<const Tensor*> InferShapeContext::MultiInput<Tensor>(
+const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
const std::string& name) const {
auto names = op().Inputs(name);
std::vector<const Tensor*> res;
@@ -225,13 +225,13 @@ const std::vector<const Tensor*> InferShapeContext::MultiInput<Tensor>(
}
template <>
-Tensor* InferShapeContext::Output<Tensor>(const std::string& name) const {
+Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
auto var = OutputVar(name);
return var == nullptr ? nullptr : var->GetMutable<LoDTensor>();
}
template <>
-std::vector<Tensor*> InferShapeContext::MultiOutput<Tensor>(
+std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
const std::string& name) const {
auto names = op().Outputs(name);
std::vector<Tensor*> res;
diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h
index d7bc9c9ffb..af8989dc4f 100644
--- a/paddle/framework/operator.h
+++ b/paddle/framework/operator.h
@@ -57,7 +57,6 @@ inline std::string GradVarName(const std::string& var_name) {
}
class OperatorBase;
-class InferShapeContext;
class ExecutionContext;
extern const Tensor* GetTensorFromVar(const Variable* var);
@@ -169,10 +168,11 @@ class NOP : public OperatorBase {
}
};
-class InferShapeContext {
+class ExecutionContext {
public:
- InferShapeContext(const OperatorBase& op, const Scope& scope)
- : op_(op), scope_(scope) {}
+ ExecutionContext(const OperatorBase& op, const Scope& scope,
+ const platform::DeviceContext& device_context)
+ : op_(op), scope_(scope), device_context_(device_context) {}
const OperatorBase& op() const { return op_; }
@@ -278,31 +278,6 @@ class InferShapeContext {
out_tensor->set_lod(in_tensor.lod());
}
- private:
- const OperatorBase& op_;
- const Scope& scope_;
-};
-
-template <>
-const Tensor* InferShapeContext::Input<Tensor>(const std::string& name) const;
-
-template <>
-const std::vector<const Tensor*> InferShapeContext::MultiInput<Tensor>(
- const std::string& name) const;
-
-template <>
-Tensor* InferShapeContext::Output<Tensor>(const std::string& name) const;
-
-template <>
-std::vector<Tensor*> InferShapeContext::MultiOutput<Tensor>(
- const std::string& name) const;
-
-class ExecutionContext : public InferShapeContext {
- public:
- ExecutionContext(const OperatorBase& op, const Scope& scope,
- const platform::DeviceContext& device_context)
- : InferShapeContext(op, scope), device_context_(device_context) {}
-
template <typename PlaceType,
typename DeviceType = typename platform::EigenDeviceConverter<
PlaceType>::EigenDeviceType>
@@ -315,9 +290,25 @@ class ExecutionContext : public InferShapeContext {
}
private:
+ const OperatorBase& op_;
+ const Scope& scope_;
const platform::DeviceContext& device_context_;
};
+template <>
+const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const;
+
+template <>
+const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
+ const std::string& name) const;
+
+template <>
+Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const;
+
+template <>
+std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
+ const std::string& name) const;
+
class CompileTimeInferShapeContext : public InferShapeContextBase {
public:
CompileTimeInferShapeContext(const OpDescBind& op, const BlockDescBind& block)
From c0a34e1c645381bbf4c80d13b257f62c828662f7 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Sat, 7 Oct 2017 00:47:55 -0700
Subject: [PATCH 26/82] rename InferShapeContextBase to InferShapeContext
---
paddle/framework/operator.h | 6 +++---
paddle/framework/operator_test.cc | 2 +-
paddle/framework/shape_inference.h | 6 +++---
paddle/operators/accuracy_op.cc | 2 +-
paddle/operators/activation_op.cc | 4 ++--
paddle/operators/adadelta_op.cc | 2 +-
paddle/operators/adagrad_op.cc | 2 +-
paddle/operators/clip_op.cc | 4 ++--
paddle/operators/concat_op.cc | 4 ++--
paddle/operators/conv2d_op.cc | 4 ++--
paddle/operators/cos_sim_op.cc | 4 ++--
paddle/operators/crop_op.cc | 4 ++--
paddle/operators/cross_entropy_op.cc | 4 ++--
paddle/operators/dropout_op.cc | 4 ++--
paddle/operators/elementwise_op.h | 4 ++--
paddle/operators/fill_zeros_like_op.cc | 2 +-
paddle/operators/gather_op.cc | 4 ++--
paddle/operators/gaussian_random_op.cc | 2 +-
paddle/operators/lookup_table_op.cc | 4 ++--
paddle/operators/lstm_unit_op.cc | 4 ++--
paddle/operators/mean_op.cc | 4 ++--
paddle/operators/minus_op.cc | 2 +-
paddle/operators/modified_huber_loss_op.cc | 4 ++--
paddle/operators/mul_op.cc | 4 ++--
paddle/operators/multiplex_op.cc | 4 ++--
paddle/operators/pad_op.cc | 4 ++--
paddle/operators/pool_op.cc | 4 ++--
paddle/operators/prelu_op.cc | 4 ++--
paddle/operators/rank_loss_op.cc | 4 ++--
paddle/operators/reduce_op.cc | 4 ++--
paddle/operators/reshape_op.cc | 4 ++--
paddle/operators/rmsprop_op.cc | 2 +-
paddle/operators/scale_op.cc | 2 +-
paddle/operators/scatter_op.cc | 4 ++--
paddle/operators/sequence_pool_op.cc | 4 ++--
paddle/operators/sequence_softmax_op.cc | 4 ++--
paddle/operators/sgd_op.cc | 2 +-
paddle/operators/sigmoid_cross_entropy_with_logits_op.cc | 4 ++--
paddle/operators/smooth_l1_loss_op.cc | 4 ++--
paddle/operators/softmax_op.cc | 4 ++--
paddle/operators/softmax_with_cross_entropy_op.cc | 4 ++--
paddle/operators/split_op.cc | 2 +-
paddle/operators/squared_l2_distance_op.cc | 4 ++--
paddle/operators/sum_op.cc | 2 +-
paddle/operators/top_k_op.cc | 2 +-
paddle/operators/transpose_op.cc | 4 ++--
paddle/operators/uniform_random_op.cc | 2 +-
47 files changed, 82 insertions(+), 82 deletions(-)
diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h
index af8989dc4f..1e9ace9987 100644
--- a/paddle/framework/operator.h
+++ b/paddle/framework/operator.h
@@ -309,7 +309,7 @@ template <>
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
const std::string& name) const;
-class CompileTimeInferShapeContext : public InferShapeContextBase {
+class CompileTimeInferShapeContext : public InferShapeContext {
public:
CompileTimeInferShapeContext(const OpDescBind& op, const BlockDescBind& block)
: op_(op), block_(block) {}
@@ -405,7 +405,7 @@ class CompileTimeInferShapeContext : public InferShapeContextBase {
const BlockDescBind& block_;
};
-class RuntimeInferShapeContext : public InferShapeContextBase {
+class RuntimeInferShapeContext : public InferShapeContext {
public:
RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope)
: op_(op), scope_(scope) {}
@@ -603,7 +603,7 @@ class OperatorWithKernel : public OperatorBase {
});
}
- virtual void InferShape(InferShapeContextBase* ctx) const = 0;
+ virtual void InferShape(InferShapeContext* ctx) const = 0;
protected:
// indicate kernel DataType by input data. Defaultly all input data must be
diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc
index a0c17b41f2..a02f4668bc 100644
--- a/paddle/framework/operator_test.cc
+++ b/paddle/framework/operator_test.cc
@@ -113,7 +113,7 @@ class OpWithKernelTest : public OperatorWithKernel {
using OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {}
+ void InferShape(framework::InferShapeContext* ctx) const override {}
DataType IndicateDataType(const ExecutionContext& ctx) const override {
return DataType::FP32;
}
diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h
index 74e0371e32..64aab16ae5 100644
--- a/paddle/framework/shape_inference.h
+++ b/paddle/framework/shape_inference.h
@@ -20,11 +20,11 @@ namespace paddle {
namespace framework {
// TODO(longfei): Once after both CompileTimeInferShapeContext and
-// RuntimeInferShapeContext get merged, we can rename InferShapeContextBase into
+// RuntimeInferShapeContext get merged, we can rename InferShapeContext into
// InferShapeContext so to replace the current InferShapeContext.
-class InferShapeContextBase {
+class InferShapeContext {
public:
- virtual ~InferShapeContextBase() {}
+ virtual ~InferShapeContext() {}
virtual bool HasInput(const std::string &name) const = 0;
virtual bool HasOutput(const std::string &name) const = 0;
diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc
index 82010bfb53..c5fb113e0f 100644
--- a/paddle/operators/accuracy_op.cc
+++ b/paddle/operators/accuracy_op.cc
@@ -22,7 +22,7 @@ class AccuracyOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Inference"),
"Input(Inference) of AccuracyOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Label"),
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index 66e9d2c401..5df875cd61 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -22,7 +22,7 @@ class ActivationOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
ctx->SetOutputDim("Y", ctx->GetInputDim("X"));
ctx->ShareLoD("X", /*->*/ "Y");
}
@@ -33,7 +33,7 @@ class ActivationOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("Y"));
}
};
diff --git a/paddle/operators/adadelta_op.cc b/paddle/operators/adadelta_op.cc
index bd8c93b4a1..cf1bca1658 100644
--- a/paddle/operators/adadelta_op.cc
+++ b/paddle/operators/adadelta_op.cc
@@ -22,7 +22,7 @@ class AdadeltaOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Param"),
"Input(Param) of AdadeltaOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Grad"),
diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc
index ea2ff3c503..a17747efb7 100644
--- a/paddle/operators/adagrad_op.cc
+++ b/paddle/operators/adagrad_op.cc
@@ -22,7 +22,7 @@ class AdagradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Param"),
"Input(Param) of AdagradOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Grad"),
diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc
index b3dd060fd7..3e9b0d82ba 100644
--- a/paddle/operators/clip_op.cc
+++ b/paddle/operators/clip_op.cc
@@ -22,7 +22,7 @@ class ClipOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of ClipOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -61,7 +61,7 @@ class ClipOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) should not be null");
diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc
index 1ffa02c8f9..235c4449ac 100644
--- a/paddle/operators/concat_op.cc
+++ b/paddle/operators/concat_op.cc
@@ -24,7 +24,7 @@ class ConcatOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE_GE(ctx->Inputs("X").size(), 1UL,
"Inputs(X) of ConcatOp should be empty.")
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -83,7 +83,7 @@ class ConcatOpGrad : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
ctx->SetOutputsDim(framework::GradVarName("X"), ctx->GetInputsDim("X"));
}
};
diff --git a/paddle/operators/conv2d_op.cc b/paddle/operators/conv2d_op.cc
index 5cc82944bb..6325d4248f 100644
--- a/paddle/operators/conv2d_op.cc
+++ b/paddle/operators/conv2d_op.cc
@@ -27,7 +27,7 @@ class Conv2DOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Input"),
"Input(Input) of Conv2DOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Filter"),
@@ -106,7 +106,7 @@ class Conv2DOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
auto in_dims = ctx->GetInputDim("Input");
auto filter_dims = ctx->GetInputDim("Filter");
if (ctx->HasOutput(framework::GradVarName("Input"))) {
diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc
index 040546f1a6..2b4c4b9c45 100644
--- a/paddle/operators/cos_sim_op.cc
+++ b/paddle/operators/cos_sim_op.cc
@@ -24,7 +24,7 @@ class CosSimOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
// notnull check
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of CosSimOp should not be null.");
@@ -98,7 +98,7 @@ class CosSimOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
// notnull check
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) must not be null.");
diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc
index 9b2305e90e..a1424993cc 100644
--- a/paddle/operators/crop_op.cc
+++ b/paddle/operators/crop_op.cc
@@ -25,7 +25,7 @@ class CropOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of CropOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -115,7 +115,7 @@ class CropOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) should not be null");
diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc
index 4b67887f36..708e80e96a 100644
--- a/paddle/operators/cross_entropy_op.cc
+++ b/paddle/operators/cross_entropy_op.cc
@@ -22,7 +22,7 @@ class CrossEntropyOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null.");
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null.");
PADDLE_ENFORCE(ctx->HasOutput("Y"), "Output(Y) should be not null.");
@@ -60,7 +60,7 @@ class CrossEntropyGradientOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null.");
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null.");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Y")),
diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc
index a669b5cf00..708ccfa0bf 100644
--- a/paddle/operators/dropout_op.cc
+++ b/paddle/operators/dropout_op.cc
@@ -24,7 +24,7 @@ class DropoutOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
PADDLE_ENFORCE_GE(ctx->Attrs().Get<float>("dropout_prob"), 0);
PADDLE_ENFORCE_LE(ctx->Attrs().Get<float>("dropout_prob"), 1);
@@ -70,7 +70,7 @@ class DropoutOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE_EQ(ctx->Attrs().Get<bool>("is_training"), 1,
"GradOp is only callable when is_training is true");
diff --git a/paddle/operators/elementwise_op.h b/paddle/operators/elementwise_op.h
index 3082f37422..66f1910a47 100644
--- a/paddle/operators/elementwise_op.h
+++ b/paddle/operators/elementwise_op.h
@@ -25,7 +25,7 @@ class ElementwiseOp : public framework::OperatorWithKernel {
protected:
using Tensor = framework::Tensor;
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of elementwise op should not be null");
PADDLE_ENFORCE(ctx->HasInput("Y"),
@@ -106,7 +106,7 @@ class ElementwiseOpGrad : public framework::OperatorWithKernel {
using Tensor = framework::Tensor;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc
index e164de6584..4c70b9a36b 100644
--- a/paddle/operators/fill_zeros_like_op.cc
+++ b/paddle/operators/fill_zeros_like_op.cc
@@ -22,7 +22,7 @@ class FillZerosLikeOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of FillZerosLikeOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Y"),
diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc
index fe305337cb..fb99c6c016 100644
--- a/paddle/operators/gather_op.cc
+++ b/paddle/operators/gather_op.cc
@@ -23,7 +23,7 @@ class GatherOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of GatherOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Index"),
@@ -51,7 +51,7 @@ class GatherGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
}
diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc
index 5cd2c7d2c0..ca7fb38505 100644
--- a/paddle/operators/gaussian_random_op.cc
+++ b/paddle/operators/gaussian_random_op.cc
@@ -43,7 +43,7 @@ class GaussianRandomOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasOutput("Out"),
"Output(Out) of GaussianRandomOp should not be null.");
auto dims = ctx->Attrs().Get<std::vector<int>>("dims");
diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc
index 929008fbcb..3f8d4ab857 100644
--- a/paddle/operators/lookup_table_op.cc
+++ b/paddle/operators/lookup_table_op.cc
@@ -22,7 +22,7 @@ class LookupTableOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("W"),
"Input(W) of LookupTableOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Ids"),
@@ -70,7 +70,7 @@ class LookupTableOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
auto table_dims = ctx->GetInputDim("W");
ctx->SetOutputDim(framework::GradVarName("W"), table_dims);
}
diff --git a/paddle/operators/lstm_unit_op.cc b/paddle/operators/lstm_unit_op.cc
index dad56731de..13a45ec246 100644
--- a/paddle/operators/lstm_unit_op.cc
+++ b/paddle/operators/lstm_unit_op.cc
@@ -22,7 +22,7 @@ class LstmUnitOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of LSTM should not be null.");
PADDLE_ENFORCE(ctx->HasInput("C_prev"),
"Input(C_prev) of LSTM should not be null.");
@@ -77,7 +77,7 @@ class LstmUnitGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("C")),
"Input(C@GRAD) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("H")),
diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc
index 2332c9546b..441543049f 100644
--- a/paddle/operators/mean_op.cc
+++ b/paddle/operators/mean_op.cc
@@ -22,7 +22,7 @@ class MeanOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of MeanOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -47,7 +47,7 @@ class MeanGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
}
};
diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc
index 7057dcbd6e..d7fd2f901b 100644
--- a/paddle/operators/minus_op.cc
+++ b/paddle/operators/minus_op.cc
@@ -26,7 +26,7 @@ class MinusOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of MinusOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Y"),
diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc
index 84212a2b3b..6522327fdc 100644
--- a/paddle/operators/modified_huber_loss_op.cc
+++ b/paddle/operators/modified_huber_loss_op.cc
@@ -22,7 +22,7 @@ class ModifiedHuberLossOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "X must be initialized.");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Y must be initialized.");
@@ -74,7 +74,7 @@ class ModifiedHuberLossGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "X must be initialized.");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Y must be initialized.");
PADDLE_ENFORCE(ctx->HasInput("IntermediateVal"),
diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc
index 3c8fe04d2e..ec0683d887 100644
--- a/paddle/operators/mul_op.cc
+++ b/paddle/operators/mul_op.cc
@@ -24,7 +24,7 @@ class MulOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of MulOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) of MulOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -97,7 +97,7 @@ class MulOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
diff --git a/paddle/operators/multiplex_op.cc b/paddle/operators/multiplex_op.cc
index a069127a19..a86685b6dd 100644
--- a/paddle/operators/multiplex_op.cc
+++ b/paddle/operators/multiplex_op.cc
@@ -24,7 +24,7 @@ class MultiplexOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Ids"), "Input(Ids) shouldn't be null.");
PADDLE_ENFORCE(!ctx->Inputs("X").empty(),
"MultiInput(X) shouldn't be empty.");
@@ -90,7 +90,7 @@ class MultiplexGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(!ctx->Inputs("X").empty(), "Input(X) should not be null.");
PADDLE_ENFORCE(!ctx->Outputs(framework::GradVarName("X")).empty(),
"Output(X@Grad) should not be null.");
diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc
index 15aa05f266..2f26ada85e 100644
--- a/paddle/operators/pad_op.cc
+++ b/paddle/operators/pad_op.cc
@@ -24,7 +24,7 @@ class PadOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of PadOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
"Output(Out) of PadOp should not be null.");
@@ -98,7 +98,7 @@ class PadOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) should not be null");
diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc
index c29f51f056..ba3b5ed207 100644
--- a/paddle/operators/pool_op.cc
+++ b/paddle/operators/pool_op.cc
@@ -27,7 +27,7 @@ class PoolOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"X(Input) of Pooling should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -74,7 +74,7 @@ class PoolOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"X(Input) of Pooling should not be null.");
PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")),
diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc
index 1692464f28..166fe26824 100644
--- a/paddle/operators/prelu_op.cc
+++ b/paddle/operators/prelu_op.cc
@@ -26,7 +26,7 @@ class PReluOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput("Alpha"), "Input(Alpha) should not be null");
PADDLE_ENFORCE(product(ctx->GetInputDim("Alpha")) == 1,
@@ -63,7 +63,7 @@ class PReluGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) should not be null");
diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc
index 1ba22006f2..e0abbc4db1 100644
--- a/paddle/operators/rank_loss_op.cc
+++ b/paddle/operators/rank_loss_op.cc
@@ -25,7 +25,7 @@ class RankLossOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
// input check
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null");
PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null");
@@ -90,7 +90,7 @@ class RankLossGradOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null.");
PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null.");
PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null.");
diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc
index 3ef443d1c7..12081ee6f0 100644
--- a/paddle/operators/reduce_op.cc
+++ b/paddle/operators/reduce_op.cc
@@ -24,7 +24,7 @@ class ReduceOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of ReduceOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -58,7 +58,7 @@ class ReduceGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null.");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) should not be null.");
diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc
index a3c3fa2716..3cd54930a0 100644
--- a/paddle/operators/reshape_op.cc
+++ b/paddle/operators/reshape_op.cc
@@ -26,7 +26,7 @@ class ReshapeOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
// input check
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of ReshapeOp should not be null.");
@@ -94,7 +94,7 @@ class ReshapeGradOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) shouldn't be null.");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) shouldn't be null.");
diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc
index 8f61c7fdda..ada6f2bc3c 100644
--- a/paddle/operators/rmsprop_op.cc
+++ b/paddle/operators/rmsprop_op.cc
@@ -22,7 +22,7 @@ class RmspropOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Param"),
"Input(Param) of RmspropOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("MeanSquare"),
diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc
index e225aecc27..ac297da6b7 100644
--- a/paddle/operators/scale_op.cc
+++ b/paddle/operators/scale_op.cc
@@ -26,7 +26,7 @@ class ScaleOp : public framework::OperatorWithKernel {
: OperatorWithKernel(type, inputs, outputs, attrs) {}
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of ScaleOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
diff --git a/paddle/operators/scatter_op.cc b/paddle/operators/scatter_op.cc
index d15ba15153..fbea01a8db 100644
--- a/paddle/operators/scatter_op.cc
+++ b/paddle/operators/scatter_op.cc
@@ -23,7 +23,7 @@ class ScatterOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Ref"),
"Input(Ref) of ScatterOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Index"),
@@ -60,7 +60,7 @@ class ScatterGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
ctx->SetOutputDim(framework::GradVarName("Updates"),
ctx->GetInputDim("Updates"));
ctx->SetOutputDim(framework::GradVarName("Ref"), ctx->GetInputDim("Ref"));
diff --git a/paddle/operators/sequence_pool_op.cc b/paddle/operators/sequence_pool_op.cc
index bc4af2f704..06c00d31ea 100644
--- a/paddle/operators/sequence_pool_op.cc
+++ b/paddle/operators/sequence_pool_op.cc
@@ -22,7 +22,7 @@ class SequencePoolOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of SequencePoolOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -74,7 +74,7 @@ class SequencePoolGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Gradient of Out should not be null.");
PADDLE_ENFORCE(ctx->HasInput("X"), "The input X should not be null.");
diff --git a/paddle/operators/sequence_softmax_op.cc b/paddle/operators/sequence_softmax_op.cc
index 621779ab61..ea217ba459 100644
--- a/paddle/operators/sequence_softmax_op.cc
+++ b/paddle/operators/sequence_softmax_op.cc
@@ -22,7 +22,7 @@ class SequenceSoftmaxOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of SequenceSoftmaxOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -67,7 +67,7 @@ class SequenceSoftmaxGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Out"),
"Input(Out) of SequenceSoftmaxGradOp should not be null.");
PADDLE_ENFORCE(
diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc
index 31d491f130..2a6a162a02 100644
--- a/paddle/operators/sgd_op.cc
+++ b/paddle/operators/sgd_op.cc
@@ -22,7 +22,7 @@ class SGDOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Param"),
"Input(Param) of SGDOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Grad"),
diff --git a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc
index ede458e011..b6653e1cc7 100644
--- a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc
+++ b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc
@@ -24,7 +24,7 @@ class SigmoidCrossEntropyWithLogitsOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null.");
PADDLE_ENFORCE(ctx->HasInput("Labels"),
"Input(Labels) should be not null.");
@@ -53,7 +53,7 @@ class SigmoidCrossEntropyWithLogitsGradOp
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null.");
PADDLE_ENFORCE(ctx->HasInput("Labels"),
"Input(Labels) should be not null.");
diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc
index 2d197e3b1b..91391dc945 100644
--- a/paddle/operators/smooth_l1_loss_op.cc
+++ b/paddle/operators/smooth_l1_loss_op.cc
@@ -22,7 +22,7 @@ class SmoothL1LossOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "X must be initialized.");
PADDLE_ENFORCE(ctx->HasInput("Y"), "Y must be initialized.");
@@ -94,7 +94,7 @@ class SmoothL1LossGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
auto in_dims = ctx->GetInputDim("X");
auto out_dims = ctx->GetInputDim(framework::GradVarName("Out"));
diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc
index e353afee3e..4c131ed44d 100644
--- a/paddle/operators/softmax_op.cc
+++ b/paddle/operators/softmax_op.cc
@@ -22,7 +22,7 @@ class SoftmaxOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of SoftmaxOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Y"),
@@ -69,7 +69,7 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should be not null.");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Y")),
"Input(Y@GRAD) should be not null.");
diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc
index 42c1ba6fdf..5431a1657c 100644
--- a/paddle/operators/softmax_with_cross_entropy_op.cc
+++ b/paddle/operators/softmax_with_cross_entropy_op.cc
@@ -83,7 +83,7 @@ class SoftmaxWithCrossEntropyOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Logits"),
"Input(Logits) should be not null.");
PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null.");
@@ -128,7 +128,7 @@ class SoftmaxWithCrossEntropyOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Loss")),
"Input(Loss@Grad) should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Softmax"),
diff --git a/paddle/operators/split_op.cc b/paddle/operators/split_op.cc
index 5f4b5539af..d5dd4df2a2 100644
--- a/paddle/operators/split_op.cc
+++ b/paddle/operators/split_op.cc
@@ -24,7 +24,7 @@ class SplitOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of SplitOp should not be null.");
PADDLE_ENFORCE_GE(ctx->Outputs("Out").size(), 1UL,
diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc
index 5a0cb59600..cce4e527c3 100644
--- a/paddle/operators/squared_l2_distance_op.cc
+++ b/paddle/operators/squared_l2_distance_op.cc
@@ -22,7 +22,7 @@ class SquaredL2DistanceOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of SquaredL2DistanceOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Y"),
@@ -86,7 +86,7 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Gradient of Out should not be null");
auto out_dims = ctx->GetInputDim(framework::GradVarName("Out"));
diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc
index c701ee8dde..ffb0cb9211 100644
--- a/paddle/operators/sum_op.cc
+++ b/paddle/operators/sum_op.cc
@@ -22,7 +22,7 @@ class SumOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInputs("X"), "Inputs(X) should not be null");
auto x_dims = ctx->GetInputsDim("X");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
diff --git a/paddle/operators/top_k_op.cc b/paddle/operators/top_k_op.cc
index 5f22bf1df8..c954819912 100644
--- a/paddle/operators/top_k_op.cc
+++ b/paddle/operators/top_k_op.cc
@@ -22,7 +22,7 @@ class TopkOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"Input(X) of TopkOp should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc
index 0672f9342d..1101bbe3ef 100644
--- a/paddle/operators/transpose_op.cc
+++ b/paddle/operators/transpose_op.cc
@@ -24,7 +24,7 @@ class TransposeOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null");
auto x_dims = ctx->GetInputDim("X");
@@ -93,7 +93,7 @@ class TransposeOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null");
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
"Input(Out@GRAD) should not be null");
diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc
index 97b1d0bed4..e330877fc4 100644
--- a/paddle/operators/uniform_random_op.cc
+++ b/paddle/operators/uniform_random_op.cc
@@ -47,7 +47,7 @@ class UniformRandomOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase* ctx) const override {
+ void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasOutput("Out"),
"Output(Out) of UniformRandomOp should not be null.");
From adad8d9ed2cd722e6ac45b18596099b31fdb9929 Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Mon, 9 Oct 2017 11:20:09 +0800
Subject: [PATCH 27/82] Open WITH_TESTING option.
---
CMakeLists.txt | 4 --
paddle/capi/tests/CMakeLists.txt | 17 ++---
paddle/gserver/tests/CMakeLists.txt | 70 +++++++++++--------
paddle/gserver/tests/LayerGradUtil.h | 1 -
paddle/gserver/tests/test_ActivationGrad.cpp | 1 -
paddle/gserver/tests/test_BatchNorm.cpp | 1 -
paddle/gserver/tests/test_CRFLayerGrad.cpp | 1 -
paddle/gserver/tests/test_ConvTrans.cpp | 1 -
paddle/gserver/tests/test_ConvUnify.cpp | 1 -
.../tests/test_CrossEntropyOverBeamGrad.cpp | 1 -
paddle/gserver/tests/test_KmaxSeqScore.cpp | 1 -
paddle/gserver/tests/test_LayerGrad.cpp | 1 -
.../gserver/tests/test_SelectiveFCLayer.cpp | 1 -
.../gserver/tests/test_SeqSliceLayerGrad.cpp | 1 -
14 files changed, 48 insertions(+), 54 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7d549b864b..4783095194 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -94,10 +94,6 @@ if(ANDROID OR IOS)
endif()
set(MOBILE_INFERENCE ON)
add_definitions(-DPADDLE_MOBILE_INFERENCE)
-
- # TODO: Need Open the WITH_TESTING
- set(WITH_TESTING OFF CACHE STRING "Disable TESTING when cross-compiling
- for Android and iOS" FORCE)
endif()
set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING
diff --git a/paddle/capi/tests/CMakeLists.txt b/paddle/capi/tests/CMakeLists.txt
index 8208808b94..bb38ace628 100644
--- a/paddle/capi/tests/CMakeLists.txt
+++ b/paddle/capi/tests/CMakeLists.txt
@@ -4,11 +4,12 @@ add_unittest(capi_test_mats test_Vector.cpp
target_include_directories(capi_test_mats PUBLIC ${PADDLE_CAPI_INC_PATH})
target_link_libraries(capi_test_mats paddle_capi)
-
-add_unittest_without_exec(capi_test_gradientMachine test_GradientMachine.cpp)
-target_include_directories(capi_test_gradientMachine PUBLIC
- ${PADDLE_CAPI_INC_PATH})
-target_link_libraries(capi_test_gradientMachine paddle_capi)
-add_test(NAME capi_test_gradientMachine
- COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}/capi_test_gradientMachine
- WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/capi/tests)
+if(NOT MOBILE_INFERENCE)
+ add_unittest_without_exec(capi_test_gradientMachine test_GradientMachine.cpp)
+ target_include_directories(capi_test_gradientMachine PUBLIC
+ ${PADDLE_CAPI_INC_PATH})
+ target_link_libraries(capi_test_gradientMachine paddle_capi)
+ add_test(NAME capi_test_gradientMachine
+ COMMAND ${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}/capi_test_gradientMachine
+ WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/capi/tests)
+endif()
diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt
index de9b8e63df..fcee19415c 100644
--- a/paddle/gserver/tests/CMakeLists.txt
+++ b/paddle/gserver/tests/CMakeLists.txt
@@ -1,15 +1,17 @@
# gserver pacakge unittests
+if(NOT MOBILE_INFERENCE)
################### test_ProtoDataProvider ############
-add_unittest_without_exec(test_ProtoDataProvider
- test_ProtoDataProvider.cpp)
-
-# test_ProtoDataProvider will mkdir as same name,
-# so if WORKING_DIRECTORY is default directory, then
-# mkdir will get error.
-add_test(NAME test_ProtoDataProvider
- COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_ProtoDataProvider
- WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
+ add_unittest_without_exec(test_ProtoDataProvider
+ test_ProtoDataProvider.cpp)
+
+ # test_ProtoDataProvider will mkdir as same name,
+ # so if WORKING_DIRECTORY is default directory, then
+ # mkdir will get error.
+ add_test(NAME test_ProtoDataProvider
+ COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_ProtoDataProvider
+ WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
+endif()
################# test_LayerGrad #######################
add_unittest_without_exec(test_LayerGrad
@@ -98,9 +100,11 @@ add_unittest_without_exec(test_KmaxSeqScore
add_test(NAME test_KmaxSeqScore
COMMAND test_KmaxSeqScore)
+if(NOT MOBILE_INFERENCE)
################## test_Evaluator #######################
-add_unittest(test_Evaluator
- test_Evaluator.cpp)
+ add_unittest(test_Evaluator
+ test_Evaluator.cpp)
+endif()
################ test_LinearChainCRF ####################
add_simple_unittest(test_LinearChainCRF)
@@ -131,27 +135,31 @@ if(NOT WITH_DOUBLE)
WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
endif()
+if(NOT MOBILE_INFERENCE)
############### test_RecurrentGradientMachine ###############
-# TODO(yuyang18): There is some bug in test_RecurrentGradientMachine
-# I will fix it.
-add_unittest_without_exec(test_RecurrentGradientMachine
- test_RecurrentGradientMachine.cpp)
-add_test(NAME test_RecurrentGradientMachine
- COMMAND .set_python_path.sh -d
- ${PADDLE_SOURCE_DIR}/python:${PADDLE_SOURCE_DIR}/paddle/gserver/tests
- ${CMAKE_CURRENT_BINARY_DIR}/test_RecurrentGradientMachine
- WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
-
-add_unittest_without_exec(test_NetworkCompare
- test_NetworkCompare.cpp)
-if(WITH_GPU)
- add_test(NAME test_NetworkCompare
- COMMAND .set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}/test_NetworkCompare --use_gpu=true
- WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
-else()
- add_test(NAME test_NetworkCompare
- COMMAND .set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}/test_NetworkCompare --use_gpu=false
- WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
+ # TODO(yuyang18): There is some bug in test_RecurrentGradientMachine
+ # I will fix it.
+ add_unittest_without_exec(test_RecurrentGradientMachine
+ test_RecurrentGradientMachine.cpp)
+ add_test(NAME test_RecurrentGradientMachine
+ COMMAND .set_python_path.sh -d
+ ${PADDLE_SOURCE_DIR}/python:${PADDLE_SOURCE_DIR}/paddle/gserver/tests
+ ${CMAKE_CURRENT_BINARY_DIR}/test_RecurrentGradientMachine
+ WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
+endif()
+
+if(NOT MOBILE_INFERENCE)
+ add_unittest_without_exec(test_NetworkCompare
+ test_NetworkCompare.cpp)
+ if(WITH_GPU)
+ add_test(NAME test_NetworkCompare
+ COMMAND .set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}/test_NetworkCompare --use_gpu=true
+ WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
+ else()
+ add_test(NAME test_NetworkCompare
+ COMMAND .set_python_path.sh -d ${PADDLE_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}/test_NetworkCompare --use_gpu=false
+ WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle)
+ endif()
endif()
diff --git a/paddle/gserver/tests/LayerGradUtil.h b/paddle/gserver/tests/LayerGradUtil.h
index 88e831f78b..e10a27eedf 100644
--- a/paddle/gserver/tests/LayerGradUtil.h
+++ b/paddle/gserver/tests/LayerGradUtil.h
@@ -15,7 +15,6 @@ limitations under the License. */
#pragma once
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
-#include "paddle/trainer/Trainer.h"
#include "paddle/testing/TestUtil.h"
using namespace std; // NOLINT
diff --git a/paddle/gserver/tests/test_ActivationGrad.cpp b/paddle/gserver/tests/test_ActivationGrad.cpp
index de93972a58..f4c2a07c44 100644
--- a/paddle/gserver/tests/test_ActivationGrad.cpp
+++ b/paddle/gserver/tests/test_ActivationGrad.cpp
@@ -17,7 +17,6 @@ limitations under the License. */
#include <vector>
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
-#include "paddle/trainer/Trainer.h"
#include "LayerGradUtil.h"
#include "paddle/testing/TestUtil.h"
diff --git a/paddle/gserver/tests/test_BatchNorm.cpp b/paddle/gserver/tests/test_BatchNorm.cpp
index 659eefa31b..38bcbb880d 100644
--- a/paddle/gserver/tests/test_BatchNorm.cpp
+++ b/paddle/gserver/tests/test_BatchNorm.cpp
@@ -17,7 +17,6 @@ limitations under the License. */
#include <vector>
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
-#include "paddle/trainer/Trainer.h"
#include "paddle/utils/GlobalConstants.h"
#include "LayerGradUtil.h"
diff --git a/paddle/gserver/tests/test_CRFLayerGrad.cpp b/paddle/gserver/tests/test_CRFLayerGrad.cpp
index df14449291..f010066ebc 100644
--- a/paddle/gserver/tests/test_CRFLayerGrad.cpp
+++ b/paddle/gserver/tests/test_CRFLayerGrad.cpp
@@ -16,7 +16,6 @@ limitations under the License. */
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
#include "paddle/gserver/layers/LinearChainCRF.h"
-#include "paddle/trainer/Trainer.h"
#include "LayerGradUtil.h"
#include "paddle/testing/TestUtil.h"
diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp
index 6035a866b4..5f2f966547 100644
--- a/paddle/gserver/tests/test_ConvTrans.cpp
+++ b/paddle/gserver/tests/test_ConvTrans.cpp
@@ -18,7 +18,6 @@ limitations under the License. */
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
#include "paddle/math/MathUtils.h"
-#include "paddle/trainer/Trainer.h"
#include "paddle/utils/GlobalConstants.h"
#include "LayerGradUtil.h"
diff --git a/paddle/gserver/tests/test_ConvUnify.cpp b/paddle/gserver/tests/test_ConvUnify.cpp
index e7325e0cc3..bcc10a6197 100644
--- a/paddle/gserver/tests/test_ConvUnify.cpp
+++ b/paddle/gserver/tests/test_ConvUnify.cpp
@@ -18,7 +18,6 @@ limitations under the License. */
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
#include "paddle/math/MathUtils.h"
-#include "paddle/trainer/Trainer.h"
#include "paddle/utils/GlobalConstants.h"
#include "LayerGradUtil.h"
diff --git a/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp b/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp
index c922237d33..477638426f 100644
--- a/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp
+++ b/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp
@@ -18,7 +18,6 @@ limitations under the License. */
#include <gtest/gtest.h>
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
-#include "paddle/trainer/Trainer.h"
#include "LayerGradUtil.h"
#include "paddle/testing/TestUtil.h"
diff --git a/paddle/gserver/tests/test_KmaxSeqScore.cpp b/paddle/gserver/tests/test_KmaxSeqScore.cpp
index 308abe6816..483e382f6d 100644
--- a/paddle/gserver/tests/test_KmaxSeqScore.cpp
+++ b/paddle/gserver/tests/test_KmaxSeqScore.cpp
@@ -18,7 +18,6 @@ limitations under the License. */
#include <vector>
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
-#include "paddle/trainer/Trainer.h"
#include "paddle/utils/GlobalConstants.h"
#include "LayerGradUtil.h"
diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp
index 090bde7b20..876a935fb0 100644
--- a/paddle/gserver/tests/test_LayerGrad.cpp
+++ b/paddle/gserver/tests/test_LayerGrad.cpp
@@ -21,7 +21,6 @@ limitations under the License. */
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
#include "paddle/math/MathUtils.h"
-#include "paddle/trainer/Trainer.h"
#include "LayerGradUtil.h"
#include "paddle/testing/TestUtil.h"
diff --git a/paddle/gserver/tests/test_SelectiveFCLayer.cpp b/paddle/gserver/tests/test_SelectiveFCLayer.cpp
index ab23d00a2c..1da935dfc9 100644
--- a/paddle/gserver/tests/test_SelectiveFCLayer.cpp
+++ b/paddle/gserver/tests/test_SelectiveFCLayer.cpp
@@ -24,7 +24,6 @@ limitations under the License. */
#include "paddle/gserver/layers/Layer.h"
#include "paddle/gserver/layers/SelectiveFullyConnectedLayer.h"
#include "paddle/math/CpuSparseMatrix.h"
-#include "paddle/trainer/Trainer.h"
using namespace paddle; // NOLINT
using namespace std; // NOLINT
diff --git a/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp b/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp
index e1d4ae1617..8e04ccd16a 100644
--- a/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp
+++ b/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp
@@ -15,7 +15,6 @@ limitations under the License. */
#include <gtest/gtest.h>
#include "ModelConfig.pb.h"
#include "paddle/gserver/layers/DataLayer.h"
-#include "paddle/trainer/Trainer.h"
#include "LayerGradUtil.h"
#include "paddle/testing/TestUtil.h"
From 5b862fedf1feb78b7dc63451e5219cef2fde33a3 Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Mon, 9 Oct 2017 11:31:13 +0800
Subject: [PATCH 28/82] remove debug log in interp_op.cc
---
paddle/operators/interp_op.cc | 1 -
1 file changed, 1 deletion(-)
diff --git a/paddle/operators/interp_op.cc b/paddle/operators/interp_op.cc
index 04bcb9ade8..fc8b9a11b8 100644
--- a/paddle/operators/interp_op.cc
+++ b/paddle/operators/interp_op.cc
@@ -58,7 +58,6 @@ class InterpOp : public NetOp {
{{"Out", {Output("Out")}}}, {}));
CompleteAddOp(false);
- LOG(INFO) << DebugString();
}
};
From 8f2c48cf4c46b3b3d1b58fb8fec13a9c95d2327c Mon Sep 17 00:00:00 2001
From: hedaoyuan <hedaoyuan@github.com>
Date: Mon, 9 Oct 2017 11:32:17 +0800
Subject: [PATCH 29/82] Fix Layer.cpp
---
paddle/gserver/layers/Layer.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp
index 075e8166ef..01f2aae6cf 100644
--- a/paddle/gserver/layers/Layer.cpp
+++ b/paddle/gserver/layers/Layer.cpp
@@ -15,11 +15,14 @@ limitations under the License. */
#include "paddle/utils/Util.h"
#include "CostLayer.h"
-#include "ValidationLayer.h"
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/Error.h"
#include "paddle/utils/Logging.h"
+#ifndef PADDLE_MOBILE_INFERENCE
+#include "ValidationLayer.h"
+#endif
+
DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
namespace paddle {
From 707d144c93aa6053cd02c58bc92bf1d7306c95c3 Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Mon, 9 Oct 2017 14:45:01 +0800
Subject: [PATCH 30/82] Unify Reduce functions and simplify register code
---
paddle/operators/activation_op.cc | 8 +++----
paddle/operators/activation_op.cu | 10 ++++-----
paddle/operators/reduce_op.cc | 34 +++++++++--------------------
paddle/operators/reduce_op.cu | 36 ++++++++-----------------------
paddle/operators/reduce_op.h | 6 ++++++
5 files changed, 33 insertions(+), 61 deletions(-)
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index 66e9d2c401..2afa8a68b0 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -285,11 +285,9 @@ REGISTER_OP(stanh, ops::ActivationOp, ops::STanhOpMaker<float>, stanh_grad,
#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, functor, grad_functor) \
REGISTER_OP_CPU_KERNEL( \
act_type, \
- paddle::operators::ActivationKernel<paddle::platform::CPUPlace, \
- paddle::operators::functor<float>>); \
+ ops::ActivationKernel<paddle::platform::CPUPlace, ops::functor<float>>); \
REGISTER_OP_CPU_KERNEL(act_type##_grad, \
- paddle::operators::ActivationGradKernel< \
- paddle::platform::CPUPlace, \
- paddle::operators::grad_functor<float>>);
+ ops::ActivationGradKernel<paddle::platform::CPUPlace, \
+ ops::grad_functor<float>>);
FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CPU_KERNEL);
diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu
index 93e9f1c694..7b7644519d 100644
--- a/paddle/operators/activation_op.cu
+++ b/paddle/operators/activation_op.cu
@@ -15,14 +15,14 @@
#define EIGEN_USE_GPU
#include "paddle/operators/activation_op.h"
+namespace ops = paddle::operators;
+
#define REGISTER_ACTIVATION_GPU_KERNEL(act_type, functor, grad_functor) \
REGISTER_OP_GPU_KERNEL( \
act_type, \
- paddle::operators::ActivationKernel<paddle::platform::GPUPlace, \
- paddle::operators::functor<float>>); \
+ ops::ActivationKernel<paddle::platform::GPUPlace, ops::functor<float>>); \
REGISTER_OP_GPU_KERNEL(act_type##_grad, \
- paddle::operators::ActivationGradKernel< \
- paddle::platform::GPUPlace, \
- paddle::operators::grad_functor<float>>);
+ ops::ActivationGradKernel<paddle::platform::GPUPlace, \
+ ops::grad_functor<float>>);
FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_GPU_KERNEL);
diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc
index 3ef443d1c7..87f66e1e93 100644
--- a/paddle/operators/reduce_op.cc
+++ b/paddle/operators/reduce_op.cc
@@ -168,36 +168,22 @@ namespace ops = paddle::operators;
REGISTER_OP(reduce_sum, ops::ReduceOp, ops::ReduceSumOpMaker, reduce_sum_grad,
ops::ReduceGradOp);
-REGISTER_OP_CPU_KERNEL(
- reduce_sum,
- ops::ReduceKernel<paddle::platform::CPUPlace, float, ops::SumFunctor>);
-REGISTER_OP_CPU_KERNEL(reduce_sum_grad,
- ops::ReduceGradKernel<paddle::platform::CPUPlace, float,
- ops::SumGradFunctor>);
REGISTER_OP(reduce_mean, ops::ReduceOp, ops::ReduceMeanOpMaker,
reduce_mean_grad, ops::ReduceGradOp);
-REGISTER_OP_CPU_KERNEL(
- reduce_mean,
- ops::ReduceKernel<paddle::platform::CPUPlace, float, ops::MeanFunctor>);
-REGISTER_OP_CPU_KERNEL(reduce_mean_grad,
- ops::ReduceGradKernel<paddle::platform::CPUPlace, float,
- ops::MeanGradFunctor>);
REGISTER_OP(reduce_max, ops::ReduceOp, ops::ReduceMaxOpMaker, reduce_max_grad,
ops::ReduceGradOp);
-REGISTER_OP_CPU_KERNEL(
- reduce_max,
- ops::ReduceKernel<paddle::platform::CPUPlace, float, ops::MaxFunctor>);
-REGISTER_OP_CPU_KERNEL(reduce_max_grad,
- ops::ReduceGradKernel<paddle::platform::CPUPlace, float,
- ops::MaxOrMinGradFunctor>);
REGISTER_OP(reduce_min, ops::ReduceOp, ops::ReduceMaxOpMaker, reduce_min_grad,
ops::ReduceGradOp);
-REGISTER_OP_CPU_KERNEL(
- reduce_min,
- ops::ReduceKernel<paddle::platform::CPUPlace, float, ops::MinFunctor>);
-REGISTER_OP_CPU_KERNEL(reduce_min_grad,
- ops::ReduceGradKernel<paddle::platform::CPUPlace, float,
- ops::MaxOrMinGradFunctor>);
+
+#define REGISTER_REDUCE_CPU_KERNEL(reduce_type, functor, grad_functor) \
+ REGISTER_OP_CPU_KERNEL( \
+ reduce_type, \
+ ops::ReduceKernel<paddle::platform::CPUPlace, float, ops::functor>); \
+ REGISTER_OP_CPU_KERNEL(reduce_type##_grad, \
+ ops::ReduceGradKernel<paddle::platform::CPUPlace, \
+ float, ops::grad_functor>);
+
+FOR_EACH_KERNEL_FUNCTOR(REGISTER_REDUCE_CPU_KERNEL);
diff --git a/paddle/operators/reduce_op.cu b/paddle/operators/reduce_op.cu
index 595127b858..d306e1a240 100644
--- a/paddle/operators/reduce_op.cu
+++ b/paddle/operators/reduce_op.cu
@@ -17,30 +17,12 @@
namespace ops = paddle::operators;
-REGISTER_OP_GPU_KERNEL(
- reduce_sum,
- ops::ReduceKernel<paddle::platform::GPUPlace, float, ops::SumFunctor>);
-REGISTER_OP_GPU_KERNEL(reduce_sum_grad,
- ops::ReduceGradKernel<paddle::platform::GPUPlace, float,
- ops::SumGradFunctor>);
-
-REGISTER_OP_GPU_KERNEL(
- reduce_mean,
- ops::ReduceKernel<paddle::platform::GPUPlace, float, ops::MeanFunctor>);
-REGISTER_OP_GPU_KERNEL(reduce_mean_grad,
- ops::ReduceGradKernel<paddle::platform::GPUPlace, float,
- ops::MeanGradFunctor>);
-
-REGISTER_OP_GPU_KERNEL(
- reduce_max,
- ops::ReduceKernel<paddle::platform::GPUPlace, float, ops::MaxFunctor>);
-REGISTER_OP_GPU_KERNEL(reduce_max_grad,
- ops::ReduceGradKernel<paddle::platform::GPUPlace, float,
- ops::MaxOrMinGradFunctor>);
-
-REGISTER_OP_GPU_KERNEL(
- reduce_min,
- ops::ReduceKernel<paddle::platform::GPUPlace, float, ops::MinFunctor>);
-REGISTER_OP_GPU_KERNEL(reduce_min_grad,
- ops::ReduceGradKernel<paddle::platform::GPUPlace, float,
- ops::MaxOrMinGradFunctor>);
+#define REGISTER_REDUCE_GPU_KERNEL(reduce_type, functor, grad_functor) \
+ REGISTER_OP_GPU_KERNEL( \
+ reduce_type, \
+ ops::ReduceKernel<paddle::platform::GPUPlace, float, ops::functor>); \
+ REGISTER_OP_GPU_KERNEL(reduce_type##_grad, \
+ ops::ReduceGradKernel<paddle::platform::GPUPlace, \
+ float, ops::grad_functor>);
+
+FOR_EACH_KERNEL_FUNCTOR(REGISTER_REDUCE_GPU_KERNEL);
diff --git a/paddle/operators/reduce_op.h b/paddle/operators/reduce_op.h
index ba3f3db81d..45043c440b 100644
--- a/paddle/operators/reduce_op.h
+++ b/paddle/operators/reduce_op.h
@@ -198,3 +198,9 @@ class ReduceGradKernel : public framework::OpKernel<T> {
} // namespace operators
} // namespace paddle
+
+#define FOR_EACH_KERNEL_FUNCTOR(__macro) \
+ __macro(reduce_sum, SumFunctor, SumGradFunctor); \
+ __macro(reduce_mean, MeanFunctor, MeanGradFunctor); \
+ __macro(reduce_max, MaxFunctor, MaxOrMinGradFunctor); \
+ __macro(reduce_min, MinFunctor, MaxOrMinGradFunctor);
From 3f874143fe62062607f341f2559840fc23f4bbd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=AD=A6=E6=AF=85?= <typhoonzero1986@gmail.com>
Date: Mon, 9 Oct 2017 14:55:03 +0800
Subject: [PATCH 31/82] fix grad debug event (#4536)
---
python/paddle/v2/event.py | 14 +++++++++++++-
python/paddle/v2/trainer.py | 9 +++++++--
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/python/paddle/v2/event.py b/python/paddle/v2/event.py
index e66bf67d79..a0ffd31c54 100644
--- a/python/paddle/v2/event.py
+++ b/python/paddle/v2/event.py
@@ -10,7 +10,8 @@ There are:
* EndPass
"""
__all__ = [
- 'EndIteration', 'BeginIteration', 'BeginPass', 'EndPass', 'TestResult'
+ 'EndIteration', 'BeginIteration', 'BeginPass', 'EndPass', 'TestResult',
+ 'EndForwardBackward'
]
@@ -73,6 +74,17 @@ class BeginIteration(object):
self.batch_id = batch_id
+class EndForwardBackward(object):
+ """
+ Event On One Batch ForwardBackward Complete.
+ """
+
+ def __init__(self, pass_id, batch_id, gm):
+ self.pass_id = pass_id
+ self.batch_id = batch_id
+ self.gm = gm
+
+
class EndIteration(WithMetric):
"""
Event On One Batch Training Complete.
diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py
index ca95ef13bd..076e755939 100644
--- a/python/paddle/v2/trainer.py
+++ b/python/paddle/v2/trainer.py
@@ -164,11 +164,18 @@ class SGD(object):
pass_type)
self.__gradient_machine__.eval(pass_evaluator)
self.__gradient_machine__.eval(batch_evaluator)
+ event_handler(
+ v2_event.EndForwardBackward(
+ pass_id=pass_id,
+ batch_id=batch_id,
+ gm=self.__gradient_machine__))
for each_param in self.__gradient_machine__.getNonStaticParameters(
):
self.__parameter_updater__.update(each_param)
cost_sum = out_args.sum()
cost = cost_sum / len(data_batch)
+ self.__parameter_updater__.finishBatch(cost)
+ batch_evaluator.finish()
event_handler(
v2_event.EndIteration(
pass_id=pass_id,
@@ -176,8 +183,6 @@ class SGD(object):
cost=cost,
evaluator=batch_evaluator,
gm=self.__gradient_machine__))
- self.__parameter_updater__.finishBatch(cost)
- batch_evaluator.finish()
self.__parameter_updater__.finishPass()
pass_evaluator.finish()
From fcfce48421650f983b484af9fe20d2e843dc042b Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Mon, 9 Oct 2017 19:02:24 +0800
Subject: [PATCH 32/82] follow coments
---
paddle/operators/CMakeLists.txt | 3 +-
paddle/operators/math/pooling.h | 42 +++++++++++++++++--
paddle/operators/pool_with_index_op.cc | 20 ++++-----
paddle/operators/pool_with_index_op.cu | 8 ++--
.../v2/framework/tests/test_pool_max_op.py | 21 +++++-----
5 files changed, 65 insertions(+), 29 deletions(-)
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index 49da132049..39af318ca5 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -75,10 +75,11 @@ function(op_library TARGET)
file(APPEND ${pybind_file} "USE_OP(reduce_sum);\n")
endif()
+ # pool_with_index_op contains several operators
if ("${TARGET}" STREQUAL "pool_with_index_op")
set(pybind_flag 1)
# It's enough to just adding one operator to pybind
- file(APPEND ${pybind_file} "USE_OP(maxPool2dWithIndex);\n")
+ file(APPEND ${pybind_file} "USE_OP(max_pool2d_with_index);\n")
endif()
# pybind USE_NO_KERNEL_OP
diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h
index d819e5986e..f15ddca69a 100644
--- a/paddle/operators/math/pooling.h
+++ b/paddle/operators/math/pooling.h
@@ -21,15 +21,26 @@ limitations under the License. */
namespace paddle {
namespace operators {
namespace math {
-//////////////////////
-#define FLT_MAX __FLT_MAX__ //
+#define FLT_MAX \
+ __FLT_MAX__ // It might need to be placed in another file, but I'm still
+ // wondering where to put it
+
+/*
+ * \brief Extracting simple operations from pooling.
+ * Both MaxPool and AvgPool need initial, compute and finalize operation.
+ * MaxPool initializes temp variable to the negative maximum to find the
+ * maximum value in the pooling field.
+ * AvgPool initializes temp variable to the zero to accumulate all values
+ * in pool pooling, and takes the average.
+ * MaxPoolGrad and AvgPoolGrad are gradient operations respectively.
+ */
template <class T>
class MaxPool {
public:
DEVICE inline T initial() { return static_cast<T>(-FLT_MAX); }
DEVICE inline void compute(T& y, const T& x) { y = y > x ? y : x; }
- DEVICE inline void finalize(T& y, const T& poo_size) {}
+ DEVICE inline void finalize(T& y, const T& pool_field) {}
};
template <class T>
@@ -37,8 +48,9 @@ class AvgPool {
public:
DEVICE inline T initial() { return static_cast<T>(0); }
DEVICE inline void compute(T& y, const T& x) { y += x; }
- DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; }
+ DEVICE inline void finalize(T& y, const T& pool_field) { y /= pool_field; }
};
+
template <class T>
class MaxPoolGrad {
public:
@@ -57,6 +69,20 @@ class AvgPoolGrad {
}
};
+/*
+ * \brief Getting pooling results, and calculating gradient.
+ *
+ * In pool2d, all tensors are in NCHW format. In pool3d, all tensors are in
+ * NCDHW format.
+ *
+ * In max pooling, it is possible that the pooling region has multiple maximum
+ * elements.
+ * In this case, we should compute the gradient of the first maximum element.
+ * This is different from average pooling. So we rewrite the max_pool_grad:
+ * MaxPool2dGradFunctor, MaxPool3dGradFunctor.
+ *
+ */
+
template <typename Place, typename PoolProcess, typename T>
class Pool2dFunctor {
public:
@@ -117,6 +143,14 @@ class MaxPool3dGradFunctor {
std::vector<int>& strides, std::vector<int>& paddings);
};
+/*
+ * \brief Getting max pooling results and corresponding max index, and
+ * calculating gradient.
+ * In sub-sampling-pooling, it is necessary to know max element index.
+ * In pool2d, all tensors are in NCHW format. In pool3d, all tensors are in
+ * NCDHW format.
+ *
+ */
template <typename Place, typename T>
class MaxPool2dWithIndexFunctor {
public:
diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc
index c51145b923..2e6a5f2555 100644
--- a/paddle/operators/pool_with_index_op.cc
+++ b/paddle/operators/pool_with_index_op.cc
@@ -17,8 +17,8 @@ limitations under the License. */
namespace paddle {
namespace operators {
-int OutputSizeMaxPool(int input_size, int filter_size, int padding,
- int stride) {
+inline int OutputSizeMaxPool(int input_size, int filter_size, int padding,
+ int stride) {
int output_size = (input_size - filter_size + 2 * padding) / stride + 1;
return output_size;
}
@@ -194,24 +194,24 @@ the input and ksize, strides, paddings parameters.
namespace ops = paddle::operators;
-REGISTER_OP(maxPool2dWithIndex, ops::MaxPoolWithIndexOp,
- ops::MaxPool2dWithIndexOpMaker, maxPool2dWithIndex_grad,
+REGISTER_OP(max_pool2d_with_index, ops::MaxPoolWithIndexOp,
+ ops::MaxPool2dWithIndexOpMaker, max_pool2d_with_index_grad,
ops::MaxPoolWithIndexOpGrad);
REGISTER_OP_CPU_KERNEL(
- maxPool2dWithIndex,
+ max_pool2d_with_index,
ops::MaxPoolWithIndexKernel<paddle::platform::CPUPlace, float>);
REGISTER_OP_CPU_KERNEL(
- maxPool2dWithIndex_grad,
+ max_pool2d_with_index_grad,
ops::MaxPoolWithIndexGradKernel<paddle::platform::CPUPlace, float>)
-REGISTER_OP(maxPool3dWithIndex, ops::MaxPoolWithIndexOp,
- ops::MaxPool3dWithIndexOpMaker, maxPool3dWithIndex_grad,
+REGISTER_OP(max_pool3d_with_index, ops::MaxPoolWithIndexOp,
+ ops::MaxPool3dWithIndexOpMaker, max_pool3d_with_index_grad,
ops::MaxPoolWithIndexOpGrad);
REGISTER_OP_CPU_KERNEL(
- maxPool3dWithIndex,
+ max_pool3d_with_index,
ops::MaxPoolWithIndexKernel<paddle::platform::CPUPlace, float>);
REGISTER_OP_CPU_KERNEL(
- maxPool3dWithIndex_grad,
+ max_pool3d_with_index_grad,
ops::MaxPoolWithIndexGradKernel<paddle::platform::CPUPlace, float>)
diff --git a/paddle/operators/pool_with_index_op.cu b/paddle/operators/pool_with_index_op.cu
index 8007fc7ccf..287657d4b1 100644
--- a/paddle/operators/pool_with_index_op.cu
+++ b/paddle/operators/pool_with_index_op.cu
@@ -17,15 +17,15 @@ limitations under the License. */
namespace ops = paddle::operators;
REGISTER_OP_GPU_KERNEL(
- maxPool2dWithIndex,
+ max_pool2d_with_index,
ops::MaxPoolWithIndexKernel<paddle::platform::GPUPlace, float>);
REGISTER_OP_GPU_KERNEL(
- maxPool2dWithIndex_grad,
+ max_pool2d_with_index_grad,
ops::MaxPoolWithIndexGradKernel<paddle::platform::GPUPlace, float>)
REGISTER_OP_GPU_KERNEL(
- maxPool3dWithIndex,
+ max_pool3d_with_index,
ops::MaxPoolWithIndexKernel<paddle::platform::GPUPlace, float>);
REGISTER_OP_GPU_KERNEL(
- maxPool3dWithIndex_grad,
+ max_pool3d_with_index_grad,
ops::MaxPoolWithIndexGradKernel<paddle::platform::GPUPlace, float>)
diff --git a/python/paddle/v2/framework/tests/test_pool_max_op.py b/python/paddle/v2/framework/tests/test_pool_max_op.py
index 17028c3bf6..f0f8aa6089 100644
--- a/python/paddle/v2/framework/tests/test_pool_max_op.py
+++ b/python/paddle/v2/framework/tests/test_pool_max_op.py
@@ -100,7 +100,8 @@ class TestMaxPoolWithIndex_Op(OpTest):
def initTestCase(self):
self.global_pool = True
- self.op_type = "maxPool3dWithIndex"
+ self.index = "max_pool3d_with_index"
+ self.op_type = "%s" % self.index
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 5, 5, 5]
self.ksize = [3, 3, 3]
@@ -111,7 +112,7 @@ class TestMaxPoolWithIndex_Op(OpTest):
class TestCase1(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
- self.op_type = "maxPool3dWithIndex"
+ self.op_type = "max_pool3d_with_index"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 5, 5, 5]
self.ksize = [3, 3, 3]
@@ -122,7 +123,7 @@ class TestCase1(TestMaxPoolWithIndex_Op):
class TestCase2(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = False
- self.op_type = "maxPool3dWithIndex"
+ self.op_type = "max_pool3d_with_index"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 7, 7, 7]
self.ksize = [3, 3, 3]
@@ -133,7 +134,7 @@ class TestCase2(TestMaxPoolWithIndex_Op):
class TestCase3(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = False
- self.op_type = "maxPool3dWithIndex"
+ self.op_type = "max_pool3d_with_index"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 7, 7, 7]
self.ksize = [3, 3, 3]
@@ -144,7 +145,7 @@ class TestCase3(TestMaxPoolWithIndex_Op):
class TestCase4(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
- self.op_type = "maxPool3dWithIndex"
+ self.op_type = "max_pool3d_with_index"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 5, 5, 5]
self.ksize = [3, 3, 3]
@@ -155,7 +156,7 @@ class TestCase4(TestMaxPoolWithIndex_Op):
class TestCase5(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
- self.op_type = "maxPool3dWithIndex"
+ self.op_type = "max_pool3d_with_index"
self.pool_forward_naive = max_pool3D_forward_naive
self.shape = [2, 3, 5, 5, 5]
self.ksize = [3, 3, 3]
@@ -166,7 +167,7 @@ class TestCase5(TestMaxPoolWithIndex_Op):
class TestCase6(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = False
- self.op_type = "maxPool2dWithIndex"
+ self.op_type = "max_pool2d_with_index"
self.pool_forward_naive = max_pool2D_forward_naive
self.shape = [2, 3, 7, 7]
self.ksize = [3, 3]
@@ -177,7 +178,7 @@ class TestCase6(TestMaxPoolWithIndex_Op):
class TestCase7(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = False
- self.op_type = "maxPool2dWithIndex"
+ self.op_type = "max_pool2d_with_index"
self.pool_forward_naive = max_pool2D_forward_naive
self.shape = [2, 3, 7, 7]
self.ksize = [3, 3]
@@ -188,7 +189,7 @@ class TestCase7(TestMaxPoolWithIndex_Op):
class TestCase8(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
- self.op_type = "maxPool2dWithIndex"
+ self.op_type = "max_pool2d_with_index"
self.pool_forward_naive = max_pool2D_forward_naive
self.shape = [2, 3, 5, 5]
self.ksize = [3, 3]
@@ -199,7 +200,7 @@ class TestCase8(TestMaxPoolWithIndex_Op):
class TestCase9(TestMaxPoolWithIndex_Op):
def initTestCase(self):
self.global_pool = True
- self.op_type = "maxPool2dWithIndex"
+ self.op_type = "max_pool2d_with_index"
self.pool_forward_naive = max_pool2D_forward_naive
self.shape = [2, 3, 5, 5]
self.ksize = [3, 3]
From a06f099d9f54b47ce4df7d1ae32c928fb8d7593e Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Mon, 9 Oct 2017 16:34:05 +0800
Subject: [PATCH 33/82] refine comment of interp_op
---
paddle/operators/interp_op.cc | 43 +++++++++++--------
.../v2/framework/tests/test_interp_op.py | 6 +--
2 files changed, 28 insertions(+), 21 deletions(-)
diff --git a/paddle/operators/interp_op.cc b/paddle/operators/interp_op.cc
index fc8b9a11b8..d02b01c3f3 100644
--- a/paddle/operators/interp_op.cc
+++ b/paddle/operators/interp_op.cc
@@ -30,27 +30,26 @@ class InterpOp : public NetOp {
"Input(Y) of InterpOp should not be null.");
PADDLE_ENFORCE_NE(Input("W"), framework::kEmptyVarName,
"Input(W) of InterpOp should not be null.");
- PADDLE_ENFORCE_NE(Output("MinusOut"), framework::kEmptyVarName,
- "Output(MinusOut) of InterpOp should not be null.");
+ PADDLE_ENFORCE_NE(Output("SubOut"), framework::kEmptyVarName,
+ "Output(SubOut) of InterpOp should not be null.");
PADDLE_ENFORCE_NE(Output("MulOut"), framework::kEmptyVarName,
"Output(MulOut) of InterpOp should not be null.");
PADDLE_ENFORCE_NE(Output("Out"), framework::kEmptyVarName,
"Output(Out) of InterpOp should not be null.");
- // MinusOut = X - Y
+ // SubOut = X - Y
auto x = Input("X");
auto y = Input("Y");
- auto minus_out = Output("MinusOut");
- AppendOp(framework::OpRegistry::CreateOp("elementwise_sub",
- {{"X", {x}}, {"Y", {y}}},
- {{"Out", {minus_out}}}, {}));
+ auto sub_out = Output("SubOut");
+ AppendOp(framework::OpRegistry::CreateOp(
+ "elementwise_sub", {{"X", {x}}, {"Y", {y}}}, {{"Out", {sub_out}}}, {}));
- // MulOut = MinusOut * W = (X - Y) * W
+ // MulOut = SubOut * W = (X - Y) * W
auto w = Input("W");
auto mul_out = Output("MulOut");
AppendOp(framework::OpRegistry::CreateOp(
- "elementwise_mul", {{"X", {minus_out}}, {"Y", {w}}},
- {{"Out", {mul_out}}}, {{"axis", 0}}));
+ "elementwise_mul", {{"X", {sub_out}}, {"Y", {w}}}, {{"Out", {mul_out}}},
+ {{"axis", 0}}));
// Out = MulOut + Y = (X - Y) * W + Y = X * W + Y * (1 - W)
AppendOp(framework::OpRegistry::CreateOp("elementwise_add",
@@ -65,18 +64,26 @@ class InterpOpMaker : public framework::OpProtoAndCheckerMaker {
public:
InterpOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
: OpProtoAndCheckerMaker(proto, op_checker) {
- AddInput("X", "A 2-D Tensor, the first input of interp_op");
- AddInput("Y", "A 2-D Tensor, the second input of interp_op");
- AddInput("W", "A 1-D Tensor, the interpolated values");
- AddOutput("MinusOut",
- "A 2-D Tensor, the intermediate outputs, saving X - Y.")
+ AddInput("X",
+ "(Tensor), 2-D Matrix of shape [batch_size, data_dim]"
+ "containing data samples, the first input of interp_op");
+ AddInput("Y",
+ "(Tensor), 2-D Matrix of shape `[batch_size, data_dim]`"
+ "containing data samples, the second input of interp_op");
+ AddInput("W",
+ "(Tensor), 1-D Vector of shape [batch_size],"
+ "the interpolated values in the half-open interval [0.0, 1.0)");
+ AddOutput("SubOut",
+ "(Tensor), the intermediate subtraction outputs, saving X - Y.")
.AsIntermediate();
AddOutput("MulOut",
- "A 2-D Tensor, the intermediate outputs,"
- "saving the mul mul of (X - Y) and W")
+ "(Tensor), the intermediate multiplication outputs,"
+ "saving the elementwise multiplication of (X - Y) and W.")
.AsIntermediate();
AddOutput("Out",
- "A 2-D Tensor, the output of interp_op, same shape with X");
+ "(Tensor), the output of interp_op, same shape with X,"
+ "returns the first-dimensional piecewise linear interpolant "
+ "between X and Y");
AddComment(R"DOC(
Linear Interpolation with two inputs, used in NEURAL TURING MACHINE.
diff --git a/python/paddle/v2/framework/tests/test_interp_op.py b/python/paddle/v2/framework/tests/test_interp_op.py
index f82dcc7f50..066569b96c 100644
--- a/python/paddle/v2/framework/tests/test_interp_op.py
+++ b/python/paddle/v2/framework/tests/test_interp_op.py
@@ -10,12 +10,12 @@ class TestInterpOp(OpTest):
y = np.random.random((2, 3)).astype("float32")
w = np.random.random(2).astype("float32")
- minus_out = x - y
- mul_out = minus_out * w.reshape(2, 1)
+ sub_out = x - y
+ mul_out = sub_out * w.reshape(2, 1)
out = mul_out + y
self.inputs = {'X': x, 'Y': y, 'W': w}
- self.outputs = {'Out': out, 'MinusOut': minus_out, 'MulOut': mul_out}
+ self.outputs = {'Out': out, 'SubOut': sub_out, 'MulOut': mul_out}
def test_check_output(self):
self.check_output()
From 597299074efb2e926954219c4afac9a6b189904d Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Mon, 9 Oct 2017 20:11:01 +0800
Subject: [PATCH 34/82] fix bug in REGISTER_OP(reduce_min)
---
paddle/operators/reduce_op.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc
index 87f66e1e93..55f294a9be 100644
--- a/paddle/operators/reduce_op.cc
+++ b/paddle/operators/reduce_op.cc
@@ -175,7 +175,7 @@ REGISTER_OP(reduce_mean, ops::ReduceOp, ops::ReduceMeanOpMaker,
REGISTER_OP(reduce_max, ops::ReduceOp, ops::ReduceMaxOpMaker, reduce_max_grad,
ops::ReduceGradOp);
-REGISTER_OP(reduce_min, ops::ReduceOp, ops::ReduceMaxOpMaker, reduce_min_grad,
+REGISTER_OP(reduce_min, ops::ReduceOp, ops::ReduceMinOpMaker, reduce_min_grad,
ops::ReduceGradOp);
#define REGISTER_REDUCE_CPU_KERNEL(reduce_type, functor, grad_functor) \
From b14c122a949402169678aa1a154349b2fbd8ddc0 Mon Sep 17 00:00:00 2001
From: fengjiayi <fengjiayi@baidu.com>
Date: Mon, 9 Oct 2017 10:28:26 -0700
Subject: [PATCH 35/82] Fix bug
---
paddle/framework/block_desc.cc | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc
index 01f50e1393..509aa235d3 100644
--- a/paddle/framework/block_desc.cc
+++ b/paddle/framework/block_desc.cc
@@ -74,6 +74,12 @@ void BlockDescBind::Sync() {
for (auto &op_desc : ops_) {
op_field.AddAllocated(op_desc->Proto());
}
+ auto &var_field = *this->desc_->mutable_vars();
+ var_field.Clear();
+ var_field.Reserve(static_cast<int>(vars_.size()));
+ for (auto &var_desc : vars_) {
+ var_field.AddAllocated(var_desc.second->Proto());
+ }
need_update_ = false;
}
}
From 3c39df197e2fbb0e8666bd8bb20e2a60e5a47d9b Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 10:30:20 -0700
Subject: [PATCH 36/82] Init Python API
Following the design
* https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md
Just written `Program`, `Block` and unittest of program.
---
python/paddle/v2/framework/graph.py | 45 +++++++++++++++++++
.../paddle/v2/framework/tests/test_program.py | 36 +++++++++++++++
2 files changed, 81 insertions(+)
create mode 100644 python/paddle/v2/framework/graph.py
create mode 100644 python/paddle/v2/framework/tests/test_program.py
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
new file mode 100644
index 0000000000..5211b0f166
--- /dev/null
+++ b/python/paddle/v2/framework/graph.py
@@ -0,0 +1,45 @@
+import paddle.v2.framework.core as core
+
+
+class Block(object):
+ def __init__(self, program, idx):
+ self.proto = program.proto.block(idx)
+ self.vars = dict() # var_name --> var
+ self.ops = list() # operator list
+ self.program = program
+
+ @property
+ def parent_idx(self):
+ return self.proto.parent
+
+ @property
+ def idx(self):
+ return self.proto.id
+
+
+class Program(object):
+ def __init__(self):
+ self.proto = core.ProgramDesc.instance()
+ assert self.proto.num_blocks() == 1
+ self.blocks = [Block(self, 0)]
+ self.current_block_idx = 0
+
+ def global_block(self):
+ return self.blocks[0]
+
+ def current_block(self):
+ return self.blocks[self.current_block_idx]
+
+ def create_block(self):
+ new_block_idx = len(self.blocks)
+ self.proto.append_block(self.current_block().proto)
+ self.current_block_idx = new_block_idx
+ self.blocks.append(Block(self, self.current_block_idx))
+ return self.current_block()
+
+ def rollback(self):
+ self.current_block_idx = self.current_block().parent_idx
+
+
+# program is a global instance.
+g_program = Program()
diff --git a/python/paddle/v2/framework/tests/test_program.py b/python/paddle/v2/framework/tests/test_program.py
new file mode 100644
index 0000000000..b82d1760d6
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_program.py
@@ -0,0 +1,36 @@
+import unittest
+from paddle.v2.framework.graph import g_program
+
+
+class TestProgram(unittest.TestCase):
+ def test_program(self):
+ b = g_program.current_block()
+ self.assertEqual(-1, b.parent_idx)
+ self.assertEqual(0, b.idx)
+
+ b = g_program.create_block()
+ self.assertEqual(1, b.idx)
+ self.assertEqual(0, b.parent_idx)
+
+ b = g_program.create_block()
+ self.assertEqual(2, b.idx)
+ self.assertEqual(1, b.parent_idx)
+
+ g_program.rollback()
+
+ b = g_program.current_block()
+ self.assertEqual(1, b.idx)
+ self.assertEqual(0, b.parent_idx)
+
+ b = g_program.create_block()
+ self.assertEqual(3, b.idx)
+ self.assertEqual(1, b.parent_idx)
+
+ g_program.rollback()
+ b = g_program.current_block()
+ self.assertEqual(1, b.idx)
+ self.assertEqual(0, b.parent_idx)
+
+
+if __name__ == '__main__':
+ unittest.main()
From ee545e47ccfa79a793bb0c7adabe6f0e852afc13 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Mon, 9 Oct 2017 10:34:59 -0700
Subject: [PATCH 37/82] gan api
---
doc/design/gan_api.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index d0f8b47ca3..0db18f92a7 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -15,6 +15,7 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
| batch norm (missing) | Zhuoyuan, Jiayi | Y |
| cudnn batch norm (missing)| Zhuoyuan, Jiayi | N |
| max-pooling (done) | ? | Y |
+| cudnn-max-pool (missing) | Chengduo | Y |
| fc (done) | ? | Y |
| softmax loss (done) | ? | Y |
| reshape op (done) | ? | Y |
From bedcf074a2c497afeb057cb8a1ecfaa3eb39a7dd Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 10:37:31 -0700
Subject: [PATCH 38/82] Implementation singleton
---
python/paddle/v2/framework/graph.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index 5211b0f166..7468279438 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -18,9 +18,18 @@ class Block(object):
class Program(object):
+ @classmethod
+ def instance(cls):
+ # From https://stackoverflow.com/questions/8212053
+ # Making Program as a Singleton class.
+ if not hasattr(cls, '_instance'):
+ cls._instance = cls()
+ return cls._instance
+
def __init__(self):
+ assert not hasattr(self.__class__,
+ '_instance'), 'Do not call constructor directly!'
self.proto = core.ProgramDesc.instance()
- assert self.proto.num_blocks() == 1
self.blocks = [Block(self, 0)]
self.current_block_idx = 0
@@ -42,4 +51,4 @@ class Program(object):
# program is a global instance.
-g_program = Program()
+g_program = Program.instance()
From f30a1f42f0b90b17c2664d7e9a65070ee1c3a473 Mon Sep 17 00:00:00 2001
From: kavyasrinet <kavyasrinet@baidu.com>
Date: Mon, 9 Oct 2017 10:49:21 -0700
Subject: [PATCH 39/82] Adding relu6 activation function (#4607)
---
paddle/operators/activation_op.cc | 16 ++++++++++
paddle/operators/activation_op.h | 31 +++++++++++++++++++
.../v2/framework/tests/test_activation_op.py | 19 +++++++-----
3 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index 2afa8a68b0..43081d2326 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -201,6 +201,19 @@ class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+template <typename AttrType>
+class Relu6OpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ Relu6OpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of Relu6 operator");
+ AddOutput("Y", "Output of Relu6 operator");
+ AddComment("Relu6 activation operator, relu6 = min(max(0, x), 6)");
+ AddAttr<AttrType>("threshold", "The threshold value of Relu6")
+ .SetDefault(static_cast<AttrType>(6));
+ }
+};
+
template <typename AttrType>
class PowOpMaker : public framework::OpProtoAndCheckerMaker {
public:
@@ -276,6 +289,9 @@ REGISTER_OP(leaky_relu, ops::ActivationOp, ops::LeakyReluOpMaker<float>,
REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker<float>,
soft_relu_grad, ops::ActivationOpGrad);
+REGISTER_OP(relu6, ops::ActivationOp, ops::Relu6OpMaker<float>, relu6_grad,
+ ops::ActivationOpGrad);
+
REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker<float>, pow_grad,
ops::ActivationOpGrad);
diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h
index 2450601742..f127468125 100644
--- a/paddle/operators/activation_op.h
+++ b/paddle/operators/activation_op.h
@@ -280,6 +280,36 @@ struct BReluGradFunctor : public BaseActivationFunctor<T> {
}
};
+// relu6(x) = min(max(0, x), 6)
+template <typename T>
+struct Relu6Functor : public BaseActivationFunctor<T> {
+ float threshold;
+
+ // NOTE: Explicit hides the `BaseActivationFunctor<T>::GetAttrs`
+ // not polymorphism for speed.
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"threshold", &threshold}};
+ }
+
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) const {
+ y.device(d) = x.cwiseMax(static_cast<T>(0)).cwiseMin(threshold);
+ }
+};
+
+template <typename T>
+struct Relu6GradFunctor : public BaseActivationFunctor<T> {
+ float threshold;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"threshold", &threshold}};
+ }
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) const {
+ dx.device(d) =
+ dy * ((x > static_cast<T>(0)) * (x < threshold)).template cast<T>();
+ }
+};
+
// softsign(x) = x / (1 + |x|)
template <typename T>
struct SoftsignFunctor : public BaseActivationFunctor<T> {
@@ -425,5 +455,6 @@ struct STanhGradFunctor : public BaseActivationFunctor<T> {
__macro(pow, PowFunctor, PowGradFunctor); \
__macro(stanh, STanhFunctor, STanhGradFunctor); \
__macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \
+ __macro(relu6, Relu6Functor, Relu6GradFunctor); \
__macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \
__macro(tanh_shrink, TanhShrinkFunctor, TanhShrinkGradFunctor)
diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py
index 701e1a1aee..8b76decaec 100644
--- a/python/paddle/v2/framework/tests/test_activation_op.py
+++ b/python/paddle/v2/framework/tests/test_activation_op.py
@@ -137,21 +137,26 @@ class TestBRelu(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.02)
-class TestLeakyRelu(OpTest):
+class TestRelu6(OpTest):
def setUp(self):
- self.op_type = "leaky_relu"
- alpha = 0.02
- self.attrs = {'alpha': alpha}
- self.inputs = {'X': np.random.uniform(-3, 3, [4, 4]).astype("float32")}
+ self.op_type = "relu6"
+ x = np.random.uniform(-1, 1, [4, 10]).astype("float32")
+ threshold = 6.0
+ # The same with TestAbs
+ x[np.abs(x) < 0.005] = 0.02
+ x[np.abs(x - threshold) < 0.005] = threshold + 0.02
+
+ self.inputs = {'X': x}
+ self.attrs = {'threshold': threshold}
self.outputs = {
- 'Y': np.maximum(self.inputs['X'], alpha * self.inputs['X'])
+ 'Y': np.minimum(np.maximum(self.inputs['X'], 0), threshold)
}
def test_check_output(self):
self.check_output()
def test_check_grad(self):
- self.check_grad(['X'], 'Y', max_relative_error=0.007)
+ self.check_grad(['X'], 'Y', max_relative_error=0.02)
class TestSoftRelu(OpTest):
From 8f4771be226e19593e0434db2293cc6bbcbbdc69 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 10:49:53 -0700
Subject: [PATCH 40/82] Add skeleton of Variable
---
python/paddle/v2/framework/graph.py | 35 +++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index 7468279438..5b93115b3e 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -1,5 +1,37 @@
import paddle.v2.framework.core as core
+__all__ = ['Block', 'Variable', 'Program']
+
+
+class Variable(object):
+ def __init__(self, block, name=None, shape=None, dtype=None,
+ lod_level=None):
+ self.block = block
+
+ if name is None:
+ name = Variable._unique_var_name_()
+ self.proto = self.block.proto.new_var(name)
+
+ if shape is not None:
+ self.proto.set_shape(shape)
+
+ if dtype is not None:
+ # TODO(yuyang18): Convert dtype from numpy.dtype
+ self.proto.set_data_type(dtype)
+
+ if lod_level is not None:
+ # TODO(yuyang18): set_lod_level is not defined.
+ self.proto.set_lod_level(lod_level)
+
+ self.block.vars[name] = self
+
+ # TODO(yuyang18): Get methods
+
+ @staticmethod
+ def _unique_var_name_():
+ uid = core.unique_integer() # unique during whole process.
+ return "_generated_var_%d" % uid
+
class Block(object):
def __init__(self, program, idx):
@@ -16,6 +48,9 @@ class Block(object):
def idx(self):
return self.proto.id
+ def create_var(self, *args, **kwargs):
+ return Variable(self, *args, **kwargs)
+
class Program(object):
@classmethod
From 4cb5bd90218082998f990d0977f05acef8da61e7 Mon Sep 17 00:00:00 2001
From: Abhinav Arora <abhinavarora28@gmail.com>
Date: Mon, 9 Oct 2017 10:56:56 -0700
Subject: [PATCH 41/82] Implementing the Adamax optimizer operator (#4538)
* Implementing the Adamax optimizer step operator
* Adding unit tests for adamax_op
* Changing learning rate and time step to inputs from attributes
* Changing learning rate and time step to input(tensors)
* Making the Adamax operator conform to naming convention
* Removing Tensor<float> from comments
* Rectifying the Adamax implementation
* Changing Unit Test values and adding comments
* Changing Unit Test to test multiple steps
---
paddle/operators/adamax_op.cc | 139 ++++++++++++++
paddle/operators/adamax_op.cu | 20 ++
paddle/operators/adamax_op.h | 72 +++++++
.../v2/framework/tests/test_adamax_op.py | 178 ++++++++++++++++++
4 files changed, 409 insertions(+)
create mode 100644 paddle/operators/adamax_op.cc
create mode 100644 paddle/operators/adamax_op.cu
create mode 100644 paddle/operators/adamax_op.h
create mode 100644 python/paddle/v2/framework/tests/test_adamax_op.py
diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc
new file mode 100644
index 0000000000..c348e0a0b2
--- /dev/null
+++ b/paddle/operators/adamax_op.cc
@@ -0,0 +1,139 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/adamax_op.h"
+
+namespace paddle {
+namespace operators {
+
+class AdamaxOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContextBase *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("Param"),
+ "Input(Param) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Grad"),
+ "Input(Grad) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Moment"),
+ "Input(Moment) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("InfNorm"),
+ "Input(InfNorm) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("LearningRate"),
+ "Input(LearningRate) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Beta1Pow"),
+ "Input(Beta1Pow) of AdamaxOp should not be null.");
+
+ PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
+ "Output(ParamOut) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("MomentOut"),
+ "Output(MomentOut) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("InfNormOut"),
+ "Output(InfNormOut) of AdamaxOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Beta1PowOut"),
+ "Output(Beta1PowOut) of AdamaxOp should not be null.");
+
+ auto lr_dims = ctx->GetInputDim("LearningRate");
+ PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1,
+ "Learning rate should have 1 dimension");
+ auto beta1_pow_dims = ctx->GetInputDim("Beta1Pow");
+ PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1,
+ "Beta1 power accumulator should have 1 dimension");
+ auto param_dims = ctx->GetInputDim("Param");
+ PADDLE_ENFORCE_EQ(
+ param_dims, ctx->GetInputDim("Grad"),
+ "Param and Grad input of AdamaxOp should have same dimension");
+ PADDLE_ENFORCE_EQ(
+ param_dims, ctx->GetInputDim("Moment"),
+ "Param and Moment input of AdamaxOp should have same dimension");
+ PADDLE_ENFORCE_EQ(
+ param_dims, ctx->GetInputDim("InfNorm"),
+ "Param and InfNorm input of AdamaxOp should have same dimension");
+
+ ctx->SetOutputDim("ParamOut", param_dims);
+ ctx->SetOutputDim("MomentOut", param_dims);
+ ctx->SetOutputDim("InfNormOut", param_dims);
+ ctx->SetOutputDim("Beta1PowOut", beta1_pow_dims);
+ }
+};
+
+class AdamaxOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ AdamaxOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("Param", "(Tensor) Input parameter");
+ AddInput("Grad", "(Tensor) Input gradient");
+ AddInput("LearningRate", "(Tensor) Learning rate");
+ AddInput("Moment", "(Tensor) First moment");
+ AddInput("InfNorm",
+ "(Tensor) "
+ "Input exponentially weighted infinity norm");
+ AddInput("Beta1Pow", "(Tensor) Input beta1 power accumulator");
+
+ AddOutput("ParamOut", "(Tensor) Output parameter");
+ AddOutput("MomentOut", "(Tensor) Output first moment");
+ AddOutput("InfNormOut",
+ "(Tensor) "
+ "Output exponentially weighted infinity norm");
+ AddOutput("Beta1PowOut", "(Tensor) Output beta1 power accumulator");
+
+ AddAttr<float>("beta1",
+ "(float, default 0.9) "
+ "Exponential decay rate for the "
+ "1st moment estimates.")
+ .SetDefault(0.9f);
+ AddAttr<float>("beta2",
+ "(float, default 0.999) "
+ "exponential decay rate for the weighted "
+ "infinity norm estimates.")
+ .SetDefault(0.999f);
+ AddAttr<float>("epsilon",
+ "(float, default 1.0e-8) "
+ "Constant for numerical stability")
+ .SetDefault(1.0e-8f);
+ AddComment(R"DOC(
+Adamax Updates Operator.
+
+This implements the Adamax optimizer from Section 7 of the Adam
+paper[1]. Adamax is a variant of the
+Adam algorithm based on the infinity norm.
+
+Adamax updates:
+
+moment_out = beta1 * moment + (1 - beta1) * grad
+inf_norm_out = max(beta2 * inf_norm + epsilon, abs(grad))
+beta1_pow_out = beta1_pow * beta1
+learning_rate_t = learning_rate/(1 - beta1_pow_out)
+param_out = param - learning_rate_t * moment_out/inf_norm_out
+
+The original paper does not have an epsilon attribute.
+However, it is added here for numerical stability
+by preventing divide by 0.
+
+References:
+ [1] Adam: A Method for Stochastic Optimization
+ (https://arxiv.org/abs/1412.6980)
+
+)DOC");
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(adamax, ops::AdamaxOp, ops::AdamaxOpMaker);
+REGISTER_OP_CPU_KERNEL(adamax,
+ ops::AdamaxOpKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/adamax_op.cu b/paddle/operators/adamax_op.cu
new file mode 100644
index 0000000000..fee3b6fc6b
--- /dev/null
+++ b/paddle/operators/adamax_op.cu
@@ -0,0 +1,20 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#define EIGEN_USE_GPU
+#include "paddle/operators/adamax_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(adamax,
+ ops::AdamaxOpKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/adamax_op.h b/paddle/operators/adamax_op.h
new file mode 100644
index 0000000000..9677b1bb78
--- /dev/null
+++ b/paddle/operators/adamax_op.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename Place, typename T>
+class AdamaxOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ auto param_out_tensor = ctx.Output<framework::Tensor>("ParamOut");
+ auto moment_out_tensor = ctx.Output<framework::Tensor>("MomentOut");
+ auto inf_norm_out_tensor = ctx.Output<framework::Tensor>("InfNormOut");
+ auto beta1_pow_out_tensor = ctx.Output<framework::Tensor>("Beta1PowOut");
+
+ param_out_tensor->mutable_data<T>(ctx.GetPlace());
+ moment_out_tensor->mutable_data<T>(ctx.GetPlace());
+ inf_norm_out_tensor->mutable_data<T>(ctx.GetPlace());
+ beta1_pow_out_tensor->mutable_data<T>(ctx.GetPlace());
+
+ float beta1 = ctx.Attr<float>("beta1");
+ float beta2 = ctx.Attr<float>("beta2");
+ float epsilon = ctx.Attr<float>("epsilon");
+
+ auto param = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Param"));
+ auto grad = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Grad"));
+ auto moment = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Moment"));
+ auto inf_norm = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("InfNorm"));
+ auto lr = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("LearningRate"));
+ auto beta1_pow = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Beta1Pow"));
+ auto param_out = framework::EigenVector<T>::Flatten(*param_out_tensor);
+ auto moment_out = framework::EigenVector<T>::Flatten(*moment_out_tensor);
+ auto inf_norm_out =
+ framework::EigenVector<T>::Flatten(*inf_norm_out_tensor);
+ auto beta1_pow_out =
+ framework::EigenVector<T>::Flatten(*beta1_pow_out_tensor);
+ auto place = ctx.GetEigenDevice<Place>();
+
+ moment_out.device(place) = beta1 * moment + (1 - beta1) * grad;
+ inf_norm_out.device(place) =
+ grad.abs().cwiseMax((beta2 * inf_norm) + epsilon);
+ beta1_pow_out.device(place) = beta1_pow * beta1;
+ auto lr_t = lr / (1 - beta1_pow_out);
+ Eigen::DSizes<int, 1> m_dsize(moment_out_tensor->numel());
+ param_out.device(place) =
+ param - lr_t.broadcast(m_dsize) * (moment_out / inf_norm_out);
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/python/paddle/v2/framework/tests/test_adamax_op.py b/python/paddle/v2/framework/tests/test_adamax_op.py
new file mode 100644
index 0000000000..af81075d6a
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_adamax_op.py
@@ -0,0 +1,178 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+class TestAdamaxOp1(OpTest):
+ def setUp(self):
+ '''Test Adamax Operator with supplied attributes
+ '''
+ self.op_type = "adamax"
+ param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ moment = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ # The infinity norm is positive
+ inf_norm = np.random.random((102, 105)).astype("float32")
+
+ learning_rate = 0.002
+ beta1 = 0.78
+ beta2 = 0.899
+ epsilon = 1e-5
+ beta1_pow = beta1**10
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment': moment,
+ 'InfNorm': inf_norm,
+ 'LearningRate': np.array([learning_rate]).astype("float32"),
+ 'Beta1Pow': np.array([beta1_pow]).astype("float32")
+ }
+
+ self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon}
+
+ param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step(
+ self.inputs, self.attrs)
+
+ self.outputs = {
+ 'ParamOut': param_out,
+ 'MomentOut': moment_out,
+ 'InfNormOut': inf_norm_out,
+ 'Beta1PowOut': beta1_pow_out
+ }
+
+ def test_check_output(self):
+ self.check_output()
+
+
+class TestAdamaxOp2(OpTest):
+ '''Test Adamax Operator with default attributes
+ '''
+
+ def setUp(self):
+ self.op_type = "adamax"
+ param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ moment = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ # The infinity norm is positive
+ inf_norm = np.random.random((102, 105)).astype("float32")
+
+ learning_rate = 0.002
+ beta1 = 0.9
+ beta2 = 0.999
+ epsilon = 1e-8
+ beta1_pow = beta1**8
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment': moment,
+ 'InfNorm': inf_norm,
+ 'LearningRate': np.array([learning_rate]).astype("float32"),
+ 'Beta1Pow': np.array([beta1_pow]).astype("float32")
+ }
+
+ attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon}
+ param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step(
+ self.inputs, attrs)
+
+ self.outputs = {
+ 'ParamOut': param_out,
+ 'MomentOut': moment_out,
+ 'InfNormOut': inf_norm_out,
+ 'Beta1PowOut': beta1_pow_out
+ }
+
+ def test_check_output(self):
+ self.check_output()
+
+
+class TestAdamaxOpMultipleSteps(OpTest):
+ def setUp(self):
+ '''Test Adamax Operator with supplied attributes
+ '''
+ self.op_type = "adamax"
+ self.num_steps = 10
+
+ param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ moment = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ # The infinity norm is positive
+ inf_norm = np.random.random((102, 105)).astype("float32")
+
+ learning_rate = 0.002
+ beta1 = 0.8
+ beta2 = 0.99
+ epsilon = 1e-5
+ beta1_pow = 1
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment': moment,
+ 'InfNorm': inf_norm,
+ 'LearningRate': np.array([learning_rate]).astype("float32"),
+ 'Beta1Pow': np.array([beta1_pow]).astype("float32")
+ }
+
+ self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon}
+
+ param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step(
+ self.inputs, self.attrs)
+
+ def test_check_output(self):
+ for _ in range(self.num_steps):
+ param_out, moment_out, inf_norm_out, beta1_pow_out = adamax_step(
+ self.inputs, self.attrs)
+
+ self.outputs = {
+ 'ParamOut': param_out,
+ 'MomentOut': moment_out,
+ 'InfNormOut': inf_norm_out,
+ 'Beta1PowOut': beta1_pow_out
+ }
+
+ # Verify output for this step
+ self.check_output()
+
+ # Output of this step becomes input for next step
+ self.inputs['Param'] = param_out
+ self.inputs['Moment'] = moment_out
+ self.inputs['InfNorm'] = inf_norm_out
+ self.inputs['Beta1Pow'] = beta1_pow_out
+
+ # Randomize gradient for next step
+ self.inputs['Grad'] = np.random.uniform(
+ -1, 1, (102, 105)).astype("float32")
+
+
+def adamax_step(inputs, attributes):
+ '''
+ Simulate one step of the adamax optimizer
+ :param inputs: dict of inputs
+ :param attributes: dict of attributes
+ :return tuple: tuple of output param, moment, inf_norm and
+ beta1 power accumulator
+ '''
+ param = inputs['Param']
+ grad = inputs['Grad']
+ moment = inputs['Moment']
+ inf_norm = inputs['InfNorm']
+ lr = inputs['LearningRate']
+ beta1_pow = inputs['Beta1Pow']
+
+ beta1 = attributes['beta1']
+ beta2 = attributes['beta2']
+ epsilon = attributes['epsilon']
+
+ moment_out = beta1 * moment + (1 - beta1) * grad
+ inf_norm_out = np.maximum(beta2 * inf_norm + epsilon, np.abs(grad))
+ beta1_pow_out = beta1_pow * beta1
+ lr_t = (lr / (1 - beta1_pow_out))
+ param_out = param - lr_t * np.divide(moment_out, inf_norm_out)
+
+ return param_out, moment_out, inf_norm_out, beta1_pow_out
+
+
+if __name__ == "__main__":
+ unittest.main()
From 61a5181e31a073a2b23cc76028fc24119d4970c7 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 11:05:25 -0700
Subject: [PATCH 42/82] Add skeleton of Operator
---
python/paddle/v2/framework/graph.py | 44 +++++++++++++++++++++++++++--
1 file changed, 42 insertions(+), 2 deletions(-)
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index 5b93115b3e..6f2a76a983 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -1,6 +1,7 @@
import paddle.v2.framework.core as core
+import collections
-__all__ = ['Block', 'Variable', 'Program']
+__all__ = ['Block', 'Variable', 'Program', 'Operator']
class Variable(object):
@@ -24,6 +25,7 @@ class Variable(object):
self.proto.set_lod_level(lod_level)
self.block.vars[name] = self
+ self.op = None
# TODO(yuyang18): Get methods
@@ -33,11 +35,37 @@ class Variable(object):
return "_generated_var_%d" % uid
+class Operator(object):
+ def __init__(self,
+ block,
+ proto,
+ type=None,
+ inputs=None,
+ outputs=None,
+ attrs=None):
+ self.block = block
+ self.proto = proto
+ if type is not None:
+ # TODO.
+ pass
+ if inputs is not None:
+ # TODO
+ pass
+ if outputs is not None:
+ # TODO
+ pass
+ if attrs is not None:
+ # TODO
+ pass
+
+ # TODO: Getters
+
+
class Block(object):
def __init__(self, program, idx):
self.proto = program.proto.block(idx)
self.vars = dict() # var_name --> var
- self.ops = list() # operator list
+ self.ops = collections.deque() # operator list
self.program = program
@property
@@ -51,6 +79,18 @@ class Block(object):
def create_var(self, *args, **kwargs):
return Variable(self, *args, **kwargs)
+ def append_op(self, *args, **kwargs):
+ op_proto = self.proto.append_op()
+ op = Operator(self, op_proto, *args, **kwargs)
+ self.ops.append(op)
+ return op
+
+ def prepend_op(self, *args, **kwargs):
+ op_proto = self.proto.prepend_op()
+ op = Operator(self, op_proto, *args, **kwargs)
+ self.ops.appendleft(op)
+ return op
+
class Program(object):
@classmethod
From c464ec21d8b0a1e7ad6da7115b78cd047d9a2041 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 12:09:39 -0700
Subject: [PATCH 43/82] Fix bug of foward default attribute not passed to
backward
---
paddle/framework/backward.cc | 2 +-
paddle/framework/op_desc.h | 5 +++++
paddle/framework/op_registry.cc | 11 ++++++++---
paddle/framework/op_registry.h | 2 +-
4 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc
index c970e01dd1..0a4688db9c 100644
--- a/paddle/framework/backward.cc
+++ b/paddle/framework/backward.cc
@@ -302,7 +302,7 @@ std::vector<std::unique_ptr<OpDescBind>> MakeOpGrad(
return grad_op_descs; // empty vector
}
- grad_op_descs = OpRegistry::CreateGradOpDescs(*op_desc);
+ grad_op_descs = OpRegistry::CreateGradOpDescs(op_desc.get());
std::list<std::unique_ptr<OpDescBind>> pending_fill_zeros_ops;
for (auto& desc : grad_op_descs) {
diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h
index b39808dad1..b729029412 100644
--- a/paddle/framework/op_desc.h
+++ b/paddle/framework/op_desc.h
@@ -97,6 +97,11 @@ class OpDescBind {
const VariableNameMap &Outputs() const { return outputs_; }
+ AttributeMap *MutableAttrMap() {
+ this->need_update_ = true;
+ return &this->attrs_;
+ }
+
private:
template <typename MapType>
static std::vector<typename MapType::key_type> MapKeys(const MapType &map) {
diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc
index 66043f6e04..b118edae17 100644
--- a/paddle/framework/op_registry.cc
+++ b/paddle/framework/op_registry.cc
@@ -60,9 +60,14 @@ std::unique_ptr<OperatorBase> OpRegistry::CreateOp(const OpDescBind& op_desc) {
}
std::vector<std::unique_ptr<OpDescBind>> OpRegistry::CreateGradOpDescs(
- const OpDescBind& op_desc) {
- auto& info = OpInfoMap::Instance().Get(op_desc.Type());
- return info.grad_op_maker_(op_desc);
+ OpDescBind* op_desc) {
+ auto& info = OpInfoMap::Instance().Get(op_desc->Type());
+
+ if (info.Checker() != nullptr) {
+ info.Checker()->Check(*op_desc->MutableAttrMap());
+ }
+
+ return info.grad_op_maker_(*op_desc);
}
} // namespace framework
diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h
index cce3605fd4..5ca3af52a6 100644
--- a/paddle/framework/op_registry.h
+++ b/paddle/framework/op_registry.h
@@ -80,7 +80,7 @@ class OpRegistry {
static std::unique_ptr<OperatorBase> CreateOp(const OpDesc& op_desc);
static std::vector<std::unique_ptr<OpDescBind>> CreateGradOpDescs(
- const OpDescBind& op_desc);
+ OpDescBind* op_desc);
static std::unique_ptr<OperatorBase> CreateOp(const OpDescBind& op_desc);
};
From 49ca0b4831037a3faa955ecf45ebedbee21ea833 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 13:10:04 -0700
Subject: [PATCH 44/82] Stash
---
paddle/framework/var_desc.cc | 8 ++++++++
paddle/framework/var_desc.h | 4 ++++
paddle/pybind/protobuf.cc | 4 +++-
3 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc
index 13b9c5f3cd..a88e813b5e 100644
--- a/paddle/framework/var_desc.cc
+++ b/paddle/framework/var_desc.cc
@@ -32,5 +32,13 @@ std::vector<int64_t> VarDescBind::Shape() const {
DataType VarDescBind::GetDataType() const {
return desc_.lod_tensor().data_type();
}
+
+void VarDescBind::SetLoDLevel(int32_t lod_level) {
+ desc_.mutable_lod_tensor()->set_lod_level(lod_level);
+}
+
+int32_t VarDescBind::GetLodLevel() const {
+ return desc_.lod_tensor().lod_level();
+}
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h
index 4763bf09d0..464fece85f 100644
--- a/paddle/framework/var_desc.h
+++ b/paddle/framework/var_desc.h
@@ -66,6 +66,10 @@ class VarDescBind {
DataType GetDataType() const;
+ void SetLoDLevel(int32_t lod_level);
+
+ int32_t GetLodLevel() const;
+
private:
VarDesc desc_;
};
diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc
index 218821b35b..7e18bf1c07 100644
--- a/paddle/pybind/protobuf.cc
+++ b/paddle/pybind/protobuf.cc
@@ -167,7 +167,9 @@ void BindVarDsec(py::module &m) {
.def("set_shape", &VarDescBind::SetShape)
.def("set_data_type", &VarDescBind::SetDataType)
.def("shape", &VarDescBind::Shape, py::return_value_policy::reference)
- .def("data_type", &VarDescBind::GetDataType);
+ .def("data_type", &VarDescBind::GetDataType)
+ .def("lod_level", &VarDescBind::GetLodLevel)
+ .def("set_lod_level", &VarDescBind::SetLoDLevel);
}
void BindOpDesc(py::module &m) {
From 92add2a29b4d0e5e5d4abe44d86e3a7c7af23645 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 13:20:08 -0700
Subject: [PATCH 45/82] Fix compile error in develop branch
---
paddle/operators/adamax_op.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc
index c348e0a0b2..5cf727742c 100644
--- a/paddle/operators/adamax_op.cc
+++ b/paddle/operators/adamax_op.cc
@@ -22,7 +22,7 @@ class AdamaxOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Param"),
"Input(Param) of AdamaxOp should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Grad"),
From 1e41a675d4111a826ffac45cbd197054d193d72e Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 13:39:38 -0700
Subject: [PATCH 46/82] Convert np.dtype to core.DataType
---
python/paddle/v2/framework/graph.py | 25 +++++++++++++++++--
.../v2/framework/tests/test_variable.py | 22 ++++++++++++++++
2 files changed, 45 insertions(+), 2 deletions(-)
create mode 100644 python/paddle/v2/framework/tests/test_variable.py
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index 6f2a76a983..a7a3ca62c7 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -1,5 +1,6 @@
import paddle.v2.framework.core as core
import collections
+import numpy as np
__all__ = ['Block', 'Variable', 'Program', 'Operator']
@@ -17,11 +18,11 @@ class Variable(object):
self.proto.set_shape(shape)
if dtype is not None:
- # TODO(yuyang18): Convert dtype from numpy.dtype
+ if not isinstance(dtype, core.DataType):
+ dtype = Variable._convert_np_dtype_to_dtype_(dtype)
self.proto.set_data_type(dtype)
if lod_level is not None:
- # TODO(yuyang18): set_lod_level is not defined.
self.proto.set_lod_level(lod_level)
self.block.vars[name] = self
@@ -34,6 +35,26 @@ class Variable(object):
uid = core.unique_integer() # unique during whole process.
return "_generated_var_%d" % uid
+ @staticmethod
+ def _convert_np_dtype_to_dtype_(np_dtype):
+ dtype = np.dtype(np_dtype)
+ if dtype == np.float32:
+ return core.DataType.FP32
+ elif dtype == np.float64:
+ return core.DataType.FP64
+ elif dtype == np.float16:
+ return core.DataType.FP16
+ elif dtype == np.int32:
+ return core.DataType.INT32
+ elif dtype == np.int16:
+ return core.DataType.INT16
+ elif dtype == np.int64:
+ return core.DataType.INT64
+ elif dtype == np.bool:
+ return core.DataType.BOOL
+ else:
+ raise ValueError("Not supported numpy dtype " + str(dtype))
+
class Operator(object):
def __init__(self,
diff --git a/python/paddle/v2/framework/tests/test_variable.py b/python/paddle/v2/framework/tests/test_variable.py
new file mode 100644
index 0000000000..dd23eac0cd
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_variable.py
@@ -0,0 +1,22 @@
+import unittest
+from paddle.v2.framework.graph import Variable
+import paddle.v2.framework.core as core
+import numpy as np
+
+
+class TestVariable(unittest.TestCase):
+ def test_np_dtype_convert(self):
+ DT = core.DataType
+ convert = Variable._convert_np_dtype_to_dtype_
+ self.assertEqual(DT.FP32, convert(np.float32))
+ self.assertEqual(DT.FP16, convert("float16"))
+ self.assertEqual(DT.FP64, convert("float64"))
+ self.assertEqual(DT.INT32, convert("int32"))
+ self.assertEqual(DT.INT16, convert("int16"))
+ self.assertEqual(DT.INT64, convert("int64"))
+ self.assertEqual(DT.BOOL, convert("bool"))
+ self.assertRaises(ValueError, lambda: convert("int8"))
+
+
+if __name__ == '__main__':
+ unittest.main()
From 569616b329db71bfc4739021d55e0a74179732e2 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 14:04:36 -0700
Subject: [PATCH 47/82] Complete Variable for Python API
---
python/paddle/v2/framework/graph.py | 59 ++++++++++++++++---
.../v2/framework/tests/test_variable.py | 20 ++++++-
2 files changed, 71 insertions(+), 8 deletions(-)
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index a7a3ca62c7..a66e7a9d73 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -12,23 +12,68 @@ class Variable(object):
if name is None:
name = Variable._unique_var_name_()
- self.proto = self.block.proto.new_var(name)
+ try:
+ self.proto = self.block.proto.var(name)
+ is_new_var = False
+ except core.EnforceNotMet:
+ self.proto = self.block.proto.new_var(name)
+ is_new_var = True
if shape is not None:
- self.proto.set_shape(shape)
-
+ if is_new_var:
+ self.proto.set_shape(shape)
+ else:
+ old_shape = self.shape
+ shape = tuple(shape)
+ if shape != old_shape:
+ raise ValueError(
+ "Variable {0} has been created before. the previous "
+ "shape is {1}; the new shape is {2}. They are not "
+ "matched.".format(self.name, old_shape, shape))
if dtype is not None:
if not isinstance(dtype, core.DataType):
dtype = Variable._convert_np_dtype_to_dtype_(dtype)
- self.proto.set_data_type(dtype)
+ if is_new_var:
+ self.proto.set_data_type(dtype)
+ else:
+ old_dtype = self.data_type()
+ if dtype != old_shape:
+ raise ValueError("Variable {0} has been created before. "
+ "The previous data type is {1}; the new "
+ "data type is {2}. They are not "
+ "matched.".format(self.name, old_dtype,
+ dtype))
if lod_level is not None:
- self.proto.set_lod_level(lod_level)
+ if is_new_var:
+ self.proto.set_lod_level(lod_level)
+ else:
+ if lod_level != self.lod_level:
+ raise ValueError("Variable {0} has been created before. "
+ "The previous lod_level is {1}; the new "
+ "lod_level is {2}. They are not "
+ "matched".format(self.name, self.lod_level,
+ lod_level))
self.block.vars[name] = self
self.op = None
- # TODO(yuyang18): Get methods
+ @property
+ def name(self):
+ return self.proto.name()
+
+ @property
+ def shape(self):
+ # convert to tuple, make it as same as numpy API.
+ return tuple(self.proto.shape())
+
+ @property
+ def data_type(self):
+ return self.proto.data_type()
+
+ @property
+ def lod_level(self):
+ return self.proto.lod_level()
@staticmethod
def _unique_var_name_():
@@ -79,7 +124,7 @@ class Operator(object):
# TODO
pass
- # TODO: Getters
+ # TODO: Getters
class Block(object):
diff --git a/python/paddle/v2/framework/tests/test_variable.py b/python/paddle/v2/framework/tests/test_variable.py
index dd23eac0cd..8ea1083ff6 100644
--- a/python/paddle/v2/framework/tests/test_variable.py
+++ b/python/paddle/v2/framework/tests/test_variable.py
@@ -1,5 +1,5 @@
import unittest
-from paddle.v2.framework.graph import Variable
+from paddle.v2.framework.graph import Variable, g_program
import paddle.v2.framework.core as core
import numpy as np
@@ -17,6 +17,24 @@ class TestVariable(unittest.TestCase):
self.assertEqual(DT.BOOL, convert("bool"))
self.assertRaises(ValueError, lambda: convert("int8"))
+ def test_var(self):
+ b = g_program.current_block()
+ w = b.create_var(
+ dtype="float64", shape=[784, 100], lod_level=0, name="fc.w")
+ self.assertEqual(core.DataType.FP64, w.data_type)
+ self.assertEqual((784, 100), w.shape)
+ self.assertEqual("fc.w", w.name)
+ self.assertEqual(0, w.lod_level)
+
+ w = b.create_var(name='fc.w')
+ self.assertEqual(core.DataType.FP64, w.data_type)
+ self.assertEqual((784, 100), w.shape)
+ self.assertEqual("fc.w", w.name)
+ self.assertEqual(0, w.lod_level)
+
+ self.assertRaises(ValueError,
+ lambda: b.create_var(name="fc.w", shape=(24, 100)))
+
if __name__ == '__main__':
unittest.main()
From dcb09e932d57701b553a5308aaab5b16bf214910 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 14:21:58 -0700
Subject: [PATCH 48/82] Use PROTO_LITE when refactoring Paddle
It will significantly reduce binary size. It is useful for mobile
deployment.
---
paddle/framework/framework.proto | 1 +
paddle/framework/op_desc.h | 2 --
paddle/framework/program_desc.h | 2 --
paddle/operators/net_op.h | 1 +
paddle/pybind/protobuf.cc | 3 ---
5 files changed, 2 insertions(+), 7 deletions(-)
diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto
index ac2827e547..b7a63f9ba1 100644
--- a/paddle/framework/framework.proto
+++ b/paddle/framework/framework.proto
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
syntax = "proto2";
+option optimize_for = LITE_RUNTIME;
package paddle.framework;
enum AttrType {
diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h
index b729029412..d0c314771c 100644
--- a/paddle/framework/op_desc.h
+++ b/paddle/framework/op_desc.h
@@ -52,8 +52,6 @@ class OpDescBind {
void SetOutput(const std::string ¶m_name,
const std::vector<std::string> &args);
- std::string DebugString() { return this->Proto()->DebugString(); }
-
bool HasAttr(const std::string &name) const {
return attrs_.find(name) != attrs_.end();
}
diff --git a/paddle/framework/program_desc.h b/paddle/framework/program_desc.h
index 9b34a06aef..d684b08d16 100644
--- a/paddle/framework/program_desc.h
+++ b/paddle/framework/program_desc.h
@@ -31,8 +31,6 @@ class ProgramDescBind {
BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); }
- std::string DebugString() { return Proto()->DebugString(); }
-
size_t Size() const { return blocks_.size(); }
ProgramDesc *Proto();
diff --git a/paddle/operators/net_op.h b/paddle/operators/net_op.h
index 2388b094d2..ebeb262d96 100644
--- a/paddle/operators/net_op.h
+++ b/paddle/operators/net_op.h
@@ -14,6 +14,7 @@ limitations under the License. */
#pragma once
+#include <set>
#include "paddle/framework/framework.pb.h"
#include "paddle/framework/op_registry.h"
diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc
index 218821b35b..47bd7bc3bb 100644
--- a/paddle/pybind/protobuf.cc
+++ b/paddle/pybind/protobuf.cc
@@ -117,7 +117,6 @@ void BindProgramDesc(py::module &m) {
.def("append_block", &ProgramDescBind::AppendBlock,
py::return_value_policy::reference)
.def("block", &ProgramDescBind::Block, py::return_value_policy::reference)
- .def("__str__", &ProgramDescBind::DebugString)
.def("num_blocks", &ProgramDescBind::Size);
}
@@ -191,8 +190,6 @@ void BindOpDesc(py::module &m) {
.def("output", &OpDescBind::Output)
.def("output_names", &OpDescBind::OutputNames)
.def("set_output", &OpDescBind::SetOutput)
- .def("__str__", &OpDescBind::DebugString)
- .def("__repr__", &OpDescBind::DebugString)
.def("has_attr", &OpDescBind::HasAttr)
.def("attr_type", &OpDescBind::GetAttrType)
.def("attr_names", &OpDescBind::AttrNames)
From 383faaf71907605b92feef2ffc91f955623fa82d Mon Sep 17 00:00:00 2001
From: Abhinav Arora <abhinavarora28@gmail.com>
Date: Mon, 9 Oct 2017 14:25:38 -0700
Subject: [PATCH 49/82] Adding implementation for copying a vector to a tensor
(#4635)
* Adding implementation for copying a vector to tensor
* Changing Tensor test to access gpu memory indirectly
---
paddle/framework/tensor.h | 13 +++++
paddle/framework/tensor_impl.h | 23 +++++++++
paddle/framework/tensor_test.cc | 87 +++++++++++++++++++++++++++++++++
3 files changed, 123 insertions(+)
diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h
index 80a3f0a393..ba82127d9c 100644
--- a/paddle/framework/tensor.h
+++ b/paddle/framework/tensor.h
@@ -95,6 +95,19 @@ class Tensor {
template <typename T>
inline void CopyFrom(const Tensor& src, const platform::Place& dst_place);
+ /**
+ * @brief Copy the content of an external vector to a tensor.
+ *
+ * @param[in] src The external vector.
+ * @param[in] ctx The device context contains place where to store.
+ *
+ * * @note CopyFromVector assumes that the tensor has been resized
+ * before invoking.
+ */
+ template <typename T>
+ inline void CopyFromVector(const std::vector<T>& src,
+ const platform::Place& dst_place);
+
/**
* @brief Return the slice of the tensor.
*
diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h
index 379eac94f9..8ee9941982 100644
--- a/paddle/framework/tensor_impl.h
+++ b/paddle/framework/tensor_impl.h
@@ -123,6 +123,29 @@ inline void Tensor::CopyFrom(const Tensor& src,
#endif
}
+template <typename T>
+inline void Tensor::CopyFromVector(const std::vector<T>& src,
+ const platform::Place& dst_place) {
+ auto src_ptr = static_cast<const void*>(src.data());
+ platform::CPUPlace src_place;
+ auto dst_ptr = static_cast<void*>(mutable_data<T>(dst_place));
+ auto size = src.size() * sizeof(T);
+
+ if (platform::is_cpu_place(dst_place)) {
+ memory::Copy(boost::get<platform::CPUPlace>(dst_place), dst_ptr, src_place,
+ src_ptr, size);
+ }
+#ifdef PADDLE_WITH_CUDA
+ else if (platform::is_gpu_place(dst_place)) {
+ memory::Copy(boost::get<platform::GPUPlace>(dst_place), dst_ptr, src_place,
+ src_ptr, size, 0);
+ }
+ PADDLE_ENFORCE(cudaStreamSynchronize(0),
+ "cudaStreamSynchronize failed in Tensor CopyFromVector");
+
+#endif
+}
+
template <typename T>
inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const {
check_memory_size<T>();
diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc
index 58cf0fc3cb..492eba69e1 100644
--- a/paddle/framework/tensor_test.cc
+++ b/paddle/framework/tensor_test.cc
@@ -263,6 +263,93 @@ TEST(Tensor, CopyFrom) {
#endif
}
+TEST(Tensor, CopyFromVector) {
+ using namespace paddle::framework;
+ using namespace paddle::platform;
+ {
+ std::vector<int> src_vec = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+ Tensor cpu_tensor;
+
+ // Copy to CPU Tensor
+ cpu_tensor.Resize(make_ddim({3, 3}));
+ auto cpu_place = new paddle::platform::CPUPlace();
+ cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+
+ // Compare Tensors
+ const int* cpu_ptr = cpu_tensor.data<int>();
+ const int* src_ptr = src_vec.data();
+ ASSERT_NE(src_ptr, cpu_ptr);
+ for (size_t i = 0; i < 9; ++i) {
+ EXPECT_EQ(src_ptr[i], cpu_ptr[i]);
+ }
+
+ src_vec.erase(src_vec.begin(), src_vec.begin() + 5);
+ cpu_tensor.Resize(make_ddim({2, 2}));
+ cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+ cpu_ptr = cpu_tensor.data<int>();
+ src_ptr = src_vec.data();
+ ASSERT_NE(src_ptr, cpu_ptr);
+ for (size_t i = 0; i < 5; ++i) {
+ EXPECT_EQ(src_ptr[i], cpu_ptr[i]);
+ }
+
+ delete cpu_place;
+ }
+
+#ifdef PADDLE_WITH_CUDA
+ {
+ std::vector<int> src_vec = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+ Tensor cpu_tensor;
+ Tensor gpu_tensor;
+ Tensor dst_tensor;
+
+ // Copy to CPU Tensor
+ cpu_tensor.Resize(make_ddim({3, 3}));
+ auto cpu_place = new paddle::platform::CPUPlace();
+ cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+
+ // Copy to GPUTensor
+ gpu_tensor.Resize(make_ddim({3, 3}));
+ auto gpu_place = new paddle::platform::GPUPlace();
+ gpu_tensor.CopyFromVector<int>(src_vec, *gpu_place);
+ // Copy from GPU to CPU tensor for comparison
+ dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place);
+
+ // Compare Tensors
+ const int* src_ptr = src_vec.data();
+ const int* cpu_ptr = cpu_tensor.data<int>();
+ const int* dst_ptr = dst_tensor.data<int>();
+ ASSERT_NE(src_ptr, cpu_ptr);
+ ASSERT_NE(src_ptr, dst_ptr);
+ for (size_t i = 0; i < 9; ++i) {
+ EXPECT_EQ(src_ptr[i], cpu_ptr[i]);
+ EXPECT_EQ(src_ptr[i], dst_ptr[i]);
+ }
+
+ src_vec.erase(src_vec.begin(), src_vec.begin() + 5);
+
+ cpu_tensor.Resize(make_ddim({2, 2}));
+ cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+ gpu_tensor.Resize(make_ddim({2, 2}));
+ gpu_tensor.CopyFromVector<int>(src_vec, *gpu_place);
+ dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place);
+
+ src_ptr = src_vec.data();
+ cpu_ptr = cpu_tensor.data<int>();
+ dst_ptr = dst_tensor.data<int>();
+ ASSERT_NE(src_ptr, cpu_ptr);
+ ASSERT_NE(src_ptr, dst_ptr);
+ for (size_t i = 0; i < 5; ++i) {
+ EXPECT_EQ(src_ptr[i], cpu_ptr[i]);
+ EXPECT_EQ(src_ptr[i], dst_ptr[i]);
+ }
+
+ delete cpu_place;
+ delete gpu_place;
+ }
+#endif
+}
+
TEST(Tensor, ReshapeToMatrix) {
using namespace paddle::framework;
using namespace paddle::platform;
From 5984cbca47a4663b47b16390fc028829dbc9f183 Mon Sep 17 00:00:00 2001
From: fengjiayi <fengjiayi@baidu.com>
Date: Mon, 9 Oct 2017 14:30:31 -0700
Subject: [PATCH 50/82] Add Attr test
---
paddle/framework/backward_test.cc | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc
index 30225a4a99..05ebf356ba 100644
--- a/paddle/framework/backward_test.cc
+++ b/paddle/framework/backward_test.cc
@@ -440,6 +440,25 @@ TEST(Backward, simple_single_op) {
std::vector<std::string>({f::GradVarName("b")}));
}
+TEST(Backward, default_attribute) {
+ f::ProgramDesc *program_desc = GetNewProgramDesc();
+ f::ProgramDescBind &program = f::ProgramDescBind::Instance(program_desc);
+ f::BlockDescBind *block = program.Block(0);
+ f::OpDescBind *op = block->AppendOp();
+ op->SetType("mul");
+ op->SetInput("X", {"x"});
+ op->SetInput("Y", {"y"});
+ op->SetOutput("Out", {"out"});
+
+ AppendBackward(program, {});
+
+ ASSERT_EQ(block->AllOps().size(), 2UL);
+ f::OpDescBind *grad_op = block->AllOps()[1];
+ ASSERT_EQ(grad_op->Type(), "mul_grad");
+ EXPECT_EQ(boost::get<int>(grad_op->GetAttr("x_num_col_dims")), 1);
+ EXPECT_EQ(boost::get<int>(grad_op->GetAttr("y_num_col_dims")), 1);
+}
+
TEST(Backward, simple_mult_op) {
f::ProgramDesc *program_desc = GetNewProgramDesc();
f::ProgramDescBind &program = f::ProgramDescBind::Instance(program_desc);
From 4238b9b95cda29618828a9a477afecb3bbed984e Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Mon, 9 Oct 2017 14:35:45 -0700
Subject: [PATCH 51/82] gan_api
---
doc/design/gan_api.md | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 0db18f92a7..4fcff8b70a 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -1,10 +1,10 @@
# Design for GAN
-GAN (General Adversarial Net) is an important model for unsupervised learning and widely used in many areas.
+GAN (General Adversarial Net [https://arxiv.org/abs/1406.2661]) is an important model for unsupervised learning and widely used in many areas.
-It contains several important machine learning concepts, including building and running subgraphs, dependency tracing, different optimizers in one executor and so forth.
+It applies several important concepts in machine learning system design, including building and running subgraphs, dependency tracing, different optimizers in one executor and so forth.
-In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN as an example due to its good performance on image generation.
+In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN (Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks [https://arxiv.org/abs/1511.06434]) as an example due to its good performance on image generation.
| important building blocks | People in Charge | Required |
|---------------------------|-------------------|----------|
@@ -51,7 +51,7 @@ Returns a 0/1 binary label.
build the whole GAN model, define training loss for both generator and discrimator.
## Discussion on Engine Functions required to build GAN
-- Trace the ternsor and variable dependency in the engine executor. (Very critical, otherwise GAN can'be be trained correctly)
+- Trace the tensor and variable dependency in the engine executor. (Very critical, otherwise GAN can'be be trained correctly)
- Different optimizers responsible for optimizing different loss.
To be more detailed, we introduce our design of DCGAN as following:
From ce901b1186b671781cd86b91ce530e2be3408f37 Mon Sep 17 00:00:00 2001
From: fengjiayi <fengjiayi@baidu.com>
Date: Mon, 9 Oct 2017 15:16:34 -0700
Subject: [PATCH 52/82] Refine unit test
---
paddle/framework/backward_test.cc | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc
index 05ebf356ba..3b7cbcd989 100644
--- a/paddle/framework/backward_test.cc
+++ b/paddle/framework/backward_test.cc
@@ -58,6 +58,8 @@ class MulOpMaker : public OpProtoAndCheckerMaker {
AddInput("X", "A");
AddInput("Y", "B");
AddOutput("Out", "Out");
+ AddAttr<int>("x_num_col_dims", "").SetDefault(1).EqualGreaterThan(1);
+ AddAttr<int>("y_num_col_dims", "").SetDefault(1).EqualGreaterThan(1);
AddComment("Mul");
}
};
@@ -453,6 +455,9 @@ TEST(Backward, default_attribute) {
AppendBackward(program, {});
ASSERT_EQ(block->AllOps().size(), 2UL);
+ EXPECT_EQ(boost::get<int>(op->GetAttr("x_num_col_dims")), 1);
+ EXPECT_EQ(boost::get<int>(op->GetAttr("y_num_col_dims")), 1);
+
f::OpDescBind *grad_op = block->AllOps()[1];
ASSERT_EQ(grad_op->Type(), "mul_grad");
EXPECT_EQ(boost::get<int>(grad_op->GetAttr("x_num_col_dims")), 1);
From d2217ffa39f30ec719e93cd2cb6b9449467bcbc1 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 16:03:39 -0700
Subject: [PATCH 53/82] Fix CI
---
paddle/framework/block_desc.h | 1 +
paddle/framework/program_desc.h | 1 +
2 files changed, 2 insertions(+)
diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h
index 2de270f60e..3437e89923 100644
--- a/paddle/framework/block_desc.h
+++ b/paddle/framework/block_desc.h
@@ -15,6 +15,7 @@ limitations under the License. */
#pragma once
#include <deque>
+#include <memory>
#include <unordered_map>
#include <vector>
#include "paddle/framework/op_desc.h"
diff --git a/paddle/framework/program_desc.h b/paddle/framework/program_desc.h
index d684b08d16..f29b1c54e7 100644
--- a/paddle/framework/program_desc.h
+++ b/paddle/framework/program_desc.h
@@ -14,6 +14,7 @@ limitations under the License. */
#pragma once
+#include <memory>
#include <vector>
#include "paddle/framework/framework.pb.h"
#include "paddle/platform/macros.h"
From 4767fb6719694ee400d3a6c9344aa21edde8bd36 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Mon, 9 Oct 2017 16:05:14 -0700
Subject: [PATCH 54/82] gan api modified
---
doc/design/gan_api.md | 67 ++++++++++++++++++++++++++++++++++---------
1 file changed, 53 insertions(+), 14 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 4fcff8b70a..77c867bac7 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -139,10 +139,10 @@ class DCGAN(object):
- Define data readers as placeholders to hold the data;
- Build generator and discriminators;
- Define two training losses for discriminator and generator, respectively.
+If we have execution dependency engine to back-trace all tensors, the module building our GAN model will be like this:
```python
class DCGAN(object):
def build_model(self):
- # input data
if self.y_dim:
self.y = pd.data(pd.float32, [self.batch_size, self.y_dim])
self.images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
@@ -151,17 +151,17 @@ class DCGAN(object):
# step 1: generate images by generator, classify real/fake images with discriminator
if self.y_dim: # if conditional GAN, includes label
- self.G = self.generator(self.z, self.y)
- self.D_t = self.discriminator(self.images)
- # generated fake images
- self.sampled = self.sampler(self.z, self.y)
- self.D_f = self.discriminator(self.images)
+ self.G = self.generator(self.z, self.y)
+ self.D_t = self.discriminator(self.images)
+ # generated fake images
+ self.sampled = self.sampler(self.z, self.y)
+ self.D_f = self.discriminator(self.G)
else: # original version of GAN
- self.G = self.generator(self.z)
- self.D_t = self.discriminator(self.images)
- # generate fake images
- self.sampled = self.sampler(self.z)
- self.D_f = self.discriminator(self.images)
+ self.G = self.generator(self.z)
+ self.D_t = self.discriminator(self.images)
+ # generate fake images
+ self.sampled = self.sampler(self.z)
+ self.D_f = self.discriminator(self.images)
# step 2: define the two losses
self.d_loss_real = pd.reduce_mean(pd.cross_entropy(self.D_t, np.ones(self.batch_size))
@@ -171,6 +171,44 @@ class DCGAN(object):
self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_f, np.ones(self.batch_szie))
```
+If we do not have dependency engine but blocks, the module building our GAN model will be like this:
+```python
+class DCGAN(object):
+ def build_model(self, default_block):
+ # input data in the default block
+ if self.y_dim:
+ self.y = pd.data(pd.float32, [self.batch_size, self.y_dim])
+ self.images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
+ # self.faked_images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
+ self.z = pd.data(tf.float32, [None, self.z_size])
+
+ # step 1: generate images by generator, classify real/fake images with discriminator
+ with pd.default_block().g_block():
+ if self.y_dim: # if conditional GAN, includes label
+ self.G = self.generator(self.z, self.y)
+ self.D_g = self.discriminator(self.G, self.y)
+ else: # original version of GAN
+ self.G = self.generator(self.z)
+ self.D_g = self.discriminator(self.G, self.y)
+ self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_g, np.ones(self.batch_szie))
+
+ with pd.default_block().d_block():
+ if self.y_dim: # if conditional GAN, includes label
+ self.D_t = self.discriminator(self.images, self.y)
+ self.D_f = self.discriminator(self.G, self.y)
+ else: # original version of GAN
+ self.D_t = self.discriminator(self.images)
+ self.D_f = self.discriminator(self.G)
+
+ # step 2: define the two losses
+ self.d_loss_real = pd.reduce_mean(pd.cross_entropy(self.D_t, np.ones(self.batch_size))
+ self.d_loss_fake = pd.reduce_mean(pd.cross_entropy(self.D_f, np.zeros(self.batch_size))
+ self.d_loss = self.d_loss_real + self.d_loss_fake
+```
+Some small confusion and problems with this design:
+- D\_g and D\_f are actually the same thing, but has to be written twice;
+- Requires ability to create a block anytime, rather than in if-else or rnn only;
+
## Main function for the demo:
Generally, the user of GAN just need to the following things:
- Define an object as DCGAN class;
@@ -183,9 +221,10 @@ import numpy as np
import logging
if __name__ == "__main__":
- # dcgan
- dcgan = DCGAN()
- dcgan.build_model()
+ # dcgan class in the default graph/block
+ with pd.block() as def_block:
+ dcgan = DCGAN()
+ dcgan.build_model(def_block)
# load mnist data
data_X, data_y = self.load_mnist()
From 35a5b9b99756188f2782ed19b4eaca57cb44ceea Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Mon, 9 Oct 2017 16:22:49 -0700
Subject: [PATCH 55/82] gan api
---
doc/design/gan_api.md | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 77c867bac7..ed7622920b 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -222,6 +222,10 @@ import logging
if __name__ == "__main__":
# dcgan class in the default graph/block
+ # if we use dependency engine as tensorflow
+ # the codes, will be slightly different like:
+ # dcgan = DCGAN()
+ # dcgan.build_model()
with pd.block() as def_block:
dcgan = DCGAN()
dcgan.build_model(def_block)
@@ -230,8 +234,12 @@ if __name__ == "__main__":
data_X, data_y = self.load_mnist()
# Two subgraphs required!!!
- d_optim = pd.train.Adam(lr = .001, beta= .1).minimize(dcgan.d_loss, dcgan.theta_D)
- g_optim = pd.train.Adam(lr = .001, beta= .1).minimize(dcgan.g_loss, dcgan.theta_G)
+ with pd.block().d_block():
+ d_optim = pd.train.Adam(lr = .001, beta= .1)
+ d_step = d_optim.minimize(dcgan.d_loss, dcgan.theta_D)
+ with pd.block.g_block():
+ g_optim = pd.train.Adam(lr = .001, beta= .1)
+ g_step = pd.minimize(dcgan.g_loss, dcgan.theta_G)
# executor
sess = pd.executor()
@@ -246,11 +254,11 @@ if __name__ == "__main__":
batch_z = np.random.uniform(-1., 1., [batch_size, z_dim])
if batch_id % 2 == 0:
- sess.run(d_optim,
+ sess.run(d_step,
feed_dict = {dcgan.images: batch_im,
dcgan.y: batch_label,
dcgan.z: batch_z})
else:
- sess.run(g_optim,
+ sess.run(g_step,
feed_dict = {dcgan.z: batch_z})
```
From 67c77b57fc8ad8eff7e9e46e842aa54b1e39047a Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Mon, 9 Oct 2017 16:47:05 -0700
Subject: [PATCH 56/82] gan api
---
doc/design/gan_api.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index ed7622920b..689f359564 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -206,7 +206,7 @@ class DCGAN(object):
self.d_loss = self.d_loss_real + self.d_loss_fake
```
Some small confusion and problems with this design:
-- D\_g and D\_f are actually the same thing, but has to be written twice;
+- D\_g and D\_f are actually the same thing, but has to be written twice; i.e., if we want to run two sub-graphs conceptually, the same codes have to be written twice if they are shared by the graph.
- Requires ability to create a block anytime, rather than in if-else or rnn only;
## Main function for the demo:
From 63912dcc198729b5e29e9080da6d76e649fd9394 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Mon, 9 Oct 2017 17:09:04 -0700
Subject: [PATCH 57/82] gan design
---
doc/design/gan_api.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 689f359564..1a7d0df116 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -262,3 +262,7 @@ if __name__ == "__main__":
sess.run(g_step,
feed_dict = {dcgan.z: batch_z})
```
+
+# More thinking about dependency engine v.s. block design:
+- What if we just want to run an intermediate result? Do we need to run the whole block/graph?
+- Should we call eval() to get the fake images in the first stage? And then train the discriminator in the second stage?
From 6efacc14d857bd117d5918bf02afc9cca702bd78 Mon Sep 17 00:00:00 2001
From: Abhinav Arora <aroraabhinav@baidu.com>
Date: Mon, 9 Oct 2017 17:24:21 -0700
Subject: [PATCH 58/82] Implementing the fill constant op for the executor
---
paddle/operators/fill_constant_op.cc | 68 +++++++++++++++++++
paddle/operators/fill_constant_op.cu | 22 ++++++
paddle/operators/fill_constant_op.h | 37 ++++++++++
.../framework/tests/test_fill_constant_op.py | 35 ++++++++++
4 files changed, 162 insertions(+)
create mode 100644 paddle/operators/fill_constant_op.cc
create mode 100644 paddle/operators/fill_constant_op.cu
create mode 100644 paddle/operators/fill_constant_op.h
create mode 100644 python/paddle/v2/framework/tests/test_fill_constant_op.py
diff --git a/paddle/operators/fill_constant_op.cc b/paddle/operators/fill_constant_op.cc
new file mode 100644
index 0000000000..65d03d5fa4
--- /dev/null
+++ b/paddle/operators/fill_constant_op.cc
@@ -0,0 +1,68 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/fill_constant_op.h"
+
+namespace paddle {
+namespace operators {
+
+class FillConstantOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasOutput("Out"),
+ "Output(Out) of FillConstantOp should not be null.");
+ auto &shape = ctx->Attrs().Get<std::vector<int>>("shape");
+ std::vector<int64_t> shape_int64(shape.size(), 0);
+ std::transform(shape.begin(), shape.end(), shape_int64.begin(),
+ [](int a) { return static_cast<int64_t>(a); });
+ auto dims = framework::make_ddim(shape_int64);
+ ctx->SetOutputDim("Out", dims);
+ }
+
+ framework::DataType IndicateDataType(
+ const framework::ExecutionContext &ctx) const override {
+ return static_cast<framework::DataType>(ctx.Attr<int>("dataType"));
+ }
+};
+
+class FillConstantOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ FillConstantOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : framework::OpProtoAndCheckerMaker(proto, op_checker) {
+ AddAttr<int>("dataType",
+ "(int, default 5 (FP32)) "
+ "Output data type")
+ .SetDefault(framework::DataType::FP32);
+ AddAttr<std::vector<int>>("shape", "(vector<int>) The shape of the output");
+ AddAttr<float>("value", "(float, default 0) The value to be filled")
+ .SetDefault(0.0f);
+ AddOutput("Out",
+ "(Tensor) Tensor of specified shape will be filled "
+ "with the specified value");
+ AddComment(R"DOC(Fill up a variable with specified constant value.)DOC");
+ }
+};
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(fill_constant, ops::FillConstantOp,
+ ops::FillConstantOpMaker);
+REGISTER_OP_CPU_KERNEL(
+ fill_constant,
+ ops::FillConstantOpKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/fill_constant_op.cu b/paddle/operators/fill_constant_op.cu
new file mode 100644
index 0000000000..eef8fcbd7f
--- /dev/null
+++ b/paddle/operators/fill_constant_op.cu
@@ -0,0 +1,22 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#define EIGEN_USE_GPU
+#include "paddle/framework/op_registry.h"
+#include "paddle/operators/fill_constant_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(
+ fill_constant,
+ ops::FillConstantOpKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/fill_constant_op.h b/paddle/operators/fill_constant_op.h
new file mode 100644
index 0000000000..53b8b548ec
--- /dev/null
+++ b/paddle/operators/fill_constant_op.h
@@ -0,0 +1,37 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename Place, typename T>
+class FillConstantOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ auto* out = ctx.Output<framework::Tensor>("Out");
+ out->mutable_data<T>(ctx.GetPlace());
+ auto value = ctx.Attr<T>("value");
+
+ auto out_eigen = framework::EigenVector<T>::Flatten(*out);
+ auto place = ctx.GetEigenDevice<Place>();
+ out_eigen.device(place) = out_eigen.constant(static_cast<T>(value));
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/python/paddle/v2/framework/tests/test_fill_constant_op.py b/python/paddle/v2/framework/tests/test_fill_constant_op.py
new file mode 100644
index 0000000000..dff7b615aa
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_fill_constant_op.py
@@ -0,0 +1,35 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+class TestFillConstantOp1(OpTest):
+ def setUp(self):
+ '''Test fill_constant op with specified value
+ '''
+ self.op_type = "fill_constant"
+
+ self.inputs = {}
+ self.attrs = {'shape': [123, 92], 'value': 3.8}
+ self.outputs = {'Out': np.full((123, 92), 3.8)}
+
+ def test_check_output(self):
+ self.check_output()
+
+
+class TestFillConstantOp2(OpTest):
+ def setUp(self):
+ '''Test fill_constant op with default value
+ '''
+ self.op_type = "fill_constant"
+
+ self.inputs = {}
+ self.attrs = {'shape': [123, 92]}
+ self.outputs = {'Out': np.full((123, 92), 0.0)}
+
+ def test_check_output(self):
+ self.check_output()
+
+
+if __name__ == "__main__":
+ unittest.main()
From c876cacc7b7936f356b0c712dff8f32534eab2ae Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Tue, 10 Oct 2017 10:24:18 +0800
Subject: [PATCH 59/82] =?UTF-8?q?fix=20compiler=20error:=20=E2=80=98unique?=
=?UTF-8?q?=5Fptr=E2=80=99=20is=20not=20a=20member=20of=20=E2=80=98std?=
=?UTF-8?q?=E2=80=99?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
paddle/framework/type_defs.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h
index a5b9472213..6f65a942ba 100644
--- a/paddle/framework/type_defs.h
+++ b/paddle/framework/type_defs.h
@@ -15,6 +15,7 @@
#pragma once
#include <functional>
#include <map>
+#include <memory>
#include "paddle/platform/variant.h"
namespace paddle {
From d350cdbc973a81d3a1e1b3cd90aaaddf2bdfac2f Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 19:59:38 -0700
Subject: [PATCH 60/82] Polish CompileTime InferShape
1. InferShape should be a method for `OpDescBind` not `Operator`, since
`OpDescBind` is a compile-time concept.
2. Pre-create operators and store them into a map. Make InferShape
faster
---
paddle/framework/op_desc.cc | 35 +++++++++++++++++++
paddle/framework/op_desc.h | 2 ++
paddle/pybind/protobuf.cc | 3 +-
paddle/pybind/pybind.cc | 15 --------
.../v2/framework/tests/test_infer_shape.py | 6 ++--
5 files changed, 42 insertions(+), 19 deletions(-)
diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc
index 02aa74a842..6ce453cc99 100644
--- a/paddle/framework/op_desc.cc
+++ b/paddle/framework/op_desc.cc
@@ -13,7 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/framework/op_desc.h"
+#include <functional>
+#include <unordered_map>
#include "paddle/framework/block_desc.h"
+#include "paddle/framework/operator.h"
namespace paddle {
namespace framework {
@@ -184,5 +187,37 @@ void OpDescBind::Sync() {
need_update_ = false;
}
}
+
+using InferShapeFuncMap =
+ std::unordered_map<std::string /*op_type*/,
+ std::function<void(InferShapeContext *)>>;
+
+static InferShapeFuncMap &InferShapeFuncs() {
+ static InferShapeFuncMap *g_map = nullptr;
+ if (g_map == nullptr) {
+ g_map = new InferShapeFuncMap();
+ auto &info_map = OpInfoMap::Instance();
+ // all registered kernels
+ for (auto &pair : OperatorWithKernel::AllOpKernels()) {
+ auto &info = info_map.Get(pair.first);
+ auto op =
+ static_cast<OperatorWithKernel *>(info.Creator()("", {}, {}, {}));
+ g_map->insert(
+ {pair.first, [op](InferShapeContext *ctx) { op->InferShape(ctx); }});
+ }
+ }
+ return *g_map;
+}
+
+void OpDescBind::InferShape(const BlockDescBind &block) const {
+ auto &funcs = InferShapeFuncs();
+ auto it = funcs.find(this->Type());
+ if (it == funcs.end()) {
+ PADDLE_THROW("Operator %s has not been registered", this->Type());
+ }
+ CompileTimeInferShapeContext ctx(*this, block);
+ it->second(&ctx);
+}
+
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h
index d0c314771c..81c4225041 100644
--- a/paddle/framework/op_desc.h
+++ b/paddle/framework/op_desc.h
@@ -100,6 +100,8 @@ class OpDescBind {
return &this->attrs_;
}
+ void InferShape(const BlockDescBind &block) const;
+
private:
template <typename MapType>
static std::vector<typename MapType::key_type> MapKeys(const MapType &map) {
diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc
index 47bd7bc3bb..6333cc332e 100644
--- a/paddle/pybind/protobuf.cc
+++ b/paddle/pybind/protobuf.cc
@@ -196,7 +196,8 @@ void BindOpDesc(py::module &m) {
.def("set_attr", &OpDescBind::SetAttr)
.def("attr", &OpDescBind::GetAttr)
.def("set_block_attr", &OpDescBind::SetBlockAttr)
- .def("get_block_attr", &OpDescBind::GetBlockAttr);
+ .def("get_block_attr", &OpDescBind::GetBlockAttr)
+ .def("infer_shape", &OpDescBind::InferShape);
}
} // namespace pybind
diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc
index 356c4986e2..0f6e3101e2 100644
--- a/paddle/pybind/pybind.cc
+++ b/paddle/pybind/pybind.cc
@@ -231,21 +231,6 @@ All parameter, weight, gradient are variables in Paddle.
desc.InitializationErrorString());
return OpRegistry::CreateOp(desc);
})
- .def_static("infer_shape",
- [](OpDescBind &op_desc, BlockDescBind &block) {
- auto op = OpRegistry::CreateOp(*op_desc.Proto());
- auto *op_with_kernel =
- dynamic_cast<OperatorWithKernel *>(op.get());
- if (op_with_kernel != nullptr) {
- auto ctx = CompileTimeInferShapeContext(op_desc, block);
- op_with_kernel->InferShape(&ctx);
- } else {
- PADDLE_THROW(
- "OP(%s) is not type of OperatorWithKernel, "
- "should not call this function",
- op_desc.Type());
- }
- })
.def("backward",
[](const OperatorBase &forwardOp,
const std::unordered_set<std::string> &no_grad_vars) {
diff --git a/python/paddle/v2/framework/tests/test_infer_shape.py b/python/paddle/v2/framework/tests/test_infer_shape.py
index b38ec9c037..99562890fd 100644
--- a/python/paddle/v2/framework/tests/test_infer_shape.py
+++ b/python/paddle/v2/framework/tests/test_infer_shape.py
@@ -1,6 +1,6 @@
import unittest
+
import paddle.v2.framework.core as core
-from paddle.v2.framework.op import Operator
class TestInferShape(unittest.TestCase):
@@ -26,7 +26,7 @@ class TestInferShape(unittest.TestCase):
sum_op_desc.set_input("X", ["x1", "x2"])
sum_op_desc.set_output("Out", ["out"])
- core.Operator.infer_shape(sum_op_desc, block)
+ sum_op_desc.infer_shape(block)
self.assertEqual(out.shape(), shape)
def test_mul_op(self):
@@ -55,7 +55,7 @@ class TestInferShape(unittest.TestCase):
mul_op_desc.set_attr("x_num_col_dims", 1)
mul_op_desc.set_attr("y_num_col_dims", 1)
- core.Operator.infer_shape(mul_op_desc, block)
+ mul_op_desc.infer_shape(block)
self.assertEqual(out.shape(), [x_shape[0], y_shape[1]])
From 6c6474cbd8514011b1c63d3439d49bd4700e46c8 Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Tue, 10 Oct 2017 10:32:19 +0800
Subject: [PATCH 61/82] follow coments
---
paddle/operators/CMakeLists.txt | 15 +++----
paddle/operators/math/pooling.h | 23 ++++++-----
paddle/operators/pool_with_index_op.cc | 57 +++++++++++++++-----------
3 files changed, 54 insertions(+), 41 deletions(-)
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index 39af318ca5..31ae4b2cc1 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -55,12 +55,20 @@ function(op_library TARGET)
set(pybind_flag 1)
endif()
+ # pool_op contains several operators
if ("${TARGET}" STREQUAL "pool_op")
set(pybind_flag 1)
# It's enough to just adding one operator to pybind
file(APPEND ${pybind_file} "USE_OP(pool2d);\n")
endif()
+ # pool_with_index_op contains several operators
+ if ("${TARGET}" STREQUAL "pool_with_index_op")
+ set(pybind_flag 1)
+ # It's enough to just adding one operator to pybind
+ file(APPEND ${pybind_file} "USE_OP(max_pool2d_with_index);\n")
+ endif()
+
# activation_op contains several operators
if ("${TARGET}" STREQUAL "activation_op")
set(pybind_flag 1)
@@ -75,13 +83,6 @@ function(op_library TARGET)
file(APPEND ${pybind_file} "USE_OP(reduce_sum);\n")
endif()
- # pool_with_index_op contains several operators
- if ("${TARGET}" STREQUAL "pool_with_index_op")
- set(pybind_flag 1)
- # It's enough to just adding one operator to pybind
- file(APPEND ${pybind_file} "USE_OP(max_pool2d_with_index);\n")
- endif()
-
# pybind USE_NO_KERNEL_OP
file(READ ${TARGET}.cc TARGET_CONTENT)
string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}")
diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h
index f15ddca69a..c50c57b5c5 100644
--- a/paddle/operators/math/pooling.h
+++ b/paddle/operators/math/pooling.h
@@ -24,15 +24,16 @@ namespace math {
#define FLT_MAX \
__FLT_MAX__ // It might need to be placed in another file, but I'm still
- // wondering where to put it
+ // wondering where to put it.
/*
* \brief Extracting simple operations from pooling.
- * Both MaxPool and AvgPool need initial, compute and finalize operation.
+ * Both MaxPool and AvgPool need "initial", "compute" and "finalize"
+ * operation.
* MaxPool initializes temp variable to the negative maximum to find the
* maximum value in the pooling field.
* AvgPool initializes temp variable to the zero to accumulate all values
- * in pool pooling, and takes the average.
+ * in pool pooling, and finally takes the average.
* MaxPoolGrad and AvgPoolGrad are gradient operations respectively.
*/
template <class T>
@@ -72,17 +73,17 @@ class AvgPoolGrad {
/*
* \brief Getting pooling results, and calculating gradient.
*
- * In pool2d, all tensors are in NCHW format. In pool3d, all tensors are in
- * NCDHW format.
+ * In pool2d, all tensors are in NCHW format. Where N is batch size, C is the
+ * number of channels, H and W is the height and width of feature.
+ * In pool3d, all tensors are in NCDHW format. Where N is batch size, C is the
+ * number of channels, D, H and W is the depth, height and width of feature.
*
* In max pooling, it is possible that the pooling region has multiple maximum
- * elements.
- * In this case, we should compute the gradient of the first maximum element.
+ * elements. In this case, we should compute the gradient of the first maximum
+ * element.
* This is different from average pooling. So we rewrite the max_pool_grad:
* MaxPool2dGradFunctor, MaxPool3dGradFunctor.
- *
*/
-
template <typename Place, typename PoolProcess, typename T>
class Pool2dFunctor {
public:
@@ -146,10 +147,9 @@ class MaxPool3dGradFunctor {
/*
* \brief Getting max pooling results and corresponding max index, and
* calculating gradient.
- * In sub-sampling-pooling, it is necessary to know max element index.
+ * In up-sampling-pooling, it is necessary to know max element index.
* In pool2d, all tensors are in NCHW format. In pool3d, all tensors are in
* NCDHW format.
- *
*/
template <typename Place, typename T>
class MaxPool2dWithIndexFunctor {
@@ -188,6 +188,7 @@ class MaxPool3dWithIndexGradFunctor {
const framework::Tensor& mask, std::vector<int>& ksize,
std::vector<int>& strides, std::vector<int>& paddings);
};
+
} // namespace math
} // namespace operators
} // namespace paddle
diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc
index 2e6a5f2555..ab933a3400 100644
--- a/paddle/operators/pool_with_index_op.cc
+++ b/paddle/operators/pool_with_index_op.cc
@@ -34,7 +34,7 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
PADDLE_ENFORCE(ctx->HasOutput("Out"),
"Out(Output) of Pooling should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Mask"),
- "Out(Output) of Pooling should not be null.");
+ "Mask(Output) of Pooling should not be null.");
auto in_x_dims = ctx->GetInputDim("X");
@@ -52,13 +52,11 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
}
PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2U,
- "Pooling intput size and pooling size should be consistent");
- PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3,
- "Pooling size size should be 2 elements. or 3 elements.");
+ "Intput size and pooling size should be consistent.");
PADDLE_ENFORCE_EQ(ksize.size(), strides.size(),
- "strides size and pooling size should be the same.");
+ "Strides size and pooling size should be the same.");
PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(),
- "paddings size and pooling size should be the same.");
+ "Paddings size and pooling size should be the same.");
std::vector<int64_t> output_shape({in_x_dims[0], in_x_dims[1]});
for (size_t i = 0; i < ksize.size(); ++i) {
@@ -76,11 +74,9 @@ class MaxPoolWithIndexOpGrad : public framework::OperatorWithKernel {
protected:
void InferShape(framework::InferShapeContextBase *ctx) const override {
- PADDLE_ENFORCE(ctx->HasInput("X"),
- "X(Input) of Pooling should not be null.");
- PADDLE_ENFORCE(
- ctx->HasOutput(framework::GradVarName("X")),
- "X@GRAD(Input@GRAD) of MaxPoolWithIndexOpGrad should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")),
+ "Input(X@GRAD) should not be null.");
ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
}
};
@@ -110,9 +106,10 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr<std::vector<int>>(
"ksize",
- "Pooling size(height, width) of pooling operator."
+ "The pooling size(height, width) of pooling operator."
"If globalPooling = true, ksize is ignored and need not be "
- "specified."); // TODO(Add checker)
+ "specified."); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
AddAttr<bool>(
"globalPooling",
"Whether to use the globalPooling."
@@ -123,15 +120,21 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr<std::vector<int>>("strides",
"Strides(height, width) of pooling operator."
"Default {1,1}.")
- .SetDefault({1, 1}); // TODO(Add checker)
+ .SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
AddAttr<std::vector<int>>("paddings",
"Paddings(height, width) of pooling operator."
"Default {0,0}.")
- .SetDefault({0, 0}); // TODO(Add checker)
+ .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
AddComment(R"DOC(
-The maxPooling2d with index operation calculates the output and the mask based on
-the input and ksize, strides, paddings parameters.
+The maxPooling2d with index operation calculates the output and the mask
+based on the input and ksize, strides, paddings parameters. Input(X) and
+output(Out, Mask) are in NCHW format. Where N is batch size, C is the
+number of channels, H and W is the height and width of feature.
+Parameters(ksize, strides, paddings) are two elements.
+These two elements represent height and width, respectively.
)DOC");
}
};
@@ -162,9 +165,10 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr<std::vector<int>>(
"ksize",
- "Pooling size(depth, height, width) of pooling operator."
+ "The pooling size(depth, height, width) of pooling operator."
"If globalPooling = true, ksize is ignored and need not be "
- "specified."); // TODO(Add checker)
+ "specified."); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
AddAttr<bool>(
"globalPooling",
"Whether to use the globalPooling."
@@ -176,19 +180,26 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
"strides",
"Strides(depth, height, width) of pooling operator."
"Default {1,1,1}.")
- .SetDefault({1, 1, 1}); // TODO(Add checker)
+ .SetDefault({1, 1, 1}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
AddAttr<std::vector<int>>(
"paddings",
"Paddings(depth, height, width) of pooling operator."
"Default {0,0,0}.")
- .SetDefault({0, 0, 0}); // TODO(Add checker)
+ .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
AddComment(R"DOC(
-The maxpooling3d with index operation calculates the output and the mask based on
-the input and ksize, strides, paddings parameters.
+The maxpooling3d with index operation calculates the output and the mask
+based on the input and ksize, strides, paddings parameters.
+Input(X) and output(Out, Mask) are in NCDHW format. Where N is batch
+size, C is the number of channels, D, H and W is the depth, height and
+width of feature. Parameters(ksize, strides, paddings) are three elements.
+These three elements represent depth, height and width, respectively.
)DOC");
}
};
+
} // namespace operators
} // namespace paddle
From 0e1f21a57050df474a4bbf47f538ddf1b4ef4f61 Mon Sep 17 00:00:00 2001
From: fengjiayi <fengjiayi@baidu.com>
Date: Mon, 9 Oct 2017 20:43:29 -0700
Subject: [PATCH 62/82] Fix bug
---
paddle/framework/op_desc.cc | 1 +
1 file changed, 1 insertion(+)
diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc
index 02aa74a842..c2e796b7c1 100644
--- a/paddle/framework/op_desc.cc
+++ b/paddle/framework/op_desc.cc
@@ -25,6 +25,7 @@ OpDescBind::OpDescBind(const std::string &type, const VariableNameMap &inputs,
inputs_ = inputs;
outputs_ = outputs;
attrs_ = attrs;
+ need_update_ = true;
}
OpDesc *OpDescBind::Proto() {
From cb2ef7d97de9b011f2dc9de060e0a641794ccbe1 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 20:46:44 -0700
Subject: [PATCH 63/82] Add comment
---
paddle/framework/op_desc.cc | 1 +
1 file changed, 1 insertion(+)
diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc
index 6ce453cc99..c4dcb2210d 100644
--- a/paddle/framework/op_desc.cc
+++ b/paddle/framework/op_desc.cc
@@ -200,6 +200,7 @@ static InferShapeFuncMap &InferShapeFuncs() {
// all registered kernels
for (auto &pair : OperatorWithKernel::AllOpKernels()) {
auto &info = info_map.Get(pair.first);
+ // use empty type here to avoid runtime checks.
auto op =
static_cast<OperatorWithKernel *>(info.Creator()("", {}, {}, {}));
g_map->insert(
From 32cb74be3ebea9c9c59602576f45086934308789 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Mon, 9 Oct 2017 21:06:01 -0700
Subject: [PATCH 64/82] Removed unreached code
---
paddle/framework/data_type.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/paddle/framework/data_type.h b/paddle/framework/data_type.h
index 55e3931f87..649899d425 100644
--- a/paddle/framework/data_type.h
+++ b/paddle/framework/data_type.h
@@ -28,7 +28,6 @@ inline DataType ToDataType(std::type_index type) {
return DataType::INT32;
} else {
PADDLE_THROW("Not supported");
- return static_cast<DataType>(-1);
}
}
From e21e5646a574b9e2fa299bacb3a8ee85472e84b5 Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Tue, 10 Oct 2017 13:55:27 +0800
Subject: [PATCH 65/82] fix atomicAdd -> CudaAtomicAdd
---
paddle/operators/math/pooling.cu | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu
index 06263737a9..4d50121de4 100644
--- a/paddle/operators/math/pooling.cu
+++ b/paddle/operators/math/pooling.cu
@@ -144,7 +144,7 @@ __global__ void KernelMaxPool2DGrad(
if (maxIndex != -1) {
// atomic add
- atomicAdd(input_grad + maxIndex, output_grad[index]);
+ platform::CudaAtomicAdd(input_grad + maxIndex, output_grad[index]);
}
}
}
@@ -278,9 +278,7 @@ class MaxPool2dGradFunctor<platform::GPUPlace, T> {
};
template class MaxPool2dGradFunctor<platform::GPUPlace, float>;
-// template class MaxPool2dGradFunctor<platform::GPUPlace, double>; // The
-// 64-bit floating-point version of atomicAdd() is only supported by devices of
-// compute capability 6.x and higher.
+template class MaxPool2dGradFunctor<platform::GPUPlace, double>;
template class Pool2dFunctor<platform::GPUPlace,
paddle::operators::math::MaxPool<float>, float>;
@@ -453,7 +451,7 @@ __global__ void KernelMaxPool3DGrad(
}
if (maxIdx != -1) {
// atomic add
- atomicAdd(input_grad + maxIdx, output_grad[index]);
+ platform::CudaAtomicAdd(input_grad + maxIdx, output_grad[index]);
}
}
}
@@ -609,9 +607,7 @@ class MaxPool3dGradFunctor<platform::GPUPlace, T> {
};
template class MaxPool3dGradFunctor<platform::GPUPlace, float>;
-// template class MaxPool3dGradFunctor<platform::GPUPlace, double>; // The
-// 64-bit floating-point version of atomicAdd() is only supported by devices of
-// compute capability 6.x and higher.
+template class MaxPool3dGradFunctor<platform::GPUPlace, double>;
template class Pool3dFunctor<platform::GPUPlace,
paddle::operators::math::MaxPool<float>, float>;
From 871a3f6e76f57432d64b0410f49277a6e4f7d477 Mon Sep 17 00:00:00 2001
From: Luo Tao <luotao02@baidu.com>
Date: Tue, 10 Oct 2017 15:18:02 +0800
Subject: [PATCH 66/82] remove unused PADDLE_ONLY_CPU comment
---
paddle/math/tests/test_GpuProfiler.cpp | 2 +-
paddle/memory/detail/buddy_allocator.cc | 2 +-
paddle/memory/detail/system_allocator.cc | 2 +-
paddle/memory/detail/system_allocator.h | 2 +-
paddle/memory/detail/system_allocator_test.cc | 2 +-
paddle/memory/memcpy.cc | 2 +-
paddle/memory/memcpy.h | 2 +-
paddle/memory/memory.cc | 2 +-
paddle/memory/memory_test.cc | 2 +-
paddle/platform/device_context.cc | 2 +-
paddle/platform/enforce.h | 2 +-
paddle/platform/gpu_info.h | 2 +-
12 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/paddle/math/tests/test_GpuProfiler.cpp b/paddle/math/tests/test_GpuProfiler.cpp
index 9402bd3ec4..d9f146f0d1 100644
--- a/paddle/math/tests/test_GpuProfiler.cpp
+++ b/paddle/math/tests/test_GpuProfiler.cpp
@@ -162,4 +162,4 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
-#endif /* PADDLE_ONLY_CPU */
+#endif
diff --git a/paddle/memory/detail/buddy_allocator.cc b/paddle/memory/detail/buddy_allocator.cc
index fdc5ed19dc..e212f7737a 100644
--- a/paddle/memory/detail/buddy_allocator.cc
+++ b/paddle/memory/detail/buddy_allocator.cc
@@ -182,7 +182,7 @@ BuddyAllocator::PoolSet::iterator BuddyAllocator::RefillPool() {
max_chunk_size_ = platform::GpuMaxChunkSize();
}
}
-#endif // PADDLE_ONLY_CPU
+#endif
// Allocate a new maximum sized block
size_t index = 0;
diff --git a/paddle/memory/detail/system_allocator.cc b/paddle/memory/detail/system_allocator.cc
index 6c9a46dd09..33166d9ce2 100644
--- a/paddle/memory/detail/system_allocator.cc
+++ b/paddle/memory/detail/system_allocator.cc
@@ -134,7 +134,7 @@ void GPUAllocator::Free(void* p, size_t size, size_t index) {
bool GPUAllocator::UseGpu() const { return true; }
-#endif // PADDLE_ONLY_CPU
+#endif
} // namespace detail
} // namespace memory
diff --git a/paddle/memory/detail/system_allocator.h b/paddle/memory/detail/system_allocator.h
index ee9b012f91..552cab4f96 100644
--- a/paddle/memory/detail/system_allocator.h
+++ b/paddle/memory/detail/system_allocator.h
@@ -51,7 +51,7 @@ class GPUAllocator : public SystemAllocator {
size_t gpu_alloc_size_ = 0;
size_t fallback_alloc_size_ = 0;
};
-#endif // PADDLE_ONLY_CPU
+#endif
} // namespace detail
} // namespace memory
diff --git a/paddle/memory/detail/system_allocator_test.cc b/paddle/memory/detail/system_allocator_test.cc
index cd563844e7..6a8558937b 100644
--- a/paddle/memory/detail/system_allocator_test.cc
+++ b/paddle/memory/detail/system_allocator_test.cc
@@ -62,4 +62,4 @@ TEST(GPUAllocator, Alloc) {
TestAllocator(a, 2048);
TestAllocator(a, 0);
}
-#endif // PADDLE_ONLY_CPU
+#endif
diff --git a/paddle/memory/memcpy.cc b/paddle/memory/memcpy.cc
index 790420a8ab..1df88a6da9 100644
--- a/paddle/memory/memcpy.cc
+++ b/paddle/memory/memcpy.cc
@@ -89,7 +89,7 @@ void Copy<platform::GPUPlace, platform::GPUPlace>(platform::GPUPlace dst_place,
platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToDevice);
}
-#endif // PADDLE_ONLY_CPU
+#endif
} // namespace memory
} // namespace paddle
diff --git a/paddle/memory/memcpy.h b/paddle/memory/memcpy.h
index 0bccee58c3..9b36182c2b 100644
--- a/paddle/memory/memcpy.h
+++ b/paddle/memory/memcpy.h
@@ -53,7 +53,7 @@ template <typename DstPlace, typename SrcPlace>
void Copy(DstPlace, void* dst, SrcPlace, const void* src, size_t num,
cudaStream_t stream);
-#endif // PADDLE_ONLY_CPU
+#endif
} // namespace memory
} // namespace paddle
diff --git a/paddle/memory/memory.cc b/paddle/memory/memory.cc
index 30ce8a82e1..5087c02385 100644
--- a/paddle/memory/memory.cc
+++ b/paddle/memory/memory.cc
@@ -111,7 +111,7 @@ size_t Used<platform::GPUPlace>(platform::GPUPlace place) {
return GetGPUBuddyAllocator(place.device)->Used();
}
-#endif // PADDLE_ONLY_CPU
+#endif
} // namespace memory
} // namespace paddle
diff --git a/paddle/memory/memory_test.cc b/paddle/memory/memory_test.cc
index 0d402038a0..2444931e26 100644
--- a/paddle/memory/memory_test.cc
+++ b/paddle/memory/memory_test.cc
@@ -135,4 +135,4 @@ TEST(BuddyAllocator, GPUMultAlloc) {
}
}
-#endif // PADDLE_ONLY_CPU
+#endif
diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc
index a9b6b79903..36450e9268 100644
--- a/paddle/platform/device_context.cc
+++ b/paddle/platform/device_context.cc
@@ -136,7 +136,7 @@ cudnnHandle_t CUDADeviceContext::cudnn_handle() const { return cudnn_handle_; }
cudaStream_t CUDADeviceContext::stream() const { return stream_; }
-#endif // PADDLE_ONLY_CPU
+#endif
} // namespace platform
} // namespace paddle
diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h
index 15d8446cd8..cd906c3fa9 100644
--- a/paddle/platform/enforce.h
+++ b/paddle/platform/enforce.h
@@ -41,7 +41,7 @@ limitations under the License. */
#include <thrust/system/cuda/error.h>
#include <thrust/system_error.h>
-#endif // PADDLE_ONLY_CPU
+#endif
namespace paddle {
namespace platform {
diff --git a/paddle/platform/gpu_info.h b/paddle/platform/gpu_info.h
index fb33db07bd..37665b97d7 100644
--- a/paddle/platform/gpu_info.h
+++ b/paddle/platform/gpu_info.h
@@ -63,4 +63,4 @@ void GpuMemcpyPeer(void *dst, int dst_device, const void *src, int src_device,
} // namespace platform
} // namespace paddle
-#endif // PADDLE_ONLY_CPU
+#endif
From 36da82550af759fcfcaec571921851dd04bc4a3b Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Tue, 10 Oct 2017 16:28:24 +0800
Subject: [PATCH 67/82] Add code comments
---
paddle/operators/math/pooling.cc | 54 ++++++++++++++++++++++++++++++--
paddle/operators/math/pooling.cu | 50 +++++++++++++++++++++++++++++
2 files changed, 102 insertions(+), 2 deletions(-)
diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc
index 5accde8b07..50cfb88bb5 100644
--- a/paddle/operators/math/pooling.cc
+++ b/paddle/operators/math/pooling.cc
@@ -18,6 +18,11 @@ namespace paddle {
namespace operators {
namespace math {
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename PoolProcess, typename T>
class Pool2dFunctor<platform::CPUPlace, PoolProcess, T> {
public:
@@ -73,6 +78,11 @@ class Pool2dFunctor<platform::CPUPlace, PoolProcess, T> {
}
};
+/*
+* All tensors are in NCHW format.
+* Ksize, strides, paddings are two elements. These two elements represent height
+* and width, respectively.
+*/
template <typename PoolProcess, class T>
class Pool2dGradFunctor<platform::CPUPlace, PoolProcess, T> {
public:
@@ -135,6 +145,11 @@ class Pool2dGradFunctor<platform::CPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <class T>
class MaxPool2dGradFunctor<platform::CPUPlace, T> {
public:
@@ -197,7 +212,7 @@ class MaxPool2dGradFunctor<platform::CPUPlace, T> {
};
template class MaxPool2dGradFunctor<platform::CPUPlace, float>;
-// template class MaxPool2dGradFunctor<platform::CPUPlace, double>;
+template class MaxPool2dGradFunctor<platform::CPUPlace, double>;
template class Pool2dFunctor<platform::CPUPlace,
paddle::operators::math::MaxPool<float>, float>;
@@ -216,6 +231,11 @@ template class Pool2dGradFunctor<
template class Pool2dGradFunctor<
platform::CPUPlace, paddle::operators::math::AvgPoolGrad<double>, double>;
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename PoolProcess, class T>
class Pool3dFunctor<platform::CPUPlace, PoolProcess, T> {
public:
@@ -286,6 +306,11 @@ class Pool3dFunctor<platform::CPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename PoolProcess, class T>
class Pool3dGradFunctor<platform::CPUPlace, PoolProcess, T> {
public:
@@ -364,6 +389,11 @@ class Pool3dGradFunctor<platform::CPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <class T>
class MaxPool3dGradFunctor<platform::CPUPlace, T> {
public:
@@ -440,7 +470,7 @@ class MaxPool3dGradFunctor<platform::CPUPlace, T> {
};
template class MaxPool3dGradFunctor<platform::CPUPlace, float>;
-// template class MaxPool3dGradFunctor<platform::CPUPlace, double>;
+template class MaxPool3dGradFunctor<platform::CPUPlace, double>;
template class Pool3dFunctor<platform::CPUPlace,
paddle::operators::math::MaxPool<float>, float>;
@@ -459,6 +489,11 @@ template class Pool3dGradFunctor<
template class Pool3dGradFunctor<
platform::CPUPlace, paddle::operators::math::AvgPoolGrad<double>, double>;
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename T>
class MaxPool2dWithIndexFunctor<platform::CPUPlace, T> {
public:
@@ -519,6 +554,11 @@ class MaxPool2dWithIndexFunctor<platform::CPUPlace, T> {
}
};
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename T>
class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, T> {
public:
@@ -563,6 +603,11 @@ template class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, float>;
template class MaxPool2dWithIndexFunctor<platform::CPUPlace, double>;
template class MaxPool2dWithIndexGradFunctor<platform::CPUPlace, double>;
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename T>
class MaxPool3dWithIndexFunctor<platform::CPUPlace, T> {
public:
@@ -637,6 +682,11 @@ class MaxPool3dWithIndexFunctor<platform::CPUPlace, T> {
}
};
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename T>
class MaxPool3dWithIndexGradFunctor<platform::CPUPlace, T> {
public:
diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu
index 4d50121de4..736327f4b7 100644
--- a/paddle/operators/math/pooling.cu
+++ b/paddle/operators/math/pooling.cu
@@ -149,6 +149,11 @@ __global__ void KernelMaxPool2DGrad(
}
}
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename PoolProcess, typename T>
class Pool2dFunctor<platform::GPUPlace, PoolProcess, T> {
public:
@@ -190,6 +195,11 @@ class Pool2dFunctor<platform::GPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename PoolProcess, typename T>
class Pool2dGradFunctor<platform::GPUPlace, PoolProcess, T> {
public:
@@ -234,6 +244,11 @@ class Pool2dGradFunctor<platform::GPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename T>
class MaxPool2dGradFunctor<platform::GPUPlace, T> {
public:
@@ -456,6 +471,11 @@ __global__ void KernelMaxPool3DGrad(
}
}
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename PoolProcess, class T>
class Pool3dFunctor<platform::GPUPlace, PoolProcess, T> {
public:
@@ -504,6 +524,11 @@ class Pool3dFunctor<platform::GPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename PoolProcess, class T>
class Pool3dGradFunctor<platform::GPUPlace, PoolProcess, T> {
public:
@@ -556,6 +581,11 @@ class Pool3dGradFunctor<platform::GPUPlace, PoolProcess, T> {
}
};
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <class T>
class MaxPool3dGradFunctor<platform::GPUPlace, T> {
public:
@@ -709,6 +739,11 @@ __global__ void KernelMaxPool2DWithIdxGrad(
}
}
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename T>
class MaxPool2dWithIndexFunctor<platform::GPUPlace, T> {
public:
@@ -750,6 +785,11 @@ class MaxPool2dWithIndexFunctor<platform::GPUPlace, T> {
}
};
+/*
+ * All tensors are in NCHW format.
+ * Ksize, strides, paddings are two elements. These two elements represent
+ * height and width, respectively.
+ */
template <typename T>
class MaxPool2dWithIndexGradFunctor<platform::GPUPlace, T> {
public:
@@ -903,6 +943,11 @@ __global__ void KernelMaxPool3DWithIdxGrad(
}
}
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename T>
class MaxPool3dWithIndexFunctor<platform::GPUPlace, T> {
public:
@@ -951,6 +996,11 @@ class MaxPool3dWithIndexFunctor<platform::GPUPlace, T> {
}
};
+/*
+ * All tensors are in NCDHW format.
+ * Ksize, strides, paddings are three elements. These three elements represent
+ * depth, height and width, respectively.
+ */
template <typename T>
class MaxPool3dWithIndexGradFunctor<platform::GPUPlace, T> {
public:
From f2e7cf21415fbdc0ae2f34b88b6cf307b37966f0 Mon Sep 17 00:00:00 2001
From: chengduoZH <zhaochengduo@163.com>
Date: Tue, 10 Oct 2017 22:08:33 +0800
Subject: [PATCH 68/82] fix InferShapeContextBase to InferShapeContext
---
paddle/operators/pool_with_index_op.cc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc
index ab933a3400..7b6afcfd1f 100644
--- a/paddle/operators/pool_with_index_op.cc
+++ b/paddle/operators/pool_with_index_op.cc
@@ -28,7 +28,7 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"),
"X(Input) of Pooling should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Out"),
@@ -73,7 +73,7 @@ class MaxPoolWithIndexOpGrad : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
- void InferShape(framework::InferShapeContextBase *ctx) const override {
+ void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")),
"Input(X@GRAD) should not be null.");
From a281b38393597e9c6342d365b3e0b7371194b97e Mon Sep 17 00:00:00 2001
From: Markus Kliegl <markus@coniri.com>
Date: Tue, 10 Oct 2017 10:53:02 -0700
Subject: [PATCH 69/82] Conv Shift Operator (#4591)
* conv_shift_op: initial implementation using Eigen
Limitations:
- both gradient outputs must be specified and are always computed
- explicit for loops => could be optimized in various ways
(e.g., different memory layout)
* conv shift - gradient fixes
fix case when not all output gradients desired
* conv shift: minor cleanup
* conv shift - more minor cleanup
* conv shift: clean up & initial GPU implementation
* fix rebase issue
---
paddle/operators/conv_shift_op.cc | 206 ++++++++++++++++++
paddle/operators/conv_shift_op.cu | 194 +++++++++++++++++
paddle/operators/conv_shift_op.h | 33 +++
.../v2/framework/tests/test_conv_shift_op.py | 47 ++++
4 files changed, 480 insertions(+)
create mode 100644 paddle/operators/conv_shift_op.cc
create mode 100644 paddle/operators/conv_shift_op.cu
create mode 100644 paddle/operators/conv_shift_op.h
create mode 100644 python/paddle/v2/framework/tests/test_conv_shift_op.py
diff --git a/paddle/operators/conv_shift_op.cc b/paddle/operators/conv_shift_op.cc
new file mode 100644
index 0000000000..e1e321ed5f
--- /dev/null
+++ b/paddle/operators/conv_shift_op.cc
@@ -0,0 +1,206 @@
+/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/operators/conv_shift_op.h"
+#include "paddle/framework/eigen.h"
+
+namespace paddle {
+namespace operators {
+
+using framework::Tensor;
+template <typename T, int MajorType = Eigen::RowMajor,
+ typename IndexType = Eigen::DenseIndex>
+using EigenMatrix = framework::EigenMatrix<T, MajorType, IndexType>;
+
+class ConvShiftOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null.");
+ PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should be not null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should be not null.");
+
+ auto x_dims = ctx->GetInputDim("X");
+ auto y_dims = ctx->GetInputDim("Y");
+ PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank should be 2.");
+ PADDLE_ENFORCE_EQ(y_dims.size(), 2, "Input(Y)'s rank should be 2.");
+ PADDLE_ENFORCE_EQ(x_dims[0], y_dims[0],
+ "The 1st dimension of Input(X) and Input(Y) should "
+ "be equal.");
+ PADDLE_ENFORCE_EQ(y_dims[1] % 2, 1,
+ "The 2nd dimension of Input(Y) should be odd.");
+ PADDLE_ENFORCE_LE(y_dims[1], x_dims[1],
+ "The 2nd dimension of Input(Y) should be less than or "
+ "equal to the 2nd dimension of Input(X).");
+ ctx->SetOutputDim("Out", x_dims);
+ ctx->ShareLoD("X", /*->*/ "Out");
+ }
+};
+
+class ConvShiftGradOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null.");
+ PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should be not null.");
+ PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
+ "Input(Out@GRAD) should be not null.");
+
+ auto x_grad_name = framework::GradVarName("X");
+ if (ctx->HasOutput(x_grad_name)) {
+ auto x_dims = ctx->GetInputDim("X");
+ ctx->SetOutputDim(x_grad_name, x_dims);
+ }
+
+ auto y_grad_name = framework::GradVarName("Y");
+ if (ctx->HasOutput(y_grad_name)) {
+ auto y_dims = ctx->GetInputDim("Y");
+ ctx->SetOutputDim(y_grad_name, y_dims);
+ }
+ }
+};
+
+class ConvShiftOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ ConvShiftOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : framework::OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X",
+ "(Tensor, default Tensor<float>), a 2-D tensor with shape B x M, "
+ "where B is the batch size and M is the data dimension.");
+ AddInput("Y",
+ "(Tensor, default Tensor<float>), a 2-D tensor with shape B x N, "
+ "where B is the batch size and N is the data dimension. N must "
+ "be odd.");
+ AddOutput("Out",
+ "(Tensor, default Tensor<float>), a 2-D tensor with shape B x M, "
+ "i.e., the same shape as X.");
+ AddComment(R"DOC(
+ConvShift Operator.
+
+A layer for circular convolution of two vectors,
+as used in the Neural Turing Machine: https://arxiv.org/abs/1410.5401
+
+The equation is:
+
+ \f[
+ Out[i] = \sum_{j=-(N-1)/2}^{(N-1)/2} X_{i+j} * Y_{j}
+ \f]
+
+where X's index is computed modulo M, and b's index is computed modulo N.
+
+Both of the input `X` and `Y` can carry LoD (Level of Details) information.
+However, the output only shares the LoD information with input `X`.
+)DOC");
+ }
+};
+
+template <typename T>
+class ConvShiftKernel<platform::CPUPlace, T> : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext &context) const override {
+ auto *X = context.Input<Tensor>("X");
+ auto *Y = context.Input<Tensor>("Y");
+ auto *Out = context.Output<Tensor>("Out");
+ Out->mutable_data<T>(context.GetPlace());
+
+ auto x = EigenMatrix<T>::From(*X);
+ auto y = EigenMatrix<T>::From(*Y);
+ auto out = EigenMatrix<T>::From(*Out);
+ out.setZero();
+
+ size_t batch_size = X->dims()[0];
+ size_t x_width = X->dims()[1];
+ size_t y_width = Y->dims()[1];
+ size_t y_half_width = (y_width - 1) / 2;
+
+ for (size_t k = 0; k < batch_size; ++k) {
+ for (size_t i = 0; i < x_width; ++i) {
+ for (size_t j = 0; j < y_width; ++j) {
+ int index = (i + j - y_half_width + x_width) % x_width;
+ out(k, i) += x(k, index) * y(k, j);
+ }
+ }
+ }
+ }
+};
+
+template <typename T>
+class ConvShiftGradKernel<platform::CPUPlace, T>
+ : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext &context) const override {
+ auto *X = context.Input<Tensor>("X");
+ auto *Y = context.Input<Tensor>("Y");
+ auto *dOut = context.Input<Tensor>(framework::GradVarName("Out"));
+ auto *dX = context.Output<Tensor>(framework::GradVarName("X"));
+ auto *dY = context.Output<Tensor>(framework::GradVarName("Y"));
+
+ auto x = EigenMatrix<T>::From(*X);
+ auto y = EigenMatrix<T>::From(*Y);
+ auto dout = EigenMatrix<T>::From(*dOut);
+
+ auto x_dims = X->dims();
+ auto y_dims = Y->dims();
+ size_t batch_size = x_dims[0];
+ size_t x_width = x_dims[1];
+ size_t y_width = y_dims[1];
+ size_t y_half_width = (y_width - 1) / 2;
+
+ // The below trades code duplication for efficiency (keeping the if
+ // statement outside of the loop).
+ if (dX) {
+ dX->mutable_data<T>(context.GetPlace());
+ auto dx = EigenMatrix<T>::From(*dX);
+ dx.setZero();
+ for (size_t k = 0; k < batch_size; ++k) {
+ for (size_t i = 0; i < x_width; ++i) {
+ for (size_t j = 0; j < y_width; ++j) {
+ int index = (i + j - y_half_width + x_width) % x_width;
+ dx(k, index) += dout(k, i) * y(k, j);
+ }
+ }
+ }
+ }
+
+ if (dY) {
+ dY->mutable_data<T>(context.GetPlace());
+ auto dy = EigenMatrix<T>::From(*dY);
+ dy.setZero();
+ for (size_t k = 0; k < batch_size; ++k) {
+ for (size_t i = 0; i < x_width; ++i) {
+ for (size_t j = 0; j < y_width; ++j) {
+ int index = (i + j - y_half_width + x_width) % x_width;
+ dy(k, j) += x(k, index) * dout(k, i);
+ }
+ }
+ }
+ }
+ }
+};
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP(conv_shift, ops::ConvShiftOp, ops::ConvShiftOpMaker,
+ conv_shift_grad, ops::ConvShiftGradOp);
+REGISTER_OP_CPU_KERNEL(conv_shift,
+ ops::ConvShiftKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+ conv_shift_grad,
+ ops::ConvShiftGradKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/conv_shift_op.cu b/paddle/operators/conv_shift_op.cu
new file mode 100644
index 0000000000..145e966fe9
--- /dev/null
+++ b/paddle/operators/conv_shift_op.cu
@@ -0,0 +1,194 @@
+/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/operators/conv_shift_op.h"
+#include "paddle/platform/cuda_helper.h"
+
+namespace paddle {
+namespace operators {
+
+using framework::Tensor;
+
+namespace {
+
+inline int div_up(int x, int y) { return (x + y - 1) / y; }
+
+// Some notes on the design:
+//
+// Each thread is responsible for computing a single output out[k, i].
+// Thread blocks are based on tiles of x with height 1 in the batch dimension.
+//
+// This design is based on the typical use case where the filter
+// y is fairly small. For large y, it would probably be more efficient
+// to also tile across y.
+template <typename T>
+__global__ void conv_shift_forward(const T *x, const T *y, T *out, int x_width,
+ int y_width, int y_half_width,
+ int batch_size) {
+ extern __shared__ T mem[];
+
+ int tx = threadIdx.x;
+ int i = blockIdx.x * blockDim.x + tx; // global x index
+ int k = blockIdx.y; // batch index
+
+ // Check if we are in a boundary block with fewer x's to process than
+ // blockDim.x.
+ int num_x =
+ (blockIdx.x == gridDim.x - 1) ? (x_width % blockDim.x) : blockDim.x;
+
+ T *sx = mem;
+ T *sx_pad = &mem[num_x];
+ T *sy = &mem[blockDim.x + y_width];
+
+ // Collaboratively load y[k, :] and length-y padding of x into shared memory.
+ int pad_start = blockIdx.x * blockDim.x + num_x + x_width - y_half_width;
+ for (int j = tx; j < y_width; j += blockDim.x) {
+ sy[j] = y[k * y_width + j];
+ sx_pad[j] = x[k * x_width + (pad_start + j) % x_width];
+ }
+
+ // Load a cyclically shifted slice of x into shared memory.
+ if (tx < num_x) {
+ int load_i = (i - y_half_width + x_width) % x_width;
+ sx[tx] = x[k * x_width + load_i];
+ } else {
+ return;
+ }
+ __syncthreads();
+
+ // Compute dot product of sx[tx:tx + y_width] and sy.
+ T sum = 0;
+ for (int j = 0; j < y_width; ++j) {
+ sum += sx[tx + j] * sy[j];
+ }
+
+ // Save to out[k, i].
+ out[k * x_width + i] = sum;
+}
+
+// Compute x gradient - initial naive implementation with atomic add.
+template <typename T>
+__global__ void conv_shift_dx(const T *dout, const T *y, T *dx, int x_width,
+ int y_width, int y_half_width, int batch_size) {
+ int i = blockIdx.x * blockDim.x + threadIdx.x; // x index
+ int j = blockIdx.y; // y index
+ int k = blockIdx.z; // batch index
+
+ if (i < x_width) {
+ int index = (i + j - y_half_width + x_width) % x_width;
+ atomicAdd(&dx[k * x_width + index],
+ dout[k * x_width + i] * y[k * y_width + j]);
+ }
+}
+
+// Compute y gradient - initial naive implementation with atomic add.
+template <typename T>
+__global__ void conv_shift_dy(const T *x, const T *dout, T *dy, int x_width,
+ int y_width, int y_half_width, int batch_size) {
+ int i = blockIdx.x * blockDim.x + threadIdx.x; // x index
+ int j = blockIdx.y; // y index
+ int k = blockIdx.z; // batch index
+
+ if (i < x_width) {
+ int index = (i + j - y_half_width + x_width) % x_width;
+ atomicAdd(&dy[k * y_width + j],
+ x[k * x_width + index] * dout[k * x_width + i]);
+ }
+}
+} // namespace
+
+template <typename T>
+class ConvShiftKernel<platform::GPUPlace, T> : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext &context) const override {
+ const Tensor *X = context.Input<Tensor>("X");
+ const Tensor *Y = context.Input<Tensor>("Y");
+ Tensor *Out = context.Output<Tensor>("Out");
+ const T *x_data = X->data<T>();
+ const T *y_data = Y->data<T>();
+ T *out_data = Out->mutable_data<T>(context.GetPlace());
+
+ int batch_size = X->dims()[0];
+ int x_width = X->dims()[1];
+ int y_width = Y->dims()[1];
+ int y_half_width = (y_width - 1) / 2;
+
+ const int x_per_block = 256;
+ int num_x_blocks = div_up(x_width, x_per_block);
+ int mem_per_block = (x_per_block + 2 * y_width) * sizeof(T);
+
+ dim3 grid_dim(num_x_blocks, batch_size);
+
+ auto stream = reinterpret_cast<const platform::CUDADeviceContext &>(
+ context.device_context())
+ .stream();
+
+ conv_shift_forward<T><<<grid_dim, x_per_block, mem_per_block, stream>>>(
+ x_data, y_data, out_data, x_width, y_width, y_half_width, batch_size);
+ }
+};
+
+template <typename T>
+class ConvShiftGradKernel<platform::GPUPlace, T>
+ : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext &context) const override {
+ const Tensor *X = context.Input<Tensor>("X");
+ const Tensor *Y = context.Input<Tensor>("Y");
+ const Tensor *dOut = context.Input<Tensor>(framework::GradVarName("Out"));
+ const T *x_data = X->data<T>();
+ const T *y_data = Y->data<T>();
+ const T *dout_data = dOut->data<T>();
+
+ Tensor *dX = context.Output<Tensor>(framework::GradVarName("X"));
+ Tensor *dY = context.Output<Tensor>(framework::GradVarName("Y"));
+
+ int batch_size = X->dims()[0];
+ int x_width = X->dims()[1];
+ int y_width = Y->dims()[1];
+ int y_half_width = (y_width - 1) / 2;
+
+ auto stream = reinterpret_cast<const platform::CUDADeviceContext &>(
+ context.device_context())
+ .stream();
+
+ const int x_per_block = 256;
+ int num_x_blocks = div_up(x_width, x_per_block);
+ dim3 grid_dim(num_x_blocks, y_width, batch_size);
+
+ if (dX) {
+ T *dx_data = dX->mutable_data<T>(context.GetPlace());
+ cudaMemsetAsync(dx_data, 0, dX->numel() * sizeof(T), stream);
+ conv_shift_dx<T><<<grid_dim, x_per_block, 0, stream>>>(
+ dout_data, y_data, dx_data, x_width, y_width, y_half_width,
+ batch_size);
+ }
+ if (dY) {
+ T *dy_data = dY->mutable_data<T>(context.GetPlace());
+ cudaMemsetAsync(dy_data, 0, dY->numel() * sizeof(T), stream);
+ conv_shift_dy<T><<<grid_dim, x_per_block, 0, stream>>>(
+ x_data, dout_data, dy_data, x_width, y_width, y_half_width,
+ batch_size);
+ }
+ }
+};
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(conv_shift,
+ ops::ConvShiftKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(
+ conv_shift_grad,
+ ops::ConvShiftGradKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/conv_shift_op.h b/paddle/operators/conv_shift_op.h
new file mode 100644
index 0000000000..5a160b0f16
--- /dev/null
+++ b/paddle/operators/conv_shift_op.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#pragma once
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename Place, typename T>
+class ConvShiftKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext &context) const override;
+};
+
+template <typename Place, typename T>
+class ConvShiftGradKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext &context) const override;
+};
+} // namespace operators
+} // namespace paddle
diff --git a/python/paddle/v2/framework/tests/test_conv_shift_op.py b/python/paddle/v2/framework/tests/test_conv_shift_op.py
new file mode 100644
index 0000000000..b9ab21a06a
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_conv_shift_op.py
@@ -0,0 +1,47 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+def conv_shift_forward(x, y):
+ out = np.zeros_like(x)
+ M = x.shape[1]
+ N = y.shape[1]
+ y_half_width = (N - 1) / 2
+ for i in xrange(M):
+ for j in xrange(N):
+ out[:, i] += x[:, (i + j + M - y_half_width) % M] * y[:, j]
+ return out
+
+
+class TestConvShiftOp(OpTest):
+ def setUp(self):
+ self.op_type = "conv_shift"
+
+ batch_size = 4
+ x_dim = 17
+ y_dim = 3 # must be odd and <= x_dim
+ x = np.random.random((batch_size, x_dim)).astype("float32")
+ y = np.random.random((batch_size, y_dim)).astype("float32")
+ self.inputs = {'X': x, 'Y': y}
+
+ out = conv_shift_forward(x, y)
+ self.outputs = {'Out': out}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad_normal(self):
+ self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.05)
+
+ def test_check_grad_ignore_x(self):
+ self.check_grad(
+ ['Y'], 'Out', max_relative_error=0.05, no_grad_set=set("X"))
+
+ def test_check_grad_ignore_y(self):
+ self.check_grad(
+ ['X'], 'Out', max_relative_error=0.05, no_grad_set=set('Y'))
+
+
+if __name__ == '__main__':
+ unittest.main()
From 805639b16c5eb8af6f689a3f5a311d389a88df07 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Wed, 11 Oct 2017 01:59:48 +0800
Subject: [PATCH 70/82] Fix compile error in linux
---
paddle/framework/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt
index 3e0e0f5903..1bf80b3e58 100644
--- a/paddle/framework/CMakeLists.txt
+++ b/paddle/framework/CMakeLists.txt
@@ -19,7 +19,7 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope)
proto_library(framework_proto SRCS framework.proto)
cc_library(attribute SRCS attribute.cc DEPS framework_proto)
-cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute)
+cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute ddim)
cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute)
cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker)
cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc)
From fb2ad4c9490925d49a2f0d9a641472137e308876 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Tue, 10 Oct 2017 13:10:58 -0700
Subject: [PATCH 71/82] Change PythonAPI `.proto` to `.desc`
---
doc/design/python_api.md | 12 ++++++------
python/paddle/v2/framework/graph.py | 30 ++++++++++++++---------------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/doc/design/python_api.md b/doc/design/python_api.md
index 6213da65c8..c4665e44fc 100644
--- a/doc/design/python_api.md
+++ b/doc/design/python_api.md
@@ -22,7 +22,7 @@ Whenever we create a block, we need to set its parent block to the current block
```python
class Program(objects):
def __init__(self):
- self.proto = core.NewProgram() # a C++ ProgramDesc pointer.
+ self.desc = core.NewProgram() # a C++ ProgramDesc pointer.
self.blocks = vector<Block>()
self.blocks.append(Block(self, -1)) # the global block
self.current_block = 0 # initialized to the global block
@@ -57,7 +57,7 @@ A [Block](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/block.m
```python
class Block(objects):
def __init__(self, program, parent_idx):
- self.proto = core.NewBlock(program.proto)
+ self.desc = core.NewBlock(program.desc)
self.program = program
self.vars = map<string, Variable>()
self.ops = vector<Operator>()
@@ -98,11 +98,11 @@ class Operator(object):
outputs,# dict<stirng, Variable>
attrs # dict<string, Any>
):
- self.proto = core.NewOpDesc(block.proto, type, inputs, outputs, attrs)
- core.infer_shape(self.proto, inputs, outputs)
+ self.desc = core.NewOpDesc(block.desc, type, inputs, outputs, attrs)
+ core.infer_shape(self.desc, inputs, outputs)
def type(self):
- return self.proto.type()
+ return self.desc.type()
```
`Operator` creates the `OpDesc` message in C++ space, so that it can call the `InferShape` function, which is in C++.
@@ -124,7 +124,7 @@ class Variable(object):
name = unique_name_generator()
self.name = name
self.block = block
- self.proto = core.NewVarDesc(block.proto, name, shape, lod_level)
+ self.desc = core.NewVarDesc(block.desc, name, shape, lod_level)
self.writer = None
```
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index 6f2a76a983..7fb72c3638 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -11,18 +11,18 @@ class Variable(object):
if name is None:
name = Variable._unique_var_name_()
- self.proto = self.block.proto.new_var(name)
+ self.desc = self.block.desc.new_var(name)
if shape is not None:
- self.proto.set_shape(shape)
+ self.desc.set_shape(shape)
if dtype is not None:
# TODO(yuyang18): Convert dtype from numpy.dtype
- self.proto.set_data_type(dtype)
+ self.desc.set_data_type(dtype)
if lod_level is not None:
# TODO(yuyang18): set_lod_level is not defined.
- self.proto.set_lod_level(lod_level)
+ self.desc.set_lod_level(lod_level)
self.block.vars[name] = self
self.op = None
@@ -38,13 +38,13 @@ class Variable(object):
class Operator(object):
def __init__(self,
block,
- proto,
+ desc,
type=None,
inputs=None,
outputs=None,
attrs=None):
self.block = block
- self.proto = proto
+ self.desc = desc
if type is not None:
# TODO.
pass
@@ -63,31 +63,31 @@ class Operator(object):
class Block(object):
def __init__(self, program, idx):
- self.proto = program.proto.block(idx)
+ self.desc = program.desc.block(idx)
self.vars = dict() # var_name --> var
self.ops = collections.deque() # operator list
self.program = program
@property
def parent_idx(self):
- return self.proto.parent
+ return self.desc.parent
@property
def idx(self):
- return self.proto.id
+ return self.desc.id
def create_var(self, *args, **kwargs):
return Variable(self, *args, **kwargs)
def append_op(self, *args, **kwargs):
- op_proto = self.proto.append_op()
- op = Operator(self, op_proto, *args, **kwargs)
+ op_desc = self.desc.append_op()
+ op = Operator(self, op_desc, *args, **kwargs)
self.ops.append(op)
return op
def prepend_op(self, *args, **kwargs):
- op_proto = self.proto.prepend_op()
- op = Operator(self, op_proto, *args, **kwargs)
+ op_desc = self.desc.prepend_op()
+ op = Operator(self, op_desc, *args, **kwargs)
self.ops.appendleft(op)
return op
@@ -104,7 +104,7 @@ class Program(object):
def __init__(self):
assert not hasattr(self.__class__,
'_instance'), 'Do not call constructor directly!'
- self.proto = core.ProgramDesc.instance()
+ self.desc = core.ProgramDesc.instance()
self.blocks = [Block(self, 0)]
self.current_block_idx = 0
@@ -116,7 +116,7 @@ class Program(object):
def create_block(self):
new_block_idx = len(self.blocks)
- self.proto.append_block(self.current_block().proto)
+ self.desc.append_block(self.current_block().desc)
self.current_block_idx = new_block_idx
self.blocks.append(Block(self, self.current_block_idx))
return self.current_block()
From ef4132051c0fc88394d75dc7c482a024fc70663f Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 10 Oct 2017 13:43:49 -0700
Subject: [PATCH 72/82] gan design with graph
---
doc/design/gan_api.md | 7 ++++++-
doc/design/test.dot.png | Bin 0 -> 59401 bytes
2 files changed, 6 insertions(+), 1 deletion(-)
create mode 100644 doc/design/test.dot.png
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 1a7d0df116..a1626e50d7 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -27,10 +27,15 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
| Concat op (done) | ? | N (Cond) |
| Repmat op (done) | ? | N (Cond) |
+<p align="center">
+<img src="./test.dot.png" width = "90%" align="center"/><br/>
+The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes.
+</p>
+
<p align="center">
<img src="./dcgan.png" width = "90%" align="center"/><br/>
-Borrow this photo from the original DC-GAN paper.
+Photo borrowed from the original DC-GAN paper.
</p>
## The Conditional-GAN might be a class.
diff --git a/doc/design/test.dot.png b/doc/design/test.dot.png
new file mode 100644
index 0000000000000000000000000000000000000000..768e5cac4b14201f378896bb3bcde9e6ee540414
GIT binary patch
literal 59401
zcmc$`byQVf*EW1mLRuuHJ%oaiD$<}HTDp;v5|9#*E(0m)5-I6Y5Cj1!0g;vxknV1f
zu5a%9e%|r^^Zxma@r`?o+n>jC_Fj9fx#qm)bzN%*sVK=16VMW%P$*(KS&SMAh2@Gu
zT}-@$3!mf;)pNrixF!lR7}PoPzmGLJ(I^x%N)B^h!!2ob%w1n&?hJ3Ui%5hZkQrV0
zah60cSfP@~N}Ee6L#<S;QY+hHu+%3m+hW+7d#tp;VtCq->+V<Vb^~>%Mz*_7^*H9y
zQaEoDHa1APqc7dK_;kL5UfkZO`rU1(lm!vvRQGwI1yPyn=nH6cFGbBT&SkR5ZJKoq
zhW4k!b#(7X0;Y9|(|_b8B_)5jUkA#nQVNWclap`#Bv=0Ex-ou<l(c!^^W%Q*pg>tZ
zO8(Khqve}-Y=<efwzhPtT(~~mvkxmYs51Gt*hAk$9?7ne>rm&iMvG=)A!A?&eR&aY
zx+RijaiUJ5_WaEAlYZruo7#n#o%zmU^LDB-1vWHiJOTN7fp(GfSBp-E&B^;eo0^_X
zHTdW1SC$-Z-G7Zc^|#p6ZL=P)CqstxWGD3;V_}gUOi4}_n>^oGV_eEeN$HPu5tmt9
zoY4SAvsc+)pxx|yer$x?>e*nSl#ULK%ob)g;ts?5KHcZ|HyLhdN9PYjt_spCY*RN<
z0_hXx8)`mf>_$0IC}YB8Z%-Bew5(x89wnm(Q*l#Fj|U(5{SA8I(a%Ev^OsBX0;9Hf
z|N9rSE_&|KewzPYxkQ_Ua@{(7K%qfZlxⅅF7B+?f?ACbr+`fJ>mcP4e6@DD8rLd
z!z*FaL9(_kSJ%S?{`)VPR8<77GK~NIjoRz6H&t5yGfo@1U5CNx{~6lnT-tT(&;f66
zZ!&oP=QF>aj8;6)HLMZ-=OMPYHu7}077MQ2f{Vck3JT&k{(<UQa=t=N&XSOjAh0rG
z%-a&fq24iJKU^w@YI#;vWAT-Cz0|s&b8CM;-eY%JPG3KX^X}!yy+sVh=Fp<VaZbMa
zU^*mMEsKDP$MD_nWRZ~)&%jG1U%yUGO}#8Fy>sW;0Fh>aRuioD<ovuvpB@1#JG;rB
zTveSqA2CC(Eh#TpKnGV>?JO#EFK%Lh=h*ib&kwiFz0Obf7yEL;xvIF)!osu=89!^i
zkGs>Q2sNL{J$i&`4rAC_NcA-tEHtnjEKJVdy&P%!^)TrezO;ET6P{sI=ac{YZjnJ1
z)lIEWKSu29K6`vAGOVHPOuQFRZ!D>=PyaeOuub{%A@kjZ5AZei{(Q}7m$l)bdShOD
z3FpNK#TAvEzZc1@60M)M=f3zA7&z%qZTJIzUTibOFDN2%5v8WCzO`09p;vQ$K~JJ6
zHSkZ?a$baW(*Ehh`5845Uk(nuRJU~ekTp2ioC*vKl$4b8(>%uI9B@CyWHKf)sc^NH
zJsaGd4&j-u+Nf=W-LO@U$MmDUU16<+qReC0`ag54U&#-1+o5tDU7&hO(0@U%)+0LO
z1-j&J75@GOy(*7rrxi>lZWW=cBwdDE*g%l1B18=i{R@h&9$~V)&ca3phfk`aGAkG}
zo>lQt!oSBiQjnitCm~<=^A4L9(z}lKoK=}J{9!1)MgN&Rvc|E+0q+k(lT)b@X7Bft
z#u~rRGXMDp7KZ}|Je%9W3zo@&^g-+mHGXCP{Y6HPM#aUp|6WQatI~l-W&A%koNc*7
zw=OyK-!D;F3Xc-Txcp~SR#N_#Tqno>JKSfh^Gc7t#k_wXsG1@D<n(AaFgUnjzB5rZ
zOP=zUVKq&%h^tnWC_3G-AY#gJvY@E8;XR)zvdE_Eqm@aouAw_~yQfG{!pS5^lte3w
z=eclTFu{4ePLRha@f(u^Wncf_l*as4ge(%5Z!{?IPT-UhCi6a#D;iB~QbA5;b@~6t
zul5E7<{G^D+>v;XG=z$avvlp>QeUoW25xnAb;rnZe?b5-oxtDkqG)uJV1a=jm$`+-
zG-N-iZ$=dkY2Ewm(XCo+WMsNh{vYDpm-BKy_ARX+(o<7Yiyp6j{l*a(*dovQr<4c<
zJDC8Flo2Dx^Ejt(s;KVlL5;_*&0`v}$XA?yS{P7oNf-#OU(YdN>RZ{h?8_#GJwk8j
zPq?3x;*tMT=tYfTc-q5&;R-YwJ*QdWKziwtWR6mDFhfnqB}nR;xxs<5x|H0bSj@u0
z-&>NZqNB;byKaQ}`JrO-(P%X#f1>@C7Fj_NYH>mh46j+s<ycPmSj(Tt)!yFzA3UVW
z4-pX&Wb1NjNntXR2(GT9kgw5%N~3)7s>;jD+ce3@I5HaEdQQ}OPc1Fsqj+gUX++(=
zv{Az6ISp^Cs;WeAiD;m%YUEw<*kAwbD22hasoK~k*qF`8oLpjgCx6%KMOkVd@hv<(
zi6PpRv4H}EFe$x^yUS@Q!{(PGlhtevUzX{F$O`@yns!Gq;_wnmK8t-;+^z0LFfmD2
z*3OI;$@<$OM(U12y}Zb!IYVR|GAyODE+@LAk{S`%vd6ia1o0L3zaL+oLTs$8xds==
zRLO|;9R*$2=~LY2uOb8Dd{8{Hs(B}#+u%W}xVQ10Z5LOJnXd*zWymGiMFH0yzC}`(
zUt%JivvBeuA<Ax|HgxFgQ&Y%Emi_sZ$PyhKxCLH<N*?h3J<XqdO+qUxD;fbCnv;{0
z?oYb6=>;8rj4I37!aCTDeW#rH75Wgu#sRXl^YS137t4R{>*>)drHUgZ-DLV_P+(Y?
zbN~s1!@osU)DLPSub1T4m6-YX$TFlu`S)W}Qo``b*^PTM<!B^)L_PLagSKXxS-7~?
zYENMir723(v<QBd%+LQ$zUfh87!@5Yg)y7&NVp*RE1b#R=XW3(%j|4x^!$eBzr|}<
zSXhoTzW`RrM##r<;-eDyOy4F+_!`FX7;&Rm(dg+~Tm3p81IX)ajEsyIBrSd?aTFI9
z^LrgQw03mF;1biy%F04wo}Hf$4i66p?DA)n1ZDO2OWNV_yLUwR#Kf^2>e=th0Y4}{
zc=(V_P*4YI<;IYa74gvo6FNN#immtY9%Zu9bs-@kH2Mk?6Avm=F;M`GE^*y3#8O!u
zd`nD+!OXtIV~h})$ausajg5^RSWr;VWSg0pNs+(NNGcHn`%n*qc_*Fm=FOYHuV23&
zZdEm&hTa>$jB=c7i+!+m>+!cosFpYQ<m+%_h^$2)jbx{do(wzzo4<acbobJI&(ENQ
z;-Q?^hAB~Be9wLESoLBzhcodMhrU`$6sg<a-EG=k8RW9du<p<I{}#?9!KkaNtMkPe
z3u;(obo6V;GFL{1%AQf9p!gJY?}aglTtK;OjC*&T1qQ~XFDD%@_GD}wE~Fwsb1+zU
z=C5t!O=(|uDv+(1h>MbyldG?ET5tgDhCYtlleBOj{#$O(gy6r3h|AT7^NE1jw#8D5
z1{cl+yHX{LEIoIZ`>{~s;^JBP34yuQ8pN&{J%OgpVNW*3YYrw~lYX%q6RzGJc<kWv
z`&_aoSCwU_zFQ<&@cD(Eg)SaVra3Fq#qKm-8luIqs(Yv(X8c$!5|{NSe@A-NU-E|d
z^dIIBq^|YcU;m%WC&wiSIk6+t1v_E}=KEE7`Ms0VQ?e^pE<AnuRP^j%<`N0XFUYwF
z_CTibx^fBvIKEhP(xN0?Tm<}1esH~CZ|Ug3M$r)Ah77l~wQ=4NuJhRC<mczVOh*^8
zGFSvRrbBJZ{JVeuey&lSc%flUa0oR|x>B;R+gjNmaw#OeKMWv|3Z=b^LJ?Bkl04p9
z<Ma5p=(#6@$)w3*-)MJ~d;zm~<?>~fS;?Hby1GxgU!^e^z*X0gI5RRb$}_C_{*GQq
zt<b2B8;yqPxVLfJ`^dTS`OI6dlda}?Ti9$dkIU_1|4M6V#Z{l~jm$5E!}m!TMJ)^M
z&rS}f*Vec`KQ50G^W1Am5^~Cfjni9XBq67y6ekzWE@y1~u~IAY<hBL-qxZ~W9{(a>
zceX?9nGKg(MI|SDPn^kz;Aub%I-^}X_+lD4KRuvGXTD+JgnpH!5Rc~Iz=r`~M@T3-
z&9zaU9IX@~;bJa+L*8F3=sMQyc}GxSaK><H$Li46E%;X!A)%q_&em3Z07Ac5!bM+w
zxce+$la?&9QGLyC8J-&pmFm5xo8~9sw#m${_~8xZO|9Jj{yW!bJlEs50RBz4rkip#
z3#f+6?Zc6_V1L55IhgWhcd3<nnLG>H(JjBTQhn(8@8rPs0EM$F{|kkl!{=*Qzs_gp
zXy<rr){g?#1qQfeLwrpQHMrJ4uKu2%vt0YdgZgkGx6tKUn{|MTb-X0T<x#O!0?8uk
zBGF}$<C&pBas?lKwQTRYUxO$=GMcobm5SponFHB8BL*_7i0p+?d&ttTjA?ph*5<?K
zPfv(Y2Um>HI2`hE(ik(%4<!4UnXITAP8IeI>?l8c_7l8w)lU!*&rB=gCuzc_gbOdC
zwj%<RH}SV4axA?un1Kcdp8BqXtG`|8?UgX-0n-|jGC&N`^NnMjoy3NQhEy9>_=(1^
zT2E>iJs5woq65+n9eCPv#ps6q1||D9j9p@Jc|sjF85qTOg-q8!yHSjd4d?9ajB2Bt
zVI=Z(5_-g7@|FAh9E=qPqm{Mf(ACo72Nk>NxCu>F=)hGnhPoJMDx@63imu98a;S6P
zVdvD$H{0@|`+`;3!nMcua<fzxUE)c!U&V?h%X$9&XEQPHe?Nj}{q*TpNaHC?0)z9u
zmlFC)G094sIs}QPmGEdgzvINa&L+FThossS4a9aRzbqZS^`pSRPf}ZKzH4&24PsdM
z-S_c~4$=woe~_mf5c*E3iUxnrC4^5n^V7M8_42II>D#6i6Xqh{sE^JB!!oXZc!F#3
zk>dlWE+biFzk}Yq_$#t^Yk$Kozqz47Nfx<l<uRL*oJ{QMy6;#vg|F-^iRTYcri55A
zd^A*=B`GNh>6TmO|ARU+mmKaTJnCM#dwK;zKWk}rrX@L$lT$H3L}neMp=u?32w(fp
zpE2*_TBW$rmtT(k{j8!1O9m6AZ8Dn@AiCP@lZn<$%ZG<j?wcy?TVh7fS!YaYc`8l5
z)I~eee;{!Mus1tT<=Yq9%inOx50Gt`RNvJ4@-o5edX+)}f(C=4H6}_jt9RMt?fv~_
zqb#mU?Uy52Oe!Yua?Ar;$oNJ#UL!w}?Nd18BwIh*Z025laQlvWE<D#tpL0c3MFoZM
z#}7~|Mo0DC+}&fp6Khwbiw!@{08CoYBj>J+T~4lDs9z5oFCr=F9khN>;EYGU*+L5U
zJ(pKjmQhk7fgbdCidYCVl6jAL$s#GO*hlLQ=jlXuI&M2WfBrI%nTMz7?}EFV8?%@g
z12;FfKp|i%mpe|9FMjx8ZSL;+*Bjs9`@8EQ_Wi}mFXQXz&b~(#Rh5-bpr70)vIBT(
zSmQDD1%~{>(n1fnCDpY^J%{9-v}sV6koVCYsNl9buzYJ)7PBNM$kM<)_y%R{fBeXd
zD?+Au=>vf)ibll6rg){bqN3s>>_DpWeGc)a9XG?E-oQW`X{V{P7M5^kQPJua;dOU+
zcPNShO=}EIs`@{0vf5bCCHS5GBB<)^wwqGlmYjv1mwKC~zePp7vChrL=wdJe8LbY!
zfX-F2<YOO<=zX&Z%GWH=8>ND_gFce|Kl_lU=3ms|kNkpyn@9ha)U&@qXv_xQ0UW&9
zK*;mnzBoQH@p1N{%YW8a?(39chBProTKWP?R7|WVTahkggtsB^P9|W4mMKdQ%@(Bf
z1PHM>T6w!rzw%W~48>@*JNeMi5XukW%?^$-RN(HFT?a=;q#yzcVD86eHzIEw!^+CZ
znJJ&}c^J+p2F=7D?tqs0JJ0>CA3V4a#v~EuwD3Ej?suw0IIuHlKEACK?N8&;w?Epa
z$@KrpzvMtF3Hb0bJ-y9Y)sx1zmVa_1-RI=gCZKudvmY0&K0iB_RaBIMK8M%)=pR3m
z(=|s866$q)eEh$)-XcEdr>^uuj@SXDOw!iYoTwjzgXJm*1D|xsA=_;3?A-Q+nXm8f
zr$kN8&1qW78ArEblhp$ZibzNZnw>RcWn;SrscEpti09w@b0|WAmqtfN5ghv_ARq(o
z>NXoGbD4??_xknm%GNlbDz=86rcRByPBg8Jl=In*3PLMFxKVd5jsRi-+hJo^ipBAo
zs^P3FLvO|cJWWi{0(PT;P(w|Cg{W|tx}dMG4;Oq(#ia+aLI32*lS_nzuZxO`I+xp8
zTkBh+uRjMUj69P}7=3;HYrM#~I9zCXL{Byw0mfWp^jzhGv5kBx4E|-zJDF%-TmL6d
z9T_?iod6MnvMyY_*x3~n9^MQHr5Jbt<d^Toy*&~N-I*P4m;@*#30}MX^mh!5Hqfw$
znv|4u2?mT5|Lu03x)zxDe(qu&=?v8{Mlnvn4Dq?SH}HvwC{SwUb_VW;bMb5f0$SVK
z+kd##$pMFc(kqjvy<^D?I}o9U;(3f}7sRVvH{$HZzT=~iORtWUcS-kO8A=&@<Ef|j
z;g)`d3Zu`#6D(#}qbm#yTm@R6qaeb{+HZaZRM!fe(rj7);}yWkcdh?K@R|N<fkyHV
zcjvW)N9?+FJ~eOr{eujP;-PxCruv>)snUtKtVRF|{qXQ?vi`MJew?gWe!%fmV}N-}
z#I@=9dG=UN&8Wpi%b^nU5NJ!v6x2tT`f|*-XIs^Blo+s>*M`fYf$np<V<ZoFe;W|x
zhli&ysdIKklZvk0AMbBBGx^@o;D|iZAl$F05L_DinuCe}aM(WG6vE)~#tuKD=f?Br
z&wJ~9eS21R)#`k_b-x(DG<*8Az~F;HaA;^O1V^IGu)NPQ#{uhQMn*fIK0&AXcP4A(
zc}??EK8xMgmLNHxB4EZ^0VDr3;~)A?RQ3GmpVig7)-i5wZn8jhnKuPfQXVZl&w>6z
z4w$UYv|0e~1-qKO0Zg>C)+vp3&P!4KM=T`*iT<8yYEcjlkN$boU@(}LZ?+>g{bdg;
z;0vPXCp*)#v#g|I{~}KIM(nNQGe1oD94kWDvY=+C0+`|;REIxpK;qW@W|M$48PI_I
z{Q0v5X2^CxB2vxf$~y2>vCx4lz{HtD#-L9af9T~^?YuIe2-VCCnl(!6{CQv;_$|Ac
zfidj;X8RafH|w(5+1Zu`e?qn!H)Nrpvm)s8&mWEc<!dD|nvJJ0OZ?7@Dp<^266e)G
zL^ggHvwGWV7UfS!9ULDY4-J>b4&ISciYP}<PY?1C5UTOO7Y=ozCSf)BpZ=yt0524r
z)}KM-ha2zdU-dVLUAuN4U=bU9FP6#Ypk8mxa`<p(Aq!?v4LGBrQmg3Uxf|;zhdW60
z{{H>Dx8k{J6~S&5uqJH)s1e9Jy|8c{XyuZT6|O+cCyI5TtkocyT8&olb&1}{SzWcc
zDq!<oP)MltX{vX&i%>8$ib&iBCMG8SDbQA?;?`f}&3T=jnTZglKSxF$LoR2BD@Ki!
z+ZPy6sgeZ;9OtR$u=4V%2)nG_SXx?=zi0FWT0=$X!f)x8VAj;yZgFtPu`4CrfZkNC
z%EeX}xEyFM%Gx=}xoW6-*`*p!d$SbO3beSappN<rJspc1_gLY2?&uhG_t`+ML9kS*
z=<thK<!hO?!Tz3xhAGgF%hBGoW`*`${;q<n`%+dsjBI3DVZC1^az1ZE+~*6Bmx_kw
zieV3n9a4o~<X2QxwZpE8wT=poOitz^6+ie09Fe@ElT$x;jt=D^QM5?ciPgZT1Op~k
zc3XAUQ69tU8+LYf+gr^{H+%GpAz2{La<sp}^I5Mf3fO-6I-gUs*_Oy{2)s4`^d`>}
z$;imIi+oSp3HtR0VM=~>c6P3le4$FBL&{e{65YCN9gnB*(^O7&sRl82cIoAT&o`l+
zmcOI@vnhl+3yKZ0fY6@CyR&M1;g;?Ea5oF;3RO8BB_&(*b!DyZ`iV4R9;MhgIIR%s
zf4G}0v|k8MZ>+2r*!rICYL_Xb6Qqw$r9?%cA){@lo}b7Bc?1J>)SLDY57OYRt<^+`
zTUi~Q_nJrL+JIunTwPstzdPqy-*pg+>-{<>EQCQ5yg80H^5KAo*Afw#2adNKRk4xS
z>^kB7H1=#w7pRv^=)(mz;%{=q8Q9o-f?cX(?yrz85-T#Ak)A%=Bg3Sb9%n-7m09CB
z7i?lXKR4If7JJhiDgZJLGP2gTwy3%G_;z(=F*YQBja9iSK#k0V{;|ZgndERLTw)fc
zomxUd0vL)Xk&*GRk)Z8vgA8d72xxkFnG@AvH1Nf;2c4&p7o|9us%lE?{Q0DIJ@eo2
zKaA_m{krKdUw8|CzaGffRPgfh3Kp9&-d`WvK0Dq>tP?*wFohv-YF1Hm>t9CAS9(AS
zeEK_y425)z5+|Fl=_Pzdy%MYU$32zpMk}rXkw}5+g#~@E)|o7#4T*{M`t^{&mX3tm
zIDYS#Bx=1R{5)^uHc@3*lx-Mh`cDjw#Z(kfsLHi5tRrkY4k6VIRfS?0M<xJ5HgP2P
zVIeig5uRSH{26}&fxnd8$g5Yc2tsFpIXO5!4hauePTM=$Te~To{M$=KNL<_yC7UQ<
zD{l~;1$!OGPcvS5yw0}{*$60E;QKYIHgc(ajM!S++oO4E4<5XF{ko~YUsF1qk&A<a
zL!jt&sg%(;oy}7Vi*(>x=4`Vu03?1vFB*}OVz@fP_4pe$3bs2b5KUQj+Bto()`{`)
z+K~$fhliATIWpmlm&t5KE67lj>+2+_#o@ANdCy8>07{{lSy&pay*u9XU4*Us5O6}V
z^O9Pr-Po|T_gDxrBG|<ZU!F96fsfrhJWQZ5{|kAmbA{RVIk){qM5+SDCN?)q$Kbnj
z{dkRMvHOk%5$&DamKh}_B^0Wrc6391j^E>-CDamQU}gRSd-FR<=rw>gI{WIaxBW+t
z9zDWFVdLWJeGZ4qnJ)Eay%(^hT^=k7Cu5PG1b#^pNSMz9Ib6A0$DDFrUZUB`sbTQV
z3}6}nTzu!W=g;JuV>l{*HmLL~jN$QF*JU;!Kw^YnEXxAElTT7^Hel*hD5yAHA0L26
zLF4@m9<ku1F<bpxEFi}F&HLB|%Q(7^l#pA6fQTptV!a)5xNU96n~d|?b(fQa=R5vN
zkQ=JIgD5oDr|e*&K>3T?7En)z@6H@=)CKfdVpaw|7eDP{47`Mr#C(Bmfbs)YL~6~#
z!s5aW(X8jkd$vwhI7GM)(-tlwJPqoH<(&`Zok>FEbOJU%06}c_DN8(oAQyxN01(ct
z{PrtQ*K?sSfWZ1RAcLi?tqrws5(?B$;1!Eu^ZY3?Vp7jhDqAj9!QwipB>+Z|@%&^#
zf($So<OyS_kAJINc_0j*?bha9;Mk~Y<f%JBqh^8An6q6@;lF)*4mNAt+39inLsBtT
z1`(GS$c$}2US6C7C92G_vL71SggaJIho$AD5+^FaopKdbZwI3;!(f5{T%`IO$wA%4
z_NzYGZjVYzx&=A#?#4aOJsUZB`730RP$d;WOiJrpLIuKf^lLr087j%D{@1IZWp(TW
zjFcJR<>{Fj7Jy^?&jxN47;qKDP7Fde?F<P41ST8D_4pShYpXSE#mdUcD-;y0Kwhb*
zGX|LPCzt`9g8XRbTXnT)rQ23I!aqWF%XUk}QRj#9rd@5T09zed&8EIS^$Qm+Ah&bz
zozQ-KUW@QjFi8}ULGvD$a}WAlPgI$;4t3v|4+3_cm6i2IkACH-*p-{>D=^)`dxz^)
z8w98wp*~k(&%IBjmOT%8&M+8Z*Y#TfkkqTkO1|IzvHT~`bP&s;U}7-Y-mA>IKT61H
z{_iOrlT`qE3i@!a7b^p?pe(R`a1uN}XqLeCgMHU{Qf!2o1uE!2pt9JsnPSZz;8uFy
zk{Lup@VA0DA~}qZ4%W8kDF@EHhaB62b6JB8p4Qlo2p|;}5DI61W}__7Vn8t|I5>Q%
zsH~K$a#=H*{PD5`qNGW?O%z7#=IJ?ddgm|XVGuT@VbL<J`m$Zh3M<QejDTcCCUR?c
zHx}lv4S<4Mft*@sdWa>$iB3;VT?3e_Qlfj53Xln(lr$ceMYyB+80PQ2@;)J9-$nU~
z|KkN9j0U)qXAm{LI(0`iy^4btX+W=By~;5@K8|qBY>9=Ny?uRbX!HZ?!9uQu-^uD#
z^dSsl9)IY!k!k{|EZ0D-a&CUU4d7aqMdyd<r6ms2U+*G+{=}5Eqpy(pHyW_|QuO60
zGv%pe1+$;weZ$i(GK__uLKgVSS!g}1<JGx6_f|QacL<&phM02Uto(&GYr@MZsy&_u
z0X^5WbVvSy%Dg=rtsD$Ywk9=Y!6jfGZ$L(k1GfFInY+Q`p<>ej_xmO5V^!_YWfi)}
z?Q33MCKcWEgQi>7*}1g8XoH#aW&xMJEyN;#SXeSvVc`!>5mfvMk4lW|FKqq$7X!b3
z<`Vnk?_V9O{`}ZQXJLT<Kyx}`)}n$b>F934<KE%vTv<&0D0}?LNBb4v#QWBNa<4(x
zzYUzbPL+t_!AenGCg9%b)zw>oBEF66G{sUNKh-O}Tw};{5s#!aQADKa*Dra<L7n#E
z(2#0Xv8JmPpK(3KuyS!JnwXe$FxAh@%-HsPZsZV*{SCtH2Laps;o&7~KHAXjUIiW`
z=G})8##t>Z4unj3SeMRo3sQQ)kMax{bMt5io-bB?><GdFO}U+rr~1chmGfwbX-ENi
z8byW%`5JlXuTQ&N-m5djnm}>|edU6rbQnFygx8K@+Qm5eaf>a(z4ft`O+2q4j1B!d
z`Y=b~Y(e5Y5}4Fr0QyMN0BxW33jgW@dHG=28V{kJM5-W?)K}TI^jv}SwwkD|(EyI1
z+kF!!hZYwHCqemc>8#!3xWMpm9_M@u(ET-!u9Z-NLjMxLv2L|nF^CKpz_QL8;|8^0
z{OB~wMivZ89ly!XO9%itI67*}l#AwE&3O+g5D6mIJ9kQdE`5g!6d15NPyfUZ3Jarx
zxrTf;I4h;4!Iww1jsvA0YN{$y<pH}RS&T*cT?8~~t(~2kRiaG*L-e6kai^uFU65>O
zY%~WMI4U6_&&4VqLC{DPfjp={IUsL1?t6CPzO#Kv@xz_sf;qCln2#P66C8Yes^UkB
zEU@9IQOM<0RaLk9wT*PwW-7`?zrRqJo}M;?UIl6Tu$ZCp^Si9-2!SPFcrI@l3GUUe
zUBS`O(T@`Nr2zj*9aG=xm|I)3-D=RM<}r-<pC)09+eSJQyNY06Bw5T;3j{0{$Y<oh
z(qS<J#E1i-O}Eyo@&q&$m%F|4^72v88ni;?z7{UJ9SErgA7#~>8T9DG9XU|1%%2BZ
z^(5SWnh9wX>0BhvPn;@;kzWOoUk?rlX!ef(^CFWiC@AJ5dF2eFxeF%?%S92_^`B4x
zzd#)Z1rG&zesg>K0jLW|9RnN)jo?S%%W}1g7=a+6C~7q0pMt`RNQ=O`2S5^<1V@6P
z$G;m87Z*^Lf4^K4c3$*@OQu6lpZmDHaQcN{K9lbW*~7O)_W==X0w|`gudfd-^*lX#
z;q|Xa<^b}5N{z>h>aAaNZfA!JQ>&}*fP_M15GL<`WTK*?ygnyx?-)d=*4NjA|0$Xk
z(W6H<T%f4ai+NB0Tf!Hx0;$^pG(MD{QqDsamCK-D;Q#^#Fxc4DMe5_@1IkxGd^{*u
zeSPEr&Uq6a$to-3p`b6FgzmHA`3!DURJ6mVGro$m+2Ci|Mc^PJY-w%9Lct_Yfi5Qr
zHXw*1GDgNQgur2DmP}>}W*{d+aIV`-=)KL3+g--+qs@Q+G^$g9JedL0$1k8cFVoTn
zr-*r)v_`SP7@8r{5t8mlW8=dupP<#qhi^UQdV702GM16wz{Db!DA1gyrl%1i4#xN1
zzQk)~fD8UhGN&&Mno<-Bm>X!xGT^_vUHfN#9OHFAljA9^`a=TKI$s1tC1kq0MuiVs
zT3UV}=n-;9M~WC7tVfwgYL0U1FQ_q_K=~L02BW!cLBy$DXtN8W3CeD$5rj%{u$U3f
z3)?ef-}|Edq(ODaH_H&Qzo!1nrE9R(f$!d707GdN8HPg8<r!R2^-t5TLOpBy{aYFU
zS3-`wfC&rAm}hU=^`y|u>cjr-pwSHUdD5^hB=mwmjIWpQDFz$^aK-xgXe#9u#ZMMS
z)Z_(fuUE<xpA`!Le(_rMvJx?h(!9na{RPGDG7U|xi!O#T4CDk*j!-I}_3no3he~}q
ztRv=~eC!p1Vdd0Hq+(-ZOMBXtf(Vwir+bfqZMqC?f_1!<#x+C$xa}A&bt`Gg=FE^;
z=Z#cwfA(D{(+}L-1g~7V^2KxC4kIP?pJ&NFbg0|;BcovRQ<}z@*Ecmlw4+gU7OmZS
z<?fT7bSZ%EHyOP)@2^hOF+oDohFnhoY<o&fj7GNVv0T#6yZSdv#3=K0Li^=ovhjgM
zr4jcMln$l!*EV#=0sZe@s+v4z*|GLqon9N5OHj{mjoaTAXuSHn<)L9@^LV2t&AWQ}
zn1ya`!|Fi5D<?302hh>=6ln95DWKCuEJ0R?jn&K!(e`Oj=B?K-tHNieZ&F=tGGuFW
z(t$qbL3<omWPE(U>GA&Uzd@@J*^Q^bl7QJEO6qtPO+E7E^Ir$NZRL=p*Fta~2Mjh;
z4r%C2yQ^I7SBJ|8uU)$avxaHvcCLswpRJ{<dQMla;5CZ6IABc7J*DQ}?sVho)vFK&
z2o&aXnt$|3r?`Q0rSWnkKbYjquU0lAVouG1(<9y|*Qnxh?gSt_+DF@wZwIg`z5sN3
zlq9IS78STC4&Mj(Lltlgnv;9W>zn#%JX6gx_{y2r6oam;jYFFfqV-v?9`-OE2|c2N
zA}&Jt`+ZN_fuio4wo2fmdHq!|B>((4I#}afqoB<%i&9BK4Jl2*`c`?$k#NAKk5WXh
zA`kfJ5z%~CYFB|`5Pe{FW6dp={WKcA#p74I=GN)#`Ssi8MD(g^YFAQxj;xSLP>APl
z`r0L4;39`HyQUc6sXITqF+ksWt@~eoi$9VnBO@anUc6`m;uT4g$jb0{|4Z^zQVP?w
zE6>q{p1c7d#5;gcPoU5cxuEf~|F6bF2|P+DG+6KB$)HKU=;iI*&m9ujcE7KWUpj+K
zNoiQ(K$j7yZe)_7cmCqCrY8!1O6zzGhM@c3m$)e(`?|BEo4{}RCneF-i@HVBD`j3I
z+9$|0zWS0;%;O`7f`4l~??E3|1Y0MQS*cxv7tDKZ>y-=tQO$RLi&PKC$Y#*2Q?|ej
zc){Wal=u{w<)ndPL}vLD1_mRa#N|X+p7xrA?#unO{6EDYxR+Y>;nvpH3VI#zzD|~%
zp;_O5Hsh;s9#{JNUCnLU(D!$&t^m;!4%{(xB|Mi12-K=>p-cD`mD_<QtGB!pR}lKj
z!S&~^8JIS<{`D!F0@yj|@;!SE_cnx}Xpx4XfQ<{f{l4$C7)|8){G=tzKI8%ukawVy
zo~UI2{sur6y5w53{4K)D@88k)?v?A?3Qm`H0QnLm!|09;wY(n4W13YAWyB1t*3}z)
zj;@(j2{2h8i(lyyKfd+r-PPBC_pkskYr+WCUS{W*$Iw)Yw3T*%M9u3kDVZede&yM~
zr+PoEi+^B3G(e+seg2%owPN8KDQeQDSt1`o`Wh!E0Za$d>e1e_`vl>V$$!ik78hMm
z@T@Wii7o6`0b5@lXmZG4He0W!egwb4Vs94Ve={K(MQJ?ivoo&<Ef4eEyOhAmLDjk6
zl_CaR+aK;CedrRmPIkJGKDYVZ)!P;CP-K+p<YbDY7K_j*x~+UJkBEseBia7<&-wkM
zY;+gs^|u~8c%YH1b`?qVFmwl`%I<XDTO9bDILt;CncUM+`pD6d9}xQ+K(CX4Iv?oi
z=@r)-LR!Vf!z)mEvXlwB4p`_ox3(Zlv!Kxzpz!{!^OXRnS8!H>gD(_qGJ1Nit|cmf
zDx!OTO<ly;c3`zlfp>o|;%b++22l*naTB?+=u3!%)$tlSXumY`<iYXs0pB%QDF8&L
zF(<C8VCaxx^8WKyQ<t5an*>~#C=`%FjDSk8Q6LQ>{^9pxo&w?kWsrnkSXgKsCy#Ma
zy6v<sd3OKTj~|kN#R~p5zYt~sYKQFdW#gH)^$oyZpeK=W8`atv=KpPO#xwFgy$d-Q
z2W|(XC^NJ_YidC)SeA&y2dXxbqCuVs_O90bYC(nP$=&9f;$@|nz>fnoA0Lmi#U0v6
z(2aafkJ3PV5OiL=3^O6Gs6It9+oiiOH%B=H8fvh(_mR%%_?<_MuiaFFdgYx#&OzXf
z)Up5#5%7qYh>87EQ&TmcF{5pJ^eY11zD<W%aB$)FBJj1d;|7nD)pJid*;h=Ts6WB<
z0oBdMID3~i^kBd4JQPAw?=vpCml4qp-_hMGY`O7wY>WYVwyzbRSy}6u&g5gYeu7iO
z0cyI5gze}OVnhYf_OliqnN<jTV-4cO#>eLie~SpCE^7q$?%fL+k$-R{KkY*=+XWN?
z=0Jm@_7p!$b{1xEbadRDPqf#|B7u7<zO7QqX#`%sGe>H9*8?%(J+ftE8M0>Ck7SV<
zzx|gmI}Tc!V8BA8Ak0*1E;FOodj8}^xVE*lNE#b6gPDvN1zt>v?k`UshKy_lNMt{u
zHT)S8JW%d+=!oc7uIC4YaU6EtAF*&r8Q<lk`h0)%f?TyJd&s}Bx~7H?zz6~dft0wR
zE=l_YL)%M{8V6>SUtsd=n4lij$OH)mag=FjP(VkMZJCPED5fCZU$vQRW_sy3t~1Q|
z&aNA^gY^%L=Q=gR)2Dp~Unj?#an3}-FAl~Qtn?r5KtvLwB#mP^1K0nOsjiL8q`hkD
zP)g}IFL^ug`K8D1r#=C)q0sa`{%3<1A|p9tGpfU}{-KPOXC@<2qF5L#OS~W1H^)SG
zDK<OreOrwT)KGDrHFS_qk<OkrQmyhc$#v2B*LwZ#R?xe^b>--%?U%_ko&R@ir3G(2
z>xmKn<~It`eQ2AiA-jX$nE|i`f!F{jzK`DX-e2MlQTfjFxL*;INzv3e2EjAa9NrAM
z-xw%VF)Dg`A|PY3s%|}4^7*9c`ryH<Y1=DgSWROV_Whj9ocW`!CeRs^0nO}Rm6+oN
zNxQK-?IFN!l93ljv(bmXMSA8{;z6>MQubeEv|N{Z*?<bblOEp=gGv%<l>i0l$;rW1
zU{Fv!pa#U$GXtPxz8RblP~rW7sZRrZI=^<jzaaw{W)s@tsm^;7Z@zxL4LJ}C1uy~V
zM&Y+aS6@Nz^bwNF``b@R04#oav7!YVf&^&FMj*y}^3<<`gooGK+A4Ka2?i3aPdYCl
zo@ha~Z3ZCTU1F~A=ksG~r0;=;--JU7KPD!apzUk`W1{@0#~7qO0@wEuh9#?@&<Fr}
za$z9=>Nw@_K0FhRpaV0=F@c~fIxlu(0lSNX^4e;m7JNR94Jn8ZLUczQgq-)hPc9%<
zBydJ$04zmpi~#hF@6alEjUm#@32l(9!mQm*?O=+pm8`h0?}J~~>}MdE34a1u0fA_w
zxPJY5<<~<(YVJ$_F&nx+wzSNJZu!H$=XUG$IiQiF5)*GG3%`g0^JZX=B?Va|n?l?r
zlx~ej86X%1=$nA!N)l2xF*P*@?iadW-Atzc2VJj^LS)9dn*H%;x_fplAU)&w0XLr4
zy;Aj6EV`8yaEMx-8XFe#ZHtZwXj9s^=h|`nK)HCo|E&AtOGGLRl(n-fJUBRz?aNl|
z44*jt3av*(cehFczj?6x2=F@4(`x{a#)jAcimEhliQCT6=+2w26<vw<lu<1(Nrbkf
z#Kc?_>+I2avL6qRj|oxG%7%e#3yO9;l()mpMp7I<Ft0JqFVGU<>Xw)VWoBmw2ew=!
z6=g=iOO{Gq)2{mHw~rq`qIcl}AOm!HkK9~5{=M)&c1rh=04ARsPz%ZeXY1<!ua9!X
z3pzM#KmAI_ER}Nr&*UinCk<~rYuDADRFG@Z%%#K)RlDEzJGm{to;gTur<c=5IkXRL
z>^*3l-a*>91{D(Gb8~O+7c`StC?FsCB?gCvu6Bv-(*w36EYpUMj`!3@1;Dz3?%LYf
zdFbKsy=e{XAF!Z+UIwdeVRI~wl=C2G0bIx3@j%*YX>Y&TtMw8DOUlRE_5XT-%_7|O
z-d|xye^;+;JcU(2`9Y5uvd;h^Nx~%l4#ImHxJ((a)owOX8%;upp!^WUzg)B*w0KhJ
zS7c;mFO35y1T4;BN*9mYZBgGdue3EF^zl%zemYr7=-zC{08en?qEHYGrEU}D;~s<n
z1DYWu-vL;l1xE3`s5?J&!&s<|>K&ytr#WzIhC_>AT+KG}6SBe+UK*sAK=Ia*RL=bA
z7LnP;Vfk0(8-}yc099x#fA>n{FN%cRVu#<S@NGoUMMPSJ+5li@1`9AfH^+vWg_~W8
zJh}K5l?kPL-djvmw8Lo;2ns}L0!6@aZ&epaK2Vpk6sQv<d}{&e>mqC$o(@oPH&Ma#
zk8TyFLf*kgxlg{t>lguCgzaZ$X7;|EvNRz+-nDCwAVu4OS!Z@b=mcbMGP13`Jt|ZQ
zNGQ6+CjJ@)T8gL^&{XC?>_=E0giwUd2x%7PF$?%xH4t(i?5rc~C;*Pc5w4{d@dOK_
zL7YT6kXe+tV)mNDFFqSUx!Q7rVUiix>US(MVa6ch=zO!e3iKL+Dc}YyH*Q=43f^sh
z%(d8QL20Mj19lM~*fnj&tFJ<;&?&Vf%hxUngWb){%8CtV9~!}#y#<ia6pXNzP&2`?
zj33?l2EfY{e2Rwx-aP}-7jiVfZoE1O;weW(0&amoDPRFZIyxSeJ{u$f-L4)|BLUfc
zEiEcYKVRIoOuUZQ%29qmEu=w8z8r~OvjsCZ@SivlzVwK11$>rQfycjyYU%83LK+<4
ztWBUA2s+MQLP|GSc~{4(sDb192>>+`eB8jArh}ozvNsbSvB1EgF0g%h?F~IeT8M8p
zLl?lWdP}eL67K7?pMwV01=<VeVwNgP``6F`W@R@uT1mDDG}8%Q|L=G*|DTro2&bTM
ziqR;=6>ftIuU?^Q<~E$D#iFNdHzg?_vfZSxz`@6tTpPC5zK;=5xik9g4=O47yp}C7
zd4$kd$WI8RZ!_vT>+|C-Vc_zGcg4>q%m)gShbb>d5>qe~x-5bd_;6q4g<nxB;Nk2^
z<BAyb^Gc^3Bd01C^W|fWn>?%QMDIfWJXwz6F!v-L`+j=*m6Q}O?iQBaSjl+TYDT!t
z4o^s+#@wdUdfn+B-7t76z9q-=be(%{jGu525@w<)6I$Mo2x}ac5M=xEGQfT>t$rIW
ze&i{CL!6O@@4}@^C_`^bLGR)1Jvg5<ImL}eY}a7aH9h=I&5gn%nLRK6dV6!NoNCPn
z_v!XCKjyyM<lf)*|5K;OJvVf<WOQhFZD=N^FT8(r2y(@bYIn)Tw~`~@4x1&2aNCF3
z(g`}#8c(4%g2nrORY?bn@2;j#?nOyM#?85Ss<pH0XK&7YC2B=yr*iOo8)loL`GDZ@
zI$Fw!>*qJcot)Khdwl;rlOPHsHOKTX`3`=CTTjWipxg(Jiw+ZCpOOOah<IlXe;fH6
zy7~p9n!~WbKjN`}TkvAM2+0%ywoIx>FyuX^44Lp>5Nh&93M!X*pEOL^3%K&r?yM*%
z;fkMJzJ!m3W_h_Xs4yW;rYT6m@SoXa^jN$G44>{dQ!i16WsNCYZtfp#u~~N0w|=8Z
z-RC!md5swc(v}Jo`P{e7s)4yXSS~1%H|l#<ge5NSr=-2@1PG+jrxBN#Bk!LrxvKw6
z_Y|T~BeH(&%)N=(fWu@WT;rwQ6IDl!q#yV+Pi~cV;w!mb<(cWUvY##+G(_;lrmfY&
z_DyZvarcFBXelIW-T1(kyo5|_>yySY%R4~llY?9AtBp+L$yc*r5y^yz&HeiW#|PxT
zRG;yslNyD9qpCh$uX3A<)#e2&%_pO}TIaG*8mu>OsgiwpKK5ZTd7&^e@r7dO{4a3$
z5E{%7(C9KCW5lghJXJo^%xil7yp^ecqM3M95|=;NyW<iIyD5L+G$%xW7!i{={cBv}
zhH!~9xAWnor0s?N&|m4{OS?aERev08?b`<ga*75J?*oFnUo@Bj4)M6PgU)+?1<$s^
zuM!<*98EXulHlark7Ffue;*jsz2uCrbSWOo93WO-f)rij>wSKJ*PW#>WySJz;pBBL
zRFnP6LkwnrvU`e=p8lPD>@A(aCQIzvGuoC;u0x^L)*s(&_ig5H{eBP-weSnp2}jGA
zbWgHq&=i<fq>v)k$nSpVn~f*0b0mFQ%54*wmaAN_y2S9_F(&(pi;}70!$iO-2fM?V
zNBZ;;A3Qb|Dk^5{utpoNCcV5VPjh<F!T%=qtMDy8)DN(p8+q?iBE*L8$qZ9qNJy0{
zG1ub@58uw7yPh8XAeBJb*+~u-{unMh#M1~Mn(|k~XrTW)Bf5<1D+6=qYt!5N@n4<J
zwCne!O?~{v8&eO+acA)g2=zOtZ&x-l*mmRcOiSHrb_5=@;~2P-sSuvpACN<PKx?!4
z`(<jptoE3j*I4q5E-JihV&r)DqJZrM0XyszH?%JUEjqm_rh2xBp&imP0H^Lhy?ua9
zarH`XRIjC;V@ro&>y;gOK33@#OKg=E7@Z1>vkVfjeC_aN#$7a=XNwK@iAvBhfQ~0H
zePH%PP%gsoLd#2eKKtSBOK`FrGz+`pezU=1^t|PJr|vn+<brG0hoD@Oo}ifRpLQZ@
zylhSYGfNDTJ+3EnpHH(9#tTx(rn&DMW3%A$xiJ(?_!^NR>a$L@8=qF8e$e$4w}$Bw
ztXU(~<HZcWI*BhYo=uU#=CmXca+Clb1N9N^K4*VRfXZ@wbb4%E{3C5}_i>7p>G;0y
z)1Jj$51?$PKq{9yKRe|r#8Ui#5^<g9AzxjyJv_r?>XOi{zkK-;&XhPF(yffwoY*o;
zjD}BJ=A{<(E%AcG6fS^7A0!nv<d9UmNa15@w`1G(eHOH$(_aUBfVRWt6ur2%5pVB=
z?17yP(J4L-uJFPBdjE)>h(l1}ZD1}v1QBIIlxlbP`Dt+sw8_wePcAJ5fgy|v{mPP?
zM-nqP3yW9l?OE2Y@&{}$X@e_UqfTA*X8ccp`$H%_l8&hcWS4{H;BwDVy8(zExDUen
zAZknHLds2}*IQdZhQ1yyG(wJ<c9`<nnwEm&NW)`ERcgrL8nSvg@np<PbD5l6eeajn
z=L;5{7iteK(1_|3(+0Ji3F%i*C*fYbawXl7Z2&n40yz({LxWM{C-6La18=vxX2Qd8
z?M6|C9&$quf?Dk0Cva*I3mlZMz&pTCytjz;D22#*IrEVxK?1<Pe67#HK(<|#laqUY
z_nFoZ?FRwkTn&7;!?}|(OK=H%Q$d#~fL|9ma0V<z>@!|&Zr?*Jk$-TGwP9q04uQsS
zV4T-=U4L=cT!+MR`sc~+q>(#LO9%G2m-?YeP)ng8jG@-+Dqv@|s&^&^O`5Cqj6^?B
zO~ds3W|zQ+d>PIzLuYWy;5)_drZL%LANEJ6(Xp4sraRQ~o~LK+td}Ea94j0Uf8^NP
z<NNd&R&?Y`BDZ>lVi(9S&%FR}A!3Gw1v`YJ)uCej%agpT%F4`wI>p0*E#g9>QFX}h
z!AaK7ouTtW1rHVBhut&+IQ*W^(b9F~+zJh^F>+MS;M?1`q+LdGQGs_trwTs~w@6Sq
z;IoZplqG?!FAzcu2e|8pU^R+}R6X(54+^B`9JL?&?hi<)DmgCi73ZcOUg30p|B>5b
zS^%QOC5d@b0q~b{bS&vVLZh4KRHjlQA|%G16u}|ZmzPSdD=tcc83XCRL4>>r%IQyN
zem^hhzw!S+UVvdHG9ktQg8f&Q)t&X&1+;pcMs3N~QQ)`+Da{2kWd<BFnh&XJn+sra
zo3TZQI>5<$II0974N+L;XxGLWat#fN=dC}!Rcv5DLHqv_c@hwa5CRjJ0L{H$zmqTG
z5?@I148u~+#DcTyKpsDolteiTkDaD|ieg-Rsb%@%{5mHqGcyVytdWytBn-mFuw%`i
z-4GN!$DJ<Fnk#x39iL;9hA~q|u25to^8!a^Xn>a>W0ThKt5qt{tB`OXVldd>otIug
zn-KdsEk$hSA{f$k1~Z=gAQ!q7K)4SFNXy#4D5RvM=#<+rAT<8Jl|fR4xLX{L9}>Bt
z6&-SKYQPb0i+vW?yGWh`Ehks^E4%bNnwQ`Quin$UDlSEVeKHfifOoPzcc?&hIkIuG
z#lRHEuxvQK3TBc?P-2>a1$)BTj@6qY_Hv{4Vxb{Uo5$T-WO={R-kQ=^#*YIZg|Gz+
zDV`*Gd3ib|W(elJ^(<4n=!6IDC`fYGQCVJI&1ZHz1MiD%Vu>vrC4Y@)3V0tGG-B~K
zv#`iFxI#7**!bocv8yL1e?Dmu<0h$aQ5+IjlP$L!LzKHoI8*?HE?rbqRIupvn>Pso
z_1!u)F~gX-(zA~eK9K7neE-ReVPNVqNuI%5T`*dI0ee(tRu)3CRDfIOkPoO(<mC$I
zXB+36P~-D-*vL3qpET4Mzr59CSM*e>kC(XMi~Jft0kr@1Am@U4Ee!m`$lDlDe#p)T
zgA~dSjE#fUK3(2di3T`h11TnXr&7SSj>^?5%WsPvO_ex_e3RbCWKBp!^b^{-V4wux
zyaP^A?GdN=@#<~l%&uRnSp#xvaU!A~<-CaMNHCoX4$A!mkD;!vJ_WQY^dfl%ZQhy>
z@3J<E$4`HTw+Q&09X<s|0fW<g$IHO?Ot)!>bjqwLT~_}--;mbV_sdDe{S}T!#a-&y
zgwdcPapE+yw$6X}ZW!wasZWI;p>@1!I&fd6jc<wlU)A`*0c1muB{q6-uY>V>GmIis
zvI-Xsy~uL)g?z<m(Is?n0v?($_<!-u`@+3QCRpf7l>r0=ZEi5}K1fX9HK6}b8S&m5
z^1LVT!ukwHsqrmVVc}tc{CG1EPl!t&*le)-U1X2OUs^hpxXHxC1O*6uL5M&Nev{43
z%^Qscte(*^F>hdBrh)zxXJmihyul$n6z9V|?NT{=H<pWgu7uU#3KKnA%z)f|r|T=V
z3<i)cHXQG0XkQV(ca(CM5EAHHA}mhL3yhvbl4m$ifn;t1eua#QDI8j{Y3OF>O|NIo
zt*nHANADwGHOuj8A%wmKMPSoJ01x=U%hc2XU;{%`2N|HIp;hIx`OAx-K(JF-$LAum
z0pSg2Z8U4A*2gIAdktmQfpG)A?4h9{V^VT*q;^BPG5QrS2*%)QKw8_4v0Np*j@0mK
zx_ck#cv1G!($iJ1@qo_pMRaYj7+4OMa=>ZqP&l=d4s07JM-L$L!|@FqkPE;yj%ulk
zyN`*8jC=`)+VpEYDB;AxFEC&M<IU#<wgExkGf@mS>=|irmmqKZ0G_)2T3C|?1tjqX
zAPKh4SI1m+G@6g_HaFH^p?ecAUFxN{|HUMTOIn4`SPs*7OBJJ-o*Tr?Vd%>~|2;FP
zhaz2C)wKR6D|;IyMU}}lfe$!YLCg$GtJ`O1lO@^|vHoI7UwKm(rc`xZ<`uTap8QTq
z0RH<bObwzaAT<J!vEfVv!V3XMXk-jU_tK?HKcMImFfcIGnwE=YUNiJLGzZd}YJ%?r
zYn;gvgGayyO=wj690db|A{hP%<XErt4h*oPLTe!odw_gM9oCYP!dd%U{2GWz*`s9P
z9<$>2hHg_h1SBNcAQ^LVaHx8oO;4ME6XP-t9$p`?EOO{b6*z`(35Os4K>|@RH_sYA
zHcKfwTWFh@NG&ZbZTb86JuLJRko(GBpB|*C_<Z{IP2egmtr~C_r#oE|qOA5+cd{!;
z8H8ivxDDD|-wDhCdBGap+Vbm{v~C9z*ZT1AFxY)Eucx?zVZj65&C&YjPfTf9nH6mC
zQlM2l;Pos8z9giKq95TY`e4_~K7O3o(b2K^npFHF^dsdSaeFt!#FEYUuMQ0lN9o*R
z=i%WYASKO#E{k1ELPR7HPG{u-hlK412lyo+7&TN>emvj)rfeGQ3DwmSNMNsK%V-J$
z90p<MK%ig$ELw`Qv%d`tR8v#C!OI(gJm~Q7hfFmNIJ>lybmqH0QK!tG@D9$mE149(
z2j>NckWj*gj;)fpIUBqdAOcJyK^Qb36>v+-gBzI3dLKfGf`&#Gj9TolhTHr5Ilu+y
zfBvkgzP%@98eA0{OX>gi?KVS3X8})yi;GL!^!n=RBgj?Pf-uOd6w)DA_`?t<R8!@-
z<NF}D<^n;?8Yv4K(LE+UzS;FwX#D&4vTAuTFgLR3NMsN&78OOsP`zUgPR?`?oI%-T
z!eR#bEvnV5`oto)6_$pMhNc@x^~10pyFUX1y|4#b`}(3WXfj<ry<QM_a18!1U5{_K
z0xV)?ZZ4IbosF!D0D*?8yL+W$42d5Y6XCGGW=ve%0OzUEcjq8@rN|?8cJ@ETrZNH_
zv|mb7><<s?Mai<3LN%$)D5V27Zgyow0rbOCuqZ2Whkc8Ku+xLG2ca67n#x4ibnpHt
zMG%y5=3r|nDXG_a%ye{gQSw<aA3n4xFe-i|YpN+NjX`xo8SI~^h^Ow^npw8jew{4j
z^!C%|&#k|Ihk@_4)0MtV=*2Px5)`4K#4z(=8-lOYB^Y|k?Ts!V=g80rkECHRUNX;^
zIXHq~c@uye=b466(JC)q+@0vbmSPF^1lJ1IM{q3V);;&L+%ViNFGxeqkpb0Bet39T
z)=EhTfBXX-GrF*l`xH!9Ud?20j#R(H-{AcPazgUd-<x?>!SoUfH+amkYecJxoWNB?
z-Qea{It_Kw7~St?49?OiD{s#NCNm0yPJ{durO&+}Lh^LhQ3wYY7x`>+)1h`|GySZg
z68vF~_$F|o;FnUJxtZAw>I;EQCbaNQ1%H44yY6xpk2A@d5S|x%GU48zJ9IKlHKe;~
z;Y+4)gV+h*^JLa5_}(YQ^JaE=lat1vE~JB?yUCEHDeAKWd6Eq7*9)ldFf3a@`Ix>)
zzp}x#R4WHA2?7bWAM(1U2|<}AZ`RPv{Y5djrau@ZdwY72L2ol;^~2BMoK*56v?X3L
zQ;#exvK>#)Gcx{_l<?PPJm<W5GX(ajkg%}8K{b=OR|{M_jjTzGpFaxauyd0|rfDqv
z#1gkt#j|QzOJDyH2?@#U%uI%AJAJ<tL_OdrC>V04IjpqmnwnBz?_l=z^@Y$fv$B$d
z$Ok>I2^p=*jYSx38?__6tLOu~xItQ9+Y&!i@Y4)vo0o$yv}HmuD)&e|M}hZIF*FPV
zMPAJFd{00g9f`1&;0Z#y&57rFt8nZdPVY+tfZ_!%hk}aA1RTC#0t<M={u&^_1li+c
zom;;)ry3FC8`M~@-G1!~co7xIk5lvWZ-LjG<4dt6m?+{_+=6VV4xDGDRZ>RevB*D?
z8d;wF8iN<W@X|xFI0B*bAgrMZ{5R3Etm&Yb>@Y}~+Yr0${3*Hu?pyModOk1u0;kWw
z4HK1>#da@LMuyz{X+g~p>C(exnbSYxg~IS6mv@F<s(*PTKU;w)bjNX4w$}8HF8J@|
zvefwb_|gDHM-&zDKw8K(&r*lTUK^{52Vyp(cPBVF*lnqt4UB&`L`C)8sS80vYc=C<
zoBzmN${u=S9ZD1cV4*FBhiuO*iG`wXPw=HIgDV|da_!y@^wd2ETQjl8y~t~YkZ!FP
zOav`mUEu%^SpV9(ukef#9!^_7f6fO%stQ0JAGNzW6b+#;H#?jAIUJ0_9t+8?F>4aa
zii*Ua4XWbc<rdOcg5E^?+`fI=3fTSGjSXcm16sK-bwx)+baNgrl;Xm$z>INicWoqo
zgN8FG{f3@Y!9m!T1{f=6*Vli`jevOt+YhpIp;zjIw2xqIjHaij2d_jBcrz3G1qQ2B
zJlx#6U*Ql6u4M@6c{cxqSzxDjP5&-t`7SgxFY9(q*YZ8E<7PeL+A2-v*%+;Cbe~vG
zCu{PCM&J(U%eAJZR~;*p-o1Ob2xlv&K85F;UhimYLn|mKI66CP9O6cUWY`Vy(N9^d
zijYB9>F5Ye=2Y(A$AXE7gm=ndF@rH5(s4{`>ICO$fu1b5pogZX&#HYM7gR_rLuO`;
zmCplUNBm7BH$UIHpFY1gaE>f1E33P&Z$?Bq@i|yEC@3lGM@FjpkJrIyhG1mCjg#O(
zw%gtT9drs%8%Px-eORD6+<)@q8kpAv*A}dnyBLz;*v2g`u3|(`XAq_UbPk8n{4_ZG
z-@l$7A%OQ+w3zXaR=JXbLHTWQG2gh`46%-$9&)6l!sk?Yf{ZQ;5(8|iPZb3{;8QoY
zw7dk5V|WAyz&L-J@TI4xe>h1(3@`AG7I3v81m4r3(Xp|#qM~r{S2+&Im_kOm4DZSS
z-4bx^3=|y1+k|`wI|`0Dlv}6#LQWQgHsY_r*#s;HVlvbR8(vU&)r%D>kgJVtY>1SV
zmFY!X9v$ic#sSF&VF-xm1uuh5;}us}IP56IpbE_(<U8hAd3Z@2G58^oZy@iWYinye
z2s@PUI$!Rl0F0jl4zk1z8r!!ToLTUeH-}-ol%18Yu;&uqzt1gGe6a*Xq4&X5K+^tB
zsgn$}hriq>6t@lrM<B2uz5)YtzkmNgL`)2(V#PzZ)6z%aVT3By14O9o_wV0XsX}6(
z=HX&vtH8O-Z6WB4&7M5D-_+Ekytc2KaIm-Nf(JN+`QpWkaJFUEg6S5e)z)D{uLiQ@
zgM$mO<WM6?q0ceAZE0n7pY_Vthbb$l`4Oq9UAq4tS??W}^V^1x-z}x3GRi0#D$$~Z
zsI<{uQlf!M(n6(F(%wTMX-Hcnr9pehYEY!DEgB>(<#${@&-e9P&mYh8deGhdzTVe$
zUgvq7$8nrXWR+1BqR|$8_bwBpz=TGF>q{B^y?x(`)1fX5f?|X^Yjb@<Lc#z7oVI5-
z>F410Q52BB#P2%cOFDMw(y{J!=nX^CCI2H4ML^A`Pq(_XeP}G7aA8@oo;9FX0b`E<
zA-*4<Py^VJh_KN&op!#FqPosiEA#A&^6T1iRI1?v8P}*p#KqNcHMo7NEGyfDSb~oj
z5UqWBX|j7Go-MJc5?TY&6sP!*G6HT}*Go(HTQ6t7DLDH^$HXLLs$z+ij*bqe!1tt>
z_rTydlPrZ0^-!)KhH@IzN3!{PA6YxTtEr)eZXghHDP*Tkz`d|xSWv~PAbgu1id21D
zdMiHOxp=lJnqNVSRmPs_CU!R?6O;Qlc|pNd($dm|;p(t4DLo~{=R@BG=y}+-Ze5dY
z5P~eCrlDaK$ibu^f_%(+`B@vTKBCg1r>BSiU%?TYM`C7W6v_pAd%UPfB=UO?h>H$2
zKi`4iN?;o7bK*oLEF$8oA&8#t0darsmZ*VKMn)M<oCe2qe8W3}E}WU+9$pP*Y~tes
z?g=?LB7_tKgxcZhZ)e*J53(*IZ%NcP#eeTOZvMTybZuN*9HBaV|Nfm|H4uGhEjdI0
z)4iXOu?kXr%DB;Q3Wfd9?0&E?DHL?iKxP*f&yo|t$wa|5BrIJFa=HW>yT+!b{1Osu
z4Gj(8#$bQQ;K0O*UwqN;@xF2gBAxKAU14_{_IH{J-q_7RdqBy$t@p|?lJ&;NZ?HB^
zj*p9Dr$G8{H@yO4{U%9E+RIbD_YiUnE`DlljlOjG^8Q1Kr4JtXArDyw7?wEuLkgEV
zqTH*R8sE9W!A(m%o4B~xD2T=PvD`7(1XL+VGGO!|gw&Nf<z(P#Q`2MUsy+ZkUC-WL
z)X=~ns=WLVRSNKy7DSM*xJw1G^@r)&Gnc;~T;%F=&E?#%K@W87S%^8Rm+2K{?;ht5
zc^(nLir^HErxBT4xT(EOM{2c$Y3rh_x|OA6Tkmf)WNjdPdp$vqUa|_d&vwwmucBLG
z0~GC@m+op-RzXC1EhdLcm!w>+>805Fx@Mu<g_mIH&AwWY*K`r+?Ym2&nVFdzkyYcn
zsiFQrJ6Rb=@&Ql)JR~1p2Bfv?=xAxl50cMj`CV4(F(-z^G8+5x^4ItNxChiw0=b>5
zR92Rhl$63bqgFr$y^e$9eNJH5!3z6);MV=cEFXgg#5RQ5VxM<QHR(EuXF5|~XN{83
zUVsi=Nc7Z;+f!V=GZ04Uq4JrJNNqc{rjw0PQ*K%btwQYkth_vbtV*A*!F?L;AYRq|
zuNNZT1M)ie1?bITL&Noby4yH{la}HW6W_W=yncO@&|Zm;!00vUxS}Fy<hUrB(642|
zX&^lJ{DOiS$fXIGQNjOge`N3W3)a^3cQgV($ms`l9WrwcoKfk4O)34M9@l<+?Rdi9
ze8hm?m*GImVLFld|9*Swg9o$_H1*3eW=+a8T33{olsG0?LDna{b7w8?gnd)T3}?S(
zgLMdXWdww8Y*?hRPmhy-PB!B1`pK=I28A{s8sWcx|K^gpNKoW_r$N<Wvl{qBe389>
z-fih#YA`BB1`79~t2ghm?(1y67u0xD<E!jqMC`M6V&brU=gyrXqN2mHj8i;vp6c=O
z@r7qzaa5#bJ@6`4H#6&5^;~7Q=FI$LBrw?RDEx{Y`bK0;LRVLmd^-I{79v5c>c{Z1
z*5kj{BY6@L5(>n&0G@e!O8AB86H7W$HUl{6(@jXv$_lEfsmVf}0EWrQlj?%8w%<o<
zi-2LE3OaM~V$;%VfkzA#C7BN%uu;m~e(>R=pCuT5&>a|)VU=FZ#Kix^_-&7a>ZX3A
zWu!=iHp5d*H*~S}MMZ_z-Gg$EN|UZ^;k|XtZb4vvAPIFNh4$$kJ=0(4RL0;nqOL$m
z(2lUI3vkvJeP8_a^PfMR&q~rLfcInfq5iuqqMR~Zs06L72CvLo3e69ALG44WAmh+G
zBzyF(p|P=gxyPK~@#CM2WEm70ApCw+U+)isk%{N;%$9a-r9%I~!G!x>2{{OcEE(D3
zpW2G)eZ?o?sz9646Ta8P6c{_Mv*^rB>*w=pqGDr}XjU<A^YHLE<zRf?Ui-Pv?U<Mv
zSE(YmPP&n2az`D)`a_ox5g2?Ll9eB=MlGO1%7^dYyU(pWl^Fq;GBh@(9uW~?Z)vFP
zf=WJCLy+Ip)O1*-9>rB0B+y+*zIRATp-ViO_o7u8>?3zqJCE&nu=!~VY6WC$o@%z7
zh;6nY6$5is_5Qmo)92JSiJK2-;ZeZIH)CUKUAG)|`_sb4&(Dtw+<+&aS2>a=CN(cl
zGaw*<U;r>xScBqHX}W$HKS=BIyw&et3EN$mmr6Dk7$SVp>dMIZ+DoEOTD^Yr#uoiG
z0ghlBq=pyl|LV8(U7YIg-`3bQ@%aj~CKkTJp-+yw#e9ahXu!td&NFk=>JJ`2*2c45
z0(w$0DAfi25?PHi=H?2>NQY!2RI|2jjNBw0f{*<<UxMta=xV&0jNXU$?@97%g8Ih{
zCJgOQZ8*X^ycA4LHzOIJ5*CpSNt<V>{K1{}`P|#v1XCk96eOy<k%vu#>@&@s`wIKw
z!P`S_)8>R64m#eDY`iL4Xg&0f4Tz{DH6yotP<VJf`cJTKC1sLUBp>ZK>kz);|FhV_
z-b1nVa&h)sE2u$QOP}04E6WZxL*9Wuj(pC#Z%b)&#^)g=xx~Vr&WJ7h74ZhhI-&LL
zTVXX1&L9CAC^pmm)rlAEyW`sdaDaNEF2K=>|JiiVsB)7EJ}9<bII_>?jJ>@kYRJjn
z^4$2}U2Cg3=b#}>($Lf_{pM3Uz@vVBNaa*w;k#SNdfrFzDlt%ICwuO~_-$K|5+ee~
z7@CUP!H=a@K$!v-IqMJZV}osNZ93>|0M|G)6VU^d135+-I5O;%ahUAAI#Y1&^n-}V
zNFhWE04r?}>;Igcwg2#K+VloCvrM0-l97>7_vI=JC+}Om>CsX#vCkH+Ae~`kU`WAb
z5{oMr%&JxdJO!`^^Wu3$MZ<~ZF}@F<sHiCF#x&@<S_u(YRLNA&#N_B&K@IYgxxZ>|
zM@6OVdYjX|Eq#)cA%hps3(8&0jT<zs+>XZGx8Cb*QdHV!YbuPa#Rk_2DZs8?y-Mrz
zePRNl!V_TO%j;gauq*E7&8+lfsEBoOz2+rJT<^BGhm*_KudN{c+kk82d!qGUWVL}E
z0XZdrt?Z39Yu?k#PVW5LUF!0@vNh<K$w93-nIndVaSap0r?fYBW*uhfliZh%GU3xH
z50=N}zc&FoaGLDiPx$phT8UMpF#t><NSTjd`p=#{%YX9ZI&$>4@7`IQIYS$*PK)zL
zQP9<84`M*_4dRgs>RnVraL5Tj2aS#W*RNkuh9`o*Mnsf&f9C>6^u7~)DZ@K>BS=Cy
zhBw!b@)j)<wWpn+RbQ{Hln2lk4Ff9VyXGK^5mtu;JbegSi=yo?bRgKMf$##LcBoho
zL;=wLXQL!#8{gH2)?w;tm^AS2-pv90c_iZozZ%8W)#rx(>*JQ&SL%{W0_@H^n671G
z$L0%#V<sW?^YhbS)s>DOWyFP=0=4C?5sEpU?1?e6l9JNxJ9nzzzvtMycQ1Kv7;qwH
zu`FHe**4z6zK#0IByGp|tqrB9(O_*EiF|(mP7Y)DNB+{cx3}+e?5B{-9_bmhZp7*q
z4M`u9C5t571r;a-$r__wF)ADKTCP5`BfLjlk)uzSb{HrxAxWTrWFT*`Lx?`V@Zr+)
z>r_!UX^l^0%b$@&Bl!kQEs0zXjNHKM*QxrVxuiM%9B?f?$to&Z-OM9>o*}@Tb!3`b
z39>oUu%`tD2Hrn!(EnZXP@$6uymsv_5q3=f_)!h+3j4;5uNXZx1I;p9>tIgp^MosO
z*tp4+xmt419wnglcaL|wxcr@)OW3qcgt4Q{@Y<Ko>!2K3VQ-Q;6BQ6*?mJh302*8>
zjQ)yNYLgm+z%^0t;zwtGQliJa{|ZuX5T$<qp`9~7oR3dEU5~Vdci&-#<SNj3*E~m+
zRBV54hG1m(olJb!8r%<t-j)}|Qqm1&&ObhG#_~3aWy_(?{OLVMv95$D=e0B;LnJJ2
z%d?*rBI+D9rp3E14E62>(^J~pWaLNdIZYPFKl}1+V@(Q*q;bFA-si-(viMCL>L8!>
zzpp4x*WU~8?h>@K`!=HertPELf*0R5VFvXhw(cc+1!QG6gA7;);-%Bip(7wLM@*?}
z>}2<J8uc8SoM1`ZS8cYWk@oQ6Ll{<NFRtBskTUeMVXMCIxKT5M$7P3H^xa9%ABCqc
zB7Q3FT~9Sjlj#0X*-`#EezozBT6^+dS65S^M#y^f$Y>&XZc4t~?Nb{iQ=6vGyYHvZ
zrb%>yxDH$r20TJeBN!(?xPxuaVuce9j(RZ_v-z?Kr}WE)^y6<1r|R^h?VWSzM{9ai
zcy=D0+&rbR(Z*{Z3n%z|gpiH=hKN9MX0cY&3e)vwv7_Cw(P8Ga#S6=46KcVRr8q*e
z*`9R!)~ER;*-KeV`p1~yt#&aagS!)sUZe{`Owsni!tvMaEcp~=@+oSQ1Ao2TKQ}HP
zLC7MFXJiBIzl7MS{`Lf}>g>GyhDYUo*#v{HuTNuR*~{icsxsG6f#RsV)0I4cB^pq0
z5{CE{7d=^_`JC)m{I09Vou3Z$?gJvyt~%+wKP8Rk_FWn|xvFF0?cr^i8reH1KYr}?
zUoYXWcpb$n|E^ukD5~mYr(V>$3aC<OPj^eZ51l&QZP(iP{?IDeY~+M@7ZqI(T(h}1
z>({hJPuUz3=Xi*xEKp5<6m0{zN$YTGCG0z;D)=R-dFO36?sax}4yfHA*~R-0tHHHv
zRrTqQN{aU&w!v6#7myYZ8|oQlKlt<q)+I)Er=@-S?J`Q)IoUiiG~|bXXK?&@n$j{|
z>xgj)n}oFCX$cm`zbn^3-$i7g-{<vbAGlAot_N9#Fg4)6836*S=J?|Kx0ZUiJAIVG
zCw)YJOn4n*cg3~M(A2zh7%QJ^<2*_OcY+$oWw6q?)BW@EL~-h=IQ|rd2{xCyD#@tx
z`ios!h~QJl&$PUMuLwA@PghD6R|gT-Ag4~;=j6No%1HLVRch*Cy%KeJl~Vp5zRe%d
zK_HwagjG+qRtE2t1tHROCDIS1_Y!;d49arwoB!K;&eloIP$ztPRMfoZoFTzlt$@&z
zQ}_co*nRAF>~vbK|Mte)nbPOapGGgafX!`=@aKlE6PXOKx5FYqd0KuuqrdO^xxH^|
zYiZDw6W+1IV$@Ae6`PqDxuFmvST8h0crlxl`Bf(`Ja~TvpLZiD>h|qM*L8=pA3m%D
zt&y-S2=V-3{vMIF|4wKVUPv73RBY7tJ<>BPOP7Uq+m`tHQWM|{&^FA;eSywm^;i&y
zPnnv&nc%IsAn$q#@Nh|4Sx#oj-%<I?ecCLjJOYqJ@b1~OmauU_2Aq$mvk|W5bk#KS
z<MZ1_`?+yMHY5dRhor(9w_#}M4}A(7D$-v>FUxVfZ3n9VY3|@wvUtk0(E&*|r~UY7
z`MEaQRMM+S)gvus=j5bc_z=--C*ta=pmfWS2kq+Gz%_wDz0q4?K|h8(f5duM)+ZD6
z3FqK@CM+)Qu%Ke->bgyI!rP<5E(pyAZa@zNO5x+@x3Fovc-6{;{6ssa>wc4ceD;Gt
z0s$LZpM>Q9EQ`Ba8~0f1;jWF~tx@5H)xU6`$wy*4Ha3Q@YRgZy#~p^t;z#{5w~W1|
zP=Fs`V|@BKaM6y5Yol%53t9FrgqXm#Zk=D;&f^2f$A+(bp?sKR@S-Y32>%u;JL5?R
ziFh+6AA{6se|=xw{USR_XG`>(VdD}<UJw-mKoqgSj=*N=+-B6hzq`4dXLG51a%QFw
zkq6(5i%UhOgPs5FF<p7F$B7dgL8KziEpYCj!M*iGmg9r!>XkW#rqJiNI9t-0gfyQ<
zdwgI4Wz4zsPw@6=^cHfYr7jrvywacs<huqa1@VAEf`NUK$nmGqEcUXhHI58RN5QjE
zCr3vge}6h;w!X;Pw*i_W)`iGpa0RzKwbn*O-T42@SLpkqZ}0uL(?`2!Q|LFI@qEcv
z_&>q{j?L@PFd%pw!L+FMDHMBq`y~BmZ;|=+4-V4cF#cJd>l>?SbqHae5jN`HcxoLd
z=OB8}<j$d@0C0G1iH#YGhTibYgu8cVVsyFT01-lD%E-Nm+^=gP##Z`5)4ebL=zsxP
z1zez@{JLhP1kDG9)2A!rK1iDZE&=C$)3po#pW9ycLk`~Zoo7c-Zy=>|pJXb|_Bmcb
z`Yt%lBrgDbv-#jNujQ+66Qr2{0V%t=l{9FJN+ziNcX2)fWITw*u-iy{lVVcL2?*mw
zL`1gTdLs$K0|J$A5YQ}Qm;_)OeAlF&#JXw4Y5B{EoUYH3WgmmR$O4!8OrFN~MRv#A
zH?ypv&(N^7D6l>TS4kTi;q&Lu>)`Q2LKwq%4sFuU&^U}gL~qX@9W$4N>lR7=>;HWY
zA#ec&xIXr3bGZP=k(m$=_x+>d9q3kA-0*%g?2IQGKvw|%;8mD}(O~<RdFCLAV-Gpv
z7!EV&MZZQI`PNq{54Z7^x?f>P^LOpqW#G`#xk-4}!<|33lsFGFf|I{gr_!I0un934
z-gB#lx6Hni&_ucV=#|Ul`YZ(rUni#G`3OF@zjNMy=jwhjvr9WP^>mg^Hb2}Yoo;7?
zqEfqDOGjV-2BsUBqsG?}?l5X*_BM7JW$W@v`AQxFyG!PFlCJQ&cRMGIj4pdS>vW@+
z?}r_KaK@ay^kf8)*uwng{8X|<LI{ZlqTYrdVEPPsx8i;gBmCgqc#wBm-cwKZ=-bKR
zeuNRzp}DFn*m<bKa4Y2&c?anF1N<?q3i0>=uQx_`bG)WLkJjN;eoIMn+w9*1D+uUB
z`W3EKjAkd2{@b^2o5P`z*n^`})Yd6qo8@aSU&XsuG_7p0eGm2`rNw+J=__NY@28|t
z)~{cmm6b)IkZbVaLuLxLCCR3>4;@<fvVl)VjP<~Q0}^}p1_TEmd8&ONCo2n-?w!eQ
z-hj<@%*^1&>q-Q2x8e0Is~OBr`!6=!(|jXk27`K7mR$#>5U^eq43EJ@BZM!4{nped
z#);`1pb<W*r8?<QDedMB{F}5ZTwGi_Iy-AVe%t^sMG5Uia|ee=&{?V*8^?lwSN!>T
zY-n_Jr#DZ%r6k%YB*wr$j{>O5_UkPli#9B@xcFq}invkt-^{{7(zgLWiw_`cR8*7?
zm}v(+=h-MJnVFfpO0bxq3HjW)vk^I4T$+QGl@%hYg0}V=g`SvKi_6xik0Q53nVmct
z27G5YsB)zlf@eHSKhphqka?ox;?hrc+MGR02Xr0x5E5$|WH3>$&b$uILctEsO$`{T
zpDjLynFH&xD~K3l*zv&8ho(>0&1I_hkg}TEASx-sw80ll(GfPcwOvit6U=zfTFCSO
zn6%NYT^qY$RtRqe26unv(!9On5oDk#=7)O&?k@GCM-`DWpl*41^6ABM=T?E_0iD`f
zXpVdzJ-QfLu$%doa$85<b!tl+n;_D_Ko`(c%_1QM%?4Bpwdhij*+8hS@c5)e!Ok_R
z07e@=m8Hpd>kYAIM<zlI<{J5`h_ESXX*cjwR>bfELcH$%ayii!1~^|A&CPxAgHM$F
z=roWPpoS&Q0GQf`qs_K&-#$3qGZBHXyw{7A;t9ic{3b^@(h!?gINQ|$6~pUZ^PklS
zvAy?HJHz#b(-qBXgd`qtKEB8@&fNC%qlFji8sfeNl$cDbAk`Y;;HEvh9d_`M)qsL1
z5XjDY<m<2%Pyy!#qMpw~w!8tlG6>#rjl<4F!Ucyt8Za;Eki8N?EPR*(NtX7*O+|Sh
zDAS<85xp4Fb>c@$XjhU@_E!u_PK5P4SiT>+JZXOfUD;<2eXPJYh<E-NKw4<geF#%)
z;pgx~MLx}CP8SRk2_T*=C|hg6$0Ac42s(rY7xKbkS-uYu`2)y-yaoAc@Wlz)ihLVV
z(17<qAY-h^@0z7`i;Ig4p#CAGI>GD|cdiAH1M&ut5jVIV%-}Y!zc(?uVgL)NdJqdG
z;p9ntUVuB+;iUBUw|L#UvNtd?at$$rhLD_G9{}u(%Px@)%zQ*cp#_Px8m#*j*?hZ7
z-np)eLsYOr6e9$7^mK@c2G}wkpFjWL)=YCpc7|7shTqnFD6Nn=^#d3hk==ls9^3tA
z;`1%eNS%V!fqKUFl)BI(O?rtn_rbs7L4I}^ww3b{?dK4<&>HJUM};KpAYla|5<cM;
z*jQv$e@`jlnNx@N%FNbZQ+-=eOA=Td@h<bpX^+%Bl9JWPO-SqLLTBM>SQb@bN6kmn
z7xF$VaZ!)m%Zk3xRw5e(FOPTc-VM<656S91pZ;j-zzcyqdS+Z`!vG9IeN@PfXScuJ
z61oHm(KPxmRxqIHcxJ!3tKeDO>{u%eB9z`m+bA$?Tm>>%*gTyanD||JWnD&?;l?%%
z`Yb<iQLD<22~EHsFbN_sJ9`Iy_k-@_B#VU5^>br3=54}S2^$s*HB?nsow_z_2q7^Q
z#9;Pua$9>Mig!RI&c@Du9VJd;nsSZBg$rS5%oFEmQa|@Oak0D%d4B77&z-wlIL<qP
zPegi7#GWsxzI$2Nz{qF-cfm^{7(9Qzb7&L^)fXU9_5aJbn|`hi#@;5nG$?BR{O&l+
z&z#0cGGoC2E<N>DnnUvuKm9;_C2V4d>kWCXR<8#k3r2LzG8elQa4W?oe}Zd@CrXDc
zf_4?K8gA+YZn2Kwo!F9T1qCrry>36#{I6K8XH#0W@oJh(pKj7Lj7X1_a=P1rd$N{X
z*Ir3%;sA6!IANN&D)ZW(rM|zaa4N?+YO<Br=&BF#ki7{<58S211PB$*KJXPlk5oYu
zAamLH#0d+S<95K1BXe28{#JkFXRCU%Wdbmaef>&CZzay^g1n+Fw?GRVAwNIg2mHG(
z#wgr*BpX16hkXQ(Ej3_pPcThuc;zb**m+Eq2ZF;}Q&&fem!bkX0P2zCg^rmEUC&rB
zxJyDxst$~ZY4p%?hF#9T#lrpo(#0nvL~pjQLaeUvuz(|l;CkT9=|gy`08+u1hs#Jq
z;OoqWCYBC5a_sCq71!<}Lzf;^WBo_<2nxd4YyABA4kU)@o(IwV0O9jhZ*LM1(cmER
z7vy?c*1o$p92+EQ>2{8L$<4cWk5Le7tP<zy3lI4w$6K_C0~Q15m00u_-;Zb?aH_L^
z7K^E=sTF+}+!6cA5iG*Dm;<sP$Hc_s3C(i}8fh!bQ{FTb{3c+OLFiU(C@Ly4hH%$;
zW#vTV>R{ZBq%B$Q_NLJ6VM%fAa`&;LN6pR6t4=Jzz>7lh@K{<kb^N9C8|)iO)mnXF
z#4K70VcSy>pykP3iXL=z%HbPmQ|J6iOrE{n!Ug{!PX|gZ1t0ikxf^_i5MW|1A6F3l
zDHL?}&O1C`p&R&N`s>^10~Qt*DheriP`60H5{!(a1Azy}vdd-b+8C@YbQ?9GAWjK~
zt+K4YXT{5Kcy|madp;nxWiH)9OrlU=hL`pG**CSZ$YL}3%U?i&O}UtznU=;+RK6s>
zi`^nkb@xdt&F#0|m@Z}?JdOj3aF?lXqp`VR?!n&~pKnY#6!yT08wwsrj5+<#-f-mz
zt-pEa&Z{F!04@bVMFg>L-NubahKGlVRmJ;HpM)rvki`oY8yXp@QW~3@v=$t2OB6hN
zmH|H0@JrHiBrMaMM~?*g$iNcP5@lyMJbpYD_#g_SvC>Fse|*cY<Kw`mO=6QjYytWv
z1Udqeu>QiW8TOX^b5|MxFyLND?CI?Vdk;$cs``2|BZ5q65j$DpgMljlSSc=}tm_!-
zUAjO<VTE*@ia_r;<|^o10`(4*>2;$)d!+_xO(e#BRK0ymkc5_xA6HQf3=GWSV28AZ
z#391)M;JXg_atL_ZjL~ixD-0V9&3F7Wr`DtKl@PqDb;ypWr4Th+AGHdILLSzVuE^{
zo8mD?y`@Opm%-x+=Z7lYm2)PjEl~hRgQSh;|G^#T{uU+ZKeGbtZ8!8n79whGwgKSD
zB<d+7jlI7wvIi?SUpuCCvBz$`;;RQb!ue=5!W#BTO_S8!{wbZaCGdj34#6e^eDWJW
zifyU%{%bUGI;c{Dar+0bF+z|9;_i<mJI0FNm37XAvy9}BkUIu|Vu}9ed_?U(dyQ#$
z)B{am8EaK?dmg+y;l}zoqmE{?ES%melijP$%*;TtxD$K%tHuXNK$soMXEzYNIYB@M
zB?tl#afH{O=_Dau5zYX#F^FHV5zKx;6>Mwf{pSZg=DkpGx08BnYM;jDsu0<iXQHZU
zcsyUa34-^ND0giWMFL#$lO>MVN0le&*#U@M#<IUe2_FvLt4e8Ntc8j1de>DHb9?)6
z!bQQ`C)ez!)83-DnZoQ1y}abejf!{LPp$_DJDa~c1!^qSc*SsS2UiAM_W?LFDPeFP
zxOv32qr$4_2n6An1h@uyMF2jO)G?&+=Q8TPvkE{ldf9+F!vFm7bOVxtgKp4uc=LoA
zY;YhWn(nlFOoA|}#4y|Jwd<}77Mx7mHo$4XP67##0EI?=aWPTPqlX7VWi)0Yu>+mJ
zX62KY-%7l8(8{_gq$U|w_GF+J@5n*qiE$VJKukVhN3f}bAZ!&Ho}Td0zYzID@Ti7H
z9a3rHb%zGi7trAA(Apv1aMRr3wu8n_4uMOhxv#-~AzU-y7&oB#9>n$NkDe~ZUcFfu
zIgtSV3?pHeL+DMKZ@@Dwz^_AoU4xJ2jaHwJXe^{bVxbYVR4Q`|@B{c%j>zN5mmpOJ
z%r^Z<l?_7p(4|PX)ZDzhdi3>(H#AJ$S@5;}kP@t5Y9?5<tI7Y3-_SL2!}bt9a~D@N
zdL5oGb#~%>lY)wnCCMV=1NqUNM>)ZD&`pAvKGwZ_Il~?60$vM&h{3=yx3dd{bIbrb
z+Jv5owS0mro`B$@4rkn|12{B72{MSCj92x?2OAw(05!mT!UO`nnpnahwvq`)gWBsr
zmB-!q5?Lsz3h?%Upi;x+j}YzAxauDtf9<LfK!v@UCJ2f;Nk7p_S%dl&FeE*$1wxL&
zg6Zn()7aVBktPuOe&h}yR3J<;tw@sz2|^kbg5tU#3<%Qi#LLn#GFpZG2-|sb=13+;
z29CI%3~;C3IT_Xo+CDs+Yfzpbla@Ss>=+?0)ipE>;1lNItpjNf3njm>&<}GwP&N6J
znz5;g6@+p0fxdX&!TAtHgPJtCkhvgjrN@=7j;WT|wFHdW5K1iJ$hRduDm1;Q5v=#N
z*D&zxoMawFOH2a3&%icJ7AjEDqJhhc8K<D<%)@94YM`W5vijp+zgDAgMZZ!)Qt~>P
zfzS+Sr>e1$L7^x7_^kKi?Ciscx;HA}!mpHcCphBHouD{%9UUDo#kUYLApxGoPeon0
zaG?d?g)l}*zXpPBY{ykJG&epj>Pd!m<`3|2vy?oaxp%+kNl_7jm617r>R_U!BE
zOUtIXMz95bAOfU9tRgHbs)8B?g_43YI4mk)no<;w9b>|sK_R}9ipP(0f=twqxv(&e
z!5IagPOE_cV}9{s1Uds`1xW|7pujTs#O=NQ%TgZhKkoUsE8}8=m}s%i{}nN5{#Ojg
z#89LC*1q}KQ<`;*o-&Tx*w#JBC}ie8d0`;RO0lomOdnS@aj|@Nmk+D2uKjpvsqC_X
zecsht$uP>3fhI{!1lP3(-KHrcBO@Rq=g99DVa%%BDJ9Ah6lUPCFMu7u<HqnPu=kjl
zLPriQOn>>iY<7%keq{5kpCuO+iiK7N@V+EuWR5QfGtxQ!vgt{tY`aAxDXAb^awFch
z&)CUYig5E1WSjvly3OZ*St=}o@JwG@B7mKMdf_eKqdhKNGL!T+&}7NKyi-RfCCQPQ
zU*TcN)8%eyMXW5k-iF=-sY;jiB_q*3Vmw$|sQcgF)*$T0$p}fGII^0JlV0jdOYzB5
zt`<dJVo4SxaPI#Y50qQq)^-PY%?~?cBY3X;PWZhvcfE4u{E7d5{r-;|5a{fc{eYq&
zlg<8J*D%txb^C^=XQMUl)`!UQy8)}w5nh)Q*}aB`HYYcEr9_b`@c`Y)%H9H>f8TPC
z?{S<WO3wPZwze`+`oh9-)v55F&mtRW8X6Rg;;;Jd4)e(mujQ(?GjfuVp<kI5Pf?<;
zs(Uaw5h?HU=GM1oTO}l^5}s0RthHgla*%&E)xFWUyzI+TOw4$O`S*$T+YuIi!34{D
z-rv5pBK2qD(%`{^J~fKc0+w7~@CqhI#oQ2@C=_n)d-2l0Uhll@!2E$VpDAaH)lzA3
z;y|U$rX%CU6KA<bN;?|wS?z9|k4X7({;vl=+^LyAT)HsS#Bgf4(QMZ_`A5tj48D6a
z@w?&ABy#Z4><9NcTXVHP`70XM|0G(rHVaoQ`5xK!MH(iXrZI&>y-uR$Wr%E7v)Odp
zUQw4n`y{MX-_S^gtwLdJbwPeEK39Cats`}Fj*c*8Z&7XGT14KkiP2u?&c8je<%=h}
z=JwL<oYau5A&U*JDJ3OipvkCPVuE(*h&8RYeSP_}`0(yKS8l9#Rh@1-;B1Y1O$fV+
zHZr_>qw3`ggA)I~PhQC!^u2fP2PiZ1d;Z%Ubokkxmzk5{7M)bs3k|<r9FhnGw{G1w
z+;>26C_XrZipZBylL`w96B!*;w69$B0@wxFi{_iyE~3{#tUz#e4$3lqI7)11Krzd$
zUc!Zp-UR~KVdbps>}ufcPG4&1lJs_m<v(G{3F~L(lA!QKXH_qaGhIcthe1bZ|Au28
z<n+xb3%^J_iX#fx7Ti?UUK)^h;5snf!Cq^_Lw10qgv7r|zTH<+Btg;TlCsg+nPJKG
zj;xS6gmzJHhQs9qH-t)}3bfwf9bh|!<yb0ZiUCU=$v{f_?2s?U;DkXkm)kq$P)YUb
zLZDX0wSPc`m@%$byGy^s07B#bY2TbEzjfag4VG72s`G$axw@Lm7ZBR$)zn@J88xBo
ztA@m(s9_yrN<N!}%yBEl0CuteEv8(wx*zV4z5A{cgdF;q9GUpLrp3i!{{EZR?hbQA
zcwy*7(!_$KZMbsS*?lHnVPxdv@+(!m?P@%W4Ib{cj6;(1*)Vm1nq%VS53VY$;c5H7
z|A8AwwvT>BeMgFd3Q>7h)!F6SvnA=hMlJ{Nik>-f@w?%UCN@<oD@zWzd|6Sm!gELo
zCmsw9Dhfzal<|?F*<ZgXZ9U9jS<!>Qe|vOD;mpORh|hMlK+uWyj>v3BE-Z%s+uHdM
z9u0t>1@5^5_zq->C@hL_`A|_lJ~9hMb0VjE@(mw*gX_4Tgsbbm0|#obL<`oi$UGt0
zVh;GUhB&^0iVPn!;;)74ya9I~YM}OhUr!Ib#jfUi$H9~#eNIvbfckI}B5{BRASMw8
z1qzTY+}s$LXanL$iud1@%lu9*E-2!J=dQ}EOkFyL9fSLmLP4pGj^JSk7Ez`mNQWMo
zo8re%D8fkrr15aekFL&60yhznH?m+N)!o^3&M&i|fP+}E5aI<Y{wk1&LD(b>N+%de
z6Vo2>K9vBS5jcyCS0eYh?3G+g(-miAO>Q&}RY;iC*lCf0fTKQ$G?Un(p#jFr5!{c7
z*r@46m@c3SNli<ubx%UW2UY;`L<5b9)CAGOeLglmZV9}RbmRoJlW@&nbxiViuK<k;
zL!zv(O~LK01LcLCiz^63j)P7zGoXL}>!yP6g4bM&E#rsG9~D9AwF_p4sp);thIr%3
zoI!^|l91fdw1dbrjG}{YvO`!n2r!`n{@hdT!nW<({UHZY1~z*DLaViuairc5zgk~{
zm1V_Xku2WEIvyS^Ag8oG00-c{Zg?{(srSL?aYMsaJYFL-hbT&*ON#+B9|j;!iHWBl
zmh4F^_5Lddtx7?0u@GfweEcYs%lA?KRHRAYf$Rfybt_m47(fz2X@M+-@R$GP<DW7x
z03#UW(<9)3q1mC1dXmT|0S>&oh+RwMDc~-bSI-qZe5eur%Ktio$OP=#zJk;y0HkQQ
zv~)Tujniq2iyzQ~A;L*mhz67IXl&eqj7bZueyGd#0D0Jf-wNOHV-%QTpMw5T7}s~+
zGJYx~B0>-%CL-2EVgbhB1LAmJv{ep=IZG2$(KM6g?x6{QE58-6Y4-j5zEj4Duu4lo
z^%q2l#dURls2er0e4a1g8_%X)!s$n0p1E<)Ih{+FF6I8ZhDJXl3k%q3#jDcXHUD}Q
z3^WxQgBm4zq5#x@!p7R#8_1Q_Q*}V!Geop*0Xwd-qfg9%_y2GKI&Q=M@hB*`Hm!w+
zJ!0bGRG@#viP^lao$zbpzqWF21dKLp5G80?-}5GPY~E?I(_s*VMI|gUI@NpN+>_j?
zsq1|bN9SF#d@>%1Wvg5wdMDMc0*9}99;j<<48qI!ioOfhEaiU`Sa4MSQDFDo;Tgc)
zNnu}lr*L-7R=EeLY=}?_qh5%brZ1ZI3nZ3p7^+1`Zvd2ka;r&TTm`7Ju+6>KVAl9>
z>$`8?zU5BTT(m=69vl$k-?&|u!+K(d7vfD!N<9p={V$}GKe*K#b8>QsymT9AnDDPl
z0nfRD|Hu8DoDeKiBXkgW#h3q6^bwqE%I2M(uI^z}ldremLFvj5Iv+A%JjJ@5lUq1l
z*=)p%?)5PPjy{IpAq6BKnk1e+Ct6YH-oO*H{WaF=Yx<@+9h>$)elL-g;$9Kv<MYKu
zMznaK?!0c+<%<@Ue+f9KB5en@M~FwPikceH9cscGncwT)24A27gujc{zYa>`KBs*%
zV_8N59MQmupaDCosW~jWDHDNuvuE#Y<vqxRP&jsuMjQxx4d2r+(#8e0QH^SB^OltA
zgZBah=?RFAI~;cLtC5#^s_Eq-cw!z>b!X=$f&~D!`+=>_2P0b4S|~(3)rJhvaQQMd
z6$yze>;rgVRfd8yfZUbk<xc=(uT+c@BIg>UMy|L&+R$qSV-f2sQZZ=r0QU|NtKp`G
zps;@YcnmP{WQq>N^G5_@TmM1df(((57@+0Rd&E$Np>??<c%$p*tf8k(M{Tc*(SjZK
zZi`;RMFae*5Y&!}G^u0DInjvyGR8DwzYO_3b56ejiMD0jZsCMeWQ5@Brjdv4n&Ojy
zRlt`X!|jnc2MTl6`<LjNd;p4T3rCMF+x02|qWIvQ-F@+OeIFwuBZ(%2FT6_&#QyW8
zUM<~Ir;mwi&HQlDmrqERv|0rbXfWY<Lq`nsr~X7TDXn8e33q@*2_R9;C$?f`3o*oZ
z6D&%C>jwno#U&-9VeG^`T<DX5w&gaUB%plu%f8k>WZxHu7f>{tIzIO}BO?G8<S`VI
z{K3lie|w=2wFT2#>hib&#Qu>sKmdqtTk3hTb#|Zp%i3C{($pjk!8BYBMZaqS!#og3
z&uuGnu^%r$OP#1V2}aqe?e4xGLbceOTRv!>OFhB_d;8A9V2HBmisj0@Ue+y|9sHKb
zhU#?xSMBzx(=Ogx3Gb)SE`;({%YDx&oJYX<x;8d8p^H~}#I964M7BC(06gshtRITo
z7tc3&n`rKuG&D3Ml42CdL$ZvohJUW&{EOYR11p75FTGI1EiT(lHotnhsHv&B9dy=3
znA-OA6M~--@|g2mb!9S2N^XNFj{yP}v_9kvBPtb`N}PQguedwB49G9SOX?FL^VKKN
z$0ZK&z=3|oe?eyjvd#b;6eM1O0NjX*0g5EpWho)xyTAF^>9sJ-iS`+^1sT;#r@=ac
z#@Y&Nn`kW|AwleE32K2B9Y!_K5gP~Sj=xMygk!)FaMs*5L0P3ZTZ1!a1h7?z_fJu=
z+5Zo#Px9#0!v$gd^c)M~3C=ljTfk1U!W5v_E=HJ#7DUW(A<y*-4qi<%A$-%ZZ{Or+
zBrpH`@#8S~wjkA;PB>2D;1G9lqGbacgAp_iV%-l7X<&KzwcImqNq1!&2GSzse39k=
z`6H+V5mWK-uta9RfQSy+QvkYxwZyT?^9LPfSg}yB?W%~CI{8t&TEx;CAQb>NdTq!!
zfK@(1O~w;&6QPhk@%V~V@+%C^L<B$tGV-d#6b)8<4bm*ceUtq|hDpmu^UX9LG6I4I
z7Zsc8>nnkba~}>Y!!HJ4cLA?~LN^nrXO!J(P0IllIA6IU2<&V4{1%+R-IyZzTOM>c
zjJ*VcJ&wdci!uiM<-Eg^6I!-JJ_ON|WAW?3C$<}LT$~*pxAEN{A3wiaN=g72$ME3n
zhuz5JA(d2-UzsOxGpquP9`UaMmHNShzZ0WF6BC*nHf&&t|0&n$0B3uKs<lFPsPWh-
zIF><+@=oh}N}O&06oeip{gvqfG&7~fVaAq*Oew3p2jW{c%29Q7n`tj=>(LFIbL4ua
z0PZom5xIKJ8X-y{n6-R-!}}j0Pyo;!19|E7-}&aN8uxJ95K20tZhP-u-JXzJXg+S*
z=a}--?iZ*xjz1OKIU+gdkT)fOYNDciIr#=y+PGkLY<(;~8R?}Be}mN&45o^OY3pi*
zG>6EG&T}@E=o4<EXm8o2f9jMLB?~ESqUYp0UC5@P4oE|kj5gkt0jhor=1pue`vss2
zg@`U{3Ms1w=Az<nqobe3gpY7@8mK}0L`f<+JOnN$tq;-VDl6k7_zxXAG#nD8_Gw!F
zo8;vWh#gQ-Xmw&Vy4dBhmg|%#un){5BRyV}tZla!0_3W?2+H`Y&d##xW)&-WAyDgJ
z$iTWq9n-Xk1~zW-o{R$zmD_!V-si576zG&4T;vZm0Vwf-p}<4WFXt~9u6dqvb%-Gf
z5(~y^Nf{Y63?}UCf=M+G1@-0W3H#9c%koY5V3yuK*n3iZQd4;;=IEFckQO6pLJ>H_
zU~UNAkO%gV0cfze*^Lq9%k3$c!zv5pjF9{AhonBi!K=ed(uG^Zk>qhlRGTHwpVxvO
zi-6^a#MC8O|L-!AMHSq%=<{`rNpJC2Y#p2F>F%Zw>tU8&Ec!J{6^K5Di(5+cKpaBj
zh*+ECKYdz@-VSC|1R*=c{WumJi6Z;|v~^Mo@Lmr7sC~o=OM8kBZoHiG2|%BFJFlQ)
zLm~763h6d7LEu6BFtID>K*d2PaHupgTYve}N8F4P_Mi6b-K&a7Mc{3SNi0y5%)?LH
z0uVZkNWY+UzKySG@&Bz7WZdLqJ@WAH3UD)mvEyE1SiRbIS`sK?2PP$^HIpk1G0h32
zh*eXO!NJ$z_W@aO_&IO6@u;{se<FE5cLx6)U6*?Fx*+5I{fjf%=<hW}m6rbUv+Evg
zV;eiZ7#I7y^cR?!ur|<L+SG@XH&&UC7qDqaSl9#bP!0g|{rc_OT1pvO?y+4tmHlU}
ztuy}=gDi5w)KuoS>uEs1<v_k)hib#kSg-e?o|&2Es#U9`U~M`om!XAwybVntBl$)p
z^}Ak&R+jnsc1uWXqmc6tE}Z}vIMfONFG(Q5D$^>IE*}PVDtfXciuj}8wHoN^roLEh
z2LWIqzQZC`?iFTn?$_=!uEZ^a0wpfc5_&|~F|5ryRbJL!!4rA4bD2Kpf}G&|fEZLL
zNLc<!H*UqAv<me=x@~J`_W_5fz4td{N@MU^$1LhY2vjMkaNmQMZ3A8PrJ+W4VAIX^
zC@TFiHReI0%Sq^Y*RNZrbL7Zs(bE-)$f0aOeInt6oT$y~bSW4sCvO6(TRDc|5G#tE
zrwx*3Zy(S^-)63~8}I&K0fagMC5i&7oJ*i93SL__9k=bPgupWf?Jf7KSKnj3AbPrq
zemhan;KEY_&P^0FAlZ;WZ30hOs2QIe9zi-6gDw)11bz7A5DSvmoR2eSO)HnHOdXcx
zJ?6MKw6`u({dw-sr>>@;!<KI$ZmiqZ!P0T;f_j_zkv4-*HXPD|PS?VF>dcRP54Q1K
z`<>(1y7lQnLHA=ighSM$SxuX!)_8rlyM+N{<^SArir*&E9h8Mx-&@)Qd2hM!E&F3I
zE7@lHzRlcJpHRBq5zvdv3?<3g{<6u?J%8w;&(MnOa?_EelY~h}#5z+FIroraBe`VP
zxZId<wIx8u@p6A)|7iDi)X*bzZ$fo871MGg@n<Ju*aI^u=^^c9?tHAw=RWu;eDlEb
z2;FmN5}~Q{5?zP|Ryv9~Orm5R`FMG$P?Qt5V`k{ednVQ6j@G;0Zr4KjhFl{MNWBFP
z48;d2P(Q-Fjz}ChSy?9Zi|$ijA~Xa)<arF!2J+8wi14zFRsK4y)hXkLR5ccCU|Xq8
z6-wC~h?)(xLn7ycoSe}$#aEo2IcC3nR77kt&sTbO!g(}5$wP8|s9fpGhK3kuMnR(s
znbJ|#PHWWxrjU(p)cpgy4#4H)=F#n_BoFC}#@tIY^*Y<WRdm;`U#FIT7}6$p4?FL-
zz8qI-6yN<fMSh+&{b<F?q8&=X4Wv%Sr%k|IavS(#A)%pzvNw;qS2THi{^6w9v#q|k
zAnpiKL|)3!@UY6!^cd7&EbrxyO>WVP4q5jg#qQF?yEPjm;*P2isc3t)ejuvo1y;42
zXwl-{|K~r$sv-`3s{(!)6USM+JQG{u!r#Ap`!)-Nfim5))YWpOn>TK3#j-50rZv2I
z^Ck_jeheHG?z-lspfMSxAvkFNPU=&V);ZkbQceTPr(!lHzwZ4GYn1u7cg;SRxyv!&
zGd6D~<~j4MH+b&@5gEsj?EU-p<?l#v))R^QpPi0pU`=Vk;NW~hfJEVOZZ+YXwj%M}
zcNj70GYv2V5hPoyJkGg+Ecz<QQb$DM-orF|f%Rp7$w9dYhqEu)_u8~5v~b?0OYYXx
z(6GT}=^0pkP$gZj6&*Vb2<gyCYCdx0q-1DBL<YDsXzFnPGyh?I3{c4iTevLU`|RxR
zslLi)SDW>EFhW7jAO$Zy>-Yf1g-z>@O=_fb?|1e?0iNx(ya*9+jqAP_aPdsX5hkx6
zmPVl%m9oHkL<gy~76|Q7ZPB^fH0GjZc>_=lp84id6mu$@r}JpRh&20?tCPgRM-`Y9
z79k66N%zf0j<4So9spXKCIaLcD=Q`R<*n~loJvD!4q~bnn1%Sapl+NH@0yq$hJ^(F
z9m#C)jywa6czv}e;l*_ywYc*qsz6x-yBK7o`dAIJVz?CPTy%qvoiF;A)S$Q`RWpju
zFX?6XZ`%}^n3P|YhT(z^YB^z)$?z>u2iW!4IrU*q2o!Vvim5vvjEs(|VertZ9S<7l
z;=A{2Ox7D{9MyYYDG<iK`ct8a=tN&-GQ5wEVabtc*yM}c2)G#?eVE`w&>lpi5&}Nl
zf{$HbJvI$AOc}a<0eBsOj??_-)L2@b{n>-=KZnw{NBLjbVsRGR=kA?5t&ls`n9*+$
zoD=%xr;w6%>1OP2|D(C>dT`pGw6cfOwk^h7q7^Dyp_D6CWTJ^Ul7i?7v}ik8ItiG6
znj)UH`W2C6VhPbMQbl9!+Ju$9o}PNPK@4WD?a9?eJ^X8f_UH3FxTx@xc}BVJuk(kR
zES#G&E>K^OXqD9J<h+4Oc@ZPy8mp_XQ*`iDpru!$B-&T~S4~7%I0Y@x_n`Y^%1XgJ
z^X^35LkdnHNYesXX=Ea8+<tmIXTLS#`>Eqa5#1*W+WScSkJiKY|44OpbsGjBP%Ds^
zldhL7JD*m@?&*wVIumX%cObvim^^V)Wm}8W0Yq8L`$oB6p}3U7Y$xnu%q)f47MNxl
z<Z><818&;Cfel{**tv*F;>3@O80>A{k5uPo`1_q>sKnYHnaLL9=cn}d_lp$G=G6Vk
zh^<NLwK)SusK*WTTOa8iwA?m7bP5BlFRqd9mngN?6FfF4p4w^yiO;bI1|r!I6cJr6
zr4e8sW}_S-O;mW1$APo@m6v|KEO<S_D6LT>9R>L*9o6V9lj4h-uyCP70iFPi5={!f
z*1VS+Lf{Ob3lmau=}&!d8or~q<8J2;S?9>mZ0^)pw$)1l%y^rGX$7=-B!ecHw&N*1
z*CCTBO>17JXZb2zH7)ele)W1?)&;W`86#ougXaG~&{o&LfR>VF_IxiS5MRc|7@(H#
z=;}%XLI7Ed5`HZe1u74aQGhQf{}VByqK2W48tfvFLIjdAZ=<7ws1|i{+bI-TS=mDS
zt_}D(B`}RzDz14*UFe7^p7QU2zm7Sk7R|$kQW670cb%@-0+vSNui2qjYf06M@~#22
zHbCY6WP$);h=H79ZfzZiElcJQx=MIx@{P*+$b9)DkdZHua`qU=eM?6)0>I$+MI<G?
z&a$o`hW|B*i3m7l+ks8XN=i=*#MFM3d$|3XC)OIojwD4Y+eT{B?SzE;2<lNs)6sGD
zMKnchBDzYXW#-5vh_fJKHUlhOIyyS0XA5%gL1k`<z=h|331uxAv!t~|^mw2hlTl(|
z7!&ZC_?~nYJ49kaJqvQ?k?-Lu>)e*Rby#s9+!u{qyv?(g;gYY`G<V26_kO(t{zk9#
z=#ezgDohJ)3ywC)6GRfBs;WwoDVWYt_3=v9HR%lB+@X=`Lw(V4P<GvqW7_l@8kA#i
z80BXX<4OQKQ^?5Pmh(l65sw-`aH}zODEEhNy#-jYgth>|);wz{RaJ~^;9CpU?Kh|W
zg!)p0`qI=C?j4A`I*ygz7W;}qK?d^@Oyk*&C&G)sX&}?GAfPRA{6dXlfhO2b^Dv`-
zf2?dRMVPx(cAvp`Ht@9%+qseRgD^I}r(jcNem)y`!$jOSjnl;sp((E8FS5Hv_J8*k
z;b)E0nA{Y;3J1L<FG4rBeIIPm9-S%=iQi(T{+P_cB#kNf{|}<zuZ0`wJZrE!9lDTp
zsWBgRBf)#0m$kzRL`~=y4xf&0XHfRTK%^=d@&epC3_aJe@6<x`Lrtoa%s%gjP6bCV
zp$`TX?ytA@0Hr(1F7Jfyc6Pm4(eJTVevp_6BZa7YP#dg523=8W+3T}?uHxEt&}tuD
zQ<aBB^|_{p(;q-wdw4g-JwY|CnzU9T8*_@_2g`&fBfpdYFj(rZVp@A@1o7KTD_5xB
zzn_orb&oa5qpXQ!JOsd-6Gu**drMaP`G*h|YzkLB&2F?)B#sf2))fD)EVt&F%79$~
z`C8yq{%uCwhMY5n8azMqncqZ(ok8Lit(Ca%N2l}zyf+jO->LXH0zrBvJq7Rww?Dhs
z6M}*Vjwv(T8}tw3!n6EMt1Yo+#&PT10&BV|w!Mc%#j>ql_Bu?dk}y}UDgMv@MM>-y
zKjHxzK|`_v88OJc>JK`L=Ze-&4}#f>29x{nGF}+?2eo*^%v^C2hunYCk`58-xeSdv
zzNod06f}oA4!F{o``0o9wDKrH9SpxXKuK-qXPGM1&eqo>q4GgrZD!pL3{K%A12IX^
zK;Z`or9AfcRYX&<a}PxXyip|Iz`e$S*pUAp+hS#Ya19}z;9rwJ0fDB^;^)ykqd^#5
zRYRFh14b7{L(>t&H?V#zxIgy!ZMDf=r86%<s>9tjyQg@&fB+dMf)SRZJ4C*a;{qla
z6qo7{hfqUjdG4}B&c^EIQWvpZ%ww%LVlzG`K2NH930-CSc7_}<O2&#@rlsq_qmDr1
zh$x~Fp3cx!=Ip2za&1zZytYV*i0Cd_v)B1?!CQ^~Kkg|Z=~u(U!|g6Pi|?G5hW!~B
zb3}BFY<NI+R{;M7lK`dod_QW&HWjl9CHCTiaMkxY;VS2yog=~K!9Y*wW`kiji#<BS
z%~314*LgIG<wA-d$E<GhluljVuF~U9T^^bmlV|BF(GLkhUJt%?{Py3Q6sKRAiNqDe
zx|}a7NcLVI`y5Vs1|o4h)Et2r_Csb7V(RV=GLg(^IyZnF;z$iat6BN;2k_lN-IXBl
zI;fR|V$upUyh6#6S!0^tHNJb}we52W63*gcVhuPkWaJ(pS%HjsF1S3vlHXNz{@2a!
zA`{UEhA)J7o&klq3Y<bRyPVK@A@=W=6$nuC{i`|Z*>tbng|mX^kWZFyzqCr#>$C$x
za|OPq31=@N;#}}Ob}whhh?~YsnUag7TPhCe)Q9ig6rRz^E%gI;1sL}6>;G7us>(RK
z?-y6(6FO4~I(_ps9u>9OKXSSn$smWam=_?AE3S#SVzOsIV7~8?q|n^D2j)B@yfhn;
zQl~5mxE7g^Bb%z7EQh%_L;`y3`&%Y-0hG?)OzPRexl&xJn#KBIH&GCP(@3nk!RRwb
z3GNwK$8fX%X~gZ==d044o(_pITW&3#g0=yWp5v7(4d{xKAx&5}T8-(-ua>>3CVQQ4
z9qvx6A6JBF@$S;mkr7KsoVX5Ml|t%Q&mpTX=DSrd#C@nK<E)=bo>Ap*4MSeghib?!
z04tm9f`x@2+#LKIzFQj{le+gm9Bqnup445z;^qNmzG3zq5Cd=Dz58v4@Wr_Ge*nl{
z2PX>%+zCr-5#{L<fipwBnQN>ypYC@(v_pjYe%ke+GYaX{1qDa#UVO@9GDzVmHIDIs
zdI&u)!eX+h_N6110?PaE)lDCpKB4yZ!|scYH?UU~kw~)Nn*9bleNMMOB~roG*0*|l
z^SYi!+I?76;#u1LE`+yqE2x=B-m_4xVYzM~tD44l*Yn-V8nnMuM%79<)o-=NJgdq{
zOG_X9Obo<+-+J{_Z)l|C?wsjM2~n{GpTK2v0|m}~WKrlFTLs_C=Fk;zeex*YLd5p>
zg?>%?!Yl)YO0V_jjj~=HLdi=ARR}M*#aPLJMW7fXvb$7=5Y3P++CXl<y~vj4(&V2-
zktpZ={ohPe&-vf(e#7X6w4P)`sF+6){Sy9*y3G1?7~DHf^JF_mirw)P-?T2QZ*^D@
zW_^Qo!9|R|$mkr*UPXPlz#5vZYV2(7XFKm$)%s(2?$vNr%Hj1F3!wBunQX}X{T*Qu
zKx4jb&z_m|Irz~rV)!;G=N+M(7mhpHnpbJ<r;Sr9c1tPG$mJ7N?x>iC^L`0Cqpz6v
zS?4KBK_UsOFV!{G<>Ugwsbc5g@Wr${d6&~Csvm<OK~%*=@$5jC8PApR0*@UY7FMD;
z`7r6ZBOP7&i-6}iM`f4&?;g|ERs%@&i>)JfN8C}EB~noU*<u(XGlj6zk*Ed}ewi^m
zfV@DthBC}~6O;Z^RJ@@JI)lb_mMkBqxdYrjchL;Kl=q;HJ<0lFd9O~j<{?~U+DQkm
zmmhMyJPXSzLc#1A7vm#D&L@`jj3gyQ+DJ&7xkHUyehASRKxz}av_G-7cs_ngqRzf4
z@xATs-6r<w8E4QeRZvb0?}oY&wiYFv7eAdZ0#}5G@M_V>Gnx*S<U-s<{ok5o{5T*X
zjwdT;uf30$*-G)Qn5<t1j?jVaK|`_TpCgilzfg)W=?9(t8?P=PA!%{<GgmFUiZkc8
z#(Udc54m4GaVAzUUHYuZ7%~M)$q~|aLl38R=VO*^T-Bz<tnl`1@#tfo5!qGhqRt|n
zJN#%(ZNRd_i3gS5=?QRu34}@ElFD0<93d}BJ?URg^e<3XjX_Q5zP@_R&iaF+kFE!V
zch4Rc6Bj)q;Vt*xq)73o2J?R^00s(SaR9jWdH;0{+Gaia_7^Sy%n3najYRz0^6he9
zq`W(Kehc@I#H~vFqTmSN3CrlK;nyursZQU)0q5DU%JEWksOFoSeSkV0!DHyI^xleA
zExC(Pe@Dl}_(CFcqo81VhmZeSq5C40EZCl)b{xij0LPROz&G;EttdeQ0cD`&o8<1|
zb`NkyKZph1!^>wqq+ls1;W_V!Xq0+-yV$Bd8mL2O9K8<5)n1*f7e2S1H#rt_d+~+{
z#~Ifxq0|yZBD81}Odu+{eB=`pdjR$UTLq#I4Bn;V$B#xbo5Xi+hcFf^PnB|M!Q#q^
zU~>v~69Os~=@G#}Vrh@4C<}QS=i&CJYVrSv3&04aK@gfPJyUm3Gr@#90M6T~n8rnf
zFXRz6&Y}S`2qAHxG9T5lcl4e+8msuI*;Ao+A_^;51rf3Ft2ZCN?O_g7j_$5MAUa61
zwT-Iy?m0g(aB1+Bhy;(2R#2rKPHQEjR8U4L!w*Ojy-m2EZ2?QfFlp^Ab%EyZilS2y
z26;xI=w0$7DiN_Dh(s`ZBwQQF>jChSToo%$<n_eg5KGfD){Q&`hEhH-b^)cCmYzOP
zq?|P%TV!?X-SfFW?NG-PC;s;xVtTsRE7fR66XKGezdxbGg8d3n8V$_QA1uMYO0F+8
zKGY;n6=^9FEk|NF_`;1HeJM(chZ%&3MCS-trnz<Z+rK~&9J))`8U4xlH{6Rad=Xv{
znDSwNNTx%O&5gc0q<A+l9Ml}BeVJ|Sct4oRfS~2cC@gTld9h5GuS!Vr;BmK1EUZoA
z3ZO><TKVTEliYkiP0x7-)qMo{M_<Yu_bHSasz@ivq(Q=fmg1ku;p4-^A#<dTM7>B9
z<quEnL1UkH+cuvW#j`ki_&8LPHP5Fic6PeYjjQGvk=bE5r!<5VfNS&%#{ZFe;U8Pt
z?FAnn(XxXiQiT!~Rk;M>_6r08o4$%eeP1M{!-V%rW*dlHY2#@&5Lpi$R0P0LEBJha
zNKAy(Wb7D#*M|*4S5kgiK7xXexS@ljU=Au!iRZ#@K;Ny9f8c9RbMxun2;wWH>A$#N
zAa``dMA;8LMAFuBn<}5{|Mo2u*a+TLH8?|}KRoDAs^ihVz!sS_On%PHz!Qm+PzE8L
z^$dL3M!~b56>4*g?>=#8?syscMHs<H<~NZ>5FwZmW#N}E)vh-8catf<cupK_H6x=9
zq~atx4786u0~-vi$OrMeN;Z{FRk0>E*jChL<;TQuyuByO+1uOOVk1Qa5>O$4?mW+`
z*#=5bC=eNQwD3DhU4)3EK1$c0s3tU@3^k=18v9%AYjEXS?~II&Bz|a?Rl!Fz2(0~7
z02H2qhW2o+(YSYDJ>-wdP#Q|WcgqvL6ZiitpYsdf?#ddR_ooVbrxn=B&6_v3Ve&8*
zi07{UXb>pXq=3P?ky#R?{{;xF&ed2$nhzk$m8!!aQ8z~$oMaDCj_t-43dP7sa}4W+
zYF!~#P;;I&Ta^$2@291anRIyNgMhY?s2Qxq8URqDrkJ@Om)xN$sbTTjmCK<JX#&w{
zf>2Y1tib|f%x~w$0`?*5U<qky6)dyk+$0in%N#0PG~f(T6cCK?6%x5J&Z6KB#2Flb
zgsB=G=g(5i+NiD_A&G`g(*j@Xg4$mFv%5+I{i#4}dw+>Ng!T}bn^A+BSi+`d8=QUG
zwq27mizp@|HDNjH>rk1uLLc7M(b13UvJSjv(B2wd_k}g$;&qV|{gB!GH^TA5p47;0
z`Nc7Q6nQ{8xQ7o2J}N+-h|y<#_t9BFD{trA41~?oKv=9GAVQ)*Z~en?6}tsuFKEnT
z?YW$w`$Pa}F)B4fyeFn3C_Nt^V(_!*4WiXbYmI`xH_Awqn-X+8#u&}M$cEj5fxQsq
zR%#JjB-OpamL>S>`sKKtyLJgs+8!KP8^M>G2v3Gsz$Z9TpeEjjo}d-pGqWF89vq2n
z?CH6On{M`_p#D>a3P&!T1JX||YFDAm0Fa*(j&iXUgng7I(#FYX4%CC@r`vjZf3UJI
z?opyq-J)XTgukF;|9ZA?WjA`NWHvHT8~>D)ZCZ=wIWpDSZz$=n%))Zd`i%p-Ae9T!
zEoP*udEgUjiOT_#BBPxN^0fckjqF}p@3%ZU2d2at+IK6?;$NqsZAt1->uX?~^`o_*
z0A?62yOdU&U(cw4BnlD11Mm@IR|MI1p-n3_m=@`eysm;*L1G+yQ);0^z*|tkL4@S;
zlt<s)EeGA$kOwGZ=kUmR2Ej0bhC(dSdhR@0+>L-l8o=nCI}VMNvFV5Zklkq@hZa}{
z<|IyYs}_+7isR6fSx@8|h>Sz#tO=k7NHRM+nPvdqp2)#bE#>C7rK3YbI%td804gq9
zf@>b4uqsTawGh*ll`*h^zu?J})>(NFN<owH7`zr29J~{l4y_L%0YUybTbFV6JctJZ
zZMnxIpZ@knpgUQ1X*E*d=!@gff9Q+1gVV4beG*Bq9(wG-x`j|e+;Oytn~Wf$_&j{L
z3*;HH$pAmiqv?0~?$cAQ$x+B&YH_pS+Q!`y#CB{_8m%G7PGNhs-|jEkRuou|%~@ak
z1U)072vrr6zA%yliVwhf8VdaEe2_y7LPDdA-5!1GR;I_h$IrEG@Z&pQ$oN=L*tfej
zjda_YADROlWB}~e#G+(~A}C-aws>@%bzp0DXcfO3`>ddVnu42#13gw4ac)JFXf5|}
zz0gxO8$P!>_-MK!kW5%x&a?Wn^P<X-Jf*Wt|JQtfyb{wF?ls1bPaV?wc1sF_mnn`A
zftMGX?N#)ym?+2&#hpl@5P=YMG6OhlL?GLCUb+ISBhxD+fagY#P1H)2Z0G8WT^S~|
z12LhM38zu^_&rws`SjQ0G0_@R6XUhsf0s^^p#rcn{ZHPAaR88kkPa+C&jHRvLH$HU
zW=!0Ao`-6LS^)x~GH*oMM0|5$X=%f`63`I!k(Ok`&^9OtlZsTpXH!-NwLlU0Ek+=`
zl<7Ur1ARWN50KtB%ZaZE-3Jubf^ZFu^dQS2u?D7`(nf!}-WP=t|ENCrmDXKdU4^zc
zh$sn3C>O-X=)7LXHc!QgBqBr9HZ0+)GYwZrkpqK2Dhk1b5f0KyO1L=1PFA6aC2S|6
zf4wd1Dh7a+7>4yc0a4taNG8z*MA=DR$REIk4lkxO2^_dEGfE?TZ<7gH>yJ*B>fLep
zuzBM1Q+nJ@YyVR_K2e$lE3yiO9`O)bmJy4lc>IQ>@LSR6IOE`U<M1W6bJr%pJM9N*
zge0t`eagab5LWzT<~;e4Uod^2a^zy?c-Lrl!DUjBr<A4&++0{pkV1d@eI@C%-b#JL
z!mwR1T=fQYTx@7p?|QN)PB4Kx$MSCcE2X5A*yo_Xj5H@lBHCHOcnDLg?;T#^9H3WC
zt^0*&1DI@c$8%TlzElZ5!H{U#%-pm4az>LvPqqbk6{mgcBI-z@o5BQV#5K>r|74Iv
zU`YxO1cMRTEvj|MOVJv|6pfdfgIj={EEY0TU>ZZbmXX2}4>X}~@9+GS1(ZrqEyS6p
zN24_2g852Aoo@9iFxPQ~A&$!+3{-=<0b5#D`26;)5@^UL$Ql_UhbJSOq3?@CyN__p
zbFk6SSVgzst*h|xUaGaztiAXEXWNc^UbHW$D6mo@t^m#K?OT-nkZqG$Z2!i;+I<r1
zan=gRX4Dm!D-a#F+*}?S92&|*U;tz-_d%3KW$AA&CSi0O8<FViAO#F1GH61C%6!Sa
zTRK$OD;5kks+%8woGY;c_)mx`@P{GwA5xSzpN}X>n?c8jW=S~uOBh$T3bA9DRPU(j
ztodDQqtyj)*)gnHqXvqKNzr+K+^HuW=<VAs5m`OxRMSneQ-oNLidhAaGGOuS0Ic!n
z#-3?p(SE33h`9l#c>AId0wGPX?A;kfC*%VF`RNJdi_gV65TCA|SGvq;4)@h30qmoT
z&>i+K9R0*uCh9%ZQtWJOJ_zVla0COh#5(!@>#rd4FfcPKBV;?m%=Hh56R_J-#K~P?
z4bL9I7fIIxMSEAt#?nt!S0~pG_xJzvZ-9p{q0o><I7HUwuW#-^{x&3guzCjoIpRTy
zlqm%#kYM!~OX~|pAnmGEih{8}kf73#1T|fMkL5;Dx$3o^fm2Z-<3-;-fI)<A1#xyn
z7c~NKD0r&uh~u@$PWR`ri(fLy+klQ>J$g$zm0s6i8c_v-8t!Tp&<XILZMQ=2chc)?
z1__p$o;VSFu4$Q<$C7HpHkx*qmzl7@ga=44UJ@w%X^?%BvB5Cih44kl^Y8K;M31Ag
z8ciVfe1XRz8Haw-%#2J30RPVlbZr7ipz$1uRPO0;IJ3C2?*G-+dxvw`{&C|UDpWEn
zBa&1^*$p$htg?j=DkEi&5VE2&N(xCK6*3x<na~hINW)&)TZ-_!&hGE;_x$s89QSeD
z_Z^>cUDtV@@AqrHr`ae_R$ju{Pi(mh=i-BJyw(j@qIEClZc|((*<e<z^iVIudFiT%
z&g9|*d(z#=_lP6$^i+dAqY+#RhE-o#R^bKoheMnstdRB&aS|v`;1+NrX9S2!HGn(6
z8Dgy{;SVtQE$6zguq&0qWN78=!^d=LZUgt{9%`45e)E3i$YI^n<NWux3u(j<Qw|6O
ztTX96VK}=+_$YLl0Id6E7H)6f-iGuA96$sC!5@MrVjD-jNDxRu7vZV6jDKdhg)2)G
zB&*G}+{QUR1a8#(dY@*Go2J&DrDnIq{jgk>4>rb7{cHH#2LmBF-zFl$2q=Z*H(?VZ
z=?bz)+)=PUcL%brivQwYoZZ}lAy-1J`u>l&BKlJZ<$F+g5vMH~?}fPyCtmw=ljQ*%
z0=h!9&g3p#<2key$eJ3&rr!+U3rkf;@{$oj6gsVLYSM_GTqE>Br_NaQ@BHiXzUsis
zv`nn4ynV`Mt*|Yt!PH1hl@tY(NTli^33T4megB3iBb^h}*jU2HoA}8vfRzY~hpBcw
zBoXcR00S2-F|tIc%zMRYq-Sk2{WXu)85r1hA))oe<5OF!0<e-~k?LhoZW?`fVw}5`
zd}73qY2B0OMge&kpV-RYzWQ51jMb~{iG82GeURHjZ}&|vSNqI%R?obqvv-1&R{Kp-
zAzPySiAni%e_%xQ*?SmBe-$xPHfXr<a9%zkeAzVn9iII>&oR^XUbJc+d_&K2pH(Hn
z(+cio8$J(iI&iIR+aVKZzj+tG5vppRaT@9|5{uf_#`V8gR4B9X2{FNnn_w(ByAhHT
z3dq8&uEIbl_0PEh$4{llcuEf>T%!M~6z^8;!+V|L=~>=UTK?gB?!`fDBvW9jh&vit
z{h!-&jI!|PUP7xbKjRHrUh?dhYv%E%;ssd@UD;}qd@W~H^(Fgd=+9qVniHCUa=HRw
zY955uWV-~hSTqzZ4uNc^-^_;5(ab4+uCVUwcK`N$$aqs)9M%MgWgS|W;ieWrJ3-JU
zGQc%K$)=v)Gs9pVh%#K~_6A;Ytw`R)zVkLVtME+0@bpIbURE{8*sig$4FFp5O_#pE
zcgQy-XBloP&6OxChpkqOm+oOK4>zS8pzDRG2Q-ClkJ=c)jPJupEJ}pR^mnXTjyJ8X
zZ0J679Ng(EOTq?kEMi@H<A-%J0;A=r6Zh}+3c6i=il7BJ2bR9u^q}s-wve8Cgd^<2
z8r_{<Xdj@dK_AnxRerV4Jk5S58q9ul_7a)bhmwbEyohrwbvH=?J3<Ud=m3T!ti!A<
zlq9R$(T|{HS~v#$ba5p~hW3S%Wnx$TJ=%AXS<b9Ds6lkYh9`3KWgO!Ao}G#)ONF~~
zA6nB(pKm<(A;0FE5xM13K?n8s{8hr{na%HVaEg`VJ0T4WD#lAd4C~(1oDwHS_Qz%g
zIc0pd>;pSh;@U=<Gx&bnCv16j2{Uc4=*$k#u77)DXL)Txxv?cxN=o^PQ-5waVLU<O
z;F7#UYeL)&Sj4LY_A+)24u%5&IMR7tf_`S*p2886J=RY*YFg1V@10psM`7Th*}O-f
z{h}$ALp~?*^z+~I3nJ=k9Ce?~buMD;&3ywwMHPbnlCbIUWUKhwOm)&;_azJ0mwF%P
z{M<#MJ@R(8Y-qq}!B7!PkzLAl2l6D~eqX*V?39H1j-UwGc!v4(!mK{38&tP9HZp@M
z>yx?Gs-euu)&0SXJxNnbD+Ev%){_fx{?=Vix4v?`n^ci3Sq&-U^+dr@H+7yKtNddG
ziY56@7+=ARM9~`uD(PYdZ}^@$Ga%8jCUn;Zd*>k5POFl^VM7JR&{jzB!a*8uK{D=p
zh#I>FkC%3SU{`yha=R^p4y0*=Wtx?hLiARjvqxi^#EZRF-IZA-WMojZ7S$c5HITrm
zc&PdOm*$L{GYvJP6~*b(j-@8Ld3k!$=oIO`E2Wm<x74hkYvbwR<?6cMx?e~;omzfr
z<MD!=7fI!A`As!KAbHW4zs0-fp%MbZU{rqzui})dG5%jn8A9T^Wo{ZLM*o?-#Mo(t
zY-&uPR&bY`9LX(@7mITkfUOEZ+F|4>0@orlQ{-qriqf;;$$%mb%qZXB)l3Po*a@b<
zR|iiH>L_H>9K9#Jcel=3!p}m;MDiHI!|AZeSiW`<r?2GDt^0TG``mTQJ+G%1m}7p4
zW`hP}cFrm5&Q~sc9X=y_NoX~G8yt(3ikf?MJG8bCQ21TuJ%M53@Q=QaUiItOCVwAj
zK2gIxEOrKjV1eDug-M`N^aM)_icyT?C3qHo$I;(E&c_t}6omVc2)rVhUw|`O-A>Y1
zvR@fg8^_icB%-)Y4YZ*ej1VypTj2BJ0Z{A1joKn<7YH~PNsl9s6piY+iB|{M8^NMT
z&w=U&ppK~4rU>!Ev-zg5Wez=3eEP-hwQKrEZOA!Dh!=L1_as;d$mE-UoQW8H<pJGY
zUjO)wT$d6prXXi#F1Gzbg4DaT1>0RrL+6&slMIg}+#JC`vaBF75=ly`620afF&>nO
zm{e3_s}D>hhwgE#*mbSy$hfq+_L?w8+8j&TU!%LtIrP^8x?{)KGsrHS2WmOs`&1-1
zzVbk0WE+*0g@LxpcFKKTPS++yEv+FtISJNR-pt<WW3$sF{~DK*0k#5gF;A;LiMHGn
zh%H9hxx6%QSyt(-ww=X_-sV+>;Yy^_52GTn3`3!9;w+QXna->#At*0M9+@}REuoT(
z8n0Vris|l}CqNe9Oux%h*Hm^?aF54~_&ywY&vVJEbMM~2n>g1UH!*w_0pOVo#3aF$
zzyP6s^f=rtMz1dyasCrGB~j8o2;B!FDl@tg?wwFq+0L1AeR<1P@zr@xsRT_TkW|Br
z3rw&MqJ01$HX@}EW)*VeCf__zn{99u9$Hg1YfyjpdEa=C-cXf)S2-22!=g+^v1^TI
z&4CG@6b}@2p#Q@&3$s+SqVs<lm%Dcr8WRwj&C>4@!T1n8MBp5c{7H_m3=WlqHf}C1
z6beG|7rdyg_&u$eXr{Q=fi5Ca6Eg1eJv5?usLK(`ILW=aQj})bM=7ta05-#!nHiF<
zffDxwMsKd)yhkBTEsR@Hb6bXwwaA{;ncj}(T3YnT+M&XB@OovLVDy=$TJ)9Sx6n;b
zZ2PF0ZNvw(9sNQDfb1C1DIk1s>cg1>Ne6*?h@ceUyQTVCnBa%bH}M?~)Ueprfoy14
zP@-^+h^7U835BQy8}=}^RQ5u)O`C`mo}icjIN|X`=>@8OTnzg&W!CFuW?5e!CI9TI
z&$nv>PoJUYJA5n0yfZ&bFQWAEV>%v`Fa8)LOYq#zi}TzML`JS!#dE4|{C9rglqx!v
z(`W|>%fdTJWqV^wGIK~0*L|zX&mS=fO#V*E${*Uvd0JOD%`Cl_XN|iDR{s3tA&3-V
zyisx!tB@Q`z38?02<veE*y=otS}d`rsm7W*{)lF;xv8}D%GiOEZ-Q3Zw{Ep4>TUkG
zF8oTSf%PHeU&Fylhq)^8@$6XGVHd&#Zpctjdt1X?nA>hL)ZMK+6;_Bo_~z;-rfcjZ
z1C7iQ0nE_)FLMA9a6h+H9UB;{ADY-=CwUw?ZxRgxg$J4l<8coOdj0H(UFg+O%p$Pu
z@8?BaGd!+(Q)+m{<)KIE>Pz-+12@knCLUm<TT3PQdv`C0J=o}(fHV+H31{W{k=S*W
zqCvcwed9=^M*PS?IvYJ0Q}4p~BKjJpz(89ka3)`i*}R8?L@0rDUtjaEi{7tbJC+9M
z2$K;Iir<A^kZ#?&OzS@SN&{PL@Dx#K+K{E9C>-$1G|x19U}xdIirsco&2)DkyN4`4
z3l<62cw~=Z)92d-y?9VP5(<fE-7rrGU6(E+&TRd=5yFKi3%~%q&NEd@Yh;%#JbPbp
z_4bAh?aN+}*p;VimamA^`Aw;FZWCb~?BwzO`#XQ^XMevHRPtayXb2X8x(RWfM$x%}
zR`J6c?$z8BKR-WFm!2v%;T+C|iXOugZ&_t8G5TpZ2?*1`s8Odc%(=mNUO3OSUx<eV
zfkDVnAp@EZ!bA+M>xVD7_+;l(>o!9==r>@Uib+wJSV_-WDU}qwb5A3Eorcr#=QQ4v
zU2oTCoc`ExO;+wwOV*|OLjgmTgjfOpUyeQ#y${)AX&e82O>5Ss8UmKXzx2B+nDKs#
zLw8@_?7LOTfZC_J-mE%hYnuik89Zs;TkxfC$Jm@*kBSoX=s{TS8ylx0$L>~2{Vt~!
z{}4FqjsbI;hx_Y=isic|EKMEmdrjPq8h3MbC86a&^l3@X7Rg~^XlV$><)$d0;PH+M
zv7$e0kbgS)5#AhrIke|#=yi!8Eu4-_@y31tJsNSs0WjYVmJ8-_kr$1$<%X1ck+7T9
zr<~p0@letdL=5(b`-FA`5r@i&igMmrnJFlbvR)(h*R8!PX1sj2Z+|M5k){R&EMZHX
zn=#LV&4{QR!PR`l4zVylk$cZcZ$n&cY%ml{oA8cm1_m1`ge?TO{}$*X>1W(R=qvZ?
z@_XHPZ$`YMEty44j5kQmL?|{rx(+;m<dh^M=Fr6G7MwRfC?i9V>HlTqAYTrA7Zkm?
z;)E(e8vBEX4~b)zWL6Tq0$;bbRtx**eUvhTzda8>S#;!%Iug~Dy~P7xQ{$0M-}L80
zOMf+3eLBlTa)oAA!s$SbzJ*PdNZ1G{g+S7MU0!)UQeaLA@d-i!8zt$k@m1DPNWuW5
z*TI;fho_9NsO~T@{|x5{<J+N!(qk5Uh8})kEd}2ECZr44L)b{nGJ<g$6Bp68-bR6`
z27NX!3=ToNKfJ9AwU(8Oplm;UIQ8w-h#RprEfy%Ch%EzjIkC}5az=L8<g5jkOlBMr
znJwip21!AtJA}PKM0P(W^Wp@|07kJo5Sxh-r2m9~QpV!uMl+FZqzME%4T8@bCxrRo
z1SH~QRP;J7E>GX|0^v$S>)kX;v%yYO;hb51Dzv`W(MWavZi8Hu7z2QZMgs)13AD>%
zi&Rs_V}@~V+lZe5_^Td<^4*p5ec5&nK~u3xDcdj!2qwdafbn5m6@UtX0qsF0leQ}(
zMz&yaGEp2rV`4C}KA440t09yQa9=RtYQ;jnPrfkRW0u}YMp@~EmqCZ~XL0ehyL)ws
zse^+cbRW~Rvwj0!DY8zFb1{kR2z2YskB;$Rc&P!vZasmG?!da_)IN!Z?fl)hm6=%)
zbRqF0;OTZpl4_Z<Pd0k11%x_~IcY$2;ivcitX&}MGaC+Z`-g{Cf=A%?fGIQ*V}0P^
z`GCAY#AXS=IXQ8WSQr13{l_3X0T{Xt&^(N887vhE>%MU^))z$uz~xBn>+lX}PO2|m
zh~_!;5K4{^ywqbnl^eOZw1`}Nc-Us#BK-f;0%-i36C$fi#C`aVt{xs_@)e*;n#C$I
zz!l#TlEcHqFqM>qfB_v0>ic<7;+j02cp+HsSfCJCjQ_bh9%%rC)&LYh#^HeOe*lW3
zOqt~clDTXj9vS|UtqC6k1Q<LW)~NZmL#>Otwq9#Z|IeQ@n^`MMBD!uYLI<ucd^;B2
zB#2RFxT7qHlK>f_=x!SWsa72i3WxwihpTM>shaIW<yXh&3`}wpx3c_rr19_!la{M-
zRtxAT6?|ZHwyLJtw<cm{g<-?T$#)nEBEr~|Rd%;Gv58aZV6m&G=rln3@ta{R@_+6X
zNS7ZMtS?3z6Le%;FzyO5Oad^Tg-3pxo8b`|9SyAH5kZ}#Tgo4yTu(&a;J2?|V_@<n
zw+?=0mBlcNmwF$Oqk$rrc(FEa-uw>4?1uNgu2AJ*UiJ=gsPG{IEvpm@$IdbR>(?&m
zrG0)+@^*H!t9%<s!D?lpI6(21|3lW`WkM!y?&?6GA#fNN-5F9<JK=q790eJ%^AN9H
zX9{~WTHD7U(1{9PaL=AT8Lm&Va+wJ0YKi=uRs5_K>vYUvYAA%6$0Bx-m=`cC39`&8
zTlSJa*yrKwOoZX_VwrMd7WlUYT(Asi{pxEEgDRWkPJ9Cko2;983KrN4=WeAUJTRRg
zX}FxK<q)-j9+adY_=RqxL-0G9d5zcTlesB)!GC82w5MP|b#XLwrmTI3{SsH54E#gp
zhhh$$btYrSjYV|TO5n&GO|~&dLaK;?dOJRv$6#Y4wgAOo8cfoal6!rt6G5X`2TQ10
z30u!svSS|fPAo3{5~4i)+(iz;m_Pypv0rg_rrgv-i3n>Bq4(Uu3f3oUBfcZs^#@6y
zfSIcSLRkHjpQTTtqInmxGXb?XB9oMOa*F1b;NxL6Xy2ih<gGpqUXK|=B1SwO_yD`)
z#F<RBjnWM4Y#3wt@|`-9F<S^+k2X>$4TgR}&zWuVJPa_sCNa&!{MCrYV0QBg&eiJn
zcJqe0Gtf#@+qcKT5z~aC$RpjmQw9DaJpR?tGM^Gp=vZuV$cK=8mjgf^Y*PBLhu8th
zN;<L8Fsr4=esAH4qkzfCvu>PoqS4n<tM24vp9Od;Xu4AXo)4Ce!7E3tfQZh9_J^=o
z0MOroz25<R#M}4pjWfso-}Lh)X~=$=DAF!HAsyeFFaH$URrKmr402c??|1%Uc_CIt
zcW8|!FvrZ{#?>Mt##lfgCGD}6Bz=E55|a$&1#lt3xw&1KoQ}JvSx@inkSQc065ZsV
zF#NWwdI{tbcE*EddnYri;APi9o8LDyWRekGg7}pbG-4xsRmSMsS+x_yL!0D3W<B{*
zh&`G_8en4x4GCd?@a2N)4SbM#HJ+cJWZNeIU)aN7P~R4SP*Z9J07i!6S+XC6^^QUQ
zosc*OA0*AtJ>~<!{PQ5m^l_uWrFuk=gvZtQ_3m+E{HowhXF(fiIFsh5q#>h9>FPQO
zO(-v=8Mawi%V4fL({L~(rVXx8O)|-Gz#j)3rV(QJt8Rappdc0jISo<IILV&7VioRv
zWSqonlrD}JBki`R3g%j};s_2f{)?pQ;Jx`Mpnz5JFKKvoT=bOtARfn+p<m|tH59A{
z>r7Q^*qxsNVJI^mF!QN3pj@zr+@=|J9NU4^nj4Gw7c!+8)DLVfcaA2@X83C1c*9Pf
zD0@MiJeSC}&_<)3J2rm-n;}?WdoXT3e)*gDU)*Ek)1o;xzIE$X+$CHO?G?qlNl712
z1%q8x#zs#i`r*4XZ}jSWD;5V}M*#~F{&+Sq63cm?Q^yIf029}H;2*dT`6m&b4?F^}
zfMxo!#$$Leo}t0yCTM+#J(mAia>=vKpP@J}kSvMFlaaA}qfi}rd$EdBlYQq;pME<O
z)qG%yHXD$1OSYLYd3($bq`D7S!tKjpNJjr1iII^c0|*ITyz!?D2mVaV!TC-mo2eK`
zGmdq4E4mi``%D4Sx<JtrDP(XP#+nWq>B_Bmz%Xn4Yf%#ZDRAXQ2SavAODE!$6oJea
z&73@|k-cl6zfUv|kU3zinud023XUjDL=D4HJNbF#4urz1D8SyNMwI_cj3;w3NlYcA
zIZ$GRKVP2hhPq1)5(xj_-L$~nQ$gjzR}%Uj#mexqJQ_nkM39rQ%fS8`A=Nrj?3w*p
zY>!0=I}V_Cs8xm9X>NnNzf6!y#N|RZFOP7htAKV;FwbyvR<dTK$h?5|(8%akO7R*B
z8S@1s9lt%0zH*?twstjzNKe1K2foS7$$1647-3joks&k@s&c83=b30MLcq>}iCaZt
zFC1EPmfrnDdq7PAj|58O5|(+%thRnyW69HOIOxj($0Ll?@NMvcKuEIon7*JRa4a32
z0y>_&b3gWj$C<)s;|u2ZLXC&O|1CvkfTC>Rg|=&YxZw<@#Z4~Pq>V0GJ2@dK1&}tu
z>QKFqS!skIhBeae+q0d}hCM~d05&w|F9D}Pnkl5~S>d)(DA`8qDce$w<@tB-UQ6lv
z`SUViMbYPAH}oA4BjXxy7+e0BM^#D$o2caF+!nFSh5ky<X5cO$UQ(5CSy0^&ijuf6
z@NnapW@2J0|G|Gsjh!B9Ay8R}6ibMGS#twnK#dXw@2jdvXsik<E0m80-S*+nm_5|h
zBmAi^hN4MFv5zSPu}%ITK`biZp+G!sC9XXA5`L3!XbEo*<5n<;<uu@9G!f~wHU445
zhXOGN-h2{X8Z>OH1h^Zav{BxEog%O#_3iujpzu707D1Bm2|2icuy7cmQ=xf;u$827
zfR!h$9R}-^;em$kTrtCdWP%kPpxRDoDJw)`LN@5Svf%=8FQbeILn%f`7qp)h$#T9V
zsP{uZ&B{bAT|x@~A7fNW*u{J-UwZr+HNmd&Y+T6%p3TPyZXzQCfkQsRiEZrg$NI#H
zr1QHUK<k9E0FwisSlEb2pjX5$+5KQ_+Q+1@11E|;7VgiBD>$D)uk2~BcP1%dFeBoi
z<A*_!_;t+@6HEfHzI^#23AGyzR57xLKvwz~Bq<5MfhKh`W_c|qY%um3Z{tP07HD{0
zWQ6-b`{qs?TJoe5{kk4(Ar=>Szr?7P{m*v$#8FhA7h4)Oyw!ogg3k`-tkF)_yAICi
z;#`Zd-{sVq%;5cBeH<!J()x=UzR~Q?!v%6Jx}H&Du}{!<VPq)3@dQcQLW<!BH^Z|P
z%omj2L=D@_sw&MUX6bSL0UI}kxMea99kq1WxJ3+^+XvbNzNV`aIu=6G(wwMA(D%rY
zz*MMf_yh!wupQEuf{zIS416S!86XduqV3zZDdg37axi!GawGjc27*F6oOz^#0nxa*
z-JoTrW%~%KRus)phrEY!k@WaDif~r`(V9CftYId)kI{)=Z0zgPU{nkiH3L^jI5)PB
zYe0W0yaeKI#l$G!*YUEI>UZad`~LLOWoKuTSU03a)q2@!?W;5+1?en+z}QEReoFj!
zcQz67j~HkfZ08y`4E^ct{jY9UifLu0-~0dj&!6_vaRhhm#n@{9Js#NMQ53FSSiL<5
zj>M}lFyV-XORpntrv&tJp!NHDdzIlv-8cOE6{4uttzF9tXsX>MWV9|yY@<c2409ut
zfg}eFLgcqz0yG{F>XK&e*rBDH6FLXyGVSSdeyyW>gCgpy<fclQ4)QBd@FN_o5ohAr
zD$Wjga57=Xgah=cT1f{HytVwI?#U7sg|hqhL_<8M22=Ij$FtN~Xza)jnMOO~UZ`U`
zl7!gWVM*HZa9U#n)Ihj8N4)&nK(3R>w}}S9J%WN)u#0I!T2!Q0jNOVxDk{dx;;<-Q
z&pi_>IS4{3qp@4fXvZ25_^bxEvi#C)6X!nbfV<DMWpGAsZcE|K)q9zdg&MqwbJ9_t
zjZZR2V_ZcA#b`Rd^L0o$v38>;oM(|7eI0rV6@<%@=%-2UsFQN$;|k%J6+7j5EuOuk
zZ{iM7(qXG6Lm@~)As}Berw>}v-2huiwzBC<$B5L_?Pzgv0K`n>t%luxJ)}d}PTt+v
zsfkh~UC6%J@tZi(y*Ok^(7PiP%~(!Bn79t5*A&16g599|Bl+4t5^CsJJ12y5-2j-A
z(WJneNF%GNZwc>36P6Gej*V7l5X(OI>C8w!z>K{RQ*8pv+X!)?M}cLrXC&%&{5M?N
zP1&-#83svjr~j_wLC6qr0hTtG5c(PEprhj|>259X+HIqyr6u<htyVOo&A%B&((z-Y
z04CZz1Z>Qb8Q2I#?zCFl0pvgtn+a)riV)zW$uskpW;fUYw28TXey3rKjMdB7K~!@_
zGYB{s%4Q4`+xUimUL1*%B!+|7ouM7Wmhuq)tg}|}n;SM<KtBQ8Htc_iR_6?M)1rF7
zYQpnPx*7Z_O!mU}#Jj*M{kv9rx~HdS-TV7SSsrQb|F=5FUbN}_RjUvc$?8%qbR$q?
zkvs`iJSCv$>@gyOw3eKXr7ujg9|CD7;TZeQeZNVG0ZO(P=4z-Q?PA!ggpqOwNRq@U
zAQ0`AM!s%bl^+}nrKO6Z*O;dCrj0F0U^psa8>mx(jt3eDsuD9XL}Kp4&1=#80=<vS
z^!s}p$`zt4hhztZ1)r=e7g~74u26+B@<icorGYhy(OBbk(KLpHgp!zVBvw~py}_bY
zBUq4t_$fFXwjDgkj@?-ipc4s2$LU*%T0vo`Fb!*j>~*kSQsFL;@gXR2gNuuO<7`f{
z^D=w524GMCCwc88%wRwml^RYeNzM&GD!PV-5Vph!nmGj$FA|guJskikJX|{PVWU1n
zZPANk_K2pYGH(C4`b76|P?ip(!K!GDdM*-Z1>rZ*!vI{ITzaM&8cuWwen>BzoSCU4
zy5-ZS(_7^s3B?}$?`V{i+=#_S0>UXLfi4r1GT<|@(;qjI+`3X;i58=L_3?TkK|utm
zQ<L;`kTu8&r}0L&4m|}1TL*&|dTbzwXY0+^uk3g=$f2>^)*#=ZyQe4UavD$1Y+h5m
zpO^Tu%@hxrOSAM$MF<l)aKsZf3t0LVL`i;H)Y8<f1e-|`J6BN%!wwTSiZj=a1C^86
z-*?T6<1s&8<|owwvTofWFD9C9x~TH4zm6!byJl6Z7}C-SE4UbZlTeDEM58kJJzoem
zQLTLA>_f4)w4Y<Pk-_i6odR#{lV~WXwzK*^HLh=Lp~s0u#_iw)A~PsiWGdN@<R2YJ
z5kMWNv2&hI<vvp=+I`xUMza(96yY;_c=YZSgou)iGLX%V(!hitJJcLmog-K+M{gw>
zCZ?f7z4t}9GM8`{b<zE+WtC4ni4GQ@3toUO^f<&tLOxeyrP_F|Gu>sKAeQ#mktajq
zj%O5n&CEK4c`|swLQV)Dh73k?l)f_7;a=O>mQ2_HuC9|4+4AjGkWhWw!=QC5Zv(D}
z?6??q{2b^cLS#bgSP6WTWU{{WOtSJe7>o5itv1H*OdZ0;#;nf!Hu|?NPl}-sTGK&v
z?j)40?#@bXsW0P|Y1zet%jQN$KkCKBUVfl*eTUK6jgz9B$HE^Ze|{l&LsF2Y&dMS>
zPB2pKtg>U0LvKa|XTt-V7YnI@`dX}4>GzAKUfTXuO{Lj@xBI?OjpFgI7dsAd%}3f^
zvNiE6l$W?xDepWH8u0g3;k}<l6W%y{A}|y<yM`rI*=^AIwM#Dz)LCbL&2#s%-yL_8
zNxiPNTzfAg-6(RJO9|3BY{h^U;B2hJz_~LcH$s>WdcSJLkmK&|#?cAQopEZM6TaC|
z(!ZYO=BIy?Q`P<Y?DTrv9gtIGR&2(z$Ur6~ZU*jgl8(wSZ+)fkV+*at0h=>?TPBJf
zn{?UfX-Gc=j0PJPIO2>l{6eY+*MjZ`<W<%@)Quitf8UV$He{%*b<@UAZppn|w$!Z6
z0!=x8^L|M3@hK<X2(f#_<ulN6!+9l0Wa~kp2^IbmE~>#{t67=_8gq^mWH9}zOtB^J
z4}0!`5eUSXCC;31P7yMnDi4+(rsEB<n8hT7qNckp*Pb739MidYQ4U1wDRg-78wZgn
zy=<eiBhNyZ7`q#~J8mQ_?mlYu?wZUvr7uRAj@R*w1l8)}X|<l(McScx#~4cY=$o1S
z7<Kk~+rly$c$)tMk7t(W#?R*jLzZqh2jIP|xBQq)>vi<m+5S-J>!HgQx9kyk=<(<Q
zo5h|}DH@yd1K@qfl)|f6+vuIrcdOIM^fz>$7f<MWXDRV_(y<BJ9eWLyiYTesXV*d&
zd#|6@=jDo4u5YV5^PDU2FC9(rCA1<geIKsxm-%5pXLS)5U)(L=;L!d}6)sY<FTR`-
zudZF9dVs%sKhRKZyZ_GV0e?2B9|qx3Z&E7FBTpx&ab5^M@PXelJ<}05V>q_v*rX&a
zpwVazF=LLkLQ%^7RhEl#OZav3(PxXjb9k-rsJAJV?eJTv+P9aDoT)8%_pE6}5bv34
z<2v~0G;lgGPO}33+A-%=>wp?ITo0A^_*&yVJ)ei2UthGlTw2$bPF|+2j3q0fna<1C
z>=yZEt3T8(T_3eKQNz_`Do}rQNQqTjVUf`budMVp=HiomFJd^mbClnEzHEy27c@+l
zCx6ja*Iii(4$IJHuwm-ZUq-pX6g|UX@^V$d2Wy{c;SNJ@kC{-e5Yv&M8w6JZ^20ug
z$w;4|audB%2gP9!=!+jkeMZ-x<GmTvbWMNsHy!=kFA_jTeU@kc{NZ#!G(l^Z)~?w8
z5p3)}rCaYFwawBq+lvS;;ImNIXj)sRO72|CJKil?BMuVeDOX18*RMyEl~<{yO+}q-
zwUfl?iQ4nQ=H%N&*H~4w*VJ4^G1xmg`nNpS#nbc5bYJzk<8eMf1=tUq=K@f;Q(it9
zGQXD#e#d!3dR^n4X~}>xBwaF=oR{RpA?8$hr5RP4B3fOjHPPzT?59==KkWM0nFsJ8
zrY@!EKU(bM|1#tb=CZ)WkP*>X90*>T@Fh<$)82laZiYNK^cO#M08i}h?yl>2EgP|8
z|1;g}>2b_7tb@hk&$oL2cbGRa{_2b<JE1b6F~IO4jH;=M_Mhd;YUT}bmStcY$N3BE
z>9QDS<UsOSk&;QJ=kFJTM1CjLw+M`F1w;7&!!u&5t6<_dg6wtYtRL=hek4hIfQO*}
zrlvsG=!h2qOa~LrY_hVn)-G0TEp=OSgVy;?>SoDaMFoi|bc!}Me7j;JAdZSc5^X4|
zcLVZGvIj)#3s-cIz%A!9dp{@V)AV=!V{&AeMxxvoJ2i>;H;y)U9Om{(N|Iy{bS8f9
zSzx|o#ru&UZAb~#qRLs7Uc^_{wRL}qY{IHC*?QF-HwBH&Ud|d$iFq`hovfqIBgYk<
zf2+m*zw;~JwkHM`5UOxZ@BRDsiGj-99GT|%3R338LrX))82wHG<RJVr3?7EQ$9PrV
z_vUqPWoJJE3WkJj@bd$SY#kNTihuQHLrX=wouoLX!Cl_^{hWA$C_6od7tW;AkFMvf
zsytZx&T9TS%4w7oYPj8Vb8{yOT|=>NA{3MgY4=Y@IlqDCqu_)^ZWy5_pGR{%&*HH3
zGt96BatfZV{prq%rTDrtpxcPU4OzsQB?U~WHZJj7GBK<P62vf6^bnXsxI6%i8jOOq
zc!FJH^bl2!oB2>&)73Kb1r{bI_`8>Y&YT2t0wHt9{PmhebTr!#QF!UO3j~L$0;sJJ
zlBkB06@d}r5Z~72n$b0Oe>?qZOQaeg#A+z9K2%rpV#F7o3<{|qAzI$UgKQ-r{HvgX
z2UIgPJ6i<^3GU&d;}!b3t!D4`v(w{%(lIenKYe;9Qr7(z#|p?mAT)f$n~*91zXuD@
zYH)5K0Z$=L4fdZ-{oLwP)}Y8x2>yy%>Yreky>02Aa1VCs&rf!Q*zGG`JCD#*JPqoa
zntXs*i;ARyv~xhckA4#Ch82}3=9ILY&B)DtbpL+&^wpjC+z<(GRJ8)o84?yIjzW%P
zM;c}IM+f}nLvKMjzozJ$<2x`3Y!=>^P;!%c25FZqBb$}nxyGd1U^bLORfZ5mpi%H5
zYT}7NO^X+MnKgDkD>D;bnJ$3de|vdWx;j^u4}dR2oq9b!{s>k8t|)SMLLtz?>pJ{d
zuSCb&{AHXxKWJnsP<W$st;F_%VSgn22F2uTx%QjycSwdJegJ#jAGk<|VCO@NfdzLH
z)X3*A{cCx-P86^83Vu_Lev2ILKBdV(ji>OltXsD(HyoxNQ^YVhRQ$wZmr_)eh)+N`
zFmwaw%oCX^6u-%~ZJ=2NlKc#WzIfD7Ay*;qMipqOqscA9=GIoX$u32xJ<5?nh6ds!
zBwZ}iN^~vX_OxwMB?GdaoG84pu;2qYsse2!s>`gsAMN$eJWl{Ji00>@*Eo?zoq`sA
z`sOtzrG5MV-90pvQWvIo5VlHKI7d6#)!wQ0j09}Hh;+FEI~B16*qgE_Jx;A-dp?iE
zaY(B$QH^Xr&}BYWrjcp;zGfQ65TxY9;43p@ei<eZqmXzhtW9V2^Sk3fM?-@G{udm?
z1xdoQd6iFOQh}EdUB~U)Z1CW9gKT~2p{_(H)3U4EAJe1!r><HOZwM5=qth|NFHy%=
zAOeFdE~kLSEkL~r{{8r=Ip^2#V*`jjoZIRKazU!=k63d{3;7J_Veryc0OVf^tJ7K8
zKH(e(P`?AH0pdH+{P5Pv)HO6WS7GfQK>#uzUoh~XA4NNOL%5%sz7o1Z;*Str&yU9%
zu)!Bjr{0EZYJ4P(>d?v`Mlf{w%JPdMA$bg0a(iQe<e=RFGIZ;t6EC#HWGLLxM+9Zv
z`{U(tx<+1;oo%l%;u@BOPf!(d$ap^4*OB@b#=?Wp76CR(&epuJ2hc4jsLNRRH$bDB
z-R29)V>L3lbzv;O6$0@c81ezYCjYY2Q^<S5fJn?>Xy$aSM7{HPo0Qq@*{?v3-&w9K
z7!zwZ1b5jr%t|lZGV9xNPZU<*JG{fS`{B81l!7kQ7H7^&;WSkwYGh<)TB93zDXdCo
zMQgw~k7KiEWTYn7e@-a>vQg-rJ9n}*?U(@xfhmTrqc~fWS?McPOXU)F@{-xlnx8K^
ztby{j56Ly%a2t`=Y5bJoGp%Z3!hsgP1Qd${!}>O#7jW5*5fMz&5OI}}qkUvALaJ3U
zK>?nOw8jrO(mBw3Oq9$ppxR#5g<HM{%s;di;AKAXEKkzT`p8M&9ZzX(MqHnOyEHce
z$#F3rX4<Uh<+_%dnp?J@L3I<3;C+`3H>&EwfSZX8bvJT`vBiA^5WPv^&s=Um`{+|j
ze}ZkYE`EldCYS}&1fQ4}e{n6vB!ac{^ajw)rVY;dsOoOqDJrTn^d<X__5M4XRCRA+
z(WXCnGJ3tWdB1g4_~xZrrSK@#meUE(o{6JF#zup0S`J&9&c?UmTaw`=qd<q5%{dRZ
zKB*BgAg^*d5jJ_mI1hkLBm-$Fr54c^xwe=9fkjVJ$!ivtDswWj0k-;`$&N)X0Ih?%
zilPDBB}N=(blnFHHq7Jf0w0k5tV13_j2l&Xp>_GavW0Z^c&}d~my2)f0Nh@s0C1c_
zxKWTuWUGRQ6$7<`0VVR~W@tZZvJz!~BL0%8!sRjNI6gbJ7*Mlm19=p3)!~DV2L=J@
zJ85b0@p_RDelhD^=|%Yw^sy7(?a*tNyGnmXh&N_e0H{bptTZ&Tb=VQ7Kb&;{r&!Ru
z3;jT<!>dk)yI6JmF0Uy+CGG+wn;c{~zhNgN%2WU#MZOgUwu1#L+h2`gLIDN)HB}xW
z+i|oJUnUX7qjq~@QM!OjZ)9pJ*ex95ExC_v(y5~)!+pHXBqBQ6*zGuO*l9dD8sHP+
zV29~Pqizi1D-MTLItpErUW`H!4D6YhaQfp1k)j4R%{Y9nnpVtFJkBBI`oLO}6SsLI
z<|p^#VZk$VrP0m+i70WnbaK;)iLf1PL|7o{z~ph^o<1&HD!;i~9~n&iVq&^=R#Kj$
z?05zRQ4@HTjE{}gq2`DONyYb`m}JQ24%(VR2na|VZ)10z0#J4XJKeE>1$`99(s!fa
z)H`0}xQ}xK2A=Uk8UauR3D#tWECy6x7~HpyK#{1MF=in9<wKDWXZp;`zDPy@q|$@s
z2M?froEQ!SENPth5Ws14eG%8Tx8DtaEG9d)u`Wa*hIS)z29eX}FrpX2G?p~Xor%Dx
zhoEA}N6U;Xgqa!VLc*bdvVMdk*5Jsh#<e_#>(yu{8T~i!QDnvVt-E)%eSD;GWaoM8
z#@>D3=T&i0(K8@+@_#3lQ41Mz%w*-3QJ)0fjwZs?qQ0(fC-R)Yh+!@A9UAqpyzu;+
z7J6C$UnGF;rSM_wKu@0aJL&7Qk~zFBEpf1kLe^!ruLJGI@Ae7*!|1P2z94O&vDnKA
zJXr#W9Q+Y9?2w0zEdp}XJMdabz>@xN|6FAxb4KLQp?iSzS+;Dc0T%pZYBRSIv{X1W
zY{7oul>7LlpN@Al#E|jB(IZEo{#*l`d?aelb;=t~bm9-ic>{JUqosa!J+CxF)<abs
zX26E~vkN*Z+S^Op$EP~BqKUvVJBlX{#Yy1mVy0A+C&}z21~Dfm3LBfgsp&^2|Cudk
zU#GVdS0lRqbl9YzR6{VZt=a7zA$bx;Qri~KBZJ^1DgcCoi3cqEIh;NFqvzb%XQ+`h
zAe;h#(u>jtlDKdGPAZ33G&X>gv5<hHva+x!A-Th>G~XO!2z%m;Ev%6ESMq1z4*s$U
zkql<HcZTHYpW8f65h?q<9h);^PsEUZf>Yr1W=05pV{qc$NKXEQvdIhP1A()nO5sNK
zP0qC5JrM-dvjIH^IUG3=ZxJkW|D7L28}A4S2%N+@i-Z($>_^~X(O(sKm?+WFHVkw#
z6ue|0l2;&ZgeXM@lQ_^!2G;47h1S<t8`1>W#q1B#q)c?2i(`Ef!o(e`U9q$uwJfYU
zL<v|gH_t^L(14X?5{g!8;Z*b4z*aorplcr;vJ$pK0f+j`4OV7Ix9bo|;Ao!ARPp9`
zendv$r+bI$K}nJ=1l<oLa%2P$cE_gSz4VU#82mSi7*ia#=p8Yjzy{%>Bn&B1&1Pge
zgoz>3Jaq9KCjI(hk`4)e=o=Uyha(2{RN-_28FQ_L;hs@&ssL(H2>)yGoDql;tUs10
zncu6R5SXpGQOTV$Cf*h>h_~aCb`M8E1}&Y7mzOQ@2OzMQ0MAN*Y`YUyidWjSX{tV8
zITkv`D4s*GymO%X?z{c^?@oy`wI@;JdoJF-%v2$tzv#S@va&q?`W8i7Q?nO2B9$nb
z@BQuLN98SnF(}FY3-;JRLFgJUxl{m^Ak)I|NMke@)a<v3TAl`u5wC$iKkwP%{-J_K
z3vf!x&y0@udm_GgJAN3w#?uf-5a$42SCxPJ#sRvIL@I%bK+Ey__wNQgS0t?s3irCY
z4cIx+e$oU}1>%ev9v(*A>3z&}M&>wb9b&L0;|bxdAT!XhwLxx7Zr;yi-%s5ITQ@6x
zWh`cxUu+ziBfSZVRiMgRxBxi6Zv{Xoi&CY+aladI_2H3`Mu+@c66zk(+(>Fcv(gK_
zE6FtR_m`b()(1n~0C!}vchA~F4#5z+RxaVTP$s3Xzooo*Lzuqs;{0G@I8tFw$IGR|
k-xYAqJcLP!&SQzErPP`I*?D~?3jV089#zRvIvMnT0I@OVP5=M^
literal 0
HcmV?d00001
From c4721f5ab15a841e4d5f5e4791e49de34d333228 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 10 Oct 2017 13:46:09 -0700
Subject: [PATCH 73/82] gan design with graph
---
doc/design/gan_api.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index a1626e50d7..f99d1e6540 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -28,8 +28,8 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
| Repmat op (done) | ? | N (Cond) |
<p align="center">
-<img src="./test.dot.png" width = "90%" align="center"/><br/>
-The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes.
+<img src="./test.dot.png" width = "50%" align="center"/><br/>
+The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers.
</p>
From ea56018907d89ac541bb1fa0e184159142a160d2 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 10 Oct 2017 13:50:06 -0700
Subject: [PATCH 74/82] gan design with graph
---
doc/design/gan_api.md | 2 +-
doc/design/test.dot | 35 +++++++++++++++++++++++++++++++++++
doc/design/test.dot.png | Bin 59401 -> 58935 bytes
3 files changed, 36 insertions(+), 1 deletion(-)
create mode 100644 doc/design/test.dot
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index f99d1e6540..2fb30432cb 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -29,7 +29,7 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
<p align="center">
<img src="./test.dot.png" width = "50%" align="center"/><br/>
-The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers.
+The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers. d\_loss and g\_loss mared in red and green are the two targets we would like to run.
</p>
diff --git a/doc/design/test.dot b/doc/design/test.dot
new file mode 100644
index 0000000000..62c69b8fc8
--- /dev/null
+++ b/doc/design/test.dot
@@ -0,0 +1,35 @@
+
+digraph Test {
+ z -> generator -> G_img;
+ G_img -> discriminator -> D_f -> d_loss_f;
+ label0 -> d_loss_f -> d_loss;
+
+ img -> discriminator -> D_t -> d_loss_t;
+ label1 -> d_loss_t -> d_loss;
+
+ d_loss -> d_loss_t[color=red, style=dashed];
+ d_loss -> d_loss_f[color=red, style=dashed];
+ d_loss_t -> D_t[color=red, style=dashed];
+ d_loss_f -> D_f[color=red, style=dashed];
+ D_t -> discriminator[color=red, style=dashed];
+ D_f -> discriminator[color=red, style=dashed];
+
+ D_f -> g_loss;
+ label2 -> g_loss;
+
+ g_loss -> D_f[color=green, style=dashed];
+ D_f -> discriminator[color=green, style=dashed];
+ discriminator -> G_img[color=green, style=dashed];
+ G_img -> generator[color=green, style=dashed];
+
+ discriminator [color=red, shape=box];
+ generator [color=green, shape=box];
+ z [shape=diamond];
+ img [shape=diamond];
+ label0 [shape=diamond];
+ label1 [shape=diamond];
+ label2 [shape=diamond];
+
+ d_loss [color=red];
+ g_loss [color=green];
+}
diff --git a/doc/design/test.dot.png b/doc/design/test.dot.png
index 768e5cac4b14201f378896bb3bcde9e6ee540414..4e121a40b9f7b2232d7cdda315bad15926446f55 100644
GIT binary patch
delta 41475
zcmag`cRZH=8$S+TA|sMf$jB^P_Ff@m%Z!jw85xntI*qLCEhHm5Nl5mJqL3|H$jaXP
zcbxCf_r4$B-~G@1*Sj~a>pIWZ`Fb74a~)UcFYL;n*dJ|fBlmTNj_}qs$+Q%9$*^uE
zGKG+jW$SB|PFdfsq1F4^!(m&L*-|27d?fTNbby$)*z<v%vGE)yn?vg@6E3_CvRhUi
zH>M9xVq#xq@jF+EI8BsL-sGK5-+K9#m32P-DDAD!Aptu(I~BhbzVMS_-_w&L%e~d{
z{e$_p8?BUP5B8DuX?%SAM!S+aTsk_s`WA;5%JQ|F9h`#RhtKL;@{_1uWgZB=UE^J1
zN&3Og!STw}dq*!;ndMz^u{vhyxVN`AD^<i%x~%xqC;pLoZ;1#Rk;v`QifjT4zKhg?
zA^rgY>`YAluU@_CzlNBbGt!AaLw{y7)#%3(qnvM2^BfU0Z^jv^a$!wKN-C<;{M%}4
zYH)gb`YtEuc3WH9WhyFVgA!BkCATXV0+-w>Dk|E4{fa(^jlH<maQ2sRGf~wt_NdVh
z+h*fWt5<MO)>w^OlokKg-^0WCu?I8lCdS68ZBg`R{ZA_$#<TV&k+*8<>UYo|M$)OK
z3fpJlT|%x29$C7tPhr`Ql!u}}9vK<2dh|$<-fJCeyxzO!x1~)4+@<o3xR~N%?$B0?
z&CSiMV&lrYHa0eOk=-2~S-*a1Elt!(U}e;MAKkUKW}BOvdrJG_M;>2$opGf@x$~S#
zeT#L><qPN0n}@40o41x07f%TZiH(b6=S6s<%gf8ZkBp?eyUU)Fle0m4(y%(&pvnI_
zT-<F{&7w5<_~Zm0VCdRMqqNOU=L?rEvEICSlPpxh>fyuNcMIa-T~hm3b5xVqczL5o
zDjhY&#l>gW)-<)XwRKHQigUjkQ%2lb)z{a*OikV4M#XKa?q6P3rgPox@#DuSVlK#y
zJ1k^NgT*u}V^wi*k&~Iyarb%hRz@pzhCZ0wVT$?n?|0eabW5;TaLxu98JRLiLhS3;
zSK&qILaeVl6&4oWwUar2{`_SM3dK6ly&IU>KY!%l@{=@!Qq(MCYyABFTiZ2Wqll#?
zTXBzVwig~Au!WdZRaHky5zF9=xWl+vXEFQH&k^t6zyIFfA1`b_@>S9AqOOrqA$1Nm
z11l@5QK`lGoZMVt)w3iXldM2O%Gl6Qf(rZ5#E8E%vdpobuC79GNh4*CFHL5aQ;Ug-
z%`Pvi8W|atyKh)%XhaGpH8;x{m0Od-Q;T`~mY$|{NWSxOXnlP>!VqhF@7}%Q0tOly
znpl;b_}JKHRml|eJ)b`1!^#N2yHwcl^yba1nQsG=A8;jH{u13YHjaDqhVG5DcgmVm
z|KI)s9V<IKRbgS_XRBjk^(~}nDT~pjbsnS`bXnW2_&eI$rzOgb$jHg>z~WU^RV}eN
zxVww$>FaOOr63vB3eK>Nm@v=&d?nZsAtqTFFFJiL)bH4EqPQ|r!7L-wKv8AEC?b+%
z`eN=%@4!Hg%;|o>niEduqxcU+MXK;OIuX5rtL@}9bP~^53UoX4RZirYV~^kw5@1@}
z+TijdV~?(l$n#y!-hE!Rz11xfr@)l?V(V|reV%zlg0}1Xobv@DB9>4EIoPms611lM
zX`*h<&Ud+Q5fKvh4KG%1Z$BH5!;~7T!D{4w{P@x1X8V&+L8dCRdau_Q$vyuR#TmA$
zy?5!`eBSTjBQ{4J8)D!iqGSfsl2bWEMU!ERFTgSN3hw1fuXX?a{d)*{hBxPacp<^L
z!rdO>1|ROj8a>J*Z9jqa#PQ`95Ez&gRciO9x|qGUAXQQMBQQCcs<)>Hz3vS<l`=vi
zqIb_%MtWc?>=OqHT)ld=aOW&4s#jG-WuG`g=k@E?EOG7MdwNO-XMDC7NGCIe2qTnN
zNl8g@d{d*sX|KD<Y-Se->a8Fvg~m{r#6(4-qoSe|xm|QAtL|%LD@HKEu8VjRo3DEN
z_IWx{Cto;*a2ld>a<1RWxm)eJOaX5x5h!;nPvgzqx|rqV<vzG@p+B{9I@Vtw(M4U8
z@>gc*&Htk%$9--0-FZTajPml!#-E?O@HyF~P+^LBusm4I#=+5yGzStM5a+c!jD7K&
zntJeTsh@!ShMKI8kM#4kiHP!Y!S7c5qGDnjbQWdgVXbHxZ*Fdme*HSMr2g;%oX~*E
zN|CX#F%mMetdX{|+tWwmInq9di>$Vj^->}cE`rW)HMbV8d)!%BDeS9qnO9~Z>w(hQ
z<;GS!+JMM}Dj-ba$f@LHrXZCu_JpWV(aY`aZJg3XvvGL5$k9rrm5oii`i!==!o7R8
zj=S=Q(H=UE6_rIA=uSv7CEKOZa-kM~#&(fLI2@V;HxExF)Unse$y!iezYh&1HMg`3
zeClQA<P6(g9ak;VFSh;l6pBUd_UPr?w{MrbBFp*%dSAM_y5MoKg$309N<%;R^k9B=
zcJ`|q)x}S*^kW3Xw&>af-bi^SV)9Qc(!YOKrWSY<UdYp_db3X9X5BW~&dO-g@$s?s
zOk0FM_YY&2Kc?SV4G0Md*}1sF54RWPG5K~wAJ7kwkTACQW?^CZb$qa;x6+`jr-xiA
zHmys+y>v~DEc9iZj{rZv_151`w18sc5?Mc^zez?<KgqS@k`kcEvUf@5IND=psTHn^
z>`qL%zy3O0*LfVA=$aY{R{O_RRwS6Ev8u8qVb~YQ@dh5ctK#CwTU%R&3hh4>Y3(lb
z1pMuK^TUQdqG#+1`Hzl{CFIRmXuObhctk`&fk0jNvWSpS#VtOe)-n#Wi7J=it@?r6
zaF+eg6Iw6)%7hkY0f&{JpFhj7{vfDKI(_FYuYf=t7dLl1Z1pC}<S}?n5Yg4F4<yUm
z0|sU>rKP0|`oz$Zv<(fZs%mP;G0v{8*I%nCLK*7pMdZm58`wmtX=7&njxN{tzKz$u
zh=T{{3LmCl6##wvQABp8pv4OXD2A{z_KCAwzxbSpU$}5#YGq~opSRn;VbZtWyH}wL
zZR`x04IO<64=36`JcQE|+_K;s!>q(4BSXKjv5~gB#U*qUsgW*uH8C;Ku=*K0E-`J$
z@GxTd_1iajjs)AEiW}U4Wn}`;zE{U<f?I+wwG0na!EN4vyW@M@MSH)&=gk_)@;fLI
zJ=scJRYg6ppC4FT;~N+lm>#|rv?2Ry!N<hRyzKomk=r;rI@<UBd#;~vB$CR8$HvDq
zOG={39(OwrDw#lSAtfU_2j|<}(-WELdLv<2=|G46l!78i+WSz@PSUmN#nz1*>M1w{
zx~~d!@+@FEzCg9nX>eN}<la9xXo5>iPfr(p{x5l{9-z<5moFa-zQ4D=zV4fpl=L8O
zCXR)SptiP_<l42!{(fEMoVyCQZ)Yk+U4y@947y08b@y(wTk_eTqijUpJN&9Z(8P;*
z0AJR8u=eO??vq~kkLrenva+zq8WbML0T8r39Zc^$(OtGnSND8{dft2JTO8sDH{tB$
z#K4r*(V@aGD99orA~J9y)W?78)~(A|uik-H!MwP%6l=((zmh{uLLwU)8d_*sTUXcK
z-0XjWknrtqWNpjh&*;n?@kqIiJT5LSAg=G~bnkI{|7=C%)V~S~gN5q`*r4d(zysjv
zMZ_&tRYInikRht{)}KF#U>l`XyiG_56tWvcOWODLc0Z^{XzhAB)0U~8Dx9`6stYGd
z{#zs+7MoIOZf@=u_>E4(-eTU{=Eg<@gV8cIjl3O3F}+;U&<5wS)TZyolacbp0!!E|
z1ZHMt4)hp2c(kybQ*(2Hc$coBA2eM1LI40Q=F+80KhvdXu&}Ud?=$3+w!i7Kx$5XR
z6{qPt-BM*^=BuKr+VjCgV&bsm{d<My&xPS#+{30{08onh^T(oV1evUHyIH^ghY-^U
zm3X6_^=;$WSX$X>|HJm!>ufwcZElm^!L_xLy1KfuJV`P;o<gB?m>X)zzHr5ijEq>6
zH%(s_6>&Pw3w(KGZte#-ta)^l#?R01i~9!C2a}p0?QF$Nm-(*FYf*|nt0*ZcT?{;?
zzLPRYq^5p8-vRrd>fqp@c88vhE~2oIBb-(&wvs9x5METDrZ7Cjkg%`-U5?gBI*A9C
zme?{fGXF*^FI@=qFY`%Vg6nm1a|;d#z)rkrW?F(}5KK&m!F>Mw`Fm$)>v)aZa#|9_
z=FEqvtSqJ{qn|HMOiV0{*NDK;Wr%B!PE2f5wZIc(L8_AC<E_UC2AD3JOARs;-t!9H
zyeuFf&}HyO5^a6bhYQTxO9MsP75A5@*$a&|G>GT>^c@@=3a?N}JpU*A;)PhQ+Us)?
z5)wFgc+A(Y2lk!PSi)Y)FD&$rin@fE>$rwt$a;J&E<K$Xv%a$v7#0?`^m1tipyWrN
z6OTwF>#zQkCtQDjzQ(q*v%|r~ZHA)-*cCPj?z_7MOPvvvUYdtP!7(wUfq{Wf`$Uy0
z9`>@ok<Aczrnq7+q6a_<Fx^xA!}53UUSgu(yfG;rwY0RfffL(b<?{GdRFtDrO=M)`
zyP~2Wi+_JY&jI4GkYwgF{XXjP9TgRXsX$kSf`X#fk=|g)#rnYmSqvLLf9&&r!?*ta
z{fpykVPV0>%^h+7{(XF3TRI=WnVqR3QEd@4N&^D}y88NxySuwMzK^=n?!anfM9G{c
z!|Bpv)qT4&^5R&!zC^#fyF2gWN8Y91Wsi%OFTMOMQliM>22^Y0(?ddN&`A843wx4m
z1{;~_y`B_K<?QV2fr(wHIo7Zwg`3z=2lHSHp`f+yaH*)==j2eap#au84v<J*s0dK9
znmX*I&^70}_V)Jj2TkV&T)sT&r0)3fBYS6gC=Sp!!0lcrm2857@!{d&)?;6uZm&<v
zzX}g$fy0GROM8hz3;Tt(n3Kbu3fu9qI#0BlejgrA7Js%N3so~04xMDB8B^B6f^~|N
zr!W=+H#heM0s@;sb;WcpnkNq)FzkP7qK+ABwcuNss7p<46H{5u(N$sig+=z4M+RtK
z4&by+x}d4rqt%-JVq=l|7A>1%8hiXe<c8xu<G$D)Y<^t>19>H-;59931w3&Q`S82(
zn{+B~ay8#ZzL9Xt%F6Oz(_$bCHQi5$kN^6D7JAxng&ieYIJ=GtZ^Gt*D$Byk%FMxW
zftZ-seXVZq!NZ4vVlIEDIuba&f`|V|`Xqb=+zt&!F_Jcv)8NBYZ;mR0Pejxo;^OPC
zxhmA22OLCFsh5{2Bm<i+J~MnxYYFASm#<tgKiXaK3O;w&$KPu;DJ+a|{^^t7pUz*u
zemz;-m1EvpNVn5t3CIx8*6Ef@G78V033}|U7FUaJLaXu$F6e=l4G`^|r>AH5H7RO9
zr-ltabxR!6>4=o)CY?pg`tIw@^lQ>y%9w?Pp6sz_joMOfOiVibIcp0N8YB9_P=%6!
z9OZxclDwwnzPHfB{HeVB9n>Je!U6rdVf_OG#d_Sa1#el4WMJ>cKpz42n!5HYMQy#k
z?(gj6Vqe|?aZZ!CHtXT5`acrkk2|`$ZeU|$Bd}P1B`!~J8-ETbx_FTt`Zhd(oQw=#
zz?u2bc5r-Q?*sj{`8P9dWMRPs=$BtaWZwHBnE}q*vADQ60*pzmdknnj$@SuQ*<X8D
ziG>IV37Ja`e~z;<UvxU&e+p!;9Ey6qymnYhPi=j@PuQ6Bhtl*4I>sbb#O{D)e>Ocj
zS{8$&qgHU;W2IsUWy8IFeQ>?Mf%Dyk&SA4T+wK*d^&ALtp%t#Lxa(4XR^0Eh`_<1v
zDR}NHgHlku)Ty!Fegc#N$Jb$^w%v`5TOUBQ>)OPd`z9u-_RRQ<(3=>c058EKiQ7B9
zT%BSQlK2tvzVF?yrn3W>V^C19Gv#nlcD-Hd#*H9Md3BGX%a8Bh8KAapZf`%ia)$M(
ztgOIsvfj$kv3vQvYUj#5RuWi%17daVbXWhXoT%Pqru|tT;Bn_NHY8%~{DDrzKzSCE
zkf=;P`{!B(tU*j%JT^PqGR*Kwk0BH;76e$z;Gn+i-KWshE>4_AJMOJb=KW7os9BQB
zhJV4MEl!w<I|33;fSH@0-=b3~icUzl?R~U+9V+!9^v!)@r4?dYQCaA&vx|#&e*gaM
z70isU=|F8^Wu*YksPLlj%Y_Yf{__8~fFdE-gphbS%Q#J8u%;XYr{``zI~EnOx622l
z(zCtT2b8+!PivTv1B*dYZ|_}{K!++YrSMte8jqe42_=n6h>GHGI(zCzW}d=k*jpJ@
z!q{G+zsnhE`=cCZ^s_y)x3{<RvuE#sEo{<hWS@ZSG&?mV3pk_P@*8n|OAkIG#5@Bx
z28>kOp5AS#zu-~S+=iSjP<LP+cL*tY-YpN6h<I|PC@^<ivbM96XNp;eEl<~~ef-Mo
zfH=D+SL4mAuU{41-QA}W3{3r?PM)3~Io7unR;niP6#M}#MFyDn#-Op|23?{eNm#40
zib^xoS6;h8F0YMNgfb*FwCPu-{Pf?yLDA798kI~q@ifkOczD@5c^b*Q=I1uIwo2`W
zc;IN81J_;vsVDEmPO^T|#MBCK&AYt3Zzg!Okz<d45p6QkYm7KJC`HmTYHDhNL?h@h
zCVI<^&ueq`&gRa}bxeN)f@rAFR9ZprD7R?60DpqOe_t8x4ZK(E>Fb-Z66%Pfq(%h>
zU5=a@x3$8YlR*X8)<<g%tU5Y6tKw6<Rd{z9Uc^wc+HcKGH3bMhXvCb8g=U}Ux;#i=
z5$7L#iN3dT(nl3KG>$JW5tS@}f<F>m5juK$w*e%<UF?i;KL0=52xt$YhRKJ80&+y{
z9V`BS=vzw4-T%?I%G)I^j>J$KC%v{Vf=u#s2rIb##Ka>hsE1c$<nL%}7$rLcgD)yO
z!4sREo4YM@yp9d0RLGd9prBy;WN$JqDTxwQUDDI9b*4+D@|aZjKjAB90Zt_N{GXvM
zGFk5(FhMmlGou$;#Ym}KaCFhXKz+U4duM3?wn`(c?Tx+G@?_j!7>yB%SAb<k>O4gb
zj*b9OT*G33f|?@jEp9*j(GNOxety0(_eq+BTO<@O>$&05SqD2+jIyR?)D6vaHP5}(
zD?eWmtE1hITGWZ*bxKO_Z=~!o&48kJo~~`~bGVJkrJI6(Vkx5fPIm5UYiB;K7_+mM
zr_sy*BSECGxV$_A9OoOHw@6Up;A4ncHPgR_lC!S=xY>~L?i~(O3_i4~AP@?lJbCiK
z#)iPf#f9qX)mtoNP2a!cW3;riFidi{Zeck)JHr7E1gi87$<1XF61t**BPAtem6&At
zDw}zgULpt!8`pYFkw?yDZ`B0&0j{2&o~Y+8SM{^S-u&m!o;{mdTf2lH2+Q^B=CBF>
zlXHRYnQt%jEWn1}{r5Wn&cycWIE|Ut#`y`4Wo{~J>cRt3J`4ksN7vxox%D0;dPXLu
zb4lVi&Hm;FFe_o)+<XH}I+85KP8;dJ1lid-8k3Wdu1?g2efc5^6pdL*iVlQhd1q&S
z;Ch)wMMWxcYsPd)wc8r;k;WSa+ffRJbI<=_hL@6VQ~^V53%?owsPzx*(S?k^XLtz-
z38GMDK-qgaJZ!jsbYuy9X<@Xo#OTO8{l@-m{j<fpX_7C<=DXj{OT^<IU4CmJ@|>A8
z)GL^Y4Z6_n64Scq<mBOXy2MXqSgNXX-MF~27EBM2hdHMV#(scH)Y3#rfsze_szF3V
zv`-wW&D>PtlJsNlS1;M|KkYZT4=|4`ZcLivM-IreiwX#oP2<V9udj7KeAuYAyJsH`
z;#uRjZ+IXf$SWw|sQ%-Q2Arl|>q~%9G{2l^!H&KXgwK)B_M=~B)8n?*y7lyAtq_*=
zt?nhl4_6Fpo^zv61%Q?XRPtYggMTGhg0SJ}TnLo2R9aOVS70IqCQFkvMlX${kcFJ7
z<8T>b-sKeBY@~P<!jPa+3Q{GAJ?~9wC@e~+bTEC&{@p1llFpCQJ3BfqKv!_w_Y)KI
zO-yv|H_iS;mduAO>QRn7|MOUvgOiibvW@5%I>9g~=@afV7whFML~ra@+6y~7z6RVt
zEyKMvYh6}&LDYwK_=`4oX@wvhKq?Uj+SB7LO<It*uJ7v0t+&647&E?MXSdv@`9s%0
z##@4v<T4HmtBBMW3KmXI!j_<mEdxacPobOvib+|EPJV<y!|OWuK0Zn#`qaN>_q8|_
z2}Wk->}fn{cg*)W*U3CBHiK?_PglpXI~p2oP-nGubzeed&2(Fv@CyDa$lS#0)}^$n
z6v;NEA<)^;)P%LTxS0R`{gYw8y)_w$7Y2G*d79Xh$mBLgXbj2?pVO2q6%EY{2pvw&
z&VLXImfY6_(p7ewX-C{+a`<u?z5o+(zO}a#TnIb|pJ6}j7j-l6y{fCwdJ~?khsR7n
zpA*mvDlumkP;vsZm7<z~@2y2sUBmzHT|CKRNy4Wn%*=B6(%!tm!azxI81}nFKj`h5
zD%`PVWF-4mx`RDM-x|7``&<HIu)b7Og6{HPLqqfM#985q!$YGy+nRsKyq5<zlPMoe
z{Ka)y|Kt+Wxzhr9Y<v5;Pf9V$_aYhC@IykhCXl~Gzp4aSpA!Basjx$zBiFr8I5KC)
z0yUxq3FKab{j)JOI2~W^IhUlCmASMit~!flWMe0%VoOUm{y=VWDlL(-a>UhC2)Q*~
zNS4b)X)G#a0Qk2&JwAXn#`m3ytGN+1W*6%zv}9QQt+Bnmz?|95-hQeldxVlh%H!Sr
zJ)dcR{`N}#D_5>03)@ql?$2EMH&Q_u$EpdtSdjroh8sSYJ1H(sr)7TZZS6TdGkT||
z+Bf#}k=%?745+1YR#q&az1*{SY)g6^oxBy}sT=<(T>{V08IcB!;$3cT)~8Qo_O==!
z!fmxLd<zN=&-GzoPX_}>ye8@HJ9FJnT^WO6z}!)|rTZ-^`*C+Vdgb9$aT4>)ZZ~gs
zr95gsIe_waE>O;Iw9>I<V4z~-V%;8+e(2weAs`@F-`MbjjsQ*Ift}qCH|1~)fsc`5
z-29kCCi%N)Im&7%B+4H?eE8MdD|i1s?PR8PI0(u_^wP9ojfg(|11J8c!2dQ<4c$M6
z0HUA=o(h9GKnP%)HN#>a5Z`7#-uOoEelVBB=f3_xK}o6Q=T9UYwhk7#m-EnUjKr43
z>gwv3=f$q8e4`{XC)?zpAiWfsaK~;q-Xw&F`wcFu@xRab{^kXro12@kGvWuB<7bKx
zA&9cl!j2O^aV6C1@K5!!wAc)t9h*G17yRPmF9QK7^_F~_p~beoTC--+m#dCXNGK1i
zYZ77PqgG&6Tz&Gc*BRu!fDq8V8r|6Z?}Dbnu9N!!Xv;n{VR$aoKm^`w#Yhy*cX2}3
zTv%Q%dTjgmWG_L2-{uPl9H52G43}DlgonQ@DY-cb@2d>Vp}IO{RNd>Il$$94zc1C*
zg{)O)Ye>huP5>Zti}fi48r%2A*8-1uVM(Cq2*#s(jZ+p&j>MnKYJ=`o#?C?yHmLjB
zO7gzAZ_^^e!djvk<uh_}0-*>rwYCPr`dx(juc1I<vk<v^o=uk5KPGap=Ba`=Ksp9t
zVT#Si_HPZ$R#$z4c#0%_jzvM<M76s#X>V#_VMG|ViM43rnAq2jOZ}P+J||j6MkCvC
zg{!Nwwo45{lR{s?NWrD2rw3_;4CH@aK`Y8SKJ{<062Z2yHMHuqF`u2_sab<u&SP3T
zTpr}Ht>v(gd25bnu-G{GPTUQ1Xr{xJj`YxUp{dv~0?rkBjo%@1y;(r`*&~pz$U<?z
z_HyuWuB<p89xiAx@g`&)Iom_<FI;E_I<m1+Iq6*E<ZI)K)8KnQP6=q{n-`@(<ELS@
z)$J5m&Q(7@?=YdOsYM&}V=h_5@!IDnzx}~SwSWc*H&-spRpnyXU586k)sUQ*$69}~
zYaAaRp8@DU!f^pvE;(;9S_jwz&T1P#5vQp}to{A{bvm*>If7FchWYmKAA}3P{Ni$d
zf3G_|+*!t}x_{aq`yKdnkdUDJN9V^kusS+B13|RwdMgu^pU;*!d{u+MXZAz;NVe3w
z=w{p}_eR4+MMWE%o6iBVZ32Kp0%8^D63oyN5D*j>Mk+Wb8HL*ch4MZ6Nu{f&Hw~WQ
zLT?Ts>>}<obVGDkE#JDt|9ziW{uhwkkeP2$%;MrkuCdx&6aeTm3JWh_GKz~kUE<YU
zRz`&2`WvC;AtyVO;!fLlV$83><7x>53$J#&SM`g>4hKdS%)^AAMMX%FHkh1;CIGZd
zEp&sSoc{qF6bS3IyEfU)VaDGX)LA7^`)gp}_MJNf(1Jsh6NcaR=c?1d0T6L{wnKU4
zO5no6g815KsU@D5mlrQUb_E56k6*h4IiWS>KI|PFY)hB&G~I8r=a@>k32I$3NX_t<
zPT(vUA$!Hpulw>eztz6j`Uq7%;;oT|U=)-snySkC-iO;wc;F<D#dgW1$KI>+5bEgY
zU`YDG2gny(2;cvA+fOt}`?@Z-ULxKVhn~`+1WtqI_V&fsG@qXR8!nxWa2UU!mNxwL
zd-~1m*Ux8WW`e{fhkzgj@5yWN6(0pa@KaP2N#rr<upuJalH{r?^H{C59sSG=>h<}o
zyqnv=#UoT#b8^xe%#r=+;EGf3q#qzIMWN4AQj)f$;Z!{DMBrx_+6_NB$1~$FD&M*=
znTc=oR@(a`pwGPA$6)3?FgNd6$1N`QLyo;3#?CjwrGA8Fo2PNkKu_;hrq+XjLOr&*
ze8-@Io3M!L=`J_rulkdQ8-Hnq+O&tB4o&rL678A4V?b>u!RxzhVjo7q?eX`8KL|nh
ztDMinb`FwjQjrokHgA6;|NdC^1l(Q<phO>pq_VZxsFTKu-^WjiA<C;jaGGJ`r`ME*
z+~@h$6o6Oy<TroCUN`SfD;#pe_Q{|{kUhZ?$fdr*dHGeF`j2M|7;77>z10FCY3L@`
zRTFESTAth6+k-g<ie6y_5@dtl#JZR0dwuWF7>s8C0(u1fKYzZTjtFV>&B+;wJJOUn
zc?rRXK}2kvi6V@VEcAheg+II#gP7P=fPXr6XGDD@?~gIT!{ur%1Mkx}z*X}#ay10a
z9bIjxxA6%HneX2dVYKeu3yqB>!}uz*_=CtPxwEZBMIMkgYzVuu5wy@7H*Q2spooa*
zN77AXX%`T52DDPBS;;bpG?J^H+V)(>0>|&V8}V{gna%4%Rdw}NC?BBk27zz;rK)Ng
zPCb>7?PX^d7x?VS4D$Fc%!Sz4=&j|F3J1rb4|HHYnL9WT8yg!BkBnqNU0&!fApf_r
z(gQ*r&=;wJ3D8SJ$HwjtETadOi^3A*nEk1MOTWqx5C=~Qs1%D0o%c5Xd{{v~&YqTl
zu>Q5J?GGpiAyH9EUS5*1v9aqQSz&wupmR)%OGv;uy#~fr0N7Ku+xAbM5P=}Nu(*h&
z#q;`ofsDKYoBNZ1e{kO86B7#+kA)`7%F2L1Miv>A{DB=R_vlfstxa}9!j=1oiOKZR
z(yMDy9)M~H_+P8Jk_`Zc1mzsXF{3|INV(7rI7ng77<if3X$q7s$cS751c=fpwBqgW
zyR%)sU}$92JU&hrfO}C+Ru%(>>MywW<m6<*0+|?P91xTT8tBgsUKAD;#lX4(xBA-J
z%E-q@itzFA4VT-**>+X~IRJBk*M3A0MHSW6)#|Sj%)hW?Rr6fDOd>BY?{%^>1W_Bf
zgxqU@*q{Cz=A)JNLbYyG<d2LbmXwm}0fAt^04(Kw;tEs9MDUo^hm*5vFi1;Z17f5E
z@*A`vUaRjU;nYGGEZf5U5Q%TbLi}p-fGTLHs2+5t2t6S;$I@ZDEb6vOySp+P3`Lol
zl@$*Z(e=$uf8gRk%$OHG)m_|~YQjzvefnQbhePvpZ9-~zbhM?r`x3-&P@j2{kw1Jr
zuBN_Tdhmlu7_|6Jx(iz`VY!aiTP~qh2Al+51oQ`#%=NyzLoLAX$iXD1rbdK*u9?rl
zc>upouCBKMr@$`NU$bEh(iTY^dyNZy4(+Hu`_p)7;;t9K{7nW?8MP5$3)Z*HTp>jp
z$S}l-(5eMT6p+l+xZC7y&<YNSqii1HAf$0ON?aW86e5L%AMHnl(OnLGRuMq?Q)qx7
zabtt1VZRefr<MMbPSTy%=XgU2eeiaJ@0;d+Bu!6GV>voHe*OA&osOyR)yLVcr79Up
zQge7c+P5Bcr$?B#1Wf}32zm9&A2cP;8E;(BvcRAC13g{z*#e1*iVCmO6e5eVbFffn
zBqVdMJZOJ2;DY4BHnkZ3Xr3(QazinKIuPoF1<*<-C+j*@37QrlnSgF@17HSasi)G>
zjLWo^3VaE#;I{GyD0@&N76bG&`lZv2?c!ai4iA4MHnp^11D^r2NdbJ^fB*jN6RT&x
z0K|p>4}vmI{}JT0FJHb)!RY|CCpVA`MWkYGVzc;)yksaM+<89VMMrd%9|sj^>d#I+
z)5KhG0k`~>c#ocFW(Ed~h{IT8zIHa+0fB<N)Qh(2kA$jXV>JEo>|_mD=*zo=?%T!Q
z>)^N<Tb}L8J+?qB{6RT>0P8mV<tcLzG2JV$3fAc;`ut%{8Pz#xKRZrJhttbkgO;xZ
zTci5<N{5?raWf=DB0vWK#mF3-8f8`086a|jphY?oyi6Yh(iMr~W{@^opl}<uk31U8
zQKf{<qN3N!045VqPYTd!>#if=V}bvEz7{|;Y#@?Ffm|AKVr(Z4@PJmvhZgiuDjA>J
zLTRZMK?>DXZSBVO=`=y^ZJXCn8f$k)?7+j~(l0R`=kk^|jjWUgftM64+isb&wCs?V
zr~!@I_@6#~;sst--(oIEtGa4x+94$(#?n7q4}=|o`iOc1Bw<ya?D0uSGmx(N21yrl
zTU(>G8B}-#o718ro*muIk2eTHK^uZdL@9FSbA0$!kr}$ge~+Aq?_G2jVf8v)hfY_T
zSis5A9<!jJZpGqx132&>t=h@ZEe)#Sj`Rl*mG^<^i-o`6=<P%L#Oyrx*QZe%0=oVy
z(51Q#&5eZr*L5l7iLP2LM6QCT7k_p<FH>kV+nEV&3`BUW*PqHsR)HjDQhV?>HEnle
z^6ZonbmQUA_B44Xyxf$ktHoOS`V<;zqJIBHZeF#ufB)_a)BwaQ(Uot2&)|`MLqInd
zTM4$Q!fEKdrn<Vc2jZBVrsgFm@PFS&foANr+0F_!YEWR{`O6mqNA!ImCT0QXAX=FP
zdYS@IF;I{q;Yb!O<(bvGlY(t?O;S=4ZKohLK>CUP@bIv^ZtXUTieJ9O0UgU?b*yS1
znApH(@)E-Tu;ViNAisBYDdI(T6Y_DB5YS>qz|@^IFJ}YeH3u>X&VxRBdRKtNMV&i$
zE+05eNQlE^y*ea5sJCLVWno3GGXWrGa(@0i_X?+e{%SXvBan~#R9TsKeC)-9{^ABS
z0lhPTCAo0?C?|$~xqpd|i$mlw3xMO(K%$iNf17CDe-m^vh=Qe6oXteZ`~fa*SnFQ8
zq>}``o>AeQ4lDk#YF9#+zdyCdQM*`E`Tz%TQ!8?IWQIXQKTr}-!jM&6O%3hdpmS1Y
zO`4GpI^~BCJbA2+uM@+r@96^@$j`?I5k0K}p}%^S0Q0f3vc$DUJkdBE4mBn{MeeHb
z6DrIU6j6*Xlvykf4-Z4o?g0k`LvJ3i^qKIMvwZO2*OVPL74C!fI=w-cwc%1`(8{~-
z9f8V+jhX%Xx3F#{MI?mvW>3lMIAK9S9E@6uAOSGBCa4i4G&EtLYgqh161i{IYAQnr
z5P~kKgIh0eE=Z!fN=tYPKQ<ZyJQc&k!%jvG&@RtooYtpqRaBY?$RVFrQVIwm#)#?2
zNe6RQ)N7v?E%jCRBVk2m1NR1!?(;Y<$-l^iV+!3_Bkxx-MflpCGfJ!>R$VM6;!!-m
z-Pn&B_-DxOs3QwS7&|j_K-}(4vwAeeqo$_D2BA}M{ujYZdskATiCKV<$BskM|FMYQ
zEKPzuZe?$;4n|zzr12LgMl`Gn#Vs@h&7{IPR#H#^#nKAwVDLmer@eQk7Z#MEr^lzH
z7_C`STY?jS{_-L6@L}3+OS#&U*@+7W`}?wHW^c=WjCz8y1d=2a<i^QKdT7Dl!2bMI
zWN;HI6`tGbn4esxySW7X<Cc>XFNOj1Ulh*g=Qo0Sxc)cQ(E_N2``O7Jie!^tzh)E{
zr-oQ6#ut(%D2?TRO(gK>XCsJdrGUTEDJKlT^El<@<!uCkjAmn~paBZI#h<UC{*u3c
z;J*zCd3kStmL9}uIPR^p?QvC0nBf}AseErzh2gNeWTMDFj_=@%qk1sVIcq?KlheGK
z5N@F|^*MM*EAgBN4Ozj@s6#*>EC8?%p`N#uBiu|e$U>mmojaYLALQ8h*e_q;GAhUS
z@$um*6^OX8OW!O+lfDmzNL87x?o^nj<7ckhVu51wM?*KMebOzi9gP(5cPE-MC<
z#z+M3j0LU+T#{}KA!VMLpAY`|Qw@qP=J@nzZU0|c_eY0u@n?%nwYFK|e__{NU8*8P
z5)_!4KxfV<Dhh+(oYN<sHw6ur-y%^o&<4Gc6yhvyqTxZ7?QbmaDo>xItZ#8gPcl$E
zh?U}jqN!uz`&F=H5;>!Tmjyzr2ek9UO;nArF;?Q6Rdgnv2ZX#_AW=FE7mx7;s5Aqo
zT2iUsgM@VOJ#r5lWcb>xE(uUx^K8XfTLjOmt`cDyM@Fc@9C!&<G3XWk(3XvOS*eT*
z0Yrz>h!Fos<ZhXNi$sFzlis&N{CY63E|%&FI6QD&Q)S)K@~|%;M*u;Yi}fwGG0-u3
z${uUpuX%nEbsRoD{DDdl=7!FX5nFKNqE&K4A+`kg9<0qvUVsN|N3F9fAPR{BK2zkx
z6*ZmFqYItsfH=&?4_>*1no$7LQD}npZovDHit>k&T`%9K42fgVFXT|wJvB89_^l=M
z;=(uCREp!EO9k$XEvo_z7%4|=l$Dhm!QEZ|{rGLu#Kbkgeb<ZlOf^B0O9r11sMFJh
zUyR55Gf`ijEt270r2gt=tw#7Q)|2!7{Dc++FZMw{bpmYi69jDttFyl^F6IMkt}E=Z
zP1dVd7yeDu(Ev9MPEDnOGaL-z3ub`bu>E}%n5HEZkpDoJr5wDOA-owr0YMY&{q?yd
zGn9TT43*HsQ3|-(a2(r5y&-2{ViE!L#=Ob@JW6o@mEyvhAFm;2@&K*T|K;Y^UkR|0
z2FZnsstIfmui#95D6}ZmLc=_;5ZRh<rQu`zz}G$8TSKDI9Ol!fPrZWMtDyPPN_kuc
z^6{~vqV=PB3u=CI-Ty)YtBLv&ic!}(6GixIAIKBr-U06|9NkR8^gfO7*Z_7Azg7!E
z^Fz>DA*+KjHb{!)Zb)2cpL2%#ISs`O&}ege9ILjzJ}xA^aBy&t2SCLDWY0;c1!2LC
z&AtJJf=b{K5x{(GK+X+kr!+A!G3e_6sA}oyorfB|y*5b?=|)*l%cfRWiOI>y>*Y(x
zNf|*_KsU+3);uXWc>n<DOwg48LIHDVhQuqcZ9g02Q$yFBuKs1`suHkejZsFX78eP?
z?DPU@6irJ(0YYE9HQzmx>R5jZ00gG7x0eir*98be!<*oOn?C&ecT-zi(AGnLKs4Vw
zI^=9^Il%9K3cCPQ5e7(A-MMpzjh)@kkV_GQ`@dkL*6s`%Q}I4Pnc1)|A+tMtd|F>$
zUkd`20u?_7?7?GHR#QU(1<E!d>u=eeE>*#yv$VL_G&o2Bx53ND=Ldvy350z62Ub=%
zvM;9fI>E)!gA4z#<6>jOK6~}5b!v<sRVdDj7cT%&w836N*G%>x$jZ_(z+t?4Iw#4r
z8R`nS#pn~DLa3+&hu*$2?+?1Zm(SKX{$`EMi0*$x+5*hD3WuBQ?Aj6U?CK#c9~~R(
z54t`%C}@fhr-YvQKSK~C9FPUDFyNqqiGYVG2cZL64Jt`WgEFW2TsX0I0~cL8P`S{6
z@bl--VS@z%|20NBo6Fh$4g8yHb;J2-2?pjR1B0PF4YgC)(=BGmN#N|P6+kB8T5|~r
z*~~__Yu9A8qEoUSQcYWs2h);!NV%!U#Ki@GsU__2YPO4xfm{7xD@91wwZbF7k_yWL
z%4UjIw+XqV`#J{lP$Z!W7iC2G$?~f>1ck1Edx1DQ5ix^8DL~B#<rGd}gan{b1R3l-
zSAU==b&D_Dw{*74`ej-gZE|X=$gynTIZ+SY13JvCGlMv_^?8F|{KYh?tKo_;f^lm`
zado~MJL=j%y}B{Y{;I?dP8;tz5OK0esimBwUfsR1^-WI?OXiGdcCLU`G>DWTaSQ=z
zIwCqL2~S6-FNofKA=;Va@0|;Q{7nLCgz(Cu`46O^4ZN!sEtq)q%6Hf><3mLKobz1h
zKuVQ+5L!X!4OI*0yYc`Ql|!<_9FI=w@K|VNZw(tc{axNjrI_*GpFnvT3&DT*@Zqn%
zKD%K**Um~A4X#H&m|i0kVb|Rh*Fk~<ah_V*S;n0t|Gn4l$%XSyOP8HaJo?UA{kMvw
zwW!8pZRo6dM>~=BSMS}(iRJ1~dR(~S5q>|_c5pg)&oy5N=JOxQp9Libpt@|Tuy1JS
z=JMX8-;bn(IXpEr-?qyPmQ)v;VkMZ}BUDybMzJ{!29bJhK5en{mX0ND3sj29P}g_H
z&X+veioorzk||KS#2{mKk(4id9UcO7OH)YeVPut*@@y{F9TskenHym}mrJmGpYGYm
zep>ZQ`L53AB=clS3ud>-c7_=VoCt%H?m%UDx@srTotfQ*nB7W~>+WC8tsg$4P+TAW
z{su9p29EgOKjB{|j^;Q~5o1?#r~h{$n)XEx@9OxOA$sx(M9g0QsgeG9pKV^xeDf>N
zNxeqWVvPamzHO;W#6<mWUbXPJHGbf^2RagJoN$}p45jvR`%d$2c+PEg><!|nuBvf1
zqjwLZI#?OGvt9aQyht&QXN~~8=d7tor{bzw(SOThOIKAk+f_~mSML8!ah0gg>Ae$G
z(L9g4t?H^tHHMtEkdY~fR621Cc2ARzvGd-Bwhx~gC))AhGut3Gu3&0<4f-i5B_$pP
z@I4-6pJA7wK~M<en;&e>Nh)>lHnU_QH_Wa!bH^yZL=o42A*E))xXK9+!kXtRg|4Zf
z=_MvoKzKC-AL<Mm{skcD{wmh|WyBk;hd6n#<={wFI27Gk0P771(aX=Dg<%TmhtA9Y
z)=R*joP@*|pt01R8^F_P$Xu(z<NOPnC@C$Zw`vhU9)+K<U3);_hH{?5#}U-Sh2Hk%
z%a?Jg?wihgTa;2UD@A%aY(7pNgh!>{Z4wPld3tW{Ja=&(<ujOQg2s?L9dUZd8U2;x
z8GkXh>gxOSng`tXKE3PesR2n9@ILj#RGL_0sikd5xcSSd>u!;y9;*=|9^2N4sJOUv
zg>xJ9EDa5fi*{<6n9VIL7Z)ML@4U@$0RLN0@=yg}K~Ib6k|BRIy}UfS-sbEaU-6KO
ztDlOo1s|wpct#!E2&nXWC&oNUA`S#r8j+hIp$in_pudC&vjx~Ng$aBBVxgHkx6pX3
z%N~GRjl9<PNB7@^_nyhXjo)=1JN}`ecUoxL+qDyqgoO27o)Iyt!NX^E2L%iz`=^J?
zH*5Jl1fyfHB)#x2Q)17}b(nQ@8~6+j;%;xp%g|7<A;JPWA42#=9*_R3cz^~k00={x
z@1l$FNgG;Hcz6c}8jFo(y(FX45TddAi!7{`cPyAr&+&Tf*gn`<3aaEJ!&(_(e`}|t
z)Y!f_xmM><TIGK6IYh^wOw?Wl{(c+!B4lI(0I~iT8{gb4bd{?Yf~Pv*GC$N3+_*6V
zR6zC#m(u~imhL-R^U+~8@{iffis4j#KlJn#2J)huSsWZb(7r~_be2to#_!%Eh>vGO
zczE<b@2qD!Lp}j}tf9oK5A?nL#XLmm`15Dk$@-2G^A?x@JhM+FMMug0%L|m-kORfX
zg5f7r+lmmGXoU8UMzB%H2%V}8D9c8d#9AXX?6r@nS2z!Tls>?jQLhcXw$S9phBQOn
zi#uovkbNfTbW-iCJD!a<HND~JzcNAht7lp|J@^t}5dHenuWg`SowKp={lLR(h0p8E
z!fWpDOvxJL28?64+LaT$&v2mH;Z%I*Ag`7Tj;xM>z+~+Ua@0rlIX!qR>9tP+kDea}
zhxp8!U+QaD!-Ee(JmFdNWH;7(H##=3liTjCc6=3ce&_t|iS>84__JykrGqwdy5d@Y
zF;={V-{i@H1j+p8ou2=dXx5(t)U^*Z;iLD_b6PQH9FQU1w>m{o@Ci+JG4Pv#zW$%}
z=@wME0W;|mG+(v1(sU4_y4XOa5f|Nvl~L!3>}}HF_EGJvk+)kZ?CvUHVv0C>UR_bR
zbH__bAGvV8r$9%6kdo})J$8Df$$ABIbA_B7FQuU@6G$$?o!o(F5+q7gfxdyqdmG?T
zgo6NZS;(lmu8zINXo2F14zYmP*iTi(3z}NKncTS^V@FaOJMz~lWOc|{{iEyKkG0uC
zi0PQ9Wzv<I7(abvI6QXAYL=^Kw&Mdm0ewA09d2XtVdtSkFs!Uf#V5e3VY-6}$O{-;
zH$a|-39xq{ITTgUOGC5B*YuL?iDy@^;EPG6>G^E0@bICRQftKav#cm-CVpN9em+K{
zs@VK|BxEgLP`{JS#K0hFduPW60(@{I16<yd4ai$LdHMFv&f-DqdgYe4nxOpVa<VZP
zm4_&+hUF_=%361tW2+SU{zFGY=qe4l>p$gkZwYPk+>1tW!24p`t!5V%3^@y=J+?zC
zldmChJv}|z>gR2k636OlLn7(tKb#~wbR*ZYi0MLp{tOX*Vl8#&<jy5}vd-7p7m})#
z{y$-Y<9^Vxdm?DOlvtECHG)B}(*{A>q{=yP?UyY(8(Rh%&IW@BivhBs8o9w=zQ#((
zewDr|IF_xXAn*vvvx3V!htfx)o*Y#L??aNsMbeXFfyo|xMGp`0cEpMwG{e3x<gbOY
zUw}Yd@n+bb7}!JF*h7|V1(n(N-li<qNnIDSXp7GdSm9Ngx6k+B(X&nN=W1N%xH&Ju
zWD1i7T99HtI6RDnnK|IHv5@_Ph{G10Q8t)F+VDF-qcBYfj0M8CD(dQU66NCXjGj8<
z`60V^2@kEDcHi?CT%wYFwss9Z!dL+(CX=xzgtTLetXSdhU9$g0c3EM5&(zfP^0jLk
zAZ>WT+#Na$@{KJ<SvRV*u<+H|FI}U@J?C?uM$&30+EEEAztE0NxH3sfFq+KQ*dEu|
znc6{EvAMY)86tpYuR*{EhKisb(Z8`Oa)HNPUr}rL%9Zm7az^2*mk-nzwlU!PQgFvH
zjq&U{kjw~!88?8>!Snd{+Rbw7dCYK4y{{yTk`6bRy?7zLWg+E(dE9+2FtBldL-yX)
zNzwv67!L6F_kUn-9|5sO8c6yAm6QP}eeSgya0x`BUVc^=<mc(NHPpO@pa~Nj8$S4i
zK`=`M7g!3&HhyC}4e&tiaGhr(X9-)F&B=D$CmwMf1gfF}PINRzEw#-}P+i~9Fa+k<
zGN37=of=Z9sQzMQWyJu-D5x6I<;_FN;LSi*>nZ4_sCm<dw6!G**-=2miWePGsu`>u
zwf{8RB}q@pYe$h$a_RVxvnYPpaJ-uR%;#v2bCN+w>;IZ|IDfvWr-u|AAYd*pbvYgi
z^`9|Pgh6i({9l%wQB;%)Joj6`WWcfTLoW^j;779i>DvgYlee4@2l9%IGpy{d(CJHC
z!{p7oM=tZCYaB2#daD73a&Cj(lcQIp|MW5`p7g%#i&!D<h&AP<>o8q{hP&anh1bJD
zd5TX>9i9D5b98(~aAQ`Gz9cne=LBeS4&aU$m>)Qgi`xr<S<&S<X#6(MoNuyg4{E+R
z^X{r<<3u>K;RAYnv9Oi7C}8>v6xtNv%2*k|mYpfyjwB?Iiiwe)$62Rq7mzAiReAE{
z{m7y9!WP)Xx4|aK1qO^pe?1$}L@acj6!6x)w|kTXp$fxP_~Y!O7wIX7Vp1O444B!a
zB^8(}h5$kSpbx<E<oI{bkc1DLbc!?YO;q1yXTM+lo(cK|zp!v_kWVxhCud(R<Vwn)
zW`D|0X&h7<(p4)*h0;uIU|*~!LfZ#+|CZlLpM1Dq6$_d!+uGZ(4WMbj`y!!orS=bn
zIcXn!!c|@US?0Q;vb|c<*{?UGc#=2peb!VvlZ(FAOKcK}iN4Qc5Ybx!Gu$vQfljr8
z%8E|by=`c#*GM07R8hGA)U*c_-btf|GTQ+T7~$1B3aRFSaTd_R6(JgZ1Je)g22{NG
zftT!-fXE9*$7>oRXfUHJw@*gOwRytAL#DV<znKX(#Ngl{m6V6z|6<7qkJaPH%v@YV
zATrK#fByT~WnKd&1VJVIG`NDcOHe`4Zr)`H@}MjfK8?515#ZsWVF6UC0ks7{Wdwvt
zZUOGfSN|N$p;H-xKewukjaOK=`2Q>5pI5Qnm9ozQXZF2O1;m$e5GW;JU&@1MT<Ue;
zOm$Y)y#yM-#)Rh@>REw#Z3w9cxK<b=I~zOd@A!s?PpGrH*{@cl#=*gXrd6H{f5ZYi
z7^Sn1ekS|Dz((W5MEyMr_ZYb0KOi3ZXJ<3RbY2Y5=e7DHVkwXQiQG-F<_tes5g>3s
z7a<J<Np4CiJ_{U71{md&K1bXz0}u)utQm5*;Q9I0)Ofz}PKU@4CVj-R;!#o_{I2lG
zTsjG6x1dVtf4E=ZQI9&1oSdAsC%dCNNPYi)yFdAZKK^6;3ry64$}64vdV1P`vDp|H
zWQ9uHk9O@LOM$4TC&JevFo9VBBkF-mr~m#vLCpe4SPj_H!cSo{>hH+B*tCLq3!iCk
z+l0hiF7Kzi?+-_P<iHt6OEbv7V>z4d7DjFD?V+@^v>_q+pFYLHFa|VFbyuXbTun=h
z<hG(>FH9Gq8Z_wLi%WWQRmHx(7(_+wxH3Hs(mVGI4Yj5%Qwt?09HCqgU=pND%k&}h
zf|UVT;`}o|kzWuBfa(3ake668>4m&Nu0u(KhTzkzkyI0yf5ggw*o{#o$TK2hRR|ox
z|Aw&bPP$;F2{7Cy)$t+FXMf^j0)v8L<uWrMmS1!_gMdF#QJ4=x-e`l_iG7@Nt11|j
zFWlYpPZiPVv52Ft`nr|#85maRpsB(;FCqZqkV4O?Zn#k;bDH)tD2NP`0$zRV*RLeL
z;ti)>@4t`FoDkEA$78g9lhgcw2ml#o3A{(vs5(B_G9Zh13j*^X3v2Dqlt=K4K0moS
zUSiqvA8qj*_wr7-a&h2icXvsbcC5HYJ~6#?8b-3>cMBX40?fs0lF1<<;7==oB~o~W
zasHZd%USJFzlKt<^|HQ3O1<YEPlZii5GFb=FYiHRlqY``IVlvMkiil&O;nwPX#S9g
zN&k*GC{&D?*=c0@7Q{W6L3F|0k?`Doa&BGMzt(LHlIdn<lWo$HT_BwJ!iY+~$4sG}
z-X)1lk6HZaH=S;mn=KuPCB5=@AFYko;9+25PtSAIqI7dEEjIS<@-Eyy1_L_)3ln!k
zy%_{|RC`0kS};q8LH;FN(<#bg%BZD9K7Q0fMltYJ5#ccH`Z6#uv$mENxM)ytFdk~#
zOJY&-);LX{g9+H_sVQIZ+2l1f*8;!wOE&ypgMyO55UE*`oapSlh=JMid&|2=pnzaN
zx$e#Q)|dca+kgUOh#dyN{1j@$H8#qk&i$=h$Suti>Ho{_xj2-Gz5X@Kgbs#=Fc|QH
zbv;KxWG^aP0Vf9o!?Bc@DVei_OSE)!Pxsf&(U7Tg742Ja3DH|hX(-OgIUGF=E-mGQ
zQu-9KU@+oC0M)7W8$Ch<(xn>A4?{=9B7HiDSgNH8BNtI#y|ICnp=OOqr+M7D1(W!M
zfaCx<F|x3{8(afL@d0=^uGeH-mvli2`vY{o!XYX$G6<;lgx6N5qRchd?>~PkC@Y5)
zQr(l2I|s!dO+&z9N*#6P{g%-bBoX|QUgLim=iG^0gHx|`p3lMUd(UM%?YwRx|Nktf
zzb}*Cy!rncP9}B}-qX#2g4Vwn2HNwWzZUT57rcMZ0#tr!sALjdgO9vfyDO>Q%uy*T
zGoW2(r=&!;TWv$Y_)$c_!s2{X$oIxZUyK&S2Udnl1CXDco$xIWU%x`^k-F;37ZQx#
zK>$3j6vz>x-^Buf6?lI&SJFY?aW*hXwGGB8l#5E+^!g7z5BaacY*LYMD-RbJjLxwd
zRXIN_JCVA!_kQ`3tp@c_Fl6_XS+b-&cb(9|4-*aNPW9FE=R1nn+}89x`&c#7kc?Xa
zxae#LjJp4VF;R5xQV@`kxbrN2``jkw=a{rKH9`t5txmBI_sW8xy@JgZ0#3CGNZjS%
zYa78NxcA{=p6;n3*?C#mxw&Bb_JMYMr=)iC)sp($i@kz@mjoSK)RK_?CF8#TB^vy_
zd>AZ7XY}z9V%q$n$%caTo#SG&hIB~&fA8yy1E)X<a1B5=1#maAVJ7A-tQR2cUf^@+
zzzPMIp)3@zk}F$K7TUYJqtN+#$Y~405gte#Dmy$uc2_013N25t_uU9@M@RZ*Fpv=L
z`d_j8O8qfuI6A*6Hosl{54Ady72584a<wM3DR$W6a)d||PieNlZ&sFdNJ3V~xnA=h
zkd3+N*{brpMeOnG?ydEF<3|(cz1*I6{glX=!-H>w(F9*5o0M7c7r=(16Sg`ycyV_N
zSRfpHQkV?Uidv_LLP<mClAzgBiIc87M`>VU=es%9XY)$4vnxD89x0>>Z_p8tCP+zZ
zFvVzr0HU0!^$YImfVfx@zF>l5>*S_X<T?~_JzIlymsJ=!a@z8ryZx(`+wbMPGRI<B
z_cK<WH!^0$<0>q%B0l3^7@L==auT_*Y3cl)&g@6xU><<dAmhXYwhJs~3uGgRz_~qH
zRZdW8bwiRBS>YGpv)_h~=7sntrV*f@)7hyvTY`%Fix+wY-k|WY9L`SEi2N!b=XK**
zpCmLM@O$=_&Fo0e!(9JaZ0s`$9hoB{UV#~j6iOJGqEg8rLmd+sWq{p{MoLdM&@b@$
zBcY(p37QeT5+ts4AQFik$BJn3xX@<acgSeQ(xcupz<#ZkJK5hz08uzf9X>zVv&=K6
zUZ0APC{})<>qe#iYsz0Wvl*8LBl2DCm`Ld;-oWlu$%%!)L5FS_G>v%)!5+5aQ)^Iq
z*O#J7V7P%N>4#WR(a(zB*;Bj|5IpoiB2cKXB{lid=*IGA7ip!e7U|$L*oUvZNH5uP
zfG>i9sc`>shStj7ggfv(LGX1rGc)q=8887_n1jGtykok;u6%1~VI-BF%ZlSlkHi|u
zK!H5Ww7LoYxlCQiDf00t^JEOwR{(~}1-h}7m2R(lj<;$8?6P{bt06`K;R#h4l@bTY
zjoi3_+$ab6qQ2$9=4OcaGltD=mcx1a`fSo9`_X5fxb8**ofVJcUIVmJPv-SYvTDog
z6{!Jvvgl{3$zt^jX-Ijd!W0z9|NF!Nj7%Nh8Ehv_-L|i#X0g-f+!e8SE0MIOENbB4
zMQheaGO(iejr799lve;9;A@aVaEX4cARlJ)7ZS9;|6ul5lVU1}x8AX>&n|d={~G<b
z%HA1#8bND@1;U?jsgE8#Du;M1TxojFUDA_fvue`CFHaTj$A7(2_C<7@G?18BVS3t2
zsig?40(3SSM!d)|-gS_KSpy~<c$tz?2bgdK49EjsgGv_-7=#52F$(~XqM%kjI3(>&
z=+Y4D1;QB#>eIQus0?6C+E<k1z{-*Bc5>)?m+dkL3T$8G<xS^ceEXdi8_a_ij0Ica
z2SSyrsuHuG-kgysnl}|%TvTXks@C>eo5(WxYjB<VVoxNUf`bEmdw_!7(2|3tY2p7V
zVc=>ZFirj)xC_JR%Eh?@XClZIz&oRH1Fv9xM!4?^+kq@FEnT2~Ry?$gcbN|vsp#wE
zyF{bN5*?<JcDE?h=OiKDMrV9EUGjBE@X{-btN~k_xsC+%OQ-;OWWbby(Z>f^10`k+
z9f6+;IT#uJAV^amZ3H>Rz(6EWPnA)K2ZSMW=bO0}cVlB6Vt5r<()w&&zqM7^WYe@?
zqRL2hp;$yB4Lmj#DlIF`<;TgsZOnvzFq@a)X!3;b#zDWp1itbKzN04oPD&8lQ{$^(
zaeCVjSY^^F8D52Z09$7pCfxq(0VnKI*GU$5>Hfa+ZtAslMedAGnkCIH4-FM>pktTJ
zMUoXM3&VU_84Tka?)v%)eYvITCUe_EW=IS`Z2*S9AW3L?Gz{O0ahD?j@?Uz7gj7sT
z)0_UVn{-;j$S1f=iWMn&mjP7V*n84(gJc1pZu*_Kwz3M-;Xi@#5qIDGhpHxG0h|WU
z9M8!i`dYHj_IZ1b)|Q{opHrx>uO82R!uqHxUzY=2HGtRDd=@WH8R9qDOq8VItHuiT
ziYNfY=_o7X!*?U;b0mV+bm8K~913nD5{&IgIXioT%1IM_o8o%=cyDhksyzNTb@^DE
z!cSIdk&SgF?3Ix^jLG(dUsd4YL8{kk$^?3bORKB@B4?e*r<+{;CKBLXjklV+xR|7^
zjF57h{#0DN@(iqeG=~lIQQFql)(ImXdKNy%L2=FL<JMNTwuMLl(5Rxjk`@g6v!*rA
z;R~?lGI3!A)H<x%PaX<Vj}sf@#O7<#GZ7MwcQ>^K<v-ROr#|dNQbU=Ts|tqi-OW<%
z55BZ4Oa=KV?IL{z=PDjz1A{CA@(`bcKxM1t(m)Fc9q7V4!z5}>!$nCtgLL+U-hwy1
z%zIywM@Yp9Vm^lD7~i{R4)q01xQoy!vp^|_L?EuCIN`mI!BR{lSjaw2t#`KeeWLsI
zME7g)zqTTar?ozG|4&WV0hVLm_U|+(X&?=yDG5nR(a@H(Bt=R~Noa|tuF%#%QfX02
zi!!28Q7IbQNtC7%8ib<t{r;Z!`;O!Nj`w+9>b|e*KhE>FP6^4@9;;_N`bM)AAAhLb
z>)EW8yel9mXk3L>@Pz4@%&h{WX(bUM#kH@SmdeOXPJUGG*3}QP@>g-4Sa-|xYsyF;
z?ePiv$ER~=ojTdPzHad^XV)g`3?i$Cdm|Vt#&)zdLt0Mm)1CaghN%NsP1V@LkRnp+
z(Y5^3hQe|m#lLe2jJphsj5eTR2JS6dF*4<_Gzm1ae+><#nLqBkbD~A??S9LKh5$AJ
z537vOfa|OKQYjBBH#b!;1);dF*QN$vm1w3U=KA^>u-$3_<EeVjJivtI-8<Ti8v*H}
zYclRwPbDjQ;TFqFlxth>>O#4^XpBnjJQbLho=yZVLn9-D5Lpna`tCep=&ZrR6CK3B
zz(4`o@Vvv1m>?ah!uhZbHDKe%Lz)*x4B1>rmfKn_bMar>l(FD>1R#j{uIQm7Z*N>{
zycAMzh5&S^NX_FZG}p&nLMkAjRHzCmdYEoPlW*5;JQJ@$$EehYD{3I_>g~1tM~;}y
zWJ6+9ffJOD24pwn312j;>^Y=Fcp=y6fos3|X3LV*sEvYw)b-kIvp$~XUbpgwkvg-B
z*OWl`3~1BD<N1Lgk5t8D=x7j!MdM21lX=K@a^ho|V^4`3t~APZVa}Tbu&iWGyttYL
zvcq7yTWgCIcOLwbjXJ$$g8(#L=sMQouOSA517Q+1sKE%MAS5o%gbQ+5ar5xbZx<y+
z*<$s^`68K6qh9*^JlC$>Nsed_(Wj>|?%{5YB$YWamBJ21@J1pqR1_PgxIjX3hUgm9
zTNJs{6=!U6TBvVtUk#D#rQ0I8oT1B*ICT-CMS!Q5fKA<a$B2{w@J76}lA)tvS7*Ti
z*x1?{4?h7tOUM}r!h*vavvBY(0}V0rv(0du=ePw_IWOM4iJT~imxu1iP_VCk<M{jI
ztaepQ6kagP?lF|-e>Wx^KOxIEH#gN39NLftv9}d@M#wpFg<ldX5F^U%CMOYQHp16(
zC8g1GbwlFgQ_No82S0B_j1IQH?`WUuJyW~taIR)Y+2HWzESzFQK7#f|1I;t1pddRE
zV9Kj(C82$_Y-p45eoa-a+^oi8Jj0fuy$iVyCp#a*)Jg+b*uisEfoMiyYkdjD@&oTF
z2Q)NLLfpW&V1YcC`_Wym@Z$K(vqepw5^~0xnlG16jWr<w@6eM|hvl@z{J<6j;mHvd
z`q9!*P*w009oUQOz;<16+}j&AGel`k!HQJ+!#`5{65cG4MtAVUN-LlLyBcCDEr^)l
zyN*C!S7|j%Rv6!rk)taP^%s1B;!&B1vC;VfLUJnn8`LYBZNdT$ipUJ^nHLUhG+uxr
z+6G_XbyL$TC=HwQt#)Ijv;O<yhYsp2AZ%?Z04!=fLWyi#T-X7)5O*F72jq`i<KN)N
zl~)bLn%}DdaC{l($Qn>Ls2o=TVwqvPyp02|Vq0|gVfT?}R8{3r!OA>cFD^a+qu%7t
zpF<lZ`-`YW6_n`FxPmtydBXb0;f?KCn}~?Mk?x(`oOC}wSL|)0PBr$Zu3Smmz1x!c
zq9hb1-edfJ`rjc%PKP1i>&Fl?4u-=OTSeFZ=acQV@~mVJdH1?3TxR=)Ga~_#H@?Wp
z$=z$IEqmP82`vi;=wAG~$EUikO%(7%6`bH@J6y%bB4NaR76!y_U3tE}xjWWm*VZaa
zNwMJ>Q(6uV;*f{^)4A!ezQS(MI9G6KrEdG4^W_?xPg{qMR-c}b+g2R6X6_4=?P$?L
z0RP>1cM=X4PlN>;G-i+LQVR$5P)v#&Wt%W9)8U6=A`sY;AKJ1REa$@&`Fz@wZ8E@K
zI~KD>@!=~)L`0<B1~lUnsDysJiT`=MI`k$*J#TdLYOJ5WDeb-$av(tc#M0m~_MR(F
z%F;HhEEw7X?HShV-G$~`?i?(Y*mR7j&T&Gz0#756Di9SD8||Kpi~1vejo?GM%hxGG
zxX~x4^`fpSKF0S%#a3vxpoe<!vT@mNs`!iw(L4k-y<Owj$(A*sy*5(Y&SR;*5?O;a
z5jKp^<GIOGRIaf*RVl7lb4;P~>V&J4lHJ<kR3&d?t83FR@PDpoF*vE6#ITqsH9V9O
zZqiU~p;Q4;l#R7gttECOTK385=@9IOfx$s|DUZI;9_KgK@H}^yvY6=|=Bwbzv?rlu
z-v?ib!m&uyebC<iVGHAws5maLZG-0u54T4{QmnRu#S>kRmZC#|WUHEJLesRCV_7rk
zO+FvB{dn7amwTLAce<9{?G{JPgl;g7+W=Y+uLn-yV`0e~xhvQ=R>KnBCR3Sanrr2o
zN#$^$jIl*CL<G<%kRfn}HjtCKg56HRaGd|`nd7xRs?NPke&;XvPUp1iL1PRT`ij-7
zpXx66hncMct{23!L9xjtGLu*3?LIv{l6PXqh_jHEmZ}B+m1|3~OjJ#8+^_NNDN$wo
za`p21VnAb}z@dpfZ<uOIB?>s*>uq~kpEIPMeW|o%kd9sAmg<)F0mapBho7yPX`%vN
zMW5ZPq<F;BxAVTeU}uS(>!@*YBA15yZlDB+gGIfvB?;Ip_#<LBzQeGapW}?XOQ1IH
zqV4f8krtuW07-38oVNDf$fvRyo>eg{hIg5F7IRaG=7BPR+_yyNhftq_myIT264#y`
zGOL%4l|8v;GLzoiL=vP9i`=I6p_KUeP=n-MBs762-Jdx3gk!%?39J!y$H``#ZSq*u
zGxM7HQ>I;Kea>^~6!RnRL;VfQzx@LYx{5f^;g7&OiwH#95KrB;kC&X888j0N-Ex%%
zZ3!2D&+y2|atBR_EFy8c5NAw0j36*a189Hac_@ODh&M8As%XbY;(9m4(bF$AEvmhE
zaS5cdCff?+pPirG0ij?du-3Sb6tZMwT|V-|{DYeP)ol|=t)~kqoGRG6#TIOZ=5Hv=
zr4VaK*M54@kWjKa)hz7KIU2yMKwwUE7k%Y$F2hkS1bzTGqU0H?6BYRgxVwdegE+IM
zSq!#p)ANRRC^0_XHIb-c`dOI0y;pc1Idbb@E{lBJ)vFhfwZkJ+V0J(<HZ(Glkro&d
z0%yW%nkyKvpWnKuRq*mHhQOlI56N9fS()<P2hwth<k7IpMeR{Cwc+Zu35`n^m3FoO
z{XXR0m+Tq9;{0WQwu#r@pBwoFis9k>KA=VJ$LA%NLrE$NIcp*DQA1iAIXu_z$uuW@
zD&E|V;?!z*Vkh7u?RIE-t6z1UYIWKqw|;7<7GV{~MW><RN07tspbt1+5VS}n!9vJ1
z;hM9}iNt3@B#EP8o|(oQU3Qa$U*D>ZG%g-lv-i$317kG=h|*qky`GOTRH;B!ax`9j
z^ZZjykqZgD9~x>*1ow-1K8V6yA%Ep~oYoHKyxffk)S?A<Gf^+}-J?T8UqGFA)cdp!
z&I(wtz>TI>%sRxI5$mE;*5NIl8a%sxQuQMA^qV>~wY5pGS;bac74IJmIPSKZ+2q5&
zH~=b)X)M*|sMcha?zX1llegf}*SJ!Dc0DuF^L*-W=#+Vp2sDY!n;B@hb%kJ!2ymwU
zRv|rT(FCwP^mU@35+J8%+U?+if(Ob<pTVQ7D8K*>lj0_oS}0btvyPAB^c(J1D&C_>
z<F_2LZIk)*??Q&w0*3rO2>2jJ8OkhhsRLMaxAHdou~=6GGH2DfT*6HWs?6xFfk4r-
zXH`8tlmJ{6dr+YCLkB7-m@4DKmj4KpE0JvHmp=E14o3bZWURrsPk!EeS#2Gh&02AM
zUE|g)ceogMs`EsK&HxnT;N}j&wMPWbD0HfbeK9d{ISpd`Cn}>p-rl#tRl0h3U|9>`
z*SpqyJi422>z4IGaa6sHy*;Dq(>dv9k2R$O&>W;8kN`~<HbMJh0z88dKw6ah@d8HJ
zV{U$PVY;ECgA3@P7SQaLhxYqmt>sivQL)!1A<M+mD0Edvb#aTkcucLTrjW>EN3C=G
zHUF<AXjwvJvuL~#k0#{Z<#Rrm>c2wHn!EOMeIdL0&b37}!xxX+amh$yvIbfbU|~#=
z+VlXVA3L@zf$F0IJa}{7;q7Q!OuP*&M6O=%3NrsFkZpc%4SD*IA){hM(n%=D>{H!E
zoXAQoc=(W%u&D1s@n-B=(zHMqbVN69*}S=#!tDy-jx7SUkyDV<`qer<=Nd;@_7lnH
z1pX%>g~?+gTtO&r|7Z47L%Iom1wg$h@F3|p3jXE;dz=z*D$eR~BKlhds6lHH8Wp9D
zPec1`Wd{0*Q1DJpb!Q&4PVL4)3G6gP{kE#9DTNqZ>HJ$Qj-(GDTJ7iazt^HKKvE-g
z02Hi&>D1!F-xI6shynCpg@y!x3-n8`+`P#G-7xv=L$9xX{yP`8pS6k}u3KUpglFZp
zzZ`=oKUoaeTEyK3o|7apk>C!<poDK;^*+eSnMS3h6+m8q+jc1$8%Ol->gwtot5?7D
zzAY2%%|(75HO~c8QHtY8Ph=#DK|I0-@DULE8~F5>#59Iu8cR8OV$w@XllUe9hXtc<
zR!wiQhKBc_(e=KiVtJLF6VLurwzwp|Whtj*n@FDODWSNq->YW6586051cS8i#x6o0
z>fuPkZvo@B_hBa>VZ7K$NO|HS3V6Ts=UeU?3L*~lyMV}nOQ9mg^nK1d#d1d$%rygX
z@T4K^k61ri%6vVM?n68oXhcqh^=>+ZWEC+n2GmiNI4X#X7+oYq9&eHRCzjs+ZF0!@
z@Ev8AOCl=2`B>r|CF=zj5OQ|5l2j~0wL${o7a}f+g-qNgu0KETNApQcVYtPjp#`q;
zbd2Ccpv6{!;>&K0=hr^z7c&6G9)#3AYgp4rgrj!48mxSvN&3NmRd_n4y~Q_5+D)wk
z6;_2s)pBR=s&Bo|Q&GW+5)ZbiHDgrYvR)gL*T|%9DLiSGcjPG>i6(7KQN9d6GeUh@
z;9-cDv1cK)EppUtn{TPjbUt5LU)SrR{SI9Jm3MXQL~wyT4r1iDwX*}<H1TLBbv6pF
zRbo!~Cn7__JrQZ+h5Ch5^Kjzzx6-pA#MC?L&Oo@4*v%mje9o-vlz7m|YMYp>=971B
z@YIp3#0!MfllV<MzP??7jffhwMhCY}XU@Upw4a|WtMBksONZ?2wJC)d>I1>Ki54tT
zo^lB;E}pd8As)#mR|%YgY%A>f0X%qQH}KHmqDIyhO%<U}O;=5lP$!b_O#H0mqJb->
zOP19TTp8&R5wc)C*;jFD&yEB3o5|^{?Bm+%!5Pk=>~4tc9JI|O{045(BJ;m~PxxKD
zkv&Ec?!Wi^@wuguAG*5w0nWdKTY{6HzX7h7*SOiwsB9UWSbQ$p&8qiVbQ%mY4|qS6
zK1Yh9!gBL)U;0h=cz*6K-j1%QZoU-GW>8}CL|}mLS##JgR!x&I_(+B)aXIAS4!=n}
z=-i32(Z<H+9R;P$N%kc+@$w=U+8kwphT?ugZla`~8cT^@+#j~V3{*ZOMjl;RPR4W8
zInYa`hcDD|ty!~@=G0xZZO_$%jd&2XK-)|txq!U4;LmQ{wd=k0m|>E_Rk)(F3ksU8
zPs_+%?pq#0SCXB8&dLVvCch9{S&9fVVPL@8D0=_*RO=mPvV!pA5nV^91audD5)Z>q
z;(rKzEtLK%ZtuQ*8XzPeAbky?y{@fG?Yfa}L;v~OGaj@0WgU2UZeQ9r;^ih!7-??W
zxig{dauaY~9^xUM9g}bTYaNl;7u`kgF*R;$u|<?JdA_)y=+t>X5{VveZ};FmlQ3D}
znicS)^F`KJZ&HQqAe^+Hux%kQ5`Gy~EZ6U_sC@Z)_tndneF%&skt;CZkO(O3#tP&@
z7fzSeVpsz)ibBfU2Qd;cwA1=<UUjC9I!OTkAS;3b_r`WS>i+U=+7Z2nZ~r*ap2fKZ
z61xfMh#Kf=vT(B}73LdfXAyD>R>@YULQTz|SSDBW#B(Q;6gguP&NS85-o_Q(iUiP~
z(<tr_;Q7>oMT6}XQUv&*arEAupxU$cUr$L{sF{^qwaN@f{4!dyf*W<Q&7WuRWv0E;
zEnKwixn|Liep5K{2pS_u%{}n{f;ij{LLFkUy6fS~kVZQ*e;P{RnSsk&iR))>ws(<<
zhVLwlFcDxq!KS&@2ph=mP5M<b8~76HD1gs81_p?z)D8p<cHvj|8FzQ6!QLF6g=mD4
zqIvoF+$iqhsNVwB!*beIz1toW9|dTxh~lS_s>}$8rytY|;3er%X#C9ob9S0Jvas-M
z&@_`Yt5DidV^@;zioB7dPn|6ZSO4fYx=T8mVT#f=uMZh698E4QTa#m+Ja|AyBidB7
zO5`vFoYxHD+o_LK>7U4H<}lq1b(D25ryz@-Qy9hKwxdS#OBby--cVoOFTj2%|ERLT
z*Erb|tqe3GdAan|a+f+)KGvGO9G4{Rg_+fPM%d8_$&M7+0NhgC5Kq+128tn0f-f?-
zZKG^W=y&B&t95wonOF;F+7whDpNy+Wq~;_2MR{vkKi(IHX<$(pVYaAak{(lVX+uzi
zq)J2RJ<_o4P(JXWF5;w>fG{flP}kJ70s4jh+l!vSHBeBHDHlJZ6Cq|hJK7oq!71Lo
z={@u72?={OHL-4qT~c6|hu7fwPv22)FHuF7!zX{rz#dAVaL9<dw8b-xHMhBQ4>ERG
z3{|qY95sIx0_h2oT*4oXjp1smK`}|L24Z0ZCGyV`S#tKIUh*z37`}tG)4lB!@m~mN
zKu3e1s0ydTehZ5Tn2GGr;6f8D0(0mp;-Ms-QUAH3g+o*gv`8f94xC{{MFlY#og44|
zQ#1Q<2v7$})1mdWHv=z@BzD^24?ndb%&)`&!|44Wj#Fj-H=}u^84#fX%A?B3$bJ8h
zd7nPF^M>7e(*c#h!>PV#Es5nHcJ5!le0lmwT+~PZ+0*zd)Dn>6LyG*`ftuUhxj!U5
zyRv5-F~^-&Bpe4gz+eUyrum~L|BIBo(Dis(z7-@-8AZ=Sbg4HT&t-+5HWz32zGyIx
zJwZNg`NH3mto#ZbxaaP@T?m@FuwP=^c^t?j4jG}^hy;iT4YM6;9ChN2`cgb1LU9+`
zyc_!*{2OM<H00zV%~lgi0C+WNB*csOF(H3aLqgXKTL|YitV{@;a5|A7WuPW1Tj*&x
zmY_BbL1hoUM`dk^N9G&>Uh%R>j&t5hu><yLI^(w88S*rGX})wE>5-%chpP|#I&}+`
z`QH^fdcXccmWe<FRZ`J`LhuWz;-SN@gBJ?aT{>io6GnBXm1+hpOY1#(KK(!*r?8C7
zN*b|wBIR?EfBzfYGzeQE*}T&qec7Kc{LS@;pVA~q;+pU14fh9;AFE$Npn?)#8_jGp
z$_O-yl%6y<Ph<yd+0ww%$<pXOKP}*ov<HJ|#bZuu_RooTy??)?x#PP-(I5mcI0kJ{
zK*uY3O5vqeb?w!VvxH#DywE9#1Omh28NdBF8*+=ZwO13II5!$SASzwR2U3*0FCvU!
z01kiocWP_pdeEixQIMS?o0pqFw2*T_htoh>{FPen=tjSEKCs&ZqzXsq3gki1<0^Xq
z$wSL&@m1ltH;JWzK*Zz+10zM|`CiI>brsJ|bQEB05#cK@I?abRsTBVe1-L6%Ou}v8
z^8=uKiSXjCyHu8Y3DOn;qO=s)Xz(j}{mUIgc?7jGce#dLL=t|Oi6jx%{qKjRB_$<I
zA`)y6h1xPUwi@|fKYBRXh_D7(CVn9s6p#!^bn(sbl0t^bO0Wl5BS}vJ@h%by%QcQU
z(eP?j<>f@F8!ZC|l&YV9cezthbng5vm}8Keu7d~~d_3X=gp%nN7BdO_zbiKoWRTl2
zfVV^t5eQzQZ`~3?)zgQI3<_9H^ch3$`%Px8BLW?-J&TNtbR6y$91tG1Q%QND;OS?6
z<OmQwUQ+F%ra&EbWb<L_Ap;6pSnx$jrUkl}NdfT@2-{teu0m)Da3>_O6-n}8haWlg
zqbz%eXaMv(uZR(oL#x(`23sW&k%b|_p67`To9;r5$eOX1b)K$ES2Vw{9%D(+&8Ymf
zwOqAg;<=sCxgG0vdoRJn0j!jz&?Qk?IH<I!v@7QKv_Viny&W7Hx{QEXXw@Cx=J68C
z28n<_(Uu8|s5DK}m}@0R=WAu|Vw%uQr47>o69xnsw>VWXS-;(^Wbu%JJKv%L^(&&v
z$UG1*0LV7EjV_HCRuRV?0HPB5$drlCR^g$p)6}>n3Fm}CoaD$vMrr}yLnViJ&}e)g
zB4y69dalyjFVnDGGp4&*c4UFUP$+2ixh%T+ZU`?@eE<!f8)wjtyvV8wBX&#qdEbWA
z-GUIDBNkT;6*3SQBJ3pYi-7x2&^Vv&8yuuZo*L5GUQ|_4^vG!)^h>yfdz;iwK=vUd
zB5R9`iMVg|JZi1HGl43Mvsa_dCNB!aT?in!+jt+Ra)LOOTEsj7)q@X+o{+Bl#GLYi
zX~&g{*JfI{&tBZw8xY6s`dy@NY`!(5<G$ruh)jqnnMT7}ok0gc0rQ6kwe7C!nKJ~O
zdjKVhOzcLC`&a=94b~Q()jIek|L)e;*Z(yuD7G?eVL~!i*Gld2&k6L_MEC_W4w08)
zrV2D_Xw}p+jOm&WzRoktUx%2lW*Dy_G+Ti_PXjk4DLuNLUa=_>xrx%@HKi=F#IdFF
zmcE0?DmGVX$Ha^341P1hlY`L!<X*I91SE8wOuYsRHTp?wFn_520xYCLaKmF4Jb+(l
zeh}i=U!tR}`K>RL09TO5g{wm{mZht!OGw1A(0u-SGX6$v?EUtKh#1<uc{2%?{hy1r
znyO~RW6e5nmkrwqvk=$`EYr~ud(XipBHYnGacAP0>1lbd7{f`1-yFjh57l(U0xr=P
z_{zgHds5H`C(0Z+F2agn)BJpz+u?*ZllY_U*SVZH5r%#LaCr>t0j)IMaodD56PHvR
zFN*txRNQkZ_x@xFiiRRS0D?4ttF1!wuzP;Lu6Rs8;$4C+UE-uFZFts&i5Gu(|6W91
zp4<GM=_L1~zqbW@g(r!e2>RX{lxTIU`FvF#?8`95O1Ht4(;*;I)y|W)cO37n*{Kk~
z_L^@$if?{{p__D{dg7PN@A;3}0pk1TzlZbFDLl%?X`K!Gh1C5(rf4sKs-fd!f+N41
z{d7kT7mV8MxQ5@qN{Ejq7Z#7KqtU^1JDy=VO~kP(`@B3OCKsT~=r^~m`Oe<rRi%33
zRfv3RhE||+-Kx8$xujdr(5jPe*c2*RyiaT;F1|^Da5vu>4(fBy+IK~qGoQF$v_0xJ
zEG#H^f#$q_VBklqeGnKG!lc9CbMkd4Z?HErmO6TsLr0FRRUyG_i)VgsZoZ!^RgtQ=
zG(zA=zc@-TY$V~hdMk<3o8sypIJSsrXw^$MoD3B&rZ2hnh{=#x^BVvDdhx-9{`7|>
z+)|W@lU*m_#{d@~!I5->z$bPu-5OI%Oz>%F*n*>yY~tfM4!Y+bxEs}X*WS60sr+S}
zgnA3~N2@Qdq1*02L-`pPLciEkXT=G=lq6u0+#)<hQE6!o@PLfBHnI18`xb%r@m=%P
z?W|KQn~T>TQW|)`LtVCE;oLH{oMj2yI)>q<&M}pHv_kVp-t7+m8r<iL^8|E&O!OD>
zn}Uq988~ga9@<MIKwiWr*ygCZ?ALBOOLuOc?*$%rs+bL~gpZzYtJ0W0s0JdI=(&;S
zNrL6#q+F74FB6C#cma=yaj;F@UX$9ME~<+cSe0MMck@x5knKr-*e;!cGM7|AI0&H)
zpMO8}W~E}4$!^XSTj_9Y-^pN}wP+3Ta{O7V3}^)fZ2&N9+?R)cIPlP|v~+K3XxMzf
z(#Wy(DjRnZk{>SLKg!5z<aHVsyw^ywC*dJUAO^hQ^V&7iw$%C(SPWoZzR=GX8A|i{
zDShb7#iScu59mL)Gaq`A*Q)O*%SSFEz>QUi2%N;uxQ()!iS?QdkH*T5q-RjY(0a6a
zn^?Xj`E8-^JVS+sxL%A=`;aFa%QExxfb|<b%bgcSobJY%VFC*wz;PG!69EKBGgf{|
zqDnegJZETpU;u`gV5n-!HM}#t<CpH9`7Xc&lVb!w^UG>FL|{g&v&?ot>P;lHyJ+YJ
zasFU8$+_NQt#VZ}skM}fOUJ2vu{>)R{q`9)x*~T*9vKFs+zT-q&aP*aB!OH!2$iw0
zwB$r$A}-LiI}Czt6e!W0oum-3Yu@Q;7ju&Z0j;u*w-g_jTnjr-9vPGd#37V~*-h)u
z1#|MYtoKB*ap%sRNr7P3;$k{Q&n0`JmqbNz`TGFYL6&>YbJr4JG$2D<Iq=;qR(3pQ
zrIB!V)DwOo!l&ryVtvffYmo(t04a(fe<-iEyIQkWnQ*s6b}K6>F<4zxjX=<NAg&>d
zBw@tW2!1Q#b(mDKykj#Bb__WFcj#TpOF_5wjt-nU>CCvLEL3s8gFUl}mt1M@-sx0`
z#uI<c_TGs9ANSZqkn*H%B=;A%4#A~t`~k@n;aTi>f?d+|#{Y35M+dzUI0A+E__=p%
zl3XHg&_1ScuhM4`$P!6i!0xI<4O!sSk;83ZV9TRjs_CiLfi4#zv=<XSo7C6ru%_3x
zX7EhDlR;M%OxKxD|LokTX^qK04Y<c=BLM%TNQDT#>;k<#PcY?@wzinQvNY>vpcw7N
z5wFXMY*I(tBj0Y{e&MrY-}=I~dOPUlNn1qT+}gGE*bF=y_ltW1dqon_Wn8&B>MQkZ
zSUii?mVe)J#A9uY`4j9qnqVr6JPLAHEYKBPBqi>vXu;!L0=zY$VKT+?PVLL&T|N#e
zQV}xOt}qySsXKp8xGzXCeDpOmo7tqqU+^H?brn<%WX(Wq8-QFcDE+@gSK9H^D1<pT
z3eW((MSwfF6e^V6myl|)SA8X4+U@?ez~%}b(Mui<+lFWujbK(1AwjA)tW918)nJ=I
zo$I30cm<;J@}1I*FDX`&rsPLM<HM$-FUt`%1{41}JWEtU_F>63P8`T6MhGu0z$VjS
zZk~?5L3!~KRkU|J7KrsEnSTkqjYq{#0Y5z>Kd4Ds?~PF1(D=|01LOo14bN6B`|v=3
ze%qA^qe-;9B;|$F9OO1YvCa%hBgWTNOY6z4`jn(N`TTMY_*7zs0XTa9XjwRnwCOfg
zUj)?l31$1(Zr#j~<8`Ws5Qu&=t~eElfF4yrp1B@tRp^bt(9j8?x_$RdMafVhDGP#K
z03#1a-ivB(hq_dp2Dqigg?VxaAslc{S2tj^?d&s3pIu;yr1u3cK(7QK0$Udd8nv=^
zuh5?PZLC$x_eXSJ4i9ezO+Y446&3NI-9s&g5@-=(q7}4Y#UmX??(PBa3+YDpIlO=>
zfm77uWG6LWXdFgTrU1nY6A>I~tri!ip9`NqY{gpDE`gR1*P9NtZ(rMnGhjF(wWlfv
zQC*e)ju09eqPdi1<SG@P_w5>*Z`!SDEp#Ojf$?2WP6s6}BKJlbPXGlUJFentqye+?
zC{;<@iiZoQ;Nr!*{hrAyh62YXneVaJ180x&GaIC-g+=G+2yCXp2XwY}n_}Mqqm4#y
zK|!0;kHP@^oAp4j#5F}4fq&5M`iwQ|Zrz$6F5hrad3xxWbv27>I^AP)%QP!c(tnQM
z@^tBW|99{e9d{gyTyBeF0pw5w7luRx3CMZfUmbXAvQW9uzW@k<Ft@+0vuMJwW-Y#J
z7w3#DjNUD|vyj*Ih&r2?Io!xtSsB3MU*|P9vkg<tmSq@+5r`76r1I~bO6dq|R1MD%
z&WE%A{3xRaN`RwMnNk%i8HBUGMYgoO81H$po!gI&rupJynd;&KSGgA-z-V%Kp-3YA
zE~-W@(c?Td@nMqqM0gebsKJX%VNa*P0q=9)ePLDeKmzBktImN74=-&E^=Ol&U%G@^
zv>aD~aU~i@A&Isp*j4lGOLQXtev(vi{V^Hr$?^ho?erECU-wScOA?y5H<c_*)bu>N
zu$((!QR<D~_q}{VTiHK%vg5lIj*TF$uQHH19upUU0*63ZMpZ$_ACMnl!()e=kc|eL
zBxYt*F6TDOovj==Xi<GYDM)b6LmozZm^_9zaz`>1`EnmBpNnNLMjuDUaG`A=lDR{W
zYaV&tD6Ll<w&TcVtMbx@YE9f|K%EbsWIgV-=52xY0ojh80w*r)*~{F<`(n#^y?rT!
zU@QTz*&1@Xr-TbDpYV+kX%N*3TiU-bYP+hf9*uxCM#Rnl>$%<GM9Q-o`V`Dg`Hm1w
zLM%a<hOXCZDT!yeV&}pRZz+Ha1x3ZGuC70kW69NrXDX0i91y|Um+0b={$nKesNgcz
z_E<jDf3q3sv(PLP7>i&_cst+GUO&>1V<oYjv}QnDDWW#<Uqo}>%QUq)72?*dM$8ZC
zD+nd083;0M5KEW<!)PH1evNA!F6y%8;N1bav=l4WjWM+-zliLfhyj3|?E$nzdbM~e
zF_E68)OUd+&{}2q>$TsR`in<|MdQ8>_arA@IMEisyFH+@-8T%J+p~%C%lMe!thjJ7
zZ*P##4#r^83ZkzBkRbxu3a#(!S3*r#BZ{fu$&=xo>cRatvSJi(tv!puEX?HL!cz+l
z=n}31WS_I5^PotNi!RCNx&L|0b>aq8jxf3q$f==UKuR!OtZooOGtkCB&n$v$JQCBh
zZ(kL@gBzL;H0(m#x39uUO7PiTrEPR4RgPTJxTfJXt8ID9`%I5VgvI^Y?Lw+?&9gtf
zy)6?KMMUzBmhmA#PYvAQgY(qvIsDqEe9te{d;)Ix4nbD1w%+P++C~zQ6J!`^1#W}d
zq)5P_hhw3V(64|-QbZkiqORy<+_}??yKN6TGf^5DID-J#G{_YXz@!U;@6xn4WDZ1)
zt`BuK(p{Gg3(2#~mwX=6XW-bWxa#~oCAVK!xA=X1V`C;v9RHJ1U(rtA=G7oat}dA6
zjB)8Nqsg6qfBm}ddS(B)nV?q=$B!Stq+l|?mSzM~#ola_Rv^1xp-DDHqB%qNJDf&e
zUlo-_&g-@MPf)GvKr!S2z)ccCMZdk^S5*hADfh;yIe=oKYtOXb_ZMyOMtS+^$fn4s
zC~@5FIcQ#!RnC|2rO8PFK*mecz|Sy&(B|QjCz;5c-*skKulNJjoDgPJVul$s$hr_2
z-2sPboZG>#-6-9mAxmoHi)6S9VdBqWfUquFW()%6Ub~jq9Pq%Xy|D9<sbl}_jc(@m
zZa}EI7dHdu2+p*&te<ki3%_F`S^MF`vxW(E?~g`S83hGb@Qz;h_NHP;BHTXFAeVZ|
zd^-cKZrA$-WnwgP2{4%q?n?3mi0h|UxXcta-+7D$1y!I0^cPL*?$NtP)=|nnnxHbR
z+oZYfv^bL3Kj4St_mh#l_=u|o1zje~F<ey*L-T-qk@3aPpYH}Zvw!d2u!>n*9x3J>
z#m7N9LB@vhyv6_*7*g9)r)R#~{T+%ieEsreAQ^%NHxaQP15WyH$nUD4|BN~maq2)c
z3Ei*R>C|_i8cEy&L|9~CG<3Dd!~>K*Q0fxAjX_O(ddi>g_5X_r0QtFldTxL;;5`y1
zdizak@hq}QMnkM_Hi&yT*s48sE}9}Az#;q&8aZMtz){z~B@<K&5zqJc@24CjB}f)u
zJkPK`LtHlkV)fjMlkoc`kRmbR%Br5*(TcOB7GDERaMQ}UePZI`#J-6_;d%mFzir0>
z`!XjVol~ij2{+ZGiQ0`w<1w`uc2>p{h34fNGDVOgko6?z@-YdsU;eO#cs3n<&7g|v
zZ3BhX0|abP@v*!QhG;`P@ALKEQ`U1faW<@bIchC4j0u}ZrcIJTBjr{7?YK0u0sWZZ
zz&LuBZ)P=?5~^+y-1h)s(h=4PDQci&Z}UDgU07N%Cg-1Z1(FJAHnaP^49Bs?#0`>k
zFdP#TE~#2ig$@ynKn1UkHI@i-IP(5F(mbZf09tIj6Rj^l?_iI0&U6CON+$&13QtaZ
zd)e#I<rB5%F(891e2Acj6|XyCRDq}y2sE-Uk;sKFy1gmN>CKxrztTPI%>I@DRXV*o
z)qv-Fzg+PmFn+u!>I9k(zu<~Qyf-o+{5+<NfR{AFP>WlzZ5o5=t8dtYVYZV3=7pOi
zBqYvQ-nOX<COf7C>eAlZ+q9#e4yAx(!O1BLlV`a)M<DhZ8J~<Ng;8IzO3$ys`56pd
z!)LrnaQ8f&`Boe7+_4SF_%McIk<e}HO4m;#IukitN$(!2ieNr2tK6w~M%SMS9WXYA
zjmQ`W+dIBVcOU;P&(U#WS)#s#c4M1%VBC;Dw3p=Jqt?3+J<d1+l=PUhvjK>Me#PtV
z8>OT&aVqd3R>gmM7z*~!U{($~lW}B^Pg0#ye-XjC>1SzN$M@_g|J*N#G|YsR#0Zzn
zudlODPj!i8mF$gB_<pS$jK3WQsmXrU!Glh3#|^zim6nA7%XVCapil)!&;W=?I0Um=
z$4(ZS1C{4`ldwI+=l8e0cqSRZ#&|X_VDVf9OL^<-8;C8an6{Elcpi6wIDFpsy&Ozj
z8cOUK2rsaPFHHl`B5$ZY=inwXFb};~D=;mEVu>8n++S_x*el>NB*er5p_Vu`Sss7$
z_WKxLbfO*E2SR|f9-DlE=Z9tF*sMw=YNy}cy*L+xd*JiErytY6AL~+uWf<^|AC0x3
z1$s>uJz98F)DDx+*W=?c%nxQ$rju<PDf#iA;cLMegzVa?!_+7Jam|wDHvn*%bfhyM
zeVdORiLl2OXc13f>;rlFG)Ew-2fuvPH$HFx_rsS)RxgK~O=K{rKkuONSnWLamDp8E
zN8!kp$()x+d)Jndk{TKtv&@bz#f6@U2ANmnrP&uCuH^D+Q>n{+{I(dWg$bDM_0Ky5
z>(ITaa4}Djo~9a(kw&Et6IgL4z_i(q??xaiosf&mgyi(Wbs5~*8}#ChG9Auis#W37
zrqCmW8>W#Lx*s$QrEqfbY#3Jb3Wi(8seC{uC9k_mlEE`c)8BW&N0foi4+y6y4fL@n
zwjL<tqxamF2^XjCJQKW5$;)Wi|DNeSEM*+*2S1fdN@6cVS&|J!p^MV#aGOvsopIDC
zJIcTbAyl{IC7FDOvG-WMrsKbVQ?PPF6||m)d_E15S%IJmcVmqq6qL{+<jn8>_GoFO
z(zBM({1x&|i>xjmOZTGQyo->5W@tXLv7_yW)Aq&8BL7n#wLiHvFcjVcxsCI7$T!E%
zfqW*0+<xuPJY7oMKh>R_J)dHlyB=O-%n-Xp(OI3YW*!@<UUr~x1Lo9h#aG0~%v`*M
zw*mImW%Aq#;f))^XeRaVU4^3=QWuXtN0cELI4#YRlL!jl(GD@#{Cr03d9XnqP~$*!
zo%$N*B{YHy`qoH}#zM*>F$d@2R||ud;BTFODmO1LF-ooGrnrw6beeOi|GD$uCjgRe
z;CZKfbhjgr@+@2$OfH=q*xSrB0cD@-k5oXMoPgen9D4ZOWVmI8Sgvssl^?g|h&Ip=
zA(0-X$8c#Qa*}+g`7TYezL!A1qK1kz!%FMdTqpE7N4q8-L9B6hg8FDF7}5-mi;N4b
zt*Ozb)}7MN?s!?6#NUwzB_8Z=IpfZBG$MR3Gz`JjyLJ9~7)F=UKnkQjQo+>^LCkV|
zY?3NZGB4q6QTW=#vG6OO65hHM6SpVn@%h06vjPgQ90b$+ixNbn4<S-5o>Tq;U;>El
z3#uE_Q69pd5;_<}QMSig3*xu}+QtZo#}6m-JB*G4#>&CVTkGk_YIvuk=Qb3D&^hEh
zJ3>R~)~nmb+0A&*NB>eJgNR13!Q<yikT@I_g!c!});~TTg;}m>(=R+y$)(5?UdNU+
zMjQ(N09SarUHqa@C(~X)A_Ex#8JKePZ)SFj<HvVz@#uiwliBS#0~KbM?%WY0{tmQI
zBy1J@%LDuRH87h@3h-5>I~k5+TVT9S*0?jF!Lj3Rqpd{TjA`Mt)w+yc?mc_<P>p|K
ziy{&GD5;1hmuMZ9v9s^NG!0VD<Bm?pEYC*^n?w<d16BdW5{-s(oJQj;`>_+6Ia&au
znFRkrO$4}SZ^A1K+7`gZIDfhq3JWb{b3#dbWcW9jm4h>9d5jUBLdH829kJUWRYQnP
zB-UNVcfltYl8^{}YWO-r+GO^z9`1El=nxFvOX=%^E3kKdj+w~hy<xBSV~2931WO3)
zkGp=IN4(jL%u7bkfU)9lPVCNrl!8n<La1y%_9=#?Dyz&*Y7h{D2w$LVX~r?`6qZ4?
z*ZT+KvQ@=gV$=vkWQcx_zB%%<rKP3tRh+>72B>a@MDh+jSVE>dz>|U^4QkaeqK$-J
zmK-i1_TBm`8PO(;#Lh3^W;c?I1#%U?pdS!Qj0u<yk%l<bZb~C11~e#1ya%33l1nMn
ziv^et$ttjGwYzuWqNhV(Ude_(W_gFm^~_}T<m_+*FG6juEvO%kUZ+f4Td0h6a9*Yt
zp6oPr`ZItk;paZ#^(&c)wH=-73O?U#9AejSgKan+*I8fpi{#T_3+$Yjg^ODjb^Cmi
zE7XKUnSuNFJ~@h<kQJJ|n@S<YH}qX3U%&@o1ljW>@f`1wh<|Q^b;Y=vCk?^52%k2>
zq5;Qr6>GI@abfoFCH13kCm=b>#)io+<v5b8RgS24M|7Hbr=$-EccgUZqw%19#!Kkv
zDZGXD2&3ew46Nnw38SA!&o(n1F|<^EcR^6^;H0ra9;m`3=VOxjW|4I%^FHTJ%f`W`
z0wLJ49~K=qT{TmT3ex}bXk6te8_rq1hY<FDzq^iOHs{6M+>>WLx9X4(3uHT^(ZS4^
zd@cYY=<{rm^n@bn1yTnvPB$$&mC8=Se8N1d=j(`u0q_yvB*;I--q}BkST^}R%J|SB
zoptN}eS4J>?j*?m!ku%z{BgR9%T@IMYggG80QjVK{qg&^)Peb{ms{89y{Y=JR}a5U
zG(pdy`C+lGiII^izETF*9Ta65Fbe&vqN%fAT#ZM1;*}`=+bI8n0t3-q#?7!rZj&ip
z?}?jjqI8N4gBs$xb{uUZs<Kn3Pw&i)KAN1eTjl<LH9dp3!0nRt8#WNzKM1^Y{K5ye
zc<n=d5S;;0h^+gNj#{ef%UC>!SoBNSr7%QP#ENhK*LYB*^B`FVa$jbO5Mqk5H68nT
z-YxLko2YE5QgKg_Gz<_ys0Z(xq@6JnMmZfQv0t~hUssv`d7xaQzdbioxaj>RAqFVr
ztlLajCs1K)YWn{ECY+$BjW#=*^Q4!kh)5RBHCbRkeBXWcA-simjPa$JrFFS4H7$iA
zQtrCYHN~srOLD`)m|--p$l9ZT*Fh$h@&Vx7|GcrY#ZlG+I{`DJTLcB=;lm9N4ee8W
z%yKz0GJ_O!Wo2dM?asjY3;FlFmd+QAA2PU;m;>c=8qRgTo@*r#F#kmaxDJ$rcT@e%
z)3v|lA1FC-mqPl~MMc@6gSzr(QMQ{}(WjjLVhiU;ZE`57Q`?agKvMjn(wP*9QiGy)
zKMaOMYv}*y6T-kb^N!aGEOUFNtoSjCGsbD_pILDNuK>XfW*3Y?OHUBuBBa9f+8or~
zzn_!bFGR_W<BYKFu9T~EuMP7JOtHh0C#N(yRW@zfXVrR+=pYOY^MA+Vej%gI;ptk5
z+5!M^6(ENYU_8W-hc0-m!9Fl-!ZWUOC#IVrME(k-IHc80%g(=TntK8p5M`Y{P;D|!
zwe#JdUqAjr1xPBSp+2fl6(K9_z5Y-z5r;d5>tYDb^B<ZS#%^osr#AVaWJ7tY>H3OO
zrs3<dUB#<LhjYkXkEKP-z&Al?VC_u_yxqsOdNnyziQrExu6`5+S2|1p<y9+7QMg~i
zy3K};!vmL8oXYT@6~^dS(2!C`|IF0sAPXnMcrOHD_NG%PZFuw8gR`X|6VHI+&JoVt
zp@9K)xW|zGt%<go`(@}1UJWiP!K=7_Nj-{F`9qd5n`CS_3Ia$su}*GdAcMQ|1lkF4
z>{Em@+P}SuUfE26snGnMU`G8XcOoRE0A-*m(+&Dy%OlLG@4oe*g@qnqxU{2XKIox@
zf#ctVZj1~b?s#$(1oQ#CTD;+CoXP66*Y)+6*Pvx)7gWi+!ShlIlSQEHC$cx(FqyiA
z`(t1FO%3Z`DpQk=;P`I*qtgiG`z4%Aws62AQ~RO`C8>o&f>2z<NI;ejj`m!?xkI=%
zv2ufPIKl8C0;LlY<Q<!D(Gf2Mp73qlrdM<sJ<pq80rwC@_!Z=Oy?b<=9YdD)f7;R3
z+Dhiy0G}Oz%9}*;qyHn5S#T9CX--z;!;})-?Sm8=4B{ojB}uGe5JQ^oWD4e7xqrj-
z@r=872Z3rRoEc^%3<}tJunw4M^y0&ZwGhp?y1A*LNQ9ta1vFs@4FmH}2m(MnB*`0u
zJ{h@$CD0x7v8v;b+3j_=nAVLp@&hZ|L}qrNOHB8sriT5VpV}|{50xVUNw{tb;5Eh6
z2cF05_e?JXQzCaIN|F#%eE|2D(q?eM68SokV~9_igtYr=I7HlN5t|-C3N9I|M8A}c
zB)7EOQllrz6Zm#e{HlQfMbU{h1uFszCcMD$LRBGo7K?+(KdSb#ySabft5>4Pym<T<
z<?8C%H)|heipl_G+XIk8eqbYwBvvO)6JH2!lL1f>O#jAOpuBZGcWxLkO@CsG__Q(Z
zP@-7+NG;B@?DiC58sK27;EdAIlGL-QQwyw_02xWxX<c1Pm)!oCi$>-c)_msub-cCp
z#F`Bo?y~HD%CKG`EQS`Kcg$A)+**Z_n}iUUoSM2660(#K&bG(noow4{=OOl?8D$Tx
z3zAr!X0$l5ShdKz`TosY;9d@7>X88v4sUJ-K~%Iupi;Uc%c)+6;l5<8@)AzzppCjl
z)RHZai!N?bh@Sc!(HP9u_8BLf&C5_uqQHX7LK@?l;8ly~3yJKu>Zh^YVp#u|N_@dk
z|DfCY0dFq4*;RxBw~!(y-o9_FRjbnfMOIvljXfBY5*K;OqUMv#)9hQ7)TE%8Xx9~C
zz7=5i$&6dzwWM07usUD8joQu{9idWe=FslX`byQyjq~bi<GE156`VKs@734Wf-;v#
zU5K3pHx=f74JvL9&Tfg=Rg8eNs>a6ql}(pb8HTnN^WVP6vb%5TkZ3PQKkvT*k)54r
zb$H=LsX-e?s97E!Z}+D9dJVD)=St}*1sa#h*0OvW<GTw22UqC-{Lk@?K7%k}5Ek;>
ziJ*ZZ|GqfZ<7?7>cH!0ckp@@Jvl0wy88TXRJ2q|2O4gma81GeU$#qY^y4v~K4f8yo
zhR4fqrex_UXAIkq6+I~y`+T@t&TNJJPLVHZMTwzV_te}TT<u@U+TiC;QK56Cq9C*X
z{P6dSesyMed8+qU!L2HO!?O#19s=W#v;Sh7=69dDAn5GVd*PDTfJ@P-HseG0OiA_+
zP-ct{03zp99vQ%q=H@Xk0ZjG8SmyqxC*_|Z<~C|Rc$zExK$7iNU*0;SEp;B7%+4id
z-npZ5;2Ou9>-Ue3Q!ALKEx(nF=5^DvmwvAaxbQ2lGfnfUNKdAE5xZqmcYR$Q#QkT=
z%12m_LwUS>>-dnt`_>!ob<Cm>1|F4y)cckwHCA_nmN!OPT8tR0V2$$&6<!EH6g!y&
z3(FfE18eX-RwNV)ViWMs^wGcIjN;JdsXuJ6Ot?{kdUQC(`G9bD%>6ifgrI*B82{+i
z$)srXY}HU~PtKMgk3^RKa<T5O&0z+4$JVPoPSVf6^TA}<^xd!a3HbBcp1OYrn^+wj
zP4-X4B_`Ixi@Ve3rvx@iOs1G{_x1Z#J<d<-mr}o6KXql|$+TV<3tZdw>tRIT?ATTO
z-k;Q-&h4HdRl-Y!IxCzCWiWgVjDE|p<B2T7Nrs*MQroAGkk9vXUuT>7vmX3o;uZ3b
z`kl2St_sJ7{a5V#U=n&mt+W4r^+P?bgQc%8^N$p1A1pS2`4DKx2DI(~Tai~KxqB|K
zd81{>vo(TQ?+YH>D0HSuQ#=wWmX?Q~cXdZ<i|Wcb_nuuEwa>12l#0dIZjl-Ndx95G
z{HZ53W6WoEcIy}Q{vgkz#eH-aB5X_f1jggo$6TK#JR|?|VX>KQN6ec3ap!uO*3*)K
zw1nc$+T<Jd<Y~Dj1o{f(WHa5QiuOIt<`~c&fAF?h)jaSS?{+Hpkd}PO36t30|Nh{X
zo)BLIqNUeJ+YPhNU10NQ&XOF{A%Dlg<I&~=`}eEiUPA3cQVPg_U@f?E{W`HE;?yUa
z{gA2(v;fR(Z0F9_Mv9Qf=v;k^@584yJwH?=gySmu={Ng{N&Hz)RF8B1JhdyPHn7^Z
z;?$03IqljX0Axds1G{%D!WK^Gu2i?!BVR9rJ0MTlI3F3ab#=O@c6>Nk;VjLAV?Xr+
z6Z!7K&T-C_Jt%}Q*<x^X)VnPo(-OsGWF9i-wi@8fN83i+Q!ouhLDMMv*YA*Up#P|N
zHysW}O6%;|U-}atPVGSVce6m`G^8aQNY%8l5yo<PNOqM$3ClEwa9+W2IMDAl*cA50
zVLJwMo*2k+MuM8%w+~Mzf~U#TYcVQe@eDw3-|PC@StcPA0{H2LPhny!4NFZwS%NAG
z1Z1D$^JODcp65E0b35v0zEgu?k&)|RF~#jVUNp#dw_P}}S+FN)76@nYNQ$pC4{K-7
zS`|O9vR5Cfr4|}{3nj*O;qC_s!tt!95gH!cwM<fH6Mui{xD0VJLM}x<EL$f0Mc=dE
zS(YyWU-)c@$Gfp}Xr8_*9~BOIe7DT>QwKtT?a{%Q9Hb7~JpJ$o?2k9Pr!EX4B+rvi
zU%si>DjOdd47m}m`q}rW=IZgyzAC?^F*V1e<Q>a!_;t|K6s!|FPH@kTmDgGvUpmUl
zPz!!YE0oyN$1KdJ=egap=x4<!Y9S}V%SgaK1`KZnUp?>kX#l(y+}~VB4^gUEF#&Y+
zV()@DBGc+!mqOX=aN5h%%xpV!UrdqdXN!Q<PGLgb$DddAFIBw24zI1RZ~LYk#IYsX
zxk_19_Bz02RwjDw^s6WcI(%6b&q=xOf8_eK>lFCAPd7r&eHc5JXS!*XxL(nx?X&93
z+%}bTDSXq$jJ`MHJ=8PiuWPk~To^}o8g9(EZnQ*oV}^;_iB6r}b2LY-O*rakOI+J6
zE-*M5H(gu54_d-<r+tBI-4ysC^n8d43st79w~Mf5eOe;B$#RK4o>#BM6O(R!eY{Km
zt;XSZcHgqPR?Hsv8Pwmft0ne+Acwkq38(rp<<Dvow|R;du2S`DYa1HwK%yp}9GR#m
ze%+WSQXB6o0lL2?r72cC{1rJzmSKC_7kwyjyK?E<@!flz?oTBxy(KWVtayOs;nJ3h
z_6w>@n1yIoekb-Fp}lWr+A0-vmvP_NJJY<?#yb1o#O_(wJEeSgr#{=xxfkpbtIKCv
zpAGv&yz;uXfteEOEAbgSrwA$XCp=oSv8~kUTf3(kwkys5Y-`OE3v`ry$~?XKp^iwH
z^0$84Qi;1sjUJg>&yOBB^D9$+VMej+28-ltu}8m`)^kR=CJjogW}lMQ;HlKn{vl>)
zTS@GZO|VJ^?7J}TssPpbF--Ts9I!A*ni1sx^(R$=_qgW~8-B&iwXJ)FF6hZHi(PT8
z<TXBX_V%Aso=5byZM;(uS2h<z{SaKx@O{6^Un(hZoY~}{DPoSR!E%O$hudOZVd%-*
z4*TW89ywh>&sMDc(A8-E_F;(f%wQ8H7+zqGux422{DXILXXAt9u~Q5F+e3PH>OJcp
zqe9p5zRBD1s3`H7rDY0-`p8ZFq9YF;UW@f>cy*{i`esY%i{+Xe7h2w=zH1P<T$uOC
za4u}6Zm3AOyw|0rUN1T_n1wy+KJ`0S*ZsIzf8mmuS5TaHf`Ny-{b7UsYK%v^>SS)X
zU5nA=tX5y6`+B*s?8rk`Gl?CK5{er#F0wehwV*uR?{2C)Va&!ac14xnEon$>dwhk+
z8n>lCu19dFuazRtK2$jP6915Q%7&;Saqqf}Vz)kPfv;Yd<R@QTivANXap%O|Wi4L%
z<?&@#BY)mzk2P@*)$pskyG7<5T?CK!1@6}(ms=tnR>t7R$kbU_-sGr!BiGeXc1_nd
zw0}8OE_CD2KxkD=qM$?1%G1Xc+Y+L7FMP9JI_;L4yKkj>QMNeq^ks{o&bgbcGI&E_
zFXq{@oh6HhWTM=Lu2l8Tteqd7_;M|f!};}SzyF2htPav^Hh(*|fn9j1QfEezyZAP&
z8=IwyJatB9abkh4+YMGsTg?QyR@RC}&wq;>rjmWKGH-6w6HYj0taE#VvZduUj><@d
zE!|w<%zD;>vkEWhO4@pOUD8kPIHkF=min_bWM^Le@x>sH#{S25?cY915xe%TW(V_c
zrXQ7;PoJuaNEp@I)3W(ySP+Nvi;evY%Dq!QZ+?jx?i$>}w;6~rs`iI4$(hBuAMyKc
zFS?fcb2I;NlaE=PXrSvVgTpue2s|I8EmdfAEB*e~XNGAO<I)d-MS}K(8JfM_P*ZuF
z-TU<Pjz@o%Q~j*RCo5icd|D=abLX3?d8Lt2ZTYta!H#K#{N}6Bh`^#<WLY#4W5`{?
zVW{+cWct3;o@Ff!C(1WJ3YKG7`P${ry|<Lc$uvL^_oZ&DU0fz|)6w)nT}Q?;;qHLp
z{_>dQ2>U#b)Lj(}h|cMC3*LTTN{jq`pC4K(e~w+LcvhPGsemStK|UZZtN1q#JK$e^
z<M8Z@zXtHGRe-!|k5xK6azh9N2EkV#6f+E{APr+xq0qR5-1_&0)X7)A6J;U)Zs6V@
zDL2-QxHZJ%nRtiA$qF1<c6+<7MsJP_7fWoedm+E+#*FrGu1(il98FLI=x)rovH5Ts
z?brL0`5nHSReR9v;NRt$<{}U6An@ayveeYne2_5*;W@ruZM$E0BNrcE4m#xPRsBZz
z2M_`jxSg-1cMJD)q<@h+xJW(XNV9M8I4qv($I9bq=({C^(HiuyM26GxDgd`keL#I9
z`8)v$0HvLR2k>qA8Xlh4a2}Fzy!&o!rh8%g!1}=<VgzCfxdd;@qjU2Mrqhq#USOMh
zX=uQUuOxUg*mq-w*WlRoU%zcAw9?l_;F6nJWY-QsUTv1!)<xH%bIkVmRWzLOOD4gm
zAmH!_iUl$uMPkdA>dwxmjlSmF?P|h1HrnCH(X6+kKbMot5+u2Qy*XR%b>q5z*Q_R$
zSv~0avLUs#gq~Tu45`o<*_89q&aL`k0|KA8ZUK~~^8en%C=JFe;Pmf1!P4AA9kTis
z0wvqg>$-eRxe=V(FR5LvV1Eili{q{{>>|qolzu$sUqi$7kJtwVuSp>YvrhGvCqsKQ
z(BD$j6u7midF!IKYZnBBbZDHHTK881SPUecCc<~sF)s><s#!k-?@Q7s8e{xkO*NCL
zl(#=#=gw1W)I$?n9*1RLWSRSZoN-gAuy(Eu9nlODi9d{V8xY(AZ3Vh?Ol?0|wxqnY
zGzp68W<;%{4%mbFK;UhO8VTOwHn^8)4vk5*V0w{`<bvb>4hRU0h8zIqh=H0Q%qYn1
zk}(kc$j3HgPA!jH^vq(?Hy+%K+7j5z{YKGbkIYIzKOG^pdYdj#FHOqR{HQa{%GPmk
zyZ{2k3prh~9j=#PNNyeh#i0>$03zUUw8)71mO|^N^s`wbUQfKiVLQOciVw4`znPgX
zT>!cR88WhuC~@ev@&J%S<JOJ!|D+(uF>PboGOMIVnOg&!m%gbwkFz$&OreT*SL^<m
zZL!AaLa)Lz2u@?Ug?T1w5h9QU!G4tcua9h?@Z&Tt#jepW#(^Afl7<CxViNQe-+$*6
zJh5XKvCHtp*g%6`n4h30K3+cSoZ>|dFauoJ#>0Ac!R>C@vHP*we6#Of@edc3tQHPj
z+pN@G_I>#5(UDau!Xf<nHFKcK>4=>g+pjPm#)Io1bx1VVb>p)OqaS;er(-?)aG+xs
zGi`fKPNwdF15f7eW!~K5z1pR;tI1z4o=@&07K#^C7EmXPQy_#tmiZ#nN)NdXAV90@
z>-VMCCEjJ93HZ~nA=zWyu+6{2k={VjVM5{6=sv6AZP8D?Kk&M=h}^kaW6d<B{bqe2
z$J46prN{D*D>qFkojhFck=Z%FVnDL%*pxzaTk3+?wa}0!v!%Ey8?sMuYOwB}q~AIv
zqcS7=L7DoPn11SEn9#-h9-sD=U$^E~Z@=8O%8v6g3XNJl@o1%m*Mi9y)G@&k!c#3#
z(j0W`+6n!W|GP?d3eB3=AF6S`u|3*b@AxkJ>vGi}N;33M*)G%gyl40ChuqZMzx!l`
z+}<Mk(z;x+YdS0U26AdM@#_D2>wG)jHsup_ICEQMoL89GHAjuLW^wBST@UE58K>Ft
zXfW|IaB5P2@uOufFMaYbcHK{0M>9(og;Kl@+i|PQ>c#{szOd}BUiTY^`^iHZo+VR^
zR(&s8+B1qQ#1mNUC1T=VN(Z|o4Qg`qe|Z?lv8#EgpUt}R<)%{gHHS4hD$Xdqs#5Bv
zY}FTf=Mvo_(l;n?%#1fN?s~y=H_3W^pkso?%JDeHm9Kr?p1HV6sZvzWL56*jCOEZi
z&F1@?_0OF9hD%zMM@{grYt$LPYO^~rjp<5XzWl&18f+KO{JX1qqe>tBlLAJ?bK){r
zH|p64(XTBFx{#YRu-a;p@#|qjsp7y@jk#2*uGm!`>7o)P;XnhqT-WD$fh!jpQvXCW
zDb(#>!hCGA9q%gzCYdXy3L}#zwn(f_ICf2QrPX(?S33<j)HGMUnNl)bCVXFN|FV`P
zvh}XdTQ?ef=d!x0x@6DJaCQ}?8aq6POT2o9J|(i>xWZ38{KLIy%kKNG{^*a|yP^Vr
zRy?O{lm>ZU(4Nk}(OA}>a`SwV)hIQWze9DpAVlyFr=ijxuIFj&=Ra$09UoLt$90q}
zM<q>e@D~Z;e^@IXqZjqo!Zh!3rX1`^8oYHJ!Oe*uc`RqI26Ln_p23$f-O<J0uXFmf
z!<7m~DOz9HhGbL+yH={w)EHV)em}mw!jp2}z{H9@t1@H#DvE0pE#tbe!Oo!7#+sar
ztFIWU9N)QbV+IZXJNplzvXl2!e)e;&t5|;CMR{|etDv*MxV&ve<><{#A^iA0p=s}*
z*Gr_w+wv>S1X*R>TCyth{r0_Y;=fmf1l-J39%B{>^(~I$Q2(qgKRzJ!M;Z-d-smo=
z?jR2JqM4nZ!hsx(!u?Wse`5ds{}A0AW^ye<JNR~yF8Qi#)vxHTUK;*yua<%4eRb=L
F{|CSBH?#l%
delta 41969
zcmZU*c|2Ba)IEL|BBGQcQ-#nVlu(2+XO<))QlXNNS&jztOiD5pGBr?S3MFHMS;;(;
zGS7tHI?wxlKfmeu<LQ&!=RW7U_O<s~Yp;EBzft+sQDs^w5(aveOSBW?WA}I(p3>1Z
zu$;`i<R_%0Eo!Q&(0EWsLF0L<-=Tdb1pi<`l3zW`b0r<~Pi1`kU$(OpUfLrbO_S`m
zo9p-Asnyl)x93<Qy-viu6Nw+{(K+ups5aSQRul6io><+PmX^jQ<1Xp3^jlyPw=`R!
z(|E;?S4NZNtQSp*rc@m|I=YV;Z;I);xVVbVb5A7gG4WWu5`FagjxNz?zDJjSbGiDA
zvjv5J9bn<&(s=V>ie8m(Yr~Hp{FI<#D=Vw|?@uM-TlEzcX+~SJ?(=KLs4T@EufLoX
zW#ZgJL(jzYqN++||Gs@XZmYe$myX{a-a<o5+gSHZUX>?IJ4KU-wVdj&p7|<VqJ8B`
za_jQtxHqdqU%I;Dv$M0ST3VvkQ&IgMY0l7ce|hSew#Uo&@5Ohrv+J6g?p~R$m6hhR
zDHMB=x0&ns!$*gVzc>Yo4|Eo}@kLZuRa7XQ|F%nLthKdO%&H+tAkOK}4<XmJmHFzX
zria-%IaZ|)HfFm_Uv(uWx=&^hOq;o-6Yw034>oR7F*17k?c2BE`<rDF+H!4jT_4Q}
zo;VSE`S8wPt*zk(2l<7Ch3PkMPMMmT;y)oPCH3@_ynLF|cqg^zhD~g8&z?Qg)7JiI
zJM~`6+~4huNs;C3NV9vTWdsX74cGD8zWf@|O~aNl{QTa&zPh@)f`o`jDEX+Zt<g!k
z`;t>qreju#v91!W!%qSWoW?aTWkvm4S=o2&*pqPsYt1WH_9`kUgg$=E>Zic-_U&7q
zG`-XN_U#k8Hs6q}#=^#?5-a7zk8PZqpHCScwoHHZ>g<`RIYo2-!pO+&Uf$kQT#4T^
zghL%29l!LB{r#(MKS+Gr<EL<{$aVJlMUO=<++>$d{OLoF8-ux|QY~uZ_`*~yJ3qKQ
zeEhi5>0r>~s3;ZPw1-$76`nBiE^ss28X7@{%lr24f1a5sGV=4yu64UvSy{uYFMs%V
zHMJVM!@<T@H~9VOVsQP<rXN2V>Jyc#8yh1Oc$m){8aC7ru~Ia~O}hjhf3;ek8@+tx
zisFkGFUXw{r`OYQamlxP$mAIl6Jyz&e)i#`M?dzjTr9Zlw?o)Oou8k-Da%}0Jo?;S
zC6@WtR>N==zHEzH?&8F3j{N-m{y%@z3=IvlZ~xZSJQV!?5n|8q?d6S#*Rc5bT^!{S
zCs$bfii?Yh-Tv1U6&3HN?dIU%2tS?tFgp5+8n;FQb9qsA_CreDr~9<cU3rgL>nHn0
zY|h<}J!bFy^3|*AnwntEc$x1`oH@rFMp($e2nu4v;s=jQ-a8}9)sSm*alIT?mFJ1N
z67I%b`TFjC`~0xOPo6$KAl$2jb#rhy+Eq_YDDwKdIXa53OV}o$nOb7&X*q5(o15{>
zV{OH0q$zFN;^JaX^>zmQF*Eg5IO6r|Lo16z@$Qw(50-Qa@HdB%Ce>r7SqmzK|2mGg
zM&cWc_RU^o*J-!uD%PL~9y+AC5@fH}K3~V}pKOqm`!)HMQEs>b&oeIu*6<65hNK84
zi+;OfG#fUMKb)AjRopl6e6=(`A|k?W<0S)ws0z)JE0-?|vaa_nzr-$ngU!p!>!^$B
zRih;4@|CH9uhiX7B$~uHl*@}Y&pr$JZH{k<>>}1;`L=Gn^Dk@g^3AmFZqtT_hUZkY
zOcPwmXQe%+L&L(D@qG;gUmq_9zaYf5mO0;C$S-xu)=$ATo6^oudm^%%x+Im$@9+gB
z%@3czMZH>D3>xa|$=jadN^ZvIbyGBBhfBQ}+?9LOuU>s&x3ZR)IP>=H;o?Nw{Ra*N
z;J6+;eq4Ov!!8;3DqQ;+bNPwGhr=nirVsEcmv@Bxv)u4q+pTbf(7$j&or#I5zppP*
z=PPHkB4WQK%e=a}TGc#-kNwQqvx>A#oVy8vKzv=cvQkC--|Q$TCvR6<a2U7yh1KrP
zojcK@=H5yd&RK2@l6=|M^Vw&Mp8|W%v4`3co88)<D+Xwrm;_t5y+7f$HYa|XCv3O0
zH0R#EdksD~6QW(VhJP!9IOv#|lyYx&h`3B$X4$dh()?J5ik6m-I{*EMh;3cWMzID5
zD<*nAGw;~p{qp5O_u1xid5)v(ABu{41_!-EL$?fw##z&My*Z>YiM)7bs3tbwDkd>(
zS!#x<NJTik$k^ey(-<d`#!_F1>_Tv9;mY!SgbLsDX=12$noIG@4MykbrZ-y~8XLEj
z8G1N0`u47l=5~cAC-0RAQdZt}#Ui8V*XG~yBg!jHov)8Ga)<|-xa+hC%e}HH^JNyZ
z>sKi@7d!9nF0GoZdstNT857s>&^K>{ksea7B%eWek9HJ3{_*mBV#Bn*zrXYE+`Xqy
zpB9vszTiTzzJB-aNcH8zUj`EnW%CCKj-g;3!+(=p@_VmaG9HUK(j^)<^7q#}YPnH~
zX$3{a`i059$mNC)3%vm|y#c}vp6jTpYHLHpty}q8tX=*Hw=*pEUbD3oMWpJit*&gO
z%>Mlqfp8e;?@xUdQdXwmIuz^la8y=HLxb^^F;V!)n~_6ltKWkNk0VEpSbcpY)IUD1
zCGWk-%5hh1L}+N;{^g<U4Y;pe9t(SBN828cv+ehLE@-Hju@F3|w{G3K{;{zd)wU;3
zo=|%tVZU~dKlLm@cI{sTCnslBb+un_U*FSa@t=j-uFlSN_h=X-M-z`3x>c589r#(D
zo)ZNpECYjr(vpu9)s9FA327v&3r%Ln3Xiwve|G5_dCpvZr~bL3*rhVMV)N{scKI=Z
zfq}zMZHjwdhNP?reEaf+M@>!5?)GiH#SIZsPGfb5k>>4LIv4NEZr{nl!Ek9n`{c=W
zSc<U1LaBAT&z(Dm%=|DuzH9$VhM|hv++X5XZ|~{{52K7c*%4%EzR1&QYL5>wzL!c(
zPrug8ncm<#z?__%T-VsxC#Ce%c4f(dW&8F|ZEc0k%VVuM2V|E29!W|{>i+Yl<mS}$
z;Uh<S{`?7OZ@<9QxX!#H@T8g9o~>KAN{$X$jns0*id(ZE6cWmEU$ooAb)3c4)>cJD
zh49qd-yC(nw~gMy!lLT(;r2qO%|4rVd1qybb~^Vl8W<X0TK%_}=dp6Ui<u)C2?A&9
zWnM;ozWtD?r6nC6$Yd%0xWoRkOQO%8Kac(wgHyIgRFwJGuU`h4rt2(zwRdzpd-Epb
zokgwP4`p*?mMvSit``sxaCUR+tBI8&S~AV3$e-e1t*$KRUW=<}|KPIiOp2zL-v08y
zokv8l^cM;pNBu$y9scb2_wS$SjT@V^w6r*-9Mu;MfO<}yIwj&Xc3~5z<Q5YXlY2tJ
zAvjnQQ&XoD6sVE;c1NCO=Hg=9ym>SE8?q}i+rE7HvJku|>%KNp$I?eshNR?nc|09w
zv-Z~8pS+H0!>v}@xamLdaP*!;MWyCwIQ(wQ6~433;}bJKoo%oD#<;M;rAuRS;b$A7
z3qj`Vn_5s1y@`>r`ulf{Mdy`lb#3h}C~5T}va2cu1qFON0wOKaHc(S*>*|J29qa7u
zyliH6vb?-pYjpnn6JlX**pU{<WH%KR75%17{a1G#+-^0wSlXaRZu`%lL)dnd%*|)c
zoH03KX=QbiZ|h`qj}<ULXncI_`DJpUb8m8$e*PTx^eKaljm_WAA`a9t1s^6Z$v?Tt
zI}Tm&eQjL0K9!?9Iy$-+nNG{SJtf|Q9^XfiyFy&q`{>~Ldv8is*~dRjGl|=D1_Fo4
zPR2wCXqEo_`M6EXlj>3y8&(!eT6&*$^A>LIU?HRT-MB>Q$GcRbHl5Pc)YKDp_Y;&@
zoHSFed?>oBC9_T`sQ1O4b^C>dl~E>~|CAIIup_7761_s7-`yT---(Bd$tiIfCrsRd
zgDIZn0jzxyS~7R@^N(4qq<wvR<*T-#q5pXCG7}9A4bH;73x|l_z7Yl~ih7*%PPIn|
zPhPsjbI9Za8G)UzEpB};EnWZhv9M2Wt^^>f?PRYaf`-w&GFTCr+~`Dz(zpD_j~?0O
z&VCHzmD6OBux5CO5T|D#7+n6y^7C&vdh{sUk*k|*cE%+qBzQJ<-MDw}-j-dvHmB+N
z16Gt)RBZ6@@My`k*#(GZvGTqNg|K^UjER<(_T`2Ax2jaH4;dFwC^xLFu~#7g_~OTf
zh5Bm?_aV<_-dT1;v?pFH@etg<|DHO(ysPVRVK1(_&|VXLLjQ#=tDc(ny`UD~8<(fK
z1~y*3dNt<FahK@y^!@HL^~%G*!0+bltZpQw-UO0~(K5pS_nkBS+n&EIGBPqs=FaiI
zfB$ab<P2<T($ieF34U_K;`xgg8)Rf;C?ooF6bkc+*<(9*?xavIUc8u>mnSHkDzhI4
zltbcXzu?haqPMqKgjjVqFf!s55!q5vQQ;dLj3AM~xrr3M`Jvmbp=2L&p5VcQfln)r
zwHoo?KTJn`T;8Io(@~AY!jB&9urP{$6dB1(dHe3&^@-KIJV|!ZOVkeIow3M0dN+U8
z*N5G9a5#1%CO;|Z0f5QM(w{dmvy)<|@qCfQ>FUzbd-HvL?91HjY;0R7$ji^M;^7E{
zbqPQnr?s>M0BF}IT=2PQWTeKxdML?nn}Dvhwa{5Tz0jhf6YCO=*$>tsmHb9&<;MXS
zplME7`76D|*5#DJx?_i~fdSj>@5Yo*E3~Jio}T0NFANT5LY4}`<Ln<J79VQu))w1n
zV^qH+rDA^RQd8$#+L;u~nwVo>MroCT*cA~&du3&1T?Zo1S@u-~efQAW$i&2ttgp#n
zbNjZKlLf69b64B!i=?DF0F8%TpB?`E&_U9-ZqEzbv}scnz&3798N}rB<;w~<nMof$
zNaHDaupjJ4iEAu0)YRlB$%`}27390k9|GD8Lv%X)Zty})PzE`2zFMN7ASx;<hU2zg
z4#1ja>(+*JpFrti>&&}%?{?=h{_yQRv?IU@(cUgrqhl+aVz>S-Q>cj6;_qMY1cXdJ
zM2)lEn;t71*99k92!7UNvbl8&y@m`iAwOY1hUiay^~wti)1{;LoQlhLkJ7O(i@^=4
z=igoJd3@31<k_>^yL5JM^^0466ch8!CHTvi(|=oY7;H*b#E0%2jm*h83ginuwFRjb
zXX3+rr(5RRx269Ur)>wmY%k`0e=MYP?+ptJ+9&GXqUIIdUm_3W-Tu9GJrzN9w3j`h
zJHYME<nNX&-da^X-g|V%>rT?_U-+M|f0mR)E#>&PTZ%q)Orpx7I6p7%R-Ad;duzc1
z2R@@dNRF$Wo&A%>%fk~I7Z<!)_TLsP)9}wX@l#_}C}dMyXA-Ymy~@MKM?E(;$0l);
z6U8Rewohrmxb)7_Ob;RT{F?J}>@z>mNjeJG3D<{DnIv0lV|}Y0s{fW+3XXW++|%LI
zwfDe*1N()92Bhf6xn<l{9zA-LeYtvTabjwcN3M#oamR*2#lsrY3u7V1g^qhc3Uh6`
z2Bi!GHWugoy!cP*&XO3`^+!-a{kfU#fK&Sg1bh%0f|8O^#JJ!GqpV9bB&)u8^H7&g
zbaHB{Z)s_1DhLF%=Y<OwRQMyT{tkW4NlM}aKe~MLX3g>!zSnfcc_~RrTPOw2lRUWg
z`)K_}o!(&yrlT7f7<jV`zIE6rv3CQhkpyyj@VmRRPQX(AhvH(7r_wU7^Wz*j_dHg+
zzR0&g^u~cyQmEfO(KVVG$#@fvtpwo!S7=6x*7^C`*s3+#3Lh_=nzFkhN+s7(;BdXM
z{I3=Aok9SsFu%EZpX2N$frVa=#i=Xn<s@u6`=w4lJ!@c~Qsg$zf9~8*eJQ$AbawXk
z)#c@0($dm~?sF-1&06x=UVVMh&pn9fbmT?G7il9un{sLI36EuEWl<+|1c@6OUA@YW
z;ue;YVo|uTIW!_do@&FU-EMAfCN>7wthEX~qr$^Woy2mSTWJR0%bd0e><(B)W}x$|
zh*x;M5gb90q!oYvwqIR+Cou@UrGI!>B{(?P`qEhidl3Byb$(tWBf@Avu^hZBGDNn&
z4WN9VnAk(t`LS0Y>IKEb^jw^-yY0oJElilLmjjA(SGHnBmb3ckx$g4St4hS$ibR)f
z(-`XD<!jeY$Hm1xjEMN`#Cg*BU)>g7US8C5J?_js3L)I#&tARK@b>m5HG;K&)0@FO
z<tNHlKHep$s67koAN)sf+p}LiBctox)$crEDhj}X&3R!L%)fm7YE>P+mv5_|<=o$2
zH>}srG<Dq^@9y4R)i(OeagzqtzSyQqW__LM^q!nvOPeR}O+n?8-n`Mm)82-@Qrsuj
zc9B>i*W<FOsT^Qxzf`bN>aLw3$UpwL?$kma09TIdBV`HQ#)bwG(i(nn7Bw#MC~AIZ
zxizT8L(a&E0~mc=aKDs);sRsIBv<0EtKXvTA}C2%C0z3FFre8)t?b$aS4K5rBk9`-
z=Lu83y?cAUOLX;1JyO~dP_Ck>S%KV2qBXMU`vwJ-`x4B&-T%X&G(&i(N0|C?lmAa1
zh5!T)t6ZA>Q(x$0qq+Rk*}bGdy7rNf5q=y}$Rx>`U;E-$zr0LLc1LS4=dn}R1ahT%
z%lz&Yn=kZNN03Sc#d4ziQ_KzP_D9W-K+?}<sPJvA#~<v{gnkLTBCZtCEEv*ddD+@p
z0|axdvnb=yT-&x!tn=d~YjNrt8u@=bKlgI0-0fF1iOK%{7&xdQbdNk8Wp=FNsrZeS
z-M;elNFc*)HYIx{BqY|$fxkSH_hFjkQtNAKYSNaIqd@sMF&*3i9fY2NA<^u!H#23V
zK9QEFzWny;#iZbn5CNnOXfI9A<Y}j7XRRCmOc>oqNK16N%j@gww>x|^vn3vPCWncI
zN4#Hj1i+!Eqf0<#i9iK*adD|f>XZBZ`}eCydxV67iq}@%a6>#oLPFq_i7wNF^~q|F
z4Ta>5bPWwpZ{t@dKbbL88bySLCLH{h-o86|jFs-i9z1c*^`GJQ?^8K(-7;u@+@Q;T
zN?y{+NDw&2619coX0EQT)SiDwN3B38XBxF+b#B}^82RvFYI3~&P>l|%?W;JEyY1H2
z8!5P*`fRI4%kj>ld*|!_%dwWjb<fDN0&UA$pF<;-&`pbP{oS5#UszJ_t7a&zIWM7l
z_H1Ou=zxh9M{DXy#x7CGWblWdCT@&xissm%E!d8=AB&N8*(#tJ^AWh!AEJY=0?+O7
zPOkUw-)ClJ-34_2_Wirr<;&DzD%3cJ6geFoW?wpB@~ZFO6~TtVlfoW7^3Ki{+d?qU
z%*=$IJ@5BBI5>~^%jUgl=W|=ye7f%)g-)U#|D3UV-5|fikj>t;*4NkHEh&+OAQ%=C
z69^P+`sc^11!PT;o9&XMvI|0YQdf5?qSZ&ZcwxOw@ghHnLG!<zqLke{t`+qI>nG}!
zckMZRc<W=~i-Rxj^C}Y*CnwMQXTF@fw0E^EKF|C5H5;RCTbL1n0SgPxe0+Snd3mX@
z@bafm)1$s7WMpI<9Hf}y*|H2a$;-<}J$dq}p<%1IxH$RbPoHuic`+{E?ASfh!n}LE
z;}~_z$m1RPpl=l+yF$PNdb>Z}AAUZ#M%&!nEP1P&LXe0bkZoak*?g=$pH0Nn({R}=
zjt@BnJds@Hq1yN%DgQ4r<tZ&e9V(}`cIi+LKoJHm6%G#%*ON8dCqSE-{c^eR?z+&2
z)CT9%WG_o?*|zQezB4aALe5y8ANRR?cU@N)+tIE6jyvUFU=kAgRLgzz8lAT()8GWb
zC4cMXs!`y(w{P``sr$5R%WjG^B0hKSJT-@iGBG*%smyO1^Uj^s^llu`yo|Tmn6CHC
zVhb2BsBzl#N%Y7O`O09fr{~)3?bqh#quF=wr*A1Zd#9)M?(CnR<_J)dgn}C({eE}i
z^QJ<9q4zpvt~{pR2L%*tn80VJrzl~=#2Wwl9YX6H8k<=&gLrsUl64On7qA?Dvgyyy
zH$Daj7q)&Bi|EdH!5`B8Bn3BHidu1&xODSSLh@OOYj4gOGk%L><UG-uIdSAPmNx2y
zo4CjFsKh^rhR!RqKNo|`ZmCZQx83OP6hD9d{87SiaI*fY2IDQ_pJb+0gyGwqqbRpg
zlD0gnOMiqpC2x6dpL0AlQ63xA-o}6Z`kz6quT4K>7Cm@&3(~2mTDjzj^GQi@@G1wF
zHD{c&o$6N&<d6s+S5ZH}>~1^c-aj_V7$f<~cv(I1<;$0$PoAVLZQCVH>7T0Dbw+SR
zzn0F;!6rjhRrQlEGr<d5dJozE8OpG`@+HHt?f|JF-g#c-ksq2zUVZuWX&uNz>g(6~
zqki+_vPI56n<yz-R9!^ZlqdUGZ+QUQ@!NtBP{>0mNvcMER#&jd(b7X647pqGZ4pf9
zmh$h$znpxg-96S(n0{iiJ~=cf?G=s80;Q_TlJJmZWaZ)t0w_!X;*oTn5yaWWa!trh
zlp9A3wMT>nno{R3u3U-!{Oi|F%B|%&>*lzq$3GXArEaG#pHX8N?kc(I|AMQv_>L6W
zH~BF2<$$E~#O|5dS@J3h<W-c%de=N$epRigB6kfpBr^aVG|vbyuUqRNd^WVUI(+3;
zd|oh0ckiBORaHS{ZOpm?ry;)F&{r2rxFK9rfQ@E4{Ly!`*Od}jNKdQgb=o*$_pY~7
z5}HY~(wN<`ghZ;Sr&KaBAJv4LgPKy*(+-UN_|fjQUBv6uT`*hTg9kT(Z<R@nfB5Xg
zr$nK?*nZTd-{fNZ^?IVJ;lu__P0jS6_MDu%KASl@Q)ed3Itu0)SV#QbrJ#R$fzb7$
zu$JNUideUb74wR&-?Lz~Q{2v`5X!t>ygl)cN$<{`kLAg)^Ky?M%dts09)uXPW&8GW
zdZ|XwZlAK4koLsHKXdj&lmlb6gZ=#vkoxq_olBHoA!zCc4f3{#96ftcgz?te>Yas!
z1*8aCNsoV$OEbSeJ4rJM9HLNmQ&bgB>hUR=;?|Gt%6D_I{UL@+{YX^aEXYFNUVMk+
z_3PK)ZH9~Hn^;d%@$vJQR#wttu~Fl_UcC~+>0h_?Uv?mWZN8JdgvuVTL$?=$c~tNj
zqOPIg6oktz9Wf=;5fCmKK<yYw+k4v`2h;vrsG1HY$Iz1-<TH=(a5SRjfMDuDhILo2
zrz2Ju1x#Fj@7=^9z6r2j<j9desja-G{{^6(MO<x12iMQnIkiWwo+V{ja|rt6VVs0o
z_8cLPaHl3T|0h7cP3IRDz6)EnfAHXeDH5SGdR19hzU~ljF^g81kljn~b#ME;__xoW
zsnE<5+_%qc$XP}SAq{R&4vt3Z!XH1%VQ<(Kc$LO(BsDnVdiQ-pqoS&uSWc$Bd|5X0
z8#iCc^l!3q*UQW!`?vjfP-_TmN+eh;!nOIx(dpG?2Z6&@dH3$EC-pH1;lBR<_n_or
z0U42#O^l4bj_xVGA?suUEjh2CAU!2-ZAjLkON$Yd#2ct$&ygeBENWstWtx3n2rjb}
z{KV>{q5pvA!EWkbRfAIg2`<jgB>5V#q^s)=csDsoOp={Rbii(nH0^^tFu~<lPp+Q=
zb(nXY<;4d))VypvElX&X#ma$lnORwB7aN0XukUwql9PXYmRr_?^Rv%pA86!g$1tLW
zL#{yZb|<E+QQGLMbqOa0g@tVv70)_3?G_p(+={RJqC3F_ZGseAczAiu2+OLQuI7g1
zN19okju}ewSoc9PguZBT;e&J56-F25Ca&RUC);-3XvtiMO)LN4GMx#yJ3Ks$d$r;v
zL*xmagYfAGDV)RKC=>_`2#%jqy*IBju<f)e`yj>Kv$i_EwuOb|LF9pRy?|%~jy;sd
zF*=WR`TQY7`=0=*Y4`gZ7)!A`hCh5&sAv9eZ2yoJr_CAmb-*C+)}Hs)ZQdGB#mx{E
zh?v&h`t8TsYm2x!@@3;wQUr=u=8PUjMm_`fLHz%DLsOLNcK-a%7aC8!!S%bbXeeSm
zQd=99Hm=Ug)dUniX01zR5YTuR=GMr_E=JJV<^S{B8-{K~`cbhP1|8Mv>!A;ChK>;<
zOK<~mAXs9y{;M*Ma8R<q$zg1H`+msw)-6vjFB;(8dqDHMp<jXODgoJ`BG<jM&;sGE
z`u_nK+ODuD@&CeosC_npcIRdH$}QOh#0XC8B2LejfIujHTm7h^22d#0*4A-n-hTrc
z?C$HM!kPTHGT${^T5sdOX<AUfeWwWvD{CJ**kl1(M}Z>o<r1PZ0FA(b%IK$0r^9u)
zg1JxlpEGfx0VHoJA!OSY!>x|qsEO_ILf;G`EaU_^1>#HTY987Zau+X_L^d8ZhN5!m
z(k1pgH~v4>U#c-3JN@LnDu@VBmdhAJZkp%0VzO0(Q%&LoNIV?U?>$yrzeOKqfJ~&|
z?3`DjB_tZH{6A&%HRR<!G?kqPAJv?S3qN1zD71h7eq#6IucBt9)JR$Pe4%+|p<xLb
z{N=@xxQ>{ziBl7@4l%4w@8hN3=)W;x6c9*B;jU`PxqhyhgK;zM3w0~Aw-#y|8j6;d
zg4eEH)5Ze$`_l(<pI|@pLj5Fuh@PMq8g2H`cg^Cww*PmX<BorQp%UC3uI4D>+e)ZT
z`@6Xu8xn3of5YrP@pZrs&(w=Hg0HXd$Lnv3Dy-HvO-qv*?d8@;x|4LeVPB9?{&bbd
zJ`%pTS~e31_<!)=K|PzgR(8RIFAq#{=GhI<&y95~mnn8fN3$b6zdZkL!vN>(cM%P6
z+1J(fW7|{ZMDE!#<SqpB5YB7!#M*q>%qHfeHZ>is1!J`@4<1dvZVASzRivqX=FI&A
z=U$sa1JD+1(XZV^7}ySNY4b?<O+E%fOd={yN08<FfiZo32f|Lf9bSM3h<oX2Q|9~&
z!Qe@k%Kf)r8;>`O_9wG~5OKEg%|+Rl23e<$?bpW#iKmZ#H_+5ItRL$*b6p1I^2*mo
zC+0soBK*M7P@^((2yY!P2t-C7{oNX08RpQ})YN=ge+@lHSKdpardDZly{ky>eaON=
znz?9`f(%4OMVbD3dqo8h2i>KnR@u*~_pD_<?hzMCELdtjf`FvdneHN+(garfc|rnZ
z`}Xarsi_nSSruQt+(bcek|0~_#0i$l3Lc3=OvjHO7ZDft_VZJHr*%9%HT5I1XuLB4
zm0jD|n9<3}>BDr5RMatheklz<X6pZPM6AYFF=InR_M7c_cV}j9L;m~t{X65&Hy10B
z*h!tZwDc4ac}V*>mEdVjr7@e5qs|26a<ai;Z*Sky+FJVK#}4Q%^5{#N+Sr7c6uEw?
zsv7p2EB-g7);~0KfZ#5_B#O=pnQS%#PeIVK`u*6`tO;u^EIih_Dy-kWmXe)K_ImOl
z6hV&-4Gs0|@4qbVzOaRokdl%@9L&RdPE1UAK6$beI4&~L#@yT-$yH8EYqMNOc=ghX
z1*lU9XQ=Uo3xUvp27F6aYhz+K@ig6_0Sn9HC1H_~$roBJuUw&l7>}X^yiEm26Iy-w
zOF%04JRLoK>9;3GewqCUG=(B|{cZ3t-M*w-o>W~r&V>DV=LrR6<vx%sNn%wRx0s+U
zXliA(kzCS=QU;h~JV{zJJ3_N<Tf~l80c^~T8!!LOClbUXWSgXv6yAONHcn1Xs+>N3
z3J?N>=H-QVH?LmZ0Hesam}A=#m4fcQetk3G?cq(26`ERJ-Ccjl($bggc%XYoxGS4Q
zC!jq68u1yuE%b_lK)kR3BzZyn>K8-L4H{3?;CcMj77`br$@+~OH<I^?WSfwfcpslK
z|J#zqChH;HeoQ7k%uEl*?53%yCq9@Z|A0mvFaZRYY#um{cLt$5Cn+iU@#|LxG*#FL
zS&t71xua*B@m05a%QkWzv!kZn%v}cY3|qb#!ZXRt7AAV!l`pgPusWgDjoid-KlrUj
ziherk!2{}l6Q4J?<XmTkq%3l|Y76=6$cXGm4%_VGA=hJWw9N<V&vwB0TMiD2u<l%~
ziTTxUwZ~gWLcg8Z8Cf-8_qQz<^1%d^X6$hqVA^j_B{nyvX!(%HZPHm$LDFsRpyS_`
zyGUDr@ut9q&hs5kqNX1y#eJfqf!#Kj8u}f^tX0v`eQ(pn1O?>7><^=1=($XslAb`m
zri)Vpqs5t24y^V*j2zy(h=0uBw4Vzc{t&mYrlf>~z8JvbfD}(-aAt4KoISzMQ;J&$
z@7_$_4oP)jognpPtjS#IGRxA^65XavWyr3!!*zR5g3TZ;kedLUT;U}Bg3Gi#c64Y}
z@7x=c@~SXiQHvVRKrX2r5Gc!VUb?&MJRUoW`w$@^n``1u1f-@4lC=cFo<YHFvTL&t
zT#;=?rd@TWW2d&<Ys>rY#1e_@V2*soCV6W&`aozu`NDM5^6S?mmqwxs02{U$jlx}-
z_CuP$Qr!@b2BmhO@+0V{VhTC!0H*v@ppQm$<lEB%I8|;YWdC{M*SLY%PL_DAECe@S
zMG8Ylts6}k63D~Q<%^t{#l^QX$GkjNKKi|F3$Z%AYi7J_7l<aQq9gOAWL<Q7c0^S4
z6Tk}DZ@ST%y^&Wb@FSvXA-Fty&&xb}l?ZVr^pbYL0rfyNf`5;=_>L29b40(CZsEia
zBbz;k4w0=kHneV_7=b!vGvn#4@74t@!;3V5o{f1=saMN;YmT<J?;~f2>#2~cbZ=UP
z!m#7S_kwYYyR8iauhSkW`@v1?*RP*gT^{F!f@)s!Bf16xMP<k?pR}}n`0}Lo_&BrZ
zfbH|cmZnXDgl6=PrEGO2r41%`X3wfeiLAT+^R;!P%&v~$+<kU-c3YU4?}GfqBr24e
z-MA5m#{M>l!X&-!vSnke^e=pTuH(s5&aKyMVN4<$RhhupzUA#Jf_nP;y{HS7u)3LD
zyVlx21c=oGxvKnsfq2c-M!#3D2q7LGo=jrR<KJA%N#0F~^dw;B_h!>osi!)z5IptZ
zh((PlYz%aajAyxBKW&G(>!8v5O%gZT<xrv`;(unG6Ltru;N-tv92mH%rWMKt3#p7E
zE)(Cr4S(km^<LwDm~8nL`3*Z=6D7KI;wIh@D{A?yF4u<%c#O#M?Gd*{7^06GtIdbW
z4Jmj2EqARcX=*azIMrCms<)N<yGcEajSWI+HN`f7`$)p^Fflu;h^9fxioyBwW(`TI
zEk$k;DJvq@kGn&DnU@={kP5``@84$adA2dLI<UZK$t+SsXUNRV^n{(SjXt!O=)FY3
z1E4+xlW$0fGPJrZyMfaKtG`)=P0HNHi+t`<(Uz8#QDa{e;U@r5iC=7)zR~uc(a+CM
zL`<v<riux)-qHu`uYJSXPC^gk5fGp?mMlJ0mVJ`%7KPORV8J`H<Dh{YTnRldSAa{{
z#Vog=^F?T*lgG^5QGDkqko(agWv2i9BEG&jf>pnMJ<)$<nCvc%eo0JhY;|X69K=&U
zU-BQwTD79}R6Gbl61N<c?vnTL=~Fcdl8$-Ie0lbXd-0FVnuH;qK^lw6S;GzX!mWhE
zoi>$wNLg7q=P&=hi0WGee2L%UWiKwuFfcH<PgM)#q5rnJGEPuY@KsPVebM9Ek&}~S
zkZ+f4x4L>hWTPJ{CE3s{a{iia{4y{1^Gdtg>C>jBrXSBQ>*(lEC~j`c8^+I7+I>ar
zP(E%uBZ%ZhO^G?-wu6FPElXBbG_XG<oV>4XNEoL7i;16&s$*kgOLe!Qe5T-vUsu}U
zc<_gGX8&Uqm4HG)NAQ|W;VMn<q|wZup(I8|#&r~u41v*zIQ}&#D=#;L5eTQV&3@Om
za4c{C;9w~{jpKrW8!nZ{n(v`Ic{m&vKu_4@QkEYhIZ-G~+|sFY#QQ(W!y&oGvJO21
z12hRY(^3)>d6{;}1d_S_&||VE?=ohtvG?&;qop+Ib2z6+d?{xv)j2BWlh&p^|1hLe
zn0-d?%eko<us%RT#A$0l`NJnqs#TXEaq`342%{j&&Yh|Q0|U2bhIJZ#{uH3x2Ds<X
zJ*%&;L?NnbYBU#ZP%_@WfByp3s)0(fSxI_V(_g>l=Xs8jM0Qh|na`d(7Xcv=oM|{e
z<fs>J^Y_RI)NI3u_(o2MbOJCm0GD-V?|NZ<iFe+y3JL^jL`+9#C)|K#Wn~}B%V}0u
zSBE<aB@SK4^DMrz3~LLqm}fu4#6n+g-v&zy)4lum*O5{^j=Lgyo)F1>Bs!g`(2*$x
z&<WXoK>y>nZ=}>v_v6O~ik_aH>9;6R039+RDFw3KMMjmL&w4R2KTnFDC=e}yZ!{a>
zzG4L-VD2wBQCd(?;QJP&wW`Lq)ePr?h=MEU92doHem!T7@I5-WEY`-4I;&R=K|H~a
z!yZ0-fEAeJ@^SeFemc>hCftJvcSieUG5E9PZiqlBASOXTlAUumnf(-M@2F|s?6|)D
zRCSWJU?zH$3yX_cr8QztyT`S!<e8b7-3^ghqRTk<rUF*mx)Op|(;vO)TOvZgw-JsG
ze?UWEfe<om%z2%d=IvPvNMM<$9O4bzCR(Qp!JjD$I7}12EaD+IFb>ztI~V#Lj=sNr
zUcaT*I2DK3ajbo#v9U3H8c!k|eycZfNjYw^DVp7Z<L^moBz+>JLJ+LF$BW^@0s#{j
zX*viDR6r(E3g%4vu+GoQnD{qI%dZ|qL9HjL*tn!R$47su%FCgjsJPuLo%vxPijld)
zP7%4$B61<`*4-h6QCeomW)4{SN>KbDxM>yhV<OB9?`?P5Krywp4k8ULhyb$c{=7&C
zMKJ_gpY`w{WXJ-GeeEV|#e5f0``gw>eYw;loNX^GHwsEhz3*ye<Zs;)f!EKruY$4I
zJV%w8nR&P|Wix<=H?B%D98wry({F#W0je@O;SfWD{{3@zhS-7wuh({xW7OEr?(l~X
zA0}F_zajH6Nz%NTxV{ih2VXGagjzL0b%VU(Ln;l8FElc9bCp$8=+OIvNi$6M%o%3r
z7YJD%S=n7FTJaTVfISpY77Z-O>g~if+3e3U2!v3`17`=aPMJmgoegM}x-Skqz7aCX
ze_CC=3;?_5@83XlrFtrN5oKrx0bL6d6I?-7eFnBRKFj$TUtppkooi4!?*Dpc(T8g3
zg?{j0P3+gj!Sm6u;n0)b^;DfivS$ay!whyk!0u99drz3kLU2_!6_AFy?{dDxW9SpG
zX59k3C+~u!9~&6iACM#*X7m2rA<oY-UcD+O&=AbDtmnD?=LaKh`~fh+>U8`X+`1dd
zzdqX0HhSOc1TK1z?Gw5q?v>gHaL!4tMfxwvb;cEW(cK5nV3T$hDP9^mP*zzv%@tt}
zcMK^f!^mTL{dzzcuW~Q?;jny)Vl}f+@kz;Ch_Jcr{K<Q(dH^XQBA&5b|2z@yKn3d|
z{*#s>ShunFqDZrWoX2$gv{>N8N2RO;G$q*rA^>9R`kOazl0JU4yShh4zBs9SuS38l
z^1}d6Vp4QpHiO(kucSqZAT2gnGMzJLsEF&=uali4Gz`gtKzfR{ZQE9yDB<r5WaSU;
z-3=23*($|`Y3u86Ku|inxR6JWM2A4o6;2ntUDh8M3v9($fU~P>DR>J2w&-a!HPQxE
zR#DN5D=Z8?wF!-lWu#<gXFtfuH~>E7Me<3ariKZ2bTp58+@Ii>2%)`A_Qrr^0A^`X
z;#Jhu6%f0mV6!8j9&m@eU9z)<R@r*wZSm$(I_?8wn}*Ox6LFofg!*B)SPZ_4=I<UE
z8E)8o7KY-9OmzG<NSzt^{reMGEgGF7qM~=RvWTpyTF5^itE%YaI)cv45^vJdP9g!{
zFM0g<v3%SUzu+fNd?Qt~wY6bG=Oo>Uq&hWX5_;ptjXK;6X+tG@Jnil6h{+99RQG=@
z>52xnX7+M(G3FIc$3E}K%E=)$XyECe-78nFT=^1qd&LMP1z|AxlekDcbDJL%6cSPd
zdjW@%Q-Bjj5&l(*oSGT~3JAp$`1jPgbF6SBRirE~PE=Qadi(RDGJG_qH*W@`mGB8`
zNVXo|zP*%jK1$qcMa<1*GS<^Y0cEadWMrw#t9tF<5QrS8a(MW9bnTis-oK+_p?8<K
zMeN?flJp{b6Yqr^y`ko(x@wKjpmaZn>FE1Y9&CV$)|=(!0tY$kSFWE$11VX(HKH2G
zv)iPdCn$r1gRnWL%O2ZLpIUN2Oo-7pP|rrvn;8n_&Y)1Z0T~zq22U(b^sKEIt1&DL
za?C!s#I_D3qMi;OG7$-hb1QxXJ<Y8d%Z_-;?#EQ3qH<Ds_a9kx8Q5BgkzT)O2|H+z
z&I_46%h@@1$BWNsiFh-U;$KkR-!Qk^w~a*!Jq^Z8hXdN$330bJ@yfl-d$-bl^b{5s
zJ@d0f?=yLaGom5rInhgh&er*#e=S7@j+78MdR}$o7FJq9%&{)_f{Bw^j>n-mGct!B
z`|${x_!%p!eXvZwY>f!!zB3m+w>*EhWcAwl|M~u7KXxGHSt~q%*am!?Hdj_L*tTo$
zj(1nW)Ss67OYu5G&e0ZRNe^k?%uSu15x<&uYF*56nhPbJZ$1CJ%_H~D;Ve<ow-fVC
zO$9=<+1VpX<3Sz2_V1vosF2ft<a+OLpl4>#XSPq*^=&02Xjf;26Xa<>mL-jihRAw;
zef;PBE)mgn(eKvn{A@{w1tEWJynUxZQ9;jgc=*T*)4!w5QNd;p{7B8Ou)BFz@w2Iz
z<vwX?&(c#z`7W{bU>8P*4mn2>k&zS%7gy|~qcdL)IM{4zWXfbn=QLl=&yDFVkziLH
z$sN7IHkjX16>EODY9Tmb^4gjk?^YbQ#@jdgYv@c?s*Dd_m3_UbQSUFo!0U{kiKOG&
z%+lg^7HrK~YeIqDSz%J2Q&Dlr8%eXyQ8#3tXtTMdQYKPb=`TgJ)f!JUi3{2PlN2#m
z`bs4lxB<M`{D$oOqYUAZrj}<M>Dq!6@tn`u+mM6<M~6D?TGu)vik8l|&5P3<7*m%j
zCD$7UR!YK)-WvUOkx}YpRSRk@>++)akAm8tINsmxq%_fV+|B|;P5_Zb9TL>OQ_0~*
zp9pzFMB1yo`QdfyPo8@#Y~pqQFGOhY;f~6bu^_Y7b%=+GIeVKpNOdeMEW1(tp!85Y
zaTZSjMuV^l3JQ9n{h<H)LbbiFH#0wT&O*(Wo9K@G1?WN{?USN!Jl*L&tpUGU#;0Y}
zXEZyI$xbSyrlo!QNZh5e?J1>+(>)xRnZ=ME*u9BOgu)Y+`e}KD!4(KFJtG6X<DiY(
z_$MBQnmq`?!`uSBtwY(e@KDDL2}*;}KIYGs++-Apiims!`gqmum>>%4FPoU9_JJ3d
z*uIl%q;m4)$<VI@K{EV10z|<OEp`yzR^!l9Etc~c!ePP(I!H*K?Z0;jrF^olGNW_Y
zret094Ij-Hcm9(y4jG`@UcWta2#Pg|Q8P!3>@G=1b;fEor3I+6Y;Cngi%4>`%F2}j
z`Zb`0pD;v`Q^7)?khw)f;+**@Z{|b)yPPs%Di2Uk#3k<%q@a#DoOFptPmOYO1HHW1
zw;c|=g*-#oiYH+NkQ*o&uydJyBsVztW2IUNJN^kzV~NL$lCX)hyAr6vSE?pll;kSE
znY-!#{sm`vcVAD1{sm?MdsLWJva|p5e)*`lzWs6R(tBGrUT2i-C_=<+{p#wn4T>=n
z7;V+puktwgaCxkwz;8tv8SI~(nV~dwY=UEp76SrNL;Z4>Z`K6=y8ao0`>nq%qBDPf
zZXUd`6!c$!XJW*u7hW$o+vVVxxY<=4^zb1ID$zR1kJrWl=w_t1kA3A~u5cQ8AmZdC
zd7L<2id9;)=qkQ*UbM~>N^2>g6=`z*`}fA};ktDw?-i(oU`Nf}_uSpElWn!J=U})Y
zJESB(fO+CVvHMP_3$QqmCJZnWPA)D{ltP$C5{R``2VPryd+<BKc~^<m@!M*M8dOsX
z1x#F!*thSbi3vAIEE0Br>ijtG-n}%Wvjw8ZOU}u*)>cxJf-QprZ0v`|?18qc4^rN~
z-CD9T$3c2Gpa^_~I~gWWvbAZu`p@M?=ZBr}lFCD2BSo(dD|2mRd6#m`xMZ|y2OQ;0
zW%Cg*TA7*J>lAUfuQH?$*wfd~?-TlFd$#&@_w?Ka=ikq81N`J!V&Z3)I5dg=0U!GY
z1jt{wz)9*}sPDrgBbOjdlASz$tvHl@*IRL3F2%5fJ@D}{N36m#Tn6I|GaH*P%plUX
z64S7j|JR6xZ3a928DT>_03`rb$iH*L_~d$8Pc$yRI&FGE1C~j8=+W$}`t~i5qK#YG
zCn)F(Jy8xncUQi#d+%N^%$z7d$32dru5FYNAo4qNzbtM~4{8+m<w#*eSh%@0A+l0?
zLOie{C)ncRI+KRZojqHR=c~eyBl<5JF$Nb7=f6?zP1~55i!bwz#N-oeGMI>Yo0}^@
z=^q(6ouU8gIfzkl;?XCVEdlMWhwq{K%NKu29fn^>XY_wC{pSl~%*Mfet_nXH+9N6;
zo8-hM#DniQ5xV3M3|!GgpXT4be5oE(?R6K)gc#krZQE~n<oF;b9X@(A85HDVBK=Y$
z`ZnaSrbe`wA2`){1A{t%C{2tFV3zF&q=|j_+NHTsHA*T3PK;a%qMW~4!KeI#gZVKo
zMGlApX~2S*1Q45%vr7hNI#uJ;sr4H1GO=hH00Pv{jS_8X&!68LH;AdKs!9Mg^d&9J
zWn~XQDmAcz?v)J&vx&En1wh<Wb{@H^ef#$9jF~$y3eYn$!t0j1A<;!+&EsuvO|}7C
zRzm0B!dsAC_4)ICAZV3m+OPwjMdGf5*{-Uk>yXW!|KlZSi7Id$JuO8z{<W;nKIs-7
z8L5boOB@^ymS)no`{;K)>rSX`1A3wSm&!Z71`IFQj<wqK!2&6dbw;22$kOkW_*M-a
z;sOrP_juWOJQU+DnHaQNJ~{E7bag*oR#oMT{rHVm4OT7Te~q;;y8LIXm3+e8yOlUF
zMqyt5E_Y?~E}0~79&#v)hG7RewbK<w-Gi~tCXi&(LkIPFic4AK@ZrO7<c7t@##R_t
zz1;Qv&!0aTqop^mBSrW19^&1(S7)om==2^8jJnPYpM;rz25fVZOZirMdOA7Nx*N{U
zBUisAz(HNiJNcYQPxr@4RbkWu+wAZ^BZQ<HHg5S{S4Zb0sOFcwPe5#WVGjf@#&fV7
z7~^Dh+F~i36Wg^3+O-<KL=4J5rk}XGY^jAXW(U)m)y#1HJ)^I+$%x$l@ePx+tf(=9
zJUsVM)j%u7VF4n()avkr3CdPNGb&EphZym9i%?WnCMO*=VkC8VJz{s<gIqxRg5e#|
z_VAGDvXz`3PSxkz8U`^5181i-GzO&DQ;@Sc+&gDWVlj^dQrS8bd_3^Wp^L?VWD6MT
z$0raq4$g#6(y=~1w4~&YlCB<2h;0N`b63{QKpv&0q50I>%1(*}koP7L@H|J4ZU+qm
zLvdH`*F%q{XM8*aleJ07$$JC^>A*(4$;w$&lm*dywRng$hi(Q~azZU>LYo#I$mbZO
zDDJaNLJ=ZI<50LU!B{?WE*uJaJcZD%jXR;rA5s6GQqE5o@|q4qL%Of&3S4KeWej3h
zx6fN(jyD<HUnhk7L`!Ux&}~#5s9f2whZHA@sjZ}kp$<qGP#u~%%j=$g@hiN!btBgo
z&)bFs^M9!seCeVgJO<?jR_tL^l9+jzqf;9y(PwFd{^Mr#J^Co;UfmMtES}i6ryqWA
z@1m!tC({aPL_VmA$$@M69(C>G7uAF{rzh>t$VSJDns315sUK+~#{e@FqVn_cB=e34
zAk7D4wgB0vJ--WUn|<{=JNx0wbbjT%{IbHLqG6pbLtF#do-fev+$}%|2*5|$E%oKh
z3G1#%>_N`#vyu5XFJ5?~UaEm*^7<(}pYs4qwStXY%wa@tX*e;&5()u1GcHybZ;{p|
zTlx93eEzdIb^b&Yik!L6P->F+k~5kL?5#)Mq9adEj*?<#tCow)F^t7Uq(AmV?_KP5
z-Zp~Sx;5Jm6X7(uG6f!$WlP4=e^R!9yB_<k)jWRD9{quaar7B6@vG@_#019@^1qIy
zrDed9EB7JC;+Md%(|VBIlAOBk<Tfu|t04?E4vw8YdzPGK27T_AqOTs9+Q7Poh}ws>
zdVR5^6T^2)E7!+rtKTgV8X6jV;gVgNFIns8CLM$FKsMKQ=}5fD%Zq{!-x33A)Sl#7
zMtXW{By#23qf>{23ZN1r&pgW#Fx_|_Jzx>520my-Q;&MkjKL_WH;xOim=AP0a!vxX
zPjn0n@<{$JUw^cEEDo@u>4xEj7oUjbi|{XDzS|tj8)G3NA|i(l9jR-ewWt2%Fb#yP
zy_k6J866F}vpfe$HlvAON<Pv`@A73ngp26yVeOn;<Nu$wALrJpi(+B$?kXel3eG$8
zu3aJMFPIN}iTq-(FN_k78I8?U1TE78KfjG6IKsUS|M??3E$T2eIe8L3UYPccMsJPb
z{J@Y%AxG!n=%9xc<XhYcj_ILVpQ55W8JC^oo=VvCCWgq|1Mq=DNXi!E+!i-CV@TQu
z%<KVAd83>7xw?AeMfXV>r4VjL3S#dg#_j^168J2(7iqN=S_b5gPRn@cTN^wDY79ny
zyyBpcP3DnRvH0rdW-=2X73q7FywHkg!hMikMiig4NeKo`PmQk=JU|Gfa-5TEbmoja
zJU!1nJt+epdPm!G@wS6(XG09>D7a7?6+ud{b9Cf~LPiW+<HSikEH17;CkuDom+ETh
z-6KE>nv`K^G}mmFZPD(<TqfRJpm!_xOJA1NPN<J|w{Gp`c|J07?XZ{_A5@TmzS+jZ
zK=_!>RFqv^Ahl_{t3dxX?`F8Plakg(hx$iHHFoUS!T4xOrqw21<__J*Z35Rx{>Mzg
z@$_AiwcXy4XL}#&A?ALPtBsDM4JtO0^V1{|;3>7J14F-CD5q6aEGIlHEQWTl&Xd)e
z0!c{!e#pknn*}J@FzoX13><rngaPq)7&fRcYYVlm>ai$2q~RS2xVckYr^Ej-x|Zyc
zw-TnV&k(Ti+?qPoyfuV%9#AS8h80Bxhxq$&fTP;%+x13xEW|?#uUObf`5tpD;*z=f
zM!7_rkehb%mL+H~?xtvQ9z0`WqDe^w(2a2)`>unDZHylzB436X9DIRMfI9!}2iT2g
zAcujW=zwd8nOAJe$^A1l^mAB{P~~FPQ^uGTB`)t|KMbPOp5#2Rf	=d*Z~20sm0t
zpA)iwL>(HDL@=nK*;<UV=P+D%ml6v9f@w&I+lRbu&DH`?XO))VN3U*eE%;QcXf7)N
zrygDk!NjS;A%24#y^UOom9XJn=loo*v(PDof(dCGdqVa^4YU+bywc#Md(X8SXEzrn
zxc1Y9VvK|Slc<D*vWZDtYg-XYD+T29;;3~%xr1yCuFTlkm8^!r#b?j<P)yNqCk5S#
zV6Fh9&_FiP^%zicLmcVBT#HI?fQ(;cO&RVil7hZQng#JglDLrPuM8WdgLhV_#*f?r
z70oLw{0!GEQr82(Rr~leYb!vIiYUD354H^-<@7pLKRn&h-cF&wNzB-Zb-zRT2*c3X
zCC)ruIFQh7G9CQ(B=g<7&*%qj0s01HMYSCE3jqiJKdqo=Z?QGflf>uOOot8~qIjbI
zr5BAtB^7UVM3;v`8WbpGpOOJnB$Dn6B8UtcSbESx?9UHLJG1ih2kPXg_0J>X;z~$a
zq!^Bg6*I7v1vCiEptxH#zUV>ceK$|d(*Jubk)GYtV({hKD%3TC)Z|g6=r(S&nh=HD
z*pg>!o>)s3IAk|l_$oGxhxqy3#fwKUj~sNBkQoV$jPxQW?yp|PucJ{@jt&{dhv#Nk
z<Mr3>oKmdoF1o^!w0q0&xuwX6x%?TpJ@Ik{on`he0O1G)o;?s){R0D&U}ib)O3VoU
z{_|%Wr2sAah_>{S?kg4+DgSceI5}@*BoXCw5qfbEgz_%|TCv9+bUR<_8XIeD*swv&
zvMymr=7lEedJ`H)`m$B>Do;I5tgi6#92ODTO(F9Dta*Ge@~oG`OcR+<u+G$S`70-(
zvI<?u3ngFGl0)R==;`P@`>^pkCJwW46H8dU>b5qcV_IDXB`7(dE|ET$Fae2ILTr0w
zQdH1f%pKy|x<Z?NLxz8$_Yh`XfMEU`4|yDM!92hXDBSA$^+p_}=FT|`L=Ahfn(W3&
zLdHq}|84MP=C#B~_wD{FW(e6ttu+{@*A5}Wq!{}P7$n}#!lJFJx>4w2aSULU6<kzg
zULg+_2a67+3Fa+BIOmEcD>%p(VjQ@r7dIPqTm!wH`TTb5|9{j7`~loW4*cgf43PYH
zR*Xihx=Juq8jhxzi>qq`Rt)1+57DkC$VnZPKV`_`<fIPFI%Lu|%(i640A<$6K|s84
zw4Q((&OAP3Ouh`|3+tPdS)-Dbk482tvTpNSJDTfP*8MB=;!#nS)83M4CTyV7)WX=J
zc0;AfRJBR(r{&h8{I++3I?7B{|N2?FZ~MDdjb(eXukZ8l7D0bPC5*|aW_+{9-|LUx
zI*k|o_x7ILZ!t8|Qc_NX->9Qy37H_yW8vaA+bt)aTh5Pn2_);P!nj;;R+K#BJ?v*)
zGW&0O_OD8<)Tl0BAdPwCP|&zYdMuDP5GS*`LshhEowxQahi;$lq4vFC*n>1*1GL$5
zskg=vylFADGRvDt%8!}iO|5SfcwG8^266Nj2J2i!X9<m<O7txjnP@1c*RG{Y-QwA^
zXB{|uB?N*^m}>7BQ;9rX?iAIm32F!M;R9vg49AAz2~gCHl&`%%685fCL)s;m@t$CH
z{>E2s1Jpje^IP}t*)m*0%v{uLm3ROE8v&cKl^A=meA*6jcn+i@hBYZYeQ0y8qn+K>
z*`6P#kfcmA<=>yT8_JAx6WtyllV4d;5spDsxbpnRwH34y>$M>|q@jiN{P@A+c;U*!
z(|bWPPn;17k4-f4xYE2!=-|N_la<M{)bZ^DcE>CnB8hzcN9}6~vI|Vfr;AsYu7fA;
zAZau%HyR*Zyc=dT|A2r#sfVXsifi0{P1>I7*j=9cHc}O^aSx?`U_kM7ayaN2V}q>P
z7^iNSKTA@=_1mN0H}4RMJgrF1r#7dZ@d3wPWKw>J?k;e^e{aKKQWVBit=wrMIb8b0
z2}Te1n7YWIhVS3LrNVJgpi}T{qf9=>&YksGm_??<imzY4CZY`xj<+~Gb$XS6mS(6r
zf1mYtv7d38S5ei)Y<m?<!gt1h>HLegv@CpkYW%CfMTQR7;NT$N<+H$af6sGhCL?vv
zsE$cWX6}o&)7>8lMnI4e?e0^W|F*AhA=+Cc`y7|D;6tnZkJ_KmSK~_|RFI?47K(4u
z?*m-B!gr;*KeC}U=lUX3Wp|#mOt8(BN@j7(I=MR5D4O_o4Rv)(lumb_PtuCXy7lPv
zsbg*r?WbB*)eE8l!ND(J!9gR4>%X@VF})$6JXs=;3zCT|vvcEJCAChL+ja3y7Jvw`
zYfY&ZkG$y@*;&-a)RVc6**yS1PxDw=!t`;eljMi5-@hm0D3c+8w=4yOM?@?#scHjZ
zYr-y%2{9Tc%c=}CJMTly!Ban64&C%@i@V*>+rgvES*~N`8-)lZC|6jARJ_6Ix$)q+
z?d(C`aLH*PS6((Zmq%aU;%Tu-BB(p;u9~na;&);HI4|5bIy-<DS>X3btb=WG_tD>T
zPirjaTJv*Ib-SPIe5CZQhXbab!FrHuh7w6*uj8k7E$2U!1KI^nO`HZb?MW_p9%Y$h
zXjtTW6fb?y2G0`&i^i)HR3M4Hv3vG1-5&#bUZ<WNNE#d(Qr@#?&xU<T6*Q09kExH9
z>#3jCZ7AUjWZw8Q+fZn<t0cbly|o%v9nixvbN_wshhbqSNev0Z4PoF&(30zL*+nL`
z3GNews{&puz=zi_!PZmz&ty2U%=(J7%T#~z-cT<`PO~eBp{Gxt)MLW3)R>l&e_mkb
zfm}l3?S~O_UZ*pfbuBGZ$IPu)mS(Lcd&|*86)Kj`5X&*tKt4%89)+T`7j2+unL7mu
z!l@gp$uUqYB3evJ=*Qg|H7CyK>Z+vah5Pta9?8%FU7p#Y_3Iip$`(HQN<YKpOJ;wK
znO*IR8|&`|*NbYlvfc-gUi$m4w(8TTyA*9a7fkOZAWM>;)|B@P3MQa=+5p>7ij-Js
zTiesovX>MbMEJ4!AYcHRG;ux9<-YQ#3OA^Ny&}kAQ_$V#U<9Yj!=F4+{q*Tm)7vX*
z>*dG|O4d!2T1YHlcDDn@PQj~bo?zrteeC>0#ocwb$B}R^J=f3pjVVtt`$2VtG7p8E
z@`d`Di0QmK#DTNc9CYN{kVTi~+HA=8gkkzHo#Pn5&%>aG1H%MZTGMM|skfP#3Ekb@
z`*UW~%l^HHD2?m1ybPPE+kNzEU+YR=+P%<kQs`)XbMw(|k$ekXezh^-XZ4ntB~nY$
z+n<KXC2}5^QUy`ab?&b!*?+a}D2Rj;_V*sz?Naau34(cpKAnb*Dj8Juv0?5_4ZKc<
zpoBuefYK61vcmdTl{@_LZUUXr)*KQ9j5pd2Jkj0vv~{19T}VJ0*RzN%8<+VuVMj^R
z7NqIH7gT=KE$?XAP8g1BSnOeVpQ*@JT1R_lrqjbg2ZmjsNOC#=V+8-J@9OC3QBzWl
z3&k;H(K9?uM_^jNrL8Ry5(1`Y<ng`hC>SMyr3C_n!hds<Al!Jtk1}3b0ZicoIUUyH
zK2S;RjAjZ&N=ho*x@`wOClAX~UGW_^u^TO+x#M0fc;Am{Mc~eb*`F6hum3oB@M1A1
zj2<yFv;EcENKOpaSTXRGcnoFT%c;|amb9C};$muH;e)^h^;L8dans-#lJb=3`N#Jn
zQzYSv8q9*rs_NdD5V(h=N_I&L+I;E$x5G?MA1CZS&b}fqpQU$5d8WwC`QHLw@xeg8
z7$re0%~Fg#DmwZ(@_Xp%WHf8<A+P==Mn}n6PCzwNBw{4vW60lhc)v_bOUuOU+l*Mu
zxL-mF!wOVlqF3Zrx18oOIsXSkvnXN?)@4%aCSSqQnrjo%<+02NBt7^yNRh>PrCpl|
zwc)u?#8MRZHoDvQG$*+H7r2h;9{19()};lApq3*h1a|v{8NT8Ns!&o=BJq{$MEAO`
zJ)&hb+5->wsXy~vfAdzK6w9QV7OlEEMeQrS>@2cf9yDr-Dad@?wVb8k*A=kKl^WR;
zc_{|$FoW5aG(KQBY=J3Y-N$e#pKoSNdmm0b5a@dLf3@}9;as-i-(M6enH9-M5>d7a
z5kh5DB9$T|87V>$a?4g($w(SVg^ZN4vnoOeO*12-Y!Tu8Ts^<v`~LHG9M5q)&lBHq
z-}iN0=lNM@a7XeUwL41D9)q~n>!YeE8VxngTWFZ4)*cJZg)9OJ21Xcn^PIZq(M2$X
z`0J}L59o|iuAqtW=PU0p8cYQY*CfJ^UI6N|LAe~>N7>oz5G2D*C_IWqCJcKkPUkM=
zEq#hVeZ_@YqxFV)Lm9E)8#2NS(pz>Qu`ScA_-YKF>E0~rJp!5towZR^v>S*(MVyS)
z3~R`xrQk0s6K^!hqUigO@p!Wp#Gi0TsGp2%Vb=6lIP<ae+qZRqz|`R#)BQ;=I@9xB
zdzi)NLyuWV<Q&*DxPK>W&jP{zTyQ}Ge&4Cprn$fS8-*c{CbAadB0|r=$W3u`msD7`
z`-n`JT)qOiZsuj}UE9vmpML*jv<Wh}C%120iq}pqBWz4~Ra)+=h1nJOZsc7FMuV$8
zAh}<zd=uV2|J4HhhYvGhzL%nmGiYq#%T)l6b7SR6@$)@yv3~;R0&l4HNe*KP@#y8(
z)@;>ST$pdnppI{cLIZBOppopj<v1L<C!XsGjAygdL|i|GS}byZ!j7@83crv6qK3Vm
zPM9+YdTK&KK#(Z%;#}KRv_c4Pnc!bzsE*^$^7oPh(rgn>g42IkQ?&8$ds>vt+Z{?>
zC1LzT+FY~z<$qx`yCq|Vu`h6mQu}+9H$J?9dhGR1XQ}k96{Fn{%);PysdpYX4GM+|
z++`M?Xla#8<4XvwgkJBV-Ys9W;>K$FJ)f1_7|a7ISO9H#=AlbQm>uw@X8WlXul88K
zevKN-2Xx%Ts?Cc)xuw1Tx|7WVO(9&HyYP^EP}o={Pd^dgNTG-J!zss!O(Oe0#m2%+
zHye?!;K#}Tfexp`Vq7P~xEuLfW$?u_Kpp1rC1)u?hl5`Chjd02#_bv2l0Wel8a<qJ
zQ*tjwHg4=ht>on9_G7bn7g;n=mBHk>7Y-D9SUXQs7cA1&meuFGNN!>2Z}f|L_$8t6
zZsl8eIa5TK(;!Rf&vhA9cnyhm7}zFa#>PH%YU!)A&1Ir)@AnK}o~4C_wdS&IbI!`C
z)$9Cw>HgIekI>N2Gp^235;F=&W`h8a7`{;>e^=fTDC}SoxH~4>pMH6>mTAdu4(er2
zsP>1nP%V3B=Wxh^^3Hd|uN%_Pa1c-G1V3km%r@sA5i1?;`*Thi+#5MqnXx7R$oX%c
z`g+6GOJ9LPxr%xpTJ9K;-@F>5)u!UnIZ-b51v&Sq)lr2=`ZE%b7NF-0+7m8NR$tF{
z&wF;W>f&D)<I(g$4)`(kHc?hnyX=+l>7fah){B>42Dv5?sx$&TFWGI$qBkn+7Sp=+
zvdDD2w6m0?<Qps;X$b#`UJC-x(;>xymcnk@Gm|%4b4|8BI9wzqaSGz-Qiv4K!>UVU
z!bltFRNff4$9GZxhu8Z%EiT+80xCX_#X1$VO5ah*N}|&_z9)$WFf8nJ$Q=%EXBe2@
z59Y79m#ffFqH^$c=r-QahhO>S$8c~!!5_2wuccY(L+c%rJmF859+zHfXwK$YQjt2A
z_N{6j`s0EdMNp<|u!UVTk^8=Jro&Z6bb8eTbAdh~hP5cP@6T>@%QYb@H}Swd)lusA
zZ{j=E{aL#Lw83usn|Iqbb1%Hi*M7{}BuhLBkV(Qw8vuo%IXZN&pi1VOozKGJq6!&P
z&OhssWU&;?AAx%U(4W)Ai*G=xH=uaIm0O}W`f9+kJ<(*FbF6x6^6Np3<i=px{2zUN
zmM~NCsCdYu9DL15DH}@q3Ld(8sptJeYk#c_<5$1+4huntR8F1*D89*)@aNAMKc_IK
zuizV}yRC{(e!Ra?c(=7=rMoAR0S>3$g3>VV_U)m~V&|e){{?n?1^O-kb;m4i#5G5c
z1x@s{KVq}df2QcBvRRxyDfvpzDYX>(oE)7qMV~UL6-M_3@{iu|gii@n*%-(wi!$G(
zB-B7t@lNHa?&z^SwN0|;oc!>_YM{F0h@Bd`vbezQ{x?rP#KtDJy*=aGv+y%bjCo%9
zt#wz0@&zIHL{0w~jT|nR9}2BeeE37Yf~>)c`P;Rv#1j3N>CLceov^g@F?>Y%<mG*6
z^e}6`7A_;3HtHH55k(LToIHMLeUeaEfpoSCxs%Fiu+i;lW{icn-EW6ZeWvG+jnrOw
zueLXST&;qJm}s-G&)`gBb?7MJ2hP!_yhXbQ+ZrlJTiE?Ya_tyghyTusM>s2X4joTC
z9T3-Av)ua@6&FVeBsA1Nun)%nw*j*lE)~2yImqx-g!BHHBlmrizu^*>T@QZt>=~3B
zJS2e(ieL?Q_Z{dR|FB+5)jsNM<8L?PRN6S!J6+vddS89@xg5C4&_o|*`B_I~2r#&>
zmy?@FnMPXP@~tM%Nw@Db-b3vZi`Hq(cxB^%0IOFrc6Wxc%O|?@ACh{`iqdC{2kzCU
zS9&SaP+cN}Oq;E&n4CsfTpXO7zSU8pyIf8lE6ak$g7}$<+1hdGqZpotMfl>O*RSX4
z4?nqE=(KccaZz9)R#U-+fZMtU4(tJvHOc-teRH&qCZ(lCM+3ae!N<o!BieVAuOURO
zb`+sR-jH*{hUUz>!tfdT;cI^mb|0;@T={X7KhXWlH-_%FyFBTmPOuiuZ#!6~uYzOk
zz}=l!idEb$Od&{%s5;vQB{vXp=u^wr%SoAtQj!Qk(|f9T{IOd{Mn+P$QFDq36)$IE
zMx-mxzEAjI7bk0SHs#?d@T_W@38AepR3cp?kNeyw`&<Z41c+Q01fcQgH5ndk!N3O^
z?;gzxjE@$0oVM+(kEH3s%Plt^Pb%$O3dINm%|>{dm>w2kM_FMQY`qw>ccZk7#qBRV
z6&za5+(T8=9T7K`OCHBgMQut^uomw}=|IcdN9a26J{1yKD^qcFAAiY;5Yh(E4web=
zE%`2cS1P3UX0lZT&O07^@QO-_M-)vEVif1JY9X71hb8euKryk|z{u4P`{Je5Wo#0!
zcmBw{5*XS#r7kJ8b)Ph)`oSbuLr0Hd`Ojv+Ov9P04v^R9!vGuDwl>4F4h{f(uA<&X
zeLgfFR}45+NJ3&L)KdnBEJ?9KB!Sn$pC2BfS1io5M{i-F3#?&ua*e#EUvskqpr{j+
z5UsB$LD0P9XhwSu`JpeIIDR=fqnmvKxI~l0saGrUkU}4#j)wxu)#bp+QG^PjK@9@f
z0lxU|B^UQQ06aROOrUz_tv%%tdMNER<AiM~@uY|(qnsX^)CnhV_2>$Z;n!lPR}0;X
zij57zO(N=PoW4Y<OFWQpzR}dsDO}k12@XLJ0l>DXAdtw=ayotdXuO?QN<svFTwHv*
z`>ua1E*=Xpr{R&p-bzP^CFEGl$!*=bl6)WM-j-*3V*Xc5upC~7VDK|-Be&2^Av!zo
zbX|Pn;dEmJ#B!1EuyzWZO*b6OL7E%r=BY@zTY8Ke=s6OtshSw#=0bQ8u{l@Qd>oQv
z3DS&geXX>$n?bOd?&a;%{*sX7;4=}+ACbPGgHum#ECro~9#azmMKWMT5zNL1smbBH
zT(NsM*8kMlIud4-VSEj&r!clB;!cDj35k?MvIBQN2mw;S-ch115PLs~V8qpF>u*Kn
zf|x5Gq|87qy@ZJ=NW7ReFjah6<8Ax&@iXZ2k0M3+!)M7u26$e|KxPvOlz%_~QQAUO
z3kw{B(uKhXb4b-v?MRHdW|E<eS{1dI6NyADa_0b<MZ5253hN_rRziO(Q{O0Cy9k)U
zu{Dolc>p~~1ssz_zSw85MP<8<nEi$Mi#+1s?P2}%Q@uQE4wS#ZE^4q78ffAMfGsZ+
z!g3*8sNvyZqQQqoyy4d$u4JA-CRC-b{*JFuo#|v~vuD=cL6`!NS>`xhtKL)ZwNYcr
z*f)z2akaegMB0W8nwpyCs3(bGlK9!59Fv1EKxqAXp9u|XtU$aTTFvspkrIiom!=1`
zGmNdsPa{W^s0whZb`?*zlRn~~5Zk?I!v^A{M}}M}`d2jV(%8a_up!uXl&Ex2T`~&S
z#kR?$8i`+Q7N|E8Uk%Gu7;sc8B$Pv<OibM8+KWPf(LQ-2dh!0GWhNYhB*`5z2XknO
z^1S}Y0w`~U#{{2ylz)SvU+{)}hR$<}8`FO*m}vTgd?aL;`$+M0=g`nKpeMLpWe^{2
z9rIj@{K*yga$SgWCLGK7!~}AhxQPZ4mR+x)x6L$sTWir~r<m4bD$_>_Kr(q<zAS@~
zV!|E~wHom<c6F7x*(S-NHznT`3w)2U@fy;O60ZliXRn|)Mpop7gxzG`=Z}=KCcLpL
zsd$`y;|3>Hdq<hOy}iA`R-Sk;;B$bIdWKcM8QP+2KyAz+0r;HnA{xK#>{T?j<7hMV
zpZ2^@JbX03YR4Nlp4HB%@<=fVj;$1lr@?RqBm%+k3Oali89I7*lvcxWsR?VNH1fc_
zkbaT$cmA}0sE8YDNXFmNMk;wvBjA^tH*c;N7hjHRByijCM=<2tq_x4Nvx0<^&}LAV
zRk|J3Q`i6;=3=Qj^x5^1MkhF~YDTr<Il3maJJ7r;LI|FDwWv+{GpwoFaS#?JB_~Is
zn8t1I1`v+g&4>$E0jLu_&Gga{na$cVdX&XGHy+35s2hkm6vCWRlo1y8;(z1PqW}#N
zTe7r*f)=i{)ATS3cFQz6oH`I6(bTXj;WH%gJhVnX9)zX%9cHOA(A8h$Sq`B2c7hul
z-cSpCE{8qa46V212L;eUB=^CU<LVL>=L6{2N&b$sZNqv*4K=U7tZEweawpU$NXYet
zh;)G|^xNmpooGEPA(@7XuF7r4^(vghE~;Cb9@Q^~IW@^8hPUpT?H5J^2rPuZS7}pb
z4r<3yWJf1~i~$8GF+BlG^(a6XD>#)<I563a9cJXPz%~qlbbkw#&JEpA7{D0gd{gZG
zB>n`AX_gAJzeRiSlKshz5eOznkBMGXdg<o=<)&TA*JF{tFA8hdo-jU{y%a>a1gc#<
zA1$$E%SKxBgMD0K8`2Yy3lRl`1q%ye$0QgQ8j*xE^>N|BzNo6UwmUfXrao>mq@FRC
zIPn}*Qg~)j!HE6?z<*jOddUhH0?@Q59K8oN;Wc!F28|~bLo|JHtP*D|pO*1?{KZm^
z0qf@rvY=yan;&xk*$KFRfBpVh^R(?{2Wn_3)uz|et^Ef9ZjhD+6&MTZ*9-_0_e-e)
zxU#~Xp$I;uIOLbwzJ#h3IH)ut`KCoyV+uc>48oIisWz9PU+V-Hp$3^5j=uYiwv#XD
z0Xd2iPX!<p5}*Xj_;cGvdWbbrGQB+@%^<rSB3vrqb-<0#!g_?u)6BEuHs4NncGLx$
zczOg>y@DykRxr>=TwB|%%vo7%O@uuL_3qTuU%+k{I;JdPqzdNjp@?qGD8F{D%;j0v
zK{(iKex)0U55i%s4=@L?GY1D5$N(Ln_|6~uHS25hfAsVm1es<FSh-*hp?=tqwGoxQ
zAKSa8CWcN3=RAGdIJJv{OciPy&+g07At4gLeU|tTH4!|gQ<V>`?IB&**qnYO{Ml~`
zTi%HR*JUV-BhL-OWMU}Q0#$?vs1+ILCfd$I%?mGvl+%y*{9a%wXg*J#Y=H)cJZZqF
zW<VBRxc$t`?Op`RnF<_pIK^?!P{HiF!wTS&V0J|um*Sa4kX<x{qD|YJ`veOmwiCM2
z6^3G{7HB>I^BHJ}+w(zD(G91LCZ74o*w{y&by<Z6Z1L+7McaKWXdEJL@B3QMn5#@f
zQt};|*@WDC@LkX#acuDXX6?vc@7O4HJL*MF4n2*+vBL>|7Kx#P*e)82J>4AI3Z_D*
zmxbu0XaMs>0_O~C0EZCT4;izA^Gu88iZQ!abn&wt%`#KjKNKr}_UkH|6CCKpFHN^;
zP$j=|Qm^5Jq|t~I33i<CSaZae+ib5;g6p%rU34S9Ikww`{fROyJXA;2LN945NM`Q#
zPzFuU-~Gj3P7WNo5ve!wYp{Y^oI6RzA|Q43KSw178o+Bp*joa&L;V{8qZJ*Qlo4B)
zfi{I+4Njy2$ZFaX@X^H-6b?`4LDy)AIwckPyTQTsEooX%$7yOpo1hUx#l&EY!`toc
zM+6WMu*3(jaLs(eJE9wcFg7PtPcJ3dvPbgGinRQyfTit4&%@X-N&qL=Qk!qzzCE||
zBlag$raZ7cgS5SZhdmJsleiku^{folo_KSSv_D87q@xl17~4W}ULFs(<cU&rxY87a
zQ^a-~r|c#P#)w3jwLOIjJ%HFWK_8+gCHLena7D+W5q*Li{+al}Aa;k>1k8NqiTp#i
z9Glkt`tpnk2Nl<U2Fs_rry#T}MH5IujpnyYM$$Zg!^1d~`z88lNXyaCIrh_+haqO|
ztcs+BQJ6Kfj9B=0W0rvXP{vF0JClg4A93R0;H;}Mb#NKh5XfgJh2|GQ7X%0x{qTyA
zw6TdbE15+$g=+i3GsX_Gd&^U~=uPqbX_mKRs(0VLA5{nuO<(=JaO`_lh(faclzB&e
zj{7^4c4Flub{&9e>poM3Zc-hI(i=8ijZ}V=ZoMPz$K7iunghLGCJ%ihhD_qo>gW)}
z4(Ap0pYxKKI!Q-@O`%VjPrDMuDp*xa19_|8%?EI0_Bjg;2{t4j8kIeXXA>RzaBEx{
zs4Ky~u-`n>iO`D<*^qzjd~7?V#!|_19eetA><49#e;`Gd70JYHj2qu%69A5dSDqQ)
zjXV$?UD7Q+VmAt*Z38C@dZE@$talq7*C=Z{UclMzbG8r&4>}s+ok%P}Jx2?lray{s
z3v4$eFyiT%Pm*oU`vX*H4;+1hKluJ{@ia_!m*f)k*QNrDC&ZD)YFq_pu7JdMQ9
z2k+rE;=m@Fsz-16WfiW8c}GDHM|bn%*y%hgkOf5Ug0LFW8=_{Z`Lb`ff+?Jn;8#LH
zN+l?|Fk%nsCCxoLJ{w`Tipaeiar7~>vF(A*g)+&t55N(7!jb80vn#RnLz*@EUU`HF
z_t8*m0e=RHo*IbD{H3aG6veeaIvbL&aC_TXU$jV&w{0q@JEd_R^#Mo$CIS!Rg>fZF
zepp+z0ZX!D(%xYZ0iYdPum9VZ#<8S{TX1TK`ozJ`?t{&}6j^1^r`X&}dN%+;5Hkyl
zCbn-UB+veZ_5v|r8d$-9SVL3$@JUh*LH@lZZ*BgkQjg))y`7!^G9(Z>Oq4i;Mu)-M
z{N2sXDCv5Xcb07iZby<bQAFLxLL|`s3x_YhP#7+u7#TG-MUA0sWgum1ilLn4+O4W(
zTwXyV5myIuhngTo0>44xCqcl50Xc>Gl>>Wv1xi-M3=Sz*lZ-W>5?_OmI{3=_GLkDw
zVTZ%ftpz;-{;ZF^mh_d$P(OH6{_$hSLQcP*7ZR|fTeF^_#pUgzloUz?X(S=IQSb~T
z!*hQ0g~M8;1jAn=>a{pO4WH!?Wxe+h2z4O`GEUBh^@OP@nQ;I?p%tXy1hD{X9ggzu
zS#Kzdl%~NrHw|LVE2ssC9eDO^oZpRC2EiIkuBH5~>J0J?#|t$c9!hnZzb18XbZ&%)
zO1KmH7Re|VFKWXA(g^xxW_I@Kf~F8KLLBM|NP8izA3`lqu)vPsLe2|NoZ0|@W>}*v
zXu%J}H?Gf&^ptd_QyUGan0)w{S=*)W{?tSL(w|@4^BmS%e*f<Dcji3l(Gai>kOY)9
zX-na5yB@}m{Ar|n0N(CZo{inGp%n=bIE@HMf<FX!#FmaCl3<mDW5SaWgnuUZB`b>5
z<f~2fT!z<H`K&th`99+o7hSzA^UW@E+hFf1?Qe{r`_~M*_WMB3zFta-4QLC=l)`RA
z0v>iE0Z7g6EOi?ivGV`IXq;SJ{2+Wn1^f1otU9_^=<vUz79$Q{l;ULg#EDnl{A7^;
zsDMxroi({p*9CV80D#kmg7u3LOlV2_kOpN;Acqc%o4Sm$=1V224%Qvf`aAuqw5Qr9
zh>4wx!QG=|(gHg)r41h?F>=z>P&JV{hy>)hkM}IY=|y5h!KnSZ_3kEqG!U>RqV8el
zT>(`^+dV+W1@mm2A)3=}(K;#V>rH-7qq7E5wn0K-1@RKq)@lOPB&n%~QfW7htDYRm
z5+ENL0cm>oRQXX<?wO$a>MZQ+s=vI9u*lmG-}C-UmC6<t+b@T*^iOZ#x|rQ`=C-fK
z60cD@BwUm}IaWH}>l0Fa=H3l>br61Kg?=1QXwV7CAd`$YSOM7=f0?v(ql>#BJdl%>
zQ<(rKE$Eu{_&~VvK)=>+fXtxn<{kV-fR<gVH5bLjMIu~VTlxMc!WAknd`3*>;wLZ+
z&T@qI1OU`HURQ1;k@V+mpTqm&1A@idZ(U*etP$r@?I9FNyLhp*y|}b0GV5|baGNpk
zTf~@+VFMr9GL6&m{9ZvPuR7rlR$u<i$LpqXr{csp4?A<$BzRIMC$$XadzD$vU7nwk
z7=bFg9Jp&XMB8NN1S4565H1VdZHL#y%AY^e#zgr7yCQA>odJ*sC$`uxrDW1~%}(&s
zOQFvokQ8$Ix^8J$)r($CHChhc8ZLEPgOIFVs8D>*IV&p$JXf$*l}X-9uLh4xb^iLb
z5@1ZO$^6&1_PHkHWW$YRyc%X<FJQrT<sRk?@zY9y&t3|9z?kpysFe-Gx&lUh(LyxG
zzhddGE&KF|8?9-ky(>#aQDFa#IV?|i{P4k4pKw+B_-$L=d}FIm5i|j3!u(gO?<mEv
zJ)~sqT^o3bT52HThQ0#Y9CSGC0;)?qrWv<6GJ5}6>f|OhZ3smW*@KZeS?qc&5rhh{
zI-xTdkh2W5Fw>B)ZbPF2!hH4spw_v?1ZAe@jwj<g>+dnW2~Bt6!nqBa9JW8Hn=hgf
zCzx|8q$CN3P6hO>S3cZ$=0TC)^Gpq0cjaX}{nuOuNz=6EH<>uoO7X3bQwCk+6+nx1
zWi_W{iI4vA@t2t;!Ztj8BAU^yL(QqeGj_Mu<z0~)6S;bDvX5!S>l-4awYN%-oTN)k
zEM0W$%?c)5DX1S@lsD*Eh<^fWnL%tTTW5cNa5$wpySF1!j%8x`mi(b(TP&Zh(zRgW
z*gCO-nZ_!}xMqu3+hr3v`&?cUDj<5#%a<s-aqxXGJ$M;oeC`{Gt7{UZm_%d;-?fN~
zWv^3myDy)=qS*Z??}ttr)85yUB?EoNvxn8OB6$>@HzL#m=KF<MNyh|~e6ti_2JAqC
z!iNGa-t9B0Zfk7h0K?X!eEpNoZeG5wDmR`4T|GU20Ag5H=fMkn+T<AXdF`7~P5GiF
z5JyJFi~n@lC-~hW_W(g{N$3=&aBv{|^v3oI=7|0qo~KXu$+avE*tF8l$(O6cqNsoH
zuo_#yCn)-YUv^O5>yYI87COn!{-ebmRXo~HG-F#sn8CL;oJ_W`Pzx9EII};ZNw(09
z;f^wcgt1Y<GE{sRIYDBu;-C)no}W6>oOW0@TwRtW`9M;<i`&JEj0X!0-W1a-i=Nc2
zpK2BSF2vV)|I;=J{S<oD`Bg_>W<F0Sb;)h2k=VYS;zr~C8gHMC;s_9gas3s%jAN3{
z@P7$vh_LHYzIh}*{LknWwhjwKYL`AXP;-?Z9m*<=lZm$PLq-<xwmryx1o}nhzo;<1
zlV;(@(*c1UI91{PYiV*aQ6ua=c{|Me52|G_?!PCwb@RbxgxVeLqAExzL~t-OcAJy0
z&dah?tTm|nR|k09amhM&=#WpQ=@rJ6I&2x4rz|`2&I`AD3~eP@+xT_RG8UTJuGMXj
z@Ir&(6(n>*QnD1L)wkgczkjdx_JEEQW!=L<CqN8l*yhf&i&aRU;A}yKnwOtXKrb&U
zNXWt4JKDnptrnaCp~&GPVPpV2KDn5)RPbEw*B-_u=qshZUK@O)HoO)Q&}-lW;~~)Q
z!M$21Z|ei-7jcwBkJOFp`H3e85FG)mNHc<Z2f&ZC-s%w9{4=>Gu!0UeQh)l<<&|^F
zJAGSuikGB|VS9N?qK<%Jmi?nyWLPS;8*Fm>$B1N|FVbW8b>iUT-X<YVze!)b?Yv3A
z)B<^)VatSL#1D9v1=LDX2?>?b*WE*gebbQWiUMuP_K~{*TO7(aU9a3bth7&mX&@U@
z=1HdCKR27MHCzUSk7pa@zys-qQ(x^`m`&Bm&9B(r7}`o_VP>SSxq)_Hh}XGET~BYo
zRz;30&z-}4->=DWlG}|d$_n2C=5al(d@tQ{Q!J_wwdlh9^vRM6ckK<F7A#hI<%btT
z9lseD$fO>YSTD;tI-cglrRk4ygCwQ7WBLB$pNtzLEz)ERHcb<_3&5vW(Aag&jph8q
z5ksC2hu#WaaqHN+_3vul<wuVl&I<ub%_6AENAfj+6+%#XVNaI~i=j%$x%d3EczHWt
zv?U1AOzpgN@5DaM4ZMkw3+n`Amt;Ss6F(M$+#C*EVuFSc^#owEAxcp#(87Q3P2p)E
zJL}<L+_kiF(y0E<v!3DahXyLWJ4@+^3m4ThN?=PoaQ5u@sCcUAeY_u@o}Hwdl%D>}
zw$QaH{|G^>xlejsAy^>7kO<^+A$N3bVCq`UTdn+jd?*|w;x4(-TZmq?WM>@XUk-{1
z{O>o%Zv5n12v4(7uOm5flz&ZyG~=dsif)}0_u+|&36k=G!uLe6$LbYpwx}g*2lA`y
zu21!_l-jZ+&E3INPmcxJKXlj@UM(&F)4z~rqQ_DZ9D`<iWc@qc3}az{^XMbW0eeS)
zfdNT`DgX{3$U-QB2N9Vn!1|N>mcfHRaIQ&skB^So`gSB*!{`!*%S4nhuw<x(&06sU
zawqZRbFW@al=cMg1QZHmDXKA0{lhZYs%evMFVaeSdT6<47(T?-`dFW)7v2+-Y1)yS
zeki2)@ndE|)HB`-e{2Z!OLbhH=6@hHbj>1;S8LV3(+p=;!KutfTL4{V?g^S38e8si
z_$Tn)x2X8=4l~YFZx=28VH%xga(2;8KlI|vk*>bQKR*NsY(@BTRN};wq{3J)eLXJ3
zGFUXKIve98i|iPyv9gXnV%%zKqNKF=YrA=w?_!$(C16(2-TZEO@YOUU%U#IrhH;h|
zLt^6Mc(A|&F9nL-P^O_Iw}hQBtL@l8SC_$9U_QF!n@gUUEVY&QF+Ls-?1ssEVJ#2?
z*R%80Q9e<I0r4%i@<*W|CmAJBiJ-YSGJHXf#V{je6IyvxxCoQ``(f_EA!;)o+CyX7
zc*%u<M=AT}?Ogh9o{Nv)&c?iqPJCu_HwZ!4_}KwP5a0<%=Zc}I<rUJtLTNq22+~Fn
zN?!^$3&>XY{I~*^8g?Ha8%NMAp9{I&2YsbzKFptA3GztaFXujx40s8{9gykYi9V5e
z`SLW&9u}&?$OfA|O&St8<hrO!Qol{IO)~mK^6!;zwjFC`zVq1Ce<8<DDp=={9pg&+
zb8X^of+#5o??jYyn2>~q%zzC?xZ#};$$XRvV2xg7n`kFD^6bn%b6<VQhK7}G3vN*C
zm8R&HE=tvTjqT%IFU8j1A?W^hhRXf*qqo-rB76`gj0CMfDTUlo<M1pWi?~4@*J^&6
zmzS6H`R|q7l9{}-<=+pVc+Iu*3Y(XXqnIQk%pi4!lDsROrX{nTdnE)pkzj;4mrmHe
zcfh?21#Q*GEPS+cNp)+WMfB>kOv3yt4CZ9vtx!zx6WKyFvefA~9(~5>KHB+uMXL3?
z_Uk)UuC%0IsozBnR1hu(0)kRBq3DXpcFWZG?~7V;HPsMo9Zsp4PCvH$E%seKJ(F)3
z?gE(~>nvk9Wn+^JRT>;{?(6WiV<W82tU#3sVs(d-Qh8(JSm>`i6^b*ZOq6Vu&!j8n
z=jrb0t(PcN?HoC2Vt?OlBsOf=#o3wUt%CtzBKcpSKkHdr8vJm@X=*5e+{64WSoRp@
zTHk$ycZWR>-Fq@RVIpq}W=6{E9_$Mcw-MtV;PnmQ#^79+dfv!XdYD!(6?n7yl#{C~
zg-4T#0B3Mu+$V%32s{*1bhL9$yV-rSX)AQ1e#dNGJT4?08~eUcS&1HSv78NkR_Zh-
zb|s>l1i$kcJH_nuNY*{aLo1`BqWnnQC*D%q$Y>Rf(2URqyao(O8XXsZmWr(gqHgzH
zn~_0j<4PfYml%AI@QE;Q7Fc8O6p}-eOwdDnWAO5v=?-OOf@A+rW)ITuz>+~JjEhd#
z6-3EDc=(VQbxE`)`VaDjYisqelio*d(?4@@&yzU^{;<7aof+#c;Ct#kvg#TCuxtLW
z4wpv<RU(xoF|ioT3|93u+_8-r$F~#c3fZX&oo?A4iXdGHX9@-adnReV@nx3MNX7wV
z-F}!6h)0bOu&%I9{|H_iD7^6yYLQ7WAcyb+%V@|3XhOV!9i)=PlOrCjGjbU{Z7hmT
zZAiZ<A$TZ!H&?x`3$Wa&5<=UsXHU}WYautHYFf-tQxWS2*mh#qk>?F<x5``wBAU!?
zBJy3@V~odw{C5C*gw)P$7)6ZpFcl!i5+7_T8Zagh)@pYTO9elgj0|I4z}3kRk={5V
zDGKW$Q9TP&d%_N$KmW9>8#q`pTJxr#j4N%W)y^KzO@fFx5<OPOOe-{~#CZVdG#pTw
zm5*&Ycc}JmJZqR$x1KmA0M#DC)WJI{o-bO@BK|5$BXRxl<HzA@d<fVdc3CmV7+BFZ
zRIuwivtj-W)+js89t0?bY0!q?B04MvP)oq!!Hx?S1%Ckf%J8b4bWbo&Pa#|mdY?aY
zbFW-otBXwR?ZqJ;8K0c=>T^rn>G(JcL)P{}72o{mfFS0y8UY$VB(};Gh?t7@d+9)t
znH~WS4t3CsQ16ooFs=xFE!pjnF*!-O&LUZY%zy*>3qQa22mO~jJtl*pov(UmA-)%u
z5;##qc@^N;x?Y7uL&SNB!K^uQnV2R2^A$uOz5$T?L7;w^L(^X@71(vdJjxSQ29W4b
zoQyCh=#K82zZ5RG>mk%2{&=a!cr;h>@#!J|>OC?ZtcJ~k|5r?)^KatGP6MJA#J6<5
zaDhzp0;tJ2S4pPT;(J2GxQBSN5)u$`a1fISe_WQkuF5Q%4~{$vFa_4+f6kFdIzX&-
z02PqwK9CDM04Y+koAWsm!)zYz9sHO<>B1@j5r<&ACF;NpPywUbt=C)H`{T#N8m@|>
zkj@)(5Sj0jjE#Z|3X+xy{xCCQRY2M)df!GLI)<aZR0yCvTyP@@-)tW4&O16~bSx`g
zfb-iUorkB{^_-8Sw}8Xa#3x4It7VcAGZHZ=i5LK!gVY{qkqFCEQqk4c#4SsA5Q~0a
z;=u;!M`l=mh5j?iq}+fg`ti#Zg=lbm_YR9kUnL$*pv#l6=#TTWJ|Y9N0jE47(3Db3
z=_6F}@km7c^7(TF%*EvH!AY$-7ijk4&^yF=pfo17uT^W-yrDqqu6*n140R6%g>RIF
z${)(ok_wq%Y$4;ne{X^m+hb-_sH2NV^GjbMRxKyZ9s;`DZ##`%+)Bd@Ug8711QrJ4
z+XGtKC)^JVqedgPAtgwlI}&-C(fB?F|4u~y;#;=#DD%DFsgi~ev6j#e>4iDeC#=_B
z_G1J79D-(1bHpEk0Z@>7R@(3s{lT6OJ83*TmX}L*H>QKPYrr*2g?O;Ob`Kb~QU3Tc
z80L1mm?mPKJ$LHTD8NJ00b+;Gv04R@A81fX5QA61dS;~7lTqJzjUMHj;+MQdLLg{e
z3ii|h=R`;1vNzZ|apB2GLxhj0XEIx+v9;frL({DRYR%!;dX7+tA9)q*H@-8Cz!pb*
z2kL%|m>jIA@+zhSk*HV=^C)Er+b>+SaT=6R6t4a}GDp2!rFOvjKoSkHaq)K~-qc0e
z2n)~$3cR2z=*0TF`p6N<aQ;SeFyI_(fWFo+@kjBKuyCQ-j5HwgjfhVr-k*Z0dH996
zjM_G8C%Eq$1|P_QQ5QoO9(Vxd<i!z9x1QDvh;0ya2Xh@e?qXUJ_#u5{Wf~3qhDJ2w
z*t0;O`ntrkf3A%xXhg>_xn>c^Y;{|kX~WcM$Stbv+M;2+X+oWJA;rBz6HX>9f@-Lw
zPs!eDpKGztg-U&sJ%AwWR)+AP*aGiLII-$*dP{-b*8CIu0j7~B{z&GDM$eOab>^cz
zW?;LZ7f%G#-e3F+);xMOM2R*~Z4Y7N00FQO=YTyJjMs199!dN4|0l0kD?v}pPE$Jn
z-M{^7bI=coO$B**5y*sreZcAC$xBhn1_MiV0Zyh3HZGAGI)Y^cHq#C(O5XFg1M%X}
zo&!4)pPJf)sqwghy7d%`yM3C3lvH?=_pQO#oz?RYJ7bgFaeV7&S|zOXI)DLs1_q9$
zh8H0UCJ{Z_kZ|P@^z~f&w`2pFRNtjP`IwLGnxraVg9z~V=Xvn)lGY7;l6q~yAMbaz
zjR4=UgM*>Il|ue1y&6EJ!^7!2-$@=i0BJdOD}D;rOU8kF9NYawr@^fm;%0${y%0iT
zD3{znbdMLaX2owhiCaMznlL@41VvV2XQw%Yrb4u4xOHJ<gOTfW!w&z5R#;JW$+*fs
zZya|RxQJ1=2E8TX;#dmgY(#<Mxby5)i(vP?!zAUSc#aw_L^^LlB}~0!;Sr!<_%})U
z!J7-yz#nVjUy|_voj<g@2g-C@8<qvZ-vfTyaOhNi3cUS;3Pj!UfJ0cj0p)`oR6EVE
z{n+#+)!dlFzmU<?po?IpxxN4H&Qwo5oORgL<9A+?B@d?uH9f44Mm+1+^d;<zppfmr
z)OmR2t`>cMk6T!e@xZWvfB^0ku897k`kjP?D%8dxW_M%Lr<1Pw>Le7t<lds$c9>p3
zTLeFzj1R@KR&Z=T;YJ;Ez6ZjBf0uUx(F?(Z0KZw9Cs!QCCF42jzuN@O5Jhacq6h94
z<#hZAz~O=XOQf$1{mL~CIH+nTQ+{f+=bW|m>w&Q5?ek0-0I*v!jvpcKkLieX_kmlu
zeB1*a>c0arG?auAAzDl*?$qJFKO<8x-jktinnp@&zq-2Ao%8>FC`!oyn0q{pOr_JJ
zG&TE;4OA8{z^HrV_nbVuTA<nscKB~nQi{h*sRPL`m@q%1ld-9<w?{e~2s^;8nua!d
z8qO<Be+|URJNjYqHWbJVG(dEUL%aXWyC?HMNth-iVi0o#KU<jWf>KNyA`0)BE+)YA
zNuYu4sodFsw<uo*7gW(TdLccX%$f#B*a+FziNcE+A7r+e74hISdV>m9qKz>Y+<y?k
zG!gF$RlX_$qpkt|K^Z;4&s)KjnyCC7Vn$=*n8d=RG%~*oz&n1sFJ*Cib#3hu8qu?U
zd<!I$gO~TJ_dGRASQ%Jx2)Bf4U2*7H8afVtuzFzd7)bht{in?NH$Tuf(9=LZfknB3
z<z6(YZ<yRz^fUu(cq!0(#HJp8?YG?r@~$l=&zT8w%gn5XUMTzQw{75l#_-{U{Y2eR
z_hDkex`N}ttE^zdrfi!Yt~`xlc%ut7$v@{T9UYO{0sx#~dZ=f}95})*!;NYCB}W88
zv!{qTz}DvUk+KHYrj~NZ0(Xu^%P?L+Tc319Rdn;_Wwg#8KY|FKiVg_-qGz8BnK^;8
z`Q*20l&VAoiUME7We%&IlK87MnTk7s5KJw?fI+=O$W7w(z$1=ho1LA#^qc4@Z5|fL
zjzEAR?Ce(5i<%oqCu)>?`L?o>MBHkk*g{Qd)MXc(!*OBXz7TKv%K_-&(F9<QLR6FY
zJ1~-JcrXyATY-yDzJ=Fl8B-hu5Ew2oKhV<2fYBI3Qfh0w1Bu%Of)Kp>C_Fo8<`~5I
z8==_Qy&;k&Hm~^Q>sR04Z2J~*5=9EFxtOG6AYo#mqJ&(Qq=JCdCv6`F1(o2bhNxXV
z)rf?Z6>O*5K)5UmBzHoG=)6d+yhM!Cs4D_do)Mx3&1v~v6;F~ZT-D3CI8w`eD^c{%
z5fo0?&pb}PxbXYuF>D_>!y3on6Mc*{DKZohfaN0`;79ELSe`hMaBlMhsHRXRU^e0t
zGb<@M^pV&vyB_=+_c)f{j?={u>-Wdy#W`x^6)2f4ZS_thpA7a#9DAa$L=vO0DT0kj
z4%f$zALSvQ!}%&hHWO%7AA?^dSv*j>uECJ9g<C6+xDB@oA@&O_z7W#QJ)op>WeO;I
z(t&pV5WFX5mxMk>XqEhr+xLlsw4o68w)5wy^GkdH9G~gWT&kaPC=x`|<*Ak<UO`C{
zX@1*2d)QZ)GkHrNF4OJG#wBtth)gXqQxNx@9U91OJV6q`5VTn3a`;R+$0d!jutBRy
zEse?e^h11qK;n%dBASeXM}-|YY!*QV6M|@ggX-GIK{E*@C0-OLZz=frNVqFhI>KUN
zd%1TRD#9{_5C~xsBMl%3{nCc@>(#PrF0SS1?B+*yeFPMWwm1PvQ3Hl@O`B26M9YSu
zmh?R+r6DnS3oRtU0B|nh<ou&IwMSCtxU>SBqo~ZU&rkhOL73I_T_vI7*hZ-9fQZW7
z#G+#&BGmBvc-@M9ccuq>{&X{EWMq)cIb>MXy4mU}R2(N&>LdWes7H@}$bEZrCLS7+
z2&f@!rW#ic{ORsSJ&$r;BjOVW%hvzvet+7{yw<OCE2edOZ@GZY9);qv*(Dn?;eotH
z^5<}5!xq>cEg}c`9drRb-QBz4fmIls$wNZc@@30}0ARJ9_y1WJCbP;cN|~b(QbQ8a
z28r`)H!;Qw5CD@#@4&A4n<EEzOr_Z|XZ~K!T!ymhv;67`<#zHba0no^tPv;VnM&Sv
zSbfr90fp=IDWzT1PV{pvzv;iHOG-mF9y`(y$E(9$edqBceL9*x@>3?^PPiTWurJ9&
z$ZbEbbaGE}V*_MGxITN`yjsD{lRUbS2JtQ8;#aYq=|iJbaHtU57NcTVgoW82X}q3m
z8WyxKG+D;KVw%yZH6W#$(uPr5b$+smSHY6HlcTSUGkZ;IqEObM7pdte%?o%(9SpgJ
z<$ZOAH8oM8rr=vgLLQ1W96sVSiM;DbNHR1Lg-i0NM)||cRZNG~f-xC)?Bew}o}!+S
z+eDa$U7Jj{AQ_5)hshv92vj!%kD*8)w8;yHkffvyXn=7TM2uuFf!%)vG)UN2-rNw;
zMQxHIVOQwzMHaDRYj={AdIxBwv8sGAZXT+!F`x|u-$7?ca>2jds$u5p7?I3!ftra-
zVFf5fx?C;8ldyL*VIh&B;An#S=SHYkSx+a1dI5KAg)nP1DB?!Q6)(IzS$HuN^*sI?
zmhq;Hod&5!39rZhE*C`X5r6~E*7N=>6H31Ohc#1NTHx(l&&0$;?kT#kaEPX7Scg*Z
zW27u5T0aDROp@8($Z_tlSk?|?K^>b4X^E%;q)O=uPW)xu3_1ZlWR{ngHN2Qnhf+5Z
z5uWh`G8P6h*aBr%m5ENvBKMNSfe@!PglO1X9^#+nmYQBugNJ9)XaM64{GXt$vqqbl
zP)%S>!4L?ID5^|@{JU=WqIeg$v}bCS#=m3!#PYZIjngkAyZ*nb9rf7ibJr~VHRY>|
z_0Xq4-9=I@wD6>W+Oxx;3(|M;Iut)Q$#@80o@8_=oc(%}76B}6D~#O`R@z4JR7xU*
z57;G1bU;>Hj83jWbfp(O5yi#o(%0F?4vil<IZPqy90jr!G_L^DeT>Alh>aPFGuOf9
zWoV6o_J^i<{XGi(3(>7Z`h!YCc;`+&wD^dCp$lXa48sje2Gtayv()XfNd!qVCHdqC
z+OEX1gB`3!{N*i#V!=bPe#Z_TY}D#NrbwPU4r8hU#e>>FelnH{+4W%eq{EFMgHTZT
z`V|&>Mq8Ql2yt9+rZDP&m%Mla21y_yOB=_QJnu?i8Jz<I2!LV(N1X_{7zy}>JP)W9
zp0I=PfTLtXvC)lFcdxGQZaf0R`$oD3ebW!J88L)YI`w=cXA7!wBC7!kIXa)C6%b66
z3|<IW9G#e`Ac|;f>y%Hb(3xVp{&!Hy%df(kBhltGb3oC=9u1UD#`@hV6027%B-dh`
zyKlH&LR=gn@bo149sCXg&Kcd&%tMladES2Dk{(k=;&{NyK7Z!H%Ry<4Rksu&6J6iG
z`vxToexJ;4it}=lU9cJxBx82wb2KF|S_DTv;lKdBuS4MF`#C*b-3suaBwCb#MhJQs
z$WgC3w{Nc)&G@=$S{4uc(GoAkc5r;lce$}Mb}@&Q3jE%yzWh36QLFCX(t&9$GVp3b
zhHj1)rvGcM1k9;=xkoZ~MaAep!@MS=nS2qkGP?vu+Nlj(o==a|H@2|g{0aioh~tQi
z+T>KO;MtqIe;7ppy^oH_G_&S?_5i>EOpA@jPbjFv)b?=yovTPYCFyBksvVSoA3vt-
zYxXY56t7lcv5*dw(K$%J^?C4aKFLn{g8SD>DxO><@>+Z_m<>A7{Sd1P`Cy?H+QV5+
z%s~fzIon=^n)^o|O?~+}EqxcJ<KQXtKOwmXVjA6_EEQ4q_gc@iq%B1f_;MAgbzky)
zCF)xr`YvC5O-1@|yc~FxItvntP@s@cRsc#R0kSVHCRn%|{fhFm*8U~xMDNed&9P7D
zb@+^dVB%p3G_L*V>`7o;-R;G!Vo$cK<2&beESMVae|IQ4D(Hb`<VNE&t45`H4+KBB
z`{B9x4S8|KIt#P#Xz@_(GrJuU?7LG#cpDyAJ)ceTG1Q~DuCZ*BPP($;v$kflkx<ut
zi5m5zpD(xX;+qb&xngtdV!o=}^$Jy|kpSv%UjDrw1tab_i9#?2IirR%X}3$i)2s8{
zj8J-=`900w&2wkiML8)_d!hDTYKn2_IG+ZTc-WPJJ-|y@hjDtRhi>?@?{Lrigt^XL
zU5!7f5nYjJZQc>jj4-9&PqT7UzNly!e9p06fjb0Bi%iB%%?b5k*WhR6A0|nztkafP
z^WU{FnQgZ^Exc}|(4onIhlP=JN`P^&bAez^Ex|7&UErHFs6wh{%|nCmA)dDlNw56}
zN<OV#_1@)fH=hkXSF>1C=HG1Uo4l~_?)V%2wvYHc`r2<eE&56c?2s7I6g_cX%P(*V
zXR}yi=H8d7?7u4#ZO8|JEqMD7)MgxVr%yN~N|=t7`Y8=E3;CN(Vwyui)1C9zpY3n_
zb@1|K6>zkt(EY*b>`S8lGK|j*<@mF+bv1Og-?%lmdB4S*>&n9vttVnPvyj7SIl3iB
zlWQ;P7w89MA7CxsVtD-cx1UaKuUj~O`dEur30_RUxaz}MasT-nP88nEa^1Jl<ZcHK
zk+z2#pLYeB#n>T}@xr4A+-6%&CF-osrC`0sAjWH0<LIhVHt%Cr?rrEgCwr^s%}Kex
zqYh2b0NLqqmQ!JhlR4M@=ei@$84B@*@2+pHJN=B$=Pxs(-xV||=X<Imw<&)!Vz#)9
z>o4m<?HJg$y8OK2<nxcGWUFiE=^o(k-u5+A+ibgS-RI4%_{}IdtSqs@G}QW*Ht!|B
z?Ny>DQ_>s&NCsn<j!H<-19XkH5fk*d%GDKJ^QiRmyIX=IO@HRhbx+~Ng2P@XR<yy}
zrDfMvGIYB3<(r(QMRB}ol9hA+BWvJ!GQ7uMiq`&e`eaGfaN~+-zQq?C{{H>Lpwp|%
zwn4>pttsSX>Pk4%Z#6T!c^;1;Uu{WM?R?}<yJOn8zU(je<?Yho(pQ^fbEELm8h;}$
zKT&ud!P}L&`|ZUSP2t|+hf`F^U$oYBRTP8yI&9V7Ft+P2n@WG;p~HjZ^(y^#)aK~n
zE<?|cNmtI$=8@bT<ZA*N#Eyz-Pw%0blfGRC38OCvk8cG%#*xqP=4{CZCf|CS_W$jb
zqAbYySDNwTo8xxrTY8)HHbwOgVUzbL7PzzDCjHRyt%wE#mP>MmEiIGeMV1K-cS+aC
zf;V}}m-^}R=e@gkGiWD|g_(b{mB#>$+H-!U<oku!Sd_Qb)LcV>*!}b8-_oq}7cZV3
z@2Ng}G};5G0nhexd;k<hR8{ZZLI3w+*6XN{f46g-6BC)ehCoiXqI2@RIMN&|t~R4`
zQ%55V1t{9Unr-wN!F!w^I|%}3#3-pm!$)%+q8|rb!Fj$sf0zx4U272rHR4GgWA?4R
z2fJ9akq|KVz8%10S65eE`>UNH8@J^cWQ-4EY-3$dH5K*eOTG6Sj7}NOJ1xyaNKmL0
zFi#1ScPhiZCxz3Sh5Vg%vT_gOIELYML54T9@9r6?qEU@!Z|8iaW)kXK#C{2Y+kAi_
zAW_woFpBI&ZoE_aH&@t85|l20xM2CBt;XEwfENLp2d~fM-Q7xho4ByW)KLOizRSHv
z4bpdBLs^L-d+JtJ!keN(ptA}?aBcvqdL!~(GWw+J^A`^yI$XtL@_uIK`|+=a2UN&V
zka(4kw%T%WWe!%i?WYvv<w-;dS{ARjoG^TH;SEUuIJAk{VWnIe&*LiUTDv}mHeqcY
z6S(Gjk-B&qEzefo8eX|+w51(fKb?k-sy+Ksi?cxFGv4=m1g;?@=eq9O6cl7Y1Fs29
zzL*D*bo{RQfdgz_rvQTx@){<R!=49(G~ae-cM0(D>;-E>LO(=B0a<?fDWeycSGKaH
zyv<f#7NhQh1iqe?y(P`VQcEQ$P9)a<Tp?6hv7`2l#q={&*{CtJam%NsrcUHL2Ve_D
zd?_8$`k(&f{Q`QBh65LgW`x*$7QXgb`r4CThf{3;vVz}hdn!^sA6Iu8EFJOFAw4;*
z=p}oS)%mz}X_!C-O=BPmeW=$ViS7fSy^6xnKiONhjo|}ynJ%US(M{J%OlLXS*<mi9
z2ZCb`aAgT2WBYVu%^Z52^+-{?^6We$j7efBw-Ds0je{1c6|xZZ)@2=MZtVJEowqJj
z8(3pCL|s+Y)k2tEhNpu@YD&nVw+JHc35);(H1<Gq#;D23O5jZJEf*XuH_ZBU{LMBV
z793Xxj~&}*Z7qU`c(1u%FUg1_G=s$bkR%3E2`8{^P<4<Ao<h_cd`2CHS=Fa3!K|S;
z{2eymJHk2{d(!*XJvg~PJlW`Pt5CRX8u_kxBKGO(3ImNTC{O}0zgCHI6+^R$<->(~
z6eCw!&ZK5#J-UCtbo`nKJ~`CKtF$ZtiTVcy%A&X<375v{z2VefVKg1Ib4v@pIJ^P%
zz-{J!1(i6da1hGbGPFj+mG75QD}ss?QK%uO5s($!kGgm&Q0U^tUZnpzm!6hZMO9UG
z0we$1Ex6d(vAD1uG#jed$hf$@SO>VM$N>s~*b6Ug|7V2;vrzNLVe$~6#VJQUj)JxV
z`wYezk}Munn3JXYWnXWToJafscD+Bat`5LHh~5M1?k32T4<CD%3GtmMT;}FiR{G*B
z^^F|%9>sS7F^}Pgxda3j8ul>#Qpcz}bOOZEmsn5`kB>m8Gjs`OP2<_i)xYd)UCGD*
zj{P)5&v@QYH&-HlM+?BJ!?Bpd&7VHGjCQI+F;a?rG&C0GkcV-OYcRKb+0wdNiwypH
zaw7l6?5qa`(5f6gCQ8oqt?%p%Pd~c_2=V*(?>pRviOT8}r1s-Cud{0?Z2NZy(QPVT
z8sA7bE=kEu{kzU~j<u)d;4enL%vCtIi08o0gj3^jQXTiRX=J5CrG;s6WFLYV^zm*+
z<<_rD$KgIfXio&bGsS_~Y#6$X%HoBjKC{J-uMU0f4Gn7eU+^x!Opu(+u6UxH1Xzt|
zLt<mO;rZ(VPy6D+J`HB&mQ7vWn2F^*cI_lFpg=kNb39`31?u~9Byo^6=19%012*>Z
z-;bY~a(V?XI8f_7S)W|MMksbte`K0lTF8e$cZ1iq0D%8Oav!sW%@f{X0Q(yOD<EeS
zZIMu&a$Q4nb0yaBUSu{43;O{x`c|+}$e;hINuI=2lE{SQeNjBzz!g67I(9c)*A^xj
zS-Tehuz}di-K{z&<)6)(uCg@>U=JE0phr}UxuXzN%4Fo-{zn9n-TLjtQRYUWV<N4u
zFxnfYhxgDau2sJHM4>(DHLQ#~AW{Olb~i)!(iXtH;H`qN0o(usdpuUyKby<=Sj?qg
zxu2ju--u}?z;tqh9G^la6bQ&-0>eSaYvt;lhFcYn$4-8Z*-9OMb8>O^2=SFez?V_O
zq4C@$t-dwuM1DEG#T#6{Zx>hpq@hK%!2$F_GFc1B9;s<*mS|I6NNO=#Fd4B;Lwx^z
zXs9O3drBfV$T;Bk?b{i;wj6+sKr=%dQkbF3q4AlnrD7hNdeP(u-4B=TmqO6ngB%^I
z3%(}uLXGcJJ;t?;9b1bQzi4i_(4KWgtH*O^bgD#4(=<RlY~-jP+KOy#Elh!c?<2Xf
z3P=1}G$kWN6Rar089H&>w?H`nxdy15_ks(fOp_ieN_U16KQ$vxP|Q_{pFrxk91qj1
zRtWK3r;?JgN@jKFZo-SK5OjEzmH~{vY1msgV|*7jyLW)CSF8P*%A(qSKBe{Evbk%3
zpJ8G2<HWqgC#HoTor^I{VHpdH5#+eZ{Zk%V2CGD*r4J5#%(!j2?e=OdgPT~pDNmmK
zjQrHR&9X9h&3vs!aF|w$^{t#7S$SS+C3YNq*HW0^4z7AFyY4QWYcyy(GkB*F7eER}
z3;?VckB3Vi+0%WXA<6t*TCrKUS(XjvSYYv!K=qp0`HIZc4B)a}r!%4uOu%HnPgC6p
z78c{R$C<l!7_FSfK?eRHBd1*zVUeq}grJz2S)|sH&L3~~TPmn9_8_2e1~p*mG(xL_
zc_Nz@th1OD<_FACI4f2EQImx<&lA}f?B(Yl^A6*~W3K_No7|U8BNrYP>^OiFP}Y-%
z7oV>i!RD9K-xOa~9YT{k;@$==c&W3}2c(Z<1cxxSb7un5upzyz!{#_%b;cfK<jdww
zXcUs{^Ez1XVDTvgEiFAIdmgwqIo5EV!@x-Nu7F4iJj-9&^uJu(koOBi7HHVI>9UbG
zkAseQL5b!b<=hjq;#pjMV-pkcE=hlPc?IrK$M&LB*WuP<A(+2=#N{Y%nl&CD9S{)F
zaNYExkv{^yEE-2v3d-MO-5Bd4?(=<Q#PN?8WRE)7Kcn%%YCd6B<k7W?&JQf*d2y>(
zVP<qM9vD0~R~v1O5Y7^fi>ERk9}i!{Dx?{b=1dir?&;%_`O=%44V~O5H&GcGgE|Yv
zi$8hrAc~_%a4Q=A^{Wm=NF1mw;kU$#LpFMl-ZVm4Ks6bQjdl!p+70Y^2dG&?l*~$Z
z!r%-%THv6-yAre9grKef?t<WLGBFo(xi9rAC=kpN)ieerWW0DN<?qCj7UW4GodKxJ
zcdSB~4h^Gaa5`XVqXK}QPN(yutiGMyX4rBuzp|BUHVmn`tB{0<AVK?~L;mc)5(fFx
z5M%Wa><#&BX`%VBQR9rbwTmk0MSf!q&aY}*%>%e%jkfaPf3qKjmJi3=xufslp@b7W
zo4T+W`}lp2yuyNl93XqDe@AztJUYB~B0Z~w-W-Z0bQdOO^>uY32$=$rh9xPy>*s}&
zm!5smL&pnvjKt%;klcf<=t)kmqoE-e89dz55)E%EBx1)E+R=y1w2gT0LF<Kj25|?C
zg>H^u*KUFPp>W2FA8uELYc2#L*W2)Q$-xl+Z!2XmRz5AYYu7zc0G#XA)c`ntGPZ_a
z1F|ffAU5DjaOS=H*vl;R)BiAA)&9MEAt_x7SbZpL%6ZHkK6qkE#o+@^EVZS6a)po*
zYx+YioNU06doy0Pm$$VQQ*Faz?E>g7u-f+H@k6=dv!swc>DZIIJS3_yGcycZoS})y
zJ4f${b!T3sv=JjEP=ORUvY>lIl(Eh6*p2?#a>k15=guJo!5kF<Qp;qi1uy$EXTE(u
z`|AT_O*+tE0b=P!tpkDGmw(5WeUuZ^Kl~Z%36LunC#MD?M2;8dnqt!8_vj;L7KkJ)
z`qOtCf7ygsh~u#${@I3S*9_C7N@g}-pGE?U41!p2B3Q3sgOE4^2k?!%ci*Fsa)Z4=
z?95M%U}L)`CnoprAq4!h0;>mc9XTIk2<W-(_BX0v)olqeF>@TdNNbVB&IAq@t=G#7
z$26Fk8v43eU%p@^N?HJOgkwc6lq_&hR<7~I*-x*qLX-#yj3FkZdl_jz8_o5^pPfHS
zzkGfhDqL8Ch_bL=WtxvXrPSgA$!jY%bFBGbWFvcPhjS0bjJ$>o5Y(Y&bAgK*y6`&W
z2s)VFWiKx~nj4au|Nh>tdXSl9FF{`f#T^-HgnhDUa4U;LFGd^wMB*w=Uo@7Ov0#N5
zRFcmWs%<qi?$6GeW*RVe7Nd>5Fd2xXPxSQlk)slmhbnPYt(BurUvFW(XY7|Gh6)wV
zf?7Om1k8kN085ojELKzV$<SS;;Y#}@+e-E0{rHqzgHfYFns?sK%?1Dkz}hQ-a^=9$
z-3~0qE3IBVR!=QNK^Yk)xC@T^wJ6GaVqg6gkvm;$j^h8~Tx<|~xoYm5(;}6)xG?=H
zhNiEp+l@q%3e?i~{`QEXAeX~Pm%HAxcGycn1|OL}Uk==Y%t6D0jiG1I%EuBhJ|k=?
zZhe1#+_S+gL}88A;FO}*ab}^f@%ZX(_+d05PyHRhy91|XDE;zfEoefL@&xX}+ne&9
znVD(8!$qRxAfT_STZv5+%_*ZFoexf|!NEZ!v);!fYs96a`XR<%GJz5P4Kn{7n;eA9
z<o^A*>-oOxrNA1E&uoRP({qhOQ>1A@1q%pT50?PPc?=nLiQ1*yVVetJ_`#u}Mthsw
z7`c5Hl=u-FgEpocGFlR^;_bb2s@V|yb^{EVciq1)%U>(*Z~KW)vNeER<MWK7n~*>I
xM<E<0?5umsO__!G4vKbCXHEUtHJC5V3$_$H@#LH{WT)ZJKCS(lnHuK4{|Br*kpKVy
From 1bb2308ff39594e081d0a1a5f6ae10752a105f57 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 10 Oct 2017 13:51:40 -0700
Subject: [PATCH 75/82] gan design with graph
---
doc/design/gan_api.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 2fb30432cb..f9bf5939f4 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -29,13 +29,13 @@ In our GAN design, we wrap it as a user-friendly easily customized python API to
<p align="center">
<img src="./test.dot.png" width = "50%" align="center"/><br/>
-The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers. d\_loss and g\_loss mared in red and green are the two targets we would like to run.
+Figure 1. The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers. d\_loss and g\_loss marked in red and green are the two targets we would like to run.
</p>
<p align="center">
<img src="./dcgan.png" width = "90%" align="center"/><br/>
-Photo borrowed from the original DC-GAN paper.
+Figure 2. Photo borrowed from the original DC-GAN paper.
</p>
## The Conditional-GAN might be a class.
From f185af8d7bbe45cf8029a5c3727e27a5d001c984 Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Tue, 10 Oct 2017 13:55:01 -0700
Subject: [PATCH 76/82] Complete parameter
---
python/paddle/v2/framework/graph.py | 50 ++++++++++++++++++-
.../v2/framework/tests/test_parameter.py | 27 ++++++++++
2 files changed, 75 insertions(+), 2 deletions(-)
create mode 100644 python/paddle/v2/framework/tests/test_parameter.py
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/graph.py
index ba14885462..0f0a2847e5 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/graph.py
@@ -1,13 +1,19 @@
import paddle.v2.framework.core as core
import collections
import numpy as np
+import copy
__all__ = ['Block', 'Variable', 'Program', 'Operator']
class Variable(object):
- def __init__(self, block, name=None, shape=None, dtype=None,
- lod_level=None):
+ def __init__(self,
+ block,
+ name=None,
+ shape=None,
+ dtype=None,
+ lod_level=None,
+ **kwargs):
self.block = block
if name is None:
@@ -144,6 +150,10 @@ class Block(object):
def create_var(self, *args, **kwargs):
return Variable(self, *args, **kwargs)
+ def create_parameter(self, *args, **kwargs):
+ global_block = self.program.global_block()
+ return Parameter(global_block, *args, **kwargs)
+
def append_op(self, *args, **kwargs):
op_desc = self.desc.append_op()
op = Operator(self, op_desc, *args, **kwargs)
@@ -190,5 +200,41 @@ class Program(object):
self.current_block_idx = self.current_block().parent_idx
+class Parameter(Variable):
+ def __init__(self, block, shape, dtype, **kwargs):
+ if shape is None or dtype is None:
+ raise ValueError("Parameter must set shape and dtype")
+ if len(shape) == 0:
+ raise ValueError("Parameter shape cannot be empty")
+
+ for each in shape:
+ if each < 0:
+ raise ValueError("Parameter shape should not be related with "
+ "batch-size")
+
+ Variable.__init__(self, block, shape=shape, dtype=dtype, **kwargs)
+ self.trainable = kwargs.get('trainable', True)
+ self.init_attr = kwargs.get('initialize_attr', {
+ 'type': 'uniform_random',
+ 'min': -1.0,
+ 'max': 1.0
+ })
+
+ self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})
+ self._append_initialize_ops_()
+
+ def _append_initialize_ops_(self):
+ attr = copy.deepcopy(self.init_attr)
+ op_type = attr.pop('type', None)
+ block = self.block
+ assert isinstance(block, Block)
+ shape = self.shape
+ attr['dims'] = shape
+ attr['data_type'] = int(self.data_type)
+ op = block.prepend_op(
+ type=op_type, inputs=None, outputs={'Out': [self]}, attrs=attr)
+ self.op = op
+
+
# program is a global instance.
g_program = Program.instance()
diff --git a/python/paddle/v2/framework/tests/test_parameter.py b/python/paddle/v2/framework/tests/test_parameter.py
new file mode 100644
index 0000000000..3b5d38f257
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_parameter.py
@@ -0,0 +1,27 @@
+import unittest
+from paddle.v2.framework.graph import g_program
+import paddle.v2.framework.core as core
+
+
+class TestParameter(unittest.TestCase):
+ def test_param(self):
+ b = g_program.create_block()
+ param = b.create_parameter(
+ name='fc.w',
+ shape=[784, 100],
+ dtype='float32',
+ initialize_attr={
+ 'type': 'uniform_random',
+ 'seed': 13,
+ 'min': -5.0,
+ 'max': 5.0
+ })
+ self.assertIsNotNone(param)
+ self.assertEqual('fc.w', param.name)
+ self.assertEqual((784, 100), param.shape)
+ self.assertEqual(core.DataType.FP32, param.data_type)
+ self.assertEqual(0, param.block.idx)
+
+
+if __name__ == '__main__':
+ unittest.main()
From e31cfcd2715feb16a5961020b9ae19c3e3013123 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 10 Oct 2017 14:06:29 -0700
Subject: [PATCH 77/82] gan
---
doc/design/gan_api.md | 22 +---------------------
1 file changed, 1 insertion(+), 21 deletions(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index f9bf5939f4..5764112f3c 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -6,32 +6,12 @@ It applies several important concepts in machine learning system design, includi
In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN (Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks [https://arxiv.org/abs/1511.06434]) as an example due to its good performance on image generation.
-| important building blocks | People in Charge | Required |
-|---------------------------|-------------------|----------|
-| convolution 2d (done) | Chengduo | Y |
-| cudnn conv 2d (missing) | Chengduo | N |
-| deconv 2d (missing) | Zhuoyuan, Zhihong | Y |
-| cudnn deconv 2d (missing) | Zhuoyuan, Zhihong | N |
-| batch norm (missing) | Zhuoyuan, Jiayi | Y |
-| cudnn batch norm (missing)| Zhuoyuan, Jiayi | N |
-| max-pooling (done) | ? | Y |
-| cudnn-max-pool (missing) | Chengduo | Y |
-| fc (done) | ? | Y |
-| softmax loss (done) | ? | Y |
-| reshape op (done) | ? | Y |
-| Dependency Engine (done) | Jiayi | Y * |
-| Python API (done) | Longfei, Jiayi | Y * |
-| Executor (done) | Tony | Y * |
-| Multi optimizer (woking) | Longfei | Y * |
-| Optimizer with any para | ? | Y * |
-| Concat op (done) | ? | N (Cond) |
-| Repmat op (done) | ? | N (Cond) |
-
<p align="center">
<img src="./test.dot.png" width = "50%" align="center"/><br/>
Figure 1. The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers. d\_loss and g\_loss marked in red and green are the two targets we would like to run.
</p>
+The operators, layers and functions required/optional to build a GAN demo is summarized in https://github.com/PaddlePaddle/Paddle/issues/4563.
<p align="center">
<img src="./dcgan.png" width = "90%" align="center"/><br/>
From 16b2f963b8a45175d2caaf1fcdc02df4cca8e6d3 Mon Sep 17 00:00:00 2001
From: zchen0211 <chenzhuoyuan07@gmail.com>
Date: Tue, 10 Oct 2017 14:08:45 -0700
Subject: [PATCH 78/82] gan
---
doc/design/gan_api.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/design/gan_api.md b/doc/design/gan_api.md
index 5764112f3c..fb41df8615 100644
--- a/doc/design/gan_api.md
+++ b/doc/design/gan_api.md
@@ -7,7 +7,7 @@ It applies several important concepts in machine learning system design, includi
In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN (Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks [https://arxiv.org/abs/1511.06434]) as an example due to its good performance on image generation.
<p align="center">
-<img src="./test.dot.png" width = "50%" align="center"/><br/>
+<img src="./test.dot.png" width = "35%" align="center"/><br/>
Figure 1. The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers. d\_loss and g\_loss marked in red and green are the two targets we would like to run.
</p>
From 843ed8e3207583a8009526cbdac43e31e917de66 Mon Sep 17 00:00:00 2001
From: Yan Chunwei <yanchunwei@outlook.com>
Date: Tue, 10 Oct 2017 18:08:33 -0400
Subject: [PATCH 79/82] dynamic recurrent op forward c++ implentation (#4597)
---
cmake/configure.cmake | 4 +
paddle/framework/operator.h | 6 +-
paddle/framework/tensor_array.h | 4 +-
paddle/operators/CMakeLists.txt | 1 +
paddle/operators/dynamic_recurrent_op.cc | 276 ++++++++++++++++++
paddle/operators/dynamic_recurrent_op.h | 158 ++++++++++
paddle/operators/dynamic_recurrent_op_test.cc | 222 ++++++++++++++
7 files changed, 666 insertions(+), 5 deletions(-)
create mode 100644 paddle/operators/dynamic_recurrent_op.cc
create mode 100644 paddle/operators/dynamic_recurrent_op.h
create mode 100644 paddle/operators/dynamic_recurrent_op_test.cc
diff --git a/cmake/configure.cmake b/cmake/configure.cmake
index c1c93e17fd..db8f5ab045 100644
--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@@ -24,6 +24,10 @@ if(WITH_DOUBLE)
add_definitions(-DPADDLE_TYPE_DOUBLE)
endif(WITH_DOUBLE)
+if(WITH_TESTING)
+ add_definitions(-DPADDLE_WITH_TESTING)
+endif(WITH_TESTING)
+
if(NOT WITH_TIMER)
add_definitions(-DPADDLE_DISABLE_TIMER)
endif(NOT WITH_TIMER)
diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h
index 1e9ace9987..15f80b5720 100644
--- a/paddle/framework/operator.h
+++ b/paddle/framework/operator.h
@@ -142,9 +142,9 @@ class OperatorBase {
// Macro for define a clone method.
// If you are writing an kernel operator, `Clone` will be defined when you
// register it. i.e. `Clone` method is not needed to define by yourself.
-#define DEFINE_OP_CLONE_METHOD(cls) \
- std::unique_ptr<OperatorBase> Clone() const final { \
- return std::unique_ptr<OperatorBase>(new cls(*this)); \
+#define DEFINE_OP_CLONE_METHOD(cls) \
+ std::unique_ptr<::paddle::framework::OperatorBase> Clone() const final { \
+ return std::unique_ptr<::paddle::framework::OperatorBase>(new cls(*this)); \
}
// Macro for define a default constructor for Operator.
diff --git a/paddle/framework/tensor_array.h b/paddle/framework/tensor_array.h
index 94a14c2df4..293da04997 100644
--- a/paddle/framework/tensor_array.h
+++ b/paddle/framework/tensor_array.h
@@ -87,12 +87,12 @@ class TensorArray {
LoDTensor Stack() const;
/*
- * Unpacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors.
+ * Unstacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors.
*/
void Unstack(const LoDTensor &source) const;
/*
- * Unpacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors,
+ * Unstacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors,
* with memory of tensors shared.
*/
void UnstackShared(const LoDTensor &source) const;
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index d132c1813e..7dae8fe2f9 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -133,3 +133,4 @@ cc_test(gather_test SRCS gather_test.cc DEPS tensor)
cc_test(net_op_test SRCS net_op_test.cc DEPS net_op)
cc_test(scatter_test SRCS scatter_test.cc DEPS tensor)
cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory)
+cc_test(dynamic_recurrent_op_test SRCS dynamic_recurrent_op_test.cc DEPS dynamic_recurrent_op recurrent_op tensor_array)
diff --git a/paddle/operators/dynamic_recurrent_op.cc b/paddle/operators/dynamic_recurrent_op.cc
new file mode 100644
index 0000000000..b919aef8fb
--- /dev/null
+++ b/paddle/operators/dynamic_recurrent_op.cc
@@ -0,0 +1,276 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve .
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/operators/dynamic_recurrent_op.h"
+
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+using framework::Scope;
+using framework::TensorArray;
+using framework::LoDTensor;
+using framework::Variable;
+
+namespace detail {
+
+inline void CreateVariables(Scope& scope,
+ const std::vector<std::string>& var_names) {
+ for (const auto& name : var_names) {
+ scope.NewVar(name);
+ }
+}
+
+} // namespace detail
+
+class DynamicRecurrentOpProtoAndCheckerMaker
+ : public framework::OpProtoAndCheckerMaker {
+ public:
+ DynamicRecurrentOpProtoAndCheckerMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ const auto& name = DynamicRecurrentOp::kArgName;
+ // inputs and outputs stored in proto
+ AddInput(name.inlinks,
+ "the inputs that need to be segmented for each step.")
+ .AsDuplicable();
+ AddInput(name.boot_memories, "variables to initialize memories.")
+ .AsDuplicable();
+
+ AddOutput(name.outlinks, "the outputs that need to concated for all steps.")
+ .AsDuplicable();
+ AddOutput(name.step_scopes, "step scopes");
+
+ // Attributes stored in AttributeMap
+ AddAttr<std::vector<std::string>>(name.pre_memories,
+ "names of pre-memories");
+ AddAttr<std::vector<std::string>>(name.memories, "names of memories");
+
+ AddComment("This is a RNN operator for varience-length sequences.");
+ }
+};
+
+void DynamicRecurrentOp::Run(const Scope& scope,
+ const platform::DeviceContext& dev_ctx) const {
+ cache_.Init(kArgName, *this, scope, &arg_);
+ SplitInputs();
+ CreateScopes();
+ WriteStepInputs();
+ InitStates();
+
+ // call stepnet in all the time steps
+ for (size_t step = 0; step < cache_.num_steps; step++) {
+ auto& step_scope = cache_.GetScope(step);
+ stepnet_->Run(step_scope, dev_ctx);
+ }
+
+ WriteStepOutputs();
+ ConcatOutputs();
+}
+
+void DynamicRecurrentOp::SplitInputs() const {
+ // TODO(superjom) make level a config
+ // TODO(superjom) check all the inputs has the same LoD
+ int level = 0;
+ const auto& inlinks = cache_.inlinks;
+ for (const auto& item : inlinks) {
+ const auto& var = item.second;
+ const auto& tensor = var->Get<LoDTensor>();
+ TensorArray& ta = step_inputs_[item.first];
+ dy_seq_metas_[item.first] =
+ ta.Unpack(tensor, level, true /*length_descend*/);
+
+ if (cache_.num_steps) {
+ PADDLE_ENFORCE_EQ(ta.size(), cache_.num_steps,
+ "inputs should have the same steps");
+ } else {
+ cache_.num_steps = ta.size();
+ }
+ }
+}
+
+void DynamicRecurrentOp::WriteStepInputs() const {
+ for (const auto& item : cache_.inlinks) {
+ auto ta_it = step_inputs_.find(item.first);
+ PADDLE_ENFORCE(ta_it != step_inputs_.end(),
+ "step_inputs_ not compatible with memory set");
+ TensorArray& ta = ta_it->second;
+ for (size_t step = 0; step < ta.size(); step++) {
+ auto tensor = ta.Read(step);
+ auto& step_scope = cache_.GetScope(step);
+ Variable* var = step_scope.FindVar(item.first);
+ if (var == nullptr) {
+ var = step_scope.NewVar(item.first);
+ }
+ var->GetMutable<LoDTensor>()->ShareDataWith<value_type>(tensor);
+ }
+ }
+}
+
+void DynamicRecurrentOp::WriteStepOutputs() const {
+ for (size_t step = 0; step < cache_.scopes->size(); step++) {
+ auto& scope = cache_.GetScope(step);
+ for (auto& item : step_outputs_) {
+ auto* var = scope.FindVar(item.first);
+ if (var == nullptr) {
+ var = scope.NewVar(item.first);
+ }
+ auto* tensor = var->GetMutable<LoDTensor>();
+ item.second.WriteShared(step, *tensor);
+ }
+ }
+}
+
+void DynamicRecurrentOp::CreateScopes() const {
+ PADDLE_ENFORCE_GT(cache_.num_steps, 0);
+ // resize scopes
+ size_t num_scopes_need_create = cache_.num_steps - cache_.scopes->size();
+ for (size_t i = 0; i < num_scopes_need_create; i++) {
+ cache_.scopes->emplace_back(&cache_.scope->NewScope());
+ }
+
+ // init temporary inputs
+ PADDLE_ENFORCE_NOT_NULL(stepnet_, "stepnet should be set first");
+ std::vector<std::string> memories;
+ std::vector<std::string> pre_memories;
+ std::transform(arg_.memories.begin(), arg_.memories.end(),
+ std::back_inserter(memories),
+ [](const rnn::MemoryAttr& m) { return m.var; });
+ std::transform(arg_.memories.begin(), arg_.memories.end(),
+ std::back_inserter(pre_memories),
+ [](const rnn::MemoryAttr& m) { return m.pre_var; });
+
+ for (size_t step = 0; step < cache_.num_steps; step++) {
+ auto& scope = cache_.GetScope(step);
+ detail::CreateVariables(scope, arg_.inlinks);
+ detail::CreateVariables(scope, arg_.outlinks);
+ detail::CreateVariables(scope, memories);
+ detail::CreateVariables(scope, pre_memories);
+ }
+}
+
+void DynamicRecurrentOp::ConcatOutputs() const {
+ // TODO(superjom) transform this to a config
+ int level = 0;
+ // TODO(superjom) pass in some lod
+ // just a placeholder
+ framework::LoD lod;
+ for (auto& item : step_outputs_) {
+ auto tensor = item.second.Pack(level, dy_seq_metas_[item.first], lod);
+ auto& output = cache_.outlinks[item.first]->Get<LoDTensor>();
+ const_cast<LoDTensor*>(&output)->ShareDataWith<value_type>(tensor);
+ }
+}
+
+void DynamicRecurrentOp::InitStates() const {
+ // init the first state
+ // TODO(superjom) parepare the scenerio that boot state not exists
+ for (auto memory : arg_.memories) {
+ auto* boot_state_var = cache_.scope->FindVar(memory.boot_var);
+ PADDLE_ENFORCE_NOT_NULL(boot_state_var);
+ auto& boot_state = boot_state_var->Get<LoDTensor>();
+ const auto& dims = boot_state.dims();
+
+ for (size_t step = 0; step < cache_.num_steps; step++) {
+ auto& cur_scope = cache_.GetScope(step);
+ // link pre-state to boot_state
+ // init state and pre-state
+ auto* pre_state = cur_scope.FindVar(memory.pre_var);
+ PADDLE_ENFORCE_NOT_NULL(pre_state);
+ pre_state->GetMutable<LoDTensor>();
+
+ auto* state = cur_scope.FindVar(memory.var);
+ PADDLE_ENFORCE_NOT_NULL(state);
+ state->GetMutable<LoDTensor>()->Resize(dims);
+ state->GetMutable<LoDTensor>()->mutable_data<value_type>(
+ platform::CPUPlace());
+
+ if (step == 0) {
+ auto* pre_state_tensor = pre_state->GetMutable<LoDTensor>();
+ pre_state_tensor->Resize(boot_state.dims());
+ pre_state_tensor->ShareDataWith<value_type>(boot_state);
+ } else {
+ auto& pre_scope = cache_.GetScope(step - 1);
+ auto* state_pre = pre_scope.FindVar(memory.var);
+ PADDLE_ENFORCE_NOT_NULL(state_pre);
+ pre_state->GetMutable<LoDTensor>()->ShareDataWith<value_type>(
+ *state_pre->GetMutable<LoDTensor>());
+ }
+ }
+ }
+}
+
+void DynamicRecurrentOp::ArgCache::Init(
+ const rnn::ArgumentName& name, const paddle::framework::OperatorBase& op,
+ const paddle::framework::Scope& scope, rnn::Argument* arg) {
+ this->scope = &scope;
+ InitArgument(name, op, arg);
+ CacheScopes(scope, *arg);
+ CacheInlinks(scope, arg->inlinks);
+ CacheOutlinks(scope, arg->outlinks);
+}
+
+void DynamicRecurrentOp::ArgCache::InitArgument(const rnn::ArgumentName& name,
+ const OperatorBase& op,
+ rnn::Argument* arg) {
+ rnn::InitArgument(name, arg, op, false /*is_grad*/);
+}
+
+void DynamicRecurrentOp::ArgCache::CacheScopes(const Scope& scope,
+ const rnn::Argument& arg) {
+ auto scopes_var = scope.FindVar(arg.step_scopes);
+ PADDLE_ENFORCE(scopes_var != nullptr,
+ "the step_scopes output argument [%s] should be created first "
+ "by framework.",
+ arg.step_scopes);
+ this->scopes = scopes_var->GetMutable<std::vector<Scope*>>();
+}
+
+void DynamicRecurrentOp::ArgCache::CacheInlinks(
+ const Scope& scope, const std::vector<std::string>& names) {
+ for (auto name : names) {
+ auto* var = GetVariable(scope, name);
+ inlinks[name] = var;
+ }
+}
+
+void DynamicRecurrentOp::ArgCache::CacheOutlinks(
+ const Scope& scope, const std::vector<std::string>& names) {
+ for (auto name : names) {
+ auto* var = GetVariable(scope, name);
+ outlinks[name] = var;
+ }
+}
+
+Variable* DynamicRecurrentOp::ArgCache::GetVariable(const Scope& scope,
+ const std::string& name) {
+ auto* var = scope.FindVar(name);
+ PADDLE_ENFORCE_NOT_NULL(var, "variable [%s] not exist in scope", name);
+ return var;
+}
+
+const rnn::ArgumentName DynamicRecurrentOp::kArgName{
+ "step_net", "step_scopes", "inlinks", "outlinks",
+ "memories", "pre_memories", "boot_memories"};
+
+void DynamicRecurrentGradientOp::Run(
+ const Scope& scope, const platform::DeviceContext& dev_ctx) const {}
+
+} // namespace operators
+} // namespace paddle
+
+REGISTER_OP_WITHOUT_GRADIENT(
+ dynamic_recurrent, paddle::operators::DynamicRecurrentOp,
+ paddle::operators::DynamicRecurrentOpProtoAndCheckerMaker);
diff --git a/paddle/operators/dynamic_recurrent_op.h b/paddle/operators/dynamic_recurrent_op.h
new file mode 100644
index 0000000000..6a2970f27f
--- /dev/null
+++ b/paddle/operators/dynamic_recurrent_op.h
@@ -0,0 +1,158 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#pragma once
+
+#ifdef PADDLE_WITH_TESTING
+#include "gtest/gtest.h"
+#endif
+
+#include "paddle/framework/lod_tensor.h"
+#include "paddle/framework/operator.h"
+#include "paddle/framework/tensor_array.h"
+#include "paddle/framework/variable.h"
+#include "paddle/operators/rnn/recurrent_op_utils.h"
+
+namespace paddle {
+namespace operators {
+
+class DynamicRecurrentOp : public framework::OperatorBase {
+ public:
+ static const rnn::ArgumentName kArgName;
+ using value_type = float;
+
+ DynamicRecurrentOp(const std::string& type,
+ const framework::VariableNameMap& inputs,
+ const framework::VariableNameMap& outputs,
+ const framework::AttributeMap& attrs)
+ : OperatorBase(type, inputs, outputs, attrs) {}
+
+ DynamicRecurrentOp(const DynamicRecurrentOp& o)
+ : framework::OperatorBase(
+ static_cast<const framework::OperatorBase&>(o)) {
+ // TODO(yuyang18): Implement copy ctor well.
+ PADDLE_THROW("Not implemented");
+ }
+
+ void Run(const framework::Scope& scope,
+ const platform::DeviceContext& dev_ctx) const override;
+
+ /*
+ * Split the inputs(LoDTensors) to segments for each time step.
+ */
+ void SplitInputs() const;
+
+ /*
+ * Create step-scopes to store temporary outputs in each time steps.
+ */
+ void CreateScopes() const;
+
+ /*
+ * Link TensorArray steps to the corresponding variables located in
+ * step-scopes.
+ */
+ void WriteStepInputs() const;
+
+ /*
+ * Write output of each step to the corresponding TensorArray.
+ */
+ void WriteStepOutputs() const;
+
+ /*
+ * Initialize the states, each state will have a corresponding pre-state,
+ * which share the memory with the state in the previous time state. The
+ * pre-state in the first time step will be initialized with an zero tensor or
+ * a tensor in parent scope if is provided.
+ */
+ void InitStates() const;
+
+ /*
+ * Concatenate outputs in each time step and generate a LoDTensor.
+ */
+ void ConcatOutputs() const;
+
+ /*
+ * set a stepnet that is created according to a RecurrentOp's stepnet.
+ */
+ void SetStepNet(std::unique_ptr<OperatorBase> net) {
+ PADDLE_ENFORCE_NOT_NULL(net);
+ stepnet_ = std::move(net);
+ }
+ const OperatorBase& GetStepNet() const { return *stepnet_; }
+
+ protected:
+ struct ArgCache {
+ framework::Scope const* scope;
+ std::vector<framework::Scope*>* scopes;
+ std::map<std::string, framework::Variable*> inlinks;
+ std::map<std::string, framework::Variable*> outlinks;
+
+ size_t num_steps{0};
+
+ void Init(const rnn::ArgumentName& name, const OperatorBase& op,
+ const framework::Scope& scope, rnn::Argument* arg);
+
+ framework::Scope& GetScope(size_t index) {
+ PADDLE_ENFORCE_LT(index, num_steps);
+ return *scopes->at(index);
+ }
+
+ private:
+ void InitArgument(const rnn::ArgumentName& name, const OperatorBase& op,
+ rnn::Argument* arg);
+ void CacheScopes(const framework::Scope& scope, const rnn::Argument& arg);
+ void CacheInlinks(const framework::Scope& scope,
+ const std::vector<std::string>& names);
+ void CacheOutlinks(const framework::Scope& scope,
+ const std::vector<std::string>& names);
+ framework::Variable* GetVariable(const framework::Scope& scope,
+ const std::string& name);
+ };
+
+ private:
+ std::unique_ptr<OperatorBase> stepnet_;
+ mutable framework::TensorArray states_;
+ mutable std::map<std::string, framework::TensorArray> step_inputs_;
+ mutable std::map<std::string, framework::TensorArray> step_outputs_;
+ mutable std::map<std::string, std::vector<framework::DySeqMeta>>
+ dy_seq_metas_;
+ mutable rnn::Argument arg_;
+ mutable ArgCache cache_;
+
+#ifdef PADDLE_WITH_TESTING
+ friend class DynamicRecurrentOpTestHelper;
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, SplitInputs);
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, CreateCache);
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, CreateScopes);
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, WriteStepInputs);
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, WriteStepOutputs);
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, InitStates);
+ FRIEND_TEST(DynamicRecurrentOpTestHelper, ConcatOutputs);
+#endif
+};
+
+class DynamicRecurrentGradientOp : public framework::OperatorBase {
+ public:
+ DynamicRecurrentGradientOp(const std::string& type,
+ const framework::VariableNameMap& inputs,
+ const framework::VariableNameMap& outputs,
+ const framework::AttributeMap& attrs)
+ : OperatorBase(type, inputs, outputs, attrs) {}
+
+ void Run(const framework::Scope& scope,
+ const platform::DeviceContext& dev_ctx) const override;
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/dynamic_recurrent_op_test.cc b/paddle/operators/dynamic_recurrent_op_test.cc
new file mode 100644
index 0000000000..675a7890f3
--- /dev/null
+++ b/paddle/operators/dynamic_recurrent_op_test.cc
@@ -0,0 +1,222 @@
+#include "paddle/operators/dynamic_recurrent_op.h"
+
+#include <gtest/gtest.h>
+
+#include "paddle/framework/ddim.h"
+#include "paddle/framework/lod_tensor.h"
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/op_registry.h"
+#include "paddle/operators/net_op.h"
+
+namespace paddle {
+namespace operators {
+
+using framework::Scope;
+using framework::TensorArray;
+using framework::LoDTensor;
+using framework::Variable;
+
+class TestOp : public framework::OperatorBase {
+ public:
+ using framework::OperatorBase::OperatorBase;
+ DEFINE_OP_CLONE_METHOD(TestOp);
+ void Run(const Scope& scope,
+ const platform::DeviceContext& dev_ctx) const override {}
+};
+
+void OpDescNewVar(const std::string& param_name,
+ std::initializer_list<const char*> arguments,
+ paddle::framework::OpDesc::Var* var) {
+ var->set_parameter(param_name);
+ for (auto& arg_name : arguments) {
+ var->add_arguments(arg_name);
+ }
+}
+
+// create a LoD tensor in scope with specific dims
+LoDTensor* CreateVar(Scope& scope, std::string name, framework::DDim dims,
+ const platform::Place& place) {
+ auto* var = scope.NewVar(name);
+ auto* tensor = var->GetMutable<LoDTensor>();
+ tensor->Resize(dims);
+ tensor->mutable_data<float>(place);
+ return tensor;
+}
+
+class DynamicRecurrentOpTestHelper : public ::testing::Test {
+ protected:
+ const rnn::ArgumentName argname = DynamicRecurrentOp::kArgName;
+
+ virtual void SetUp() override {
+ CreateGlobalVariables();
+
+ auto op_desc = CreateOpDesc();
+ op = paddle::framework::OpRegistry::CreateOp(op_desc);
+ dop = dynamic_cast<DynamicRecurrentOp*>(op.get());
+ InitCacheManually();
+ InitStepNet();
+ }
+
+ framework::OpDesc CreateOpDesc() {
+ // create op
+ paddle::framework::OpDesc op_desc;
+ op_desc.set_type("dynamic_recurrent");
+
+ OpDescNewVar(argname.inlinks, {"in0"}, op_desc.add_inputs());
+ OpDescNewVar(argname.boot_memories, {"boot_mem"}, op_desc.add_inputs());
+ OpDescNewVar(argname.step_scopes, {"step_scopes"}, op_desc.add_outputs());
+ OpDescNewVar(argname.outlinks, {"out0"}, op_desc.add_outputs());
+
+ // set pre-memories
+ auto pre_memories = op_desc.mutable_attrs()->Add();
+ pre_memories->set_name(argname.pre_memories);
+ pre_memories->set_type(paddle::framework::AttrType::STRINGS);
+ auto pre_memories_item = pre_memories->add_strings();
+ *pre_memories_item = "mem@pre";
+
+ // set memories
+ auto memories = op_desc.mutable_attrs()->Add();
+ memories->set_name(argname.memories);
+ memories->set_type(paddle::framework::AttrType::STRINGS);
+ auto memories_item = memories->add_strings();
+ *memories_item = "mem";
+ return op_desc;
+ }
+
+ void CreateGlobalVariables() {
+ platform::CPUPlace place;
+ scope.NewVar("step_scopes");
+ CreateVar(scope, "boot_mem", framework::make_ddim({10, 20}), place);
+ // auto* out0 =
+ CreateVar(scope, "out0", framework::make_ddim({10, 20}), place);
+ auto* in0 = CreateVar(scope, "in0", framework::make_ddim({10, 8}), place);
+ // 10 instanes with 4 sentences, length is 4, 3, 2, 1 respectively.
+ framework::LoD in0_lod(1);
+ for (int x : std::vector<int>{0, 4, 7, 9, 10}) {
+ in0_lod[0].push_back(x);
+ }
+ in0->set_lod(in0_lod);
+ in0->Resize(framework::make_ddim({10, 8}));
+ // set the content, each sentence content is seqid.batchid
+ // the seqid starts from 0
+ int start = 0;
+ for (size_t seqid = 0; seqid < in0_lod.size() - 1; seqid++) {
+ for (size_t batchid = 0;
+ batchid < in0_lod[0][seqid + 1] - in0_lod[0][seqid]; batchid++) {
+ float v = seqid + batchid * 0.1;
+
+ for (size_t dim = 0; dim < 8; dim++) {
+ in0->data<float>()[start * 8 + dim] = v;
+ }
+ start++;
+ }
+ }
+ }
+
+ void InitCacheManually() {
+ dop->cache_.Init(DynamicRecurrentOp::kArgName, *dop, scope, &dop->arg_);
+ }
+
+ void InitStepNet() {
+ std::unique_ptr<framework::OperatorBase> stepnet{new NetOp};
+ dynamic_cast<NetOp*>(stepnet.get())
+ ->AppendOp(std::unique_ptr<TestOp>(new TestOp(
+ "test", {{"inlinks", {"in0"}}, {"boot_memories", {"boot_mem"}}},
+ {{"outlinks", {"out0"}}, {"step_scopes", {"step_scopes"}}}, {})));
+ dop->SetStepNet(std::move(stepnet));
+ }
+
+ protected:
+ DynamicRecurrentOp* dop;
+ std::unique_ptr<framework::OperatorBase> op;
+ paddle::platform::CPUDeviceContext device_context;
+ paddle::framework::Scope scope;
+};
+
+TEST_F(DynamicRecurrentOpTestHelper, CreateCache) {
+ const rnn::Argument& arg = dop->arg_;
+ ASSERT_EQ(arg.inlinks.size(), 1UL);
+ ASSERT_EQ(arg.outlinks.size(), 1UL);
+}
+
+TEST_F(DynamicRecurrentOpTestHelper, SplitInputs) {
+ dop->SplitInputs();
+ auto& in0_ta = dop->step_inputs_["in0"];
+ ASSERT_EQ(in0_ta.size(), 4UL);
+
+ const auto& batch0 = in0_ta.Read(0);
+ const auto& batch1 = in0_ta.Read(1);
+ const auto& batch2 = in0_ta.Read(2);
+ const auto& batch3 = in0_ta.Read(3);
+ EXPECT_EQ(batch0.dims()[0], 4);
+ EXPECT_EQ(batch1.dims()[0], 3);
+ EXPECT_EQ(batch2.dims()[0], 2);
+ EXPECT_EQ(batch3.dims()[0], 1);
+}
+
+TEST_F(DynamicRecurrentOpTestHelper, CreateScopes) {
+ dop->SplitInputs();
+ dop->CreateScopes();
+ ASSERT_EQ(dop->cache_.num_steps, 4UL);
+ ASSERT_EQ(dop->cache_.scopes->size(), 4UL);
+}
+
+TEST_F(DynamicRecurrentOpTestHelper, WriteStepInputs) {
+ dop->SplitInputs();
+ dop->CreateScopes();
+ dop->WriteStepInputs();
+
+ for (size_t step = 0; step < dop->cache_.num_steps; step++) {
+ auto& scope = dop->cache_.GetScope(step);
+ for (auto name : std::vector<std::string>({"in0"})) {
+ ASSERT_TRUE(scope.FindVar(name) != nullptr);
+ }
+ }
+}
+
+TEST_F(DynamicRecurrentOpTestHelper, WriteStepOutputs) {
+ dop->SplitInputs();
+ dop->CreateScopes();
+ dop->WriteStepInputs();
+ dop->WriteStepOutputs();
+
+ for (size_t step = 0; step < dop->cache_.num_steps; step++) {
+ auto& scope = dop->cache_.GetScope(step);
+ for (auto name : std::vector<std::string>({"out0"})) {
+ ASSERT_TRUE(scope.FindVar(name));
+ }
+ }
+}
+
+TEST_F(DynamicRecurrentOpTestHelper, ConcatOutputs) {
+ // Let's leave this test to python unittest.
+}
+
+TEST_F(DynamicRecurrentOpTestHelper, InitStates) {
+ dop->SplitInputs();
+ dop->CreateScopes();
+ dop->WriteStepInputs();
+ dop->WriteStepOutputs();
+ dop->InitStates();
+
+ for (size_t step = 0; step < dop->cache_.num_steps; step++) {
+ auto& scope = dop->cache_.GetScope(step);
+ auto state = scope.FindVar("mem");
+ ASSERT_TRUE(state != nullptr);
+
+ auto* pre_state = scope.FindVar("mem@pre");
+ ASSERT_TRUE(pre_state != nullptr);
+
+ auto* boot_state = scope.FindVar("boot_mem");
+ ASSERT_TRUE(boot_state != nullptr);
+
+ if (step == 0) {
+ // check pre_state is a reference of boot_state
+ ASSERT_EQ(boot_state->Get<LoDTensor>().data<float>(),
+ pre_state->Get<LoDTensor>().data<float>());
+ }
+ }
+}
+
+} // operators
+} // namespace paddle
From d1479d930eb2067ae08b63f3ad41706cc7265dc8 Mon Sep 17 00:00:00 2001
From: Abhinav Arora <abhinavarora28@gmail.com>
Date: Tue, 10 Oct 2017 16:57:50 -0700
Subject: [PATCH 80/82] Fixing errors in the refactorization doc (#4680)
---
doc/design/refactorization.md | 45 ++++++++++++++++++-----------------
1 file changed, 23 insertions(+), 22 deletions(-)
diff --git a/doc/design/refactorization.md b/doc/design/refactorization.md
index 629422e774..ec51aa1a0e 100644
--- a/doc/design/refactorization.md
+++ b/doc/design/refactorization.md
@@ -17,22 +17,22 @@ The goals of refactoring include:
1. A graph is composed of *variables* and *operators*.
-1. The description of graphs must be capable of being serialized/deserialized, so that:
+1. The description of graphs must be serializable/deserializable, so that:
- 1. It can to be sent to the cloud for distributed execution, and
+ 1. It can be sent to the cloud for distributed execution, and
1. It can be sent to clients for mobile or enterprise deployment.
-1. The Python program does the following steps
+1. The Python program does two things
- 1. *compilation*: run a Python program to generate a protobuf message representation of the graph and send it to
+ 1. *Compilation* runs a Python program to generate a protobuf message representation of the graph and send it to
1. the C++ library `libpaddle.so` for local execution,
1. the master process of a distributed training job for training, or
1. the server process of a Kubernetes serving job for distributed serving.
- 1. *execution*: execute the graph by constructing instances of class [`Variable`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/variable.h#L24) and [`OperatorBase`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L70), according to the protobuf message.
+ 1. *Execution* executes the graph by constructing instances of class [`Variable`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/variable.h#L24) and [`OperatorBase`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L70), according to the protobuf message.
## Description and Realization of Computation Graph
-At compile time, the Python program generates a protobuf message representation of the graph, or the description of the graph.
+At compile time, the Python program generates a protobuf message representation of the graph, or a description of the graph.
At runtime, the C++ program realizes the graph and runs it.
@@ -42,11 +42,11 @@ At runtime, the C++ program realizes the graph and runs it.
|Operation|[OpDesc](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L35)|[Operator](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L64)|
|Block|BlockDesc|Block|
-The word *graph* is interchangeable with *block* in this document. A graph represents computation steps and local variables similar to a C++/Java program block, or a pair of parentheses(`{` and `}`).
+The word *graph* is interchangeable with *block* in this document. A graph consists of computation steps and local variables similar to a C++/Java program block, or a pair of parentheses(`{` and `}`).
## Compilation and Execution
-1. Run an application Python program to describe the graph. In particular, the Python application program does the following:
+1. Run a Python program to describe the graph. In particular, the Python application program does the following:
1. Create `VarDesc` to represent local/intermediate variables,
1. Create operators and set attributes,
@@ -54,10 +54,10 @@ The word *graph* is interchangeable with *block* in this document. A graph repr
1. Infer the type and the shape of variables,
1. Plan memory-reuse for variables,
1. Generate the backward graph
- 1. Optimize the computation graph.
- 1. Potentially, split the graph for distributed training.
+ 1. Add optimization operators to the computation graph.
+ 1. Optionally, split the graph for distributed training.
-1. The invocation of `train` or [`infer`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/inference.py#L108) methods in the application Python program does the following:
+1. The invocation of `train` or [`infer`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/inference.py#L108) methods in the Python program does the following:
1. Create a new Scope instance in the [scope hierarchy](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/scope.md) for each run of a block,
1. realize local variables defined in the BlockDesc message in the new scope,
@@ -107,8 +107,8 @@ Compile Time -> IR -> Runtime
* `Operator` is the fundamental building block of the user interface.
- * Operator stores input/output variable names, and attributes.
- * The `InferShape` interface is used to infer the shape of the output variable shapes based on the shapes of the input variables.
+ * Operator stores input/output variable names and attributes.
+ * The `InferShape` interface is used to infer the shape of the output variables based on the shapes of the input variables.
* Use `Run` to compute the `output` variables from the `input` variables.
---
@@ -139,7 +139,7 @@ Compile Time -> IR -> Runtime
* Limit the number of `tensor.device(dev) = ` in your code.
* `thrust::transform` and `std::transform`.
* `thrust` has the same API as C++ standard library. Using `transform`, one can quickly implement customized element-wise kernels.
- * `thrust` also has more complex APIs, like `scan`, `reduce`, `reduce_by_key`.
+ * `thrust`, in addition, supports more complex APIs, like `scan`, `reduce`, `reduce_by_key`.
* Hand-writing `GPUKernel` and `CPU` code
* Do not write in header (`.h`) files. CPU Kernel should be in cpp source (`.cc`) and GPU kernels should be in cuda (`.cu`) files. (GCC cannot compile GPU code.)
---
@@ -185,10 +185,10 @@ Make sure the registration process is executed and linked.
1. Write an Op class and its gradient Op class, if required.
2. Write an Op maker class. In the constructor of this class, describe the inputs, outputs and attributes of the operator.
3. Invoke the macro `REGISTER_OP`. This macro will
- 1. Call maker class to complete the `proto` and the `checker`
+ 1. Call maker class to complete `proto` and `checker`
2. Using the completed `proto` and `checker`, it will add a new key-value pair to the `OpInfoMap`
-4. Invoke the `USE` macro in which the Op is used, to make sure that it is linked.
+4. Invoke the `USE` macro in which the Op is used to make sure that it is linked.
---
# Backward Module (1/2)
@@ -199,13 +199,14 @@ Make sure the registration process is executed and linked.
---
# Backward Module (2/2)
### Build Backward Network
-- **Input**: graph of forward operators
-- **Output**: graph of backward operators
+- **Input**: a graph of forward operators
+- **Output**: a graph of backward operators
- **Corner cases in construction**
- Shared Variables => insert an `Add` operator to combine gradients
- No Gradient => insert a `fill_zero_grad` operator
- Recursive NetOp => call `Backward` recursively
- RNN Op => recursively call `Backward` on stepnet
+ - RNN Op => recursively call `Backward` on stepnet
---
@@ -215,10 +216,10 @@ Make sure the registration process is executed and linked.
* Only dims and data pointers are stored in `Tensor`.
* All operations on `Tensor` are written in `Operator` or global functions.
* Variable length Tensor design [LoDTensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/lod_tensor.md)
-* `Variable` instances are the inputs and the outputs of an operator. Not just `Tensor`.
+* `Variable` instances are the inputs and the outputs of an operator, not just `Tensor`.
* `step_scopes` in RNN is a variable and not a tensor.
-* `Scope` is where variables are stores.
- * map<string `variable_name`, Variable>
+* `Scope` is where variables are stored.
+ * map<string `var name`, Variable>
* `Scope` has a hierarchical structure. The local scope can get variables from its parent scope.
---
@@ -246,7 +247,7 @@ Make sure the registration process is executed and linked.
---
# Control the migration quality
- Compare the performance of migrated models with old ones.
-- Follow the google C++ style
+- Follow the google C++ style guide.
- Build the automatic workflow of generating Python/C++ documentations.
- The documentation of layers and ops should be written inside the code.
- Take the documentation quality into account when submitting pull requests.
From 0a74fed181bea14e6b797e59261631556401d29b Mon Sep 17 00:00:00 2001
From: kavyasrinet <kavyasrinet@baidu.com>
Date: Tue, 10 Oct 2017 17:08:01 -0700
Subject: [PATCH 81/82] Correcting few mistakes in the block doc (#4681)
---
doc/design/block.md | 74 ++++++++++++++++++++++-----------------------
1 file changed, 36 insertions(+), 38 deletions(-)
diff --git a/doc/design/block.md b/doc/design/block.md
index 4d5dd4ba95..9c812732d6 100644
--- a/doc/design/block.md
+++ b/doc/design/block.md
@@ -5,12 +5,12 @@
Both deep learning systems and programming languages help users describe computation procedures. These systems use various representations of computation:
- Caffe, Torch, and Paddle: sequences of layers.
-- TensorFlow, Caffe2, Mxnet: graphs of operators.
+- TensorFlow, Caffe2, Mxnet: graph of operators.
- PaddlePaddle: nested blocks, like C++ and Java programs.
## Block in Programming Languages and Deep Learning
-In programming languages, a block is a pair of curly braces that includes local variables definitions and a sequence of instructions, or operators.
+In programming languages, a block is a pair of curly braces that includes local variables definitions and a sequence of instructions or operators.
Blocks work with control flow structures like `if`, `else`, and `for`, which have equivalents in deep learning:
@@ -24,14 +24,14 @@ A key difference is that a C++ program describes a one pass computation, whereas
## Stack Frames and the Scope Hierarchy
-The existence of the backward makes the execution of a block of traditional programs and PaddlePaddle different to each other:
+The existence of the backward pass makes the execution of a block of PaddlePaddle different from traditional programs:
-| programming languages | PaddlePaddle |
-|-----------------------|-------------------------------|
-| stack | scope hierarchy |
-| stack frame | scope |
-| push at entering block| push at entering block |
-| pop at leaving block | destroy at minibatch completes|
+| programming languages | PaddlePaddle |
+|-----------------------|---------------------------------|
+| stack | scope hierarchy |
+| stack frame | scope |
+| push at entering block| push at entering block |
+| pop at leaving block | destroy when minibatch completes|
1. In traditional programs:
@@ -42,9 +42,9 @@ The existence of the backward makes the execution of a block of traditional prog
1. In PaddlePaddle
- When the execution enters a block, PaddlePaddle adds a new scope, where it realizes variables.
- - PaddlePaddle doesn't pop a scope after the execution of the block because variables therein are to be used by the backward pass. So it has a stack forest known as a *scope hierarchy*.
+ - PaddlePaddle doesn't pop a scope after the execution of the block because variables therein are used by the backward pass. So it has a stack forest known as a *scope hierarchy*.
- The height of the highest tree is the maximum depth of nested blocks.
- - After the process of a minibatch, PaddlePaddle destroys the scope hierarchy.
+ - After the processing of a minibatch, PaddlePaddle destroys the scope hierarchy.
## Use Blocks in C++ and PaddlePaddle Programs
@@ -94,14 +94,14 @@ with ie.false_block():
o1, o2 = ie(cond)
```
-In both examples, the left branch computes `x+y` and `softmax(x+y)`, the right branch computes `x+1` and `fc(x)`.
+In both examples, the left branch computes `x+y` and `softmax(x+y)`, the right branch computes `fc(x)` and `x+1` .
-A difference is that variables in the C++ program contain scalar values, whereas those in the PaddlePaddle programs are mini-batches of instances. The `ie.input(true, 0)` invocation returns instances in the 0-th input, `x`, that corresponds to true values in `cond` as the local variable `x`, where `ie.input(false, 0)` returns instances corresponding to false values.
+The difference is that variables in the C++ program contain scalar values, whereas those in the PaddlePaddle programs are mini-batches of instances.
### Blocks with `for` and `RNNOp`
-The following RNN model from the [RNN design doc](./rnn.md)
+The following RNN model in PaddlePaddle from the [RNN design doc](./rnn.md) :
```python
x = sequence([10, 20, 30]) # shape=[None, 1]
@@ -112,9 +112,9 @@ U = var(0.375, param=true) # shape=[1]
rnn = pd.rnn()
with rnn.step():
h = rnn.memory(init = m)
- hh = rnn.previous_memory(h)
+ h_prev = rnn.previous_memory(h)
a = layer.fc(W, x)
- b = layer.fc(U, hh)
+ b = layer.fc(U, h_prev)
s = pd.add(a, b)
act = pd.sigmoid(s)
rnn.update_memory(h, act)
@@ -147,9 +147,9 @@ for (int i = 1; i <= sizeof(x)/sizeof(x[0]); ++i) {
## Compilation and Execution
-Like TensorFlow programs, a PaddlePaddle program is written in Python. The first part describes a neural network as a protobuf message, and the rest part executes the message for training or inference.
+Like TensorFlow, a PaddlePaddle program is written in Python. The first part describes a neural network as a protobuf message, and the rest executes the message for training or inference.
-The generation of this protobuf message is like what a compiler generates a binary executable file. The execution of the message that the OS executes the binary file.
+The generation of this protobuf message is similar to how a compiler generates a binary executable file. The execution of the message is similar to how the OS executes the binary file.
## The "Binary Executable File Format"
@@ -186,8 +186,8 @@ Also, the RNN operator in above example is serialized into a protobuf message of
```
OpDesc {
- inputs = {0} // the index of x
- outputs = {5, 3} // indices of act and hidden_out
+ inputs = {0} // the index of x in vars of BlockDesc above
+ outputs = {5, 3} // indices of act and hidden_out in vars of BlockDesc above
attrs {
"memories" : {1} // the index of h
"step_net" : <above step net>
@@ -203,14 +203,14 @@ This `OpDesc` value is in the `ops` field of the `BlockDesc` value representing
During the generation of the Protobuf message, the Block should store VarDesc (the Protobuf message which describes Variable) and OpDesc (the Protobuf message which describes Operator).
VarDesc in a block should have its name scope to avoid local variables affect parent block's name scope.
-Child block's name scopes should inherit the parent's so that OpDesc in child block can reference a VarDesc that stored in parent block. For example
+Child block's name scopes should inherit the parent's so that OpDesc in child block can reference a VarDesc that stored in parent block. For example:
```python
-a = pd.Varaible(shape=[20, 20])
+a = pd.Variable(shape=[20, 20])
b = pd.fc(a, params=["fc.w", "fc.b"])
rnn = pd.create_rnn()
-with rnn.stepnet()
+with rnn.stepnet():
x = a.as_step_input()
# reuse fc's parameter
fc_without_b = pd.get_variable("fc.w")
@@ -218,17 +218,17 @@ with rnn.stepnet()
out = rnn()
```
-the method `pd.get_variable` can help retrieve a Variable by a name, a Variable may store in a parent block, but might be retrieved in a child block, so block should have a variable scope that supports inheritance.
+The method `pd.get_variable` can help retrieve a Variable by the name. The Variable may be stored in a parent block, but might be retrieved in a child block, so block should have a variable scope that supports inheritance.
In compiler design, the symbol table is a data structure created and maintained by compilers to store information about the occurrence of various entities such as variable names, function names, classes, etc.
To store the definition of variables and operators, we define a C++ class `SymbolTable`, like the one used in compilers.
-`SymbolTable` can do the following stuff:
+`SymbolTable` can do the following:
- store the definitions (some names and attributes) of variables and operators,
-- to verify if a variable was declared,
-- to make it possible to implement type checking (offer Protobuf message pointers to `InferShape` handlers).
+- verify if a variable was declared,
+- make it possible to implement type checking (offer Protobuf message pointers to `InferShape` handlers).
```c++
@@ -240,19 +240,18 @@ class SymbolTable {
OpDesc* NewOp(const string& name="");
- // TODO determine whether name is generated by python or C++
- // currently assume that a unique name will be generated by C++ if the
- // argument name left default.
+ // TODO determine whether name is generated by python or C++.
+ // Currently assume that a unique name will be generated by C++ if the
+ // argument name is left default.
VarDesc* NewVar(const string& name="");
- // find a VarDesc by name, if recursive true, find parent's SymbolTable
+ // find a VarDesc by name, if recursive is true, find parent's SymbolTable
// recursively.
// this interface is introduced to support InferShape, find protobuf messages
// of variables and operators, pass pointers into InferShape.
- // operator
//
// NOTE maybe some C++ classes such as VarDescBuilder and OpDescBuilder should
- // be proposed and embedded into pybind to enable python operate on C++ pointers.
+ // be proposed and embedded into pybind to enable python operation on C++ pointers.
VarDesc* FindVar(const string& name, bool recursive=true);
OpDesc* FindOp(const string& name);
@@ -270,7 +269,7 @@ class SymbolTable {
After all the description of variables and operators is added into SymbolTable,
the block has enough information to run.
-The `Block` class takes a `BlockDesc` as input, and provide `Run` and `InferShape` functions.
+The `Block` class takes a `BlockDesc` as input, and provides `Run` and `InferShape` functions.
```c++
@@ -302,7 +301,7 @@ public:
void CreateVariables(const framework::Scope& scope);
void CreateOperators();
- // some other necessary interfaces of NetOp are list below
+ // some other necessary interfaces of NetOp are listed below
// ...
private:
@@ -316,15 +315,14 @@ private:
Block inherits from OperatorBase, which has a Run method.
Block's Run method will run its operators sequentially.
-There is another important interface called `Eval`, which take some arguments called targets, and generate a minimal graph which takes targets as the end points and creates a new Block,
-after `Run`, `Eval` will get the latest value and return the targets.
+There is another important interface called `Eval`, which takes some arguments called targets and generates a minimal graph which treats targets as the end points and creates a new Block. After `Run`, `Eval` will get the latest value and return the targets.
The definition of Eval is as follows:
```c++
// clean a block description by targets using the corresponding dependency graph.
// return a new BlockDesc with minimal number of operators.
-// NOTE not return a Block but the block's description so that this can be distributed
+// NOTE: The return type is not a Block but the block's description so that this can be distributed
// to a cluster.
BlockDesc Prune(const BlockDesc& desc, vector<string> targets);
From 6604d7cda295cc7978ff227a91a0128a497f14be Mon Sep 17 00:00:00 2001
From: Siddharth Goyal <vi.siddharth78@gmail.com>
Date: Tue, 10 Oct 2017 17:57:02 -0700
Subject: [PATCH 82/82] Add logsigmoid (numerically stable) and softshrink
(#4663)
* Add numerically-stable logsigmoid activation
* Add softshrink operator
* Adjust relative tolerance for grad-check
* Address review comments
---
paddle/operators/activation_op.cc | 35 ++++++++++
paddle/operators/activation_op.h | 70 ++++++++++++++++++-
.../v2/framework/tests/test_activation_op.py | 35 ++++++++++
3 files changed, 139 insertions(+), 1 deletion(-)
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index 92db629079..a6bb738af3 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -49,6 +49,18 @@ class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+class LogSigmoidOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ LogSigmoidOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of LogSigmoid operator");
+ AddOutput("Y", "Output of LogSigmoid operator");
+ AddComment(
+ "Logsigmoid activation operator, logsigmoid = log (1 / (1 + exp(-x)))");
+ }
+};
+
class ExpOpMaker : public framework::OpProtoAndCheckerMaker {
public:
ExpOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
@@ -85,6 +97,23 @@ class LeakyReluOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+template <typename AttrType>
+class SoftShrinkOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ SoftShrinkOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of Softshrink operator");
+ AddOutput("Y", "Output of Softshrink operator");
+ AddComment(
+ "Softshrink activation operator, "
+ "softshrink = x - lambda, if x > lambda;"
+ " x + lambda, if x < lambda; 0 otherwise");
+ AddAttr<AttrType>("lambda", "non-negative offset")
+ .SetDefault(static_cast<AttrType>(0.5f));
+ }
+};
+
class TanhOpMaker : public framework::OpProtoAndCheckerMaker {
public:
TanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
@@ -271,6 +300,9 @@ namespace ops = paddle::operators;
REGISTER_OP(sigmoid, ops::ActivationOp, ops::SigmoidOpMaker, sigmoid_grad,
ops::ActivationOpGrad);
+REGISTER_OP(logsigmoid, ops::ActivationOp, ops::LogSigmoidOpMaker,
+ logsigmoid_grad, ops::ActivationOpGrad);
+
REGISTER_OP(exp, ops::ActivationOp, ops::ExpOpMaker, exp_grad,
ops::ActivationOpGrad);
@@ -283,6 +315,9 @@ REGISTER_OP(tanh, ops::ActivationOp, ops::TanhOpMaker, tanh_grad,
REGISTER_OP(tanh_shrink, ops::ActivationOp, ops::TanhShrinkOpMaker,
tanh_shrink_grad, ops::ActivationOpGrad);
+REGISTER_OP(softshrink, ops::ActivationOp, ops::SoftShrinkOpMaker<float>,
+ softshrink_grad, ops::ActivationOpGrad);
+
REGISTER_OP(sqrt, ops::ActivationOp, ops::SqrtOpMaker, sqrt_grad,
ops::ActivationOpGrad);
diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h
index 123f0c4dbc..70d5a62052 100644
--- a/paddle/operators/activation_op.h
+++ b/paddle/operators/activation_op.h
@@ -95,6 +95,41 @@ struct SigmoidGradFunctor : public BaseActivationFunctor<T> {
}
};
+// Originally: logsigmoid(x) = -log (1 + exp(-x))
+// For numerical stability, we can use the log-sum-exp trick:
+// https://hips.seas.harvard.edu/blog/2013/01/09/computing-log-sum-exp/
+// We can rewrite the above equation as:
+// y = -log( exp(0) + exp(-x)) [since exp(0) = 1]
+// = -log( exp(max(-x, 0) - max(-x, 0)) + exp(-x + max(-x, 0) - max(-x, 0)))
+// = -log( exp(max(-x, 0)) * exp(-max(-x, 0)) - exp(max(-x, 0)) * exp(-x -
+// max(-x, 0)))
+// = -log( exp(max(-x, 0)) * (exp(-max(-x, 0)) + exp(-x - max(-x, 0))))
+// = -log( exp(max(-x, 0)) - log(exp(-max(-x, 0)) + exp(-x - max(-x, 0)))
+//
+// Hence, logsigmoid(x) = - (max(-x, 0) + log(exp(-max(-x, 0))
+// + exp(-x - max(-x, 0))))
+template <typename T>
+struct LogSigmoidFunctor : public BaseActivationFunctor<T> {
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) const {
+ auto temp = (-x).cwiseMax(static_cast<T>(0)); // temp = max(-x, 0)
+ y.device(d) = -temp - (((-temp).exp() + (-x - temp).exp()).log());
+ }
+};
+
+// Originally: f' = exp(-x) / (1 + exp(-x))
+// For numerical stability: f' = exp(-x - max(-x, 0)) / (exp(-max(-x, 0)) +
+// exp(-x - max(-x, 0)))
+template <typename T>
+struct LogSigmoidGradFunctor : public BaseActivationFunctor<T> {
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) const {
+ auto temp = (-x).cwiseMax(static_cast<T>(0)); // temp = max(-x, 0)
+ dx.device(d) =
+ dy * ((-x - temp).exp() / ((-temp).exp() + (-x - temp).exp()));
+ }
+};
+
// exp(x) = e^x
template <typename T>
struct ExpFunctor : public BaseActivationFunctor<T> {
@@ -164,6 +199,37 @@ struct TanhShrinkGradFunctor : public BaseActivationFunctor<T> {
}
};
+// softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < lambda; 0
+// otherwise
+template <typename T>
+struct SoftShrinkFunctor : public BaseActivationFunctor<T> {
+ float lambda;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"lambda", &lambda}};
+ }
+
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) const {
+ auto temp1 = (x > lambda).template cast<T>().eval();
+ auto temp2 = (x < -lambda).template cast<T>().eval();
+ y.device(d) = temp1 * (x - lambda) + temp2 * (x + lambda);
+ }
+};
+
+template <typename T>
+struct SoftShrinkGradFunctor : public BaseActivationFunctor<T> {
+ float lambda;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"lambda", &lambda}};
+ }
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) const {
+ auto temp1 = (x > lambda).template cast<T>().eval();
+ auto temp2 = (x < -lambda).template cast<T>().eval();
+ dx.device(d) = dy * (temp1 + temp2).template cast<T>();
+ }
+};
+
// sqrt(x) = x^(1/2)
template <typename T>
struct SqrtFunctor : public BaseActivationFunctor<T> {
@@ -471,9 +537,11 @@ struct STanhGradFunctor : public BaseActivationFunctor<T> {
#define FOR_EACH_KERNEL_FUNCTOR(__macro) \
__macro(sigmoid, SigmoidFunctor, SigmoidGradFunctor); \
+ __macro(logsigmoid, LogSigmoidFunctor, LogSigmoidGradFunctor); \
__macro(exp, ExpFunctor, ExpGradFunctor); \
__macro(relu, ReluFunctor, ReluGradFunctor); \
__macro(tanh, TanhFunctor, TanhGradFunctor); \
+ __macro(softshrink, SoftShrinkFunctor, SoftShrinkGradFunctor); \
__macro(sqrt, SqrtFunctor, SqrtGradFunctor); \
__macro(abs, AbsFunctor, AbsGradFunctor); \
__macro(reciprocal, ReciprocalFunctor, ReciprocalGradFunctor); \
@@ -484,7 +552,7 @@ struct STanhGradFunctor : public BaseActivationFunctor<T> {
__macro(pow, PowFunctor, PowGradFunctor); \
__macro(stanh, STanhFunctor, STanhGradFunctor); \
__macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \
- __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \
__macro(relu6, Relu6Functor, Relu6GradFunctor); \
+ __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \
__macro(tanh_shrink, TanhShrinkFunctor, TanhShrinkGradFunctor); \
__macro(elu, ELUFunctor, ELUGradFunctor)
diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py
index 4528ed555d..9157e00f6e 100644
--- a/python/paddle/v2/framework/tests/test_activation_op.py
+++ b/python/paddle/v2/framework/tests/test_activation_op.py
@@ -33,6 +33,21 @@ class TestSigmoid(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.008)
+class TestLogSigmoid(OpTest):
+ def setUp(self):
+ self.op_type = "logsigmoid"
+ self.inputs = {
+ 'X': np.random.uniform(-1, 1, [11, 17]).astype("float32")
+ }
+ self.outputs = {'Y': np.log(1 / (1 + np.exp(-self.inputs['X'])))}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=0.008)
+
+
class TestTanh(OpTest):
def setUp(self):
self.op_type = "tanh"
@@ -63,6 +78,26 @@ class TestTanhShrink(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.008)
+class TestSoftShrink(OpTest):
+ def setUp(self):
+ self.op_type = "softshrink"
+ lambda_val = 0.1
+ self.attrs = {'lambda': lambda_val}
+ self.inputs = {
+ 'X': np.random.uniform(0.25, 10, [4, 4]).astype("float32")
+ }
+ y = np.copy(self.inputs['X'])
+ y = (y < -lambda_val) * (y + lambda_val) + (y > lambda_val) * (
+ y - lambda_val)
+ self.outputs = {'Y': y}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=0.007)
+
+
class TestSqrt(OpTest):
def setUp(self):
self.op_type = "sqrt"