From 2bdd3e43fa86d614b8235eab55182e15ead98743 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 7 Dec 2017 09:09:32 +0000 Subject: [PATCH 01/35] Update the version of openblas. --- cmake/external/openblas.cmake | 8 ++------ paddle/math/tests/CMakeLists.txt | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 97857a686b..da25574793 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -30,23 +30,21 @@ IF(NOT ${CBLAS_FOUND}) CACHE FILEPATH "openblas library." FORCE) SET(OPENBLAS_CC "${CMAKE_C_COMPILER} -Wno-unused-but-set-variable -Wno-unused-variable") + SET(OPENBLAS_COMMIT "v0.2.20") IF(CMAKE_CROSSCOMPILING) SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER}) GET_FILENAME_COMPONENT(CROSS_SUFFIX ${CMAKE_C_COMPILER} DIRECTORY) SET(CROSS_SUFFIX ${CROSS_SUFFIX}/) IF(ANDROID) - # arm_soft_fp_abi branch of OpenBLAS to support softfp - # https://github.com/xianyi/OpenBLAS/tree/arm_soft_fp_abi - SET(OPENBLAS_COMMIT "b5c96fcfcdc82945502a2303116a64d89985daf5") IF(ANDROID_ABI MATCHES "^armeabi(-v7a)?$") + # use softfp SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV7 ARM_SOFTFP_ABI=1 USE_THREAD=0) ELSEIF(ANDROID_ABI STREQUAL "arm64-v8a") SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV8 BINARY=64 USE_THREAD=0) ENDIF() ELSEIF(IOS) IF(CMAKE_OSX_ARCHITECTURES MATCHES "arm64") - SET(OPENBLAS_COMMIT "b5c96fcfcdc82945502a2303116a64d89985daf5") SET(OPENBLAS_CC "${OPENBLAS_CC} ${CMAKE_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") SET(OPENBLAS_CC "${OPENBLAS_CC} -arch arm64") SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV8 BINARY=64 USE_THREAD=0 CROSS_SUFFIX=${CROSS_SUFFIX}) @@ -56,14 +54,12 @@ IF(NOT ${CBLAS_FOUND}) ENDIF() ELSEIF(RPI) # use hardfp - SET(OPENBLAS_COMMIT "v0.2.20") SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV7 USE_THREAD=0) ENDIF() ELSE() IF(APPLE) SET(OPENBLAS_CC "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}") ENDIF() - SET(OPENBLAS_COMMIT "v0.2.20") SET(OPTIONAL_ARGS "") IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^x86(_64)?$") SET(OPTIONAL_ARGS DYNAMIC_ARCH=1 NUM_THREADS=64) diff --git a/paddle/math/tests/CMakeLists.txt b/paddle/math/tests/CMakeLists.txt index 215bac1271..41c1bdf785 100644 --- a/paddle/math/tests/CMakeLists.txt +++ b/paddle/math/tests/CMakeLists.txt @@ -34,4 +34,6 @@ add_simple_unittest(test_FPException) add_simple_unittest(test_GpuProfiler) add_simple_unittest(test_BaseMatrix) add_simple_unittest(test_Matrix) -cc_test(test_float16 SRCS test_float16.cpp) +if(WITH_ARM_FP16) + cc_test(test_float16 SRCS test_float16.cpp) +endif() From d13d787bcdc324ec5d31a1043742ea31061c00dc Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 21 Dec 2017 15:24:42 +0800 Subject: [PATCH 02/35] Refine the cross-compiling documentations. --- doc/mobile/cross_compiling_for_android_cn.md | 27 +++++++++++++------- doc/mobile/cross_compiling_for_android_en.md | 20 ++++++++++----- doc/mobile/cross_compiling_for_ios_en.md | 6 ++--- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/doc/mobile/cross_compiling_for_android_cn.md b/doc/mobile/cross_compiling_for_android_cn.md index 424d7718c6..ae24ced770 100644 --- a/doc/mobile/cross_compiling_for_android_cn.md +++ b/doc/mobile/cross_compiling_for_android_cn.md @@ -1,8 +1,9 @@ # Android平台编译指南 用户可通过如下两种方式，交叉编译Android平台上适用的PaddlePaddle库： -- 基于Docker容器的编译方式 -- 基于Linux交叉编译环境的编译方式 + +- [基于Docker容器的编译方式](#基于docker容器的编译方式) +- [基于Linux交叉编译环境的编译方式](#基于linux交叉编译环境的编译方式) ## 基于Docker容器的编译方式 Docker能在所有主要操作系统（包括Linux，Mac OS X和Windows）上运行，因此，使用基于Docker容器的编译方式，用户可在自己熟悉的开发平台上编译Android平台上适用的PaddlePaddle库。 @@ -16,6 +17,12 @@ $ cd Paddle $ docker build -t username/paddle-android:dev . -f Dockerfile.android ``` +用户也可以使用PaddlePaddle提供的官方开发镜像： + +```bash +$ docker pull paddlepaddle/paddle:latest-dev-android +``` + ### 编译PaddlePaddle C-API库 构建好开发镜像后，即可使用开发镜像来编译Android版PaddlePaddle C-API库。 Android的Docker开发镜像向用户提供两个可配置的参数： @@ -41,23 +48,25 @@ Android的Docker开发镜像向用户提供两个可配置的参数： ANDROID_API - >= 21 + >= 16 21 - 编译`armeabi-v7a`，`Android API 21`的PaddlePaddle库 + ```bash $ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" username/paddle-android:dev ``` - 编译`arm64-v8a`，`Android API 21`的PaddlePaddle库 + ```bash $ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=arm64-v8a" -e "ANDROID_API=21" username/paddle-android:dev ``` -执行上述`docker run`命令时，容器默认执行[paddle/scripts/docker/build_android.sh](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/build_android.sh)脚本。该脚本中记录了交叉编译Android版PaddlePaddle库常用的CMake配置，并且会根据`ANDROID_ABI`和`ANDROID_API`自动构建独立工具链、进行编译和安装。由于arm64架构要求Android API不小于21。因此当`ANDROID_ABI=arm64-v8a`，`ANDROID_API<21`时，Docker容器中将默认使用`Android API 21`的编译工具链。用户可以参考下文**配置交叉编译参数**章节，根据个人的需求修改定制Docker容器所执行的脚本。编译安装结束之后，PaddlePaddle的C-API库将被安装到`$PWD/install_android`目录，所依赖的第三方库同时也被安装到`$PWD/install_android/third_party`目录。 +执行上述`docker run`命令时，容器默认执行[paddle/scripts/docker/build_android.sh](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/build_android.sh)脚本。该脚本中记录了交叉编译Android版PaddlePaddle库常用的CMake配置，并且会根据`ANDROID_ABI`和`ANDROID_API`自动构建独立工具链、进行编译和安装。由于arm64架构要求Android API不小于21。因此当`ANDROID_ABI=arm64-v8a`，`ANDROID_API<21`时，Docker容器中将默认使用`Android API 21`的编译工具链。用户可以参考下文[配置交叉编译参数](#配置交叉编译参数)章节，根据个人的需求修改定制Docker容器所执行的脚本。编译安装结束之后，PaddlePaddle的C-API库将被安装到`$PWD/install_android`目录，所依赖的第三方库同时也被安装到`$PWD/install_android/third_party`目录。 ## 基于Linux交叉编译环境的编译方式 本文档将以Linux x86-64平台为例，介绍交叉编译Android平台上适用的PaddlePaddle库的方法和步骤。 @@ -83,6 +92,7 @@ your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain 此命令将在`your/path/to/arm_standalone_toolchain`目录生成一套独立编译工具链，面向架构为32位ARM架构，支持的最小的Android API级别为21，支持编译器`arm-linux-androideabi-gcc (GCC) 4.9`和`clang 3.8`。 - 构建`arm64-v8a`、 `Android API 21`的独立工具链： + ```bash your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain.sh \ --arch=arm64 --platform=android-21 --install-dir=your/path/to/arm64_standalone_toolchain @@ -90,14 +100,12 @@ your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain 此命令将在`your/path/to/arm64_standalone_toolchain`目录生成一套独立编译工具链，面向架构为64位ARM64架构，支持的最小Android API级别为21，支持编译器`arm-linux-androideabi-gcc (GCC) 4.9`和`clang 3.8`。 -注意：**PaddlePaddle要求使用的编译工具链所支持的Android API级别不小于21**。 - ### 配置交叉编译参数 CMake系统对交叉编译提供了支持[cmake-toolchains](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling)。为了简化cmake配置，PaddlePaddle为交叉编译提供了工具链配置文档[cmake/cross_compiling/android.cmake](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/android.cmake)，以提供一些默认的编译器和编译参数相关配置。注意，从CMake 3.7版本开始，CMake官方对Android平台的交叉编译提供了通用的支持。PaddlePaddle若检测到用户使用的CMake版本不低于3.7时，将会将用户传进来的配置参数传递CMake系统，交由CMake系统本身来处理。有关参数配置的详细说明见[cmake-toolchains](https://cmake.org/cmake/help/v3.7/manual/cmake-toolchains.7.html#cross-compiling)。 交叉编译Android版本的PaddlePaddle库时，有一些必须配置的参数： -- `CMAKE_SYSTEM_NAME`，CMake编译的目标平台，必须设置为`Android`。在设置`CMAKE_SYSTEM_NAME=Android`后，PaddlePaddle的CMake系统才认为是在交叉编译Android系统的版本，并自动编译宿主机版protoc可执行文件、目标机版protobuf库、以及Android所需`arm_soft_fp_abi`分支的目标机版OpenBLAS库。此外，还会强制设置一些PaddlePaddle参数的值（`WITH_GPU=OFF`、`WITH_AVX=OFF`、`WITH_PYTHON=OFF`、`WITH_RDMA=OFF`）。 +- `CMAKE_SYSTEM_NAME`，CMake编译的目标平台，必须设置为`Android`。在设置`CMAKE_SYSTEM_NAME=Android`后，PaddlePaddle的CMake系统才认为是在交叉编译Android系统的版本，并自动编译PaddlePaddle所需的所有第三方库。此外，还会强制设置一些PaddlePaddle参数的值（`WITH_GPU=OFF`、`WITH_AVX=OFF`、`WITH_PYTHON=OFF`、`WITH_RDMA=OFF`、`WITH_MKL=OFF`、`WITH_GOLANG=OFF`）。 - `WITH_C_API`，必须设置为`ON`。在Android平台上只支持使用C-API来预测。 - `WITH_SWIG_PY`，必须设置为`OFF`。在Android平台上不支持通过swig调用来训练或者预测。 @@ -119,7 +127,7 @@ Android平台可选配置参数： 其他配置参数： - `USE_EIGEN_FOR_BLAS`，是否使用Eigen库进行矩阵计算。可设置`ON/OFF`，默认值为`OFF`。 -- `HOST_C/CXX_COMPILER`，宿主机的C/C++编译器。在编译宿主机版protoc可执行文件和目标机版OpenBLAS库时需要用到。默认设置成环境变量`CC`的值；若环境变量`CC`没有设置，则设置成`cc`编译器。 +- `HOST_C/CXX_COMPILER`，宿主机的C/C++编译器。在编译宿主机版protoc可执行文件和目标机版OpenBLAS库时需要用到。默认设置成环境变量`CC/CXX`的值；若环境变量`CC/CXX`没有设置，则设置成`cc/c++`编译器。 常用的cmake配置如下： @@ -147,9 +155,10 @@ cmake -DCMAKE_SYSTEM_NAME=Android \ .. ``` -用户还可根据自己的需求设置其他编译参数。比如希望最小化生成的库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。亦可以通过手动设置`CMAKE_C/CXX_FLAGS_MINSIZEREL/RELEASE`来影响PaddlePaddle的编译过程。 +用户还可根据自己的需求设置其他编译参数。比如希望最小化生成的库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。亦可以通过手动设置`CMAKE_C/CXX_FLAGS`来影响PaddlePaddle的编译过程。 **性能TIPS**，为了达到最快的计算速度，在CMake参数配置上，有以下建议： + - 设置`CMAKE_BUILD_TYPE`为`Release` - 使用`clang`编译工具链 - `armeabi-v7a`时，设置`USE_EIGEN_BLAS=ON`，使用Eigen进行矩阵计算；`arm64-v8a`时，设置`USE_EIGEN_FOR_BLAS=OFF`，使用OpenBLAS进行矩阵计算 diff --git a/doc/mobile/cross_compiling_for_android_en.md b/doc/mobile/cross_compiling_for_android_en.md index 26858581fc..0cf50181df 100644 --- a/doc/mobile/cross_compiling_for_android_en.md +++ b/doc/mobile/cross_compiling_for_android_en.md @@ -1,6 +1,9 @@ # Build PaddlePaddle for Android -There are two approaches to build PaddlePaddle for Android: using Docker and on Linux without Docker. +There are two approaches to build PaddlePaddle for Android: + +- [Cross-Compiling Using Docker](#cross-compiling-using-docker) +- [Cross-Compiling on Linux](#cross-compiling-on-linux) ## Cross-Compiling Using Docker @@ -16,6 +19,12 @@ $ cd Paddle $ docker build -t paddle:dev-android . -f Dockerfile.android ``` +Users can directly use the published Docker image. + +```bash +$ docker pull paddlepaddle/paddle:latest-dev-android +``` + ### Build the Inference Library We can run the Docker image we just created to build the inference library of PaddlePaddle for Android using the command below: @@ -47,7 +56,7 @@ The Docker image accepts two arguments `ANDROID_ABI` and `ANDROID_API`: ANDROID_API - >= 21 + >= 16 21 @@ -93,15 +102,13 @@ Android NDK includes everything we need to build the [*standalone toolchain*](ht The generated standalone toolchain will be in `your/path/to/arm64_standalone_toolchain`. -**Please be aware that the minimum level of Android API required by PaddlePaddle is 21.** - ### Cross-Compiling Arguments CMake supports [choosing the toolchain](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling). PaddlePaddle provides [`android.cmake`](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/android.cmake), which configures the Android cross-compiling toolchain for CMake. `android.cmake` is not required for CMake >= 3.7, which support Android cross-compiling. PaddlePaddle detects the CMake version, for those newer than 3.7, it uses [the official version](https://cmake.org/cmake/help/v3.7/manual/cmake-toolchains.7.html#cross-compiling). Some other CMake arguments you need to know: -- `CMAKE_SYSTEM_NAME` must be `Android`. This tells PaddlePaddle's CMake system to cross-compile third-party dependencies. This also changes some other CMake arguments like `WITH_GPU=OFF`, `WITH_AVX=OFF`, `WITH_PYTHON=OFF`, and `WITH_RDMA=OFF`. +- `CMAKE_SYSTEM_NAME` must be `Android`. This tells PaddlePaddle's CMake system to cross-compile third-party dependencies. This also changes some other CMake arguments like `WITH_GPU=OFF`, `WITH_AVX=OFF`, `WITH_PYTHON=OFF`, `WITH_RDMA=OFF`, `WITH_MKL=OFF` and `WITH_GOLANG=OFF`. - `WITH_C_API` must be `ON`, to build the C-based inference library for Android. - `WITH_SWIG_PY` must be `OFF` because the Android platform doesn't support SWIG-based API. @@ -123,7 +130,7 @@ Some Android-specific arguments: Other useful arguments: - `USE_EIGEN_FOR_BLAS`: indicates if using Eigen. Could be `ON` or `OFF`, defaults to `OFF`. -- `HOST_C/CXX_COMPILER`: specifies the host compiler, which is used to build the host-specific protoc and target-specific OpenBLAS. It defaults to the value of the environment variable `CC`, or `cc`. +- `HOST_C/CXX_COMPILER`: specifies the host compiler, which is used to build the host-specific protoc and target-specific OpenBLAS. It defaults to the value of the environment variable `CC/C++`, or `cc/c++`. Some frequent configurations for your reference: @@ -158,6 +165,7 @@ There are some other arguments you might want to configure. - `CMAKE_BUILD_TYPE-Release` optimizes the runtime performance. Our own tip for performance optimization to use clang and Eigen or OpenBLAS: + - `CMAKE_BUILD_TYPE=Release` - `ANDROID_TOOLCHAIN=clang` - `USE_EIGEN_BLAS=ON` for `armeabi-v7a`, or `USE_EIGEN_FOR_BLAS=OFF` for `arm64-v8a`. diff --git a/doc/mobile/cross_compiling_for_ios_en.md b/doc/mobile/cross_compiling_for_ios_en.md index aa390cd61f..19bfe86c51 100644 --- a/doc/mobile/cross_compiling_for_ios_en.md +++ b/doc/mobile/cross_compiling_for_ios_en.md @@ -1,4 +1,4 @@ -# PaddlePaddle Compiling Guide for iOS +# Build PaddlePaddle for iOS This tutorial will walk you through cross compiling the PaddlePaddle library for iOS from the source in MacOS. @@ -98,7 +98,7 @@ You can set other compiling parameters for your own need. I.E. if you are trying - set `CMAKE_BUILD_TYPE` with `Release` - set `IOS_USE_VECLIB_FOR_BLAS` with `ON` -## Compile and install +## Build and install After CMake, run following commands, PaddlePaddle will download the compile 3rd party dependencies, compile and install PaddlePaddle inference library. @@ -109,7 +109,7 @@ $ make install Please Note: if you compiled PaddlePaddle in the source directory for other platforms, do remove `third_party` and `build` directory within the source with `rm -rf` to ensure that all the 3rd party libraries dependencies and PaddlePaddle is newly compiled with current CMake configuration. -`your/path/to/install` directory will have following directories after `compile` and `install`: +`your/path/to/install` directory will have following directories after `make install`: - `include`, contains all the C-API header files. - `lib`, contains PaddlePaddle C-API static library. From 9b3f2c39f2de4ea0f67dc151c3b59647334c3c3d Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 28 Dec 2017 02:29:58 +0000 Subject: [PATCH 03/35] Add a simple example for fluid to do inference in C++ code. --- CMakeLists.txt | 6 +- cmake/external/eigen.cmake | 2 +- cmake/external/openblas.cmake | 2 +- cmake/generic.cmake | 6 +- paddle/CMakeLists.txt | 1 + paddle/framework/op_desc.cc | 2 +- paddle/framework/var_desc.cc | 2 +- paddle/framework/var_desc.h | 2 +- paddle/inference/CMakeLists.txt | 36 ++++ paddle/inference/example.cc | 56 +++++ paddle/inference/inference.cc | 203 ++++++++++++++++++ paddle/inference/inference.h | 50 +++++ paddle/pybind/protobuf.cc | 2 +- python/paddle/v2/fluid/io.py | 2 +- .../tests/book/test_recognize_digits_mlp.py | 5 + 15 files changed, 365 insertions(+), 12 deletions(-) create mode 100644 paddle/inference/CMakeLists.txt create mode 100644 paddle/inference/example.cc create mode 100644 paddle/inference/inference.cc create mode 100644 paddle/inference/inference.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5df83499d5..00996cb7ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,8 +20,10 @@ set(PADDLE_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) include(system) project(paddle CXX C Go) -message(STATUS "CXX compiler: " ${CMAKE_CXX_COMPILER} ", version: " ${CMAKE_CXX_COMPILER_VERSION}) -message(STATUS "C compiler: " ${CMAKE_C_COMPILER} ", version: " ${CMAKE_C_COMPILER_VERSION}) +message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}, version: " + "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") +message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: " + "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}") find_package(Sphinx) if(NOT CMAKE_CROSSCOMPILING) diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake index 96fc886a34..c4712f19eb 100644 --- a/cmake/external/eigen.cmake +++ b/cmake/external/eigen.cmake @@ -19,7 +19,7 @@ ExternalProject_Add( if (${CMAKE_VERSION} VERSION_LESS "3.3.0") set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/eigen3_dummy.c) - file(WRITE ${dummyfile} "const char * dummy_eigen3 = \"${dummyfile}\";") + file(WRITE ${dummyfile} "const char *dummy_eigen3 = \"${dummyfile}\";") add_library(eigen3 STATIC ${dummyfile}) else() add_library(eigen3 INTERFACE) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 97857a686b..d8dc79c19e 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -113,7 +113,7 @@ INCLUDE_DIRECTORIES(${CBLAS_INC_DIR}) # FIXME(gangliao): generate cblas target to track all high performance # linear algebra libraries for cc_library(xxx SRCS xxx.c DEPS cblas) SET(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/cblas_dummy.c) -FILE(WRITE ${dummyfile} "const char * dummy = \"${dummyfile}\";") +FILE(WRITE ${dummyfile} "const char *dummy_cblas = \"${dummyfile}\";") ADD_LIBRARY(cblas STATIC ${dummyfile}) TARGET_LINK_LIBRARIES(cblas ${CBLAS_LIBRARIES}) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 66c8e3ad7e..585db019d5 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -120,7 +120,7 @@ function(merge_static_libs TARGET_NAME) DEPENDS ${libs}) # Generate dummy staic lib - file(WRITE ${target_SRCS} "const char *dummy = \"${target_SRCS}\";") + file(WRITE ${target_SRCS} "const char *dummy_${TARGET_NAME} = \"${target_SRCS}\";") add_library(${TARGET_NAME} STATIC ${target_SRCS}) target_link_libraries(${TARGET_NAME} ${libs_deps}) @@ -160,7 +160,7 @@ function(merge_static_libs TARGET_NAME) DEPENDS ${libs} ${target_OBJS}) # Generate dummy staic lib - file(WRITE ${target_SRCS} "const char *dummy = \"${target_SRCS}\";") + file(WRITE ${target_SRCS} "const char *dummy_${TARGET_NAME} = \"${target_SRCS}\";") add_library(${TARGET_NAME} STATIC ${target_SRCS}) target_link_libraries(${TARGET_NAME} ${libs_deps}) @@ -324,7 +324,7 @@ function(go_library TARGET_NAME) ) # Add dummy code to support `make target_name` under Terminal Command - file(WRITE ${dummyfile} "const char * dummy = \"${dummyfile}\";") + file(WRITE ${dummyfile} "const char *dummy_${TARGET_NAME} = \"${dummyfile}\";") if (go_library_SHARED OR go_library_shared) add_library(${TARGET_NAME} SHARED ${dummyfile}) else() diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index 7d2becbdd7..4a98ede278 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -24,6 +24,7 @@ else() add_subdirectory(framework) add_subdirectory(operators) add_subdirectory(pybind) + add_subdirectory(inference) endif() if(WITH_SWIG_PY) diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 781bbb4c19..f9b82180df 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -64,7 +64,7 @@ class CompileTimeInferShapeContext : public InferShapeContext { PADDLE_ENFORCE_EQ(in_var->GetType(), proto::VarDesc::LOD_TENSOR, "The %d-th output of Output(%s) must be LoDTensor.", j, out); - out_var->SetLoDLevel(in_var->GetLodLevel()); + out_var->SetLoDLevel(in_var->GetLoDLevel()); } bool IsRuntime() const override; diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc index 7d002b9ea0..aeab18d721 100644 --- a/paddle/framework/var_desc.cc +++ b/paddle/framework/var_desc.cc @@ -52,7 +52,7 @@ void VarDesc::SetLoDLevel(int32_t lod_level) { } } -int32_t VarDesc::GetLodLevel() const { +int32_t VarDesc::GetLoDLevel() const { switch (desc_.type()) { case proto::VarDesc::LOD_TENSOR: return desc_.lod_tensor().lod_level(); diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h index 4fd2abe7fb..fc482c4674 100644 --- a/paddle/framework/var_desc.h +++ b/paddle/framework/var_desc.h @@ -76,7 +76,7 @@ class VarDesc { void SetLoDLevel(int32_t lod_level); - int32_t GetLodLevel() const; + int32_t GetLoDLevel() const; proto::VarDesc::VarType GetType() const; diff --git a/paddle/inference/CMakeLists.txt b/paddle/inference/CMakeLists.txt new file mode 100644 index 0000000000..73c875409a --- /dev/null +++ b/paddle/inference/CMakeLists.txt @@ -0,0 +1,36 @@ +set(FLUID_CORE_MODULES + backward proto_desc paddle_memory executor prune init ${GLOB_OP_LIB}) + +cc_library(paddle_fluid_api + SRCS inference.cc + DEPS ${FLUID_CORE_MODULES}) + +# Merge all modules into a simgle static library +cc_library(paddle_fluid DEPS paddle_fluid_api ${FLUID_CORE_MODULES}) + +# ptools +# just for testing, we may need to change the storing format for inference_model +# and move the dependent of pickle. +# download from http://www.picklingtools.com/ +# build in the C++ sub-directory, using command +# make -f Makefile.Linux libptools.so +set(PTOOLS_LIB) +set(PTOOLS_ROOT $ENV{PTOOLS_ROOT} CACHE PATH "Folder contains PicklingTools") +find_path(PTOOLS_INC_DIR chooseser.h PATHS ${PTOOLS_ROOT}/C++) +find_library(PTOOLS_SHARED_LIB NAMES ptools PATHS ${PTOOLS_ROOT}/C++) +if(PTOOLS_INC_DIR AND PTOOLS_SHARED_LIB) + add_definitions(-DPADDLE_USE_PTOOLS) + set(PTOOLS_LIB ptools) + message(STATUS "Found PicklingTools: ${PTOOLS_SHARED_LIB}") + add_library(${PTOOLS_LIB} SHARED IMPORTED GLOBAL) + set_property(TARGET ${PTOOLS_LIB} PROPERTY IMPORTED_LOCATION ${PTOOLS_SHARED_LIB}) + include_directories(${PTOOLS_ROOT}/C++) + include_directories(${PTOOLS_ROOT}/C++/opencontainers_1_8_5/include) + add_definitions(-DOC_NEW_STYLE_INCLUDES) # used in ptools +endif() + +add_executable(example example.cc) +target_link_libraries(example + -Wl,--start-group -Wl,--whole-archive paddle_fluid + -Wl,--no-whole-archive -Wl,--end-group + ${PTOOLS_LIB}) diff --git a/paddle/inference/example.cc b/paddle/inference/example.cc new file mode 100644 index 0000000000..30cdd96327 --- /dev/null +++ b/paddle/inference/example.cc @@ -0,0 +1,56 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include "paddle/inference/inference.h" + +int main(int argc, char* argv[]) { + std::string dirname = + "/home/work/liuyiqun/PaddlePaddle/Paddle/paddle/inference/" + "recognize_digits_mlp.inference.model"; + std::vector feed_var_names = {"x"}; + std::vector fetch_var_names = {"fc_2.tmp_2"}; + paddle::InferenceEngine* desc = new paddle::InferenceEngine(); + desc->LoadInferenceModel(dirname, feed_var_names, fetch_var_names); + + paddle::framework::LoDTensor input; + srand(time(0)); + float* input_ptr = + input.mutable_data({1, 784}, paddle::platform::CPUPlace()); + for (int i = 0; i < 784; ++i) { + input_ptr[i] = rand() / (static_cast(RAND_MAX)); + } + + std::vector feeds; + feeds.push_back(input); + std::vector fetchs; + desc->Execute(feeds, fetchs); + + for (size_t i = 0; i < fetchs.size(); ++i) { + auto dims_i = fetchs[i].dims(); + std::cout << "dims_i:"; + for (int j = 0; j < dims_i.size(); ++j) { + std::cout << " " << dims_i[j]; + } + std::cout << std::endl; + std::cout << "result:"; + float* output_ptr = fetchs[i].data(); + for (int j = 0; j < paddle::framework::product(dims_i); ++j) { + std::cout << " " << output_ptr[j]; + } + std::cout << std::endl; + } + return 0; +} diff --git a/paddle/inference/inference.cc b/paddle/inference/inference.cc new file mode 100644 index 0000000000..ebfdcd7456 --- /dev/null +++ b/paddle/inference/inference.cc @@ -0,0 +1,203 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "inference.h" +#include +#include "paddle/framework/executor.h" +#include "paddle/framework/feed_fetch_method.h" +#include "paddle/framework/init.h" +#include "paddle/framework/scope.h" + +#ifdef PADDLE_USE_PTOOLS +#include "chooseser.h" +#endif + +namespace paddle { + +void InferenceEngine::LoadInferenceModel( + const std::string& dirname, + const std::vector& feed_var_names, + const std::vector& fetch_var_names) { +#ifdef PADDLE_USE_PTOOLS + std::string model_filename = dirname + "/__model__"; + LOG(INFO) << "Using PicklingTools, loading model from " << model_filename; + Val v; + LoadValFromFile(model_filename.c_str(), v, SERIALIZE_P0); + std::string program_desc_str = v["program_desc_str"]; + LOG(INFO) << "program_desc_str's size: " << program_desc_str.size(); +// PicklingTools cannot parse the vector of strings correctly. +#else + // program_desc_str + // the inference.model is stored by following python codes: + // inference_program = fluid.io.get_inference_program(predict) + // model_filename = "recognize_digits_mlp.inference.model/inference.model" + // with open(model_filename, "w") as f: + // program_str = inference_program.desc.serialize_to_string() + // f.write(struct.pack('q', len(program_str))) + // f.write(program_str) + std::string model_filename = dirname + "/inference.model"; + LOG(INFO) << "loading model from " << model_filename; + std::ifstream fs(model_filename, std::ios_base::binary); + int64_t size = 0; + fs.read(reinterpret_cast(&size), sizeof(int64_t)); + LOG(INFO) << "program_desc_str's size: " << size; + std::string program_desc_str; + program_desc_str.resize(size); + fs.read(&program_desc_str[0], size); +#endif + program_ = new framework::ProgramDesc(program_desc_str); + GenerateLoadProgram(dirname); + + if (feed_var_names.empty() || fetch_var_names.empty()) { + LOG(FATAL) << "Please specify the feed_var_names and fetch_var_names."; + } + feed_var_names_ = feed_var_names; + fetch_var_names_ = fetch_var_names; + PrependFeedOp(); + AppendFetchOp(); +} + +bool InferenceEngine::IsParameter(const framework::VarDesc* var) { + if (var->Persistable()) { + // There are many unreachable variables in the program + for (size_t i = 0; i < program_->Size(); ++i) { + const framework::BlockDesc& block = program_->Block(i); + for (auto* op : block.AllOps()) { + for (auto input_argument_name : op->InputArgumentNames()) { + if (input_argument_name == var->Name()) { + return true; + } + } + } + } + } + return false; +} + +void InferenceEngine::GenerateLoadProgram(const std::string& dirname) { + framework::BlockDesc* global_block = program_->MutableBlock(0); + + load_program_ = new framework::ProgramDesc(); + framework::BlockDesc* load_block = load_program_->MutableBlock(0); + for (auto* var : global_block->AllVars()) { + if (IsParameter(var)) { + LOG(INFO) << "parameter's name: " << var->Name(); + + // framework::VarDesc new_var = *var; + framework::VarDesc* new_var = load_block->Var(var->Name()); + new_var->SetShape(var->Shape()); + new_var->SetDataType(var->GetDataType()); + new_var->SetType(var->GetType()); + new_var->SetLoDLevel(var->GetLoDLevel()); + new_var->SetPersistable(true); + + // append_op + framework::OpDesc* op = load_block->AppendOp(); + op->SetType("load"); + op->SetOutput("Out", {new_var->Name()}); + op->SetAttr("file_path", {dirname + "/" + new_var->Name()}); + op->CheckAttrs(); + } + } +} + +void InferenceEngine::PrependFeedOp() { + if (!program_) { + LOG(FATAL) << "Please initialize the program_ first."; + } + + framework::BlockDesc* global_block = program_->MutableBlock(0); + + // create_var + framework::VarDesc* feed_var = global_block->Var("feed"); + feed_var->SetType(framework::proto::VarDesc::FEED_MINIBATCH); + feed_var->SetPersistable(true); + + // prepend feed_op + for (size_t i = 0; i < feed_var_names_.size(); ++i) { + std::string var_name = feed_var_names_[i]; + LOG(INFO) << "feed var's name: " << var_name; + + // prepend_op + framework::OpDesc* op = global_block->PrependOp(); + op->SetType("feed"); + op->SetInput("X", {"feed"}); + op->SetOutput("Out", {var_name}); + op->SetAttr("col", {static_cast(i)}); + op->CheckAttrs(); + } +} + +void InferenceEngine::AppendFetchOp() { + if (!program_) { + LOG(FATAL) << "Please initialize the program_ first."; + } + + framework::BlockDesc* global_block = program_->MutableBlock(0); + + // create_var + framework::VarDesc* fetch_var = global_block->Var("fetch"); + fetch_var->SetType(framework::proto::VarDesc::FETCH_LIST); + fetch_var->SetPersistable(true); + + // append fetch_op + for (size_t i = 0; i < fetch_var_names_.size(); ++i) { + std::string var_name = fetch_var_names_[i]; + LOG(INFO) << "fetch var's name: " << var_name; + + // append_op + framework::OpDesc* op = global_block->AppendOp(); + op->SetType("fetch"); + op->SetInput("X", {var_name}); + op->SetOutput("Out", {"fetch"}); + op->SetAttr("col", {static_cast(i)}); + op->CheckAttrs(); + } +} + +void InferenceEngine::Execute(const std::vector& feeds, + std::vector& fetchs) { + if (!program_ || !load_program_) { + LOG(FATAL) << "Please initialize the program_ and load_program_ first."; + } + + if (feeds.size() < feed_var_names_.size()) { + LOG(FATAL) << "Please feed " << feed_var_names_.size() << " input Tensors."; + } + + auto* place = new platform::CPUPlace(); + framework::InitDevices({"CPU"}); + framework::Executor* executor = new framework::Executor(*place); + framework::Scope* scope = new framework::Scope(); + + executor->Run(*load_program_, scope, 0, true, true); + + // set_feed_variable + for (size_t i = 0; i < feed_var_names_.size(); ++i) { + framework::SetFeedVariable(scope, feeds[i], "feed", i); + } + + executor->Run(*program_, scope, 0, true, true); + + // get_fetch_variable + fetchs.resize(fetch_var_names_.size()); + for (size_t i = 0; i < fetch_var_names_.size(); ++i) { + fetchs[i] = framework::GetFetchVariable(*scope, "fetch", i); + } + + delete place; + delete scope; + delete executor; +} +} // namespace paddle diff --git a/paddle/inference/inference.h b/paddle/inference/inference.h new file mode 100644 index 0000000000..a3f3ef4b44 --- /dev/null +++ b/paddle/inference/inference.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/block_desc.h" +#include "paddle/framework/lod_tensor.h" +#include "paddle/framework/program_desc.h" + +namespace paddle { + +class InferenceEngine { +public: + InferenceEngine() : program_(nullptr), load_program_(nullptr) {} + ~InferenceEngine() { + delete program_; + delete load_program_; + } + + void LoadInferenceModel(const std::string& dirname, + const std::vector& feed_var_names, + const std::vector& fetch_var_names); + void Execute(const std::vector& feeds, + std::vector& fetchs); + +private: + bool IsParameter(const framework::VarDesc* var); + void GenerateLoadProgram(const std::string& dirname); + void PrependFeedOp(); + void AppendFetchOp(); + +private: + framework::ProgramDesc* program_; + framework::ProgramDesc* load_program_; + std::vector feed_var_names_; + std::vector fetch_var_names_; +}; + +} // namespace paddle diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 07292d47e9..564a370001 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -216,7 +216,7 @@ void BindVarDsec(py::module &m) { .def("set_dtype", &VarDesc::SetDataType) .def("shape", &VarDesc::Shape, py::return_value_policy::reference) .def("dtype", &VarDesc::GetDataType, py::return_value_policy::reference) - .def("lod_level", &VarDesc::GetLodLevel) + .def("lod_level", &VarDesc::GetLoDLevel) .def("set_lod_level", &VarDesc::SetLoDLevel) .def("type", &VarDesc::GetType) .def("set_type", &VarDesc::SetType) diff --git a/python/paddle/v2/fluid/io.py b/python/paddle/v2/fluid/io.py index 69a732fc45..78b8d97b2e 100644 --- a/python/paddle/v2/fluid/io.py +++ b/python/paddle/v2/fluid/io.py @@ -36,7 +36,7 @@ def save_vars(executor, dirname, main_program=None, vars=None, predicate=None): :param executor: executor that save variable :param dirname: directory path :param main_program: program. If vars is None, then filter all variables in this - program which fit `predicate`. Default g_program. + program which fit `predicate`. Default default_main_program. :param predicate: The Predicate describes a callable that returns a variable as a bool. If it returns true, the variables will be saved. :param vars: variables need to be saved. If specify vars, program & predicate diff --git a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py index fc073f6be8..51bfe2973d 100644 --- a/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py +++ b/python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py @@ -14,6 +14,7 @@ hidden1 = fluid.layers.fc(input=image, param_attr=fluid.ParamAttr( regularizer=regularizer, clip=fluid.clip.ClipByValue(10))) + hidden2 = fluid.layers.fc(input=hidden1, size=64, act='relu', @@ -73,5 +74,9 @@ for pass_id in range(PASS_NUM): + " test_acc=" + str(test_pass_acc)) if test_pass_acc > 0.7: + fluid.io.save_inference_model( + "./recognize_digits_mlp.inference.model/", ["x"], [predict], + exe) exit(0) + exit(1) From 0c5202cbb562b7d070c807d38de5b54db06833b4 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Tue, 2 Jan 2018 14:42:27 +0800 Subject: [PATCH 04/35] Tiny enhance of while_op --- paddle/operators/while_op.cc | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/paddle/operators/while_op.cc b/paddle/operators/while_op.cc index 728ef60794..65d827e0e0 100644 --- a/paddle/operators/while_op.cc +++ b/paddle/operators/while_op.cc @@ -25,12 +25,12 @@ namespace operators { using StepScopeVar = std::vector; using LoDTensor = framework::LoDTensor; -constexpr char kStepBlock[] = "sub_block"; -constexpr char kCondition[] = "Condition"; -constexpr char kStepScopes[] = "StepScopes"; -constexpr char kParameters[] = "X"; -constexpr char kParamGrads[] = "X@GRAD"; -constexpr char kOutputs[] = "Out"; +static constexpr char kStepBlock[] = "sub_block"; +static constexpr char kCondition[] = "Condition"; +static constexpr char kStepScopes[] = "StepScopes"; +static constexpr char kX[] = "X"; +static constexpr char kXGRAD[] = "X@GRAD"; +static constexpr char kOutputs[] = "Out"; class WhileOp : public framework::OperatorBase { public: @@ -67,7 +67,7 @@ class WhileOpMaker : public framework::OpProtoAndCheckerMaker { public: WhileOpMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput(kParameters, + AddInput(kX, "A set of variables, which are required by operators inside the " "block of While Op.") .AsDuplicable(); @@ -158,8 +158,8 @@ class WhileGradOp : public framework::OperatorBase { executor.Run(*program, *cur_scope_iter, block->ID(), false); - auto &pg_names = Outputs(kParamGrads); - auto &p_names = Inputs(kParameters); + auto &pg_names = Outputs(kXGRAD); + auto &p_names = Inputs(kX); PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size()); for (size_t param_id = 0; param_id < pg_names.size(); ++param_id) { if (pg_names[param_id] == framework::kEmptyVarName) { @@ -213,11 +213,11 @@ class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { std::unique_ptr Apply() const override { auto *grad = new framework::OpDesc(); grad->SetType("while_grad"); - grad->SetInput(kParameters, Input(kParameters)); + grad->SetInput(kX, Input(kX)); // Not all of IGs will be generated by inner gradient operators of while op. // Ignore IGs that is not generated by the inside block. - auto igs = InputGrad(kParameters, /*do not drop empty gradient*/ false); + auto igs = InputGrad(kX, /*do not drop empty gradient*/ false); std::unordered_set all_outs; for (size_t i = 0; i < grad_block_[0]->OpSize(); ++i) { for (auto &oname : grad_block_[0]->Op(i)->OutputArgumentNames()) { @@ -231,7 +231,7 @@ class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { } } - grad->SetOutput(framework::GradVarName(kParameters), igs); + grad->SetOutput(framework::GradVarName(kX), igs); grad->SetInput(kOutputs, Output(kOutputs)); @@ -240,7 +240,7 @@ class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { std::unordered_set block_ins; auto *fwd_block = this->grad_block_[0]->ParentBlock(); { - for (auto &p : Input(kParameters)) { + for (auto &p : Input(kX)) { block_ins.insert(p); } for (auto &o : Output(kOutputs)) { @@ -288,8 +288,8 @@ class WhileGradOpVarTypeInference : public framework::VarTypeInference { public: void operator()(const framework::OpDesc &op_desc, framework::BlockDesc *block) const override { - auto p_names = op_desc.Input(kParameters); - auto pg_names = op_desc.Output(framework::GradVarName(kParameters)); + auto p_names = op_desc.Input(kX); + auto pg_names = op_desc.Output(framework::GradVarName(kX)); for (size_t i = 0; i < p_names.size(); ++i) { auto &p_var = detail::Ref(block->FindVarRecursive(p_names[i])); @@ -307,21 +307,21 @@ class WhileGradOpVarTypeInference : public framework::VarTypeInference { class WhileGradOpShapeInference : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext *ctx) const override { - ctx->HasInputs(kParameters); - ctx->HasOutputs(framework::GradVarName(kParameters)); + ctx->HasInputs(kX); + ctx->HasOutputs(framework::GradVarName(kX)); ctx->HasInputs(kOutputs); ctx->HasInputs(framework::GradVarName(kOutputs)); - auto p_names = ctx->Inputs(kParameters); - auto pg_names = ctx->Outputs(kParamGrads); - auto var_types = ctx->GetInputsVarType(kParameters); + auto p_names = ctx->Inputs(kX); + auto pg_names = ctx->Outputs(kXGRAD); + auto var_types = ctx->GetInputsVarType(kX); std::vector names_to_set; std::vector dims_to_set; for (size_t i = 0; i < p_names.size(); ++i) { if (pg_names[i] == framework::kEmptyVarName) { continue; } - auto dims = ctx->GetInputsElementDim(kParameters, i); + auto dims = ctx->GetInputsElementDim(kX, i); if (var_types[i] == framework::proto::VarDesc::LOD_TENSOR) { names_to_set.push_back(pg_names[i]); dims_to_set.push_back(dims); From f3851fe58dbae3d5d6a450af76b97fb49aa4f4ba Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 2 Jan 2018 21:18:26 +0800 Subject: [PATCH 05/35] auto pybind when *_op.cc contains several operators --- paddle/operators/CMakeLists.txt | 83 +++++---------------------------- 1 file changed, 11 insertions(+), 72 deletions(-) diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 9f603474de..467963f666 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -71,74 +71,11 @@ function(op_library TARGET) file(APPEND ${pybind_file} "USE_OP(less_than);\nUSE_OP(equal);\n") endif() - # conv_op contains several operators - if ("${TARGET}" STREQUAL "conv_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(conv2d);\n") - endif() - - # conv_cudnn_op contains several operators - if ("${TARGET}" STREQUAL "conv_cudnn_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(conv2d_cudnn);\n") - endif() - - # pool_op contains several operators - if ("${TARGET}" STREQUAL "pool_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(pool2d);\n") - endif() - - # pool_cudnn_op contains several operators - if ("${TARGET}" STREQUAL "pool_cudnn_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(pool2d_cudnn);\n") - endif() - if ("${TARGET}" STREQUAL "logical_op") set(pybind_flag 1) file(APPEND ${pybind_file} "USE_OP(logical_and);\n") endif() - # pool_with_index_op contains several operators - if ("${TARGET}" STREQUAL "pool_with_index_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(max_pool2d_with_index);\n") - endif() - - # conv_transpose_op contains several operators - if ("${TARGET}" STREQUAL "conv_transpose_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(conv2d_transpose);\n") - endif() - - # conv_transpose_cudnn_op contains two operators - if ("${TARGET}" STREQUAL "conv_transpose_cudnn_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(conv2d_transpose_cudnn);\n") - endif() - - # save_restore_op contains several operators - if ("${TARGET}" STREQUAL "save_restore_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_NO_KERNEL_OP(save);\n") - endif() - - # activation_op contains several operators - if ("${TARGET}" STREQUAL "activation_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(sigmoid);\n") - endif() - # nccl_op contains several operators if ("${TARGET}" STREQUAL "nccl_op") set(pybind_flag 1) @@ -146,21 +83,24 @@ function(op_library TARGET) file(APPEND ${pybind_file} "USE_CUDA_ONLY_OP(ncclAllReduce);\n") endif() - # reduce_op contains several operators - if ("${TARGET}" STREQUAL "reduce_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_OP(reduce_sum);\n") - endif() - if ("${TARGET}" STREQUAL "tensor_array_read_write_op") set(pybind_flag 1) file(APPEND ${pybind_file} "USE_NO_KERNEL_OP(read_from_array);\nUSE_NO_KERNEL_OP(write_to_array);\n") endif() + file(READ ${TARGET}.cc TARGET_CONTENT) + # It's enough to just adding one operator to pybind + string(REGEX MATCH "REGISTER_OP\\(.*REGISTER_OP\\(" multi_register "${TARGET_CONTENT}") + string(REGEX MATCH "REGISTER_OP\\([a-z0-9_]*," one_register "${multi_register}") + if (one_register STREQUAL "") + string(REPLACE "_op" "" TARGET "${TARGET}") + else () + string(REPLACE "REGISTER_OP(" "" TARGET "${one_register}") + string(REPLACE "," "" TARGET "${TARGET}") + endif() + # pybind USE_NO_KERNEL_OP # HACK: if REGISTER_OP_CPU_KERNEL presents the operator must have kernel - file(READ ${TARGET}.cc TARGET_CONTENT) string(REGEX MATCH "REGISTER_OP_CPU_KERNEL" regex_result "${TARGET_CONTENT}") string(REPLACE "_op" "" TARGET "${TARGET}") if (${pybind_flag} EQUAL 0 AND regex_result STREQUAL "") @@ -171,7 +111,6 @@ function(op_library TARGET) # pybind USE_CPU_ONLY_OP list(LENGTH cu_srcs cu_srcs_len) list(LENGTH cu_cc_srcs cu_cc_srcs_len) - if (${pybind_flag} EQUAL 0 AND ${cu_srcs_len} EQUAL 0 AND ${cu_cc_srcs_len} EQUAL 0) file(APPEND ${pybind_file} "USE_CPU_ONLY_OP(${TARGET});\n") set(pybind_flag 1) From e4e95beedc3cabd73e3d37faf6c6d95c96f955df Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 2 Jan 2018 21:21:42 +0800 Subject: [PATCH 06/35] manually pybind some specific operators --- paddle/operators/CMakeLists.txt | 35 +++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 467963f666..df737ed9b0 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -61,32 +61,12 @@ function(op_library TARGET) ${op_common_deps}) endif() - # net_op doesn't need pybind - if ("${TARGET}" STREQUAL "net_op") - set(pybind_flag 1) - endif() - - if ("${TARGET}" STREQUAL "compare_op") - set(pybind_flag 1) - file(APPEND ${pybind_file} "USE_OP(less_than);\nUSE_OP(equal);\n") - endif() - - if ("${TARGET}" STREQUAL "logical_op") - set(pybind_flag 1) - file(APPEND ${pybind_file} "USE_OP(logical_and);\n") - endif() - - # nccl_op contains several operators - if ("${TARGET}" STREQUAL "nccl_op") - set(pybind_flag 1) - # It's enough to just adding one operator to pybind - file(APPEND ${pybind_file} "USE_CUDA_ONLY_OP(ncclAllReduce);\n") - endif() - - if ("${TARGET}" STREQUAL "tensor_array_read_write_op") - set(pybind_flag 1) - file(APPEND ${pybind_file} "USE_NO_KERNEL_OP(read_from_array);\nUSE_NO_KERNEL_OP(write_to_array);\n") - endif() + # net_op doesn't need pybind, others will be pybind manually + foreach(manual_pybind_op "net_op" "compare_op" "logical_op" "nccl_op" "tensor_array_read_write_op") + if ("${TARGET}" STREQUAL "${manual_pybind_op}") + set(pybind_flag 1) + endif() + endforeach() file(READ ${TARGET}.cc TARGET_CONTENT) # It's enough to just adding one operator to pybind @@ -127,6 +107,7 @@ add_subdirectory(nccl) if(WITH_GPU) op_library(nccl_op DEPS nccl_common) + file(APPEND ${pybind_file} "USE_CUDA_ONLY_OP(ncclAllReduce);\n") else() set(DEPS_OPS ${DEPS_OPS} nccl_op) endif() @@ -177,6 +158,8 @@ list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) foreach(src ${GENERAL_OPS}) op_library(${src}) endforeach() +file(APPEND ${pybind_file} "USE_OP(less_than);\nUSE_OP(logical_and);\nUSE_NO_KERNEL_OP(read_from_array);\n") + set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") From f0e797e5b70bf098b407f0ef4983b2bd8f853609 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 3 Jan 2018 14:12:15 +0800 Subject: [PATCH 07/35] Doc fix and enhancement for lstm_unit python wrapper. --- python/paddle/v2/fluid/layers/nn.py | 126 +++++++++++++++------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 55d8bf8a8a..1a2019d1f2 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -151,7 +151,7 @@ def embedding(input, size, is_sparse=False, param_attr=None, dtype='float32'): Args: input(Variable): Input to the function - size(tuple|list|None): Shape of the look up table parameter + size(tuple|list|None): Shape of the look up table parameter is_sparse(bool): Boolean flag that specifying whether the input is sparse param_attr(ParamAttr): Parameters for this layer dtype(np.dtype|core.DataType|str): The type of data : float32, float_16, int etc @@ -366,9 +366,9 @@ def cross_entropy(input, label, **kwargs): 1) One-hot cross-entropy: `soft_label = False`, `Label[i, 0]` indicates the class index for sample i: - + .. math:: - + Y[i] = -\log(X[i, Label[i]]) 2) Soft-label cross-entropy: @@ -386,15 +386,15 @@ def cross_entropy(input, label, **kwargs): As a special case of 2), when each row of 'label' has only one non-zero element which is equal to 1, soft-label cross-entropy degenerates to a one-hot cross-entropy with one-hot label representation. - + Args: - input (Variable|list): a 2-D tensor with shape [N x D], where N is the - batch size and D is the number of classes. This input is a probability + input (Variable|list): a 2-D tensor with shape [N x D], where N is the + batch size and D is the number of classes. This input is a probability computed by the previous operator, which is almost always the result of a softmax operator. - label (Variable|list): the ground truth which is a 2-D tensor. When - `soft_label` is set to `False`, `label` is a tensor with shape - [N x 1]. When `soft_label` is set to `True`, `label` is a + label (Variable|list): the ground truth which is a 2-D tensor. When + `soft_label` is set to `False`, `label` is a tensor with shape + [N x 1]. When `soft_label` is set to `True`, `label` is a tensor with shape [N x D]. soft_label (bool, via `**kwargs`): a flag indicating whether to interpretate the given labels as soft labels, default `False`. @@ -403,7 +403,7 @@ def cross_entropy(input, label, **kwargs): A 2-D tensor with shape [N x 1], the cross entropy loss. Raises: - `ValueError`: 1) the 1st dimension of `input` and `label` are not equal; 2) when \ + `ValueError`: 1) the 1st dimension of `input` and `label` are not equal; 2) when \ `soft_label == True`, and the 2nd dimension of `input` and `label` are not \ equal; 3) when `soft_label == False`, and the 2nd dimension of `label` is not 1. @@ -727,9 +727,9 @@ def conv2d(input, def sequence_pool(input, pool_type, **kwargs): """ - This function add the operator for sequence pooling. - It pools features of all time-steps of each instance, and is applied - on top of the input using pool_type mentioned in the parameters. + This function add the operator for sequence pooling. + It pools features of all time-steps of each instance, and is applied + on top of the input using pool_type mentioned in the parameters. It supports four pool_type: @@ -758,7 +758,7 @@ def sequence_pool(input, pool_type, **kwargs): Args: input(variable): The input variable which is a LoDTensor. - pool_type (string): The pooling type of sequence_pool. + pool_type (string): The pooling type of sequence_pool. It supports average, sum, sqrt and max. Returns: @@ -768,7 +768,7 @@ def sequence_pool(input, pool_type, **kwargs): .. code-block:: python - x = fluid.layers.data(name='x', shape=[7, 1], + x = fluid.layers.data(name='x', shape=[7, 1], dtype='float32', lod_level=1) avg_x = fluid.layers.sequence_pool(input=x, pool_type='average') sum_x = fluid.layers.sequence_pool(input=x, pool_type='sum') @@ -816,7 +816,7 @@ def sequence_first_step(input, **kwargs): .. code-block:: python - x = fluid.layers.data(name='x', shape=[7, 1], + x = fluid.layers.data(name='x', shape=[7, 1], dtype='float32', lod_level=1) x_first_step = fluid.layers.sequence_first_step(input=x) """ @@ -849,7 +849,7 @@ def sequence_last_step(input, **kwargs): .. code-block:: python - x = fluid.layers.data(name='x', shape=[7, 1], + x = fluid.layers.data(name='x', shape=[7, 1], dtype='float32', lod_level=1) x_last_step = fluid.layers.sequence_last_step(input=x) """ @@ -1168,25 +1168,26 @@ def lstm_unit(x_t, .. math:: - i_t & = \sigma(W_{x_i}x_{t} + W_{h_i}h_{t-1} + W_{c_i}c_{t-1} + b_i) + i_t & = \sigma(W_{x_i}x_{t} + W_{h_i}h_{t-1} + b_i) - f_t & = \sigma(W_{x_f}x_{t} + W_{h_f}h_{t-1} + W_{c_f}c_{t-1} + b_f) + f_t & = \sigma(W_{x_f}x_{t} + W_{h_f}h_{t-1} + b_f) - c_t & = f_tc_{t-1} + i_t tanh (W_{x_c}x_t+W_{h_c}h_{t-1} + b_c) + c_t & = f_tc_{t-1} + i_t tanh (W_{x_c}x_t + W_{h_c}h_{t-1} + b_c) - o_t & = \sigma(W_{x_o}x_{t} + W_{h_o}h_{t-1} + W_{c_o}c_t + b_o) + o_t & = \sigma(W_{x_o}x_{t} + W_{h_o}h_{t-1} + b_o) h_t & = o_t tanh(c_t) - The inputs of lstm unit includes :math:`x_t`, :math:`h_{t-1}` and - :math:`c_{t-1}`. The implementation separates the linear transformation - and non-linear transformation apart. Here, we take :math:`i_t` as an - example. The linear transformation is applied by calling a `fc` layer and - the equation is: + The inputs of lstm unit include :math:`x_t`, :math:`h_{t-1}` and + :math:`c_{t-1}`. The 2nd dimensions of :math:`h_{t-1}` and :math:`c_{t-1}` + should be same. The implementation separates the linear transformation and + non-linear transformation apart. Here, we take :math:`i_t` as an example. + The linear transformation is applied by calling a `fc` layer and the + equation is: .. math:: - L_{i_t} = W_{x_i}x_{t} + W_{h_i}h_{t-1} + W_{c_i}c_{t-1} + b_i + L_{i_t} = W_{x_i}x_{t} + W_{h_i}h_{t-1} + b_i The non-linear transformation is applied by calling `lstm_unit_op` and the equation is: @@ -1213,14 +1214,15 @@ def lstm_unit(x_t, Raises: ValueError: The ranks of **x_t**, **hidden_t_prev** and **cell_t_prev**\ not be 2 or the 1st dimensions of **x_t**, **hidden_t_prev** \ - and **cell_t_prev** not be the same. + and **cell_t_prev** not be the same or the 2nd dimensions of \ + **hidden_t_prev** and **cell_t_prev** not be the same. Examples: .. code-block:: python x_t = fluid.layers.fc(input=x_t_data, size=10) - prev_hidden = fluid.layers.fc(input=prev_hidden_data, size=20) + prev_hidden = fluid.layers.fc(input=prev_hidden_data, size=30) prev_cell = fluid.layers.fc(input=prev_cell_data, size=30) hidden_value, cell_value = fluid.layers.lstm_unit(x_t=x_t, hidden_t_prev=prev_hidden, @@ -1239,7 +1241,11 @@ def lstm_unit(x_t, if x_t.shape[0] != hidden_t_prev.shape[0] or x_t.shape[ 0] != cell_t_prev.shape[0]: - raise ValueError("The 1s dimension of x_t, hidden_t_prev and " + raise ValueError("The 1s dimensions of x_t, hidden_t_prev and " + "cell_t_prev must be the same.") + + if hidden_t_prev.shape[1] != cell_t_prev.shape[1]: + raise ValueError("The 2nd dimensions of hidden_t_prev and " "cell_t_prev must be the same.") if bias_attr is None: @@ -1268,17 +1274,17 @@ def lstm_unit(x_t, def reduce_sum(input, dim=None, keep_dim=False): """ - Computes the sum of tensor elements over the given dimension. + Computes the sum of tensor elements over the given dimension. Args: input (Variable): The input variable which is a Tensor or LoDTensor. - dim (int|None): The dimension along which the sum is performed. If - :attr:`None`, sum all elements of :attr:`input` and return a - Tensor variable with a single element, otherwise must be in the - range :math:`[-rank(input), rank(input))`. If :math:`dim < 0`, + dim (int|None): The dimension along which the sum is performed. If + :attr:`None`, sum all elements of :attr:`input` and return a + Tensor variable with a single element, otherwise must be in the + range :math:`[-rank(input), rank(input))`. If :math:`dim < 0`, the dimension to reduce is :math:`rank + dim`. - keep_dim (bool): Whether to reserve the reduced dimension in the - output Tensor. The result tensor will have one fewer dimension + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. Returns: @@ -1312,17 +1318,17 @@ def reduce_sum(input, dim=None, keep_dim=False): def reduce_mean(input, dim=None, keep_dim=False): """ - Computes the mean of tensor elements over the given dimension. + Computes the mean of tensor elements over the given dimension. Args: input (Variable): The input variable which is a Tensor or LoDTensor. - dim (int|None): The dimension along which the mean is computed. If - :attr:`None`, compute the mean over all elements of :attr:`input` - and return a Tensor variable with a single element, otherwise - must be in the range :math:`[-rank(input), rank(input))`. If + dim (int|None): The dimension along which the mean is computed. If + :attr:`None`, compute the mean over all elements of :attr:`input` + and return a Tensor variable with a single element, otherwise + must be in the range :math:`[-rank(input), rank(input))`. If :math:`dim < 0`, the dimension to reduce is :math:`rank + dim`. - keep_dim (bool): Whether to reserve the reduced dimension in the - output Tensor. The result tensor will have one fewer dimension + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. Returns: @@ -1356,22 +1362,22 @@ def reduce_mean(input, dim=None, keep_dim=False): def reduce_max(input, dim=None, keep_dim=False): """ - Computes the maximum of tensor elements over the given dimension. + Computes the maximum of tensor elements over the given dimension. Args: input (Variable): The input variable which is a Tensor or LoDTensor. - dim (int|None): The dimension along which the maximum is computed. - If :attr:`None`, compute the maximum over all elements of - :attr:`input` and return a Tensor variable with a single element, - otherwise must be in the range :math:`[-rank(input), rank(input))`. + dim (int|None): The dimension along which the maximum is computed. + If :attr:`None`, compute the maximum over all elements of + :attr:`input` and return a Tensor variable with a single element, + otherwise must be in the range :math:`[-rank(input), rank(input))`. If :math:`dim < 0`, the dimension to reduce is :math:`rank + dim`. - keep_dim (bool): Whether to reserve the reduced dimension in the - output Tensor. The result tensor will have one fewer dimension + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. Returns: Variable: The reduced Tensor variable. - + Examples: .. code-block:: python @@ -1400,22 +1406,22 @@ def reduce_max(input, dim=None, keep_dim=False): def reduce_min(input, dim=None, keep_dim=False): """ - Computes the minimum of tensor elements over the given dimension. + Computes the minimum of tensor elements over the given dimension. Args: input (Variable): The input variable which is a Tensor or LoDTensor. - dim (int|None): The dimension along which the minimum is computed. - If :attr:`None`, compute the minimum over all elements of - :attr:`input` and return a Tensor variable with a single element, - otherwise must be in the range :math:`[-rank(input), rank(input))`. + dim (int|None): The dimension along which the minimum is computed. + If :attr:`None`, compute the minimum over all elements of + :attr:`input` and return a Tensor variable with a single element, + otherwise must be in the range :math:`[-rank(input), rank(input))`. If :math:`dim < 0`, the dimension to reduce is :math:`rank + dim`. - keep_dim (bool): Whether to reserve the reduced dimension in the - output Tensor. The result tensor will have one fewer dimension + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension than the :attr:`input` unless :attr:`keep_dim` is true. Returns: Variable: The reduced Tensor variable. - + Examples: .. code-block:: python From d6ec9630473712bf0a61b121030369b63a9996b8 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 3 Jan 2018 14:20:33 +0800 Subject: [PATCH 08/35] Minor correction. --- python/paddle/v2/fluid/layers/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 1a2019d1f2..09b71cc371 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1241,7 +1241,7 @@ def lstm_unit(x_t, if x_t.shape[0] != hidden_t_prev.shape[0] or x_t.shape[ 0] != cell_t_prev.shape[0]: - raise ValueError("The 1s dimensions of x_t, hidden_t_prev and " + raise ValueError("The 1st dimensions of x_t, hidden_t_prev and " "cell_t_prev must be the same.") if hidden_t_prev.shape[1] != cell_t_prev.shape[1]: From c0f6f492bcc86dcb2a5702332915852734884b9a Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 3 Jan 2018 14:24:31 +0800 Subject: [PATCH 09/35] Add shape info for arguments. --- python/paddle/v2/fluid/layers/nn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 09b71cc371..5442cce494 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1199,9 +1199,9 @@ def lstm_unit(x_t, This layer has two outputs including :math:`h_t` and :math:`o_t`. Args: - x_t (Variable): The input value of current step. - hidden_t_prev (Variable): The hidden value of lstm unit. - cell_t_prev (Variable): The cell value of lstm unit. + x_t (Variable): The input value of current step, a 2-D tensor. + hidden_t_prev (Variable): The hidden value of lstm unit, a 2-D tensor. + cell_t_prev (Variable): The cell value of lstm unit, a 2-D tensor. forget_bias (float): The forget bias of lstm unit. param_attr (ParamAttr): The attributes of parameter weights, used to set initializer, name etc. From 5974c1b76e400da3b6f3e1dd8884fb006d48cc59 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 3 Jan 2018 15:09:24 +0800 Subject: [PATCH 10/35] refine comments in CMakelists.txt of operator --- paddle/operators/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index df737ed9b0..a0b61640e5 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -61,7 +61,7 @@ function(op_library TARGET) ${op_common_deps}) endif() - # net_op doesn't need pybind, others will be pybind manually + # Define operators that don't need pybind here. foreach(manual_pybind_op "net_op" "compare_op" "logical_op" "nccl_op" "tensor_array_read_write_op") if ("${TARGET}" STREQUAL "${manual_pybind_op}") set(pybind_flag 1) @@ -69,7 +69,8 @@ function(op_library TARGET) endforeach() file(READ ${TARGET}.cc TARGET_CONTENT) - # It's enough to just adding one operator to pybind + # It's enough to just adding one operator to pybind. + # And for detail pybind information, please see paddle/pybind/pybind.h. string(REGEX MATCH "REGISTER_OP\\(.*REGISTER_OP\\(" multi_register "${TARGET_CONTENT}") string(REGEX MATCH "REGISTER_OP\\([a-z0-9_]*," one_register "${multi_register}") if (one_register STREQUAL "") From 90a5a55a6c68585707d012346ff055fbe534eba3 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 3 Jan 2018 15:22:24 +0800 Subject: [PATCH 11/35] Expose some activations --- python/paddle/v2/fluid/layers/ops.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index d2ff6841a3..23fe13f9bb 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -1,9 +1,24 @@ from ..registry import register_layer -__all__ = [ - 'mean', 'mul', 'dropout', 'reshape', 'sigmoid', 'scale', 'transpose', - 'sigmoid_cross_entropy_with_logits', 'elementwise_add', 'elementwise_div', - 'elementwise_sub', 'elementwise_mul', 'clip', 'abs', 'sequence_softmax' + +__activations__ = [ + 'abs', 'tanh', 'sigmoid', 'relu', 'sqrt', 'ceil', 'floor', 'log', 'round' ] +__all__ = [ + 'mean', + 'mul', + 'dropout', + 'reshape', + 'scale', + 'transpose', + 'sigmoid_cross_entropy_with_logits', + 'elementwise_add', + 'elementwise_div', + 'elementwise_sub', + 'elementwise_mul', + 'clip', + 'sequence_softmax', +] + __activations__ + for _OP in set(__all__): globals()[_OP] = register_layer(_OP) From 60fecce43db68281112a91198d85a79a972f03f9 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 3 Jan 2018 15:20:00 +0800 Subject: [PATCH 12/35] Fix unit test for lstm_unit. --- python/paddle/v2/fluid/layers/nn.py | 9 ++++++--- python/paddle/v2/fluid/tests/test_layers.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 5442cce494..1c1c09dd28 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -1199,9 +1199,12 @@ def lstm_unit(x_t, This layer has two outputs including :math:`h_t` and :math:`o_t`. Args: - x_t (Variable): The input value of current step, a 2-D tensor. - hidden_t_prev (Variable): The hidden value of lstm unit, a 2-D tensor. - cell_t_prev (Variable): The cell value of lstm unit, a 2-D tensor. + x_t (Variable): The input value of current step, a 2-D tensor with shape + M x N, M for batch size and N for input size. + hidden_t_prev (Variable): The hidden value of lstm unit, a 2-D tensor + with shape M x S, M for batch size and S for size of lstm unit. + cell_t_prev (Variable): The cell value of lstm unit, a 2-D tensor with + shape M x S, M for batch size and S for size of lstm unit. forget_bias (float): The forget bias of lstm unit. param_attr (ParamAttr): The attributes of parameter weights, used to set initializer, name etc. diff --git a/python/paddle/v2/fluid/tests/test_layers.py b/python/paddle/v2/fluid/tests/test_layers.py index 9d2dcca56d..77f0f11f1b 100644 --- a/python/paddle/v2/fluid/tests/test_layers.py +++ b/python/paddle/v2/fluid/tests/test_layers.py @@ -177,8 +177,8 @@ class TestBook(unittest.TestCase): name='x_t_data', shape=[10, 10], dtype='float32') x_t = layers.fc(input=x_t_data, size=10) prev_hidden_data = layers.data( - name='prev_hidden_data', shape=[10, 20], dtype='float32') - prev_hidden = layers.fc(input=prev_hidden_data, size=20) + name='prev_hidden_data', shape=[10, 30], dtype='float32') + prev_hidden = layers.fc(input=prev_hidden_data, size=30) prev_cell_data = layers.data( name='prev_cell', shape=[10, 30], dtype='float32') prev_cell = layers.fc(input=prev_cell_data, size=30) From 059096741e85d8595b1d6767de936afb051d7f06 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 3 Jan 2018 16:09:30 +0800 Subject: [PATCH 13/35] Add init glog --- paddle/framework/init.cc | 4 ++++ paddle/framework/init.h | 2 ++ paddle/pybind/pybind.cc | 1 + python/paddle/v2/fluid/__init__.py | 25 +++++++++++++------------ 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/paddle/framework/init.cc b/paddle/framework/init.cc index 682cff168d..1980023372 100644 --- a/paddle/framework/init.cc +++ b/paddle/framework/init.cc @@ -75,5 +75,9 @@ bool InitDevices(const std::vector &devices) { return true; } +void InitGLOG(const std::string &prog_name) { + google::InitGoogleLogging(prog_name.c_str()); +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/init.h b/paddle/framework/init.h index 33907f9eb0..9c84a03ded 100644 --- a/paddle/framework/init.h +++ b/paddle/framework/init.h @@ -22,6 +22,8 @@ namespace framework { void InitGflags(std::vector &argv); +void InitGLOG(const std::string &prog_name); + bool InitDevices(const std::vector &devices); } // namespace framework diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 04485ce7c1..364db62cba 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -427,6 +427,7 @@ All parameter, weight, gradient are variables in Paddle. m.def("unique_integer", UniqueIntegerGenerator); m.def("init_gflags", framework::InitGflags); + m.def("init_glog", framework::InitGLOG); m.def("init_devices", &framework::InitDevices); m.def("is_compile_gpu", IsCompileGPU); diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index 225b41c504..b00892d91a 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -1,23 +1,23 @@ # import all class inside framework into fluid module -import framework -from framework import * -# import all class inside executor into fluid module -import executor -from executor import * +from core import LoDTensor -import io +import backward +import clip import evaluator +# import all class inside executor into fluid module +import executor +import framework import initializer +import io import layers import nets import optimizer -import backward import regularizer -from param_attr import ParamAttr from data_feeder import DataFeeder -from core import LoDTensor, CPUPlace, CUDAPlace from distribute_transpiler import DistributeTranspiler -import clip +from executor import * +from framework import * +from param_attr import ParamAttr Tensor = LoDTensor __all__ = framework.__all__ + executor.__all__ + [ @@ -27,7 +27,7 @@ __all__ = framework.__all__ + executor.__all__ + [ ] -def __read_gflags_from_env__(): +def __bootstrap__(): """ Enable reading gflags from environment variables. @@ -41,6 +41,7 @@ def __read_gflags_from_env__(): read_env_flags.append('fraction_of_gpu_memory_to_use') core.init_gflags([sys.argv[0]] + ["--tryfromenv=" + ",".join(read_env_flags)]) + core.init_glog(sys.argv[0]) if core.is_compile_gpu(): core.init_devices(["CPU", "GPU:0"]) @@ -48,4 +49,4 @@ def __read_gflags_from_env__(): core.init_devices(["CPU"]) -__read_gflags_from_env__() +__bootstrap__() From 0c16f4f6029871c1fadd97b60e93feba2990e75c Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 3 Jan 2018 16:18:10 +0800 Subject: [PATCH 14/35] Update --- python/paddle/v2/fluid/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index b00892d91a..e115529665 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -1,23 +1,23 @@ # import all class inside framework into fluid module -from core import LoDTensor - -import backward -import clip -import evaluator +import framework +from framework import * # import all class inside executor into fluid module import executor -import framework -import initializer +from executor import * + import io +import evaluator +import initializer import layers import nets import optimizer +import backward import regularizer +from param_attr import ParamAttr from data_feeder import DataFeeder +from core import LoDTensor, CPUPlace, CUDAPlace from distribute_transpiler import DistributeTranspiler -from executor import * -from framework import * -from param_attr import ParamAttr +import clip Tensor = LoDTensor __all__ = framework.__all__ + executor.__all__ + [ From 2b3d94691bd9a2cf5ddadce1bb9e7227b0f891da Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 3 Jan 2018 16:38:49 +0800 Subject: [PATCH 15/35] Update init.cc --- paddle/framework/init.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/framework/init.cc b/paddle/framework/init.cc index 1980023372..3bea8f3d0a 100644 --- a/paddle/framework/init.cc +++ b/paddle/framework/init.cc @@ -77,6 +77,7 @@ bool InitDevices(const std::vector &devices) { void InitGLOG(const std::string &prog_name) { google::InitGoogleLogging(prog_name.c_str()); + google::InstallFailureSignalHandler(); } } // namespace framework From 63e3150772557e311e32914c44c0f22420ecb52d Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 3 Jan 2018 17:13:35 +0800 Subject: [PATCH 16/35] Update code --- paddle/framework/lod_tensor.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 7b6dc09bdb..92b3d7fccd 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -177,6 +177,9 @@ void AppendLoD(LoD *lod, const LoD &lod_length) { lod->empty() || lod->size() == lod_length.size(), "The lod_length should has the same size with the appended lod."); if (lod->empty()) { + for (size_t i = 0; i < lod_length.size(); ++i) { + lod->emplace_back(1, 0); // size = 1, value = 0; + } *lod = LoD(lod_length.size(), std::vector({0})); } for (size_t i = 0; i < lod->size(); ++i) { From 5b3cf4ee61482b44c89ac8ebe9cf656e9d6fac7c Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Wed, 3 Jan 2018 17:26:22 +0800 Subject: [PATCH 17/35] Use gflags to parse arguments from command-line. --- paddle/inference/example.cc | 41 +++++++++++++++++++++++++++-------- paddle/inference/inference.cc | 1 - 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/paddle/inference/example.cc b/paddle/inference/example.cc index 30cdd96327..9711b20e6f 100644 --- a/paddle/inference/example.cc +++ b/paddle/inference/example.cc @@ -14,16 +14,37 @@ limitations under the License. */ #include #include +#include "gflags/gflags.h" #include "paddle/inference/inference.h" -int main(int argc, char* argv[]) { - std::string dirname = - "/home/work/liuyiqun/PaddlePaddle/Paddle/paddle/inference/" - "recognize_digits_mlp.inference.model"; - std::vector feed_var_names = {"x"}; - std::vector fetch_var_names = {"fc_2.tmp_2"}; - paddle::InferenceEngine* desc = new paddle::InferenceEngine(); - desc->LoadInferenceModel(dirname, feed_var_names, fetch_var_names); +DEFINE_string(dirname, "", "Directory of the inference model."); +DEFINE_string(feed_var_names, "", "Names of feeding variables"); +DEFINE_string(fetch_var_names, "", "Names of fetching variables"); + +int main(int argc, char** argv) { + google::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_dirname.empty() || FLAGS_feed_var_names.empty() || + FLAGS_fetch_var_names.empty()) { + // Example: + // ./example --dirname=recognize_digits_mlp.inference.model + // --feed_var_names="x" + // --fetch_var_names="fc_2.tmp_2" + std::cout << "Usage: ./example --dirname=path/to/your/model " + "--feed_var_names=x --fetch_var_names=y" + << std::endl; + exit(1); + } + + std::cout << "FLAGS_dirname: " << FLAGS_dirname << std::endl; + std::cout << "FLAGS_feed_var_names: " << FLAGS_feed_var_names << std::endl; + std::cout << "FLAGS_fetch_var_names: " << FLAGS_fetch_var_names << std::endl; + + std::string dirname = FLAGS_dirname; + std::vector feed_var_names = {FLAGS_feed_var_names}; + std::vector fetch_var_names = {FLAGS_fetch_var_names}; + + paddle::InferenceEngine* engine = new paddle::InferenceEngine(); + engine->LoadInferenceModel(dirname, feed_var_names, fetch_var_names); paddle::framework::LoDTensor input; srand(time(0)); @@ -36,7 +57,7 @@ int main(int argc, char* argv[]) { std::vector feeds; feeds.push_back(input); std::vector fetchs; - desc->Execute(feeds, fetchs); + engine->Execute(feeds, fetchs); for (size_t i = 0; i < fetchs.size(); ++i) { auto dims_i = fetchs[i].dims(); @@ -52,5 +73,7 @@ int main(int argc, char* argv[]) { } std::cout << std::endl; } + + delete engine; return 0; } diff --git a/paddle/inference/inference.cc b/paddle/inference/inference.cc index ebfdcd7456..48a51efcd2 100644 --- a/paddle/inference/inference.cc +++ b/paddle/inference/inference.cc @@ -94,7 +94,6 @@ void InferenceEngine::GenerateLoadProgram(const std::string& dirname) { if (IsParameter(var)) { LOG(INFO) << "parameter's name: " << var->Name(); - // framework::VarDesc new_var = *var; framework::VarDesc* new_var = load_block->Var(var->Name()); new_var->SetShape(var->Shape()); new_var->SetDataType(var->GetDataType()); From 5a4367bb16c14fb039dbeedcfe44051663ea77d0 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Wed, 3 Jan 2018 17:33:16 +0800 Subject: [PATCH 18/35] Update --- paddle/operators/activation_op.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index 0885f7c570..88c3d1c597 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/detail/safe_ref.h" namespace paddle { namespace operators { @@ -26,12 +27,16 @@ class ActivationKernel using T = typename Functor::ELEMENT_TYPE; void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* Out = context.Output("Out"); - Out->mutable_data(context.GetPlace()); - - auto x = framework::EigenVector::Flatten(*X); - auto out = framework::EigenVector::Flatten(*Out); + auto& X = detail::Ref(context.Input("X"), + "Cannot get input tensor X, variable name = %s", + context.op().Input("X")); + + auto& Out = detail::Ref(context.Output("Out"), + "Cannot get output tensor Out, variable name = %s", + context.op().Output("Out")); + Out.mutable_data(context.GetPlace()); + auto x = framework::EigenVector::Flatten(X); + auto out = framework::EigenVector::Flatten(Out); auto* place = context.template device_context().eigen_device(); Functor functor; From 907e6d04de0c5ccc41b84952e5cc18d1f1a85531 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Wed, 3 Jan 2018 17:57:33 +0800 Subject: [PATCH 19/35] Fix bug in SetAttrDescVisitor (#7165) * fix bug in SetAttrDescVisitor * add comments --- paddle/framework/op_desc.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 781bbb4c19..3e58e6442e 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -260,7 +260,13 @@ struct SetAttrDescVisitor : public boost::static_visitor { void operator()(int v) const { attr_->set_i(v); } void operator()(float v) const { attr_->set_f(v); } void operator()(const std::string &v) const { attr_->set_s(v); } - void operator()(bool b) const { attr_->set_b(b); } + + // Please refer to https://github.com/PaddlePaddle/Paddle/issues/7162 + template ::value>::type> + void operator()(T b) const { + attr_->set_b(b); + } void operator()(const std::vector &v) const { VectorToRepeated(v, attr_->mutable_ints()); @@ -274,9 +280,7 @@ struct SetAttrDescVisitor : public boost::static_visitor { void operator()(const std::vector &v) const { VectorToRepeated(v, attr_->mutable_bools()); } - void operator()(proto::BlockDesc *desc) const { - attr_->set_block_idx(desc->idx()); - } + void operator()(BlockDesc *desc) const { attr_->set_block_idx(desc->ID()); } void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } }; From 2d2b633282523c494a99e02da092c87da0c87dc0 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 3 Jan 2018 19:53:22 +0800 Subject: [PATCH 20/35] add more comments in CMakelists.txt of operator --- paddle/framework/op_registry.h | 4 ++-- paddle/operators/CMakeLists.txt | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index bdaa259181..d75c0233e8 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -37,8 +37,8 @@ class Registrar { public: // In our design, various kinds of classes, e.g., operators and kernels, // have their corresponding registry and registrar. The action of - // registration is in the constructor of a global registrar variable, which, - // however, are not used in the code that calls package framework, and would + // registration is in the constructor of a global registrar variable, which + // are not used in the code that calls package framework, and would // be removed from the generated binary file by the linker. To avoid such // removal, we add Touch to all registrar classes and make USE_OP macros to // call this method. So, as long as the callee code calls USE_OP, the global diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index a0b61640e5..77b52eb176 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -68,9 +68,10 @@ function(op_library TARGET) endif() endforeach() + # The registration of USE_OP, please refer to paddle/framework/op_registry.h. + # Note that it's enough to just adding one operator to pybind in a *_op.cc file. + # And for detail pybind information, please see generated paddle/pybind/pybind.h. file(READ ${TARGET}.cc TARGET_CONTENT) - # It's enough to just adding one operator to pybind. - # And for detail pybind information, please see paddle/pybind/pybind.h. string(REGEX MATCH "REGISTER_OP\\(.*REGISTER_OP\\(" multi_register "${TARGET_CONTENT}") string(REGEX MATCH "REGISTER_OP\\([a-z0-9_]*," one_register "${multi_register}") if (one_register STREQUAL "") From 19541468b6a99b57a3ef130fba841fac721b75c8 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 3 Jan 2018 22:04:35 +0800 Subject: [PATCH 21/35] "fix frigled test gradient of rnn" (#7166) * "fix frigled test gradient of rnn" * "fix based on comments" --- paddle/gserver/tests/test_LayerGrad.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index a2f07937b8..ba83667ebc 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -1472,7 +1472,8 @@ TEST(Layer, RecurrentLayer) { for (auto reversed : {false, true}) { config.layerConfig.set_reversed(reversed); config.testState = !reversed; - testLayerGrad(config, "recurrent", 50, /* trans= */ false, useGpu); + testLayerGrad( + config, "recurrent", 50, /* trans= */ false, useGpu, false, 1.0); } } } @@ -1494,7 +1495,8 @@ TEST(Layer, LstmLayer) { for (auto reversed : {false, true}) { config.layerConfig.set_reversed(reversed); config.testState = !reversed; - testLayerGrad(config, "lstmemory", 100, /* trans= */ false, useGpu); + testLayerGrad( + config, "lstmemory", 100, /* trans= */ false, useGpu, false, 0.02); } } for (auto useGpu : {true}) { From 042f3524d2327075e0d88993e8c75bb80e8d8f69 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 4 Jan 2018 11:04:27 +0800 Subject: [PATCH 22/35] add flag use_mkl_packed --- paddle/trainer/TrainerConfigHelper.cpp | 2 ++ paddle/utils/Flags.cpp | 6 ++++++ paddle/utils/Flags.h | 1 + python/paddle/v2/__init__.py | 2 ++ 4 files changed, 11 insertions(+) diff --git a/paddle/trainer/TrainerConfigHelper.cpp b/paddle/trainer/TrainerConfigHelper.cpp index a0a365aa0b..2b68d89e48 100644 --- a/paddle/trainer/TrainerConfigHelper.cpp +++ b/paddle/trainer/TrainerConfigHelper.cpp @@ -29,6 +29,7 @@ DECLARE_bool(with_gpu); DECLARE_bool(parallel_nn); DECLARE_string(config_args); DECLARE_bool(use_mkldnn); +DECLARE_bool(use_mkl_packed); const char *kConfigParserModuleName = "paddle.trainer.config_parser"; const char *kConfigParserFuncName = "parse_config_and_serialize"; @@ -46,6 +47,7 @@ TrainerConfigHelper::TrainerConfigHelper(const std::string &configFilePath) << ",with_cost=" << FLAGS_with_cost << ",use_gpu=" << FLAGS_use_gpu << ",parallel_nn=" << FLAGS_parallel_nn << ",use_mkldnn=" << FLAGS_use_mkldnn + << ",use_mkl_packed=" << FLAGS_use_mkl_packed << ",cudnn_version=" << hl_get_cudnn_lib_version(); if (!FLAGS_config_args.empty()) { configArgs << "," << FLAGS_config_args; diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index 9a7dc0e356..10db81cbda 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -27,6 +27,12 @@ DEFINE_bool(use_mkldnn, false, "Default still keep use CPU training"); DEFINE_bool(use_mkldnn, false, "Only support CPU training"); #endif +#ifdef PADDLE_WITH_MKLML +DEFINE_bool(use_mkl_packed, true, "Default use MKL Packed Optimization"); +#else +DEFINE_bool(use_mkl_packed, false, "Whether to use MKL Packed Optimization"); +#endif + DEFINE_bool(parallel_nn, false, "Whether to use multi-threads to calculate one neural network." diff --git a/paddle/utils/Flags.h b/paddle/utils/Flags.h index 1832bb515e..b64295bca0 100644 --- a/paddle/utils/Flags.h +++ b/paddle/utils/Flags.h @@ -41,3 +41,4 @@ DECLARE_string(predict_file); DECLARE_bool(prev_batch_state); DECLARE_string(init_model_path); DECLARE_bool(use_mkldnn); +DECLARE_bool(use_mkl_packed); diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 70f61e8499..0de417df2c 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -135,6 +135,8 @@ def init(**kwargs): cp.g_command_config_args['use_gpu'] = kwargs['use_gpu'] if 'use_mkldnn' in kwargs: cp.g_command_config_args['use_mkldnn'] = kwargs['use_mkldnn'] + if 'use_mkl_packed' in kwargs: + cp.g_command_config_args['use_mkl_packed'] = kwargs['use_mkl_packed'] assert 'parallel_nn' not in kwargs, ("currently 'parallel_nn' is not " "supported in v2 APIs.") From d3f867e776c9ff4c56401ca005257c091cfa6c31 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 4 Jan 2018 11:12:16 +0800 Subject: [PATCH 23/35] enable mkl_packed_recurrent python interface --- python/paddle/trainer/config_parser.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 239fe4204b..4fdf409021 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -3622,8 +3622,13 @@ class ConcatenateLayer2(LayerBase): @config_layer('recurrent') class RecurrentLayer(LayerBase): + layer_type = 'recurrent' + def __init__(self, name, inputs, reversed=False, bias=True, **xargs): - super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, + use_mkl_packed = bool( + int(g_command_config_args.get("use_mkl_packed", 0))) + self.layer_type = 'mkl_packed_recurrent' if use_mkl_packed else 'recurrent' + super(RecurrentLayer, self).__init__(name, self.layer_type, 0, inputs, **xargs) config_assert(len(self.inputs) == 1, 'RecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) From a893f156527942d9172d51ab6662748b7000d5bc Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 4 Jan 2018 13:30:51 +0800 Subject: [PATCH 24/35] fix layout transform (#7149) * "fix typo" * "fix based on comments" * "follow gogle style" * "fix based on comemnts" --- paddle/framework/data_transform.cc | 36 +++++++++++++++++++------ paddle/framework/data_transform.h | 8 +++--- paddle/framework/data_transform_test.cc | 16 +++++++++-- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/paddle/framework/data_transform.cc b/paddle/framework/data_transform.cc index 9d6a842442..ac6e40a3ae 100644 --- a/paddle/framework/data_transform.cc +++ b/paddle/framework/data_transform.cc @@ -11,6 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include #include "paddle/framework/data_transform.h" #include "paddle/framework/lod_tensor.h" @@ -74,26 +75,28 @@ void TransDataType(const platform::DeviceContext* ctx, } } -void TransDataLayout(const platform::DeviceContext* ctx, +void TransDataLayout(const std::vector& axis, + const platform::DeviceContext* ctx, const KernelTypePair& kernel_pair, const Variable& in, Variable* out) { - PADDLE_ENFORCE(in.IsType(), "Only Support Tensor transform!."); + PADDLE_ENFORCE(in.IsType(), "Only support Tensor transform!."); PADDLE_ENFORCE( platform::places_are_same_class(kernel_pair.first.place_, kernel_pair.second.place_), - "TransDataType Only Support DataType transform on same place!"); + "TransDataLayout only support DataLayout transform on same place!"); + PADDLE_ENFORCE(kernel_pair.first.data_type_ == kernel_pair.second.data_type_, + "TransDataLayout only support Datatype are same!"); auto src = in.Get(); auto* dst = out->GetMutable(); PADDLE_ENFORCE(arity(src.dims()) == 4, "Input Arity Only Suppport 4!"); - auto src_dim = src.dims(); - dst->Resize(src_dim); auto place = kernel_pair.second.place_; CopyFrom(src, place, *ctx, dst); - const std::vector axis = {0, 2, 3, 1}; + auto src_dim = src.dims(); std::vector dst_dim; + dst_dim.resize(axis.size()); for (size_t i = 0; i < axis.size(); i++) { dst_dim[i] = src_dim[axis[i]]; @@ -102,7 +105,7 @@ void TransDataLayout(const platform::DeviceContext* ctx, dst->Resize(make_ddim(dst_dim)); auto src_type = kernel_pair.first.data_type_; - framework::VisitDataType(src_type, CastDataLayout(src, dst, ctx, axis)); + framework::VisitDataType(src_type, CastDataLayout(ctx, axis, src, dst)); dst->set_layout(kernel_pair.second.data_layout_); } @@ -111,5 +114,22 @@ void TransDataLayout(const platform::DeviceContext* ctx, } // namespace paddle namespace f = paddle::framework; + +namespace { +std::vector NHWC2NCHW = {0, 3, 1, 2}; +std::vector NCHW2NHWC = {0, 2, 3, 1}; +} + REGISTER_DATA_TRANSFORM_FN(f::KernelFP32, f::KernelFP64, f::TransDataType); -REGISTER_DATA_TRANSFORM_FN(f::KernelNHWC, f::KernelNCHW, f::TransDataLayout); +REGISTER_DATA_TRANSFORM_FN(f::KernelNHWC, f::KernelNCHW, + std::bind(f::TransDataLayout, NHWC2NCHW, + std::placeholders::_1, + std::placeholders::_2, + std::placeholders::_3, + std::placeholders::_4)); +REGISTER_DATA_TRANSFORM_FN(f::KernelNCHW, f::KernelNHWC, + std::bind(f::TransDataLayout, NCHW2NHWC, + std::placeholders::_1, + std::placeholders::_2, + std::placeholders::_3, + std::placeholders::_4)); diff --git a/paddle/framework/data_transform.h b/paddle/framework/data_transform.h index 9abb3c99bf..56ebc80f43 100644 --- a/paddle/framework/data_transform.h +++ b/paddle/framework/data_transform.h @@ -73,6 +73,7 @@ struct CastDataType { auto numel = in_.numel(); auto* in_end = in_begin + numel; auto* out_begin = out_->mutable_data(place); + if (platform::is_cpu_place(place)) { platform::Transform trans; auto* context = static_cast(ctx_); @@ -86,9 +87,9 @@ struct CastDataType { }; struct CastDataLayout { - CastDataLayout(const framework::Tensor& in, framework::Tensor* out, - const platform::DeviceContext* ctx, - const std::vector& axis) + CastDataLayout(const platform::DeviceContext* ctx, + const std::vector& axis, const framework::Tensor& in, + framework::Tensor* out) : in_(in), out_(out), ctx_(ctx), axis_(axis) {} const framework::Tensor in_; framework::Tensor* out_; @@ -98,6 +99,7 @@ struct CastDataLayout { template void operator()() { auto place = ctx_->GetPlace(); + if (platform::is_cpu_place(place)) { operators::math::Transpose trans4; auto* context = static_cast(ctx_); diff --git a/paddle/framework/data_transform_test.cc b/paddle/framework/data_transform_test.cc index 8665b6248f..edd305fd17 100644 --- a/paddle/framework/data_transform_test.cc +++ b/paddle/framework/data_transform_test.cc @@ -106,7 +106,7 @@ TEST(DataTransform, Register) { ASSERT_EQ(test_value, 2); } -TEST(DataTransform, Layout) { +TEST(DataTransform, DataLayout) { using namespace paddle::framework; using namespace paddle::platform; @@ -127,7 +127,19 @@ TEST(DataTransform, Layout) { } Tensor dst = out.Get(); - EXPECT_TRUE(dst.layout() != src->layout()); + + EXPECT_TRUE(dst.layout() == DataLayout::kNCHW); + EXPECT_TRUE(dst.dims() == make_ddim({2, 2, 3, 1})); + + { + auto kernel1 = GenFromBit({1, 0, 1, 0}); + auto kernel2 = GenFromBit({1, 0, 0, 0}); + auto pair0 = std::make_pair(kernel1, kernel2); + instance.Get(pair0)(ctx, pair0, out, &in); + } + + EXPECT_TRUE(src->layout() == DataLayout::kNHWC); + EXPECT_TRUE(src->dims() == make_ddim({2, 3, 1, 2})); } TEST(DataTransform, DataType) { From 8ae84a5745591a2937d3004831050f8fe77ce1ef Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 4 Jan 2018 14:47:35 +0800 Subject: [PATCH 25/35] Async to drop kid --- paddle/framework/scope.cc | 4 +++- paddle/framework/threadpool.h | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/paddle/framework/scope.cc b/paddle/framework/scope.cc index 0c01d605bc..4e80e3d974 100644 --- a/paddle/framework/scope.cc +++ b/paddle/framework/scope.cc @@ -17,6 +17,7 @@ limitations under the License. */ #include // for unique_ptr #include // for call_once #include "glog/logging.h" +#include "paddle/framework/threadpool.h" #include "paddle/string/printf.h" namespace paddle { @@ -87,7 +88,8 @@ void Scope::DeleteScope(Scope* scope) { auto it = std::find(this->kids_.begin(), this->kids_.end(), scope); PADDLE_ENFORCE(it != this->kids_.end(), "Cannot find %p as kid scope", scope); this->kids_.erase(it); - delete scope; + // Make delete async. + Async([scope] { delete scope; }); } void Scope::Rename(const std::string& origin_name, diff --git a/paddle/framework/threadpool.h b/paddle/framework/threadpool.h index bcd8190755..c644e7d296 100644 --- a/paddle/framework/threadpool.h +++ b/paddle/framework/threadpool.h @@ -159,5 +159,14 @@ class ThreadPool { std::condition_variable completed_; }; +// Run a function asynchronously. +// NOTE: The function must return void. If the function need to return a value, +// you can use lambda to capture a value pointer. +template +std::future Async(Callback callback, ARGS... args) { + return ThreadPool::GetInstance()->Run( + [&] { callback(std::forward(args)...); }); +}; + } // namespace framework } // namespace paddle From e138bcf45080a0288293197195d453283c1396bf Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 4 Jan 2018 15:16:14 +0800 Subject: [PATCH 26/35] Update cmake of scope --- paddle/framework/CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index b4458eb955..fb8c9ab96d 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -26,7 +26,10 @@ nv_test(lod_tensor_gpu_test SRCS lod_tensor_test.cu DEPS lod_tensor) cc_test(variable_test SRCS variable_test.cc) -cc_library(scope SRCS scope.cc DEPS glog) +cc_library(threadpool SRCS threadpool.cc) +cc_test(threadpool_test SRCS threadpool_test.cc DEPS threadpool) + +cc_library(scope SRCS scope.cc DEPS glog threadpool) cc_test(scope_test SRCS scope_test.cc DEPS scope) cc_library(data_transform SRCS data_transform.cc DEPS math_function tensor framework_proto) @@ -70,8 +73,7 @@ cc_test(var_type_inference_test SRCS var_type_inference_test.cc DEPS op_registry cc_library(selected_rows SRCS selected_rows.cc DEPS tensor) cc_test(selected_rows_test SRCS selected_rows_test.cc DEPS selected_rows) -cc_library(threadpool SRCS threadpool.cc) -cc_test(threadpool_test SRCS threadpool_test.cc DEPS threadpool) + cc_library(init SRCS init.cc DEPS gflags device_context place stringpiece) cc_test(init_test SRCS init_test.cc DEPS init) From 7e10b8181804295ddecf9b10201bb12e2d88f745 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 4 Jan 2018 15:17:53 +0800 Subject: [PATCH 27/35] Fix style check --- paddle/framework/threadpool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/threadpool.h b/paddle/framework/threadpool.h index 31c12e9e1f..3ac345851c 100644 --- a/paddle/framework/threadpool.h +++ b/paddle/framework/threadpool.h @@ -165,7 +165,7 @@ class ThreadPool { template std::future Async(Callback callback) { return ThreadPool::GetInstance()->Run(callback); -}; +} } // namespace framework } // namespace paddle From 3b5e4e0a83e0229a70a8ef232d41f01e5c965896 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 4 Jan 2018 16:15:52 +0800 Subject: [PATCH 28/35] default disable use_mkl_packed --- paddle/utils/Flags.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index 10db81cbda..ea47cf23eb 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -28,9 +28,10 @@ DEFINE_bool(use_mkldnn, false, "Only support CPU training"); #endif #ifdef PADDLE_WITH_MKLML -DEFINE_bool(use_mkl_packed, true, "Default use MKL Packed Optimization"); -#else +// TODO(TJ): change to true when fully confirmed DEFINE_bool(use_mkl_packed, false, "Whether to use MKL Packed Optimization"); +#else +DEFINE_bool(use_mkl_packed, false, "Not to use MKL Packed Optimization"); #endif DEFINE_bool(parallel_nn, From b585c93ab013f0c0fd860f1cee0b0cc4e4760398 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 4 Jan 2018 17:36:52 +0800 Subject: [PATCH 29/35] Default use one thread in fluid --- python/paddle/v2/fluid/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/paddle/v2/fluid/__init__.py b/python/paddle/v2/fluid/__init__.py index e115529665..5e01b87198 100644 --- a/python/paddle/v2/fluid/__init__.py +++ b/python/paddle/v2/fluid/__init__.py @@ -1,3 +1,4 @@ +from __future__ import print_function # import all class inside framework into fluid module import framework from framework import * @@ -36,6 +37,24 @@ def __bootstrap__(): """ import sys import core + import os + + try: + num_threads = int(os.getenv('OMP_NUM_THREADS', '1')) + except ValueError: + num_threads = 1 + + if num_threads > 1: + print( + 'WARNING: OMP_NUM_THREADS set to {0}, not 1. The computation ' + 'speed will not be optimized if you use data parallel. It will ' + 'fail if this PaddlePaddle binary is compiled with OpenBlas since' + ' OpenBlas does not support multi-threads.'.format(num_threads), + file=sys.stderr) + print('PLEASE USE OMP_NUM_THREADS WISELY.', file=sys.stderr) + + os.environ['OMP_NUM_THREADS'] = str(num_threads) + read_env_flags = ['use_pinned_memory', 'check_nan_inf'] if core.is_compile_gpu(): read_env_flags.append('fraction_of_gpu_memory_to_use') From a4024a5f3dc852bc955f8c4c7aa30750d8da4c6f Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 4 Jan 2018 18:36:51 +0800 Subject: [PATCH 30/35] "remove cudnn devicecontext" (#7207) * "remove cudnndevicecontext" * "remove unused init code" * "fix hash functions" --- doc/design/support_new_device.md | 14 ++------------ paddle/framework/library_type.h | 4 ++++ paddle/framework/op_kernel_type.h | 13 ++++++------- paddle/platform/device_context.cc | 26 +++++++++----------------- paddle/platform/device_context.h | 14 +------------- paddle/platform/device_context_test.cu | 15 --------------- python/paddle/v2/fluid/executor.py | 7 ------- 7 files changed, 22 insertions(+), 71 deletions(-) diff --git a/doc/design/support_new_device.md b/doc/design/support_new_device.md index f54b2b3694..4c5f10e2ec 100644 --- a/doc/design/support_new_device.md +++ b/doc/design/support_new_device.md @@ -48,8 +48,8 @@ Fluid uses class [DeviceContext](https://github.com/PaddlePaddle/Paddle/blob/dev ``` - /-> CPUDeviceContext --> MKLDeviceContext -DeviceContext ----> CUDADeviceContext --> CUDNNDeviceContext + /-> CPUDeviceContext +DeviceContext ----> CUDADeviceContext \-> FPGADeviceContext ``` @@ -79,16 +79,6 @@ private: }; ``` -- CUDNNDeviceContext - -``` -class CUDNNDeviceContext : public CUDADeviceContext { - private: - cudnnHandle_t cudnn_handle_; -}; -``` - - ### Memory and Tensor diff --git a/paddle/framework/library_type.h b/paddle/framework/library_type.h index 7707799cae..1e30848354 100644 --- a/paddle/framework/library_type.h +++ b/paddle/framework/library_type.h @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#include namespace paddle { namespace framework { @@ -41,6 +42,9 @@ inline std::string LibraryTypeToString(const LibraryType& library_type) { inline LibraryType StringToLibraryType(const char* ctype) { std::string s(ctype); + for (size_t i = 0; i < s.size(); ++i) { + s[i] = toupper(s[i]); + } if (s == std::string("PLAIN")) { return LibraryType::kPlain; } else if (s == std::string("MKLDNN")) { diff --git a/paddle/framework/op_kernel_type.h b/paddle/framework/op_kernel_type.h index b06002096f..053897784c 100644 --- a/paddle/framework/op_kernel_type.h +++ b/paddle/framework/op_kernel_type.h @@ -26,13 +26,12 @@ namespace framework { struct OpKernelType { struct Hash { size_t operator()(const OpKernelType& key) const { - int place = key.place_.which() + (1 << LEFT_SHIFT); - int data_type = - static_cast(key.data_type_) + (1 << (LEFT_SHIFT + 1)); - int data_layout = - static_cast(key.data_layout_) + (1 << (LEFT_SHIFT + 2)); - int library_type = - static_cast(key.library_type_) + (1 << (LEFT_SHIFT + 3)); + int place = key.place_.which(); + int data_type = static_cast(key.data_type_) << LEFT_SHIFT; + int data_layout = static_cast(key.data_layout_) << (LEFT_SHIFT * 2); + int library_type = static_cast(key.library_type_) + << (LEFT_SHIFT * 3); + std::hash hasher; return hasher(place + data_type + data_layout + library_type); } diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index ea07f2e002..4bf643e048 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -127,15 +127,21 @@ CUDADeviceContext::CUDADeviceContext(CUDAPlace place) : place_(place) { eigen_device_.reset(new Eigen::GpuDevice(eigen_stream_.get())); PADDLE_ENFORCE(dynload::cublasCreate(&cublas_handle_)); PADDLE_ENFORCE(dynload::cublasSetStream(cublas_handle_, stream_)); - PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); - PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream_)); + if (dynload::HasCUDNN()) { + PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); + PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream_)); + } else { + cudnn_handle_ = nullptr; + } } CUDADeviceContext::~CUDADeviceContext() { SetDeviceId(place_.device); Wait(); PADDLE_ENFORCE(dynload::cublasDestroy(cublas_handle_)); - PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); + if (cudnn_handle_ != nullptr) { + PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); + } eigen_stream_.reset(); eigen_device_.reset(); PADDLE_ENFORCE(cudaStreamDestroy(stream_)); @@ -160,20 +166,6 @@ cudnnHandle_t CUDADeviceContext::cudnn_handle() const { return cudnn_handle_; } cudaStream_t CUDADeviceContext::stream() const { return stream_; } -CUDNNDeviceContext::CUDNNDeviceContext(CUDAPlace place) - : CUDADeviceContext(place) { - PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); - PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream())); -} - -CUDNNDeviceContext::~CUDNNDeviceContext() { - SetDeviceId(boost::get(GetPlace()).device); - Wait(); - PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); -} - -cudnnHandle_t CUDNNDeviceContext::cudnn_handle() const { return cudnn_handle_; } - #endif } // namespace platform diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index 2b366e6383..609ea4bd3a 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -103,18 +103,6 @@ struct DefaultDeviceContextType { using TYPE = CUDADeviceContext; }; -class CUDNNDeviceContext : public CUDADeviceContext { - public: - explicit CUDNNDeviceContext(CUDAPlace place); - virtual ~CUDNNDeviceContext(); - - /*! \brief Return cudnn handle in the device context. */ - cudnnHandle_t cudnn_handle() const; - - private: - cudnnHandle_t cudnn_handle_; -}; - #endif /*! \brief device context pool singleton */ @@ -151,7 +139,7 @@ class DeviceContextPool { struct Hash { std::hash hash_; size_t operator()(const platform::Place& place) const { - int pre_hash = place.which() + (1 << LEFT_SHIFT); + int pre_hash = place.which() << LEFT_SHIFT; if (platform::is_gpu_place(place)) { pre_hash += boost::get(place).GetDeviceId(); } diff --git a/paddle/platform/device_context_test.cu b/paddle/platform/device_context_test.cu index ca10cf3463..767fe9b24a 100644 --- a/paddle/platform/device_context_test.cu +++ b/paddle/platform/device_context_test.cu @@ -49,21 +49,6 @@ TEST(Device, CUDADeviceContext) { } } -TEST(Device, CUDNNDeviceContext) { - using paddle::platform::CUDNNDeviceContext; - using paddle::platform::CUDAPlace; - if (paddle::platform::dynload::HasCUDNN()) { - int count = paddle::platform::GetCUDADeviceCount(); - for (int i = 0; i < count; ++i) { - CUDNNDeviceContext* device_context = new CUDNNDeviceContext(CUDAPlace(i)); - cudnnHandle_t cudnn_handle = device_context->cudnn_handle(); - ASSERT_NE(nullptr, cudnn_handle); - ASSERT_NE(nullptr, device_context->stream()); - delete device_context; - } - } -} - TEST(Device, DeviceContextPool) { using paddle::platform::DeviceContextPool; using paddle::platform::CUDADeviceContext; diff --git a/python/paddle/v2/fluid/executor.py b/python/paddle/v2/fluid/executor.py index 1d6c594b41..1b2075dcd5 100644 --- a/python/paddle/v2/fluid/executor.py +++ b/python/paddle/v2/fluid/executor.py @@ -65,13 +65,6 @@ class Executor(object): p.set_place(each) act_places.append(p) - # TODO(dzhwinter) : consider that our fluid tests all written in - # CUDAPlace(gpu_id), this will be changed in the future - if core.is_compile_gpu(): - core.init_devices(["CPU", "GPU:0"]) - else: - core.init_devices(["CPU"]) - # TODO(dzhwinter) : only use the first place self.executor = core.Executor(act_places[0]) self.places = places From c7bd77792edd5d5e9b33c98ba2c99d9f5c3e6188 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 4 Jan 2018 11:59:48 +0000 Subject: [PATCH 31/35] Support the link of inference library on mac. --- paddle/inference/CMakeLists.txt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/paddle/inference/CMakeLists.txt b/paddle/inference/CMakeLists.txt index 73c875409a..8437b2b219 100644 --- a/paddle/inference/CMakeLists.txt +++ b/paddle/inference/CMakeLists.txt @@ -30,7 +30,18 @@ if(PTOOLS_INC_DIR AND PTOOLS_SHARED_LIB) endif() add_executable(example example.cc) -target_link_libraries(example - -Wl,--start-group -Wl,--whole-archive paddle_fluid - -Wl,--no-whole-archive -Wl,--end-group - ${PTOOLS_LIB}) +if(APPLE) + set(OPTIONAL_LINK_FLAGS) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + set(OPTIONAL_LINK_FLAGS "-undefined dynamic_lookup") + endif() + target_link_libraries(example + -Wl,-force_load paddle_fluid + ${OPTIONAL_LINK_FLAGS} + ${PTOOLS_LIB}) +else() + target_link_libraries(example + -Wl,--start-group -Wl,--whole-archive paddle_fluid + -Wl,--no-whole-archive -Wl,--end-group + ${PTOOLS_LIB}) +endif() From 040dc59b0f3558baee90627936716823569700d5 Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Thu, 4 Jan 2018 21:57:16 +0800 Subject: [PATCH 32/35] Correctly handle image operators --- paddle/operators/batch_norm_op.cc | 3 ++- paddle/operators/conv_op.cc | 7 +++---- paddle/operators/pool_op.cc | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paddle/operators/batch_norm_op.cc b/paddle/operators/batch_norm_op.cc index 98db28ddee..dd7b038b00 100644 --- a/paddle/operators/batch_norm_op.cc +++ b/paddle/operators/batch_norm_op.cc @@ -64,7 +64,7 @@ class BatchNormOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(x_dims.size() >= 2 && x_dims.size() <= 5, "Input X must have 2 to 5 dimensions."); - const int C = + const int64_t C = (data_layout == DataLayout::kNCHW ? x_dims[1] : x_dims[x_dims.size() - 1]); @@ -78,6 +78,7 @@ class BatchNormOp : public framework::OperatorWithKernel { ctx->SetOutputDim("VarianceOut", {C}); ctx->SetOutputDim("SavedMean", {C}); ctx->SetOutputDim("SavedVariance", {C}); + ctx->ShareLoD("X", "Y"); } }; diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc index e65a5dce52..ad84524e17 100644 --- a/paddle/operators/conv_op.cc +++ b/paddle/operators/conv_op.cc @@ -44,14 +44,12 @@ void ConvOp::InferShape(framework::InferShapeContext* ctx) const { paddings.size(), strides.size(), "Conv paddings dimension and Conv strides dimension should be the same."); - int input_channels = in_dims[1]; - PADDLE_ENFORCE_EQ(input_channels, filter_dims[1] * groups, + PADDLE_ENFORCE_EQ(in_dims[1], filter_dims[1] * groups, "The number of input channels should be equal to filter " "channels * groups."); - int output_channels = filter_dims[0]; PADDLE_ENFORCE_EQ( - output_channels % groups, 0, + filter_dims[0] % groups, 0, "The number of output channels should be divided by groups."); std::vector output_shape({in_dims[0], filter_dims[0]}); @@ -66,6 +64,7 @@ void ConvOp::InferShape(framework::InferShapeContext* ctx) const { dilations[i], paddings[i], strides[i])); } ctx->SetOutputDim("Output", framework::make_ddim(output_shape)); + ctx->ShareLoD("Input", "Output"); } Conv2DOpMaker::Conv2DOpMaker(OpProto* proto, OpAttrChecker* op_checker) diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 50057eb648..d3cf5fa638 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -58,6 +58,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { OutputSizePool(in_x_dims[i + 2], ksize[i], paddings[i], strides[i])); } ctx->SetOutputDim("Out", framework::make_ddim(output_shape)); + ctx->ShareLoD("X", "Out"); } void PoolOpGrad::InferShape(framework::InferShapeContext *ctx) const { From f3c42f607c91f7fe6c9c3b04151baab60e92c16c Mon Sep 17 00:00:00 2001 From: Siddharth Goyal Date: Thu, 4 Jan 2018 13:52:04 -0800 Subject: [PATCH 33/35] Add doc for gru_unit op (in fluid) (#7151) * Add squared error layers doc * Add doc for gru_unit * Remove cdot which isn't supported * Update layers.rst * Update layers.rst (minor) --- doc/api/v2/fluid/layers.rst | 6 ++++ python/paddle/v2/fluid/layers/nn.py | 48 ++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 004ee2d8c8..a7c8670f66 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -307,6 +307,12 @@ sequence_expand :noindex: +gru_unit +-------- +.. autofunction:: paddle.v2.fluid.layers.gru_unit + :noindex: + + lstm_unit --------- .. autofunction:: paddle.v2.fluid.layers.lstm_unit diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 1c1c09dd28..6883630ac6 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -236,21 +236,47 @@ def gru_unit(input, activation='tanh', gate_activation='sigmoid'): """ - GRUUnit Operator implements partial calculations of the GRU unit as following: + GRU unit layer. The equation of a gru step is: - $$ - update \ gate: u_t = actGate(xu_t + W_u * h_{t-1} + b_u) \\ - reset \ gate: r_t = actGate(xr_t + W_r * h_{t-1} + b_r) \\ - output \ candidate: {h}_t = actNode(xc_t + W_c * dot(r_t, h_{t-1}) + b_c) \\ - output: h_t = dot((1 - u_t), h_{t-1}) + dot(u_t, {h}_t) - $$ + .. math:: + u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u) + + r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r) + + ch_t & = actNode(xc_t + W_c dot(r_t, h_{t-1}) + b_c) + + h_t & = dot((1-u_t), ch_{t-1}) + dot(u_t, h_t) - which is same as one time step of GRU Operator. + The inputs of gru unit includes :math:`z_t`, :math:`h_{t-1}`. In terms + of the equation above, the :math:`z_t` is split into 3 parts - + :math:`xu_t`, :math:`xr_t` and :math:`xc_t`. This means that in order to + implement a full GRU unit operator for an input, a fully + connected layer has to be applied, such that :math:`z_t = W_{fc}x_t`. + + This layer has three outputs :math:`h_t`, :math:`dot(r_t, h_{t - 1})` + and concatenation of :math:`u_t`, :math:`r_t` and :math:`ch_t`. + + Args: + input (Variable): The fc transformed input value of current step. + hidden (Variable): The hidden value of lstm unit from previous step. + size (integer): The input dimension value. + weight (ParamAttr): The weight parameters for gru unit. Default: None + bias (ParamAttr): The bias parameters for gru unit. Default: None + activation (string): The activation type for cell (actNode). Default: 'tanh' + gate_activation (string): The activation type for gates (actGate). Default: 'sigmoid' + + Returns: + tuple: The hidden value, reset-hidden value and gate values. + + Examples: + + .. code-block:: python - @note To implement the complete GRU unit, fully-connected operator must be - used before to feed xu, xr and xc as the Input of GRUUnit operator. + # assuming we have x_t_data and prev_hidden of size=10 + x_t = fluid.layers.fc(input=x_t_data, size=30) + hidden_val, r_h_val, gate_val = fluid.layers.gru_unit(input=x_t, + hidden = prev_hidden) - TODO(ChunweiYan) add more document here """ activation_dict = dict( identity=0, From 7508d52b3c532208d14b96465084107288e2f461 Mon Sep 17 00:00:00 2001 From: QI JUN Date: Fri, 5 Jan 2018 13:03:08 +0800 Subject: [PATCH 34/35] add memory optimization design doc (#7206) --- doc/design/images/control_flow_graph.png | Bin 0 -> 85311 bytes doc/design/images/dataflow_equations.png | Bin 0 -> 23064 bytes doc/design/images/deep_learning.png | Bin 0 -> 40605 bytes doc/design/memory_optimization.md | 217 +++++++++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 doc/design/images/control_flow_graph.png create mode 100644 doc/design/images/dataflow_equations.png create mode 100644 doc/design/images/deep_learning.png create mode 100644 doc/design/memory_optimization.md diff --git a/doc/design/images/control_flow_graph.png b/doc/design/images/control_flow_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..3579998e58d07abc50bd3332128d4733a391cb3b GIT binary patch literal 85311 zcmZ5|cQn=iAAd%2|Szu))#{&0@tT;A{3>-l<)$9Rc&qM=N3o8k728#hQ)ppd6GZr}>uxPj9OxP|?R zwu<=o8#nMt>=YE9s3<6~K5=t?Vdr3d;|4e;=9#fA@ENsCgS!no$Dp=@NT|mLS(3XX ziO)seRn-U1&51gtsEE#h)~$|q2Ge>6!NLva?=%&k_zP}iQg^h}zGme2AH#)nei`E>go4)g_cNXTK6jB@Qme#7!|KXnQTgiw5j+m( zcj2>sFC1SCf<<=2fceN`y;ks1hk);wMa#*r?lJ)}b}v@=md4;WpJ~5@r;^7FbpBBy zJ{2sRFnz23a`WT*D$*j;bM9@q%dYuFu49{R=z-(c!lCqBnUGy`qf_38ch?rRI#fR< zvlVJ)Xv#oTitk=(eLL2Im0yiK#yQ z|Mi2A6n=uU_o+$2G#$b4z;{Y=M{7$LHKs3Lq$Km32Y{*B)rL;_8`3J zwz?i3g%$UsTMM*Pwnf3Dw}hxmsP9CYzI#E(xYjDQFHx2Lt*yB|>7u8{H2CDpVAVaV zaG4<53Wl@TV%jn5y-%S=mQ=Cal~OiImvHd1JKj(}GOJIq|B7w`P~Z_tcEECyV(=Yr@$3c}{N{H=T8vmK&k?#I$Ob z?qt{N4xX5vY@P`n@t(cR{~5TVyI1ShZ7|SKq4sslLC@X$8H3}kW!m2(>75asOb4!e zE4pM#0V%!dYo%yoDShYrq)Nr9H!&|yp5M`kffS>I(cx8mO@=%SXn&^{M2z*1go z>P`_FMAPRnu8>KD(R>7ZPac$l6$tAm^6sG>O(tfG@d zb&kL29=ro3Ypy=EBiEMHrYb5Qmm>?WEbw~kh>OvpUC{B*-6CJ^UsuK%+qW*5%b`atLc)px&fgO2j*3&$pR6k9$X-B5wZ>Ab?( znj=W!f0ius+s(=i_inc`X#xTW=}^H2Kl+8P`_jkgn&@#g0tjhRjaj<#FI;DK8KH z?Kkpo$+_`Ue}Dc!0f1(IaL$QH2Z_KS zVQXKY1HX-;D)O<9#6H=KmLe^>V*siD9nArtj!4JP>bv*f8{K}hsQ4E9X_L%~;(wnG zBTwVLEv)cOS2!x7RVYfo^drsRXH(<4GvjEot8?b)eLQ@S$R^U@lW5vBU0q@z@ZX<% zar2GAfFy63fLorT1?`=RkN;l0Ooa6bMcQIZpQ1(BB|~!P-?d2NS~6Ws9lYCpbO+2u z!A&X5C17~R_-aY5TOc#G;zQ6STJv&z-sQA1_q_OY_wr)5CSYaZ@_eBooF(9uM)BQ| z@M6C^K6nn26LFVkLzfQch~vzblgttUczVF>#aiG+;VN&YqgQFq=(I2Yma#;s-(mgr zGVjX&G`(0gH8__&`zq_8;XiGi>bM3x-#r`L^=LVtZJ8TQ)qjz$+|(5q!H^x+*8@NX zrFH!3mc7`^RyXB4Q?yMwsSP}>B^@~_S-_O+90%+kGh4@0^sz~KBo7ed(nO@+$@-Lh z9rRx`xLKaSnjX~<;k9){a2;#UZ6qi|erku_4izQD^JxK7%(t{e@2WqQr~!j{N$82JS zW2l8JZ$GZPI&Iql*rOPjG{BD#eCA_@AK5`n{EUY0Z*T#)6HVkDt8^`Y6r zhsw-8`F=}Oxl>5mBH?I1t0=wM;1BQ$Rt_XIR>1&0S=W4iAbaUuh1IN5l$3RsjhBUIT7&3E;XP ze%JLD`{AN++^5@IA%1DT%Qs6w3+c>T0;o*Xt|vBc19HzIE6j@ylI{ zf>oD!HyibCC|_^CF#Lv%({iKwdzTGt?Bg1u1e(8`E|LXK!LaM5&rV5~tEBzJ!=Ypv zpVoiBp}1i+#ZXD*HZHs79sJItpTT5ax+Xr>QGKjJ8@=Jdu-*YSY~ubZ`<(K3`CPG}%ko{SqvtCAz9U zX<`7Ydv?JjqU)rxXTL{2M0*MW8`klWtQO|qqEV?A22>n!;@0aKcI5@(Ws|t8&z|eQ zKb@#;{?6n+AUL<*FR-v(HKq~M%F5p}Fz3B62gtneZOc`!?`+K(W$+pm9s8rVSdL0ZY2>V`Q&;Hd&8Hv20JZ{JW#nHc1;tjTfxq-UyWfKMiE zrOdBE$z6y1evv!!Ves-}<6U3f+Iip2hq&xbv`2%3G{va=>9!^V{_BFK!5-hEGVunE zgjP~$tMMepx?PA`Q+cAOvXU}UFevftb3OCW8a6RNYWHV#DmH5n6h2Lr^E<>TQC-xo z8wtfAi+Kw7pfhU6Hg+#dp#!5H)t^eQYmyoTE>I>RsKaF?%MTEF06rk6UDFpvX2sA6{zp`L5jc>wlgb{rLaHOwvNn_ht(E3j zSV~p{?`(814Lm<=fE`WSBt^4XWjZ(e)J~fCRdYx0gEZjZSkBs5sxH_d7(*d%9v@oHWvN}k%szOBSLr8tR6U1{QN_u zC?~()Wg-x)U?J7!-g`Gy1Y{|=C9zp7Tu+}eMrIe0lQkq_)8cahseKw0@QW4Z+95oyf`E`EGg3W}9mD)3Rl4sa}TL z$h(pxRYR)n+|E_Sm&RmrUT`Jv-hVwg(-*=WGUL2hje{E0H`#N-E;^OW_9$#gMvpv?4rMd|gq zVS_wEzj68Cn#9JGVnF==RgMlW`OjB!>NNPg1$wC1Ocj7=ib&Pf2!dQPp(gyB18_qe{V6nsVWhl z`Eb&xvMF>*CQVH_u)x3ukyfNsg}$Dn%BWde#ohKNqWqT`7i=&8M3=$ey3~ut@t!K? zR`j`N0r?6eMLJHpU1xZQM*1oXiPFg*zRlasO6GDM7+n7Io?nep8kuT5BZoir!jh<+ zToR?4Il{!)-LIP7Iky>AWB&FM4tN?G-J0o0tIZDDs^N)yRG}J(9uY(;GbXu#tkBJ8J)hoqhO2c8G6V=DLnaE zZ`$y>Vv@7A-kAY^j}e{uAo(lJa>gS>go2nf;eeR!=ab}Q7Q%9sJ191)BQ}t07f`5a z@N}D(m)~(%j5{4Jqmko2SzsG(nQvEY8_WuZNZRq9rU)vsl;w*-!eAYB>I;W&O6*LT zwCL7cZ_vd1}*h+bg*D_gHG`!EE(0Ay)EvZQ5g-2ILPmvl=Wd#21&T9 zM0O0k|MaZpIyMhD051MnY?Q4OT_?fat5lhI^F?xm{iMDQGV3GCn# zTPYp~HKcyWs4oX_a2OUi-SfE1?X-6MGKUq~yx*_go5b2X+Iwvd+d4%+jtK z=Z(NP8|8h@nKApZ0;n=nt;K(9voNN*JXb8a8xvv{+PW$=#U_a49$bWUF-wvK9C_rc zVRoc~rdrMKLlJ3pF3Eje*ZUhrjH3~YN7Gbx5yV~{8>FpfaB)Y11473MVqkmF*Rfbo z3Q)a`u`K3L9WT14Vk=&`bFw{{20xi5fJnj8wFtP9qSyOt^j$ibsOzqHCGETda zq-~!gp^;q5i&dJ!GpX9iH5X~AcZ>xjhkZ%Lf9r)fE-<$!53K#` zF;Uy1j<~|05UHp!g9!iXcHXDl7ZgtOiisW?ImU?;kzhOgyP%ID58?xIrBGj|3(1Ny zeW&dOAZ(Bz=6ns~ZV2pz!Uyw?L}FC4BUBxez$4+4^K^q~bR;zbQ{n`&nem7o*3(t6 znOWatI9dh?ehmoiYyNl8wD7@R?S$PF?D*45XfC&vyhvYyEXjQ&DRME(y_W4Rbo6bz zjPDdJBLuvj9|eP|?@w(Ij;8Xu-#PeGy(v@XBxzj6>zL zXnAVP7@edR`$*A(v%ApkdYt6O4+G$k=7jizm7o>kraCIdejRNxh$OjQ)fuZ?422iv z%Iab`i<7gcsAC8{n{a{aguYF-RzkkiQV8*TCy(zJe*Sm8OHeCSR9vnVDSe;e?@gIM59m)yOK+*Kg=GZLOU5kgEC_${0K6>;(PWW<{mf zmCTGhkMLi}lM`{ULD;orW|R!#0r){>t{_lev)e@ZpIj~?!=@@tCT%&iL28iRu*y(7 zhjZh5)v8XN1hFM%7A@8ld+6+WC@xV$*_-t`&stC!qutn}OUTbP|u_*fig zKIGQVUt9Jo1#=c^ig?eA#JyqzuAsR&Ao{Kd5pN5}$;>`JW&7(8R=NUH@=WrS=8Ikj zZ5CHOyZ6ZkXm-qc-rhF1XucMN zfDT;ErWcP6&|{j;*5BoBDO#glg=V|$1MO6KDJ!dt_colbkA*h;QGt2Elhyo>;J|zZ zD@fAUa(K9HBwQuOgkk9O-sm_N?RSairs5nOoFEYF2_?HPd*oM6&(HN6A9qGo1svBh z9V%1|JEA^t*G?GNdt#%YdVGa=)^hxuW zW^3*48OnaNoUyI`{@i{fCDW7UpT)Iup^hB~0dRT#9J?WRDv!CIquA?TdI_IRZx3TwPD4mxwlc&x# zVD#IzH=U)Ohzxl}^x2cgQsvt#jJ$!G9FG8HaG8*MwmzZi-kbSUWyPyOW9 z)YoEg1W945!xKbWM0ruxzdRts)&`reZEi;$Nfh#nbG!v_M5oMxy;FB2Cyw1TCs3(vyEC1c8~Md`~flpf%oym^y6b8*=d2`(Z>U_bjp8&w-xWz4!l; z;t_rqyYQHbQ#?)(zRMAoWw6RD&<{9SWh4Y+pKYkKicn^@%a^|sa#J5}9IF6|kMx{0 zuBBimazrq1>g4Q2+f+3jih_iDB77r0eAx95rFp*o+!S{Bf@mK!m(!z};R~QQgjgnx zrAjIB$H8WFhs0-rscO;K1N1-pqCyVfNz3o4vbJNSn8G5c*$gU;g~Uh4^M20kr2CF+ z6i4NBTC5}H8&D1od_tco+q@Wtw%;r9{+qf2X99_{_%@4U3&VGqW5_y!^i~oF$ zXW+pxrkEo`siKn4ZiVyatwR1SACy4Xvo4X5d!LF&>(>amtsqRRYE(sem!nf(@7qnY z%S?2@H*AA@3MGlWcCi?vCPfLRdB2DXlox}3lYMvjibV7`x>x;g*8vp2LE=G=UWs5@ z@yHy;gs;Gs_Yc?ZheBrWN_iMJRmt%ncs%r~_2jql*+Kw|6wd@{JWp0rYkLN?gpk)SQx|b#jTyzl<@oU!bdMtL`NFH^nC(#4`4IbZXXqn9 zbdsUa@K* z`G*3IDjTRY!0)KOV+kR@nK2!YIc(>cyGds>oKw7Sl?08&Y{|!0jdkl~Cn#g%_iNuj2Ujsw%#g#-L#_?djNGxqv#w`%EL?2;61hcFqXIceDQ5s{x{WoR=V}}wxVDYJUVYb&x zZEaL$6-Xd>E=1Ad*4LuP|CnJkc7K*_-5rK-Gz+a5lopiM!Fmn%x6}FKW+gZz8SXVs!=J%3^2;a*BpdoiF1>G<_OC3`QKUT2|x{gBOEB-w#SagMxdhW;|gyLiHWrefpZ7bPpNIA2?6bt`|Y$= zpu?o7?9uPLlj^ghE1@AAF!R%O?c6kwg(j2|m&(6tA$1&8_O`ndx3szJ&MQU}a_WP3wBs(O z$md3<(u9IHirwO_Bn3;sqNIQ66UzF!QO|#ahqXJ)EqB;ud<>OOfxAtyonJy8T0BV@ z5idKzX`P&yx(>)5O|HJ1P)QC}svHZT91kZUlo>p&0pHh`!jjQ^%udffUDt>A3^;@7 z2dnW+Ye2Sm+HUFZq!QTPOdgzH$2wXfMi^n9T1B2iPUmOWJJcCHCvG|p+ZbbEoL&R3 zjZAAYW)=?U$tU?%Up~ZDv;;Zl+=}u%ZW#%Q2U`Lw=n9d5vxV6jTk#3k!t_A<*R%Y4NvFv5rNlRpZ$X zr7ehc;I6-Uhqc4p`iJwptM+Isweg=o^9X-wo$gmWfsWXd@@9AMEXOB|nAP~C_}KMi z=cxHN(Hk^KVMnWl{?A`|l(2!I#xvOXdmtIef9qA&j`;G-PI0{fyJ;n$;O*!ha>DVNjQh@g9PI>B<)5vqjb^WaJBuAbIRZF(p=QWU$qKBg4AJr>|`To^kgcZ*Go zXZD`jqViT@t|KPf8>40}EL`vrIuI>qOme-S4+6=5{sC`4(>1O>MQH}kWP8v1%=Ph> z{*a(Aq~Cu4AIo`ZRh9isgwAb$Sb?niq0~hKG^d5`_vPb)E{7}(mZlq1+Y40dTku|h zZI=?P51U$_@#LpYoMY>>^BOd^u&VJ@oQX98Z}+3zZ~2@RHpPjhspGr!)(b=cz4c*6vfI zFAU;wG{j@?L@bsmnkD1}rl%-P^*V73a2m6GUq&OyNSRgkhQ9YiyIaT8I!=uT6H>AZ zP-X_{%hrzxBwBiDgy&y6*yIJCV7cp(1}qRMJ?u6pY~*v1n8|&d9mj~{>iZ<)F|Vrb zzoPwwjWwf9%YFT7{A9l|DE=~*k&S%y=eW&sDhz-pkN5yM1;|qdSqv4iQ*Xp4xeo>e z&iom$k*PFv|Mt+ZC7xl-7Ab%U`(gVvhq)X2q+llhE=JB98eKD5aF!mwhan8MPBpGM zAEL(H@GscXPmqZ-Yt&b?NWpLW_?KmQ8vQ7scen9iFprzVx0TTCh)pE#=8HeC6_lzV zv_L8^c7`KZ#d2DrCQH^Vc^rx8)Z1B)G4)LMNde3j;$eP>jNf5(VXV+-)$HVZ<5f6% z%l!vs0V>#-O=oK~{AYZ#1lg@?9fzM%*!oEd^133VFypH=xC9mGsFlD6;1HL90`?X6 zC9Ftxq=p>JZFd`Z?&UIG9`P-GM}?W7w9#z?&YA7IQ8d?Yda?iO?7edC1$W*MNWf#5 zfMAuJ9%Ra_;&XzCh)tAD_Jd(1|3eycb74x(xRQctjHtTxsdmXHY*}VE!2DO1e-4OC z#R_P|S&H2JDi{fMNxO>r&32VpfFz+|rI#X0{%oNUugHGe4wuv-Va(h7oF=z@`R8s~ zLUyy_BRN9TqrXbs0-0iZPds+iT~VbZcIJu8@d8!@KA+b-W`tHj7ddPcrrPdAy=HwX9>>o{J(@g1FakgwbqE$N?BZopSaDDm z=897JxXzc3O}=;9P#jX*saADx)7aQCf$TJKFTlba)71G>U+r)Hh8e8-ee}T_YiB$Dj?4tXp z_*~mPlU``+1!vxFHaR<9lxlsPi7(PHcY1VpD4|AJ`!K3wfO_=%UpCuX5_a?B z`Sk(##fqDrv-ne;NB^>&O@a>L8a*&jP$swXwb?n&(u-2ttV4bUqKQ^-R&Po8Y+_ZM zrz^U`WnG~wY6wukl1;pYP37BaIswDz^?qiV0>l>8-HpWIW)f83#?e8QtZ3kq%#{2e zTl40s7Cztq^|Z8y_!#`q19>Bm>x*eFnCMZtaDFI>iu;e+S#f1%%CIEvsCDp8|0~#7 zmZ=z-Rn)S zzG5S+2NW{8K16N25LP60pHev4Hs$lb{^t|^2t;=9&Jy|J!smx6XK~f&V@C>MWY7RB z7wtpCB**MT_ilVPIX!ZkmCic4WuGBp|9GG^6L0W1xZtbF2`U~WIQHfx>Yoss<_?IC zsad~4AeD=6-m4Am3F{%kP2lS18&0_&Bo9e-i1@&9TY;>7*~lbk{jn7h4^r27cP9O_ z_t3y#rSkzQ_CxbOGXYl6$H#$?ls}RI2#wPDu(`u(sVwWR3 z)CfMlulxrRG}y@3q;f`JHZmp`MvoOB@=rMR)z^BkHX% zDoobGG>T+%5nbG+#dd|gzl|x+uw9(c_gU@VzlIeh6#!{Va(wX=_ImrxELI)zCIZa| zDHbcguJHc-mb3kxAMQijJJ|9hYRz$OBYUdPE~@FjM8)_9OD1{NbIG3c&SAYrNlwc# zVmnu&pVEs_(pOcIOiwvpNIc>HTkx^Vy#1yZ$v;M;m`3tCo-U;Q1rssIeMtJaO0faN zd9k~3r8atxs&G|zwjlX49c%QDf2_&WD%w0Z@mX~$sh-#%Pm8)=|H1Ef8$ko3b7Q&W za{sVRj;je8&c#@jcQV+Ik z1rxgqe9;u?BVt5uSmpC%2P^h%HXoKadvfKxvWRMBWn*-#(mKxPk}g8f9+GFAm2+ht zWA{MmDc_Itt2QeAyA^MakSl8#wywakF@Y7}cBY2{yu7imR$IjF$;BQS6(xsA>s0)_ z@o%ibE}s|&g4E=MJl(-U5nA=SrAS@|3Vq9%hxOYvv!Ac(c>{^qk1|E$M)q42y;bpC zY4I!;7+b7x;-IaCZNJ{^kSo;dje4IdO36V+xKS(ltW4S|1ff3kbynNj9Q7Z&V}O5_ zkvkxMH{spDE9-96%-%uH{dSp5_t=^F4AOR$s0Ulv`=NBO(F0fSQtNNSC~Bk!%}Ka6zSEICJlaAy=g%j%XB$Jx#r=KZluwIW@y`bEywpA zM-en*Y76H(-VN@9;)RO-^Vn~_C3n79MlB8BVG?&DjM@{d2(zj5?eGOj+GDZQf5x|+}B)L(XxUG-dt8;>fT2S zdZN`aIz-tsfG}~Js3Z2lWTLkC&tzGq8hA^b>b9Gg6MY{R}lT9~8SDW^}mvb38glMng zf6pc{&pYCOY3&VcDWZxpl1^UcuVu3yMR#gT11n+<4u!-}V6&yi1;v(kZ~~A z%p@6V1FUOv<*Bf<$%T0x(*^F)@z3C%Ubz`0G3$A=?Z$^=bf~F zs(^J?Ixev3y!p9tzfV-LP0kTZ<;@Y9{0?$z>(Rc)JiDqUZb-_f@sy!E)%1|a-uvSL z1x^A05BCxHdB;sT1&^uj7bu{J>!7P-A=l-daiPNXV5e={g>n*c!hmu{9?0W_Zl^Ep zhPmVf+R8bs^0qk#S!*5yK$k~;86Pyel02meXeOUFcu4K{u+K&Eex7<@k&4D@_a%Pk zo2Cm*yRhBO=HsT_YB%SAgG^<@JTG+Tn1)aA@3)S;x`;GIxleu9YRnUW2tK$0>`!`h zb};0uv2ld(j(TMSKV%1Cl1$pZk{EJ}6EqMs}{Ua0D;rHz#ru@)3q+o%RRHXkE zeY${U2n}+;f2vVY{f&-S{-KsdUntrx<6(xFz1 zpS_z|X+1{xEfUpD5s`AG5u!<@>&{iyi#pH;?)17|!U_y%= z|Fv+{wV$CyDKX z1O^YShwa-)I^#@b)<0J9!+hD z*6!tnlR@uh``2#I{+ai%Xk77zPr0iXkhc((<&u__!W~wG7u(xER`F`0o`kVi0xp;R z;*&@yi$kqlZj%GLnHtiHe5Ne|+$*V_p8*NN@d|loR*dfzK+&Dl#0hx8ik=;bp5ex9 zLNmjnORpU$9R2nIf@YOcabUw;(1yirl}ior)uf{j;6^GQM6AOwD#K3Il}-}MRbIj}A@`>xxYl@%06%m>E8&XZ0aGKW@b8i~!@jG%Co4A!cIF5^FwTO?YfJo8eqe#u9p}(`iP+J9L`w)Cib|+3MvkN7_8lZk%GP5>!1Er2sPy1=xs#@1dJy_E!?8Rxwjv7j zFuRJ|_Sy8mSxpOn$px&FAWiTJYZ$3Ay`+uv9iNyreqR{dv>$Ps1EzCA%;YlT0MJ@| z-Bd(-(pRMmVB~W6*h!^3aRKIXT|r%Xq%&Jk0IS1AjJ)SOW5~mFK%^iuq`7`KyG2MA zDi?z(ZkG9A(V#)TZV9$R0s5C=?qx_B4*FGL9$lQm@ejXMSJF)zN4)Dimsn}^glY&un3 ztvX~uhR_Yu_{$^b%bHp9aHg4h0t!40Q*jO6u?JB6HCkoxn4hEjTL*UsXSw|=%Ts|F zgch*2b$;)H0LmekD{kzm&^{FlMW5p3#)qI|WL7K0xq55kI1J>u7^@gnBqz2*)Hvge z;lW&qQld_`yDh9o-Y0pIkZToHw(+M;hxv3F!h^u0k4tmEq^ibw`DX z`$@H_q?Iz*5vy^OdG290erUqK@GHu*lJb&v8bo6YKuw8f*#p{$a?-Fwc{4Shl4i_C z%&?0akdlixVc0aV0;;6uYEEht2tLGe8+D38XY7@S%)n+qOf@G^On$`sj7m`;NK3KGeKC{VUIVnIo;|bYPjgxZernU8Ehie*fGKgPx=@*JD`SE zLYW3{I?sm5((H2Q;KY0-QG2`7epr7IH)3m+!fz^7Jpk|&JLWaAJ}dbPGnsHSLKS}L zHK~2e<#2l<;5!nPp?z32;KLqw?gpN)bg{^~zm{%$gP6c|V=UCO2e*3%jCgm2=M|$H zMRQ&1=S!CE24KDgEPdIveXm#uLlTWfh!}$IZE!W2ic*l~b9g2x-kOppPZbE*iz1O- z+h24x#EPxnR|d6&?rPSq@fx8Sh1j4#cyeP=O8f)STaucZbu4P0*;l3#tmL<_ zl~L@dhkg%LV~LTt|JPVb4|oGMC+$VYSCdSbxrEFUF0Z~s(uvK#vh*X15ZVV10Jvtn zf2ic5q`Mh}x>VfI@Svi+t9kWMj`~(P#WdD7SxL4C4N+*0xqk|c-9F6ZDA~Jq@E?j4y%=re05jEQ! z@zahsdGM!kP1@N=DvjK)J42h}1s}-MfSxHQ>>BGOQ)3QepS9ycJ4RTZb8 z*MQ?C(-I86`N%aS%7{F7*m8^JE9*{#GS@Yq{K}l=#ZUw%jnKh7KWlRMZhf!S^p#vs zCfq|EORgq|W**46x<PpHBUtBvsP#1Ji?e|ADB!$K zl;8ZeY>+Z$b%s3>9X< z4)MPm*>(cL-Kx!n1JI|1Bx^IzO{BXyR=3Ov#Snb?_k1p1w2qlVqc7zoR^k;HMkH2S|Z_v_b_vsAk*a%y14!Nk195AZ`$I1UBjPq4U?g4|&=TxHAca0S`U5EUUb3f)EvAIqbIioA^6 zYxJqyr_95vFFkyx`%Nb1?f@%pzuMRD{W) zJjYgWrpKzoXnexBwwT7OpWO;u3{sE{zNMX`-t&4NZO<}4nq*VRf8tQEkg^$4Rorrc z*(PHs!5cO4sBr(tF~wQ}e-0=q>b{bb=CJDe*ijEsCL{^PH4+XFi2hllug-N7F>`H{ z#NH@~zQGlg&-%a==p*0dlSM)h(%~WzOBmi;G{Y_md9&KlQkJWq zv}QV7r;Kj;HjW&}6f@;Jslu(mR5T(7ZLHtT#1ClzY}_*x-)6A@aE2XI&_K!d zc%g!2C=_9;X7E?wPn zkB;QX6eZoQmr5FK z1jLrXmNn$t>xpy#gYM;$cBJM-&g(gtnuXFr4MT@*;=k)G+BVsJ#-(>`QW?=@--T%|8fM zCS5;x8T^MKLtD)NF5CeRxgy`vDlgMD_sHdjs1;~24Qj8nca5mziDimth-tutUZd;>xmH?Jp)ZSSWogTQ-XWkxEw@ncoivLDIGa*_1CwyPvM?ORN2V?Mq?g zIFL|n%A=uDKf7r#^I*}gOV{QtE=90-=02Q=E9INGJ_BKNwWte(KEjsR2+FdV@}PsI zZcq@h4gX}%t9cnu9~V0-KmODteSLj_K+z)T$@V|P(JiId?2E6r9D&3({o;pXx3oUo zHy)wu4jZ0YSBRulg5R>U|D23kMqXyYeuq8W$|7s{5)jgaX*)uz*Ng0a8KH}dU%)a9 z{H8wEDwraZl}TllBG_N}fb~9-<1azgtyjR$mK-Wl=Vdj-Nh7DYHWH+K#YKHYH6Cb4 zTBn4;kTd{Lxe0gL)K^<6hj0?c$++y#Y?cbjjFwBi7&Nsuboy z)#8i)J)yM~yYpF5g(_eg((gHyl>85vUZu`_B; z`l|&ODvo03>ILpM<*c+$6R9p2HRg1aByY-BWqR|*G3ystKvr5ff0mk1wNjL5S9Kx% zM3al;G+E&E8qm={#Rqd(ea&&X@A?2}Um(+!F(i&4Pa|j55RcMrqEZY1-m&@RZ=P@K zVCa4h^r6cBeanO+F3%eXT$&0-l^uyqPMt3rBsBgWd`K7S$|8MOr0Ll4ry$eGcYI6_ zRVuU-jBdcj-M6#rZC)csSE)c{3a6VZpBTJ#&?#j zHtF9VOd>@UvHw8`^m;{qMHR>^Hqaih)ZB9(loo>imE$v-m@_^v&cWcmmhD|rQJ4C( zbYT+fNIZBZF#F%SmYzs4Y^DisNamkCHAy>hh1SNMo#fukz7U*Y|1wXrL?@Bm2Y-Z~ z)k~W{N!Vp+E5#0OqMb(>lZ^$6W)Leg%4qQuGuh&E4eFRVXzJ-Lt?vXrfRCFynhJ^; zKeU;vkGuh+$>lsOPxDHz+<}p)|IZy*2qj)yN^@V#r+oMmjv=*a+~F>54zy}Mnz3)d zatD6f<=e*IbJgmCf2!~2CaO{gF&{2{Sxr>(iR>tO_?mG0YBw9CSH1p8Gs#Ptr=C& ztbqEiJ~qMgO(TvMfcp;V*O#nXjUcau4{!U&h6cM1vaMxRE>=MowAS=;J_MQm(lz#c zcAUp@{y`E;CMvU3SHFFB$GK>AYX2VW5WzqicaW#RkdW7Cc@$61Yq!M?Ulc4+E{HrzB9;MZW3+e@M_o>BMJ7Tw2*6CMQTU9(;f zDBc-ZCqFaDoqG3KRKaBG9n6vRQ<(s06t@N5{KX5O6DfctQC*L}vRsr&Vy!I zwPJab;`6sr2kaYRaIok@IWo${Nk6DMT=~Ktu910!iWat_lNLRiX1|4gA23FGDGhjb)Zo<%c4v-~sKJwh5SFjP4 zMuv>*7ijTsIJz*>X)(r|WM2_j(F(+>BjKM+V1EsLDdhC321{xRRx8K-E{$xWYQNsf zhpo8k02@6H!GSA}3;=MqK}TSQ`!)#wBDk9w@dz;WE+8jlk(#hVF4vds-j>^E;||Uf zG*x|)qWt=VQXe~OxvIo0+_D~C0T{lzd9%UwEyoy1f&TP~e^Vj9JlqjIwXR}}YOB9H z7#w${w;EzA^sM&d`n)(ui{tslgvZn10-1j_4=n*stfYp_X#~21@bKWh>yMDiU#$po z&4M%&N=3wAxil7tIB_WI4;o1u0vp^`D*EqlN+EeQuO|4y z{Un8yto}(-%7ZJ0X}S;=^+eae?)?srrq5$`mYMj^!1j(kiJD()SlxHa!($9yX#_tD^wJy)$V3tDd6qHgZC2q!uIYCNIi zU5317%a!hym%KI^gb{L-SC<#W+~aS1B7(?+Kv*;!{2lp)$0(ud0oDB%*YNTM4$fd=aBLzO!t8O695AK| z82-wov9EK4oiO6DUe8Nuz}%s^k`JwH;Y83>#@>DQxm_GW%dzDPUG4W^~`==8=qt{echsq%Q8*6A3X8^ zzsx7XNfa3B@>w%74K^-wR2teNgM}k5O0nea9M*~`#Bj6!9Jzpq?7)xsZ=U26#m#cy zzz-c`bFJueSwF8RVy8sdhG4jI{oQJRW!Tp^A8MoGH zE`~ScjVpHrP+nNhuD}`VE_5E8Z+L#OFK}7Wfc;G+l~7M^INqWN%FStGqq-F4=-tB> zUR6YVz2Z;l>u$<`$FBdKVr`SIwpcNc04w#Wh=k6j*Ze@3z0a55kk|W|M4- z*6^&bo1~R45qAHhV2U1rS@>$`Y!6{?kj_fukxgQ+vjCf0t$nXYgRie2WpNwIYk+gI zW{;WdFTc=WE?G9*He5$RB(!Uzp?Xu929GS7!IUxLSQR}ZkqQT)x%68+&_atFL`kXG z!et`$xnWF0vKkHUSG&Uo>Inmo?T4I!X`AE{z#6^~o0ZIszVQ@K$t$5+3P0^;qYYfke~HevoXeWIosV^7vurJZ+=dt2=+76KjuTg@83WkJ z^J|0U57p)Vve{i*rYVv6>%xElTt12ph6s}oqB7Ia;H-e0s_|rIg6s;mJ2OAWv6{o! z5id}SVh+x(OQ-ULl#LB4AB##12W(>#KPHL$t0-!wyf8JMQ>^zh9(8ORLDN2?-osKZ|HcWlH%zy}xPINF7I|ChnN| zqT-Q=b>;3XKfk37(7Ff57^@Z)e=0B2%@3Q$8a-C|h4PG76f!RsXAR-g!8#*skMjbr z7>BVuX$q!Q3Ld|p-`tu><>k+ePi0b|(G}%Q&oc5@Fr_RmMh_}}-)RQvmARfHBsqkh za&SV4du7D(7}NfM>TI2!_&6DLPaFLvzID_r)wdcpbVG<^j#n64e9dxL&4`!M#LYB1 z(-H;mEyoGn$?b|&?Cas`1J@#wR;!|2fh{5!^@k_Va)kDiwNsf}#wln~z-EcqrodnM zi5eSDEIyHdI)M2i-}lPFP^&i=SU+j&dDRlG3v(gQdy-3}v zaF_Q`91lt1(GVQndMVU|e22j#62+Wz$h953Mazc7lKN|@5pjinCFbH@B}gC2Jt zg}Uc`T5Fd$x>=!{Gx(QFE5Lgbt$)X6No{*M`j%sAo)S0sp6m~K{Ag=n#&H?+V4g0{ z5PK2zYs_#DS6AyrsmFFVvoA&X6=vlZLdJ?vP+F*cmq~aV^G$4_QTkrfh!W#N`TUY` z3JUPWpXJOP*^@-9H`%Dmz_LQf14{uv^{Soovd6sl+jx299xokRc=@SB;*zY>$+Mbq zVBOqr^e_shqT4OlP_kMcVATxWVcNC(!T#U>0?_+XI-zG7bA(QV8bOivsrK0B1OMC@ z6=dG9$L{o0tcCHrVztV_U01e)&9+WO+aFtXb;4y3-M{ln+Dm+}LFQf;JuMeCm$0oL zh&up&=PRGn!%$WsbE)MsGVDrYmutavy-<(#S17P!)c;}XEF7Y2+Hg;YG$;*A2_hZR zAq`R@uL6>?q;%IVodQcLARU5AcQ-8Ey@15h-O`+g_xrwc&VR5wGtbOD_jO&r(ePZj zQZ>GQN5lNEQ+oe(6mV6*hi_h+u8h_WT#ct{z8#8?Va zp}X;wR50jhbZPnd$LA*@ZyOG-Be!;>fxML&#aQ$f6XQ#mp)w0RnrE@wjgw2bmHX0p zEq}NaZJi|j|IMZ-oV`bNRXN%#|NWyw{Nh1rI@OaFG0RHIRB_xv(Nfmj_jMwMfvf+g zgPPz0BlUCBFb?+DN{tDrT&RqKW=wByO9%;_QNWti0+q222kh(vtH|d9v z0)>c4g^yL6PWE|VAS_{<%%Bc5Waw-Oe62N8h1NywANJ_Pzn*U>EBg~ z)YT3X%_;Ftu(51h#JQ=ZTnEMNlCYDKv4mP>^4PVgwV;GYxeOZ5lHUAXcM59k1)9Ph z^G&U7(R+$YCf5``;SIX^j|2sY*KL=iXq{K_BIDguK1Dv4U2IZgw9~)^c;-)9q#%P3 z6@};HWoa5HcorvPn^Q@;y98ZP$AoyQVTmL+DqMdaAW}UAZ=$Jl4v)5mE8I%F*|r z&B(Urcasi>*nw;d&aLM+VwZH&rSE$tLj5Dddh1f4*#;7;c&9*xQhakg)B0z_^f8Sg zoWt~lxoA7-3k46IQy0m8(rc1r>qhw>bwGKmNW5%yqY=V9i+(@(u+3X9G`Yu*E zqx4|?BNICQ;c+h9Th#0)@zGjPU?b8Gw--w0Exq@dofKW_bQ?Tttlr?^$!^yAxv-y+ z=5K%!X^;#fisz0m->*r8a7x!wO#r?=BT_K(Yo3N;|7*GEhp-@3Iw27v(Xv{VgaQpA zvFj__4~pJr`JnOGr`s%AsO!1t?9Q}tVQFR3LS=H_o0MT+na3 zn+3H}W`nzkHH7vVD0jvRGQ)r$%T4&H z#I`uT)|e9EQh0uv!@{DdS2J7b)ZV+Bl76%6#j(K^E5Kv$Z~m^$mC?kue^uAc3?gt( zCB1(`X!*&V@-zw@2l%H)JolP1XG)n%m-4~Lm!C#-`zX^Se6*e+tG$8PqwOGdO5U$Z z;KEcr5jBq>kO@b+X)Nbyv<~Vr<=D2i?wAw6&dR+`ut}h*P-yM3*G0rCQH~{Re;Ka3 zk!Me-ck=UxHyOJpvm9b%-?(IH>Twq+x$89e{E}^xC#a z`cvz(Ccva&=b(P}{54vco-rE5yZTt%hCypmysEWDzTp4je+p2WWS@nJ?Y0lvH zMR4GO&En?G!gEn!p9KLx_i{Qfyjh2Tm;euGw7SZs))+JNDR{ReMC$QFK<=p8Yt#==I9Y9;i%(0Csvx{*Ac%j0)7FcjyvA&_ZZ znszbR-0Jc-o~f%nI@i}Y^9$l+9X*HV#k-YePUQflWS+X^zysjoZ7aXc<={221xi>~ z{3pY=!6gAz^5RY9RnAR^RpeSn9!$~@=S#!cE{Cjwf$f#O5mfg3>=n>w5@gzJ6*>Fy zpD*K=qW`JDFjayGx%6d}44f8h>1#!}V^Dx-kvXTboU@W(EuTrI$zgREEl-pJy9G;G zJARSZx%idfvQAscc%iJ#y;|q@XC{*$xlcl2z8y0nLu{bwRgw$pgf1etQ%VHy%mFpA z@Zs%hYqN4}ZzR-(iL`Te0S8vuU4GM1H#ZFU!5?jaHpV|6eG_i9xX0bH|F|y9Gmv1r z0SBzTa5|ncCPlysEFHg+bU}v9!gV88&p7RE+$0&c9)xN@qnWQ8CMXjH8-1IMCq%M}LKO6dMDyfkmpoVRAX+>o%#Z%&(! zNtzxRDVMV55kr<<62bVj;uB1Ntviis+I>_8JNll~k(s^VG~d*JEp3hU2aj;9tN$3s8`IM#HeKtz%rUly zpO5L${hAMCZ!+<_l<35qk)N6O%k=f@o#+~0Czti)LFW<*pzGj-i|zbKAf|UKrtEW) zR=A0kX!O55Al7M0*FqrOevgmbNeCvut&v2aot=~p_XhLL-zC5Lvk~^wcPCW>5T}^N zZ&IBMQtl~BIMao^;FkOObatY`8wDjowq;)xs}i6v{SGGIE<&-}pIVnl|#;$MUBUVleyiBrZI7AJ2{m$RP$cN0HUy*?6X;n z5b9ojOz|GLl%{K_koIo6*fVa##`p$d->&>mm4o`cQBdwczU(jFJ~Q;&$uXPqLz`t8gk z`bZ5Ojkf4`=F;t^*KfWFpb|M~%v_qQt+GUehr{$%2j{3;O(is>qXu9JLNz?#8%umx z<~wroIgZq!+2L9%V>;aqLG!r|&@T`jSLI*6nth|`;4lQ9=(}D~XLGEe82R68Hhx0# z^Vd^sA8~1q+QZ40$kjqKy6T+#Q(&07bSxQYqybLlh8PZiU0fJ&xZ#hDstYqBVq!|~ zwO^V*Rwxcjq^<7f1x`>ZKI0`2RGn+SSv)+D0z8Qxk0Kjh!lT=Df9j!s><4U!N!XfE z)wPnnOPbVL0M8+;?o<+gV3}^do+`Bmf2J*W8c0s!tCr4JPm)y45%y)=WjvRAh+4UJo=_Klh3(M9I?i-G zeVBP{4t^2}}zPcfAi(0a*| znY(mG6`m!@yE^XjckWV$=z*P{MO7%(u)Vrzlu!ovP|tm^Xe1q-e%Z<)yY>qLXk zi>yAtI~Wd~am(K>VfII_)wN!^kuWR<_mY?%1yVF+1#HJq-gWMvV$+Ux1hemAsk+L4 zcd7Y=VAMqPMfcKg_cLhjX~jVMw<>8C&`fuoV=4C=!u!dD(^@JfPS5NWy;pd*q8Fb| z{fCH!&J=qb;&Kcg7455ASIPq;7kO3pUSTowU4Nn-rUaZ&e$#e!E1`>lBp|s&EgsUP z6$#&$UdOMf{%Xg-CU=vxG^zA=fTLk7&cz3O&j64Na3!yYEW5F+A$_)QuJ^-4r8n=N z1OSUCX)c1|oJ+UY>A_UgeZ{V;o7ZrSXTT801vZ-t=~@O{(RMD-h_?)U{To=(gx&$6 z62YFqYI%|rQv~;q-bk)3=&(!11Wt&M8uG;my^ndW63wSMgkx^q9IdKHQHDDGrx@4G z@d&?PHFbT7lh2b3#>Wi}%t>ghcP@#ZQy4TaPd=Ue&!5(g@;~me;ADnNBI7}JaGun1 z`($CEiymLF<=(oIebX`g05EHONQL7-PHxbGa10@@;U3%VeNL`Dnj(iYI-x9FZZ|i^ zUv-;C3@Ci00!N&OoHYt}g==CWXAn6vJAV_5J%%$lN&kXdc;rA4sGPiulxbCxqbet^ zi6&MZrc7kyYxF*Q<@*NdYIt-2c3?3(j9%>edG3T_fus`|+tZf>;p0`C5Fk5mgJ&s> zu*y9EqWJb&)N81t)5EhIpvIHPE_KzdT#g>xPj5Vs%rEcWX7I%S;@#iLHO3m>8ik2= zRE|hBuAGf0wN$BVTF*!8{r5~@h~qU_i}m9} z2w;$%(xj|GO8xYAD+ywJwx?iF6iH%U4>b!&dBtqx`8S&g`ty!4r>9i(+NPvFzxdN} zvfCL-zq)LuU=JcLgG*S;FEi4CP-{(2`57vz*o~~o?54z>qg2YzP1BSca6%+2BCkcE z&B(-39ma+VX}=xv_ba_PX(oa!qb~xe7ldrkwg1~^aVhxw{f`NnA9Scl0lpTRQ>5kI zo$i{>un01WoZzomRQ$GrKane!?Gf88=wTmQ!p}yrtD3srT?rq`5}M730b{0Za*28O zXnGzx_R%jr<*J#@x!G;K-z8`C8jCcV+y{EnTzA%*J|Fc%hRV@<)W7puoi$|tn27nA zr|$p8gC4E2|7D>o6DZ$hkewByph#$oA&8XH{O&tR1WB`2j3i7UNdmTS4h{d((|D?b zF})kk8-cGkkZUM(=0`PV`0qTuW<0XOh}vaAHag9SWDe<4=bUbRcN3z+X-g*x5|!>R zs2741J%4taVUxHkbjHG(*AZ1k_b z42~Pe<1N%yoB#LA32Cdtq2!4!AnAd>)0dXKs?L4gsH3%Xjviity_CUoE! z*8yf|x{3`|0%SC(tcx5f6C3*+FL5p(^r$a#P>37a>f4UVpRih&RQxM{>Ha1OH|DDm zW2M@R&W7vX$n+fjpdMu3ktjlHUrEnkMH~kizLhm9=_xIlefmm*?z6M@xm(YeDxT9Q zADx?q8V4iF4l`!2YQE0oVW!J{KkD;k(E<*AZNv>uTMl=J;q>|}`ghp9_@^zsr!+HcX~c5g#`TDWt)+&ukh}S3B|!T zn&=$4SiL-t!ExzERC@-UA5}6HaT0#l8zLDtl&u<*{LVR}BL5?2&6HPgHvzjCdRA<} zDt$(EU*^#s4%P0#JVmHSAzAOsOK|1 zgeP6TJ^jB6LFHby;{S8E9Tgta5mI&TN7qw{3uMp?sOE!EFMBlc2pM?BFGZwA%3U|t zW;ji*?ezD^7v%AkgmBmnS&=_og8BsPo(6WVG;WLdtjpm{CTV24uDeu@&$W7erQ_I9 z*m{($zDt>*TN9IzqRPJ5tCs#bS}c(|Ycs(kG!~P_V7ix|$?$5s&_$EKyi@(Hu~>NG zf1ogYXvdgT^GJ}h{p~B(O^B)bSAl0ZZJ9C5`7z6y@}G)4);F&f6REsO$>e(eSQYCq zgH)8{MWE{f(Ruh`FdSuc4g0t2IpQRg4Xlk?6CV)yk@rRQpMqSxQDaQT_VIYbM++c3 zHkz4R3ih-U)(#UqUP5kZHpcXZ7tjA|-63O+b&|eso2638I9_n-J7XsH>+{3(>+wzA z#P<0XQ?{XHWZx9|^!bFt^(EJtUH{OlX;F>|Hl|PZg3dd`qMq>$_PXb?w0k`l?>{1J zzbZ++1G)fNXog~o!?=bxVO++rZ{@K>ci~)L#FXAu$>?<0_UEbZi%n}@u|RFuD$n@H zH7&}ty;UAtEp`}?B)dGny#)`6;od7=Rll@xB5)z!`v$)1ht)A(>32LnUNF^f_^2}s z$s4oh%Xr8KPKr3CZ%*y2XYbc|d-AITnV%?g$MeStt{H%}+3QnW8RA1qMK>2Bd|xD0 z*qU3OX#bHir!CM_8SrZ5QBh5q)Mj;frj|mCYtH)F_>ZToQfmwUIR8VEP#&{Y9PNMy zfl8}w|InmS>rEGlMvZ*ma_{h5l~aD%PxmJQSK+U3Ll4Uj#iUJWPZs|+Uo4}d5%<-t zNi6x8Gph_-sqIIrmh1f^sfx~DeW>zQs=yd#cRjLk)}D+Nl$gqXv#BqBz=$!6rSC~P z+mR++TgCKufWMyj$*)+v)h0i_)!00xcTxm%N>2TQlNnG(;cGLy@YLAT8Ea??*Sy7( zpikD=!{kPc9wi|G?bSyPixCF(MmXvvb7iwe?vEW(zzZcri&xQcoFHMfyb$!v2xCWl zxV?w!>VSs%5)WK?XZxlVdf#C@ec5Ru&3YOX-wdyqE#cYLT)K!&_Wel5Lyg9(=t;)w z=+};@F6*871I66^P%;90wQj0K>YNby!=Tlc`*$;!aEB*_`&dJn=Z|Wdjlql}Qy5nuc=~C*ND`Q>q0XxB0bW_S_b?xwb z>z#ewPJvlTPx-S;hDJt{{&3kNZ7;~@+oGIV z3ayCBO1bcaCk3nd6f-x2A@b|p`-ttkN$y6MpmH8Y_U*n*BaSyd#5fzGBhZbRR38Q3 z|E{&DO64d}OAE4PMhNLBPZiJCrb{c$$g|BZJ>j!YlMwF9(!pX_bqN^n>0d=O+r|uC z+}-yLTv`1P(`CU}kanzY7Z?-!n>cjdr`_{pWF`Vi35 zUKA_J9o}CIc1qiOmv44W;%HolRj;%cfy(n&>mYl(Hvi+Jv0_`k&0{e}QKQEq?fmY* zuP3@^7@26Q*{U{f;am#sIJiA><4hld13GY;YkWh$^6t&|4_Q(GZAZ70|DHLnB`=~& zw}g04ssG{gSX8ixZ;z-F8W4MU&dp!*QZjlpJA3P+Sz$p|pqD^ReF`LCwxDvuytvaO zlmEFukiQSQehtciTLg}P>Htq|-b@S7vE3j?gXla z_PsYpdY*xJ$_f$qYjpGU$=b1@JF~$qAlc;l(yG>W|N6Z<`+@%)XN|txBN`A3rvEH2 zW@yCoWhmD&vT<8c0|P>=I4WrPyAcCbj(4gGD+IP1r|%0i3#+WxLGY$e+o( z@x2+l>ryWtvw`(tDA@Lcf9d`76Ap6@E~!&L0~dtNq@h_bnbsy`B?4uC+ZL#n8NanU3HWyq*z2i!&} zG{0FWXO z-!O}%Zq0~$+M0EW6ejGQHBtK}Jx}_uY@G+FE42y-KT^=ZxxR=>I=vM($-X?IJ)#IL z_=gHloHnWq$53WzdeAEEFf>Ob{-ID*l>&43EE5C3rrL!O*Lh+%%>fXZ?;S7thGvAd zbGf~#!%4sc$~vsV>fD~KC1T)_8*ey$rD4_9qcKNA`tIx?l*WFmX3iq@pqt0n>82Ia z?j<@(j}WQ;ja?sDTbgVCe8O4H;fS)-=xi*`-6id!XQv>QMcPh90CQbFd_R`}%K7phEtdvx$${RJXkD)=SCo zjK{R*-S}T=$y>~K7PLs?1Hf40&fV_^IR4!4Jw>`WVIElOnKSD3tU@zg-)zjT`NtqA zRg@;Q#IndsfJ~|7@!0hF8P)%9R2C433Pi;>xrIVLWjjGUXR?`R14vxU`06P;_B}#s0%Mnwup>)Pgg3=1L}iIXZd`RMrw=yo{F(4hj`)) z1(ek`CVF@(!2y*+@>7(ncul!_vq@n_RaFFoV&a5_Z9F|^o~0N*_jvlmGpKDrP&3R9 z_I1X=ZJeTEj-9J!Tc94WL?DvhztFuhAnQ3RXdRy*Tre!$D$oJNB|Y++LsAH=9zA)r zfN@Y*a;LKSvCFzak9Ku{xcPGkuz&2*oc0*d580!@53TL0UF`(M+uhYw8#3VoHMPj5^hBzMC7llMLba-#{4H}#F z-XA*`q`U7tMzY6tnDEXjvo&E=*_vl-iqjOs2&3=TLKjS2dSE%wCYQnQ65s392AqBc z09&G5AWRXV-j5N)3lYoP+^_X&__E(0sy14f=BqHa?rRkZwbqw(=U`sEdIY5% zoMt{im`kv-C%=}a-N#pq9VIIgi`FsWcaC zRB+_s@gp=U{|h8D8YE31A5LL`=H6Kq<_8ykqotOGS z|JKwt7dHZo;`yg3$D_$U!0A_ee=Vc(*q*m<55ycw60O~cG2V?i9akkM2)LA-3DoQp zwY7sTsh@5w5!QtZJrm6mp`r*zcSeV&uq){q3$kW*e(h2Dyg>dgUT6;~xb1;HZ!TH}3V$y21;f4ST7WRuCoTHN^Mb@hOhEYbW~ zDopu0|FC9C$GG{co0?MaVwchUQI3A2{P(8WLV}AcBGN9_m}a)t_rD*dkYsHfO6e$% zmWP%Z_;i%YjDd@~2T6 zfJ{Y0H*sqcNhuyZmct8+E~Xkd8*uV7P;W*M!Cv1oL00CZDfXS|{e)H>eD5w~gh?!H z-RfoS-x4alP@-oucNq)Ed$xYjMX+!2)`+?PZ1-i7#QJ)rj9zN0zW{l3+Y$D&Jb6Qm z&;;X0Hp+W3)OK#4c&S}g+r$!Mp{rm!MwhFvGBJ=;z_;i6?#3-!^W^$2JIHr^lE{91 z`E{6gO%=Q3nxbgW4%}Vr2Z^y1IttD9g+dT2-S9fLjWtWnz(DmRp;M(*nD4p)wwJ4N zZGn|I zWhZLOWP{sRHy{06e9n0)&0ATSd34?oy{{_YAy8P?w>6syH@>gWZ(kbIa~9u>Nt-W6 zJ9Z4+7O|nr56|XE=42qNdg!ffbL8~I!d<+k;dMl@@6PksXKJ;*wpkjqHJWKH-)eK; z#TNYAXixURSrfN2;KPboqt>*A>kg0-2Xd6u<*eHQZl3+d!<$n4>`qEcwbbHc25?0= z7jhNAeU8%;l0<*vXtgThzdu#wfvn&u-Ta0@C|~G<5%G%y2Qe)%Swqz;#>?$7Mmp!OG?Y|1sn(;Dd zl)Qv*e-&gPgihmtIa_aI2}R$SRUmTC^`hRG$h>#Ov#wCUIRYHt#??yM9p5}H>lTuePl1~wF`NG#MM)JZ6Tz&tyM`E*b zipXvRJHYs$;F8m~i%*(~^cxlSot+=bc8Ak!KG&fB7b{1jj5i3I(&mpw%J}P074Y~1 zzP(jPg!L%qMwtlzYFg)@H-Yuem}a7!x5!;ybyNtXY({<4kOe{3(v@R;mP6t8Xt>r_ zK8+F>-jcc=i>bNNi=)oks^z*vHvwpkxG18o#;@f!$~p;c@LJ?0TaH3_s#3`#82(%+ zgBE{fw%m*t(A}{i0{uWb=80dw{gT-CZW>`;yFPk^B`E}K)p#BNDyZrWX@ik3yOqAN zY=ub&_clUcbd*PZsi@HjTxVfhFk@d(g#tmGDi8}ln>E-aE&E5B>a)dnQs+^jf1YmiVSe-p%j=fP2(r0x`_f@ zx9)nUtGF|zzUPH@YX86m6F}|P>jsA4RHp`b<*$0@gP`!6?3hBAnO>sRyxOEi6;~u9 zA8>?zZ2TnxF6_6fCvge-W$SW)M!MjvQ#F6=7#{A-|lr_@7wB~+C|LLvIaaSK^zuO;-bKC z79@jsly;5^4nn8pgUX^yg~`6u=$ha%;0tcbFQNB?$9mLs#htsSm30MSi#-O)$X&Is>6KJc*L{kHd; zSk6BY^1qO2T!)@L;th|zcLlvi7C~)5j^8H|!cQ7D$+L1gIB^$t^3o*0wki5!j$3_h zb~|!wDD}cOAUP*6p~j?}^}4vxYezd1kz;_+lD^Zf@iVs4r@5vvRY-DvY+&&Ulk_I< zV`uOUaBxRGz@=BkHASG(*HJ_*e{rI~t4$zO`-sZhxZ{b5l$^0?E$Tzb{m%`5eBR7D za84?L)48+`^@RM41V=X8l46#nol8~1s+BV(t!+#Vh*NZr_nAxgJ!~6(Jiwkkdr_@v z{=`|J-$lT=A%~H=+f*ej51Cjy;v{W29|a3?$v`H8=A~p!cEazNq-~Wt61988sA!Hk z7zq>Yub|N4(Ozttt@Fb6GwL4L3+1p#@3NdE@V5+}pJKKaL~#d^*-qmcs-TjYc}^E<#Agn;aacaT7*lX!+7!~X8Mh8N>Yq9 zc2YWFum!a}ZE`G}IA=uEVmtqxZRow9z6%{Vf!{T5(uR#0G;XIdB6aoeZF)|j?XT#P z@shRlQ-R^j7R2pO0i)u|;RoamOyLe{kVIB03xPWC>di~#>xx(Kge-@j) ze;gP!yB|7oLrl0kQ=s(JSE0;f;XLS-4qFoS)7$S`jkq(HTqpl1Q223#J8=wZjJH5< zLNy1MhMZ3`mra_K*12yOZG@b-hs5?1>&)J8^RuM7tUqvs^JTH%y~}(zEpvd05Sy4q zD6zNwF>_5VvF%eEaihC`%%w8GWx+!@` zg_>A;@n_FMn1<|$dfg7hnne9!hRkjs6W4J|iO3Db*qq5p>?S(z@@4$B%|=@CbftV#}yr?dfH0!;;1fJ${-`T=FLS2c=RJ%4xz0S0? zg|#@>pv*a#T|AVbA{=|4-R&eHO#`~o#BBmLO6YJ}9u$Qe#&*1C8vi8MZg7H723Pz+ zyrri!x0so{LHz#k1$Vlz7=Q88zL=jAz8ty*>@kFXUmt$ZWHN_~Zs#S0KAh(lrBhZ- z(YuVkBB7W7*M_8c?CLhKrG9%LHrAyHM%S zpklt&R5`KXP?i~(7i02kqz&ZP!QOd14`G<+^uTjrl*wzDi_Y0W7j8e99p~j6fY%2l z+<5?Y^i9;}=TYbyGf33_FxH%qcJ$k9z#tl>v+BuZRk;3IfU}+JyUnA$D9_z5hFeqE z--89L9?oqQH8Y$(ut&P<7ETef|IQ1w^4%nH)s zsja(IB^N7}6DvtNtdS5+_&Pi{BAWVjiM{?9+tv`on##{`=)e%wxU@}$U#li@njMFd zFJm@3^|nXP43m$%Axv`b6Z**O3X^sh`A)CU4ngW+)Q?oE>4unSdJNT>xh_|8N35vN zSzH$`%4k-Me$b&s<~ciTH$fg;IOs(LR3ImR^d-%zlCjiT$YMFE%25a%@JfEtHtQU= z9_~hGC1^@{xm|A;#9&+F1=n?>ZSqSTzwl&7Vc>x@^<@)C)VS@2apsKtk?268 z5_z_bM%xCGHY&PXNEomwza}fD#Y6P>$|spQ$rITrd0xwn1`z6JucWE%B|-X%(lM$u z{|-;2NV#Tv2!6&(T3luo@cLmzgf%eo@7lIUby1f&JB4-j6$Wu_a9QM6-pZxJ=at#*$wTKN2elKSJh~T?*SPpA)8?c&*M6}GIg3H1}uL#2IQuHZjf?a{kJ|MD2JD^xYCVWSc2kha!JJ^-}c%*1gh;PNE;rF z5)#s(BKlWWKPg2BFbj+Ai13{q%i$69pl}I}>6(<}=x=MP; zj9f9hHGdsfj`kksuUfekeZHJ+1Zx4Lf=?qWO%089^XR!d@j7}yCeaRir*SiGWX3J0 zkLwg>AE}>`pard0Egb*lQX9#_Hjz?Xr4A*5thdimoj)_ZKAq)tAF&ZXQWg7roBUwG zsBsOZXinE>S)kdP6A6skk^HRqKTY8IW{C1t_avUwe%`cmv z^$G2jW)EV+(jqz+m8~f-9fIR$_el|?zFy2>HH^D|Detop))UqGY&fcQX86jrrlOHM ztIBIRXG!7tJg&X_lbVWZ{=RFLIO$8efQU`d<2wjZ!?npxXFqr(kG{&v;G45s{#9r* zvFw~`!JJ2ntRtK`4j&L6HT@?8#c_rAn=>wvdC4@U)(&TX7`k1rEjGNvN{hk-x9S%2 zw%7r>`*l-ykGi^WG*E>wf$a~L^yn{j{tL#+VR8oD*f9#_tUmhTZS{>kOkrxPco%v~ zRulJLRv02qla3)$=Vs5}ORleE;%4dL{+M%I0HD#A`)r7^ID2sn&Uj0DN_g;#9q%^X zC!2SqA_G*7?si|Y`&irbScI-kVhx=jVrA-0#&=iTOP;Wt7juY=9LLIZkO8}xvzYP! zU5Nk0wmsB;J*1Ku(fa;(=q?cFop9+@y;q6Hf$o!5p_8jX$2G|}%jDs&(0Qchx_^jm zLOxcB1`YBXG9Md z8ss&211UAXE}1-7CelER%B$y1P-{iJ$$CtVA6|<+C5t4X9ZQvAm}3?6q((X7+>;3! zHA&#S#4U{=)07AY`rV+S2}s`^O6f=22>Py zR4%khodq?(MR7EPgcA`+>aS@s+EH%R$!6?jhmRt5vqftqUU>l3@JC!Y#5*O7} zlBfb(UU$?3<%#H@-_G64hp0Qq3u!afG6;}jklM3)>rEeCv?nJV5l#CQ_5AqBOq>Rm zA}ZazYobm8RERRz_mqr45J{F%j(%(TrRD7XimkfQ-C?iLR<44jD~dSF-v!}Z=PNzV zGF=lTt~Tl9PEx-Wq2N~?2Jf*iC0xYb_DBo%q@M1iZ)(!6Sn2z;_&?MPzi_b+qr=JR zI6vuZH@RRQ>sESGa>*Ja)ZexsMSoRtGRD<94?AN4i_ImD8c)+V9QEv+Hurq=1^;q- zja3m#)vKRSG+?hVzT)B9G84Rssem4-8ol;|ypo&(!|=SbI-I7A{DUmR@Qe7GU^LhX ziY8t`yOrZlGOCY9C{q5Ur_0u#-Ya4?^1|qq51xH?9a9S8bo_z8`T}*`@dfHLcbrVueb+kpJxXg7aAzGtag@(R>`Xx_6f7z=nb3 zD3Ocd?YI9FVh+H*Dd8bldn&yl_`sT3+Nad+pq>>VIz00$OdP#o!_Tg8WHTFy76AT0eEugepU(L`+ ztC})%QvgxAc%HflIk|&n1{V;^_o1nX_L=-7(>F^*kCY5zc!K{DuRsRV(mu?7f0E{M zi|Xh&qSXDe!^XLdPb5`@in7F+KMQ+}9SOMA))AMotwq0j?p|nIB#>d4c4Jv^W+KNfK0^WPFY=pydWAl1fBX~u zMN$sQyAM$F#Jx1WhG$cOyWbK~pB2x&z=$wfKN%|=)S8QGeD=qCE~<6dI9=Se>Ys+P7!NhT^UtsV&uXVy6CQr%JCtU~}_y%QedQwkn5^&ccXzt={yjxB`A+}K%wxZKmxgjG#PsLb zJ70Z-WdYRYLrr06zjy~O9Q3+BIO|=!LwSi7R$}R?p^CPFAa`J8H!?Gq>mMf5!)c$$ ze8V7wTL`!GDdRwz5}PlTkk@ixe|jOhl`9M`_K!|IM-T%RZnjo8wJ>Vu zY(}I=zazwzckRj0qK8nxX|n-^Jn+A9=vj^Us*@_mv`V7OeOmb;9Iri?Cqokn$! zclW4OCx$up=_h~Q@DyxYI}-li8ZSf`Vk`QUa@Q>%r|R)EzsSSngpfbQD~{B~i*{>A zn+QbKZKLguVF(t}Oq1EI>kE&CB{(tfl#3Pl{r-3aK1Zrufd-B?oV7G<*)>&eHhyQ z2E*#xy0G7!;&gNgqA7{xF>P)2bVR7eBQ&9w0pw;^Z!-_muD*-XE$dgIraZ6PB|bbq zDLz?I#FG@K*+Z?#X?>Os*7^qRGcZ-XI2C%9N~cAfaDuz^I>K|E>C28B2y-&D=xFpk_KZeU>mo_hta%`XNP%{R4Oui5 z&uuYmj@Wp=x>Hs3S!3UW@;MwJiMao8 zcA%ly6|2D3D&QDaxL8lz*NU~R>W9cmkBHn|Iq-;@w%2GKBGOR&HrKwypeX9!rXSAN zIvj;zXL1fhtEO~j?hFH4KRXRXlpqHExfiP=YL^RYyDS!phpSaB#YUg&&6x?yoWxm6 z@oc?oaUz!M!%7Hs%Q<~b71SA&nN3S$HkOR7xBs68%D>wa0iG@d6Px~BwX!}+F)9QS zS>J7;J^JV4C%YFo#C0#^W>)8~16!UNp@!C6>Q4p;N#g9TATkuR3hv@wqY`78CHWhRgXQ|~{&{}k|Lk!Z86V>ZWyMeXD3>SL2a=|(pj1%C$Oqa^+o7W?)Z7!DK5 zX7MYtqZ!tF$La6L5%#-+1cr`4P>DNv7juB?mn5FNc|OKm%x8y@hI^)kHio+&k)@Ee zzC~{04EjIkAZLJ=?8oQ2J#XX+jILThCg%0v-2?WPX7rrg-4xL7p8~=g6A#api+HZX zCUAYlL}p@OYx}65u55ioyyp!^hR?|meZSKPSj@YrXt4QOqq8W710r1lZP^CTj?k%c z8}vfeJ#CCr@e2&kRx5Ff)tyFRhia`ADX;*=7Anmi>cn;fow;n-oUQ%e`JD_vUh|0u zZ@9du|JLyD6|59<5V;}BK6H9J5%*^QjlSb@J@MxF;~VGEi_cJ_9^!0z&@|aI!%A;0 zfr}K0#NNE$t|D<~sb1pLy6VctVZ4^4;mgcBZ8yh0c`1;`Guxl1Y<+)amoTA|q9D;BC^eixOQhR}zbC6MctyO3Ms_8Xmsaa7|k7jcO1 zoG7JwO6HXrC*lNs-)VPG>}k7-c%(>N4J^m~zwdsQP7+<>=G}UozWU+>c4*MTt{J50 z`eftObI9BN&8efO6-9mSl6-EY3^&ghw$Wd#>L7=CEkQ$ZJ0Vo)&*(n?n$t6Z_O-Dc z1NuMsry_5aESwea>~w5V?Dwd8%EFek!gy`AmVnY~^mdf%h21vuNZ3j7)d(af*Bu}`)_#cb>|$2r zbt({FwBBoJ0?Zah>CXM^g^#Z%6=F*)qas%fn?~iZp*0mY$V0EKCl$GdG~)bbEMcz) zi^UpW*>3wqV`!i?Q8aHUAV-o;i*Kdqccr{|b#KO47q`Q8S{CVE?`tq?G zV*@1o7j~w5CM-A>YQo)JBi?HCC@0R~GMO?spimvzP0?vk*Re7&f3|5$!C+NZeBV4 z+3j(RUy~T%>Vf<5Sz{N-)ICerYyEG<2lQn{AIwPB9R_{YZ>RS-gvSD7Tz1z!5&6u{ zdR-lbNkg(yUavharb`-{rlXb9%ja#HD;&pK_A@_`KFc>et*HPg1+0Xw5$WoyPDSlc z^cV|mZP<8KD(c#%>e#+?Vhs7t5CQ|9ONYX_=v(mbv^6E&FzmOEe%a>0G~ol^nGU~E zG~E6Wt8HFZMWZvp-Uf12J39cs;K(z5R@m$SaMZw-Hvpi~!|c#;l9a#gmK~aFukV!! z{|jbgp<;Wo@ADD4pdu1dy

2lgXYv@-%QDC?iiVxEc}XZsaKW-@ zK~%+2P3d&e(~xsgqI9c{EdvqOEU`_@k7l{09H5Gsxqc$-EgNBRw5qW<6mKl=gb>5( z)C3`03z{=~kH-1Bhr+$bwyz`8{j>S$LJrvVn>(Nq14>6pyd$|=qRiOR<)jnI4flvl z5!y6>1B0Ytw;x*o78iXZn|)u_P$?h_mQpnmQ8X=MhVjU{T*M8pd_<=Q_|q^k{*C!z zK=_IgYpHr20Ghxuf?UeTgk(L*kjM-HJvb{cQUCBH`)-k&5HH$Tb;w{wcbrKbs}v6l9!~C5@X$pZ6o5 z*jdN%KYrO;PbUw-X%%Mo-pzb@mqUN8H;4Dlvg@CHQoVd_4qpAO7i!bO)|1}G<=3l@ z=GgVh_t6Z4q>;9vpW-wpN~A#s@$Ls0$1O!BxkXIw(AjC=r{D}63mrt_BH=h^m@t9! zyoc2?FjYF929Z(@5Qb_r>`sK!DO>(Olav~|HaR@5_;HOaDM*jzSDJ&%Hh%da_`8&g z@||pLN7;^)t?Js9yCR2}jrJ2>fbFB5EKm2Z4NW~&Y{I6x!2S3l^oK9&jW8Hic5t`z zE;%-7V!!^3x140jJHGaS_rp#9>p~4wQ-XgYDyHZ;K3F1tlV2XhiET}`MpTGhy_^lQ zu=+!N?lP#$oC>-xodM4m?w zb(}k~M}lqWGzoFuuY6ImBgj$LL(bVNy!TX&_JxHL-E~N1$F-~|8A#VSJJF*zhTlwm zYm+l;>e(O&9vUaL)Wg8fN+fT6IRT472CShIG0V?%Y<{0Rs5P%cT>j0 zRqmT2p-S#rI<0U*xqhp*Ll7|~NO@;_9_WkXGfygJVH7yhX0s+@Qr8uR%jJlcPhZ-K zbU`zEmYTiGJBnHyp7#?;@5Y-B3IsAdIS89NbbCSZQ1fNKHO3Jr_vDWbjxf}YEUEHf z5Fn_ZOJ96yZY7uW+|%i+TJ)+U0<8}Uq4fCnwOk9AuiYk)#h#W0jIbhSvq332LA@Lo zbxD2h5uPWSL6bZg=JsduT6Y$*LdX?L-R!THb=Vwi^bNwwj@dNkN5d7F9k^b3cgThAv=%=!&6-J>nf zSo#-?0_Jb1KWlxqWz61j47)ufs!Z!;>5<8L`tU3jv|v#S7B(=F zCr936*ScIn$>fkIZibbJH;XJxT%f{H!O`zE%$J}+a6*5P3V z>&OLC=fhsXJ3iQlwPnI}4cgYVmO!BPgKyHfH;Yb*zz2Dz6>O5T|I*3(9B5>4_i3jS z;mIDN`K3c<3+*1+9D8cn#GoVG)A((G2HSa$qCfoG3AV(DIZKR{t;@3?t#x?FZh(x<%I#oFp*>&%)xvguZQi3$q8ik+bo z{)EcW<(f8;MYj;mp}-3bv`N z)U-(+5La=w+8B7&{TVdSz~k>RI}=3U-8YrJAW}CAXm0h0e3_v_cxGjxYt+7P#9WsK z!I|Hd9FnVQT-6Mulx~&Yn;?p=7y@OH)S0N{hWW$du;Mi%L^iVq*Xb9AJ0YRfL{*M_ zSezw%MSK0`o~8FCkEymm5=?6K2M|+~s5znhR?|8Z-w>{9@Qr693S?V_PX#l#@~n83 zvF(v~BWgAapHvX7Qe#uitp}+` z?^=}REMwM}|4{7$WbOT6hfq?51}EDFpzd9Ruf(#kv~4=MR;lt_6%b}c_$$Qe#|qbY zR-$iuUC;XcHdy=FHkCwmFz8J|R^Z77X<7Dwcv#$9k4&H2a)@Me{rVp^kn%jZrF0Un zjCj`E$CJVVuMBH?PwxSuBt=e{fPp@pZpuoly45v{ys15ph8b>k1WOMG9W5c}^b)_0 zlxo|&cRAJ)P*AC(D=JqOMy^OWb+?%P?9BwP&eTsnED8AIYlbLTi3fyRwAF~jRUp%7 zjaF&vXfmM8{4vWNsRJ_2e$AYI6L!RiF>2;+bKmA2B7-%deFKwd$Vbj#aahy1BH3+j z3|u%dqH*dPzgjBKDwJDxWI}n`&gW-jMR)wL8e_lEH(^TRLoROf!=mQuYCzwr@vgV@ z?dP{ixIRc>1^R?| z4RXG|8%Fc!v~{K9vHQn1`mYjyv8@YJ&Ut?e+4Y*ha?}pSwNjS@vcfg@g;}x&OD3QC?%6tq?{=zA5AHj{aZws;tqZsj1ejRKriWSXQ>6%G z(>%oL>=_v{`+1kZstRMG?sI&A+2Nk-#M~ffY<3AN$%Relu9vX&bi32Q+u3?PwNYb) z!F81)4cXC5l>;<|i5SHYmjb3Q>gSgpjK?c?Y4#=}%Ch>`_Nq6-uJV4}zxpABm-|Q3 zuWNmaejQ!RKO;2`KD{0?u5>Z3I$Px{d^e@AEF7Fq9$73pc4v#5Y{+lW>sd#G$ykhxfOxunhLAO!>24!>x%-+DvRHNo)%-~8-Rn22K)E1cgCsv7$mV6YLRU!Q z9%PY;343o0`wX3^9y9My;GE+?v2eDhUt@2gUhnP~<^f@T&c!S8b>BR=d#k1W`#_eY zZQJCy<>IWHU9#?M;vHl^3dju(Ou&2sM%wiqhC~1AaACsB`g+L z-l{+ASom|NWJdqg6b{!pWey6HMi*useAkxt%sX#Pb@ca6(l2M_NOJ#?LLYQku5x3r(6fo(etKV~ov)*eq$@b&=-6!u~-o zhY3%6JA%VBm3!<3Q@wkque0HFdXo^;OlgapTP`okb+IogP3B$mG5&Zh zaLjL+Mk@2vDTJ}5YnMU|-9O=$UpJ<_5J{azkUbx(kalw);nhDWp*+D{V+h1(iKHuy z8sE`g>aHr2`#j6V%5+dcHw|~|^tFt|W8=aYr$0$rKe|7GdVBDRKWKCTFQ1XyJHPZ@W^XEr3NRvGYLbafNHbwcH?Vxb5 zB{>xeWWCWtLv<@NTNCKri45quNn18IB>?lD=?<{0Dd-td5H@3SI+!Z~&JbCuc<+bq zYGif8uQvsY2dz5 zEQ*RK60aY8_~a~UN)yP@nmEW@>UEUe|2^GPR(N6RQ&8nhT+BcZQB(YEWYu3`eNzwO z6af~^ru6x;W}^XEkeI{oc6DzDxenx&-^g$3&1>o{qjRz~@@E>XxDE5;D&>>X=2l1T z&gUDrqlKk4n4CPpw!8dEeZiJ|H4*4^T(AK)S>lAB*aLm038-_r+^DV8BbSp|I7jYT z-@1`%WH`M$sEO- z*J=2-PxBCjUMc>jI$n!y+&ooIWY*>1?s8chu#MF3azXVJo5@u{6ErdvxXTZ}M!Vb8 z-Ck_YyEmURCwawHvRI{nT;`YR@4Ao;zVGJ7IsepXY;~npQgAZVXe~PhE4&}pQ&{Q! z&NzUhVo?q@ixuWq3wb{y#}i@$`I8n{{Xnsv{b?a(AC;;%D>~b`a6Fcd6SE#(ZPUH4 z5fBzKx0Mrz(qB{snxw_tjp!ONP5)wF(YpJcy+!)h)8_S5{Jj?o6vS+^#G<~{lQ7hp zCNahbT>{bQmO1sAofe@{TYKb#eOf(#ZbQTopKXskQt0pxEH>9H3Gf~hv0jsF;&exi z3lp6xpw*8Zw#%5av-Hx3XT&lNCgrpkJ7jCMIwljeFh@}+`lvjCu6;gSW83nxh}=fd z)SX5#+FNQNJK&n(teBidGX950+K6DH#pd$jv~vLm5{F)2TorFohmd zBTRCv4W(pAxIqOaW%nTM{1ip3idM!^-`@IBlJwoaj$ebD5wTl>WR?AbN)>Njq)@d& zA9KsLgXUfdhN<`mQ>fRv(DVq>S)DRSdh`!@>4GeBJ^qr@hGyAwR?S@&3cKmM2!@O7 z(+2|lRr3+M%g%V`jUvz`zCLm_dk=F=My_tPM@FRijIrOltW){R+vZV8P|ATp@co<3 zyuJDQ-P2F4dDW{!52nAl70RA-NuKi<`t|x%ySPq%I~$S8J!V`{edajuBE6{S(zuio zm#9E4+j9!x_Va+)#&G zqyQ+6R|*wO&cH7ThdmE&gxs5Jw0#??Qm$(ks;!9k-|Q(;3`w%-k-CDg&#dz6H=mc) zs0je79Aj#jV)Ha3Z2oGd8oDT8W;eXDT5e4MjB2LzQWzl`L0F_3a@>m~3TPPL;pcA_ zU+B2V9zf!>A!+_AcD=h@+*Wi_{8)HiFjOOJmfY<6Y2w%V!(|Enx$&9L{r;-8F4tcs zN^HVu`a4pFMRIwAj$oNmj@`mE%f7I_URh+i&zo<);=R-O3v1UQ=I-3dh%z6pPvk-q zi;R-f$Cn$J)txWW)#j&67Fs!6Cbft_<)osD+hb$>c46UZVdR=C;?cC=_7wAl!z+RD zJvrC=B2XRNaQ0`mk9Gx=JWz7_Tgpl_s#0*e>uR&Qn*QNlztZQu)uIc9dZP;Lj(fA^ zwMJm+w1)B^acb8@Zt42ED|s0_dtFd;m@uz$0rrbf6fZj)ouTLOW9zAPEt4Pu{DYy( zo30-7XRsQqG@1GT9_O01me_v!1csFujo&lUuw`S)rQuLNNA4zO>}#D;U$h(US)N|a zM8y4AP|jK5DaP>lY6z$V)urs#Pf;94~{QcsUlF_xxTlG`Y7C-2M%2 z6?DFKyqE`wJ+=zHa%9RT@kyUPCSA+6mnHI4cd96l(_G4wVclG2MBR~RwXEynIA+DY#iUi^giM~P(fO$p+9T;(5NnfS|Bf0`CAUF32|?D zld1xhvR1SUVO~U)7(DKVzY9IqnesdL^e7lxF$isAK97rMs-}Q(JO6(6q2a4bA$c5v zaex5hXbe~$v$9Elzm!1p;GT0R7q9R2?6cYYw$%<@w?F6#vA7DYLVbT_*gRTB2Y0&` zP~IB7W(HfYwQ@t6jkAS7*GJlQ|D-KA;nK6H?-9msKvh(4ii4$jpx%89xJFrhjRxsZ zv9sJ+IR(2bE)gY?b&k`$WjrKWaPo}(R!aKNs{grQFVfp>Errk)RuyBdHoo8Y;huP- zHZM`f6`#&-;3pdwt_h3B=EMxqf77+v&fwr_O?S;fj5}*Mws@!Qjpo+Fhq~c|@~bC@ zUh>Y%89Y)Mdoc_C>8KvZtk_xwW2G)A#SkUU8-``ZYVLp!6Y00zqD&#` zzr}ZzFUgNH$<%00)+zYB+Get&v#cqDD9NF4YqDVyMw-$~DmR*brx{`Chxq`vO>B_n z?cpT4o-O~M0;MCxWR57S+G!E<+Plk?Arej}lEf^Q%Niuw(Yhk4mUw6IZmCddA;rjGNsfbT(790z~bfETvb;v zp5B7WY7pco)u%cdzLS3P$E;>sDa-XivP+NK;3174C`Cc?eCV9q*Y4P~TDvb7qFr#+ zrlMWt#UOVa^^#izPjIe*xO?e9DT<9;(&UQpw>9UF(OX|*U77NII34`PhbkJ#)4ul& z#~W%A|Nh`N>>f8-4y&rWwb>`$zgS)j~v!QZR zWv}!gL*Tg^OB(ijib>FSB+=R*6SVzfa*<()`$;|6l1kb7+D|@O>J;^0h+ny)~z?c%%e* zT0)&HR8?j^R=ry-^_Dm)r*E|~o#vUWZ^p{}!x~)mrc+!hV$WXOeujJ`RBx_-p~29 z@tASN<>bi`9-i(5h0orh;qdtYR%7uAnZ5}oPYL_kw>*}U(+R$obgngb zg)jcvR$6P#)@e5Oc*pOW;3r;5y_A1Yol}-1NfHovv6tfZ8r&53uUJy0{UScs$i<@) z4~8=&-<|?`^JCEP!sJmuiGWJXm-qF^_^Vt@P({6p~2j168enUoIW|yn@ zs@g$+1*2VXDT=LY@B5MgTv%Z3_Oh_t2~-;p^LjRuWFsS0WD9*FWJ4Wf?op!4HHeaL z%9=NQ*9~i$upRCzUhAKtcf)PZusI1<9=aaHN{@`)XII@1wvG=& zrS@E0dN&ihE~OWG74GsJ!V=mjMJGZ_zR!iLT+>ckaD}2iCsN=CRo0E{^(%3KGs`Y5 z@NCYYwS9pU&8tqiU>@!`50v}EWt++r{D1UqB#6N1g|Ipuc-`E)#zk@Nazy!oe|tH< zIU?u-R*+mpRH&xh=Q&Y!g*fP)v7TMVyjeNEiGM*J6)P9$o#yLx2z>roh_%*CpIj^W@4tcH58}z%U}ol zzC^XaQb;5}CGTd@>5YYy!F~VB^FTOif^?>Df6w6Svbw2AAv@1duRZ@0rH_$Bwx%{0z+5KV{Nf{nm`1`>qq{boO8svkMnhM`Mne z!)t!|9Oz{Ynm_JI=7l{P#8fo@ERWYgm90p-4jR`aI((K3v|l-HAHif%zCFy&H;}Y3 zvM$(g(b9{}vw6YRRKr!fX%7!eqd8Yc)9@S8xwR#!ebJX` zKcxg=h@e|3$WY6p#1_403>b+DufUtuwx=PbkKk*P*Z1B5b;uW`%s=TN7jM(xW=l_#%<%7m4TSFzBxk`6VyMs=< zH^3J-aJPyt@DWuiHf#pz7FBo~fjiBxXSVXVgBFl=nhivNEJOU)IHkEI9TqTg*a}=E zYr<*Os01lEWhbA?WFx3QB0||inEiU^!^S?)xsfSXLB0OlCI1RwS}DfqH&d$YB9!0l zh5rM$p?l#VzKyFhVB?v58ZUukKcQed9DHMHKsI6!%^cy*K@&@s-&Sh*Y7w~;biNkE z2M>5eb|ctWq}AAvz3pF=KshkE+J+!u9>A%41h(EV{GUvhbcLvj;Hn?f|F~8JlC6z% zIMYsUVd5S7{4v3{&TSSDGsM)ebn*lzL&2~nf0iKn81sOms-tLB1XA^+y8-I~OJ zlrVVfn;VR3VKEZ1Zc$=kE&n5aBV9|Xq|J7MgJ_(w!**%r+ls9pKa-~yy-+U{JGS$3 z47yNR#3{5|H-4Lxukx9J>>#H7q{=ZtbCg%`8%Tl)oG%(9RuQCf|$rJ$xj>D>0 zvEVXGVF(NDgPb9fTwr;q$7=uM#P|7&^W&01pJK0~!JzYfez?_r8#4b=-v<7VH;|4V zYP(B~=!Yy`fOn;Y)W%cX4A_UF*Zfb;TLIjmWz5yhD}8JQ$QWiN#F>@a8jb!PcK$xb zTjme}Av3lj=kX}~j*%&NrASMcMb6xfBZI?d4UFP{$Xc@H0;yaJxobXeqW zd4^AEu~5Gt4)GJtm*$G2kIr`~BZlG~r~-&tvyGMEr3fqDI_sIZB$;ra)2F|S!W#fdx%)A7nRxfS-NK|RC3S!7vyaBtQv;sf zb-$+O{(FOn54nv=hL~l1%(^|(*8u~phrwcR=9W{2ouMslX^VWv#a$||bt#pCx1BA3 z0p*p=ZFKuJ4eiM>7P#q;5s##-nBZsImGQx{bgpQpI7+r>j~qnWcc6ZUPG8bzx}Z(_ zX&jA&k359-mw0DNV4Ak1qJlZhT6uaquK_8FEfG90j~piJkJ~9OeF?}dzRT<`rG|~@ zgnLc4uKLfEC~hh=18=-sC}f2?Q)TGLV)OX~7nrYK*DE_U(EEQ*v4GI{b}um!0Sfvj zkWz!P)yFFE^3Fv_fy8Ma@-Uw9R8$!o?-mR3D}PoJ;|2Yy(Ea?#i%V0%-q~4)qXE<$ zlWVqS*}|snRAb^GYU5;MJe;#N{zEMl@phoN^B&N(Yka(Lakfyn8+hDMNGc}bynlx+ zV0+{@d$v7mbv8#W`tVK5@ta&raM5!@)=_0a42pIrECV2jxRtQ>-rkZ2u(v%ERH6wS zWQ$L2vU#m=9|fkx*uKDhh~Fy)xm?w(@jx}bJEjHT>%L2QrS*R=ejVc+%?ozLwPu~E z>4`6Vh&VZ5Z&br-fw^iz&wt$^EKH@4r$iiaJ^)x1`yN_qzeGq_2Y(+wX zsGG@jMVb@bZFfip%Qk-X^tTZQI7bCXGuWP1Re-flrAh*cq1<_Bs zScm_i-p1-^lK0m%9~QBi&K)-&TdLziz!m|3O?Y)-UZ^Nnuj`wYJIzfjLeJ5A)!p@U#hEi;0Tzv?l= z1tMOLy-laQ5{^%|93JH!HZd`K;e?bRHT1+@A68FPYw9;tY!sZfzzLD2Hz>0(Dv9>M^A)_tYN{xltGTu7T*fbmB>Mpc8{AL^Z z;7@1@E^_HFO7~`v0$~~$Z&<$BUO=v~_4~0QF9%eLK)BjQp5?GpLaFjd!=dTiX_Y8b zU)x&Fy*$uY2rSwiCR49pvrdOwd$zAQZU;I)R)3CIAl=HCPeol}vbOpywUe-BfPM-K z6fyaAA>iW z0oh9O01!GgG6vAHgT{dX!ssb6=wnigw;Sy;zQg@n%j4cL1%tf@y$d!%Gn+vL4|{=# z1l3Zo1Y4}!G~o=14DM94rQ!p^=Th%>D?ea%XGK+ABqv3Bf?jm<`p$k~Lrj{ruK}4N z2k3!kTabuQaoP9<;~eg(U)J;7I;2erstH$!5NcicKp&iLm230A$Zsp)RtqC$NuzzY z==Wt>LwA>i`6p$9e!mvjZ3UkrZdw17Ci#KQNc0SM!zSXh=gHLPbKt4SoaRQ|TY;gs zZ;ha`23od*xz|d&8zEMKrF)Zl`g0~Zk zFjW|0HS6=~%cb&d!&SjGG1#Lx@K55-``s)HfE?!~e+$Xha>M6{-0bF%l>K57>EPd! zyS#CiOm_J?Hs~)^V%0VUkhRr}=emG;%s}QPa|dhxtJ33OUy1{c$wTpXZHA`*QeDoD2K_?I zy+m}5OTCSRNoGJQsk_&RH{xeL6nlRzyIKf+{340{@qOoOT{DViv-$qB%Y2H|zQ=0Y zvS@pTvDS&qk*aIy0VHo*+jGmJ`y;%y!8;@ojaJE;zZ#h&M0xKAn@J7uR(FqJK;AFd zR9+g=tYN<yQ|41yY`G?kxsLM-*^r4(LG!3vnCM2n3&SHR_F0a8}!-hYB9u;HVGYe{=zi zdvG-`uRgJY@5P7}aimHc(+O^BT6yk3^8dEU?9MLkCve!6|>dEmiVgnS9#P=y->U^|FEhFDbbv1~m0zXMi`%J77`1?J89a zwe70n1JF&MCuqy`LUUD2A=yv;)LH}XeZNR&?7_7zPHBeVaDD5 z;-vW?K)^eMzl~^b09dlcUB08zNZ!luam^{h*^VI*b%26(+1Pf7DBz2`ZoYL7nMUcYau+Nuh+#4J%ki z+_*O67FwS^ixDo=wZbQKJfqMYVxxPm_oVvLyX%z74(aH&K?)(T>KogBwq=TI2zBV& zlw6x%JBG*cz0!ZJKl7Dj)~z%cadq)_4xEsbKS3_A8^jmX0p1r!bZtGxDDHs>!Lf2N zJ`AhQ6a6)PYsis-l6jkp0wn@mgN6T~XI*egWtm}RoZOy_+493cbAWhYq`0mod=UIu zgr{5w;p#eJZ)C+eJ6NQ@e^0$}do;|&k)|h^k9U*9Z_VY=+{O24$H*BOrw%mLxK@(w z0ZbLcIeW1$s_0kW^pEcnu=?&!el9hKyk@OdbeE`9VRqftQ|hL+p_H?gO@5maX?bra zG=gm2h@CF(2(6hEG5~^TAg~`bk&{B?E^y5`E<_Vt@ z^ugsKnboyqYp<+6LrN~V5ozUz*;{Gd5V%3M6YPj0YWRi70jlEtd$;Z=J__}k5-NF* zN}-0xCG?4vp6tt#zX=mZnV_F@Q(%HOWy>q*gn80xjitGJY<=)vV@8x-Ll05pdB;p2 z$4@0(pD$1cW~ayUR)$!Z<9miqyYjFCw<>n0iS7(E-!>Y{VpV5j!GcY zr%Y(EjRzo?nBD!(e*)2?Q9Mp)5gSk4hiP}EOl}t=GJ(Mj%mW@J zZWOQC){Afll>;@hP9K9clGdEa7$}svRdp$Zj`lKQOIRx8cx*g6qMDmoPiME?e7V8< zIos8jt>6`|J@_j^xdBIPht36t8Nrlqo^D`JKD5@Ru@%WkEVj z`j!h>B|)QE2)*ghctb__28$pMvvcl)CpgWsHtJ8i$rXP4M*p+Bx%*z)bC?r~LuCO3yS^aKgQ z)4)86-Md-N?0V_B3rM^VCo4L$9q<0Aw*3L${i@t+rk5V**gZyad}J(=tYb*7RK4sJ z)slzO*-=P+zdU}m0IkeZ??IdC$ZABiVm#*w50?Bh{GRbZ8;R(v^kJ@ijWl`F7Eyg5 zToK%hE|ydut5HXcyElXW@$E-X3}Pz7mOr?gm&d{oxVw)9-kQWN8XcK#kWhe06uVQI z+?u5Xpd|zIHix$?Qu2`QIy3*zNZz2+y^a)|*r^Wul#aiH|0RcLmKJw5Pw~+s26!_Q z^F{lREB$o6&94aSJ1LFLpjE*uyNELLoe=|SBF0y5Tzk)OM|<&$Hu(Fdddf2N95-ov zq4dobOn{eHRU!HBCmNa&jF$if6@pcu?7b{7L`<49yblm|^w) ze`l!>cDHjuOKt*}3~pcg?33 zvlUL0sd$JA94~=*$I)8e)lTN1FgcJG*RpDkZqzq^m+$zI%a+Lp$b>t5GZn= z#yFeV!G^j(AXnTsYZUrkZ9hlb;tJUz(Dm7W6$bTokRKBRYkrgi+b=5GpdIbP?)cgF z^(v-P3IWMCmPyZgRn^azHjpnx;Ifv6>1;rloK!28<{@?#Zd`Ecuy%( z`$m7o^Mam)vQdJ;!I}F`iz4wutgdmfLKUa~p-lx6ancX42m7BzmIgOO#Ha8xiY`KifL3_0p>{ydwNV95 z5Ywp0z>4}t2W>O^ywweK?F|>U3~BU&9zw_-HyK>XuJd{P2H!r!jQNxZtBUN;-y+X& zRs_GOe%duiUf+jv)Fe#bq_G2w%)|+CE#7ngOGDI14D$??q|UHEAdeb?C=F$`qR!&f zdMaxSvlnducs!qV7;k3dninoPQl#vPYHet*!4`i^8~g&P@E42(1h~@Vf!8%{ z0~Nc8!w6b6FVrrkpTMQ@V^EntS3hu$sL~1^54i8+r93@SIM)_?da0-Tu++JEheUd7 z=-pbrquevG*LjLMjFd{phu1JqmE`&(u)?3fSf1GOEl$2ERE>l3gf)Ca`Y=;`F&$+t zL~n!5MgHEVilZDx?%?ehlSU?3w@w^iY(%R_e16npEW!>qGO#LW`;Z|%@rhA(;n6FB zuiBLIL|n$^$kPHI`3K(BvTBTEYwRbu)%_71iza5Qc=?3vX~B1Y{Nj3@K+v;Q3r@9{ zd@LWn9@`X$Y-Y@cavgj1FOzG|UE)0JJ77{X2v*zb7Cv}7fqon*#hv(TU&aF${ujds z3N^E%wb*JNfZ+#EQekgw> zLhdsN686SL6qktn`Bv(fbkO?_THgGE5~`o(GHNBplcg<%2tyJxg7wzlDK@~fr_f`?%!d4I+)mPM zyYsHz%KR%X`Mf7sJAO|)k;j?)*8_0Hkuqqb#yM8m$~wu`Dldo zzbQu}=oVaD_k^EHq}TOxm(@=wwKt_GA->ox=zUKyFOB}E-1zHkW+1CmHPqx=RJoTz zFY=A)k>SOGp$Uoo(Fjd*xinv<;b5W?hR572>zve58E@wmYf(|j%@6kcil>J7L8(QN zyLwT^d1RSXuPUq}GTYct1o2&jo2>_m_uaV!gV_j7PnrnF8c|kOSMYe5;j>xw?eW5` zdNazYFO_N5Hz%nEUw;J;z0OY*`og#K|1RDt<*3uc`ttFHOwf;#%2N6l67mQt%~tE5 z=a6|&Y-bXf4RJ8$!%z|>Z$tjfWQP4Uf^a!lnt!tMiBc~9mL^e5W0<&=bu&GH*Av_E zn3+B}DgH*!kT6osnl3{etQmk6`xM?n6&aJHnl&S;O>_MXvF|2S{KO@CFcE-B_mmk` z5Af^qw0uDSN#ciS0lyCwGDH`a$(4OZ?3B4Z^3x;EdE=bm+(G867Jgx7b!t4_DtXl# zxhAOcYz5vA6R=fdqQ9BvM{g)S;kVzQR(W#P=5kT-dzlGm>v0TVdo8f;4JVnEeu73+3+RIy!n_(y=}1hfp;fJ>HRm3FB>a+%&mT_P zHOZpCecgv5ZQ7}3aNpwyl4wE56}uoIvN&y%x)U>G$H?s9iQlY}BaMn`eofh;vL4dt z)bM&K>XAg{|8Om)XdZX+s3LSb1Ka%kUW}BzWu+;LBB(|`7}QU(Q602Ubm{uGe4JQ7 zPilr_C1!K>1q1!9$}3l$!XnUyD6jVe4*nY>m$sZ3Y8akXGAJZ;v(5{Xl^o&r?pu$# zL+?1V1w#K#EGqABY`Fid!3#$A%a`s&?C*Al^S@ab>mw?cwu0jiM9PmT!xkcm`xu*vXWHMRAfT!LCD+Dbfae^Ib%O&j}Sec+tXzu|uBA#AE>n z5&xHqb$_ES)ZNIV^229FmR_}qL0(r5Kc|({NDu9#&mL9As;MxCYSA2KSSH{nr2g|4J0ej?70%>QQTL$b0@Q=%q8|o)7QH2c z%ap1hV(I-IbG^1Imy>!#N;+!UTW`z_ zw9-C*cqHMgrRd&KT&Qqb9;(!RG~xYrxuV}> zyOL+a4(Wic1P-EmQU1fz;(rO@7)f-xtv6-%7K9Bi)PYemnz; zq;g?LD63I8)K?gHlbSJTK=D5J ze%P|rvqNzehTl;4YXfH0N5YhH)w10xKhZUv2lW$8ZlXRwL7O)M{+`u4K?^ht2jT*6 zdvM$0|KpA_sY4&Qho0 ziE3(|tuvQB$uxluN|Ob^aN3o^xpxdihgfbrDkL%E+DF)r9vA1k7aAHOu{4}_o6nJ? zKJ}}O>0f{i`2k0LzPs@u!l%7?Dvrq$F^&Gf^h@SB{HOgv-`4@P#V;LlX_fch@S<-@ zr`;Nyf_m~9#U6M)dr>>EqWaxL~7a4rUxX5Vby`9})xnBRQjeIk`5xmJ!N z1#->Vk=s_jRv8+xbI};GZK^*MvViZa4T42GpiOlcQL*L7p8(AHs-Rif)u7F|OPWmU z(Y^(DFkcYn-Thz@Yl_oPxA;#Y6dV6J1pM(qK4ILmHdXEy874>#t6r7N|4j8Es1K)tn){@dhQoxEMmrqaPg(>Hx2LJKd<8JOju=dB^xQu6_00X8(D@AaVFw#ZvYrW6fJOx~ zS8n9RR;8qj&s={jzsOIIPOlZFMM*YaoRnOsnd8W%(qnxpjxLp=aXwWK^^2Rkszk+>2lPA-lm_34(kq+T;4jU6YIU zue{GUEa^TWQ?{1@HeJ1t#MCT*!;;*`1n5((XxP+U)io#4?i3**sNX5}9=pPC>Mx`) zN2}6SSj|KfD$LJihJ ze9;!i;H=sRbL=?FzPI)2B#EK`2b>o zrY-}@`DxKajB+MM{u64Ar$<@!nh(f!TDgEmBvJoE`mk_u_N9T)loOPTi~A$M6}(eV#-{c(p2;MBevGEeWPsxQDDmDr9YZu{^CB3d$Bk zjst}g_5L$6^VKInB}DZ}dJ8dKX5s>P_HMZBnjqvV_f~KNy8J@l{egZBv60jAZdcTP^Mv;XblEC#b8T z>?}%(P7;@s;uXofl|Yu)9%rz6e{0K|Jz5}v*fJgxaT`@~wj`NSycO`iEdO!0MZ1T`miBSXrL!B<}6ClZF$nToU2I9&uI z=>$q`ISvgGOKdcnt~le>n;{6;%dN>_t){!o(CX_oUMeG+mU}O9PsC<%Nr3njmzbg? z5&M47sf?x-O4cf67c~7VQ-P3Mz&kt$hajU9H2UT@Ryx{Lrxr^eyKd^*>MK8=ev>`K z%s*p+Pe`$_+>Tq*8QG4#OOl%?WOazFW6Q{%=h)FX z$KK!LzCXY3_wo4s1Lyrd@9Vl=*K0h73spq9>1aOqUqy#~o17_9m?G~9H?w*GNIEw& zwS zsUs5uZWc*Z)aK#1|0>~ZqD(-@eHQVqqZ&u6v+@0THA8^xa&pA(Q7+oIXz=Z`b3C(R z{1w%0KSoPwUhX|;?@N&IWy9a$KTd%C=}9?xF_h2LQA33;FkG0suJsk_h`89fYE*sg zxz_#h&+O|kkl+{74wBU#@a}Ux6kj1MVRJ<= ztO}6$xfiZ=2uHIwD>q2^tz5{Z5*1>)QToEoa{wz(r>GR7rO}dkTJ?|P_2P=1nWq&5 z+tkkA*V4%xn9-l!r^Byit_noGu6q4Zp>vgC>EI`J+y1GS!*hv#d$FjR{760RN|>}` z*RYl=rX(xI;xX8@YL7bS`0_6xWJ9btc+Fx)g(_(7erDpJV?k)@O&8;_qv2-;&QP<1 zBeA|cqCGz4czDG9T+tw6J5v1?7I!c{%%YPX2;Q8j^vap~mb%=TkQg4c?5N?>@#^Gk zHDA$153h5UEpi3-Y1GeOF5~=XvurWTLfn`C%-`XwS7h$K-A0u zvDCF(lazVI??Y7|QZ(JA=AvOgLJ`81ygD?~8fK1$RBY&G+ryqJ9d^^NnBw$OlOdnfPy%y;Jtqb0%r z>^QmTAa#!Zm<3S3Ft=t1;0YEgj3mON7_U%CDW}IjPG8aj-JawHw{ba^l==-9GpBd_ zyh6kCU7CrLlCP!SP81Rv7tWH^%+ibeNIy!lVUIeFp~`A_*o!qqPN2un$*WU=oH4~x zrZ;J5HP88{k+MR9-ZP9(%W84VocYv*rWEs;-rVD1@im}@(I5iXSDCG=FX1s z`k7c9?xQInU(L+Ytpr;10gv1I>{I@wU!GTUcSxFcAB1>aIjAAkRQwAcVHiiS+vn?x znIy-)b^Lf;$x&GxL0r3^(HhRsnr&fIw#CepVfJ)4bpO(V+Bx=shQ z($O)JP*c^93gW7V_t8-2nf~pcp(rXPzd8Jer)GQV*@%;8eZ7(N zZEw?MFHuL(VvRk6=G{7@ZCLzwqDXe6N$|2pV2k9R#;W};Dgno=`<(mivSr6BAfr)J zWFB$9FZhI$6&yE$srRsdQFjoIcoKRm9n<)8d5o1IR{@FymneV>KXIH^?EE!L(=DT- z2-{i`)MCPA_K7A8y{Ek-!8UR z^Sy_6rg7(Pbo#q`XUI+ZT-Bf0w8>64|A`6abet<8=(89Qn;JlbQB4Q_MXtj;joKP$#6za1sF=iIs$0Y(YDeu8%@7xStAl1CbD&g zFkm|TehW>d5z0e6*=JFojalcDQ9B1`Ue>=HA>Li)@m+z0Ln==(|F64O=0wR^s==hn zBScf|sBt)^#q7>G=4{Ker9`K)%3-8Ykf{>eZ((upGxKp(Ow46;;FyvfgFNVX?6tV) z(BDoP*SQcMs;Ci_OZG>BfwQ{pE5RMK7|%evIXF3|W$8p>33%JsCRrE7&%yknTUmYF zQWnu;rT$N~==x85=J!|hYjnJu-L08*#E+q95fnK(kHK+Zpa zbD3(W!)QmCUCCwp`lb`pFP{~skz0SVW<%pHFONoEOsYL@(GV`Y$xIU1a;a}-={Zxd z(t7E4soxp?;#%SFl|ZuRAS+9}ntLTW~NV(pZ8#4q)oJ<+WZ+Xxm5;TnZ= ze2#LGP>H=_U#(;Lz&7ZV|47V zb42Eu%$sXTSvEclmlxL_)FBUkxr8HiMjqvggU}m`q*oJg-oO+}@70 zyPYlDi>Y)L)~Mok_54R~6}RliVUW}|4H=W$Qi&f-G!nh>3< zqT39lzMM&ivixs@VRNuP0H5R#s$V zH{2~1doy!UwV%#6Cy+h{;dL?fFFd@Am>);G%&M`z9QU%}udWjxlV!Oc(Sx@4+VQtS zUt31;TOjS6VtAF$6eJ|fNpH8cue7ijF;2s;ByhecHi+ZwsQ^lGf zO%;h7pfYtCp!|o16Yo~fX~YEEnswNF#Z~^eyH1ef>Urts%5|wOw;uCK9Gz*ZwX0cq zUzLxwZycEU@HCSeE`-M-zs5MTnaNMx@;v>$@hbbm(PN^T;w%3B3T2*W^veRkz$QuG z*kz%e{qUzq)U#@ZxHv@_FB#_}s&J8ZHH$P_^0f&n6yk z!(t>J!>+kjWGlE?k7c9h7_60i(SOu$DlFp-w+aedd=rCwcw$Qqm|ZnD8;+aAtnmLx zj8z_C?|HlGHY*~g;n->)3A*jv2)u32ddY~jMMhKL25YjObqZ%8XTH8Xck-7j z+;UD=F9_2zMm)Jh|Ke>wG!#?;5)A$PDeryT>1=)OPj{1J;`GLnug0!AV&?nqilxlT z#CQcmG!kaXHPe<$PVy)#g|Vi+jFzxBl!@(Wk{OC!JU+w1NyL3!9X}FtZ*$T(=@I(_ zk~QTXS#;ROUUNsJ|G?|8w8^BS9DCRypNv-TBc+$d7cph~*X`6jskY!m`BYNk#7L2$ zlnq{o+2_KdltbIK^-kV9ifV7-U?@ZsUc|{o^wL)*Ambq8A$W@}0UUx&p5?on>}yi;y|-5)n~I9XzEwUuvroMW+TW2*_jp@VcM z?^KU1y;t{oOSXUR&|RdjHh{+*7Z=kr~%C-|d^tb=}mI5pY zJV0jsD>|F47stU2Fs3*90MS%0FNofm6^OC)Mg$9A3cOHr)kJiGtuU*q$l}Fk4Vqi< z_l0mYRwMK>|Mul%@KY<$%4VD1B9{l=_=h#)$}Wxr_X{Vz$bWbz`l6i(OiqJyx9!{4 z1>SW1=!a<5obwO}jA(YjmNiDx=BH&i3&w`|2^3w7e!~hEl2|vW-}@&jVBo$Y+A>Pf z^otDf9gAP0)bw-LL=I)`U=-@bLWlxljJ`wb8q(Z(FQH&*gwu6ir*AYgo zvyMcszDW+1+Z)-nSmqFf>_EF#1t~GI!XbL_$@MgniZ<9_eiAP_<*@~h)DIsNAh*z8 zyN6?r77;44TZSumuD&+^Ko;jQ@;m&Bf0_3LY0Yq7Oaug^w-j~i6`IOSjq1REA` zV(~o1zq3jw5-V~$?q8X`VZ}oWxxs$pKKPC4_fNS4jW_x+GE17B+5v>QGxAtwrg#&# zS6jS{d#~(oIi#8;=g6X{T}PV2qZqyY?$7halLXydN^(P{>eR6Jr zZnK2mW%KP3AyMIVCD$AemnBwnQXXIbBDk_60cSrL6vFJEfu<@(bHCH9HRgWb41IgW z+mPw0w{1k?$Nc37jNoQ=Bt^%3K3QvE*nc6}RDsYfEr+Ar|Hyp);=u%s;skwEp6`-< zb62Brp$jOIaW4uRC-RdS$*FgC4>1~G(PmK?~R{Hp=RDau#WdH+G%`T9p2Sc zosW)@lm>vHu*X!Ujj_eco>|gW!A~N5qM*sEw$Nw#fMl0#e7>t=ZlEuA>OODW73IpV zhATebgK=yq=eQ8(`Gk>n0_p7uKnDP}Mo;k^#<%cntqgUF>d(O6AKvpl*2$Pvk2gLh zHg)CwGL-HA8ZdvcR_@6^h{nPDnWz!s2d(m1Es>oC)c4iOK|_}SQqQqhi>ZcjocgUc zqBGrUq5PLlc^gYE4dAs!b3#{n;_<6C`o2R5zx8%|!n%_EFtm$K9BB8&{H_yF1O>1zq|Zz$^H@pFMp!GWNB(z`>wo z-3O7`&xO=5_?quwP3&bM>{d;xyjlAyQ)1IQ!Z!4lx?*Pxh%u|U{oaE|;rNddD=xoF zoSl=LypsF2yWA`OF?54NLuq6Z?LP#-*zp3>m?WL{C0RN_POF^s&0f_qAU98)I+a-? z|2J%eDTLco;$vumD?Bm8UV|~TSSvo#ZTU?xh05V@^(&vWV|{DNVfBvFDwNUM;oBR- zW!~U*5EOTwc=>6WE_cAH=YyAkY?r#K2lSIDlTOUTYL%opV{^sotU`hxQ>KBiB0J0R zreH#X`07jM0ALzUIOR`eOlb$>gLGE%rhORw*th0-qeAM!1=*XcF5MB#bqQp8G0eQM z{Dw*>Pc3^x3GfJyGR+Ct%XmKNc$8cq704=u-sW&77;4|cA6#cD0Wfjq%E`RglX3?W_$~y}AGFX`pC1hu}ZSxC9;>~!+QgX}gW1GkFL8ouzH*bcjNUj(uU_f$?yHyC_B$-jQFvhUaA zT@4&!{?;H?p6&U(?x?7hmMCX|NexiNtaO2c^PJ*OWg#(9sp*~z)(i2O9E5@j>C#U9 z_FMrvqu;$;a)`+Rz=bY3WFSbI^8sZ}h|^}Z1ra!aj~dv|(mDlWyu(<}k*csfZOIp) zE2B9#%D0-0;AazZ^m)ADg=k5L z6lnwO-WC3fE&r9d?W>Q%U7fy9lY=jSybkx;y|JsGX@J=yvDFri$A$023mo6(J=wY2 zu>_gd8tEujhb2b^4S%-oGn_L*Bcw%fRjIshBjijhe;t=c4ka7u4CNfEN8yfIdG z7xpEizB=Z#w5MGLT(m<35v+IX(~`J(*<+cY8dkX?D8i<;IioB?wJIV|GKH+y160 zJ=JyKa6N%i84-SOoZMM^ike3Rj~8U+A8acx3@Sce*f!xdbKsVCi+0Xci*RgexWppe z7Pa3MnqS+TSgB;!zUSqg?csYgc|Tfdvq%{q`icDXh|6VFwu(=VB`nn=f^<3BdY>&W zcb|Tg@@4VPIhM#pE1lLHzYK|42IKlc@SlF=exDx>7L{*c>(EcdS0YSOs$WU+f%=!z z73tI7aj=hB9}io9tFtPU5T@U?oPu9f1B!Wry8$pv_yGW3Y7OWd;2kMHj5VOuJ#EbF zUhSZrkJRTU9s^GRQj`umil+uDu~O?09SSJD>GHeMfANHno`?8{9YrwLN{=48e(B%D z^57epzJ{j{!r09s>eXdC9$$+!9oJ$svVn0GwuOg7yi9N%ZCKQ1qQ}~#(&fJV{E~gE zjIVFYl!a{CCdCZ>IxCpwyQ!*`gAt#lt}+fOh9U3}xLSK<-X$ zNs8c60y>FtKH2mUl3Ys8WB1qq3Z|ma99oOq4&8d*Pd8-U`#O0qDKB@0mWDqrhD@Cv z1^(GxRGIJ&(6)Ov-t!BfD9YqX5S_mF`#P6ycKbJI&pqEkCgD&mTs}`OhJRX@tBVT{ zO|V`9g|%#sqVmIdJF!uz?O0%bB&iXhw(9%epQW)PC`Jx2P zek6i*CmL|YUU_%YE7K7DU}#P~!?N<1a^T0~4Y~SG5o91H0hoL8&a$^)jo+D{PrupJ z0?4{MT$mjw=o9my-#csA*d}3Ijc8K@ZMHm$X30Zy2#5? zUa(HY<7Oj7eriGX>0q+CB8V=fMHXZy@CoiO&ITfykChoSfbpc$9tbc@?Omv-MxM@0 z&Wwx>9er2!s!re4Sb?cn1$%jI-ZT;!C|dYs#Pl*JY;CV|)gY()B0&q-xpt0`0h$LxEj3Bdgv&$|8Cb~U?@kySF(44l9k7p!)_Bo9LZYW- zWvnwjdPq}!37k+zieVhT#bPfvnylkZ%L!dih?NS0*``d<=CI=yMxOf2{awL(vS4VJ zXW!)jH20I<`P*ol2e8&Bngq$Fd84%b1gi^gEp!Ouiy;9Y2EQMEFg$7%SaGgUNVsDZ z2iv;JVwKi{dU|W`g8?OexXm7ZeS1FSJ%ZfX5k*S4lK$BKI0^hH0lXDR=#?Q1!kpdM z97}W+W371Bj%#F3x`B)O1Z1O#UDAyLw6sj2J&x)f^G2gnDNEpxv%^>T8g)Om!^WOyu!K;S&lOErBwf4DXzkgZPbH+TzbsL71x z;KhHKH++>}=!gyy3h((!>E}v?LIwm)S0Cm235L2X=_0r9Y+Tv@&hL?K#5S{(oI_D_ zM^lwNh`JD1l72%@;gK$K!Bavq@OGAna+eV?U(r9%AZ@aLfXN7UKaO&zAjEtaexOQN zn>5zw9Tj*PO|hL|osLC4#m&X=_OQeb5E&wQvO&?;$sJ3RquoB8DsQo|(Sy-CI^}u4 zR9f})>+Q7?uEBnqX88S-8X3TfI0(k=wOm8QcJ3~fD&AQ&%!IwB(V6eo8dqfaaF51l z^+^)0igwww_N4_=#L8vvIgYp39N6X*BcH_8T#-}OHwB;_kEB%Qt4Q@w!1fW;?b1SZ zRBbLE>(Y+wRS&8>X%KJN{8l72le;Us8=$>r^s#TvuK}ilunq<*^KsJV-1@^sepaEB z!{*TmkYlVy`9FHykh3iAx#Q=`6}+oex^J(aGns0+OL^NIY=>JA&@YSobri@)`53(! zWZP+y?u`xYX>H`GN7Gc`p04L2-@G4a{QVwyY^l-98y}SGQNlNHu-01nnP0!ne=l5T z{vbeEaWOmGGpF`W00fCGM0k}pye2m3EHCk+E7{XUK4hxO25)(_f8s4QagQzP+OF}q zF4rqV0iTNp0*1lP`Cdm#uv9^o+V@h$*ICFClp)nsJ^gH;F4n=NbiyuUIo7?47mw#501x35XyEJ{% z)C>21Y8wrynC^3Z3&@~;WN{ZLi(E95mb>0hcDaFex-e2CWs%7y*&* zRmyQoH|-wdeChslKbu&@M*RVbj@zVLWVU`U^tRD>uk6gd+sn&U-&QsHiWQn3m{Q9@ zXS?{l>vDA-cwpa0GFw+cFzt0MU#>7RcbOWnkLxvrM+!)Q@-q&Im4ye?Pf(2ISkKApby z0Z}ovI>YIpJ^3@!2xhx=-{`_Xy! ze4?RE-wX?}1zud27!?vtA^UQ=EqxTY$c(M-q0R(Bg?yYBeUfPC=&gmwhg}iUT99Be z&;lLE*^-Q-qrR5TfD{LpM+m>2r)5tL_6<}~DXK)BwuNd3-Kl*+hqUUlC8b}uL~-@p8Fg!zVQW%3BMXVaZtBTHhM%yN0Z zY1ytM#?Fe3txGPp6DJQoI*{I*|LB_&GM?MTe;oTz)8Ph_fhOocT(%*wY^AO?E@7d9 zZk|j3QlIvBv&CF|&DUr3+qOtq^%%9bsn9nuA83UBX2>wXJ8D`LeUugh?6&Wz_5^-? z&@%Df6`o-SxwQh%aSrJljT&XyEKo$Q{4P{BDqEtF8Qd_oa6Yboq5w?$#EB%+*t_>d zHA(UP>{F@eEz0kldD-^)!DXJynUa;iWDef3vuk@hczVYgeEI}8Fwr{h{sP{zYeMa; zygn0xUw>#ea!^Jt{hP^P8=q34Oz4$B>vyEtOUK;NwnT3xDUEQIaTPm9!EQwuwtZH0 zNI4v>4k_8VEES$#H7YU)8dyj|%;?Q@X00@T*wAXAnOUobcY97}UpC_SLKCkfn0+1E z)+i0|hGDO~b~l1=QPR&Pv1j>rl5c;xugi1TA+WwmkX2fN<7DS42T-eD@Am%P`S8_e< zx@3fO>^+6{TqM8P0y|!@aDs7rMOW`^fu}*4r=Ew^!H3mzXFUo2$6b3&W-|0);WFz1 zDHGij`fc5F(-Gc(0ZK{As_R*8YF#*8Z5Et8?G<#?`!)PGTeyMBJ?_7{K+)`;S{~Xd zx|P}#DfGhmmF~#Deb`b%VA)9+;kx128_Mj(z7|mp9{~q)V;_3;MfPTr)iUQ;QX%_D z!sjIcSKh?96Qj;q!lcZ$E&}Iu)wRIHTh>r@Md;u$QyrI%NV4PF04B&q-Nr^XmzZTd z`S{?QhfC?T?>4dBqw+K3fJ~V4;PS6*_S8M|dhGZL(jmzA)V3He9yIP65xoI=JEG2C zr*Dm|qQ3CVN>=-)XZG%9?=N}u`eWh)E?q_XB@O2i#!NaRy&%(Wv_Bc<1oDQSG`>z^ zpBY)O3}AVmD7B#A`#>Ee=6TJqwr8az=aJvr*n#N}1=E%z9$HMdca`1J9-9_~jQxhU zsa-!*7Z37|OL&=2iHRlgcmH($yJo#O&xm);tqXeQD~)cl1?DVF zfF{$RfR>KTO7B-K9fRK&YyOf1g&GE|{Ck)@?7$`QQb#S8z#mXz+7833Fs0NiMV<1PZB3h!EvuUlXGfOGj!@^Xl80g<%@ z0es?t!}nhE+q@V5vXJH2dK1GQ)R_>2>kWZ4DXVv$ADEiFmmupq)>SM)jK(EgG!k^g z-O8LdU!lGL{NGPtc7y55y+5KBmzP46s^oevRLNCRKL8-Stql%R;LcQ9x7uS%9Hqn{OAX|A( zX@D?*&Q0`hl7%pkMtp{pNK2F^V&c4)aD0?4YlA>4##hGD``4Xjs&C9x6*E069`O|@ z)Oo&WhWS~ZuGOuSFaW0u)kiak-~7H~<7*|#`FyAlRL6qEmr6>=#d&Kf;D$00hWcXv zZ|zH*FlzSs%Ivu07G(Lud4KM|2@O&*`Ahp47A=V_rk)@U8#-fJElCz3_stS&qYXnj z14{#JN`IUj%uxw-Zv0AYEO2<}B_1e&n?DZ#RBi-H3$fgnVg+SIn4X@wGo+Yi6YmTh z|4Qy@RzHlMR9YDqnNo)IwhOFRiU{YO`*b0$;i%^5K-LHO?=d+0NO~@qdk)yqr&WnP zb_`>tLlec9fwq`8{719&&89yNhQZoLR@H66RHGP34%MrLk&3T&;b8?zQb}1Sh3md2 zRLt8DyiEu~j+n!Mm^sh%FlYG{G-#lD@l2y?d`UCdWLXO2&{_3ab zfpg4xPUqy?Txf98J14Ih?^i^(xOi^%C@MC3v|QAwtg&#f_t~SOU`F7fr%x~>l*Myi z8s?pu;M2u#q=#^Qlj%QChzr~N0?|JRXWAT#u5-fxNx<7Pn?xoYRb9f`DIiW|^;27v z<3f3tBoF2aAkxz4TI~<27+O!sU==#W@^A1oukrn{{EdGDa)Qeei; z?#Q51-*2Sj0fR*cu<;|+sofBCxNuSvTR-?aeTE$yTP7#dS$b1)2`7eQgo$8iq?-wV zulw!#pcdu6u2v4~N!+!wWuA~t(2xLymEtI!b0)b54?z5v&cN<2$JRlAzqUh~HR<_= z0I?2`krY3wX<+P5q0P#d`Il2ZBjAYVw<#lANuS-&KX$&AF=Vt5j~23#|0#+X(Wr9pwv-%2sfL zMkp@bq_(iXQ0EYgh#(3!^ZloJJ%rzVxsywH_RwGzi3Z}FYl1LD^u1H|j7G=?bqC$k z4i*fv)L3^$@K80iOus4Sk{0;pT!BiJT?4{fLR+l`1mLk*QTS9~UAnQBD5yV#oz{qZB! zqhh`c@g9i&?oiHKtJJx&Ho=m<-sMtrPqDys?Xeis;q_|6-p6B=h4*4}=zfQDz4_Bk&aP79@nk5R9Ta1aaZOUN~S z6E3J`%4$`*Ss;q$d1JrNh}8(O5i3co7_!%61S&=##p8gtm$JMJK{jVveHL5}5)Ro^ zaJVbW#p7kxEqENY%|HzlHeb1=!)2rwo8y(d9yH#amkS1qrtNk+c8Up!D;6Lr+HSHn zR~CiH&Y@eRG%5F12_)!wDZQJ8%Jb!zY_`7N<|{Tlzo`Ef0M|h*#Wj8Vdncopg?HJg zv=sz^zt$et_!;^{Z|O`_l0>x#DfUqcWk7(W+%Jr==ZoK&>-e%s1l&l97y^}?Q;{`- zP9d`x&89Ds0eAX^kOLk0NS^0t8&-2V*x;jcLybng_qHx#LW*pLoe3iKf4_| zme#HQ)7AyH^O+B&_jGIany1NoY)kDzoke~b`S|_YzfmUL*A@klwk-D+??LI(Pi6V5 zB4%3CcR%V8SDWqzmr9gkbuxQq<`Xt^4$1=qvg68tD!=2FkEjQTZ)ei0&zFU|tXIG! zK<h*0F@i|6N8=^)p^$1%B1abXCaB9e zP?&LhtpB+NBkL{|(nsa4yupX?_Zbrla*;ZOm)|wXZ~WOWjSJz@pcxsoR=#T%S9Z^FZ$3w$RsSsf}_4Hh^2RCnZg z^Z2Y~2Ezzu-z5wbwQ#Y{XPID*z;Uoh7Jej$|6NX$$EEjKxM-YLNO=noO#W$PX8p1k zYpRl?n}Gsqkc--X*@%bg>BYwf7+Gxi3Pwn3tmE zTONAFcK>E)oTi|~ly3j)c0n?Q${BVEYyBlX8bq#Z9*eWxQ-O78IWHf6>m`d);Bn+M zU;VP}HNhQAYflZosXmxJ@A@BZ`W^-#kQ5v*81dH?*RES6y|`|z`3oj z3wMGnQHsxR`2DftpMWW1=pi@()OQ7GkYthx2Otq>fVBBp9#`Jk5l&nPInUyW7m7XF zd#u8?#Os{zr?w4s7;dk)8dsf~Wb6)muxcu`)ybe06aauY3_bk6Vbsff(RMRNxhKG3 zL;eQ|?*C%T=1sT0=w=xued6)YOF`7pC%f^YqpYxmt+Vw2ADzb?>>zxr)A0C@qlQcF z%_ni&V#ks7gFoKyhUbxq_GzLbKV?^S35?O0@D?swmCUx*^$$(GbkLTznE<)gObr!r zg4=a`P!b_lwY0v8(DRNH1p_d=CDh88EjVuZkT6%b>zE?|(g1L|SZGUvHCnNS>1Spk zEkEt@s9A|br=Vz3)~ih6GCxa@7&X&XUr0t+R6dcnw*J`LK7vty|Cjh=LIg2t92mJi6g+CV3+#bK=(YEYW(xzf zK?MaBT=KZw~`_;oU6pLm8``4 z;p_HuyNzFee^6(1=)dBMmhIxiSFk#s=JxeMrfF6hH%T1n=T4Iv^bW)Wt-qS#Qp)wa_t+uiXe-Zk`WeN~ zH;1-Td0qHN6$KdyDm^IU+LY(>?$?yyLnQU~OH#ohXN70>f+M=PLn`kDXJ2pH{Y{-iYRvM-{%E(0|uuOcz0{nAc!Wi}#$4D~&+VMsT9{ZKqg44A7di5EA+FmU1qE-fy0dylok3!u{es zq)d}12_DmQK?!gZ{F9oYZ!LPvae1su2sm`~hQ1nrxU#mph5>|+3a}mRheL6A%%s6W zA|lJT^DhWP)I53H)L4#NSix660enqLrRVpdBGS(89gmyw%FS6<2(CF~)&+O7?t6IN z$i$fdyOZ9J6+FOp#m*hdbh_lVaZ> z$Bl0a_1 zWMm6v#J}*CkU(1z*zIED{JiZkc%O5+zlxF}O-O{4FQE zv>=KG5ZO#d5-UC&8n^<#2w6=nE#c9xN@_NhewSBy5Bb^=49&5>{$Hg>c-EL@%Phl^ z9vl#lP5o(kbTIOx;p;{B2HEyBuPjGwnR0(XtJ!s-K6%k~=^q&r=w1z~L=@B7BJRk( zd(ye|9&)qbbdKOKAWTl_qvEwJEBCqcw1PpzxKvjHipnOgp5AGXT{Z1p0#MxK{zkrg z>&?h_M`!+<&ECyhj^&m4AZ{bPjm#I6?jP5c^NnzsMHC_O{T$fg_Bzd$5QSe$%F)hktNkCo9we1_Jz)+31lGJ8LSrgqX z9nRsWM^X;(?#BCnYp5@rOli6CwRqe!2o(gADuk+djrQ;Q6p9T^nrfXZW%&hlF= z>d6m0Q-us(miv)j(Vg7+RGq(KHS(z(ePS!j&25YHOcfAU6BqWZ5H|iS2;6q0Xbq80 z9%ObNeBORLS@zD;EV`XHtEP>%e@vwk4=+t?*j~j4{bbvlS>-)NryF@{sg5fS2V|CED0x27&Jx>X#0i>(f)aHdgmDn;MPw_Sjbd+u8mV!91Bc!FpOch+T2>}43 z-6QnboC5%%M`}jWX0K8X%i(^XkpG_+fC`jN1RtJ@3SlFAZ?y=-GNOQDy#?FATzp7% zuY@F}Hntlj%A<_WQd-*6Iv9|9kv9~tw9kvqAbB#u<_JnEUr1Nuygl12F6H=$|67k| z4q(u&N49RASx-+ojvOSvwf9-%sGC0tl6~pmB^7T6xzl+sl;%DDNzfIHOs4Pp3fVjs z^*CT{ky;H@{Q_#cNWG_4pmoVHS2;Kj1UX)T9_5aR}rG0~E zH;%ppRoi5t!(sVC)BpPXnfunkeqRumSFCbxK)Uj8%c{vs>BBA%2YNiA;tD+kHT$BO zNm1VJ*~Zpqs>ko)wblf4i$4p}Wrq1oU7sroO8Oi%NA+{BJjvZ+sX|y^Tvn+{;bk%J zDM)(8KoubslGg=cGAvbhm=Yyliwcc&7KYLV)NH8A7}*2btT@$Ufq-7J_vey>Y@H_X zE8se}gAwOY{Uqg+HLvW&4-#k<0Txfbem20osD?S|J%>+}iDA}Sr3QfI2S#IwLf>5@ z`CpLF9Dpw>dZj6E4CqU_mht&l3@nmQtKX3bY(VUe#=xHUi%B@x$$8Y*cxD^9r z$gf&BLBcB}b382kFK>O$WPf7P2uhRMI?%uDzCC)$ z#Z8qMuQd)y1aNF#%BK}g%j~w=I|pP-|3PGOfM7M2^fFt11&Dq!v0zQdyUJ0}OW7j| z5`U{_f_mhj{LmD1k|irDNDa3uoZF|RKHCDV+ISw|*u6i+kalAJE@KxcVLZ=k0 zim3Eh>Ui-AvNUxsRf{8!x{@oV_ycq;F#Fp%R@6BqN1 z)sKC8y&g1Zyj0_#LB^g~1`TbdNgX#?^3dcZt#t`o$MWdw=TN7|XZgh&c~&i3$tU)I z+~?9sdgpamtfnQuez+9>6E15IRlq6ZBQZQVym@Z|Rer1%{e$ld&fd;_;DElrwaW22 zMMn~sEwZ(Kw_eXn?Sh#l{fn!oS7Y-jbLmPt+cfcyZY^8=(ERGZ`AzHK5zzMBhn}7_r|stWyGuh25*GX<~~Eh9;lvmtDc&4U7LZd5iCY zWYoEPW`)|`Bs}@_VS*kHl~I}xxs3k6Aai?w=53Z(SHuO1!FZm+6Dr_*A z%hHJs3x-@LrF=zDws4AJVJr*P#`g)zjuBAeNJXi9GtXcgPuv4s8mUzrCtC8(&bMCi z+53`J{k7cs`oSn~`jVKIq9|PZxW1vX!`hVOyRYjK+?Q{$kiAaimbMRk74$OnX6Ucy{yW{10zCZi|0!D7x=1;?Z zoYqUI>v%tbW89n8_I0+vx| z+~R;>%%0D;!bBJF?%v7zVGMA34Y;#@ufE7mKMs{>v5dyDW600Gc^!HJ(uoY`u)=w~ z+0-SZB4GYE=b|p5?(~(JH&k9@P({!WFrnE@R(|J&bgWFt=0hmIRx+$JpVve^KxFcWap`YTt6 zE1zfNR#{hNtJ=ULeZ5{fc=*Ybi35h)1AFyIFn8xum+@>QdFJu=Kjr!^;Riw@wSH>Y zGWQYtf-L)}DAWJVK)Xx8w(*=?VZU!SUjlLGx9Yy*&F*5ijZggDzPnWaD^YR3*{;Ek zGRv-&TOODVlql*mR!4LCuisa9Wt3Xi9-tc;$STYs#Rd$Z%23=_v=Ss74%U#0$_PL2 zyeu#B_-p(s)y$qcL6F^Dc)WQ2XqLZTNvHAc0VXZet9U077kPo%e-s)kqV*eYnSo>fL8Z+m-JlH7lUl~ zyt2L5o5i$|>TQZP(0y{s@2(|FB#e#N?sDJeU0GS`wI$`w)c7i*iff;Tdd^(MHJu~& z(ba*%0~Z^pHt=N=)*Q6r*UBgZraT3?w4^#@wcFH-wa?WG zr1{SuT2~9;-OtwU3{U@i2BZa;q?Ke&qHe0QvpvyNXrxUCT=_p;{dZgw-S-6wPZA&? zAP6X-_onpTi6BMkDgx5G2uM*hK!5;Bkx->7O+m5JyOdx;2a%#QrFW%;-tR=8@9(|$ z{Z~FFlbJK;?7i07YXLCrTlUU_q^~mdcD5q?Opu25HaF~!t9-QiTouNV1Iz8y*!re5 z_Tsjxm?gu8IXM_L&m48Q*3^`Ww+qPK+by1G{7`;7dg0>kmte>u&`FxB$CLK3DH267 z;0?^z1d7nh6ur4<_uq2+I0USP<0OS7ji`Rh7%h>}^H4iV!c44Xa{3{eiJSX4{eG6A z6nt#@9EtOUiQ~>_k$B*SrN_4GupAOB(|^QRMb`z9Ao;Q@?BT4JEjn~}xes1z)NNhc zRoneJfE;I=*lXS1^^gk8_@v$4qGubR4^{&t82>J-eRP=t!z9Bu`j^uy?c08w3m;UF zE$6mzu1;m{=6h0vdWlB2FOo392@nyB`cT{$NV1G7<&-3Ew@@b9A?p=YrB4i!;jRyy z)?1hDsNB^sG53JWVIjF7O6Un(Z1cJwmfigf1X8V4q-%&&m}rkKUD9w!R0N$mY8d2(Ug%DdeuhsoIkPM`V`56Z zM9@eV#i`i#TV1ugDA3nobZ2PG|6}Cl(TiFab^W{1U%}a7uF9UT^4sDMMW_;P8@>8} zZ-1(mLWI3t23k{Y@B+*(? zD_{YMB1Q73{&Ar5D8Bkkqik1aP)UnT0z1c>+q$b8%e9Atv*tNqNnEJ|IEu~+T16#P zRC|ttUR1L9qc;i-@4iDywm2zrC~@zH26WiUG#O@sUs8`t2G-1_S&d5fm**b;d1H?8 zE(I55xu&w7T>jtJP_&u?5gMt=RFdqJo2L3q`8}*1x$JO+(3y-Vob8tBWxvqaR?a7eGEG?(G}rl70xDWM(o|sB!EL6Hf>|nz7=* zmW8gR8HaGsgHuLJ>!f8}17>c#7@7k;C%m6s%^{O6sydcjHKm*V$#)oiwDtou_E_I} z1glLaibtLyi0W9SR{!620Z8%?^mb!n)@G`6{SAARo+t)S?PV=xf|Lw5S@p_j$+6axTU6mVF*bne93x zsO{Xbl{Q4Sur;eL`u}%DZ{9b2!Z{wl_4@kL<27&Zi#GRrEKU;2 zL&IeP6Ggq+-J#E~8L|zMD}E|FBtOKvK*_xk*}YVltBwYJ2=8*YB_{;S=(76~)Q~a&4XcFcXKjG8F6@$lgBo3>JyE6tke&~;ZJ+; z#rt&hzcz0|qIN_elI_7(qM%msW9!Ax|8_zHLzpU;WwQO&G&#f>t=gl0{!kQsnc2Ri zc1Y(Zcz=TSz;7N?6(k-Ij!8ZxQw7hA?%Fa>1YHK&K*xq*`QOrY$OP{%>x05@G$mGg z9Eb^)U8e9u!{-BfgKR$>9&Pq=eN|e>sw$Yh)P$|3o7tEgMzQCs%eja~vHxl}yK#fZWL z#L&$v=D6*3p4UT=BuR(ND@^pM#Q{mPg9eRCyjp+Qvf{2}qhL}PN+;?;$$BJX0=7Xm z?Su8EB#OFo-Yoa~>7uU2=rS$n9#Qfog!+A`M*rx13U& z{#*WU-asgf0Z~6A#1IDJwDgJj#odN4zc1A$1pS<$z&vJtyui;%Z_Cle5}y`l%((UoWfkZbJQ)VBGW_y;UcNLrHoaA9yZXzA#Wppl zBcRQ70=xUFy2nf_ETiYUb?T-6L|ZOHz&Zw?oP-G5Q<4LC#s-OITl7W?9;BMTHO`bR z@AJJv2?WxgW77ou)H&cS&luJkf9{j~2?CIppY-t?M>yJaXS z;Qq9iX`9CBrbczo7qHP1Whg02ag5PtBa)A~m`|o$B>P#xVY1bEP@c2>@eNG*9Lhl+ zK6ySmv!C|U-T$3x8?2PkRNN53=_IO>^`lfb*@pB;d%FBgnDdC~lCP$z?`7FQ1bwm?Ky;A*m|m@pPX% z^M_fX;+iq82U~r*3fUgY4Slrsb$@qxf$YNP|2WoL;S^<$^TlX5G0f^k!&b7te$8P*-%&9w6+c!oV{3FX~iuF<@i@_V%+aB-*_Wjx1E8Wp^!FwEASD5{z3%pO1n0qpt$u=z zc+5;e39cE(lf{ISw*gi<6AWjX*f;LS({YaLF^I+j*J!-4HOJDyN-nTzfakP$Hcnt) zcP{mj$7>+HuL@S$PoP%4owklqtb4%(E}Rs>*nhyU#S>QhL>3VW6`dC^U2$o!cF_Vq zeLdV$g;=6xkZzZ@$GoZ}CfU^SMV+VT$~9@46i5lnc4@u#VTdn)>>G2lF5RB)*2~WJ|&5 zO|TE*+0v|YE+NTXGQ)Ya1C}3&_ttRhgmClQ!dF(PiU%|6r+&5Vy4?Dr)X*PW^`_x~ z+{f)Sc8f%fHjFn;gdA3e9=_4yWMx`sNUXKU_CzsXu&Vpty~Dj91hn`Gs1}T=4$Xk< zc}C!Tt>YaIt>b{{3(8R5sZ2kmH#EiJ2Lz#>(jndH8?GK2cdaH|?a6Sz!^TKqe^aFi z!Ja`8U6)p0F7gKY8(0%Pmo2)kzUVi3{omukg|jZk8Dagg*K@15BLuFQRMT4mMFbZr zKERFrT*_74!2X>eysjDwJ}aQEyL9PEe;Rvv{hQzi+}7ZkzS7lptPn7JLq|{DH}*C0 z|FeUS`h^v)BE*cxHL;fQ1J_dzAhca9Qppf5=i49Amg?7(yjmvYPOc`vdG`ke8VZ3A z>i0a23R9G1Y&mVO9QhI#lZa z{X0(kfczeNsu9U9VRB)`x||Cxp=V8+Go-ODsIjr>G6T_EEe4Z-J!@}}F> zp?Gq*_HFd(==QC<_#PKO7?MiF7NQ}N)0Nulv(}UvIgyE~#Mt`;>O4le46>|Ai30Hj zVc-1yG zvoC>++w|3|nHQ8==`t57xay+ee@KNS$%jfllTL9Gvm^K-&@|pbBQSf!~`XU%PVHf%7ct5DF zMkxC6?eZNo4}@OrpuRHAs8-eObo7?*;I+&l)^pzG^TU;OuoEfqS~Rm;-9U&kr_Sxp zlkGKqdtfSh@&5nXd3jn%Tb>0qMPH75mNiWM2BLsHKMJ9^?}%Y!<_@#N=`E+u({UVY z$u%y|Bw2z0#X6|>4FoN{_oDu)X!4-Qn3uHPXh#bU8=2WXi{xZ3Zpc}Zo72wIl2AMK zbsWqp7{CU0aM<%mM5v7#f&RvE#HaDRGL7#hY99g@g#%l(YG?Jwmgs$avEs9k`3enx zIalhVb}cv5#r##$=nS4@tK|TOYqFD~nzu22kP9z3)Pxr9&Bcs(Zksj#eJmw`a`X}H zfOH^KB6@Ee)k@jxM$hsMF@w0>Dp~)UYGZ!=%Osyfg!_Ip#J`8yk*<{#@Jpx5;eYe6 z?M?t04!YesT3MH3*v{-4#=K3eNu0|Dn}qeG1hn(tcCRlKjB0&{dZy2|Zu)~uOJ^s| zfhzP>)XmGb{I)NG7<6@Z^{nRn4IdF*mlATUW6fRJ&t%}4qw&8%>)wa3L{eU>6?A)TAvlxByydCj!@EydrEEGdLFnuov6I-rg*v= zdRpxz-ncg%8>MkFqrqtl!Z_6u1SlV|z~GYN?m*Ws*o*c{Hkl~1dfs}lj+K|_C9yhb zX<*1!Tg8I_7{&V2pYJ^M6CN<|K0DA$lyW(^-4tT&ZtKeXjaMAGlS%;2QJP5k5jy81VNi?8)d;7qb&KD zUa0^-Zw4+KU2llAb8k+@qLY;43|GMVj&6cH@g68A_EK6UWwr?xJ+o~z#Lucf%Kl&7 z;%;vQLz2S;oh2wEdKHNnLbS9GQS{1HYHpuT23RjUBi0iLPbXWr}3! zF*a|fI;;Ja+u?3X+~%$09x2p8@)h97?gx1^k%c*W`H`F(ft`4G{< zclsGjN#-j37tr}K?rQE%%V$fH%oEca+cW%#uBUV-ME7zOMsWW?S*JHlLG_O=(RDaZ zVO$ZtsKv$laKt6Osw4em{r4Q#T&chixntFpAhW8!oBT&dT7u;KVh>YFsIaSETW}rS zVp49YO*BADrg*$EapDD&3f)u)mH7j9p*=9Ca*4c`tAiR`8Wbw}*JsU(kFbjSPPn-k z(`x2O-}xknBPt9PlCxDHBu9TmP$&4Oyu@Iuy1IRQ{a!M;wz{snuc`k$<6!+snW<|rZyhlEcIdgDa8=ne&n5ooY+lhoqWu+`Vg;8+_xZV%?%P$mvB5CChd}HLA}KOE=mRj(w8T zrO$dZT*0YZ5_WERWzOpDswtkeWPoj~Pt}=(JhM{P5IZliC6>bB(aaEVP*qZXs}2r%O{@up)KGUc)^5;k*YwPp%>4(IfP@m*9B@0KSk&wx z(GE1^LjWE`w3eobT@v1rM3RWzs>(zzvH2S>k|!@;+hDR#p|f5yE%B_z(fUsIbCTmZ zo@4=9CBeUcvjYPIMyIlTd;NyVuH3$U!TGCNOt+=$^vk3oHT662o1=`IZlODFpy!T; z$R)ajxFh|`b1{}@KKk&}aP(97KW^*B0H!dw zwI4pi9!4=c<8B9zx-RyLUk}D0w?QYwe~s?DyO@`5vMeZW$lHxM^H5&Y&WKyJ zhRE#7FiG@&s=<)|T!8Tcz%Rs{6R zKFa;=4Hx`0IBovlCLcQlIJGTGJ*0UjC99TT0fECJ(GL(U7AiD`qNQR~>wR72(cP>k?+;ckH4tbg3}yNn9#vspN$j;pPd zbS!K-1k5I;Be~0u1_w4%+e!|1>vL_?)P#;+1T^)@9V@ESo(TpF{U6WenFb<6s{4m` zM!%cesi9Xq?AXA`J3Com6SR6M788OfMLyWu8g3a^4woE ze2HI7q>GT3Pb=2gaY+;=TVQf=oJ0M_Y(OcO4o8|9hGARz5mUj4jALlGOS;XEv{Xz9 z_~aVJw4WrP%dfd}sVyXkRVF<$$cx(S)o2Q1^GeJWulO`X4lu2KFBrS4^bXlwPPbYl*`T-2rtl=()-phw5r$=qJr_xgx?~1YzFDq%d+}3|p&AmIpSI^6!7o{8gM6O~(Y>({ZM;YfKc*9L3ScPqQqxS@9e zE@*+wHW7Vo;vE?}mS3W95Dz^oX3p`iX&L~W?o2O)?EZ)nLQ`ZmBfqjh_a7rvbe@=E zrP9+&4LFCou8v~byGq(JV!8$t_6fSJgsy~sS1%{-1|6?=;cz^ggeoxV!RNI%`Gmct z&v;M3R;|yfOLZzYr>Ty7n@5_*cPDw6&*^@d!bdbcI2;sY9rSsb zQ=z$&f?r9DYXrG@{XH0A@R4JhNpnlx-;H0iAk1KkCV3O_5_Qh+lPaDb#$=!9lBzzT z=`UucTH78`5ylKnqScG-o^BU0#+H^5snZ1hGRW)mX3Kv?+wOKDkhge!7IeR>`_wO^ zklg{cef5y*3u>pfoJV|4tG_`Ue{B2~wDqX)St#xVgjAJ5;2nY|sgF*qr2$QqN-Da2 znC5!i3H#id$N_z!r?Z}Fj#Aq|fF>rkZ2Hv=;^(Ne-9N^}C$nyM0ab9qpyu9tJ~|1l*;Mlc$hGn5WlP9_CX?5cF_6jzNmu`dcAL~7gOUMdubZKi}c zGO@@KdA;}^o$@Fz1#`}!M9^-tO{a%VV>gZma&{`Fz<`@OcFo&LLB+Q(;jLV#A~;>M z0qjp>B%$`fSj?1u+FBnyZDA%=@u$+uQCl?u_4Fi3uY`ycwO_ z@{i*iRc5|>4n?n~yw>G5)rjF-)DJaDPppp@-cO;WPfk0U&VmWz&hLNs1xY>{u!Y0HZ^w1PeNDk7QkG zwc{tI7HnqowNRVf@y+Y~_ar*YqLNi|sf}Jz8A}V=mAxhPH!J6FwPK^fA-X#^hJD`A zzy`e=+2yW(!ZDmCUbaF=ORIs+V2?0S9@X;WcONd}^m4y#t5wKEh!Qx7IC}lbP+{=) zcR|yo_A<-Sc7QKWk9~YV_PJq(;n8|t;`d`7jRt2+u>-%RUmjg@{cGT0;e1>vS0|VL z?OQ?MQKB(?#FjUWu%Bj6`1xp8Gh6szsdy>(?DwltOWwrkB~B>zq`S{N3UbL-BTN)i zUROaSG7~aar)@L^kA9~nb;AU0_74lc!`;fifbB6ew~_l(zxmZ`ppeh0d5@6SHDf)} zc;x4iLZha*!QU%9+T{mw^=Ca68ZA6VoO{)FJ&pJ4$yLvS2>9uD?Boq_$ad$Iyb;Ej&a0j=1#C4KqLB^jCBaAgl*fWYN)^J4r6(D^+7f0eSV=a~?Hz2mR@z zwmdIF#kip1dyxv3>#h1W0CmXptK# z;mdm0DT@U;`ItQogU}r(_b#_shTcZoJubv0x1^fej--LY!@+vhbt%$Z>F3cxt+bGu$HI70nv9ZZ?>TUVe1=Jdg* z`*#RDpj}*Jzl`0r<7)7-i3aal#T}z!aI?7GSsX|L2flxRv|=5c=!uW@*)TD*IgMh| zFW6Nt(8QlkZloonTv(VR-zivhed z9S9lfZc<8yyTfcBOx;#>>hPeq$LoEFD0<^A+tw$wUvb$d0Yc?h7+f?5PNC+V6KdKc z+qHbLL_1uP9$if~*+=*Ar*GR7l`%^5a2z|7lU=AmP|=Cn`^LAH@%kDLftwA3oFP&A zQ+rBuCgT+MB~bS5QGUkl$@0<0pR*f%-o(PhB^9~|>LY~Jzr ztq}En=BHe9#%GnGe?-Mb(H^~hB)Wh8Q+X5xX3#kC@Urc}13LzI0V%wF74>=gCX?VR%p15#q4;0ekad1gZ&aj{TS)b_| zqOi7%aCT~3vD3KV4a?Ce-PH{W&F*#Wund+)C4t+dqn#*=vj;9=DPfg8@?*02sdp@T zBcSkM)UR3m^Phg}6J(e~-EG(DW|PQsP%;EZ%1mp;Ygw z9K45a@mj6kkWRD%&U8FK1Ra*n=M5z@8lUyc1=x4n8kVk(_#WJ}bG^N={jk46)^3}m zO8#&iRxgT$&k(ex5E0GPuyzvkA$As5YB0GpV#S{Zp~n-3#ao97H6pnOS^4TK%^C2` zxB0zn_?#oli%+{%#YQRUEySS@vaA?3=gbcs>usFeBQ^-Sbwo>6{o^iwgTwaeEYbJ} z`Z{5pP`@<&qCuD^AkuffMEcqT&UHj>OOln6FI0V+BT4b7!n}Tx{cV!`Z`q;wQAU~H z3h!n~WcRH!is@{7NyL3HITtWMFt-xH1ivdV#l}xx@~3FIybC0+g`;;gg*A#w9kLS3 zzOqz5zN5HNbXd{oNyF_VL5K0{(`K%8;ddXsEIy$!WkyU(jcXtoHEfP1DoDdFk9y}u zDpz>biX7xUl1?=I%v-1!5Ev|4h*i78qq|d^7-ldl_~AtMqEqO^>}S;-q+%zZ%?TpE1#vR? z!s&V9Z}yfpZXI#o;6Nm^MAEUi>a}_?Al5vjvYAcNmb}WjK&{gQo*z>oP<%&a?e|@l zw5h0*c7bmmp%U#Ew^nQ^lH}jM8loC`g(v8aw-@=0GGuqYI2wJiSs}JG#iJe|Pwz@0 zafZ-6X(*CJx~|j@5Qbu~IffdTxk^8$xxd`&d=E`rYWlRXpN3>cd%pZBv-GtiDhr-iv$|Hhb7i+bachg*#S8e+B+3{+R?InDan!U! z05xMF4d!vA>%I5`fPubNEG=Wo=GMO2FsPIwHdVD~&dCCiLXf&bLuVT=r5S7L(fJl; zM_~|c>c$>`LEPWwxB2CqWXvu(51QK&+B>@#GCSsP+p4B&7n$Z3Yl{vT+cjP6$f)W+ z2e@1IvN?{wy?u|h!FZWjXp1{yHK=w#d_$Aif8S2t<%_~x+UM*SH@N7_{r6(!3;oUB z{O#qs^C%Ek!yFtWe|)kUf|mGA#|X*eke*A?&AvHK)9laM_W6akC+F(v)I!U)2D^T= zsq~I>VA>m5@-WpgQgf*r+ZLQJX6KqaJa7!5F(_YSjnbAi}ee+*={w4ECW0 z=R3i5>7%U`4st{=t7)K`wyf9E+K_OIie9vIkfuX zjoH(5jD2sXMxL`3$Kj#{&U6`KXz20xXOy{!Gb>5V7|6XuzqjxU4VUl5Fz)$kKwSr) z48jP*c{I@iA<*yVW2x@uA-l;^EkFKytKNXSYOA8m=*azMm2%K{Le@m4&fDRJs+*FH z6@)C0oE{kdlj%z9k@8v5qx*B8{Mbkz z9eMEf!#|d#fEBPt=;~zcHYdG=TQgANGFtP*-y!w8t*t6A>EC%j6Gv6j_fQ$)8kmQa zc$;ve)t_rf+G~mq6yjbPHIaS6Doii)V)u0c3HG{GLqVMFwb-@SR>gD9G*y0Re&b9= zxWf!U2k&sL^WGagOpYTEcL2x!$tTv!XD~boi5|O8(DJ-OMwzch`jBB6Bel!|S4ro{ zHCY@&CXMDZ{Y*y~xjPhz069paRP_2|*uODMCaZQBOX!me@=l-+1QWJi zjSP@J;>iT?;=XF!au|EHLu5= zQ3}GkLPlR<8aQ#wT`fAvjuUsUpDAbW16+v?B3W6gFq+Ze^#8dWoJnimZWkZ6#ynn& z#@*Y1u}O}zGBf>ZP~V-;6&!b1UhhxqY#I=svqPk(*pQV!%L-1E?^(mh9p#I1ul>`{ zFn$NLIGy69@>y_>&x$M1I})#cL_d}p1q=APw9%)i&?uj@9v6lJl}Ah|8#b&95dNurn`NWmW!-qLAi(<@JB|Vahj(>;hiyU zrzGs#g$^{byrpBgG@ByCjcl0uBu>n$&{hhVU31~_u)EOy#XN*Ro4pg_v^VmBmy3m< z!t2|q#Ooejy!QSED3p`zyX<^Zr;gh@e2=tvE_eNJ@tCk8tY8pv8d)c3$Y}_dP8`}e$LKSAwG4gPm@0rRKvFxjYddc9^mM@mXmVjFZgEf zNxP0)zs)^EAi^qrHSU?=t9cx`4I*`J_N z6TdU&Jwd#PXM1IPL{bS<+_#%DD}uA4TJ~ZS9?gP^RR5W^(ByX3Z+RhQ_dsgrQxU)r z(66%{o!)C65=OJI_pF1qWrIKM2MSnrVOYY3GTOJ*gfK{Otph zV6^5o{$?^Ky+Pxfk?Eij5T1mW(HxPwI8fh&nacpf%$hiJHnSFFCw)k!PMEx}GYJ|Y z!@g}CQTM+pxz+qG`utp#E@&7ZNebYO2z`AvLkWymg9G!E5p?$s!D-VO?5(qQ`nO1$ zaVs^FUV89E=>T*FaQ7Ms-u)Hz%t+b&xXFr<(_+BLo(u_b?BBUmRO7e@I$Vs^>&;Sp z`)fi(YF0xPsZcRp46K};QHxB8=hA~^g9C}*eGf>x-f@r#!#}9M=La%P1*My12jU%g zfplwWGzesSj>AGPimZT_=(4FtqDh|9_*iYsK#r%SAwMkvci z6DHEe8RQAm#WB$PEo48cLNnW<96Tf>eMxhhKIA?dM*!V)&dF9`br~>NyO5&-lFm&C zQU<7k2@}!*|4HCAhW<1@81!|bB{@$}_^*M6QnbUQKw?iXyc4KM=CB3JAvv3kQNr)p zx2vZU;|-rJQpp?ujgr%_Xu6w{K36uvcqzj(VkEA}(&G(j3VSeJvvOI>>mq}0<5NbP z!5DJK3*Q?A>QzlRh3P~a;bayN3vLQY&*OGXeaLa0mr%BPa2*JLT)U# zm z+q-?!!_ja>}{Gtg7=R0?+=_Wc}d~7XfnnM?Z%{f3>C>`;}DH} zy74r-tG_&U%3Ihcb+Q?1ee2zRXZ`6#eV(w znE4Z*b>lr+jwm*dXk#sLX3O31V80KmPn*j7k;6C$@Y^8Aa| zn=h-LY^AAZ^CE4J=M%LOKAt^6znlfi-F^?Y38Jg2${H~O(n|uDo-{nekZWXH(&fC` zK;^B*u=IWw*6)PPg%NR~I~aflMTTyFNLdrEV8U%e-8}{4B@1=4^PHgE+1rAhfy#&Y z`T6|FaHNq$W)^a&1?Lq?EEr>-y`y*vFk;9$qDy{Rt1hpX3@D@fu3S)8g-GA^>vo{a z3rEKY!0@h>o63C-KUurAlcLhI7T3FtqzAY^kK$#MBL%ii`#7R|~zlbiFxh9%UHH83@a4U*R{mpO$Qa8vs1K zDa7)e{1yfxNtGd$t2Un;C4p%}C^ifX9t}B*)SHLBeQBKged+C`2RmNTSU(ezJfHLdqmCH9HHbbGx(sn=E7pDJ|Miv_9o>dn*CC=%?JQ6t7 zz(kwlljB1{1@(NPCkC@h4@)kKuol9X4Z0O4^exT6ohAkTh6qSl*~naoXLv-&*g9a9acjp-85q;LvsK_X(O(K+b(?#wrKNapphEIKZASs(ju zp&|7ditF7Q`*Iw}@I#C4QneOAFC?<(f|XV&6}Mx#Zyg{YC;3W#HJ;5ws;#^-PzM9& zsKT;4J~Yi$aVxTI7(KgH$4pUNX;b&$UGOL^Q<7oMLxB3QJzp#4)1SG|uQmcEt1p=s z9Z9K|cfJB2cAjd9sv0+nxB2>dg};-P?l<)SqVsR+URlmX8S=98;z7^+eU`Fms7LPg zwJco9HhdvZ3qQ+##~zmFTbdQSdT*lfSDC1I^I`A_g>CMQ@&pct`5;mh0;$EGlPEUvyb54XB~ z_Nc5&Xkfk+AJ=%eyXKu)fT3YQj3Dd@RRdG_C^jxhZ4C|J?mg;LU-%X>(~EIgr{&a8;sr$oVf;@!aeqQ z4i))7;ITx$nr>|<#G z3KK|l63?b1?ZNPsDEW1s{8{)V2#?=lnS52IR^%ty`+@7y`+nF`?N3Z`17H>3x^LV0 z{_g{lOofxVXwx%$a@-@=bhdF8CVPm){h(8K+9P{7Bh3j!0^XN&CUsEf@-9;#^H5cl z(AhnMz_BVX!?Qk+!9+vm{Uq{|ng_;j2tUp>{4rbPX+zu7*qdVLs+Oru26P@wN-^h# zc#tK2gwgYTLh`sbId&^aPv(>>RqTlL{xjluN4Ile}S; z_V8-}G|b%O!tApi+@*Hv?$K6oz(N4W4H=UGXDt}WGp^nIL#0Ast2m(4_b0rG4-#GJjs6kwYDI@gGO&rh7JWV0cWnr zFE>;F%G9+jliC}8PZv$uUJ}_|2+O3yIg4li@%I@zz|NTQA)jdMbq?uB%N;mAayr;) zLx(uP?e(e_z!w>8%=gBxTHYiTI!<(ySx{bjZrok!ADGM1(D33{8o(>MR~={g^_y)V zv=AYx@q>=1oc9F-$o!-?w)sDY4clY8F|mvCu40+`Mzfi*Ad$*_W-5Urr(DqM{wre* z`)B*_@C!=vvqLpYi~GZj{aP9*yM4Mi=sOogjU`rHbfAzsH%ksO)Fikt-!Rb*LY*a8hSgsqRG*6>@k z#?%I)F+_D`ZN$;DtSLRc_8EE}Z9G&Uy7|S;Yt+)|XfWXgopog4klo7bU9hfr9SV23 zOxn7_RTJ|>B#kJ>$$pcW-n+W~C&G84FuSyoBYjyvR6=29{^HcqWttOCzF4&w-S z*3avY=`h{$DROOvWu|3P8(oN@x@515^ySGDA)gvZl~yhM$a|vp+~q!&IL{W!ysx$Z z)o}phkuae9ZRPL)`H>^<<*kDNs^R&8XUJch1Rj>nth^k9Bz0-t-un|fs`AT217pk9 zaZ+v`o-8b|+aJw*9}2oTw^qF=b32LB>!SL5oAOj8Rpah|OQp%Psww?|+hTx_3aiVJJ@1 z&BDIc+|v|dP$n%kgYNkMZ5;M2LEN~@lMgUV7N&<>O@9o|4nnCZa~aO$nRLA+Ly_e@ zwwp9uZ(%lJPq(j@mIxxzWr~yt6BC?HCuE zYx*{#A$B7&B(57O1s3H3|F%_wIY_hFIYsj8UK&1&kHW;QYrf^c91oCLd1IIQ(t+$c zuA#H-z7%j_;>Sder=BxqbkLeV^qEMk+tED48gp*EebIk+WicVg@<+p3NWsu;3=-F} zFgfo++j_*0|M}C}>M&t;aJU}@uKE7GhKe-s)w}c)2I2l{UE^X-zx_fB&_WuZYTnaz z1}Dm;EetNfkY!>5$YmIPeJ$o_>K)rRk*!j zei!RU%NXM_3OARBe}?MF=f`2#7Ql73m0}D4|!Wp@~Qr0qIRerAhA{0tkd4 zCG;+Y-a-tay?lS?{bTp+*>iT!?Ci{)d!P9{F^>&(=xMlUNJvQNbss)>NFczta_ zMR9$u^7J+#At9%Kt*QA~S5uSkv8S8EYq&iL2_!!Lxup}&a~5c`m!rUKlrdZ`%KN(- zy&!$c3%P=trqKC$dDk>u`5AsJ48Mo^Jb;48HeVDNX*~{--pXR>X|G4{-8o^Lh1N)> z(g@Cs^WA~g5F#F9W4~=@D9dKNZmL|c-wQg<+0PB!YjoTPVjF+HOiN}VWy>rVXO~&= zicWm5Dw-4-ozG5DVDbbcYWYJSAa6wLS8Kkg*%fF#Ju!coS5~hLV&(Gy+g7iCr+uGD zdQkj)(3doTm<(3iGULVDV(X>Vdn5xImhCXzUeJ*E*A8o9D`W40&yC-_PiIOT>LqB? zok{W%&$hfTWwjq}g=|F1AB(MszaLT9 z_-))dRq`{XH@nlfr~aPV(UCHFM@*A>=3$@bnTv7m(Z=7cdzpiQZy%f!Ti!mz2Ycoz z?rsKQk0aB>DbK&Ae~Mu3iFhAcpsj)5Sh=j6d&ES>`G)JSZm7h^A_jBGRZEv|*bzNC zc7<`U9@{oC>}mehgX&mMM1?6u$&v~G2GeK_35&|-#X&70abGs z77Yz&tKvFOC}kV7KP-u|uwzyW5mNxD332k>oQ@c=lF`K{d&rRTzx|+MfKTD6VutM# zryjwztW*>xFIwVcpMJ~Inpdo8&bPUey{m6wj42gksCy?cC~^k0Y&}O>S6Xa}DLkoL z;mm0=L7iHkZlBBGMbF=SSqfbF1mLW^Y@S{6%THd(90}7=+1-CK z6_!@Eea4n2nX3w7umi6bpINj?ydaG^s|4T-E}uVtw(0&!1oXnq0Me$x9Cuq27WYa1 z*}n#p;{yIvqrXpIGZ`xzGykX@*I8vnP0Zzsz5hg^(r&h>beZipjl6_j(@ z7x%po$V)EdY<5vzq3tl9I~cGW zOTh~x2_=r@WnP`NUzOanfRKnEt&UOhW1cm5w{_CHK9(s0DJ$MZ=!afEmg#JLHbdW> z98+4~G*R~Rx|vsjpxW)@>sHq8Xl?YD?BeQ`^>N#; z{mgPy8=QyxDr$y|Gr+31gRcmE!+*Z}v&F{Km=JrGWIbXlE{TbL(RF)nYA(<3~ zqT!s%Y&p!B;^I~8_c`okSBM6O_J~(>+oc(%G@b{aO=oEp%Q*CBxUm;hJ3sH;FCHTLlEjL z#we)p6kp$Nc`{@0J#g`CVK@SOzSI@diIe^n&b{yfNRmXEcYTL)n7y6Q%VlxIbw|!E zaCTPSD0s^_y$n_iyu6d-!_sBk>Z_EJYQg!yWlid_Li6cd#I%YpX{RB7SSmVg~^E$i_-d*T4rA&x)r|U@iZ?rn|doGFBO%<93?$!#sFL86c6OcozAsBc# z-%rFXm#(uoyeG#?SO>2RIW=j6AIOv48Iv+i{pvD#kh33DEabYl#KT#vqgJhjSqiQauZkl#;Hs{iKJ zUoi&0G%)Kt+Vg7F(I!;AwAC=G&HWr9#RwaATGC8s zsp}9v@;QlgmZ0+V&wnj8M7{d4*9h$SVDF{_g<3!8G^w{(wAf2sXmz?*P(f8JAK-|a z5SI6mc&vOEE4s7dB+WtwTVf1ANBpi31hJ_f(io`_rhb-`&&;jz{=3s|lQ-4-d3854 z2zw+D>}e29hy!YJ^Hhl-rH>RpJYfUV7L)(2Obmt#CSk)~12;J6U(rYlb!DFyt6jV{ zU`kMThai}@DZx&$WF>wQ-Go2sZO?vO3V=`hWs+EP4x||ZC*ImHu6)FjSj%zJCdKGM36gj;8_z_yY^NQ!|7L`Q za()pwryFdDScUP_NV?>tS3_5BBs%dDIw;fNJM2yV9#So=OpT!K2t%PLac)jh-nCzl zkmIaHv4Bo`7ehUT5R$?bt1Y$x}H{?i(R+aZE_LYPq<@VuB+OM?t+&8i7j=f%bv zUt-+WSl*?#{f}rK&xZy~u`zAuELQLmaBhN#dp~#3`5b9R(i*=c6!rofyj(zQq#F2+ z;nFgT%Q`j-@x^)u%bS5+=K@0rcQc%4hJd~wGpK*pQHS8G^HMEW5}i^M?4A!@e|YI( zMn8LVWp!OYiq|`g68DuXv<`6>wd@p|doXRu&-}Qr?}VKUhGM&nS6FD+Q-|9x33GIl z?cCby&kPHIAmg&vP>RGIR#T8yu7{8SSEJW&2BaT?sr4z{7&o9974d z9%|02dwDZc8?*qO3}3rLuTzQixdQ2lQ9kdt6lyJ&&W^O6E7Z7%6jFZW*8Z;sq@%-; z8kEp1`;O_G63tGIvb%_Kl3lTf1ZRx!v^>wQQ;J5Gb5u76S-zVq>2y4t_^a4UCy6dy zC7Ug@szDo^|2NZ(q-Czl)M;L6=7W1PRdgO&A*7@Dz{a@7PPC!Z#7L;Y|94#xyLpwY znpQdkk#u7V7}|D_R^Ref26u1rWf8GMhtX>15gzxqduvoc;F6lbeTsgGF+Et{*B?%c zAz&-$bLmXc1gwtQ439TjgSj5Pwlp@q)rqM+lySwTu#0>1Oo!3YfTkj|iOY@_7S4X< z?H5xh7^ovJ&3>lC`G0#f0~VO-{*@K7=vRngM4f zgRcGvLkbAl>i3EZ%R0IK?DzyCwG8gTVD05{8Mu#hX7hoyR%i`qniq62p?G)gm9zK3 zZBfv8ceja}T_Na~!sX1;bl2132W+baNn{}Zy>Ave6h7EHTUg%3e9B<-i(<(@K-Hgt za4jL+Xs`LV?nQk=UplNNulMNk&hNJyIhccq%F4>7uc<%sqoFgf@pMgz})mFGjtMj{CHFX`Js~guo>b>FN7nolfGeXD1kp|qvghOT} z9aojO9J`9=byeCziQiXJ)z%7U8)M_L4HBw$s$nl#Cd!CIz^-E7rV2-`E9G= z`90tIm?efxG7aC@6|+S-`N$}0vUXN z@3XSV{8Q^#ek z&gr^^d8N%_C&(BVpu?q&D~BjPpLxzLXqDUE8oG(vA!YPeUtavz_Rq!YI)ldylHxN@ zGb$I~l$2%my4-B+W>6l7wjC-M!aFA69~3&G=Ly6AWv%N&vLmG%LXtttw9d@~wg=Z4 zZR^3Ff1mOsEnQGV8@96EtEw=T*K)H0+7>htxKO?RUb=SPL9GRj!QkWKq{8W5`^aXr za%zxvy={W`9;0tDBl%QzUmKOQzD~(1Bv19AWr%!?7Lw|QsRQ5CC^qPS@OrAx$Ao~? zrw&C38O?w%usTK@q~c%o@;Zs(A(njLercD3sqW;%Y2lJe=My4Yw1N2N$+FaSnq@Yf z(&Fk&ZsTOFh=)kC|uIRyXiU%Vty~ z*XyrY%M=1UHj0m|w~RZxE+Nkgb$L54jPr=krVZY?Squ!i;yL0aXQ{sN5|TC5)Y&|% z4c3a=d;7U1*E6D~cz>R1I1okFyAzc6UdEbZ3kuq|SyWgsGlzMv)wAMl?QrpyxFSYN zLb8m!{3O+jJ;)I4po&MnuV9f0lsuFOTsb7=f&s(EX808)$he} z9zU@ne`gkjaOOdeA|)yr11S%L>VZQ5>}e)+J-o8+`v<#)M>1*iw{IHQ>5WK8;?-rN zB)1$7lUa0oa8EGGUZ~srkZ`&;C-ni1J_nx`l>-5Hik^}%{JJUjnhem#rcaIJ1bQ2e zNX3zt!ahkO!kKOX*~2tQIszAi|B26p-{_O$qlif1CVv;RtO+)H^pBL{(+$9eNMrfC zK`2`8Jw;Nyg_qbnUuQk`rq_Nf{iKckH8sO^bHCBOHZ#J-%0Zm9B=D8QTElIP9J=YX zI`)maDP}8|wOYT%ziH-djGjreUs5lqkKm%!Qrdt^<~%|V7uiuYJ6};7oMT#?9PkTz zz5ry%mku_zt>`{<7j+nE#MY8SX~NATPqtGOALP_+ z#I&ksNd?vkn}&UjsC+euo>XhVr55}#nKC zv(U-H*Oi$SvxlF0d$wC&LFqoO5uI2NDT6 z=v>s2UBrcAn^04q2XQY{>|V|w$y^QsDIICN9wU-UG0h(v=_>t>LWAKO*+CgLVco<{V5da2UUT^L`b& z@zJDJp9=;x6gkW}ACm$MGV1c*lc5DOzs^OC^wX;^MUn}UMA_@}U@%6qa5YLiM zo{sg}pKL379VGs1&t}QYU;Gz9o^z*hfZ^m7W0R+RL)8cJJET-bL{A%)!?8r`AHu6x ze!Uk&b)dIO|Qd_@53_+O7q`U`YicokmT+3HJLZ{SCXrw{ORL{CVY7m`Y-LkV*xI4fr>gJ^y%HJf0AWa#Ojv&CHFL-ja;s}!aqNx#X3 zI2(Z$R&@uTn;Q6AS#2;I&8i}|wvW*{r|*g+%Nt@}fjwh`QJUN!gpg8@fc*15o&J9L ztubxhCOiYm)aOQgwb@r{o7gac?t)J-k{uc9LZzfLc8RKQp4UhB@Ws(9Q0TCK4%%uu zC+9SMAM#21EgfCN;7zo&F63hyT^PexP8ye}Wg;MB6T#NzRJ8n1{v=tos!;qpT50Ij$9aD!^9zgm%fCJFZBMErD3{Z=%5kG3TW8(5<>u}3W>-Nr z9p1MvLD9VhppD-xcpp@6w$VNzBgI=lS+SpbMc7*wN=Ty5?<|g==!4=bxAx~i#k;q6 z*Y5-lEV1GS64NW)AIKh4t3eB)A>|UQM*Bm33!NDqJQ5KiTe3I1QWc3F8`&I4WLwp}IjhrbHh9HqSMSQS@{-4ctg|rz5QU$B+<%jYMP9I0-KPij*R~h*M!!r={ z3L2T&PD&J0lsVXa9>~g%oU1=E=Salxa<_&kuK!yCeSA*Hd0t4+9oo3dgMQKFo6iR5 zMnF<%q>)%s)$>^$|1IYsLzTmxsvHQPIh*%k)kz=aso)YL8+%iO^Kyi@lq*b~5#8B< zzST4e$}d2@Jf#2S)%<(~6`A0jba*TrHl236iMX28&>?}$5g!*#*2-Gp;s~W?BDO}z z-jdCyGVRqjLv^TU3q~NN4J1UdE4{J8yf-~y5n+w>WOT5&rypy~>tZ8}6~A?SVH5+|p9 zlfQRP24g;V3mSDJ81XJ9V@XHhGMz8BPQjF%ggHJK%MoZkMJSI#TMoYDh7Z^>aFBW1 zk(rk2>w$6e&#Ba5PsEZEs?{=tU3yC3%)FC^e>VenPKWMS4+F;%K}<;8ugcZBUUR>yY+gd~UC~T8p6Io|1(y z_XMWUF*d=dsYKd3u34*pGV_p+%tB?(^r!@`_`p?#UoT;+Fjh(BKv*sBm!LM|Y!bnt z?d5~snh-y>G3B=SUj8qX8&+$Uj9NqMP7{)(L1CbfW$ zr{uG_i#8dcDQ*T)GN|tPn=0*jl$R8Vk_>a$y!sTz4*n?L-V^IK0r+Guwo8FjY*bf} zde$ogT{34K<+qFbsiyfmpXBY*Pe*R#J0pjLVF{=2CgVk46GDho`8y9yQq*$>B>04w zWg4ex>Cah-RlwP#6T)g@MVWxNqF`}nVt2!Q?QV}NFFYyUrQd)}C8x*;jHzs=qf_Oq znwZ1KIA|c z;~Zr{oItxBjg?b}24lWs)hgo%$7g~QP3GErII)Qg=1luTnNt_gcK*lqLwUUt;a$wm zEf9AHr!$#i4t78d9p=Up$EbK)VGT{0TPej}7$I<7S#xIlX0*C_Q7qcNtCyA+-lQ~Y zm+;=)RI`4jx&+(#<@NMBmi zuk6v?`WRO!G>j!&=DiX!9$ik)wo%>Zeb7nCE=loA%hr=7Je;BzEhCk%Mdq|UMoA80 z{KKIqzoHb8YGWi3@qpr5gyOXBbE%)N!~4wN!a93?lvJutANrlk+f40RWtOm>%BOUj z`xA91J5^2h#_f@}D6hLogCBuTofhB*|H{?tUFJ_SI)e29RXoZJo)&w zgRD_`73+WSLm&ZC$V)}L=HIIW3X`Uz$`w*IB_9t0ai~@uO9zb6e-2vf#boL3sQb+- zZN=qDzoN{@t2}WPH|H|a;9`VELM;tE!>qz>B_6&ML)M*6WtHRJ(>eW_n4)&rV;K_n z9Ys12%;gS#b*zra?>MR)PUMNRs5A+lRHkhAcO(!EoMmRAF^t?*(M&$ZmYD z)*w@QfO1j}G$Y7cs*^5v z!p!>JBqcUwKr(*-P3}h^cAoqtt?$E04`5f=Mxfcv?3 z!dK?7pNW@1;`b7p()l2;SJdiBY}-yhz-_sfW7OE1QBmBc%2=+&uz!(2+lGB`X^RfQ z3X^0$z`ZhRAtw)sZkLuq;l>ufccm;6PHymf%(?n=L(39SeBUXR|BKzHEu$<)Vrnq#`3P*P1~n@!$+cZ{%5>tWMV#{x_DlPHsrjD@Lu2 zI8jl~(Om)ie%?W7JxZ;}et3jMU175$<5W+%mCgqXwS#KqLu)UVVXw6RO zdo#!SGQ0mn+I?Onao311Vzc~YQ#bPI(Z&Gy187PN;}#QB6uiph=CWd0PGR~T}m z74pHsLxQ*i8%aN|5S_l}p*p-+z{i`6XR3EtYNowBl(nm#bWZTB+pQ-(Fv4#o=q=%A zZzI{jZ5mKNPPoALZA}LGQMR`eDSh{Xaf$QTSNgXf0=%r05VS`6RZB-E6_6$}w289n zGJ5fbxDWhG0v(%;(r7P$Z!{I2Jn;F10rlP%A-z&A%_KF>j5wmZt5;=aeh4p3Vi0+%_By-yUQ1mv*q2o1h+hl9{@9sF|7c+p>DVLP0 zdW6akBUU*cs#25?W`HeQFMu=(T*&|!W>$15Ji&o&6jx63^a3Ghw@^!T@>bPrGcJY2Q= z*pt>B%Wt`ScG=UBAqV}1;7rBhl(y0C+K}_7$p^|j=&-10nZb1fSa)em*Wix9#hpr;(eq|cukV+Bd+3!psZ4=rA?3p}r->p=P zpIWMu^Q-cNPrdb;XIxJ6#aONqL|t{(cSI9IXsk|yiHwf47>_`MX7A-pF(f$qNg+*N z8STovx$M_Ds6oUU*tyL01ET$412=}rx5o=g63*8ibaU8;X)XrIEQDXn>8JsIZthBr z^Pius@X5Y7Ql0_&tMx&rk3ZIl_QBs&ZZk=JocJK?e!q)3Zj{_|qj9&31KP`xt^Oy} z(BVqE4rwD)(ec3jktkgcS|}+R+$jD?vUaj9I5?UrNoDQTbK{ z`&2T6;_Q$Z4kwcu5rlQ62yqMNsM*;!8CyD5t3MvuopSZMSZx}M2q8wb1;~&eBwf%p zNCt1}IZLL@?PyRi&*w_d($FC;!^O;!O;uaP^y>;>?Q2Yk!=*svSmHWuoAW&hZfD$q7HEg^bWuZyS@K8-|%sLqNNA0ql z;ffH^xXnX<(d;{6v7Z55bx=yOX2mXqgz4VH?GAU=Vzc39IqO(R)f}kG)|kpV!87E_V~Q3w;-+R z=!dqg{@%U=dIt0SUQA1vT{e6Io8HTHY7A)|2B7sJ4ve8a$hnGD2DBc0vyfp76aoqj`# zZ%NOeWF~7}O@{phwc_Y=c#61r2xZ6YE?`dfb2Ca#H|rZ*ZlJ*P;Qhcw|K^u*It;6S zTlB^hr^k!RNY4MPXD+zUv`!r2HR7FA79;@otcYa6B-6G!tFN<1ql$U+qeF<_w|_Hb zCeX|wjtQ*o&h7K}7F<)}>-R(1E#(fWT`s1nARh(s(W^==-Pc3Vp9ZvaxBO)D$lc%@ z33tunH;{Gk$wc4jo95If0a#kJG;aa8Edlmm@a+dBnN7;6T1#BtEJAi6WV@W$AxIg# zjGU}|zPA6{vS)_Tg?OjZ&q>RnVMcw;b10iC1iou#4OzQqbfCM)KORy;>vc#!@4NlF zWxg4FUDCI!Ex6u@t0xUBg4ID zVrqKr3~Im^*tg)@R6WvTeo?+%>W*of0eBZ%pWm(9pOHT2HzxiB*XGE3_B!Bd>qP|c zC$G*9F4w{q!{psr=vO~jVO~U`Lts2!^ls1C+PL!d;HwsC4-JC}!3i@5&Sb{(T8+l; z|6qC3)qrVL>jXxQ`l*tfse6X7?!if##dIy+>Ah76m->46#IGXg4tQ}wK?#COZXpGqXy%9Y*toM6nh_FeW>}r+XCfD;vMOQ#RAG)DZa^f+UxBj+M0Q1U+#G? zqnG3GC5yal+;#zfqJogQJgpI4k*C&)Pljxm5&1L)RKm5zw}o-FtG%)8)?xNUKhb5{ z9srXW&mT?j?`GD@_V?EK(&pgk^PYOcw zIIGT@>EIjX5*8^+3Nx({;@+@dHmv7$ssB_b)^BJ2t~^`@{IrjK7VgXs(NCv5e*&_o z{yH!yj{r@Pxz+h)x#>MByb)^X&L?Y8C4v%;Acqjb85iq!@7(G#zwr#M;I`B~2r<(3 zf{u@-GgdK!$Xnz7rp(s!#HH_oX03R6W{y~Jw!#gj`nGDTh+Kt`@kV6CMW5LsW|2Rs za>zp_(8*4(_n4OU;d-3B=l4}-|Cy)G`yU?(?H0rfXPYvjo_x4P)yj6crX<53AX2W#L&hQ1M_LXiWf;sTUC)l0kB_n0Ras>cZkLG?#dO$v$w9KzZB!r< z0Z8isSffl-NjXJmkRS=3#el^#IeHFtnzWrk;oXUiOmwk-f-et~mSV@nNYXyqoCR?c z<~=-fZA2R?MxhPkiH6~yJs6CFbD|TsPpM$H)mmj5I_Ng#{u749rivqTNbtsIeSKJ)I^p!pzOv z%Us5gt=h5KM(IsJb7f*%jLtlM05z&%WW5@b*R}?X(c8)?!Stz+lzaV~+-jW3CU*>f zjCf4fOF~|^rReDN=@mpa{9!cc6|=$*mVPfw5NJ! z%r4NlM@uo7U!XHUrye)Rm+j#lwWAk$Yl*tY<6|kFC%uT9U4C+ba>|^I5Bo)p3powp zFh?n-GtFbQtl9t;jxvUQlxNs3iu3U`zXYSLsk=UF(jKdW=UlGv>D1|<(UB=w5YEI? zTs>b49L(y1w0o8Y4HyC%hFijKfHnbXsVCJ67R2tDOKfErkpNHz_;yCSXn@t!89~7Z zqu|#57(<1}S(I=${jd;NbKL%$v02$WH)a26+(F)A`pW;3M;sFjk@*$E5?dA5Wg|i- z-Lb}Iql|y0W7qWr@>HnW%42;%#G*27*Zx)Z9Hh`$L))m%vBitluQ#{aPm9g-Px>On zF`5v2TmqQlS3&nhw;J0jGxNCe?Pe$tzy9*@ev+P_(&qF-fx+oFOBtMG=yo1aJ6H%4ViC;s07KI?iFP4k)JSN!|^XMu> z`~D6dJ+p9ZQ03&bGZndGBPg6bVVRbc8mv@W1|^ympluP560abNhAQAk)k@aBo9bmN zaJJ|+&kB1?KUExA6IKLiAupln$KY7bml73cS_5jHbt!k8v7VBF>QY4IRn-bKFTFN| zP`cx2A^V1x1Rv}ub$-f9CKS$Sq^eX9>z6KceoC?C!0wf`KUJPzvJwSoQrzF7lMMKw z(8jqx#n;BE5>OrOtKwxTJa~#oZ1;#&oyZ`ol;vGc*#JHKDVw0a$hDewnyn@Jfr(ry zfW@wb>QN(G2I0?vJ>IK_YhJ^sKqL$h5#0IT3L>$braA z+KzTnZca--c4p6$iTVl#0-$Aze39H_R^@zhEn!!ta@pRwOHqmX!E`f5STGDEVdKZB zKCueQeNCI!$d!1rY9qSUU#TyhX8`POpw>5~?1D6I<)g=PbK9$UmZ^GY8v%*-b7DOH zBFLKS{I7955M@`MEOLmhjA{(zgZy-g6ka2dbtJ_PMKnUV`rH{HDN)iXp58p3^a>0t zj(XN9In3{b=3Lbb z$7(IXnU>jYGQNkNds7m;UT5mRN`@vnMVgMREK&AU^Fwz@pQ*FKfDu-o4^Y zhdbqAOLSrYY*F;yQNEtvTYuUgE^p;CN?od&{!TmJJ^wM2t`jIf>u?{Hkzc8jbM|L{ z<*!QL-;w;k9^jevRv@|2bQwa#oVT^t{00zW7+!T5?wc@E%(S9kaH zcsG+#hqw^cIIRjU{hbEfCWm*LZ#s+&e$-$}i*j@m3sc%(4M3DgoD3Gqjx(}xh)LA~ z-z}-s#+TP|)bX9)&0gAMp9&)FBZGDtBc07PlPKol{&%CQ1;g7y84d?sLzufHn@=sN zVmqJBav2|T%uv#bdDGH$z7@oisA*qaLvu}SxWoTwyUPQ01Y36XaB^89G(*;ydT81h zPcPg~NY7KnrMwUE5oM$^u^nUhNzc~sTU?5KqvYBo(Wg#=HyW$5yY3ds>2eo34b!Ay zHXUmbAYO&U+cma(|5i~n_;Gq%i{w11+pi9rFi|rbe?EV$z*29M@Gtp_)6He!B{CqU z$P^c48e+JrdI|TJVokpekeJlVr4MWz;m54Il}|0H(?a?Rxe><)l8cAwe`kJT$*5bZ z+$yktwL)qRDsx`l>Ss44qIul^zh}GakZZoCB5#1ki|UWi*$ldiygMJfctV2 zeCqWq?`F#sEy@CL346p;H=S`-2=!e+JF%N;(x-z+)Y=8xra4jBi5FAj@0 zoPyUTD58gG@*ZV5Z#;5}N`RtHli#k0qSbQ>CDE94ch5buJanXaEFdOnDyX$TFgojMZOb-^SzVgu0@uNXTHnB(K z5R1SU-%kmBYF43T)Ndf6<*Y;``G7e_PopXmZLuwrDk z?_I3SYK`HO0~KvNe?1t6pXdfKcV-y7-(~3b$SEWwshQUO0Q}Lb-a0UM&QxQUy73 zo0h&39`S2C&LKuDkAz+x`;@!D;iZ>h86)KVkn4%@gjQ_jJEB{Jhl$FS`jHHki_8c) zuL~fLP674%RG&PbfuWK#YtcECC|7Tgm&#z6x3osg4+K+ z+{15j8H>8z-4}UO9FJ=_=%US%9Z{)no5!y!v^T;s_jJ5RB}s2-XmID!TzYpe-f#{7 z=lzy}kZ#^FHF)0+6Gz!@d%SpwuOBwL_WZ2RE?)Yc_LevAnO()a2ZZWMtrh7q5a_Qo zQWoE=dN~oR(U!i`U&E`M{gIf@G%ct5s=;VRacEfD_J4%uIqn+fcl26`5tiS`4Rk$K z$~d4yixaVliA|(zbwIXhGUjs#h{+v@JMW#p8QSX;czcy(O`ji zdJ=?E5Q*$6azK;%&}*dGTz1ld9Z^&`;IX`=Bf&gOKAN5~+ojG83hEcU^V_%o?;uCs z#g1dF_Km&60yXt?9-98!o?jH2Zxgyac+LN$>bp0mR8rA{$EwufPD2{y(-;nJKfA+^ z=`CLx|F!PV&z++Dau1$g7O1XvM~lsr1B2|4;8o@FlfxlEa7cPZneeYbC_sm460@>I zl{oTQ;GD0I06aYvK3x`>(Y}D2Avu#w9$o*_DW0e?s)syshDBC6;Hq`lmg3K2eF_a7 zlIInM>QC7>m>($S#dM5X&e}roQ+8R`UXI4p`3?h{dZwx04=E}tfaESHH<{CPY`-?t z+kmFVfXau^NxF5jr&#%e6Hq2>>Y6|a@=my>!%dofzt>f0rY(T2Bj#q-EOJ z{AW4&G^)7iCd}7>>sj+V4nAwf?gYuJ(e>|_zV*#H5?Xrs>k+0@{iKwk8kt5Pb=D2i z$h|hU&^^KkuqyUlxtHK&Q3xW+B}^-vR%dxBJKEU2gkqcG(bQhS4Tto^|05r#@!-&RnsZG^V&2=L*ZVnEH@{-hXIv@Aqz4 zTAt;*WY=7J^}mg} z#O}@Fg|)|NVy1t1f^nn6Mb{Wz0D_D%xMgWT`{3b=o&wdVKAm@4#Jc)c7yG+a@Uw|6 zgWT&tu=iCnJ6misBKi;e&HfXOXgJMV4WVS`4K~1T0?dckd^F01@}&nY2N}AySqXE# z8%`_Xbd9LFJ#%01UyY|tPYgHYD_ZZmgZ<__PBK%X`PC_ce~T2$Wx2JTK3J-fa({yo{golM)Vd$+rmEja zSc!GEoidT5)=y##C3K$*I{(M@9M53d>Hzg|zsYZ~056^dXnB7~9KD36*w};@W(>ec z`J9DndwZZEys$scR48MPAjRRGY-Ooy#V9B7NMsiFpIhg*Bq6DCC3^3Vq4I7G7kB5W zsP*Zi(|QCd6gG4-mQod~!RkM1DCOfCXplsKRboG|J_u)*v%&=SYw|vIi&i-dv0B0Rzh2!k_JBPYkTkm9llBV-q6p$Zi2yxkiEJ zcYbv<3}3s*q?2nsaYYGAKSN}mx*sKJ+N|5`UsJ=spX^`T@g|KP((Ik>R2u=`PRr*0 zXY^C8k-h;_C3?JJ?jXNSmh=@~JuIIJ3dU-J?-u(6HeFqwDcHHNd0zv`^=tsfbBJ6( z0uPUP%Y@+Srfg66X4~1G`E-&p$q@~%V@=oHmjRnM@Fkwt9A2q){JBr%+XBbHZSl2b zyC|^^aGV~S?xk(e#e*Lv3j`@}&BJ5Pw*JeIyC>Icuv7Iy!1e?`Bc>Tywe2%$T8td_ zKfGZsK;(4pCEWF2wU5gQ47Cd9ztwo{DKncDyk;t`|A+DUbGvy(^Ed?>eiX|N&5ySZ z8oy!Pa9GAF^9TLzf$jS;44usMC}C!{9>|YN-a7AUYKGnVFarvi406|X@ei>DXF@8u5Qb->L4ys%|pNjqz^_FJe_yT?iG|zSHZH3+Lg^=KHpBiNxL`_Nt=a8YxwcO|@06Rwq)k6&1DjCW)#VHLCWm z8l_ecszp?-TCrD=pfyu5q6F<5$M4M_aOZfQ2BNW>SsWPl`kfJ)@n)4J_!UN-AbIf`a^o%GGc9?bE2@>iU#>N~w z`pLH0*BQK4tHIC*0Rg7_)=Kkjji+B#e^@WdFaXxOp5Z@hMDQczq;xI=!U=Ovc^=cV zmM72e-Sya-Im?$b%!Cy@NtxD~oP58*C_j~E$G#@nfKyws*qjW|+>WDRDn7_tu$% z0SWqc{Yt&$u_Rw058HJKe@px!;nYm*OThj*9ofVE)_L*b&r~-(+6U0wX0A1}sh6_( z^x(2My>sdV#<>~=1}ezbGdGfPv*w=sMA$pmtLEe%aVMB6EB&LuoVx`L4Kj+qDA<^> zY!0$9eIrhZ&n?U5cjDhCO?%K|tCwnfWPB!u+=~0*DD%D2 z6jvz;Kza60l|JbYlV%ae07+?nj<7xKt83v@`A@;YpGqPmxgg2#V`&dxd|Ll-dxH;d zjeRXnel9+}$RIkwe}8alzgSyP@(378c`%h;8K#^6`mj`4!{O(hK3tbY8*O}tedxB% z!Nn39&mb?3I@pU_FkOOozT$Di`v;sJXZF&@B0IhMo*cylR<6}WDy*L1jpW<{qJ zN$29XX1@Jq+?x;kGv|WI?qL`*g&l(!u1R09@N!do9zkT3vc%5Jo7k^)gvfhMOLNi}!PK-1w?jS8$$z zeWvx{8RE{prsJz$iK*J^2X}BAAippA)DLFkb?K2@T4lH|pks#w&604T>pP?YSzK{M zXy>Jwb%KO_je#Azb%#Hod+9we#s}Tsn)MOy;;J^Gd+Bcwnt?&E_>={(KUK628b=7b z%aAMClIUr#k#GAigSnK?A^C~qflRzRlzLgy^vvFvk3Cl<1Nu&`{Z@OCkFVo8rPmo? z*v;>wzJ1|dLne}z$s^Aq0h-b);0|6bZ8Aq7mNmB5jKlYDn*X%UEy%kP{fQwsiMfXL z#DNhVbS<(IsqYY_m(?|NXljIfH#Fh;{jcOh?O-m|36=TkpaGZXwY+bk^z_x6-jM!1 z`F!Q1Q?JzT_uMn?uM4z^#)*)jRm*-pYm!)>QWd7zO*{PO_=A%zdcm0LkqHGhx3Xn_ zn^=BrH&k!K6s%YchOnlIo8XX%38Oye2g&64)3|U0H(5{)aQ`kw;oMAqo${HwK@h2D z-j)()jnf3(Y07z!G!|iURLn5#lLJG`KGQgS&||0pPjD03yDbX155UU9Ic@00n;Lwz zh-EblkSG!GqgK8o226-E9R%3!d2`rN*C&ut!nKS&XDj32f9)&WiSd%le)Hpn)yPdA zu^MI3E(kK)*-Xcz0J9{G5N&R^^p*Q5e{1pJJC84f z@!zkwb?O-;kLRr<^M(T--LL5~D%k%nDYjZ&{j{|JZ9MtxqFX+C{<|~!n(g})YfMEY zD;%JZk{n2shaECIasM&(9#qICmSH)dJJ}3KmI86UyqK@Gy0C5KR@fJMF2aq^@a_rH zGvVUJPn#0@naQjS^?gr!UHOtMG!T8?KzC3zEoV^KQ>@r)_OaYrfzjjNfQ2hJY=bq8 zM~55q1|7?*)?QXO*WYg6KkqEho(UG(sWHKDeU1dZZv|d;TCyjpGlk0R(MHzdqNS#n z6i7w3q=?WL`G@ZWf^=Bkr-MVb7dtO8A%%tFN-8;E$9DGn``n^k++pej|KAy2#dYsH zTzrnF#AIk5HSJ74Lnw=*U2_e339j4AqfXkr$8|77_cnQvk9;yn-x*XNV{S&Nqr_W^*j zJj{bHY92jNJG>GXeOX?JCy5_PXf)m|vvM1F5s3%u=DAE#<$&?(aVgtGn5i%z-NM);Yq-ml|xc zE2>%W8%*>ATEtwr@9^*xz|=zg1GEOq(RZdaSDff!y-pTx!434P3ZM`>^!kiEVgc^9j)~DI? zuDw>NK;BElAR=@LaNq)cjdC1bv+r{emYLC-K>d=BFQP8endObUi0a1dCw)v(@%ei*7gmbZ^jD-zQz^o?Y+E` zIltY)A*t?d(=de;H@SeTxH6CvR^+mwB2f!)d8r^lrAwu05FOm9+~UNwzRgMoH4*Y- zina%vx~mtBAt5VlMCf3zFp-~@{3u(s(8AYOz;&%)J}pXCr}c)u@AN~$qb-)F7s*KC zliK~CDjGXH!?|&-u>~GVQ?U9yuLSeGSDPJPPafN!8OgXCi-(DP12lNO@1+pqncnU3 zfBTH*9UgXH3~WL^6^}UHZfM)on4XojMYFx$oAeMsD6ebcTzeu#xAMz(&*#NF+s@=N z)iAm4$a*JXwWIKsy;{(y|078-`>gpX1Y;eLDczxhb=rSrJ;*P7-#B{gniD`N_}vmZ zt3Vd{{h^POoPmuXKnvUXEV;tP#VuKtT;#V7z8*+pk_w|wdRm3V#h3-VJ^ba$epNdg zYk|6rO)5-r!TC6$TRoI_9LWw2|4vL8Pe?D#E#%#jdk-1dM}g_aN$`mU5K|6e8FFC) z`G*U9KV=>lrOe;{i93S@!|2p_dy!P~@s0-B4;E6r51}`$BR@zzO5g|V6}g!nQ&8Ahi8_p$MwtxSD0S8N|WX~%dEr`4_Aa|* z)f``QTJgCsbWvXxHMKokN1iV$$JSprYx+-dm|rLVIo&ZTEgiLFmF?chFEI7$wa-BK zrrd>2|2!iGk+T&1fwNi2W8JG0^jv#H_=&fSEe&DyA!&tV92yU$2x_tPZ)@T_t7;gt zn>L6>7%hE1#9BqZTC+4sMwK`^!?irz%#;4GdWWh zMBt!i+1D6NAE_kez8kPFZ)Tr|f}~PheSy@%5WnsxMhk;`PDT6orh7bij=b+}cB)XO zZlLVjKy@p5mOrE+iLmlwXBnK-A<|he)?R#y^C=h4 zd%3TOYjIUeeseBee5*7Bx;0S*b=qPQnq=p66$`r66<+?Qi(DG)_ofoe1uwAg@OKpy zle-Y&I5I2gL5HYphy~)*i3hlng?~5+xxlrdU$m&M(1uHgiw=Xmw*%l_jkY7wfjiJZ z&qXsDzPQn0#t2HLm3-?4uMkw3W=P%ta=B|f>21*16Cd14zpCaXV{=oahs9Z0Ri#m{ zyD73Mu|<)lZ%hluES7*Y`v-QHaETsqUlYv_eK4WBBjL3g+<4G)|8(W&f68<0^H3@m zLTc76roNz;(6ZVcTTE3IkwM@qSf}JOZOTmHAGj!)2akMjKI5>9=q5HXxt101hf-^K zb;9UjRsWTnt7T&L^c_8jx#;5r{Psz++=nXetJMzQ2bjvETj_`83xAvzn-?IgCaTeH zGlKS09WXJObW+LxxhB1TMob7;kC(REZ#X&@SI`k@l49KXg^BpO9mTEQ;20rTUGcz^ z__5k^)&?s6TP4lHCgBfN}MA$YXTYW`h0XVCTUhbCzCZ9$&~ zpb3>XNG*k0?_W=`5eQIbsATx;7guX2btkr!QOTNp$|tXm*#E(iH$gnL$mh98hR@)4 z=Oau^p_y8Nskk};JjXhc{(D>^@i#_3B!l8QIFGpjd=YK`SLB<;8(oUS#aVI4P5q8F ztNh+1-DSQ_Dgaogy6c%vrhBxlvsu5l1$VG!z*-jr=9Ts71ov1M>P<|YzdFIU%5qiQ zs9Q5MaGk(b@Bx}N!)!SP$rvSyulPQMsq}z`uC8idEo9bT(cCXqm7=WXnurFOS~q9Z zXA%-vIVe-ml*dHYvVb{qX{C!8c^=%D%6nwB3i8fG*fK{>N+|e(QhH{KWAe4nsOKoo zj2FJXBc!i(Z6Bm}lGBmvY;aNSs-8|xfDG)zhHc^qOF|J&s;=`M0GLl?9B|t|x$;sLLBnPy7HY zkm9{99M}oQ1+j!HOT)Outae>}9TPBV4ujwl#J~)q9B7Bh)-Y6&=8+dJamOVhWQ~@| z>wI(G@68%>Tb+Hz^yt@X)HAf^tdA}7mDLe7o-951OPPXd&+Fy@#E@M8mxMSq@_);qWne=c z$&e&a1xUfrB-cJ2WlSGHXyO7@i-8(g!|ON^l+o&N{P&RI`DIF4bU-E8&O60|H*igq zFv)#zbiy^Qu1lr5eWTY-N&@twszY0Q*R}pLIu;4^X?$;k+Hi+AK~^yp9-g^F@>5xq zg%ly%z$YgLEzqWtya8VEzyan{_fjwZiZRnH zANCgbEK0M{4FGD~Tw_C4fmMx<8Znu{1^-+b;q!D2f5Jo8gKih3p1hfx08jJf^xn{G zW*Y%En%Z6X^Eh9hOx>FLyWx>;p;Lz1tQfVCVv&cf`KuRzW0JIddqmChF@#>>;n5NVUEJ^+?RY!GSu#BD&}EVvGC5 z$&>1c)_LF2+u8JKx5dK4%dig6al62d6%Zs;uQU~f`yQIq*G z;f>V#AETRp=WpWo6@!2+5%pYp+O@AwtA3@cr{7xIb(Q~&)rWP)3FxqCv-HL{mCq@pGJq#h1GRJ ztOQz>86z~F%S<_!=EmyN{$^Tv&Ic@nDa!s)^gHOSSvcsrJb{aqT8kL3uziFU7)qy$ zCW2*+bZi){?>zVK1U=d+ElRt2gwvA4wXF$~M3y@ug?G;3Uy8X~wu_pvCqI>%d77fT z`sK6kEu%{{B1#vaN_4j#F3MyG2Vh<+J*eSBpfB^CW-os^LPOxPHx%jw2`B&ZrEo&l zU?ma7$JpEC)r0O;f-(Pmrc)lg{XPo!Q<@R?2%s@#AE`y_u$Yx&NHBdQt#?f6P!tY{ zx_!acrLsX3E!^g$vm`Pw?UZkWo{wO~RN`jJ+2&UH4wbJ+K9jd(?>U4TakC3*9Z|L6 zdehY8wb=={-}s}rzZBK~-ywDCi)vTUX67GQEb)i?vmSQny}ePhYP{mzUX&Xn3j)Un zG8L)9%P|Ds$ha&9x-(rL=0t3`D6%dSEk3eImvRM<$7VMK#mV>B_aQ9k`Lvk$$CUKW zj@m7yCCI}%Im2*~O1x5JI`w;C1ACX`kKyh!GFBy#R|2q)#j)eqUis{?wz?@uZQ?2R ztZ3U~+;jVVzI3%4-C}1SK0O$U?+G;t0NAEp@`N6{W8=sERxn5Y?WL6!B|Pt%39~X} z%j+M9L>0K?PuqSBHk|o{KbOIVk{o0mQ*-|L_Q4(JeY!98J(OhU-76>f%I)$&dl-5` zZjD3Dg6u+`(g+KXMa3z^6KH@^;gf~5msAc)?wwqWdZoqWf$Y&Y{v zHSsnX&ggJo#^vGY7Rvj7|Jy$F!H`?BdG><-m_#>LqRRlLkj%PJPB$Kuu6gb-MHQU4 zoC&7*Apty`zUTs^BHp6JvC$JTbjlza?+Ye%@PbIaJz#S~0k=t(h}P zAIq>l^SNjKSC31?di^yG@9$aYsBZr#tvFU|c8spC_ozq@zE|V){z0NilZa=|V~*LZ zIIOARy@_*msfHE1u{rk3mo(e&QR+^-rk+GU;?bs#J;;{3mG+Ql)b3@b5`XJuDUgH5 z^B#Z4qSZxCsE17t*3E8eG}VT^4D(KRy$7jU^HauN`#h%96fQ0F1KLEy1gBgjG}>`l z;N{q}28VBWeW&>?1P!5fkNw6hMli2uJ`Wt20Ed4)XZW`np$<(0(?42LUM?BVNQP|QJE~jB4Y1Yac-#4| zaVbH2Lg*nVXu2UFtL5J@9k^Km#J0E|H3QokFV@f6ZfkT+Sb5k0Y%fqP9sxBe26zAQ z-+>j*?<9dkDva={$|4{pxm2^Kt2P$S%H2C0OP7HLW~(-(2d zs1kfh%h6x#rqC1(rSMrU+DD|K2$h&fbRNEuZE8+F^;yIm#vm^W$$(ZqTES8dz&+8;eWVQnXDMN{bljX5!l^oFDFLIbW2$S6S%K$sd@5KTD;ccp- z^cVRK!Fb0d2#6h&i8he12~)jp7dq_k{v@e)_7`Q{4uV?O3FUxTN<%Ei?!2NuYL!n_ z32KZp19U%~Vde4BYC~{=El*1OGyVNG<$ovQWHGHd8f7wVmyjgLJmMsvK-Mg2xn-97 z^j1fBa|==gCf;t59o4St2N$UCJ7{_8G^@bPO5!?wyOssF?}q>RH^9R^2YaV-m4W^< NxpwDj)fIcp{{g{6PO$(0 literal 0 HcmV?d00001 diff --git a/doc/design/images/deep_learning.png b/doc/design/images/deep_learning.png new file mode 100644 index 0000000000000000000000000000000000000000..026becc4d94e01e407dacb2a5314a0e5723334ff GIT binary patch literal 40605 zcmcfoWmr_-8#WAsiYQ16NJ}?JOG|g>45hTh(A^+0bcb|@gmi;QcX!9o-3&0iqrboR zd%VYSzsK|K`2>5`UVC5bT34Ru+F)fxDKr#96a)kWG#P1eRRo0R<_HLg>#v@{e~DK| zQA0rRMUW8}QFmQ9Tz;+l7V?CAdU81Y`x)x94L=Ik!h=9$I!KILKCYVyVB6YsKHs@W z&*J-4MayJnVsY`86scgTzzr;Y^9|+@I(stI!IcZ2BXp)truXj|Hg+C2@~cNj?tV6V zUZ=Qa#QbEEgTE93p`KC{_wPUclz;xb|M*81g#Z5+|BpBP|4jXV`R-q>v?B1;{1W6~ zHC-R>+FZJf4S|U37vcCKzXB2?BCM^96P*)JG69sy5ne^Y*O||8Q9xh_zrw&vD|zYr zT}~;n72HnT?D$@gKPBYq>hfT^#Kq+aCTj5Idhc-#^75AS zqi@#>m|@lsBrZ0&hg?ajwblRdSoPg06@M}%fj)L6P8}L!Ri+2ro@jTUrTJtdqQI3I#{WG($=|_B*qxV>X<1K|rMRz@eMCd&*28z&E1almp zYmDkR2Oc(cjSr_Ok)Q%vfmIp-nj=39!7PL<4{m(y^j8*b`f}pl8Ety0LD#R}4MqQW zA8Lo3A1@p9B8m&An%{2GkE0p&5||)5UpTzk`drd>_(e*d9@)ikV3#(QO0>aRH8~~W zz+ytvQvmQWIU#S(bJ4p=fQ+vxqKWowA|s=;ndRy8hrM@(lt3X$O5qhY zi<`82KXUsRh71%^#k>|{#fBKXq;&g^alA6-b-0~O7dJ(ZEGM-em} zU~C9;WHp_MV1i;$V@zO-p>M&{_1tQnqGjgzQ*F?IvbPK&#OK|eeKG$$%X?SdzQV}h z$r}Y8lhv()1J@Zj*6%dztl+tUqt0Mtv%-8`TYZg8O>(7L&#=@H^3Cu*`~j$`u=0iW z(^`LI@I%1>C2F=<)2;AMvW*LH>bNNXq3||=GD}r^2V)7?T5C-`UZc0!m(hr{vddoy zF2P?hM9}b6n?_K^!Np97A#B88neonv^8xFEPkxyQQm=dM9Pu!oZ_C{@GdrFIQu*K!=Z%n8f*Eqr3OlUEHxiRlL&7L zwQ4RT@U&8pD*(ykdZmr$7K011Y=w|>XFRsLO!zYHb1reO->o||$$gF?a6-4rc&9PK z-|hCkLbc(=Ef;gjN_k?_-eSqlvUEPCvDD0Lb@>m`>1-A0s~K)NNA_bEseVhZLKg5` z1D&B>Lo1=33H7IhbwVkGqLk3o*HVt&iG32{Q@Se;VSYc3l$P%@y?bD-^Ph#on5%=v zZu{0*y7#u$;9E|fBVg!erZs$l>^-Wdf2Vuu@ERZ^(K+y8r~0~98G|lG@BFY(Gc^J= z#nR@!#w7iUjnS?8U84AdIMv>Un3%tm7twtLzsYme&k(Mahi4NC1mJi(AkUwM&>6PnUSR(Nsq3Idv>0(gq5x=Y4QT^ZSzyBmy0-EitVvXZiwAU6sPo$U5>RDZaCv zRxi$s%$;KC!Y%lN20u(@HT)r-L940uuw7(Yd(jH4F=%$=&Xx(rt=D zu@806B|Xkp&IC1oE#6$OOE#$QejwbMwY9hb z8lz;3hB*PciyuC6bPP(x9hXBy6~m!ZKkVXf{~`D5{3N=);0Hp6jcn^2TDVWr{`KB8*(e_mwtjCSv z*O}It9Ud|bSSBk0i^kPV&5WSIlbw2AS#P1%6`j4@;#GnZkn1P7XJrY%)wy$UDIQp*50006kl7|4LNWz zUUs2WC^VqT<86M=H8@wx_3At%!}sne<3Z@a%EGhfAc)D2nvTM58GV?DawGSsth;Gj2 zL`l0)Sz`gsl8YBQUoNHd@L>xtng%u`Xg zGtyvTiVZ+>4|QrqX}De$Z1m2gY)VH%05%f)xVk+SK?w*RF@6NJ)(&GUgcHN2JL=`H zMIsNy!iW^Q=acB&CJ&oW_rCN!^6hV`@kF8wq}~afTSVOuycCo99e`3;f;g~c4%)Jf zh-5`4`u4q%)@uDT1ePGqn2roOT*_nQ={PZw7biluF1f`wO!%$vW0c?XZfku|4M|CG zy6QB@+dfAMD2SysHBva8S|G`nDa@s1Yu=yTK}k#!2d3jb`%MJKaUInGzq`k~@5aT@ zXLjQ`x9%PJ*n$Z?T!tak->{kC9X^OF=PNuugpO>+Aln2*aI;{WFIa8Ko;tDxsv=ILYH52 zHi**XL3ix*>X7pA;x(y~SabDNfXPX0i+PxluHazv0~=FA#Kk9pX&9D4Pu2EplTW|! zc@i`+Xl4Sqm=Eqo@0~_?bH+`P<#Q=R6m}zY`^`uIUJ>{T~S7IsT(_bT0M8)&rn$+Au<^C@@I5t-s-2{TYT@fAdYIc680-}P%%MVO zzhZ4!gp3n?SGmfNhzUH$WWM)C?W_JSzgqOd>ap-+wciht^r-zNe2i5}%I`##lFBLT z!ycM8)*RwCZvb>IC1yTK6%~}44D`D!t)M8qDA@L|UOqLjx0a#ggJIu~03b&1J)s5S z%cR|B9|s3#Bfg(!IM`BAXyp$EnPld8TnQRcS0^yxvG+@l#4l@nr)F0c5-Uy99U$;$ zOO-#75;BM<*Ihk$6i)0Dr9magr@~q;?|dOwn({`oN?K+hI=OgAJ4&&)8`$@2PBpnE zjifF*q4p>PAa4k^zBxXwwL5`V6Yn@ZWV%#y_%s*G3EvrZLqO?T5vnndrUJM{h>UWk zSm0##kwz^o>(nEDXAB10lRyKXJDY$kWqPM{OsKPMdZH_<*pV-{gRg4*!ct#HLgYZQsbndvGv_j!1&3NlMbSNfF0gyImh@>DB8 zTb&7!2y4opCz!+Z*U|`|w|;M%zQDp+$w`NQP%GqxLuJtedYPkOO5sjI0g0rSms#Md<5E6wiI)Z5SEy zJZG!QnHHXo%a*7_+(y?BSzX-p7!tEXv+%rwava_uth(qpm#+e|Q~TRKIKD4;^uz?f z?GsEMlYHvZ`j%#5kySf>jpFt^z0J+)sNEoiNzywz1PjfQr_LizsEDkn?6eU<*?&NT z*M0X>Z|8|zkTgsUR4%JN8_*|&9r5l;IdOmgQLUMz;e01i?a4?E9iJhIuE#lLr3A!3 zg)wDlZuf#tDKk6n4&w%-7s-d{nhXF~8Xvkm>E7unAMb5&@miUBMM)M>6fMKcD+B~( za>?p1a6!^#gjnco*2Y~`=iVscv#MkL*Vn^4f+Q){h71x`Ivc(E$3Z>)BstqqGPt|K zfebp{8)#L)MKuch3rKc^N5t+2{jSd;*l?A@`|g|z`a&(TJ5d`JwnJgDAvNoaljS#J zjQ`;db0mtlmXlx&&AP6lMAewf?vcrb>636M;tMJG)c1)dKKwV6#}2};2bQ;v<3!Ck zuTdm%5&z;2(fx+d^|`7|ncx=??-HomZ&UYXXig|RlOZ4+zmmILD7VvWt<(Fo))0+U z6pgH&A=12mn)8DI{{=+B3mA0*El}MK{j=E!2t`3_=3|!m;`EKhwQIKW;#!V{?8PB+=x=`8w;FU^G zbg=|nKGd}6_@9p;uIKV8z70Ed!TF=5EWQud0_v7SydU{_@_$a36=$AUF1Ood>^$mm z7`rpan8&S`g;};)Hk)F7q@A~0C|9iiB=y?-MlZD*Kicnz>X+pQfysIZ<)XW<*-lG``_qn(N zp|aC3WZPd{kqFA0$>0%-C$3a9k^P{IsX4cxC&$W{VxXT**~ljV#&MI*KqIXu?supm zb2M#!*)YNUEply5sqKc+P8MuCOO2wK-O;YTZ}AsGpnWqS`XI%&yC!i=V=h!>=VY2U zuRRS{%y_Q|y<(yMICUm(E|0PwFiSwes^xSaMDl#A4Cx1h*-f5$aB8n~BIQ>f>U;z4 zcTQK+XwMEfonneZ`aZLjibtJ~L{oPt}^GStjZyE8c<;Wxh^t zI2r>|Ko9Fjlf)f=+-{716Sz1?5j1U0l=c>~bV0rHMPYHBCidB#Ae(n1C}&U7yaKwM z?(IMPD8)mfS^iFGkf9{VOlNaA?P$2m&(O%f67>g|QqIN2yg$keYkbp4wL$BV8D~hJ zr*pO_;NcR%SK(wZfUD0*>)rZj4YTTHC7YK`Z9=Li$#;t~fVnKJFRRFfgp)~?da|Bt zupp+>TBsDRY@^aPw7ri#Wr`T(@NfNIX<5>R=Ub#F&cQbQV+!jrt-;G+NK#}yV{SEM z&W1vhJ+J)N@ifGlp55-=phbhEFqeAPLL2)mj8ewO-PT!F7R+XX-t{Qf<&AZg5dOf} z_SE`t?JZXN!ShAk<2KEe6}Lf1M;_O!l_Bielz$*ofOPVkO> zs2Hcdn~lqQ_M?==p{}9nc0}JS_?+=7Y2%1I(*a|K3>w&{+(BIn9m0!!xsbEEeUhZL z+ncs99dh;sqTp9E;4Nac`W} ziMh9yG4VL;iTOSsX`i@jyhL{Wx#MN=?}k_;VWrKSo-KXWdQD*waX8>8kKPaCw5Pz5 zFo+~7?x$t&8ryb39xkP|$yAs>rux!1{K?oLfk=fWM$67{vd5kcq}_)pg1T!>fAnMO z1t%FkdmK5nRuSbOQOxMPuzvzm{s+a}>5nI*ay1+Eye!w}qZHeXs+d@USeR_^g;xbH zR8S^{^abmj`f?|R#+G|!5$n9!C&R4#uwC8Ng(F@=@66V2XwwS~$;;Nrt|~M$RXg;L z`Ob@rk?ug5$Q+p_D$Z|&af#}k63Wn#!W_NeqLYs~O4gA5vxI9B-9(~Zg{Pzke=cQf z(=y82q$Xba*BiFZg{Y(PExQ&(!~qYv=Ll1rdeh`f;+GI@?~@g(fpvC6HDO_v&2$c0 zJL;on&l;!t&S*zN^lj`X3nyT7PK;#eZuJ>0f>>^`g&|Ata+HTBm(*0i!WTQbWEvm8 zeR2I2JwO^3YcA8ZeMU~k*77Tl%I;~*4&w6`)j5h8T7Gh}H;^w0>L+;KI0<#+JMuM} zpN4|~G>PJ#z4?+OI^-Pk`5IOdLj<=PrkKf(b z?hRw?X77KX<|-=Jn(g@Ep-BRyl8#`n^n~ZmJV^w2YH>Ynz{Hh6lPSMy$@QFVlm=}C zmIoFeL{%>~f-A`ru`U&0hqH`J*cN*nsEP>6qTfzO7qKC~-S9iF2=?x7cmc2Id0EvqgegcWOMZv!NmAd(hlF@O5a0L~>)KvYYK`WYwDaAH ztO|KS@${J)4kFE7{|Hv)Mmw)$d0u_;#O%^uEYx z`P%gWF&+cn_GASSExJj;`7c3c< z5pqXuYROC1G1AByPn^zg|u9j%gsi~!B zsrBWb#bq&Ui6bJC*epEzMm6aR%aWsZN|yUJ4%Mdp7*tt#UFJxq|cnKMp@JF z&CWH$@JkQbMI!u1^+BHMkaeiqy+_Hw-n0rjGeDS5G2$?{%$>Uwhbf_6XU(+rB#u39^* zY7d&I+PwobIf}U5{1SqP5?rp_8cc8J zurT4i$zrQ)kG~Fw1dx+KWq|yosbLFWQlVU@NiVccXj{|FT~#t~qFja(83h>I4o`hM ze0r^-7l=q^dp*A}-@mEhr#C=NF+u;*9--%ceU9i6b=VF?iefyx(kH-IeY(Fs-EnvD z%X;Xaq0BHp<6JTMTNTK?06)l!tLCVI@keStj!3+`_x;%2#C_!UYbC^q;i$q{G1pmZ zI4rxu^Xh_=0_bkrf0g!zCGD>N$DD(xP2c9m65Vh!^}ukDY#Hq|aISD`Fl01uYHp>q zb6_qf=`_AMbW4|}Io^KKo4x(J&J?Ly@WbbmU-7xDn0u(|C<&UG8Zk%`;-fd|wRfX; z41tR&UE1SEPvUHGTx$65oUWZUV;&1c6Hp_iuwg5%h6xGGc4hwRR3s$qwO#?`u3?Dk zcndp-S&}}YwKiAZ9z`YR2ZZrU-ZJbU+L5B}4l9&KE@uv@YkGB?_w|M zFdFMsT{?OJh~dDKV%lOsK%+XL743z6IkEMkhd}RRY&5d52Uh!epSMFlotehF zY{ka?PKmG^SB8OJ9cKK(SWH*2EEmSP-DH~~9jDt1#=V^vkPauH;_h{#>UK~)E}XPc z=R%lr#9EJ>p_k*&^c&&m3rUo+IxZ(Td{9sIoynAB+V+|0rwXiB`7($WpK*;y(zqw` zT?kiLX`pOMK|f6~Ui^Y`W-xT7$E{SOo*F9^RhG#ybvbN2eMdD{mK=zEwh3pcmPXC6 z4xWV_4?o_uZ4fyK@2$FMcJb&-)LYpp)5lYa1_$3_v>#O#X?R1-pxLb}bil$=acZ?M zwDy5r`BNss2aoxAMWTcr_&07R`RD|1p|KhbouPqP22%;a>Uprqa}YD^z-wpf-Nes( zc=trf^=yee$OJI;>=OX`CVj}K!R5eyN(Rl0990ev3oP#9DQPkQr+oHK|ayW)8qUYhBALi`-4`TVGm!1qXWCC%COm`xz+gaC<;4tlEZv zD)z1gmxi3;M~j|rjbOZOfvH(NZN$%TO!Ok+nAM#UNu_T3+`uVp|Ln%r zvAz;x=fv_UU+;+@>f=APOQE571Y=R9{KO?1Mv34X?YH|ILU(+jvN>5FbU1x0wW?k9 z$V?HcAaLWp@o|USx4_#QE#V8sBI}@Gtrc%A4WD;r){@*F2ep-{-pxx_!RLR_iX{mk zyFFPXLtsK*ZP1|xs!V4Tg{DuVjdP+}uSJ3=Z`)2D%>WC=qK&K#gSNxvpaXrS|GDIT zmP|H>ItoThGGYlnUoS#Da=H+J!z;66FI%$dp-RUtwa_h zw@d%jMzFuVLTfZWzfv67#FhHFI)4pZh>u?RG_f;NeSKIl+A2ohMNh)=4!=RhQSH+} zPPgXD?XQNzC|+uv{uLMjGhIQba(nlQfH^4EE|XvZB=0(gQ$+TsfDx+EonVviT@5uI5OV!obTQd zG;xZ!X@5TXa?}$lVW^JvA84Z|&wqckK8h;OCvumi*$J%;|Md9ZX_7hO*6Y!pO-AO@ z{H|5z7)QrGMiN3tv45iv-k`;ihGBnEtrQEx-9?3#)LifXNovJq^RbPB=FB&1X?EDh zeomgOzNv`fKLtz>T(k+c+^V{#WeM~(cYhnJ*E0eA8kS zZ%AMA>MsoBi_Okj_Oa%}Xfz|8Ps{uoxwT~W|6XXNNJDZbWD4%L*uW2cch-l#+01ne z{-08R-W^OF_f4+8M#05SI$K_`I$nHv_1t$24wHS-oN0ngJ)!F_Q_^WM9>~8=g`BJ) zXuSAu@MX34@c2_$vy-Ya>Ds(8ybW*(r2QvM6)O852rb4zL3yR8eKL7u(ioff-(YMN zjwUgbsrkK{GU-a*N%c`r$UCfmgBzf9pu?qH0AQ5WI0QV_a;t6YGllVYGkk~PeFt}@ zZYTC2YdjcY#HVDbR)MpR{NMVu6PN9-dxcpaDCk_YToqB$v9dt^4A1_voVe~=y`k8@ zw8!Hzks^BPUVdr7zriwc7f5#C>y)Gy5N6NC@FQqYm^%1>gIRBnhE2vVIL_6OAj^a| z#N*yA+Eo9{Y7A1-kPs})`T>GEfLue&mo;7ee+)Uf>$xzo4(JOrHEchGYSiWXMDdq* z|C9CCcGQLN$;bQLtTByk?(D@y5hq~QKkG2$vJq%IH#dyWTIwx3j{gY|0y(Mt&s6g` zwkw3HZVeXzB5pV*+c-_FM>Ja5|MLJkN?|9!rT^O_G!eJnm)Y=FQsBz{o9W1WaSpGR zYlDbI+)hlDgs%GNg1-FofVUo-t4?gI$^YT95$3`|oImmZkE|zy?GioSy>O+gOY8PH zd*$XE==^U-5CDJ!rNS9Wy+<|Wa_6%QV(#h;DTIGp`Ns5dA=lv1<=|UZOSais-WUNX zi^V_3g1j|mj?#~gk9$w&)78ESw~#Gtq<=F(TU0FYLA_{lx|5ZyvoA;bcx)8^X8s|0 z9k%3J8+{G>FfZg6#|2Z-|Bq9O_ns_z+Sex3o}=a)&cmO&FoP=cKcB=Ltm5CSd$99P z&C;d5Nq)XX8LdAc`n?H zJ?nY&@?`(XlW!51Br&Qqs)QBqVtqLWcdu@W2dw{vtfn+?&~&y0pITd1i_f4Lavfpl zKgZ*B7KFl!m?2FEa617-Kir$eo;Cdo5>~L5pO3Ro+$bmssB>tWbg7%RFnM)xTx(3sZ?|}2JMTx(e$d7Z!})~jf0=y zAb)8HOy05&EB?lUZ+DM-l;aKDNB+j8tVy1zvd0_7xb8mIw`@7T*7w;>*PTf1&5BsMffdJ>}bqOsFmRAF*&>GU!D~0?a+Sn(hP} z6QHpGI#8G}FQKBnIP7v*k&lY=j7Nh6_%f7Skz*ZXk{Y9u$T$< z8Sa^Xl=sW@9~U(yvCo&ir~`-k;UWKTH0*@41aj@Z!brv$P7F#8FRaT!6P@pjJH)3O%&Xm|H@;9uDe_f*&m zD}xA6c7Q6dD^Er4hb7=I2?DminrB33mLF=OptFI@f~;ou?7`18UL^d+1Xzx*%de(q zWOxRof{ruq$9+{n3760)d~^iTf&^5`1e1)_jB*kGz?5`2U>q z@EW-CLB5-QV*Q4Kx>&e#YuJQmG3DO3F!6~e1y9+Vg4){R^furz9+_%R9 zlH!S1MqKPGNPa#v6WfB7;*ACo$81OU+P!6=z&pszt5;MxC!>=rD%$Dy7D`_+}4Kw4k zu)HqiTemGT7cNWwAo0H68CR2UhW(V%)QsSCuFciMi(;(fjJFJm5e$J1I$s_^7sioSsHLOE55AC<BM+&Mj|1BszG*oQD`sPm|J*E67 zP)M;#56j`j2Xh_@l}s9gUy-fZ;y2De9bwY>Mh5Rze??dIJn5!U7#(nhu@s~CtCLH+$o>`U`9 z(er)hvwANsLd-3@7HbD_Qr+l zl3Och*oce6wKzsDZd9m%-d!CUc6t$m*l}F<2@ou8G3u{7@S{+1N0ky^Xgh+gh5N?b zFxF^(l5a!RsP~g#5+N^I_vM7|?;}!a`O+yrCQ^-t?4k?%#Bdj*`NJF|;-&HW4_pgZ z*R-|T{(lNrmsxOLr-77LyxCOZZHs9atK;``;{&WBvWBRZ&Z{eB%#Z2z2YhVuNXVpL&t-s=IxbfD)*;~`c<&cfL4 z+>UUXj*F0GIuGXz|FD2>o1_yqo7?7xZm^~>mW&$3ld9?HC#{sMHipN|26>epM5R$h zUh7Or)#%E$SaDcIcASriofWc7Cuns-4Ai`VN|h%r^991?Kiu>Fdq}- zg`f21o)`E{L}apu^(plh##icjnP7-Zyj_vOaT==Y=J64za#v@qYf5^oNm@L76v5wY z)ZwESUcr+R=RoH9dXg*p$I15#d2ma4p^(P6>$|iOQzCE@yXY-x5e4Q49YA_VtWC|M z`IYHnR7b4Z=aKPL2NZb#pMz%GUV_wfPPCF(J^K)Sdc}xDA!{OJXfZ@}J8YTyWqNT4 zbOoff(Z6C&>kU!>Yuz$NdCXUQd+W|pDsq5eldIN?9T2!s%%$-$&5jUykj(65akWA~ zbAE9u&D=!mRLeAo>9sV-9LKqOI2s<-K7z|~d$m8AZcNF^{5RP0ne4H{J+2{zv1Q*t zjdcso${QOJloVZ8Y6Al1EjV@1uE}x(xjq&QYql2K0&S}H74TZUa()SwaR5-m6QA9` zsb7|rY^$_r=JD|%BlSNW4ZyW)CS(jFzGy(gyUt9&2fdh0J zK1>4_?Q<%ha~?P^UK{r$s&Hp7I966>cM)= zO}VOGi{unYpvc2jirTJg(7$dH8x~})`>2{FdkBJDRXfk*Y6Pt2avbUvs{KBq4A#o^ zhSf}Dtl3|MYi8!EHke;EHCL9#o0}#-=skDw*Z&>+tyrSM=TA1i-?KUgb4fjonzNmv z5*P;65#f+tzS!R(9ZO|ubeGHMNE=PR=F&bteA*AcCX37eWzPHofFfV#b4BxGZ-uJu zo`g7z4#3V47_b%XXRbi4q+uDRzffIokUCWuv5=tKMO9@c6*xJhRFk!uYmFm!lXvxw zXiY#r_YU13RgBZJXgjPdP%Pf^3Te8wG?H%l6C$+^Zi4-BWpc6J3I1ORY4!)mbd<_W zip4Qwz!3P^l}D^6^rB3Q*Maj$aBCci|G~5KHre55_SyD=H`9`YA0DGlnC1KGk`9$( zx*?}cp0o#db$tAsWG1MmPCl!x)q#HUHOLfTX5%DeR;I(w*Cho^mfXj}jc8;htK4p% zMXuT1WyPN#bs=jJC4xno^i{c8H8b339&a3L`#7=OH$O&(F(BQK#2xuBS@YUkhp&e} zs@Jw^HNEDuQ)0ll9WkR;N;Nsi zg$aWePhU!aS{N_>lo3n(_4YPQQ5l)payaq9NY|t@pi*rr?>UApLld|VGV?)1^Uyr2 z#|SGORi+Qili_r@91M-(aLXAuEcAk4^8nWG4D#g67d z4T*s0^k7*I`doq$rE+$0&2-XlTh}a_j%B|e0}EDk70^PlvDA7M_BUxRRQf%`U{}s@C3<-jSOK%z z5z~_J4*;e;8<^>ri;Ze!$-TWktx;NNO=0_Qw%VR;bWcsG2a1jtC=fO3!fatM<<|&Y zY4a8_Qg>0fR=7C zHJ^PtO3cV?5^(npy)MXVQS1DvLP9IoJ%P;1WO^{VG7u#<5+*k?$nrtUn%W*r_e*ZP zZ!o_yDJOdY56_N(0^T++^)xvh5KpFZTbBUgpY9Mcl3z@~@vVzDjsE9t!G!$At=^is`(a=m606ck-u z-IwUZ+7O3%1Y3hNdA@V6A?&!?EC-H%tkkaNh64O@B~tmlRvtieg;8*2ciyt=MsxR z6rh8D7f^e2dE+k{*kuBTlv3~Oiu*k>Vr&HL^gP(vZiW=B^Z6P+h@(w~nzjvWdzTM; z>OBgMsbIi6>Jp7Y#dLNKj>(A$V|GBUWWc zSO(rTb6u$n*mmN=(Tk$jw`JiEGk?BE`oE#UZV>}v<66~GG{%-!i6|`7eRAVf9PcXx*0>~(+bKVP7f$r2M^gc8v46Mmg9my{vowNPobSZ8hE21y_n3ySBMy4JC2 z&7Zaa?;cdNCk{ehzkaPGc%^P-#pEJ>Fk5l5(#p@Xv()Gi7Z(RMpD6__nM4B1oZ;RH zw_{`sQpZx;i9DId`qph%^C68Egik;tfho7jwc;`ySw4T}?6+wcEz zfSwiz21{jXmXY3HKO{BqMU&HrUThe3Jf7SiFAs1LH>6#S=9kt87)fl~93D+%XP0=1 z-Fzm?e-*He;2U?pw9(*lvU09QIs0yJbF&jt7V2?1E|r^;!|;*g$_l7s8FhdwMOdeZ&v zM?(}r!24z=(BU}!ujgVr2%jR*BWacJ45Ud(NnstxWagZNfub?d(I1@l)um1cMn;4j zw&hc~aY_0&);ag0&7Z)bRs8nt+v$#Ov$Hi2_&qa|?8bJZ3;cj2C@82^ zu1kQ2XUd*Z2U;GmL5ExBPL;`EfqaVUjF7m5gqyp&ePLr~XQzX`y`hoO-A-0I8VTR! zR)5^p{&W}^Y>e^tGX|vLDx8yLo3_pi4l)_;?mX*%$;(faw{P6R@Q*y;GlQ4-z zZ=ue5@n*#%$^iGl@nCQ7?(t?7XMA@&xAFPbxmUio6aoe({HU^E^ViY9BY%6Bz#TZQ zkGM~BW7#{hJh12?v}rCHx#zsOC*3XI{IVokl9c56lHCa&LQ&SeX9*O(Wk`O+POhHbBpMt*0$jF%JP3^Wc!XKI`?6YDgld~4${dlDX>ki3h z@rDfGO0C0nb~j!boi>ZImR4KW;fYm$fbYgTsm9%5 z3R2p-!vWe}+O8Wn#F{K`W8mzoR`YEZ4P|pg$g)4akodaX>{74I#P57)s~Djhh1Pqy z-Wkleg1r{PP8cCM_L2r0e0@kyY{JaXpPKp1Y$R<8RjaT1%}v z{@mFEl3ITizv$iVWh&Tn7 zaHv8mcUf9lZ&%mS9l3gRVc|}(T4CCtR3uXyBgoj;*aY9_VwmS{a-?wE3ofr9DsqvW zwDeD4nX8%zb7D^G-JQV+wXr^Syo=?MgZ&&6e(`z65qRD(qAC9F_&V(5q`~u4wr(D{ z-IyC>(Orc31fc}B&axJnajL&G>nXICQws>RYOlCHNz!sUf4Y^u)I?&%2h#=&$*WG)i z9HyrjrsH;d@ht9C%|!Xfmq(!FQuT5l%ZEp-q{o6Q~d2R z;wR=ek$maL0||`1QDi%rLTzzV(&B$2J1I}_YUp;K>O*xWJhp>)4;(pv+Ih*AYFZXY z6hxv=-AEQ=0p8Qn?kM6_+{!rkF3{4P{le%?)S1H$FqIRJ-Qic*oBD9La@ZMR^*F-u zRVAr<;Hv>u&I}CHn(E>7^^hFP=qCuFD5Ap-o|43Z(aXv;8H2t^MiMuEouoNt=mF!< zo(F$eg2#f9bbdJnKaCaLgxq%s0+`;y_10!frXtxthO9;F>jU;{8j$%hlhW!p_2uS37d;nVZxaM%%K)Qh=c54vk)IauSl*8$1$xz~@k5hYw_$90+G#vT z;$eGeIF*MZfowgoBdY-c0}akYo!t)Pf`6-#!Ryh&Z^P$yKRmSRN2;ubYL^TQ)ynhm z<2iJcNs_$F_P=qTiQ`ZrOr5WNwA&E(wwN;K%=Ce|4JI-t$WqUu@xFhbg~A-u{&=05 z)pZxi6z~#_klRYTI4nt+f(Pn@AVB5V^Dr%iXCTkdNN2OL>WSRf%bfWR#jbZ z!3te-ZcTD_xnIU3aQ5r=L8t*9!W3-HoICZ7Bn5-`@5XIwu0UB@Q4lO_pAeHIt% zK=OpAawaol8=(lvfS1y~K>Gg+gSuMAU;1u%rk%wU=1#*USTXb?RL^9iSI70nQctR^ zD?%S(E&Fxk(qjy-P1~KR(-n=3f`XI$!IgWly8HDZ;P_1~une@)(ijyLb)ot|+6%8( z7k+*b1AuLxV2?LdiMRD)FMK^Vf!h#x-)MpM8)h8ELnLq;vvWK#D-kf?w4)2HD4et( zIA%H8q#?SMPR|%_`G7)+AmG>$Lk7O>Y>U6WZF4TG`mx;PWO?S|lB>d!RA0&HLthIV z_Gz$LZ4>f>#`Epmo^6s$$|W*2F4n#!A<>(mcS2|&GPHSUX4xkCAE}3R)EoTrx`{|< zDwWA&#N@%9rYsH1*?+%5V7{kb&Ilvsl^HL5v2P;b*8XRsF6P&z$h>M$@IS&UuVJ>M zV_>u@QT=KmInx}Z6%61i*g{~AIY=0O03%cdUSFf z-&1exh3n0aaH@N4^+Q6H`fR%j&5a^_44u(+ud&HuPHO?97&b=Fv9_q4#g^i z^X~5M7vGKC+_(gt)R;O|lYSwfnSSmC!V@?Q0UZs0L zGbY{$wzjdC^H4LfVT_Di#0L=rIX*#f!2@p;0x5f!2oV7V5A-}Aw{eik-*Z<0tQ8qa|6cLYAi_j@TJ=F>$=k-Xm>sRC@5Xgs_8-@}A@ z&H~{OP_Kbp0dLirK1`ka+XIlis*8dX{y&yq%-Oo%(B!2#4%eMFzl>-~$2iV82eaaFf~Tm3g&8^Ri*JY|_`xhhlEWU1JlIp1*(X(1x1FYO=h6+|oJn8W%{h zh>0N%JA;J0F0iXHL7*lIjfw*HWMmvhr(p=NL_Uv)cL$;jjf@a-I~W%)Ih36@1gU}6 zPC~~P+CA@p(Q+kToyF-!kA#Sl_N~~zdGm(0wGpV#9Ay({Agu=7fo+1GO;*H2UpXBx z1YiEi+vann*50qImQ+ml*Bob;YTAls=(~R{o}TXi`iZFQnI)&EG>Tmq^wu{eb=Oid z?9))q!otGxNF14wXDU(1Ie0! zQbvvZb9{@sYR31%DH9Wu0b>?*b#)t^YuYk@nO>|)k1z4 zf+2}i(Vm0~9^TsA0d`}OAZcHd&-J>+3h`K$X|YI<6Cj=gcDAz}Qoa(ZxWw}(gzgoF z4j0$8cE`uDj#=YWN+MRoN27z<0o7KjLPbLj#pyc=tf5|~jM)Y~$$G0zP>YR`>}7JQ zyUEBn!?>dc0>mQ&3C06(!WYkP75ki2RHm0~A-acFTIC$Hv?bH|ud;4AjjE0#YXzU( zH*hvw#hUv7O#nb8xORj9z!`DW3J-z4>0O8X>(g$YS?lc`KYfJhB+kH}hTuXODyylC z)jaobucdh=8hVE<@|`?IJI|Ph=z8BAurcN+3lc^Bscbp?Cw*9geE_PMvP+3I1;0eJ zHm-3O59xhbCiP5S9dJe`fji3%uySEr9^kyGFW%z-%ulMK zFd)c9+?NU`b+Bu(kQ}fN2}?5=qsK&oPU-QI6HD}TOulBY?UG}ZE`#;<%LM72!ZX`% zz8(QZ`HSxocTKMDaT#gd`qj^W?s^4h-{1Xd;GXq9RUp6Ut%U2mSOrv-n3S@1%Wpj6 z3FnG?R#u*XH38G9m@)^8PHo-!)%kD0cAISFZx|>3|PV*(~;R?n&v20UKgXt(uqQlP>AEfi%RlSld#=MFl;CLpHX9i;ysFV$ zf@oU#Gys6}hJs$hW-(`8dH0aM)Z>6))0ymPtYG$IwrJkz1q-g@{{C*wC}FE5&(yfP zC@IH{-|Z1bI^s$pSY)+4UsS48??RH#jP7yAGUeRc;P`jT{I11npt!d%=~5y!P4qNK zGJR#ur?}EDe1QC_&lbD$BI9u{6O4g@q33-wQzITg08~3|?pJom*uFu1JJ*h-s0@?I zm+yAqv``7h4;Of~98+voqVW@zo%02b4oKY_+Vv}UR}Ghq@JeEk>(#6a7|?f37BLJq zqP!O~a0U!pM({3cY+B=ow>*zKC!}w9aS{jliu(K~zi;I}9b9xVz74m%3g1m{XMzn` zk9glw#?0(!QMXgkPe9hRh2v5`pPiaF8 zislPx?ShV?;*W+{NphexeP9HPd|C-xiJrQvO-wCA41Q7kKf#i^dwudlfm~lqE@R?z zkR~HKip|lLF}{BD+c~HRZtp1st%Qk~1G zgb@E(lO5%-NV(483=ke#ap;#*dW*&Ig_@e$4K4?)+ZfzZDzkF)?T_4Q^I!t{S!w~d)#XJ*jE50o z@4}38>UydBGs2n&H;QW$+S(}}Xrl9Y<;w)+ya5(c~o;b2$1ByB&m2fS7 zu|~2FS|^P8&)9uEh?*NE8`#lvqDW$w*JK3 z4ud;iWeN!;e_9dAc?BYka&n>#{+?aX=FPttY1?#}zH;51-h;ok$CB1im(=DCIZV$? zHqRzUAZlszK5n|3oN&S3|G;y;CioWf7yHB=17sn&!KzFnxgw7iCJpC%nNv z2;;lQ`|koi&J*K(l>+pS_w_#*t>9lyI|i0OF;p`QI6DpJy_U_aD{3rH-agd>R(}>i z|G3cxmxA?soW5s^29Ecy*~A1{&KHzcPHj$S8RPc_1>+J`4`Djnf9x{3yM>vjoPuP6 zX0WVOnZMuZORWfdf1@|4&-brxc$h28V|-Wgrl>`9U%~NUyf0JvVU9xVERwm->_QV` zI(NW1(8;n74&9*6ze13AB~c&!z?Bq;8uEA zux$E+{x6D9ATW*H*18+dS|9Yd(soz|@r_54t@;>ePt9>g7?wUE-8gRgHd=C$kNJ;8 zgh%*bvCrw%Gu5U>y4Yp(rX>~3@jcEHRhM|$IGB~_Va+FzzjCMp3K(%p-r!AU3!Y&9 zuw?&`h1JpMMDg9<&PWV&XG7uH=n#OCt{$E@Hk~TkeMu*asM0>&Q<(1d>=AHUX!r>u1W|ZNg)5E@?z(<;| zJVs#!BBztxgDr|0iKuj8NiDt_!)m%lM|(mxm>BNfjN4B##cz(rSi)?UEH!9^g6r)gHU)h zFOPX*V?%|?)Dm{h#o@cNI^+hyfdJ=%J*dp%(DAB!CZuueOGe z>M3wiTF}tjo%bte?p)6_DVNt)m(^({V`zJ-`{}Z0yA;{PxCCCdXRkigPqq1KDE(Ez zX2ue;GEveMFQYcBr|>seq$YfXpXd+*BAzoCIrzpvDfH(;Wg=1$2QoZc_Bb*8ze!WB z)8w*SO9tlDNO5v6KxEHo&~DULKT~I@O72ale*z!#b^d)-c81kYgZYy^Y0h+N2c9*jqRQjs=#3{~`^T7R zg(e-P{iFs!FSyaxqo2b zV75H9*7i$*NO;e?aa`dKLj`F|ZChP;eh&Gv?BH!nB_Dpi1xd9KpOenEG!`m2Hfgcx zx}Um~H^|3nPX)NT?z+_ULuP}>^Pz7F%6yBdjV)=lT`O769(9UNg_oYUVo38Z3(dpJ zP=WJVP98YPek@S<{dD}JBS zNdQGIZ$u^Jd9focJ)IB>t4FMNa8O!j3vql=x+F}E$P1F!BB>}q&tfpU?w)fr0#M=_ zv%~{}zjwCd=*0T@+nE7HVBtK;$mi^!0MF5Q(<0UoHOlACSWxXm4)S2_p4TMKo z$Z7YTeu)N{eC2*bF|HT#b>Pyr5(%XYH*jpvzr7Bms-eYBD951 zUFHzQMZr3oY(~mJ<-;#d%7F5@&-Aw>9)d$@aj{UqtGbAET)}5I7ob@}CMSv4&|vJ3 zz$pv%&?%RG19ScOfjRG!NW7y&5W+)0kCSYjR&q_L&Nt(Vov%IIvYI|J4U~gez_utD zev4Ka74mFQhf%2bum0Qx`^(^aYSMl}QbGJ?S<5mjS17<6SF5dfT8XIj^)P}_AlrsW zTf;=B{pB#gQr>5aCNGC7FevimU~a>ZwEOGMJLiKc0UA@*eD(K6*4b#DG=v|85d4=6 z4MkTVuhHweaBNv$HTC#y0fetJPkL0;7y!~XMGgQ6vVyeqVxui_qp+}W7r+VfdR&{| zElnIa0ZgaHf7}nAItCWj&0kD~HrVA*MTO9|iyY#^{Z^C-i>AE%=r1&Zette$S`dJ2 z*<0^H`Zj~=4cMU^_IA~xo@eSd+}}0@ABu4UM>?MUUw_5 z%RKh>_F(AJ8l{{AXRA0cjsynnj>p4NI`s-W1#)0Ge}mD%ur1=C54f8fwO{-JzV)QM zadk5zSgSk(0DV2r+RY%6)A0aX8YS@9@AD_)_z`{;Zrord<>p1;aw1@p)$fuTl5r)@ zco|)+!~ff0K65m|ibvD9fd|*t%-HoNwp+xzeGyFgIl*LM&=N&1WMS5FK~~B}#@Vqz z<+alfa>9=^B~Z>Z)Zi?yUtcfbUe*2SLmsL*Y4Qv-5Y2f!G7Sw2>oKg(%)ByvRj;M1 ztW1g>rW+8CCI}m!(c2wOZ#5mk<7^!r8sc@o+y=O_cb;WMMHB2HBu?|tN51sT6_RBepF;tvu3GkJa4wN5xHBP(O+!OtR3~+wL4UH^3C1quC zlH~G6HeOzBslSB~H313$V2E!|SB^k)wWcEu%j|P)b7e!g0GWKWOJj%4pj8K8%GOH_ z&Bef#lWAQ2L?E)eS~fB=vIK$j0D2t&QCqaAczAetd0VA87VC$+LECs_aBy(VUycqn zn*rVboY%^_z8raIrv)75STP`DnIz0hlF%hqgeR>gW)VZ2$QQ=l}cRYHuu!o0Wzpgsx6zb3^NRcO?V5$*5JV>9WcrKGgDT^I@l zHfYfMN!%PS#DZCwnVHGR$as07VN4z7m*vCI_YWx`LE`qk=b^n3Yk^9eOCQ4!Scm)M&%01zqu+S1 zb)F3{VxX(P7CHN(QIHk!8S>K=RBP$!xD-m%O0TZ2+S~E_R501LE>cryX=(2Z8%};X zES&)8Jb)m}PX%sOye9!h)7KYdd6#2->>3{zH$T==p;0Ytr<=Pj`R_KQrO~?;0MzRq z<6Noj7YmQb`Kv8^9ai9IbaX>{Noi@Y+a+6M4hpV303}{PGu<6d#^>a?)#A5X|C7XQ zgn@|(Xps6(bbn8uNU5BEQ}1)x=z0g8N$?9@Y;mT0;Takp9-g1qP`|?v=CEBIjtDTW zY)^GZb9+-*vNX9W1My02h@_CyBLmt~ud4*FQHb`hNz?*YJPO=~XsA#kK9OKT;rV#T zMJa6_^!v*OelQFgA~9+qGtM&T3{k2%B3sQ3NlfSC6MpJ>l5q{mX`cYUTZd=%|J2Sc zFBMGtP44-epEgg-sZ4K1NlD3Xs=)vEnh$iuX91Xi#|yQ9=43BHAN=I(({Fcd%j^TY z|NB?eM(taL^N3vLhhLpP#yBI@&yr;$aLRczAiKm;hK4!+7-zfMnwoos?ZQkShc(Wg zr;B-DlBElH{}aIj_Kc^1YUP;w{(Pew)+!!FbaLG#G1^BAwnB^aL-260!i(4iCRmle zU9;<%o(~1z-Lm8D4xlOFnH>wqWu=r&0(1)Rp&Rwbp05^9uF_4OQ=fNE!vG^Q@12xA ziFTq2m**JW@~>sfiJl7!SP=#tH4pF6nS~fQUcIs8&4uz@3GCD7dC=e$Zc0 zW>V7SITm+4P=l;*1#!ZC3#*SnHxy#1b1QdA!o&Y(Z~Wy9NXXx_qp%l?BSb>*)~A)3Xh=O0XSdn-7~FnirY z$mpj$9)C#1Sd@qE$qu;;^9QJr$3-Jmr6vJB9u14hxAAVcR!LFLrSbl_9qr1DMxCdX z6wk=!1hJ%4{BKJ1`Ead^sTfDt)4Jz}aCL^gU$zfZx26fRdL}>hE9MP}YO8v2S5sim%prhuqGfW1XsOADm0RytFm zOh#fn7)KK!N7K;(X!Ze4x?<|`bftAE5l}v16&^XAEY1)Sq*9QOY_xd$E)(E`MP+>` z&P$p9{i3+qWDRayuP$(KCMaQ7&G`Kw)F}jPdQZnvy?jCPEalsy{I?znb<&pF**hISg2z$BvWE_vU6{iP8S}VR6%k zEg?0EtGrVc*-|2D(r&IocdAeUQcXFa_Jq|r=Xdtn+!MFQOpw2A`3E`&f&2ldrSS2m zzMC-T0#&HP7#6p?t^dOyc6h|`L|UM@T4lYCY`qSTO|w(lO#Dj!S|z2%r^)Z1@I-=c zKL!BC1h9*gE(ZFSPsJ_b9v{dj8U5K~re9*{_TC#v&1Gyy7W+GRH`_g1I9!7NW>xCD zk3WeX+qu6t`hs)&vFo?M-TNTNYL(bwI8jm({xeWpm^P&1+$K=t;OWJ{(Wh|wd3&|+ z2(|M0Cr^xfuyML6T^LQX-*q^dH%2g9Rridqn8aXZzvHN*99h9jMZC*_LYcTBM@-dE zIZ*CB0e4G8Ieqr_sEp~?ZviMpfyWohJV6v^H$H#jc&rZwREM1XY-$(!(1@AXsM?+W*<;*tFkAw(u|tz z=40&VQ#>o0V+Fo<{LHD+%c;^(a${05|FDz)_Xx2OA@KL4s+5DU3fdS|sN%^Z(Jw6TWMl~%P+BBfLenZ!XLNS$i9wsw(oZ!=%_A$q!RxnzEt@-G!5gymw0P;q@`z;j3 zBX%R#{YGJL3fmf^W&2cI@h``UN^le{WUY^NN!iBmc`DeYt_@>O> zl;(l3qb2_z8h@Lbns$Z~?@m`z@yVpTD=?$CJ6Il9DRAlvy3Q=9hQZReUGu>iL{8}qyJkt?HCyh+Oq%w^G&D&mUD@z` zy3k{?(EVQ=VokbA{+|(}F>{_=oVS+NmilrkUs|UsJigz^BauWzJnq({Ed~yg->FIV zjj@=eRg6fMYjd9@h0>Ex;q#hIP6kehTWr8k<(g)Sl_9v~TR6m);Msgxa zA|OTD)O5wusUHu?L*w!a3Y0(XNV$g2{(%Ia1Id=(Pg|Xku&^Uvl;7B40fHmL&reW7 zrlz(HJ@$-NuLlQ5;>nJ)*XP$Ly&4^S9f$X-8|JsVUc#Bg4<5eC^F#M`E<1$?mmpbr zAGGB;s`YAK;XIo@^EzxDw#)@<`Im@t+S}`oCaaqjPhv`NqPozqE<>8?x0QHK93}Gt z#CijGdgN^rP?QhsuVv-xSos@5? zA{7M~9Qno|5?z6qBJZo;NoQ7lR`yW6IbI?Mg)>~DLk{vldmzx+Zc9C9hwBw#k#-e) z%ZQ$GV@I~|-KdB&-5PIv_=DhcN4Wo60D``ziGQ|eJ>f%AlB3#QM#aEgdc8(OSaj1f z(ya7MJr%{T_EK}`+E%=+Y@0FiFTt%%qt|n3pe{wwunnp}#W=F-B zF$GV4L=#O>7pR07DQs!J5wSP*HVr-LF`-1wKNcQ&bnH%ioK4`)NKBZ`?``>kC25XD zLT>n&TH@+Mru1G1RA%d$@^x+er=>4CCtDd$_>6R_)ABm{Bs1E1Ci;ORlak9{r669# zt6?B(#Kgo*j|>?h9`yjK&l~5J>og-%Q&U-4*;ayP@!X9y*v*_iP;s9*$vwZC%k;XG zLhuqfZ%sW1w0c{duNt#_eP2C~TwgmuCC}dQq(|ak6>YU4thfAwYzTFfb#$qzA!LTv zMj{zFQ*r`fqu45y%Zb`*TgKmfd5vHhpRo^5te(HkYFG-i_IZJV*!RMzfVbvfsh>f@>8biXw2)7<@6a6q_0)6`HFFlr2t zs&a~m1GQA?3@9@=zlTAuZ9c<>N;YZT>BITCIsM}U9>Y(%0M%o8_ z>|#oQ2B$ugn)vGv-!0^lse>ju`mnGtpitwT_I$P6q?6EmJ3a}DJ?H6xddSHfFB$Ix zUd#`VZW9V-G6bBPM-&;HJYIHrv0SL+j8)W4AY|d}w{E=*9 zd*wCh`(r~t-1`#Cduy#SKj#@I>$Z|rwOY#v#7uW){dq2r!jAOe15oM^gr+0 zM{_CRS2Ghhw#Z)u^-}d^oZwB;6mT_a9pi6D>=#WGEI56#Rxa{g`Vw2}_#16d{KxbR zLkSn{Wb&S@s&7Xwwl`mRc0KdRZpkI3q?C$PS zmQ@XuJ4<1SbPQO$kx?;Hhl4ENHh^ z%)99MD#e($x8>m)f@bbBQ?^n`YQ2>U%ReZ^m0q{X+|`E-ZG8n#YKeq*&t)m+EF^?SNTsMwB|MCbHUR7d zC=DI(K79_fcXnO`G+8L{BFf4!r3_S*lm!I^Kv6I^g9Q7u3Fr`R9!_RT6HAYaWJqbAma99TP?AB9y^RDey}fpbN+ z@nm=2Jf)!##x3VyIa>y3AR|H@CR43|;v0ZfmW_89phUh2CST_&1g?J?fsRW;d~PaS zABr!=cS6;{{dTeuhAAFi=KcDoQM<}ymdvU0R+Z0t-yd%(n`I�*v>KdtmW{i2{ic z?m%;X7W-Qi8?FII(^je@)>|9Wg<`oybVQo9nlg8&4VQa)3XDsT-Q^r_Js{*r-j3oj z`B_c%DcLf#svz?A(6y<##B3gj-I^K}#uUUydR~gyfrO>~o5#>lq>7**;cUfXm5NPD zF(vvTmJcEeZWt4v0}V81pB@DS1n{ko6@GeR17f`=Jw8Dqg;@eYA3oPQo8H#d25Wp* zhO8|fHn~bbWeDi*vbemUK>a&Jgs_95#-Tl_sjO zu2|>4hV~j78kP`*ER!A8-7=o0yumNwzg%ycz=ysIY^pCU`b}zA0~vG_GhMG;sf971 z2m(A@8uU5Ihe%V|wsI1U7`aEyL-~3!_wOgOV?_b0J}+O6P*exG=tkNm#;b|N#)2YK zifBDo3@zags@ub-m*tj!{Z-qqN}tLK$f?EE1=S*luk~`S0~93d8=Pk7UUOER1;(x{ zh9j349Sw~4g^1}t-0(VygCr_mBNs!&Wm;W>Xv@$JCJS#{^Yz9UDe7!37Aq%h6NlQd zCaIoDl_FV$2Co4(?JGx(eo@Rs0v@UrN4^c2Y3>j)N6N>8;NnRkOq%95>}ca27Jm+O`Jb=7 zCsMbX6?0eaNl1t}bThH6gjgQl)i?)O*X(L_*O*CmCxylM+SQigKBS>fOqMj+cQoka zue4-X=`yt)jxO!>D$AG7`pT9yiAst-4HDZ2=ruT-FMX+j_l_`_vGmg|$^b*)ICuaV z!ndLM#CoHl1E%=`;zZ@>;{yZmK92HQ>v_uT{YA#I424HuP}Ke|^tj0GY_VpsJTqCr zib}0^$TrgEVlo;n#7IIh$(P0uIW6qdexgp@P`0( zxuX}-&kB7bSD-Dtedy?D*By~ETSkf*qV63VyLWH&00oxz+&NtG@{;!BBvQ`t;_Efq zI4EO9c?39pc@l81@xD=o-3SM<_&i?0QvV~OxePUS`>rc}+NqDd&*oD5e@lHc3%j%J8 z;DuuW!=f62iWtJ8obkkmx5lJp@rs7R8kA(Kdv?*~OxqyOPXTPI^TzaFlCtY#0%i@%CMxqOE3dU?rPl|6&-tDDSCKFLu0 zI*pA;k%tClcfIB@OvvJk!kol@FQzhNU1DwN+YlgAD9VXj05}+A?28)Y^eGYsQI9Y5gtl-u|$1KUiDH*#z zUbL!kX6{UsB_U*1<+&sozJCL_2lYc;sygXj+gT3KKRiu!4_S_M*)YaJN81YLel2@_ zeOZk4+(_w`#&rxal z(MOC+!;kOJ>n4ti^odhj z|I7A&|Cv5~rxdgFyW<`5;IB9m;BoOQ;(Fz`-Db8{ZmY8&90=FW+PA==S-enj@Zxqv zE7lp0d)KsP=Bd6eGI;gRwyBA5Uv4Io==t36fLqAD4qVwxrmrd)&pR4#!=?74frA^F z(7NBJWOIoz@Xa3ax9V<_yHTunV%Z)`pN)*_wUXiG6Pw@lrNZ8I)&O)>%a?g%&K>?2 zeO{Y5edyP2`9A#JF%N0>b*8Hcwh{fRmm{|+tZ2z?C!zDosdt5(CjoApZvexHWgBVB zP<2ADmlXM=mq5{Q53BPlapZb0b(cddh8ITL(&y|@>-R0W@_Rm6zjAlb+Ne*l=N3Sd zlYcU*?_XNb^!(x|5ncS;_MAF9yzr;V1?3vfWU)B`UCV?Vrl_h<%gx;#8sax?0YIYi z4S$dZ()0+_3%P2FJ_^UbVxc2^`YKPA{LLopa)A~EI*uM{3wN}owb?FhUfWxl3Cq!v z6FJ)X@Uk4@xf8}L$s?xU;cOO;{}@(tJr8v*y?=YrsJiyqs5iVBdfKWq_RpHidRt{R z`X$fD&@^RRI0a*b;Ci{L!qvyM#9U^|0B>)KP{KaO(q-$HU6W=&FK1E)swQWCkTO?aK&8YUf1H*Ptr$S0CCakkD&hIFd5f&uLt zoD`9oHfC7SphDw5E*zl38*ibt98xFLiP3i!Q(P4^mNZ=3OFW}*Q<>x<$X!%^5Ix|0 z0SvOE95RF_78J&79O=QY-wx*Q~sJhJsg0LIOFXpEok z+PX&~rDbG*mWiv&OQ5?P;GeY{grW;9@|A(+)m#-?ETngLOnt4d{l5Hf?P-EbbhY8R zv28f`u>$PMozKWV@^?dwboDs56Z&o?*Ov9FM@{Gh>mipR$kffkia@IsKoYsQ+yWSy zgQKIRrKRn&Z@hGV%g@2yxw*N%K8b9_b*HcY9lQ<+Xxrp#wO8L&6Ka;OAi}!o+CwPT za`Ya>WZey-YlxUSbsaW|i-LF^3QG-t^)V0=6HAbDacSuM(UctCFXy{)c6Mf#J7I38 z`T2hbq^X^$6S#?;dXRXx(^-NDeu#bHHmq0p{MnS&t-?(tskwh(uiw{RFIKOh&8tZ- zr@6T~IXM{@7Z>Ph;IxxO3Lp)p`ts$^&=7Lvr~e(1DqX2{cYHGKGR*Jt^!X{2j#zfg zdOxqLW~&}r5N{j%G028aPe3pB<;wvWr~B(dB2y;6=PjTdAK}M5IXc3}#>NI@*H&9w zTfY*4Ck6x|&u_~Xzj%Z-rcxU0I0C48K#Q6l>VHf7eimZ_@RB|^_j1k8H^Sbhf-zXO zkOg({#z9P_ifny1Rgga}E0hb%<8bN*h}>BILF1bK{(kDz_4Oaumj8Ey`%}IAq)(`C zVv?HXiamw#J~_pu9>MkdU#&tln3KYcZRFFB;luH#!|cL4ebVh1D526PEa1wlF|n}& zs}TRQ>1&M99|b2m*wkhzQmTP03oI=~no?iyPB!=8wL^I}D6#l2Vmj2Z31s#y{usAdU9R;Z{G!{Cqxi5pmamPbft3U}jdE*+BG zv=pSk*@h0E1vd?Ui&kqa*P4w1d`<7)zd%T+aQ$z>##|Sqj754SXbvsgzFgcZFIA9=Hq?}IVN!!`=VR;=B{5ujOc7JgPxKJ%=wo{rPmegEvT(I8(Pp ztT*NDkqXAcSdN~UN^AmcwB&TF>sjY71PL)Q0|y9dltrKS_mfsn1TEGL#^&Z%nidI|+F7iRCz}dunG1h7hH>Y= zsb!~v8l0&7otSaPAiLYwbpeW=`}?d;MeXF&)aC#>3NXhwVHer28)+%T^q#J?3qa+) zePJIL?E?-|4jGFmfy~8sG|(W~RsmdncKPDJ6JAo|5g7k|(?dZ}sYU8^+_NRkEkD>J zQTZm3!R2h#8|b+OTDiZKcUgZUL8oEEGqC7>mCHio;rh>W*@;CwY$Gi9g-_c_S<$B9 zt)->u*D~e5TBdPNT=K4eE~@v6v(lXukX7%*@CXULfu?%R~lw~DXd0i@)=)#J%zV|R%bQxM)qKPC@l zGClJRk{=sblT6z>MLF@jjjoP2PE@JXuTImCNl0j5H|ZO1DuNaY3@}wT{JvY2!v5eL zmyjTjh|9~iTLCq@z??C&R11Bd%x}GUmVZ3p4YAuPoRYsLv+-TS`c z2XbHJ%oF2dcU#{y2|s@&3KZ#$jGq^vc4KYg78hFuXaJ6z&3VQBh3RitQLEg*BR z`P7c;Iq)!?^u-Z^lQ_QZ?Qcz>2VgaAG#6YI9Bgx&`l4~AYAuzpS8t#% zv@qT}w8qrG^D{KXcq>Oam!JKTH|WV|WX}I{8euo8C+PaNAxfP>PTuF`Y1^2Qoz{KH zzO1E3CXq8we7C}X!g~vThFHq}^>bDhA@DNHbq@Pytwl(bmvPlJ$3u*%l-$X^)|6rj zv!G6!yVr?*--aVgk?+d{b7%BQMjQ0BsEB&nILzq1=FQk$)5r7aT+G&6JaI%^E@Mgi zk-GSyXK6(46t%DRAO@0~!Y|$3ExYm9!{En@_5RNl#)s1gFRjpVqt8u%=Pi+jl&Fx( zf%glUf7#mg3=AhW{gj%g3w+DURO4SVtJ(s>NxB?A&g3j3^|yNT89Fw@zCgQ#Rkt_% zsxv1wTH#B_7vPikuxZYDHdn39d;l~Lf2m`XxSo&LU%vRiMfeEqw!XO)P`8CM^eY@m z=}YG|MSNI~oe|fT*+8IEqoQJJ-B`SbXigy}tJ#nfL-ZS`XfOfSANHP}pRE4lx7}6q zDR9l^$}4WpMOICGRFH z0tbi3>48E$ZtFbR}%;}u&E z_H%VHh*BQzS;zQ(P0vd+#!qwP5rY}o9sE9Bw?_%c9d=7>`SXv-rzfSs)II69$RqJB z>|1g28{KbNUnH#H>Y?{Vy5-n7D*={YhFePNcVa!}K*4P!W9XX|QT-8w8R-=ppH7as zBdGR`M5e{HYju>02Lv2)+E)>aQ#dR;oy4H>Su|$YP%$_wvI2 z%9-26JpAeFUw^*4%giQTm(!)a>(_S_AF6cy74P_$7!(hPjRYaXF($vl3r93}=-R>ErFdFy#dL7^tM8 zHikWOA3+a0E6sm7?M2#aAIDfw+X)V)xx%5>7%x?4nl?UK@ce<+_|^K+$glOWSKI-a zGWLLHqhN!e2zj_m%+}rED~oU(`ouEjkRv|X-0X|>9yU=j#}jAtJy@4r`ILR@afGWv&}*5^XCO=H zhD>xc*@Y^a%~X?VX(yZm!_!Vh>hS2QFTGIOfoVMokq0%c1imRCxxC;duYxb0DNT3L z5M^9lU_5_wa6bjpm06Q?Lot$EeOsUV@gUjgEI~TIsCQOPQ*V%g90I*dN)mvzj+&S{ zWOjK*<=qlFWS;+QN%4cV)zm1b{Px>KYCh7PPkFyXL^Ni^YU=&Cv4%*J+E_3Qw*TUa z%|UfibW;A~H>kzqv5ilFParoWwO&k1JLUvSW39eNQc~o;q`uLdjm_$7$#0v;C0vX- zsAA~JNSZ6V$DfF~z@V0UWQC7QRPn{d<+sL9MOD8ai9J!5F2aIm^+zzgID>jWP>=G7 zAZlJbaQY|h0=coBu`Ao%Khx^vEKsh?NVdtsPI4EUMP%IW-C;35OGfj2VmqbRXEELP z_^T(?bMdF6gF-T!?C-)@Sy^?x-f||twLLn8qq;k}^um#^4Wr8zApOOlffb)(O~||b zz(aQjw>g^q#1}$2!q3+s)SlbS+Dcq@f4JlB$VRe)f_?eCmdHu)=SPqSE4SRwz+OtN ziIHh>%jv4AoA!3mk;Sg7+5+#E=uh)CWF(U_nP4fzWZ-Qt=4>r0aKM<^rr32mfzA+V zOaJEI#i!Uwxz|Q4RN7De5hT6 z!GcbjEhL(EY|ssu+dUp05cKFuSKC?h#ZZ5d8vp3>-n)AeMof3z5Ou3m;&qvc!UFHD z-09wJaJsv#VrX+>E?o-F!TNk8b>x=aUvg2SvL2@37HRM+y4gOXH2y7R0zD70xz5(% zUDYvYbL<(<>Z??_bl=@k7=IwuA$HWbhwvmZJ>VKk-lLQjfTHBc1liGokuz& ziEVpwPc|xFe7S-m_DxeK3>LOiX2Ot^jXZIgjlVQjZ9rEVMu~~IGZ&1x_Mglc8?0QT zBn~Cs{;<37rmA3Wt?l%Knf`I-M?O*ZLUf!2e zWa_twU>Gh%2RFKIc*4MJEQqviK5X)>=h$N@bshq372&)7;00}(Vr1FP?QOJOzDLEo z`40b_8}X~;Ye&IC*f&f{Zti5{w@%&pc4jO?9z<(P;C_u?0bb1yOKroo44x`*o?@p} zC&~xqW&fwSD-VZq4f}IWI)ozGvW;ZP5?bseOCf97vL752vX3+jC6t61OLi&QDa#3C zISARZB{CQink-{qGWhP%Ro{1=>s$W)T=QPRvUiqNVcnB$cwoVY^mX9AdggecgS{*w@~590CnS#&Ea+|VW(9s{Mh%mqqcTm z_jbwJmxn#wc0TpD5CWb!-?$`DKc|4?LSOlmJ#(87q(maN(oBt7lalT`iIcW>m9MxH zv1LdGn(*^xPmA`pD<=4UQ;XO*PO-#Q7JW!w1JkKwnfm6aSn<*|t3xCQ@%qHJIN8)% zrdIluY4UD3|D$qygl6-W;b1pf1xDqx^T74b@DG`@QD|0 zS$Yg2XSzJ+ME298thKvbG2t->d-}0BZBdMosTxzEoZ3)8tA-XS`6TBC)ujo#mugR# zmi4@ryKx&x!bZV(@r5&^Li)+?7#Lu`->lF4M@?5=IAsitvNezKoDGu-_Oxbq^6b*% zi=nY&%)ngKP<<-S!?5gi?m;Wl@;bh4L@GSO+OJXJmFL}-&XFt$WnlZu7C3$;@mM%g zMivURDeu;-biTf?yIk;Qvnt1N`w!X&sXjKGZYG<&a(CMLrgSEGzp$!mmEgL}iUQd3#1EXt{!!KdG8r=5;YomQEwqN0#>J>k-6MCC7Ei}U1x%4;zd z-^Z!t?c*~?3f^{?7@P~QVr(;^MV>VE)7>lM_m#D-iwfWF%vQV}>D=;7hR?=nD%jp9 z#B2Q=UCOqY+xTL8*O$rL*h41_h3KO=rL;Uq1ew5+io+I9$ur8bzTl^bLY$65a1SO0 zPug&8=X`Qz3a}Slxqq^w8C#TNR8tiu=VK7JVITB-r^&#ddt(E=V&c*)J*ht!)P7n~ z@lJc>g@|eHv8}@IV?qrLdSRjY!V)d`1DW^LX2k9uaqiBZ&)~5Z2kj7VGNn#0a!$!v zY5plk^A5?V)8^g>n)KWgA?JN`)@dKRrKGJ#6=$UQ4}`^i9?@R+64kzd-@K)!LFC|D zYNn&=IYUW0-(>p>Si1ZETsX$Nv2}KZy7XXUt3ml(`pjSUJSy||9{6%AHlL=CaKRma zBE{q{!;H?pbwo*{&AsN{dnDyuZ~W+g!GLC&n)R-`zPs(v!-ew|V_b%~U-x&%EjHD%={8yX=x{ ziSlS0Xm?eqQZJZ`nO(`;3T8N}=dJT#bv1oWy68|##KEpcCcBEK&32v&(xfwD9oLIO zUfDg?WAKRhhe6?PR92?Q?STdXmI{YpuRBSO*Ll2Uu;RW?&DEHcRW1mG$7){n@0oKv zpgOEn+{=6^Vo7R$eRTOu3@3F*kw^c$DUamVjtjlVz-;>Vb?6bBs`7x(zY`zc%mtPZ zPzYbuk2sKar$6M-<9!}FhS7E&M!(>b`LL9AIdpioxB0W1=g)#-*AsN*-rQ^rMTYeD zh|4red1XCTHue&8;Hp2Q?;fY=0i`o75nZAfb6e9u{H*2!>x5BvMG3E( zT+bf$tuKkfXT+W1Ed{J0de4V*!PfU_s+8XLeBpX>>Mbr;&!>UOtZhiYtMab2`CtV3;x+Pm+p+(YEuZ2RvCoc-wITWVoVSGR<2ek_ztPQ$mVz3v9qphYR`yT6rW@Og8 zI_qwZmsHK7iqUkuZP`>`Kmf*pC(QNV!7 zlro}i_h-YluFwgqYYA&FR68ddme7zH$;IJ859*^eNz}vVm7K%OM7+?EpXP&3>v$Kt_IvJd#$-YD< zm8wo1r4(aU6Bm{^?J#|qlZ)Z%KP5#SNL#8u1f>aUvc$PKf2uNbsB?TtBxI;xn2H7< zOV$`9B7*_;?&T0MMVcpD#0Z><&M#GrF+AbI!68iGCAIE~&_@3J)tPI#T^63>JMF{Q zRJNX6Yr1cW^66K-x^d4snjC7f`Kz|JQy0Lt?S-xQ8~Q59Y7{M8%u*=Et)%AL?adq+ z!SQ2Ov2w9xGX=+7uJ?U0Dt~B;Z0lVwaF!Q+Y_uS?W~{Y`Gr0LeJS270a}EAx~Q2PBjSBb+#E%kowXixVxwq4opx2|{K0WF1`&-sV~j zFnDtam}dU$YTg;+J#P~A!C@VhJ2Gj+CD)?Fc$tGzmpTL$MYa8y2*jwmyPB=fm)>sM^-EWU1s}ggGrT)`)d@4WW<3*iPSukUQ4c{BdXa|?) z1zk0Kp@V8A+URIi)DQV~`p7SpANdhF7W~-6{}4CF=th$lZpvw2NYM`$$mQCv{Vp0K5*hvFhQNByDO0qr1wZi|hZH!FDgBIze=UC+X{}($ zMPm>?cv-3BKQ4VKu-iMXlhYU;r0f7ugD(=V9zq~`ZuaK>4yE>=gd0@)zkc~Y3s73{ ztDgU#rN4#*Dmai%e?CU81Bs5TD6 zQt?|C-(?m=mM9YZ^uGgoZFaGJv_O9MH;+eS65B=TgMU^GH481D1sEUBaAy zlS9x5g!<)+9_o~lt<-fTU>2K_+?%EA(#mKG^r1ypno)%o1+02s2ZtqnSO z!;F$Z(*svdWe|rkduEWS`t$dPsfz`Ezl&#Wl2qPVHN17JC^2#H0E6P6$|qn`1ZhR+ zewm)uOIJ*?c>nI*!|-sGGz4OVp1k|VU3`K%-)jl*=R!Luk`8~Ap%|eG6g^qL1~Ug; z%T!YUv>rq%=l!v?v_yS`i;o!IMbJTwzRE?2(XK*^5+PbhdG=Si$?+bk?Bn+m_*a1ujY6zzAeD83L3F|*)l z-f;wX`uJ;rp`!Lqu(Gf?08b?Yb>Z|jTUcBI!>BfLBbMyZ1 zH#8{37#|(Za{%FPfxoDz=>6be%Tb)mpt%2=8#0jr_wL=33Gh3OivYsIzB)?GJT5so zf-ubnJO=LGK(05Jbtz(m%kD{5`_6y+R!q&~#|!x9fhch8 zt)X`SQJaqbCTu#ZuAzae7QJv`w~0a7XGQ&Eu@zKxoox17IVCg44Xax8+Gm1}rsk0Q zF*df_qpj)ks7HJM074hU)um$ioYQ-Le*VRNCp5ZxXL^;`57DVfadB^Lgn$}6uK@tE zz@UwSp%RdkG$=B`yIa@z^%9+tnAI4}7NF!`Npu6fJ)sUmcC3e>W(Y)eaB`wcJ^_K< zz;OBrU1Dx*9AALz>+75A5H%?rst;1G&#L?r`eeRsION<^l8~Cpl@l-LKBlILRVP0WA55d9nQ`_tFFn!#+rHEl_4y)fX*JuRO$I5RW+ZzK0{|2GhaicX@Z--8V zhlc~A)pL?8Ik8$JxySEkZJpkKvZl>S*8H_yGYZ)l)c(i)d!k|z5;KVk=?0GMf+8Z1 z>Z8U}OHvF6$k>kbR3d8)BN;a*y0}pX0N_>^vc|rPCOr_`^M8iIfPL zXF!p%-Y~}6a!+G#mW*U0zWbx_<@$<_T6##J5^g%me z6N23F0z*1pj6^qNdW585N217W@t61Ed3kjT$9c!-V; z1kU5OQsnHN-+`2%!?LtL3%9tmG$B)zrsQw(su2Xq&gO5MZlT-);Et@8Hww(bCR(HO z=50(W%zB`cYIEw>cd_tPsn>OT#0G^242=--<$!VSswYU@iK8 zq6>ix$?481 zB_%az)#@rOc@G6;eUOVEO3VByOK;Z7$` zG^ESwrz^_+^^Vbv!nXaUWZLH|X?ppu&DAJ<$r6TocPWI=DT*S#)qr&r$pHlj@Q$AR zyw_5`4a&%_`pxJd0UN-)Tz%#Jg^OpDUjB45)vgNmkS1#yYXXB zO@^C+gnPaNIWkiwWzW99B?!QFTlO*wlLA!B}x>e(3PG6*m*(SSEB%>tYG3tE*aAK`s7RUstY!U*{xe z`4HtID!xAYlA)fKmZ^V+f_Cwboz##wJ*I^lj=|JTrw~E`1r5M~v|Y zu~qp84KLvuxL>7Ir_5p*z{zB5MlcO95M_2`IWYcxxS zgnJgjPjL)4m6B2?fQ#if6Jwn(T^_q=f0enPxSINOXjGOXa#@jdBv_iemuCn+`w;6i zmO}62ab$a7U|@E3wv|9VJ9m{C)<^o)Z?aJ?N}o9eyki&Oe!U^S>aB~Rbf+S@pB`BBd1|GxH5ew5SSe|J&NkN?v}IV=AAT1?qNA`p9fm76cay)T 6 and 5 --> 2, and *succ[5]* = {2, 6}. The in-edges of 2 are 5 --> 2 and 1 --> 2, and *pred[2]* = {1, 5}. + +- Uses and Defs + +An assignmemt to a variable or temporary defines that variable. An occurence of a variable on the right-hand side of an assginment(or in other expressions) uses the variable. We can speak the *def* of a variable as the set of graph nodes that define it; or the *def* of a graph node as the set of variables that it defines; and the similarly for the *use* of a variable or graph node. In former control flow graph, *def(3)* = {c}, *use(3)* = {b, c}. + +- Liveness + +A variable is *live* on an edge if there is a directed path from that edge to a *use* of the variable that does not go through any *def*. A variable is *live-in* at a node if it is live on any of the in-edges of that node; it is *live-out* at a node if it is live on any of the out-edges of the node. + + +The calcution of liveness can be solved by iteration until a fixed pointer is reached. Following is the recursive formula: + +![](images/dataflow_equations.png) + +### Memory optimization transpiler + +At last, we take basic strategy and liveness analysis techniques learning from compilers to implement our memory optimization transpiler. + +#### add in-place attribute + +In-place is a built-in attribute of an operator. Since we treat in-place and other operators differently, we have to add an in-place attribute for every operator. + + +#### contruct control flow graph + +Following is the ProgramDesc protobuf of [machine translation](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/tests/book/test_machine_translation.py) example. + +- Block0: + +``` +lookup_table +mul +... +while(sub-block idx 1) +... +array_to_lod_tensor +cross_entropy +... +while_grad(sub-block idx 2) +read_from_array +array_to_lod_tensor +... +``` + +- Block1 + +``` +read_from_array +read_from_array +... +write_to_array +increment +write_to_array +less_than +``` + +- Block2 + +``` +read_from_array +increment +... +write_to_array +write_to_array +``` + +We can transfer all the operators and variables in ProgramDesc to build a control flow graph. + +```python +class ControlFlowGraph(object): + def __init__(self, Program): + self._sucessors = defaultdict(set) + self._presucessors = defaultdict(set) + self._uses = defaultdict(set) + self._defs = defaultdict(set) + self._live_in = defaultdict(set) + self._live_out = defaultdict(set) + self._program = Program + + def build(self): + pass + + def dataflow_analysis(self): + pass + + def memory_optimization(self): + pass + + def get_program(self): + return self._program +``` + +#### make dataflow analysis + +We follow guide from compilers and try to solve the dataflow equation to get liveness of every variable. If the live-in of an operator node is different from the live-out, then we can make memory sharing. + +For example: + +``` +a = op1(b, c); +d = op2(a) +e = op3(d, f) +``` + +The dataflow analysis result is: + +``` +live_in(op1) = {b, c, f} +live_out(op1) = {a, f} + +live_in(op2) = {a, f} +live_out(op2) = {d, f} + +live_in(op3) = {d, f} +live_out(op3) = {} +``` + +After op1, we can process variable b and variable c; After op2, we can process variable a. After op3, we can process variable d and variable f. + +#### memory sharing policy + +A memory pool will be mantained in the stage of memory optimization. Each operator node will be scanned to determine memory optimization is done or not. If an operator satifies the requirement, following policy will be taken to handle input/output variables. + +``` +if op.support_inplace(): + i --> pool + pool --> o +else: + pool --> o + i --> pool +``` + + + +## Reference + +- [Lecture Notes From Artificial Intelligence Is The New Electricity By Andrew Ng](https://manavsehgal.com/lecture-notes-from-artificial-intelligence-is-the-new-electricity-by-andrew-ng-4712dcbf26e5) +- Modern compiler implementation in ML, by Andrew W. Appel +- [Optimizing Memory Consumption in Deep learning](https://mxnet.incubator.apache.org/architecture/note_memory.html) From e5fe8935fb8aedd8b7ecd63136bba56e02dbd96c Mon Sep 17 00:00:00 2001 From: Yancey Date: Fri, 5 Jan 2018 13:41:34 +0800 Subject: [PATCH 35/35] send_recv variables (#7161) * send_recv variable * delete unused logs * fix ci failed * update * resize tensor before tensor copy * add selectedrows unit test * check rows --- paddle/framework/lod_tensor.cc | 7 +- paddle/framework/lod_tensor.h | 3 +- paddle/framework/lod_tensor_test.cc | 2 +- paddle/framework/selected_rows.cc | 6 +- paddle/framework/selected_rows.h | 3 +- paddle/framework/selected_rows_test.cc | 4 +- paddle/framework/tensor_util.h | 57 ++++++--- paddle/framework/tensor_util_test.cc | 6 +- paddle/framework/var_type.h | 4 +- paddle/operators/detail/recv_impl.cc | 19 +-- paddle/operators/detail/send_impl.cc | 18 +-- paddle/operators/detail/send_recv.proto | 18 +-- paddle/operators/detail/send_recv_impl.h | 57 ++++++++- paddle/operators/load_op.cc | 2 +- paddle/operators/recv_op.cc | 13 +- paddle/operators/send_recv_op_test.cc | 148 +++++++++++++++++------ 16 files changed, 249 insertions(+), 118 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 92b3d7fccd..8f6944c241 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -217,9 +217,10 @@ void SerializeToStream(std::ostream &os, const LoDTensor &tensor, SerializeToStream(os, static_cast(tensor), dev_ctx); } -void DeserializeFromStream(std::istream &is, LoDTensor *tensor) { +void DeserializeFromStream(std::istream &is, LoDTensor *tensor, + const platform::DeviceContext &dev_ctx) { { - // the 1st field, unit32_t version for SelectedRows + // the 1st field, unit32_t version for LoDTensor uint32_t version; is.read(reinterpret_cast(&version), sizeof(version)); PADDLE_ENFORCE_EQ(version, 0U, "Only version 0 is supported"); @@ -240,7 +241,7 @@ void DeserializeFromStream(std::istream &is, LoDTensor *tensor) { } } // the 3st filed, Tensor - DeserializeFromStream(is, static_cast(tensor)); + DeserializeFromStream(is, static_cast(tensor), dev_ctx); } } // namespace framework diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index 147db3ab08..d0b6befffe 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -208,7 +208,8 @@ void AppendLoD(LoD* lod, const LoD& lod_length); */ void SerializeToStream(std::ostream& os, const LoDTensor& tensor, const platform::DeviceContext& dev_ctx); -void DeserializeFromStream(std::istream& is, LoDTensor* tensor); +void DeserializeFromStream(std::istream& is, LoDTensor* tensor, + const platform::DeviceContext& dev_ctx); } // namespace framework } // namespace paddle diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 0747c8db53..0868c1f6e6 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -132,7 +132,7 @@ TEST_F(LoDTensorTester, SerializeAndDeserialize) { std::ostringstream oss; SerializeToStream(oss, lod_tensor_, cpu_ctx); std::istringstream iss(oss.str()); - DeserializeFromStream(iss, &dst_tensor); + DeserializeFromStream(iss, &dst_tensor, cpu_ctx); float* dst_ptr = dst_tensor.mutable_data(platform::CPUPlace()); for (int i = 0; i < kLodTensorSize; ++i) { EXPECT_EQ(dst_ptr[i], i); diff --git a/paddle/framework/selected_rows.cc b/paddle/framework/selected_rows.cc index 82adfa7123..3b3e60177a 100644 --- a/paddle/framework/selected_rows.cc +++ b/paddle/framework/selected_rows.cc @@ -37,8 +37,8 @@ void SerializeToStream(std::ostream& os, const SelectedRows& selected_rows, SerializeToStream(os, selected_rows.value(), dev_ctx); } -void DeserializeFromStream(std::istream& is, SelectedRows* selected_rows) { - auto tensor = *selected_rows->mutable_value(); +void DeserializeFromStream(std::istream& is, SelectedRows* selected_rows, + const platform::DeviceContext& dev_ctx) { { // the 1st field, unit32_t version for SelectedRows uint32_t version; @@ -62,7 +62,7 @@ void DeserializeFromStream(std::istream& is, SelectedRows* selected_rows) { selected_rows->set_height(height); } // the 4st field, tensor which contains the data - DeserializeFromStream(is, &tensor); + DeserializeFromStream(is, selected_rows->mutable_value(), dev_ctx); } } // namespace framework diff --git a/paddle/framework/selected_rows.h b/paddle/framework/selected_rows.h index 699e392688..30d3dfc1e8 100644 --- a/paddle/framework/selected_rows.h +++ b/paddle/framework/selected_rows.h @@ -66,7 +66,8 @@ class SelectedRows { */ void SerializeToStream(std::ostream& os, const SelectedRows& selected_rows, const platform::DeviceContext& dev_ctx); -void DeserializeFromStream(std::istream& is, SelectedRows* selected_rows); +void DeserializeFromStream(std::istream& is, SelectedRows* selected_rows, + const platform::DeviceContext& dev_ctx); } // namespace framework } // namespace paddle diff --git a/paddle/framework/selected_rows_test.cc b/paddle/framework/selected_rows_test.cc index 75487c4010..8ff3fb6a97 100644 --- a/paddle/framework/selected_rows_test.cc +++ b/paddle/framework/selected_rows_test.cc @@ -51,10 +51,12 @@ TEST_F(SelectedRowsTester, SerializeAndDeseralize) { SerializeToStream(oss, *selected_rows_, cpu_ctx); std::istringstream iss(oss.str()); - DeserializeFromStream(iss, &dst_tensor); + DeserializeFromStream(iss, &dst_tensor, cpu_ctx); ASSERT_EQ(selected_rows_->rows(), dst_tensor.rows()); ASSERT_EQ(selected_rows_->height(), dst_tensor.height()); + ASSERT_EQ(selected_rows_->value().dims(), dst_tensor.value().dims()); + ASSERT_EQ(selected_rows_->GetCompleteDims(), dst_tensor.GetCompleteDims()); } } // namespace framework diff --git a/paddle/framework/tensor_util.h b/paddle/framework/tensor_util.h index 6a21f8db1e..5ac13cba4d 100644 --- a/paddle/framework/tensor_util.h +++ b/paddle/framework/tensor_util.h @@ -270,7 +270,23 @@ inline void SerializeToStream(std::ostream& os, const Tensor& tensor, } } -inline void DeserializeFromStream(std::istream& is, Tensor* tensor) { +struct DeserializedDataFunctor { + DeserializedDataFunctor(void** buf, Tensor* tensor, + const platform::Place& place) + : buf_(buf), tensor_(tensor), place_(place) {} + + template + void operator()() { + *buf_ = tensor_->mutable_data(place_); + } + + void** buf_; + Tensor* tensor_; + platform::Place place_; +}; + +inline void DeserializeFromStream(std::istream& is, Tensor* tensor, + const platform::DeviceContext& dev_ctx) { uint32_t version; is.read(reinterpret_cast(&version), sizeof(version)); PADDLE_ENFORCE_EQ(version, 0U, "Only version 0 is supported"); @@ -289,27 +305,28 @@ inline void DeserializeFromStream(std::istream& is, Tensor* tensor) { dims.reserve(static_cast(desc.dims().size())); std::copy(desc.dims().begin(), desc.dims().end(), std::back_inserter(dims)); tensor->Resize(framework::make_ddim(dims)); - void* buf; - platform::Place cpu = platform::CPUPlace(); - // TODO(Yancey1989): use VisiterDataType instead of DataType switch - switch (desc.data_type()) { - case proto::FP32: - buf = tensor->mutable_data(cpu); - break; - case proto::FP64: - buf = tensor->mutable_data(cpu); - break; - case proto::INT32: - buf = tensor->mutable_data(cpu); - break; - case proto::INT64: - buf = tensor->mutable_data(cpu); - break; - default: - PADDLE_THROW("DataType %d not supported", desc.data_type()); + auto ctx = platform::CPUDeviceContext(); + if (platform::is_gpu_place(dev_ctx.GetPlace())) { +#ifdef PADDLE_WITH_CUDA + Tensor cpu_tensor; + cpu_tensor.Resize(framework::make_ddim(dims)); + framework::VisitDataType( + desc.data_type(), + DeserializedDataFunctor(&buf, &cpu_tensor, ctx.GetPlace())); + is.read(static_cast(buf), cpu_tensor.memory_size()); + auto cpu_place = new platform::CPUPlace(); + framework::CopyFrom(cpu_tensor, *cpu_place, dev_ctx, tensor); + delete cpu_place; +#else + PADDLE_THROW("Unexpected branch"); +#endif + } else { + framework::VisitDataType( + desc.data_type(), + DeserializedDataFunctor(&buf, tensor, ctx.GetPlace())); + is.read(static_cast(buf), tensor->memory_size()); } - is.read(static_cast(buf), tensor->memory_size()); } } diff --git a/paddle/framework/tensor_util_test.cc b/paddle/framework/tensor_util_test.cc index 0dc5166fca..15cd2bd09c 100644 --- a/paddle/framework/tensor_util_test.cc +++ b/paddle/framework/tensor_util_test.cc @@ -270,11 +270,12 @@ TEST(Tensor, SerializeAndDeserialize) { SerializeToStream(oss, src_tensor, cpu_ctx); std::istringstream iss(oss.str()); - DeserializeFromStream(iss, &dst_tensor); + DeserializeFromStream(iss, &dst_tensor, cpu_ctx); int* dst_ptr = dst_tensor.mutable_data(platform::CPUPlace()); for (int i = 0; i < 5; ++i) { ASSERT_EQ(dst_ptr[i], array[i]); } + ASSERT_EQ(dst_tensor.dims(), src_tensor.dims()); delete place; } #ifdef PADDLE_WITH_CUDA @@ -292,13 +293,12 @@ TEST(Tensor, SerializeAndDeserialize) { SerializeToStream(oss, gpu_tensor, gpu_ctx); std::istringstream iss(oss.str()); - DeserializeFromStream(iss, &dst_tensor); + DeserializeFromStream(iss, &dst_tensor, gpu_ctx); int* dst_ptr = dst_tensor.mutable_data(platform::CPUPlace()); for (int i = 0; i < 6; ++i) { ASSERT_EQ(dst_ptr[i], array[i]); } - delete gpu_place; } #endif diff --git a/paddle/framework/var_type.h b/paddle/framework/var_type.h index 0e6ea8dc69..5b7a08a087 100644 --- a/paddle/framework/var_type.h +++ b/paddle/framework/var_type.h @@ -17,6 +17,8 @@ limitations under the License. */ #include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/selected_rows.h" +#include "paddle/framework/variable.h" namespace paddle { namespace framework { @@ -35,7 +37,7 @@ inline proto::VarDesc::VarType ToVarType(std::type_index type) { } template -inline void VisitVarType(const Variable& var, Visitor visitor) { +inline void VisitVarType(const framework::Variable& var, Visitor visitor) { switch (ToVarType(var.Type())) { case proto::VarDesc_VarType_LOD_TENSOR: visitor(var.Get()); diff --git a/paddle/operators/detail/recv_impl.cc b/paddle/operators/detail/recv_impl.cc index b746f9df46..319404e56a 100644 --- a/paddle/operators/detail/recv_impl.cc +++ b/paddle/operators/detail/recv_impl.cc @@ -21,14 +21,9 @@ namespace detail { Status SendRecvServerImpl::SendVariable(ServerContext *context, const VariableMessage *in_var, VoidMessage *out_var) { - // TODO(typhoonzero): support different variable types. - std::istringstream iss(in_var->serialized()); - framework::LoDTensor t; - framework::DeserializeFromStream(iss, &t); - TensorWithName tensor_with_name = - std::make_pair(in_var->varname(), std::move(t)); - - var_recv_queue_.Push(std::move(tensor_with_name)); + MessageWithName msg_with_name = + std::make_pair(in_var->varname(), std::move(*in_var)); + var_recv_queue_.Push(std::move(msg_with_name)); return Status::OK; } @@ -37,14 +32,8 @@ Status SendRecvServerImpl::GetVariable(ServerContext *context, VariableMessage *out_var) { std::string get_var_name = in_var->varname(); auto *var = scope_->FindVar(get_var_name); - auto tensor = var->Get(); - std::ostringstream oss; - framework::SerializeToStream(oss, tensor, platform::CPUDeviceContext()); - std::string *varname = out_var->mutable_varname(); - *varname = get_var_name; - std::string *serialized = out_var->mutable_serialized(); - *serialized = oss.str(); + SerializeToMessage(get_var_name, var, platform::CPUDeviceContext(), out_var); return Status::OK; } diff --git a/paddle/operators/detail/send_impl.cc b/paddle/operators/detail/send_impl.cc index a812fcf39b..ae85cf2cec 100644 --- a/paddle/operators/detail/send_impl.cc +++ b/paddle/operators/detail/send_impl.cc @@ -27,14 +27,8 @@ bool RPCClient::SendVariable(const framework::Scope& scope, auto ctx = platform::CPUDeviceContext(); auto* var = scope.FindVar(inname); PADDLE_ENFORCE(var); - // TODO(typhoonzero): support SelectedRows - PADDLE_ENFORCE(var->IsType(), - "Only support LoDTensor, %s has wrong type", inname); - const framework::LoDTensor& tensor = var->Get(); - std::ostringstream oss; - framework::SerializeToStream(oss, tensor, ctx); - msg.set_varname(inname); - msg.set_serialized(oss.str()); + SerializeToMessage(inname, var, ctx, &msg); + Status status = stub_->SendVariable(&context, msg, &out_msg); if (!status.ok()) { LOG(ERROR) << "gRPC error: " << status.error_message(); @@ -50,19 +44,15 @@ bool RPCClient::GetVariable(const framework::Scope& scope, call_msg.set_varname(outname); auto ctx = platform::CPUDeviceContext(); Status status = stub_->GetVariable(&context, call_msg, &ret_msg); + auto* outvar = scope.FindVar(outname); if (!status.ok()) { LOG(ERROR) << "gRPC error: " << status.error_message(); return false; } std::istringstream iss(ret_msg.serialized()); + DeserializeFromMessage(ret_msg, ctx, outvar); - framework::LoDTensor ret_tensor; - framework::DeserializeFromStream(iss, &ret_tensor); - auto* outvar = scope.FindVar(outname); - framework::LoDTensor* out_tensor = outvar->GetMutable(); - // FIXME(typhoonzero): do not copy. - framework::CopyFrom(ret_tensor, ctx.GetPlace(), ctx, out_tensor); return true; } diff --git a/paddle/operators/detail/send_recv.proto b/paddle/operators/detail/send_recv.proto index 95c8e70898..f141c755ce 100644 --- a/paddle/operators/detail/send_recv.proto +++ b/paddle/operators/detail/send_recv.proto @@ -1,7 +1,6 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. Licensed under +the Apache License, Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 @@ -13,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ syntax = "proto3"; - package sendrecv; service SendRecvService { @@ -29,12 +27,18 @@ service SendRecvService { // VariableMessage is serialized paddle variable message. // It can be: -// Tensor // LoDTensor // SelectedRows +enum VarType { + LOD_TENSOR = 0; + SELECTED_ROWS = 1; +} + message VariableMessage { string varname = 1; - bytes serialized = 2; + // TODO(Yancey1989): reference framework::proto::VarDesc::VarType + VarType type = 2; + bytes serialized = 3; } message VoidMessage {} diff --git a/paddle/operators/detail/send_recv_impl.h b/paddle/operators/detail/send_recv_impl.h index 47f730f7ae..1fe54f1f05 100644 --- a/paddle/operators/detail/send_recv_impl.h +++ b/paddle/operators/detail/send_recv_impl.h @@ -14,10 +14,10 @@ limitations under the License. */ #pragma once -#include "paddle/framework/data_type.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/scope.h" #include "paddle/framework/selected_rows.h" +#include "paddle/framework/var_type.h" #include "paddle/operators/detail/simple_block_queue.h" #include "paddle/operators/detail/send_recv.grpc.pb.h" @@ -44,7 +44,7 @@ namespace paddle { namespace operators { namespace detail { -typedef std::pair TensorWithName; +typedef std::pair MessageWithName; class SendRecvServerImpl final : public SendRecvService::Service { public: @@ -60,13 +60,13 @@ class SendRecvServerImpl final : public SendRecvService::Service { void Done(); void SetScope(framework::Scope *scope) { scope_ = scope; }; - const TensorWithName Get() { return this->var_recv_queue_.Pop(); } + const MessageWithName Get() { return this->var_recv_queue_.Pop(); } - void Push(const TensorWithName &msg) { this->var_recv_queue_.Push(msg); } + void Push(const MessageWithName &msg) { this->var_recv_queue_.Push(msg); } private: // received variable from RPC, operators fetch variable from this queue. - SimpleBlockQueue var_recv_queue_; + SimpleBlockQueue var_recv_queue_; framework::Scope *scope_; // condition of the sub program std::mutex mutex_; @@ -89,6 +89,53 @@ class RPCClient { std::unique_ptr stub_; }; +inline void SerializeToMessage(const std::string &name, + const framework::Variable *var, + const platform::DeviceContext &ctx, + VariableMessage *msg) { + msg->set_varname(name); + std::ostringstream oss; + switch (framework::ToVarType(var->Type())) { + case framework::proto::VarDesc_VarType_LOD_TENSOR: + msg->set_type(sendrecv::VarType::LOD_TENSOR); + framework::SerializeToStream(oss, var->Get(), ctx); + break; + case framework::proto::VarDesc_VarType_SELECTED_ROWS: + msg->set_type(sendrecv::VarType::SELECTED_ROWS); + framework::SerializeToStream(oss, var->Get(), + ctx); + break; + default: { + PADDLE_THROW("Serialize does not support type: %s", + typeid(var->Type()).name()); + break; + } + } + msg->set_serialized(oss.str()); +} + +inline void DeserializeFromMessage(const VariableMessage &msg, + const platform::DeviceContext &ctx, + framework::Variable *var) { + using namespace paddle::framework::proto; + std::istringstream iss(msg.serialized()); + switch (msg.type()) { + case sendrecv::VarType::LOD_TENSOR: + DeserializeFromStream(iss, var->GetMutable(), ctx); + break; + case sendrecv::VarType::SELECTED_ROWS: { + DeserializeFromStream(iss, var->GetMutable(), + ctx); + break; + } + default: { + PADDLE_THROW("Deserialize does not support type: %s", + typeid(var->Type()).name()); + break; + } + } +} + } // namespace detail } // namespace operators } // namespace paddle diff --git a/paddle/operators/load_op.cc b/paddle/operators/load_op.cc index 08b972a233..7f551f101f 100644 --- a/paddle/operators/load_op.cc +++ b/paddle/operators/load_op.cc @@ -38,10 +38,10 @@ class LoadOp : public framework::OperatorBase { out_var_name); auto *tensor = out_var->GetMutable(); - DeserializeFromStream(fin, tensor); platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); auto &dev_ctx = *pool.Get(place); + DeserializeFromStream(fin, tensor, dev_ctx); if (platform::is_gpu_place(place)) { // copy CPU to GPU diff --git a/paddle/operators/recv_op.cc b/paddle/operators/recv_op.cc index 322f8571cf..82fceb3da7 100644 --- a/paddle/operators/recv_op.cc +++ b/paddle/operators/recv_op.cc @@ -60,7 +60,7 @@ class RecvOp : public framework::OperatorBase { } void Stop() override { - detail::TensorWithName term_msg; + detail::MessageWithName term_msg; term_msg.first = LISTEN_TERMINATE_MESSAGE; rpc_service_->Push(term_msg); rpc_server_->Shutdown(); @@ -94,7 +94,7 @@ class RecvOp : public framework::OperatorBase { // the gradient arrives, just add suffix 0~n then average the gradient. for (size_t i = 0; i < param_count * trainer_count; ++i) { // blocking get one var from client. - const detail::TensorWithName &v = rpc_service_->Get(); + const detail::MessageWithName &v = rpc_service_->Get(); auto grad_var_name = v.first; if (grad_var_name == LISTEN_TERMINATE_MESSAGE) { exit_flag = true; @@ -121,11 +121,10 @@ class RecvOp : public framework::OperatorBase { } auto *var = recv_scope.Var(grad_var_name); - auto *tensor = var->GetMutable(); - // FIXME(typhoonzero): do not copy - platform::DeviceContextPool &pool = platform::DeviceContextPool::Get(); - auto &dev_ctx = *pool.Borrow(dev_place); - framework::CopyFrom(v.second, dev_place, dev_ctx, tensor); + platform::DeviceContextPool &pool = + platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(dev_place); + detail::DeserializeFromMessage(v.second, dev_ctx, var); } if (exit_flag) { break; diff --git a/paddle/operators/send_recv_op_test.cc b/paddle/operators/send_recv_op_test.cc index 108e2dec6b..fa94424bf9 100644 --- a/paddle/operators/send_recv_op_test.cc +++ b/paddle/operators/send_recv_op_test.cc @@ -20,22 +20,27 @@ limitations under the License. */ #include "paddle/framework/op_registry.h" #include "paddle/framework/operator.h" #include "paddle/framework/program_desc.h" +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/selected_rows_functor.h" #include "paddle/string/printf.h" USE_NO_KERNEL_OP(send); USE_NO_KERNEL_OP(recv); USE_OP(sum); +namespace f = paddle::framework; +namespace p = paddle::platform; +namespace m = paddle::operators::math; + // global for simplicity. -std::unique_ptr recv_op; +std::unique_ptr recv_op; -void InitTensorsInScope(paddle::framework::Scope &scope, - paddle::platform::CPUPlace &place) { - paddle::platform::CPUDeviceContext ctx(place); +void InitTensorsInScope(f::Scope &scope, p::CPUPlace &place) { + p::CPUDeviceContext ctx(place); for (int i = 0; i < 2; ++i) { auto var_name = paddle::string::Sprintf("x%d", i); auto var = scope.Var(var_name); - auto tensor = var->GetMutable(); + auto tensor = var->GetMutable(); tensor->Resize({10, 10}); float *expect = tensor->mutable_data(place); for (int64_t i = 0; i < tensor->numel(); ++i) { @@ -44,21 +49,53 @@ void InitTensorsInScope(paddle::framework::Scope &scope, } auto out_var = scope.Var("Out"); - auto out_tensor = out_var->GetMutable(); + auto out_tensor = out_var->GetMutable(); out_tensor->Resize({10, 10}); out_tensor->mutable_data(place); // allocate } -void AddOp(const std::string &type, - const paddle::framework::VariableNameMap &inputs, - const paddle::framework::VariableNameMap &outputs, - paddle::framework::AttributeMap attrs, - paddle::framework::BlockDesc *block) { +void InitSelectedRowsInScope(f::Scope &scope, p::CPUPlace &place) { + p::CPUDeviceContext ctx(place); + int64_t height = 10; + int64_t row_numel = 10; + m::SetConstant set_one; + // init x0 + std::vector rows0{0, 4, 7}; + auto x0_var = scope.Var("x0"); + auto x0 = x0_var->GetMutable(); + x0->set_rows(rows0); + x0->set_height(height); + auto x0_value = x0->mutable_value(); + x0_value->mutable_data( + f::make_ddim({static_cast(rows0.size()), row_numel}), place); + set_one(ctx, x0_value, 1.0); + + // init x1 + std::vector rows1{2, 9}; + auto x1_var = scope.Var("x1"); + auto x1 = x1_var->GetMutable(); + x1->set_rows(rows1); + x1->set_height(height); + auto x1_value = x1->mutable_value(); + x1_value->mutable_data( + f::make_ddim({static_cast(rows1.size()), row_numel}), place); + set_one(ctx, x1_value, 1.0); + + auto out_var = scope.Var("Out"); + auto out = out_var->GetMutable(); + auto out_value = out->mutable_value(); + out->set_height(height); + out_value->mutable_data(f::make_ddim({5, 10}), place); +} + +void AddOp(const std::string &type, const f::VariableNameMap &inputs, + const f::VariableNameMap &outputs, f::AttributeMap attrs, + f::BlockDesc *block) { // insert output for (auto kv : outputs) { for (auto v : kv.second) { auto var = block->Var(v); - var->SetDataType(paddle::framework::proto::DataType::FP32); + var->SetDataType(f::proto::DataType::FP32); } } @@ -74,58 +111,99 @@ void AddOp(const std::string &type, op->SetAttrMap(attrs); } -void StartServerNet() { - paddle::framework::Scope scope; - paddle::platform::CPUPlace place; - InitTensorsInScope(scope, place); +void StartServerNet(bool is_sparse) { + f::Scope scope; + p::CPUPlace place; + if (is_sparse) { + InitSelectedRowsInScope(scope, place); + } else { + InitTensorsInScope(scope, place); + } // sub program run in recv_op, for simple test we use sum - paddle::framework::ProgramDesc program; - paddle::framework::BlockDesc *block = program.MutableBlock(0); + f::ProgramDesc program; + f::BlockDesc *block = program.MutableBlock(0); // X for server side tensors, RX for received tensers, must be of same shape. - AddOp("sum", {{"X", {"x0", "x1"}}}, {{"Out", {"x0"}}}, {}, block); + AddOp("sum", {{"X", {"x0", "x1"}}}, {{"Out", {"Out"}}}, {}, block); - paddle::framework::AttributeMap attrs; + f::AttributeMap attrs; attrs.insert({"endpoint", std::string("127.0.0.1:6174")}); - attrs.insert({"ParamList", std::vector({"x0"})}); + attrs.insert({"ParamList", std::vector({"Out"})}); attrs.insert({"GradList", std::vector({"x1"})}); std::string program_proto; PADDLE_ENFORCE(program.Proto()->SerializeToString(&program_proto)); attrs.insert({"OptimizeProgram", program_proto}); - recv_op = paddle::framework::OpRegistry::CreateOp("recv", {{"RX", {"x1"}}}, - {}, attrs); + recv_op = f::OpRegistry::CreateOp("recv", {{"RX", {"x1"}}}, {}, attrs); recv_op->Run(scope, place); } -TEST(SendRecvOp, CPU) { - std::thread server_thread(StartServerNet); - sleep(5); // wait server to start +TEST(SendRecvOp, CPUDense) { + std::thread server_thread(StartServerNet, false); + sleep(3); // wait server to start // local net - paddle::framework::Scope scope; - paddle::platform::CPUPlace place; + f::Scope scope; + p::CPUPlace place; InitTensorsInScope(scope, place); - paddle::framework::AttributeMap attrs; + f::AttributeMap attrs; attrs.insert({"endpoints", std::vector({"127.0.0.1:6174"})}); attrs.insert({"epmap", std::vector({"127.0.0.1:6174"})}); - auto send_op = paddle::framework::OpRegistry::CreateOp( - "send", {{"X", {"x1"}}}, {{"Out", {"x0"}}}, attrs); + auto send_op = f::OpRegistry::CreateOp("send", {{"X", {"x1"}}}, + {{"Out", {"Out"}}}, attrs); send_op->Run(scope, place); auto in_var = scope.Var("x1"); - auto tensor = in_var->GetMutable(); + auto tensor = in_var->GetMutable(); float *expected = tensor->data(); - auto out_var = scope.Var("x0"); - auto target = out_var->GetMutable(); + auto out_var = scope.Var("Out"); + auto target = out_var->GetMutable(); // x1 * 2 == x0 EXPECT_NE(target->memory_size(), size_t(0)); float *actual = target->data(); for (int64_t i = 0; i < target->numel(); ++i) { EXPECT_EQ(expected[i] * 2, actual[i]); } + recv_op->Stop(); + server_thread.join(); + recv_op.reset(nullptr); +} +TEST(SendRecvOp, CPUSparse) { + std::thread server_thread(StartServerNet, true); + sleep(3); // wait server to start + // local net + f::Scope scope; + p::CPUPlace place; + p::CPUDeviceContext ctx(place); + InitSelectedRowsInScope(scope, place); + f::AttributeMap attrs; + attrs.insert({"endpoints", std::vector({"127.0.0.1:6174"})}); + attrs.insert({"epmap", std::vector({"127.0.0.1:6174"})}); + auto send_op = f::OpRegistry::CreateOp("send", {{"X", {"x1"}}}, + {{"Out", {"Out"}}}, attrs); + send_op->Run(scope, place); + + auto x0 = scope.Var("x0")->GetMutable(); + auto x1 = scope.Var("x1")->GetMutable(); + auto out = scope.Var("Out")->GetMutable(); + auto actual = out->mutable_value(); + + std::unique_ptr expect{new f::SelectedRows()}; + auto expect_value = expect->mutable_value(); + expect_value->mutable_data(f::make_ddim({5, 10}), place); + + m::SelectedRowsAdd add_functor; + add_functor(ctx, *x0, *x1, expect.get()); + + EXPECT_EQ(actual->numel(), expect_value->numel()); + EXPECT_EQ(out->rows().size(), x0->rows().size() + x1->rows().size()); + + for (int64_t i = 0; i < expect_value->numel(); ++i) { + EXPECT_EQ(expect_value->mutable_data(place)[i], + actual->mutable_data(place)[i]); + } recv_op->Stop(); server_thread.join(); - // recv_op.reset(); + recv_op.reset(); }