From daea7b2956d181d9ae53922183a6c3f4aa3771a8 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Fri, 9 Dec 2016 15:47:58 +0800 Subject: [PATCH 01/48] fix dead link for quick start --- doc/demo/quick_start/index_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index 659485d9be..6e2056b69f 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -164,7 +164,7 @@ You will describe four kinds of network architectures in this section.
![](./PipelineNetwork_en.jpg)
First, you will build a logistic regression model. Later, you will also get chance to build other more powerful network architectures. -For more detailed documentation, you could refer to: Layer documentation。All configuration files are in `demo/quick_start` directory. +For more detailed documentation, you could refer to: Layer documentation。All configuration files are in `demo/quick_start` directory. ### Logistic Regression The architecture is illustrated in the following picture: From af7f407ecf468c0b28d5969e2a26d01511f09af2 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Mon, 12 Dec 2016 11:58:56 +0800 Subject: [PATCH 02/48] update --- doc/demo/quick_start/index_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index 6e2056b69f..c2d6f3a6b6 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -164,7 +164,7 @@ You will describe four kinds of network architectures in this section.
![](./PipelineNetwork_en.jpg)
First, you will build a logistic regression model. Later, you will also get chance to build other more powerful network architectures. -For more detailed documentation, you could refer to: Layer documentation。All configuration files are in `demo/quick_start` directory. +For more detailed documentation, you could refer to: layer documentation. All configuration files are in `demo/quick_start` directory. ### Logistic Regression The architecture is illustrated in the following picture: From 88b7ed8f16f0f76a5d1b5256b9af9de7bbd16bc1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 14 Dec 2016 16:21:56 +0800 Subject: [PATCH 03/48] Fix Travis-CI build for release --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ffe3bc193b..a30857b600 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ before_install: fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - - pip install wheel protobuf sphinx breathe recommonmark virtualenv numpy + - pip install wheel protobuf 'sphinx==1.4.9' breathe recommonmark virtualenv numpy script: - paddle/scripts/travis/main.sh notifications: From 576176d4b232cb4cf17b0ecddefc21cafbbeb203 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 14 Dec 2016 16:24:45 +0800 Subject: [PATCH 04/48] Remove typo in documentation. --- doc/ui/data_provider/pydataprovider2.rst | 8 ++++---- doc/ui/index.md | 2 +- doc_cn/ui/data_provider/pydataprovider2.rst | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/ui/data_provider/pydataprovider2.rst b/doc/ui/data_provider/pydataprovider2.rst index e105d3be30..fcf230522f 100644 --- a/doc/ui/data_provider/pydataprovider2.rst +++ b/doc/ui/data_provider/pydataprovider2.rst @@ -174,12 +174,12 @@ input_types +++++++++++ PaddlePaddle has four data types, and three sequence types. -The four data types are: +The four data types are: * :code:`dense_vector`: dense float vector. * :code:`sparse_binary_vector`: sparse binary vector, most of the value is 0, and the non zero elements are fixed to 1. -* :code:`sparse_float_vector`: sparse float vector, most of the value is 0, and some +* :code:`sparse_vector`: sparse float vector, most of the value is 0, and some non zero elements can be any float value. They are given by the user. * :code:`integer`: an integer scalar, that is especially used for label or word index. @@ -200,7 +200,7 @@ in the above table. +----------------------+---------------------+-----------------------------------+------------------------------------------------+ | sparse_binary_vector | [i, i, ...] | [[i, ...], [i, ...], ...] | [[[i, ...], ...], [[i, ...], ...],...] | +----------------------+---------------------+-----------------------------------+------------------------------------------------+ -| sparse_float_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | +| sparse_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | +----------------------+---------------------+-----------------------------------+------------------------------------------------+ | integer_value | i | [i, i, ...] | [[i, ...], [i, ...], ...] | +----------------------+---------------------+-----------------------------------+------------------------------------------------+ @@ -227,7 +227,7 @@ Its parameters lists as follows: * :code:`is_train` is a bool parameter that indicates the DataProvider is used in training or testing. * :code:`file_list` is the list of all files. - + * User-defined parameters args can be set in training configuration. Note, PaddlePaddle reserves the right to add pre-defined parameter, so please diff --git a/doc/ui/index.md b/doc/ui/index.md index 9c1ba27bdc..5aa051b4e9 100644 --- a/doc/ui/index.md +++ b/doc/ui/index.md @@ -7,7 +7,7 @@ ## API Reference -* [Model Config Interface](api/trainer_config_helpers/index.md) +* [Model Config Interface](api/trainer_config_helpers/index.rst) ## Command Line Argument diff --git a/doc_cn/ui/data_provider/pydataprovider2.rst b/doc_cn/ui/data_provider/pydataprovider2.rst index 80b40084d8..34db4a97d9 100644 --- a/doc_cn/ui/data_provider/pydataprovider2.rst +++ b/doc_cn/ui/data_provider/pydataprovider2.rst @@ -53,7 +53,7 @@ process函数调用多次 :code:`yield` 即可。 :code:`yield` 是Python的一 .. literalinclude:: mnist_config.py -这里说明了训练数据是 'train.list'，而没有测试数据。引用的DataProvider是 'mnist_provider' +这里说明了训练数据是 'train.list'，而没有测试数据。引用的DataProvider是 'mnist_provider' 这个模块中的 'process' 函数。 同时，根据模型配置文件中 :code:`data_layer` 的名字，用户也可以显式指定返回的数据对应关系。例如: @@ -152,7 +152,7 @@ PaddlePaddle的数据包括四种主要类型，和三种序列模式。其中 * dense_vector 表示稠密的浮点数向量。 * sparse_binary_vector 表示稀疏的零一向量，即大部分值为0，有值的位置只能取1 -* sparse_float_vector 表示稀疏的向量，即大部分值为0，有值的部分可以是任何浮点数 +* sparse_vector 表示稀疏的向量，即大部分值为0，有值的部分可以是任何浮点数 * integer 表示整数标签。 而三种序列模式为 @@ -170,7 +170,7 @@ PaddlePaddle的数据包括四种主要类型，和三种序列模式。其中 +----------------------+---------------------+-----------------------------------+------------------------------------------------+ | sparse_binary_vector | [i, i, ...] | [[i, ...], [i, ...], ...] | [[[i, ...], ...], [[i, ...], ...],...] | +----------------------+---------------------+-----------------------------------+------------------------------------------------+ -| sparse_float_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | +| sparse_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | +----------------------+---------------------+-----------------------------------+------------------------------------------------+ | integer_value | i | [i, i, ...] | [[i, ...], [i, ...], ...] | +----------------------+---------------------+-----------------------------------+------------------------------------------------+ @@ -202,7 +202,7 @@ DataProvider提供了两种简单的Cache策略。他们是 * CacheType.NO_CACHE 不缓存任何数据，每次都会从python端读取数据 * CacheType.CACHE_PASS_IN_MEM 第一个pass会从python端读取数据，剩下的pass会直接从内存里 - 读取数据。 + 读取数据。 注意事项 From d1d50412209836624d6220ef9825f417af75cac4 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Mon, 19 Dec 2016 16:32:45 +0800 Subject: [PATCH 05/48] fix typo --- doc_cn/build_and_install/install/docker_install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc_cn/build_and_install/install/docker_install.rst b/doc_cn/build_and_install/install/docker_install.rst index a5f5fb117e..8298b79bf7 100644 --- a/doc_cn/build_and_install/install/docker_install.rst +++ b/doc_cn/build_and_install/install/docker_install.rst @@ -67,7 +67,7 @@ mac osx或者是windows机器，请参考 .. code-block:: bash - $ docker run -it paddledev/paddlepaddle:cpu-latest + $ docker run -it paddledev/paddle:cpu-latest 即可启动和进入PaddlePaddle的container。如果运行GPU版本的PaddlePaddle，则需要先将 cuda相关的Driver和设备映射进container中，脚本类似于 From 9572e11d73ede8040d6495f8e588b61fb9ea20ed Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Mon, 1 May 2017 18:27:28 -0700 Subject: [PATCH 06/48] Design doc: master process --- doc/design/cluster_train/data_dispatch.md | 6 +- doc/design/cluster_train/master_process.md | 89 +++++++++++++++++++ doc/design/cluster_train/src/dataset.graffle | Bin 0 -> 2770 bytes doc/design/cluster_train/src/dataset.png | Bin 0 -> 10845 bytes 4 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 doc/design/cluster_train/master_process.md create mode 100644 doc/design/cluster_train/src/dataset.graffle create mode 100644 doc/design/cluster_train/src/dataset.png diff --git a/doc/design/cluster_train/data_dispatch.md b/doc/design/cluster_train/data_dispatch.md index a3eb4e28db..f60c3b843d 100644 --- a/doc/design/cluster_train/data_dispatch.md +++ b/doc/design/cluster_train/data_dispatch.md @@ -21,7 +21,7 @@ ### 文件预处理 -在数据集可以被训练之前，文件需要预先被转换成PaddlePaddle集群内部的存储格式（SSTable）。我们提供两个转换方式： +在数据集可以被训练之前，文件需要预先被转换成PaddlePaddle集群内部的存储格式（RecordIO）。我们提供两个转换方式： - 提供给用户本地转换的库，用户可以编写程序完成转换。 - 用户可以上传自己的数据集，在集群运行MapReduce job完成转换。 @@ -92,11 +92,11 @@ random_images-00099-of-00099 #### 进行训练 -PaddlePaddle提供专用的[data reader creator](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/reader/README.md#python-data-reader-design-doc)，生成给定SSTable文件对应的data reader。**无论在本地还是在云端，reader的使用方式都是一致的**： +PaddlePaddle提供专用的[data reader creator](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/reader/README.md#python-data-reader-design-doc)，生成给定RecordIO文件对应的data reader。**无论在本地还是在云端，reader的使用方式都是一致的**： ```python # ... -reader = paddle.reader.creator.SSTable("/home/random_images-*-of-*") +reader = paddle.reader.creator.RecordIO("/home/random_images-*-of-*") batch_reader = paddle.batch(paddle.dataset.mnist.train(), 128) trainer.train(batch_reader, ...) ``` diff --git a/doc/design/cluster_train/master_process.md b/doc/design/cluster_train/master_process.md new file mode 100644 index 0000000000..949811b4f7 --- /dev/null +++ b/doc/design/cluster_train/master_process.md @@ -0,0 +1,89 @@ +# Design Doc: Master Process + +For an overview of master process' role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master process in more details. The master will be implemented in [golang](https://golang.org/). + +## Dataset + + + +A dataset is represented by a list of files in *RecordIO* format on the distributed filesystem, each RecordIO file consists of multiple *blocks*, and each block has multiple data instances. + +## Task Queue + +As mentioned in [distributed training design doc](./README.md), a *task* is a data shard that the master process assigns to the trainer process to train on. A task consists of one or multiple *blocks* from one or multiple files. The master process maintains *task queues* to track the training progress. + +### Task Queue Creation + +1. Each trainer will make an RPC call (using [golang rpc](https://golang.org/pkg/net/rpc/)) to the master process, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master process the same dataset, only the first RPC call will be honored. + + The RPC interface is: + ```go + func (m *RPCServer) ReportDataset(Paths []string, dummy *int) error { + } + ``` +1. The master process will scan through each RecordIO file to generate the *block index* and know how many blocks does each file have. A block can be referenced by the file path and the index of the block within the file. The block index is in memory data structure that enables fast access to each block, and the index of the block with the file is an integer start from 0, representing the n-th block within the file. + + The definition of the block is: + ```go + type Block struct { + Idx int // index of the block within the file + Path string + Index recordio.Index // block index + } + ``` +1. Blocks are grouped into tasks, and tasks are filled into the todo queue. The pending queue and the done queue are initialized with no element. + + The definition of the task is: + ```go + type Task struct { + Index int + Blocks []Block + } + ``` + + The elements in the tasks queues is of type `TaskEntry`, containing a timeout counter (described in [task retry logic](#task-retry-logic)), and a task: + ```go + type TaskEntry struct { + NumTimeout int + Task Task + } + ``` + + The definition of task queues is: + ```go + type TaskQueues struct { + Todo []TaskEntry + Pending map[int]TaskEntry // map from task index to task entry + Done []TaskEntry + } + ``` + +### Task Queue Persistence + +The task queues need to be persisted on [etcd](https://github.com/coreos/etcd) for fault recovery. Since the task queues only change once a task is completed or timed out, which is not very frequent, we can afford to synchronize with etcd every time the task queues change. + +We will serialize the task queues data structure with [gob encoding](https://golang.org/pkg/encoding/gob/), compress with gzip, and save into etcd synchronously under key `/task_queues`. + +### Task Dispatch + +The trainer will make an RPC call to master to get a new task when: + +- the trainer first started, or +- the trainer finishes a task. + +The RPC interface is: +```go +func (m *RPCServer) GetTask(finished *Task, result *Task) error { +} +``` +Argument `finished` will be `nil` when the trainer is just started. + +During the RPC call the master will do the following: + +- Make a copy of the task queues, and update the copy reflecting the finished tasks and the new pending tasks. +- Synchronize the copy of task queues with etcd using a transaction conditioned on holding the master lock. +- Replace the task queues with the copy and report to the trainer with the new tasks if succeeded, or discard the copy and report the error to the trainer if failed. + +### Task Retry Logic + +When a task is dispatched to the trainer, the master will schedule a function for execution after the timeout duration (based on the moving average of task completion time). If the task entry in still in the pending queue, its timeout counter will increase by one, and the task will be moved to todo queue. If the timeout counter is above the threshold, the master will log the error and discard the task. diff --git a/doc/design/cluster_train/src/dataset.graffle b/doc/design/cluster_train/src/dataset.graffle new file mode 100644 index 0000000000000000000000000000000000000000..c10a423ed16a23229a9ee33d11bfc82bb59646c8 GIT binary patch literal 2770 zcmV;@3N7^?iwFP!000030PS3BbK1xf{T%-at-r1fk^lo)$6F)bj*ZRB1d6I#gESx+ zJ%}WXNy`8I_UHk6fyvG0+NqVRj2?4(rl&Q1x;4gq`S-J?GEc;`6hr@X%4OJ7hUlUp zDf;}=>7ZXfFP;8GmOJdra8|dsD1TvB>ox(6~?;&;IuB3?e z)yap;1=(C7(N>)JZR4sbLDbAfVj;1J-ZsL zYy(M;E6a|l%N1h!3Rqmw@pTEtUdoEN74i(>!6?G=+qOD=|9;M~Jb%WVm-yn#8S_2I zUR<0pe7?lJyo4q}1uH7H;(hdFyd=BPBQmY(PQX{456;4c_3VvGc1T}$w-LHD7Q5C! zv)f)Z@p`A%PP6u1P#v)I)+GMf?!>jFMaG7q9kEO*SbxISovF-~-2(BA%APT7(0T!h zp7M>v@a`j^Paq~_s4GoJsxsH(iy^L*gYGS{2wiOPCc)}e?&31*ocK_W%QSu0<{Sf!t4?))rS#66B zyKiJhE@Y;tY7h~IY!~ouIVGW7*fb3BmN=U!l*2!z?V1lt&wonV{3&S*e3L|b@QiPn z^J(Ns1UW0=%IQ(YlCgfxuu+EGPmw==w-o0?T+TVZ zz)wU&G%#(-vw8QzT20|PUt~|dPiBU0+f#KiV<$6Rfuks6(;_y@0Mk4X8^U#)tjJ$4 zfEDQq0`OlGlSq7_FqxZ#==5Z&t_U0fHCbX)n%K{U3l=^(j&}|hC3@ibymKhA^pMLr zkX+e;2%Y_CkLPQHrthONwHo9Htv0#mXlC(bOa9M4R| z8OsQ&AueK`8!&n3{C{ByzF2qzmdK|pk+MX}62A{i@VPf&i9*T}DNCd*@%~wYEuJx4 znR6(jP|6hO75o2UiHmp45+yeLlm8)>I8NI9Dd~R)ON4ev!s~~PE}xWFcD(W`snBhg zL!7cR=Di9Fjc&LkdU5nA1p3%*eiqS zdiSZ6&k}F4EyI;lmPlD5Wr^R1CEjFPhD})_Wr>s}-akv^a_&~l8*R*R>BdaDF_Ui0 z{L43HxZ*qCnBgzVZ})wf9G50jX)={2Q|~^R@>$|dHfD0^D_6=ADNFoDEb%5AGwJqi z$`Y@!M7LkBMl`}U_Adlvf0LTm>S)NZkLn)K*9Mynd)m8|6ow$2Fv>*B%6Q zhCLa^t=0`R6B~F-Odw5nJeY__8(4 zwGbU_w7!Eu#u`G6h6h;~y`d@#cLPq4MXj#G?;6+JEEztl{yPyOuqO*pR_gsx^ig$f#TGutFcrD|OVu zJ%6BUJQNl%VXeH9TJB8zYu|Lj#X#%9?FZ{yTWd?g93dpE;iH=MAlbznjXQ#cnj?B- zQDcEdYe7b}y#I{gHixLHsIJwnEu^g*Qv^rZz>VEn3oMQIA7f|(>)vLao9ufEJNz#V zJM0_o^Oe$lzI31OAK&MDmw~F=zz`H0Uq1kiBU^B{ioqcad{?P{lfi%<)K1*q&nAmlbc<;IjRu-;6z*B+#~?2WHm zo%--q#yU$KRrkf2`Xx=ku-;6vx9I)ctnjF>ch2fy5;~;Rnc(N(bknI z8q;7^X~@RtQyQrL>5XIl9etzk$c^J@U<`!Cd=3I(fzLQwHU;`J2Dc(%@zCb;{*bs| z?XcE~u`v4o$uYu?1^*aX@1yFFmBV%lyF^rnC_(RkjtS%(pyorw9A#ujAV|xH_?TiG zfgKx5wGrgvBLBepj4TwwRO|=@`S=hKQwDZN*h9lf{Obt#Bz%Z{ecE*=oxr{ONcY5l z0XamuL9TTKqJ;Y@)4s~HL6%M8*py+9WY{X05oY$Xr})4@g5Bemfr160S`iIXky`r9 zNG|$7TO-u~yyTVJrK%3{7C#y$g@@De2ovKtgR&Ve&YRN&TpTq&3S}OJiX_?l^6Nm1 z3C%iYB;WL*|6>goi)gGAiC8!A%15MVQGyU_2hb0SrA*_c4@R|K6yLnZb7IAc7Vxo} z1uYGL6$p_wF^?z_@tKfi-v}3Am|@i-Ss&q4OR-)pe}KS2!QDfFdE1ePp~gAT+{RkM zGjqmg5?0kcTn0VGHdn#k9u%@pYTf5LFpjaZ_d5^L6)jL4V^2hOprT@m#;u}P19c;O zM=M1<&B68vkcd-%BLZhLYy_`5YS#3*qQ`ZS3_GTbPHS@90Q!dLb6buLz&M7?! z@LU(x5m@TD`yH*_6Aarnv|DTjis=Nu@8NenQw?l~iNQP1vtgRwv6eeR(=e59hK^Ox zThSv6H9nANNR-qQ20EDjA{jEn0yiCAtB_v8HDrq1Se`Z?R0Qo6NG<#P_17^ZuBbJv zq25=()Sp$z!Var{LuU5Rn1`~^q5kjTHOLH>CQ9nwa@rGm|9-*oXH2oozq~RET?bu4 z3wDu&#rZoth%Qb1e&v(s8_QqOeVBTk2XUtp>co4pfG)w(ovvCYY=F?cJ41%)J13LR0)&^H7n3+0X938xm&ZiTtOJ zr$<@9vwoM%L;SV11bfDu=iI;6Y)iYv4G}Cgx97c#LLM-@-&%=$5d2Eyeh>MO(0kIY YAREP)V|R4QE?%Ad7qtPw^$k)00GL^19RL6T literal 0 HcmV?d00001 diff --git a/doc/design/cluster_train/src/dataset.png b/doc/design/cluster_train/src/dataset.png new file mode 100644 index 0000000000000000000000000000000000000000..2fb7f1cce3b6dd21489392557826e95a9f207c34 GIT binary patch literal 10845 zcmeI21yt03_vZ%$1q6hlq`QS7MUa&4Zb@mRQ=~&e5u`(g2FU>dK@boTk?xR?knWad zKZC!0p8xLIXV3FL=f7w7?D=_)Iy2l)-S7R}`?~KTQcXn`2a5~~0)gPj%SmZKAgGk! z^#KMd__t#8#t`@q#Z5!@5v07AViml>be7Y1gFtQ*ApfF3(%#+$1|zmwdhUA4Nd~NwZeD=?E z{@AP8IJr9kFSy!1k#}^rbOnRm&5*kj<@sy$zqa_F=TddGwFIZW9?kvN=>PcHU-rT< zd!#X$XXcR$fX%%Nu1g<5>mnkCS#c(la5?VEpxQ z#_|kq&#(xQ+*npiQaUq**tEM6q;V1a7Q-Bpok@7>G)Zbnl~QjoE4SnH9&McUZa;MM zA9Ct@ZnD()d(+=z=JHd(Cy&#E)uCly_%hrnbN{XXb}d{i03I8=!y$?n2tlKP1f^ig zsTpfB#u59Omq%mKKu|EmX;#qk*jIyl11}{ZsCa>axoG`FvMw5=Nb}DG;<~8Ys1W4f zSr!!cHubk!H<4p@(E1ZUH$WiB86GG=j9dtr6D7cK3}F;^#vM)w1VbEx`WhNh%I?&B z@!^OjUG+7EVAJo$?Zd6g%D$%PNa8U3s`xz==@b-< zPtHxy7cvGscEj(gtozCzf7B>+oN2lk%Dk+$8OR~!wetNHE%5C%Wl8t1_L#AE>c3`N zLSB-bW?4}O)}qL!vm`>? zR|i?YXIFpW2sj`JoieS5Mpkh`Szhz_m&HRVvY zmNl4x>1rTX7Fwn%cA*tMUGJeev9NB)pQoC6Ip6kDLG|qeb%FOPZ?0n&1%cmBbM@s| zK?)}!XQ4O4(B4GA=_en0wfqJm;9@1Ey}eba`>s8E}4FiOO4L)x%@!r&#NM zbv9$&lU8HY;0eqwH>&!&7mGJ##ZCv7GXorPn=dw783%nj`GLdf=clL)hV4Qr={di)<-Sa@tLCc_%M;AV1u2EBjws?H zzCN{cdUIx?ZiG?8hc6rxzw=QGRWf+m2s@mn>kq{)cVnV)Mx!jii~9D^@rf!@s$7|Fnah$H__gK^N5P#To zP{wuE`205^|H<5)0Po*hMm@rC;b%F%$`5d3g%bU0 zbKSOc`36Wy5L7C+7UM$u)_$Ll={t+ zI~DI}&E5$(4#B75aF)N{nZ#jVtYLn3v>Da9u0~5>8;K94v};c`)*EPGE$|-!&f8;D zkD8Y=Qo39nD(p&N%_SW?+irGpJ$`Y0IaKlc`!lj`%F`Z{g%J8N+(>I#$(N(&N|Ti1 zRhlr4u^XLCNV8iJkzKSOP};v6KbmbNn09ydf@n4Vf-Rq@WX#yYK@+!Y84sji2%4~> zI9sfb(e6`(&D}rQi48R_k>s1Mv37bL?D~`AG)Vot{r-SA^4M_tz;f?i20jdA@3jsyw z>P&Ht*Nu{xqG2MVtKZF6)5T#@X|%B^DJeNkvo~44Earak4vu-XXsoHZcC&!7G~}Z- zz8&&O;7OsbDBzW;WYU5Y*U5si6vM%ftZEo^pe#k`6!swj!mg7|> z(L_&Iavy(m{c)Be^4xQg&vEL3V$!`M-$C0H$c;y2_qVzlmh#Hwq}PUv;?Mo5{g2J| zlw1e7Piy+^F>%Z2?l## z_6#N$&GB^b%=o4EO6Tscc7$;w*=Rz(V)E;|+&Q2!Q2XvkY)yR($0q#(3l61KcoP15 zYx<$`Pie!UxKHzF5yvPUMW`MGw&t|pjD)CBsnO`_&IP^v$gH#Q)Umw z)6s>Vblor_b4*SADwyD@56h|->Ru>7m9R~MDn#~4V@S0a8Hla{wLSIJemZYqa-#wz z(r8L##@zQkJUQR*t6^m^=VF56{4#c7%F;&gTKBHxCm|?k`?^zkL9wJ*wsLOJRSZO# z8`<+Z-$Kaf@=1&q49=V{2^bjbDoM`}FsqbmSSWzlX4^d@f5-_b?_&NwD>aHq3ZEV4- z1?@r%-uEJf)#k}*L@zFb?-Nl=KA%rWS(0|1_dgF=0RY)bL z%YsKdV3jqIx<_GHZP_JLbY!R#Ym|+=4;F9QrTlb!cDysUj%cP*re`I%g^Kt}0Om0L zK{!n(?PzK@TCvl)#9n}o^^S&(mB-D1qcrlRsBMkc0lNb5oqF(|ccm_Y$?gNfv|HEJ zqKGN5Jdcc$NJqk-XgRt51$t9coSQ6#zDe{PY-C(|8u zHKM2F(@lQQ0TmtXrSe#^J+5<3<}^}BWXnGX_b!@R%ztCN@?>wZisYrJ$kXu(Cgn5$ z-E^mlETjRb=>ks1bR zzM^6>r-$Nq7L&j$_ zTAHdeQd-4S`J^3}HR)|0NO)$24-UvsZlV?5gHF|5oa`@;l^G_3D;~$F_+sFb_ihex zd%1CA@m$uk{iXI8YTc>DpN-Gi3QG9eJ_cO*1BfRVaP$S25$u)RtD5bj>tbh~d^}a;$yMg8#-RA?*6GZv@7jn z&}%^xLG&E0kHMbxP+35`nW9N~zrBl~A|W&%ShHE~&1hBIm~PNTIR`DH_B%C+Lc7sY z)-F3@LvT>wa5?F;`wn*HaZIeT`*EPr(NA0nXy{Gll?@)QFq34D4=u#tqN%fZZMqso zTyPJiKk&WNOj9HQvjT+ShG$b+n9V(|oSbvvXM5l=OQ#MJ`3cw!e}`gkqW04}e0xRO zvp8`(D_fR--Q>A}9S9-41-7Q@`Yim*w_mjcv#^Ixrv_!V-Teu~fP=*HPd;0IC)=ScMSq60~OeIS0 zVyc3R>D)ol5{u_+#? zF%p?nj>!Y#yLfNkt7Lgd-(H6jIV@EU4Ik503OdgQCKj1?zoC~+NaZvVa!?xHi||lL zW`C4RY0gv#5)2fMODXhSg5;i_DG~jg3+j6W@z)LByZm~@y7zRe2ugPMuvnv*xT8u~ zhqf@gl3|%#X2Fp^RuZT}YkCW2qpB4qzm<~7d8a~d1Z$3BCY7&5?)B?ZJ3vN`sFeAu z8K5$nXLZ$UHQu}Co17!nPmVTz7-%^Cc%L6-3oE)wvWY%;R!{b-`nFsNg--Xk;arj} z!rqarTq7AXezz5TR3F=p2f;^p_O5~fm%cqJK&>!yf`7mT9WFmB>*~gjDxVf<*qXgl zN|FG7#>lU*oZnGd-)}8o?&W7Z^a59GY>-GZ=8IEQchPv6VeNHjfh3TOjbD1)X>H<< ztq_gtv!?ZK^iz=2atUoNxWI!%n<$!1!L|T9xFag zb@5(d!?ZY+^!uA2jl>bO?bMXD`hS$S?*uXo%d){B-MKJ&SjCO;M(b7jQTR>d*T>6 z*P2kdBvg{nWLRO?G=F>9UDl40!Y@3H>}zJBKzd`W(7KJR-dUh}2a>@Zk$sJGWFULo8RB;O31*4SSax$)9~b7%Y1%GBzi!>tpeC3w8VMH zm*O;@fQdG+i%`I>A!v_)8QyJ;KQliB2;uo+7-M`Qq~dMIfJ27pEJ zP?iIz?{RZ;Yl*O`LL>TA@6Tbv`7X!Eh;FHV#%Yq7iof-`A!xsz&QV@o9-b$g+eCxc zN1fb8@M*)%Wgl&d#>VlpP)ag~$|Q0_$1$Ke7E2#6N3LWHD9`(cP&E$H?^j^gmsO-3 z*DkdLzD_Qa{69XV{Cq3#)C^rBJ4^~&Q_;X&jn~-fD?&f#-BXaN1+TQB%mvt0ZJ?I zn(J~eQbPs#$X2^Y$R7I8b0gNgL%uT^M(r6Ti9i;mPP&%p$xN{Te}Ha)gkS(J@AiP^ z(9UH8^wMOwX#!6E=f^H>MxaO*n%666z6I>!1(+3ZWcVFwxTs&fi)=|8AXB9@KKWg8 z97!D}OebipD}{)UJMc!0{TQw$I!6+ad8Iqmfr~&pso=q+3jYL9YUmA+8T--&5PFs7 zVEy><74;{Z6E$XjQ#JO>eYf9g{Gi z|1qhv8dTc=0E-*jkYH+gI+()!M z%7EdAr>j338xo6dsbWfwJ3gUY8yI=p2_B2K9%hx`$imZ!>LL)a$uJY03y3S#RQ&5~YQiTAOgfB6YiBgIB+KA=L>aNCisNXeFv;YAz zrjV9%@5nr_QAxZ?Pgz#w?E^EXynFB|94&EjkyoMPA`*QRduZ%~1zBC6gKlVR>=HfC z0LI|vv96;T=vG&7st;IEJyec{LDO&C!qrQ~`UZeYDi!3Qp>I+gF?xmWAL;leXsuj< z63RZe+MgXl;ocIAuEPzbVx~E>ljs%=Q4g_nX>;X&SV;y$#OByQbL-N0HsKYZnE=2il4^(U;-JuEAR|4n%m54wXSw5dIB-+VVS7MJQjyM#uO>#Z#2DPL z(Jc+-hgN;?GR8xY*NRh);_RFUjA zzT$8ilZlUR*5Hb97;zu6oELn58wK8UU56HuOz}WP_dQx{&l+CIDpU!x>mfh(Y$D?U z>5Ri@fQoC8-MeIjqfx4;+Vj#TLch_chEO`3Q6XVvrny;U+ z-M2~Pktx`bj-myP2b(v2L;CrPQ zZ+z7gPR)Iy4pk{LwFRA*M)<1FKBZ5T9MMe{E%KNg^zJ|fxU}{YeXD)$tO+Dt!f#qL&W6Ayi5jY!WCCwl0rxJXF`fs6@ z^X2arK(&%1@p}wG$LAIV@LBm^0cGmfnvnfhk!OD-q4}hQgavsYzO~z^yXP+Y!Yy5* zBm#hvf|qWVhCKT*|9_Em{v}V%c5bWqRJ4MsOvPgn(P;L!4D`1Q^tTN3@5(@8pZ=DC z{+5CMmVy43f&Rajf&L|GiGKmOPXKK?#$A+tGsvi(kP`vJl-kCw3xeTLc;xPezEjU% z3?66*%FZ&QO>BxrVYu?ucg!AZp$F%a}9TWuQmJzwtY3;<^-V@|H*zMO6hL z_vtnSiF^O0zZUn;C1|p!U;Waxb-8gZi#dk|qAgxQk`l7cuvL8ok-zgPFRda~{>U`= FzX23JaCHCx literal 0 HcmV?d00001 From 6e9f9ae75c09d8a78470c5bfb4f0bb88701a7c3a Mon Sep 17 00:00:00 2001 From: qijun Date: Wed, 3 May 2017 17:46:58 +0800 Subject: [PATCH 07/48] refactor use concepts doc --- doc/getstarted/concepts/src/train.py | 52 ++++++ doc/getstarted/concepts/use_concepts_cn.rst | 149 ++++++++++++++++++ doc/getstarted/index_cn.rst | 1 + doc/howto/index_cn.rst | 1 - .../usage/concepts/src/pserver_topology.dot | 68 -------- .../usage/concepts/src/trainer_config.py | 29 ---- doc/howto/usage/concepts/use_concepts_cn.rst | 139 ---------------- 7 files changed, 202 insertions(+), 237 deletions(-) create mode 100644 doc/getstarted/concepts/src/train.py create mode 100644 doc/getstarted/concepts/use_concepts_cn.rst delete mode 100644 doc/howto/usage/concepts/src/pserver_topology.dot delete mode 100644 doc/howto/usage/concepts/src/trainer_config.py delete mode 100644 doc/howto/usage/concepts/use_concepts_cn.rst diff --git a/doc/getstarted/concepts/src/train.py b/doc/getstarted/concepts/src/train.py new file mode 100644 index 0000000000..679d0a931a --- /dev/null +++ b/doc/getstarted/concepts/src/train.py @@ -0,0 +1,52 @@ +import paddle.v2 as paddle +import numpy as np + +# init paddle +paddle.init(use_gpu=False) + +# network config +x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) +y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) +y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) +cost = paddle.layer.mse_cost(input=y_predict, label=y) + +# create parameters +parameters = paddle.parameters.create(cost) +# create optimizer +optimizer = paddle.optimizer.Momentum(momentum=0) +# create trainer +trainer = paddle.trainer.SGD(cost=cost, + parameters=parameters, + update_equation=optimizer) + + +# event_handler to print training info +def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 1 == 0: + print "Pass %d, Batch %d, Cost %f" % (event.pass_id, event.batch_id, + event.cost) + + +# define training dataset reader +def train_reader(): + train_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]) + train_y = np.array([-2, -3, -7, -7]) + + def reader(): + for i in xrange(train_y.shape[0]): + yield train_x[i], train_y[i] + + return reader + + +# define feeding map +feeding = {'x': 0, 'y': 1} + +# training +trainer.train( + reader=paddle.batch( + train_reader(), batch_size=1), + feeding=feeding, + event_handler=event_handler, + num_passes=100) diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst new file mode 100644 index 0000000000..c4b32cc1a8 --- /dev/null +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -0,0 +1,149 @@ +############ +新手入门 +############ + +PaddlePaddle是源于百度的一个深度学习平台。PaddlePaddle为深度学习研究人员提供了丰富的API，可以轻松的完成神经网络配置，模型训练等任务。 +这里将介绍PaddlePaddle的基本使用概念，并且展示了如何利用PaddlePaddle来解决一个经典的线性回归问题。 +在使用该文档之前，请参考 `安装文档 `_ 完成PaddlePaddle的安装。 + + +配置网络 +============ + +加载PaddlePaddle +---------------------- + +在进行网络配置之前，首先需要加载相应的Python库，并进行初始化操作。 + +.. code-block:: bash + + import paddle.v2 as paddle + import numpy as np + paddle.init(use_gpu=False) + + +搭建神经网络 +----------------------- + +搭建神经网络就像使用积木搭建宝塔一样。在PaddlePaddle中，layer是我们的积木，而神经网络是我们要搭建的宝塔。我们使用不同的layer进行组合，来搭建神经网络。 +宝塔的底端需要坚实的基座来支撑，同样，神经网络也需要一些特定的layer作为输入接口，来完成网络的训练。 + +例如，我们可以定义如下layer来描述神经网络的输入： + +.. code-block:: bash + + x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) + y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) + +其中x表示输入数据是一个维度为2的稠密向量，y表示输入数据是一个维度为1的稠密向量。 + +PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和三种序列模式。 + +四种数据类型： + +* dense_vector：稠密的浮点数向量。 +* sparse_binary_vector：稀疏的01向量，即大部分值为0，但有值的地方必须为1。 +* sparse_float_vector：稀疏的向量，即大部分值为0，但有值的部分可以是任何浮点数。 +* integer：整数标签。 + +三种序列模式： + +* SequenceType.NO_SEQUENCE：不是一条序列 +* SequenceType.SEQUENCE：是一条时间序列 +* SequenceType.SUB_SEQUENCE： 是一条时间序列，且序列的每一个元素还是一个时间序列。 + +不同的数据类型和序列模式返回的格式不同，列表如下： + ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| | NO_SEQUENCE | SEQUENCE | SUB_SEQUENCE | ++======================+=====================+===================================+================================================+ +| dense_vector | [f, f, ...] | [[f, ...], [f, ...], ...] | [[[f, ...], ...], [[f, ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| sparse_binary_vector | [i, i, ...] | [[i, ...], [i, ...], ...] | [[[i, ...], ...], [[i, ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| sparse_float_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| integer_value | i | [i, i, ...] | [[i, ...], [i, ...], ...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ + +其中，f代表一个浮点数，i代表一个整数。 + +注意：对sparse_binary_vector和sparse_float_vector，PaddlePaddle存的是有值位置的索引。例如， + +- 对一个5维非序列的稀疏01向量 ``[0, 1, 1, 0, 0]`` ，类型是sparse_binary_vector，返回的是 ``[1, 2]`` 。 +- 对一个5维非序列的稀疏浮点向量 ``[0, 0.5, 0.7, 0, 0]`` ，类型是sparse_float_vector，返回的是 ``[(1, 0.5), (2, 0.7)]`` 。 + + +在定义输入layer之后，我们可以使用其他layer进行组合。在组合时，需要指定layer的输入来源。 + +例如，我们可以定义如下的layer组合： + +.. code-block:: bash + + y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) + cost = paddle.layer.mse_cost(input=y_predict, label=y) + +其中，x与y为之前描述的输入层；而y_predict是接收x作为输入，接上一个全连接层；cost接收y_predict与y作为输入，接上均方误差层。 + +最后一层cost中记录了神经网络的所有拓扑结构，通过组合不同的layer，我们即可完成神经网络的搭建。 + + +训练模型 +============ + +在完成神经网络的搭建之后，我们首先需要根据神经网络结构来创建所需要优化的parameters，并创建optimizer。 +之后，我们可以创建trainer来对网络进行训练。 + +.. code-block:: bash + + parameters = paddle.parameters.create(cost) + optimizer = paddle.optimizer.Momentum(momentum=0) + trainer = paddle.trainer.SGD(cost=cost, + parameters=parameters, + update_equation=optimizer) + +其中，trianer接收三个参数，包括神经网络拓扑结构，神经网络参数以及迭代方程。 + +在搭建神经网络的过程中，我们仅仅对神经网络的输入进行了描述。而trainer需要读取训练数据进行训练，PaddlePaddle中通过reader来加载数据。 + +.. code-block:: bash + + # define training dataset reader + def train_reader(): + train_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]) + train_y = np.array([-2, -3, -7, -7]) + def reader(): + for i in xrange(train_y.shape[0]): + yield train_x[i], train_y[i] + return reader + +最终我们可以调用trainer的train方法启动训练： + +.. code-block:: bash + + # define feeding map + feeding = {'x': 0, 'y': 1} + + # event_handler to print training info + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 1 == 0: + print "Pass %d, Batch %d, Cost %f" % ( + event.pass_id, event.batch_id, event.cost) + # training + trainer.train( + reader=paddle.batch(train_reader(), batch_size=1), + feeding=feeding, + event_handler=event_handler, + num_passes=100) + + +线性回归完整示例 +============== + +下面给出在三维空间中使用线性回归拟合一条直线的例子： + +.. literalinclude:: src/train.py + :linenos: + +有关线性回归的实际应用，可以参考Paddle book的 `第一章节 `_ \ No newline at end of file diff --git a/doc/getstarted/index_cn.rst b/doc/getstarted/index_cn.rst index cadf092f8f..0cb27f802c 100644 --- a/doc/getstarted/index_cn.rst +++ b/doc/getstarted/index_cn.rst @@ -5,5 +5,6 @@ :maxdepth: 1 build_and_install/index_cn.rst + concepts/use_concepts_cn.rst - `深度学习入门课程 `_ diff --git a/doc/howto/index_cn.rst b/doc/howto/index_cn.rst index 5b84eea491..26449a6365 100644 --- a/doc/howto/index_cn.rst +++ b/doc/howto/index_cn.rst @@ -8,7 +8,6 @@ :maxdepth: 1 usage/cmd_parameter/index_cn.rst - usage/concepts/use_concepts_cn.rst usage/cluster/cluster_train_cn.md usage/k8s/k8s_basis_cn.md usage/k8s/k8s_cn.md diff --git a/doc/howto/usage/concepts/src/pserver_topology.dot b/doc/howto/usage/concepts/src/pserver_topology.dot deleted file mode 100644 index 9ff658b849..0000000000 --- a/doc/howto/usage/concepts/src/pserver_topology.dot +++ /dev/null @@ -1,68 +0,0 @@ -graph pp_topology { - rankdir=BT; - subgraph cluster_node0 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器0" - - pserver0 [label="Parameter \n Server 0"] - trainer0 [label="Trainer 0"] - } - subgraph cluster_node1 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器1" - - pserver1 [label="Parameter \n Server 1"] - trainer1 [label="Trainer 1"] - } - - subgraph cluster_node2 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器2" - - pserver2 [label="Parameter \n Server 2"] - trainer2 [label="Trainer 2"] - } - - subgraph cluster_node3 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器3" - - pserver3 [label="Parameter \n Server 3"] - trainer3 [label="Trainer 3"] - } - - data [label="数据", shape=hexagon] - - trainer0 -- pserver0 - trainer0 -- pserver1 - trainer0 -- pserver2 - trainer0 -- pserver3 - - trainer1 -- pserver0 - trainer1 -- pserver1 - trainer1 -- pserver2 - trainer1 -- pserver3 - - trainer2 -- pserver0 - trainer2 -- pserver1 - trainer2 -- pserver2 - trainer2 -- pserver3 - - trainer3 -- pserver0 - trainer3 -- pserver1 - trainer3 -- pserver2 - trainer3 -- pserver3 - - data -- trainer0 - data -- trainer1 - data -- trainer2 - data -- trainer3 -} diff --git a/doc/howto/usage/concepts/src/trainer_config.py b/doc/howto/usage/concepts/src/trainer_config.py deleted file mode 100644 index 3eccbd7bc1..0000000000 --- a/doc/howto/usage/concepts/src/trainer_config.py +++ /dev/null @@ -1,29 +0,0 @@ -from paddle.trainer_config_helpers import * - -define_py_data_sources2( - train_list='train.list', - test_list='test.list', - module='provider', - obj='process') -settings( - batch_size=128, - learning_rate=1e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(0.5)) - -img = data_layer(name='pixel', size=28 * 28) - -hidden1 = simple_img_conv_pool( - input=img, filter_size=3, num_filters=32, pool_size=3, num_channel=1) - -hidden2 = fc_layer( - input=hidden1, - size=200, - act=TanhActivation(), - layer_attr=ExtraAttr(drop_rate=0.5)) -predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) - -outputs( - classification_cost( - input=predict, label=data_layer( - name='label', size=10))) diff --git a/doc/howto/usage/concepts/use_concepts_cn.rst b/doc/howto/usage/concepts/use_concepts_cn.rst deleted file mode 100644 index fa334bcbb9..0000000000 --- a/doc/howto/usage/concepts/use_concepts_cn.rst +++ /dev/null @@ -1,139 +0,0 @@ -############ -基本使用概念 -############ - -PaddlePaddle是一个深度学习框架，支持单机模式和多机模式。 - -单机模式用命令 ``paddle train`` 可以启动一个trainer进程，单机训练通常只包括一个trainer进程。如果数据规模比较大，希望加速训练，可以启动分布式作业。一个分布式作业里包括若干trainer进程和若干Parameter Server（或称pserver）进程。用命令 ``paddle pserver`` 可以启动 pserver 进程，pserver进程用于协调多个trainer进程之间的通信。 - -本文首先介绍trainer进程中的一些使用概念，然后介绍pserver进程中概念。 - -.. contents:: - -系统框图 -======== - -下图描述了用户使用框图，PaddlePaddle的trainer进程里内嵌了Python解释器，trainer进程可以利用这个解释器执行Python脚本，Python脚本里定义了模型配置、训练算法、以及数据读取函数。其中，数据读取程序往往定义在一个单独Python脚本文件里，被称为数据提供器（DataProvider），通常是一个Python函数。模型配置、训练算法通常定义在另一单独Python文件中, 称为训练配置文件。下面将分别介绍这两部分。 - -.. graphviz:: - - digraph pp_process { - rankdir=LR; - config_file [label="用户神经网络配置"]; - subgraph cluster_pp { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "PaddlePaddle C++"; - py [label="Python解释器"]; - } - data_provider [label="用户数据解析"]; - config_file -> py; - py -> data_provider [dir="back"]; - } - -数据提供器 -========== - -DataProvider是PaddlePaddle系统的数据提供器，将用户的原始数据转换成系统可以识别的数据类型。每当系统需要新的数据训练时, trainer进程会调用DataProvider函数返回数据。当所有数据读取完一轮后，DataProvider返回空数据，通知系统一轮数据读取结束，并且系统每一轮训练开始时会重置DataProvider。需要注意的是，DataProvider是被系统调用，而不是新数据驱动系统，一些随机化噪声添加都应该在DataProvider中完成。 - -在不同的应用里，训练数据的格式往往各不相同。因此，为了用户能够灵活的处理数据，我们提供了Python处理数据的接口，称为 ``PyDataProvider`` 。在 ``PyDataProvider`` 中，系统C++模块接管了shuffle、处理batch、GPU和CPU通信、双缓冲、异步读取等问题，一些情况下(如：``min_pool_size=0``)需要Python接口里处理shuffle，可以参考 :ref:`api_pydataprovider2` 继续深入了解。 - - -训练配置文件 -============ - -训练配置文件主要包括数据源、优化算法、网络结构配置三部分。 其中数据源配置与DataProvider的关系是：DataProvider里定义数据读取函数，训练配置文件的数据源配置中指定DataProvider文件名字、生成数据函数接口，请不要混淆。 - -一个简单的训练配置文件为： - -.. literalinclude:: src/trainer_config.py - :linenos: - -文件开头 ``from paddle.trainer_config_helpers import *`` ，是因为PaddlePaddle配置文件与C++模块通信的最基础协议是protobuf，为了避免用户直接写复杂的protobuf string，我们为用户定以Python接口来配置网络，该Python代码可以生成protobuf包，这就是 :ref:`api_trainer_config` 的作用。因此，在文件的开始，需要import这些函数。 这个包里面包含了模型配置需要的各个模块。 - -下面分别介绍数据源配置、优化算法配置、网络结构配置这三部分该概念。 - -数据源配置 ----------- - -使用 ``PyDataProvider2`` 的函数 ``define_py_data_sources2`` 配置数据源。``define_py_data_sources2`` 里通过train_list和test_list指定是训练文件列表和测试文件列表。 如果传入字符串的话，是指一个数据列表文件。这个数据列表文件中包含的是每一个训练或者测试文件的路径。如果传入一个list的话，则会默认生成一个list文件，再传入给train.list或者test.list。 - -``module`` 和 ``obj`` 指定了DataProvider的文件名和返回数据的函数名。更详细的使用，请参考 :ref:`api_pydataprovider2` 。 - -优化算法配置 ------------- - -通过 :ref:`api_trainer_config_helpers_optimizers_settings` 接口设置神经网络所使用的训练参数和 :ref:`api_trainer_config_helpers_optimizers` ，包括学习率、batch_size、优化算法、正则方法等，具体的使用方法请参考 :ref:`api_trainer_config_helpers_optimizers_settings` 文档。 - -网络结构配置 ------------- - -神经网络配置主要包括网络连接、激活函数、损失函数、评估器。 - -- 网络连接： 主要由Layer组成，每个Layer返回的都是一个 ``LayerOutput`` 对象，Layer里面可以定义参数属性、激活类型等。 - - 为了更灵活的配置，PaddlePaddle提供了基于 Projection 或者 Operator 的配置，这两个需要与 ``mixed_layer`` 配合使用。这里简单介绍Layer、Projection、Operator的概念: - - - Layer: 神经网络的某一层，可以有可学习的参数，一般是封装了许多复杂操作的集合。 - - Projection：需要与 ``mixed_layer`` 配合使用，含可学习参数。 - - Operator： 需要与 ``mixed_layer`` 配合使用，不含可学习参数，输入全是其他Layer的输出。 - - - 这个配置文件网络由 ``data_layer`` 、 ``simple_img_conv_pool`` 、 ``fc_layer`` 组成。 - - - :ref:`api_trainer_config_helpers_layers_data_layer` ： 通常每个配置文件都会包括 ``data_layer`` ，定义输入数据大小。 - - :ref:`api_trainer_config_helpers_network_simple_img_conv_pool` ：是一个组合层，包括了图像的卷积 (convolution)和池化(pooling)。 - - :ref:`api_trainer_config_helpers_layers_fc_layer` ：全连接层，激活函数为Softmax，这里也可叫分类层。 - -- 损失函数和评估器：损失函数即为网络的优化目标，评估器可以评价模型结果。 - - PaddlePaddle包括很多损失函数和评估起，详细可以参考 :ref:`api_trainer_config_helpers_layers_cost_layers` 和 :ref:`api_trainer_config_helpers_evaluators` 。这里 ``classification_cost`` 默认使用多类交叉熵损失函数和分类错误率统计评估器。 - -- ``outputs``: 标记网络输出的函数为 ``outputs`` 。 - - 训练阶段，网络的输出为神经网络的优化目标；预测阶段，网络的输出也可通过 ``outputs`` 标记。 - - -这里对 ``mixed_layer`` 稍做详细说明， 该Layer将多个输入(Projection 或 Operator)累加求和，具体计算是通过内部的 Projection 和 Operator 完成，然后加 Bias 和 activation 操作， - -例如，和 ``fc_layer`` 同样功能的 ``mixed_layer`` 是: - -.. code-block:: python - - data = data_layer(name='data', size=200) - with mixed_layer(size=200) as out: - out += full_matrix_projection(input=data) - -PaddlePaddle 可以使用 ``mixed layer`` 配置出非常复杂的网络，甚至可以直接配置一个完整的LSTM。用户可以参考 :ref:`api_trainer_config_helpers_layers_mixed_layer` 的相关文档进行配置。 - - -分布式训练 -========== - -PaddlePaddle多机采用经典的 Parameter Server 架构对多个节点的 trainer 进行同步。多机训练的经典拓扑结构如下\: - -.. graphviz:: src/pserver_topology.dot - -图中每个灰色方块是一台机器，在每个机器中，先使用命令 ``paddle pserver`` 启动一个pserver进程，并指定端口号，可能的参数是\: - -.. code-block:: bash - - paddle pserver --port=5000 --num_gradient_servers=4 --tcp_rdma='tcp' --nics='eth0' - -* ``--port=5000`` : 指定 pserver 进程端口是 5000 。 -* ``--gradient_servers=4`` : 有四个训练进程(PaddlePaddle 将 trainer 也称作 GradientServer ，因为其为负责提供Gradient) 。 -* ``--tcp_rdma='tcp' --nics=`eth0```: 指定以太网类型为TCP网络，指定网络接口名字为eth0。 - -启动之后 pserver 进程之后，需要启动 trainer 训练进程，在各个机器上运行如下命令\: - -.. code-block:: bash - - paddle train --port=5000 --pservers=192.168.100.101,192.168.100.102,192.168.100.103,192.168.100.104 --config=... - -对于简单的多机协同训练使用上述方式即可。另外，pserver/train 通常在高级情况下，还需要设置下面两个参数\： - -* --ports_num\: 一个 pserver 进程共绑定多少个端口用来做稠密更新，默认是1。 -* --ports_num_for_sparse\: 一个pserver进程共绑定多少端口用来做稀疏更新，默认是0。 - -使用手工指定端口数量，是因为Paddle的网络通信中，使用了 int32 作为消息长度，比较容易在大模型下溢出。所以，在 pserver 进程中可以启动多个子线程去接受 trainer 的数据，这样单个子线程的长度就不会溢出了。但是这个值不可以调的过大，因为增加这个值，对性能尤其是内存占用有一定的开销，另外稀疏更新的端口如果太大的话，很容易导致某一个参数服务器没有分配到任何参数。 From c3c6acd3465cf2364959c7f96b4e0998f6ecf8bb Mon Sep 17 00:00:00 2001 From: qijun Date: Wed, 3 May 2017 19:05:36 +0800 Subject: [PATCH 08/48] update some links --- doc/getstarted/concepts/use_concepts_cn.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst index c4b32cc1a8..ea0b78f3c3 100644 --- a/doc/getstarted/concepts/use_concepts_cn.rst +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -1,5 +1,5 @@ ############ -新手入门 +基本使用概念 ############ PaddlePaddle是源于百度的一个深度学习平台。PaddlePaddle为深度学习研究人员提供了丰富的API，可以轻松的完成神经网络配置，模型训练等任务。 @@ -146,4 +146,4 @@ PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和 .. literalinclude:: src/train.py :linenos: -有关线性回归的实际应用，可以参考Paddle book的 `第一章节 `_ \ No newline at end of file +有关线性回归的实际应用，可以参考Paddle book的 `第一章节 `_ \ No newline at end of file From 74ad98e993d05042f59307bb41d420cd1b1cfe4e Mon Sep 17 00:00:00 2001 From: chrisxu2016 <823254351@qq.com> Date: Fri, 5 May 2017 11:13:22 +0800 Subject: [PATCH 09/48] add release.cn.md --- RELEASE.cn.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 RELEASE.cn.md diff --git a/RELEASE.cn.md b/RELEASE.cn.md new file mode 100755 index 0000000000..e28586cf54 --- /dev/null +++ b/RELEASE.cn.md @@ -0,0 +1,83 @@ +# v0.10.0版本 + +我们非常高兴发布了PaddlePaddleV0.10.0版。在这一版，开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 + +- 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，即定义数据生成器和网络拓扑结构的文件。用户通过运行paddle_trainer的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的配置脚本，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 + +- 新版的API被称为 *V2 API*，允许我们在单个.py文件中编辑更短的Python程序来定义网络结构和数据。此外，该Python程序也可以在Jupyter Notebook中运行，因为PaddlePaddle可以作为共享库来被Python程序加载和使用，这也是入门级的Python程序使用方式。 + +基于新的API，我们提供了一个在线的学习文档 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 + +我们还致力于迭代更新新版API的在线文档。我们将在下一个版本中发布更多的改进文档。 + +我们还致力于将新版API引入分布式模型训练中（通过MPI和Kubernetes）。这项工作正在进行中。我们将在下一个版本中发布更多内容。 + +## 新特点 + +* 发布新版[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 +* 学习文档 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 +* 支持矩形输入的CNN。 +* 为seqlastin和seqfirstin提供stride pooling。 +* 在`trainer_config_helpers`中暴露`seq_concat_layer/seq_reshape_layer`。 +* 添加公共数据集包：CIFAR，MNIST，IMDB，WMT14，CONLL05，movielens，imikolov。 +* 针对Single Shot Multibox Detection增加 Prior box layer。 +* 增加光滑的L1损失。 +* 在V2 API中增加 data reader 创建器和修饰器。 +* 增加cmrnorm投影的CPU实现。 + + +## 改进 + +* 提供`paddle_trainer`的Python virtualenv支持。 +* 增加用于自动格式化代码pre-commit hooks。 +* 升级protobuf到3.x版本。 +* 在Python数据生成器中提供一个检测数据类型的选项。 +* 加速GPU中average层的后向反馈。 +* 细化文档。 +* 使用Travis-CI检查文档中的死链接。 +* 增加解释`sparse_vector的示例。 +* 在layer_math.py中添加ReLU。 +* 简化Quick Start示例中的数据处理流程。 +* 支持CUDNN Deconv。 +* 在v2 API中增加数据feeder。 +* 在情感分析示例的演示中增加对标准输入流中样本的预测。 +* 提供图像预处理的多进程接口。 +* 增加V1 API的基准文档。 +* 在`layer_math.py`中增加ReLU。 +* 提供公共数据集的自动下载包。 +* 将`Argument::sumCost`重新命名为`Argument::sum`。 +* 将Argument::sum暴露给python。 +* 为矩阵相关的表达式评估增加一个新的`TensorExpression`实现。 +* 增加延迟分配来优化批处理多表达式计算。 +* 增加抽象的类函数及其实现。 + * `PadFunc` 和 `PadGradFunc`。 + * `ContextProjectionForwardFunc` 和 `ContextProjectionBackwardFunc`。 + * `CosSimBackward` 和 `CosSimBackwardFunc`。 + * `CrossMapNormalFunc` 和 `CrossMapNormalGradFunc`。 + * `MulFunc`。 +* 增加`AutoCompare`和`FunctionCompare`类，使得编写比较gpu和cpu版本函数的单元测试更容易。 +* 生成`libpaddle_test_main.a`并删除测试文件内的主函数。 +* 支持PyDataProvider2中numpy中的稠密向量。 +* 清理代码库，删除一些复制粘贴的代码片段。 + * 增加`SparseRowMatrix`的抽样类`RowBuffer`。 + * 清理`GradientMachine`的接口。 + * 在layer中增加`override`关键字。 + * 简化`Evaluator::create`，使用`ClassRegister`来创建`Evaluator`。 +* 下载演示的数据集时检查MD5校验。 +* 添加`paddle::Error`，用于替代Paddle中的`LOG(FATAL)`。 + + +## 错误修复 + +* 检查`recurrent_group`的layer输入类型。 +* 不要用.cu源文件运行`clang-format`。 +* 修复`LogActivation`的使用错误。 +* 修复运行`test_layerHelpers`多次的错误。 +* 修复seq2seq示例超出原型消息大小限制的错误。 +* 修复在GPU模式下dataprovider转换的错误。 +* 修复`GatedRecurrentLayer`中的错误。 +* 修复在测试多个模型时`BatchNorm`的错误。 +* 修复paramRelu在单元测试时崩溃的错误。 +* 修复`CpuSparseMatrix`编译时相关的警告。 +* 修复`MultiGradientMachine`在`trainer_count > batch_size`时的错误。 +* 修复`PyDataProvider2`阻止异步加载数据的错误。 From 7302d40a376673d935f7bd682ee6748ec1e15ad9 Mon Sep 17 00:00:00 2001 From: chrisxu2016 <823254351@qq.com> Date: Fri, 5 May 2017 11:16:42 +0800 Subject: [PATCH 10/48] add RELEASE.cn.md --- RELEASE.cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.cn.md b/RELEASE.cn.md index e28586cf54..e27b1f480f 100755 --- a/RELEASE.cn.md +++ b/RELEASE.cn.md @@ -1,6 +1,6 @@ # v0.10.0版本 -我们非常高兴发布了PaddlePaddleV0.10.0版。在这一版，开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 +我们非常高兴发布了PaddlePaddle V0.10.0版。并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 - 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，即定义数据生成器和网络拓扑结构的文件。用户通过运行paddle_trainer的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的配置脚本，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 From 4b6d891903494fe888abebfebe603a58a4cb381a Mon Sep 17 00:00:00 2001 From: xushaoyong <823254351@qq.com> Date: Fri, 5 May 2017 12:02:44 +0800 Subject: [PATCH 11/48] Update RELEASE.cn.md --- RELEASE.cn.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE.cn.md b/RELEASE.cn.md index e27b1f480f..d9eb8af98b 100755 --- a/RELEASE.cn.md +++ b/RELEASE.cn.md @@ -1,8 +1,8 @@ # v0.10.0版本 -我们非常高兴发布了PaddlePaddle V0.10.0版。并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 +我们非常高兴发布了PaddlePaddle V0.10.0版,并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 -- 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，即定义数据生成器和网络拓扑结构的文件。用户通过运行paddle_trainer的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的配置脚本，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 +- 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，分别是定义数据生成器和定义网络拓扑结构的文件。用户通过运行`paddle_trainer`的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的文件，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 - 新版的API被称为 *V2 API*，允许我们在单个.py文件中编辑更短的Python程序来定义网络结构和数据。此外，该Python程序也可以在Jupyter Notebook中运行，因为PaddlePaddle可以作为共享库来被Python程序加载和使用，这也是入门级的Python程序使用方式。 From 3aa9ebd5b434749ad819d0095ba7ce233ddd2ca4 Mon Sep 17 00:00:00 2001 From: xushaoyong <823254351@qq.com> Date: Fri, 5 May 2017 12:09:33 +0800 Subject: [PATCH 12/48] Update RELEASE.cn.md --- RELEASE.cn.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/RELEASE.cn.md b/RELEASE.cn.md index d9eb8af98b..c67e9756a8 100755 --- a/RELEASE.cn.md +++ b/RELEASE.cn.md @@ -4,18 +4,16 @@ - 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，分别是定义数据生成器和定义网络拓扑结构的文件。用户通过运行`paddle_trainer`的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的文件，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 -- 新版的API被称为 *V2 API*，允许我们在单个.py文件中编辑更短的Python程序来定义网络结构和数据。此外，该Python程序也可以在Jupyter Notebook中运行，因为PaddlePaddle可以作为共享库来被Python程序加载和使用，这也是入门级的Python程序使用方式。 +- 新版的API被称为 *V2 API*，允许我们在单个.py文件中，通过编辑更短的Python程序来定义网络结构和数据。此外，该Python程序也可以在Jupyter Notebook中运行，因为PaddlePaddle可以作为共享库来被Python程序加载和使用。 基于新的API，我们提供了一个在线的学习文档 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 -我们还致力于迭代更新新版API的在线文档。我们将在下一个版本中发布更多的改进文档。 - -我们还致力于将新版API引入分布式模型训练中（通过MPI和Kubernetes）。这项工作正在进行中。我们将在下一个版本中发布更多内容。 +我们还致力于迭代更新新版API的在线文档，并将新版API引入分布式集群（包括MPI和Kubernetes）训练中。我们将在下一个版本中发布更多的内容。 ## 新特点 * 发布新版[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 -* 学习文档 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 +* 学习深度学习系列课程 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 * 支持矩形输入的CNN。 * 为seqlastin和seqfirstin提供stride pooling。 * 在`trainer_config_helpers`中暴露`seq_concat_layer/seq_reshape_layer`。 @@ -29,13 +27,13 @@ ## 改进 * 提供`paddle_trainer`的Python virtualenv支持。 -* 增加用于自动格式化代码pre-commit hooks。 +* 增加代码自动格式化的pre-commit hooks。 * 升级protobuf到3.x版本。 * 在Python数据生成器中提供一个检测数据类型的选项。 -* 加速GPU中average层的后向反馈。 +* 加速GPU中average层的后向反馈计算。 * 细化文档。 * 使用Travis-CI检查文档中的死链接。 -* 增加解释`sparse_vector的示例。 +* 增加解释`sparse_vector`的示例。 * 在layer_math.py中添加ReLU。 * 简化Quick Start示例中的数据处理流程。 * 支持CUDNN Deconv。 @@ -45,11 +43,10 @@ * 增加V1 API的基准文档。 * 在`layer_math.py`中增加ReLU。 * 提供公共数据集的自动下载包。 -* 将`Argument::sumCost`重新命名为`Argument::sum`。 -* 将Argument::sum暴露给python。 +* 将`Argument::sumCost`重新命名为`Argument::sum`，并暴露给python。 * 为矩阵相关的表达式评估增加一个新的`TensorExpression`实现。 * 增加延迟分配来优化批处理多表达式计算。 -* 增加抽象的类函数及其实现。 +* 增加抽象的类函数及其实现： * `PadFunc` 和 `PadGradFunc`。 * `ContextProjectionForwardFunc` 和 `ContextProjectionBackwardFunc`。 * `CosSimBackward` 和 `CosSimBackwardFunc`。 @@ -57,8 +54,8 @@ * `MulFunc`。 * 增加`AutoCompare`和`FunctionCompare`类，使得编写比较gpu和cpu版本函数的单元测试更容易。 * 生成`libpaddle_test_main.a`并删除测试文件内的主函数。 -* 支持PyDataProvider2中numpy中的稠密向量。 -* 清理代码库，删除一些复制粘贴的代码片段。 +* 支持PyDataProvider2中numpy的稠密向量。 +* 清理代码库，删除一些复制粘贴的代码片段： * 增加`SparseRowMatrix`的抽样类`RowBuffer`。 * 清理`GradientMachine`的接口。 * 在layer中增加`override`关键字。 @@ -73,7 +70,7 @@ * 不要用.cu源文件运行`clang-format`。 * 修复`LogActivation`的使用错误。 * 修复运行`test_layerHelpers`多次的错误。 -* 修复seq2seq示例超出原型消息大小限制的错误。 +* 修复seq2seq示例超出消息大小限制的错误。 * 修复在GPU模式下dataprovider转换的错误。 * 修复`GatedRecurrentLayer`中的错误。 * 修复在测试多个模型时`BatchNorm`的错误。 From c003efda6963ecd924e4990c4877807ee88c47ab Mon Sep 17 00:00:00 2001 From: xushaoyong <823254351@qq.com> Date: Fri, 5 May 2017 12:12:00 +0800 Subject: [PATCH 13/48] Update RELEASE.cn.md --- RELEASE.cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.cn.md b/RELEASE.cn.md index c67e9756a8..5c467a853e 100755 --- a/RELEASE.cn.md +++ b/RELEASE.cn.md @@ -1,6 +1,6 @@ # v0.10.0版本 -我们非常高兴发布了PaddlePaddle V0.10.0版,并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 +我们非常高兴发布了PaddlePaddle V0.10.0版，并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 - 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，分别是定义数据生成器和定义网络拓扑结构的文件。用户通过运行`paddle_trainer`的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的文件，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 From a687a0b4efb1c03ef2cc19c12250630dc0717603 Mon Sep 17 00:00:00 2001 From: xushaoyong <823254351@qq.com> Date: Fri, 5 May 2017 12:14:32 +0800 Subject: [PATCH 14/48] Update RELEASE.cn.md --- RELEASE.cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.cn.md b/RELEASE.cn.md index 5c467a853e..5deaf230a8 100755 --- a/RELEASE.cn.md +++ b/RELEASE.cn.md @@ -13,7 +13,7 @@ ## 新特点 * 发布新版[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 -* 学习深度学习系列课程 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 +* 发布深度学习系列课程 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 * 支持矩形输入的CNN。 * 为seqlastin和seqfirstin提供stride pooling。 * 在`trainer_config_helpers`中暴露`seq_concat_layer/seq_reshape_layer`。 From 70d15e84bebfc9bbb7d1a1aa37e32ed9806033ac Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 5 May 2017 12:29:10 +0800 Subject: [PATCH 15/48] Add dataset ptb --- python/paddle/v2/dataset/imikolov.py | 4 +- python/paddle/v2/dataset/ptb.py | 169 +++++++++++++++++++++ python/paddle/v2/dataset/tests/ptb_test.py | 53 +++++++ 3 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 python/paddle/v2/dataset/ptb.py create mode 100644 python/paddle/v2/dataset/tests/ptb_test.py diff --git a/python/paddle/v2/dataset/imikolov.py b/python/paddle/v2/dataset/imikolov.py index bf88fe1557..97b9f7d915 100644 --- a/python/paddle/v2/dataset/imikolov.py +++ b/python/paddle/v2/dataset/imikolov.py @@ -41,7 +41,7 @@ def word_count(f, word_freq=None): return word_freq -def build_dict(typo_freq=50): +def build_dict(min_word_freq=50): """ Build a word dictionary from the corpus, Keys of the dictionary are words, and values are zero-based IDs of these words. @@ -59,7 +59,7 @@ def build_dict(typo_freq=50): # remove for now, since we will set it as last index del word_freq[''] - word_freq = filter(lambda x: x[1] > typo_freq, word_freq.items()) + word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*word_freq_sorted)) diff --git a/python/paddle/v2/dataset/ptb.py b/python/paddle/v2/dataset/ptb.py new file mode 100644 index 0000000000..ea68daf9c6 --- /dev/null +++ b/python/paddle/v2/dataset/ptb.py @@ -0,0 +1,169 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +langauge model's simple dataset. + +This module will download dataset from +http://www.fit.vutbr.cz/~imikolov/rnnlm/ and parse training set and test set +into paddle reader creators. +""" +import paddle.v2.dataset.common +import collections +import tarfile + +__all__ = ['train', 'test', 'build_dict'] + +URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' +MD5 = '30177ea32e27c525793142b6bf2c8e2d' + + +def word_count(f, word_freq=None): + if word_freq is None: + word_freq = collections.defaultdict(int) + + for l in f: + for w in l.strip().split(): + word_freq[w] += 1 + word_freq[''] += 1 + word_freq[''] += 1 + + return word_freq + + +def build_dict(min_word_freq=50): + """ + Build a word dictionary from the corpus, Keys of the dictionary are words, + and values are zero-based IDs of these words. + """ + train_filename = './simple-examples/data/ptb.train.txt' + test_filename = './simple-examples/data/ptb.valid.txt' + with tarfile.open( + paddle.v2.dataset.common.download( + paddle.v2.dataset.imikolov.URL, 'imikolov', + paddle.v2.dataset.imikolov.MD5)) as tf: + trainf = tf.extractfile(train_filename) + testf = tf.extractfile(test_filename) + word_freq = word_count(testf, word_count(trainf)) + if '' in word_freq: + # remove for now, since we will set it as last index + del word_freq[''] + + word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) + + word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) + words, _ = list(zip(*word_freq_sorted)) + word_idx = dict(zip(words, xrange(len(words)))) + word_idx[''] = len(words) + + return word_idx + + +def reader_creator(filename, reader_type, word_idx, n=-1): + def reader(): + with tarfile.open( + paddle.v2.dataset.common.download( + paddle.v2.dataset.imikolov.URL, 'imikolov', + paddle.v2.dataset.imikolov.MD5)) as tf: + f = tf.extractfile(filename) + + UNK = word_idx[''] + + for l in f: + if 'ngram' == reader_type: + assert n > -1, 'Invalid gram length' + l = [''] + l.strip().split() + [''] + if len(l) < n: continue + l = [word_idx.get(w, UNK) for w in l] + for i in range(n, len(l) + 1): + yield tuple(l[i - n:i]) + elif 'seq' == reader_type: + l = l.strip().split() + l = [word_idx.get(w, UNK) for w in l] + src_seq = [word_idx['']] + l + trg_seq = l + [word_idx['']] + yield src_seq, trg_seq + + return reader + + +def ngram_train(word_idx, n): + """ + ptb ngram type training set creator. + + It returns a reader creator, each sample in the reader is a word ID + tuple. + + :param word_idx: word dictionary + :type word_idx: dict + :param n: sliding window size + :type n: int + :return: Training reader creator + :rtype: callable + """ + return reader_creator('./simple-examples/data/ptb.train.txt', 'ngram', + word_idx, n) + + +def ngram_test(word_idx, n): + """ + ptb ngram test set creator. + + It returns a reader creator, each sample in the reader is a word ID + tuple. + + :param word_idx: word dictionary + :type word_idx: dict + :param n: sliding window size + :type n: int + :return: Test reader creator + :rtype: callable + """ + return reader_creator('./simple-examples/data/ptb.valid.txt', 'ngram', + word_idx, n) + + +def seq_train(word_idx): + """ + ptb sequence type training set creator. + + It returns a reader creator, each sample in the reader is a word ID + pair. + + :param word_idx: word dictionary + :type word_idx: dict + :return: Test reader creator + :rtype: callable + """ + return reader_creator('./simple-examples/data/ptb.train.txt', 'seq', + word_idx) + + +def seq_test(word_idx): + """ + ptb sequence type test set creator. + + It returns a reader creator, each sample in the reader is a word ID + pair. + + :param word_idx: word dictionary + :type word_idx: dict + :return: Test reader creator + :rtype: callable + """ + return reader_creator('./simple-examples/data/ptb.valid.txt', 'seq', + word_idx) + + +def fetch(): + paddle.v2.dataset.common.download(URL, "imikolov", MD5) diff --git a/python/paddle/v2/dataset/tests/ptb_test.py b/python/paddle/v2/dataset/tests/ptb_test.py new file mode 100644 index 0000000000..5e584a734d --- /dev/null +++ b/python/paddle/v2/dataset/tests/ptb_test.py @@ -0,0 +1,53 @@ +import paddle.v2.dataset.ptb +import unittest + +WORD_DICT = paddle.v2.dataset.ptb.build_dict() + + +class TestMikolov(unittest.TestCase): + def check_reader(self, reader, n): + for l in reader(): + self.assertEqual(len(l), n) + + def test_ngram_train(self): + n = 5 + self.check_reader(paddle.v2.dataset.ptb.ngram_train(WORD_DICT, n), n) + + def test_ngram_test(self): + n = 5 + self.check_reader(paddle.v2.dataset.ptb.ngram_test(WORD_DICT, n), n) + + def test_seq_train(self): + first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ + 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ + 'rake regatta rubens sim snack-food ssangyong swapo wachter' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.ptb.seq_train(WORD_DICT)(): + read_line = l[0][1:] + break + + self.assertEqual(first_line, read_line) + + def test_seq_test(self): + first_line = 'consumers may want to move their telephones a little '\ + 'closer to the tv set' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.ptb.seq_test(WORD_DICT)(): + read_line = l[0][1:] + break + + self.assertEqual(first_line, read_line) + + def test_total(self): + _, idx = zip(*WORD_DICT.items()) + self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1) + + +if __name__ == '__main__': + unittest.main() From 286696aa2b300b63d460f117c834500e87ca407a Mon Sep 17 00:00:00 2001 From: yangyaming Date: Sat, 6 May 2017 17:33:12 +0800 Subject: [PATCH 16/48] extend imikolov instead of adding ptb --- python/paddle/v2/dataset/imikolov.py | 44 +++-- python/paddle/v2/dataset/ptb.py | 169 ------------------ .../paddle/v2/dataset/tests/imikolov_test.py | 27 +++ python/paddle/v2/dataset/tests/ptb_test.py | 53 ------ 4 files changed, 60 insertions(+), 233 deletions(-) delete mode 100644 python/paddle/v2/dataset/ptb.py delete mode 100644 python/paddle/v2/dataset/tests/ptb_test.py diff --git a/python/paddle/v2/dataset/imikolov.py b/python/paddle/v2/dataset/imikolov.py index 97b9f7d915..dd3a4552d2 100644 --- a/python/paddle/v2/dataset/imikolov.py +++ b/python/paddle/v2/dataset/imikolov.py @@ -28,6 +28,11 @@ URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' +class DataType(object): + NGRAM = 1 + SEQ = 2 + + def word_count(f, word_freq=None): if word_freq is None: word_freq = collections.defaultdict(int) @@ -69,7 +74,7 @@ def build_dict(min_word_freq=50): return word_idx -def reader_creator(filename, word_idx, n): +def reader_creator(filename, word_idx, n, data_type): def reader(): with tarfile.open( paddle.v2.dataset.common.download( @@ -79,16 +84,27 @@ def reader_creator(filename, word_idx, n): UNK = word_idx[''] for l in f: - l = [''] + l.strip().split() + [''] - if len(l) >= n: + if DataType.NGRAM == data_type: + assert n > -1, 'Invalid gram length' + l = [''] + l.strip().split() + [''] + if len(l) >= n: + l = [word_idx.get(w, UNK) for w in l] + for i in range(n, len(l) + 1): + yield tuple(l[i - n:i]) + elif DataType.SEQ == data_type: + l = l.strip().split() l = [word_idx.get(w, UNK) for w in l] - for i in range(n, len(l) + 1): - yield tuple(l[i - n:i]) + src_seq = [word_idx['']] + l + trg_seq = l + [word_idx['']] + if n > 0 and len(src_seq) > n: continue + yield src_seq, trg_seq + else: + assert False, 'Unknow data type' return reader -def train(word_idx, n): +def train(word_idx, n, data_type=DataType.NGRAM): """ imikolov training set creator. @@ -97,15 +113,18 @@ def train(word_idx, n): :param word_idx: word dictionary :type word_idx: dict - :param n: sliding window size + :param n: sliding window size if type is ngram, otherwise max length of sequence :type n: int + :param data_type: data type (ngram or sequence) + :type data_type: member variable of DataType (NGRAM or SEQ) :return: Training reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n) + return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n, + data_type) -def test(word_idx, n): +def test(word_idx, n, data_type=DataType.NGRAM): """ imikolov test set creator. @@ -114,12 +133,15 @@ def test(word_idx, n): :param word_idx: word dictionary :type word_idx: dict - :param n: sliding window size + :param n: sliding window size if type is ngram, otherwise max length of sequence :type n: int + :param data_type: data type (ngram or sequence) + :type data_type: member variable of DataType (NGRAM or SEQ) :return: Test reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n) + return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n, + data_type) def fetch(): diff --git a/python/paddle/v2/dataset/ptb.py b/python/paddle/v2/dataset/ptb.py deleted file mode 100644 index ea68daf9c6..0000000000 --- a/python/paddle/v2/dataset/ptb.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -langauge model's simple dataset. - -This module will download dataset from -http://www.fit.vutbr.cz/~imikolov/rnnlm/ and parse training set and test set -into paddle reader creators. -""" -import paddle.v2.dataset.common -import collections -import tarfile - -__all__ = ['train', 'test', 'build_dict'] - -URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' -MD5 = '30177ea32e27c525793142b6bf2c8e2d' - - -def word_count(f, word_freq=None): - if word_freq is None: - word_freq = collections.defaultdict(int) - - for l in f: - for w in l.strip().split(): - word_freq[w] += 1 - word_freq[''] += 1 - word_freq[''] += 1 - - return word_freq - - -def build_dict(min_word_freq=50): - """ - Build a word dictionary from the corpus, Keys of the dictionary are words, - and values are zero-based IDs of these words. - """ - train_filename = './simple-examples/data/ptb.train.txt' - test_filename = './simple-examples/data/ptb.valid.txt' - with tarfile.open( - paddle.v2.dataset.common.download( - paddle.v2.dataset.imikolov.URL, 'imikolov', - paddle.v2.dataset.imikolov.MD5)) as tf: - trainf = tf.extractfile(train_filename) - testf = tf.extractfile(test_filename) - word_freq = word_count(testf, word_count(trainf)) - if '' in word_freq: - # remove for now, since we will set it as last index - del word_freq[''] - - word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) - - word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) - words, _ = list(zip(*word_freq_sorted)) - word_idx = dict(zip(words, xrange(len(words)))) - word_idx[''] = len(words) - - return word_idx - - -def reader_creator(filename, reader_type, word_idx, n=-1): - def reader(): - with tarfile.open( - paddle.v2.dataset.common.download( - paddle.v2.dataset.imikolov.URL, 'imikolov', - paddle.v2.dataset.imikolov.MD5)) as tf: - f = tf.extractfile(filename) - - UNK = word_idx[''] - - for l in f: - if 'ngram' == reader_type: - assert n > -1, 'Invalid gram length' - l = [''] + l.strip().split() + [''] - if len(l) < n: continue - l = [word_idx.get(w, UNK) for w in l] - for i in range(n, len(l) + 1): - yield tuple(l[i - n:i]) - elif 'seq' == reader_type: - l = l.strip().split() - l = [word_idx.get(w, UNK) for w in l] - src_seq = [word_idx['']] + l - trg_seq = l + [word_idx['']] - yield src_seq, trg_seq - - return reader - - -def ngram_train(word_idx, n): - """ - ptb ngram type training set creator. - - It returns a reader creator, each sample in the reader is a word ID - tuple. - - :param word_idx: word dictionary - :type word_idx: dict - :param n: sliding window size - :type n: int - :return: Training reader creator - :rtype: callable - """ - return reader_creator('./simple-examples/data/ptb.train.txt', 'ngram', - word_idx, n) - - -def ngram_test(word_idx, n): - """ - ptb ngram test set creator. - - It returns a reader creator, each sample in the reader is a word ID - tuple. - - :param word_idx: word dictionary - :type word_idx: dict - :param n: sliding window size - :type n: int - :return: Test reader creator - :rtype: callable - """ - return reader_creator('./simple-examples/data/ptb.valid.txt', 'ngram', - word_idx, n) - - -def seq_train(word_idx): - """ - ptb sequence type training set creator. - - It returns a reader creator, each sample in the reader is a word ID - pair. - - :param word_idx: word dictionary - :type word_idx: dict - :return: Test reader creator - :rtype: callable - """ - return reader_creator('./simple-examples/data/ptb.train.txt', 'seq', - word_idx) - - -def seq_test(word_idx): - """ - ptb sequence type test set creator. - - It returns a reader creator, each sample in the reader is a word ID - pair. - - :param word_idx: word dictionary - :type word_idx: dict - :return: Test reader creator - :rtype: callable - """ - return reader_creator('./simple-examples/data/ptb.valid.txt', 'seq', - word_idx) - - -def fetch(): - paddle.v2.dataset.common.download(URL, "imikolov", MD5) diff --git a/python/paddle/v2/dataset/tests/imikolov_test.py b/python/paddle/v2/dataset/tests/imikolov_test.py index 009e55243a..4e52810e6b 100644 --- a/python/paddle/v2/dataset/tests/imikolov_test.py +++ b/python/paddle/v2/dataset/tests/imikolov_test.py @@ -13,10 +13,37 @@ class TestMikolov(unittest.TestCase): n = 5 self.check_reader(paddle.v2.dataset.imikolov.train(WORD_DICT, n), n) + first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ + 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ + 'rake regatta rubens sim snack-food ssangyong swapo wachter' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.imikolov.train( + WORD_DICT, n=-1, + data_type=paddle.v2.dataset.imikolov.DataType.SEQ)(): + read_line = l[0][1:] + break + self.assertEqual(first_line, read_line) + def test_test(self): n = 5 self.check_reader(paddle.v2.dataset.imikolov.test(WORD_DICT, n), n) + first_line = 'consumers may want to move their telephones a little '\ + 'closer to the tv set' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.imikolov.test( + WORD_DICT, n=-1, + data_type=paddle.v2.dataset.imikolov.DataType.SEQ)(): + read_line = l[0][1:] + break + self.assertEqual(first_line, read_line) + def test_total(self): _, idx = zip(*WORD_DICT.items()) self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1) diff --git a/python/paddle/v2/dataset/tests/ptb_test.py b/python/paddle/v2/dataset/tests/ptb_test.py deleted file mode 100644 index 5e584a734d..0000000000 --- a/python/paddle/v2/dataset/tests/ptb_test.py +++ /dev/null @@ -1,53 +0,0 @@ -import paddle.v2.dataset.ptb -import unittest - -WORD_DICT = paddle.v2.dataset.ptb.build_dict() - - -class TestMikolov(unittest.TestCase): - def check_reader(self, reader, n): - for l in reader(): - self.assertEqual(len(l), n) - - def test_ngram_train(self): - n = 5 - self.check_reader(paddle.v2.dataset.ptb.ngram_train(WORD_DICT, n), n) - - def test_ngram_test(self): - n = 5 - self.check_reader(paddle.v2.dataset.ptb.ngram_test(WORD_DICT, n), n) - - def test_seq_train(self): - first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ - 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ - 'rake regatta rubens sim snack-food ssangyong swapo wachter' - first_line = [ - WORD_DICT.get(ch, WORD_DICT['']) - for ch in first_line.split(' ') - ] - for l in paddle.v2.dataset.ptb.seq_train(WORD_DICT)(): - read_line = l[0][1:] - break - - self.assertEqual(first_line, read_line) - - def test_seq_test(self): - first_line = 'consumers may want to move their telephones a little '\ - 'closer to the tv set' - first_line = [ - WORD_DICT.get(ch, WORD_DICT['']) - for ch in first_line.split(' ') - ] - for l in paddle.v2.dataset.ptb.seq_test(WORD_DICT)(): - read_line = l[0][1:] - break - - self.assertEqual(first_line, read_line) - - def test_total(self): - _, idx = zip(*WORD_DICT.items()) - self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1) - - -if __name__ == '__main__': - unittest.main() From fa288b70afe934cafe368429b85227c67dfc5171 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Thu, 4 May 2017 10:45:33 -0700 Subject: [PATCH 17/48] Ongong work --- Dockerfile | 25 +++-- doc/howto/dev/write_docs_cn.rst | 9 +- doc/howto/dev/write_docs_en.rst | 77 +++++++++++++ paddle/scripts/docker/build.sh | 106 +++++++++++++----- paddle/scripts/tools/build_docs/build_docs.sh | 42 ++----- 5 files changed, 180 insertions(+), 79 deletions(-) create mode 100644 doc/howto/dev/write_docs_en.rst diff --git a/Dockerfile b/Dockerfile index c3ad0c9c2f..d24042d63e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # A image for building paddle binaries # Use cuda devel base image for both cpu and gpu environment -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 MAINTAINER PaddlePaddle Authors ARG UBUNTU_MIRROR @@ -23,11 +23,13 @@ ENV HOME /root COPY ./paddle/scripts/docker/root/ /root/ RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 swig doxygen && \ + apt-get install -y \ + git python-pip python-dev openssh-server bison \ + wget unzip tar xz-utils bzip2 gzip coreutils \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-numpy python-matplotlib gcc g++ gfortran \ + automake locales clang-format-3.8 swig doxygen cmake \ + clang-3.8 llvm-3.8 libclang-3.8-dev && \ apt-get clean -y # git credential to skip password typing @@ -51,11 +53,12 @@ RUN pip install --upgrade pip && \ RUN apt-get install -y libssl-dev libffi-dev RUN pip install certifi urllib3[secure] -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -VOLUME ["/woboq_out"] +# Install woboq_codebrowser to /woboq +RUN git clone https://github.com/woboq/woboq_codebrowser /woboq && \ + (cd /woboq \ + cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ + -DCMAKE_BUILD_TYPE=Release . \ + make) # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd diff --git a/doc/howto/dev/write_docs_cn.rst b/doc/howto/dev/write_docs_cn.rst index d536f53abc..5d1297d079 100644 --- a/doc/howto/dev/write_docs_cn.rst +++ b/doc/howto/dev/write_docs_cn.rst @@ -2,14 +2,13 @@ 如何贡献/修改文档 ################## -PaddlePaddle的文档包括英文文档 ``doc`` 和中文文档 ``doc_cn`` 两个部分。文档都是通过 `cmake`_ 驱动 `sphinx`_ 编译生成，生成后的文档分别存储在编译目录的 ``doc`` 和 ``doc_cn`` 两个子目录下。 +PaddlePaddle的文档文件都在 :code:`doc` 这个子目录里。源文件是 `RST `_ 格式的。 在编译PaddlePaddle源码的时候，可以选择让 `cmake`_ 调用 `sphinx `_ 从 RST 文件生成 HTML 格式的文档。 如何构建PaddlePaddle的文档 ========================== -PaddlePaddle的文档构建有直接构建和基于Docker构建两种方式，我们提供了一个构建脚本build_docs.sh来进行构建。 -PaddlePaddle文档需要准备的环境相对较复杂，所以我们推荐使用基于Docker来构建PaddlePaddle的文档。 +为了简化大家安装文档构建工具的过程，我们提供一个Docker image。 使用Docker构建PaddlePaddle的文档 @@ -74,5 +73,5 @@ PaddlePaddle文档使用 `sphinx`_ 自动生成，用户可以参考sphinx教程 -.. _cmake: https://cmake.org/ -.. _sphinx: http://www.sphinx-doc.org/en/1.4.8/ +.. _cmake: +.. _sphinx: diff --git a/doc/howto/dev/write_docs_en.rst b/doc/howto/dev/write_docs_en.rst new file mode 100644 index 0000000000..65e7edca94 --- /dev/null +++ b/doc/howto/dev/write_docs_en.rst @@ -0,0 +1,77 @@ +############### +Build Documents +############### + +Document files of PaddlePaddle are in sub-directory :code:`doc`. Source files are in `RST `_ format. We can build the document by letting `cmake`_ invoke `sphinx `_ to convert RST files into HTML files. + + +How to Build Documents +====================== + +To save the time of installing building tools, we provide a Docker image. + + +使用Docker构建PaddlePaddle的文档 +-------------------------------- + +使用Docker构建PaddlePaddle的文档，需要在系统里先安装好Docker工具包。Docker安装请参考 `Docker的官网 `_ 。安装好Docker之后可以使用源码目录下的脚本构建文档，即 + +.. code-block:: bash + + cd TO_YOUR_PADDLE_CLONE_PATH + cd paddle/scripts/tools/build_docs + bash build_docs.sh with_docker + +编译完成后，会在当前目录生成两个子目录\: + +* doc 英文文档目录 +* doc_cn 中文文档目录 + +打开浏览器访问对应目录下的index.html即可访问本地文档。 + + + +直接构建PaddlePaddle的文档 +-------------------------- + +因为PaddlePaddle的v2 api文档生成过程依赖于py_paddle Python包，用户需要首先确认py_paddle包已经安装。 + +.. code-block:: bash + + python -c "import py_paddle" + +如果提示错误，那么用户需要在本地编译安装PaddlePaddle，请参考 `源码编译文档 `_ 。 +注意，用户在首次编译安装PaddlePaddle时，请将WITH_DOC选项关闭。在编译安装正确之后，请再次确认py_paddle包已经安装，即可进行下一步操作。 + +如果提示正确，可以执行以下命令编译生成文档，即 + +.. code-block:: bash + + cd TO_YOUR_PADDLE_CLONE_PATH + cd paddle/scripts/tools/build_docs + bash build_docs.sh local + +编译完成之后，会在当前目录生成两个子目录\: + +* doc 英文文档目录 +* doc_cn 中文文档目录 + +打开浏览器访问对应目录下的index.html即可访问本地文档。 + + +如何书写PaddlePaddle的文档 +========================== + +PaddlePaddle文档使用 `sphinx`_ 自动生成，用户可以参考sphinx教程进行书写。 + +如何更新www.paddlepaddle.org文档 +================================ + +开发者给PaddlePaddle代码增加的注释以PR的形式提交到github中，提交方式可参见 `贡献文档 `_ 。 +目前PaddlePaddle的develop分支的文档是自动触发更新的，用户可以分别查看最新的 `中文文档 `_ 和 +`英文文档 `_ 。 + + + +.. _cmake: https://cmake.org/ +.. _sphinx: http://www.sphinx-doc.org/en/1.4.8/ diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index bc8eef4ea8..3b458b8864 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -4,9 +4,9 @@ set -xe # Set BASE_IMAGE according to env variables if [ ${WITH_GPU} == "ON" ]; then - BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu14.04" + BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04" else - BASE_IMAGE="ubuntu:14.04" + BASE_IMAGE="ubuntu:16.04" fi DOCKERFILE_GPU_ENV="" @@ -22,6 +22,20 @@ cd /paddle/build # build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true +cat < /paddle/build/Dockerfile <]" - echo "This script generates doc and doc_cn in the script's directory." - echo "These are common commands used in various situations:" - echo " with_docker build doc and doc_cn with docker" - echo " local build doc and doc_cn locally" -} - - -case "$1" in - "with_docker") - docker run --rm -v $PWD/../../../../:/paddle \ - -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_DOC=ON" paddledev/paddle:dev - ;; - "local") - mkdir -p doc - mkdir -p doc_cn - PADDLE_SOURCE_DIR=$PWD/../../../../ - mkdir -p $PADDLE_SOURCE_DIR/build_doc - pushd $PADDLE_SOURCE_DIR/build_doc - cmake .. -DWITH_DOC=ON - make paddle_docs paddle_docs_cn - popd - cp -r $PADDLE_SOURCE_DIR/build_doc/doc/en/html/* doc - cp -r $PADDLE_SOURCE_DIR/build_doc/doc/cn/html/* doc_cn - rm -rf $PADDLE_SOURCE_DIR/build_doc - ;; - "--help") - usage - ;; - *) - usage - ;; -esac +docker run --rm \ + -v $(git rev-parse --show-toplevel):/paddle \ + -e "WITH_GPU=OFF" \ + -e "WITH_AVX=ON" \ + -e "WITH_DOC=ON" \ + -e "WOBOQ=ON" \ + ${1:-"paddledev/paddle:dev"} From 6066b36e1d3a1ff7afb3bfe0295ec6c3b6dd1af7 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 7 May 2017 05:42:47 -0700 Subject: [PATCH 18/48] Update base image --- paddle/api/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index 4d0dacae90..f2c56a4a36 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -92,7 +92,9 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so # TODO(yuyang18) : make wheel name calculated by cmake add_custom_target(python_api_wheel ALL DEPENDS ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so) -install(DIRECTORY ${PROJ_ROOT}/paddle/dist/ DESTINATION opt/paddle/share/wheels) +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ + DESTINATION opt/paddle/share/wheels +) if(WITH_TESTING) IF(NOT PY_PIP_FOUND) From e84e69e1416c33db368fcace76d7db1791d6264b Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 7 May 2017 07:27:32 -0700 Subject: [PATCH 19/48] Update Dockerfile and build.sh --- paddle/api/CMakeLists.txt | 2 +- paddle/scripts/docker/build.sh | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index f2c56a4a36..1cec77c0ca 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -92,7 +92,7 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so # TODO(yuyang18) : make wheel name calculated by cmake add_custom_target(python_api_wheel ALL DEPENDS ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so) -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ +install(DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/dist/ DESTINATION opt/paddle/share/wheels ) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 3b458b8864..9739ec9555 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -128,17 +128,6 @@ Generate /paddle/build/Dockerfile ... ======================================== EOF -if [ ${WITH_GPU} == "ON" ]; then - BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu14.04" - # additional packages to install when building gpu images - GPU_DOCKER_PKG="python-pip python-dev" -else - BASE_IMAGE="python:2.7.13-slim" - # FIXME: Python base image uses different python version than WITH_GPU - # need to change PYTHONHOME to /usr/local when using python base image - CPU_DOCKER_PYTHON_HOME_ENV="ENV PYTHONHOME /usr/local" -fi - cat > /paddle/build/Dockerfile < From e7ae0ecd69831f1cd7560fa16e057eca96551f8e Mon Sep 17 00:00:00 2001 From: qijun Date: Mon, 8 May 2017 15:34:22 +0800 Subject: [PATCH 20/48] update some links --- doc/getstarted/concepts/use_concepts_cn.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst index ea0b78f3c3..e63ca11102 100644 --- a/doc/getstarted/concepts/use_concepts_cn.rst +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -2,9 +2,9 @@ 基本使用概念 ############ -PaddlePaddle是源于百度的一个深度学习平台。PaddlePaddle为深度学习研究人员提供了丰富的API，可以轻松的完成神经网络配置，模型训练等任务。 +PaddlePaddle是源于百度的一个深度学习平台。PaddlePaddle为深度学习研究人员提供了丰富的API，可以轻松地完成神经网络配置，模型训练等任务。 这里将介绍PaddlePaddle的基本使用概念，并且展示了如何利用PaddlePaddle来解决一个经典的线性回归问题。 -在使用该文档之前，请参考 `安装文档 `_ 完成PaddlePaddle的安装。 +在使用该文档之前，请参考 `安装文档 <../build_and_install/index_cn.html>`_ 完成PaddlePaddle的安装。 配置网络 @@ -102,7 +102,7 @@ PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和 parameters=parameters, update_equation=optimizer) -其中，trianer接收三个参数，包括神经网络拓扑结构，神经网络参数以及迭代方程。 +其中，trainer接收三个参数，包括神经网络拓扑结构、神经网络参数以及迭代方程。 在搭建神经网络的过程中，我们仅仅对神经网络的输入进行了描述。而trainer需要读取训练数据进行训练，PaddlePaddle中通过reader来加载数据。 @@ -137,6 +137,7 @@ PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和 event_handler=event_handler, num_passes=100) +关于PaddlePaddle的更多使用方法请参考 `进阶指南 <../../howto/index_cn.html>`_。 线性回归完整示例 ============== @@ -146,4 +147,4 @@ PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和 .. literalinclude:: src/train.py :linenos: -有关线性回归的实际应用，可以参考Paddle book的 `第一章节 `_ \ No newline at end of file +有关线性回归的实际应用，可以参考PaddlePaddle book的 `第一章节 `_。 \ No newline at end of file From 191a3268da8c22bab5bbdeced5e684fefcf7c8d3 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Mon, 8 May 2017 17:42:10 -0700 Subject: [PATCH 21/48] fix according to comments --- doc/design/cluster_train/data_dispatch.md | 4 ++-- doc/design/cluster_train/master_process.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/design/cluster_train/data_dispatch.md b/doc/design/cluster_train/data_dispatch.md index f60c3b843d..241902cca4 100644 --- a/doc/design/cluster_train/data_dispatch.md +++ b/doc/design/cluster_train/data_dispatch.md @@ -23,8 +23,8 @@ 在数据集可以被训练之前，文件需要预先被转换成PaddlePaddle集群内部的存储格式（RecordIO）。我们提供两个转换方式： -- 提供给用户本地转换的库，用户可以编写程序完成转换。 -- 用户可以上传自己的数据集，在集群运行MapReduce job完成转换。 +1. 用户在本地转换好再上传 +1. 用户上传数据后，在机群上运行转换程序 转换生成的文件名会是以下格式： diff --git a/doc/design/cluster_train/master_process.md b/doc/design/cluster_train/master_process.md index 949811b4f7..e0be8df634 100644 --- a/doc/design/cluster_train/master_process.md +++ b/doc/design/cluster_train/master_process.md @@ -1,12 +1,12 @@ # Design Doc: Master Process -For an overview of master process' role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master process in more details. The master will be implemented in [golang](https://golang.org/). +For an overview of master process' role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master process in more details. The master will be implemented in [Go](https://golang.org/). ## Dataset -A dataset is represented by a list of files in *RecordIO* format on the distributed filesystem, each RecordIO file consists of multiple *blocks*, and each block has multiple data instances. +A dataset is a list of files in *RecordIO* format. A RecordIO file consists of chunks, whereas each chunk consists some records. ## Task Queue @@ -14,7 +14,7 @@ As mentioned in [distributed training design doc](./README.md), a *task* is a da ### Task Queue Creation -1. Each trainer will make an RPC call (using [golang rpc](https://golang.org/pkg/net/rpc/)) to the master process, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master process the same dataset, only the first RPC call will be honored. +1. Each trainer will make an RPC call (using Go's [rpc](https://golang.org/pkg/net/rpc/) package) to the master process, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master process the same dataset, only the first RPC call will be honored. The RPC interface is: ```go From adb6d43e0bbcc75c1a051bd4a64fc66779d79454 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Mon, 8 May 2017 18:34:50 -0700 Subject: [PATCH 22/48] fix according to comments --- doc/design/cluster_train/master_process.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/design/cluster_train/master_process.md b/doc/design/cluster_train/master_process.md index e0be8df634..d9c6655954 100644 --- a/doc/design/cluster_train/master_process.md +++ b/doc/design/cluster_train/master_process.md @@ -87,3 +87,5 @@ During the RPC call the master will do the following: ### Task Retry Logic When a task is dispatched to the trainer, the master will schedule a function for execution after the timeout duration (based on the moving average of task completion time). If the task entry in still in the pending queue, its timeout counter will increase by one, and the task will be moved to todo queue. If the timeout counter is above the threshold, the master will log the error and discard the task. + +Please note that since a timed out task could be completed after it has been dispatched for retry, so it is possible for a task to be processed multiple times. We do not try to prevent it from happening since it's fine to train on the same task multiple times due to the stochastic nature of the stochastic gradient decent algorithm. From a6f248f5476f0a5ce38bbdd2e85276d929d3f55c Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 9 May 2017 13:54:57 -0700 Subject: [PATCH 23/48] change master process to master server process --- doc/design/cluster_train/README.md | 26 +++++++++++----------- doc/design/cluster_train/master_process.md | 10 ++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/design/cluster_train/README.md b/doc/design/cluster_train/README.md index b88a8f382b..74961f8005 100644 --- a/doc/design/cluster_train/README.md +++ b/doc/design/cluster_train/README.md @@ -15,7 +15,7 @@ This poses technical challenges to PaddlePaddle: A training job will be created once user asks Paddle cloud to train a model. The training job is made up of different processes that collaboratively consume data and produce a trained model. There are three kinds of processes: -1. the *master process*, which dispatches tasks to +1. the *master server process*, which dispatches tasks to 1. one or more *trainer processes*, which run distributed training and synchronize gradients/models via 1. one or more *parameter server processes*, where each holds a shard of the global model, and receive the uploaded gradients from every *trainer process*, so they can run the optimize functions to update their parameters. @@ -27,9 +27,9 @@ By coordinating these processes, PaddlePaddle supports use both Synchronize Stoc When training with sync SGD, parameter servers wait for all trainers to finish gradients update and then send the updated parameters to trainers, training can not proceed until the trainer received the updated parameters. This creates a synchronization point between trainers. When training with async SGD, each trainer upload gradient and download new parameters individually, without the synchronization with other trainers. Using asyc SGD will be faster in terms of time per pass, but have more noise in gradient since trainers are likely to have a stale model. -### Master Process +### Master Server Process -The master process will: +The master server process will: - Partition a dataset into [tasks](#task) and dispatch tasks to trainers. - Keep track of training progress on the dataset with [task queue](#task-queue). A training job will iterate on the dataset for a full pass until it goes into next pass. @@ -41,11 +41,11 @@ A task is a data shard to be trained. The total number of tasks will be much big #### Task Queue -The master process has three task queues to track training progress. As illustrated in the graph below, Job A and Job B both have one master process. Each master process has three task queues. +The master server has three task queues to track training progress. As illustrated in the graph below, Job A and Job B both have one master server. Each master server process has three task queues. -- The todo queue holds tasks to be dispatched. When a job starts, the master process fills in the todo queue with all tasks. +- The todo queue holds tasks to be dispatched. When a job starts, the master server fills in the todo queue with all tasks. - The pending queue holds tasks that are currently training by trainers. - the done queue holds tasks that are already trained. @@ -54,10 +54,10 @@ The life cycle of a single task is illustrated below: 1. When a new pass of training starts, all tasks will be placed in the todo queue. -1. The master process will dispatch few tasks to each trainer at a time, puts them in the pending queue and waits for completion. -1. The trainer will work on its tasks and tell the master process once a task is completed. The master process will dispatch a new task to that trainer. -1. If a task timeout. the master process will move it back to the todo queue. The timeout count will increase by one. If the timeout count is above a threshold, the task is likely to cause a trainer to crash, so it will be discarded. -1. The master process will move completed task to the done queue. When the todo queue is empty, the master process will start a new pass by moving all tasks in the done queue to todo queue and reset the timeout counter of all tasks to zero. +1. The master server will dispatch few tasks to each trainer at a time, puts them in the pending queue and waits for completion. +1. The trainer will work on its tasks and tell the master server once a task is completed. The master server will dispatch a new task to that trainer. +1. If a task timeout. the master server will move it back to the todo queue. The timeout count will increase by one. If the timeout count is above a threshold, the task is likely to cause a trainer to crash, so it will be discarded. +1. The master server will move completed task to the done queue. When the todo queue is empty, the master server will start a new pass by moving all tasks in the done queue to todo queue and reset the timeout counter of all tasks to zero. ### Trainer Process @@ -93,7 +93,7 @@ The communication pattern between the trainers and the parameter servers depends ## Fault Tolerant -The training job will pause if the master processes is dead, or any of the parameter server process is dead. They will be started by [Kubernetes](https://kubernetes.io/) and recover in few minutes. Please refer to [fault recovery](#fault-recovery). +The training job will pause if the master server processes is dead, or any of the parameter server process is dead. They will be started by [Kubernetes](https://kubernetes.io/) and recover in few minutes. Please refer to [fault recovery](#fault-recovery). The training job will continue to make progress if there is at least one training process running. The strategy depends on the type of optimization algorithm: @@ -113,7 +113,7 @@ Now we will introduce how each process recovers from a failure, the graph below -### Master Process +### Master Server Process When the master is started by the Kubernetes, it executes the following steps at startup: @@ -122,7 +122,7 @@ When the master is started by the Kubernetes, it executes the following steps at 1. Watches the trainer prefix keys `/trainer/` on etcd to find the live trainers. 1. Starts dispatching the tasks to the trainers, and updates task queue using an etcd transaction to ensure lock is held during the update. -When the master process is dead for any reason, Kubernetes will restart it. It will be online again with all states recovered from etcd in few minutes. +When the master server process is dead for any reason, Kubernetes will restart it. It will be online again with all states recovered from etcd in few minutes. ### Trainer Process @@ -132,7 +132,7 @@ When the trainer is started by the Kubernetes, it executes the following steps a 1. Generates a unique ID, and sets key `/trainer/` with its contact address as value. The key will be deleted when the lease expires, so the master will be aware of the trainer being online and offline. 1. Waits for tasks from the master to start training. -If trainer's etcd lease expires, it will try set key `/trainer/` again so that the master process can discover the trainer again. +If trainer's etcd lease expires, it will try set key `/trainer/` again so that the master server can discover the trainer again. When a trainer fails, Kuberentes would try to restart it. The recovered trainer would fetch tasks from the TODO queue and go on training. diff --git a/doc/design/cluster_train/master_process.md b/doc/design/cluster_train/master_process.md index d9c6655954..2334f51390 100644 --- a/doc/design/cluster_train/master_process.md +++ b/doc/design/cluster_train/master_process.md @@ -1,6 +1,6 @@ -# Design Doc: Master Process +# Design Doc: Master Server -For an overview of master process' role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master process in more details. The master will be implemented in [Go](https://golang.org/). +For an overview of master server' role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master server in more details. The master will be implemented in [Go](https://golang.org/). ## Dataset @@ -10,18 +10,18 @@ A dataset is a list of files in *RecordIO* format. A RecordIO file consists of c ## Task Queue -As mentioned in [distributed training design doc](./README.md), a *task* is a data shard that the master process assigns to the trainer process to train on. A task consists of one or multiple *blocks* from one or multiple files. The master process maintains *task queues* to track the training progress. +As mentioned in [distributed training design doc](./README.md), a *task* is a data shard that the master server assigns to the trainer process to train on. A task consists of one or multiple *blocks* from one or multiple files. The master server maintains *task queues* to track the training progress. ### Task Queue Creation -1. Each trainer will make an RPC call (using Go's [rpc](https://golang.org/pkg/net/rpc/) package) to the master process, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master process the same dataset, only the first RPC call will be honored. +1. Each trainer will make an RPC call (using Go's [rpc](https://golang.org/pkg/net/rpc/) package) to the master server, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master server the same dataset, only the first RPC call will be honored. The RPC interface is: ```go func (m *RPCServer) ReportDataset(Paths []string, dummy *int) error { } ``` -1. The master process will scan through each RecordIO file to generate the *block index* and know how many blocks does each file have. A block can be referenced by the file path and the index of the block within the file. The block index is in memory data structure that enables fast access to each block, and the index of the block with the file is an integer start from 0, representing the n-th block within the file. +1. The master server will scan through each RecordIO file to generate the *block index* and know how many blocks does each file have. A block can be referenced by the file path and the index of the block within the file. The block index is in memory data structure that enables fast access to each block, and the index of the block with the file is an integer start from 0, representing the n-th block within the file. The definition of the block is: ```go From 7a78e02d5299a6d7ecdac43348d03aa786e80e55 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 9 May 2017 14:04:17 -0700 Subject: [PATCH 24/48] file rename --- .../cluster_train/{master_process.md => master_server.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename doc/design/cluster_train/{master_process.md => master_server.md} (94%) diff --git a/doc/design/cluster_train/master_process.md b/doc/design/cluster_train/master_server.md similarity index 94% rename from doc/design/cluster_train/master_process.md rename to doc/design/cluster_train/master_server.md index 2334f51390..bb83076525 100644 --- a/doc/design/cluster_train/master_process.md +++ b/doc/design/cluster_train/master_server.md @@ -1,6 +1,6 @@ # Design Doc: Master Server -For an overview of master server' role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master server in more details. The master will be implemented in [Go](https://golang.org/). +For an overview of master server's role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master server in more details. The master will be implemented in [Go](https://golang.org/). ## Dataset From 70c30efcaabb030c9c07045ad3f9cb0a133183d4 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Wed, 10 May 2017 10:03:49 +0800 Subject: [PATCH 25/48] image preprcoess module --- python/paddle/v2/__init__.py | 3 +- python/paddle/v2/image.py | 223 ++++++++++++++++++++++++++ python/paddle/v2/tests/CMakeLists.txt | 3 +- python/paddle/v2/tests/cat.jpg | Bin 0 -> 57218 bytes python/paddle/v2/tests/test_image.py | 42 +++++ python/setup.py.in | 1 + 6 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 python/paddle/v2/image.py create mode 100644 python/paddle/v2/tests/cat.jpg create mode 100644 python/paddle/v2/tests/test_image.py diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 7c8f6ea62f..35131594f0 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -32,11 +32,12 @@ import networks import py_paddle.swig_paddle as api import minibatch import plot +import image __all__ = [ 'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', 'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'dataset', 'reader', - 'topology', 'networks', 'infer', 'plot', 'evaluator' + 'topology', 'networks', 'infer', 'plot', 'evaluator', 'image' ] diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py new file mode 100644 index 0000000000..20dcc96782 --- /dev/null +++ b/python/paddle/v2/image.py @@ -0,0 +1,223 @@ +import numpy as np +try: + import cv2 +except ImportError: + cv2 = None + +from cv2 import resize + +__all__ = [ + "load_image", "resize_short", "to_chw", "center_crop", "random_crop", + "left_right_flip", "simple_transform", "load_and_transform" +] +""" +This file contains some common interface for image preprocess. +Many users are confused about the image layout. We introduce +the image layout firstly. + +- CHW Layout + - The abbreviations: C=channel, H=Height, W=Width + - The default image layout is HWC opened by cv2 or PIL. + PaddlePaddle only support the image layout with CHW. + CHW is simply a transpose of HWC. It must transpose + the input image. + +- Color format: RGB or BGR + OpenCV use BGR color format. PIL use RGB color format. Both + formats can be used for training. But it must be noted that, + the format should be keep consistent between the training and + inference peroid. +""" + + +def load_image(file, is_color=True): + """ + Load an color or gray image from the file path. + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + + :param file: the input image path. + :type file: string + :param is_color: If set is_color True, it will load and + return a color image. Otherwise, it will + load and return a gray image. + """ + flag = cv2.CV_LOAD_IMAGE_COLOR if is_color else \ + cv2.CV_LOAD_IMAGE_GRAYSCALE + im = cv2.imread(file, flag) + return im + + +def resize_short(im, size): + """ + Resize an image so that the length of shorter edge is size. + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + im = resize_short(im, 256) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the shorter edge size of image after resizing. + :type size: int + """ + assert im.shape[-1] == 1 or im.shape[-1] == 3 + h, w = im.shape[:2] + h_new, w_new = size, size + if h > w: + h_new = size * h / w + else: + w_new = size * w / h + im = resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) + return im + + +def to_chw(im, order=(2, 0, 1)): + """ + Transpose the input image order. The image layout is HWC format + opened by cv2 or PIL. Transposed the input image to CHW layouts + by order (2,0,1). + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + im = resize_short(im, 256) + im = to_chw(im) + + :param im: the input image with HWC layout. + :type im: ndarray + :param order: the transposed order. + :type order: tuple|list + """ + assert len(im.shape) == len(order) + im = im.transpose(order) + return im + + +def center_crop(im, size, is_color=True): + """ + Crop the center of image with size. + + Example usage: + + .. code-block:: python + im = center_crop(im, 224) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the cropping size + :type size: int + :param is_color: whether the image is color or not. + :type is_color: bool + """ + h, w = im.shape[:2] + h_start = (h - size) / 2 + w_start = (w - size) / 2 + h_end, w_end = h_start + size, w_start + size + if is_color: + im = im[h_start:h_end, w_start:w_end, :] + else: + im = im[h_start:h_end, w_start:w_end] + return im + + +def random_crop(im, size, is_color=True): + """ + Randomly crop input image with size. + + Example usage: + + .. code-block:: python + im = random_crop(im, 224) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the cropping size + :type size: int + :param is_color: whether the image is color or not. + :type is_color: bool + """ + h, w = im.shape[:2] + h_start = np.random.randint(0, h - size + 1) + w_start = np.random.randint(0, w - size + 1) + h_end, w_end = h_start + size, w_start + size + if is_color: + im = im[h_start:h_end, w_start:w_end, :] + else: + im = im[h_start:h_end, w_start:w_end] + return im + + +def left_right_flip(im): + """ + Flip an image along the horizontal direction. + Return the flipped image. + + Example usage: + + .. code-block:: python + im = left_right_flip(im) + + :paam im: input image with HWC layout + :type im: ndarray + """ + if len(im.shape) == 3: + return im[:, ::-1, :] + else: + return im[:, ::-1, :] + + +def simple_transform(im, resize_size, crop_size, is_train, is_color=True): + """ + Simply data argumentation for traing. These operations includes + resizing, croping and flipping. + + :param im: The input image with HWC layout. + :type im: ndarray + :param resize_size: The shorter edge length of the resized image. + :type resize_size: int + :param crop_size: The cropping size. + :type crop_size: int + :param is_train: Whether it is training or not. + :type is_train: bool + """ + im = resize_short(im, resize_size) + if is_train: + im = random_crop(im, crop_size) + if np.random.randint(2) == 0: + im = left_right_flip(im) + else: + im = center_crop(im, crop_size) + im = to_chw(im) + + return im + + +def load_and_transform(filename, + resize_size, + crop_size, + is_train, + is_color=True): + """ + Load image from the input file `filename` and transform image for + data argumentation. Please refer the `simple_transform` interface + for the transform operation. + + :param filename: The file name of input image. + :type filename: string + :param resize_size: The shorter edge length of the resized image. + :type resize_size: int + :param crop_size: The cropping size. + :type crop_size: int + :param is_train: Whether it is training or not. + :type is_train: bool + """ + im = load_image(filename) + im = simple_transform(im, resize_size, crop_size, is_train, is_color) + return im diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index 5554a37df0..0b8c78b465 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -1 +1,2 @@ -add_python_test(test_v2_api test_data_feeder.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py) +add_python_test(test_v2_api test_data_feeder.py test_parameters.py +test_layer.py test_rnn_layer.py test_topology.py test_image) diff --git a/python/paddle/v2/tests/cat.jpg b/python/paddle/v2/tests/cat.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bc1fbbd371216b9904b522ed302700c79d2e4876 GIT binary patch literal 57218 zcmbTcXH?T&^fefI??LIk1rR9#>Ai#!Nv*34S-X=ZL#@+H5Ob$9ML`|P{V`M30M4ZvzJCU& zUI7dS3rebKsVZv9DS;LL=SRrs=;#<37`d33xDwQv@{jUZ1KNlG}1tk?V4J{o#!}SAgtN?N{3JP*c3Mwi}%Il}2ufGRS zvQe=ME9g;k*tpS%1OgS~vnyyt^_%-SVY5HPl-z?7=;&|U;I?-54vtRF9-a@qynPVJ;E>R;C*cv1iAl*RscFwpIk|cH1%*Y$n93?F zuDS+aTlc1=wXMCQv#Wbx@cq#6$mrPk9Fa7?u(D?8Gzz{!@Bz*zXO)={|njw1?>OEwG3dSAiF+13O0Za z;ByTOwm4ChWV29^1G&sL$D?W+5rz7snkNFI#nfYUAyBkLzPqV#6w1I?s2?(ioJO`>i6jf|D{&5RM_ zJ#OwIiUAAW(MtKNQ2VeqUn3pSJNR$YWNun@s8Fk*2+ zmoS(skO#yh!~<33al=t#xTU~d_>&2Vg zGE*_qqSi=wkdqog%^pj~>Rv;GdWN(PZ@>=6ciTl!IZ9L@Mf&rdB1uumG5q++-D=Yt zpFlXIFHC+tRfwxi8il6MhI%cH0dN(Ou(?j^@aiWx2^1o3PPLl34a8>~zWu_3YXVXa zW7FlVj3j5h1?8$H&b8#FK&Uif^fC$sMz_GBRmviQ5U9AYvJnFmHx0}?6b>|rv@tbJ zbC#;aQi6$=XV?*7-?L=h^@b+AKwDZdlN8XslnU;eN*`E_3WW|!lTgyiwe~4}0l+m7 zm3l=Jj}aIZQJssCqhN71FbKsGY($~JPJ)C2T!?Xwh!`k$v#ci8E>VEJa&o~l2`^E) zpnN=M05Kj>Zn$0ud_MQmHsv~NkzSnKbz}-d;UUFN)U!EF$%Ft&!x6eY_@rHuitUIzmdT>SzHfN_rE zYTREK;Va2$tG8?1vaR1-+8!sSO`e#k!>)stt*F&25vm+WW?BVa*)FLJFoeeJLQ7U3COuuf$R{mDpfw4e&WUII2pL4> z0y)bKRmL1g8m4$MvMT$_ldW(XfbsK(wiyG;cvQe=;D z6{>+N8cC_uS#*xVK*@-8wB5}h_P29>L(ukIKMC}*h)B`A(k){n@h&k}Pf{I70VX}) zPKJWENqhn+zyrzYi;>El(kQNxb+k1IDx8ibfU2M1i;}^E3okK<+B7CGJrn&S^RxgU z7KU`&OJhq=Z~u z9WalXK7z3s6Dkj{BVz{g_#^b{zRDB0u~-5mHPPIH`XvV7GRH$DCq+iKT?l?DRZb;> zG1AWEwDI6>M+d^#AV!XC3e{dEq;YCBD`j;|ob3dxFZz~Bp1nJT1^5K2454-}p%SX0 zub>jH%mp(du)yl9&s7&;qen@241@22ohjQB2Nia9`hV9Tuu)skYfxp zfL+*Ky+&4As4-L(gT`3sZ=>gIV8VJejq>mXyk{H?Lq&t}5CJbo!I=Ol7?(hElSo}i zz7*b;RDN`K!~;?R!QX8(v*P3s@F^I`q7MMhfYzIvubI>hhH)#Wg;dAG*q}q?00e0U zL^=eIxaacCp)}@Qu)=({7Wp3dYBmc$)`MCgEPF_L&XalMI?OCdbJ74;K2klDytEe) z_eGBz2wP85RM!V>!gz|Q(srv$5c%Znd7;&FsQR=lo=|z70A*KE+fY$kuSyHo>vS%z zVb04XV=+UN6HV(n`!EJ`If*ceC}OfwuPRR~<;4FwofKgNPeB%GiEc;^(OawJ)Htl# zCUwm$0}T!W&^urbXsxRzPdNU>iaLoh5d#BR2U?0Wiln7sfj{xTNn?&&U(7%>W&zkD z28ao8&%jworW3#C%_#sv+rVC%q3N?=KJIfrv6U5-Mp_7v`YZ+ku)!d}Ue{DqjFpIo z=;;EV;5dEUXN@Qt0m9Y9CD%1lsK|{%@XC6|=?8PSyEw&?NVv44JPwPiV80~{Ze38W zWE}=C!xzvnAHD13p#lMO$ck%f@kPWuB6$EJ3JxUhqCH^lrA8`QOhdwwitaHCYW ziT;{ID2}skmPdHFh>T#moE3`wAgf?v<}!oeaHCtt_(&9E3dkTv!r(8av_`;n)$91M zFo?9lI3xD;c|1sS*^t(~`ochL;B{pn`ndZ9l+)`iluA>%b>fDkfUN`lW=vez%0bpS zfxIp(@)=;z!XLCJ#!3S4B z^)P~%xUpa{a8eDumk*P@$N(|^I>h>Bgjexk<~&=lA}qW*R}cK@EQ`5@-lxW^L3cP8 zoDL;GSg5i0+h~aD;6JhA|OF%#v7_z;xSk#Qby zCfX zhy`i=16-bUH2b;iWw4EL6^U|xx9o4stsYd&$)E^bjvcOtR*wPQjkc5)=c;Y7&n{cn ztsicPR#dZnYViA*wE|lE9YwOhIo7VB2vu}LHTZ$I3FG zjsq?S&C>>8kpa6)BftT*+9)`rvB8EJ#0AfErX|oZvGOQcJ;Z0|RXocD z)#lB6YLfQmMI?Yp>lmH{lB361Igb}&;7yal8nL#rpq5<}PKN^%&HAJ?Np(mz7zj(b zhcLHwol{t^+vvQpT~dR})Fj3a8TNC4YPKukRs^X-g%9hPVKk9z;ST@-<}lU8?@=tg zTrL8ASZ}qUlz3r=sD2ZbW+sJIHbpC0T{GYS-XfqfUJne&d$|i0mlvRkS>1<2-<&M9 z@Jasz+>FI}4V^ddJF~AUzshq04DwO?&*QjiZdtws+ZO$SJ0(og<}#~C#ygFHJER3} zHDF&R1%Ir;YuYOi#=KTGGps~fLmFpB-FLTxXf-`UUd@ZGX5O=DeM~C2W$ZIG;^1IV zA`ojp@cn=cTl7&o{?IAL_bU2NT_;MWvOh&^O646ROLc6;m2?lH`FZ+;_J>rvhoYi0 z8D}oNX-tN^+a&yj5DR02#cMl94N#&TCVXUZ)vx!1e72XX^vQQ}z(BN6^BL0Ov&rvwuY5oT|i!oD%}4UOelECB{R$*D+S$b+yZo>J3q4jERx~!bp{AgbT;kQLWu(&p@kI z-64WQOJ$LNxP-;tm;q4QiGkB-3G}V&oi$SH@k`k>oD!YmAtr+vG2uXLf11w}2nr_o8}RXzFqv2q~4G@lcl7hRKrrs zio8Q;Hb@C{(Zf8r59e2zVtZyjZn~LET4)*^+auWXKueC_J0_@2M3|mY-ck z*;1C))Hd?|B3B@{yl{^y({suQ*z{I?g=}t5)vVm8Vw{7g^>Ob*IWhJ^h;n!umi(D8 z^E8kHD)9o7hcak^Ae4-e-y|d>^I14KwVh_`2oIthqmO7Jv!bpd zXxjVI=~dKH2`fV=!E1=p(=L1tk73`D6?KJu&32ts#fR!#V+8t58%gb zkkFW&@#ch8YBI8n&ygs0hS256M7y_^_dbc(1X+8YQFlvyeWAtopR^sleGrx3WPsk) zzPl<(Na292N5|lxbY$Pr{tJVYaB<|qeEV9=z5c)irG>VP)GnBcGD$W&sgpj344tLl z(R_M2r|x1RSingNv-cbPq4Os)*)yQLR$=3vX6$Djl3m$>VY^WV>>y6fNO08S>gTJx z^>Z~bY`b(sPJ#5yRE~!bezhb9xD&C+BE})1lWuZxRX4Nnb~3qR{HKhhPFf!}`|Zjb zN`s6FvXM=zA$^cXEYy=_npk>%_fTtEWb|YT7E3BDZx|)a(w(xB~D@YUXXs( z_OMM)r0tV1wpBxigr(s}rM8XDV{pdOW;I!D+r<9r?ySIje9pRn*DgWQy`M7nruMo3 zp7xe6+NL6`bBtvn&Y^%Owc_#l@gu%G%gu9R0l!Y`7oD9)lS#2RcP+uIsPvnUO7iew zpzr88s*6~juaCiDY%i-M@mKo_4W1U{3Is{&VkIn5I-?aE414>Kc0C)eIvtF8!F8;$ z^z83GPa}H%gI;VNa~sb<*xC+vZF|+`6PX6F?5_wj&)n}0NsdDCeaeCbP8sSH@y=on z;;0G*b5z^r8nZg-p+ebsUZwdnW+kXdwcXf|a$fvdE#x^Hy#cJ@VY*x&nRebcK3o{V za1}WSnv$lW%pFl?qN}7BtL2RF`o^THZ=U?!;r@zH|5qLObzSIhb;y(?Z#-Bk{%-Dv z4022yQun1M>j`UQe{(x${97E+$~i)fFVHkIk9p_9c4=zR1iTkwQ|OTae4>?X)^zcr zN&fk#TH`Hzy8(~CKH-<}NqOjNGnY%-K2i`wCq z)cR8oAwKqaQ^0FJihEoWz?O2aJAP4cuJu{M`0W=HR8bUtb-8=iAGcK(ceFY@|3Kk3 z%KHUUljR{_X-JDk>Ii-_w#2xp&&e4=zkG=C`UeP! z{2%^?FGoEJOo=`m^vSrL7p=4?@L$`DC$YV8c2;?A`f{AwrGL-bNcK2se=9?W9@2NQ zThe~K6!anMlecoh8rTmIvsr~d$}r{p)qYfty4+96^45#es2#2ZNC2Ws^8W+9}^vRuquV z_9=mJKv(s^#RjI;+bQ4`Qd&0RLp6mN6WyNhDbW&`WS_>vU5ZdLL$nUL8>*_E*0zl% ze!M$c1t`b6i-yzm-hZ1pN%WA`1LbcXoxZ~GRA+pnY_#M1(_p@xTd?uE4Uux38!mX~ zNA8r?UI}tnXG(?9`|5H&MLfe^WT$2n@(-Ys|H$KWZkN~-@eD7fGx*|<+7?=>tHqee zI!wroJM5h-p5qER_fTsw`7IA0F;en|XKzm{O1RXXQXV<$GaY)U4FT7iS8kXZzKCKe z4n9-cI3_@fvwT}xVk4B=biP;8T|K(z_fi)jahw+vp#J-bO?Cc(aD)iA^j=Rq$E{Li0io?cVOn!Iu zUBxc_|?#6Si{oclK`lNAiZ$}&2e?Lm3ZrI)>i@1 z(v3q{e-f#r_Q5Lwp8({@oz3Z=ax903aWD;ZBb-_Sk*NrF}z4+2~{ z>>d)kwbf4XAxr3mL_@8sjt@Ip(XG_0eOsQ{bvLL^{#m`Vt#G7veiYOY9wL|_PbQy@!s@%~;k@_083`{<_q=~?}#fIWS$R`M+Z zp)DsUSh9Q*vj8Nj*nVMwwKUkz6y$nww1iN%;7I??cM0E;P%j!8*J-?|sJsDESu)ak z$Ss%!OsDiuP~&%}sH|Onmn}HDFq>(J*dDEF6@Is9z@}9yqV%h@)x~+rgS)_V@pp5l zV8zH;rMZ?RV|1~5j)d5JfXcT^kPSCPBcG*P3{+_HAeaYiFfLws@fJ|V zn%n$7AI*%ZXxZ6bpc+MOU#-{$1tGJR*@B>!<)pw@P-nxMaah`9pbbZYk&_InQIg5P zaEz^M$fT3_sklR;dH`|<9>4ygiI@s>DjY}hm<0moWH=a@z{AKu{IoMxxQ{rZ%&H9b z5VKWa7de4%kSDp&2Z!L?=S0B6;8%e*O4dt7*Z%3sEGAA2eyuj)R;v!IVg&Gp}f2JyllBETYRwX`R07u3lxsY z#C>KLVnmM^uFvBU_HP}CsOiGj-|3Rgc8Zk0((7-0#O+WJQy<@#kUG(;M+lL1TW1 zC;vUZwG_n9nOOEd^a{Bgo@wQl-;rLX6B*pE;}WZMKQAJ(^kY!Y+feYWwI>t4y1f(S zHe~^HW0wnA;TskJda?-xX&t#U)aAx_yp#6y$oS~!puop;y;d`eOnmB!V%K1dCZB6G zvMXmP!~X9rrbms2HB@%M4PlV2c!v_&qxz34rD#bMz42~hO7 zneYAo{w77*)#y&@+DY?lE3ZEL;Db6~3qV=wZh64p=^O}reyNZ*!kb`D zG3DZ>aC)dCJ3k)>xc5<%+sl>gv3}r8q!hkLwkAaH=)48Z9vu{0Ch?qO((My@9CsfD zpU{2C&-;l-)gu85{Wl`SlX+j}Y-InP`4+8kmmHWXIsOj-3C`nXkUtaEd9oMic&?7T z$*%PFZsuhtJgVc0;c|nsq>8klXk{GIPSlI`r-$$lYy^Bis|%75O&h)FHHjQ)yK^Jx zMb3MZubJ?iMKrp*j_bNn;|!V-jO5nJikQ3$zmol?N8YDYjNouf>X0rz1lO&07+$S@X4H z|FlLj^SzY9ppXE^%~($8_P1f_1`BGdr$o|WkST6*#8Yc~*cd5%XWTL}48N%(7i~IZ zU%DnnLt=iUGH$k2rC`LRSt>30GJfB-z#)sAY%08##OImu5)bozTFb1+R#H}!Aw7g; z67|fEi&gh)D=YfsTeHPiG8=*C5>*&_qpj*5A!x+6xj@?e?9g1Y<0slBe!BaJLn=#z z?Zkucx90+5A-P99QfyiuFukTkusl|J)%6DeLrCa&0x-ogMHfdtJ5 zA?%-3e@gOYj(|@X1IncDJ~dpmi~7s2BbU-@BRs~<5*&!sgH*_*_a~5ZMIK!Utj+z` z5r9svdRCF;Z)%(Ui|+|%EuZd<2 zCySS87}9>W@FvZXa6P`|hVgAS&D6A=qmc*WmEFwE_1VB#?XWSf;phmi{;5aGZ}*=Z z7ainViuw=r#Fhm*2WeP&a2T{XO(+=5oNfqO&$IbEE17WKXz+M2K=AiWr@-^Au}^$6 zId1qJBD-74dtukplQNi;-B0-xdt6h^vWB?4_hvd;V4hy-MCz;7ko#1rH&0|isNs_P z^KwIe=);oRVt+Yn>pCBe(cZuE#(Dl1>~a>P@^s52LgiYy8Yxqmy{s3Jbr!IH+@Z^ zt&b0{J9*hFnvNRTTP$}|<{|=EDTr*U#pDT9z@@?Cem6AW zPrJLvO6R%F_!Ud&(uHy0luk>wqH4tx7!gwURfElg6Rg>0I4>*7c|&5TDZC;RNFbpH zfsS+UVFE}IpX>|#d7$;EAjaj$X1h?+cFBN5Y_R}(V_GTtYrLv}UDd-nV4^WX36qzS zb!2&90ld!OK5)DAO=J|$Yti*up_`LKqNefI+|nvJZ78r&($hAV7hVk- zCkTLk*U^4NMIbGFeTyj*Z7`)qipH-rN(4P@R)kH}18mtY>@Bq{mPI6Lo`bG0TFZEM z-ja+OeEp0u7w!0=@t}lORQz`b+RwK1cbM!bXND9@JUz6}65{>EJ!J|tS(z8EG&t@( z_(1mN;R&6!K@fG|{vv0Y+{=d`-fb4iSUJshEqUmZp*xRCr*6DH{5ILv;zRpDMX%=& zo7F^fOx^8wTh9}1g*W9wyS?O8=$VBG1s>S9)`LU=21OMxI)q@b$I*3Sr#ymgk)@?b zy&do18L03m3y^2Td zD#t^|`kQvAUTx;n=kLa_A`c*Po7M(G!PXz96 z34A)M@P`9lTkSeMVfz-Fx2g7(MFf}fSdN`JT=ml7);Gd(H+Z|>g}DZWNQomKVu028f|u=QgPrEtQ<5mI&ip!r8)r-?_Us69G>7wwrvIyR{3dj^|;0 z4PWIba;H^Q8?a~14R-Sc@!T!#US!+Qgb$@vn;Fa*7m9oJ`O_l*VZ;`GW{w8j=7DG;V zHCgHK&L{M?0(wIZ%4uz5JDtlfOzzL?DovCGPXm4*S&I+4@K>q9%W(J7e3|-IZb~Z4 ze0q2#RUB4+3+=ZSEB)@VAKF|+nYD}6C_^Y4G)-i)E{w(sG-(m*v01=1IUmU0SV>&)JiO(cGw znSb3RCE(8Ip(-AEnV=YfqbAB*QB9m!pK^ zRO@d(Fk9W2;qMWRBp{~hYQ0h9aiF|Sael-`P-%u#XB+D85;j&0Wgz61$ouosd*KDs zN4XnDlB*GJyV?H$`}H!ik56RC;?I+n-BgHOV|xK@SNA;w%hZ6WmbRVzfnUdi{|4H= zJw0~UDAANvgiHm|?Hm!p&h29nXb8SO;B9uP;L+Ks^4MO12vvUH@gUPkz#3VW70Hui z#&6TBUi;M*IsU8;@-g$fs0o0@S_iKjHf+`Mq4jNQU|V#cU3KzzWlB+IT4{*TsXNWC zgQN?8+6xZnC=d!lc{y}NMhyZKzXq};Q*V-M|ow?a;ej7OQlV~`{iMqx-Em7_3p zL;C#$JpP#EqXbW0$2>!dF_qXl)*C4zs?}!=`5+2VSLAe|#qkO3OKlm*PHz?N$5nIT z2*x2d$nzlFNEJ)u6bKjeD7sjjv3rsAZABgkxzi{~6wp)Pp7C zZGP3} zsUoi>&8|)Gb{ge7-uw%7DbjtHkqg)xzXz0i*460)EyDCz^CRI(K?7@!nve7q1p|-bUsvr4^ zhfU6d_tgIZN|7Zo@8jjrJEb6D*FEi!exb-P?e*8olP}`g{27Bh`2Q?k>AVb4`YHpy z9$JW*v+$>;|NCG9Qy%iXsHkeGUz%egb?uJ{CC?mKoTkF1@n%ChZs1LQ?ewP%YXgc4 z*sVrf&o|V{gi1JD4n$ywf*kiIVebwtjC5Pq@(J7S6!d<$OT5%w8=a#8ZM#yi$kon>-9t5UVtctB~^_~wx$>b3YF zH+^N#Ws<(_wdZvBDC(Qo!hk;0AfUJMep$M0>7%eL!?Zb-*m=-~cFp2ZJ)6_yltp7R ziyckwulb@4)Q3R&2#6r%lNS4Yx#C@`VXMBH>hp*^aHu19#00#FOuR4 z){|O1)Ac51x*%#KRDPHfLfOMkE0}MhdJt@EDw-+nLDBK_ZwdYAJJJCyo7Vh5ZlAfQ zx8yMwr-@C5UO{KKuRtAYN#M#uGZI56*cr<+c+5EI=&ICCcgY+(j2oSk2j7oYoVWa_ zkwK$qHXe8iIN7xsjP~p-Rl`0Mr634O=^TWPtOGdn#T=N~sN~88%%!>Rdp=>$mkCpk%zb~ zz7{HTUO&A!^rI|~=Ye4J`(?`D2m8c@?tP7T*%W;;Uyl`U%EW!x2gUSfa*cTkWX5y{ z!&F{H6f!0yIeHsyzSAcOx@#G$^%9aN_U<>&Q&k1WyLR}YZnx+O3^Kl%+|z0`apvsm z@fqrNDQQxDzFSQ!F;pgUU;FF?!;>#d>{Y;>`cFGcgSwWI%Fp~0uYQv-ri>@rdK3Ub z=K}YFcGzeHTg|wZmEPt2Eujl*rkz{~)3d(mUv|p``t#>^K3RaYf18}uIXEB{%YVF$ z0T;4=nG@Gi)qxW{?z_1ARaO22sE3MsLRD7_N1vFs3`N zZ@2#kK%gWgx9sJjk7i=3F~Its?2{WhnHNLe;w{QnA?AKLOPc)eS+?xXj1-+#sI+W5 z{+wihQr1jD(wMa$EHq?cft!bnItrOT>eDg8s~Sa@PdClQp#5Be8B~b(Ize&F$*H?e z^riPJq`q%^|KRLFR#dKLFi`x_8E#2f7$TfWPdOT+6d`LwM)A*)86|%r=dno*=!W2l zK&YVx7R|~Wq{Cr1ZkDy=7nX2@?l|Fp7+KMPRE_Gjy3 z;&k1mpv}l}-CXOBu_=YpRqcvYwV$FVb#D~VSO?Z(4qlr?H5t5dmi!zT(J^*fnYfz^OrZ>TFP6ms&1){v!FlKlXJz~5 z^u71|I3_sxY3~t3{_~jsbs>XMCA&0oOJwY@jdqICBuQ@m~KUH|fQT9x25mYt?#x=^hDapm=SL z3>(}S^t3zQfwMc_o-L(V}3u(Sli;a~*eJw%#T24dp?vyIRZNSuUr%Tye|nz((OUnSh(S zHqyPqT zxu(&210x9+3@5Tb1H*xt1tzBjI{=@yIo+hR!|}m&&AD#%Jy6rYs|Nm!<>~d@reR|E zPLWf!<#r}W;+d40LXO{4kMEQ%+TCW%{&m)9I~SQB+)O;Y_0ZRpS9mz6D)CorvT4MZ z&fBJ#;SlyapNxOMK0DPu8y{@c)J^F~??3bU+ttVr%Tf7isP&=tGtkJFp%7^jwo!ys z2)A6Gwr?tYG`#e!@4H|ce@yQqhl4vG^{n-FPlR>wJEzL|e(PPL_gb7?IKMKCzndh@ zg%8gu-uVb&A?m!M^5fRKH4c@-^#{jZS)(KAXmnDj*iFDm0=9pU4KbjE-GLG2&laRY z=e*`&E`0wkK|8$nCBgM|Q@T6%JTCfum|NKnCY+90FL)olabX|)*=$3@kujE|Y|9D0 zSHV};--$k@5S+qIezksV5VT?KxVUMP35&h_P^b7=?RQ+W7A-vU zx`W!g%h_K;1T>W*wj!1MhO%0}hj%ufU+KbjHLo(GHWZu<(~jUW z8S4{mZn!uDS@iQWuZ5`um;BTWz3zBME-LdER6Gupb{PreX}&P`@X<rG!E{~5;rQVP9ekZ!8fx6L?j&0L_ULp2oISD;h!k%8z@laH|tkLD!Lt$|_ zF$<;_9W=)0LYtW<2acDo(xbftWoM0DRGz56o@CbJZ9e#Xbm@;!J{qubK1pI=Xw3KO`CL_S9$8Zu`7MbMST4=df;UOxh z8}$dt)jr`T@GK5(%{_3By_5-o2C_l;_)v+9iEJ(e1>sn`-To^WK44A{I(+;TfR zw9nM#L?s(|J_T} zcu{t8X}`7j9lYe8DPH4Fb8W|-u`D!~uB3YFL}Uu;Bayy#TFUBC?83w^To^fh~s`nLxXBa~$-fcijm0J_XLj~;D zPE18hIKHZfY%I6k+Hl{IjnPXHZ9Ww^a}K}S>F0+0fQX_?gQ9g_TEBvu?}`CR)=Hn3 zu;>+Bi&W(2pUItWZ-obf4vx{$*J_C9Y<%eH@pi+Vxi2z!q}nP*hJANr8LjZ2ordJ3 zrFwzaT8ypljV{tiFt9fNVm(IQs_5N@<*~K;TXs#cAkDW&zapa?BvOF9AM6+yHxo?x zJ2iv%GjUpNAYnI=RW*24MFfKtrhjbdK)9 zs42@s{AJi$WKQ@<@1w!`j~$~qG8?}#xh`YVUAzp-XYogxQj_sDGzsN#n^8Y?(qYsZ zsqc+89t!TQ%9}LP)Q|#@w)tJ*pJnzJPwKf%e~3{%XrV~_#nUDdrtoR$2U3bh11a`t8Ud^7KD1g$v?lM^Gn%fjRz%VoX()wfhHFW4i~I+7JG=*|p6S zn|;lfn^Xio+~__ZdEMFY^0C!}vmbwmx$$#Y@ih&8hLI9H4HZjW>B!akD?RxF@{A{j zD-m(g#61&_J!$7{doS7bv1$*@%)IRK+*bU(@?!!rj>K zh~QVywdAijhV1gZ+LO$U12pIHP~F{jGmM%0koD_00rB-kdD@P*qR<>a(brBCHE0?Vce?-uZQ+zaJLMOtE95}KOH!ZO* z6dRCEm4yJD9nS2zoDxz@-Fi?G=mJ)V;VXLT2cFBPh@yAf)q?&Wc*2 za0txAq(=8)){Rh_-&HmL0M2cpTqaBHl+(XOd#z_1)-~=uFAHMwTQyE#ugWrsdDPBe zP@U7#HRWIICiGtSpu6jj<=V%t2w@Mwx`ZI@l3uR zi{P&9?1!6;Q+yA&wq+Yotfltif)=^C_Eei4rYUW*Rw90=y_)^_KQcJX+?2t#Z(Du_ z+3g#1T8btQMo^WX1Um{~vqPE|Yn%)q&_4j#G{tU~wC@8=Gj^?2Vp94S^S22-DR*)Y z%6bnduL7brNS8#O z(P7;lrE61pq~M{K>CN_|*^%D{*`E^pQ%w-!TPR*X)4_TwQ;#eekKbr{J?*#+tyC_|p;eO#lLhG@?8?t1-{>;~!8`m=-jc`Uq_2O?8 zP7Kip#SU7-U&hPu)dtx-dp0z2W(*%J!(=~4mo5dz8p}OOq1GGB zs=sSmn00D!&gxH@3DUw7cqdL%;W$3eC#tIFUy<*9EFPNP>g(U+$n+n%0x4QgJSuB9 zV4S@(wIofedEF&HCFr^xwp%uNtyB09aQ}5GfKxMB?Q*+Cjgpb9gI^;~?{C|~ zhpVi&KG*WM5DXqVzL^W)&>BJ3YQ88$p2aAfZCrR&ouJ}7H#h$#>U=S$WpB{P6ny<| z7xR3jwZR1crZJmVcXZ0iZU3tNWgDD1Fz0ZSBBMs<@9VjaHqjma8Ug2*--6NcA5RFL zX6VNv&x}>7ce%o*#bVwy8W}#adc-qZ_l7ZuEsR>=PoNFd*YoMYuxTH^Qij4Qrw0_e z$M>f%h+2J<(eC-A0Y!&ssTA@^;-{DPYxN=e5w5YR*)y=($0|AIJ8@)JG^b;AN-_`3 zYE*esw3BwOkdH1LzSYtgv@(YAk6zW?=E^nDHhRNmf} zDR~pjD@CK}t@;n3-8QFh{^3i628|W*3O}I=zz|>xUM&3cX>h@=1hlm zCAoI@ZOqBj?f$#ZNq1j$Aj4|gNIWqj%5)=eb((nP6G(cx_>AL%tX$Th5PI z#%A?h)%C1geHhznN5`7AZA&XWxIm62u(<8oV?iPgPayVEX{#M-`*x@XJ!j+JU2LzR zBRxF6=ae0XSuMf?TTKiewLdBT0p{&I$1Uo^>>UTlw(pR=*xg-3VU5xxJrpN`HQ!(X zto5ycaTYP=i{0JKMvqm?ET=K8F1zPX6#25Ongqjt|8;D+|2w;lM-;(6fd<;Y_U}A0Z#U6i%OsWI zKphbY(=vZb=L>lg6p3k;U9n~el4k7uxbbfj@9Pb=$70S)T@xY z<}W&O!aYelx~%c7vaVqRAp(iu#+6xUh|*PkeR|Ue7gIiI#pc4*+`W-{!QfwH_Eqhj z&ul06g=w}wrW1Z8C|r`NWfHrDQy4Vd6}Ik5Y`Yz<#3YdnhVxT{ zEf=lBxun7P!J48Ee;(N=zq$EGTEgydja-KM(}$`DrIwNTLOgE^vY;m^o`Ii~(0@50 zDOJ&yWmcrMF7AXaahDD0;ERQVgZ+_777v&IvL8~ibBm_iO@H#-(6Mo5iv8&nG<{sU z)F>*^p|&UWS$~xzBo}tmX7RDzKLAg~&*Y8($DEqO(In4IgVJ~Dh`~^*HAZHxz|wps zk{XS$Crtti<@~CWMA|FSj`?p-TOYFHFU%QZ_!dE6#eRvg>}=A3s}U0H_Yl;Tuyzp} zQdi=!ryP$k z?KS6Xscd7IJfzxg;PIA0$US;?4@e%5-Ip#b7f$WK8LPp5r_Fex8_S1M z7`ivGW$M8nH78N7!@5zR~FT;UnVyH9$>4#sCRb4jD=^m!tIR*qn=eFj0!;0OHb)AthhcX)6 zjuA**L@_}v7$XZaO1rczS}$f@ulYD+AfDBPZx6=2v9A(&PHV_1RE@6%R153kgkW<# z01IM%oni!zIz_^&8%=U0#RWpixz5;}nU^#8KlhZ9B@NJ#~`Jo+AJFoC3{vnXB>WJ}nS-z)@u(Ei1s?U*~)7A6i^M%brCC+xQ3F zo02)|-Ya3?gQ=RWEI+N5kuQHaDN<#6udO9>Ny=yw8;cXqoyXHtw8~7a&XBivr(EB! zKjcsS#DASNPYr>Tg4wD5^Lo7XIkvT`(~bAi)`f*#*v{kl|CXNJ={#_|(|*#iSmQDB z4^a1v&Nh4T@L>0JNchh(x%S~9?WuO_`GiSZ%)SEO&ox_g{G!}MS-uHnM)Alm(k#T# z=4fnX<>%Nn?Ozh_3V5Qa?QB_wUz|H}XyscGYimE`opFaL$p#2s_!xirT%j`)6Y3nw z0{*76RpGC9r87N;RoecsqWGf@GW{xcpx#-hXhlGtX8#{xu{Q73Hy68n!bivV_JLAs ziMfAtaC-F#Kcj3t^Sv%l$s0V4IQz8>T%!Ua;g9vx91ljVfgbzLcg?vzf9je_o3t16 z-Wi?={62l6tdG##eVFBa9wB;k-9qe%+!86%1X9KEr|RlhE!3JY+-kO_^P+Wpq9*va zZs1-<(Yfd|dKD{KTD6gm>%XMC=l=iKmJ@Rw>%~evPQDg%tyI}X@{Bu{AA1s7? z+qW_NIQ==QyH+-725^~d`kYZ9iK`5lwa7hNulfG~dZ@`eJhI4sRNOY7Z%@*ti*sqg zM{~jS8%Amsk1R9Fr_2Yh`OQ_^Ng3MAo-zw#I3tzF>Hc$8%a1UCJK+BSpUSJeeqQFs zIl%P>pN3fa4u2u*`qd#aPm&Ze0fHABvCwrqdXrFxSjxWF+(r~&Mv*pVnL$FzX;}0Q!GAnf zWb>`y^2_;4*uXb&@3>KN(8-O`#Ko3&UhJ{_x%31Ml`V|YHL-1gD)E&Z{VA~9TBCs! z(+L-*@09-lc{%jT9mvp?Ew4stB%JlS9fy^%3^;q7UX{jt$Q6&ZsRe^yI6N4IR60aS3h+t zbPh*7xT=C_A(duEB0zJRSGLA?I{T|w?iHRRwsHwyn}^45f1&2N$=FC9Foqf9Zc(}) zr=?tjQCXlRl)C-rjZO&1b@v^rJxKkcE$3^>sg;#S`qcG5?FznYpxCJdY&+Nk-NInO=Mtxx9psgL<+2OsSZPpwR3QX3<4Rhtad*PE_sBYd?}MKoHE zfm0!>>e#74s35eamvA*y%d}#b%)-?Sw6EHoyCSK1+|-LEYdJd**)tl6n+G(k!x^g5 zNHA%}Wkj4wNMx#Beml|Ud)A6Zwk??ltvMvB23l2==9`#U)DnzTGRO$3hFnx;Vrw?S z!b)<~%+ne2)P<^BGm-{8)77}B*nHFjgI;<(;$YP%HC!5yHxz=v?bfSEYG&rEN^wF! zLg$)jYHPPO9{y?Aq>G4hMOX)lr4w;h194Rt*wQPgNNNL8zG?60Ynvi?b5LBRX zP|U?qPXdq%QPzMg$5W1#UOJdJy=FsT!lNRjV^I@KR}@r$)FG+rYCx28NWy?=tkp

{4ciM8UKAoy%x!4*r7zKKxkGg$EJ9~<_VU5>!1-lRie@?W`j7Ud` z9CY2*oyU8PhIt)E03$pr4CClWU*}fQno+lQ)*Xqe%VHUu%Kh7SQb_#A^%Q1U=UC@a zpW$^pgYW7+>4>%t8+$62Uf|WB0bQYp$jGX4A#`aMg1nRJr@!Y`ZCJgyW?!BxsUF|f zkOHf#ql|`Ajmmi+l}@OWFb{=&*xon?>N|Z!F;x}UkKH~-b5pbhW&wFt8B^;(3^t7z zbZqho!5s}tY~%$DzcP#!>$qo;{uMkdSxi#oE^w!zT1K~TGbDWKHjqA?F!k+8uBa2} z#`#-f+O4=_gPxspxs*^vL{aBPh`Dy}`iknoSKd z2T-udfOC_^Ju0lL%Q!qPPCrlatv@Uk+n&7R^Zu2cJYXv-G5LWR_dn!Tb4A6Ow=;Qd zh2v=J_!Cw%*ob)){lNMB2pInWKGjVS&od(rodFp3^!zJI1p?HL?o?!Y0LG@8ySa0A zRxj?%2PcP@anGj|>wpSKI5-*hI6Xh{+M)K4NFd}nLG}LtIjsl(097;OoUqTf1#F$n z8nMbq%{SXY8B)EEU-QK}Fv3OwiDg3n0FVGwHk;U7CT3jzB^FGpkY>f8f{{XLAy1;2!RY^GA{(m7+=&@|HS=*ku>g^35rf zo5CK56u3RocdU}^)?qF5HFz^9{3pjD$Gy?W&jQc#Sk)dtU;5g%f=*RjG#;GUu??lYfJFz2Z0020`gVWemSzWMIMgbd`^uQFZI}eiOix~0&?iZ)$RBo=t zqkN<^%rTzGJx};k(0dMZ1 z>V3^IB)YiDENdEeW;oB#cBlD)7#O44y_tqtsf^PfnbT2+FYN!?6`D^)R^9ncdByd7GD>$E{bH>8WZsi6s93Hc2@(Bob8?Vogm85!S8F<+f@oRH6P}wMkzh zpTCMpkjK2)f29U)D!J>LYqkYiVA*6#?^DGtd)8QXCapk7BC@E=*`Upps&X;rtY<4# zqGU;9OWV9vr#isOJbN?H2Hy! z-OVXD6rJf9@rqY5k!6KP-kwb}_^Xa50z<&!f_b1(ift6ojCiV&w>4%?l}ba_nnYNJ zLsi3Nsq(c(qZO)&q!r6Z_!T-nYLc}|uZpfyXB`VgsxmpI2hBWlQqhwfwK~)TgHtKu zrHaI&lnk0`oYaV<)RM^*U{mu*1_dWMtjGpxVd9M`wsx9cfYvwK-6MJ2Y4YPxGcSDC3f0%u)s&(R28ZYMJcC z#APN$+tfbw1%3MTq-$wJq1j1R zVZmkjz53I{E}>>vTmi$V=V|;pkN&k-R0;B98R!oq>FrVZIk@v9A!Gv*MtI2e6b7SA zOsqs)dvw9;`qc4&>SJZY2Fj7f4ND&7KvjuV{uksCM2iX9Lb8H!xej*c)Ou3TE;D8( zRmmH_fP;+vdsNX#>&U_9Di2Tp08!~omNL+lc@&U-`kn#pPsq_=x z-3G*s_onTmk`GnE{$iKQ*(vA)3>-mbz)pu-FAZ`dZeiS_-+AV6* zWD9%dNUEo}Y<&(ZpqgE}Sz(Nk&*lFB)~-WMQ!LMr!Icl?&Obq2P1)O{x}1Z62ewcD z0A7k)HY&+oBQhvdMDh+cw$u4%`I?3m@^ovt{ULpzae`Q{^{K7dTHVtiG>l0dh{s>^ z=}ftdu8k$XLo*`&Y(K3Q*5TxA>F7Sdha;-Drg-$MY1pmwu21@)q&Lu}8fP(P(E zodJd=W&|uFEuN*Zz#l?IJ57;%#0qhT^A0+ZkMYG^*lg_NE%$~oo_XZ{ILSX;R3&wO z#*NzOu^R*n3Xi%zZM`$ztSD2vc5+85AQeeAE$riLj-0?6M12njnBzaK*{Z&YO4*C&gC}~yOv?; z_5Qu9UG1Jp_V?=Lx{q)1sOQ+Bp^898aK!%r5znun^rVK_iF(S9h}Uxux=#v#clIcUIey$NVd!dqwiemn05wJJxaYT%VUEjsgx90QEi3wF6AXOGW^&Cx3UC z?mv*=RLg8v9%fzh(Nt&r1!(=5iw(*Uo`m)Kf!pe7t8EzJGTq5yNorwpIK!4X*n{1- zAJVln>vRMkynUAyEOy`!Ku|uFqi(1$Lvun`6K0*D+Moei4Fb&;4^vt2KTKAPQX1M8 zFrKG3s_3VdMF;6zoXA#2w37n|DBYF4K8C$r&^%;U3*tKsk`srIVZ{z z7!XGU9@NEPYUR{sL=tjXe6Pn}LsWJZAZEFe1+ge>(mHOW1DccqX^lv%(fd%VgT-0c zNv5)5nk_O&`cj&SSoEipni~c<42q4#NDmYh$e~Dr6kcl53{w?&sZ>&sNfRN+rpeNp z4h=i4W`!NdcGOP0TQy{WDhZDqRU@L~RM1XpBG?rhN>-hR7D&ex7;svnk;xRu$sEvK z%OuhRkxVa24-^>4vRaco(+g0i*12MECQ7v^=}{$2I#sgBWEE0n;MLX~)GnE(g<~6^fwYs&zT2wh-LZs;9MEdKyXSJrt8x6>untI#k;+SvFwE zg7Z>=PDts+aZzzxatgAGP(&)qQe?5)c&M0Ttyt|)$yt&_d8V3CN>??kQ9}SUy(!^w zNW;AnLK8gF5k@E{6p|_^a+;ZVs1l?N?@mK#lygYf6oIJk(>|3cU5K}57elvmo`cig zyDb(4C^H!czH6J%?H)^j>|76VUFEb$k%9oO`cbmEtRtx-C{@6!lH^sX8mY=#r%F<1 zR8@-3>N zs)a^;vIk?I{{USHgB*}VP%Li624$Nhlmlzr69^Xop4$uie!#zh@m4d$` zBmJCpswAqrSLHcn9XRD_?F!4W*Co3UdHg+UK@el|oa3=LBAl@W$N++Rqww^kG*tpbZZVF7>FJN@(xy+G zfT~gZW2fQ%6h1d#g;VtYpW#grVRrz#eF5YD0IgAXa_p=;t474~q;u_1%B-YIAAA+` zXQ7BLym-Zs3cH|LCKA~4l!C=LNBSAcHl-Oi{%7hkMrr%@~TlvvdmPTUyywd zU*}e?zD9UlfQ&eP-pBm=)@_(?JjBOuC3^B0bovlMtR<|GymX0sU}tOsRo9$$&-oRb zs-7?j+{jm_Q(E(H3UPvR7&k%d{xzGbL|Ig+<$){pP`pNu13JPQgAnQ_X8aMAJVC7 zVPb(3k{z}c9Y{bhzf#2oX#Nv8{6so?he>h6k@I_d56ITf+11uP-!M~_J(ub4$N9x` z8hnJ@Fj;qDh-Oj4Hhy9~he7nNwlTI9L?CgT_Fmu5rDIF6rCk}8vhBD9u&(EY1BDf* zYQ|>?9~mUE^d6?MH93|!#B2VroE2Vnf$RAGxE-rf%$Va_bRfuo)^or=-TXWI{U~1* zg)fHYz+iU4ZUY{yTR8fk=qka881QhzsUTx_{{UK`g~V8i6MJCio~EaeJALv#fY(ay zbFm-G35h;#+d}@Hn5wvUEXcv%yN-%IGu!Z}XSzi@R0cnw_Vp%|Nb(K`KU@LqDjb%@ za=phm3m(LjAnrovpRG1KR6&p!w`OCEew8{~p00r7pev9lk}?#MOJly@=qaLEPGYP! zq79whn*-_wDH;em6C#XpgOYzrkhu+&j41x|a(@r-6p}0K+!S(oXW)G)+zJ;E+qOV* zqqsjVe-WSNDx)lL@Gaag_lHCH)`XBdj1~ZQ!0S+$OdP?GM^-13`BHAFG@l3uZ+>R2clnPvukX!fsKO{_y_*JWzzmp_#wC_|}v%oD9}V=chGd7iIOT zOqPwLlOnOT)k7XisruH03{Em?rmtr6!gkiGO`{n^;AbGpz>pO1YJ)~qh&@5YYTj79 z1eP^MRn8kdct2Xl)Xl3RxI$Y3tSopGM7cFN;8Zgt^&{8-sNWqbLe&c&nyDKmFrXaM zbInLk%TN}hI%_7QzGl~p&t3f%6c9M-(7L4z*H2 zR^&MqRmU~FO`=Aq4M`jv=7`rOuEifTrpc2T99T6cms5!8+iM2Qn<3eOQI)BmiijMV z7OY5maZ^S)s7D5-PAb_=4<{8AoYJ|)M9;-%ZAq});i>&<27YPi-K#`^oOGlkB7si> zG?>U3VAN-mR%A?yghoeN5oi^0NuF^}>BTc{TPB7kJTb*AQkdopG5F9p9@T2sPd84x z4h=XbeF8e2W{rFbC}j;@DoPnR7_1E&NBce+Ndq5B=t6~D0bTdd>4If>AFWqn4stlD zrXA{|$U1ZQRt)GvWXL>Y=}`Ga+m!>{ig``^{KM-?p_jHt)~4m80u~&Azg*H4OksdN zqMl!9EqF_!6F!awft~7!;;zPj#t*GPg4t-RLhCU7pZ9neSg;vH^R{C+xR7D{( zw;&J+@6YQ(mdWP?bs+Ih$>c~6)|wRw7}|dkR7miw!*}XNX~75wE5YlGe=N|E?ci=+ z{2H7B4o(lgX{0NV1@5HeQZl*fFnvcuR!H5!VV^^h{V59ik_hY16gY=5QKMooFvHMx zs1+U~xdVmzXCB6{H<(W+Bh*v`6=8?idnw21{{ZW#aVebwf=iS(zWH!32&8 z91Q(&kHW5M=^Mnv@Z&jePyV%M=oc}`B*nB()%7sS^FdH0>D`{+F4~XUx zR!f%1A2V>?r2ha))s{jmnj^+T4^znh01xL`Vkc>%a!x~V3C;&>dsW%oODeLj%JHcD zNF($h*E}z&(>q-gDk{vFW0wFD2VuebedAEw%v)5k;PEVnk9Q;dDp30+A#uWw$PvwD zeW3pUY1xsS1Z;!b=LfHT->q!)iJH}$;zF0FbE_bH^am$D{bBjhs!Bhz(aSk)#~kD4 zIXV0RC+ZfcJ4)7zaJb&eFfoi0O*dMeQ5~!Z2}aAauVaosK2Ok6zT!&l5oxC0Nj9Qj zZrTS-V0Nyy*>Mncz){ft6~*YPk-DGzsuI`DXZ`6T7&tut057j! zO3JZyR@$Uzxg2NM4$J)N=B&{WcTDMRV^xflBc7~#(`FdQvLlEppQ^_FUnzbB201yxdrfF3qke2fcj@9GV+U>q^&L8M{{Ysgj0)U3@5ew+YSRoKnUF7h9MbMpIZ)63e+smS zxm<={yBI<0asGcQVFZk^+}*G`eJW>^x9|n`-ZNEM8-C+3!R^!eP{9i7fOC!B=Rc)W zkT@SJaC(wOT~&1>c_e!e@u=h^gSWS(A(`xj+YcE0Y2q}5PiZp409Pq28#g9%-n}CJ-^`4ITy_1Qmnk7>NXl{0#8I}9 z+XAMNHd1#CRcQzVn!@^=$qg(3$*S)t&sv7!Vmclwy2x18h}joB&w67V^HG9vPgA>! z3P@DRHF6`;t0xR9`{%`qZjxGIk%f zowp*Kqau`y3Rlz=tVYN&P%x`JnvK_*cQocR=3G^LRoT}fs%<`%ab;TQz6TVlywtmB z#szYtiLx>&F-pYdl}CECE8I{ikKUxHV7oaVAyCser(wMZl+#RM~8= zDdMWL8n-B_j;AyrNRPcrl#3o}RE&dG^bT=K9M$hE z3S+MnBCKZ>=cOjfu<3(LBADz&u(9H{w8%!`nF5N;jB|n3wzRtmCVcUV(vs9^S)FBs zXs9rA?^nne;;LEMn4FyMV^Jn)7?w`(+?M<*MP|lJ06ot&RzG>O>^Q5DgN?up z`*3PeM6q%P&|DtjxT{FA4ns%n$3LxCiC7%5V1L>*bS_sUllXsHxRD|jGW6WP_thXI z@Ji>~nwJVk2k{*$Bd`O4N%aScWRDpGzCRj~TqyaAkHDI&z$os_dm4?OV9bQ5?~_)9 z(V!Vv{pRPfrnpm(u6XyXnYD{sY=@I>J$U2ssx9K1GRMoAN$k>sqw}hAk5MQ~p}V{X z$C!BPGyWo!qk)AP9-XTh{6++U9t(+EyIZi31N7aKQ^Bji$8n#_W1b{Nk3Udb-`N1++P9{&JJxv1@)0t1F8rs7BPs`@;E z5<mNL(gN*Kb2`d-!|NUaIq7C{{Ve+`Tn%qm=)e2 zqiFlX^CbTODv_af0RI5fE>1Iy4nL)51#6p1SFsFi8tLDHk%9_?{5d%OMyTDu!u|jO z$C&DI=uX^zXEmiAs;rlClZAIx>CgF2xWa;Z%G5D#oDh&WxWTlwQ5C2|v!MM)8Rx2YMt+ z<)OeOh~zKRIXUT?=bo(E7Dm34=C0p4$vNGy@CTqjLDst4R%TGo_nQRcx1g?POoB93 z50o3XF&{C`I{yGKO6l#vW*%Nu7mdy7=s(7)lv$lwCdn@C8fMD24i8bBd*k}{tcW9q z8MdZdpIq)I)YjeI)Nw${#|4ft)Sqk`#)jf{n6WGde(J73`mySNPfFTp8A43lg-AeI zOS$zp1OEW6L6MG7I%hw1dB@>Y((7_O1F?j56vWTuW|+l^ZvBfsEt!C zqe+316nDwT{{UL2QyE}eV|)Jq`s)F;+ZeYoodW0Y$luHV0M%9_)*z7;i}1raJFsX; z4cyxTGCn|`&YJvn4Ufkap1OoB(cxk24OvT=vGQ$Sp=JJ6S`(CDeGs zK<({FqdHa#_esxegT+#~XWO{T_+qrCaS_KSzCr3MF6QzR{0_u_bW}*RK^*6KaHN=I7tNX+YTmxyw1rumo1^yYMS`vlfeT zAZ0a?djxws#PY0w@+_iDmu8wLXNO&Tg}S(i;5hyH8*xC z3sPk6;y$k$&I12+}5OO zxEbf@aYRr_mLYi+0b`$R)S;we3lOW=0h3U9>kz;g_a>9=MYvp(>(Eu1*t)@S&g1x0 zPO6K%ug%zh^{R{A>;$vt9kOv$C)6f}bg6X`DN}5{G|r#J$^Pf{tX*2@#;utf zrgPu$AO5Oi-OTPhypz*CaY3+ViOh#{9QDO0%b!BrRCQ;bYnsf~kUYx}`H#xd63zY< zhZd!JmdkfN#%?9%0IQ9I{(5@nu>AcirSV;iM^9;H2g!u)a5|Ho6n%PqGhASm?qo7a zjm^#gz#f2m4wcfO7{>M`&9|XtH4CrexVY<*M5G$i`$nf6YEto~1Z7c)Vt*0O>scUS zG7mKy-Oq0m#}uxp!vt=}y(seD%uUGIvGK&Vb00R{kx$-eGByHRzTJ2gr*vy5Mwy^^seRG_8`q!QKW5n+Bt)1%(kU~m&pJQEF z-30rYxxosE3rtw$uwG{{4Ldk5Q$*F9&bI+&E? z$XE=4+ZgBbHKA`hNopKFAd$yjrz81QGBz(NDV+mjDuiR->q?-4p}&ST>EE1V^%Th5 zJ4G^_=Q;Y~ts)k=X&eoO0*rS%s8I&NZG&3KiSutevFV<1RIcIQZU(}qYXOgKr~GTC zir5F=)HXjtYK`T(E-Rhm$Ga8D$1SM?^Ng2ZllW!&4@m! z$JZwnywL5Dvw^+F3I70iW~oa4TE>HsjifiOFu%_=k0GzLELr9+owk-F@t%Mjb^ic9 zm8JcpakX0q9P^xig>usB2Vv!i3xSTLo&`a7sXHqwouR%^0psiXev~aXA?kEDdZb_# z8xS`Hmcc*Q@~rgMp-_JA#Pu8xObX>DpD5QC3{o{8jz7=;0A9I0TV0y?`A`!Ly6^xz zx4AUbT$s_`-~2$fv|@?@W&BC{dJ5;SJZEu!a*(D!+gzM|Pxx08C;B=wj57hBQJT=P zz9^xwv@jU}4!wBvtyMK^h0=CL*wpXT86q&T&niPB{y^aVAC)#Q6nShK>raV*>d06V z_2e3%bkR#YG-Zk$U>aa&*73dmQ+HBxcCS-w+%zL5rALp9Nad3J)3(`vO zbI>BVE`Dd;eOLm2Ig&41vmmKQZbLTGEG5-bPO!m1ob~9W$U? zkN`Pj?~Kwtz-)|j^sMOT7#v`GQ;dL~G70U_)uT3vWt%I8WeR;y@)bhrV5bBr2NgCc zvqHhoaa;D%aacC-F~G^iYukdV25Q!XGg%uVm6Q=x<9?jglsT;vGia$Q3V;an&MNJ~ z{JE@Y+@l;CndoaxCSL^yKq_QKJmRsE=LeHklK27Evgb8)%@Yc@Y78g`097@;2OQMN zKI7J_oe<#1Gzv!oqHARXrCHt$E;CNzv7Xk7dsHuA2*xX4U>wsJQ?O`ZzM4HMvuR+S zoL5D#R8jy1CV@pW5BoVl=}ucUQ_{3pQyD<(P|XxkPstrIQlttg%zY^VIp(J2@}hM` z>?%PPYADW4T2`in)rJVb>qzOIf~Np=G#p}?Cz?G%%bKvlm=~JC9tt4<(uL-y37TIo zj+LQEgS`Z1rZHlTqNR+^YxuhhCA?brtgv~Kmik>nkTACyqKop<~QfL(H zQ$#YZ4MKWVw_U=kNSLOHv0cqZUWqa)A(BlnW4C0~sA-D4Q@T(nlWxT@y)$(tY1pyF z=vgvqCp1P5X;?+nb5d@|sE~0_98$0WCWQOcq*BCyxaOKeH+0sCO63DLrFC}93UHf6 zU}-QQP;r{*)PREn71BeAMYa8sZSk7{+y zrzBQn%QUO#Lo0!~u+f@4sjhr#aEC-;fkyuN(ZgIEQoSLyDk=ud>N$*2p zVnw;wK*`VhMy^A5BLR%#@fFTJ@M(bB>-GbNSqF&Fh9>Vx915HK3nz1Mn9qEw1gwMoaZH>O>E+fWt1*>z!*Q) zl-nqTq+z)ne7O~1tcQg2zJP820QJ&{Sd+Ye-)zLzv5H$5%1xeg;|oJJmvd^X^93i4 zK|ZzS)&Na!f-{bz(!H<579$0NDb7LL85#co>(`1aO{ME|LozgI>~WEding1JE;^UI zfD4Vvo#`K74$De>&)t?nGhAjF}Y2F6AXy z9FJpOf#LmE`%=L-ureUp!1V|G^IRq2z$Lgfr{S4e%K99!Rhxz${i$-^Hyhr^a$s+k zNdExV;DSN*&-AO1lOrT*avAaO*R43h&uqa)BqL$K>T%cdH7daH%+3J$co;e72m1d2 zFM5MEYFo1xK%+l4Fg-E<0M@N&NGbw=c7S;4(?8a;t>f7d$>kJh9CpQPTSu?}y~fk+ z$NvCaRisNcAOp%3LjEGfu({4gJ#o!wqHO?n0~HkMxer{9mAj4Aj8v@7UhJrpf;cL~ z{$L;HpTe*;Yp*5U2nAPw-FfJLophRvUo+$dWys^d9Y^7c6s+DI5Hl0=j)D#`qLst@=`}5=I!m%HELKHBs`J{+A-^2Dvx@`I?Bb^*6@v}ds${HoW2L=UV0jyidp=|#7-NESV)AOo=S)l+J!CuE5D?7NFSxD$E1Xf}ncU$S$08U`;-B ziS}n)lY@?NR^pR5!Oe3v)5zE)*43<;12ie!icqpNiQK%_KA|4pm@R9m$f}oU#Z7Ix zA}-A49!wKan}R#lo5TT+t!KB=xn`_(#oZSox;Yhk*7Q8GN6us0pTuUc)rhCVKbLy{ z0PV&PtdaF$*W5!Ju91r@DeiBqYdJL*0hYwr2}7Zqpebo<99i!#GF>D zMou;)ViZ+o1-_Lp-ZbsG6<4^nDKhQmp=PVHpv6^a@sfH}G-(!B$flE;MrsHFoGA*T_MOh$Z>^H3GRtMac}sLRa+whO?flTQ^J z^`J?MjwvxfJYtjzcP`=;ry#+lq{tLpo0F;HotlAOqz02HaeQ%4f^Cj7(x856V;MbZ zp`EUiDzXd;-BZU}4A?qN0rEQe^~?7n;Gdp&)69UKp|VtofvA81CSD=cQSe7TSJoze=RAgC-hK z2Yi|?Qb=n6x%solUA0s^JF;aZhgN@-Rm+=D>Ohf*_7$3_ut|l4wmnTDrfEcrAlriD zsl{kP8X)<$?mxmYRkY_bN1dBfIL9B(uDn}vcLm4rf-1RmGji6&_-05~a4WZ=1$q4V zt8v}L;9y{S@%}Z6hTteMgd@IlpVFd{h}>DQ-|UXG?$E5B=Tsz*IpqQVB1RAEN`@&p zGVlKCg~a@!i)t zEzyGdAbG;e&@lc$`&9Q+!E(Dm7$Z5Zt6uP(%&JUeOoWry<~i-3O5=xzFW|d)f;2Jh z$Z-5;>5Nd}8EY3}(`ol0hT7aozdWz47n5zvoMC#L^Hzt2^&PGLoUAdD6*>Ist)GQ2 zuWq)ryrM4VLz2hX4k;*Ed(3ZFqA&*Q`Mk}i%Lli5x{D-taJIs}TaGb`^qn)nQzCD0 z?m*~!W80vl)OVjZ z!Ny4(aon1U-U5uf7a(ISPY3*u>sm2F0rRoA94Z6ew1jSFCt_d;L+wnJOAnXUtg7}H z&$ThvBn8K){OhNoiO%Y_%_JwwVhQAW51m~I4%_fH15EN4uDa6uf@(RgQ|xdY?|i>smU0gOUQb+O(oc17#sh4Dfkg!=C(~rFKx;wb_LQxNg z*&f{u9;R~rNS@y2){{x!1Kg~r?R- zpDD?-vJRQgq0KS;$-pvYaay-=6Cuj&Ur;(zVzqL9f1FhE*uG;jC`Rxy3H2haqDvpk z#tmEk%SHL)-IM$|rx+PHagWC|IE_s*7()}uKSNq}-zyFQsNl6nAeSbzp|p$vf@?c5 zwp+GfK;sp$ZO~R^5QQKBO>00f2NiKfb7q81#X3M8X_1kNbMh;pY+=;H)Gfk-O6IU8 zkSQX%d&MUsHG_9(GDxhdF2`Lb+|Fx^bQKUK#@n!63=I@V(ZjGPRSO%k?q(xIyw=+V-_eoso$g6DQJYmv9OImK$* zTpW&QQfP@l>LZXfOdB15j*HhK!7Zy(Cd+@4KOdck5?!R3yBDs&(vs*Gfk zO=R8Y%45%Z$#x?7fvF>Nnq!lXN@_PWo`+rTb0l*{DpIt~rl~XLlNCDDpi>b@H9k11 zD20Y;SLLY!(iG=3(P@Iv)V&83#%u~15@IuAp)Ho7nuHlSsku>l5)PEB!KVeM1*y0; zJsO{%T8kJo@N-s}l!nPrGS!9|6%iB(vb%9n9%^?L7Ac7_gRMPi3(Y(&OJLa2YBtEJ zSW|Y;n3E$`Ak^Ebfhp>=+cdaltXr!Y1{RnUdRB#u5^n`insC*WjmtPyW4U$<@0XDRwZ$t;>SUqA}g1?N;E^?GTN}&5-v4 zfkR0&@AHv>dJdIhIT=RO{Do4CNtBsZ@l^p-U5Yxf>sgNg19t7J>PIK&D`HsZk+KF& zQ<}zh<(RkV1xtt~Lu%0(Lgxq3m=^y4>sG{3B#ZK*f!vk<02~^%I1);@=MTsp;-$FL_c3HKsq``xAC*^FriH#; zqz-eOkL6P0*=k7Hj!`NuBu&5G3<~}$#a)MhIp5zX*&jj<4l54a4x@56bA>;xZCa=? zgD;fn*LjZwenyD4f>vgNmh#8SnLWdDY6)!PowFny0gc$l<=(79k-ss_G2aq#{EbeL zV*!G>JuIS`^+#ZqLDIeCJ(m=pbu8y53O=q+)We`?Q{;vLh*nz>0M0Hhb*eH9CAP$oc{nX z=Um3VjDRx*MiJ4wa>$7KWFrkib=){08y#|{=I6Vxg4lb=nZz$gT2lf$!t|H zjiii#Mm|;P{uN$GnTX6l0pE5AAJ?^Oh{kh_nubXBp#T7Vx>n5@+{cm}kD0?292{~* zVBgwHzk0D?pEd_OhI#yPkF9mmh6(d-+wI3nrz~;1zXT4172T0>Qu1ExDx9V$~()0<&qXpf( z?Gh`aH_P)f;1a*gR}+^jcRS$=#^5w&=$08%654^RFci~89-jBCyAD52R(AxQt zMZB<<+}Xj?x!V2+BTH7&K&(MseIX|N0vnKArbe@@k8m5l9obWVVP zf2C8{p-W}=XNLf&AC_u2ymVYFtfX`Ij|bP9j&z60*=~E|Z^B;64nF9{e!n+0LTy&< zlF6u?; z)}nSdr*kG?xjAw46;Xng3NlGJ8LP9EBZ19Om0NIO)EtWEZ0K?3i%dz_#6s;Ez~t3e zl#|U_yKguNSAn^Q_5QUGe zfm*U6fyt^9l~rc!22DdqwJzpu%Wj>IWy1deQ&$moz^Xdqn#@egARpdr{(RN*BZFM@ z+H!I^@ZR*LF|tN_lTo69-iLBeCYWRk(wwY!#KlpQOShVGigi)*19tOLfyFaC(}PnT z#oJ9`QZHIY9MN*FLPlyC22EPqQMzK2V%)2?o{D`cK$$f#kBXau8jI^s#wp=2DH@%I z*up7>pPEk8$he^7Q(0*`Q-SPgY%V~{QI)F>DiS%Ra7=2gDHPx_N<~eXpn$NuG~6C(04YI z0ALDJlSrHnY5DZ7rxT$vyT`^&Rh9OPD65ilRV4vOtzkV4Vr5*Ha(5cfkvZozr#?wF zLS8<#&i6W_u@Y@&2(je*R-`s*vo3b~R#3Z&*RL(tinSi0B*q3&~Hx{=o&lF?tGJ=2E#b&cd9Dr<j)hbEu&5~-Ywz9ug0DUTFzbKW-;hBAkCfafg8fKyAM#TtAqJ-{HtA20bQ~#@Oc6?{zQXaF{Zkl@|(E$6gNwD z8S1H({(}^$19`+nZWQO1FO&UG@~bgMq=3;kKAFemnzmL9gtsy=Wf%i2$G@?;r>X2X$I#btq-pWmTgDJs&-JG% z-^X0{9+gz6sOrM4XFU--ah;%UBQqAx{^7=oM-f=mCjGSJ3RZ* zkVbpZ12$ym12szKA^{;XGti#3d>kAaWXrt>9B03!4GQ|6apLjcnbgEHksSw=~obK*>dRHA< zO}m|x>ZKbpyf8}`3=hr@9Y=HOJ!)Wqn`kI|l_Lxfw|dhSVPe4<0AL(;^dhQHY~U3H zNHf4;mS3l>OQ&Q?nrx!#th}qE^dLBI(}Psaq-I>ROSE(*7ay%^q;CHJD?2_qf(Qff zAk;S!TEvGUHa*E80r?7_W|}YBTSE%-WMHUeBeL`Qj8+}Z&&*4bn|KPqbNbfR_3e{x z21Ll`=kLOv3S#$s@ zC)aIHYpq8q#DD|rF;{d$i`>zOsN)1xo#>+y9zAnVE5|fs#OVBTGlN-^T{XLvP)0ou z_|w=*H!s|^yn_hjei_X^($NkNB-KlMnP*{q5r!EM^9#%O9tcC%xO>zjQ5&iBDtwAVp z&}3Di6mHNrLs*wsLPkw%-8lq;F<94akVqWXRSueE2^>{dk%cA9?1b=YnZHcn^{!^l zsG@bx*;xnhoc{nytm%`w z62OcKmX6DtwqvI?w{E};3g_(|xZu{bEax0m!Pw`hu4@TY4z(N3PYX1O=oHn6^KXO&OZv;byM3l&*~C+my7`7F|>V6Wm)pUvnuoDru@cQM$a)S zKIp4bLm=jxVB#nXW0pUlsaof8gv3Y9e{3FG>>5^o{c3E*bZCn!jla2V*d0{l{)hSsk|{3Xak@2Q{@hqUf#Wq+*3$7xma~RI z{pDs-e=qQ@Yc#_y=q_MAep!nt{KaVq>vTk{-j8MgUWKo`& zjK$I)E}i1JqlA^qWiKRzpF?&HZ|B)E}D>aj!e*E^}r z1c!%tTy;=b5%mCpT$K_!VA~@r&leC%dSs|y%ZyY^FJf$?06P8PL8=kYEMw%2K_{e{ zKdIub#XRHYHvRc3I(}8pYn$G~n(E~9Ap_sH@~oXg?b%P3L;8T3tMc6!f89t>{{VHc zr}L`LWRPLqhtTG!CPy+bHkNJ1IRVd5D%=8C$Y5Bn(zOMwilBj!*NmD$f%26fT9*?= zG4R|Cxfuj?+t#f4o^SyC$KI{ZbWnMV?RYo1mWoIk&|~L8a zJ*q{U1D+{a1~}rQJC}|Ksu4Ee=}AWZblU4;V>cGPfN8qkUrRpii>&Zi>G1r&4@r38TE7_7uZ zQPi5d0Ry0`S{X}ISsdb=Mpvy=S#Ubj{?MRuYh=tC9L=y2M*w8kEj*ju-}}s3x34cF z`#ghi+*I_gcwwbLdJ;4JYnu}8)KSNYyOM#k9k`C#A^br`ezhAajw!NSGv^2Js_nd< zwWaK=k4iDJu=x}aamlJ|LLZn?$R!6M_^70EMh{w+Xo|1FsE=B(SFJ$es>jTyjcq-~;l*_pvJIlSdlbnP(^;kzU~^rx zo3YbP7u4i&QUifaRUmUpYik>j$(H(6ck}|SFgWI`NZWq3gpp~L<-#vLYcAk!B%0D- zTO^(;q_O8Gpsbp?q+N_jlLVgiojbbloO{;n&~@UdOKYA;{3||IYNU>SM3Av17wx)n5Ey?t%uLB>Lt~~`DrE?hp92_s>nt6fSt~2UuL2a3jPpw#30*r0RrlL!z zVZD)d$wKGy6+Bvb7%zao-R)ZeG1Wj6;U?3F)1OmRp<-P^D@dh6Jh*`azRo{0TTuDf z`JGefNA#@afBER!kEL6NQ=G1G>~UJhLn%9#gGldzb0&QOAIh~XCcO*>-h@W4^t^@Y$UQV{{T{l)PtIxWgz_9RDYjSRAP-A7!99N z4O&Q}L6%oz?l}Exp&2sbKH-HU=nj9CSd2F8AJ5zOi` zHn1bt{{XL4C6JN#BxOFb59v}|OOSBOk^R^I02<4Zbn4m2?~s3$Ln#FC+DyMPu7BPD zkHWI9ba+@2kh>@SGmtp{0KQKn@T>Dn9l~LVf6$NXSyMc4xdmk>i59O(R&gWZhcY z+qohs&O44jPPIl!EyCn1tLRLdf2Cc#xO0#bgWNH!8SN#KLdCKP?_BLAvAk~gCz9z< z5)s&Y$_+vNr4hjLi1g(Clt*l;dh_qiOt#Um;YNKiQ0`UYh9;y{zE!~W=B=#Wcmk9S<3%GJkfJzF)k!q}(ao1oH^% z`P6S99MjvL)f1^8jC~CV2&8bsKdn`gP0Ce|IN;NsMumuF;P({-_;uWeBhrG>GUbR8 z@g7vyKYe7;Ny~0u;bGpjFXm{8bR-aZ8iL;T3-ZYL^FN7CM)$5di;?ND({x`l);&z9 z?YHrtrC5m`E(yr=G+kS@teF`xlfm?rT|9-FV|Y4nHcpVHp~HXBaQ@etkbm=1ZQd?2amb zysqT>5VDw$12r2G-npyJgbqzWRH(=wjcnuC^puK{BV&rEDP?X}uE@lm4M4*=$<0eN zMP1pbo2@iTDOH0S-Nwl9oCgJ`W4$3@yii*;nVNwn{0vdLq1}_kNUBJr;u47}H4c3# zm^q|wl@gAG@{BywPZVdGTvkz9fVeHPOmz za25b0O&QPZ+(3!0VF5iAOZOST;-}WN&p3z^ilp5(Az^KLx$@28J1*)?_UTL!It+QA zDD>izE46;iGt;$Br5}@2gfcnUj0IvpomdE13;?Io9Moj42@XPztPLt$CU${RX8ztlS9kJJ*wcJjz$^e&<*b2whETEd%)DOHdo%j`< zOQm#aO{?m8$NVMo>Junc*r<8!o;^Evt1GQG(%bft&PQtD{AHm{q}(*t*Do625w*6Q z)g24OlFGu;{{Rfq|o~1a!t622-JW40E%1ki8G;V_+k6+4&H95J< z<&JXM;l84{S?~6IirZJ(tXgV}gzj&UbCPPKTU#>T9VK*jGhIRk=4x)A;xTJ<_U|_E z!zelv+XLxdZGGZf zmybi*p`U7@DT2eMYBKIu1ExDs8h31C)OM;7og^~c0vbm87_WneTqqa_o=G4T9u>#@6q`6bN>L=tD2pZ z02YolUilnXH8suLE&Gd^P!C5x%cXP4wYj26l1btV5J*i5#Gm$?Kgd)++mOq_@b;K9 z-w*yYtEpZ$6Mdd>{{T&`(Z5_~t;H-ZM*S|t`>L{kCaAspeZ^YECAfk{{{WCKoVoN8 z5B-|0!XaWeX;y>KhLisQimeD@v6BI;UO#x|Kh#yo?JUpTl4L%TAS3ezj#uO^Vzdbc zL47t1dWj?-)~}>?rx>tB_YPX6iq9YF75aMAVY;g*Bh*$+wYj3O@7*x@b5TCn`P|t50CzMp z;N*0m9E={7De5w`{!5N!QJ+w2J>mczvqnMx0BaRu_Acsq^sHIr*iLirO}lh7itItC zS%JwNlk=wX&(62SMGi39~HH&to8*-eA!fi9AntK}dlN-xzlf=sS z!z&DCqH8}Oh6L>=gXvi|((RR97=|9Ex)?6EW1Xx==Ugw`dTesZPYEt%@V~&W5wX2` zcOI1s^x6 z@}z~{#M8?Vy(-jppssOGdu%Tvp+v^TD1K-5tE-H0P{#%&>BU4IEX9yJ@>{YRODf zHiBvL+?ygb>rdO7aF3dlM^vem$Wvsfj|Q*2dF@r+6O7W8Fm%ef=AXqhsRtCJrDEV_ z;5%;>6Py}(YK4wkmnbL_BUQ)=HAx3GZXrU3r6?O#Wj6tlT@|q#KnA(JJ`K(}=DMpp zZ~nw za?PLiSNv(4+oAx2{hq&{T6qNfRBpmC#YL<;jFEy3tJKmYVLoT&y*(+3o%@g8Jw2*M z8TmyoF_1_Mt)9Z6Xn^fnA~30eP`r8hK;+Qk7c*WWLB&GQ5D7J|&|rI14FgC%Rmk~F zPQTKfmql>(BmU7~JgZm)N_!_6`s0)MlUmwjZX;6I8~Q4659QXW<4x1f6^+FGK2RLz z+xSQRJZZ0}T=~0Lt*-ty0%V>20mnaw$||FExSqQmD2aMG!Sn-~VYXKLi};h$vQtnH zFo)$50rLiJ%s#l_A8u;eYA)Q6Saccx01DReBFvskR7K>HPi5ovsFfHlN~lhsH%e8% ziH|2O-_oA*#@{f;@9#P3{(Y*Ix(H+3zTl^5QR!1hDgrPtPXir)3aYHE3n?r+zCS@o zK|!2j>rUWmTIUJC%hAoaC=0DL5X3{Pm?5jHl(xIUKR2kB8y z8!f<64svU6MDYBV_X_d`STo7*_)!(9**b|tx#1rVu6))|a@(*+KZSb6lWMRgLXnOu z2TzG$i)e@jPfoSF9nf_cZNv-^c*ytv02=CqQq=Nl)N*Fh$jU}a^y4G1(9_I{#H4_A z$ie2an)mlx8Fw#noDuyqRNCQ*>db)hz;ZsKwz@_ZxzK&8D6@`n?Nx4XgJ3F;#2zaw zz>T&*O9RG39DntVU4=efx!e!%p1=Kn!j53E5BfvNl-R)b546{3+gWh-Nn1>IvZg07_P4F?9$ekpnF63wCgRT>JV~6tcWhn6(R4k-xk~$Na|! z@-^4Zd1ojwOo@Jo8RBi7nR#E#aaPk!gpJ7*!F^$MnSY`D>cz4xx&6(|$bB|r z^v!3=x|MRYF>zO7OmIb8MvhVQh@ATEDAgc>KFsKQ2N@rjt5FrmQ9xAT9oL$2f({SoPSz1^U9o~s<5GjWt`Ff=V>mS^$mAT;xC-&B4D27BR*_TgkmKBQ zR>S9~2hyW(G03TuQyOJFi1GWuk46TV@fZ1{+JD_OG>VwUYLuu`l{l;W$*s#Xk>!w1M`Kubm;~pLYU5w7 zK6_VCd!zpVjZ~E_E(Qg|p*^HdKQUQ4)ZUjW6`;2=%5eDS_LN}7GI z%0}Afnz7LNXB#te+VCU!Q}?S^*5KJQ$Ije1`gf{pYPWEJcyaez(xbN2&85K8uZ9)Q z(8soW{VE|FY0&cJ2SK<$f&K=Diy)DnGJ9lJ^h(Rn8s#*pzRjqrIq1a?@~+4*1LYN^ zHF4o*`@UYo(42EpF4NwkErCq?Kta;CH?_+_MPcjmfYHP|DPxa*D?{yy+z@LIQn-#; zoaZ=feE<~|TI6;)u`Q?;G@D}3%`dA2U%+$vnzYb5pS{g3jv%p#7~!$F{ZBOcBva7S zrxc)(wI$00i1R9rlq`hjate-Hhe7g;imunW2O|cl=xnaZriaZ0g#xHOqh#dqPHWd6 zbmpLv@vw4EO+&G2=6XU4XBB!Ya5=?A8lOtP82M|i$CW8+J_P{a)U!l3^r%Q+K^0+} zF@k#2anO~iSs@rS;TgwDa=8{tFI=Ssot!zb4U;^X$XoV5i?Q* zspCM^DT51|)YUSPk0zs$Fe>p(4ns?k4;1Xuj8#Vknyo)I3F4wah}Ch=dVGAgYBF=c>U}DBlq&rEdk#$`40ARq(Zzwe z&I#?_qw@(kQG$3F>T1r_iGO*^e{g?Vv>~N850yYs)!(3~-S+VjCBNsRb^HfvSCF{N zV`%6HN^-zS8?)2yj=%kH#(@-!Wt$2MG5krVeUC4-*zW$~4bAxWAB|a(SmPUaaor<5 zey#o$PI(!VV-XBpe6$>&!@tzhX>}U7tu?)&{n`hYpa^#y`?vH__|+?^69yn}(H%sPE)LVm$f4)agpy%mO<=CqU8c!^2VA*MI zygbB__-^V?u&S+aux8&&j!!Z8X~_NK`zZbq^yl8Jzngfgad{_~ZoEdSxX-9X>G<^e z(xBgN&Zq1)JsH=TCH4oYKadp~y}z$d^DES4g{s?@6Ed`lH#BN;5PG|I`gN^LpowLb zzWU`<3}t5{sN=&ebU9g-+b0qoYdf|_+y%S$5DL~uL@i6Fp zhqYDEtXlh7KonyGxE(%~AKTVe!NMTwPp7q1lI0?ZF#x*{k$t^SxAUy|WK!5>C!TRb zoqRXHwLFH`L;nEQs`K24*ACbKVB__wH(Hv-FP5is^2hoLl-EM6%$xfNUsp%K z`T<>30yq?7g#eOArEt%vIleOIAUIm-Z8a#^FC*^Y=lRpUnR2^I=SK^Llc+p!YDBro z+Sm<_IjkkP4yvf#K2y)-`qYN)mXkxyO{cwUd5J@u^#FDHj{gA8kl)E6%x8jlBiGQ1 zq=?QybDVTFY86G!;s`xy^Ea@pTZ>V+!C6-$x#&mZO`hf2CV0qX95)?4qM&=xYE&>1 z#heTr5%fKew@RDr=_)PVoXVu}mKa9=0A+{ppV0eLO^Br(!yl4W^J4zvbzh;aTS=vY z0$Hikq|>nu%#?bNkyz zFu!=;oP7^p=T2cfKnce1dKVjk`ZxaoUZRdjr6bH9TlSF$ABg_|3c9jQ5c%-v-^REh zzP*p*&04b&3IuLID1E+R{xxbfZM;#Au_9}frsd}6^4e#EJ=aK>U?@s_Qz~}kX5gx_Q4TJAPD=$4g4{Ccyk;hR{g2~gN zpkjG|@J0;=Onu{#>rlxc-G@wiGXDUDFtA~TaL3T_e=3j?;jvWA!#hqX(#x>yV}s~x zI_g#AWCPop&PK6Bva*Kc;Ea!Wg>XOvv3ckWna&|fo0m&o@LxNkSmg@+AdNp{QCZ zqZv76^;2BUyv+mQT&-!tsLvnXZNjm1?eOShJRFv=WyWdFe7o7=K83dr)}Rt<$o~NA z(slPm&+?~@O4-NH^1gbSUotl!qbc;-YF$dP-Q0y%y&Wd6^8HJe+JyfAc0~aH0JsH7 zsM*1*+Ie$utkx2&7bo|t-}}SV zS9Pb}toCtB+>U;txx3+Os@wC3=TY)W>Hh%Mr)avnLpRuO6d_IqKcBr+WgFcy8gqpy zHl5S_4zqsl)Yit{l?j*2#!f-#M|x{YPaxn{_eXMRtDzclpYqRv-mp_M+{YhX?~lW& zt$T)H1F?S9ou#9BvP+EZ8$tBXTEt&>Mp zh#oAid8(4ymd##iS3i4-X#n!CK+%lwv}|w^)x`R8rJI*f%{X>`4{RQEE(;gkB9lUg}g zZbgWQHGbcw?1fRmW7GctuUSaZ#Bk?(kKj|l`k&}OI<}I$gfkrAa0dVnKs!}NUCK`8 z%ri3`v4R)79;f^(UIz-G4ozWMu{?CGc!&T2S8-Uh(lb+A5w7Mmo3<`VtgS-$E=Fm& zB(BDy-iVTGwevdFyLn5D3g+NuxtwD-&2-k0VFuAt2{|#;ii#y}l?$jGxxn=ARvQb` zlj~I^%f1Fn6Wgh)mWD`ZW_H?xYW}D3t4k3nfq>!K7xg&;K%u@hY*507d=nY(vU04`w(aQ$FTaEdczzm=1$*4Q_1{lyw?pN+5koD zi_m|Gqix0LS~Ecd1&{Z#58~*f=}|P#BDPp64!wu}0A8XBv5dc8?*9OdK{Sl`T>f1J z4ptq92}Vgy-^6?rPnG3g{RKS-ss2K#s4pkkZCy;P2$thNDlc#kW%X|T2OV2> z{=cud5iCj(=5sM(KjZ4@PDv;JS;l*l(?8I272Vmja&W717id-cw#nHC`(CH|)b}&T zVGKzMkjg)Bj5%EOm529_Vc1q|5E6XWUop2~5dQ!h{{VpR_)=}HKd^0ncB*PQC z{G_g5)l~Ff_nY(|T8qxYeWFk>Psp8*{{UEj^}T5>SUkQ@m`qWU^2z=7{wDr{smDLs zZnsE2MH{yE3Of(w_||RPLXs@YEAPvaT#e38sj8-9yr1rhagfQEWAia2{c1BG?xgp? z{eSw^E$UK4Qe3x_D*|~qC)=<1R%2f|bqB6%LRiAvk?+Z>MgYx~91cZGk)lY+y}6c2 znRj(K{LeqqsXUT>qA5!sm#Fo?z%{oup7v(iJ&jVD*A3GY`HHeKUM=E8>6XZ9d~V2h zDLZfmK9zi1mWyuRn2hF^>{-Ca7|kwVu^qM4u|DF&@%h&3-DC#OC#_^9xl^@);#`>ygj$%~~^Ny502v^Nf#r zciIKaYysp9@%-xrZWt=CC*>i59jfv(M$2&#$2^bjpHHEyQX!)3R}J@E2UEkNe-P|P zzrAQ$yV)?77B6o65&XgT{&kx!(9z0^`>8sAu-(-AdR5ywwljRYk+^uzfAi~9o}{y& zj3mn&#Q5zS4oV;H`;YcdBer@~pDk{9nF7s>?p|^|bNoZ~6_*$`*#)>qeT3v&mzW*(hwC!OCG|h zF%|-;&JI0^;+j;W4CGT|aOxdT1Y{B1denkUt_}ztsEA0~tJO=zAK?lpa-ptIsOGD*d`vRrG*IyCWTltmITtPJ8*ICN|pYIxK^5{{RtH zKGkJ?`=&BlPC6X#L;Y$SjY2GeCJuNfy=Tg*?Y=TP(a_P5DrJ^ZtpWcM- z$Klqo?st61cv~^SKsgx>+UOJWif>)+mMWdT(#0Q@~cKx zITMgb{#AcX)THvv7YntA1x;xyCApeaFzepf(zuZQowH8U{!1~>EOKg{)9zJO9ZgFR zlmG^BIIPpu-iPlhxvs{LAf9SB7#SHf*umhMm}j0ityZU7(4iyX^dgZaVmLLR@Z@~z z6neK62l_N;fLQzRC~$j?NedIm+yyO8cSw$0?mmnDG#NK81_JsH1xCjM1PW=`R=E_a z!Lv?{s?^bfII60IHF{3RlG;p|HbJVFF&8yvlfaw(}8-JI2snL&!mak%tmeCdJ3 zSB=O#R*cc*Y6xKj^sKpT#bPyQYItV{nz*+9!jCKi$gPsl5*Uw#sgitgOk`fw6DbEH zr7^TgE9JV?8JSw_{xOJ>6L?q%}tM=ARs3RA{Wtgc+%U5#w*p-lw4Y4)ox! zsA$Q2ye{TohU|S(qyC3t-MYt*35G$Upe)0bR3X(=?4?lqP zrWFzJS9FJ>e~Q8e=n-`$zrBH?#G9y2I&sYBd z0qs>;K*j+lpaZ$3S1rPf=Yfv(D@-iTBKhu*%eZs1*?mvFE8KRl^(&_?Hi>WFWQ!Swe&y@x%;RGLScW|8rnZ2)i-dJkb!vb!7C zLKfX|cb0NV%R9H~O-}O&t^Ck7mr}5=bx!T|JpQ#_La0n`NI<@1wj9W9D*xXex^p)K7*6}DK7W}C$>-H zP+XFt7>Dr23HrAu`HITrq)+GYxg&v-_|t{RSXqh2SSb9BP@Y003O$GEPeLSxx8y1i z#Qt=yGTNUu5xb9X#*jyhckfje;hoPI2a2^6z^)1U(irl9;Yh|tK9x>b0SbLjO1NJK zIpowkx149AP&8ML05(2)91%`1g~obzse%o>9&?&=sPgDSY@pHYa%wfY2`E#+Clt3c z43MqPDulU_44oZ#`ubHNGgRFyp@8e05!_T~_iY@bqPF5coo6dH^#`cyT5+j(fIfu& zw9_SoX|m=>NcnjJuG=@3N=g2$xNmQIh(W8)rZWyb*RyI;Nz7TKH&7N2$V?}Q~t3BAMG#rkL6k=av1wNri$mu zDjGJ7Gb!)RD_$7P5(y(YL!GDk8pMrH+rS;5&;C7Ic8|3PZdCOY;`%a8OL8{lN6Xz$ zZ~nDe357d(Q`8!x8yH`%RP;4$N;d9Z-9>Dla#{caWas7Yp4EXp%jc*viB=dcbPI#K>2x*~i>U}f89;2=*-MkRl+n*r&AbjXWdDJGA}%4vredE&mf+KTcP$fT*+u?qS?7D+fG5+PwP-U z>`r!1>BA19vYPYl7VRyoGX?wcH!PpxJ?kO$WRr1aZOPXiYFzA6r*om6IienHG6^Ca z<$ABwRz>~B$c*XN_|bYdzCWc$ZK%kYlHOd7xXnuU7I*4*e&%EMiVTD9RDUt~8qKu! zG>lq$ob;}fatKm-2IxoEx8qHP_Ayhq9_ndz3mNRnIr-sp@=AwtpT?qD0a20&9>3>` zUm#UWv%VylL;*D zWQ3lfTZ8o-4I;_rsa8F@)h1?wU9I;?;MBL(7MDhJF!+Kw+ZXzD5}t-ge{}x7DwaaNbhnL%^~$A0y% zGysp1IDqVSXZh7eRtE)$`f*uJC9^T9QZ*xDrt*>n!vF?BZYxe7nh_wCI()z!pVG2# z?0(Y;8N&~{$F*kZTBNo&Ceq{;aC&w>N~pBa%kR$lo!;d(~-dh~!|3P2`Z+%}TTG`Fdy8 zuIgKrs@SR*X~Co_ry$jdQ9;2K7QhB^QsZKiC4q#|gG7-5&MP;=_vvt(3FKDQzyT}O zRPL^M3LVO*!*Bwx$AYzvB?$Pdu-x&AOG52p@dFi3bHNoapY2trW+x)8G$zW7gttn5 z&;>je^rUUoPQ=}fP!^OsX9Ad8Ja?%fQd>Bzw`RApB@wfVdO{Ruy<3R}-juRN3e#~X zn5q~Krl68k+;C`>2q&7)bZz;`snHdwb9G3hHBb@}#Y-Icp|m>^%r=V5k%4}4eT7-N zy2{{HTXgd!1lOHjZc|Z5HAi=3TUgo93}AW+O^m0dXhj%Q6v>wdfnD^~k)&f}Qpc@1 zW6dr(=~9njRLe+bmw{2^X{nCttuqy>T8Xm%3ZdKzikYX_nB?B5fNVK^SW%|uwc1zu0JQdv(3qbz$=i69>;9Ov&hudw#2=D4{_pKyG< zx$ZlCKgO#{;Q>|Ze)E4ydrBAOA9(fls_t<^VzCj)Y*cT+z&v+8wJt^~76ph331h$m zl1*mHAl>uhwM@u~iEfDtjHE!}pWONnf69^FEJ-9Rw_crAKk+}HsuHuv#pGhn+_5K< zOKrn)OPlwd$3Uy{qNu=8>?$U9cUY4wNx%SPlj%@n=g(nQ$uJShI^pXO^O{^VraKyrkSF~{+rPxnV}&Z)jt{eM^dgq6(;Mn;K}AN514k%J-i z_8x=r>sBqTqLJR>N=oOmBYYpdHEf9co*dmU$wzha%b&^5=rb zyB~8xdR>FP?1Z|wdzNYB7#$UVx+=0D3%&m8jmO{Gq?LopRetL6MZ+q_sDAD@ztDF7 z06w&vySoc$W!)100J|sVW9CYI`wze$)|nh<810PI=^H$I-Dsk0Jl1~72_KbI5VX(FK}n4n%>ii(@{rj#Xz(yKP*keD@ce~Wgpt? zZ^^aIP`}ZeOVCKQ4a?)Pn3tnM26!LTg`8Q8&_c!B#wX z{{R}RsOpDGw(|xaMmUX6M*jfoRamaG*QPtxQ(cWx=)&qmEO8Sac6y(o?@Dz!SD!*B zriRvyZ4>X+Audd$kRR_M@6UXmhtjsvZRC|0an*ml^#1?~$cE?aWg`G8fwOOJJ5Hf7xMjzf4%xd>7J@zcow&QUFo;`hswM|=C)|{;DX0_GDw7z6+%OK=)gHLk~v=xCC zx`-bujuKPoeGlkrIbCNsVS;^oA4<;+hUeD@+PNJ@wCHxIC$CZBdz)K_j#OypPO5v= zw4Eja+qW`_PGhWW6{F%`}^UqIfY`Bh1+X0n*9s4{s39IFpdXl?Gc4K{n4 zkX}fkmvB~Zyc}*xJ(zSqTCF{t7j8wiEpGZ7c^l+^_Za^GBl()IygG5g6y;@_KPq0_ zD<0n8O0_GiG8}EHDJ+sjBSA7UNQVrLFjk?wD(SU{2fw9Qp7cj5+iGa?#-j-`{{XFs zdmrLoWB6jBy0>(VU0r5@_>ST}RlSdJeUCIXZBWy`=Ra{9+eyL79%>huIQh9#S0{Wi zNd_xAXz&R`$E8CyiY!FoSFS}^NXQ}MJa?>^q$tDRt!P0pLJ0gtVI!g_*|cC-1{J?+ z3a}N4WD;BvTaY6r2&~rRR4tov-E&puu!=+XdHPkz4Q?@i&O44$512Wen-D*xZ7mpR#=5YmR74NBdDSj9 zb=kDL1zk_zN2so@FqccZG6VM>wNC!V4Nl+6kZhHCj#1T3WLR9>X*Y(}@L}_i-JE)Z zT3+htoBKDtL@PKa2Bc6h zN|E%Yv;zgUV-&&)jIME2siudTXxiDa)Iz&_q~g6AJM_4PST=Gi#xFrhOAdh6&W++X zg#=Q$!02n!r&CKDT&JUzQzmNMN&@D&IJK2^1WBL9t;ei!_o^i;u{fP8N~_kH_Jk*s zSifnKRNi=B=T&CAW#K{1E@LFMI%SqJDme~%S3hrgH*jkDU6a=$t`6pkD0}&WO!ljm z((}(1Rc2Q1D#9ouiq$|;Pob)lAYQb|Za`CmS&q)0wNhKA8Km8af}O>feS1 zEyR=DW6pn(e(Y*Cs{H_~D-b;fDbEnaODP$V$sv3W6jqg`;gV5r1{sGW_8z33!n3A= zX{GX^A2H`SzyrA_uoW$;BnV)DorLB|2*H&907V^#;%aO=of^!hEvVap`_KLF@D(gm z2++uVaHG`LI!?^%1bz6cZ7LT4ACWj;e|k^3N%S`;q> zGc*4Hs9a|qw`2Kx)n{+8hEUnNek6S7U_DfO`k%wrrn|XloXpr4EarKzR+G=FDL^{>R3{t^A(@%*ZQi!x;N6zGm$ zapxzkXj;OF9G8;-VTYoh`Do{&{X3ujy+iky_9QR%TX7%!wv|DD_7k%GPgD3+WS3^< z9eY)FndOn4iDqM#C%L3z#ACH5AV`|szG7JQIW)49BxLuIvG@=Ezw)GtJVwXx6>DHk zqx*~c(4K%pDOeD9W6)G%I1JrSYJ7-gpK<3L3Y0=eBvQB)1Vw|KaezNsks%1RKEfkD zl^rLVy3T26?WnAET)#r?=liZrDS9H0g1Pu-& zHBI8j&{o68i0Xu?oG+=O!%V92;Dg$!%7Y*eTGn*=YMiNtUt>ZdsHM;xRTqJX5g6Sn$UqhF)saE0zjJY*O2`y~hzsu2-lBkSf~1JitKW zor$>%0*Jp;{VGWFw1eE#(?$ps#auVg(qt2-DhI7Z(x@d#zz2i%qDI*0z5f8inxm*V zSfq43vWCad=B+ZJ8?wFNk{f`*hVQ%&uealhjdcMAs9@-r00ZgNe>$xl>AQ#)z+^c3 zj%v12OK6?j_p7-#W3D+Lp{!K(b~dLCWSO$pb2B_jPRAI}r!^!6W6LW5K*H4s7{_d8 zla|L{TFGmvkgs5N1DuWcrL^ce&GdLZ=oAIs@euCmaC zAw+=uj)$X;)`CNBl-ns)JC~g>RE&(b1(~dx`-CJwas+OjU(&~=| z*n4+%K-r00^N)Jfg61O;xwZ!f=9(eZPI`}Q{{TNjT)a*mLh>f~*kPBxJ-rQCQ7)Y_ zG;-!9`I205>5@M}Ju~f58k*hz0K*dqTUl;t-`K}6`)#2GPt7E2+tr7+sO?xXj7(T{ z?kiS1#ZkPSMB#l=(+S%~ zP>GbQDEz9`z2c;QWw}EovE%{wqdu$t6_{pBkKsMOl(b6k6p|6R-yp1 z#~^qg?n=E6vGhLGSg{<`mUmKHNb!&)g~!bsbZ`E>SeDv0h}_ze@P87Yyob>L0F7Ho z>WNO;GdAH??mr>^{{UFq)7rBv?rwDHoFP)J#E{4`k`HnHdWzGYEDgZbS)&(7W$7aD z`0wvZ?6i>Ut6P0W-R86mHVnxVa`Air0JOgU0G)GF#`fnp%AWPP3p^Jp;o3J-m61*e zHDd18EkJ(kM-AJ5%RZ6Jf7$GXVc3E@nym`uI_79(A1T^tqbff2P_nvN6bC%|invoehRCd>mCkv+Nl}Sx92&Q8 za`F79k?&c8Go8xZ`c=4#0hT<9o>Oi}$~KXrr6lD@2hi0^>sart%rh@eRlA|}6(lo8 zK~@LrPmo}6cQ5KGG}fr|>fxnL-WwQpH`f|n(6_li?!RUg*FkW%k%FuP{{T9|xw1>h zN=P>;=togm_V+g$J<7#!#iBoQPR5AN#t@TE=<4_FB^Z$1!Jv*#R+|*C>JctmY=CgW znTeosk{pWNO6Q|WqMGwOuTj&$fI5z%xl9z8fO;DA3*EnEz~|-SxOR^h1oq8#LF!=_ zdzyA$V8$5NNv6Wm&Y_9gIW>!?sK_F^T_|k@o2Oc-SSI;Z&3(PIg%s#*QbsuyS1%wq zJOP@0ozY2f=e;DfxS_G99fB`WQ#GV&xxl9_!W7`uE8E?Xk_|LALv)K6EgN9gYU`3) zOSQ&8tlPatc_b>J;;MOa%5re1vH>FsYM zjxCdpeQRjSGeWiYDwu>)b5cVX&V!3Q|PcpibSl2Qc zWhyg)TNX%@3^D6c-02F6TzzUR<*_X$W?RMb4At1})s1JHpCVv{a1Cl(OtOrMTWHGB zXkCKI2{ibm#Rx)opE9;3y=X0IHgk z{hAhr5td#~;%It|*6<+!{;xNVY=kLBTy#D})p!{jjj3Hyo8p0oFlH@MK zjPqO3tbz~#!Nn-H^<>3gQSmpC9FqS45&2j8{{TvTxM?3G@jf9L0lD3}B95f{3b}tj zn(k@7(=pD7J_!@s~e6$sXK<{5$)fYK3>R7jL}BHCrnScX8_&104r* zO~GUGjy-C0h>@2)YGEHhI#k$WVThfodz_k>p?OFjjXp3Gk6%ijCFF5V#N1?d;Xv+b z?GUR$Fe$7CD!ECyNzfv5>zbz~UBK)r+~l97S6I$@?^4P`qmPc9RC0`*Rjgqu76T-j zVkt}jUwVe#4E)BdI++-HW~WHPrU0Z-PGrFLthq)VkEcr7n~()%yhM@@QfOd^f^Nk_ zG+9!8dRC8;RwCWU0+}>ncARwLlH3eHq&9sHYi7lPw@^C@&yq3bx(6*{DP@GDbLbxZ;id@d;RT;#N+Dv?B08;{{V$lkIR}8l4mEO@0yPGe9s^(c0bF{BO}t8r^76_PSPm_I486J z0QKp`B^7bgwJKL-`xrcl+U%9xw5qn=fQ%o;oA#x+x;DV_Mh86)Z}aa?yE6TnQ3oul zTmJyBj`*%h-f5wqd1fpdC!Vy{x|vgpto@_;9lSUMDH!cnQI}zcU-fKq5BCqR_|`Oc z%QSFY=ZQY>{_BpvpI^qcB5>fTBC~D9gzwaBqH9Op8isUTu7{+t^~d-g+__8Lq22^-jm!5K{{SQHQ29!Q;&6P4dV~BY*YN&TtSZ=0pK_k3 zX0jt(+eT4A+cZ7lZ|bM9tk~J3BX?ehHNS4MthfXVsD|I0kt%<5`q6WyLo{A|3AGPY z?-}$y-iE46n^bkeX=ut2NT(U(q85(2Ati?yKvHK_jjJBryGIRIF&@ zb5?t3uHk8|hSCOJEcEvt)rtP5Z=0du`-;)DxoyFL*z3^L*S0Br8%YX?<`MSoHL*HP{(BI4R>8@9K&LXJNp=t1s1>h08PaTee(%Si>uo)xpc4Dhe| zxfEsLKHjV9zm+*PC9dVh@!Mf7&Hz1YR^5rgQb6xn%V#yL!5oHG8wG!_H2OL%`GRG?HakSOo14IC9k?TzvHFDL(ycZ*K^)-oi zq{q0E!1k?JU87bMGnaIBcN>dxKa8;jN^@ZYx%j{{U_ZrjcvaoLSZGgTjwuUhCl0`b|#yMjmyWpLT zX}u3Tw-O}F7|$ma*F-#vTy!K;ZM1MNH_&|3Q=~T;PFfLv5|(R;Kg*yw)Tn{c+M)uqX<|F4@z4(gGZbSTZj8Z1wRV7 zDf&I;5V@{`eIX>cAdYig2AieD6e9{La$Jedn-1`S$^Tax6N3|5H4rA)R84;?D;&aN@sW>#oHI$^8N(MjnMYx{RKYb_i>n^IbY&$?;r52sik!U z5JhFpt%3_t#n(G?ienB=1yar&xnf0HNkf2pQj*ZRnk$irH_8iitFgrMPaU)ix<@zz z-kTWbC$(uTm4@dm39TzQOc9&L19&@O2m9UI@cwx8sBJA8dxTD& zr1k@;tw}=2;GiNj8&Q1+`RDPgit1-B#&W~9H)G2=$3L0=l=O=ovTD;rb}l}(EKpz% zS}X)97jyYj%5zqhFKSkgXz4|XxmGyIV^<=NVD~h{z#LTR&ot5#V0hxAZgM*gwIg@M zCNP+xf+{k1rUMEtz=kkwVPqaaN#}2T36^-EzzsLlNkg7(=^XB9+FuJNIG_McU5L8P~kT!|u(scycyIxGjk9edSTVNo+HcjOV; zr8@*|EzJA9Ijv+4OEMGkvZ!6Y#PzGT(g@{3!v*;O_Bh2$9npt$7emc~_|JS*StCJr zZjeYGC2~O=@kG7r7g92mQkPDp1QM&3R7Tw2G8~m``qt_!cJOiXs2%ymQ?ZGbM`s`e z9KULUd!uu3fk{}<{F&~3O;gu%N|gPpQClK8<(*{nK|kv| zA75(LSlp-%fzeOQzr?>o=~nGvNafud@Ou%~pEpCHMMY?1Y4AUnECdp+yOiUd%l&bh z)3h!|(s>o4Z6jA(h(H7!vW}F=tkoZSa6PJJ8=_>Ci6WDJ6~6al>;C}Pt7Sy7^Y?{8 z6h2_d7(Ma)>dFQO=KJ5yx!P%*bEf056QN~SQ}f6Czxvf;8H%4d06nBb(EkASjSPSs zj`YB$H^>|wzV%V5VqsUK8+l>~`$euyXR7y8=xZ_J>_!i*ZbIfKByy}UN2sPGiF-l$FlPbr!}M1qj>*M|!mj94jjj!;nX6ds~Cor@c4IXt`6Bq?ys93GKz% zXN(1$j5-dEJxBOf45Y4Ah8P@Tw`I#HWjkDQ20xyNZfp7o>mmzgWw zentbGx!cgw5+ECJs|=N0xsFuxA%DKU{-fTSchQ-@Ab65h-3bK#wImX{0J3xa zDjSO<3oF9^04s8iIv@VOwIo*=ZL)Gf`@)k=J%+5&qK>3s{Mf1O{!&7o4{E5YjKS29 zd;8V6Q;x zV|0F%+e>V+AS82Cp2?FW46mWCo>QtbIL27rikz!et4Ur1jmK%=irKu# z$DX35hIL%SCLIq2TB%1H54qG-&9*~^;r%FUnHf%;WUV6Yyd_(uSZKlRPK+O#pq^r! z;=L$Z);#p|xZ@Ila5~cksK9Ns*zPx0qDEwkfyOFh9%(jZ8D3X1u?t!|gr|d1NoH7s*A-HB98AQuc{w#Q*_G+m zw7jsZjC86q+?b!I6)tYVvblEI+6YK)Y8bDCb27Q%jYOBK$&y7^HJm$5h?-|ZVc>D~~q1>F- zY70X=H{w1M~oHBB@^Xs;!*$oo%2PhCD+XA(eir>LzjDM-!^eJNsWjiDS-9R{7A zg_~2pKbwP6n2A4gBkNMyS~)v_I6aMOrK%1{=CpE8P;TaY8hSU)$Tgsmgct^%Kwbqj zw>*mHoXSxH$0z>)*HJM!=A6x*wHux@ST<;o(%9=!vCn!!bKaQEg(uRY<=A0yAS51@ zUiHi{Iv>8sNzY;F{xp^kv`H!K#3((_{{UK{zJ2?H9F|f(yiv_%a$CAFyy7V)MV2~; zc3aw~+45&Kj2jd+W*x*NHbLN1tcw?I!+}rKW{Eyo0blloL2L4iAue5j7_F1KG^rnT znz69KW*Jq%BiGurUP88!D;&u=GdcYi^dGHdGXRAYj4x6-X{!}F8*Q#A9OlULzSI(1Rhwgpkt^d^okv)X4EdARy0{{R#GszgkI zPo0CvcKy-vuh4X*jC_Rmtvrmp#Av}Ij+J6G80k^=%}*apQ(Kb9B6r6%9Fga(SqF-2 zfB~AT5>fJqH6Y+%Q;$6;E4GtBG>{HE(iWhpCyu6^Av_aOP~7Jjrn3Yd=B8x|ed;Zv zIX!3@6E1l@Jt_%CIVYO6BZXS3=OnQoN{L9)F)2Ji9G*!%O;?jF%onFX2YR+onMdi%cW&mv+MMrRakn%Twx=%s=H3X~!Jj5IxgQ4kCn=Z(!HPZd1 z)F{U8JMmdMg~OOh1mw8{dk$$WWRiQCi;O8?bM2fFO4jD&ODlcn$-wP`PxPR*WJNBj z(gJO%B>JRkTvJ05g1ZUGce?P*Zg3TehfT#@TEx)p2q*-H@NFcW5Oqr08 zjo;om^aB_k{Y=k2T9lmY&;~!7!Rug^zV1FLb6H$L zDkveJYbo!?Ut?QP**wk)BL?^Ae!i6ss|+^LG>sva;a76`W8^?R`}M1YTh#ATQjaN_ z46|PsK@-NYovgm6*NVQD0H7^~$6;A|JaRM=GReo8jk)0Q*0n-OY(VHwJanp+u3U6l z4vmaXayjU7dWx{E9}F@v)Ou4E7L;xs`xvU@E#cK*(TeUEan{Nz}JtvF;dX$@+R#<#6aSpdV_k ziGnuZA4--|(R|p+^rF`lW}Tw9Wa}7lfPS9FqhLn_Q5(lEsyC--Z7g zx+65sx^=o4w-)>oo%>c5^lW0>G94R&E2?xXK3d1WfCYb-2N}g~wmKs#nMo>YspZZ@ zcVJ*hcn|D-{p&JwJ*=%1szFnX8j|4|PU!meAW$!o@ddr(?P0`o5z6~FsHt|A`suYZhh1rCcRA}3=SjsT-A#c8! z$KpK=A%pBrtbj=i@~w`(-pAUi8B}1ddGAlOx-qjO<>6!VAwQ_}H2JNeE;lt`xwm`< z8%Kd<;7YP5(^D;ZsGa#f=ktHrbuvu-@oV!4qR5CIvcIp(Lf zi+9qZ7jHv~Dx-IcEvB?o;i5S9s}bs!(u@`=eZ^Li%rbnq$f|N$73eExIm2?@T9Mq@ z#7@~qQ}i`T_e~cADn4=0ip~*Ar{&1{RG(?NhxpIqN3~CMX&hH{Rhk*%;ei76mf(|eM_vB}D|-)gY-ir^3iM{!L^&>XI6 z!i)-!Xk%9uAF|6aTd)PNNc5_AknUlQ^if{SqLeOkrul4C7VORydz@EKYXYPoA4({w zq40_c6PQCG=~#D`8{GDyirPx$r>RaG285DG+f%9u4il{uQbLn=8uo6@hmC;s6n|g~ zfC8{=^v@JgSIC+b+GEqfwcC7uLqy ze+a1syi3tV6jO3xWIb@50HiWviYgqLN+@Y;ifK6*2Z|`EWmt`cLm}=@6j4yiMQI-& zcA+vBQlsx*$9Ta8XymBe?7c~Z+!rm60TO^YlOYi(g@Emv=x|`MbOY!7@xW^S@5tH?z zik0ymWi2X~p!qZOtqT|Vq?7xL!S_6WQ$-cB-0P7ja4<45dJ3?I6~Emilp|gmPI)Su;U-rszs-}x)mIz zcIW%ub43(fYhr7CA>o4I)xLRxf)5~M<+%Rgf! z%!P|O?c3A}D6Jswc2hZx4>tEwMI&P*sXq15C`9=VHucCkZ~nCuR*!Q;D%(QNUKt5- z*qXYNsX())03Cg(qLq2rg;=mSlixa6t!|>k?m8tl#F*K znA(lIzS$MWZNrg86uGRfZ9#L*x6ofR=E{*G_Bg>6rD*Eo89a`2MHO8Uw4$BH&mPvn z%ChmE#CE1HBOa7dSk#KS$v7(x#wqCBVDzGj=T6T=dsU9t1L+J38A~L?&rwCN?2kS)@%Tb(_p%t+z#s_MC0HTVBE3=KC|Jir} Bp^E?j literal 0 HcmV?d00001 diff --git a/python/paddle/v2/tests/test_image.py b/python/paddle/v2/tests/test_image.py new file mode 100644 index 0000000000..5aa612f91d --- /dev/null +++ b/python/paddle/v2/tests/test_image.py @@ -0,0 +1,42 @@ +# Copyright PaddlePaddle contributors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import numpy as np + +import paddle.v2.image as image + + +class Image(unittest.TestCase): + def test_resize_flip_chw(self): + # resize + im = image.load_image('cat.jpg') + im = image.resize_short(im, 256) + self.assertEqual(256, min(im.shape[:2])) + self.assertEqual(3, im.shape[2]) + + # flip + im = left_right_flip(im) + im2 = np.flip(im, 1) + self.assertEqual(im.all(), im2.all()) + + # to_chw + h, w, c = im.shape + im = image.to_chw(im) + self.assertEqual(c, im.shape[0]) + self.assertEqual(h, im.shape[1]) + self.assertEqual(w, im.shape[2]) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/setup.py.in b/python/setup.py.in index 5dfb46192a..7d9438e3f8 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -18,6 +18,7 @@ setup(name='paddle', "numpy", "protobuf==${PROTOBUF_VERSION}", "matplotlib", + "opencv-python", ], packages=packages, package_dir={ From e3fd0d56f46b26894c682213f17272c122d7c2a8 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Wed, 10 May 2017 16:34:24 +0800 Subject: [PATCH 26/48] Pass unit test --- paddle/scripts/run_python_tests.sh | 2 +- python/paddle/v2/image.py | 19 +++++++++++++++++-- python/paddle/v2/tests/CMakeLists.txt | 2 +- python/paddle/v2/tests/test_image.py | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/paddle/scripts/run_python_tests.sh b/paddle/scripts/run_python_tests.sh index 05c187522c..02d2cdb977 100755 --- a/paddle/scripts/run_python_tests.sh +++ b/paddle/scripts/run_python_tests.sh @@ -29,7 +29,7 @@ if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then fi export PYTHONPATH=$SCRIPTPATH/../../python/ -$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib ipython==5.3 +$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib opencv-python ipython==5.3 for fn in "$@" do diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py index 20dcc96782..ca75efd90d 100644 --- a/python/paddle/v2/image.py +++ b/python/paddle/v2/image.py @@ -45,8 +45,13 @@ def load_image(file, is_color=True): return a color image. Otherwise, it will load and return a gray image. """ - flag = cv2.CV_LOAD_IMAGE_COLOR if is_color else \ - cv2.CV_LOAD_IMAGE_GRAYSCALE + # cv2.IMAGE_COLOR for OpenCV3 + # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version + # cv2.IMAGE_GRAYSCALE for OpenCV3 + # cv2.CV_LOAD_IMAGE_GRAYSCALE for older OpenCV Version + # Here, use constant 1 and 0 + # 1: COLOR, 0: GRAYSCALE + flag = 1 if is_color else 0 im = cv2.imread(file, flag) return im @@ -178,6 +183,11 @@ def simple_transform(im, resize_size, crop_size, is_train, is_color=True): Simply data argumentation for traing. These operations includes resizing, croping and flipping. + Example usage: + + .. code-block:: python + im = simple_transform(im, 256, 224, True) + :param im: The input image with HWC layout. :type im: ndarray :param resize_size: The shorter edge length of the resized image. @@ -209,6 +219,11 @@ def load_and_transform(filename, data argumentation. Please refer the `simple_transform` interface for the transform operation. + Example usage: + + .. code-block:: python + im = load_and_transform('cat.jpg', 256, 224, True) + :param filename: The file name of input image. :type filename: string :param resize_size: The shorter edge length of the resized image. diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index 0b8c78b465..eb02e53706 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -1,2 +1,2 @@ add_python_test(test_v2_api test_data_feeder.py test_parameters.py -test_layer.py test_rnn_layer.py test_topology.py test_image) +test_layer.py test_rnn_layer.py test_topology.py test_image.py) diff --git a/python/paddle/v2/tests/test_image.py b/python/paddle/v2/tests/test_image.py index 5aa612f91d..b2d773510d 100644 --- a/python/paddle/v2/tests/test_image.py +++ b/python/paddle/v2/tests/test_image.py @@ -26,7 +26,7 @@ class Image(unittest.TestCase): self.assertEqual(3, im.shape[2]) # flip - im = left_right_flip(im) + im = image.left_right_flip(im) im2 = np.flip(im, 1) self.assertEqual(im.all(), im2.all()) From ad0608847e8eef659f57d580c3816af653275339 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Wed, 10 May 2017 19:30:04 +0800 Subject: [PATCH 27/48] Fix paddle.init bug --- python/paddle/v2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 70c8ec0baa..d13850597e 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -52,7 +52,7 @@ def init(**kwargs): args_dict.update(kwargs) # NOTE: overwrite arguments from ENV if it is in kwargs for key in args_dict.keys(): - args.append('--%s=%s' % (key, str(kwargs[key]))) + args.append('--%s=%s' % (key, str(args_dict[key]))) api.initPaddle(*args) From 3513935f7ee4a6a9d6eb06b37dd681c015a37694 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Wed, 10 May 2017 19:30:46 +0800 Subject: [PATCH 28/48] refine the annotation. --- python/paddle/v2/image.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py index ca75efd90d..13f53919aa 100644 --- a/python/paddle/v2/image.py +++ b/python/paddle/v2/image.py @@ -11,22 +11,20 @@ __all__ = [ "left_right_flip", "simple_transform", "load_and_transform" ] """ -This file contains some common interface for image preprocess. +This file contains some common interfaces for image preprocess. Many users are confused about the image layout. We introduce -the image layout firstly. +the image layout as follows. - CHW Layout - The abbreviations: C=channel, H=Height, W=Width - - The default image layout is HWC opened by cv2 or PIL. - PaddlePaddle only support the image layout with CHW. - CHW is simply a transpose of HWC. It must transpose - the input image. + - The default layout of image opened by cv2 or PIL is HWC. + PaddlePaddle only supports the CHW layout. And CHW is simply + a transpose of HWC. It must transpose the input image. - Color format: RGB or BGR OpenCV use BGR color format. PIL use RGB color format. Both - formats can be used for training. But it must be noted that, - the format should be keep consistent between the training and - inference peroid. + formats can be used for training. Noted that, the format should + be keep consistent between the training and inference peroid. """ @@ -85,8 +83,8 @@ def resize_short(im, size): def to_chw(im, order=(2, 0, 1)): """ Transpose the input image order. The image layout is HWC format - opened by cv2 or PIL. Transposed the input image to CHW layouts - by order (2,0,1). + opened by cv2 or PIL. Transpose the input image to CHW layout + according the order (2,0,1). Example usage: @@ -116,7 +114,7 @@ def center_crop(im, size, is_color=True): :param im: the input image with HWC layout. :type im: ndarray - :param size: the cropping size + :param size: the cropping size. :type size: int :param is_color: whether the image is color or not. :type is_color: bool @@ -143,7 +141,7 @@ def random_crop(im, size, is_color=True): :param im: the input image with HWC layout. :type im: ndarray - :param size: the cropping size + :param size: the cropping size. :type size: int :param is_color: whether the image is color or not. :type is_color: bool @@ -180,7 +178,7 @@ def left_right_flip(im): def simple_transform(im, resize_size, crop_size, is_train, is_color=True): """ - Simply data argumentation for traing. These operations includes + Simply data argumentation for training. These operations include resizing, croping and flipping. Example usage: @@ -216,8 +214,8 @@ def load_and_transform(filename, is_color=True): """ Load image from the input file `filename` and transform image for - data argumentation. Please refer the `simple_transform` interface - for the transform operation. + data argumentation. Please refer to the `simple_transform` interface + for the transform operations. Example usage: From e59c183afb5c86055f0b29b77e37b1d5f0673c9d Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 10 May 2017 11:12:32 -0700 Subject: [PATCH 29/48] Remove -D CPACK_DEBIAN_PACKAGE_DEPENDS="" from the invocation of cpack --- paddle/scripts/docker/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index b1a8274b1d..101b44e6c6 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -113,14 +113,12 @@ fi # generate deb package for current build # FIXME(typhoonzero): should we remove paddle/scripts/deb ? -# FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must -# install them in docker cat < Date: Wed, 10 May 2017 11:15:49 -0700 Subject: [PATCH 30/48] Remove the incomplete file write_docs_en.rst. Will create it in another PR. --- doc/howto/dev/write_docs_en.rst | 77 --------------------------------- 1 file changed, 77 deletions(-) delete mode 100644 doc/howto/dev/write_docs_en.rst diff --git a/doc/howto/dev/write_docs_en.rst b/doc/howto/dev/write_docs_en.rst deleted file mode 100644 index 65e7edca94..0000000000 --- a/doc/howto/dev/write_docs_en.rst +++ /dev/null @@ -1,77 +0,0 @@ -############### -Build Documents -############### - -Document files of PaddlePaddle are in sub-directory :code:`doc`. Source files are in `RST `_ format. We can build the document by letting `cmake`_ invoke `sphinx `_ to convert RST files into HTML files. - - -How to Build Documents -====================== - -To save the time of installing building tools, we provide a Docker image. - - -使用Docker构建PaddlePaddle的文档 --------------------------------- - -使用Docker构建PaddlePaddle的文档，需要在系统里先安装好Docker工具包。Docker安装请参考 `Docker的官网 `_ 。安装好Docker之后可以使用源码目录下的脚本构建文档，即 - -.. code-block:: bash - - cd TO_YOUR_PADDLE_CLONE_PATH - cd paddle/scripts/tools/build_docs - bash build_docs.sh with_docker - -编译完成后，会在当前目录生成两个子目录\: - -* doc 英文文档目录 -* doc_cn 中文文档目录 - -打开浏览器访问对应目录下的index.html即可访问本地文档。 - - - -直接构建PaddlePaddle的文档 --------------------------- - -因为PaddlePaddle的v2 api文档生成过程依赖于py_paddle Python包，用户需要首先确认py_paddle包已经安装。 - -.. code-block:: bash - - python -c "import py_paddle" - -如果提示错误，那么用户需要在本地编译安装PaddlePaddle，请参考 `源码编译文档 `_ 。 -注意，用户在首次编译安装PaddlePaddle时，请将WITH_DOC选项关闭。在编译安装正确之后，请再次确认py_paddle包已经安装，即可进行下一步操作。 - -如果提示正确，可以执行以下命令编译生成文档，即 - -.. code-block:: bash - - cd TO_YOUR_PADDLE_CLONE_PATH - cd paddle/scripts/tools/build_docs - bash build_docs.sh local - -编译完成之后，会在当前目录生成两个子目录\: - -* doc 英文文档目录 -* doc_cn 中文文档目录 - -打开浏览器访问对应目录下的index.html即可访问本地文档。 - - -如何书写PaddlePaddle的文档 -========================== - -PaddlePaddle文档使用 `sphinx`_ 自动生成，用户可以参考sphinx教程进行书写。 - -如何更新www.paddlepaddle.org文档 -================================ - -开发者给PaddlePaddle代码增加的注释以PR的形式提交到github中，提交方式可参见 `贡献文档 `_ 。 -目前PaddlePaddle的develop分支的文档是自动触发更新的，用户可以分别查看最新的 `中文文档 `_ 和 -`英文文档 `_ 。 - - - -.. _cmake: https://cmake.org/ -.. _sphinx: http://www.sphinx-doc.org/en/1.4.8/ From 8673366329b11bb102a5639b516c84d938f25863 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 10 May 2017 18:17:11 -0700 Subject: [PATCH 31/48] Abandon my change to the Chinese document --- doc/howto/dev/write_docs_cn.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/howto/dev/write_docs_cn.rst b/doc/howto/dev/write_docs_cn.rst index 5d1297d079..d536f53abc 100644 --- a/doc/howto/dev/write_docs_cn.rst +++ b/doc/howto/dev/write_docs_cn.rst @@ -2,13 +2,14 @@ 如何贡献/修改文档 ################## -PaddlePaddle的文档文件都在 :code:`doc` 这个子目录里。源文件是 `RST `_ 格式的。 在编译PaddlePaddle源码的时候，可以选择让 `cmake`_ 调用 `sphinx `_ 从 RST 文件生成 HTML 格式的文档。 +PaddlePaddle的文档包括英文文档 ``doc`` 和中文文档 ``doc_cn`` 两个部分。文档都是通过 `cmake`_ 驱动 `sphinx`_ 编译生成，生成后的文档分别存储在编译目录的 ``doc`` 和 ``doc_cn`` 两个子目录下。 如何构建PaddlePaddle的文档 ========================== -为了简化大家安装文档构建工具的过程，我们提供一个Docker image。 +PaddlePaddle的文档构建有直接构建和基于Docker构建两种方式，我们提供了一个构建脚本build_docs.sh来进行构建。 +PaddlePaddle文档需要准备的环境相对较复杂，所以我们推荐使用基于Docker来构建PaddlePaddle的文档。 使用Docker构建PaddlePaddle的文档 @@ -73,5 +74,5 @@ PaddlePaddle文档使用 `sphinx`_ 自动生成，用户可以参考sphinx教程 -.. _cmake: -.. _sphinx: +.. _cmake: https://cmake.org/ +.. _sphinx: http://www.sphinx-doc.org/en/1.4.8/ From 45d800b34a880b34d0a69b9e0e698d0e7dea5dd6 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 11 May 2017 14:16:41 +0800 Subject: [PATCH 32/48] add tensor doc --- paddle/tensor/README.md | 125 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 paddle/tensor/README.md diff --git a/paddle/tensor/README.md b/paddle/tensor/README.md new file mode 100644 index 0000000000..ea1fddb665 --- /dev/null +++ b/paddle/tensor/README.md @@ -0,0 +1,125 @@ +# Tensor: An Unified Data Type in PaddlePaddle + +## Pain Point + +In this week, we discussed several potential weaknesses of PaddlePaddle caused by rapid iteration and development to promote new business products on the line in recent four years. For instance, current Matrix/Vector implementation in PaddlePaddle are long and tedious to read, which interfered seriously with the contribution of both fresh and professional engineers. More seriously for this issue, it will also become too challenging to maintain over time. + + +## Learn from Majel + +Consequently, we decide to refactor PaddlePaddle step-by-step. First, refactor and replace Matrix/Vector to Tensor, a modern terminology in the deep learning system. Fortunately, we can learn from Majel how to define a Tensor. + +To simplify heterogeneous resource allocation in any dimensions (1-9) and types (double, float, float16), Majel consists of several primitives such as `Dim`, `Place` and `Array`, all of them are standard C++ classes. + +1. `Place`: memory location [i.e. CPU/GPU]. +2. `Allocation`: heterogeneous resource allocator [i.e. 20MB in GPU]. +3. `Dim`: size of each dimension. [i.e. Dim<4>({10, 2, 5, 1})] +4. `Array`: dynamic array consists of `Place`, `Dim`, and a pointer to memory. + +If you dig deeper into Majel source code, you will find Majel heavily use `boost.variant`. The variant class template is a safe, generic, stack-based discriminated union container, **offering a simple solution for manipulating an object from a heterogeneous set of types in a uniform manner**. Whereas standard containers such as std::vector may be thought of as "multi-value, single type," variant is "multi-type, single value." + +As a simple example, consider the following: + +```c++ +#include "boost/variant.hpp" +#include + +class my_visitor : public boost::static_visitor +{ +public: + int operator()(int i) const + { + return i; + } + + int operator()(const std::string & str) const + { + return str.length(); + } +}; + +int main() +{ + boost::variant< int, std::string > u("hello world"); + std::cout << u; // output: hello world + + int result = boost::apply_visitor( my_visitor(), u ); + std::cout << result; // output: 11 (i.e., length of "hello world") +} +``` + +In Majel, `DDimVar` is derived from `Dim`, `DArrayVar` is from `Array`. + +```c++ +template +struct Dim { +... +int head; +Dim tail; +} +``` + +```c++ +template +class Array : public Buffer { + ... +private: + Dim size_; + Dim stride_; + T* ptr_; +}; +``` + +```c++ +typedef boost::variant Place; +typedef boost::variant, Dim<2>, Dim<3>, Dim<4>, Dim<5>, + Dim<6>, Dim<7>, Dim<8>, Dim<9>> DDimVar; +typedef boost::variant< + Array, + Array, + Array, + Array, + + Array, + Array, + Array, + Array, + + Array, + Array, + Array, + Array > DArrayVar; +``` + +Because `variant` may be thought of as "multi-type, single value", we can utilize it to implement unified interfaces for PaddlePaddle. + +## implement Tensor in Paddle + +Before writing code, please make sure you already look through Majel Source Code and grabbed the design philosophy of `DArray` in Majel. + +To assign subtasks to our colleagues, we have to discuss how to divide it to independent subtasks. + +- [ ] 1. First, we need to consider the third-party dependencies in Majel. + +Majel heavily use `boost.variant`, but we don't want to integrate `boost` into PaddlePaddle. It's better to replace boost using the lightweight implementation. https://github.com/mapbox/variant Mapbox variant has the same speedy performance of `boost::variant `but is faster to compile, results in smaller binaries, and has no dependencies. + +> @gangliao + +- [ ] 2. Re-implement `Place` and `Allocation/Memory` + +I found @wangkuiyi submitted a pull request includes `Place`. @gangliao and @qijun could re-implement `Allocation', because we have the GPU development experience before joining Paddle team. + +> @wangkuiyi @gangliao @qijun + +- [ ] 3. Re-implement `Dim`. +`Dim` is an excellent implementation in Majel. + +> ??? + +- [ ] 4. Re-implement `Array/Tensor`. + +> Prerequisites: 1 - 3 + +- [ ] 5. Re-implement fundamental operators for `Array/Tensor`. + +> Prerequisites: 1 - 4 From 85502783aaa85a43c82a851e9944f535f09c394f Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Thu, 11 May 2017 14:24:33 +0800 Subject: [PATCH 33/48] Bugfix: checkout input_type failed --- python/paddle/trainer/PyDataProvider2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 0e752c117c..a36f0ebfdc 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -270,7 +270,7 @@ class CheckWrapper(object): assert isinstance(each, collections.Sequence) for d in each: assert isinstance(d, float) - assert len(each, input_type.dim) + assert len(each) == input_type.dim elif input_type.type == DataType.Index: assert isinstance(each, int) assert each < input_type.dim @@ -304,7 +304,7 @@ class CheckInputTypeWrapper(object): def __call__(self, obj, filename): for items in self.generator(obj, filename): try: - # dict type is required for input_types when item is dict type + # dict type is required for input_types when item is dict type assert (isinstance(items, dict) and \ not isinstance(self.input_types, dict))==False yield items From ea25aab57592bcc4f3645b417c2ea000147f800d Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 11 May 2017 14:26:48 +0800 Subject: [PATCH 34/48] update doc --- paddle/tensor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/tensor/README.md b/paddle/tensor/README.md index ea1fddb665..71074e9b6b 100644 --- a/paddle/tensor/README.md +++ b/paddle/tensor/README.md @@ -5,7 +5,7 @@ In this week, we discussed several potential weaknesses of PaddlePaddle caused by rapid iteration and development to promote new business products on the line in recent four years. For instance, current Matrix/Vector implementation in PaddlePaddle are long and tedious to read, which interfered seriously with the contribution of both fresh and professional engineers. More seriously for this issue, it will also become too challenging to maintain over time. -## Learn from Majel +## Learn from Majel Consequently, we decide to refactor PaddlePaddle step-by-step. First, refactor and replace Matrix/Vector to Tensor, a modern terminology in the deep learning system. Fortunately, we can learn from Majel how to define a Tensor. From 1ba60de83610c3e37062bb479c94f9b94f6aa883 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 11 May 2017 14:31:03 +0800 Subject: [PATCH 35/48] update docs --- paddle/tensor/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/tensor/README.md b/paddle/tensor/README.md index 71074e9b6b..b37ed72216 100644 --- a/paddle/tensor/README.md +++ b/paddle/tensor/README.md @@ -101,18 +101,19 @@ To assign subtasks to our colleagues, we have to discuss how to divide it to ind - [ ] 1. First, we need to consider the third-party dependencies in Majel. -Majel heavily use `boost.variant`, but we don't want to integrate `boost` into PaddlePaddle. It's better to replace boost using the lightweight implementation. https://github.com/mapbox/variant Mapbox variant has the same speedy performance of `boost::variant `but is faster to compile, results in smaller binaries, and has no dependencies. + Majel heavily use `boost.variant`, but we don't want to integrate `boost` into PaddlePaddle. It's better to replace boost using the lightweight implementation. https://github.com/mapbox/variant Mapbox variant has the same speedy performance of `boost::variant `but is faster to compile, results in smaller binaries, and has no dependencies. > @gangliao - [ ] 2. Re-implement `Place` and `Allocation/Memory` -I found @wangkuiyi submitted a pull request includes `Place`. @gangliao and @qijun could re-implement `Allocation', because we have the GPU development experience before joining Paddle team. + I found @wangkuiyi submitted a pull request includes `Place`. @gangliao and @qijun could re-implement `Allocation`, because we have the GPU development experience before joining Paddle team. > @wangkuiyi @gangliao @qijun - [ ] 3. Re-implement `Dim`. -`Dim` is an excellent implementation in Majel. + + `Dim` is an excellent implementation in Majel. > ??? From 3c5e0f8a5db328205d29916a8fc9a6ced9a2e2ba Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Thu, 11 May 2017 15:18:06 +0800 Subject: [PATCH 36/48] remove g_pass_height_width in config_parse.py --- python/paddle/trainer/config_parser.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 32e31fe2c4..57d30b088b 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -138,14 +138,7 @@ def init_config_environment( g_root_submodel=None, g_submodel_map={}, g_submodel_stack=[], - g_add_submodel_suffix=False, - - # Whether current layer needs to pass the image height and width. - # Default value is true, but if it encounters recurrent_layer_group, - # it will be false. The reason is that image is converted to be sequence, - # image height will be sequence length, and image width will be feature - # length of each timestep. - g_pass_height_width=True, ): + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -1437,12 +1430,6 @@ class LayerBase(object): g_current_submodel.layer_names.append(self.config.name) - if self.config.type != 'data' and g_pass_height_width: - height = self.get_input_layer(0).height - width = self.get_input_layer(0).width - if height and width: - self.set_layer_height_width(height, width) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] @@ -3164,8 +3151,6 @@ class WarpCTCLayer(LayerBase): @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): def __init__(self, name, device=None): - global g_pass_height_width - g_pass_height_width = False super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) From 805088c615c32437cbdcd910582eb8e8bf174025 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Thu, 11 May 2017 16:54:07 +0800 Subject: [PATCH 37/48] pass unit test --- .../tests/configs/protostr/test_bilinear_interp.protostr | 2 -- .../tests/configs/protostr/test_maxout.protostr | 4 ---- 2 files changed, 6 deletions(-) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr index 9fae596f28..fd5224ca55 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr @@ -90,8 +90,6 @@ layers { input_layer_name: "__pool_0__" input_parameter_name: "___fc_layer_0__.w0" } - height: 32 - width: 32 } parameters { name: "___conv_0__.w0" diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr index c763a95f9d..03f4f3a31d 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr @@ -153,8 +153,6 @@ layers { img_size_y: 0 } } - height: 24 - width: 24 } layers { name: "__fc_layer_0__" @@ -165,8 +163,6 @@ layers { input_layer_name: "__block_expand_layer_0__" input_parameter_name: "___fc_layer_0__.w0" } - height: 24 - width: 24 } parameters { name: "___conv_0__.w0" From 02990b824d64ede5d038019f78f1dc34503e1f45 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 10 May 2017 11:15:49 -0700 Subject: [PATCH 38/48] Initial trial to port Majel to Paddle --- Dockerfile | 11 +++++++- majel/Makefile | 10 ++++++++ majel/place.cu | 61 +++++++++++++++++++++++++++++++++++++++++++++ majel/place.h | 50 +++++++++++++++++++++++++++++++++++++ majel/place_test.cu | 40 +++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 majel/Makefile create mode 100644 majel/place.cu create mode 100644 majel/place.h create mode 100644 majel/place_test.cu diff --git a/Dockerfile b/Dockerfile index 0d2f569114..dc44388732 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN apt-get update && \ curl sed grep graphviz libjpeg-dev zlib1g-dev \ python-numpy python-matplotlib gcc g++ \ automake locales clang-format-3.8 swig doxygen cmake \ - liblapack-dev liblapacke-dev \ + liblapack-dev liblapacke-dev libboost-dev \ clang-3.8 llvm-3.8 libclang-3.8-dev && \ apt-get clean -y @@ -61,6 +61,15 @@ RUN git clone https://github.com/woboq/woboq_codebrowser /woboq && \ -DCMAKE_BUILD_TYPE=Release . \ make) +# Install gtest. +# +# NOTE: This is added for quick hack of the development work of +# majel-in-paddle. +RUN git clone https://github.com/google/googletest /gtest && \ + cd /gtest && \ + git checkout -b release-1.8.0 && \ + cmake . && make install + # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd RUN echo 'root:root' | chpasswd diff --git a/majel/Makefile b/majel/Makefile new file mode 100644 index 0000000000..403c1d3a6a --- /dev/null +++ b/majel/Makefile @@ -0,0 +1,10 @@ +CCFLAGS = -std=c++11 -I/work/paddle +CC = nvcc + +all : place_test + +place.o : place.h place.cu + $(CC) $(CCFLAGS) -c place.cu -o $@ + +place_test : place.o place_test.cu + $(CC) $(CCFLAGS) -lgtest -lgtest_main $^ -o $@ diff --git a/majel/place.cu b/majel/place.cu new file mode 100644 index 0000000000..ee84e96048 --- /dev/null +++ b/majel/place.cu @@ -0,0 +1,61 @@ +#include + +namespace majel { + +namespace detail { + +class PlacePrinter + : public boost::static_visitor<> { +private: + std::ostream& os_; +public: + PlacePrinter(std::ostream& os) : os_(os) {} + + void operator()(const CpuPlace&) { + os_ << "CpuPlace"; + } + + void operator()(const GpuPlace& p) { + os_ << "GpuPlace(" << p.device << ")"; + } +}; + +} // namespace majel + +static Place the_default_place; + +void set_place(const Place& place) { + the_default_place = place; +} + +const Place& get_place() { + return the_default_place; +} + +const GpuPlace default_gpu() { + return GpuPlace(0); +} + +const CpuPlace default_cpu() { + return CpuPlace(); +} + +bool is_gpu_place(const Place& p) { + return boost::apply_visitor(IsGpuPlace(), p); +} + +bool is_cpu_place(const Place& p) { + return !boost::apply_visitor(IsGpuPlace(), p); +} + +bool places_are_same_class(const Place& p1, const Place& p2) { + return is_gpu_place(p1) == is_gpu_place(p2); +} + +std::ostream& operator<<(std::ostream& os, const majel::Place& p) { + majel::detail::PlacePrinter printer(os); + boost::apply_visitor(printer, p); + return os; +} + +} // namespace majel diff --git a/majel/place.h b/majel/place.h new file mode 100644 index 0000000000..ad3dc3fe0b --- /dev/null +++ b/majel/place.h @@ -0,0 +1,50 @@ +#pragma once +#include +#include + +namespace majel { + +struct CpuPlace { + CpuPlace() {} // WORKAROUND: for some reason, omitting this constructor + // causes errors with boost 1.59 and OSX + // needed for variant equality comparison + inline bool operator==(const CpuPlace&) const { return true; } + + inline bool operator!=(const CpuPlace&) const { return false; } +}; + +struct GpuPlace { + GpuPlace(int d) : device(d) {} + + // needed for variant equality comparison + inline bool operator==(const GpuPlace& o) const { return device == o.device; } + + inline bool operator!=(const GpuPlace& o) const { return !(*this == o); } + + GpuPlace() : GpuPlace(0) {} + int device; +}; + +class IsGpuPlace : public boost::static_visitor { +public: + bool operator()(const CpuPlace&) const { return false; } + + bool operator()(const GpuPlace& gpu) const { return true; } +}; + +typedef boost::variant Place; + +void set_place(const Place&); + +const Place& get_place(); + +const GpuPlace default_gpu(); +const CpuPlace default_cpu(); + +bool is_gpu_place(const Place&); +bool is_cpu_place(const Place&); +bool places_are_same_class(const Place&, const Place&); + +std::ostream& operator<<(std::ostream&, const majel::Place&); + +} // namespace majel diff --git a/majel/place_test.cu b/majel/place_test.cu new file mode 100644 index 0000000000..cb82946130 --- /dev/null +++ b/majel/place_test.cu @@ -0,0 +1,40 @@ +#include "gtest/gtest.h" +#include "majel/place.h" +#include + +TEST(Place, Equality) { + majel::CpuPlace cpu; + majel::GpuPlace g0(0), g1(1), gg0(0); + + EXPECT_EQ(cpu, cpu); + EXPECT_EQ(g0, g0); + EXPECT_EQ(g1, g1); + EXPECT_EQ(g0, gg0); + + EXPECT_NE(g0, g1); + + EXPECT_TRUE(majel::places_are_same_class(g0, gg0)); + EXPECT_FALSE(majel::places_are_same_class(g0, cpu)); +} + +TEST(Place, Default) { + EXPECT_TRUE(majel::is_gpu_place( majel::get_place())); + EXPECT_TRUE(majel::is_gpu_place( majel::default_gpu())); + EXPECT_TRUE(majel::is_cpu_place( majel::default_cpu())); + + majel::set_place(majel::CpuPlace()); + EXPECT_TRUE(majel::is_cpu_place( majel::get_place())); +} + +TEST(Place, Print) { + { + std::stringstream ss; + ss << majel::GpuPlace(1); + EXPECT_EQ("GpuPlace(1)", ss.str()); + } + { + std::stringstream ss; + ss << majel::CpuPlace(); + EXPECT_EQ("CpuPlace", ss.str()); + } +} From 97955f063e39c3b580a70ed42c2baa77a0efbeeb Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 12 May 2017 00:23:54 +0800 Subject: [PATCH 39/48] rename majel --- paddle/{tensor => majel}/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename paddle/{tensor => majel}/README.md (99%) diff --git a/paddle/tensor/README.md b/paddle/majel/README.md similarity index 99% rename from paddle/tensor/README.md rename to paddle/majel/README.md index b37ed72216..5539853056 100644 --- a/paddle/tensor/README.md +++ b/paddle/majel/README.md @@ -9,7 +9,7 @@ In this week, we discussed several potential weaknesses of PaddlePaddle caused b Consequently, we decide to refactor PaddlePaddle step-by-step. First, refactor and replace Matrix/Vector to Tensor, a modern terminology in the deep learning system. Fortunately, we can learn from Majel how to define a Tensor. -To simplify heterogeneous resource allocation in any dimensions (1-9) and types (double, float, float16), Majel consists of several primitives such as `Dim`, `Place` and `Array`, all of them are standard C++ classes. +To simplify heterogeneous resource allocation in any dimensions (1-9) and types (double, float, float16), Majel consists of several primitives such as `Dim`, `Place` and `Array`, all of them are standard C++ class templates. 1. `Place`: memory location [i.e. CPU/GPU]. 2. `Allocation`: heterogeneous resource allocator [i.e. 20MB in GPU]. From 3f347ef539e5e6a81e5ba2747d2fc306b15f759d Mon Sep 17 00:00:00 2001 From: luotao02 Date: Fri, 12 May 2017 16:50:53 +0800 Subject: [PATCH 40/48] fix error cmake infomation --- paddle/utils/DynamicLoader.cpp | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/paddle/utils/DynamicLoader.cpp b/paddle/utils/DynamicLoader.cpp index 76cf3c3006..5aa1fe12d8 100644 --- a/paddle/utils/DynamicLoader.cpp +++ b/paddle/utils/DynamicLoader.cpp @@ -104,20 +104,9 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, CHECK(nullptr != *dso_handle) << "Failed to find dynamic library: " << dlPath << " (" << dlerror() << ") \n" << "Please specify its path correctly using " - "one of the following ways: \n" - - << "Method 1. set cuda and cudnn lib path at " - "runtime. " - << "http://www.paddlepaddle.org/doc/ui/" - "cmd_argument/" - "argument_outline.html \n" - << "For instance, issue command: paddle train " - "--use_gpu=1 " - << "--cuda_dir=/usr/local/cuda/lib64 " - "--cudnn_dir=/usr/local/cudnn/lib " - "...\n" - - << "Method 2. set environment variable " + "following ways: \n" + + << "Method. set environment variable " "LD_LIBRARY_PATH on Linux or " << "DYLD_LIBRARY_PATH on Mac OS. \n" << "For instance, issue command: export " From df00e2928d1f69f55ef6aba1a91ea820a2c2d612 Mon Sep 17 00:00:00 2001 From: luotao02 Date: Fri, 12 May 2017 17:51:05 +0800 Subject: [PATCH 41/48] follow comments --- paddle/utils/DynamicLoader.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/paddle/utils/DynamicLoader.cpp b/paddle/utils/DynamicLoader.cpp index 5aa1fe12d8..5604a90038 100644 --- a/paddle/utils/DynamicLoader.cpp +++ b/paddle/utils/DynamicLoader.cpp @@ -115,9 +115,7 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, << "Note: After Mac OS 10.11, using the " "DYLD_LIBRARY_PATH is impossible " << "unless System Integrity Protection (SIP) " - "is disabled. However, " - "method 1 " - << "always work well."; + "is disabled."; } void GetCublasDsoHandle(void** dso_handle) { From b0f070e013fea9e93728e3cfe1bfc5045132c51b Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 00:11:59 +0800 Subject: [PATCH 42/48] add cmake for majel --- paddle/CMakeLists.txt | 8 ++++++++ paddle/majel/CMakeLists.txt | 15 +++++++++++++++ paddle/majel/test/CMakeLists.txt | 10 ++++++++++ 3 files changed, 33 insertions(+) create mode 100644 paddle/majel/CMakeLists.txt create mode 100644 paddle/majel/test/CMakeLists.txt diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index eff296bcb0..c6fd9cc54a 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -9,6 +9,14 @@ add_subdirectory(pserver) add_subdirectory(trainer) add_subdirectory(scripts) +find_package(boost QUIET) + +if(Boost_FOUND) + include_directories(${Boost_INCLUDE_DIRS}) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + add_subdirectory(majel) +endif() + if(WITH_C_API) add_subdirectory(capi) endif() diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt new file mode 100644 index 0000000000..f2f57900c8 --- /dev/null +++ b/paddle/majel/CMakeLists.txt @@ -0,0 +1,15 @@ +set(MAJEL_CXX_FILES place.cpp) +set(MAJEL_CUDA_FILES "") + +if(CUDA_FOUND) + cuda_add_library(majel ${MAJEL_CUDA_FILES} ${MAJEL_CXX_FILES}) +else() + add_library(majel ${MAJEL_CXX_FILES}) +endif() + +file(GLOB_RECURSE CHECK_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cpp" "*.h") +add_style_check_target(majel ${CHECK_FILES}) + +if(WITH_TESTING) + add_subdirectory(test) +endif() \ No newline at end of file diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt new file mode 100644 index 0000000000..a07b1bf5d8 --- /dev/null +++ b/paddle/majel/test/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE ALL_TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cpp" "*.cc") + +add_executable(majel_tests ${ALL_TEST_FILES}) +add_dependencies(majel_tests majel ${external_project_dependencies}) +target_link_libraries(majel_tests + ${Boost_LIBRARIES} + ${GTEST_LIBRARIES} + majel + ) +add_test(majel_tests majel_tests) From 3a734557d80129cd0f557ee590b2f4db91b63276 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 00:12:28 +0800 Subject: [PATCH 43/48] move majel into paddle/majel --- majel/Makefile | 10 ---------- majel/place.cu => paddle/majel/place.cpp | 2 +- {majel => paddle/majel}/place.h | 0 .../place_test.cu => paddle/majel/test/place_test.cpp | 0 paddle/majel/test/test_framework.cpp | 6 ++++++ 5 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 majel/Makefile rename majel/place.cu => paddle/majel/place.cpp (97%) rename {majel => paddle/majel}/place.h (100%) rename majel/place_test.cu => paddle/majel/test/place_test.cpp (100%) create mode 100644 paddle/majel/test/test_framework.cpp diff --git a/majel/Makefile b/majel/Makefile deleted file mode 100644 index 403c1d3a6a..0000000000 --- a/majel/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -CCFLAGS = -std=c++11 -I/work/paddle -CC = nvcc - -all : place_test - -place.o : place.h place.cu - $(CC) $(CCFLAGS) -c place.cu -o $@ - -place_test : place.o place_test.cu - $(CC) $(CCFLAGS) -lgtest -lgtest_main $^ -o $@ diff --git a/majel/place.cu b/paddle/majel/place.cpp similarity index 97% rename from majel/place.cu rename to paddle/majel/place.cpp index ee84e96048..396534b84b 100644 --- a/majel/place.cu +++ b/paddle/majel/place.cpp @@ -1,4 +1,4 @@ -#include +#include "majel/place.h" namespace majel { diff --git a/majel/place.h b/paddle/majel/place.h similarity index 100% rename from majel/place.h rename to paddle/majel/place.h diff --git a/majel/place_test.cu b/paddle/majel/test/place_test.cpp similarity index 100% rename from majel/place_test.cu rename to paddle/majel/test/place_test.cpp diff --git a/paddle/majel/test/test_framework.cpp b/paddle/majel/test/test_framework.cpp new file mode 100644 index 0000000000..a62216ba85 --- /dev/null +++ b/paddle/majel/test/test_framework.cpp @@ -0,0 +1,6 @@ +#include "gtest/gtest.h" + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 9c501e90cf35ce159c5ddd26faea1fe955e4abdd Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 00:12:40 +0800 Subject: [PATCH 44/48] remove gtest in dockerfile --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc44388732..ad0d086d3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,15 +61,6 @@ RUN git clone https://github.com/woboq/woboq_codebrowser /woboq && \ -DCMAKE_BUILD_TYPE=Release . \ make) -# Install gtest. -# -# NOTE: This is added for quick hack of the development work of -# majel-in-paddle. -RUN git clone https://github.com/google/googletest /gtest && \ - cd /gtest && \ - git checkout -b release-1.8.0 && \ - cmake . && make install - # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd RUN echo 'root:root' | chpasswd From 92e0c893e4101eeb8303a07777cb7d097f705a9d Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 00:15:07 +0800 Subject: [PATCH 45/48] precommit --- paddle/majel/CMakeLists.txt | 2 +- paddle/majel/place.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt index f2f57900c8..f1cd4ee9a1 100644 --- a/paddle/majel/CMakeLists.txt +++ b/paddle/majel/CMakeLists.txt @@ -12,4 +12,4 @@ add_style_check_target(majel ${CHECK_FILES}) if(WITH_TESTING) add_subdirectory(test) -endif() \ No newline at end of file +endif() diff --git a/paddle/majel/place.cpp b/paddle/majel/place.cpp index 396534b84b..ee84e96048 100644 --- a/paddle/majel/place.cpp +++ b/paddle/majel/place.cpp @@ -1,4 +1,4 @@ -#include "majel/place.h" +#include namespace majel { From d8c8f419f974a9d02bddd937330e8201a3a2a699 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 01:09:24 +0800 Subject: [PATCH 46/48] support build majel in subdir --- paddle/majel/.gitignore | 2 ++ paddle/majel/CMakeLists.txt | 31 +++++++++++++++++++++++++------ paddle/majel/test/CMakeLists.txt | 2 +- 3 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 paddle/majel/.gitignore diff --git a/paddle/majel/.gitignore b/paddle/majel/.gitignore new file mode 100644 index 0000000000..1f5acdebb5 --- /dev/null +++ b/paddle/majel/.gitignore @@ -0,0 +1,2 @@ +build +third-party \ No newline at end of file diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt index f1cd4ee9a1..f3bc984a26 100644 --- a/paddle/majel/CMakeLists.txt +++ b/paddle/majel/CMakeLists.txt @@ -1,3 +1,26 @@ +cmake_minimum_required(VERSION 3.0) + +if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) + message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})") +else() + # find #include + get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) + include_directories(${PARENT_DIR}) + + # find cmake directory modules + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + + # enable c++11 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + + # enable gtest + set(THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/third_party) + set(WITH_TESTING ON) + include(external/gtest) +endif() + +########################### Build Majel ############################# set(MAJEL_CXX_FILES place.cpp) set(MAJEL_CUDA_FILES "") @@ -6,10 +29,6 @@ if(CUDA_FOUND) else() add_library(majel ${MAJEL_CXX_FILES}) endif() +########################### Build Majel ############################# -file(GLOB_RECURSE CHECK_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cpp" "*.h") -add_style_check_target(majel ${CHECK_FILES}) - -if(WITH_TESTING) - add_subdirectory(test) -endif() +add_subdirectory(test) diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt index a07b1bf5d8..0cc7103b03 100644 --- a/paddle/majel/test/CMakeLists.txt +++ b/paddle/majel/test/CMakeLists.txt @@ -1,7 +1,7 @@ file(GLOB_RECURSE ALL_TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cpp" "*.cc") add_executable(majel_tests ${ALL_TEST_FILES}) -add_dependencies(majel_tests majel ${external_project_dependencies}) +add_dependencies(majel_tests majel) target_link_libraries(majel_tests ${Boost_LIBRARIES} ${GTEST_LIBRARIES} From 05b3196e0a9e92e7123fb51fe1a7f46992843607 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 01:10:48 +0800 Subject: [PATCH 47/48] clang-format --- paddle/majel/place.cpp | 44 ++++++++++------------------ paddle/majel/test/place_test.cpp | 16 +++++----- paddle/majel/test/test_framework.cpp | 4 +-- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/paddle/majel/place.cpp b/paddle/majel/place.cpp index ee84e96048..eecd8e5b73 100644 --- a/paddle/majel/place.cpp +++ b/paddle/majel/place.cpp @@ -4,58 +4,46 @@ namespace majel { namespace detail { -class PlacePrinter - : public boost::static_visitor<> { +class PlacePrinter : public boost::static_visitor<> { private: - std::ostream& os_; + std::ostream& os_; + public: - PlacePrinter(std::ostream& os) : os_(os) {} + PlacePrinter(std::ostream& os) : os_(os) {} - void operator()(const CpuPlace&) { - os_ << "CpuPlace"; - } + void operator()(const CpuPlace&) { os_ << "CpuPlace"; } - void operator()(const GpuPlace& p) { - os_ << "GpuPlace(" << p.device << ")"; - } + void operator()(const GpuPlace& p) { os_ << "GpuPlace(" << p.device << ")"; } }; } // namespace majel static Place the_default_place; -void set_place(const Place& place) { - the_default_place = place; -} +void set_place(const Place& place) { the_default_place = place; } -const Place& get_place() { - return the_default_place; -} +const Place& get_place() { return the_default_place; } -const GpuPlace default_gpu() { - return GpuPlace(0); -} +const GpuPlace default_gpu() { return GpuPlace(0); } -const CpuPlace default_cpu() { - return CpuPlace(); -} +const CpuPlace default_cpu() { return CpuPlace(); } bool is_gpu_place(const Place& p) { - return boost::apply_visitor(IsGpuPlace(), p); + return boost::apply_visitor(IsGpuPlace(), p); } bool is_cpu_place(const Place& p) { - return !boost::apply_visitor(IsGpuPlace(), p); + return !boost::apply_visitor(IsGpuPlace(), p); } bool places_are_same_class(const Place& p1, const Place& p2) { - return is_gpu_place(p1) == is_gpu_place(p2); + return is_gpu_place(p1) == is_gpu_place(p2); } std::ostream& operator<<(std::ostream& os, const majel::Place& p) { - majel::detail::PlacePrinter printer(os); - boost::apply_visitor(printer, p); - return os; + majel::detail::PlacePrinter printer(os); + boost::apply_visitor(printer, p); + return os; } } // namespace majel diff --git a/paddle/majel/test/place_test.cpp b/paddle/majel/test/place_test.cpp index cb82946130..c9a53802b2 100644 --- a/paddle/majel/test/place_test.cpp +++ b/paddle/majel/test/place_test.cpp @@ -1,6 +1,6 @@ -#include "gtest/gtest.h" #include "majel/place.h" #include +#include "gtest/gtest.h" TEST(Place, Equality) { majel::CpuPlace cpu; @@ -18,12 +18,12 @@ TEST(Place, Equality) { } TEST(Place, Default) { - EXPECT_TRUE(majel::is_gpu_place( majel::get_place())); - EXPECT_TRUE(majel::is_gpu_place( majel::default_gpu())); - EXPECT_TRUE(majel::is_cpu_place( majel::default_cpu())); + EXPECT_TRUE(majel::is_gpu_place(majel::get_place())); + EXPECT_TRUE(majel::is_gpu_place(majel::default_gpu())); + EXPECT_TRUE(majel::is_cpu_place(majel::default_cpu())); majel::set_place(majel::CpuPlace()); - EXPECT_TRUE(majel::is_cpu_place( majel::get_place())); + EXPECT_TRUE(majel::is_cpu_place(majel::get_place())); } TEST(Place, Print) { @@ -33,8 +33,8 @@ TEST(Place, Print) { EXPECT_EQ("GpuPlace(1)", ss.str()); } { - std::stringstream ss; - ss << majel::CpuPlace(); - EXPECT_EQ("CpuPlace", ss.str()); + std::stringstream ss; + ss << majel::CpuPlace(); + EXPECT_EQ("CpuPlace", ss.str()); } } diff --git a/paddle/majel/test/test_framework.cpp b/paddle/majel/test/test_framework.cpp index a62216ba85..443e2dbb3f 100644 --- a/paddle/majel/test/test_framework.cpp +++ b/paddle/majel/test/test_framework.cpp @@ -1,6 +1,6 @@ #include "gtest/gtest.h" int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } From d444251c4a43facb2d15f0952b6312a616210c22 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 13 May 2017 01:15:04 +0800 Subject: [PATCH 48/48] refine format --- paddle/majel/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt index f3bc984a26..0c91fa72da 100644 --- a/paddle/majel/CMakeLists.txt +++ b/paddle/majel/CMakeLists.txt @@ -29,6 +29,6 @@ if(CUDA_FOUND) else() add_library(majel ${MAJEL_CXX_FILES}) endif() -########################### Build Majel ############################# +##################################################################### add_subdirectory(test)